drivers/net/ethernet/freescale/enetc/enetc.c

   1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2 /* Copyright 2017-2019 NXP */
   3
   4 #include "enetc.h"
   5 #include <linux/bpf_trace.h>
   6 #include <linux/tcp.h>
   7 #include <linux/udp.h>
   8 #include <linux/vmalloc.h>
   9 #include <linux/ptp_classify.h>
  10 #include <net/ip6_checksum.h>
  11 #include <net/pkt_sched.h>
  12 #include <net/tso.h>
  13
  14 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
  15 {
  16         return enetc_port_rd(&si->hw, reg);
  17 }
  18 EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
  19
  20 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
  21 {
  22         enetc_port_wr(&si->hw, reg, val);
  23         if (si->hw_features & ENETC_SI_F_QBU)
  24                 enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val);
  25 }
  26 EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
  27
  28 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
  29                                          u8 preemptible_tcs)
  30 {
  31         priv->preemptible_tcs = preemptible_tcs;
  32         enetc_mm_commit_preemptible_tcs(priv);
  33 }
  34
  35 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
  36 {
  37         int num_tx_rings = priv->num_tx_rings;
  38
  39         if (priv->xdp_prog)
  40                 return num_tx_rings - num_possible_cpus();
  41
  42         return num_tx_rings;
  43 }
  44
  45 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
  46                                                         struct enetc_bdr *tx_ring)
  47 {
  48         int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
  49
  50         return priv->rx_ring[index];
  51 }
  52
  53 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
  54 {
  55         if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
  56                 return NULL;
  57
  58         return tx_swbd->skb;
  59 }
  60
  61 static struct xdp_frame *
  62 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
  63 {
  64         if (tx_swbd->is_xdp_redirect)
  65                 return tx_swbd->xdp_frame;
  66
  67         return NULL;
  68 }
  69
  70 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
  71                                 struct enetc_tx_swbd *tx_swbd)
  72 {
  73         /* For XDP_TX, pages come from RX, whereas for the other contexts where
  74          * we have is_dma_page_set, those come from skb_frag_dma_map. We need
  75          * to match the DMA mapping length, so we need to differentiate those.
  76          */
  77         if (tx_swbd->is_dma_page)
  78                 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
  79                                tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
  80                                tx_swbd->dir);
  81         else
  82                 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
  83                                  tx_swbd->len, tx_swbd->dir);
  84         tx_swbd->dma = 0;
  85 }
  86
  87 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
  88                                 struct enetc_tx_swbd *tx_swbd)
  89 {
  90         struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
  91         struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
  92
  93         if (tx_swbd->dma)
  94                 enetc_unmap_tx_buff(tx_ring, tx_swbd);
  95
  96         if (xdp_frame) {
  97                 xdp_return_frame(tx_swbd->xdp_frame);
  98                 tx_swbd->xdp_frame = NULL;
  99         } else if (skb) {
 100                 dev_kfree_skb_any(skb);
 101                 tx_swbd->skb = NULL;
 102         }
 103 }
 104
 105 /* Let H/W know BD ring has been updated */
 106 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
 107 {
 108         /* includes wmb() */
 109         enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
 110 }
 111
 112 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
 113                            u8 *msgtype, u8 *twostep,
 114                            u16 *correction_offset, u16 *body_offset)
 115 {
 116         unsigned int ptp_class;
 117         struct ptp_header *hdr;
 118         unsigned int type;
 119         u8 *base;
 120
 121         ptp_class = ptp_classify_raw(skb);
 122         if (ptp_class == PTP_CLASS_NONE)
 123                 return -EINVAL;
 124
 125         hdr = ptp_parse_header(skb, ptp_class);
 126         if (!hdr)
 127                 return -EINVAL;
 128
 129         type = ptp_class & PTP_CLASS_PMASK;
 130         if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
 131                 *udp = 1;
 132         else
 133                 *udp = 0;
 134
 135         *msgtype = ptp_get_msgtype(hdr, ptp_class);
 136         *twostep = hdr->flag_field[0] & 0x2;
 137
 138         base = skb_mac_header(skb);
 139         *correction_offset = (u8 *)&hdr->correction - base;
 140         *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
 141
 142         return 0;
 143 }
 144
 145 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
 146 {
 147         bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
 148         struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
 149         struct enetc_hw *hw = &priv->si->hw;
 150         struct enetc_tx_swbd *tx_swbd;
 151         int len = skb_headlen(skb);
 152         union enetc_tx_bd temp_bd;
 153         u8 msgtype, twostep, udp;
 154         union enetc_tx_bd *txbd;
 155         u16 offset1, offset2;
 156         int i, count = 0;
 157         skb_frag_t *frag;
 158         unsigned int f;
 159         dma_addr_t dma;
 160         u8 flags = 0;
 161
 162         i = tx_ring->next_to_use;
 163         txbd = ENETC_TXBD(*tx_ring, i);
 164         prefetchw(txbd);
 165
 166         dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
 167         if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
 168                 goto dma_err;
 169
 170         temp_bd.addr = cpu_to_le64(dma);
 171         temp_bd.buf_len = cpu_to_le16(len);
 172         temp_bd.lstatus = 0;
 173
 174         tx_swbd = &tx_ring->tx_swbd[i];
 175         tx_swbd->dma = dma;
 176         tx_swbd->len = len;
 177         tx_swbd->is_dma_page = 0;
 178         tx_swbd->dir = DMA_TO_DEVICE;
 179         count++;
 180
 181         do_vlan = skb_vlan_tag_present(skb);
 182         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 183                 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
 184                                     &offset2) ||
 185                     msgtype != PTP_MSGTYPE_SYNC || twostep)
 186                         WARN_ONCE(1, "Bad packet for one-step timestamping\n");
 187                 else
 188                         do_onestep_tstamp = true;
 189         } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
 190                 do_twostep_tstamp = true;
 191         }
 192
 193         tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
 194         tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
 195         tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
 196
 197         if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
 198                 flags |= ENETC_TXBD_FLAGS_EX;
 199
 200         if (tx_ring->tsd_enable)
 201                 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
 202
 203         /* first BD needs frm_len and offload flags set */
 204         temp_bd.frm_len = cpu_to_le16(skb->len);
 205         temp_bd.flags = flags;
 206
 207         if (flags & ENETC_TXBD_FLAGS_TSE)
 208                 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
 209                                                           flags);
 210
 211         if (flags & ENETC_TXBD_FLAGS_EX) {
 212                 u8 e_flags = 0;
 213                 *txbd = temp_bd;
 214                 enetc_clear_tx_bd(&temp_bd);
 215
 216                 /* add extension BD for VLAN and/or timestamping */
 217                 flags = 0;
 218                 tx_swbd++;
 219                 txbd++;
 220                 i++;
 221                 if (unlikely(i == tx_ring->bd_count)) {
 222                         i = 0;
 223                         tx_swbd = tx_ring->tx_swbd;
 224                         txbd = ENETC_TXBD(*tx_ring, 0);
 225                 }
 226                 prefetchw(txbd);
 227
 228                 if (do_vlan) {
 229                         temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
 230                         temp_bd.ext.tpid = 0; /* < C-TAG */
 231                         e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
 232                 }
 233
 234                 if (do_onestep_tstamp) {
 235                         u32 lo, hi, val;
 236                         u64 sec, nsec;
 237                         u8 *data;
 238
 239                         lo = enetc_rd_hot(hw, ENETC_SICTR0);
 240                         hi = enetc_rd_hot(hw, ENETC_SICTR1);
 241                         sec = (u64)hi << 32 | lo;
 242                         nsec = do_div(sec, 1000000000);
 243
 244                         /* Configure extension BD */
 245                         temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
 246                         e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
 247
 248                         /* Update originTimestamp field of Sync packet
 249                          * - 48 bits seconds field
 250                          * - 32 bits nanseconds field
 251                          */
 252                         data = skb_mac_header(skb);
 253                         *(__be16 *)(data + offset2) =
 254                                 htons((sec >> 32) & 0xffff);
 255                         *(__be32 *)(data + offset2 + 2) =
 256                                 htonl(sec & 0xffffffff);
 257                         *(__be32 *)(data + offset2 + 6) = htonl(nsec);
 258
 259                         /* Configure single-step register */
 260                         val = ENETC_PM0_SINGLE_STEP_EN;
 261                         val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
 262                         if (udp)
 263                                 val |= ENETC_PM0_SINGLE_STEP_CH;
 264
 265                         enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,
 266                                           val);
 267                 } else if (do_twostep_tstamp) {
 268                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 269                         e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
 270                 }
 271
 272                 temp_bd.ext.e_flags = e_flags;
 273                 count++;
 274         }
 275
 276         frag = &skb_shinfo(skb)->frags[0];
 277         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
 278                 len = skb_frag_size(frag);
 279                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
 280                                        DMA_TO_DEVICE);
 281                 if (dma_mapping_error(tx_ring->dev, dma))
 282                         goto dma_err;
 283
 284                 *txbd = temp_bd;
 285                 enetc_clear_tx_bd(&temp_bd);
 286
 287                 flags = 0;
 288                 tx_swbd++;
 289                 txbd++;
 290                 i++;
 291                 if (unlikely(i == tx_ring->bd_count)) {
 292                         i = 0;
 293                         tx_swbd = tx_ring->tx_swbd;
 294                         txbd = ENETC_TXBD(*tx_ring, 0);
 295                 }
 296                 prefetchw(txbd);
 297
 298                 temp_bd.addr = cpu_to_le64(dma);
 299                 temp_bd.buf_len = cpu_to_le16(len);
 300
 301                 tx_swbd->dma = dma;
 302                 tx_swbd->len = len;
 303                 tx_swbd->is_dma_page = 1;
 304                 tx_swbd->dir = DMA_TO_DEVICE;
 305                 count++;
 306         }
 307
 308         /* last BD needs 'F' bit set */
 309         flags |= ENETC_TXBD_FLAGS_F;
 310         temp_bd.flags = flags;
 311         *txbd = temp_bd;
 312
 313         tx_ring->tx_swbd[i].is_eof = true;
 314         tx_ring->tx_swbd[i].skb = skb;
 315
 316         enetc_bdr_idx_inc(tx_ring, &i);
 317         tx_ring->next_to_use = i;
 318
 319         skb_tx_timestamp(skb);
 320
 321         enetc_update_tx_ring_tail(tx_ring);
 322
 323         return count;
 324
 325 dma_err:
 326         dev_err(tx_ring->dev, "DMA map error");
 327
 328         do {
 329                 tx_swbd = &tx_ring->tx_swbd[i];
 330                 enetc_free_tx_frame(tx_ring, tx_swbd);
 331                 if (i == 0)
 332                         i = tx_ring->bd_count;
 333                 i--;
 334         } while (count--);
 335
 336         return 0;
 337 }
 338
 339 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
 340                                  struct enetc_tx_swbd *tx_swbd,
 341                                  union enetc_tx_bd *txbd, int *i, int hdr_len,
 342                                  int data_len)
 343 {
 344         union enetc_tx_bd txbd_tmp;
 345         u8 flags = 0, e_flags = 0;
 346         dma_addr_t addr;
 347
 348         enetc_clear_tx_bd(&txbd_tmp);
 349         addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
 350
 351         if (skb_vlan_tag_present(skb))
 352                 flags |= ENETC_TXBD_FLAGS_EX;
 353
 354         txbd_tmp.addr = cpu_to_le64(addr);
 355         txbd_tmp.buf_len = cpu_to_le16(hdr_len);
 356
 357         /* first BD needs frm_len and offload flags set */
 358         txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
 359         txbd_tmp.flags = flags;
 360
 361         /* For the TSO header we do not set the dma address since we do not
 362          * want it unmapped when we do cleanup. We still set len so that we
 363          * count the bytes sent.
 364          */
 365         tx_swbd->len = hdr_len;
 366         tx_swbd->do_twostep_tstamp = false;
 367         tx_swbd->check_wb = false;
 368
 369         /* Actually write the header in the BD */
 370         *txbd = txbd_tmp;
 371
 372         /* Add extension BD for VLAN */
 373         if (flags & ENETC_TXBD_FLAGS_EX) {
 374                 /* Get the next BD */
 375                 enetc_bdr_idx_inc(tx_ring, i);
 376                 txbd = ENETC_TXBD(*tx_ring, *i);
 377                 tx_swbd = &tx_ring->tx_swbd[*i];
 378                 prefetchw(txbd);
 379
 380                 /* Setup the VLAN fields */
 381                 enetc_clear_tx_bd(&txbd_tmp);
 382                 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
 383                 txbd_tmp.ext.tpid = 0; /* < C-TAG */
 384                 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
 385
 386                 /* Write the BD */
 387                 txbd_tmp.ext.e_flags = e_flags;
 388                 *txbd = txbd_tmp;
 389         }
 390 }
 391
 392 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
 393                                  struct enetc_tx_swbd *tx_swbd,
 394                                  union enetc_tx_bd *txbd, char *data,
 395                                  int size, bool last_bd)
 396 {
 397         union enetc_tx_bd txbd_tmp;
 398         dma_addr_t addr;
 399         u8 flags = 0;
 400
 401         enetc_clear_tx_bd(&txbd_tmp);
 402
 403         addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
 404         if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
 405                 netdev_err(tx_ring->ndev, "DMA map error\n");
 406                 return -ENOMEM;
 407         }
 408
 409         if (last_bd) {
 410                 flags |= ENETC_TXBD_FLAGS_F;
 411                 tx_swbd->is_eof = 1;
 412         }
 413
 414         txbd_tmp.addr = cpu_to_le64(addr);
 415         txbd_tmp.buf_len = cpu_to_le16(size);
 416         txbd_tmp.flags = flags;
 417
 418         tx_swbd->dma = addr;
 419         tx_swbd->len = size;
 420         tx_swbd->dir = DMA_TO_DEVICE;
 421
 422         *txbd = txbd_tmp;
 423
 424         return 0;
 425 }
 426
 427 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
 428                                  char *hdr, int hdr_len, int *l4_hdr_len)
 429 {
 430         char *l4_hdr = hdr + skb_transport_offset(skb);
 431         int mac_hdr_len = skb_network_offset(skb);
 432
 433         if (tso->tlen != sizeof(struct udphdr)) {
 434                 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
 435
 436                 tcph->check = 0;
 437         } else {
 438                 struct udphdr *udph = (struct udphdr *)(l4_hdr);
 439
 440                 udph->check = 0;
 441         }
 442
 443         /* Compute the IP checksum. This is necessary since tso_build_hdr()
 444          * already incremented the IP ID field.
 445          */
 446         if (!tso->ipv6) {
 447                 struct iphdr *iph = (void *)(hdr + mac_hdr_len);
 448
 449                 iph->check = 0;
 450                 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
 451         }
 452
 453         /* Compute the checksum over the L4 header. */
 454         *l4_hdr_len = hdr_len - skb_transport_offset(skb);
 455         return csum_partial(l4_hdr, *l4_hdr_len, 0);
 456 }
 457
 458 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
 459                                     struct sk_buff *skb, char *hdr, int len,
 460                                     __wsum sum)
 461 {
 462         char *l4_hdr = hdr + skb_transport_offset(skb);
 463         __sum16 csum_final;
 464
 465         /* Complete the L4 checksum by appending the pseudo-header to the
 466          * already computed checksum.
 467          */
 468         if (!tso->ipv6)
 469                 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
 470                                                ip_hdr(skb)->daddr,
 471                                                len, ip_hdr(skb)->protocol, sum);
 472         else
 473                 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
 474                                              &ipv6_hdr(skb)->daddr,
 475                                              len, ipv6_hdr(skb)->nexthdr, sum);
 476
 477         if (tso->tlen != sizeof(struct udphdr)) {
 478                 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
 479
 480                 tcph->check = csum_final;
 481         } else {
 482                 struct udphdr *udph = (struct udphdr *)(l4_hdr);
 483
 484                 udph->check = csum_final;
 485         }
 486 }
 487
 488 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
 489 {
 490         int hdr_len, total_len, data_len;
 491         struct enetc_tx_swbd *tx_swbd;
 492         union enetc_tx_bd *txbd;
 493         struct tso_t tso;
 494         __wsum csum, csum2;
 495         int count = 0, pos;
 496         int err, i, bd_data_num;
 497
 498         /* Initialize the TSO handler, and prepare the first payload */
 499         hdr_len = tso_start(skb, &tso);
 500         total_len = skb->len - hdr_len;
 501         i = tx_ring->next_to_use;
 502
 503         while (total_len > 0) {
 504                 char *hdr;
 505
 506                 /* Get the BD */
 507                 txbd = ENETC_TXBD(*tx_ring, i);
 508                 tx_swbd = &tx_ring->tx_swbd[i];
 509                 prefetchw(txbd);
 510
 511                 /* Determine the length of this packet */
 512                 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
 513                 total_len -= data_len;
 514
 515                 /* prepare packet headers: MAC + IP + TCP */
 516                 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
 517                 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
 518
 519                 /* compute the csum over the L4 header */
 520                 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
 521                 enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
 522                 bd_data_num = 0;
 523                 count++;
 524
 525                 while (data_len > 0) {
 526                         int size;
 527
 528                         size = min_t(int, tso.size, data_len);
 529
 530                         /* Advance the index in the BDR */
 531                         enetc_bdr_idx_inc(tx_ring, &i);
 532                         txbd = ENETC_TXBD(*tx_ring, i);
 533                         tx_swbd = &tx_ring->tx_swbd[i];
 534                         prefetchw(txbd);
 535
 536                         /* Compute the checksum over this segment of data and
 537                          * add it to the csum already computed (over the L4
 538                          * header and possible other data segments).
 539                          */
 540                         csum2 = csum_partial(tso.data, size, 0);
 541                         csum = csum_block_add(csum, csum2, pos);
 542                         pos += size;
 543
 544                         err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
 545                                                     tso.data, size,
 546                                                     size == data_len);
 547                         if (err)
 548                                 goto err_map_data;
 549
 550                         data_len -= size;
 551                         count++;
 552                         bd_data_num++;
 553                         tso_build_data(skb, &tso, size);
 554
 555                         if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
 556                                 goto err_chained_bd;
 557                 }
 558
 559                 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
 560
 561                 if (total_len == 0)
 562                         tx_swbd->skb = skb;
 563
 564                 /* Go to the next BD */
 565                 enetc_bdr_idx_inc(tx_ring, &i);
 566         }
 567
 568         tx_ring->next_to_use = i;
 569         enetc_update_tx_ring_tail(tx_ring);
 570
 571         return count;
 572
 573 err_map_data:
 574         dev_err(tx_ring->dev, "DMA map error");
 575
 576 err_chained_bd:
 577         do {
 578                 tx_swbd = &tx_ring->tx_swbd[i];
 579                 enetc_free_tx_frame(tx_ring, tx_swbd);
 580                 if (i == 0)
 581                         i = tx_ring->bd_count;
 582                 i--;
 583         } while (count--);
 584
 585         return 0;
 586 }
 587
 588 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
 589                                     struct net_device *ndev)
 590 {
 591         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 592         struct enetc_bdr *tx_ring;
 593         int count, err;
 594
 595         /* Queue one-step Sync packet if already locked */
 596         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 597                 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
 598                                           &priv->flags)) {
 599                         skb_queue_tail(&priv->tx_skbs, skb);
 600                         return NETDEV_TX_OK;
 601                 }
 602         }
 603
 604         tx_ring = priv->tx_ring[skb->queue_mapping];
 605
 606         if (skb_is_gso(skb)) {
 607                 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
 608                         netif_stop_subqueue(ndev, tx_ring->index);
 609                         return NETDEV_TX_BUSY;
 610                 }
 611
 612                 enetc_lock_mdio();
 613                 count = enetc_map_tx_tso_buffs(tx_ring, skb);
 614                 enetc_unlock_mdio();
 615         } else {
 616                 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
 617                         if (unlikely(skb_linearize(skb)))
 618                                 goto drop_packet_err;
 619
 620                 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
 621                 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
 622                         netif_stop_subqueue(ndev, tx_ring->index);
 623                         return NETDEV_TX_BUSY;
 624                 }
 625
 626                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
 627                         err = skb_checksum_help(skb);
 628                         if (err)
 629                                 goto drop_packet_err;
 630                 }
 631                 enetc_lock_mdio();
 632                 count = enetc_map_tx_buffs(tx_ring, skb);
 633                 enetc_unlock_mdio();
 634         }
 635
 636         if (unlikely(!count))
 637                 goto drop_packet_err;
 638
 639         if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
 640                 netif_stop_subqueue(ndev, tx_ring->index);
 641
 642         return NETDEV_TX_OK;
 643
 644 drop_packet_err:
 645         dev_kfree_skb_any(skb);
 646         return NETDEV_TX_OK;
 647 }
 648
 649 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
 650 {
 651         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 652         u8 udp, msgtype, twostep;
 653         u16 offset1, offset2;
 654
 655         /* Mark tx timestamp type on skb->cb[0] if requires */
 656         if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
 657             (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
 658                 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
 659         } else {
 660                 skb->cb[0] = 0;
 661         }
 662
 663         /* Fall back to two-step timestamp if not one-step Sync packet */
 664         if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
 665                 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
 666                                     &offset1, &offset2) ||
 667                     msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
 668                         skb->cb[0] = ENETC_F_TX_TSTAMP;
 669         }
 670
 671         return enetc_start_xmit(skb, ndev);
 672 }
 673 EXPORT_SYMBOL_GPL(enetc_xmit);
 674
 675 static irqreturn_t enetc_msix(int irq, void *data)
 676 {
 677         struct enetc_int_vector *v = data;
 678         int i;
 679
 680         enetc_lock_mdio();
 681
 682         /* disable interrupts */
 683         enetc_wr_reg_hot(v->rbier, 0);
 684         enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
 685
 686         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
 687                 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
 688
 689         enetc_unlock_mdio();
 690
 691         napi_schedule(&v->napi);
 692
 693         return IRQ_HANDLED;
 694 }
 695
 696 static void enetc_rx_dim_work(struct work_struct *w)
 697 {
 698         struct dim *dim = container_of(w, struct dim, work);
 699         struct dim_cq_moder moder =
 700                 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
 701         struct enetc_int_vector *v =
 702                 container_of(dim, struct enetc_int_vector, rx_dim);
 703
 704         v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
 705         dim->state = DIM_START_MEASURE;
 706 }
 707
 708 static void enetc_rx_net_dim(struct enetc_int_vector *v)
 709 {
 710         struct dim_sample dim_sample = {};
 711
 712         v->comp_cnt++;
 713
 714         if (!v->rx_napi_work)
 715                 return;
 716
 717         dim_update_sample(v->comp_cnt,
 718                           v->rx_ring.stats.packets,
 719                           v->rx_ring.stats.bytes,
 720                           &dim_sample);
 721         net_dim(&v->rx_dim, dim_sample);
 722 }
 723
 724 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
 725 {
 726         int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
 727
 728         return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
 729 }
 730
 731 static bool enetc_page_reusable(struct page *page)
 732 {
 733         return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
 734 }
 735
 736 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
 737                              struct enetc_rx_swbd *old)
 738 {
 739         struct enetc_rx_swbd *new;
 740
 741         new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
 742
 743         /* next buf that may reuse a page */
 744         enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
 745
 746         /* copy page reference */
 747         *new = *old;
 748 }
 749
 750 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
 751                                 u64 *tstamp)
 752 {
 753         u32 lo, hi, tstamp_lo;
 754
 755         lo = enetc_rd_hot(hw, ENETC_SICTR0);
 756         hi = enetc_rd_hot(hw, ENETC_SICTR1);
 757         tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
 758         if (lo <= tstamp_lo)
 759                 hi -= 1;
 760         *tstamp = (u64)hi << 32 | tstamp_lo;
 761 }
 762
 763 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
 764 {
 765         struct skb_shared_hwtstamps shhwtstamps;
 766
 767         if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
 768                 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
 769                 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
 770                 skb_txtime_consumed(skb);
 771                 skb_tstamp_tx(skb, &shhwtstamps);
 772         }
 773 }
 774
 775 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
 776                                       struct enetc_tx_swbd *tx_swbd)
 777 {
 778         struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
 779         struct enetc_rx_swbd rx_swbd = {
 780                 .dma = tx_swbd->dma,
 781                 .page = tx_swbd->page,
 782                 .page_offset = tx_swbd->page_offset,
 783                 .dir = tx_swbd->dir,
 784                 .len = tx_swbd->len,
 785         };
 786         struct enetc_bdr *rx_ring;
 787
 788         rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
 789
 790         if (likely(enetc_swbd_unused(rx_ring))) {
 791                 enetc_reuse_page(rx_ring, &rx_swbd);
 792
 793                 /* sync for use by the device */
 794                 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
 795                                                  rx_swbd.page_offset,
 796                                                  ENETC_RXB_DMA_SIZE_XDP,
 797                                                  rx_swbd.dir);
 798
 799                 rx_ring->stats.recycles++;
 800         } else {
 801                 /* RX ring is already full, we need to unmap and free the
 802                  * page, since there's nothing useful we can do with it.
 803                  */
 804                 rx_ring->stats.recycle_failures++;
 805
 806                 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
 807                                rx_swbd.dir);
 808                 __free_page(rx_swbd.page);
 809         }
 810
 811         rx_ring->xdp.xdp_tx_in_flight--;
 812 }
 813
 814 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
 815 {
 816         int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
 817         struct net_device *ndev = tx_ring->ndev;
 818         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 819         struct enetc_tx_swbd *tx_swbd;
 820         int i, bds_to_clean;
 821         bool do_twostep_tstamp;
 822         u64 tstamp = 0;
 823
 824         i = tx_ring->next_to_clean;
 825         tx_swbd = &tx_ring->tx_swbd[i];
 826
 827         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 828
 829         do_twostep_tstamp = false;
 830
 831         while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
 832                 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
 833                 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
 834                 bool is_eof = tx_swbd->is_eof;
 835
 836                 if (unlikely(tx_swbd->check_wb)) {
 837                         union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
 838
 839                         if (txbd->flags & ENETC_TXBD_FLAGS_W &&
 840                             tx_swbd->do_twostep_tstamp) {
 841                                 enetc_get_tx_tstamp(&priv->si->hw, txbd,
 842                                                     &tstamp);
 843                                 do_twostep_tstamp = true;
 844                         }
 845
 846                         if (tx_swbd->qbv_en &&
 847                             txbd->wb.status & ENETC_TXBD_STATS_WIN)
 848                                 tx_win_drop++;
 849                 }
 850
 851                 if (tx_swbd->is_xdp_tx)
 852                         enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
 853                 else if (likely(tx_swbd->dma))
 854                         enetc_unmap_tx_buff(tx_ring, tx_swbd);
 855
 856                 if (xdp_frame) {
 857                         xdp_return_frame(xdp_frame);
 858                 } else if (skb) {
 859                         if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
 860                                 /* Start work to release lock for next one-step
 861                                  * timestamping packet. And send one skb in
 862                                  * tx_skbs queue if has.
 863                                  */
 864                                 schedule_work(&priv->tx_onestep_tstamp);
 865                         } else if (unlikely(do_twostep_tstamp)) {
 866                                 enetc_tstamp_tx(skb, tstamp);
 867                                 do_twostep_tstamp = false;
 868                         }
 869                         napi_consume_skb(skb, napi_budget);
 870                 }
 871
 872                 tx_byte_cnt += tx_swbd->len;
 873                 /* Scrub the swbd here so we don't have to do that
 874                  * when we reuse it during xmit
 875                  */
 876                 memset(tx_swbd, 0, sizeof(*tx_swbd));
 877
 878                 bds_to_clean--;
 879                 tx_swbd++;
 880                 i++;
 881                 if (unlikely(i == tx_ring->bd_count)) {
 882                         i = 0;
 883                         tx_swbd = tx_ring->tx_swbd;
 884                 }
 885
 886                 /* BD iteration loop end */
 887                 if (is_eof) {
 888                         tx_frm_cnt++;
 889                         /* re-arm interrupt source */
 890                         enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
 891                                          BIT(16 + tx_ring->index));
 892                 }
 893
 894                 if (unlikely(!bds_to_clean))
 895                         bds_to_clean = enetc_bd_ready_count(tx_ring, i);
 896         }
 897
 898         tx_ring->next_to_clean = i;
 899         tx_ring->stats.packets += tx_frm_cnt;
 900         tx_ring->stats.bytes += tx_byte_cnt;
 901         tx_ring->stats.win_drop += tx_win_drop;
 902
 903         if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
 904                      __netif_subqueue_stopped(ndev, tx_ring->index) &&
 905                      (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
 906                 netif_wake_subqueue(ndev, tx_ring->index);
 907         }
 908
 909         return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
 910 }
 911
 912 static bool enetc_new_page(struct enetc_bdr *rx_ring,
 913                            struct enetc_rx_swbd *rx_swbd)
 914 {
 915         bool xdp = !!(rx_ring->xdp.prog);
 916         struct page *page;
 917         dma_addr_t addr;
 918
 919         page = dev_alloc_page();
 920         if (unlikely(!page))
 921                 return false;
 922
 923         /* For XDP_TX, we forgo dma_unmap -> dma_map */
 924         rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
 925
 926         addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
 927         if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
 928                 __free_page(page);
 929
 930                 return false;
 931         }
 932
 933         rx_swbd->dma = addr;
 934         rx_swbd->page = page;
 935         rx_swbd->page_offset = rx_ring->buffer_offset;
 936
 937         return true;
 938 }
 939
 940 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
 941 {
 942         struct enetc_rx_swbd *rx_swbd;
 943         union enetc_rx_bd *rxbd;
 944         int i, j;
 945
 946         i = rx_ring->next_to_use;
 947         rx_swbd = &rx_ring->rx_swbd[i];
 948         rxbd = enetc_rxbd(rx_ring, i);
 949
 950         for (j = 0; j < buff_cnt; j++) {
 951                 /* try reuse page */
 952                 if (unlikely(!rx_swbd->page)) {
 953                         if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
 954                                 rx_ring->stats.rx_alloc_errs++;
 955                                 break;
 956                         }
 957                 }
 958
 959                 /* update RxBD */
 960                 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
 961                                            rx_swbd->page_offset);
 962                 /* clear 'R" as well */
 963                 rxbd->r.lstatus = 0;
 964
 965                 enetc_rxbd_next(rx_ring, &rxbd, &i);
 966                 rx_swbd = &rx_ring->rx_swbd[i];
 967         }
 968
 969         if (likely(j)) {
 970                 rx_ring->next_to_alloc = i; /* keep track from page reuse */
 971                 rx_ring->next_to_use = i;
 972
 973                 /* update ENETC's consumer index */
 974                 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
 975         }
 976
 977         return j;
 978 }
 979
 980 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
 981 static void enetc_get_rx_tstamp(struct net_device *ndev,
 982                                 union enetc_rx_bd *rxbd,
 983                                 struct sk_buff *skb)
 984 {
 985         struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
 986         struct enetc_ndev_priv *priv = netdev_priv(ndev);
 987         struct enetc_hw *hw = &priv->si->hw;
 988         u32 lo, hi, tstamp_lo;
 989         u64 tstamp;
 990
 991         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
 992                 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
 993                 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
 994                 rxbd = enetc_rxbd_ext(rxbd);
 995                 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
 996                 if (lo <= tstamp_lo)
 997                         hi -= 1;
 998
 999                 tstamp = (u64)hi << 32 | tstamp_lo;
1000                 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1001                 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1002         }
1003 }
1004 #endif
1005
1006 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1007                                union enetc_rx_bd *rxbd, struct sk_buff *skb)
1008 {
1009         struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1010
1011         /* TODO: hashing */
1012         if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1013                 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1014
1015                 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1016                 skb->ip_summed = CHECKSUM_COMPLETE;
1017         }
1018
1019         if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1020                 __be16 tpid = 0;
1021
1022                 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1023                 case 0:
1024                         tpid = htons(ETH_P_8021Q);
1025                         break;
1026                 case 1:
1027                         tpid = htons(ETH_P_8021AD);
1028                         break;
1029                 case 2:
1030                         tpid = htons(enetc_port_rd(&priv->si->hw,
1031                                                    ENETC_PCVLANR1));
1032                         break;
1033                 case 3:
1034                         tpid = htons(enetc_port_rd(&priv->si->hw,
1035                                                    ENETC_PCVLANR2));
1036                         break;
1037                 default:
1038                         break;
1039                 }
1040
1041                 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1042         }
1043
1044 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1045         if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1046                 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1047 #endif
1048 }
1049
1050 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1051  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1052  * mapped buffers.
1053  */
1054 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1055                                                int i, u16 size)
1056 {
1057         struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1058
1059         dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1060                                       rx_swbd->page_offset,
1061                                       size, rx_swbd->dir);
1062         return rx_swbd;
1063 }
1064
1065 /* Reuse the current page without performing half-page buffer flipping */
1066 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1067                               struct enetc_rx_swbd *rx_swbd)
1068 {
1069         size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1070
1071         enetc_reuse_page(rx_ring, rx_swbd);
1072
1073         dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1074                                          rx_swbd->page_offset,
1075                                          buffer_size, rx_swbd->dir);
1076
1077         rx_swbd->page = NULL;
1078 }
1079
1080 /* Reuse the current page by performing half-page buffer flipping */
1081 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1082                                struct enetc_rx_swbd *rx_swbd)
1083 {
1084         if (likely(enetc_page_reusable(rx_swbd->page))) {
1085                 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1086                 page_ref_inc(rx_swbd->page);
1087
1088                 enetc_put_rx_buff(rx_ring, rx_swbd);
1089         } else {
1090                 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1091                                rx_swbd->dir);
1092                 rx_swbd->page = NULL;
1093         }
1094 }
1095
1096 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1097                                                 int i, u16 size)
1098 {
1099         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1100         struct sk_buff *skb;
1101         void *ba;
1102
1103         ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1104         skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1105         if (unlikely(!skb)) {
1106                 rx_ring->stats.rx_alloc_errs++;
1107                 return NULL;
1108         }
1109
1110         skb_reserve(skb, rx_ring->buffer_offset);
1111         __skb_put(skb, size);
1112
1113         enetc_flip_rx_buff(rx_ring, rx_swbd);
1114
1115         return skb;
1116 }
1117
1118 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1119                                      u16 size, struct sk_buff *skb)
1120 {
1121         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1122
1123         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1124                         rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1125
1126         enetc_flip_rx_buff(rx_ring, rx_swbd);
1127 }
1128
1129 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1130                                               u32 bd_status,
1131                                               union enetc_rx_bd **rxbd, int *i)
1132 {
1133         if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1134                 return false;
1135
1136         enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1137         enetc_rxbd_next(rx_ring, rxbd, i);
1138
1139         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1140                 dma_rmb();
1141                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1142
1143                 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1144                 enetc_rxbd_next(rx_ring, rxbd, i);
1145         }
1146
1147         rx_ring->ndev->stats.rx_dropped++;
1148         rx_ring->ndev->stats.rx_errors++;
1149
1150         return true;
1151 }
1152
1153 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1154                                        u32 bd_status, union enetc_rx_bd **rxbd,
1155                                        int *i, int *cleaned_cnt, int buffer_size)
1156 {
1157         struct sk_buff *skb;
1158         u16 size;
1159
1160         size = le16_to_cpu((*rxbd)->r.buf_len);
1161         skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1162         if (!skb)
1163                 return NULL;
1164
1165         enetc_get_offloads(rx_ring, *rxbd, skb);
1166
1167         (*cleaned_cnt)++;
1168
1169         enetc_rxbd_next(rx_ring, rxbd, i);
1170
1171         /* not last BD in frame? */
1172         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1173                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1174                 size = buffer_size;
1175
1176                 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1177                         dma_rmb();
1178                         size = le16_to_cpu((*rxbd)->r.buf_len);
1179                 }
1180
1181                 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1182
1183                 (*cleaned_cnt)++;
1184
1185                 enetc_rxbd_next(rx_ring, rxbd, i);
1186         }
1187
1188         skb_record_rx_queue(skb, rx_ring->index);
1189         skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1190
1191         return skb;
1192 }
1193
1194 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1195
1196 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1197                                struct napi_struct *napi, int work_limit)
1198 {
1199         int rx_frm_cnt = 0, rx_byte_cnt = 0;
1200         int cleaned_cnt, i;
1201
1202         cleaned_cnt = enetc_bd_unused(rx_ring);
1203         /* next descriptor to process */
1204         i = rx_ring->next_to_clean;
1205
1206         while (likely(rx_frm_cnt < work_limit)) {
1207                 union enetc_rx_bd *rxbd;
1208                 struct sk_buff *skb;
1209                 u32 bd_status;
1210
1211                 if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1212                         cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1213                                                             cleaned_cnt);
1214
1215                 rxbd = enetc_rxbd(rx_ring, i);
1216                 bd_status = le32_to_cpu(rxbd->r.lstatus);
1217                 if (!bd_status)
1218                         break;
1219
1220                 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1221                 dma_rmb(); /* for reading other rxbd fields */
1222
1223                 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1224                                                       &rxbd, &i))
1225                         break;
1226
1227                 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1228                                       &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1229                 if (!skb)
1230                         break;
1231
1232                 /* When set, the outer VLAN header is extracted and reported
1233                  * in the receive buffer descriptor. So rx_byte_cnt should
1234                  * add the length of the extracted VLAN header.
1235                  */
1236                 if (bd_status & ENETC_RXBD_FLAG_VLAN)
1237                         rx_byte_cnt += VLAN_HLEN;
1238                 rx_byte_cnt += skb->len + ETH_HLEN;
1239                 rx_frm_cnt++;
1240
1241                 napi_gro_receive(napi, skb);
1242         }
1243
1244         rx_ring->next_to_clean = i;
1245
1246         rx_ring->stats.packets += rx_frm_cnt;
1247         rx_ring->stats.bytes += rx_byte_cnt;
1248
1249         return rx_frm_cnt;
1250 }
1251
1252 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1253                                   struct enetc_tx_swbd *tx_swbd,
1254                                   int frm_len)
1255 {
1256         union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1257
1258         prefetchw(txbd);
1259
1260         enetc_clear_tx_bd(txbd);
1261         txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1262         txbd->buf_len = cpu_to_le16(tx_swbd->len);
1263         txbd->frm_len = cpu_to_le16(frm_len);
1264
1265         memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1266 }
1267
1268 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1269  * descriptors.
1270  */
1271 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1272                          struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1273 {
1274         struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1275         int i, k, frm_len = tmp_tx_swbd->len;
1276
1277         if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1278                 return false;
1279
1280         while (unlikely(!tmp_tx_swbd->is_eof)) {
1281                 tmp_tx_swbd++;
1282                 frm_len += tmp_tx_swbd->len;
1283         }
1284
1285         i = tx_ring->next_to_use;
1286
1287         for (k = 0; k < num_tx_swbd; k++) {
1288                 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1289
1290                 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1291
1292                 /* last BD needs 'F' bit set */
1293                 if (xdp_tx_swbd->is_eof) {
1294                         union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1295
1296                         txbd->flags = ENETC_TXBD_FLAGS_F;
1297                 }
1298
1299                 enetc_bdr_idx_inc(tx_ring, &i);
1300         }
1301
1302         tx_ring->next_to_use = i;
1303
1304         return true;
1305 }
1306
1307 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1308                                           struct enetc_tx_swbd *xdp_tx_arr,
1309                                           struct xdp_frame *xdp_frame)
1310 {
1311         struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1312         struct skb_shared_info *shinfo;
1313         void *data = xdp_frame->data;
1314         int len = xdp_frame->len;
1315         skb_frag_t *frag;
1316         dma_addr_t dma;
1317         unsigned int f;
1318         int n = 0;
1319
1320         dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1321         if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1322                 netdev_err(tx_ring->ndev, "DMA map error\n");
1323                 return -1;
1324         }
1325
1326         xdp_tx_swbd->dma = dma;
1327         xdp_tx_swbd->dir = DMA_TO_DEVICE;
1328         xdp_tx_swbd->len = len;
1329         xdp_tx_swbd->is_xdp_redirect = true;
1330         xdp_tx_swbd->is_eof = false;
1331         xdp_tx_swbd->xdp_frame = NULL;
1332
1333         n++;
1334
1335         if (!xdp_frame_has_frags(xdp_frame))
1336                 goto out;
1337
1338         xdp_tx_swbd = &xdp_tx_arr[n];
1339
1340         shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1341
1342         for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1343              f++, frag++) {
1344                 data = skb_frag_address(frag);
1345                 len = skb_frag_size(frag);
1346
1347                 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1348                 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1349                         /* Undo the DMA mapping for all fragments */
1350                         while (--n >= 0)
1351                                 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1352
1353                         netdev_err(tx_ring->ndev, "DMA map error\n");
1354                         return -1;
1355                 }
1356
1357                 xdp_tx_swbd->dma = dma;
1358                 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1359                 xdp_tx_swbd->len = len;
1360                 xdp_tx_swbd->is_xdp_redirect = true;
1361                 xdp_tx_swbd->is_eof = false;
1362                 xdp_tx_swbd->xdp_frame = NULL;
1363
1364                 n++;
1365                 xdp_tx_swbd = &xdp_tx_arr[n];
1366         }
1367 out:
1368         xdp_tx_arr[n - 1].is_eof = true;
1369         xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1370
1371         return n;
1372 }
1373
1374 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1375                    struct xdp_frame **frames, u32 flags)
1376 {
1377         struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1378         struct enetc_ndev_priv *priv = netdev_priv(ndev);
1379         struct enetc_bdr *tx_ring;
1380         int xdp_tx_bd_cnt, i, k;
1381         int xdp_tx_frm_cnt = 0;
1382
1383         enetc_lock_mdio();
1384
1385         tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1386
1387         prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1388
1389         for (k = 0; k < num_frames; k++) {
1390                 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1391                                                                xdp_redirect_arr,
1392                                                                frames[k]);
1393                 if (unlikely(xdp_tx_bd_cnt < 0))
1394                         break;
1395
1396                 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1397                                            xdp_tx_bd_cnt))) {
1398                         for (i = 0; i < xdp_tx_bd_cnt; i++)
1399                                 enetc_unmap_tx_buff(tx_ring,
1400                                                     &xdp_redirect_arr[i]);
1401                         tx_ring->stats.xdp_tx_drops++;
1402                         break;
1403                 }
1404
1405                 xdp_tx_frm_cnt++;
1406         }
1407
1408         if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1409                 enetc_update_tx_ring_tail(tx_ring);
1410
1411         tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1412
1413         enetc_unlock_mdio();
1414
1415         return xdp_tx_frm_cnt;
1416 }
1417 EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
1418
1419 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1420                                      struct xdp_buff *xdp_buff, u16 size)
1421 {
1422         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1423         void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1424
1425         /* To be used for XDP_TX */
1426         rx_swbd->len = size;
1427
1428         xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1429                          rx_ring->buffer_offset, size, false);
1430 }
1431
1432 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1433                                      u16 size, struct xdp_buff *xdp_buff)
1434 {
1435         struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1436         struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1437         skb_frag_t *frag;
1438
1439         /* To be used for XDP_TX */
1440         rx_swbd->len = size;
1441
1442         if (!xdp_buff_has_frags(xdp_buff)) {
1443                 xdp_buff_set_frags_flag(xdp_buff);
1444                 shinfo->xdp_frags_size = size;
1445                 shinfo->nr_frags = 0;
1446         } else {
1447                 shinfo->xdp_frags_size += size;
1448         }
1449
1450         if (page_is_pfmemalloc(rx_swbd->page))
1451                 xdp_buff_set_frag_pfmemalloc(xdp_buff);
1452
1453         frag = &shinfo->frags[shinfo->nr_frags];
1454         skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1455                                 size);
1456
1457         shinfo->nr_frags++;
1458 }
1459
1460 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1461                                  union enetc_rx_bd **rxbd, int *i,
1462                                  int *cleaned_cnt, struct xdp_buff *xdp_buff)
1463 {
1464         u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1465
1466         xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1467
1468         enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1469         (*cleaned_cnt)++;
1470         enetc_rxbd_next(rx_ring, rxbd, i);
1471
1472         /* not last BD in frame? */
1473         while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1474                 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1475                 size = ENETC_RXB_DMA_SIZE_XDP;
1476
1477                 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1478                         dma_rmb();
1479                         size = le16_to_cpu((*rxbd)->r.buf_len);
1480                 }
1481
1482                 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1483                 (*cleaned_cnt)++;
1484                 enetc_rxbd_next(rx_ring, rxbd, i);
1485         }
1486 }
1487
1488 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1489  * recycled back into the RX ring in enetc_clean_tx_ring.
1490  */
1491 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1492                                         struct enetc_bdr *rx_ring,
1493                                         int rx_ring_first, int rx_ring_last)
1494 {
1495         int n = 0;
1496
1497         for (; rx_ring_first != rx_ring_last;
1498              n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1499                 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1500                 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1501
1502                 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1503                 tx_swbd->dma = rx_swbd->dma;
1504                 tx_swbd->dir = rx_swbd->dir;
1505                 tx_swbd->page = rx_swbd->page;
1506                 tx_swbd->page_offset = rx_swbd->page_offset;
1507                 tx_swbd->len = rx_swbd->len;
1508                 tx_swbd->is_dma_page = true;
1509                 tx_swbd->is_xdp_tx = true;
1510                 tx_swbd->is_eof = false;
1511         }
1512
1513         /* We rely on caller providing an rx_ring_last > rx_ring_first */
1514         xdp_tx_arr[n - 1].is_eof = true;
1515
1516         return n;
1517 }
1518
1519 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1520                            int rx_ring_last)
1521 {
1522         while (rx_ring_first != rx_ring_last) {
1523                 enetc_put_rx_buff(rx_ring,
1524                                   &rx_ring->rx_swbd[rx_ring_first]);
1525                 enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1526         }
1527         rx_ring->stats.xdp_drops++;
1528 }
1529
1530 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1531                                    struct napi_struct *napi, int work_limit,
1532                                    struct bpf_prog *prog)
1533 {
1534         int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1535         struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1536         struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1537         int rx_frm_cnt = 0, rx_byte_cnt = 0;
1538         struct enetc_bdr *tx_ring;
1539         int cleaned_cnt, i;
1540         u32 xdp_act;
1541
1542         cleaned_cnt = enetc_bd_unused(rx_ring);
1543         /* next descriptor to process */
1544         i = rx_ring->next_to_clean;
1545
1546         while (likely(rx_frm_cnt < work_limit)) {
1547                 union enetc_rx_bd *rxbd, *orig_rxbd;
1548                 int orig_i, orig_cleaned_cnt;
1549                 struct xdp_buff xdp_buff;
1550                 struct sk_buff *skb;
1551                 u32 bd_status;
1552                 int err;
1553
1554                 rxbd = enetc_rxbd(rx_ring, i);
1555                 bd_status = le32_to_cpu(rxbd->r.lstatus);
1556                 if (!bd_status)
1557                         break;
1558
1559                 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1560                 dma_rmb(); /* for reading other rxbd fields */
1561
1562                 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1563                                                       &rxbd, &i))
1564                         break;
1565
1566                 orig_rxbd = rxbd;
1567                 orig_cleaned_cnt = cleaned_cnt;
1568                 orig_i = i;
1569
1570                 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1571                                      &cleaned_cnt, &xdp_buff);
1572
1573                 /* When set, the outer VLAN header is extracted and reported
1574                  * in the receive buffer descriptor. So rx_byte_cnt should
1575                  * add the length of the extracted VLAN header.
1576                  */
1577                 if (bd_status & ENETC_RXBD_FLAG_VLAN)
1578                         rx_byte_cnt += VLAN_HLEN;
1579                 rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
1580
1581                 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1582
1583                 switch (xdp_act) {
1584                 default:
1585                         bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1586                         fallthrough;
1587                 case XDP_ABORTED:
1588                         trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1589                         fallthrough;
1590                 case XDP_DROP:
1591                         enetc_xdp_drop(rx_ring, orig_i, i);
1592                         break;
1593                 case XDP_PASS:
1594                         rxbd = orig_rxbd;
1595                         cleaned_cnt = orig_cleaned_cnt;
1596                         i = orig_i;
1597
1598                         skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1599                                               &i, &cleaned_cnt,
1600                                               ENETC_RXB_DMA_SIZE_XDP);
1601                         if (unlikely(!skb))
1602                                 goto out;
1603
1604                         napi_gro_receive(napi, skb);
1605                         break;
1606                 case XDP_TX:
1607                         tx_ring = priv->xdp_tx_ring[rx_ring->index];
1608                         xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1609                                                                      rx_ring,
1610                                                                      orig_i, i);
1611
1612                         if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1613                                 enetc_xdp_drop(rx_ring, orig_i, i);
1614                                 tx_ring->stats.xdp_tx_drops++;
1615                         } else {
1616                                 tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1617                                 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1618                                 xdp_tx_frm_cnt++;
1619                                 /* The XDP_TX enqueue was successful, so we
1620                                  * need to scrub the RX software BDs because
1621                                  * the ownership of the buffers no longer
1622                                  * belongs to the RX ring, and we must prevent
1623                                  * enetc_refill_rx_ring() from reusing
1624                                  * rx_swbd->page.
1625                                  */
1626                                 while (orig_i != i) {
1627                                         rx_ring->rx_swbd[orig_i].page = NULL;
1628                                         enetc_bdr_idx_inc(rx_ring, &orig_i);
1629                                 }
1630                         }
1631                         break;
1632                 case XDP_REDIRECT:
1633                         err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1634                         if (unlikely(err)) {
1635                                 enetc_xdp_drop(rx_ring, orig_i, i);
1636                                 rx_ring->stats.xdp_redirect_failures++;
1637                         } else {
1638                                 while (orig_i != i) {
1639                                         enetc_flip_rx_buff(rx_ring,
1640                                                            &rx_ring->rx_swbd[orig_i]);
1641                                         enetc_bdr_idx_inc(rx_ring, &orig_i);
1642                                 }
1643                                 xdp_redirect_frm_cnt++;
1644                                 rx_ring->stats.xdp_redirect++;
1645                         }
1646                 }
1647
1648                 rx_frm_cnt++;
1649         }
1650
1651 out:
1652         rx_ring->next_to_clean = i;
1653
1654         rx_ring->stats.packets += rx_frm_cnt;
1655         rx_ring->stats.bytes += rx_byte_cnt;
1656
1657         if (xdp_redirect_frm_cnt)
1658                 xdp_do_flush();
1659
1660         if (xdp_tx_frm_cnt)
1661                 enetc_update_tx_ring_tail(tx_ring);
1662
1663         if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1664                 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1665                                      rx_ring->xdp.xdp_tx_in_flight);
1666
1667         return rx_frm_cnt;
1668 }
1669
1670 static int enetc_poll(struct napi_struct *napi, int budget)
1671 {
1672         struct enetc_int_vector
1673                 *v = container_of(napi, struct enetc_int_vector, napi);
1674         struct enetc_bdr *rx_ring = &v->rx_ring;
1675         struct bpf_prog *prog;
1676         bool complete = true;
1677         int work_done;
1678         int i;
1679
1680         enetc_lock_mdio();
1681
1682         for (i = 0; i < v->count_tx_rings; i++)
1683                 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1684                         complete = false;
1685
1686         prog = rx_ring->xdp.prog;
1687         if (prog)
1688                 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1689         else
1690                 work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1691         if (work_done == budget)
1692                 complete = false;
1693         if (work_done)
1694                 v->rx_napi_work = true;
1695
1696         if (!complete) {
1697                 enetc_unlock_mdio();
1698                 return budget;
1699         }
1700
1701         napi_complete_done(napi, work_done);
1702
1703         if (likely(v->rx_dim_en))
1704                 enetc_rx_net_dim(v);
1705
1706         v->rx_napi_work = false;
1707
1708         /* enable interrupts */
1709         enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1710
1711         for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1712                 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1713                                  ENETC_TBIER_TXTIE);
1714
1715         enetc_unlock_mdio();
1716
1717         return work_done;
1718 }
1719
1720 /* Probing and Init */
1721 #define ENETC_MAX_RFS_SIZE 64
1722 void enetc_get_si_caps(struct enetc_si *si)
1723 {
1724         struct enetc_hw *hw = &si->hw;
1725         u32 val;
1726
1727         /* find out how many of various resources we have to work with */
1728         val = enetc_rd(hw, ENETC_SICAPR0);
1729         si->num_rx_rings = (val >> 16) & 0xff;
1730         si->num_tx_rings = val & 0xff;
1731
1732         val = enetc_rd(hw, ENETC_SIRFSCAPR);
1733         si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1734         si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1735
1736         si->num_rss = 0;
1737         val = enetc_rd(hw, ENETC_SIPCAPR0);
1738         if (val & ENETC_SIPCAPR0_RSS) {
1739                 u32 rss;
1740
1741                 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1742                 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1743         }
1744
1745         if (val & ENETC_SIPCAPR0_QBV)
1746                 si->hw_features |= ENETC_SI_F_QBV;
1747
1748         if (val & ENETC_SIPCAPR0_QBU)
1749                 si->hw_features |= ENETC_SI_F_QBU;
1750
1751         if (val & ENETC_SIPCAPR0_PSFP)
1752                 si->hw_features |= ENETC_SI_F_PSFP;
1753 }
1754 EXPORT_SYMBOL_GPL(enetc_get_si_caps);
1755
1756 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
1757 {
1758         size_t bd_base_size = res->bd_count * res->bd_size;
1759
1760         res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
1761                                           &res->bd_dma_base, GFP_KERNEL);
1762         if (!res->bd_base)
1763                 return -ENOMEM;
1764
1765         /* h/w requires 128B alignment */
1766         if (!IS_ALIGNED(res->bd_dma_base, 128)) {
1767                 dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1768                                   res->bd_dma_base);
1769                 return -EINVAL;
1770         }
1771
1772         return 0;
1773 }
1774
1775 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
1776 {
1777         size_t bd_base_size = res->bd_count * res->bd_size;
1778
1779         dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1780                           res->bd_dma_base);
1781 }
1782
1783 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
1784                                    struct device *dev, size_t bd_count)
1785 {
1786         int err;
1787
1788         res->dev = dev;
1789         res->bd_count = bd_count;
1790         res->bd_size = sizeof(union enetc_tx_bd);
1791
1792         res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
1793         if (!res->tx_swbd)
1794                 return -ENOMEM;
1795
1796         err = enetc_dma_alloc_bdr(res);
1797         if (err)
1798                 goto err_alloc_bdr;
1799
1800         res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
1801                                               &res->tso_headers_dma,
1802                                               GFP_KERNEL);
1803         if (!res->tso_headers) {
1804                 err = -ENOMEM;
1805                 goto err_alloc_tso;
1806         }
1807
1808         return 0;
1809
1810 err_alloc_tso:
1811         enetc_dma_free_bdr(res);
1812 err_alloc_bdr:
1813         vfree(res->tx_swbd);
1814         res->tx_swbd = NULL;
1815
1816         return err;
1817 }
1818
1819 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
1820 {
1821         dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
1822                           res->tso_headers, res->tso_headers_dma);
1823         enetc_dma_free_bdr(res);
1824         vfree(res->tx_swbd);
1825 }
1826
1827 static struct enetc_bdr_resource *
1828 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1829 {
1830         struct enetc_bdr_resource *tx_res;
1831         int i, err;
1832
1833         tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);
1834         if (!tx_res)
1835                 return ERR_PTR(-ENOMEM);
1836
1837         for (i = 0; i < priv->num_tx_rings; i++) {
1838                 struct enetc_bdr *tx_ring = priv->tx_ring[i];
1839
1840                 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
1841                                               tx_ring->bd_count);
1842                 if (err)
1843                         goto fail;
1844         }
1845
1846         return tx_res;
1847
1848 fail:
1849         while (i-- > 0)
1850                 enetc_free_tx_resource(&tx_res[i]);
1851
1852         kfree(tx_res);
1853
1854         return ERR_PTR(err);
1855 }
1856
1857 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
1858                                     size_t num_resources)
1859 {
1860         size_t i;
1861
1862         for (i = 0; i < num_resources; i++)
1863                 enetc_free_tx_resource(&tx_res[i]);
1864
1865         kfree(tx_res);
1866 }
1867
1868 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
1869                                    struct device *dev, size_t bd_count,
1870                                    bool extended)
1871 {
1872         int err;
1873
1874         res->dev = dev;
1875         res->bd_count = bd_count;
1876         res->bd_size = sizeof(union enetc_rx_bd);
1877         if (extended)
1878                 res->bd_size *= 2;
1879
1880         res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
1881         if (!res->rx_swbd)
1882                 return -ENOMEM;
1883
1884         err = enetc_dma_alloc_bdr(res);
1885         if (err) {
1886                 vfree(res->rx_swbd);
1887                 return err;
1888         }
1889
1890         return 0;
1891 }
1892
1893 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
1894 {
1895         enetc_dma_free_bdr(res);
1896         vfree(res->rx_swbd);
1897 }
1898
1899 static struct enetc_bdr_resource *
1900 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
1901 {
1902         struct enetc_bdr_resource *rx_res;
1903         int i, err;
1904
1905         rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);
1906         if (!rx_res)
1907                 return ERR_PTR(-ENOMEM);
1908
1909         for (i = 0; i < priv->num_rx_rings; i++) {
1910                 struct enetc_bdr *rx_ring = priv->rx_ring[i];
1911
1912                 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
1913                                               rx_ring->bd_count, extended);
1914                 if (err)
1915                         goto fail;
1916         }
1917
1918         return rx_res;
1919
1920 fail:
1921         while (i-- > 0)
1922                 enetc_free_rx_resource(&rx_res[i]);
1923
1924         kfree(rx_res);
1925
1926         return ERR_PTR(err);
1927 }
1928
1929 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
1930                                     size_t num_resources)
1931 {
1932         size_t i;
1933
1934         for (i = 0; i < num_resources; i++)
1935                 enetc_free_rx_resource(&rx_res[i]);
1936
1937         kfree(rx_res);
1938 }
1939
1940 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
1941                                      const struct enetc_bdr_resource *res)
1942 {
1943         tx_ring->bd_base = res ? res->bd_base : NULL;
1944         tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1945         tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
1946         tx_ring->tso_headers = res ? res->tso_headers : NULL;
1947         tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
1948 }
1949
1950 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
1951                                      const struct enetc_bdr_resource *res)
1952 {
1953         rx_ring->bd_base = res ? res->bd_base : NULL;
1954         rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1955         rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
1956 }
1957
1958 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
1959                                       const struct enetc_bdr_resource *res)
1960 {
1961         int i;
1962
1963         if (priv->tx_res)
1964                 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
1965
1966         for (i = 0; i < priv->num_tx_rings; i++) {
1967                 enetc_assign_tx_resource(priv->tx_ring[i],
1968                                          res ? &res[i] : NULL);
1969         }
1970
1971         priv->tx_res = res;
1972 }
1973
1974 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
1975                                       const struct enetc_bdr_resource *res)
1976 {
1977         int i;
1978
1979         if (priv->rx_res)
1980                 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
1981
1982         for (i = 0; i < priv->num_rx_rings; i++) {
1983                 enetc_assign_rx_resource(priv->rx_ring[i],
1984                                          res ? &res[i] : NULL);
1985         }
1986
1987         priv->rx_res = res;
1988 }
1989
1990 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
1991 {
1992         int i;
1993
1994         for (i = 0; i < tx_ring->bd_count; i++) {
1995                 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
1996
1997                 enetc_free_tx_frame(tx_ring, tx_swbd);
1998         }
1999 }
2000
2001 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
2002 {
2003         int i;
2004
2005         for (i = 0; i < rx_ring->bd_count; i++) {
2006                 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2007
2008                 if (!rx_swbd->page)
2009                         continue;
2010
2011                 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
2012                                rx_swbd->dir);
2013                 __free_page(rx_swbd->page);
2014                 rx_swbd->page = NULL;
2015         }
2016 }
2017
2018 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2019 {
2020         int i;
2021
2022         for (i = 0; i < priv->num_rx_rings; i++)
2023                 enetc_free_rx_ring(priv->rx_ring[i]);
2024
2025         for (i = 0; i < priv->num_tx_rings; i++)
2026                 enetc_free_tx_ring(priv->tx_ring[i]);
2027 }
2028
2029 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2030 {
2031         int *rss_table;
2032         int i;
2033
2034         rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
2035         if (!rss_table)
2036                 return -ENOMEM;
2037
2038         /* Set up RSS table defaults */
2039         for (i = 0; i < si->num_rss; i++)
2040                 rss_table[i] = i % num_groups;
2041
2042         enetc_set_rss_table(si, rss_table, si->num_rss);
2043
2044         kfree(rss_table);
2045
2046         return 0;
2047 }
2048
2049 int enetc_configure_si(struct enetc_ndev_priv *priv)
2050 {
2051         struct enetc_si *si = priv->si;
2052         struct enetc_hw *hw = &si->hw;
2053         int err;
2054
2055         /* set SI cache attributes */
2056         enetc_wr(hw, ENETC_SICAR0,
2057                  ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2058         enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2059         /* enable SI */
2060         enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2061
2062         if (si->num_rss) {
2063                 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2064                 if (err)
2065                         return err;
2066         }
2067
2068         return 0;
2069 }
2070 EXPORT_SYMBOL_GPL(enetc_configure_si);
2071
2072 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2073 {
2074         struct enetc_si *si = priv->si;
2075         int cpus = num_online_cpus();
2076
2077         priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
2078         priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2079
2080         /* Enable all available TX rings in order to configure as many
2081          * priorities as possible, when needed.
2082          * TODO: Make # of TX rings run-time configurable
2083          */
2084         priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2085         priv->num_tx_rings = si->num_tx_rings;
2086         priv->bdr_int_num = cpus;
2087         priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2088         priv->tx_ictt = ENETC_TXIC_TIMETHR;
2089 }
2090 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2091
2092 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2093 {
2094         struct enetc_si *si = priv->si;
2095
2096         priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2097                                   GFP_KERNEL);
2098         if (!priv->cls_rules)
2099                 return -ENOMEM;
2100
2101         return 0;
2102 }
2103 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2104
2105 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2106 {
2107         kfree(priv->cls_rules);
2108 }
2109 EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2110
2111 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2112 {
2113         int idx = tx_ring->index;
2114         u32 tbmr;
2115
2116         enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2117                        lower_32_bits(tx_ring->bd_dma_base));
2118
2119         enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2120                        upper_32_bits(tx_ring->bd_dma_base));
2121
2122         WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2123         enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2124                        ENETC_RTBLENR_LEN(tx_ring->bd_count));
2125
2126         /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2127         tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2128         tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2129
2130         /* enable Tx ints by setting pkt thr to 1 */
2131         enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2132
2133         tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2134         if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2135                 tbmr |= ENETC_TBMR_VIH;
2136
2137         /* enable ring */
2138         enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2139
2140         tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2141         tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2142         tx_ring->idr = hw->reg + ENETC_SITXIDR;
2143 }
2144
2145 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2146                               bool extended)
2147 {
2148         int idx = rx_ring->index;
2149         u32 rbmr = 0;
2150
2151         enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2152                        lower_32_bits(rx_ring->bd_dma_base));
2153
2154         enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2155                        upper_32_bits(rx_ring->bd_dma_base));
2156
2157         WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2158         enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2159                        ENETC_RTBLENR_LEN(rx_ring->bd_count));
2160
2161         if (rx_ring->xdp.prog)
2162                 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2163         else
2164                 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2165
2166         /* Also prepare the consumer index in case page allocation never
2167          * succeeds. In that case, hardware will never advance producer index
2168          * to match consumer index, and will drop all frames.
2169          */
2170         enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2171         enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2172
2173         /* enable Rx ints by setting pkt thr to 1 */
2174         enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2175
2176         rx_ring->ext_en = extended;
2177         if (rx_ring->ext_en)
2178                 rbmr |= ENETC_RBMR_BDS;
2179
2180         if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2181                 rbmr |= ENETC_RBMR_VTE;
2182
2183         rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2184         rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2185
2186         rx_ring->next_to_clean = 0;
2187         rx_ring->next_to_use = 0;
2188         rx_ring->next_to_alloc = 0;
2189
2190         enetc_lock_mdio();
2191         enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2192         enetc_unlock_mdio();
2193
2194         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2195 }
2196
2197 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2198 {
2199         struct enetc_hw *hw = &priv->si->hw;
2200         int i;
2201
2202         for (i = 0; i < priv->num_tx_rings; i++)
2203                 enetc_setup_txbdr(hw, priv->tx_ring[i]);
2204
2205         for (i = 0; i < priv->num_rx_rings; i++)
2206                 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2207 }
2208
2209 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2210 {
2211         int idx = tx_ring->index;
2212         u32 tbmr;
2213
2214         tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2215         tbmr |= ENETC_TBMR_EN;
2216         enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2217 }
2218
2219 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2220 {
2221         int idx = rx_ring->index;
2222         u32 rbmr;
2223
2224         rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);
2225         rbmr |= ENETC_RBMR_EN;
2226         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2227 }
2228
2229 static void enetc_enable_bdrs(struct enetc_ndev_priv *priv)
2230 {
2231         struct enetc_hw *hw = &priv->si->hw;
2232         int i;
2233
2234         for (i = 0; i < priv->num_tx_rings; i++)
2235                 enetc_enable_txbdr(hw, priv->tx_ring[i]);
2236
2237         for (i = 0; i < priv->num_rx_rings; i++)
2238                 enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2239 }
2240
2241 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2242 {
2243         int idx = rx_ring->index;
2244
2245         /* disable EN bit on ring */
2246         enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2247 }
2248
2249 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2250 {
2251         int idx = rx_ring->index;
2252
2253         /* disable EN bit on ring */
2254         enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2255 }
2256
2257 static void enetc_disable_bdrs(struct enetc_ndev_priv *priv)
2258 {
2259         struct enetc_hw *hw = &priv->si->hw;
2260         int i;
2261
2262         for (i = 0; i < priv->num_tx_rings; i++)
2263                 enetc_disable_txbdr(hw, priv->tx_ring[i]);
2264
2265         for (i = 0; i < priv->num_rx_rings; i++)
2266                 enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2267 }
2268
2269 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2270 {
2271         int delay = 8, timeout = 100;
2272         int idx = tx_ring->index;
2273
2274         /* wait for busy to clear */
2275         while (delay < timeout &&
2276                enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2277                 msleep(delay);
2278                 delay *= 2;
2279         }
2280
2281         if (delay >= timeout)
2282                 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2283                             idx);
2284 }
2285
2286 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2287 {
2288         struct enetc_hw *hw = &priv->si->hw;
2289         int i;
2290
2291         for (i = 0; i < priv->num_tx_rings; i++)
2292                 enetc_wait_txbdr(hw, priv->tx_ring[i]);
2293 }
2294
2295 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2296 {
2297         struct pci_dev *pdev = priv->si->pdev;
2298         struct enetc_hw *hw = &priv->si->hw;
2299         int i, j, err;
2300
2301         for (i = 0; i < priv->bdr_int_num; i++) {
2302                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2303                 struct enetc_int_vector *v = priv->int_vector[i];
2304                 int entry = ENETC_BDR_INT_BASE_IDX + i;
2305
2306                 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2307                          priv->ndev->name, i);
2308                 err = request_irq(irq, enetc_msix, 0, v->name, v);
2309                 if (err) {
2310                         dev_err(priv->dev, "request_irq() failed!\n");
2311                         goto irq_err;
2312                 }
2313                 disable_irq(irq);
2314
2315                 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2316                 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2317                 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2318
2319                 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2320
2321                 for (j = 0; j < v->count_tx_rings; j++) {
2322                         int idx = v->tx_ring[j].index;
2323
2324                         enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2325                 }
2326                 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2327         }
2328
2329         return 0;
2330
2331 irq_err:
2332         while (i--) {
2333                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2334
2335                 irq_set_affinity_hint(irq, NULL);
2336                 free_irq(irq, priv->int_vector[i]);
2337         }
2338
2339         return err;
2340 }
2341
2342 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2343 {
2344         struct pci_dev *pdev = priv->si->pdev;
2345         int i;
2346
2347         for (i = 0; i < priv->bdr_int_num; i++) {
2348                 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2349
2350                 irq_set_affinity_hint(irq, NULL);
2351                 free_irq(irq, priv->int_vector[i]);
2352         }
2353 }
2354
2355 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2356 {
2357         struct enetc_hw *hw = &priv->si->hw;
2358         u32 icpt, ictt;
2359         int i;
2360
2361         /* enable Tx & Rx event indication */
2362         if (priv->ic_mode &
2363             (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2364                 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2365                 /* init to non-0 minimum, will be adjusted later */
2366                 ictt = 0x1;
2367         } else {
2368                 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2369                 ictt = 0;
2370         }
2371
2372         for (i = 0; i < priv->num_rx_rings; i++) {
2373                 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2374                 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2375                 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2376         }
2377
2378         if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2379                 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2380         else
2381                 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2382
2383         for (i = 0; i < priv->num_tx_rings; i++) {
2384                 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2385                 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2386                 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2387         }
2388 }
2389
2390 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2391 {
2392         struct enetc_hw *hw = &priv->si->hw;
2393         int i;
2394
2395         for (i = 0; i < priv->num_tx_rings; i++)
2396                 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2397
2398         for (i = 0; i < priv->num_rx_rings; i++)
2399                 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2400 }
2401
2402 static int enetc_phylink_connect(struct net_device *ndev)
2403 {
2404         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2405         struct ethtool_eee edata;
2406         int err;
2407
2408         if (!priv->phylink) {
2409                 /* phy-less mode */
2410                 netif_carrier_on(ndev);
2411                 return 0;
2412         }
2413
2414         err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2415         if (err) {
2416                 dev_err(&ndev->dev, "could not attach to PHY\n");
2417                 return err;
2418         }
2419
2420         /* disable EEE autoneg, until ENETC driver supports it */
2421         memset(&edata, 0, sizeof(struct ethtool_eee));
2422         phylink_ethtool_set_eee(priv->phylink, &edata);
2423
2424         phylink_start(priv->phylink);
2425
2426         return 0;
2427 }
2428
2429 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2430 {
2431         struct enetc_ndev_priv *priv;
2432         struct sk_buff *skb;
2433
2434         priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2435
2436         netif_tx_lock_bh(priv->ndev);
2437
2438         clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2439         skb = skb_dequeue(&priv->tx_skbs);
2440         if (skb)
2441                 enetc_start_xmit(skb, priv->ndev);
2442
2443         netif_tx_unlock_bh(priv->ndev);
2444 }
2445
2446 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2447 {
2448         INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2449         skb_queue_head_init(&priv->tx_skbs);
2450 }
2451
2452 void enetc_start(struct net_device *ndev)
2453 {
2454         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2455         int i;
2456
2457         enetc_setup_interrupts(priv);
2458
2459         for (i = 0; i < priv->bdr_int_num; i++) {
2460                 int irq = pci_irq_vector(priv->si->pdev,
2461                                          ENETC_BDR_INT_BASE_IDX + i);
2462
2463                 napi_enable(&priv->int_vector[i]->napi);
2464                 enable_irq(irq);
2465         }
2466
2467         enetc_enable_bdrs(priv);
2468
2469         netif_tx_start_all_queues(ndev);
2470 }
2471 EXPORT_SYMBOL_GPL(enetc_start);
2472
2473 int enetc_open(struct net_device *ndev)
2474 {
2475         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2476         struct enetc_bdr_resource *tx_res, *rx_res;
2477         bool extended;
2478         int err;
2479
2480         extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2481
2482         err = enetc_setup_irqs(priv);
2483         if (err)
2484                 return err;
2485
2486         err = enetc_phylink_connect(ndev);
2487         if (err)
2488                 goto err_phy_connect;
2489
2490         tx_res = enetc_alloc_tx_resources(priv);
2491         if (IS_ERR(tx_res)) {
2492                 err = PTR_ERR(tx_res);
2493                 goto err_alloc_tx;
2494         }
2495
2496         rx_res = enetc_alloc_rx_resources(priv, extended);
2497         if (IS_ERR(rx_res)) {
2498                 err = PTR_ERR(rx_res);
2499                 goto err_alloc_rx;
2500         }
2501
2502         enetc_tx_onestep_tstamp_init(priv);
2503         enetc_assign_tx_resources(priv, tx_res);
2504         enetc_assign_rx_resources(priv, rx_res);
2505         enetc_setup_bdrs(priv, extended);
2506         enetc_start(ndev);
2507
2508         return 0;
2509
2510 err_alloc_rx:
2511         enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2512 err_alloc_tx:
2513         if (priv->phylink)
2514                 phylink_disconnect_phy(priv->phylink);
2515 err_phy_connect:
2516         enetc_free_irqs(priv);
2517
2518         return err;
2519 }
2520 EXPORT_SYMBOL_GPL(enetc_open);
2521
2522 void enetc_stop(struct net_device *ndev)
2523 {
2524         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2525         int i;
2526
2527         netif_tx_stop_all_queues(ndev);
2528
2529         enetc_disable_bdrs(priv);
2530
2531         for (i = 0; i < priv->bdr_int_num; i++) {
2532                 int irq = pci_irq_vector(priv->si->pdev,
2533                                          ENETC_BDR_INT_BASE_IDX + i);
2534
2535                 disable_irq(irq);
2536                 napi_synchronize(&priv->int_vector[i]->napi);
2537                 napi_disable(&priv->int_vector[i]->napi);
2538         }
2539
2540         enetc_wait_bdrs(priv);
2541
2542         enetc_clear_interrupts(priv);
2543 }
2544 EXPORT_SYMBOL_GPL(enetc_stop);
2545
2546 int enetc_close(struct net_device *ndev)
2547 {
2548         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2549
2550         enetc_stop(ndev);
2551
2552         if (priv->phylink) {
2553                 phylink_stop(priv->phylink);
2554                 phylink_disconnect_phy(priv->phylink);
2555         } else {
2556                 netif_carrier_off(ndev);
2557         }
2558
2559         enetc_free_rxtx_rings(priv);
2560
2561         /* Avoids dangling pointers and also frees old resources */
2562         enetc_assign_rx_resources(priv, NULL);
2563         enetc_assign_tx_resources(priv, NULL);
2564
2565         enetc_free_irqs(priv);
2566
2567         return 0;
2568 }
2569 EXPORT_SYMBOL_GPL(enetc_close);
2570
2571 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
2572                              int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
2573                              void *ctx)
2574 {
2575         struct enetc_bdr_resource *tx_res, *rx_res;
2576         int err;
2577
2578         ASSERT_RTNL();
2579
2580         /* If the interface is down, run the callback right away,
2581          * without reconfiguration.
2582          */
2583         if (!netif_running(priv->ndev)) {
2584                 if (cb) {
2585                         err = cb(priv, ctx);
2586                         if (err)
2587                                 return err;
2588                 }
2589
2590                 return 0;
2591         }
2592
2593         tx_res = enetc_alloc_tx_resources(priv);
2594         if (IS_ERR(tx_res)) {
2595                 err = PTR_ERR(tx_res);
2596                 goto out;
2597         }
2598
2599         rx_res = enetc_alloc_rx_resources(priv, extended);
2600         if (IS_ERR(rx_res)) {
2601                 err = PTR_ERR(rx_res);
2602                 goto out_free_tx_res;
2603         }
2604
2605         enetc_stop(priv->ndev);
2606         enetc_free_rxtx_rings(priv);
2607
2608         /* Interface is down, run optional callback now */
2609         if (cb) {
2610                 err = cb(priv, ctx);
2611                 if (err)
2612                         goto out_restart;
2613         }
2614
2615         enetc_assign_tx_resources(priv, tx_res);
2616         enetc_assign_rx_resources(priv, rx_res);
2617         enetc_setup_bdrs(priv, extended);
2618         enetc_start(priv->ndev);
2619
2620         return 0;
2621
2622 out_restart:
2623         enetc_setup_bdrs(priv, extended);
2624         enetc_start(priv->ndev);
2625         enetc_free_rx_resources(rx_res, priv->num_rx_rings);
2626 out_free_tx_res:
2627         enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2628 out:
2629         return err;
2630 }
2631
2632 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
2633 {
2634         int i;
2635
2636         for (i = 0; i < priv->num_tx_rings; i++)
2637                 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
2638                            priv->tx_ring[i]->prio);
2639 }
2640
2641 void enetc_reset_tc_mqprio(struct net_device *ndev)
2642 {
2643         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2644         struct enetc_hw *hw = &priv->si->hw;
2645         struct enetc_bdr *tx_ring;
2646         int num_stack_tx_queues;
2647         int i;
2648
2649         num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2650
2651         netdev_reset_tc(ndev);
2652         netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2653         priv->min_num_stack_tx_queues = num_possible_cpus();
2654
2655         /* Reset all ring priorities to 0 */
2656         for (i = 0; i < priv->num_tx_rings; i++) {
2657                 tx_ring = priv->tx_ring[i];
2658                 tx_ring->prio = 0;
2659                 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2660         }
2661
2662         enetc_debug_tx_ring_prios(priv);
2663
2664         enetc_change_preemptible_tcs(priv, 0);
2665 }
2666 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
2667
2668 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2669 {
2670         struct tc_mqprio_qopt_offload *mqprio = type_data;
2671         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2672         struct tc_mqprio_qopt *qopt = &mqprio->qopt;
2673         struct enetc_hw *hw = &priv->si->hw;
2674         int num_stack_tx_queues = 0;
2675         struct enetc_bdr *tx_ring;
2676         u8 num_tc = qopt->num_tc;
2677         int offset, count;
2678         int err, tc, q;
2679
2680         if (!num_tc) {
2681                 enetc_reset_tc_mqprio(ndev);
2682                 return 0;
2683         }
2684
2685         err = netdev_set_num_tc(ndev, num_tc);
2686         if (err)
2687                 return err;
2688
2689         for (tc = 0; tc < num_tc; tc++) {
2690                 offset = qopt->offset[tc];
2691                 count = qopt->count[tc];
2692                 num_stack_tx_queues += count;
2693
2694                 err = netdev_set_tc_queue(ndev, tc, count, offset);
2695                 if (err)
2696                         goto err_reset_tc;
2697
2698                 for (q = offset; q < offset + count; q++) {
2699                         tx_ring = priv->tx_ring[q];
2700                         /* The prio_tc_map is skb_tx_hash()'s way of selecting
2701                          * between TX queues based on skb->priority. As such,
2702                          * there's nothing to offload based on it.
2703                          * Make the mqprio "traffic class" be the priority of
2704                          * this ring group, and leave the Tx IPV to traffic
2705                          * class mapping as its default mapping value of 1:1.
2706                          */
2707                         tx_ring->prio = tc;
2708                         enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2709                 }
2710         }
2711
2712         err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2713         if (err)
2714                 goto err_reset_tc;
2715
2716         priv->min_num_stack_tx_queues = num_stack_tx_queues;
2717
2718         enetc_debug_tx_ring_prios(priv);
2719
2720         enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
2721
2722         return 0;
2723
2724 err_reset_tc:
2725         enetc_reset_tc_mqprio(ndev);
2726         return err;
2727 }
2728 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
2729
2730 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
2731 {
2732         struct bpf_prog *old_prog, *prog = ctx;
2733         int num_stack_tx_queues;
2734         int err, i;
2735
2736         old_prog = xchg(&priv->xdp_prog, prog);
2737
2738         num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2739         err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
2740         if (err) {
2741                 xchg(&priv->xdp_prog, old_prog);
2742                 return err;
2743         }
2744
2745         if (old_prog)
2746                 bpf_prog_put(old_prog);
2747
2748         for (i = 0; i < priv->num_rx_rings; i++) {
2749                 struct enetc_bdr *rx_ring = priv->rx_ring[i];
2750
2751                 rx_ring->xdp.prog = prog;
2752
2753                 if (prog)
2754                         rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2755                 else
2756                         rx_ring->buffer_offset = ENETC_RXB_PAD;
2757         }
2758
2759         return 0;
2760 }
2761
2762 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
2763                                 struct netlink_ext_ack *extack)
2764 {
2765         int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
2766         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2767         bool extended;
2768
2769         if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
2770             priv->num_tx_rings) {
2771                 NL_SET_ERR_MSG_FMT_MOD(extack,
2772                                        "Reserving %d XDP TXQs does not leave a minimum of %d for stack (total %d)",
2773                                        num_xdp_tx_queues,
2774                                        priv->min_num_stack_tx_queues,
2775                                        priv->num_tx_rings);
2776                 return -EBUSY;
2777         }
2778
2779         extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2780
2781         /* The buffer layout is changing, so we need to drain the old
2782          * RX buffers and seed new ones.
2783          */
2784         return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
2785 }
2786
2787 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
2788 {
2789         switch (bpf->command) {
2790         case XDP_SETUP_PROG:
2791                 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
2792         default:
2793                 return -EINVAL;
2794         }
2795
2796         return 0;
2797 }
2798 EXPORT_SYMBOL_GPL(enetc_setup_bpf);
2799
2800 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2801 {
2802         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2803         struct net_device_stats *stats = &ndev->stats;
2804         unsigned long packets = 0, bytes = 0;
2805         unsigned long tx_dropped = 0;
2806         int i;
2807
2808         for (i = 0; i < priv->num_rx_rings; i++) {
2809                 packets += priv->rx_ring[i]->stats.packets;
2810                 bytes   += priv->rx_ring[i]->stats.bytes;
2811         }
2812
2813         stats->rx_packets = packets;
2814         stats->rx_bytes = bytes;
2815         bytes = 0;
2816         packets = 0;
2817
2818         for (i = 0; i < priv->num_tx_rings; i++) {
2819                 packets += priv->tx_ring[i]->stats.packets;
2820                 bytes   += priv->tx_ring[i]->stats.bytes;
2821                 tx_dropped += priv->tx_ring[i]->stats.win_drop;
2822         }
2823
2824         stats->tx_packets = packets;
2825         stats->tx_bytes = bytes;
2826         stats->tx_dropped = tx_dropped;
2827
2828         return stats;
2829 }
2830 EXPORT_SYMBOL_GPL(enetc_get_stats);
2831
2832 static int enetc_set_rss(struct net_device *ndev, int en)
2833 {
2834         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2835         struct enetc_hw *hw = &priv->si->hw;
2836         u32 reg;
2837
2838         enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2839
2840         reg = enetc_rd(hw, ENETC_SIMR);
2841         reg &= ~ENETC_SIMR_RSSE;
2842         reg |= (en) ? ENETC_SIMR_RSSE : 0;
2843         enetc_wr(hw, ENETC_SIMR, reg);
2844
2845         return 0;
2846 }
2847
2848 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2849 {
2850         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2851         struct enetc_hw *hw = &priv->si->hw;
2852         int i;
2853
2854         for (i = 0; i < priv->num_rx_rings; i++)
2855                 enetc_bdr_enable_rxvlan(hw, i, en);
2856 }
2857
2858 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2859 {
2860         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2861         struct enetc_hw *hw = &priv->si->hw;
2862         int i;
2863
2864         for (i = 0; i < priv->num_tx_rings; i++)
2865                 enetc_bdr_enable_txvlan(hw, i, en);
2866 }
2867
2868 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2869 {
2870         netdev_features_t changed = ndev->features ^ features;
2871
2872         if (changed & NETIF_F_RXHASH)
2873                 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2874
2875         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2876                 enetc_enable_rxvlan(ndev,
2877                                     !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2878
2879         if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2880                 enetc_enable_txvlan(ndev,
2881                                     !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2882 }
2883 EXPORT_SYMBOL_GPL(enetc_set_features);
2884
2885 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2886 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2887 {
2888         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2889         int err, new_offloads = priv->active_offloads;
2890         struct hwtstamp_config config;
2891
2892         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2893                 return -EFAULT;
2894
2895         switch (config.tx_type) {
2896         case HWTSTAMP_TX_OFF:
2897                 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2898                 break;
2899         case HWTSTAMP_TX_ON:
2900                 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2901                 new_offloads |= ENETC_F_TX_TSTAMP;
2902                 break;
2903         case HWTSTAMP_TX_ONESTEP_SYNC:
2904                 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2905                 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2906                 break;
2907         default:
2908                 return -ERANGE;
2909         }
2910
2911         switch (config.rx_filter) {
2912         case HWTSTAMP_FILTER_NONE:
2913                 new_offloads &= ~ENETC_F_RX_TSTAMP;
2914                 break;
2915         default:
2916                 new_offloads |= ENETC_F_RX_TSTAMP;
2917                 config.rx_filter = HWTSTAMP_FILTER_ALL;
2918         }
2919
2920         if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
2921                 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
2922
2923                 err = enetc_reconfigure(priv, extended, NULL, NULL);
2924                 if (err)
2925                         return err;
2926         }
2927
2928         priv->active_offloads = new_offloads;
2929
2930         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2931                -EFAULT : 0;
2932 }
2933
2934 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2935 {
2936         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2937         struct hwtstamp_config config;
2938
2939         config.flags = 0;
2940
2941         if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2942                 config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2943         else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2944                 config.tx_type = HWTSTAMP_TX_ON;
2945         else
2946                 config.tx_type = HWTSTAMP_TX_OFF;
2947
2948         config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2949                             HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2950
2951         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2952                -EFAULT : 0;
2953 }
2954 #endif
2955
2956 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2957 {
2958         struct enetc_ndev_priv *priv = netdev_priv(ndev);
2959 #ifdef CONFIG_FSL_ENETC_PTP_CLOCK
2960         if (cmd == SIOCSHWTSTAMP)
2961                 return enetc_hwtstamp_set(ndev, rq);
2962         if (cmd == SIOCGHWTSTAMP)
2963                 return enetc_hwtstamp_get(ndev, rq);
2964 #endif
2965
2966         if (!priv->phylink)
2967                 return -EOPNOTSUPP;
2968
2969         return phylink_mii_ioctl(priv->phylink, rq, cmd);
2970 }
2971 EXPORT_SYMBOL_GPL(enetc_ioctl);
2972
2973 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
2974 {
2975         struct pci_dev *pdev = priv->si->pdev;
2976         int num_stack_tx_queues;
2977         int first_xdp_tx_ring;
2978         int i, n, err, nvec;
2979         int v_tx_rings;
2980
2981         nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
2982         /* allocate MSIX for both messaging and Rx/Tx interrupts */
2983         n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
2984
2985         if (n < 0)
2986                 return n;
2987
2988         if (n != nvec)
2989                 return -EPERM;
2990
2991         /* # of tx rings per int vector */
2992         v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
2993
2994         for (i = 0; i < priv->bdr_int_num; i++) {
2995                 struct enetc_int_vector *v;
2996                 struct enetc_bdr *bdr;
2997                 int j;
2998
2999                 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
3000                 if (!v) {
3001                         err = -ENOMEM;
3002                         goto fail;
3003                 }
3004
3005                 priv->int_vector[i] = v;
3006
3007                 bdr = &v->rx_ring;
3008                 bdr->index = i;
3009                 bdr->ndev = priv->ndev;
3010                 bdr->dev = priv->dev;
3011                 bdr->bd_count = priv->rx_bd_count;
3012                 bdr->buffer_offset = ENETC_RXB_PAD;
3013                 priv->rx_ring[i] = bdr;
3014
3015                 err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
3016                 if (err) {
3017                         kfree(v);
3018                         goto fail;
3019                 }
3020
3021                 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
3022                                                  MEM_TYPE_PAGE_SHARED, NULL);
3023                 if (err) {
3024                         xdp_rxq_info_unreg(&bdr->xdp.rxq);
3025                         kfree(v);
3026                         goto fail;
3027                 }
3028
3029                 /* init defaults for adaptive IC */
3030                 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3031                         v->rx_ictt = 0x1;
3032                         v->rx_dim_en = true;
3033                 }
3034                 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3035                 netif_napi_add(priv->ndev, &v->napi, enetc_poll);
3036                 v->count_tx_rings = v_tx_rings;
3037
3038                 for (j = 0; j < v_tx_rings; j++) {
3039                         int idx;
3040
3041                         /* default tx ring mapping policy */
3042                         idx = priv->bdr_int_num * j + i;
3043                         __set_bit(idx, &v->tx_rings_map);
3044                         bdr = &v->tx_ring[j];
3045                         bdr->index = idx;
3046                         bdr->ndev = priv->ndev;
3047                         bdr->dev = priv->dev;
3048                         bdr->bd_count = priv->tx_bd_count;
3049                         priv->tx_ring[idx] = bdr;
3050                 }
3051         }
3052
3053         num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3054
3055         err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3056         if (err)
3057                 goto fail;
3058
3059         err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
3060         if (err)
3061                 goto fail;
3062
3063         priv->min_num_stack_tx_queues = num_possible_cpus();
3064         first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
3065         priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
3066
3067         return 0;
3068
3069 fail:
3070         while (i--) {
3071                 struct enetc_int_vector *v = priv->int_vector[i];
3072                 struct enetc_bdr *rx_ring = &v->rx_ring;
3073
3074                 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3075                 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3076                 netif_napi_del(&v->napi);
3077                 cancel_work_sync(&v->rx_dim.work);
3078                 kfree(v);
3079         }
3080
3081         pci_free_irq_vectors(pdev);
3082
3083         return err;
3084 }
3085 EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3086
3087 void enetc_free_msix(struct enetc_ndev_priv *priv)
3088 {
3089         int i;
3090
3091         for (i = 0; i < priv->bdr_int_num; i++) {
3092                 struct enetc_int_vector *v = priv->int_vector[i];
3093                 struct enetc_bdr *rx_ring = &v->rx_ring;
3094
3095                 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3096                 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3097                 netif_napi_del(&v->napi);
3098                 cancel_work_sync(&v->rx_dim.work);
3099         }
3100
3101         for (i = 0; i < priv->num_rx_rings; i++)
3102                 priv->rx_ring[i] = NULL;
3103
3104         for (i = 0; i < priv->num_tx_rings; i++)
3105                 priv->tx_ring[i] = NULL;
3106
3107         for (i = 0; i < priv->bdr_int_num; i++) {
3108                 kfree(priv->int_vector[i]);
3109                 priv->int_vector[i] = NULL;
3110         }
3111
3112         /* disable all MSIX for this device */
3113         pci_free_irq_vectors(priv->si->pdev);
3114 }
3115 EXPORT_SYMBOL_GPL(enetc_free_msix);
3116
3117 static void enetc_kfree_si(struct enetc_si *si)
3118 {
3119         char *p = (char *)si - si->pad;
3120
3121         kfree(p);
3122 }
3123
3124 static void enetc_detect_errata(struct enetc_si *si)
3125 {
3126         if (si->pdev->revision == ENETC_REV1)
3127                 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
3128 }
3129
3130 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3131 {
3132         struct enetc_si *si, *p;
3133         struct enetc_hw *hw;
3134         size_t alloc_size;
3135         int err, len;
3136
3137         pcie_flr(pdev);
3138         err = pci_enable_device_mem(pdev);
3139         if (err)
3140                 return dev_err_probe(&pdev->dev, err, "device enable failed\n");
3141
3142         /* set up for high or low dma */
3143         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3144         if (err) {
3145                 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3146                 goto err_dma;
3147         }
3148
3149         err = pci_request_mem_regions(pdev, name);
3150         if (err) {
3151                 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3152                 goto err_pci_mem_reg;
3153         }
3154
3155         pci_set_master(pdev);
3156
3157         alloc_size = sizeof(struct enetc_si);
3158         if (sizeof_priv) {
3159                 /* align priv to 32B */
3160                 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3161                 alloc_size += sizeof_priv;
3162         }
3163         /* force 32B alignment for enetc_si */
3164         alloc_size += ENETC_SI_ALIGN - 1;
3165
3166         p = kzalloc(alloc_size, GFP_KERNEL);
3167         if (!p) {
3168                 err = -ENOMEM;
3169                 goto err_alloc_si;
3170         }
3171
3172         si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3173         si->pad = (char *)si - (char *)p;
3174
3175         pci_set_drvdata(pdev, si);
3176         si->pdev = pdev;
3177         hw = &si->hw;
3178
3179         len = pci_resource_len(pdev, ENETC_BAR_REGS);
3180         hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3181         if (!hw->reg) {
3182                 err = -ENXIO;
3183                 dev_err(&pdev->dev, "ioremap() failed\n");
3184                 goto err_ioremap;
3185         }
3186         if (len > ENETC_PORT_BASE)
3187                 hw->port = hw->reg + ENETC_PORT_BASE;
3188         if (len > ENETC_GLOBAL_BASE)
3189                 hw->global = hw->reg + ENETC_GLOBAL_BASE;
3190
3191         enetc_detect_errata(si);
3192
3193         return 0;
3194
3195 err_ioremap:
3196         enetc_kfree_si(si);
3197 err_alloc_si:
3198         pci_release_mem_regions(pdev);
3199 err_pci_mem_reg:
3200 err_dma:
3201         pci_disable_device(pdev);
3202
3203         return err;
3204 }
3205 EXPORT_SYMBOL_GPL(enetc_pci_probe);
3206
3207 void enetc_pci_remove(struct pci_dev *pdev)
3208 {
3209         struct enetc_si *si = pci_get_drvdata(pdev);
3210         struct enetc_hw *hw = &si->hw;
3211
3212         iounmap(hw->reg);
3213         enetc_kfree_si(si);
3214         pci_release_mem_regions(pdev);
3215         pci_disable_device(pdev);
3216 }
3217 EXPORT_SYMBOL_GPL(enetc_pci_remove);
3218
3219 MODULE_DESCRIPTION("NXP ENETC Ethernet driver");
3220 MODULE_LICENSE("Dual BSD/GPL");