drivers/net/ethernet/qualcomm/emac/emac-mac.c

   1 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License version 2 and
   5  * only version 2 as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12
  13 /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support
  14  */
  15
  16 #include <linux/tcp.h>
  17 #include <linux/ip.h>
  18 #include <linux/ipv6.h>
  19 #include <linux/crc32.h>
  20 #include <linux/if_vlan.h>
  21 #include <linux/jiffies.h>
  22 #include <linux/phy.h>
  23 #include <linux/of.h>
  24 #include <net/ip6_checksum.h>
  25 #include "emac.h"
  26 #include "emac-sgmii.h"
  27
  28 /* EMAC_MAC_CTRL */
  29 #define SINGLE_PAUSE_MODE               0x10000000
  30 #define DEBUG_MODE                      0x08000000
  31 #define BROAD_EN                        0x04000000
  32 #define MULTI_ALL                       0x02000000
  33 #define RX_CHKSUM_EN                    0x01000000
  34 #define HUGE                            0x00800000
  35 #define SPEED(x)                        (((x) & 0x3) << 20)
  36 #define SPEED_MASK                      SPEED(0x3)
  37 #define SIMR                            0x00080000
  38 #define TPAUSE                          0x00010000
  39 #define PROM_MODE                       0x00008000
  40 #define VLAN_STRIP                      0x00004000
  41 #define PRLEN_BMSK                      0x00003c00
  42 #define PRLEN_SHFT                      10
  43 #define HUGEN                           0x00000200
  44 #define FLCHK                           0x00000100
  45 #define PCRCE                           0x00000080
  46 #define CRCE                            0x00000040
  47 #define FULLD                           0x00000020
  48 #define MAC_LP_EN                       0x00000010
  49 #define RXFC                            0x00000008
  50 #define TXFC                            0x00000004
  51 #define RXEN                            0x00000002
  52 #define TXEN                            0x00000001
  53
  54 /* EMAC_DESC_CTRL_3 */
  55 #define RFD_RING_SIZE_BMSK                                       0xfff
  56
  57 /* EMAC_DESC_CTRL_4 */
  58 #define RX_BUFFER_SIZE_BMSK                                     0xffff
  59
  60 /* EMAC_DESC_CTRL_6 */
  61 #define RRD_RING_SIZE_BMSK                                       0xfff
  62
  63 /* EMAC_DESC_CTRL_9 */
  64 #define TPD_RING_SIZE_BMSK                                      0xffff
  65
  66 /* EMAC_TXQ_CTRL_0 */
  67 #define NUM_TXF_BURST_PREF_BMSK                             0xffff0000
  68 #define NUM_TXF_BURST_PREF_SHFT                                     16
  69 #define LS_8023_SP                                                0x80
  70 #define TXQ_MODE                                                  0x40
  71 #define TXQ_EN                                                    0x20
  72 #define IP_OP_SP                                                  0x10
  73 #define NUM_TPD_BURST_PREF_BMSK                                    0xf
  74 #define NUM_TPD_BURST_PREF_SHFT                                      0
  75
  76 /* EMAC_TXQ_CTRL_1 */
  77 #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK                        0x7ff
  78
  79 /* EMAC_TXQ_CTRL_2 */
  80 #define TXF_HWM_BMSK                                         0xfff0000
  81 #define TXF_LWM_BMSK                                             0xfff
  82
  83 /* EMAC_RXQ_CTRL_0 */
  84 #define RXQ_EN                                                 BIT(31)
  85 #define CUT_THRU_EN                                            BIT(30)
  86 #define RSS_HASH_EN                                            BIT(29)
  87 #define NUM_RFD_BURST_PREF_BMSK                              0x3f00000
  88 #define NUM_RFD_BURST_PREF_SHFT                                     20
  89 #define IDT_TABLE_SIZE_BMSK                                    0x1ff00
  90 #define IDT_TABLE_SIZE_SHFT                                          8
  91 #define SP_IPV6                                                   0x80
  92
  93 /* EMAC_RXQ_CTRL_1 */
  94 #define JUMBO_1KAH_BMSK                                         0xf000
  95 #define JUMBO_1KAH_SHFT                                             12
  96 #define RFD_PREF_LOW_TH                                           0x10
  97 #define RFD_PREF_LOW_THRESHOLD_BMSK                              0xfc0
  98 #define RFD_PREF_LOW_THRESHOLD_SHFT                                  6
  99 #define RFD_PREF_UP_TH                                            0x10
 100 #define RFD_PREF_UP_THRESHOLD_BMSK                                0x3f
 101 #define RFD_PREF_UP_THRESHOLD_SHFT                                   0
 102
 103 /* EMAC_RXQ_CTRL_2 */
 104 #define RXF_DOF_THRESFHOLD                                       0x1a0
 105 #define RXF_DOF_THRESHOLD_BMSK                               0xfff0000
 106 #define RXF_DOF_THRESHOLD_SHFT                                      16
 107 #define RXF_UOF_THRESFHOLD                                        0xbe
 108 #define RXF_UOF_THRESHOLD_BMSK                                   0xfff
 109 #define RXF_UOF_THRESHOLD_SHFT                                       0
 110
 111 /* EMAC_RXQ_CTRL_3 */
 112 #define RXD_TIMER_BMSK                                      0xffff0000
 113 #define RXD_THRESHOLD_BMSK                                       0xfff
 114 #define RXD_THRESHOLD_SHFT                                           0
 115
 116 /* EMAC_DMA_CTRL */
 117 #define DMAW_DLY_CNT_BMSK                                      0xf0000
 118 #define DMAW_DLY_CNT_SHFT                                           16
 119 #define DMAR_DLY_CNT_BMSK                                       0xf800
 120 #define DMAR_DLY_CNT_SHFT                                           11
 121 #define DMAR_REQ_PRI                                             0x400
 122 #define REGWRBLEN_BMSK                                           0x380
 123 #define REGWRBLEN_SHFT                                               7
 124 #define REGRDBLEN_BMSK                                            0x70
 125 #define REGRDBLEN_SHFT                                               4
 126 #define OUT_ORDER_MODE                                             0x4
 127 #define ENH_ORDER_MODE                                             0x2
 128 #define IN_ORDER_MODE                                              0x1
 129
 130 /* EMAC_MAILBOX_13 */
 131 #define RFD3_PROC_IDX_BMSK                                   0xfff0000
 132 #define RFD3_PROC_IDX_SHFT                                          16
 133 #define RFD3_PROD_IDX_BMSK                                       0xfff
 134 #define RFD3_PROD_IDX_SHFT                                           0
 135
 136 /* EMAC_MAILBOX_2 */
 137 #define NTPD_CONS_IDX_BMSK                                  0xffff0000
 138 #define NTPD_CONS_IDX_SHFT                                          16
 139
 140 /* EMAC_MAILBOX_3 */
 141 #define RFD0_CONS_IDX_BMSK                                       0xfff
 142 #define RFD0_CONS_IDX_SHFT                                           0
 143
 144 /* EMAC_MAILBOX_11 */
 145 #define H3TPD_PROD_IDX_BMSK                                 0xffff0000
 146 #define H3TPD_PROD_IDX_SHFT                                         16
 147
 148 /* EMAC_AXI_MAST_CTRL */
 149 #define DATA_BYTE_SWAP                                             0x8
 150 #define MAX_BOUND                                                  0x2
 151 #define MAX_BTYPE                                                  0x1
 152
 153 /* EMAC_MAILBOX_12 */
 154 #define H3TPD_CONS_IDX_BMSK                                 0xffff0000
 155 #define H3TPD_CONS_IDX_SHFT                                         16
 156
 157 /* EMAC_MAILBOX_9 */
 158 #define H2TPD_PROD_IDX_BMSK                                     0xffff
 159 #define H2TPD_PROD_IDX_SHFT                                          0
 160
 161 /* EMAC_MAILBOX_10 */
 162 #define H1TPD_CONS_IDX_BMSK                                 0xffff0000
 163 #define H1TPD_CONS_IDX_SHFT                                         16
 164 #define H2TPD_CONS_IDX_BMSK                                     0xffff
 165 #define H2TPD_CONS_IDX_SHFT                                          0
 166
 167 /* EMAC_ATHR_HEADER_CTRL */
 168 #define HEADER_CNT_EN                                              0x2
 169 #define HEADER_ENABLE                                              0x1
 170
 171 /* EMAC_MAILBOX_0 */
 172 #define RFD0_PROC_IDX_BMSK                                   0xfff0000
 173 #define RFD0_PROC_IDX_SHFT                                          16
 174 #define RFD0_PROD_IDX_BMSK                                       0xfff
 175 #define RFD0_PROD_IDX_SHFT                                           0
 176
 177 /* EMAC_MAILBOX_5 */
 178 #define RFD1_PROC_IDX_BMSK                                   0xfff0000
 179 #define RFD1_PROC_IDX_SHFT                                          16
 180 #define RFD1_PROD_IDX_BMSK                                       0xfff
 181 #define RFD1_PROD_IDX_SHFT                                           0
 182
 183 /* EMAC_MISC_CTRL */
 184 #define RX_UNCPL_INT_EN                                            0x1
 185
 186 /* EMAC_MAILBOX_7 */
 187 #define RFD2_CONS_IDX_BMSK                                   0xfff0000
 188 #define RFD2_CONS_IDX_SHFT                                          16
 189 #define RFD1_CONS_IDX_BMSK                                       0xfff
 190 #define RFD1_CONS_IDX_SHFT                                           0
 191
 192 /* EMAC_MAILBOX_8 */
 193 #define RFD3_CONS_IDX_BMSK                                       0xfff
 194 #define RFD3_CONS_IDX_SHFT                                           0
 195
 196 /* EMAC_MAILBOX_15 */
 197 #define NTPD_PROD_IDX_BMSK                                      0xffff
 198 #define NTPD_PROD_IDX_SHFT                                           0
 199
 200 /* EMAC_MAILBOX_16 */
 201 #define H1TPD_PROD_IDX_BMSK                                     0xffff
 202 #define H1TPD_PROD_IDX_SHFT                                          0
 203
 204 #define RXQ0_RSS_HSTYP_IPV6_TCP_EN                                0x20
 205 #define RXQ0_RSS_HSTYP_IPV6_EN                                    0x10
 206 #define RXQ0_RSS_HSTYP_IPV4_TCP_EN                                 0x8
 207 #define RXQ0_RSS_HSTYP_IPV4_EN                                     0x4
 208
 209 /* EMAC_EMAC_WRAPPER_TX_TS_INX */
 210 #define EMAC_WRAPPER_TX_TS_EMPTY                               BIT(31)
 211 #define EMAC_WRAPPER_TX_TS_INX_BMSK                             0xffff
 212
 213 struct emac_skb_cb {
 214         u32           tpd_idx;
 215         unsigned long jiffies;
 216 };
 217
 218 #define EMAC_SKB_CB(skb)        ((struct emac_skb_cb *)(skb)->cb)
 219 #define EMAC_RSS_IDT_SIZE       256
 220 #define JUMBO_1KAH              0x4
 221 #define RXD_TH                  0x100
 222 #define EMAC_TPD_LAST_FRAGMENT  0x80000000
 223 #define EMAC_TPD_TSTAMP_SAVE    0x80000000
 224
 225 /* EMAC Errors in emac_rrd.word[3] */
 226 #define EMAC_RRD_L4F            BIT(14)
 227 #define EMAC_RRD_IPF            BIT(15)
 228 #define EMAC_RRD_CRC            BIT(21)
 229 #define EMAC_RRD_FAE            BIT(22)
 230 #define EMAC_RRD_TRN            BIT(23)
 231 #define EMAC_RRD_RNT            BIT(24)
 232 #define EMAC_RRD_INC            BIT(25)
 233 #define EMAC_RRD_FOV            BIT(29)
 234 #define EMAC_RRD_LEN            BIT(30)
 235
 236 /* Error bits that will result in a received frame being discarded */
 237 #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
 238                         EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
 239                         EMAC_RRD_FOV | EMAC_RRD_LEN)
 240 #define EMAC_RRD_STATS_DW_IDX 3
 241
 242 #define EMAC_RRD(RXQ, SIZE, IDX)        ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
 243 #define EMAC_RFD(RXQ, SIZE, IDX)        ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
 244 #define EMAC_TPD(TXQ, SIZE, IDX)        ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
 245
 246 #define GET_RFD_BUFFER(RXQ, IDX)        (&((RXQ)->rfd.rfbuff[(IDX)]))
 247 #define GET_TPD_BUFFER(RTQ, IDX)        (&((RTQ)->tpd.tpbuff[(IDX)]))
 248
 249 #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD       8
 250
 251 #define ISR_RX_PKT      (\
 252         RX_PKT_INT0     |\
 253         RX_PKT_INT1     |\
 254         RX_PKT_INT2     |\
 255         RX_PKT_INT3)
 256
 257 void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)
 258 {
 259         u32 crc32, bit, reg, mta;
 260
 261         /* Calculate the CRC of the MAC address */
 262         crc32 = ether_crc(ETH_ALEN, addr);
 263
 264         /* The HASH Table is an array of 2 32-bit registers. It is
 265          * treated like an array of 64 bits (BitArray[hash_value]).
 266          * Use the upper 6 bits of the above CRC as the hash value.
 267          */
 268         reg = (crc32 >> 31) & 0x1;
 269         bit = (crc32 >> 26) & 0x1F;
 270
 271         mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
 272         mta |= BIT(bit);
 273         writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
 274 }
 275
 276 void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)
 277 {
 278         writel(0, adpt->base + EMAC_HASH_TAB_REG0);
 279         writel(0, adpt->base + EMAC_HASH_TAB_REG1);
 280 }
 281
 282 /* definitions for RSS */
 283 #define EMAC_RSS_KEY(_i, _type) \
 284                 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
 285 #define EMAC_RSS_TBL(_i, _type) \
 286                 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
 287
 288 /* Config MAC modes */
 289 void emac_mac_mode_config(struct emac_adapter *adpt)
 290 {
 291         struct net_device *netdev = adpt->netdev;
 292         u32 mac;
 293
 294         mac = readl(adpt->base + EMAC_MAC_CTRL);
 295         mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);
 296
 297         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
 298                 mac |= VLAN_STRIP;
 299
 300         if (netdev->flags & IFF_PROMISC)
 301                 mac |= PROM_MODE;
 302
 303         if (netdev->flags & IFF_ALLMULTI)
 304                 mac |= MULTI_ALL;
 305
 306         writel(mac, adpt->base + EMAC_MAC_CTRL);
 307 }
 308
 309 /* Config descriptor rings */
 310 static void emac_mac_dma_rings_config(struct emac_adapter *adpt)
 311 {
 312         static const unsigned short tpd_q_offset[] = {
 313                 EMAC_DESC_CTRL_8,        EMAC_H1TPD_BASE_ADDR_LO,
 314                 EMAC_H2TPD_BASE_ADDR_LO, EMAC_H3TPD_BASE_ADDR_LO};
 315         static const unsigned short rfd_q_offset[] = {
 316                 EMAC_DESC_CTRL_2,        EMAC_DESC_CTRL_10,
 317                 EMAC_DESC_CTRL_12,       EMAC_DESC_CTRL_13};
 318         static const unsigned short rrd_q_offset[] = {
 319                 EMAC_DESC_CTRL_5,        EMAC_DESC_CTRL_14,
 320                 EMAC_DESC_CTRL_15,       EMAC_DESC_CTRL_16};
 321
 322         /* TPD (Transmit Packet Descriptor) */
 323         writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),
 324                adpt->base + EMAC_DESC_CTRL_1);
 325
 326         writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),
 327                adpt->base + tpd_q_offset[0]);
 328
 329         writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,
 330                adpt->base + EMAC_DESC_CTRL_9);
 331
 332         /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */
 333         writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),
 334                adpt->base + EMAC_DESC_CTRL_0);
 335
 336         writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),
 337                adpt->base + rfd_q_offset[0]);
 338         writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),
 339                adpt->base + rrd_q_offset[0]);
 340
 341         writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,
 342                adpt->base + EMAC_DESC_CTRL_3);
 343         writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,
 344                adpt->base + EMAC_DESC_CTRL_6);
 345
 346         writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,
 347                adpt->base + EMAC_DESC_CTRL_4);
 348
 349         writel(0, adpt->base + EMAC_DESC_CTRL_11);
 350
 351         /* Load all of the base addresses above and ensure that triggering HW to
 352          * read ring pointers is flushed
 353          */
 354         writel(1, adpt->base + EMAC_INTER_SRAM_PART9);
 355 }
 356
 357 /* Config transmit parameters */
 358 static void emac_mac_tx_config(struct emac_adapter *adpt)
 359 {
 360         u32 val;
 361
 362         writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &
 363                JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);
 364
 365         val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &
 366                NUM_TPD_BURST_PREF_BMSK;
 367
 368         val |= TXQ_MODE | LS_8023_SP;
 369         val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &
 370                 NUM_TXF_BURST_PREF_BMSK;
 371
 372         writel(val, adpt->base + EMAC_TXQ_CTRL_0);
 373         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,
 374                           (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);
 375 }
 376
 377 /* Config receive parameters */
 378 static void emac_mac_rx_config(struct emac_adapter *adpt)
 379 {
 380         u32 val;
 381
 382         val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &
 383                NUM_RFD_BURST_PREF_BMSK;
 384         val |= (SP_IPV6 | CUT_THRU_EN);
 385
 386         writel(val, adpt->base + EMAC_RXQ_CTRL_0);
 387
 388         val = readl(adpt->base + EMAC_RXQ_CTRL_1);
 389         val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |
 390                  RFD_PREF_UP_THRESHOLD_BMSK);
 391         val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |
 392                 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |
 393                 (RFD_PREF_UP_TH  << RFD_PREF_UP_THRESHOLD_SHFT);
 394         writel(val, adpt->base + EMAC_RXQ_CTRL_1);
 395
 396         val = readl(adpt->base + EMAC_RXQ_CTRL_2);
 397         val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);
 398         val |= (RXF_DOF_THRESFHOLD  << RXF_DOF_THRESHOLD_SHFT) |
 399                 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);
 400         writel(val, adpt->base + EMAC_RXQ_CTRL_2);
 401
 402         val = readl(adpt->base + EMAC_RXQ_CTRL_3);
 403         val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);
 404         val |= RXD_TH << RXD_THRESHOLD_SHFT;
 405         writel(val, adpt->base + EMAC_RXQ_CTRL_3);
 406 }
 407
 408 /* Config dma */
 409 static void emac_mac_dma_config(struct emac_adapter *adpt)
 410 {
 411         u32 dma_ctrl = DMAR_REQ_PRI;
 412
 413         switch (adpt->dma_order) {
 414         case emac_dma_ord_in:
 415                 dma_ctrl |= IN_ORDER_MODE;
 416                 break;
 417         case emac_dma_ord_enh:
 418                 dma_ctrl |= ENH_ORDER_MODE;
 419                 break;
 420         case emac_dma_ord_out:
 421                 dma_ctrl |= OUT_ORDER_MODE;
 422                 break;
 423         default:
 424                 break;
 425         }
 426
 427         dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &
 428                                                 REGRDBLEN_BMSK;
 429         dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &
 430                                                 REGWRBLEN_BMSK;
 431         dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &
 432                                                 DMAR_DLY_CNT_BMSK;
 433         dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &
 434                                                 DMAW_DLY_CNT_BMSK;
 435
 436         /* config DMA and ensure that configuration is flushed to HW */
 437         writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);
 438 }
 439
 440 /* set MAC address */
 441 static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr)
 442 {
 443         u32 sta;
 444
 445         /* for example: 00-A0-C6-11-22-33
 446          * 0<-->C6112233, 1<-->00A0.
 447          */
 448
 449         /* low 32bit word */
 450         sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |
 451               (((u32)addr[4]) << 8)  | (((u32)addr[5]));
 452         writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);
 453
 454         /* hight 32bit word */
 455         sta = (((u32)addr[0]) << 8) | (u32)addr[1];
 456         writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);
 457 }
 458
 459 static void emac_mac_config(struct emac_adapter *adpt)
 460 {
 461         struct net_device *netdev = adpt->netdev;
 462         unsigned int max_frame;
 463         u32 val;
 464
 465         emac_set_mac_address(adpt, netdev->dev_addr);
 466
 467         max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
 468         adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
 469                 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
 470
 471         emac_mac_dma_rings_config(adpt);
 472
 473         writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,
 474                adpt->base + EMAC_MAX_FRAM_LEN_CTRL);
 475
 476         emac_mac_tx_config(adpt);
 477         emac_mac_rx_config(adpt);
 478         emac_mac_dma_config(adpt);
 479
 480         val = readl(adpt->base + EMAC_AXI_MAST_CTRL);
 481         val &= ~(DATA_BYTE_SWAP | MAX_BOUND);
 482         val |= MAX_BTYPE;
 483         writel(val, adpt->base + EMAC_AXI_MAST_CTRL);
 484         writel(0, adpt->base + EMAC_CLK_GATE_CTRL);
 485         writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);
 486 }
 487
 488 void emac_mac_reset(struct emac_adapter *adpt)
 489 {
 490         emac_mac_stop(adpt);
 491
 492         emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);
 493         usleep_range(100, 150); /* reset may take up to 100usec */
 494
 495         /* interrupt clear-on-read */
 496         emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);
 497 }
 498
 499 static void emac_mac_start(struct emac_adapter *adpt)
 500 {
 501         struct phy_device *phydev = adpt->phydev;
 502         u32 mac, csr1;
 503
 504         /* enable tx queue */
 505         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);
 506
 507         /* enable rx queue */
 508         emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);
 509
 510         /* enable mac control */
 511         mac = readl(adpt->base + EMAC_MAC_CTRL);
 512         csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
 513
 514         mac |= TXEN | RXEN;     /* enable RX/TX */
 515
 516         /* Configure MAC flow control. If set to automatic, then match
 517          * whatever the PHY does. Otherwise, enable or disable it, depending
 518          * on what the user configured via ethtool.
 519          */
 520         mac &= ~(RXFC | TXFC);
 521
 522         if (adpt->automatic) {
 523                 /* If it's set to automatic, then update our local values */
 524                 adpt->rx_flow_control = phydev->pause;
 525                 adpt->tx_flow_control = phydev->pause != phydev->asym_pause;
 526         }
 527         mac |= adpt->rx_flow_control ? RXFC : 0;
 528         mac |= adpt->tx_flow_control ? TXFC : 0;
 529
 530         /* setup link speed */
 531         mac &= ~SPEED_MASK;
 532         if (phydev->speed == SPEED_1000) {
 533                 mac |= SPEED(2);
 534                 csr1 |= FREQ_MODE;
 535         } else {
 536                 mac |= SPEED(1);
 537                 csr1 &= ~FREQ_MODE;
 538         }
 539
 540         if (phydev->duplex == DUPLEX_FULL)
 541                 mac |= FULLD;
 542         else
 543                 mac &= ~FULLD;
 544
 545         /* other parameters */
 546         mac |= (CRCE | PCRCE);
 547         mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);
 548         mac |= BROAD_EN;
 549         mac |= FLCHK;
 550         mac &= ~RX_CHKSUM_EN;
 551         mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |
 552                  DEBUG_MODE | SINGLE_PAUSE_MODE);
 553
 554         /* Enable single-pause-frame mode if requested.
 555          *
 556          * If enabled, the EMAC will send a single pause frame when the RX
 557          * queue is full.  This normally leads to packet loss because
 558          * the pause frame disables the remote MAC only for 33ms (the quanta),
 559          * and then the remote MAC continues sending packets even though
 560          * the RX queue is still full.
 561          *
 562          * If disabled, the EMAC sends a pause frame every 31ms until the RX
 563          * queue is no longer full.  Normally, this is the preferred
 564          * method of operation.  However, when the system is hung (e.g.
 565          * cores are halted), the EMAC interrupt handler is never called
 566          * and so the RX queue fills up quickly and stays full.  The resuling
 567          * non-stop "flood" of pause frames sometimes has the effect of
 568          * disabling nearby switches.  In some cases, other nearby switches
 569          * are also affected, shutting down the entire network.
 570          *
 571          * The user can enable or disable single-pause-frame mode
 572          * via ethtool.
 573          */
 574         mac |= adpt->single_pause_mode ? SINGLE_PAUSE_MODE : 0;
 575
 576         writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
 577
 578         writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);
 579
 580         /* enable interrupt read clear, low power sleep mode and
 581          * the irq moderators
 582          */
 583
 584         writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);
 585         writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |
 586                         IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);
 587
 588         emac_mac_mode_config(adpt);
 589
 590         emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,
 591                           (HEADER_ENABLE | HEADER_CNT_EN), 0);
 592 }
 593
 594 void emac_mac_stop(struct emac_adapter *adpt)
 595 {
 596         emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);
 597         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);
 598         emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);
 599         usleep_range(1000, 1050); /* stopping mac may take upto 1msec */
 600 }
 601
 602 /* Free all descriptors of given transmit queue */
 603 static void emac_tx_q_descs_free(struct emac_adapter *adpt)
 604 {
 605         struct emac_tx_queue *tx_q = &adpt->tx_q;
 606         unsigned int i;
 607         size_t size;
 608
 609         /* ring already cleared, nothing to do */
 610         if (!tx_q->tpd.tpbuff)
 611                 return;
 612
 613         for (i = 0; i < tx_q->tpd.count; i++) {
 614                 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);
 615
 616                 if (tpbuf->dma_addr) {
 617                         dma_unmap_single(adpt->netdev->dev.parent,
 618                                          tpbuf->dma_addr, tpbuf->length,
 619                                          DMA_TO_DEVICE);
 620                         tpbuf->dma_addr = 0;
 621                 }
 622                 if (tpbuf->skb) {
 623                         dev_kfree_skb_any(tpbuf->skb);
 624                         tpbuf->skb = NULL;
 625                 }
 626         }
 627
 628         size = sizeof(struct emac_buffer) * tx_q->tpd.count;
 629         memset(tx_q->tpd.tpbuff, 0, size);
 630
 631         /* clear the descriptor ring */
 632         memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);
 633
 634         tx_q->tpd.consume_idx = 0;
 635         tx_q->tpd.produce_idx = 0;
 636 }
 637
 638 /* Free all descriptors of given receive queue */
 639 static void emac_rx_q_free_descs(struct emac_adapter *adpt)
 640 {
 641         struct device *dev = adpt->netdev->dev.parent;
 642         struct emac_rx_queue *rx_q = &adpt->rx_q;
 643         unsigned int i;
 644         size_t size;
 645
 646         /* ring already cleared, nothing to do */
 647         if (!rx_q->rfd.rfbuff)
 648                 return;
 649
 650         for (i = 0; i < rx_q->rfd.count; i++) {
 651                 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);
 652
 653                 if (rfbuf->dma_addr) {
 654                         dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,
 655                                          DMA_FROM_DEVICE);
 656                         rfbuf->dma_addr = 0;
 657                 }
 658                 if (rfbuf->skb) {
 659                         dev_kfree_skb(rfbuf->skb);
 660                         rfbuf->skb = NULL;
 661                 }
 662         }
 663
 664         size =  sizeof(struct emac_buffer) * rx_q->rfd.count;
 665         memset(rx_q->rfd.rfbuff, 0, size);
 666
 667         /* clear the descriptor rings */
 668         memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);
 669         rx_q->rrd.produce_idx = 0;
 670         rx_q->rrd.consume_idx = 0;
 671
 672         memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);
 673         rx_q->rfd.produce_idx = 0;
 674         rx_q->rfd.consume_idx = 0;
 675 }
 676
 677 /* Free all buffers associated with given transmit queue */
 678 static void emac_tx_q_bufs_free(struct emac_adapter *adpt)
 679 {
 680         struct emac_tx_queue *tx_q = &adpt->tx_q;
 681
 682         emac_tx_q_descs_free(adpt);
 683
 684         kfree(tx_q->tpd.tpbuff);
 685         tx_q->tpd.tpbuff = NULL;
 686         tx_q->tpd.v_addr = NULL;
 687         tx_q->tpd.dma_addr = 0;
 688         tx_q->tpd.size = 0;
 689 }
 690
 691 /* Allocate TX descriptor ring for the given transmit queue */
 692 static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
 693                                 struct emac_tx_queue *tx_q)
 694 {
 695         struct emac_ring_header *ring_header = &adpt->ring_header;
 696         size_t size;
 697
 698         size = sizeof(struct emac_buffer) * tx_q->tpd.count;
 699         tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
 700         if (!tx_q->tpd.tpbuff)
 701                 return -ENOMEM;
 702
 703         tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);
 704         tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;
 705         tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;
 706         ring_header->used += ALIGN(tx_q->tpd.size, 8);
 707         tx_q->tpd.produce_idx = 0;
 708         tx_q->tpd.consume_idx = 0;
 709
 710         return 0;
 711 }
 712
 713 /* Free all buffers associated with given transmit queue */
 714 static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
 715 {
 716         struct emac_rx_queue *rx_q = &adpt->rx_q;
 717
 718         emac_rx_q_free_descs(adpt);
 719
 720         kfree(rx_q->rfd.rfbuff);
 721         rx_q->rfd.rfbuff   = NULL;
 722
 723         rx_q->rfd.v_addr   = NULL;
 724         rx_q->rfd.dma_addr = 0;
 725         rx_q->rfd.size     = 0;
 726
 727         rx_q->rrd.v_addr   = NULL;
 728         rx_q->rrd.dma_addr = 0;
 729         rx_q->rrd.size     = 0;
 730 }
 731
 732 /* Allocate RX descriptor rings for the given receive queue */
 733 static int emac_rx_descs_alloc(struct emac_adapter *adpt)
 734 {
 735         struct emac_ring_header *ring_header = &adpt->ring_header;
 736         struct emac_rx_queue *rx_q = &adpt->rx_q;
 737         size_t size;
 738
 739         size = sizeof(struct emac_buffer) * rx_q->rfd.count;
 740         rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
 741         if (!rx_q->rfd.rfbuff)
 742                 return -ENOMEM;
 743
 744         rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);
 745         rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);
 746
 747         rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;
 748         rx_q->rrd.v_addr   = ring_header->v_addr + ring_header->used;
 749         ring_header->used += ALIGN(rx_q->rrd.size, 8);
 750
 751         rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;
 752         rx_q->rfd.v_addr   = ring_header->v_addr + ring_header->used;
 753         ring_header->used += ALIGN(rx_q->rfd.size, 8);
 754
 755         rx_q->rrd.produce_idx = 0;
 756         rx_q->rrd.consume_idx = 0;
 757
 758         rx_q->rfd.produce_idx = 0;
 759         rx_q->rfd.consume_idx = 0;
 760
 761         return 0;
 762 }
 763
 764 /* Allocate all TX and RX descriptor rings */
 765 int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)
 766 {
 767         struct emac_ring_header *ring_header = &adpt->ring_header;
 768         struct device *dev = adpt->netdev->dev.parent;
 769         unsigned int num_tx_descs = adpt->tx_desc_cnt;
 770         unsigned int num_rx_descs = adpt->rx_desc_cnt;
 771         int ret;
 772
 773         adpt->tx_q.tpd.count = adpt->tx_desc_cnt;
 774
 775         adpt->rx_q.rrd.count = adpt->rx_desc_cnt;
 776         adpt->rx_q.rfd.count = adpt->rx_desc_cnt;
 777
 778         /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
 779          * hence the additional padding bytes are allocated.
 780          */
 781         ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +
 782                             num_rx_descs * (adpt->rfd_size * 4) +
 783                             num_rx_descs * (adpt->rrd_size * 4) +
 784                             8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */
 785
 786         ring_header->used = 0;
 787         ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size,
 788                                                  &ring_header->dma_addr,
 789                                                  GFP_KERNEL);
 790         if (!ring_header->v_addr)
 791                 return -ENOMEM;
 792
 793         ring_header->used = ALIGN(ring_header->dma_addr, 8) -
 794                                                         ring_header->dma_addr;
 795
 796         ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);
 797         if (ret) {
 798                 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");
 799                 goto err_alloc_tx;
 800         }
 801
 802         ret = emac_rx_descs_alloc(adpt);
 803         if (ret) {
 804                 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");
 805                 goto err_alloc_rx;
 806         }
 807
 808         return 0;
 809
 810 err_alloc_rx:
 811         emac_tx_q_bufs_free(adpt);
 812 err_alloc_tx:
 813         dma_free_coherent(dev, ring_header->size,
 814                           ring_header->v_addr, ring_header->dma_addr);
 815
 816         ring_header->v_addr   = NULL;
 817         ring_header->dma_addr = 0;
 818         ring_header->size     = 0;
 819         ring_header->used     = 0;
 820
 821         return ret;
 822 }
 823
 824 /* Free all TX and RX descriptor rings */
 825 void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)
 826 {
 827         struct emac_ring_header *ring_header = &adpt->ring_header;
 828         struct device *dev = adpt->netdev->dev.parent;
 829
 830         emac_tx_q_bufs_free(adpt);
 831         emac_rx_q_bufs_free(adpt);
 832
 833         dma_free_coherent(dev, ring_header->size,
 834                           ring_header->v_addr, ring_header->dma_addr);
 835
 836         ring_header->v_addr   = NULL;
 837         ring_header->dma_addr = 0;
 838         ring_header->size     = 0;
 839         ring_header->used     = 0;
 840 }
 841
 842 /* Initialize descriptor rings */
 843 static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)
 844 {
 845         unsigned int i;
 846
 847         adpt->tx_q.tpd.produce_idx = 0;
 848         adpt->tx_q.tpd.consume_idx = 0;
 849         for (i = 0; i < adpt->tx_q.tpd.count; i++)
 850                 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;
 851
 852         adpt->rx_q.rrd.produce_idx = 0;
 853         adpt->rx_q.rrd.consume_idx = 0;
 854         adpt->rx_q.rfd.produce_idx = 0;
 855         adpt->rx_q.rfd.consume_idx = 0;
 856         for (i = 0; i < adpt->rx_q.rfd.count; i++)
 857                 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;
 858 }
 859
 860 /* Produce new receive free descriptor */
 861 static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,
 862                                    struct emac_rx_queue *rx_q,
 863                                    dma_addr_t addr)
 864 {
 865         u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);
 866
 867         *(hw_rfd++) = lower_32_bits(addr);
 868         *hw_rfd = upper_32_bits(addr);
 869
 870         if (++rx_q->rfd.produce_idx == rx_q->rfd.count)
 871                 rx_q->rfd.produce_idx = 0;
 872 }
 873
 874 /* Fill up receive queue's RFD with preallocated receive buffers */
 875 static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
 876                                     struct emac_rx_queue *rx_q)
 877 {
 878         struct emac_buffer *curr_rxbuf;
 879         struct emac_buffer *next_rxbuf;
 880         unsigned int count = 0;
 881         u32 next_produce_idx;
 882
 883         next_produce_idx = rx_q->rfd.produce_idx + 1;
 884         if (next_produce_idx == rx_q->rfd.count)
 885                 next_produce_idx = 0;
 886
 887         curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
 888         next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
 889
 890         /* this always has a blank rx_buffer*/
 891         while (!next_rxbuf->dma_addr) {
 892                 struct sk_buff *skb;
 893                 int ret;
 894
 895                 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);
 896                 if (!skb)
 897                         break;
 898
 899                 curr_rxbuf->dma_addr =
 900                         dma_map_single(adpt->netdev->dev.parent, skb->data,
 901                                        adpt->rxbuf_size, DMA_FROM_DEVICE);
 902
 903                 ret = dma_mapping_error(adpt->netdev->dev.parent,
 904                                         curr_rxbuf->dma_addr);
 905                 if (ret) {
 906                         dev_kfree_skb(skb);
 907                         break;
 908                 }
 909                 curr_rxbuf->skb = skb;
 910                 curr_rxbuf->length = adpt->rxbuf_size;
 911
 912                 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);
 913                 next_produce_idx = rx_q->rfd.produce_idx + 1;
 914                 if (next_produce_idx == rx_q->rfd.count)
 915                         next_produce_idx = 0;
 916
 917                 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
 918                 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
 919                 count++;
 920         }
 921
 922         if (count) {
 923                 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &
 924                                 rx_q->produce_mask;
 925                 emac_reg_update32(adpt->base + rx_q->produce_reg,
 926                                   rx_q->produce_mask, prod_idx);
 927         }
 928 }
 929
 930 static void emac_adjust_link(struct net_device *netdev)
 931 {
 932         struct emac_adapter *adpt = netdev_priv(netdev);
 933         struct emac_sgmii *sgmii = &adpt->phy;
 934         struct phy_device *phydev = netdev->phydev;
 935
 936         if (phydev->link) {
 937                 emac_mac_start(adpt);
 938                 sgmii->link_up(adpt);
 939         } else {
 940                 sgmii->link_down(adpt);
 941                 emac_mac_stop(adpt);
 942         }
 943
 944         phy_print_status(phydev);
 945 }
 946
 947 /* Bringup the interface/HW */
 948 int emac_mac_up(struct emac_adapter *adpt)
 949 {
 950         struct net_device *netdev = adpt->netdev;
 951         int ret;
 952
 953         emac_mac_rx_tx_ring_reset_all(adpt);
 954         emac_mac_config(adpt);
 955         emac_mac_rx_descs_refill(adpt, &adpt->rx_q);
 956
 957         adpt->phydev->irq = PHY_POLL;
 958         ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,
 959                                  PHY_INTERFACE_MODE_SGMII);
 960         if (ret) {
 961                 netdev_err(adpt->netdev, "could not connect phy\n");
 962                 return ret;
 963         }
 964
 965         phy_attached_print(adpt->phydev, NULL);
 966
 967         /* enable mac irq */
 968         writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);
 969         writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);
 970
 971         phy_start(adpt->phydev);
 972
 973         napi_enable(&adpt->rx_q.napi);
 974         netif_start_queue(netdev);
 975
 976         return 0;
 977 }
 978
 979 /* Bring down the interface/HW */
 980 void emac_mac_down(struct emac_adapter *adpt)
 981 {
 982         struct net_device *netdev = adpt->netdev;
 983
 984         netif_stop_queue(netdev);
 985         napi_disable(&adpt->rx_q.napi);
 986
 987         phy_stop(adpt->phydev);
 988
 989         /* Interrupts must be disabled before the PHY is disconnected, to
 990          * avoid a race condition where adjust_link is null when we get
 991          * an interrupt.
 992          */
 993         writel(DIS_INT, adpt->base + EMAC_INT_STATUS);
 994         writel(0, adpt->base + EMAC_INT_MASK);
 995         synchronize_irq(adpt->irq.irq);
 996
 997         phy_disconnect(adpt->phydev);
 998
 999         emac_mac_reset(adpt);
1000
1001         emac_tx_q_descs_free(adpt);
1002         netdev_reset_queue(adpt->netdev);
1003         emac_rx_q_free_descs(adpt);
1004 }
1005
1006 /* Consume next received packet descriptor */
1007 static bool emac_rx_process_rrd(struct emac_adapter *adpt,
1008                                 struct emac_rx_queue *rx_q,
1009                                 struct emac_rrd *rrd)
1010 {
1011         u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);
1012
1013         rrd->word[3] = *(hw_rrd + 3);
1014
1015         if (!RRD_UPDT(rrd))
1016                 return false;
1017
1018         rrd->word[4] = 0;
1019         rrd->word[5] = 0;
1020
1021         rrd->word[0] = *(hw_rrd++);
1022         rrd->word[1] = *(hw_rrd++);
1023         rrd->word[2] = *(hw_rrd++);
1024
1025         if (unlikely(RRD_NOR(rrd) != 1)) {
1026                 netdev_err(adpt->netdev,
1027                            "error: multi-RFD not support yet! nor:%lu\n",
1028                            RRD_NOR(rrd));
1029         }
1030
1031         /* mark rrd as processed */
1032         RRD_UPDT_SET(rrd, 0);
1033         *hw_rrd = rrd->word[3];
1034
1035         if (++rx_q->rrd.consume_idx == rx_q->rrd.count)
1036                 rx_q->rrd.consume_idx = 0;
1037
1038         return true;
1039 }
1040
1041 /* Produce new transmit descriptor */
1042 static void emac_tx_tpd_create(struct emac_adapter *adpt,
1043                                struct emac_tx_queue *tx_q, struct emac_tpd *tpd)
1044 {
1045         u32 *hw_tpd;
1046
1047         tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;
1048         hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);
1049
1050         if (++tx_q->tpd.produce_idx == tx_q->tpd.count)
1051                 tx_q->tpd.produce_idx = 0;
1052
1053         *(hw_tpd++) = tpd->word[0];
1054         *(hw_tpd++) = tpd->word[1];
1055         *(hw_tpd++) = tpd->word[2];
1056         *hw_tpd = tpd->word[3];
1057 }
1058
1059 /* Mark the last transmit descriptor as such (for the transmit packet) */
1060 static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,
1061                                   struct emac_tx_queue *tx_q)
1062 {
1063         u32 *hw_tpd =
1064                 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);
1065         u32 tmp_tpd;
1066
1067         tmp_tpd = *(hw_tpd + 1);
1068         tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
1069         *(hw_tpd + 1) = tmp_tpd;
1070 }
1071
1072 static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)
1073 {
1074         struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;
1075         u32 consume_idx = RRD_SI(rrd);
1076         unsigned int i;
1077
1078         for (i = 0; i < RRD_NOR(rrd); i++) {
1079                 rfbuf[consume_idx].skb = NULL;
1080                 if (++consume_idx == rx_q->rfd.count)
1081                         consume_idx = 0;
1082         }
1083
1084         rx_q->rfd.consume_idx = consume_idx;
1085         rx_q->rfd.process_idx = consume_idx;
1086 }
1087
1088 /* Push the received skb to upper layers */
1089 static void emac_receive_skb(struct emac_rx_queue *rx_q,
1090                              struct sk_buff *skb,
1091                              u16 vlan_tag, bool vlan_flag)
1092 {
1093         if (vlan_flag) {
1094                 u16 vlan;
1095
1096                 EMAC_TAG_TO_VLAN(vlan_tag, vlan);
1097                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
1098         }
1099
1100         napi_gro_receive(&rx_q->napi, skb);
1101 }
1102
1103 /* Process receive event */
1104 void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
1105                          int *num_pkts, int max_pkts)
1106 {
1107         u32 proc_idx, hw_consume_idx, num_consume_pkts;
1108         struct net_device *netdev  = adpt->netdev;
1109         struct emac_buffer *rfbuf;
1110         unsigned int count = 0;
1111         struct emac_rrd rrd;
1112         struct sk_buff *skb;
1113         u32 reg;
1114
1115         reg = readl_relaxed(adpt->base + rx_q->consume_reg);
1116
1117         hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;
1118         num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?
1119                 (hw_consume_idx -  rx_q->rrd.consume_idx) :
1120                 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);
1121
1122         do {
1123                 if (!num_consume_pkts)
1124                         break;
1125
1126                 if (!emac_rx_process_rrd(adpt, rx_q, &rrd))
1127                         break;
1128
1129                 if (likely(RRD_NOR(&rrd) == 1)) {
1130                         /* good receive */
1131                         rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));
1132                         dma_unmap_single(adpt->netdev->dev.parent,
1133                                          rfbuf->dma_addr, rfbuf->length,
1134                                          DMA_FROM_DEVICE);
1135                         rfbuf->dma_addr = 0;
1136                         skb = rfbuf->skb;
1137                 } else {
1138                         netdev_err(adpt->netdev,
1139                                    "error: multi-RFD not support yet!\n");
1140                         break;
1141                 }
1142                 emac_rx_rfd_clean(rx_q, &rrd);
1143                 num_consume_pkts--;
1144                 count++;
1145
1146                 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags
1147                  * with DF set are marked as error), drop packets based on the
1148                  * error mask rather than the summary bit (ignoring L4F errors)
1149                  */
1150                 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {
1151                         netif_dbg(adpt, rx_status, adpt->netdev,
1152                                   "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
1153                                   rrd.word[0], rrd.word[1],
1154                                   rrd.word[2], rrd.word[3]);
1155
1156                         dev_kfree_skb(skb);
1157                         continue;
1158                 }
1159
1160                 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);
1161                 skb->dev = netdev;
1162                 skb->protocol = eth_type_trans(skb, skb->dev);
1163                 if (netdev->features & NETIF_F_RXCSUM)
1164                         skb->ip_summed = RRD_L4F(&rrd) ?
1165                                           CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
1166                 else
1167                         skb_checksum_none_assert(skb);
1168
1169                 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),
1170                                  (bool)RRD_CVTAG(&rrd));
1171
1172                 (*num_pkts)++;
1173         } while (*num_pkts < max_pkts);
1174
1175         if (count) {
1176                 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &
1177                                 rx_q->process_mask;
1178                 emac_reg_update32(adpt->base + rx_q->process_reg,
1179                                   rx_q->process_mask, proc_idx);
1180                 emac_mac_rx_descs_refill(adpt, rx_q);
1181         }
1182 }
1183
1184 /* get the number of free transmit descriptors */
1185 static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)
1186 {
1187         u32 produce_idx = tx_q->tpd.produce_idx;
1188         u32 consume_idx = tx_q->tpd.consume_idx;
1189
1190         return (consume_idx > produce_idx) ?
1191                 (consume_idx - produce_idx - 1) :
1192                 (tx_q->tpd.count + consume_idx - produce_idx - 1);
1193 }
1194
1195 /* Process transmit event */
1196 void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
1197 {
1198         u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);
1199         u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;
1200         struct emac_buffer *tpbuf;
1201
1202         hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;
1203
1204         while (tx_q->tpd.consume_idx != hw_consume_idx) {
1205                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
1206                 if (tpbuf->dma_addr) {
1207                         dma_unmap_single(adpt->netdev->dev.parent,
1208                                          tpbuf->dma_addr, tpbuf->length,
1209                                          DMA_TO_DEVICE);
1210                         tpbuf->dma_addr = 0;
1211                 }
1212
1213                 if (tpbuf->skb) {
1214                         pkts_compl++;
1215                         bytes_compl += tpbuf->skb->len;
1216                         dev_kfree_skb_irq(tpbuf->skb);
1217                         tpbuf->skb = NULL;
1218                 }
1219
1220                 if (++tx_q->tpd.consume_idx == tx_q->tpd.count)
1221                         tx_q->tpd.consume_idx = 0;
1222         }
1223
1224         netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
1225
1226         if (netif_queue_stopped(adpt->netdev))
1227                 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))
1228                         netif_wake_queue(adpt->netdev);
1229 }
1230
1231 /* Initialize all queue data structures */
1232 void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
1233                                   struct emac_adapter *adpt)
1234 {
1235         adpt->rx_q.netdev = adpt->netdev;
1236
1237         adpt->rx_q.produce_reg  = EMAC_MAILBOX_0;
1238         adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;
1239         adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;
1240
1241         adpt->rx_q.process_reg  = EMAC_MAILBOX_0;
1242         adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;
1243         adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;
1244
1245         adpt->rx_q.consume_reg  = EMAC_MAILBOX_3;
1246         adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;
1247         adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;
1248
1249         adpt->rx_q.irq          = &adpt->irq;
1250         adpt->rx_q.intr         = adpt->irq.mask & ISR_RX_PKT;
1251
1252         adpt->tx_q.produce_reg  = EMAC_MAILBOX_15;
1253         adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;
1254         adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;
1255
1256         adpt->tx_q.consume_reg  = EMAC_MAILBOX_2;
1257         adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;
1258         adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;
1259 }
1260
1261 /* Fill up transmit descriptors with TSO and Checksum offload information */
1262 static int emac_tso_csum(struct emac_adapter *adpt,
1263                          struct emac_tx_queue *tx_q,
1264                          struct sk_buff *skb,
1265                          struct emac_tpd *tpd)
1266 {
1267         unsigned int hdr_len;
1268         int ret;
1269
1270         if (skb_is_gso(skb)) {
1271                 if (skb_header_cloned(skb)) {
1272                         ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1273                         if (unlikely(ret))
1274                                 return ret;
1275                 }
1276
1277                 if (skb->protocol == htons(ETH_P_IP)) {
1278                         u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)
1279                                        + ntohs(ip_hdr(skb)->tot_len);
1280                         if (skb->len > pkt_len)
1281                                 pskb_trim(skb, pkt_len);
1282                 }
1283
1284                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1285                 if (unlikely(skb->len == hdr_len)) {
1286                         /* we only need to do csum */
1287                         netif_warn(adpt, tx_err, adpt->netdev,
1288                                    "tso not needed for packet with 0 data\n");
1289                         goto do_csum;
1290                 }
1291
1292                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1293                         ip_hdr(skb)->check = 0;
1294                         tcp_hdr(skb)->check =
1295                                 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1296                                                    ip_hdr(skb)->daddr,
1297                                                    0, IPPROTO_TCP, 0);
1298                         TPD_IPV4_SET(tpd, 1);
1299                 }
1300
1301                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1302                         /* ipv6 tso need an extra tpd */
1303                         struct emac_tpd extra_tpd;
1304
1305                         memset(tpd, 0, sizeof(*tpd));
1306                         memset(&extra_tpd, 0, sizeof(extra_tpd));
1307
1308                         ipv6_hdr(skb)->payload_len = 0;
1309                         tcp_hdr(skb)->check =
1310                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1311                                                  &ipv6_hdr(skb)->daddr,
1312                                                  0, IPPROTO_TCP, 0);
1313                         TPD_PKT_LEN_SET(&extra_tpd, skb->len);
1314                         TPD_LSO_SET(&extra_tpd, 1);
1315                         TPD_LSOV_SET(&extra_tpd, 1);
1316                         emac_tx_tpd_create(adpt, tx_q, &extra_tpd);
1317                         TPD_LSOV_SET(tpd, 1);
1318                 }
1319
1320                 TPD_LSO_SET(tpd, 1);
1321                 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));
1322                 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);
1323                 return 0;
1324         }
1325
1326 do_csum:
1327         if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1328                 unsigned int css, cso;
1329
1330                 cso = skb_transport_offset(skb);
1331                 if (unlikely(cso & 0x1)) {
1332                         netdev_err(adpt->netdev,
1333                                    "error: payload offset should be even\n");
1334                         return -EINVAL;
1335                 }
1336                 css = cso + skb->csum_offset;
1337
1338                 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);
1339                 TPD_CXSUM_OFFSET_SET(tpd, css >> 1);
1340                 TPD_CSX_SET(tpd, 1);
1341         }
1342
1343         return 0;
1344 }
1345
1346 /* Fill up transmit descriptors */
1347 static void emac_tx_fill_tpd(struct emac_adapter *adpt,
1348                              struct emac_tx_queue *tx_q, struct sk_buff *skb,
1349                              struct emac_tpd *tpd)
1350 {
1351         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1352         unsigned int first = tx_q->tpd.produce_idx;
1353         unsigned int len = skb_headlen(skb);
1354         struct emac_buffer *tpbuf = NULL;
1355         unsigned int mapped_len = 0;
1356         unsigned int i;
1357         int count = 0;
1358         int ret;
1359
1360         /* if Large Segment Offload is (in TCP Segmentation Offload struct) */
1361         if (TPD_LSO(tpd)) {
1362                 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1363
1364                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1365                 tpbuf->length = mapped_len;
1366                 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1367                                                  skb->data, tpbuf->length,
1368                                                  DMA_TO_DEVICE);
1369                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1370                                         tpbuf->dma_addr);
1371                 if (ret)
1372                         goto error;
1373
1374                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1375                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1376                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1377                 emac_tx_tpd_create(adpt, tx_q, tpd);
1378                 count++;
1379         }
1380
1381         if (mapped_len < len) {
1382                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1383                 tpbuf->length = len - mapped_len;
1384                 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1385                                                  skb->data + mapped_len,
1386                                                  tpbuf->length, DMA_TO_DEVICE);
1387                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1388                                         tpbuf->dma_addr);
1389                 if (ret)
1390                         goto error;
1391
1392                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1393                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1394                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1395                 emac_tx_tpd_create(adpt, tx_q, tpd);
1396                 count++;
1397         }
1398
1399         for (i = 0; i < nr_frags; i++) {
1400                 struct skb_frag_struct *frag;
1401
1402                 frag = &skb_shinfo(skb)->frags[i];
1403
1404                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1405                 tpbuf->length = frag->size;
1406                 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1407                                                frag->page.p, frag->page_offset,
1408                                                tpbuf->length, DMA_TO_DEVICE);
1409                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1410                                         tpbuf->dma_addr);
1411                 if (ret)
1412                         goto error;
1413
1414                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1415                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1416                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1417                 emac_tx_tpd_create(adpt, tx_q, tpd);
1418                 count++;
1419         }
1420
1421         /* The last tpd */
1422         wmb();
1423         emac_tx_tpd_mark_last(adpt, tx_q);
1424
1425         /* The last buffer info contain the skb address,
1426          * so it will be freed after unmap
1427          */
1428         tpbuf->skb = skb;
1429
1430         return;
1431
1432 error:
1433         /* One of the memory mappings failed, so undo everything */
1434         tx_q->tpd.produce_idx = first;
1435
1436         while (count--) {
1437                 tpbuf = GET_TPD_BUFFER(tx_q, first);
1438                 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,
1439                                tpbuf->length, DMA_TO_DEVICE);
1440                 tpbuf->dma_addr = 0;
1441                 tpbuf->length = 0;
1442
1443                 if (++first == tx_q->tpd.count)
1444                         first = 0;
1445         }
1446
1447         dev_kfree_skb(skb);
1448 }
1449
1450 /* Transmit the packet using specified transmit queue */
1451 int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
1452                          struct sk_buff *skb)
1453 {
1454         struct emac_tpd tpd;
1455         u32 prod_idx;
1456
1457         memset(&tpd, 0, sizeof(tpd));
1458
1459         if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {
1460                 dev_kfree_skb_any(skb);
1461                 return NETDEV_TX_OK;
1462         }
1463
1464         if (skb_vlan_tag_present(skb)) {
1465                 u16 tag;
1466
1467                 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);
1468                 TPD_CVLAN_TAG_SET(&tpd, tag);
1469                 TPD_INSTC_SET(&tpd, 1);
1470         }
1471
1472         if (skb_network_offset(skb) != ETH_HLEN)
1473                 TPD_TYP_SET(&tpd, 1);
1474
1475         emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);
1476
1477         netdev_sent_queue(adpt->netdev, skb->len);
1478
1479         /* Make sure the are enough free descriptors to hold one
1480          * maximum-sized SKB.  We need one desc for each fragment,
1481          * one for the checksum (emac_tso_csum), one for TSO, and
1482          * and one for the SKB header.
1483          */
1484         if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))
1485                 netif_stop_queue(adpt->netdev);
1486
1487         /* update produce idx */
1488         prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &
1489                     tx_q->produce_mask;
1490         emac_reg_update32(adpt->base + tx_q->produce_reg,
1491                           tx_q->produce_mask, prod_idx);
1492
1493         return NETDEV_TX_OK;
1494 }