drivers/net/ethernet/qualcomm/emac/emac-mac.c

   1 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License version 2 and
   5  * only version 2 as published by the Free Software Foundation.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  */
  12
  13 /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support
  14  */
  15
  16 #include <linux/tcp.h>
  17 #include <linux/ip.h>
  18 #include <linux/ipv6.h>
  19 #include <linux/crc32.h>
  20 #include <linux/if_vlan.h>
  21 #include <linux/jiffies.h>
  22 #include <linux/phy.h>
  23 #include <linux/of.h>
  24 #include <net/ip6_checksum.h>
  25 #include "emac.h"
  26 #include "emac-sgmii.h"
  27
  28 /* EMAC_MAC_CTRL */
  29 #define SINGLE_PAUSE_MODE               0x10000000
  30 #define DEBUG_MODE                      0x08000000
  31 #define BROAD_EN                        0x04000000
  32 #define MULTI_ALL                       0x02000000
  33 #define RX_CHKSUM_EN                    0x01000000
  34 #define HUGE                            0x00800000
  35 #define SPEED(x)                        (((x) & 0x3) << 20)
  36 #define SPEED_MASK                      SPEED(0x3)
  37 #define SIMR                            0x00080000
  38 #define TPAUSE                          0x00010000
  39 #define PROM_MODE                       0x00008000
  40 #define VLAN_STRIP                      0x00004000
  41 #define PRLEN_BMSK                      0x00003c00
  42 #define PRLEN_SHFT                      10
  43 #define HUGEN                           0x00000200
  44 #define FLCHK                           0x00000100
  45 #define PCRCE                           0x00000080
  46 #define CRCE                            0x00000040
  47 #define FULLD                           0x00000020
  48 #define MAC_LP_EN                       0x00000010
  49 #define RXFC                            0x00000008
  50 #define TXFC                            0x00000004
  51 #define RXEN                            0x00000002
  52 #define TXEN                            0x00000001
  53
  54 /* EMAC_DESC_CTRL_3 */
  55 #define RFD_RING_SIZE_BMSK                                       0xfff
  56
  57 /* EMAC_DESC_CTRL_4 */
  58 #define RX_BUFFER_SIZE_BMSK                                     0xffff
  59
  60 /* EMAC_DESC_CTRL_6 */
  61 #define RRD_RING_SIZE_BMSK                                       0xfff
  62
  63 /* EMAC_DESC_CTRL_9 */
  64 #define TPD_RING_SIZE_BMSK                                      0xffff
  65
  66 /* EMAC_TXQ_CTRL_0 */
  67 #define NUM_TXF_BURST_PREF_BMSK                             0xffff0000
  68 #define NUM_TXF_BURST_PREF_SHFT                                     16
  69 #define LS_8023_SP                                                0x80
  70 #define TXQ_MODE                                                  0x40
  71 #define TXQ_EN                                                    0x20
  72 #define IP_OP_SP                                                  0x10
  73 #define NUM_TPD_BURST_PREF_BMSK                                    0xf
  74 #define NUM_TPD_BURST_PREF_SHFT                                      0
  75
  76 /* EMAC_TXQ_CTRL_1 */
  77 #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK                        0x7ff
  78
  79 /* EMAC_TXQ_CTRL_2 */
  80 #define TXF_HWM_BMSK                                         0xfff0000
  81 #define TXF_LWM_BMSK                                             0xfff
  82
  83 /* EMAC_RXQ_CTRL_0 */
  84 #define RXQ_EN                                                 BIT(31)
  85 #define CUT_THRU_EN                                            BIT(30)
  86 #define RSS_HASH_EN                                            BIT(29)
  87 #define NUM_RFD_BURST_PREF_BMSK                              0x3f00000
  88 #define NUM_RFD_BURST_PREF_SHFT                                     20
  89 #define IDT_TABLE_SIZE_BMSK                                    0x1ff00
  90 #define IDT_TABLE_SIZE_SHFT                                          8
  91 #define SP_IPV6                                                   0x80
  92
  93 /* EMAC_RXQ_CTRL_1 */
  94 #define JUMBO_1KAH_BMSK                                         0xf000
  95 #define JUMBO_1KAH_SHFT                                             12
  96 #define RFD_PREF_LOW_TH                                           0x10
  97 #define RFD_PREF_LOW_THRESHOLD_BMSK                              0xfc0
  98 #define RFD_PREF_LOW_THRESHOLD_SHFT                                  6
  99 #define RFD_PREF_UP_TH                                            0x10
 100 #define RFD_PREF_UP_THRESHOLD_BMSK                                0x3f
 101 #define RFD_PREF_UP_THRESHOLD_SHFT                                   0
 102
 103 /* EMAC_RXQ_CTRL_2 */
 104 #define RXF_DOF_THRESFHOLD                                       0x1a0
 105 #define RXF_DOF_THRESHOLD_BMSK                               0xfff0000
 106 #define RXF_DOF_THRESHOLD_SHFT                                      16
 107 #define RXF_UOF_THRESFHOLD                                        0xbe
 108 #define RXF_UOF_THRESHOLD_BMSK                                   0xfff
 109 #define RXF_UOF_THRESHOLD_SHFT                                       0
 110
 111 /* EMAC_RXQ_CTRL_3 */
 112 #define RXD_TIMER_BMSK                                      0xffff0000
 113 #define RXD_THRESHOLD_BMSK                                       0xfff
 114 #define RXD_THRESHOLD_SHFT                                           0
 115
 116 /* EMAC_DMA_CTRL */
 117 #define DMAW_DLY_CNT_BMSK                                      0xf0000
 118 #define DMAW_DLY_CNT_SHFT                                           16
 119 #define DMAR_DLY_CNT_BMSK                                       0xf800
 120 #define DMAR_DLY_CNT_SHFT                                           11
 121 #define DMAR_REQ_PRI                                             0x400
 122 #define REGWRBLEN_BMSK                                           0x380
 123 #define REGWRBLEN_SHFT                                               7
 124 #define REGRDBLEN_BMSK                                            0x70
 125 #define REGRDBLEN_SHFT                                               4
 126 #define OUT_ORDER_MODE                                             0x4
 127 #define ENH_ORDER_MODE                                             0x2
 128 #define IN_ORDER_MODE                                              0x1
 129
 130 /* EMAC_MAILBOX_13 */
 131 #define RFD3_PROC_IDX_BMSK                                   0xfff0000
 132 #define RFD3_PROC_IDX_SHFT                                          16
 133 #define RFD3_PROD_IDX_BMSK                                       0xfff
 134 #define RFD3_PROD_IDX_SHFT                                           0
 135
 136 /* EMAC_MAILBOX_2 */
 137 #define NTPD_CONS_IDX_BMSK                                  0xffff0000
 138 #define NTPD_CONS_IDX_SHFT                                          16
 139
 140 /* EMAC_MAILBOX_3 */
 141 #define RFD0_CONS_IDX_BMSK                                       0xfff
 142 #define RFD0_CONS_IDX_SHFT                                           0
 143
 144 /* EMAC_MAILBOX_11 */
 145 #define H3TPD_PROD_IDX_BMSK                                 0xffff0000
 146 #define H3TPD_PROD_IDX_SHFT                                         16
 147
 148 /* EMAC_AXI_MAST_CTRL */
 149 #define DATA_BYTE_SWAP                                             0x8
 150 #define MAX_BOUND                                                  0x2
 151 #define MAX_BTYPE                                                  0x1
 152
 153 /* EMAC_MAILBOX_12 */
 154 #define H3TPD_CONS_IDX_BMSK                                 0xffff0000
 155 #define H3TPD_CONS_IDX_SHFT                                         16
 156
 157 /* EMAC_MAILBOX_9 */
 158 #define H2TPD_PROD_IDX_BMSK                                     0xffff
 159 #define H2TPD_PROD_IDX_SHFT                                          0
 160
 161 /* EMAC_MAILBOX_10 */
 162 #define H1TPD_CONS_IDX_BMSK                                 0xffff0000
 163 #define H1TPD_CONS_IDX_SHFT                                         16
 164 #define H2TPD_CONS_IDX_BMSK                                     0xffff
 165 #define H2TPD_CONS_IDX_SHFT                                          0
 166
 167 /* EMAC_ATHR_HEADER_CTRL */
 168 #define HEADER_CNT_EN                                              0x2
 169 #define HEADER_ENABLE                                              0x1
 170
 171 /* EMAC_MAILBOX_0 */
 172 #define RFD0_PROC_IDX_BMSK                                   0xfff0000
 173 #define RFD0_PROC_IDX_SHFT                                          16
 174 #define RFD0_PROD_IDX_BMSK                                       0xfff
 175 #define RFD0_PROD_IDX_SHFT                                           0
 176
 177 /* EMAC_MAILBOX_5 */
 178 #define RFD1_PROC_IDX_BMSK                                   0xfff0000
 179 #define RFD1_PROC_IDX_SHFT                                          16
 180 #define RFD1_PROD_IDX_BMSK                                       0xfff
 181 #define RFD1_PROD_IDX_SHFT                                           0
 182
 183 /* EMAC_MISC_CTRL */
 184 #define RX_UNCPL_INT_EN                                            0x1
 185
 186 /* EMAC_MAILBOX_7 */
 187 #define RFD2_CONS_IDX_BMSK                                   0xfff0000
 188 #define RFD2_CONS_IDX_SHFT                                          16
 189 #define RFD1_CONS_IDX_BMSK                                       0xfff
 190 #define RFD1_CONS_IDX_SHFT                                           0
 191
 192 /* EMAC_MAILBOX_8 */
 193 #define RFD3_CONS_IDX_BMSK                                       0xfff
 194 #define RFD3_CONS_IDX_SHFT                                           0
 195
 196 /* EMAC_MAILBOX_15 */
 197 #define NTPD_PROD_IDX_BMSK                                      0xffff
 198 #define NTPD_PROD_IDX_SHFT                                           0
 199
 200 /* EMAC_MAILBOX_16 */
 201 #define H1TPD_PROD_IDX_BMSK                                     0xffff
 202 #define H1TPD_PROD_IDX_SHFT                                          0
 203
 204 #define RXQ0_RSS_HSTYP_IPV6_TCP_EN                                0x20
 205 #define RXQ0_RSS_HSTYP_IPV6_EN                                    0x10
 206 #define RXQ0_RSS_HSTYP_IPV4_TCP_EN                                 0x8
 207 #define RXQ0_RSS_HSTYP_IPV4_EN                                     0x4
 208
 209 /* EMAC_EMAC_WRAPPER_TX_TS_INX */
 210 #define EMAC_WRAPPER_TX_TS_EMPTY                               BIT(31)
 211 #define EMAC_WRAPPER_TX_TS_INX_BMSK                             0xffff
 212
 213 struct emac_skb_cb {
 214         u32           tpd_idx;
 215         unsigned long jiffies;
 216 };
 217
 218 #define EMAC_SKB_CB(skb)        ((struct emac_skb_cb *)(skb)->cb)
 219 #define EMAC_RSS_IDT_SIZE       256
 220 #define JUMBO_1KAH              0x4
 221 #define RXD_TH                  0x100
 222 #define EMAC_TPD_LAST_FRAGMENT  0x80000000
 223 #define EMAC_TPD_TSTAMP_SAVE    0x80000000
 224
 225 /* EMAC Errors in emac_rrd.word[3] */
 226 #define EMAC_RRD_L4F            BIT(14)
 227 #define EMAC_RRD_IPF            BIT(15)
 228 #define EMAC_RRD_CRC            BIT(21)
 229 #define EMAC_RRD_FAE            BIT(22)
 230 #define EMAC_RRD_TRN            BIT(23)
 231 #define EMAC_RRD_RNT            BIT(24)
 232 #define EMAC_RRD_INC            BIT(25)
 233 #define EMAC_RRD_FOV            BIT(29)
 234 #define EMAC_RRD_LEN            BIT(30)
 235
 236 /* Error bits that will result in a received frame being discarded */
 237 #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
 238                         EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
 239                         EMAC_RRD_FOV | EMAC_RRD_LEN)
 240 #define EMAC_RRD_STATS_DW_IDX 3
 241
 242 #define EMAC_RRD(RXQ, SIZE, IDX)        ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
 243 #define EMAC_RFD(RXQ, SIZE, IDX)        ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
 244 #define EMAC_TPD(TXQ, SIZE, IDX)        ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
 245
 246 #define GET_RFD_BUFFER(RXQ, IDX)        (&((RXQ)->rfd.rfbuff[(IDX)]))
 247 #define GET_TPD_BUFFER(RTQ, IDX)        (&((RTQ)->tpd.tpbuff[(IDX)]))
 248
 249 #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD       8
 250
 251 #define ISR_RX_PKT      (\
 252         RX_PKT_INT0     |\
 253         RX_PKT_INT1     |\
 254         RX_PKT_INT2     |\
 255         RX_PKT_INT3)
 256
 257 void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)
 258 {
 259         u32 crc32, bit, reg, mta;
 260
 261         /* Calculate the CRC of the MAC address */
 262         crc32 = ether_crc(ETH_ALEN, addr);
 263
 264         /* The HASH Table is an array of 2 32-bit registers. It is
 265          * treated like an array of 64 bits (BitArray[hash_value]).
 266          * Use the upper 6 bits of the above CRC as the hash value.
 267          */
 268         reg = (crc32 >> 31) & 0x1;
 269         bit = (crc32 >> 26) & 0x1F;
 270
 271         mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
 272         mta |= BIT(bit);
 273         writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
 274 }
 275
 276 void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)
 277 {
 278         writel(0, adpt->base + EMAC_HASH_TAB_REG0);
 279         writel(0, adpt->base + EMAC_HASH_TAB_REG1);
 280 }
 281
 282 /* definitions for RSS */
 283 #define EMAC_RSS_KEY(_i, _type) \
 284                 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
 285 #define EMAC_RSS_TBL(_i, _type) \
 286                 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
 287
 288 /* Config MAC modes */
 289 void emac_mac_mode_config(struct emac_adapter *adpt)
 290 {
 291         struct net_device *netdev = adpt->netdev;
 292         u32 mac;
 293
 294         mac = readl(adpt->base + EMAC_MAC_CTRL);
 295         mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);
 296
 297         if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
 298                 mac |= VLAN_STRIP;
 299
 300         if (netdev->flags & IFF_PROMISC)
 301                 mac |= PROM_MODE;
 302
 303         if (netdev->flags & IFF_ALLMULTI)
 304                 mac |= MULTI_ALL;
 305
 306         writel(mac, adpt->base + EMAC_MAC_CTRL);
 307 }
 308
 309 /* Config descriptor rings */
 310 static void emac_mac_dma_rings_config(struct emac_adapter *adpt)
 311 {
 312         /* TPD (Transmit Packet Descriptor) */
 313         writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),
 314                adpt->base + EMAC_DESC_CTRL_1);
 315
 316         writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),
 317                adpt->base + EMAC_DESC_CTRL_8);
 318
 319         writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,
 320                adpt->base + EMAC_DESC_CTRL_9);
 321
 322         /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */
 323         writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),
 324                adpt->base + EMAC_DESC_CTRL_0);
 325
 326         writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),
 327                adpt->base + EMAC_DESC_CTRL_2);
 328         writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),
 329                adpt->base + EMAC_DESC_CTRL_5);
 330
 331         writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,
 332                adpt->base + EMAC_DESC_CTRL_3);
 333         writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,
 334                adpt->base + EMAC_DESC_CTRL_6);
 335
 336         writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,
 337                adpt->base + EMAC_DESC_CTRL_4);
 338
 339         writel(0, adpt->base + EMAC_DESC_CTRL_11);
 340
 341         /* Load all of the base addresses above and ensure that triggering HW to
 342          * read ring pointers is flushed
 343          */
 344         writel(1, adpt->base + EMAC_INTER_SRAM_PART9);
 345 }
 346
 347 /* Config transmit parameters */
 348 static void emac_mac_tx_config(struct emac_adapter *adpt)
 349 {
 350         u32 val;
 351
 352         writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &
 353                JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);
 354
 355         val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &
 356                NUM_TPD_BURST_PREF_BMSK;
 357
 358         val |= TXQ_MODE | LS_8023_SP;
 359         val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &
 360                 NUM_TXF_BURST_PREF_BMSK;
 361
 362         writel(val, adpt->base + EMAC_TXQ_CTRL_0);
 363         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,
 364                           (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);
 365 }
 366
 367 /* Config receive parameters */
 368 static void emac_mac_rx_config(struct emac_adapter *adpt)
 369 {
 370         u32 val;
 371
 372         val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &
 373                NUM_RFD_BURST_PREF_BMSK;
 374         val |= (SP_IPV6 | CUT_THRU_EN);
 375
 376         writel(val, adpt->base + EMAC_RXQ_CTRL_0);
 377
 378         val = readl(adpt->base + EMAC_RXQ_CTRL_1);
 379         val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |
 380                  RFD_PREF_UP_THRESHOLD_BMSK);
 381         val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |
 382                 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |
 383                 (RFD_PREF_UP_TH  << RFD_PREF_UP_THRESHOLD_SHFT);
 384         writel(val, adpt->base + EMAC_RXQ_CTRL_1);
 385
 386         val = readl(adpt->base + EMAC_RXQ_CTRL_2);
 387         val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);
 388         val |= (RXF_DOF_THRESFHOLD  << RXF_DOF_THRESHOLD_SHFT) |
 389                 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);
 390         writel(val, adpt->base + EMAC_RXQ_CTRL_2);
 391
 392         val = readl(adpt->base + EMAC_RXQ_CTRL_3);
 393         val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);
 394         val |= RXD_TH << RXD_THRESHOLD_SHFT;
 395         writel(val, adpt->base + EMAC_RXQ_CTRL_3);
 396 }
 397
 398 /* Config dma */
 399 static void emac_mac_dma_config(struct emac_adapter *adpt)
 400 {
 401         u32 dma_ctrl = DMAR_REQ_PRI;
 402
 403         switch (adpt->dma_order) {
 404         case emac_dma_ord_in:
 405                 dma_ctrl |= IN_ORDER_MODE;
 406                 break;
 407         case emac_dma_ord_enh:
 408                 dma_ctrl |= ENH_ORDER_MODE;
 409                 break;
 410         case emac_dma_ord_out:
 411                 dma_ctrl |= OUT_ORDER_MODE;
 412                 break;
 413         default:
 414                 break;
 415         }
 416
 417         dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &
 418                                                 REGRDBLEN_BMSK;
 419         dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &
 420                                                 REGWRBLEN_BMSK;
 421         dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &
 422                                                 DMAR_DLY_CNT_BMSK;
 423         dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &
 424                                                 DMAW_DLY_CNT_BMSK;
 425
 426         /* config DMA and ensure that configuration is flushed to HW */
 427         writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);
 428 }
 429
 430 /* set MAC address */
 431 static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr)
 432 {
 433         u32 sta;
 434
 435         /* for example: 00-A0-C6-11-22-33
 436          * 0<-->C6112233, 1<-->00A0.
 437          */
 438
 439         /* low 32bit word */
 440         sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |
 441               (((u32)addr[4]) << 8)  | (((u32)addr[5]));
 442         writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);
 443
 444         /* hight 32bit word */
 445         sta = (((u32)addr[0]) << 8) | (u32)addr[1];
 446         writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);
 447 }
 448
 449 static void emac_mac_config(struct emac_adapter *adpt)
 450 {
 451         struct net_device *netdev = adpt->netdev;
 452         unsigned int max_frame;
 453         u32 val;
 454
 455         emac_set_mac_address(adpt, netdev->dev_addr);
 456
 457         max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
 458         adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
 459                 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
 460
 461         emac_mac_dma_rings_config(adpt);
 462
 463         writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,
 464                adpt->base + EMAC_MAX_FRAM_LEN_CTRL);
 465
 466         emac_mac_tx_config(adpt);
 467         emac_mac_rx_config(adpt);
 468         emac_mac_dma_config(adpt);
 469
 470         val = readl(adpt->base + EMAC_AXI_MAST_CTRL);
 471         val &= ~(DATA_BYTE_SWAP | MAX_BOUND);
 472         val |= MAX_BTYPE;
 473         writel(val, adpt->base + EMAC_AXI_MAST_CTRL);
 474         writel(0, adpt->base + EMAC_CLK_GATE_CTRL);
 475         writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);
 476 }
 477
 478 void emac_mac_reset(struct emac_adapter *adpt)
 479 {
 480         emac_mac_stop(adpt);
 481
 482         emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);
 483         usleep_range(100, 150); /* reset may take up to 100usec */
 484
 485         /* interrupt clear-on-read */
 486         emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);
 487 }
 488
 489 static void emac_mac_start(struct emac_adapter *adpt)
 490 {
 491         struct phy_device *phydev = adpt->phydev;
 492         u32 mac, csr1;
 493
 494         /* enable tx queue */
 495         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);
 496
 497         /* enable rx queue */
 498         emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);
 499
 500         /* enable mac control */
 501         mac = readl(adpt->base + EMAC_MAC_CTRL);
 502         csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
 503
 504         mac |= TXEN | RXEN;     /* enable RX/TX */
 505
 506         /* Configure MAC flow control. If set to automatic, then match
 507          * whatever the PHY does. Otherwise, enable or disable it, depending
 508          * on what the user configured via ethtool.
 509          */
 510         mac &= ~(RXFC | TXFC);
 511
 512         if (adpt->automatic) {
 513                 /* If it's set to automatic, then update our local values */
 514                 adpt->rx_flow_control = phydev->pause;
 515                 adpt->tx_flow_control = phydev->pause != phydev->asym_pause;
 516         }
 517         mac |= adpt->rx_flow_control ? RXFC : 0;
 518         mac |= adpt->tx_flow_control ? TXFC : 0;
 519
 520         /* setup link speed */
 521         mac &= ~SPEED_MASK;
 522         if (phydev->speed == SPEED_1000) {
 523                 mac |= SPEED(2);
 524                 csr1 |= FREQ_MODE;
 525         } else {
 526                 mac |= SPEED(1);
 527                 csr1 &= ~FREQ_MODE;
 528         }
 529
 530         if (phydev->duplex == DUPLEX_FULL)
 531                 mac |= FULLD;
 532         else
 533                 mac &= ~FULLD;
 534
 535         /* other parameters */
 536         mac |= (CRCE | PCRCE);
 537         mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);
 538         mac |= BROAD_EN;
 539         mac |= FLCHK;
 540         mac &= ~RX_CHKSUM_EN;
 541         mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |
 542                  DEBUG_MODE | SINGLE_PAUSE_MODE);
 543
 544         /* Enable single-pause-frame mode if requested.
 545          *
 546          * If enabled, the EMAC will send a single pause frame when the RX
 547          * queue is full.  This normally leads to packet loss because
 548          * the pause frame disables the remote MAC only for 33ms (the quanta),
 549          * and then the remote MAC continues sending packets even though
 550          * the RX queue is still full.
 551          *
 552          * If disabled, the EMAC sends a pause frame every 31ms until the RX
 553          * queue is no longer full.  Normally, this is the preferred
 554          * method of operation.  However, when the system is hung (e.g.
 555          * cores are halted), the EMAC interrupt handler is never called
 556          * and so the RX queue fills up quickly and stays full.  The resuling
 557          * non-stop "flood" of pause frames sometimes has the effect of
 558          * disabling nearby switches.  In some cases, other nearby switches
 559          * are also affected, shutting down the entire network.
 560          *
 561          * The user can enable or disable single-pause-frame mode
 562          * via ethtool.
 563          */
 564         mac |= adpt->single_pause_mode ? SINGLE_PAUSE_MODE : 0;
 565
 566         writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
 567
 568         writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);
 569
 570         /* enable interrupt read clear, low power sleep mode and
 571          * the irq moderators
 572          */
 573
 574         writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);
 575         writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |
 576                         IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);
 577
 578         emac_mac_mode_config(adpt);
 579
 580         emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,
 581                           (HEADER_ENABLE | HEADER_CNT_EN), 0);
 582 }
 583
 584 void emac_mac_stop(struct emac_adapter *adpt)
 585 {
 586         emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);
 587         emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);
 588         emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);
 589         usleep_range(1000, 1050); /* stopping mac may take upto 1msec */
 590 }
 591
 592 /* Free all descriptors of given transmit queue */
 593 static void emac_tx_q_descs_free(struct emac_adapter *adpt)
 594 {
 595         struct emac_tx_queue *tx_q = &adpt->tx_q;
 596         unsigned int i;
 597         size_t size;
 598
 599         /* ring already cleared, nothing to do */
 600         if (!tx_q->tpd.tpbuff)
 601                 return;
 602
 603         for (i = 0; i < tx_q->tpd.count; i++) {
 604                 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);
 605
 606                 if (tpbuf->dma_addr) {
 607                         dma_unmap_single(adpt->netdev->dev.parent,
 608                                          tpbuf->dma_addr, tpbuf->length,
 609                                          DMA_TO_DEVICE);
 610                         tpbuf->dma_addr = 0;
 611                 }
 612                 if (tpbuf->skb) {
 613                         dev_kfree_skb_any(tpbuf->skb);
 614                         tpbuf->skb = NULL;
 615                 }
 616         }
 617
 618         size = sizeof(struct emac_buffer) * tx_q->tpd.count;
 619         memset(tx_q->tpd.tpbuff, 0, size);
 620
 621         /* clear the descriptor ring */
 622         memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);
 623
 624         tx_q->tpd.consume_idx = 0;
 625         tx_q->tpd.produce_idx = 0;
 626 }
 627
 628 /* Free all descriptors of given receive queue */
 629 static void emac_rx_q_free_descs(struct emac_adapter *adpt)
 630 {
 631         struct device *dev = adpt->netdev->dev.parent;
 632         struct emac_rx_queue *rx_q = &adpt->rx_q;
 633         unsigned int i;
 634         size_t size;
 635
 636         /* ring already cleared, nothing to do */
 637         if (!rx_q->rfd.rfbuff)
 638                 return;
 639
 640         for (i = 0; i < rx_q->rfd.count; i++) {
 641                 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);
 642
 643                 if (rfbuf->dma_addr) {
 644                         dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,
 645                                          DMA_FROM_DEVICE);
 646                         rfbuf->dma_addr = 0;
 647                 }
 648                 if (rfbuf->skb) {
 649                         dev_kfree_skb(rfbuf->skb);
 650                         rfbuf->skb = NULL;
 651                 }
 652         }
 653
 654         size =  sizeof(struct emac_buffer) * rx_q->rfd.count;
 655         memset(rx_q->rfd.rfbuff, 0, size);
 656
 657         /* clear the descriptor rings */
 658         memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);
 659         rx_q->rrd.produce_idx = 0;
 660         rx_q->rrd.consume_idx = 0;
 661
 662         memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);
 663         rx_q->rfd.produce_idx = 0;
 664         rx_q->rfd.consume_idx = 0;
 665 }
 666
 667 /* Free all buffers associated with given transmit queue */
 668 static void emac_tx_q_bufs_free(struct emac_adapter *adpt)
 669 {
 670         struct emac_tx_queue *tx_q = &adpt->tx_q;
 671
 672         emac_tx_q_descs_free(adpt);
 673
 674         kfree(tx_q->tpd.tpbuff);
 675         tx_q->tpd.tpbuff = NULL;
 676         tx_q->tpd.v_addr = NULL;
 677         tx_q->tpd.dma_addr = 0;
 678         tx_q->tpd.size = 0;
 679 }
 680
 681 /* Allocate TX descriptor ring for the given transmit queue */
 682 static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
 683                                 struct emac_tx_queue *tx_q)
 684 {
 685         struct emac_ring_header *ring_header = &adpt->ring_header;
 686         int node = dev_to_node(adpt->netdev->dev.parent);
 687         size_t size;
 688
 689         size = sizeof(struct emac_buffer) * tx_q->tpd.count;
 690         tx_q->tpd.tpbuff = kzalloc_node(size, GFP_KERNEL, node);
 691         if (!tx_q->tpd.tpbuff)
 692                 return -ENOMEM;
 693
 694         tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);
 695         tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;
 696         tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;
 697         ring_header->used += ALIGN(tx_q->tpd.size, 8);
 698         tx_q->tpd.produce_idx = 0;
 699         tx_q->tpd.consume_idx = 0;
 700
 701         return 0;
 702 }
 703
 704 /* Free all buffers associated with given transmit queue */
 705 static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
 706 {
 707         struct emac_rx_queue *rx_q = &adpt->rx_q;
 708
 709         emac_rx_q_free_descs(adpt);
 710
 711         kfree(rx_q->rfd.rfbuff);
 712         rx_q->rfd.rfbuff   = NULL;
 713
 714         rx_q->rfd.v_addr   = NULL;
 715         rx_q->rfd.dma_addr = 0;
 716         rx_q->rfd.size     = 0;
 717
 718         rx_q->rrd.v_addr   = NULL;
 719         rx_q->rrd.dma_addr = 0;
 720         rx_q->rrd.size     = 0;
 721 }
 722
 723 /* Allocate RX descriptor rings for the given receive queue */
 724 static int emac_rx_descs_alloc(struct emac_adapter *adpt)
 725 {
 726         struct emac_ring_header *ring_header = &adpt->ring_header;
 727         int node = dev_to_node(adpt->netdev->dev.parent);
 728         struct emac_rx_queue *rx_q = &adpt->rx_q;
 729         size_t size;
 730
 731         size = sizeof(struct emac_buffer) * rx_q->rfd.count;
 732         rx_q->rfd.rfbuff = kzalloc_node(size, GFP_KERNEL, node);
 733         if (!rx_q->rfd.rfbuff)
 734                 return -ENOMEM;
 735
 736         rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);
 737         rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);
 738
 739         rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;
 740         rx_q->rrd.v_addr   = ring_header->v_addr + ring_header->used;
 741         ring_header->used += ALIGN(rx_q->rrd.size, 8);
 742
 743         rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;
 744         rx_q->rfd.v_addr   = ring_header->v_addr + ring_header->used;
 745         ring_header->used += ALIGN(rx_q->rfd.size, 8);
 746
 747         rx_q->rrd.produce_idx = 0;
 748         rx_q->rrd.consume_idx = 0;
 749
 750         rx_q->rfd.produce_idx = 0;
 751         rx_q->rfd.consume_idx = 0;
 752
 753         return 0;
 754 }
 755
 756 /* Allocate all TX and RX descriptor rings */
 757 int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)
 758 {
 759         struct emac_ring_header *ring_header = &adpt->ring_header;
 760         struct device *dev = adpt->netdev->dev.parent;
 761         unsigned int num_tx_descs = adpt->tx_desc_cnt;
 762         unsigned int num_rx_descs = adpt->rx_desc_cnt;
 763         int ret;
 764
 765         adpt->tx_q.tpd.count = adpt->tx_desc_cnt;
 766
 767         adpt->rx_q.rrd.count = adpt->rx_desc_cnt;
 768         adpt->rx_q.rfd.count = adpt->rx_desc_cnt;
 769
 770         /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,
 771          * hence the additional padding bytes are allocated.
 772          */
 773         ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +
 774                             num_rx_descs * (adpt->rfd_size * 4) +
 775                             num_rx_descs * (adpt->rrd_size * 4) +
 776                             8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */
 777
 778         ring_header->used = 0;
 779         ring_header->v_addr = dma_alloc_coherent(dev, ring_header->size,
 780                                                  &ring_header->dma_addr,
 781                                                  GFP_KERNEL);
 782         if (!ring_header->v_addr)
 783                 return -ENOMEM;
 784
 785         ring_header->used = ALIGN(ring_header->dma_addr, 8) -
 786                                                         ring_header->dma_addr;
 787
 788         ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);
 789         if (ret) {
 790                 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");
 791                 goto err_alloc_tx;
 792         }
 793
 794         ret = emac_rx_descs_alloc(adpt);
 795         if (ret) {
 796                 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");
 797                 goto err_alloc_rx;
 798         }
 799
 800         return 0;
 801
 802 err_alloc_rx:
 803         emac_tx_q_bufs_free(adpt);
 804 err_alloc_tx:
 805         dma_free_coherent(dev, ring_header->size,
 806                           ring_header->v_addr, ring_header->dma_addr);
 807
 808         ring_header->v_addr   = NULL;
 809         ring_header->dma_addr = 0;
 810         ring_header->size     = 0;
 811         ring_header->used     = 0;
 812
 813         return ret;
 814 }
 815
 816 /* Free all TX and RX descriptor rings */
 817 void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)
 818 {
 819         struct emac_ring_header *ring_header = &adpt->ring_header;
 820         struct device *dev = adpt->netdev->dev.parent;
 821
 822         emac_tx_q_bufs_free(adpt);
 823         emac_rx_q_bufs_free(adpt);
 824
 825         dma_free_coherent(dev, ring_header->size,
 826                           ring_header->v_addr, ring_header->dma_addr);
 827
 828         ring_header->v_addr   = NULL;
 829         ring_header->dma_addr = 0;
 830         ring_header->size     = 0;
 831         ring_header->used     = 0;
 832 }
 833
 834 /* Initialize descriptor rings */
 835 static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)
 836 {
 837         unsigned int i;
 838
 839         adpt->tx_q.tpd.produce_idx = 0;
 840         adpt->tx_q.tpd.consume_idx = 0;
 841         for (i = 0; i < adpt->tx_q.tpd.count; i++)
 842                 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;
 843
 844         adpt->rx_q.rrd.produce_idx = 0;
 845         adpt->rx_q.rrd.consume_idx = 0;
 846         adpt->rx_q.rfd.produce_idx = 0;
 847         adpt->rx_q.rfd.consume_idx = 0;
 848         for (i = 0; i < adpt->rx_q.rfd.count; i++)
 849                 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;
 850 }
 851
 852 /* Produce new receive free descriptor */
 853 static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,
 854                                    struct emac_rx_queue *rx_q,
 855                                    dma_addr_t addr)
 856 {
 857         u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);
 858
 859         *(hw_rfd++) = lower_32_bits(addr);
 860         *hw_rfd = upper_32_bits(addr);
 861
 862         if (++rx_q->rfd.produce_idx == rx_q->rfd.count)
 863                 rx_q->rfd.produce_idx = 0;
 864 }
 865
 866 /* Fill up receive queue's RFD with preallocated receive buffers */
 867 static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
 868                                     struct emac_rx_queue *rx_q)
 869 {
 870         struct emac_buffer *curr_rxbuf;
 871         struct emac_buffer *next_rxbuf;
 872         unsigned int count = 0;
 873         u32 next_produce_idx;
 874
 875         next_produce_idx = rx_q->rfd.produce_idx + 1;
 876         if (next_produce_idx == rx_q->rfd.count)
 877                 next_produce_idx = 0;
 878
 879         curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
 880         next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
 881
 882         /* this always has a blank rx_buffer*/
 883         while (!next_rxbuf->dma_addr) {
 884                 struct sk_buff *skb;
 885                 int ret;
 886
 887                 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);
 888                 if (!skb)
 889                         break;
 890
 891                 curr_rxbuf->dma_addr =
 892                         dma_map_single(adpt->netdev->dev.parent, skb->data,
 893                                        adpt->rxbuf_size, DMA_FROM_DEVICE);
 894
 895                 ret = dma_mapping_error(adpt->netdev->dev.parent,
 896                                         curr_rxbuf->dma_addr);
 897                 if (ret) {
 898                         dev_kfree_skb(skb);
 899                         break;
 900                 }
 901                 curr_rxbuf->skb = skb;
 902                 curr_rxbuf->length = adpt->rxbuf_size;
 903
 904                 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);
 905                 next_produce_idx = rx_q->rfd.produce_idx + 1;
 906                 if (next_produce_idx == rx_q->rfd.count)
 907                         next_produce_idx = 0;
 908
 909                 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
 910                 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
 911                 count++;
 912         }
 913
 914         if (count) {
 915                 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &
 916                                 rx_q->produce_mask;
 917                 emac_reg_update32(adpt->base + rx_q->produce_reg,
 918                                   rx_q->produce_mask, prod_idx);
 919         }
 920 }
 921
 922 static void emac_adjust_link(struct net_device *netdev)
 923 {
 924         struct emac_adapter *adpt = netdev_priv(netdev);
 925         struct phy_device *phydev = netdev->phydev;
 926
 927         if (phydev->link) {
 928                 emac_mac_start(adpt);
 929                 emac_sgmii_link_change(adpt, true);
 930         } else {
 931                 emac_sgmii_link_change(adpt, false);
 932                 emac_mac_stop(adpt);
 933         }
 934
 935         phy_print_status(phydev);
 936 }
 937
 938 /* Bringup the interface/HW */
 939 int emac_mac_up(struct emac_adapter *adpt)
 940 {
 941         struct net_device *netdev = adpt->netdev;
 942         int ret;
 943
 944         emac_mac_rx_tx_ring_reset_all(adpt);
 945         emac_mac_config(adpt);
 946         emac_mac_rx_descs_refill(adpt, &adpt->rx_q);
 947
 948         adpt->phydev->irq = PHY_POLL;
 949         ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,
 950                                  PHY_INTERFACE_MODE_SGMII);
 951         if (ret) {
 952                 netdev_err(adpt->netdev, "could not connect phy\n");
 953                 return ret;
 954         }
 955
 956         phy_attached_print(adpt->phydev, NULL);
 957
 958         /* enable mac irq */
 959         writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);
 960         writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);
 961
 962         phy_start(adpt->phydev);
 963
 964         napi_enable(&adpt->rx_q.napi);
 965         netif_start_queue(netdev);
 966
 967         return 0;
 968 }
 969
 970 /* Bring down the interface/HW */
 971 void emac_mac_down(struct emac_adapter *adpt)
 972 {
 973         struct net_device *netdev = adpt->netdev;
 974
 975         netif_stop_queue(netdev);
 976         napi_disable(&adpt->rx_q.napi);
 977
 978         phy_stop(adpt->phydev);
 979
 980         /* Interrupts must be disabled before the PHY is disconnected, to
 981          * avoid a race condition where adjust_link is null when we get
 982          * an interrupt.
 983          */
 984         writel(DIS_INT, adpt->base + EMAC_INT_STATUS);
 985         writel(0, adpt->base + EMAC_INT_MASK);
 986         synchronize_irq(adpt->irq.irq);
 987
 988         phy_disconnect(adpt->phydev);
 989
 990         emac_mac_reset(adpt);
 991
 992         emac_tx_q_descs_free(adpt);
 993         netdev_reset_queue(adpt->netdev);
 994         emac_rx_q_free_descs(adpt);
 995 }
 996
 997 /* Consume next received packet descriptor */
 998 static bool emac_rx_process_rrd(struct emac_adapter *adpt,
 999                                 struct emac_rx_queue *rx_q,
1000                                 struct emac_rrd *rrd)
1001 {
1002         u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);
1003
1004         rrd->word[3] = *(hw_rrd + 3);
1005
1006         if (!RRD_UPDT(rrd))
1007                 return false;
1008
1009         rrd->word[4] = 0;
1010         rrd->word[5] = 0;
1011
1012         rrd->word[0] = *(hw_rrd++);
1013         rrd->word[1] = *(hw_rrd++);
1014         rrd->word[2] = *(hw_rrd++);
1015
1016         if (unlikely(RRD_NOR(rrd) != 1)) {
1017                 netdev_err(adpt->netdev,
1018                            "error: multi-RFD not support yet! nor:%lu\n",
1019                            RRD_NOR(rrd));
1020         }
1021
1022         /* mark rrd as processed */
1023         RRD_UPDT_SET(rrd, 0);
1024         *hw_rrd = rrd->word[3];
1025
1026         if (++rx_q->rrd.consume_idx == rx_q->rrd.count)
1027                 rx_q->rrd.consume_idx = 0;
1028
1029         return true;
1030 }
1031
1032 /* Produce new transmit descriptor */
1033 static void emac_tx_tpd_create(struct emac_adapter *adpt,
1034                                struct emac_tx_queue *tx_q, struct emac_tpd *tpd)
1035 {
1036         u32 *hw_tpd;
1037
1038         tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;
1039         hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);
1040
1041         if (++tx_q->tpd.produce_idx == tx_q->tpd.count)
1042                 tx_q->tpd.produce_idx = 0;
1043
1044         *(hw_tpd++) = tpd->word[0];
1045         *(hw_tpd++) = tpd->word[1];
1046         *(hw_tpd++) = tpd->word[2];
1047         *hw_tpd = tpd->word[3];
1048 }
1049
1050 /* Mark the last transmit descriptor as such (for the transmit packet) */
1051 static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,
1052                                   struct emac_tx_queue *tx_q)
1053 {
1054         u32 *hw_tpd =
1055                 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);
1056         u32 tmp_tpd;
1057
1058         tmp_tpd = *(hw_tpd + 1);
1059         tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
1060         *(hw_tpd + 1) = tmp_tpd;
1061 }
1062
1063 static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)
1064 {
1065         struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;
1066         u32 consume_idx = RRD_SI(rrd);
1067         unsigned int i;
1068
1069         for (i = 0; i < RRD_NOR(rrd); i++) {
1070                 rfbuf[consume_idx].skb = NULL;
1071                 if (++consume_idx == rx_q->rfd.count)
1072                         consume_idx = 0;
1073         }
1074
1075         rx_q->rfd.consume_idx = consume_idx;
1076         rx_q->rfd.process_idx = consume_idx;
1077 }
1078
1079 /* Push the received skb to upper layers */
1080 static void emac_receive_skb(struct emac_rx_queue *rx_q,
1081                              struct sk_buff *skb,
1082                              u16 vlan_tag, bool vlan_flag)
1083 {
1084         if (vlan_flag) {
1085                 u16 vlan;
1086
1087                 EMAC_TAG_TO_VLAN(vlan_tag, vlan);
1088                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
1089         }
1090
1091         napi_gro_receive(&rx_q->napi, skb);
1092 }
1093
1094 /* Process receive event */
1095 void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
1096                          int *num_pkts, int max_pkts)
1097 {
1098         u32 proc_idx, hw_consume_idx, num_consume_pkts;
1099         struct net_device *netdev  = adpt->netdev;
1100         struct emac_buffer *rfbuf;
1101         unsigned int count = 0;
1102         struct emac_rrd rrd;
1103         struct sk_buff *skb;
1104         u32 reg;
1105
1106         reg = readl_relaxed(adpt->base + rx_q->consume_reg);
1107
1108         hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;
1109         num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?
1110                 (hw_consume_idx -  rx_q->rrd.consume_idx) :
1111                 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);
1112
1113         do {
1114                 if (!num_consume_pkts)
1115                         break;
1116
1117                 if (!emac_rx_process_rrd(adpt, rx_q, &rrd))
1118                         break;
1119
1120                 if (likely(RRD_NOR(&rrd) == 1)) {
1121                         /* good receive */
1122                         rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));
1123                         dma_unmap_single(adpt->netdev->dev.parent,
1124                                          rfbuf->dma_addr, rfbuf->length,
1125                                          DMA_FROM_DEVICE);
1126                         rfbuf->dma_addr = 0;
1127                         skb = rfbuf->skb;
1128                 } else {
1129                         netdev_err(adpt->netdev,
1130                                    "error: multi-RFD not support yet!\n");
1131                         break;
1132                 }
1133                 emac_rx_rfd_clean(rx_q, &rrd);
1134                 num_consume_pkts--;
1135                 count++;
1136
1137                 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags
1138                  * with DF set are marked as error), drop packets based on the
1139                  * error mask rather than the summary bit (ignoring L4F errors)
1140                  */
1141                 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {
1142                         netif_dbg(adpt, rx_status, adpt->netdev,
1143                                   "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
1144                                   rrd.word[0], rrd.word[1],
1145                                   rrd.word[2], rrd.word[3]);
1146
1147                         dev_kfree_skb(skb);
1148                         continue;
1149                 }
1150
1151                 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);
1152                 skb->dev = netdev;
1153                 skb->protocol = eth_type_trans(skb, skb->dev);
1154                 if (netdev->features & NETIF_F_RXCSUM)
1155                         skb->ip_summed = RRD_L4F(&rrd) ?
1156                                           CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
1157                 else
1158                         skb_checksum_none_assert(skb);
1159
1160                 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),
1161                                  (bool)RRD_CVTAG(&rrd));
1162
1163                 (*num_pkts)++;
1164         } while (*num_pkts < max_pkts);
1165
1166         if (count) {
1167                 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &
1168                                 rx_q->process_mask;
1169                 emac_reg_update32(adpt->base + rx_q->process_reg,
1170                                   rx_q->process_mask, proc_idx);
1171                 emac_mac_rx_descs_refill(adpt, rx_q);
1172         }
1173 }
1174
1175 /* get the number of free transmit descriptors */
1176 static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)
1177 {
1178         u32 produce_idx = tx_q->tpd.produce_idx;
1179         u32 consume_idx = tx_q->tpd.consume_idx;
1180
1181         return (consume_idx > produce_idx) ?
1182                 (consume_idx - produce_idx - 1) :
1183                 (tx_q->tpd.count + consume_idx - produce_idx - 1);
1184 }
1185
1186 /* Process transmit event */
1187 void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
1188 {
1189         u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);
1190         u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;
1191         struct emac_buffer *tpbuf;
1192
1193         hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;
1194
1195         while (tx_q->tpd.consume_idx != hw_consume_idx) {
1196                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
1197                 if (tpbuf->dma_addr) {
1198                         dma_unmap_page(adpt->netdev->dev.parent,
1199                                        tpbuf->dma_addr, tpbuf->length,
1200                                        DMA_TO_DEVICE);
1201                         tpbuf->dma_addr = 0;
1202                 }
1203
1204                 if (tpbuf->skb) {
1205                         pkts_compl++;
1206                         bytes_compl += tpbuf->skb->len;
1207                         dev_kfree_skb_irq(tpbuf->skb);
1208                         tpbuf->skb = NULL;
1209                 }
1210
1211                 if (++tx_q->tpd.consume_idx == tx_q->tpd.count)
1212                         tx_q->tpd.consume_idx = 0;
1213         }
1214
1215         netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
1216
1217         if (netif_queue_stopped(adpt->netdev))
1218                 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))
1219                         netif_wake_queue(adpt->netdev);
1220 }
1221
1222 /* Initialize all queue data structures */
1223 void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
1224                                   struct emac_adapter *adpt)
1225 {
1226         adpt->rx_q.netdev = adpt->netdev;
1227
1228         adpt->rx_q.produce_reg  = EMAC_MAILBOX_0;
1229         adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;
1230         adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;
1231
1232         adpt->rx_q.process_reg  = EMAC_MAILBOX_0;
1233         adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;
1234         adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;
1235
1236         adpt->rx_q.consume_reg  = EMAC_MAILBOX_3;
1237         adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;
1238         adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;
1239
1240         adpt->rx_q.irq          = &adpt->irq;
1241         adpt->rx_q.intr         = adpt->irq.mask & ISR_RX_PKT;
1242
1243         adpt->tx_q.produce_reg  = EMAC_MAILBOX_15;
1244         adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;
1245         adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;
1246
1247         adpt->tx_q.consume_reg  = EMAC_MAILBOX_2;
1248         adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;
1249         adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;
1250 }
1251
1252 /* Fill up transmit descriptors with TSO and Checksum offload information */
1253 static int emac_tso_csum(struct emac_adapter *adpt,
1254                          struct emac_tx_queue *tx_q,
1255                          struct sk_buff *skb,
1256                          struct emac_tpd *tpd)
1257 {
1258         unsigned int hdr_len;
1259         int ret;
1260
1261         if (skb_is_gso(skb)) {
1262                 if (skb_header_cloned(skb)) {
1263                         ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1264                         if (unlikely(ret))
1265                                 return ret;
1266                 }
1267
1268                 if (skb->protocol == htons(ETH_P_IP)) {
1269                         u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)
1270                                        + ntohs(ip_hdr(skb)->tot_len);
1271                         if (skb->len > pkt_len)
1272                                 pskb_trim(skb, pkt_len);
1273                 }
1274
1275                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1276                 if (unlikely(skb->len == hdr_len)) {
1277                         /* we only need to do csum */
1278                         netif_warn(adpt, tx_err, adpt->netdev,
1279                                    "tso not needed for packet with 0 data\n");
1280                         goto do_csum;
1281                 }
1282
1283                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1284                         ip_hdr(skb)->check = 0;
1285                         tcp_hdr(skb)->check =
1286                                 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1287                                                    ip_hdr(skb)->daddr,
1288                                                    0, IPPROTO_TCP, 0);
1289                         TPD_IPV4_SET(tpd, 1);
1290                 }
1291
1292                 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1293                         /* ipv6 tso need an extra tpd */
1294                         struct emac_tpd extra_tpd;
1295
1296                         memset(tpd, 0, sizeof(*tpd));
1297                         memset(&extra_tpd, 0, sizeof(extra_tpd));
1298
1299                         ipv6_hdr(skb)->payload_len = 0;
1300                         tcp_hdr(skb)->check =
1301                                 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1302                                                  &ipv6_hdr(skb)->daddr,
1303                                                  0, IPPROTO_TCP, 0);
1304                         TPD_PKT_LEN_SET(&extra_tpd, skb->len);
1305                         TPD_LSO_SET(&extra_tpd, 1);
1306                         TPD_LSOV_SET(&extra_tpd, 1);
1307                         emac_tx_tpd_create(adpt, tx_q, &extra_tpd);
1308                         TPD_LSOV_SET(tpd, 1);
1309                 }
1310
1311                 TPD_LSO_SET(tpd, 1);
1312                 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));
1313                 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);
1314                 return 0;
1315         }
1316
1317 do_csum:
1318         if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1319                 unsigned int css, cso;
1320
1321                 cso = skb_transport_offset(skb);
1322                 if (unlikely(cso & 0x1)) {
1323                         netdev_err(adpt->netdev,
1324                                    "error: payload offset should be even\n");
1325                         return -EINVAL;
1326                 }
1327                 css = cso + skb->csum_offset;
1328
1329                 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);
1330                 TPD_CXSUM_OFFSET_SET(tpd, css >> 1);
1331                 TPD_CSX_SET(tpd, 1);
1332         }
1333
1334         return 0;
1335 }
1336
1337 /* Fill up transmit descriptors */
1338 static void emac_tx_fill_tpd(struct emac_adapter *adpt,
1339                              struct emac_tx_queue *tx_q, struct sk_buff *skb,
1340                              struct emac_tpd *tpd)
1341 {
1342         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1343         unsigned int first = tx_q->tpd.produce_idx;
1344         unsigned int len = skb_headlen(skb);
1345         struct emac_buffer *tpbuf = NULL;
1346         unsigned int mapped_len = 0;
1347         unsigned int i;
1348         int count = 0;
1349         int ret;
1350
1351         /* if Large Segment Offload is (in TCP Segmentation Offload struct) */
1352         if (TPD_LSO(tpd)) {
1353                 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1354
1355                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1356                 tpbuf->length = mapped_len;
1357                 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1358                                                virt_to_page(skb->data),
1359                                                offset_in_page(skb->data),
1360                                                tpbuf->length,
1361                                                DMA_TO_DEVICE);
1362                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1363                                         tpbuf->dma_addr);
1364                 if (ret)
1365                         goto error;
1366
1367                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1368                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1369                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1370                 emac_tx_tpd_create(adpt, tx_q, tpd);
1371                 count++;
1372         }
1373
1374         if (mapped_len < len) {
1375                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1376                 tpbuf->length = len - mapped_len;
1377                 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1378                                                virt_to_page(skb->data +
1379                                                             mapped_len),
1380                                                offset_in_page(skb->data +
1381                                                               mapped_len),
1382                                                tpbuf->length, DMA_TO_DEVICE);
1383                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1384                                         tpbuf->dma_addr);
1385                 if (ret)
1386                         goto error;
1387
1388                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1389                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1390                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1391                 emac_tx_tpd_create(adpt, tx_q, tpd);
1392                 count++;
1393         }
1394
1395         for (i = 0; i < nr_frags; i++) {
1396                 struct skb_frag_struct *frag;
1397
1398                 frag = &skb_shinfo(skb)->frags[i];
1399
1400                 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1401                 tpbuf->length = frag->size;
1402                 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1403                                                frag->page.p, frag->page_offset,
1404                                                tpbuf->length, DMA_TO_DEVICE);
1405                 ret = dma_mapping_error(adpt->netdev->dev.parent,
1406                                         tpbuf->dma_addr);
1407                 if (ret)
1408                         goto error;
1409
1410                 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1411                 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1412                 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1413                 emac_tx_tpd_create(adpt, tx_q, tpd);
1414                 count++;
1415         }
1416
1417         /* The last tpd */
1418         wmb();
1419         emac_tx_tpd_mark_last(adpt, tx_q);
1420
1421         /* The last buffer info contain the skb address,
1422          * so it will be freed after unmap
1423          */
1424         tpbuf->skb = skb;
1425
1426         return;
1427
1428 error:
1429         /* One of the memory mappings failed, so undo everything */
1430         tx_q->tpd.produce_idx = first;
1431
1432         while (count--) {
1433                 tpbuf = GET_TPD_BUFFER(tx_q, first);
1434                 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,
1435                                tpbuf->length, DMA_TO_DEVICE);
1436                 tpbuf->dma_addr = 0;
1437                 tpbuf->length = 0;
1438
1439                 if (++first == tx_q->tpd.count)
1440                         first = 0;
1441         }
1442
1443         dev_kfree_skb(skb);
1444 }
1445
1446 /* Transmit the packet using specified transmit queue */
1447 int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
1448                          struct sk_buff *skb)
1449 {
1450         struct emac_tpd tpd;
1451         u32 prod_idx;
1452
1453         memset(&tpd, 0, sizeof(tpd));
1454
1455         if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {
1456                 dev_kfree_skb_any(skb);
1457                 return NETDEV_TX_OK;
1458         }
1459
1460         if (skb_vlan_tag_present(skb)) {
1461                 u16 tag;
1462
1463                 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);
1464                 TPD_CVLAN_TAG_SET(&tpd, tag);
1465                 TPD_INSTC_SET(&tpd, 1);
1466         }
1467
1468         if (skb_network_offset(skb) != ETH_HLEN)
1469                 TPD_TYP_SET(&tpd, 1);
1470
1471         emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);
1472
1473         netdev_sent_queue(adpt->netdev, skb->len);
1474
1475         /* Make sure the are enough free descriptors to hold one
1476          * maximum-sized SKB.  We need one desc for each fragment,
1477          * one for the checksum (emac_tso_csum), one for TSO, and
1478          * and one for the SKB header.
1479          */
1480         if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))
1481                 netif_stop_queue(adpt->netdev);
1482
1483         /* update produce idx */
1484         prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &
1485                     tx_q->produce_mask;
1486         emac_reg_update32(adpt->base + tx_q->produce_reg,
1487                           tx_q->produce_mask, prod_idx);
1488
1489         return NETDEV_TX_OK;
1490 }