drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 /*******************************************************************************
   2   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   3   ST Ethernet IPs are built around a Synopsys IP Core.
   4
   5         Copyright(C) 2007-2011 STMicroelectronics Ltd
   6
   7   This program is free software; you can redistribute it and/or modify it
   8   under the terms and conditions of the GNU General Public License,
   9   version 2, as published by the Free Software Foundation.
  10
  11   This program is distributed in the hope it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   The full GNU General Public License is included in this distribution in
  17   the file called "COPYING".
  18
  19   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  20
  21   Documentation available at:
  22         http://www.stlinux.com
  23   Support available at:
  24         https://bugzilla.stlinux.com/
  25 *******************************************************************************/
  26
  27 #include <linux/clk.h>
  28 #include <linux/kernel.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/ip.h>
  31 #include <linux/tcp.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/if_ether.h>
  35 #include <linux/crc32.h>
  36 #include <linux/mii.h>
  37 #include <linux/if.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/pinctrl/consumer.h>
  43 #ifdef CONFIG_DEBUG_FS
  44 #include <linux/debugfs.h>
  45 #include <linux/seq_file.h>
  46 #endif /* CONFIG_DEBUG_FS */
  47 #include <linux/net_tstamp.h>
  48 #include "stmmac_ptp.h"
  49 #include "stmmac.h"
  50 #include <linux/reset.h>
  51 #include <linux/of_mdio.h>
  52 #include "dwmac1000.h"
  53
  54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x)
  55 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  56
  57 /* Module parameters */
  58 #define TX_TIMEO        5000
  59 static int watchdog = TX_TIMEO;
  60 module_param(watchdog, int, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  62
  63 static int debug = -1;
  64 module_param(debug, int, S_IRUGO | S_IWUSR);
  65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  66
  67 static int phyaddr = -1;
  68 module_param(phyaddr, int, S_IRUGO);
  69 MODULE_PARM_DESC(phyaddr, "Physical device address");
  70
  71 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  72 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  73
  74 static int flow_ctrl = FLOW_OFF;
  75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR);
  76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  77
  78 static int pause = PAUSE_TIME;
  79 module_param(pause, int, S_IRUGO | S_IWUSR);
  80 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  81
  82 #define TC_DEFAULT 64
  83 static int tc = TC_DEFAULT;
  84 module_param(tc, int, S_IRUGO | S_IWUSR);
  85 MODULE_PARM_DESC(tc, "DMA threshold control value");
  86
  87 #define DEFAULT_BUFSIZE 1536
  88 static int buf_sz = DEFAULT_BUFSIZE;
  89 module_param(buf_sz, int, S_IRUGO | S_IWUSR);
  90 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  91
  92 #define STMMAC_RX_COPYBREAK     256
  93
  94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  95                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  96                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
  97
  98 #define STMMAC_DEFAULT_LPI_TIMER        1000
  99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR);
 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
 103
 104 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 105  * but allow user to force to use the chain instead of the ring
 106  */
 107 static unsigned int chain_mode;
 108 module_param(chain_mode, int, S_IRUGO);
 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 110
 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 112
 113 #ifdef CONFIG_DEBUG_FS
 114 static int stmmac_init_fs(struct net_device *dev);
 115 static void stmmac_exit_fs(struct net_device *dev);
 116 #endif
 117
 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 119
 120 /**
 121  * stmmac_verify_args - verify the driver parameters.
 122  * Description: it checks the driver parameters and set a default in case of
 123  * errors.
 124  */
 125 static void stmmac_verify_args(void)
 126 {
 127         if (unlikely(watchdog < 0))
 128                 watchdog = TX_TIMEO;
 129         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 130                 buf_sz = DEFAULT_BUFSIZE;
 131         if (unlikely(flow_ctrl > 1))
 132                 flow_ctrl = FLOW_AUTO;
 133         else if (likely(flow_ctrl < 0))
 134                 flow_ctrl = FLOW_OFF;
 135         if (unlikely((pause < 0) || (pause > 0xffff)))
 136                 pause = PAUSE_TIME;
 137         if (eee_timer < 0)
 138                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 139 }
 140
 141 /**
 142  * stmmac_disable_all_queues - Disable all queues
 143  * @priv: driver private structure
 144  */
 145 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
 146 {
 147         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 148         u32 queue;
 149
 150         for (queue = 0; queue < rx_queues_cnt; queue++) {
 151                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 152
 153                 napi_disable(&rx_q->napi);
 154         }
 155 }
 156
 157 /**
 158  * stmmac_enable_all_queues - Enable all queues
 159  * @priv: driver private structure
 160  */
 161 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
 162 {
 163         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 164         u32 queue;
 165
 166         for (queue = 0; queue < rx_queues_cnt; queue++) {
 167                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 168
 169                 napi_enable(&rx_q->napi);
 170         }
 171 }
 172
 173 /**
 174  * stmmac_stop_all_queues - Stop all queues
 175  * @priv: driver private structure
 176  */
 177 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
 178 {
 179         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 180         u32 queue;
 181
 182         for (queue = 0; queue < tx_queues_cnt; queue++)
 183                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
 184 }
 185
 186 /**
 187  * stmmac_start_all_queues - Start all queues
 188  * @priv: driver private structure
 189  */
 190 static void stmmac_start_all_queues(struct stmmac_priv *priv)
 191 {
 192         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 193         u32 queue;
 194
 195         for (queue = 0; queue < tx_queues_cnt; queue++)
 196                 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
 197 }
 198
 199 /**
 200  * stmmac_clk_csr_set - dynamically set the MDC clock
 201  * @priv: driver private structure
 202  * Description: this is to dynamically set the MDC clock according to the csr
 203  * clock input.
 204  * Note:
 205  *      If a specific clk_csr value is passed from the platform
 206  *      this means that the CSR Clock Range selection cannot be
 207  *      changed at run-time and it is fixed (as reported in the driver
 208  *      documentation). Viceversa the driver will try to set the MDC
 209  *      clock dynamically according to the actual clock input.
 210  */
 211 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 212 {
 213         u32 clk_rate;
 214
 215         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 216
 217         /* Platform provided default clk_csr would be assumed valid
 218          * for all other cases except for the below mentioned ones.
 219          * For values higher than the IEEE 802.3 specified frequency
 220          * we can not estimate the proper divider as it is not known
 221          * the frequency of clk_csr_i. So we do not change the default
 222          * divider.
 223          */
 224         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 225                 if (clk_rate < CSR_F_35M)
 226                         priv->clk_csr = STMMAC_CSR_20_35M;
 227                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 228                         priv->clk_csr = STMMAC_CSR_35_60M;
 229                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 230                         priv->clk_csr = STMMAC_CSR_60_100M;
 231                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 232                         priv->clk_csr = STMMAC_CSR_100_150M;
 233                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 234                         priv->clk_csr = STMMAC_CSR_150_250M;
 235                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 236                         priv->clk_csr = STMMAC_CSR_250_300M;
 237         }
 238
 239         if (priv->plat->has_sun8i) {
 240                 if (clk_rate > 160000000)
 241                         priv->clk_csr = 0x03;
 242                 else if (clk_rate > 80000000)
 243                         priv->clk_csr = 0x02;
 244                 else if (clk_rate > 40000000)
 245                         priv->clk_csr = 0x01;
 246                 else
 247                         priv->clk_csr = 0;
 248         }
 249 }
 250
 251 static void print_pkt(unsigned char *buf, int len)
 252 {
 253         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 254         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 255 }
 256
 257 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 258 {
 259         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 260         u32 avail;
 261
 262         if (tx_q->dirty_tx > tx_q->cur_tx)
 263                 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
 264         else
 265                 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
 266
 267         return avail;
 268 }
 269
 270 /**
 271  * stmmac_rx_dirty - Get RX queue dirty
 272  * @priv: driver private structure
 273  * @queue: RX queue index
 274  */
 275 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 276 {
 277         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 278         u32 dirty;
 279
 280         if (rx_q->dirty_rx <= rx_q->cur_rx)
 281                 dirty = rx_q->cur_rx - rx_q->dirty_rx;
 282         else
 283                 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
 284
 285         return dirty;
 286 }
 287
 288 /**
 289  * stmmac_hw_fix_mac_speed - callback for speed selection
 290  * @priv: driver private structure
 291  * Description: on some platforms (e.g. ST), some HW system configuration
 292  * registers have to be set according to the link speed negotiated.
 293  */
 294 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
 295 {
 296         struct net_device *ndev = priv->dev;
 297         struct phy_device *phydev = ndev->phydev;
 298
 299         if (likely(priv->plat->fix_mac_speed))
 300                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
 301 }
 302
 303 /**
 304  * stmmac_enable_eee_mode - check and enter in LPI mode
 305  * @priv: driver private structure
 306  * Description: this function is to verify and enter in LPI mode in case of
 307  * EEE.
 308  */
 309 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 310 {
 311         u32 tx_cnt = priv->plat->tx_queues_to_use;
 312         u32 queue;
 313
 314         /* check if all TX queues have the work finished */
 315         for (queue = 0; queue < tx_cnt; queue++) {
 316                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 317
 318                 if (tx_q->dirty_tx != tx_q->cur_tx)
 319                         return; /* still unfinished work */
 320         }
 321
 322         /* Check and enter in LPI mode */
 323         if (!priv->tx_path_in_lpi_mode)
 324                 priv->hw->mac->set_eee_mode(priv->hw,
 325                                             priv->plat->en_tx_lpi_clockgating);
 326 }
 327
 328 /**
 329  * stmmac_disable_eee_mode - disable and exit from LPI mode
 330  * @priv: driver private structure
 331  * Description: this function is to exit and disable EEE in case of
 332  * LPI state is true. This is called by the xmit.
 333  */
 334 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 335 {
 336         priv->hw->mac->reset_eee_mode(priv->hw);
 337         del_timer_sync(&priv->eee_ctrl_timer);
 338         priv->tx_path_in_lpi_mode = false;
 339 }
 340
 341 /**
 342  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 343  * @arg : data hook
 344  * Description:
 345  *  if there is no data transfer and if we are not in LPI state,
 346  *  then MAC Transmitter can be moved to LPI state.
 347  */
 348 static void stmmac_eee_ctrl_timer(struct timer_list *t)
 349 {
 350         struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
 351
 352         stmmac_enable_eee_mode(priv);
 353         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 354 }
 355
 356 /**
 357  * stmmac_eee_init - init EEE
 358  * @priv: driver private structure
 359  * Description:
 360  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 361  *  can also manage EEE, this function enable the LPI state and start related
 362  *  timer.
 363  */
 364 bool stmmac_eee_init(struct stmmac_priv *priv)
 365 {
 366         struct net_device *ndev = priv->dev;
 367         unsigned long flags;
 368         bool ret = false;
 369
 370         /* Using PCS we cannot dial with the phy registers at this stage
 371          * so we do not support extra feature like EEE.
 372          */
 373         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 374             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 375             (priv->hw->pcs == STMMAC_PCS_RTBI))
 376                 goto out;
 377
 378         /* MAC core supports the EEE feature. */
 379         if (priv->dma_cap.eee) {
 380                 int tx_lpi_timer = priv->tx_lpi_timer;
 381
 382                 /* Check if the PHY supports EEE */
 383                 if (phy_init_eee(ndev->phydev, 1)) {
 384                         /* To manage at run-time if the EEE cannot be supported
 385                          * anymore (for example because the lp caps have been
 386                          * changed).
 387                          * In that case the driver disable own timers.
 388                          */
 389                         spin_lock_irqsave(&priv->lock, flags);
 390                         if (priv->eee_active) {
 391                                 netdev_dbg(priv->dev, "disable EEE\n");
 392                                 del_timer_sync(&priv->eee_ctrl_timer);
 393                                 priv->hw->mac->set_eee_timer(priv->hw, 0,
 394                                                              tx_lpi_timer);
 395                         }
 396                         priv->eee_active = 0;
 397                         spin_unlock_irqrestore(&priv->lock, flags);
 398                         goto out;
 399                 }
 400                 /* Activate the EEE and start timers */
 401                 spin_lock_irqsave(&priv->lock, flags);
 402                 if (!priv->eee_active) {
 403                         priv->eee_active = 1;
 404                         timer_setup(&priv->eee_ctrl_timer,
 405                                     stmmac_eee_ctrl_timer, 0);
 406                         mod_timer(&priv->eee_ctrl_timer,
 407                                   STMMAC_LPI_T(eee_timer));
 408
 409                         priv->hw->mac->set_eee_timer(priv->hw,
 410                                                      STMMAC_DEFAULT_LIT_LS,
 411                                                      tx_lpi_timer);
 412                 }
 413                 /* Set HW EEE according to the speed */
 414                 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
 415
 416                 ret = true;
 417                 spin_unlock_irqrestore(&priv->lock, flags);
 418
 419                 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 420         }
 421 out:
 422         return ret;
 423 }
 424
 425 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 426  * @priv: driver private structure
 427  * @p : descriptor pointer
 428  * @skb : the socket buffer
 429  * Description :
 430  * This function will read timestamp from the descriptor & pass it to stack.
 431  * and also perform some sanity checks.
 432  */
 433 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 434                                    struct dma_desc *p, struct sk_buff *skb)
 435 {
 436         struct skb_shared_hwtstamps shhwtstamp;
 437         u64 ns;
 438
 439         if (!priv->hwts_tx_en)
 440                 return;
 441
 442         /* exit if skb doesn't support hw tstamp */
 443         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 444                 return;
 445
 446         /* check tx tstamp status */
 447         if (priv->hw->desc->get_tx_timestamp_status(p)) {
 448                 /* get the valid tstamp */
 449                 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 450
 451                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 452                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 453
 454                 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 455                 /* pass tstamp to stack */
 456                 skb_tstamp_tx(skb, &shhwtstamp);
 457         }
 458
 459         return;
 460 }
 461
 462 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 463  * @priv: driver private structure
 464  * @p : descriptor pointer
 465  * @np : next descriptor pointer
 466  * @skb : the socket buffer
 467  * Description :
 468  * This function will read received packet's timestamp from the descriptor
 469  * and pass it to stack. It also perform some sanity checks.
 470  */
 471 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 472                                    struct dma_desc *np, struct sk_buff *skb)
 473 {
 474         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 475         struct dma_desc *desc = p;
 476         u64 ns;
 477
 478         if (!priv->hwts_rx_en)
 479                 return;
 480         /* For GMAC4, the valid timestamp is from CTX next desc. */
 481         if (priv->plat->has_gmac4)
 482                 desc = np;
 483
 484         /* Check if timestamp is available */
 485         if (priv->hw->desc->get_rx_timestamp_status(p, np, priv->adv_ts)) {
 486                 ns = priv->hw->desc->get_timestamp(desc, priv->adv_ts);
 487                 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 488                 shhwtstamp = skb_hwtstamps(skb);
 489                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 490                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 491         } else  {
 492                 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
 493         }
 494 }
 495
 496 /**
 497  *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 498  *  @dev: device pointer.
 499  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 500  *  a proprietary structure used to pass information to the driver.
 501  *  Description:
 502  *  This function configures the MAC to enable/disable both outgoing(TX)
 503  *  and incoming(RX) packets time stamping based on user input.
 504  *  Return Value:
 505  *  0 on success and an appropriate -ve integer on failure.
 506  */
 507 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 508 {
 509         struct stmmac_priv *priv = netdev_priv(dev);
 510         struct hwtstamp_config config;
 511         struct timespec64 now;
 512         u64 temp = 0;
 513         u32 ptp_v2 = 0;
 514         u32 tstamp_all = 0;
 515         u32 ptp_over_ipv4_udp = 0;
 516         u32 ptp_over_ipv6_udp = 0;
 517         u32 ptp_over_ethernet = 0;
 518         u32 snap_type_sel = 0;
 519         u32 ts_master_en = 0;
 520         u32 ts_event_en = 0;
 521         u32 value = 0;
 522         u32 sec_inc;
 523
 524         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 525                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 526                 priv->hwts_tx_en = 0;
 527                 priv->hwts_rx_en = 0;
 528
 529                 return -EOPNOTSUPP;
 530         }
 531
 532         if (copy_from_user(&config, ifr->ifr_data,
 533                            sizeof(struct hwtstamp_config)))
 534                 return -EFAULT;
 535
 536         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 537                    __func__, config.flags, config.tx_type, config.rx_filter);
 538
 539         /* reserved for future extensions */
 540         if (config.flags)
 541                 return -EINVAL;
 542
 543         if (config.tx_type != HWTSTAMP_TX_OFF &&
 544             config.tx_type != HWTSTAMP_TX_ON)
 545                 return -ERANGE;
 546
 547         if (priv->adv_ts) {
 548                 switch (config.rx_filter) {
 549                 case HWTSTAMP_FILTER_NONE:
 550                         /* time stamp no incoming packet at all */
 551                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 552                         break;
 553
 554                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 555                         /* PTP v1, UDP, any kind of event packet */
 556                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 557                         /* take time stamp for all event messages */
 558                         if (priv->plat->has_gmac4)
 559                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 560                         else
 561                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 562
 563                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 564                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 565                         break;
 566
 567                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 568                         /* PTP v1, UDP, Sync packet */
 569                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 570                         /* take time stamp for SYNC messages only */
 571                         ts_event_en = PTP_TCR_TSEVNTENA;
 572
 573                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 574                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 575                         break;
 576
 577                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 578                         /* PTP v1, UDP, Delay_req packet */
 579                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 580                         /* take time stamp for Delay_Req messages only */
 581                         ts_master_en = PTP_TCR_TSMSTRENA;
 582                         ts_event_en = PTP_TCR_TSEVNTENA;
 583
 584                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 585                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 586                         break;
 587
 588                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 589                         /* PTP v2, UDP, any kind of event packet */
 590                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 591                         ptp_v2 = PTP_TCR_TSVER2ENA;
 592                         /* take time stamp for all event messages */
 593                         if (priv->plat->has_gmac4)
 594                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 595                         else
 596                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 597
 598                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 599                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 600                         break;
 601
 602                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 603                         /* PTP v2, UDP, Sync packet */
 604                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 605                         ptp_v2 = PTP_TCR_TSVER2ENA;
 606                         /* take time stamp for SYNC messages only */
 607                         ts_event_en = PTP_TCR_TSEVNTENA;
 608
 609                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 610                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 611                         break;
 612
 613                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 614                         /* PTP v2, UDP, Delay_req packet */
 615                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 616                         ptp_v2 = PTP_TCR_TSVER2ENA;
 617                         /* take time stamp for Delay_Req messages only */
 618                         ts_master_en = PTP_TCR_TSMSTRENA;
 619                         ts_event_en = PTP_TCR_TSEVNTENA;
 620
 621                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 622                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 623                         break;
 624
 625                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 626                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 627                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 628                         ptp_v2 = PTP_TCR_TSVER2ENA;
 629                         /* take time stamp for all event messages */
 630                         if (priv->plat->has_gmac4)
 631                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 632                         else
 633                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 634
 635                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 636                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 637                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 638                         break;
 639
 640                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 641                         /* PTP v2/802.AS1, any layer, Sync packet */
 642                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 643                         ptp_v2 = PTP_TCR_TSVER2ENA;
 644                         /* take time stamp for SYNC messages only */
 645                         ts_event_en = PTP_TCR_TSEVNTENA;
 646
 647                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 648                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 649                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 650                         break;
 651
 652                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 653                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 654                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 655                         ptp_v2 = PTP_TCR_TSVER2ENA;
 656                         /* take time stamp for Delay_Req messages only */
 657                         ts_master_en = PTP_TCR_TSMSTRENA;
 658                         ts_event_en = PTP_TCR_TSEVNTENA;
 659
 660                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 661                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 662                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 663                         break;
 664
 665                 case HWTSTAMP_FILTER_NTP_ALL:
 666                 case HWTSTAMP_FILTER_ALL:
 667                         /* time stamp any incoming packet */
 668                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 669                         tstamp_all = PTP_TCR_TSENALL;
 670                         break;
 671
 672                 default:
 673                         return -ERANGE;
 674                 }
 675         } else {
 676                 switch (config.rx_filter) {
 677                 case HWTSTAMP_FILTER_NONE:
 678                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 679                         break;
 680                 default:
 681                         /* PTP v1, UDP, any kind of event packet */
 682                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 683                         break;
 684                 }
 685         }
 686         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 687         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 688
 689         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 690                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
 691         else {
 692                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 693                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 694                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 695                          ts_master_en | snap_type_sel);
 696                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
 697
 698                 /* program Sub Second Increment reg */
 699                 sec_inc = priv->hw->ptp->config_sub_second_increment(
 700                         priv->ptpaddr, priv->plat->clk_ptp_rate,
 701                         priv->plat->has_gmac4);
 702                 temp = div_u64(1000000000ULL, sec_inc);
 703
 704                 /* calculate default added value:
 705                  * formula is :
 706                  * addend = (2^32)/freq_div_ratio;
 707                  * where, freq_div_ratio = 1e9ns/sec_inc
 708                  */
 709                 temp = (u64)(temp << 32);
 710                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 711                 priv->hw->ptp->config_addend(priv->ptpaddr,
 712                                              priv->default_addend);
 713
 714                 /* initialize system time */
 715                 ktime_get_real_ts64(&now);
 716
 717                 /* lower 32 bits of tv_sec are safe until y2106 */
 718                 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
 719                                             now.tv_nsec);
 720         }
 721
 722         return copy_to_user(ifr->ifr_data, &config,
 723                             sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
 724 }
 725
 726 /**
 727  * stmmac_init_ptp - init PTP
 728  * @priv: driver private structure
 729  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 730  * This is done by looking at the HW cap. register.
 731  * This function also registers the ptp driver.
 732  */
 733 static int stmmac_init_ptp(struct stmmac_priv *priv)
 734 {
 735         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 736                 return -EOPNOTSUPP;
 737
 738         priv->adv_ts = 0;
 739         /* Check if adv_ts can be enabled for dwmac 4.x core */
 740         if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
 741                 priv->adv_ts = 1;
 742         /* Dwmac 3.x core with extend_desc can support adv_ts */
 743         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 744                 priv->adv_ts = 1;
 745
 746         if (priv->dma_cap.time_stamp)
 747                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 748
 749         if (priv->adv_ts)
 750                 netdev_info(priv->dev,
 751                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 752
 753         priv->hw->ptp = &stmmac_ptp;
 754         priv->hwts_tx_en = 0;
 755         priv->hwts_rx_en = 0;
 756
 757         stmmac_ptp_register(priv);
 758
 759         return 0;
 760 }
 761
 762 static void stmmac_release_ptp(struct stmmac_priv *priv)
 763 {
 764         if (priv->plat->clk_ptp_ref)
 765                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 766         stmmac_ptp_unregister(priv);
 767 }
 768
 769 /**
 770  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 771  *  @priv: driver private structure
 772  *  Description: It is used for configuring the flow control in all queues
 773  */
 774 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 775 {
 776         u32 tx_cnt = priv->plat->tx_queues_to_use;
 777
 778         priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl,
 779                                  priv->pause, tx_cnt);
 780 }
 781
 782 /**
 783  * stmmac_adjust_link - adjusts the link parameters
 784  * @dev: net device structure
 785  * Description: this is the helper called by the physical abstraction layer
 786  * drivers to communicate the phy link status. According the speed and duplex
 787  * this driver can invoke registered glue-logic as well.
 788  * It also invoke the eee initialization because it could happen when switch
 789  * on different networks (that are eee capable).
 790  */
 791 static void stmmac_adjust_link(struct net_device *dev)
 792 {
 793         struct stmmac_priv *priv = netdev_priv(dev);
 794         struct phy_device *phydev = dev->phydev;
 795         unsigned long flags;
 796         bool new_state = false;
 797
 798         if (!phydev)
 799                 return;
 800
 801         spin_lock_irqsave(&priv->lock, flags);
 802
 803         if (phydev->link) {
 804                 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 805
 806                 /* Now we make sure that we can be in full duplex mode.
 807                  * If not, we operate in half-duplex mode. */
 808                 if (phydev->duplex != priv->oldduplex) {
 809                         new_state = true;
 810                         if (!phydev->duplex)
 811                                 ctrl &= ~priv->hw->link.duplex;
 812                         else
 813                                 ctrl |= priv->hw->link.duplex;
 814                         priv->oldduplex = phydev->duplex;
 815                 }
 816                 /* Flow Control operation */
 817                 if (phydev->pause)
 818                         stmmac_mac_flow_ctrl(priv, phydev->duplex);
 819
 820                 if (phydev->speed != priv->speed) {
 821                         new_state = true;
 822                         ctrl &= ~priv->hw->link.speed_mask;
 823                         switch (phydev->speed) {
 824                         case SPEED_1000:
 825                                 ctrl |= priv->hw->link.speed1000;
 826                                 break;
 827                         case SPEED_100:
 828                                 ctrl |= priv->hw->link.speed100;
 829                                 break;
 830                         case SPEED_10:
 831                                 ctrl |= priv->hw->link.speed10;
 832                                 break;
 833                         default:
 834                                 netif_warn(priv, link, priv->dev,
 835                                            "broken speed: %d\n", phydev->speed);
 836                                 phydev->speed = SPEED_UNKNOWN;
 837                                 break;
 838                         }
 839                         if (phydev->speed != SPEED_UNKNOWN)
 840                                 stmmac_hw_fix_mac_speed(priv);
 841                         priv->speed = phydev->speed;
 842                 }
 843
 844                 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 845
 846                 if (!priv->oldlink) {
 847                         new_state = true;
 848                         priv->oldlink = true;
 849                 }
 850         } else if (priv->oldlink) {
 851                 new_state = true;
 852                 priv->oldlink = false;
 853                 priv->speed = SPEED_UNKNOWN;
 854                 priv->oldduplex = DUPLEX_UNKNOWN;
 855         }
 856
 857         if (new_state && netif_msg_link(priv))
 858                 phy_print_status(phydev);
 859
 860         spin_unlock_irqrestore(&priv->lock, flags);
 861
 862         if (phydev->is_pseudo_fixed_link)
 863                 /* Stop PHY layer to call the hook to adjust the link in case
 864                  * of a switch is attached to the stmmac driver.
 865                  */
 866                 phydev->irq = PHY_IGNORE_INTERRUPT;
 867         else
 868                 /* At this stage, init the EEE if supported.
 869                  * Never called in case of fixed_link.
 870                  */
 871                 priv->eee_enabled = stmmac_eee_init(priv);
 872 }
 873
 874 /**
 875  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 876  * @priv: driver private structure
 877  * Description: this is to verify if the HW supports the PCS.
 878  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 879  * configured for the TBI, RTBI, or SGMII PHY interface.
 880  */
 881 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 882 {
 883         int interface = priv->plat->interface;
 884
 885         if (priv->dma_cap.pcs) {
 886                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 887                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 888                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 889                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 890                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 891                         priv->hw->pcs = STMMAC_PCS_RGMII;
 892                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 893                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 894                         priv->hw->pcs = STMMAC_PCS_SGMII;
 895                 }
 896         }
 897 }
 898
 899 /**
 900  * stmmac_init_phy - PHY initialization
 901  * @dev: net device structure
 902  * Description: it initializes the driver's PHY state, and attaches the PHY
 903  * to the mac driver.
 904  *  Return value:
 905  *  0 on success
 906  */
 907 static int stmmac_init_phy(struct net_device *dev)
 908 {
 909         struct stmmac_priv *priv = netdev_priv(dev);
 910         struct phy_device *phydev;
 911         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
 912         char bus_id[MII_BUS_ID_SIZE];
 913         int interface = priv->plat->interface;
 914         int max_speed = priv->plat->max_speed;
 915         priv->oldlink = false;
 916         priv->speed = SPEED_UNKNOWN;
 917         priv->oldduplex = DUPLEX_UNKNOWN;
 918
 919         if (priv->plat->phy_node) {
 920                 phydev = of_phy_connect(dev, priv->plat->phy_node,
 921                                         &stmmac_adjust_link, 0, interface);
 922         } else {
 923                 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
 924                          priv->plat->bus_id);
 925
 926                 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
 927                          priv->plat->phy_addr);
 928                 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
 929                            phy_id_fmt);
 930
 931                 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
 932                                      interface);
 933         }
 934
 935         if (IS_ERR_OR_NULL(phydev)) {
 936                 netdev_err(priv->dev, "Could not attach to PHY\n");
 937                 if (!phydev)
 938                         return -ENODEV;
 939
 940                 return PTR_ERR(phydev);
 941         }
 942
 943         /* Stop Advertising 1000BASE Capability if interface is not GMII */
 944         if ((interface == PHY_INTERFACE_MODE_MII) ||
 945             (interface == PHY_INTERFACE_MODE_RMII) ||
 946                 (max_speed < 1000 && max_speed > 0))
 947                 phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
 948                                          SUPPORTED_1000baseT_Full);
 949
 950         /*
 951          * Broken HW is sometimes missing the pull-up resistor on the
 952          * MDIO line, which results in reads to non-existent devices returning
 953          * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
 954          * device as well.
 955          * Note: phydev->phy_id is the result of reading the UID PHY registers.
 956          */
 957         if (!priv->plat->phy_node && phydev->phy_id == 0) {
 958                 phy_disconnect(phydev);
 959                 return -ENODEV;
 960         }
 961
 962         /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
 963          * subsequent PHY polling, make sure we force a link transition if
 964          * we have a UP/DOWN/UP transition
 965          */
 966         if (phydev->is_pseudo_fixed_link)
 967                 phydev->irq = PHY_POLL;
 968
 969         phy_attached_info(phydev);
 970         return 0;
 971 }
 972
 973 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
 974 {
 975         u32 rx_cnt = priv->plat->rx_queues_to_use;
 976         void *head_rx;
 977         u32 queue;
 978
 979         /* Display RX rings */
 980         for (queue = 0; queue < rx_cnt; queue++) {
 981                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 982
 983                 pr_info("\tRX Queue %u rings\n", queue);
 984
 985                 if (priv->extend_desc)
 986                         head_rx = (void *)rx_q->dma_erx;
 987                 else
 988                         head_rx = (void *)rx_q->dma_rx;
 989
 990                 /* Display RX ring */
 991                 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
 992         }
 993 }
 994
 995 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
 996 {
 997         u32 tx_cnt = priv->plat->tx_queues_to_use;
 998         void *head_tx;
 999         u32 queue;
1000
1001         /* Display TX rings */
1002         for (queue = 0; queue < tx_cnt; queue++) {
1003                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1004
1005                 pr_info("\tTX Queue %d rings\n", queue);
1006
1007                 if (priv->extend_desc)
1008                         head_tx = (void *)tx_q->dma_etx;
1009                 else
1010                         head_tx = (void *)tx_q->dma_tx;
1011
1012                 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
1013         }
1014 }
1015
1016 static void stmmac_display_rings(struct stmmac_priv *priv)
1017 {
1018         /* Display RX ring */
1019         stmmac_display_rx_rings(priv);
1020
1021         /* Display TX ring */
1022         stmmac_display_tx_rings(priv);
1023 }
1024
1025 static int stmmac_set_bfsize(int mtu, int bufsize)
1026 {
1027         int ret = bufsize;
1028
1029         if (mtu >= BUF_SIZE_4KiB)
1030                 ret = BUF_SIZE_8KiB;
1031         else if (mtu >= BUF_SIZE_2KiB)
1032                 ret = BUF_SIZE_4KiB;
1033         else if (mtu > DEFAULT_BUFSIZE)
1034                 ret = BUF_SIZE_2KiB;
1035         else
1036                 ret = DEFAULT_BUFSIZE;
1037
1038         return ret;
1039 }
1040
1041 /**
1042  * stmmac_clear_rx_descriptors - clear RX descriptors
1043  * @priv: driver private structure
1044  * @queue: RX queue index
1045  * Description: this function is called to clear the RX descriptors
1046  * in case of both basic and extended descriptors are used.
1047  */
1048 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1049 {
1050         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1051         int i;
1052
1053         /* Clear the RX descriptors */
1054         for (i = 0; i < DMA_RX_SIZE; i++)
1055                 if (priv->extend_desc)
1056                         priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic,
1057                                                      priv->use_riwt, priv->mode,
1058                                                      (i == DMA_RX_SIZE - 1));
1059                 else
1060                         priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i],
1061                                                      priv->use_riwt, priv->mode,
1062                                                      (i == DMA_RX_SIZE - 1));
1063 }
1064
1065 /**
1066  * stmmac_clear_tx_descriptors - clear tx descriptors
1067  * @priv: driver private structure
1068  * @queue: TX queue index.
1069  * Description: this function is called to clear the TX descriptors
1070  * in case of both basic and extended descriptors are used.
1071  */
1072 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1073 {
1074         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1075         int i;
1076
1077         /* Clear the TX descriptors */
1078         for (i = 0; i < DMA_TX_SIZE; i++)
1079                 if (priv->extend_desc)
1080                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1081                                                      priv->mode,
1082                                                      (i == DMA_TX_SIZE - 1));
1083                 else
1084                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1085                                                      priv->mode,
1086                                                      (i == DMA_TX_SIZE - 1));
1087 }
1088
1089 /**
1090  * stmmac_clear_descriptors - clear descriptors
1091  * @priv: driver private structure
1092  * Description: this function is called to clear the TX and RX descriptors
1093  * in case of both basic and extended descriptors are used.
1094  */
1095 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1096 {
1097         u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1098         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1099         u32 queue;
1100
1101         /* Clear the RX descriptors */
1102         for (queue = 0; queue < rx_queue_cnt; queue++)
1103                 stmmac_clear_rx_descriptors(priv, queue);
1104
1105         /* Clear the TX descriptors */
1106         for (queue = 0; queue < tx_queue_cnt; queue++)
1107                 stmmac_clear_tx_descriptors(priv, queue);
1108 }
1109
1110 /**
1111  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1112  * @priv: driver private structure
1113  * @p: descriptor pointer
1114  * @i: descriptor index
1115  * @flags: gfp flag
1116  * @queue: RX queue index
1117  * Description: this function is called to allocate a receive buffer, perform
1118  * the DMA mapping and init the descriptor.
1119  */
1120 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1121                                   int i, gfp_t flags, u32 queue)
1122 {
1123         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1124         struct sk_buff *skb;
1125
1126         skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
1127         if (!skb) {
1128                 netdev_err(priv->dev,
1129                            "%s: Rx init fails; skb is NULL\n", __func__);
1130                 return -ENOMEM;
1131         }
1132         rx_q->rx_skbuff[i] = skb;
1133         rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
1134                                                 priv->dma_buf_sz,
1135                                                 DMA_FROM_DEVICE);
1136         if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) {
1137                 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
1138                 dev_kfree_skb_any(skb);
1139                 return -EINVAL;
1140         }
1141
1142         if (priv->synopsys_id >= DWMAC_CORE_4_00)
1143                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1144         else
1145                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1146
1147         if ((priv->hw->mode->init_desc3) &&
1148             (priv->dma_buf_sz == BUF_SIZE_16KiB))
1149                 priv->hw->mode->init_desc3(p);
1150
1151         return 0;
1152 }
1153
1154 /**
1155  * stmmac_free_rx_buffer - free RX dma buffers
1156  * @priv: private structure
1157  * @queue: RX queue index
1158  * @i: buffer index.
1159  */
1160 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1161 {
1162         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1163
1164         if (rx_q->rx_skbuff[i]) {
1165                 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i],
1166                                  priv->dma_buf_sz, DMA_FROM_DEVICE);
1167                 dev_kfree_skb_any(rx_q->rx_skbuff[i]);
1168         }
1169         rx_q->rx_skbuff[i] = NULL;
1170 }
1171
1172 /**
1173  * stmmac_free_tx_buffer - free RX dma buffers
1174  * @priv: private structure
1175  * @queue: RX queue index
1176  * @i: buffer index.
1177  */
1178 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1179 {
1180         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1181
1182         if (tx_q->tx_skbuff_dma[i].buf) {
1183                 if (tx_q->tx_skbuff_dma[i].map_as_page)
1184                         dma_unmap_page(priv->device,
1185                                        tx_q->tx_skbuff_dma[i].buf,
1186                                        tx_q->tx_skbuff_dma[i].len,
1187                                        DMA_TO_DEVICE);
1188                 else
1189                         dma_unmap_single(priv->device,
1190                                          tx_q->tx_skbuff_dma[i].buf,
1191                                          tx_q->tx_skbuff_dma[i].len,
1192                                          DMA_TO_DEVICE);
1193         }
1194
1195         if (tx_q->tx_skbuff[i]) {
1196                 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1197                 tx_q->tx_skbuff[i] = NULL;
1198                 tx_q->tx_skbuff_dma[i].buf = 0;
1199                 tx_q->tx_skbuff_dma[i].map_as_page = false;
1200         }
1201 }
1202
1203 /**
1204  * init_dma_rx_desc_rings - init the RX descriptor rings
1205  * @dev: net device structure
1206  * @flags: gfp flag.
1207  * Description: this function initializes the DMA RX descriptors
1208  * and allocates the socket buffers. It supports the chained and ring
1209  * modes.
1210  */
1211 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1212 {
1213         struct stmmac_priv *priv = netdev_priv(dev);
1214         u32 rx_count = priv->plat->rx_queues_to_use;
1215         unsigned int bfsize = 0;
1216         int ret = -ENOMEM;
1217         int queue;
1218         int i;
1219
1220         if (priv->hw->mode->set_16kib_bfsize)
1221                 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1222
1223         if (bfsize < BUF_SIZE_16KiB)
1224                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1225
1226         priv->dma_buf_sz = bfsize;
1227
1228         /* RX INITIALIZATION */
1229         netif_dbg(priv, probe, priv->dev,
1230                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1231
1232         for (queue = 0; queue < rx_count; queue++) {
1233                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1234
1235                 netif_dbg(priv, probe, priv->dev,
1236                           "(%s) dma_rx_phy=0x%08x\n", __func__,
1237                           (u32)rx_q->dma_rx_phy);
1238
1239                 for (i = 0; i < DMA_RX_SIZE; i++) {
1240                         struct dma_desc *p;
1241
1242                         if (priv->extend_desc)
1243                                 p = &((rx_q->dma_erx + i)->basic);
1244                         else
1245                                 p = rx_q->dma_rx + i;
1246
1247                         ret = stmmac_init_rx_buffers(priv, p, i, flags,
1248                                                      queue);
1249                         if (ret)
1250                                 goto err_init_rx_buffers;
1251
1252                         netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
1253                                   rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data,
1254                                   (unsigned int)rx_q->rx_skbuff_dma[i]);
1255                 }
1256
1257                 rx_q->cur_rx = 0;
1258                 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1259
1260                 stmmac_clear_rx_descriptors(priv, queue);
1261
1262                 /* Setup the chained descriptor addresses */
1263                 if (priv->mode == STMMAC_CHAIN_MODE) {
1264                         if (priv->extend_desc)
1265                                 priv->hw->mode->init(rx_q->dma_erx,
1266                                                      rx_q->dma_rx_phy,
1267                                                      DMA_RX_SIZE, 1);
1268                         else
1269                                 priv->hw->mode->init(rx_q->dma_rx,
1270                                                      rx_q->dma_rx_phy,
1271                                                      DMA_RX_SIZE, 0);
1272                 }
1273         }
1274
1275         buf_sz = bfsize;
1276
1277         return 0;
1278
1279 err_init_rx_buffers:
1280         while (queue >= 0) {
1281                 while (--i >= 0)
1282                         stmmac_free_rx_buffer(priv, queue, i);
1283
1284                 if (queue == 0)
1285                         break;
1286
1287                 i = DMA_RX_SIZE;
1288                 queue--;
1289         }
1290
1291         return ret;
1292 }
1293
1294 /**
1295  * init_dma_tx_desc_rings - init the TX descriptor rings
1296  * @dev: net device structure.
1297  * Description: this function initializes the DMA TX descriptors
1298  * and allocates the socket buffers. It supports the chained and ring
1299  * modes.
1300  */
1301 static int init_dma_tx_desc_rings(struct net_device *dev)
1302 {
1303         struct stmmac_priv *priv = netdev_priv(dev);
1304         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1305         u32 queue;
1306         int i;
1307
1308         for (queue = 0; queue < tx_queue_cnt; queue++) {
1309                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1310
1311                 netif_dbg(priv, probe, priv->dev,
1312                           "(%s) dma_tx_phy=0x%08x\n", __func__,
1313                          (u32)tx_q->dma_tx_phy);
1314
1315                 /* Setup the chained descriptor addresses */
1316                 if (priv->mode == STMMAC_CHAIN_MODE) {
1317                         if (priv->extend_desc)
1318                                 priv->hw->mode->init(tx_q->dma_etx,
1319                                                      tx_q->dma_tx_phy,
1320                                                      DMA_TX_SIZE, 1);
1321                         else
1322                                 priv->hw->mode->init(tx_q->dma_tx,
1323                                                      tx_q->dma_tx_phy,
1324                                                      DMA_TX_SIZE, 0);
1325                 }
1326
1327                 for (i = 0; i < DMA_TX_SIZE; i++) {
1328                         struct dma_desc *p;
1329                         if (priv->extend_desc)
1330                                 p = &((tx_q->dma_etx + i)->basic);
1331                         else
1332                                 p = tx_q->dma_tx + i;
1333
1334                         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1335                                 p->des0 = 0;
1336                                 p->des1 = 0;
1337                                 p->des2 = 0;
1338                                 p->des3 = 0;
1339                         } else {
1340                                 p->des2 = 0;
1341                         }
1342
1343                         tx_q->tx_skbuff_dma[i].buf = 0;
1344                         tx_q->tx_skbuff_dma[i].map_as_page = false;
1345                         tx_q->tx_skbuff_dma[i].len = 0;
1346                         tx_q->tx_skbuff_dma[i].last_segment = false;
1347                         tx_q->tx_skbuff[i] = NULL;
1348                 }
1349
1350                 tx_q->dirty_tx = 0;
1351                 tx_q->cur_tx = 0;
1352
1353                 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1354         }
1355
1356         return 0;
1357 }
1358
1359 /**
1360  * init_dma_desc_rings - init the RX/TX descriptor rings
1361  * @dev: net device structure
1362  * @flags: gfp flag.
1363  * Description: this function initializes the DMA RX/TX descriptors
1364  * and allocates the socket buffers. It supports the chained and ring
1365  * modes.
1366  */
1367 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1368 {
1369         struct stmmac_priv *priv = netdev_priv(dev);
1370         int ret;
1371
1372         ret = init_dma_rx_desc_rings(dev, flags);
1373         if (ret)
1374                 return ret;
1375
1376         ret = init_dma_tx_desc_rings(dev);
1377
1378         stmmac_clear_descriptors(priv);
1379
1380         if (netif_msg_hw(priv))
1381                 stmmac_display_rings(priv);
1382
1383         return ret;
1384 }
1385
1386 /**
1387  * dma_free_rx_skbufs - free RX dma buffers
1388  * @priv: private structure
1389  * @queue: RX queue index
1390  */
1391 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1392 {
1393         int i;
1394
1395         for (i = 0; i < DMA_RX_SIZE; i++)
1396                 stmmac_free_rx_buffer(priv, queue, i);
1397 }
1398
1399 /**
1400  * dma_free_tx_skbufs - free TX dma buffers
1401  * @priv: private structure
1402  * @queue: TX queue index
1403  */
1404 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1405 {
1406         int i;
1407
1408         for (i = 0; i < DMA_TX_SIZE; i++)
1409                 stmmac_free_tx_buffer(priv, queue, i);
1410 }
1411
1412 /**
1413  * free_dma_rx_desc_resources - free RX dma desc resources
1414  * @priv: private structure
1415  */
1416 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1417 {
1418         u32 rx_count = priv->plat->rx_queues_to_use;
1419         u32 queue;
1420
1421         /* Free RX queue resources */
1422         for (queue = 0; queue < rx_count; queue++) {
1423                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1424
1425                 /* Release the DMA RX socket buffers */
1426                 dma_free_rx_skbufs(priv, queue);
1427
1428                 /* Free DMA regions of consistent memory previously allocated */
1429                 if (!priv->extend_desc)
1430                         dma_free_coherent(priv->device,
1431                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1432                                           rx_q->dma_rx, rx_q->dma_rx_phy);
1433                 else
1434                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1435                                           sizeof(struct dma_extended_desc),
1436                                           rx_q->dma_erx, rx_q->dma_rx_phy);
1437
1438                 kfree(rx_q->rx_skbuff_dma);
1439                 kfree(rx_q->rx_skbuff);
1440         }
1441 }
1442
1443 /**
1444  * free_dma_tx_desc_resources - free TX dma desc resources
1445  * @priv: private structure
1446  */
1447 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1448 {
1449         u32 tx_count = priv->plat->tx_queues_to_use;
1450         u32 queue;
1451
1452         /* Free TX queue resources */
1453         for (queue = 0; queue < tx_count; queue++) {
1454                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1455
1456                 /* Release the DMA TX socket buffers */
1457                 dma_free_tx_skbufs(priv, queue);
1458
1459                 /* Free DMA regions of consistent memory previously allocated */
1460                 if (!priv->extend_desc)
1461                         dma_free_coherent(priv->device,
1462                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1463                                           tx_q->dma_tx, tx_q->dma_tx_phy);
1464                 else
1465                         dma_free_coherent(priv->device, DMA_TX_SIZE *
1466                                           sizeof(struct dma_extended_desc),
1467                                           tx_q->dma_etx, tx_q->dma_tx_phy);
1468
1469                 kfree(tx_q->tx_skbuff_dma);
1470                 kfree(tx_q->tx_skbuff);
1471         }
1472 }
1473
1474 /**
1475  * alloc_dma_rx_desc_resources - alloc RX resources.
1476  * @priv: private structure
1477  * Description: according to which descriptor can be used (extend or basic)
1478  * this function allocates the resources for TX and RX paths. In case of
1479  * reception, for example, it pre-allocated the RX socket buffer in order to
1480  * allow zero-copy mechanism.
1481  */
1482 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1483 {
1484         u32 rx_count = priv->plat->rx_queues_to_use;
1485         int ret = -ENOMEM;
1486         u32 queue;
1487
1488         /* RX queues buffers and DMA */
1489         for (queue = 0; queue < rx_count; queue++) {
1490                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1491
1492                 rx_q->queue_index = queue;
1493                 rx_q->priv_data = priv;
1494
1495                 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE,
1496                                                     sizeof(dma_addr_t),
1497                                                     GFP_KERNEL);
1498                 if (!rx_q->rx_skbuff_dma)
1499                         goto err_dma;
1500
1501                 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE,
1502                                                 sizeof(struct sk_buff *),
1503                                                 GFP_KERNEL);
1504                 if (!rx_q->rx_skbuff)
1505                         goto err_dma;
1506
1507                 if (priv->extend_desc) {
1508                         rx_q->dma_erx = dma_zalloc_coherent(priv->device,
1509                                                             DMA_RX_SIZE *
1510                                                             sizeof(struct
1511                                                             dma_extended_desc),
1512                                                             &rx_q->dma_rx_phy,
1513                                                             GFP_KERNEL);
1514                         if (!rx_q->dma_erx)
1515                                 goto err_dma;
1516
1517                 } else {
1518                         rx_q->dma_rx = dma_zalloc_coherent(priv->device,
1519                                                            DMA_RX_SIZE *
1520                                                            sizeof(struct
1521                                                            dma_desc),
1522                                                            &rx_q->dma_rx_phy,
1523                                                            GFP_KERNEL);
1524                         if (!rx_q->dma_rx)
1525                                 goto err_dma;
1526                 }
1527         }
1528
1529         return 0;
1530
1531 err_dma:
1532         free_dma_rx_desc_resources(priv);
1533
1534         return ret;
1535 }
1536
1537 /**
1538  * alloc_dma_tx_desc_resources - alloc TX resources.
1539  * @priv: private structure
1540  * Description: according to which descriptor can be used (extend or basic)
1541  * this function allocates the resources for TX and RX paths. In case of
1542  * reception, for example, it pre-allocated the RX socket buffer in order to
1543  * allow zero-copy mechanism.
1544  */
1545 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1546 {
1547         u32 tx_count = priv->plat->tx_queues_to_use;
1548         int ret = -ENOMEM;
1549         u32 queue;
1550
1551         /* TX queues buffers and DMA */
1552         for (queue = 0; queue < tx_count; queue++) {
1553                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1554
1555                 tx_q->queue_index = queue;
1556                 tx_q->priv_data = priv;
1557
1558                 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
1559                                                     sizeof(*tx_q->tx_skbuff_dma),
1560                                                     GFP_KERNEL);
1561                 if (!tx_q->tx_skbuff_dma)
1562                         goto err_dma;
1563
1564                 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE,
1565                                                 sizeof(struct sk_buff *),
1566                                                 GFP_KERNEL);
1567                 if (!tx_q->tx_skbuff)
1568                         goto err_dma;
1569
1570                 if (priv->extend_desc) {
1571                         tx_q->dma_etx = dma_zalloc_coherent(priv->device,
1572                                                             DMA_TX_SIZE *
1573                                                             sizeof(struct
1574                                                             dma_extended_desc),
1575                                                             &tx_q->dma_tx_phy,
1576                                                             GFP_KERNEL);
1577                         if (!tx_q->dma_etx)
1578                                 goto err_dma;
1579                 } else {
1580                         tx_q->dma_tx = dma_zalloc_coherent(priv->device,
1581                                                            DMA_TX_SIZE *
1582                                                            sizeof(struct
1583                                                                   dma_desc),
1584                                                            &tx_q->dma_tx_phy,
1585                                                            GFP_KERNEL);
1586                         if (!tx_q->dma_tx)
1587                                 goto err_dma;
1588                 }
1589         }
1590
1591         return 0;
1592
1593 err_dma:
1594         free_dma_tx_desc_resources(priv);
1595
1596         return ret;
1597 }
1598
1599 /**
1600  * alloc_dma_desc_resources - alloc TX/RX resources.
1601  * @priv: private structure
1602  * Description: according to which descriptor can be used (extend or basic)
1603  * this function allocates the resources for TX and RX paths. In case of
1604  * reception, for example, it pre-allocated the RX socket buffer in order to
1605  * allow zero-copy mechanism.
1606  */
1607 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1608 {
1609         /* RX Allocation */
1610         int ret = alloc_dma_rx_desc_resources(priv);
1611
1612         if (ret)
1613                 return ret;
1614
1615         ret = alloc_dma_tx_desc_resources(priv);
1616
1617         return ret;
1618 }
1619
1620 /**
1621  * free_dma_desc_resources - free dma desc resources
1622  * @priv: private structure
1623  */
1624 static void free_dma_desc_resources(struct stmmac_priv *priv)
1625 {
1626         /* Release the DMA RX socket buffers */
1627         free_dma_rx_desc_resources(priv);
1628
1629         /* Release the DMA TX socket buffers */
1630         free_dma_tx_desc_resources(priv);
1631 }
1632
1633 /**
1634  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1635  *  @priv: driver private structure
1636  *  Description: It is used for enabling the rx queues in the MAC
1637  */
1638 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1639 {
1640         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1641         int queue;
1642         u8 mode;
1643
1644         for (queue = 0; queue < rx_queues_count; queue++) {
1645                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1646                 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue);
1647         }
1648 }
1649
1650 /**
1651  * stmmac_start_rx_dma - start RX DMA channel
1652  * @priv: driver private structure
1653  * @chan: RX channel index
1654  * Description:
1655  * This starts a RX DMA channel
1656  */
1657 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1658 {
1659         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1660         priv->hw->dma->start_rx(priv->ioaddr, chan);
1661 }
1662
1663 /**
1664  * stmmac_start_tx_dma - start TX DMA channel
1665  * @priv: driver private structure
1666  * @chan: TX channel index
1667  * Description:
1668  * This starts a TX DMA channel
1669  */
1670 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1671 {
1672         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1673         priv->hw->dma->start_tx(priv->ioaddr, chan);
1674 }
1675
1676 /**
1677  * stmmac_stop_rx_dma - stop RX DMA channel
1678  * @priv: driver private structure
1679  * @chan: RX channel index
1680  * Description:
1681  * This stops a RX DMA channel
1682  */
1683 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1684 {
1685         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1686         priv->hw->dma->stop_rx(priv->ioaddr, chan);
1687 }
1688
1689 /**
1690  * stmmac_stop_tx_dma - stop TX DMA channel
1691  * @priv: driver private structure
1692  * @chan: TX channel index
1693  * Description:
1694  * This stops a TX DMA channel
1695  */
1696 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1697 {
1698         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1699         priv->hw->dma->stop_tx(priv->ioaddr, chan);
1700 }
1701
1702 /**
1703  * stmmac_start_all_dma - start all RX and TX DMA channels
1704  * @priv: driver private structure
1705  * Description:
1706  * This starts all the RX and TX DMA channels
1707  */
1708 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1709 {
1710         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1711         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1712         u32 chan = 0;
1713
1714         for (chan = 0; chan < rx_channels_count; chan++)
1715                 stmmac_start_rx_dma(priv, chan);
1716
1717         for (chan = 0; chan < tx_channels_count; chan++)
1718                 stmmac_start_tx_dma(priv, chan);
1719 }
1720
1721 /**
1722  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1723  * @priv: driver private structure
1724  * Description:
1725  * This stops the RX and TX DMA channels
1726  */
1727 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1728 {
1729         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1730         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1731         u32 chan = 0;
1732
1733         for (chan = 0; chan < rx_channels_count; chan++)
1734                 stmmac_stop_rx_dma(priv, chan);
1735
1736         for (chan = 0; chan < tx_channels_count; chan++)
1737                 stmmac_stop_tx_dma(priv, chan);
1738 }
1739
1740 /**
1741  *  stmmac_dma_operation_mode - HW DMA operation mode
1742  *  @priv: driver private structure
1743  *  Description: it is used for configuring the DMA operation mode register in
1744  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1745  */
1746 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1747 {
1748         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1749         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1750         int rxfifosz = priv->plat->rx_fifo_size;
1751         int txfifosz = priv->plat->tx_fifo_size;
1752         u32 txmode = 0;
1753         u32 rxmode = 0;
1754         u32 chan = 0;
1755         u8 qmode = 0;
1756
1757         if (rxfifosz == 0)
1758                 rxfifosz = priv->dma_cap.rx_fifo_size;
1759         if (txfifosz == 0)
1760                 txfifosz = priv->dma_cap.tx_fifo_size;
1761
1762         /* Adjust for real per queue fifo size */
1763         rxfifosz /= rx_channels_count;
1764         txfifosz /= tx_channels_count;
1765
1766         if (priv->plat->force_thresh_dma_mode) {
1767                 txmode = tc;
1768                 rxmode = tc;
1769         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1770                 /*
1771                  * In case of GMAC, SF mode can be enabled
1772                  * to perform the TX COE in HW. This depends on:
1773                  * 1) TX COE if actually supported
1774                  * 2) There is no bugged Jumbo frame support
1775                  *    that needs to not insert csum in the TDES.
1776                  */
1777                 txmode = SF_DMA_MODE;
1778                 rxmode = SF_DMA_MODE;
1779                 priv->xstats.threshold = SF_DMA_MODE;
1780         } else {
1781                 txmode = tc;
1782                 rxmode = SF_DMA_MODE;
1783         }
1784
1785         /* configure all channels */
1786         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1787                 for (chan = 0; chan < rx_channels_count; chan++) {
1788                         qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1789
1790                         priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1791                                                    rxfifosz, qmode);
1792                 }
1793
1794                 for (chan = 0; chan < tx_channels_count; chan++) {
1795                         qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1796
1797                         priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan,
1798                                                    txfifosz, qmode);
1799                 }
1800         } else {
1801                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1802                                         rxfifosz);
1803         }
1804 }
1805
1806 /**
1807  * stmmac_tx_clean - to manage the transmission completion
1808  * @priv: driver private structure
1809  * @queue: TX queue index
1810  * Description: it reclaims the transmit resources after transmission completes.
1811  */
1812 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
1813 {
1814         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1815         unsigned int bytes_compl = 0, pkts_compl = 0;
1816         unsigned int entry;
1817
1818         netif_tx_lock(priv->dev);
1819
1820         priv->xstats.tx_clean++;
1821
1822         entry = tx_q->dirty_tx;
1823         while (entry != tx_q->cur_tx) {
1824                 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1825                 struct dma_desc *p;
1826                 int status;
1827
1828                 if (priv->extend_desc)
1829                         p = (struct dma_desc *)(tx_q->dma_etx + entry);
1830                 else
1831                         p = tx_q->dma_tx + entry;
1832
1833                 status = priv->hw->desc->tx_status(&priv->dev->stats,
1834                                                       &priv->xstats, p,
1835                                                       priv->ioaddr);
1836                 /* Check if the descriptor is owned by the DMA */
1837                 if (unlikely(status & tx_dma_own))
1838                         break;
1839
1840                 /* Just consider the last segment and ...*/
1841                 if (likely(!(status & tx_not_ls))) {
1842                         /* ... verify the status error condition */
1843                         if (unlikely(status & tx_err)) {
1844                                 priv->dev->stats.tx_errors++;
1845                         } else {
1846                                 priv->dev->stats.tx_packets++;
1847                                 priv->xstats.tx_pkt_n++;
1848                         }
1849                         stmmac_get_tx_hwtstamp(priv, p, skb);
1850                 }
1851
1852                 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1853                         if (tx_q->tx_skbuff_dma[entry].map_as_page)
1854                                 dma_unmap_page(priv->device,
1855                                                tx_q->tx_skbuff_dma[entry].buf,
1856                                                tx_q->tx_skbuff_dma[entry].len,
1857                                                DMA_TO_DEVICE);
1858                         else
1859                                 dma_unmap_single(priv->device,
1860                                                  tx_q->tx_skbuff_dma[entry].buf,
1861                                                  tx_q->tx_skbuff_dma[entry].len,
1862                                                  DMA_TO_DEVICE);
1863                         tx_q->tx_skbuff_dma[entry].buf = 0;
1864                         tx_q->tx_skbuff_dma[entry].len = 0;
1865                         tx_q->tx_skbuff_dma[entry].map_as_page = false;
1866                 }
1867
1868                 if (priv->hw->mode->clean_desc3)
1869                         priv->hw->mode->clean_desc3(tx_q, p);
1870
1871                 tx_q->tx_skbuff_dma[entry].last_segment = false;
1872                 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1873
1874                 if (likely(skb != NULL)) {
1875                         pkts_compl++;
1876                         bytes_compl += skb->len;
1877                         dev_consume_skb_any(skb);
1878                         tx_q->tx_skbuff[entry] = NULL;
1879                 }
1880
1881                 priv->hw->desc->release_tx_desc(p, priv->mode);
1882
1883                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1884         }
1885         tx_q->dirty_tx = entry;
1886
1887         netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1888                                   pkts_compl, bytes_compl);
1889
1890         if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1891                                                                 queue))) &&
1892             stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1893
1894                 netif_dbg(priv, tx_done, priv->dev,
1895                           "%s: restart transmit\n", __func__);
1896                 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1897         }
1898
1899         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1900                 stmmac_enable_eee_mode(priv);
1901                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1902         }
1903         netif_tx_unlock(priv->dev);
1904 }
1905
1906 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan)
1907 {
1908         priv->hw->dma->enable_dma_irq(priv->ioaddr, chan);
1909 }
1910
1911 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan)
1912 {
1913         priv->hw->dma->disable_dma_irq(priv->ioaddr, chan);
1914 }
1915
1916 /**
1917  * stmmac_tx_err - to manage the tx error
1918  * @priv: driver private structure
1919  * @chan: channel index
1920  * Description: it cleans the descriptors and restarts the transmission
1921  * in case of transmission errors.
1922  */
1923 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1924 {
1925         struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
1926         int i;
1927
1928         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
1929
1930         stmmac_stop_tx_dma(priv, chan);
1931         dma_free_tx_skbufs(priv, chan);
1932         for (i = 0; i < DMA_TX_SIZE; i++)
1933                 if (priv->extend_desc)
1934                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1935                                                      priv->mode,
1936                                                      (i == DMA_TX_SIZE - 1));
1937                 else
1938                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1939                                                      priv->mode,
1940                                                      (i == DMA_TX_SIZE - 1));
1941         tx_q->dirty_tx = 0;
1942         tx_q->cur_tx = 0;
1943         netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
1944         stmmac_start_tx_dma(priv, chan);
1945
1946         priv->dev->stats.tx_errors++;
1947         netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
1948 }
1949
1950 /**
1951  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
1952  *  @priv: driver private structure
1953  *  @txmode: TX operating mode
1954  *  @rxmode: RX operating mode
1955  *  @chan: channel index
1956  *  Description: it is used for configuring of the DMA operation mode in
1957  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
1958  *  mode.
1959  */
1960 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
1961                                           u32 rxmode, u32 chan)
1962 {
1963         u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1964         u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1965         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1966         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1967         int rxfifosz = priv->plat->rx_fifo_size;
1968         int txfifosz = priv->plat->tx_fifo_size;
1969
1970         if (rxfifosz == 0)
1971                 rxfifosz = priv->dma_cap.rx_fifo_size;
1972         if (txfifosz == 0)
1973                 txfifosz = priv->dma_cap.tx_fifo_size;
1974
1975         /* Adjust for real per queue fifo size */
1976         rxfifosz /= rx_channels_count;
1977         txfifosz /= tx_channels_count;
1978
1979         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1980                 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1981                                            rxfifosz, rxqmode);
1982                 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan,
1983                                            txfifosz, txqmode);
1984         } else {
1985                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1986                                         rxfifosz);
1987         }
1988 }
1989
1990 /**
1991  * stmmac_dma_interrupt - DMA ISR
1992  * @priv: driver private structure
1993  * Description: this is the DMA ISR. It is called by the main ISR.
1994  * It calls the dwmac dma routine and schedule poll method in case of some
1995  * work can be done.
1996  */
1997 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
1998 {
1999         u32 tx_channel_count = priv->plat->tx_queues_to_use;
2000         int status;
2001         u32 chan;
2002
2003         for (chan = 0; chan < tx_channel_count; chan++) {
2004                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
2005
2006                 status = priv->hw->dma->dma_interrupt(priv->ioaddr,
2007                                                       &priv->xstats, chan);
2008                 if (likely((status & handle_rx)) || (status & handle_tx)) {
2009                         if (likely(napi_schedule_prep(&rx_q->napi))) {
2010                                 stmmac_disable_dma_irq(priv, chan);
2011                                 __napi_schedule(&rx_q->napi);
2012                         }
2013                 }
2014
2015                 if (unlikely(status & tx_hard_error_bump_tc)) {
2016                         /* Try to bump up the dma threshold on this failure */
2017                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2018                             (tc <= 256)) {
2019                                 tc += 64;
2020                                 if (priv->plat->force_thresh_dma_mode)
2021                                         stmmac_set_dma_operation_mode(priv,
2022                                                                       tc,
2023                                                                       tc,
2024                                                                       chan);
2025                                 else
2026                                         stmmac_set_dma_operation_mode(priv,
2027                                                                     tc,
2028                                                                     SF_DMA_MODE,
2029                                                                     chan);
2030                                 priv->xstats.threshold = tc;
2031                         }
2032                 } else if (unlikely(status == tx_hard_error)) {
2033                         stmmac_tx_err(priv, chan);
2034                 }
2035         }
2036 }
2037
2038 /**
2039  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2040  * @priv: driver private structure
2041  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2042  */
2043 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2044 {
2045         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2046                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2047
2048         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2049                 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
2050                 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
2051         } else {
2052                 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
2053                 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
2054         }
2055
2056         dwmac_mmc_intr_all_mask(priv->mmcaddr);
2057
2058         if (priv->dma_cap.rmon) {
2059                 dwmac_mmc_ctrl(priv->mmcaddr, mode);
2060                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2061         } else
2062                 netdev_info(priv->dev, "No MAC Management Counters available\n");
2063 }
2064
2065 /**
2066  * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
2067  * @priv: driver private structure
2068  * Description: select the Enhanced/Alternate or Normal descriptors.
2069  * In case of Enhanced/Alternate, it checks if the extended descriptors are
2070  * supported by the HW capability register.
2071  */
2072 static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
2073 {
2074         if (priv->plat->enh_desc) {
2075                 dev_info(priv->device, "Enhanced/Alternate descriptors\n");
2076
2077                 /* GMAC older than 3.50 has no extended descriptors */
2078                 if (priv->synopsys_id >= DWMAC_CORE_3_50) {
2079                         dev_info(priv->device, "Enabled extended descriptors\n");
2080                         priv->extend_desc = 1;
2081                 } else
2082                         dev_warn(priv->device, "Extended descriptors not supported\n");
2083
2084                 priv->hw->desc = &enh_desc_ops;
2085         } else {
2086                 dev_info(priv->device, "Normal descriptors\n");
2087                 priv->hw->desc = &ndesc_ops;
2088         }
2089 }
2090
2091 /**
2092  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2093  * @priv: driver private structure
2094  * Description:
2095  *  new GMAC chip generations have a new register to indicate the
2096  *  presence of the optional feature/functions.
2097  *  This can be also used to override the value passed through the
2098  *  platform and necessary for old MAC10/100 and GMAC chips.
2099  */
2100 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2101 {
2102         u32 ret = 0;
2103
2104         if (priv->hw->dma->get_hw_feature) {
2105                 priv->hw->dma->get_hw_feature(priv->ioaddr,
2106                                               &priv->dma_cap);
2107                 ret = 1;
2108         }
2109
2110         return ret;
2111 }
2112
2113 /**
2114  * stmmac_check_ether_addr - check if the MAC addr is valid
2115  * @priv: driver private structure
2116  * Description:
2117  * it is to verify if the MAC address is valid, in case of failures it
2118  * generates a random MAC address
2119  */
2120 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2121 {
2122         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2123                 priv->hw->mac->get_umac_addr(priv->hw,
2124                                              priv->dev->dev_addr, 0);
2125                 if (!is_valid_ether_addr(priv->dev->dev_addr))
2126                         eth_hw_addr_random(priv->dev);
2127                 netdev_info(priv->dev, "device MAC address %pM\n",
2128                             priv->dev->dev_addr);
2129         }
2130 }
2131
2132 /**
2133  * stmmac_init_dma_engine - DMA init.
2134  * @priv: driver private structure
2135  * Description:
2136  * It inits the DMA invoking the specific MAC/GMAC callback.
2137  * Some DMA parameters can be passed from the platform;
2138  * in case of these are not passed a default is kept for the MAC or GMAC.
2139  */
2140 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2141 {
2142         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2143         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2144         struct stmmac_rx_queue *rx_q;
2145         struct stmmac_tx_queue *tx_q;
2146         u32 dummy_dma_rx_phy = 0;
2147         u32 dummy_dma_tx_phy = 0;
2148         u32 chan = 0;
2149         int atds = 0;
2150         int ret = 0;
2151
2152         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2153                 dev_err(priv->device, "Invalid DMA configuration\n");
2154                 return -EINVAL;
2155         }
2156
2157         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2158                 atds = 1;
2159
2160         ret = priv->hw->dma->reset(priv->ioaddr);
2161         if (ret) {
2162                 dev_err(priv->device, "Failed to reset the dma\n");
2163                 return ret;
2164         }
2165
2166         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2167                 /* DMA Configuration */
2168                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2169                                     dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
2170
2171                 /* DMA RX Channel Configuration */
2172                 for (chan = 0; chan < rx_channels_count; chan++) {
2173                         rx_q = &priv->rx_queue[chan];
2174
2175                         priv->hw->dma->init_rx_chan(priv->ioaddr,
2176                                                     priv->plat->dma_cfg,
2177                                                     rx_q->dma_rx_phy, chan);
2178
2179                         rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2180                                     (DMA_RX_SIZE * sizeof(struct dma_desc));
2181                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
2182                                                        rx_q->rx_tail_addr,
2183                                                        chan);
2184                 }
2185
2186                 /* DMA TX Channel Configuration */
2187                 for (chan = 0; chan < tx_channels_count; chan++) {
2188                         tx_q = &priv->tx_queue[chan];
2189
2190                         priv->hw->dma->init_chan(priv->ioaddr,
2191                                                  priv->plat->dma_cfg,
2192                                                  chan);
2193
2194                         priv->hw->dma->init_tx_chan(priv->ioaddr,
2195                                                     priv->plat->dma_cfg,
2196                                                     tx_q->dma_tx_phy, chan);
2197
2198                         tx_q->tx_tail_addr = tx_q->dma_tx_phy +
2199                                     (DMA_TX_SIZE * sizeof(struct dma_desc));
2200                         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr,
2201                                                        tx_q->tx_tail_addr,
2202                                                        chan);
2203                 }
2204         } else {
2205                 rx_q = &priv->rx_queue[chan];
2206                 tx_q = &priv->tx_queue[chan];
2207                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2208                                     tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds);
2209         }
2210
2211         if (priv->plat->axi && priv->hw->dma->axi)
2212                 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);
2213
2214         return ret;
2215 }
2216
2217 /**
2218  * stmmac_tx_timer - mitigation sw timer for tx.
2219  * @data: data pointer
2220  * Description:
2221  * This is the timer handler to directly invoke the stmmac_tx_clean.
2222  */
2223 static void stmmac_tx_timer(struct timer_list *t)
2224 {
2225         struct stmmac_priv *priv = from_timer(priv, t, txtimer);
2226         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2227         u32 queue;
2228
2229         /* let's scan all the tx queues */
2230         for (queue = 0; queue < tx_queues_count; queue++)
2231                 stmmac_tx_clean(priv, queue);
2232 }
2233
2234 /**
2235  * stmmac_init_tx_coalesce - init tx mitigation options.
2236  * @priv: driver private structure
2237  * Description:
2238  * This inits the transmit coalesce parameters: i.e. timer rate,
2239  * timer handler and default threshold used for enabling the
2240  * interrupt on completion bit.
2241  */
2242 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
2243 {
2244         priv->tx_coal_frames = STMMAC_TX_FRAMES;
2245         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2246         timer_setup(&priv->txtimer, stmmac_tx_timer, 0);
2247         priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
2248         add_timer(&priv->txtimer);
2249 }
2250
2251 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2252 {
2253         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2254         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2255         u32 chan;
2256
2257         /* set TX ring length */
2258         if (priv->hw->dma->set_tx_ring_len) {
2259                 for (chan = 0; chan < tx_channels_count; chan++)
2260                         priv->hw->dma->set_tx_ring_len(priv->ioaddr,
2261                                                        (DMA_TX_SIZE - 1), chan);
2262         }
2263
2264         /* set RX ring length */
2265         if (priv->hw->dma->set_rx_ring_len) {
2266                 for (chan = 0; chan < rx_channels_count; chan++)
2267                         priv->hw->dma->set_rx_ring_len(priv->ioaddr,
2268                                                        (DMA_RX_SIZE - 1), chan);
2269         }
2270 }
2271
2272 /**
2273  *  stmmac_set_tx_queue_weight - Set TX queue weight
2274  *  @priv: driver private structure
2275  *  Description: It is used for setting TX queues weight
2276  */
2277 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2278 {
2279         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2280         u32 weight;
2281         u32 queue;
2282
2283         for (queue = 0; queue < tx_queues_count; queue++) {
2284                 weight = priv->plat->tx_queues_cfg[queue].weight;
2285                 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue);
2286         }
2287 }
2288
2289 /**
2290  *  stmmac_configure_cbs - Configure CBS in TX queue
2291  *  @priv: driver private structure
2292  *  Description: It is used for configuring CBS in AVB TX queues
2293  */
2294 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2295 {
2296         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2297         u32 mode_to_use;
2298         u32 queue;
2299
2300         /* queue 0 is reserved for legacy traffic */
2301         for (queue = 1; queue < tx_queues_count; queue++) {
2302                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2303                 if (mode_to_use == MTL_QUEUE_DCB)
2304                         continue;
2305
2306                 priv->hw->mac->config_cbs(priv->hw,
2307                                 priv->plat->tx_queues_cfg[queue].send_slope,
2308                                 priv->plat->tx_queues_cfg[queue].idle_slope,
2309                                 priv->plat->tx_queues_cfg[queue].high_credit,
2310                                 priv->plat->tx_queues_cfg[queue].low_credit,
2311                                 queue);
2312         }
2313 }
2314
2315 /**
2316  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2317  *  @priv: driver private structure
2318  *  Description: It is used for mapping RX queues to RX dma channels
2319  */
2320 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2321 {
2322         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2323         u32 queue;
2324         u32 chan;
2325
2326         for (queue = 0; queue < rx_queues_count; queue++) {
2327                 chan = priv->plat->rx_queues_cfg[queue].chan;
2328                 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan);
2329         }
2330 }
2331
2332 /**
2333  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2334  *  @priv: driver private structure
2335  *  Description: It is used for configuring the RX Queue Priority
2336  */
2337 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2338 {
2339         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2340         u32 queue;
2341         u32 prio;
2342
2343         for (queue = 0; queue < rx_queues_count; queue++) {
2344                 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2345                         continue;
2346
2347                 prio = priv->plat->rx_queues_cfg[queue].prio;
2348                 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue);
2349         }
2350 }
2351
2352 /**
2353  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2354  *  @priv: driver private structure
2355  *  Description: It is used for configuring the TX Queue Priority
2356  */
2357 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2358 {
2359         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2360         u32 queue;
2361         u32 prio;
2362
2363         for (queue = 0; queue < tx_queues_count; queue++) {
2364                 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2365                         continue;
2366
2367                 prio = priv->plat->tx_queues_cfg[queue].prio;
2368                 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue);
2369         }
2370 }
2371
2372 /**
2373  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2374  *  @priv: driver private structure
2375  *  Description: It is used for configuring the RX queue routing
2376  */
2377 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2378 {
2379         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2380         u32 queue;
2381         u8 packet;
2382
2383         for (queue = 0; queue < rx_queues_count; queue++) {
2384                 /* no specific packet type routing specified for the queue */
2385                 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2386                         continue;
2387
2388                 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2389                 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue);
2390         }
2391 }
2392
2393 /**
2394  *  stmmac_mtl_configuration - Configure MTL
2395  *  @priv: driver private structure
2396  *  Description: It is used for configurring MTL
2397  */
2398 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2399 {
2400         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2401         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2402
2403         if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight)
2404                 stmmac_set_tx_queue_weight(priv);
2405
2406         /* Configure MTL RX algorithms */
2407         if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms)
2408                 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw,
2409                                                 priv->plat->rx_sched_algorithm);
2410
2411         /* Configure MTL TX algorithms */
2412         if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms)
2413                 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw,
2414                                                 priv->plat->tx_sched_algorithm);
2415
2416         /* Configure CBS in AVB TX queues */
2417         if (tx_queues_count > 1 && priv->hw->mac->config_cbs)
2418                 stmmac_configure_cbs(priv);
2419
2420         /* Map RX MTL to DMA channels */
2421         if (priv->hw->mac->map_mtl_to_dma)
2422                 stmmac_rx_queue_dma_chan_map(priv);
2423
2424         /* Enable MAC RX Queues */
2425         if (priv->hw->mac->rx_queue_enable)
2426                 stmmac_mac_enable_rx_queues(priv);
2427
2428         /* Set RX priorities */
2429         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio)
2430                 stmmac_mac_config_rx_queues_prio(priv);
2431
2432         /* Set TX priorities */
2433         if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio)
2434                 stmmac_mac_config_tx_queues_prio(priv);
2435
2436         /* Set RX routing */
2437         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing)
2438                 stmmac_mac_config_rx_queues_routing(priv);
2439 }
2440
2441 /**
2442  * stmmac_hw_setup - setup mac in a usable state.
2443  *  @dev : pointer to the device structure.
2444  *  Description:
2445  *  this is the main function to setup the HW in a usable state because the
2446  *  dma engine is reset, the core registers are configured (e.g. AXI,
2447  *  Checksum features, timers). The DMA is ready to start receiving and
2448  *  transmitting.
2449  *  Return value:
2450  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2451  *  file on failure.
2452  */
2453 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2454 {
2455         struct stmmac_priv *priv = netdev_priv(dev);
2456         u32 rx_cnt = priv->plat->rx_queues_to_use;
2457         u32 tx_cnt = priv->plat->tx_queues_to_use;
2458         u32 chan;
2459         int ret;
2460
2461         /* DMA initialization and SW reset */
2462         ret = stmmac_init_dma_engine(priv);
2463         if (ret < 0) {
2464                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2465                            __func__);
2466                 return ret;
2467         }
2468
2469         /* Copy the MAC addr into the HW  */
2470         priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
2471
2472         /* PS and related bits will be programmed according to the speed */
2473         if (priv->hw->pcs) {
2474                 int speed = priv->plat->mac_port_sel_speed;
2475
2476                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2477                     (speed == SPEED_1000)) {
2478                         priv->hw->ps = speed;
2479                 } else {
2480                         dev_warn(priv->device, "invalid port speed\n");
2481                         priv->hw->ps = 0;
2482                 }
2483         }
2484
2485         /* Initialize the MAC Core */
2486         priv->hw->mac->core_init(priv->hw, dev->mtu);
2487
2488         /* Initialize MTL*/
2489         if (priv->synopsys_id >= DWMAC_CORE_4_00)
2490                 stmmac_mtl_configuration(priv);
2491
2492         ret = priv->hw->mac->rx_ipc(priv->hw);
2493         if (!ret) {
2494                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2495                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2496                 priv->hw->rx_csum = 0;
2497         }
2498
2499         /* Enable the MAC Rx/Tx */
2500         priv->hw->mac->set_mac(priv->ioaddr, true);
2501
2502         /* Set the HW DMA mode and the COE */
2503         stmmac_dma_operation_mode(priv);
2504
2505         stmmac_mmc_setup(priv);
2506
2507         if (init_ptp) {
2508                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2509                 if (ret < 0)
2510                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2511
2512                 ret = stmmac_init_ptp(priv);
2513                 if (ret == -EOPNOTSUPP)
2514                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2515                 else if (ret)
2516                         netdev_warn(priv->dev, "PTP init failed\n");
2517         }
2518
2519 #ifdef CONFIG_DEBUG_FS
2520         ret = stmmac_init_fs(dev);
2521         if (ret < 0)
2522                 netdev_warn(priv->dev, "%s: failed debugFS registration\n",
2523                             __func__);
2524 #endif
2525         /* Start the ball rolling... */
2526         stmmac_start_all_dma(priv);
2527
2528         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2529
2530         if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
2531                 priv->rx_riwt = MAX_DMA_RIWT;
2532                 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
2533         }
2534
2535         if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
2536                 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
2537
2538         /* set TX and RX rings length */
2539         stmmac_set_rings_length(priv);
2540
2541         /* Enable TSO */
2542         if (priv->tso) {
2543                 for (chan = 0; chan < tx_cnt; chan++)
2544                         priv->hw->dma->enable_tso(priv->ioaddr, 1, chan);
2545         }
2546
2547         return 0;
2548 }
2549
2550 static void stmmac_hw_teardown(struct net_device *dev)
2551 {
2552         struct stmmac_priv *priv = netdev_priv(dev);
2553
2554         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2555 }
2556
2557 /**
2558  *  stmmac_open - open entry point of the driver
2559  *  @dev : pointer to the device structure.
2560  *  Description:
2561  *  This function is the open entry point of the driver.
2562  *  Return value:
2563  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2564  *  file on failure.
2565  */
2566 static int stmmac_open(struct net_device *dev)
2567 {
2568         struct stmmac_priv *priv = netdev_priv(dev);
2569         int ret;
2570
2571         stmmac_check_ether_addr(priv);
2572
2573         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2574             priv->hw->pcs != STMMAC_PCS_TBI &&
2575             priv->hw->pcs != STMMAC_PCS_RTBI) {
2576                 ret = stmmac_init_phy(dev);
2577                 if (ret) {
2578                         netdev_err(priv->dev,
2579                                    "%s: Cannot attach to PHY (error: %d)\n",
2580                                    __func__, ret);
2581                         return ret;
2582                 }
2583         }
2584
2585         /* Extra statistics */
2586         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2587         priv->xstats.threshold = tc;
2588
2589         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
2590         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2591         priv->mss = 0;
2592
2593         ret = alloc_dma_desc_resources(priv);
2594         if (ret < 0) {
2595                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2596                            __func__);
2597                 goto dma_desc_error;
2598         }
2599
2600         ret = init_dma_desc_rings(dev, GFP_KERNEL);
2601         if (ret < 0) {
2602                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2603                            __func__);
2604                 goto init_error;
2605         }
2606
2607         ret = stmmac_hw_setup(dev, true);
2608         if (ret < 0) {
2609                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2610                 goto init_error;
2611         }
2612
2613         stmmac_init_tx_coalesce(priv);
2614
2615         if (dev->phydev)
2616                 phy_start(dev->phydev);
2617
2618         /* Request the IRQ lines */
2619         ret = request_irq(dev->irq, stmmac_interrupt,
2620                           IRQF_SHARED, dev->name, dev);
2621         if (unlikely(ret < 0)) {
2622                 netdev_err(priv->dev,
2623                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2624                            __func__, dev->irq, ret);
2625                 goto irq_error;
2626         }
2627
2628         /* Request the Wake IRQ in case of another line is used for WoL */
2629         if (priv->wol_irq != dev->irq) {
2630                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2631                                   IRQF_SHARED, dev->name, dev);
2632                 if (unlikely(ret < 0)) {
2633                         netdev_err(priv->dev,
2634                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2635                                    __func__, priv->wol_irq, ret);
2636                         goto wolirq_error;
2637                 }
2638         }
2639
2640         /* Request the IRQ lines */
2641         if (priv->lpi_irq > 0) {
2642                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2643                                   dev->name, dev);
2644                 if (unlikely(ret < 0)) {
2645                         netdev_err(priv->dev,
2646                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2647                                    __func__, priv->lpi_irq, ret);
2648                         goto lpiirq_error;
2649                 }
2650         }
2651
2652         stmmac_enable_all_queues(priv);
2653         stmmac_start_all_queues(priv);
2654
2655         return 0;
2656
2657 lpiirq_error:
2658         if (priv->wol_irq != dev->irq)
2659                 free_irq(priv->wol_irq, dev);
2660 wolirq_error:
2661         free_irq(dev->irq, dev);
2662 irq_error:
2663         if (dev->phydev)
2664                 phy_stop(dev->phydev);
2665
2666         del_timer_sync(&priv->txtimer);
2667         stmmac_hw_teardown(dev);
2668 init_error:
2669         free_dma_desc_resources(priv);
2670 dma_desc_error:
2671         if (dev->phydev)
2672                 phy_disconnect(dev->phydev);
2673
2674         return ret;
2675 }
2676
2677 /**
2678  *  stmmac_release - close entry point of the driver
2679  *  @dev : device pointer.
2680  *  Description:
2681  *  This is the stop entry point of the driver.
2682  */
2683 static int stmmac_release(struct net_device *dev)
2684 {
2685         struct stmmac_priv *priv = netdev_priv(dev);
2686
2687         if (priv->eee_enabled)
2688                 del_timer_sync(&priv->eee_ctrl_timer);
2689
2690         /* Stop and disconnect the PHY */
2691         if (dev->phydev) {
2692                 phy_stop(dev->phydev);
2693                 phy_disconnect(dev->phydev);
2694         }
2695
2696         stmmac_stop_all_queues(priv);
2697
2698         stmmac_disable_all_queues(priv);
2699
2700         del_timer_sync(&priv->txtimer);
2701
2702         /* Free the IRQ lines */
2703         free_irq(dev->irq, dev);
2704         if (priv->wol_irq != dev->irq)
2705                 free_irq(priv->wol_irq, dev);
2706         if (priv->lpi_irq > 0)
2707                 free_irq(priv->lpi_irq, dev);
2708
2709         /* Stop TX/RX DMA and clear the descriptors */
2710         stmmac_stop_all_dma(priv);
2711
2712         /* Release and free the Rx/Tx resources */
2713         free_dma_desc_resources(priv);
2714
2715         /* Disable the MAC Rx/Tx */
2716         priv->hw->mac->set_mac(priv->ioaddr, false);
2717
2718         netif_carrier_off(dev);
2719
2720 #ifdef CONFIG_DEBUG_FS
2721         stmmac_exit_fs(dev);
2722 #endif
2723
2724         stmmac_release_ptp(priv);
2725
2726         return 0;
2727 }
2728
2729 /**
2730  *  stmmac_tso_allocator - close entry point of the driver
2731  *  @priv: driver private structure
2732  *  @des: buffer start address
2733  *  @total_len: total length to fill in descriptors
2734  *  @last_segmant: condition for the last descriptor
2735  *  @queue: TX queue index
2736  *  Description:
2737  *  This function fills descriptor and request new descriptors according to
2738  *  buffer length to fill
2739  */
2740 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2741                                  int total_len, bool last_segment, u32 queue)
2742 {
2743         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2744         struct dma_desc *desc;
2745         u32 buff_size;
2746         int tmp_len;
2747
2748         tmp_len = total_len;
2749
2750         while (tmp_len > 0) {
2751                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2752                 desc = tx_q->dma_tx + tx_q->cur_tx;
2753
2754                 desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
2755                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2756                             TSO_MAX_BUFF_SIZE : tmp_len;
2757
2758                 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
2759                         0, 1,
2760                         (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
2761                         0, 0);
2762
2763                 tmp_len -= TSO_MAX_BUFF_SIZE;
2764         }
2765 }
2766
2767 /**
2768  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2769  *  @skb : the socket buffer
2770  *  @dev : device pointer
2771  *  Description: this is the transmit function that is called on TSO frames
2772  *  (support available on GMAC4 and newer chips).
2773  *  Diagram below show the ring programming in case of TSO frames:
2774  *
2775  *  First Descriptor
2776  *   --------
2777  *   | DES0 |---> buffer1 = L2/L3/L4 header
2778  *   | DES1 |---> TCP Payload (can continue on next descr...)
2779  *   | DES2 |---> buffer 1 and 2 len
2780  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2781  *   --------
2782  *      |
2783  *     ...
2784  *      |
2785  *   --------
2786  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2787  *   | DES1 | --|
2788  *   | DES2 | --> buffer 1 and 2 len
2789  *   | DES3 |
2790  *   --------
2791  *
2792  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2793  */
2794 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2795 {
2796         struct dma_desc *desc, *first, *mss_desc = NULL;
2797         struct stmmac_priv *priv = netdev_priv(dev);
2798         int nfrags = skb_shinfo(skb)->nr_frags;
2799         u32 queue = skb_get_queue_mapping(skb);
2800         unsigned int first_entry, des;
2801         struct stmmac_tx_queue *tx_q;
2802         int tmp_pay_len = 0;
2803         u32 pay_len, mss;
2804         u8 proto_hdr_len;
2805         int i;
2806
2807         tx_q = &priv->tx_queue[queue];
2808
2809         /* Compute header lengths */
2810         proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2811
2812         /* Desc availability based on threshold should be enough safe */
2813         if (unlikely(stmmac_tx_avail(priv, queue) <
2814                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2815                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2816                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2817                                                                 queue));
2818                         /* This is a hard error, log it. */
2819                         netdev_err(priv->dev,
2820                                    "%s: Tx Ring full when queue awake\n",
2821                                    __func__);
2822                 }
2823                 return NETDEV_TX_BUSY;
2824         }
2825
2826         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2827
2828         mss = skb_shinfo(skb)->gso_size;
2829
2830         /* set new MSS value if needed */
2831         if (mss != priv->mss) {
2832                 mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2833                 priv->hw->desc->set_mss(mss_desc, mss);
2834                 priv->mss = mss;
2835                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2836         }
2837
2838         if (netif_msg_tx_queued(priv)) {
2839                 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2840                         __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
2841                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2842                         skb->data_len);
2843         }
2844
2845         first_entry = tx_q->cur_tx;
2846
2847         desc = tx_q->dma_tx + first_entry;
2848         first = desc;
2849
2850         /* first descriptor: fill Headers on Buf1 */
2851         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2852                              DMA_TO_DEVICE);
2853         if (dma_mapping_error(priv->device, des))
2854                 goto dma_map_err;
2855
2856         tx_q->tx_skbuff_dma[first_entry].buf = des;
2857         tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2858
2859         first->des0 = cpu_to_le32(des);
2860
2861         /* Fill start of payload in buff2 of first descriptor */
2862         if (pay_len)
2863                 first->des1 = cpu_to_le32(des + proto_hdr_len);
2864
2865         /* If needed take extra descriptors to fill the remaining payload */
2866         tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
2867
2868         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
2869
2870         /* Prepare fragments */
2871         for (i = 0; i < nfrags; i++) {
2872                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2873
2874                 des = skb_frag_dma_map(priv->device, frag, 0,
2875                                        skb_frag_size(frag),
2876                                        DMA_TO_DEVICE);
2877                 if (dma_mapping_error(priv->device, des))
2878                         goto dma_map_err;
2879
2880                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2881                                      (i == nfrags - 1), queue);
2882
2883                 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
2884                 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
2885                 tx_q->tx_skbuff[tx_q->cur_tx] = NULL;
2886                 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
2887         }
2888
2889         tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
2890
2891         /* Only the last descriptor gets to point to the skb. */
2892         tx_q->tx_skbuff[tx_q->cur_tx] = skb;
2893
2894         /* We've used all descriptors we need for this skb, however,
2895          * advance cur_tx so that it references a fresh descriptor.
2896          * ndo_start_xmit will fill this descriptor the next time it's
2897          * called and stmmac_tx_clean may clean up to this descriptor.
2898          */
2899         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2900
2901         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2902                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2903                           __func__);
2904                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
2905         }
2906
2907         dev->stats.tx_bytes += skb->len;
2908         priv->xstats.tx_tso_frames++;
2909         priv->xstats.tx_tso_nfrags += nfrags;
2910
2911         /* Manage tx mitigation */
2912         priv->tx_count_frames += nfrags + 1;
2913         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2914                 mod_timer(&priv->txtimer,
2915                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2916         } else {
2917                 priv->tx_count_frames = 0;
2918                 priv->hw->desc->set_tx_ic(desc);
2919                 priv->xstats.tx_set_ic_bit++;
2920         }
2921
2922         skb_tx_timestamp(skb);
2923
2924         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2925                      priv->hwts_tx_en)) {
2926                 /* declare that device is doing timestamping */
2927                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2928                 priv->hw->desc->enable_tx_timestamp(first);
2929         }
2930
2931         /* Complete the first descriptor before granting the DMA */
2932         priv->hw->desc->prepare_tso_tx_desc(first, 1,
2933                         proto_hdr_len,
2934                         pay_len,
2935                         1, tx_q->tx_skbuff_dma[first_entry].last_segment,
2936                         tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));
2937
2938         /* If context desc is used to change MSS */
2939         if (mss_desc)
2940                 priv->hw->desc->set_tx_owner(mss_desc);
2941
2942         /* The own bit must be the latest setting done when prepare the
2943          * descriptor and then barrier is needed to make sure that
2944          * all is coherent before granting the DMA engine.
2945          */
2946         dma_wmb();
2947
2948         if (netif_msg_pktdata(priv)) {
2949                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
2950                         __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
2951                         tx_q->cur_tx, first, nfrags);
2952
2953                 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE,
2954                                              0);
2955
2956                 pr_info(">>> frame to be transmitted: ");
2957                 print_pkt(skb->data, skb_headlen(skb));
2958         }
2959
2960         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
2961
2962         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
2963                                        queue);
2964
2965         return NETDEV_TX_OK;
2966
2967 dma_map_err:
2968         dev_err(priv->device, "Tx dma map failed\n");
2969         dev_kfree_skb(skb);
2970         priv->dev->stats.tx_dropped++;
2971         return NETDEV_TX_OK;
2972 }
2973
2974 /**
2975  *  stmmac_xmit - Tx entry point of the driver
2976  *  @skb : the socket buffer
2977  *  @dev : device pointer
2978  *  Description : this is the tx entry point of the driver.
2979  *  It programs the chain or the ring and supports oversized frames
2980  *  and SG feature.
2981  */
2982 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
2983 {
2984         struct stmmac_priv *priv = netdev_priv(dev);
2985         unsigned int nopaged_len = skb_headlen(skb);
2986         int i, csum_insertion = 0, is_jumbo = 0;
2987         u32 queue = skb_get_queue_mapping(skb);
2988         int nfrags = skb_shinfo(skb)->nr_frags;
2989         int entry;
2990         unsigned int first_entry;
2991         struct dma_desc *desc, *first;
2992         struct stmmac_tx_queue *tx_q;
2993         unsigned int enh_desc;
2994         unsigned int des;
2995
2996         tx_q = &priv->tx_queue[queue];
2997
2998         /* Manage oversized TCP frames for GMAC4 device */
2999         if (skb_is_gso(skb) && priv->tso) {
3000                 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
3001                         return stmmac_tso_xmit(skb, dev);
3002         }
3003
3004         if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
3005                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3006                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3007                                                                 queue));
3008                         /* This is a hard error, log it. */
3009                         netdev_err(priv->dev,
3010                                    "%s: Tx Ring full when queue awake\n",
3011                                    __func__);
3012                 }
3013                 return NETDEV_TX_BUSY;
3014         }
3015
3016         if (priv->tx_path_in_lpi_mode)
3017                 stmmac_disable_eee_mode(priv);
3018
3019         entry = tx_q->cur_tx;
3020         first_entry = entry;
3021
3022         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
3023
3024         if (likely(priv->extend_desc))
3025                 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3026         else
3027                 desc = tx_q->dma_tx + entry;
3028
3029         first = desc;
3030
3031         enh_desc = priv->plat->enh_desc;
3032         /* To program the descriptors according to the size of the frame */
3033         if (enh_desc)
3034                 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);
3035
3036         if (unlikely(is_jumbo) && likely(priv->synopsys_id <
3037                                          DWMAC_CORE_4_00)) {
3038                 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion);
3039                 if (unlikely(entry < 0))
3040                         goto dma_map_err;
3041         }
3042
3043         for (i = 0; i < nfrags; i++) {
3044                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3045                 int len = skb_frag_size(frag);
3046                 bool last_segment = (i == (nfrags - 1));
3047
3048                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3049
3050                 if (likely(priv->extend_desc))
3051                         desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3052                 else
3053                         desc = tx_q->dma_tx + entry;
3054
3055                 des = skb_frag_dma_map(priv->device, frag, 0, len,
3056                                        DMA_TO_DEVICE);
3057                 if (dma_mapping_error(priv->device, des))
3058                         goto dma_map_err; /* should reuse desc w/o issues */
3059
3060                 tx_q->tx_skbuff[entry] = NULL;
3061
3062                 tx_q->tx_skbuff_dma[entry].buf = des;
3063                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3064                         desc->des0 = cpu_to_le32(des);
3065                 else
3066                         desc->des2 = cpu_to_le32(des);
3067
3068                 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3069                 tx_q->tx_skbuff_dma[entry].len = len;
3070                 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3071
3072                 /* Prepare the descriptor and set the own bit too */
3073                 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
3074                                                 priv->mode, 1, last_segment,
3075                                                 skb->len);
3076         }
3077
3078         /* Only the last descriptor gets to point to the skb. */
3079         tx_q->tx_skbuff[entry] = skb;
3080
3081         /* We've used all descriptors we need for this skb, however,
3082          * advance cur_tx so that it references a fresh descriptor.
3083          * ndo_start_xmit will fill this descriptor the next time it's
3084          * called and stmmac_tx_clean may clean up to this descriptor.
3085          */
3086         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3087         tx_q->cur_tx = entry;
3088
3089         if (netif_msg_pktdata(priv)) {
3090                 void *tx_head;
3091
3092                 netdev_dbg(priv->dev,
3093                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3094                            __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3095                            entry, first, nfrags);
3096
3097                 if (priv->extend_desc)
3098                         tx_head = (void *)tx_q->dma_etx;
3099                 else
3100                         tx_head = (void *)tx_q->dma_tx;
3101
3102                 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
3103
3104                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3105                 print_pkt(skb->data, skb->len);
3106         }
3107
3108         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3109                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3110                           __func__);
3111                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3112         }
3113
3114         dev->stats.tx_bytes += skb->len;
3115
3116         /* According to the coalesce parameter the IC bit for the latest
3117          * segment is reset and the timer re-started to clean the tx status.
3118          * This approach takes care about the fragments: desc is the first
3119          * element in case of no SG.
3120          */
3121         priv->tx_count_frames += nfrags + 1;
3122         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
3123                 mod_timer(&priv->txtimer,
3124                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
3125         } else {
3126                 priv->tx_count_frames = 0;
3127                 priv->hw->desc->set_tx_ic(desc);
3128                 priv->xstats.tx_set_ic_bit++;
3129         }
3130
3131         skb_tx_timestamp(skb);
3132
3133         /* Ready to fill the first descriptor and set the OWN bit w/o any
3134          * problems because all the descriptors are actually ready to be
3135          * passed to the DMA engine.
3136          */
3137         if (likely(!is_jumbo)) {
3138                 bool last_segment = (nfrags == 0);
3139
3140                 des = dma_map_single(priv->device, skb->data,
3141                                      nopaged_len, DMA_TO_DEVICE);
3142                 if (dma_mapping_error(priv->device, des))
3143                         goto dma_map_err;
3144
3145                 tx_q->tx_skbuff_dma[first_entry].buf = des;
3146                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3147                         first->des0 = cpu_to_le32(des);
3148                 else
3149                         first->des2 = cpu_to_le32(des);
3150
3151                 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3152                 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3153
3154                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3155                              priv->hwts_tx_en)) {
3156                         /* declare that device is doing timestamping */
3157                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3158                         priv->hw->desc->enable_tx_timestamp(first);
3159                 }
3160
3161                 /* Prepare the first descriptor setting the OWN bit too */
3162                 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
3163                                                 csum_insertion, priv->mode, 1,
3164                                                 last_segment, skb->len);
3165
3166                 /* The own bit must be the latest setting done when prepare the
3167                  * descriptor and then barrier is needed to make sure that
3168                  * all is coherent before granting the DMA engine.
3169                  */
3170                 dma_wmb();
3171         }
3172
3173         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3174
3175         if (priv->synopsys_id < DWMAC_CORE_4_00)
3176                 priv->hw->dma->enable_dma_transmission(priv->ioaddr);
3177         else
3178                 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
3179                                                queue);
3180
3181         return NETDEV_TX_OK;
3182
3183 dma_map_err:
3184         netdev_err(priv->dev, "Tx DMA map failed\n");
3185         dev_kfree_skb(skb);
3186         priv->dev->stats.tx_dropped++;
3187         return NETDEV_TX_OK;
3188 }
3189
3190 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3191 {
3192         struct ethhdr *ehdr;
3193         u16 vlanid;
3194
3195         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
3196             NETIF_F_HW_VLAN_CTAG_RX &&
3197             !__vlan_get_tag(skb, &vlanid)) {
3198                 /* pop the vlan tag */
3199                 ehdr = (struct ethhdr *)skb->data;
3200                 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
3201                 skb_pull(skb, VLAN_HLEN);
3202                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
3203         }
3204 }
3205
3206
3207 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3208 {
3209         if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3210                 return 0;
3211
3212         return 1;
3213 }
3214
3215 /**
3216  * stmmac_rx_refill - refill used skb preallocated buffers
3217  * @priv: driver private structure
3218  * @queue: RX queue index
3219  * Description : this is to reallocate the skb for the reception process
3220  * that is based on zero-copy.
3221  */
3222 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3223 {
3224         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3225         int dirty = stmmac_rx_dirty(priv, queue);
3226         unsigned int entry = rx_q->dirty_rx;
3227
3228         int bfsize = priv->dma_buf_sz;
3229
3230         while (dirty-- > 0) {
3231                 struct dma_desc *p;
3232
3233                 if (priv->extend_desc)
3234                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3235                 else
3236                         p = rx_q->dma_rx + entry;
3237
3238                 if (likely(!rx_q->rx_skbuff[entry])) {
3239                         struct sk_buff *skb;
3240
3241                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
3242                         if (unlikely(!skb)) {
3243                                 /* so for a while no zero-copy! */
3244                                 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH;
3245                                 if (unlikely(net_ratelimit()))
3246                                         dev_err(priv->device,
3247                                                 "fail to alloc skb entry %d\n",
3248                                                 entry);
3249                                 break;
3250                         }
3251
3252                         rx_q->rx_skbuff[entry] = skb;
3253                         rx_q->rx_skbuff_dma[entry] =
3254                             dma_map_single(priv->device, skb->data, bfsize,
3255                                            DMA_FROM_DEVICE);
3256                         if (dma_mapping_error(priv->device,
3257                                               rx_q->rx_skbuff_dma[entry])) {
3258                                 netdev_err(priv->dev, "Rx DMA map failed\n");
3259                                 dev_kfree_skb(skb);
3260                                 break;
3261                         }
3262
3263                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
3264                                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3265                                 p->des1 = 0;
3266                         } else {
3267                                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3268                         }
3269                         if (priv->hw->mode->refill_desc3)
3270                                 priv->hw->mode->refill_desc3(rx_q, p);
3271
3272                         if (rx_q->rx_zeroc_thresh > 0)
3273                                 rx_q->rx_zeroc_thresh--;
3274
3275                         netif_dbg(priv, rx_status, priv->dev,
3276                                   "refill entry #%d\n", entry);
3277                 }
3278                 dma_wmb();
3279
3280                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3281                         priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
3282                 else
3283                         priv->hw->desc->set_rx_owner(p);
3284
3285                 dma_wmb();
3286
3287                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3288         }
3289         rx_q->dirty_rx = entry;
3290 }
3291
3292 /**
3293  * stmmac_rx - manage the receive process
3294  * @priv: driver private structure
3295  * @limit: napi bugget
3296  * @queue: RX queue index.
3297  * Description :  this the function called by the napi poll method.
3298  * It gets all the frames inside the ring.
3299  */
3300 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3301 {
3302         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3303         unsigned int entry = rx_q->cur_rx;
3304         int coe = priv->hw->rx_csum;
3305         unsigned int next_entry;
3306         unsigned int count = 0;
3307
3308         if (netif_msg_rx_status(priv)) {
3309                 void *rx_head;
3310
3311                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3312                 if (priv->extend_desc)
3313                         rx_head = (void *)rx_q->dma_erx;
3314                 else
3315                         rx_head = (void *)rx_q->dma_rx;
3316
3317                 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
3318         }
3319         while (count < limit) {
3320                 int status;
3321                 struct dma_desc *p;
3322                 struct dma_desc *np;
3323
3324                 if (priv->extend_desc)
3325                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3326                 else
3327                         p = rx_q->dma_rx + entry;
3328
3329                 /* read the status of the incoming frame */
3330                 status = priv->hw->desc->rx_status(&priv->dev->stats,
3331                                                    &priv->xstats, p);
3332                 /* check if managed by the DMA otherwise go ahead */
3333                 if (unlikely(status & dma_own))
3334                         break;
3335
3336                 count++;
3337
3338                 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3339                 next_entry = rx_q->cur_rx;
3340
3341                 if (priv->extend_desc)
3342                         np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3343                 else
3344                         np = rx_q->dma_rx + next_entry;
3345
3346                 prefetch(np);
3347
3348                 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
3349                         priv->hw->desc->rx_extended_status(&priv->dev->stats,
3350                                                            &priv->xstats,
3351                                                            rx_q->dma_erx +
3352                                                            entry);
3353                 if (unlikely(status == discard_frame)) {
3354                         priv->dev->stats.rx_errors++;
3355                         if (priv->hwts_rx_en && !priv->extend_desc) {
3356                                 /* DESC2 & DESC3 will be overwritten by device
3357                                  * with timestamp value, hence reinitialize
3358                                  * them in stmmac_rx_refill() function so that
3359                                  * device can reuse it.
3360                                  */
3361                                 dev_kfree_skb_any(rx_q->rx_skbuff[entry]);
3362                                 rx_q->rx_skbuff[entry] = NULL;
3363                                 dma_unmap_single(priv->device,
3364                                                  rx_q->rx_skbuff_dma[entry],
3365                                                  priv->dma_buf_sz,
3366                                                  DMA_FROM_DEVICE);
3367                         }
3368                 } else {
3369                         struct sk_buff *skb;
3370                         int frame_len;
3371                         unsigned int des;
3372
3373                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3374                                 des = le32_to_cpu(p->des0);
3375                         else
3376                                 des = le32_to_cpu(p->des2);
3377
3378                         frame_len = priv->hw->desc->get_rx_frame_len(p, coe);
3379
3380                         /*  If frame length is greater than skb buffer size
3381                          *  (preallocated during init) then the packet is
3382                          *  ignored
3383                          */
3384                         if (frame_len > priv->dma_buf_sz) {
3385                                 netdev_err(priv->dev,
3386                                            "len %d larger than size (%d)\n",
3387                                            frame_len, priv->dma_buf_sz);
3388                                 priv->dev->stats.rx_length_errors++;
3389                                 break;
3390                         }
3391
3392                         /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3393                          * Type frames (LLC/LLC-SNAP)
3394                          */
3395                         if (unlikely(status != llc_snap))
3396                                 frame_len -= ETH_FCS_LEN;
3397
3398                         if (netif_msg_rx_status(priv)) {
3399                                 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
3400                                            p, entry, des);
3401                                 if (frame_len > ETH_FRAME_LEN)
3402                                         netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
3403                                                    frame_len, status);
3404                         }
3405
3406                         /* The zero-copy is always used for all the sizes
3407                          * in case of GMAC4 because it needs
3408                          * to refill the used descriptors, always.
3409                          */
3410                         if (unlikely(!priv->plat->has_gmac4 &&
3411                                      ((frame_len < priv->rx_copybreak) ||
3412                                      stmmac_rx_threshold_count(rx_q)))) {
3413                                 skb = netdev_alloc_skb_ip_align(priv->dev,
3414                                                                 frame_len);
3415                                 if (unlikely(!skb)) {
3416                                         if (net_ratelimit())
3417                                                 dev_warn(priv->device,
3418                                                          "packet dropped\n");
3419                                         priv->dev->stats.rx_dropped++;
3420                                         break;
3421                                 }
3422
3423                                 dma_sync_single_for_cpu(priv->device,
3424                                                         rx_q->rx_skbuff_dma
3425                                                         [entry], frame_len,
3426                                                         DMA_FROM_DEVICE);
3427                                 skb_copy_to_linear_data(skb,
3428                                                         rx_q->
3429                                                         rx_skbuff[entry]->data,
3430                                                         frame_len);
3431
3432                                 skb_put(skb, frame_len);
3433                                 dma_sync_single_for_device(priv->device,
3434                                                            rx_q->rx_skbuff_dma
3435                                                            [entry], frame_len,
3436                                                            DMA_FROM_DEVICE);
3437                         } else {
3438                                 skb = rx_q->rx_skbuff[entry];
3439                                 if (unlikely(!skb)) {
3440                                         netdev_err(priv->dev,
3441                                                    "%s: Inconsistent Rx chain\n",
3442                                                    priv->dev->name);
3443                                         priv->dev->stats.rx_dropped++;
3444                                         break;
3445                                 }
3446                                 prefetch(skb->data - NET_IP_ALIGN);
3447                                 rx_q->rx_skbuff[entry] = NULL;
3448                                 rx_q->rx_zeroc_thresh++;
3449
3450                                 skb_put(skb, frame_len);
3451                                 dma_unmap_single(priv->device,
3452                                                  rx_q->rx_skbuff_dma[entry],
3453                                                  priv->dma_buf_sz,
3454                                                  DMA_FROM_DEVICE);
3455                         }
3456
3457                         if (netif_msg_pktdata(priv)) {
3458                                 netdev_dbg(priv->dev, "frame received (%dbytes)",
3459                                            frame_len);
3460                                 print_pkt(skb->data, frame_len);
3461                         }
3462
3463                         stmmac_get_rx_hwtstamp(priv, p, np, skb);
3464
3465                         stmmac_rx_vlan(priv->dev, skb);
3466
3467                         skb->protocol = eth_type_trans(skb, priv->dev);
3468
3469                         if (unlikely(!coe))
3470                                 skb_checksum_none_assert(skb);
3471                         else
3472                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3473
3474                         napi_gro_receive(&rx_q->napi, skb);
3475
3476                         priv->dev->stats.rx_packets++;
3477                         priv->dev->stats.rx_bytes += frame_len;
3478                 }
3479                 entry = next_entry;
3480         }
3481
3482         stmmac_rx_refill(priv, queue);
3483
3484         priv->xstats.rx_pkt_n += count;
3485
3486         return count;
3487 }
3488
3489 /**
3490  *  stmmac_poll - stmmac poll method (NAPI)
3491  *  @napi : pointer to the napi structure.
3492  *  @budget : maximum number of packets that the current CPU can receive from
3493  *            all interfaces.
3494  *  Description :
3495  *  To look at the incoming frames and clear the tx resources.
3496  */
3497 static int stmmac_poll(struct napi_struct *napi, int budget)
3498 {
3499         struct stmmac_rx_queue *rx_q =
3500                 container_of(napi, struct stmmac_rx_queue, napi);
3501         struct stmmac_priv *priv = rx_q->priv_data;
3502         u32 tx_count = priv->plat->tx_queues_to_use;
3503         u32 chan = rx_q->queue_index;
3504         int work_done = 0;
3505         u32 queue;
3506
3507         priv->xstats.napi_poll++;
3508
3509         /* check all the queues */
3510         for (queue = 0; queue < tx_count; queue++)
3511                 stmmac_tx_clean(priv, queue);
3512
3513         work_done = stmmac_rx(priv, budget, rx_q->queue_index);
3514         if (work_done < budget) {
3515                 napi_complete_done(napi, work_done);
3516                 stmmac_enable_dma_irq(priv, chan);
3517         }
3518         return work_done;
3519 }
3520
3521 /**
3522  *  stmmac_tx_timeout
3523  *  @dev : Pointer to net device structure
3524  *  Description: this function is called when a packet transmission fails to
3525  *   complete within a reasonable time. The driver will mark the error in the
3526  *   netdev structure and arrange for the device to be reset to a sane state
3527  *   in order to transmit a new packet.
3528  */
3529 static void stmmac_tx_timeout(struct net_device *dev)
3530 {
3531         struct stmmac_priv *priv = netdev_priv(dev);
3532         u32 tx_count = priv->plat->tx_queues_to_use;
3533         u32 chan;
3534
3535         /* Clear Tx resources and restart transmitting again */
3536         for (chan = 0; chan < tx_count; chan++)
3537                 stmmac_tx_err(priv, chan);
3538 }
3539
3540 /**
3541  *  stmmac_set_rx_mode - entry point for multicast addressing
3542  *  @dev : pointer to the device structure
3543  *  Description:
3544  *  This function is a driver entry point which gets called by the kernel
3545  *  whenever multicast addresses must be enabled/disabled.
3546  *  Return value:
3547  *  void.
3548  */
3549 static void stmmac_set_rx_mode(struct net_device *dev)
3550 {
3551         struct stmmac_priv *priv = netdev_priv(dev);
3552
3553         priv->hw->mac->set_filter(priv->hw, dev);
3554 }
3555
3556 /**
3557  *  stmmac_change_mtu - entry point to change MTU size for the device.
3558  *  @dev : device pointer.
3559  *  @new_mtu : the new MTU size for the device.
3560  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3561  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3562  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3563  *  Return value:
3564  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3565  *  file on failure.
3566  */
3567 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3568 {
3569         struct stmmac_priv *priv = netdev_priv(dev);
3570
3571         if (netif_running(dev)) {
3572                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3573                 return -EBUSY;
3574         }
3575
3576         dev->mtu = new_mtu;
3577
3578         netdev_update_features(dev);
3579
3580         return 0;
3581 }
3582
3583 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3584                                              netdev_features_t features)
3585 {
3586         struct stmmac_priv *priv = netdev_priv(dev);
3587
3588         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3589                 features &= ~NETIF_F_RXCSUM;
3590
3591         if (!priv->plat->tx_coe)
3592                 features &= ~NETIF_F_CSUM_MASK;
3593
3594         /* Some GMAC devices have a bugged Jumbo frame support that
3595          * needs to have the Tx COE disabled for oversized frames
3596          * (due to limited buffer sizes). In this case we disable
3597          * the TX csum insertion in the TDES and not use SF.
3598          */
3599         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3600                 features &= ~NETIF_F_CSUM_MASK;
3601
3602         /* Disable tso if asked by ethtool */
3603         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3604                 if (features & NETIF_F_TSO)
3605                         priv->tso = true;
3606                 else
3607                         priv->tso = false;
3608         }
3609
3610         return features;
3611 }
3612
3613 static int stmmac_set_features(struct net_device *netdev,
3614                                netdev_features_t features)
3615 {
3616         struct stmmac_priv *priv = netdev_priv(netdev);
3617
3618         /* Keep the COE Type in case of csum is supporting */
3619         if (features & NETIF_F_RXCSUM)
3620                 priv->hw->rx_csum = priv->plat->rx_coe;
3621         else
3622                 priv->hw->rx_csum = 0;
3623         /* No check needed because rx_coe has been set before and it will be
3624          * fixed in case of issue.
3625          */
3626         priv->hw->mac->rx_ipc(priv->hw);
3627
3628         return 0;
3629 }
3630
3631 /**
3632  *  stmmac_interrupt - main ISR
3633  *  @irq: interrupt number.
3634  *  @dev_id: to pass the net device pointer.
3635  *  Description: this is the main driver interrupt service routine.
3636  *  It can call:
3637  *  o DMA service routine (to manage incoming frame reception and transmission
3638  *    status)
3639  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3640  *    interrupts.
3641  */
3642 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3643 {
3644         struct net_device *dev = (struct net_device *)dev_id;
3645         struct stmmac_priv *priv = netdev_priv(dev);
3646         u32 rx_cnt = priv->plat->rx_queues_to_use;
3647         u32 tx_cnt = priv->plat->tx_queues_to_use;
3648         u32 queues_count;
3649         u32 queue;
3650
3651         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3652
3653         if (priv->irq_wake)
3654                 pm_wakeup_event(priv->device, 0);
3655
3656         if (unlikely(!dev)) {
3657                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3658                 return IRQ_NONE;
3659         }
3660
3661         /* To handle GMAC own interrupts */
3662         if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
3663                 int status = priv->hw->mac->host_irq_status(priv->hw,
3664                                                             &priv->xstats);
3665
3666                 if (unlikely(status)) {
3667                         /* For LPI we need to save the tx status */
3668                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3669                                 priv->tx_path_in_lpi_mode = true;
3670                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3671                                 priv->tx_path_in_lpi_mode = false;
3672                 }
3673
3674                 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3675                         for (queue = 0; queue < queues_count; queue++) {
3676                                 struct stmmac_rx_queue *rx_q =
3677                                 &priv->rx_queue[queue];
3678
3679                                 status |=
3680                                 priv->hw->mac->host_mtl_irq_status(priv->hw,
3681                                                                    queue);
3682
3683                                 if (status & CORE_IRQ_MTL_RX_OVERFLOW &&
3684                                     priv->hw->dma->set_rx_tail_ptr)
3685                                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
3686                                                                 rx_q->rx_tail_addr,
3687                                                                 queue);
3688                         }
3689                 }
3690
3691                 /* PCS link status */
3692                 if (priv->hw->pcs) {
3693                         if (priv->xstats.pcs_link)
3694                                 netif_carrier_on(dev);
3695                         else
3696                                 netif_carrier_off(dev);
3697                 }
3698         }
3699
3700         /* To handle DMA interrupts */
3701         stmmac_dma_interrupt(priv);
3702
3703         return IRQ_HANDLED;
3704 }
3705
3706 #ifdef CONFIG_NET_POLL_CONTROLLER
3707 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3708  * to allow network I/O with interrupts disabled.
3709  */
3710 static void stmmac_poll_controller(struct net_device *dev)
3711 {
3712         disable_irq(dev->irq);
3713         stmmac_interrupt(dev->irq, dev);
3714         enable_irq(dev->irq);
3715 }
3716 #endif
3717
3718 /**
3719  *  stmmac_ioctl - Entry point for the Ioctl
3720  *  @dev: Device pointer.
3721  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3722  *  a proprietary structure used to pass information to the driver.
3723  *  @cmd: IOCTL command
3724  *  Description:
3725  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3726  */
3727 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3728 {
3729         int ret = -EOPNOTSUPP;
3730
3731         if (!netif_running(dev))
3732                 return -EINVAL;
3733
3734         switch (cmd) {
3735         case SIOCGMIIPHY:
3736         case SIOCGMIIREG:
3737         case SIOCSMIIREG:
3738                 if (!dev->phydev)
3739                         return -EINVAL;
3740                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3741                 break;
3742         case SIOCSHWTSTAMP:
3743                 ret = stmmac_hwtstamp_ioctl(dev, rq);
3744                 break;
3745         default:
3746                 break;
3747         }
3748
3749         return ret;
3750 }
3751
3752 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
3753 {
3754         struct stmmac_priv *priv = netdev_priv(ndev);
3755         int ret = 0;
3756
3757         ret = eth_mac_addr(ndev, addr);
3758         if (ret)
3759                 return ret;
3760
3761         priv->hw->mac->set_umac_addr(priv->hw, ndev->dev_addr, 0);
3762
3763         return ret;
3764 }
3765
3766 #ifdef CONFIG_DEBUG_FS
3767 static struct dentry *stmmac_fs_dir;
3768
3769 static void sysfs_display_ring(void *head, int size, int extend_desc,
3770                                struct seq_file *seq)
3771 {
3772         int i;
3773         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
3774         struct dma_desc *p = (struct dma_desc *)head;
3775
3776         for (i = 0; i < size; i++) {
3777                 if (extend_desc) {
3778                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3779                                    i, (unsigned int)virt_to_phys(ep),
3780                                    le32_to_cpu(ep->basic.des0),
3781                                    le32_to_cpu(ep->basic.des1),
3782                                    le32_to_cpu(ep->basic.des2),
3783                                    le32_to_cpu(ep->basic.des3));
3784                         ep++;
3785                 } else {
3786                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3787                                    i, (unsigned int)virt_to_phys(p),
3788                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
3789                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3790                         p++;
3791                 }
3792                 seq_printf(seq, "\n");
3793         }
3794 }
3795
3796 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
3797 {
3798         struct net_device *dev = seq->private;
3799         struct stmmac_priv *priv = netdev_priv(dev);
3800         u32 rx_count = priv->plat->rx_queues_to_use;
3801         u32 tx_count = priv->plat->tx_queues_to_use;
3802         u32 queue;
3803
3804         for (queue = 0; queue < rx_count; queue++) {
3805                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3806
3807                 seq_printf(seq, "RX Queue %d:\n", queue);
3808
3809                 if (priv->extend_desc) {
3810                         seq_printf(seq, "Extended descriptor ring:\n");
3811                         sysfs_display_ring((void *)rx_q->dma_erx,
3812                                            DMA_RX_SIZE, 1, seq);
3813                 } else {
3814                         seq_printf(seq, "Descriptor ring:\n");
3815                         sysfs_display_ring((void *)rx_q->dma_rx,
3816                                            DMA_RX_SIZE, 0, seq);
3817                 }
3818         }
3819
3820         for (queue = 0; queue < tx_count; queue++) {
3821                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3822
3823                 seq_printf(seq, "TX Queue %d:\n", queue);
3824
3825                 if (priv->extend_desc) {
3826                         seq_printf(seq, "Extended descriptor ring:\n");
3827                         sysfs_display_ring((void *)tx_q->dma_etx,
3828                                            DMA_TX_SIZE, 1, seq);
3829                 } else {
3830                         seq_printf(seq, "Descriptor ring:\n");
3831                         sysfs_display_ring((void *)tx_q->dma_tx,
3832                                            DMA_TX_SIZE, 0, seq);
3833                 }
3834         }
3835
3836         return 0;
3837 }
3838
3839 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
3840 {
3841         return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
3842 }
3843
3844 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */
3845
3846 static const struct file_operations stmmac_rings_status_fops = {
3847         .owner = THIS_MODULE,
3848         .open = stmmac_sysfs_ring_open,
3849         .read = seq_read,
3850         .llseek = seq_lseek,
3851         .release = single_release,
3852 };
3853
3854 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
3855 {
3856         struct net_device *dev = seq->private;
3857         struct stmmac_priv *priv = netdev_priv(dev);
3858
3859         if (!priv->hw_cap_support) {
3860                 seq_printf(seq, "DMA HW features not supported\n");
3861                 return 0;
3862         }
3863
3864         seq_printf(seq, "==============================\n");
3865         seq_printf(seq, "\tDMA HW features\n");
3866         seq_printf(seq, "==============================\n");
3867
3868         seq_printf(seq, "\t10/100 Mbps: %s\n",
3869                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3870         seq_printf(seq, "\t1000 Mbps: %s\n",
3871                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
3872         seq_printf(seq, "\tHalf duplex: %s\n",
3873                    (priv->dma_cap.half_duplex) ? "Y" : "N");
3874         seq_printf(seq, "\tHash Filter: %s\n",
3875                    (priv->dma_cap.hash_filter) ? "Y" : "N");
3876         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
3877                    (priv->dma_cap.multi_addr) ? "Y" : "N");
3878         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3879                    (priv->dma_cap.pcs) ? "Y" : "N");
3880         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
3881                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
3882         seq_printf(seq, "\tPMT Remote wake up: %s\n",
3883                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
3884         seq_printf(seq, "\tPMT Magic Frame: %s\n",
3885                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
3886         seq_printf(seq, "\tRMON module: %s\n",
3887                    (priv->dma_cap.rmon) ? "Y" : "N");
3888         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
3889                    (priv->dma_cap.time_stamp) ? "Y" : "N");
3890         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3891                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
3892         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3893                    (priv->dma_cap.eee) ? "Y" : "N");
3894         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
3895         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
3896                    (priv->dma_cap.tx_coe) ? "Y" : "N");
3897         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3898                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
3899                            (priv->dma_cap.rx_coe) ? "Y" : "N");
3900         } else {
3901                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
3902                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
3903                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
3904                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
3905         }
3906         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
3907                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
3908         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
3909                    priv->dma_cap.number_rx_channel);
3910         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
3911                    priv->dma_cap.number_tx_channel);
3912         seq_printf(seq, "\tEnhanced descriptors: %s\n",
3913                    (priv->dma_cap.enh_desc) ? "Y" : "N");
3914
3915         return 0;
3916 }
3917
3918 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
3919 {
3920         return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
3921 }
3922
3923 static const struct file_operations stmmac_dma_cap_fops = {
3924         .owner = THIS_MODULE,
3925         .open = stmmac_sysfs_dma_cap_open,
3926         .read = seq_read,
3927         .llseek = seq_lseek,
3928         .release = single_release,
3929 };
3930
3931 static int stmmac_init_fs(struct net_device *dev)
3932 {
3933         struct stmmac_priv *priv = netdev_priv(dev);
3934
3935         /* Create per netdev entries */
3936         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3937
3938         if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3939                 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3940
3941                 return -ENOMEM;
3942         }
3943
3944         /* Entry to report DMA RX/TX rings */
3945         priv->dbgfs_rings_status =
3946                 debugfs_create_file("descriptors_status", S_IRUGO,
3947                                     priv->dbgfs_dir, dev,
3948                                     &stmmac_rings_status_fops);
3949
3950         if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
3951                 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
3952                 debugfs_remove_recursive(priv->dbgfs_dir);
3953
3954                 return -ENOMEM;
3955         }
3956
3957         /* Entry to report the DMA HW features */
3958         priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO,
3959                                             priv->dbgfs_dir,
3960                                             dev, &stmmac_dma_cap_fops);
3961
3962         if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
3963                 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
3964                 debugfs_remove_recursive(priv->dbgfs_dir);
3965
3966                 return -ENOMEM;
3967         }
3968
3969         return 0;
3970 }
3971
3972 static void stmmac_exit_fs(struct net_device *dev)
3973 {
3974         struct stmmac_priv *priv = netdev_priv(dev);
3975
3976         debugfs_remove_recursive(priv->dbgfs_dir);
3977 }
3978 #endif /* CONFIG_DEBUG_FS */
3979
3980 static const struct net_device_ops stmmac_netdev_ops = {
3981         .ndo_open = stmmac_open,
3982         .ndo_start_xmit = stmmac_xmit,
3983         .ndo_stop = stmmac_release,
3984         .ndo_change_mtu = stmmac_change_mtu,
3985         .ndo_fix_features = stmmac_fix_features,
3986         .ndo_set_features = stmmac_set_features,
3987         .ndo_set_rx_mode = stmmac_set_rx_mode,
3988         .ndo_tx_timeout = stmmac_tx_timeout,
3989         .ndo_do_ioctl = stmmac_ioctl,
3990 #ifdef CONFIG_NET_POLL_CONTROLLER
3991         .ndo_poll_controller = stmmac_poll_controller,
3992 #endif
3993         .ndo_set_mac_address = stmmac_set_mac_address,
3994 };
3995
3996 /**
3997  *  stmmac_hw_init - Init the MAC device
3998  *  @priv: driver private structure
3999  *  Description: this function is to configure the MAC device according to
4000  *  some platform parameters or the HW capability register. It prepares the
4001  *  driver to use either ring or chain modes and to setup either enhanced or
4002  *  normal descriptors.
4003  */
4004 static int stmmac_hw_init(struct stmmac_priv *priv)
4005 {
4006         struct mac_device_info *mac;
4007
4008         /* Identify the MAC HW device */
4009         if (priv->plat->setup) {
4010                 mac = priv->plat->setup(priv);
4011         } else if (priv->plat->has_gmac) {
4012                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
4013                 mac = dwmac1000_setup(priv->ioaddr,
4014                                       priv->plat->multicast_filter_bins,
4015                                       priv->plat->unicast_filter_entries,
4016                                       &priv->synopsys_id);
4017         } else if (priv->plat->has_gmac4) {
4018                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
4019                 mac = dwmac4_setup(priv->ioaddr,
4020                                    priv->plat->multicast_filter_bins,
4021                                    priv->plat->unicast_filter_entries,
4022                                    &priv->synopsys_id);
4023         } else {
4024                 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
4025         }
4026         if (!mac)
4027                 return -ENOMEM;
4028
4029         priv->hw = mac;
4030
4031         /* dwmac-sun8i only work in chain mode */
4032         if (priv->plat->has_sun8i)
4033                 chain_mode = 1;
4034
4035         /* To use the chained or ring mode */
4036         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
4037                 priv->hw->mode = &dwmac4_ring_mode_ops;
4038         } else {
4039                 if (chain_mode) {
4040                         priv->hw->mode = &chain_mode_ops;
4041                         dev_info(priv->device, "Chain mode enabled\n");
4042                         priv->mode = STMMAC_CHAIN_MODE;
4043                 } else {
4044                         priv->hw->mode = &ring_mode_ops;
4045                         dev_info(priv->device, "Ring mode enabled\n");
4046                         priv->mode = STMMAC_RING_MODE;
4047                 }
4048         }
4049
4050         /* Get the HW capability (new GMAC newer than 3.50a) */
4051         priv->hw_cap_support = stmmac_get_hw_features(priv);
4052         if (priv->hw_cap_support) {
4053                 dev_info(priv->device, "DMA HW capability register supported\n");
4054
4055                 /* We can override some gmac/dma configuration fields: e.g.
4056                  * enh_desc, tx_coe (e.g. that are passed through the
4057                  * platform) with the values from the HW capability
4058                  * register (if supported).
4059                  */
4060                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
4061                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4062                 priv->hw->pmt = priv->plat->pmt;
4063
4064                 /* TXCOE doesn't work in thresh DMA mode */
4065                 if (priv->plat->force_thresh_dma_mode)
4066                         priv->plat->tx_coe = 0;
4067                 else
4068                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
4069
4070                 /* In case of GMAC4 rx_coe is from HW cap register. */
4071                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
4072
4073                 if (priv->dma_cap.rx_coe_type2)
4074                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4075                 else if (priv->dma_cap.rx_coe_type1)
4076                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4077
4078         } else {
4079                 dev_info(priv->device, "No HW DMA feature register supported\n");
4080         }
4081
4082         /* To use alternate (extended), normal or GMAC4 descriptor structures */
4083         if (priv->synopsys_id >= DWMAC_CORE_4_00)
4084                 priv->hw->desc = &dwmac4_desc_ops;
4085         else
4086                 stmmac_selec_desc_mode(priv);
4087
4088         if (priv->plat->rx_coe) {
4089                 priv->hw->rx_csum = priv->plat->rx_coe;
4090                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4091                 if (priv->synopsys_id < DWMAC_CORE_4_00)
4092                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4093         }
4094         if (priv->plat->tx_coe)
4095                 dev_info(priv->device, "TX Checksum insertion supported\n");
4096
4097         if (priv->plat->pmt) {
4098                 dev_info(priv->device, "Wake-Up On Lan supported\n");
4099                 device_set_wakeup_capable(priv->device, 1);
4100         }
4101
4102         if (priv->dma_cap.tsoen)
4103                 dev_info(priv->device, "TSO supported\n");
4104
4105         return 0;
4106 }
4107
4108 /**
4109  * stmmac_dvr_probe
4110  * @device: device pointer
4111  * @plat_dat: platform data pointer
4112  * @res: stmmac resource pointer
4113  * Description: this is the main probe function used to
4114  * call the alloc_etherdev, allocate the priv structure.
4115  * Return:
4116  * returns 0 on success, otherwise errno.
4117  */
4118 int stmmac_dvr_probe(struct device *device,
4119                      struct plat_stmmacenet_data *plat_dat,
4120                      struct stmmac_resources *res)
4121 {
4122         struct net_device *ndev = NULL;
4123         struct stmmac_priv *priv;
4124         int ret = 0;
4125         u32 queue;
4126
4127         ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
4128                                   MTL_MAX_TX_QUEUES,
4129                                   MTL_MAX_RX_QUEUES);
4130         if (!ndev)
4131                 return -ENOMEM;
4132
4133         SET_NETDEV_DEV(ndev, device);
4134
4135         priv = netdev_priv(ndev);
4136         priv->device = device;
4137         priv->dev = ndev;
4138
4139         stmmac_set_ethtool_ops(ndev);
4140         priv->pause = pause;
4141         priv->plat = plat_dat;
4142         priv->ioaddr = res->addr;
4143         priv->dev->base_addr = (unsigned long)res->addr;
4144
4145         priv->dev->irq = res->irq;
4146         priv->wol_irq = res->wol_irq;
4147         priv->lpi_irq = res->lpi_irq;
4148
4149         if (res->mac)
4150                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4151
4152         dev_set_drvdata(device, priv->dev);
4153
4154         /* Verify driver arguments */
4155         stmmac_verify_args();
4156
4157         /* Override with kernel parameters if supplied XXX CRS XXX
4158          * this needs to have multiple instances
4159          */
4160         if ((phyaddr >= 0) && (phyaddr <= 31))
4161                 priv->plat->phy_addr = phyaddr;
4162
4163         if (priv->plat->stmmac_rst) {
4164                 ret = reset_control_assert(priv->plat->stmmac_rst);
4165                 reset_control_deassert(priv->plat->stmmac_rst);
4166                 /* Some reset controllers have only reset callback instead of
4167                  * assert + deassert callbacks pair.
4168                  */
4169                 if (ret == -ENOTSUPP)
4170                         reset_control_reset(priv->plat->stmmac_rst);
4171         }
4172
4173         /* Init MAC and get the capabilities */
4174         ret = stmmac_hw_init(priv);
4175         if (ret)
4176                 goto error_hw_init;
4177
4178         /* Configure real RX and TX queues */
4179         netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4180         netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4181
4182         ndev->netdev_ops = &stmmac_netdev_ops;
4183
4184         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4185                             NETIF_F_RXCSUM;
4186
4187         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4188                 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
4189                 priv->tso = true;
4190                 dev_info(priv->device, "TSO feature enabled\n");
4191         }
4192         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4193         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4194 #ifdef STMMAC_VLAN_TAG_USED
4195         /* Both mac100 and gmac support receive VLAN tag detection */
4196         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
4197 #endif
4198         priv->msg_enable = netif_msg_init(debug, default_msg_level);
4199
4200         /* MTU range: 46 - hw-specific max */
4201         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4202         if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4203                 ndev->max_mtu = JUMBO_LEN;
4204         else
4205                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4206         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4207          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4208          */
4209         if ((priv->plat->maxmtu < ndev->max_mtu) &&
4210             (priv->plat->maxmtu >= ndev->min_mtu))
4211                 ndev->max_mtu = priv->plat->maxmtu;
4212         else if (priv->plat->maxmtu < ndev->min_mtu)
4213                 dev_warn(priv->device,
4214                          "%s: warning: maxmtu having invalid value (%d)\n",
4215                          __func__, priv->plat->maxmtu);
4216
4217         if (flow_ctrl)
4218                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
4219
4220         /* Rx Watchdog is available in the COREs newer than the 3.40.
4221          * In some case, for example on bugged HW this feature
4222          * has to be disable and this can be done by passing the
4223          * riwt_off field from the platform.
4224          */
4225         if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
4226                 priv->use_riwt = 1;
4227                 dev_info(priv->device,
4228                          "Enable RX Mitigation via HW Watchdog Timer\n");
4229         }
4230
4231         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4232                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4233
4234                 netif_napi_add(ndev, &rx_q->napi, stmmac_poll,
4235                                (8 * priv->plat->rx_queues_to_use));
4236         }
4237
4238         spin_lock_init(&priv->lock);
4239
4240         /* If a specific clk_csr value is passed from the platform
4241          * this means that the CSR Clock Range selection cannot be
4242          * changed at run-time and it is fixed. Viceversa the driver'll try to
4243          * set the MDC clock dynamically according to the csr actual
4244          * clock input.
4245          */
4246         if (!priv->plat->clk_csr)
4247                 stmmac_clk_csr_set(priv);
4248         else
4249                 priv->clk_csr = priv->plat->clk_csr;
4250
4251         stmmac_check_pcs_mode(priv);
4252
4253         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4254             priv->hw->pcs != STMMAC_PCS_TBI &&
4255             priv->hw->pcs != STMMAC_PCS_RTBI) {
4256                 /* MDIO bus Registration */
4257                 ret = stmmac_mdio_register(ndev);
4258                 if (ret < 0) {
4259                         dev_err(priv->device,
4260                                 "%s: MDIO bus (id: %d) registration failed",
4261                                 __func__, priv->plat->bus_id);
4262                         goto error_mdio_register;
4263                 }
4264         }
4265
4266         ret = register_netdev(ndev);
4267         if (ret) {
4268                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4269                         __func__, ret);
4270                 goto error_netdev_register;
4271         }
4272
4273         return ret;
4274
4275 error_netdev_register:
4276         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4277             priv->hw->pcs != STMMAC_PCS_TBI &&
4278             priv->hw->pcs != STMMAC_PCS_RTBI)
4279                 stmmac_mdio_unregister(ndev);
4280 error_mdio_register:
4281         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4282                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4283
4284                 netif_napi_del(&rx_q->napi);
4285         }
4286 error_hw_init:
4287         free_netdev(ndev);
4288
4289         return ret;
4290 }
4291 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4292
4293 /**
4294  * stmmac_dvr_remove
4295  * @dev: device pointer
4296  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4297  * changes the link status, releases the DMA descriptor rings.
4298  */
4299 int stmmac_dvr_remove(struct device *dev)
4300 {
4301         struct net_device *ndev = dev_get_drvdata(dev);
4302         struct stmmac_priv *priv = netdev_priv(ndev);
4303
4304         netdev_info(priv->dev, "%s: removing driver", __func__);
4305
4306         stmmac_stop_all_dma(priv);
4307
4308         priv->hw->mac->set_mac(priv->ioaddr, false);
4309         netif_carrier_off(ndev);
4310         unregister_netdev(ndev);
4311         if (priv->plat->stmmac_rst)
4312                 reset_control_assert(priv->plat->stmmac_rst);
4313         clk_disable_unprepare(priv->plat->pclk);
4314         clk_disable_unprepare(priv->plat->stmmac_clk);
4315         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4316             priv->hw->pcs != STMMAC_PCS_TBI &&
4317             priv->hw->pcs != STMMAC_PCS_RTBI)
4318                 stmmac_mdio_unregister(ndev);
4319         free_netdev(ndev);
4320
4321         return 0;
4322 }
4323 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4324
4325 /**
4326  * stmmac_suspend - suspend callback
4327  * @dev: device pointer
4328  * Description: this is the function to suspend the device and it is called
4329  * by the platform driver to stop the network queue, release the resources,
4330  * program the PMT register (for WoL), clean and release driver resources.
4331  */
4332 int stmmac_suspend(struct device *dev)
4333 {
4334         struct net_device *ndev = dev_get_drvdata(dev);
4335         struct stmmac_priv *priv = netdev_priv(ndev);
4336         unsigned long flags;
4337
4338         if (!ndev || !netif_running(ndev))
4339                 return 0;
4340
4341         if (ndev->phydev)
4342                 phy_stop(ndev->phydev);
4343
4344         spin_lock_irqsave(&priv->lock, flags);
4345
4346         netif_device_detach(ndev);
4347         stmmac_stop_all_queues(priv);
4348
4349         stmmac_disable_all_queues(priv);
4350
4351         /* Stop TX/RX DMA */
4352         stmmac_stop_all_dma(priv);
4353
4354         /* Enable Power down mode by programming the PMT regs */
4355         if (device_may_wakeup(priv->device)) {
4356                 priv->hw->mac->pmt(priv->hw, priv->wolopts);
4357                 priv->irq_wake = 1;
4358         } else {
4359                 priv->hw->mac->set_mac(priv->ioaddr, false);
4360                 pinctrl_pm_select_sleep_state(priv->device);
4361                 /* Disable clock in case of PWM is off */
4362                 clk_disable(priv->plat->pclk);
4363                 clk_disable(priv->plat->stmmac_clk);
4364         }
4365         spin_unlock_irqrestore(&priv->lock, flags);
4366
4367         priv->oldlink = false;
4368         priv->speed = SPEED_UNKNOWN;
4369         priv->oldduplex = DUPLEX_UNKNOWN;
4370         return 0;
4371 }
4372 EXPORT_SYMBOL_GPL(stmmac_suspend);
4373
4374 /**
4375  * stmmac_reset_queues_param - reset queue parameters
4376  * @dev: device pointer
4377  */
4378 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4379 {
4380         u32 rx_cnt = priv->plat->rx_queues_to_use;
4381         u32 tx_cnt = priv->plat->tx_queues_to_use;
4382         u32 queue;
4383
4384         for (queue = 0; queue < rx_cnt; queue++) {
4385                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4386
4387                 rx_q->cur_rx = 0;
4388                 rx_q->dirty_rx = 0;
4389         }
4390
4391         for (queue = 0; queue < tx_cnt; queue++) {
4392                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4393
4394                 tx_q->cur_tx = 0;
4395                 tx_q->dirty_tx = 0;
4396         }
4397 }
4398
4399 /**
4400  * stmmac_resume - resume callback
4401  * @dev: device pointer
4402  * Description: when resume this function is invoked to setup the DMA and CORE
4403  * in a usable state.
4404  */
4405 int stmmac_resume(struct device *dev)
4406 {
4407         struct net_device *ndev = dev_get_drvdata(dev);
4408         struct stmmac_priv *priv = netdev_priv(ndev);
4409         unsigned long flags;
4410
4411         if (!netif_running(ndev))
4412                 return 0;
4413
4414         /* Power Down bit, into the PM register, is cleared
4415          * automatically as soon as a magic packet or a Wake-up frame
4416          * is received. Anyway, it's better to manually clear
4417          * this bit because it can generate problems while resuming
4418          * from another devices (e.g. serial console).
4419          */
4420         if (device_may_wakeup(priv->device)) {
4421                 spin_lock_irqsave(&priv->lock, flags);
4422                 priv->hw->mac->pmt(priv->hw, 0);
4423                 spin_unlock_irqrestore(&priv->lock, flags);
4424                 priv->irq_wake = 0;
4425         } else {
4426                 pinctrl_pm_select_default_state(priv->device);
4427                 /* enable the clk previously disabled */
4428                 clk_enable(priv->plat->stmmac_clk);
4429                 clk_enable(priv->plat->pclk);
4430                 /* reset the phy so that it's ready */
4431                 if (priv->mii)
4432                         stmmac_mdio_reset(priv->mii);
4433         }
4434
4435         netif_device_attach(ndev);
4436
4437         spin_lock_irqsave(&priv->lock, flags);
4438
4439         stmmac_reset_queues_param(priv);
4440
4441         /* reset private mss value to force mss context settings at
4442          * next tso xmit (only used for gmac4).
4443          */
4444         priv->mss = 0;
4445
4446         stmmac_clear_descriptors(priv);
4447
4448         stmmac_hw_setup(ndev, false);
4449         stmmac_init_tx_coalesce(priv);
4450         stmmac_set_rx_mode(ndev);
4451
4452         stmmac_enable_all_queues(priv);
4453
4454         stmmac_start_all_queues(priv);
4455
4456         spin_unlock_irqrestore(&priv->lock, flags);
4457
4458         if (ndev->phydev)
4459                 phy_start(ndev->phydev);
4460
4461         return 0;
4462 }
4463 EXPORT_SYMBOL_GPL(stmmac_resume);
4464
4465 #ifndef MODULE
4466 static int __init stmmac_cmdline_opt(char *str)
4467 {
4468         char *opt;
4469
4470         if (!str || !*str)
4471                 return -EINVAL;
4472         while ((opt = strsep(&str, ",")) != NULL) {
4473                 if (!strncmp(opt, "debug:", 6)) {
4474                         if (kstrtoint(opt + 6, 0, &debug))
4475                                 goto err;
4476                 } else if (!strncmp(opt, "phyaddr:", 8)) {
4477                         if (kstrtoint(opt + 8, 0, &phyaddr))
4478                                 goto err;
4479                 } else if (!strncmp(opt, "buf_sz:", 7)) {
4480                         if (kstrtoint(opt + 7, 0, &buf_sz))
4481                                 goto err;
4482                 } else if (!strncmp(opt, "tc:", 3)) {
4483                         if (kstrtoint(opt + 3, 0, &tc))
4484                                 goto err;
4485                 } else if (!strncmp(opt, "watchdog:", 9)) {
4486                         if (kstrtoint(opt + 9, 0, &watchdog))
4487                                 goto err;
4488                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
4489                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
4490                                 goto err;
4491                 } else if (!strncmp(opt, "pause:", 6)) {
4492                         if (kstrtoint(opt + 6, 0, &pause))
4493                                 goto err;
4494                 } else if (!strncmp(opt, "eee_timer:", 10)) {
4495                         if (kstrtoint(opt + 10, 0, &eee_timer))
4496                                 goto err;
4497                 } else if (!strncmp(opt, "chain_mode:", 11)) {
4498                         if (kstrtoint(opt + 11, 0, &chain_mode))
4499                                 goto err;
4500                 }
4501         }
4502         return 0;
4503
4504 err:
4505         pr_err("%s: ERROR broken module parameter conversion", __func__);
4506         return -EINVAL;
4507 }
4508
4509 __setup("stmmaceth=", stmmac_cmdline_opt);
4510 #endif /* MODULE */
4511
4512 static int __init stmmac_init(void)
4513 {
4514 #ifdef CONFIG_DEBUG_FS
4515         /* Create debugfs main directory if it doesn't exist yet */
4516         if (!stmmac_fs_dir) {
4517                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
4518
4519                 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
4520                         pr_err("ERROR %s, debugfs create directory failed\n",
4521                                STMMAC_RESOURCE_NAME);
4522
4523                         return -ENOMEM;
4524                 }
4525         }
4526 #endif
4527
4528         return 0;
4529 }
4530
4531 static void __exit stmmac_exit(void)
4532 {
4533 #ifdef CONFIG_DEBUG_FS
4534         debugfs_remove_recursive(stmmac_fs_dir);
4535 #endif
4536 }
4537
4538 module_init(stmmac_init)
4539 module_exit(stmmac_exit)
4540
4541 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
4542 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
4543 MODULE_LICENSE("GPL");