drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 /*******************************************************************************
   2   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   3   ST Ethernet IPs are built around a Synopsys IP Core.
   4
   5         Copyright(C) 2007-2011 STMicroelectronics Ltd
   6
   7   This program is free software; you can redistribute it and/or modify it
   8   under the terms and conditions of the GNU General Public License,
   9   version 2, as published by the Free Software Foundation.
  10
  11   This program is distributed in the hope it will be useful, but WITHOUT
  12   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14   more details.
  15
  16   The full GNU General Public License is included in this distribution in
  17   the file called "COPYING".
  18
  19   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  20
  21   Documentation available at:
  22         http://www.stlinux.com
  23   Support available at:
  24         https://bugzilla.stlinux.com/
  25 *******************************************************************************/
  26
  27 #include <linux/clk.h>
  28 #include <linux/kernel.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/ip.h>
  31 #include <linux/tcp.h>
  32 #include <linux/skbuff.h>
  33 #include <linux/ethtool.h>
  34 #include <linux/if_ether.h>
  35 #include <linux/crc32.h>
  36 #include <linux/mii.h>
  37 #include <linux/if.h>
  38 #include <linux/if_vlan.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/slab.h>
  41 #include <linux/prefetch.h>
  42 #include <linux/pinctrl/consumer.h>
  43 #ifdef CONFIG_DEBUG_FS
  44 #include <linux/debugfs.h>
  45 #include <linux/seq_file.h>
  46 #endif /* CONFIG_DEBUG_FS */
  47 #include <linux/net_tstamp.h>
  48 #include "stmmac_ptp.h"
  49 #include "stmmac.h"
  50 #include <linux/reset.h>
  51 #include <linux/of_mdio.h>
  52 #include "dwmac1000.h"
  53
  54 #define STMMAC_ALIGN(x) L1_CACHE_ALIGN(x)
  55 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  56
  57 /* Module parameters */
  58 #define TX_TIMEO        5000
  59 static int watchdog = TX_TIMEO;
  60 module_param(watchdog, int, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  62
  63 static int debug = -1;
  64 module_param(debug, int, S_IRUGO | S_IWUSR);
  65 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  66
  67 static int phyaddr = -1;
  68 module_param(phyaddr, int, S_IRUGO);
  69 MODULE_PARM_DESC(phyaddr, "Physical device address");
  70
  71 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  72 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  73
  74 static int flow_ctrl = FLOW_OFF;
  75 module_param(flow_ctrl, int, S_IRUGO | S_IWUSR);
  76 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  77
  78 static int pause = PAUSE_TIME;
  79 module_param(pause, int, S_IRUGO | S_IWUSR);
  80 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  81
  82 #define TC_DEFAULT 64
  83 static int tc = TC_DEFAULT;
  84 module_param(tc, int, S_IRUGO | S_IWUSR);
  85 MODULE_PARM_DESC(tc, "DMA threshold control value");
  86
  87 #define DEFAULT_BUFSIZE 1536
  88 static int buf_sz = DEFAULT_BUFSIZE;
  89 module_param(buf_sz, int, S_IRUGO | S_IWUSR);
  90 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  91
  92 #define STMMAC_RX_COPYBREAK     256
  93
  94 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  95                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  96                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
  97
  98 #define STMMAC_DEFAULT_LPI_TIMER        1000
  99 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 100 module_param(eee_timer, int, S_IRUGO | S_IWUSR);
 101 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
 102 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
 103
 104 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 105  * but allow user to force to use the chain instead of the ring
 106  */
 107 static unsigned int chain_mode;
 108 module_param(chain_mode, int, S_IRUGO);
 109 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 110
 111 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 112
 113 #ifdef CONFIG_DEBUG_FS
 114 static int stmmac_init_fs(struct net_device *dev);
 115 static void stmmac_exit_fs(struct net_device *dev);
 116 #endif
 117
 118 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 119
 120 /**
 121  * stmmac_verify_args - verify the driver parameters.
 122  * Description: it checks the driver parameters and set a default in case of
 123  * errors.
 124  */
 125 static void stmmac_verify_args(void)
 126 {
 127         if (unlikely(watchdog < 0))
 128                 watchdog = TX_TIMEO;
 129         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 130                 buf_sz = DEFAULT_BUFSIZE;
 131         if (unlikely(flow_ctrl > 1))
 132                 flow_ctrl = FLOW_AUTO;
 133         else if (likely(flow_ctrl < 0))
 134                 flow_ctrl = FLOW_OFF;
 135         if (unlikely((pause < 0) || (pause > 0xffff)))
 136                 pause = PAUSE_TIME;
 137         if (eee_timer < 0)
 138                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 139 }
 140
 141 /**
 142  * stmmac_disable_all_queues - Disable all queues
 143  * @priv: driver private structure
 144  */
 145 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
 146 {
 147         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 148         u32 queue;
 149
 150         for (queue = 0; queue < rx_queues_cnt; queue++) {
 151                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 152
 153                 napi_disable(&rx_q->napi);
 154         }
 155 }
 156
 157 /**
 158  * stmmac_enable_all_queues - Enable all queues
 159  * @priv: driver private structure
 160  */
 161 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
 162 {
 163         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 164         u32 queue;
 165
 166         for (queue = 0; queue < rx_queues_cnt; queue++) {
 167                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 168
 169                 napi_enable(&rx_q->napi);
 170         }
 171 }
 172
 173 /**
 174  * stmmac_stop_all_queues - Stop all queues
 175  * @priv: driver private structure
 176  */
 177 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
 178 {
 179         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 180         u32 queue;
 181
 182         for (queue = 0; queue < tx_queues_cnt; queue++)
 183                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
 184 }
 185
 186 /**
 187  * stmmac_start_all_queues - Start all queues
 188  * @priv: driver private structure
 189  */
 190 static void stmmac_start_all_queues(struct stmmac_priv *priv)
 191 {
 192         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 193         u32 queue;
 194
 195         for (queue = 0; queue < tx_queues_cnt; queue++)
 196                 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
 197 }
 198
 199 /**
 200  * stmmac_clk_csr_set - dynamically set the MDC clock
 201  * @priv: driver private structure
 202  * Description: this is to dynamically set the MDC clock according to the csr
 203  * clock input.
 204  * Note:
 205  *      If a specific clk_csr value is passed from the platform
 206  *      this means that the CSR Clock Range selection cannot be
 207  *      changed at run-time and it is fixed (as reported in the driver
 208  *      documentation). Viceversa the driver will try to set the MDC
 209  *      clock dynamically according to the actual clock input.
 210  */
 211 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 212 {
 213         u32 clk_rate;
 214
 215         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 216
 217         /* Platform provided default clk_csr would be assumed valid
 218          * for all other cases except for the below mentioned ones.
 219          * For values higher than the IEEE 802.3 specified frequency
 220          * we can not estimate the proper divider as it is not known
 221          * the frequency of clk_csr_i. So we do not change the default
 222          * divider.
 223          */
 224         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 225                 if (clk_rate < CSR_F_35M)
 226                         priv->clk_csr = STMMAC_CSR_20_35M;
 227                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 228                         priv->clk_csr = STMMAC_CSR_35_60M;
 229                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 230                         priv->clk_csr = STMMAC_CSR_60_100M;
 231                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 232                         priv->clk_csr = STMMAC_CSR_100_150M;
 233                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 234                         priv->clk_csr = STMMAC_CSR_150_250M;
 235                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 236                         priv->clk_csr = STMMAC_CSR_250_300M;
 237         }
 238
 239         if (priv->plat->has_sun8i) {
 240                 if (clk_rate > 160000000)
 241                         priv->clk_csr = 0x03;
 242                 else if (clk_rate > 80000000)
 243                         priv->clk_csr = 0x02;
 244                 else if (clk_rate > 40000000)
 245                         priv->clk_csr = 0x01;
 246                 else
 247                         priv->clk_csr = 0;
 248         }
 249 }
 250
 251 static void print_pkt(unsigned char *buf, int len)
 252 {
 253         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 254         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 255 }
 256
 257 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 258 {
 259         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 260         u32 avail;
 261
 262         if (tx_q->dirty_tx > tx_q->cur_tx)
 263                 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
 264         else
 265                 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
 266
 267         return avail;
 268 }
 269
 270 /**
 271  * stmmac_rx_dirty - Get RX queue dirty
 272  * @priv: driver private structure
 273  * @queue: RX queue index
 274  */
 275 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 276 {
 277         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 278         u32 dirty;
 279
 280         if (rx_q->dirty_rx <= rx_q->cur_rx)
 281                 dirty = rx_q->cur_rx - rx_q->dirty_rx;
 282         else
 283                 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
 284
 285         return dirty;
 286 }
 287
 288 /**
 289  * stmmac_hw_fix_mac_speed - callback for speed selection
 290  * @priv: driver private structure
 291  * Description: on some platforms (e.g. ST), some HW system configuration
 292  * registers have to be set according to the link speed negotiated.
 293  */
 294 static inline void stmmac_hw_fix_mac_speed(struct stmmac_priv *priv)
 295 {
 296         struct net_device *ndev = priv->dev;
 297         struct phy_device *phydev = ndev->phydev;
 298
 299         if (likely(priv->plat->fix_mac_speed))
 300                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, phydev->speed);
 301 }
 302
 303 /**
 304  * stmmac_enable_eee_mode - check and enter in LPI mode
 305  * @priv: driver private structure
 306  * Description: this function is to verify and enter in LPI mode in case of
 307  * EEE.
 308  */
 309 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 310 {
 311         u32 tx_cnt = priv->plat->tx_queues_to_use;
 312         u32 queue;
 313
 314         /* check if all TX queues have the work finished */
 315         for (queue = 0; queue < tx_cnt; queue++) {
 316                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 317
 318                 if (tx_q->dirty_tx != tx_q->cur_tx)
 319                         return; /* still unfinished work */
 320         }
 321
 322         /* Check and enter in LPI mode */
 323         if (!priv->tx_path_in_lpi_mode)
 324                 priv->hw->mac->set_eee_mode(priv->hw,
 325                                             priv->plat->en_tx_lpi_clockgating);
 326 }
 327
 328 /**
 329  * stmmac_disable_eee_mode - disable and exit from LPI mode
 330  * @priv: driver private structure
 331  * Description: this function is to exit and disable EEE in case of
 332  * LPI state is true. This is called by the xmit.
 333  */
 334 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 335 {
 336         priv->hw->mac->reset_eee_mode(priv->hw);
 337         del_timer_sync(&priv->eee_ctrl_timer);
 338         priv->tx_path_in_lpi_mode = false;
 339 }
 340
 341 /**
 342  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 343  * @arg : data hook
 344  * Description:
 345  *  if there is no data transfer and if we are not in LPI state,
 346  *  then MAC Transmitter can be moved to LPI state.
 347  */
 348 static void stmmac_eee_ctrl_timer(struct timer_list *t)
 349 {
 350         struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
 351
 352         stmmac_enable_eee_mode(priv);
 353         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 354 }
 355
 356 /**
 357  * stmmac_eee_init - init EEE
 358  * @priv: driver private structure
 359  * Description:
 360  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 361  *  can also manage EEE, this function enable the LPI state and start related
 362  *  timer.
 363  */
 364 bool stmmac_eee_init(struct stmmac_priv *priv)
 365 {
 366         struct net_device *ndev = priv->dev;
 367         int interface = priv->plat->interface;
 368         unsigned long flags;
 369         bool ret = false;
 370
 371         if ((interface != PHY_INTERFACE_MODE_MII) &&
 372             (interface != PHY_INTERFACE_MODE_GMII) &&
 373             !phy_interface_mode_is_rgmii(interface))
 374                 goto out;
 375
 376         /* Using PCS we cannot dial with the phy registers at this stage
 377          * so we do not support extra feature like EEE.
 378          */
 379         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 380             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 381             (priv->hw->pcs == STMMAC_PCS_RTBI))
 382                 goto out;
 383
 384         /* MAC core supports the EEE feature. */
 385         if (priv->dma_cap.eee) {
 386                 int tx_lpi_timer = priv->tx_lpi_timer;
 387
 388                 /* Check if the PHY supports EEE */
 389                 if (phy_init_eee(ndev->phydev, 1)) {
 390                         /* To manage at run-time if the EEE cannot be supported
 391                          * anymore (for example because the lp caps have been
 392                          * changed).
 393                          * In that case the driver disable own timers.
 394                          */
 395                         spin_lock_irqsave(&priv->lock, flags);
 396                         if (priv->eee_active) {
 397                                 netdev_dbg(priv->dev, "disable EEE\n");
 398                                 del_timer_sync(&priv->eee_ctrl_timer);
 399                                 priv->hw->mac->set_eee_timer(priv->hw, 0,
 400                                                              tx_lpi_timer);
 401                         }
 402                         priv->eee_active = 0;
 403                         spin_unlock_irqrestore(&priv->lock, flags);
 404                         goto out;
 405                 }
 406                 /* Activate the EEE and start timers */
 407                 spin_lock_irqsave(&priv->lock, flags);
 408                 if (!priv->eee_active) {
 409                         priv->eee_active = 1;
 410                         timer_setup(&priv->eee_ctrl_timer,
 411                                     stmmac_eee_ctrl_timer, 0);
 412                         mod_timer(&priv->eee_ctrl_timer,
 413                                   STMMAC_LPI_T(eee_timer));
 414
 415                         priv->hw->mac->set_eee_timer(priv->hw,
 416                                                      STMMAC_DEFAULT_LIT_LS,
 417                                                      tx_lpi_timer);
 418                 }
 419                 /* Set HW EEE according to the speed */
 420                 priv->hw->mac->set_eee_pls(priv->hw, ndev->phydev->link);
 421
 422                 ret = true;
 423                 spin_unlock_irqrestore(&priv->lock, flags);
 424
 425                 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 426         }
 427 out:
 428         return ret;
 429 }
 430
 431 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 432  * @priv: driver private structure
 433  * @p : descriptor pointer
 434  * @skb : the socket buffer
 435  * Description :
 436  * This function will read timestamp from the descriptor & pass it to stack.
 437  * and also perform some sanity checks.
 438  */
 439 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 440                                    struct dma_desc *p, struct sk_buff *skb)
 441 {
 442         struct skb_shared_hwtstamps shhwtstamp;
 443         u64 ns;
 444
 445         if (!priv->hwts_tx_en)
 446                 return;
 447
 448         /* exit if skb doesn't support hw tstamp */
 449         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 450                 return;
 451
 452         /* check tx tstamp status */
 453         if (priv->hw->desc->get_tx_timestamp_status(p)) {
 454                 /* get the valid tstamp */
 455                 ns = priv->hw->desc->get_timestamp(p, priv->adv_ts);
 456
 457                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 458                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 459
 460                 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 461                 /* pass tstamp to stack */
 462                 skb_tstamp_tx(skb, &shhwtstamp);
 463         }
 464
 465         return;
 466 }
 467
 468 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 469  * @priv: driver private structure
 470  * @p : descriptor pointer
 471  * @np : next descriptor pointer
 472  * @skb : the socket buffer
 473  * Description :
 474  * This function will read received packet's timestamp from the descriptor
 475  * and pass it to stack. It also perform some sanity checks.
 476  */
 477 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 478                                    struct dma_desc *np, struct sk_buff *skb)
 479 {
 480         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 481         struct dma_desc *desc = p;
 482         u64 ns;
 483
 484         if (!priv->hwts_rx_en)
 485                 return;
 486         /* For GMAC4, the valid timestamp is from CTX next desc. */
 487         if (priv->plat->has_gmac4)
 488                 desc = np;
 489
 490         /* Check if timestamp is available */
 491         if (priv->hw->desc->get_rx_timestamp_status(p, np, priv->adv_ts)) {
 492                 ns = priv->hw->desc->get_timestamp(desc, priv->adv_ts);
 493                 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 494                 shhwtstamp = skb_hwtstamps(skb);
 495                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 496                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 497         } else  {
 498                 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
 499         }
 500 }
 501
 502 /**
 503  *  stmmac_hwtstamp_ioctl - control hardware timestamping.
 504  *  @dev: device pointer.
 505  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 506  *  a proprietary structure used to pass information to the driver.
 507  *  Description:
 508  *  This function configures the MAC to enable/disable both outgoing(TX)
 509  *  and incoming(RX) packets time stamping based on user input.
 510  *  Return Value:
 511  *  0 on success and an appropriate -ve integer on failure.
 512  */
 513 static int stmmac_hwtstamp_ioctl(struct net_device *dev, struct ifreq *ifr)
 514 {
 515         struct stmmac_priv *priv = netdev_priv(dev);
 516         struct hwtstamp_config config;
 517         struct timespec64 now;
 518         u64 temp = 0;
 519         u32 ptp_v2 = 0;
 520         u32 tstamp_all = 0;
 521         u32 ptp_over_ipv4_udp = 0;
 522         u32 ptp_over_ipv6_udp = 0;
 523         u32 ptp_over_ethernet = 0;
 524         u32 snap_type_sel = 0;
 525         u32 ts_master_en = 0;
 526         u32 ts_event_en = 0;
 527         u32 value = 0;
 528         u32 sec_inc;
 529
 530         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 531                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 532                 priv->hwts_tx_en = 0;
 533                 priv->hwts_rx_en = 0;
 534
 535                 return -EOPNOTSUPP;
 536         }
 537
 538         if (copy_from_user(&config, ifr->ifr_data,
 539                            sizeof(struct hwtstamp_config)))
 540                 return -EFAULT;
 541
 542         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 543                    __func__, config.flags, config.tx_type, config.rx_filter);
 544
 545         /* reserved for future extensions */
 546         if (config.flags)
 547                 return -EINVAL;
 548
 549         if (config.tx_type != HWTSTAMP_TX_OFF &&
 550             config.tx_type != HWTSTAMP_TX_ON)
 551                 return -ERANGE;
 552
 553         if (priv->adv_ts) {
 554                 switch (config.rx_filter) {
 555                 case HWTSTAMP_FILTER_NONE:
 556                         /* time stamp no incoming packet at all */
 557                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 558                         break;
 559
 560                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 561                         /* PTP v1, UDP, any kind of event packet */
 562                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 563                         /* take time stamp for all event messages */
 564                         if (priv->plat->has_gmac4)
 565                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 566                         else
 567                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 568
 569                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 570                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 571                         break;
 572
 573                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 574                         /* PTP v1, UDP, Sync packet */
 575                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 576                         /* take time stamp for SYNC messages only */
 577                         ts_event_en = PTP_TCR_TSEVNTENA;
 578
 579                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 580                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 581                         break;
 582
 583                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 584                         /* PTP v1, UDP, Delay_req packet */
 585                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 586                         /* take time stamp for Delay_Req messages only */
 587                         ts_master_en = PTP_TCR_TSMSTRENA;
 588                         ts_event_en = PTP_TCR_TSEVNTENA;
 589
 590                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 591                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 592                         break;
 593
 594                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 595                         /* PTP v2, UDP, any kind of event packet */
 596                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 597                         ptp_v2 = PTP_TCR_TSVER2ENA;
 598                         /* take time stamp for all event messages */
 599                         if (priv->plat->has_gmac4)
 600                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 601                         else
 602                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 603
 604                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 605                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 606                         break;
 607
 608                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 609                         /* PTP v2, UDP, Sync packet */
 610                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 611                         ptp_v2 = PTP_TCR_TSVER2ENA;
 612                         /* take time stamp for SYNC messages only */
 613                         ts_event_en = PTP_TCR_TSEVNTENA;
 614
 615                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 616                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 617                         break;
 618
 619                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 620                         /* PTP v2, UDP, Delay_req packet */
 621                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 622                         ptp_v2 = PTP_TCR_TSVER2ENA;
 623                         /* take time stamp for Delay_Req messages only */
 624                         ts_master_en = PTP_TCR_TSMSTRENA;
 625                         ts_event_en = PTP_TCR_TSEVNTENA;
 626
 627                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 628                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 629                         break;
 630
 631                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 632                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 633                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 634                         ptp_v2 = PTP_TCR_TSVER2ENA;
 635                         /* take time stamp for all event messages */
 636                         if (priv->plat->has_gmac4)
 637                                 snap_type_sel = PTP_GMAC4_TCR_SNAPTYPSEL_1;
 638                         else
 639                                 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 640
 641                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 642                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 643                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 644                         break;
 645
 646                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 647                         /* PTP v2/802.AS1, any layer, Sync packet */
 648                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 649                         ptp_v2 = PTP_TCR_TSVER2ENA;
 650                         /* take time stamp for SYNC messages only */
 651                         ts_event_en = PTP_TCR_TSEVNTENA;
 652
 653                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 654                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 655                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 656                         break;
 657
 658                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 659                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 660                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 661                         ptp_v2 = PTP_TCR_TSVER2ENA;
 662                         /* take time stamp for Delay_Req messages only */
 663                         ts_master_en = PTP_TCR_TSMSTRENA;
 664                         ts_event_en = PTP_TCR_TSEVNTENA;
 665
 666                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 667                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 668                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 669                         break;
 670
 671                 case HWTSTAMP_FILTER_NTP_ALL:
 672                 case HWTSTAMP_FILTER_ALL:
 673                         /* time stamp any incoming packet */
 674                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 675                         tstamp_all = PTP_TCR_TSENALL;
 676                         break;
 677
 678                 default:
 679                         return -ERANGE;
 680                 }
 681         } else {
 682                 switch (config.rx_filter) {
 683                 case HWTSTAMP_FILTER_NONE:
 684                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 685                         break;
 686                 default:
 687                         /* PTP v1, UDP, any kind of event packet */
 688                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 689                         break;
 690                 }
 691         }
 692         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 693         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 694
 695         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 696                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, 0);
 697         else {
 698                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 699                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 700                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 701                          ts_master_en | snap_type_sel);
 702                 priv->hw->ptp->config_hw_tstamping(priv->ptpaddr, value);
 703
 704                 /* program Sub Second Increment reg */
 705                 sec_inc = priv->hw->ptp->config_sub_second_increment(
 706                         priv->ptpaddr, priv->plat->clk_ptp_rate,
 707                         priv->plat->has_gmac4);
 708                 temp = div_u64(1000000000ULL, sec_inc);
 709
 710                 /* calculate default added value:
 711                  * formula is :
 712                  * addend = (2^32)/freq_div_ratio;
 713                  * where, freq_div_ratio = 1e9ns/sec_inc
 714                  */
 715                 temp = (u64)(temp << 32);
 716                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 717                 priv->hw->ptp->config_addend(priv->ptpaddr,
 718                                              priv->default_addend);
 719
 720                 /* initialize system time */
 721                 ktime_get_real_ts64(&now);
 722
 723                 /* lower 32 bits of tv_sec are safe until y2106 */
 724                 priv->hw->ptp->init_systime(priv->ptpaddr, (u32)now.tv_sec,
 725                                             now.tv_nsec);
 726         }
 727
 728         return copy_to_user(ifr->ifr_data, &config,
 729                             sizeof(struct hwtstamp_config)) ? -EFAULT : 0;
 730 }
 731
 732 /**
 733  * stmmac_init_ptp - init PTP
 734  * @priv: driver private structure
 735  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 736  * This is done by looking at the HW cap. register.
 737  * This function also registers the ptp driver.
 738  */
 739 static int stmmac_init_ptp(struct stmmac_priv *priv)
 740 {
 741         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 742                 return -EOPNOTSUPP;
 743
 744         priv->adv_ts = 0;
 745         /* Check if adv_ts can be enabled for dwmac 4.x core */
 746         if (priv->plat->has_gmac4 && priv->dma_cap.atime_stamp)
 747                 priv->adv_ts = 1;
 748         /* Dwmac 3.x core with extend_desc can support adv_ts */
 749         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 750                 priv->adv_ts = 1;
 751
 752         if (priv->dma_cap.time_stamp)
 753                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 754
 755         if (priv->adv_ts)
 756                 netdev_info(priv->dev,
 757                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 758
 759         priv->hw->ptp = &stmmac_ptp;
 760         priv->hwts_tx_en = 0;
 761         priv->hwts_rx_en = 0;
 762
 763         stmmac_ptp_register(priv);
 764
 765         return 0;
 766 }
 767
 768 static void stmmac_release_ptp(struct stmmac_priv *priv)
 769 {
 770         if (priv->plat->clk_ptp_ref)
 771                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 772         stmmac_ptp_unregister(priv);
 773 }
 774
 775 /**
 776  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 777  *  @priv: driver private structure
 778  *  Description: It is used for configuring the flow control in all queues
 779  */
 780 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 781 {
 782         u32 tx_cnt = priv->plat->tx_queues_to_use;
 783
 784         priv->hw->mac->flow_ctrl(priv->hw, duplex, priv->flow_ctrl,
 785                                  priv->pause, tx_cnt);
 786 }
 787
 788 /**
 789  * stmmac_adjust_link - adjusts the link parameters
 790  * @dev: net device structure
 791  * Description: this is the helper called by the physical abstraction layer
 792  * drivers to communicate the phy link status. According the speed and duplex
 793  * this driver can invoke registered glue-logic as well.
 794  * It also invoke the eee initialization because it could happen when switch
 795  * on different networks (that are eee capable).
 796  */
 797 static void stmmac_adjust_link(struct net_device *dev)
 798 {
 799         struct stmmac_priv *priv = netdev_priv(dev);
 800         struct phy_device *phydev = dev->phydev;
 801         unsigned long flags;
 802         bool new_state = false;
 803
 804         if (!phydev)
 805                 return;
 806
 807         spin_lock_irqsave(&priv->lock, flags);
 808
 809         if (phydev->link) {
 810                 u32 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 811
 812                 /* Now we make sure that we can be in full duplex mode.
 813                  * If not, we operate in half-duplex mode. */
 814                 if (phydev->duplex != priv->oldduplex) {
 815                         new_state = true;
 816                         if (!phydev->duplex)
 817                                 ctrl &= ~priv->hw->link.duplex;
 818                         else
 819                                 ctrl |= priv->hw->link.duplex;
 820                         priv->oldduplex = phydev->duplex;
 821                 }
 822                 /* Flow Control operation */
 823                 if (phydev->pause)
 824                         stmmac_mac_flow_ctrl(priv, phydev->duplex);
 825
 826                 if (phydev->speed != priv->speed) {
 827                         new_state = true;
 828                         ctrl &= ~priv->hw->link.speed_mask;
 829                         switch (phydev->speed) {
 830                         case SPEED_1000:
 831                                 ctrl |= priv->hw->link.speed1000;
 832                                 break;
 833                         case SPEED_100:
 834                                 ctrl |= priv->hw->link.speed100;
 835                                 break;
 836                         case SPEED_10:
 837                                 ctrl |= priv->hw->link.speed10;
 838                                 break;
 839                         default:
 840                                 netif_warn(priv, link, priv->dev,
 841                                            "broken speed: %d\n", phydev->speed);
 842                                 phydev->speed = SPEED_UNKNOWN;
 843                                 break;
 844                         }
 845                         if (phydev->speed != SPEED_UNKNOWN)
 846                                 stmmac_hw_fix_mac_speed(priv);
 847                         priv->speed = phydev->speed;
 848                 }
 849
 850                 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 851
 852                 if (!priv->oldlink) {
 853                         new_state = true;
 854                         priv->oldlink = true;
 855                 }
 856         } else if (priv->oldlink) {
 857                 new_state = true;
 858                 priv->oldlink = false;
 859                 priv->speed = SPEED_UNKNOWN;
 860                 priv->oldduplex = DUPLEX_UNKNOWN;
 861         }
 862
 863         if (new_state && netif_msg_link(priv))
 864                 phy_print_status(phydev);
 865
 866         spin_unlock_irqrestore(&priv->lock, flags);
 867
 868         if (phydev->is_pseudo_fixed_link)
 869                 /* Stop PHY layer to call the hook to adjust the link in case
 870                  * of a switch is attached to the stmmac driver.
 871                  */
 872                 phydev->irq = PHY_IGNORE_INTERRUPT;
 873         else
 874                 /* At this stage, init the EEE if supported.
 875                  * Never called in case of fixed_link.
 876                  */
 877                 priv->eee_enabled = stmmac_eee_init(priv);
 878 }
 879
 880 /**
 881  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 882  * @priv: driver private structure
 883  * Description: this is to verify if the HW supports the PCS.
 884  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 885  * configured for the TBI, RTBI, or SGMII PHY interface.
 886  */
 887 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 888 {
 889         int interface = priv->plat->interface;
 890
 891         if (priv->dma_cap.pcs) {
 892                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 893                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 894                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 895                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 896                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 897                         priv->hw->pcs = STMMAC_PCS_RGMII;
 898                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 899                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 900                         priv->hw->pcs = STMMAC_PCS_SGMII;
 901                 }
 902         }
 903 }
 904
 905 /**
 906  * stmmac_init_phy - PHY initialization
 907  * @dev: net device structure
 908  * Description: it initializes the driver's PHY state, and attaches the PHY
 909  * to the mac driver.
 910  *  Return value:
 911  *  0 on success
 912  */
 913 static int stmmac_init_phy(struct net_device *dev)
 914 {
 915         struct stmmac_priv *priv = netdev_priv(dev);
 916         struct phy_device *phydev;
 917         char phy_id_fmt[MII_BUS_ID_SIZE + 3];
 918         char bus_id[MII_BUS_ID_SIZE];
 919         int interface = priv->plat->interface;
 920         int max_speed = priv->plat->max_speed;
 921         priv->oldlink = false;
 922         priv->speed = SPEED_UNKNOWN;
 923         priv->oldduplex = DUPLEX_UNKNOWN;
 924
 925         if (priv->plat->phy_node) {
 926                 phydev = of_phy_connect(dev, priv->plat->phy_node,
 927                                         &stmmac_adjust_link, 0, interface);
 928         } else {
 929                 snprintf(bus_id, MII_BUS_ID_SIZE, "stmmac-%x",
 930                          priv->plat->bus_id);
 931
 932                 snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT, bus_id,
 933                          priv->plat->phy_addr);
 934                 netdev_dbg(priv->dev, "%s: trying to attach to %s\n", __func__,
 935                            phy_id_fmt);
 936
 937                 phydev = phy_connect(dev, phy_id_fmt, &stmmac_adjust_link,
 938                                      interface);
 939         }
 940
 941         if (IS_ERR_OR_NULL(phydev)) {
 942                 netdev_err(priv->dev, "Could not attach to PHY\n");
 943                 if (!phydev)
 944                         return -ENODEV;
 945
 946                 return PTR_ERR(phydev);
 947         }
 948
 949         /* Stop Advertising 1000BASE Capability if interface is not GMII */
 950         if ((interface == PHY_INTERFACE_MODE_MII) ||
 951             (interface == PHY_INTERFACE_MODE_RMII) ||
 952                 (max_speed < 1000 && max_speed > 0))
 953                 phydev->advertising &= ~(SUPPORTED_1000baseT_Half |
 954                                          SUPPORTED_1000baseT_Full);
 955
 956         /*
 957          * Broken HW is sometimes missing the pull-up resistor on the
 958          * MDIO line, which results in reads to non-existent devices returning
 959          * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
 960          * device as well.
 961          * Note: phydev->phy_id is the result of reading the UID PHY registers.
 962          */
 963         if (!priv->plat->phy_node && phydev->phy_id == 0) {
 964                 phy_disconnect(phydev);
 965                 return -ENODEV;
 966         }
 967
 968         /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
 969          * subsequent PHY polling, make sure we force a link transition if
 970          * we have a UP/DOWN/UP transition
 971          */
 972         if (phydev->is_pseudo_fixed_link)
 973                 phydev->irq = PHY_POLL;
 974
 975         phy_attached_info(phydev);
 976         return 0;
 977 }
 978
 979 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
 980 {
 981         u32 rx_cnt = priv->plat->rx_queues_to_use;
 982         void *head_rx;
 983         u32 queue;
 984
 985         /* Display RX rings */
 986         for (queue = 0; queue < rx_cnt; queue++) {
 987                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 988
 989                 pr_info("\tRX Queue %u rings\n", queue);
 990
 991                 if (priv->extend_desc)
 992                         head_rx = (void *)rx_q->dma_erx;
 993                 else
 994                         head_rx = (void *)rx_q->dma_rx;
 995
 996                 /* Display RX ring */
 997                 priv->hw->desc->display_ring(head_rx, DMA_RX_SIZE, true);
 998         }
 999 }
1000
1001 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1002 {
1003         u32 tx_cnt = priv->plat->tx_queues_to_use;
1004         void *head_tx;
1005         u32 queue;
1006
1007         /* Display TX rings */
1008         for (queue = 0; queue < tx_cnt; queue++) {
1009                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1010
1011                 pr_info("\tTX Queue %d rings\n", queue);
1012
1013                 if (priv->extend_desc)
1014                         head_tx = (void *)tx_q->dma_etx;
1015                 else
1016                         head_tx = (void *)tx_q->dma_tx;
1017
1018                 priv->hw->desc->display_ring(head_tx, DMA_TX_SIZE, false);
1019         }
1020 }
1021
1022 static void stmmac_display_rings(struct stmmac_priv *priv)
1023 {
1024         /* Display RX ring */
1025         stmmac_display_rx_rings(priv);
1026
1027         /* Display TX ring */
1028         stmmac_display_tx_rings(priv);
1029 }
1030
1031 static int stmmac_set_bfsize(int mtu, int bufsize)
1032 {
1033         int ret = bufsize;
1034
1035         if (mtu >= BUF_SIZE_4KiB)
1036                 ret = BUF_SIZE_8KiB;
1037         else if (mtu >= BUF_SIZE_2KiB)
1038                 ret = BUF_SIZE_4KiB;
1039         else if (mtu > DEFAULT_BUFSIZE)
1040                 ret = BUF_SIZE_2KiB;
1041         else
1042                 ret = DEFAULT_BUFSIZE;
1043
1044         return ret;
1045 }
1046
1047 /**
1048  * stmmac_clear_rx_descriptors - clear RX descriptors
1049  * @priv: driver private structure
1050  * @queue: RX queue index
1051  * Description: this function is called to clear the RX descriptors
1052  * in case of both basic and extended descriptors are used.
1053  */
1054 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1055 {
1056         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1057         int i;
1058
1059         /* Clear the RX descriptors */
1060         for (i = 0; i < DMA_RX_SIZE; i++)
1061                 if (priv->extend_desc)
1062                         priv->hw->desc->init_rx_desc(&rx_q->dma_erx[i].basic,
1063                                                      priv->use_riwt, priv->mode,
1064                                                      (i == DMA_RX_SIZE - 1));
1065                 else
1066                         priv->hw->desc->init_rx_desc(&rx_q->dma_rx[i],
1067                                                      priv->use_riwt, priv->mode,
1068                                                      (i == DMA_RX_SIZE - 1));
1069 }
1070
1071 /**
1072  * stmmac_clear_tx_descriptors - clear tx descriptors
1073  * @priv: driver private structure
1074  * @queue: TX queue index.
1075  * Description: this function is called to clear the TX descriptors
1076  * in case of both basic and extended descriptors are used.
1077  */
1078 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1079 {
1080         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1081         int i;
1082
1083         /* Clear the TX descriptors */
1084         for (i = 0; i < DMA_TX_SIZE; i++)
1085                 if (priv->extend_desc)
1086                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1087                                                      priv->mode,
1088                                                      (i == DMA_TX_SIZE - 1));
1089                 else
1090                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1091                                                      priv->mode,
1092                                                      (i == DMA_TX_SIZE - 1));
1093 }
1094
1095 /**
1096  * stmmac_clear_descriptors - clear descriptors
1097  * @priv: driver private structure
1098  * Description: this function is called to clear the TX and RX descriptors
1099  * in case of both basic and extended descriptors are used.
1100  */
1101 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1102 {
1103         u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1104         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1105         u32 queue;
1106
1107         /* Clear the RX descriptors */
1108         for (queue = 0; queue < rx_queue_cnt; queue++)
1109                 stmmac_clear_rx_descriptors(priv, queue);
1110
1111         /* Clear the TX descriptors */
1112         for (queue = 0; queue < tx_queue_cnt; queue++)
1113                 stmmac_clear_tx_descriptors(priv, queue);
1114 }
1115
1116 /**
1117  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1118  * @priv: driver private structure
1119  * @p: descriptor pointer
1120  * @i: descriptor index
1121  * @flags: gfp flag
1122  * @queue: RX queue index
1123  * Description: this function is called to allocate a receive buffer, perform
1124  * the DMA mapping and init the descriptor.
1125  */
1126 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1127                                   int i, gfp_t flags, u32 queue)
1128 {
1129         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1130         struct sk_buff *skb;
1131
1132         skb = __netdev_alloc_skb_ip_align(priv->dev, priv->dma_buf_sz, flags);
1133         if (!skb) {
1134                 netdev_err(priv->dev,
1135                            "%s: Rx init fails; skb is NULL\n", __func__);
1136                 return -ENOMEM;
1137         }
1138         rx_q->rx_skbuff[i] = skb;
1139         rx_q->rx_skbuff_dma[i] = dma_map_single(priv->device, skb->data,
1140                                                 priv->dma_buf_sz,
1141                                                 DMA_FROM_DEVICE);
1142         if (dma_mapping_error(priv->device, rx_q->rx_skbuff_dma[i])) {
1143                 netdev_err(priv->dev, "%s: DMA mapping error\n", __func__);
1144                 dev_kfree_skb_any(skb);
1145                 return -EINVAL;
1146         }
1147
1148         if (priv->synopsys_id >= DWMAC_CORE_4_00)
1149                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1150         else
1151                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[i]);
1152
1153         if ((priv->hw->mode->init_desc3) &&
1154             (priv->dma_buf_sz == BUF_SIZE_16KiB))
1155                 priv->hw->mode->init_desc3(p);
1156
1157         return 0;
1158 }
1159
1160 /**
1161  * stmmac_free_rx_buffer - free RX dma buffers
1162  * @priv: private structure
1163  * @queue: RX queue index
1164  * @i: buffer index.
1165  */
1166 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1167 {
1168         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1169
1170         if (rx_q->rx_skbuff[i]) {
1171                 dma_unmap_single(priv->device, rx_q->rx_skbuff_dma[i],
1172                                  priv->dma_buf_sz, DMA_FROM_DEVICE);
1173                 dev_kfree_skb_any(rx_q->rx_skbuff[i]);
1174         }
1175         rx_q->rx_skbuff[i] = NULL;
1176 }
1177
1178 /**
1179  * stmmac_free_tx_buffer - free RX dma buffers
1180  * @priv: private structure
1181  * @queue: RX queue index
1182  * @i: buffer index.
1183  */
1184 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1185 {
1186         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1187
1188         if (tx_q->tx_skbuff_dma[i].buf) {
1189                 if (tx_q->tx_skbuff_dma[i].map_as_page)
1190                         dma_unmap_page(priv->device,
1191                                        tx_q->tx_skbuff_dma[i].buf,
1192                                        tx_q->tx_skbuff_dma[i].len,
1193                                        DMA_TO_DEVICE);
1194                 else
1195                         dma_unmap_single(priv->device,
1196                                          tx_q->tx_skbuff_dma[i].buf,
1197                                          tx_q->tx_skbuff_dma[i].len,
1198                                          DMA_TO_DEVICE);
1199         }
1200
1201         if (tx_q->tx_skbuff[i]) {
1202                 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1203                 tx_q->tx_skbuff[i] = NULL;
1204                 tx_q->tx_skbuff_dma[i].buf = 0;
1205                 tx_q->tx_skbuff_dma[i].map_as_page = false;
1206         }
1207 }
1208
1209 /**
1210  * init_dma_rx_desc_rings - init the RX descriptor rings
1211  * @dev: net device structure
1212  * @flags: gfp flag.
1213  * Description: this function initializes the DMA RX descriptors
1214  * and allocates the socket buffers. It supports the chained and ring
1215  * modes.
1216  */
1217 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1218 {
1219         struct stmmac_priv *priv = netdev_priv(dev);
1220         u32 rx_count = priv->plat->rx_queues_to_use;
1221         unsigned int bfsize = 0;
1222         int ret = -ENOMEM;
1223         int queue;
1224         int i;
1225
1226         if (priv->hw->mode->set_16kib_bfsize)
1227                 bfsize = priv->hw->mode->set_16kib_bfsize(dev->mtu);
1228
1229         if (bfsize < BUF_SIZE_16KiB)
1230                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1231
1232         priv->dma_buf_sz = bfsize;
1233
1234         /* RX INITIALIZATION */
1235         netif_dbg(priv, probe, priv->dev,
1236                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1237
1238         for (queue = 0; queue < rx_count; queue++) {
1239                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1240
1241                 netif_dbg(priv, probe, priv->dev,
1242                           "(%s) dma_rx_phy=0x%08x\n", __func__,
1243                           (u32)rx_q->dma_rx_phy);
1244
1245                 for (i = 0; i < DMA_RX_SIZE; i++) {
1246                         struct dma_desc *p;
1247
1248                         if (priv->extend_desc)
1249                                 p = &((rx_q->dma_erx + i)->basic);
1250                         else
1251                                 p = rx_q->dma_rx + i;
1252
1253                         ret = stmmac_init_rx_buffers(priv, p, i, flags,
1254                                                      queue);
1255                         if (ret)
1256                                 goto err_init_rx_buffers;
1257
1258                         netif_dbg(priv, probe, priv->dev, "[%p]\t[%p]\t[%x]\n",
1259                                   rx_q->rx_skbuff[i], rx_q->rx_skbuff[i]->data,
1260                                   (unsigned int)rx_q->rx_skbuff_dma[i]);
1261                 }
1262
1263                 rx_q->cur_rx = 0;
1264                 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1265
1266                 stmmac_clear_rx_descriptors(priv, queue);
1267
1268                 /* Setup the chained descriptor addresses */
1269                 if (priv->mode == STMMAC_CHAIN_MODE) {
1270                         if (priv->extend_desc)
1271                                 priv->hw->mode->init(rx_q->dma_erx,
1272                                                      rx_q->dma_rx_phy,
1273                                                      DMA_RX_SIZE, 1);
1274                         else
1275                                 priv->hw->mode->init(rx_q->dma_rx,
1276                                                      rx_q->dma_rx_phy,
1277                                                      DMA_RX_SIZE, 0);
1278                 }
1279         }
1280
1281         buf_sz = bfsize;
1282
1283         return 0;
1284
1285 err_init_rx_buffers:
1286         while (queue >= 0) {
1287                 while (--i >= 0)
1288                         stmmac_free_rx_buffer(priv, queue, i);
1289
1290                 if (queue == 0)
1291                         break;
1292
1293                 i = DMA_RX_SIZE;
1294                 queue--;
1295         }
1296
1297         return ret;
1298 }
1299
1300 /**
1301  * init_dma_tx_desc_rings - init the TX descriptor rings
1302  * @dev: net device structure.
1303  * Description: this function initializes the DMA TX descriptors
1304  * and allocates the socket buffers. It supports the chained and ring
1305  * modes.
1306  */
1307 static int init_dma_tx_desc_rings(struct net_device *dev)
1308 {
1309         struct stmmac_priv *priv = netdev_priv(dev);
1310         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1311         u32 queue;
1312         int i;
1313
1314         for (queue = 0; queue < tx_queue_cnt; queue++) {
1315                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1316
1317                 netif_dbg(priv, probe, priv->dev,
1318                           "(%s) dma_tx_phy=0x%08x\n", __func__,
1319                          (u32)tx_q->dma_tx_phy);
1320
1321                 /* Setup the chained descriptor addresses */
1322                 if (priv->mode == STMMAC_CHAIN_MODE) {
1323                         if (priv->extend_desc)
1324                                 priv->hw->mode->init(tx_q->dma_etx,
1325                                                      tx_q->dma_tx_phy,
1326                                                      DMA_TX_SIZE, 1);
1327                         else
1328                                 priv->hw->mode->init(tx_q->dma_tx,
1329                                                      tx_q->dma_tx_phy,
1330                                                      DMA_TX_SIZE, 0);
1331                 }
1332
1333                 for (i = 0; i < DMA_TX_SIZE; i++) {
1334                         struct dma_desc *p;
1335                         if (priv->extend_desc)
1336                                 p = &((tx_q->dma_etx + i)->basic);
1337                         else
1338                                 p = tx_q->dma_tx + i;
1339
1340                         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1341                                 p->des0 = 0;
1342                                 p->des1 = 0;
1343                                 p->des2 = 0;
1344                                 p->des3 = 0;
1345                         } else {
1346                                 p->des2 = 0;
1347                         }
1348
1349                         tx_q->tx_skbuff_dma[i].buf = 0;
1350                         tx_q->tx_skbuff_dma[i].map_as_page = false;
1351                         tx_q->tx_skbuff_dma[i].len = 0;
1352                         tx_q->tx_skbuff_dma[i].last_segment = false;
1353                         tx_q->tx_skbuff[i] = NULL;
1354                 }
1355
1356                 tx_q->dirty_tx = 0;
1357                 tx_q->cur_tx = 0;
1358
1359                 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1360         }
1361
1362         return 0;
1363 }
1364
1365 /**
1366  * init_dma_desc_rings - init the RX/TX descriptor rings
1367  * @dev: net device structure
1368  * @flags: gfp flag.
1369  * Description: this function initializes the DMA RX/TX descriptors
1370  * and allocates the socket buffers. It supports the chained and ring
1371  * modes.
1372  */
1373 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1374 {
1375         struct stmmac_priv *priv = netdev_priv(dev);
1376         int ret;
1377
1378         ret = init_dma_rx_desc_rings(dev, flags);
1379         if (ret)
1380                 return ret;
1381
1382         ret = init_dma_tx_desc_rings(dev);
1383
1384         stmmac_clear_descriptors(priv);
1385
1386         if (netif_msg_hw(priv))
1387                 stmmac_display_rings(priv);
1388
1389         return ret;
1390 }
1391
1392 /**
1393  * dma_free_rx_skbufs - free RX dma buffers
1394  * @priv: private structure
1395  * @queue: RX queue index
1396  */
1397 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1398 {
1399         int i;
1400
1401         for (i = 0; i < DMA_RX_SIZE; i++)
1402                 stmmac_free_rx_buffer(priv, queue, i);
1403 }
1404
1405 /**
1406  * dma_free_tx_skbufs - free TX dma buffers
1407  * @priv: private structure
1408  * @queue: TX queue index
1409  */
1410 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1411 {
1412         int i;
1413
1414         for (i = 0; i < DMA_TX_SIZE; i++)
1415                 stmmac_free_tx_buffer(priv, queue, i);
1416 }
1417
1418 /**
1419  * free_dma_rx_desc_resources - free RX dma desc resources
1420  * @priv: private structure
1421  */
1422 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1423 {
1424         u32 rx_count = priv->plat->rx_queues_to_use;
1425         u32 queue;
1426
1427         /* Free RX queue resources */
1428         for (queue = 0; queue < rx_count; queue++) {
1429                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1430
1431                 /* Release the DMA RX socket buffers */
1432                 dma_free_rx_skbufs(priv, queue);
1433
1434                 /* Free DMA regions of consistent memory previously allocated */
1435                 if (!priv->extend_desc)
1436                         dma_free_coherent(priv->device,
1437                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1438                                           rx_q->dma_rx, rx_q->dma_rx_phy);
1439                 else
1440                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1441                                           sizeof(struct dma_extended_desc),
1442                                           rx_q->dma_erx, rx_q->dma_rx_phy);
1443
1444                 kfree(rx_q->rx_skbuff_dma);
1445                 kfree(rx_q->rx_skbuff);
1446         }
1447 }
1448
1449 /**
1450  * free_dma_tx_desc_resources - free TX dma desc resources
1451  * @priv: private structure
1452  */
1453 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1454 {
1455         u32 tx_count = priv->plat->tx_queues_to_use;
1456         u32 queue;
1457
1458         /* Free TX queue resources */
1459         for (queue = 0; queue < tx_count; queue++) {
1460                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1461
1462                 /* Release the DMA TX socket buffers */
1463                 dma_free_tx_skbufs(priv, queue);
1464
1465                 /* Free DMA regions of consistent memory previously allocated */
1466                 if (!priv->extend_desc)
1467                         dma_free_coherent(priv->device,
1468                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1469                                           tx_q->dma_tx, tx_q->dma_tx_phy);
1470                 else
1471                         dma_free_coherent(priv->device, DMA_TX_SIZE *
1472                                           sizeof(struct dma_extended_desc),
1473                                           tx_q->dma_etx, tx_q->dma_tx_phy);
1474
1475                 kfree(tx_q->tx_skbuff_dma);
1476                 kfree(tx_q->tx_skbuff);
1477         }
1478 }
1479
1480 /**
1481  * alloc_dma_rx_desc_resources - alloc RX resources.
1482  * @priv: private structure
1483  * Description: according to which descriptor can be used (extend or basic)
1484  * this function allocates the resources for TX and RX paths. In case of
1485  * reception, for example, it pre-allocated the RX socket buffer in order to
1486  * allow zero-copy mechanism.
1487  */
1488 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1489 {
1490         u32 rx_count = priv->plat->rx_queues_to_use;
1491         int ret = -ENOMEM;
1492         u32 queue;
1493
1494         /* RX queues buffers and DMA */
1495         for (queue = 0; queue < rx_count; queue++) {
1496                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1497
1498                 rx_q->queue_index = queue;
1499                 rx_q->priv_data = priv;
1500
1501                 rx_q->rx_skbuff_dma = kmalloc_array(DMA_RX_SIZE,
1502                                                     sizeof(dma_addr_t),
1503                                                     GFP_KERNEL);
1504                 if (!rx_q->rx_skbuff_dma)
1505                         goto err_dma;
1506
1507                 rx_q->rx_skbuff = kmalloc_array(DMA_RX_SIZE,
1508                                                 sizeof(struct sk_buff *),
1509                                                 GFP_KERNEL);
1510                 if (!rx_q->rx_skbuff)
1511                         goto err_dma;
1512
1513                 if (priv->extend_desc) {
1514                         rx_q->dma_erx = dma_zalloc_coherent(priv->device,
1515                                                             DMA_RX_SIZE *
1516                                                             sizeof(struct
1517                                                             dma_extended_desc),
1518                                                             &rx_q->dma_rx_phy,
1519                                                             GFP_KERNEL);
1520                         if (!rx_q->dma_erx)
1521                                 goto err_dma;
1522
1523                 } else {
1524                         rx_q->dma_rx = dma_zalloc_coherent(priv->device,
1525                                                            DMA_RX_SIZE *
1526                                                            sizeof(struct
1527                                                            dma_desc),
1528                                                            &rx_q->dma_rx_phy,
1529                                                            GFP_KERNEL);
1530                         if (!rx_q->dma_rx)
1531                                 goto err_dma;
1532                 }
1533         }
1534
1535         return 0;
1536
1537 err_dma:
1538         free_dma_rx_desc_resources(priv);
1539
1540         return ret;
1541 }
1542
1543 /**
1544  * alloc_dma_tx_desc_resources - alloc TX resources.
1545  * @priv: private structure
1546  * Description: according to which descriptor can be used (extend or basic)
1547  * this function allocates the resources for TX and RX paths. In case of
1548  * reception, for example, it pre-allocated the RX socket buffer in order to
1549  * allow zero-copy mechanism.
1550  */
1551 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1552 {
1553         u32 tx_count = priv->plat->tx_queues_to_use;
1554         int ret = -ENOMEM;
1555         u32 queue;
1556
1557         /* TX queues buffers and DMA */
1558         for (queue = 0; queue < tx_count; queue++) {
1559                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1560
1561                 tx_q->queue_index = queue;
1562                 tx_q->priv_data = priv;
1563
1564                 tx_q->tx_skbuff_dma = kmalloc_array(DMA_TX_SIZE,
1565                                                     sizeof(*tx_q->tx_skbuff_dma),
1566                                                     GFP_KERNEL);
1567                 if (!tx_q->tx_skbuff_dma)
1568                         goto err_dma;
1569
1570                 tx_q->tx_skbuff = kmalloc_array(DMA_TX_SIZE,
1571                                                 sizeof(struct sk_buff *),
1572                                                 GFP_KERNEL);
1573                 if (!tx_q->tx_skbuff)
1574                         goto err_dma;
1575
1576                 if (priv->extend_desc) {
1577                         tx_q->dma_etx = dma_zalloc_coherent(priv->device,
1578                                                             DMA_TX_SIZE *
1579                                                             sizeof(struct
1580                                                             dma_extended_desc),
1581                                                             &tx_q->dma_tx_phy,
1582                                                             GFP_KERNEL);
1583                         if (!tx_q->dma_etx)
1584                                 goto err_dma;
1585                 } else {
1586                         tx_q->dma_tx = dma_zalloc_coherent(priv->device,
1587                                                            DMA_TX_SIZE *
1588                                                            sizeof(struct
1589                                                                   dma_desc),
1590                                                            &tx_q->dma_tx_phy,
1591                                                            GFP_KERNEL);
1592                         if (!tx_q->dma_tx)
1593                                 goto err_dma;
1594                 }
1595         }
1596
1597         return 0;
1598
1599 err_dma:
1600         free_dma_tx_desc_resources(priv);
1601
1602         return ret;
1603 }
1604
1605 /**
1606  * alloc_dma_desc_resources - alloc TX/RX resources.
1607  * @priv: private structure
1608  * Description: according to which descriptor can be used (extend or basic)
1609  * this function allocates the resources for TX and RX paths. In case of
1610  * reception, for example, it pre-allocated the RX socket buffer in order to
1611  * allow zero-copy mechanism.
1612  */
1613 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1614 {
1615         /* RX Allocation */
1616         int ret = alloc_dma_rx_desc_resources(priv);
1617
1618         if (ret)
1619                 return ret;
1620
1621         ret = alloc_dma_tx_desc_resources(priv);
1622
1623         return ret;
1624 }
1625
1626 /**
1627  * free_dma_desc_resources - free dma desc resources
1628  * @priv: private structure
1629  */
1630 static void free_dma_desc_resources(struct stmmac_priv *priv)
1631 {
1632         /* Release the DMA RX socket buffers */
1633         free_dma_rx_desc_resources(priv);
1634
1635         /* Release the DMA TX socket buffers */
1636         free_dma_tx_desc_resources(priv);
1637 }
1638
1639 /**
1640  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1641  *  @priv: driver private structure
1642  *  Description: It is used for enabling the rx queues in the MAC
1643  */
1644 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1645 {
1646         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1647         int queue;
1648         u8 mode;
1649
1650         for (queue = 0; queue < rx_queues_count; queue++) {
1651                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1652                 priv->hw->mac->rx_queue_enable(priv->hw, mode, queue);
1653         }
1654 }
1655
1656 /**
1657  * stmmac_start_rx_dma - start RX DMA channel
1658  * @priv: driver private structure
1659  * @chan: RX channel index
1660  * Description:
1661  * This starts a RX DMA channel
1662  */
1663 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1664 {
1665         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1666         priv->hw->dma->start_rx(priv->ioaddr, chan);
1667 }
1668
1669 /**
1670  * stmmac_start_tx_dma - start TX DMA channel
1671  * @priv: driver private structure
1672  * @chan: TX channel index
1673  * Description:
1674  * This starts a TX DMA channel
1675  */
1676 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1677 {
1678         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1679         priv->hw->dma->start_tx(priv->ioaddr, chan);
1680 }
1681
1682 /**
1683  * stmmac_stop_rx_dma - stop RX DMA channel
1684  * @priv: driver private structure
1685  * @chan: RX channel index
1686  * Description:
1687  * This stops a RX DMA channel
1688  */
1689 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1690 {
1691         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1692         priv->hw->dma->stop_rx(priv->ioaddr, chan);
1693 }
1694
1695 /**
1696  * stmmac_stop_tx_dma - stop TX DMA channel
1697  * @priv: driver private structure
1698  * @chan: TX channel index
1699  * Description:
1700  * This stops a TX DMA channel
1701  */
1702 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1703 {
1704         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1705         priv->hw->dma->stop_tx(priv->ioaddr, chan);
1706 }
1707
1708 /**
1709  * stmmac_start_all_dma - start all RX and TX DMA channels
1710  * @priv: driver private structure
1711  * Description:
1712  * This starts all the RX and TX DMA channels
1713  */
1714 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1715 {
1716         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1717         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1718         u32 chan = 0;
1719
1720         for (chan = 0; chan < rx_channels_count; chan++)
1721                 stmmac_start_rx_dma(priv, chan);
1722
1723         for (chan = 0; chan < tx_channels_count; chan++)
1724                 stmmac_start_tx_dma(priv, chan);
1725 }
1726
1727 /**
1728  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1729  * @priv: driver private structure
1730  * Description:
1731  * This stops the RX and TX DMA channels
1732  */
1733 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1734 {
1735         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1736         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1737         u32 chan = 0;
1738
1739         for (chan = 0; chan < rx_channels_count; chan++)
1740                 stmmac_stop_rx_dma(priv, chan);
1741
1742         for (chan = 0; chan < tx_channels_count; chan++)
1743                 stmmac_stop_tx_dma(priv, chan);
1744 }
1745
1746 /**
1747  *  stmmac_dma_operation_mode - HW DMA operation mode
1748  *  @priv: driver private structure
1749  *  Description: it is used for configuring the DMA operation mode register in
1750  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1751  */
1752 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1753 {
1754         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1755         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1756         int rxfifosz = priv->plat->rx_fifo_size;
1757         int txfifosz = priv->plat->tx_fifo_size;
1758         u32 txmode = 0;
1759         u32 rxmode = 0;
1760         u32 chan = 0;
1761         u8 qmode = 0;
1762
1763         if (rxfifosz == 0)
1764                 rxfifosz = priv->dma_cap.rx_fifo_size;
1765         if (txfifosz == 0)
1766                 txfifosz = priv->dma_cap.tx_fifo_size;
1767
1768         /* Adjust for real per queue fifo size */
1769         rxfifosz /= rx_channels_count;
1770         txfifosz /= tx_channels_count;
1771
1772         if (priv->plat->force_thresh_dma_mode) {
1773                 txmode = tc;
1774                 rxmode = tc;
1775         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1776                 /*
1777                  * In case of GMAC, SF mode can be enabled
1778                  * to perform the TX COE in HW. This depends on:
1779                  * 1) TX COE if actually supported
1780                  * 2) There is no bugged Jumbo frame support
1781                  *    that needs to not insert csum in the TDES.
1782                  */
1783                 txmode = SF_DMA_MODE;
1784                 rxmode = SF_DMA_MODE;
1785                 priv->xstats.threshold = SF_DMA_MODE;
1786         } else {
1787                 txmode = tc;
1788                 rxmode = SF_DMA_MODE;
1789         }
1790
1791         /* configure all channels */
1792         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1793                 for (chan = 0; chan < rx_channels_count; chan++) {
1794                         qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1795
1796                         priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1797                                                    rxfifosz, qmode);
1798                 }
1799
1800                 for (chan = 0; chan < tx_channels_count; chan++) {
1801                         qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1802
1803                         priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan,
1804                                                    txfifosz, qmode);
1805                 }
1806         } else {
1807                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1808                                         rxfifosz);
1809         }
1810 }
1811
1812 /**
1813  * stmmac_tx_clean - to manage the transmission completion
1814  * @priv: driver private structure
1815  * @queue: TX queue index
1816  * Description: it reclaims the transmit resources after transmission completes.
1817  */
1818 static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
1819 {
1820         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1821         unsigned int bytes_compl = 0, pkts_compl = 0;
1822         unsigned int entry;
1823
1824         netif_tx_lock(priv->dev);
1825
1826         priv->xstats.tx_clean++;
1827
1828         entry = tx_q->dirty_tx;
1829         while (entry != tx_q->cur_tx) {
1830                 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1831                 struct dma_desc *p;
1832                 int status;
1833
1834                 if (priv->extend_desc)
1835                         p = (struct dma_desc *)(tx_q->dma_etx + entry);
1836                 else
1837                         p = tx_q->dma_tx + entry;
1838
1839                 status = priv->hw->desc->tx_status(&priv->dev->stats,
1840                                                       &priv->xstats, p,
1841                                                       priv->ioaddr);
1842                 /* Check if the descriptor is owned by the DMA */
1843                 if (unlikely(status & tx_dma_own))
1844                         break;
1845
1846                 /* Just consider the last segment and ...*/
1847                 if (likely(!(status & tx_not_ls))) {
1848                         /* ... verify the status error condition */
1849                         if (unlikely(status & tx_err)) {
1850                                 priv->dev->stats.tx_errors++;
1851                         } else {
1852                                 priv->dev->stats.tx_packets++;
1853                                 priv->xstats.tx_pkt_n++;
1854                         }
1855                         stmmac_get_tx_hwtstamp(priv, p, skb);
1856                 }
1857
1858                 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1859                         if (tx_q->tx_skbuff_dma[entry].map_as_page)
1860                                 dma_unmap_page(priv->device,
1861                                                tx_q->tx_skbuff_dma[entry].buf,
1862                                                tx_q->tx_skbuff_dma[entry].len,
1863                                                DMA_TO_DEVICE);
1864                         else
1865                                 dma_unmap_single(priv->device,
1866                                                  tx_q->tx_skbuff_dma[entry].buf,
1867                                                  tx_q->tx_skbuff_dma[entry].len,
1868                                                  DMA_TO_DEVICE);
1869                         tx_q->tx_skbuff_dma[entry].buf = 0;
1870                         tx_q->tx_skbuff_dma[entry].len = 0;
1871                         tx_q->tx_skbuff_dma[entry].map_as_page = false;
1872                 }
1873
1874                 if (priv->hw->mode->clean_desc3)
1875                         priv->hw->mode->clean_desc3(tx_q, p);
1876
1877                 tx_q->tx_skbuff_dma[entry].last_segment = false;
1878                 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1879
1880                 if (likely(skb != NULL)) {
1881                         pkts_compl++;
1882                         bytes_compl += skb->len;
1883                         dev_consume_skb_any(skb);
1884                         tx_q->tx_skbuff[entry] = NULL;
1885                 }
1886
1887                 priv->hw->desc->release_tx_desc(p, priv->mode);
1888
1889                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1890         }
1891         tx_q->dirty_tx = entry;
1892
1893         netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1894                                   pkts_compl, bytes_compl);
1895
1896         if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1897                                                                 queue))) &&
1898             stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1899
1900                 netif_dbg(priv, tx_done, priv->dev,
1901                           "%s: restart transmit\n", __func__);
1902                 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1903         }
1904
1905         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1906                 stmmac_enable_eee_mode(priv);
1907                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1908         }
1909         netif_tx_unlock(priv->dev);
1910 }
1911
1912 static inline void stmmac_enable_dma_irq(struct stmmac_priv *priv, u32 chan)
1913 {
1914         priv->hw->dma->enable_dma_irq(priv->ioaddr, chan);
1915 }
1916
1917 static inline void stmmac_disable_dma_irq(struct stmmac_priv *priv, u32 chan)
1918 {
1919         priv->hw->dma->disable_dma_irq(priv->ioaddr, chan);
1920 }
1921
1922 /**
1923  * stmmac_tx_err - to manage the tx error
1924  * @priv: driver private structure
1925  * @chan: channel index
1926  * Description: it cleans the descriptors and restarts the transmission
1927  * in case of transmission errors.
1928  */
1929 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1930 {
1931         struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
1932         int i;
1933
1934         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
1935
1936         stmmac_stop_tx_dma(priv, chan);
1937         dma_free_tx_skbufs(priv, chan);
1938         for (i = 0; i < DMA_TX_SIZE; i++)
1939                 if (priv->extend_desc)
1940                         priv->hw->desc->init_tx_desc(&tx_q->dma_etx[i].basic,
1941                                                      priv->mode,
1942                                                      (i == DMA_TX_SIZE - 1));
1943                 else
1944                         priv->hw->desc->init_tx_desc(&tx_q->dma_tx[i],
1945                                                      priv->mode,
1946                                                      (i == DMA_TX_SIZE - 1));
1947         tx_q->dirty_tx = 0;
1948         tx_q->cur_tx = 0;
1949         netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
1950         stmmac_start_tx_dma(priv, chan);
1951
1952         priv->dev->stats.tx_errors++;
1953         netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
1954 }
1955
1956 /**
1957  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
1958  *  @priv: driver private structure
1959  *  @txmode: TX operating mode
1960  *  @rxmode: RX operating mode
1961  *  @chan: channel index
1962  *  Description: it is used for configuring of the DMA operation mode in
1963  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
1964  *  mode.
1965  */
1966 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
1967                                           u32 rxmode, u32 chan)
1968 {
1969         u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1970         u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1971         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1972         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1973         int rxfifosz = priv->plat->rx_fifo_size;
1974         int txfifosz = priv->plat->tx_fifo_size;
1975
1976         if (rxfifosz == 0)
1977                 rxfifosz = priv->dma_cap.rx_fifo_size;
1978         if (txfifosz == 0)
1979                 txfifosz = priv->dma_cap.tx_fifo_size;
1980
1981         /* Adjust for real per queue fifo size */
1982         rxfifosz /= rx_channels_count;
1983         txfifosz /= tx_channels_count;
1984
1985         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
1986                 priv->hw->dma->dma_rx_mode(priv->ioaddr, rxmode, chan,
1987                                            rxfifosz, rxqmode);
1988                 priv->hw->dma->dma_tx_mode(priv->ioaddr, txmode, chan,
1989                                            txfifosz, txqmode);
1990         } else {
1991                 priv->hw->dma->dma_mode(priv->ioaddr, txmode, rxmode,
1992                                         rxfifosz);
1993         }
1994 }
1995
1996 /**
1997  * stmmac_dma_interrupt - DMA ISR
1998  * @priv: driver private structure
1999  * Description: this is the DMA ISR. It is called by the main ISR.
2000  * It calls the dwmac dma routine and schedule poll method in case of some
2001  * work can be done.
2002  */
2003 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2004 {
2005         u32 tx_channel_count = priv->plat->tx_queues_to_use;
2006         int status;
2007         u32 chan;
2008
2009         for (chan = 0; chan < tx_channel_count; chan++) {
2010                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
2011
2012                 status = priv->hw->dma->dma_interrupt(priv->ioaddr,
2013                                                       &priv->xstats, chan);
2014                 if (likely((status & handle_rx)) || (status & handle_tx)) {
2015                         if (likely(napi_schedule_prep(&rx_q->napi))) {
2016                                 stmmac_disable_dma_irq(priv, chan);
2017                                 __napi_schedule(&rx_q->napi);
2018                         }
2019                 }
2020
2021                 if (unlikely(status & tx_hard_error_bump_tc)) {
2022                         /* Try to bump up the dma threshold on this failure */
2023                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2024                             (tc <= 256)) {
2025                                 tc += 64;
2026                                 if (priv->plat->force_thresh_dma_mode)
2027                                         stmmac_set_dma_operation_mode(priv,
2028                                                                       tc,
2029                                                                       tc,
2030                                                                       chan);
2031                                 else
2032                                         stmmac_set_dma_operation_mode(priv,
2033                                                                     tc,
2034                                                                     SF_DMA_MODE,
2035                                                                     chan);
2036                                 priv->xstats.threshold = tc;
2037                         }
2038                 } else if (unlikely(status == tx_hard_error)) {
2039                         stmmac_tx_err(priv, chan);
2040                 }
2041         }
2042 }
2043
2044 /**
2045  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2046  * @priv: driver private structure
2047  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2048  */
2049 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2050 {
2051         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2052                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2053
2054         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2055                 priv->ptpaddr = priv->ioaddr + PTP_GMAC4_OFFSET;
2056                 priv->mmcaddr = priv->ioaddr + MMC_GMAC4_OFFSET;
2057         } else {
2058                 priv->ptpaddr = priv->ioaddr + PTP_GMAC3_X_OFFSET;
2059                 priv->mmcaddr = priv->ioaddr + MMC_GMAC3_X_OFFSET;
2060         }
2061
2062         dwmac_mmc_intr_all_mask(priv->mmcaddr);
2063
2064         if (priv->dma_cap.rmon) {
2065                 dwmac_mmc_ctrl(priv->mmcaddr, mode);
2066                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2067         } else
2068                 netdev_info(priv->dev, "No MAC Management Counters available\n");
2069 }
2070
2071 /**
2072  * stmmac_selec_desc_mode - to select among: normal/alternate/extend descriptors
2073  * @priv: driver private structure
2074  * Description: select the Enhanced/Alternate or Normal descriptors.
2075  * In case of Enhanced/Alternate, it checks if the extended descriptors are
2076  * supported by the HW capability register.
2077  */
2078 static void stmmac_selec_desc_mode(struct stmmac_priv *priv)
2079 {
2080         if (priv->plat->enh_desc) {
2081                 dev_info(priv->device, "Enhanced/Alternate descriptors\n");
2082
2083                 /* GMAC older than 3.50 has no extended descriptors */
2084                 if (priv->synopsys_id >= DWMAC_CORE_3_50) {
2085                         dev_info(priv->device, "Enabled extended descriptors\n");
2086                         priv->extend_desc = 1;
2087                 } else
2088                         dev_warn(priv->device, "Extended descriptors not supported\n");
2089
2090                 priv->hw->desc = &enh_desc_ops;
2091         } else {
2092                 dev_info(priv->device, "Normal descriptors\n");
2093                 priv->hw->desc = &ndesc_ops;
2094         }
2095 }
2096
2097 /**
2098  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2099  * @priv: driver private structure
2100  * Description:
2101  *  new GMAC chip generations have a new register to indicate the
2102  *  presence of the optional feature/functions.
2103  *  This can be also used to override the value passed through the
2104  *  platform and necessary for old MAC10/100 and GMAC chips.
2105  */
2106 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2107 {
2108         u32 ret = 0;
2109
2110         if (priv->hw->dma->get_hw_feature) {
2111                 priv->hw->dma->get_hw_feature(priv->ioaddr,
2112                                               &priv->dma_cap);
2113                 ret = 1;
2114         }
2115
2116         return ret;
2117 }
2118
2119 /**
2120  * stmmac_check_ether_addr - check if the MAC addr is valid
2121  * @priv: driver private structure
2122  * Description:
2123  * it is to verify if the MAC address is valid, in case of failures it
2124  * generates a random MAC address
2125  */
2126 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2127 {
2128         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2129                 priv->hw->mac->get_umac_addr(priv->hw,
2130                                              priv->dev->dev_addr, 0);
2131                 if (!is_valid_ether_addr(priv->dev->dev_addr))
2132                         eth_hw_addr_random(priv->dev);
2133                 netdev_info(priv->dev, "device MAC address %pM\n",
2134                             priv->dev->dev_addr);
2135         }
2136 }
2137
2138 /**
2139  * stmmac_init_dma_engine - DMA init.
2140  * @priv: driver private structure
2141  * Description:
2142  * It inits the DMA invoking the specific MAC/GMAC callback.
2143  * Some DMA parameters can be passed from the platform;
2144  * in case of these are not passed a default is kept for the MAC or GMAC.
2145  */
2146 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2147 {
2148         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2149         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2150         struct stmmac_rx_queue *rx_q;
2151         struct stmmac_tx_queue *tx_q;
2152         u32 dummy_dma_rx_phy = 0;
2153         u32 dummy_dma_tx_phy = 0;
2154         u32 chan = 0;
2155         int atds = 0;
2156         int ret = 0;
2157
2158         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2159                 dev_err(priv->device, "Invalid DMA configuration\n");
2160                 return -EINVAL;
2161         }
2162
2163         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2164                 atds = 1;
2165
2166         ret = priv->hw->dma->reset(priv->ioaddr);
2167         if (ret) {
2168                 dev_err(priv->device, "Failed to reset the dma\n");
2169                 return ret;
2170         }
2171
2172         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
2173                 /* DMA Configuration */
2174                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2175                                     dummy_dma_tx_phy, dummy_dma_rx_phy, atds);
2176
2177                 /* DMA RX Channel Configuration */
2178                 for (chan = 0; chan < rx_channels_count; chan++) {
2179                         rx_q = &priv->rx_queue[chan];
2180
2181                         priv->hw->dma->init_rx_chan(priv->ioaddr,
2182                                                     priv->plat->dma_cfg,
2183                                                     rx_q->dma_rx_phy, chan);
2184
2185                         rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2186                                     (DMA_RX_SIZE * sizeof(struct dma_desc));
2187                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
2188                                                        rx_q->rx_tail_addr,
2189                                                        chan);
2190                 }
2191
2192                 /* DMA TX Channel Configuration */
2193                 for (chan = 0; chan < tx_channels_count; chan++) {
2194                         tx_q = &priv->tx_queue[chan];
2195
2196                         priv->hw->dma->init_chan(priv->ioaddr,
2197                                                  priv->plat->dma_cfg,
2198                                                  chan);
2199
2200                         priv->hw->dma->init_tx_chan(priv->ioaddr,
2201                                                     priv->plat->dma_cfg,
2202                                                     tx_q->dma_tx_phy, chan);
2203
2204                         tx_q->tx_tail_addr = tx_q->dma_tx_phy +
2205                                     (DMA_TX_SIZE * sizeof(struct dma_desc));
2206                         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr,
2207                                                        tx_q->tx_tail_addr,
2208                                                        chan);
2209                 }
2210         } else {
2211                 rx_q = &priv->rx_queue[chan];
2212                 tx_q = &priv->tx_queue[chan];
2213                 priv->hw->dma->init(priv->ioaddr, priv->plat->dma_cfg,
2214                                     tx_q->dma_tx_phy, rx_q->dma_rx_phy, atds);
2215         }
2216
2217         if (priv->plat->axi && priv->hw->dma->axi)
2218                 priv->hw->dma->axi(priv->ioaddr, priv->plat->axi);
2219
2220         return ret;
2221 }
2222
2223 /**
2224  * stmmac_tx_timer - mitigation sw timer for tx.
2225  * @data: data pointer
2226  * Description:
2227  * This is the timer handler to directly invoke the stmmac_tx_clean.
2228  */
2229 static void stmmac_tx_timer(struct timer_list *t)
2230 {
2231         struct stmmac_priv *priv = from_timer(priv, t, txtimer);
2232         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2233         u32 queue;
2234
2235         /* let's scan all the tx queues */
2236         for (queue = 0; queue < tx_queues_count; queue++)
2237                 stmmac_tx_clean(priv, queue);
2238 }
2239
2240 /**
2241  * stmmac_init_tx_coalesce - init tx mitigation options.
2242  * @priv: driver private structure
2243  * Description:
2244  * This inits the transmit coalesce parameters: i.e. timer rate,
2245  * timer handler and default threshold used for enabling the
2246  * interrupt on completion bit.
2247  */
2248 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
2249 {
2250         priv->tx_coal_frames = STMMAC_TX_FRAMES;
2251         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2252         timer_setup(&priv->txtimer, stmmac_tx_timer, 0);
2253         priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
2254         add_timer(&priv->txtimer);
2255 }
2256
2257 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2258 {
2259         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2260         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2261         u32 chan;
2262
2263         /* set TX ring length */
2264         if (priv->hw->dma->set_tx_ring_len) {
2265                 for (chan = 0; chan < tx_channels_count; chan++)
2266                         priv->hw->dma->set_tx_ring_len(priv->ioaddr,
2267                                                        (DMA_TX_SIZE - 1), chan);
2268         }
2269
2270         /* set RX ring length */
2271         if (priv->hw->dma->set_rx_ring_len) {
2272                 for (chan = 0; chan < rx_channels_count; chan++)
2273                         priv->hw->dma->set_rx_ring_len(priv->ioaddr,
2274                                                        (DMA_RX_SIZE - 1), chan);
2275         }
2276 }
2277
2278 /**
2279  *  stmmac_set_tx_queue_weight - Set TX queue weight
2280  *  @priv: driver private structure
2281  *  Description: It is used for setting TX queues weight
2282  */
2283 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2284 {
2285         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2286         u32 weight;
2287         u32 queue;
2288
2289         for (queue = 0; queue < tx_queues_count; queue++) {
2290                 weight = priv->plat->tx_queues_cfg[queue].weight;
2291                 priv->hw->mac->set_mtl_tx_queue_weight(priv->hw, weight, queue);
2292         }
2293 }
2294
2295 /**
2296  *  stmmac_configure_cbs - Configure CBS in TX queue
2297  *  @priv: driver private structure
2298  *  Description: It is used for configuring CBS in AVB TX queues
2299  */
2300 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2301 {
2302         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2303         u32 mode_to_use;
2304         u32 queue;
2305
2306         /* queue 0 is reserved for legacy traffic */
2307         for (queue = 1; queue < tx_queues_count; queue++) {
2308                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2309                 if (mode_to_use == MTL_QUEUE_DCB)
2310                         continue;
2311
2312                 priv->hw->mac->config_cbs(priv->hw,
2313                                 priv->plat->tx_queues_cfg[queue].send_slope,
2314                                 priv->plat->tx_queues_cfg[queue].idle_slope,
2315                                 priv->plat->tx_queues_cfg[queue].high_credit,
2316                                 priv->plat->tx_queues_cfg[queue].low_credit,
2317                                 queue);
2318         }
2319 }
2320
2321 /**
2322  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2323  *  @priv: driver private structure
2324  *  Description: It is used for mapping RX queues to RX dma channels
2325  */
2326 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2327 {
2328         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2329         u32 queue;
2330         u32 chan;
2331
2332         for (queue = 0; queue < rx_queues_count; queue++) {
2333                 chan = priv->plat->rx_queues_cfg[queue].chan;
2334                 priv->hw->mac->map_mtl_to_dma(priv->hw, queue, chan);
2335         }
2336 }
2337
2338 /**
2339  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2340  *  @priv: driver private structure
2341  *  Description: It is used for configuring the RX Queue Priority
2342  */
2343 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2344 {
2345         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2346         u32 queue;
2347         u32 prio;
2348
2349         for (queue = 0; queue < rx_queues_count; queue++) {
2350                 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2351                         continue;
2352
2353                 prio = priv->plat->rx_queues_cfg[queue].prio;
2354                 priv->hw->mac->rx_queue_prio(priv->hw, prio, queue);
2355         }
2356 }
2357
2358 /**
2359  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2360  *  @priv: driver private structure
2361  *  Description: It is used for configuring the TX Queue Priority
2362  */
2363 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2364 {
2365         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2366         u32 queue;
2367         u32 prio;
2368
2369         for (queue = 0; queue < tx_queues_count; queue++) {
2370                 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2371                         continue;
2372
2373                 prio = priv->plat->tx_queues_cfg[queue].prio;
2374                 priv->hw->mac->tx_queue_prio(priv->hw, prio, queue);
2375         }
2376 }
2377
2378 /**
2379  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2380  *  @priv: driver private structure
2381  *  Description: It is used for configuring the RX queue routing
2382  */
2383 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2384 {
2385         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2386         u32 queue;
2387         u8 packet;
2388
2389         for (queue = 0; queue < rx_queues_count; queue++) {
2390                 /* no specific packet type routing specified for the queue */
2391                 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2392                         continue;
2393
2394                 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2395                 priv->hw->mac->rx_queue_prio(priv->hw, packet, queue);
2396         }
2397 }
2398
2399 /**
2400  *  stmmac_mtl_configuration - Configure MTL
2401  *  @priv: driver private structure
2402  *  Description: It is used for configurring MTL
2403  */
2404 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2405 {
2406         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2407         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2408
2409         if (tx_queues_count > 1 && priv->hw->mac->set_mtl_tx_queue_weight)
2410                 stmmac_set_tx_queue_weight(priv);
2411
2412         /* Configure MTL RX algorithms */
2413         if (rx_queues_count > 1 && priv->hw->mac->prog_mtl_rx_algorithms)
2414                 priv->hw->mac->prog_mtl_rx_algorithms(priv->hw,
2415                                                 priv->plat->rx_sched_algorithm);
2416
2417         /* Configure MTL TX algorithms */
2418         if (tx_queues_count > 1 && priv->hw->mac->prog_mtl_tx_algorithms)
2419                 priv->hw->mac->prog_mtl_tx_algorithms(priv->hw,
2420                                                 priv->plat->tx_sched_algorithm);
2421
2422         /* Configure CBS in AVB TX queues */
2423         if (tx_queues_count > 1 && priv->hw->mac->config_cbs)
2424                 stmmac_configure_cbs(priv);
2425
2426         /* Map RX MTL to DMA channels */
2427         if (priv->hw->mac->map_mtl_to_dma)
2428                 stmmac_rx_queue_dma_chan_map(priv);
2429
2430         /* Enable MAC RX Queues */
2431         if (priv->hw->mac->rx_queue_enable)
2432                 stmmac_mac_enable_rx_queues(priv);
2433
2434         /* Set RX priorities */
2435         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_prio)
2436                 stmmac_mac_config_rx_queues_prio(priv);
2437
2438         /* Set TX priorities */
2439         if (tx_queues_count > 1 && priv->hw->mac->tx_queue_prio)
2440                 stmmac_mac_config_tx_queues_prio(priv);
2441
2442         /* Set RX routing */
2443         if (rx_queues_count > 1 && priv->hw->mac->rx_queue_routing)
2444                 stmmac_mac_config_rx_queues_routing(priv);
2445 }
2446
2447 /**
2448  * stmmac_hw_setup - setup mac in a usable state.
2449  *  @dev : pointer to the device structure.
2450  *  Description:
2451  *  this is the main function to setup the HW in a usable state because the
2452  *  dma engine is reset, the core registers are configured (e.g. AXI,
2453  *  Checksum features, timers). The DMA is ready to start receiving and
2454  *  transmitting.
2455  *  Return value:
2456  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2457  *  file on failure.
2458  */
2459 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2460 {
2461         struct stmmac_priv *priv = netdev_priv(dev);
2462         u32 rx_cnt = priv->plat->rx_queues_to_use;
2463         u32 tx_cnt = priv->plat->tx_queues_to_use;
2464         u32 chan;
2465         int ret;
2466
2467         /* DMA initialization and SW reset */
2468         ret = stmmac_init_dma_engine(priv);
2469         if (ret < 0) {
2470                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2471                            __func__);
2472                 return ret;
2473         }
2474
2475         /* Copy the MAC addr into the HW  */
2476         priv->hw->mac->set_umac_addr(priv->hw, dev->dev_addr, 0);
2477
2478         /* PS and related bits will be programmed according to the speed */
2479         if (priv->hw->pcs) {
2480                 int speed = priv->plat->mac_port_sel_speed;
2481
2482                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2483                     (speed == SPEED_1000)) {
2484                         priv->hw->ps = speed;
2485                 } else {
2486                         dev_warn(priv->device, "invalid port speed\n");
2487                         priv->hw->ps = 0;
2488                 }
2489         }
2490
2491         /* Initialize the MAC Core */
2492         priv->hw->mac->core_init(priv->hw, dev->mtu);
2493
2494         /* Initialize MTL*/
2495         if (priv->synopsys_id >= DWMAC_CORE_4_00)
2496                 stmmac_mtl_configuration(priv);
2497
2498         ret = priv->hw->mac->rx_ipc(priv->hw);
2499         if (!ret) {
2500                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2501                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2502                 priv->hw->rx_csum = 0;
2503         }
2504
2505         /* Enable the MAC Rx/Tx */
2506         priv->hw->mac->set_mac(priv->ioaddr, true);
2507
2508         /* Set the HW DMA mode and the COE */
2509         stmmac_dma_operation_mode(priv);
2510
2511         stmmac_mmc_setup(priv);
2512
2513         if (init_ptp) {
2514                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2515                 if (ret < 0)
2516                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2517
2518                 ret = stmmac_init_ptp(priv);
2519                 if (ret == -EOPNOTSUPP)
2520                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2521                 else if (ret)
2522                         netdev_warn(priv->dev, "PTP init failed\n");
2523         }
2524
2525 #ifdef CONFIG_DEBUG_FS
2526         ret = stmmac_init_fs(dev);
2527         if (ret < 0)
2528                 netdev_warn(priv->dev, "%s: failed debugFS registration\n",
2529                             __func__);
2530 #endif
2531         /* Start the ball rolling... */
2532         stmmac_start_all_dma(priv);
2533
2534         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2535
2536         if ((priv->use_riwt) && (priv->hw->dma->rx_watchdog)) {
2537                 priv->rx_riwt = MAX_DMA_RIWT;
2538                 priv->hw->dma->rx_watchdog(priv->ioaddr, MAX_DMA_RIWT, rx_cnt);
2539         }
2540
2541         if (priv->hw->pcs && priv->hw->mac->pcs_ctrl_ane)
2542                 priv->hw->mac->pcs_ctrl_ane(priv->hw, 1, priv->hw->ps, 0);
2543
2544         /* set TX and RX rings length */
2545         stmmac_set_rings_length(priv);
2546
2547         /* Enable TSO */
2548         if (priv->tso) {
2549                 for (chan = 0; chan < tx_cnt; chan++)
2550                         priv->hw->dma->enable_tso(priv->ioaddr, 1, chan);
2551         }
2552
2553         return 0;
2554 }
2555
2556 static void stmmac_hw_teardown(struct net_device *dev)
2557 {
2558         struct stmmac_priv *priv = netdev_priv(dev);
2559
2560         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2561 }
2562
2563 /**
2564  *  stmmac_open - open entry point of the driver
2565  *  @dev : pointer to the device structure.
2566  *  Description:
2567  *  This function is the open entry point of the driver.
2568  *  Return value:
2569  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2570  *  file on failure.
2571  */
2572 static int stmmac_open(struct net_device *dev)
2573 {
2574         struct stmmac_priv *priv = netdev_priv(dev);
2575         int ret;
2576
2577         stmmac_check_ether_addr(priv);
2578
2579         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2580             priv->hw->pcs != STMMAC_PCS_TBI &&
2581             priv->hw->pcs != STMMAC_PCS_RTBI) {
2582                 ret = stmmac_init_phy(dev);
2583                 if (ret) {
2584                         netdev_err(priv->dev,
2585                                    "%s: Cannot attach to PHY (error: %d)\n",
2586                                    __func__, ret);
2587                         return ret;
2588                 }
2589         }
2590
2591         /* Extra statistics */
2592         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2593         priv->xstats.threshold = tc;
2594
2595         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
2596         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2597         priv->mss = 0;
2598
2599         ret = alloc_dma_desc_resources(priv);
2600         if (ret < 0) {
2601                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2602                            __func__);
2603                 goto dma_desc_error;
2604         }
2605
2606         ret = init_dma_desc_rings(dev, GFP_KERNEL);
2607         if (ret < 0) {
2608                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2609                            __func__);
2610                 goto init_error;
2611         }
2612
2613         ret = stmmac_hw_setup(dev, true);
2614         if (ret < 0) {
2615                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2616                 goto init_error;
2617         }
2618
2619         stmmac_init_tx_coalesce(priv);
2620
2621         if (dev->phydev)
2622                 phy_start(dev->phydev);
2623
2624         /* Request the IRQ lines */
2625         ret = request_irq(dev->irq, stmmac_interrupt,
2626                           IRQF_SHARED, dev->name, dev);
2627         if (unlikely(ret < 0)) {
2628                 netdev_err(priv->dev,
2629                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2630                            __func__, dev->irq, ret);
2631                 goto irq_error;
2632         }
2633
2634         /* Request the Wake IRQ in case of another line is used for WoL */
2635         if (priv->wol_irq != dev->irq) {
2636                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2637                                   IRQF_SHARED, dev->name, dev);
2638                 if (unlikely(ret < 0)) {
2639                         netdev_err(priv->dev,
2640                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2641                                    __func__, priv->wol_irq, ret);
2642                         goto wolirq_error;
2643                 }
2644         }
2645
2646         /* Request the IRQ lines */
2647         if (priv->lpi_irq > 0) {
2648                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2649                                   dev->name, dev);
2650                 if (unlikely(ret < 0)) {
2651                         netdev_err(priv->dev,
2652                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2653                                    __func__, priv->lpi_irq, ret);
2654                         goto lpiirq_error;
2655                 }
2656         }
2657
2658         stmmac_enable_all_queues(priv);
2659         stmmac_start_all_queues(priv);
2660
2661         return 0;
2662
2663 lpiirq_error:
2664         if (priv->wol_irq != dev->irq)
2665                 free_irq(priv->wol_irq, dev);
2666 wolirq_error:
2667         free_irq(dev->irq, dev);
2668 irq_error:
2669         if (dev->phydev)
2670                 phy_stop(dev->phydev);
2671
2672         del_timer_sync(&priv->txtimer);
2673         stmmac_hw_teardown(dev);
2674 init_error:
2675         free_dma_desc_resources(priv);
2676 dma_desc_error:
2677         if (dev->phydev)
2678                 phy_disconnect(dev->phydev);
2679
2680         return ret;
2681 }
2682
2683 /**
2684  *  stmmac_release - close entry point of the driver
2685  *  @dev : device pointer.
2686  *  Description:
2687  *  This is the stop entry point of the driver.
2688  */
2689 static int stmmac_release(struct net_device *dev)
2690 {
2691         struct stmmac_priv *priv = netdev_priv(dev);
2692
2693         if (priv->eee_enabled)
2694                 del_timer_sync(&priv->eee_ctrl_timer);
2695
2696         /* Stop and disconnect the PHY */
2697         if (dev->phydev) {
2698                 phy_stop(dev->phydev);
2699                 phy_disconnect(dev->phydev);
2700         }
2701
2702         stmmac_stop_all_queues(priv);
2703
2704         stmmac_disable_all_queues(priv);
2705
2706         del_timer_sync(&priv->txtimer);
2707
2708         /* Free the IRQ lines */
2709         free_irq(dev->irq, dev);
2710         if (priv->wol_irq != dev->irq)
2711                 free_irq(priv->wol_irq, dev);
2712         if (priv->lpi_irq > 0)
2713                 free_irq(priv->lpi_irq, dev);
2714
2715         /* Stop TX/RX DMA and clear the descriptors */
2716         stmmac_stop_all_dma(priv);
2717
2718         /* Release and free the Rx/Tx resources */
2719         free_dma_desc_resources(priv);
2720
2721         /* Disable the MAC Rx/Tx */
2722         priv->hw->mac->set_mac(priv->ioaddr, false);
2723
2724         netif_carrier_off(dev);
2725
2726 #ifdef CONFIG_DEBUG_FS
2727         stmmac_exit_fs(dev);
2728 #endif
2729
2730         stmmac_release_ptp(priv);
2731
2732         return 0;
2733 }
2734
2735 /**
2736  *  stmmac_tso_allocator - close entry point of the driver
2737  *  @priv: driver private structure
2738  *  @des: buffer start address
2739  *  @total_len: total length to fill in descriptors
2740  *  @last_segmant: condition for the last descriptor
2741  *  @queue: TX queue index
2742  *  Description:
2743  *  This function fills descriptor and request new descriptors according to
2744  *  buffer length to fill
2745  */
2746 static void stmmac_tso_allocator(struct stmmac_priv *priv, unsigned int des,
2747                                  int total_len, bool last_segment, u32 queue)
2748 {
2749         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2750         struct dma_desc *desc;
2751         u32 buff_size;
2752         int tmp_len;
2753
2754         tmp_len = total_len;
2755
2756         while (tmp_len > 0) {
2757                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2758                 desc = tx_q->dma_tx + tx_q->cur_tx;
2759
2760                 desc->des0 = cpu_to_le32(des + (total_len - tmp_len));
2761                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2762                             TSO_MAX_BUFF_SIZE : tmp_len;
2763
2764                 priv->hw->desc->prepare_tso_tx_desc(desc, 0, buff_size,
2765                         0, 1,
2766                         (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
2767                         0, 0);
2768
2769                 tmp_len -= TSO_MAX_BUFF_SIZE;
2770         }
2771 }
2772
2773 /**
2774  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2775  *  @skb : the socket buffer
2776  *  @dev : device pointer
2777  *  Description: this is the transmit function that is called on TSO frames
2778  *  (support available on GMAC4 and newer chips).
2779  *  Diagram below show the ring programming in case of TSO frames:
2780  *
2781  *  First Descriptor
2782  *   --------
2783  *   | DES0 |---> buffer1 = L2/L3/L4 header
2784  *   | DES1 |---> TCP Payload (can continue on next descr...)
2785  *   | DES2 |---> buffer 1 and 2 len
2786  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2787  *   --------
2788  *      |
2789  *     ...
2790  *      |
2791  *   --------
2792  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2793  *   | DES1 | --|
2794  *   | DES2 | --> buffer 1 and 2 len
2795  *   | DES3 |
2796  *   --------
2797  *
2798  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2799  */
2800 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2801 {
2802         struct dma_desc *desc, *first, *mss_desc = NULL;
2803         struct stmmac_priv *priv = netdev_priv(dev);
2804         int nfrags = skb_shinfo(skb)->nr_frags;
2805         u32 queue = skb_get_queue_mapping(skb);
2806         unsigned int first_entry, des;
2807         struct stmmac_tx_queue *tx_q;
2808         int tmp_pay_len = 0;
2809         u32 pay_len, mss;
2810         u8 proto_hdr_len;
2811         int i;
2812
2813         tx_q = &priv->tx_queue[queue];
2814
2815         /* Compute header lengths */
2816         proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2817
2818         /* Desc availability based on threshold should be enough safe */
2819         if (unlikely(stmmac_tx_avail(priv, queue) <
2820                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2821                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2822                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2823                                                                 queue));
2824                         /* This is a hard error, log it. */
2825                         netdev_err(priv->dev,
2826                                    "%s: Tx Ring full when queue awake\n",
2827                                    __func__);
2828                 }
2829                 return NETDEV_TX_BUSY;
2830         }
2831
2832         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2833
2834         mss = skb_shinfo(skb)->gso_size;
2835
2836         /* set new MSS value if needed */
2837         if (mss != priv->mss) {
2838                 mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2839                 priv->hw->desc->set_mss(mss_desc, mss);
2840                 priv->mss = mss;
2841                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2842         }
2843
2844         if (netif_msg_tx_queued(priv)) {
2845                 pr_info("%s: tcphdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2846                         __func__, tcp_hdrlen(skb), proto_hdr_len, pay_len, mss);
2847                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2848                         skb->data_len);
2849         }
2850
2851         first_entry = tx_q->cur_tx;
2852
2853         desc = tx_q->dma_tx + first_entry;
2854         first = desc;
2855
2856         /* first descriptor: fill Headers on Buf1 */
2857         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2858                              DMA_TO_DEVICE);
2859         if (dma_mapping_error(priv->device, des))
2860                 goto dma_map_err;
2861
2862         tx_q->tx_skbuff_dma[first_entry].buf = des;
2863         tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2864
2865         first->des0 = cpu_to_le32(des);
2866
2867         /* Fill start of payload in buff2 of first descriptor */
2868         if (pay_len)
2869                 first->des1 = cpu_to_le32(des + proto_hdr_len);
2870
2871         /* If needed take extra descriptors to fill the remaining payload */
2872         tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
2873
2874         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
2875
2876         /* Prepare fragments */
2877         for (i = 0; i < nfrags; i++) {
2878                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2879
2880                 des = skb_frag_dma_map(priv->device, frag, 0,
2881                                        skb_frag_size(frag),
2882                                        DMA_TO_DEVICE);
2883                 if (dma_mapping_error(priv->device, des))
2884                         goto dma_map_err;
2885
2886                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
2887                                      (i == nfrags - 1), queue);
2888
2889                 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
2890                 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
2891                 tx_q->tx_skbuff[tx_q->cur_tx] = NULL;
2892                 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
2893         }
2894
2895         tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
2896
2897         /* Only the last descriptor gets to point to the skb. */
2898         tx_q->tx_skbuff[tx_q->cur_tx] = skb;
2899
2900         /* We've used all descriptors we need for this skb, however,
2901          * advance cur_tx so that it references a fresh descriptor.
2902          * ndo_start_xmit will fill this descriptor the next time it's
2903          * called and stmmac_tx_clean may clean up to this descriptor.
2904          */
2905         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2906
2907         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
2908                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
2909                           __func__);
2910                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
2911         }
2912
2913         dev->stats.tx_bytes += skb->len;
2914         priv->xstats.tx_tso_frames++;
2915         priv->xstats.tx_tso_nfrags += nfrags;
2916
2917         /* Manage tx mitigation */
2918         priv->tx_count_frames += nfrags + 1;
2919         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2920                 mod_timer(&priv->txtimer,
2921                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
2922         } else {
2923                 priv->tx_count_frames = 0;
2924                 priv->hw->desc->set_tx_ic(desc);
2925                 priv->xstats.tx_set_ic_bit++;
2926         }
2927
2928         skb_tx_timestamp(skb);
2929
2930         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
2931                      priv->hwts_tx_en)) {
2932                 /* declare that device is doing timestamping */
2933                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2934                 priv->hw->desc->enable_tx_timestamp(first);
2935         }
2936
2937         /* Complete the first descriptor before granting the DMA */
2938         priv->hw->desc->prepare_tso_tx_desc(first, 1,
2939                         proto_hdr_len,
2940                         pay_len,
2941                         1, tx_q->tx_skbuff_dma[first_entry].last_segment,
2942                         tcp_hdrlen(skb) / 4, (skb->len - proto_hdr_len));
2943
2944         /* If context desc is used to change MSS */
2945         if (mss_desc)
2946                 priv->hw->desc->set_tx_owner(mss_desc);
2947
2948         /* The own bit must be the latest setting done when prepare the
2949          * descriptor and then barrier is needed to make sure that
2950          * all is coherent before granting the DMA engine.
2951          */
2952         dma_wmb();
2953
2954         if (netif_msg_pktdata(priv)) {
2955                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
2956                         __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
2957                         tx_q->cur_tx, first, nfrags);
2958
2959                 priv->hw->desc->display_ring((void *)tx_q->dma_tx, DMA_TX_SIZE,
2960                                              0);
2961
2962                 pr_info(">>> frame to be transmitted: ");
2963                 print_pkt(skb->data, skb_headlen(skb));
2964         }
2965
2966         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
2967
2968         priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
2969                                        queue);
2970
2971         return NETDEV_TX_OK;
2972
2973 dma_map_err:
2974         dev_err(priv->device, "Tx dma map failed\n");
2975         dev_kfree_skb(skb);
2976         priv->dev->stats.tx_dropped++;
2977         return NETDEV_TX_OK;
2978 }
2979
2980 /**
2981  *  stmmac_xmit - Tx entry point of the driver
2982  *  @skb : the socket buffer
2983  *  @dev : device pointer
2984  *  Description : this is the tx entry point of the driver.
2985  *  It programs the chain or the ring and supports oversized frames
2986  *  and SG feature.
2987  */
2988 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
2989 {
2990         struct stmmac_priv *priv = netdev_priv(dev);
2991         unsigned int nopaged_len = skb_headlen(skb);
2992         int i, csum_insertion = 0, is_jumbo = 0;
2993         u32 queue = skb_get_queue_mapping(skb);
2994         int nfrags = skb_shinfo(skb)->nr_frags;
2995         int entry;
2996         unsigned int first_entry;
2997         struct dma_desc *desc, *first;
2998         struct stmmac_tx_queue *tx_q;
2999         unsigned int enh_desc;
3000         unsigned int des;
3001
3002         tx_q = &priv->tx_queue[queue];
3003
3004         /* Manage oversized TCP frames for GMAC4 device */
3005         if (skb_is_gso(skb) && priv->tso) {
3006                 if (skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
3007                         return stmmac_tso_xmit(skb, dev);
3008         }
3009
3010         if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
3011                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3012                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3013                                                                 queue));
3014                         /* This is a hard error, log it. */
3015                         netdev_err(priv->dev,
3016                                    "%s: Tx Ring full when queue awake\n",
3017                                    __func__);
3018                 }
3019                 return NETDEV_TX_BUSY;
3020         }
3021
3022         if (priv->tx_path_in_lpi_mode)
3023                 stmmac_disable_eee_mode(priv);
3024
3025         entry = tx_q->cur_tx;
3026         first_entry = entry;
3027
3028         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
3029
3030         if (likely(priv->extend_desc))
3031                 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3032         else
3033                 desc = tx_q->dma_tx + entry;
3034
3035         first = desc;
3036
3037         enh_desc = priv->plat->enh_desc;
3038         /* To program the descriptors according to the size of the frame */
3039         if (enh_desc)
3040                 is_jumbo = priv->hw->mode->is_jumbo_frm(skb->len, enh_desc);
3041
3042         if (unlikely(is_jumbo) && likely(priv->synopsys_id <
3043                                          DWMAC_CORE_4_00)) {
3044                 entry = priv->hw->mode->jumbo_frm(tx_q, skb, csum_insertion);
3045                 if (unlikely(entry < 0))
3046                         goto dma_map_err;
3047         }
3048
3049         for (i = 0; i < nfrags; i++) {
3050                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3051                 int len = skb_frag_size(frag);
3052                 bool last_segment = (i == (nfrags - 1));
3053
3054                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3055
3056                 if (likely(priv->extend_desc))
3057                         desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3058                 else
3059                         desc = tx_q->dma_tx + entry;
3060
3061                 des = skb_frag_dma_map(priv->device, frag, 0, len,
3062                                        DMA_TO_DEVICE);
3063                 if (dma_mapping_error(priv->device, des))
3064                         goto dma_map_err; /* should reuse desc w/o issues */
3065
3066                 tx_q->tx_skbuff[entry] = NULL;
3067
3068                 tx_q->tx_skbuff_dma[entry].buf = des;
3069                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3070                         desc->des0 = cpu_to_le32(des);
3071                 else
3072                         desc->des2 = cpu_to_le32(des);
3073
3074                 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3075                 tx_q->tx_skbuff_dma[entry].len = len;
3076                 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3077
3078                 /* Prepare the descriptor and set the own bit too */
3079                 priv->hw->desc->prepare_tx_desc(desc, 0, len, csum_insertion,
3080                                                 priv->mode, 1, last_segment,
3081                                                 skb->len);
3082         }
3083
3084         /* Only the last descriptor gets to point to the skb. */
3085         tx_q->tx_skbuff[entry] = skb;
3086
3087         /* We've used all descriptors we need for this skb, however,
3088          * advance cur_tx so that it references a fresh descriptor.
3089          * ndo_start_xmit will fill this descriptor the next time it's
3090          * called and stmmac_tx_clean may clean up to this descriptor.
3091          */
3092         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3093         tx_q->cur_tx = entry;
3094
3095         if (netif_msg_pktdata(priv)) {
3096                 void *tx_head;
3097
3098                 netdev_dbg(priv->dev,
3099                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3100                            __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3101                            entry, first, nfrags);
3102
3103                 if (priv->extend_desc)
3104                         tx_head = (void *)tx_q->dma_etx;
3105                 else
3106                         tx_head = (void *)tx_q->dma_tx;
3107
3108                 priv->hw->desc->display_ring(tx_head, DMA_TX_SIZE, false);
3109
3110                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3111                 print_pkt(skb->data, skb->len);
3112         }
3113
3114         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3115                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3116                           __func__);
3117                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3118         }
3119
3120         dev->stats.tx_bytes += skb->len;
3121
3122         /* According to the coalesce parameter the IC bit for the latest
3123          * segment is reset and the timer re-started to clean the tx status.
3124          * This approach takes care about the fragments: desc is the first
3125          * element in case of no SG.
3126          */
3127         priv->tx_count_frames += nfrags + 1;
3128         if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
3129                 mod_timer(&priv->txtimer,
3130                           STMMAC_COAL_TIMER(priv->tx_coal_timer));
3131         } else {
3132                 priv->tx_count_frames = 0;
3133                 priv->hw->desc->set_tx_ic(desc);
3134                 priv->xstats.tx_set_ic_bit++;
3135         }
3136
3137         skb_tx_timestamp(skb);
3138
3139         /* Ready to fill the first descriptor and set the OWN bit w/o any
3140          * problems because all the descriptors are actually ready to be
3141          * passed to the DMA engine.
3142          */
3143         if (likely(!is_jumbo)) {
3144                 bool last_segment = (nfrags == 0);
3145
3146                 des = dma_map_single(priv->device, skb->data,
3147                                      nopaged_len, DMA_TO_DEVICE);
3148                 if (dma_mapping_error(priv->device, des))
3149                         goto dma_map_err;
3150
3151                 tx_q->tx_skbuff_dma[first_entry].buf = des;
3152                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3153                         first->des0 = cpu_to_le32(des);
3154                 else
3155                         first->des2 = cpu_to_le32(des);
3156
3157                 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3158                 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3159
3160                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3161                              priv->hwts_tx_en)) {
3162                         /* declare that device is doing timestamping */
3163                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3164                         priv->hw->desc->enable_tx_timestamp(first);
3165                 }
3166
3167                 /* Prepare the first descriptor setting the OWN bit too */
3168                 priv->hw->desc->prepare_tx_desc(first, 1, nopaged_len,
3169                                                 csum_insertion, priv->mode, 1,
3170                                                 last_segment, skb->len);
3171
3172                 /* The own bit must be the latest setting done when prepare the
3173                  * descriptor and then barrier is needed to make sure that
3174                  * all is coherent before granting the DMA engine.
3175                  */
3176                 dma_wmb();
3177         }
3178
3179         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3180
3181         if (priv->synopsys_id < DWMAC_CORE_4_00)
3182                 priv->hw->dma->enable_dma_transmission(priv->ioaddr);
3183         else
3184                 priv->hw->dma->set_tx_tail_ptr(priv->ioaddr, tx_q->tx_tail_addr,
3185                                                queue);
3186
3187         return NETDEV_TX_OK;
3188
3189 dma_map_err:
3190         netdev_err(priv->dev, "Tx DMA map failed\n");
3191         dev_kfree_skb(skb);
3192         priv->dev->stats.tx_dropped++;
3193         return NETDEV_TX_OK;
3194 }
3195
3196 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3197 {
3198         struct ethhdr *ehdr;
3199         u16 vlanid;
3200
3201         if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) ==
3202             NETIF_F_HW_VLAN_CTAG_RX &&
3203             !__vlan_get_tag(skb, &vlanid)) {
3204                 /* pop the vlan tag */
3205                 ehdr = (struct ethhdr *)skb->data;
3206                 memmove(skb->data + VLAN_HLEN, ehdr, ETH_ALEN * 2);
3207                 skb_pull(skb, VLAN_HLEN);
3208                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlanid);
3209         }
3210 }
3211
3212
3213 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3214 {
3215         if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3216                 return 0;
3217
3218         return 1;
3219 }
3220
3221 /**
3222  * stmmac_rx_refill - refill used skb preallocated buffers
3223  * @priv: driver private structure
3224  * @queue: RX queue index
3225  * Description : this is to reallocate the skb for the reception process
3226  * that is based on zero-copy.
3227  */
3228 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3229 {
3230         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3231         int dirty = stmmac_rx_dirty(priv, queue);
3232         unsigned int entry = rx_q->dirty_rx;
3233
3234         int bfsize = priv->dma_buf_sz;
3235
3236         while (dirty-- > 0) {
3237                 struct dma_desc *p;
3238
3239                 if (priv->extend_desc)
3240                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3241                 else
3242                         p = rx_q->dma_rx + entry;
3243
3244                 if (likely(!rx_q->rx_skbuff[entry])) {
3245                         struct sk_buff *skb;
3246
3247                         skb = netdev_alloc_skb_ip_align(priv->dev, bfsize);
3248                         if (unlikely(!skb)) {
3249                                 /* so for a while no zero-copy! */
3250                                 rx_q->rx_zeroc_thresh = STMMAC_RX_THRESH;
3251                                 if (unlikely(net_ratelimit()))
3252                                         dev_err(priv->device,
3253                                                 "fail to alloc skb entry %d\n",
3254                                                 entry);
3255                                 break;
3256                         }
3257
3258                         rx_q->rx_skbuff[entry] = skb;
3259                         rx_q->rx_skbuff_dma[entry] =
3260                             dma_map_single(priv->device, skb->data, bfsize,
3261                                            DMA_FROM_DEVICE);
3262                         if (dma_mapping_error(priv->device,
3263                                               rx_q->rx_skbuff_dma[entry])) {
3264                                 netdev_err(priv->dev, "Rx DMA map failed\n");
3265                                 dev_kfree_skb(skb);
3266                                 break;
3267                         }
3268
3269                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00)) {
3270                                 p->des0 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3271                                 p->des1 = 0;
3272                         } else {
3273                                 p->des2 = cpu_to_le32(rx_q->rx_skbuff_dma[entry]);
3274                         }
3275                         if (priv->hw->mode->refill_desc3)
3276                                 priv->hw->mode->refill_desc3(rx_q, p);
3277
3278                         if (rx_q->rx_zeroc_thresh > 0)
3279                                 rx_q->rx_zeroc_thresh--;
3280
3281                         netif_dbg(priv, rx_status, priv->dev,
3282                                   "refill entry #%d\n", entry);
3283                 }
3284                 dma_wmb();
3285
3286                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3287                         priv->hw->desc->init_rx_desc(p, priv->use_riwt, 0, 0);
3288                 else
3289                         priv->hw->desc->set_rx_owner(p);
3290
3291                 dma_wmb();
3292
3293                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3294         }
3295         rx_q->dirty_rx = entry;
3296 }
3297
3298 /**
3299  * stmmac_rx - manage the receive process
3300  * @priv: driver private structure
3301  * @limit: napi bugget
3302  * @queue: RX queue index.
3303  * Description :  this the function called by the napi poll method.
3304  * It gets all the frames inside the ring.
3305  */
3306 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3307 {
3308         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3309         unsigned int entry = rx_q->cur_rx;
3310         int coe = priv->hw->rx_csum;
3311         unsigned int next_entry;
3312         unsigned int count = 0;
3313
3314         if (netif_msg_rx_status(priv)) {
3315                 void *rx_head;
3316
3317                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3318                 if (priv->extend_desc)
3319                         rx_head = (void *)rx_q->dma_erx;
3320                 else
3321                         rx_head = (void *)rx_q->dma_rx;
3322
3323                 priv->hw->desc->display_ring(rx_head, DMA_RX_SIZE, true);
3324         }
3325         while (count < limit) {
3326                 int status;
3327                 struct dma_desc *p;
3328                 struct dma_desc *np;
3329
3330                 if (priv->extend_desc)
3331                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3332                 else
3333                         p = rx_q->dma_rx + entry;
3334
3335                 /* read the status of the incoming frame */
3336                 status = priv->hw->desc->rx_status(&priv->dev->stats,
3337                                                    &priv->xstats, p);
3338                 /* check if managed by the DMA otherwise go ahead */
3339                 if (unlikely(status & dma_own))
3340                         break;
3341
3342                 count++;
3343
3344                 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3345                 next_entry = rx_q->cur_rx;
3346
3347                 if (priv->extend_desc)
3348                         np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3349                 else
3350                         np = rx_q->dma_rx + next_entry;
3351
3352                 prefetch(np);
3353
3354                 if ((priv->extend_desc) && (priv->hw->desc->rx_extended_status))
3355                         priv->hw->desc->rx_extended_status(&priv->dev->stats,
3356                                                            &priv->xstats,
3357                                                            rx_q->dma_erx +
3358                                                            entry);
3359                 if (unlikely(status == discard_frame)) {
3360                         priv->dev->stats.rx_errors++;
3361                         if (priv->hwts_rx_en && !priv->extend_desc) {
3362                                 /* DESC2 & DESC3 will be overwritten by device
3363                                  * with timestamp value, hence reinitialize
3364                                  * them in stmmac_rx_refill() function so that
3365                                  * device can reuse it.
3366                                  */
3367                                 dev_kfree_skb_any(rx_q->rx_skbuff[entry]);
3368                                 rx_q->rx_skbuff[entry] = NULL;
3369                                 dma_unmap_single(priv->device,
3370                                                  rx_q->rx_skbuff_dma[entry],
3371                                                  priv->dma_buf_sz,
3372                                                  DMA_FROM_DEVICE);
3373                         }
3374                 } else {
3375                         struct sk_buff *skb;
3376                         int frame_len;
3377                         unsigned int des;
3378
3379                         if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00))
3380                                 des = le32_to_cpu(p->des0);
3381                         else
3382                                 des = le32_to_cpu(p->des2);
3383
3384                         frame_len = priv->hw->desc->get_rx_frame_len(p, coe);
3385
3386                         /*  If frame length is greater than skb buffer size
3387                          *  (preallocated during init) then the packet is
3388                          *  ignored
3389                          */
3390                         if (frame_len > priv->dma_buf_sz) {
3391                                 netdev_err(priv->dev,
3392                                            "len %d larger than size (%d)\n",
3393                                            frame_len, priv->dma_buf_sz);
3394                                 priv->dev->stats.rx_length_errors++;
3395                                 break;
3396                         }
3397
3398                         /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3399                          * Type frames (LLC/LLC-SNAP)
3400                          */
3401                         if (unlikely(status != llc_snap))
3402                                 frame_len -= ETH_FCS_LEN;
3403
3404                         if (netif_msg_rx_status(priv)) {
3405                                 netdev_dbg(priv->dev, "\tdesc: %p [entry %d] buff=0x%x\n",
3406                                            p, entry, des);
3407                                 if (frame_len > ETH_FRAME_LEN)
3408                                         netdev_dbg(priv->dev, "frame size %d, COE: %d\n",
3409                                                    frame_len, status);
3410                         }
3411
3412                         /* The zero-copy is always used for all the sizes
3413                          * in case of GMAC4 because it needs
3414                          * to refill the used descriptors, always.
3415                          */
3416                         if (unlikely(!priv->plat->has_gmac4 &&
3417                                      ((frame_len < priv->rx_copybreak) ||
3418                                      stmmac_rx_threshold_count(rx_q)))) {
3419                                 skb = netdev_alloc_skb_ip_align(priv->dev,
3420                                                                 frame_len);
3421                                 if (unlikely(!skb)) {
3422                                         if (net_ratelimit())
3423                                                 dev_warn(priv->device,
3424                                                          "packet dropped\n");
3425                                         priv->dev->stats.rx_dropped++;
3426                                         break;
3427                                 }
3428
3429                                 dma_sync_single_for_cpu(priv->device,
3430                                                         rx_q->rx_skbuff_dma
3431                                                         [entry], frame_len,
3432                                                         DMA_FROM_DEVICE);
3433                                 skb_copy_to_linear_data(skb,
3434                                                         rx_q->
3435                                                         rx_skbuff[entry]->data,
3436                                                         frame_len);
3437
3438                                 skb_put(skb, frame_len);
3439                                 dma_sync_single_for_device(priv->device,
3440                                                            rx_q->rx_skbuff_dma
3441                                                            [entry], frame_len,
3442                                                            DMA_FROM_DEVICE);
3443                         } else {
3444                                 skb = rx_q->rx_skbuff[entry];
3445                                 if (unlikely(!skb)) {
3446                                         netdev_err(priv->dev,
3447                                                    "%s: Inconsistent Rx chain\n",
3448                                                    priv->dev->name);
3449                                         priv->dev->stats.rx_dropped++;
3450                                         break;
3451                                 }
3452                                 prefetch(skb->data - NET_IP_ALIGN);
3453                                 rx_q->rx_skbuff[entry] = NULL;
3454                                 rx_q->rx_zeroc_thresh++;
3455
3456                                 skb_put(skb, frame_len);
3457                                 dma_unmap_single(priv->device,
3458                                                  rx_q->rx_skbuff_dma[entry],
3459                                                  priv->dma_buf_sz,
3460                                                  DMA_FROM_DEVICE);
3461                         }
3462
3463                         if (netif_msg_pktdata(priv)) {
3464                                 netdev_dbg(priv->dev, "frame received (%dbytes)",
3465                                            frame_len);
3466                                 print_pkt(skb->data, frame_len);
3467                         }
3468
3469                         stmmac_get_rx_hwtstamp(priv, p, np, skb);
3470
3471                         stmmac_rx_vlan(priv->dev, skb);
3472
3473                         skb->protocol = eth_type_trans(skb, priv->dev);
3474
3475                         if (unlikely(!coe))
3476                                 skb_checksum_none_assert(skb);
3477                         else
3478                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
3479
3480                         napi_gro_receive(&rx_q->napi, skb);
3481
3482                         priv->dev->stats.rx_packets++;
3483                         priv->dev->stats.rx_bytes += frame_len;
3484                 }
3485                 entry = next_entry;
3486         }
3487
3488         stmmac_rx_refill(priv, queue);
3489
3490         priv->xstats.rx_pkt_n += count;
3491
3492         return count;
3493 }
3494
3495 /**
3496  *  stmmac_poll - stmmac poll method (NAPI)
3497  *  @napi : pointer to the napi structure.
3498  *  @budget : maximum number of packets that the current CPU can receive from
3499  *            all interfaces.
3500  *  Description :
3501  *  To look at the incoming frames and clear the tx resources.
3502  */
3503 static int stmmac_poll(struct napi_struct *napi, int budget)
3504 {
3505         struct stmmac_rx_queue *rx_q =
3506                 container_of(napi, struct stmmac_rx_queue, napi);
3507         struct stmmac_priv *priv = rx_q->priv_data;
3508         u32 tx_count = priv->plat->tx_queues_to_use;
3509         u32 chan = rx_q->queue_index;
3510         int work_done = 0;
3511         u32 queue;
3512
3513         priv->xstats.napi_poll++;
3514
3515         /* check all the queues */
3516         for (queue = 0; queue < tx_count; queue++)
3517                 stmmac_tx_clean(priv, queue);
3518
3519         work_done = stmmac_rx(priv, budget, rx_q->queue_index);
3520         if (work_done < budget) {
3521                 napi_complete_done(napi, work_done);
3522                 stmmac_enable_dma_irq(priv, chan);
3523         }
3524         return work_done;
3525 }
3526
3527 /**
3528  *  stmmac_tx_timeout
3529  *  @dev : Pointer to net device structure
3530  *  Description: this function is called when a packet transmission fails to
3531  *   complete within a reasonable time. The driver will mark the error in the
3532  *   netdev structure and arrange for the device to be reset to a sane state
3533  *   in order to transmit a new packet.
3534  */
3535 static void stmmac_tx_timeout(struct net_device *dev)
3536 {
3537         struct stmmac_priv *priv = netdev_priv(dev);
3538         u32 tx_count = priv->plat->tx_queues_to_use;
3539         u32 chan;
3540
3541         /* Clear Tx resources and restart transmitting again */
3542         for (chan = 0; chan < tx_count; chan++)
3543                 stmmac_tx_err(priv, chan);
3544 }
3545
3546 /**
3547  *  stmmac_set_rx_mode - entry point for multicast addressing
3548  *  @dev : pointer to the device structure
3549  *  Description:
3550  *  This function is a driver entry point which gets called by the kernel
3551  *  whenever multicast addresses must be enabled/disabled.
3552  *  Return value:
3553  *  void.
3554  */
3555 static void stmmac_set_rx_mode(struct net_device *dev)
3556 {
3557         struct stmmac_priv *priv = netdev_priv(dev);
3558
3559         priv->hw->mac->set_filter(priv->hw, dev);
3560 }
3561
3562 /**
3563  *  stmmac_change_mtu - entry point to change MTU size for the device.
3564  *  @dev : device pointer.
3565  *  @new_mtu : the new MTU size for the device.
3566  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3567  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3568  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3569  *  Return value:
3570  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3571  *  file on failure.
3572  */
3573 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3574 {
3575         struct stmmac_priv *priv = netdev_priv(dev);
3576
3577         if (netif_running(dev)) {
3578                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3579                 return -EBUSY;
3580         }
3581
3582         dev->mtu = new_mtu;
3583
3584         netdev_update_features(dev);
3585
3586         return 0;
3587 }
3588
3589 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3590                                              netdev_features_t features)
3591 {
3592         struct stmmac_priv *priv = netdev_priv(dev);
3593
3594         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3595                 features &= ~NETIF_F_RXCSUM;
3596
3597         if (!priv->plat->tx_coe)
3598                 features &= ~NETIF_F_CSUM_MASK;
3599
3600         /* Some GMAC devices have a bugged Jumbo frame support that
3601          * needs to have the Tx COE disabled for oversized frames
3602          * (due to limited buffer sizes). In this case we disable
3603          * the TX csum insertion in the TDES and not use SF.
3604          */
3605         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3606                 features &= ~NETIF_F_CSUM_MASK;
3607
3608         /* Disable tso if asked by ethtool */
3609         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3610                 if (features & NETIF_F_TSO)
3611                         priv->tso = true;
3612                 else
3613                         priv->tso = false;
3614         }
3615
3616         return features;
3617 }
3618
3619 static int stmmac_set_features(struct net_device *netdev,
3620                                netdev_features_t features)
3621 {
3622         struct stmmac_priv *priv = netdev_priv(netdev);
3623
3624         /* Keep the COE Type in case of csum is supporting */
3625         if (features & NETIF_F_RXCSUM)
3626                 priv->hw->rx_csum = priv->plat->rx_coe;
3627         else
3628                 priv->hw->rx_csum = 0;
3629         /* No check needed because rx_coe has been set before and it will be
3630          * fixed in case of issue.
3631          */
3632         priv->hw->mac->rx_ipc(priv->hw);
3633
3634         return 0;
3635 }
3636
3637 /**
3638  *  stmmac_interrupt - main ISR
3639  *  @irq: interrupt number.
3640  *  @dev_id: to pass the net device pointer.
3641  *  Description: this is the main driver interrupt service routine.
3642  *  It can call:
3643  *  o DMA service routine (to manage incoming frame reception and transmission
3644  *    status)
3645  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3646  *    interrupts.
3647  */
3648 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3649 {
3650         struct net_device *dev = (struct net_device *)dev_id;
3651         struct stmmac_priv *priv = netdev_priv(dev);
3652         u32 rx_cnt = priv->plat->rx_queues_to_use;
3653         u32 tx_cnt = priv->plat->tx_queues_to_use;
3654         u32 queues_count;
3655         u32 queue;
3656
3657         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3658
3659         if (priv->irq_wake)
3660                 pm_wakeup_event(priv->device, 0);
3661
3662         if (unlikely(!dev)) {
3663                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3664                 return IRQ_NONE;
3665         }
3666
3667         /* To handle GMAC own interrupts */
3668         if ((priv->plat->has_gmac) || (priv->plat->has_gmac4)) {
3669                 int status = priv->hw->mac->host_irq_status(priv->hw,
3670                                                             &priv->xstats);
3671
3672                 if (unlikely(status)) {
3673                         /* For LPI we need to save the tx status */
3674                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3675                                 priv->tx_path_in_lpi_mode = true;
3676                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3677                                 priv->tx_path_in_lpi_mode = false;
3678                 }
3679
3680                 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3681                         for (queue = 0; queue < queues_count; queue++) {
3682                                 struct stmmac_rx_queue *rx_q =
3683                                 &priv->rx_queue[queue];
3684
3685                                 status |=
3686                                 priv->hw->mac->host_mtl_irq_status(priv->hw,
3687                                                                    queue);
3688
3689                                 if (status & CORE_IRQ_MTL_RX_OVERFLOW &&
3690                                     priv->hw->dma->set_rx_tail_ptr)
3691                                         priv->hw->dma->set_rx_tail_ptr(priv->ioaddr,
3692                                                                 rx_q->rx_tail_addr,
3693                                                                 queue);
3694                         }
3695                 }
3696
3697                 /* PCS link status */
3698                 if (priv->hw->pcs) {
3699                         if (priv->xstats.pcs_link)
3700                                 netif_carrier_on(dev);
3701                         else
3702                                 netif_carrier_off(dev);
3703                 }
3704         }
3705
3706         /* To handle DMA interrupts */
3707         stmmac_dma_interrupt(priv);
3708
3709         return IRQ_HANDLED;
3710 }
3711
3712 #ifdef CONFIG_NET_POLL_CONTROLLER
3713 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3714  * to allow network I/O with interrupts disabled.
3715  */
3716 static void stmmac_poll_controller(struct net_device *dev)
3717 {
3718         disable_irq(dev->irq);
3719         stmmac_interrupt(dev->irq, dev);
3720         enable_irq(dev->irq);
3721 }
3722 #endif
3723
3724 /**
3725  *  stmmac_ioctl - Entry point for the Ioctl
3726  *  @dev: Device pointer.
3727  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3728  *  a proprietary structure used to pass information to the driver.
3729  *  @cmd: IOCTL command
3730  *  Description:
3731  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3732  */
3733 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3734 {
3735         int ret = -EOPNOTSUPP;
3736
3737         if (!netif_running(dev))
3738                 return -EINVAL;
3739
3740         switch (cmd) {
3741         case SIOCGMIIPHY:
3742         case SIOCGMIIREG:
3743         case SIOCSMIIREG:
3744                 if (!dev->phydev)
3745                         return -EINVAL;
3746                 ret = phy_mii_ioctl(dev->phydev, rq, cmd);
3747                 break;
3748         case SIOCSHWTSTAMP:
3749                 ret = stmmac_hwtstamp_ioctl(dev, rq);
3750                 break;
3751         default:
3752                 break;
3753         }
3754
3755         return ret;
3756 }
3757
3758 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
3759 {
3760         struct stmmac_priv *priv = netdev_priv(ndev);
3761         int ret = 0;
3762
3763         ret = eth_mac_addr(ndev, addr);
3764         if (ret)
3765                 return ret;
3766
3767         priv->hw->mac->set_umac_addr(priv->hw, ndev->dev_addr, 0);
3768
3769         return ret;
3770 }
3771
3772 #ifdef CONFIG_DEBUG_FS
3773 static struct dentry *stmmac_fs_dir;
3774
3775 static void sysfs_display_ring(void *head, int size, int extend_desc,
3776                                struct seq_file *seq)
3777 {
3778         int i;
3779         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
3780         struct dma_desc *p = (struct dma_desc *)head;
3781
3782         for (i = 0; i < size; i++) {
3783                 if (extend_desc) {
3784                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3785                                    i, (unsigned int)virt_to_phys(ep),
3786                                    le32_to_cpu(ep->basic.des0),
3787                                    le32_to_cpu(ep->basic.des1),
3788                                    le32_to_cpu(ep->basic.des2),
3789                                    le32_to_cpu(ep->basic.des3));
3790                         ep++;
3791                 } else {
3792                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
3793                                    i, (unsigned int)virt_to_phys(p),
3794                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
3795                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
3796                         p++;
3797                 }
3798                 seq_printf(seq, "\n");
3799         }
3800 }
3801
3802 static int stmmac_sysfs_ring_read(struct seq_file *seq, void *v)
3803 {
3804         struct net_device *dev = seq->private;
3805         struct stmmac_priv *priv = netdev_priv(dev);
3806         u32 rx_count = priv->plat->rx_queues_to_use;
3807         u32 tx_count = priv->plat->tx_queues_to_use;
3808         u32 queue;
3809
3810         for (queue = 0; queue < rx_count; queue++) {
3811                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3812
3813                 seq_printf(seq, "RX Queue %d:\n", queue);
3814
3815                 if (priv->extend_desc) {
3816                         seq_printf(seq, "Extended descriptor ring:\n");
3817                         sysfs_display_ring((void *)rx_q->dma_erx,
3818                                            DMA_RX_SIZE, 1, seq);
3819                 } else {
3820                         seq_printf(seq, "Descriptor ring:\n");
3821                         sysfs_display_ring((void *)rx_q->dma_rx,
3822                                            DMA_RX_SIZE, 0, seq);
3823                 }
3824         }
3825
3826         for (queue = 0; queue < tx_count; queue++) {
3827                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3828
3829                 seq_printf(seq, "TX Queue %d:\n", queue);
3830
3831                 if (priv->extend_desc) {
3832                         seq_printf(seq, "Extended descriptor ring:\n");
3833                         sysfs_display_ring((void *)tx_q->dma_etx,
3834                                            DMA_TX_SIZE, 1, seq);
3835                 } else {
3836                         seq_printf(seq, "Descriptor ring:\n");
3837                         sysfs_display_ring((void *)tx_q->dma_tx,
3838                                            DMA_TX_SIZE, 0, seq);
3839                 }
3840         }
3841
3842         return 0;
3843 }
3844
3845 static int stmmac_sysfs_ring_open(struct inode *inode, struct file *file)
3846 {
3847         return single_open(file, stmmac_sysfs_ring_read, inode->i_private);
3848 }
3849
3850 /* Debugfs files, should appear in /sys/kernel/debug/stmmaceth/eth0 */
3851
3852 static const struct file_operations stmmac_rings_status_fops = {
3853         .owner = THIS_MODULE,
3854         .open = stmmac_sysfs_ring_open,
3855         .read = seq_read,
3856         .llseek = seq_lseek,
3857         .release = single_release,
3858 };
3859
3860 static int stmmac_sysfs_dma_cap_read(struct seq_file *seq, void *v)
3861 {
3862         struct net_device *dev = seq->private;
3863         struct stmmac_priv *priv = netdev_priv(dev);
3864
3865         if (!priv->hw_cap_support) {
3866                 seq_printf(seq, "DMA HW features not supported\n");
3867                 return 0;
3868         }
3869
3870         seq_printf(seq, "==============================\n");
3871         seq_printf(seq, "\tDMA HW features\n");
3872         seq_printf(seq, "==============================\n");
3873
3874         seq_printf(seq, "\t10/100 Mbps: %s\n",
3875                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
3876         seq_printf(seq, "\t1000 Mbps: %s\n",
3877                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
3878         seq_printf(seq, "\tHalf duplex: %s\n",
3879                    (priv->dma_cap.half_duplex) ? "Y" : "N");
3880         seq_printf(seq, "\tHash Filter: %s\n",
3881                    (priv->dma_cap.hash_filter) ? "Y" : "N");
3882         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
3883                    (priv->dma_cap.multi_addr) ? "Y" : "N");
3884         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
3885                    (priv->dma_cap.pcs) ? "Y" : "N");
3886         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
3887                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
3888         seq_printf(seq, "\tPMT Remote wake up: %s\n",
3889                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
3890         seq_printf(seq, "\tPMT Magic Frame: %s\n",
3891                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
3892         seq_printf(seq, "\tRMON module: %s\n",
3893                    (priv->dma_cap.rmon) ? "Y" : "N");
3894         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
3895                    (priv->dma_cap.time_stamp) ? "Y" : "N");
3896         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
3897                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
3898         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
3899                    (priv->dma_cap.eee) ? "Y" : "N");
3900         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
3901         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
3902                    (priv->dma_cap.tx_coe) ? "Y" : "N");
3903         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
3904                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
3905                            (priv->dma_cap.rx_coe) ? "Y" : "N");
3906         } else {
3907                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
3908                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
3909                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
3910                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
3911         }
3912         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
3913                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
3914         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
3915                    priv->dma_cap.number_rx_channel);
3916         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
3917                    priv->dma_cap.number_tx_channel);
3918         seq_printf(seq, "\tEnhanced descriptors: %s\n",
3919                    (priv->dma_cap.enh_desc) ? "Y" : "N");
3920
3921         return 0;
3922 }
3923
3924 static int stmmac_sysfs_dma_cap_open(struct inode *inode, struct file *file)
3925 {
3926         return single_open(file, stmmac_sysfs_dma_cap_read, inode->i_private);
3927 }
3928
3929 static const struct file_operations stmmac_dma_cap_fops = {
3930         .owner = THIS_MODULE,
3931         .open = stmmac_sysfs_dma_cap_open,
3932         .read = seq_read,
3933         .llseek = seq_lseek,
3934         .release = single_release,
3935 };
3936
3937 static int stmmac_init_fs(struct net_device *dev)
3938 {
3939         struct stmmac_priv *priv = netdev_priv(dev);
3940
3941         /* Create per netdev entries */
3942         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
3943
3944         if (!priv->dbgfs_dir || IS_ERR(priv->dbgfs_dir)) {
3945                 netdev_err(priv->dev, "ERROR failed to create debugfs directory\n");
3946
3947                 return -ENOMEM;
3948         }
3949
3950         /* Entry to report DMA RX/TX rings */
3951         priv->dbgfs_rings_status =
3952                 debugfs_create_file("descriptors_status", S_IRUGO,
3953                                     priv->dbgfs_dir, dev,
3954                                     &stmmac_rings_status_fops);
3955
3956         if (!priv->dbgfs_rings_status || IS_ERR(priv->dbgfs_rings_status)) {
3957                 netdev_err(priv->dev, "ERROR creating stmmac ring debugfs file\n");
3958                 debugfs_remove_recursive(priv->dbgfs_dir);
3959
3960                 return -ENOMEM;
3961         }
3962
3963         /* Entry to report the DMA HW features */
3964         priv->dbgfs_dma_cap = debugfs_create_file("dma_cap", S_IRUGO,
3965                                             priv->dbgfs_dir,
3966                                             dev, &stmmac_dma_cap_fops);
3967
3968         if (!priv->dbgfs_dma_cap || IS_ERR(priv->dbgfs_dma_cap)) {
3969                 netdev_err(priv->dev, "ERROR creating stmmac MMC debugfs file\n");
3970                 debugfs_remove_recursive(priv->dbgfs_dir);
3971
3972                 return -ENOMEM;
3973         }
3974
3975         return 0;
3976 }
3977
3978 static void stmmac_exit_fs(struct net_device *dev)
3979 {
3980         struct stmmac_priv *priv = netdev_priv(dev);
3981
3982         debugfs_remove_recursive(priv->dbgfs_dir);
3983 }
3984 #endif /* CONFIG_DEBUG_FS */
3985
3986 static const struct net_device_ops stmmac_netdev_ops = {
3987         .ndo_open = stmmac_open,
3988         .ndo_start_xmit = stmmac_xmit,
3989         .ndo_stop = stmmac_release,
3990         .ndo_change_mtu = stmmac_change_mtu,
3991         .ndo_fix_features = stmmac_fix_features,
3992         .ndo_set_features = stmmac_set_features,
3993         .ndo_set_rx_mode = stmmac_set_rx_mode,
3994         .ndo_tx_timeout = stmmac_tx_timeout,
3995         .ndo_do_ioctl = stmmac_ioctl,
3996 #ifdef CONFIG_NET_POLL_CONTROLLER
3997         .ndo_poll_controller = stmmac_poll_controller,
3998 #endif
3999         .ndo_set_mac_address = stmmac_set_mac_address,
4000 };
4001
4002 /**
4003  *  stmmac_hw_init - Init the MAC device
4004  *  @priv: driver private structure
4005  *  Description: this function is to configure the MAC device according to
4006  *  some platform parameters or the HW capability register. It prepares the
4007  *  driver to use either ring or chain modes and to setup either enhanced or
4008  *  normal descriptors.
4009  */
4010 static int stmmac_hw_init(struct stmmac_priv *priv)
4011 {
4012         struct mac_device_info *mac;
4013
4014         /* Identify the MAC HW device */
4015         if (priv->plat->setup) {
4016                 mac = priv->plat->setup(priv);
4017         } else if (priv->plat->has_gmac) {
4018                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
4019                 mac = dwmac1000_setup(priv->ioaddr,
4020                                       priv->plat->multicast_filter_bins,
4021                                       priv->plat->unicast_filter_entries,
4022                                       &priv->synopsys_id);
4023         } else if (priv->plat->has_gmac4) {
4024                 priv->dev->priv_flags |= IFF_UNICAST_FLT;
4025                 mac = dwmac4_setup(priv->ioaddr,
4026                                    priv->plat->multicast_filter_bins,
4027                                    priv->plat->unicast_filter_entries,
4028                                    &priv->synopsys_id);
4029         } else {
4030                 mac = dwmac100_setup(priv->ioaddr, &priv->synopsys_id);
4031         }
4032         if (!mac)
4033                 return -ENOMEM;
4034
4035         priv->hw = mac;
4036
4037         /* dwmac-sun8i only work in chain mode */
4038         if (priv->plat->has_sun8i)
4039                 chain_mode = 1;
4040
4041         /* To use the chained or ring mode */
4042         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
4043                 priv->hw->mode = &dwmac4_ring_mode_ops;
4044         } else {
4045                 if (chain_mode) {
4046                         priv->hw->mode = &chain_mode_ops;
4047                         dev_info(priv->device, "Chain mode enabled\n");
4048                         priv->mode = STMMAC_CHAIN_MODE;
4049                 } else {
4050                         priv->hw->mode = &ring_mode_ops;
4051                         dev_info(priv->device, "Ring mode enabled\n");
4052                         priv->mode = STMMAC_RING_MODE;
4053                 }
4054         }
4055
4056         /* Get the HW capability (new GMAC newer than 3.50a) */
4057         priv->hw_cap_support = stmmac_get_hw_features(priv);
4058         if (priv->hw_cap_support) {
4059                 dev_info(priv->device, "DMA HW capability register supported\n");
4060
4061                 /* We can override some gmac/dma configuration fields: e.g.
4062                  * enh_desc, tx_coe (e.g. that are passed through the
4063                  * platform) with the values from the HW capability
4064                  * register (if supported).
4065                  */
4066                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
4067                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4068                 priv->hw->pmt = priv->plat->pmt;
4069
4070                 /* TXCOE doesn't work in thresh DMA mode */
4071                 if (priv->plat->force_thresh_dma_mode)
4072                         priv->plat->tx_coe = 0;
4073                 else
4074                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
4075
4076                 /* In case of GMAC4 rx_coe is from HW cap register. */
4077                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
4078
4079                 if (priv->dma_cap.rx_coe_type2)
4080                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4081                 else if (priv->dma_cap.rx_coe_type1)
4082                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4083
4084         } else {
4085                 dev_info(priv->device, "No HW DMA feature register supported\n");
4086         }
4087
4088         /* To use alternate (extended), normal or GMAC4 descriptor structures */
4089         if (priv->synopsys_id >= DWMAC_CORE_4_00)
4090                 priv->hw->desc = &dwmac4_desc_ops;
4091         else
4092                 stmmac_selec_desc_mode(priv);
4093
4094         if (priv->plat->rx_coe) {
4095                 priv->hw->rx_csum = priv->plat->rx_coe;
4096                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4097                 if (priv->synopsys_id < DWMAC_CORE_4_00)
4098                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4099         }
4100         if (priv->plat->tx_coe)
4101                 dev_info(priv->device, "TX Checksum insertion supported\n");
4102
4103         if (priv->plat->pmt) {
4104                 dev_info(priv->device, "Wake-Up On Lan supported\n");
4105                 device_set_wakeup_capable(priv->device, 1);
4106         }
4107
4108         if (priv->dma_cap.tsoen)
4109                 dev_info(priv->device, "TSO supported\n");
4110
4111         return 0;
4112 }
4113
4114 /**
4115  * stmmac_dvr_probe
4116  * @device: device pointer
4117  * @plat_dat: platform data pointer
4118  * @res: stmmac resource pointer
4119  * Description: this is the main probe function used to
4120  * call the alloc_etherdev, allocate the priv structure.
4121  * Return:
4122  * returns 0 on success, otherwise errno.
4123  */
4124 int stmmac_dvr_probe(struct device *device,
4125                      struct plat_stmmacenet_data *plat_dat,
4126                      struct stmmac_resources *res)
4127 {
4128         struct net_device *ndev = NULL;
4129         struct stmmac_priv *priv;
4130         int ret = 0;
4131         u32 queue;
4132
4133         ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
4134                                   MTL_MAX_TX_QUEUES,
4135                                   MTL_MAX_RX_QUEUES);
4136         if (!ndev)
4137                 return -ENOMEM;
4138
4139         SET_NETDEV_DEV(ndev, device);
4140
4141         priv = netdev_priv(ndev);
4142         priv->device = device;
4143         priv->dev = ndev;
4144
4145         stmmac_set_ethtool_ops(ndev);
4146         priv->pause = pause;
4147         priv->plat = plat_dat;
4148         priv->ioaddr = res->addr;
4149         priv->dev->base_addr = (unsigned long)res->addr;
4150
4151         priv->dev->irq = res->irq;
4152         priv->wol_irq = res->wol_irq;
4153         priv->lpi_irq = res->lpi_irq;
4154
4155         if (res->mac)
4156                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4157
4158         dev_set_drvdata(device, priv->dev);
4159
4160         /* Verify driver arguments */
4161         stmmac_verify_args();
4162
4163         /* Override with kernel parameters if supplied XXX CRS XXX
4164          * this needs to have multiple instances
4165          */
4166         if ((phyaddr >= 0) && (phyaddr <= 31))
4167                 priv->plat->phy_addr = phyaddr;
4168
4169         if (priv->plat->stmmac_rst) {
4170                 ret = reset_control_assert(priv->plat->stmmac_rst);
4171                 reset_control_deassert(priv->plat->stmmac_rst);
4172                 /* Some reset controllers have only reset callback instead of
4173                  * assert + deassert callbacks pair.
4174                  */
4175                 if (ret == -ENOTSUPP)
4176                         reset_control_reset(priv->plat->stmmac_rst);
4177         }
4178
4179         /* Init MAC and get the capabilities */
4180         ret = stmmac_hw_init(priv);
4181         if (ret)
4182                 goto error_hw_init;
4183
4184         /* Configure real RX and TX queues */
4185         netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4186         netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4187
4188         ndev->netdev_ops = &stmmac_netdev_ops;
4189
4190         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4191                             NETIF_F_RXCSUM;
4192
4193         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4194                 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
4195                 priv->tso = true;
4196                 dev_info(priv->device, "TSO feature enabled\n");
4197         }
4198         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4199         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4200 #ifdef STMMAC_VLAN_TAG_USED
4201         /* Both mac100 and gmac support receive VLAN tag detection */
4202         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
4203 #endif
4204         priv->msg_enable = netif_msg_init(debug, default_msg_level);
4205
4206         /* MTU range: 46 - hw-specific max */
4207         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4208         if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4209                 ndev->max_mtu = JUMBO_LEN;
4210         else
4211                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4212         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4213          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4214          */
4215         if ((priv->plat->maxmtu < ndev->max_mtu) &&
4216             (priv->plat->maxmtu >= ndev->min_mtu))
4217                 ndev->max_mtu = priv->plat->maxmtu;
4218         else if (priv->plat->maxmtu < ndev->min_mtu)
4219                 dev_warn(priv->device,
4220                          "%s: warning: maxmtu having invalid value (%d)\n",
4221                          __func__, priv->plat->maxmtu);
4222
4223         if (flow_ctrl)
4224                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
4225
4226         /* Rx Watchdog is available in the COREs newer than the 3.40.
4227          * In some case, for example on bugged HW this feature
4228          * has to be disable and this can be done by passing the
4229          * riwt_off field from the platform.
4230          */
4231         if ((priv->synopsys_id >= DWMAC_CORE_3_50) && (!priv->plat->riwt_off)) {
4232                 priv->use_riwt = 1;
4233                 dev_info(priv->device,
4234                          "Enable RX Mitigation via HW Watchdog Timer\n");
4235         }
4236
4237         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4238                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4239
4240                 netif_napi_add(ndev, &rx_q->napi, stmmac_poll,
4241                                (8 * priv->plat->rx_queues_to_use));
4242         }
4243
4244         spin_lock_init(&priv->lock);
4245
4246         /* If a specific clk_csr value is passed from the platform
4247          * this means that the CSR Clock Range selection cannot be
4248          * changed at run-time and it is fixed. Viceversa the driver'll try to
4249          * set the MDC clock dynamically according to the csr actual
4250          * clock input.
4251          */
4252         if (!priv->plat->clk_csr)
4253                 stmmac_clk_csr_set(priv);
4254         else
4255                 priv->clk_csr = priv->plat->clk_csr;
4256
4257         stmmac_check_pcs_mode(priv);
4258
4259         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4260             priv->hw->pcs != STMMAC_PCS_TBI &&
4261             priv->hw->pcs != STMMAC_PCS_RTBI) {
4262                 /* MDIO bus Registration */
4263                 ret = stmmac_mdio_register(ndev);
4264                 if (ret < 0) {
4265                         dev_err(priv->device,
4266                                 "%s: MDIO bus (id: %d) registration failed",
4267                                 __func__, priv->plat->bus_id);
4268                         goto error_mdio_register;
4269                 }
4270         }
4271
4272         ret = register_netdev(ndev);
4273         if (ret) {
4274                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4275                         __func__, ret);
4276                 goto error_netdev_register;
4277         }
4278
4279         return ret;
4280
4281 error_netdev_register:
4282         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4283             priv->hw->pcs != STMMAC_PCS_TBI &&
4284             priv->hw->pcs != STMMAC_PCS_RTBI)
4285                 stmmac_mdio_unregister(ndev);
4286 error_mdio_register:
4287         for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4288                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4289
4290                 netif_napi_del(&rx_q->napi);
4291         }
4292 error_hw_init:
4293         free_netdev(ndev);
4294
4295         return ret;
4296 }
4297 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4298
4299 /**
4300  * stmmac_dvr_remove
4301  * @dev: device pointer
4302  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4303  * changes the link status, releases the DMA descriptor rings.
4304  */
4305 int stmmac_dvr_remove(struct device *dev)
4306 {
4307         struct net_device *ndev = dev_get_drvdata(dev);
4308         struct stmmac_priv *priv = netdev_priv(ndev);
4309
4310         netdev_info(priv->dev, "%s: removing driver", __func__);
4311
4312         stmmac_stop_all_dma(priv);
4313
4314         priv->hw->mac->set_mac(priv->ioaddr, false);
4315         netif_carrier_off(ndev);
4316         unregister_netdev(ndev);
4317         if (priv->plat->stmmac_rst)
4318                 reset_control_assert(priv->plat->stmmac_rst);
4319         clk_disable_unprepare(priv->plat->pclk);
4320         clk_disable_unprepare(priv->plat->stmmac_clk);
4321         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4322             priv->hw->pcs != STMMAC_PCS_TBI &&
4323             priv->hw->pcs != STMMAC_PCS_RTBI)
4324                 stmmac_mdio_unregister(ndev);
4325         free_netdev(ndev);
4326
4327         return 0;
4328 }
4329 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4330
4331 /**
4332  * stmmac_suspend - suspend callback
4333  * @dev: device pointer
4334  * Description: this is the function to suspend the device and it is called
4335  * by the platform driver to stop the network queue, release the resources,
4336  * program the PMT register (for WoL), clean and release driver resources.
4337  */
4338 int stmmac_suspend(struct device *dev)
4339 {
4340         struct net_device *ndev = dev_get_drvdata(dev);
4341         struct stmmac_priv *priv = netdev_priv(ndev);
4342         unsigned long flags;
4343
4344         if (!ndev || !netif_running(ndev))
4345                 return 0;
4346
4347         if (ndev->phydev)
4348                 phy_stop(ndev->phydev);
4349
4350         spin_lock_irqsave(&priv->lock, flags);
4351
4352         netif_device_detach(ndev);
4353         stmmac_stop_all_queues(priv);
4354
4355         stmmac_disable_all_queues(priv);
4356
4357         /* Stop TX/RX DMA */
4358         stmmac_stop_all_dma(priv);
4359
4360         /* Enable Power down mode by programming the PMT regs */
4361         if (device_may_wakeup(priv->device)) {
4362                 priv->hw->mac->pmt(priv->hw, priv->wolopts);
4363                 priv->irq_wake = 1;
4364         } else {
4365                 priv->hw->mac->set_mac(priv->ioaddr, false);
4366                 pinctrl_pm_select_sleep_state(priv->device);
4367                 /* Disable clock in case of PWM is off */
4368                 clk_disable(priv->plat->pclk);
4369                 clk_disable(priv->plat->stmmac_clk);
4370         }
4371         spin_unlock_irqrestore(&priv->lock, flags);
4372
4373         priv->oldlink = false;
4374         priv->speed = SPEED_UNKNOWN;
4375         priv->oldduplex = DUPLEX_UNKNOWN;
4376         return 0;
4377 }
4378 EXPORT_SYMBOL_GPL(stmmac_suspend);
4379
4380 /**
4381  * stmmac_reset_queues_param - reset queue parameters
4382  * @dev: device pointer
4383  */
4384 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4385 {
4386         u32 rx_cnt = priv->plat->rx_queues_to_use;
4387         u32 tx_cnt = priv->plat->tx_queues_to_use;
4388         u32 queue;
4389
4390         for (queue = 0; queue < rx_cnt; queue++) {
4391                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4392
4393                 rx_q->cur_rx = 0;
4394                 rx_q->dirty_rx = 0;
4395         }
4396
4397         for (queue = 0; queue < tx_cnt; queue++) {
4398                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4399
4400                 tx_q->cur_tx = 0;
4401                 tx_q->dirty_tx = 0;
4402         }
4403 }
4404
4405 /**
4406  * stmmac_resume - resume callback
4407  * @dev: device pointer
4408  * Description: when resume this function is invoked to setup the DMA and CORE
4409  * in a usable state.
4410  */
4411 int stmmac_resume(struct device *dev)
4412 {
4413         struct net_device *ndev = dev_get_drvdata(dev);
4414         struct stmmac_priv *priv = netdev_priv(ndev);
4415         unsigned long flags;
4416
4417         if (!netif_running(ndev))
4418                 return 0;
4419
4420         /* Power Down bit, into the PM register, is cleared
4421          * automatically as soon as a magic packet or a Wake-up frame
4422          * is received. Anyway, it's better to manually clear
4423          * this bit because it can generate problems while resuming
4424          * from another devices (e.g. serial console).
4425          */
4426         if (device_may_wakeup(priv->device)) {
4427                 spin_lock_irqsave(&priv->lock, flags);
4428                 priv->hw->mac->pmt(priv->hw, 0);
4429                 spin_unlock_irqrestore(&priv->lock, flags);
4430                 priv->irq_wake = 0;
4431         } else {
4432                 pinctrl_pm_select_default_state(priv->device);
4433                 /* enable the clk previously disabled */
4434                 clk_enable(priv->plat->stmmac_clk);
4435                 clk_enable(priv->plat->pclk);
4436                 /* reset the phy so that it's ready */
4437                 if (priv->mii)
4438                         stmmac_mdio_reset(priv->mii);
4439         }
4440
4441         netif_device_attach(ndev);
4442
4443         spin_lock_irqsave(&priv->lock, flags);
4444
4445         stmmac_reset_queues_param(priv);
4446
4447         /* reset private mss value to force mss context settings at
4448          * next tso xmit (only used for gmac4).
4449          */
4450         priv->mss = 0;
4451
4452         stmmac_clear_descriptors(priv);
4453
4454         stmmac_hw_setup(ndev, false);
4455         stmmac_init_tx_coalesce(priv);
4456         stmmac_set_rx_mode(ndev);
4457
4458         stmmac_enable_all_queues(priv);
4459
4460         stmmac_start_all_queues(priv);
4461
4462         spin_unlock_irqrestore(&priv->lock, flags);
4463
4464         if (ndev->phydev)
4465                 phy_start(ndev->phydev);
4466
4467         return 0;
4468 }
4469 EXPORT_SYMBOL_GPL(stmmac_resume);
4470
4471 #ifndef MODULE
4472 static int __init stmmac_cmdline_opt(char *str)
4473 {
4474         char *opt;
4475
4476         if (!str || !*str)
4477                 return -EINVAL;
4478         while ((opt = strsep(&str, ",")) != NULL) {
4479                 if (!strncmp(opt, "debug:", 6)) {
4480                         if (kstrtoint(opt + 6, 0, &debug))
4481                                 goto err;
4482                 } else if (!strncmp(opt, "phyaddr:", 8)) {
4483                         if (kstrtoint(opt + 8, 0, &phyaddr))
4484                                 goto err;
4485                 } else if (!strncmp(opt, "buf_sz:", 7)) {
4486                         if (kstrtoint(opt + 7, 0, &buf_sz))
4487                                 goto err;
4488                 } else if (!strncmp(opt, "tc:", 3)) {
4489                         if (kstrtoint(opt + 3, 0, &tc))
4490                                 goto err;
4491                 } else if (!strncmp(opt, "watchdog:", 9)) {
4492                         if (kstrtoint(opt + 9, 0, &watchdog))
4493                                 goto err;
4494                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
4495                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
4496                                 goto err;
4497                 } else if (!strncmp(opt, "pause:", 6)) {
4498                         if (kstrtoint(opt + 6, 0, &pause))
4499                                 goto err;
4500                 } else if (!strncmp(opt, "eee_timer:", 10)) {
4501                         if (kstrtoint(opt + 10, 0, &eee_timer))
4502                                 goto err;
4503                 } else if (!strncmp(opt, "chain_mode:", 11)) {
4504                         if (kstrtoint(opt + 11, 0, &chain_mode))
4505                                 goto err;
4506                 }
4507         }
4508         return 0;
4509
4510 err:
4511         pr_err("%s: ERROR broken module parameter conversion", __func__);
4512         return -EINVAL;
4513 }
4514
4515 __setup("stmmaceth=", stmmac_cmdline_opt);
4516 #endif /* MODULE */
4517
4518 static int __init stmmac_init(void)
4519 {
4520 #ifdef CONFIG_DEBUG_FS
4521         /* Create debugfs main directory if it doesn't exist yet */
4522         if (!stmmac_fs_dir) {
4523                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
4524
4525                 if (!stmmac_fs_dir || IS_ERR(stmmac_fs_dir)) {
4526                         pr_err("ERROR %s, debugfs create directory failed\n",
4527                                STMMAC_RESOURCE_NAME);
4528
4529                         return -ENOMEM;
4530                 }
4531         }
4532 #endif
4533
4534         return 0;
4535 }
4536
4537 static void __exit stmmac_exit(void)
4538 {
4539 #ifdef CONFIG_DEBUG_FS
4540         debugfs_remove_recursive(stmmac_fs_dir);
4541 #endif
4542 }
4543
4544 module_init(stmmac_init)
4545 module_exit(stmmac_exit)
4546
4547 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
4548 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
4549 MODULE_LICENSE("GPL");