USB: serial: option: add Fibocom NL678 series
[sfrench/cifs-2.6.git] / drivers / net / ethernet / ti / cpsw.c
1 /*
2  * Texas Instruments Ethernet Switch Driver
3  *
4  * Copyright (C) 2012 Texas Instruments
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License as
8  * published by the Free Software Foundation version 2.
9  *
10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11  * kind, whether express or implied; without even the implied warranty
12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  */
15
16 #include <linux/kernel.h>
17 #include <linux/io.h>
18 #include <linux/clk.h>
19 #include <linux/timer.h>
20 #include <linux/module.h>
21 #include <linux/platform_device.h>
22 #include <linux/irqreturn.h>
23 #include <linux/interrupt.h>
24 #include <linux/if_ether.h>
25 #include <linux/etherdevice.h>
26 #include <linux/netdevice.h>
27 #include <linux/net_tstamp.h>
28 #include <linux/phy.h>
29 #include <linux/phy/phy.h>
30 #include <linux/workqueue.h>
31 #include <linux/delay.h>
32 #include <linux/pm_runtime.h>
33 #include <linux/gpio/consumer.h>
34 #include <linux/of.h>
35 #include <linux/of_mdio.h>
36 #include <linux/of_net.h>
37 #include <linux/of_device.h>
38 #include <linux/if_vlan.h>
39 #include <linux/kmemleak.h>
40 #include <linux/sys_soc.h>
41
42 #include <linux/pinctrl/consumer.h>
43 #include <net/pkt_cls.h>
44
45 #include "cpsw.h"
46 #include "cpsw_ale.h"
47 #include "cpts.h"
48 #include "davinci_cpdma.h"
49
50 #include <net/pkt_sched.h>
51
52 #define CPSW_DEBUG      (NETIF_MSG_HW           | NETIF_MSG_WOL         | \
53                          NETIF_MSG_DRV          | NETIF_MSG_LINK        | \
54                          NETIF_MSG_IFUP         | NETIF_MSG_INTR        | \
55                          NETIF_MSG_PROBE        | NETIF_MSG_TIMER       | \
56                          NETIF_MSG_IFDOWN       | NETIF_MSG_RX_ERR      | \
57                          NETIF_MSG_TX_ERR       | NETIF_MSG_TX_DONE     | \
58                          NETIF_MSG_PKTDATA      | NETIF_MSG_TX_QUEUED   | \
59                          NETIF_MSG_RX_STATUS)
60
61 #define cpsw_info(priv, type, format, ...)              \
62 do {                                                            \
63         if (netif_msg_##type(priv) && net_ratelimit())          \
64                 dev_info(priv->dev, format, ## __VA_ARGS__);    \
65 } while (0)
66
67 #define cpsw_err(priv, type, format, ...)               \
68 do {                                                            \
69         if (netif_msg_##type(priv) && net_ratelimit())          \
70                 dev_err(priv->dev, format, ## __VA_ARGS__);     \
71 } while (0)
72
73 #define cpsw_dbg(priv, type, format, ...)               \
74 do {                                                            \
75         if (netif_msg_##type(priv) && net_ratelimit())          \
76                 dev_dbg(priv->dev, format, ## __VA_ARGS__);     \
77 } while (0)
78
79 #define cpsw_notice(priv, type, format, ...)            \
80 do {                                                            \
81         if (netif_msg_##type(priv) && net_ratelimit())          \
82                 dev_notice(priv->dev, format, ## __VA_ARGS__);  \
83 } while (0)
84
85 #define ALE_ALL_PORTS           0x7
86
87 #define CPSW_MAJOR_VERSION(reg)         (reg >> 8 & 0x7)
88 #define CPSW_MINOR_VERSION(reg)         (reg & 0xff)
89 #define CPSW_RTL_VERSION(reg)           ((reg >> 11) & 0x1f)
90
91 #define CPSW_VERSION_1          0x19010a
92 #define CPSW_VERSION_2          0x19010c
93 #define CPSW_VERSION_3          0x19010f
94 #define CPSW_VERSION_4          0x190112
95
96 #define HOST_PORT_NUM           0
97 #define CPSW_ALE_PORTS_NUM      3
98 #define SLIVER_SIZE             0x40
99
100 #define CPSW1_HOST_PORT_OFFSET  0x028
101 #define CPSW1_SLAVE_OFFSET      0x050
102 #define CPSW1_SLAVE_SIZE        0x040
103 #define CPSW1_CPDMA_OFFSET      0x100
104 #define CPSW1_STATERAM_OFFSET   0x200
105 #define CPSW1_HW_STATS          0x400
106 #define CPSW1_CPTS_OFFSET       0x500
107 #define CPSW1_ALE_OFFSET        0x600
108 #define CPSW1_SLIVER_OFFSET     0x700
109
110 #define CPSW2_HOST_PORT_OFFSET  0x108
111 #define CPSW2_SLAVE_OFFSET      0x200
112 #define CPSW2_SLAVE_SIZE        0x100
113 #define CPSW2_CPDMA_OFFSET      0x800
114 #define CPSW2_HW_STATS          0x900
115 #define CPSW2_STATERAM_OFFSET   0xa00
116 #define CPSW2_CPTS_OFFSET       0xc00
117 #define CPSW2_ALE_OFFSET        0xd00
118 #define CPSW2_SLIVER_OFFSET     0xd80
119 #define CPSW2_BD_OFFSET         0x2000
120
121 #define CPDMA_RXTHRESH          0x0c0
122 #define CPDMA_RXFREE            0x0e0
123 #define CPDMA_TXHDP             0x00
124 #define CPDMA_RXHDP             0x20
125 #define CPDMA_TXCP              0x40
126 #define CPDMA_RXCP              0x60
127
128 #define CPSW_POLL_WEIGHT        64
129 #define CPSW_RX_VLAN_ENCAP_HDR_SIZE             4
130 #define CPSW_MIN_PACKET_SIZE    (VLAN_ETH_ZLEN)
131 #define CPSW_MAX_PACKET_SIZE    (VLAN_ETH_FRAME_LEN +\
132                                  ETH_FCS_LEN +\
133                                  CPSW_RX_VLAN_ENCAP_HDR_SIZE)
134
135 #define RX_PRIORITY_MAPPING     0x76543210
136 #define TX_PRIORITY_MAPPING     0x33221100
137 #define CPDMA_TX_PRIORITY_MAP   0x76543210
138
139 #define CPSW_VLAN_AWARE         BIT(1)
140 #define CPSW_RX_VLAN_ENCAP      BIT(2)
141 #define CPSW_ALE_VLAN_AWARE     1
142
143 #define CPSW_FIFO_NORMAL_MODE           (0 << 16)
144 #define CPSW_FIFO_DUAL_MAC_MODE         (1 << 16)
145 #define CPSW_FIFO_RATE_LIMIT_MODE       (2 << 16)
146
147 #define CPSW_INTPACEEN          (0x3f << 16)
148 #define CPSW_INTPRESCALE_MASK   (0x7FF << 0)
149 #define CPSW_CMINTMAX_CNT       63
150 #define CPSW_CMINTMIN_CNT       2
151 #define CPSW_CMINTMAX_INTVL     (1000 / CPSW_CMINTMIN_CNT)
152 #define CPSW_CMINTMIN_INTVL     ((1000 / CPSW_CMINTMAX_CNT) + 1)
153
154 #define cpsw_slave_index(cpsw, priv)                            \
155                 ((cpsw->data.dual_emac) ? priv->emac_port :     \
156                 cpsw->data.active_slave)
157 #define IRQ_NUM                 2
158 #define CPSW_MAX_QUEUES         8
159 #define CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT 256
160 #define CPSW_FIFO_QUEUE_TYPE_SHIFT      16
161 #define CPSW_FIFO_SHAPE_EN_SHIFT        16
162 #define CPSW_FIFO_RATE_EN_SHIFT         20
163 #define CPSW_TC_NUM                     4
164 #define CPSW_FIFO_SHAPERS_NUM           (CPSW_TC_NUM - 1)
165 #define CPSW_PCT_MASK                   0x7f
166
167 #define CPSW_RX_VLAN_ENCAP_HDR_PRIO_SHIFT       29
168 #define CPSW_RX_VLAN_ENCAP_HDR_PRIO_MSK         GENMASK(2, 0)
169 #define CPSW_RX_VLAN_ENCAP_HDR_VID_SHIFT        16
170 #define CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_SHIFT   8
171 #define CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_MSK     GENMASK(1, 0)
172 enum {
173         CPSW_RX_VLAN_ENCAP_HDR_PKT_VLAN_TAG = 0,
174         CPSW_RX_VLAN_ENCAP_HDR_PKT_RESERV,
175         CPSW_RX_VLAN_ENCAP_HDR_PKT_PRIO_TAG,
176         CPSW_RX_VLAN_ENCAP_HDR_PKT_UNTAG,
177 };
178
179 static int debug_level;
180 module_param(debug_level, int, 0);
181 MODULE_PARM_DESC(debug_level, "cpsw debug level (NETIF_MSG bits)");
182
183 static int ale_ageout = 10;
184 module_param(ale_ageout, int, 0);
185 MODULE_PARM_DESC(ale_ageout, "cpsw ale ageout interval (seconds)");
186
187 static int rx_packet_max = CPSW_MAX_PACKET_SIZE;
188 module_param(rx_packet_max, int, 0);
189 MODULE_PARM_DESC(rx_packet_max, "maximum receive packet size (bytes)");
190
191 static int descs_pool_size = CPSW_CPDMA_DESCS_POOL_SIZE_DEFAULT;
192 module_param(descs_pool_size, int, 0444);
193 MODULE_PARM_DESC(descs_pool_size, "Number of CPDMA CPPI descriptors in pool");
194
195 struct cpsw_wr_regs {
196         u32     id_ver;
197         u32     soft_reset;
198         u32     control;
199         u32     int_control;
200         u32     rx_thresh_en;
201         u32     rx_en;
202         u32     tx_en;
203         u32     misc_en;
204         u32     mem_allign1[8];
205         u32     rx_thresh_stat;
206         u32     rx_stat;
207         u32     tx_stat;
208         u32     misc_stat;
209         u32     mem_allign2[8];
210         u32     rx_imax;
211         u32     tx_imax;
212
213 };
214
215 struct cpsw_ss_regs {
216         u32     id_ver;
217         u32     control;
218         u32     soft_reset;
219         u32     stat_port_en;
220         u32     ptype;
221         u32     soft_idle;
222         u32     thru_rate;
223         u32     gap_thresh;
224         u32     tx_start_wds;
225         u32     flow_control;
226         u32     vlan_ltype;
227         u32     ts_ltype;
228         u32     dlr_ltype;
229 };
230
231 /* CPSW_PORT_V1 */
232 #define CPSW1_MAX_BLKS      0x00 /* Maximum FIFO Blocks */
233 #define CPSW1_BLK_CNT       0x04 /* FIFO Block Usage Count (Read Only) */
234 #define CPSW1_TX_IN_CTL     0x08 /* Transmit FIFO Control */
235 #define CPSW1_PORT_VLAN     0x0c /* VLAN Register */
236 #define CPSW1_TX_PRI_MAP    0x10 /* Tx Header Priority to Switch Pri Mapping */
237 #define CPSW1_TS_CTL        0x14 /* Time Sync Control */
238 #define CPSW1_TS_SEQ_LTYPE  0x18 /* Time Sync Sequence ID Offset and Msg Type */
239 #define CPSW1_TS_VLAN       0x1c /* Time Sync VLAN1 and VLAN2 */
240
241 /* CPSW_PORT_V2 */
242 #define CPSW2_CONTROL       0x00 /* Control Register */
243 #define CPSW2_MAX_BLKS      0x08 /* Maximum FIFO Blocks */
244 #define CPSW2_BLK_CNT       0x0c /* FIFO Block Usage Count (Read Only) */
245 #define CPSW2_TX_IN_CTL     0x10 /* Transmit FIFO Control */
246 #define CPSW2_PORT_VLAN     0x14 /* VLAN Register */
247 #define CPSW2_TX_PRI_MAP    0x18 /* Tx Header Priority to Switch Pri Mapping */
248 #define CPSW2_TS_SEQ_MTYPE  0x1c /* Time Sync Sequence ID Offset and Msg Type */
249
250 /* CPSW_PORT_V1 and V2 */
251 #define SA_LO               0x20 /* CPGMAC_SL Source Address Low */
252 #define SA_HI               0x24 /* CPGMAC_SL Source Address High */
253 #define SEND_PERCENT        0x28 /* Transmit Queue Send Percentages */
254
255 /* CPSW_PORT_V2 only */
256 #define RX_DSCP_PRI_MAP0    0x30 /* Rx DSCP Priority to Rx Packet Mapping */
257 #define RX_DSCP_PRI_MAP1    0x34 /* Rx DSCP Priority to Rx Packet Mapping */
258 #define RX_DSCP_PRI_MAP2    0x38 /* Rx DSCP Priority to Rx Packet Mapping */
259 #define RX_DSCP_PRI_MAP3    0x3c /* Rx DSCP Priority to Rx Packet Mapping */
260 #define RX_DSCP_PRI_MAP4    0x40 /* Rx DSCP Priority to Rx Packet Mapping */
261 #define RX_DSCP_PRI_MAP5    0x44 /* Rx DSCP Priority to Rx Packet Mapping */
262 #define RX_DSCP_PRI_MAP6    0x48 /* Rx DSCP Priority to Rx Packet Mapping */
263 #define RX_DSCP_PRI_MAP7    0x4c /* Rx DSCP Priority to Rx Packet Mapping */
264
265 /* Bit definitions for the CPSW2_CONTROL register */
266 #define PASS_PRI_TAGGED     BIT(24) /* Pass Priority Tagged */
267 #define VLAN_LTYPE2_EN      BIT(21) /* VLAN LTYPE 2 enable */
268 #define VLAN_LTYPE1_EN      BIT(20) /* VLAN LTYPE 1 enable */
269 #define DSCP_PRI_EN         BIT(16) /* DSCP Priority Enable */
270 #define TS_107              BIT(15) /* Tyme Sync Dest IP Address 107 */
271 #define TS_320              BIT(14) /* Time Sync Dest Port 320 enable */
272 #define TS_319              BIT(13) /* Time Sync Dest Port 319 enable */
273 #define TS_132              BIT(12) /* Time Sync Dest IP Addr 132 enable */
274 #define TS_131              BIT(11) /* Time Sync Dest IP Addr 131 enable */
275 #define TS_130              BIT(10) /* Time Sync Dest IP Addr 130 enable */
276 #define TS_129              BIT(9)  /* Time Sync Dest IP Addr 129 enable */
277 #define TS_TTL_NONZERO      BIT(8)  /* Time Sync Time To Live Non-zero enable */
278 #define TS_ANNEX_F_EN       BIT(6)  /* Time Sync Annex F enable */
279 #define TS_ANNEX_D_EN       BIT(4)  /* Time Sync Annex D enable */
280 #define TS_LTYPE2_EN        BIT(3)  /* Time Sync LTYPE 2 enable */
281 #define TS_LTYPE1_EN        BIT(2)  /* Time Sync LTYPE 1 enable */
282 #define TS_TX_EN            BIT(1)  /* Time Sync Transmit Enable */
283 #define TS_RX_EN            BIT(0)  /* Time Sync Receive Enable */
284
285 #define CTRL_V2_TS_BITS \
286         (TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
287          TS_TTL_NONZERO  | TS_ANNEX_D_EN | TS_LTYPE1_EN)
288
289 #define CTRL_V2_ALL_TS_MASK (CTRL_V2_TS_BITS | TS_TX_EN | TS_RX_EN)
290 #define CTRL_V2_TX_TS_BITS  (CTRL_V2_TS_BITS | TS_TX_EN)
291 #define CTRL_V2_RX_TS_BITS  (CTRL_V2_TS_BITS | TS_RX_EN)
292
293
294 #define CTRL_V3_TS_BITS \
295         (TS_107 | TS_320 | TS_319 | TS_132 | TS_131 | TS_130 | TS_129 |\
296          TS_TTL_NONZERO | TS_ANNEX_F_EN | TS_ANNEX_D_EN |\
297          TS_LTYPE1_EN)
298
299 #define CTRL_V3_ALL_TS_MASK (CTRL_V3_TS_BITS | TS_TX_EN | TS_RX_EN)
300 #define CTRL_V3_TX_TS_BITS  (CTRL_V3_TS_BITS | TS_TX_EN)
301 #define CTRL_V3_RX_TS_BITS  (CTRL_V3_TS_BITS | TS_RX_EN)
302
303 /* Bit definitions for the CPSW2_TS_SEQ_MTYPE register */
304 #define TS_SEQ_ID_OFFSET_SHIFT   (16)    /* Time Sync Sequence ID Offset */
305 #define TS_SEQ_ID_OFFSET_MASK    (0x3f)
306 #define TS_MSG_TYPE_EN_SHIFT     (0)     /* Time Sync Message Type Enable */
307 #define TS_MSG_TYPE_EN_MASK      (0xffff)
308
309 /* The PTP event messages - Sync, Delay_Req, Pdelay_Req, and Pdelay_Resp. */
310 #define EVENT_MSG_BITS ((1<<0) | (1<<1) | (1<<2) | (1<<3))
311
312 /* Bit definitions for the CPSW1_TS_CTL register */
313 #define CPSW_V1_TS_RX_EN                BIT(0)
314 #define CPSW_V1_TS_TX_EN                BIT(4)
315 #define CPSW_V1_MSG_TYPE_OFS            16
316
317 /* Bit definitions for the CPSW1_TS_SEQ_LTYPE register */
318 #define CPSW_V1_SEQ_ID_OFS_SHIFT        16
319
320 #define CPSW_MAX_BLKS_TX                15
321 #define CPSW_MAX_BLKS_TX_SHIFT          4
322 #define CPSW_MAX_BLKS_RX                5
323
324 struct cpsw_host_regs {
325         u32     max_blks;
326         u32     blk_cnt;
327         u32     tx_in_ctl;
328         u32     port_vlan;
329         u32     tx_pri_map;
330         u32     cpdma_tx_pri_map;
331         u32     cpdma_rx_chan_map;
332 };
333
334 struct cpsw_sliver_regs {
335         u32     id_ver;
336         u32     mac_control;
337         u32     mac_status;
338         u32     soft_reset;
339         u32     rx_maxlen;
340         u32     __reserved_0;
341         u32     rx_pause;
342         u32     tx_pause;
343         u32     __reserved_1;
344         u32     rx_pri_map;
345 };
346
347 struct cpsw_hw_stats {
348         u32     rxgoodframes;
349         u32     rxbroadcastframes;
350         u32     rxmulticastframes;
351         u32     rxpauseframes;
352         u32     rxcrcerrors;
353         u32     rxaligncodeerrors;
354         u32     rxoversizedframes;
355         u32     rxjabberframes;
356         u32     rxundersizedframes;
357         u32     rxfragments;
358         u32     __pad_0[2];
359         u32     rxoctets;
360         u32     txgoodframes;
361         u32     txbroadcastframes;
362         u32     txmulticastframes;
363         u32     txpauseframes;
364         u32     txdeferredframes;
365         u32     txcollisionframes;
366         u32     txsinglecollframes;
367         u32     txmultcollframes;
368         u32     txexcessivecollisions;
369         u32     txlatecollisions;
370         u32     txunderrun;
371         u32     txcarriersenseerrors;
372         u32     txoctets;
373         u32     octetframes64;
374         u32     octetframes65t127;
375         u32     octetframes128t255;
376         u32     octetframes256t511;
377         u32     octetframes512t1023;
378         u32     octetframes1024tup;
379         u32     netoctets;
380         u32     rxsofoverruns;
381         u32     rxmofoverruns;
382         u32     rxdmaoverruns;
383 };
384
385 struct cpsw_slave_data {
386         struct device_node *phy_node;
387         char            phy_id[MII_BUS_ID_SIZE];
388         int             phy_if;
389         u8              mac_addr[ETH_ALEN];
390         u16             dual_emac_res_vlan;     /* Reserved VLAN for DualEMAC */
391         struct phy      *ifphy;
392 };
393
394 struct cpsw_platform_data {
395         struct cpsw_slave_data  *slave_data;
396         u32     ss_reg_ofs;     /* Subsystem control register offset */
397         u32     channels;       /* number of cpdma channels (symmetric) */
398         u32     slaves;         /* number of slave cpgmac ports */
399         u32     active_slave; /* time stamping, ethtool and SIOCGMIIPHY slave */
400         u32     ale_entries;    /* ale table size */
401         u32     bd_ram_size;  /*buffer descriptor ram size */
402         u32     mac_control;    /* Mac control register */
403         u16     default_vlan;   /* Def VLAN for ALE lookup in VLAN aware mode*/
404         bool    dual_emac;      /* Enable Dual EMAC mode */
405 };
406
407 struct cpsw_slave {
408         void __iomem                    *regs;
409         struct cpsw_sliver_regs __iomem *sliver;
410         int                             slave_num;
411         u32                             mac_control;
412         struct cpsw_slave_data          *data;
413         struct phy_device               *phy;
414         struct net_device               *ndev;
415         u32                             port_vlan;
416 };
417
418 static inline u32 slave_read(struct cpsw_slave *slave, u32 offset)
419 {
420         return readl_relaxed(slave->regs + offset);
421 }
422
423 static inline void slave_write(struct cpsw_slave *slave, u32 val, u32 offset)
424 {
425         writel_relaxed(val, slave->regs + offset);
426 }
427
428 struct cpsw_vector {
429         struct cpdma_chan *ch;
430         int budget;
431 };
432
433 struct cpsw_common {
434         struct device                   *dev;
435         struct cpsw_platform_data       data;
436         struct napi_struct              napi_rx;
437         struct napi_struct              napi_tx;
438         struct cpsw_ss_regs __iomem     *regs;
439         struct cpsw_wr_regs __iomem     *wr_regs;
440         u8 __iomem                      *hw_stats;
441         struct cpsw_host_regs __iomem   *host_port_regs;
442         u32                             version;
443         u32                             coal_intvl;
444         u32                             bus_freq_mhz;
445         int                             rx_packet_max;
446         struct cpsw_slave               *slaves;
447         struct cpdma_ctlr               *dma;
448         struct cpsw_vector              txv[CPSW_MAX_QUEUES];
449         struct cpsw_vector              rxv[CPSW_MAX_QUEUES];
450         struct cpsw_ale                 *ale;
451         bool                            quirk_irq;
452         bool                            rx_irq_disabled;
453         bool                            tx_irq_disabled;
454         u32 irqs_table[IRQ_NUM];
455         struct cpts                     *cpts;
456         int                             rx_ch_num, tx_ch_num;
457         int                             speed;
458         int                             usage_count;
459 };
460
461 struct cpsw_priv {
462         struct net_device               *ndev;
463         struct device                   *dev;
464         u32                             msg_enable;
465         u8                              mac_addr[ETH_ALEN];
466         bool                            rx_pause;
467         bool                            tx_pause;
468         bool                            mqprio_hw;
469         int                             fifo_bw[CPSW_TC_NUM];
470         int                             shp_cfg_speed;
471         u32 emac_port;
472         struct cpsw_common *cpsw;
473 };
474
475 struct cpsw_stats {
476         char stat_string[ETH_GSTRING_LEN];
477         int type;
478         int sizeof_stat;
479         int stat_offset;
480 };
481
482 enum {
483         CPSW_STATS,
484         CPDMA_RX_STATS,
485         CPDMA_TX_STATS,
486 };
487
488 #define CPSW_STAT(m)            CPSW_STATS,                             \
489                                 FIELD_SIZEOF(struct cpsw_hw_stats, m), \
490                                 offsetof(struct cpsw_hw_stats, m)
491 #define CPDMA_RX_STAT(m)        CPDMA_RX_STATS,                            \
492                                 FIELD_SIZEOF(struct cpdma_chan_stats, m), \
493                                 offsetof(struct cpdma_chan_stats, m)
494 #define CPDMA_TX_STAT(m)        CPDMA_TX_STATS,                            \
495                                 FIELD_SIZEOF(struct cpdma_chan_stats, m), \
496                                 offsetof(struct cpdma_chan_stats, m)
497
498 static const struct cpsw_stats cpsw_gstrings_stats[] = {
499         { "Good Rx Frames", CPSW_STAT(rxgoodframes) },
500         { "Broadcast Rx Frames", CPSW_STAT(rxbroadcastframes) },
501         { "Multicast Rx Frames", CPSW_STAT(rxmulticastframes) },
502         { "Pause Rx Frames", CPSW_STAT(rxpauseframes) },
503         { "Rx CRC Errors", CPSW_STAT(rxcrcerrors) },
504         { "Rx Align/Code Errors", CPSW_STAT(rxaligncodeerrors) },
505         { "Oversize Rx Frames", CPSW_STAT(rxoversizedframes) },
506         { "Rx Jabbers", CPSW_STAT(rxjabberframes) },
507         { "Undersize (Short) Rx Frames", CPSW_STAT(rxundersizedframes) },
508         { "Rx Fragments", CPSW_STAT(rxfragments) },
509         { "Rx Octets", CPSW_STAT(rxoctets) },
510         { "Good Tx Frames", CPSW_STAT(txgoodframes) },
511         { "Broadcast Tx Frames", CPSW_STAT(txbroadcastframes) },
512         { "Multicast Tx Frames", CPSW_STAT(txmulticastframes) },
513         { "Pause Tx Frames", CPSW_STAT(txpauseframes) },
514         { "Deferred Tx Frames", CPSW_STAT(txdeferredframes) },
515         { "Collisions", CPSW_STAT(txcollisionframes) },
516         { "Single Collision Tx Frames", CPSW_STAT(txsinglecollframes) },
517         { "Multiple Collision Tx Frames", CPSW_STAT(txmultcollframes) },
518         { "Excessive Collisions", CPSW_STAT(txexcessivecollisions) },
519         { "Late Collisions", CPSW_STAT(txlatecollisions) },
520         { "Tx Underrun", CPSW_STAT(txunderrun) },
521         { "Carrier Sense Errors", CPSW_STAT(txcarriersenseerrors) },
522         { "Tx Octets", CPSW_STAT(txoctets) },
523         { "Rx + Tx 64 Octet Frames", CPSW_STAT(octetframes64) },
524         { "Rx + Tx 65-127 Octet Frames", CPSW_STAT(octetframes65t127) },
525         { "Rx + Tx 128-255 Octet Frames", CPSW_STAT(octetframes128t255) },
526         { "Rx + Tx 256-511 Octet Frames", CPSW_STAT(octetframes256t511) },
527         { "Rx + Tx 512-1023 Octet Frames", CPSW_STAT(octetframes512t1023) },
528         { "Rx + Tx 1024-Up Octet Frames", CPSW_STAT(octetframes1024tup) },
529         { "Net Octets", CPSW_STAT(netoctets) },
530         { "Rx Start of Frame Overruns", CPSW_STAT(rxsofoverruns) },
531         { "Rx Middle of Frame Overruns", CPSW_STAT(rxmofoverruns) },
532         { "Rx DMA Overruns", CPSW_STAT(rxdmaoverruns) },
533 };
534
535 static const struct cpsw_stats cpsw_gstrings_ch_stats[] = {
536         { "head_enqueue", CPDMA_RX_STAT(head_enqueue) },
537         { "tail_enqueue", CPDMA_RX_STAT(tail_enqueue) },
538         { "pad_enqueue", CPDMA_RX_STAT(pad_enqueue) },
539         { "misqueued", CPDMA_RX_STAT(misqueued) },
540         { "desc_alloc_fail", CPDMA_RX_STAT(desc_alloc_fail) },
541         { "pad_alloc_fail", CPDMA_RX_STAT(pad_alloc_fail) },
542         { "runt_receive_buf", CPDMA_RX_STAT(runt_receive_buff) },
543         { "runt_transmit_buf", CPDMA_RX_STAT(runt_transmit_buff) },
544         { "empty_dequeue", CPDMA_RX_STAT(empty_dequeue) },
545         { "busy_dequeue", CPDMA_RX_STAT(busy_dequeue) },
546         { "good_dequeue", CPDMA_RX_STAT(good_dequeue) },
547         { "requeue", CPDMA_RX_STAT(requeue) },
548         { "teardown_dequeue", CPDMA_RX_STAT(teardown_dequeue) },
549 };
550
551 #define CPSW_STATS_COMMON_LEN   ARRAY_SIZE(cpsw_gstrings_stats)
552 #define CPSW_STATS_CH_LEN       ARRAY_SIZE(cpsw_gstrings_ch_stats)
553
554 #define ndev_to_cpsw(ndev) (((struct cpsw_priv *)netdev_priv(ndev))->cpsw)
555 #define napi_to_cpsw(napi)      container_of(napi, struct cpsw_common, napi)
556 #define for_each_slave(priv, func, arg...)                              \
557         do {                                                            \
558                 struct cpsw_slave *slave;                               \
559                 struct cpsw_common *cpsw = (priv)->cpsw;                \
560                 int n;                                                  \
561                 if (cpsw->data.dual_emac)                               \
562                         (func)((cpsw)->slaves + priv->emac_port, ##arg);\
563                 else                                                    \
564                         for (n = cpsw->data.slaves,                     \
565                                         slave = cpsw->slaves;           \
566                                         n; n--)                         \
567                                 (func)(slave++, ##arg);                 \
568         } while (0)
569
570 static inline int cpsw_get_slave_port(u32 slave_num)
571 {
572         return slave_num + 1;
573 }
574
575 static void cpsw_add_mcast(struct cpsw_priv *priv, const u8 *addr)
576 {
577         struct cpsw_common *cpsw = priv->cpsw;
578
579         if (cpsw->data.dual_emac) {
580                 struct cpsw_slave *slave = cpsw->slaves + priv->emac_port;
581
582                 cpsw_ale_add_mcast(cpsw->ale, addr, ALE_PORT_HOST,
583                                    ALE_VLAN, slave->port_vlan, 0);
584                 return;
585         }
586
587         cpsw_ale_add_mcast(cpsw->ale, addr, ALE_ALL_PORTS, 0, 0, 0);
588 }
589
590 static void cpsw_set_promiscious(struct net_device *ndev, bool enable)
591 {
592         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
593         struct cpsw_ale *ale = cpsw->ale;
594         int i;
595
596         if (cpsw->data.dual_emac) {
597                 bool flag = false;
598
599                 /* Enabling promiscuous mode for one interface will be
600                  * common for both the interface as the interface shares
601                  * the same hardware resource.
602                  */
603                 for (i = 0; i < cpsw->data.slaves; i++)
604                         if (cpsw->slaves[i].ndev->flags & IFF_PROMISC)
605                                 flag = true;
606
607                 if (!enable && flag) {
608                         enable = true;
609                         dev_err(&ndev->dev, "promiscuity not disabled as the other interface is still in promiscuity mode\n");
610                 }
611
612                 if (enable) {
613                         /* Enable Bypass */
614                         cpsw_ale_control_set(ale, 0, ALE_BYPASS, 1);
615
616                         dev_dbg(&ndev->dev, "promiscuity enabled\n");
617                 } else {
618                         /* Disable Bypass */
619                         cpsw_ale_control_set(ale, 0, ALE_BYPASS, 0);
620                         dev_dbg(&ndev->dev, "promiscuity disabled\n");
621                 }
622         } else {
623                 if (enable) {
624                         unsigned long timeout = jiffies + HZ;
625
626                         /* Disable Learn for all ports (host is port 0 and slaves are port 1 and up */
627                         for (i = 0; i <= cpsw->data.slaves; i++) {
628                                 cpsw_ale_control_set(ale, i,
629                                                      ALE_PORT_NOLEARN, 1);
630                                 cpsw_ale_control_set(ale, i,
631                                                      ALE_PORT_NO_SA_UPDATE, 1);
632                         }
633
634                         /* Clear All Untouched entries */
635                         cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
636                         do {
637                                 cpu_relax();
638                                 if (cpsw_ale_control_get(ale, 0, ALE_AGEOUT))
639                                         break;
640                         } while (time_after(timeout, jiffies));
641                         cpsw_ale_control_set(ale, 0, ALE_AGEOUT, 1);
642
643                         /* Clear all mcast from ALE */
644                         cpsw_ale_flush_multicast(ale, ALE_ALL_PORTS, -1);
645                         __dev_mc_unsync(ndev, NULL);
646
647                         /* Flood All Unicast Packets to Host port */
648                         cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 1);
649                         dev_dbg(&ndev->dev, "promiscuity enabled\n");
650                 } else {
651                         /* Don't Flood All Unicast Packets to Host port */
652                         cpsw_ale_control_set(ale, 0, ALE_P0_UNI_FLOOD, 0);
653
654                         /* Enable Learn for all ports (host is port 0 and slaves are port 1 and up */
655                         for (i = 0; i <= cpsw->data.slaves; i++) {
656                                 cpsw_ale_control_set(ale, i,
657                                                      ALE_PORT_NOLEARN, 0);
658                                 cpsw_ale_control_set(ale, i,
659                                                      ALE_PORT_NO_SA_UPDATE, 0);
660                         }
661                         dev_dbg(&ndev->dev, "promiscuity disabled\n");
662                 }
663         }
664 }
665
666 static int cpsw_add_mc_addr(struct net_device *ndev, const u8 *addr)
667 {
668         struct cpsw_priv *priv = netdev_priv(ndev);
669
670         cpsw_add_mcast(priv, addr);
671         return 0;
672 }
673
674 static int cpsw_del_mc_addr(struct net_device *ndev, const u8 *addr)
675 {
676         struct cpsw_priv *priv = netdev_priv(ndev);
677         struct cpsw_common *cpsw = priv->cpsw;
678         int vid, flags;
679
680         if (cpsw->data.dual_emac) {
681                 vid = cpsw->slaves[priv->emac_port].port_vlan;
682                 flags = ALE_VLAN;
683         } else {
684                 vid = 0;
685                 flags = 0;
686         }
687
688         cpsw_ale_del_mcast(cpsw->ale, addr, 0, flags, vid);
689         return 0;
690 }
691
692 static void cpsw_ndo_set_rx_mode(struct net_device *ndev)
693 {
694         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
695
696         if (ndev->flags & IFF_PROMISC) {
697                 /* Enable promiscuous mode */
698                 cpsw_set_promiscious(ndev, true);
699                 cpsw_ale_set_allmulti(cpsw->ale, IFF_ALLMULTI);
700                 return;
701         } else {
702                 /* Disable promiscuous mode */
703                 cpsw_set_promiscious(ndev, false);
704         }
705
706         /* Restore allmulti on vlans if necessary */
707         cpsw_ale_set_allmulti(cpsw->ale, ndev->flags & IFF_ALLMULTI);
708
709         __dev_mc_sync(ndev, cpsw_add_mc_addr, cpsw_del_mc_addr);
710 }
711
712 static void cpsw_intr_enable(struct cpsw_common *cpsw)
713 {
714         writel_relaxed(0xFF, &cpsw->wr_regs->tx_en);
715         writel_relaxed(0xFF, &cpsw->wr_regs->rx_en);
716
717         cpdma_ctlr_int_ctrl(cpsw->dma, true);
718         return;
719 }
720
721 static void cpsw_intr_disable(struct cpsw_common *cpsw)
722 {
723         writel_relaxed(0, &cpsw->wr_regs->tx_en);
724         writel_relaxed(0, &cpsw->wr_regs->rx_en);
725
726         cpdma_ctlr_int_ctrl(cpsw->dma, false);
727         return;
728 }
729
730 static void cpsw_tx_handler(void *token, int len, int status)
731 {
732         struct netdev_queue     *txq;
733         struct sk_buff          *skb = token;
734         struct net_device       *ndev = skb->dev;
735         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
736
737         /* Check whether the queue is stopped due to stalled tx dma, if the
738          * queue is stopped then start the queue as we have free desc for tx
739          */
740         txq = netdev_get_tx_queue(ndev, skb_get_queue_mapping(skb));
741         if (unlikely(netif_tx_queue_stopped(txq)))
742                 netif_tx_wake_queue(txq);
743
744         cpts_tx_timestamp(cpsw->cpts, skb);
745         ndev->stats.tx_packets++;
746         ndev->stats.tx_bytes += len;
747         dev_kfree_skb_any(skb);
748 }
749
750 static void cpsw_rx_vlan_encap(struct sk_buff *skb)
751 {
752         struct cpsw_priv *priv = netdev_priv(skb->dev);
753         struct cpsw_common *cpsw = priv->cpsw;
754         u32 rx_vlan_encap_hdr = *((u32 *)skb->data);
755         u16 vtag, vid, prio, pkt_type;
756
757         /* Remove VLAN header encapsulation word */
758         skb_pull(skb, CPSW_RX_VLAN_ENCAP_HDR_SIZE);
759
760         pkt_type = (rx_vlan_encap_hdr >>
761                     CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_SHIFT) &
762                     CPSW_RX_VLAN_ENCAP_HDR_PKT_TYPE_MSK;
763         /* Ignore unknown & Priority-tagged packets*/
764         if (pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_RESERV ||
765             pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_PRIO_TAG)
766                 return;
767
768         vid = (rx_vlan_encap_hdr >>
769                CPSW_RX_VLAN_ENCAP_HDR_VID_SHIFT) &
770                VLAN_VID_MASK;
771         /* Ignore vid 0 and pass packet as is */
772         if (!vid)
773                 return;
774         /* Ignore default vlans in dual mac mode */
775         if (cpsw->data.dual_emac &&
776             vid == cpsw->slaves[priv->emac_port].port_vlan)
777                 return;
778
779         prio = (rx_vlan_encap_hdr >>
780                 CPSW_RX_VLAN_ENCAP_HDR_PRIO_SHIFT) &
781                 CPSW_RX_VLAN_ENCAP_HDR_PRIO_MSK;
782
783         vtag = (prio << VLAN_PRIO_SHIFT) | vid;
784         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vtag);
785
786         /* strip vlan tag for VLAN-tagged packet */
787         if (pkt_type == CPSW_RX_VLAN_ENCAP_HDR_PKT_VLAN_TAG) {
788                 memmove(skb->data + VLAN_HLEN, skb->data, 2 * ETH_ALEN);
789                 skb_pull(skb, VLAN_HLEN);
790         }
791 }
792
793 static void cpsw_rx_handler(void *token, int len, int status)
794 {
795         struct cpdma_chan       *ch;
796         struct sk_buff          *skb = token;
797         struct sk_buff          *new_skb;
798         struct net_device       *ndev = skb->dev;
799         int                     ret = 0, port;
800         struct cpsw_common      *cpsw = ndev_to_cpsw(ndev);
801
802         if (cpsw->data.dual_emac) {
803                 port = CPDMA_RX_SOURCE_PORT(status);
804                 if (port) {
805                         ndev = cpsw->slaves[--port].ndev;
806                         skb->dev = ndev;
807                 }
808         }
809
810         if (unlikely(status < 0) || unlikely(!netif_running(ndev))) {
811                 /* In dual emac mode check for all interfaces */
812                 if (cpsw->data.dual_emac && cpsw->usage_count &&
813                     (status >= 0)) {
814                         /* The packet received is for the interface which
815                          * is already down and the other interface is up
816                          * and running, instead of freeing which results
817                          * in reducing of the number of rx descriptor in
818                          * DMA engine, requeue skb back to cpdma.
819                          */
820                         new_skb = skb;
821                         goto requeue;
822                 }
823
824                 /* the interface is going down, skbs are purged */
825                 dev_kfree_skb_any(skb);
826                 return;
827         }
828
829         new_skb = netdev_alloc_skb_ip_align(ndev, cpsw->rx_packet_max);
830         if (new_skb) {
831                 skb_copy_queue_mapping(new_skb, skb);
832                 skb_put(skb, len);
833                 if (status & CPDMA_RX_VLAN_ENCAP)
834                         cpsw_rx_vlan_encap(skb);
835                 cpts_rx_timestamp(cpsw->cpts, skb);
836                 skb->protocol = eth_type_trans(skb, ndev);
837                 netif_receive_skb(skb);
838                 ndev->stats.rx_bytes += len;
839                 ndev->stats.rx_packets++;
840                 kmemleak_not_leak(new_skb);
841         } else {
842                 ndev->stats.rx_dropped++;
843                 new_skb = skb;
844         }
845
846 requeue:
847         if (netif_dormant(ndev)) {
848                 dev_kfree_skb_any(new_skb);
849                 return;
850         }
851
852         ch = cpsw->rxv[skb_get_queue_mapping(new_skb)].ch;
853         ret = cpdma_chan_submit(ch, new_skb, new_skb->data,
854                                 skb_tailroom(new_skb), 0);
855         if (WARN_ON(ret < 0))
856                 dev_kfree_skb_any(new_skb);
857 }
858
859 static void cpsw_split_res(struct net_device *ndev)
860 {
861         struct cpsw_priv *priv = netdev_priv(ndev);
862         u32 consumed_rate = 0, bigest_rate = 0;
863         struct cpsw_common *cpsw = priv->cpsw;
864         struct cpsw_vector *txv = cpsw->txv;
865         int i, ch_weight, rlim_ch_num = 0;
866         int budget, bigest_rate_ch = 0;
867         u32 ch_rate, max_rate;
868         int ch_budget = 0;
869
870         for (i = 0; i < cpsw->tx_ch_num; i++) {
871                 ch_rate = cpdma_chan_get_rate(txv[i].ch);
872                 if (!ch_rate)
873                         continue;
874
875                 rlim_ch_num++;
876                 consumed_rate += ch_rate;
877         }
878
879         if (cpsw->tx_ch_num == rlim_ch_num) {
880                 max_rate = consumed_rate;
881         } else if (!rlim_ch_num) {
882                 ch_budget = CPSW_POLL_WEIGHT / cpsw->tx_ch_num;
883                 bigest_rate = 0;
884                 max_rate = consumed_rate;
885         } else {
886                 max_rate = cpsw->speed * 1000;
887
888                 /* if max_rate is less then expected due to reduced link speed,
889                  * split proportionally according next potential max speed
890                  */
891                 if (max_rate < consumed_rate)
892                         max_rate *= 10;
893
894                 if (max_rate < consumed_rate)
895                         max_rate *= 10;
896
897                 ch_budget = (consumed_rate * CPSW_POLL_WEIGHT) / max_rate;
898                 ch_budget = (CPSW_POLL_WEIGHT - ch_budget) /
899                             (cpsw->tx_ch_num - rlim_ch_num);
900                 bigest_rate = (max_rate - consumed_rate) /
901                               (cpsw->tx_ch_num - rlim_ch_num);
902         }
903
904         /* split tx weight/budget */
905         budget = CPSW_POLL_WEIGHT;
906         for (i = 0; i < cpsw->tx_ch_num; i++) {
907                 ch_rate = cpdma_chan_get_rate(txv[i].ch);
908                 if (ch_rate) {
909                         txv[i].budget = (ch_rate * CPSW_POLL_WEIGHT) / max_rate;
910                         if (!txv[i].budget)
911                                 txv[i].budget++;
912                         if (ch_rate > bigest_rate) {
913                                 bigest_rate_ch = i;
914                                 bigest_rate = ch_rate;
915                         }
916
917                         ch_weight = (ch_rate * 100) / max_rate;
918                         if (!ch_weight)
919                                 ch_weight++;
920                         cpdma_chan_set_weight(cpsw->txv[i].ch, ch_weight);
921                 } else {
922                         txv[i].budget = ch_budget;
923                         if (!bigest_rate_ch)
924                                 bigest_rate_ch = i;
925                         cpdma_chan_set_weight(cpsw->txv[i].ch, 0);
926                 }
927
928                 budget -= txv[i].budget;
929         }
930
931         if (budget)
932                 txv[bigest_rate_ch].budget += budget;
933
934         /* split rx budget */
935         budget = CPSW_POLL_WEIGHT;
936         ch_budget = budget / cpsw->rx_ch_num;
937         for (i = 0; i < cpsw->rx_ch_num; i++) {
938                 cpsw->rxv[i].budget = ch_budget;
939                 budget -= ch_budget;
940         }
941
942         if (budget)
943                 cpsw->rxv[0].budget += budget;
944 }
945
946 static irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id)
947 {
948         struct cpsw_common *cpsw = dev_id;
949
950         writel(0, &cpsw->wr_regs->tx_en);
951         cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_TX);
952
953         if (cpsw->quirk_irq) {
954                 disable_irq_nosync(cpsw->irqs_table[1]);
955                 cpsw->tx_irq_disabled = true;
956         }
957
958         napi_schedule(&cpsw->napi_tx);
959         return IRQ_HANDLED;
960 }
961
962 static irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id)
963 {
964         struct cpsw_common *cpsw = dev_id;
965
966         cpdma_ctlr_eoi(cpsw->dma, CPDMA_EOI_RX);
967         writel(0, &cpsw->wr_regs->rx_en);
968
969         if (cpsw->quirk_irq) {
970                 disable_irq_nosync(cpsw->irqs_table[0]);
971                 cpsw->rx_irq_disabled = true;
972         }
973
974         napi_schedule(&cpsw->napi_rx);
975         return IRQ_HANDLED;
976 }
977
978 static int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget)
979 {
980         u32                     ch_map;
981         int                     num_tx, cur_budget, ch;
982         struct cpsw_common      *cpsw = napi_to_cpsw(napi_tx);
983         struct cpsw_vector      *txv;
984
985         /* process every unprocessed channel */
986         ch_map = cpdma_ctrl_txchs_state(cpsw->dma);
987         for (ch = 0, num_tx = 0; ch_map & 0xff; ch_map <<= 1, ch++) {
988                 if (!(ch_map & 0x80))
989                         continue;
990
991                 txv = &cpsw->txv[ch];
992                 if (unlikely(txv->budget > budget - num_tx))
993                         cur_budget = budget - num_tx;
994                 else
995                         cur_budget = txv->budget;
996
997                 num_tx += cpdma_chan_process(txv->ch, cur_budget);
998                 if (num_tx >= budget)
999                         break;
1000         }
1001
1002         if (num_tx < budget) {
1003                 napi_complete(napi_tx);
1004                 writel(0xff, &cpsw->wr_regs->tx_en);
1005         }
1006
1007         return num_tx;
1008 }
1009
1010 static int cpsw_tx_poll(struct napi_struct *napi_tx, int budget)
1011 {
1012         struct cpsw_common *cpsw = napi_to_cpsw(napi_tx);
1013         int num_tx;
1014
1015         num_tx = cpdma_chan_process(cpsw->txv[0].ch, budget);
1016         if (num_tx < budget) {
1017                 napi_complete(napi_tx);
1018                 writel(0xff, &cpsw->wr_regs->tx_en);
1019                 if (cpsw->tx_irq_disabled) {
1020                         cpsw->tx_irq_disabled = false;
1021                         enable_irq(cpsw->irqs_table[1]);
1022                 }
1023         }
1024
1025         return num_tx;
1026 }
1027
1028 static int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget)
1029 {
1030         u32                     ch_map;
1031         int                     num_rx, cur_budget, ch;
1032         struct cpsw_common      *cpsw = napi_to_cpsw(napi_rx);
1033         struct cpsw_vector      *rxv;
1034
1035         /* process every unprocessed channel */
1036         ch_map = cpdma_ctrl_rxchs_state(cpsw->dma);
1037         for (ch = 0, num_rx = 0; ch_map; ch_map >>= 1, ch++) {
1038                 if (!(ch_map & 0x01))
1039                         continue;
1040
1041                 rxv = &cpsw->rxv[ch];
1042                 if (unlikely(rxv->budget > budget - num_rx))
1043                         cur_budget = budget - num_rx;
1044                 else
1045                         cur_budget = rxv->budget;
1046
1047                 num_rx += cpdma_chan_process(rxv->ch, cur_budget);
1048                 if (num_rx >= budget)
1049                         break;
1050         }
1051
1052         if (num_rx < budget) {
1053                 napi_complete_done(napi_rx, num_rx);
1054                 writel(0xff, &cpsw->wr_regs->rx_en);
1055         }
1056
1057         return num_rx;
1058 }
1059
1060 static int cpsw_rx_poll(struct napi_struct *napi_rx, int budget)
1061 {
1062         struct cpsw_common *cpsw = napi_to_cpsw(napi_rx);
1063         int num_rx;
1064
1065         num_rx = cpdma_chan_process(cpsw->rxv[0].ch, budget);
1066         if (num_rx < budget) {
1067                 napi_complete_done(napi_rx, num_rx);
1068                 writel(0xff, &cpsw->wr_regs->rx_en);
1069                 if (cpsw->rx_irq_disabled) {
1070                         cpsw->rx_irq_disabled = false;
1071                         enable_irq(cpsw->irqs_table[0]);
1072                 }
1073         }
1074
1075         return num_rx;
1076 }
1077
1078 static inline void soft_reset(const char *module, void __iomem *reg)
1079 {
1080         unsigned long timeout = jiffies + HZ;
1081
1082         writel_relaxed(1, reg);
1083         do {
1084                 cpu_relax();
1085         } while ((readl_relaxed(reg) & 1) && time_after(timeout, jiffies));
1086
1087         WARN(readl_relaxed(reg) & 1, "failed to soft-reset %s\n", module);
1088 }
1089
1090 static void cpsw_set_slave_mac(struct cpsw_slave *slave,
1091                                struct cpsw_priv *priv)
1092 {
1093         slave_write(slave, mac_hi(priv->mac_addr), SA_HI);
1094         slave_write(slave, mac_lo(priv->mac_addr), SA_LO);
1095 }
1096
1097 static bool cpsw_shp_is_off(struct cpsw_priv *priv)
1098 {
1099         struct cpsw_common *cpsw = priv->cpsw;
1100         struct cpsw_slave *slave;
1101         u32 shift, mask, val;
1102
1103         val = readl_relaxed(&cpsw->regs->ptype);
1104
1105         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1106         shift = CPSW_FIFO_SHAPE_EN_SHIFT + 3 * slave->slave_num;
1107         mask = 7 << shift;
1108         val = val & mask;
1109
1110         return !val;
1111 }
1112
1113 static void cpsw_fifo_shp_on(struct cpsw_priv *priv, int fifo, int on)
1114 {
1115         struct cpsw_common *cpsw = priv->cpsw;
1116         struct cpsw_slave *slave;
1117         u32 shift, mask, val;
1118
1119         val = readl_relaxed(&cpsw->regs->ptype);
1120
1121         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1122         shift = CPSW_FIFO_SHAPE_EN_SHIFT + 3 * slave->slave_num;
1123         mask = (1 << --fifo) << shift;
1124         val = on ? val | mask : val & ~mask;
1125
1126         writel_relaxed(val, &cpsw->regs->ptype);
1127 }
1128
1129 static void _cpsw_adjust_link(struct cpsw_slave *slave,
1130                               struct cpsw_priv *priv, bool *link)
1131 {
1132         struct phy_device       *phy = slave->phy;
1133         u32                     mac_control = 0;
1134         u32                     slave_port;
1135         struct cpsw_common *cpsw = priv->cpsw;
1136
1137         if (!phy)
1138                 return;
1139
1140         slave_port = cpsw_get_slave_port(slave->slave_num);
1141
1142         if (phy->link) {
1143                 mac_control = cpsw->data.mac_control;
1144
1145                 /* enable forwarding */
1146                 cpsw_ale_control_set(cpsw->ale, slave_port,
1147                                      ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1148
1149                 if (phy->speed == 1000)
1150                         mac_control |= BIT(7);  /* GIGABITEN    */
1151                 if (phy->duplex)
1152                         mac_control |= BIT(0);  /* FULLDUPLEXEN */
1153
1154                 /* set speed_in input in case RMII mode is used in 100Mbps */
1155                 if (phy->speed == 100)
1156                         mac_control |= BIT(15);
1157                 /* in band mode only works in 10Mbps RGMII mode */
1158                 else if ((phy->speed == 10) && phy_interface_is_rgmii(phy))
1159                         mac_control |= BIT(18); /* In Band mode */
1160
1161                 if (priv->rx_pause)
1162                         mac_control |= BIT(3);
1163
1164                 if (priv->tx_pause)
1165                         mac_control |= BIT(4);
1166
1167                 *link = true;
1168
1169                 if (priv->shp_cfg_speed &&
1170                     priv->shp_cfg_speed != slave->phy->speed &&
1171                     !cpsw_shp_is_off(priv))
1172                         dev_warn(priv->dev,
1173                                  "Speed was changed, CBS shaper speeds are changed!");
1174         } else {
1175                 mac_control = 0;
1176                 /* disable forwarding */
1177                 cpsw_ale_control_set(cpsw->ale, slave_port,
1178                                      ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1179         }
1180
1181         if (mac_control != slave->mac_control) {
1182                 phy_print_status(phy);
1183                 writel_relaxed(mac_control, &slave->sliver->mac_control);
1184         }
1185
1186         slave->mac_control = mac_control;
1187 }
1188
1189 static int cpsw_get_common_speed(struct cpsw_common *cpsw)
1190 {
1191         int i, speed;
1192
1193         for (i = 0, speed = 0; i < cpsw->data.slaves; i++)
1194                 if (cpsw->slaves[i].phy && cpsw->slaves[i].phy->link)
1195                         speed += cpsw->slaves[i].phy->speed;
1196
1197         return speed;
1198 }
1199
1200 static int cpsw_need_resplit(struct cpsw_common *cpsw)
1201 {
1202         int i, rlim_ch_num;
1203         int speed, ch_rate;
1204
1205         /* re-split resources only in case speed was changed */
1206         speed = cpsw_get_common_speed(cpsw);
1207         if (speed == cpsw->speed || !speed)
1208                 return 0;
1209
1210         cpsw->speed = speed;
1211
1212         for (i = 0, rlim_ch_num = 0; i < cpsw->tx_ch_num; i++) {
1213                 ch_rate = cpdma_chan_get_rate(cpsw->txv[i].ch);
1214                 if (!ch_rate)
1215                         break;
1216
1217                 rlim_ch_num++;
1218         }
1219
1220         /* cases not dependent on speed */
1221         if (!rlim_ch_num || rlim_ch_num == cpsw->tx_ch_num)
1222                 return 0;
1223
1224         return 1;
1225 }
1226
1227 static void cpsw_adjust_link(struct net_device *ndev)
1228 {
1229         struct cpsw_priv        *priv = netdev_priv(ndev);
1230         struct cpsw_common      *cpsw = priv->cpsw;
1231         bool                    link = false;
1232
1233         for_each_slave(priv, _cpsw_adjust_link, priv, &link);
1234
1235         if (link) {
1236                 if (cpsw_need_resplit(cpsw))
1237                         cpsw_split_res(ndev);
1238
1239                 netif_carrier_on(ndev);
1240                 if (netif_running(ndev))
1241                         netif_tx_wake_all_queues(ndev);
1242         } else {
1243                 netif_carrier_off(ndev);
1244                 netif_tx_stop_all_queues(ndev);
1245         }
1246 }
1247
1248 static int cpsw_get_coalesce(struct net_device *ndev,
1249                                 struct ethtool_coalesce *coal)
1250 {
1251         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1252
1253         coal->rx_coalesce_usecs = cpsw->coal_intvl;
1254         return 0;
1255 }
1256
1257 static int cpsw_set_coalesce(struct net_device *ndev,
1258                                 struct ethtool_coalesce *coal)
1259 {
1260         struct cpsw_priv *priv = netdev_priv(ndev);
1261         u32 int_ctrl;
1262         u32 num_interrupts = 0;
1263         u32 prescale = 0;
1264         u32 addnl_dvdr = 1;
1265         u32 coal_intvl = 0;
1266         struct cpsw_common *cpsw = priv->cpsw;
1267
1268         coal_intvl = coal->rx_coalesce_usecs;
1269
1270         int_ctrl =  readl(&cpsw->wr_regs->int_control);
1271         prescale = cpsw->bus_freq_mhz * 4;
1272
1273         if (!coal->rx_coalesce_usecs) {
1274                 int_ctrl &= ~(CPSW_INTPRESCALE_MASK | CPSW_INTPACEEN);
1275                 goto update_return;
1276         }
1277
1278         if (coal_intvl < CPSW_CMINTMIN_INTVL)
1279                 coal_intvl = CPSW_CMINTMIN_INTVL;
1280
1281         if (coal_intvl > CPSW_CMINTMAX_INTVL) {
1282                 /* Interrupt pacer works with 4us Pulse, we can
1283                  * throttle further by dilating the 4us pulse.
1284                  */
1285                 addnl_dvdr = CPSW_INTPRESCALE_MASK / prescale;
1286
1287                 if (addnl_dvdr > 1) {
1288                         prescale *= addnl_dvdr;
1289                         if (coal_intvl > (CPSW_CMINTMAX_INTVL * addnl_dvdr))
1290                                 coal_intvl = (CPSW_CMINTMAX_INTVL
1291                                                 * addnl_dvdr);
1292                 } else {
1293                         addnl_dvdr = 1;
1294                         coal_intvl = CPSW_CMINTMAX_INTVL;
1295                 }
1296         }
1297
1298         num_interrupts = (1000 * addnl_dvdr) / coal_intvl;
1299         writel(num_interrupts, &cpsw->wr_regs->rx_imax);
1300         writel(num_interrupts, &cpsw->wr_regs->tx_imax);
1301
1302         int_ctrl |= CPSW_INTPACEEN;
1303         int_ctrl &= (~CPSW_INTPRESCALE_MASK);
1304         int_ctrl |= (prescale & CPSW_INTPRESCALE_MASK);
1305
1306 update_return:
1307         writel(int_ctrl, &cpsw->wr_regs->int_control);
1308
1309         cpsw_notice(priv, timer, "Set coalesce to %d usecs.\n", coal_intvl);
1310         cpsw->coal_intvl = coal_intvl;
1311
1312         return 0;
1313 }
1314
1315 static int cpsw_get_sset_count(struct net_device *ndev, int sset)
1316 {
1317         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1318
1319         switch (sset) {
1320         case ETH_SS_STATS:
1321                 return (CPSW_STATS_COMMON_LEN +
1322                        (cpsw->rx_ch_num + cpsw->tx_ch_num) *
1323                        CPSW_STATS_CH_LEN);
1324         default:
1325                 return -EOPNOTSUPP;
1326         }
1327 }
1328
1329 static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir)
1330 {
1331         int ch_stats_len;
1332         int line;
1333         int i;
1334
1335         ch_stats_len = CPSW_STATS_CH_LEN * ch_num;
1336         for (i = 0; i < ch_stats_len; i++) {
1337                 line = i % CPSW_STATS_CH_LEN;
1338                 snprintf(*p, ETH_GSTRING_LEN,
1339                          "%s DMA chan %ld: %s", rx_dir ? "Rx" : "Tx",
1340                          (long)(i / CPSW_STATS_CH_LEN),
1341                          cpsw_gstrings_ch_stats[line].stat_string);
1342                 *p += ETH_GSTRING_LEN;
1343         }
1344 }
1345
1346 static void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
1347 {
1348         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1349         u8 *p = data;
1350         int i;
1351
1352         switch (stringset) {
1353         case ETH_SS_STATS:
1354                 for (i = 0; i < CPSW_STATS_COMMON_LEN; i++) {
1355                         memcpy(p, cpsw_gstrings_stats[i].stat_string,
1356                                ETH_GSTRING_LEN);
1357                         p += ETH_GSTRING_LEN;
1358                 }
1359
1360                 cpsw_add_ch_strings(&p, cpsw->rx_ch_num, 1);
1361                 cpsw_add_ch_strings(&p, cpsw->tx_ch_num, 0);
1362                 break;
1363         }
1364 }
1365
1366 static void cpsw_get_ethtool_stats(struct net_device *ndev,
1367                                     struct ethtool_stats *stats, u64 *data)
1368 {
1369         u8 *p;
1370         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
1371         struct cpdma_chan_stats ch_stats;
1372         int i, l, ch;
1373
1374         /* Collect Davinci CPDMA stats for Rx and Tx Channel */
1375         for (l = 0; l < CPSW_STATS_COMMON_LEN; l++)
1376                 data[l] = readl(cpsw->hw_stats +
1377                                 cpsw_gstrings_stats[l].stat_offset);
1378
1379         for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1380                 cpdma_chan_get_stats(cpsw->rxv[ch].ch, &ch_stats);
1381                 for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1382                         p = (u8 *)&ch_stats +
1383                                 cpsw_gstrings_ch_stats[i].stat_offset;
1384                         data[l] = *(u32 *)p;
1385                 }
1386         }
1387
1388         for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
1389                 cpdma_chan_get_stats(cpsw->txv[ch].ch, &ch_stats);
1390                 for (i = 0; i < CPSW_STATS_CH_LEN; i++, l++) {
1391                         p = (u8 *)&ch_stats +
1392                                 cpsw_gstrings_ch_stats[i].stat_offset;
1393                         data[l] = *(u32 *)p;
1394                 }
1395         }
1396 }
1397
1398 static inline int cpsw_tx_packet_submit(struct cpsw_priv *priv,
1399                                         struct sk_buff *skb,
1400                                         struct cpdma_chan *txch)
1401 {
1402         struct cpsw_common *cpsw = priv->cpsw;
1403
1404         skb_tx_timestamp(skb);
1405         return cpdma_chan_submit(txch, skb, skb->data, skb->len,
1406                                  priv->emac_port + cpsw->data.dual_emac);
1407 }
1408
1409 static inline void cpsw_add_dual_emac_def_ale_entries(
1410                 struct cpsw_priv *priv, struct cpsw_slave *slave,
1411                 u32 slave_port)
1412 {
1413         struct cpsw_common *cpsw = priv->cpsw;
1414         u32 port_mask = 1 << slave_port | ALE_PORT_HOST;
1415
1416         if (cpsw->version == CPSW_VERSION_1)
1417                 slave_write(slave, slave->port_vlan, CPSW1_PORT_VLAN);
1418         else
1419                 slave_write(slave, slave->port_vlan, CPSW2_PORT_VLAN);
1420         cpsw_ale_add_vlan(cpsw->ale, slave->port_vlan, port_mask,
1421                           port_mask, port_mask, 0);
1422         cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1423                            ALE_PORT_HOST, ALE_VLAN, slave->port_vlan, 0);
1424         cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
1425                            HOST_PORT_NUM, ALE_VLAN |
1426                            ALE_SECURE, slave->port_vlan);
1427         cpsw_ale_control_set(cpsw->ale, slave_port,
1428                              ALE_PORT_DROP_UNKNOWN_VLAN, 1);
1429 }
1430
1431 static void soft_reset_slave(struct cpsw_slave *slave)
1432 {
1433         char name[32];
1434
1435         snprintf(name, sizeof(name), "slave-%d", slave->slave_num);
1436         soft_reset(name, &slave->sliver->soft_reset);
1437 }
1438
1439 static void cpsw_slave_open(struct cpsw_slave *slave, struct cpsw_priv *priv)
1440 {
1441         u32 slave_port;
1442         struct phy_device *phy;
1443         struct cpsw_common *cpsw = priv->cpsw;
1444
1445         soft_reset_slave(slave);
1446
1447         /* setup priority mapping */
1448         writel_relaxed(RX_PRIORITY_MAPPING, &slave->sliver->rx_pri_map);
1449
1450         switch (cpsw->version) {
1451         case CPSW_VERSION_1:
1452                 slave_write(slave, TX_PRIORITY_MAPPING, CPSW1_TX_PRI_MAP);
1453                 /* Increase RX FIFO size to 5 for supporting fullduplex
1454                  * flow control mode
1455                  */
1456                 slave_write(slave,
1457                             (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1458                             CPSW_MAX_BLKS_RX, CPSW1_MAX_BLKS);
1459                 break;
1460         case CPSW_VERSION_2:
1461         case CPSW_VERSION_3:
1462         case CPSW_VERSION_4:
1463                 slave_write(slave, TX_PRIORITY_MAPPING, CPSW2_TX_PRI_MAP);
1464                 /* Increase RX FIFO size to 5 for supporting fullduplex
1465                  * flow control mode
1466                  */
1467                 slave_write(slave,
1468                             (CPSW_MAX_BLKS_TX << CPSW_MAX_BLKS_TX_SHIFT) |
1469                             CPSW_MAX_BLKS_RX, CPSW2_MAX_BLKS);
1470                 break;
1471         }
1472
1473         /* setup max packet size, and mac address */
1474         writel_relaxed(cpsw->rx_packet_max, &slave->sliver->rx_maxlen);
1475         cpsw_set_slave_mac(slave, priv);
1476
1477         slave->mac_control = 0; /* no link yet */
1478
1479         slave_port = cpsw_get_slave_port(slave->slave_num);
1480
1481         if (cpsw->data.dual_emac)
1482                 cpsw_add_dual_emac_def_ale_entries(priv, slave, slave_port);
1483         else
1484                 cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1485                                    1 << slave_port, 0, 0, ALE_MCAST_FWD_2);
1486
1487         if (slave->data->phy_node) {
1488                 phy = of_phy_connect(priv->ndev, slave->data->phy_node,
1489                                  &cpsw_adjust_link, 0, slave->data->phy_if);
1490                 if (!phy) {
1491                         dev_err(priv->dev, "phy \"%pOF\" not found on slave %d\n",
1492                                 slave->data->phy_node,
1493                                 slave->slave_num);
1494                         return;
1495                 }
1496         } else {
1497                 phy = phy_connect(priv->ndev, slave->data->phy_id,
1498                                  &cpsw_adjust_link, slave->data->phy_if);
1499                 if (IS_ERR(phy)) {
1500                         dev_err(priv->dev,
1501                                 "phy \"%s\" not found on slave %d, err %ld\n",
1502                                 slave->data->phy_id, slave->slave_num,
1503                                 PTR_ERR(phy));
1504                         return;
1505                 }
1506         }
1507
1508         slave->phy = phy;
1509
1510         phy_attached_info(slave->phy);
1511
1512         phy_start(slave->phy);
1513
1514         /* Configure GMII_SEL register */
1515         if (!IS_ERR(slave->data->ifphy))
1516                 phy_set_mode_ext(slave->data->ifphy, PHY_MODE_ETHERNET,
1517                                  slave->data->phy_if);
1518         else
1519                 cpsw_phy_sel(cpsw->dev, slave->phy->interface,
1520                              slave->slave_num);
1521 }
1522
1523 static inline void cpsw_add_default_vlan(struct cpsw_priv *priv)
1524 {
1525         struct cpsw_common *cpsw = priv->cpsw;
1526         const int vlan = cpsw->data.default_vlan;
1527         u32 reg;
1528         int i;
1529         int unreg_mcast_mask;
1530
1531         reg = (cpsw->version == CPSW_VERSION_1) ? CPSW1_PORT_VLAN :
1532                CPSW2_PORT_VLAN;
1533
1534         writel(vlan, &cpsw->host_port_regs->port_vlan);
1535
1536         for (i = 0; i < cpsw->data.slaves; i++)
1537                 slave_write(cpsw->slaves + i, vlan, reg);
1538
1539         if (priv->ndev->flags & IFF_ALLMULTI)
1540                 unreg_mcast_mask = ALE_ALL_PORTS;
1541         else
1542                 unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
1543
1544         cpsw_ale_add_vlan(cpsw->ale, vlan, ALE_ALL_PORTS,
1545                           ALE_ALL_PORTS, ALE_ALL_PORTS,
1546                           unreg_mcast_mask);
1547 }
1548
1549 static void cpsw_init_host_port(struct cpsw_priv *priv)
1550 {
1551         u32 fifo_mode;
1552         u32 control_reg;
1553         struct cpsw_common *cpsw = priv->cpsw;
1554
1555         /* soft reset the controller and initialize ale */
1556         soft_reset("cpsw", &cpsw->regs->soft_reset);
1557         cpsw_ale_start(cpsw->ale);
1558
1559         /* switch to vlan unaware mode */
1560         cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_VLAN_AWARE,
1561                              CPSW_ALE_VLAN_AWARE);
1562         control_reg = readl(&cpsw->regs->control);
1563         control_reg |= CPSW_VLAN_AWARE | CPSW_RX_VLAN_ENCAP;
1564         writel(control_reg, &cpsw->regs->control);
1565         fifo_mode = (cpsw->data.dual_emac) ? CPSW_FIFO_DUAL_MAC_MODE :
1566                      CPSW_FIFO_NORMAL_MODE;
1567         writel(fifo_mode, &cpsw->host_port_regs->tx_in_ctl);
1568
1569         /* setup host port priority mapping */
1570         writel_relaxed(CPDMA_TX_PRIORITY_MAP,
1571                        &cpsw->host_port_regs->cpdma_tx_pri_map);
1572         writel_relaxed(0, &cpsw->host_port_regs->cpdma_rx_chan_map);
1573
1574         cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM,
1575                              ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
1576
1577         if (!cpsw->data.dual_emac) {
1578                 cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
1579                                    0, 0);
1580                 cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
1581                                    ALE_PORT_HOST, 0, 0, ALE_MCAST_FWD_2);
1582         }
1583 }
1584
1585 static int cpsw_fill_rx_channels(struct cpsw_priv *priv)
1586 {
1587         struct cpsw_common *cpsw = priv->cpsw;
1588         struct sk_buff *skb;
1589         int ch_buf_num;
1590         int ch, i, ret;
1591
1592         for (ch = 0; ch < cpsw->rx_ch_num; ch++) {
1593                 ch_buf_num = cpdma_chan_get_rx_buf_num(cpsw->rxv[ch].ch);
1594                 for (i = 0; i < ch_buf_num; i++) {
1595                         skb = __netdev_alloc_skb_ip_align(priv->ndev,
1596                                                           cpsw->rx_packet_max,
1597                                                           GFP_KERNEL);
1598                         if (!skb) {
1599                                 cpsw_err(priv, ifup, "cannot allocate skb\n");
1600                                 return -ENOMEM;
1601                         }
1602
1603                         skb_set_queue_mapping(skb, ch);
1604                         ret = cpdma_chan_submit(cpsw->rxv[ch].ch, skb,
1605                                                 skb->data, skb_tailroom(skb),
1606                                                 0);
1607                         if (ret < 0) {
1608                                 cpsw_err(priv, ifup,
1609                                          "cannot submit skb to channel %d rx, error %d\n",
1610                                          ch, ret);
1611                                 kfree_skb(skb);
1612                                 return ret;
1613                         }
1614                         kmemleak_not_leak(skb);
1615                 }
1616
1617                 cpsw_info(priv, ifup, "ch %d rx, submitted %d descriptors\n",
1618                           ch, ch_buf_num);
1619         }
1620
1621         return 0;
1622 }
1623
1624 static void cpsw_slave_stop(struct cpsw_slave *slave, struct cpsw_common *cpsw)
1625 {
1626         u32 slave_port;
1627
1628         slave_port = cpsw_get_slave_port(slave->slave_num);
1629
1630         if (!slave->phy)
1631                 return;
1632         phy_stop(slave->phy);
1633         phy_disconnect(slave->phy);
1634         slave->phy = NULL;
1635         cpsw_ale_control_set(cpsw->ale, slave_port,
1636                              ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1637         soft_reset_slave(slave);
1638 }
1639
1640 static int cpsw_tc_to_fifo(int tc, int num_tc)
1641 {
1642         if (tc == num_tc - 1)
1643                 return 0;
1644
1645         return CPSW_FIFO_SHAPERS_NUM - tc;
1646 }
1647
1648 static int cpsw_set_fifo_bw(struct cpsw_priv *priv, int fifo, int bw)
1649 {
1650         struct cpsw_common *cpsw = priv->cpsw;
1651         u32 val = 0, send_pct, shift;
1652         struct cpsw_slave *slave;
1653         int pct = 0, i;
1654
1655         if (bw > priv->shp_cfg_speed * 1000)
1656                 goto err;
1657
1658         /* shaping has to stay enabled for highest fifos linearly
1659          * and fifo bw no more then interface can allow
1660          */
1661         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1662         send_pct = slave_read(slave, SEND_PERCENT);
1663         for (i = CPSW_FIFO_SHAPERS_NUM; i > 0; i--) {
1664                 if (!bw) {
1665                         if (i >= fifo || !priv->fifo_bw[i])
1666                                 continue;
1667
1668                         dev_warn(priv->dev, "Prev FIFO%d is shaped", i);
1669                         continue;
1670                 }
1671
1672                 if (!priv->fifo_bw[i] && i > fifo) {
1673                         dev_err(priv->dev, "Upper FIFO%d is not shaped", i);
1674                         return -EINVAL;
1675                 }
1676
1677                 shift = (i - 1) * 8;
1678                 if (i == fifo) {
1679                         send_pct &= ~(CPSW_PCT_MASK << shift);
1680                         val = DIV_ROUND_UP(bw, priv->shp_cfg_speed * 10);
1681                         if (!val)
1682                                 val = 1;
1683
1684                         send_pct |= val << shift;
1685                         pct += val;
1686                         continue;
1687                 }
1688
1689                 if (priv->fifo_bw[i])
1690                         pct += (send_pct >> shift) & CPSW_PCT_MASK;
1691         }
1692
1693         if (pct >= 100)
1694                 goto err;
1695
1696         slave_write(slave, send_pct, SEND_PERCENT);
1697         priv->fifo_bw[fifo] = bw;
1698
1699         dev_warn(priv->dev, "set FIFO%d bw = %d\n", fifo,
1700                  DIV_ROUND_CLOSEST(val * priv->shp_cfg_speed, 100));
1701
1702         return 0;
1703 err:
1704         dev_err(priv->dev, "Bandwidth doesn't fit in tc configuration");
1705         return -EINVAL;
1706 }
1707
1708 static int cpsw_set_fifo_rlimit(struct cpsw_priv *priv, int fifo, int bw)
1709 {
1710         struct cpsw_common *cpsw = priv->cpsw;
1711         struct cpsw_slave *slave;
1712         u32 tx_in_ctl_rg, val;
1713         int ret;
1714
1715         ret = cpsw_set_fifo_bw(priv, fifo, bw);
1716         if (ret)
1717                 return ret;
1718
1719         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1720         tx_in_ctl_rg = cpsw->version == CPSW_VERSION_1 ?
1721                        CPSW1_TX_IN_CTL : CPSW2_TX_IN_CTL;
1722
1723         if (!bw)
1724                 cpsw_fifo_shp_on(priv, fifo, bw);
1725
1726         val = slave_read(slave, tx_in_ctl_rg);
1727         if (cpsw_shp_is_off(priv)) {
1728                 /* disable FIFOs rate limited queues */
1729                 val &= ~(0xf << CPSW_FIFO_RATE_EN_SHIFT);
1730
1731                 /* set type of FIFO queues to normal priority mode */
1732                 val &= ~(3 << CPSW_FIFO_QUEUE_TYPE_SHIFT);
1733
1734                 /* set type of FIFO queues to be rate limited */
1735                 if (bw)
1736                         val |= 2 << CPSW_FIFO_QUEUE_TYPE_SHIFT;
1737                 else
1738                         priv->shp_cfg_speed = 0;
1739         }
1740
1741         /* toggle a FIFO rate limited queue */
1742         if (bw)
1743                 val |= BIT(fifo + CPSW_FIFO_RATE_EN_SHIFT);
1744         else
1745                 val &= ~BIT(fifo + CPSW_FIFO_RATE_EN_SHIFT);
1746         slave_write(slave, val, tx_in_ctl_rg);
1747
1748         /* FIFO transmit shape enable */
1749         cpsw_fifo_shp_on(priv, fifo, bw);
1750         return 0;
1751 }
1752
1753 /* Defaults:
1754  * class A - prio 3
1755  * class B - prio 2
1756  * shaping for class A should be set first
1757  */
1758 static int cpsw_set_cbs(struct net_device *ndev,
1759                         struct tc_cbs_qopt_offload *qopt)
1760 {
1761         struct cpsw_priv *priv = netdev_priv(ndev);
1762         struct cpsw_common *cpsw = priv->cpsw;
1763         struct cpsw_slave *slave;
1764         int prev_speed = 0;
1765         int tc, ret, fifo;
1766         u32 bw = 0;
1767
1768         tc = netdev_txq_to_tc(priv->ndev, qopt->queue);
1769
1770         /* enable channels in backward order, as highest FIFOs must be rate
1771          * limited first and for compliance with CPDMA rate limited channels
1772          * that also used in bacward order. FIFO0 cannot be rate limited.
1773          */
1774         fifo = cpsw_tc_to_fifo(tc, ndev->num_tc);
1775         if (!fifo) {
1776                 dev_err(priv->dev, "Last tc%d can't be rate limited", tc);
1777                 return -EINVAL;
1778         }
1779
1780         /* do nothing, it's disabled anyway */
1781         if (!qopt->enable && !priv->fifo_bw[fifo])
1782                 return 0;
1783
1784         /* shapers can be set if link speed is known */
1785         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
1786         if (slave->phy && slave->phy->link) {
1787                 if (priv->shp_cfg_speed &&
1788                     priv->shp_cfg_speed != slave->phy->speed)
1789                         prev_speed = priv->shp_cfg_speed;
1790
1791                 priv->shp_cfg_speed = slave->phy->speed;
1792         }
1793
1794         if (!priv->shp_cfg_speed) {
1795                 dev_err(priv->dev, "Link speed is not known");
1796                 return -1;
1797         }
1798
1799         ret = pm_runtime_get_sync(cpsw->dev);
1800         if (ret < 0) {
1801                 pm_runtime_put_noidle(cpsw->dev);
1802                 return ret;
1803         }
1804
1805         bw = qopt->enable ? qopt->idleslope : 0;
1806         ret = cpsw_set_fifo_rlimit(priv, fifo, bw);
1807         if (ret) {
1808                 priv->shp_cfg_speed = prev_speed;
1809                 prev_speed = 0;
1810         }
1811
1812         if (bw && prev_speed)
1813                 dev_warn(priv->dev,
1814                          "Speed was changed, CBS shaper speeds are changed!");
1815
1816         pm_runtime_put_sync(cpsw->dev);
1817         return ret;
1818 }
1819
1820 static void cpsw_cbs_resume(struct cpsw_slave *slave, struct cpsw_priv *priv)
1821 {
1822         int fifo, bw;
1823
1824         for (fifo = CPSW_FIFO_SHAPERS_NUM; fifo > 0; fifo--) {
1825                 bw = priv->fifo_bw[fifo];
1826                 if (!bw)
1827                         continue;
1828
1829                 cpsw_set_fifo_rlimit(priv, fifo, bw);
1830         }
1831 }
1832
1833 static void cpsw_mqprio_resume(struct cpsw_slave *slave, struct cpsw_priv *priv)
1834 {
1835         struct cpsw_common *cpsw = priv->cpsw;
1836         u32 tx_prio_map = 0;
1837         int i, tc, fifo;
1838         u32 tx_prio_rg;
1839
1840         if (!priv->mqprio_hw)
1841                 return;
1842
1843         for (i = 0; i < 8; i++) {
1844                 tc = netdev_get_prio_tc_map(priv->ndev, i);
1845                 fifo = CPSW_FIFO_SHAPERS_NUM - tc;
1846                 tx_prio_map |= fifo << (4 * i);
1847         }
1848
1849         tx_prio_rg = cpsw->version == CPSW_VERSION_1 ?
1850                      CPSW1_TX_PRI_MAP : CPSW2_TX_PRI_MAP;
1851
1852         slave_write(slave, tx_prio_map, tx_prio_rg);
1853 }
1854
1855 /* restore resources after port reset */
1856 static void cpsw_restore(struct cpsw_priv *priv)
1857 {
1858         /* restore MQPRIO offload */
1859         for_each_slave(priv, cpsw_mqprio_resume, priv);
1860
1861         /* restore CBS offload */
1862         for_each_slave(priv, cpsw_cbs_resume, priv);
1863 }
1864
1865 static int cpsw_ndo_open(struct net_device *ndev)
1866 {
1867         struct cpsw_priv *priv = netdev_priv(ndev);
1868         struct cpsw_common *cpsw = priv->cpsw;
1869         int ret;
1870         u32 reg;
1871
1872         ret = pm_runtime_get_sync(cpsw->dev);
1873         if (ret < 0) {
1874                 pm_runtime_put_noidle(cpsw->dev);
1875                 return ret;
1876         }
1877
1878         netif_carrier_off(ndev);
1879
1880         /* Notify the stack of the actual queue counts. */
1881         ret = netif_set_real_num_tx_queues(ndev, cpsw->tx_ch_num);
1882         if (ret) {
1883                 dev_err(priv->dev, "cannot set real number of tx queues\n");
1884                 goto err_cleanup;
1885         }
1886
1887         ret = netif_set_real_num_rx_queues(ndev, cpsw->rx_ch_num);
1888         if (ret) {
1889                 dev_err(priv->dev, "cannot set real number of rx queues\n");
1890                 goto err_cleanup;
1891         }
1892
1893         reg = cpsw->version;
1894
1895         dev_info(priv->dev, "initializing cpsw version %d.%d (%d)\n",
1896                  CPSW_MAJOR_VERSION(reg), CPSW_MINOR_VERSION(reg),
1897                  CPSW_RTL_VERSION(reg));
1898
1899         /* Initialize host and slave ports */
1900         if (!cpsw->usage_count)
1901                 cpsw_init_host_port(priv);
1902         for_each_slave(priv, cpsw_slave_open, priv);
1903
1904         /* Add default VLAN */
1905         if (!cpsw->data.dual_emac)
1906                 cpsw_add_default_vlan(priv);
1907         else
1908                 cpsw_ale_add_vlan(cpsw->ale, cpsw->data.default_vlan,
1909                                   ALE_ALL_PORTS, ALE_ALL_PORTS, 0, 0);
1910
1911         /* initialize shared resources for every ndev */
1912         if (!cpsw->usage_count) {
1913                 /* disable priority elevation */
1914                 writel_relaxed(0, &cpsw->regs->ptype);
1915
1916                 /* enable statistics collection only on all ports */
1917                 writel_relaxed(0x7, &cpsw->regs->stat_port_en);
1918
1919                 /* Enable internal fifo flow control */
1920                 writel(0x7, &cpsw->regs->flow_control);
1921
1922                 napi_enable(&cpsw->napi_rx);
1923                 napi_enable(&cpsw->napi_tx);
1924
1925                 if (cpsw->tx_irq_disabled) {
1926                         cpsw->tx_irq_disabled = false;
1927                         enable_irq(cpsw->irqs_table[1]);
1928                 }
1929
1930                 if (cpsw->rx_irq_disabled) {
1931                         cpsw->rx_irq_disabled = false;
1932                         enable_irq(cpsw->irqs_table[0]);
1933                 }
1934
1935                 ret = cpsw_fill_rx_channels(priv);
1936                 if (ret < 0)
1937                         goto err_cleanup;
1938
1939                 if (cpts_register(cpsw->cpts))
1940                         dev_err(priv->dev, "error registering cpts device\n");
1941
1942         }
1943
1944         cpsw_restore(priv);
1945
1946         /* Enable Interrupt pacing if configured */
1947         if (cpsw->coal_intvl != 0) {
1948                 struct ethtool_coalesce coal;
1949
1950                 coal.rx_coalesce_usecs = cpsw->coal_intvl;
1951                 cpsw_set_coalesce(ndev, &coal);
1952         }
1953
1954         cpdma_ctlr_start(cpsw->dma);
1955         cpsw_intr_enable(cpsw);
1956         cpsw->usage_count++;
1957
1958         return 0;
1959
1960 err_cleanup:
1961         cpdma_ctlr_stop(cpsw->dma);
1962         for_each_slave(priv, cpsw_slave_stop, cpsw);
1963         pm_runtime_put_sync(cpsw->dev);
1964         netif_carrier_off(priv->ndev);
1965         return ret;
1966 }
1967
1968 static int cpsw_ndo_stop(struct net_device *ndev)
1969 {
1970         struct cpsw_priv *priv = netdev_priv(ndev);
1971         struct cpsw_common *cpsw = priv->cpsw;
1972
1973         cpsw_info(priv, ifdown, "shutting down cpsw device\n");
1974         __dev_mc_unsync(priv->ndev, cpsw_del_mc_addr);
1975         netif_tx_stop_all_queues(priv->ndev);
1976         netif_carrier_off(priv->ndev);
1977
1978         if (cpsw->usage_count <= 1) {
1979                 napi_disable(&cpsw->napi_rx);
1980                 napi_disable(&cpsw->napi_tx);
1981                 cpts_unregister(cpsw->cpts);
1982                 cpsw_intr_disable(cpsw);
1983                 cpdma_ctlr_stop(cpsw->dma);
1984                 cpsw_ale_stop(cpsw->ale);
1985         }
1986         for_each_slave(priv, cpsw_slave_stop, cpsw);
1987
1988         if (cpsw_need_resplit(cpsw))
1989                 cpsw_split_res(ndev);
1990
1991         cpsw->usage_count--;
1992         pm_runtime_put_sync(cpsw->dev);
1993         return 0;
1994 }
1995
1996 static netdev_tx_t cpsw_ndo_start_xmit(struct sk_buff *skb,
1997                                        struct net_device *ndev)
1998 {
1999         struct cpsw_priv *priv = netdev_priv(ndev);
2000         struct cpsw_common *cpsw = priv->cpsw;
2001         struct cpts *cpts = cpsw->cpts;
2002         struct netdev_queue *txq;
2003         struct cpdma_chan *txch;
2004         int ret, q_idx;
2005
2006         if (skb_padto(skb, CPSW_MIN_PACKET_SIZE)) {
2007                 cpsw_err(priv, tx_err, "packet pad failed\n");
2008                 ndev->stats.tx_dropped++;
2009                 return NET_XMIT_DROP;
2010         }
2011
2012         if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
2013             cpts_is_tx_enabled(cpts) && cpts_can_timestamp(cpts, skb))
2014                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2015
2016         q_idx = skb_get_queue_mapping(skb);
2017         if (q_idx >= cpsw->tx_ch_num)
2018                 q_idx = q_idx % cpsw->tx_ch_num;
2019
2020         txch = cpsw->txv[q_idx].ch;
2021         txq = netdev_get_tx_queue(ndev, q_idx);
2022         ret = cpsw_tx_packet_submit(priv, skb, txch);
2023         if (unlikely(ret != 0)) {
2024                 cpsw_err(priv, tx_err, "desc submit failed\n");
2025                 goto fail;
2026         }
2027
2028         /* If there is no more tx desc left free then we need to
2029          * tell the kernel to stop sending us tx frames.
2030          */
2031         if (unlikely(!cpdma_check_free_tx_desc(txch))) {
2032                 netif_tx_stop_queue(txq);
2033
2034                 /* Barrier, so that stop_queue visible to other cpus */
2035                 smp_mb__after_atomic();
2036
2037                 if (cpdma_check_free_tx_desc(txch))
2038                         netif_tx_wake_queue(txq);
2039         }
2040
2041         return NETDEV_TX_OK;
2042 fail:
2043         ndev->stats.tx_dropped++;
2044         netif_tx_stop_queue(txq);
2045
2046         /* Barrier, so that stop_queue visible to other cpus */
2047         smp_mb__after_atomic();
2048
2049         if (cpdma_check_free_tx_desc(txch))
2050                 netif_tx_wake_queue(txq);
2051
2052         return NETDEV_TX_BUSY;
2053 }
2054
2055 #if IS_ENABLED(CONFIG_TI_CPTS)
2056
2057 static void cpsw_hwtstamp_v1(struct cpsw_common *cpsw)
2058 {
2059         struct cpsw_slave *slave = &cpsw->slaves[cpsw->data.active_slave];
2060         u32 ts_en, seq_id;
2061
2062         if (!cpts_is_tx_enabled(cpsw->cpts) &&
2063             !cpts_is_rx_enabled(cpsw->cpts)) {
2064                 slave_write(slave, 0, CPSW1_TS_CTL);
2065                 return;
2066         }
2067
2068         seq_id = (30 << CPSW_V1_SEQ_ID_OFS_SHIFT) | ETH_P_1588;
2069         ts_en = EVENT_MSG_BITS << CPSW_V1_MSG_TYPE_OFS;
2070
2071         if (cpts_is_tx_enabled(cpsw->cpts))
2072                 ts_en |= CPSW_V1_TS_TX_EN;
2073
2074         if (cpts_is_rx_enabled(cpsw->cpts))
2075                 ts_en |= CPSW_V1_TS_RX_EN;
2076
2077         slave_write(slave, ts_en, CPSW1_TS_CTL);
2078         slave_write(slave, seq_id, CPSW1_TS_SEQ_LTYPE);
2079 }
2080
2081 static void cpsw_hwtstamp_v2(struct cpsw_priv *priv)
2082 {
2083         struct cpsw_slave *slave;
2084         struct cpsw_common *cpsw = priv->cpsw;
2085         u32 ctrl, mtype;
2086
2087         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
2088
2089         ctrl = slave_read(slave, CPSW2_CONTROL);
2090         switch (cpsw->version) {
2091         case CPSW_VERSION_2:
2092                 ctrl &= ~CTRL_V2_ALL_TS_MASK;
2093
2094                 if (cpts_is_tx_enabled(cpsw->cpts))
2095                         ctrl |= CTRL_V2_TX_TS_BITS;
2096
2097                 if (cpts_is_rx_enabled(cpsw->cpts))
2098                         ctrl |= CTRL_V2_RX_TS_BITS;
2099                 break;
2100         case CPSW_VERSION_3:
2101         default:
2102                 ctrl &= ~CTRL_V3_ALL_TS_MASK;
2103
2104                 if (cpts_is_tx_enabled(cpsw->cpts))
2105                         ctrl |= CTRL_V3_TX_TS_BITS;
2106
2107                 if (cpts_is_rx_enabled(cpsw->cpts))
2108                         ctrl |= CTRL_V3_RX_TS_BITS;
2109                 break;
2110         }
2111
2112         mtype = (30 << TS_SEQ_ID_OFFSET_SHIFT) | EVENT_MSG_BITS;
2113
2114         slave_write(slave, mtype, CPSW2_TS_SEQ_MTYPE);
2115         slave_write(slave, ctrl, CPSW2_CONTROL);
2116         writel_relaxed(ETH_P_1588, &cpsw->regs->ts_ltype);
2117 }
2118
2119 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
2120 {
2121         struct cpsw_priv *priv = netdev_priv(dev);
2122         struct hwtstamp_config cfg;
2123         struct cpsw_common *cpsw = priv->cpsw;
2124         struct cpts *cpts = cpsw->cpts;
2125
2126         if (cpsw->version != CPSW_VERSION_1 &&
2127             cpsw->version != CPSW_VERSION_2 &&
2128             cpsw->version != CPSW_VERSION_3)
2129                 return -EOPNOTSUPP;
2130
2131         if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
2132                 return -EFAULT;
2133
2134         /* reserved for future extensions */
2135         if (cfg.flags)
2136                 return -EINVAL;
2137
2138         if (cfg.tx_type != HWTSTAMP_TX_OFF && cfg.tx_type != HWTSTAMP_TX_ON)
2139                 return -ERANGE;
2140
2141         switch (cfg.rx_filter) {
2142         case HWTSTAMP_FILTER_NONE:
2143                 cpts_rx_enable(cpts, 0);
2144                 break;
2145         case HWTSTAMP_FILTER_ALL:
2146         case HWTSTAMP_FILTER_NTP_ALL:
2147                 return -ERANGE;
2148         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2149         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2150         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2151                 cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V1_L4_EVENT);
2152                 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
2153                 break;
2154         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2155         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2156         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2157         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2158         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2159         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2160         case HWTSTAMP_FILTER_PTP_V2_EVENT:
2161         case HWTSTAMP_FILTER_PTP_V2_SYNC:
2162         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2163                 cpts_rx_enable(cpts, HWTSTAMP_FILTER_PTP_V2_EVENT);
2164                 cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
2165                 break;
2166         default:
2167                 return -ERANGE;
2168         }
2169
2170         cpts_tx_enable(cpts, cfg.tx_type == HWTSTAMP_TX_ON);
2171
2172         switch (cpsw->version) {
2173         case CPSW_VERSION_1:
2174                 cpsw_hwtstamp_v1(cpsw);
2175                 break;
2176         case CPSW_VERSION_2:
2177         case CPSW_VERSION_3:
2178                 cpsw_hwtstamp_v2(priv);
2179                 break;
2180         default:
2181                 WARN_ON(1);
2182         }
2183
2184         return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
2185 }
2186
2187 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
2188 {
2189         struct cpsw_common *cpsw = ndev_to_cpsw(dev);
2190         struct cpts *cpts = cpsw->cpts;
2191         struct hwtstamp_config cfg;
2192
2193         if (cpsw->version != CPSW_VERSION_1 &&
2194             cpsw->version != CPSW_VERSION_2 &&
2195             cpsw->version != CPSW_VERSION_3)
2196                 return -EOPNOTSUPP;
2197
2198         cfg.flags = 0;
2199         cfg.tx_type = cpts_is_tx_enabled(cpts) ?
2200                       HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
2201         cfg.rx_filter = (cpts_is_rx_enabled(cpts) ?
2202                          cpts->rx_enable : HWTSTAMP_FILTER_NONE);
2203
2204         return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
2205 }
2206 #else
2207 static int cpsw_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
2208 {
2209         return -EOPNOTSUPP;
2210 }
2211
2212 static int cpsw_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
2213 {
2214         return -EOPNOTSUPP;
2215 }
2216 #endif /*CONFIG_TI_CPTS*/
2217
2218 static int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2219 {
2220         struct cpsw_priv *priv = netdev_priv(dev);
2221         struct cpsw_common *cpsw = priv->cpsw;
2222         int slave_no = cpsw_slave_index(cpsw, priv);
2223
2224         if (!netif_running(dev))
2225                 return -EINVAL;
2226
2227         switch (cmd) {
2228         case SIOCSHWTSTAMP:
2229                 return cpsw_hwtstamp_set(dev, req);
2230         case SIOCGHWTSTAMP:
2231                 return cpsw_hwtstamp_get(dev, req);
2232         }
2233
2234         if (!cpsw->slaves[slave_no].phy)
2235                 return -EOPNOTSUPP;
2236         return phy_mii_ioctl(cpsw->slaves[slave_no].phy, req, cmd);
2237 }
2238
2239 static void cpsw_ndo_tx_timeout(struct net_device *ndev)
2240 {
2241         struct cpsw_priv *priv = netdev_priv(ndev);
2242         struct cpsw_common *cpsw = priv->cpsw;
2243         int ch;
2244
2245         cpsw_err(priv, tx_err, "transmit timeout, restarting dma\n");
2246         ndev->stats.tx_errors++;
2247         cpsw_intr_disable(cpsw);
2248         for (ch = 0; ch < cpsw->tx_ch_num; ch++) {
2249                 cpdma_chan_stop(cpsw->txv[ch].ch);
2250                 cpdma_chan_start(cpsw->txv[ch].ch);
2251         }
2252
2253         cpsw_intr_enable(cpsw);
2254         netif_trans_update(ndev);
2255         netif_tx_wake_all_queues(ndev);
2256 }
2257
2258 static int cpsw_ndo_set_mac_address(struct net_device *ndev, void *p)
2259 {
2260         struct cpsw_priv *priv = netdev_priv(ndev);
2261         struct sockaddr *addr = (struct sockaddr *)p;
2262         struct cpsw_common *cpsw = priv->cpsw;
2263         int flags = 0;
2264         u16 vid = 0;
2265         int ret;
2266
2267         if (!is_valid_ether_addr(addr->sa_data))
2268                 return -EADDRNOTAVAIL;
2269
2270         ret = pm_runtime_get_sync(cpsw->dev);
2271         if (ret < 0) {
2272                 pm_runtime_put_noidle(cpsw->dev);
2273                 return ret;
2274         }
2275
2276         if (cpsw->data.dual_emac) {
2277                 vid = cpsw->slaves[priv->emac_port].port_vlan;
2278                 flags = ALE_VLAN;
2279         }
2280
2281         cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr, HOST_PORT_NUM,
2282                            flags, vid);
2283         cpsw_ale_add_ucast(cpsw->ale, addr->sa_data, HOST_PORT_NUM,
2284                            flags, vid);
2285
2286         memcpy(priv->mac_addr, addr->sa_data, ETH_ALEN);
2287         memcpy(ndev->dev_addr, priv->mac_addr, ETH_ALEN);
2288         for_each_slave(priv, cpsw_set_slave_mac, priv);
2289
2290         pm_runtime_put(cpsw->dev);
2291
2292         return 0;
2293 }
2294
2295 #ifdef CONFIG_NET_POLL_CONTROLLER
2296 static void cpsw_ndo_poll_controller(struct net_device *ndev)
2297 {
2298         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2299
2300         cpsw_intr_disable(cpsw);
2301         cpsw_rx_interrupt(cpsw->irqs_table[0], cpsw);
2302         cpsw_tx_interrupt(cpsw->irqs_table[1], cpsw);
2303         cpsw_intr_enable(cpsw);
2304 }
2305 #endif
2306
2307 static inline int cpsw_add_vlan_ale_entry(struct cpsw_priv *priv,
2308                                 unsigned short vid)
2309 {
2310         int ret;
2311         int unreg_mcast_mask = 0;
2312         int mcast_mask;
2313         u32 port_mask;
2314         struct cpsw_common *cpsw = priv->cpsw;
2315
2316         if (cpsw->data.dual_emac) {
2317                 port_mask = (1 << (priv->emac_port + 1)) | ALE_PORT_HOST;
2318
2319                 mcast_mask = ALE_PORT_HOST;
2320                 if (priv->ndev->flags & IFF_ALLMULTI)
2321                         unreg_mcast_mask = mcast_mask;
2322         } else {
2323                 port_mask = ALE_ALL_PORTS;
2324                 mcast_mask = port_mask;
2325
2326                 if (priv->ndev->flags & IFF_ALLMULTI)
2327                         unreg_mcast_mask = ALE_ALL_PORTS;
2328                 else
2329                         unreg_mcast_mask = ALE_PORT_1 | ALE_PORT_2;
2330         }
2331
2332         ret = cpsw_ale_add_vlan(cpsw->ale, vid, port_mask, 0, port_mask,
2333                                 unreg_mcast_mask);
2334         if (ret != 0)
2335                 return ret;
2336
2337         ret = cpsw_ale_add_ucast(cpsw->ale, priv->mac_addr,
2338                                  HOST_PORT_NUM, ALE_VLAN, vid);
2339         if (ret != 0)
2340                 goto clean_vid;
2341
2342         ret = cpsw_ale_add_mcast(cpsw->ale, priv->ndev->broadcast,
2343                                  mcast_mask, ALE_VLAN, vid, 0);
2344         if (ret != 0)
2345                 goto clean_vlan_ucast;
2346         return 0;
2347
2348 clean_vlan_ucast:
2349         cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
2350                            HOST_PORT_NUM, ALE_VLAN, vid);
2351 clean_vid:
2352         cpsw_ale_del_vlan(cpsw->ale, vid, 0);
2353         return ret;
2354 }
2355
2356 static int cpsw_ndo_vlan_rx_add_vid(struct net_device *ndev,
2357                                     __be16 proto, u16 vid)
2358 {
2359         struct cpsw_priv *priv = netdev_priv(ndev);
2360         struct cpsw_common *cpsw = priv->cpsw;
2361         int ret;
2362
2363         if (vid == cpsw->data.default_vlan)
2364                 return 0;
2365
2366         ret = pm_runtime_get_sync(cpsw->dev);
2367         if (ret < 0) {
2368                 pm_runtime_put_noidle(cpsw->dev);
2369                 return ret;
2370         }
2371
2372         if (cpsw->data.dual_emac) {
2373                 /* In dual EMAC, reserved VLAN id should not be used for
2374                  * creating VLAN interfaces as this can break the dual
2375                  * EMAC port separation
2376                  */
2377                 int i;
2378
2379                 for (i = 0; i < cpsw->data.slaves; i++) {
2380                         if (vid == cpsw->slaves[i].port_vlan) {
2381                                 ret = -EINVAL;
2382                                 goto err;
2383                         }
2384                 }
2385         }
2386
2387         dev_info(priv->dev, "Adding vlanid %d to vlan filter\n", vid);
2388         ret = cpsw_add_vlan_ale_entry(priv, vid);
2389 err:
2390         pm_runtime_put(cpsw->dev);
2391         return ret;
2392 }
2393
2394 static int cpsw_ndo_vlan_rx_kill_vid(struct net_device *ndev,
2395                                      __be16 proto, u16 vid)
2396 {
2397         struct cpsw_priv *priv = netdev_priv(ndev);
2398         struct cpsw_common *cpsw = priv->cpsw;
2399         int ret;
2400
2401         if (vid == cpsw->data.default_vlan)
2402                 return 0;
2403
2404         ret = pm_runtime_get_sync(cpsw->dev);
2405         if (ret < 0) {
2406                 pm_runtime_put_noidle(cpsw->dev);
2407                 return ret;
2408         }
2409
2410         if (cpsw->data.dual_emac) {
2411                 int i;
2412
2413                 for (i = 0; i < cpsw->data.slaves; i++) {
2414                         if (vid == cpsw->slaves[i].port_vlan)
2415                                 goto err;
2416                 }
2417         }
2418
2419         dev_info(priv->dev, "removing vlanid %d from vlan filter\n", vid);
2420         ret = cpsw_ale_del_vlan(cpsw->ale, vid, 0);
2421         ret |= cpsw_ale_del_ucast(cpsw->ale, priv->mac_addr,
2422                                   HOST_PORT_NUM, ALE_VLAN, vid);
2423         ret |= cpsw_ale_del_mcast(cpsw->ale, priv->ndev->broadcast,
2424                                   0, ALE_VLAN, vid);
2425 err:
2426         pm_runtime_put(cpsw->dev);
2427         return ret;
2428 }
2429
2430 static int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate)
2431 {
2432         struct cpsw_priv *priv = netdev_priv(ndev);
2433         struct cpsw_common *cpsw = priv->cpsw;
2434         struct cpsw_slave *slave;
2435         u32 min_rate;
2436         u32 ch_rate;
2437         int i, ret;
2438
2439         ch_rate = netdev_get_tx_queue(ndev, queue)->tx_maxrate;
2440         if (ch_rate == rate)
2441                 return 0;
2442
2443         ch_rate = rate * 1000;
2444         min_rate = cpdma_chan_get_min_rate(cpsw->dma);
2445         if ((ch_rate < min_rate && ch_rate)) {
2446                 dev_err(priv->dev, "The channel rate cannot be less than %dMbps",
2447                         min_rate);
2448                 return -EINVAL;
2449         }
2450
2451         if (rate > cpsw->speed) {
2452                 dev_err(priv->dev, "The channel rate cannot be more than 2Gbps");
2453                 return -EINVAL;
2454         }
2455
2456         ret = pm_runtime_get_sync(cpsw->dev);
2457         if (ret < 0) {
2458                 pm_runtime_put_noidle(cpsw->dev);
2459                 return ret;
2460         }
2461
2462         ret = cpdma_chan_set_rate(cpsw->txv[queue].ch, ch_rate);
2463         pm_runtime_put(cpsw->dev);
2464
2465         if (ret)
2466                 return ret;
2467
2468         /* update rates for slaves tx queues */
2469         for (i = 0; i < cpsw->data.slaves; i++) {
2470                 slave = &cpsw->slaves[i];
2471                 if (!slave->ndev)
2472                         continue;
2473
2474                 netdev_get_tx_queue(slave->ndev, queue)->tx_maxrate = rate;
2475         }
2476
2477         cpsw_split_res(ndev);
2478         return ret;
2479 }
2480
2481 static int cpsw_set_mqprio(struct net_device *ndev, void *type_data)
2482 {
2483         struct tc_mqprio_qopt_offload *mqprio = type_data;
2484         struct cpsw_priv *priv = netdev_priv(ndev);
2485         struct cpsw_common *cpsw = priv->cpsw;
2486         int fifo, num_tc, count, offset;
2487         struct cpsw_slave *slave;
2488         u32 tx_prio_map = 0;
2489         int i, tc, ret;
2490
2491         num_tc = mqprio->qopt.num_tc;
2492         if (num_tc > CPSW_TC_NUM)
2493                 return -EINVAL;
2494
2495         if (mqprio->mode != TC_MQPRIO_MODE_DCB)
2496                 return -EINVAL;
2497
2498         ret = pm_runtime_get_sync(cpsw->dev);
2499         if (ret < 0) {
2500                 pm_runtime_put_noidle(cpsw->dev);
2501                 return ret;
2502         }
2503
2504         if (num_tc) {
2505                 for (i = 0; i < 8; i++) {
2506                         tc = mqprio->qopt.prio_tc_map[i];
2507                         fifo = cpsw_tc_to_fifo(tc, num_tc);
2508                         tx_prio_map |= fifo << (4 * i);
2509                 }
2510
2511                 netdev_set_num_tc(ndev, num_tc);
2512                 for (i = 0; i < num_tc; i++) {
2513                         count = mqprio->qopt.count[i];
2514                         offset = mqprio->qopt.offset[i];
2515                         netdev_set_tc_queue(ndev, i, count, offset);
2516                 }
2517         }
2518
2519         if (!mqprio->qopt.hw) {
2520                 /* restore default configuration */
2521                 netdev_reset_tc(ndev);
2522                 tx_prio_map = TX_PRIORITY_MAPPING;
2523         }
2524
2525         priv->mqprio_hw = mqprio->qopt.hw;
2526
2527         offset = cpsw->version == CPSW_VERSION_1 ?
2528                  CPSW1_TX_PRI_MAP : CPSW2_TX_PRI_MAP;
2529
2530         slave = &cpsw->slaves[cpsw_slave_index(cpsw, priv)];
2531         slave_write(slave, tx_prio_map, offset);
2532
2533         pm_runtime_put_sync(cpsw->dev);
2534
2535         return 0;
2536 }
2537
2538 static int cpsw_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type,
2539                              void *type_data)
2540 {
2541         switch (type) {
2542         case TC_SETUP_QDISC_CBS:
2543                 return cpsw_set_cbs(ndev, type_data);
2544
2545         case TC_SETUP_QDISC_MQPRIO:
2546                 return cpsw_set_mqprio(ndev, type_data);
2547
2548         default:
2549                 return -EOPNOTSUPP;
2550         }
2551 }
2552
2553 static const struct net_device_ops cpsw_netdev_ops = {
2554         .ndo_open               = cpsw_ndo_open,
2555         .ndo_stop               = cpsw_ndo_stop,
2556         .ndo_start_xmit         = cpsw_ndo_start_xmit,
2557         .ndo_set_mac_address    = cpsw_ndo_set_mac_address,
2558         .ndo_do_ioctl           = cpsw_ndo_ioctl,
2559         .ndo_validate_addr      = eth_validate_addr,
2560         .ndo_tx_timeout         = cpsw_ndo_tx_timeout,
2561         .ndo_set_rx_mode        = cpsw_ndo_set_rx_mode,
2562         .ndo_set_tx_maxrate     = cpsw_ndo_set_tx_maxrate,
2563 #ifdef CONFIG_NET_POLL_CONTROLLER
2564         .ndo_poll_controller    = cpsw_ndo_poll_controller,
2565 #endif
2566         .ndo_vlan_rx_add_vid    = cpsw_ndo_vlan_rx_add_vid,
2567         .ndo_vlan_rx_kill_vid   = cpsw_ndo_vlan_rx_kill_vid,
2568         .ndo_setup_tc           = cpsw_ndo_setup_tc,
2569 };
2570
2571 static int cpsw_get_regs_len(struct net_device *ndev)
2572 {
2573         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2574
2575         return cpsw->data.ale_entries * ALE_ENTRY_WORDS * sizeof(u32);
2576 }
2577
2578 static void cpsw_get_regs(struct net_device *ndev,
2579                           struct ethtool_regs *regs, void *p)
2580 {
2581         u32 *reg = p;
2582         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2583
2584         /* update CPSW IP version */
2585         regs->version = cpsw->version;
2586
2587         cpsw_ale_dump(cpsw->ale, reg);
2588 }
2589
2590 static void cpsw_get_drvinfo(struct net_device *ndev,
2591                              struct ethtool_drvinfo *info)
2592 {
2593         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2594         struct platform_device  *pdev = to_platform_device(cpsw->dev);
2595
2596         strlcpy(info->driver, "cpsw", sizeof(info->driver));
2597         strlcpy(info->version, "1.0", sizeof(info->version));
2598         strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
2599 }
2600
2601 static u32 cpsw_get_msglevel(struct net_device *ndev)
2602 {
2603         struct cpsw_priv *priv = netdev_priv(ndev);
2604         return priv->msg_enable;
2605 }
2606
2607 static void cpsw_set_msglevel(struct net_device *ndev, u32 value)
2608 {
2609         struct cpsw_priv *priv = netdev_priv(ndev);
2610         priv->msg_enable = value;
2611 }
2612
2613 #if IS_ENABLED(CONFIG_TI_CPTS)
2614 static int cpsw_get_ts_info(struct net_device *ndev,
2615                             struct ethtool_ts_info *info)
2616 {
2617         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2618
2619         info->so_timestamping =
2620                 SOF_TIMESTAMPING_TX_HARDWARE |
2621                 SOF_TIMESTAMPING_TX_SOFTWARE |
2622                 SOF_TIMESTAMPING_RX_HARDWARE |
2623                 SOF_TIMESTAMPING_RX_SOFTWARE |
2624                 SOF_TIMESTAMPING_SOFTWARE |
2625                 SOF_TIMESTAMPING_RAW_HARDWARE;
2626         info->phc_index = cpsw->cpts->phc_index;
2627         info->tx_types =
2628                 (1 << HWTSTAMP_TX_OFF) |
2629                 (1 << HWTSTAMP_TX_ON);
2630         info->rx_filters =
2631                 (1 << HWTSTAMP_FILTER_NONE) |
2632                 (1 << HWTSTAMP_FILTER_PTP_V1_L4_EVENT) |
2633                 (1 << HWTSTAMP_FILTER_PTP_V2_EVENT);
2634         return 0;
2635 }
2636 #else
2637 static int cpsw_get_ts_info(struct net_device *ndev,
2638                             struct ethtool_ts_info *info)
2639 {
2640         info->so_timestamping =
2641                 SOF_TIMESTAMPING_TX_SOFTWARE |
2642                 SOF_TIMESTAMPING_RX_SOFTWARE |
2643                 SOF_TIMESTAMPING_SOFTWARE;
2644         info->phc_index = -1;
2645         info->tx_types = 0;
2646         info->rx_filters = 0;
2647         return 0;
2648 }
2649 #endif
2650
2651 static int cpsw_get_link_ksettings(struct net_device *ndev,
2652                                    struct ethtool_link_ksettings *ecmd)
2653 {
2654         struct cpsw_priv *priv = netdev_priv(ndev);
2655         struct cpsw_common *cpsw = priv->cpsw;
2656         int slave_no = cpsw_slave_index(cpsw, priv);
2657
2658         if (!cpsw->slaves[slave_no].phy)
2659                 return -EOPNOTSUPP;
2660
2661         phy_ethtool_ksettings_get(cpsw->slaves[slave_no].phy, ecmd);
2662         return 0;
2663 }
2664
2665 static int cpsw_set_link_ksettings(struct net_device *ndev,
2666                                    const struct ethtool_link_ksettings *ecmd)
2667 {
2668         struct cpsw_priv *priv = netdev_priv(ndev);
2669         struct cpsw_common *cpsw = priv->cpsw;
2670         int slave_no = cpsw_slave_index(cpsw, priv);
2671
2672         if (cpsw->slaves[slave_no].phy)
2673                 return phy_ethtool_ksettings_set(cpsw->slaves[slave_no].phy,
2674                                                  ecmd);
2675         else
2676                 return -EOPNOTSUPP;
2677 }
2678
2679 static void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2680 {
2681         struct cpsw_priv *priv = netdev_priv(ndev);
2682         struct cpsw_common *cpsw = priv->cpsw;
2683         int slave_no = cpsw_slave_index(cpsw, priv);
2684
2685         wol->supported = 0;
2686         wol->wolopts = 0;
2687
2688         if (cpsw->slaves[slave_no].phy)
2689                 phy_ethtool_get_wol(cpsw->slaves[slave_no].phy, wol);
2690 }
2691
2692 static int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
2693 {
2694         struct cpsw_priv *priv = netdev_priv(ndev);
2695         struct cpsw_common *cpsw = priv->cpsw;
2696         int slave_no = cpsw_slave_index(cpsw, priv);
2697
2698         if (cpsw->slaves[slave_no].phy)
2699                 return phy_ethtool_set_wol(cpsw->slaves[slave_no].phy, wol);
2700         else
2701                 return -EOPNOTSUPP;
2702 }
2703
2704 static void cpsw_get_pauseparam(struct net_device *ndev,
2705                                 struct ethtool_pauseparam *pause)
2706 {
2707         struct cpsw_priv *priv = netdev_priv(ndev);
2708
2709         pause->autoneg = AUTONEG_DISABLE;
2710         pause->rx_pause = priv->rx_pause ? true : false;
2711         pause->tx_pause = priv->tx_pause ? true : false;
2712 }
2713
2714 static int cpsw_set_pauseparam(struct net_device *ndev,
2715                                struct ethtool_pauseparam *pause)
2716 {
2717         struct cpsw_priv *priv = netdev_priv(ndev);
2718         bool link;
2719
2720         priv->rx_pause = pause->rx_pause ? true : false;
2721         priv->tx_pause = pause->tx_pause ? true : false;
2722
2723         for_each_slave(priv, _cpsw_adjust_link, priv, &link);
2724         return 0;
2725 }
2726
2727 static int cpsw_ethtool_op_begin(struct net_device *ndev)
2728 {
2729         struct cpsw_priv *priv = netdev_priv(ndev);
2730         struct cpsw_common *cpsw = priv->cpsw;
2731         int ret;
2732
2733         ret = pm_runtime_get_sync(cpsw->dev);
2734         if (ret < 0) {
2735                 cpsw_err(priv, drv, "ethtool begin failed %d\n", ret);
2736                 pm_runtime_put_noidle(cpsw->dev);
2737         }
2738
2739         return ret;
2740 }
2741
2742 static void cpsw_ethtool_op_complete(struct net_device *ndev)
2743 {
2744         struct cpsw_priv *priv = netdev_priv(ndev);
2745         int ret;
2746
2747         ret = pm_runtime_put(priv->cpsw->dev);
2748         if (ret < 0)
2749                 cpsw_err(priv, drv, "ethtool complete failed %d\n", ret);
2750 }
2751
2752 static void cpsw_get_channels(struct net_device *ndev,
2753                               struct ethtool_channels *ch)
2754 {
2755         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2756
2757         ch->max_rx = cpsw->quirk_irq ? 1 : CPSW_MAX_QUEUES;
2758         ch->max_tx = cpsw->quirk_irq ? 1 : CPSW_MAX_QUEUES;
2759         ch->max_combined = 0;
2760         ch->max_other = 0;
2761         ch->other_count = 0;
2762         ch->rx_count = cpsw->rx_ch_num;
2763         ch->tx_count = cpsw->tx_ch_num;
2764         ch->combined_count = 0;
2765 }
2766
2767 static int cpsw_check_ch_settings(struct cpsw_common *cpsw,
2768                                   struct ethtool_channels *ch)
2769 {
2770         if (cpsw->quirk_irq) {
2771                 dev_err(cpsw->dev, "Maximum one tx/rx queue is allowed");
2772                 return -EOPNOTSUPP;
2773         }
2774
2775         if (ch->combined_count)
2776                 return -EINVAL;
2777
2778         /* verify we have at least one channel in each direction */
2779         if (!ch->rx_count || !ch->tx_count)
2780                 return -EINVAL;
2781
2782         if (ch->rx_count > cpsw->data.channels ||
2783             ch->tx_count > cpsw->data.channels)
2784                 return -EINVAL;
2785
2786         return 0;
2787 }
2788
2789 static int cpsw_update_channels_res(struct cpsw_priv *priv, int ch_num, int rx)
2790 {
2791         struct cpsw_common *cpsw = priv->cpsw;
2792         void (*handler)(void *, int, int);
2793         struct netdev_queue *queue;
2794         struct cpsw_vector *vec;
2795         int ret, *ch, vch;
2796
2797         if (rx) {
2798                 ch = &cpsw->rx_ch_num;
2799                 vec = cpsw->rxv;
2800                 handler = cpsw_rx_handler;
2801         } else {
2802                 ch = &cpsw->tx_ch_num;
2803                 vec = cpsw->txv;
2804                 handler = cpsw_tx_handler;
2805         }
2806
2807         while (*ch < ch_num) {
2808                 vch = rx ? *ch : 7 - *ch;
2809                 vec[*ch].ch = cpdma_chan_create(cpsw->dma, vch, handler, rx);
2810                 queue = netdev_get_tx_queue(priv->ndev, *ch);
2811                 queue->tx_maxrate = 0;
2812
2813                 if (IS_ERR(vec[*ch].ch))
2814                         return PTR_ERR(vec[*ch].ch);
2815
2816                 if (!vec[*ch].ch)
2817                         return -EINVAL;
2818
2819                 cpsw_info(priv, ifup, "created new %d %s channel\n", *ch,
2820                           (rx ? "rx" : "tx"));
2821                 (*ch)++;
2822         }
2823
2824         while (*ch > ch_num) {
2825                 (*ch)--;
2826
2827                 ret = cpdma_chan_destroy(vec[*ch].ch);
2828                 if (ret)
2829                         return ret;
2830
2831                 cpsw_info(priv, ifup, "destroyed %d %s channel\n", *ch,
2832                           (rx ? "rx" : "tx"));
2833         }
2834
2835         return 0;
2836 }
2837
2838 static int cpsw_update_channels(struct cpsw_priv *priv,
2839                                 struct ethtool_channels *ch)
2840 {
2841         int ret;
2842
2843         ret = cpsw_update_channels_res(priv, ch->rx_count, 1);
2844         if (ret)
2845                 return ret;
2846
2847         ret = cpsw_update_channels_res(priv, ch->tx_count, 0);
2848         if (ret)
2849                 return ret;
2850
2851         return 0;
2852 }
2853
2854 static void cpsw_suspend_data_pass(struct net_device *ndev)
2855 {
2856         struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
2857         struct cpsw_slave *slave;
2858         int i;
2859
2860         /* Disable NAPI scheduling */
2861         cpsw_intr_disable(cpsw);
2862
2863         /* Stop all transmit queues for every network device.
2864          * Disable re-using rx descriptors with dormant_on.
2865          */
2866         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2867                 if (!(slave->ndev && netif_running(slave->ndev)))
2868                         continue;
2869
2870                 netif_tx_stop_all_queues(slave->ndev);
2871                 netif_dormant_on(slave->ndev);
2872         }
2873
2874         /* Handle rest of tx packets and stop cpdma channels */
2875         cpdma_ctlr_stop(cpsw->dma);
2876 }
2877
2878 static int cpsw_resume_data_pass(struct net_device *ndev)
2879 {
2880         struct cpsw_priv *priv = netdev_priv(ndev);
2881         struct cpsw_common *cpsw = priv->cpsw;
2882         struct cpsw_slave *slave;
2883         int i, ret;
2884
2885         /* Allow rx packets handling */
2886         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++)
2887                 if (slave->ndev && netif_running(slave->ndev))
2888                         netif_dormant_off(slave->ndev);
2889
2890         /* After this receive is started */
2891         if (cpsw->usage_count) {
2892                 ret = cpsw_fill_rx_channels(priv);
2893                 if (ret)
2894                         return ret;
2895
2896                 cpdma_ctlr_start(cpsw->dma);
2897                 cpsw_intr_enable(cpsw);
2898         }
2899
2900         /* Resume transmit for every affected interface */
2901         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++)
2902                 if (slave->ndev && netif_running(slave->ndev))
2903                         netif_tx_start_all_queues(slave->ndev);
2904
2905         return 0;
2906 }
2907
2908 static int cpsw_set_channels(struct net_device *ndev,
2909                              struct ethtool_channels *chs)
2910 {
2911         struct cpsw_priv *priv = netdev_priv(ndev);
2912         struct cpsw_common *cpsw = priv->cpsw;
2913         struct cpsw_slave *slave;
2914         int i, ret;
2915
2916         ret = cpsw_check_ch_settings(cpsw, chs);
2917         if (ret < 0)
2918                 return ret;
2919
2920         cpsw_suspend_data_pass(ndev);
2921         ret = cpsw_update_channels(priv, chs);
2922         if (ret)
2923                 goto err;
2924
2925         for (i = cpsw->data.slaves, slave = cpsw->slaves; i; i--, slave++) {
2926                 if (!(slave->ndev && netif_running(slave->ndev)))
2927                         continue;
2928
2929                 /* Inform stack about new count of queues */
2930                 ret = netif_set_real_num_tx_queues(slave->ndev,
2931                                                    cpsw->tx_ch_num);
2932                 if (ret) {
2933                         dev_err(priv->dev, "cannot set real number of tx queues\n");
2934                         goto err;
2935                 }
2936
2937                 ret = netif_set_real_num_rx_queues(slave->ndev,
2938                                                    cpsw->rx_ch_num);
2939                 if (ret) {
2940                         dev_err(priv->dev, "cannot set real number of rx queues\n");
2941                         goto err;
2942                 }
2943         }
2944
2945         if (cpsw->usage_count)
2946                 cpsw_split_res(ndev);
2947
2948         ret = cpsw_resume_data_pass(ndev);
2949         if (!ret)
2950                 return 0;
2951 err:
2952         dev_err(priv->dev, "cannot update channels number, closing device\n");
2953         dev_close(ndev);
2954         return ret;
2955 }
2956
2957 static int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata)
2958 {
2959         struct cpsw_priv *priv = netdev_priv(ndev);
2960         struct cpsw_common *cpsw = priv->cpsw;
2961         int slave_no = cpsw_slave_index(cpsw, priv);
2962
2963         if (cpsw->slaves[slave_no].phy)
2964                 return phy_ethtool_get_eee(cpsw->slaves[slave_no].phy, edata);
2965         else
2966                 return -EOPNOTSUPP;
2967 }
2968
2969 static int cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata)
2970 {
2971         struct cpsw_priv *priv = netdev_priv(ndev);
2972         struct cpsw_common *cpsw = priv->cpsw;
2973         int slave_no = cpsw_slave_index(cpsw, priv);
2974
2975         if (cpsw->slaves[slave_no].phy)
2976                 return phy_ethtool_set_eee(cpsw->slaves[slave_no].phy, edata);
2977         else
2978                 return -EOPNOTSUPP;
2979 }
2980
2981 static int cpsw_nway_reset(struct net_device *ndev)
2982 {
2983         struct cpsw_priv *priv = netdev_priv(ndev);
2984         struct cpsw_common *cpsw = priv->cpsw;
2985         int slave_no = cpsw_slave_index(cpsw, priv);
2986
2987         if (cpsw->slaves[slave_no].phy)
2988                 return genphy_restart_aneg(cpsw->slaves[slave_no].phy);
2989         else
2990                 return -EOPNOTSUPP;
2991 }
2992
2993 static void cpsw_get_ringparam(struct net_device *ndev,
2994                                struct ethtool_ringparam *ering)
2995 {
2996         struct cpsw_priv *priv = netdev_priv(ndev);
2997         struct cpsw_common *cpsw = priv->cpsw;
2998
2999         /* not supported */
3000         ering->tx_max_pending = 0;
3001         ering->tx_pending = cpdma_get_num_tx_descs(cpsw->dma);
3002         ering->rx_max_pending = descs_pool_size - CPSW_MAX_QUEUES;
3003         ering->rx_pending = cpdma_get_num_rx_descs(cpsw->dma);
3004 }
3005
3006 static int cpsw_set_ringparam(struct net_device *ndev,
3007                               struct ethtool_ringparam *ering)
3008 {
3009         struct cpsw_priv *priv = netdev_priv(ndev);
3010         struct cpsw_common *cpsw = priv->cpsw;
3011         int ret;
3012
3013         /* ignore ering->tx_pending - only rx_pending adjustment is supported */
3014
3015         if (ering->rx_mini_pending || ering->rx_jumbo_pending ||
3016             ering->rx_pending < CPSW_MAX_QUEUES ||
3017             ering->rx_pending > (descs_pool_size - CPSW_MAX_QUEUES))
3018                 return -EINVAL;
3019
3020         if (ering->rx_pending == cpdma_get_num_rx_descs(cpsw->dma))
3021                 return 0;
3022
3023         cpsw_suspend_data_pass(ndev);
3024
3025         cpdma_set_num_rx_descs(cpsw->dma, ering->rx_pending);
3026
3027         if (cpsw->usage_count)
3028                 cpdma_chan_split_pool(cpsw->dma);
3029
3030         ret = cpsw_resume_data_pass(ndev);
3031         if (!ret)
3032                 return 0;
3033
3034         dev_err(&ndev->dev, "cannot set ring params, closing device\n");
3035         dev_close(ndev);
3036         return ret;
3037 }
3038
3039 static const struct ethtool_ops cpsw_ethtool_ops = {
3040         .get_drvinfo    = cpsw_get_drvinfo,
3041         .get_msglevel   = cpsw_get_msglevel,
3042         .set_msglevel   = cpsw_set_msglevel,
3043         .get_link       = ethtool_op_get_link,
3044         .get_ts_info    = cpsw_get_ts_info,
3045         .get_coalesce   = cpsw_get_coalesce,
3046         .set_coalesce   = cpsw_set_coalesce,
3047         .get_sset_count         = cpsw_get_sset_count,
3048         .get_strings            = cpsw_get_strings,
3049         .get_ethtool_stats      = cpsw_get_ethtool_stats,
3050         .get_pauseparam         = cpsw_get_pauseparam,
3051         .set_pauseparam         = cpsw_set_pauseparam,
3052         .get_wol        = cpsw_get_wol,
3053         .set_wol        = cpsw_set_wol,
3054         .get_regs_len   = cpsw_get_regs_len,
3055         .get_regs       = cpsw_get_regs,
3056         .begin          = cpsw_ethtool_op_begin,
3057         .complete       = cpsw_ethtool_op_complete,
3058         .get_channels   = cpsw_get_channels,
3059         .set_channels   = cpsw_set_channels,
3060         .get_link_ksettings     = cpsw_get_link_ksettings,
3061         .set_link_ksettings     = cpsw_set_link_ksettings,
3062         .get_eee        = cpsw_get_eee,
3063         .set_eee        = cpsw_set_eee,
3064         .nway_reset     = cpsw_nway_reset,
3065         .get_ringparam = cpsw_get_ringparam,
3066         .set_ringparam = cpsw_set_ringparam,
3067 };
3068
3069 static void cpsw_slave_init(struct cpsw_slave *slave, struct cpsw_common *cpsw,
3070                             u32 slave_reg_ofs, u32 sliver_reg_ofs)
3071 {
3072         void __iomem            *regs = cpsw->regs;
3073         int                     slave_num = slave->slave_num;
3074         struct cpsw_slave_data  *data = cpsw->data.slave_data + slave_num;
3075
3076         slave->data     = data;
3077         slave->regs     = regs + slave_reg_ofs;
3078         slave->sliver   = regs + sliver_reg_ofs;
3079         slave->port_vlan = data->dual_emac_res_vlan;
3080 }
3081
3082 static int cpsw_probe_dt(struct cpsw_platform_data *data,
3083                          struct platform_device *pdev)
3084 {
3085         struct device_node *node = pdev->dev.of_node;
3086         struct device_node *slave_node;
3087         int i = 0, ret;
3088         u32 prop;
3089
3090         if (!node)
3091                 return -EINVAL;
3092
3093         if (of_property_read_u32(node, "slaves", &prop)) {
3094                 dev_err(&pdev->dev, "Missing slaves property in the DT.\n");
3095                 return -EINVAL;
3096         }
3097         data->slaves = prop;
3098
3099         if (of_property_read_u32(node, "active_slave", &prop)) {
3100                 dev_err(&pdev->dev, "Missing active_slave property in the DT.\n");
3101                 return -EINVAL;
3102         }
3103         data->active_slave = prop;
3104
3105         data->slave_data = devm_kcalloc(&pdev->dev,
3106                                         data->slaves,
3107                                         sizeof(struct cpsw_slave_data),
3108                                         GFP_KERNEL);
3109         if (!data->slave_data)
3110                 return -ENOMEM;
3111
3112         if (of_property_read_u32(node, "cpdma_channels", &prop)) {
3113                 dev_err(&pdev->dev, "Missing cpdma_channels property in the DT.\n");
3114                 return -EINVAL;
3115         }
3116         data->channels = prop;
3117
3118         if (of_property_read_u32(node, "ale_entries", &prop)) {
3119                 dev_err(&pdev->dev, "Missing ale_entries property in the DT.\n");
3120                 return -EINVAL;
3121         }
3122         data->ale_entries = prop;
3123
3124         if (of_property_read_u32(node, "bd_ram_size", &prop)) {
3125                 dev_err(&pdev->dev, "Missing bd_ram_size property in the DT.\n");
3126                 return -EINVAL;
3127         }
3128         data->bd_ram_size = prop;
3129
3130         if (of_property_read_u32(node, "mac_control", &prop)) {
3131                 dev_err(&pdev->dev, "Missing mac_control property in the DT.\n");
3132                 return -EINVAL;
3133         }
3134         data->mac_control = prop;
3135
3136         if (of_property_read_bool(node, "dual_emac"))
3137                 data->dual_emac = 1;
3138
3139         /*
3140          * Populate all the child nodes here...
3141          */
3142         ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
3143         /* We do not want to force this, as in some cases may not have child */
3144         if (ret)
3145                 dev_warn(&pdev->dev, "Doesn't have any child node\n");
3146
3147         for_each_available_child_of_node(node, slave_node) {
3148                 struct cpsw_slave_data *slave_data = data->slave_data + i;
3149                 const void *mac_addr = NULL;
3150                 int lenp;
3151                 const __be32 *parp;
3152
3153                 /* This is no slave child node, continue */
3154                 if (strcmp(slave_node->name, "slave"))
3155                         continue;
3156
3157                 slave_data->ifphy = devm_of_phy_get(&pdev->dev, slave_node,
3158                                                     NULL);
3159                 if (!IS_ENABLED(CONFIG_TI_CPSW_PHY_SEL) &&
3160                     IS_ERR(slave_data->ifphy)) {
3161                         ret = PTR_ERR(slave_data->ifphy);