1
2
3
4
5
6
7
8#include <linux/clk.h>
9#include <linux/etherdevice.h>
10#include <linux/if_vlan.h>
11#include <linux/interrupt.h>
12#include <linux/kernel.h>
13#include <linux/kmemleak.h>
14#include <linux/module.h>
15#include <linux/netdevice.h>
16#include <linux/net_tstamp.h>
17#include <linux/of.h>
18#include <linux/of_mdio.h>
19#include <linux/of_net.h>
20#include <linux/of_device.h>
21#include <linux/phy.h>
22#include <linux/phy/phy.h>
23#include <linux/platform_device.h>
24#include <linux/pm_runtime.h>
25#include <linux/regmap.h>
26#include <linux/mfd/syscon.h>
27#include <linux/sys_soc.h>
28#include <linux/dma/ti-cppi5.h>
29#include <linux/dma/k3-udma-glue.h>
30#include <net/switchdev.h>
31
32#include "cpsw_ale.h"
33#include "cpsw_sl.h"
34#include "am65-cpsw-nuss.h"
35#include "am65-cpsw-switchdev.h"
36#include "k3-cppi-desc-pool.h"
37#include "am65-cpts.h"
38
39#define AM65_CPSW_SS_BASE 0x0
40#define AM65_CPSW_SGMII_BASE 0x100
41#define AM65_CPSW_XGMII_BASE 0x2100
42#define AM65_CPSW_CPSW_NU_BASE 0x20000
43#define AM65_CPSW_NU_PORTS_BASE 0x1000
44#define AM65_CPSW_NU_FRAM_BASE 0x12000
45#define AM65_CPSW_NU_STATS_BASE 0x1a000
46#define AM65_CPSW_NU_ALE_BASE 0x1e000
47#define AM65_CPSW_NU_CPTS_BASE 0x1d000
48
49#define AM65_CPSW_NU_PORTS_OFFSET 0x1000
50#define AM65_CPSW_NU_STATS_PORT_OFFSET 0x200
51#define AM65_CPSW_NU_FRAM_PORT_OFFSET 0x200
52
53#define AM65_CPSW_MAX_PORTS 8
54
55#define AM65_CPSW_MIN_PACKET_SIZE VLAN_ETH_ZLEN
56#define AM65_CPSW_MAX_PACKET_SIZE (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)
57
58#define AM65_CPSW_REG_CTL 0x004
59#define AM65_CPSW_REG_STAT_PORT_EN 0x014
60#define AM65_CPSW_REG_PTYPE 0x018
61
62#define AM65_CPSW_P0_REG_CTL 0x004
63#define AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET 0x008
64
65#define AM65_CPSW_PORT_REG_PRI_CTL 0x01c
66#define AM65_CPSW_PORT_REG_RX_PRI_MAP 0x020
67#define AM65_CPSW_PORT_REG_RX_MAXLEN 0x024
68
69#define AM65_CPSW_PORTN_REG_SA_L 0x308
70#define AM65_CPSW_PORTN_REG_SA_H 0x30c
71#define AM65_CPSW_PORTN_REG_TS_CTL 0x310
72#define AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG 0x314
73#define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG 0x318
74#define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2 0x31C
75
76#define AM65_CPSW_CTL_VLAN_AWARE BIT(1)
77#define AM65_CPSW_CTL_P0_ENABLE BIT(2)
78#define AM65_CPSW_CTL_P0_TX_CRC_REMOVE BIT(13)
79#define AM65_CPSW_CTL_P0_RX_PAD BIT(14)
80
81
82#define AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN BIT(0)
83
84
85#define AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN BIT(8)
86
87
88#define AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN BIT(4)
89#define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN BIT(5)
90#define AM65_CPSW_PN_TS_CTL_TX_VLAN_LT2_EN BIT(6)
91#define AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN BIT(7)
92#define AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN BIT(10)
93#define AM65_CPSW_PN_TS_CTL_TX_HOST_TS_EN BIT(11)
94#define AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT 16
95
96
97#define AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT 16
98
99
100#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 BIT(16)
101#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 BIT(17)
102#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 BIT(18)
103#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 BIT(19)
104#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 BIT(20)
105#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 BIT(21)
106#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 BIT(22)
107#define AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO BIT(23)
108
109
110#define AM65_CPSW_TS_EVENT_MSG_TYPE_BITS (BIT(0) | BIT(1) | BIT(2) | BIT(3))
111
112#define AM65_CPSW_TS_SEQ_ID_OFFSET (0x1e)
113
114#define AM65_CPSW_TS_TX_ANX_ALL_EN \
115 (AM65_CPSW_PN_TS_CTL_TX_ANX_D_EN | \
116 AM65_CPSW_PN_TS_CTL_TX_ANX_E_EN | \
117 AM65_CPSW_PN_TS_CTL_TX_ANX_F_EN)
118
119#define AM65_CPSW_ALE_AGEOUT_DEFAULT 30
120
121#define AM65_CPSW_MAX_TX_DESC 500
122#define AM65_CPSW_MAX_RX_DESC 500
123
124#define AM65_CPSW_NAV_PS_DATA_SIZE 16
125#define AM65_CPSW_NAV_SW_DATA_SIZE 16
126
127#define AM65_CPSW_DEBUG (NETIF_MSG_HW | NETIF_MSG_DRV | NETIF_MSG_LINK | \
128 NETIF_MSG_IFUP | NETIF_MSG_PROBE | NETIF_MSG_IFDOWN | \
129 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
130
131static void am65_cpsw_port_set_sl_mac(struct am65_cpsw_port *slave,
132 const u8 *dev_addr)
133{
134 u32 mac_hi = (dev_addr[0] << 0) | (dev_addr[1] << 8) |
135 (dev_addr[2] << 16) | (dev_addr[3] << 24);
136 u32 mac_lo = (dev_addr[4] << 0) | (dev_addr[5] << 8);
137
138 writel(mac_hi, slave->port_base + AM65_CPSW_PORTN_REG_SA_H);
139 writel(mac_lo, slave->port_base + AM65_CPSW_PORTN_REG_SA_L);
140}
141
142static void am65_cpsw_sl_ctl_reset(struct am65_cpsw_port *port)
143{
144 cpsw_sl_reset(port->slave.mac_sl, 100);
145
146 writel(AM65_CPSW_MAX_PACKET_SIZE,
147 port->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
148}
149
150static void am65_cpsw_nuss_get_ver(struct am65_cpsw_common *common)
151{
152 common->nuss_ver = readl(common->ss_base);
153 common->cpsw_ver = readl(common->cpsw_base);
154 dev_info(common->dev,
155 "initializing am65 cpsw nuss version 0x%08X, cpsw version 0x%08X Ports: %u quirks:%08x\n",
156 common->nuss_ver,
157 common->cpsw_ver,
158 common->port_num + 1,
159 common->pdata.quirks);
160}
161
162void am65_cpsw_nuss_adjust_link(struct net_device *ndev)
163{
164 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
165 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
166 struct phy_device *phy = port->slave.phy;
167 u32 mac_control = 0;
168
169 if (!phy)
170 return;
171
172 if (phy->link) {
173 mac_control = CPSW_SL_CTL_GMII_EN;
174
175 if (phy->speed == 1000)
176 mac_control |= CPSW_SL_CTL_GIG;
177 if (phy->speed == 10 && phy_interface_is_rgmii(phy))
178
179 mac_control |= CPSW_SL_CTL_EXT_EN;
180 if (phy->speed == 100 && phy->interface == PHY_INTERFACE_MODE_RMII)
181 mac_control |= CPSW_SL_CTL_IFCTL_A;
182 if (phy->duplex)
183 mac_control |= CPSW_SL_CTL_FULLDUPLEX;
184
185
186
187
188 if (port->slave.rx_pause)
189 mac_control |= CPSW_SL_CTL_RX_FLOW_EN;
190
191 if (port->slave.tx_pause)
192 mac_control |= CPSW_SL_CTL_TX_FLOW_EN;
193
194 cpsw_sl_ctl_set(port->slave.mac_sl, mac_control);
195
196
197 cpsw_ale_control_set(common->ale, port->port_id,
198 ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
199
200 am65_cpsw_qos_link_up(ndev, phy->speed);
201 netif_tx_wake_all_queues(ndev);
202 } else {
203 int tmo;
204
205
206 cpsw_ale_control_set(common->ale, port->port_id,
207 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
208
209 cpsw_sl_ctl_set(port->slave.mac_sl, CPSW_SL_CTL_CMD_IDLE);
210
211 tmo = cpsw_sl_wait_for_idle(port->slave.mac_sl, 100);
212 dev_dbg(common->dev, "donw msc_sl %08x tmo %d\n",
213 cpsw_sl_reg_read(port->slave.mac_sl, CPSW_SL_MACSTATUS),
214 tmo);
215
216 cpsw_sl_ctl_reset(port->slave.mac_sl);
217
218 am65_cpsw_qos_link_down(ndev);
219 netif_tx_stop_all_queues(ndev);
220 }
221
222 phy_print_status(phy);
223}
224
225static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
226 __be16 proto, u16 vid)
227{
228 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
229 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
230 u32 port_mask, unreg_mcast = 0;
231 int ret;
232
233 if (!common->is_emac_mode)
234 return 0;
235
236 if (!netif_running(ndev) || !vid)
237 return 0;
238
239 ret = pm_runtime_get_sync(common->dev);
240 if (ret < 0) {
241 pm_runtime_put_noidle(common->dev);
242 return ret;
243 }
244
245 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
246 if (!vid)
247 unreg_mcast = port_mask;
248 dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);
249 ret = cpsw_ale_vlan_add_modify(common->ale, vid, port_mask,
250 unreg_mcast, port_mask, 0);
251
252 pm_runtime_put(common->dev);
253 return ret;
254}
255
256static int am65_cpsw_nuss_ndo_slave_kill_vid(struct net_device *ndev,
257 __be16 proto, u16 vid)
258{
259 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
260 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
261 int ret;
262
263 if (!common->is_emac_mode)
264 return 0;
265
266 if (!netif_running(ndev) || !vid)
267 return 0;
268
269 ret = pm_runtime_get_sync(common->dev);
270 if (ret < 0) {
271 pm_runtime_put_noidle(common->dev);
272 return ret;
273 }
274
275 dev_info(common->dev, "Removing vlan %d from vlan filter\n", vid);
276 ret = cpsw_ale_del_vlan(common->ale, vid,
277 BIT(port->port_id) | ALE_PORT_HOST);
278
279 pm_runtime_put(common->dev);
280 return ret;
281}
282
283static void am65_cpsw_slave_set_promisc(struct am65_cpsw_port *port,
284 bool promisc)
285{
286 struct am65_cpsw_common *common = port->common;
287
288 if (promisc && !common->is_emac_mode) {
289 dev_dbg(common->dev, "promisc mode requested in switch mode");
290 return;
291 }
292
293 if (promisc) {
294
295 cpsw_ale_control_set(common->ale, port->port_id,
296 ALE_PORT_MACONLY_CAF, 1);
297 dev_dbg(common->dev, "promisc enabled\n");
298 } else {
299
300 cpsw_ale_control_set(common->ale, port->port_id,
301 ALE_PORT_MACONLY_CAF, 0);
302 dev_dbg(common->dev, "promisc disabled\n");
303 }
304}
305
306static void am65_cpsw_nuss_ndo_slave_set_rx_mode(struct net_device *ndev)
307{
308 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
309 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
310 u32 port_mask;
311 bool promisc;
312
313 promisc = !!(ndev->flags & IFF_PROMISC);
314 am65_cpsw_slave_set_promisc(port, promisc);
315
316 if (promisc)
317 return;
318
319
320 cpsw_ale_set_allmulti(common->ale,
321 ndev->flags & IFF_ALLMULTI, port->port_id);
322
323 port_mask = ALE_PORT_HOST;
324
325 cpsw_ale_flush_multicast(common->ale, port_mask, -1);
326
327 if (!netdev_mc_empty(ndev)) {
328 struct netdev_hw_addr *ha;
329
330
331 netdev_for_each_mc_addr(ha, ndev) {
332 cpsw_ale_add_mcast(common->ale, ha->addr,
333 port_mask, 0, 0, 0);
334 }
335 }
336}
337
338static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev,
339 unsigned int txqueue)
340{
341 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
342 struct am65_cpsw_tx_chn *tx_chn;
343 struct netdev_queue *netif_txq;
344 unsigned long trans_start;
345
346 netif_txq = netdev_get_tx_queue(ndev, txqueue);
347 tx_chn = &common->tx_chns[txqueue];
348 trans_start = netif_txq->trans_start;
349
350 netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n",
351 txqueue,
352 netif_tx_queue_stopped(netif_txq),
353 jiffies_to_msecs(jiffies - trans_start),
354 dql_avail(&netif_txq->dql),
355 k3_cppi_desc_pool_avail(tx_chn->desc_pool));
356
357 if (netif_tx_queue_stopped(netif_txq)) {
358
359 txq_trans_update(netif_txq);
360 netif_tx_wake_queue(netif_txq);
361 }
362}
363
364static int am65_cpsw_nuss_rx_push(struct am65_cpsw_common *common,
365 struct sk_buff *skb)
366{
367 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
368 struct cppi5_host_desc_t *desc_rx;
369 struct device *dev = common->dev;
370 u32 pkt_len = skb_tailroom(skb);
371 dma_addr_t desc_dma;
372 dma_addr_t buf_dma;
373 void *swdata;
374
375 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);
376 if (!desc_rx) {
377 dev_err(dev, "Failed to allocate RXFDQ descriptor\n");
378 return -ENOMEM;
379 }
380 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
381
382 buf_dma = dma_map_single(rx_chn->dma_dev, skb->data, pkt_len,
383 DMA_FROM_DEVICE);
384 if (unlikely(dma_mapping_error(rx_chn->dma_dev, buf_dma))) {
385 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
386 dev_err(dev, "Failed to map rx skb buffer\n");
387 return -EINVAL;
388 }
389
390 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
391 AM65_CPSW_NAV_PS_DATA_SIZE);
392 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
393 cppi5_hdesc_attach_buf(desc_rx, buf_dma, skb_tailroom(skb), buf_dma, skb_tailroom(skb));
394 swdata = cppi5_hdesc_get_swdata(desc_rx);
395 *((void **)swdata) = skb;
396
397 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, 0, desc_rx, desc_dma);
398}
399
400void am65_cpsw_nuss_set_p0_ptype(struct am65_cpsw_common *common)
401{
402 struct am65_cpsw_host *host_p = am65_common_get_host(common);
403 u32 val, pri_map;
404
405
406 val = readl(host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
407
408 if (common->pf_p0_rx_ptype_rrobin) {
409 val |= AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
410
411
412
413 pri_map = 0x0;
414 } else {
415 val &= ~AM65_CPSW_PORT_REG_PRI_CTL_RX_PTYPE_RROBIN;
416
417 pri_map = 0x76543210;
418 }
419
420 writel(pri_map, host_p->port_base + AM65_CPSW_PORT_REG_RX_PRI_MAP);
421 writel(val, host_p->port_base + AM65_CPSW_PORT_REG_PRI_CTL);
422}
423
424static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common);
425static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
426static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
427static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
428
429static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common,
430 netdev_features_t features)
431{
432 struct am65_cpsw_host *host_p = am65_common_get_host(common);
433 int port_idx, i, ret;
434 struct sk_buff *skb;
435 u32 val, port_mask;
436
437 if (common->usage_count)
438 return 0;
439
440
441 writel(AM65_CPSW_CTL_P0_ENABLE | AM65_CPSW_CTL_P0_TX_CRC_REMOVE |
442 AM65_CPSW_CTL_VLAN_AWARE | AM65_CPSW_CTL_P0_RX_PAD,
443 common->cpsw_base + AM65_CPSW_REG_CTL);
444
445 writel(AM65_CPSW_MAX_PACKET_SIZE,
446 host_p->port_base + AM65_CPSW_PORT_REG_RX_MAXLEN);
447
448 writel(common->rx_flow_id_base,
449 host_p->port_base + AM65_CPSW_PORT0_REG_FLOW_ID_OFFSET);
450
451 writel(AM65_CPSW_P0_REG_CTL_RX_CHECKSUM_EN, host_p->port_base + AM65_CPSW_P0_REG_CTL);
452
453 am65_cpsw_nuss_set_p0_ptype(common);
454
455
456 val = BIT(HOST_PORT_NUM);
457 for (port_idx = 0; port_idx < common->port_num; port_idx++) {
458 struct am65_cpsw_port *port = &common->ports[port_idx];
459
460 if (!port->disabled)
461 val |= BIT(port->port_id);
462 }
463 writel(val, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
464
465
466 writel(0, common->cpsw_base + AM65_CPSW_REG_PTYPE);
467
468 cpsw_ale_start(common->ale);
469
470
471 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
472 ALE_DEFAULT_THREAD_ID, 0);
473 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
474 ALE_DEFAULT_THREAD_ENABLE, 1);
475
476 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_VLAN_AWARE, 1);
477 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
478 ALE_PORT_STATE, ALE_PORT_STATE_FORWARD);
479
480
481 port_mask = GENMASK(common->port_num, 0) &
482 ~common->disabled_ports_mask;
483
484 cpsw_ale_add_vlan(common->ale, 0, port_mask,
485 port_mask, port_mask,
486 port_mask & ~ALE_PORT_HOST);
487
488 if (common->is_emac_mode)
489 am65_cpsw_init_host_port_emac(common);
490 else
491 am65_cpsw_init_host_port_switch(common);
492
493 for (i = 0; i < common->rx_chns.descs_num; i++) {
494 skb = __netdev_alloc_skb_ip_align(NULL,
495 AM65_CPSW_MAX_PACKET_SIZE,
496 GFP_KERNEL);
497 if (!skb) {
498 dev_err(common->dev, "cannot allocate skb\n");
499 return -ENOMEM;
500 }
501
502 ret = am65_cpsw_nuss_rx_push(common, skb);
503 if (ret < 0) {
504 dev_err(common->dev,
505 "cannot submit skb to channel rx, error %d\n",
506 ret);
507 kfree_skb(skb);
508 return ret;
509 }
510 kmemleak_not_leak(skb);
511 }
512 k3_udma_glue_enable_rx_chn(common->rx_chns.rx_chn);
513
514 for (i = 0; i < common->tx_ch_num; i++) {
515 ret = k3_udma_glue_enable_tx_chn(common->tx_chns[i].tx_chn);
516 if (ret)
517 return ret;
518 napi_enable(&common->tx_chns[i].napi_tx);
519 }
520
521 napi_enable(&common->napi_rx);
522 if (common->rx_irq_disabled) {
523 common->rx_irq_disabled = false;
524 enable_irq(common->rx_chns.irq);
525 }
526
527 dev_dbg(common->dev, "cpsw_nuss started\n");
528 return 0;
529}
530
531static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma);
532static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma);
533
534static int am65_cpsw_nuss_common_stop(struct am65_cpsw_common *common)
535{
536 int i;
537
538 if (common->usage_count != 1)
539 return 0;
540
541 cpsw_ale_control_set(common->ale, HOST_PORT_NUM,
542 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
543
544
545 atomic_set(&common->tdown_cnt, common->tx_ch_num);
546
547 smp_mb__after_atomic();
548 reinit_completion(&common->tdown_complete);
549
550 for (i = 0; i < common->tx_ch_num; i++)
551 k3_udma_glue_tdown_tx_chn(common->tx_chns[i].tx_chn, false);
552
553 i = wait_for_completion_timeout(&common->tdown_complete,
554 msecs_to_jiffies(1000));
555 if (!i)
556 dev_err(common->dev, "tx timeout\n");
557 for (i = 0; i < common->tx_ch_num; i++)
558 napi_disable(&common->tx_chns[i].napi_tx);
559
560 for (i = 0; i < common->tx_ch_num; i++) {
561 k3_udma_glue_reset_tx_chn(common->tx_chns[i].tx_chn,
562 &common->tx_chns[i],
563 am65_cpsw_nuss_tx_cleanup);
564 k3_udma_glue_disable_tx_chn(common->tx_chns[i].tx_chn);
565 }
566
567 k3_udma_glue_tdown_rx_chn(common->rx_chns.rx_chn, true);
568 napi_disable(&common->napi_rx);
569
570 for (i = 0; i < AM65_CPSW_MAX_RX_FLOWS; i++)
571 k3_udma_glue_reset_rx_chn(common->rx_chns.rx_chn, i,
572 &common->rx_chns,
573 am65_cpsw_nuss_rx_cleanup, !!i);
574
575 k3_udma_glue_disable_rx_chn(common->rx_chns.rx_chn);
576
577 cpsw_ale_stop(common->ale);
578
579 writel(0, common->cpsw_base + AM65_CPSW_REG_CTL);
580 writel(0, common->cpsw_base + AM65_CPSW_REG_STAT_PORT_EN);
581
582 dev_dbg(common->dev, "cpsw_nuss stopped\n");
583 return 0;
584}
585
586static int am65_cpsw_nuss_ndo_slave_stop(struct net_device *ndev)
587{
588 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
589 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
590 int ret;
591
592 if (port->slave.phy)
593 phy_stop(port->slave.phy);
594
595 netif_tx_stop_all_queues(ndev);
596
597 if (port->slave.phy) {
598 phy_disconnect(port->slave.phy);
599 port->slave.phy = NULL;
600 }
601
602 ret = am65_cpsw_nuss_common_stop(common);
603 if (ret)
604 return ret;
605
606 common->usage_count--;
607 pm_runtime_put(common->dev);
608 return 0;
609}
610
611static int cpsw_restore_vlans(struct net_device *vdev, int vid, void *arg)
612{
613 struct am65_cpsw_port *port = arg;
614
615 if (!vdev)
616 return 0;
617
618 return am65_cpsw_nuss_ndo_slave_add_vid(port->ndev, 0, vid);
619}
620
621static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
622{
623 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
624 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
625 int ret, i;
626
627 ret = pm_runtime_get_sync(common->dev);
628 if (ret < 0) {
629 pm_runtime_put_noidle(common->dev);
630 return ret;
631 }
632
633
634 ret = netif_set_real_num_tx_queues(ndev, common->tx_ch_num);
635 if (ret) {
636 dev_err(common->dev, "cannot set real number of tx queues\n");
637 return ret;
638 }
639
640 ret = netif_set_real_num_rx_queues(ndev, AM65_CPSW_MAX_RX_QUEUES);
641 if (ret) {
642 dev_err(common->dev, "cannot set real number of rx queues\n");
643 return ret;
644 }
645
646 for (i = 0; i < common->tx_ch_num; i++)
647 netdev_tx_reset_queue(netdev_get_tx_queue(ndev, i));
648
649 ret = am65_cpsw_nuss_common_open(common, ndev->features);
650 if (ret)
651 return ret;
652
653 common->usage_count++;
654
655 am65_cpsw_port_set_sl_mac(port, ndev->dev_addr);
656
657 if (common->is_emac_mode)
658 am65_cpsw_init_port_emac_ale(port);
659 else
660 am65_cpsw_init_port_switch_ale(port);
661
662
663 am65_cpsw_sl_ctl_reset(port);
664
665 ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET,
666 port->slave.phy_if);
667 if (ret)
668 goto error_cleanup;
669
670 if (port->slave.phy_node) {
671 port->slave.phy = of_phy_connect(ndev,
672 port->slave.phy_node,
673 &am65_cpsw_nuss_adjust_link,
674 0, port->slave.phy_if);
675 if (!port->slave.phy) {
676 dev_err(common->dev, "phy %pOF not found on slave %d\n",
677 port->slave.phy_node,
678 port->port_id);
679 ret = -ENODEV;
680 goto error_cleanup;
681 }
682 }
683
684
685 vlan_for_each(ndev, cpsw_restore_vlans, port);
686
687 phy_attached_info(port->slave.phy);
688 phy_start(port->slave.phy);
689
690 return 0;
691
692error_cleanup:
693 am65_cpsw_nuss_ndo_slave_stop(ndev);
694 return ret;
695}
696
697static void am65_cpsw_nuss_rx_cleanup(void *data, dma_addr_t desc_dma)
698{
699 struct am65_cpsw_rx_chn *rx_chn = data;
700 struct cppi5_host_desc_t *desc_rx;
701 struct sk_buff *skb;
702 dma_addr_t buf_dma;
703 u32 buf_dma_len;
704 void **swdata;
705
706 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
707 swdata = cppi5_hdesc_get_swdata(desc_rx);
708 skb = *swdata;
709 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
710 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
711
712 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
713 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
714
715 dev_kfree_skb_any(skb);
716}
717
718static void am65_cpsw_nuss_rx_ts(struct sk_buff *skb, u32 *psdata)
719{
720 struct skb_shared_hwtstamps *ssh;
721 u64 ns;
722
723 ns = ((u64)psdata[1] << 32) | psdata[0];
724
725 ssh = skb_hwtstamps(skb);
726 memset(ssh, 0, sizeof(*ssh));
727 ssh->hwtstamp = ns_to_ktime(ns);
728}
729
730
731#define AM65_CPSW_RX_PSD_CSUM_ADD GENMASK(15, 0)
732#define AM65_CPSW_RX_PSD_CSUM_ERR BIT(16)
733#define AM65_CPSW_RX_PSD_IS_FRAGMENT BIT(17)
734#define AM65_CPSW_RX_PSD_IS_TCP BIT(18)
735#define AM65_CPSW_RX_PSD_IPV6_VALID BIT(19)
736#define AM65_CPSW_RX_PSD_IPV4_VALID BIT(20)
737
738static void am65_cpsw_nuss_rx_csum(struct sk_buff *skb, u32 csum_info)
739{
740
741
742
743
744
745
746
747
748
749 skb_checksum_none_assert(skb);
750
751 if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM)))
752 return;
753
754 if ((csum_info & (AM65_CPSW_RX_PSD_IPV6_VALID |
755 AM65_CPSW_RX_PSD_IPV4_VALID)) &&
756 !(csum_info & AM65_CPSW_RX_PSD_CSUM_ERR)) {
757
758 if (!(csum_info & AM65_CPSW_RX_PSD_IS_FRAGMENT))
759 skb->ip_summed = CHECKSUM_UNNECESSARY;
760 }
761}
762
763static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_common *common,
764 u32 flow_idx)
765{
766 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
767 u32 buf_dma_len, pkt_len, port_id = 0, csum_info;
768 struct am65_cpsw_ndev_priv *ndev_priv;
769 struct am65_cpsw_ndev_stats *stats;
770 struct cppi5_host_desc_t *desc_rx;
771 struct device *dev = common->dev;
772 struct sk_buff *skb, *new_skb;
773 dma_addr_t desc_dma, buf_dma;
774 struct am65_cpsw_port *port;
775 struct net_device *ndev;
776 void **swdata;
777 u32 *psdata;
778 int ret = 0;
779
780 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_idx, &desc_dma);
781 if (ret) {
782 if (ret != -ENODATA)
783 dev_err(dev, "RX: pop chn fail %d\n", ret);
784 return ret;
785 }
786
787 if (cppi5_desc_is_tdcm(desc_dma)) {
788 dev_dbg(dev, "%s RX tdown flow: %u\n", __func__, flow_idx);
789 return 0;
790 }
791
792 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
793 dev_dbg(dev, "%s flow_idx: %u desc %pad\n",
794 __func__, flow_idx, &desc_dma);
795
796 swdata = cppi5_hdesc_get_swdata(desc_rx);
797 skb = *swdata;
798 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
799 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
800 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
801 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
802 dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id);
803 port = am65_common_get_port(common, port_id);
804 ndev = port->ndev;
805 skb->dev = ndev;
806
807 psdata = cppi5_hdesc_get_psdata(desc_rx);
808
809 if (port->rx_ts_enabled)
810 am65_cpsw_nuss_rx_ts(skb, psdata);
811 csum_info = psdata[2];
812 dev_dbg(dev, "%s rx csum_info:%#x\n", __func__, csum_info);
813
814 dma_unmap_single(rx_chn->dma_dev, buf_dma, buf_dma_len, DMA_FROM_DEVICE);
815
816 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
817
818 new_skb = netdev_alloc_skb_ip_align(ndev, AM65_CPSW_MAX_PACKET_SIZE);
819 if (new_skb) {
820 ndev_priv = netdev_priv(ndev);
821 am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark);
822 skb_put(skb, pkt_len);
823 skb->protocol = eth_type_trans(skb, ndev);
824 am65_cpsw_nuss_rx_csum(skb, csum_info);
825 napi_gro_receive(&common->napi_rx, skb);
826
827 stats = this_cpu_ptr(ndev_priv->stats);
828
829 u64_stats_update_begin(&stats->syncp);
830 stats->rx_packets++;
831 stats->rx_bytes += pkt_len;
832 u64_stats_update_end(&stats->syncp);
833 kmemleak_not_leak(new_skb);
834 } else {
835 ndev->stats.rx_dropped++;
836 new_skb = skb;
837 }
838
839 if (netif_dormant(ndev)) {
840 dev_kfree_skb_any(new_skb);
841 ndev->stats.rx_dropped++;
842 return 0;
843 }
844
845 ret = am65_cpsw_nuss_rx_push(common, new_skb);
846 if (WARN_ON(ret < 0)) {
847 dev_kfree_skb_any(new_skb);
848 ndev->stats.rx_errors++;
849 ndev->stats.rx_dropped++;
850 }
851
852 return ret;
853}
854
855static int am65_cpsw_nuss_rx_poll(struct napi_struct *napi_rx, int budget)
856{
857 struct am65_cpsw_common *common = am65_cpsw_napi_to_common(napi_rx);
858 int flow = AM65_CPSW_MAX_RX_FLOWS;
859 int cur_budget, ret;
860 int num_rx = 0;
861
862
863 while (flow--) {
864 cur_budget = budget - num_rx;
865
866 while (cur_budget--) {
867 ret = am65_cpsw_nuss_rx_packets(common, flow);
868 if (ret)
869 break;
870 num_rx++;
871 }
872
873 if (num_rx >= budget)
874 break;
875 }
876
877 dev_dbg(common->dev, "%s num_rx:%d %d\n", __func__, num_rx, budget);
878
879 if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) {
880 if (common->rx_irq_disabled) {
881 common->rx_irq_disabled = false;
882 enable_irq(common->rx_chns.irq);
883 }
884 }
885
886 return num_rx;
887}
888
889static void am65_cpsw_nuss_xmit_free(struct am65_cpsw_tx_chn *tx_chn,
890 struct cppi5_host_desc_t *desc)
891{
892 struct cppi5_host_desc_t *first_desc, *next_desc;
893 dma_addr_t buf_dma, next_desc_dma;
894 u32 buf_dma_len;
895
896 first_desc = desc;
897 next_desc = first_desc;
898
899 cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
900 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
901
902 dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len, DMA_TO_DEVICE);
903
904 next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
905 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
906 while (next_desc_dma) {
907 next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
908 next_desc_dma);
909 cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
910 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
911
912 dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
913 DMA_TO_DEVICE);
914
915 next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
916 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
917
918 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
919 }
920
921 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
922}
923
924static void am65_cpsw_nuss_tx_cleanup(void *data, dma_addr_t desc_dma)
925{
926 struct am65_cpsw_tx_chn *tx_chn = data;
927 struct cppi5_host_desc_t *desc_tx;
928 struct sk_buff *skb;
929 void **swdata;
930
931 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
932 swdata = cppi5_hdesc_get_swdata(desc_tx);
933 skb = *(swdata);
934 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
935
936 dev_kfree_skb_any(skb);
937}
938
939static struct sk_buff *
940am65_cpsw_nuss_tx_compl_packet(struct am65_cpsw_tx_chn *tx_chn,
941 dma_addr_t desc_dma)
942{
943 struct am65_cpsw_ndev_priv *ndev_priv;
944 struct am65_cpsw_ndev_stats *stats;
945 struct cppi5_host_desc_t *desc_tx;
946 struct net_device *ndev;
947 struct sk_buff *skb;
948 void **swdata;
949
950 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
951 desc_dma);
952 swdata = cppi5_hdesc_get_swdata(desc_tx);
953 skb = *(swdata);
954 am65_cpsw_nuss_xmit_free(tx_chn, desc_tx);
955
956 ndev = skb->dev;
957
958 am65_cpts_tx_timestamp(tx_chn->common->cpts, skb);
959
960 ndev_priv = netdev_priv(ndev);
961 stats = this_cpu_ptr(ndev_priv->stats);
962 u64_stats_update_begin(&stats->syncp);
963 stats->tx_packets++;
964 stats->tx_bytes += skb->len;
965 u64_stats_update_end(&stats->syncp);
966
967 return skb;
968}
969
970static void am65_cpsw_nuss_tx_wake(struct am65_cpsw_tx_chn *tx_chn, struct net_device *ndev,
971 struct netdev_queue *netif_txq)
972{
973 if (netif_tx_queue_stopped(netif_txq)) {
974
975
976
977
978 __netif_tx_lock(netif_txq, smp_processor_id());
979 if (netif_running(ndev) &&
980 (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >= MAX_SKB_FRAGS))
981 netif_tx_wake_queue(netif_txq);
982
983 __netif_tx_unlock(netif_txq);
984 }
985}
986
987static int am65_cpsw_nuss_tx_compl_packets(struct am65_cpsw_common *common,
988 int chn, unsigned int budget)
989{
990 struct device *dev = common->dev;
991 struct am65_cpsw_tx_chn *tx_chn;
992 struct netdev_queue *netif_txq;
993 unsigned int total_bytes = 0;
994 struct net_device *ndev;
995 struct sk_buff *skb;
996 dma_addr_t desc_dma;
997 int res, num_tx = 0;
998
999 tx_chn = &common->tx_chns[chn];
1000
1001 while (true) {
1002 spin_lock(&tx_chn->lock);
1003 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1004 spin_unlock(&tx_chn->lock);
1005 if (res == -ENODATA)
1006 break;
1007
1008 if (cppi5_desc_is_tdcm(desc_dma)) {
1009 if (atomic_dec_and_test(&common->tdown_cnt))
1010 complete(&common->tdown_complete);
1011 break;
1012 }
1013
1014 skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);
1015 total_bytes = skb->len;
1016 ndev = skb->dev;
1017 napi_consume_skb(skb, budget);
1018 num_tx++;
1019
1020 netif_txq = netdev_get_tx_queue(ndev, chn);
1021
1022 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1023
1024 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1025 }
1026
1027 dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1028
1029 return num_tx;
1030}
1031
1032static int am65_cpsw_nuss_tx_compl_packets_2g(struct am65_cpsw_common *common,
1033 int chn, unsigned int budget)
1034{
1035 struct device *dev = common->dev;
1036 struct am65_cpsw_tx_chn *tx_chn;
1037 struct netdev_queue *netif_txq;
1038 unsigned int total_bytes = 0;
1039 struct net_device *ndev;
1040 struct sk_buff *skb;
1041 dma_addr_t desc_dma;
1042 int res, num_tx = 0;
1043
1044 tx_chn = &common->tx_chns[chn];
1045
1046 while (true) {
1047 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
1048 if (res == -ENODATA)
1049 break;
1050
1051 if (cppi5_desc_is_tdcm(desc_dma)) {
1052 if (atomic_dec_and_test(&common->tdown_cnt))
1053 complete(&common->tdown_complete);
1054 break;
1055 }
1056
1057 skb = am65_cpsw_nuss_tx_compl_packet(tx_chn, desc_dma);
1058
1059 ndev = skb->dev;
1060 total_bytes += skb->len;
1061 napi_consume_skb(skb, budget);
1062 num_tx++;
1063 }
1064
1065 if (!num_tx)
1066 return 0;
1067
1068 netif_txq = netdev_get_tx_queue(ndev, chn);
1069
1070 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
1071
1072 am65_cpsw_nuss_tx_wake(tx_chn, ndev, netif_txq);
1073
1074 dev_dbg(dev, "%s:%u pkt:%d\n", __func__, chn, num_tx);
1075
1076 return num_tx;
1077}
1078
1079static int am65_cpsw_nuss_tx_poll(struct napi_struct *napi_tx, int budget)
1080{
1081 struct am65_cpsw_tx_chn *tx_chn = am65_cpsw_napi_to_tx_chn(napi_tx);
1082 int num_tx;
1083
1084 if (AM65_CPSW_IS_CPSW2G(tx_chn->common))
1085 num_tx = am65_cpsw_nuss_tx_compl_packets_2g(tx_chn->common, tx_chn->id, budget);
1086 else
1087 num_tx = am65_cpsw_nuss_tx_compl_packets(tx_chn->common, tx_chn->id, budget);
1088
1089 if (num_tx >= budget)
1090 return budget;
1091
1092 if (napi_complete_done(napi_tx, num_tx))
1093 enable_irq(tx_chn->irq);
1094
1095 return 0;
1096}
1097
1098static irqreturn_t am65_cpsw_nuss_rx_irq(int irq, void *dev_id)
1099{
1100 struct am65_cpsw_common *common = dev_id;
1101
1102 common->rx_irq_disabled = true;
1103 disable_irq_nosync(irq);
1104 napi_schedule(&common->napi_rx);
1105
1106 return IRQ_HANDLED;
1107}
1108
1109static irqreturn_t am65_cpsw_nuss_tx_irq(int irq, void *dev_id)
1110{
1111 struct am65_cpsw_tx_chn *tx_chn = dev_id;
1112
1113 disable_irq_nosync(irq);
1114 napi_schedule(&tx_chn->napi_tx);
1115
1116 return IRQ_HANDLED;
1117}
1118
1119static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb,
1120 struct net_device *ndev)
1121{
1122 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1123 struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
1124 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1125 struct device *dev = common->dev;
1126 struct am65_cpsw_tx_chn *tx_chn;
1127 struct netdev_queue *netif_txq;
1128 dma_addr_t desc_dma, buf_dma;
1129 int ret, q_idx, i;
1130 void **swdata;
1131 u32 *psdata;
1132 u32 pkt_len;
1133
1134
1135 pkt_len = skb_headlen(skb);
1136
1137
1138 if (port->tx_ts_enabled)
1139 am65_cpts_prep_tx_timestamp(common->cpts, skb);
1140
1141 q_idx = skb_get_queue_mapping(skb);
1142 dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx);
1143
1144 tx_chn = &common->tx_chns[q_idx];
1145 netif_txq = netdev_get_tx_queue(ndev, q_idx);
1146
1147
1148 buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len,
1149 DMA_TO_DEVICE);
1150 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1151 dev_err(dev, "Failed to map tx skb buffer\n");
1152 ndev->stats.tx_errors++;
1153 goto err_free_skb;
1154 }
1155
1156 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1157 if (!first_desc) {
1158 dev_dbg(dev, "Failed to allocate descriptor\n");
1159 dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len,
1160 DMA_TO_DEVICE);
1161 goto busy_stop_q;
1162 }
1163
1164 cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1165 AM65_CPSW_NAV_PS_DATA_SIZE);
1166 cppi5_desc_set_pktids(&first_desc->hdr, 0, 0x3FFF);
1167 cppi5_hdesc_set_pkttype(first_desc, 0x7);
1168 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, port->port_id);
1169
1170 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1171 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
1172 swdata = cppi5_hdesc_get_swdata(first_desc);
1173 *(swdata) = skb;
1174 psdata = cppi5_hdesc_get_psdata(first_desc);
1175
1176
1177 psdata[2] = 0;
1178 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1179 unsigned int cs_start, cs_offset;
1180
1181 cs_start = skb_transport_offset(skb);
1182 cs_offset = cs_start + skb->csum_offset;
1183
1184 psdata[2] = ((cs_offset + 1) << 24) |
1185 ((cs_start + 1) << 16) | (skb->len - cs_start);
1186 dev_dbg(dev, "%s tx psdata:%#x\n", __func__, psdata[2]);
1187 }
1188
1189 if (!skb_is_nonlinear(skb))
1190 goto done_tx;
1191
1192 dev_dbg(dev, "fragmented SKB\n");
1193
1194
1195 cur_desc = first_desc;
1196 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1197 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1198 u32 frag_size = skb_frag_size(frag);
1199
1200 next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1201 if (!next_desc) {
1202 dev_err(dev, "Failed to allocate descriptor\n");
1203 goto busy_free_descs;
1204 }
1205
1206 buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,
1207 DMA_TO_DEVICE);
1208 if (unlikely(dma_mapping_error(tx_chn->dma_dev, buf_dma))) {
1209 dev_err(dev, "Failed to map tx skb page\n");
1210 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
1211 ndev->stats.tx_errors++;
1212 goto err_free_descs;
1213 }
1214
1215 cppi5_hdesc_reset_hbdesc(next_desc);
1216 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1217 cppi5_hdesc_attach_buf(next_desc,
1218 buf_dma, frag_size, buf_dma, frag_size);
1219
1220 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,
1221 next_desc);
1222 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);
1223 cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
1224
1225 pkt_len += frag_size;
1226 cur_desc = next_desc;
1227 }
1228 WARN_ON(pkt_len != skb->len);
1229
1230done_tx:
1231 skb_tx_timestamp(skb);
1232
1233
1234 netdev_tx_sent_queue(netif_txq, pkt_len);
1235
1236 cppi5_hdesc_set_pktlen(first_desc, pkt_len);
1237 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);
1238 if (AM65_CPSW_IS_CPSW2G(common)) {
1239 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1240 } else {
1241 spin_lock_bh(&tx_chn->lock);
1242 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1243 spin_unlock_bh(&tx_chn->lock);
1244 }
1245 if (ret) {
1246 dev_err(dev, "can't push desc %d\n", ret);
1247
1248 netdev_tx_completed_queue(netif_txq, 1, pkt_len);
1249 ndev->stats.tx_errors++;
1250 goto err_free_descs;
1251 }
1252
1253 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
1254 netif_tx_stop_queue(netif_txq);
1255
1256 smp_mb__after_atomic();
1257 dev_dbg(dev, "netif_tx_stop_queue %d\n", q_idx);
1258
1259
1260 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
1261 MAX_SKB_FRAGS) {
1262 netif_tx_wake_queue(netif_txq);
1263 dev_dbg(dev, "netif_tx_wake_queue %d\n", q_idx);
1264 }
1265 }
1266
1267 return NETDEV_TX_OK;
1268
1269err_free_descs:
1270 am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1271err_free_skb:
1272 ndev->stats.tx_dropped++;
1273 dev_kfree_skb_any(skb);
1274 return NETDEV_TX_OK;
1275
1276busy_free_descs:
1277 am65_cpsw_nuss_xmit_free(tx_chn, first_desc);
1278busy_stop_q:
1279 netif_tx_stop_queue(netif_txq);
1280 return NETDEV_TX_BUSY;
1281}
1282
1283static int am65_cpsw_nuss_ndo_slave_set_mac_address(struct net_device *ndev,
1284 void *addr)
1285{
1286 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1287 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1288 struct sockaddr *sockaddr = (struct sockaddr *)addr;
1289 int ret;
1290
1291 ret = eth_prepare_mac_addr_change(ndev, addr);
1292 if (ret < 0)
1293 return ret;
1294
1295 ret = pm_runtime_get_sync(common->dev);
1296 if (ret < 0) {
1297 pm_runtime_put_noidle(common->dev);
1298 return ret;
1299 }
1300
1301 cpsw_ale_del_ucast(common->ale, ndev->dev_addr,
1302 HOST_PORT_NUM, 0, 0);
1303 cpsw_ale_add_ucast(common->ale, sockaddr->sa_data,
1304 HOST_PORT_NUM, ALE_SECURE, 0);
1305
1306 am65_cpsw_port_set_sl_mac(port, addr);
1307 eth_commit_mac_addr_change(ndev, sockaddr);
1308
1309 pm_runtime_put(common->dev);
1310
1311 return 0;
1312}
1313
1314static int am65_cpsw_nuss_hwtstamp_set(struct net_device *ndev,
1315 struct ifreq *ifr)
1316{
1317 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
1318 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1319 u32 ts_ctrl, seq_id, ts_ctrl_ltype2, ts_vlan_ltype;
1320 struct hwtstamp_config cfg;
1321
1322 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1323 return -EOPNOTSUPP;
1324
1325 if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1326 return -EFAULT;
1327
1328
1329 switch (cfg.tx_type) {
1330 case HWTSTAMP_TX_OFF:
1331 case HWTSTAMP_TX_ON:
1332 break;
1333 default:
1334 return -ERANGE;
1335 }
1336
1337 switch (cfg.rx_filter) {
1338 case HWTSTAMP_FILTER_NONE:
1339 port->rx_ts_enabled = false;
1340 break;
1341 case HWTSTAMP_FILTER_ALL:
1342 case HWTSTAMP_FILTER_SOME:
1343 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1344 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1345 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1346 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1347 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1348 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1349 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1350 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1351 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1352 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1353 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1354 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1355 case HWTSTAMP_FILTER_NTP_ALL:
1356 port->rx_ts_enabled = true;
1357 cfg.rx_filter = HWTSTAMP_FILTER_ALL;
1358 break;
1359 default:
1360 return -ERANGE;
1361 }
1362
1363 port->tx_ts_enabled = (cfg.tx_type == HWTSTAMP_TX_ON);
1364
1365
1366 seq_id = (AM65_CPSW_TS_SEQ_ID_OFFSET <<
1367 AM65_CPSW_PN_TS_SEQ_ID_OFFSET_SHIFT) | ETH_P_1588;
1368
1369 ts_vlan_ltype = ETH_P_8021Q;
1370
1371 ts_ctrl_ltype2 = ETH_P_1588 |
1372 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_107 |
1373 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_129 |
1374 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_130 |
1375 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_131 |
1376 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_132 |
1377 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_319 |
1378 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_320 |
1379 AM65_CPSW_PN_TS_CTL_LTYPE2_TS_TTL_NONZERO;
1380
1381 ts_ctrl = AM65_CPSW_TS_EVENT_MSG_TYPE_BITS <<
1382 AM65_CPSW_PN_TS_CTL_MSG_TYPE_EN_SHIFT;
1383
1384 if (port->tx_ts_enabled)
1385 ts_ctrl |= AM65_CPSW_TS_TX_ANX_ALL_EN |
1386 AM65_CPSW_PN_TS_CTL_TX_VLAN_LT1_EN;
1387
1388 writel(seq_id, port->port_base + AM65_CPSW_PORTN_REG_TS_SEQ_LTYPE_REG);
1389 writel(ts_vlan_ltype, port->port_base +
1390 AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG);
1391 writel(ts_ctrl_ltype2, port->port_base +
1392 AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2);
1393 writel(ts_ctrl, port->port_base + AM65_CPSW_PORTN_REG_TS_CTL);
1394
1395
1396 am65_cpts_rx_enable(common->cpts, port->rx_ts_enabled);
1397
1398 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1399}
1400
1401static int am65_cpsw_nuss_hwtstamp_get(struct net_device *ndev,
1402 struct ifreq *ifr)
1403{
1404 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1405 struct hwtstamp_config cfg;
1406
1407 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1408 return -EOPNOTSUPP;
1409
1410 cfg.flags = 0;
1411 cfg.tx_type = port->tx_ts_enabled ?
1412 HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1413 cfg.rx_filter = port->rx_ts_enabled ?
1414 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1415
1416 return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1417}
1418
1419static int am65_cpsw_nuss_ndo_slave_ioctl(struct net_device *ndev,
1420 struct ifreq *req, int cmd)
1421{
1422 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1423
1424 if (!netif_running(ndev))
1425 return -EINVAL;
1426
1427 switch (cmd) {
1428 case SIOCSHWTSTAMP:
1429 return am65_cpsw_nuss_hwtstamp_set(ndev, req);
1430 case SIOCGHWTSTAMP:
1431 return am65_cpsw_nuss_hwtstamp_get(ndev, req);
1432 }
1433
1434 if (!port->slave.phy)
1435 return -EOPNOTSUPP;
1436
1437 return phy_mii_ioctl(port->slave.phy, req, cmd);
1438}
1439
1440static void am65_cpsw_nuss_ndo_get_stats(struct net_device *dev,
1441 struct rtnl_link_stats64 *stats)
1442{
1443 struct am65_cpsw_ndev_priv *ndev_priv = netdev_priv(dev);
1444 unsigned int start;
1445 int cpu;
1446
1447 for_each_possible_cpu(cpu) {
1448 struct am65_cpsw_ndev_stats *cpu_stats;
1449 u64 rx_packets;
1450 u64 rx_bytes;
1451 u64 tx_packets;
1452 u64 tx_bytes;
1453
1454 cpu_stats = per_cpu_ptr(ndev_priv->stats, cpu);
1455 do {
1456 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
1457 rx_packets = cpu_stats->rx_packets;
1458 rx_bytes = cpu_stats->rx_bytes;
1459 tx_packets = cpu_stats->tx_packets;
1460 tx_bytes = cpu_stats->tx_bytes;
1461 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
1462
1463 stats->rx_packets += rx_packets;
1464 stats->rx_bytes += rx_bytes;
1465 stats->tx_packets += tx_packets;
1466 stats->tx_bytes += tx_bytes;
1467 }
1468
1469 stats->rx_errors = dev->stats.rx_errors;
1470 stats->rx_dropped = dev->stats.rx_dropped;
1471 stats->tx_dropped = dev->stats.tx_dropped;
1472}
1473
1474static struct devlink_port *am65_cpsw_ndo_get_devlink_port(struct net_device *ndev)
1475{
1476 struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
1477
1478 return &port->devlink_port;
1479}
1480
1481static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
1482 .ndo_open = am65_cpsw_nuss_ndo_slave_open,
1483 .ndo_stop = am65_cpsw_nuss_ndo_slave_stop,
1484 .ndo_start_xmit = am65_cpsw_nuss_ndo_slave_xmit,
1485 .ndo_set_rx_mode = am65_cpsw_nuss_ndo_slave_set_rx_mode,
1486 .ndo_get_stats64 = am65_cpsw_nuss_ndo_get_stats,
1487 .ndo_validate_addr = eth_validate_addr,
1488 .ndo_set_mac_address = am65_cpsw_nuss_ndo_slave_set_mac_address,
1489 .ndo_tx_timeout = am65_cpsw_nuss_ndo_host_tx_timeout,
1490 .ndo_vlan_rx_add_vid = am65_cpsw_nuss_ndo_slave_add_vid,
1491 .ndo_vlan_rx_kill_vid = am65_cpsw_nuss_ndo_slave_kill_vid,
1492 .ndo_eth_ioctl = am65_cpsw_nuss_ndo_slave_ioctl,
1493 .ndo_setup_tc = am65_cpsw_qos_ndo_setup_tc,
1494 .ndo_get_devlink_port = am65_cpsw_ndo_get_devlink_port,
1495};
1496
1497static void am65_cpsw_nuss_slave_disable_unused(struct am65_cpsw_port *port)
1498{
1499 struct am65_cpsw_common *common = port->common;
1500
1501 if (!port->disabled)
1502 return;
1503
1504 cpsw_ale_control_set(common->ale, port->port_id,
1505 ALE_PORT_STATE, ALE_PORT_STATE_DISABLE);
1506
1507 cpsw_sl_reset(port->slave.mac_sl, 100);
1508 cpsw_sl_ctl_reset(port->slave.mac_sl);
1509}
1510
1511static void am65_cpsw_nuss_free_tx_chns(void *data)
1512{
1513 struct am65_cpsw_common *common = data;
1514 int i;
1515
1516 for (i = 0; i < common->tx_ch_num; i++) {
1517 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1518
1519 if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
1520 k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
1521
1522 if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
1523 k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
1524
1525 memset(tx_chn, 0, sizeof(*tx_chn));
1526 }
1527}
1528
1529void am65_cpsw_nuss_remove_tx_chns(struct am65_cpsw_common *common)
1530{
1531 struct device *dev = common->dev;
1532 int i;
1533
1534 devm_remove_action(dev, am65_cpsw_nuss_free_tx_chns, common);
1535
1536 for (i = 0; i < common->tx_ch_num; i++) {
1537 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1538
1539 if (tx_chn->irq)
1540 devm_free_irq(dev, tx_chn->irq, tx_chn);
1541
1542 netif_napi_del(&tx_chn->napi_tx);
1543
1544 if (!IS_ERR_OR_NULL(tx_chn->desc_pool))
1545 k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
1546
1547 if (!IS_ERR_OR_NULL(tx_chn->tx_chn))
1548 k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
1549
1550 memset(tx_chn, 0, sizeof(*tx_chn));
1551 }
1552}
1553
1554static int am65_cpsw_nuss_init_tx_chns(struct am65_cpsw_common *common)
1555{
1556 u32 max_desc_num = ALIGN(AM65_CPSW_MAX_TX_DESC, MAX_SKB_FRAGS);
1557 struct k3_udma_glue_tx_channel_cfg tx_cfg = { 0 };
1558 struct device *dev = common->dev;
1559 struct k3_ring_cfg ring_cfg = {
1560 .elm_size = K3_RINGACC_RING_ELSIZE_8,
1561 .mode = K3_RINGACC_RING_MODE_RING,
1562 .flags = 0
1563 };
1564 u32 hdesc_size;
1565 int i, ret = 0;
1566
1567 hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
1568 AM65_CPSW_NAV_SW_DATA_SIZE);
1569
1570 tx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
1571 tx_cfg.tx_cfg = ring_cfg;
1572 tx_cfg.txcq_cfg = ring_cfg;
1573 tx_cfg.tx_cfg.size = max_desc_num;
1574 tx_cfg.txcq_cfg.size = max_desc_num;
1575
1576 for (i = 0; i < common->tx_ch_num; i++) {
1577 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
1578
1579 snprintf(tx_chn->tx_chn_name,
1580 sizeof(tx_chn->tx_chn_name), "tx%d", i);
1581
1582 spin_lock_init(&tx_chn->lock);
1583 tx_chn->common = common;
1584 tx_chn->id = i;
1585 tx_chn->descs_num = max_desc_num;
1586
1587 tx_chn->tx_chn =
1588 k3_udma_glue_request_tx_chn(dev,
1589 tx_chn->tx_chn_name,
1590 &tx_cfg);
1591 if (IS_ERR(tx_chn->tx_chn)) {
1592 ret = dev_err_probe(dev, PTR_ERR(tx_chn->tx_chn),
1593 "Failed to request tx dma channel\n");
1594 goto err;
1595 }
1596 tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);
1597
1598 tx_chn->desc_pool = k3_cppi_desc_pool_create_name(tx_chn->dma_dev,
1599 tx_chn->descs_num,
1600 hdesc_size,
1601 tx_chn->tx_chn_name);
1602 if (IS_ERR(tx_chn->desc_pool)) {
1603 ret = PTR_ERR(tx_chn->desc_pool);
1604 dev_err(dev, "Failed to create poll %d\n", ret);
1605 goto err;
1606 }
1607
1608 tx_chn->irq = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);
1609 if (tx_chn->irq <= 0) {
1610 dev_err(dev, "Failed to get tx dma irq %d\n",
1611 tx_chn->irq);
1612 goto err;
1613 }
1614
1615 snprintf(tx_chn->tx_chn_name,
1616 sizeof(tx_chn->tx_chn_name), "%s-tx%d",
1617 dev_name(dev), tx_chn->id);
1618 }
1619
1620err:
1621 i = devm_add_action(dev, am65_cpsw_nuss_free_tx_chns, common);
1622 if (i) {
1623 dev_err(dev, "Failed to add free_tx_chns action %d\n", i);
1624 return i;
1625 }
1626
1627 return ret;
1628}
1629
1630static void am65_cpsw_nuss_free_rx_chns(void *data)
1631{
1632 struct am65_cpsw_common *common = data;
1633 struct am65_cpsw_rx_chn *rx_chn;
1634
1635 rx_chn = &common->rx_chns;
1636
1637 if (!IS_ERR_OR_NULL(rx_chn->desc_pool))
1638 k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
1639
1640 if (!IS_ERR_OR_NULL(rx_chn->rx_chn))
1641 k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
1642}
1643
1644static int am65_cpsw_nuss_init_rx_chns(struct am65_cpsw_common *common)
1645{
1646 struct am65_cpsw_rx_chn *rx_chn = &common->rx_chns;
1647 struct k3_udma_glue_rx_channel_cfg rx_cfg = { 0 };
1648 u32 max_desc_num = AM65_CPSW_MAX_RX_DESC;
1649 struct device *dev = common->dev;
1650 u32 hdesc_size;
1651 u32 fdqring_id;
1652 int i, ret = 0;
1653
1654 hdesc_size = cppi5_hdesc_calc_size(true, AM65_CPSW_NAV_PS_DATA_SIZE,
1655 AM65_CPSW_NAV_SW_DATA_SIZE);
1656
1657 rx_cfg.swdata_size = AM65_CPSW_NAV_SW_DATA_SIZE;
1658 rx_cfg.flow_id_num = AM65_CPSW_MAX_RX_FLOWS;
1659 rx_cfg.flow_id_base = common->rx_flow_id_base;
1660
1661
1662 rx_chn->dev = dev;
1663 rx_chn->descs_num = max_desc_num;
1664
1665 rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, "rx", &rx_cfg);
1666 if (IS_ERR(rx_chn->rx_chn)) {
1667 ret = dev_err_probe(dev, PTR_ERR(rx_chn->rx_chn),
1668 "Failed to request rx dma channel\n");
1669 goto err;
1670 }
1671 rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);
1672
1673 rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,
1674 rx_chn->descs_num,
1675 hdesc_size, "rx");
1676 if (IS_ERR(rx_chn->desc_pool)) {
1677 ret = PTR_ERR(rx_chn->desc_pool);
1678 dev_err(dev, "Failed to create rx poll %d\n", ret);
1679 goto err;
1680 }
1681
1682 common->rx_flow_id_base =
1683 k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);
1684 dev_info(dev, "set new flow-id-base %u\n", common->rx_flow_id_base);
1685
1686 fdqring_id = K3_RINGACC_RING_ID_ANY;
1687 for (i = 0; i < rx_cfg.flow_id_num; i++) {
1688 struct k3_ring_cfg rxring_cfg = {
1689 .elm_size = K3_RINGACC_RING_ELSIZE_8,
1690 .mode = K3_RINGACC_RING_MODE_RING,
1691 .flags = 0,
1692 };
1693 struct k3_ring_cfg fdqring_cfg = {
1694 .elm_size = K3_RINGACC_RING_ELSIZE_8,
1695 .flags = K3_RINGACC_RING_SHARED,
1696 };
1697 struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = {
1698 .rx_cfg = rxring_cfg,
1699 .rxfdq_cfg = fdqring_cfg,
1700 .ring_rxq_id = K3_RINGACC_RING_ID_ANY,
1701 .src_tag_lo_sel =
1702 K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
1703 };
1704
1705 rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
1706 rx_flow_cfg.rx_cfg.size = max_desc_num;
1707 rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
1708 rx_flow_cfg.rxfdq_cfg.mode = common->pdata.fdqring_mode;
1709
1710 ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,
1711 i, &rx_flow_cfg);
1712 if (ret) {
1713 dev_err(dev, "Failed to init rx flow%d %d\n", i, ret);
1714 goto err;
1715 }
1716 if (!i)
1717 fdqring_id =
1718 k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,
1719 i);
1720
1721 rx_chn->irq = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);
1722
1723 if (rx_chn->irq <= 0) {
1724 dev_err(dev, "Failed to get rx dma irq %d\n",
1725 rx_chn->irq);
1726 ret = -ENXIO;
1727 goto err;
1728 }
1729 }
1730
1731err:
1732 i = devm_add_action(dev, am65_cpsw_nuss_free_rx_chns, common);
1733 if (i) {
1734 dev_err(dev, "Failed to add free_rx_chns action %d\n", i);
1735 return i;
1736 }
1737
1738 return ret;
1739}
1740
1741static int am65_cpsw_nuss_init_host_p(struct am65_cpsw_common *common)
1742{
1743 struct am65_cpsw_host *host_p = am65_common_get_host(common);
1744
1745 host_p->common = common;
1746 host_p->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE;
1747 host_p->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE;
1748
1749 return 0;
1750}
1751
1752static int am65_cpsw_am654_get_efuse_macid(struct device_node *of_node,
1753 int slave, u8 *mac_addr)
1754{
1755 u32 mac_lo, mac_hi, offset;
1756 struct regmap *syscon;
1757 int ret;
1758
1759 syscon = syscon_regmap_lookup_by_phandle(of_node, "ti,syscon-efuse");
1760 if (IS_ERR(syscon)) {
1761 if (PTR_ERR(syscon) == -ENODEV)
1762 return 0;
1763 return PTR_ERR(syscon);
1764 }
1765
1766 ret = of_property_read_u32_index(of_node, "ti,syscon-efuse", 1,
1767 &offset);
1768 if (ret)
1769 return ret;
1770
1771 regmap_read(syscon, offset, &mac_lo);
1772 regmap_read(syscon, offset + 4, &mac_hi);
1773
1774 mac_addr[0] = (mac_hi >> 8) & 0xff;
1775 mac_addr[1] = mac_hi & 0xff;
1776 mac_addr[2] = (mac_lo >> 24) & 0xff;
1777 mac_addr[3] = (mac_lo >> 16) & 0xff;
1778 mac_addr[4] = (mac_lo >> 8) & 0xff;
1779 mac_addr[5] = mac_lo & 0xff;
1780
1781 return 0;
1782}
1783
1784static int am65_cpsw_init_cpts(struct am65_cpsw_common *common)
1785{
1786 struct device *dev = common->dev;
1787 struct device_node *node;
1788 struct am65_cpts *cpts;
1789 void __iomem *reg_base;
1790
1791 if (!IS_ENABLED(CONFIG_TI_K3_AM65_CPTS))
1792 return 0;
1793
1794 node = of_get_child_by_name(dev->of_node, "cpts");
1795 if (!node) {
1796 dev_err(dev, "%s cpts not found\n", __func__);
1797 return -ENOENT;
1798 }
1799
1800 reg_base = common->cpsw_base + AM65_CPSW_NU_CPTS_BASE;
1801 cpts = am65_cpts_create(dev, reg_base, node);
1802 if (IS_ERR(cpts)) {
1803 int ret = PTR_ERR(cpts);
1804
1805 if (ret == -EOPNOTSUPP) {
1806 dev_info(dev, "cpts disabled\n");
1807 return 0;
1808 }
1809
1810 dev_err(dev, "cpts create err %d\n", ret);
1811 return ret;
1812 }
1813 common->cpts = cpts;
1814
1815
1816
1817
1818
1819
1820 pm_runtime_forbid(dev);
1821
1822 return 0;
1823}
1824
1825static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
1826{
1827 struct device_node *node, *port_np;
1828 struct device *dev = common->dev;
1829 int ret;
1830
1831 node = of_get_child_by_name(dev->of_node, "ethernet-ports");
1832 if (!node)
1833 return -ENOENT;
1834
1835 for_each_child_of_node(node, port_np) {
1836 struct am65_cpsw_port *port;
1837 u32 port_id;
1838
1839
1840 if (strcmp(port_np->name, "port"))
1841 continue;
1842
1843 ret = of_property_read_u32(port_np, "reg", &port_id);
1844 if (ret < 0) {
1845 dev_err(dev, "%pOF error reading port_id %d\n",
1846 port_np, ret);
1847 return ret;
1848 }
1849
1850 if (!port_id || port_id > common->port_num) {
1851 dev_err(dev, "%pOF has invalid port_id %u %s\n",
1852 port_np, port_id, port_np->name);
1853 return -EINVAL;
1854 }
1855
1856 port = am65_common_get_port(common, port_id);
1857 port->port_id = port_id;
1858 port->common = common;
1859 port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +
1860 AM65_CPSW_NU_PORTS_OFFSET * (port_id);
1861 port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +
1862 (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id);
1863 port->name = of_get_property(port_np, "label", NULL);
1864 port->fetch_ram_base =
1865 common->cpsw_base + AM65_CPSW_NU_FRAM_BASE +
1866 (AM65_CPSW_NU_FRAM_PORT_OFFSET * (port_id - 1));
1867
1868 port->slave.mac_sl = cpsw_sl_get("am65", dev, port->port_base);
1869 if (IS_ERR(port->slave.mac_sl))
1870 return PTR_ERR(port->slave.mac_sl);
1871
1872 port->disabled = !of_device_is_available(port_np);
1873 if (port->disabled) {
1874 common->disabled_ports_mask |= BIT(port->port_id);
1875 continue;
1876 }
1877
1878 port->slave.ifphy = devm_of_phy_get(dev, port_np, NULL);
1879 if (IS_ERR(port->slave.ifphy)) {
1880 ret = PTR_ERR(port->slave.ifphy);
1881 dev_err(dev, "%pOF error retrieving port phy: %d\n",
1882 port_np, ret);
1883 return ret;
1884 }
1885
1886 port->slave.mac_only =
1887 of_property_read_bool(port_np, "ti,mac-only");
1888
1889
1890 if (of_phy_is_fixed_link(port_np)) {
1891 ret = of_phy_register_fixed_link(port_np);
1892 if (ret)
1893 return dev_err_probe(dev, ret,
1894 "failed to register fixed-link phy %pOF\n",
1895 port_np);
1896 port->slave.phy_node = of_node_get(port_np);
1897 } else {
1898 port->slave.phy_node =
1899 of_parse_phandle(port_np, "phy-handle", 0);
1900 }
1901
1902 if (!port->slave.phy_node) {
1903 dev_err(dev,
1904 "slave[%d] no phy found\n", port_id);
1905 return -ENODEV;
1906 }
1907
1908 ret = of_get_phy_mode(port_np, &port->slave.phy_if);
1909 if (ret) {
1910 dev_err(dev, "%pOF read phy-mode err %d\n",
1911 port_np, ret);
1912 return ret;
1913 }
1914
1915 ret = of_get_mac_address(port_np, port->slave.mac_addr);
1916 if (ret) {
1917 am65_cpsw_am654_get_efuse_macid(port_np,
1918 port->port_id,
1919 port->slave.mac_addr);
1920 if (!is_valid_ether_addr(port->slave.mac_addr)) {
1921 random_ether_addr(port->slave.mac_addr);
1922 dev_err(dev, "Use random MAC address\n");
1923 }
1924 }
1925 }
1926 of_node_put(node);
1927
1928
1929 if (!(~common->disabled_ports_mask & GENMASK(common->port_num, 1))) {
1930 dev_err(dev, "No Ext. port are available\n");
1931 return -ENODEV;
1932 }
1933
1934 return 0;
1935}
1936
1937static void am65_cpsw_pcpu_stats_free(void *data)
1938{
1939 struct am65_cpsw_ndev_stats __percpu *stats = data;
1940
1941 free_percpu(stats);
1942}
1943
1944static int
1945am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
1946{
1947 struct am65_cpsw_ndev_priv *ndev_priv;
1948 struct device *dev = common->dev;
1949 struct am65_cpsw_port *port;
1950 int ret;
1951
1952 port = &common->ports[port_idx];
1953
1954 if (port->disabled)
1955 return 0;
1956
1957
1958 port->ndev = devm_alloc_etherdev_mqs(common->dev,
1959 sizeof(struct am65_cpsw_ndev_priv),
1960 AM65_CPSW_MAX_TX_QUEUES,
1961 AM65_CPSW_MAX_RX_QUEUES);
1962 if (!port->ndev) {
1963 dev_err(dev, "error allocating slave net_device %u\n",
1964 port->port_id);
1965 return -ENOMEM;
1966 }
1967
1968 ndev_priv = netdev_priv(port->ndev);
1969 ndev_priv->port = port;
1970 ndev_priv->msg_enable = AM65_CPSW_DEBUG;
1971 SET_NETDEV_DEV(port->ndev, dev);
1972
1973 ether_addr_copy(port->ndev->dev_addr, port->slave.mac_addr);
1974
1975 port->ndev->min_mtu = AM65_CPSW_MIN_PACKET_SIZE;
1976 port->ndev->max_mtu = AM65_CPSW_MAX_PACKET_SIZE;
1977 port->ndev->hw_features = NETIF_F_SG |
1978 NETIF_F_RXCSUM |
1979 NETIF_F_HW_CSUM |
1980 NETIF_F_HW_TC;
1981 port->ndev->features = port->ndev->hw_features |
1982 NETIF_F_HW_VLAN_CTAG_FILTER;
1983 port->ndev->vlan_features |= NETIF_F_SG;
1984 port->ndev->netdev_ops = &am65_cpsw_nuss_netdev_ops;
1985 port->ndev->ethtool_ops = &am65_cpsw_ethtool_ops_slave;
1986
1987
1988 if (common->pdata.quirks & AM65_CPSW_QUIRK_I2027_NO_TX_CSUM)
1989 port->ndev->features &= ~NETIF_F_HW_CSUM;
1990
1991 ndev_priv->stats = netdev_alloc_pcpu_stats(struct am65_cpsw_ndev_stats);
1992 if (!ndev_priv->stats)
1993 return -ENOMEM;
1994
1995 ret = devm_add_action_or_reset(dev, am65_cpsw_pcpu_stats_free,
1996 ndev_priv->stats);
1997 if (ret)
1998 dev_err(dev, "failed to add percpu stat free action %d\n", ret);
1999
2000 if (!common->dma_ndev)
2001 common->dma_ndev = port->ndev;
2002
2003 return ret;
2004}
2005
2006static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common)
2007{
2008 int ret;
2009 int i;
2010
2011 for (i = 0; i < common->port_num; i++) {
2012 ret = am65_cpsw_nuss_init_port_ndev(common, i);
2013 if (ret)
2014 return ret;
2015 }
2016
2017 netif_napi_add(common->dma_ndev, &common->napi_rx,
2018 am65_cpsw_nuss_rx_poll, NAPI_POLL_WEIGHT);
2019
2020 return ret;
2021}
2022
2023static int am65_cpsw_nuss_ndev_add_tx_napi(struct am65_cpsw_common *common)
2024{
2025 struct device *dev = common->dev;
2026 int i, ret = 0;
2027
2028 for (i = 0; i < common->tx_ch_num; i++) {
2029 struct am65_cpsw_tx_chn *tx_chn = &common->tx_chns[i];
2030
2031 netif_tx_napi_add(common->dma_ndev, &tx_chn->napi_tx,
2032 am65_cpsw_nuss_tx_poll, NAPI_POLL_WEIGHT);
2033
2034 ret = devm_request_irq(dev, tx_chn->irq,
2035 am65_cpsw_nuss_tx_irq,
2036 IRQF_TRIGGER_HIGH,
2037 tx_chn->tx_chn_name, tx_chn);
2038 if (ret) {
2039 dev_err(dev, "failure requesting tx%u irq %u, %d\n",
2040 tx_chn->id, tx_chn->irq, ret);
2041 goto err;
2042 }
2043 }
2044
2045err:
2046 return ret;
2047}
2048
2049static void am65_cpsw_nuss_cleanup_ndev(struct am65_cpsw_common *common)
2050{
2051 struct am65_cpsw_port *port;
2052 int i;
2053
2054 for (i = 0; i < common->port_num; i++) {
2055 port = &common->ports[i];
2056 if (port->ndev)
2057 unregister_netdev(port->ndev);
2058 }
2059}
2060
2061static void am65_cpsw_port_offload_fwd_mark_update(struct am65_cpsw_common *common)
2062{
2063 int set_val = 0;
2064 int i;
2065
2066 if (common->br_members == (GENMASK(common->port_num, 1) & ~common->disabled_ports_mask))
2067 set_val = 1;
2068
2069 dev_dbg(common->dev, "set offload_fwd_mark %d\n", set_val);
2070
2071 for (i = 1; i <= common->port_num; i++) {
2072 struct am65_cpsw_port *port = am65_common_get_port(common, i);
2073 struct am65_cpsw_ndev_priv *priv;
2074
2075 if (!port->ndev)
2076 continue;
2077
2078 priv = am65_ndev_to_priv(port->ndev);
2079 priv->offload_fwd_mark = set_val;
2080 }
2081}
2082
2083bool am65_cpsw_port_dev_check(const struct net_device *ndev)
2084{
2085 if (ndev->netdev_ops == &am65_cpsw_nuss_netdev_ops) {
2086 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2087
2088 return !common->is_emac_mode;
2089 }
2090
2091 return false;
2092}
2093
2094static int am65_cpsw_netdevice_port_link(struct net_device *ndev,
2095 struct net_device *br_ndev,
2096 struct netlink_ext_ack *extack)
2097{
2098 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2099 struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2100 int err;
2101
2102 if (!common->br_members) {
2103 common->hw_bridge_dev = br_ndev;
2104 } else {
2105
2106
2107
2108 if (common->hw_bridge_dev != br_ndev)
2109 return -EOPNOTSUPP;
2110 }
2111
2112 err = switchdev_bridge_port_offload(ndev, ndev, NULL, NULL, NULL,
2113 false, extack);
2114 if (err)
2115 return err;
2116
2117 common->br_members |= BIT(priv->port->port_id);
2118
2119 am65_cpsw_port_offload_fwd_mark_update(common);
2120
2121 return NOTIFY_DONE;
2122}
2123
2124static void am65_cpsw_netdevice_port_unlink(struct net_device *ndev)
2125{
2126 struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
2127 struct am65_cpsw_ndev_priv *priv = am65_ndev_to_priv(ndev);
2128
2129 switchdev_bridge_port_unoffload(ndev, NULL, NULL, NULL);
2130
2131 common->br_members &= ~BIT(priv->port->port_id);
2132
2133 am65_cpsw_port_offload_fwd_mark_update(common);
2134
2135 if (!common->br_members)
2136 common->hw_bridge_dev = NULL;
2137}
2138
2139
2140static int am65_cpsw_netdevice_event(struct notifier_block *unused,
2141 unsigned long event, void *ptr)
2142{
2143 struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(ptr);
2144 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
2145 struct netdev_notifier_changeupper_info *info;
2146 int ret = NOTIFY_DONE;
2147
2148 if (!am65_cpsw_port_dev_check(ndev))
2149 return NOTIFY_DONE;
2150
2151 switch (event) {
2152 case NETDEV_CHANGEUPPER:
2153 info = ptr;
2154
2155 if (netif_is_bridge_master(info->upper_dev)) {
2156 if (info->linking)
2157 ret = am65_cpsw_netdevice_port_link(ndev,
2158 info->upper_dev,
2159 extack);
2160 else
2161 am65_cpsw_netdevice_port_unlink(ndev);
2162 }
2163 break;
2164 default:
2165 return NOTIFY_DONE;
2166 }
2167
2168 return notifier_from_errno(ret);
2169}
2170
2171static int am65_cpsw_register_notifiers(struct am65_cpsw_common *cpsw)
2172{
2173 int ret = 0;
2174
2175 if (AM65_CPSW_IS_CPSW2G(cpsw) ||
2176 !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2177 return 0;
2178
2179 cpsw->am65_cpsw_netdevice_nb.notifier_call = &am65_cpsw_netdevice_event;
2180 ret = register_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2181 if (ret) {
2182 dev_err(cpsw->dev, "can't register netdevice notifier\n");
2183 return ret;
2184 }
2185
2186 ret = am65_cpsw_switchdev_register_notifiers(cpsw);
2187 if (ret)
2188 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2189
2190 return ret;
2191}
2192
2193static void am65_cpsw_unregister_notifiers(struct am65_cpsw_common *cpsw)
2194{
2195 if (AM65_CPSW_IS_CPSW2G(cpsw) ||
2196 !IS_REACHABLE(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV))
2197 return;
2198
2199 am65_cpsw_switchdev_unregister_notifiers(cpsw);
2200 unregister_netdevice_notifier(&cpsw->am65_cpsw_netdevice_nb);
2201}
2202
2203static const struct devlink_ops am65_cpsw_devlink_ops = {};
2204
2205static void am65_cpsw_init_stp_ale_entry(struct am65_cpsw_common *cpsw)
2206{
2207 cpsw_ale_add_mcast(cpsw->ale, eth_stp_addr, ALE_PORT_HOST, ALE_SUPER, 0,
2208 ALE_MCAST_BLOCK_LEARN_FWD);
2209}
2210
2211static void am65_cpsw_init_host_port_switch(struct am65_cpsw_common *common)
2212{
2213 struct am65_cpsw_host *host = am65_common_get_host(common);
2214
2215 writel(common->default_vlan, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2216
2217 am65_cpsw_init_stp_ale_entry(common);
2218
2219 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 1);
2220 dev_dbg(common->dev, "Set P0_UNI_FLOOD\n");
2221 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 0);
2222}
2223
2224static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common)
2225{
2226 struct am65_cpsw_host *host = am65_common_get_host(common);
2227
2228 writel(0, host->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2229
2230 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_P0_UNI_FLOOD, 0);
2231 dev_dbg(common->dev, "unset P0_UNI_FLOOD\n");
2232
2233
2234 cpsw_ale_control_set(common->ale, HOST_PORT_NUM, ALE_PORT_NOLEARN, 1);
2235}
2236
2237static int am65_cpsw_dl_switch_mode_get(struct devlink *dl, u32 id,
2238 struct devlink_param_gset_ctx *ctx)
2239{
2240 struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
2241 struct am65_cpsw_common *common = dl_priv->common;
2242
2243 dev_dbg(common->dev, "%s id:%u\n", __func__, id);
2244
2245 if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
2246 return -EOPNOTSUPP;
2247
2248 ctx->val.vbool = !common->is_emac_mode;
2249
2250 return 0;
2251}
2252
2253static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port)
2254{
2255 struct am65_cpsw_slave_data *slave = &port->slave;
2256 struct am65_cpsw_common *common = port->common;
2257 u32 port_mask;
2258
2259 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2260
2261 if (slave->mac_only)
2262
2263 cpsw_ale_control_set(common->ale, port->port_id,
2264 ALE_PORT_MACONLY, 1);
2265
2266 cpsw_ale_control_set(common->ale, port->port_id, ALE_PORT_NOLEARN, 1);
2267
2268 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
2269
2270 cpsw_ale_add_ucast(common->ale, port->ndev->dev_addr,
2271 HOST_PORT_NUM, ALE_SECURE, slave->port_vlan);
2272 cpsw_ale_add_mcast(common->ale, port->ndev->broadcast,
2273 port_mask, ALE_VLAN, slave->port_vlan, ALE_MCAST_FWD_2);
2274}
2275
2276static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port)
2277{
2278 struct am65_cpsw_slave_data *slave = &port->slave;
2279 struct am65_cpsw_common *cpsw = port->common;
2280 u32 port_mask;
2281
2282 cpsw_ale_control_set(cpsw->ale, port->port_id,
2283 ALE_PORT_NOLEARN, 0);
2284
2285 cpsw_ale_add_ucast(cpsw->ale, port->ndev->dev_addr,
2286 HOST_PORT_NUM, ALE_SECURE | ALE_BLOCKED | ALE_VLAN,
2287 slave->port_vlan);
2288
2289 port_mask = BIT(port->port_id) | ALE_PORT_HOST;
2290
2291 cpsw_ale_add_mcast(cpsw->ale, port->ndev->broadcast,
2292 port_mask, ALE_VLAN, slave->port_vlan,
2293 ALE_MCAST_FWD_2);
2294
2295 writel(slave->port_vlan, port->port_base + AM65_CPSW_PORT_VLAN_REG_OFFSET);
2296
2297 cpsw_ale_control_set(cpsw->ale, port->port_id,
2298 ALE_PORT_MACONLY, 0);
2299}
2300
2301static int am65_cpsw_dl_switch_mode_set(struct devlink *dl, u32 id,
2302 struct devlink_param_gset_ctx *ctx)
2303{
2304 struct am65_cpsw_devlink *dl_priv = devlink_priv(dl);
2305 struct am65_cpsw_common *cpsw = dl_priv->common;
2306 bool switch_en = ctx->val.vbool;
2307 bool if_running = false;
2308 int i;
2309
2310 dev_dbg(cpsw->dev, "%s id:%u\n", __func__, id);
2311
2312 if (id != AM65_CPSW_DL_PARAM_SWITCH_MODE)
2313 return -EOPNOTSUPP;
2314
2315 if (switch_en == !cpsw->is_emac_mode)
2316 return 0;
2317
2318 if (!switch_en && cpsw->br_members) {
2319 dev_err(cpsw->dev, "Remove ports from bridge before disabling switch mode\n");
2320 return -EINVAL;
2321 }
2322
2323 rtnl_lock();
2324
2325 cpsw->is_emac_mode = !switch_en;
2326
2327 for (i = 0; i < cpsw->port_num; i++) {
2328 struct net_device *sl_ndev = cpsw->ports[i].ndev;
2329
2330 if (!sl_ndev || !netif_running(sl_ndev))
2331 continue;
2332
2333 if_running = true;
2334 }
2335
2336 if (!if_running) {
2337
2338 for (i = 0; i < cpsw->port_num; i++) {
2339 struct net_device *sl_ndev = cpsw->ports[i].ndev;
2340 struct am65_cpsw_slave_data *slave;
2341
2342 if (!sl_ndev)
2343 continue;
2344
2345 slave = am65_ndev_to_slave(sl_ndev);
2346 if (switch_en)
2347 slave->port_vlan = cpsw->default_vlan;
2348 else
2349 slave->port_vlan = 0;
2350 }
2351
2352 goto exit;
2353 }
2354
2355 cpsw_ale_control_set(cpsw->ale, 0, ALE_BYPASS, 1);
2356
2357 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_CLEAR, 1);
2358 cpsw_ale_control_get(cpsw->ale, HOST_PORT_NUM, ALE_AGEOUT);
2359
2360 if (switch_en) {
2361 dev_info(cpsw->dev, "Enable switch mode\n");
2362
2363 am65_cpsw_init_host_port_switch(cpsw);
2364
2365 for (i = 0; i < cpsw->port_num; i++) {
2366 struct net_device *sl_ndev = cpsw->ports[i].ndev;
2367 struct am65_cpsw_slave_data *slave;
2368 struct am65_cpsw_port *port;
2369
2370 if (!sl_ndev)
2371 continue;
2372
2373 port = am65_ndev_to_port(sl_ndev);
2374 slave = am65_ndev_to_slave(sl_ndev);
2375 slave->port_vlan = cpsw->default_vlan;
2376
2377 if (netif_running(sl_ndev))
2378 am65_cpsw_init_port_switch_ale(port);
2379 }
2380
2381 } else {
2382 dev_info(cpsw->dev, "Disable switch mode\n");
2383
2384 am65_cpsw_init_host_port_emac(cpsw);
2385
2386 for (i = 0; i < cpsw->port_num; i++) {
2387 struct net_device *sl_ndev = cpsw->ports[i].ndev;
2388 struct am65_cpsw_port *port;
2389
2390 if (!sl_ndev)
2391 continue;
2392
2393 port = am65_ndev_to_port(sl_ndev);
2394 port->slave.port_vlan = 0;
2395 if (netif_running(sl_ndev))
2396 am65_cpsw_init_port_emac_ale(port);
2397 }
2398 }
2399 cpsw_ale_control_set(cpsw->ale, HOST_PORT_NUM, ALE_BYPASS, 0);
2400exit:
2401 rtnl_unlock();
2402
2403 return 0;
2404}
2405
2406static const struct devlink_param am65_cpsw_devlink_params[] = {
2407 DEVLINK_PARAM_DRIVER(AM65_CPSW_DL_PARAM_SWITCH_MODE, "switch_mode",
2408 DEVLINK_PARAM_TYPE_BOOL,
2409 BIT(DEVLINK_PARAM_CMODE_RUNTIME),
2410 am65_cpsw_dl_switch_mode_get,
2411 am65_cpsw_dl_switch_mode_set, NULL),
2412};
2413
2414static int am65_cpsw_nuss_register_devlink(struct am65_cpsw_common *common)
2415{
2416 struct devlink_port_attrs attrs = {};
2417 struct am65_cpsw_devlink *dl_priv;
2418 struct device *dev = common->dev;
2419 struct devlink_port *dl_port;
2420 struct am65_cpsw_port *port;
2421 int ret = 0;
2422 int i;
2423
2424 common->devlink =
2425 devlink_alloc(&am65_cpsw_devlink_ops, sizeof(*dl_priv), dev);
2426 if (!common->devlink)
2427 return -ENOMEM;
2428
2429 dl_priv = devlink_priv(common->devlink);
2430 dl_priv->common = common;
2431
2432 ret = devlink_register(common->devlink);
2433 if (ret) {
2434 dev_err(dev, "devlink reg fail ret:%d\n", ret);
2435 goto dl_free;
2436 }
2437
2438
2439
2440
2441 if (!AM65_CPSW_IS_CPSW2G(common) &&
2442 IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) {
2443 ret = devlink_params_register(common->devlink,
2444 am65_cpsw_devlink_params,
2445 ARRAY_SIZE(am65_cpsw_devlink_params));
2446 if (ret) {
2447 dev_err(dev, "devlink params reg fail ret:%d\n", ret);
2448 goto dl_unreg;
2449 }
2450 devlink_params_publish(common->devlink);
2451 }
2452
2453 for (i = 1; i <= common->port_num; i++) {
2454 port = am65_common_get_port(common, i);
2455 dl_port = &port->devlink_port;
2456
2457 attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
2458 attrs.phys.port_number = port->port_id;
2459 attrs.switch_id.id_len = sizeof(resource_size_t);
2460 memcpy(attrs.switch_id.id, common->switch_id, attrs.switch_id.id_len);
2461 devlink_port_attrs_set(dl_port, &attrs);
2462
2463 ret = devlink_port_register(common->devlink, dl_port, port->port_id);
2464 if (ret) {
2465 dev_err(dev, "devlink_port reg fail for port %d, ret:%d\n",
2466 port->port_id, ret);
2467 goto dl_port_unreg;
2468 }
2469 devlink_port_type_eth_set(dl_port, port->ndev);
2470 }
2471
2472 return ret;
2473
2474dl_port_unreg:
2475 for (i = i - 1; i >= 1; i--) {
2476 port = am65_common_get_port(common, i);
2477 dl_port = &port->devlink_port;
2478
2479 devlink_port_unregister(dl_port);
2480 }
2481dl_unreg:
2482 devlink_unregister(common->devlink);
2483dl_free:
2484 devlink_free(common->devlink);
2485
2486 return ret;
2487}
2488
2489static void am65_cpsw_unregister_devlink(struct am65_cpsw_common *common)
2490{
2491 struct devlink_port *dl_port;
2492 struct am65_cpsw_port *port;
2493 int i;
2494
2495 for (i = 1; i <= common->port_num; i++) {
2496 port = am65_common_get_port(common, i);
2497 dl_port = &port->devlink_port;
2498
2499 devlink_port_unregister(dl_port);
2500 }
2501
2502 if (!AM65_CPSW_IS_CPSW2G(common) &&
2503 IS_ENABLED(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV)) {
2504 devlink_params_unpublish(common->devlink);
2505 devlink_params_unregister(common->devlink, am65_cpsw_devlink_params,
2506 ARRAY_SIZE(am65_cpsw_devlink_params));
2507 }
2508
2509 devlink_unregister(common->devlink);
2510 devlink_free(common->devlink);
2511}
2512
2513static int am65_cpsw_nuss_register_ndevs(struct am65_cpsw_common *common)
2514{
2515 struct device *dev = common->dev;
2516 struct am65_cpsw_port *port;
2517 int ret = 0, i;
2518
2519 ret = am65_cpsw_nuss_ndev_add_tx_napi(common);
2520 if (ret)
2521 return ret;
2522
2523 ret = devm_request_irq(dev, common->rx_chns.irq,
2524 am65_cpsw_nuss_rx_irq,
2525 IRQF_TRIGGER_HIGH, dev_name(dev), common);
2526 if (ret) {
2527 dev_err(dev, "failure requesting rx irq %u, %d\n",
2528 common->rx_chns.irq, ret);
2529 return ret;
2530 }
2531
2532 for (i = 0; i < common->port_num; i++) {
2533 port = &common->ports[i];
2534
2535 if (!port->ndev)
2536 continue;
2537
2538 ret = register_netdev(port->ndev);
2539 if (ret) {
2540 dev_err(dev, "error registering slave net device%i %d\n",
2541 i, ret);
2542 goto err_cleanup_ndev;
2543 }
2544 }
2545
2546 ret = am65_cpsw_register_notifiers(common);
2547 if (ret)
2548 goto err_cleanup_ndev;
2549
2550 ret = am65_cpsw_nuss_register_devlink(common);
2551 if (ret)
2552 goto clean_unregister_notifiers;
2553
2554
2555
2556
2557
2558 return 0;
2559clean_unregister_notifiers:
2560 am65_cpsw_unregister_notifiers(common);
2561err_cleanup_ndev:
2562 am65_cpsw_nuss_cleanup_ndev(common);
2563
2564 return ret;
2565}
2566
2567int am65_cpsw_nuss_update_tx_chns(struct am65_cpsw_common *common, int num_tx)
2568{
2569 int ret;
2570
2571 common->tx_ch_num = num_tx;
2572 ret = am65_cpsw_nuss_init_tx_chns(common);
2573 if (ret)
2574 return ret;
2575
2576 return am65_cpsw_nuss_ndev_add_tx_napi(common);
2577}
2578
2579struct am65_cpsw_soc_pdata {
2580 u32 quirks_dis;
2581};
2582
2583static const struct am65_cpsw_soc_pdata am65x_soc_sr2_0 = {
2584 .quirks_dis = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
2585};
2586
2587static const struct soc_device_attribute am65_cpsw_socinfo[] = {
2588 { .family = "AM65X",
2589 .revision = "SR2.0",
2590 .data = &am65x_soc_sr2_0
2591 },
2592 {}
2593};
2594
2595static const struct am65_cpsw_pdata am65x_sr1_0 = {
2596 .quirks = AM65_CPSW_QUIRK_I2027_NO_TX_CSUM,
2597 .ale_dev_id = "am65x-cpsw2g",
2598 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2599};
2600
2601static const struct am65_cpsw_pdata j721e_pdata = {
2602 .quirks = 0,
2603 .ale_dev_id = "am65x-cpsw2g",
2604 .fdqring_mode = K3_RINGACC_RING_MODE_MESSAGE,
2605};
2606
2607static const struct am65_cpsw_pdata am64x_cpswxg_pdata = {
2608 .quirks = 0,
2609 .ale_dev_id = "am64-cpswxg",
2610 .fdqring_mode = K3_RINGACC_RING_MODE_RING,
2611};
2612
2613static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
2614 { .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
2615 { .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
2616 { .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
2617 { },
2618};
2619MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
2620
2621static void am65_cpsw_nuss_apply_socinfo(struct am65_cpsw_common *common)
2622{
2623 const struct soc_device_attribute *soc;
2624
2625 soc = soc_device_match(am65_cpsw_socinfo);
2626 if (soc && soc->data) {
2627 const struct am65_cpsw_soc_pdata *socdata = soc->data;
2628
2629
2630 common->pdata.quirks &= ~socdata->quirks_dis;
2631 }
2632}
2633
2634static int am65_cpsw_nuss_probe(struct platform_device *pdev)
2635{
2636 struct cpsw_ale_params ale_params = { 0 };
2637 const struct of_device_id *of_id;
2638 struct device *dev = &pdev->dev;
2639 struct am65_cpsw_common *common;
2640 struct device_node *node;
2641 struct resource *res;
2642 struct clk *clk;
2643 u64 id_temp;
2644 int ret, i;
2645
2646 common = devm_kzalloc(dev, sizeof(struct am65_cpsw_common), GFP_KERNEL);
2647 if (!common)
2648 return -ENOMEM;
2649 common->dev = dev;
2650
2651 of_id = of_match_device(am65_cpsw_nuss_of_mtable, dev);
2652 if (!of_id)
2653 return -EINVAL;
2654 common->pdata = *(const struct am65_cpsw_pdata *)of_id->data;
2655
2656 am65_cpsw_nuss_apply_socinfo(common);
2657
2658 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cpsw_nuss");
2659 common->ss_base = devm_ioremap_resource(&pdev->dev, res);
2660 if (IS_ERR(common->ss_base))
2661 return PTR_ERR(common->ss_base);
2662 common->cpsw_base = common->ss_base + AM65_CPSW_CPSW_NU_BASE;
2663
2664 id_temp = cpu_to_be64(res->start);
2665 memcpy(common->switch_id, &id_temp, sizeof(res->start));
2666
2667 node = of_get_child_by_name(dev->of_node, "ethernet-ports");
2668 if (!node)
2669 return -ENOENT;
2670 common->port_num = of_get_child_count(node);
2671 if (common->port_num < 1 || common->port_num > AM65_CPSW_MAX_PORTS)
2672 return -ENOENT;
2673 of_node_put(node);
2674
2675 common->rx_flow_id_base = -1;
2676 init_completion(&common->tdown_complete);
2677 common->tx_ch_num = 1;
2678 common->pf_p0_rx_ptype_rrobin = false;
2679 common->default_vlan = 1;
2680
2681 common->ports = devm_kcalloc(dev, common->port_num,
2682 sizeof(*common->ports),
2683 GFP_KERNEL);
2684 if (!common->ports)
2685 return -ENOMEM;
2686
2687 clk = devm_clk_get(dev, "fck");
2688 if (IS_ERR(clk))
2689 return dev_err_probe(dev, PTR_ERR(clk), "getting fck clock\n");
2690 common->bus_freq = clk_get_rate(clk);
2691
2692 pm_runtime_enable(dev);
2693 ret = pm_runtime_get_sync(dev);
2694 if (ret < 0) {
2695 pm_runtime_put_noidle(dev);
2696 pm_runtime_disable(dev);
2697 return ret;
2698 }
2699
2700 node = of_get_child_by_name(dev->of_node, "mdio");
2701 if (!node) {
2702 dev_warn(dev, "MDIO node not found\n");
2703 } else if (of_device_is_available(node)) {
2704 struct platform_device *mdio_pdev;
2705
2706 mdio_pdev = of_platform_device_create(node, NULL, dev);
2707 if (!mdio_pdev) {
2708 ret = -ENODEV;
2709 goto err_pm_clear;
2710 }
2711
2712 common->mdio_dev = &mdio_pdev->dev;
2713 }
2714 of_node_put(node);
2715
2716 am65_cpsw_nuss_get_ver(common);
2717
2718
2719 ret = am65_cpsw_nuss_init_tx_chns(common);
2720 if (ret)
2721 goto err_of_clear;
2722 ret = am65_cpsw_nuss_init_rx_chns(common);
2723 if (ret)
2724 goto err_of_clear;
2725
2726 ret = am65_cpsw_nuss_init_host_p(common);
2727 if (ret)
2728 goto err_of_clear;
2729
2730 ret = am65_cpsw_nuss_init_slave_ports(common);
2731 if (ret)
2732 goto err_of_clear;
2733
2734
2735 ale_params.dev = dev;
2736 ale_params.ale_ageout = AM65_CPSW_ALE_AGEOUT_DEFAULT;
2737 ale_params.ale_ports = common->port_num + 1;
2738 ale_params.ale_regs = common->cpsw_base + AM65_CPSW_NU_ALE_BASE;
2739 ale_params.dev_id = common->pdata.ale_dev_id;
2740 ale_params.bus_freq = common->bus_freq;
2741
2742 common->ale = cpsw_ale_create(&ale_params);
2743 if (IS_ERR(common->ale)) {
2744 dev_err(dev, "error initializing ale engine\n");
2745 ret = PTR_ERR(common->ale);
2746 goto err_of_clear;
2747 }
2748
2749 ret = am65_cpsw_init_cpts(common);
2750 if (ret)
2751 goto err_of_clear;
2752
2753
2754 for (i = 0; i < common->port_num; i++)
2755 am65_cpsw_nuss_slave_disable_unused(&common->ports[i]);
2756
2757 dev_set_drvdata(dev, common);
2758
2759 common->is_emac_mode = true;
2760
2761 ret = am65_cpsw_nuss_init_ndevs(common);
2762 if (ret)
2763 goto err_of_clear;
2764
2765 ret = am65_cpsw_nuss_register_ndevs(common);
2766 if (ret)
2767 goto err_of_clear;
2768
2769 pm_runtime_put(dev);
2770 return 0;
2771
2772err_of_clear:
2773 of_platform_device_destroy(common->mdio_dev, NULL);
2774err_pm_clear:
2775 pm_runtime_put_sync(dev);
2776 pm_runtime_disable(dev);
2777 return ret;
2778}
2779
2780static int am65_cpsw_nuss_remove(struct platform_device *pdev)
2781{
2782 struct device *dev = &pdev->dev;
2783 struct am65_cpsw_common *common;
2784 int ret;
2785
2786 common = dev_get_drvdata(dev);
2787
2788 ret = pm_runtime_get_sync(&pdev->dev);
2789 if (ret < 0) {
2790 pm_runtime_put_noidle(&pdev->dev);
2791 return ret;
2792 }
2793
2794 am65_cpsw_unregister_devlink(common);
2795 am65_cpsw_unregister_notifiers(common);
2796
2797
2798
2799
2800 am65_cpsw_nuss_cleanup_ndev(common);
2801
2802 of_platform_device_destroy(common->mdio_dev, NULL);
2803
2804 pm_runtime_put_sync(&pdev->dev);
2805 pm_runtime_disable(&pdev->dev);
2806 return 0;
2807}
2808
2809static struct platform_driver am65_cpsw_nuss_driver = {
2810 .driver = {
2811 .name = AM65_CPSW_DRV_NAME,
2812 .of_match_table = am65_cpsw_nuss_of_mtable,
2813 },
2814 .probe = am65_cpsw_nuss_probe,
2815 .remove = am65_cpsw_nuss_remove,
2816};
2817
2818module_platform_driver(am65_cpsw_nuss_driver);
2819
2820MODULE_LICENSE("GPL v2");
2821MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
2822MODULE_DESCRIPTION("TI AM65 CPSW Ethernet driver");
2823
