1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/clk.h>
18#include <linux/kernel.h>
19#include <linux/interrupt.h>
20#include <linux/ip.h>
21#include <linux/tcp.h>
22#include <linux/skbuff.h>
23#include <linux/ethtool.h>
24#include <linux/if_ether.h>
25#include <linux/crc32.h>
26#include <linux/mii.h>
27#include <linux/if.h>
28#include <linux/if_vlan.h>
29#include <linux/dma-mapping.h>
30#include <linux/slab.h>
31#include <linux/pm_runtime.h>
32#include <linux/prefetch.h>
33#include <linux/pinctrl/consumer.h>
34#ifdef CONFIG_DEBUG_FS
35#include <linux/debugfs.h>
36#include <linux/seq_file.h>
37#endif
38#include <linux/net_tstamp.h>
39#include <linux/phylink.h>
40#include <linux/udp.h>
41#include <linux/bpf_trace.h>
42#include <net/pkt_cls.h>
43#include <net/xdp_sock_drv.h>
44#include "stmmac_ptp.h"
45#include "stmmac.h"
46#include "stmmac_xdp.h"
47#include <linux/reset.h>
48#include <linux/of_mdio.h>
49#include "dwmac1000.h"
50#include "dwxgmac2.h"
51#include "hwif.h"
52
53#define STMMAC_ALIGN(x) ALIGN(ALIGN(x, SMP_CACHE_BYTES), 16)
54#define TSO_MAX_BUFF_SIZE (SZ_16K - 1)
55
56
57#define TX_TIMEO 5000
58static int watchdog = TX_TIMEO;
59module_param(watchdog, int, 0644);
60MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
61
62static int debug = -1;
63module_param(debug, int, 0644);
64MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
65
66static int phyaddr = -1;
67module_param(phyaddr, int, 0444);
68MODULE_PARM_DESC(phyaddr, "Physical device address");
69
70#define STMMAC_TX_THRESH(x) ((x)->dma_tx_size / 4)
71#define STMMAC_RX_THRESH(x) ((x)->dma_rx_size / 4)
72
73
74#define STMMAC_XSK_TX_BUDGET_MAX 256
75#define STMMAC_TX_XSK_AVAIL 16
76#define STMMAC_RX_FILL_BATCH 16
77
78#define STMMAC_XDP_PASS 0
79#define STMMAC_XDP_CONSUMED BIT(0)
80#define STMMAC_XDP_TX BIT(1)
81#define STMMAC_XDP_REDIRECT BIT(2)
82
83static int flow_ctrl = FLOW_AUTO;
84module_param(flow_ctrl, int, 0644);
85MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
86
87static int pause = PAUSE_TIME;
88module_param(pause, int, 0644);
89MODULE_PARM_DESC(pause, "Flow Control Pause Time");
90
91#define TC_DEFAULT 64
92static int tc = TC_DEFAULT;
93module_param(tc, int, 0644);
94MODULE_PARM_DESC(tc, "DMA threshold control value");
95
96#define DEFAULT_BUFSIZE 1536
97static int buf_sz = DEFAULT_BUFSIZE;
98module_param(buf_sz, int, 0644);
99MODULE_PARM_DESC(buf_sz, "DMA buffer size");
100
101#define STMMAC_RX_COPYBREAK 256
102
103static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
104 NETIF_MSG_LINK | NETIF_MSG_IFUP |
105 NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
106
107#define STMMAC_DEFAULT_LPI_TIMER 1000
108static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
109module_param(eee_timer, int, 0644);
110MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
111#define STMMAC_LPI_T(x) (jiffies + usecs_to_jiffies(x))
112
113
114
115
116static unsigned int chain_mode;
117module_param(chain_mode, int, 0444);
118MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
119
120static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
121
122static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id);
123static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id);
124static irqreturn_t stmmac_msi_intr_tx(int irq, void *data);
125static irqreturn_t stmmac_msi_intr_rx(int irq, void *data);
126static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue);
127static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue);
128
129#ifdef CONFIG_DEBUG_FS
130static const struct net_device_ops stmmac_netdev_ops;
131static void stmmac_init_fs(struct net_device *dev);
132static void stmmac_exit_fs(struct net_device *dev);
133#endif
134
135#define STMMAC_COAL_TIMER(x) (ns_to_ktime((x) * NSEC_PER_USEC))
136
137int stmmac_bus_clks_config(struct stmmac_priv *priv, bool enabled)
138{
139 int ret = 0;
140
141 if (enabled) {
142 ret = clk_prepare_enable(priv->plat->stmmac_clk);
143 if (ret)
144 return ret;
145 ret = clk_prepare_enable(priv->plat->pclk);
146 if (ret) {
147 clk_disable_unprepare(priv->plat->stmmac_clk);
148 return ret;
149 }
150 if (priv->plat->clks_config) {
151 ret = priv->plat->clks_config(priv->plat->bsp_priv, enabled);
152 if (ret) {
153 clk_disable_unprepare(priv->plat->stmmac_clk);
154 clk_disable_unprepare(priv->plat->pclk);
155 return ret;
156 }
157 }
158 } else {
159 clk_disable_unprepare(priv->plat->stmmac_clk);
160 clk_disable_unprepare(priv->plat->pclk);
161 if (priv->plat->clks_config)
162 priv->plat->clks_config(priv->plat->bsp_priv, enabled);
163 }
164
165 return ret;
166}
167EXPORT_SYMBOL_GPL(stmmac_bus_clks_config);
168
169
170
171
172
173
174static void stmmac_verify_args(void)
175{
176 if (unlikely(watchdog < 0))
177 watchdog = TX_TIMEO;
178 if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
179 buf_sz = DEFAULT_BUFSIZE;
180 if (unlikely(flow_ctrl > 1))
181 flow_ctrl = FLOW_AUTO;
182 else if (likely(flow_ctrl < 0))
183 flow_ctrl = FLOW_OFF;
184 if (unlikely((pause < 0) || (pause > 0xffff)))
185 pause = PAUSE_TIME;
186 if (eee_timer < 0)
187 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
188}
189
190static void __stmmac_disable_all_queues(struct stmmac_priv *priv)
191{
192 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
193 u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
194 u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
195 u32 queue;
196
197 for (queue = 0; queue < maxq; queue++) {
198 struct stmmac_channel *ch = &priv->channel[queue];
199
200 if (stmmac_xdp_is_enabled(priv) &&
201 test_bit(queue, priv->af_xdp_zc_qps)) {
202 napi_disable(&ch->rxtx_napi);
203 continue;
204 }
205
206 if (queue < rx_queues_cnt)
207 napi_disable(&ch->rx_napi);
208 if (queue < tx_queues_cnt)
209 napi_disable(&ch->tx_napi);
210 }
211}
212
213
214
215
216
217static void stmmac_disable_all_queues(struct stmmac_priv *priv)
218{
219 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
220 struct stmmac_rx_queue *rx_q;
221 u32 queue;
222
223
224 for (queue = 0; queue < rx_queues_cnt; queue++) {
225 rx_q = &priv->rx_queue[queue];
226 if (rx_q->xsk_pool) {
227 synchronize_rcu();
228 break;
229 }
230 }
231
232 __stmmac_disable_all_queues(priv);
233}
234
235
236
237
238
239static void stmmac_enable_all_queues(struct stmmac_priv *priv)
240{
241 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
242 u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
243 u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
244 u32 queue;
245
246 for (queue = 0; queue < maxq; queue++) {
247 struct stmmac_channel *ch = &priv->channel[queue];
248
249 if (stmmac_xdp_is_enabled(priv) &&
250 test_bit(queue, priv->af_xdp_zc_qps)) {
251 napi_enable(&ch->rxtx_napi);
252 continue;
253 }
254
255 if (queue < rx_queues_cnt)
256 napi_enable(&ch->rx_napi);
257 if (queue < tx_queues_cnt)
258 napi_enable(&ch->tx_napi);
259 }
260}
261
262static void stmmac_service_event_schedule(struct stmmac_priv *priv)
263{
264 if (!test_bit(STMMAC_DOWN, &priv->state) &&
265 !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
266 queue_work(priv->wq, &priv->service_task);
267}
268
269static void stmmac_global_err(struct stmmac_priv *priv)
270{
271 netif_carrier_off(priv->dev);
272 set_bit(STMMAC_RESET_REQUESTED, &priv->state);
273 stmmac_service_event_schedule(priv);
274}
275
276
277
278
279
280
281
282
283
284
285
286
287
288static void stmmac_clk_csr_set(struct stmmac_priv *priv)
289{
290 u32 clk_rate;
291
292 clk_rate = clk_get_rate(priv->plat->stmmac_clk);
293
294
295
296
297
298
299
300
301 if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
302 if (clk_rate < CSR_F_35M)
303 priv->clk_csr = STMMAC_CSR_20_35M;
304 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
305 priv->clk_csr = STMMAC_CSR_35_60M;
306 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
307 priv->clk_csr = STMMAC_CSR_60_100M;
308 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
309 priv->clk_csr = STMMAC_CSR_100_150M;
310 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
311 priv->clk_csr = STMMAC_CSR_150_250M;
312 else if ((clk_rate >= CSR_F_250M) && (clk_rate <= CSR_F_300M))
313 priv->clk_csr = STMMAC_CSR_250_300M;
314 }
315
316 if (priv->plat->has_sun8i) {
317 if (clk_rate > 160000000)
318 priv->clk_csr = 0x03;
319 else if (clk_rate > 80000000)
320 priv->clk_csr = 0x02;
321 else if (clk_rate > 40000000)
322 priv->clk_csr = 0x01;
323 else
324 priv->clk_csr = 0;
325 }
326
327 if (priv->plat->has_xgmac) {
328 if (clk_rate > 400000000)
329 priv->clk_csr = 0x5;
330 else if (clk_rate > 350000000)
331 priv->clk_csr = 0x4;
332 else if (clk_rate > 300000000)
333 priv->clk_csr = 0x3;
334 else if (clk_rate > 250000000)
335 priv->clk_csr = 0x2;
336 else if (clk_rate > 150000000)
337 priv->clk_csr = 0x1;
338 else
339 priv->clk_csr = 0x0;
340 }
341}
342
343static void print_pkt(unsigned char *buf, int len)
344{
345 pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
346 print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
347}
348
349static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
350{
351 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
352 u32 avail;
353
354 if (tx_q->dirty_tx > tx_q->cur_tx)
355 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
356 else
357 avail = priv->dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1;
358
359 return avail;
360}
361
362
363
364
365
366
367static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
368{
369 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
370 u32 dirty;
371
372 if (rx_q->dirty_rx <= rx_q->cur_rx)
373 dirty = rx_q->cur_rx - rx_q->dirty_rx;
374 else
375 dirty = priv->dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx;
376
377 return dirty;
378}
379
380static void stmmac_lpi_entry_timer_config(struct stmmac_priv *priv, bool en)
381{
382 int tx_lpi_timer;
383
384
385 priv->eee_sw_timer_en = en ? 0 : 1;
386 tx_lpi_timer = en ? priv->tx_lpi_timer : 0;
387 stmmac_set_eee_lpi_timer(priv, priv->hw, tx_lpi_timer);
388}
389
390
391
392
393
394
395
396static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
397{
398 u32 tx_cnt = priv->plat->tx_queues_to_use;
399 u32 queue;
400
401
402 for (queue = 0; queue < tx_cnt; queue++) {
403 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
404
405 if (tx_q->dirty_tx != tx_q->cur_tx)
406 return;
407 }
408
409
410 if (!priv->tx_path_in_lpi_mode)
411 stmmac_set_eee_mode(priv, priv->hw,
412 priv->plat->en_tx_lpi_clockgating);
413}
414
415
416
417
418
419
420
421void stmmac_disable_eee_mode(struct stmmac_priv *priv)
422{
423 if (!priv->eee_sw_timer_en) {
424 stmmac_lpi_entry_timer_config(priv, 0);
425 return;
426 }
427
428 stmmac_reset_eee_mode(priv, priv->hw);
429 del_timer_sync(&priv->eee_ctrl_timer);
430 priv->tx_path_in_lpi_mode = false;
431}
432
433
434
435
436
437
438
439
440static void stmmac_eee_ctrl_timer(struct timer_list *t)
441{
442 struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
443
444 stmmac_enable_eee_mode(priv);
445 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
446}
447
448
449
450
451
452
453
454
455
456bool stmmac_eee_init(struct stmmac_priv *priv)
457{
458 int eee_tw_timer = priv->eee_tw_timer;
459
460
461
462
463 if (priv->hw->pcs == STMMAC_PCS_TBI ||
464 priv->hw->pcs == STMMAC_PCS_RTBI)
465 return false;
466
467
468 if (!priv->dma_cap.eee)
469 return false;
470
471 mutex_lock(&priv->lock);
472
473
474 if (!priv->eee_active) {
475 if (priv->eee_enabled) {
476 netdev_dbg(priv->dev, "disable EEE\n");
477 stmmac_lpi_entry_timer_config(priv, 0);
478 del_timer_sync(&priv->eee_ctrl_timer);
479 stmmac_set_eee_timer(priv, priv->hw, 0, eee_tw_timer);
480 if (priv->hw->xpcs)
481 xpcs_config_eee(priv->hw->xpcs,
482 priv->plat->mult_fact_100ns,
483 false);
484 }
485 mutex_unlock(&priv->lock);
486 return false;
487 }
488
489 if (priv->eee_active && !priv->eee_enabled) {
490 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
491 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
492 eee_tw_timer);
493 if (priv->hw->xpcs)
494 xpcs_config_eee(priv->hw->xpcs,
495 priv->plat->mult_fact_100ns,
496 true);
497 }
498
499 if (priv->plat->has_gmac4 && priv->tx_lpi_timer <= STMMAC_ET_MAX) {
500 del_timer_sync(&priv->eee_ctrl_timer);
501 priv->tx_path_in_lpi_mode = false;
502 stmmac_lpi_entry_timer_config(priv, 1);
503 } else {
504 stmmac_lpi_entry_timer_config(priv, 0);
505 mod_timer(&priv->eee_ctrl_timer,
506 STMMAC_LPI_T(priv->tx_lpi_timer));
507 }
508
509 mutex_unlock(&priv->lock);
510 netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
511 return true;
512}
513
514
515
516
517
518
519
520
521
522static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
523 struct dma_desc *p, struct sk_buff *skb)
524{
525 struct skb_shared_hwtstamps shhwtstamp;
526 bool found = false;
527 s64 adjust = 0;
528 u64 ns = 0;
529
530 if (!priv->hwts_tx_en)
531 return;
532
533
534 if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
535 return;
536
537
538 if (stmmac_get_tx_timestamp_status(priv, p)) {
539 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
540 found = true;
541 } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
542 found = true;
543 }
544
545 if (found) {
546
547 if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) {
548 adjust += -(2 * (NSEC_PER_SEC /
549 priv->plat->clk_ptp_rate));
550 ns += adjust;
551 }
552
553 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
554 shhwtstamp.hwtstamp = ns_to_ktime(ns);
555
556 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
557
558 skb_tstamp_tx(skb, &shhwtstamp);
559 }
560}
561
562
563
564
565
566
567
568
569
570
571static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
572 struct dma_desc *np, struct sk_buff *skb)
573{
574 struct skb_shared_hwtstamps *shhwtstamp = NULL;
575 struct dma_desc *desc = p;
576 u64 adjust = 0;
577 u64 ns = 0;
578
579 if (!priv->hwts_rx_en)
580 return;
581
582 if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
583 desc = np;
584
585
586 if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
587 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
588
589
590 if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) {
591 adjust += 2 * (NSEC_PER_SEC / priv->plat->clk_ptp_rate);
592 ns -= adjust;
593 }
594
595 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
596 shhwtstamp = skb_hwtstamps(skb);
597 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
598 shhwtstamp->hwtstamp = ns_to_ktime(ns);
599 } else {
600 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
601 }
602}
603
604
605
606
607
608
609
610
611
612
613
614
615static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
616{
617 struct stmmac_priv *priv = netdev_priv(dev);
618 struct hwtstamp_config config;
619 struct timespec64 now;
620 u64 temp = 0;
621 u32 ptp_v2 = 0;
622 u32 tstamp_all = 0;
623 u32 ptp_over_ipv4_udp = 0;
624 u32 ptp_over_ipv6_udp = 0;
625 u32 ptp_over_ethernet = 0;
626 u32 snap_type_sel = 0;
627 u32 ts_master_en = 0;
628 u32 ts_event_en = 0;
629 u32 sec_inc = 0;
630 u32 value = 0;
631 bool xmac;
632
633 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
634
635 if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
636 netdev_alert(priv->dev, "No support for HW time stamping\n");
637 priv->hwts_tx_en = 0;
638 priv->hwts_rx_en = 0;
639
640 return -EOPNOTSUPP;
641 }
642
643 if (copy_from_user(&config, ifr->ifr_data,
644 sizeof(config)))
645 return -EFAULT;
646
647 netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
648 __func__, config.flags, config.tx_type, config.rx_filter);
649
650
651 if (config.flags)
652 return -EINVAL;
653
654 if (config.tx_type != HWTSTAMP_TX_OFF &&
655 config.tx_type != HWTSTAMP_TX_ON)
656 return -ERANGE;
657
658 if (priv->adv_ts) {
659 switch (config.rx_filter) {
660 case HWTSTAMP_FILTER_NONE:
661
662 config.rx_filter = HWTSTAMP_FILTER_NONE;
663 break;
664
665 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
666
667 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
668
669
670
671
672
673
674 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
675 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
676 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
677 break;
678
679 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
680
681 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
682
683 ts_event_en = PTP_TCR_TSEVNTENA;
684
685 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
686 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
687 break;
688
689 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
690
691 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
692
693 ts_master_en = PTP_TCR_TSMSTRENA;
694 ts_event_en = PTP_TCR_TSEVNTENA;
695
696 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
697 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
698 break;
699
700 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
701
702 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
703 ptp_v2 = PTP_TCR_TSVER2ENA;
704
705 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
706
707 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
708 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
709 break;
710
711 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
712
713 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
714 ptp_v2 = PTP_TCR_TSVER2ENA;
715
716 ts_event_en = PTP_TCR_TSEVNTENA;
717
718 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
719 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
720 break;
721
722 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
723
724 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
725 ptp_v2 = PTP_TCR_TSVER2ENA;
726
727 ts_master_en = PTP_TCR_TSMSTRENA;
728 ts_event_en = PTP_TCR_TSEVNTENA;
729
730 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
731 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
732 break;
733
734 case HWTSTAMP_FILTER_PTP_V2_EVENT:
735
736 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
737 ptp_v2 = PTP_TCR_TSVER2ENA;
738 snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
739 if (priv->synopsys_id < DWMAC_CORE_4_10)
740 ts_event_en = PTP_TCR_TSEVNTENA;
741 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
742 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
743 ptp_over_ethernet = PTP_TCR_TSIPENA;
744 break;
745
746 case HWTSTAMP_FILTER_PTP_V2_SYNC:
747
748 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
749 ptp_v2 = PTP_TCR_TSVER2ENA;
750
751 ts_event_en = PTP_TCR_TSEVNTENA;
752
753 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
754 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
755 ptp_over_ethernet = PTP_TCR_TSIPENA;
756 break;
757
758 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
759
760 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
761 ptp_v2 = PTP_TCR_TSVER2ENA;
762
763 ts_master_en = PTP_TCR_TSMSTRENA;
764 ts_event_en = PTP_TCR_TSEVNTENA;
765
766 ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
767 ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
768 ptp_over_ethernet = PTP_TCR_TSIPENA;
769 break;
770
771 case HWTSTAMP_FILTER_NTP_ALL:
772 case HWTSTAMP_FILTER_ALL:
773
774 config.rx_filter = HWTSTAMP_FILTER_ALL;
775 tstamp_all = PTP_TCR_TSENALL;
776 break;
777
778 default:
779 return -ERANGE;
780 }
781 } else {
782 switch (config.rx_filter) {
783 case HWTSTAMP_FILTER_NONE:
784 config.rx_filter = HWTSTAMP_FILTER_NONE;
785 break;
786 default:
787
788 config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
789 break;
790 }
791 }
792 priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
793 priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
794
795 if (!priv->hwts_tx_en && !priv->hwts_rx_en)
796 stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
797 else {
798 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
799 tstamp_all | ptp_v2 | ptp_over_ethernet |
800 ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
801 ts_master_en | snap_type_sel);
802 stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
803
804
805 stmmac_config_sub_second_increment(priv,
806 priv->ptpaddr, priv->plat->clk_ptp_rate,
807 xmac, &sec_inc);
808 temp = div_u64(1000000000ULL, sec_inc);
809
810
811 priv->sub_second_inc = sec_inc;
812 priv->systime_flags = value;
813
814
815
816
817
818
819 temp = (u64)(temp << 32);
820 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
821 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
822
823
824 ktime_get_real_ts64(&now);
825
826
827 stmmac_init_systime(priv, priv->ptpaddr,
828 (u32)now.tv_sec, now.tv_nsec);
829 }
830
831 memcpy(&priv->tstamp_config, &config, sizeof(config));
832
833 return copy_to_user(ifr->ifr_data, &config,
834 sizeof(config)) ? -EFAULT : 0;
835}
836
837
838
839
840
841
842
843
844
845
846static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
847{
848 struct stmmac_priv *priv = netdev_priv(dev);
849 struct hwtstamp_config *config = &priv->tstamp_config;
850
851 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
852 return -EOPNOTSUPP;
853
854 return copy_to_user(ifr->ifr_data, config,
855 sizeof(*config)) ? -EFAULT : 0;
856}
857
858
859
860
861
862
863
864
865static int stmmac_init_ptp(struct stmmac_priv *priv)
866{
867 bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
868
869 if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
870 return -EOPNOTSUPP;
871
872 priv->adv_ts = 0;
873
874 if (xmac && priv->dma_cap.atime_stamp)
875 priv->adv_ts = 1;
876
877 else if (priv->extend_desc && priv->dma_cap.atime_stamp)
878 priv->adv_ts = 1;
879
880 if (priv->dma_cap.time_stamp)
881 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
882
883 if (priv->adv_ts)
884 netdev_info(priv->dev,
885 "IEEE 1588-2008 Advanced Timestamp supported\n");
886
887 priv->hwts_tx_en = 0;
888 priv->hwts_rx_en = 0;
889
890 stmmac_ptp_register(priv);
891
892 return 0;
893}
894
895static void stmmac_release_ptp(struct stmmac_priv *priv)
896{
897 clk_disable_unprepare(priv->plat->clk_ptp_ref);
898 stmmac_ptp_unregister(priv);
899}
900
901
902
903
904
905
906
907static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
908{
909 u32 tx_cnt = priv->plat->tx_queues_to_use;
910
911 stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
912 priv->pause, tx_cnt);
913}
914
915static void stmmac_validate(struct phylink_config *config,
916 unsigned long *supported,
917 struct phylink_link_state *state)
918{
919 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
920 __ETHTOOL_DECLARE_LINK_MODE_MASK(mac_supported) = { 0, };
921 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
922 int tx_cnt = priv->plat->tx_queues_to_use;
923 int max_speed = priv->plat->max_speed;
924
925 phylink_set(mac_supported, 10baseT_Half);
926 phylink_set(mac_supported, 10baseT_Full);
927 phylink_set(mac_supported, 100baseT_Half);
928 phylink_set(mac_supported, 100baseT_Full);
929 phylink_set(mac_supported, 1000baseT_Half);
930 phylink_set(mac_supported, 1000baseT_Full);
931 phylink_set(mac_supported, 1000baseKX_Full);
932
933 phylink_set(mac_supported, Autoneg);
934 phylink_set(mac_supported, Pause);
935 phylink_set(mac_supported, Asym_Pause);
936 phylink_set_port_modes(mac_supported);
937
938
939 if ((max_speed > 0) && (max_speed < 1000)) {
940 phylink_set(mask, 1000baseT_Full);
941 phylink_set(mask, 1000baseX_Full);
942 } else if (priv->plat->has_gmac4) {
943 if (!max_speed || max_speed >= 2500) {
944 phylink_set(mac_supported, 2500baseT_Full);
945 phylink_set(mac_supported, 2500baseX_Full);
946 }
947 } else if (priv->plat->has_xgmac) {
948 if (!max_speed || (max_speed >= 2500)) {
949 phylink_set(mac_supported, 2500baseT_Full);
950 phylink_set(mac_supported, 2500baseX_Full);
951 }
952 if (!max_speed || (max_speed >= 5000)) {
953 phylink_set(mac_supported, 5000baseT_Full);
954 }
955 if (!max_speed || (max_speed >= 10000)) {
956 phylink_set(mac_supported, 10000baseSR_Full);
957 phylink_set(mac_supported, 10000baseLR_Full);
958 phylink_set(mac_supported, 10000baseER_Full);
959 phylink_set(mac_supported, 10000baseLRM_Full);
960 phylink_set(mac_supported, 10000baseT_Full);
961 phylink_set(mac_supported, 10000baseKX4_Full);
962 phylink_set(mac_supported, 10000baseKR_Full);
963 }
964 if (!max_speed || (max_speed >= 25000)) {
965 phylink_set(mac_supported, 25000baseCR_Full);
966 phylink_set(mac_supported, 25000baseKR_Full);
967 phylink_set(mac_supported, 25000baseSR_Full);
968 }
969 if (!max_speed || (max_speed >= 40000)) {
970 phylink_set(mac_supported, 40000baseKR4_Full);
971 phylink_set(mac_supported, 40000baseCR4_Full);
972 phylink_set(mac_supported, 40000baseSR4_Full);
973 phylink_set(mac_supported, 40000baseLR4_Full);
974 }
975 if (!max_speed || (max_speed >= 50000)) {
976 phylink_set(mac_supported, 50000baseCR2_Full);
977 phylink_set(mac_supported, 50000baseKR2_Full);
978 phylink_set(mac_supported, 50000baseSR2_Full);
979 phylink_set(mac_supported, 50000baseKR_Full);
980 phylink_set(mac_supported, 50000baseSR_Full);
981 phylink_set(mac_supported, 50000baseCR_Full);
982 phylink_set(mac_supported, 50000baseLR_ER_FR_Full);
983 phylink_set(mac_supported, 50000baseDR_Full);
984 }
985 if (!max_speed || (max_speed >= 100000)) {
986 phylink_set(mac_supported, 100000baseKR4_Full);
987 phylink_set(mac_supported, 100000baseSR4_Full);
988 phylink_set(mac_supported, 100000baseCR4_Full);
989 phylink_set(mac_supported, 100000baseLR4_ER4_Full);
990 phylink_set(mac_supported, 100000baseKR2_Full);
991 phylink_set(mac_supported, 100000baseSR2_Full);
992 phylink_set(mac_supported, 100000baseCR2_Full);
993 phylink_set(mac_supported, 100000baseLR2_ER2_FR2_Full);
994 phylink_set(mac_supported, 100000baseDR2_Full);
995 }
996 }
997
998
999 if (tx_cnt > 1) {
1000 phylink_set(mask, 10baseT_Half);
1001 phylink_set(mask, 100baseT_Half);
1002 phylink_set(mask, 1000baseT_Half);
1003 }
1004
1005 linkmode_and(supported, supported, mac_supported);
1006 linkmode_andnot(supported, supported, mask);
1007
1008 linkmode_and(state->advertising, state->advertising, mac_supported);
1009 linkmode_andnot(state->advertising, state->advertising, mask);
1010
1011
1012 if (priv->hw->xpcs)
1013 xpcs_validate(priv->hw->xpcs, supported, state);
1014}
1015
1016static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
1017 const struct phylink_link_state *state)
1018{
1019
1020}
1021
1022static void stmmac_fpe_link_state_handle(struct stmmac_priv *priv, bool is_up)
1023{
1024 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg;
1025 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state;
1026 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state;
1027 bool *hs_enable = &fpe_cfg->hs_enable;
1028
1029 if (is_up && *hs_enable) {
1030 stmmac_fpe_send_mpacket(priv, priv->ioaddr, MPACKET_VERIFY);
1031 } else {
1032 *lo_state = FPE_STATE_OFF;
1033 *lp_state = FPE_STATE_OFF;
1034 }
1035}
1036
1037static void stmmac_mac_link_down(struct phylink_config *config,
1038 unsigned int mode, phy_interface_t interface)
1039{
1040 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
1041
1042 stmmac_mac_set(priv, priv->ioaddr, false);
1043 priv->eee_active = false;
1044 priv->tx_lpi_enabled = false;
1045 priv->eee_enabled = stmmac_eee_init(priv);
1046 stmmac_set_eee_pls(priv, priv->hw, false);
1047
1048 if (priv->dma_cap.fpesel)
1049 stmmac_fpe_link_state_handle(priv, false);
1050}
1051
1052static void stmmac_mac_link_up(struct phylink_config *config,
1053 struct phy_device *phy,
1054 unsigned int mode, phy_interface_t interface,
1055 int speed, int duplex,
1056 bool tx_pause, bool rx_pause)
1057{
1058 struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
1059 u32 ctrl;
1060
1061 ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
1062 ctrl &= ~priv->hw->link.speed_mask;
1063
1064 if (interface == PHY_INTERFACE_MODE_USXGMII) {
1065 switch (speed) {
1066 case SPEED_10000:
1067 ctrl |= priv->hw->link.xgmii.speed10000;
1068 break;
1069 case SPEED_5000:
1070 ctrl |= priv->hw->link.xgmii.speed5000;
1071 break;
1072 case SPEED_2500:
1073 ctrl |= priv->hw->link.xgmii.speed2500;
1074 break;
1075 default:
1076 return;
1077 }
1078 } else if (interface == PHY_INTERFACE_MODE_XLGMII) {
1079 switch (speed) {
1080 case SPEED_100000:
1081 ctrl |= priv->hw->link.xlgmii.speed100000;
1082 break;
1083 case SPEED_50000:
1084 ctrl |= priv->hw->link.xlgmii.speed50000;
1085 break;
1086 case SPEED_40000:
1087 ctrl |= priv->hw->link.xlgmii.speed40000;
1088 break;
1089 case SPEED_25000:
1090 ctrl |= priv->hw->link.xlgmii.speed25000;
1091 break;
1092 case SPEED_10000:
1093 ctrl |= priv->hw->link.xgmii.speed10000;
1094 break;
1095 case SPEED_2500:
1096 ctrl |= priv->hw->link.speed2500;
1097 break;
1098 case SPEED_1000:
1099 ctrl |= priv->hw->link.speed1000;
1100 break;
1101 default:
1102 return;
1103 }
1104 } else {
1105 switch (speed) {
1106 case SPEED_2500:
1107 ctrl |= priv->hw->link.speed2500;
1108 break;
1109 case SPEED_1000:
1110 ctrl |= priv->hw->link.speed1000;
1111 break;
1112 case SPEED_100:
1113 ctrl |= priv->hw->link.speed100;
1114 break;
1115 case SPEED_10:
1116 ctrl |= priv->hw->link.speed10;
1117 break;
1118 default:
1119 return;
1120 }
1121 }
1122
1123 priv->speed = speed;
1124
1125 if (priv->plat->fix_mac_speed)
1126 priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed);
1127
1128 if (!duplex)
1129 ctrl &= ~priv->hw->link.duplex;
1130 else
1131 ctrl |= priv->hw->link.duplex;
1132
1133
1134 if (tx_pause && rx_pause)
1135 stmmac_mac_flow_ctrl(priv, duplex);
1136
1137 writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
1138
1139 stmmac_mac_set(priv, priv->ioaddr, true);
1140 if (phy && priv->dma_cap.eee) {
1141 priv->eee_active = phy_init_eee(phy, 1) >= 0;
1142 priv->eee_enabled = stmmac_eee_init(priv);
1143 priv->tx_lpi_enabled = priv->eee_enabled;
1144 stmmac_set_eee_pls(priv, priv->hw, true);
1145 }
1146
1147 if (priv->dma_cap.fpesel)
1148 stmmac_fpe_link_state_handle(priv, true);
1149}
1150
1151static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
1152 .validate = stmmac_validate,
1153 .mac_config = stmmac_mac_config,
1154 .mac_link_down = stmmac_mac_link_down,
1155 .mac_link_up = stmmac_mac_link_up,
1156};
1157
1158
1159
1160
1161
1162
1163
1164
1165static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
1166{
1167 int interface = priv->plat->interface;
1168
1169 if (priv->dma_cap.pcs) {
1170 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
1171 (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
1172 (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
1173 (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
1174 netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
1175 priv->hw->pcs = STMMAC_PCS_RGMII;
1176 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
1177 netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
1178 priv->hw->pcs = STMMAC_PCS_SGMII;
1179 }
1180 }
1181}
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191static int stmmac_init_phy(struct net_device *dev)
1192{
1193 struct stmmac_priv *priv = netdev_priv(dev);
1194 struct device_node *node;
1195 int ret;
1196
1197 node = priv->plat->phylink_node;
1198
1199 if (node)
1200 ret = phylink_of_phy_connect(priv->phylink, node, 0);
1201
1202
1203
1204
1205 if (!node || ret) {
1206 int addr = priv->plat->phy_addr;
1207 struct phy_device *phydev;
1208
1209 phydev = mdiobus_get_phy(priv->mii, addr);
1210 if (!phydev) {
1211 netdev_err(priv->dev, "no phy at addr %d\n", addr);
1212 return -ENODEV;
1213 }
1214
1215 ret = phylink_connect_phy(priv->phylink, phydev);
1216 }
1217
1218 if (!priv->plat->pmt) {
1219 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL };
1220
1221 phylink_ethtool_get_wol(priv->phylink, &wol);
1222 device_set_wakeup_capable(priv->device, !!wol.supported);
1223 }
1224
1225 return ret;
1226}
1227
1228static int stmmac_phy_setup(struct stmmac_priv *priv)
1229{
1230 struct stmmac_mdio_bus_data *mdio_bus_data = priv->plat->mdio_bus_data;
1231 struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node);
1232 int mode = priv->plat->phy_interface;
1233 struct phylink *phylink;
1234
1235 priv->phylink_config.dev = &priv->dev->dev;
1236 priv->phylink_config.type = PHYLINK_NETDEV;
1237 priv->phylink_config.pcs_poll = true;
1238 if (priv->plat->mdio_bus_data)
1239 priv->phylink_config.ovr_an_inband =
1240 mdio_bus_data->xpcs_an_inband;
1241
1242 if (!fwnode)
1243 fwnode = dev_fwnode(priv->device);
1244
1245 phylink = phylink_create(&priv->phylink_config, fwnode,
1246 mode, &stmmac_phylink_mac_ops);
1247 if (IS_ERR(phylink))
1248 return PTR_ERR(phylink);
1249
1250 if (priv->hw->xpcs)
1251 phylink_set_pcs(phylink, &priv->hw->xpcs->pcs);
1252
1253 priv->phylink = phylink;
1254 return 0;
1255}
1256
1257static void stmmac_display_rx_rings(struct stmmac_priv *priv)
1258{
1259 u32 rx_cnt = priv->plat->rx_queues_to_use;
1260 unsigned int desc_size;
1261 void *head_rx;
1262 u32 queue;
1263
1264
1265 for (queue = 0; queue < rx_cnt; queue++) {
1266 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1267
1268 pr_info("\tRX Queue %u rings\n", queue);
1269
1270 if (priv->extend_desc) {
1271 head_rx = (void *)rx_q->dma_erx;
1272 desc_size = sizeof(struct dma_extended_desc);
1273 } else {
1274 head_rx = (void *)rx_q->dma_rx;
1275 desc_size = sizeof(struct dma_desc);
1276 }
1277
1278
1279 stmmac_display_ring(priv, head_rx, priv->dma_rx_size, true,
1280 rx_q->dma_rx_phy, desc_size);
1281 }
1282}
1283
1284static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1285{
1286 u32 tx_cnt = priv->plat->tx_queues_to_use;
1287 unsigned int desc_size;
1288 void *head_tx;
1289 u32 queue;
1290
1291
1292 for (queue = 0; queue < tx_cnt; queue++) {
1293 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1294
1295 pr_info("\tTX Queue %d rings\n", queue);
1296
1297 if (priv->extend_desc) {
1298 head_tx = (void *)tx_q->dma_etx;
1299 desc_size = sizeof(struct dma_extended_desc);
1300 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
1301 head_tx = (void *)tx_q->dma_entx;
1302 desc_size = sizeof(struct dma_edesc);
1303 } else {
1304 head_tx = (void *)tx_q->dma_tx;
1305 desc_size = sizeof(struct dma_desc);
1306 }
1307
1308 stmmac_display_ring(priv, head_tx, priv->dma_tx_size, false,
1309 tx_q->dma_tx_phy, desc_size);
1310 }
1311}
1312
1313static void stmmac_display_rings(struct stmmac_priv *priv)
1314{
1315
1316 stmmac_display_rx_rings(priv);
1317
1318
1319 stmmac_display_tx_rings(priv);
1320}
1321
1322static int stmmac_set_bfsize(int mtu, int bufsize)
1323{
1324 int ret = bufsize;
1325
1326 if (mtu >= BUF_SIZE_8KiB)
1327 ret = BUF_SIZE_16KiB;
1328 else if (mtu >= BUF_SIZE_4KiB)
1329 ret = BUF_SIZE_8KiB;
1330 else if (mtu >= BUF_SIZE_2KiB)
1331 ret = BUF_SIZE_4KiB;
1332 else if (mtu > DEFAULT_BUFSIZE)
1333 ret = BUF_SIZE_2KiB;
1334 else
1335 ret = DEFAULT_BUFSIZE;
1336
1337 return ret;
1338}
1339
1340
1341
1342
1343
1344
1345
1346
1347static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1348{
1349 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1350 int i;
1351
1352
1353 for (i = 0; i < priv->dma_rx_size; i++)
1354 if (priv->extend_desc)
1355 stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1356 priv->use_riwt, priv->mode,
1357 (i == priv->dma_rx_size - 1),
1358 priv->dma_buf_sz);
1359 else
1360 stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1361 priv->use_riwt, priv->mode,
1362 (i == priv->dma_rx_size - 1),
1363 priv->dma_buf_sz);
1364}
1365
1366
1367
1368
1369
1370
1371
1372
1373static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1374{
1375 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1376 int i;
1377
1378
1379 for (i = 0; i < priv->dma_tx_size; i++) {
1380 int last = (i == (priv->dma_tx_size - 1));
1381 struct dma_desc *p;
1382
1383 if (priv->extend_desc)
1384 p = &tx_q->dma_etx[i].basic;
1385 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1386 p = &tx_q->dma_entx[i].basic;
1387 else
1388 p = &tx_q->dma_tx[i];
1389
1390 stmmac_init_tx_desc(priv, p, priv->mode, last);
1391 }
1392}
1393
1394
1395
1396
1397
1398
1399
1400static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1401{
1402 u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1403 u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1404 u32 queue;
1405
1406
1407 for (queue = 0; queue < rx_queue_cnt; queue++)
1408 stmmac_clear_rx_descriptors(priv, queue);
1409
1410
1411 for (queue = 0; queue < tx_queue_cnt; queue++)
1412 stmmac_clear_tx_descriptors(priv, queue);
1413}
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1426 int i, gfp_t flags, u32 queue)
1427{
1428 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1429 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1430
1431 if (!buf->page) {
1432 buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
1433 if (!buf->page)
1434 return -ENOMEM;
1435 buf->page_offset = stmmac_rx_offset(priv);
1436 }
1437
1438 if (priv->sph && !buf->sec_page) {
1439 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
1440 if (!buf->sec_page)
1441 return -ENOMEM;
1442
1443 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
1444 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
1445 } else {
1446 buf->sec_page = NULL;
1447 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
1448 }
1449
1450 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset;
1451
1452 stmmac_set_desc_addr(priv, p, buf->addr);
1453 if (priv->dma_buf_sz == BUF_SIZE_16KiB)
1454 stmmac_init_desc3(priv, p);
1455
1456 return 0;
1457}
1458
1459
1460
1461
1462
1463
1464
1465static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1466{
1467 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1468 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1469
1470 if (buf->page)
1471 page_pool_put_full_page(rx_q->page_pool, buf->page, false);
1472 buf->page = NULL;
1473
1474 if (buf->sec_page)
1475 page_pool_put_full_page(rx_q->page_pool, buf->sec_page, false);
1476 buf->sec_page = NULL;
1477}
1478
1479
1480
1481
1482
1483
1484
1485static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1486{
1487 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1488
1489 if (tx_q->tx_skbuff_dma[i].buf &&
1490 tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) {
1491 if (tx_q->tx_skbuff_dma[i].map_as_page)
1492 dma_unmap_page(priv->device,
1493 tx_q->tx_skbuff_dma[i].buf,
1494 tx_q->tx_skbuff_dma[i].len,
1495 DMA_TO_DEVICE);
1496 else
1497 dma_unmap_single(priv->device,
1498 tx_q->tx_skbuff_dma[i].buf,
1499 tx_q->tx_skbuff_dma[i].len,
1500 DMA_TO_DEVICE);
1501 }
1502
1503 if (tx_q->xdpf[i] &&
1504 (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_TX ||
1505 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XDP_NDO)) {
1506 xdp_return_frame(tx_q->xdpf[i]);
1507 tx_q->xdpf[i] = NULL;
1508 }
1509
1510 if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_XSK_TX)
1511 tx_q->xsk_frames_done++;
1512
1513 if (tx_q->tx_skbuff[i] &&
1514 tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) {
1515 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1516 tx_q->tx_skbuff[i] = NULL;
1517 }
1518
1519 tx_q->tx_skbuff_dma[i].buf = 0;
1520 tx_q->tx_skbuff_dma[i].map_as_page = false;
1521}
1522
1523
1524
1525
1526
1527
1528static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1529{
1530 int i;
1531
1532 for (i = 0; i < priv->dma_rx_size; i++)
1533 stmmac_free_rx_buffer(priv, queue, i);
1534}
1535
1536static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, u32 queue,
1537 gfp_t flags)
1538{
1539 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1540 int i;
1541
1542 for (i = 0; i < priv->dma_rx_size; i++) {
1543 struct dma_desc *p;
1544 int ret;
1545
1546 if (priv->extend_desc)
1547 p = &((rx_q->dma_erx + i)->basic);
1548 else
1549 p = rx_q->dma_rx + i;
1550
1551 ret = stmmac_init_rx_buffers(priv, p, i, flags,
1552 queue);
1553 if (ret)
1554 return ret;
1555
1556 rx_q->buf_alloc_num++;
1557 }
1558
1559 return 0;
1560}
1561
1562
1563
1564
1565
1566
1567static void dma_free_rx_xskbufs(struct stmmac_priv *priv, u32 queue)
1568{
1569 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1570 int i;
1571
1572 for (i = 0; i < priv->dma_rx_size; i++) {
1573 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1574
1575 if (!buf->xdp)
1576 continue;
1577
1578 xsk_buff_free(buf->xdp);
1579 buf->xdp = NULL;
1580 }
1581}
1582
1583static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, u32 queue)
1584{
1585 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1586 int i;
1587
1588 for (i = 0; i < priv->dma_rx_size; i++) {
1589 struct stmmac_rx_buffer *buf;
1590 dma_addr_t dma_addr;
1591 struct dma_desc *p;
1592
1593 if (priv->extend_desc)
1594 p = (struct dma_desc *)(rx_q->dma_erx + i);
1595 else
1596 p = rx_q->dma_rx + i;
1597
1598 buf = &rx_q->buf_pool[i];
1599
1600 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool);
1601 if (!buf->xdp)
1602 return -ENOMEM;
1603
1604 dma_addr = xsk_buff_xdp_get_dma(buf->xdp);
1605 stmmac_set_desc_addr(priv, p, dma_addr);
1606 rx_q->buf_alloc_num++;
1607 }
1608
1609 return 0;
1610}
1611
1612static struct xsk_buff_pool *stmmac_get_xsk_pool(struct stmmac_priv *priv, u32 queue)
1613{
1614 if (!stmmac_xdp_is_enabled(priv) || !test_bit(queue, priv->af_xdp_zc_qps))
1615 return NULL;
1616
1617 return xsk_get_pool_from_qid(priv->dev, queue);
1618}
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, u32 queue, gfp_t flags)
1630{
1631 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1632 int ret;
1633
1634 netif_dbg(priv, probe, priv->dev,
1635 "(%s) dma_rx_phy=0x%08x\n", __func__,
1636 (u32)rx_q->dma_rx_phy);
1637
1638 stmmac_clear_rx_descriptors(priv, queue);
1639
1640 xdp_rxq_info_unreg_mem_model(&rx_q->xdp_rxq);
1641
1642 rx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
1643
1644 if (rx_q->xsk_pool) {
1645 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
1646 MEM_TYPE_XSK_BUFF_POOL,
1647 NULL));
1648 netdev_info(priv->dev,
1649 "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n",
1650 rx_q->queue_index);
1651 xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq);
1652 } else {
1653 WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq,
1654 MEM_TYPE_PAGE_POOL,
1655 rx_q->page_pool));
1656 netdev_info(priv->dev,
1657 "Register MEM_TYPE_PAGE_POOL RxQ-%d\n",
1658 rx_q->queue_index);
1659 }
1660
1661 if (rx_q->xsk_pool) {
1662
1663
1664
1665 stmmac_alloc_rx_buffers_zc(priv, queue);
1666 } else {
1667 ret = stmmac_alloc_rx_buffers(priv, queue, flags);
1668 if (ret < 0)
1669 return -ENOMEM;
1670 }
1671
1672 rx_q->cur_rx = 0;
1673 rx_q->dirty_rx = 0;
1674
1675
1676 if (priv->mode == STMMAC_CHAIN_MODE) {
1677 if (priv->extend_desc)
1678 stmmac_mode_init(priv, rx_q->dma_erx,
1679 rx_q->dma_rx_phy,
1680 priv->dma_rx_size, 1);
1681 else
1682 stmmac_mode_init(priv, rx_q->dma_rx,
1683 rx_q->dma_rx_phy,
1684 priv->dma_rx_size, 0);
1685 }
1686
1687 return 0;
1688}
1689
1690static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1691{
1692 struct stmmac_priv *priv = netdev_priv(dev);
1693 u32 rx_count = priv->plat->rx_queues_to_use;
1694 u32 queue;
1695 int ret;
1696
1697
1698 netif_dbg(priv, probe, priv->dev,
1699 "SKB addresses:\nskb\t\tskb data\tdma data\n");
1700
1701 for (queue = 0; queue < rx_count; queue++) {
1702 ret = __init_dma_rx_desc_rings(priv, queue, flags);
1703 if (ret)
1704 goto err_init_rx_buffers;
1705 }
1706
1707 return 0;
1708
1709err_init_rx_buffers:
1710 while (queue >= 0) {
1711 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1712
1713 if (rx_q->xsk_pool)
1714 dma_free_rx_xskbufs(priv, queue);
1715 else
1716 dma_free_rx_skbufs(priv, queue);
1717
1718 rx_q->buf_alloc_num = 0;
1719 rx_q->xsk_pool = NULL;
1720
1721 if (queue == 0)
1722 break;
1723
1724 queue--;
1725 }
1726
1727 return ret;
1728}
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, u32 queue)
1739{
1740 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1741 int i;
1742
1743 netif_dbg(priv, probe, priv->dev,
1744 "(%s) dma_tx_phy=0x%08x\n", __func__,
1745 (u32)tx_q->dma_tx_phy);
1746
1747
1748 if (priv->mode == STMMAC_CHAIN_MODE) {
1749 if (priv->extend_desc)
1750 stmmac_mode_init(priv, tx_q->dma_etx,
1751 tx_q->dma_tx_phy,
1752 priv->dma_tx_size, 1);
1753 else if (!(tx_q->tbs & STMMAC_TBS_AVAIL))
1754 stmmac_mode_init(priv, tx_q->dma_tx,
1755 tx_q->dma_tx_phy,
1756 priv->dma_tx_size, 0);
1757 }
1758
1759 tx_q->xsk_pool = stmmac_get_xsk_pool(priv, queue);
1760
1761 for (i = 0; i < priv->dma_tx_size; i++) {
1762 struct dma_desc *p;
1763
1764 if (priv->extend_desc)
1765 p = &((tx_q->dma_etx + i)->basic);
1766 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
1767 p = &((tx_q->dma_entx + i)->basic);
1768 else
1769 p = tx_q->dma_tx + i;
1770
1771 stmmac_clear_desc(priv, p);
1772
1773 tx_q->tx_skbuff_dma[i].buf = 0;
1774 tx_q->tx_skbuff_dma[i].map_as_page = false;
1775 tx_q->tx_skbuff_dma[i].len = 0;
1776 tx_q->tx_skbuff_dma[i].last_segment = false;
1777 tx_q->tx_skbuff[i] = NULL;
1778 }
1779
1780 tx_q->dirty_tx = 0;
1781 tx_q->cur_tx = 0;
1782 tx_q->mss = 0;
1783
1784 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1785
1786 return 0;
1787}
1788
1789static int init_dma_tx_desc_rings(struct net_device *dev)
1790{
1791 struct stmmac_priv *priv = netdev_priv(dev);
1792 u32 tx_queue_cnt;
1793 u32 queue;
1794
1795 tx_queue_cnt = priv->plat->tx_queues_to_use;
1796
1797 for (queue = 0; queue < tx_queue_cnt; queue++)
1798 __init_dma_tx_desc_rings(priv, queue);
1799
1800 return 0;
1801}
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1812{
1813 struct stmmac_priv *priv = netdev_priv(dev);
1814 int ret;
1815
1816 ret = init_dma_rx_desc_rings(dev, flags);
1817 if (ret)
1818 return ret;
1819
1820 ret = init_dma_tx_desc_rings(dev);
1821
1822 stmmac_clear_descriptors(priv);
1823
1824 if (netif_msg_hw(priv))
1825 stmmac_display_rings(priv);
1826
1827 return ret;
1828}
1829
1830
1831
1832
1833
1834
1835static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1836{
1837 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1838 int i;
1839
1840 tx_q->xsk_frames_done = 0;
1841
1842 for (i = 0; i < priv->dma_tx_size; i++)
1843 stmmac_free_tx_buffer(priv, queue, i);
1844
1845 if (tx_q->xsk_pool && tx_q->xsk_frames_done) {
1846 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done);
1847 tx_q->xsk_frames_done = 0;
1848 tx_q->xsk_pool = NULL;
1849 }
1850}
1851
1852
1853
1854
1855
1856static void stmmac_free_tx_skbufs(struct stmmac_priv *priv)
1857{
1858 u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1859 u32 queue;
1860
1861 for (queue = 0; queue < tx_queue_cnt; queue++)
1862 dma_free_tx_skbufs(priv, queue);
1863}
1864
1865
1866
1867
1868
1869
1870static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue)
1871{
1872 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1873
1874
1875 if (rx_q->xsk_pool)
1876 dma_free_rx_xskbufs(priv, queue);
1877 else
1878 dma_free_rx_skbufs(priv, queue);
1879
1880 rx_q->buf_alloc_num = 0;
1881 rx_q->xsk_pool = NULL;
1882
1883
1884 if (!priv->extend_desc)
1885 dma_free_coherent(priv->device, priv->dma_rx_size *
1886 sizeof(struct dma_desc),
1887 rx_q->dma_rx, rx_q->dma_rx_phy);
1888 else
1889 dma_free_coherent(priv->device, priv->dma_rx_size *
1890 sizeof(struct dma_extended_desc),
1891 rx_q->dma_erx, rx_q->dma_rx_phy);
1892
1893 if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq))
1894 xdp_rxq_info_unreg(&rx_q->xdp_rxq);
1895
1896 kfree(rx_q->buf_pool);
1897 if (rx_q->page_pool)
1898 page_pool_destroy(rx_q->page_pool);
1899}
1900
1901static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1902{
1903 u32 rx_count = priv->plat->rx_queues_to_use;
1904 u32 queue;
1905
1906
1907 for (queue = 0; queue < rx_count; queue++)
1908 __free_dma_rx_desc_resources(priv, queue);
1909}
1910
1911
1912
1913
1914
1915
1916static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue)
1917{
1918 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1919 size_t size;
1920 void *addr;
1921
1922
1923 dma_free_tx_skbufs(priv, queue);
1924
1925 if (priv->extend_desc) {
1926 size = sizeof(struct dma_extended_desc);
1927 addr = tx_q->dma_etx;
1928 } else if (tx_q->tbs & STMMAC_TBS_AVAIL) {
1929 size = sizeof(struct dma_edesc);
1930 addr = tx_q->dma_entx;
1931 } else {
1932 size = sizeof(struct dma_desc);
1933 addr = tx_q->dma_tx;
1934 }
1935
1936 size *= priv->dma_tx_size;
1937
1938 dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy);
1939
1940 kfree(tx_q->tx_skbuff_dma);
1941 kfree(tx_q->tx_skbuff);
1942}
1943
1944static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1945{
1946 u32 tx_count = priv->plat->tx_queues_to_use;
1947 u32 queue;
1948
1949
1950 for (queue = 0; queue < tx_count; queue++)
1951 __free_dma_tx_desc_resources(priv, queue);
1952}
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue)
1964{
1965 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1966 struct stmmac_channel *ch = &priv->channel[queue];
1967 bool xdp_prog = stmmac_xdp_is_enabled(priv);
1968 struct page_pool_params pp_params = { 0 };
1969 unsigned int num_pages;
1970 unsigned int napi_id;
1971 int ret;
1972
1973 rx_q->queue_index = queue;
1974 rx_q->priv_data = priv;
1975
1976 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
1977 pp_params.pool_size = priv->dma_rx_size;
1978 num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE);
1979 pp_params.order = ilog2(num_pages);
1980 pp_params.nid = dev_to_node(priv->device);
1981 pp_params.dev = priv->device;
1982 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
1983 pp_params.offset = stmmac_rx_offset(priv);
1984 pp_params.max_len = STMMAC_MAX_RX_BUF_SIZE(num_pages);
1985
1986 rx_q->page_pool = page_pool_create(&pp_params);
1987 if (IS_ERR(rx_q->page_pool)) {
1988 ret = PTR_ERR(rx_q->page_pool);
1989 rx_q->page_pool = NULL;
1990 return ret;
1991 }
1992
1993 rx_q->buf_pool = kcalloc(priv->dma_rx_size,
1994 sizeof(*rx_q->buf_pool),
1995 GFP_KERNEL);
1996 if (!rx_q->buf_pool)
1997 return -ENOMEM;
1998
1999 if (priv->extend_desc) {
2000 rx_q->dma_erx = dma_alloc_coherent(priv->device,
2001 priv->dma_rx_size *
2002 sizeof(struct dma_extended_desc),
2003 &rx_q->dma_rx_phy,
2004 GFP_KERNEL);
2005 if (!rx_q->dma_erx)
2006 return -ENOMEM;
2007
2008 } else {
2009 rx_q->dma_rx = dma_alloc_coherent(priv->device,
2010 priv->dma_rx_size *
2011 sizeof(struct dma_desc),
2012 &rx_q->dma_rx_phy,
2013 GFP_KERNEL);
2014 if (!rx_q->dma_rx)
2015 return -ENOMEM;
2016 }
2017
2018 if (stmmac_xdp_is_enabled(priv) &&
2019 test_bit(queue, priv->af_xdp_zc_qps))
2020 napi_id = ch->rxtx_napi.napi_id;
2021 else
2022 napi_id = ch->rx_napi.napi_id;
2023
2024 ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev,
2025 rx_q->queue_index,
2026 napi_id);
2027 if (ret) {
2028 netdev_err(priv->dev, "Failed to register xdp rxq info\n");
2029 return -EINVAL;
2030 }
2031
2032 return 0;
2033}
2034
2035static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
2036{
2037 u32 rx_count = priv->plat->rx_queues_to_use;
2038 u32 queue;
2039 int ret;
2040
2041
2042 for (queue = 0; queue < rx_count; queue++) {
2043 ret = __alloc_dma_rx_desc_resources(priv, queue);
2044 if (ret)
2045 goto err_dma;
2046 }
2047
2048 return 0;
2049
2050err_dma:
2051 free_dma_rx_desc_resources(priv);
2052
2053 return ret;
2054}
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue)
2066{
2067 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2068 size_t size;
2069 void *addr;
2070
2071 tx_q->queue_index = queue;
2072 tx_q->priv_data = priv;
2073
2074 tx_q->tx_skbuff_dma = kcalloc(priv->dma_tx_size,
2075 sizeof(*tx_q->tx_skbuff_dma),
2076 GFP_KERNEL);
2077 if (!tx_q->tx_skbuff_dma)
2078 return -ENOMEM;
2079
2080 tx_q->tx_skbuff = kcalloc(priv->dma_tx_size,
2081 sizeof(struct sk_buff *),
2082 GFP_KERNEL);
2083 if (!tx_q->tx_skbuff)
2084 return -ENOMEM;
2085
2086 if (priv->extend_desc)
2087 size = sizeof(struct dma_extended_desc);
2088 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2089 size = sizeof(struct dma_edesc);
2090 else
2091 size = sizeof(struct dma_desc);
2092
2093 size *= priv->dma_tx_size;
2094
2095 addr = dma_alloc_coherent(priv->device, size,
2096 &tx_q->dma_tx_phy, GFP_KERNEL);
2097 if (!addr)
2098 return -ENOMEM;
2099
2100 if (priv->extend_desc)
2101 tx_q->dma_etx = addr;
2102 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2103 tx_q->dma_entx = addr;
2104 else
2105 tx_q->dma_tx = addr;
2106
2107 return 0;
2108}
2109
2110static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
2111{
2112 u32 tx_count = priv->plat->tx_queues_to_use;
2113 u32 queue;
2114 int ret;
2115
2116
2117 for (queue = 0; queue < tx_count; queue++) {
2118 ret = __alloc_dma_tx_desc_resources(priv, queue);
2119 if (ret)
2120 goto err_dma;
2121 }
2122
2123 return 0;
2124
2125err_dma:
2126 free_dma_tx_desc_resources(priv);
2127 return ret;
2128}
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138static int alloc_dma_desc_resources(struct stmmac_priv *priv)
2139{
2140
2141 int ret = alloc_dma_rx_desc_resources(priv);
2142
2143 if (ret)
2144 return ret;
2145
2146 ret = alloc_dma_tx_desc_resources(priv);
2147
2148 return ret;
2149}
2150
2151
2152
2153
2154
2155static void free_dma_desc_resources(struct stmmac_priv *priv)
2156{
2157
2158 free_dma_tx_desc_resources(priv);
2159
2160
2161
2162
2163 free_dma_rx_desc_resources(priv);
2164}
2165
2166
2167
2168
2169
2170
2171static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
2172{
2173 u32 rx_queues_count = priv->plat->rx_queues_to_use;
2174 int queue;
2175 u8 mode;
2176
2177 for (queue = 0; queue < rx_queues_count; queue++) {
2178 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
2179 stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
2180 }
2181}
2182
2183
2184
2185
2186
2187
2188
2189
2190static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
2191{
2192 netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
2193 stmmac_start_rx(priv, priv->ioaddr, chan);
2194}
2195
2196
2197
2198
2199
2200
2201
2202
2203static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
2204{
2205 netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
2206 stmmac_start_tx(priv, priv->ioaddr, chan);
2207}
2208
2209
2210
2211
2212
2213
2214
2215
2216static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
2217{
2218 netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
2219 stmmac_stop_rx(priv, priv->ioaddr, chan);
2220}
2221
2222
2223
2224
2225
2226
2227
2228
2229static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
2230{
2231 netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
2232 stmmac_stop_tx(priv, priv->ioaddr, chan);
2233}
2234
2235
2236
2237
2238
2239
2240
2241static void stmmac_start_all_dma(struct stmmac_priv *priv)
2242{
2243 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2244 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2245 u32 chan = 0;
2246
2247 for (chan = 0; chan < rx_channels_count; chan++)
2248 stmmac_start_rx_dma(priv, chan);
2249
2250 for (chan = 0; chan < tx_channels_count; chan++)
2251 stmmac_start_tx_dma(priv, chan);
2252}
2253
2254
2255
2256
2257
2258
2259
2260static void stmmac_stop_all_dma(struct stmmac_priv *priv)
2261{
2262 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2263 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2264 u32 chan = 0;
2265
2266 for (chan = 0; chan < rx_channels_count; chan++)
2267 stmmac_stop_rx_dma(priv, chan);
2268
2269 for (chan = 0; chan < tx_channels_count; chan++)
2270 stmmac_stop_tx_dma(priv, chan);
2271}
2272
2273
2274
2275
2276
2277
2278
2279static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
2280{
2281 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2282 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2283 int rxfifosz = priv->plat->rx_fifo_size;
2284 int txfifosz = priv->plat->tx_fifo_size;
2285 u32 txmode = 0;
2286 u32 rxmode = 0;
2287 u32 chan = 0;
2288 u8 qmode = 0;
2289
2290 if (rxfifosz == 0)
2291 rxfifosz = priv->dma_cap.rx_fifo_size;
2292 if (txfifosz == 0)
2293 txfifosz = priv->dma_cap.tx_fifo_size;
2294
2295
2296 rxfifosz /= rx_channels_count;
2297 txfifosz /= tx_channels_count;
2298
2299 if (priv->plat->force_thresh_dma_mode) {
2300 txmode = tc;
2301 rxmode = tc;
2302 } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
2303
2304
2305
2306
2307
2308
2309
2310 txmode = SF_DMA_MODE;
2311 rxmode = SF_DMA_MODE;
2312 priv->xstats.threshold = SF_DMA_MODE;
2313 } else {
2314 txmode = tc;
2315 rxmode = SF_DMA_MODE;
2316 }
2317
2318
2319 for (chan = 0; chan < rx_channels_count; chan++) {
2320 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
2321 u32 buf_size;
2322
2323 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2324
2325 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
2326 rxfifosz, qmode);
2327
2328 if (rx_q->xsk_pool) {
2329 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
2330 stmmac_set_dma_bfsize(priv, priv->ioaddr,
2331 buf_size,
2332 chan);
2333 } else {
2334 stmmac_set_dma_bfsize(priv, priv->ioaddr,
2335 priv->dma_buf_sz,
2336 chan);
2337 }
2338 }
2339
2340 for (chan = 0; chan < tx_channels_count; chan++) {
2341 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2342
2343 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
2344 txfifosz, qmode);
2345 }
2346}
2347
2348static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
2349{
2350 struct netdev_queue *nq = netdev_get_tx_queue(priv->dev, queue);
2351 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2352 struct xsk_buff_pool *pool = tx_q->xsk_pool;
2353 unsigned int entry = tx_q->cur_tx;
2354 struct dma_desc *tx_desc = NULL;
2355 struct xdp_desc xdp_desc;
2356 bool work_done = true;
2357
2358
2359 nq->trans_start = jiffies;
2360
2361 budget = min(budget, stmmac_tx_avail(priv, queue));
2362
2363 while (budget-- > 0) {
2364 dma_addr_t dma_addr;
2365 bool set_ic;
2366
2367
2368
2369
2370 if (unlikely(stmmac_tx_avail(priv, queue) < STMMAC_TX_XSK_AVAIL) ||
2371 !netif_carrier_ok(priv->dev)) {
2372 work_done = false;
2373 break;
2374 }
2375
2376 if (!xsk_tx_peek_desc(pool, &xdp_desc))
2377 break;
2378
2379 if (likely(priv->extend_desc))
2380 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
2381 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2382 tx_desc = &tx_q->dma_entx[entry].basic;
2383 else
2384 tx_desc = tx_q->dma_tx + entry;
2385
2386 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
2387 xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len);
2388
2389 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX;
2390
2391
2392
2393
2394
2395 tx_q->tx_skbuff_dma[entry].buf = 0;
2396 tx_q->xdpf[entry] = NULL;
2397
2398 tx_q->tx_skbuff_dma[entry].map_as_page = false;
2399 tx_q->tx_skbuff_dma[entry].len = xdp_desc.len;
2400 tx_q->tx_skbuff_dma[entry].last_segment = true;
2401 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
2402
2403 stmmac_set_desc_addr(priv, tx_desc, dma_addr);
2404
2405 tx_q->tx_count_frames++;
2406
2407 if (!priv->tx_coal_frames[queue])
2408 set_ic = false;
2409 else if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)
2410 set_ic = true;
2411 else
2412 set_ic = false;
2413
2414 if (set_ic) {
2415 tx_q->tx_count_frames = 0;
2416 stmmac_set_tx_ic(priv, tx_desc);
2417 priv->xstats.tx_set_ic_bit++;
2418 }
2419
2420 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len,
2421 true, priv->mode, true, true,
2422 xdp_desc.len);
2423
2424 stmmac_enable_dma_transmission(priv, priv->ioaddr);
2425
2426 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
2427 entry = tx_q->cur_tx;
2428 }
2429
2430 if (tx_desc) {
2431 stmmac_flush_tx_descriptors(priv, queue);
2432 xsk_tx_release(pool);
2433 }
2434
2435
2436
2437
2438
2439
2440 return !!budget && work_done;
2441}
2442
2443
2444
2445
2446
2447
2448
2449
2450static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
2451{
2452 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2453 unsigned int bytes_compl = 0, pkts_compl = 0;
2454 unsigned int entry, xmits = 0, count = 0;
2455
2456 __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
2457
2458 priv->xstats.tx_clean++;
2459
2460 tx_q->xsk_frames_done = 0;
2461
2462 entry = tx_q->dirty_tx;
2463
2464
2465 while ((entry != tx_q->cur_tx) && count < priv->dma_tx_size) {
2466 struct xdp_frame *xdpf;
2467 struct sk_buff *skb;
2468 struct dma_desc *p;
2469 int status;
2470
2471 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX ||
2472 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
2473 xdpf = tx_q->xdpf[entry];
2474 skb = NULL;
2475 } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
2476 xdpf = NULL;
2477 skb = tx_q->tx_skbuff[entry];
2478 } else {
2479 xdpf = NULL;
2480 skb = NULL;
2481 }
2482
2483 if (priv->extend_desc)
2484 p = (struct dma_desc *)(tx_q->dma_etx + entry);
2485 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
2486 p = &tx_q->dma_entx[entry].basic;
2487 else
2488 p = tx_q->dma_tx + entry;
2489
2490 status = stmmac_tx_status(priv, &priv->dev->stats,
2491 &priv->xstats, p, priv->ioaddr);
2492
2493 if (unlikely(status & tx_dma_own))
2494 break;
2495
2496 count++;
2497
2498
2499
2500
2501 dma_rmb();
2502
2503
2504 if (likely(!(status & tx_not_ls))) {
2505
2506 if (unlikely(status & tx_err)) {
2507 priv->dev->stats.tx_errors++;
2508 } else {
2509 priv->dev->stats.tx_packets++;
2510 priv->xstats.tx_pkt_n++;
2511 priv->xstats.txq_stats[queue].tx_pkt_n++;
2512 }
2513 if (skb)
2514 stmmac_get_tx_hwtstamp(priv, p, skb);
2515 }
2516
2517 if (likely(tx_q->tx_skbuff_dma[entry].buf &&
2518 tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) {
2519 if (tx_q->tx_skbuff_dma[entry].map_as_page)
2520 dma_unmap_page(priv->device,
2521 tx_q->tx_skbuff_dma[entry].buf,
2522 tx_q->tx_skbuff_dma[entry].len,
2523 DMA_TO_DEVICE);
2524 else
2525 dma_unmap_single(priv->device,
2526 tx_q->tx_skbuff_dma[entry].buf,
2527 tx_q->tx_skbuff_dma[entry].len,
2528 DMA_TO_DEVICE);
2529 tx_q->tx_skbuff_dma[entry].buf = 0;
2530 tx_q->tx_skbuff_dma[entry].len = 0;
2531 tx_q->tx_skbuff_dma[entry].map_as_page = false;
2532 }
2533
2534 stmmac_clean_desc3(priv, tx_q, p);
2535
2536 tx_q->tx_skbuff_dma[entry].last_segment = false;
2537 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
2538
2539 if (xdpf &&
2540 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_TX) {
2541 xdp_return_frame_rx_napi(xdpf);
2542 tx_q->xdpf[entry] = NULL;
2543 }
2544
2545 if (xdpf &&
2546 tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) {
2547 xdp_return_frame(xdpf);
2548 tx_q->xdpf[entry] = NULL;
2549 }
2550
2551 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX)
2552 tx_q->xsk_frames_done++;
2553
2554 if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) {
2555 if (likely(skb)) {
2556 pkts_compl++;
2557 bytes_compl += skb->len;
2558 dev_consume_skb_any(skb);
2559 tx_q->tx_skbuff[entry] = NULL;
2560 }
2561 }
2562
2563 stmmac_release_tx_desc(priv, p, priv->mode);
2564
2565 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
2566 }
2567 tx_q->dirty_tx = entry;
2568
2569 netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
2570 pkts_compl, bytes_compl);
2571
2572 if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
2573 queue))) &&
2574 stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH(priv)) {
2575
2576 netif_dbg(priv, tx_done, priv->dev,
2577 "%s: restart transmit\n", __func__);
2578 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
2579 }
2580
2581 if (tx_q->xsk_pool) {
2582 bool work_done;
2583
2584 if (tx_q->xsk_frames_done)
2585 xsk_tx_completed(tx_q->xsk_pool, tx_q->xsk_frames_done);
2586
2587 if (xsk_uses_need_wakeup(tx_q->xsk_pool))
2588 xsk_set_tx_need_wakeup(tx_q->xsk_pool);
2589
2590
2591
2592
2593
2594
2595 work_done = stmmac_xdp_xmit_zc(priv, queue,
2596 STMMAC_XSK_TX_BUDGET_MAX);
2597 if (work_done)
2598 xmits = budget - 1;
2599 else
2600 xmits = budget;
2601 }
2602
2603 if (priv->eee_enabled && !priv->tx_path_in_lpi_mode &&
2604 priv->eee_sw_timer_en) {
2605 stmmac_enable_eee_mode(priv);
2606 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(priv->tx_lpi_timer));
2607 }
2608
2609
2610 if (tx_q->dirty_tx != tx_q->cur_tx)
2611 hrtimer_start(&tx_q->txtimer,
2612 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]),
2613 HRTIMER_MODE_REL);
2614
2615 __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
2616
2617
2618 return max(count, xmits);
2619}
2620
2621
2622
2623
2624
2625
2626
2627
2628static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
2629{
2630 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2631
2632 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
2633
2634 stmmac_stop_tx_dma(priv, chan);
2635 dma_free_tx_skbufs(priv, chan);
2636 stmmac_clear_tx_descriptors(priv, chan);
2637 tx_q->dirty_tx = 0;
2638 tx_q->cur_tx = 0;
2639 tx_q->mss = 0;
2640 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
2641 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2642 tx_q->dma_tx_phy, chan);
2643 stmmac_start_tx_dma(priv, chan);
2644
2645 priv->dev->stats.tx_errors++;
2646 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
2647}
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
2660 u32 rxmode, u32 chan)
2661{
2662 u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2663 u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2664 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2665 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2666 int rxfifosz = priv->plat->rx_fifo_size;
2667 int txfifosz = priv->plat->tx_fifo_size;
2668
2669 if (rxfifosz == 0)
2670 rxfifosz = priv->dma_cap.rx_fifo_size;
2671 if (txfifosz == 0)
2672 txfifosz = priv->dma_cap.tx_fifo_size;
2673
2674
2675 rxfifosz /= rx_channels_count;
2676 txfifosz /= tx_channels_count;
2677
2678 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2679 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
2680}
2681
2682static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2683{
2684 int ret;
2685
2686 ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2687 priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2688 if (ret && (ret != -EINVAL)) {
2689 stmmac_global_err(priv);
2690 return true;
2691 }
2692
2693 return false;
2694}
2695
2696static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir)
2697{
2698 int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2699 &priv->xstats, chan, dir);
2700 struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
2701 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2702 struct stmmac_channel *ch = &priv->channel[chan];
2703 struct napi_struct *rx_napi;
2704 struct napi_struct *tx_napi;
2705 unsigned long flags;
2706
2707 rx_napi = rx_q->xsk_pool ? &ch->rxtx_napi : &ch->rx_napi;
2708 tx_napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
2709
2710 if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
2711 if (napi_schedule_prep(rx_napi)) {
2712 spin_lock_irqsave(&ch->lock, flags);
2713 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
2714 spin_unlock_irqrestore(&ch->lock, flags);
2715 __napi_schedule(rx_napi);
2716 }
2717 }
2718
2719 if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use)) {
2720 if (napi_schedule_prep(tx_napi)) {
2721 spin_lock_irqsave(&ch->lock, flags);
2722 stmmac_disable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
2723 spin_unlock_irqrestore(&ch->lock, flags);
2724 __napi_schedule(tx_napi);
2725 }
2726 }
2727
2728 return status;
2729}
2730
2731
2732
2733
2734
2735
2736
2737
2738static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2739{
2740 u32 tx_channel_count = priv->plat->tx_queues_to_use;
2741 u32 rx_channel_count = priv->plat->rx_queues_to_use;
2742 u32 channels_to_check = tx_channel_count > rx_channel_count ?
2743 tx_channel_count : rx_channel_count;
2744 u32 chan;
2745 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
2746
2747
2748 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
2749 channels_to_check = ARRAY_SIZE(status);
2750
2751 for (chan = 0; chan < channels_to_check; chan++)
2752 status[chan] = stmmac_napi_check(priv, chan,
2753 DMA_DIR_RXTX);
2754
2755 for (chan = 0; chan < tx_channel_count; chan++) {
2756 if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
2757
2758 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2759 (tc <= 256)) {
2760 tc += 64;
2761 if (priv->plat->force_thresh_dma_mode)
2762 stmmac_set_dma_operation_mode(priv,
2763 tc,
2764 tc,
2765 chan);
2766 else
2767 stmmac_set_dma_operation_mode(priv,
2768 tc,
2769 SF_DMA_MODE,
2770 chan);
2771 priv->xstats.threshold = tc;
2772 }
2773 } else if (unlikely(status[chan] == tx_hard_error)) {
2774 stmmac_tx_err(priv, chan);
2775 }
2776 }
2777}
2778
2779
2780
2781
2782
2783
2784static void stmmac_mmc_setup(struct stmmac_priv *priv)
2785{
2786 unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2787 MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2788
2789 stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
2790
2791 if (priv->dma_cap.rmon) {
2792 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
2793 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2794 } else
2795 netdev_info(priv->dev, "No MAC Management Counters available\n");
2796}
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807static int stmmac_get_hw_features(struct stmmac_priv *priv)
2808{
2809 return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
2810}
2811
2812
2813
2814
2815
2816
2817
2818
2819static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2820{
2821 if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2822 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
2823 if (!is_valid_ether_addr(priv->dev->dev_addr))
2824 eth_hw_addr_random(priv->dev);
2825 dev_info(priv->device, "device MAC address %pM\n",
2826 priv->dev->dev_addr);
2827 }
2828}
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2839{
2840 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2841 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2842 u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2843 struct stmmac_rx_queue *rx_q;
2844 struct stmmac_tx_queue *tx_q;
2845 u32 chan = 0;
2846 int atds = 0;
2847 int ret = 0;
2848
2849 if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2850 dev_err(priv->device, "Invalid DMA configuration\n");
2851 return -EINVAL;
2852 }
2853
2854 if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2855 atds = 1;
2856
2857 ret = stmmac_reset(priv, priv->ioaddr);
2858 if (ret) {
2859 dev_err(priv->device, "Failed to reset the dma\n");
2860 return ret;
2861 }
2862
2863
2864 stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
2865
2866 if (priv->plat->axi)
2867 stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2868
2869
2870 for (chan = 0; chan < dma_csr_ch; chan++)
2871 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
2872
2873
2874 for (chan = 0; chan < rx_channels_count; chan++) {
2875 rx_q = &priv->rx_queue[chan];
2876
2877 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2878 rx_q->dma_rx_phy, chan);
2879
2880 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2881 (rx_q->buf_alloc_num *
2882 sizeof(struct dma_desc));
2883 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
2884 rx_q->rx_tail_addr, chan);
2885 }
2886
2887
2888 for (chan = 0; chan < tx_channels_count; chan++) {
2889 tx_q = &priv->tx_queue[chan];
2890
2891 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2892 tx_q->dma_tx_phy, chan);
2893
2894 tx_q->tx_tail_addr = tx_q->dma_tx_phy;
2895 stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
2896 tx_q->tx_tail_addr, chan);
2897 }
2898
2899 return ret;
2900}
2901
2902static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2903{
2904 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2905
2906 hrtimer_start(&tx_q->txtimer,
2907 STMMAC_COAL_TIMER(priv->tx_coal_timer[queue]),
2908 HRTIMER_MODE_REL);
2909}
2910
2911
2912
2913
2914
2915
2916
2917static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t)
2918{
2919 struct stmmac_tx_queue *tx_q = container_of(t, struct stmmac_tx_queue, txtimer);
2920 struct stmmac_priv *priv = tx_q->priv_data;
2921 struct stmmac_channel *ch;
2922 struct napi_struct *napi;
2923
2924 ch = &priv->channel[tx_q->queue_index];
2925 napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi;
2926
2927 if (likely(napi_schedule_prep(napi))) {
2928 unsigned long flags;
2929
2930 spin_lock_irqsave(&ch->lock, flags);
2931 stmmac_disable_dma_irq(priv, priv->ioaddr, ch->index, 0, 1);
2932 spin_unlock_irqrestore(&ch->lock, flags);
2933 __napi_schedule(napi);
2934 }
2935
2936 return HRTIMER_NORESTART;
2937}
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947static void stmmac_init_coalesce(struct stmmac_priv *priv)
2948{
2949 u32 tx_channel_count = priv->plat->tx_queues_to_use;
2950 u32 rx_channel_count = priv->plat->rx_queues_to_use;
2951 u32 chan;
2952
2953 for (chan = 0; chan < tx_channel_count; chan++) {
2954 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2955
2956 priv->tx_coal_frames[chan] = STMMAC_TX_FRAMES;
2957 priv->tx_coal_timer[chan] = STMMAC_COAL_TX_TIMER;
2958
2959 hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2960 tx_q->txtimer.function = stmmac_tx_timer;
2961 }
2962
2963 for (chan = 0; chan < rx_channel_count; chan++)
2964 priv->rx_coal_frames[chan] = STMMAC_RX_FRAMES;
2965}
2966
2967static void stmmac_set_rings_length(struct stmmac_priv *priv)
2968{
2969 u32 rx_channels_count = priv->plat->rx_queues_to_use;
2970 u32 tx_channels_count = priv->plat->tx_queues_to_use;
2971 u32 chan;
2972
2973
2974 for (chan = 0; chan < tx_channels_count; chan++)
2975 stmmac_set_tx_ring_len(priv, priv->ioaddr,
2976 (priv->dma_tx_size - 1), chan);
2977
2978
2979 for (chan = 0; chan < rx_channels_count; chan++)
2980 stmmac_set_rx_ring_len(priv, priv->ioaddr,
2981 (priv->dma_rx_size - 1), chan);
2982}
2983
2984
2985
2986
2987
2988
2989static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2990{
2991 u32 tx_queues_count = priv->plat->tx_queues_to_use;
2992 u32 weight;
2993 u32 queue;
2994
2995 for (queue = 0; queue < tx_queues_count; queue++) {
2996 weight = priv->plat->tx_queues_cfg[queue].weight;
2997 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
2998 }
2999}
3000
3001
3002
3003
3004
3005
3006static void stmmac_configure_cbs(struct stmmac_priv *priv)
3007{
3008 u32 tx_queues_count = priv->plat->tx_queues_to_use;
3009 u32 mode_to_use;
3010 u32 queue;
3011
3012
3013 for (queue = 1; queue < tx_queues_count; queue++) {
3014 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
3015 if (mode_to_use == MTL_QUEUE_DCB)
3016 continue;
3017
3018 stmmac_config_cbs(priv, priv->hw,
3019 priv->plat->tx_queues_cfg[queue].send_slope,
3020 priv->plat->tx_queues_cfg[queue].idle_slope,
3021 priv->plat->tx_queues_cfg[queue].high_credit,
3022 priv->plat->tx_queues_cfg[queue].low_credit,
3023 queue);
3024 }
3025}
3026
3027
3028
3029
3030
3031
3032static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
3033{
3034 u32 rx_queues_count = priv->plat->rx_queues_to_use;
3035 u32 queue;
3036 u32 chan;
3037
3038 for (queue = 0; queue < rx_queues_count; queue++) {
3039 chan = priv->plat->rx_queues_cfg[queue].chan;
3040 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
3041 }
3042}
3043
3044
3045
3046
3047
3048
3049static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
3050{
3051 u32 rx_queues_count = priv->plat->rx_queues_to_use;
3052 u32 queue;
3053 u32 prio;
3054
3055 for (queue = 0; queue < rx_queues_count; queue++) {
3056 if (!priv->plat->rx_queues_cfg[queue].use_prio)
3057 continue;
3058
3059 prio = priv->plat->rx_queues_cfg[queue].prio;
3060 stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
3061 }
3062}
3063
3064
3065
3066
3067
3068
3069static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
3070{
3071 u32 tx_queues_count = priv->plat->tx_queues_to_use;
3072 u32 queue;
3073 u32 prio;
3074
3075 for (queue = 0; queue < tx_queues_count; queue++) {
3076 if (!priv->plat->tx_queues_cfg[queue].use_prio)
3077 continue;
3078
3079 prio = priv->plat->tx_queues_cfg[queue].prio;
3080 stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
3081 }
3082}
3083
3084
3085
3086
3087
3088
3089static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
3090{
3091 u32 rx_queues_count = priv->plat->rx_queues_to_use;
3092 u32 queue;
3093 u8 packet;
3094
3095 for (queue = 0; queue < rx_queues_count; queue++) {
3096
3097 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
3098 continue;
3099
3100 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
3101 stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
3102 }
3103}
3104
3105static void stmmac_mac_config_rss(struct stmmac_priv *priv)
3106{
3107 if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
3108 priv->rss.enable = false;
3109 return;
3110 }
3111
3112 if (priv->dev->features & NETIF_F_RXHASH)
3113 priv->rss.enable = true;
3114 else
3115 priv->rss.enable = false;
3116
3117 stmmac_rss_configure(priv, priv->hw, &priv->rss,
3118 priv->plat->rx_queues_to_use);
3119}
3120
3121
3122
3123
3124
3125
3126static void stmmac_mtl_configuration(struct stmmac_priv *priv)
3127{
3128 u32 rx_queues_count = priv->plat->rx_queues_to_use;
3129 u32 tx_queues_count = priv->plat->tx_queues_to_use;
3130
3131 if (tx_queues_count > 1)
3132 stmmac_set_tx_queue_weight(priv);
3133
3134
3135 if (rx_queues_count > 1)
3136 stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
3137 priv->plat->rx_sched_algorithm);
3138
3139
3140 if (tx_queues_count > 1)
3141 stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
3142 priv->plat->tx_sched_algorithm);
3143
3144
3145 if (tx_queues_count > 1)
3146 stmmac_configure_cbs(priv);
3147
3148
3149 stmmac_rx_queue_dma_chan_map(priv);
3150
3151
3152 stmmac_mac_enable_rx_queues(priv);
3153
3154
3155 if (rx_queues_count > 1)
3156 stmmac_mac_config_rx_queues_prio(priv);
3157
3158
3159 if (tx_queues_count > 1)
3160 stmmac_mac_config_tx_queues_prio(priv);
3161
3162
3163 if (rx_queues_count > 1)
3164 stmmac_mac_config_rx_queues_routing(priv);
3165
3166
3167 if (rx_queues_count > 1)
3168 stmmac_mac_config_rss(priv);
3169}
3170
3171static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
3172{
3173 if (priv->dma_cap.asp) {
3174 netdev_info(priv->dev, "Enabling Safety Features\n");
3175 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp,
3176 priv->plat->safety_feat_cfg);
3177 } else {
3178 netdev_info(priv->dev, "No Safety Features support found\n");
3179 }
3180}
3181
3182static int stmmac_fpe_start_wq(struct stmmac_priv *priv)
3183{
3184 char *name;
3185
3186 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state);
3187 clear_bit(__FPE_REMOVING, &priv->fpe_task_state);
3188
3189 name = priv->wq_name;
3190 sprintf(name, "%s-fpe", priv->dev->name);
3191
3192 priv->fpe_wq = create_singlethread_workqueue(name);
3193 if (!priv->fpe_wq) {
3194 netdev_err(priv->dev, "%s: Failed to create workqueue\n", name);
3195
3196 return -ENOMEM;
3197 }
3198 netdev_info(priv->dev, "FPE workqueue start");
3199
3200 return 0;
3201}
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
3217{
3218 struct stmmac_priv *priv = netdev_priv(dev);
3219 u32 rx_cnt = priv->plat->rx_queues_to_use;
3220 u32 tx_cnt = priv->plat->tx_queues_to_use;
3221 bool sph_en;
3222 u32 chan;
3223 int ret;
3224
3225
3226 ret = stmmac_init_dma_engine(priv);
3227 if (ret < 0) {
3228 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
3229 __func__);
3230 return ret;
3231 }
3232
3233
3234 stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
3235
3236
3237 if (priv->hw->pcs) {
3238 int speed = priv->plat->mac_port_sel_speed;
3239
3240 if ((speed == SPEED_10) || (speed == SPEED_100) ||
3241 (speed == SPEED_1000)) {
3242 priv->hw->ps = speed;
3243 } else {
3244 dev_warn(priv->device, "invalid port speed\n");
3245 priv->hw->ps = 0;
3246 }
3247 }
3248
3249
3250 stmmac_core_init(priv, priv->hw, dev);
3251
3252
3253 stmmac_mtl_configuration(priv);
3254
3255
3256 stmmac_safety_feat_configuration(priv);
3257
3258 ret = stmmac_rx_ipc(priv, priv->hw);
3259 if (!ret) {
3260 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
3261 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
3262 priv->hw->rx_csum = 0;
3263 }
3264
3265
3266 stmmac_mac_set(priv, priv->ioaddr, true);
3267
3268
3269 stmmac_dma_operation_mode(priv);
3270
3271 stmmac_mmc_setup(priv);
3272
3273 if (init_ptp) {
3274 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
3275 if (ret < 0)
3276 netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
3277
3278 ret = stmmac_init_ptp(priv);
3279 if (ret == -EOPNOTSUPP)
3280 netdev_warn(priv->dev, "PTP not supported by HW\n");
3281 else if (ret)
3282 netdev_warn(priv->dev, "PTP init failed\n");
3283 }
3284
3285 priv->eee_tw_timer = STMMAC_DEFAULT_TWT_LS;
3286
3287
3288 if (!priv->tx_lpi_timer)
3289 priv->tx_lpi_timer = eee_timer * 1000;
3290
3291 if (priv->use_riwt) {
3292 u32 queue;
3293
3294 for (queue = 0; queue < rx_cnt; queue++) {
3295 if (!priv->rx_riwt[queue])
3296 priv->rx_riwt[queue] = DEF_DMA_RIWT;
3297
3298 stmmac_rx_watchdog(priv, priv->ioaddr,
3299 priv->rx_riwt[queue], queue);
3300 }
3301 }
3302
3303 if (priv->hw->pcs)
3304 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
3305
3306
3307 stmmac_set_rings_length(priv);
3308
3309
3310 if (priv->tso) {
3311 for (chan = 0; chan < tx_cnt; chan++) {
3312 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
3313
3314
3315 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3316 continue;
3317
3318 stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
3319 }
3320 }
3321
3322
3323 sph_en = (priv->hw->rx_csum > 0) && priv->sph;
3324 for (chan = 0; chan < rx_cnt; chan++)
3325 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
3326
3327
3328
3329 if (priv->dma_cap.vlins)
3330 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
3331
3332
3333 for (chan = 0; chan < tx_cnt; chan++) {
3334 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
3335 int enable = tx_q->tbs & STMMAC_TBS_AVAIL;
3336
3337 stmmac_enable_tbs(priv, priv->ioaddr, enable, chan);
3338 }
3339
3340
3341 netif_set_real_num_rx_queues(dev, priv->plat->rx_queues_to_use);
3342 netif_set_real_num_tx_queues(dev, priv->plat->tx_queues_to_use);
3343
3344
3345 stmmac_start_all_dma(priv);
3346
3347 if (priv->dma_cap.fpesel) {
3348 stmmac_fpe_start_wq(priv);
3349
3350 if (priv->plat->fpe_cfg->enable)
3351 stmmac_fpe_handshake(priv, true);
3352 }
3353
3354 return 0;
3355}
3356
3357static void stmmac_hw_teardown(struct net_device *dev)
3358{
3359 struct stmmac_priv *priv = netdev_priv(dev);
3360
3361 clk_disable_unprepare(priv->plat->clk_ptp_ref);
3362}
3363
3364static void stmmac_free_irq(struct net_device *dev,
3365 enum request_irq_err irq_err, int irq_idx)
3366{
3367 struct stmmac_priv *priv = netdev_priv(dev);
3368 int j;
3369
3370 switch (irq_err) {
3371 case REQ_IRQ_ERR_ALL:
3372 irq_idx = priv->plat->tx_queues_to_use;
3373 fallthrough;
3374 case REQ_IRQ_ERR_TX:
3375 for (j = irq_idx - 1; j >= 0; j--) {
3376 if (priv->tx_irq[j] > 0) {
3377 irq_set_affinity_hint(priv->tx_irq[j], NULL);
3378 free_irq(priv->tx_irq[j], &priv->tx_queue[j]);
3379 }
3380 }
3381 irq_idx = priv->plat->rx_queues_to_use;
3382 fallthrough;
3383 case REQ_IRQ_ERR_RX:
3384 for (j = irq_idx - 1; j >= 0; j--) {
3385 if (priv->rx_irq[j] > 0) {
3386 irq_set_affinity_hint(priv->rx_irq[j], NULL);
3387 free_irq(priv->rx_irq[j], &priv->rx_queue[j]);
3388 }
3389 }
3390
3391 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq)
3392 free_irq(priv->sfty_ue_irq, dev);
3393 fallthrough;
3394 case REQ_IRQ_ERR_SFTY_UE:
3395 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq)
3396 free_irq(priv->sfty_ce_irq, dev);
3397 fallthrough;
3398 case REQ_IRQ_ERR_SFTY_CE:
3399 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq)
3400 free_irq(priv->lpi_irq, dev);
3401 fallthrough;
3402 case REQ_IRQ_ERR_LPI:
3403 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq)
3404 free_irq(priv->wol_irq, dev);
3405 fallthrough;
3406 case REQ_IRQ_ERR_WOL:
3407 free_irq(dev->irq, dev);
3408 fallthrough;
3409 case REQ_IRQ_ERR_MAC:
3410 case REQ_IRQ_ERR_NO:
3411
3412 break;
3413 }
3414}
3415
3416static int stmmac_request_irq_multi_msi(struct net_device *dev)
3417{
3418 struct stmmac_priv *priv = netdev_priv(dev);
3419 enum request_irq_err irq_err;
3420 cpumask_t cpu_mask;
3421 int irq_idx = 0;
3422 char *int_name;
3423 int ret;
3424 int i;
3425
3426
3427 int_name = priv->int_name_mac;
3428 sprintf(int_name, "%s:%s", dev->name, "mac");
3429 ret = request_irq(dev->irq, stmmac_mac_interrupt,
3430 0, int_name, dev);
3431 if (unlikely(ret < 0)) {
3432 netdev_err(priv->dev,
3433 "%s: alloc mac MSI %d (error: %d)\n",
3434 __func__, dev->irq, ret);
3435 irq_err = REQ_IRQ_ERR_MAC;
3436 goto irq_error;
3437 }
3438
3439
3440
3441
3442 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
3443 int_name = priv->int_name_wol;
3444 sprintf(int_name, "%s:%s", dev->name, "wol");
3445 ret = request_irq(priv->wol_irq,
3446 stmmac_mac_interrupt,
3447 0, int_name, dev);
3448 if (unlikely(ret < 0)) {
3449 netdev_err(priv->dev,
3450 "%s: alloc wol MSI %d (error: %d)\n",
3451 __func__, priv->wol_irq, ret);
3452 irq_err = REQ_IRQ_ERR_WOL;
3453 goto irq_error;
3454 }
3455 }
3456
3457
3458
3459
3460 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {
3461 int_name = priv->int_name_lpi;
3462 sprintf(int_name, "%s:%s", dev->name, "lpi");
3463 ret = request_irq(priv->lpi_irq,
3464 stmmac_mac_interrupt,
3465 0, int_name, dev);
3466 if (unlikely(ret < 0)) {
3467 netdev_err(priv->dev,
3468 "%s: alloc lpi MSI %d (error: %d)\n",
3469 __func__, priv->lpi_irq, ret);
3470 irq_err = REQ_IRQ_ERR_LPI;
3471 goto irq_error;
3472 }
3473 }
3474
3475
3476
3477
3478 if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) {
3479 int_name = priv->int_name_sfty_ce;
3480 sprintf(int_name, "%s:%s", dev->name, "safety-ce");
3481 ret = request_irq(priv->sfty_ce_irq,
3482 stmmac_safety_interrupt,
3483 0, int_name, dev);
3484 if (unlikely(ret < 0)) {
3485 netdev_err(priv->dev,
3486 "%s: alloc sfty ce MSI %d (error: %d)\n",
3487 __func__, priv->sfty_ce_irq, ret);
3488 irq_err = REQ_IRQ_ERR_SFTY_CE;
3489 goto irq_error;
3490 }
3491 }
3492
3493
3494
3495
3496 if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) {
3497 int_name = priv->int_name_sfty_ue;
3498 sprintf(int_name, "%s:%s", dev->name, "safety-ue");
3499 ret = request_irq(priv->sfty_ue_irq,
3500 stmmac_safety_interrupt,
3501 0, int_name, dev);
3502 if (unlikely(ret < 0)) {
3503 netdev_err(priv->dev,
3504 "%s: alloc sfty ue MSI %d (error: %d)\n",
3505 __func__, priv->sfty_ue_irq, ret);
3506 irq_err = REQ_IRQ_ERR_SFTY_UE;
3507 goto irq_error;
3508 }
3509 }
3510
3511
3512 for (i = 0; i < priv->plat->rx_queues_to_use; i++) {
3513 if (priv->rx_irq[i] == 0)
3514 continue;
3515
3516 int_name = priv->int_name_rx_irq[i];
3517 sprintf(int_name, "%s:%s-%d", dev->name, "rx", i);
3518 ret = request_irq(priv->rx_irq[i],
3519 stmmac_msi_intr_rx,
3520 0, int_name, &priv->rx_queue[i]);
3521 if (unlikely(ret < 0)) {
3522 netdev_err(priv->dev,
3523 "%s: alloc rx-%d MSI %d (error: %d)\n",
3524 __func__, i, priv->rx_irq[i], ret);
3525 irq_err = REQ_IRQ_ERR_RX;
3526 irq_idx = i;
3527 goto irq_error;
3528 }
3529 cpumask_clear(&cpu_mask);
3530 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
3531 irq_set_affinity_hint(priv->rx_irq[i], &cpu_mask);
3532 }
3533
3534
3535 for (i = 0; i < priv->plat->tx_queues_to_use; i++) {
3536 if (priv->tx_irq[i] == 0)
3537 continue;
3538
3539 int_name = priv->int_name_tx_irq[i];
3540 sprintf(int_name, "%s:%s-%d", dev->name, "tx", i);
3541 ret = request_irq(priv->tx_irq[i],
3542 stmmac_msi_intr_tx,
3543 0, int_name, &priv->tx_queue[i]);
3544 if (unlikely(ret < 0)) {
3545 netdev_err(priv->dev,
3546 "%s: alloc tx-%d MSI %d (error: %d)\n",
3547 __func__, i, priv->tx_irq[i], ret);
3548 irq_err = REQ_IRQ_ERR_TX;
3549 irq_idx = i;
3550 goto irq_error;
3551 }
3552 cpumask_clear(&cpu_mask);
3553 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
3554 irq_set_affinity_hint(priv->tx_irq[i], &cpu_mask);
3555 }
3556
3557 return 0;
3558
3559irq_error:
3560 stmmac_free_irq(dev, irq_err, irq_idx);
3561 return ret;
3562}
3563
3564static int stmmac_request_irq_single(struct net_device *dev)
3565{
3566 struct stmmac_priv *priv = netdev_priv(dev);
3567 enum request_irq_err irq_err;
3568 int ret;
3569
3570 ret = request_irq(dev->irq, stmmac_interrupt,
3571 IRQF_SHARED, dev->name, dev);
3572 if (unlikely(ret < 0)) {
3573 netdev_err(priv->dev,
3574 "%s: ERROR: allocating the IRQ %d (error: %d)\n",
3575 __func__, dev->irq, ret);
3576 irq_err = REQ_IRQ_ERR_MAC;
3577 goto irq_error;
3578 }
3579
3580
3581
3582
3583 if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) {
3584 ret = request_irq(priv->wol_irq, stmmac_interrupt,
3585 IRQF_SHARED, dev->name, dev);
3586 if (unlikely(ret < 0)) {
3587 netdev_err(priv->dev,
3588 "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
3589 __func__, priv->wol_irq, ret);
3590 irq_err = REQ_IRQ_ERR_WOL;
3591 goto irq_error;
3592 }
3593 }
3594
3595
3596 if (priv->lpi_irq > 0 && priv->lpi_irq != dev->irq) {
3597 ret = request_irq(priv->lpi_irq, stmmac_interrupt,
3598 IRQF_SHARED, dev->name, dev);
3599 if (unlikely(ret < 0)) {
3600 netdev_err(priv->dev,
3601 "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
3602 __func__, priv->lpi_irq, ret);
3603 irq_err = REQ_IRQ_ERR_LPI;
3604 goto irq_error;
3605 }
3606 }
3607
3608 return 0;
3609
3610irq_error:
3611 stmmac_free_irq(dev, irq_err, 0);
3612 return ret;
3613}
3614
3615static int stmmac_request_irq(struct net_device *dev)
3616{
3617 struct stmmac_priv *priv = netdev_priv(dev);
3618 int ret;
3619
3620
3621 if (priv->plat->multi_msi_en)
3622 ret = stmmac_request_irq_multi_msi(dev);
3623 else
3624 ret = stmmac_request_irq_single(dev);
3625
3626 return ret;
3627}
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638int stmmac_open(struct net_device *dev)
3639{
3640 struct stmmac_priv *priv = netdev_priv(dev);
3641 int mode = priv->plat->phy_interface;
3642 int bfsize = 0;
3643 u32 chan;
3644 int ret;
3645
3646 ret = pm_runtime_get_sync(priv->device);
3647 if (ret < 0) {
3648 pm_runtime_put_noidle(priv->device);
3649 return ret;
3650 }
3651
3652 if (priv->hw->pcs != STMMAC_PCS_TBI &&
3653 priv->hw->pcs != STMMAC_PCS_RTBI &&
3654 (!priv->hw->xpcs ||
3655 xpcs_get_an_mode(priv->hw->xpcs, mode) != DW_AN_C73)) {
3656 ret = stmmac_init_phy(dev);
3657 if (ret) {
3658 netdev_err(priv->dev,
3659 "%s: Cannot attach to PHY (error: %d)\n",
3660 __func__, ret);
3661 goto init_phy_error;
3662 }
3663 }
3664
3665
3666 memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
3667 priv->xstats.threshold = tc;
3668
3669 bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
3670 if (bfsize < 0)
3671 bfsize = 0;
3672
3673 if (bfsize < BUF_SIZE_16KiB)
3674 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
3675
3676 priv->dma_buf_sz = bfsize;
3677 buf_sz = bfsize;
3678
3679 priv->rx_copybreak = STMMAC_RX_COPYBREAK;
3680
3681 if (!priv->dma_tx_size)
3682 priv->dma_tx_size = DMA_DEFAULT_TX_SIZE;
3683 if (!priv->dma_rx_size)
3684 priv->dma_rx_size = DMA_DEFAULT_RX_SIZE;
3685
3686
3687 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) {
3688 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
3689 int tbs_en = priv->plat->tx_queues_cfg[chan].tbs_en;
3690
3691
3692 tx_q->tbs |= tbs_en ? STMMAC_TBS_AVAIL : 0;
3693 }
3694
3695 ret = alloc_dma_desc_resources(priv);
3696 if (ret < 0) {
3697 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
3698 __func__);
3699 goto dma_desc_error;
3700 }
3701
3702 ret = init_dma_desc_rings(dev, GFP_KERNEL);
3703 if (ret < 0) {
3704 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
3705 __func__);
3706 goto init_error;
3707 }
3708
3709 ret = stmmac_hw_setup(dev, true);
3710 if (ret < 0) {
3711 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
3712 goto init_error;
3713 }
3714
3715 stmmac_init_coalesce(priv);
3716
3717 phylink_start(priv->phylink);
3718
3719 phylink_speed_up(priv->phylink);
3720
3721 ret = stmmac_request_irq(dev);
3722 if (ret)
3723 goto irq_error;
3724
3725 stmmac_enable_all_queues(priv);
3726 netif_tx_start_all_queues(priv->dev);
3727
3728 return 0;
3729
3730irq_error:
3731 phylink_stop(priv->phylink);
3732
3733 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
3734 hrtimer_cancel(&priv->tx_queue[chan].txtimer);
3735
3736 stmmac_hw_teardown(dev);
3737init_error:
3738 free_dma_desc_resources(priv);
3739dma_desc_error:
3740 phylink_disconnect_phy(priv->phylink);
3741init_phy_error:
3742 pm_runtime_put(priv->device);
3743 return ret;
3744}
3745
3746static void stmmac_fpe_stop_wq(struct stmmac_priv *priv)
3747{
3748 set_bit(__FPE_REMOVING, &priv->fpe_task_state);
3749
3750 if (priv->fpe_wq)
3751 destroy_workqueue(priv->fpe_wq);
3752
3753 netdev_info(priv->dev, "FPE workqueue stop");
3754}
3755
3756
3757
3758
3759
3760
3761
3762int stmmac_release(struct net_device *dev)
3763{
3764 struct stmmac_priv *priv = netdev_priv(dev);
3765 u32 chan;
3766
3767 if (device_may_wakeup(priv->device))
3768 phylink_speed_down(priv->phylink, false);
3769
3770 phylink_stop(priv->phylink);
3771 phylink_disconnect_phy(priv->phylink);
3772
3773 stmmac_disable_all_queues(priv);
3774
3775 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
3776 hrtimer_cancel(&priv->tx_queue[chan].txtimer);
3777
3778
3779 stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0);
3780
3781 if (priv->eee_enabled) {
3782 priv->tx_path_in_lpi_mode = false;
3783 del_timer_sync(&priv->eee_ctrl_timer);
3784 }
3785
3786
3787 stmmac_stop_all_dma(priv);
3788
3789
3790 free_dma_desc_resources(priv);
3791
3792
3793 stmmac_mac_set(priv, priv->ioaddr, false);
3794
3795 netif_carrier_off(dev);
3796
3797 stmmac_release_ptp(priv);
3798
3799 pm_runtime_put(priv->device);
3800
3801 if (priv->dma_cap.fpesel)
3802 stmmac_fpe_stop_wq(priv);
3803
3804 return 0;
3805}
3806
3807static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
3808 struct stmmac_tx_queue *tx_q)
3809{
3810 u16 tag = 0x0, inner_tag = 0x0;
3811 u32 inner_type = 0x0;
3812 struct dma_desc *p;
3813
3814 if (!priv->dma_cap.vlins)
3815 return false;
3816 if (!skb_vlan_tag_present(skb))
3817 return false;
3818 if (skb->vlan_proto == htons(ETH_P_8021AD)) {
3819 inner_tag = skb_vlan_tag_get(skb);
3820 inner_type = STMMAC_VLAN_INSERT;
3821 }
3822
3823 tag = skb_vlan_tag_get(skb);
3824
3825 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3826 p = &tx_q->dma_entx[tx_q->cur_tx].basic;
3827 else
3828 p = &tx_q->dma_tx[tx_q->cur_tx];
3829
3830 if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))
3831 return false;
3832
3833 stmmac_set_tx_owner(priv, p);
3834 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
3835 return true;
3836}
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
3850 int total_len, bool last_segment, u32 queue)
3851{
3852 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3853 struct dma_desc *desc;
3854 u32 buff_size;
3855 int tmp_len;
3856
3857 tmp_len = total_len;
3858
3859 while (tmp_len > 0) {
3860 dma_addr_t curr_addr;
3861
3862 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
3863 priv->dma_tx_size);
3864 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
3865
3866 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3867 desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3868 else
3869 desc = &tx_q->dma_tx[tx_q->cur_tx];
3870
3871 curr_addr = des + (total_len - tmp_len);
3872 if (priv->dma_cap.addr64 <= 32)
3873 desc->des0 = cpu_to_le32(curr_addr);
3874 else
3875 stmmac_set_desc_addr(priv, desc, curr_addr);
3876
3877 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
3878 TSO_MAX_BUFF_SIZE : tmp_len;
3879
3880 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
3881 0, 1,
3882 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
3883 0, 0);
3884
3885 tmp_len -= TSO_MAX_BUFF_SIZE;
3886 }
3887}
3888
3889static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue)
3890{
3891 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
3892 int desc_size;
3893
3894 if (likely(priv->extend_desc))
3895 desc_size = sizeof(struct dma_extended_desc);
3896 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
3897 desc_size = sizeof(struct dma_edesc);
3898 else
3899 desc_size = sizeof(struct dma_desc);
3900
3901
3902
3903
3904
3905 wmb();
3906
3907 tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size);
3908 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3909}
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
3939{
3940 struct dma_desc *desc, *first, *mss_desc = NULL;
3941 struct stmmac_priv *priv = netdev_priv(dev);
3942 int nfrags = skb_shinfo(skb)->nr_frags;
3943 u32 queue = skb_get_queue_mapping(skb);
3944 unsigned int first_entry, tx_packets;
3945 int tmp_pay_len = 0, first_tx;
3946 struct stmmac_tx_queue *tx_q;
3947 bool has_vlan, set_ic;
3948 u8 proto_hdr_len, hdr;
3949 u32 pay_len, mss;
3950 dma_addr_t des;
3951 int i;
3952
3953 tx_q = &priv->tx_queue[queue];
3954 first_tx = tx_q->cur_tx;
3955
3956
3957 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
3958 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
3959 hdr = sizeof(struct udphdr);
3960 } else {
3961 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
3962 hdr = tcp_hdrlen(skb);
3963 }
3964
3965
3966 if (unlikely(stmmac_tx_avail(priv, queue) <
3967 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
3968 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3969 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3970 queue));
3971
3972 netdev_err(priv->dev,
3973 "%s: Tx Ring full when queue awake\n",
3974 __func__);
3975 }
3976 return NETDEV_TX_BUSY;
3977 }
3978
3979 pay_len = skb_headlen(skb) - proto_hdr_len;
3980
3981 mss = skb_shinfo(skb)->gso_size;
3982
3983
3984 if (mss != tx_q->mss) {
3985 if (tx_q->tbs & STMMAC_TBS_AVAIL)
3986 mss_desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
3987 else
3988 mss_desc = &tx_q->dma_tx[tx_q->cur_tx];
3989
3990 stmmac_set_mss(priv, mss_desc, mss);
3991 tx_q->mss = mss;
3992 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx,
3993 priv->dma_tx_size);
3994 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
3995 }
3996
3997 if (netif_msg_tx_queued(priv)) {
3998 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
3999 __func__, hdr, proto_hdr_len, pay_len, mss);
4000 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
4001 skb->data_len);
4002 }
4003
4004
4005 has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
4006
4007 first_entry = tx_q->cur_tx;
4008 WARN_ON(tx_q->tx_skbuff[first_entry]);
4009
4010 if (tx_q->tbs & STMMAC_TBS_AVAIL)
4011 desc = &tx_q->dma_entx[first_entry].basic;
4012 else
4013 desc = &tx_q->dma_tx[first_entry];
4014 first = desc;
4015
4016 if (has_vlan)
4017 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
4018
4019
4020 des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
4021 DMA_TO_DEVICE);
4022 if (dma_mapping_error(priv->device, des))
4023 goto dma_map_err;
4024
4025 tx_q->tx_skbuff_dma[first_entry].buf = des;
4026 tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
4027 tx_q->tx_skbuff_dma[first_entry].map_as_page = false;
4028 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB;
4029
4030 if (priv->dma_cap.addr64 <= 32) {
4031 first->des0 = cpu_to_le32(des);
4032
4033
4034 if (pay_len)
4035 first->des1 = cpu_to_le32(des + proto_hdr_len);
4036
4037
4038 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
4039 } else {
4040 stmmac_set_desc_addr(priv, first, des);
4041 tmp_pay_len = pay_len;
4042 des += proto_hdr_len;
4043 pay_len = 0;
4044 }
4045
4046 stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
4047
4048
4049 for (i = 0; i < nfrags; i++) {
4050 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4051
4052 des = skb_frag_dma_map(priv->device, frag, 0,
4053 skb_frag_size(frag),
4054 DMA_TO_DEVICE);
4055 if (dma_mapping_error(priv->device, des))
4056 goto dma_map_err;
4057
4058 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
4059 (i == nfrags - 1), queue);
4060
4061 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
4062 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
4063 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
4064 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
4065 }
4066
4067 tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
4068
4069
4070 tx_q->tx_skbuff[tx_q->cur_tx] = skb;
4071 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB;
4072
4073
4074 tx_packets = (tx_q->cur_tx + 1) - first_tx;
4075 tx_q->tx_count_frames += tx_packets;
4076
4077 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
4078 set_ic = true;
4079 else if (!priv->tx_coal_frames[queue])
4080 set_ic = false;
4081 else if (tx_packets > priv->tx_coal_frames[queue])
4082 set_ic = true;
4083 else if ((tx_q->tx_count_frames %
4084 priv->tx_coal_frames[queue]) < tx_packets)
4085 set_ic = true;
4086 else
4087 set_ic = false;
4088
4089 if (set_ic) {
4090 if (tx_q->tbs & STMMAC_TBS_AVAIL)
4091 desc = &tx_q->dma_entx[tx_q->cur_tx].basic;
4092 else
4093 desc = &tx_q->dma_tx[tx_q->cur_tx];
4094
4095 tx_q->tx_count_frames = 0;
4096 stmmac_set_tx_ic(priv, desc);
4097 priv->xstats.tx_set_ic_bit++;
4098 }
4099
4100
4101
4102
4103
4104
4105 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_tx_size);
4106
4107 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
4108 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
4109 __func__);
4110 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
4111 }
4112
4113 dev->stats.tx_bytes += skb->len;
4114 priv->xstats.tx_tso_frames++;
4115 priv->xstats.tx_tso_nfrags += nfrags;
4116
4117 if (priv->sarc_type)
4118 stmmac_set_desc_sarc(priv, first, priv->sarc_type);
4119
4120 skb_tx_timestamp(skb);
4121
4122 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
4123 priv->hwts_tx_en)) {
4124
4125 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4126 stmmac_enable_tx_timestamp(priv, first);
4127 }
4128
4129
4130 stmmac_prepare_tso_tx_desc(priv, first, 1,
4131 proto_hdr_len,
4132 pay_len,
4133 1, tx_q->tx_skbuff_dma[first_entry].last_segment,
4134 hdr / 4, (skb->len - proto_hdr_len));
4135
4136
4137 if (mss_desc) {
4138
4139
4140
4141
4142
4143 dma_wmb();
4144 stmmac_set_tx_owner(priv, mss_desc);
4145 }
4146
4147 if (netif_msg_pktdata(priv)) {
4148 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
4149 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
4150 tx_q->cur_tx, first, nfrags);
4151 pr_info(">>> frame to be transmitted: ");
4152 print_pkt(skb->data, skb_headlen(skb));
4153 }
4154
4155 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
4156
4157 stmmac_flush_tx_descriptors(priv, queue);
4158 stmmac_tx_timer_arm(priv, queue);
4159
4160 return NETDEV_TX_OK;
4161
4162dma_map_err:
4163 dev_err(priv->device, "Tx dma map failed\n");
4164 dev_kfree_skb(skb);
4165 priv->dev->stats.tx_dropped++;
4166 return NETDEV_TX_OK;
4167}
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
4178{
4179 unsigned int first_entry, tx_packets, enh_desc;
4180 struct stmmac_priv *priv = netdev_priv(dev);
4181 unsigned int nopaged_len = skb_headlen(skb);
4182 int i, csum_insertion = 0, is_jumbo = 0;
4183 u32 queue = skb_get_queue_mapping(skb);
4184 int nfrags = skb_shinfo(skb)->nr_frags;
4185 int gso = skb_shinfo(skb)->gso_type;
4186 struct dma_edesc *tbs_desc = NULL;
4187 struct dma_desc *desc, *first;
4188 struct stmmac_tx_queue *tx_q;
4189 bool has_vlan, set_ic;
4190 int entry, first_tx;
4191 dma_addr_t des;
4192
4193 tx_q = &priv->tx_queue[queue];
4194 first_tx = tx_q->cur_tx;
4195
4196 if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en)
4197 stmmac_disable_eee_mode(priv);
4198
4199
4200 if (skb_is_gso(skb) && priv->tso) {
4201 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
4202 return stmmac_tso_xmit(skb, dev);
4203 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
4204 return stmmac_tso_xmit(skb, dev);
4205 }
4206
4207 if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
4208 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
4209 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
4210 queue));
4211
4212 netdev_err(priv->dev,
4213 "%s: Tx Ring full when queue awake\n",
4214 __func__);
4215 }
4216 return NETDEV_TX_BUSY;
4217 }
4218
4219
4220 has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
4221
4222 entry = tx_q->cur_tx;
4223 first_entry = entry;
4224 WARN_ON(tx_q->tx_skbuff[first_entry]);
4225
4226 csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
4227
4228 if (likely(priv->extend_desc))
4229 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
4230 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4231 desc = &tx_q->dma_entx[entry].basic;
4232 else
4233 desc = tx_q->dma_tx + entry;
4234
4235 first = desc;
4236
4237 if (has_vlan)
4238 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
4239
4240 enh_desc = priv->plat->enh_desc;
4241
4242 if (enh_desc)
4243 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
4244
4245 if (unlikely(is_jumbo)) {
4246 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
4247 if (unlikely(entry < 0) && (entry != -EINVAL))
4248 goto dma_map_err;
4249 }
4250
4251 for (i = 0; i < nfrags; i++) {
4252 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
4253 int len = skb_frag_size(frag);
4254 bool last_segment = (i == (nfrags - 1));
4255
4256 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
4257 WARN_ON(tx_q->tx_skbuff[entry]);
4258
4259 if (likely(priv->extend_desc))
4260 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
4261 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4262 desc = &tx_q->dma_entx[entry].basic;
4263 else
4264 desc = tx_q->dma_tx + entry;
4265
4266 des = skb_frag_dma_map(priv->device, frag, 0, len,
4267 DMA_TO_DEVICE);
4268 if (dma_mapping_error(priv->device, des))
4269 goto dma_map_err;
4270
4271 tx_q->tx_skbuff_dma[entry].buf = des;
4272
4273 stmmac_set_desc_addr(priv, desc, des);
4274
4275 tx_q->tx_skbuff_dma[entry].map_as_page = true;
4276 tx_q->tx_skbuff_dma[entry].len = len;
4277 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
4278 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
4279
4280
4281 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
4282 priv->mode, 1, last_segment, skb->len);
4283 }
4284
4285
4286 tx_q->tx_skbuff[entry] = skb;
4287 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB;
4288
4289
4290
4291
4292
4293
4294 tx_packets = (entry + 1) - first_tx;
4295 tx_q->tx_count_frames += tx_packets;
4296
4297 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en)
4298 set_ic = true;
4299 else if (!priv->tx_coal_frames[queue])
4300 set_ic = false;
4301 else if (tx_packets > priv->tx_coal_frames[queue])
4302 set_ic = true;
4303 else if ((tx_q->tx_count_frames %
4304 priv->tx_coal_frames[queue]) < tx_packets)
4305 set_ic = true;
4306 else
4307 set_ic = false;
4308
4309 if (set_ic) {
4310 if (likely(priv->extend_desc))
4311 desc = &tx_q->dma_etx[entry].basic;
4312 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4313 desc = &tx_q->dma_entx[entry].basic;
4314 else
4315 desc = &tx_q->dma_tx[entry];
4316
4317 tx_q->tx_count_frames = 0;
4318 stmmac_set_tx_ic(priv, desc);
4319 priv->xstats.tx_set_ic_bit++;
4320 }
4321
4322
4323
4324
4325
4326
4327 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
4328 tx_q->cur_tx = entry;
4329
4330 if (netif_msg_pktdata(priv)) {
4331 netdev_dbg(priv->dev,
4332 "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
4333 __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
4334 entry, first, nfrags);
4335
4336 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
4337 print_pkt(skb->data, skb->len);
4338 }
4339
4340 if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
4341 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
4342 __func__);
4343 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
4344 }
4345
4346 dev->stats.tx_bytes += skb->len;
4347
4348 if (priv->sarc_type)
4349 stmmac_set_desc_sarc(priv, first, priv->sarc_type);
4350
4351 skb_tx_timestamp(skb);
4352
4353
4354
4355
4356
4357 if (likely(!is_jumbo)) {
4358 bool last_segment = (nfrags == 0);
4359
4360 des = dma_map_single(priv->device, skb->data,
4361 nopaged_len, DMA_TO_DEVICE);
4362 if (dma_mapping_error(priv->device, des))
4363 goto dma_map_err;
4364
4365 tx_q->tx_skbuff_dma[first_entry].buf = des;
4366 tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB;
4367 tx_q->tx_skbuff_dma[first_entry].map_as_page = false;
4368
4369 stmmac_set_desc_addr(priv, first, des);
4370
4371 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
4372 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
4373
4374 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
4375 priv->hwts_tx_en)) {
4376
4377 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4378 stmmac_enable_tx_timestamp(priv, first);
4379 }
4380
4381
4382 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
4383 csum_insertion, priv->mode, 0, last_segment,
4384 skb->len);
4385 }
4386
4387 if (tx_q->tbs & STMMAC_TBS_EN) {
4388 struct timespec64 ts = ns_to_timespec64(skb->tstamp);
4389
4390 tbs_desc = &tx_q->dma_entx[first_entry];
4391 stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec);
4392 }
4393
4394 stmmac_set_tx_owner(priv, first);
4395
4396 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
4397
4398 stmmac_enable_dma_transmission(priv, priv->ioaddr);
4399
4400 stmmac_flush_tx_descriptors(priv, queue);
4401 stmmac_tx_timer_arm(priv, queue);
4402
4403 return NETDEV_TX_OK;
4404
4405dma_map_err:
4406 netdev_err(priv->dev, "Tx DMA map failed\n");
4407 dev_kfree_skb(skb);
4408 priv->dev->stats.tx_dropped++;
4409 return NETDEV_TX_OK;
4410}
4411
4412static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
4413{
4414 struct vlan_ethhdr *veth;
4415 __be16 vlan_proto;
4416 u16 vlanid;
4417
4418 veth = (struct vlan_ethhdr *)skb->data;
4419 vlan_proto = veth->h_vlan_proto;
4420
4421 if ((vlan_proto == htons(ETH_P_8021Q) &&
4422 dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
4423 (vlan_proto == htons(ETH_P_8021AD) &&
4424 dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
4425
4426 vlanid = ntohs(veth->h_vlan_TCI);
4427 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
4428 skb_pull(skb, VLAN_HLEN);
4429 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
4430 }
4431}
4432
4433
4434
4435
4436
4437
4438
4439
4440static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
4441{
4442 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4443 int dirty = stmmac_rx_dirty(priv, queue);
4444 unsigned int entry = rx_q->dirty_rx;
4445
4446 while (dirty-- > 0) {
4447 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
4448 struct dma_desc *p;
4449 bool use_rx_wd;
4450
4451 if (priv->extend_desc)
4452 p = (struct dma_desc *)(rx_q->dma_erx + entry);
4453 else
4454 p = rx_q->dma_rx + entry;
4455
4456 if (!buf->page) {
4457 buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
4458 if (!buf->page)
4459 break;
4460 }
4461
4462 if (priv->sph && !buf->sec_page) {
4463 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
4464 if (!buf->sec_page)
4465 break;
4466
4467 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
4468 }
4469
4470 buf->addr = page_pool_get_dma_addr(buf->page) + buf->page_offset;
4471
4472 stmmac_set_desc_addr(priv, p, buf->addr);
4473 if (priv->sph)
4474 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, true);
4475 else
4476 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr, false);
4477 stmmac_refill_desc3(priv, rx_q, p);
4478
4479 rx_q->rx_count_frames++;
4480 rx_q->rx_count_frames += priv->rx_coal_frames[queue];
4481 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])
4482 rx_q->rx_count_frames = 0;
4483
4484 use_rx_wd = !priv->rx_coal_frames[queue];
4485 use_rx_wd |= rx_q->rx_count_frames > 0;
4486 if (!priv->use_riwt)
4487 use_rx_wd = false;
4488
4489 dma_wmb();
4490 stmmac_set_rx_owner(priv, p, use_rx_wd);
4491
4492 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size);
4493 }
4494 rx_q->dirty_rx = entry;
4495 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
4496 (rx_q->dirty_rx * sizeof(struct dma_desc));
4497 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
4498}
4499
4500static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
4501 struct dma_desc *p,
4502 int status, unsigned int len)
4503{
4504 unsigned int plen = 0, hlen = 0;
4505 int coe = priv->hw->rx_csum;
4506
4507
4508 if (priv->sph && len)
4509 return 0;
4510
4511
4512 stmmac_get_rx_header_len(priv, p, &hlen);
4513 if (priv->sph && hlen) {
4514 priv->xstats.rx_split_hdr_pkt_n++;
4515 return hlen;
4516 }
4517
4518
4519 if (status & rx_not_ls)
4520 return priv->dma_buf_sz;
4521
4522 plen = stmmac_get_rx_frame_len(priv, p, coe);
4523
4524
4525 return min_t(unsigned int, priv->dma_buf_sz, plen);
4526}
4527
4528static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
4529 struct dma_desc *p,
4530 int status, unsigned int len)
4531{
4532 int coe = priv->hw->rx_csum;
4533 unsigned int plen = 0;
4534
4535
4536 if (!priv->sph)
4537 return 0;
4538
4539
4540 if (status & rx_not_ls)
4541 return priv->dma_buf_sz;
4542
4543 plen = stmmac_get_rx_frame_len(priv, p, coe);
4544
4545
4546 return plen - len;
4547}
4548
4549static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue,
4550 struct xdp_frame *xdpf, bool dma_map)
4551{
4552 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4553 unsigned int entry = tx_q->cur_tx;
4554 struct dma_desc *tx_desc;
4555 dma_addr_t dma_addr;
4556 bool set_ic;
4557
4558 if (stmmac_tx_avail(priv, queue) < STMMAC_TX_THRESH(priv))
4559 return STMMAC_XDP_CONSUMED;
4560
4561 if (likely(priv->extend_desc))
4562 tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry);
4563 else if (tx_q->tbs & STMMAC_TBS_AVAIL)
4564 tx_desc = &tx_q->dma_entx[entry].basic;
4565 else
4566 tx_desc = tx_q->dma_tx + entry;
4567
4568 if (dma_map) {
4569 dma_addr = dma_map_single(priv->device, xdpf->data,
4570 xdpf->len, DMA_TO_DEVICE);
4571 if (dma_mapping_error(priv->device, dma_addr))
4572 return STMMAC_XDP_CONSUMED;
4573
4574 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO;
4575 } else {
4576 struct page *page = virt_to_page(xdpf->data);
4577
4578 dma_addr = page_pool_get_dma_addr(page) + sizeof(*xdpf) +
4579 xdpf->headroom;
4580 dma_sync_single_for_device(priv->device, dma_addr,
4581 xdpf->len, DMA_BIDIRECTIONAL);
4582
4583 tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX;
4584 }
4585
4586 tx_q->tx_skbuff_dma[entry].buf = dma_addr;
4587 tx_q->tx_skbuff_dma[entry].map_as_page = false;
4588 tx_q->tx_skbuff_dma[entry].len = xdpf->len;
4589 tx_q->tx_skbuff_dma[entry].last_segment = true;
4590 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
4591
4592 tx_q->xdpf[entry] = xdpf;
4593
4594 stmmac_set_desc_addr(priv, tx_desc, dma_addr);
4595
4596 stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len,
4597 true, priv->mode, true, true,
4598 xdpf->len);
4599
4600 tx_q->tx_count_frames++;
4601
4602 if (tx_q->tx_count_frames % priv->tx_coal_frames[queue] == 0)
4603 set_ic = true;
4604 else
4605 set_ic = false;
4606
4607 if (set_ic) {
4608 tx_q->tx_count_frames = 0;
4609 stmmac_set_tx_ic(priv, tx_desc);
4610 priv->xstats.tx_set_ic_bit++;
4611 }
4612
4613 stmmac_enable_dma_transmission(priv, priv->ioaddr);
4614
4615 entry = STMMAC_GET_ENTRY(entry, priv->dma_tx_size);
4616 tx_q->cur_tx = entry;
4617
4618 return STMMAC_XDP_TX;
4619}
4620
4621static int stmmac_xdp_get_tx_queue(struct stmmac_priv *priv,
4622 int cpu)
4623{
4624 int index = cpu;
4625
4626 if (unlikely(index < 0))
4627 index = 0;
4628
4629 while (index >= priv->plat->tx_queues_to_use)
4630 index -= priv->plat->tx_queues_to_use;
4631
4632 return index;
4633}
4634
4635static int stmmac_xdp_xmit_back(struct stmmac_priv *priv,
4636 struct xdp_buff *xdp)
4637{
4638 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
4639 int cpu = smp_processor_id();
4640 struct netdev_queue *nq;
4641 int queue;
4642 int res;
4643
4644 if (unlikely(!xdpf))
4645 return STMMAC_XDP_CONSUMED;
4646
4647 queue = stmmac_xdp_get_tx_queue(priv, cpu);
4648 nq = netdev_get_tx_queue(priv->dev, queue);
4649
4650 __netif_tx_lock(nq, cpu);
4651
4652 nq->trans_start = jiffies;
4653
4654 res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false);
4655 if (res == STMMAC_XDP_TX)
4656 stmmac_flush_tx_descriptors(priv, queue);
4657
4658 __netif_tx_unlock(nq);
4659
4660 return res;
4661}
4662
4663static int __stmmac_xdp_run_prog(struct stmmac_priv *priv,
4664 struct bpf_prog *prog,
4665 struct xdp_buff *xdp)
4666{
4667 u32 act;
4668 int res;
4669
4670 act = bpf_prog_run_xdp(prog, xdp);
4671 switch (act) {
4672 case XDP_PASS:
4673 res = STMMAC_XDP_PASS;
4674 break;
4675 case XDP_TX:
4676 res = stmmac_xdp_xmit_back(priv, xdp);
4677 break;
4678 case XDP_REDIRECT:
4679 if (xdp_do_redirect(priv->dev, xdp, prog) < 0)
4680 res = STMMAC_XDP_CONSUMED;
4681 else
4682 res = STMMAC_XDP_REDIRECT;
4683 break;
4684 default:
4685 bpf_warn_invalid_xdp_action(act);
4686 fallthrough;
4687 case XDP_ABORTED:
4688 trace_xdp_exception(priv->dev, prog, act);
4689 fallthrough;
4690 case XDP_DROP:
4691 res = STMMAC_XDP_CONSUMED;
4692 break;
4693 }
4694
4695 return res;
4696}
4697
4698static struct sk_buff *stmmac_xdp_run_prog(struct stmmac_priv *priv,
4699 struct xdp_buff *xdp)
4700{
4701 struct bpf_prog *prog;
4702 int res;
4703
4704 prog = READ_ONCE(priv->xdp_prog);
4705 if (!prog) {
4706 res = STMMAC_XDP_PASS;
4707 goto out;
4708 }
4709
4710 res = __stmmac_xdp_run_prog(priv, prog, xdp);
4711out:
4712 return ERR_PTR(-res);
4713}
4714
4715static void stmmac_finalize_xdp_rx(struct stmmac_priv *priv,
4716 int xdp_status)
4717{
4718 int cpu = smp_processor_id();
4719 int queue;
4720
4721 queue = stmmac_xdp_get_tx_queue(priv, cpu);
4722
4723 if (xdp_status & STMMAC_XDP_TX)
4724 stmmac_tx_timer_arm(priv, queue);
4725
4726 if (xdp_status & STMMAC_XDP_REDIRECT)
4727 xdp_do_flush();
4728}
4729
4730static struct sk_buff *stmmac_construct_skb_zc(struct stmmac_channel *ch,
4731 struct xdp_buff *xdp)
4732{
4733 unsigned int metasize = xdp->data - xdp->data_meta;
4734 unsigned int datasize = xdp->data_end - xdp->data;
4735 struct sk_buff *skb;
4736
4737 skb = __napi_alloc_skb(&ch->rxtx_napi,
4738 xdp->data_end - xdp->data_hard_start,
4739 GFP_ATOMIC | __GFP_NOWARN);
4740 if (unlikely(!skb))
4741 return NULL;
4742
4743 skb_reserve(skb, xdp->data - xdp->data_hard_start);
4744 memcpy(__skb_put(skb, datasize), xdp->data, datasize);
4745 if (metasize)
4746 skb_metadata_set(skb, metasize);
4747
4748 return skb;
4749}
4750
4751static void stmmac_dispatch_skb_zc(struct stmmac_priv *priv, u32 queue,
4752 struct dma_desc *p, struct dma_desc *np,
4753 struct xdp_buff *xdp)
4754{
4755 struct stmmac_channel *ch = &priv->channel[queue];
4756 unsigned int len = xdp->data_end - xdp->data;
4757 enum pkt_hash_types hash_type;
4758 int coe = priv->hw->rx_csum;
4759 struct sk_buff *skb;
4760 u32 hash;
4761
4762 skb = stmmac_construct_skb_zc(ch, xdp);
4763 if (!skb) {
4764 priv->dev->stats.rx_dropped++;
4765 return;
4766 }
4767
4768 stmmac_get_rx_hwtstamp(priv, p, np, skb);
4769 stmmac_rx_vlan(priv->dev, skb);
4770 skb->protocol = eth_type_trans(skb, priv->dev);
4771
4772 if (unlikely(!coe))
4773 skb_checksum_none_assert(skb);
4774 else
4775 skb->ip_summed = CHECKSUM_UNNECESSARY;
4776
4777 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
4778 skb_set_hash(skb, hash, hash_type);
4779
4780 skb_record_rx_queue(skb, queue);
4781 napi_gro_receive(&ch->rxtx_napi, skb);
4782
4783 priv->dev->stats.rx_packets++;
4784 priv->dev->stats.rx_bytes += len;
4785}
4786
4787static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget)
4788{
4789 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4790 unsigned int entry = rx_q->dirty_rx;
4791 struct dma_desc *rx_desc = NULL;
4792 bool ret = true;
4793
4794 budget = min(budget, stmmac_rx_dirty(priv, queue));
4795
4796 while (budget-- > 0 && entry != rx_q->cur_rx) {
4797 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
4798 dma_addr_t dma_addr;
4799 bool use_rx_wd;
4800
4801 if (!buf->xdp) {
4802 buf->xdp = xsk_buff_alloc(rx_q->xsk_pool);
4803 if (!buf->xdp) {
4804 ret = false;
4805 break;
4806 }
4807 }
4808
4809 if (priv->extend_desc)
4810 rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry);
4811 else
4812 rx_desc = rx_q->dma_rx + entry;
4813
4814 dma_addr = xsk_buff_xdp_get_dma(buf->xdp);
4815 stmmac_set_desc_addr(priv, rx_desc, dma_addr);
4816 stmmac_set_desc_sec_addr(priv, rx_desc, 0, false);
4817 stmmac_refill_desc3(priv, rx_q, rx_desc);
4818
4819 rx_q->rx_count_frames++;
4820 rx_q->rx_count_frames += priv->rx_coal_frames[queue];
4821 if (rx_q->rx_count_frames > priv->rx_coal_frames[queue])
4822 rx_q->rx_count_frames = 0;
4823
4824 use_rx_wd = !priv->rx_coal_frames[queue];
4825 use_rx_wd |= rx_q->rx_count_frames > 0;
4826 if (!priv->use_riwt)
4827 use_rx_wd = false;
4828
4829 dma_wmb();
4830 stmmac_set_rx_owner(priv, rx_desc, use_rx_wd);
4831
4832 entry = STMMAC_GET_ENTRY(entry, priv->dma_rx_size);
4833 }
4834
4835 if (rx_desc) {
4836 rx_q->dirty_rx = entry;
4837 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
4838 (rx_q->dirty_rx * sizeof(struct dma_desc));
4839 stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
4840 }
4841
4842 return ret;
4843}
4844
4845static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue)
4846{
4847 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4848 unsigned int count = 0, error = 0, len = 0;
4849 int dirty = stmmac_rx_dirty(priv, queue);
4850 unsigned int next_entry = rx_q->cur_rx;
4851 unsigned int desc_size;
4852 struct bpf_prog *prog;
4853 bool failure = false;
4854 int xdp_status = 0;
4855 int status = 0;
4856
4857 if (netif_msg_rx_status(priv)) {
4858 void *rx_head;
4859
4860 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
4861 if (priv->extend_desc) {
4862 rx_head = (void *)rx_q->dma_erx;
4863 desc_size = sizeof(struct dma_extended_desc);
4864 } else {
4865 rx_head = (void *)rx_q->dma_rx;
4866 desc_size = sizeof(struct dma_desc);
4867 }
4868
4869 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true,
4870 rx_q->dma_rx_phy, desc_size);
4871 }
4872 while (count < limit) {
4873 struct stmmac_rx_buffer *buf;
4874 unsigned int buf1_len = 0;
4875 struct dma_desc *np, *p;
4876 int entry;
4877 int res;
4878
4879 if (!count && rx_q->state_saved) {
4880 error = rx_q->state.error;
4881 len = rx_q->state.len;
4882 } else {
4883 rx_q->state_saved = false;
4884 error = 0;
4885 len = 0;
4886 }
4887
4888 if (count >= limit)
4889 break;
4890
4891read_again:
4892 buf1_len = 0;
4893 entry = next_entry;
4894 buf = &rx_q->buf_pool[entry];
4895
4896 if (dirty >= STMMAC_RX_FILL_BATCH) {
4897 failure = failure ||
4898 !stmmac_rx_refill_zc(priv, queue, dirty);
4899 dirty = 0;
4900 }
4901
4902 if (priv->extend_desc)
4903 p = (struct dma_desc *)(rx_q->dma_erx + entry);
4904 else
4905 p = rx_q->dma_rx + entry;
4906
4907
4908 status = stmmac_rx_status(priv, &priv->dev->stats,
4909 &priv->xstats, p);
4910
4911 if (unlikely(status & dma_own))
4912 break;
4913
4914
4915 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
4916 priv->dma_rx_size);
4917 next_entry = rx_q->cur_rx;
4918
4919 if (priv->extend_desc)
4920 np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
4921 else
4922 np = rx_q->dma_rx + next_entry;
4923
4924 prefetch(np);
4925
4926
4927 if (!buf->xdp)
4928 break;
4929
4930 if (priv->extend_desc)
4931 stmmac_rx_extended_status(priv, &priv->dev->stats,
4932 &priv->xstats,
4933 rx_q->dma_erx + entry);
4934 if (unlikely(status == discard_frame)) {
4935 xsk_buff_free(buf->xdp);
4936 buf->xdp = NULL;
4937 dirty++;
4938 error = 1;
4939 if (!priv->hwts_rx_en)
4940 priv->dev->stats.rx_errors++;
4941 }
4942
4943 if (unlikely(error && (status & rx_not_ls)))
4944 goto read_again;
4945 if (unlikely(error)) {
4946 count++;
4947 continue;
4948 }
4949
4950
4951 if (likely(status & rx_not_ls)) {
4952 xsk_buff_free(buf->xdp);
4953 buf->xdp = NULL;
4954 dirty++;
4955 count++;
4956 goto read_again;
4957 }
4958
4959
4960 buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
4961 len += buf1_len;
4962
4963
4964
4965
4966
4967
4968
4969
4970 if (likely(!(status & rx_not_ls)) &&
4971 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
4972 unlikely(status != llc_snap))) {
4973 buf1_len -= ETH_FCS_LEN;
4974 len -= ETH_FCS_LEN;
4975 }
4976
4977
4978 buf->xdp->data_end = buf->xdp->data + buf1_len;
4979 xsk_buff_dma_sync_for_cpu(buf->xdp, rx_q->xsk_pool);
4980
4981 prog = READ_ONCE(priv->xdp_prog);
4982 res = __stmmac_xdp_run_prog(priv, prog, buf->xdp);
4983
4984 switch (res) {
4985 case STMMAC_XDP_PASS:
4986 stmmac_dispatch_skb_zc(priv, queue, p, np, buf->xdp);
4987 xsk_buff_free(buf->xdp);
4988 break;
4989 case STMMAC_XDP_CONSUMED:
4990 xsk_buff_free(buf->xdp);
4991 priv->dev->stats.rx_dropped++;
4992 break;
4993 case STMMAC_XDP_TX:
4994 case STMMAC_XDP_REDIRECT:
4995 xdp_status |= res;
4996 break;
4997 }
4998
4999 buf->xdp = NULL;
5000 dirty++;
5001 count++;
5002 }
5003
5004 if (status & rx_not_ls) {
5005 rx_q->state_saved = true;
5006 rx_q->state.error = error;
5007 rx_q->state.len = len;
5008 }
5009
5010 stmmac_finalize_xdp_rx(priv, xdp_status);
5011
5012 priv->xstats.rx_pkt_n += count;
5013 priv->xstats.rxq_stats[queue].rx_pkt_n += count;
5014
5015 if (xsk_uses_need_wakeup(rx_q->xsk_pool)) {
5016 if (failure || stmmac_rx_dirty(priv, queue) > 0)
5017 xsk_set_rx_need_wakeup(rx_q->xsk_pool);
5018 else
5019 xsk_clear_rx_need_wakeup(rx_q->xsk_pool);
5020
5021 return (int)count;
5022 }
5023
5024 return failure ? limit : (int)count;
5025}
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
5036{
5037 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
5038 struct stmmac_channel *ch = &priv->channel[queue];
5039 unsigned int count = 0, error = 0, len = 0;
5040 int status = 0, coe = priv->hw->rx_csum;
5041 unsigned int next_entry = rx_q->cur_rx;
5042 enum dma_data_direction dma_dir;
5043 unsigned int desc_size;
5044 struct sk_buff *skb = NULL;
5045 struct xdp_buff xdp;
5046 int xdp_status = 0;
5047 int buf_sz;
5048
5049 dma_dir = page_pool_get_dma_dir(rx_q->page_pool);
5050 buf_sz = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
5051
5052 if (netif_msg_rx_status(priv)) {
5053 void *rx_head;
5054
5055 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
5056 if (priv->extend_desc) {
5057 rx_head = (void *)rx_q->dma_erx;
5058 desc_size = sizeof(struct dma_extended_desc);
5059 } else {
5060 rx_head = (void *)rx_q->dma_rx;
5061 desc_size = sizeof(struct dma_desc);
5062 }
5063
5064 stmmac_display_ring(priv, rx_head, priv->dma_rx_size, true,
5065 rx_q->dma_rx_phy, desc_size);
5066 }
5067 while (count < limit) {
5068 unsigned int buf1_len = 0, buf2_len = 0;
5069 enum pkt_hash_types hash_type;
5070 struct stmmac_rx_buffer *buf;
5071 struct dma_desc *np, *p;
5072 int entry;
5073 u32 hash;
5074
5075 if (!count && rx_q->state_saved) {
5076 skb = rx_q->state.skb;
5077 error = rx_q->state.error;
5078 len = rx_q->state.len;
5079 } else {
5080 rx_q->state_saved = false;
5081 skb = NULL;
5082 error = 0;
5083 len = 0;
5084 }
5085
5086 if (count >= limit)
5087 break;
5088
5089read_again:
5090 buf1_len = 0;
5091 buf2_len = 0;
5092 entry = next_entry;
5093 buf = &rx_q->buf_pool[entry];
5094
5095 if (priv->extend_desc)
5096 p = (struct dma_desc *)(rx_q->dma_erx + entry);
5097 else
5098 p = rx_q->dma_rx + entry;
5099
5100
5101 status = stmmac_rx_status(priv, &priv->dev->stats,
5102 &priv->xstats, p);
5103
5104 if (unlikely(status & dma_own))
5105 break;
5106
5107 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx,
5108 priv->dma_rx_size);
5109 next_entry = rx_q->cur_rx;
5110
5111 if (priv->extend_desc)
5112 np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
5113 else
5114 np = rx_q->dma_rx + next_entry;
5115
5116 prefetch(np);
5117
5118 if (priv->extend_desc)
5119 stmmac_rx_extended_status(priv, &priv->dev->stats,
5120 &priv->xstats, rx_q->dma_erx + entry);
5121 if (unlikely(status == discard_frame)) {
5122 page_pool_recycle_direct(rx_q->page_pool, buf->page);
5123 buf->page = NULL;
5124 error = 1;
5125 if (!priv->hwts_rx_en)
5126 priv->dev->stats.rx_errors++;
5127 }
5128
5129 if (unlikely(error && (status & rx_not_ls)))
5130 goto read_again;
5131 if (unlikely(error)) {
5132 dev_kfree_skb(skb);
5133 skb = NULL;
5134 count++;
5135 continue;
5136 }
5137
5138
5139
5140 prefetch(page_address(buf->page) + buf->page_offset);
5141 if (buf->sec_page)
5142 prefetch(page_address(buf->sec_page));
5143
5144 buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
5145 len += buf1_len;
5146 buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
5147 len += buf2_len;
5148
5149
5150
5151
5152
5153
5154
5155
5156 if (likely(!(status & rx_not_ls)) &&
5157 (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
5158 unlikely(status != llc_snap))) {
5159 if (buf2_len)
5160 buf2_len -= ETH_FCS_LEN;
5161 else
5162 buf1_len -= ETH_FCS_LEN;
5163
5164 len -= ETH_FCS_LEN;
5165 }
5166
5167 if (!skb) {
5168 unsigned int pre_len, sync_len;
5169
5170 dma_sync_single_for_cpu(priv->device, buf->addr,
5171 buf1_len, dma_dir);
5172
5173 xdp_init_buff(&xdp, buf_sz, &rx_q->xdp_rxq);
5174 xdp_prepare_buff(&xdp, page_address(buf->page),
5175 buf->page_offset, buf1_len, false);
5176
5177 pre_len = xdp.data_end - xdp.data_hard_start -
5178 buf->page_offset;
5179 skb = stmmac_xdp_run_prog(priv, &xdp);
5180
5181
5182
5183 sync_len = xdp.data_end - xdp.data_hard_start -
5184 buf->page_offset;
5185 sync_len = max(sync_len, pre_len);
5186
5187
5188 if (IS_ERR(skb)) {
5189 unsigned int xdp_res = -PTR_ERR(skb);
5190
5191 if (xdp_res & STMMAC_XDP_CONSUMED) {
5192 page_pool_put_page(rx_q->page_pool,
5193 virt_to_head_page(xdp.data),
5194 sync_len, true);
5195 buf->page = NULL;
5196 priv->dev->stats.rx_dropped++;
5197
5198
5199
5200
5201 skb = NULL;
5202
5203 if (unlikely((status & rx_not_ls)))
5204 goto read_again;
5205
5206 count++;
5207 continue;
5208 } else if (xdp_res & (STMMAC_XDP_TX |
5209 STMMAC_XDP_REDIRECT)) {
5210 xdp_status |= xdp_res;
5211 buf->page = NULL;
5212 skb = NULL;
5213 count++;
5214 continue;
5215 }
5216 }
5217 }
5218
5219 if (!skb) {
5220
5221 buf1_len = xdp.data_end - xdp.data;
5222
5223 skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
5224 if (!skb) {
5225 priv->dev->stats.rx_dropped++;
5226 count++;
5227 goto drain_data;
5228 }
5229
5230
5231 skb_copy_to_linear_data(skb, xdp.data, buf1_len);
5232 skb_put(skb, buf1_len);
5233
5234
5235 page_pool_recycle_direct(rx_q->page_pool, buf->page);
5236 buf->page = NULL;
5237 } else if (buf1_len) {
5238 dma_sync_single_for_cpu(priv->device, buf->addr,
5239 buf1_len, dma_dir);
5240 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
5241 buf->page, buf->page_offset, buf1_len,
5242 priv->dma_buf_sz);
5243
5244
5245 page_pool_release_page(rx_q->page_pool, buf->page);
5246 buf->page = NULL;
5247 }
5248
5249 if (buf2_len) {
5250 dma_sync_single_for_cpu(priv->device, buf->sec_addr,
5251 buf2_len, dma_dir);
5252 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
5253 buf->sec_page, 0, buf2_len,
5254 priv->dma_buf_sz);
5255
5256
5257 page_pool_release_page(rx_q->page_pool, buf->sec_page);
5258 buf->sec_page = NULL;
5259 }
5260
5261drain_data:
5262 if (likely(status & rx_not_ls))
5263 goto read_again;
5264 if (!skb)
5265 continue;
5266
5267
5268
5269 stmmac_get_rx_hwtstamp(priv, p, np, skb);
5270 stmmac_rx_vlan(priv->dev, skb);
5271 skb->protocol = eth_type_trans(skb, priv->dev);
5272
5273 if (unlikely(!coe))
5274 skb_checksum_none_assert(skb);
5275 else
5276 skb->ip_summed = CHECKSUM_UNNECESSARY;
5277
5278 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
5279 skb_set_hash(skb, hash, hash_type);
5280
5281 skb_record_rx_queue(skb, queue);
5282 napi_gro_receive(&ch->rx_napi, skb);
5283 skb = NULL;
5284
5285 priv->dev->stats.rx_packets++;
5286 priv->dev->stats.rx_bytes += len;
5287 count++;
5288 }
5289
5290 if (status & rx_not_ls || skb) {
5291 rx_q->state_saved = true;
5292 rx_q->state.skb = skb;
5293 rx_q->state.error = error;
5294 rx_q->state.len = len;
5295 }
5296
5297 stmmac_finalize_xdp_rx(priv, xdp_status);
5298
5299 stmmac_rx_refill(priv, queue);
5300
5301 priv->xstats.rx_pkt_n += count;
5302 priv->xstats.rxq_stats[queue].rx_pkt_n += count;
5303
5304 return count;
5305}
5306
5307static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
5308{
5309 struct stmmac_channel *ch =
5310 container_of(napi, struct stmmac_channel, rx_napi);
5311 struct stmmac_priv *priv = ch->priv_data;
5312 u32 chan = ch->index;
5313 int work_done;
5314
5315 priv->xstats.napi_poll++;
5316
5317 work_done = stmmac_rx(priv, budget, chan);
5318 if (work_done < budget && napi_complete_done(napi, work_done)) {
5319 unsigned long flags;
5320
5321 spin_lock_irqsave(&ch->lock, flags);
5322 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 0);
5323 spin_unlock_irqrestore(&ch->lock, flags);
5324 }
5325
5326 return work_done;
5327}
5328
5329static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
5330{
5331 struct stmmac_channel *ch =
5332 container_of(napi, struct stmmac_channel, tx_napi);
5333 struct stmmac_priv *priv = ch->priv_data;
5334 u32 chan = ch->index;
5335 int work_done;
5336
5337 priv->xstats.napi_poll++;
5338
5339 work_done = stmmac_tx_clean(priv, budget, chan);
5340 work_done = min(work_done, budget);
5341
5342 if (work_done < budget && napi_complete_done(napi, work_done)) {
5343 unsigned long flags;
5344
5345 spin_lock_irqsave(&ch->lock, flags);
5346 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 0, 1);
5347 spin_unlock_irqrestore(&ch->lock, flags);
5348 }
5349
5350 return work_done;
5351}
5352
5353static int stmmac_napi_poll_rxtx(struct napi_struct *napi, int budget)
5354{
5355 struct stmmac_channel *ch =
5356 container_of(napi, struct stmmac_channel, rxtx_napi);
5357 struct stmmac_priv *priv = ch->priv_data;
5358 int rx_done, tx_done, rxtx_done;
5359 u32 chan = ch->index;
5360
5361 priv->xstats.napi_poll++;
5362
5363 tx_done = stmmac_tx_clean(priv, budget, chan);
5364 tx_done = min(tx_done, budget);
5365
5366 rx_done = stmmac_rx_zc(priv, budget, chan);
5367
5368 rxtx_done = max(tx_done, rx_done);
5369
5370
5371
5372
5373 if (rxtx_done >= budget)
5374 return budget;
5375
5376
5377 if (napi_complete_done(napi, rxtx_done)) {
5378 unsigned long flags;
5379
5380 spin_lock_irqsave(&ch->lock, flags);
5381
5382
5383
5384 stmmac_enable_dma_irq(priv, priv->ioaddr, chan, 1, 1);
5385 spin_unlock_irqrestore(&ch->lock, flags);
5386 }
5387
5388 return min(rxtx_done, budget - 1);
5389}
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400static void stmmac_tx_timeout(struct net_device *dev, unsigned int txqueue)
5401{
5402 struct stmmac_priv *priv = netdev_priv(dev);
5403
5404 stmmac_global_err(priv);
5405}
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416static void stmmac_set_rx_mode(struct net_device *dev)
5417{
5418 struct stmmac_priv *priv = netdev_priv(dev);
5419
5420 stmmac_set_filter(priv, priv->hw, dev);
5421}
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
5435{
5436 struct stmmac_priv *priv = netdev_priv(dev);
5437 int txfifosz = priv->plat->tx_fifo_size;
5438 const int mtu = new_mtu;
5439
5440 if (txfifosz == 0)
5441 txfifosz = priv->dma_cap.tx_fifo_size;
5442
5443 txfifosz /= priv->plat->tx_queues_to_use;
5444
5445 if (netif_running(dev)) {
5446 netdev_err(priv->dev, "must be stopped to change its MTU\n");
5447 return -EBUSY;
5448 }
5449
5450 if (stmmac_xdp_is_enabled(priv) && new_mtu > ETH_DATA_LEN) {
5451 netdev_dbg(priv->dev, "Jumbo frames not supported for XDP\n");
5452 return -EINVAL;
5453 }
5454
5455 new_mtu = STMMAC_ALIGN(new_mtu);
5456
5457
5458 if ((txfifosz < new_mtu) || (new_mtu > BUF_SIZE_16KiB))
5459 return -EINVAL;
5460
5461 dev->mtu = mtu;
5462
5463 netdev_update_features(dev);
5464
5465 return 0;
5466}
5467
5468static netdev_features_t stmmac_fix_features(struct net_device *dev,
5469 netdev_features_t features)
5470{
5471 struct stmmac_priv *priv = netdev_priv(dev);
5472
5473 if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
5474 features &= ~NETIF_F_RXCSUM;
5475
5476 if (!priv->plat->tx_coe)
5477 features &= ~NETIF_F_CSUM_MASK;
5478
5479
5480
5481
5482
5483
5484 if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
5485 features &= ~NETIF_F_CSUM_MASK;
5486
5487
5488 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
5489 if (features & NETIF_F_TSO)
5490 priv->tso = true;
5491 else
5492 priv->tso = false;
5493 }
5494
5495 return features;
5496}
5497
5498static int stmmac_set_features(struct net_device *netdev,
5499 netdev_features_t features)
5500{
5501 struct stmmac_priv *priv = netdev_priv(netdev);
5502 bool sph_en;
5503 u32 chan;
5504
5505
5506 if (features & NETIF_F_RXCSUM)
5507 priv->hw->rx_csum = priv->plat->rx_coe;
5508 else
5509 priv->hw->rx_csum = 0;
5510
5511
5512
5513 stmmac_rx_ipc(priv, priv->hw);
5514
5515 sph_en = (priv->hw->rx_csum > 0) && priv->sph;
5516
5517 for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
5518 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
5519
5520 return 0;
5521}
5522
5523static void stmmac_fpe_event_status(struct stmmac_priv *priv, int status)
5524{
5525 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg;
5526 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state;
5527 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state;
5528 bool *hs_enable = &fpe_cfg->hs_enable;
5529
5530 if (status == FPE_EVENT_UNKNOWN || !*hs_enable)
5531 return;
5532
5533
5534 if ((status & FPE_EVENT_RVER) == FPE_EVENT_RVER) {
5535 if (*lp_state < FPE_STATE_CAPABLE)
5536 *lp_state = FPE_STATE_CAPABLE;
5537
5538
5539 if (*hs_enable)
5540 stmmac_fpe_send_mpacket(priv, priv->ioaddr,
5541 MPACKET_RESPONSE);
5542 }
5543
5544
5545 if ((status & FPE_EVENT_TVER) == FPE_EVENT_TVER) {
5546 if (*lo_state < FPE_STATE_CAPABLE)
5547 *lo_state = FPE_STATE_CAPABLE;
5548 }
5549
5550
5551 if ((status & FPE_EVENT_RRSP) == FPE_EVENT_RRSP)
5552 *lp_state = FPE_STATE_ENTERING_ON;
5553
5554
5555 if ((status & FPE_EVENT_TRSP) == FPE_EVENT_TRSP)
5556 *lo_state = FPE_STATE_ENTERING_ON;
5557
5558 if (!test_bit(__FPE_REMOVING, &priv->fpe_task_state) &&
5559 !test_and_set_bit(__FPE_TASK_SCHED, &priv->fpe_task_state) &&
5560 priv->fpe_wq) {
5561 queue_work(priv->fpe_wq, &priv->fpe_task);
5562 }
5563}
5564
5565static void stmmac_common_interrupt(struct stmmac_priv *priv)
5566{
5567 u32 rx_cnt = priv->plat->rx_queues_to_use;
5568 u32 tx_cnt = priv->plat->tx_queues_to_use;
5569 u32 queues_count;
5570 u32 queue;
5571 bool xmac;
5572
5573 xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
5574 queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
5575
5576 if (priv->irq_wake)
5577 pm_wakeup_event(priv->device, 0);
5578
5579 if (priv->dma_cap.estsel)
5580 stmmac_est_irq_status(priv, priv->ioaddr, priv->dev,
5581 &priv->xstats, tx_cnt);
5582
5583 if (priv->dma_cap.fpesel) {
5584 int status = stmmac_fpe_irq_status(priv, priv->ioaddr,
5585 priv->dev);
5586
5587 stmmac_fpe_event_status(priv, status);
5588 }
5589
5590
5591 if ((priv->plat->has_gmac) || xmac) {
5592 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
5593
5594 if (unlikely(status)) {
5595
5596 if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
5597 priv->tx_path_in_lpi_mode = true;
5598 if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
5599 priv->tx_path_in_lpi_mode = false;
5600 }
5601
5602 for (queue = 0; queue < queues_count; queue++) {
5603 status = stmmac_host_mtl_irq_status(priv, priv->hw,
5604 queue);
5605 }
5606
5607
5608 if (priv->hw->pcs) {
5609 if (priv->xstats.pcs_link)
5610 netif_carrier_on(priv->dev);
5611 else
5612 netif_carrier_off(priv->dev);
5613 }
5614
5615 stmmac_timestamp_interrupt(priv, priv);
5616 }
5617}
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
5631{
5632 struct net_device *dev = (struct net_device *)dev_id;
5633 struct stmmac_priv *priv = netdev_priv(dev);
5634
5635
5636 if (test_bit(STMMAC_DOWN, &priv->state))
5637 return IRQ_HANDLED;
5638
5639
5640 if (stmmac_safety_feat_interrupt(priv))
5641 return IRQ_HANDLED;
5642
5643
5644 stmmac_common_interrupt(priv);
5645
5646
5647 stmmac_dma_interrupt(priv);
5648
5649 return IRQ_HANDLED;
5650}
5651
5652static irqreturn_t stmmac_mac_interrupt(int irq, void *dev_id)
5653{
5654 struct net_device *dev = (struct net_device *)dev_id;
5655 struct stmmac_priv *priv = netdev_priv(dev);
5656
5657 if (unlikely(!dev)) {
5658 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
5659 return IRQ_NONE;
5660 }
5661
5662
5663 if (test_bit(STMMAC_DOWN, &priv->state))
5664 return IRQ_HANDLED;
5665
5666
5667 stmmac_common_interrupt(priv);
5668
5669 return IRQ_HANDLED;
5670}
5671
5672static irqreturn_t stmmac_safety_interrupt(int irq, void *dev_id)
5673{
5674 struct net_device *dev = (struct net_device *)dev_id;
5675 struct stmmac_priv *priv = netdev_priv(dev);
5676
5677 if (unlikely(!dev)) {
5678 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
5679 return IRQ_NONE;
5680 }
5681
5682
5683 if (test_bit(STMMAC_DOWN, &priv->state))
5684 return IRQ_HANDLED;
5685
5686
5687 stmmac_safety_feat_interrupt(priv);
5688
5689 return IRQ_HANDLED;
5690}
5691
5692static irqreturn_t stmmac_msi_intr_tx(int irq, void *data)
5693{
5694 struct stmmac_tx_queue *tx_q = (struct stmmac_tx_queue *)data;
5695 int chan = tx_q->queue_index;
5696 struct stmmac_priv *priv;
5697 int status;
5698
5699 priv = container_of(tx_q, struct stmmac_priv, tx_queue[chan]);
5700
5701 if (unlikely(!data)) {
5702 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
5703 return IRQ_NONE;
5704 }
5705
5706
5707 if (test_bit(STMMAC_DOWN, &priv->state))
5708 return IRQ_HANDLED;
5709
5710 status = stmmac_napi_check(priv, chan, DMA_DIR_TX);
5711
5712 if (unlikely(status & tx_hard_error_bump_tc)) {
5713
5714 if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
5715 tc <= 256) {
5716 tc += 64;
5717 if (priv->plat->force_thresh_dma_mode)
5718 stmmac_set_dma_operation_mode(priv,
5719 tc,
5720 tc,
5721 chan);
5722 else
5723 stmmac_set_dma_operation_mode(priv,
5724 tc,
5725 SF_DMA_MODE,
5726 chan);
5727 priv->xstats.threshold = tc;
5728 }
5729 } else if (unlikely(status == tx_hard_error)) {
5730 stmmac_tx_err(priv, chan);
5731 }
5732
5733 return IRQ_HANDLED;
5734}
5735
5736static irqreturn_t stmmac_msi_intr_rx(int irq, void *data)
5737{
5738 struct stmmac_rx_queue *rx_q = (struct stmmac_rx_queue *)data;
5739 int chan = rx_q->queue_index;
5740 struct stmmac_priv *priv;
5741
5742 priv = container_of(rx_q, struct stmmac_priv, rx_queue[chan]);
5743
5744 if (unlikely(!data)) {
5745 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
5746 return IRQ_NONE;
5747 }
5748
5749
5750 if (test_bit(STMMAC_DOWN, &priv->state))
5751 return IRQ_HANDLED;
5752
5753 stmmac_napi_check(priv, chan, DMA_DIR_RX);
5754
5755 return IRQ_HANDLED;
5756}
5757
5758#ifdef CONFIG_NET_POLL_CONTROLLER
5759
5760
5761
5762static void stmmac_poll_controller(struct net_device *dev)
5763{
5764 struct stmmac_priv *priv = netdev_priv(dev);
5765 int i;
5766
5767
5768 if (test_bit(STMMAC_DOWN, &priv->state))
5769 return;
5770
5771 if (priv->plat->multi_msi_en) {
5772 for (i = 0; i < priv->plat->rx_queues_to_use; i++)
5773 stmmac_msi_intr_rx(0, &priv->rx_queue[i]);
5774
5775 for (i = 0; i < priv->plat->tx_queues_to_use; i++)
5776 stmmac_msi_intr_tx(0, &priv->tx_queue[i]);
5777 } else {
5778 disable_irq(dev->irq);
5779 stmmac_interrupt(dev->irq, dev);
5780 enable_irq(dev->irq);
5781 }
5782}
5783#endif
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
5795{
5796 struct stmmac_priv *priv = netdev_priv (dev);
5797 int ret = -EOPNOTSUPP;
5798
5799 if (!netif_running(dev))
5800 return -EINVAL;
5801
5802 switch (cmd) {
5803 case SIOCGMIIPHY:
5804 case SIOCGMIIREG:
5805 case SIOCSMIIREG:
5806 ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
5807 break;
5808 case SIOCSHWTSTAMP:
5809 ret = stmmac_hwtstamp_set(dev, rq);
5810 break;
5811 case SIOCGHWTSTAMP:
5812 ret = stmmac_hwtstamp_get(dev, rq);
5813 break;
5814 default:
5815 break;
5816 }
5817
5818 return ret;
5819}
5820
5821static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
5822 void *cb_priv)
5823{
5824 struct stmmac_priv *priv = cb_priv;
5825 int ret = -EOPNOTSUPP;
5826
5827 if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
5828 return ret;
5829
5830 __stmmac_disable_all_queues(priv);
5831
5832 switch (type) {
5833 case TC_SETUP_CLSU32:
5834 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
5835 break;
5836 case TC_SETUP_CLSFLOWER:
5837 ret = stmmac_tc_setup_cls(priv, priv, type_data);
5838 break;
5839 default:
5840 break;
5841 }
5842
5843 stmmac_enable_all_queues(priv);
5844 return ret;
5845}
5846
5847static LIST_HEAD(stmmac_block_cb_list);
5848
5849static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
5850 void *type_data)
5851{
5852 struct stmmac_priv *priv = netdev_priv(ndev);
5853
5854 switch (type) {
5855 case TC_SETUP_BLOCK:
5856 return flow_block_cb_setup_simple(type_data,
5857 &stmmac_block_cb_list,
5858 stmmac_setup_tc_block_cb,
5859 priv, priv, true);
5860 case TC_SETUP_QDISC_CBS:
5861 return stmmac_tc_setup_cbs(priv, priv, type_data);
5862 case TC_SETUP_QDISC_TAPRIO:
5863 return stmmac_tc_setup_taprio(priv, priv, type_data);
5864 case TC_SETUP_QDISC_ETF:
5865 return stmmac_tc_setup_etf(priv, priv, type_data);
5866 default:
5867 return -EOPNOTSUPP;
5868 }
5869}
5870
5871static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
5872 struct net_device *sb_dev)
5873{
5874 int gso = skb_shinfo(skb)->gso_type;
5875
5876 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
5877
5878
5879
5880
5881
5882
5883 return 0;
5884 }
5885
5886 return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
5887}
5888
5889static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
5890{
5891 struct stmmac_priv *priv = netdev_priv(ndev);
5892 int ret = 0;
5893
5894 ret = pm_runtime_get_sync(priv->device);
5895 if (ret < 0) {
5896 pm_runtime_put_noidle(priv->device);
5897 return ret;
5898 }
5899
5900 ret = eth_mac_addr(ndev, addr);
5901 if (ret)
5902 goto set_mac_error;
5903
5904 stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
5905
5906set_mac_error:
5907 pm_runtime_put(priv->device);
5908
5909 return ret;
5910}
5911
5912#ifdef CONFIG_DEBUG_FS
5913static struct dentry *stmmac_fs_dir;
5914
5915static void sysfs_display_ring(void *head, int size, int extend_desc,
5916 struct seq_file *seq, dma_addr_t dma_phy_addr)
5917{
5918 int i;
5919 struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
5920 struct dma_desc *p = (struct dma_desc *)head;
5921 dma_addr_t dma_addr;
5922
5923 for (i = 0; i < size; i++) {
5924 if (extend_desc) {
5925 dma_addr = dma_phy_addr + i * sizeof(*ep);
5926 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
5927 i, &dma_addr,
5928 le32_to_cpu(ep->basic.des0),
5929 le32_to_cpu(ep->basic.des1),
5930 le32_to_cpu(ep->basic.des2),
5931 le32_to_cpu(ep->basic.des3));
5932 ep++;
5933 } else {
5934 dma_addr = dma_phy_addr + i * sizeof(*p);
5935 seq_printf(seq, "%d [%pad]: 0x%x 0x%x 0x%x 0x%x\n",
5936 i, &dma_addr,
5937 le32_to_cpu(p->des0), le32_to_cpu(p->des1),
5938 le32_to_cpu(p->des2), le32_to_cpu(p->des3));
5939 p++;
5940 }
5941 seq_printf(seq, "\n");
5942 }
5943}
5944
5945static int stmmac_rings_status_show(struct seq_file *seq, void *v)
5946{
5947 struct net_device *dev = seq->private;
5948 struct stmmac_priv *priv = netdev_priv(dev);
5949 u32 rx_count = priv->plat->rx_queues_to_use;
5950 u32 tx_count = priv->plat->tx_queues_to_use;
5951 u32 queue;
5952
5953 if ((dev->flags & IFF_UP) == 0)
5954 return 0;
5955
5956 for (queue = 0; queue < rx_count; queue++) {
5957 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
5958
5959 seq_printf(seq, "RX Queue %d:\n", queue);
5960
5961 if (priv->extend_desc) {
5962 seq_printf(seq, "Extended descriptor ring:\n");
5963 sysfs_display_ring((void *)rx_q->dma_erx,
5964 priv->dma_rx_size, 1, seq, rx_q->dma_rx_phy);
5965 } else {
5966 seq_printf(seq, "Descriptor ring:\n");
5967 sysfs_display_ring((void *)rx_q->dma_rx,
5968 priv->dma_rx_size, 0, seq, rx_q->dma_rx_phy);
5969 }
5970 }
5971
5972 for (queue = 0; queue < tx_count; queue++) {
5973 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
5974
5975 seq_printf(seq, "TX Queue %d:\n", queue);
5976
5977 if (priv->extend_desc) {
5978 seq_printf(seq, "Extended descriptor ring:\n");
5979 sysfs_display_ring((void *)tx_q->dma_etx,
5980 priv->dma_tx_size, 1, seq, tx_q->dma_tx_phy);
5981 } else if (!(tx_q->tbs & STMMAC_TBS_AVAIL)) {
5982 seq_printf(seq, "Descriptor ring:\n");
5983 sysfs_display_ring((void *)tx_q->dma_tx,
5984 priv->dma_tx_size, 0, seq, tx_q->dma_tx_phy);
5985 }
5986 }
5987
5988 return 0;
5989}
5990DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
5991
5992static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
5993{
5994 struct net_device *dev = seq->private;
5995 struct stmmac_priv *priv = netdev_priv(dev);
5996
5997 if (!priv->hw_cap_support) {
5998 seq_printf(seq, "DMA HW features not supported\n");
5999 return 0;
6000 }
6001
6002 seq_printf(seq, "==============================\n");
6003 seq_printf(seq, "\tDMA HW features\n");
6004 seq_printf(seq, "==============================\n");
6005
6006 seq_printf(seq, "\t10/100 Mbps: %s\n",
6007 (priv->dma_cap.mbps_10_100) ? "Y" : "N");
6008 seq_printf(seq, "\t1000 Mbps: %s\n",
6009 (priv->dma_cap.mbps_1000) ? "Y" : "N");
6010 seq_printf(seq, "\tHalf duplex: %s\n",
6011 (priv->dma_cap.half_duplex) ? "Y" : "N");
6012 seq_printf(seq, "\tHash Filter: %s\n",
6013 (priv->dma_cap.hash_filter) ? "Y" : "N");
6014 seq_printf(seq, "\tMultiple MAC address registers: %s\n",
6015 (priv->dma_cap.multi_addr) ? "Y" : "N");
6016 seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
6017 (priv->dma_cap.pcs) ? "Y" : "N");
6018 seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
6019 (priv->dma_cap.sma_mdio) ? "Y" : "N");
6020 seq_printf(seq, "\tPMT Remote wake up: %s\n",
6021 (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
6022 seq_printf(seq, "\tPMT Magic Frame: %s\n",
6023 (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
6024 seq_printf(seq, "\tRMON module: %s\n",
6025 (priv->dma_cap.rmon) ? "Y" : "N");
6026 seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
6027 (priv->dma_cap.time_stamp) ? "Y" : "N");
6028 seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
6029 (priv->dma_cap.atime_stamp) ? "Y" : "N");
6030 seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
6031 (priv->dma_cap.eee) ? "Y" : "N");
6032 seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
6033 seq_printf(seq, "\tChecksum Offload in TX: %s\n",
6034 (priv->dma_cap.tx_coe) ? "Y" : "N");
6035 if (priv->synopsys_id >= DWMAC_CORE_4_00) {
6036 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
6037 (priv->dma_cap.rx_coe) ? "Y" : "N");
6038 } else {
6039 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
6040 (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
6041 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
6042 (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
6043 }
6044 seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
6045 (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
6046 seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
6047 priv->dma_cap.number_rx_channel);
6048 seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
6049 priv->dma_cap.number_tx_channel);
6050 seq_printf(seq, "\tNumber of Additional RX queues: %d\n",
6051 priv->dma_cap.number_rx_queues);
6052 seq_printf(seq, "\tNumber of Additional TX queues: %d\n",
6053 priv->dma_cap.number_tx_queues);
6054 seq_printf(seq, "\tEnhanced descriptors: %s\n",
6055 (priv->dma_cap.enh_desc) ? "Y" : "N");
6056 seq_printf(seq, "\tTX Fifo Size: %d\n", priv->dma_cap.tx_fifo_size);
6057 seq_printf(seq, "\tRX Fifo Size: %d\n", priv->dma_cap.rx_fifo_size);
6058 seq_printf(seq, "\tHash Table Size: %d\n", priv->dma_cap.hash_tb_sz);
6059 seq_printf(seq, "\tTSO: %s\n", priv->dma_cap.tsoen ? "Y" : "N");
6060 seq_printf(seq, "\tNumber of PPS Outputs: %d\n",
6061 priv->dma_cap.pps_out_num);
6062 seq_printf(seq, "\tSafety Features: %s\n",
6063 priv->dma_cap.asp ? "Y" : "N");
6064 seq_printf(seq, "\tFlexible RX Parser: %s\n",
6065 priv->dma_cap.frpsel ? "Y" : "N");
6066 seq_printf(seq, "\tEnhanced Addressing: %d\n",
6067 priv->dma_cap.addr64);
6068 seq_printf(seq, "\tReceive Side Scaling: %s\n",
6069 priv->dma_cap.rssen ? "Y" : "N");
6070 seq_printf(seq, "\tVLAN Hash Filtering: %s\n",
6071 priv->dma_cap.vlhash ? "Y" : "N");
6072 seq_printf(seq, "\tSplit Header: %s\n",
6073 priv->dma_cap.sphen ? "Y" : "N");
6074 seq_printf(seq, "\tVLAN TX Insertion: %s\n",
6075 priv->dma_cap.vlins ? "Y" : "N");
6076 seq_printf(seq, "\tDouble VLAN: %s\n",
6077 priv->dma_cap.dvlan ? "Y" : "N");
6078 seq_printf(seq, "\tNumber of L3/L4 Filters: %d\n",
6079 priv->dma_cap.l3l4fnum);
6080 seq_printf(seq, "\tARP Offloading: %s\n",
6081 priv->dma_cap.arpoffsel ? "Y" : "N");
6082 seq_printf(seq, "\tEnhancements to Scheduled Traffic (EST): %s\n",
6083 priv->dma_cap.estsel ? "Y" : "N");
6084 seq_printf(seq, "\tFrame Preemption (FPE): %s\n",
6085 priv->dma_cap.fpesel ? "Y" : "N");
6086 seq_printf(seq, "\tTime-Based Scheduling (TBS): %s\n",
6087 priv->dma_cap.tbssel ? "Y" : "N");
6088 return 0;
6089}
6090DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
6091
6092
6093
6094static int stmmac_device_event(struct notifier_block *unused,
6095 unsigned long event, void *ptr)
6096{
6097 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
6098 struct stmmac_priv *priv = netdev_priv(dev);
6099
6100 if (dev->netdev_ops != &stmmac_netdev_ops)
6101 goto done;
6102
6103 switch (event) {
6104 case NETDEV_CHANGENAME:
6105 if (priv->dbgfs_dir)
6106 priv->dbgfs_dir = debugfs_rename(stmmac_fs_dir,
6107 priv->dbgfs_dir,
6108 stmmac_fs_dir,
6109 dev->name);
6110 break;
6111 }
6112done:
6113 return NOTIFY_DONE;
6114}
6115
6116static struct notifier_block stmmac_notifier = {
6117 .notifier_call = stmmac_device_event,
6118};
6119
6120static void stmmac_init_fs(struct net_device *dev)
6121{
6122 struct stmmac_priv *priv = netdev_priv(dev);
6123
6124 rtnl_lock();
6125
6126
6127 priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
6128
6129
6130 debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
6131 &stmmac_rings_status_fops);
6132
6133
6134 debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
6135 &stmmac_dma_cap_fops);
6136
6137 rtnl_unlock();
6138}
6139
6140static void stmmac_exit_fs(struct net_device *dev)
6141{
6142 struct stmmac_priv *priv = netdev_priv(dev);
6143
6144 debugfs_remove_recursive(priv->dbgfs_dir);
6145}
6146#endif
6147
6148static u32 stmmac_vid_crc32_le(__le16 vid_le)
6149{
6150 unsigned char *data = (unsigned char *)&vid_le;
6151 unsigned char data_byte = 0;
6152 u32 crc = ~0x0;
6153 u32 temp = 0;
6154 int i, bits;
6155
6156 bits = get_bitmask_order(VLAN_VID_MASK);
6157 for (i = 0; i < bits; i++) {
6158 if ((i % 8) == 0)
6159 data_byte = data[i / 8];
6160
6161 temp = ((crc & 1) ^ data_byte) & 1;
6162 crc >>= 1;
6163 data_byte >>= 1;
6164
6165 if (temp)
6166 crc ^= 0xedb88320;
6167 }
6168
6169 return crc;
6170}
6171
6172static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
6173{
6174 u32 crc, hash = 0;
6175 __le16 pmatch = 0;
6176 int count = 0;
6177 u16 vid = 0;
6178
6179 for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
6180 __le16 vid_le = cpu_to_le16(vid);
6181 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
6182 hash |= (1 << crc);
6183 count++;
6184 }
6185
6186 if (!priv->dma_cap.vlhash) {
6187 if (count > 2)
6188 return -EOPNOTSUPP;
6189
6190 pmatch = cpu_to_le16(vid);
6191 hash = 0;
6192 }
6193
6194 return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
6195}
6196
6197static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
6198{
6199 struct stmmac_priv *priv = netdev_priv(ndev);
6200 bool is_double = false;
6201 int ret;
6202
6203 if (be16_to_cpu(proto) == ETH_P_8021AD)
6204 is_double = true;
6205
6206 set_bit(vid, priv->active_vlans);
6207 ret = stmmac_vlan_update(priv, is_double);
6208 if (ret) {
6209 clear_bit(vid, priv->active_vlans);
6210 return ret;
6211 }
6212
6213 if (priv->hw->num_vlan) {
6214 ret = stmmac_add_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
6215 if (ret)
6216 return ret;
6217 }
6218
6219 return 0;
6220}
6221
6222static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
6223{
6224 struct stmmac_priv *priv = netdev_priv(ndev);
6225 bool is_double = false;
6226 int ret;
6227
6228 ret = pm_runtime_get_sync(priv->device);
6229 if (ret < 0) {
6230 pm_runtime_put_noidle(priv->device);
6231 return ret;
6232 }
6233
6234 if (be16_to_cpu(proto) == ETH_P_8021AD)
6235 is_double = true;
6236
6237 clear_bit(vid, priv->active_vlans);
6238
6239 if (priv->hw->num_vlan) {
6240 ret = stmmac_del_hw_vlan_rx_fltr(priv, ndev, priv->hw, proto, vid);
6241 if (ret)
6242 goto del_vlan_error;
6243 }
6244
6245 ret = stmmac_vlan_update(priv, is_double);
6246
6247del_vlan_error:
6248 pm_runtime_put(priv->device);
6249
6250 return ret;
6251}
6252
6253static int stmmac_bpf(struct net_device *dev, struct netdev_bpf *bpf)
6254{
6255 struct stmmac_priv *priv = netdev_priv(dev);
6256
6257 switch (bpf->command) {
6258 case XDP_SETUP_PROG:
6259 return stmmac_xdp_set_prog(priv, bpf->prog, bpf->extack);
6260 case XDP_SETUP_XSK_POOL:
6261 return stmmac_xdp_setup_pool(priv, bpf->xsk.pool,
6262 bpf->xsk.queue_id);
6263 default:
6264 return -EOPNOTSUPP;
6265 }
6266}
6267
6268static int stmmac_xdp_xmit(struct net_device *dev, int num_frames,
6269 struct xdp_frame **frames, u32 flags)
6270{
6271 struct stmmac_priv *priv = netdev_priv(dev);
6272 int cpu = smp_processor_id();
6273 struct netdev_queue *nq;
6274 int i, nxmit = 0;
6275 int queue;
6276
6277 if (unlikely(test_bit(STMMAC_DOWN, &priv->state)))
6278 return -ENETDOWN;
6279
6280 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
6281 return -EINVAL;
6282
6283 queue = stmmac_xdp_get_tx_queue(priv, cpu);
6284 nq = netdev_get_tx_queue(priv->dev, queue);
6285
6286 __netif_tx_lock(nq, cpu);
6287
6288 nq->trans_start = jiffies;
6289
6290 for (i = 0; i < num_frames; i++) {
6291 int res;
6292
6293 res = stmmac_xdp_xmit_xdpf(priv, queue, frames[i], true);
6294 if (res == STMMAC_XDP_CONSUMED)
6295 break;
6296
6297 nxmit++;
6298 }
6299
6300 if (flags & XDP_XMIT_FLUSH) {
6301 stmmac_flush_tx_descriptors(priv, queue);
6302 stmmac_tx_timer_arm(priv, queue);
6303 }
6304
6305 __netif_tx_unlock(nq);
6306
6307 return nxmit;
6308}
6309
6310void stmmac_disable_rx_queue(struct stmmac_priv *priv, u32 queue)
6311{
6312 struct stmmac_channel *ch = &priv->channel[queue];
6313 unsigned long flags;
6314
6315 spin_lock_irqsave(&ch->lock, flags);
6316 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
6317 spin_unlock_irqrestore(&ch->lock, flags);
6318
6319 stmmac_stop_rx_dma(priv, queue);
6320 __free_dma_rx_desc_resources(priv, queue);
6321}
6322
6323void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue)
6324{
6325 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
6326 struct stmmac_channel *ch = &priv->channel[queue];
6327 unsigned long flags;
6328 u32 buf_size;
6329 int ret;
6330
6331 ret = __alloc_dma_rx_desc_resources(priv, queue);
6332 if (ret) {
6333 netdev_err(priv->dev, "Failed to alloc RX desc.\n");
6334 return;
6335 }
6336
6337 ret = __init_dma_rx_desc_rings(priv, queue, GFP_KERNEL);
6338 if (ret) {
6339 __free_dma_rx_desc_resources(priv, queue);
6340 netdev_err(priv->dev, "Failed to init RX desc.\n");
6341 return;
6342 }
6343
6344 stmmac_clear_rx_descriptors(priv, queue);
6345
6346 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
6347 rx_q->dma_rx_phy, rx_q->queue_index);
6348
6349 rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num *
6350 sizeof(struct dma_desc));
6351 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
6352 rx_q->rx_tail_addr, rx_q->queue_index);
6353
6354 if (rx_q->xsk_pool && rx_q->buf_alloc_num) {
6355 buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool);
6356 stmmac_set_dma_bfsize(priv, priv->ioaddr,
6357 buf_size,
6358 rx_q->queue_index);
6359 } else {
6360 stmmac_set_dma_bfsize(priv, priv->ioaddr,
6361 priv->dma_buf_sz,
6362 rx_q->queue_index);
6363 }
6364
6365 stmmac_start_rx_dma(priv, queue);
6366
6367 spin_lock_irqsave(&ch->lock, flags);
6368 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 1, 0);
6369 spin_unlock_irqrestore(&ch->lock, flags);
6370}
6371
6372void stmmac_disable_tx_queue(struct stmmac_priv *priv, u32 queue)
6373{
6374 struct stmmac_channel *ch = &priv->channel[queue];
6375 unsigned long flags;
6376
6377 spin_lock_irqsave(&ch->lock, flags);
6378 stmmac_disable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
6379 spin_unlock_irqrestore(&ch->lock, flags);
6380
6381 stmmac_stop_tx_dma(priv, queue);
6382 __free_dma_tx_desc_resources(priv, queue);
6383}
6384
6385void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue)
6386{
6387 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
6388 struct stmmac_channel *ch = &priv->channel[queue];
6389 unsigned long flags;
6390 int ret;
6391
6392 ret = __alloc_dma_tx_desc_resources(priv, queue);
6393 if (ret) {
6394 netdev_err(priv->dev, "Failed to alloc TX desc.\n");
6395 return;
6396 }
6397
6398 ret = __init_dma_tx_desc_rings(priv, queue);
6399 if (ret) {
6400 __free_dma_tx_desc_resources(priv, queue);
6401 netdev_err(priv->dev, "Failed to init TX desc.\n");
6402 return;
6403 }
6404
6405 stmmac_clear_tx_descriptors(priv, queue);
6406
6407 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
6408 tx_q->dma_tx_phy, tx_q->queue_index);
6409
6410 if (tx_q->tbs & STMMAC_TBS_AVAIL)
6411 stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index);
6412
6413 tx_q->tx_tail_addr = tx_q->dma_tx_phy;
6414 stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
6415 tx_q->tx_tail_addr, tx_q->queue_index);
6416
6417 stmmac_start_tx_dma(priv, queue);
6418
6419 spin_lock_irqsave(&ch->lock, flags);
6420 stmmac_enable_dma_irq(priv, priv->ioaddr, queue, 0, 1);
6421 spin_unlock_irqrestore(&ch->lock, flags);
6422}
6423
6424int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags)
6425{
6426 struct stmmac_priv *priv = netdev_priv(dev);
6427 struct stmmac_rx_queue *rx_q;
6428 struct stmmac_tx_queue *tx_q;
6429 struct stmmac_channel *ch;
6430
6431 if (test_bit(STMMAC_DOWN, &priv->state) ||
6432 !netif_carrier_ok(priv->dev))
6433 return -ENETDOWN;
6434
6435 if (!stmmac_xdp_is_enabled(priv))
6436 return -ENXIO;
6437
6438 if (queue >= priv->plat->rx_queues_to_use ||
6439 queue >= priv->plat->tx_queues_to_use)
6440 return -EINVAL;
6441
6442 rx_q = &priv->rx_queue[queue];
6443 tx_q = &priv->tx_queue[queue];
6444 ch = &priv->channel[queue];
6445
6446 if (!rx_q->xsk_pool && !tx_q->xsk_pool)
6447 return -ENXIO;
6448
6449 if (!napi_if_scheduled_mark_missed(&ch->rxtx_napi)) {
6450
6451
6452
6453 if (likely(napi_schedule_prep(&ch->rxtx_napi)))
6454 __napi_schedule(&ch->rxtx_napi);
6455 }
6456
6457 return 0;
6458}
6459
6460static const struct net_device_ops stmmac_netdev_ops = {
6461 .ndo_open = stmmac_open,
6462 .ndo_start_xmit = stmmac_xmit,
6463 .ndo_stop = stmmac_release,
6464 .ndo_change_mtu = stmmac_change_mtu,
6465 .ndo_fix_features = stmmac_fix_features,
6466 .ndo_set_features = stmmac_set_features,
6467 .ndo_set_rx_mode = stmmac_set_rx_mode,
6468 .ndo_tx_timeout = stmmac_tx_timeout,
6469 .ndo_eth_ioctl = stmmac_ioctl,
6470 .ndo_setup_tc = stmmac_setup_tc,
6471 .ndo_select_queue = stmmac_select_queue,
6472#ifdef CONFIG_NET_POLL_CONTROLLER
6473 .ndo_poll_controller = stmmac_poll_controller,
6474#endif
6475 .ndo_set_mac_address = stmmac_set_mac_address,
6476 .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
6477 .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
6478 .ndo_bpf = stmmac_bpf,
6479 .ndo_xdp_xmit = stmmac_xdp_xmit,
6480 .ndo_xsk_wakeup = stmmac_xsk_wakeup,
6481};
6482
6483static void stmmac_reset_subtask(struct stmmac_priv *priv)
6484{
6485 if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
6486 return;
6487 if (test_bit(STMMAC_DOWN, &priv->state))
6488 return;
6489
6490 netdev_err(priv->dev, "Reset adapter.\n");
6491
6492 rtnl_lock();
6493 netif_trans_update(priv->dev);
6494 while (test_and_set_bit(STMMAC_RESETING, &priv->state))
6495 usleep_range(1000, 2000);
6496
6497 set_bit(STMMAC_DOWN, &priv->state);
6498 dev_close(priv->dev);
6499 dev_open(priv->dev, NULL);
6500 clear_bit(STMMAC_DOWN, &priv->state);
6501 clear_bit(STMMAC_RESETING, &priv->state);
6502 rtnl_unlock();
6503}
6504
6505static void stmmac_service_task(struct work_struct *work)
6506{
6507 struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
6508 service_task);
6509
6510 stmmac_reset_subtask(priv);
6511 clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
6512}
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522static int stmmac_hw_init(struct stmmac_priv *priv)
6523{
6524 int ret;
6525
6526
6527 if (priv->plat->has_sun8i)
6528 chain_mode = 1;
6529 priv->chain_mode = chain_mode;
6530
6531
6532 ret = stmmac_hwif_init(priv);
6533 if (ret)
6534 return ret;
6535
6536
6537 priv->hw_cap_support = stmmac_get_hw_features(priv);
6538 if (priv->hw_cap_support) {
6539 dev_info(priv->device, "DMA HW capability register supported\n");
6540
6541
6542
6543
6544
6545
6546 priv->plat->enh_desc = priv->dma_cap.enh_desc;
6547 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up &&
6548 !priv->plat->use_phy_wol;
6549 priv->hw->pmt = priv->plat->pmt;
6550 if (priv->dma_cap.hash_tb_sz) {
6551 priv->hw->multicast_filter_bins =
6552 (BIT(priv->dma_cap.hash_tb_sz) << 5);
6553 priv->hw->mcast_bits_log2 =
6554 ilog2(priv->hw->multicast_filter_bins);
6555 }
6556
6557
6558 if (priv->plat->force_thresh_dma_mode)
6559 priv->plat->tx_coe = 0;
6560 else
6561 priv->plat->tx_coe = priv->dma_cap.tx_coe;
6562
6563
6564 priv->plat->rx_coe = priv->dma_cap.rx_coe;
6565
6566 if (priv->dma_cap.rx_coe_type2)
6567 priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
6568 else if (priv->dma_cap.rx_coe_type1)
6569 priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
6570
6571 } else {
6572 dev_info(priv->device, "No HW DMA feature register supported\n");
6573 }
6574
6575 if (priv->plat->rx_coe) {
6576 priv->hw->rx_csum = priv->plat->rx_coe;
6577 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
6578 if (priv->synopsys_id < DWMAC_CORE_4_00)
6579 dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
6580 }
6581 if (priv->plat->tx_coe)
6582 dev_info(priv->device, "TX Checksum insertion supported\n");
6583
6584 if (priv->plat->pmt) {
6585 dev_info(priv->device, "Wake-Up On Lan supported\n");
6586 device_set_wakeup_capable(priv->device, 1);
6587 }
6588
6589 if (priv->dma_cap.tsoen)
6590 dev_info(priv->device, "TSO supported\n");
6591
6592 priv->hw->vlan_fail_q_en = priv->plat->vlan_fail_q_en;
6593 priv->hw->vlan_fail_q = priv->plat->vlan_fail_q;
6594
6595
6596 if (priv->hwif_quirks) {
6597 ret = priv->hwif_quirks(priv);
6598 if (ret)
6599 return ret;
6600 }
6601
6602
6603
6604
6605
6606
6607 if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
6608 (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
6609 priv->use_riwt = 1;
6610 dev_info(priv->device,
6611 "Enable RX Mitigation via HW Watchdog Timer\n");
6612 }
6613
6614 return 0;
6615}
6616
6617static void stmmac_napi_add(struct net_device *dev)
6618{
6619 struct stmmac_priv *priv = netdev_priv(dev);
6620 u32 queue, maxq;
6621
6622 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
6623
6624 for (queue = 0; queue < maxq; queue++) {
6625 struct stmmac_channel *ch = &priv->channel[queue];
6626
6627 ch->priv_data = priv;
6628 ch->index = queue;
6629 spin_lock_init(&ch->lock);
6630
6631 if (queue < priv->plat->rx_queues_to_use) {
6632 netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx,
6633 NAPI_POLL_WEIGHT);
6634 }
6635 if (queue < priv->plat->tx_queues_to_use) {
6636 netif_tx_napi_add(dev, &ch->tx_napi,
6637 stmmac_napi_poll_tx,
6638 NAPI_POLL_WEIGHT);
6639 }
6640 if (queue < priv->plat->rx_queues_to_use &&
6641 queue < priv->plat->tx_queues_to_use) {
6642 netif_napi_add(dev, &ch->rxtx_napi,
6643 stmmac_napi_poll_rxtx,
6644 NAPI_POLL_WEIGHT);
6645 }
6646 }
6647}
6648
6649static void stmmac_napi_del(struct net_device *dev)
6650{
6651 struct stmmac_priv *priv = netdev_priv(dev);
6652 u32 queue, maxq;
6653
6654 maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
6655
6656 for (queue = 0; queue < maxq; queue++) {
6657 struct stmmac_channel *ch = &priv->channel[queue];
6658
6659 if (queue < priv->plat->rx_queues_to_use)
6660 netif_napi_del(&ch->rx_napi);
6661 if (queue < priv->plat->tx_queues_to_use)
6662 netif_napi_del(&ch->tx_napi);
6663 if (queue < priv->plat->rx_queues_to_use &&
6664 queue < priv->plat->tx_queues_to_use) {
6665 netif_napi_del(&ch->rxtx_napi);
6666 }
6667 }
6668}
6669
6670int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt)
6671{
6672 struct stmmac_priv *priv = netdev_priv(dev);
6673 int ret = 0;
6674
6675 if (netif_running(dev))
6676 stmmac_release(dev);
6677
6678 stmmac_napi_del(dev);
6679
6680 priv->plat->rx_queues_to_use = rx_cnt;
6681 priv->plat->tx_queues_to_use = tx_cnt;
6682
6683 stmmac_napi_add(dev);
6684
6685 if (netif_running(dev))
6686 ret = stmmac_open(dev);
6687
6688 return ret;
6689}
6690
6691int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size)
6692{
6693 struct stmmac_priv *priv = netdev_priv(dev);
6694 int ret = 0;
6695
6696 if (netif_running(dev))
6697 stmmac_release(dev);
6698
6699 priv->dma_rx_size = rx_size;
6700 priv->dma_tx_size = tx_size;
6701
6702 if (netif_running(dev))
6703 ret = stmmac_open(dev);
6704
6705 return ret;
6706}
6707
6708#define SEND_VERIFY_MPAKCET_FMT "Send Verify mPacket lo_state=%d lp_state=%d\n"
6709static void stmmac_fpe_lp_task(struct work_struct *work)
6710{
6711 struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
6712 fpe_task);
6713 struct stmmac_fpe_cfg *fpe_cfg = priv->plat->fpe_cfg;
6714 enum stmmac_fpe_state *lo_state = &fpe_cfg->lo_fpe_state;
6715 enum stmmac_fpe_state *lp_state = &fpe_cfg->lp_fpe_state;
6716 bool *hs_enable = &fpe_cfg->hs_enable;
6717 bool *enable = &fpe_cfg->enable;
6718 int retries = 20;
6719
6720 while (retries-- > 0) {
6721
6722 if (*lo_state == FPE_STATE_OFF || !*hs_enable)
6723 break;
6724
6725 if (*lo_state == FPE_STATE_ENTERING_ON &&
6726 *lp_state == FPE_STATE_ENTERING_ON) {
6727 stmmac_fpe_configure(priv, priv->ioaddr,
6728 priv->plat->tx_queues_to_use,
6729 priv->plat->rx_queues_to_use,
6730 *enable);
6731
6732 netdev_info(priv->dev, "configured FPE\n");
6733
6734 *lo_state = FPE_STATE_ON;
6735 *lp_state = FPE_STATE_ON;
6736 netdev_info(priv->dev, "!!! BOTH FPE stations ON\n");
6737 break;
6738 }
6739
6740 if ((*lo_state == FPE_STATE_CAPABLE ||
6741 *lo_state == FPE_STATE_ENTERING_ON) &&
6742 *lp_state != FPE_STATE_ON) {
6743 netdev_info(priv->dev, SEND_VERIFY_MPAKCET_FMT,
6744 *lo_state, *lp_state);
6745 stmmac_fpe_send_mpacket(priv, priv->ioaddr,
6746 MPACKET_VERIFY);
6747 }
6748
6749 msleep(500);
6750 }
6751
6752 clear_bit(__FPE_TASK_SCHED, &priv->fpe_task_state);
6753}
6754
6755void stmmac_fpe_handshake(struct stmmac_priv *priv, bool enable)
6756{
6757 if (priv->plat->fpe_cfg->hs_enable != enable) {
6758 if (enable) {
6759 stmmac_fpe_send_mpacket(priv, priv->ioaddr,
6760 MPACKET_VERIFY);
6761 } else {
6762 priv->plat->fpe_cfg->lo_fpe_state = FPE_STATE_OFF;
6763 priv->plat->fpe_cfg->lp_fpe_state = FPE_STATE_OFF;
6764 }
6765
6766 priv->plat->fpe_cfg->hs_enable = enable;
6767 }
6768}
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780int stmmac_dvr_probe(struct device *device,
6781 struct plat_stmmacenet_data *plat_dat,
6782 struct stmmac_resources *res)
6783{
6784 struct net_device *ndev = NULL;
6785 struct stmmac_priv *priv;
6786 u32 rxq;
6787 int i, ret = 0;
6788
6789 ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
6790 MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
6791 if (!ndev)
6792 return -ENOMEM;
6793
6794 SET_NETDEV_DEV(ndev, device);
6795
6796 priv = netdev_priv(ndev);
6797 priv->device = device;
6798 priv->dev = ndev;
6799
6800 stmmac_set_ethtool_ops(ndev);
6801 priv->pause = pause;
6802 priv->plat = plat_dat;
6803 priv->ioaddr = res->addr;
6804 priv->dev->base_addr = (unsigned long)res->addr;
6805 priv->plat->dma_cfg->multi_msi_en = priv->plat->multi_msi_en;
6806
6807 priv->dev->irq = res->irq;
6808 priv->wol_irq = res->wol_irq;
6809 priv->lpi_irq = res->lpi_irq;
6810 priv->sfty_ce_irq = res->sfty_ce_irq;
6811 priv->sfty_ue_irq = res->sfty_ue_irq;
6812 for (i = 0; i < MTL_MAX_RX_QUEUES; i++)
6813 priv->rx_irq[i] = res->rx_irq[i];
6814 for (i = 0; i < MTL_MAX_TX_QUEUES; i++)
6815 priv->tx_irq[i] = res->tx_irq[i];
6816
6817 if (!is_zero_ether_addr(res->mac))
6818 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
6819
6820 dev_set_drvdata(device, priv->dev);
6821
6822
6823 stmmac_verify_args();
6824
6825 priv->af_xdp_zc_qps = bitmap_zalloc(MTL_MAX_TX_QUEUES, GFP_KERNEL);
6826 if (!priv->af_xdp_zc_qps)
6827 return -ENOMEM;
6828
6829
6830 priv->wq = create_singlethread_workqueue("stmmac_wq");
6831 if (!priv->wq) {
6832 dev_err(priv->device, "failed to create workqueue\n");
6833 return -ENOMEM;
6834 }
6835
6836 INIT_WORK(&priv->service_task, stmmac_service_task);
6837
6838
6839 INIT_WORK(&priv->fpe_task, stmmac_fpe_lp_task);
6840
6841
6842
6843
6844 if ((phyaddr >= 0) && (phyaddr <= 31))
6845 priv->plat->phy_addr = phyaddr;
6846
6847 if (priv->plat->stmmac_rst) {
6848 ret = reset_control_assert(priv->plat->stmmac_rst);
6849 reset_control_deassert(priv->plat->stmmac_rst);
6850
6851
6852
6853 if (ret == -ENOTSUPP)
6854 reset_control_reset(priv->plat->stmmac_rst);
6855 }
6856
6857 ret = reset_control_deassert(priv->plat->stmmac_ahb_rst);
6858 if (ret == -ENOTSUPP)
6859 dev_err(priv->device, "unable to bring out of ahb reset: %pe\n",
6860 ERR_PTR(ret));
6861
6862
6863 ret = stmmac_hw_init(priv);
6864 if (ret)
6865 goto error_hw_init;
6866
6867
6868
6869 if (priv->synopsys_id < DWMAC_CORE_5_20)
6870 priv->plat->dma_cfg->dche = false;
6871
6872 stmmac_check_ether_addr(priv);
6873
6874 ndev->netdev_ops = &stmmac_netdev_ops;
6875
6876 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6877 NETIF_F_RXCSUM;
6878
6879 ret = stmmac_tc_init(priv, priv);
6880 if (!ret) {
6881 ndev->hw_features |= NETIF_F_HW_TC;
6882 }
6883
6884 if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
6885 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
6886 if (priv->plat->has_gmac4)
6887 ndev->hw_features |= NETIF_F_GSO_UDP_L4;
6888 priv->tso = true;
6889 dev_info(priv->device, "TSO feature enabled\n");
6890 }
6891
6892 if (priv->dma_cap.sphen) {
6893 ndev->hw_features |= NETIF_F_GRO;
6894 priv->sph_cap = true;
6895 priv->sph = priv->sph_cap;
6896 dev_info(priv->device, "SPH feature enabled\n");
6897 }
6898
6899
6900
6901
6902
6903
6904 if (priv->plat->addr64)
6905 priv->dma_cap.addr64 = priv->plat->addr64;
6906
6907 if (priv->dma_cap.addr64) {
6908 ret = dma_set_mask_and_coherent(device,
6909 DMA_BIT_MASK(priv->dma_cap.addr64));
6910 if (!ret) {
6911 dev_info(priv->device, "Using %d bits DMA width\n",
6912 priv->dma_cap.addr64);
6913
6914
6915
6916
6917
6918 if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
6919 priv->plat->dma_cfg->eame = true;
6920 } else {
6921 ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
6922 if (ret) {
6923 dev_err(priv->device, "Failed to set DMA Mask\n");
6924 goto error_hw_init;
6925 }
6926
6927 priv->dma_cap.addr64 = 32;
6928 }
6929 }
6930
6931 ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
6932 ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
6933#ifdef STMMAC_VLAN_TAG_USED
6934
6935 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
6936 if (priv->dma_cap.vlhash) {
6937 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
6938 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
6939 }
6940 if (priv->dma_cap.vlins) {
6941 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
6942 if (priv->dma_cap.dvlan)
6943 ndev->features |= NETIF_F_HW_VLAN_STAG_TX;
6944 }
6945#endif
6946 priv->msg_enable = netif_msg_init(debug, default_msg_level);
6947
6948
6949 rxq = priv->plat->rx_queues_to_use;
6950 netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
6951 for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
6952 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
6953
6954 if (priv->dma_cap.rssen && priv->plat->rss_en)
6955 ndev->features |= NETIF_F_RXHASH;
6956
6957
6958 ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
6959 if (priv->plat->has_xgmac)
6960 ndev->max_mtu = XGMAC_JUMBO_LEN;
6961 else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
6962 ndev->max_mtu = JUMBO_LEN;
6963 else
6964 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
6965
6966
6967
6968 if ((priv->plat->maxmtu < ndev->max_mtu) &&
6969 (priv->plat->maxmtu >= ndev->min_mtu))
6970 ndev->max_mtu = priv->plat->maxmtu;
6971 else if (priv->plat->maxmtu < ndev->min_mtu)
6972 dev_warn(priv->device,
6973 "%s: warning: maxmtu having invalid value (%d)\n",
6974 __func__, priv->plat->maxmtu);
6975
6976 if (flow_ctrl)
6977 priv->flow_ctrl = FLOW_AUTO;
6978
6979
6980 stmmac_napi_add(ndev);
6981
6982 mutex_init(&priv->lock);
6983
6984
6985
6986
6987
6988
6989
6990 if (priv->plat->clk_csr >= 0)
6991 priv->clk_csr = priv->plat->clk_csr;
6992 else
6993 stmmac_clk_csr_set(priv);
6994
6995 stmmac_check_pcs_mode(priv);
6996
6997 pm_runtime_get_noresume(device);
6998 pm_runtime_set_active(device);
6999 pm_runtime_enable(device);
7000
7001 if (priv->hw->pcs != STMMAC_PCS_TBI &&
7002 priv->hw->pcs != STMMAC_PCS_RTBI) {
7003
7004 ret = stmmac_mdio_register(ndev);
7005 if (ret < 0) {
7006 dev_err(priv->device,
7007 "%s: MDIO bus (id: %d) registration failed",
7008 __func__, priv->plat->bus_id);
7009 goto error_mdio_register;
7010 }
7011 }
7012
7013 if (priv->plat->speed_mode_2500)
7014 priv->plat->speed_mode_2500(ndev, priv->plat->bsp_priv);
7015
7016 if (priv->plat->mdio_bus_data && priv->plat->mdio_bus_data->has_xpcs) {
7017 ret = stmmac_xpcs_setup(priv->mii);
7018 if (ret)
7019 goto error_xpcs_setup;
7020 }
7021
7022 ret = stmmac_phy_setup(priv);
7023 if (ret) {
7024 netdev_err(ndev, "failed to setup phy (%d)\n", ret);
7025 goto error_phy_setup;
7026 }
7027
7028 ret = register_netdev(ndev);
7029 if (ret) {
7030 dev_err(priv->device, "%s: ERROR %i registering the device\n",
7031 __func__, ret);
7032 goto error_netdev_register;
7033 }
7034
7035 if (priv->plat->serdes_powerup) {
7036 ret = priv->plat->serdes_powerup(ndev,
7037 priv->plat->bsp_priv);
7038
7039 if (ret < 0)
7040 goto error_serdes_powerup;
7041 }
7042
7043#ifdef CONFIG_DEBUG_FS
7044 stmmac_init_fs(ndev);
7045#endif
7046
7047
7048
7049
7050 pm_runtime_put(device);
7051
7052 return ret;
7053
7054error_serdes_powerup:
7055 unregister_netdev(ndev);
7056error_netdev_register:
7057 phylink_destroy(priv->phylink);
7058error_xpcs_setup:
7059error_phy_setup:
7060 if (priv->hw->pcs != STMMAC_PCS_TBI &&
7061 priv->hw->pcs != STMMAC_PCS_RTBI)
7062 stmmac_mdio_unregister(ndev);
7063error_mdio_register:
7064 stmmac_napi_del(ndev);
7065error_hw_init:
7066 destroy_workqueue(priv->wq);
7067 bitmap_free(priv->af_xdp_zc_qps);
7068
7069 return ret;
7070}
7071EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
7072
7073
7074
7075
7076
7077
7078
7079int stmmac_dvr_remove(struct device *dev)
7080{
7081 struct net_device *ndev = dev_get_drvdata(dev);
7082 struct stmmac_priv *priv = netdev_priv(ndev);
7083
7084 netdev_info(priv->dev, "%s: removing driver", __func__);
7085
7086 stmmac_stop_all_dma(priv);
7087 stmmac_mac_set(priv, priv->ioaddr, false);
7088 netif_carrier_off(ndev);
7089 unregister_netdev(ndev);
7090
7091
7092
7093
7094 if (priv->plat->serdes_powerdown)
7095 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
7096
7097#ifdef CONFIG_DEBUG_FS
7098 stmmac_exit_fs(ndev);
7099#endif
7100 phylink_destroy(priv->phylink);
7101 if (priv->plat->stmmac_rst)
7102 reset_control_assert(priv->plat->stmmac_rst);
7103 reset_control_assert(priv->plat->stmmac_ahb_rst);
7104 pm_runtime_put(dev);
7105 pm_runtime_disable(dev);
7106 if (priv->hw->pcs != STMMAC_PCS_TBI &&
7107 priv->hw->pcs != STMMAC_PCS_RTBI)
7108 stmmac_mdio_unregister(ndev);
7109 destroy_workqueue(priv->wq);
7110 mutex_destroy(&priv->lock);
7111 bitmap_free(priv->af_xdp_zc_qps);
7112
7113 return 0;
7114}
7115EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
7116
7117
7118
7119
7120
7121
7122
7123
7124int stmmac_suspend(struct device *dev)
7125{
7126 struct net_device *ndev = dev_get_drvdata(dev);
7127 struct stmmac_priv *priv = netdev_priv(ndev);
7128 u32 chan;
7129
7130 if (!ndev || !netif_running(ndev))
7131 return 0;
7132
7133 mutex_lock(&priv->lock);
7134
7135 netif_device_detach(ndev);
7136
7137 stmmac_disable_all_queues(priv);
7138
7139 for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
7140 hrtimer_cancel(&priv->tx_queue[chan].txtimer);
7141
7142 if (priv->eee_enabled) {
7143 priv->tx_path_in_lpi_mode = false;
7144 del_timer_sync(&priv->eee_ctrl_timer);
7145 }
7146
7147
7148 stmmac_stop_all_dma(priv);
7149
7150 if (priv->plat->serdes_powerdown)
7151 priv->plat->serdes_powerdown(ndev, priv->plat->bsp_priv);
7152
7153
7154 if (device_may_wakeup(priv->device) && priv->plat->pmt) {
7155 stmmac_pmt(priv, priv->hw, priv->wolopts);
7156 priv->irq_wake = 1;
7157 } else {
7158 stmmac_mac_set(priv, priv->ioaddr, false);
7159 pinctrl_pm_select_sleep_state(priv->device);
7160 }
7161
7162 mutex_unlock(&priv->lock);
7163
7164 rtnl_lock();
7165 if (device_may_wakeup(priv->device) && priv->plat->pmt) {
7166 phylink_suspend(priv->phylink, true);
7167 } else {
7168 if (device_may_wakeup(priv->device))
7169 phylink_speed_down(priv->phylink, false);
7170 phylink_suspend(priv->phylink, false);
7171 }
7172 rtnl_unlock();
7173
7174 if (priv->dma_cap.fpesel) {
7175
7176 stmmac_fpe_configure(priv, priv->ioaddr,
7177 priv->plat->tx_queues_to_use,
7178 priv->plat->rx_queues_to_use, false);
7179
7180 stmmac_fpe_handshake(priv, false);
7181 stmmac_fpe_stop_wq(priv);
7182 }
7183
7184 priv->speed = SPEED_UNKNOWN;
7185 return 0;
7186}
7187EXPORT_SYMBOL_GPL(stmmac_suspend);
7188
7189
7190
7191
7192
7193static void stmmac_reset_queues_param(struct stmmac_priv *priv)
7194{
7195 u32 rx_cnt = priv->plat->rx_queues_to_use;
7196 u32 tx_cnt = priv->plat->tx_queues_to_use;
7197 u32 queue;
7198
7199 for (queue = 0; queue < rx_cnt; queue++) {
7200 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
7201
7202 rx_q->cur_rx = 0;
7203 rx_q->dirty_rx = 0;
7204 }
7205
7206 for (queue = 0; queue < tx_cnt; queue++) {
7207 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
7208
7209 tx_q->cur_tx = 0;
7210 tx_q->dirty_tx = 0;
7211 tx_q->mss = 0;
7212
7213 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
7214 }
7215}
7216
7217
7218
7219
7220
7221
7222
7223int stmmac_resume(struct device *dev)
7224{
7225 struct net_device *ndev = dev_get_drvdata(dev);
7226 struct stmmac_priv *priv = netdev_priv(ndev);
7227 int ret;
7228
7229 if (!netif_running(ndev))
7230 return 0;
7231
7232
7233
7234
7235
7236
7237
7238 if (device_may_wakeup(priv->device) && priv->plat->pmt) {
7239 mutex_lock(&priv->lock);
7240 stmmac_pmt(priv, priv->hw, 0);
7241 mutex_unlock(&priv->lock);
7242 priv->irq_wake = 0;
7243 } else {
7244 pinctrl_pm_select_default_state(priv->device);
7245
7246 if (priv->mii)
7247 stmmac_mdio_reset(priv->mii);
7248 }
7249
7250 if (priv->plat->serdes_powerup) {
7251 ret = priv->plat->serdes_powerup(ndev,
7252 priv->plat->bsp_priv);
7253
7254 if (ret < 0)
7255 return ret;
7256 }
7257
7258 rtnl_lock();
7259 if (device_may_wakeup(priv->device) && priv->plat->pmt) {
7260 phylink_resume(priv->phylink);
7261 } else {
7262 phylink_resume(priv->phylink);
7263 if (device_may_wakeup(priv->device))
7264 phylink_speed_up(priv->phylink);
7265 }
7266 rtnl_unlock();
7267
7268 rtnl_lock();
7269 mutex_lock(&priv->lock);
7270
7271 stmmac_reset_queues_param(priv);
7272
7273 stmmac_free_tx_skbufs(priv);
7274 stmmac_clear_descriptors(priv);
7275
7276 stmmac_hw_setup(ndev, false);
7277 stmmac_init_coalesce(priv);
7278 stmmac_set_rx_mode(ndev);
7279
7280 stmmac_restore_hw_vlan_rx_fltr(priv, ndev, priv->hw);
7281
7282 stmmac_enable_all_queues(priv);
7283
7284 mutex_unlock(&priv->lock);
7285 rtnl_unlock();
7286
7287 netif_device_attach(ndev);
7288
7289 return 0;
7290}
7291EXPORT_SYMBOL_GPL(stmmac_resume);
7292
7293#ifndef MODULE
7294static int __init stmmac_cmdline_opt(char *str)
7295{
7296 char *opt;
7297
7298 if (!str || !*str)
7299 return -EINVAL;
7300 while ((opt = strsep(&str, ",")) != NULL) {
7301 if (!strncmp(opt, "debug:", 6)) {
7302 if (kstrtoint(opt + 6, 0, &debug))
7303 goto err;
7304 } else if (!strncmp(opt, "phyaddr:", 8)) {
7305 if (kstrtoint(opt + 8, 0, &phyaddr))
7306 goto err;
7307 } else if (!strncmp(opt, "buf_sz:", 7)) {
7308 if (kstrtoint(opt + 7, 0, &buf_sz))
7309 goto err;
7310 } else if (!strncmp(opt, "tc:", 3)) {
7311 if (kstrtoint(opt + 3, 0, &tc))
7312 goto err;
7313 } else if (!strncmp(opt, "watchdog:", 9)) {
7314 if (kstrtoint(opt + 9, 0, &watchdog))
7315 goto err;
7316 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
7317 if (kstrtoint(opt + 10, 0, &flow_ctrl))
7318 goto err;
7319 } else if (!strncmp(opt, "pause:", 6)) {
7320 if (kstrtoint(opt + 6, 0, &pause))
7321 goto err;
7322 } else if (!strncmp(opt, "eee_timer:", 10)) {
7323 if (kstrtoint(opt + 10, 0, &eee_timer))
7324 goto err;
7325 } else if (!strncmp(opt, "chain_mode:", 11)) {
7326 if (kstrtoint(opt + 11, 0, &chain_mode))
7327 goto err;
7328 }
7329 }
7330 return 0;
7331
7332err:
7333 pr_err("%s: ERROR broken module parameter conversion", __func__);
7334 return -EINVAL;
7335}
7336
7337__setup("stmmaceth=", stmmac_cmdline_opt);
7338#endif
7339
7340static int __init stmmac_init(void)
7341{
7342#ifdef CONFIG_DEBUG_FS
7343
7344 if (!stmmac_fs_dir)
7345 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
7346 register_netdevice_notifier(&stmmac_notifier);
7347#endif
7348
7349 return 0;
7350}
7351
7352static void __exit stmmac_exit(void)
7353{
7354#ifdef CONFIG_DEBUG_FS
7355 unregister_netdevice_notifier(&stmmac_notifier);
7356 debugfs_remove_recursive(stmmac_fs_dir);
7357#endif
7358}
7359
7360module_init(stmmac_init)
7361module_exit(stmmac_exit)
7362
7363MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
7364MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
7365MODULE_LICENSE("GPL");
7366