1
2
3
4#include "enetc.h"
5#include <linux/tcp.h>
6#include <linux/udp.h>
7#include <linux/of_mdio.h>
8#include <linux/vmalloc.h>
9
10
11#define ENETC_TXBDS_NEEDED(val) ((val) + 2)
12
13#define ENETC_MAX_SKB_FRAGS 13
14#define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
15
16static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
17 int active_offloads);
18
19netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
20{
21 struct enetc_ndev_priv *priv = netdev_priv(ndev);
22 struct enetc_bdr *tx_ring;
23 int count;
24
25 tx_ring = priv->tx_ring[skb->queue_mapping];
26
27 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
28 if (unlikely(skb_linearize(skb)))
29 goto drop_packet_err;
30
31 count = skb_shinfo(skb)->nr_frags + 1;
32 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
33 netif_stop_subqueue(ndev, tx_ring->index);
34 return NETDEV_TX_BUSY;
35 }
36
37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
38 if (unlikely(!count))
39 goto drop_packet_err;
40
41 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
42 netif_stop_subqueue(ndev, tx_ring->index);
43
44 return NETDEV_TX_OK;
45
46drop_packet_err:
47 dev_kfree_skb_any(skb);
48 return NETDEV_TX_OK;
49}
50
51static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd)
52{
53 int l3_start, l3_hsize;
54 u16 l3_flags, l4_flags;
55
56 if (skb->ip_summed != CHECKSUM_PARTIAL)
57 return false;
58
59 switch (skb->csum_offset) {
60 case offsetof(struct tcphdr, check):
61 l4_flags = ENETC_TXBD_L4_TCP;
62 break;
63 case offsetof(struct udphdr, check):
64 l4_flags = ENETC_TXBD_L4_UDP;
65 break;
66 default:
67 skb_checksum_help(skb);
68 return false;
69 }
70
71 l3_start = skb_network_offset(skb);
72 l3_hsize = skb_network_header_len(skb);
73
74 l3_flags = 0;
75 if (skb->protocol == htons(ETH_P_IPV6))
76 l3_flags = ENETC_TXBD_L3_IPV6;
77
78
79 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags);
80 txbd->l4_csoff = l4_flags;
81
82 return true;
83}
84
85static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
86 struct enetc_tx_swbd *tx_swbd)
87{
88 if (tx_swbd->is_dma_page)
89 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
90 tx_swbd->len, DMA_TO_DEVICE);
91 else
92 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
93 tx_swbd->len, DMA_TO_DEVICE);
94 tx_swbd->dma = 0;
95}
96
97static void enetc_free_tx_skb(struct enetc_bdr *tx_ring,
98 struct enetc_tx_swbd *tx_swbd)
99{
100 if (tx_swbd->dma)
101 enetc_unmap_tx_buff(tx_ring, tx_swbd);
102
103 if (tx_swbd->skb) {
104 dev_kfree_skb_any(tx_swbd->skb);
105 tx_swbd->skb = NULL;
106 }
107}
108
109static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
110 int active_offloads)
111{
112 struct enetc_tx_swbd *tx_swbd;
113 skb_frag_t *frag;
114 int len = skb_headlen(skb);
115 union enetc_tx_bd temp_bd;
116 union enetc_tx_bd *txbd;
117 bool do_vlan, do_tstamp;
118 int i, count = 0;
119 unsigned int f;
120 dma_addr_t dma;
121 u8 flags = 0;
122
123 i = tx_ring->next_to_use;
124 txbd = ENETC_TXBD(*tx_ring, i);
125 prefetchw(txbd);
126
127 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
128 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
129 goto dma_err;
130
131 temp_bd.addr = cpu_to_le64(dma);
132 temp_bd.buf_len = cpu_to_le16(len);
133 temp_bd.lstatus = 0;
134
135 tx_swbd = &tx_ring->tx_swbd[i];
136 tx_swbd->dma = dma;
137 tx_swbd->len = len;
138 tx_swbd->is_dma_page = 0;
139 count++;
140
141 do_vlan = skb_vlan_tag_present(skb);
142 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) &&
143 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP);
144 tx_swbd->do_tstamp = do_tstamp;
145 tx_swbd->check_wb = tx_swbd->do_tstamp;
146
147 if (do_vlan || do_tstamp)
148 flags |= ENETC_TXBD_FLAGS_EX;
149
150 if (enetc_tx_csum(skb, &temp_bd))
151 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS;
152 else if (tx_ring->tsd_enable)
153 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
154
155
156 temp_bd.frm_len = cpu_to_le16(skb->len);
157 temp_bd.flags = flags;
158
159 if (flags & ENETC_TXBD_FLAGS_TSE) {
160 u32 temp;
161
162 temp = (skb->skb_mstamp_ns >> 5 & ENETC_TXBD_TXSTART_MASK)
163 | (flags << ENETC_TXBD_FLAGS_OFFSET);
164 temp_bd.txstart = cpu_to_le32(temp);
165 }
166
167 if (flags & ENETC_TXBD_FLAGS_EX) {
168 u8 e_flags = 0;
169 *txbd = temp_bd;
170 enetc_clear_tx_bd(&temp_bd);
171
172
173 flags = 0;
174 tx_swbd++;
175 txbd++;
176 i++;
177 if (unlikely(i == tx_ring->bd_count)) {
178 i = 0;
179 tx_swbd = tx_ring->tx_swbd;
180 txbd = ENETC_TXBD(*tx_ring, 0);
181 }
182 prefetchw(txbd);
183
184 if (do_vlan) {
185 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
186 temp_bd.ext.tpid = 0;
187 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
188 }
189
190 if (do_tstamp) {
191 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
192 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
193 }
194
195 temp_bd.ext.e_flags = e_flags;
196 count++;
197 }
198
199 frag = &skb_shinfo(skb)->frags[0];
200 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
201 len = skb_frag_size(frag);
202 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
203 DMA_TO_DEVICE);
204 if (dma_mapping_error(tx_ring->dev, dma))
205 goto dma_err;
206
207 *txbd = temp_bd;
208 enetc_clear_tx_bd(&temp_bd);
209
210 flags = 0;
211 tx_swbd++;
212 txbd++;
213 i++;
214 if (unlikely(i == tx_ring->bd_count)) {
215 i = 0;
216 tx_swbd = tx_ring->tx_swbd;
217 txbd = ENETC_TXBD(*tx_ring, 0);
218 }
219 prefetchw(txbd);
220
221 temp_bd.addr = cpu_to_le64(dma);
222 temp_bd.buf_len = cpu_to_le16(len);
223
224 tx_swbd->dma = dma;
225 tx_swbd->len = len;
226 tx_swbd->is_dma_page = 1;
227 count++;
228 }
229
230
231 flags |= ENETC_TXBD_FLAGS_F;
232 temp_bd.flags = flags;
233 *txbd = temp_bd;
234
235 tx_ring->tx_swbd[i].skb = skb;
236
237 enetc_bdr_idx_inc(tx_ring, &i);
238 tx_ring->next_to_use = i;
239
240 skb_tx_timestamp(skb);
241
242
243 enetc_wr_reg(tx_ring->tpir, i);
244
245 return count;
246
247dma_err:
248 dev_err(tx_ring->dev, "DMA map error");
249
250 do {
251 tx_swbd = &tx_ring->tx_swbd[i];
252 enetc_free_tx_skb(tx_ring, tx_swbd);
253 if (i == 0)
254 i = tx_ring->bd_count;
255 i--;
256 } while (count--);
257
258 return 0;
259}
260
261static irqreturn_t enetc_msix(int irq, void *data)
262{
263 struct enetc_int_vector *v = data;
264 int i;
265
266
267 enetc_wr_reg(v->rbier, 0);
268
269 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings)
270 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0);
271
272 napi_schedule_irqoff(&v->napi);
273
274 return IRQ_HANDLED;
275}
276
277static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
278static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
279 struct napi_struct *napi, int work_limit);
280
281static int enetc_poll(struct napi_struct *napi, int budget)
282{
283 struct enetc_int_vector
284 *v = container_of(napi, struct enetc_int_vector, napi);
285 bool complete = true;
286 int work_done;
287 int i;
288
289 for (i = 0; i < v->count_tx_rings; i++)
290 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
291 complete = false;
292
293 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
294 if (work_done == budget)
295 complete = false;
296
297 if (!complete)
298 return budget;
299
300 napi_complete_done(napi, work_done);
301
302
303 enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE);
304
305 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings)
306 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i),
307 ENETC_TBIER_TXTIE);
308
309 return work_done;
310}
311
312static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
313{
314 int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
315
316 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
317}
318
319static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
320 u64 *tstamp)
321{
322 u32 lo, hi, tstamp_lo;
323
324 lo = enetc_rd(hw, ENETC_SICTR0);
325 hi = enetc_rd(hw, ENETC_SICTR1);
326 tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
327 if (lo <= tstamp_lo)
328 hi -= 1;
329 *tstamp = (u64)hi << 32 | tstamp_lo;
330}
331
332static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
333{
334 struct skb_shared_hwtstamps shhwtstamps;
335
336 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
337 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
338 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
339 skb_tstamp_tx(skb, &shhwtstamps);
340 }
341}
342
343static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
344{
345 struct net_device *ndev = tx_ring->ndev;
346 int tx_frm_cnt = 0, tx_byte_cnt = 0;
347 struct enetc_tx_swbd *tx_swbd;
348 int i, bds_to_clean;
349 bool do_tstamp;
350 u64 tstamp = 0;
351
352 i = tx_ring->next_to_clean;
353 tx_swbd = &tx_ring->tx_swbd[i];
354 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
355
356 do_tstamp = false;
357
358 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
359 bool is_eof = !!tx_swbd->skb;
360
361 if (unlikely(tx_swbd->check_wb)) {
362 struct enetc_ndev_priv *priv = netdev_priv(ndev);
363 union enetc_tx_bd *txbd;
364
365 txbd = ENETC_TXBD(*tx_ring, i);
366
367 if (txbd->flags & ENETC_TXBD_FLAGS_W &&
368 tx_swbd->do_tstamp) {
369 enetc_get_tx_tstamp(&priv->si->hw, txbd,
370 &tstamp);
371 do_tstamp = true;
372 }
373 }
374
375 if (likely(tx_swbd->dma))
376 enetc_unmap_tx_buff(tx_ring, tx_swbd);
377
378 if (is_eof) {
379 if (unlikely(do_tstamp)) {
380 enetc_tstamp_tx(tx_swbd->skb, tstamp);
381 do_tstamp = false;
382 }
383 napi_consume_skb(tx_swbd->skb, napi_budget);
384 tx_swbd->skb = NULL;
385 }
386
387 tx_byte_cnt += tx_swbd->len;
388
389 bds_to_clean--;
390 tx_swbd++;
391 i++;
392 if (unlikely(i == tx_ring->bd_count)) {
393 i = 0;
394 tx_swbd = tx_ring->tx_swbd;
395 }
396
397
398 if (is_eof) {
399 tx_frm_cnt++;
400
401 enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) |
402 BIT(16 + tx_ring->index));
403 }
404
405 if (unlikely(!bds_to_clean))
406 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
407 }
408
409 tx_ring->next_to_clean = i;
410 tx_ring->stats.packets += tx_frm_cnt;
411 tx_ring->stats.bytes += tx_byte_cnt;
412
413 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
414 __netif_subqueue_stopped(ndev, tx_ring->index) &&
415 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
416 netif_wake_subqueue(ndev, tx_ring->index);
417 }
418
419 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
420}
421
422static bool enetc_new_page(struct enetc_bdr *rx_ring,
423 struct enetc_rx_swbd *rx_swbd)
424{
425 struct page *page;
426 dma_addr_t addr;
427
428 page = dev_alloc_page();
429 if (unlikely(!page))
430 return false;
431
432 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
433 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
434 __free_page(page);
435
436 return false;
437 }
438
439 rx_swbd->dma = addr;
440 rx_swbd->page = page;
441 rx_swbd->page_offset = ENETC_RXB_PAD;
442
443 return true;
444}
445
446static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
447{
448 struct enetc_rx_swbd *rx_swbd;
449 union enetc_rx_bd *rxbd;
450 int i, j;
451
452 i = rx_ring->next_to_use;
453 rx_swbd = &rx_ring->rx_swbd[i];
454 rxbd = enetc_rxbd(rx_ring, i);
455
456 for (j = 0; j < buff_cnt; j++) {
457
458 if (unlikely(!rx_swbd->page)) {
459 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
460 rx_ring->stats.rx_alloc_errs++;
461 break;
462 }
463 }
464
465
466 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
467 rx_swbd->page_offset);
468
469 rxbd->r.lstatus = 0;
470
471 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
472 rx_swbd++;
473 i++;
474 if (unlikely(i == rx_ring->bd_count)) {
475 i = 0;
476 rx_swbd = rx_ring->rx_swbd;
477 }
478 }
479
480 if (likely(j)) {
481 rx_ring->next_to_alloc = i;
482 rx_ring->next_to_use = i;
483
484 enetc_wr_reg(rx_ring->rcir, i);
485 }
486
487 return j;
488}
489
490#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
491static void enetc_get_rx_tstamp(struct net_device *ndev,
492 union enetc_rx_bd *rxbd,
493 struct sk_buff *skb)
494{
495 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
496 struct enetc_ndev_priv *priv = netdev_priv(ndev);
497 struct enetc_hw *hw = &priv->si->hw;
498 u32 lo, hi, tstamp_lo;
499 u64 tstamp;
500
501 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
502 lo = enetc_rd(hw, ENETC_SICTR0);
503 hi = enetc_rd(hw, ENETC_SICTR1);
504 rxbd = enetc_rxbd_ext(rxbd);
505 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
506 if (lo <= tstamp_lo)
507 hi -= 1;
508
509 tstamp = (u64)hi << 32 | tstamp_lo;
510 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
511 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
512 }
513}
514#endif
515
516static void enetc_get_offloads(struct enetc_bdr *rx_ring,
517 union enetc_rx_bd *rxbd, struct sk_buff *skb)
518{
519#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
520 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
521#endif
522
523 if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
524 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
525
526 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
527 skb->ip_summed = CHECKSUM_COMPLETE;
528 }
529
530
531
532
533 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN)
534 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
535 le16_to_cpu(rxbd->r.vlan_opt));
536#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
537 if (priv->active_offloads & ENETC_F_RX_TSTAMP)
538 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
539#endif
540}
541
542static void enetc_process_skb(struct enetc_bdr *rx_ring,
543 struct sk_buff *skb)
544{
545 skb_record_rx_queue(skb, rx_ring->index);
546 skb->protocol = eth_type_trans(skb, rx_ring->ndev);
547}
548
549static bool enetc_page_reusable(struct page *page)
550{
551 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
552}
553
554static void enetc_reuse_page(struct enetc_bdr *rx_ring,
555 struct enetc_rx_swbd *old)
556{
557 struct enetc_rx_swbd *new;
558
559 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
560
561
562 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
563
564
565 *new = *old;
566}
567
568static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
569 int i, u16 size)
570{
571 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
572
573 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
574 rx_swbd->page_offset,
575 size, DMA_FROM_DEVICE);
576 return rx_swbd;
577}
578
579static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
580 struct enetc_rx_swbd *rx_swbd)
581{
582 if (likely(enetc_page_reusable(rx_swbd->page))) {
583 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
584 page_ref_inc(rx_swbd->page);
585
586 enetc_reuse_page(rx_ring, rx_swbd);
587
588
589 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
590 rx_swbd->page_offset,
591 ENETC_RXB_DMA_SIZE,
592 DMA_FROM_DEVICE);
593 } else {
594 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
595 PAGE_SIZE, DMA_FROM_DEVICE);
596 }
597
598 rx_swbd->page = NULL;
599}
600
601static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
602 int i, u16 size)
603{
604 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
605 struct sk_buff *skb;
606 void *ba;
607
608 ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
609 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE);
610 if (unlikely(!skb)) {
611 rx_ring->stats.rx_alloc_errs++;
612 return NULL;
613 }
614
615 skb_reserve(skb, ENETC_RXB_PAD);
616 __skb_put(skb, size);
617
618 enetc_put_rx_buff(rx_ring, rx_swbd);
619
620 return skb;
621}
622
623static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
624 u16 size, struct sk_buff *skb)
625{
626 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
627
628 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
629 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
630
631 enetc_put_rx_buff(rx_ring, rx_swbd);
632}
633
634#define ENETC_RXBD_BUNDLE 16
635
636static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
637 struct napi_struct *napi, int work_limit)
638{
639 int rx_frm_cnt = 0, rx_byte_cnt = 0;
640 int cleaned_cnt, i;
641
642 cleaned_cnt = enetc_bd_unused(rx_ring);
643
644 i = rx_ring->next_to_clean;
645
646 while (likely(rx_frm_cnt < work_limit)) {
647 union enetc_rx_bd *rxbd;
648 struct sk_buff *skb;
649 u32 bd_status;
650 u16 size;
651
652 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
653 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
654
655 cleaned_cnt -= count;
656 }
657
658 rxbd = enetc_rxbd(rx_ring, i);
659 bd_status = le32_to_cpu(rxbd->r.lstatus);
660 if (!bd_status)
661 break;
662
663 enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index));
664 dma_rmb();
665 size = le16_to_cpu(rxbd->r.buf_len);
666 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
667 if (!skb)
668 break;
669
670 enetc_get_offloads(rx_ring, rxbd, skb);
671
672 cleaned_cnt++;
673
674 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
675 if (unlikely(++i == rx_ring->bd_count))
676 i = 0;
677
678 if (unlikely(bd_status &
679 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
680 dev_kfree_skb(skb);
681 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
682 dma_rmb();
683 bd_status = le32_to_cpu(rxbd->r.lstatus);
684
685 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
686 if (unlikely(++i == rx_ring->bd_count))
687 i = 0;
688 }
689
690 rx_ring->ndev->stats.rx_dropped++;
691 rx_ring->ndev->stats.rx_errors++;
692
693 break;
694 }
695
696
697 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
698 bd_status = le32_to_cpu(rxbd->r.lstatus);
699 size = ENETC_RXB_DMA_SIZE;
700
701 if (bd_status & ENETC_RXBD_LSTATUS_F) {
702 dma_rmb();
703 size = le16_to_cpu(rxbd->r.buf_len);
704 }
705
706 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);
707
708 cleaned_cnt++;
709
710 rxbd = enetc_rxbd_next(rx_ring, rxbd, i);
711 if (unlikely(++i == rx_ring->bd_count))
712 i = 0;
713 }
714
715 rx_byte_cnt += skb->len;
716
717 enetc_process_skb(rx_ring, skb);
718
719 napi_gro_receive(napi, skb);
720
721 rx_frm_cnt++;
722 }
723
724 rx_ring->next_to_clean = i;
725
726 rx_ring->stats.packets += rx_frm_cnt;
727 rx_ring->stats.bytes += rx_byte_cnt;
728
729 return rx_frm_cnt;
730}
731
732
733#define ENETC_MAX_RFS_SIZE 64
734void enetc_get_si_caps(struct enetc_si *si)
735{
736 struct enetc_hw *hw = &si->hw;
737 u32 val;
738
739
740 val = enetc_rd(hw, ENETC_SICAPR0);
741 si->num_rx_rings = (val >> 16) & 0xff;
742 si->num_tx_rings = val & 0xff;
743
744 val = enetc_rd(hw, ENETC_SIRFSCAPR);
745 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
746 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
747
748 si->num_rss = 0;
749 val = enetc_rd(hw, ENETC_SIPCAPR0);
750 if (val & ENETC_SIPCAPR0_RSS) {
751 u32 rss;
752
753 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
754 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
755 }
756
757 if (val & ENETC_SIPCAPR0_QBV)
758 si->hw_features |= ENETC_SI_F_QBV;
759}
760
761static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
762{
763 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
764 &r->bd_dma_base, GFP_KERNEL);
765 if (!r->bd_base)
766 return -ENOMEM;
767
768
769 if (!IS_ALIGNED(r->bd_dma_base, 128)) {
770 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
771 r->bd_dma_base);
772 return -EINVAL;
773 }
774
775 return 0;
776}
777
778static int enetc_alloc_txbdr(struct enetc_bdr *txr)
779{
780 int err;
781
782 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
783 if (!txr->tx_swbd)
784 return -ENOMEM;
785
786 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
787 if (err) {
788 vfree(txr->tx_swbd);
789 return err;
790 }
791
792 txr->next_to_clean = 0;
793 txr->next_to_use = 0;
794
795 return 0;
796}
797
798static void enetc_free_txbdr(struct enetc_bdr *txr)
799{
800 int size, i;
801
802 for (i = 0; i < txr->bd_count; i++)
803 enetc_free_tx_skb(txr, &txr->tx_swbd[i]);
804
805 size = txr->bd_count * sizeof(union enetc_tx_bd);
806
807 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
808 txr->bd_base = NULL;
809
810 vfree(txr->tx_swbd);
811 txr->tx_swbd = NULL;
812}
813
814static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
815{
816 int i, err;
817
818 for (i = 0; i < priv->num_tx_rings; i++) {
819 err = enetc_alloc_txbdr(priv->tx_ring[i]);
820
821 if (err)
822 goto fail;
823 }
824
825 return 0;
826
827fail:
828 while (i-- > 0)
829 enetc_free_txbdr(priv->tx_ring[i]);
830
831 return err;
832}
833
834static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
835{
836 int i;
837
838 for (i = 0; i < priv->num_tx_rings; i++)
839 enetc_free_txbdr(priv->tx_ring[i]);
840}
841
842static int enetc_alloc_rxbdr(struct enetc_bdr *rxr, bool extended)
843{
844 size_t size = sizeof(union enetc_rx_bd);
845 int err;
846
847 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
848 if (!rxr->rx_swbd)
849 return -ENOMEM;
850
851 if (extended)
852 size *= 2;
853
854 err = enetc_dma_alloc_bdr(rxr, size);
855 if (err) {
856 vfree(rxr->rx_swbd);
857 return err;
858 }
859
860 rxr->next_to_clean = 0;
861 rxr->next_to_use = 0;
862 rxr->next_to_alloc = 0;
863 rxr->ext_en = extended;
864
865 return 0;
866}
867
868static void enetc_free_rxbdr(struct enetc_bdr *rxr)
869{
870 int size;
871
872 size = rxr->bd_count * sizeof(union enetc_rx_bd);
873
874 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
875 rxr->bd_base = NULL;
876
877 vfree(rxr->rx_swbd);
878 rxr->rx_swbd = NULL;
879}
880
881static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
882{
883 bool extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
884 int i, err;
885
886 for (i = 0; i < priv->num_rx_rings; i++) {
887 err = enetc_alloc_rxbdr(priv->rx_ring[i], extended);
888
889 if (err)
890 goto fail;
891 }
892
893 return 0;
894
895fail:
896 while (i-- > 0)
897 enetc_free_rxbdr(priv->rx_ring[i]);
898
899 return err;
900}
901
902static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
903{
904 int i;
905
906 for (i = 0; i < priv->num_rx_rings; i++)
907 enetc_free_rxbdr(priv->rx_ring[i]);
908}
909
910static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
911{
912 int i;
913
914 if (!tx_ring->tx_swbd)
915 return;
916
917 for (i = 0; i < tx_ring->bd_count; i++) {
918 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
919
920 enetc_free_tx_skb(tx_ring, tx_swbd);
921 }
922
923 tx_ring->next_to_clean = 0;
924 tx_ring->next_to_use = 0;
925}
926
927static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
928{
929 int i;
930
931 if (!rx_ring->rx_swbd)
932 return;
933
934 for (i = 0; i < rx_ring->bd_count; i++) {
935 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
936
937 if (!rx_swbd->page)
938 continue;
939
940 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
941 PAGE_SIZE, DMA_FROM_DEVICE);
942 __free_page(rx_swbd->page);
943 rx_swbd->page = NULL;
944 }
945
946 rx_ring->next_to_clean = 0;
947 rx_ring->next_to_use = 0;
948 rx_ring->next_to_alloc = 0;
949}
950
951static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
952{
953 int i;
954
955 for (i = 0; i < priv->num_rx_rings; i++)
956 enetc_free_rx_ring(priv->rx_ring[i]);
957
958 for (i = 0; i < priv->num_tx_rings; i++)
959 enetc_free_tx_ring(priv->tx_ring[i]);
960}
961
962static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
963{
964 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
965
966 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base,
967 GFP_KERNEL);
968 if (!cbdr->bd_base)
969 return -ENOMEM;
970
971
972 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) {
973 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
974 return -EINVAL;
975 }
976
977 cbdr->next_to_clean = 0;
978 cbdr->next_to_use = 0;
979
980 return 0;
981}
982
983static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
984{
985 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
986
987 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
988 cbdr->bd_base = NULL;
989}
990
991static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr)
992{
993
994 enetc_wr(hw, ENETC_SICAR2,
995 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
996
997 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base));
998 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base));
999 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count));
1000
1001 enetc_wr(hw, ENETC_SICBDRPIR, 0);
1002 enetc_wr(hw, ENETC_SICBDRCIR, 0);
1003
1004
1005 enetc_wr(hw, ENETC_SICBDRMR, BIT(31));
1006
1007 cbdr->pir = hw->reg + ENETC_SICBDRPIR;
1008 cbdr->cir = hw->reg + ENETC_SICBDRCIR;
1009}
1010
1011static void enetc_clear_cbdr(struct enetc_hw *hw)
1012{
1013 enetc_wr(hw, ENETC_SICBDRMR, 0);
1014}
1015
1016static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1017{
1018 int *rss_table;
1019 int i;
1020
1021 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1022 if (!rss_table)
1023 return -ENOMEM;
1024
1025
1026 for (i = 0; i < si->num_rss; i++)
1027 rss_table[i] = i % num_groups;
1028
1029 enetc_set_rss_table(si, rss_table, si->num_rss);
1030
1031 kfree(rss_table);
1032
1033 return 0;
1034}
1035
1036static int enetc_configure_si(struct enetc_ndev_priv *priv)
1037{
1038 struct enetc_si *si = priv->si;
1039 struct enetc_hw *hw = &si->hw;
1040 int err;
1041
1042 enetc_setup_cbdr(hw, &si->cbd_ring);
1043
1044 enetc_wr(hw, ENETC_SICAR0,
1045 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1046 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1047
1048 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1049
1050 if (si->num_rss) {
1051 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1052 if (err)
1053 return err;
1054 }
1055
1056 return 0;
1057}
1058
1059void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1060{
1061 struct enetc_si *si = priv->si;
1062 int cpus = num_online_cpus();
1063
1064 priv->tx_bd_count = ENETC_BDR_DEFAULT_SIZE;
1065 priv->rx_bd_count = ENETC_BDR_DEFAULT_SIZE;
1066
1067
1068
1069
1070
1071 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1072 priv->num_tx_rings = si->num_tx_rings;
1073 priv->bdr_int_num = cpus;
1074
1075
1076 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
1077}
1078
1079int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1080{
1081 struct enetc_si *si = priv->si;
1082 int err;
1083
1084 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring);
1085 if (err)
1086 return err;
1087
1088 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1089 GFP_KERNEL);
1090 if (!priv->cls_rules) {
1091 err = -ENOMEM;
1092 goto err_alloc_cls;
1093 }
1094
1095 err = enetc_configure_si(priv);
1096 if (err)
1097 goto err_config_si;
1098
1099 return 0;
1100
1101err_config_si:
1102 kfree(priv->cls_rules);
1103err_alloc_cls:
1104 enetc_clear_cbdr(&si->hw);
1105 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1106
1107 return err;
1108}
1109
1110void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1111{
1112 struct enetc_si *si = priv->si;
1113
1114 enetc_clear_cbdr(&si->hw);
1115 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1116
1117 kfree(priv->cls_rules);
1118}
1119
1120static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1121{
1122 int idx = tx_ring->index;
1123 u32 tbmr;
1124
1125 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1126 lower_32_bits(tx_ring->bd_dma_base));
1127
1128 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1129 upper_32_bits(tx_ring->bd_dma_base));
1130
1131 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64));
1132 enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1133 ENETC_RTBLENR_LEN(tx_ring->bd_count));
1134
1135
1136 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1137 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1138
1139
1140 enetc_txbdr_wr(hw, idx, ENETC_TBICIR0, ENETC_TBICIR0_ICEN | 0x1);
1141
1142 tbmr = ENETC_TBMR_EN;
1143 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1144 tbmr |= ENETC_TBMR_VIH;
1145
1146
1147 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1148
1149 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1150 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1151 tx_ring->idr = hw->reg + ENETC_SITXIDR;
1152}
1153
1154static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1155{
1156 int idx = rx_ring->index;
1157 u32 rbmr;
1158
1159 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1160 lower_32_bits(rx_ring->bd_dma_base));
1161
1162 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1163 upper_32_bits(rx_ring->bd_dma_base));
1164
1165 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64));
1166 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1167 ENETC_RTBLENR_LEN(rx_ring->bd_count));
1168
1169 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1170
1171 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1172
1173
1174 enetc_rxbdr_wr(hw, idx, ENETC_RBICIR0, ENETC_RBICIR0_ICEN | 0x1);
1175
1176 rbmr = ENETC_RBMR_EN;
1177
1178 if (rx_ring->ext_en)
1179 rbmr |= ENETC_RBMR_BDS;
1180
1181 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1182 rbmr |= ENETC_RBMR_VTE;
1183
1184 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1185 rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1186
1187 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1188
1189
1190 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1191}
1192
1193static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1194{
1195 int i;
1196
1197 for (i = 0; i < priv->num_tx_rings; i++)
1198 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]);
1199
1200 for (i = 0; i < priv->num_rx_rings; i++)
1201 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1202}
1203
1204static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1205{
1206 int idx = rx_ring->index;
1207
1208
1209 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1210}
1211
1212static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1213{
1214 int delay = 8, timeout = 100;
1215 int idx = tx_ring->index;
1216
1217
1218 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1219
1220
1221 while (delay < timeout &&
1222 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1223 msleep(delay);
1224 delay *= 2;
1225 }
1226
1227 if (delay >= timeout)
1228 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1229 idx);
1230}
1231
1232static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1233{
1234 int i;
1235
1236 for (i = 0; i < priv->num_tx_rings; i++)
1237 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]);
1238
1239 for (i = 0; i < priv->num_rx_rings; i++)
1240 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1241
1242 udelay(1);
1243}
1244
1245static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1246{
1247 struct pci_dev *pdev = priv->si->pdev;
1248 cpumask_t cpu_mask;
1249 int i, j, err;
1250
1251 for (i = 0; i < priv->bdr_int_num; i++) {
1252 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1253 struct enetc_int_vector *v = priv->int_vector[i];
1254 int entry = ENETC_BDR_INT_BASE_IDX + i;
1255 struct enetc_hw *hw = &priv->si->hw;
1256
1257 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1258 priv->ndev->name, i);
1259 err = request_irq(irq, enetc_msix, 0, v->name, v);
1260 if (err) {
1261 dev_err(priv->dev, "request_irq() failed!\n");
1262 goto irq_err;
1263 }
1264
1265 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1266 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1267
1268 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1269
1270 for (j = 0; j < v->count_tx_rings; j++) {
1271 int idx = v->tx_ring[j].index;
1272
1273 enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1274 }
1275 cpumask_clear(&cpu_mask);
1276 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
1277 irq_set_affinity_hint(irq, &cpu_mask);
1278 }
1279
1280 return 0;
1281
1282irq_err:
1283 while (i--) {
1284 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1285
1286 irq_set_affinity_hint(irq, NULL);
1287 free_irq(irq, priv->int_vector[i]);
1288 }
1289
1290 return err;
1291}
1292
1293static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1294{
1295 struct pci_dev *pdev = priv->si->pdev;
1296 int i;
1297
1298 for (i = 0; i < priv->bdr_int_num; i++) {
1299 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1300
1301 irq_set_affinity_hint(irq, NULL);
1302 free_irq(irq, priv->int_vector[i]);
1303 }
1304}
1305
1306static void enetc_enable_interrupts(struct enetc_ndev_priv *priv)
1307{
1308 int i;
1309
1310
1311 for (i = 0; i < priv->num_rx_rings; i++) {
1312 enetc_rxbdr_wr(&priv->si->hw, i,
1313 ENETC_RBIER, ENETC_RBIER_RXTIE);
1314 }
1315
1316 for (i = 0; i < priv->num_tx_rings; i++) {
1317 enetc_txbdr_wr(&priv->si->hw, i,
1318 ENETC_TBIER, ENETC_TBIER_TXTIE);
1319 }
1320}
1321
1322static void enetc_disable_interrupts(struct enetc_ndev_priv *priv)
1323{
1324 int i;
1325
1326 for (i = 0; i < priv->num_tx_rings; i++)
1327 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0);
1328
1329 for (i = 0; i < priv->num_rx_rings; i++)
1330 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0);
1331}
1332
1333static void adjust_link(struct net_device *ndev)
1334{
1335 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1336 struct phy_device *phydev = ndev->phydev;
1337
1338 if (priv->active_offloads & ENETC_F_QBV)
1339 enetc_sched_speed_set(ndev);
1340
1341 phy_print_status(phydev);
1342}
1343
1344static int enetc_phy_connect(struct net_device *ndev)
1345{
1346 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1347 struct phy_device *phydev;
1348 struct ethtool_eee edata;
1349
1350 if (!priv->phy_node)
1351 return 0;
1352
1353 phydev = of_phy_connect(ndev, priv->phy_node, &adjust_link,
1354 0, priv->if_mode);
1355 if (!phydev) {
1356 dev_err(&ndev->dev, "could not attach to PHY\n");
1357 return -ENODEV;
1358 }
1359
1360 phy_attached_info(phydev);
1361
1362
1363 memset(&edata, 0, sizeof(struct ethtool_eee));
1364 phy_ethtool_set_eee(phydev, &edata);
1365
1366 return 0;
1367}
1368
1369int enetc_open(struct net_device *ndev)
1370{
1371 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1372 int i, err;
1373
1374 err = enetc_setup_irqs(priv);
1375 if (err)
1376 return err;
1377
1378 err = enetc_phy_connect(ndev);
1379 if (err)
1380 goto err_phy_connect;
1381
1382 err = enetc_alloc_tx_resources(priv);
1383 if (err)
1384 goto err_alloc_tx;
1385
1386 err = enetc_alloc_rx_resources(priv);
1387 if (err)
1388 goto err_alloc_rx;
1389
1390 enetc_setup_bdrs(priv);
1391
1392 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1393 if (err)
1394 goto err_set_queues;
1395
1396 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
1397 if (err)
1398 goto err_set_queues;
1399
1400 for (i = 0; i < priv->bdr_int_num; i++)
1401 napi_enable(&priv->int_vector[i]->napi);
1402
1403 enetc_enable_interrupts(priv);
1404
1405 if (ndev->phydev)
1406 phy_start(ndev->phydev);
1407 else
1408 netif_carrier_on(ndev);
1409
1410 netif_tx_start_all_queues(ndev);
1411
1412 return 0;
1413
1414err_set_queues:
1415 enetc_free_rx_resources(priv);
1416err_alloc_rx:
1417 enetc_free_tx_resources(priv);
1418err_alloc_tx:
1419 if (ndev->phydev)
1420 phy_disconnect(ndev->phydev);
1421err_phy_connect:
1422 enetc_free_irqs(priv);
1423
1424 return err;
1425}
1426
1427int enetc_close(struct net_device *ndev)
1428{
1429 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1430 int i;
1431
1432 netif_tx_stop_all_queues(ndev);
1433
1434 if (ndev->phydev) {
1435 phy_stop(ndev->phydev);
1436 phy_disconnect(ndev->phydev);
1437 } else {
1438 netif_carrier_off(ndev);
1439 }
1440
1441 for (i = 0; i < priv->bdr_int_num; i++) {
1442 napi_synchronize(&priv->int_vector[i]->napi);
1443 napi_disable(&priv->int_vector[i]->napi);
1444 }
1445
1446 enetc_disable_interrupts(priv);
1447 enetc_clear_bdrs(priv);
1448
1449 enetc_free_rxtx_rings(priv);
1450 enetc_free_rx_resources(priv);
1451 enetc_free_tx_resources(priv);
1452 enetc_free_irqs(priv);
1453
1454 return 0;
1455}
1456
1457static int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
1458{
1459 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1460 struct tc_mqprio_qopt *mqprio = type_data;
1461 struct enetc_bdr *tx_ring;
1462 u8 num_tc;
1463 int i;
1464
1465 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1466 num_tc = mqprio->num_tc;
1467
1468 if (!num_tc) {
1469 netdev_reset_tc(ndev);
1470 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1471
1472
1473 for (i = 0; i < priv->num_tx_rings; i++) {
1474 tx_ring = priv->tx_ring[i];
1475 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0);
1476 }
1477
1478 return 0;
1479 }
1480
1481
1482 if (num_tc > priv->num_tx_rings) {
1483 netdev_err(ndev, "Max %d traffic classes supported\n",
1484 priv->num_tx_rings);
1485 return -EINVAL;
1486 }
1487
1488
1489
1490
1491
1492 for (i = 0; i < num_tc; i++) {
1493 tx_ring = priv->tx_ring[i];
1494 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i);
1495 }
1496
1497
1498 netif_set_real_num_tx_queues(ndev, num_tc);
1499
1500 netdev_set_num_tc(ndev, num_tc);
1501
1502
1503 for (i = 0; i < num_tc; i++)
1504 netdev_set_tc_queue(ndev, i, 1, i);
1505
1506 return 0;
1507}
1508
1509int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1510 void *type_data)
1511{
1512 switch (type) {
1513 case TC_SETUP_QDISC_MQPRIO:
1514 return enetc_setup_tc_mqprio(ndev, type_data);
1515 case TC_SETUP_QDISC_TAPRIO:
1516 return enetc_setup_tc_taprio(ndev, type_data);
1517 case TC_SETUP_QDISC_CBS:
1518 return enetc_setup_tc_cbs(ndev, type_data);
1519 case TC_SETUP_QDISC_ETF:
1520 return enetc_setup_tc_txtime(ndev, type_data);
1521 default:
1522 return -EOPNOTSUPP;
1523 }
1524}
1525
1526struct net_device_stats *enetc_get_stats(struct net_device *ndev)
1527{
1528 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1529 struct net_device_stats *stats = &ndev->stats;
1530 unsigned long packets = 0, bytes = 0;
1531 int i;
1532
1533 for (i = 0; i < priv->num_rx_rings; i++) {
1534 packets += priv->rx_ring[i]->stats.packets;
1535 bytes += priv->rx_ring[i]->stats.bytes;
1536 }
1537
1538 stats->rx_packets = packets;
1539 stats->rx_bytes = bytes;
1540 bytes = 0;
1541 packets = 0;
1542
1543 for (i = 0; i < priv->num_tx_rings; i++) {
1544 packets += priv->tx_ring[i]->stats.packets;
1545 bytes += priv->tx_ring[i]->stats.bytes;
1546 }
1547
1548 stats->tx_packets = packets;
1549 stats->tx_bytes = bytes;
1550
1551 return stats;
1552}
1553
1554static int enetc_set_rss(struct net_device *ndev, int en)
1555{
1556 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1557 struct enetc_hw *hw = &priv->si->hw;
1558 u32 reg;
1559
1560 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
1561
1562 reg = enetc_rd(hw, ENETC_SIMR);
1563 reg &= ~ENETC_SIMR_RSSE;
1564 reg |= (en) ? ENETC_SIMR_RSSE : 0;
1565 enetc_wr(hw, ENETC_SIMR, reg);
1566
1567 return 0;
1568}
1569
1570int enetc_set_features(struct net_device *ndev,
1571 netdev_features_t features)
1572{
1573 netdev_features_t changed = ndev->features ^ features;
1574
1575 if (changed & NETIF_F_RXHASH)
1576 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
1577
1578 return 0;
1579}
1580
1581#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1582static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
1583{
1584 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1585 struct hwtstamp_config config;
1586 int ao;
1587
1588 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1589 return -EFAULT;
1590
1591 switch (config.tx_type) {
1592 case HWTSTAMP_TX_OFF:
1593 priv->active_offloads &= ~ENETC_F_TX_TSTAMP;
1594 break;
1595 case HWTSTAMP_TX_ON:
1596 priv->active_offloads |= ENETC_F_TX_TSTAMP;
1597 break;
1598 default:
1599 return -ERANGE;
1600 }
1601
1602 ao = priv->active_offloads;
1603 switch (config.rx_filter) {
1604 case HWTSTAMP_FILTER_NONE:
1605 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
1606 break;
1607 default:
1608 priv->active_offloads |= ENETC_F_RX_TSTAMP;
1609 config.rx_filter = HWTSTAMP_FILTER_ALL;
1610 }
1611
1612 if (netif_running(ndev) && ao != priv->active_offloads) {
1613 enetc_close(ndev);
1614 enetc_open(ndev);
1615 }
1616
1617 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1618 -EFAULT : 0;
1619}
1620
1621static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
1622{
1623 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1624 struct hwtstamp_config config;
1625
1626 config.flags = 0;
1627
1628 if (priv->active_offloads & ENETC_F_TX_TSTAMP)
1629 config.tx_type = HWTSTAMP_TX_ON;
1630 else
1631 config.tx_type = HWTSTAMP_TX_OFF;
1632
1633 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
1634 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1635
1636 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1637 -EFAULT : 0;
1638}
1639#endif
1640
1641int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
1642{
1643#ifdef CONFIG_FSL_ENETC_PTP_CLOCK
1644 if (cmd == SIOCSHWTSTAMP)
1645 return enetc_hwtstamp_set(ndev, rq);
1646 if (cmd == SIOCGHWTSTAMP)
1647 return enetc_hwtstamp_get(ndev, rq);
1648#endif
1649
1650 if (!ndev->phydev)
1651 return -EOPNOTSUPP;
1652 return phy_mii_ioctl(ndev->phydev, rq, cmd);
1653}
1654
1655int enetc_alloc_msix(struct enetc_ndev_priv *priv)
1656{
1657 struct pci_dev *pdev = priv->si->pdev;
1658 int size, v_tx_rings;
1659 int i, n, err, nvec;
1660
1661 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
1662
1663 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1664
1665 if (n < 0)
1666 return n;
1667
1668 if (n != nvec)
1669 return -EPERM;
1670
1671
1672 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
1673 size = sizeof(struct enetc_int_vector) +
1674 sizeof(struct enetc_bdr) * v_tx_rings;
1675
1676 for (i = 0; i < priv->bdr_int_num; i++) {
1677 struct enetc_int_vector *v;
1678 struct enetc_bdr *bdr;
1679 int j;
1680
1681 v = kzalloc(size, GFP_KERNEL);
1682 if (!v) {
1683 err = -ENOMEM;
1684 goto fail;
1685 }
1686
1687 priv->int_vector[i] = v;
1688
1689 netif_napi_add(priv->ndev, &v->napi, enetc_poll,
1690 NAPI_POLL_WEIGHT);
1691 v->count_tx_rings = v_tx_rings;
1692
1693 for (j = 0; j < v_tx_rings; j++) {
1694 int idx;
1695
1696
1697 if (priv->bdr_int_num == ENETC_MAX_BDR_INT)
1698 idx = 2 * j + i;
1699 else
1700 idx = j + i * v_tx_rings;
1701
1702 __set_bit(idx, &v->tx_rings_map);
1703 bdr = &v->tx_ring[j];
1704 bdr->index = idx;
1705 bdr->ndev = priv->ndev;
1706 bdr->dev = priv->dev;
1707 bdr->bd_count = priv->tx_bd_count;
1708 priv->tx_ring[idx] = bdr;
1709 }
1710
1711 bdr = &v->rx_ring;
1712 bdr->index = i;
1713 bdr->ndev = priv->ndev;
1714 bdr->dev = priv->dev;
1715 bdr->bd_count = priv->rx_bd_count;
1716 priv->rx_ring[i] = bdr;
1717 }
1718
1719 return 0;
1720
1721fail:
1722 while (i--) {
1723 netif_napi_del(&priv->int_vector[i]->napi);
1724 kfree(priv->int_vector[i]);
1725 }
1726
1727 pci_free_irq_vectors(pdev);
1728
1729 return err;
1730}
1731
1732void enetc_free_msix(struct enetc_ndev_priv *priv)
1733{
1734 int i;
1735
1736 for (i = 0; i < priv->bdr_int_num; i++) {
1737 struct enetc_int_vector *v = priv->int_vector[i];
1738
1739 netif_napi_del(&v->napi);
1740 }
1741
1742 for (i = 0; i < priv->num_rx_rings; i++)
1743 priv->rx_ring[i] = NULL;
1744
1745 for (i = 0; i < priv->num_tx_rings; i++)
1746 priv->tx_ring[i] = NULL;
1747
1748 for (i = 0; i < priv->bdr_int_num; i++) {
1749 kfree(priv->int_vector[i]);
1750 priv->int_vector[i] = NULL;
1751 }
1752
1753
1754 pci_free_irq_vectors(priv->si->pdev);
1755}
1756
1757static void enetc_kfree_si(struct enetc_si *si)
1758{
1759 char *p = (char *)si - si->pad;
1760
1761 kfree(p);
1762}
1763
1764static void enetc_detect_errata(struct enetc_si *si)
1765{
1766 if (si->pdev->revision == ENETC_REV1)
1767 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL |
1768 ENETC_ERR_UCMCSWP;
1769}
1770
1771int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
1772{
1773 struct enetc_si *si, *p;
1774 struct enetc_hw *hw;
1775 size_t alloc_size;
1776 int err, len;
1777
1778 pcie_flr(pdev);
1779 err = pci_enable_device_mem(pdev);
1780 if (err) {
1781 dev_err(&pdev->dev, "device enable failed\n");
1782 return err;
1783 }
1784
1785
1786 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1787 if (err) {
1788 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1789 if (err) {
1790 dev_err(&pdev->dev,
1791 "DMA configuration failed: 0x%x\n", err);
1792 goto err_dma;
1793 }
1794 }
1795
1796 err = pci_request_mem_regions(pdev, name);
1797 if (err) {
1798 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
1799 goto err_pci_mem_reg;
1800 }
1801
1802 pci_set_master(pdev);
1803
1804 alloc_size = sizeof(struct enetc_si);
1805 if (sizeof_priv) {
1806
1807 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
1808 alloc_size += sizeof_priv;
1809 }
1810
1811 alloc_size += ENETC_SI_ALIGN - 1;
1812
1813 p = kzalloc(alloc_size, GFP_KERNEL);
1814 if (!p) {
1815 err = -ENOMEM;
1816 goto err_alloc_si;
1817 }
1818
1819 si = PTR_ALIGN(p, ENETC_SI_ALIGN);
1820 si->pad = (char *)si - (char *)p;
1821
1822 pci_set_drvdata(pdev, si);
1823 si->pdev = pdev;
1824 hw = &si->hw;
1825
1826 len = pci_resource_len(pdev, ENETC_BAR_REGS);
1827 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
1828 if (!hw->reg) {
1829 err = -ENXIO;
1830 dev_err(&pdev->dev, "ioremap() failed\n");
1831 goto err_ioremap;
1832 }
1833 if (len > ENETC_PORT_BASE)
1834 hw->port = hw->reg + ENETC_PORT_BASE;
1835 if (len > ENETC_GLOBAL_BASE)
1836 hw->global = hw->reg + ENETC_GLOBAL_BASE;
1837
1838 enetc_detect_errata(si);
1839
1840 return 0;
1841
1842err_ioremap:
1843 enetc_kfree_si(si);
1844err_alloc_si:
1845 pci_release_mem_regions(pdev);
1846err_pci_mem_reg:
1847err_dma:
1848 pci_disable_device(pdev);
1849
1850 return err;
1851}
1852
1853void enetc_pci_remove(struct pci_dev *pdev)
1854{
1855 struct enetc_si *si = pci_get_drvdata(pdev);
1856 struct enetc_hw *hw = &si->hw;
1857
1858 iounmap(hw->reg);
1859 enetc_kfree_si(si);
1860 pci_release_mem_regions(pdev);
1861 pci_disable_device(pdev);
1862}
1863
