1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/prefetch.h>
29#include <net/busy_poll.h>
30
31#include "i40evf.h"
32#include "i40e_trace.h"
33#include "i40e_prototype.h"
34
35static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
36 u32 td_tag)
37{
38 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
39 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
40 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
41 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
42 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
43}
44
45#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
46
47
48
49
50
51
52static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
53 struct i40e_tx_buffer *tx_buffer)
54{
55 if (tx_buffer->skb) {
56 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
57 kfree(tx_buffer->raw_buf);
58 else
59 dev_kfree_skb_any(tx_buffer->skb);
60 if (dma_unmap_len(tx_buffer, len))
61 dma_unmap_single(ring->dev,
62 dma_unmap_addr(tx_buffer, dma),
63 dma_unmap_len(tx_buffer, len),
64 DMA_TO_DEVICE);
65 } else if (dma_unmap_len(tx_buffer, len)) {
66 dma_unmap_page(ring->dev,
67 dma_unmap_addr(tx_buffer, dma),
68 dma_unmap_len(tx_buffer, len),
69 DMA_TO_DEVICE);
70 }
71
72 tx_buffer->next_to_watch = NULL;
73 tx_buffer->skb = NULL;
74 dma_unmap_len_set(tx_buffer, len, 0);
75
76}
77
78
79
80
81
82void i40evf_clean_tx_ring(struct i40e_ring *tx_ring)
83{
84 unsigned long bi_size;
85 u16 i;
86
87
88 if (!tx_ring->tx_bi)
89 return;
90
91
92 for (i = 0; i < tx_ring->count; i++)
93 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
94
95 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
96 memset(tx_ring->tx_bi, 0, bi_size);
97
98
99 memset(tx_ring->desc, 0, tx_ring->size);
100
101 tx_ring->next_to_use = 0;
102 tx_ring->next_to_clean = 0;
103
104 if (!tx_ring->netdev)
105 return;
106
107
108 netdev_tx_reset_queue(txring_txq(tx_ring));
109}
110
111
112
113
114
115
116
117void i40evf_free_tx_resources(struct i40e_ring *tx_ring)
118{
119 i40evf_clean_tx_ring(tx_ring);
120 kfree(tx_ring->tx_bi);
121 tx_ring->tx_bi = NULL;
122
123 if (tx_ring->desc) {
124 dma_free_coherent(tx_ring->dev, tx_ring->size,
125 tx_ring->desc, tx_ring->dma);
126 tx_ring->desc = NULL;
127 }
128}
129
130
131
132
133
134
135
136
137
138u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw)
139{
140 u32 head, tail;
141
142 head = ring->next_to_clean;
143 tail = readl(ring->tail);
144
145 if (head != tail)
146 return (head < tail) ?
147 tail - head : (tail + ring->count - head);
148
149 return 0;
150}
151
152
153
154
155
156
157
158
159void i40evf_detect_recover_hung(struct i40e_vsi *vsi)
160{
161 struct i40e_ring *tx_ring = NULL;
162 struct net_device *netdev;
163 unsigned int i;
164 int packets;
165
166 if (!vsi)
167 return;
168
169 if (test_bit(__I40E_VSI_DOWN, vsi->state))
170 return;
171
172 netdev = vsi->netdev;
173 if (!netdev)
174 return;
175
176 if (!netif_carrier_ok(netdev))
177 return;
178
179 for (i = 0; i < vsi->back->num_active_queues; i++) {
180 tx_ring = &vsi->back->tx_rings[i];
181 if (tx_ring && tx_ring->desc) {
182
183
184
185
186
187
188
189 packets = tx_ring->stats.packets & INT_MAX;
190 if (tx_ring->tx_stats.prev_pkt_ctr == packets) {
191 i40evf_force_wb(vsi, tx_ring->q_vector);
192 continue;
193 }
194
195
196
197
198 smp_rmb();
199 tx_ring->tx_stats.prev_pkt_ctr =
200 i40evf_get_tx_pending(tx_ring, true) ? packets : -1;
201 }
202 }
203}
204
205#define WB_STRIDE 4
206
207
208
209
210
211
212
213
214
215static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
216 struct i40e_ring *tx_ring, int napi_budget)
217{
218 u16 i = tx_ring->next_to_clean;
219 struct i40e_tx_buffer *tx_buf;
220 struct i40e_tx_desc *tx_desc;
221 unsigned int total_bytes = 0, total_packets = 0;
222 unsigned int budget = vsi->work_limit;
223
224 tx_buf = &tx_ring->tx_bi[i];
225 tx_desc = I40E_TX_DESC(tx_ring, i);
226 i -= tx_ring->count;
227
228 do {
229 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
230
231
232 if (!eop_desc)
233 break;
234
235
236 smp_rmb();
237
238 i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf);
239
240 if (!(eop_desc->cmd_type_offset_bsz &
241 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
242 break;
243
244
245 tx_buf->next_to_watch = NULL;
246
247
248 total_bytes += tx_buf->bytecount;
249 total_packets += tx_buf->gso_segs;
250
251
252 napi_consume_skb(tx_buf->skb, napi_budget);
253
254
255 dma_unmap_single(tx_ring->dev,
256 dma_unmap_addr(tx_buf, dma),
257 dma_unmap_len(tx_buf, len),
258 DMA_TO_DEVICE);
259
260
261 tx_buf->skb = NULL;
262 dma_unmap_len_set(tx_buf, len, 0);
263
264
265 while (tx_desc != eop_desc) {
266 i40e_trace(clean_tx_irq_unmap,
267 tx_ring, tx_desc, tx_buf);
268
269 tx_buf++;
270 tx_desc++;
271 i++;
272 if (unlikely(!i)) {
273 i -= tx_ring->count;
274 tx_buf = tx_ring->tx_bi;
275 tx_desc = I40E_TX_DESC(tx_ring, 0);
276 }
277
278
279 if (dma_unmap_len(tx_buf, len)) {
280 dma_unmap_page(tx_ring->dev,
281 dma_unmap_addr(tx_buf, dma),
282 dma_unmap_len(tx_buf, len),
283 DMA_TO_DEVICE);
284 dma_unmap_len_set(tx_buf, len, 0);
285 }
286 }
287
288
289 tx_buf++;
290 tx_desc++;
291 i++;
292 if (unlikely(!i)) {
293 i -= tx_ring->count;
294 tx_buf = tx_ring->tx_bi;
295 tx_desc = I40E_TX_DESC(tx_ring, 0);
296 }
297
298 prefetch(tx_desc);
299
300
301 budget--;
302 } while (likely(budget));
303
304 i += tx_ring->count;
305 tx_ring->next_to_clean = i;
306 u64_stats_update_begin(&tx_ring->syncp);
307 tx_ring->stats.bytes += total_bytes;
308 tx_ring->stats.packets += total_packets;
309 u64_stats_update_end(&tx_ring->syncp);
310 tx_ring->q_vector->tx.total_bytes += total_bytes;
311 tx_ring->q_vector->tx.total_packets += total_packets;
312
313 if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {
314
315
316
317
318
319 unsigned int j = i40evf_get_tx_pending(tx_ring, false);
320
321 if (budget &&
322 ((j / WB_STRIDE) == 0) && (j > 0) &&
323 !test_bit(__I40E_VSI_DOWN, vsi->state) &&
324 (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))
325 tx_ring->arm_wb = true;
326 }
327
328
329 netdev_tx_completed_queue(txring_txq(tx_ring),
330 total_packets, total_bytes);
331
332#define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))
333 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
334 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
335
336
337
338 smp_mb();
339 if (__netif_subqueue_stopped(tx_ring->netdev,
340 tx_ring->queue_index) &&
341 !test_bit(__I40E_VSI_DOWN, vsi->state)) {
342 netif_wake_subqueue(tx_ring->netdev,
343 tx_ring->queue_index);
344 ++tx_ring->tx_stats.restart_queue;
345 }
346 }
347
348 return !!budget;
349}
350
351
352
353
354
355
356
357static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
358 struct i40e_q_vector *q_vector)
359{
360 u16 flags = q_vector->tx.ring[0].flags;
361 u32 val;
362
363 if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR))
364 return;
365
366 if (q_vector->arm_wb_state)
367 return;
368
369 val = I40E_VFINT_DYN_CTLN1_WB_ON_ITR_MASK |
370 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK;
371
372 wr32(&vsi->back->hw,
373 I40E_VFINT_DYN_CTLN1(q_vector->reg_idx), val);
374 q_vector->arm_wb_state = true;
375}
376
377
378
379
380
381
382
383void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
384{
385 u32 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
386 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK |
387 I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
388 I40E_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK
389 ;
390
391 wr32(&vsi->back->hw,
392 I40E_VFINT_DYN_CTLN1(q_vector->reg_idx),
393 val);
394}
395
396static inline bool i40e_container_is_rx(struct i40e_q_vector *q_vector,
397 struct i40e_ring_container *rc)
398{
399 return &q_vector->rx == rc;
400}
401
402static inline unsigned int i40e_itr_divisor(struct i40e_q_vector *q_vector)
403{
404 unsigned int divisor;
405
406 switch (q_vector->adapter->link_speed) {
407 case I40E_LINK_SPEED_40GB:
408 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 1024;
409 break;
410 case I40E_LINK_SPEED_25GB:
411 case I40E_LINK_SPEED_20GB:
412 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 512;
413 break;
414 default:
415 case I40E_LINK_SPEED_10GB:
416 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 256;
417 break;
418 case I40E_LINK_SPEED_1GB:
419 case I40E_LINK_SPEED_100MB:
420 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 32;
421 break;
422 }
423
424 return divisor;
425}
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440static void i40e_update_itr(struct i40e_q_vector *q_vector,
441 struct i40e_ring_container *rc)
442{
443 unsigned int avg_wire_size, packets, bytes, itr;
444 unsigned long next_update = jiffies;
445
446
447
448
449 if (!rc->ring || !ITR_IS_DYNAMIC(rc->ring->itr_setting))
450 return;
451
452
453
454
455 itr = i40e_container_is_rx(q_vector, rc) ?
456 I40E_ITR_ADAPTIVE_MIN_USECS | I40E_ITR_ADAPTIVE_LATENCY :
457 I40E_ITR_ADAPTIVE_MAX_USECS | I40E_ITR_ADAPTIVE_LATENCY;
458
459
460
461
462
463
464 if (time_after(next_update, rc->next_update))
465 goto clear_counts;
466
467
468
469
470
471
472
473 if (q_vector->itr_countdown) {
474 itr = rc->target_itr;
475 goto clear_counts;
476 }
477
478 packets = rc->total_packets;
479 bytes = rc->total_bytes;
480
481 if (i40e_container_is_rx(q_vector, rc)) {
482
483
484
485
486
487 if (packets && packets < 4 && bytes < 9000 &&
488 (q_vector->tx.target_itr & I40E_ITR_ADAPTIVE_LATENCY)) {
489 itr = I40E_ITR_ADAPTIVE_LATENCY;
490 goto adjust_by_size;
491 }
492 } else if (packets < 4) {
493
494
495
496
497
498 if (rc->target_itr == I40E_ITR_ADAPTIVE_MAX_USECS &&
499 (q_vector->rx.target_itr & I40E_ITR_MASK) ==
500 I40E_ITR_ADAPTIVE_MAX_USECS)
501 goto clear_counts;
502 } else if (packets > 32) {
503
504
505
506 rc->target_itr &= ~I40E_ITR_ADAPTIVE_LATENCY;
507 }
508
509
510
511
512
513
514
515
516
517 if (packets < 56) {
518 itr = rc->target_itr + I40E_ITR_ADAPTIVE_MIN_INC;
519 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
520 itr &= I40E_ITR_ADAPTIVE_LATENCY;
521 itr += I40E_ITR_ADAPTIVE_MAX_USECS;
522 }
523 goto clear_counts;
524 }
525
526 if (packets <= 256) {
527 itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr);
528 itr &= I40E_ITR_MASK;
529
530
531
532
533
534 if (packets <= 112)
535 goto clear_counts;
536
537
538
539
540
541
542 itr /= 2;
543 itr &= I40E_ITR_MASK;
544 if (itr < I40E_ITR_ADAPTIVE_MIN_USECS)
545 itr = I40E_ITR_ADAPTIVE_MIN_USECS;
546
547 goto clear_counts;
548 }
549
550
551
552
553
554
555
556 itr = I40E_ITR_ADAPTIVE_BULK;
557
558adjust_by_size:
559
560
561
562
563
564 avg_wire_size = bytes / packets;
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581 if (avg_wire_size <= 60) {
582
583 avg_wire_size = 4096;
584 } else if (avg_wire_size <= 380) {
585
586 avg_wire_size *= 40;
587 avg_wire_size += 1696;
588 } else if (avg_wire_size <= 1084) {
589
590 avg_wire_size *= 15;
591 avg_wire_size += 11452;
592 } else if (avg_wire_size <= 1980) {
593
594 avg_wire_size *= 5;
595 avg_wire_size += 22420;
596 } else {
597
598 avg_wire_size = 32256;
599 }
600
601
602
603
604 if (itr & I40E_ITR_ADAPTIVE_LATENCY)
605 avg_wire_size /= 2;
606
607
608
609
610
611
612
613
614 itr += DIV_ROUND_UP(avg_wire_size, i40e_itr_divisor(q_vector)) *
615 I40E_ITR_ADAPTIVE_MIN_INC;
616
617 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
618 itr &= I40E_ITR_ADAPTIVE_LATENCY;
619 itr += I40E_ITR_ADAPTIVE_MAX_USECS;
620 }
621
622clear_counts:
623
624 rc->target_itr = itr;
625
626
627 rc->next_update = next_update + 1;
628
629 rc->total_bytes = 0;
630 rc->total_packets = 0;
631}
632
633
634
635
636
637
638
639int i40evf_setup_tx_descriptors(struct i40e_ring *tx_ring)
640{
641 struct device *dev = tx_ring->dev;
642 int bi_size;
643
644 if (!dev)
645 return -ENOMEM;
646
647
648 WARN_ON(tx_ring->tx_bi);
649 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
650 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
651 if (!tx_ring->tx_bi)
652 goto err;
653
654
655 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
656 tx_ring->size = ALIGN(tx_ring->size, 4096);
657 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
658 &tx_ring->dma, GFP_KERNEL);
659 if (!tx_ring->desc) {
660 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
661 tx_ring->size);
662 goto err;
663 }
664
665 tx_ring->next_to_use = 0;
666 tx_ring->next_to_clean = 0;
667 tx_ring->tx_stats.prev_pkt_ctr = -1;
668 return 0;
669
670err:
671 kfree(tx_ring->tx_bi);
672 tx_ring->tx_bi = NULL;
673 return -ENOMEM;
674}
675
676
677
678
679
680void i40evf_clean_rx_ring(struct i40e_ring *rx_ring)
681{
682 unsigned long bi_size;
683 u16 i;
684
685
686 if (!rx_ring->rx_bi)
687 return;
688
689 if (rx_ring->skb) {
690 dev_kfree_skb(rx_ring->skb);
691 rx_ring->skb = NULL;
692 }
693
694
695 for (i = 0; i < rx_ring->count; i++) {
696 struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
697
698 if (!rx_bi->page)
699 continue;
700
701
702
703
704 dma_sync_single_range_for_cpu(rx_ring->dev,
705 rx_bi->dma,
706 rx_bi->page_offset,
707 rx_ring->rx_buf_len,
708 DMA_FROM_DEVICE);
709
710
711 dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma,
712 i40e_rx_pg_size(rx_ring),
713 DMA_FROM_DEVICE,
714 I40E_RX_DMA_ATTR);
715
716 __page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias);
717
718 rx_bi->page = NULL;
719 rx_bi->page_offset = 0;
720 }
721
722 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
723 memset(rx_ring->rx_bi, 0, bi_size);
724
725
726 memset(rx_ring->desc, 0, rx_ring->size);
727
728 rx_ring->next_to_alloc = 0;
729 rx_ring->next_to_clean = 0;
730 rx_ring->next_to_use = 0;
731}
732
733
734
735
736
737
738
739void i40evf_free_rx_resources(struct i40e_ring *rx_ring)
740{
741 i40evf_clean_rx_ring(rx_ring);
742 kfree(rx_ring->rx_bi);
743 rx_ring->rx_bi = NULL;
744
745 if (rx_ring->desc) {
746 dma_free_coherent(rx_ring->dev, rx_ring->size,
747 rx_ring->desc, rx_ring->dma);
748 rx_ring->desc = NULL;
749 }
750}
751
752
753
754
755
756
757
758int i40evf_setup_rx_descriptors(struct i40e_ring *rx_ring)
759{
760 struct device *dev = rx_ring->dev;
761 int bi_size;
762
763
764 WARN_ON(rx_ring->rx_bi);
765 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
766 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
767 if (!rx_ring->rx_bi)
768 goto err;
769
770 u64_stats_init(&rx_ring->syncp);
771
772
773 rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc);
774 rx_ring->size = ALIGN(rx_ring->size, 4096);
775 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
776 &rx_ring->dma, GFP_KERNEL);
777
778 if (!rx_ring->desc) {
779 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
780 rx_ring->size);
781 goto err;
782 }
783
784 rx_ring->next_to_alloc = 0;
785 rx_ring->next_to_clean = 0;
786 rx_ring->next_to_use = 0;
787
788 return 0;
789err:
790 kfree(rx_ring->rx_bi);
791 rx_ring->rx_bi = NULL;
792 return -ENOMEM;
793}
794
795
796
797
798
799
800static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
801{
802 rx_ring->next_to_use = val;
803
804
805 rx_ring->next_to_alloc = val;
806
807
808
809
810
811
812 wmb();
813 writel(val, rx_ring->tail);
814}
815
816
817
818
819
820
821
822static inline unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
823{
824 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
825}
826
827
828
829
830
831
832
833
834
835static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring,
836 struct i40e_rx_buffer *bi)
837{
838 struct page *page = bi->page;
839 dma_addr_t dma;
840
841
842 if (likely(page)) {
843 rx_ring->rx_stats.page_reuse_count++;
844 return true;
845 }
846
847
848 page = dev_alloc_pages(i40e_rx_pg_order(rx_ring));
849 if (unlikely(!page)) {
850 rx_ring->rx_stats.alloc_page_failed++;
851 return false;
852 }
853
854
855 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
856 i40e_rx_pg_size(rx_ring),
857 DMA_FROM_DEVICE,
858 I40E_RX_DMA_ATTR);
859
860
861
862
863 if (dma_mapping_error(rx_ring->dev, dma)) {
864 __free_pages(page, i40e_rx_pg_order(rx_ring));
865 rx_ring->rx_stats.alloc_page_failed++;
866 return false;
867 }
868
869 bi->dma = dma;
870 bi->page = page;
871 bi->page_offset = i40e_rx_offset(rx_ring);
872
873
874 bi->pagecnt_bias = 1;
875
876 return true;
877}
878
879
880
881
882
883
884
885static void i40e_receive_skb(struct i40e_ring *rx_ring,
886 struct sk_buff *skb, u16 vlan_tag)
887{
888 struct i40e_q_vector *q_vector = rx_ring->q_vector;
889
890 if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
891 (vlan_tag & VLAN_VID_MASK))
892 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
893
894 napi_gro_receive(&q_vector->napi, skb);
895}
896
897
898
899
900
901
902
903
904bool i40evf_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
905{
906 u16 ntu = rx_ring->next_to_use;
907 union i40e_rx_desc *rx_desc;
908 struct i40e_rx_buffer *bi;
909
910
911 if (!rx_ring->netdev || !cleaned_count)
912 return false;
913
914 rx_desc = I40E_RX_DESC(rx_ring, ntu);
915 bi = &rx_ring->rx_bi[ntu];
916
917 do {
918 if (!i40e_alloc_mapped_page(rx_ring, bi))
919 goto no_buffers;
920
921
922 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
923 bi->page_offset,
924 rx_ring->rx_buf_len,
925 DMA_FROM_DEVICE);
926
927
928
929
930 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
931
932 rx_desc++;
933 bi++;
934 ntu++;
935 if (unlikely(ntu == rx_ring->count)) {
936 rx_desc = I40E_RX_DESC(rx_ring, 0);
937 bi = rx_ring->rx_bi;
938 ntu = 0;
939 }
940
941
942 rx_desc->wb.qword1.status_error_len = 0;
943
944 cleaned_count--;
945 } while (cleaned_count);
946
947 if (rx_ring->next_to_use != ntu)
948 i40e_release_rx_desc(rx_ring, ntu);
949
950 return false;
951
952no_buffers:
953 if (rx_ring->next_to_use != ntu)
954 i40e_release_rx_desc(rx_ring, ntu);
955
956
957
958
959 return true;
960}
961
962
963
964
965
966
967
968static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
969 struct sk_buff *skb,
970 union i40e_rx_desc *rx_desc)
971{
972 struct i40e_rx_ptype_decoded decoded;
973 u32 rx_error, rx_status;
974 bool ipv4, ipv6;
975 u8 ptype;
976 u64 qword;
977
978 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
979 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT;
980 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
981 I40E_RXD_QW1_ERROR_SHIFT;
982 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
983 I40E_RXD_QW1_STATUS_SHIFT;
984 decoded = decode_rx_desc_ptype(ptype);
985
986 skb->ip_summed = CHECKSUM_NONE;
987
988 skb_checksum_none_assert(skb);
989
990
991 if (!(vsi->netdev->features & NETIF_F_RXCSUM))
992 return;
993
994
995 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
996 return;
997
998
999 if (!(decoded.known && decoded.outer_ip))
1000 return;
1001
1002 ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
1003 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4);
1004 ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
1005 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6);
1006
1007 if (ipv4 &&
1008 (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) |
1009 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT))))
1010 goto checksum_fail;
1011
1012
1013 if (ipv6 &&
1014 rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1015
1016 return;
1017
1018
1019 if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT))
1020 goto checksum_fail;
1021
1022
1023
1024
1025
1026 if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT))
1027 return;
1028
1029
1030 switch (decoded.inner_prot) {
1031 case I40E_RX_PTYPE_INNER_PROT_TCP:
1032 case I40E_RX_PTYPE_INNER_PROT_UDP:
1033 case I40E_RX_PTYPE_INNER_PROT_SCTP:
1034 skb->ip_summed = CHECKSUM_UNNECESSARY;
1035
1036 default:
1037 break;
1038 }
1039
1040 return;
1041
1042checksum_fail:
1043 vsi->back->hw_csum_rx_error++;
1044}
1045
1046
1047
1048
1049
1050
1051
1052static inline int i40e_ptype_to_htype(u8 ptype)
1053{
1054 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1055
1056 if (!decoded.known)
1057 return PKT_HASH_TYPE_NONE;
1058
1059 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1060 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1061 return PKT_HASH_TYPE_L4;
1062 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1063 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1064 return PKT_HASH_TYPE_L3;
1065 else
1066 return PKT_HASH_TYPE_L2;
1067}
1068
1069
1070
1071
1072
1073
1074static inline void i40e_rx_hash(struct i40e_ring *ring,
1075 union i40e_rx_desc *rx_desc,
1076 struct sk_buff *skb,
1077 u8 rx_ptype)
1078{
1079 u32 hash;
1080 const __le64 rss_mask =
1081 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1082 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1083
1084 if (ring->netdev->features & NETIF_F_RXHASH)
1085 return;
1086
1087 if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
1088 hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1089 skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype));
1090 }
1091}
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104static inline
1105void i40evf_process_skb_fields(struct i40e_ring *rx_ring,
1106 union i40e_rx_desc *rx_desc, struct sk_buff *skb,
1107 u8 rx_ptype)
1108{
1109 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
1110
1111 i40e_rx_checksum(rx_ring->vsi, skb, rx_desc);
1112
1113 skb_record_rx_queue(skb, rx_ring->queue_index);
1114
1115
1116 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1117}
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb)
1133{
1134
1135 if (eth_skb_pad(skb))
1136 return true;
1137
1138 return false;
1139}
1140
1141
1142
1143
1144
1145
1146
1147
1148static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
1149 struct i40e_rx_buffer *old_buff)
1150{
1151 struct i40e_rx_buffer *new_buff;
1152 u16 nta = rx_ring->next_to_alloc;
1153
1154 new_buff = &rx_ring->rx_bi[nta];
1155
1156
1157 nta++;
1158 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1159
1160
1161 new_buff->dma = old_buff->dma;
1162 new_buff->page = old_buff->page;
1163 new_buff->page_offset = old_buff->page_offset;
1164 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
1165}
1166
1167
1168
1169
1170
1171
1172
1173
1174static inline bool i40e_page_is_reusable(struct page *page)
1175{
1176 return (page_to_nid(page) == numa_mem_id()) &&
1177 !page_is_pfmemalloc(page);
1178}
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer)
1208{
1209 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1210 struct page *page = rx_buffer->page;
1211
1212
1213 if (unlikely(!i40e_page_is_reusable(page)))
1214 return false;
1215
1216#if (PAGE_SIZE < 8192)
1217
1218 if (unlikely((page_count(page) - pagecnt_bias) > 1))
1219 return false;
1220#else
1221#define I40E_LAST_OFFSET \
1222 (SKB_WITH_OVERHEAD(PAGE_SIZE) - I40E_RXBUFFER_2048)
1223 if (rx_buffer->page_offset > I40E_LAST_OFFSET)
1224 return false;
1225#endif
1226
1227
1228
1229
1230
1231 if (unlikely(!pagecnt_bias)) {
1232 page_ref_add(page, USHRT_MAX);
1233 rx_buffer->pagecnt_bias = USHRT_MAX;
1234 }
1235
1236 return true;
1237}
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
1252 struct i40e_rx_buffer *rx_buffer,
1253 struct sk_buff *skb,
1254 unsigned int size)
1255{
1256#if (PAGE_SIZE < 8192)
1257 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1258#else
1259 unsigned int truesize = SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring));
1260#endif
1261
1262 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1263 rx_buffer->page_offset, size, truesize);
1264
1265
1266#if (PAGE_SIZE < 8192)
1267 rx_buffer->page_offset ^= truesize;
1268#else
1269 rx_buffer->page_offset += truesize;
1270#endif
1271}
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
1282 const unsigned int size)
1283{
1284 struct i40e_rx_buffer *rx_buffer;
1285
1286 rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
1287 prefetchw(rx_buffer->page);
1288
1289
1290 dma_sync_single_range_for_cpu(rx_ring->dev,
1291 rx_buffer->dma,
1292 rx_buffer->page_offset,
1293 size,
1294 DMA_FROM_DEVICE);
1295
1296
1297 rx_buffer->pagecnt_bias--;
1298
1299 return rx_buffer;
1300}
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
1313 struct i40e_rx_buffer *rx_buffer,
1314 unsigned int size)
1315{
1316 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1317#if (PAGE_SIZE < 8192)
1318 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1319#else
1320 unsigned int truesize = SKB_DATA_ALIGN(size);
1321#endif
1322 unsigned int headlen;
1323 struct sk_buff *skb;
1324
1325
1326 prefetch(va);
1327#if L1_CACHE_BYTES < 128
1328 prefetch(va + L1_CACHE_BYTES);
1329#endif
1330
1331
1332 skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
1333 I40E_RX_HDR_SIZE,
1334 GFP_ATOMIC | __GFP_NOWARN);
1335 if (unlikely(!skb))
1336 return NULL;
1337
1338
1339 headlen = size;
1340 if (headlen > I40E_RX_HDR_SIZE)
1341 headlen = eth_get_headlen(va, I40E_RX_HDR_SIZE);
1342
1343
1344 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
1345
1346
1347 size -= headlen;
1348 if (size) {
1349 skb_add_rx_frag(skb, 0, rx_buffer->page,
1350 rx_buffer->page_offset + headlen,
1351 size, truesize);
1352
1353
1354#if (PAGE_SIZE < 8192)
1355 rx_buffer->page_offset ^= truesize;
1356#else
1357 rx_buffer->page_offset += truesize;
1358#endif
1359 } else {
1360
1361 rx_buffer->pagecnt_bias++;
1362 }
1363
1364 return skb;
1365}
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
1377 struct i40e_rx_buffer *rx_buffer,
1378 unsigned int size)
1379{
1380 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1381#if (PAGE_SIZE < 8192)
1382 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1383#else
1384 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1385 SKB_DATA_ALIGN(I40E_SKB_PAD + size);
1386#endif
1387 struct sk_buff *skb;
1388
1389
1390 prefetch(va);
1391#if L1_CACHE_BYTES < 128
1392 prefetch(va + L1_CACHE_BYTES);
1393#endif
1394
1395 skb = build_skb(va - I40E_SKB_PAD, truesize);
1396 if (unlikely(!skb))
1397 return NULL;
1398
1399
1400 skb_reserve(skb, I40E_SKB_PAD);
1401 __skb_put(skb, size);
1402
1403
1404#if (PAGE_SIZE < 8192)
1405 rx_buffer->page_offset ^= truesize;
1406#else
1407 rx_buffer->page_offset += truesize;
1408#endif
1409
1410 return skb;
1411}
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
1422 struct i40e_rx_buffer *rx_buffer)
1423{
1424 if (i40e_can_reuse_rx_page(rx_buffer)) {
1425
1426 i40e_reuse_rx_page(rx_ring, rx_buffer);
1427 rx_ring->rx_stats.page_reuse_count++;
1428 } else {
1429
1430 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1431 i40e_rx_pg_size(rx_ring),
1432 DMA_FROM_DEVICE, I40E_RX_DMA_ATTR);
1433 __page_frag_cache_drain(rx_buffer->page,
1434 rx_buffer->pagecnt_bias);
1435 }
1436
1437
1438 rx_buffer->page = NULL;
1439}
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452static bool i40e_is_non_eop(struct i40e_ring *rx_ring,
1453 union i40e_rx_desc *rx_desc,
1454 struct sk_buff *skb)
1455{
1456 u32 ntc = rx_ring->next_to_clean + 1;
1457
1458
1459 ntc = (ntc < rx_ring->count) ? ntc : 0;
1460 rx_ring->next_to_clean = ntc;
1461
1462 prefetch(I40E_RX_DESC(rx_ring, ntc));
1463
1464
1465#define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)
1466 if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF)))
1467 return false;
1468
1469 rx_ring->rx_stats.non_eop_descs++;
1470
1471 return true;
1472}
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1487{
1488 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1489 struct sk_buff *skb = rx_ring->skb;
1490 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1491 bool failure = false;
1492
1493 while (likely(total_rx_packets < (unsigned int)budget)) {
1494 struct i40e_rx_buffer *rx_buffer;
1495 union i40e_rx_desc *rx_desc;
1496 unsigned int size;
1497 u16 vlan_tag;
1498 u8 rx_ptype;
1499 u64 qword;
1500
1501
1502 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1503 failure = failure ||
1504 i40evf_alloc_rx_buffers(rx_ring, cleaned_count);
1505 cleaned_count = 0;
1506 }
1507
1508 rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);
1509
1510
1511
1512
1513
1514
1515 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1516
1517
1518
1519
1520
1521 dma_rmb();
1522
1523 size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1524 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1525 if (!size)
1526 break;
1527
1528 i40e_trace(clean_rx_irq, rx_ring, rx_desc, skb);
1529 rx_buffer = i40e_get_rx_buffer(rx_ring, size);
1530
1531
1532 if (skb)
1533 i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
1534 else if (ring_uses_build_skb(rx_ring))
1535 skb = i40e_build_skb(rx_ring, rx_buffer, size);
1536 else
1537 skb = i40e_construct_skb(rx_ring, rx_buffer, size);
1538
1539
1540 if (!skb) {
1541 rx_ring->rx_stats.alloc_buff_failed++;
1542 rx_buffer->pagecnt_bias++;
1543 break;
1544 }
1545
1546 i40e_put_rx_buffer(rx_ring, rx_buffer);
1547 cleaned_count++;
1548
1549 if (i40e_is_non_eop(rx_ring, rx_desc, skb))
1550 continue;
1551
1552
1553
1554
1555
1556
1557 if (unlikely(i40e_test_staterr(rx_desc, BIT(I40E_RXD_QW1_ERROR_SHIFT)))) {
1558 dev_kfree_skb_any(skb);
1559 skb = NULL;
1560 continue;
1561 }
1562
1563 if (i40e_cleanup_headers(rx_ring, skb)) {
1564 skb = NULL;
1565 continue;
1566 }
1567
1568
1569 total_rx_bytes += skb->len;
1570
1571 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1572 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1573 I40E_RXD_QW1_PTYPE_SHIFT;
1574
1575
1576 i40evf_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
1577
1578
1579 vlan_tag = (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) ?
1580 le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0;
1581
1582 i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb);
1583 i40e_receive_skb(rx_ring, skb, vlan_tag);
1584 skb = NULL;
1585
1586
1587 total_rx_packets++;
1588 }
1589
1590 rx_ring->skb = skb;
1591
1592 u64_stats_update_begin(&rx_ring->syncp);
1593 rx_ring->stats.packets += total_rx_packets;
1594 rx_ring->stats.bytes += total_rx_bytes;
1595 u64_stats_update_end(&rx_ring->syncp);
1596 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1597 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1598
1599
1600 return failure ? budget : (int)total_rx_packets;
1601}
1602
1603static inline u32 i40e_buildreg_itr(const int type, u16 itr)
1604{
1605 u32 val;
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622 itr &= I40E_ITR_MASK;
1623
1624 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1625 (type << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1626 (itr << (I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT - 1));
1627
1628 return val;
1629}
1630
1631
1632#define INTREG I40E_VFINT_DYN_CTLN1
1633
1634
1635
1636
1637
1638
1639
1640
1641#define ITR_COUNTDOWN_START 3
1642
1643
1644
1645
1646
1647
1648
1649static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
1650 struct i40e_q_vector *q_vector)
1651{
1652 struct i40e_hw *hw = &vsi->back->hw;
1653 u32 intval;
1654
1655
1656 i40e_update_itr(q_vector, &q_vector->tx);
1657 i40e_update_itr(q_vector, &q_vector->rx);
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667 if (q_vector->rx.target_itr < q_vector->rx.current_itr) {
1668
1669 intval = i40e_buildreg_itr(I40E_RX_ITR,
1670 q_vector->rx.target_itr);
1671 q_vector->rx.current_itr = q_vector->rx.target_itr;
1672 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1673 } else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) ||
1674 ((q_vector->rx.target_itr - q_vector->rx.current_itr) <
1675 (q_vector->tx.target_itr - q_vector->tx.current_itr))) {
1676
1677
1678
1679 intval = i40e_buildreg_itr(I40E_TX_ITR,
1680 q_vector->tx.target_itr);
1681 q_vector->tx.current_itr = q_vector->tx.target_itr;
1682 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1683 } else if (q_vector->rx.current_itr != q_vector->rx.target_itr) {
1684
1685 intval = i40e_buildreg_itr(I40E_RX_ITR,
1686 q_vector->rx.target_itr);
1687 q_vector->rx.current_itr = q_vector->rx.target_itr;
1688 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1689 } else {
1690
1691 intval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
1692 if (q_vector->itr_countdown)
1693 q_vector->itr_countdown--;
1694 }
1695
1696 if (!test_bit(__I40E_VSI_DOWN, vsi->state))
1697 wr32(hw, INTREG(q_vector->reg_idx), intval);
1698}
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709int i40evf_napi_poll(struct napi_struct *napi, int budget)
1710{
1711 struct i40e_q_vector *q_vector =
1712 container_of(napi, struct i40e_q_vector, napi);
1713 struct i40e_vsi *vsi = q_vector->vsi;
1714 struct i40e_ring *ring;
1715 bool clean_complete = true;
1716 bool arm_wb = false;
1717 int budget_per_ring;
1718 int work_done = 0;
1719
1720 if (test_bit(__I40E_VSI_DOWN, vsi->state)) {
1721 napi_complete(napi);
1722 return 0;
1723 }
1724
1725
1726
1727
1728 i40e_for_each_ring(ring, q_vector->tx) {
1729 if (!i40e_clean_tx_irq(vsi, ring, budget)) {
1730 clean_complete = false;
1731 continue;
1732 }
1733 arm_wb |= ring->arm_wb;
1734 ring->arm_wb = false;
1735 }
1736
1737
1738 if (budget <= 0)
1739 goto tx_only;
1740
1741
1742
1743
1744 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1745
1746 i40e_for_each_ring(ring, q_vector->rx) {
1747 int cleaned = i40e_clean_rx_irq(ring, budget_per_ring);
1748
1749 work_done += cleaned;
1750
1751 if (cleaned >= budget_per_ring)
1752 clean_complete = false;
1753 }
1754
1755
1756 if (!clean_complete) {
1757 int cpu_id = smp_processor_id();
1758
1759
1760
1761
1762
1763
1764
1765
1766 if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) {
1767
1768 napi_complete_done(napi, work_done);
1769
1770
1771 i40evf_force_wb(vsi, q_vector);
1772
1773
1774 return budget - 1;
1775 }
1776tx_only:
1777 if (arm_wb) {
1778 q_vector->tx.ring[0].tx_stats.tx_force_wb++;
1779 i40e_enable_wb_on_itr(vsi, q_vector);
1780 }
1781 return budget;
1782 }
1783
1784 if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
1785 q_vector->arm_wb_state = false;
1786
1787
1788 napi_complete_done(napi, work_done);
1789
1790 i40e_update_enable_itr(vsi, q_vector);
1791
1792 return min(work_done, budget - 1);
1793}
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807static inline int i40evf_tx_prepare_vlan_flags(struct sk_buff *skb,
1808 struct i40e_ring *tx_ring,
1809 u32 *flags)
1810{
1811 __be16 protocol = skb->protocol;
1812 u32 tx_flags = 0;
1813
1814 if (protocol == htons(ETH_P_8021Q) &&
1815 !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
1816
1817
1818
1819
1820
1821
1822
1823 skb->protocol = vlan_get_protocol(skb);
1824 goto out;
1825 }
1826
1827
1828 if (skb_vlan_tag_present(skb)) {
1829 tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1830 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1831
1832 } else if (protocol == htons(ETH_P_8021Q)) {
1833 struct vlan_hdr *vhdr, _vhdr;
1834
1835 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1836 if (!vhdr)
1837 return -EINVAL;
1838
1839 protocol = vhdr->h_vlan_encapsulated_proto;
1840 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1841 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1842 }
1843
1844out:
1845 *flags = tx_flags;
1846 return 0;
1847}
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len,
1858 u64 *cd_type_cmd_tso_mss)
1859{
1860 struct sk_buff *skb = first->skb;
1861 u64 cd_cmd, cd_tso_len, cd_mss;
1862 union {
1863 struct iphdr *v4;
1864 struct ipv6hdr *v6;
1865 unsigned char *hdr;
1866 } ip;
1867 union {
1868 struct tcphdr *tcp;
1869 struct udphdr *udp;
1870 unsigned char *hdr;
1871 } l4;
1872 u32 paylen, l4_offset;
1873 u16 gso_segs, gso_size;
1874 int err;
1875
1876 if (skb->ip_summed != CHECKSUM_PARTIAL)
1877 return 0;
1878
1879 if (!skb_is_gso(skb))
1880 return 0;
1881
1882 err = skb_cow_head(skb, 0);
1883 if (err < 0)
1884 return err;
1885
1886 ip.hdr = skb_network_header(skb);
1887 l4.hdr = skb_transport_header(skb);
1888
1889
1890 if (ip.v4->version == 4) {
1891 ip.v4->tot_len = 0;
1892 ip.v4->check = 0;
1893 } else {
1894 ip.v6->payload_len = 0;
1895 }
1896
1897 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1898 SKB_GSO_GRE_CSUM |
1899 SKB_GSO_IPXIP4 |
1900 SKB_GSO_IPXIP6 |
1901 SKB_GSO_UDP_TUNNEL |
1902 SKB_GSO_UDP_TUNNEL_CSUM)) {
1903 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1904 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
1905 l4.udp->len = 0;
1906
1907
1908 l4_offset = l4.hdr - skb->data;
1909
1910
1911 paylen = skb->len - l4_offset;
1912 csum_replace_by_diff(&l4.udp->check,
1913 (__force __wsum)htonl(paylen));
1914 }
1915
1916
1917 ip.hdr = skb_inner_network_header(skb);
1918 l4.hdr = skb_inner_transport_header(skb);
1919
1920
1921 if (ip.v4->version == 4) {
1922 ip.v4->tot_len = 0;
1923 ip.v4->check = 0;
1924 } else {
1925 ip.v6->payload_len = 0;
1926 }
1927 }
1928
1929
1930 l4_offset = l4.hdr - skb->data;
1931
1932
1933 paylen = skb->len - l4_offset;
1934 csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
1935
1936
1937 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
1938
1939
1940 gso_size = skb_shinfo(skb)->gso_size;
1941 gso_segs = skb_shinfo(skb)->gso_segs;
1942
1943
1944 first->gso_segs = gso_segs;
1945 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1946
1947
1948 cd_cmd = I40E_TX_CTX_DESC_TSO;
1949 cd_tso_len = skb->len - *hdr_len;
1950 cd_mss = gso_size;
1951 *cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1952 (cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1953 (cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1954 return 1;
1955}
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
1967 u32 *td_cmd, u32 *td_offset,
1968 struct i40e_ring *tx_ring,
1969 u32 *cd_tunneling)
1970{
1971 union {
1972 struct iphdr *v4;
1973 struct ipv6hdr *v6;
1974 unsigned char *hdr;
1975 } ip;
1976 union {
1977 struct tcphdr *tcp;
1978 struct udphdr *udp;
1979 unsigned char *hdr;
1980 } l4;
1981 unsigned char *exthdr;
1982 u32 offset, cmd = 0;
1983 __be16 frag_off;
1984 u8 l4_proto = 0;
1985
1986 if (skb->ip_summed != CHECKSUM_PARTIAL)
1987 return 0;
1988
1989 ip.hdr = skb_network_header(skb);
1990 l4.hdr = skb_transport_header(skb);
1991
1992
1993 offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1994
1995 if (skb->encapsulation) {
1996 u32 tunnel = 0;
1997
1998 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1999 tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
2000 I40E_TX_CTX_EXT_IP_IPV4 :
2001 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
2002
2003 l4_proto = ip.v4->protocol;
2004 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
2005 tunnel |= I40E_TX_CTX_EXT_IP_IPV6;
2006
2007 exthdr = ip.hdr + sizeof(*ip.v6);
2008 l4_proto = ip.v6->nexthdr;
2009 if (l4.hdr != exthdr)
2010 ipv6_skip_exthdr(skb, exthdr - skb->data,
2011 &l4_proto, &frag_off);
2012 }
2013
2014
2015 switch (l4_proto) {
2016 case IPPROTO_UDP:
2017 tunnel |= I40E_TXD_CTX_UDP_TUNNELING;
2018 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
2019 break;
2020 case IPPROTO_GRE:
2021 tunnel |= I40E_TXD_CTX_GRE_TUNNELING;
2022 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
2023 break;
2024 case IPPROTO_IPIP:
2025 case IPPROTO_IPV6:
2026 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
2027 l4.hdr = skb_inner_network_header(skb);
2028 break;
2029 default:
2030 if (*tx_flags & I40E_TX_FLAGS_TSO)
2031 return -1;
2032
2033 skb_checksum_help(skb);
2034 return 0;
2035 }
2036
2037
2038 tunnel |= ((l4.hdr - ip.hdr) / 4) <<
2039 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;
2040
2041
2042 ip.hdr = skb_inner_network_header(skb);
2043
2044
2045 tunnel |= ((ip.hdr - l4.hdr) / 2) <<
2046 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
2047
2048
2049 if ((*tx_flags & I40E_TX_FLAGS_TSO) &&
2050 !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
2051 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
2052 tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK;
2053
2054
2055 *cd_tunneling |= tunnel;
2056
2057
2058 l4.hdr = skb_inner_transport_header(skb);
2059 l4_proto = 0;
2060
2061
2062 *tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6);
2063 if (ip.v4->version == 4)
2064 *tx_flags |= I40E_TX_FLAGS_IPV4;
2065 if (ip.v6->version == 6)
2066 *tx_flags |= I40E_TX_FLAGS_IPV6;
2067 }
2068
2069
2070 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
2071 l4_proto = ip.v4->protocol;
2072
2073
2074
2075 cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
2076 I40E_TX_DESC_CMD_IIPT_IPV4_CSUM :
2077 I40E_TX_DESC_CMD_IIPT_IPV4;
2078 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
2079 cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
2080
2081 exthdr = ip.hdr + sizeof(*ip.v6);
2082 l4_proto = ip.v6->nexthdr;
2083 if (l4.hdr != exthdr)
2084 ipv6_skip_exthdr(skb, exthdr - skb->data,
2085 &l4_proto, &frag_off);
2086 }
2087
2088
2089 offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
2090
2091
2092 switch (l4_proto) {
2093 case IPPROTO_TCP:
2094
2095 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
2096 offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2097 break;
2098 case IPPROTO_SCTP:
2099
2100 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
2101 offset |= (sizeof(struct sctphdr) >> 2) <<
2102 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2103 break;
2104 case IPPROTO_UDP:
2105
2106 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
2107 offset |= (sizeof(struct udphdr) >> 2) <<
2108 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2109 break;
2110 default:
2111 if (*tx_flags & I40E_TX_FLAGS_TSO)
2112 return -1;
2113 skb_checksum_help(skb);
2114 return 0;
2115 }
2116
2117 *td_cmd |= cmd;
2118 *td_offset |= offset;
2119
2120 return 1;
2121}
2122
2123
2124
2125
2126
2127
2128
2129
2130static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
2131 const u64 cd_type_cmd_tso_mss,
2132 const u32 cd_tunneling, const u32 cd_l2tag2)
2133{
2134 struct i40e_tx_context_desc *context_desc;
2135 int i = tx_ring->next_to_use;
2136
2137 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
2138 !cd_tunneling && !cd_l2tag2)
2139 return;
2140
2141
2142 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
2143
2144 i++;
2145 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2146
2147
2148 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
2149 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
2150 context_desc->rsvd = cpu_to_le16(0);
2151 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
2152}
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167bool __i40evf_chk_linearize(struct sk_buff *skb)
2168{
2169 const struct skb_frag_struct *frag, *stale;
2170 int nr_frags, sum;
2171
2172
2173 nr_frags = skb_shinfo(skb)->nr_frags;
2174 if (nr_frags < (I40E_MAX_BUFFER_TXD - 1))
2175 return false;
2176
2177
2178
2179
2180 nr_frags -= I40E_MAX_BUFFER_TXD - 2;
2181 frag = &skb_shinfo(skb)->frags[0];
2182
2183
2184
2185
2186
2187
2188
2189 sum = 1 - skb_shinfo(skb)->gso_size;
2190
2191
2192 sum += skb_frag_size(frag++);
2193 sum += skb_frag_size(frag++);
2194 sum += skb_frag_size(frag++);
2195 sum += skb_frag_size(frag++);
2196 sum += skb_frag_size(frag++);
2197
2198
2199
2200
2201 for (stale = &skb_shinfo(skb)->frags[0];; stale++) {
2202 int stale_size = skb_frag_size(stale);
2203
2204 sum += skb_frag_size(frag++);
2205
2206
2207
2208
2209
2210
2211
2212 if (stale_size > I40E_MAX_DATA_PER_TXD) {
2213 int align_pad = -(stale->page_offset) &
2214 (I40E_MAX_READ_REQ_SIZE - 1);
2215
2216 sum -= align_pad;
2217 stale_size -= align_pad;
2218
2219 do {
2220 sum -= I40E_MAX_DATA_PER_TXD_ALIGNED;
2221 stale_size -= I40E_MAX_DATA_PER_TXD_ALIGNED;
2222 } while (stale_size > I40E_MAX_DATA_PER_TXD);
2223 }
2224
2225
2226 if (sum < 0)
2227 return true;
2228
2229 if (!nr_frags--)
2230 break;
2231
2232 sum -= stale_size;
2233 }
2234
2235 return false;
2236}
2237
2238
2239
2240
2241
2242
2243
2244
2245int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2246{
2247 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2248
2249 smp_mb();
2250
2251
2252 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2253 return -EBUSY;
2254
2255
2256 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2257 ++tx_ring->tx_stats.restart_queue;
2258 return 0;
2259}
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
2272 struct i40e_tx_buffer *first, u32 tx_flags,
2273 const u8 hdr_len, u32 td_cmd, u32 td_offset)
2274{
2275 unsigned int data_len = skb->data_len;
2276 unsigned int size = skb_headlen(skb);
2277 struct skb_frag_struct *frag;
2278 struct i40e_tx_buffer *tx_bi;
2279 struct i40e_tx_desc *tx_desc;
2280 u16 i = tx_ring->next_to_use;
2281 u32 td_tag = 0;
2282 dma_addr_t dma;
2283
2284 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
2285 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2286 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2287 I40E_TX_FLAGS_VLAN_SHIFT;
2288 }
2289
2290 first->tx_flags = tx_flags;
2291
2292 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2293
2294 tx_desc = I40E_TX_DESC(tx_ring, i);
2295 tx_bi = first;
2296
2297 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2298 unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2299
2300 if (dma_mapping_error(tx_ring->dev, dma))
2301 goto dma_error;
2302
2303
2304 dma_unmap_len_set(tx_bi, len, size);
2305 dma_unmap_addr_set(tx_bi, dma, dma);
2306
2307
2308 max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1);
2309 tx_desc->buffer_addr = cpu_to_le64(dma);
2310
2311 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
2312 tx_desc->cmd_type_offset_bsz =
2313 build_ctob(td_cmd, td_offset,
2314 max_data, td_tag);
2315
2316 tx_desc++;
2317 i++;
2318
2319 if (i == tx_ring->count) {
2320 tx_desc = I40E_TX_DESC(tx_ring, 0);
2321 i = 0;
2322 }
2323
2324 dma += max_data;
2325 size -= max_data;
2326
2327 max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2328 tx_desc->buffer_addr = cpu_to_le64(dma);
2329 }
2330
2331 if (likely(!data_len))
2332 break;
2333
2334 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2335 size, td_tag);
2336
2337 tx_desc++;
2338 i++;
2339
2340 if (i == tx_ring->count) {
2341 tx_desc = I40E_TX_DESC(tx_ring, 0);
2342 i = 0;
2343 }
2344
2345 size = skb_frag_size(frag);
2346 data_len -= size;
2347
2348 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2349 DMA_TO_DEVICE);
2350
2351 tx_bi = &tx_ring->tx_bi[i];
2352 }
2353
2354 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
2355
2356 i++;
2357 if (i == tx_ring->count)
2358 i = 0;
2359
2360 tx_ring->next_to_use = i;
2361
2362 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2363
2364
2365 td_cmd |= I40E_TXD_CMD;
2366 tx_desc->cmd_type_offset_bsz =
2367 build_ctob(td_cmd, td_offset, size, td_tag);
2368
2369
2370
2371
2372
2373
2374
2375 wmb();
2376
2377
2378 first->next_to_watch = tx_desc;
2379
2380
2381 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
2382 writel(i, tx_ring->tail);
2383
2384
2385
2386
2387 mmiowb();
2388 }
2389
2390 return;
2391
2392dma_error:
2393 dev_info(tx_ring->dev, "TX DMA map failed\n");
2394
2395
2396 for (;;) {
2397 tx_bi = &tx_ring->tx_bi[i];
2398 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
2399 if (tx_bi == first)
2400 break;
2401 if (i == 0)
2402 i = tx_ring->count;
2403 i--;
2404 }
2405
2406 tx_ring->next_to_use = i;
2407}
2408
2409
2410
2411
2412
2413
2414
2415
2416static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2417 struct i40e_ring *tx_ring)
2418{
2419 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2420 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2421 struct i40e_tx_buffer *first;
2422 u32 td_offset = 0;
2423 u32 tx_flags = 0;
2424 __be16 protocol;
2425 u32 td_cmd = 0;
2426 u8 hdr_len = 0;
2427 int tso, count;
2428
2429
2430 prefetch(skb->data);
2431
2432 i40e_trace(xmit_frame_ring, skb, tx_ring);
2433
2434 count = i40e_xmit_descriptor_count(skb);
2435 if (i40e_chk_linearize(skb, count)) {
2436 if (__skb_linearize(skb)) {
2437 dev_kfree_skb_any(skb);
2438 return NETDEV_TX_OK;
2439 }
2440 count = i40e_txd_use_count(skb->len);
2441 tx_ring->tx_stats.tx_linearize++;
2442 }
2443
2444
2445
2446
2447
2448
2449
2450 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
2451 tx_ring->tx_stats.tx_busy++;
2452 return NETDEV_TX_BUSY;
2453 }
2454
2455
2456 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2457 first->skb = skb;
2458 first->bytecount = skb->len;
2459 first->gso_segs = 1;
2460
2461
2462 if (i40evf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2463 goto out_drop;
2464
2465
2466 protocol = vlan_get_protocol(skb);
2467
2468
2469 if (protocol == htons(ETH_P_IP))
2470 tx_flags |= I40E_TX_FLAGS_IPV4;
2471 else if (protocol == htons(ETH_P_IPV6))
2472 tx_flags |= I40E_TX_FLAGS_IPV6;
2473
2474 tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss);
2475
2476 if (tso < 0)
2477 goto out_drop;
2478 else if (tso)
2479 tx_flags |= I40E_TX_FLAGS_TSO;
2480
2481
2482 tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
2483 tx_ring, &cd_tunneling);
2484 if (tso < 0)
2485 goto out_drop;
2486
2487 skb_tx_timestamp(skb);
2488
2489
2490 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2491
2492 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2493 cd_tunneling, cd_l2tag2);
2494
2495 i40evf_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2496 td_cmd, td_offset);
2497
2498 return NETDEV_TX_OK;
2499
2500out_drop:
2501 i40e_trace(xmit_frame_ring_drop, first->skb, tx_ring);
2502 dev_kfree_skb_any(first->skb);
2503 first->skb = NULL;
2504 return NETDEV_TX_OK;
2505}
2506
2507
2508
2509
2510
2511
2512
2513
2514netdev_tx_t i40evf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2515{
2516 struct i40evf_adapter *adapter = netdev_priv(netdev);
2517 struct i40e_ring *tx_ring = &adapter->tx_rings[skb->queue_mapping];
2518
2519
2520
2521
2522 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2523 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2524 return NETDEV_TX_OK;
2525 skb->len = I40E_MIN_TX_LEN;
2526 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2527 }
2528
2529 return i40e_xmit_frame_ring(skb, tx_ring);
2530}
2531