1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/prefetch.h>
28#include <net/busy_poll.h>
29
30#include "i40evf.h"
31#include "i40e_trace.h"
32#include "i40e_prototype.h"
33
34static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
35 u32 td_tag)
36{
37 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
38 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
39 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
40 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
41 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
42}
43
44#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
45
46
47
48
49
50
51static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
52 struct i40e_tx_buffer *tx_buffer)
53{
54 if (tx_buffer->skb) {
55 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
56 kfree(tx_buffer->raw_buf);
57 else
58 dev_kfree_skb_any(tx_buffer->skb);
59 if (dma_unmap_len(tx_buffer, len))
60 dma_unmap_single(ring->dev,
61 dma_unmap_addr(tx_buffer, dma),
62 dma_unmap_len(tx_buffer, len),
63 DMA_TO_DEVICE);
64 } else if (dma_unmap_len(tx_buffer, len)) {
65 dma_unmap_page(ring->dev,
66 dma_unmap_addr(tx_buffer, dma),
67 dma_unmap_len(tx_buffer, len),
68 DMA_TO_DEVICE);
69 }
70
71 tx_buffer->next_to_watch = NULL;
72 tx_buffer->skb = NULL;
73 dma_unmap_len_set(tx_buffer, len, 0);
74
75}
76
77
78
79
80
81void i40evf_clean_tx_ring(struct i40e_ring *tx_ring)
82{
83 unsigned long bi_size;
84 u16 i;
85
86
87 if (!tx_ring->tx_bi)
88 return;
89
90
91 for (i = 0; i < tx_ring->count; i++)
92 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
93
94 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
95 memset(tx_ring->tx_bi, 0, bi_size);
96
97
98 memset(tx_ring->desc, 0, tx_ring->size);
99
100 tx_ring->next_to_use = 0;
101 tx_ring->next_to_clean = 0;
102
103 if (!tx_ring->netdev)
104 return;
105
106
107 netdev_tx_reset_queue(txring_txq(tx_ring));
108}
109
110
111
112
113
114
115
116void i40evf_free_tx_resources(struct i40e_ring *tx_ring)
117{
118 i40evf_clean_tx_ring(tx_ring);
119 kfree(tx_ring->tx_bi);
120 tx_ring->tx_bi = NULL;
121
122 if (tx_ring->desc) {
123 dma_free_coherent(tx_ring->dev, tx_ring->size,
124 tx_ring->desc, tx_ring->dma);
125 tx_ring->desc = NULL;
126 }
127}
128
129
130
131
132
133
134
135
136
137u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw)
138{
139 u32 head, tail;
140
141 head = ring->next_to_clean;
142 tail = readl(ring->tail);
143
144 if (head != tail)
145 return (head < tail) ?
146 tail - head : (tail + ring->count - head);
147
148 return 0;
149}
150
151#define WB_STRIDE 4
152
153
154
155
156
157
158
159
160
161static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
162 struct i40e_ring *tx_ring, int napi_budget)
163{
164 u16 i = tx_ring->next_to_clean;
165 struct i40e_tx_buffer *tx_buf;
166 struct i40e_tx_desc *tx_desc;
167 unsigned int total_bytes = 0, total_packets = 0;
168 unsigned int budget = vsi->work_limit;
169
170 tx_buf = &tx_ring->tx_bi[i];
171 tx_desc = I40E_TX_DESC(tx_ring, i);
172 i -= tx_ring->count;
173
174 do {
175 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
176
177
178 if (!eop_desc)
179 break;
180
181
182 smp_rmb();
183
184 i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf);
185
186 if (!(eop_desc->cmd_type_offset_bsz &
187 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
188 break;
189
190
191 tx_buf->next_to_watch = NULL;
192
193
194 total_bytes += tx_buf->bytecount;
195 total_packets += tx_buf->gso_segs;
196
197
198 napi_consume_skb(tx_buf->skb, napi_budget);
199
200
201 dma_unmap_single(tx_ring->dev,
202 dma_unmap_addr(tx_buf, dma),
203 dma_unmap_len(tx_buf, len),
204 DMA_TO_DEVICE);
205
206
207 tx_buf->skb = NULL;
208 dma_unmap_len_set(tx_buf, len, 0);
209
210
211 while (tx_desc != eop_desc) {
212 i40e_trace(clean_tx_irq_unmap,
213 tx_ring, tx_desc, tx_buf);
214
215 tx_buf++;
216 tx_desc++;
217 i++;
218 if (unlikely(!i)) {
219 i -= tx_ring->count;
220 tx_buf = tx_ring->tx_bi;
221 tx_desc = I40E_TX_DESC(tx_ring, 0);
222 }
223
224
225 if (dma_unmap_len(tx_buf, len)) {
226 dma_unmap_page(tx_ring->dev,
227 dma_unmap_addr(tx_buf, dma),
228 dma_unmap_len(tx_buf, len),
229 DMA_TO_DEVICE);
230 dma_unmap_len_set(tx_buf, len, 0);
231 }
232 }
233
234
235 tx_buf++;
236 tx_desc++;
237 i++;
238 if (unlikely(!i)) {
239 i -= tx_ring->count;
240 tx_buf = tx_ring->tx_bi;
241 tx_desc = I40E_TX_DESC(tx_ring, 0);
242 }
243
244 prefetch(tx_desc);
245
246
247 budget--;
248 } while (likely(budget));
249
250 i += tx_ring->count;
251 tx_ring->next_to_clean = i;
252 u64_stats_update_begin(&tx_ring->syncp);
253 tx_ring->stats.bytes += total_bytes;
254 tx_ring->stats.packets += total_packets;
255 u64_stats_update_end(&tx_ring->syncp);
256 tx_ring->q_vector->tx.total_bytes += total_bytes;
257 tx_ring->q_vector->tx.total_packets += total_packets;
258
259 if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {
260
261
262
263
264
265 unsigned int j = i40evf_get_tx_pending(tx_ring, false);
266
267 if (budget &&
268 ((j / WB_STRIDE) == 0) && (j > 0) &&
269 !test_bit(__I40E_VSI_DOWN, vsi->state) &&
270 (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))
271 tx_ring->arm_wb = true;
272 }
273
274
275 netdev_tx_completed_queue(txring_txq(tx_ring),
276 total_packets, total_bytes);
277
278#define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))
279 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
280 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
281
282
283
284 smp_mb();
285 if (__netif_subqueue_stopped(tx_ring->netdev,
286 tx_ring->queue_index) &&
287 !test_bit(__I40E_VSI_DOWN, vsi->state)) {
288 netif_wake_subqueue(tx_ring->netdev,
289 tx_ring->queue_index);
290 ++tx_ring->tx_stats.restart_queue;
291 }
292 }
293
294 return !!budget;
295}
296
297
298
299
300
301
302
303static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
304 struct i40e_q_vector *q_vector)
305{
306 u16 flags = q_vector->tx.ring[0].flags;
307 u32 val;
308
309 if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR))
310 return;
311
312 if (q_vector->arm_wb_state)
313 return;
314
315 val = I40E_VFINT_DYN_CTLN1_WB_ON_ITR_MASK |
316 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK;
317
318 wr32(&vsi->back->hw,
319 I40E_VFINT_DYN_CTLN1(q_vector->v_idx +
320 vsi->base_vector - 1), val);
321 q_vector->arm_wb_state = true;
322}
323
324
325
326
327
328
329
330void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
331{
332 u32 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
333 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK |
334 I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
335 I40E_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK
336 ;
337
338 wr32(&vsi->back->hw,
339 I40E_VFINT_DYN_CTLN1(q_vector->v_idx + vsi->base_vector - 1),
340 val);
341}
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
358{
359 enum i40e_latency_range new_latency_range = rc->latency_range;
360 u32 new_itr = rc->itr;
361 int bytes_per_usec;
362 unsigned int usecs, estimated_usecs;
363
364 if (rc->total_packets == 0 || !rc->itr)
365 return false;
366
367 usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
368 bytes_per_usec = rc->total_bytes / usecs;
369
370
371
372
373
374
375 estimated_usecs = jiffies_to_usecs(jiffies - rc->last_itr_update);
376 if (estimated_usecs > usecs) {
377 new_latency_range = I40E_LOW_LATENCY;
378 goto reset_latency;
379 }
380
381
382
383
384
385
386
387
388
389
390
391
392 switch (new_latency_range) {
393 case I40E_LOWEST_LATENCY:
394 if (bytes_per_usec > 10)
395 new_latency_range = I40E_LOW_LATENCY;
396 break;
397 case I40E_LOW_LATENCY:
398 if (bytes_per_usec > 20)
399 new_latency_range = I40E_BULK_LATENCY;
400 else if (bytes_per_usec <= 10)
401 new_latency_range = I40E_LOWEST_LATENCY;
402 break;
403 case I40E_BULK_LATENCY:
404 default:
405 if (bytes_per_usec <= 20)
406 new_latency_range = I40E_LOW_LATENCY;
407 break;
408 }
409
410reset_latency:
411 rc->latency_range = new_latency_range;
412
413 switch (new_latency_range) {
414 case I40E_LOWEST_LATENCY:
415 new_itr = I40E_ITR_50K;
416 break;
417 case I40E_LOW_LATENCY:
418 new_itr = I40E_ITR_20K;
419 break;
420 case I40E_BULK_LATENCY:
421 new_itr = I40E_ITR_18K;
422 break;
423 default:
424 break;
425 }
426
427 rc->total_bytes = 0;
428 rc->total_packets = 0;
429 rc->last_itr_update = jiffies;
430
431 if (new_itr != rc->itr) {
432 rc->itr = new_itr;
433 return true;
434 }
435 return false;
436}
437
438
439
440
441
442
443
444int i40evf_setup_tx_descriptors(struct i40e_ring *tx_ring)
445{
446 struct device *dev = tx_ring->dev;
447 int bi_size;
448
449 if (!dev)
450 return -ENOMEM;
451
452
453 WARN_ON(tx_ring->tx_bi);
454 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
455 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
456 if (!tx_ring->tx_bi)
457 goto err;
458
459
460 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
461 tx_ring->size = ALIGN(tx_ring->size, 4096);
462 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
463 &tx_ring->dma, GFP_KERNEL);
464 if (!tx_ring->desc) {
465 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
466 tx_ring->size);
467 goto err;
468 }
469
470 tx_ring->next_to_use = 0;
471 tx_ring->next_to_clean = 0;
472 return 0;
473
474err:
475 kfree(tx_ring->tx_bi);
476 tx_ring->tx_bi = NULL;
477 return -ENOMEM;
478}
479
480
481
482
483
484void i40evf_clean_rx_ring(struct i40e_ring *rx_ring)
485{
486 unsigned long bi_size;
487 u16 i;
488
489
490 if (!rx_ring->rx_bi)
491 return;
492
493 if (rx_ring->skb) {
494 dev_kfree_skb(rx_ring->skb);
495 rx_ring->skb = NULL;
496 }
497
498
499 for (i = 0; i < rx_ring->count; i++) {
500 struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
501
502 if (!rx_bi->page)
503 continue;
504
505
506
507
508 dma_sync_single_range_for_cpu(rx_ring->dev,
509 rx_bi->dma,
510 rx_bi->page_offset,
511 rx_ring->rx_buf_len,
512 DMA_FROM_DEVICE);
513
514
515 dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma,
516 i40e_rx_pg_size(rx_ring),
517 DMA_FROM_DEVICE,
518 I40E_RX_DMA_ATTR);
519
520 __page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias);
521
522 rx_bi->page = NULL;
523 rx_bi->page_offset = 0;
524 }
525
526 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
527 memset(rx_ring->rx_bi, 0, bi_size);
528
529
530 memset(rx_ring->desc, 0, rx_ring->size);
531
532 rx_ring->next_to_alloc = 0;
533 rx_ring->next_to_clean = 0;
534 rx_ring->next_to_use = 0;
535}
536
537
538
539
540
541
542
543void i40evf_free_rx_resources(struct i40e_ring *rx_ring)
544{
545 i40evf_clean_rx_ring(rx_ring);
546 kfree(rx_ring->rx_bi);
547 rx_ring->rx_bi = NULL;
548
549 if (rx_ring->desc) {
550 dma_free_coherent(rx_ring->dev, rx_ring->size,
551 rx_ring->desc, rx_ring->dma);
552 rx_ring->desc = NULL;
553 }
554}
555
556
557
558
559
560
561
562int i40evf_setup_rx_descriptors(struct i40e_ring *rx_ring)
563{
564 struct device *dev = rx_ring->dev;
565 int bi_size;
566
567
568 WARN_ON(rx_ring->rx_bi);
569 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
570 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
571 if (!rx_ring->rx_bi)
572 goto err;
573
574 u64_stats_init(&rx_ring->syncp);
575
576
577 rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc);
578 rx_ring->size = ALIGN(rx_ring->size, 4096);
579 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
580 &rx_ring->dma, GFP_KERNEL);
581
582 if (!rx_ring->desc) {
583 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
584 rx_ring->size);
585 goto err;
586 }
587
588 rx_ring->next_to_alloc = 0;
589 rx_ring->next_to_clean = 0;
590 rx_ring->next_to_use = 0;
591
592 return 0;
593err:
594 kfree(rx_ring->rx_bi);
595 rx_ring->rx_bi = NULL;
596 return -ENOMEM;
597}
598
599
600
601
602
603
604static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
605{
606 rx_ring->next_to_use = val;
607
608
609 rx_ring->next_to_alloc = val;
610
611
612
613
614
615
616 wmb();
617 writel(val, rx_ring->tail);
618}
619
620
621
622
623
624
625
626static inline unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
627{
628 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
629}
630
631
632
633
634
635
636
637
638
639static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring,
640 struct i40e_rx_buffer *bi)
641{
642 struct page *page = bi->page;
643 dma_addr_t dma;
644
645
646 if (likely(page)) {
647 rx_ring->rx_stats.page_reuse_count++;
648 return true;
649 }
650
651
652 page = dev_alloc_pages(i40e_rx_pg_order(rx_ring));
653 if (unlikely(!page)) {
654 rx_ring->rx_stats.alloc_page_failed++;
655 return false;
656 }
657
658
659 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
660 i40e_rx_pg_size(rx_ring),
661 DMA_FROM_DEVICE,
662 I40E_RX_DMA_ATTR);
663
664
665
666
667 if (dma_mapping_error(rx_ring->dev, dma)) {
668 __free_pages(page, i40e_rx_pg_order(rx_ring));
669 rx_ring->rx_stats.alloc_page_failed++;
670 return false;
671 }
672
673 bi->dma = dma;
674 bi->page = page;
675 bi->page_offset = i40e_rx_offset(rx_ring);
676
677
678 bi->pagecnt_bias = 1;
679
680 return true;
681}
682
683
684
685
686
687
688
689static void i40e_receive_skb(struct i40e_ring *rx_ring,
690 struct sk_buff *skb, u16 vlan_tag)
691{
692 struct i40e_q_vector *q_vector = rx_ring->q_vector;
693
694 if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
695 (vlan_tag & VLAN_VID_MASK))
696 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
697
698 napi_gro_receive(&q_vector->napi, skb);
699}
700
701
702
703
704
705
706
707
708bool i40evf_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
709{
710 u16 ntu = rx_ring->next_to_use;
711 union i40e_rx_desc *rx_desc;
712 struct i40e_rx_buffer *bi;
713
714
715 if (!rx_ring->netdev || !cleaned_count)
716 return false;
717
718 rx_desc = I40E_RX_DESC(rx_ring, ntu);
719 bi = &rx_ring->rx_bi[ntu];
720
721 do {
722 if (!i40e_alloc_mapped_page(rx_ring, bi))
723 goto no_buffers;
724
725
726 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
727 bi->page_offset,
728 rx_ring->rx_buf_len,
729 DMA_FROM_DEVICE);
730
731
732
733
734 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
735
736 rx_desc++;
737 bi++;
738 ntu++;
739 if (unlikely(ntu == rx_ring->count)) {
740 rx_desc = I40E_RX_DESC(rx_ring, 0);
741 bi = rx_ring->rx_bi;
742 ntu = 0;
743 }
744
745
746 rx_desc->wb.qword1.status_error_len = 0;
747
748 cleaned_count--;
749 } while (cleaned_count);
750
751 if (rx_ring->next_to_use != ntu)
752 i40e_release_rx_desc(rx_ring, ntu);
753
754 return false;
755
756no_buffers:
757 if (rx_ring->next_to_use != ntu)
758 i40e_release_rx_desc(rx_ring, ntu);
759
760
761
762
763 return true;
764}
765
766
767
768
769
770
771
772static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
773 struct sk_buff *skb,
774 union i40e_rx_desc *rx_desc)
775{
776 struct i40e_rx_ptype_decoded decoded;
777 u32 rx_error, rx_status;
778 bool ipv4, ipv6;
779 u8 ptype;
780 u64 qword;
781
782 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
783 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT;
784 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
785 I40E_RXD_QW1_ERROR_SHIFT;
786 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
787 I40E_RXD_QW1_STATUS_SHIFT;
788 decoded = decode_rx_desc_ptype(ptype);
789
790 skb->ip_summed = CHECKSUM_NONE;
791
792 skb_checksum_none_assert(skb);
793
794
795 if (!(vsi->netdev->features & NETIF_F_RXCSUM))
796 return;
797
798
799 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
800 return;
801
802
803 if (!(decoded.known && decoded.outer_ip))
804 return;
805
806 ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
807 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4);
808 ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
809 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6);
810
811 if (ipv4 &&
812 (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) |
813 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT))))
814 goto checksum_fail;
815
816
817 if (ipv6 &&
818 rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
819
820 return;
821
822
823 if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT))
824 goto checksum_fail;
825
826
827
828
829
830 if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT))
831 return;
832
833
834 switch (decoded.inner_prot) {
835 case I40E_RX_PTYPE_INNER_PROT_TCP:
836 case I40E_RX_PTYPE_INNER_PROT_UDP:
837 case I40E_RX_PTYPE_INNER_PROT_SCTP:
838 skb->ip_summed = CHECKSUM_UNNECESSARY;
839
840 default:
841 break;
842 }
843
844 return;
845
846checksum_fail:
847 vsi->back->hw_csum_rx_error++;
848}
849
850
851
852
853
854
855
856static inline int i40e_ptype_to_htype(u8 ptype)
857{
858 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
859
860 if (!decoded.known)
861 return PKT_HASH_TYPE_NONE;
862
863 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
864 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
865 return PKT_HASH_TYPE_L4;
866 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
867 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
868 return PKT_HASH_TYPE_L3;
869 else
870 return PKT_HASH_TYPE_L2;
871}
872
873
874
875
876
877
878static inline void i40e_rx_hash(struct i40e_ring *ring,
879 union i40e_rx_desc *rx_desc,
880 struct sk_buff *skb,
881 u8 rx_ptype)
882{
883 u32 hash;
884 const __le64 rss_mask =
885 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
886 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
887
888 if (ring->netdev->features & NETIF_F_RXHASH)
889 return;
890
891 if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
892 hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
893 skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype));
894 }
895}
896
897
898
899
900
901
902
903
904
905
906
907
908static inline
909void i40evf_process_skb_fields(struct i40e_ring *rx_ring,
910 union i40e_rx_desc *rx_desc, struct sk_buff *skb,
911 u8 rx_ptype)
912{
913 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
914
915 i40e_rx_checksum(rx_ring->vsi, skb, rx_desc);
916
917 skb_record_rx_queue(skb, rx_ring->queue_index);
918
919
920 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
921}
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb)
937{
938
939 if (eth_skb_pad(skb))
940 return true;
941
942 return false;
943}
944
945
946
947
948
949
950
951
952static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
953 struct i40e_rx_buffer *old_buff)
954{
955 struct i40e_rx_buffer *new_buff;
956 u16 nta = rx_ring->next_to_alloc;
957
958 new_buff = &rx_ring->rx_bi[nta];
959
960
961 nta++;
962 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
963
964
965 new_buff->dma = old_buff->dma;
966 new_buff->page = old_buff->page;
967 new_buff->page_offset = old_buff->page_offset;
968 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
969}
970
971
972
973
974
975
976
977
978static inline bool i40e_page_is_reusable(struct page *page)
979{
980 return (page_to_nid(page) == numa_mem_id()) &&
981 !page_is_pfmemalloc(page);
982}
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer)
1012{
1013 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1014 struct page *page = rx_buffer->page;
1015
1016
1017 if (unlikely(!i40e_page_is_reusable(page)))
1018 return false;
1019
1020#if (PAGE_SIZE < 8192)
1021
1022 if (unlikely((page_count(page) - pagecnt_bias) > 1))
1023 return false;
1024#else
1025#define I40E_LAST_OFFSET \
1026 (SKB_WITH_OVERHEAD(PAGE_SIZE) - I40E_RXBUFFER_2048)
1027 if (rx_buffer->page_offset > I40E_LAST_OFFSET)
1028 return false;
1029#endif
1030
1031
1032
1033
1034
1035 if (unlikely(!pagecnt_bias)) {
1036 page_ref_add(page, USHRT_MAX);
1037 rx_buffer->pagecnt_bias = USHRT_MAX;
1038 }
1039
1040 return true;
1041}
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
1056 struct i40e_rx_buffer *rx_buffer,
1057 struct sk_buff *skb,
1058 unsigned int size)
1059{
1060#if (PAGE_SIZE < 8192)
1061 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1062#else
1063 unsigned int truesize = SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring));
1064#endif
1065
1066 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1067 rx_buffer->page_offset, size, truesize);
1068
1069
1070#if (PAGE_SIZE < 8192)
1071 rx_buffer->page_offset ^= truesize;
1072#else
1073 rx_buffer->page_offset += truesize;
1074#endif
1075}
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
1086 const unsigned int size)
1087{
1088 struct i40e_rx_buffer *rx_buffer;
1089
1090 rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
1091 prefetchw(rx_buffer->page);
1092
1093
1094 dma_sync_single_range_for_cpu(rx_ring->dev,
1095 rx_buffer->dma,
1096 rx_buffer->page_offset,
1097 size,
1098 DMA_FROM_DEVICE);
1099
1100
1101 rx_buffer->pagecnt_bias--;
1102
1103 return rx_buffer;
1104}
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
1117 struct i40e_rx_buffer *rx_buffer,
1118 unsigned int size)
1119{
1120 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1121#if (PAGE_SIZE < 8192)
1122 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1123#else
1124 unsigned int truesize = SKB_DATA_ALIGN(size);
1125#endif
1126 unsigned int headlen;
1127 struct sk_buff *skb;
1128
1129
1130 prefetch(va);
1131#if L1_CACHE_BYTES < 128
1132 prefetch(va + L1_CACHE_BYTES);
1133#endif
1134
1135
1136 skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
1137 I40E_RX_HDR_SIZE,
1138 GFP_ATOMIC | __GFP_NOWARN);
1139 if (unlikely(!skb))
1140 return NULL;
1141
1142
1143 headlen = size;
1144 if (headlen > I40E_RX_HDR_SIZE)
1145 headlen = eth_get_headlen(va, I40E_RX_HDR_SIZE);
1146
1147
1148 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
1149
1150
1151 size -= headlen;
1152 if (size) {
1153 skb_add_rx_frag(skb, 0, rx_buffer->page,
1154 rx_buffer->page_offset + headlen,
1155 size, truesize);
1156
1157
1158#if (PAGE_SIZE < 8192)
1159 rx_buffer->page_offset ^= truesize;
1160#else
1161 rx_buffer->page_offset += truesize;
1162#endif
1163 } else {
1164
1165 rx_buffer->pagecnt_bias++;
1166 }
1167
1168 return skb;
1169}
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
1181 struct i40e_rx_buffer *rx_buffer,
1182 unsigned int size)
1183{
1184 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1185#if (PAGE_SIZE < 8192)
1186 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1187#else
1188 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1189 SKB_DATA_ALIGN(I40E_SKB_PAD + size);
1190#endif
1191 struct sk_buff *skb;
1192
1193
1194 prefetch(va);
1195#if L1_CACHE_BYTES < 128
1196 prefetch(va + L1_CACHE_BYTES);
1197#endif
1198
1199 skb = build_skb(va - I40E_SKB_PAD, truesize);
1200 if (unlikely(!skb))
1201 return NULL;
1202
1203
1204 skb_reserve(skb, I40E_SKB_PAD);
1205 __skb_put(skb, size);
1206
1207
1208#if (PAGE_SIZE < 8192)
1209 rx_buffer->page_offset ^= truesize;
1210#else
1211 rx_buffer->page_offset += truesize;
1212#endif
1213
1214 return skb;
1215}
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
1226 struct i40e_rx_buffer *rx_buffer)
1227{
1228 if (i40e_can_reuse_rx_page(rx_buffer)) {
1229
1230 i40e_reuse_rx_page(rx_ring, rx_buffer);
1231 rx_ring->rx_stats.page_reuse_count++;
1232 } else {
1233
1234 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1235 i40e_rx_pg_size(rx_ring),
1236 DMA_FROM_DEVICE, I40E_RX_DMA_ATTR);
1237 __page_frag_cache_drain(rx_buffer->page,
1238 rx_buffer->pagecnt_bias);
1239 }
1240
1241
1242 rx_buffer->page = NULL;
1243}
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256static bool i40e_is_non_eop(struct i40e_ring *rx_ring,
1257 union i40e_rx_desc *rx_desc,
1258 struct sk_buff *skb)
1259{
1260 u32 ntc = rx_ring->next_to_clean + 1;
1261
1262
1263 ntc = (ntc < rx_ring->count) ? ntc : 0;
1264 rx_ring->next_to_clean = ntc;
1265
1266 prefetch(I40E_RX_DESC(rx_ring, ntc));
1267
1268
1269#define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)
1270 if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF)))
1271 return false;
1272
1273 rx_ring->rx_stats.non_eop_descs++;
1274
1275 return true;
1276}
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1291{
1292 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1293 struct sk_buff *skb = rx_ring->skb;
1294 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1295 bool failure = false;
1296
1297 while (likely(total_rx_packets < (unsigned int)budget)) {
1298 struct i40e_rx_buffer *rx_buffer;
1299 union i40e_rx_desc *rx_desc;
1300 unsigned int size;
1301 u16 vlan_tag;
1302 u8 rx_ptype;
1303 u64 qword;
1304
1305
1306 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1307 failure = failure ||
1308 i40evf_alloc_rx_buffers(rx_ring, cleaned_count);
1309 cleaned_count = 0;
1310 }
1311
1312 rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);
1313
1314
1315
1316
1317
1318
1319 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1320
1321
1322
1323
1324
1325 dma_rmb();
1326
1327 size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1328 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1329 if (!size)
1330 break;
1331
1332 i40e_trace(clean_rx_irq, rx_ring, rx_desc, skb);
1333 rx_buffer = i40e_get_rx_buffer(rx_ring, size);
1334
1335
1336 if (skb)
1337 i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
1338 else if (ring_uses_build_skb(rx_ring))
1339 skb = i40e_build_skb(rx_ring, rx_buffer, size);
1340 else
1341 skb = i40e_construct_skb(rx_ring, rx_buffer, size);
1342
1343
1344 if (!skb) {
1345 rx_ring->rx_stats.alloc_buff_failed++;
1346 rx_buffer->pagecnt_bias++;
1347 break;
1348 }
1349
1350 i40e_put_rx_buffer(rx_ring, rx_buffer);
1351 cleaned_count++;
1352
1353 if (i40e_is_non_eop(rx_ring, rx_desc, skb))
1354 continue;
1355
1356
1357
1358
1359
1360
1361 if (unlikely(i40e_test_staterr(rx_desc, BIT(I40E_RXD_QW1_ERROR_SHIFT)))) {
1362 dev_kfree_skb_any(skb);
1363 skb = NULL;
1364 continue;
1365 }
1366
1367 if (i40e_cleanup_headers(rx_ring, skb)) {
1368 skb = NULL;
1369 continue;
1370 }
1371
1372
1373 total_rx_bytes += skb->len;
1374
1375 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1376 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1377 I40E_RXD_QW1_PTYPE_SHIFT;
1378
1379
1380 i40evf_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
1381
1382
1383 vlan_tag = (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) ?
1384 le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0;
1385
1386 i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb);
1387 i40e_receive_skb(rx_ring, skb, vlan_tag);
1388 skb = NULL;
1389
1390
1391 total_rx_packets++;
1392 }
1393
1394 rx_ring->skb = skb;
1395
1396 u64_stats_update_begin(&rx_ring->syncp);
1397 rx_ring->stats.packets += total_rx_packets;
1398 rx_ring->stats.bytes += total_rx_bytes;
1399 u64_stats_update_end(&rx_ring->syncp);
1400 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1401 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1402
1403
1404 return failure ? budget : (int)total_rx_packets;
1405}
1406
1407static u32 i40e_buildreg_itr(const int type, const u16 itr)
1408{
1409 u32 val;
1410
1411 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1412 I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK |
1413 (type << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1414 (itr << I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT);
1415
1416 return val;
1417}
1418
1419
1420#define INTREG I40E_VFINT_DYN_CTLN1
1421static inline int get_rx_itr(struct i40e_vsi *vsi, int idx)
1422{
1423 struct i40evf_adapter *adapter = vsi->back;
1424
1425 return adapter->rx_rings[idx].rx_itr_setting;
1426}
1427
1428static inline int get_tx_itr(struct i40e_vsi *vsi, int idx)
1429{
1430 struct i40evf_adapter *adapter = vsi->back;
1431
1432 return adapter->tx_rings[idx].tx_itr_setting;
1433}
1434
1435
1436
1437
1438
1439
1440
1441static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
1442 struct i40e_q_vector *q_vector)
1443{
1444 struct i40e_hw *hw = &vsi->back->hw;
1445 bool rx = false, tx = false;
1446 u32 rxval, txval;
1447 int vector;
1448 int idx = q_vector->v_idx;
1449 int rx_itr_setting, tx_itr_setting;
1450
1451 vector = (q_vector->v_idx + vsi->base_vector);
1452
1453
1454
1455
1456 rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
1457
1458 rx_itr_setting = get_rx_itr(vsi, idx);
1459 tx_itr_setting = get_tx_itr(vsi, idx);
1460
1461 if (q_vector->itr_countdown > 0 ||
1462 (!ITR_IS_DYNAMIC(rx_itr_setting) &&
1463 !ITR_IS_DYNAMIC(tx_itr_setting))) {
1464 goto enable_int;
1465 }
1466
1467 if (ITR_IS_DYNAMIC(rx_itr_setting)) {
1468 rx = i40e_set_new_dynamic_itr(&q_vector->rx);
1469 rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
1470 }
1471
1472 if (ITR_IS_DYNAMIC(tx_itr_setting)) {
1473 tx = i40e_set_new_dynamic_itr(&q_vector->tx);
1474 txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr);
1475 }
1476
1477 if (rx || tx) {
1478
1479
1480
1481
1482 u16 itr = max(q_vector->tx.itr, q_vector->rx.itr);
1483
1484 q_vector->tx.itr = q_vector->rx.itr = itr;
1485 txval = i40e_buildreg_itr(I40E_TX_ITR, itr);
1486 tx = true;
1487 rxval = i40e_buildreg_itr(I40E_RX_ITR, itr);
1488 rx = true;
1489 }
1490
1491
1492
1493
1494 if (rx) {
1495
1496
1497
1498
1499 rxval |= BIT(31);
1500
1501 wr32(hw, INTREG(vector - 1), rxval);
1502 }
1503
1504enable_int:
1505 if (!test_bit(__I40E_VSI_DOWN, vsi->state))
1506 wr32(hw, INTREG(vector - 1), txval);
1507
1508 if (q_vector->itr_countdown)
1509 q_vector->itr_countdown--;
1510 else
1511 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1512}
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523int i40evf_napi_poll(struct napi_struct *napi, int budget)
1524{
1525 struct i40e_q_vector *q_vector =
1526 container_of(napi, struct i40e_q_vector, napi);
1527 struct i40e_vsi *vsi = q_vector->vsi;
1528 struct i40e_ring *ring;
1529 bool clean_complete = true;
1530 bool arm_wb = false;
1531 int budget_per_ring;
1532 int work_done = 0;
1533
1534 if (test_bit(__I40E_VSI_DOWN, vsi->state)) {
1535 napi_complete(napi);
1536 return 0;
1537 }
1538
1539
1540
1541
1542 i40e_for_each_ring(ring, q_vector->tx) {
1543 if (!i40e_clean_tx_irq(vsi, ring, budget)) {
1544 clean_complete = false;
1545 continue;
1546 }
1547 arm_wb |= ring->arm_wb;
1548 ring->arm_wb = false;
1549 }
1550
1551
1552 if (budget <= 0)
1553 goto tx_only;
1554
1555
1556
1557
1558 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1559
1560 i40e_for_each_ring(ring, q_vector->rx) {
1561 int cleaned = i40e_clean_rx_irq(ring, budget_per_ring);
1562
1563 work_done += cleaned;
1564
1565 if (cleaned >= budget_per_ring)
1566 clean_complete = false;
1567 }
1568
1569
1570 if (!clean_complete) {
1571 int cpu_id = smp_processor_id();
1572
1573
1574
1575
1576
1577
1578
1579
1580 if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) {
1581
1582 napi_complete_done(napi, work_done);
1583
1584
1585 i40evf_force_wb(vsi, q_vector);
1586
1587
1588 return budget - 1;
1589 }
1590tx_only:
1591 if (arm_wb) {
1592 q_vector->tx.ring[0].tx_stats.tx_force_wb++;
1593 i40e_enable_wb_on_itr(vsi, q_vector);
1594 }
1595 return budget;
1596 }
1597
1598 if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
1599 q_vector->arm_wb_state = false;
1600
1601
1602 napi_complete_done(napi, work_done);
1603
1604 i40e_update_enable_itr(vsi, q_vector);
1605
1606 return min(work_done, budget - 1);
1607}
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621static inline int i40evf_tx_prepare_vlan_flags(struct sk_buff *skb,
1622 struct i40e_ring *tx_ring,
1623 u32 *flags)
1624{
1625 __be16 protocol = skb->protocol;
1626 u32 tx_flags = 0;
1627
1628 if (protocol == htons(ETH_P_8021Q) &&
1629 !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
1630
1631
1632
1633
1634
1635
1636
1637 skb->protocol = vlan_get_protocol(skb);
1638 goto out;
1639 }
1640
1641
1642 if (skb_vlan_tag_present(skb)) {
1643 tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1644 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1645
1646 } else if (protocol == htons(ETH_P_8021Q)) {
1647 struct vlan_hdr *vhdr, _vhdr;
1648
1649 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1650 if (!vhdr)
1651 return -EINVAL;
1652
1653 protocol = vhdr->h_vlan_encapsulated_proto;
1654 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1655 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1656 }
1657
1658out:
1659 *flags = tx_flags;
1660 return 0;
1661}
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len,
1672 u64 *cd_type_cmd_tso_mss)
1673{
1674 struct sk_buff *skb = first->skb;
1675 u64 cd_cmd, cd_tso_len, cd_mss;
1676 union {
1677 struct iphdr *v4;
1678 struct ipv6hdr *v6;
1679 unsigned char *hdr;
1680 } ip;
1681 union {
1682 struct tcphdr *tcp;
1683 struct udphdr *udp;
1684 unsigned char *hdr;
1685 } l4;
1686 u32 paylen, l4_offset;
1687 u16 gso_segs, gso_size;
1688 int err;
1689
1690 if (skb->ip_summed != CHECKSUM_PARTIAL)
1691 return 0;
1692
1693 if (!skb_is_gso(skb))
1694 return 0;
1695
1696 err = skb_cow_head(skb, 0);
1697 if (err < 0)
1698 return err;
1699
1700 ip.hdr = skb_network_header(skb);
1701 l4.hdr = skb_transport_header(skb);
1702
1703
1704 if (ip.v4->version == 4) {
1705 ip.v4->tot_len = 0;
1706 ip.v4->check = 0;
1707 } else {
1708 ip.v6->payload_len = 0;
1709 }
1710
1711 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1712 SKB_GSO_GRE_CSUM |
1713 SKB_GSO_IPXIP4 |
1714 SKB_GSO_IPXIP6 |
1715 SKB_GSO_UDP_TUNNEL |
1716 SKB_GSO_UDP_TUNNEL_CSUM)) {
1717 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1718 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
1719 l4.udp->len = 0;
1720
1721
1722 l4_offset = l4.hdr - skb->data;
1723
1724
1725 paylen = skb->len - l4_offset;
1726 csum_replace_by_diff(&l4.udp->check,
1727 (__force __wsum)htonl(paylen));
1728 }
1729
1730
1731 ip.hdr = skb_inner_network_header(skb);
1732 l4.hdr = skb_inner_transport_header(skb);
1733
1734
1735 if (ip.v4->version == 4) {
1736 ip.v4->tot_len = 0;
1737 ip.v4->check = 0;
1738 } else {
1739 ip.v6->payload_len = 0;
1740 }
1741 }
1742
1743
1744 l4_offset = l4.hdr - skb->data;
1745
1746
1747 paylen = skb->len - l4_offset;
1748 csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
1749
1750
1751 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
1752
1753
1754 gso_size = skb_shinfo(skb)->gso_size;
1755 gso_segs = skb_shinfo(skb)->gso_segs;
1756
1757
1758 first->gso_segs = gso_segs;
1759 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1760
1761
1762 cd_cmd = I40E_TX_CTX_DESC_TSO;
1763 cd_tso_len = skb->len - *hdr_len;
1764 cd_mss = gso_size;
1765 *cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1766 (cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1767 (cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1768 return 1;
1769}
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
1781 u32 *td_cmd, u32 *td_offset,
1782 struct i40e_ring *tx_ring,
1783 u32 *cd_tunneling)
1784{
1785 union {
1786 struct iphdr *v4;
1787 struct ipv6hdr *v6;
1788 unsigned char *hdr;
1789 } ip;
1790 union {
1791 struct tcphdr *tcp;
1792 struct udphdr *udp;
1793 unsigned char *hdr;
1794 } l4;
1795 unsigned char *exthdr;
1796 u32 offset, cmd = 0;
1797 __be16 frag_off;
1798 u8 l4_proto = 0;
1799
1800 if (skb->ip_summed != CHECKSUM_PARTIAL)
1801 return 0;
1802
1803 ip.hdr = skb_network_header(skb);
1804 l4.hdr = skb_transport_header(skb);
1805
1806
1807 offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1808
1809 if (skb->encapsulation) {
1810 u32 tunnel = 0;
1811
1812 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1813 tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
1814 I40E_TX_CTX_EXT_IP_IPV4 :
1815 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1816
1817 l4_proto = ip.v4->protocol;
1818 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
1819 tunnel |= I40E_TX_CTX_EXT_IP_IPV6;
1820
1821 exthdr = ip.hdr + sizeof(*ip.v6);
1822 l4_proto = ip.v6->nexthdr;
1823 if (l4.hdr != exthdr)
1824 ipv6_skip_exthdr(skb, exthdr - skb->data,
1825 &l4_proto, &frag_off);
1826 }
1827
1828
1829 switch (l4_proto) {
1830 case IPPROTO_UDP:
1831 tunnel |= I40E_TXD_CTX_UDP_TUNNELING;
1832 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
1833 break;
1834 case IPPROTO_GRE:
1835 tunnel |= I40E_TXD_CTX_GRE_TUNNELING;
1836 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
1837 break;
1838 case IPPROTO_IPIP:
1839 case IPPROTO_IPV6:
1840 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
1841 l4.hdr = skb_inner_network_header(skb);
1842 break;
1843 default:
1844 if (*tx_flags & I40E_TX_FLAGS_TSO)
1845 return -1;
1846
1847 skb_checksum_help(skb);
1848 return 0;
1849 }
1850
1851
1852 tunnel |= ((l4.hdr - ip.hdr) / 4) <<
1853 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;
1854
1855
1856 ip.hdr = skb_inner_network_header(skb);
1857
1858
1859 tunnel |= ((ip.hdr - l4.hdr) / 2) <<
1860 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1861
1862
1863 if ((*tx_flags & I40E_TX_FLAGS_TSO) &&
1864 !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1865 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
1866 tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK;
1867
1868
1869 *cd_tunneling |= tunnel;
1870
1871
1872 l4.hdr = skb_inner_transport_header(skb);
1873 l4_proto = 0;
1874
1875
1876 *tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6);
1877 if (ip.v4->version == 4)
1878 *tx_flags |= I40E_TX_FLAGS_IPV4;
1879 if (ip.v6->version == 6)
1880 *tx_flags |= I40E_TX_FLAGS_IPV6;
1881 }
1882
1883
1884 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1885 l4_proto = ip.v4->protocol;
1886
1887
1888
1889 cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
1890 I40E_TX_DESC_CMD_IIPT_IPV4_CSUM :
1891 I40E_TX_DESC_CMD_IIPT_IPV4;
1892 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
1893 cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1894
1895 exthdr = ip.hdr + sizeof(*ip.v6);
1896 l4_proto = ip.v6->nexthdr;
1897 if (l4.hdr != exthdr)
1898 ipv6_skip_exthdr(skb, exthdr - skb->data,
1899 &l4_proto, &frag_off);
1900 }
1901
1902
1903 offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1904
1905
1906 switch (l4_proto) {
1907 case IPPROTO_TCP:
1908
1909 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1910 offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1911 break;
1912 case IPPROTO_SCTP:
1913
1914 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1915 offset |= (sizeof(struct sctphdr) >> 2) <<
1916 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1917 break;
1918 case IPPROTO_UDP:
1919
1920 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1921 offset |= (sizeof(struct udphdr) >> 2) <<
1922 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1923 break;
1924 default:
1925 if (*tx_flags & I40E_TX_FLAGS_TSO)
1926 return -1;
1927 skb_checksum_help(skb);
1928 return 0;
1929 }
1930
1931 *td_cmd |= cmd;
1932 *td_offset |= offset;
1933
1934 return 1;
1935}
1936
1937
1938
1939
1940
1941
1942
1943
1944static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1945 const u64 cd_type_cmd_tso_mss,
1946 const u32 cd_tunneling, const u32 cd_l2tag2)
1947{
1948 struct i40e_tx_context_desc *context_desc;
1949 int i = tx_ring->next_to_use;
1950
1951 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
1952 !cd_tunneling && !cd_l2tag2)
1953 return;
1954
1955
1956 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1957
1958 i++;
1959 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1960
1961
1962 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
1963 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1964 context_desc->rsvd = cpu_to_le16(0);
1965 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
1966}
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981bool __i40evf_chk_linearize(struct sk_buff *skb)
1982{
1983 const struct skb_frag_struct *frag, *stale;
1984 int nr_frags, sum;
1985
1986
1987 nr_frags = skb_shinfo(skb)->nr_frags;
1988 if (nr_frags < (I40E_MAX_BUFFER_TXD - 1))
1989 return false;
1990
1991
1992
1993
1994 nr_frags -= I40E_MAX_BUFFER_TXD - 2;
1995 frag = &skb_shinfo(skb)->frags[0];
1996
1997
1998
1999
2000
2001
2002
2003 sum = 1 - skb_shinfo(skb)->gso_size;
2004
2005
2006 sum += skb_frag_size(frag++);
2007 sum += skb_frag_size(frag++);
2008 sum += skb_frag_size(frag++);
2009 sum += skb_frag_size(frag++);
2010 sum += skb_frag_size(frag++);
2011
2012
2013
2014
2015 for (stale = &skb_shinfo(skb)->frags[0];; stale++) {
2016 int stale_size = skb_frag_size(stale);
2017
2018 sum += skb_frag_size(frag++);
2019
2020
2021
2022
2023
2024
2025
2026 if (stale_size > I40E_MAX_DATA_PER_TXD) {
2027 int align_pad = -(stale->page_offset) &
2028 (I40E_MAX_READ_REQ_SIZE - 1);
2029
2030 sum -= align_pad;
2031 stale_size -= align_pad;
2032
2033 do {
2034 sum -= I40E_MAX_DATA_PER_TXD_ALIGNED;
2035 stale_size -= I40E_MAX_DATA_PER_TXD_ALIGNED;
2036 } while (stale_size > I40E_MAX_DATA_PER_TXD);
2037 }
2038
2039
2040 if (sum < 0)
2041 return true;
2042
2043 if (!nr_frags--)
2044 break;
2045
2046 sum -= stale_size;
2047 }
2048
2049 return false;
2050}
2051
2052
2053
2054
2055
2056
2057
2058
2059int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2060{
2061 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2062
2063 smp_mb();
2064
2065
2066 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2067 return -EBUSY;
2068
2069
2070 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2071 ++tx_ring->tx_stats.restart_queue;
2072 return 0;
2073}
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
2086 struct i40e_tx_buffer *first, u32 tx_flags,
2087 const u8 hdr_len, u32 td_cmd, u32 td_offset)
2088{
2089 unsigned int data_len = skb->data_len;
2090 unsigned int size = skb_headlen(skb);
2091 struct skb_frag_struct *frag;
2092 struct i40e_tx_buffer *tx_bi;
2093 struct i40e_tx_desc *tx_desc;
2094 u16 i = tx_ring->next_to_use;
2095 u32 td_tag = 0;
2096 dma_addr_t dma;
2097
2098 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
2099 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2100 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2101 I40E_TX_FLAGS_VLAN_SHIFT;
2102 }
2103
2104 first->tx_flags = tx_flags;
2105
2106 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2107
2108 tx_desc = I40E_TX_DESC(tx_ring, i);
2109 tx_bi = first;
2110
2111 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2112 unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2113
2114 if (dma_mapping_error(tx_ring->dev, dma))
2115 goto dma_error;
2116
2117
2118 dma_unmap_len_set(tx_bi, len, size);
2119 dma_unmap_addr_set(tx_bi, dma, dma);
2120
2121
2122 max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1);
2123 tx_desc->buffer_addr = cpu_to_le64(dma);
2124
2125 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
2126 tx_desc->cmd_type_offset_bsz =
2127 build_ctob(td_cmd, td_offset,
2128 max_data, td_tag);
2129
2130 tx_desc++;
2131 i++;
2132
2133 if (i == tx_ring->count) {
2134 tx_desc = I40E_TX_DESC(tx_ring, 0);
2135 i = 0;
2136 }
2137
2138 dma += max_data;
2139 size -= max_data;
2140
2141 max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2142 tx_desc->buffer_addr = cpu_to_le64(dma);
2143 }
2144
2145 if (likely(!data_len))
2146 break;
2147
2148 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2149 size, td_tag);
2150
2151 tx_desc++;
2152 i++;
2153
2154 if (i == tx_ring->count) {
2155 tx_desc = I40E_TX_DESC(tx_ring, 0);
2156 i = 0;
2157 }
2158
2159 size = skb_frag_size(frag);
2160 data_len -= size;
2161
2162 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2163 DMA_TO_DEVICE);
2164
2165 tx_bi = &tx_ring->tx_bi[i];
2166 }
2167
2168 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
2169
2170 i++;
2171 if (i == tx_ring->count)
2172 i = 0;
2173
2174 tx_ring->next_to_use = i;
2175
2176 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2177
2178
2179 td_cmd |= I40E_TXD_CMD;
2180 tx_desc->cmd_type_offset_bsz =
2181 build_ctob(td_cmd, td_offset, size, td_tag);
2182
2183
2184
2185
2186
2187
2188
2189 wmb();
2190
2191
2192 first->next_to_watch = tx_desc;
2193
2194
2195 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
2196 writel(i, tx_ring->tail);
2197
2198
2199
2200
2201 mmiowb();
2202 }
2203
2204 return;
2205
2206dma_error:
2207 dev_info(tx_ring->dev, "TX DMA map failed\n");
2208
2209
2210 for (;;) {
2211 tx_bi = &tx_ring->tx_bi[i];
2212 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
2213 if (tx_bi == first)
2214 break;
2215 if (i == 0)
2216 i = tx_ring->count;
2217 i--;
2218 }
2219
2220 tx_ring->next_to_use = i;
2221}
2222
2223
2224
2225
2226
2227
2228
2229
2230static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2231 struct i40e_ring *tx_ring)
2232{
2233 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2234 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2235 struct i40e_tx_buffer *first;
2236 u32 td_offset = 0;
2237 u32 tx_flags = 0;
2238 __be16 protocol;
2239 u32 td_cmd = 0;
2240 u8 hdr_len = 0;
2241 int tso, count;
2242
2243
2244 prefetch(skb->data);
2245
2246 i40e_trace(xmit_frame_ring, skb, tx_ring);
2247
2248 count = i40e_xmit_descriptor_count(skb);
2249 if (i40e_chk_linearize(skb, count)) {
2250 if (__skb_linearize(skb)) {
2251 dev_kfree_skb_any(skb);
2252 return NETDEV_TX_OK;
2253 }
2254 count = i40e_txd_use_count(skb->len);
2255 tx_ring->tx_stats.tx_linearize++;
2256 }
2257
2258
2259
2260
2261
2262
2263
2264 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
2265 tx_ring->tx_stats.tx_busy++;
2266 return NETDEV_TX_BUSY;
2267 }
2268
2269
2270 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2271 first->skb = skb;
2272 first->bytecount = skb->len;
2273 first->gso_segs = 1;
2274
2275
2276 if (i40evf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2277 goto out_drop;
2278
2279
2280 protocol = vlan_get_protocol(skb);
2281
2282
2283 if (protocol == htons(ETH_P_IP))
2284 tx_flags |= I40E_TX_FLAGS_IPV4;
2285 else if (protocol == htons(ETH_P_IPV6))
2286 tx_flags |= I40E_TX_FLAGS_IPV6;
2287
2288 tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss);
2289
2290 if (tso < 0)
2291 goto out_drop;
2292 else if (tso)
2293 tx_flags |= I40E_TX_FLAGS_TSO;
2294
2295
2296 tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
2297 tx_ring, &cd_tunneling);
2298 if (tso < 0)
2299 goto out_drop;
2300
2301 skb_tx_timestamp(skb);
2302
2303
2304 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2305
2306 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2307 cd_tunneling, cd_l2tag2);
2308
2309 i40evf_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2310 td_cmd, td_offset);
2311
2312 return NETDEV_TX_OK;
2313
2314out_drop:
2315 i40e_trace(xmit_frame_ring_drop, first->skb, tx_ring);
2316 dev_kfree_skb_any(first->skb);
2317 first->skb = NULL;
2318 return NETDEV_TX_OK;
2319}
2320
2321
2322
2323
2324
2325
2326
2327
2328netdev_tx_t i40evf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2329{
2330 struct i40evf_adapter *adapter = netdev_priv(netdev);
2331 struct i40e_ring *tx_ring = &adapter->tx_rings[skb->queue_mapping];
2332
2333
2334
2335
2336 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2337 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2338 return NETDEV_TX_OK;
2339 skb->len = I40E_MIN_TX_LEN;
2340 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2341 }
2342
2343 return i40e_xmit_frame_ring(skb, tx_ring);
2344}
2345