1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/prefetch.h>
28#include <net/busy_poll.h>
29
30#include "i40evf.h"
31#include "i40e_trace.h"
32#include "i40e_prototype.h"
33
34static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
35 u32 td_tag)
36{
37 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
38 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
39 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
40 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
41 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
42}
43
44#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
45
46
47
48
49
50
51static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
52 struct i40e_tx_buffer *tx_buffer)
53{
54 if (tx_buffer->skb) {
55 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
56 kfree(tx_buffer->raw_buf);
57 else
58 dev_kfree_skb_any(tx_buffer->skb);
59 if (dma_unmap_len(tx_buffer, len))
60 dma_unmap_single(ring->dev,
61 dma_unmap_addr(tx_buffer, dma),
62 dma_unmap_len(tx_buffer, len),
63 DMA_TO_DEVICE);
64 } else if (dma_unmap_len(tx_buffer, len)) {
65 dma_unmap_page(ring->dev,
66 dma_unmap_addr(tx_buffer, dma),
67 dma_unmap_len(tx_buffer, len),
68 DMA_TO_DEVICE);
69 }
70
71 tx_buffer->next_to_watch = NULL;
72 tx_buffer->skb = NULL;
73 dma_unmap_len_set(tx_buffer, len, 0);
74
75}
76
77
78
79
80
81void i40evf_clean_tx_ring(struct i40e_ring *tx_ring)
82{
83 unsigned long bi_size;
84 u16 i;
85
86
87 if (!tx_ring->tx_bi)
88 return;
89
90
91 for (i = 0; i < tx_ring->count; i++)
92 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
93
94 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
95 memset(tx_ring->tx_bi, 0, bi_size);
96
97
98 memset(tx_ring->desc, 0, tx_ring->size);
99
100 tx_ring->next_to_use = 0;
101 tx_ring->next_to_clean = 0;
102
103 if (!tx_ring->netdev)
104 return;
105
106
107 netdev_tx_reset_queue(txring_txq(tx_ring));
108}
109
110
111
112
113
114
115
116void i40evf_free_tx_resources(struct i40e_ring *tx_ring)
117{
118 i40evf_clean_tx_ring(tx_ring);
119 kfree(tx_ring->tx_bi);
120 tx_ring->tx_bi = NULL;
121
122 if (tx_ring->desc) {
123 dma_free_coherent(tx_ring->dev, tx_ring->size,
124 tx_ring->desc, tx_ring->dma);
125 tx_ring->desc = NULL;
126 }
127}
128
129
130
131
132
133
134
135
136
137u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw)
138{
139 u32 head, tail;
140
141 head = ring->next_to_clean;
142 tail = readl(ring->tail);
143
144 if (head != tail)
145 return (head < tail) ?
146 tail - head : (tail + ring->count - head);
147
148 return 0;
149}
150
151#define WB_STRIDE 4
152
153
154
155
156
157
158
159
160
161static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
162 struct i40e_ring *tx_ring, int napi_budget)
163{
164 u16 i = tx_ring->next_to_clean;
165 struct i40e_tx_buffer *tx_buf;
166 struct i40e_tx_desc *tx_desc;
167 unsigned int total_bytes = 0, total_packets = 0;
168 unsigned int budget = vsi->work_limit;
169
170 tx_buf = &tx_ring->tx_bi[i];
171 tx_desc = I40E_TX_DESC(tx_ring, i);
172 i -= tx_ring->count;
173
174 do {
175 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
176
177
178 if (!eop_desc)
179 break;
180
181
182 read_barrier_depends();
183
184 i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf);
185
186 if (!(eop_desc->cmd_type_offset_bsz &
187 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
188 break;
189
190
191 tx_buf->next_to_watch = NULL;
192
193
194 total_bytes += tx_buf->bytecount;
195 total_packets += tx_buf->gso_segs;
196
197
198 napi_consume_skb(tx_buf->skb, napi_budget);
199
200
201 dma_unmap_single(tx_ring->dev,
202 dma_unmap_addr(tx_buf, dma),
203 dma_unmap_len(tx_buf, len),
204 DMA_TO_DEVICE);
205
206
207 tx_buf->skb = NULL;
208 dma_unmap_len_set(tx_buf, len, 0);
209
210
211 while (tx_desc != eop_desc) {
212 i40e_trace(clean_tx_irq_unmap,
213 tx_ring, tx_desc, tx_buf);
214
215 tx_buf++;
216 tx_desc++;
217 i++;
218 if (unlikely(!i)) {
219 i -= tx_ring->count;
220 tx_buf = tx_ring->tx_bi;
221 tx_desc = I40E_TX_DESC(tx_ring, 0);
222 }
223
224
225 if (dma_unmap_len(tx_buf, len)) {
226 dma_unmap_page(tx_ring->dev,
227 dma_unmap_addr(tx_buf, dma),
228 dma_unmap_len(tx_buf, len),
229 DMA_TO_DEVICE);
230 dma_unmap_len_set(tx_buf, len, 0);
231 }
232 }
233
234
235 tx_buf++;
236 tx_desc++;
237 i++;
238 if (unlikely(!i)) {
239 i -= tx_ring->count;
240 tx_buf = tx_ring->tx_bi;
241 tx_desc = I40E_TX_DESC(tx_ring, 0);
242 }
243
244 prefetch(tx_desc);
245
246
247 budget--;
248 } while (likely(budget));
249
250 i += tx_ring->count;
251 tx_ring->next_to_clean = i;
252 u64_stats_update_begin(&tx_ring->syncp);
253 tx_ring->stats.bytes += total_bytes;
254 tx_ring->stats.packets += total_packets;
255 u64_stats_update_end(&tx_ring->syncp);
256 tx_ring->q_vector->tx.total_bytes += total_bytes;
257 tx_ring->q_vector->tx.total_packets += total_packets;
258
259 if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {
260
261
262
263
264
265 unsigned int j = i40evf_get_tx_pending(tx_ring, false);
266
267 if (budget &&
268 ((j / WB_STRIDE) == 0) && (j > 0) &&
269 !test_bit(__I40E_VSI_DOWN, vsi->state) &&
270 (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))
271 tx_ring->arm_wb = true;
272 }
273
274
275 netdev_tx_completed_queue(txring_txq(tx_ring),
276 total_packets, total_bytes);
277
278#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
279 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
280 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
281
282
283
284 smp_mb();
285 if (__netif_subqueue_stopped(tx_ring->netdev,
286 tx_ring->queue_index) &&
287 !test_bit(__I40E_VSI_DOWN, vsi->state)) {
288 netif_wake_subqueue(tx_ring->netdev,
289 tx_ring->queue_index);
290 ++tx_ring->tx_stats.restart_queue;
291 }
292 }
293
294 return !!budget;
295}
296
297
298
299
300
301
302
303static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
304 struct i40e_q_vector *q_vector)
305{
306 u16 flags = q_vector->tx.ring[0].flags;
307 u32 val;
308
309 if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR))
310 return;
311
312 if (q_vector->arm_wb_state)
313 return;
314
315 val = I40E_VFINT_DYN_CTLN1_WB_ON_ITR_MASK |
316 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK;
317
318 wr32(&vsi->back->hw,
319 I40E_VFINT_DYN_CTLN1(q_vector->v_idx +
320 vsi->base_vector - 1), val);
321 q_vector->arm_wb_state = true;
322}
323
324
325
326
327
328
329
330void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
331{
332 u32 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
333 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK |
334 I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
335 I40E_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK
336 ;
337
338 wr32(&vsi->back->hw,
339 I40E_VFINT_DYN_CTLN1(q_vector->v_idx + vsi->base_vector - 1),
340 val);
341}
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357static bool i40e_set_new_dynamic_itr(struct i40e_ring_container *rc)
358{
359 enum i40e_latency_range new_latency_range = rc->latency_range;
360 struct i40e_q_vector *qv = rc->ring->q_vector;
361 u32 new_itr = rc->itr;
362 int bytes_per_int;
363 int usecs;
364
365 if (rc->total_packets == 0 || !rc->itr)
366 return false;
367
368
369
370
371
372
373
374
375
376
377
378
379
380 usecs = (rc->itr << 1) * ITR_COUNTDOWN_START;
381 bytes_per_int = rc->total_bytes / usecs;
382
383 switch (new_latency_range) {
384 case I40E_LOWEST_LATENCY:
385 if (bytes_per_int > 10)
386 new_latency_range = I40E_LOW_LATENCY;
387 break;
388 case I40E_LOW_LATENCY:
389 if (bytes_per_int > 20)
390 new_latency_range = I40E_BULK_LATENCY;
391 else if (bytes_per_int <= 10)
392 new_latency_range = I40E_LOWEST_LATENCY;
393 break;
394 case I40E_BULK_LATENCY:
395 case I40E_ULTRA_LATENCY:
396 default:
397 if (bytes_per_int <= 20)
398 new_latency_range = I40E_LOW_LATENCY;
399 break;
400 }
401
402
403
404
405
406
407#define RX_ULTRA_PACKET_RATE 40000
408
409 if ((((rc->total_packets * 1000000) / usecs) > RX_ULTRA_PACKET_RATE) &&
410 (&qv->rx == rc))
411 new_latency_range = I40E_ULTRA_LATENCY;
412
413 rc->latency_range = new_latency_range;
414
415 switch (new_latency_range) {
416 case I40E_LOWEST_LATENCY:
417 new_itr = I40E_ITR_50K;
418 break;
419 case I40E_LOW_LATENCY:
420 new_itr = I40E_ITR_20K;
421 break;
422 case I40E_BULK_LATENCY:
423 new_itr = I40E_ITR_18K;
424 break;
425 case I40E_ULTRA_LATENCY:
426 new_itr = I40E_ITR_8K;
427 break;
428 default:
429 break;
430 }
431
432 rc->total_bytes = 0;
433 rc->total_packets = 0;
434
435 if (new_itr != rc->itr) {
436 rc->itr = new_itr;
437 return true;
438 }
439
440 return false;
441}
442
443
444
445
446
447
448
449int i40evf_setup_tx_descriptors(struct i40e_ring *tx_ring)
450{
451 struct device *dev = tx_ring->dev;
452 int bi_size;
453
454 if (!dev)
455 return -ENOMEM;
456
457
458 WARN_ON(tx_ring->tx_bi);
459 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
460 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
461 if (!tx_ring->tx_bi)
462 goto err;
463
464
465 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
466 tx_ring->size = ALIGN(tx_ring->size, 4096);
467 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
468 &tx_ring->dma, GFP_KERNEL);
469 if (!tx_ring->desc) {
470 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
471 tx_ring->size);
472 goto err;
473 }
474
475 tx_ring->next_to_use = 0;
476 tx_ring->next_to_clean = 0;
477 return 0;
478
479err:
480 kfree(tx_ring->tx_bi);
481 tx_ring->tx_bi = NULL;
482 return -ENOMEM;
483}
484
485
486
487
488
489void i40evf_clean_rx_ring(struct i40e_ring *rx_ring)
490{
491 unsigned long bi_size;
492 u16 i;
493
494
495 if (!rx_ring->rx_bi)
496 return;
497
498 if (rx_ring->skb) {
499 dev_kfree_skb(rx_ring->skb);
500 rx_ring->skb = NULL;
501 }
502
503
504 for (i = 0; i < rx_ring->count; i++) {
505 struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
506
507 if (!rx_bi->page)
508 continue;
509
510
511
512
513 dma_sync_single_range_for_cpu(rx_ring->dev,
514 rx_bi->dma,
515 rx_bi->page_offset,
516 rx_ring->rx_buf_len,
517 DMA_FROM_DEVICE);
518
519
520 dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma,
521 i40e_rx_pg_size(rx_ring),
522 DMA_FROM_DEVICE,
523 I40E_RX_DMA_ATTR);
524
525 __page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias);
526
527 rx_bi->page = NULL;
528 rx_bi->page_offset = 0;
529 }
530
531 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
532 memset(rx_ring->rx_bi, 0, bi_size);
533
534
535 memset(rx_ring->desc, 0, rx_ring->size);
536
537 rx_ring->next_to_alloc = 0;
538 rx_ring->next_to_clean = 0;
539 rx_ring->next_to_use = 0;
540}
541
542
543
544
545
546
547
548void i40evf_free_rx_resources(struct i40e_ring *rx_ring)
549{
550 i40evf_clean_rx_ring(rx_ring);
551 kfree(rx_ring->rx_bi);
552 rx_ring->rx_bi = NULL;
553
554 if (rx_ring->desc) {
555 dma_free_coherent(rx_ring->dev, rx_ring->size,
556 rx_ring->desc, rx_ring->dma);
557 rx_ring->desc = NULL;
558 }
559}
560
561
562
563
564
565
566
567int i40evf_setup_rx_descriptors(struct i40e_ring *rx_ring)
568{
569 struct device *dev = rx_ring->dev;
570 int bi_size;
571
572
573 WARN_ON(rx_ring->rx_bi);
574 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
575 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
576 if (!rx_ring->rx_bi)
577 goto err;
578
579 u64_stats_init(&rx_ring->syncp);
580
581
582 rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc);
583 rx_ring->size = ALIGN(rx_ring->size, 4096);
584 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
585 &rx_ring->dma, GFP_KERNEL);
586
587 if (!rx_ring->desc) {
588 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
589 rx_ring->size);
590 goto err;
591 }
592
593 rx_ring->next_to_alloc = 0;
594 rx_ring->next_to_clean = 0;
595 rx_ring->next_to_use = 0;
596
597 return 0;
598err:
599 kfree(rx_ring->rx_bi);
600 rx_ring->rx_bi = NULL;
601 return -ENOMEM;
602}
603
604
605
606
607
608
609static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
610{
611 rx_ring->next_to_use = val;
612
613
614 rx_ring->next_to_alloc = val;
615
616
617
618
619
620
621 wmb();
622 writel(val, rx_ring->tail);
623}
624
625
626
627
628
629
630
631static inline unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
632{
633 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
634}
635
636
637
638
639
640
641
642
643
644static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring,
645 struct i40e_rx_buffer *bi)
646{
647 struct page *page = bi->page;
648 dma_addr_t dma;
649
650
651 if (likely(page)) {
652 rx_ring->rx_stats.page_reuse_count++;
653 return true;
654 }
655
656
657 page = dev_alloc_pages(i40e_rx_pg_order(rx_ring));
658 if (unlikely(!page)) {
659 rx_ring->rx_stats.alloc_page_failed++;
660 return false;
661 }
662
663
664 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
665 i40e_rx_pg_size(rx_ring),
666 DMA_FROM_DEVICE,
667 I40E_RX_DMA_ATTR);
668
669
670
671
672 if (dma_mapping_error(rx_ring->dev, dma)) {
673 __free_pages(page, i40e_rx_pg_order(rx_ring));
674 rx_ring->rx_stats.alloc_page_failed++;
675 return false;
676 }
677
678 bi->dma = dma;
679 bi->page = page;
680 bi->page_offset = i40e_rx_offset(rx_ring);
681
682
683 bi->pagecnt_bias = 1;
684
685 return true;
686}
687
688
689
690
691
692
693
694static void i40e_receive_skb(struct i40e_ring *rx_ring,
695 struct sk_buff *skb, u16 vlan_tag)
696{
697 struct i40e_q_vector *q_vector = rx_ring->q_vector;
698
699 if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
700 (vlan_tag & VLAN_VID_MASK))
701 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
702
703 napi_gro_receive(&q_vector->napi, skb);
704}
705
706
707
708
709
710
711
712
713bool i40evf_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
714{
715 u16 ntu = rx_ring->next_to_use;
716 union i40e_rx_desc *rx_desc;
717 struct i40e_rx_buffer *bi;
718
719
720 if (!rx_ring->netdev || !cleaned_count)
721 return false;
722
723 rx_desc = I40E_RX_DESC(rx_ring, ntu);
724 bi = &rx_ring->rx_bi[ntu];
725
726 do {
727 if (!i40e_alloc_mapped_page(rx_ring, bi))
728 goto no_buffers;
729
730
731 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
732 bi->page_offset,
733 rx_ring->rx_buf_len,
734 DMA_FROM_DEVICE);
735
736
737
738
739 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
740
741 rx_desc++;
742 bi++;
743 ntu++;
744 if (unlikely(ntu == rx_ring->count)) {
745 rx_desc = I40E_RX_DESC(rx_ring, 0);
746 bi = rx_ring->rx_bi;
747 ntu = 0;
748 }
749
750
751 rx_desc->wb.qword1.status_error_len = 0;
752
753 cleaned_count--;
754 } while (cleaned_count);
755
756 if (rx_ring->next_to_use != ntu)
757 i40e_release_rx_desc(rx_ring, ntu);
758
759 return false;
760
761no_buffers:
762 if (rx_ring->next_to_use != ntu)
763 i40e_release_rx_desc(rx_ring, ntu);
764
765
766
767
768 return true;
769}
770
771
772
773
774
775
776
777static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
778 struct sk_buff *skb,
779 union i40e_rx_desc *rx_desc)
780{
781 struct i40e_rx_ptype_decoded decoded;
782 u32 rx_error, rx_status;
783 bool ipv4, ipv6;
784 u8 ptype;
785 u64 qword;
786
787 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
788 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT;
789 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
790 I40E_RXD_QW1_ERROR_SHIFT;
791 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
792 I40E_RXD_QW1_STATUS_SHIFT;
793 decoded = decode_rx_desc_ptype(ptype);
794
795 skb->ip_summed = CHECKSUM_NONE;
796
797 skb_checksum_none_assert(skb);
798
799
800 if (!(vsi->netdev->features & NETIF_F_RXCSUM))
801 return;
802
803
804 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
805 return;
806
807
808 if (!(decoded.known && decoded.outer_ip))
809 return;
810
811 ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
812 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4);
813 ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
814 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6);
815
816 if (ipv4 &&
817 (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) |
818 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT))))
819 goto checksum_fail;
820
821
822 if (ipv6 &&
823 rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
824
825 return;
826
827
828 if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT))
829 goto checksum_fail;
830
831
832
833
834
835 if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT))
836 return;
837
838
839 switch (decoded.inner_prot) {
840 case I40E_RX_PTYPE_INNER_PROT_TCP:
841 case I40E_RX_PTYPE_INNER_PROT_UDP:
842 case I40E_RX_PTYPE_INNER_PROT_SCTP:
843 skb->ip_summed = CHECKSUM_UNNECESSARY;
844
845 default:
846 break;
847 }
848
849 return;
850
851checksum_fail:
852 vsi->back->hw_csum_rx_error++;
853}
854
855
856
857
858
859
860
861static inline int i40e_ptype_to_htype(u8 ptype)
862{
863 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
864
865 if (!decoded.known)
866 return PKT_HASH_TYPE_NONE;
867
868 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
869 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
870 return PKT_HASH_TYPE_L4;
871 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
872 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
873 return PKT_HASH_TYPE_L3;
874 else
875 return PKT_HASH_TYPE_L2;
876}
877
878
879
880
881
882
883static inline void i40e_rx_hash(struct i40e_ring *ring,
884 union i40e_rx_desc *rx_desc,
885 struct sk_buff *skb,
886 u8 rx_ptype)
887{
888 u32 hash;
889 const __le64 rss_mask =
890 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
891 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
892
893 if (ring->netdev->features & NETIF_F_RXHASH)
894 return;
895
896 if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
897 hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
898 skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype));
899 }
900}
901
902
903
904
905
906
907
908
909
910
911
912
913static inline
914void i40evf_process_skb_fields(struct i40e_ring *rx_ring,
915 union i40e_rx_desc *rx_desc, struct sk_buff *skb,
916 u8 rx_ptype)
917{
918 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
919
920 i40e_rx_checksum(rx_ring->vsi, skb, rx_desc);
921
922 skb_record_rx_queue(skb, rx_ring->queue_index);
923
924
925 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
926}
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb)
942{
943
944 if (eth_skb_pad(skb))
945 return true;
946
947 return false;
948}
949
950
951
952
953
954
955
956
957static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
958 struct i40e_rx_buffer *old_buff)
959{
960 struct i40e_rx_buffer *new_buff;
961 u16 nta = rx_ring->next_to_alloc;
962
963 new_buff = &rx_ring->rx_bi[nta];
964
965
966 nta++;
967 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
968
969
970 new_buff->dma = old_buff->dma;
971 new_buff->page = old_buff->page;
972 new_buff->page_offset = old_buff->page_offset;
973 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
974}
975
976
977
978
979
980
981
982
983static inline bool i40e_page_is_reusable(struct page *page)
984{
985 return (page_to_nid(page) == numa_mem_id()) &&
986 !page_is_pfmemalloc(page);
987}
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer)
1017{
1018 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1019 struct page *page = rx_buffer->page;
1020
1021
1022 if (unlikely(!i40e_page_is_reusable(page)))
1023 return false;
1024
1025#if (PAGE_SIZE < 8192)
1026
1027 if (unlikely((page_count(page) - pagecnt_bias) > 1))
1028 return false;
1029#else
1030#define I40E_LAST_OFFSET \
1031 (SKB_WITH_OVERHEAD(PAGE_SIZE) - I40E_RXBUFFER_2048)
1032 if (rx_buffer->page_offset > I40E_LAST_OFFSET)
1033 return false;
1034#endif
1035
1036
1037
1038
1039
1040 if (unlikely(!pagecnt_bias)) {
1041 page_ref_add(page, USHRT_MAX);
1042 rx_buffer->pagecnt_bias = USHRT_MAX;
1043 }
1044
1045 return true;
1046}
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
1061 struct i40e_rx_buffer *rx_buffer,
1062 struct sk_buff *skb,
1063 unsigned int size)
1064{
1065#if (PAGE_SIZE < 8192)
1066 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1067#else
1068 unsigned int truesize = SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring));
1069#endif
1070
1071 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1072 rx_buffer->page_offset, size, truesize);
1073
1074
1075#if (PAGE_SIZE < 8192)
1076 rx_buffer->page_offset ^= truesize;
1077#else
1078 rx_buffer->page_offset += truesize;
1079#endif
1080}
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
1091 const unsigned int size)
1092{
1093 struct i40e_rx_buffer *rx_buffer;
1094
1095 rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
1096 prefetchw(rx_buffer->page);
1097
1098
1099 dma_sync_single_range_for_cpu(rx_ring->dev,
1100 rx_buffer->dma,
1101 rx_buffer->page_offset,
1102 size,
1103 DMA_FROM_DEVICE);
1104
1105
1106 rx_buffer->pagecnt_bias--;
1107
1108 return rx_buffer;
1109}
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
1122 struct i40e_rx_buffer *rx_buffer,
1123 unsigned int size)
1124{
1125 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1126#if (PAGE_SIZE < 8192)
1127 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1128#else
1129 unsigned int truesize = SKB_DATA_ALIGN(size);
1130#endif
1131 unsigned int headlen;
1132 struct sk_buff *skb;
1133
1134
1135 prefetch(va);
1136#if L1_CACHE_BYTES < 128
1137 prefetch(va + L1_CACHE_BYTES);
1138#endif
1139
1140
1141 skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
1142 I40E_RX_HDR_SIZE,
1143 GFP_ATOMIC | __GFP_NOWARN);
1144 if (unlikely(!skb))
1145 return NULL;
1146
1147
1148 headlen = size;
1149 if (headlen > I40E_RX_HDR_SIZE)
1150 headlen = eth_get_headlen(va, I40E_RX_HDR_SIZE);
1151
1152
1153 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
1154
1155
1156 size -= headlen;
1157 if (size) {
1158 skb_add_rx_frag(skb, 0, rx_buffer->page,
1159 rx_buffer->page_offset + headlen,
1160 size, truesize);
1161
1162
1163#if (PAGE_SIZE < 8192)
1164 rx_buffer->page_offset ^= truesize;
1165#else
1166 rx_buffer->page_offset += truesize;
1167#endif
1168 } else {
1169
1170 rx_buffer->pagecnt_bias++;
1171 }
1172
1173 return skb;
1174}
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
1186 struct i40e_rx_buffer *rx_buffer,
1187 unsigned int size)
1188{
1189 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1190#if (PAGE_SIZE < 8192)
1191 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1192#else
1193 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1194 SKB_DATA_ALIGN(I40E_SKB_PAD + size);
1195#endif
1196 struct sk_buff *skb;
1197
1198
1199 prefetch(va);
1200#if L1_CACHE_BYTES < 128
1201 prefetch(va + L1_CACHE_BYTES);
1202#endif
1203
1204 skb = build_skb(va - I40E_SKB_PAD, truesize);
1205 if (unlikely(!skb))
1206 return NULL;
1207
1208
1209 skb_reserve(skb, I40E_SKB_PAD);
1210 __skb_put(skb, size);
1211
1212
1213#if (PAGE_SIZE < 8192)
1214 rx_buffer->page_offset ^= truesize;
1215#else
1216 rx_buffer->page_offset += truesize;
1217#endif
1218
1219 return skb;
1220}
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
1231 struct i40e_rx_buffer *rx_buffer)
1232{
1233 if (i40e_can_reuse_rx_page(rx_buffer)) {
1234
1235 i40e_reuse_rx_page(rx_ring, rx_buffer);
1236 rx_ring->rx_stats.page_reuse_count++;
1237 } else {
1238
1239 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1240 i40e_rx_pg_size(rx_ring),
1241 DMA_FROM_DEVICE, I40E_RX_DMA_ATTR);
1242 __page_frag_cache_drain(rx_buffer->page,
1243 rx_buffer->pagecnt_bias);
1244 }
1245
1246
1247 rx_buffer->page = NULL;
1248}
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261static bool i40e_is_non_eop(struct i40e_ring *rx_ring,
1262 union i40e_rx_desc *rx_desc,
1263 struct sk_buff *skb)
1264{
1265 u32 ntc = rx_ring->next_to_clean + 1;
1266
1267
1268 ntc = (ntc < rx_ring->count) ? ntc : 0;
1269 rx_ring->next_to_clean = ntc;
1270
1271 prefetch(I40E_RX_DESC(rx_ring, ntc));
1272
1273
1274#define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)
1275 if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF)))
1276 return false;
1277
1278 rx_ring->rx_stats.non_eop_descs++;
1279
1280 return true;
1281}
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1296{
1297 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1298 struct sk_buff *skb = rx_ring->skb;
1299 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1300 bool failure = false;
1301
1302 while (likely(total_rx_packets < budget)) {
1303 struct i40e_rx_buffer *rx_buffer;
1304 union i40e_rx_desc *rx_desc;
1305 unsigned int size;
1306 u16 vlan_tag;
1307 u8 rx_ptype;
1308 u64 qword;
1309
1310
1311 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1312 failure = failure ||
1313 i40evf_alloc_rx_buffers(rx_ring, cleaned_count);
1314 cleaned_count = 0;
1315 }
1316
1317 rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);
1318
1319
1320
1321
1322
1323
1324 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1325
1326
1327
1328
1329
1330 dma_rmb();
1331
1332 size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1333 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1334 if (!size)
1335 break;
1336
1337 i40e_trace(clean_rx_irq, rx_ring, rx_desc, skb);
1338 rx_buffer = i40e_get_rx_buffer(rx_ring, size);
1339
1340
1341 if (skb)
1342 i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
1343 else if (ring_uses_build_skb(rx_ring))
1344 skb = i40e_build_skb(rx_ring, rx_buffer, size);
1345 else
1346 skb = i40e_construct_skb(rx_ring, rx_buffer, size);
1347
1348
1349 if (!skb) {
1350 rx_ring->rx_stats.alloc_buff_failed++;
1351 rx_buffer->pagecnt_bias++;
1352 break;
1353 }
1354
1355 i40e_put_rx_buffer(rx_ring, rx_buffer);
1356 cleaned_count++;
1357
1358 if (i40e_is_non_eop(rx_ring, rx_desc, skb))
1359 continue;
1360
1361
1362
1363
1364
1365
1366 if (unlikely(i40e_test_staterr(rx_desc, BIT(I40E_RXD_QW1_ERROR_SHIFT)))) {
1367 dev_kfree_skb_any(skb);
1368 skb = NULL;
1369 continue;
1370 }
1371
1372 if (i40e_cleanup_headers(rx_ring, skb)) {
1373 skb = NULL;
1374 continue;
1375 }
1376
1377
1378 total_rx_bytes += skb->len;
1379
1380 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1381 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1382 I40E_RXD_QW1_PTYPE_SHIFT;
1383
1384
1385 i40evf_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
1386
1387
1388 vlan_tag = (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) ?
1389 le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0;
1390
1391 i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb);
1392 i40e_receive_skb(rx_ring, skb, vlan_tag);
1393 skb = NULL;
1394
1395
1396 total_rx_packets++;
1397 }
1398
1399 rx_ring->skb = skb;
1400
1401 u64_stats_update_begin(&rx_ring->syncp);
1402 rx_ring->stats.packets += total_rx_packets;
1403 rx_ring->stats.bytes += total_rx_bytes;
1404 u64_stats_update_end(&rx_ring->syncp);
1405 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1406 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1407
1408
1409 return failure ? budget : total_rx_packets;
1410}
1411
1412static u32 i40e_buildreg_itr(const int type, const u16 itr)
1413{
1414 u32 val;
1415
1416 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1417
1418
1419
1420 (type << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1421 (itr << I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT);
1422
1423 return val;
1424}
1425
1426
1427#define INTREG I40E_VFINT_DYN_CTLN1
1428static inline int get_rx_itr(struct i40e_vsi *vsi, int idx)
1429{
1430 struct i40evf_adapter *adapter = vsi->back;
1431
1432 return adapter->rx_rings[idx].rx_itr_setting;
1433}
1434
1435static inline int get_tx_itr(struct i40e_vsi *vsi, int idx)
1436{
1437 struct i40evf_adapter *adapter = vsi->back;
1438
1439 return adapter->tx_rings[idx].tx_itr_setting;
1440}
1441
1442
1443
1444
1445
1446
1447
1448static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
1449 struct i40e_q_vector *q_vector)
1450{
1451 struct i40e_hw *hw = &vsi->back->hw;
1452 bool rx = false, tx = false;
1453 u32 rxval, txval;
1454 int vector;
1455 int idx = q_vector->v_idx;
1456 int rx_itr_setting, tx_itr_setting;
1457
1458 vector = (q_vector->v_idx + vsi->base_vector);
1459
1460
1461
1462
1463 rxval = txval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
1464
1465 rx_itr_setting = get_rx_itr(vsi, idx);
1466 tx_itr_setting = get_tx_itr(vsi, idx);
1467
1468 if (q_vector->itr_countdown > 0 ||
1469 (!ITR_IS_DYNAMIC(rx_itr_setting) &&
1470 !ITR_IS_DYNAMIC(tx_itr_setting))) {
1471 goto enable_int;
1472 }
1473
1474 if (ITR_IS_DYNAMIC(rx_itr_setting)) {
1475 rx = i40e_set_new_dynamic_itr(&q_vector->rx);
1476 rxval = i40e_buildreg_itr(I40E_RX_ITR, q_vector->rx.itr);
1477 }
1478
1479 if (ITR_IS_DYNAMIC(tx_itr_setting)) {
1480 tx = i40e_set_new_dynamic_itr(&q_vector->tx);
1481 txval = i40e_buildreg_itr(I40E_TX_ITR, q_vector->tx.itr);
1482 }
1483
1484 if (rx || tx) {
1485
1486
1487
1488
1489 u16 itr = max(q_vector->tx.itr, q_vector->rx.itr);
1490
1491 q_vector->tx.itr = q_vector->rx.itr = itr;
1492 txval = i40e_buildreg_itr(I40E_TX_ITR, itr);
1493 tx = true;
1494 rxval = i40e_buildreg_itr(I40E_RX_ITR, itr);
1495 rx = true;
1496 }
1497
1498
1499
1500
1501 if (rx) {
1502
1503
1504
1505
1506 rxval |= BIT(31);
1507
1508 wr32(hw, INTREG(vector - 1), rxval);
1509 }
1510
1511enable_int:
1512 if (!test_bit(__I40E_VSI_DOWN, vsi->state))
1513 wr32(hw, INTREG(vector - 1), txval);
1514
1515 if (q_vector->itr_countdown)
1516 q_vector->itr_countdown--;
1517 else
1518 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1519}
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530int i40evf_napi_poll(struct napi_struct *napi, int budget)
1531{
1532 struct i40e_q_vector *q_vector =
1533 container_of(napi, struct i40e_q_vector, napi);
1534 struct i40e_vsi *vsi = q_vector->vsi;
1535 struct i40e_ring *ring;
1536 bool clean_complete = true;
1537 bool arm_wb = false;
1538 int budget_per_ring;
1539 int work_done = 0;
1540
1541 if (test_bit(__I40E_VSI_DOWN, vsi->state)) {
1542 napi_complete(napi);
1543 return 0;
1544 }
1545
1546
1547
1548
1549 i40e_for_each_ring(ring, q_vector->tx) {
1550 if (!i40e_clean_tx_irq(vsi, ring, budget)) {
1551 clean_complete = false;
1552 continue;
1553 }
1554 arm_wb |= ring->arm_wb;
1555 ring->arm_wb = false;
1556 }
1557
1558
1559 if (budget <= 0)
1560 goto tx_only;
1561
1562
1563
1564
1565 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1566
1567 i40e_for_each_ring(ring, q_vector->rx) {
1568 int cleaned = i40e_clean_rx_irq(ring, budget_per_ring);
1569
1570 work_done += cleaned;
1571
1572 if (cleaned >= budget_per_ring)
1573 clean_complete = false;
1574 }
1575
1576
1577 if (!clean_complete) {
1578 const cpumask_t *aff_mask = &q_vector->affinity_mask;
1579 int cpu_id = smp_processor_id();
1580
1581
1582
1583
1584
1585
1586
1587
1588 if (likely(cpumask_test_cpu(cpu_id, aff_mask))) {
1589tx_only:
1590 if (arm_wb) {
1591 q_vector->tx.ring[0].tx_stats.tx_force_wb++;
1592 i40e_enable_wb_on_itr(vsi, q_vector);
1593 }
1594 return budget;
1595 }
1596 }
1597
1598 if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
1599 q_vector->arm_wb_state = false;
1600
1601
1602 napi_complete_done(napi, work_done);
1603
1604
1605
1606
1607
1608 if (!clean_complete)
1609 i40evf_force_wb(vsi, q_vector);
1610 else
1611 i40e_update_enable_itr(vsi, q_vector);
1612
1613 return min(work_done, budget - 1);
1614}
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628static inline int i40evf_tx_prepare_vlan_flags(struct sk_buff *skb,
1629 struct i40e_ring *tx_ring,
1630 u32 *flags)
1631{
1632 __be16 protocol = skb->protocol;
1633 u32 tx_flags = 0;
1634
1635 if (protocol == htons(ETH_P_8021Q) &&
1636 !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
1637
1638
1639
1640
1641
1642
1643
1644 skb->protocol = vlan_get_protocol(skb);
1645 goto out;
1646 }
1647
1648
1649 if (skb_vlan_tag_present(skb)) {
1650 tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1651 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1652
1653 } else if (protocol == htons(ETH_P_8021Q)) {
1654 struct vlan_hdr *vhdr, _vhdr;
1655
1656 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1657 if (!vhdr)
1658 return -EINVAL;
1659
1660 protocol = vhdr->h_vlan_encapsulated_proto;
1661 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1662 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1663 }
1664
1665out:
1666 *flags = tx_flags;
1667 return 0;
1668}
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len,
1679 u64 *cd_type_cmd_tso_mss)
1680{
1681 struct sk_buff *skb = first->skb;
1682 u64 cd_cmd, cd_tso_len, cd_mss;
1683 union {
1684 struct iphdr *v4;
1685 struct ipv6hdr *v6;
1686 unsigned char *hdr;
1687 } ip;
1688 union {
1689 struct tcphdr *tcp;
1690 struct udphdr *udp;
1691 unsigned char *hdr;
1692 } l4;
1693 u32 paylen, l4_offset;
1694 u16 gso_segs, gso_size;
1695 int err;
1696
1697 if (skb->ip_summed != CHECKSUM_PARTIAL)
1698 return 0;
1699
1700 if (!skb_is_gso(skb))
1701 return 0;
1702
1703 err = skb_cow_head(skb, 0);
1704 if (err < 0)
1705 return err;
1706
1707 ip.hdr = skb_network_header(skb);
1708 l4.hdr = skb_transport_header(skb);
1709
1710
1711 if (ip.v4->version == 4) {
1712 ip.v4->tot_len = 0;
1713 ip.v4->check = 0;
1714 } else {
1715 ip.v6->payload_len = 0;
1716 }
1717
1718 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1719 SKB_GSO_GRE_CSUM |
1720 SKB_GSO_IPXIP4 |
1721 SKB_GSO_IPXIP6 |
1722 SKB_GSO_UDP_TUNNEL |
1723 SKB_GSO_UDP_TUNNEL_CSUM)) {
1724 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1725 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
1726 l4.udp->len = 0;
1727
1728
1729 l4_offset = l4.hdr - skb->data;
1730
1731
1732 paylen = skb->len - l4_offset;
1733 csum_replace_by_diff(&l4.udp->check,
1734 (__force __wsum)htonl(paylen));
1735 }
1736
1737
1738 ip.hdr = skb_inner_network_header(skb);
1739 l4.hdr = skb_inner_transport_header(skb);
1740
1741
1742 if (ip.v4->version == 4) {
1743 ip.v4->tot_len = 0;
1744 ip.v4->check = 0;
1745 } else {
1746 ip.v6->payload_len = 0;
1747 }
1748 }
1749
1750
1751 l4_offset = l4.hdr - skb->data;
1752
1753
1754 paylen = skb->len - l4_offset;
1755 csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
1756
1757
1758 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
1759
1760
1761 gso_size = skb_shinfo(skb)->gso_size;
1762 gso_segs = skb_shinfo(skb)->gso_segs;
1763
1764
1765 first->gso_segs = gso_segs;
1766 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1767
1768
1769 cd_cmd = I40E_TX_CTX_DESC_TSO;
1770 cd_tso_len = skb->len - *hdr_len;
1771 cd_mss = gso_size;
1772 *cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1773 (cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1774 (cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1775 return 1;
1776}
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
1788 u32 *td_cmd, u32 *td_offset,
1789 struct i40e_ring *tx_ring,
1790 u32 *cd_tunneling)
1791{
1792 union {
1793 struct iphdr *v4;
1794 struct ipv6hdr *v6;
1795 unsigned char *hdr;
1796 } ip;
1797 union {
1798 struct tcphdr *tcp;
1799 struct udphdr *udp;
1800 unsigned char *hdr;
1801 } l4;
1802 unsigned char *exthdr;
1803 u32 offset, cmd = 0;
1804 __be16 frag_off;
1805 u8 l4_proto = 0;
1806
1807 if (skb->ip_summed != CHECKSUM_PARTIAL)
1808 return 0;
1809
1810 ip.hdr = skb_network_header(skb);
1811 l4.hdr = skb_transport_header(skb);
1812
1813
1814 offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1815
1816 if (skb->encapsulation) {
1817 u32 tunnel = 0;
1818
1819 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1820 tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
1821 I40E_TX_CTX_EXT_IP_IPV4 :
1822 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
1823
1824 l4_proto = ip.v4->protocol;
1825 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
1826 tunnel |= I40E_TX_CTX_EXT_IP_IPV6;
1827
1828 exthdr = ip.hdr + sizeof(*ip.v6);
1829 l4_proto = ip.v6->nexthdr;
1830 if (l4.hdr != exthdr)
1831 ipv6_skip_exthdr(skb, exthdr - skb->data,
1832 &l4_proto, &frag_off);
1833 }
1834
1835
1836 switch (l4_proto) {
1837 case IPPROTO_UDP:
1838 tunnel |= I40E_TXD_CTX_UDP_TUNNELING;
1839 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
1840 break;
1841 case IPPROTO_GRE:
1842 tunnel |= I40E_TXD_CTX_GRE_TUNNELING;
1843 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
1844 break;
1845 case IPPROTO_IPIP:
1846 case IPPROTO_IPV6:
1847 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
1848 l4.hdr = skb_inner_network_header(skb);
1849 break;
1850 default:
1851 if (*tx_flags & I40E_TX_FLAGS_TSO)
1852 return -1;
1853
1854 skb_checksum_help(skb);
1855 return 0;
1856 }
1857
1858
1859 tunnel |= ((l4.hdr - ip.hdr) / 4) <<
1860 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;
1861
1862
1863 ip.hdr = skb_inner_network_header(skb);
1864
1865
1866 tunnel |= ((ip.hdr - l4.hdr) / 2) <<
1867 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
1868
1869
1870 if ((*tx_flags & I40E_TX_FLAGS_TSO) &&
1871 !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1872 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
1873 tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK;
1874
1875
1876 *cd_tunneling |= tunnel;
1877
1878
1879 l4.hdr = skb_inner_transport_header(skb);
1880 l4_proto = 0;
1881
1882
1883 *tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6);
1884 if (ip.v4->version == 4)
1885 *tx_flags |= I40E_TX_FLAGS_IPV4;
1886 if (ip.v6->version == 6)
1887 *tx_flags |= I40E_TX_FLAGS_IPV6;
1888 }
1889
1890
1891 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1892 l4_proto = ip.v4->protocol;
1893
1894
1895
1896 cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
1897 I40E_TX_DESC_CMD_IIPT_IPV4_CSUM :
1898 I40E_TX_DESC_CMD_IIPT_IPV4;
1899 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
1900 cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
1901
1902 exthdr = ip.hdr + sizeof(*ip.v6);
1903 l4_proto = ip.v6->nexthdr;
1904 if (l4.hdr != exthdr)
1905 ipv6_skip_exthdr(skb, exthdr - skb->data,
1906 &l4_proto, &frag_off);
1907 }
1908
1909
1910 offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
1911
1912
1913 switch (l4_proto) {
1914 case IPPROTO_TCP:
1915
1916 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
1917 offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1918 break;
1919 case IPPROTO_SCTP:
1920
1921 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
1922 offset |= (sizeof(struct sctphdr) >> 2) <<
1923 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1924 break;
1925 case IPPROTO_UDP:
1926
1927 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
1928 offset |= (sizeof(struct udphdr) >> 2) <<
1929 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
1930 break;
1931 default:
1932 if (*tx_flags & I40E_TX_FLAGS_TSO)
1933 return -1;
1934 skb_checksum_help(skb);
1935 return 0;
1936 }
1937
1938 *td_cmd |= cmd;
1939 *td_offset |= offset;
1940
1941 return 1;
1942}
1943
1944
1945
1946
1947
1948
1949
1950
1951static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
1952 const u64 cd_type_cmd_tso_mss,
1953 const u32 cd_tunneling, const u32 cd_l2tag2)
1954{
1955 struct i40e_tx_context_desc *context_desc;
1956 int i = tx_ring->next_to_use;
1957
1958 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
1959 !cd_tunneling && !cd_l2tag2)
1960 return;
1961
1962
1963 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
1964
1965 i++;
1966 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1967
1968
1969 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
1970 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
1971 context_desc->rsvd = cpu_to_le16(0);
1972 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
1973}
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988bool __i40evf_chk_linearize(struct sk_buff *skb)
1989{
1990 const struct skb_frag_struct *frag, *stale;
1991 int nr_frags, sum;
1992
1993
1994 nr_frags = skb_shinfo(skb)->nr_frags;
1995 if (nr_frags < (I40E_MAX_BUFFER_TXD - 1))
1996 return false;
1997
1998
1999
2000
2001 nr_frags -= I40E_MAX_BUFFER_TXD - 2;
2002 frag = &skb_shinfo(skb)->frags[0];
2003
2004
2005
2006
2007
2008
2009
2010 sum = 1 - skb_shinfo(skb)->gso_size;
2011
2012
2013 sum += skb_frag_size(frag++);
2014 sum += skb_frag_size(frag++);
2015 sum += skb_frag_size(frag++);
2016 sum += skb_frag_size(frag++);
2017 sum += skb_frag_size(frag++);
2018
2019
2020
2021
2022 stale = &skb_shinfo(skb)->frags[0];
2023 for (;;) {
2024 sum += skb_frag_size(frag++);
2025
2026
2027 if (sum < 0)
2028 return true;
2029
2030 if (!nr_frags--)
2031 break;
2032
2033 sum -= skb_frag_size(stale++);
2034 }
2035
2036 return false;
2037}
2038
2039
2040
2041
2042
2043
2044
2045
2046int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2047{
2048 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2049
2050 smp_mb();
2051
2052
2053 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2054 return -EBUSY;
2055
2056
2057 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2058 ++tx_ring->tx_stats.restart_queue;
2059 return 0;
2060}
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
2073 struct i40e_tx_buffer *first, u32 tx_flags,
2074 const u8 hdr_len, u32 td_cmd, u32 td_offset)
2075{
2076 unsigned int data_len = skb->data_len;
2077 unsigned int size = skb_headlen(skb);
2078 struct skb_frag_struct *frag;
2079 struct i40e_tx_buffer *tx_bi;
2080 struct i40e_tx_desc *tx_desc;
2081 u16 i = tx_ring->next_to_use;
2082 u32 td_tag = 0;
2083 dma_addr_t dma;
2084
2085 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
2086 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2087 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2088 I40E_TX_FLAGS_VLAN_SHIFT;
2089 }
2090
2091 first->tx_flags = tx_flags;
2092
2093 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2094
2095 tx_desc = I40E_TX_DESC(tx_ring, i);
2096 tx_bi = first;
2097
2098 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2099 unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2100
2101 if (dma_mapping_error(tx_ring->dev, dma))
2102 goto dma_error;
2103
2104
2105 dma_unmap_len_set(tx_bi, len, size);
2106 dma_unmap_addr_set(tx_bi, dma, dma);
2107
2108
2109 max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1);
2110 tx_desc->buffer_addr = cpu_to_le64(dma);
2111
2112 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
2113 tx_desc->cmd_type_offset_bsz =
2114 build_ctob(td_cmd, td_offset,
2115 max_data, td_tag);
2116
2117 tx_desc++;
2118 i++;
2119
2120 if (i == tx_ring->count) {
2121 tx_desc = I40E_TX_DESC(tx_ring, 0);
2122 i = 0;
2123 }
2124
2125 dma += max_data;
2126 size -= max_data;
2127
2128 max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2129 tx_desc->buffer_addr = cpu_to_le64(dma);
2130 }
2131
2132 if (likely(!data_len))
2133 break;
2134
2135 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2136 size, td_tag);
2137
2138 tx_desc++;
2139 i++;
2140
2141 if (i == tx_ring->count) {
2142 tx_desc = I40E_TX_DESC(tx_ring, 0);
2143 i = 0;
2144 }
2145
2146 size = skb_frag_size(frag);
2147 data_len -= size;
2148
2149 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2150 DMA_TO_DEVICE);
2151
2152 tx_bi = &tx_ring->tx_bi[i];
2153 }
2154
2155 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
2156
2157 i++;
2158 if (i == tx_ring->count)
2159 i = 0;
2160
2161 tx_ring->next_to_use = i;
2162
2163 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2164
2165
2166 td_cmd |= I40E_TXD_CMD;
2167 tx_desc->cmd_type_offset_bsz =
2168 build_ctob(td_cmd, td_offset, size, td_tag);
2169
2170
2171
2172
2173
2174
2175
2176 wmb();
2177
2178
2179 first->next_to_watch = tx_desc;
2180
2181
2182 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
2183 writel(i, tx_ring->tail);
2184
2185
2186
2187
2188 mmiowb();
2189 }
2190
2191 return;
2192
2193dma_error:
2194 dev_info(tx_ring->dev, "TX DMA map failed\n");
2195
2196
2197 for (;;) {
2198 tx_bi = &tx_ring->tx_bi[i];
2199 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
2200 if (tx_bi == first)
2201 break;
2202 if (i == 0)
2203 i = tx_ring->count;
2204 i--;
2205 }
2206
2207 tx_ring->next_to_use = i;
2208}
2209
2210
2211
2212
2213
2214
2215
2216
2217static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2218 struct i40e_ring *tx_ring)
2219{
2220 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2221 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2222 struct i40e_tx_buffer *first;
2223 u32 td_offset = 0;
2224 u32 tx_flags = 0;
2225 __be16 protocol;
2226 u32 td_cmd = 0;
2227 u8 hdr_len = 0;
2228 int tso, count;
2229
2230
2231 prefetch(skb->data);
2232
2233 i40e_trace(xmit_frame_ring, skb, tx_ring);
2234
2235 count = i40e_xmit_descriptor_count(skb);
2236 if (i40e_chk_linearize(skb, count)) {
2237 if (__skb_linearize(skb)) {
2238 dev_kfree_skb_any(skb);
2239 return NETDEV_TX_OK;
2240 }
2241 count = i40e_txd_use_count(skb->len);
2242 tx_ring->tx_stats.tx_linearize++;
2243 }
2244
2245
2246
2247
2248
2249
2250
2251 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
2252 tx_ring->tx_stats.tx_busy++;
2253 return NETDEV_TX_BUSY;
2254 }
2255
2256
2257 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2258 first->skb = skb;
2259 first->bytecount = skb->len;
2260 first->gso_segs = 1;
2261
2262
2263 if (i40evf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2264 goto out_drop;
2265
2266
2267 protocol = vlan_get_protocol(skb);
2268
2269
2270 if (protocol == htons(ETH_P_IP))
2271 tx_flags |= I40E_TX_FLAGS_IPV4;
2272 else if (protocol == htons(ETH_P_IPV6))
2273 tx_flags |= I40E_TX_FLAGS_IPV6;
2274
2275 tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss);
2276
2277 if (tso < 0)
2278 goto out_drop;
2279 else if (tso)
2280 tx_flags |= I40E_TX_FLAGS_TSO;
2281
2282
2283 tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
2284 tx_ring, &cd_tunneling);
2285 if (tso < 0)
2286 goto out_drop;
2287
2288 skb_tx_timestamp(skb);
2289
2290
2291 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2292
2293 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2294 cd_tunneling, cd_l2tag2);
2295
2296 i40evf_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2297 td_cmd, td_offset);
2298
2299 return NETDEV_TX_OK;
2300
2301out_drop:
2302 i40e_trace(xmit_frame_ring_drop, first->skb, tx_ring);
2303 dev_kfree_skb_any(first->skb);
2304 first->skb = NULL;
2305 return NETDEV_TX_OK;
2306}
2307
2308
2309
2310
2311
2312
2313
2314
2315netdev_tx_t i40evf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2316{
2317 struct i40evf_adapter *adapter = netdev_priv(netdev);
2318 struct i40e_ring *tx_ring = &adapter->tx_rings[skb->queue_mapping];
2319
2320
2321
2322
2323 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2324 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2325 return NETDEV_TX_OK;
2326 skb->len = I40E_MIN_TX_LEN;
2327 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2328 }
2329
2330 return i40e_xmit_frame_ring(skb, tx_ring);
2331}
2332