1
2
3
4
5
6#include <stdarg.h>
7#include <stdio.h>
8#include <errno.h>
9#include <stdint.h>
10#include <unistd.h>
11#include <inttypes.h>
12
13#include <sys/queue.h>
14#include <sys/stat.h>
15
16#include <rte_common.h>
17#include <rte_byteorder.h>
18#include <rte_log.h>
19#include <rte_debug.h>
20#include <rte_cycles.h>
21#include <rte_memory.h>
22#include <rte_memcpy.h>
23#include <rte_launch.h>
24#include <rte_eal.h>
25#include <rte_per_lcore.h>
26#include <rte_lcore.h>
27#include <rte_atomic.h>
28#include <rte_branch_prediction.h>
29#include <rte_mempool.h>
30#include <rte_mbuf.h>
31#include <rte_interrupts.h>
32#include <rte_pci.h>
33#include <rte_ether.h>
34#include <rte_ethdev.h>
35#include <rte_ip.h>
36#include <rte_tcp.h>
37#include <rte_udp.h>
38#include <rte_vxlan.h>
39#include <rte_sctp.h>
40#include <rte_gtp.h>
41#include <rte_prefetch.h>
42#include <rte_string_fns.h>
43#include <rte_flow.h>
44#include <rte_gro.h>
45#include <rte_gso.h>
46#include <rte_geneve.h>
47
48#include "testpmd.h"
49
50#define IP_DEFTTL 64
51
52#define GRE_CHECKSUM_PRESENT 0x8000
53#define GRE_KEY_PRESENT 0x2000
54#define GRE_SEQUENCE_PRESENT 0x1000
55#define GRE_EXT_LEN 4
56#define GRE_SUPPORTED_FIELDS (GRE_CHECKSUM_PRESENT | GRE_KEY_PRESENT |\
57 GRE_SEQUENCE_PRESENT)
58
59
60#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
61#define _htons(x) ((uint16_t)((((x) & 0x00ffU) << 8) | (((x) & 0xff00U) >> 8)))
62#else
63#define _htons(x) (x)
64#endif
65
66uint16_t vxlan_gpe_udp_port = RTE_VXLAN_GPE_DEFAULT_PORT;
67uint16_t geneve_udp_port = RTE_GENEVE_DEFAULT_PORT;
68
69
70struct testpmd_offload_info {
71 uint16_t ethertype;
72 uint8_t gso_enable;
73 uint16_t l2_len;
74 uint16_t l3_len;
75 uint16_t l4_len;
76 uint8_t l4_proto;
77 uint8_t is_tunnel;
78 uint16_t outer_ethertype;
79 uint16_t outer_l2_len;
80 uint16_t outer_l3_len;
81 uint8_t outer_l4_proto;
82 uint16_t tso_segsz;
83 uint16_t tunnel_tso_segsz;
84 uint32_t pkt_len;
85};
86
87
88struct simple_gre_hdr {
89 uint16_t flags;
90 uint16_t proto;
91} __rte_packed;
92
93static uint16_t
94get_udptcp_checksum(void *l3_hdr, void *l4_hdr, uint16_t ethertype)
95{
96 if (ethertype == _htons(RTE_ETHER_TYPE_IPV4))
97 return rte_ipv4_udptcp_cksum(l3_hdr, l4_hdr);
98 else
99 return rte_ipv6_udptcp_cksum(l3_hdr, l4_hdr);
100}
101
102
103static void
104parse_ipv4(struct rte_ipv4_hdr *ipv4_hdr, struct testpmd_offload_info *info)
105{
106 struct rte_tcp_hdr *tcp_hdr;
107
108 info->l3_len = rte_ipv4_hdr_len(ipv4_hdr);
109 info->l4_proto = ipv4_hdr->next_proto_id;
110
111
112 if (info->l4_proto == IPPROTO_TCP) {
113 tcp_hdr = (struct rte_tcp_hdr *)
114 ((char *)ipv4_hdr + info->l3_len);
115 info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
116 } else if (info->l4_proto == IPPROTO_UDP)
117 info->l4_len = sizeof(struct rte_udp_hdr);
118 else
119 info->l4_len = 0;
120}
121
122
123static void
124parse_ipv6(struct rte_ipv6_hdr *ipv6_hdr, struct testpmd_offload_info *info)
125{
126 struct rte_tcp_hdr *tcp_hdr;
127
128 info->l3_len = sizeof(struct rte_ipv6_hdr);
129 info->l4_proto = ipv6_hdr->proto;
130
131
132 if (info->l4_proto == IPPROTO_TCP) {
133 tcp_hdr = (struct rte_tcp_hdr *)
134 ((char *)ipv6_hdr + info->l3_len);
135 info->l4_len = (tcp_hdr->data_off & 0xf0) >> 2;
136 } else if (info->l4_proto == IPPROTO_UDP)
137 info->l4_len = sizeof(struct rte_udp_hdr);
138 else
139 info->l4_len = 0;
140}
141
142
143
144
145
146
147static void
148parse_ethernet(struct rte_ether_hdr *eth_hdr, struct testpmd_offload_info *info)
149{
150 struct rte_ipv4_hdr *ipv4_hdr;
151 struct rte_ipv6_hdr *ipv6_hdr;
152 struct rte_vlan_hdr *vlan_hdr;
153
154 info->l2_len = sizeof(struct rte_ether_hdr);
155 info->ethertype = eth_hdr->ether_type;
156
157 while (info->ethertype == _htons(RTE_ETHER_TYPE_VLAN) ||
158 info->ethertype == _htons(RTE_ETHER_TYPE_QINQ)) {
159 vlan_hdr = (struct rte_vlan_hdr *)
160 ((char *)eth_hdr + info->l2_len);
161 info->l2_len += sizeof(struct rte_vlan_hdr);
162 info->ethertype = vlan_hdr->eth_proto;
163 }
164
165 switch (info->ethertype) {
166 case _htons(RTE_ETHER_TYPE_IPV4):
167 ipv4_hdr = (struct rte_ipv4_hdr *)
168 ((char *)eth_hdr + info->l2_len);
169 parse_ipv4(ipv4_hdr, info);
170 break;
171 case _htons(RTE_ETHER_TYPE_IPV6):
172 ipv6_hdr = (struct rte_ipv6_hdr *)
173 ((char *)eth_hdr + info->l2_len);
174 parse_ipv6(ipv6_hdr, info);
175 break;
176 default:
177 info->l4_len = 0;
178 info->l3_len = 0;
179 info->l4_proto = 0;
180 break;
181 }
182}
183
184
185static void
186update_tunnel_outer(struct testpmd_offload_info *info)
187{
188 info->is_tunnel = 1;
189 info->outer_ethertype = info->ethertype;
190 info->outer_l2_len = info->l2_len;
191 info->outer_l3_len = info->l3_len;
192 info->outer_l4_proto = info->l4_proto;
193}
194
195
196
197
198
199static void
200parse_gtp(struct rte_udp_hdr *udp_hdr,
201 struct testpmd_offload_info *info)
202{
203 struct rte_ipv4_hdr *ipv4_hdr;
204 struct rte_ipv6_hdr *ipv6_hdr;
205 struct rte_gtp_hdr *gtp_hdr;
206 uint8_t gtp_len = sizeof(*gtp_hdr);
207 uint8_t ip_ver;
208
209
210 if (udp_hdr->dst_port != _htons(RTE_GTPC_UDP_PORT) &&
211 udp_hdr->src_port != _htons(RTE_GTPC_UDP_PORT) &&
212 udp_hdr->dst_port != _htons(RTE_GTPU_UDP_PORT))
213 return;
214
215 update_tunnel_outer(info);
216 info->l2_len = 0;
217
218 gtp_hdr = (struct rte_gtp_hdr *)((char *)udp_hdr +
219 sizeof(struct rte_udp_hdr));
220
221
222
223
224
225 if (gtp_hdr->msg_type == 0xff) {
226 ip_ver = *(uint8_t *)((char *)udp_hdr +
227 sizeof(struct rte_udp_hdr) +
228 sizeof(struct rte_gtp_hdr));
229 ip_ver = (ip_ver) & 0xf0;
230
231 if (ip_ver == RTE_GTP_TYPE_IPV4) {
232 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)gtp_hdr +
233 gtp_len);
234 info->ethertype = _htons(RTE_ETHER_TYPE_IPV4);
235 parse_ipv4(ipv4_hdr, info);
236 } else if (ip_ver == RTE_GTP_TYPE_IPV6) {
237 ipv6_hdr = (struct rte_ipv6_hdr *)((char *)gtp_hdr +
238 gtp_len);
239 info->ethertype = _htons(RTE_ETHER_TYPE_IPV6);
240 parse_ipv6(ipv6_hdr, info);
241 }
242 } else {
243 info->ethertype = 0;
244 info->l4_len = 0;
245 info->l3_len = 0;
246 info->l4_proto = 0;
247 }
248
249 info->l2_len += RTE_ETHER_GTP_HLEN;
250}
251
252
253static void
254parse_vxlan(struct rte_udp_hdr *udp_hdr,
255 struct testpmd_offload_info *info,
256 uint32_t pkt_type)
257{
258 struct rte_ether_hdr *eth_hdr;
259
260
261
262
263
264 if (udp_hdr->dst_port != _htons(RTE_VXLAN_DEFAULT_PORT) &&
265 RTE_ETH_IS_TUNNEL_PKT(pkt_type) == 0)
266 return;
267
268 update_tunnel_outer(info);
269
270 eth_hdr = (struct rte_ether_hdr *)((char *)udp_hdr +
271 sizeof(struct rte_udp_hdr) +
272 sizeof(struct rte_vxlan_hdr));
273
274 parse_ethernet(eth_hdr, info);
275 info->l2_len += RTE_ETHER_VXLAN_HLEN;
276}
277
278
279static void
280parse_vxlan_gpe(struct rte_udp_hdr *udp_hdr,
281 struct testpmd_offload_info *info)
282{
283 struct rte_ether_hdr *eth_hdr;
284 struct rte_ipv4_hdr *ipv4_hdr;
285 struct rte_ipv6_hdr *ipv6_hdr;
286 struct rte_vxlan_gpe_hdr *vxlan_gpe_hdr;
287 uint8_t vxlan_gpe_len = sizeof(*vxlan_gpe_hdr);
288
289
290 if (udp_hdr->dst_port != _htons(vxlan_gpe_udp_port))
291 return;
292
293 vxlan_gpe_hdr = (struct rte_vxlan_gpe_hdr *)((char *)udp_hdr +
294 sizeof(struct rte_udp_hdr));
295
296 if (!vxlan_gpe_hdr->proto || vxlan_gpe_hdr->proto ==
297 RTE_VXLAN_GPE_TYPE_IPV4) {
298 update_tunnel_outer(info);
299
300 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)vxlan_gpe_hdr +
301 vxlan_gpe_len);
302
303 parse_ipv4(ipv4_hdr, info);
304 info->ethertype = _htons(RTE_ETHER_TYPE_IPV4);
305 info->l2_len = 0;
306
307 } else if (vxlan_gpe_hdr->proto == RTE_VXLAN_GPE_TYPE_IPV6) {
308 update_tunnel_outer(info);
309
310 ipv6_hdr = (struct rte_ipv6_hdr *)((char *)vxlan_gpe_hdr +
311 vxlan_gpe_len);
312
313 info->ethertype = _htons(RTE_ETHER_TYPE_IPV6);
314 parse_ipv6(ipv6_hdr, info);
315 info->l2_len = 0;
316
317 } else if (vxlan_gpe_hdr->proto == RTE_VXLAN_GPE_TYPE_ETH) {
318 update_tunnel_outer(info);
319
320 eth_hdr = (struct rte_ether_hdr *)((char *)vxlan_gpe_hdr +
321 vxlan_gpe_len);
322
323 parse_ethernet(eth_hdr, info);
324 } else
325 return;
326
327 info->l2_len += RTE_ETHER_VXLAN_GPE_HLEN;
328}
329
330
331static void
332parse_geneve(struct rte_udp_hdr *udp_hdr,
333 struct testpmd_offload_info *info)
334{
335 struct rte_ether_hdr *eth_hdr;
336 struct rte_ipv4_hdr *ipv4_hdr;
337 struct rte_ipv6_hdr *ipv6_hdr;
338 struct rte_geneve_hdr *geneve_hdr;
339 uint16_t geneve_len;
340
341
342 if (udp_hdr->dst_port != _htons(geneve_udp_port))
343 return;
344
345 geneve_hdr = (struct rte_geneve_hdr *)((char *)udp_hdr +
346 sizeof(struct rte_udp_hdr));
347 geneve_len = sizeof(struct rte_geneve_hdr) + geneve_hdr->opt_len * 4;
348 if (!geneve_hdr->proto || geneve_hdr->proto ==
349 _htons(RTE_ETHER_TYPE_IPV4)) {
350 update_tunnel_outer(info);
351 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)geneve_hdr +
352 geneve_len);
353 parse_ipv4(ipv4_hdr, info);
354 info->ethertype = _htons(RTE_ETHER_TYPE_IPV4);
355 info->l2_len = 0;
356 } else if (geneve_hdr->proto == _htons(RTE_ETHER_TYPE_IPV6)) {
357 update_tunnel_outer(info);
358 ipv6_hdr = (struct rte_ipv6_hdr *)((char *)geneve_hdr +
359 geneve_len);
360 info->ethertype = _htons(RTE_ETHER_TYPE_IPV6);
361 parse_ipv6(ipv6_hdr, info);
362 info->l2_len = 0;
363
364 } else if (geneve_hdr->proto == _htons(RTE_GENEVE_TYPE_ETH)) {
365 update_tunnel_outer(info);
366 eth_hdr = (struct rte_ether_hdr *)((char *)geneve_hdr +
367 geneve_len);
368 parse_ethernet(eth_hdr, info);
369 } else
370 return;
371
372 info->l2_len +=
373 (sizeof(struct rte_udp_hdr) + sizeof(struct rte_geneve_hdr) +
374 ((struct rte_geneve_hdr *)geneve_hdr)->opt_len * 4);
375}
376
377
378static void
379parse_gre(struct simple_gre_hdr *gre_hdr, struct testpmd_offload_info *info)
380{
381 struct rte_ether_hdr *eth_hdr;
382 struct rte_ipv4_hdr *ipv4_hdr;
383 struct rte_ipv6_hdr *ipv6_hdr;
384 uint8_t gre_len = 0;
385
386 gre_len += sizeof(struct simple_gre_hdr);
387
388 if (gre_hdr->flags & _htons(GRE_KEY_PRESENT))
389 gre_len += GRE_EXT_LEN;
390 if (gre_hdr->flags & _htons(GRE_SEQUENCE_PRESENT))
391 gre_len += GRE_EXT_LEN;
392 if (gre_hdr->flags & _htons(GRE_CHECKSUM_PRESENT))
393 gre_len += GRE_EXT_LEN;
394
395 if (gre_hdr->proto == _htons(RTE_ETHER_TYPE_IPV4)) {
396 update_tunnel_outer(info);
397
398 ipv4_hdr = (struct rte_ipv4_hdr *)((char *)gre_hdr + gre_len);
399
400 parse_ipv4(ipv4_hdr, info);
401 info->ethertype = _htons(RTE_ETHER_TYPE_IPV4);
402 info->l2_len = 0;
403
404 } else if (gre_hdr->proto == _htons(RTE_ETHER_TYPE_IPV6)) {
405 update_tunnel_outer(info);
406
407 ipv6_hdr = (struct rte_ipv6_hdr *)((char *)gre_hdr + gre_len);
408
409 info->ethertype = _htons(RTE_ETHER_TYPE_IPV6);
410 parse_ipv6(ipv6_hdr, info);
411 info->l2_len = 0;
412
413 } else if (gre_hdr->proto == _htons(RTE_ETHER_TYPE_TEB)) {
414 update_tunnel_outer(info);
415
416 eth_hdr = (struct rte_ether_hdr *)((char *)gre_hdr + gre_len);
417
418 parse_ethernet(eth_hdr, info);
419 } else
420 return;
421
422 info->l2_len += gre_len;
423}
424
425
426
427static void
428parse_encap_ip(void *encap_ip, struct testpmd_offload_info *info)
429{
430 struct rte_ipv4_hdr *ipv4_hdr = encap_ip;
431 struct rte_ipv6_hdr *ipv6_hdr = encap_ip;
432 uint8_t ip_version;
433
434 ip_version = (ipv4_hdr->version_ihl & 0xf0) >> 4;
435
436 if (ip_version != 4 && ip_version != 6)
437 return;
438
439 info->is_tunnel = 1;
440 info->outer_ethertype = info->ethertype;
441 info->outer_l2_len = info->l2_len;
442 info->outer_l3_len = info->l3_len;
443
444 if (ip_version == 4) {
445 parse_ipv4(ipv4_hdr, info);
446 info->ethertype = _htons(RTE_ETHER_TYPE_IPV4);
447 } else {
448 parse_ipv6(ipv6_hdr, info);
449 info->ethertype = _htons(RTE_ETHER_TYPE_IPV6);
450 }
451 info->l2_len = 0;
452}
453
454
455
456static uint64_t
457process_inner_cksums(void *l3_hdr, const struct testpmd_offload_info *info,
458 uint64_t tx_offloads)
459{
460 struct rte_ipv4_hdr *ipv4_hdr = l3_hdr;
461 struct rte_udp_hdr *udp_hdr;
462 struct rte_tcp_hdr *tcp_hdr;
463 struct rte_sctp_hdr *sctp_hdr;
464 uint64_t ol_flags = 0;
465 uint32_t max_pkt_len, tso_segsz = 0;
466
467
468 if (!info->is_tunnel) {
469 max_pkt_len = info->l2_len + info->l3_len + info->l4_len +
470 info->tso_segsz;
471 if (info->tso_segsz != 0 && info->pkt_len > max_pkt_len)
472 tso_segsz = info->tso_segsz;
473 } else {
474 max_pkt_len = info->outer_l2_len + info->outer_l3_len +
475 info->l2_len + info->l3_len + info->l4_len +
476 info->tunnel_tso_segsz;
477 if (info->tunnel_tso_segsz != 0 && info->pkt_len > max_pkt_len)
478 tso_segsz = info->tunnel_tso_segsz;
479 }
480
481 if (info->ethertype == _htons(RTE_ETHER_TYPE_IPV4)) {
482 ipv4_hdr = l3_hdr;
483
484 ol_flags |= PKT_TX_IPV4;
485 if (info->l4_proto == IPPROTO_TCP && tso_segsz) {
486 ol_flags |= PKT_TX_IP_CKSUM;
487 } else {
488 if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) {
489 ol_flags |= PKT_TX_IP_CKSUM;
490 } else {
491 ipv4_hdr->hdr_checksum = 0;
492 ipv4_hdr->hdr_checksum =
493 rte_ipv4_cksum(ipv4_hdr);
494 }
495 }
496 } else if (info->ethertype == _htons(RTE_ETHER_TYPE_IPV6))
497 ol_flags |= PKT_TX_IPV6;
498 else
499 return 0;
500
501 if (info->l4_proto == IPPROTO_UDP) {
502 udp_hdr = (struct rte_udp_hdr *)((char *)l3_hdr + info->l3_len);
503
504 if (udp_hdr->dgram_cksum != 0) {
505 if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
506 ol_flags |= PKT_TX_UDP_CKSUM;
507 } else {
508 udp_hdr->dgram_cksum = 0;
509 udp_hdr->dgram_cksum =
510 get_udptcp_checksum(l3_hdr, udp_hdr,
511 info->ethertype);
512 }
513 }
514 if (info->gso_enable)
515 ol_flags |= PKT_TX_UDP_SEG;
516 } else if (info->l4_proto == IPPROTO_TCP) {
517 tcp_hdr = (struct rte_tcp_hdr *)((char *)l3_hdr + info->l3_len);
518 if (tso_segsz)
519 ol_flags |= PKT_TX_TCP_SEG;
520 else if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) {
521 ol_flags |= PKT_TX_TCP_CKSUM;
522 } else {
523 tcp_hdr->cksum = 0;
524 tcp_hdr->cksum =
525 get_udptcp_checksum(l3_hdr, tcp_hdr,
526 info->ethertype);
527 }
528 if (info->gso_enable)
529 ol_flags |= PKT_TX_TCP_SEG;
530 } else if (info->l4_proto == IPPROTO_SCTP) {
531 sctp_hdr = (struct rte_sctp_hdr *)
532 ((char *)l3_hdr + info->l3_len);
533
534
535 if ((tx_offloads & DEV_TX_OFFLOAD_SCTP_CKSUM) &&
536 ((ipv4_hdr->total_length & 0x3) == 0)) {
537 ol_flags |= PKT_TX_SCTP_CKSUM;
538 } else {
539 sctp_hdr->cksum = 0;
540
541
542 }
543 }
544
545 return ol_flags;
546}
547
548
549static uint64_t
550process_outer_cksums(void *outer_l3_hdr, struct testpmd_offload_info *info,
551 uint64_t tx_offloads, int tso_enabled)
552{
553 struct rte_ipv4_hdr *ipv4_hdr = outer_l3_hdr;
554 struct rte_ipv6_hdr *ipv6_hdr = outer_l3_hdr;
555 struct rte_udp_hdr *udp_hdr;
556 uint64_t ol_flags = 0;
557
558 if (info->outer_ethertype == _htons(RTE_ETHER_TYPE_IPV4)) {
559 ipv4_hdr->hdr_checksum = 0;
560 ol_flags |= PKT_TX_OUTER_IPV4;
561
562 if (tx_offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)
563 ol_flags |= PKT_TX_OUTER_IP_CKSUM;
564 else
565 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr);
566 } else
567 ol_flags |= PKT_TX_OUTER_IPV6;
568
569 if (info->outer_l4_proto != IPPROTO_UDP)
570 return ol_flags;
571
572 udp_hdr = (struct rte_udp_hdr *)
573 ((char *)outer_l3_hdr + info->outer_l3_len);
574
575 if (tso_enabled)
576 ol_flags |= PKT_TX_TCP_SEG;
577
578
579 if (tx_offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) {
580 if (info->outer_ethertype == _htons(RTE_ETHER_TYPE_IPV4))
581 udp_hdr->dgram_cksum
582 = rte_ipv4_phdr_cksum(ipv4_hdr, ol_flags);
583 else
584 udp_hdr->dgram_cksum
585 = rte_ipv6_phdr_cksum(ipv6_hdr, ol_flags);
586
587 ol_flags |= PKT_TX_OUTER_UDP_CKSUM;
588 return ol_flags;
589 }
590
591
592
593
594
595
596
597
598
599 if (tso_enabled)
600 udp_hdr->dgram_cksum = 0;
601
602
603 if (udp_hdr->dgram_cksum != 0) {
604 udp_hdr->dgram_cksum = 0;
605 if (info->outer_ethertype == _htons(RTE_ETHER_TYPE_IPV4))
606 udp_hdr->dgram_cksum =
607 rte_ipv4_udptcp_cksum(ipv4_hdr, udp_hdr);
608 else
609 udp_hdr->dgram_cksum =
610 rte_ipv6_udptcp_cksum(ipv6_hdr, udp_hdr);
611 }
612
613 return ol_flags;
614}
615
616
617
618
619
620
621
622static int
623mbuf_copy_split(const struct rte_mbuf *ms, struct rte_mbuf *md[],
624 uint16_t seglen[], uint8_t nb_seg)
625{
626 uint32_t dlen, slen, tlen;
627 uint32_t i, len;
628 const struct rte_mbuf *m;
629 const uint8_t *src;
630 uint8_t *dst;
631
632 dlen = 0;
633 slen = 0;
634 tlen = 0;
635
636 dst = NULL;
637 src = NULL;
638
639 m = ms;
640 i = 0;
641 while (ms != NULL && i != nb_seg) {
642
643 if (slen == 0) {
644 slen = rte_pktmbuf_data_len(ms);
645 src = rte_pktmbuf_mtod(ms, const uint8_t *);
646 }
647
648 if (dlen == 0) {
649 dlen = RTE_MIN(seglen[i], slen);
650 md[i]->data_len = dlen;
651 md[i]->next = (i + 1 == nb_seg) ? NULL : md[i + 1];
652 dst = rte_pktmbuf_mtod(md[i], uint8_t *);
653 }
654
655 len = RTE_MIN(slen, dlen);
656 memcpy(dst, src, len);
657 tlen += len;
658 slen -= len;
659 dlen -= len;
660 src += len;
661 dst += len;
662
663 if (slen == 0)
664 ms = ms->next;
665 if (dlen == 0)
666 i++;
667 }
668
669 if (ms != NULL)
670 return -ENOBUFS;
671 else if (tlen != m->pkt_len)
672 return -EINVAL;
673
674 md[0]->nb_segs = nb_seg;
675 md[0]->pkt_len = tlen;
676 md[0]->vlan_tci = m->vlan_tci;
677 md[0]->vlan_tci_outer = m->vlan_tci_outer;
678 md[0]->ol_flags = m->ol_flags;
679 md[0]->tx_offload = m->tx_offload;
680
681 return nb_seg;
682}
683
684
685
686
687
688static struct rte_mbuf *
689pkt_copy_split(const struct rte_mbuf *pkt)
690{
691 int32_t n, rc;
692 uint32_t i, len, nb_seg;
693 struct rte_mempool *mp;
694 uint16_t seglen[RTE_MAX_SEGS_PER_PKT];
695 struct rte_mbuf *p, *md[RTE_MAX_SEGS_PER_PKT];
696
697 mp = current_fwd_lcore()->mbp;
698
699 if (tx_pkt_split == TX_PKT_SPLIT_RND)
700 nb_seg = rte_rand() % tx_pkt_nb_segs + 1;
701 else
702 nb_seg = tx_pkt_nb_segs;
703
704 memcpy(seglen, tx_pkt_seg_lengths, nb_seg * sizeof(seglen[0]));
705
706
707 len = 0;
708 for (i = 0; i != nb_seg && len < pkt->pkt_len; i++) {
709 len += seglen[i];
710 md[i] = NULL;
711 }
712
713 n = pkt->pkt_len - len;
714
715
716 if (n >= 0) {
717 seglen[i - 1] += n;
718 len += n;
719 }
720
721 nb_seg = i;
722 while (i != 0) {
723 p = rte_pktmbuf_alloc(mp);
724 if (p == NULL) {
725 TESTPMD_LOG(ERR,
726 "failed to allocate %u-th of %u mbuf "
727 "from mempool: %s\n",
728 nb_seg - i, nb_seg, mp->name);
729 break;
730 }
731
732 md[--i] = p;
733 if (rte_pktmbuf_tailroom(md[i]) < seglen[i]) {
734 TESTPMD_LOG(ERR, "mempool %s, %u-th segment: "
735 "expected seglen: %u, "
736 "actual mbuf tailroom: %u\n",
737 mp->name, i, seglen[i],
738 rte_pktmbuf_tailroom(md[i]));
739 break;
740 }
741 }
742
743
744 if (i == 0) {
745 rc = mbuf_copy_split(pkt, md, seglen, nb_seg);
746 if (rc < 0)
747 TESTPMD_LOG(ERR,
748 "mbuf_copy_split for %p(len=%u, nb_seg=%u) "
749 "into %u segments failed with error code: %d\n",
750 pkt, pkt->pkt_len, pkt->nb_segs, nb_seg, rc);
751
752
753 i = RTE_MAX(rc, 0);
754 }
755
756
757 for (; i != nb_seg; i++) {
758 rte_pktmbuf_free_seg(md[i]);
759 md[i] = NULL;
760 }
761
762 return md[0];
763}
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794static void
795pkt_burst_checksum_forward(struct fwd_stream *fs)
796{
797 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
798 struct rte_mbuf *gso_segments[GSO_MAX_PKT_BURST];
799 struct rte_gso_ctx *gso_ctx;
800 struct rte_mbuf **tx_pkts_burst;
801 struct rte_port *txp;
802 struct rte_mbuf *m, *p;
803 struct rte_ether_hdr *eth_hdr;
804 void *l3_hdr = NULL, *outer_l3_hdr = NULL;
805 void **gro_ctx;
806 uint16_t gro_pkts_num;
807 uint8_t gro_enable;
808 uint16_t nb_rx;
809 uint16_t nb_tx;
810 uint16_t nb_prep;
811 uint16_t i;
812 uint64_t rx_ol_flags, tx_ol_flags;
813 uint64_t tx_offloads;
814 uint32_t retry;
815 uint32_t rx_bad_ip_csum;
816 uint32_t rx_bad_l4_csum;
817 uint32_t rx_bad_outer_l4_csum;
818 uint32_t rx_bad_outer_ip_csum;
819 struct testpmd_offload_info info;
820 uint16_t nb_segments = 0;
821 int ret;
822
823 uint64_t start_tsc = 0;
824
825 get_start_cycles(&start_tsc);
826
827
828 nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, pkts_burst,
829 nb_pkt_per_burst);
830 inc_rx_burst_stats(fs, nb_rx);
831 if (unlikely(nb_rx == 0))
832 return;
833
834 fs->rx_packets += nb_rx;
835 rx_bad_ip_csum = 0;
836 rx_bad_l4_csum = 0;
837 rx_bad_outer_l4_csum = 0;
838 rx_bad_outer_ip_csum = 0;
839 gro_enable = gro_ports[fs->rx_port].enable;
840
841 txp = &ports[fs->tx_port];
842 tx_offloads = txp->dev_conf.txmode.offloads;
843 memset(&info, 0, sizeof(info));
844 info.tso_segsz = txp->tso_segsz;
845 info.tunnel_tso_segsz = txp->tunnel_tso_segsz;
846 if (gso_ports[fs->tx_port].enable)
847 info.gso_enable = 1;
848
849 for (i = 0; i < nb_rx; i++) {
850 if (likely(i < nb_rx - 1))
851 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1],
852 void *));
853
854 m = pkts_burst[i];
855 info.is_tunnel = 0;
856 info.pkt_len = rte_pktmbuf_pkt_len(m);
857 tx_ol_flags = m->ol_flags &
858 (IND_ATTACHED_MBUF | EXT_ATTACHED_MBUF);
859 rx_ol_flags = m->ol_flags;
860
861
862 if ((rx_ol_flags & PKT_RX_IP_CKSUM_MASK) == PKT_RX_IP_CKSUM_BAD)
863 rx_bad_ip_csum += 1;
864 if ((rx_ol_flags & PKT_RX_L4_CKSUM_MASK) == PKT_RX_L4_CKSUM_BAD)
865 rx_bad_l4_csum += 1;
866 if (rx_ol_flags & PKT_RX_OUTER_L4_CKSUM_BAD)
867 rx_bad_outer_l4_csum += 1;
868 if (rx_ol_flags & PKT_RX_OUTER_IP_CKSUM_BAD)
869 rx_bad_outer_ip_csum += 1;
870
871
872
873
874 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
875 rte_ether_addr_copy(&peer_eth_addrs[fs->peer_addr],
876 ð_hdr->d_addr);
877 rte_ether_addr_copy(&ports[fs->tx_port].eth_addr,
878 ð_hdr->s_addr);
879 parse_ethernet(eth_hdr, &info);
880 l3_hdr = (char *)eth_hdr + info.l2_len;
881
882
883 if (txp->parse_tunnel) {
884 if (info.l4_proto == IPPROTO_UDP) {
885 struct rte_udp_hdr *udp_hdr;
886
887 udp_hdr = (struct rte_udp_hdr *)
888 ((char *)l3_hdr + info.l3_len);
889 parse_gtp(udp_hdr, &info);
890 if (info.is_tunnel) {
891 tx_ol_flags |= PKT_TX_TUNNEL_GTP;
892 goto tunnel_update;
893 }
894 parse_vxlan_gpe(udp_hdr, &info);
895 if (info.is_tunnel) {
896 tx_ol_flags |=
897 PKT_TX_TUNNEL_VXLAN_GPE;
898 goto tunnel_update;
899 }
900 parse_vxlan(udp_hdr, &info,
901 m->packet_type);
902 if (info.is_tunnel) {
903 tx_ol_flags |=
904 PKT_TX_TUNNEL_VXLAN;
905 goto tunnel_update;
906 }
907 parse_geneve(udp_hdr, &info);
908 if (info.is_tunnel) {
909 tx_ol_flags |=
910 PKT_TX_TUNNEL_GENEVE;
911 goto tunnel_update;
912 }
913 } else if (info.l4_proto == IPPROTO_GRE) {
914 struct simple_gre_hdr *gre_hdr;
915
916 gre_hdr = (struct simple_gre_hdr *)
917 ((char *)l3_hdr + info.l3_len);
918 parse_gre(gre_hdr, &info);
919 if (info.is_tunnel)
920 tx_ol_flags |= PKT_TX_TUNNEL_GRE;
921 } else if (info.l4_proto == IPPROTO_IPIP) {
922 void *encap_ip_hdr;
923
924 encap_ip_hdr = (char *)l3_hdr + info.l3_len;
925 parse_encap_ip(encap_ip_hdr, &info);
926 if (info.is_tunnel)
927 tx_ol_flags |= PKT_TX_TUNNEL_IPIP;
928 }
929 }
930
931tunnel_update:
932
933 if (info.is_tunnel) {
934 outer_l3_hdr = l3_hdr;
935 l3_hdr = (char *)l3_hdr + info.outer_l3_len + info.l2_len;
936 }
937
938
939
940
941
942
943
944 tx_ol_flags |= process_inner_cksums(l3_hdr, &info,
945 tx_offloads);
946
947
948
949
950 if (info.is_tunnel == 1) {
951 tx_ol_flags |= process_outer_cksums(outer_l3_hdr, &info,
952 tx_offloads,
953 !!(tx_ol_flags & PKT_TX_TCP_SEG));
954 }
955
956
957
958 m->tx_offload = 0;
959 if (info.is_tunnel == 1) {
960 if (info.tunnel_tso_segsz ||
961 (tx_offloads &
962 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) ||
963 (tx_offloads &
964 DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) ||
965 (tx_ol_flags & PKT_TX_OUTER_IPV6)) {
966 m->outer_l2_len = info.outer_l2_len;
967 m->outer_l3_len = info.outer_l3_len;
968 m->l2_len = info.l2_len;
969 m->l3_len = info.l3_len;
970 m->l4_len = info.l4_len;
971 m->tso_segsz = info.tunnel_tso_segsz;
972 }
973 else {
974
975
976
977
978
979 m->l2_len = info.outer_l2_len +
980 info.outer_l3_len + info.l2_len;
981 m->l3_len = info.l3_len;
982 m->l4_len = info.l4_len;
983 }
984 } else {
985
986
987
988 m->l2_len = info.l2_len;
989 m->l3_len = info.l3_len;
990 m->l4_len = info.l4_len;
991 m->tso_segsz = info.tso_segsz;
992 }
993 m->ol_flags = tx_ol_flags;
994
995
996 if (tx_pkt_split != TX_PKT_SPLIT_OFF) {
997 p = pkt_copy_split(m);
998 if (p != NULL) {
999 rte_pktmbuf_free(m);
1000 m = p;
1001 pkts_burst[i] = m;
1002 }
1003 }
1004
1005
1006 if (verbose_level > 0) {
1007 char buf[256];
1008
1009 printf("-----------------\n");
1010 printf("port=%u, mbuf=%p, pkt_len=%u, nb_segs=%u:\n",
1011 fs->rx_port, m, m->pkt_len, m->nb_segs);
1012
1013 rte_get_rx_ol_flag_list(rx_ol_flags, buf, sizeof(buf));
1014 printf("rx: l2_len=%d ethertype=%x l3_len=%d "
1015 "l4_proto=%d l4_len=%d flags=%s\n",
1016 info.l2_len, rte_be_to_cpu_16(info.ethertype),
1017 info.l3_len, info.l4_proto, info.l4_len, buf);
1018 if (rx_ol_flags & PKT_RX_LRO)
1019 printf("rx: m->lro_segsz=%u\n", m->tso_segsz);
1020 if (info.is_tunnel == 1)
1021 printf("rx: outer_l2_len=%d outer_ethertype=%x "
1022 "outer_l3_len=%d\n", info.outer_l2_len,
1023 rte_be_to_cpu_16(info.outer_ethertype),
1024 info.outer_l3_len);
1025
1026 if ((tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM |
1027 DEV_TX_OFFLOAD_UDP_CKSUM |
1028 DEV_TX_OFFLOAD_TCP_CKSUM |
1029 DEV_TX_OFFLOAD_SCTP_CKSUM)) ||
1030 info.tso_segsz != 0)
1031 printf("tx: m->l2_len=%d m->l3_len=%d "
1032 "m->l4_len=%d\n",
1033 m->l2_len, m->l3_len, m->l4_len);
1034 if (info.is_tunnel == 1) {
1035 if ((tx_offloads &
1036 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) ||
1037 (tx_offloads &
1038 DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) ||
1039 (tx_ol_flags & PKT_TX_OUTER_IPV6))
1040 printf("tx: m->outer_l2_len=%d "
1041 "m->outer_l3_len=%d\n",
1042 m->outer_l2_len,
1043 m->outer_l3_len);
1044 if (info.tunnel_tso_segsz != 0 &&
1045 (m->ol_flags & PKT_TX_TCP_SEG))
1046 printf("tx: m->tso_segsz=%d\n",
1047 m->tso_segsz);
1048 } else if (info.tso_segsz != 0 &&
1049 (m->ol_flags & PKT_TX_TCP_SEG))
1050 printf("tx: m->tso_segsz=%d\n", m->tso_segsz);
1051 rte_get_tx_ol_flag_list(m->ol_flags, buf, sizeof(buf));
1052 printf("tx: flags=%s", buf);
1053 printf("\n");
1054 }
1055 }
1056
1057 if (unlikely(gro_enable)) {
1058 if (gro_flush_cycles == GRO_DEFAULT_FLUSH_CYCLES) {
1059 nb_rx = rte_gro_reassemble_burst(pkts_burst, nb_rx,
1060 &(gro_ports[fs->rx_port].param));
1061 } else {
1062 gro_ctx = current_fwd_lcore()->gro_ctx;
1063 nb_rx = rte_gro_reassemble(pkts_burst, nb_rx, gro_ctx);
1064
1065 if (++fs->gro_times >= gro_flush_cycles) {
1066 gro_pkts_num = rte_gro_get_pkt_count(gro_ctx);
1067 if (gro_pkts_num > MAX_PKT_BURST - nb_rx)
1068 gro_pkts_num = MAX_PKT_BURST - nb_rx;
1069
1070 nb_rx += rte_gro_timeout_flush(gro_ctx, 0,
1071 RTE_GRO_TCP_IPV4,
1072 &pkts_burst[nb_rx],
1073 gro_pkts_num);
1074 fs->gro_times = 0;
1075 }
1076 }
1077 }
1078
1079 if (gso_ports[fs->tx_port].enable == 0)
1080 tx_pkts_burst = pkts_burst;
1081 else {
1082 gso_ctx = &(current_fwd_lcore()->gso_ctx);
1083 gso_ctx->gso_size = gso_max_segment_size;
1084 for (i = 0; i < nb_rx; i++) {
1085 ret = rte_gso_segment(pkts_burst[i], gso_ctx,
1086 &gso_segments[nb_segments],
1087 GSO_MAX_PKT_BURST - nb_segments);
1088 if (ret >= 1) {
1089
1090 rte_pktmbuf_free(pkts_burst[i]);
1091 nb_segments += ret;
1092 } else if (ret == 0) {
1093
1094
1095
1096 gso_segments[nb_segments] = pkts_burst[i];
1097 nb_segments += 1;
1098 } else {
1099 TESTPMD_LOG(DEBUG, "Unable to segment packet");
1100 rte_pktmbuf_free(pkts_burst[i]);
1101 }
1102 }
1103
1104 tx_pkts_burst = gso_segments;
1105 nb_rx = nb_segments;
1106 }
1107
1108 nb_prep = rte_eth_tx_prepare(fs->tx_port, fs->tx_queue,
1109 tx_pkts_burst, nb_rx);
1110 if (nb_prep != nb_rx)
1111 fprintf(stderr,
1112 "Preparing packet burst to transmit failed: %s\n",
1113 rte_strerror(rte_errno));
1114
1115 nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, tx_pkts_burst,
1116 nb_prep);
1117
1118
1119
1120
1121 if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
1122 retry = 0;
1123 while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
1124 rte_delay_us(burst_tx_delay_time);
1125 nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
1126 &tx_pkts_burst[nb_tx], nb_rx - nb_tx);
1127 }
1128 }
1129 fs->tx_packets += nb_tx;
1130 fs->rx_bad_ip_csum += rx_bad_ip_csum;
1131 fs->rx_bad_l4_csum += rx_bad_l4_csum;
1132 fs->rx_bad_outer_l4_csum += rx_bad_outer_l4_csum;
1133 fs->rx_bad_outer_ip_csum += rx_bad_outer_ip_csum;
1134
1135 inc_tx_burst_stats(fs, nb_tx);
1136 if (unlikely(nb_tx < nb_rx)) {
1137 fs->fwd_dropped += (nb_rx - nb_tx);
1138 do {
1139 rte_pktmbuf_free(tx_pkts_burst[nb_tx]);
1140 } while (++nb_tx < nb_rx);
1141 }
1142
1143 get_end_cycles(fs, start_tsc);
1144}
1145
1146struct fwd_engine csum_fwd_engine = {
1147 .fwd_mode_name = "csum",
1148 .port_fwd_begin = NULL,
1149 .port_fwd_end = NULL,
1150 .packet_fwd = pkt_burst_checksum_forward,
1151};
1152
