1
2
3
4
5#include <stdarg.h>
6#include <stdio.h>
7#include <stdlib.h>
8#include <signal.h>
9#include <string.h>
10#include <time.h>
11#include <fcntl.h>
12#ifndef RTE_EXEC_ENV_WINDOWS
13#include <sys/mman.h>
14#endif
15#include <sys/types.h>
16#include <errno.h>
17#include <stdbool.h>
18
19#include <sys/queue.h>
20#include <sys/stat.h>
21
22#include <stdint.h>
23#include <unistd.h>
24#include <inttypes.h>
25
26#include <rte_common.h>
27#include <rte_errno.h>
28#include <rte_byteorder.h>
29#include <rte_log.h>
30#include <rte_debug.h>
31#include <rte_cycles.h>
32#include <rte_memory.h>
33#include <rte_memcpy.h>
34#include <rte_launch.h>
35#include <rte_eal.h>
36#include <rte_alarm.h>
37#include <rte_per_lcore.h>
38#include <rte_lcore.h>
39#include <rte_branch_prediction.h>
40#include <rte_mempool.h>
41#include <rte_malloc.h>
42#include <rte_mbuf.h>
43#include <rte_mbuf_pool_ops.h>
44#include <rte_interrupts.h>
45#include <rte_pci.h>
46#include <rte_ether.h>
47#include <rte_ethdev.h>
48#include <rte_dev.h>
49#include <rte_string_fns.h>
50#ifdef RTE_NET_IXGBE
51#include <rte_pmd_ixgbe.h>
52#endif
53#ifdef RTE_LIB_PDUMP
54#include <rte_pdump.h>
55#endif
56#include <rte_flow.h>
57#ifdef RTE_LIB_METRICS
58#include <rte_metrics.h>
59#endif
60#ifdef RTE_LIB_BITRATESTATS
61#include <rte_bitrate.h>
62#endif
63#ifdef RTE_LIB_LATENCYSTATS
64#include <rte_latencystats.h>
65#endif
66#ifdef RTE_EXEC_ENV_WINDOWS
67#include <process.h>
68#endif
69
70#include "testpmd.h"
71
72#ifndef MAP_HUGETLB
73
74#define HUGE_FLAG (0x40000)
75#else
76#define HUGE_FLAG MAP_HUGETLB
77#endif
78
79#ifndef MAP_HUGE_SHIFT
80
81#define HUGE_SHIFT (26)
82#else
83#define HUGE_SHIFT MAP_HUGE_SHIFT
84#endif
85
86#define EXTMEM_HEAP_NAME "extmem"
87#define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
88
89uint16_t verbose_level = 0;
90int testpmd_logtype;
91
92
93uint8_t interactive = 0;
94uint8_t auto_start = 0;
95uint8_t tx_first;
96char cmdline_filename[PATH_MAX] = {0};
97
98
99
100
101
102
103
104
105uint8_t numa_support = 1;
106
107
108
109
110
111uint8_t socket_num = UMA_NO_CONFIG;
112
113
114
115
116
117
118
119
120uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
121
122
123
124
125
126uint8_t port_numa[RTE_MAX_ETHPORTS];
127
128
129
130
131
132uint8_t rxring_numa[RTE_MAX_ETHPORTS];
133
134
135
136
137
138uint8_t txring_numa[RTE_MAX_ETHPORTS];
139
140
141
142
143
144
145
146struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
147portid_t nb_peer_eth_addrs = 0;
148
149
150
151
152struct rte_port *ports;
153portid_t nb_ports;
154struct fwd_lcore **fwd_lcores;
155lcoreid_t nb_lcores;
156
157portid_t ports_ids[RTE_MAX_ETHPORTS];
158
159
160
161
162
163
164lcoreid_t nb_cfg_lcores;
165lcoreid_t nb_fwd_lcores;
166portid_t nb_cfg_ports;
167portid_t nb_fwd_ports;
168
169unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE];
170portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];
171
172struct fwd_stream **fwd_streams;
173streamid_t nb_fwd_streams;
174
175
176
177
178struct fwd_engine * fwd_engines[] = {
179 &io_fwd_engine,
180 &mac_fwd_engine,
181 &mac_swap_engine,
182 &flow_gen_engine,
183 &rx_only_engine,
184 &tx_only_engine,
185 &csum_fwd_engine,
186 &icmp_echo_engine,
187 &noisy_vnf_engine,
188 &five_tuple_swap_fwd_engine,
189#ifdef RTE_LIBRTE_IEEE1588
190 &ieee1588_fwd_engine,
191#endif
192 &shared_rxq_engine,
193 NULL,
194};
195
196struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
197uint16_t mempool_flags;
198
199struct fwd_config cur_fwd_config;
200struct fwd_engine *cur_fwd_eng = &io_fwd_engine;
201uint32_t retry_enabled;
202uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
203uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
204
205uint32_t mbuf_data_size_n = 1;
206uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
207 DEFAULT_MBUF_DATA_SIZE
208};
209uint32_t param_total_num_mbufs = 0;
210
211uint16_t stats_period;
212
213
214struct rte_eth_xstat_name *xstats_display;
215
216unsigned int xstats_display_num;
217
218
219
220
221
222uint8_t f_quit;
223
224
225
226
227uint32_t max_rx_pkt_len;
228
229
230
231
232
233uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
234uint8_t rx_pkt_nb_segs;
235uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
236uint8_t rx_pkt_nb_offs;
237
238
239
240
241uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN;
242uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
243 TXONLY_DEF_PACKET_LEN,
244};
245uint8_t tx_pkt_nb_segs = 1;
246
247enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
248
249
250uint8_t txonly_multi_flow;
251
252
253uint32_t tx_pkt_times_inter;
254
255
256uint32_t tx_pkt_times_intra;
257
258
259uint16_t nb_pkt_per_burst = DEF_PKT_BURST;
260uint16_t nb_pkt_flowgen_clones;
261int nb_flows_flowgen = 1024;
262uint16_t mb_mempool_cache = DEF_MBUF_CACHE;
263
264
265uint8_t dcb_config = 0;
266
267
268
269
270queueid_t nb_hairpinq;
271queueid_t nb_rxq = 1;
272queueid_t nb_txq = 1;
273
274
275
276
277
278#define RTE_TEST_RX_DESC_DEFAULT 0
279#define RTE_TEST_TX_DESC_DEFAULT 0
280uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
281uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
282
283#define RTE_PMD_PARAM_UNSET -1
284
285
286
287
288int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
289int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
290int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
291
292int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
293int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
294int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
295
296
297
298
299int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
300
301
302
303
304int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
305
306
307
308
309int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
310
311
312
313
314int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
315
316
317
318
319uint16_t noisy_tx_sw_bufsz;
320
321
322
323
324uint16_t noisy_tx_sw_buf_flush_time;
325
326
327
328
329
330uint64_t noisy_lkup_mem_sz;
331
332
333
334
335
336uint64_t noisy_lkup_num_writes;
337
338
339
340
341
342uint64_t noisy_lkup_num_reads;
343
344
345
346
347
348uint64_t noisy_lkup_num_reads_writes;
349
350
351
352
353uint64_t rss_hf = RTE_ETH_RSS_IP;
354
355
356
357
358uint16_t port_topology = PORT_TOPOLOGY_PAIRED;
359
360
361
362
363uint8_t no_flush_rx = 0;
364
365
366
367
368uint8_t flow_isolate_all;
369
370
371
372
373uint8_t no_link_check = 0;
374
375
376
377
378uint8_t no_device_start = 0;
379
380
381
382
383uint8_t lsc_interrupt = 1;
384
385
386
387
388uint8_t rmv_interrupt = 1;
389
390uint8_t hot_plug = 0;
391
392
393bool setup_on_probe_event = true;
394
395
396uint8_t clear_ptypes = true;
397
398
399uint16_t hairpin_mode;
400
401
402static const char * const eth_event_desc[] = {
403 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
404 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
405 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
406 [RTE_ETH_EVENT_INTR_RESET] = "reset",
407 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
408 [RTE_ETH_EVENT_IPSEC] = "IPsec",
409 [RTE_ETH_EVENT_MACSEC] = "MACsec",
410 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
411 [RTE_ETH_EVENT_NEW] = "device probed",
412 [RTE_ETH_EVENT_DESTROY] = "device released",
413 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
414 [RTE_ETH_EVENT_MAX] = NULL,
415};
416
417
418
419
420
421uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
422 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
423 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
424 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
425 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
426 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
427 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
428 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
429
430
431
432int do_mlockall = 0;
433
434
435
436
437
438#if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
439
440uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
441#endif
442
443
444#ifdef RTE_LIB_LATENCYSTATS
445
446
447
448
449uint8_t latencystats_enabled;
450
451
452
453
454lcoreid_t latencystats_lcore_id = -1;
455
456#endif
457
458
459
460
461struct rte_eth_rxmode rx_mode;
462
463struct rte_eth_txmode tx_mode = {
464 .offloads = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE,
465};
466
467struct rte_eth_fdir_conf fdir_conf = {
468 .mode = RTE_FDIR_MODE_NONE,
469 .pballoc = RTE_ETH_FDIR_PBALLOC_64K,
470 .status = RTE_FDIR_REPORT_STATUS,
471 .mask = {
472 .vlan_tci_mask = 0xFFEF,
473 .ipv4_mask = {
474 .src_ip = 0xFFFFFFFF,
475 .dst_ip = 0xFFFFFFFF,
476 },
477 .ipv6_mask = {
478 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
479 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
480 },
481 .src_port_mask = 0xFFFF,
482 .dst_port_mask = 0xFFFF,
483 .mac_addr_byte_mask = 0xFF,
484 .tunnel_type_mask = 1,
485 .tunnel_id_mask = 0xFFFFFFFF,
486 },
487 .drop_queue = 127,
488};
489
490volatile int test_done = 1;
491
492
493
494
495uint8_t xstats_hide_zero;
496
497
498
499
500uint8_t record_core_cycles;
501
502
503
504
505uint8_t record_burst_stats;
506
507
508
509
510uint32_t rxq_share;
511
512unsigned int num_sockets = 0;
513unsigned int socket_ids[RTE_MAX_NUMA_NODES];
514
515#ifdef RTE_LIB_BITRATESTATS
516
517struct rte_stats_bitrates *bitrate_data;
518lcoreid_t bitrate_lcore_id;
519uint8_t bitrate_enabled;
520#endif
521
522#ifdef RTE_LIB_GRO
523struct gro_status gro_ports[RTE_MAX_ETHPORTS];
524uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
525#endif
526
527
528
529
530enum rte_eth_rx_mq_mode rx_mq_mode = RTE_ETH_MQ_RX_VMDQ_DCB_RSS;
531
532
533
534
535uint32_t eth_link_speed;
536
537
538
539
540
541int proc_id;
542
543
544
545
546
547unsigned int num_procs = 1;
548
549static void
550eth_rx_metadata_negotiate_mp(uint16_t port_id)
551{
552 uint64_t rx_meta_features = 0;
553 int ret;
554
555 if (!is_proc_primary())
556 return;
557
558 rx_meta_features |= RTE_ETH_RX_METADATA_USER_FLAG;
559 rx_meta_features |= RTE_ETH_RX_METADATA_USER_MARK;
560 rx_meta_features |= RTE_ETH_RX_METADATA_TUNNEL_ID;
561
562 ret = rte_eth_rx_metadata_negotiate(port_id, &rx_meta_features);
563 if (ret == 0) {
564 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_FLAG)) {
565 TESTPMD_LOG(DEBUG, "Flow action FLAG will not affect Rx mbufs on port %u\n",
566 port_id);
567 }
568
569 if (!(rx_meta_features & RTE_ETH_RX_METADATA_USER_MARK)) {
570 TESTPMD_LOG(DEBUG, "Flow action MARK will not affect Rx mbufs on port %u\n",
571 port_id);
572 }
573
574 if (!(rx_meta_features & RTE_ETH_RX_METADATA_TUNNEL_ID)) {
575 TESTPMD_LOG(DEBUG, "Flow tunnel offload support might be limited or unavailable on port %u\n",
576 port_id);
577 }
578 } else if (ret != -ENOTSUP) {
579 rte_exit(EXIT_FAILURE, "Error when negotiating Rx meta features on port %u: %s\n",
580 port_id, rte_strerror(-ret));
581 }
582}
583
584static int
585eth_dev_configure_mp(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
586 const struct rte_eth_conf *dev_conf)
587{
588 if (is_proc_primary())
589 return rte_eth_dev_configure(port_id, nb_rx_q, nb_tx_q,
590 dev_conf);
591 return 0;
592}
593
594static int
595eth_dev_start_mp(uint16_t port_id)
596{
597 if (is_proc_primary())
598 return rte_eth_dev_start(port_id);
599
600 return 0;
601}
602
603static int
604eth_dev_stop_mp(uint16_t port_id)
605{
606 if (is_proc_primary())
607 return rte_eth_dev_stop(port_id);
608
609 return 0;
610}
611
612static void
613mempool_free_mp(struct rte_mempool *mp)
614{
615 if (is_proc_primary())
616 rte_mempool_free(mp);
617}
618
619static int
620eth_dev_set_mtu_mp(uint16_t port_id, uint16_t mtu)
621{
622 if (is_proc_primary())
623 return rte_eth_dev_set_mtu(port_id, mtu);
624
625 return 0;
626}
627
628
629static void setup_attached_port(portid_t pi);
630static void check_all_ports_link_status(uint32_t port_mask);
631static int eth_event_callback(portid_t port_id,
632 enum rte_eth_event_type type,
633 void *param, void *ret_param);
634static void dev_event_callback(const char *device_name,
635 enum rte_dev_event_type type,
636 void *param);
637static void fill_xstats_display_info(void);
638
639
640
641
642
643static int all_ports_started(void);
644
645#ifdef RTE_LIB_GSO
646struct gso_status gso_ports[RTE_MAX_ETHPORTS];
647uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
648#endif
649
650
651char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
652
653
654
655
656
657
658int
659new_socket_id(unsigned int socket_id)
660{
661 unsigned int i;
662
663 for (i = 0; i < num_sockets; i++) {
664 if (socket_ids[i] == socket_id)
665 return 0;
666 }
667 return 1;
668}
669
670
671
672
673static void
674set_default_fwd_lcores_config(void)
675{
676 unsigned int i;
677 unsigned int nb_lc;
678 unsigned int sock_num;
679
680 nb_lc = 0;
681 for (i = 0; i < RTE_MAX_LCORE; i++) {
682 if (!rte_lcore_is_enabled(i))
683 continue;
684 sock_num = rte_lcore_to_socket_id(i);
685 if (new_socket_id(sock_num)) {
686 if (num_sockets >= RTE_MAX_NUMA_NODES) {
687 rte_exit(EXIT_FAILURE,
688 "Total sockets greater than %u\n",
689 RTE_MAX_NUMA_NODES);
690 }
691 socket_ids[num_sockets++] = sock_num;
692 }
693 if (i == rte_get_main_lcore())
694 continue;
695 fwd_lcores_cpuids[nb_lc++] = i;
696 }
697 nb_lcores = (lcoreid_t) nb_lc;
698 nb_cfg_lcores = nb_lcores;
699 nb_fwd_lcores = 1;
700}
701
702static void
703set_def_peer_eth_addrs(void)
704{
705 portid_t i;
706
707 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
708 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
709 peer_eth_addrs[i].addr_bytes[5] = i;
710 }
711}
712
713static void
714set_default_fwd_ports_config(void)
715{
716 portid_t pt_id;
717 int i = 0;
718
719 RTE_ETH_FOREACH_DEV(pt_id) {
720 fwd_ports_ids[i++] = pt_id;
721
722
723 int socket_id = rte_eth_dev_socket_id(pt_id);
724 if (socket_id >= 0 && new_socket_id(socket_id)) {
725 if (num_sockets >= RTE_MAX_NUMA_NODES) {
726 rte_exit(EXIT_FAILURE,
727 "Total sockets greater than %u\n",
728 RTE_MAX_NUMA_NODES);
729 }
730 socket_ids[num_sockets++] = socket_id;
731 }
732 }
733
734 nb_cfg_ports = nb_ports;
735 nb_fwd_ports = nb_ports;
736}
737
738void
739set_def_fwd_config(void)
740{
741 set_default_fwd_lcores_config();
742 set_def_peer_eth_addrs();
743 set_default_fwd_ports_config();
744}
745
746#ifndef RTE_EXEC_ENV_WINDOWS
747
748static int
749calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
750{
751 unsigned int n_pages, mbuf_per_pg, leftover;
752 uint64_t total_mem, mbuf_mem, obj_sz;
753
754
755
756
757
758
759 uint64_t hdr_mem = 128 << 20;
760
761
762 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
763 if (obj_sz > pgsz) {
764 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
765 return -1;
766 }
767
768 mbuf_per_pg = pgsz / obj_sz;
769 leftover = (nb_mbufs % mbuf_per_pg) > 0;
770 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
771
772 mbuf_mem = n_pages * pgsz;
773
774 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
775
776 if (total_mem > SIZE_MAX) {
777 TESTPMD_LOG(ERR, "Memory size too big\n");
778 return -1;
779 }
780 *out = (size_t)total_mem;
781
782 return 0;
783}
784
785static int
786pagesz_flags(uint64_t page_sz)
787{
788
789
790
791 int log2 = rte_log2_u64(page_sz);
792
793 return (log2 << HUGE_SHIFT);
794}
795
796static void *
797alloc_mem(size_t memsz, size_t pgsz, bool huge)
798{
799 void *addr;
800 int flags;
801
802
803 flags = MAP_ANONYMOUS | MAP_PRIVATE;
804 if (huge)
805 flags |= HUGE_FLAG | pagesz_flags(pgsz);
806
807 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
808 if (addr == MAP_FAILED)
809 return NULL;
810
811 return addr;
812}
813
814struct extmem_param {
815 void *addr;
816 size_t len;
817 size_t pgsz;
818 rte_iova_t *iova_table;
819 unsigned int iova_table_len;
820};
821
822static int
823create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
824 bool huge)
825{
826 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G,
827 RTE_PGSIZE_16M, RTE_PGSIZE_16G};
828 unsigned int cur_page, n_pages, pgsz_idx;
829 size_t mem_sz, cur_pgsz;
830 rte_iova_t *iovas = NULL;
831 void *addr;
832 int ret;
833
834 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
835
836 if (pgsizes[pgsz_idx] > SIZE_MAX)
837 continue;
838
839 cur_pgsz = pgsizes[pgsz_idx];
840
841
842 if (!huge)
843 cur_pgsz = sysconf(_SC_PAGESIZE);
844
845 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
846 if (ret < 0) {
847 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
848 return -1;
849 }
850
851
852 addr = alloc_mem(mem_sz, cur_pgsz, huge);
853
854
855
856
857
858 if (addr == NULL)
859 continue;
860
861
862 n_pages = mem_sz / cur_pgsz;
863
864 iovas = malloc(sizeof(*iovas) * n_pages);
865
866 if (iovas == NULL) {
867 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
868 goto fail;
869 }
870
871 if (!huge)
872 mlock(addr, mem_sz);
873
874
875 for (cur_page = 0; cur_page < n_pages; cur_page++) {
876 rte_iova_t iova;
877 size_t offset;
878 void *cur;
879
880 offset = cur_pgsz * cur_page;
881 cur = RTE_PTR_ADD(addr, offset);
882
883
884 *(volatile char *)cur = 0;
885
886 iova = rte_mem_virt2iova(cur);
887
888 iovas[cur_page] = iova;
889 }
890
891 break;
892 }
893
894 if (iovas == NULL)
895 return -1;
896
897 param->addr = addr;
898 param->len = mem_sz;
899 param->pgsz = cur_pgsz;
900 param->iova_table = iovas;
901 param->iova_table_len = n_pages;
902
903 return 0;
904fail:
905 if (iovas)
906 free(iovas);
907 if (addr)
908 munmap(addr, mem_sz);
909
910 return -1;
911}
912
913static int
914setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
915{
916 struct extmem_param param;
917 int socket_id, ret;
918
919 memset(¶m, 0, sizeof(param));
920
921
922 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
923 if (socket_id < 0) {
924
925 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
926 if (ret < 0) {
927 TESTPMD_LOG(ERR, "Cannot create heap\n");
928 return -1;
929 }
930 }
931
932 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
933 if (ret < 0) {
934 TESTPMD_LOG(ERR, "Cannot create memory area\n");
935 return -1;
936 }
937
938
939 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
940 param.addr, param.len, param.iova_table,
941 param.iova_table_len, param.pgsz);
942
943
944
945
946 free(param.iova_table);
947
948 if (ret < 0) {
949 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
950 munmap(param.addr, param.len);
951 return -1;
952 }
953
954
955
956 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
957 param.len >> 20);
958
959 return 0;
960}
961static void
962dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
963 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
964{
965 uint16_t pid = 0;
966 int ret;
967
968 RTE_ETH_FOREACH_DEV(pid) {
969 struct rte_eth_dev_info dev_info;
970
971 ret = eth_dev_info_get_print_err(pid, &dev_info);
972 if (ret != 0) {
973 TESTPMD_LOG(DEBUG,
974 "unable to get device info for port %d on addr 0x%p,"
975 "mempool unmapping will not be performed\n",
976 pid, memhdr->addr);
977 continue;
978 }
979
980 ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
981 if (ret) {
982 TESTPMD_LOG(DEBUG,
983 "unable to DMA unmap addr 0x%p "
984 "for device %s\n",
985 memhdr->addr, dev_info.device->name);
986 }
987 }
988 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
989 if (ret) {
990 TESTPMD_LOG(DEBUG,
991 "unable to un-register addr 0x%p\n", memhdr->addr);
992 }
993}
994
995static void
996dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
997 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
998{
999 uint16_t pid = 0;
1000 size_t page_size = sysconf(_SC_PAGESIZE);
1001 int ret;
1002
1003 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
1004 page_size);
1005 if (ret) {
1006 TESTPMD_LOG(DEBUG,
1007 "unable to register addr 0x%p\n", memhdr->addr);
1008 return;
1009 }
1010 RTE_ETH_FOREACH_DEV(pid) {
1011 struct rte_eth_dev_info dev_info;
1012
1013 ret = eth_dev_info_get_print_err(pid, &dev_info);
1014 if (ret != 0) {
1015 TESTPMD_LOG(DEBUG,
1016 "unable to get device info for port %d on addr 0x%p,"
1017 "mempool mapping will not be performed\n",
1018 pid, memhdr->addr);
1019 continue;
1020 }
1021 ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
1022 if (ret) {
1023 TESTPMD_LOG(DEBUG,
1024 "unable to DMA map addr 0x%p "
1025 "for device %s\n",
1026 memhdr->addr, dev_info.device->name);
1027 }
1028 }
1029}
1030#endif
1031
1032static unsigned int
1033setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
1034 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
1035{
1036 struct rte_pktmbuf_extmem *xmem;
1037 unsigned int ext_num, zone_num, elt_num;
1038 uint16_t elt_size;
1039
1040 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
1041 elt_num = EXTBUF_ZONE_SIZE / elt_size;
1042 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
1043
1044 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
1045 if (xmem == NULL) {
1046 TESTPMD_LOG(ERR, "Cannot allocate memory for "
1047 "external buffer descriptors\n");
1048 *ext_mem = NULL;
1049 return 0;
1050 }
1051 for (ext_num = 0; ext_num < zone_num; ext_num++) {
1052 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
1053 const struct rte_memzone *mz;
1054 char mz_name[RTE_MEMZONE_NAMESIZE];
1055 int ret;
1056
1057 ret = snprintf(mz_name, sizeof(mz_name),
1058 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
1059 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
1060 errno = ENAMETOOLONG;
1061 ext_num = 0;
1062 break;
1063 }
1064 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
1065 socket_id,
1066 RTE_MEMZONE_IOVA_CONTIG |
1067 RTE_MEMZONE_1GB |
1068 RTE_MEMZONE_SIZE_HINT_ONLY,
1069 EXTBUF_ZONE_SIZE);
1070 if (mz == NULL) {
1071
1072
1073
1074
1075 errno = ENOMEM;
1076 ext_num = 0;
1077 break;
1078 }
1079 xseg->buf_ptr = mz->addr;
1080 xseg->buf_iova = mz->iova;
1081 xseg->buf_len = EXTBUF_ZONE_SIZE;
1082 xseg->elt_size = elt_size;
1083 }
1084 if (ext_num == 0 && xmem != NULL) {
1085 free(xmem);
1086 xmem = NULL;
1087 }
1088 *ext_mem = xmem;
1089 return ext_num;
1090}
1091
1092
1093
1094
1095static struct rte_mempool *
1096mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
1097 unsigned int socket_id, uint16_t size_idx)
1098{
1099 char pool_name[RTE_MEMPOOL_NAMESIZE];
1100 struct rte_mempool *rte_mp = NULL;
1101#ifndef RTE_EXEC_ENV_WINDOWS
1102 uint32_t mb_size;
1103
1104 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
1105#endif
1106 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
1107 if (!is_proc_primary()) {
1108 rte_mp = rte_mempool_lookup(pool_name);
1109 if (rte_mp == NULL)
1110 rte_exit(EXIT_FAILURE,
1111 "Get mbuf pool for socket %u failed: %s\n",
1112 socket_id, rte_strerror(rte_errno));
1113 return rte_mp;
1114 }
1115
1116 TESTPMD_LOG(INFO,
1117 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
1118 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
1119
1120 switch (mp_alloc_type) {
1121 case MP_ALLOC_NATIVE:
1122 {
1123
1124 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1125 rte_mbuf_best_mempool_ops());
1126 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1127 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
1128 break;
1129 }
1130#ifndef RTE_EXEC_ENV_WINDOWS
1131 case MP_ALLOC_ANON:
1132 {
1133 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1134 mb_size, (unsigned int) mb_mempool_cache,
1135 sizeof(struct rte_pktmbuf_pool_private),
1136 socket_id, mempool_flags);
1137 if (rte_mp == NULL)
1138 goto err;
1139
1140 if (rte_mempool_populate_anon(rte_mp) == 0) {
1141 rte_mempool_free(rte_mp);
1142 rte_mp = NULL;
1143 goto err;
1144 }
1145 rte_pktmbuf_pool_init(rte_mp, NULL);
1146 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1147 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1148 break;
1149 }
1150 case MP_ALLOC_XMEM:
1151 case MP_ALLOC_XMEM_HUGE:
1152 {
1153 int heap_socket;
1154 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1155
1156 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1157 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1158
1159 heap_socket =
1160 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1161 if (heap_socket < 0)
1162 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1163
1164 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1165 rte_mbuf_best_mempool_ops());
1166 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1167 mb_mempool_cache, 0, mbuf_seg_size,
1168 heap_socket);
1169 break;
1170 }
1171#endif
1172 case MP_ALLOC_XBUF:
1173 {
1174 struct rte_pktmbuf_extmem *ext_mem;
1175 unsigned int ext_num;
1176
1177 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1178 socket_id, pool_name, &ext_mem);
1179 if (ext_num == 0)
1180 rte_exit(EXIT_FAILURE,
1181 "Can't create pinned data buffers\n");
1182
1183 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1184 rte_mbuf_best_mempool_ops());
1185 rte_mp = rte_pktmbuf_pool_create_extbuf
1186 (pool_name, nb_mbuf, mb_mempool_cache,
1187 0, mbuf_seg_size, socket_id,
1188 ext_mem, ext_num);
1189 free(ext_mem);
1190 break;
1191 }
1192 default:
1193 {
1194 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1195 }
1196 }
1197
1198#ifndef RTE_EXEC_ENV_WINDOWS
1199err:
1200#endif
1201 if (rte_mp == NULL) {
1202 rte_exit(EXIT_FAILURE,
1203 "Creation of mbuf pool for socket %u failed: %s\n",
1204 socket_id, rte_strerror(rte_errno));
1205 } else if (verbose_level > 0) {
1206 rte_mempool_dump(stdout, rte_mp);
1207 }
1208 return rte_mp;
1209}
1210
1211
1212
1213
1214
1215static int
1216check_socket_id(const unsigned int socket_id)
1217{
1218 static int warning_once = 0;
1219
1220 if (new_socket_id(socket_id)) {
1221 if (!warning_once && numa_support)
1222 fprintf(stderr,
1223 "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
1224 warning_once = 1;
1225 return -1;
1226 }
1227 return 0;
1228}
1229
1230
1231
1232
1233
1234
1235queueid_t
1236get_allowed_max_nb_rxq(portid_t *pid)
1237{
1238 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1239 bool max_rxq_valid = false;
1240 portid_t pi;
1241 struct rte_eth_dev_info dev_info;
1242
1243 RTE_ETH_FOREACH_DEV(pi) {
1244 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1245 continue;
1246
1247 max_rxq_valid = true;
1248 if (dev_info.max_rx_queues < allowed_max_rxq) {
1249 allowed_max_rxq = dev_info.max_rx_queues;
1250 *pid = pi;
1251 }
1252 }
1253 return max_rxq_valid ? allowed_max_rxq : 0;
1254}
1255
1256
1257
1258
1259
1260
1261
1262int
1263check_nb_rxq(queueid_t rxq)
1264{
1265 queueid_t allowed_max_rxq;
1266 portid_t pid = 0;
1267
1268 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1269 if (rxq > allowed_max_rxq) {
1270 fprintf(stderr,
1271 "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
1272 rxq, allowed_max_rxq, pid);
1273 return -1;
1274 }
1275 return 0;
1276}
1277
1278
1279
1280
1281
1282
1283queueid_t
1284get_allowed_max_nb_txq(portid_t *pid)
1285{
1286 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1287 bool max_txq_valid = false;
1288 portid_t pi;
1289 struct rte_eth_dev_info dev_info;
1290
1291 RTE_ETH_FOREACH_DEV(pi) {
1292 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1293 continue;
1294
1295 max_txq_valid = true;
1296 if (dev_info.max_tx_queues < allowed_max_txq) {
1297 allowed_max_txq = dev_info.max_tx_queues;
1298 *pid = pi;
1299 }
1300 }
1301 return max_txq_valid ? allowed_max_txq : 0;
1302}
1303
1304
1305
1306
1307
1308
1309
1310int
1311check_nb_txq(queueid_t txq)
1312{
1313 queueid_t allowed_max_txq;
1314 portid_t pid = 0;
1315
1316 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1317 if (txq > allowed_max_txq) {
1318 fprintf(stderr,
1319 "Fail: input txq (%u) can't be greater than max_tx_queues (%u) of port %u\n",
1320 txq, allowed_max_txq, pid);
1321 return -1;
1322 }
1323 return 0;
1324}
1325
1326
1327
1328
1329
1330
1331static uint16_t
1332get_allowed_max_nb_rxd(portid_t *pid)
1333{
1334 uint16_t allowed_max_rxd = UINT16_MAX;
1335 portid_t pi;
1336 struct rte_eth_dev_info dev_info;
1337
1338 RTE_ETH_FOREACH_DEV(pi) {
1339 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1340 continue;
1341
1342 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1343 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1344 *pid = pi;
1345 }
1346 }
1347 return allowed_max_rxd;
1348}
1349
1350
1351
1352
1353
1354
1355static uint16_t
1356get_allowed_min_nb_rxd(portid_t *pid)
1357{
1358 uint16_t allowed_min_rxd = 0;
1359 portid_t pi;
1360 struct rte_eth_dev_info dev_info;
1361
1362 RTE_ETH_FOREACH_DEV(pi) {
1363 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1364 continue;
1365
1366 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1367 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1368 *pid = pi;
1369 }
1370 }
1371
1372 return allowed_min_rxd;
1373}
1374
1375
1376
1377
1378
1379
1380
1381
1382int
1383check_nb_rxd(queueid_t rxd)
1384{
1385 uint16_t allowed_max_rxd;
1386 uint16_t allowed_min_rxd;
1387 portid_t pid = 0;
1388
1389 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1390 if (rxd > allowed_max_rxd) {
1391 fprintf(stderr,
1392 "Fail: input rxd (%u) can't be greater than max_rxds (%u) of port %u\n",
1393 rxd, allowed_max_rxd, pid);
1394 return -1;
1395 }
1396
1397 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1398 if (rxd < allowed_min_rxd) {
1399 fprintf(stderr,
1400 "Fail: input rxd (%u) can't be less than min_rxds (%u) of port %u\n",
1401 rxd, allowed_min_rxd, pid);
1402 return -1;
1403 }
1404
1405 return 0;
1406}
1407
1408
1409
1410
1411
1412
1413static uint16_t
1414get_allowed_max_nb_txd(portid_t *pid)
1415{
1416 uint16_t allowed_max_txd = UINT16_MAX;
1417 portid_t pi;
1418 struct rte_eth_dev_info dev_info;
1419
1420 RTE_ETH_FOREACH_DEV(pi) {
1421 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1422 continue;
1423
1424 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1425 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1426 *pid = pi;
1427 }
1428 }
1429 return allowed_max_txd;
1430}
1431
1432
1433
1434
1435
1436
1437static uint16_t
1438get_allowed_min_nb_txd(portid_t *pid)
1439{
1440 uint16_t allowed_min_txd = 0;
1441 portid_t pi;
1442 struct rte_eth_dev_info dev_info;
1443
1444 RTE_ETH_FOREACH_DEV(pi) {
1445 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1446 continue;
1447
1448 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1449 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1450 *pid = pi;
1451 }
1452 }
1453
1454 return allowed_min_txd;
1455}
1456
1457
1458
1459
1460
1461
1462
1463int
1464check_nb_txd(queueid_t txd)
1465{
1466 uint16_t allowed_max_txd;
1467 uint16_t allowed_min_txd;
1468 portid_t pid = 0;
1469
1470 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1471 if (txd > allowed_max_txd) {
1472 fprintf(stderr,
1473 "Fail: input txd (%u) can't be greater than max_txds (%u) of port %u\n",
1474 txd, allowed_max_txd, pid);
1475 return -1;
1476 }
1477
1478 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1479 if (txd < allowed_min_txd) {
1480 fprintf(stderr,
1481 "Fail: input txd (%u) can't be less than min_txds (%u) of port %u\n",
1482 txd, allowed_min_txd, pid);
1483 return -1;
1484 }
1485 return 0;
1486}
1487
1488
1489
1490
1491
1492
1493
1494queueid_t
1495get_allowed_max_nb_hairpinq(portid_t *pid)
1496{
1497 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1498 portid_t pi;
1499 struct rte_eth_hairpin_cap cap;
1500
1501 RTE_ETH_FOREACH_DEV(pi) {
1502 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1503 *pid = pi;
1504 return 0;
1505 }
1506 if (cap.max_nb_queues < allowed_max_hairpinq) {
1507 allowed_max_hairpinq = cap.max_nb_queues;
1508 *pid = pi;
1509 }
1510 }
1511 return allowed_max_hairpinq;
1512}
1513
1514
1515
1516
1517
1518
1519
1520int
1521check_nb_hairpinq(queueid_t hairpinq)
1522{
1523 queueid_t allowed_max_hairpinq;
1524 portid_t pid = 0;
1525
1526 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1527 if (hairpinq > allowed_max_hairpinq) {
1528 fprintf(stderr,
1529 "Fail: input hairpin (%u) can't be greater than max_hairpin_queues (%u) of port %u\n",
1530 hairpinq, allowed_max_hairpinq, pid);
1531 return -1;
1532 }
1533 return 0;
1534}
1535
1536static int
1537get_eth_overhead(struct rte_eth_dev_info *dev_info)
1538{
1539 uint32_t eth_overhead;
1540
1541 if (dev_info->max_mtu != UINT16_MAX &&
1542 dev_info->max_rx_pktlen > dev_info->max_mtu)
1543 eth_overhead = dev_info->max_rx_pktlen - dev_info->max_mtu;
1544 else
1545 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1546
1547 return eth_overhead;
1548}
1549
1550static void
1551init_config_port_offloads(portid_t pid, uint32_t socket_id)
1552{
1553 struct rte_port *port = &ports[pid];
1554 int ret;
1555 int i;
1556
1557 eth_rx_metadata_negotiate_mp(pid);
1558
1559 port->dev_conf.txmode = tx_mode;
1560 port->dev_conf.rxmode = rx_mode;
1561
1562 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1563 if (ret != 0)
1564 rte_exit(EXIT_FAILURE, "rte_eth_dev_info_get() failed\n");
1565
1566 if (!(port->dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE))
1567 port->dev_conf.txmode.offloads &=
1568 ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
1569
1570
1571 for (i = 0; i < port->dev_info.max_rx_queues; i++)
1572 port->rx_conf[i].offloads = port->dev_conf.rxmode.offloads;
1573
1574 for (i = 0; i < port->dev_info.max_tx_queues; i++)
1575 port->tx_conf[i].offloads = port->dev_conf.txmode.offloads;
1576
1577 if (eth_link_speed)
1578 port->dev_conf.link_speeds = eth_link_speed;
1579
1580 if (max_rx_pkt_len)
1581 port->dev_conf.rxmode.mtu = max_rx_pkt_len -
1582 get_eth_overhead(&port->dev_info);
1583
1584
1585 port->need_reconfig = 1;
1586 port->need_reconfig_queues = 1;
1587 port->socket_id = socket_id;
1588 port->tx_metadata = 0;
1589
1590
1591
1592
1593
1594 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1595 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1596 uint32_t eth_overhead = get_eth_overhead(&port->dev_info);
1597 uint16_t mtu;
1598
1599 if (rte_eth_dev_get_mtu(pid, &mtu) == 0) {
1600 uint16_t data_size = (mtu + eth_overhead) /
1601 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1602 uint16_t buffer_size = data_size + RTE_PKTMBUF_HEADROOM;
1603
1604 if (buffer_size > mbuf_data_size[0]) {
1605 mbuf_data_size[0] = buffer_size;
1606 TESTPMD_LOG(WARNING,
1607 "Configured mbuf size of the first segment %hu\n",
1608 mbuf_data_size[0]);
1609 }
1610 }
1611 }
1612}
1613
1614static void
1615init_config(void)
1616{
1617 portid_t pid;
1618 struct rte_mempool *mbp;
1619 unsigned int nb_mbuf_per_pool;
1620 lcoreid_t lc_id;
1621#ifdef RTE_LIB_GRO
1622 struct rte_gro_param gro_param;
1623#endif
1624#ifdef RTE_LIB_GSO
1625 uint32_t gso_types;
1626#endif
1627
1628
1629 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1630 sizeof(struct fwd_lcore *) * nb_lcores,
1631 RTE_CACHE_LINE_SIZE);
1632 if (fwd_lcores == NULL) {
1633 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1634 "failed\n", nb_lcores);
1635 }
1636 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1637 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1638 sizeof(struct fwd_lcore),
1639 RTE_CACHE_LINE_SIZE);
1640 if (fwd_lcores[lc_id] == NULL) {
1641 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1642 "failed\n");
1643 }
1644 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1645 }
1646
1647 RTE_ETH_FOREACH_DEV(pid) {
1648 uint32_t socket_id;
1649
1650 if (numa_support) {
1651 socket_id = port_numa[pid];
1652 if (port_numa[pid] == NUMA_NO_CONFIG) {
1653 socket_id = rte_eth_dev_socket_id(pid);
1654
1655
1656
1657
1658
1659 if (check_socket_id(socket_id) < 0)
1660 socket_id = socket_ids[0];
1661 }
1662 } else {
1663 socket_id = (socket_num == UMA_NO_CONFIG) ?
1664 0 : socket_num;
1665 }
1666
1667 init_config_port_offloads(pid, socket_id);
1668 }
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678 if (param_total_num_mbufs)
1679 nb_mbuf_per_pool = param_total_num_mbufs;
1680 else {
1681 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1682 (nb_lcores * mb_mempool_cache) +
1683 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1684 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1685 }
1686
1687 if (numa_support) {
1688 uint8_t i, j;
1689
1690 for (i = 0; i < num_sockets; i++)
1691 for (j = 0; j < mbuf_data_size_n; j++)
1692 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1693 mbuf_pool_create(mbuf_data_size[j],
1694 nb_mbuf_per_pool,
1695 socket_ids[i], j);
1696 } else {
1697 uint8_t i;
1698
1699 for (i = 0; i < mbuf_data_size_n; i++)
1700 mempools[i] = mbuf_pool_create
1701 (mbuf_data_size[i],
1702 nb_mbuf_per_pool,
1703 socket_num == UMA_NO_CONFIG ?
1704 0 : socket_num, i);
1705 }
1706
1707 init_port_config();
1708
1709#ifdef RTE_LIB_GSO
1710 gso_types = RTE_ETH_TX_OFFLOAD_TCP_TSO | RTE_ETH_TX_OFFLOAD_VXLAN_TNL_TSO |
1711 RTE_ETH_TX_OFFLOAD_GRE_TNL_TSO | RTE_ETH_TX_OFFLOAD_UDP_TSO;
1712#endif
1713
1714
1715
1716 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1717 mbp = mbuf_pool_find(
1718 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1719
1720 if (mbp == NULL)
1721 mbp = mbuf_pool_find(0, 0);
1722 fwd_lcores[lc_id]->mbp = mbp;
1723#ifdef RTE_LIB_GSO
1724
1725 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1726 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1727 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1728 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1729 RTE_ETHER_CRC_LEN;
1730 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1731#endif
1732 }
1733
1734 fwd_config_setup();
1735
1736#ifdef RTE_LIB_GRO
1737
1738 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1739 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1740 gro_param.max_item_per_flow = MAX_PKT_BURST;
1741 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1742 gro_param.socket_id = rte_lcore_to_socket_id(
1743 fwd_lcores_cpuids[lc_id]);
1744 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1745 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1746 rte_exit(EXIT_FAILURE,
1747 "rte_gro_ctx_create() failed\n");
1748 }
1749 }
1750#endif
1751}
1752
1753
1754void
1755reconfig(portid_t new_port_id, unsigned socket_id)
1756{
1757
1758 init_config_port_offloads(new_port_id, socket_id);
1759 init_port_config();
1760}
1761
1762
1763int
1764init_fwd_streams(void)
1765{
1766 portid_t pid;
1767 struct rte_port *port;
1768 streamid_t sm_id, nb_fwd_streams_new;
1769 queueid_t q;
1770
1771
1772 RTE_ETH_FOREACH_DEV(pid) {
1773 port = &ports[pid];
1774 if (nb_rxq > port->dev_info.max_rx_queues) {
1775 fprintf(stderr,
1776 "Fail: nb_rxq(%d) is greater than max_rx_queues(%d)\n",
1777 nb_rxq, port->dev_info.max_rx_queues);
1778 return -1;
1779 }
1780 if (nb_txq > port->dev_info.max_tx_queues) {
1781 fprintf(stderr,
1782 "Fail: nb_txq(%d) is greater than max_tx_queues(%d)\n",
1783 nb_txq, port->dev_info.max_tx_queues);
1784 return -1;
1785 }
1786 if (numa_support) {
1787 if (port_numa[pid] != NUMA_NO_CONFIG)
1788 port->socket_id = port_numa[pid];
1789 else {
1790 port->socket_id = rte_eth_dev_socket_id(pid);
1791
1792
1793
1794
1795
1796 if (check_socket_id(port->socket_id) < 0)
1797 port->socket_id = socket_ids[0];
1798 }
1799 }
1800 else {
1801 if (socket_num == UMA_NO_CONFIG)
1802 port->socket_id = 0;
1803 else
1804 port->socket_id = socket_num;
1805 }
1806 }
1807
1808 q = RTE_MAX(nb_rxq, nb_txq);
1809 if (q == 0) {
1810 fprintf(stderr,
1811 "Fail: Cannot allocate fwd streams as number of queues is 0\n");
1812 return -1;
1813 }
1814 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1815 if (nb_fwd_streams_new == nb_fwd_streams)
1816 return 0;
1817
1818 if (fwd_streams != NULL) {
1819 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1820 if (fwd_streams[sm_id] == NULL)
1821 continue;
1822 rte_free(fwd_streams[sm_id]);
1823 fwd_streams[sm_id] = NULL;
1824 }
1825 rte_free(fwd_streams);
1826 fwd_streams = NULL;
1827 }
1828
1829
1830 nb_fwd_streams = nb_fwd_streams_new;
1831 if (nb_fwd_streams) {
1832 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1833 sizeof(struct fwd_stream *) * nb_fwd_streams,
1834 RTE_CACHE_LINE_SIZE);
1835 if (fwd_streams == NULL)
1836 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1837 " (struct fwd_stream *)) failed\n",
1838 nb_fwd_streams);
1839
1840 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1841 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1842 " struct fwd_stream", sizeof(struct fwd_stream),
1843 RTE_CACHE_LINE_SIZE);
1844 if (fwd_streams[sm_id] == NULL)
1845 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1846 "(struct fwd_stream) failed\n");
1847 }
1848 }
1849
1850 return 0;
1851}
1852
1853static void
1854pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1855{
1856 uint64_t total_burst, sburst;
1857 uint64_t nb_burst;
1858 uint64_t burst_stats[4];
1859 uint16_t pktnb_stats[4];
1860 uint16_t nb_pkt;
1861 int burst_percent[4], sburstp;
1862 int i;
1863
1864
1865
1866
1867
1868 memset(&burst_stats, 0x0, sizeof(burst_stats));
1869 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1870
1871
1872 total_burst = pbs->pkt_burst_spread[0];
1873 burst_stats[0] = pbs->pkt_burst_spread[0];
1874 pktnb_stats[0] = 0;
1875
1876
1877 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST + 1; nb_pkt++) {
1878 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1879
1880 if (nb_burst == 0)
1881 continue;
1882
1883 total_burst += nb_burst;
1884
1885 if (nb_burst > burst_stats[1]) {
1886 burst_stats[2] = burst_stats[1];
1887 pktnb_stats[2] = pktnb_stats[1];
1888 burst_stats[1] = nb_burst;
1889 pktnb_stats[1] = nb_pkt;
1890 } else if (nb_burst > burst_stats[2]) {
1891 burst_stats[2] = nb_burst;
1892 pktnb_stats[2] = nb_pkt;
1893 }
1894 }
1895 if (total_burst == 0)
1896 return;
1897
1898 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1899 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1900 if (i == 3) {
1901 printf("%d%% of other]\n", 100 - sburstp);
1902 return;
1903 }
1904
1905 sburst += burst_stats[i];
1906 if (sburst == total_burst) {
1907 printf("%d%% of %d pkts]\n",
1908 100 - sburstp, (int) pktnb_stats[i]);
1909 return;
1910 }
1911
1912 burst_percent[i] =
1913 (double)burst_stats[i] / total_burst * 100;
1914 printf("%d%% of %d pkts + ",
1915 burst_percent[i], (int) pktnb_stats[i]);
1916 sburstp += burst_percent[i];
1917 }
1918}
1919
1920static void
1921fwd_stream_stats_display(streamid_t stream_id)
1922{
1923 struct fwd_stream *fs;
1924 static const char *fwd_top_stats_border = "-------";
1925
1926 fs = fwd_streams[stream_id];
1927 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1928 (fs->fwd_dropped == 0))
1929 return;
1930 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1931 "TX Port=%2d/Queue=%2d %s\n",
1932 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1933 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1934 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1935 " TX-dropped: %-14"PRIu64,
1936 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1937
1938
1939 if (cur_fwd_eng == &csum_fwd_engine) {
1940 printf(" RX- bad IP checksum: %-14"PRIu64
1941 " Rx- bad L4 checksum: %-14"PRIu64
1942 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1943 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1944 fs->rx_bad_outer_l4_csum);
1945 printf(" RX- bad outer IP checksum: %-14"PRIu64"\n",
1946 fs->rx_bad_outer_ip_csum);
1947 } else {
1948 printf("\n");
1949 }
1950
1951 if (record_burst_stats) {
1952 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1953 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1954 }
1955}
1956
1957void
1958fwd_stats_display(void)
1959{
1960 static const char *fwd_stats_border = "----------------------";
1961 static const char *acc_stats_border = "+++++++++++++++";
1962 struct {
1963 struct fwd_stream *rx_stream;
1964 struct fwd_stream *tx_stream;
1965 uint64_t tx_dropped;
1966 uint64_t rx_bad_ip_csum;
1967 uint64_t rx_bad_l4_csum;
1968 uint64_t rx_bad_outer_l4_csum;
1969 uint64_t rx_bad_outer_ip_csum;
1970 } ports_stats[RTE_MAX_ETHPORTS];
1971 uint64_t total_rx_dropped = 0;
1972 uint64_t total_tx_dropped = 0;
1973 uint64_t total_rx_nombuf = 0;
1974 struct rte_eth_stats stats;
1975 uint64_t fwd_cycles = 0;
1976 uint64_t total_recv = 0;
1977 uint64_t total_xmit = 0;
1978 struct rte_port *port;
1979 streamid_t sm_id;
1980 portid_t pt_id;
1981 int i;
1982
1983 memset(ports_stats, 0, sizeof(ports_stats));
1984
1985 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1986 struct fwd_stream *fs = fwd_streams[sm_id];
1987
1988 if (cur_fwd_config.nb_fwd_streams >
1989 cur_fwd_config.nb_fwd_ports) {
1990 fwd_stream_stats_display(sm_id);
1991 } else {
1992 ports_stats[fs->tx_port].tx_stream = fs;
1993 ports_stats[fs->rx_port].rx_stream = fs;
1994 }
1995
1996 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1997
1998 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1999 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
2000 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
2001 fs->rx_bad_outer_l4_csum;
2002 ports_stats[fs->rx_port].rx_bad_outer_ip_csum +=
2003 fs->rx_bad_outer_ip_csum;
2004
2005 if (record_core_cycles)
2006 fwd_cycles += fs->core_cycles;
2007 }
2008 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2009 pt_id = fwd_ports_ids[i];
2010 port = &ports[pt_id];
2011
2012 rte_eth_stats_get(pt_id, &stats);
2013 stats.ipackets -= port->stats.ipackets;
2014 stats.opackets -= port->stats.opackets;
2015 stats.ibytes -= port->stats.ibytes;
2016 stats.obytes -= port->stats.obytes;
2017 stats.imissed -= port->stats.imissed;
2018 stats.oerrors -= port->stats.oerrors;
2019 stats.rx_nombuf -= port->stats.rx_nombuf;
2020
2021 total_recv += stats.ipackets;
2022 total_xmit += stats.opackets;
2023 total_rx_dropped += stats.imissed;
2024 total_tx_dropped += ports_stats[pt_id].tx_dropped;
2025 total_tx_dropped += stats.oerrors;
2026 total_rx_nombuf += stats.rx_nombuf;
2027
2028 printf("\n %s Forward statistics for port %-2d %s\n",
2029 fwd_stats_border, pt_id, fwd_stats_border);
2030
2031 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
2032 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
2033 stats.ipackets + stats.imissed);
2034
2035 if (cur_fwd_eng == &csum_fwd_engine) {
2036 printf(" Bad-ipcsum: %-14"PRIu64
2037 " Bad-l4csum: %-14"PRIu64
2038 "Bad-outer-l4csum: %-14"PRIu64"\n",
2039 ports_stats[pt_id].rx_bad_ip_csum,
2040 ports_stats[pt_id].rx_bad_l4_csum,
2041 ports_stats[pt_id].rx_bad_outer_l4_csum);
2042 printf(" Bad-outer-ipcsum: %-14"PRIu64"\n",
2043 ports_stats[pt_id].rx_bad_outer_ip_csum);
2044 }
2045 if (stats.ierrors + stats.rx_nombuf > 0) {
2046 printf(" RX-error: %-"PRIu64"\n", stats.ierrors);
2047 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
2048 }
2049
2050 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
2051 "TX-total: %-"PRIu64"\n",
2052 stats.opackets, ports_stats[pt_id].tx_dropped,
2053 stats.opackets + ports_stats[pt_id].tx_dropped);
2054
2055 if (record_burst_stats) {
2056 if (ports_stats[pt_id].rx_stream)
2057 pkt_burst_stats_display("RX",
2058 &ports_stats[pt_id].rx_stream->rx_burst_stats);
2059 if (ports_stats[pt_id].tx_stream)
2060 pkt_burst_stats_display("TX",
2061 &ports_stats[pt_id].tx_stream->tx_burst_stats);
2062 }
2063
2064 printf(" %s--------------------------------%s\n",
2065 fwd_stats_border, fwd_stats_border);
2066 }
2067
2068 printf("\n %s Accumulated forward statistics for all ports"
2069 "%s\n",
2070 acc_stats_border, acc_stats_border);
2071 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
2072 "%-"PRIu64"\n"
2073 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
2074 "%-"PRIu64"\n",
2075 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
2076 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
2077 if (total_rx_nombuf > 0)
2078 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
2079 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
2080 "%s\n",
2081 acc_stats_border, acc_stats_border);
2082 if (record_core_cycles) {
2083#define CYC_PER_MHZ 1E6
2084 if (total_recv > 0 || total_xmit > 0) {
2085 uint64_t total_pkts = 0;
2086 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
2087 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
2088 total_pkts = total_xmit;
2089 else
2090 total_pkts = total_recv;
2091
2092 printf("\n CPU cycles/packet=%.2F (total cycles="
2093 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
2094 " MHz Clock\n",
2095 (double) fwd_cycles / total_pkts,
2096 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
2097 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
2098 }
2099 }
2100}
2101
2102void
2103fwd_stats_reset(void)
2104{
2105 streamid_t sm_id;
2106 portid_t pt_id;
2107 int i;
2108
2109 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2110 pt_id = fwd_ports_ids[i];
2111 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
2112 }
2113 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
2114 struct fwd_stream *fs = fwd_streams[sm_id];
2115
2116 fs->rx_packets = 0;
2117 fs->tx_packets = 0;
2118 fs->fwd_dropped = 0;
2119 fs->rx_bad_ip_csum = 0;
2120 fs->rx_bad_l4_csum = 0;
2121 fs->rx_bad_outer_l4_csum = 0;
2122 fs->rx_bad_outer_ip_csum = 0;
2123
2124 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
2125 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
2126 fs->core_cycles = 0;
2127 }
2128}
2129
2130static void
2131flush_fwd_rx_queues(void)
2132{
2133 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
2134 portid_t rxp;
2135 portid_t port_id;
2136 queueid_t rxq;
2137 uint16_t nb_rx;
2138 uint16_t i;
2139 uint8_t j;
2140 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
2141 uint64_t timer_period;
2142
2143 if (num_procs > 1) {
2144 printf("multi-process not support for flushing fwd Rx queues, skip the below lines and return.\n");
2145 return;
2146 }
2147
2148
2149 timer_period = rte_get_timer_hz();
2150
2151 for (j = 0; j < 2; j++) {
2152 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
2153 for (rxq = 0; rxq < nb_rxq; rxq++) {
2154 port_id = fwd_ports_ids[rxp];
2155
2156
2157
2158
2159
2160
2161 prev_tsc = rte_rdtsc();
2162 do {
2163 nb_rx = rte_eth_rx_burst(port_id, rxq,
2164 pkts_burst, MAX_PKT_BURST);
2165 for (i = 0; i < nb_rx; i++)
2166 rte_pktmbuf_free(pkts_burst[i]);
2167
2168 cur_tsc = rte_rdtsc();
2169 diff_tsc = cur_tsc - prev_tsc;
2170 timer_tsc += diff_tsc;
2171 } while ((nb_rx > 0) &&
2172 (timer_tsc < timer_period));
2173 timer_tsc = 0;
2174 }
2175 }
2176 rte_delay_ms(10);
2177 }
2178}
2179
2180static void
2181run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2182{
2183 struct fwd_stream **fsm;
2184 streamid_t nb_fs;
2185 streamid_t sm_id;
2186#ifdef RTE_LIB_BITRATESTATS
2187 uint64_t tics_per_1sec;
2188 uint64_t tics_datum;
2189 uint64_t tics_current;
2190 uint16_t i, cnt_ports;
2191
2192 cnt_ports = nb_ports;
2193 tics_datum = rte_rdtsc();
2194 tics_per_1sec = rte_get_timer_hz();
2195#endif
2196 fsm = &fwd_streams[fc->stream_idx];
2197 nb_fs = fc->stream_nb;
2198 do {
2199 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2200 (*pkt_fwd)(fsm[sm_id]);
2201#ifdef RTE_LIB_BITRATESTATS
2202 if (bitrate_enabled != 0 &&
2203 bitrate_lcore_id == rte_lcore_id()) {
2204 tics_current = rte_rdtsc();
2205 if (tics_current - tics_datum >= tics_per_1sec) {
2206
2207 for (i = 0; i < cnt_ports; i++)
2208 rte_stats_bitrate_calc(bitrate_data,
2209 ports_ids[i]);
2210 tics_datum = tics_current;
2211 }
2212 }
2213#endif
2214#ifdef RTE_LIB_LATENCYSTATS
2215 if (latencystats_enabled != 0 &&
2216 latencystats_lcore_id == rte_lcore_id())
2217 rte_latencystats_update();
2218#endif
2219
2220 } while (! fc->stopped);
2221}
2222
2223static int
2224start_pkt_forward_on_core(void *fwd_arg)
2225{
2226 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2227 cur_fwd_config.fwd_eng->packet_fwd);
2228 return 0;
2229}
2230
2231
2232
2233
2234
2235static int
2236run_one_txonly_burst_on_core(void *fwd_arg)
2237{
2238 struct fwd_lcore *fwd_lc;
2239 struct fwd_lcore tmp_lcore;
2240
2241 fwd_lc = (struct fwd_lcore *) fwd_arg;
2242 tmp_lcore = *fwd_lc;
2243 tmp_lcore.stopped = 1;
2244 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2245 return 0;
2246}
2247
2248
2249
2250
2251
2252
2253static void
2254launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2255{
2256 unsigned int i;
2257 unsigned int lc_id;
2258 int diag;
2259
2260 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2261 lc_id = fwd_lcores_cpuids[i];
2262 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2263 fwd_lcores[i]->stopped = 0;
2264 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2265 fwd_lcores[i], lc_id);
2266 if (diag != 0)
2267 fprintf(stderr,
2268 "launch lcore %u failed - diag=%d\n",
2269 lc_id, diag);
2270 }
2271 }
2272}
2273
2274
2275
2276
2277void
2278start_packet_forwarding(int with_tx_first)
2279{
2280 port_fwd_begin_t port_fwd_begin;
2281 port_fwd_end_t port_fwd_end;
2282 unsigned int i;
2283
2284 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2285 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2286
2287 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2288 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2289
2290 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2291 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2292 (!nb_rxq || !nb_txq))
2293 rte_exit(EXIT_FAILURE,
2294 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2295 cur_fwd_eng->fwd_mode_name);
2296
2297 if (all_ports_started() == 0) {
2298 fprintf(stderr, "Not all ports were started\n");
2299 return;
2300 }
2301 if (test_done == 0) {
2302 fprintf(stderr, "Packet forwarding already started\n");
2303 return;
2304 }
2305
2306 fwd_config_setup();
2307
2308 pkt_fwd_config_display(&cur_fwd_config);
2309 if (!pkt_fwd_shared_rxq_check())
2310 return;
2311
2312 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2313 if (port_fwd_begin != NULL) {
2314 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2315 if (port_fwd_begin(fwd_ports_ids[i])) {
2316 fprintf(stderr,
2317 "Packet forwarding is not ready\n");
2318 return;
2319 }
2320 }
2321 }
2322
2323 if (with_tx_first) {
2324 port_fwd_begin = tx_only_engine.port_fwd_begin;
2325 if (port_fwd_begin != NULL) {
2326 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2327 if (port_fwd_begin(fwd_ports_ids[i])) {
2328 fprintf(stderr,
2329 "Packet forwarding is not ready\n");
2330 return;
2331 }
2332 }
2333 }
2334 }
2335
2336 test_done = 0;
2337
2338 if(!no_flush_rx)
2339 flush_fwd_rx_queues();
2340
2341 rxtx_config_display();
2342
2343 fwd_stats_reset();
2344 if (with_tx_first) {
2345 while (with_tx_first--) {
2346 launch_packet_forwarding(
2347 run_one_txonly_burst_on_core);
2348 rte_eal_mp_wait_lcore();
2349 }
2350 port_fwd_end = tx_only_engine.port_fwd_end;
2351 if (port_fwd_end != NULL) {
2352 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2353 (*port_fwd_end)(fwd_ports_ids[i]);
2354 }
2355 }
2356 launch_packet_forwarding(start_pkt_forward_on_core);
2357}
2358
2359void
2360stop_packet_forwarding(void)
2361{
2362 port_fwd_end_t port_fwd_end;
2363 lcoreid_t lc_id;
2364 portid_t pt_id;
2365 int i;
2366
2367 if (test_done) {
2368 fprintf(stderr, "Packet forwarding not started\n");
2369 return;
2370 }
2371 printf("Telling cores to stop...");
2372 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2373 fwd_lcores[lc_id]->stopped = 1;
2374 printf("\nWaiting for lcores to finish...\n");
2375 rte_eal_mp_wait_lcore();
2376 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2377 if (port_fwd_end != NULL) {
2378 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2379 pt_id = fwd_ports_ids[i];
2380 (*port_fwd_end)(pt_id);
2381 }
2382 }
2383
2384 fwd_stats_display();
2385
2386 printf("\nDone.\n");
2387 test_done = 1;
2388}
2389
2390void
2391dev_set_link_up(portid_t pid)
2392{
2393 if (rte_eth_dev_set_link_up(pid) < 0)
2394 fprintf(stderr, "\nSet link up fail.\n");
2395}
2396
2397void
2398dev_set_link_down(portid_t pid)
2399{
2400 if (rte_eth_dev_set_link_down(pid) < 0)
2401 fprintf(stderr, "\nSet link down fail.\n");
2402}
2403
2404static int
2405all_ports_started(void)
2406{
2407 portid_t pi;
2408 struct rte_port *port;
2409
2410 RTE_ETH_FOREACH_DEV(pi) {
2411 port = &ports[pi];
2412
2413 if ((port->port_status != RTE_PORT_STARTED) &&
2414 (port->slave_flag == 0))
2415 return 0;
2416 }
2417
2418
2419 return 1;
2420}
2421
2422int
2423port_is_stopped(portid_t port_id)
2424{
2425 struct rte_port *port = &ports[port_id];
2426
2427 if ((port->port_status != RTE_PORT_STOPPED) &&
2428 (port->slave_flag == 0))
2429 return 0;
2430 return 1;
2431}
2432
2433int
2434all_ports_stopped(void)
2435{
2436 portid_t pi;
2437
2438 RTE_ETH_FOREACH_DEV(pi) {
2439 if (!port_is_stopped(pi))
2440 return 0;
2441 }
2442
2443 return 1;
2444}
2445
2446int
2447port_is_started(portid_t port_id)
2448{
2449 if (port_id_is_invalid(port_id, ENABLED_WARN))
2450 return 0;
2451
2452 if (ports[port_id].port_status != RTE_PORT_STARTED)
2453 return 0;
2454
2455 return 1;
2456}
2457
2458
2459static int
2460setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2461{
2462 queueid_t qi;
2463 struct rte_eth_hairpin_conf hairpin_conf = {
2464 .peer_count = 1,
2465 };
2466 int i;
2467 int diag;
2468 struct rte_port *port = &ports[pi];
2469 uint16_t peer_rx_port = pi;
2470 uint16_t peer_tx_port = pi;
2471 uint32_t manual = 1;
2472 uint32_t tx_exp = hairpin_mode & 0x10;
2473
2474 if (!(hairpin_mode & 0xf)) {
2475 peer_rx_port = pi;
2476 peer_tx_port = pi;
2477 manual = 0;
2478 } else if (hairpin_mode & 0x1) {
2479 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2480 RTE_ETH_DEV_NO_OWNER);
2481 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2482 peer_tx_port = rte_eth_find_next_owned_by(0,
2483 RTE_ETH_DEV_NO_OWNER);
2484 if (p_pi != RTE_MAX_ETHPORTS) {
2485 peer_rx_port = p_pi;
2486 } else {
2487 uint16_t next_pi;
2488
2489
2490 RTE_ETH_FOREACH_DEV(next_pi)
2491 peer_rx_port = next_pi;
2492 }
2493 manual = 1;
2494 } else if (hairpin_mode & 0x2) {
2495 if (cnt_pi & 0x1) {
2496 peer_rx_port = p_pi;
2497 } else {
2498 peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2499 RTE_ETH_DEV_NO_OWNER);
2500 if (peer_rx_port >= RTE_MAX_ETHPORTS)
2501 peer_rx_port = pi;
2502 }
2503 peer_tx_port = peer_rx_port;
2504 manual = 1;
2505 }
2506
2507 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2508 hairpin_conf.peers[0].port = peer_rx_port;
2509 hairpin_conf.peers[0].queue = i + nb_rxq;
2510 hairpin_conf.manual_bind = !!manual;
2511 hairpin_conf.tx_explicit = !!tx_exp;
2512 diag = rte_eth_tx_hairpin_queue_setup
2513 (pi, qi, nb_txd, &hairpin_conf);
2514 i++;
2515 if (diag == 0)
2516 continue;
2517
2518
2519 if (port->port_status == RTE_PORT_HANDLING)
2520 port->port_status = RTE_PORT_STOPPED;
2521 else
2522 fprintf(stderr,
2523 "Port %d can not be set back to stopped\n", pi);
2524 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2525 pi);
2526
2527 port->need_reconfig_queues = 1;
2528 return -1;
2529 }
2530 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2531 hairpin_conf.peers[0].port = peer_tx_port;
2532 hairpin_conf.peers[0].queue = i + nb_txq;
2533 hairpin_conf.manual_bind = !!manual;
2534 hairpin_conf.tx_explicit = !!tx_exp;
2535 diag = rte_eth_rx_hairpin_queue_setup
2536 (pi, qi, nb_rxd, &hairpin_conf);
2537 i++;
2538 if (diag == 0)
2539 continue;
2540
2541
2542 if (port->port_status == RTE_PORT_HANDLING)
2543 port->port_status = RTE_PORT_STOPPED;
2544 else
2545 fprintf(stderr,
2546 "Port %d can not be set back to stopped\n", pi);
2547 fprintf(stderr, "Fail to configure port %d hairpin queues\n",
2548 pi);
2549
2550 port->need_reconfig_queues = 1;
2551 return -1;
2552 }
2553 return 0;
2554}
2555
2556
2557int
2558rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2559 uint16_t nb_rx_desc, unsigned int socket_id,
2560 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2561{
2562 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2563 unsigned int i, mp_n;
2564 int ret;
2565
2566 if (rx_pkt_nb_segs <= 1 ||
2567 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2568 rx_conf->rx_seg = NULL;
2569 rx_conf->rx_nseg = 0;
2570 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2571 nb_rx_desc, socket_id,
2572 rx_conf, mp);
2573 return ret;
2574 }
2575 for (i = 0; i < rx_pkt_nb_segs; i++) {
2576 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2577 struct rte_mempool *mpx;
2578
2579
2580
2581
2582 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2583 mpx = mbuf_pool_find(socket_id, mp_n);
2584
2585 rx_seg->length = rx_pkt_seg_lengths[i] ?
2586 rx_pkt_seg_lengths[i] :
2587 mbuf_data_size[mp_n];
2588 rx_seg->offset = i < rx_pkt_nb_offs ?
2589 rx_pkt_seg_offsets[i] : 0;
2590 rx_seg->mp = mpx ? mpx : mp;
2591 }
2592 rx_conf->rx_nseg = rx_pkt_nb_segs;
2593 rx_conf->rx_seg = rx_useg;
2594 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2595 socket_id, rx_conf, NULL);
2596 rx_conf->rx_seg = NULL;
2597 rx_conf->rx_nseg = 0;
2598 return ret;
2599}
2600
2601static int
2602alloc_xstats_display_info(portid_t pi)
2603{
2604 uint64_t **ids_supp = &ports[pi].xstats_info.ids_supp;
2605 uint64_t **prev_values = &ports[pi].xstats_info.prev_values;
2606 uint64_t **curr_values = &ports[pi].xstats_info.curr_values;
2607
2608 if (xstats_display_num == 0)
2609 return 0;
2610
2611 *ids_supp = calloc(xstats_display_num, sizeof(**ids_supp));
2612 if (*ids_supp == NULL)
2613 goto fail_ids_supp;
2614
2615 *prev_values = calloc(xstats_display_num,
2616 sizeof(**prev_values));
2617 if (*prev_values == NULL)
2618 goto fail_prev_values;
2619
2620 *curr_values = calloc(xstats_display_num,
2621 sizeof(**curr_values));
2622 if (*curr_values == NULL)
2623 goto fail_curr_values;
2624
2625 ports[pi].xstats_info.allocated = true;
2626
2627 return 0;
2628
2629fail_curr_values:
2630 free(*prev_values);
2631fail_prev_values:
2632 free(*ids_supp);
2633fail_ids_supp:
2634 return -ENOMEM;
2635}
2636
2637static void
2638free_xstats_display_info(portid_t pi)
2639{
2640 if (!ports[pi].xstats_info.allocated)
2641 return;
2642 free(ports[pi].xstats_info.ids_supp);
2643 free(ports[pi].xstats_info.prev_values);
2644 free(ports[pi].xstats_info.curr_values);
2645 ports[pi].xstats_info.allocated = false;
2646}
2647
2648
2649static void
2650fill_xstats_display_info_for_port(portid_t pi)
2651{
2652 unsigned int stat, stat_supp;
2653 const char *xstat_name;
2654 struct rte_port *port;
2655 uint64_t *ids_supp;
2656 int rc;
2657
2658 if (xstats_display_num == 0)
2659 return;
2660
2661 if (pi == (portid_t)RTE_PORT_ALL) {
2662 fill_xstats_display_info();
2663 return;
2664 }
2665
2666 port = &ports[pi];
2667 if (port->port_status != RTE_PORT_STARTED)
2668 return;
2669
2670 if (!port->xstats_info.allocated && alloc_xstats_display_info(pi) != 0)
2671 rte_exit(EXIT_FAILURE,
2672 "Failed to allocate xstats display memory\n");
2673
2674 ids_supp = port->xstats_info.ids_supp;
2675 for (stat = stat_supp = 0; stat < xstats_display_num; stat++) {
2676 xstat_name = xstats_display[stat].name;
2677 rc = rte_eth_xstats_get_id_by_name(pi, xstat_name,
2678 ids_supp + stat_supp);
2679 if (rc != 0) {
2680 fprintf(stderr, "No xstat '%s' on port %u - skip it %u\n",
2681 xstat_name, pi, stat);
2682 continue;
2683 }
2684 stat_supp++;
2685 }
2686
2687 port->xstats_info.ids_supp_sz = stat_supp;
2688}
2689
2690
2691static void
2692fill_xstats_display_info(void)
2693{
2694 portid_t pi;
2695
2696 if (xstats_display_num == 0)
2697 return;
2698
2699 RTE_ETH_FOREACH_DEV(pi)
2700 fill_xstats_display_info_for_port(pi);
2701}
2702
2703int
2704start_port(portid_t pid)
2705{
2706 int diag, need_check_link_status = -1;
2707 portid_t pi;
2708 portid_t p_pi = RTE_MAX_ETHPORTS;
2709 portid_t pl[RTE_MAX_ETHPORTS];
2710 portid_t peer_pl[RTE_MAX_ETHPORTS];
2711 uint16_t cnt_pi = 0;
2712 uint16_t cfg_pi = 0;
2713 int peer_pi;
2714 queueid_t qi;
2715 struct rte_port *port;
2716 struct rte_eth_hairpin_cap cap;
2717
2718 if (port_id_is_invalid(pid, ENABLED_WARN))
2719 return 0;
2720
2721 RTE_ETH_FOREACH_DEV(pi) {
2722 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2723 continue;
2724
2725 need_check_link_status = 0;
2726 port = &ports[pi];
2727 if (port->port_status == RTE_PORT_STOPPED)
2728 port->port_status = RTE_PORT_HANDLING;
2729 else {
2730 fprintf(stderr, "Port %d is now not stopped\n", pi);
2731 continue;
2732 }
2733
2734 if (port->need_reconfig > 0) {
2735 struct rte_eth_conf dev_conf;
2736 int k;
2737
2738 port->need_reconfig = 0;
2739
2740 if (flow_isolate_all) {
2741 int ret = port_flow_isolate(pi, 1);
2742 if (ret) {
2743 fprintf(stderr,
2744 "Failed to apply isolated mode on port %d\n",
2745 pi);
2746 return -1;
2747 }
2748 }
2749 configure_rxtx_dump_callbacks(0);
2750 printf("Configuring Port %d (socket %u)\n", pi,
2751 port->socket_id);
2752 if (nb_hairpinq > 0 &&
2753 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2754 fprintf(stderr,
2755 "Port %d doesn't support hairpin queues\n",
2756 pi);
2757 return -1;
2758 }
2759
2760
2761 diag = eth_dev_configure_mp(pi, nb_rxq + nb_hairpinq,
2762 nb_txq + nb_hairpinq,
2763 &(port->dev_conf));
2764 if (diag != 0) {
2765 if (port->port_status == RTE_PORT_HANDLING)
2766 port->port_status = RTE_PORT_STOPPED;
2767 else
2768 fprintf(stderr,
2769 "Port %d can not be set back to stopped\n",
2770 pi);
2771 fprintf(stderr, "Fail to configure port %d\n",
2772 pi);
2773
2774 port->need_reconfig = 1;
2775 return -1;
2776 }
2777
2778 if (0 !=
2779 eth_dev_conf_get_print_err(pi, &dev_conf)) {
2780 fprintf(stderr,
2781 "port %d can not get device configuration\n",
2782 pi);
2783 return -1;
2784 }
2785
2786 if (dev_conf.rxmode.offloads !=
2787 port->dev_conf.rxmode.offloads) {
2788 port->dev_conf.rxmode.offloads |=
2789 dev_conf.rxmode.offloads;
2790 for (k = 0;
2791 k < port->dev_info.max_rx_queues;
2792 k++)
2793 port->rx_conf[k].offloads |=
2794 dev_conf.rxmode.offloads;
2795 }
2796
2797 if (dev_conf.txmode.offloads !=
2798 port->dev_conf.txmode.offloads) {
2799 port->dev_conf.txmode.offloads |=
2800 dev_conf.txmode.offloads;
2801 for (k = 0;
2802 k < port->dev_info.max_tx_queues;
2803 k++)
2804 port->tx_conf[k].offloads |=
2805 dev_conf.txmode.offloads;
2806 }
2807 }
2808 if (port->need_reconfig_queues > 0 && is_proc_primary()) {
2809 port->need_reconfig_queues = 0;
2810
2811 for (qi = 0; qi < nb_txq; qi++) {
2812 if ((numa_support) &&
2813 (txring_numa[pi] != NUMA_NO_CONFIG))
2814 diag = rte_eth_tx_queue_setup(pi, qi,
2815 port->nb_tx_desc[qi],
2816 txring_numa[pi],
2817 &(port->tx_conf[qi]));
2818 else
2819 diag = rte_eth_tx_queue_setup(pi, qi,
2820 port->nb_tx_desc[qi],
2821 port->socket_id,
2822 &(port->tx_conf[qi]));
2823
2824 if (diag == 0)
2825 continue;
2826
2827
2828 if (port->port_status == RTE_PORT_HANDLING)
2829 port->port_status = RTE_PORT_STOPPED;
2830 else
2831 fprintf(stderr,
2832 "Port %d can not be set back to stopped\n",
2833 pi);
2834 fprintf(stderr,
2835 "Fail to configure port %d tx queues\n",
2836 pi);
2837
2838 port->need_reconfig_queues = 1;
2839 return -1;
2840 }
2841 for (qi = 0; qi < nb_rxq; qi++) {
2842
2843 if ((numa_support) &&
2844 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2845 struct rte_mempool * mp =
2846 mbuf_pool_find
2847 (rxring_numa[pi], 0);
2848 if (mp == NULL) {
2849 fprintf(stderr,
2850 "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2851 rxring_numa[pi]);
2852 return -1;
2853 }
2854
2855 diag = rx_queue_setup(pi, qi,
2856 port->nb_rx_desc[qi],
2857 rxring_numa[pi],
2858 &(port->rx_conf[qi]),
2859 mp);
2860 } else {
2861 struct rte_mempool *mp =
2862 mbuf_pool_find
2863 (port->socket_id, 0);
2864 if (mp == NULL) {
2865 fprintf(stderr,
2866 "Failed to setup RX queue: No mempool allocation on the socket %d\n",
2867 port->socket_id);
2868 return -1;
2869 }
2870 diag = rx_queue_setup(pi, qi,
2871 port->nb_rx_desc[qi],
2872 port->socket_id,
2873 &(port->rx_conf[qi]),
2874 mp);
2875 }
2876 if (diag == 0)
2877 continue;
2878
2879
2880 if (port->port_status == RTE_PORT_HANDLING)
2881 port->port_status = RTE_PORT_STOPPED;
2882 else
2883 fprintf(stderr,
2884 "Port %d can not be set back to stopped\n",
2885 pi);
2886 fprintf(stderr,
2887 "Fail to configure port %d rx queues\n",
2888 pi);
2889
2890 port->need_reconfig_queues = 1;
2891 return -1;
2892 }
2893
2894 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2895 return -1;
2896 }
2897 configure_rxtx_dump_callbacks(verbose_level);
2898 if (clear_ptypes) {
2899 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2900 NULL, 0);
2901 if (diag < 0)
2902 fprintf(stderr,
2903 "Port %d: Failed to disable Ptype parsing\n",
2904 pi);
2905 }
2906
2907 p_pi = pi;
2908 cnt_pi++;
2909
2910
2911 diag = eth_dev_start_mp(pi);
2912 if (diag < 0) {
2913 fprintf(stderr, "Fail to start port %d: %s\n",
2914 pi, rte_strerror(-diag));
2915
2916
2917 if (port->port_status == RTE_PORT_HANDLING)
2918 port->port_status = RTE_PORT_STOPPED;
2919 else
2920 fprintf(stderr,
2921 "Port %d can not be set back to stopped\n",
2922 pi);
2923 continue;
2924 }
2925
2926 if (port->port_status == RTE_PORT_HANDLING)
2927 port->port_status = RTE_PORT_STARTED;
2928 else
2929 fprintf(stderr, "Port %d can not be set into started\n",
2930 pi);
2931
2932 if (eth_macaddr_get_print_err(pi, &port->eth_addr) == 0)
2933 printf("Port %d: " RTE_ETHER_ADDR_PRT_FMT "\n", pi,
2934 RTE_ETHER_ADDR_BYTES(&port->eth_addr));
2935
2936
2937 need_check_link_status = 1;
2938
2939 pl[cfg_pi++] = pi;
2940 }
2941
2942 if (need_check_link_status == 1 && !no_link_check)
2943 check_all_ports_link_status(RTE_PORT_ALL);
2944 else if (need_check_link_status == 0)
2945 fprintf(stderr, "Please stop the ports first\n");
2946
2947 if (hairpin_mode & 0xf) {
2948 uint16_t i;
2949 int j;
2950
2951
2952 for (i = 0; i < cfg_pi; i++) {
2953 pi = pl[i];
2954
2955 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2956 RTE_MAX_ETHPORTS, 1);
2957 if (peer_pi < 0)
2958 return peer_pi;
2959 for (j = 0; j < peer_pi; j++) {
2960 if (!port_is_started(peer_pl[j]))
2961 continue;
2962 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2963 if (diag < 0) {
2964 fprintf(stderr,
2965 "Error during binding hairpin Tx port %u to %u: %s\n",
2966 pi, peer_pl[j],
2967 rte_strerror(-diag));
2968 return -1;
2969 }
2970 }
2971
2972 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2973 RTE_MAX_ETHPORTS, 0);
2974 if (peer_pi < 0)
2975 return peer_pi;
2976 for (j = 0; j < peer_pi; j++) {
2977 if (!port_is_started(peer_pl[j]))
2978 continue;
2979 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2980 if (diag < 0) {
2981 fprintf(stderr,
2982 "Error during binding hairpin Tx port %u to %u: %s\n",
2983 peer_pl[j], pi,
2984 rte_strerror(-diag));
2985 return -1;
2986 }
2987 }
2988 }
2989 }
2990
2991 fill_xstats_display_info_for_port(pid);
2992
2993 printf("Done\n");
2994 return 0;
2995}
2996
2997void
2998stop_port(portid_t pid)
2999{
3000 portid_t pi;
3001 struct rte_port *port;
3002 int need_check_link_status = 0;
3003 portid_t peer_pl[RTE_MAX_ETHPORTS];
3004 int peer_pi;
3005
3006 if (port_id_is_invalid(pid, ENABLED_WARN))
3007 return;
3008
3009 printf("Stopping ports...\n");
3010
3011 RTE_ETH_FOREACH_DEV(pi) {
3012 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3013 continue;
3014
3015 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3016 fprintf(stderr,
3017 "Please remove port %d from forwarding configuration.\n",
3018 pi);
3019 continue;
3020 }
3021
3022 if (port_is_bonding_slave(pi)) {
3023 fprintf(stderr,
3024 "Please remove port %d from bonded device.\n",
3025 pi);
3026 continue;
3027 }
3028
3029 port = &ports[pi];
3030 if (port->port_status == RTE_PORT_STARTED)
3031 port->port_status = RTE_PORT_HANDLING;
3032 else
3033 continue;
3034
3035 if (hairpin_mode & 0xf) {
3036 int j;
3037
3038 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
3039
3040 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
3041 RTE_MAX_ETHPORTS, 0);
3042 if (peer_pi < 0)
3043 continue;
3044 for (j = 0; j < peer_pi; j++) {
3045 if (!port_is_started(peer_pl[j]))
3046 continue;
3047 rte_eth_hairpin_unbind(peer_pl[j], pi);
3048 }
3049 }
3050
3051 if (port->flow_list)
3052 port_flow_flush(pi);
3053
3054 if (eth_dev_stop_mp(pi) != 0)
3055 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
3056 pi);
3057
3058 if (port->port_status == RTE_PORT_HANDLING)
3059 port->port_status = RTE_PORT_STOPPED;
3060 else
3061 fprintf(stderr, "Port %d can not be set into stopped\n",
3062 pi);
3063 need_check_link_status = 1;
3064 }
3065 if (need_check_link_status && !no_link_check)
3066 check_all_ports_link_status(RTE_PORT_ALL);
3067
3068 printf("Done\n");
3069}
3070
3071static void
3072remove_invalid_ports_in(portid_t *array, portid_t *total)
3073{
3074 portid_t i;
3075 portid_t new_total = 0;
3076
3077 for (i = 0; i < *total; i++)
3078 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
3079 array[new_total] = array[i];
3080 new_total++;
3081 }
3082 *total = new_total;
3083}
3084
3085static void
3086remove_invalid_ports(void)
3087{
3088 remove_invalid_ports_in(ports_ids, &nb_ports);
3089 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
3090 nb_cfg_ports = nb_fwd_ports;
3091}
3092
3093void
3094close_port(portid_t pid)
3095{
3096 portid_t pi;
3097 struct rte_port *port;
3098
3099 if (port_id_is_invalid(pid, ENABLED_WARN))
3100 return;
3101
3102 printf("Closing ports...\n");
3103
3104 RTE_ETH_FOREACH_DEV(pi) {
3105 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3106 continue;
3107
3108 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3109 fprintf(stderr,
3110 "Please remove port %d from forwarding configuration.\n",
3111 pi);
3112 continue;
3113 }
3114
3115 if (port_is_bonding_slave(pi)) {
3116 fprintf(stderr,
3117 "Please remove port %d from bonded device.\n",
3118 pi);
3119 continue;
3120 }
3121
3122 port = &ports[pi];
3123 if (port->port_status == RTE_PORT_CLOSED) {
3124 fprintf(stderr, "Port %d is already closed\n", pi);
3125 continue;
3126 }
3127
3128 if (is_proc_primary()) {
3129 port_flow_flush(pi);
3130 port_flex_item_flush(pi);
3131 rte_eth_dev_close(pi);
3132 }
3133
3134 free_xstats_display_info(pi);
3135 }
3136
3137 remove_invalid_ports();
3138 printf("Done\n");
3139}
3140
3141void
3142reset_port(portid_t pid)
3143{
3144 int diag;
3145 portid_t pi;
3146 struct rte_port *port;
3147
3148 if (port_id_is_invalid(pid, ENABLED_WARN))
3149 return;
3150
3151 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
3152 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
3153 fprintf(stderr,
3154 "Can not reset port(s), please stop port(s) first.\n");
3155 return;
3156 }
3157
3158 printf("Resetting ports...\n");
3159
3160 RTE_ETH_FOREACH_DEV(pi) {
3161 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
3162 continue;
3163
3164 if (port_is_forwarding(pi) != 0 && test_done == 0) {
3165 fprintf(stderr,
3166 "Please remove port %d from forwarding configuration.\n",
3167 pi);
3168 continue;
3169 }
3170
3171 if (port_is_bonding_slave(pi)) {
3172 fprintf(stderr,
3173 "Please remove port %d from bonded device.\n",
3174 pi);
3175 continue;
3176 }
3177
3178 diag = rte_eth_dev_reset(pi);
3179 if (diag == 0) {
3180 port = &ports[pi];
3181 port->need_reconfig = 1;
3182 port->need_reconfig_queues = 1;
3183 } else {
3184 fprintf(stderr, "Failed to reset port %d. diag=%d\n",
3185 pi, diag);
3186 }
3187 }
3188
3189 printf("Done\n");
3190}
3191
3192void
3193attach_port(char *identifier)
3194{
3195 portid_t pi;
3196 struct rte_dev_iterator iterator;
3197
3198 printf("Attaching a new port...\n");
3199
3200 if (identifier == NULL) {
3201 fprintf(stderr, "Invalid parameters are specified\n");
3202 return;
3203 }
3204
3205 if (rte_dev_probe(identifier) < 0) {
3206 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
3207 return;
3208 }
3209
3210
3211 if (setup_on_probe_event) {
3212
3213 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
3214 if (ports[pi].port_status == RTE_PORT_HANDLING &&
3215 ports[pi].need_setup != 0)
3216 setup_attached_port(pi);
3217 return;
3218 }
3219
3220
3221 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
3222
3223 if (port_is_forwarding(pi))
3224 continue;
3225 setup_attached_port(pi);
3226 }
3227}
3228
3229static void
3230setup_attached_port(portid_t pi)
3231{
3232 unsigned int socket_id;
3233 int ret;
3234
3235 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
3236
3237 if (check_socket_id(socket_id) < 0)
3238 socket_id = socket_ids[0];
3239 reconfig(pi, socket_id);
3240 ret = rte_eth_promiscuous_enable(pi);
3241 if (ret != 0)
3242 fprintf(stderr,
3243 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
3244 pi, rte_strerror(-ret));
3245
3246 ports_ids[nb_ports++] = pi;
3247 fwd_ports_ids[nb_fwd_ports++] = pi;
3248 nb_cfg_ports = nb_fwd_ports;
3249 ports[pi].need_setup = 0;
3250 ports[pi].port_status = RTE_PORT_STOPPED;
3251
3252 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
3253 printf("Done\n");
3254}
3255
3256static void
3257detach_device(struct rte_device *dev)
3258{
3259 portid_t sibling;
3260
3261 if (dev == NULL) {
3262 fprintf(stderr, "Device already removed\n");
3263 return;
3264 }
3265
3266 printf("Removing a device...\n");
3267
3268 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
3269 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
3270 if (ports[sibling].port_status != RTE_PORT_STOPPED) {
3271 fprintf(stderr, "Port %u not stopped\n",
3272 sibling);
3273 return;
3274 }
3275 port_flow_flush(sibling);
3276 }
3277 }
3278
3279 if (rte_dev_remove(dev) < 0) {
3280 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
3281 return;
3282 }
3283 remove_invalid_ports();
3284
3285 printf("Device is detached\n");
3286 printf("Now total ports is %d\n", nb_ports);
3287 printf("Done\n");
3288 return;
3289}
3290
3291void
3292detach_port_device(portid_t port_id)
3293{
3294 int ret;
3295 struct rte_eth_dev_info dev_info;
3296
3297 if (port_id_is_invalid(port_id, ENABLED_WARN))
3298 return;
3299
3300 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3301 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3302 fprintf(stderr, "Port not stopped\n");
3303 return;
3304 }
3305 fprintf(stderr, "Port was not closed\n");
3306 }
3307
3308 ret = eth_dev_info_get_print_err(port_id, &dev_info);
3309 if (ret != 0) {
3310 TESTPMD_LOG(ERR,
3311 "Failed to get device info for port %d, not detaching\n",
3312 port_id);
3313 return;
3314 }
3315 detach_device(dev_info.device);
3316}
3317
3318void
3319detach_devargs(char *identifier)
3320{
3321 struct rte_dev_iterator iterator;
3322 struct rte_devargs da;
3323 portid_t port_id;
3324
3325 printf("Removing a device...\n");
3326
3327 memset(&da, 0, sizeof(da));
3328 if (rte_devargs_parsef(&da, "%s", identifier)) {
3329 fprintf(stderr, "cannot parse identifier\n");
3330 return;
3331 }
3332
3333 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3334 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3335 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3336 fprintf(stderr, "Port %u not stopped\n",
3337 port_id);
3338 rte_eth_iterator_cleanup(&iterator);
3339 rte_devargs_reset(&da);
3340 return;
3341 }
3342 port_flow_flush(port_id);
3343 }
3344 }
3345
3346 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3347 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3348 da.name, da.bus->name);
3349 rte_devargs_reset(&da);
3350 return;
3351 }
3352
3353 remove_invalid_ports();
3354
3355 printf("Device %s is detached\n", identifier);
3356 printf("Now total ports is %d\n", nb_ports);
3357 printf("Done\n");
3358 rte_devargs_reset(&da);
3359}
3360
3361void
3362pmd_test_exit(void)
3363{
3364 portid_t pt_id;
3365 unsigned int i;
3366 int ret;
3367
3368 if (test_done == 0)
3369 stop_packet_forwarding();
3370
3371#ifndef RTE_EXEC_ENV_WINDOWS
3372 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3373 if (mempools[i]) {
3374 if (mp_alloc_type == MP_ALLOC_ANON)
3375 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3376 NULL);
3377 }
3378 }
3379#endif
3380 if (ports != NULL) {
3381 no_link_check = 1;
3382 RTE_ETH_FOREACH_DEV(pt_id) {
3383 printf("\nStopping port %d...\n", pt_id);
3384 fflush(stdout);
3385 stop_port(pt_id);
3386 }
3387 RTE_ETH_FOREACH_DEV(pt_id) {
3388 printf("\nShutting down port %d...\n", pt_id);
3389 fflush(stdout);
3390 close_port(pt_id);
3391 }
3392 }
3393
3394 if (hot_plug) {
3395 ret = rte_dev_event_monitor_stop();
3396 if (ret) {
3397 RTE_LOG(ERR, EAL,
3398 "fail to stop device event monitor.");
3399 return;
3400 }
3401
3402 ret = rte_dev_event_callback_unregister(NULL,
3403 dev_event_callback, NULL);
3404 if (ret < 0) {
3405 RTE_LOG(ERR, EAL,
3406 "fail to unregister device event callback.\n");
3407 return;
3408 }
3409
3410 ret = rte_dev_hotplug_handle_disable();
3411 if (ret) {
3412 RTE_LOG(ERR, EAL,
3413 "fail to disable hotplug handling.\n");
3414 return;
3415 }
3416 }
3417 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3418 if (mempools[i])
3419 mempool_free_mp(mempools[i]);
3420 }
3421 free(xstats_display);
3422
3423 printf("\nBye...\n");
3424}
3425
3426typedef void (*cmd_func_t)(void);
3427struct pmd_test_command {
3428 const char *cmd_name;
3429 cmd_func_t cmd_func;
3430};
3431
3432
3433static void
3434check_all_ports_link_status(uint32_t port_mask)
3435{
3436#define CHECK_INTERVAL 100
3437#define MAX_CHECK_TIME 90
3438 portid_t portid;
3439 uint8_t count, all_ports_up, print_flag = 0;
3440 struct rte_eth_link link;
3441 int ret;
3442 char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3443
3444 printf("Checking link statuses...\n");
3445 fflush(stdout);
3446 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3447 all_ports_up = 1;
3448 RTE_ETH_FOREACH_DEV(portid) {
3449 if ((port_mask & (1 << portid)) == 0)
3450 continue;
3451 memset(&link, 0, sizeof(link));
3452 ret = rte_eth_link_get_nowait(portid, &link);
3453 if (ret < 0) {
3454 all_ports_up = 0;
3455 if (print_flag == 1)
3456 fprintf(stderr,
3457 "Port %u link get failed: %s\n",
3458 portid, rte_strerror(-ret));
3459 continue;
3460 }
3461
3462 if (print_flag == 1) {
3463 rte_eth_link_to_str(link_status,
3464 sizeof(link_status), &link);
3465 printf("Port %d %s\n", portid, link_status);
3466 continue;
3467 }
3468
3469 if (link.link_status == RTE_ETH_LINK_DOWN) {
3470 all_ports_up = 0;
3471 break;
3472 }
3473 }
3474
3475 if (print_flag == 1)
3476 break;
3477
3478 if (all_ports_up == 0) {
3479 fflush(stdout);
3480 rte_delay_ms(CHECK_INTERVAL);
3481 }
3482
3483
3484 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3485 print_flag = 1;
3486 }
3487
3488 if (lsc_interrupt)
3489 break;
3490 }
3491}
3492
3493static void
3494rmv_port_callback(void *arg)
3495{
3496 int need_to_start = 0;
3497 int org_no_link_check = no_link_check;
3498 portid_t port_id = (intptr_t)arg;
3499 struct rte_eth_dev_info dev_info;
3500 int ret;
3501
3502 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3503
3504 if (!test_done && port_is_forwarding(port_id)) {
3505 need_to_start = 1;
3506 stop_packet_forwarding();
3507 }
3508 no_link_check = 1;
3509 stop_port(port_id);
3510 no_link_check = org_no_link_check;
3511
3512 ret = eth_dev_info_get_print_err(port_id, &dev_info);
3513 if (ret != 0)
3514 TESTPMD_LOG(ERR,
3515 "Failed to get device info for port %d, not detaching\n",
3516 port_id);
3517 else {
3518 struct rte_device *device = dev_info.device;
3519 close_port(port_id);
3520 detach_device(device);
3521 }
3522 if (need_to_start)
3523 start_packet_forwarding(0);
3524}
3525
3526
3527static int
3528eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3529 void *ret_param)
3530{
3531 RTE_SET_USED(param);
3532 RTE_SET_USED(ret_param);
3533
3534 if (type >= RTE_ETH_EVENT_MAX) {
3535 fprintf(stderr,
3536 "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3537 port_id, __func__, type);
3538 fflush(stderr);
3539 } else if (event_print_mask & (UINT32_C(1) << type)) {
3540 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3541 eth_event_desc[type]);
3542 fflush(stdout);
3543 }
3544
3545 switch (type) {
3546 case RTE_ETH_EVENT_NEW:
3547 ports[port_id].need_setup = 1;
3548 ports[port_id].port_status = RTE_PORT_HANDLING;
3549 break;
3550 case RTE_ETH_EVENT_INTR_RMV:
3551 if (port_id_is_invalid(port_id, DISABLED_WARN))
3552 break;
3553 if (rte_eal_alarm_set(100000,
3554 rmv_port_callback, (void *)(intptr_t)port_id))
3555 fprintf(stderr,
3556 "Could not set up deferred device removal\n");
3557 break;
3558 case RTE_ETH_EVENT_DESTROY:
3559 ports[port_id].port_status = RTE_PORT_CLOSED;
3560 printf("Port %u is closed\n", port_id);
3561 break;
3562 default:
3563 break;
3564 }
3565 return 0;
3566}
3567
3568static int
3569register_eth_event_callback(void)
3570{
3571 int ret;
3572 enum rte_eth_event_type event;
3573
3574 for (event = RTE_ETH_EVENT_UNKNOWN;
3575 event < RTE_ETH_EVENT_MAX; event++) {
3576 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3577 event,
3578 eth_event_callback,
3579 NULL);
3580 if (ret != 0) {
3581 TESTPMD_LOG(ERR, "Failed to register callback for "
3582 "%s event\n", eth_event_desc[event]);
3583 return -1;
3584 }
3585 }
3586
3587 return 0;
3588}
3589
3590
3591static void
3592dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3593 __rte_unused void *arg)
3594{
3595 uint16_t port_id;
3596 int ret;
3597
3598 if (type >= RTE_DEV_EVENT_MAX) {
3599 fprintf(stderr, "%s called upon invalid event %d\n",
3600 __func__, type);
3601 fflush(stderr);
3602 }
3603
3604 switch (type) {
3605 case RTE_DEV_EVENT_REMOVE:
3606 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3607 device_name);
3608 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3609 if (ret) {
3610 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3611 device_name);
3612 return;
3613 }
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623 if (rte_eal_alarm_set(100000,
3624 rmv_port_callback, (void *)(intptr_t)port_id))
3625 RTE_LOG(ERR, EAL,
3626 "Could not set up deferred device removal\n");
3627 break;
3628 case RTE_DEV_EVENT_ADD:
3629 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3630 device_name);
3631
3632
3633
3634 break;
3635 default:
3636 break;
3637 }
3638}
3639
3640static void
3641rxtx_port_config(portid_t pid)
3642{
3643 uint16_t qid;
3644 uint64_t offloads;
3645 struct rte_port *port = &ports[pid];
3646
3647 for (qid = 0; qid < nb_rxq; qid++) {
3648 offloads = port->rx_conf[qid].offloads;
3649 port->rx_conf[qid] = port->dev_info.default_rxconf;
3650
3651 if (rxq_share > 0 &&
3652 (port->dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE)) {
3653
3654 port->rx_conf[qid].share_group = pid / rxq_share + 1;
3655 port->rx_conf[qid].share_qid = qid;
3656 }
3657
3658 if (offloads != 0)
3659 port->rx_conf[qid].offloads = offloads;
3660
3661
3662 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3663 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3664
3665 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3666 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3667
3668 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3669 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3670
3671 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3672 port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3673
3674 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3675 port->rx_conf[qid].rx_drop_en = rx_drop_en;
3676
3677 port->nb_rx_desc[qid] = nb_rxd;
3678 }
3679
3680 for (qid = 0; qid < nb_txq; qid++) {
3681 offloads = port->tx_conf[qid].offloads;
3682 port->tx_conf[qid] = port->dev_info.default_txconf;
3683 if (offloads != 0)
3684 port->tx_conf[qid].offloads = offloads;
3685
3686
3687 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3688 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3689
3690 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3691 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3692
3693 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3694 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3695
3696 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3697 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3698
3699 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3700 port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3701
3702 port->nb_tx_desc[qid] = nb_txd;
3703 }
3704}
3705
3706
3707
3708
3709
3710
3711
3712
3713int
3714update_mtu_from_frame_size(portid_t portid, uint32_t max_rx_pktlen)
3715{
3716 struct rte_port *port = &ports[portid];
3717 uint32_t eth_overhead;
3718 uint16_t mtu, new_mtu;
3719
3720 eth_overhead = get_eth_overhead(&port->dev_info);
3721
3722 if (rte_eth_dev_get_mtu(portid, &mtu) != 0) {
3723 printf("Failed to get MTU for port %u\n", portid);
3724 return -1;
3725 }
3726
3727 new_mtu = max_rx_pktlen - eth_overhead;
3728
3729 if (mtu == new_mtu)
3730 return 0;
3731
3732 if (eth_dev_set_mtu_mp(portid, new_mtu) != 0) {
3733 fprintf(stderr,
3734 "Failed to set MTU to %u for port %u\n",
3735 new_mtu, portid);
3736 return -1;
3737 }
3738
3739 port->dev_conf.rxmode.mtu = new_mtu;
3740
3741 return 0;
3742}
3743
3744void
3745init_port_config(void)
3746{
3747 portid_t pid;
3748 struct rte_port *port;
3749 int ret, i;
3750
3751 RTE_ETH_FOREACH_DEV(pid) {
3752 port = &ports[pid];
3753 port->dev_conf.fdir_conf = fdir_conf;
3754
3755 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3756 if (ret != 0)
3757 return;
3758
3759 if (nb_rxq > 1) {
3760 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3761 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3762 rss_hf & port->dev_info.flow_type_rss_offloads;
3763 } else {
3764 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3765 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3766 }
3767
3768 if (port->dcb_flag == 0) {
3769 if (port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) {
3770 port->dev_conf.rxmode.mq_mode =
3771 (enum rte_eth_rx_mq_mode)
3772 (rx_mq_mode & RTE_ETH_MQ_RX_RSS);
3773 } else {
3774 port->dev_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
3775 port->dev_conf.rxmode.offloads &=
3776 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3777
3778 for (i = 0;
3779 i < port->dev_info.nb_rx_queues;
3780 i++)
3781 port->rx_conf[i].offloads &=
3782 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3783 }
3784 }
3785
3786 rxtx_port_config(pid);
3787
3788 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3789 if (ret != 0)
3790 return;
3791
3792#if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3793 rte_pmd_ixgbe_bypass_init(pid);
3794#endif
3795
3796 if (lsc_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_LSC))
3797 port->dev_conf.intr_conf.lsc = 1;
3798 if (rmv_interrupt && (*port->dev_info.dev_flags & RTE_ETH_DEV_INTR_RMV))
3799 port->dev_conf.intr_conf.rmv = 1;
3800 }
3801}
3802
3803void set_port_slave_flag(portid_t slave_pid)
3804{
3805 struct rte_port *port;
3806
3807 port = &ports[slave_pid];
3808 port->slave_flag = 1;
3809}
3810
3811void clear_port_slave_flag(portid_t slave_pid)
3812{
3813 struct rte_port *port;
3814
3815 port = &ports[slave_pid];
3816 port->slave_flag = 0;
3817}
3818
3819uint8_t port_is_bonding_slave(portid_t slave_pid)
3820{
3821 struct rte_port *port;
3822 struct rte_eth_dev_info dev_info;
3823 int ret;
3824
3825 port = &ports[slave_pid];
3826 ret = eth_dev_info_get_print_err(slave_pid, &dev_info);
3827 if (ret != 0) {
3828 TESTPMD_LOG(ERR,
3829 "Failed to get device info for port id %d,"
3830 "cannot determine if the port is a bonded slave",
3831 slave_pid);
3832 return 0;
3833 }
3834 if ((*dev_info.dev_flags & RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3835 return 1;
3836 return 0;
3837}
3838
3839const uint16_t vlan_tags[] = {
3840 0, 1, 2, 3, 4, 5, 6, 7,
3841 8, 9, 10, 11, 12, 13, 14, 15,
3842 16, 17, 18, 19, 20, 21, 22, 23,
3843 24, 25, 26, 27, 28, 29, 30, 31
3844};
3845
3846static int
3847get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3848 enum dcb_mode_enable dcb_mode,
3849 enum rte_eth_nb_tcs num_tcs,
3850 uint8_t pfc_en)
3851{
3852 uint8_t i;
3853 int32_t rc;
3854 struct rte_eth_rss_conf rss_conf;
3855
3856
3857
3858
3859
3860 if (dcb_mode == DCB_VT_ENABLED) {
3861 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3862 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3863 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3864 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3865
3866
3867 vmdq_rx_conf->enable_default_pool = 0;
3868 vmdq_rx_conf->default_pool = 0;
3869 vmdq_rx_conf->nb_queue_pools =
3870 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3871 vmdq_tx_conf->nb_queue_pools =
3872 (num_tcs == RTE_ETH_4_TCS ? RTE_ETH_32_POOLS : RTE_ETH_16_POOLS);
3873
3874 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3875 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3876 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3877 vmdq_rx_conf->pool_map[i].pools =
3878 1 << (i % vmdq_rx_conf->nb_queue_pools);
3879 }
3880 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3881 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3882 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3883 }
3884
3885
3886 eth_conf->rxmode.mq_mode =
3887 (enum rte_eth_rx_mq_mode)
3888 (rx_mq_mode & RTE_ETH_MQ_RX_VMDQ_DCB);
3889 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_VMDQ_DCB;
3890 } else {
3891 struct rte_eth_dcb_rx_conf *rx_conf =
3892 ð_conf->rx_adv_conf.dcb_rx_conf;
3893 struct rte_eth_dcb_tx_conf *tx_conf =
3894 ð_conf->tx_adv_conf.dcb_tx_conf;
3895
3896 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3897
3898 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3899 if (rc != 0)
3900 return rc;
3901
3902 rx_conf->nb_tcs = num_tcs;
3903 tx_conf->nb_tcs = num_tcs;
3904
3905 for (i = 0; i < RTE_ETH_DCB_NUM_USER_PRIORITIES; i++) {
3906 rx_conf->dcb_tc[i] = i % num_tcs;
3907 tx_conf->dcb_tc[i] = i % num_tcs;
3908 }
3909
3910 eth_conf->rxmode.mq_mode =
3911 (enum rte_eth_rx_mq_mode)
3912 (rx_mq_mode & RTE_ETH_MQ_RX_DCB_RSS);
3913 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3914 eth_conf->txmode.mq_mode = RTE_ETH_MQ_TX_DCB;
3915 }
3916
3917 if (pfc_en)
3918 eth_conf->dcb_capability_en =
3919 RTE_ETH_DCB_PG_SUPPORT | RTE_ETH_DCB_PFC_SUPPORT;
3920 else
3921 eth_conf->dcb_capability_en = RTE_ETH_DCB_PG_SUPPORT;
3922
3923 return 0;
3924}
3925
3926int
3927init_port_dcb_config(portid_t pid,
3928 enum dcb_mode_enable dcb_mode,
3929 enum rte_eth_nb_tcs num_tcs,
3930 uint8_t pfc_en)
3931{
3932 struct rte_eth_conf port_conf;
3933 struct rte_port *rte_port;
3934 int retval;
3935 uint16_t i;
3936
3937 if (num_procs > 1) {
3938 printf("The multi-process feature doesn't support dcb.\n");
3939 return -ENOTSUP;
3940 }
3941 rte_port = &ports[pid];
3942
3943
3944 memcpy(&port_conf, &rte_port->dev_conf, sizeof(struct rte_eth_conf));
3945
3946
3947 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3948 if (retval < 0)
3949 return retval;
3950 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
3951
3952 if (port_conf.rxmode.mq_mode == RTE_ETH_MQ_RX_VMDQ_DCB) {
3953 port_conf.rxmode.offloads &= ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3954 for (i = 0; i < nb_rxq; i++)
3955 rte_port->rx_conf[i].offloads &=
3956 ~RTE_ETH_RX_OFFLOAD_RSS_HASH;
3957 }
3958
3959
3960 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3961 if (retval < 0)
3962 return retval;
3963
3964 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3965 if (retval != 0)
3966 return retval;
3967
3968
3969
3970
3971 if (dcb_mode == DCB_VT_ENABLED &&
3972 rte_port->dev_info.vmdq_pool_base > 0) {
3973 fprintf(stderr,
3974 "VMDQ_DCB multi-queue mode is nonsensical for port %d.\n",
3975 pid);
3976 return -1;
3977 }
3978
3979
3980
3981
3982 if (dcb_mode == DCB_VT_ENABLED) {
3983 if (rte_port->dev_info.max_vfs > 0) {
3984 nb_rxq = rte_port->dev_info.nb_rx_queues;
3985 nb_txq = rte_port->dev_info.nb_tx_queues;
3986 } else {
3987 nb_rxq = rte_port->dev_info.max_rx_queues;
3988 nb_txq = rte_port->dev_info.max_tx_queues;
3989 }
3990 } else {
3991
3992 if (rte_port->dev_info.vmdq_pool_base == 0) {
3993 nb_rxq = rte_port->dev_info.max_rx_queues;
3994 nb_txq = rte_port->dev_info.max_tx_queues;
3995 } else {
3996 nb_rxq = (queueid_t)num_tcs;
3997 nb_txq = (queueid_t)num_tcs;
3998
3999 }
4000 }
4001 rx_free_thresh = 64;
4002
4003 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
4004
4005 rxtx_port_config(pid);
4006
4007 rte_port->dev_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_VLAN_FILTER;
4008 for (i = 0; i < RTE_DIM(vlan_tags); i++)
4009 rx_vft_set(pid, vlan_tags[i], 1);
4010
4011 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
4012 if (retval != 0)
4013 return retval;
4014
4015 rte_port->dcb_flag = 1;
4016
4017
4018 dcb_config = 1;
4019
4020 return 0;
4021}
4022
4023static void
4024init_port(void)
4025{
4026 int i;
4027
4028
4029 ports = rte_zmalloc("testpmd: ports",
4030 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
4031 RTE_CACHE_LINE_SIZE);
4032 if (ports == NULL) {
4033 rte_exit(EXIT_FAILURE,
4034 "rte_zmalloc(%d struct rte_port) failed\n",
4035 RTE_MAX_ETHPORTS);
4036 }
4037 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4038 ports[i].xstats_info.allocated = false;
4039 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4040 LIST_INIT(&ports[i].flow_tunnel_list);
4041
4042 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4043 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4044 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
4045}
4046
4047static void
4048force_quit(void)
4049{
4050 pmd_test_exit();
4051 prompt_exit();
4052}
4053
4054static void
4055print_stats(void)
4056{
4057 uint8_t i;
4058 const char clr[] = { 27, '[', '2', 'J', '\0' };
4059 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
4060
4061
4062 printf("%s%s", clr, top_left);
4063
4064 printf("\nPort statistics ====================================");
4065 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
4066 nic_stats_display(fwd_ports_ids[i]);
4067
4068 fflush(stdout);
4069}
4070
4071static void
4072signal_handler(int signum)
4073{
4074 if (signum == SIGINT || signum == SIGTERM) {
4075 fprintf(stderr, "\nSignal %d received, preparing to exit...\n",
4076 signum);
4077#ifdef RTE_LIB_PDUMP
4078
4079 rte_pdump_uninit();
4080#endif
4081#ifdef RTE_LIB_LATENCYSTATS
4082 if (latencystats_enabled != 0)
4083 rte_latencystats_uninit();
4084#endif
4085 force_quit();
4086
4087 f_quit = 1;
4088
4089#ifndef RTE_EXEC_ENV_WINDOWS
4090 signal(signum, SIG_DFL);
4091 kill(getpid(), signum);
4092#endif
4093 }
4094}
4095
4096int
4097main(int argc, char** argv)
4098{
4099 int diag;
4100 portid_t port_id;
4101 uint16_t count;
4102 int ret;
4103
4104 signal(SIGINT, signal_handler);
4105 signal(SIGTERM, signal_handler);
4106
4107 testpmd_logtype = rte_log_register("testpmd");
4108 if (testpmd_logtype < 0)
4109 rte_exit(EXIT_FAILURE, "Cannot register log type");
4110 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
4111
4112 diag = rte_eal_init(argc, argv);
4113 if (diag < 0)
4114 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
4115 rte_strerror(rte_errno));
4116
4117 ret = register_eth_event_callback();
4118 if (ret != 0)
4119 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
4120
4121#ifdef RTE_LIB_PDUMP
4122
4123 rte_pdump_init();
4124#endif
4125
4126 count = 0;
4127 RTE_ETH_FOREACH_DEV(port_id) {
4128 ports_ids[count] = port_id;
4129 count++;
4130 }
4131 nb_ports = (portid_t) count;
4132 if (nb_ports == 0)
4133 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
4134
4135
4136 init_port();
4137
4138 set_def_fwd_config();
4139 if (nb_lcores == 0)
4140 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
4141 "Check the core mask argument\n");
4142
4143
4144#ifdef RTE_LIB_BITRATESTATS
4145 bitrate_enabled = 0;
4146#endif
4147#ifdef RTE_LIB_LATENCYSTATS
4148 latencystats_enabled = 0;
4149#endif
4150
4151
4152#ifdef RTE_EXEC_ENV_FREEBSD
4153 do_mlockall = 0;
4154#else
4155 do_mlockall = 1;
4156#endif
4157
4158 argc -= diag;
4159 argv += diag;
4160 if (argc > 1)
4161 launch_args_parse(argc, argv);
4162
4163#ifndef RTE_EXEC_ENV_WINDOWS
4164 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
4165 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
4166 strerror(errno));
4167 }
4168#endif
4169
4170 if (tx_first && interactive)
4171 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
4172 "interactive mode.\n");
4173
4174 if (tx_first && lsc_interrupt) {
4175 fprintf(stderr,
4176 "Warning: lsc_interrupt needs to be off when using tx_first. Disabling.\n");
4177 lsc_interrupt = 0;
4178 }
4179
4180 if (!nb_rxq && !nb_txq)
4181 fprintf(stderr,
4182 "Warning: Either rx or tx queues should be non-zero\n");
4183
4184 if (nb_rxq > 1 && nb_rxq > nb_txq)
4185 fprintf(stderr,
4186 "Warning: nb_rxq=%d enables RSS configuration, but nb_txq=%d will prevent to fully test it.\n",
4187 nb_rxq, nb_txq);
4188
4189 init_config();
4190
4191 if (hot_plug) {
4192 ret = rte_dev_hotplug_handle_enable();
4193 if (ret) {
4194 RTE_LOG(ERR, EAL,
4195 "fail to enable hotplug handling.");
4196 return -1;
4197 }
4198
4199 ret = rte_dev_event_monitor_start();
4200 if (ret) {
4201 RTE_LOG(ERR, EAL,
4202 "fail to start device event monitoring.");
4203 return -1;
4204 }
4205
4206 ret = rte_dev_event_callback_register(NULL,
4207 dev_event_callback, NULL);
4208 if (ret) {
4209 RTE_LOG(ERR, EAL,
4210 "fail to register device event callback\n");
4211 return -1;
4212 }
4213 }
4214
4215 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
4216 rte_exit(EXIT_FAILURE, "Start ports failed\n");
4217
4218
4219 RTE_ETH_FOREACH_DEV(port_id) {
4220 ret = rte_eth_promiscuous_enable(port_id);
4221 if (ret != 0)
4222 fprintf(stderr,
4223 "Error during enabling promiscuous mode for port %u: %s - ignore\n",
4224 port_id, rte_strerror(-ret));
4225 }
4226
4227#ifdef RTE_LIB_METRICS
4228
4229 rte_metrics_init(rte_socket_id());
4230#endif
4231
4232#ifdef RTE_LIB_LATENCYSTATS
4233 if (latencystats_enabled != 0) {
4234 int ret = rte_latencystats_init(1, NULL);
4235 if (ret)
4236 fprintf(stderr,
4237 "Warning: latencystats init() returned error %d\n",
4238 ret);
4239 fprintf(stderr, "Latencystats running on lcore %d\n",
4240 latencystats_lcore_id);
4241 }
4242#endif
4243
4244
4245#ifdef RTE_LIB_BITRATESTATS
4246 if (bitrate_enabled != 0) {
4247 bitrate_data = rte_stats_bitrate_create();
4248 if (bitrate_data == NULL)
4249 rte_exit(EXIT_FAILURE,
4250 "Could not allocate bitrate data.\n");
4251 rte_stats_bitrate_reg(bitrate_data);
4252 }
4253#endif
4254#ifdef RTE_LIB_CMDLINE
4255 if (strlen(cmdline_filename) != 0)
4256 cmdline_read_from_file(cmdline_filename);
4257
4258 if (interactive == 1) {
4259 if (auto_start) {
4260 printf("Start automatic packet forwarding\n");
4261 start_packet_forwarding(0);
4262 }
4263 prompt();
4264 pmd_test_exit();
4265 } else
4266#endif
4267 {
4268 char c;
4269 int rc;
4270
4271 f_quit = 0;
4272
4273 printf("No commandline core given, start packet forwarding\n");
4274 start_packet_forwarding(tx_first);
4275 if (stats_period != 0) {
4276 uint64_t prev_time = 0, cur_time, diff_time = 0;
4277 uint64_t timer_period;
4278
4279
4280 timer_period = stats_period * rte_get_timer_hz();
4281
4282 while (f_quit == 0) {
4283 cur_time = rte_get_timer_cycles();
4284 diff_time += cur_time - prev_time;
4285
4286 if (diff_time >= timer_period) {
4287 print_stats();
4288
4289 diff_time = 0;
4290 }
4291
4292 prev_time = cur_time;
4293 rte_delay_us_sleep(US_PER_S);
4294 }
4295 }
4296
4297 printf("Press enter to exit\n");
4298 rc = read(0, &c, 1);
4299 pmd_test_exit();
4300 if (rc < 0)
4301 return 1;
4302 }
4303
4304 ret = rte_eal_cleanup();
4305 if (ret != 0)
4306 rte_exit(EXIT_FAILURE,
4307 "EAL cleanup failed: %s\n", strerror(-ret));
4308
4309 return EXIT_SUCCESS;
4310}
4311