1
2
3
4
5#include <stdarg.h>
6#include <stdio.h>
7#include <stdlib.h>
8#include <signal.h>
9#include <string.h>
10#include <time.h>
11#include <fcntl.h>
12#include <sys/mman.h>
13#include <sys/types.h>
14#include <errno.h>
15#include <stdbool.h>
16
17#include <sys/queue.h>
18#include <sys/stat.h>
19
20#include <stdint.h>
21#include <unistd.h>
22#include <inttypes.h>
23
24#include <rte_common.h>
25#include <rte_errno.h>
26#include <rte_byteorder.h>
27#include <rte_log.h>
28#include <rte_debug.h>
29#include <rte_cycles.h>
30#include <rte_memory.h>
31#include <rte_memcpy.h>
32#include <rte_launch.h>
33#include <rte_eal.h>
34#include <rte_alarm.h>
35#include <rte_per_lcore.h>
36#include <rte_lcore.h>
37#include <rte_atomic.h>
38#include <rte_branch_prediction.h>
39#include <rte_mempool.h>
40#include <rte_malloc.h>
41#include <rte_mbuf.h>
42#include <rte_mbuf_pool_ops.h>
43#include <rte_interrupts.h>
44#include <rte_pci.h>
45#include <rte_ether.h>
46#include <rte_ethdev.h>
47#include <rte_dev.h>
48#include <rte_string_fns.h>
49#ifdef RTE_NET_IXGBE
50#include <rte_pmd_ixgbe.h>
51#endif
52#ifdef RTE_LIB_PDUMP
53#include <rte_pdump.h>
54#endif
55#include <rte_flow.h>
56#include <rte_metrics.h>
57#ifdef RTE_LIB_BITRATESTATS
58#include <rte_bitrate.h>
59#endif
60#ifdef RTE_LIB_LATENCYSTATS
61#include <rte_latencystats.h>
62#endif
63
64#include "testpmd.h"
65
66#ifndef MAP_HUGETLB
67
68#define HUGE_FLAG (0x40000)
69#else
70#define HUGE_FLAG MAP_HUGETLB
71#endif
72
73#ifndef MAP_HUGE_SHIFT
74
75#define HUGE_SHIFT (26)
76#else
77#define HUGE_SHIFT MAP_HUGE_SHIFT
78#endif
79
80#define EXTMEM_HEAP_NAME "extmem"
81#define EXTBUF_ZONE_SIZE RTE_PGSIZE_2M
82
83uint16_t verbose_level = 0;
84int testpmd_logtype;
85
86
87uint8_t interactive = 0;
88uint8_t auto_start = 0;
89uint8_t tx_first;
90char cmdline_filename[PATH_MAX] = {0};
91
92
93
94
95
96
97
98
99uint8_t numa_support = 1;
100
101
102
103
104
105uint8_t socket_num = UMA_NO_CONFIG;
106
107
108
109
110
111
112
113
114uint8_t mp_alloc_type = MP_ALLOC_NATIVE;
115
116
117
118
119
120uint8_t port_numa[RTE_MAX_ETHPORTS];
121
122
123
124
125
126uint8_t rxring_numa[RTE_MAX_ETHPORTS];
127
128
129
130
131
132uint8_t txring_numa[RTE_MAX_ETHPORTS];
133
134
135
136
137
138
139
140struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS];
141portid_t nb_peer_eth_addrs = 0;
142
143
144
145
146struct rte_port *ports;
147portid_t nb_ports;
148struct fwd_lcore **fwd_lcores;
149lcoreid_t nb_lcores;
150
151portid_t ports_ids[RTE_MAX_ETHPORTS];
152
153
154
155
156
157
158lcoreid_t nb_cfg_lcores;
159lcoreid_t nb_fwd_lcores;
160portid_t nb_cfg_ports;
161portid_t nb_fwd_ports;
162
163unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE];
164portid_t fwd_ports_ids[RTE_MAX_ETHPORTS];
165
166struct fwd_stream **fwd_streams;
167streamid_t nb_fwd_streams;
168
169
170
171
172struct fwd_engine * fwd_engines[] = {
173 &io_fwd_engine,
174 &mac_fwd_engine,
175 &mac_swap_engine,
176 &flow_gen_engine,
177 &rx_only_engine,
178 &tx_only_engine,
179 &csum_fwd_engine,
180 &icmp_echo_engine,
181 &noisy_vnf_engine,
182 &five_tuple_swap_fwd_engine,
183#ifdef RTE_LIBRTE_IEEE1588
184 &ieee1588_fwd_engine,
185#endif
186 NULL,
187};
188
189struct rte_mempool *mempools[RTE_MAX_NUMA_NODES * MAX_SEGS_BUFFER_SPLIT];
190uint16_t mempool_flags;
191
192struct fwd_config cur_fwd_config;
193struct fwd_engine *cur_fwd_eng = &io_fwd_engine;
194uint32_t retry_enabled;
195uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
196uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
197
198uint32_t mbuf_data_size_n = 1;
199uint16_t mbuf_data_size[MAX_SEGS_BUFFER_SPLIT] = {
200 DEFAULT_MBUF_DATA_SIZE
201};
202uint32_t param_total_num_mbufs = 0;
203
204uint16_t stats_period;
205
206
207
208
209
210uint8_t f_quit;
211
212
213
214
215
216uint16_t rx_pkt_seg_lengths[MAX_SEGS_BUFFER_SPLIT];
217uint8_t rx_pkt_nb_segs;
218uint16_t rx_pkt_seg_offsets[MAX_SEGS_BUFFER_SPLIT];
219uint8_t rx_pkt_nb_offs;
220
221
222
223
224uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN;
225uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = {
226 TXONLY_DEF_PACKET_LEN,
227};
228uint8_t tx_pkt_nb_segs = 1;
229
230enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF;
231
232
233uint8_t txonly_multi_flow;
234
235
236uint32_t tx_pkt_times_inter;
237
238
239uint32_t tx_pkt_times_intra;
240
241
242uint16_t nb_pkt_per_burst = DEF_PKT_BURST;
243uint16_t nb_pkt_flowgen_clones;
244uint16_t mb_mempool_cache = DEF_MBUF_CACHE;
245
246
247uint8_t dcb_config = 0;
248
249
250uint8_t dcb_test = 0;
251
252
253
254
255queueid_t nb_hairpinq;
256queueid_t nb_rxq = 1;
257queueid_t nb_txq = 1;
258
259
260
261
262
263#define RTE_TEST_RX_DESC_DEFAULT 0
264#define RTE_TEST_TX_DESC_DEFAULT 0
265uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
266uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
267
268#define RTE_PMD_PARAM_UNSET -1
269
270
271
272
273int8_t rx_pthresh = RTE_PMD_PARAM_UNSET;
274int8_t rx_hthresh = RTE_PMD_PARAM_UNSET;
275int8_t rx_wthresh = RTE_PMD_PARAM_UNSET;
276
277int8_t tx_pthresh = RTE_PMD_PARAM_UNSET;
278int8_t tx_hthresh = RTE_PMD_PARAM_UNSET;
279int8_t tx_wthresh = RTE_PMD_PARAM_UNSET;
280
281
282
283
284int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET;
285
286
287
288
289int8_t rx_drop_en = RTE_PMD_PARAM_UNSET;
290
291
292
293
294int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET;
295
296
297
298
299int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET;
300
301
302
303
304uint16_t noisy_tx_sw_bufsz;
305
306
307
308
309uint16_t noisy_tx_sw_buf_flush_time;
310
311
312
313
314
315uint64_t noisy_lkup_mem_sz;
316
317
318
319
320
321uint64_t noisy_lkup_num_writes;
322
323
324
325
326
327uint64_t noisy_lkup_num_reads;
328
329
330
331
332
333uint64_t noisy_lkup_num_reads_writes;
334
335
336
337
338uint64_t rss_hf = ETH_RSS_IP;
339
340
341
342
343uint16_t port_topology = PORT_TOPOLOGY_PAIRED;
344
345
346
347
348uint8_t no_flush_rx = 0;
349
350
351
352
353uint8_t flow_isolate_all;
354
355
356
357
358uint8_t no_link_check = 0;
359
360
361
362
363uint8_t no_device_start = 0;
364
365
366
367
368uint8_t lsc_interrupt = 1;
369
370
371
372
373uint8_t rmv_interrupt = 1;
374
375uint8_t hot_plug = 0;
376
377
378bool setup_on_probe_event = true;
379
380
381uint8_t clear_ptypes = true;
382
383
384uint16_t hairpin_mode;
385
386
387static const char * const eth_event_desc[] = {
388 [RTE_ETH_EVENT_UNKNOWN] = "unknown",
389 [RTE_ETH_EVENT_INTR_LSC] = "link state change",
390 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state",
391 [RTE_ETH_EVENT_INTR_RESET] = "reset",
392 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox",
393 [RTE_ETH_EVENT_IPSEC] = "IPsec",
394 [RTE_ETH_EVENT_MACSEC] = "MACsec",
395 [RTE_ETH_EVENT_INTR_RMV] = "device removal",
396 [RTE_ETH_EVENT_NEW] = "device probed",
397 [RTE_ETH_EVENT_DESTROY] = "device released",
398 [RTE_ETH_EVENT_FLOW_AGED] = "flow aged",
399 [RTE_ETH_EVENT_MAX] = NULL,
400};
401
402
403
404
405
406uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) |
407 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) |
408 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) |
409 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) |
410 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) |
411 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) |
412 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV) |
413 (UINT32_C(1) << RTE_ETH_EVENT_FLOW_AGED);
414
415
416
417int do_mlockall = 0;
418
419
420
421
422
423#if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
424
425uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF;
426#endif
427
428
429#ifdef RTE_LIB_LATENCYSTATS
430
431
432
433
434uint8_t latencystats_enabled;
435
436
437
438
439lcoreid_t latencystats_lcore_id = -1;
440
441#endif
442
443
444
445
446struct rte_eth_rxmode rx_mode = {
447
448
449
450
451 .max_rx_pkt_len = 0,
452};
453
454struct rte_eth_txmode tx_mode = {
455 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE,
456};
457
458struct rte_fdir_conf fdir_conf = {
459 .mode = RTE_FDIR_MODE_NONE,
460 .pballoc = RTE_FDIR_PBALLOC_64K,
461 .status = RTE_FDIR_REPORT_STATUS,
462 .mask = {
463 .vlan_tci_mask = 0xFFEF,
464 .ipv4_mask = {
465 .src_ip = 0xFFFFFFFF,
466 .dst_ip = 0xFFFFFFFF,
467 },
468 .ipv6_mask = {
469 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
470 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF},
471 },
472 .src_port_mask = 0xFFFF,
473 .dst_port_mask = 0xFFFF,
474 .mac_addr_byte_mask = 0xFF,
475 .tunnel_type_mask = 1,
476 .tunnel_id_mask = 0xFFFFFFFF,
477 },
478 .drop_queue = 127,
479};
480
481volatile int test_done = 1;
482
483
484
485
486uint8_t xstats_hide_zero;
487
488
489
490
491uint8_t record_core_cycles;
492
493
494
495
496uint8_t record_burst_stats;
497
498unsigned int num_sockets = 0;
499unsigned int socket_ids[RTE_MAX_NUMA_NODES];
500
501#ifdef RTE_LIB_BITRATESTATS
502
503struct rte_stats_bitrates *bitrate_data;
504lcoreid_t bitrate_lcore_id;
505uint8_t bitrate_enabled;
506#endif
507
508struct gro_status gro_ports[RTE_MAX_ETHPORTS];
509uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES;
510
511
512
513
514enum rte_eth_rx_mq_mode rx_mq_mode = ETH_MQ_RX_VMDQ_DCB_RSS;
515
516
517static void setup_attached_port(portid_t pi);
518static void check_all_ports_link_status(uint32_t port_mask);
519static int eth_event_callback(portid_t port_id,
520 enum rte_eth_event_type type,
521 void *param, void *ret_param);
522static void dev_event_callback(const char *device_name,
523 enum rte_dev_event_type type,
524 void *param);
525
526
527
528
529
530static int all_ports_started(void);
531
532struct gso_status gso_ports[RTE_MAX_ETHPORTS];
533uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN;
534
535
536char dynf_names[64][RTE_MBUF_DYN_NAMESIZE];
537
538
539
540
541
542int
543new_socket_id(unsigned int socket_id)
544{
545 unsigned int i;
546
547 for (i = 0; i < num_sockets; i++) {
548 if (socket_ids[i] == socket_id)
549 return 0;
550 }
551 return 1;
552}
553
554
555
556
557static void
558set_default_fwd_lcores_config(void)
559{
560 unsigned int i;
561 unsigned int nb_lc;
562 unsigned int sock_num;
563
564 nb_lc = 0;
565 for (i = 0; i < RTE_MAX_LCORE; i++) {
566 if (!rte_lcore_is_enabled(i))
567 continue;
568 sock_num = rte_lcore_to_socket_id(i);
569 if (new_socket_id(sock_num)) {
570 if (num_sockets >= RTE_MAX_NUMA_NODES) {
571 rte_exit(EXIT_FAILURE,
572 "Total sockets greater than %u\n",
573 RTE_MAX_NUMA_NODES);
574 }
575 socket_ids[num_sockets++] = sock_num;
576 }
577 if (i == rte_get_main_lcore())
578 continue;
579 fwd_lcores_cpuids[nb_lc++] = i;
580 }
581 nb_lcores = (lcoreid_t) nb_lc;
582 nb_cfg_lcores = nb_lcores;
583 nb_fwd_lcores = 1;
584}
585
586static void
587set_def_peer_eth_addrs(void)
588{
589 portid_t i;
590
591 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
592 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR;
593 peer_eth_addrs[i].addr_bytes[5] = i;
594 }
595}
596
597static void
598set_default_fwd_ports_config(void)
599{
600 portid_t pt_id;
601 int i = 0;
602
603 RTE_ETH_FOREACH_DEV(pt_id) {
604 fwd_ports_ids[i++] = pt_id;
605
606
607 int socket_id = rte_eth_dev_socket_id(pt_id);
608 if (socket_id >= 0 && new_socket_id(socket_id)) {
609 if (num_sockets >= RTE_MAX_NUMA_NODES) {
610 rte_exit(EXIT_FAILURE,
611 "Total sockets greater than %u\n",
612 RTE_MAX_NUMA_NODES);
613 }
614 socket_ids[num_sockets++] = socket_id;
615 }
616 }
617
618 nb_cfg_ports = nb_ports;
619 nb_fwd_ports = nb_ports;
620}
621
622void
623set_def_fwd_config(void)
624{
625 set_default_fwd_lcores_config();
626 set_def_peer_eth_addrs();
627 set_default_fwd_ports_config();
628}
629
630
631static int
632calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out)
633{
634 unsigned int n_pages, mbuf_per_pg, leftover;
635 uint64_t total_mem, mbuf_mem, obj_sz;
636
637
638
639
640
641
642 uint64_t hdr_mem = 128 << 20;
643
644
645 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL);
646 if (obj_sz > pgsz) {
647 TESTPMD_LOG(ERR, "Object size is bigger than page size\n");
648 return -1;
649 }
650
651 mbuf_per_pg = pgsz / obj_sz;
652 leftover = (nb_mbufs % mbuf_per_pg) > 0;
653 n_pages = (nb_mbufs / mbuf_per_pg) + leftover;
654
655 mbuf_mem = n_pages * pgsz;
656
657 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz);
658
659 if (total_mem > SIZE_MAX) {
660 TESTPMD_LOG(ERR, "Memory size too big\n");
661 return -1;
662 }
663 *out = (size_t)total_mem;
664
665 return 0;
666}
667
668static int
669pagesz_flags(uint64_t page_sz)
670{
671
672
673
674 int log2 = rte_log2_u64(page_sz);
675
676 return (log2 << HUGE_SHIFT);
677}
678
679static void *
680alloc_mem(size_t memsz, size_t pgsz, bool huge)
681{
682 void *addr;
683 int flags;
684
685
686 flags = MAP_ANONYMOUS | MAP_PRIVATE;
687 if (huge)
688 flags |= HUGE_FLAG | pagesz_flags(pgsz);
689
690 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0);
691 if (addr == MAP_FAILED)
692 return NULL;
693
694 return addr;
695}
696
697struct extmem_param {
698 void *addr;
699 size_t len;
700 size_t pgsz;
701 rte_iova_t *iova_table;
702 unsigned int iova_table_len;
703};
704
705static int
706create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param,
707 bool huge)
708{
709 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G,
710 RTE_PGSIZE_16M, RTE_PGSIZE_16G};
711 unsigned int cur_page, n_pages, pgsz_idx;
712 size_t mem_sz, cur_pgsz;
713 rte_iova_t *iovas = NULL;
714 void *addr;
715 int ret;
716
717 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) {
718
719 if (pgsizes[pgsz_idx] > SIZE_MAX)
720 continue;
721
722 cur_pgsz = pgsizes[pgsz_idx];
723
724
725 if (!huge)
726 cur_pgsz = sysconf(_SC_PAGESIZE);
727
728 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz);
729 if (ret < 0) {
730 TESTPMD_LOG(ERR, "Cannot calculate memory size\n");
731 return -1;
732 }
733
734
735 addr = alloc_mem(mem_sz, cur_pgsz, huge);
736
737
738
739
740
741 if (addr == NULL)
742 continue;
743
744
745 n_pages = mem_sz / cur_pgsz;
746
747 iovas = malloc(sizeof(*iovas) * n_pages);
748
749 if (iovas == NULL) {
750 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n");
751 goto fail;
752 }
753
754 if (!huge)
755 mlock(addr, mem_sz);
756
757
758 for (cur_page = 0; cur_page < n_pages; cur_page++) {
759 rte_iova_t iova;
760 size_t offset;
761 void *cur;
762
763 offset = cur_pgsz * cur_page;
764 cur = RTE_PTR_ADD(addr, offset);
765
766
767 *(volatile char *)cur = 0;
768
769 iova = rte_mem_virt2iova(cur);
770
771 iovas[cur_page] = iova;
772 }
773
774 break;
775 }
776
777 if (iovas == NULL)
778 return -1;
779
780 param->addr = addr;
781 param->len = mem_sz;
782 param->pgsz = cur_pgsz;
783 param->iova_table = iovas;
784 param->iova_table_len = n_pages;
785
786 return 0;
787fail:
788 if (iovas)
789 free(iovas);
790 if (addr)
791 munmap(addr, mem_sz);
792
793 return -1;
794}
795
796static int
797setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge)
798{
799 struct extmem_param param;
800 int socket_id, ret;
801
802 memset(¶m, 0, sizeof(param));
803
804
805 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
806 if (socket_id < 0) {
807
808 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME);
809 if (ret < 0) {
810 TESTPMD_LOG(ERR, "Cannot create heap\n");
811 return -1;
812 }
813 }
814
815 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
816 if (ret < 0) {
817 TESTPMD_LOG(ERR, "Cannot create memory area\n");
818 return -1;
819 }
820
821
822 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
823 param.addr, param.len, param.iova_table,
824 param.iova_table_len, param.pgsz);
825
826
827
828
829 free(param.iova_table);
830
831 if (ret < 0) {
832 TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
833 munmap(param.addr, param.len);
834 return -1;
835 }
836
837
838
839 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
840 param.len >> 20);
841
842 return 0;
843}
844static void
845dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
846 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
847{
848 uint16_t pid = 0;
849 int ret;
850
851 RTE_ETH_FOREACH_DEV(pid) {
852 struct rte_eth_dev *dev =
853 &rte_eth_devices[pid];
854
855 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0,
856 memhdr->len);
857 if (ret) {
858 TESTPMD_LOG(DEBUG,
859 "unable to DMA unmap addr 0x%p "
860 "for device %s\n",
861 memhdr->addr, dev->data->name);
862 }
863 }
864 ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
865 if (ret) {
866 TESTPMD_LOG(DEBUG,
867 "unable to un-register addr 0x%p\n", memhdr->addr);
868 }
869}
870
871static void
872dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
873 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
874{
875 uint16_t pid = 0;
876 size_t page_size = sysconf(_SC_PAGESIZE);
877 int ret;
878
879 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
880 page_size);
881 if (ret) {
882 TESTPMD_LOG(DEBUG,
883 "unable to register addr 0x%p\n", memhdr->addr);
884 return;
885 }
886 RTE_ETH_FOREACH_DEV(pid) {
887 struct rte_eth_dev *dev =
888 &rte_eth_devices[pid];
889
890 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0,
891 memhdr->len);
892 if (ret) {
893 TESTPMD_LOG(DEBUG,
894 "unable to DMA map addr 0x%p "
895 "for device %s\n",
896 memhdr->addr, dev->data->name);
897 }
898 }
899}
900
901static unsigned int
902setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
903 char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
904{
905 struct rte_pktmbuf_extmem *xmem;
906 unsigned int ext_num, zone_num, elt_num;
907 uint16_t elt_size;
908
909 elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
910 elt_num = EXTBUF_ZONE_SIZE / elt_size;
911 zone_num = (nb_mbufs + elt_num - 1) / elt_num;
912
913 xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
914 if (xmem == NULL) {
915 TESTPMD_LOG(ERR, "Cannot allocate memory for "
916 "external buffer descriptors\n");
917 *ext_mem = NULL;
918 return 0;
919 }
920 for (ext_num = 0; ext_num < zone_num; ext_num++) {
921 struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
922 const struct rte_memzone *mz;
923 char mz_name[RTE_MEMZONE_NAMESIZE];
924 int ret;
925
926 ret = snprintf(mz_name, sizeof(mz_name),
927 RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
928 if (ret < 0 || ret >= (int)sizeof(mz_name)) {
929 errno = ENAMETOOLONG;
930 ext_num = 0;
931 break;
932 }
933 mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
934 socket_id,
935 RTE_MEMZONE_IOVA_CONTIG |
936 RTE_MEMZONE_1GB |
937 RTE_MEMZONE_SIZE_HINT_ONLY,
938 EXTBUF_ZONE_SIZE);
939 if (mz == NULL) {
940
941
942
943
944 errno = ENOMEM;
945 ext_num = 0;
946 break;
947 }
948 xseg->buf_ptr = mz->addr;
949 xseg->buf_iova = mz->iova;
950 xseg->buf_len = EXTBUF_ZONE_SIZE;
951 xseg->elt_size = elt_size;
952 }
953 if (ext_num == 0 && xmem != NULL) {
954 free(xmem);
955 xmem = NULL;
956 }
957 *ext_mem = xmem;
958 return ext_num;
959}
960
961
962
963
964static struct rte_mempool *
965mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
966 unsigned int socket_id, uint16_t size_idx)
967{
968 char pool_name[RTE_MEMPOOL_NAMESIZE];
969 struct rte_mempool *rte_mp = NULL;
970 uint32_t mb_size;
971
972 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
973 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
974
975 TESTPMD_LOG(INFO,
976 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
977 pool_name, nb_mbuf, mbuf_seg_size, socket_id);
978
979 switch (mp_alloc_type) {
980 case MP_ALLOC_NATIVE:
981 {
982
983 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
984 rte_mbuf_best_mempool_ops());
985 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
986 mb_mempool_cache, 0, mbuf_seg_size, socket_id);
987 break;
988 }
989 case MP_ALLOC_ANON:
990 {
991 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
992 mb_size, (unsigned int) mb_mempool_cache,
993 sizeof(struct rte_pktmbuf_pool_private),
994 socket_id, mempool_flags);
995 if (rte_mp == NULL)
996 goto err;
997
998 if (rte_mempool_populate_anon(rte_mp) == 0) {
999 rte_mempool_free(rte_mp);
1000 rte_mp = NULL;
1001 goto err;
1002 }
1003 rte_pktmbuf_pool_init(rte_mp, NULL);
1004 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
1005 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
1006 break;
1007 }
1008 case MP_ALLOC_XMEM:
1009 case MP_ALLOC_XMEM_HUGE:
1010 {
1011 int heap_socket;
1012 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
1013
1014 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
1015 rte_exit(EXIT_FAILURE, "Could not create external memory\n");
1016
1017 heap_socket =
1018 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
1019 if (heap_socket < 0)
1020 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
1021
1022 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1023 rte_mbuf_best_mempool_ops());
1024 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
1025 mb_mempool_cache, 0, mbuf_seg_size,
1026 heap_socket);
1027 break;
1028 }
1029 case MP_ALLOC_XBUF:
1030 {
1031 struct rte_pktmbuf_extmem *ext_mem;
1032 unsigned int ext_num;
1033
1034 ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
1035 socket_id, pool_name, &ext_mem);
1036 if (ext_num == 0)
1037 rte_exit(EXIT_FAILURE,
1038 "Can't create pinned data buffers\n");
1039
1040 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
1041 rte_mbuf_best_mempool_ops());
1042 rte_mp = rte_pktmbuf_pool_create_extbuf
1043 (pool_name, nb_mbuf, mb_mempool_cache,
1044 0, mbuf_seg_size, socket_id,
1045 ext_mem, ext_num);
1046 free(ext_mem);
1047 break;
1048 }
1049 default:
1050 {
1051 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
1052 }
1053 }
1054
1055err:
1056 if (rte_mp == NULL) {
1057 rte_exit(EXIT_FAILURE,
1058 "Creation of mbuf pool for socket %u failed: %s\n",
1059 socket_id, rte_strerror(rte_errno));
1060 } else if (verbose_level > 0) {
1061 rte_mempool_dump(stdout, rte_mp);
1062 }
1063 return rte_mp;
1064}
1065
1066
1067
1068
1069
1070static int
1071check_socket_id(const unsigned int socket_id)
1072{
1073 static int warning_once = 0;
1074
1075 if (new_socket_id(socket_id)) {
1076 if (!warning_once && numa_support)
1077 printf("Warning: NUMA should be configured manually by"
1078 " using --port-numa-config and"
1079 " --ring-numa-config parameters along with"
1080 " --numa.\n");
1081 warning_once = 1;
1082 return -1;
1083 }
1084 return 0;
1085}
1086
1087
1088
1089
1090
1091
1092queueid_t
1093get_allowed_max_nb_rxq(portid_t *pid)
1094{
1095 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
1096 bool max_rxq_valid = false;
1097 portid_t pi;
1098 struct rte_eth_dev_info dev_info;
1099
1100 RTE_ETH_FOREACH_DEV(pi) {
1101 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1102 continue;
1103
1104 max_rxq_valid = true;
1105 if (dev_info.max_rx_queues < allowed_max_rxq) {
1106 allowed_max_rxq = dev_info.max_rx_queues;
1107 *pid = pi;
1108 }
1109 }
1110 return max_rxq_valid ? allowed_max_rxq : 0;
1111}
1112
1113
1114
1115
1116
1117
1118
1119int
1120check_nb_rxq(queueid_t rxq)
1121{
1122 queueid_t allowed_max_rxq;
1123 portid_t pid = 0;
1124
1125 allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
1126 if (rxq > allowed_max_rxq) {
1127 printf("Fail: input rxq (%u) can't be greater "
1128 "than max_rx_queues (%u) of port %u\n",
1129 rxq,
1130 allowed_max_rxq,
1131 pid);
1132 return -1;
1133 }
1134 return 0;
1135}
1136
1137
1138
1139
1140
1141
1142queueid_t
1143get_allowed_max_nb_txq(portid_t *pid)
1144{
1145 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
1146 bool max_txq_valid = false;
1147 portid_t pi;
1148 struct rte_eth_dev_info dev_info;
1149
1150 RTE_ETH_FOREACH_DEV(pi) {
1151 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1152 continue;
1153
1154 max_txq_valid = true;
1155 if (dev_info.max_tx_queues < allowed_max_txq) {
1156 allowed_max_txq = dev_info.max_tx_queues;
1157 *pid = pi;
1158 }
1159 }
1160 return max_txq_valid ? allowed_max_txq : 0;
1161}
1162
1163
1164
1165
1166
1167
1168
1169int
1170check_nb_txq(queueid_t txq)
1171{
1172 queueid_t allowed_max_txq;
1173 portid_t pid = 0;
1174
1175 allowed_max_txq = get_allowed_max_nb_txq(&pid);
1176 if (txq > allowed_max_txq) {
1177 printf("Fail: input txq (%u) can't be greater "
1178 "than max_tx_queues (%u) of port %u\n",
1179 txq,
1180 allowed_max_txq,
1181 pid);
1182 return -1;
1183 }
1184 return 0;
1185}
1186
1187
1188
1189
1190
1191
1192static uint16_t
1193get_allowed_max_nb_rxd(portid_t *pid)
1194{
1195 uint16_t allowed_max_rxd = UINT16_MAX;
1196 portid_t pi;
1197 struct rte_eth_dev_info dev_info;
1198
1199 RTE_ETH_FOREACH_DEV(pi) {
1200 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1201 continue;
1202
1203 if (dev_info.rx_desc_lim.nb_max < allowed_max_rxd) {
1204 allowed_max_rxd = dev_info.rx_desc_lim.nb_max;
1205 *pid = pi;
1206 }
1207 }
1208 return allowed_max_rxd;
1209}
1210
1211
1212
1213
1214
1215
1216static uint16_t
1217get_allowed_min_nb_rxd(portid_t *pid)
1218{
1219 uint16_t allowed_min_rxd = 0;
1220 portid_t pi;
1221 struct rte_eth_dev_info dev_info;
1222
1223 RTE_ETH_FOREACH_DEV(pi) {
1224 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1225 continue;
1226
1227 if (dev_info.rx_desc_lim.nb_min > allowed_min_rxd) {
1228 allowed_min_rxd = dev_info.rx_desc_lim.nb_min;
1229 *pid = pi;
1230 }
1231 }
1232
1233 return allowed_min_rxd;
1234}
1235
1236
1237
1238
1239
1240
1241
1242
1243int
1244check_nb_rxd(queueid_t rxd)
1245{
1246 uint16_t allowed_max_rxd;
1247 uint16_t allowed_min_rxd;
1248 portid_t pid = 0;
1249
1250 allowed_max_rxd = get_allowed_max_nb_rxd(&pid);
1251 if (rxd > allowed_max_rxd) {
1252 printf("Fail: input rxd (%u) can't be greater "
1253 "than max_rxds (%u) of port %u\n",
1254 rxd,
1255 allowed_max_rxd,
1256 pid);
1257 return -1;
1258 }
1259
1260 allowed_min_rxd = get_allowed_min_nb_rxd(&pid);
1261 if (rxd < allowed_min_rxd) {
1262 printf("Fail: input rxd (%u) can't be less "
1263 "than min_rxds (%u) of port %u\n",
1264 rxd,
1265 allowed_min_rxd,
1266 pid);
1267 return -1;
1268 }
1269
1270 return 0;
1271}
1272
1273
1274
1275
1276
1277
1278static uint16_t
1279get_allowed_max_nb_txd(portid_t *pid)
1280{
1281 uint16_t allowed_max_txd = UINT16_MAX;
1282 portid_t pi;
1283 struct rte_eth_dev_info dev_info;
1284
1285 RTE_ETH_FOREACH_DEV(pi) {
1286 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1287 continue;
1288
1289 if (dev_info.tx_desc_lim.nb_max < allowed_max_txd) {
1290 allowed_max_txd = dev_info.tx_desc_lim.nb_max;
1291 *pid = pi;
1292 }
1293 }
1294 return allowed_max_txd;
1295}
1296
1297
1298
1299
1300
1301
1302static uint16_t
1303get_allowed_min_nb_txd(portid_t *pid)
1304{
1305 uint16_t allowed_min_txd = 0;
1306 portid_t pi;
1307 struct rte_eth_dev_info dev_info;
1308
1309 RTE_ETH_FOREACH_DEV(pi) {
1310 if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
1311 continue;
1312
1313 if (dev_info.tx_desc_lim.nb_min > allowed_min_txd) {
1314 allowed_min_txd = dev_info.tx_desc_lim.nb_min;
1315 *pid = pi;
1316 }
1317 }
1318
1319 return allowed_min_txd;
1320}
1321
1322
1323
1324
1325
1326
1327
1328int
1329check_nb_txd(queueid_t txd)
1330{
1331 uint16_t allowed_max_txd;
1332 uint16_t allowed_min_txd;
1333 portid_t pid = 0;
1334
1335 allowed_max_txd = get_allowed_max_nb_txd(&pid);
1336 if (txd > allowed_max_txd) {
1337 printf("Fail: input txd (%u) can't be greater "
1338 "than max_txds (%u) of port %u\n",
1339 txd,
1340 allowed_max_txd,
1341 pid);
1342 return -1;
1343 }
1344
1345 allowed_min_txd = get_allowed_min_nb_txd(&pid);
1346 if (txd < allowed_min_txd) {
1347 printf("Fail: input txd (%u) can't be less "
1348 "than min_txds (%u) of port %u\n",
1349 txd,
1350 allowed_min_txd,
1351 pid);
1352 return -1;
1353 }
1354 return 0;
1355}
1356
1357
1358
1359
1360
1361
1362
1363queueid_t
1364get_allowed_max_nb_hairpinq(portid_t *pid)
1365{
1366 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT;
1367 portid_t pi;
1368 struct rte_eth_hairpin_cap cap;
1369
1370 RTE_ETH_FOREACH_DEV(pi) {
1371 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) {
1372 *pid = pi;
1373 return 0;
1374 }
1375 if (cap.max_nb_queues < allowed_max_hairpinq) {
1376 allowed_max_hairpinq = cap.max_nb_queues;
1377 *pid = pi;
1378 }
1379 }
1380 return allowed_max_hairpinq;
1381}
1382
1383
1384
1385
1386
1387
1388
1389int
1390check_nb_hairpinq(queueid_t hairpinq)
1391{
1392 queueid_t allowed_max_hairpinq;
1393 portid_t pid = 0;
1394
1395 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid);
1396 if (hairpinq > allowed_max_hairpinq) {
1397 printf("Fail: input hairpin (%u) can't be greater "
1398 "than max_hairpin_queues (%u) of port %u\n",
1399 hairpinq, allowed_max_hairpinq, pid);
1400 return -1;
1401 }
1402 return 0;
1403}
1404
1405static void
1406init_config(void)
1407{
1408 portid_t pid;
1409 struct rte_port *port;
1410 struct rte_mempool *mbp;
1411 unsigned int nb_mbuf_per_pool;
1412 lcoreid_t lc_id;
1413 uint8_t port_per_socket[RTE_MAX_NUMA_NODES];
1414 struct rte_gro_param gro_param;
1415 uint32_t gso_types;
1416 uint16_t data_size;
1417 bool warning = 0;
1418 int k;
1419 int ret;
1420
1421 memset(port_per_socket,0,RTE_MAX_NUMA_NODES);
1422
1423
1424 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores",
1425 sizeof(struct fwd_lcore *) * nb_lcores,
1426 RTE_CACHE_LINE_SIZE);
1427 if (fwd_lcores == NULL) {
1428 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) "
1429 "failed\n", nb_lcores);
1430 }
1431 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1432 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore",
1433 sizeof(struct fwd_lcore),
1434 RTE_CACHE_LINE_SIZE);
1435 if (fwd_lcores[lc_id] == NULL) {
1436 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) "
1437 "failed\n");
1438 }
1439 fwd_lcores[lc_id]->cpuid_idx = lc_id;
1440 }
1441
1442 RTE_ETH_FOREACH_DEV(pid) {
1443 port = &ports[pid];
1444
1445 port->dev_conf.txmode = tx_mode;
1446 port->dev_conf.rxmode = rx_mode;
1447
1448 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
1449 if (ret != 0)
1450 rte_exit(EXIT_FAILURE,
1451 "rte_eth_dev_info_get() failed\n");
1452
1453 ret = update_jumbo_frame_offload(pid);
1454 if (ret != 0)
1455 printf("Updating jumbo frame offload failed for port %u\n",
1456 pid);
1457
1458 if (!(port->dev_info.tx_offload_capa &
1459 DEV_TX_OFFLOAD_MBUF_FAST_FREE))
1460 port->dev_conf.txmode.offloads &=
1461 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1462 if (numa_support) {
1463 if (port_numa[pid] != NUMA_NO_CONFIG)
1464 port_per_socket[port_numa[pid]]++;
1465 else {
1466 uint32_t socket_id = rte_eth_dev_socket_id(pid);
1467
1468
1469
1470
1471
1472 if (check_socket_id(socket_id) < 0)
1473 socket_id = socket_ids[0];
1474 port_per_socket[socket_id]++;
1475 }
1476 }
1477
1478
1479 for (k = 0; k < port->dev_info.max_rx_queues; k++)
1480 port->rx_conf[k].offloads =
1481 port->dev_conf.rxmode.offloads;
1482
1483 for (k = 0; k < port->dev_info.max_tx_queues; k++)
1484 port->tx_conf[k].offloads =
1485 port->dev_conf.txmode.offloads;
1486
1487
1488 port->need_reconfig = 1;
1489 port->need_reconfig_queues = 1;
1490 port->tx_metadata = 0;
1491
1492
1493
1494
1495 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
1496 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
1497 data_size = rx_mode.max_rx_pkt_len /
1498 port->dev_info.rx_desc_lim.nb_mtu_seg_max;
1499
1500 if ((data_size + RTE_PKTMBUF_HEADROOM) >
1501 mbuf_data_size[0]) {
1502 mbuf_data_size[0] = data_size +
1503 RTE_PKTMBUF_HEADROOM;
1504 warning = 1;
1505 }
1506 }
1507 }
1508
1509 if (warning)
1510 TESTPMD_LOG(WARNING,
1511 "Configured mbuf size of the first segment %hu\n",
1512 mbuf_data_size[0]);
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522 if (param_total_num_mbufs)
1523 nb_mbuf_per_pool = param_total_num_mbufs;
1524 else {
1525 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX +
1526 (nb_lcores * mb_mempool_cache) +
1527 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST;
1528 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS;
1529 }
1530
1531 if (numa_support) {
1532 uint8_t i, j;
1533
1534 for (i = 0; i < num_sockets; i++)
1535 for (j = 0; j < mbuf_data_size_n; j++)
1536 mempools[i * MAX_SEGS_BUFFER_SPLIT + j] =
1537 mbuf_pool_create(mbuf_data_size[j],
1538 nb_mbuf_per_pool,
1539 socket_ids[i], j);
1540 } else {
1541 uint8_t i;
1542
1543 for (i = 0; i < mbuf_data_size_n; i++)
1544 mempools[i] = mbuf_pool_create
1545 (mbuf_data_size[i],
1546 nb_mbuf_per_pool,
1547 socket_num == UMA_NO_CONFIG ?
1548 0 : socket_num, i);
1549 }
1550
1551 init_port_config();
1552
1553 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1554 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO;
1555
1556
1557
1558 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1559 mbp = mbuf_pool_find(
1560 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id]), 0);
1561
1562 if (mbp == NULL)
1563 mbp = mbuf_pool_find(0, 0);
1564 fwd_lcores[lc_id]->mbp = mbp;
1565
1566 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp;
1567 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp;
1568 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types;
1569 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN -
1570 RTE_ETHER_CRC_LEN;
1571 fwd_lcores[lc_id]->gso_ctx.flag = 0;
1572 }
1573
1574
1575 if (init_fwd_streams() < 0)
1576 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
1577
1578 fwd_config_setup();
1579
1580
1581 gro_param.gro_types = RTE_GRO_TCP_IPV4;
1582 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES;
1583 gro_param.max_item_per_flow = MAX_PKT_BURST;
1584 for (lc_id = 0; lc_id < nb_lcores; lc_id++) {
1585 gro_param.socket_id = rte_lcore_to_socket_id(
1586 fwd_lcores_cpuids[lc_id]);
1587 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param);
1588 if (fwd_lcores[lc_id]->gro_ctx == NULL) {
1589 rte_exit(EXIT_FAILURE,
1590 "rte_gro_ctx_create() failed\n");
1591 }
1592 }
1593}
1594
1595
1596void
1597reconfig(portid_t new_port_id, unsigned socket_id)
1598{
1599 struct rte_port *port;
1600 int ret;
1601
1602
1603 port = &ports[new_port_id];
1604
1605 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info);
1606 if (ret != 0)
1607 return;
1608
1609
1610 port->need_reconfig = 1;
1611 port->need_reconfig_queues = 1;
1612 port->socket_id = socket_id;
1613
1614 init_port_config();
1615}
1616
1617
1618int
1619init_fwd_streams(void)
1620{
1621 portid_t pid;
1622 struct rte_port *port;
1623 streamid_t sm_id, nb_fwd_streams_new;
1624 queueid_t q;
1625
1626
1627 RTE_ETH_FOREACH_DEV(pid) {
1628 port = &ports[pid];
1629 if (nb_rxq > port->dev_info.max_rx_queues) {
1630 printf("Fail: nb_rxq(%d) is greater than "
1631 "max_rx_queues(%d)\n", nb_rxq,
1632 port->dev_info.max_rx_queues);
1633 return -1;
1634 }
1635 if (nb_txq > port->dev_info.max_tx_queues) {
1636 printf("Fail: nb_txq(%d) is greater than "
1637 "max_tx_queues(%d)\n", nb_txq,
1638 port->dev_info.max_tx_queues);
1639 return -1;
1640 }
1641 if (numa_support) {
1642 if (port_numa[pid] != NUMA_NO_CONFIG)
1643 port->socket_id = port_numa[pid];
1644 else {
1645 port->socket_id = rte_eth_dev_socket_id(pid);
1646
1647
1648
1649
1650
1651 if (check_socket_id(port->socket_id) < 0)
1652 port->socket_id = socket_ids[0];
1653 }
1654 }
1655 else {
1656 if (socket_num == UMA_NO_CONFIG)
1657 port->socket_id = 0;
1658 else
1659 port->socket_id = socket_num;
1660 }
1661 }
1662
1663 q = RTE_MAX(nb_rxq, nb_txq);
1664 if (q == 0) {
1665 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n");
1666 return -1;
1667 }
1668 nb_fwd_streams_new = (streamid_t)(nb_ports * q);
1669 if (nb_fwd_streams_new == nb_fwd_streams)
1670 return 0;
1671
1672 if (fwd_streams != NULL) {
1673 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1674 if (fwd_streams[sm_id] == NULL)
1675 continue;
1676 rte_free(fwd_streams[sm_id]);
1677 fwd_streams[sm_id] = NULL;
1678 }
1679 rte_free(fwd_streams);
1680 fwd_streams = NULL;
1681 }
1682
1683
1684 nb_fwd_streams = nb_fwd_streams_new;
1685 if (nb_fwd_streams) {
1686 fwd_streams = rte_zmalloc("testpmd: fwd_streams",
1687 sizeof(struct fwd_stream *) * nb_fwd_streams,
1688 RTE_CACHE_LINE_SIZE);
1689 if (fwd_streams == NULL)
1690 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d"
1691 " (struct fwd_stream *)) failed\n",
1692 nb_fwd_streams);
1693
1694 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) {
1695 fwd_streams[sm_id] = rte_zmalloc("testpmd:"
1696 " struct fwd_stream", sizeof(struct fwd_stream),
1697 RTE_CACHE_LINE_SIZE);
1698 if (fwd_streams[sm_id] == NULL)
1699 rte_exit(EXIT_FAILURE, "rte_zmalloc"
1700 "(struct fwd_stream) failed\n");
1701 }
1702 }
1703
1704 return 0;
1705}
1706
1707static void
1708pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs)
1709{
1710 uint64_t total_burst, sburst;
1711 uint64_t nb_burst;
1712 uint64_t burst_stats[4];
1713 uint16_t pktnb_stats[4];
1714 uint16_t nb_pkt;
1715 int burst_percent[4], sburstp;
1716 int i;
1717
1718
1719
1720
1721
1722 memset(&burst_stats, 0x0, sizeof(burst_stats));
1723 memset(&pktnb_stats, 0x0, sizeof(pktnb_stats));
1724
1725
1726 total_burst = pbs->pkt_burst_spread[0];
1727 burst_stats[0] = pbs->pkt_burst_spread[0];
1728 pktnb_stats[0] = 0;
1729
1730
1731 for (nb_pkt = 1; nb_pkt < MAX_PKT_BURST; nb_pkt++) {
1732 nb_burst = pbs->pkt_burst_spread[nb_pkt];
1733
1734 if (nb_burst == 0)
1735 continue;
1736
1737 total_burst += nb_burst;
1738
1739 if (nb_burst > burst_stats[1]) {
1740 burst_stats[2] = burst_stats[1];
1741 pktnb_stats[2] = pktnb_stats[1];
1742 burst_stats[1] = nb_burst;
1743 pktnb_stats[1] = nb_pkt;
1744 } else if (nb_burst > burst_stats[2]) {
1745 burst_stats[2] = nb_burst;
1746 pktnb_stats[2] = nb_pkt;
1747 }
1748 }
1749 if (total_burst == 0)
1750 return;
1751
1752 printf(" %s-bursts : %"PRIu64" [", rx_tx, total_burst);
1753 for (i = 0, sburst = 0, sburstp = 0; i < 4; i++) {
1754 if (i == 3) {
1755 printf("%d%% of other]\n", 100 - sburstp);
1756 return;
1757 }
1758
1759 sburst += burst_stats[i];
1760 if (sburst == total_burst) {
1761 printf("%d%% of %d pkts]\n",
1762 100 - sburstp, (int) pktnb_stats[i]);
1763 return;
1764 }
1765
1766 burst_percent[i] =
1767 (double)burst_stats[i] / total_burst * 100;
1768 printf("%d%% of %d pkts + ",
1769 burst_percent[i], (int) pktnb_stats[i]);
1770 sburstp += burst_percent[i];
1771 }
1772}
1773
1774static void
1775fwd_stream_stats_display(streamid_t stream_id)
1776{
1777 struct fwd_stream *fs;
1778 static const char *fwd_top_stats_border = "-------";
1779
1780 fs = fwd_streams[stream_id];
1781 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) &&
1782 (fs->fwd_dropped == 0))
1783 return;
1784 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> "
1785 "TX Port=%2d/Queue=%2d %s\n",
1786 fwd_top_stats_border, fs->rx_port, fs->rx_queue,
1787 fs->tx_port, fs->tx_queue, fwd_top_stats_border);
1788 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64
1789 " TX-dropped: %-14"PRIu64,
1790 fs->rx_packets, fs->tx_packets, fs->fwd_dropped);
1791
1792
1793 if (cur_fwd_eng == &csum_fwd_engine) {
1794 printf(" RX- bad IP checksum: %-14"PRIu64
1795 " Rx- bad L4 checksum: %-14"PRIu64
1796 " Rx- bad outer L4 checksum: %-14"PRIu64"\n",
1797 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum,
1798 fs->rx_bad_outer_l4_csum);
1799 } else {
1800 printf("\n");
1801 }
1802
1803 if (record_burst_stats) {
1804 pkt_burst_stats_display("RX", &fs->rx_burst_stats);
1805 pkt_burst_stats_display("TX", &fs->tx_burst_stats);
1806 }
1807}
1808
1809void
1810fwd_stats_display(void)
1811{
1812 static const char *fwd_stats_border = "----------------------";
1813 static const char *acc_stats_border = "+++++++++++++++";
1814 struct {
1815 struct fwd_stream *rx_stream;
1816 struct fwd_stream *tx_stream;
1817 uint64_t tx_dropped;
1818 uint64_t rx_bad_ip_csum;
1819 uint64_t rx_bad_l4_csum;
1820 uint64_t rx_bad_outer_l4_csum;
1821 } ports_stats[RTE_MAX_ETHPORTS];
1822 uint64_t total_rx_dropped = 0;
1823 uint64_t total_tx_dropped = 0;
1824 uint64_t total_rx_nombuf = 0;
1825 struct rte_eth_stats stats;
1826 uint64_t fwd_cycles = 0;
1827 uint64_t total_recv = 0;
1828 uint64_t total_xmit = 0;
1829 struct rte_port *port;
1830 streamid_t sm_id;
1831 portid_t pt_id;
1832 int i;
1833
1834 memset(ports_stats, 0, sizeof(ports_stats));
1835
1836 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1837 struct fwd_stream *fs = fwd_streams[sm_id];
1838
1839 if (cur_fwd_config.nb_fwd_streams >
1840 cur_fwd_config.nb_fwd_ports) {
1841 fwd_stream_stats_display(sm_id);
1842 } else {
1843 ports_stats[fs->tx_port].tx_stream = fs;
1844 ports_stats[fs->rx_port].rx_stream = fs;
1845 }
1846
1847 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped;
1848
1849 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum;
1850 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum;
1851 ports_stats[fs->rx_port].rx_bad_outer_l4_csum +=
1852 fs->rx_bad_outer_l4_csum;
1853
1854 if (record_core_cycles)
1855 fwd_cycles += fs->core_cycles;
1856 }
1857 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1858 pt_id = fwd_ports_ids[i];
1859 port = &ports[pt_id];
1860
1861 rte_eth_stats_get(pt_id, &stats);
1862 stats.ipackets -= port->stats.ipackets;
1863 stats.opackets -= port->stats.opackets;
1864 stats.ibytes -= port->stats.ibytes;
1865 stats.obytes -= port->stats.obytes;
1866 stats.imissed -= port->stats.imissed;
1867 stats.oerrors -= port->stats.oerrors;
1868 stats.rx_nombuf -= port->stats.rx_nombuf;
1869
1870 total_recv += stats.ipackets;
1871 total_xmit += stats.opackets;
1872 total_rx_dropped += stats.imissed;
1873 total_tx_dropped += ports_stats[pt_id].tx_dropped;
1874 total_tx_dropped += stats.oerrors;
1875 total_rx_nombuf += stats.rx_nombuf;
1876
1877 printf("\n %s Forward statistics for port %-2d %s\n",
1878 fwd_stats_border, pt_id, fwd_stats_border);
1879
1880 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64
1881 "RX-total: %-"PRIu64"\n", stats.ipackets, stats.imissed,
1882 stats.ipackets + stats.imissed);
1883
1884 if (cur_fwd_eng == &csum_fwd_engine)
1885 printf(" Bad-ipcsum: %-14"PRIu64
1886 " Bad-l4csum: %-14"PRIu64
1887 "Bad-outer-l4csum: %-14"PRIu64"\n",
1888 ports_stats[pt_id].rx_bad_ip_csum,
1889 ports_stats[pt_id].rx_bad_l4_csum,
1890 ports_stats[pt_id].rx_bad_outer_l4_csum);
1891 if (stats.ierrors + stats.rx_nombuf > 0) {
1892 printf(" RX-error: %-"PRIu64"\n", stats.ierrors);
1893 printf(" RX-nombufs: %-14"PRIu64"\n", stats.rx_nombuf);
1894 }
1895
1896 printf(" TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64
1897 "TX-total: %-"PRIu64"\n",
1898 stats.opackets, ports_stats[pt_id].tx_dropped,
1899 stats.opackets + ports_stats[pt_id].tx_dropped);
1900
1901 if (record_burst_stats) {
1902 if (ports_stats[pt_id].rx_stream)
1903 pkt_burst_stats_display("RX",
1904 &ports_stats[pt_id].rx_stream->rx_burst_stats);
1905 if (ports_stats[pt_id].tx_stream)
1906 pkt_burst_stats_display("TX",
1907 &ports_stats[pt_id].tx_stream->tx_burst_stats);
1908 }
1909
1910 printf(" %s--------------------------------%s\n",
1911 fwd_stats_border, fwd_stats_border);
1912 }
1913
1914 printf("\n %s Accumulated forward statistics for all ports"
1915 "%s\n",
1916 acc_stats_border, acc_stats_border);
1917 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: "
1918 "%-"PRIu64"\n"
1919 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: "
1920 "%-"PRIu64"\n",
1921 total_recv, total_rx_dropped, total_recv + total_rx_dropped,
1922 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped);
1923 if (total_rx_nombuf > 0)
1924 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf);
1925 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++"
1926 "%s\n",
1927 acc_stats_border, acc_stats_border);
1928 if (record_core_cycles) {
1929#define CYC_PER_MHZ 1E6
1930 if (total_recv > 0 || total_xmit > 0) {
1931 uint64_t total_pkts = 0;
1932 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 ||
1933 strcmp(cur_fwd_eng->fwd_mode_name, "flowgen") == 0)
1934 total_pkts = total_xmit;
1935 else
1936 total_pkts = total_recv;
1937
1938 printf("\n CPU cycles/packet=%.2F (total cycles="
1939 "%"PRIu64" / total %s packets=%"PRIu64") at %"PRIu64
1940 " MHz Clock\n",
1941 (double) fwd_cycles / total_pkts,
1942 fwd_cycles, cur_fwd_eng->fwd_mode_name, total_pkts,
1943 (uint64_t)(rte_get_tsc_hz() / CYC_PER_MHZ));
1944 }
1945 }
1946}
1947
1948void
1949fwd_stats_reset(void)
1950{
1951 streamid_t sm_id;
1952 portid_t pt_id;
1953 int i;
1954
1955 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
1956 pt_id = fwd_ports_ids[i];
1957 rte_eth_stats_get(pt_id, &ports[pt_id].stats);
1958 }
1959 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) {
1960 struct fwd_stream *fs = fwd_streams[sm_id];
1961
1962 fs->rx_packets = 0;
1963 fs->tx_packets = 0;
1964 fs->fwd_dropped = 0;
1965 fs->rx_bad_ip_csum = 0;
1966 fs->rx_bad_l4_csum = 0;
1967 fs->rx_bad_outer_l4_csum = 0;
1968
1969 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats));
1970 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats));
1971 fs->core_cycles = 0;
1972 }
1973}
1974
1975static void
1976flush_fwd_rx_queues(void)
1977{
1978 struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1979 portid_t rxp;
1980 portid_t port_id;
1981 queueid_t rxq;
1982 uint16_t nb_rx;
1983 uint16_t i;
1984 uint8_t j;
1985 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
1986 uint64_t timer_period;
1987
1988
1989 timer_period = rte_get_timer_hz();
1990
1991 for (j = 0; j < 2; j++) {
1992 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) {
1993 for (rxq = 0; rxq < nb_rxq; rxq++) {
1994 port_id = fwd_ports_ids[rxp];
1995
1996
1997
1998
1999
2000
2001 prev_tsc = rte_rdtsc();
2002 do {
2003 nb_rx = rte_eth_rx_burst(port_id, rxq,
2004 pkts_burst, MAX_PKT_BURST);
2005 for (i = 0; i < nb_rx; i++)
2006 rte_pktmbuf_free(pkts_burst[i]);
2007
2008 cur_tsc = rte_rdtsc();
2009 diff_tsc = cur_tsc - prev_tsc;
2010 timer_tsc += diff_tsc;
2011 } while ((nb_rx > 0) &&
2012 (timer_tsc < timer_period));
2013 timer_tsc = 0;
2014 }
2015 }
2016 rte_delay_ms(10);
2017 }
2018}
2019
2020static void
2021run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
2022{
2023 struct fwd_stream **fsm;
2024 streamid_t nb_fs;
2025 streamid_t sm_id;
2026#ifdef RTE_LIB_BITRATESTATS
2027 uint64_t tics_per_1sec;
2028 uint64_t tics_datum;
2029 uint64_t tics_current;
2030 uint16_t i, cnt_ports;
2031
2032 cnt_ports = nb_ports;
2033 tics_datum = rte_rdtsc();
2034 tics_per_1sec = rte_get_timer_hz();
2035#endif
2036 fsm = &fwd_streams[fc->stream_idx];
2037 nb_fs = fc->stream_nb;
2038 do {
2039 for (sm_id = 0; sm_id < nb_fs; sm_id++)
2040 (*pkt_fwd)(fsm[sm_id]);
2041#ifdef RTE_LIB_BITRATESTATS
2042 if (bitrate_enabled != 0 &&
2043 bitrate_lcore_id == rte_lcore_id()) {
2044 tics_current = rte_rdtsc();
2045 if (tics_current - tics_datum >= tics_per_1sec) {
2046
2047 for (i = 0; i < cnt_ports; i++)
2048 rte_stats_bitrate_calc(bitrate_data,
2049 ports_ids[i]);
2050 tics_datum = tics_current;
2051 }
2052 }
2053#endif
2054#ifdef RTE_LIB_LATENCYSTATS
2055 if (latencystats_enabled != 0 &&
2056 latencystats_lcore_id == rte_lcore_id())
2057 rte_latencystats_update();
2058#endif
2059
2060 } while (! fc->stopped);
2061}
2062
2063static int
2064start_pkt_forward_on_core(void *fwd_arg)
2065{
2066 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg,
2067 cur_fwd_config.fwd_eng->packet_fwd);
2068 return 0;
2069}
2070
2071
2072
2073
2074
2075static int
2076run_one_txonly_burst_on_core(void *fwd_arg)
2077{
2078 struct fwd_lcore *fwd_lc;
2079 struct fwd_lcore tmp_lcore;
2080
2081 fwd_lc = (struct fwd_lcore *) fwd_arg;
2082 tmp_lcore = *fwd_lc;
2083 tmp_lcore.stopped = 1;
2084 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd);
2085 return 0;
2086}
2087
2088
2089
2090
2091
2092
2093static void
2094launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore)
2095{
2096 port_fwd_begin_t port_fwd_begin;
2097 unsigned int i;
2098 unsigned int lc_id;
2099 int diag;
2100
2101 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin;
2102 if (port_fwd_begin != NULL) {
2103 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2104 (*port_fwd_begin)(fwd_ports_ids[i]);
2105 }
2106 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) {
2107 lc_id = fwd_lcores_cpuids[i];
2108 if ((interactive == 0) || (lc_id != rte_lcore_id())) {
2109 fwd_lcores[i]->stopped = 0;
2110 diag = rte_eal_remote_launch(pkt_fwd_on_lcore,
2111 fwd_lcores[i], lc_id);
2112 if (diag != 0)
2113 printf("launch lcore %u failed - diag=%d\n",
2114 lc_id, diag);
2115 }
2116 }
2117}
2118
2119
2120
2121
2122void
2123start_packet_forwarding(int with_tx_first)
2124{
2125 port_fwd_begin_t port_fwd_begin;
2126 port_fwd_end_t port_fwd_end;
2127 struct rte_port *port;
2128 unsigned int i;
2129 portid_t pt_id;
2130
2131 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq)
2132 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n");
2133
2134 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq)
2135 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n");
2136
2137 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 &&
2138 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) &&
2139 (!nb_rxq || !nb_txq))
2140 rte_exit(EXIT_FAILURE,
2141 "Either rxq or txq are 0, cannot use %s fwd mode\n",
2142 cur_fwd_eng->fwd_mode_name);
2143
2144 if (all_ports_started() == 0) {
2145 printf("Not all ports were started\n");
2146 return;
2147 }
2148 if (test_done == 0) {
2149 printf("Packet forwarding already started\n");
2150 return;
2151 }
2152
2153
2154 if(dcb_test) {
2155 for (i = 0; i < nb_fwd_ports; i++) {
2156 pt_id = fwd_ports_ids[i];
2157 port = &ports[pt_id];
2158 if (!port->dcb_flag) {
2159 printf("In DCB mode, all forwarding ports must "
2160 "be configured in this mode.\n");
2161 return;
2162 }
2163 }
2164 if (nb_fwd_lcores == 1) {
2165 printf("In DCB mode,the nb forwarding cores "
2166 "should be larger than 1.\n");
2167 return;
2168 }
2169 }
2170 test_done = 0;
2171
2172 fwd_config_setup();
2173
2174 if(!no_flush_rx)
2175 flush_fwd_rx_queues();
2176
2177 pkt_fwd_config_display(&cur_fwd_config);
2178 rxtx_config_display();
2179
2180 fwd_stats_reset();
2181 if (with_tx_first) {
2182 port_fwd_begin = tx_only_engine.port_fwd_begin;
2183 if (port_fwd_begin != NULL) {
2184 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2185 (*port_fwd_begin)(fwd_ports_ids[i]);
2186 }
2187 while (with_tx_first--) {
2188 launch_packet_forwarding(
2189 run_one_txonly_burst_on_core);
2190 rte_eal_mp_wait_lcore();
2191 }
2192 port_fwd_end = tx_only_engine.port_fwd_end;
2193 if (port_fwd_end != NULL) {
2194 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
2195 (*port_fwd_end)(fwd_ports_ids[i]);
2196 }
2197 }
2198 launch_packet_forwarding(start_pkt_forward_on_core);
2199}
2200
2201void
2202stop_packet_forwarding(void)
2203{
2204 port_fwd_end_t port_fwd_end;
2205 lcoreid_t lc_id;
2206 portid_t pt_id;
2207 int i;
2208
2209 if (test_done) {
2210 printf("Packet forwarding not started\n");
2211 return;
2212 }
2213 printf("Telling cores to stop...");
2214 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++)
2215 fwd_lcores[lc_id]->stopped = 1;
2216 printf("\nWaiting for lcores to finish...\n");
2217 rte_eal_mp_wait_lcore();
2218 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end;
2219 if (port_fwd_end != NULL) {
2220 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
2221 pt_id = fwd_ports_ids[i];
2222 (*port_fwd_end)(pt_id);
2223 }
2224 }
2225
2226 fwd_stats_display();
2227
2228 printf("\nDone.\n");
2229 test_done = 1;
2230}
2231
2232void
2233dev_set_link_up(portid_t pid)
2234{
2235 if (rte_eth_dev_set_link_up(pid) < 0)
2236 printf("\nSet link up fail.\n");
2237}
2238
2239void
2240dev_set_link_down(portid_t pid)
2241{
2242 if (rte_eth_dev_set_link_down(pid) < 0)
2243 printf("\nSet link down fail.\n");
2244}
2245
2246static int
2247all_ports_started(void)
2248{
2249 portid_t pi;
2250 struct rte_port *port;
2251
2252 RTE_ETH_FOREACH_DEV(pi) {
2253 port = &ports[pi];
2254
2255 if ((port->port_status != RTE_PORT_STARTED) &&
2256 (port->slave_flag == 0))
2257 return 0;
2258 }
2259
2260
2261 return 1;
2262}
2263
2264int
2265port_is_stopped(portid_t port_id)
2266{
2267 struct rte_port *port = &ports[port_id];
2268
2269 if ((port->port_status != RTE_PORT_STOPPED) &&
2270 (port->slave_flag == 0))
2271 return 0;
2272 return 1;
2273}
2274
2275int
2276all_ports_stopped(void)
2277{
2278 portid_t pi;
2279
2280 RTE_ETH_FOREACH_DEV(pi) {
2281 if (!port_is_stopped(pi))
2282 return 0;
2283 }
2284
2285 return 1;
2286}
2287
2288int
2289port_is_started(portid_t port_id)
2290{
2291 if (port_id_is_invalid(port_id, ENABLED_WARN))
2292 return 0;
2293
2294 if (ports[port_id].port_status != RTE_PORT_STARTED)
2295 return 0;
2296
2297 return 1;
2298}
2299
2300
2301static int
2302setup_hairpin_queues(portid_t pi, portid_t p_pi, uint16_t cnt_pi)
2303{
2304 queueid_t qi;
2305 struct rte_eth_hairpin_conf hairpin_conf = {
2306 .peer_count = 1,
2307 };
2308 int i;
2309 int diag;
2310 struct rte_port *port = &ports[pi];
2311 uint16_t peer_rx_port = pi;
2312 uint16_t peer_tx_port = pi;
2313 uint32_t manual = 1;
2314 uint32_t tx_exp = hairpin_mode & 0x10;
2315
2316 if (!(hairpin_mode & 0xf)) {
2317 peer_rx_port = pi;
2318 peer_tx_port = pi;
2319 manual = 0;
2320 } else if (hairpin_mode & 0x1) {
2321 peer_tx_port = rte_eth_find_next_owned_by(pi + 1,
2322 RTE_ETH_DEV_NO_OWNER);
2323 if (peer_tx_port >= RTE_MAX_ETHPORTS)
2324 peer_tx_port = rte_eth_find_next_owned_by(0,
2325 RTE_ETH_DEV_NO_OWNER);
2326 if (p_pi != RTE_MAX_ETHPORTS) {
2327 peer_rx_port = p_pi;
2328 } else {
2329 uint16_t next_pi;
2330
2331
2332 RTE_ETH_FOREACH_DEV(next_pi)
2333 peer_rx_port = next_pi;
2334 }
2335 manual = 1;
2336 } else if (hairpin_mode & 0x2) {
2337 if (cnt_pi & 0x1) {
2338 peer_rx_port = p_pi;
2339 } else {
2340 peer_rx_port = rte_eth_find_next_owned_by(pi + 1,
2341 RTE_ETH_DEV_NO_OWNER);
2342 if (peer_rx_port >= RTE_MAX_ETHPORTS)
2343 peer_rx_port = pi;
2344 }
2345 peer_tx_port = peer_rx_port;
2346 manual = 1;
2347 }
2348
2349 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) {
2350 hairpin_conf.peers[0].port = peer_rx_port;
2351 hairpin_conf.peers[0].queue = i + nb_rxq;
2352 hairpin_conf.manual_bind = !!manual;
2353 hairpin_conf.tx_explicit = !!tx_exp;
2354 diag = rte_eth_tx_hairpin_queue_setup
2355 (pi, qi, nb_txd, &hairpin_conf);
2356 i++;
2357 if (diag == 0)
2358 continue;
2359
2360
2361 if (rte_atomic16_cmpset(&(port->port_status),
2362 RTE_PORT_HANDLING,
2363 RTE_PORT_STOPPED) == 0)
2364 printf("Port %d can not be set back "
2365 "to stopped\n", pi);
2366 printf("Fail to configure port %d hairpin "
2367 "queues\n", pi);
2368
2369 port->need_reconfig_queues = 1;
2370 return -1;
2371 }
2372 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) {
2373 hairpin_conf.peers[0].port = peer_tx_port;
2374 hairpin_conf.peers[0].queue = i + nb_txq;
2375 hairpin_conf.manual_bind = !!manual;
2376 hairpin_conf.tx_explicit = !!tx_exp;
2377 diag = rte_eth_rx_hairpin_queue_setup
2378 (pi, qi, nb_rxd, &hairpin_conf);
2379 i++;
2380 if (diag == 0)
2381 continue;
2382
2383
2384 if (rte_atomic16_cmpset(&(port->port_status),
2385 RTE_PORT_HANDLING,
2386 RTE_PORT_STOPPED) == 0)
2387 printf("Port %d can not be set back "
2388 "to stopped\n", pi);
2389 printf("Fail to configure port %d hairpin "
2390 "queues\n", pi);
2391
2392 port->need_reconfig_queues = 1;
2393 return -1;
2394 }
2395 return 0;
2396}
2397
2398
2399int
2400rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2401 uint16_t nb_rx_desc, unsigned int socket_id,
2402 struct rte_eth_rxconf *rx_conf, struct rte_mempool *mp)
2403{
2404 union rte_eth_rxseg rx_useg[MAX_SEGS_BUFFER_SPLIT] = {};
2405 unsigned int i, mp_n;
2406 int ret;
2407
2408 if (rx_pkt_nb_segs <= 1 ||
2409 (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) == 0) {
2410 rx_conf->rx_seg = NULL;
2411 rx_conf->rx_nseg = 0;
2412 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id,
2413 nb_rx_desc, socket_id,
2414 rx_conf, mp);
2415 return ret;
2416 }
2417 for (i = 0; i < rx_pkt_nb_segs; i++) {
2418 struct rte_eth_rxseg_split *rx_seg = &rx_useg[i].split;
2419 struct rte_mempool *mpx;
2420
2421
2422
2423
2424 mp_n = (i > mbuf_data_size_n) ? mbuf_data_size_n - 1 : i;
2425 mpx = mbuf_pool_find(socket_id, mp_n);
2426
2427 rx_seg->length = rx_pkt_seg_lengths[i] ?
2428 rx_pkt_seg_lengths[i] :
2429 mbuf_data_size[mp_n];
2430 rx_seg->offset = i < rx_pkt_nb_offs ?
2431 rx_pkt_seg_offsets[i] : 0;
2432 rx_seg->mp = mpx ? mpx : mp;
2433 }
2434 rx_conf->rx_nseg = rx_pkt_nb_segs;
2435 rx_conf->rx_seg = rx_useg;
2436 ret = rte_eth_rx_queue_setup(port_id, rx_queue_id, nb_rx_desc,
2437 socket_id, rx_conf, NULL);
2438 rx_conf->rx_seg = NULL;
2439 rx_conf->rx_nseg = 0;
2440 return ret;
2441}
2442
2443int
2444start_port(portid_t pid)
2445{
2446 int diag, need_check_link_status = -1;
2447 portid_t pi;
2448 portid_t p_pi = RTE_MAX_ETHPORTS;
2449 portid_t pl[RTE_MAX_ETHPORTS];
2450 portid_t peer_pl[RTE_MAX_ETHPORTS];
2451 uint16_t cnt_pi = 0;
2452 uint16_t cfg_pi = 0;
2453 int peer_pi;
2454 queueid_t qi;
2455 struct rte_port *port;
2456 struct rte_ether_addr mac_addr;
2457 struct rte_eth_hairpin_cap cap;
2458
2459 if (port_id_is_invalid(pid, ENABLED_WARN))
2460 return 0;
2461
2462 if(dcb_config)
2463 dcb_test = 1;
2464 RTE_ETH_FOREACH_DEV(pi) {
2465 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2466 continue;
2467
2468 need_check_link_status = 0;
2469 port = &ports[pi];
2470 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED,
2471 RTE_PORT_HANDLING) == 0) {
2472 printf("Port %d is now not stopped\n", pi);
2473 continue;
2474 }
2475
2476 if (port->need_reconfig > 0) {
2477 port->need_reconfig = 0;
2478
2479 if (flow_isolate_all) {
2480 int ret = port_flow_isolate(pi, 1);
2481 if (ret) {
2482 printf("Failed to apply isolated"
2483 " mode on port %d\n", pi);
2484 return -1;
2485 }
2486 }
2487 configure_rxtx_dump_callbacks(0);
2488 printf("Configuring Port %d (socket %u)\n", pi,
2489 port->socket_id);
2490 if (nb_hairpinq > 0 &&
2491 rte_eth_dev_hairpin_capability_get(pi, &cap)) {
2492 printf("Port %d doesn't support hairpin "
2493 "queues\n", pi);
2494 return -1;
2495 }
2496
2497 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq,
2498 nb_txq + nb_hairpinq,
2499 &(port->dev_conf));
2500 if (diag != 0) {
2501 if (rte_atomic16_cmpset(&(port->port_status),
2502 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2503 printf("Port %d can not be set back "
2504 "to stopped\n", pi);
2505 printf("Fail to configure port %d\n", pi);
2506
2507 port->need_reconfig = 1;
2508 return -1;
2509 }
2510 }
2511 if (port->need_reconfig_queues > 0) {
2512 port->need_reconfig_queues = 0;
2513
2514 for (qi = 0; qi < nb_txq; qi++) {
2515 if ((numa_support) &&
2516 (txring_numa[pi] != NUMA_NO_CONFIG))
2517 diag = rte_eth_tx_queue_setup(pi, qi,
2518 port->nb_tx_desc[qi],
2519 txring_numa[pi],
2520 &(port->tx_conf[qi]));
2521 else
2522 diag = rte_eth_tx_queue_setup(pi, qi,
2523 port->nb_tx_desc[qi],
2524 port->socket_id,
2525 &(port->tx_conf[qi]));
2526
2527 if (diag == 0)
2528 continue;
2529
2530
2531 if (rte_atomic16_cmpset(&(port->port_status),
2532 RTE_PORT_HANDLING,
2533 RTE_PORT_STOPPED) == 0)
2534 printf("Port %d can not be set back "
2535 "to stopped\n", pi);
2536 printf("Fail to configure port %d tx queues\n",
2537 pi);
2538
2539 port->need_reconfig_queues = 1;
2540 return -1;
2541 }
2542 for (qi = 0; qi < nb_rxq; qi++) {
2543
2544 if ((numa_support) &&
2545 (rxring_numa[pi] != NUMA_NO_CONFIG)) {
2546 struct rte_mempool * mp =
2547 mbuf_pool_find
2548 (rxring_numa[pi], 0);
2549 if (mp == NULL) {
2550 printf("Failed to setup RX queue:"
2551 "No mempool allocation"
2552 " on the socket %d\n",
2553 rxring_numa[pi]);
2554 return -1;
2555 }
2556
2557 diag = rx_queue_setup(pi, qi,
2558 port->nb_rx_desc[qi],
2559 rxring_numa[pi],
2560 &(port->rx_conf[qi]),
2561 mp);
2562 } else {
2563 struct rte_mempool *mp =
2564 mbuf_pool_find
2565 (port->socket_id, 0);
2566 if (mp == NULL) {
2567 printf("Failed to setup RX queue:"
2568 "No mempool allocation"
2569 " on the socket %d\n",
2570 port->socket_id);
2571 return -1;
2572 }
2573 diag = rx_queue_setup(pi, qi,
2574 port->nb_rx_desc[qi],
2575 port->socket_id,
2576 &(port->rx_conf[qi]),
2577 mp);
2578 }
2579 if (diag == 0)
2580 continue;
2581
2582
2583 if (rte_atomic16_cmpset(&(port->port_status),
2584 RTE_PORT_HANDLING,
2585 RTE_PORT_STOPPED) == 0)
2586 printf("Port %d can not be set back "
2587 "to stopped\n", pi);
2588 printf("Fail to configure port %d rx queues\n",
2589 pi);
2590
2591 port->need_reconfig_queues = 1;
2592 return -1;
2593 }
2594
2595 if (setup_hairpin_queues(pi, p_pi, cnt_pi) != 0)
2596 return -1;
2597 }
2598 configure_rxtx_dump_callbacks(verbose_level);
2599 if (clear_ptypes) {
2600 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN,
2601 NULL, 0);
2602 if (diag < 0)
2603 printf(
2604 "Port %d: Failed to disable Ptype parsing\n",
2605 pi);
2606 }
2607
2608 p_pi = pi;
2609 cnt_pi++;
2610
2611
2612 if (rte_eth_dev_start(pi) < 0) {
2613 printf("Fail to start port %d\n", pi);
2614
2615
2616 if (rte_atomic16_cmpset(&(port->port_status),
2617 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2618 printf("Port %d can not be set back to "
2619 "stopped\n", pi);
2620 continue;
2621 }
2622
2623 if (rte_atomic16_cmpset(&(port->port_status),
2624 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0)
2625 printf("Port %d can not be set into started\n", pi);
2626
2627 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0)
2628 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi,
2629 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1],
2630 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3],
2631 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]);
2632
2633
2634 need_check_link_status = 1;
2635
2636 pl[cfg_pi++] = pi;
2637 }
2638
2639 if (need_check_link_status == 1 && !no_link_check)
2640 check_all_ports_link_status(RTE_PORT_ALL);
2641 else if (need_check_link_status == 0)
2642 printf("Please stop the ports first\n");
2643
2644 if (hairpin_mode & 0xf) {
2645 uint16_t i;
2646 int j;
2647
2648
2649 for (i = 0; i < cfg_pi; i++) {
2650 pi = pl[i];
2651
2652 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2653 RTE_MAX_ETHPORTS, 1);
2654 if (peer_pi < 0)
2655 return peer_pi;
2656 for (j = 0; j < peer_pi; j++) {
2657 if (!port_is_started(peer_pl[j]))
2658 continue;
2659 diag = rte_eth_hairpin_bind(pi, peer_pl[j]);
2660 if (diag < 0) {
2661 printf("Error during binding hairpin"
2662 " Tx port %u to %u: %s\n",
2663 pi, peer_pl[j],
2664 rte_strerror(-diag));
2665 return -1;
2666 }
2667 }
2668
2669 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2670 RTE_MAX_ETHPORTS, 0);
2671 if (peer_pi < 0)
2672 return peer_pi;
2673 for (j = 0; j < peer_pi; j++) {
2674 if (!port_is_started(peer_pl[j]))
2675 continue;
2676 diag = rte_eth_hairpin_bind(peer_pl[j], pi);
2677 if (diag < 0) {
2678 printf("Error during binding hairpin"
2679 " Tx port %u to %u: %s\n",
2680 peer_pl[j], pi,
2681 rte_strerror(-diag));
2682 return -1;
2683 }
2684 }
2685 }
2686 }
2687
2688 printf("Done\n");
2689 return 0;
2690}
2691
2692void
2693stop_port(portid_t pid)
2694{
2695 portid_t pi;
2696 struct rte_port *port;
2697 int need_check_link_status = 0;
2698 portid_t peer_pl[RTE_MAX_ETHPORTS];
2699 int peer_pi;
2700
2701 if (dcb_test) {
2702 dcb_test = 0;
2703 dcb_config = 0;
2704 }
2705
2706 if (port_id_is_invalid(pid, ENABLED_WARN))
2707 return;
2708
2709 printf("Stopping ports...\n");
2710
2711 RTE_ETH_FOREACH_DEV(pi) {
2712 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2713 continue;
2714
2715 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2716 printf("Please remove port %d from forwarding configuration.\n", pi);
2717 continue;
2718 }
2719
2720 if (port_is_bonding_slave(pi)) {
2721 printf("Please remove port %d from bonded device.\n", pi);
2722 continue;
2723 }
2724
2725 port = &ports[pi];
2726 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
2727 RTE_PORT_HANDLING) == 0)
2728 continue;
2729
2730 if (hairpin_mode & 0xf) {
2731 int j;
2732
2733 rte_eth_hairpin_unbind(pi, RTE_MAX_ETHPORTS);
2734
2735 peer_pi = rte_eth_hairpin_get_peer_ports(pi, peer_pl,
2736 RTE_MAX_ETHPORTS, 0);
2737 if (peer_pi < 0)
2738 continue;
2739 for (j = 0; j < peer_pi; j++) {
2740 if (!port_is_started(peer_pl[j]))
2741 continue;
2742 rte_eth_hairpin_unbind(peer_pl[j], pi);
2743 }
2744 }
2745
2746 if (port->flow_list)
2747 port_flow_flush(pi);
2748
2749 if (rte_eth_dev_stop(pi) != 0)
2750 RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n",
2751 pi);
2752
2753 if (rte_atomic16_cmpset(&(port->port_status),
2754 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0)
2755 printf("Port %d can not be set into stopped\n", pi);
2756 need_check_link_status = 1;
2757 }
2758 if (need_check_link_status && !no_link_check)
2759 check_all_ports_link_status(RTE_PORT_ALL);
2760
2761 printf("Done\n");
2762}
2763
2764static void
2765remove_invalid_ports_in(portid_t *array, portid_t *total)
2766{
2767 portid_t i;
2768 portid_t new_total = 0;
2769
2770 for (i = 0; i < *total; i++)
2771 if (!port_id_is_invalid(array[i], DISABLED_WARN)) {
2772 array[new_total] = array[i];
2773 new_total++;
2774 }
2775 *total = new_total;
2776}
2777
2778static void
2779remove_invalid_ports(void)
2780{
2781 remove_invalid_ports_in(ports_ids, &nb_ports);
2782 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports);
2783 nb_cfg_ports = nb_fwd_ports;
2784}
2785
2786void
2787close_port(portid_t pid)
2788{
2789 portid_t pi;
2790 struct rte_port *port;
2791
2792 if (port_id_is_invalid(pid, ENABLED_WARN))
2793 return;
2794
2795 printf("Closing ports...\n");
2796
2797 RTE_ETH_FOREACH_DEV(pi) {
2798 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2799 continue;
2800
2801 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2802 printf("Please remove port %d from forwarding configuration.\n", pi);
2803 continue;
2804 }
2805
2806 if (port_is_bonding_slave(pi)) {
2807 printf("Please remove port %d from bonded device.\n", pi);
2808 continue;
2809 }
2810
2811 port = &ports[pi];
2812 if (rte_atomic16_cmpset(&(port->port_status),
2813 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
2814 printf("Port %d is already closed\n", pi);
2815 continue;
2816 }
2817
2818 port_flow_flush(pi);
2819 rte_eth_dev_close(pi);
2820 }
2821
2822 remove_invalid_ports();
2823 printf("Done\n");
2824}
2825
2826void
2827reset_port(portid_t pid)
2828{
2829 int diag;
2830 portid_t pi;
2831 struct rte_port *port;
2832
2833 if (port_id_is_invalid(pid, ENABLED_WARN))
2834 return;
2835
2836 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) ||
2837 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) {
2838 printf("Can not reset port(s), please stop port(s) first.\n");
2839 return;
2840 }
2841
2842 printf("Resetting ports...\n");
2843
2844 RTE_ETH_FOREACH_DEV(pi) {
2845 if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
2846 continue;
2847
2848 if (port_is_forwarding(pi) != 0 && test_done == 0) {
2849 printf("Please remove port %d from forwarding "
2850 "configuration.\n", pi);
2851 continue;
2852 }
2853
2854 if (port_is_bonding_slave(pi)) {
2855 printf("Please remove port %d from bonded device.\n",
2856 pi);
2857 continue;
2858 }
2859
2860 diag = rte_eth_dev_reset(pi);
2861 if (diag == 0) {
2862 port = &ports[pi];
2863 port->need_reconfig = 1;
2864 port->need_reconfig_queues = 1;
2865 } else {
2866 printf("Failed to reset port %d. diag=%d\n", pi, diag);
2867 }
2868 }
2869
2870 printf("Done\n");
2871}
2872
2873void
2874attach_port(char *identifier)
2875{
2876 portid_t pi;
2877 struct rte_dev_iterator iterator;
2878
2879 printf("Attaching a new port...\n");
2880
2881 if (identifier == NULL) {
2882 printf("Invalid parameters are specified\n");
2883 return;
2884 }
2885
2886 if (rte_dev_probe(identifier) < 0) {
2887 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
2888 return;
2889 }
2890
2891
2892 if (setup_on_probe_event) {
2893
2894 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++)
2895 if (ports[pi].port_status == RTE_PORT_HANDLING &&
2896 ports[pi].need_setup != 0)
2897 setup_attached_port(pi);
2898 return;
2899 }
2900
2901
2902 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) {
2903
2904 if (port_is_forwarding(pi))
2905 continue;
2906 setup_attached_port(pi);
2907 }
2908}
2909
2910static void
2911setup_attached_port(portid_t pi)
2912{
2913 unsigned int socket_id;
2914 int ret;
2915
2916 socket_id = (unsigned)rte_eth_dev_socket_id(pi);
2917
2918 if (check_socket_id(socket_id) < 0)
2919 socket_id = socket_ids[0];
2920 reconfig(pi, socket_id);
2921 ret = rte_eth_promiscuous_enable(pi);
2922 if (ret != 0)
2923 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
2924 pi, rte_strerror(-ret));
2925
2926 ports_ids[nb_ports++] = pi;
2927 fwd_ports_ids[nb_fwd_ports++] = pi;
2928 nb_cfg_ports = nb_fwd_ports;
2929 ports[pi].need_setup = 0;
2930 ports[pi].port_status = RTE_PORT_STOPPED;
2931
2932 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
2933 printf("Done\n");
2934}
2935
2936static void
2937detach_device(struct rte_device *dev)
2938{
2939 portid_t sibling;
2940
2941 if (dev == NULL) {
2942 printf("Device already removed\n");
2943 return;
2944 }
2945
2946 printf("Removing a device...\n");
2947
2948 RTE_ETH_FOREACH_DEV_OF(sibling, dev) {
2949 if (ports[sibling].port_status != RTE_PORT_CLOSED) {
2950 if (ports[sibling].port_status != RTE_PORT_STOPPED) {
2951 printf("Port %u not stopped\n", sibling);
2952 return;
2953 }
2954 port_flow_flush(sibling);
2955 }
2956 }
2957
2958 if (rte_dev_remove(dev) < 0) {
2959 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name);
2960 return;
2961 }
2962 remove_invalid_ports();
2963
2964 printf("Device is detached\n");
2965 printf("Now total ports is %d\n", nb_ports);
2966 printf("Done\n");
2967 return;
2968}
2969
2970void
2971detach_port_device(portid_t port_id)
2972{
2973 if (port_id_is_invalid(port_id, ENABLED_WARN))
2974 return;
2975
2976 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
2977 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
2978 printf("Port not stopped\n");
2979 return;
2980 }
2981 printf("Port was not closed\n");
2982 }
2983
2984 detach_device(rte_eth_devices[port_id].device);
2985}
2986
2987void
2988detach_devargs(char *identifier)
2989{
2990 struct rte_dev_iterator iterator;
2991 struct rte_devargs da;
2992 portid_t port_id;
2993
2994 printf("Removing a device...\n");
2995
2996 memset(&da, 0, sizeof(da));
2997 if (rte_devargs_parsef(&da, "%s", identifier)) {
2998 printf("cannot parse identifier\n");
2999 if (da.args)
3000 free(da.args);
3001 return;
3002 }
3003
3004 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) {
3005 if (ports[port_id].port_status != RTE_PORT_CLOSED) {
3006 if (ports[port_id].port_status != RTE_PORT_STOPPED) {
3007 printf("Port %u not stopped\n", port_id);
3008 rte_eth_iterator_cleanup(&iterator);
3009 return;
3010 }
3011 port_flow_flush(port_id);
3012 }
3013 }
3014
3015 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) {
3016 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n",
3017 da.name, da.bus->name);
3018 return;
3019 }
3020
3021 remove_invalid_ports();
3022
3023 printf("Device %s is detached\n", identifier);
3024 printf("Now total ports is %d\n", nb_ports);
3025 printf("Done\n");
3026}
3027
3028void
3029pmd_test_exit(void)
3030{
3031 portid_t pt_id;
3032 unsigned int i;
3033 int ret;
3034
3035 if (test_done == 0)
3036 stop_packet_forwarding();
3037
3038 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3039 if (mempools[i]) {
3040 if (mp_alloc_type == MP_ALLOC_ANON)
3041 rte_mempool_mem_iter(mempools[i], dma_unmap_cb,
3042 NULL);
3043 }
3044 }
3045 if (ports != NULL) {
3046 no_link_check = 1;
3047 RTE_ETH_FOREACH_DEV(pt_id) {
3048 printf("\nStopping port %d...\n", pt_id);
3049 fflush(stdout);
3050 stop_port(pt_id);
3051 }
3052 RTE_ETH_FOREACH_DEV(pt_id) {
3053 printf("\nShutting down port %d...\n", pt_id);
3054 fflush(stdout);
3055 close_port(pt_id);
3056 }
3057 }
3058
3059 if (hot_plug) {
3060 ret = rte_dev_event_monitor_stop();
3061 if (ret) {
3062 RTE_LOG(ERR, EAL,
3063 "fail to stop device event monitor.");
3064 return;
3065 }
3066
3067 ret = rte_dev_event_callback_unregister(NULL,
3068 dev_event_callback, NULL);
3069 if (ret < 0) {
3070 RTE_LOG(ERR, EAL,
3071 "fail to unregister device event callback.\n");
3072 return;
3073 }
3074
3075 ret = rte_dev_hotplug_handle_disable();
3076 if (ret) {
3077 RTE_LOG(ERR, EAL,
3078 "fail to disable hotplug handling.\n");
3079 return;
3080 }
3081 }
3082 for (i = 0 ; i < RTE_DIM(mempools) ; i++) {
3083 if (mempools[i])
3084 rte_mempool_free(mempools[i]);
3085 }
3086
3087 printf("\nBye...\n");
3088}
3089
3090typedef void (*cmd_func_t)(void);
3091struct pmd_test_command {
3092 const char *cmd_name;
3093 cmd_func_t cmd_func;
3094};
3095
3096
3097static void
3098check_all_ports_link_status(uint32_t port_mask)
3099{
3100#define CHECK_INTERVAL 100
3101#define MAX_CHECK_TIME 90
3102 portid_t portid;
3103 uint8_t count, all_ports_up, print_flag = 0;
3104 struct rte_eth_link link;
3105 int ret;
3106 char link_status[RTE_ETH_LINK_MAX_STR_LEN];
3107
3108 printf("Checking link statuses...\n");
3109 fflush(stdout);
3110 for (count = 0; count <= MAX_CHECK_TIME; count++) {
3111 all_ports_up = 1;
3112 RTE_ETH_FOREACH_DEV(portid) {
3113 if ((port_mask & (1 << portid)) == 0)
3114 continue;
3115 memset(&link, 0, sizeof(link));
3116 ret = rte_eth_link_get_nowait(portid, &link);
3117 if (ret < 0) {
3118 all_ports_up = 0;
3119 if (print_flag == 1)
3120 printf("Port %u link get failed: %s\n",
3121 portid, rte_strerror(-ret));
3122 continue;
3123 }
3124
3125 if (print_flag == 1) {
3126 rte_eth_link_to_str(link_status,
3127 sizeof(link_status), &link);
3128 printf("Port %d %s\n", portid, link_status);
3129 continue;
3130 }
3131
3132 if (link.link_status == ETH_LINK_DOWN) {
3133 all_ports_up = 0;
3134 break;
3135 }
3136 }
3137
3138 if (print_flag == 1)
3139 break;
3140
3141 if (all_ports_up == 0) {
3142 fflush(stdout);
3143 rte_delay_ms(CHECK_INTERVAL);
3144 }
3145
3146
3147 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
3148 print_flag = 1;
3149 }
3150
3151 if (lsc_interrupt)
3152 break;
3153 }
3154}
3155
3156static void
3157rmv_port_callback(void *arg)
3158{
3159 int need_to_start = 0;
3160 int org_no_link_check = no_link_check;
3161 portid_t port_id = (intptr_t)arg;
3162 struct rte_device *dev;
3163
3164 RTE_ETH_VALID_PORTID_OR_RET(port_id);
3165
3166 if (!test_done && port_is_forwarding(port_id)) {
3167 need_to_start = 1;
3168 stop_packet_forwarding();
3169 }
3170 no_link_check = 1;
3171 stop_port(port_id);
3172 no_link_check = org_no_link_check;
3173
3174
3175 dev = rte_eth_devices[port_id].device;
3176 close_port(port_id);
3177 detach_device(dev);
3178
3179 if (need_to_start)
3180 start_packet_forwarding(0);
3181}
3182
3183
3184static int
3185eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param,
3186 void *ret_param)
3187{
3188 RTE_SET_USED(param);
3189 RTE_SET_USED(ret_param);
3190
3191 if (type >= RTE_ETH_EVENT_MAX) {
3192 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n",
3193 port_id, __func__, type);
3194 fflush(stderr);
3195 } else if (event_print_mask & (UINT32_C(1) << type)) {
3196 printf("\nPort %" PRIu16 ": %s event\n", port_id,
3197 eth_event_desc[type]);
3198 fflush(stdout);
3199 }
3200
3201 switch (type) {
3202 case RTE_ETH_EVENT_NEW:
3203 ports[port_id].need_setup = 1;
3204 ports[port_id].port_status = RTE_PORT_HANDLING;
3205 break;
3206 case RTE_ETH_EVENT_INTR_RMV:
3207 if (port_id_is_invalid(port_id, DISABLED_WARN))
3208 break;
3209 if (rte_eal_alarm_set(100000,
3210 rmv_port_callback, (void *)(intptr_t)port_id))
3211 fprintf(stderr, "Could not set up deferred device removal\n");
3212 break;
3213 case RTE_ETH_EVENT_DESTROY:
3214 ports[port_id].port_status = RTE_PORT_CLOSED;
3215 printf("Port %u is closed\n", port_id);
3216 break;
3217 default:
3218 break;
3219 }
3220 return 0;
3221}
3222
3223static int
3224register_eth_event_callback(void)
3225{
3226 int ret;
3227 enum rte_eth_event_type event;
3228
3229 for (event = RTE_ETH_EVENT_UNKNOWN;
3230 event < RTE_ETH_EVENT_MAX; event++) {
3231 ret = rte_eth_dev_callback_register(RTE_ETH_ALL,
3232 event,
3233 eth_event_callback,
3234 NULL);
3235 if (ret != 0) {
3236 TESTPMD_LOG(ERR, "Failed to register callback for "
3237 "%s event\n", eth_event_desc[event]);
3238 return -1;
3239 }
3240 }
3241
3242 return 0;
3243}
3244
3245
3246static void
3247dev_event_callback(const char *device_name, enum rte_dev_event_type type,
3248 __rte_unused void *arg)
3249{
3250 uint16_t port_id;
3251 int ret;
3252
3253 if (type >= RTE_DEV_EVENT_MAX) {
3254 fprintf(stderr, "%s called upon invalid event %d\n",
3255 __func__, type);
3256 fflush(stderr);
3257 }
3258
3259 switch (type) {
3260 case RTE_DEV_EVENT_REMOVE:
3261 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n",
3262 device_name);
3263 ret = rte_eth_dev_get_port_by_name(device_name, &port_id);
3264 if (ret) {
3265 RTE_LOG(ERR, EAL, "can not get port by device %s!\n",
3266 device_name);
3267 return;
3268 }
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278 if (rte_eal_alarm_set(100000,
3279 rmv_port_callback, (void *)(intptr_t)port_id))
3280 RTE_LOG(ERR, EAL,
3281 "Could not set up deferred device removal\n");
3282 break;
3283 case RTE_DEV_EVENT_ADD:
3284 RTE_LOG(ERR, EAL, "The device: %s has been added!\n",
3285 device_name);
3286
3287
3288
3289 break;
3290 default:
3291 break;
3292 }
3293}
3294
3295static void
3296rxtx_port_config(struct rte_port *port)
3297{
3298 uint16_t qid;
3299 uint64_t offloads;
3300
3301 for (qid = 0; qid < nb_rxq; qid++) {
3302 offloads = port->rx_conf[qid].offloads;
3303 port->rx_conf[qid] = port->dev_info.default_rxconf;
3304 if (offloads != 0)
3305 port->rx_conf[qid].offloads = offloads;
3306
3307
3308 if (rx_pthresh != RTE_PMD_PARAM_UNSET)
3309 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh;
3310
3311 if (rx_hthresh != RTE_PMD_PARAM_UNSET)
3312 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh;
3313
3314 if (rx_wthresh != RTE_PMD_PARAM_UNSET)
3315 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh;
3316
3317 if (rx_free_thresh != RTE_PMD_PARAM_UNSET)
3318 port->rx_conf[qid].rx_free_thresh = rx_free_thresh;
3319
3320 if (rx_drop_en != RTE_PMD_PARAM_UNSET)
3321 port->rx_conf[qid].rx_drop_en = rx_drop_en;
3322
3323 port->nb_rx_desc[qid] = nb_rxd;
3324 }
3325
3326 for (qid = 0; qid < nb_txq; qid++) {
3327 offloads = port->tx_conf[qid].offloads;
3328 port->tx_conf[qid] = port->dev_info.default_txconf;
3329 if (offloads != 0)
3330 port->tx_conf[qid].offloads = offloads;
3331
3332
3333 if (tx_pthresh != RTE_PMD_PARAM_UNSET)
3334 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh;
3335
3336 if (tx_hthresh != RTE_PMD_PARAM_UNSET)
3337 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh;
3338
3339 if (tx_wthresh != RTE_PMD_PARAM_UNSET)
3340 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh;
3341
3342 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET)
3343 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh;
3344
3345 if (tx_free_thresh != RTE_PMD_PARAM_UNSET)
3346 port->tx_conf[qid].tx_free_thresh = tx_free_thresh;
3347
3348 port->nb_tx_desc[qid] = nb_txd;
3349 }
3350}
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360int
3361update_jumbo_frame_offload(portid_t portid)
3362{
3363 struct rte_port *port = &ports[portid];
3364 uint32_t eth_overhead;
3365 uint64_t rx_offloads;
3366 int ret;
3367 bool on;
3368
3369
3370 if (port->dev_info.max_mtu != UINT16_MAX &&
3371 port->dev_info.max_rx_pktlen > port->dev_info.max_mtu)
3372 eth_overhead = port->dev_info.max_rx_pktlen -
3373 port->dev_info.max_mtu;
3374 else
3375 eth_overhead = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
3376
3377 rx_offloads = port->dev_conf.rxmode.offloads;
3378
3379
3380 if (port->dev_conf.rxmode.max_rx_pkt_len == 0)
3381 port->dev_conf.rxmode.max_rx_pkt_len = RTE_ETHER_MTU + eth_overhead;
3382
3383 if (port->dev_conf.rxmode.max_rx_pkt_len <= RTE_ETHER_MTU + eth_overhead) {
3384 rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3385 on = false;
3386 } else {
3387 if ((port->dev_info.rx_offload_capa & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3388 printf("Frame size (%u) is not supported by port %u\n",
3389 port->dev_conf.rxmode.max_rx_pkt_len,
3390 portid);
3391 return -1;
3392 }
3393 rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3394 on = true;
3395 }
3396
3397 if (rx_offloads != port->dev_conf.rxmode.offloads) {
3398 uint16_t qid;
3399
3400 port->dev_conf.rxmode.offloads = rx_offloads;
3401
3402
3403 for (qid = 0; qid < port->dev_info.nb_rx_queues; qid++) {
3404 if (on)
3405 port->rx_conf[qid].offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
3406 else
3407 port->rx_conf[qid].offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3408 }
3409 }
3410
3411
3412
3413
3414 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) == 0) {
3415 ret = rte_eth_dev_set_mtu(portid,
3416 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead);
3417 if (ret)
3418 printf("Failed to set MTU to %u for port %u\n",
3419 port->dev_conf.rxmode.max_rx_pkt_len - eth_overhead,
3420 portid);
3421 }
3422
3423 return 0;
3424}
3425
3426void
3427init_port_config(void)
3428{
3429 portid_t pid;
3430 struct rte_port *port;
3431 int ret;
3432
3433 RTE_ETH_FOREACH_DEV(pid) {
3434 port = &ports[pid];
3435 port->dev_conf.fdir_conf = fdir_conf;
3436
3437 ret = eth_dev_info_get_print_err(pid, &port->dev_info);
3438 if (ret != 0)
3439 return;
3440
3441 if (nb_rxq > 1) {
3442 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3443 port->dev_conf.rx_adv_conf.rss_conf.rss_hf =
3444 rss_hf & port->dev_info.flow_type_rss_offloads;
3445 } else {
3446 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL;
3447 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0;
3448 }
3449
3450 if (port->dcb_flag == 0) {
3451 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0)
3452 port->dev_conf.rxmode.mq_mode =
3453 (enum rte_eth_rx_mq_mode)
3454 (rx_mq_mode & ETH_MQ_RX_RSS);
3455 else
3456 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE;
3457 }
3458
3459 rxtx_port_config(port);
3460
3461 ret = eth_macaddr_get_print_err(pid, &port->eth_addr);
3462 if (ret != 0)
3463 return;
3464
3465#if defined RTE_NET_IXGBE && defined RTE_LIBRTE_IXGBE_BYPASS
3466 rte_pmd_ixgbe_bypass_init(pid);
3467#endif
3468
3469 if (lsc_interrupt &&
3470 (rte_eth_devices[pid].data->dev_flags &
3471 RTE_ETH_DEV_INTR_LSC))
3472 port->dev_conf.intr_conf.lsc = 1;
3473 if (rmv_interrupt &&
3474 (rte_eth_devices[pid].data->dev_flags &
3475 RTE_ETH_DEV_INTR_RMV))
3476 port->dev_conf.intr_conf.rmv = 1;
3477 }
3478}
3479
3480void set_port_slave_flag(portid_t slave_pid)
3481{
3482 struct rte_port *port;
3483
3484 port = &ports[slave_pid];
3485 port->slave_flag = 1;
3486}
3487
3488void clear_port_slave_flag(portid_t slave_pid)
3489{
3490 struct rte_port *port;
3491
3492 port = &ports[slave_pid];
3493 port->slave_flag = 0;
3494}
3495
3496uint8_t port_is_bonding_slave(portid_t slave_pid)
3497{
3498 struct rte_port *port;
3499
3500 port = &ports[slave_pid];
3501 if ((rte_eth_devices[slave_pid].data->dev_flags &
3502 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1))
3503 return 1;
3504 return 0;
3505}
3506
3507const uint16_t vlan_tags[] = {
3508 0, 1, 2, 3, 4, 5, 6, 7,
3509 8, 9, 10, 11, 12, 13, 14, 15,
3510 16, 17, 18, 19, 20, 21, 22, 23,
3511 24, 25, 26, 27, 28, 29, 30, 31
3512};
3513
3514static int
3515get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf,
3516 enum dcb_mode_enable dcb_mode,
3517 enum rte_eth_nb_tcs num_tcs,
3518 uint8_t pfc_en)
3519{
3520 uint8_t i;
3521 int32_t rc;
3522 struct rte_eth_rss_conf rss_conf;
3523
3524
3525
3526
3527
3528 if (dcb_mode == DCB_VT_ENABLED) {
3529 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf =
3530 ð_conf->rx_adv_conf.vmdq_dcb_conf;
3531 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf =
3532 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf;
3533
3534
3535 vmdq_rx_conf->enable_default_pool = 0;
3536 vmdq_rx_conf->default_pool = 0;
3537 vmdq_rx_conf->nb_queue_pools =
3538 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3539 vmdq_tx_conf->nb_queue_pools =
3540 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS);
3541
3542 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools;
3543 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) {
3544 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i];
3545 vmdq_rx_conf->pool_map[i].pools =
3546 1 << (i % vmdq_rx_conf->nb_queue_pools);
3547 }
3548 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3549 vmdq_rx_conf->dcb_tc[i] = i % num_tcs;
3550 vmdq_tx_conf->dcb_tc[i] = i % num_tcs;
3551 }
3552
3553
3554 eth_conf->rxmode.mq_mode =
3555 (enum rte_eth_rx_mq_mode)
3556 (rx_mq_mode & ETH_MQ_RX_VMDQ_DCB);
3557 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
3558 } else {
3559 struct rte_eth_dcb_rx_conf *rx_conf =
3560 ð_conf->rx_adv_conf.dcb_rx_conf;
3561 struct rte_eth_dcb_tx_conf *tx_conf =
3562 ð_conf->tx_adv_conf.dcb_tx_conf;
3563
3564 memset(&rss_conf, 0, sizeof(struct rte_eth_rss_conf));
3565
3566 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf);
3567 if (rc != 0)
3568 return rc;
3569
3570 rx_conf->nb_tcs = num_tcs;
3571 tx_conf->nb_tcs = num_tcs;
3572
3573 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) {
3574 rx_conf->dcb_tc[i] = i % num_tcs;
3575 tx_conf->dcb_tc[i] = i % num_tcs;
3576 }
3577
3578 eth_conf->rxmode.mq_mode =
3579 (enum rte_eth_rx_mq_mode)
3580 (rx_mq_mode & ETH_MQ_RX_DCB_RSS);
3581 eth_conf->rx_adv_conf.rss_conf = rss_conf;
3582 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB;
3583 }
3584
3585 if (pfc_en)
3586 eth_conf->dcb_capability_en =
3587 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT;
3588 else
3589 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT;
3590
3591 return 0;
3592}
3593
3594int
3595init_port_dcb_config(portid_t pid,
3596 enum dcb_mode_enable dcb_mode,
3597 enum rte_eth_nb_tcs num_tcs,
3598 uint8_t pfc_en)
3599{
3600 struct rte_eth_conf port_conf;
3601 struct rte_port *rte_port;
3602 int retval;
3603 uint16_t i;
3604
3605 rte_port = &ports[pid];
3606
3607 memset(&port_conf, 0, sizeof(struct rte_eth_conf));
3608
3609 dcb_config = 1;
3610
3611 port_conf.rxmode = rte_port->dev_conf.rxmode;
3612 port_conf.txmode = rte_port->dev_conf.txmode;
3613
3614
3615 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en);
3616 if (retval < 0)
3617 return retval;
3618 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3619
3620
3621 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf);
3622 if (retval < 0)
3623 return retval;
3624
3625 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info);
3626 if (retval != 0)
3627 return retval;
3628
3629
3630
3631
3632 if (dcb_mode == DCB_VT_ENABLED &&
3633 rte_port->dev_info.vmdq_pool_base > 0) {
3634 printf("VMDQ_DCB multi-queue mode is nonsensical"
3635 " for port %d.", pid);
3636 return -1;
3637 }
3638
3639
3640
3641
3642 if (dcb_mode == DCB_VT_ENABLED) {
3643 if (rte_port->dev_info.max_vfs > 0) {
3644 nb_rxq = rte_port->dev_info.nb_rx_queues;
3645 nb_txq = rte_port->dev_info.nb_tx_queues;
3646 } else {
3647 nb_rxq = rte_port->dev_info.max_rx_queues;
3648 nb_txq = rte_port->dev_info.max_tx_queues;
3649 }
3650 } else {
3651
3652 if (rte_port->dev_info.vmdq_pool_base == 0) {
3653 nb_rxq = rte_port->dev_info.max_rx_queues;
3654 nb_txq = rte_port->dev_info.max_tx_queues;
3655 } else {
3656 nb_rxq = (queueid_t)num_tcs;
3657 nb_txq = (queueid_t)num_tcs;
3658
3659 }
3660 }
3661 rx_free_thresh = 64;
3662
3663 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf));
3664
3665 rxtx_port_config(rte_port);
3666
3667 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3668 for (i = 0; i < RTE_DIM(vlan_tags); i++)
3669 rx_vft_set(pid, vlan_tags[i], 1);
3670
3671 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr);
3672 if (retval != 0)
3673 return retval;
3674
3675 rte_port->dcb_flag = 1;
3676
3677 return 0;
3678}
3679
3680static void
3681init_port(void)
3682{
3683 int i;
3684
3685
3686 ports = rte_zmalloc("testpmd: ports",
3687 sizeof(struct rte_port) * RTE_MAX_ETHPORTS,
3688 RTE_CACHE_LINE_SIZE);
3689 if (ports == NULL) {
3690 rte_exit(EXIT_FAILURE,
3691 "rte_zmalloc(%d struct rte_port) failed\n",
3692 RTE_MAX_ETHPORTS);
3693 }
3694 for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3695 LIST_INIT(&ports[i].flow_tunnel_list);
3696
3697 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3698 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3699 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS);
3700}
3701
3702static void
3703force_quit(void)
3704{
3705 pmd_test_exit();
3706 prompt_exit();
3707}
3708
3709static void
3710print_stats(void)
3711{
3712 uint8_t i;
3713 const char clr[] = { 27, '[', '2', 'J', '\0' };
3714 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' };
3715
3716
3717 printf("%s%s", clr, top_left);
3718
3719 printf("\nPort statistics ====================================");
3720 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
3721 nic_stats_display(fwd_ports_ids[i]);
3722
3723 fflush(stdout);
3724}
3725
3726static void
3727signal_handler(int signum)
3728{
3729 if (signum == SIGINT || signum == SIGTERM) {
3730 printf("\nSignal %d received, preparing to exit...\n",
3731 signum);
3732#ifdef RTE_LIB_PDUMP
3733
3734 rte_pdump_uninit();
3735#endif
3736#ifdef RTE_LIB_LATENCYSTATS
3737 if (latencystats_enabled != 0)
3738 rte_latencystats_uninit();
3739#endif
3740 force_quit();
3741
3742 f_quit = 1;
3743
3744 signal(signum, SIG_DFL);
3745 kill(getpid(), signum);
3746 }
3747}
3748
3749int
3750main(int argc, char** argv)
3751{
3752 int diag;
3753 portid_t port_id;
3754 uint16_t count;
3755 int ret;
3756
3757 signal(SIGINT, signal_handler);
3758 signal(SIGTERM, signal_handler);
3759
3760 testpmd_logtype = rte_log_register("testpmd");
3761 if (testpmd_logtype < 0)
3762 rte_exit(EXIT_FAILURE, "Cannot register log type");
3763 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG);
3764
3765 diag = rte_eal_init(argc, argv);
3766 if (diag < 0)
3767 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n",
3768 rte_strerror(rte_errno));
3769
3770 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
3771 rte_exit(EXIT_FAILURE,
3772 "Secondary process type not supported.\n");
3773
3774 ret = register_eth_event_callback();
3775 if (ret != 0)
3776 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events");
3777
3778#ifdef RTE_LIB_PDUMP
3779
3780 rte_pdump_init();
3781#endif
3782
3783 count = 0;
3784 RTE_ETH_FOREACH_DEV(port_id) {
3785 ports_ids[count] = port_id;
3786 count++;
3787 }
3788 nb_ports = (portid_t) count;
3789 if (nb_ports == 0)
3790 TESTPMD_LOG(WARNING, "No probed ethernet devices\n");
3791
3792
3793 init_port();
3794
3795 set_def_fwd_config();
3796 if (nb_lcores == 0)
3797 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n"
3798 "Check the core mask argument\n");
3799
3800
3801#ifdef RTE_LIB_BITRATESTATS
3802 bitrate_enabled = 0;
3803#endif
3804#ifdef RTE_LIB_LATENCYSTATS
3805 latencystats_enabled = 0;
3806#endif
3807
3808
3809#ifdef RTE_EXEC_ENV_FREEBSD
3810 do_mlockall = 0;
3811#else
3812 do_mlockall = 1;
3813#endif
3814
3815 argc -= diag;
3816 argv += diag;
3817 if (argc > 1)
3818 launch_args_parse(argc, argv);
3819
3820 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) {
3821 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n",
3822 strerror(errno));
3823 }
3824
3825 if (tx_first && interactive)
3826 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on "
3827 "interactive mode.\n");
3828
3829 if (tx_first && lsc_interrupt) {
3830 printf("Warning: lsc_interrupt needs to be off when "
3831 " using tx_first. Disabling.\n");
3832 lsc_interrupt = 0;
3833 }
3834
3835 if (!nb_rxq && !nb_txq)
3836 printf("Warning: Either rx or tx queues should be non-zero\n");
3837
3838 if (nb_rxq > 1 && nb_rxq > nb_txq)
3839 printf("Warning: nb_rxq=%d enables RSS configuration, "
3840 "but nb_txq=%d will prevent to fully test it.\n",
3841 nb_rxq, nb_txq);
3842
3843 init_config();
3844
3845 if (hot_plug) {
3846 ret = rte_dev_hotplug_handle_enable();
3847 if (ret) {
3848 RTE_LOG(ERR, EAL,
3849 "fail to enable hotplug handling.");
3850 return -1;
3851 }
3852
3853 ret = rte_dev_event_monitor_start();
3854 if (ret) {
3855 RTE_LOG(ERR, EAL,
3856 "fail to start device event monitoring.");
3857 return -1;
3858 }
3859
3860 ret = rte_dev_event_callback_register(NULL,
3861 dev_event_callback, NULL);
3862 if (ret) {
3863 RTE_LOG(ERR, EAL,
3864 "fail to register device event callback\n");
3865 return -1;
3866 }
3867 }
3868
3869 if (!no_device_start && start_port(RTE_PORT_ALL) != 0)
3870 rte_exit(EXIT_FAILURE, "Start ports failed\n");
3871
3872
3873 RTE_ETH_FOREACH_DEV(port_id) {
3874 ret = rte_eth_promiscuous_enable(port_id);
3875 if (ret != 0)
3876 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n",
3877 port_id, rte_strerror(-ret));
3878 }
3879
3880
3881 rte_metrics_init(rte_socket_id());
3882
3883#ifdef RTE_LIB_LATENCYSTATS
3884 if (latencystats_enabled != 0) {
3885 int ret = rte_latencystats_init(1, NULL);
3886 if (ret)
3887 printf("Warning: latencystats init()"
3888 " returned error %d\n", ret);
3889 printf("Latencystats running on lcore %d\n",
3890 latencystats_lcore_id);
3891 }
3892#endif
3893
3894
3895#ifdef RTE_LIB_BITRATESTATS
3896 if (bitrate_enabled != 0) {
3897 bitrate_data = rte_stats_bitrate_create();
3898 if (bitrate_data == NULL)
3899 rte_exit(EXIT_FAILURE,
3900 "Could not allocate bitrate data.\n");
3901 rte_stats_bitrate_reg(bitrate_data);
3902 }
3903#endif
3904
3905#ifdef RTE_LIB_CMDLINE
3906 if (strlen(cmdline_filename) != 0)
3907 cmdline_read_from_file(cmdline_filename);
3908
3909 if (interactive == 1) {
3910 if (auto_start) {
3911 printf("Start automatic packet forwarding\n");
3912 start_packet_forwarding(0);
3913 }
3914 prompt();
3915 pmd_test_exit();
3916 } else
3917#endif
3918 {
3919 char c;
3920 int rc;
3921
3922 f_quit = 0;
3923
3924 printf("No commandline core given, start packet forwarding\n");
3925 start_packet_forwarding(tx_first);
3926 if (stats_period != 0) {
3927 uint64_t prev_time = 0, cur_time, diff_time = 0;
3928 uint64_t timer_period;
3929
3930
3931 timer_period = stats_period * rte_get_timer_hz();
3932
3933 while (f_quit == 0) {
3934 cur_time = rte_get_timer_cycles();
3935 diff_time += cur_time - prev_time;
3936
3937 if (diff_time >= timer_period) {
3938 print_stats();
3939
3940 diff_time = 0;
3941 }
3942
3943 prev_time = cur_time;
3944 sleep(1);
3945 }
3946 }
3947
3948 printf("Press enter to exit\n");
3949 rc = read(0, &c, 1);
3950 pmd_test_exit();
3951 if (rc < 0)
3952 return 1;
3953 }
3954
3955 ret = rte_eal_cleanup();
3956 if (ret != 0)
3957 rte_exit(EXIT_FAILURE,
3958 "EAL cleanup failed: %s\n", strerror(-ret));
3959
3960 return EXIT_SUCCESS;
3961}
3962