1
2
3
4
5
6#include "test_pipeline_common.h"
7
8int
9pipeline_test_result(struct evt_test *test, struct evt_options *opt)
10{
11 RTE_SET_USED(opt);
12 int i;
13 uint64_t total = 0;
14 struct test_pipeline *t = evt_test_priv(test);
15
16 evt_info("Packet distribution across worker cores :");
17 for (i = 0; i < t->nb_workers; i++)
18 total += t->worker[i].processed_pkts;
19 for (i = 0; i < t->nb_workers; i++)
20 evt_info("Worker %d packets: "CLGRN"%"PRIx64""CLNRM" percentage:"
21 CLGRN" %3.2f"CLNRM, i,
22 t->worker[i].processed_pkts,
23 (((double)t->worker[i].processed_pkts)/total)
24 * 100);
25 return t->result;
26}
27
28void
29pipeline_opt_dump(struct evt_options *opt, uint8_t nb_queues)
30{
31 evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores));
32 evt_dump_worker_lcores(opt);
33 evt_dump_nb_stages(opt);
34 evt_dump("nb_evdev_ports", "%d", pipeline_nb_event_ports(opt));
35 evt_dump("nb_evdev_queues", "%d", nb_queues);
36 evt_dump_queue_priority(opt);
37 evt_dump_sched_type_list(opt);
38 evt_dump_producer_type(opt);
39 evt_dump("nb_eth_rx_queues", "%d", opt->eth_queues);
40 evt_dump("event_vector", "%d", opt->ena_vector);
41 if (opt->ena_vector) {
42 evt_dump("vector_size", "%d", opt->vector_size);
43 evt_dump("vector_tmo_ns", "%" PRIu64 "", opt->vector_tmo_nsec);
44 }
45}
46
47static inline uint64_t
48processed_pkts(struct test_pipeline *t)
49{
50 uint8_t i;
51 uint64_t total = 0;
52
53 for (i = 0; i < t->nb_workers; i++)
54 total += t->worker[i].processed_pkts;
55
56 return total;
57}
58
59int
60pipeline_launch_lcores(struct evt_test *test, struct evt_options *opt,
61 int (*worker)(void *))
62{
63 int ret, lcore_id;
64 struct test_pipeline *t = evt_test_priv(test);
65
66 int port_idx = 0;
67
68 RTE_LCORE_FOREACH_WORKER(lcore_id) {
69 if (!(opt->wlcores[lcore_id]))
70 continue;
71
72 ret = rte_eal_remote_launch(worker,
73 &t->worker[port_idx], lcore_id);
74 if (ret) {
75 evt_err("failed to launch worker %d", lcore_id);
76 return ret;
77 }
78 port_idx++;
79 }
80
81 uint64_t perf_cycles = rte_get_timer_cycles();
82 const uint64_t perf_sample = rte_get_timer_hz();
83
84 static float total_mpps;
85 static uint64_t samples;
86
87 uint64_t prev_pkts = 0;
88
89 while (t->done == false) {
90 const uint64_t new_cycles = rte_get_timer_cycles();
91
92 if ((new_cycles - perf_cycles) > perf_sample) {
93 const uint64_t curr_pkts = processed_pkts(t);
94
95 float mpps = (float)(curr_pkts - prev_pkts)/1000000;
96
97 prev_pkts = curr_pkts;
98 perf_cycles = new_cycles;
99 total_mpps += mpps;
100 ++samples;
101 printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM,
102 mpps, total_mpps/samples);
103 fflush(stdout);
104 }
105 }
106 printf("\n");
107 return 0;
108}
109
110int
111pipeline_opt_check(struct evt_options *opt, uint64_t nb_queues)
112{
113 unsigned int lcores;
114
115
116 lcores = 2;
117
118 if (opt->prod_type != EVT_PROD_TYPE_ETH_RX_ADPTR) {
119 evt_err("Invalid producer type '%s' valid producer '%s'",
120 evt_prod_id_to_name(opt->prod_type),
121 evt_prod_id_to_name(EVT_PROD_TYPE_ETH_RX_ADPTR));
122 return -1;
123 }
124
125 if (!rte_eth_dev_count_avail()) {
126 evt_err("test needs minimum 1 ethernet dev");
127 return -1;
128 }
129
130 if (rte_lcore_count() < lcores) {
131 evt_err("test need minimum %d lcores", lcores);
132 return -1;
133 }
134
135
136 if (evt_lcores_has_overlap(opt->wlcores, rte_get_main_lcore())) {
137 evt_err("worker lcores overlaps with main lcore");
138 return -1;
139 }
140 if (evt_has_disabled_lcore(opt->wlcores)) {
141 evt_err("one or more workers lcores are not enabled");
142 return -1;
143 }
144 if (!evt_has_active_lcore(opt->wlcores)) {
145 evt_err("minimum one worker is required");
146 return -1;
147 }
148
149 if (nb_queues > EVT_MAX_QUEUES) {
150 evt_err("number of queues exceeds %d", EVT_MAX_QUEUES);
151 return -1;
152 }
153 if (pipeline_nb_event_ports(opt) > EVT_MAX_PORTS) {
154 evt_err("number of ports exceeds %d", EVT_MAX_PORTS);
155 return -1;
156 }
157
158 if (evt_has_invalid_stage(opt))
159 return -1;
160
161 if (evt_has_invalid_sched_type(opt))
162 return -1;
163
164 return 0;
165}
166
167#define NB_RX_DESC 128
168#define NB_TX_DESC 512
169int
170pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt)
171{
172 uint16_t i, j;
173 int ret;
174 uint8_t nb_queues = 1;
175 struct test_pipeline *t = evt_test_priv(test);
176 struct rte_eth_rxconf rx_conf;
177 struct rte_eth_conf port_conf = {
178 .rxmode = {
179 .mq_mode = ETH_MQ_RX_RSS,
180 },
181 .rx_adv_conf = {
182 .rss_conf = {
183 .rss_key = NULL,
184 .rss_hf = ETH_RSS_IP,
185 },
186 },
187 };
188
189 if (!rte_eth_dev_count_avail()) {
190 evt_err("No ethernet ports found.");
191 return -ENODEV;
192 }
193
194 if (opt->max_pkt_sz < RTE_ETHER_MIN_LEN) {
195 evt_err("max_pkt_sz can not be less than %d",
196 RTE_ETHER_MIN_LEN);
197 return -EINVAL;
198 }
199
200 port_conf.rxmode.max_rx_pkt_len = opt->max_pkt_sz;
201 if (opt->max_pkt_sz > RTE_ETHER_MAX_LEN)
202 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
203
204 t->internal_port = 1;
205 RTE_ETH_FOREACH_DEV(i) {
206 struct rte_eth_dev_info dev_info;
207 struct rte_eth_conf local_port_conf = port_conf;
208 uint32_t caps = 0;
209
210 ret = rte_event_eth_tx_adapter_caps_get(opt->dev_id, i, &caps);
211 if (ret != 0) {
212 evt_err("failed to get event tx adapter[%d] caps", i);
213 return ret;
214 }
215
216 if (!(caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT))
217 t->internal_port = 0;
218
219 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, i, &caps);
220 if (ret != 0) {
221 evt_err("failed to get event tx adapter[%d] caps", i);
222 return ret;
223 }
224
225 if (!(caps & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT))
226 local_port_conf.rxmode.offloads |=
227 DEV_RX_OFFLOAD_RSS_HASH;
228
229 ret = rte_eth_dev_info_get(i, &dev_info);
230 if (ret != 0) {
231 evt_err("Error during getting device (port %u) info: %s\n",
232 i, strerror(-ret));
233 return ret;
234 }
235
236
237 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
238 local_port_conf.txmode.offloads |=
239 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
240
241 rx_conf = dev_info.default_rxconf;
242 rx_conf.offloads = port_conf.rxmode.offloads;
243
244 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
245 dev_info.flow_type_rss_offloads;
246 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
247 port_conf.rx_adv_conf.rss_conf.rss_hf) {
248 evt_info("Port %u modified RSS hash function based on hardware support,"
249 "requested:%#"PRIx64" configured:%#"PRIx64"",
250 i,
251 port_conf.rx_adv_conf.rss_conf.rss_hf,
252 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
253 }
254
255 if (rte_eth_dev_configure(i, opt->eth_queues, nb_queues,
256 &local_port_conf) < 0) {
257 evt_err("Failed to configure eth port [%d]", i);
258 return -EINVAL;
259 }
260
261 for (j = 0; j < opt->eth_queues; j++) {
262 if (rte_eth_rx_queue_setup(
263 i, j, NB_RX_DESC, rte_socket_id(), &rx_conf,
264 opt->per_port_pool ? t->pool[i] :
265 t->pool[0]) < 0) {
266 evt_err("Failed to setup eth port [%d] rx_queue: %d.",
267 i, 0);
268 return -EINVAL;
269 }
270 }
271
272 if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC,
273 rte_socket_id(), NULL) < 0) {
274 evt_err("Failed to setup eth port [%d] tx_queue: %d.",
275 i, 0);
276 return -EINVAL;
277 }
278
279 ret = rte_eth_promiscuous_enable(i);
280 if (ret != 0) {
281 evt_err("Failed to enable promiscuous mode for eth port [%d]: %s",
282 i, rte_strerror(-ret));
283 return ret;
284 }
285 }
286
287 return 0;
288}
289
290int
291pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt,
292 uint8_t *queue_arr, uint8_t nb_queues,
293 const struct rte_event_port_conf p_conf)
294{
295 int ret;
296 uint8_t port;
297 struct test_pipeline *t = evt_test_priv(test);
298
299
300
301 for (port = 0; port < evt_nr_active_lcores(opt->wlcores); port++) {
302 struct worker_data *w = &t->worker[port];
303
304 w->dev_id = opt->dev_id;
305 w->port_id = port;
306 w->t = t;
307 w->processed_pkts = 0;
308
309 ret = rte_event_port_setup(opt->dev_id, port, &p_conf);
310 if (ret) {
311 evt_err("failed to setup port %d", port);
312 return ret;
313 }
314
315 if (rte_event_port_link(opt->dev_id, port, queue_arr, NULL,
316 nb_queues) != nb_queues)
317 goto link_fail;
318 }
319
320 return 0;
321
322link_fail:
323 evt_err("failed to link queues to port %d", port);
324 return -EINVAL;
325}
326
327int
328pipeline_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride,
329 struct rte_event_port_conf prod_conf)
330{
331 int ret = 0;
332 uint16_t prod;
333 struct rte_mempool *vector_pool = NULL;
334 struct rte_event_eth_rx_adapter_queue_conf queue_conf;
335 struct rte_event_eth_rx_adapter_event_vector_config vec_conf;
336
337 memset(&queue_conf, 0,
338 sizeof(struct rte_event_eth_rx_adapter_queue_conf));
339 queue_conf.ev.sched_type = opt->sched_type_list[0];
340 if (opt->ena_vector) {
341 unsigned int nb_elem = (opt->pool_sz / opt->vector_size) << 1;
342
343 nb_elem = nb_elem ? nb_elem : 1;
344 vector_pool = rte_event_vector_pool_create(
345 "vector_pool", nb_elem, 0, opt->vector_size,
346 opt->socket_id);
347 if (vector_pool == NULL) {
348 evt_err("failed to create event vector pool");
349 return -ENOMEM;
350 }
351 }
352 RTE_ETH_FOREACH_DEV(prod) {
353 struct rte_event_eth_rx_adapter_vector_limits limits;
354 uint32_t cap;
355
356 ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id,
357 prod, &cap);
358 if (ret) {
359 evt_err("failed to get event rx adapter[%d]"
360 " capabilities",
361 opt->dev_id);
362 return ret;
363 }
364
365 if (opt->ena_vector) {
366 memset(&limits, 0, sizeof(limits));
367 ret = rte_event_eth_rx_adapter_vector_limits_get(
368 opt->dev_id, prod, &limits);
369 if (ret) {
370 evt_err("failed to get vector limits");
371 return ret;
372 }
373
374 if (opt->vector_size < limits.min_sz ||
375 opt->vector_size > limits.max_sz) {
376 evt_err("Vector size [%d] not within limits max[%d] min[%d]",
377 opt->vector_size, limits.min_sz,
378 limits.max_sz);
379 return -EINVAL;
380 }
381
382 if (limits.log2_sz &&
383 !rte_is_power_of_2(opt->vector_size)) {
384 evt_err("Vector size [%d] not power of 2",
385 opt->vector_size);
386 return -EINVAL;
387 }
388
389 if (opt->vector_tmo_nsec > limits.max_timeout_ns ||
390 opt->vector_tmo_nsec < limits.min_timeout_ns) {
391 evt_err("Vector timeout [%" PRIu64
392 "] not within limits max[%" PRIu64
393 "] min[%" PRIu64 "]",
394 opt->vector_tmo_nsec,
395 limits.max_timeout_ns,
396 limits.min_timeout_ns);
397 return -EINVAL;
398 }
399
400 if (cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_EVENT_VECTOR) {
401 queue_conf.rx_queue_flags |=
402 RTE_EVENT_ETH_RX_ADAPTER_QUEUE_EVENT_VECTOR;
403 } else {
404 evt_err("Rx adapter doesn't support event vector");
405 return -EINVAL;
406 }
407 }
408 queue_conf.ev.queue_id = prod * stride;
409 ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id,
410 &prod_conf);
411 if (ret) {
412 evt_err("failed to create rx adapter[%d]", prod);
413 return ret;
414 }
415 ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1,
416 &queue_conf);
417 if (ret) {
418 evt_err("failed to add rx queues to adapter[%d]", prod);
419 return ret;
420 }
421
422 if (opt->ena_vector) {
423 vec_conf.vector_sz = opt->vector_size;
424 vec_conf.vector_timeout_ns = opt->vector_tmo_nsec;
425 vec_conf.vector_mp = vector_pool;
426 if (rte_event_eth_rx_adapter_queue_event_vector_config(
427 prod, prod, -1, &vec_conf) < 0) {
428 evt_err("Failed to configure event vectorization for Rx adapter");
429 return -EINVAL;
430 }
431 }
432
433 if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) {
434 uint32_t service_id = -1U;
435
436 rte_event_eth_rx_adapter_service_id_get(prod,
437 &service_id);
438 ret = evt_service_setup(service_id);
439 if (ret) {
440 evt_err("Failed to setup service core"
441 " for Rx adapter");
442 return ret;
443 }
444 }
445
446 evt_info("Port[%d] using Rx adapter[%d] configured", prod,
447 prod);
448 }
449
450 return ret;
451}
452
453int
454pipeline_event_tx_adapter_setup(struct evt_options *opt,
455 struct rte_event_port_conf port_conf)
456{
457 int ret = 0;
458 uint16_t consm;
459
460 RTE_ETH_FOREACH_DEV(consm) {
461 uint32_t cap;
462
463 ret = rte_event_eth_tx_adapter_caps_get(opt->dev_id,
464 consm, &cap);
465 if (ret) {
466 evt_err("failed to get event tx adapter[%d] caps",
467 consm);
468 return ret;
469 }
470
471 if (opt->ena_vector) {
472 if (!(cap &
473 RTE_EVENT_ETH_TX_ADAPTER_CAP_EVENT_VECTOR)) {
474 evt_err("Tx adapter doesn't support event vector");
475 return -EINVAL;
476 }
477 }
478
479 ret = rte_event_eth_tx_adapter_create(consm, opt->dev_id,
480 &port_conf);
481 if (ret) {
482 evt_err("failed to create tx adapter[%d]", consm);
483 return ret;
484 }
485
486 ret = rte_event_eth_tx_adapter_queue_add(consm, consm, -1);
487 if (ret) {
488 evt_err("failed to add tx queues to adapter[%d]",
489 consm);
490 return ret;
491 }
492
493 if (!(cap & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT)) {
494 uint32_t service_id = -1U;
495
496 ret = rte_event_eth_tx_adapter_service_id_get(consm,
497 &service_id);
498 if (ret != -ESRCH && ret != 0) {
499 evt_err("Failed to get Tx adptr service ID");
500 return ret;
501 }
502 ret = evt_service_setup(service_id);
503 if (ret) {
504 evt_err("Failed to setup service core"
505 " for Tx adapter");
506 return ret;
507 }
508 }
509
510 evt_info("Port[%d] using Tx adapter[%d] Configured", consm,
511 consm);
512 }
513
514 return ret;
515}
516
517void
518pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt)
519{
520 uint16_t i;
521 RTE_SET_USED(test);
522 RTE_SET_USED(opt);
523
524 RTE_ETH_FOREACH_DEV(i) {
525 rte_event_eth_rx_adapter_stop(i);
526 rte_event_eth_tx_adapter_stop(i);
527 rte_eth_dev_stop(i);
528 }
529}
530
531void
532pipeline_eventdev_destroy(struct evt_test *test, struct evt_options *opt)
533{
534 RTE_SET_USED(test);
535
536 rte_event_dev_stop(opt->dev_id);
537 rte_event_dev_close(opt->dev_id);
538}
539
540int
541pipeline_mempool_setup(struct evt_test *test, struct evt_options *opt)
542{
543 struct test_pipeline *t = evt_test_priv(test);
544 int i, ret;
545
546 if (!opt->mbuf_sz)
547 opt->mbuf_sz = RTE_MBUF_DEFAULT_BUF_SIZE;
548
549 if (!opt->max_pkt_sz)
550 opt->max_pkt_sz = RTE_ETHER_MAX_LEN;
551
552 RTE_ETH_FOREACH_DEV(i) {
553 struct rte_eth_dev_info dev_info;
554 uint16_t data_size = 0;
555
556 memset(&dev_info, 0, sizeof(dev_info));
557 ret = rte_eth_dev_info_get(i, &dev_info);
558 if (ret != 0) {
559 evt_err("Error during getting device (port %u) info: %s\n",
560 i, strerror(-ret));
561 return ret;
562 }
563
564 if (dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX &&
565 dev_info.rx_desc_lim.nb_mtu_seg_max != 0) {
566 data_size = opt->max_pkt_sz /
567 dev_info.rx_desc_lim.nb_mtu_seg_max;
568 data_size += RTE_PKTMBUF_HEADROOM;
569
570 if (data_size > opt->mbuf_sz)
571 opt->mbuf_sz = data_size;
572 }
573 if (opt->per_port_pool) {
574 char name[RTE_MEMPOOL_NAMESIZE];
575
576 snprintf(name, RTE_MEMPOOL_NAMESIZE, "%s-%d",
577 test->name, i);
578 t->pool[i] = rte_pktmbuf_pool_create(
579 name,
580 opt->pool_sz,
581 0,
582 0, opt->mbuf_sz, opt->socket_id);
583
584 if (t->pool[i] == NULL) {
585 evt_err("failed to create mempool %s", name);
586 return -ENOMEM;
587 }
588 }
589 }
590
591 if (!opt->per_port_pool) {
592 t->pool[0] = rte_pktmbuf_pool_create(
593 test->name,
594 opt->pool_sz,
595 0,
596 0, opt->mbuf_sz, opt->socket_id);
597
598 if (t->pool[0] == NULL) {
599 evt_err("failed to create mempool");
600 return -ENOMEM;
601 }
602 }
603
604 return 0;
605}
606
607void
608pipeline_mempool_destroy(struct evt_test *test, struct evt_options *opt)
609{
610 struct test_pipeline *t = evt_test_priv(test);
611 int i;
612
613 RTE_SET_USED(opt);
614 if (opt->per_port_pool) {
615 RTE_ETH_FOREACH_DEV(i)
616 rte_mempool_free(t->pool[i]);
617 } else {
618 rte_mempool_free(t->pool[0]);
619 }
620}
621
622int
623pipeline_test_setup(struct evt_test *test, struct evt_options *opt)
624{
625 void *test_pipeline;
626
627 test_pipeline = rte_zmalloc_socket(test->name,
628 sizeof(struct test_pipeline), RTE_CACHE_LINE_SIZE,
629 opt->socket_id);
630 if (test_pipeline == NULL) {
631 evt_err("failed to allocate test_pipeline memory");
632 goto nomem;
633 }
634 test->test_priv = test_pipeline;
635
636 struct test_pipeline *t = evt_test_priv(test);
637
638 t->nb_workers = evt_nr_active_lcores(opt->wlcores);
639 t->outstand_pkts = opt->nb_pkts * evt_nr_active_lcores(opt->wlcores);
640 t->done = false;
641 t->nb_flows = opt->nb_flows;
642 t->result = EVT_TEST_FAILED;
643 t->opt = opt;
644 opt->prod_type = EVT_PROD_TYPE_ETH_RX_ADPTR;
645 memcpy(t->sched_type_list, opt->sched_type_list,
646 sizeof(opt->sched_type_list));
647 return 0;
648nomem:
649 return -ENOMEM;
650}
651
652void
653pipeline_test_destroy(struct evt_test *test, struct evt_options *opt)
654{
655 RTE_SET_USED(opt);
656
657 rte_free(test->test_priv);
658}
659
