1
2
3
4
5#include "test_perf_common.h"
6
7
8
9static inline int
10atq_nb_event_queues(struct evt_options *opt)
11{
12
13 return opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR ?
14 rte_eth_dev_count_avail() : evt_nr_active_lcores(opt->plcores);
15}
16
17static __rte_always_inline void
18atq_mark_fwd_latency(struct rte_event *const ev)
19{
20 if (unlikely(ev->sub_event_type == 0)) {
21 struct perf_elt *const m = ev->event_ptr;
22
23 m->timestamp = rte_get_timer_cycles();
24 }
25}
26
27static __rte_always_inline void
28atq_fwd_event(struct rte_event *const ev, uint8_t *const sched_type_list,
29 const uint8_t nb_stages)
30{
31 ev->sub_event_type++;
32 ev->sched_type = sched_type_list[ev->sub_event_type % nb_stages];
33 ev->op = RTE_EVENT_OP_FORWARD;
34 ev->event_type = RTE_EVENT_TYPE_CPU;
35}
36
37static int
38perf_atq_worker(void *arg, const int enable_fwd_latency)
39{
40 PERF_WORKER_INIT;
41 struct rte_event ev;
42
43 while (t->done == false) {
44 uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0);
45
46 if (!event) {
47 rte_pause();
48 continue;
49 }
50
51 if (enable_fwd_latency && !prod_timer_type)
52
53 atq_mark_fwd_latency(&ev);
54
55
56 if (unlikely((ev.sub_event_type % nb_stages) == laststage)) {
57 if (enable_fwd_latency)
58 cnt = perf_process_last_stage_latency(pool,
59 &ev, w, bufs, sz, cnt);
60 else
61 cnt = perf_process_last_stage(pool, &ev, w,
62 bufs, sz, cnt);
63 } else {
64 atq_fwd_event(&ev, sched_type_list, nb_stages);
65 while (rte_event_enqueue_burst(dev, port, &ev, 1) != 1)
66 rte_pause();
67 }
68 }
69 return 0;
70}
71
72static int
73perf_atq_worker_burst(void *arg, const int enable_fwd_latency)
74{
75 PERF_WORKER_INIT;
76 uint16_t i;
77
78 struct rte_event ev[BURST_SIZE + 1];
79
80 while (t->done == false) {
81 uint16_t const nb_rx = rte_event_dequeue_burst(dev, port, ev,
82 BURST_SIZE, 0);
83
84 if (!nb_rx) {
85 rte_pause();
86 continue;
87 }
88
89 for (i = 0; i < nb_rx; i++) {
90 if (enable_fwd_latency && !prod_timer_type) {
91 rte_prefetch0(ev[i+1].event_ptr);
92
93
94
95 atq_mark_fwd_latency(&ev[i]);
96 }
97
98 if (unlikely((ev[i].sub_event_type % nb_stages)
99 == laststage)) {
100 if (enable_fwd_latency)
101 cnt = perf_process_last_stage_latency(
102 pool, &ev[i], w, bufs, sz, cnt);
103 else
104 cnt = perf_process_last_stage(pool,
105 &ev[i], w, bufs, sz, cnt);
106
107 ev[i].op = RTE_EVENT_OP_RELEASE;
108 } else {
109 atq_fwd_event(&ev[i], sched_type_list,
110 nb_stages);
111 }
112 }
113
114 uint16_t enq;
115
116 enq = rte_event_enqueue_burst(dev, port, ev, nb_rx);
117 while (enq < nb_rx) {
118 enq += rte_event_enqueue_burst(dev, port,
119 ev + enq, nb_rx - enq);
120 }
121 }
122 return 0;
123}
124
125static int
126worker_wrapper(void *arg)
127{
128 struct worker_data *w = arg;
129 struct evt_options *opt = w->t->opt;
130
131 const bool burst = evt_has_burst_mode(w->dev_id);
132 const int fwd_latency = opt->fwd_latency;
133
134
135 if (!burst && !fwd_latency)
136 return perf_atq_worker(arg, 0);
137 else if (!burst && fwd_latency)
138 return perf_atq_worker(arg, 1);
139 else if (burst && !fwd_latency)
140 return perf_atq_worker_burst(arg, 0);
141 else if (burst && fwd_latency)
142 return perf_atq_worker_burst(arg, 1);
143
144 rte_panic("invalid worker\n");
145}
146
147static int
148perf_atq_launch_lcores(struct evt_test *test, struct evt_options *opt)
149{
150 return perf_launch_lcores(test, opt, worker_wrapper);
151}
152
153static int
154perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt)
155{
156 int ret;
157 uint8_t queue;
158 uint8_t nb_queues;
159 uint8_t nb_ports;
160 uint16_t prod;
161 struct rte_event_dev_info dev_info;
162 struct test_perf *t = evt_test_priv(test);
163
164 nb_ports = evt_nr_active_lcores(opt->wlcores);
165 nb_ports += (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR ||
166 opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) ? 0 :
167 evt_nr_active_lcores(opt->plcores);
168
169 nb_queues = atq_nb_event_queues(opt);
170
171 memset(&dev_info, 0, sizeof(struct rte_event_dev_info));
172 ret = rte_event_dev_info_get(opt->dev_id, &dev_info);
173 if (ret) {
174 evt_err("failed to get eventdev info %d", opt->dev_id);
175 return ret;
176 }
177
178 ret = evt_configure_eventdev(opt, nb_queues, nb_ports);
179 if (ret) {
180 evt_err("failed to configure eventdev %d", opt->dev_id);
181 return ret;
182 }
183
184 struct rte_event_queue_conf q_conf = {
185 .priority = RTE_EVENT_DEV_PRIORITY_NORMAL,
186 .event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES,
187 .nb_atomic_flows = opt->nb_flows,
188 .nb_atomic_order_sequences = opt->nb_flows,
189 };
190
191 for (queue = 0; queue < nb_queues; queue++) {
192 ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf);
193 if (ret) {
194 evt_err("failed to setup queue=%d", queue);
195 return ret;
196 }
197 }
198
199 if (opt->wkr_deq_dep > dev_info.max_event_port_dequeue_depth)
200 opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth;
201
202
203 const struct rte_event_port_conf p_conf = {
204 .dequeue_depth = opt->wkr_deq_dep,
205 .enqueue_depth = dev_info.max_event_port_dequeue_depth,
206 .new_event_threshold = dev_info.max_num_events,
207 };
208
209 ret = perf_event_dev_port_setup(test, opt, 1 , nb_queues,
210 &p_conf);
211 if (ret)
212 return ret;
213
214 if (!evt_has_distributed_sched(opt->dev_id)) {
215 uint32_t service_id;
216 rte_event_dev_service_id_get(opt->dev_id, &service_id);
217 ret = evt_service_setup(service_id);
218 if (ret) {
219 evt_err("No service lcore found to run event dev.");
220 return ret;
221 }
222 }
223
224 ret = rte_event_dev_start(opt->dev_id);
225 if (ret) {
226 evt_err("failed to start eventdev %d", opt->dev_id);
227 return ret;
228 }
229
230 if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) {
231 RTE_ETH_FOREACH_DEV(prod) {
232 ret = rte_eth_dev_start(prod);
233 if (ret) {
234 evt_err("Ethernet dev [%d] failed to start. Using synthetic producer",
235 prod);
236 return ret;
237 }
238
239 ret = rte_event_eth_rx_adapter_start(prod);
240 if (ret) {
241 evt_err("Rx adapter[%d] start failed", prod);
242 return ret;
243 }
244 printf("%s: Port[%d] using Rx adapter[%d] started\n",
245 __func__, prod, prod);
246 }
247 } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) {
248 for (prod = 0; prod < opt->nb_timer_adptrs; prod++) {
249 ret = rte_event_timer_adapter_start(
250 t->timer_adptr[prod]);
251 if (ret) {
252 evt_err("failed to Start event timer adapter %d"
253 , prod);
254 return ret;
255 }
256 }
257 }
258
259 return 0;
260}
261
262static void
263perf_atq_opt_dump(struct evt_options *opt)
264{
265 perf_opt_dump(opt, atq_nb_event_queues(opt));
266}
267
268static int
269perf_atq_opt_check(struct evt_options *opt)
270{
271 return perf_opt_check(opt, atq_nb_event_queues(opt));
272}
273
274static bool
275perf_atq_capability_check(struct evt_options *opt)
276{
277 struct rte_event_dev_info dev_info;
278
279 rte_event_dev_info_get(opt->dev_id, &dev_info);
280 if (dev_info.max_event_queues < atq_nb_event_queues(opt) ||
281 dev_info.max_event_ports < perf_nb_event_ports(opt)) {
282 evt_err("not enough eventdev queues=%d/%d or ports=%d/%d",
283 atq_nb_event_queues(opt), dev_info.max_event_queues,
284 perf_nb_event_ports(opt), dev_info.max_event_ports);
285 }
286 if (!evt_has_all_types_queue(opt->dev_id))
287 return false;
288
289 return true;
290}
291
292static const struct evt_test_ops perf_atq = {
293 .cap_check = perf_atq_capability_check,
294 .opt_check = perf_atq_opt_check,
295 .opt_dump = perf_atq_opt_dump,
296 .test_setup = perf_test_setup,
297 .ethdev_setup = perf_ethdev_setup,
298 .mempool_setup = perf_mempool_setup,
299 .eventdev_setup = perf_atq_eventdev_setup,
300 .launch_lcores = perf_atq_launch_lcores,
301 .eventdev_destroy = perf_eventdev_destroy,
302 .mempool_destroy = perf_mempool_destroy,
303 .ethdev_destroy = perf_ethdev_destroy,
304 .test_result = perf_test_result,
305 .test_destroy = perf_test_destroy,
306};
307
308EVT_TEST_REGISTER(perf_atq);
309
