1
2
3
4
5#include <rte_malloc.h>
6#include <rte_eal.h>
7#include <rte_log.h>
8#include <rte_compressdev.h>
9
10#include "comp_perf_test_verify.h"
11#include "comp_perf_test_common.h"
12
13void
14cperf_verify_test_destructor(void *arg)
15{
16 if (arg) {
17 comp_perf_free_memory(
18 ((struct cperf_verify_ctx *)arg)->options,
19 &((struct cperf_verify_ctx *)arg)->mem);
20 rte_free(arg);
21 }
22}
23
24void *
25cperf_verify_test_constructor(uint8_t dev_id, uint16_t qp_id,
26 struct comp_test_data *options)
27{
28 struct cperf_verify_ctx *ctx = NULL;
29
30 ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0);
31
32 if (ctx == NULL)
33 return NULL;
34
35 ctx->mem.dev_id = dev_id;
36 ctx->mem.qp_id = qp_id;
37 ctx->options = options;
38
39 if (!comp_perf_allocate_memory(ctx->options, &ctx->mem) &&
40 !prepare_bufs(ctx->options, &ctx->mem))
41 return ctx;
42
43 cperf_verify_test_destructor(ctx);
44 return NULL;
45}
46
47static int
48main_loop(struct cperf_verify_ctx *ctx, enum rte_comp_xform_type type)
49{
50 struct comp_test_data *test_data = ctx->options;
51 uint8_t *output_data_ptr = NULL;
52 size_t *output_data_sz = NULL;
53 struct cperf_mem_resources *mem = &ctx->mem;
54
55 uint8_t dev_id = mem->dev_id;
56 uint32_t i, iter, num_iter;
57 struct rte_comp_op **ops, **deq_ops;
58 void *priv_xform = NULL;
59 struct rte_comp_xform xform;
60 size_t output_size = 0;
61 struct rte_mbuf **input_bufs, **output_bufs;
62 int res = 0;
63 int allocated = 0;
64 uint32_t out_seg_sz;
65
66 if (test_data == NULL || !test_data->burst_sz) {
67 RTE_LOG(ERR, USER1,
68 "Unknown burst size\n");
69 return -1;
70 }
71
72 ops = rte_zmalloc_socket(NULL,
73 2 * mem->total_bufs * sizeof(struct rte_comp_op *),
74 0, rte_socket_id());
75
76 if (ops == NULL) {
77 RTE_LOG(ERR, USER1,
78 "Can't allocate memory for ops strucures\n");
79 return -1;
80 }
81
82 deq_ops = &ops[mem->total_bufs];
83
84 if (type == RTE_COMP_COMPRESS) {
85 xform = (struct rte_comp_xform) {
86 .type = RTE_COMP_COMPRESS,
87 .compress = {
88 .algo = RTE_COMP_ALGO_DEFLATE,
89 .deflate.huffman = test_data->huffman_enc,
90 .level = test_data->level,
91 .window_size = test_data->window_sz,
92 .chksum = RTE_COMP_CHECKSUM_NONE,
93 .hash_algo = RTE_COMP_HASH_ALGO_NONE
94 }
95 };
96 output_data_ptr = ctx->mem.compressed_data;
97 output_data_sz = &ctx->comp_data_sz;
98 input_bufs = mem->decomp_bufs;
99 output_bufs = mem->comp_bufs;
100 out_seg_sz = test_data->out_seg_sz;
101 } else {
102 xform = (struct rte_comp_xform) {
103 .type = RTE_COMP_DECOMPRESS,
104 .decompress = {
105 .algo = RTE_COMP_ALGO_DEFLATE,
106 .chksum = RTE_COMP_CHECKSUM_NONE,
107 .window_size = test_data->window_sz,
108 .hash_algo = RTE_COMP_HASH_ALGO_NONE
109 }
110 };
111 output_data_ptr = ctx->mem.decompressed_data;
112 output_data_sz = &ctx->decomp_data_sz;
113 input_bufs = mem->comp_bufs;
114 output_bufs = mem->decomp_bufs;
115 out_seg_sz = test_data->seg_sz;
116 }
117
118
119 if (rte_compressdev_private_xform_create(dev_id, &xform,
120 &priv_xform) < 0) {
121 RTE_LOG(ERR, USER1, "Private xform could not be created\n");
122 res = -1;
123 goto end;
124 }
125
126 num_iter = 1;
127
128 for (iter = 0; iter < num_iter; iter++) {
129 uint32_t total_ops = mem->total_bufs;
130 uint32_t remaining_ops = mem->total_bufs;
131 uint32_t total_deq_ops = 0;
132 uint32_t total_enq_ops = 0;
133 uint16_t ops_unused = 0;
134 uint16_t num_enq = 0;
135 uint16_t num_deq = 0;
136
137 output_size = 0;
138
139 while (remaining_ops > 0) {
140 uint16_t num_ops = RTE_MIN(remaining_ops,
141 test_data->burst_sz);
142 uint16_t ops_needed = num_ops - ops_unused;
143
144
145
146
147
148 if ((ops_unused > 0) && (num_enq > 0)) {
149 size_t nb_b_to_mov =
150 ops_unused * sizeof(struct rte_comp_op *);
151
152 memmove(ops, &ops[num_enq], nb_b_to_mov);
153 }
154
155
156 if (ops_needed && !rte_comp_op_bulk_alloc(
157 mem->op_pool,
158 &ops[ops_unused],
159 ops_needed)) {
160 RTE_LOG(ERR, USER1,
161 "Could not allocate enough operations\n");
162 res = -1;
163 goto end;
164 }
165 allocated += ops_needed;
166
167 for (i = 0; i < ops_needed; i++) {
168
169
170
171 uint32_t buf_id = total_enq_ops + i +
172 ops_unused;
173 uint16_t op_id = ops_unused + i;
174
175 struct rte_mbuf *m = output_bufs[buf_id];
176
177 m->pkt_len = out_seg_sz * m->nb_segs;
178 while (m) {
179 m->data_len = m->buf_len - m->data_off;
180 m = m->next;
181 }
182 ops[op_id]->m_src = input_bufs[buf_id];
183 ops[op_id]->m_dst = output_bufs[buf_id];
184 ops[op_id]->src.offset = 0;
185 ops[op_id]->src.length =
186 rte_pktmbuf_pkt_len(input_bufs[buf_id]);
187 ops[op_id]->dst.offset = 0;
188 ops[op_id]->flush_flag = RTE_COMP_FLUSH_FINAL;
189 ops[op_id]->input_chksum = buf_id;
190 ops[op_id]->private_xform = priv_xform;
191 }
192
193 if (unlikely(test_data->perf_comp_force_stop))
194 goto end;
195
196 num_enq = rte_compressdev_enqueue_burst(dev_id,
197 mem->qp_id, ops,
198 num_ops);
199 if (num_enq == 0) {
200 struct rte_compressdev_stats stats;
201
202 rte_compressdev_stats_get(dev_id, &stats);
203 if (stats.enqueue_err_count) {
204 res = -1;
205 goto end;
206 }
207 }
208
209 ops_unused = num_ops - num_enq;
210 remaining_ops -= num_enq;
211 total_enq_ops += num_enq;
212
213 num_deq = rte_compressdev_dequeue_burst(dev_id,
214 mem->qp_id,
215 deq_ops,
216 test_data->burst_sz);
217 total_deq_ops += num_deq;
218
219 for (i = 0; i < num_deq; i++) {
220 struct rte_comp_op *op = deq_ops[i];
221
222 if (op->status ==
223 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED ||
224 op->status ==
225 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
226 RTE_LOG(ERR, USER1,
227"Out of space error occurred due to uncompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n");
228 res = -1;
229 goto end;
230 } else if (op->status !=
231 RTE_COMP_OP_STATUS_SUCCESS) {
232 RTE_LOG(ERR, USER1,
233 "Some operations were not successful\n");
234 goto end;
235 }
236
237 const void *read_data_addr =
238 rte_pktmbuf_read(op->m_dst, 0,
239 op->produced, output_data_ptr);
240 if (read_data_addr == NULL) {
241 RTE_LOG(ERR, USER1,
242 "Could not copy buffer in destination\n");
243 res = -1;
244 goto end;
245 }
246
247 if (read_data_addr != output_data_ptr)
248 rte_memcpy(output_data_ptr,
249 rte_pktmbuf_mtod(op->m_dst,
250 uint8_t *),
251 op->produced);
252 output_data_ptr += op->produced;
253 output_size += op->produced;
254
255 }
256
257
258 if (iter == num_iter - 1) {
259 for (i = 0; i < num_deq; i++) {
260 struct rte_comp_op *op = deq_ops[i];
261 struct rte_mbuf *m = op->m_dst;
262
263 m->pkt_len = op->produced;
264 uint32_t remaining_data = op->produced;
265 uint16_t data_to_append;
266
267 while (remaining_data > 0) {
268 data_to_append =
269 RTE_MIN(remaining_data,
270 out_seg_sz);
271 m->data_len = data_to_append;
272 remaining_data -=
273 data_to_append;
274 m = m->next;
275 }
276 }
277 }
278 rte_mempool_put_bulk(mem->op_pool,
279 (void **)deq_ops, num_deq);
280 allocated -= num_deq;
281 }
282
283
284 while (total_deq_ops < total_ops) {
285 if (unlikely(test_data->perf_comp_force_stop))
286 goto end;
287
288 num_deq = rte_compressdev_dequeue_burst(dev_id,
289 mem->qp_id,
290 deq_ops,
291 test_data->burst_sz);
292 if (num_deq == 0) {
293 struct rte_compressdev_stats stats;
294
295 rte_compressdev_stats_get(dev_id, &stats);
296 if (stats.dequeue_err_count) {
297 res = -1;
298 goto end;
299 }
300 }
301
302 total_deq_ops += num_deq;
303
304 for (i = 0; i < num_deq; i++) {
305 struct rte_comp_op *op = deq_ops[i];
306
307 if (op->status ==
308 RTE_COMP_OP_STATUS_OUT_OF_SPACE_TERMINATED ||
309 op->status ==
310 RTE_COMP_OP_STATUS_OUT_OF_SPACE_RECOVERABLE) {
311 RTE_LOG(ERR, USER1,
312"Out of space error occurred due to uncompressible input data expanding to larger than destination buffer. Increase the EXPANSE_RATIO constant to use this data.\n");
313 res = -1;
314 goto end;
315 } else if (op->status !=
316 RTE_COMP_OP_STATUS_SUCCESS) {
317 RTE_LOG(ERR, USER1,
318 "Some operations were not successful\n");
319 goto end;
320 }
321 const void *read_data_addr =
322 rte_pktmbuf_read(op->m_dst,
323 op->dst.offset,
324 op->produced, output_data_ptr);
325 if (read_data_addr == NULL) {
326 RTE_LOG(ERR, USER1,
327 "Could not copy buffer in destination\n");
328 res = -1;
329 goto end;
330 }
331
332 if (read_data_addr != output_data_ptr)
333 rte_memcpy(output_data_ptr,
334 rte_pktmbuf_mtod(
335 op->m_dst, uint8_t *),
336 op->produced);
337 output_data_ptr += op->produced;
338 output_size += op->produced;
339
340 }
341
342 if (iter == num_iter - 1) {
343 for (i = 0; i < num_deq; i++) {
344 struct rte_comp_op *op = deq_ops[i];
345 struct rte_mbuf *m = op->m_dst;
346
347 m->pkt_len = op->produced;
348 uint32_t remaining_data = op->produced;
349 uint16_t data_to_append;
350
351 while (remaining_data > 0) {
352 data_to_append =
353 RTE_MIN(remaining_data,
354 out_seg_sz);
355 m->data_len = data_to_append;
356 remaining_data -=
357 data_to_append;
358 m = m->next;
359 }
360 }
361 }
362 rte_mempool_put_bulk(mem->op_pool,
363 (void **)deq_ops, num_deq);
364 allocated -= num_deq;
365 }
366 }
367
368 if (output_data_sz)
369 *output_data_sz = output_size;
370end:
371 rte_mempool_put_bulk(mem->op_pool, (void **)ops, allocated);
372 rte_compressdev_private_xform_free(dev_id, priv_xform);
373 rte_free(ops);
374
375 if (test_data->perf_comp_force_stop) {
376 RTE_LOG(ERR, USER1,
377 "lcore: %d Perf. test has been aborted by user\n",
378 mem->lcore_id);
379 res = -1;
380 }
381
382 return res;
383}
384
385int
386cperf_verify_test_runner(void *test_ctx)
387{
388 struct cperf_verify_ctx *ctx = test_ctx;
389 struct comp_test_data *test_data = ctx->options;
390 int ret = EXIT_SUCCESS;
391 static rte_atomic16_t display_once = RTE_ATOMIC16_INIT(0);
392 uint32_t lcore = rte_lcore_id();
393
394 ctx->mem.lcore_id = lcore;
395
396 test_data->ratio = 0;
397
398 if (main_loop(ctx, RTE_COMP_COMPRESS) < 0) {
399 ret = EXIT_FAILURE;
400 goto end;
401 }
402
403 if (main_loop(ctx, RTE_COMP_DECOMPRESS) < 0) {
404 ret = EXIT_FAILURE;
405 goto end;
406 }
407
408 if (ctx->decomp_data_sz != test_data->input_data_sz) {
409 RTE_LOG(ERR, USER1,
410 "Decompressed data length not equal to input data length\n");
411 RTE_LOG(ERR, USER1,
412 "Decompressed size = %zu, expected = %zu\n",
413 ctx->decomp_data_sz, test_data->input_data_sz);
414 ret = EXIT_FAILURE;
415 goto end;
416 } else {
417 if (memcmp(ctx->mem.decompressed_data,
418 test_data->input_data,
419 test_data->input_data_sz) != 0) {
420 RTE_LOG(ERR, USER1,
421 "Decompressed data is not the same as file data\n");
422 ret = EXIT_FAILURE;
423 goto end;
424 }
425 }
426
427 ctx->ratio = (double) ctx->comp_data_sz /
428 test_data->input_data_sz * 100;
429
430 if (!ctx->silent) {
431 if (rte_atomic16_test_and_set(&display_once)) {
432 printf("%12s%6s%12s%17s\n",
433 "lcore id", "Level", "Comp size", "Comp ratio [%]");
434 }
435 printf("%12u%6u%12zu%17.2f\n",
436 ctx->mem.lcore_id,
437 test_data->level, ctx->comp_data_sz, ctx->ratio);
438 }
439
440end:
441 return ret;
442}
443
