1
2
3
4
5#include <rte_malloc.h>
6#include <rte_cycles.h>
7#include <rte_crypto.h>
8#include <rte_cryptodev.h>
9
10#include "cperf_test_verify.h"
11#include "cperf_ops.h"
12#include "cperf_test_common.h"
13
14struct cperf_verify_ctx {
15 uint8_t dev_id;
16 uint16_t qp_id;
17 uint8_t lcore_id;
18
19 struct rte_mempool *pool;
20
21 struct rte_cryptodev_sym_session *sess;
22
23 cperf_populate_ops_t populate_ops;
24
25 uint32_t src_buf_offset;
26 uint32_t dst_buf_offset;
27
28 const struct cperf_options *options;
29 const struct cperf_test_vector *test_vector;
30};
31
32struct cperf_op_result {
33 enum rte_crypto_op_status status;
34};
35
36static void
37cperf_verify_test_free(struct cperf_verify_ctx *ctx)
38{
39 if (ctx) {
40 if (ctx->sess) {
41 rte_cryptodev_sym_session_clear(ctx->dev_id, ctx->sess);
42 rte_cryptodev_sym_session_free(ctx->sess);
43 }
44
45 if (ctx->pool)
46 rte_mempool_free(ctx->pool);
47
48 rte_free(ctx);
49 }
50}
51
52void *
53cperf_verify_test_constructor(struct rte_mempool *sess_mp,
54 struct rte_mempool *sess_priv_mp,
55 uint8_t dev_id, uint16_t qp_id,
56 const struct cperf_options *options,
57 const struct cperf_test_vector *test_vector,
58 const struct cperf_op_fns *op_fns)
59{
60 struct cperf_verify_ctx *ctx = NULL;
61
62 ctx = rte_malloc(NULL, sizeof(struct cperf_verify_ctx), 0);
63 if (ctx == NULL)
64 goto err;
65
66 ctx->dev_id = dev_id;
67 ctx->qp_id = qp_id;
68
69 ctx->populate_ops = op_fns->populate_ops;
70 ctx->options = options;
71 ctx->test_vector = test_vector;
72
73
74 uint16_t iv_offset = sizeof(struct rte_crypto_op) +
75 sizeof(struct rte_crypto_sym_op);
76
77 ctx->sess = op_fns->sess_create(sess_mp, sess_priv_mp, dev_id, options,
78 test_vector, iv_offset);
79 if (ctx->sess == NULL)
80 goto err;
81
82 if (cperf_alloc_common_memory(options, test_vector, dev_id, qp_id, 0,
83 &ctx->src_buf_offset, &ctx->dst_buf_offset,
84 &ctx->pool) < 0)
85 goto err;
86
87 return ctx;
88err:
89 cperf_verify_test_free(ctx);
90
91 return NULL;
92}
93
94static int
95cperf_verify_op(struct rte_crypto_op *op,
96 const struct cperf_options *options,
97 const struct cperf_test_vector *vector)
98{
99 const struct rte_mbuf *m;
100 uint32_t len;
101 uint16_t nb_segs;
102 uint8_t *data;
103 uint32_t cipher_offset, auth_offset;
104 uint8_t cipher, auth;
105 int res = 0;
106
107 if (op->status != RTE_CRYPTO_OP_STATUS_SUCCESS)
108 return 1;
109
110 if (op->sym->m_dst)
111 m = op->sym->m_dst;
112 else
113 m = op->sym->m_src;
114 nb_segs = m->nb_segs;
115 len = 0;
116 while (m && nb_segs != 0) {
117 len += m->data_len;
118 m = m->next;
119 nb_segs--;
120 }
121
122 data = rte_malloc(NULL, len, 0);
123 if (data == NULL)
124 return 1;
125
126 if (op->sym->m_dst)
127 m = op->sym->m_dst;
128 else
129 m = op->sym->m_src;
130 nb_segs = m->nb_segs;
131 len = 0;
132 while (m && nb_segs != 0) {
133 memcpy(data + len, rte_pktmbuf_mtod(m, uint8_t *),
134 m->data_len);
135 len += m->data_len;
136 m = m->next;
137 nb_segs--;
138 }
139
140 switch (options->op_type) {
141 case CPERF_CIPHER_ONLY:
142 cipher = 1;
143 cipher_offset = 0;
144 auth = 0;
145 auth_offset = 0;
146 break;
147 case CPERF_CIPHER_THEN_AUTH:
148 cipher = 1;
149 cipher_offset = 0;
150 auth = 1;
151 auth_offset = options->test_buffer_size;
152 break;
153 case CPERF_AUTH_ONLY:
154 cipher = 0;
155 cipher_offset = 0;
156 auth = 1;
157 auth_offset = options->test_buffer_size;
158 break;
159 case CPERF_AUTH_THEN_CIPHER:
160 cipher = 1;
161 cipher_offset = 0;
162 auth = 1;
163 auth_offset = options->test_buffer_size;
164 break;
165 case CPERF_AEAD:
166 cipher = 1;
167 cipher_offset = 0;
168 auth = 1;
169 auth_offset = options->test_buffer_size;
170 break;
171 default:
172 res = 1;
173 goto out;
174 }
175
176 if (cipher == 1) {
177 if (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
178 res += memcmp(data + cipher_offset,
179 vector->ciphertext.data,
180 options->test_buffer_size);
181 else
182 res += memcmp(data + cipher_offset,
183 vector->plaintext.data,
184 options->test_buffer_size);
185 }
186
187 if (auth == 1) {
188 if (options->auth_op == RTE_CRYPTO_AUTH_OP_GENERATE)
189 res += memcmp(data + auth_offset,
190 vector->digest.data,
191 options->digest_sz);
192 }
193
194out:
195 rte_free(data);
196 return !!res;
197}
198
199static void
200cperf_mbuf_set(struct rte_mbuf *mbuf,
201 const struct cperf_options *options,
202 const struct cperf_test_vector *test_vector)
203{
204 uint32_t segment_sz = options->segment_sz;
205 uint8_t *mbuf_data;
206 uint8_t *test_data;
207 uint32_t remaining_bytes = options->max_buffer_size;
208
209 if (options->op_type == CPERF_AEAD) {
210 test_data = (options->aead_op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
211 test_vector->plaintext.data :
212 test_vector->ciphertext.data;
213 } else {
214 test_data =
215 (options->cipher_op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
216 test_vector->plaintext.data :
217 test_vector->ciphertext.data;
218 }
219
220 while (remaining_bytes) {
221 mbuf_data = rte_pktmbuf_mtod(mbuf, uint8_t *);
222
223 if (remaining_bytes <= segment_sz) {
224 memcpy(mbuf_data, test_data, remaining_bytes);
225 return;
226 }
227
228 memcpy(mbuf_data, test_data, segment_sz);
229 remaining_bytes -= segment_sz;
230 test_data += segment_sz;
231 mbuf = mbuf->next;
232 }
233}
234
235int
236cperf_verify_test_runner(void *test_ctx)
237{
238 struct cperf_verify_ctx *ctx = test_ctx;
239
240 uint64_t ops_enqd = 0, ops_enqd_total = 0, ops_enqd_failed = 0;
241 uint64_t ops_deqd = 0, ops_deqd_total = 0, ops_deqd_failed = 0;
242 uint64_t ops_failed = 0;
243
244 static uint16_t display_once;
245
246 uint64_t i;
247 uint16_t ops_unused = 0;
248 uint32_t imix_idx = 0;
249
250 struct rte_crypto_op *ops[ctx->options->max_burst_size];
251 struct rte_crypto_op *ops_processed[ctx->options->max_burst_size];
252
253 uint32_t lcore = rte_lcore_id();
254
255#ifdef CPERF_LINEARIZATION_ENABLE
256 struct rte_cryptodev_info dev_info;
257 int linearize = 0;
258
259
260 if (ctx->options->segment_sz < ctx->options->max_buffer_size) {
261 rte_cryptodev_info_get(ctx->dev_id, &dev_info);
262 if ((dev_info.feature_flags &
263 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER) == 0)
264 linearize = 1;
265 }
266#endif
267
268 ctx->lcore_id = lcore;
269
270 if (!ctx->options->csv)
271 printf("\n# Running verify test on device: %u, lcore: %u\n",
272 ctx->dev_id, lcore);
273
274 uint16_t iv_offset = sizeof(struct rte_crypto_op) +
275 sizeof(struct rte_crypto_sym_op);
276
277 while (ops_enqd_total < ctx->options->total_ops) {
278
279 uint16_t burst_size = ((ops_enqd_total + ctx->options->max_burst_size)
280 <= ctx->options->total_ops) ?
281 ctx->options->max_burst_size :
282 ctx->options->total_ops -
283 ops_enqd_total;
284
285 uint16_t ops_needed = burst_size - ops_unused;
286
287
288 if (rte_mempool_get_bulk(ctx->pool, (void **)ops,
289 ops_needed) != 0) {
290 RTE_LOG(ERR, USER1,
291 "Failed to allocate more crypto operations "
292 "from the crypto operation pool.\n"
293 "Consider increasing the pool size "
294 "with --pool-sz\n");
295 return -1;
296 }
297
298
299 (ctx->populate_ops)(ops, ctx->src_buf_offset,
300 ctx->dst_buf_offset,
301 ops_needed, ctx->sess, ctx->options,
302 ctx->test_vector, iv_offset, &imix_idx, NULL);
303
304
305
306 for (i = 0; i < ops_needed; i++)
307 cperf_mbuf_set(ops[i]->sym->m_src,
308 ctx->options,
309 ctx->test_vector);
310
311#ifdef CPERF_LINEARIZATION_ENABLE
312 if (linearize) {
313
314
315
316
317 for (i = 0; i < burst_size; i++)
318 rte_pktmbuf_linearize(ops[i]->sym->m_src);
319 }
320#endif
321
322
323 ops_enqd = rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id,
324 ops, burst_size);
325 if (ops_enqd < burst_size)
326 ops_enqd_failed++;
327
328
329
330
331
332 ops_unused = burst_size - ops_enqd;
333 ops_enqd_total += ops_enqd;
334
335
336
337 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id,
338 ops_processed, ctx->options->max_burst_size);
339
340 if (ops_deqd == 0) {
341
342
343
344
345
346 ops_deqd_failed++;
347 continue;
348 }
349
350 for (i = 0; i < ops_deqd; i++) {
351 if (cperf_verify_op(ops_processed[i], ctx->options,
352 ctx->test_vector))
353 ops_failed++;
354 }
355
356 rte_mempool_put_bulk(ctx->pool,
357 (void **)ops_processed, ops_deqd);
358 ops_deqd_total += ops_deqd;
359 }
360
361
362
363 while (ops_deqd_total < ctx->options->total_ops) {
364
365 rte_cryptodev_enqueue_burst(ctx->dev_id, ctx->qp_id, NULL, 0);
366
367
368 ops_deqd = rte_cryptodev_dequeue_burst(ctx->dev_id, ctx->qp_id,
369 ops_processed, ctx->options->max_burst_size);
370 if (ops_deqd == 0) {
371 ops_deqd_failed++;
372 continue;
373 }
374
375 for (i = 0; i < ops_deqd; i++) {
376 if (cperf_verify_op(ops_processed[i], ctx->options,
377 ctx->test_vector))
378 ops_failed++;
379 }
380
381 rte_mempool_put_bulk(ctx->pool,
382 (void **)ops_processed, ops_deqd);
383 ops_deqd_total += ops_deqd;
384 }
385
386 uint16_t exp = 0;
387 if (!ctx->options->csv) {
388 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0,
389 __ATOMIC_RELAXED, __ATOMIC_RELAXED))
390 printf("%12s%12s%12s%12s%12s%12s%12s%12s\n\n",
391 "lcore id", "Buf Size", "Burst size",
392 "Enqueued", "Dequeued", "Failed Enq",
393 "Failed Deq", "Failed Ops");
394
395 printf("%12u%12u%12u%12"PRIu64"%12"PRIu64"%12"PRIu64
396 "%12"PRIu64"%12"PRIu64"\n",
397 ctx->lcore_id,
398 ctx->options->max_buffer_size,
399 ctx->options->max_burst_size,
400 ops_enqd_total,
401 ops_deqd_total,
402 ops_enqd_failed,
403 ops_deqd_failed,
404 ops_failed);
405 } else {
406 if (__atomic_compare_exchange_n(&display_once, &exp, 1, 0,
407 __ATOMIC_RELAXED, __ATOMIC_RELAXED))
408 printf("\n# lcore id, Buffer Size(B), "
409 "Burst Size,Enqueued,Dequeued,Failed Enq,"
410 "Failed Deq,Failed Ops\n");
411
412 printf("%10u,%10u,%u,%"PRIu64",%"PRIu64",%"PRIu64",%"PRIu64","
413 "%"PRIu64"\n",
414 ctx->lcore_id,
415 ctx->options->max_buffer_size,
416 ctx->options->max_burst_size,
417 ops_enqd_total,
418 ops_deqd_total,
419 ops_enqd_failed,
420 ops_deqd_failed,
421 ops_failed);
422 }
423
424 return 0;
425}
426
427
428
429void
430cperf_verify_test_destructor(void *arg)
431{
432 struct cperf_verify_ctx *ctx = arg;
433
434 if (ctx == NULL)
435 return;
436
437 cperf_verify_test_free(ctx);
438}
439