1
2
3
4
5#include <rte_common.h>
6#include <rte_hexdump.h>
7#include <rte_cryptodev.h>
8#include <rte_cryptodev_pmd.h>
9#include <rte_bus_vdev.h>
10#include <rte_malloc.h>
11#include <rte_cpuflags.h>
12
13#include "zuc_pmd_private.h"
14#define ZUC_MAX_BURST 16
15#define BYTE_LEN 8
16
17static uint8_t cryptodev_driver_id;
18
19
20static enum zuc_operation
21zuc_get_mode(const struct rte_crypto_sym_xform *xform)
22{
23 if (xform == NULL)
24 return ZUC_OP_NOT_SUPPORTED;
25
26 if (xform->next)
27 if (xform->next->next != NULL)
28 return ZUC_OP_NOT_SUPPORTED;
29
30 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
31 if (xform->next == NULL)
32 return ZUC_OP_ONLY_AUTH;
33 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
34 return ZUC_OP_AUTH_CIPHER;
35 else
36 return ZUC_OP_NOT_SUPPORTED;
37 }
38
39 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
40 if (xform->next == NULL)
41 return ZUC_OP_ONLY_CIPHER;
42 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
43 return ZUC_OP_CIPHER_AUTH;
44 else
45 return ZUC_OP_NOT_SUPPORTED;
46 }
47
48 return ZUC_OP_NOT_SUPPORTED;
49}
50
51
52
53int
54zuc_set_session_parameters(struct zuc_session *sess,
55 const struct rte_crypto_sym_xform *xform)
56{
57 const struct rte_crypto_sym_xform *auth_xform = NULL;
58 const struct rte_crypto_sym_xform *cipher_xform = NULL;
59 enum zuc_operation mode;
60
61
62 mode = zuc_get_mode(xform);
63
64 switch (mode) {
65 case ZUC_OP_CIPHER_AUTH:
66 auth_xform = xform->next;
67
68
69 case ZUC_OP_ONLY_CIPHER:
70 cipher_xform = xform;
71 break;
72 case ZUC_OP_AUTH_CIPHER:
73 cipher_xform = xform->next;
74
75 case ZUC_OP_ONLY_AUTH:
76 auth_xform = xform;
77 break;
78 case ZUC_OP_NOT_SUPPORTED:
79 default:
80 ZUC_LOG(ERR, "Unsupported operation chain order parameter");
81 return -ENOTSUP;
82 }
83
84 if (cipher_xform) {
85
86 if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_ZUC_EEA3)
87 return -ENOTSUP;
88
89 if (cipher_xform->cipher.iv.length != ZUC_IV_KEY_LENGTH) {
90 ZUC_LOG(ERR, "Wrong IV length");
91 return -EINVAL;
92 }
93 sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
94
95
96 memcpy(sess->pKey_cipher, cipher_xform->cipher.key.data,
97 ZUC_IV_KEY_LENGTH);
98 }
99
100 if (auth_xform) {
101
102 if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_ZUC_EIA3)
103 return -ENOTSUP;
104
105 if (auth_xform->auth.digest_length != ZUC_DIGEST_LENGTH) {
106 ZUC_LOG(ERR, "Wrong digest length");
107 return -EINVAL;
108 }
109
110 sess->auth_op = auth_xform->auth.op;
111
112 if (auth_xform->auth.iv.length != ZUC_IV_KEY_LENGTH) {
113 ZUC_LOG(ERR, "Wrong IV length");
114 return -EINVAL;
115 }
116 sess->auth_iv_offset = auth_xform->auth.iv.offset;
117
118
119 memcpy(sess->pKey_hash, auth_xform->auth.key.data,
120 ZUC_IV_KEY_LENGTH);
121 }
122
123
124 sess->op = mode;
125
126 return 0;
127}
128
129
130static struct zuc_session *
131zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op)
132{
133 struct zuc_session *sess = NULL;
134
135 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
136 if (likely(op->sym->session != NULL))
137 sess = (struct zuc_session *)get_sym_session_private_data(
138 op->sym->session,
139 cryptodev_driver_id);
140 } else {
141 void *_sess = NULL;
142 void *_sess_private_data = NULL;
143
144 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
145 return NULL;
146
147 if (rte_mempool_get(qp->sess_mp_priv,
148 (void **)&_sess_private_data))
149 return NULL;
150
151 sess = (struct zuc_session *)_sess_private_data;
152
153 if (unlikely(zuc_set_session_parameters(sess,
154 op->sym->xform) != 0)) {
155 rte_mempool_put(qp->sess_mp, _sess);
156 rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
157 sess = NULL;
158 }
159 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
160 set_sym_session_private_data(op->sym->session,
161 cryptodev_driver_id, _sess_private_data);
162 }
163
164 if (unlikely(sess == NULL))
165 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
166
167
168 return sess;
169}
170
171
172static uint8_t
173process_zuc_cipher_op(struct zuc_qp *qp, struct rte_crypto_op **ops,
174 struct zuc_session **sessions,
175 uint8_t num_ops)
176{
177 unsigned i;
178 uint8_t processed_ops = 0;
179 const void *src[ZUC_MAX_BURST];
180 void *dst[ZUC_MAX_BURST];
181 const void *iv[ZUC_MAX_BURST];
182 uint32_t num_bytes[ZUC_MAX_BURST];
183 const void *cipher_keys[ZUC_MAX_BURST];
184 struct zuc_session *sess;
185
186 for (i = 0; i < num_ops; i++) {
187 if (((ops[i]->sym->cipher.data.length % BYTE_LEN) != 0)
188 || ((ops[i]->sym->cipher.data.offset
189 % BYTE_LEN) != 0)) {
190 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
191 ZUC_LOG(ERR, "Data Length or offset");
192 break;
193 }
194
195 sess = sessions[i];
196
197#ifdef RTE_LIBRTE_PMD_ZUC_DEBUG
198 if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) ||
199 (ops[i]->sym->m_dst != NULL &&
200 !rte_pktmbuf_is_contiguous(
201 ops[i]->sym->m_dst))) {
202 ZUC_LOG(ERR, "PMD supports only contiguous mbufs, "
203 "op (%p) provides noncontiguous mbuf as "
204 "source/destination buffer.\n", ops[i]);
205 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
206 break;
207 }
208#endif
209
210 src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
211 (ops[i]->sym->cipher.data.offset >> 3);
212 dst[i] = ops[i]->sym->m_dst ?
213 rte_pktmbuf_mtod(ops[i]->sym->m_dst, uint8_t *) +
214 (ops[i]->sym->cipher.data.offset >> 3) :
215 rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
216 (ops[i]->sym->cipher.data.offset >> 3);
217 iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
218 sess->cipher_iv_offset);
219 num_bytes[i] = ops[i]->sym->cipher.data.length >> 3;
220
221 cipher_keys[i] = sess->pKey_cipher;
222
223 processed_ops++;
224 }
225
226 IMB_ZUC_EEA3_N_BUFFER(qp->mb_mgr, (const void **)cipher_keys,
227 (const void **)iv, (const void **)src, (void **)dst,
228 num_bytes, processed_ops);
229
230 return processed_ops;
231}
232
233
234static int
235process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops,
236 struct zuc_session **sessions,
237 uint8_t num_ops)
238{
239 unsigned int i;
240 uint8_t processed_ops = 0;
241 uint8_t *src[ZUC_MAX_BURST];
242 uint32_t *dst[ZUC_MAX_BURST];
243 uint32_t length_in_bits[ZUC_MAX_BURST];
244 uint8_t *iv[ZUC_MAX_BURST];
245 const void *hash_keys[ZUC_MAX_BURST];
246 struct zuc_session *sess;
247
248 for (i = 0; i < num_ops; i++) {
249
250 if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
251 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
252 ZUC_LOG(ERR, "Offset");
253 break;
254 }
255
256 sess = sessions[i];
257
258 length_in_bits[i] = ops[i]->sym->auth.data.length;
259
260 src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
261 (ops[i]->sym->auth.data.offset >> 3);
262 iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
263 sess->auth_iv_offset);
264
265 hash_keys[i] = sess->pKey_hash;
266 if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
267 dst[i] = (uint32_t *)qp->temp_digest;
268 else
269 dst[i] = (uint32_t *)ops[i]->sym->auth.digest.data;
270
271#if IMB_VERSION_NUM < IMB_VERSION(0, 53, 3)
272 IMB_ZUC_EIA3_1_BUFFER(qp->mb_mgr, hash_keys[i],
273 iv[i], src[i], length_in_bits[i], dst[i]);
274#endif
275 processed_ops++;
276 }
277
278#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
279 IMB_ZUC_EIA3_N_BUFFER(qp->mb_mgr, (const void **)hash_keys,
280 (const void * const *)iv, (const void * const *)src,
281 length_in_bits, dst, processed_ops);
282#endif
283
284
285
286
287
288 for (i = 0; i < processed_ops; i++)
289 if (sessions[i]->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
290 if (memcmp(dst[i], ops[i]->sym->auth.digest.data,
291 ZUC_DIGEST_LENGTH) != 0)
292 ops[i]->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
293
294 return processed_ops;
295}
296
297
298static int
299process_ops(struct rte_crypto_op **ops, enum zuc_operation op_type,
300 struct zuc_session **sessions,
301 struct zuc_qp *qp, uint8_t num_ops,
302 uint16_t *accumulated_enqueued_ops)
303{
304 unsigned i;
305 unsigned enqueued_ops, processed_ops;
306
307 switch (op_type) {
308 case ZUC_OP_ONLY_CIPHER:
309 processed_ops = process_zuc_cipher_op(qp, ops,
310 sessions, num_ops);
311 break;
312 case ZUC_OP_ONLY_AUTH:
313 processed_ops = process_zuc_hash_op(qp, ops, sessions,
314 num_ops);
315 break;
316 case ZUC_OP_CIPHER_AUTH:
317 processed_ops = process_zuc_cipher_op(qp, ops, sessions,
318 num_ops);
319 process_zuc_hash_op(qp, ops, sessions, processed_ops);
320 break;
321 case ZUC_OP_AUTH_CIPHER:
322 processed_ops = process_zuc_hash_op(qp, ops, sessions,
323 num_ops);
324 process_zuc_cipher_op(qp, ops, sessions, processed_ops);
325 break;
326 default:
327
328 processed_ops = 0;
329 }
330
331 for (i = 0; i < num_ops; i++) {
332
333
334
335
336 if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
337 ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
338
339 if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
340 memset(sessions[i], 0, sizeof(struct zuc_session));
341 memset(ops[i]->sym->session, 0,
342 rte_cryptodev_sym_get_existing_header_session_size(
343 ops[i]->sym->session));
344 rte_mempool_put(qp->sess_mp_priv, sessions[i]);
345 rte_mempool_put(qp->sess_mp, ops[i]->sym->session);
346 ops[i]->sym->session = NULL;
347 }
348 }
349
350 enqueued_ops = rte_ring_enqueue_burst(qp->processed_ops,
351 (void **)ops, processed_ops, NULL);
352 qp->qp_stats.enqueued_count += enqueued_ops;
353 *accumulated_enqueued_ops += enqueued_ops;
354
355 return enqueued_ops;
356}
357
358static uint16_t
359zuc_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
360 uint16_t nb_ops)
361{
362 struct rte_crypto_op *c_ops[ZUC_MAX_BURST];
363 struct rte_crypto_op *curr_c_op;
364
365 struct zuc_session *curr_sess;
366 struct zuc_session *sessions[ZUC_MAX_BURST];
367 enum zuc_operation prev_zuc_op = ZUC_OP_NOT_SUPPORTED;
368 enum zuc_operation curr_zuc_op;
369 struct zuc_qp *qp = queue_pair;
370 unsigned i;
371 uint8_t burst_size = 0;
372 uint16_t enqueued_ops = 0;
373 uint8_t processed_ops;
374
375 for (i = 0; i < nb_ops; i++) {
376 curr_c_op = ops[i];
377
378 curr_sess = zuc_get_session(qp, curr_c_op);
379 if (unlikely(curr_sess == NULL)) {
380 curr_c_op->status =
381 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
382 break;
383 }
384
385 curr_zuc_op = curr_sess->op;
386
387
388
389
390
391 if (burst_size == 0) {
392 prev_zuc_op = curr_zuc_op;
393 c_ops[0] = curr_c_op;
394 sessions[0] = curr_sess;
395 burst_size++;
396 } else if (curr_zuc_op == prev_zuc_op) {
397 c_ops[burst_size] = curr_c_op;
398 sessions[burst_size] = curr_sess;
399 burst_size++;
400
401
402
403
404 if (burst_size == ZUC_MAX_BURST) {
405 processed_ops = process_ops(c_ops, curr_zuc_op,
406 sessions, qp, burst_size,
407 &enqueued_ops);
408 if (processed_ops < burst_size) {
409 burst_size = 0;
410 break;
411 }
412
413 burst_size = 0;
414 }
415 } else {
416
417
418
419
420 processed_ops = process_ops(c_ops, prev_zuc_op,
421 sessions, qp, burst_size,
422 &enqueued_ops);
423 if (processed_ops < burst_size) {
424 burst_size = 0;
425 break;
426 }
427
428 burst_size = 0;
429 prev_zuc_op = curr_zuc_op;
430
431 c_ops[0] = curr_c_op;
432 sessions[0] = curr_sess;
433 burst_size++;
434 }
435 }
436
437 if (burst_size != 0) {
438
439 processed_ops = process_ops(c_ops, prev_zuc_op,
440 sessions, qp, burst_size,
441 &enqueued_ops);
442 }
443
444 qp->qp_stats.enqueue_err_count += nb_ops - enqueued_ops;
445 return enqueued_ops;
446}
447
448static uint16_t
449zuc_pmd_dequeue_burst(void *queue_pair,
450 struct rte_crypto_op **c_ops, uint16_t nb_ops)
451{
452 struct zuc_qp *qp = queue_pair;
453
454 unsigned nb_dequeued;
455
456 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
457 (void **)c_ops, nb_ops, NULL);
458 qp->qp_stats.dequeued_count += nb_dequeued;
459
460 return nb_dequeued;
461}
462
463static int cryptodev_zuc_remove(struct rte_vdev_device *vdev);
464
465static int
466cryptodev_zuc_create(const char *name,
467 struct rte_vdev_device *vdev,
468 struct rte_cryptodev_pmd_init_params *init_params)
469{
470 struct rte_cryptodev *dev;
471 struct zuc_private *internals;
472 MB_MGR *mb_mgr;
473
474 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
475 if (dev == NULL) {
476 ZUC_LOG(ERR, "failed to create cryptodev vdev");
477 goto init_error;
478 }
479
480 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
481 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
482 RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
483 RTE_CRYPTODEV_FF_SYM_SESSIONLESS |
484 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
485
486 mb_mgr = alloc_mb_mgr(0);
487 if (mb_mgr == NULL)
488 return -ENOMEM;
489
490 if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F)) {
491 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512;
492 init_mb_mgr_avx512(mb_mgr);
493 } else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) {
494 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
495 init_mb_mgr_avx2(mb_mgr);
496 } else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX)) {
497 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
498 init_mb_mgr_avx(mb_mgr);
499 } else {
500 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
501 init_mb_mgr_sse(mb_mgr);
502 }
503
504 dev->driver_id = cryptodev_driver_id;
505 dev->dev_ops = rte_zuc_pmd_ops;
506
507
508 dev->dequeue_burst = zuc_pmd_dequeue_burst;
509 dev->enqueue_burst = zuc_pmd_enqueue_burst;
510
511 internals = dev->data->dev_private;
512 internals->mb_mgr = mb_mgr;
513
514 internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
515
516 return 0;
517init_error:
518 ZUC_LOG(ERR, "driver %s: failed",
519 init_params->name);
520
521 cryptodev_zuc_remove(vdev);
522 return -EFAULT;
523}
524
525static int
526cryptodev_zuc_probe(struct rte_vdev_device *vdev)
527{
528 struct rte_cryptodev_pmd_init_params init_params = {
529 "",
530 sizeof(struct zuc_private),
531 rte_socket_id(),
532 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
533 };
534 const char *name;
535 const char *input_args;
536
537 name = rte_vdev_device_name(vdev);
538 if (name == NULL)
539 return -EINVAL;
540 input_args = rte_vdev_device_args(vdev);
541
542 rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
543
544 return cryptodev_zuc_create(name, vdev, &init_params);
545}
546
547static int
548cryptodev_zuc_remove(struct rte_vdev_device *vdev)
549{
550
551 struct rte_cryptodev *cryptodev;
552 const char *name;
553 struct zuc_private *internals;
554
555 name = rte_vdev_device_name(vdev);
556 if (name == NULL)
557 return -EINVAL;
558
559 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
560 if (cryptodev == NULL)
561 return -ENODEV;
562
563 internals = cryptodev->data->dev_private;
564
565 free_mb_mgr(internals->mb_mgr);
566
567 return rte_cryptodev_pmd_destroy(cryptodev);
568}
569
570static struct rte_vdev_driver cryptodev_zuc_pmd_drv = {
571 .probe = cryptodev_zuc_probe,
572 .remove = cryptodev_zuc_remove
573};
574
575static struct cryptodev_driver zuc_crypto_drv;
576
577RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ZUC_PMD, cryptodev_zuc_pmd_drv);
578RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD,
579 "max_nb_queue_pairs=<int> "
580 "socket_id=<int>");
581RTE_PMD_REGISTER_CRYPTO_DRIVER(zuc_crypto_drv, cryptodev_zuc_pmd_drv.driver,
582 cryptodev_driver_id);
583RTE_LOG_REGISTER(zuc_logtype_driver, pmd.crypto.zuc, INFO);
584
