1
2
3
4
5#include <rte_common.h>
6#include <rte_hexdump.h>
7#include <rte_cryptodev.h>
8#include <cryptodev_pmd.h>
9#include <rte_bus_vdev.h>
10#include <rte_malloc.h>
11#include <rte_cpuflags.h>
12
13#include <openssl/hmac.h>
14#include <openssl/evp.h>
15
16#include "openssl_pmd_private.h"
17#include "compat.h"
18
19#define DES_BLOCK_SIZE 8
20
21static uint8_t cryptodev_driver_id;
22
23#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
24static HMAC_CTX *HMAC_CTX_new(void)
25{
26 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
27
28 if (ctx != NULL)
29 HMAC_CTX_init(ctx);
30 return ctx;
31}
32
33static void HMAC_CTX_free(HMAC_CTX *ctx)
34{
35 if (ctx != NULL) {
36 HMAC_CTX_cleanup(ctx);
37 OPENSSL_free(ctx);
38 }
39}
40#endif
41
42static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
43
44
45
46
47
48
49
50static void
51ctr_inc(uint8_t *ctr)
52{
53 uint64_t *ctr64 = (uint64_t *)ctr;
54
55 *ctr64 = __builtin_bswap64(*ctr64);
56 (*ctr64)++;
57 *ctr64 = __builtin_bswap64(*ctr64);
58}
59
60
61
62
63
64
65
66
67static enum openssl_chain_order
68openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
69{
70 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
71
72 if (xform != NULL) {
73 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
74 if (xform->next == NULL)
75 res = OPENSSL_CHAIN_ONLY_AUTH;
76 else if (xform->next->type ==
77 RTE_CRYPTO_SYM_XFORM_CIPHER)
78 res = OPENSSL_CHAIN_AUTH_CIPHER;
79 }
80 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
81 if (xform->next == NULL)
82 res = OPENSSL_CHAIN_ONLY_CIPHER;
83 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
84 res = OPENSSL_CHAIN_CIPHER_AUTH;
85 }
86 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
87 res = OPENSSL_CHAIN_COMBINED;
88 }
89
90 return res;
91}
92
93
94static void
95get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key)
96{
97 memcpy(session_key, input_key, keylen);
98}
99
100
101static int
102get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede)
103{
104 int res = 0;
105
106
107 switch (keylen) {
108 case 24:
109 memcpy(key_ede, key, 24);
110 break;
111 case 16:
112
113 memcpy(key_ede, key, 16);
114 memcpy(key_ede + 16, key, 8);
115 break;
116 case 8:
117
118 memcpy(key_ede, key, 8);
119 memcpy(key_ede + 8, key, 8);
120 memcpy(key_ede + 16, key, 8);
121 break;
122 default:
123 OPENSSL_LOG(ERR, "Unsupported key size");
124 res = -EINVAL;
125 }
126
127 return res;
128}
129
130
131static uint8_t
132get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
133 const EVP_CIPHER **algo)
134{
135 int res = 0;
136
137 if (algo != NULL) {
138 switch (sess_algo) {
139 case RTE_CRYPTO_CIPHER_3DES_CBC:
140 switch (keylen) {
141 case 8:
142 *algo = EVP_des_cbc();
143 break;
144 case 16:
145 *algo = EVP_des_ede_cbc();
146 break;
147 case 24:
148 *algo = EVP_des_ede3_cbc();
149 break;
150 default:
151 res = -EINVAL;
152 }
153 break;
154 case RTE_CRYPTO_CIPHER_3DES_CTR:
155 break;
156 case RTE_CRYPTO_CIPHER_AES_CBC:
157 switch (keylen) {
158 case 16:
159 *algo = EVP_aes_128_cbc();
160 break;
161 case 24:
162 *algo = EVP_aes_192_cbc();
163 break;
164 case 32:
165 *algo = EVP_aes_256_cbc();
166 break;
167 default:
168 res = -EINVAL;
169 }
170 break;
171 case RTE_CRYPTO_CIPHER_AES_CTR:
172 switch (keylen) {
173 case 16:
174 *algo = EVP_aes_128_ctr();
175 break;
176 case 24:
177 *algo = EVP_aes_192_ctr();
178 break;
179 case 32:
180 *algo = EVP_aes_256_ctr();
181 break;
182 default:
183 res = -EINVAL;
184 }
185 break;
186 default:
187 res = -EINVAL;
188 break;
189 }
190 } else {
191 res = -EINVAL;
192 }
193
194 return res;
195}
196
197
198static uint8_t
199get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
200 const EVP_MD **algo)
201{
202 int res = 0;
203
204 if (algo != NULL) {
205 switch (sessalgo) {
206 case RTE_CRYPTO_AUTH_MD5:
207 case RTE_CRYPTO_AUTH_MD5_HMAC:
208 *algo = EVP_md5();
209 break;
210 case RTE_CRYPTO_AUTH_SHA1:
211 case RTE_CRYPTO_AUTH_SHA1_HMAC:
212 *algo = EVP_sha1();
213 break;
214 case RTE_CRYPTO_AUTH_SHA224:
215 case RTE_CRYPTO_AUTH_SHA224_HMAC:
216 *algo = EVP_sha224();
217 break;
218 case RTE_CRYPTO_AUTH_SHA256:
219 case RTE_CRYPTO_AUTH_SHA256_HMAC:
220 *algo = EVP_sha256();
221 break;
222 case RTE_CRYPTO_AUTH_SHA384:
223 case RTE_CRYPTO_AUTH_SHA384_HMAC:
224 *algo = EVP_sha384();
225 break;
226 case RTE_CRYPTO_AUTH_SHA512:
227 case RTE_CRYPTO_AUTH_SHA512_HMAC:
228 *algo = EVP_sha512();
229 break;
230 default:
231 res = -EINVAL;
232 break;
233 }
234 } else {
235 res = -EINVAL;
236 }
237
238 return res;
239}
240
241
242static uint8_t
243get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
244 const EVP_CIPHER **algo)
245{
246 int res = 0;
247
248 if (algo != NULL) {
249 switch (sess_algo) {
250 case RTE_CRYPTO_AEAD_AES_GCM:
251 switch (keylen) {
252 case 16:
253 *algo = EVP_aes_128_gcm();
254 break;
255 case 24:
256 *algo = EVP_aes_192_gcm();
257 break;
258 case 32:
259 *algo = EVP_aes_256_gcm();
260 break;
261 default:
262 res = -EINVAL;
263 }
264 break;
265 case RTE_CRYPTO_AEAD_AES_CCM:
266 switch (keylen) {
267 case 16:
268 *algo = EVP_aes_128_ccm();
269 break;
270 case 24:
271 *algo = EVP_aes_192_ccm();
272 break;
273 case 32:
274 *algo = EVP_aes_256_ccm();
275 break;
276 default:
277 res = -EINVAL;
278 }
279 break;
280 default:
281 res = -EINVAL;
282 break;
283 }
284 } else {
285 res = -EINVAL;
286 }
287
288 return res;
289}
290
291
292static int
293openssl_set_sess_aead_enc_param(struct openssl_session *sess,
294 enum rte_crypto_aead_algorithm algo,
295 uint8_t tag_len, const uint8_t *key)
296{
297 int iv_type = 0;
298 unsigned int do_ccm;
299
300 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
301 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
302
303
304 switch (algo) {
305 case RTE_CRYPTO_AEAD_AES_GCM:
306 iv_type = EVP_CTRL_GCM_SET_IVLEN;
307 if (tag_len != 16)
308 return -EINVAL;
309 do_ccm = 0;
310 break;
311 case RTE_CRYPTO_AEAD_AES_CCM:
312 iv_type = EVP_CTRL_CCM_SET_IVLEN;
313
314 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
315 return -EINVAL;
316 do_ccm = 1;
317 break;
318 default:
319 return -ENOTSUP;
320 }
321
322 sess->cipher.mode = OPENSSL_CIPHER_LIB;
323 sess->cipher.ctx = EVP_CIPHER_CTX_new();
324
325 if (get_aead_algo(algo, sess->cipher.key.length,
326 &sess->cipher.evp_algo) != 0)
327 return -EINVAL;
328
329 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
330
331 sess->chain_order = OPENSSL_CHAIN_COMBINED;
332
333 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
334 NULL, NULL, NULL) <= 0)
335 return -EINVAL;
336
337 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
338 NULL) <= 0)
339 return -EINVAL;
340
341 if (do_ccm)
342 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
343 tag_len, NULL);
344
345 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
346 return -EINVAL;
347
348 return 0;
349}
350
351
352static int
353openssl_set_sess_aead_dec_param(struct openssl_session *sess,
354 enum rte_crypto_aead_algorithm algo,
355 uint8_t tag_len, const uint8_t *key)
356{
357 int iv_type = 0;
358 unsigned int do_ccm = 0;
359
360 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
361 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
362
363
364 switch (algo) {
365 case RTE_CRYPTO_AEAD_AES_GCM:
366 iv_type = EVP_CTRL_GCM_SET_IVLEN;
367 if (tag_len != 16)
368 return -EINVAL;
369 break;
370 case RTE_CRYPTO_AEAD_AES_CCM:
371 iv_type = EVP_CTRL_CCM_SET_IVLEN;
372
373 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
374 return -EINVAL;
375 do_ccm = 1;
376 break;
377 default:
378 return -ENOTSUP;
379 }
380
381 sess->cipher.mode = OPENSSL_CIPHER_LIB;
382 sess->cipher.ctx = EVP_CIPHER_CTX_new();
383
384 if (get_aead_algo(algo, sess->cipher.key.length,
385 &sess->cipher.evp_algo) != 0)
386 return -EINVAL;
387
388 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
389
390 sess->chain_order = OPENSSL_CHAIN_COMBINED;
391
392 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
393 NULL, NULL, NULL) <= 0)
394 return -EINVAL;
395
396 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
397 sess->iv.length, NULL) <= 0)
398 return -EINVAL;
399
400 if (do_ccm)
401 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
402 tag_len, NULL);
403
404 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
405 return -EINVAL;
406
407 return 0;
408}
409
410
411static int
412openssl_set_session_cipher_parameters(struct openssl_session *sess,
413 const struct rte_crypto_sym_xform *xform)
414{
415
416 sess->cipher.direction = xform->cipher.op;
417
418 sess->cipher.key.length = xform->cipher.key.length;
419
420
421 sess->iv.offset = xform->cipher.iv.offset;
422 sess->iv.length = xform->cipher.iv.length;
423
424
425 switch (xform->cipher.algo) {
426 case RTE_CRYPTO_CIPHER_3DES_CBC:
427 case RTE_CRYPTO_CIPHER_AES_CBC:
428 case RTE_CRYPTO_CIPHER_AES_CTR:
429 sess->cipher.mode = OPENSSL_CIPHER_LIB;
430 sess->cipher.algo = xform->cipher.algo;
431 sess->cipher.ctx = EVP_CIPHER_CTX_new();
432
433 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
434 &sess->cipher.evp_algo) != 0)
435 return -EINVAL;
436
437 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
438 sess->cipher.key.data);
439 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
440 if (EVP_EncryptInit_ex(sess->cipher.ctx,
441 sess->cipher.evp_algo,
442 NULL, xform->cipher.key.data,
443 NULL) != 1) {
444 return -EINVAL;
445 }
446 } else if (sess->cipher.direction ==
447 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
448 if (EVP_DecryptInit_ex(sess->cipher.ctx,
449 sess->cipher.evp_algo,
450 NULL, xform->cipher.key.data,
451 NULL) != 1) {
452 return -EINVAL;
453 }
454 }
455
456 break;
457
458 case RTE_CRYPTO_CIPHER_3DES_CTR:
459 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
460 sess->cipher.ctx = EVP_CIPHER_CTX_new();
461
462 if (get_cipher_key_ede(xform->cipher.key.data,
463 sess->cipher.key.length,
464 sess->cipher.key.data) != 0)
465 return -EINVAL;
466 break;
467
468 case RTE_CRYPTO_CIPHER_DES_CBC:
469 sess->cipher.algo = xform->cipher.algo;
470 sess->cipher.ctx = EVP_CIPHER_CTX_new();
471 sess->cipher.evp_algo = EVP_des_cbc();
472
473 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
474 sess->cipher.key.data);
475 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
476 if (EVP_EncryptInit_ex(sess->cipher.ctx,
477 sess->cipher.evp_algo,
478 NULL, xform->cipher.key.data,
479 NULL) != 1) {
480 return -EINVAL;
481 }
482 } else if (sess->cipher.direction ==
483 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
484 if (EVP_DecryptInit_ex(sess->cipher.ctx,
485 sess->cipher.evp_algo,
486 NULL, xform->cipher.key.data,
487 NULL) != 1) {
488 return -EINVAL;
489 }
490 }
491
492 break;
493
494 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
495 sess->cipher.algo = xform->cipher.algo;
496 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
497 sess->cipher.ctx = EVP_CIPHER_CTX_new();
498 sess->cipher.evp_algo = EVP_des_cbc();
499
500 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
501
502 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
503 NULL, xform->cipher.key.data, 0) != 1)
504 return -EINVAL;
505
506 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
507 sess->cipher.key.data);
508 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
509 if (EVP_EncryptInit_ex(sess->cipher.ctx,
510 sess->cipher.evp_algo,
511 NULL, xform->cipher.key.data,
512 NULL) != 1) {
513 return -EINVAL;
514 }
515 } else if (sess->cipher.direction ==
516 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
517 if (EVP_DecryptInit_ex(sess->cipher.ctx,
518 sess->cipher.evp_algo,
519 NULL, xform->cipher.key.data,
520 NULL) != 1) {
521 return -EINVAL;
522 }
523 }
524
525 break;
526 default:
527 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
528 return -ENOTSUP;
529 }
530
531 return 0;
532}
533
534
535static int
536openssl_set_session_auth_parameters(struct openssl_session *sess,
537 const struct rte_crypto_sym_xform *xform)
538{
539
540 sess->auth.operation = xform->auth.op;
541 sess->auth.algo = xform->auth.algo;
542
543 sess->auth.digest_length = xform->auth.digest_length;
544
545
546 switch (xform->auth.algo) {
547 case RTE_CRYPTO_AUTH_AES_GMAC:
548
549
550
551
552 sess->cipher.key.length = xform->auth.key.length;
553
554
555 sess->iv.offset = xform->auth.iv.offset;
556 sess->iv.length = xform->auth.iv.length;
557
558 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
559 return openssl_set_sess_aead_enc_param(sess,
560 RTE_CRYPTO_AEAD_AES_GCM,
561 xform->auth.digest_length,
562 xform->auth.key.data);
563 else
564 return openssl_set_sess_aead_dec_param(sess,
565 RTE_CRYPTO_AEAD_AES_GCM,
566 xform->auth.digest_length,
567 xform->auth.key.data);
568 break;
569
570 case RTE_CRYPTO_AUTH_MD5:
571 case RTE_CRYPTO_AUTH_SHA1:
572 case RTE_CRYPTO_AUTH_SHA224:
573 case RTE_CRYPTO_AUTH_SHA256:
574 case RTE_CRYPTO_AUTH_SHA384:
575 case RTE_CRYPTO_AUTH_SHA512:
576 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
577 if (get_auth_algo(xform->auth.algo,
578 &sess->auth.auth.evp_algo) != 0)
579 return -EINVAL;
580 sess->auth.auth.ctx = EVP_MD_CTX_create();
581 break;
582
583 case RTE_CRYPTO_AUTH_MD5_HMAC:
584 case RTE_CRYPTO_AUTH_SHA1_HMAC:
585 case RTE_CRYPTO_AUTH_SHA224_HMAC:
586 case RTE_CRYPTO_AUTH_SHA256_HMAC:
587 case RTE_CRYPTO_AUTH_SHA384_HMAC:
588 case RTE_CRYPTO_AUTH_SHA512_HMAC:
589 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
590 sess->auth.hmac.ctx = HMAC_CTX_new();
591 if (get_auth_algo(xform->auth.algo,
592 &sess->auth.hmac.evp_algo) != 0)
593 return -EINVAL;
594
595 if (HMAC_Init_ex(sess->auth.hmac.ctx,
596 xform->auth.key.data,
597 xform->auth.key.length,
598 sess->auth.hmac.evp_algo, NULL) != 1)
599 return -EINVAL;
600 break;
601
602 default:
603 return -ENOTSUP;
604 }
605
606 return 0;
607}
608
609
610static int
611openssl_set_session_aead_parameters(struct openssl_session *sess,
612 const struct rte_crypto_sym_xform *xform)
613{
614
615 sess->cipher.key.length = xform->aead.key.length;
616
617
618 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
619
620
621
622
623
624 sess->iv.offset = xform->aead.iv.offset + 1;
625 else
626 sess->iv.offset = xform->aead.iv.offset;
627
628 sess->iv.length = xform->aead.iv.length;
629
630 sess->auth.aad_length = xform->aead.aad_length;
631 sess->auth.digest_length = xform->aead.digest_length;
632
633 sess->aead_algo = xform->aead.algo;
634
635 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
636 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
637 xform->aead.digest_length, xform->aead.key.data);
638 else
639 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
640 xform->aead.digest_length, xform->aead.key.data);
641}
642
643
644int
645openssl_set_session_parameters(struct openssl_session *sess,
646 const struct rte_crypto_sym_xform *xform)
647{
648 const struct rte_crypto_sym_xform *cipher_xform = NULL;
649 const struct rte_crypto_sym_xform *auth_xform = NULL;
650 const struct rte_crypto_sym_xform *aead_xform = NULL;
651 int ret;
652
653 sess->chain_order = openssl_get_chain_order(xform);
654 switch (sess->chain_order) {
655 case OPENSSL_CHAIN_ONLY_CIPHER:
656 cipher_xform = xform;
657 break;
658 case OPENSSL_CHAIN_ONLY_AUTH:
659 auth_xform = xform;
660 break;
661 case OPENSSL_CHAIN_CIPHER_AUTH:
662 cipher_xform = xform;
663 auth_xform = xform->next;
664 break;
665 case OPENSSL_CHAIN_AUTH_CIPHER:
666 auth_xform = xform;
667 cipher_xform = xform->next;
668 break;
669 case OPENSSL_CHAIN_COMBINED:
670 aead_xform = xform;
671 break;
672 default:
673 return -EINVAL;
674 }
675
676
677 sess->iv.length = 0;
678
679
680 if (cipher_xform) {
681 ret = openssl_set_session_cipher_parameters(
682 sess, cipher_xform);
683 if (ret != 0) {
684 OPENSSL_LOG(ERR,
685 "Invalid/unsupported cipher parameters");
686 return ret;
687 }
688 }
689
690 if (auth_xform) {
691 ret = openssl_set_session_auth_parameters(sess, auth_xform);
692 if (ret != 0) {
693 OPENSSL_LOG(ERR,
694 "Invalid/unsupported auth parameters");
695 return ret;
696 }
697 }
698
699 if (aead_xform) {
700 ret = openssl_set_session_aead_parameters(sess, aead_xform);
701 if (ret != 0) {
702 OPENSSL_LOG(ERR,
703 "Invalid/unsupported AEAD parameters");
704 return ret;
705 }
706 }
707
708 return 0;
709}
710
711
712void
713openssl_reset_session(struct openssl_session *sess)
714{
715 EVP_CIPHER_CTX_free(sess->cipher.ctx);
716
717 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
718 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
719
720 switch (sess->auth.mode) {
721 case OPENSSL_AUTH_AS_AUTH:
722 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
723 break;
724 case OPENSSL_AUTH_AS_HMAC:
725 EVP_PKEY_free(sess->auth.hmac.pkey);
726 HMAC_CTX_free(sess->auth.hmac.ctx);
727 break;
728 default:
729 break;
730 }
731}
732
733
734static void *
735get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
736{
737 struct openssl_session *sess = NULL;
738 struct openssl_asym_session *asym_sess = NULL;
739
740 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
741 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
742
743 if (likely(op->sym->session != NULL))
744 sess = (struct openssl_session *)
745 get_sym_session_private_data(
746 op->sym->session,
747 cryptodev_driver_id);
748 } else {
749 if (likely(op->asym->session != NULL))
750 asym_sess = (struct openssl_asym_session *)
751 get_asym_session_private_data(
752 op->asym->session,
753 cryptodev_driver_id);
754 if (asym_sess == NULL)
755 op->status =
756 RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
757 return asym_sess;
758 }
759 } else {
760
761 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)
762 return NULL;
763
764
765 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
766 void *_sess_private_data = NULL;
767
768 if (_sess == NULL)
769 return NULL;
770
771 if (rte_mempool_get(qp->sess_mp_priv,
772 (void **)&_sess_private_data))
773 return NULL;
774
775 sess = (struct openssl_session *)_sess_private_data;
776
777 if (unlikely(openssl_set_session_parameters(sess,
778 op->sym->xform) != 0)) {
779 rte_mempool_put(qp->sess_mp, _sess);
780 rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
781 sess = NULL;
782 }
783 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
784 set_sym_session_private_data(op->sym->session,
785 cryptodev_driver_id, _sess_private_data);
786 }
787
788 if (sess == NULL)
789 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
790
791 return sess;
792}
793
794
795
796
797
798
799static inline int
800process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
801 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
802{
803 struct rte_mbuf *m;
804 int dstlen;
805 int l, n = srclen;
806 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
807
808 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
809 m = m->next)
810 offset -= rte_pktmbuf_data_len(m);
811
812 if (m == 0)
813 return -1;
814
815 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
816 if (inplace)
817 *dst = src;
818
819 l = rte_pktmbuf_data_len(m) - offset;
820 if (srclen <= l) {
821 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
822 return -1;
823 *dst += l;
824 return 0;
825 }
826
827 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
828 return -1;
829
830 *dst += dstlen;
831 n -= l;
832
833 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
834 uint8_t diff = l - dstlen, rem;
835
836 src = rte_pktmbuf_mtod(m, uint8_t *);
837 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
838 if (diff && inplace) {
839 rem = RTE_MIN(l,
840 (EVP_CIPHER_CTX_block_size(ctx) - diff));
841 if (EVP_EncryptUpdate(ctx, temp,
842 &dstlen, src, rem) <= 0)
843 return -1;
844 n -= rem;
845 rte_memcpy(*dst, temp, diff);
846 rte_memcpy(src, temp + diff, rem);
847 src += rem;
848 l -= rem;
849 }
850 if (inplace)
851 *dst = src;
852 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
853 return -1;
854 *dst += dstlen;
855 n -= l;
856 }
857
858 return 0;
859}
860
861static inline int
862process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
863 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace)
864{
865 struct rte_mbuf *m;
866 int dstlen;
867 int l, n = srclen;
868 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)];
869
870 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
871 m = m->next)
872 offset -= rte_pktmbuf_data_len(m);
873
874 if (m == 0)
875 return -1;
876
877 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
878 if (inplace)
879 *dst = src;
880
881 l = rte_pktmbuf_data_len(m) - offset;
882 if (srclen <= l) {
883 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
884 return -1;
885 *dst += l;
886 return 0;
887 }
888
889 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
890 return -1;
891
892 *dst += dstlen;
893 n -= l;
894
895 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
896 uint8_t diff = l - dstlen, rem;
897
898 src = rte_pktmbuf_mtod(m, uint8_t *);
899 l = RTE_MIN(rte_pktmbuf_data_len(m), n);
900 if (diff && inplace) {
901 rem = RTE_MIN(l,
902 (EVP_CIPHER_CTX_block_size(ctx) - diff));
903 if (EVP_DecryptUpdate(ctx, temp,
904 &dstlen, src, rem) <= 0)
905 return -1;
906 n -= rem;
907 rte_memcpy(*dst, temp, diff);
908 rte_memcpy(src, temp + diff, rem);
909 src += rem;
910 l -= rem;
911 }
912 if (inplace)
913 *dst = src;
914 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
915 return -1;
916 *dst += dstlen;
917 n -= l;
918 }
919
920 return 0;
921}
922
923
924static int
925process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
926 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
927 uint8_t inplace)
928{
929 int totlen;
930
931 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
932 goto process_cipher_encrypt_err;
933
934 EVP_CIPHER_CTX_set_padding(ctx, 0);
935
936 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
937 srclen, ctx, inplace))
938 goto process_cipher_encrypt_err;
939
940 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
941 goto process_cipher_encrypt_err;
942
943 return 0;
944
945process_cipher_encrypt_err:
946 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed");
947 return -EINVAL;
948}
949
950
951static int
952process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
953 uint8_t *iv, int srclen,
954 EVP_CIPHER_CTX *ctx)
955{
956 uint8_t i;
957 uint8_t encrypted_iv[DES_BLOCK_SIZE];
958 int encrypted_ivlen;
959
960 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
961 iv, DES_BLOCK_SIZE) <= 0)
962 goto process_cipher_encrypt_err;
963
964 for (i = 0; i < srclen; i++)
965 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
966
967 return 0;
968
969process_cipher_encrypt_err:
970 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed");
971 return -EINVAL;
972}
973
974static int
975process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
976 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx,
977 uint8_t inplace)
978{
979 int totlen;
980
981 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
982 goto process_cipher_decrypt_err;
983
984 EVP_CIPHER_CTX_set_padding(ctx, 0);
985
986 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
987 srclen, ctx, inplace))
988 goto process_cipher_decrypt_err;
989
990 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
991 goto process_cipher_decrypt_err;
992 return 0;
993
994process_cipher_decrypt_err:
995 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed");
996 return -EINVAL;
997}
998
999
1000static int
1001process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
1002 int offset, uint8_t *iv, uint8_t *key, int srclen,
1003 EVP_CIPHER_CTX *ctx)
1004{
1005 uint8_t ebuf[8], ctr[8];
1006 int unused, n;
1007 struct rte_mbuf *m;
1008 uint8_t *src;
1009 int l;
1010
1011 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1012 m = m->next)
1013 offset -= rte_pktmbuf_data_len(m);
1014
1015 if (m == 0)
1016 goto process_cipher_des3ctr_err;
1017
1018 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1019 l = rte_pktmbuf_data_len(m) - offset;
1020
1021
1022
1023
1024 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
1025 goto process_cipher_des3ctr_err;
1026
1027 memcpy(ctr, iv, 8);
1028
1029 for (n = 0; n < srclen; n++) {
1030 if (n % 8 == 0) {
1031 if (EVP_EncryptUpdate(ctx,
1032 (unsigned char *)&ebuf, &unused,
1033 (const unsigned char *)&ctr, 8) <= 0)
1034 goto process_cipher_des3ctr_err;
1035 ctr_inc(ctr);
1036 }
1037 dst[n] = *(src++) ^ ebuf[n % 8];
1038
1039 l--;
1040 if (!l) {
1041 m = m->next;
1042 if (m) {
1043 src = rte_pktmbuf_mtod(m, uint8_t *);
1044 l = rte_pktmbuf_data_len(m);
1045 }
1046 }
1047 }
1048
1049 return 0;
1050
1051process_cipher_des3ctr_err:
1052 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed");
1053 return -EINVAL;
1054}
1055
1056
1057static int
1058process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1059 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1060 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1061{
1062 int len = 0, unused = 0;
1063 uint8_t empty[] = {};
1064
1065 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1066 goto process_auth_encryption_gcm_err;
1067
1068 if (aadlen > 0)
1069 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1070 goto process_auth_encryption_gcm_err;
1071
1072 if (srclen > 0)
1073 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1074 srclen, ctx, 0))
1075 goto process_auth_encryption_gcm_err;
1076
1077
1078 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1079 goto process_auth_encryption_gcm_err;
1080
1081 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1082 goto process_auth_encryption_gcm_err;
1083
1084 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1085 goto process_auth_encryption_gcm_err;
1086
1087 return 0;
1088
1089process_auth_encryption_gcm_err:
1090 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed");
1091 return -EINVAL;
1092}
1093
1094
1095static int
1096process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1097 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1098 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1099{
1100 int len = 0;
1101
1102 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1103 goto process_auth_encryption_ccm_err;
1104
1105 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1106 goto process_auth_encryption_ccm_err;
1107
1108 if (aadlen > 0)
1109
1110
1111
1112
1113
1114 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1115 goto process_auth_encryption_ccm_err;
1116
1117 if (srclen >= 0)
1118 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1119 srclen, ctx, 0))
1120 goto process_auth_encryption_ccm_err;
1121
1122 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1123 goto process_auth_encryption_ccm_err;
1124
1125 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1126 goto process_auth_encryption_ccm_err;
1127
1128 return 0;
1129
1130process_auth_encryption_ccm_err:
1131 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed");
1132 return -EINVAL;
1133}
1134
1135
1136static int
1137process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1138 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1139 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1140{
1141 int len = 0, unused = 0;
1142 uint8_t empty[] = {};
1143
1144 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1145 goto process_auth_decryption_gcm_err;
1146
1147 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1148 goto process_auth_decryption_gcm_err;
1149
1150 if (aadlen > 0)
1151 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1152 goto process_auth_decryption_gcm_err;
1153
1154 if (srclen > 0)
1155 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1156 srclen, ctx, 0))
1157 goto process_auth_decryption_gcm_err;
1158
1159
1160 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1161 goto process_auth_decryption_gcm_err;
1162
1163 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1164 return -EFAULT;
1165
1166 return 0;
1167
1168process_auth_decryption_gcm_err:
1169 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed");
1170 return -EINVAL;
1171}
1172
1173
1174static int
1175process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1176 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1177 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1178 EVP_CIPHER_CTX *ctx)
1179{
1180 int len = 0;
1181
1182 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1183 goto process_auth_decryption_ccm_err;
1184
1185 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1186 goto process_auth_decryption_ccm_err;
1187
1188 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1189 goto process_auth_decryption_ccm_err;
1190
1191 if (aadlen > 0)
1192
1193
1194
1195
1196
1197 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1198 goto process_auth_decryption_ccm_err;
1199
1200 if (srclen >= 0)
1201 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1202 srclen, ctx, 0))
1203 return -EFAULT;
1204
1205 return 0;
1206
1207process_auth_decryption_ccm_err:
1208 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed");
1209 return -EINVAL;
1210}
1211
1212
1213static int
1214process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1215 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1216 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1217{
1218 size_t dstlen;
1219 struct rte_mbuf *m;
1220 int l, n = srclen;
1221 uint8_t *src;
1222
1223 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1224 m = m->next)
1225 offset -= rte_pktmbuf_data_len(m);
1226
1227 if (m == 0)
1228 goto process_auth_err;
1229
1230 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1231 goto process_auth_err;
1232
1233 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1234
1235 l = rte_pktmbuf_data_len(m) - offset;
1236 if (srclen <= l) {
1237 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1238 goto process_auth_err;
1239 goto process_auth_final;
1240 }
1241
1242 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1243 goto process_auth_err;
1244
1245 n -= l;
1246
1247 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1248 src = rte_pktmbuf_mtod(m, uint8_t *);
1249 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1250 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1251 goto process_auth_err;
1252 n -= l;
1253 }
1254
1255process_auth_final:
1256 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1257 goto process_auth_err;
1258 return 0;
1259
1260process_auth_err:
1261 OPENSSL_LOG(ERR, "Process openssl auth failed");
1262 return -EINVAL;
1263}
1264
1265
1266static int
1267process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1268 int srclen, HMAC_CTX *ctx)
1269{
1270 unsigned int dstlen;
1271 struct rte_mbuf *m;
1272 int l, n = srclen;
1273 uint8_t *src;
1274
1275 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1276 m = m->next)
1277 offset -= rte_pktmbuf_data_len(m);
1278
1279 if (m == 0)
1280 goto process_auth_err;
1281
1282 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1283
1284 l = rte_pktmbuf_data_len(m) - offset;
1285 if (srclen <= l) {
1286 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1287 goto process_auth_err;
1288 goto process_auth_final;
1289 }
1290
1291 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1292 goto process_auth_err;
1293
1294 n -= l;
1295
1296 for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1297 src = rte_pktmbuf_mtod(m, uint8_t *);
1298 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1299 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1300 goto process_auth_err;
1301 n -= l;
1302 }
1303
1304process_auth_final:
1305 if (HMAC_Final(ctx, dst, &dstlen) != 1)
1306 goto process_auth_err;
1307
1308 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1309 goto process_auth_err;
1310
1311 return 0;
1312
1313process_auth_err:
1314 OPENSSL_LOG(ERR, "Process openssl auth failed");
1315 return -EINVAL;
1316}
1317
1318
1319
1320
1321static void
1322process_openssl_combined_op
1323 (struct rte_crypto_op *op, struct openssl_session *sess,
1324 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1325{
1326
1327 uint8_t *dst = NULL, *iv, *tag, *aad;
1328 int srclen, aadlen, status = -1;
1329 uint32_t offset;
1330 uint8_t taglen;
1331 EVP_CIPHER_CTX *ctx_copy;
1332
1333
1334
1335
1336
1337 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1338 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1339 return;
1340 }
1341
1342 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1343 sess->iv.offset);
1344 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1345 srclen = 0;
1346 offset = op->sym->auth.data.offset;
1347 aadlen = op->sym->auth.data.length;
1348 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1349 op->sym->auth.data.offset);
1350 tag = op->sym->auth.digest.data;
1351 if (tag == NULL)
1352 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1353 offset + aadlen);
1354 } else {
1355 srclen = op->sym->aead.data.length;
1356 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1357 op->sym->aead.data.offset);
1358 offset = op->sym->aead.data.offset;
1359 aad = op->sym->aead.aad.data;
1360 aadlen = sess->auth.aad_length;
1361 tag = op->sym->aead.digest.data;
1362 if (tag == NULL)
1363 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1364 offset + srclen);
1365 }
1366
1367 taglen = sess->auth.digest_length;
1368 ctx_copy = EVP_CIPHER_CTX_new();
1369 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx);
1370
1371 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1372 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1373 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1374 status = process_openssl_auth_encryption_gcm(
1375 mbuf_src, offset, srclen,
1376 aad, aadlen, iv,
1377 dst, tag, ctx_copy);
1378 else
1379 status = process_openssl_auth_encryption_ccm(
1380 mbuf_src, offset, srclen,
1381 aad, aadlen, iv,
1382 dst, tag, taglen, ctx_copy);
1383
1384 } else {
1385 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1386 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1387 status = process_openssl_auth_decryption_gcm(
1388 mbuf_src, offset, srclen,
1389 aad, aadlen, iv,
1390 dst, tag, ctx_copy);
1391 else
1392 status = process_openssl_auth_decryption_ccm(
1393 mbuf_src, offset, srclen,
1394 aad, aadlen, iv,
1395 dst, tag, taglen, ctx_copy);
1396 }
1397
1398 EVP_CIPHER_CTX_free(ctx_copy);
1399 if (status != 0) {
1400 if (status == (-EFAULT) &&
1401 sess->auth.operation ==
1402 RTE_CRYPTO_AUTH_OP_VERIFY)
1403 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1404 else
1405 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1406 }
1407}
1408
1409
1410static void
1411process_openssl_cipher_op
1412 (struct rte_crypto_op *op, struct openssl_session *sess,
1413 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1414{
1415 uint8_t *dst, *iv;
1416 int srclen, status;
1417 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0;
1418 EVP_CIPHER_CTX *ctx_copy;
1419
1420
1421
1422
1423
1424
1425 if (!rte_pktmbuf_is_contiguous(mbuf_dst) &&
1426 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) {
1427 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1428 return;
1429 }
1430
1431 srclen = op->sym->cipher.data.length;
1432 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1433 op->sym->cipher.data.offset);
1434
1435 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1436 sess->iv.offset);
1437 ctx_copy = EVP_CIPHER_CTX_new();
1438 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx);
1439
1440 if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1441 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1442 status = process_openssl_cipher_encrypt(mbuf_src, dst,
1443 op->sym->cipher.data.offset, iv,
1444 srclen, ctx_copy, inplace);
1445 else
1446 status = process_openssl_cipher_decrypt(mbuf_src, dst,
1447 op->sym->cipher.data.offset, iv,
1448 srclen, ctx_copy, inplace);
1449 else
1450 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1451 op->sym->cipher.data.offset, iv,
1452 sess->cipher.key.data, srclen,
1453 ctx_copy);
1454
1455 EVP_CIPHER_CTX_free(ctx_copy);
1456 if (status != 0)
1457 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1458}
1459
1460
1461static void
1462process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1463 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1464 struct rte_mbuf *mbuf_dst)
1465{
1466 uint8_t *src, *dst, *iv;
1467 uint8_t block_size, last_block_len;
1468 int srclen, status = 0;
1469
1470 srclen = op->sym->cipher.data.length;
1471 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1472 op->sym->cipher.data.offset);
1473 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1474 op->sym->cipher.data.offset);
1475
1476 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1477 sess->iv.offset);
1478
1479 block_size = DES_BLOCK_SIZE;
1480
1481 last_block_len = srclen % block_size;
1482 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1483
1484 if (srclen < block_size) {
1485 status = process_openssl_cipher_bpi_encrypt(src, dst,
1486 iv, srclen,
1487 sess->cipher.bpi_ctx);
1488 } else {
1489 srclen -= last_block_len;
1490
1491 status = process_openssl_cipher_encrypt(mbuf_src, dst,
1492 op->sym->cipher.data.offset, iv,
1493 srclen, sess->cipher.ctx, 0);
1494 if (last_block_len) {
1495
1496 dst += srclen;
1497
1498
1499
1500
1501 iv = dst - block_size;
1502 src += srclen;
1503 srclen = last_block_len;
1504
1505 status |= process_openssl_cipher_bpi_encrypt(src,
1506 dst, iv,
1507 srclen, sess->cipher.bpi_ctx);
1508 }
1509 }
1510 } else {
1511
1512 if (srclen < block_size) {
1513 status = process_openssl_cipher_bpi_encrypt(src, dst,
1514 iv,
1515 srclen,
1516 sess->cipher.bpi_ctx);
1517 } else {
1518 if (last_block_len) {
1519
1520 dst += srclen - last_block_len;
1521 src += srclen - last_block_len;
1522
1523
1524
1525 iv = src - block_size;
1526
1527
1528
1529
1530 status = process_openssl_cipher_bpi_encrypt(src,
1531 dst, iv,
1532 last_block_len, sess->cipher.bpi_ctx);
1533
1534 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1535 sess->iv.offset);
1536 dst += last_block_len - srclen;
1537 srclen -= last_block_len;
1538 }
1539
1540
1541 status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1542 op->sym->cipher.data.offset, iv,
1543 srclen, sess->cipher.ctx, 0);
1544 }
1545 }
1546
1547 if (status != 0)
1548 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1549}
1550
1551
1552static void
1553process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1554 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1555 struct rte_mbuf *mbuf_dst)
1556{
1557 uint8_t *dst;
1558 int srclen, status;
1559 EVP_MD_CTX *ctx_a;
1560 HMAC_CTX *ctx_h;
1561
1562 srclen = op->sym->auth.data.length;
1563
1564 dst = qp->temp_digest;
1565
1566 switch (sess->auth.mode) {
1567 case OPENSSL_AUTH_AS_AUTH:
1568 ctx_a = EVP_MD_CTX_create();
1569 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx);
1570 status = process_openssl_auth(mbuf_src, dst,
1571 op->sym->auth.data.offset, NULL, NULL, srclen,
1572 ctx_a, sess->auth.auth.evp_algo);
1573 EVP_MD_CTX_destroy(ctx_a);
1574 break;
1575 case OPENSSL_AUTH_AS_HMAC:
1576 ctx_h = HMAC_CTX_new();
1577 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx);
1578 status = process_openssl_auth_hmac(mbuf_src, dst,
1579 op->sym->auth.data.offset, srclen,
1580 ctx_h);
1581 HMAC_CTX_free(ctx_h);
1582 break;
1583 default:
1584 status = -1;
1585 break;
1586 }
1587
1588 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1589 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data,
1590 sess->auth.digest_length) != 0) {
1591 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1592 }
1593 } else {
1594 uint8_t *auth_dst;
1595
1596 auth_dst = op->sym->auth.digest.data;
1597 if (auth_dst == NULL)
1598 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1599 op->sym->auth.data.offset +
1600 op->sym->auth.data.length);
1601 memcpy(auth_dst, dst, sess->auth.digest_length);
1602 }
1603
1604 if (status != 0)
1605 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1606}
1607
1608
1609static int
1610process_openssl_dsa_sign_op(struct rte_crypto_op *cop,
1611 struct openssl_asym_session *sess)
1612{
1613 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1614 DSA *dsa = sess->u.s.dsa;
1615 DSA_SIG *sign = NULL;
1616
1617 sign = DSA_do_sign(op->message.data,
1618 op->message.length,
1619 dsa);
1620
1621 if (sign == NULL) {
1622 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__);
1623 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1624 } else {
1625 const BIGNUM *r = NULL, *s = NULL;
1626 get_dsa_sign(sign, &r, &s);
1627
1628 op->r.length = BN_bn2bin(r, op->r.data);
1629 op->s.length = BN_bn2bin(s, op->s.data);
1630 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1631 }
1632
1633 DSA_SIG_free(sign);
1634
1635 return 0;
1636}
1637
1638
1639static int
1640process_openssl_dsa_verify_op(struct rte_crypto_op *cop,
1641 struct openssl_asym_session *sess)
1642{
1643 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa;
1644 DSA *dsa = sess->u.s.dsa;
1645 int ret;
1646 DSA_SIG *sign = DSA_SIG_new();
1647 BIGNUM *r = NULL, *s = NULL;
1648 BIGNUM *pub_key = NULL;
1649
1650 if (sign == NULL) {
1651 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__);
1652 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1653 return -1;
1654 }
1655
1656 r = BN_bin2bn(op->r.data,
1657 op->r.length,
1658 r);
1659 s = BN_bin2bn(op->s.data,
1660 op->s.length,
1661 s);
1662 pub_key = BN_bin2bn(op->y.data,
1663 op->y.length,
1664 pub_key);
1665 if (!r || !s || !pub_key) {
1666 BN_free(r);
1667 BN_free(s);
1668 BN_free(pub_key);
1669
1670 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1671 return -1;
1672 }
1673 set_dsa_sign(sign, r, s);
1674 set_dsa_pub_key(dsa, pub_key);
1675
1676 ret = DSA_do_verify(op->message.data,
1677 op->message.length,
1678 sign,
1679 dsa);
1680
1681 if (ret != 1)
1682 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1683 else
1684 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1685
1686 DSA_SIG_free(sign);
1687
1688 return 0;
1689}
1690
1691
1692static int
1693process_openssl_dh_op(struct rte_crypto_op *cop,
1694 struct openssl_asym_session *sess)
1695{
1696 struct rte_crypto_dh_op_param *op = &cop->asym->dh;
1697 DH *dh_key = sess->u.dh.dh_key;
1698 BIGNUM *priv_key = NULL;
1699 int ret = 0;
1700
1701 if (sess->u.dh.key_op &
1702 (1 << RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE)) {
1703
1704
1705
1706
1707 BIGNUM *peer_key = NULL;
1708
1709
1710 peer_key = BN_bin2bn(op->pub_key.data,
1711 op->pub_key.length,
1712 peer_key);
1713 if (peer_key == NULL) {
1714 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1715 return -1;
1716 }
1717 priv_key = BN_bin2bn(op->priv_key.data,
1718 op->priv_key.length,
1719 priv_key);
1720 if (priv_key == NULL) {
1721 BN_free(peer_key);
1722 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1723 return -1;
1724 }
1725 ret = set_dh_priv_key(dh_key, priv_key);
1726 if (ret) {
1727 OPENSSL_LOG(ERR, "Failed to set private key\n");
1728 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1729 BN_free(peer_key);
1730 BN_free(priv_key);
1731 return 0;
1732 }
1733
1734 ret = DH_compute_key(
1735 op->shared_secret.data,
1736 peer_key, dh_key);
1737 if (ret < 0) {
1738 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1739 BN_free(peer_key);
1740
1741
1742
1743
1744 return 0;
1745 }
1746 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1747 op->shared_secret.length = ret;
1748 BN_free(peer_key);
1749 return 0;
1750 }
1751
1752
1753
1754
1755
1756
1757
1758 if ((sess->u.dh.key_op &
1759 (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) &&
1760 !(sess->u.dh.key_op &
1761 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE))) {
1762
1763
1764
1765
1766
1767 priv_key = BN_bin2bn(op->priv_key.data,
1768 op->priv_key.length,
1769 priv_key);
1770 if (priv_key == NULL) {
1771 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1772 return -1;
1773 }
1774 ret = set_dh_priv_key(dh_key, priv_key);
1775 if (ret) {
1776 OPENSSL_LOG(ERR, "Failed to set private key\n");
1777 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1778 BN_free(priv_key);
1779 return 0;
1780 }
1781 }
1782
1783
1784
1785
1786
1787
1788
1789
1790 if (!DH_generate_key(dh_key)) {
1791 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1792 return 0;
1793 }
1794
1795 if (sess->u.dh.key_op & (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) {
1796 const BIGNUM *pub_key = NULL;
1797
1798 OPENSSL_LOG(DEBUG, "%s:%d update public key\n",
1799 __func__, __LINE__);
1800
1801
1802 get_dh_pub_key(dh_key, &pub_key);
1803
1804
1805 op->pub_key.length = BN_bn2bin(pub_key,
1806 op->pub_key.data);
1807 }
1808
1809 if (sess->u.dh.key_op &
1810 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE)) {
1811 const BIGNUM *priv_key = NULL;
1812
1813 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n",
1814 __func__, __LINE__);
1815
1816
1817 get_dh_priv_key(dh_key, &priv_key);
1818
1819
1820 op->priv_key.length = BN_bn2bin(priv_key,
1821 op->priv_key.data);
1822 }
1823
1824 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1825
1826 return 0;
1827}
1828
1829
1830static int
1831process_openssl_modinv_op(struct rte_crypto_op *cop,
1832 struct openssl_asym_session *sess)
1833{
1834 struct rte_crypto_asym_op *op = cop->asym;
1835 BIGNUM *base = BN_CTX_get(sess->u.m.ctx);
1836 BIGNUM *res = BN_CTX_get(sess->u.m.ctx);
1837
1838 if (unlikely(base == NULL || res == NULL)) {
1839 BN_free(base);
1840 BN_free(res);
1841 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1842 return -1;
1843 }
1844
1845 base = BN_bin2bn((const unsigned char *)op->modinv.base.data,
1846 op->modinv.base.length, base);
1847
1848 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) {
1849 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1850 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data);
1851 } else {
1852 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1853 }
1854
1855 BN_clear(res);
1856 BN_clear(base);
1857
1858 return 0;
1859}
1860
1861
1862static int
1863process_openssl_modexp_op(struct rte_crypto_op *cop,
1864 struct openssl_asym_session *sess)
1865{
1866 struct rte_crypto_asym_op *op = cop->asym;
1867 BIGNUM *base = BN_CTX_get(sess->u.e.ctx);
1868 BIGNUM *res = BN_CTX_get(sess->u.e.ctx);
1869
1870 if (unlikely(base == NULL || res == NULL)) {
1871 BN_free(base);
1872 BN_free(res);
1873 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1874 return -1;
1875 }
1876
1877 base = BN_bin2bn((const unsigned char *)op->modex.base.data,
1878 op->modex.base.length, base);
1879
1880 if (BN_mod_exp(res, base, sess->u.e.exp,
1881 sess->u.e.mod, sess->u.e.ctx)) {
1882 op->modex.result.length = BN_bn2bin(res, op->modex.result.data);
1883 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1884 } else {
1885 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1886 }
1887
1888 BN_clear(res);
1889 BN_clear(base);
1890
1891 return 0;
1892}
1893
1894
1895static int
1896process_openssl_rsa_op(struct rte_crypto_op *cop,
1897 struct openssl_asym_session *sess)
1898{
1899 int ret = 0;
1900 struct rte_crypto_asym_op *op = cop->asym;
1901 RSA *rsa = sess->u.r.rsa;
1902 uint32_t pad = (op->rsa.pad);
1903 uint8_t *tmp;
1904
1905 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1906
1907 switch (pad) {
1908 case RTE_CRYPTO_RSA_PADDING_PKCS1_5:
1909 pad = RSA_PKCS1_PADDING;
1910 break;
1911 case RTE_CRYPTO_RSA_PADDING_NONE:
1912 pad = RSA_NO_PADDING;
1913 break;
1914 default:
1915 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1916 OPENSSL_LOG(ERR,
1917 "rsa pad type not supported %d\n", pad);
1918 return 0;
1919 }
1920
1921 switch (op->rsa.op_type) {
1922 case RTE_CRYPTO_ASYM_OP_ENCRYPT:
1923 ret = RSA_public_encrypt(op->rsa.message.length,
1924 op->rsa.message.data,
1925 op->rsa.cipher.data,
1926 rsa,
1927 pad);
1928
1929 if (ret > 0)
1930 op->rsa.cipher.length = ret;
1931 OPENSSL_LOG(DEBUG,
1932 "length of encrypted text %d\n", ret);
1933 break;
1934
1935 case RTE_CRYPTO_ASYM_OP_DECRYPT:
1936 ret = RSA_private_decrypt(op->rsa.cipher.length,
1937 op->rsa.cipher.data,
1938 op->rsa.message.data,
1939 rsa,
1940 pad);
1941 if (ret > 0)
1942 op->rsa.message.length = ret;
1943 break;
1944
1945 case RTE_CRYPTO_ASYM_OP_SIGN:
1946 ret = RSA_private_encrypt(op->rsa.message.length,
1947 op->rsa.message.data,
1948 op->rsa.sign.data,
1949 rsa,
1950 pad);
1951 if (ret > 0)
1952 op->rsa.sign.length = ret;
1953 break;
1954
1955 case RTE_CRYPTO_ASYM_OP_VERIFY:
1956 tmp = rte_malloc(NULL, op->rsa.sign.length, 0);
1957 if (tmp == NULL) {
1958 OPENSSL_LOG(ERR, "Memory allocation failed");
1959 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1960 break;
1961 }
1962 ret = RSA_public_decrypt(op->rsa.sign.length,
1963 op->rsa.sign.data,
1964 tmp,
1965 rsa,
1966 pad);
1967
1968 OPENSSL_LOG(DEBUG,
1969 "Length of public_decrypt %d "
1970 "length of message %zd\n",
1971 ret, op->rsa.message.length);
1972 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data,
1973 op->rsa.message.length))) {
1974 OPENSSL_LOG(ERR, "RSA sign Verification failed");
1975 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1976 }
1977 rte_free(tmp);
1978 break;
1979
1980 default:
1981
1982
1983
1984 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1985 break;
1986 }
1987
1988 if (ret < 0)
1989 cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
1990
1991 return 0;
1992}
1993
1994static int
1995process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1996 struct openssl_asym_session *sess)
1997{
1998 int retval = 0;
1999
2000 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2001
2002 switch (sess->xfrm_type) {
2003 case RTE_CRYPTO_ASYM_XFORM_RSA:
2004 retval = process_openssl_rsa_op(op, sess);
2005 break;
2006 case RTE_CRYPTO_ASYM_XFORM_MODEX:
2007 retval = process_openssl_modexp_op(op, sess);
2008 break;
2009 case RTE_CRYPTO_ASYM_XFORM_MODINV:
2010 retval = process_openssl_modinv_op(op, sess);
2011 break;
2012 case RTE_CRYPTO_ASYM_XFORM_DH:
2013 retval = process_openssl_dh_op(op, sess);
2014 break;
2015 case RTE_CRYPTO_ASYM_XFORM_DSA:
2016 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN)
2017 retval = process_openssl_dsa_sign_op(op, sess);
2018 else if (op->asym->dsa.op_type ==
2019 RTE_CRYPTO_ASYM_OP_VERIFY)
2020 retval =
2021 process_openssl_dsa_verify_op(op, sess);
2022 else
2023 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2024 break;
2025 default:
2026 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
2027 break;
2028 }
2029 if (!retval) {
2030
2031 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2032 if (retval)
2033
2034 retval = -1;
2035 }
2036
2037 return retval;
2038}
2039
2040static void
2041copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst,
2042 struct rte_crypto_op *op)
2043{
2044 uint8_t *p_src, *p_dst;
2045
2046 p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
2047 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
2048
2049
2050
2051
2052
2053 if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
2054 memcpy(p_dst + op->sym->auth.data.offset,
2055 p_src + op->sym->auth.data.offset,
2056 op->sym->cipher.data.offset -
2057 op->sym->auth.data.offset);
2058}
2059
2060
2061static int
2062process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
2063 struct openssl_session *sess)
2064{
2065 struct rte_mbuf *msrc, *mdst;
2066 int retval;
2067
2068 msrc = op->sym->m_src;
2069 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
2070
2071 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
2072
2073 switch (sess->chain_order) {
2074 case OPENSSL_CHAIN_ONLY_CIPHER:
2075 process_openssl_cipher_op(op, sess, msrc, mdst);
2076 break;
2077 case OPENSSL_CHAIN_ONLY_AUTH:
2078 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2079 break;
2080 case OPENSSL_CHAIN_CIPHER_AUTH:
2081 process_openssl_cipher_op(op, sess, msrc, mdst);
2082
2083 if (msrc != mdst)
2084 copy_plaintext(msrc, mdst, op);
2085 process_openssl_auth_op(qp, op, sess, mdst, mdst);
2086 break;
2087 case OPENSSL_CHAIN_AUTH_CIPHER:
2088 process_openssl_auth_op(qp, op, sess, msrc, mdst);
2089 process_openssl_cipher_op(op, sess, msrc, mdst);
2090 break;
2091 case OPENSSL_CHAIN_COMBINED:
2092 process_openssl_combined_op(op, sess, msrc, mdst);
2093 break;
2094 case OPENSSL_CHAIN_CIPHER_BPI:
2095 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
2096 break;
2097 default:
2098 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
2099 break;
2100 }
2101
2102
2103 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
2104 openssl_reset_session(sess);
2105 memset(sess, 0, sizeof(struct openssl_session));
2106 memset(op->sym->session, 0,
2107 rte_cryptodev_sym_get_existing_header_session_size(
2108 op->sym->session));
2109 rte_mempool_put(qp->sess_mp_priv, sess);
2110 rte_mempool_put(qp->sess_mp, op->sym->session);
2111 op->sym->session = NULL;
2112 }
2113
2114 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
2115 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2116
2117 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
2118 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
2119 else
2120 retval = -1;
2121
2122 return retval;
2123}
2124
2125
2126
2127
2128
2129
2130
2131
2132static uint16_t
2133openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
2134 uint16_t nb_ops)
2135{
2136 void *sess;
2137 struct openssl_qp *qp = queue_pair;
2138 int i, retval;
2139
2140 for (i = 0; i < nb_ops; i++) {
2141 sess = get_session(qp, ops[i]);
2142 if (unlikely(sess == NULL))
2143 goto enqueue_err;
2144
2145 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)
2146 retval = process_op(qp, ops[i],
2147 (struct openssl_session *) sess);
2148 else
2149 retval = process_asym_op(qp, ops[i],
2150 (struct openssl_asym_session *) sess);
2151 if (unlikely(retval < 0))
2152 goto enqueue_err;
2153 }
2154
2155 qp->stats.enqueued_count += i;
2156 return i;
2157
2158enqueue_err:
2159 qp->stats.enqueue_err_count++;
2160 return i;
2161}
2162
2163
2164static uint16_t
2165openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
2166 uint16_t nb_ops)
2167{
2168 struct openssl_qp *qp = queue_pair;
2169
2170 unsigned int nb_dequeued = 0;
2171
2172 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
2173 (void **)ops, nb_ops, NULL);
2174 qp->stats.dequeued_count += nb_dequeued;
2175
2176 return nb_dequeued;
2177}
2178
2179
2180static int
2181cryptodev_openssl_create(const char *name,
2182 struct rte_vdev_device *vdev,
2183 struct rte_cryptodev_pmd_init_params *init_params)
2184{
2185 struct rte_cryptodev *dev;
2186 struct openssl_private *internals;
2187
2188 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
2189 if (dev == NULL) {
2190 OPENSSL_LOG(ERR, "failed to create cryptodev vdev");
2191 goto init_error;
2192 }
2193
2194 dev->driver_id = cryptodev_driver_id;
2195 dev->dev_ops = rte_openssl_pmd_ops;
2196
2197
2198 dev->dequeue_burst = openssl_pmd_dequeue_burst;
2199 dev->enqueue_burst = openssl_pmd_enqueue_burst;
2200
2201 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
2202 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
2203 RTE_CRYPTODEV_FF_CPU_AESNI |
2204 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
2205 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
2206 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
2207 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
2208 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP |
2209 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT |
2210 RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
2211
2212 internals = dev->data->dev_private;
2213
2214 internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
2215
2216 rte_cryptodev_pmd_probing_finish(dev);
2217
2218 return 0;
2219
2220init_error:
2221 OPENSSL_LOG(ERR, "driver %s: create failed",
2222 init_params->name);
2223
2224 cryptodev_openssl_remove(vdev);
2225 return -EFAULT;
2226}
2227
2228
2229static int
2230cryptodev_openssl_probe(struct rte_vdev_device *vdev)
2231{
2232 struct rte_cryptodev_pmd_init_params init_params = {
2233 "",
2234 sizeof(struct openssl_private),
2235 rte_socket_id(),
2236 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
2237 };
2238 const char *name;
2239 const char *input_args;
2240
2241 name = rte_vdev_device_name(vdev);
2242 if (name == NULL)
2243 return -EINVAL;
2244 input_args = rte_vdev_device_args(vdev);
2245
2246 rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
2247
2248 return cryptodev_openssl_create(name, vdev, &init_params);
2249}
2250
2251
2252static int
2253cryptodev_openssl_remove(struct rte_vdev_device *vdev)
2254{
2255 struct rte_cryptodev *cryptodev;
2256 const char *name;
2257
2258 name = rte_vdev_device_name(vdev);
2259 if (name == NULL)
2260 return -EINVAL;
2261
2262 cryptodev = rte_cryptodev_pmd_get_named_dev(name);
2263 if (cryptodev == NULL)
2264 return -ENODEV;
2265
2266 return rte_cryptodev_pmd_destroy(cryptodev);
2267}
2268
2269static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
2270 .probe = cryptodev_openssl_probe,
2271 .remove = cryptodev_openssl_remove
2272};
2273
2274static struct cryptodev_driver openssl_crypto_drv;
2275
2276RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
2277 cryptodev_openssl_pmd_drv);
2278RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
2279 "max_nb_queue_pairs=<int> "
2280 "socket_id=<int>");
2281RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv,
2282 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id);
2283RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO);
2284
