1
2
3
4#include <crypto/aes.h>
5#include <crypto/authenc.h>
6#include <crypto/cryptd.h>
7#include <crypto/des.h>
8#include <crypto/internal/aead.h>
9#include <crypto/sha1.h>
10#include <crypto/sha2.h>
11#include <crypto/xts.h>
12#include <crypto/gcm.h>
13#include <crypto/scatterwalk.h>
14#include <linux/rtnetlink.h>
15#include <linux/sort.h>
16#include <linux/module.h>
17#include "otx2_cptvf.h"
18#include "otx2_cptvf_algs.h"
19#include "otx2_cpt_reqmgr.h"
20
21
22#define AES_GCM_SALT_SIZE 4
23
24#define AES_GCM_IV_SIZE 8
25
26#define AES_GCM_ICV_SIZE 16
27
28#define AES_GCM_IV_OFFSET 8
29#define CONTROL_WORD_LEN 8
30#define KEY2_OFFSET 48
31#define DMA_MODE_FLAG(dma_mode) \
32 (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0)
33
34
35#define SHA1_TRUNC_DIGEST_SIZE 12
36#define SHA256_TRUNC_DIGEST_SIZE 16
37#define SHA384_TRUNC_DIGEST_SIZE 24
38#define SHA512_TRUNC_DIGEST_SIZE 32
39
40static DEFINE_MUTEX(mutex);
41static int is_crypto_registered;
42
43struct cpt_device_desc {
44 struct pci_dev *dev;
45 int num_queues;
46};
47
48struct cpt_device_table {
49 atomic_t count;
50 struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM];
51};
52
53static struct cpt_device_table se_devices = {
54 .count = ATOMIC_INIT(0)
55};
56
57static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
58{
59 int count;
60
61 count = atomic_read(&se_devices.count);
62 if (count < 1)
63 return -ENODEV;
64
65 *cpu_num = get_cpu();
66
67
68
69
70
71
72
73 if (*cpu_num >= se_devices.desc[0].num_queues)
74 *cpu_num %= se_devices.desc[0].num_queues;
75 *pdev = se_devices.desc[0].dev;
76
77 put_cpu();
78
79 return 0;
80}
81
82static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
83{
84 struct otx2_cpt_req_ctx *rctx;
85 struct aead_request *req;
86 struct crypto_aead *tfm;
87
88 req = container_of(cpt_req->areq, struct aead_request, base);
89 tfm = crypto_aead_reqtfm(req);
90 rctx = aead_request_ctx(req);
91 if (memcmp(rctx->fctx.hmac.s.hmac_calc,
92 rctx->fctx.hmac.s.hmac_recv,
93 crypto_aead_authsize(tfm)) != 0)
94 return -EBADMSG;
95
96 return 0;
97}
98
99static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2)
100{
101 struct otx2_cpt_inst_info *inst_info = arg2;
102 struct crypto_async_request *areq = arg1;
103 struct otx2_cpt_req_info *cpt_req;
104 struct pci_dev *pdev;
105
106 if (inst_info) {
107 cpt_req = inst_info->req;
108 if (!status) {
109
110
111
112
113
114 if (cpt_req->req_type ==
115 OTX2_CPT_AEAD_ENC_DEC_NULL_REQ &&
116 !cpt_req->is_enc)
117 status = validate_hmac_cipher_null(cpt_req);
118 }
119 pdev = inst_info->pdev;
120 otx2_cpt_info_destroy(pdev, inst_info);
121 }
122 if (areq)
123 areq->complete(areq, status);
124}
125
126static void output_iv_copyback(struct crypto_async_request *areq)
127{
128 struct otx2_cpt_req_info *req_info;
129 struct otx2_cpt_req_ctx *rctx;
130 struct skcipher_request *sreq;
131 struct crypto_skcipher *stfm;
132 struct otx2_cpt_enc_ctx *ctx;
133 u32 start, ivsize;
134
135 sreq = container_of(areq, struct skcipher_request, base);
136 stfm = crypto_skcipher_reqtfm(sreq);
137 ctx = crypto_skcipher_ctx(stfm);
138 if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
139 ctx->cipher_type == OTX2_CPT_DES3_CBC) {
140 rctx = skcipher_request_ctx(sreq);
141 req_info = &rctx->cpt_req;
142 ivsize = crypto_skcipher_ivsize(stfm);
143 start = sreq->cryptlen - ivsize;
144
145 if (req_info->is_enc) {
146 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,
147 ivsize, 0);
148 } else {
149 if (sreq->src != sreq->dst) {
150 scatterwalk_map_and_copy(sreq->iv, sreq->src,
151 start, ivsize, 0);
152 } else {
153 memcpy(sreq->iv, req_info->iv_out, ivsize);
154 kfree(req_info->iv_out);
155 }
156 }
157 }
158}
159
160static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2)
161{
162 struct otx2_cpt_inst_info *inst_info = arg2;
163 struct crypto_async_request *areq = arg1;
164 struct pci_dev *pdev;
165
166 if (areq) {
167 if (!status)
168 output_iv_copyback(areq);
169 if (inst_info) {
170 pdev = inst_info->pdev;
171 otx2_cpt_info_destroy(pdev, inst_info);
172 }
173 areq->complete(areq, status);
174 }
175}
176
177static inline void update_input_data(struct otx2_cpt_req_info *req_info,
178 struct scatterlist *inp_sg,
179 u32 nbytes, u32 *argcnt)
180{
181 req_info->req.dlen += nbytes;
182
183 while (nbytes) {
184 u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length;
185 u8 *ptr = sg_virt(inp_sg);
186
187 req_info->in[*argcnt].vptr = (void *)ptr;
188 req_info->in[*argcnt].size = len;
189 nbytes -= len;
190 ++(*argcnt);
191 inp_sg = sg_next(inp_sg);
192 }
193}
194
195static inline void update_output_data(struct otx2_cpt_req_info *req_info,
196 struct scatterlist *outp_sg,
197 u32 offset, u32 nbytes, u32 *argcnt)
198{
199 u32 len, sg_len;
200 u8 *ptr;
201
202 req_info->rlen += nbytes;
203
204 while (nbytes) {
205 sg_len = outp_sg->length - offset;
206 len = (nbytes < sg_len) ? nbytes : sg_len;
207 ptr = sg_virt(outp_sg);
208
209 req_info->out[*argcnt].vptr = (void *) (ptr + offset);
210 req_info->out[*argcnt].size = len;
211 nbytes -= len;
212 ++(*argcnt);
213 offset = 0;
214 outp_sg = sg_next(outp_sg);
215 }
216}
217
218static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
219 u32 *argcnt)
220{
221 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
222 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
223 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
224 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
225 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
226 int ivsize = crypto_skcipher_ivsize(stfm);
227 u32 start = req->cryptlen - ivsize;
228 gfp_t flags;
229
230 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
231 GFP_KERNEL : GFP_ATOMIC;
232 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
233 req_info->ctrl.s.se_req = 1;
234
235 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
236 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
237 if (enc) {
238 req_info->req.opcode.s.minor = 2;
239 } else {
240 req_info->req.opcode.s.minor = 3;
241 if ((ctx->cipher_type == OTX2_CPT_AES_CBC ||
242 ctx->cipher_type == OTX2_CPT_DES3_CBC) &&
243 req->src == req->dst) {
244 req_info->iv_out = kmalloc(ivsize, flags);
245 if (!req_info->iv_out)
246 return -ENOMEM;
247
248 scatterwalk_map_and_copy(req_info->iv_out, req->src,
249 start, ivsize, 0);
250 }
251 }
252
253 req_info->req.param1 = req->cryptlen;
254
255 req_info->req.param2 = 0;
256
257 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
258 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
259 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
260
261 if (ctx->cipher_type == OTX2_CPT_AES_XTS)
262 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
263 else
264 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
265
266 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
267
268 cpu_to_be64s(&fctx->enc.enc_ctrl.u);
269
270
271
272
273
274 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
275 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
276 req_info->req.dlen += CONTROL_WORD_LEN;
277 ++(*argcnt);
278
279 req_info->in[*argcnt].vptr = (u8 *)fctx;
280 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
281 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
282
283 ++(*argcnt);
284
285 return 0;
286}
287
288static inline int create_input_list(struct skcipher_request *req, u32 enc,
289 u32 enc_iv_len)
290{
291 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
292 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
293 u32 argcnt = 0;
294 int ret;
295
296 ret = create_ctx_hdr(req, enc, &argcnt);
297 if (ret)
298 return ret;
299
300 update_input_data(req_info, req->src, req->cryptlen, &argcnt);
301 req_info->in_cnt = argcnt;
302
303 return 0;
304}
305
306static inline void create_output_list(struct skcipher_request *req,
307 u32 enc_iv_len)
308{
309 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
310 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
311 u32 argcnt = 0;
312
313
314
315
316
317
318
319
320
321 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);
322 req_info->out_cnt = argcnt;
323}
324
325static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
326{
327 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
328 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
329 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
330 int ret;
331
332 if (ctx->fbk_cipher) {
333 skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher);
334 skcipher_request_set_callback(&rctx->sk_fbk_req,
335 req->base.flags,
336 req->base.complete,
337 req->base.data);
338 skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src,
339 req->dst, req->cryptlen, req->iv);
340 ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) :
341 crypto_skcipher_decrypt(&rctx->sk_fbk_req);
342 } else {
343 ret = -EINVAL;
344 }
345 return ret;
346}
347
348static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
349{
350 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
351 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req);
352 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
353 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
354 u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
355 struct pci_dev *pdev;
356 int status, cpu_num;
357
358 if (req->cryptlen == 0)
359 return 0;
360
361 if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len))
362 return -EINVAL;
363
364 if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE)
365 return skcipher_do_fallback(req, enc);
366
367
368 rctx->ctrl_word.flags = 0;
369 rctx->fctx.enc.enc_ctrl.u = 0;
370
371 status = create_input_list(req, enc, enc_iv_len);
372 if (status)
373 return status;
374 create_output_list(req, enc_iv_len);
375
376 status = get_se_device(&pdev, &cpu_num);
377 if (status)
378 return status;
379
380 req_info->callback = otx2_cpt_skcipher_callback;
381 req_info->areq = &req->base;
382 req_info->req_type = OTX2_CPT_ENC_DEC_REQ;
383 req_info->is_enc = enc;
384 req_info->is_trunc_hmac = false;
385 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
386
387
388
389
390
391
392 status = otx2_cpt_do_request(pdev, req_info, cpu_num);
393
394 return status;
395}
396
397static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req)
398{
399 return cpt_enc_dec(req, true);
400}
401
402static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req)
403{
404 return cpt_enc_dec(req, false);
405}
406
407static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
408 const u8 *key, u32 keylen)
409{
410 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
411 const u8 *key2 = key + (keylen / 2);
412 const u8 *key1 = key;
413 int ret;
414
415 ret = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen);
416 if (ret)
417 return ret;
418 ctx->key_len = keylen;
419 ctx->enc_align_len = 1;
420 memcpy(ctx->enc_key, key1, keylen / 2);
421 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
422 ctx->cipher_type = OTX2_CPT_AES_XTS;
423 switch (ctx->key_len) {
424 case 2 * AES_KEYSIZE_128:
425 ctx->key_type = OTX2_CPT_AES_128_BIT;
426 break;
427 case 2 * AES_KEYSIZE_192:
428 ctx->key_type = OTX2_CPT_AES_192_BIT;
429 break;
430 case 2 * AES_KEYSIZE_256:
431 ctx->key_type = OTX2_CPT_AES_256_BIT;
432 break;
433 default:
434 return -EINVAL;
435 }
436 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
437}
438
439static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
440 u32 keylen, u8 cipher_type)
441{
442 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
443
444 if (keylen != DES3_EDE_KEY_SIZE)
445 return -EINVAL;
446
447 ctx->key_len = keylen;
448 ctx->cipher_type = cipher_type;
449 ctx->enc_align_len = 8;
450
451 memcpy(ctx->enc_key, key, keylen);
452
453 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
454}
455
456static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
457 u32 keylen, u8 cipher_type)
458{
459 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
460
461 switch (keylen) {
462 case AES_KEYSIZE_128:
463 ctx->key_type = OTX2_CPT_AES_128_BIT;
464 break;
465 case AES_KEYSIZE_192:
466 ctx->key_type = OTX2_CPT_AES_192_BIT;
467 break;
468 case AES_KEYSIZE_256:
469 ctx->key_type = OTX2_CPT_AES_256_BIT;
470 break;
471 default:
472 return -EINVAL;
473 }
474 if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB)
475 ctx->enc_align_len = 16;
476 else
477 ctx->enc_align_len = 1;
478
479 ctx->key_len = keylen;
480 ctx->cipher_type = cipher_type;
481
482 memcpy(ctx->enc_key, key, keylen);
483
484 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
485}
486
487static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
488 const u8 *key, u32 keylen)
489{
490 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC);
491}
492
493static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
494 const u8 *key, u32 keylen)
495{
496 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB);
497}
498
499static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
500 const u8 *key, u32 keylen)
501{
502 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC);
503}
504
505static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
506 const u8 *key, u32 keylen)
507{
508 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB);
509}
510
511static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx,
512 struct crypto_alg *alg)
513{
514 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
515 ctx->fbk_cipher =
516 crypto_alloc_skcipher(alg->cra_name, 0,
517 CRYPTO_ALG_ASYNC |
518 CRYPTO_ALG_NEED_FALLBACK);
519 if (IS_ERR(ctx->fbk_cipher)) {
520 pr_err("%s() failed to allocate fallback for %s\n",
521 __func__, alg->cra_name);
522 return PTR_ERR(ctx->fbk_cipher);
523 }
524 }
525 return 0;
526}
527
528static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
529{
530 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
531 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
532 struct crypto_alg *alg = tfm->__crt_alg;
533
534 memset(ctx, 0, sizeof(*ctx));
535
536
537
538
539
540 crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) +
541 sizeof(struct skcipher_request));
542
543 return cpt_skcipher_fallback_init(ctx, alg);
544}
545
546static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)
547{
548 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
549
550 if (ctx->fbk_cipher) {
551 crypto_free_skcipher(ctx->fbk_cipher);
552 ctx->fbk_cipher = NULL;
553 }
554}
555
556static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
557 struct crypto_alg *alg)
558{
559 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
560 ctx->fbk_cipher =
561 crypto_alloc_aead(alg->cra_name, 0,
562 CRYPTO_ALG_ASYNC |
563 CRYPTO_ALG_NEED_FALLBACK);
564 if (IS_ERR(ctx->fbk_cipher)) {
565 pr_err("%s() failed to allocate fallback for %s\n",
566 __func__, alg->cra_name);
567 return PTR_ERR(ctx->fbk_cipher);
568 }
569 }
570 return 0;
571}
572
573static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
574{
575 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm);
576 struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
577 struct crypto_alg *alg = tfm->__crt_alg;
578
579 ctx->cipher_type = cipher_type;
580 ctx->mac_type = mac_type;
581
582
583
584
585
586
587 if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) {
588 switch (ctx->mac_type) {
589 case OTX2_CPT_SHA1:
590 ctx->hashalg = crypto_alloc_shash("sha1", 0,
591 CRYPTO_ALG_ASYNC);
592 if (IS_ERR(ctx->hashalg))
593 return PTR_ERR(ctx->hashalg);
594 break;
595
596 case OTX2_CPT_SHA256:
597 ctx->hashalg = crypto_alloc_shash("sha256", 0,
598 CRYPTO_ALG_ASYNC);
599 if (IS_ERR(ctx->hashalg))
600 return PTR_ERR(ctx->hashalg);
601 break;
602
603 case OTX2_CPT_SHA384:
604 ctx->hashalg = crypto_alloc_shash("sha384", 0,
605 CRYPTO_ALG_ASYNC);
606 if (IS_ERR(ctx->hashalg))
607 return PTR_ERR(ctx->hashalg);
608 break;
609
610 case OTX2_CPT_SHA512:
611 ctx->hashalg = crypto_alloc_shash("sha512", 0,
612 CRYPTO_ALG_ASYNC);
613 if (IS_ERR(ctx->hashalg))
614 return PTR_ERR(ctx->hashalg);
615 break;
616 }
617 }
618 switch (ctx->cipher_type) {
619 case OTX2_CPT_AES_CBC:
620 case OTX2_CPT_AES_ECB:
621 ctx->enc_align_len = 16;
622 break;
623 case OTX2_CPT_DES3_CBC:
624 case OTX2_CPT_DES3_ECB:
625 ctx->enc_align_len = 8;
626 break;
627 case OTX2_CPT_AES_GCM:
628 case OTX2_CPT_CIPHER_NULL:
629 ctx->enc_align_len = 1;
630 break;
631 }
632 crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx));
633
634 return cpt_aead_fallback_init(ctx, alg);
635}
636
637static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
638{
639 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1);
640}
641
642static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
643{
644 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256);
645}
646
647static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
648{
649 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384);
650}
651
652static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
653{
654 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512);
655}
656
657static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
658{
659 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1);
660}
661
662static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
663{
664 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256);
665}
666
667static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
668{
669 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384);
670}
671
672static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
673{
674 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512);
675}
676
677static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
678{
679 return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL);
680}
681
682static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
683{
684 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
685
686 kfree(ctx->ipad);
687 kfree(ctx->opad);
688 if (ctx->hashalg)
689 crypto_free_shash(ctx->hashalg);
690 kfree(ctx->sdesc);
691
692 if (ctx->fbk_cipher) {
693 crypto_free_aead(ctx->fbk_cipher);
694 ctx->fbk_cipher = NULL;
695 }
696}
697
698static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
699 unsigned int authsize)
700{
701 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
702
703 if (crypto_rfc4106_check_authsize(authsize))
704 return -EINVAL;
705
706 tfm->authsize = authsize;
707
708 if (ctx->fbk_cipher)
709 ctx->fbk_cipher->authsize = authsize;
710
711 return 0;
712}
713
714static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
715 unsigned int authsize)
716{
717 tfm->authsize = authsize;
718
719 return 0;
720}
721
722static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
723 unsigned int authsize)
724{
725 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
726
727 ctx->is_trunc_hmac = true;
728 tfm->authsize = authsize;
729
730 return 0;
731}
732
733static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
734{
735 struct otx2_cpt_sdesc *sdesc;
736 int size;
737
738 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
739 sdesc = kmalloc(size, GFP_KERNEL);
740 if (!sdesc)
741 return NULL;
742
743 sdesc->shash.tfm = alg;
744
745 return sdesc;
746}
747
748static inline void swap_data32(void *buf, u32 len)
749{
750 cpu_to_be32_array(buf, buf, len / 4);
751}
752
753static inline void swap_data64(void *buf, u32 len)
754{
755 u64 *src = buf;
756 int i = 0;
757
758 for (i = 0 ; i < len / 8; i++, src++)
759 cpu_to_be64s(src);
760}
761
762static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
763{
764 struct sha512_state *sha512;
765 struct sha256_state *sha256;
766 struct sha1_state *sha1;
767
768 switch (mac_type) {
769 case OTX2_CPT_SHA1:
770 sha1 = (struct sha1_state *) in_pad;
771 swap_data32(sha1->state, SHA1_DIGEST_SIZE);
772 memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE);
773 break;
774
775 case OTX2_CPT_SHA256:
776 sha256 = (struct sha256_state *) in_pad;
777 swap_data32(sha256->state, SHA256_DIGEST_SIZE);
778 memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE);
779 break;
780
781 case OTX2_CPT_SHA384:
782 case OTX2_CPT_SHA512:
783 sha512 = (struct sha512_state *) in_pad;
784 swap_data64(sha512->state, SHA512_DIGEST_SIZE);
785 memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE);
786 break;
787
788 default:
789 return -EINVAL;
790 }
791
792 return 0;
793}
794
795static int aead_hmac_init(struct crypto_aead *cipher)
796{
797 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
798 int state_size = crypto_shash_statesize(ctx->hashalg);
799 int ds = crypto_shash_digestsize(ctx->hashalg);
800 int bs = crypto_shash_blocksize(ctx->hashalg);
801 int authkeylen = ctx->auth_key_len;
802 u8 *ipad = NULL, *opad = NULL;
803 int ret = 0, icount = 0;
804
805 ctx->sdesc = alloc_sdesc(ctx->hashalg);
806 if (!ctx->sdesc)
807 return -ENOMEM;
808
809 ctx->ipad = kzalloc(bs, GFP_KERNEL);
810 if (!ctx->ipad) {
811 ret = -ENOMEM;
812 goto calc_fail;
813 }
814
815 ctx->opad = kzalloc(bs, GFP_KERNEL);
816 if (!ctx->opad) {
817 ret = -ENOMEM;
818 goto calc_fail;
819 }
820
821 ipad = kzalloc(state_size, GFP_KERNEL);
822 if (!ipad) {
823 ret = -ENOMEM;
824 goto calc_fail;
825 }
826
827 opad = kzalloc(state_size, GFP_KERNEL);
828 if (!opad) {
829 ret = -ENOMEM;
830 goto calc_fail;
831 }
832
833 if (authkeylen > bs) {
834 ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key,
835 authkeylen, ipad);
836 if (ret)
837 goto calc_fail;
838
839 authkeylen = ds;
840 } else {
841 memcpy(ipad, ctx->key, authkeylen);
842 }
843
844 memset(ipad + authkeylen, 0, bs - authkeylen);
845 memcpy(opad, ipad, bs);
846
847 for (icount = 0; icount < bs; icount++) {
848 ipad[icount] ^= 0x36;
849 opad[icount] ^= 0x5c;
850 }
851
852
853
854
855
856
857
858 crypto_shash_init(&ctx->sdesc->shash);
859 crypto_shash_update(&ctx->sdesc->shash, ipad, bs);
860 crypto_shash_export(&ctx->sdesc->shash, ipad);
861 ret = copy_pad(ctx->mac_type, ctx->ipad, ipad);
862 if (ret)
863 goto calc_fail;
864
865
866 crypto_shash_init(&ctx->sdesc->shash);
867 crypto_shash_update(&ctx->sdesc->shash, opad, bs);
868 crypto_shash_export(&ctx->sdesc->shash, opad);
869 ret = copy_pad(ctx->mac_type, ctx->opad, opad);
870 if (ret)
871 goto calc_fail;
872
873 kfree(ipad);
874 kfree(opad);
875
876 return 0;
877
878calc_fail:
879 kfree(ctx->ipad);
880 ctx->ipad = NULL;
881 kfree(ctx->opad);
882 ctx->opad = NULL;
883 kfree(ipad);
884 kfree(opad);
885 kfree(ctx->sdesc);
886 ctx->sdesc = NULL;
887
888 return ret;
889}
890
891static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
892 const unsigned char *key,
893 unsigned int keylen)
894{
895 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
896 struct crypto_authenc_key_param *param;
897 int enckeylen = 0, authkeylen = 0;
898 struct rtattr *rta = (void *)key;
899 int status;
900
901 if (!RTA_OK(rta, keylen))
902 return -EINVAL;
903
904 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
905 return -EINVAL;
906
907 if (RTA_PAYLOAD(rta) < sizeof(*param))
908 return -EINVAL;
909
910 param = RTA_DATA(rta);
911 enckeylen = be32_to_cpu(param->enckeylen);
912 key += RTA_ALIGN(rta->rta_len);
913 keylen -= RTA_ALIGN(rta->rta_len);
914 if (keylen < enckeylen)
915 return -EINVAL;
916
917 if (keylen > OTX2_CPT_MAX_KEY_SIZE)
918 return -EINVAL;
919
920 authkeylen = keylen - enckeylen;
921 memcpy(ctx->key, key, keylen);
922
923 switch (enckeylen) {
924 case AES_KEYSIZE_128:
925 ctx->key_type = OTX2_CPT_AES_128_BIT;
926 break;
927 case AES_KEYSIZE_192:
928 ctx->key_type = OTX2_CPT_AES_192_BIT;
929 break;
930 case AES_KEYSIZE_256:
931 ctx->key_type = OTX2_CPT_AES_256_BIT;
932 break;
933 default:
934
935 return -EINVAL;
936 }
937
938 ctx->enc_key_len = enckeylen;
939 ctx->auth_key_len = authkeylen;
940
941 status = aead_hmac_init(cipher);
942 if (status)
943 return status;
944
945 return 0;
946}
947
948static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
949 const unsigned char *key,
950 unsigned int keylen)
951{
952 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
953 struct crypto_authenc_key_param *param;
954 struct rtattr *rta = (void *)key;
955 int enckeylen = 0;
956
957 if (!RTA_OK(rta, keylen))
958 return -EINVAL;
959
960 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
961 return -EINVAL;
962
963 if (RTA_PAYLOAD(rta) < sizeof(*param))
964 return -EINVAL;
965
966 param = RTA_DATA(rta);
967 enckeylen = be32_to_cpu(param->enckeylen);
968 key += RTA_ALIGN(rta->rta_len);
969 keylen -= RTA_ALIGN(rta->rta_len);
970 if (enckeylen != 0)
971 return -EINVAL;
972
973 if (keylen > OTX2_CPT_MAX_KEY_SIZE)
974 return -EINVAL;
975
976 memcpy(ctx->key, key, keylen);
977 ctx->enc_key_len = enckeylen;
978 ctx->auth_key_len = keylen;
979
980 return 0;
981}
982
983static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
984 const unsigned char *key,
985 unsigned int keylen)
986{
987 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
988
989
990
991
992
993 switch (keylen) {
994 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
995 ctx->key_type = OTX2_CPT_AES_128_BIT;
996 ctx->enc_key_len = AES_KEYSIZE_128;
997 break;
998 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
999 ctx->key_type = OTX2_CPT_AES_192_BIT;
1000 ctx->enc_key_len = AES_KEYSIZE_192;
1001 break;
1002 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
1003 ctx->key_type = OTX2_CPT_AES_256_BIT;
1004 ctx->enc_key_len = AES_KEYSIZE_256;
1005 break;
1006 default:
1007
1008 return -EINVAL;
1009 }
1010
1011
1012 memcpy(ctx->key, key, keylen);
1013
1014 return crypto_aead_setkey(ctx->fbk_cipher, key, keylen);
1015}
1016
1017static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
1018 u32 *argcnt)
1019{
1020 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1021 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1022 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1023 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1024 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
1025 int mac_len = crypto_aead_authsize(tfm);
1026 int ds;
1027
1028 rctx->ctrl_word.e.enc_data_offset = req->assoclen;
1029
1030 switch (ctx->cipher_type) {
1031 case OTX2_CPT_AES_CBC:
1032 if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8))
1033 return -EINVAL;
1034
1035 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
1036
1037 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
1038 ctx->enc_key_len);
1039
1040 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
1041
1042 ds = crypto_shash_digestsize(ctx->hashalg);
1043 if (ctx->mac_type == OTX2_CPT_SHA384)
1044 ds = SHA512_DIGEST_SIZE;
1045 if (ctx->ipad)
1046 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
1047 if (ctx->opad)
1048 memcpy(fctx->hmac.e.opad, ctx->opad, ds);
1049 break;
1050
1051 case OTX2_CPT_AES_GCM:
1052 if (crypto_ipsec_check_assoclen(req->assoclen))
1053 return -EINVAL;
1054
1055 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR;
1056
1057 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
1058
1059 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
1060 AES_GCM_SALT_SIZE);
1061
1062 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
1063 break;
1064
1065 default:
1066
1067 return -EINVAL;
1068 }
1069 cpu_to_be64s(&rctx->ctrl_word.flags);
1070
1071 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1072 req_info->ctrl.s.se_req = 1;
1073 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
1074 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1075 if (enc) {
1076 req_info->req.opcode.s.minor = 2;
1077 req_info->req.param1 = req->cryptlen;
1078 req_info->req.param2 = req->cryptlen + req->assoclen;
1079 } else {
1080 req_info->req.opcode.s.minor = 3;
1081 req_info->req.param1 = req->cryptlen - mac_len;
1082 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
1083 }
1084
1085 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
1086 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
1087 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
1088 fctx->enc.enc_ctrl.e.mac_len = mac_len;
1089 cpu_to_be64s(&fctx->enc.enc_ctrl.u);
1090
1091
1092
1093
1094
1095 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
1096 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
1097 req_info->req.dlen += CONTROL_WORD_LEN;
1098 ++(*argcnt);
1099
1100 req_info->in[*argcnt].vptr = (u8 *)fctx;
1101 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
1102 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
1103 ++(*argcnt);
1104
1105 return 0;
1106}
1107
1108static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
1109 u32 enc)
1110{
1111 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1112 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1113 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1114 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1115
1116 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1117 req_info->ctrl.s.se_req = 1;
1118 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC |
1119 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1120 req_info->is_trunc_hmac = ctx->is_trunc_hmac;
1121
1122 req_info->req.opcode.s.minor = 0;
1123 req_info->req.param1 = ctx->auth_key_len;
1124 req_info->req.param2 = ctx->mac_type << 8;
1125
1126
1127 req_info->in[*argcnt].vptr = ctx->key;
1128 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
1129 req_info->req.dlen += round_up(ctx->auth_key_len, 8);
1130 ++(*argcnt);
1131}
1132
1133static inline int create_aead_input_list(struct aead_request *req, u32 enc)
1134{
1135 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1136 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1137 u32 inputlen = req->cryptlen + req->assoclen;
1138 u32 status, argcnt = 0;
1139
1140 status = create_aead_ctx_hdr(req, enc, &argcnt);
1141 if (status)
1142 return status;
1143 update_input_data(req_info, req->src, inputlen, &argcnt);
1144 req_info->in_cnt = argcnt;
1145
1146 return 0;
1147}
1148
1149static inline void create_aead_output_list(struct aead_request *req, u32 enc,
1150 u32 mac_len)
1151{
1152 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1153 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1154 u32 argcnt = 0, outputlen = 0;
1155
1156 if (enc)
1157 outputlen = req->cryptlen + req->assoclen + mac_len;
1158 else
1159 outputlen = req->cryptlen + req->assoclen - mac_len;
1160
1161 update_output_data(req_info, req->dst, 0, outputlen, &argcnt);
1162 req_info->out_cnt = argcnt;
1163}
1164
1165static inline void create_aead_null_input_list(struct aead_request *req,
1166 u32 enc, u32 mac_len)
1167{
1168 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1169 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1170 u32 inputlen, argcnt = 0;
1171
1172 if (enc)
1173 inputlen = req->cryptlen + req->assoclen;
1174 else
1175 inputlen = req->cryptlen + req->assoclen - mac_len;
1176
1177 create_hmac_ctx_hdr(req, &argcnt, enc);
1178 update_input_data(req_info, req->src, inputlen, &argcnt);
1179 req_info->in_cnt = argcnt;
1180}
1181
1182static inline int create_aead_null_output_list(struct aead_request *req,
1183 u32 enc, u32 mac_len)
1184{
1185 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1186 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1187 struct scatterlist *dst;
1188 u8 *ptr = NULL;
1189 int argcnt = 0, status, offset;
1190 u32 inputlen;
1191
1192 if (enc)
1193 inputlen = req->cryptlen + req->assoclen;
1194 else
1195 inputlen = req->cryptlen + req->assoclen - mac_len;
1196
1197
1198
1199
1200
1201 if (req->src != req->dst) {
1202
1203 ptr = kmalloc(inputlen, (req_info->areq->flags &
1204 CRYPTO_TFM_REQ_MAY_SLEEP) ?
1205 GFP_KERNEL : GFP_ATOMIC);
1206 if (!ptr)
1207 return -ENOMEM;
1208
1209 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,
1210 inputlen);
1211 if (status != inputlen) {
1212 status = -EINVAL;
1213 goto error_free;
1214 }
1215 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,
1216 inputlen);
1217 if (status != inputlen) {
1218 status = -EINVAL;
1219 goto error_free;
1220 }
1221 kfree(ptr);
1222 }
1223
1224 if (enc) {
1225
1226
1227
1228
1229 dst = req->dst;
1230 offset = inputlen;
1231 while (offset >= dst->length) {
1232 offset -= dst->length;
1233 dst = sg_next(dst);
1234 if (!dst)
1235 return -ENOENT;
1236 }
1237
1238 update_output_data(req_info, dst, offset, mac_len, &argcnt);
1239 } else {
1240
1241
1242
1243
1244 status = sg_copy_buffer(req->src, sg_nents(req->src),
1245 rctx->fctx.hmac.s.hmac_recv, mac_len,
1246 inputlen, true);
1247 if (status != mac_len)
1248 return -EINVAL;
1249
1250 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1251 req_info->out[argcnt].size = mac_len;
1252 argcnt++;
1253 }
1254
1255 req_info->out_cnt = argcnt;
1256 return 0;
1257
1258error_free:
1259 kfree(ptr);
1260 return status;
1261}
1262
1263static int aead_do_fallback(struct aead_request *req, bool is_enc)
1264{
1265 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1266 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1267 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead);
1268 int ret;
1269
1270 if (ctx->fbk_cipher) {
1271
1272 aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher);
1273 aead_request_set_callback(&rctx->fbk_req, req->base.flags,
1274 req->base.complete, req->base.data);
1275 aead_request_set_crypt(&rctx->fbk_req, req->src,
1276 req->dst, req->cryptlen, req->iv);
1277 ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) :
1278 crypto_aead_decrypt(&rctx->fbk_req);
1279 } else {
1280 ret = -EINVAL;
1281 }
1282
1283 return ret;
1284}
1285
1286static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1287{
1288 struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req);
1289 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1290 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1291 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1292 struct pci_dev *pdev;
1293 int status, cpu_num;
1294
1295
1296 rctx->ctrl_word.flags = 0;
1297 rctx->fctx.enc.enc_ctrl.u = 0;
1298
1299 req_info->callback = otx2_cpt_aead_callback;
1300 req_info->areq = &req->base;
1301 req_info->req_type = reg_type;
1302 req_info->is_enc = enc;
1303 req_info->is_trunc_hmac = false;
1304
1305 switch (reg_type) {
1306 case OTX2_CPT_AEAD_ENC_DEC_REQ:
1307 status = create_aead_input_list(req, enc);
1308 if (status)
1309 return status;
1310 create_aead_output_list(req, enc, crypto_aead_authsize(tfm));
1311 break;
1312
1313 case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ:
1314 create_aead_null_input_list(req, enc,
1315 crypto_aead_authsize(tfm));
1316 status = create_aead_null_output_list(req, enc,
1317 crypto_aead_authsize(tfm));
1318 if (status)
1319 return status;
1320 break;
1321
1322 default:
1323 return -EINVAL;
1324 }
1325 if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len))
1326 return -EINVAL;
1327
1328 if (!req_info->req.param2 ||
1329 (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) ||
1330 (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE))
1331 return aead_do_fallback(req, enc);
1332
1333 status = get_se_device(&pdev, &cpu_num);
1334 if (status)
1335 return status;
1336
1337 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
1338
1339
1340
1341
1342
1343
1344 return otx2_cpt_do_request(pdev, req_info, cpu_num);
1345}
1346
1347static int otx2_cpt_aead_encrypt(struct aead_request *req)
1348{
1349 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true);
1350}
1351
1352static int otx2_cpt_aead_decrypt(struct aead_request *req)
1353{
1354 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false);
1355}
1356
1357static int otx2_cpt_aead_null_encrypt(struct aead_request *req)
1358{
1359 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true);
1360}
1361
1362static int otx2_cpt_aead_null_decrypt(struct aead_request *req)
1363{
1364 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false);
1365}
1366
1367static struct skcipher_alg otx2_cpt_skciphers[] = { {
1368 .base.cra_name = "xts(aes)",
1369 .base.cra_driver_name = "cpt_xts_aes",
1370 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1371 .base.cra_blocksize = AES_BLOCK_SIZE,
1372 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1373 .base.cra_alignmask = 7,
1374 .base.cra_priority = 4001,
1375 .base.cra_module = THIS_MODULE,
1376
1377 .init = otx2_cpt_enc_dec_init,
1378 .exit = otx2_cpt_skcipher_exit,
1379 .ivsize = AES_BLOCK_SIZE,
1380 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1381 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1382 .setkey = otx2_cpt_skcipher_xts_setkey,
1383 .encrypt = otx2_cpt_skcipher_encrypt,
1384 .decrypt = otx2_cpt_skcipher_decrypt,
1385}, {
1386 .base.cra_name = "cbc(aes)",
1387 .base.cra_driver_name = "cpt_cbc_aes",
1388 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1389 .base.cra_blocksize = AES_BLOCK_SIZE,
1390 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1391 .base.cra_alignmask = 7,
1392 .base.cra_priority = 4001,
1393 .base.cra_module = THIS_MODULE,
1394
1395 .init = otx2_cpt_enc_dec_init,
1396 .exit = otx2_cpt_skcipher_exit,
1397 .ivsize = AES_BLOCK_SIZE,
1398 .min_keysize = AES_MIN_KEY_SIZE,
1399 .max_keysize = AES_MAX_KEY_SIZE,
1400 .setkey = otx2_cpt_skcipher_cbc_aes_setkey,
1401 .encrypt = otx2_cpt_skcipher_encrypt,
1402 .decrypt = otx2_cpt_skcipher_decrypt,
1403}, {
1404 .base.cra_name = "ecb(aes)",
1405 .base.cra_driver_name = "cpt_ecb_aes",
1406 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1407 .base.cra_blocksize = AES_BLOCK_SIZE,
1408 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1409 .base.cra_alignmask = 7,
1410 .base.cra_priority = 4001,
1411 .base.cra_module = THIS_MODULE,
1412
1413 .init = otx2_cpt_enc_dec_init,
1414 .exit = otx2_cpt_skcipher_exit,
1415 .ivsize = 0,
1416 .min_keysize = AES_MIN_KEY_SIZE,
1417 .max_keysize = AES_MAX_KEY_SIZE,
1418 .setkey = otx2_cpt_skcipher_ecb_aes_setkey,
1419 .encrypt = otx2_cpt_skcipher_encrypt,
1420 .decrypt = otx2_cpt_skcipher_decrypt,
1421}, {
1422 .base.cra_name = "cbc(des3_ede)",
1423 .base.cra_driver_name = "cpt_cbc_des3_ede",
1424 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1425 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1426 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1427 .base.cra_alignmask = 7,
1428 .base.cra_priority = 4001,
1429 .base.cra_module = THIS_MODULE,
1430
1431 .init = otx2_cpt_enc_dec_init,
1432 .exit = otx2_cpt_skcipher_exit,
1433 .min_keysize = DES3_EDE_KEY_SIZE,
1434 .max_keysize = DES3_EDE_KEY_SIZE,
1435 .ivsize = DES_BLOCK_SIZE,
1436 .setkey = otx2_cpt_skcipher_cbc_des3_setkey,
1437 .encrypt = otx2_cpt_skcipher_encrypt,
1438 .decrypt = otx2_cpt_skcipher_decrypt,
1439}, {
1440 .base.cra_name = "ecb(des3_ede)",
1441 .base.cra_driver_name = "cpt_ecb_des3_ede",
1442 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1443 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1444 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1445 .base.cra_alignmask = 7,
1446 .base.cra_priority = 4001,
1447 .base.cra_module = THIS_MODULE,
1448
1449 .init = otx2_cpt_enc_dec_init,
1450 .exit = otx2_cpt_skcipher_exit,
1451 .min_keysize = DES3_EDE_KEY_SIZE,
1452 .max_keysize = DES3_EDE_KEY_SIZE,
1453 .ivsize = 0,
1454 .setkey = otx2_cpt_skcipher_ecb_des3_setkey,
1455 .encrypt = otx2_cpt_skcipher_encrypt,
1456 .decrypt = otx2_cpt_skcipher_decrypt,
1457} };
1458
1459static struct aead_alg otx2_cpt_aeads[] = { {
1460 .base = {
1461 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1462 .cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1463 .cra_blocksize = AES_BLOCK_SIZE,
1464 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1465 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1466 .cra_priority = 4001,
1467 .cra_alignmask = 0,
1468 .cra_module = THIS_MODULE,
1469 },
1470 .init = otx2_cpt_aead_cbc_aes_sha1_init,
1471 .exit = otx2_cpt_aead_exit,
1472 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1473 .setauthsize = otx2_cpt_aead_set_authsize,
1474 .encrypt = otx2_cpt_aead_encrypt,
1475 .decrypt = otx2_cpt_aead_decrypt,
1476 .ivsize = AES_BLOCK_SIZE,
1477 .maxauthsize = SHA1_DIGEST_SIZE,
1478}, {
1479 .base = {
1480 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1481 .cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1482 .cra_blocksize = AES_BLOCK_SIZE,
1483 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1484 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1485 .cra_priority = 4001,
1486 .cra_alignmask = 0,
1487 .cra_module = THIS_MODULE,
1488 },
1489 .init = otx2_cpt_aead_cbc_aes_sha256_init,
1490 .exit = otx2_cpt_aead_exit,
1491 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1492 .setauthsize = otx2_cpt_aead_set_authsize,
1493 .encrypt = otx2_cpt_aead_encrypt,
1494 .decrypt = otx2_cpt_aead_decrypt,
1495 .ivsize = AES_BLOCK_SIZE,
1496 .maxauthsize = SHA256_DIGEST_SIZE,
1497}, {
1498 .base = {
1499 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1500 .cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1501 .cra_blocksize = AES_BLOCK_SIZE,
1502 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1503 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1504 .cra_priority = 4001,
1505 .cra_alignmask = 0,
1506 .cra_module = THIS_MODULE,
1507 },
1508 .init = otx2_cpt_aead_cbc_aes_sha384_init,
1509 .exit = otx2_cpt_aead_exit,
1510 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1511 .setauthsize = otx2_cpt_aead_set_authsize,
1512 .encrypt = otx2_cpt_aead_encrypt,
1513 .decrypt = otx2_cpt_aead_decrypt,
1514 .ivsize = AES_BLOCK_SIZE,
1515 .maxauthsize = SHA384_DIGEST_SIZE,
1516}, {
1517 .base = {
1518 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1519 .cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1520 .cra_blocksize = AES_BLOCK_SIZE,
1521 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1522 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1523 .cra_priority = 4001,
1524 .cra_alignmask = 0,
1525 .cra_module = THIS_MODULE,
1526 },
1527 .init = otx2_cpt_aead_cbc_aes_sha512_init,
1528 .exit = otx2_cpt_aead_exit,
1529 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1530 .setauthsize = otx2_cpt_aead_set_authsize,
1531 .encrypt = otx2_cpt_aead_encrypt,
1532 .decrypt = otx2_cpt_aead_decrypt,
1533 .ivsize = AES_BLOCK_SIZE,
1534 .maxauthsize = SHA512_DIGEST_SIZE,
1535}, {
1536 .base = {
1537 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1538 .cra_driver_name = "cpt_hmac_sha1_ecb_null",
1539 .cra_blocksize = 1,
1540 .cra_flags = CRYPTO_ALG_ASYNC,
1541 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1542 .cra_priority = 4001,
1543 .cra_alignmask = 0,
1544 .cra_module = THIS_MODULE,
1545 },
1546 .init = otx2_cpt_aead_ecb_null_sha1_init,
1547 .exit = otx2_cpt_aead_exit,
1548 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1549 .setauthsize = otx2_cpt_aead_null_set_authsize,
1550 .encrypt = otx2_cpt_aead_null_encrypt,
1551 .decrypt = otx2_cpt_aead_null_decrypt,
1552 .ivsize = 0,
1553 .maxauthsize = SHA1_DIGEST_SIZE,
1554}, {
1555 .base = {
1556 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1557 .cra_driver_name = "cpt_hmac_sha256_ecb_null",
1558 .cra_blocksize = 1,
1559 .cra_flags = CRYPTO_ALG_ASYNC,
1560 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1561 .cra_priority = 4001,
1562 .cra_alignmask = 0,
1563 .cra_module = THIS_MODULE,
1564 },
1565 .init = otx2_cpt_aead_ecb_null_sha256_init,
1566 .exit = otx2_cpt_aead_exit,
1567 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1568 .setauthsize = otx2_cpt_aead_null_set_authsize,
1569 .encrypt = otx2_cpt_aead_null_encrypt,
1570 .decrypt = otx2_cpt_aead_null_decrypt,
1571 .ivsize = 0,
1572 .maxauthsize = SHA256_DIGEST_SIZE,
1573}, {
1574 .base = {
1575 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1576 .cra_driver_name = "cpt_hmac_sha384_ecb_null",
1577 .cra_blocksize = 1,
1578 .cra_flags = CRYPTO_ALG_ASYNC,
1579 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1580 .cra_priority = 4001,
1581 .cra_alignmask = 0,
1582 .cra_module = THIS_MODULE,
1583 },
1584 .init = otx2_cpt_aead_ecb_null_sha384_init,
1585 .exit = otx2_cpt_aead_exit,
1586 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1587 .setauthsize = otx2_cpt_aead_null_set_authsize,
1588 .encrypt = otx2_cpt_aead_null_encrypt,
1589 .decrypt = otx2_cpt_aead_null_decrypt,
1590 .ivsize = 0,
1591 .maxauthsize = SHA384_DIGEST_SIZE,
1592}, {
1593 .base = {
1594 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1595 .cra_driver_name = "cpt_hmac_sha512_ecb_null",
1596 .cra_blocksize = 1,
1597 .cra_flags = CRYPTO_ALG_ASYNC,
1598 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1599 .cra_priority = 4001,
1600 .cra_alignmask = 0,
1601 .cra_module = THIS_MODULE,
1602 },
1603 .init = otx2_cpt_aead_ecb_null_sha512_init,
1604 .exit = otx2_cpt_aead_exit,
1605 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1606 .setauthsize = otx2_cpt_aead_null_set_authsize,
1607 .encrypt = otx2_cpt_aead_null_encrypt,
1608 .decrypt = otx2_cpt_aead_null_decrypt,
1609 .ivsize = 0,
1610 .maxauthsize = SHA512_DIGEST_SIZE,
1611}, {
1612 .base = {
1613 .cra_name = "rfc4106(gcm(aes))",
1614 .cra_driver_name = "cpt_rfc4106_gcm_aes",
1615 .cra_blocksize = 1,
1616 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1617 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx),
1618 .cra_priority = 4001,
1619 .cra_alignmask = 0,
1620 .cra_module = THIS_MODULE,
1621 },
1622 .init = otx2_cpt_aead_gcm_aes_init,
1623 .exit = otx2_cpt_aead_exit,
1624 .setkey = otx2_cpt_aead_gcm_aes_setkey,
1625 .setauthsize = otx2_cpt_aead_gcm_set_authsize,
1626 .encrypt = otx2_cpt_aead_encrypt,
1627 .decrypt = otx2_cpt_aead_decrypt,
1628 .ivsize = AES_GCM_IV_SIZE,
1629 .maxauthsize = AES_GCM_ICV_SIZE,
1630} };
1631
1632static inline int cpt_register_algs(void)
1633{
1634 int i, err = 0;
1635
1636 if (!IS_ENABLED(CONFIG_DM_CRYPT)) {
1637 for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
1638 otx2_cpt_skciphers[i].base.cra_flags &=
1639 ~CRYPTO_ALG_DEAD;
1640
1641 err = crypto_register_skciphers(otx2_cpt_skciphers,
1642 ARRAY_SIZE(otx2_cpt_skciphers));
1643 if (err)
1644 return err;
1645 }
1646
1647 for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)
1648 otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1649
1650 err = crypto_register_aeads(otx2_cpt_aeads,
1651 ARRAY_SIZE(otx2_cpt_aeads));
1652 if (err) {
1653 crypto_unregister_skciphers(otx2_cpt_skciphers,
1654 ARRAY_SIZE(otx2_cpt_skciphers));
1655 return err;
1656 }
1657
1658 return 0;
1659}
1660
1661static inline void cpt_unregister_algs(void)
1662{
1663 crypto_unregister_skciphers(otx2_cpt_skciphers,
1664 ARRAY_SIZE(otx2_cpt_skciphers));
1665 crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads));
1666}
1667
1668static int compare_func(const void *lptr, const void *rptr)
1669{
1670 const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1671 const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1672
1673 if (ldesc->dev->devfn < rdesc->dev->devfn)
1674 return -1;
1675 if (ldesc->dev->devfn > rdesc->dev->devfn)
1676 return 1;
1677 return 0;
1678}
1679
1680static void swap_func(void *lptr, void *rptr, int size)
1681{
1682 struct cpt_device_desc *ldesc = lptr;
1683 struct cpt_device_desc *rdesc = rptr;
1684 struct cpt_device_desc desc;
1685
1686 desc = *ldesc;
1687 *ldesc = *rdesc;
1688 *rdesc = desc;
1689}
1690
1691int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1692 int num_queues, int num_devices)
1693{
1694 int ret = 0;
1695 int count;
1696
1697 mutex_lock(&mutex);
1698 count = atomic_read(&se_devices.count);
1699 if (count >= OTX2_CPT_MAX_LFS_NUM) {
1700 dev_err(&pdev->dev, "No space to add a new device\n");
1701 ret = -ENOSPC;
1702 goto unlock;
1703 }
1704 se_devices.desc[count].num_queues = num_queues;
1705 se_devices.desc[count++].dev = pdev;
1706 atomic_inc(&se_devices.count);
1707
1708 if (atomic_read(&se_devices.count) == num_devices &&
1709 is_crypto_registered == false) {
1710 if (cpt_register_algs()) {
1711 dev_err(&pdev->dev,
1712 "Error in registering crypto algorithms\n");
1713 ret = -EINVAL;
1714 goto unlock;
1715 }
1716 try_module_get(mod);
1717 is_crypto_registered = true;
1718 }
1719 sort(se_devices.desc, count, sizeof(struct cpt_device_desc),
1720 compare_func, swap_func);
1721
1722unlock:
1723 mutex_unlock(&mutex);
1724 return ret;
1725}
1726
1727void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod)
1728{
1729 struct cpt_device_table *dev_tbl;
1730 bool dev_found = false;
1731 int i, j, count;
1732
1733 mutex_lock(&mutex);
1734
1735 dev_tbl = &se_devices;
1736 count = atomic_read(&dev_tbl->count);
1737 for (i = 0; i < count; i++) {
1738 if (pdev == dev_tbl->desc[i].dev) {
1739 for (j = i; j < count-1; j++)
1740 dev_tbl->desc[j] = dev_tbl->desc[j+1];
1741 dev_found = true;
1742 break;
1743 }
1744 }
1745
1746 if (!dev_found) {
1747 dev_err(&pdev->dev, "%s device not found\n", __func__);
1748 goto unlock;
1749 }
1750 if (atomic_dec_and_test(&se_devices.count)) {
1751 cpt_unregister_algs();
1752 module_put(mod);
1753 is_crypto_registered = false;
1754 }
1755
1756unlock:
1757 mutex_unlock(&mutex);
1758}
1759
