1
2
3
4
5
6
7
8
9
10#include "compat.h"
11#include "ctrl.h"
12#include "regs.h"
13#include "intern.h"
14#include "desc_constr.h"
15#include "error.h"
16#include "sg_sw_qm.h"
17#include "key_gen.h"
18#include "qi.h"
19#include "jr.h"
20#include "caamalg_desc.h"
21
22
23
24
25#define CAAM_CRA_PRIORITY 2000
26
27#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
28 SHA512_DIGEST_SIZE * 2)
29
30#define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \
31 CAAM_MAX_KEY_SIZE)
32#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
33
34struct caam_alg_entry {
35 int class1_alg_type;
36 int class2_alg_type;
37 bool rfc3686;
38 bool geniv;
39 bool nodkp;
40};
41
42struct caam_aead_alg {
43 struct aead_alg aead;
44 struct caam_alg_entry caam;
45 bool registered;
46};
47
48struct caam_skcipher_alg {
49 struct skcipher_alg skcipher;
50 struct caam_alg_entry caam;
51 bool registered;
52};
53
54
55
56
57struct caam_ctx {
58 struct device *jrdev;
59 u32 sh_desc_enc[DESC_MAX_USED_LEN];
60 u32 sh_desc_dec[DESC_MAX_USED_LEN];
61 u8 key[CAAM_MAX_KEY_SIZE];
62 dma_addr_t key_dma;
63 enum dma_data_direction dir;
64 struct alginfo adata;
65 struct alginfo cdata;
66 unsigned int authsize;
67 struct device *qidev;
68 spinlock_t lock;
69 struct caam_drv_ctx *drv_ctx[NUM_OP];
70};
71
72static int aead_set_sh_desc(struct crypto_aead *aead)
73{
74 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
75 typeof(*alg), aead);
76 struct caam_ctx *ctx = crypto_aead_ctx(aead);
77 unsigned int ivsize = crypto_aead_ivsize(aead);
78 u32 ctx1_iv_off = 0;
79 u32 *nonce = NULL;
80 unsigned int data_len[2];
81 u32 inl_mask;
82 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
83 OP_ALG_AAI_CTR_MOD128);
84 const bool is_rfc3686 = alg->caam.rfc3686;
85 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
86
87 if (!ctx->cdata.keylen || !ctx->authsize)
88 return 0;
89
90
91
92
93
94
95 if (ctr_mode)
96 ctx1_iv_off = 16;
97
98
99
100
101
102 if (is_rfc3686) {
103 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
104 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
105 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
106 }
107
108 data_len[0] = ctx->adata.keylen_pad;
109 data_len[1] = ctx->cdata.keylen;
110
111 if (alg->caam.geniv)
112 goto skip_enc;
113
114
115 if (desc_inline_query(DESC_QI_AEAD_ENC_LEN +
116 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
117 DESC_JOB_IO_LEN, data_len, &inl_mask,
118 ARRAY_SIZE(data_len)) < 0)
119 return -EINVAL;
120
121 if (inl_mask & 1)
122 ctx->adata.key_virt = ctx->key;
123 else
124 ctx->adata.key_dma = ctx->key_dma;
125
126 if (inl_mask & 2)
127 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
128 else
129 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
130
131 ctx->adata.key_inline = !!(inl_mask & 1);
132 ctx->cdata.key_inline = !!(inl_mask & 2);
133
134 cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
135 ivsize, ctx->authsize, is_rfc3686, nonce,
136 ctx1_iv_off, true, ctrlpriv->era);
137
138skip_enc:
139
140 if (desc_inline_query(DESC_QI_AEAD_DEC_LEN +
141 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
142 DESC_JOB_IO_LEN, data_len, &inl_mask,
143 ARRAY_SIZE(data_len)) < 0)
144 return -EINVAL;
145
146 if (inl_mask & 1)
147 ctx->adata.key_virt = ctx->key;
148 else
149 ctx->adata.key_dma = ctx->key_dma;
150
151 if (inl_mask & 2)
152 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
153 else
154 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
155
156 ctx->adata.key_inline = !!(inl_mask & 1);
157 ctx->cdata.key_inline = !!(inl_mask & 2);
158
159 cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata,
160 ivsize, ctx->authsize, alg->caam.geniv,
161 is_rfc3686, nonce, ctx1_iv_off, true,
162 ctrlpriv->era);
163
164 if (!alg->caam.geniv)
165 goto skip_givenc;
166
167
168 if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN +
169 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
170 DESC_JOB_IO_LEN, data_len, &inl_mask,
171 ARRAY_SIZE(data_len)) < 0)
172 return -EINVAL;
173
174 if (inl_mask & 1)
175 ctx->adata.key_virt = ctx->key;
176 else
177 ctx->adata.key_dma = ctx->key_dma;
178
179 if (inl_mask & 2)
180 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
181 else
182 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
183
184 ctx->adata.key_inline = !!(inl_mask & 1);
185 ctx->cdata.key_inline = !!(inl_mask & 2);
186
187 cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata,
188 ivsize, ctx->authsize, is_rfc3686, nonce,
189 ctx1_iv_off, true, ctrlpriv->era);
190
191skip_givenc:
192 return 0;
193}
194
195static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
196{
197 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
198
199 ctx->authsize = authsize;
200 aead_set_sh_desc(authenc);
201
202 return 0;
203}
204
205static int aead_setkey(struct crypto_aead *aead, const u8 *key,
206 unsigned int keylen)
207{
208 struct caam_ctx *ctx = crypto_aead_ctx(aead);
209 struct device *jrdev = ctx->jrdev;
210 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
211 struct crypto_authenc_keys keys;
212 int ret = 0;
213
214 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
215 goto badkey;
216
217 dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
218 keys.authkeylen + keys.enckeylen, keys.enckeylen,
219 keys.authkeylen);
220 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
221 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
222
223
224
225
226
227 if (ctrlpriv->era >= 6) {
228 ctx->adata.keylen = keys.authkeylen;
229 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
230 OP_ALG_ALGSEL_MASK);
231
232 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
233 goto badkey;
234
235 memcpy(ctx->key, keys.authkey, keys.authkeylen);
236 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
237 keys.enckeylen);
238 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
239 ctx->adata.keylen_pad +
240 keys.enckeylen, ctx->dir);
241 goto skip_split_key;
242 }
243
244 ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey,
245 keys.authkeylen, CAAM_MAX_KEY_SIZE -
246 keys.enckeylen);
247 if (ret)
248 goto badkey;
249
250
251 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
252 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
253 ctx->adata.keylen_pad + keys.enckeylen,
254 ctx->dir);
255#ifdef DEBUG
256 print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ",
257 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
258 ctx->adata.keylen_pad + keys.enckeylen, 1);
259#endif
260
261skip_split_key:
262 ctx->cdata.keylen = keys.enckeylen;
263
264 ret = aead_set_sh_desc(aead);
265 if (ret)
266 goto badkey;
267
268
269 if (ctx->drv_ctx[ENCRYPT]) {
270 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
271 ctx->sh_desc_enc);
272 if (ret) {
273 dev_err(jrdev, "driver enc context update failed\n");
274 goto badkey;
275 }
276 }
277
278 if (ctx->drv_ctx[DECRYPT]) {
279 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
280 ctx->sh_desc_dec);
281 if (ret) {
282 dev_err(jrdev, "driver dec context update failed\n");
283 goto badkey;
284 }
285 }
286
287 memzero_explicit(&keys, sizeof(keys));
288 return ret;
289badkey:
290 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
291 memzero_explicit(&keys, sizeof(keys));
292 return -EINVAL;
293}
294
295static int des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
296 unsigned int keylen)
297{
298 struct crypto_authenc_keys keys;
299 u32 flags;
300 int err;
301
302 err = crypto_authenc_extractkeys(&keys, key, keylen);
303 if (unlikely(err))
304 goto badkey;
305
306 err = -EINVAL;
307 if (keys.enckeylen != DES3_EDE_KEY_SIZE)
308 goto badkey;
309
310 flags = crypto_aead_get_flags(aead);
311 err = __des3_verify_key(&flags, keys.enckey);
312 if (unlikely(err)) {
313 crypto_aead_set_flags(aead, flags);
314 goto out;
315 }
316
317 err = aead_setkey(aead, key, keylen);
318
319out:
320 memzero_explicit(&keys, sizeof(keys));
321 return err;
322
323badkey:
324 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
325 goto out;
326}
327
328static int gcm_set_sh_desc(struct crypto_aead *aead)
329{
330 struct caam_ctx *ctx = crypto_aead_ctx(aead);
331 unsigned int ivsize = crypto_aead_ivsize(aead);
332 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
333 ctx->cdata.keylen;
334
335 if (!ctx->cdata.keylen || !ctx->authsize)
336 return 0;
337
338
339
340
341
342 if (rem_bytes >= DESC_QI_GCM_ENC_LEN) {
343 ctx->cdata.key_inline = true;
344 ctx->cdata.key_virt = ctx->key;
345 } else {
346 ctx->cdata.key_inline = false;
347 ctx->cdata.key_dma = ctx->key_dma;
348 }
349
350 cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
351 ctx->authsize, true);
352
353
354
355
356
357 if (rem_bytes >= DESC_QI_GCM_DEC_LEN) {
358 ctx->cdata.key_inline = true;
359 ctx->cdata.key_virt = ctx->key;
360 } else {
361 ctx->cdata.key_inline = false;
362 ctx->cdata.key_dma = ctx->key_dma;
363 }
364
365 cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
366 ctx->authsize, true);
367
368 return 0;
369}
370
371static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
372{
373 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
374
375 ctx->authsize = authsize;
376 gcm_set_sh_desc(authenc);
377
378 return 0;
379}
380
381static int gcm_setkey(struct crypto_aead *aead,
382 const u8 *key, unsigned int keylen)
383{
384 struct caam_ctx *ctx = crypto_aead_ctx(aead);
385 struct device *jrdev = ctx->jrdev;
386 int ret;
387
388 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
389 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
390
391 memcpy(ctx->key, key, keylen);
392 dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen,
393 ctx->dir);
394 ctx->cdata.keylen = keylen;
395
396 ret = gcm_set_sh_desc(aead);
397 if (ret)
398 return ret;
399
400
401 if (ctx->drv_ctx[ENCRYPT]) {
402 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
403 ctx->sh_desc_enc);
404 if (ret) {
405 dev_err(jrdev, "driver enc context update failed\n");
406 return ret;
407 }
408 }
409
410 if (ctx->drv_ctx[DECRYPT]) {
411 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
412 ctx->sh_desc_dec);
413 if (ret) {
414 dev_err(jrdev, "driver dec context update failed\n");
415 return ret;
416 }
417 }
418
419 return 0;
420}
421
422static int rfc4106_set_sh_desc(struct crypto_aead *aead)
423{
424 struct caam_ctx *ctx = crypto_aead_ctx(aead);
425 unsigned int ivsize = crypto_aead_ivsize(aead);
426 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
427 ctx->cdata.keylen;
428
429 if (!ctx->cdata.keylen || !ctx->authsize)
430 return 0;
431
432 ctx->cdata.key_virt = ctx->key;
433
434
435
436
437
438 if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) {
439 ctx->cdata.key_inline = true;
440 } else {
441 ctx->cdata.key_inline = false;
442 ctx->cdata.key_dma = ctx->key_dma;
443 }
444
445 cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
446 ctx->authsize, true);
447
448
449
450
451
452 if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) {
453 ctx->cdata.key_inline = true;
454 } else {
455 ctx->cdata.key_inline = false;
456 ctx->cdata.key_dma = ctx->key_dma;
457 }
458
459 cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
460 ctx->authsize, true);
461
462 return 0;
463}
464
465static int rfc4106_setauthsize(struct crypto_aead *authenc,
466 unsigned int authsize)
467{
468 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
469
470 ctx->authsize = authsize;
471 rfc4106_set_sh_desc(authenc);
472
473 return 0;
474}
475
476static int rfc4106_setkey(struct crypto_aead *aead,
477 const u8 *key, unsigned int keylen)
478{
479 struct caam_ctx *ctx = crypto_aead_ctx(aead);
480 struct device *jrdev = ctx->jrdev;
481 int ret;
482
483 if (keylen < 4)
484 return -EINVAL;
485
486 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
487 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
488
489 memcpy(ctx->key, key, keylen);
490
491
492
493
494 ctx->cdata.keylen = keylen - 4;
495 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
496 ctx->cdata.keylen, ctx->dir);
497
498 ret = rfc4106_set_sh_desc(aead);
499 if (ret)
500 return ret;
501
502
503 if (ctx->drv_ctx[ENCRYPT]) {
504 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
505 ctx->sh_desc_enc);
506 if (ret) {
507 dev_err(jrdev, "driver enc context update failed\n");
508 return ret;
509 }
510 }
511
512 if (ctx->drv_ctx[DECRYPT]) {
513 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
514 ctx->sh_desc_dec);
515 if (ret) {
516 dev_err(jrdev, "driver dec context update failed\n");
517 return ret;
518 }
519 }
520
521 return 0;
522}
523
524static int rfc4543_set_sh_desc(struct crypto_aead *aead)
525{
526 struct caam_ctx *ctx = crypto_aead_ctx(aead);
527 unsigned int ivsize = crypto_aead_ivsize(aead);
528 int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN -
529 ctx->cdata.keylen;
530
531 if (!ctx->cdata.keylen || !ctx->authsize)
532 return 0;
533
534 ctx->cdata.key_virt = ctx->key;
535
536
537
538
539
540 if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) {
541 ctx->cdata.key_inline = true;
542 } else {
543 ctx->cdata.key_inline = false;
544 ctx->cdata.key_dma = ctx->key_dma;
545 }
546
547 cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
548 ctx->authsize, true);
549
550
551
552
553
554 if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) {
555 ctx->cdata.key_inline = true;
556 } else {
557 ctx->cdata.key_inline = false;
558 ctx->cdata.key_dma = ctx->key_dma;
559 }
560
561 cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
562 ctx->authsize, true);
563
564 return 0;
565}
566
567static int rfc4543_setauthsize(struct crypto_aead *authenc,
568 unsigned int authsize)
569{
570 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
571
572 ctx->authsize = authsize;
573 rfc4543_set_sh_desc(authenc);
574
575 return 0;
576}
577
578static int rfc4543_setkey(struct crypto_aead *aead,
579 const u8 *key, unsigned int keylen)
580{
581 struct caam_ctx *ctx = crypto_aead_ctx(aead);
582 struct device *jrdev = ctx->jrdev;
583 int ret;
584
585 if (keylen < 4)
586 return -EINVAL;
587
588 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
589 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
590
591 memcpy(ctx->key, key, keylen);
592
593
594
595
596 ctx->cdata.keylen = keylen - 4;
597 dma_sync_single_for_device(jrdev->parent, ctx->key_dma,
598 ctx->cdata.keylen, ctx->dir);
599
600 ret = rfc4543_set_sh_desc(aead);
601 if (ret)
602 return ret;
603
604
605 if (ctx->drv_ctx[ENCRYPT]) {
606 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
607 ctx->sh_desc_enc);
608 if (ret) {
609 dev_err(jrdev, "driver enc context update failed\n");
610 return ret;
611 }
612 }
613
614 if (ctx->drv_ctx[DECRYPT]) {
615 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
616 ctx->sh_desc_dec);
617 if (ret) {
618 dev_err(jrdev, "driver dec context update failed\n");
619 return ret;
620 }
621 }
622
623 return 0;
624}
625
626static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
627 unsigned int keylen)
628{
629 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
630 struct caam_skcipher_alg *alg =
631 container_of(crypto_skcipher_alg(skcipher), typeof(*alg),
632 skcipher);
633 struct device *jrdev = ctx->jrdev;
634 unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
635 u32 ctx1_iv_off = 0;
636 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
637 OP_ALG_AAI_CTR_MOD128);
638 const bool is_rfc3686 = alg->caam.rfc3686;
639 int ret = 0;
640
641 print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
642 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
643
644
645
646
647
648
649 if (ctr_mode)
650 ctx1_iv_off = 16;
651
652
653
654
655
656
657 if (is_rfc3686) {
658 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
659 keylen -= CTR_RFC3686_NONCE_SIZE;
660 }
661
662 ctx->cdata.keylen = keylen;
663 ctx->cdata.key_virt = key;
664 ctx->cdata.key_inline = true;
665
666
667 cnstr_shdsc_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize,
668 is_rfc3686, ctx1_iv_off);
669 cnstr_shdsc_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize,
670 is_rfc3686, ctx1_iv_off);
671
672
673 if (ctx->drv_ctx[ENCRYPT]) {
674 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
675 ctx->sh_desc_enc);
676 if (ret) {
677 dev_err(jrdev, "driver enc context update failed\n");
678 goto badkey;
679 }
680 }
681
682 if (ctx->drv_ctx[DECRYPT]) {
683 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
684 ctx->sh_desc_dec);
685 if (ret) {
686 dev_err(jrdev, "driver dec context update failed\n");
687 goto badkey;
688 }
689 }
690
691 return ret;
692badkey:
693 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
694 return -EINVAL;
695}
696
697static int des3_skcipher_setkey(struct crypto_skcipher *skcipher,
698 const u8 *key, unsigned int keylen)
699{
700 return unlikely(des3_verify_key(skcipher, key)) ?:
701 skcipher_setkey(skcipher, key, keylen);
702}
703
704static int xts_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
705 unsigned int keylen)
706{
707 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
708 struct device *jrdev = ctx->jrdev;
709 int ret = 0;
710
711 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
712 dev_err(jrdev, "key size mismatch\n");
713 goto badkey;
714 }
715
716 ctx->cdata.keylen = keylen;
717 ctx->cdata.key_virt = key;
718 ctx->cdata.key_inline = true;
719
720
721 cnstr_shdsc_xts_skcipher_encap(ctx->sh_desc_enc, &ctx->cdata);
722 cnstr_shdsc_xts_skcipher_decap(ctx->sh_desc_dec, &ctx->cdata);
723
724
725 if (ctx->drv_ctx[ENCRYPT]) {
726 ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT],
727 ctx->sh_desc_enc);
728 if (ret) {
729 dev_err(jrdev, "driver enc context update failed\n");
730 goto badkey;
731 }
732 }
733
734 if (ctx->drv_ctx[DECRYPT]) {
735 ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT],
736 ctx->sh_desc_dec);
737 if (ret) {
738 dev_err(jrdev, "driver dec context update failed\n");
739 goto badkey;
740 }
741 }
742
743 return ret;
744badkey:
745 crypto_skcipher_set_flags(skcipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
746 return -EINVAL;
747}
748
749
750
751
752
753
754
755
756
757
758
759
760
761struct aead_edesc {
762 int src_nents;
763 int dst_nents;
764 dma_addr_t iv_dma;
765 int qm_sg_bytes;
766 dma_addr_t qm_sg_dma;
767 unsigned int assoclen;
768 dma_addr_t assoclen_dma;
769 struct caam_drv_req drv_req;
770 struct qm_sg_entry sgt[0];
771};
772
773
774
775
776
777
778
779
780
781
782
783struct skcipher_edesc {
784 int src_nents;
785 int dst_nents;
786 dma_addr_t iv_dma;
787 int qm_sg_bytes;
788 dma_addr_t qm_sg_dma;
789 struct caam_drv_req drv_req;
790 struct qm_sg_entry sgt[0];
791};
792
793static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx,
794 enum optype type)
795{
796
797
798
799
800
801 struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type];
802 u32 *desc;
803
804 if (unlikely(!drv_ctx)) {
805 spin_lock(&ctx->lock);
806
807
808 drv_ctx = ctx->drv_ctx[type];
809 if (!drv_ctx) {
810 int cpu;
811
812 if (type == ENCRYPT)
813 desc = ctx->sh_desc_enc;
814 else
815 desc = ctx->sh_desc_dec;
816
817 cpu = smp_processor_id();
818 drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc);
819 if (!IS_ERR_OR_NULL(drv_ctx))
820 drv_ctx->op_type = type;
821
822 ctx->drv_ctx[type] = drv_ctx;
823 }
824
825 spin_unlock(&ctx->lock);
826 }
827
828 return drv_ctx;
829}
830
831static void caam_unmap(struct device *dev, struct scatterlist *src,
832 struct scatterlist *dst, int src_nents,
833 int dst_nents, dma_addr_t iv_dma, int ivsize,
834 enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma,
835 int qm_sg_bytes)
836{
837 if (dst != src) {
838 if (src_nents)
839 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
840 if (dst_nents)
841 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
842 } else {
843 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
844 }
845
846 if (iv_dma)
847 dma_unmap_single(dev, iv_dma, ivsize, iv_dir);
848 if (qm_sg_bytes)
849 dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE);
850}
851
852static void aead_unmap(struct device *dev,
853 struct aead_edesc *edesc,
854 struct aead_request *req)
855{
856 struct crypto_aead *aead = crypto_aead_reqtfm(req);
857 int ivsize = crypto_aead_ivsize(aead);
858
859 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
860 edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma,
861 edesc->qm_sg_bytes);
862 dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
863}
864
865static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc,
866 struct skcipher_request *req)
867{
868 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
869 int ivsize = crypto_skcipher_ivsize(skcipher);
870
871 caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents,
872 edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma,
873 edesc->qm_sg_bytes);
874}
875
876static void aead_done(struct caam_drv_req *drv_req, u32 status)
877{
878 struct device *qidev;
879 struct aead_edesc *edesc;
880 struct aead_request *aead_req = drv_req->app_ctx;
881 struct crypto_aead *aead = crypto_aead_reqtfm(aead_req);
882 struct caam_ctx *caam_ctx = crypto_aead_ctx(aead);
883 int ecode = 0;
884
885 qidev = caam_ctx->qidev;
886
887 if (unlikely(status)) {
888 u32 ssrc = status & JRSTA_SSRC_MASK;
889 u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
890
891 caam_jr_strstatus(qidev, status);
892
893
894
895 if (ssrc == JRSTA_SSRC_CCB_ERROR &&
896 err_id == JRSTA_CCBERR_ERRID_ICVCHK)
897 ecode = -EBADMSG;
898 else
899 ecode = -EIO;
900 }
901
902 edesc = container_of(drv_req, typeof(*edesc), drv_req);
903 aead_unmap(qidev, edesc, aead_req);
904
905 aead_request_complete(aead_req, ecode);
906 qi_cache_free(edesc);
907}
908
909
910
911
912static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
913 bool encrypt)
914{
915 struct crypto_aead *aead = crypto_aead_reqtfm(req);
916 struct caam_ctx *ctx = crypto_aead_ctx(aead);
917 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
918 typeof(*alg), aead);
919 struct device *qidev = ctx->qidev;
920 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
921 GFP_KERNEL : GFP_ATOMIC;
922 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
923 int src_len, dst_len = 0;
924 struct aead_edesc *edesc;
925 dma_addr_t qm_sg_dma, iv_dma = 0;
926 int ivsize = 0;
927 unsigned int authsize = ctx->authsize;
928 int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes;
929 int in_len, out_len;
930 struct qm_sg_entry *sg_table, *fd_sgt;
931 struct caam_drv_ctx *drv_ctx;
932
933 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
934 if (IS_ERR_OR_NULL(drv_ctx))
935 return (struct aead_edesc *)drv_ctx;
936
937
938 edesc = qi_cache_alloc(GFP_DMA | flags);
939 if (unlikely(!edesc)) {
940 dev_err(qidev, "could not allocate extended descriptor\n");
941 return ERR_PTR(-ENOMEM);
942 }
943
944 if (likely(req->src == req->dst)) {
945 src_len = req->assoclen + req->cryptlen +
946 (encrypt ? authsize : 0);
947
948 src_nents = sg_nents_for_len(req->src, src_len);
949 if (unlikely(src_nents < 0)) {
950 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
951 src_len);
952 qi_cache_free(edesc);
953 return ERR_PTR(src_nents);
954 }
955
956 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
957 DMA_BIDIRECTIONAL);
958 if (unlikely(!mapped_src_nents)) {
959 dev_err(qidev, "unable to map source\n");
960 qi_cache_free(edesc);
961 return ERR_PTR(-ENOMEM);
962 }
963 } else {
964 src_len = req->assoclen + req->cryptlen;
965 dst_len = src_len + (encrypt ? authsize : (-authsize));
966
967 src_nents = sg_nents_for_len(req->src, src_len);
968 if (unlikely(src_nents < 0)) {
969 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
970 src_len);
971 qi_cache_free(edesc);
972 return ERR_PTR(src_nents);
973 }
974
975 dst_nents = sg_nents_for_len(req->dst, dst_len);
976 if (unlikely(dst_nents < 0)) {
977 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
978 dst_len);
979 qi_cache_free(edesc);
980 return ERR_PTR(dst_nents);
981 }
982
983 if (src_nents) {
984 mapped_src_nents = dma_map_sg(qidev, req->src,
985 src_nents, DMA_TO_DEVICE);
986 if (unlikely(!mapped_src_nents)) {
987 dev_err(qidev, "unable to map source\n");
988 qi_cache_free(edesc);
989 return ERR_PTR(-ENOMEM);
990 }
991 } else {
992 mapped_src_nents = 0;
993 }
994
995 if (dst_nents) {
996 mapped_dst_nents = dma_map_sg(qidev, req->dst,
997 dst_nents,
998 DMA_FROM_DEVICE);
999 if (unlikely(!mapped_dst_nents)) {
1000 dev_err(qidev, "unable to map destination\n");
1001 dma_unmap_sg(qidev, req->src, src_nents,
1002 DMA_TO_DEVICE);
1003 qi_cache_free(edesc);
1004 return ERR_PTR(-ENOMEM);
1005 }
1006 } else {
1007 mapped_dst_nents = 0;
1008 }
1009 }
1010
1011 if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv)
1012 ivsize = crypto_aead_ivsize(aead);
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026 qm_sg_ents = 1 + !!ivsize + mapped_src_nents;
1027 if (mapped_dst_nents > 1)
1028 qm_sg_ents += pad_sg_nents(mapped_dst_nents);
1029 else if ((req->src == req->dst) && (mapped_src_nents > 1))
1030 qm_sg_ents = max(pad_sg_nents(qm_sg_ents),
1031 1 + !!ivsize + pad_sg_nents(mapped_src_nents));
1032 else
1033 qm_sg_ents = pad_sg_nents(qm_sg_ents);
1034
1035 sg_table = &edesc->sgt[0];
1036 qm_sg_bytes = qm_sg_ents * sizeof(*sg_table);
1037 if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize >
1038 CAAM_QI_MEMCACHE_SIZE)) {
1039 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1040 qm_sg_ents, ivsize);
1041 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1042 0, DMA_NONE, 0, 0);
1043 qi_cache_free(edesc);
1044 return ERR_PTR(-ENOMEM);
1045 }
1046
1047 if (ivsize) {
1048 u8 *iv = (u8 *)(sg_table + qm_sg_ents);
1049
1050
1051 memcpy(iv, req->iv, ivsize);
1052
1053 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE);
1054 if (dma_mapping_error(qidev, iv_dma)) {
1055 dev_err(qidev, "unable to map IV\n");
1056 caam_unmap(qidev, req->src, req->dst, src_nents,
1057 dst_nents, 0, 0, DMA_NONE, 0, 0);
1058 qi_cache_free(edesc);
1059 return ERR_PTR(-ENOMEM);
1060 }
1061 }
1062
1063 edesc->src_nents = src_nents;
1064 edesc->dst_nents = dst_nents;
1065 edesc->iv_dma = iv_dma;
1066 edesc->drv_req.app_ctx = req;
1067 edesc->drv_req.cbk = aead_done;
1068 edesc->drv_req.drv_ctx = drv_ctx;
1069
1070 edesc->assoclen = cpu_to_caam32(req->assoclen);
1071 edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4,
1072 DMA_TO_DEVICE);
1073 if (dma_mapping_error(qidev, edesc->assoclen_dma)) {
1074 dev_err(qidev, "unable to map assoclen\n");
1075 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1076 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
1077 qi_cache_free(edesc);
1078 return ERR_PTR(-ENOMEM);
1079 }
1080
1081 dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0);
1082 qm_sg_index++;
1083 if (ivsize) {
1084 dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0);
1085 qm_sg_index++;
1086 }
1087 sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0);
1088 qm_sg_index += mapped_src_nents;
1089
1090 if (mapped_dst_nents > 1)
1091 sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0);
1092
1093 qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE);
1094 if (dma_mapping_error(qidev, qm_sg_dma)) {
1095 dev_err(qidev, "unable to map S/G table\n");
1096 dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE);
1097 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1098 iv_dma, ivsize, DMA_TO_DEVICE, 0, 0);
1099 qi_cache_free(edesc);
1100 return ERR_PTR(-ENOMEM);
1101 }
1102
1103 edesc->qm_sg_dma = qm_sg_dma;
1104 edesc->qm_sg_bytes = qm_sg_bytes;
1105
1106 out_len = req->assoclen + req->cryptlen +
1107 (encrypt ? ctx->authsize : (-ctx->authsize));
1108 in_len = 4 + ivsize + req->assoclen + req->cryptlen;
1109
1110 fd_sgt = &edesc->drv_req.fd_sgt[0];
1111 dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0);
1112
1113 if (req->dst == req->src) {
1114 if (mapped_src_nents == 1)
1115 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src),
1116 out_len, 0);
1117 else
1118 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma +
1119 (1 + !!ivsize) * sizeof(*sg_table),
1120 out_len, 0);
1121 } else if (mapped_dst_nents <= 1) {
1122 dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len,
1123 0);
1124 } else {
1125 dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) *
1126 qm_sg_index, out_len, 0);
1127 }
1128
1129 return edesc;
1130}
1131
1132static inline int aead_crypt(struct aead_request *req, bool encrypt)
1133{
1134 struct aead_edesc *edesc;
1135 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1136 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1137 int ret;
1138
1139 if (unlikely(caam_congested))
1140 return -EAGAIN;
1141
1142
1143 edesc = aead_edesc_alloc(req, encrypt);
1144 if (IS_ERR_OR_NULL(edesc))
1145 return PTR_ERR(edesc);
1146
1147
1148 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1149 if (!ret) {
1150 ret = -EINPROGRESS;
1151 } else {
1152 aead_unmap(ctx->qidev, edesc, req);
1153 qi_cache_free(edesc);
1154 }
1155
1156 return ret;
1157}
1158
1159static int aead_encrypt(struct aead_request *req)
1160{
1161 return aead_crypt(req, true);
1162}
1163
1164static int aead_decrypt(struct aead_request *req)
1165{
1166 return aead_crypt(req, false);
1167}
1168
1169static int ipsec_gcm_encrypt(struct aead_request *req)
1170{
1171 if (req->assoclen < 8)
1172 return -EINVAL;
1173
1174 return aead_crypt(req, true);
1175}
1176
1177static int ipsec_gcm_decrypt(struct aead_request *req)
1178{
1179 if (req->assoclen < 8)
1180 return -EINVAL;
1181
1182 return aead_crypt(req, false);
1183}
1184
1185static void skcipher_done(struct caam_drv_req *drv_req, u32 status)
1186{
1187 struct skcipher_edesc *edesc;
1188 struct skcipher_request *req = drv_req->app_ctx;
1189 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1190 struct caam_ctx *caam_ctx = crypto_skcipher_ctx(skcipher);
1191 struct device *qidev = caam_ctx->qidev;
1192 int ivsize = crypto_skcipher_ivsize(skcipher);
1193
1194 dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status);
1195
1196 edesc = container_of(drv_req, typeof(*edesc), drv_req);
1197
1198 if (status)
1199 caam_jr_strstatus(qidev, status);
1200
1201 print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ",
1202 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
1203 edesc->src_nents > 1 ? 100 : ivsize, 1);
1204 caam_dump_sg("dst @" __stringify(__LINE__)": ",
1205 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
1206 edesc->dst_nents > 1 ? 100 : req->cryptlen, 1);
1207
1208 skcipher_unmap(qidev, edesc, req);
1209
1210
1211
1212
1213
1214
1215 memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize);
1216
1217 qi_cache_free(edesc);
1218 skcipher_request_complete(req, status);
1219}
1220
1221static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req,
1222 bool encrypt)
1223{
1224 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1225 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1226 struct device *qidev = ctx->qidev;
1227 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1228 GFP_KERNEL : GFP_ATOMIC;
1229 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1230 struct skcipher_edesc *edesc;
1231 dma_addr_t iv_dma;
1232 u8 *iv;
1233 int ivsize = crypto_skcipher_ivsize(skcipher);
1234 int dst_sg_idx, qm_sg_ents, qm_sg_bytes;
1235 struct qm_sg_entry *sg_table, *fd_sgt;
1236 struct caam_drv_ctx *drv_ctx;
1237
1238 drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT);
1239 if (IS_ERR_OR_NULL(drv_ctx))
1240 return (struct skcipher_edesc *)drv_ctx;
1241
1242 src_nents = sg_nents_for_len(req->src, req->cryptlen);
1243 if (unlikely(src_nents < 0)) {
1244 dev_err(qidev, "Insufficient bytes (%d) in src S/G\n",
1245 req->cryptlen);
1246 return ERR_PTR(src_nents);
1247 }
1248
1249 if (unlikely(req->src != req->dst)) {
1250 dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
1251 if (unlikely(dst_nents < 0)) {
1252 dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n",
1253 req->cryptlen);
1254 return ERR_PTR(dst_nents);
1255 }
1256
1257 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1258 DMA_TO_DEVICE);
1259 if (unlikely(!mapped_src_nents)) {
1260 dev_err(qidev, "unable to map source\n");
1261 return ERR_PTR(-ENOMEM);
1262 }
1263
1264 mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents,
1265 DMA_FROM_DEVICE);
1266 if (unlikely(!mapped_dst_nents)) {
1267 dev_err(qidev, "unable to map destination\n");
1268 dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE);
1269 return ERR_PTR(-ENOMEM);
1270 }
1271 } else {
1272 mapped_src_nents = dma_map_sg(qidev, req->src, src_nents,
1273 DMA_BIDIRECTIONAL);
1274 if (unlikely(!mapped_src_nents)) {
1275 dev_err(qidev, "unable to map source\n");
1276 return ERR_PTR(-ENOMEM);
1277 }
1278 }
1279
1280 qm_sg_ents = 1 + mapped_src_nents;
1281 dst_sg_idx = qm_sg_ents;
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291 if (req->src != req->dst)
1292 qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1);
1293 else
1294 qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents);
1295
1296 qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry);
1297 if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes +
1298 ivsize > CAAM_QI_MEMCACHE_SIZE)) {
1299 dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n",
1300 qm_sg_ents, ivsize);
1301 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1302 0, DMA_NONE, 0, 0);
1303 return ERR_PTR(-ENOMEM);
1304 }
1305
1306
1307 edesc = qi_cache_alloc(GFP_DMA | flags);
1308 if (unlikely(!edesc)) {
1309 dev_err(qidev, "could not allocate extended descriptor\n");
1310 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1311 0, DMA_NONE, 0, 0);
1312 return ERR_PTR(-ENOMEM);
1313 }
1314
1315
1316 sg_table = &edesc->sgt[0];
1317 iv = (u8 *)(sg_table + qm_sg_ents);
1318 memcpy(iv, req->iv, ivsize);
1319
1320 iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL);
1321 if (dma_mapping_error(qidev, iv_dma)) {
1322 dev_err(qidev, "unable to map IV\n");
1323 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0,
1324 0, DMA_NONE, 0, 0);
1325 qi_cache_free(edesc);
1326 return ERR_PTR(-ENOMEM);
1327 }
1328
1329 edesc->src_nents = src_nents;
1330 edesc->dst_nents = dst_nents;
1331 edesc->iv_dma = iv_dma;
1332 edesc->qm_sg_bytes = qm_sg_bytes;
1333 edesc->drv_req.app_ctx = req;
1334 edesc->drv_req.cbk = skcipher_done;
1335 edesc->drv_req.drv_ctx = drv_ctx;
1336
1337 dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0);
1338 sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0);
1339
1340 if (req->src != req->dst)
1341 sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0);
1342
1343 dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma,
1344 ivsize, 0);
1345
1346 edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes,
1347 DMA_TO_DEVICE);
1348 if (dma_mapping_error(qidev, edesc->qm_sg_dma)) {
1349 dev_err(qidev, "unable to map S/G table\n");
1350 caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents,
1351 iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0);
1352 qi_cache_free(edesc);
1353 return ERR_PTR(-ENOMEM);
1354 }
1355
1356 fd_sgt = &edesc->drv_req.fd_sgt[0];
1357
1358 dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma,
1359 ivsize + req->cryptlen, 0);
1360
1361 if (req->src == req->dst)
1362 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma +
1363 sizeof(*sg_table), req->cryptlen + ivsize,
1364 0);
1365 else
1366 dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx *
1367 sizeof(*sg_table), req->cryptlen + ivsize,
1368 0);
1369
1370 return edesc;
1371}
1372
1373static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt)
1374{
1375 struct skcipher_edesc *edesc;
1376 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
1377 struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher);
1378 int ret;
1379
1380 if (unlikely(caam_congested))
1381 return -EAGAIN;
1382
1383
1384 edesc = skcipher_edesc_alloc(req, encrypt);
1385 if (IS_ERR(edesc))
1386 return PTR_ERR(edesc);
1387
1388 ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req);
1389 if (!ret) {
1390 ret = -EINPROGRESS;
1391 } else {
1392 skcipher_unmap(ctx->qidev, edesc, req);
1393 qi_cache_free(edesc);
1394 }
1395
1396 return ret;
1397}
1398
1399static int skcipher_encrypt(struct skcipher_request *req)
1400{
1401 return skcipher_crypt(req, true);
1402}
1403
1404static int skcipher_decrypt(struct skcipher_request *req)
1405{
1406 return skcipher_crypt(req, false);
1407}
1408
1409static struct caam_skcipher_alg driver_algs[] = {
1410 {
1411 .skcipher = {
1412 .base = {
1413 .cra_name = "cbc(aes)",
1414 .cra_driver_name = "cbc-aes-caam-qi",
1415 .cra_blocksize = AES_BLOCK_SIZE,
1416 },
1417 .setkey = skcipher_setkey,
1418 .encrypt = skcipher_encrypt,
1419 .decrypt = skcipher_decrypt,
1420 .min_keysize = AES_MIN_KEY_SIZE,
1421 .max_keysize = AES_MAX_KEY_SIZE,
1422 .ivsize = AES_BLOCK_SIZE,
1423 },
1424 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1425 },
1426 {
1427 .skcipher = {
1428 .base = {
1429 .cra_name = "cbc(des3_ede)",
1430 .cra_driver_name = "cbc-3des-caam-qi",
1431 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1432 },
1433 .setkey = des3_skcipher_setkey,
1434 .encrypt = skcipher_encrypt,
1435 .decrypt = skcipher_decrypt,
1436 .min_keysize = DES3_EDE_KEY_SIZE,
1437 .max_keysize = DES3_EDE_KEY_SIZE,
1438 .ivsize = DES3_EDE_BLOCK_SIZE,
1439 },
1440 .caam.class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1441 },
1442 {
1443 .skcipher = {
1444 .base = {
1445 .cra_name = "cbc(des)",
1446 .cra_driver_name = "cbc-des-caam-qi",
1447 .cra_blocksize = DES_BLOCK_SIZE,
1448 },
1449 .setkey = skcipher_setkey,
1450 .encrypt = skcipher_encrypt,
1451 .decrypt = skcipher_decrypt,
1452 .min_keysize = DES_KEY_SIZE,
1453 .max_keysize = DES_KEY_SIZE,
1454 .ivsize = DES_BLOCK_SIZE,
1455 },
1456 .caam.class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1457 },
1458 {
1459 .skcipher = {
1460 .base = {
1461 .cra_name = "ctr(aes)",
1462 .cra_driver_name = "ctr-aes-caam-qi",
1463 .cra_blocksize = 1,
1464 },
1465 .setkey = skcipher_setkey,
1466 .encrypt = skcipher_encrypt,
1467 .decrypt = skcipher_decrypt,
1468 .min_keysize = AES_MIN_KEY_SIZE,
1469 .max_keysize = AES_MAX_KEY_SIZE,
1470 .ivsize = AES_BLOCK_SIZE,
1471 .chunksize = AES_BLOCK_SIZE,
1472 },
1473 .caam.class1_alg_type = OP_ALG_ALGSEL_AES |
1474 OP_ALG_AAI_CTR_MOD128,
1475 },
1476 {
1477 .skcipher = {
1478 .base = {
1479 .cra_name = "rfc3686(ctr(aes))",
1480 .cra_driver_name = "rfc3686-ctr-aes-caam-qi",
1481 .cra_blocksize = 1,
1482 },
1483 .setkey = skcipher_setkey,
1484 .encrypt = skcipher_encrypt,
1485 .decrypt = skcipher_decrypt,
1486 .min_keysize = AES_MIN_KEY_SIZE +
1487 CTR_RFC3686_NONCE_SIZE,
1488 .max_keysize = AES_MAX_KEY_SIZE +
1489 CTR_RFC3686_NONCE_SIZE,
1490 .ivsize = CTR_RFC3686_IV_SIZE,
1491 .chunksize = AES_BLOCK_SIZE,
1492 },
1493 .caam = {
1494 .class1_alg_type = OP_ALG_ALGSEL_AES |
1495 OP_ALG_AAI_CTR_MOD128,
1496 .rfc3686 = true,
1497 },
1498 },
1499 {
1500 .skcipher = {
1501 .base = {
1502 .cra_name = "xts(aes)",
1503 .cra_driver_name = "xts-aes-caam-qi",
1504 .cra_blocksize = AES_BLOCK_SIZE,
1505 },
1506 .setkey = xts_skcipher_setkey,
1507 .encrypt = skcipher_encrypt,
1508 .decrypt = skcipher_decrypt,
1509 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1510 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1511 .ivsize = AES_BLOCK_SIZE,
1512 },
1513 .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1514 },
1515};
1516
1517static struct caam_aead_alg driver_aeads[] = {
1518 {
1519 .aead = {
1520 .base = {
1521 .cra_name = "rfc4106(gcm(aes))",
1522 .cra_driver_name = "rfc4106-gcm-aes-caam-qi",
1523 .cra_blocksize = 1,
1524 },
1525 .setkey = rfc4106_setkey,
1526 .setauthsize = rfc4106_setauthsize,
1527 .encrypt = ipsec_gcm_encrypt,
1528 .decrypt = ipsec_gcm_decrypt,
1529 .ivsize = 8,
1530 .maxauthsize = AES_BLOCK_SIZE,
1531 },
1532 .caam = {
1533 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1534 .nodkp = true,
1535 },
1536 },
1537 {
1538 .aead = {
1539 .base = {
1540 .cra_name = "rfc4543(gcm(aes))",
1541 .cra_driver_name = "rfc4543-gcm-aes-caam-qi",
1542 .cra_blocksize = 1,
1543 },
1544 .setkey = rfc4543_setkey,
1545 .setauthsize = rfc4543_setauthsize,
1546 .encrypt = ipsec_gcm_encrypt,
1547 .decrypt = ipsec_gcm_decrypt,
1548 .ivsize = 8,
1549 .maxauthsize = AES_BLOCK_SIZE,
1550 },
1551 .caam = {
1552 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1553 .nodkp = true,
1554 },
1555 },
1556
1557 {
1558 .aead = {
1559 .base = {
1560 .cra_name = "gcm(aes)",
1561 .cra_driver_name = "gcm-aes-caam-qi",
1562 .cra_blocksize = 1,
1563 },
1564 .setkey = gcm_setkey,
1565 .setauthsize = gcm_setauthsize,
1566 .encrypt = aead_encrypt,
1567 .decrypt = aead_decrypt,
1568 .ivsize = 12,
1569 .maxauthsize = AES_BLOCK_SIZE,
1570 },
1571 .caam = {
1572 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1573 .nodkp = true,
1574 }
1575 },
1576
1577 {
1578 .aead = {
1579 .base = {
1580 .cra_name = "authenc(hmac(md5),cbc(aes))",
1581 .cra_driver_name = "authenc-hmac-md5-"
1582 "cbc-aes-caam-qi",
1583 .cra_blocksize = AES_BLOCK_SIZE,
1584 },
1585 .setkey = aead_setkey,
1586 .setauthsize = aead_setauthsize,
1587 .encrypt = aead_encrypt,
1588 .decrypt = aead_decrypt,
1589 .ivsize = AES_BLOCK_SIZE,
1590 .maxauthsize = MD5_DIGEST_SIZE,
1591 },
1592 .caam = {
1593 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1594 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1595 OP_ALG_AAI_HMAC_PRECOMP,
1596 }
1597 },
1598 {
1599 .aead = {
1600 .base = {
1601 .cra_name = "echainiv(authenc(hmac(md5),"
1602 "cbc(aes)))",
1603 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1604 "cbc-aes-caam-qi",
1605 .cra_blocksize = AES_BLOCK_SIZE,
1606 },
1607 .setkey = aead_setkey,
1608 .setauthsize = aead_setauthsize,
1609 .encrypt = aead_encrypt,
1610 .decrypt = aead_decrypt,
1611 .ivsize = AES_BLOCK_SIZE,
1612 .maxauthsize = MD5_DIGEST_SIZE,
1613 },
1614 .caam = {
1615 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1616 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1617 OP_ALG_AAI_HMAC_PRECOMP,
1618 .geniv = true,
1619 }
1620 },
1621 {
1622 .aead = {
1623 .base = {
1624 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1625 .cra_driver_name = "authenc-hmac-sha1-"
1626 "cbc-aes-caam-qi",
1627 .cra_blocksize = AES_BLOCK_SIZE,
1628 },
1629 .setkey = aead_setkey,
1630 .setauthsize = aead_setauthsize,
1631 .encrypt = aead_encrypt,
1632 .decrypt = aead_decrypt,
1633 .ivsize = AES_BLOCK_SIZE,
1634 .maxauthsize = SHA1_DIGEST_SIZE,
1635 },
1636 .caam = {
1637 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1638 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1639 OP_ALG_AAI_HMAC_PRECOMP,
1640 }
1641 },
1642 {
1643 .aead = {
1644 .base = {
1645 .cra_name = "echainiv(authenc(hmac(sha1),"
1646 "cbc(aes)))",
1647 .cra_driver_name = "echainiv-authenc-"
1648 "hmac-sha1-cbc-aes-caam-qi",
1649 .cra_blocksize = AES_BLOCK_SIZE,
1650 },
1651 .setkey = aead_setkey,
1652 .setauthsize = aead_setauthsize,
1653 .encrypt = aead_encrypt,
1654 .decrypt = aead_decrypt,
1655 .ivsize = AES_BLOCK_SIZE,
1656 .maxauthsize = SHA1_DIGEST_SIZE,
1657 },
1658 .caam = {
1659 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1660 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1661 OP_ALG_AAI_HMAC_PRECOMP,
1662 .geniv = true,
1663 },
1664 },
1665 {
1666 .aead = {
1667 .base = {
1668 .cra_name = "authenc(hmac(sha224),cbc(aes))",
1669 .cra_driver_name = "authenc-hmac-sha224-"
1670 "cbc-aes-caam-qi",
1671 .cra_blocksize = AES_BLOCK_SIZE,
1672 },
1673 .setkey = aead_setkey,
1674 .setauthsize = aead_setauthsize,
1675 .encrypt = aead_encrypt,
1676 .decrypt = aead_decrypt,
1677 .ivsize = AES_BLOCK_SIZE,
1678 .maxauthsize = SHA224_DIGEST_SIZE,
1679 },
1680 .caam = {
1681 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1682 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1683 OP_ALG_AAI_HMAC_PRECOMP,
1684 }
1685 },
1686 {
1687 .aead = {
1688 .base = {
1689 .cra_name = "echainiv(authenc(hmac(sha224),"
1690 "cbc(aes)))",
1691 .cra_driver_name = "echainiv-authenc-"
1692 "hmac-sha224-cbc-aes-caam-qi",
1693 .cra_blocksize = AES_BLOCK_SIZE,
1694 },
1695 .setkey = aead_setkey,
1696 .setauthsize = aead_setauthsize,
1697 .encrypt = aead_encrypt,
1698 .decrypt = aead_decrypt,
1699 .ivsize = AES_BLOCK_SIZE,
1700 .maxauthsize = SHA224_DIGEST_SIZE,
1701 },
1702 .caam = {
1703 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1704 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1705 OP_ALG_AAI_HMAC_PRECOMP,
1706 .geniv = true,
1707 }
1708 },
1709 {
1710 .aead = {
1711 .base = {
1712 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1713 .cra_driver_name = "authenc-hmac-sha256-"
1714 "cbc-aes-caam-qi",
1715 .cra_blocksize = AES_BLOCK_SIZE,
1716 },
1717 .setkey = aead_setkey,
1718 .setauthsize = aead_setauthsize,
1719 .encrypt = aead_encrypt,
1720 .decrypt = aead_decrypt,
1721 .ivsize = AES_BLOCK_SIZE,
1722 .maxauthsize = SHA256_DIGEST_SIZE,
1723 },
1724 .caam = {
1725 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1726 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1727 OP_ALG_AAI_HMAC_PRECOMP,
1728 }
1729 },
1730 {
1731 .aead = {
1732 .base = {
1733 .cra_name = "echainiv(authenc(hmac(sha256),"
1734 "cbc(aes)))",
1735 .cra_driver_name = "echainiv-authenc-"
1736 "hmac-sha256-cbc-aes-"
1737 "caam-qi",
1738 .cra_blocksize = AES_BLOCK_SIZE,
1739 },
1740 .setkey = aead_setkey,
1741 .setauthsize = aead_setauthsize,
1742 .encrypt = aead_encrypt,
1743 .decrypt = aead_decrypt,
1744 .ivsize = AES_BLOCK_SIZE,
1745 .maxauthsize = SHA256_DIGEST_SIZE,
1746 },
1747 .caam = {
1748 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1749 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1750 OP_ALG_AAI_HMAC_PRECOMP,
1751 .geniv = true,
1752 }
1753 },
1754 {
1755 .aead = {
1756 .base = {
1757 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1758 .cra_driver_name = "authenc-hmac-sha384-"
1759 "cbc-aes-caam-qi",
1760 .cra_blocksize = AES_BLOCK_SIZE,
1761 },
1762 .setkey = aead_setkey,
1763 .setauthsize = aead_setauthsize,
1764 .encrypt = aead_encrypt,
1765 .decrypt = aead_decrypt,
1766 .ivsize = AES_BLOCK_SIZE,
1767 .maxauthsize = SHA384_DIGEST_SIZE,
1768 },
1769 .caam = {
1770 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1771 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1772 OP_ALG_AAI_HMAC_PRECOMP,
1773 }
1774 },
1775 {
1776 .aead = {
1777 .base = {
1778 .cra_name = "echainiv(authenc(hmac(sha384),"
1779 "cbc(aes)))",
1780 .cra_driver_name = "echainiv-authenc-"
1781 "hmac-sha384-cbc-aes-"
1782 "caam-qi",
1783 .cra_blocksize = AES_BLOCK_SIZE,
1784 },
1785 .setkey = aead_setkey,
1786 .setauthsize = aead_setauthsize,
1787 .encrypt = aead_encrypt,
1788 .decrypt = aead_decrypt,
1789 .ivsize = AES_BLOCK_SIZE,
1790 .maxauthsize = SHA384_DIGEST_SIZE,
1791 },
1792 .caam = {
1793 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1794 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
1795 OP_ALG_AAI_HMAC_PRECOMP,
1796 .geniv = true,
1797 }
1798 },
1799 {
1800 .aead = {
1801 .base = {
1802 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1803 .cra_driver_name = "authenc-hmac-sha512-"
1804 "cbc-aes-caam-qi",
1805 .cra_blocksize = AES_BLOCK_SIZE,
1806 },
1807 .setkey = aead_setkey,
1808 .setauthsize = aead_setauthsize,
1809 .encrypt = aead_encrypt,
1810 .decrypt = aead_decrypt,
1811 .ivsize = AES_BLOCK_SIZE,
1812 .maxauthsize = SHA512_DIGEST_SIZE,
1813 },
1814 .caam = {
1815 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1816 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1817 OP_ALG_AAI_HMAC_PRECOMP,
1818 }
1819 },
1820 {
1821 .aead = {
1822 .base = {
1823 .cra_name = "echainiv(authenc(hmac(sha512),"
1824 "cbc(aes)))",
1825 .cra_driver_name = "echainiv-authenc-"
1826 "hmac-sha512-cbc-aes-"
1827 "caam-qi",
1828 .cra_blocksize = AES_BLOCK_SIZE,
1829 },
1830 .setkey = aead_setkey,
1831 .setauthsize = aead_setauthsize,
1832 .encrypt = aead_encrypt,
1833 .decrypt = aead_decrypt,
1834 .ivsize = AES_BLOCK_SIZE,
1835 .maxauthsize = SHA512_DIGEST_SIZE,
1836 },
1837 .caam = {
1838 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1839 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
1840 OP_ALG_AAI_HMAC_PRECOMP,
1841 .geniv = true,
1842 }
1843 },
1844 {
1845 .aead = {
1846 .base = {
1847 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1848 .cra_driver_name = "authenc-hmac-md5-"
1849 "cbc-des3_ede-caam-qi",
1850 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1851 },
1852 .setkey = des3_aead_setkey,
1853 .setauthsize = aead_setauthsize,
1854 .encrypt = aead_encrypt,
1855 .decrypt = aead_decrypt,
1856 .ivsize = DES3_EDE_BLOCK_SIZE,
1857 .maxauthsize = MD5_DIGEST_SIZE,
1858 },
1859 .caam = {
1860 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1861 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1862 OP_ALG_AAI_HMAC_PRECOMP,
1863 }
1864 },
1865 {
1866 .aead = {
1867 .base = {
1868 .cra_name = "echainiv(authenc(hmac(md5),"
1869 "cbc(des3_ede)))",
1870 .cra_driver_name = "echainiv-authenc-hmac-md5-"
1871 "cbc-des3_ede-caam-qi",
1872 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1873 },
1874 .setkey = des3_aead_setkey,
1875 .setauthsize = aead_setauthsize,
1876 .encrypt = aead_encrypt,
1877 .decrypt = aead_decrypt,
1878 .ivsize = DES3_EDE_BLOCK_SIZE,
1879 .maxauthsize = MD5_DIGEST_SIZE,
1880 },
1881 .caam = {
1882 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1883 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
1884 OP_ALG_AAI_HMAC_PRECOMP,
1885 .geniv = true,
1886 }
1887 },
1888 {
1889 .aead = {
1890 .base = {
1891 .cra_name = "authenc(hmac(sha1),"
1892 "cbc(des3_ede))",
1893 .cra_driver_name = "authenc-hmac-sha1-"
1894 "cbc-des3_ede-caam-qi",
1895 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1896 },
1897 .setkey = des3_aead_setkey,
1898 .setauthsize = aead_setauthsize,
1899 .encrypt = aead_encrypt,
1900 .decrypt = aead_decrypt,
1901 .ivsize = DES3_EDE_BLOCK_SIZE,
1902 .maxauthsize = SHA1_DIGEST_SIZE,
1903 },
1904 .caam = {
1905 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1906 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1907 OP_ALG_AAI_HMAC_PRECOMP,
1908 },
1909 },
1910 {
1911 .aead = {
1912 .base = {
1913 .cra_name = "echainiv(authenc(hmac(sha1),"
1914 "cbc(des3_ede)))",
1915 .cra_driver_name = "echainiv-authenc-"
1916 "hmac-sha1-"
1917 "cbc-des3_ede-caam-qi",
1918 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1919 },
1920 .setkey = des3_aead_setkey,
1921 .setauthsize = aead_setauthsize,
1922 .encrypt = aead_encrypt,
1923 .decrypt = aead_decrypt,
1924 .ivsize = DES3_EDE_BLOCK_SIZE,
1925 .maxauthsize = SHA1_DIGEST_SIZE,
1926 },
1927 .caam = {
1928 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1929 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
1930 OP_ALG_AAI_HMAC_PRECOMP,
1931 .geniv = true,
1932 }
1933 },
1934 {
1935 .aead = {
1936 .base = {
1937 .cra_name = "authenc(hmac(sha224),"
1938 "cbc(des3_ede))",
1939 .cra_driver_name = "authenc-hmac-sha224-"
1940 "cbc-des3_ede-caam-qi",
1941 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1942 },
1943 .setkey = des3_aead_setkey,
1944 .setauthsize = aead_setauthsize,
1945 .encrypt = aead_encrypt,
1946 .decrypt = aead_decrypt,
1947 .ivsize = DES3_EDE_BLOCK_SIZE,
1948 .maxauthsize = SHA224_DIGEST_SIZE,
1949 },
1950 .caam = {
1951 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1952 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1953 OP_ALG_AAI_HMAC_PRECOMP,
1954 },
1955 },
1956 {
1957 .aead = {
1958 .base = {
1959 .cra_name = "echainiv(authenc(hmac(sha224),"
1960 "cbc(des3_ede)))",
1961 .cra_driver_name = "echainiv-authenc-"
1962 "hmac-sha224-"
1963 "cbc-des3_ede-caam-qi",
1964 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1965 },
1966 .setkey = des3_aead_setkey,
1967 .setauthsize = aead_setauthsize,
1968 .encrypt = aead_encrypt,
1969 .decrypt = aead_decrypt,
1970 .ivsize = DES3_EDE_BLOCK_SIZE,
1971 .maxauthsize = SHA224_DIGEST_SIZE,
1972 },
1973 .caam = {
1974 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1975 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
1976 OP_ALG_AAI_HMAC_PRECOMP,
1977 .geniv = true,
1978 }
1979 },
1980 {
1981 .aead = {
1982 .base = {
1983 .cra_name = "authenc(hmac(sha256),"
1984 "cbc(des3_ede))",
1985 .cra_driver_name = "authenc-hmac-sha256-"
1986 "cbc-des3_ede-caam-qi",
1987 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1988 },
1989 .setkey = des3_aead_setkey,
1990 .setauthsize = aead_setauthsize,
1991 .encrypt = aead_encrypt,
1992 .decrypt = aead_decrypt,
1993 .ivsize = DES3_EDE_BLOCK_SIZE,
1994 .maxauthsize = SHA256_DIGEST_SIZE,
1995 },
1996 .caam = {
1997 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1998 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
1999 OP_ALG_AAI_HMAC_PRECOMP,
2000 },
2001 },
2002 {
2003 .aead = {
2004 .base = {
2005 .cra_name = "echainiv(authenc(hmac(sha256),"
2006 "cbc(des3_ede)))",
2007 .cra_driver_name = "echainiv-authenc-"
2008 "hmac-sha256-"
2009 "cbc-des3_ede-caam-qi",
2010 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2011 },
2012 .setkey = des3_aead_setkey,
2013 .setauthsize = aead_setauthsize,
2014 .encrypt = aead_encrypt,
2015 .decrypt = aead_decrypt,
2016 .ivsize = DES3_EDE_BLOCK_SIZE,
2017 .maxauthsize = SHA256_DIGEST_SIZE,
2018 },
2019 .caam = {
2020 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2021 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2022 OP_ALG_AAI_HMAC_PRECOMP,
2023 .geniv = true,
2024 }
2025 },
2026 {
2027 .aead = {
2028 .base = {
2029 .cra_name = "authenc(hmac(sha384),"
2030 "cbc(des3_ede))",
2031 .cra_driver_name = "authenc-hmac-sha384-"
2032 "cbc-des3_ede-caam-qi",
2033 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2034 },
2035 .setkey = des3_aead_setkey,
2036 .setauthsize = aead_setauthsize,
2037 .encrypt = aead_encrypt,
2038 .decrypt = aead_decrypt,
2039 .ivsize = DES3_EDE_BLOCK_SIZE,
2040 .maxauthsize = SHA384_DIGEST_SIZE,
2041 },
2042 .caam = {
2043 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2044 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2045 OP_ALG_AAI_HMAC_PRECOMP,
2046 },
2047 },
2048 {
2049 .aead = {
2050 .base = {
2051 .cra_name = "echainiv(authenc(hmac(sha384),"
2052 "cbc(des3_ede)))",
2053 .cra_driver_name = "echainiv-authenc-"
2054 "hmac-sha384-"
2055 "cbc-des3_ede-caam-qi",
2056 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2057 },
2058 .setkey = des3_aead_setkey,
2059 .setauthsize = aead_setauthsize,
2060 .encrypt = aead_encrypt,
2061 .decrypt = aead_decrypt,
2062 .ivsize = DES3_EDE_BLOCK_SIZE,
2063 .maxauthsize = SHA384_DIGEST_SIZE,
2064 },
2065 .caam = {
2066 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2067 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2068 OP_ALG_AAI_HMAC_PRECOMP,
2069 .geniv = true,
2070 }
2071 },
2072 {
2073 .aead = {
2074 .base = {
2075 .cra_name = "authenc(hmac(sha512),"
2076 "cbc(des3_ede))",
2077 .cra_driver_name = "authenc-hmac-sha512-"
2078 "cbc-des3_ede-caam-qi",
2079 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2080 },
2081 .setkey = des3_aead_setkey,
2082 .setauthsize = aead_setauthsize,
2083 .encrypt = aead_encrypt,
2084 .decrypt = aead_decrypt,
2085 .ivsize = DES3_EDE_BLOCK_SIZE,
2086 .maxauthsize = SHA512_DIGEST_SIZE,
2087 },
2088 .caam = {
2089 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2090 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2091 OP_ALG_AAI_HMAC_PRECOMP,
2092 },
2093 },
2094 {
2095 .aead = {
2096 .base = {
2097 .cra_name = "echainiv(authenc(hmac(sha512),"
2098 "cbc(des3_ede)))",
2099 .cra_driver_name = "echainiv-authenc-"
2100 "hmac-sha512-"
2101 "cbc-des3_ede-caam-qi",
2102 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2103 },
2104 .setkey = des3_aead_setkey,
2105 .setauthsize = aead_setauthsize,
2106 .encrypt = aead_encrypt,
2107 .decrypt = aead_decrypt,
2108 .ivsize = DES3_EDE_BLOCK_SIZE,
2109 .maxauthsize = SHA512_DIGEST_SIZE,
2110 },
2111 .caam = {
2112 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2113 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2114 OP_ALG_AAI_HMAC_PRECOMP,
2115 .geniv = true,
2116 }
2117 },
2118 {
2119 .aead = {
2120 .base = {
2121 .cra_name = "authenc(hmac(md5),cbc(des))",
2122 .cra_driver_name = "authenc-hmac-md5-"
2123 "cbc-des-caam-qi",
2124 .cra_blocksize = DES_BLOCK_SIZE,
2125 },
2126 .setkey = aead_setkey,
2127 .setauthsize = aead_setauthsize,
2128 .encrypt = aead_encrypt,
2129 .decrypt = aead_decrypt,
2130 .ivsize = DES_BLOCK_SIZE,
2131 .maxauthsize = MD5_DIGEST_SIZE,
2132 },
2133 .caam = {
2134 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2135 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2136 OP_ALG_AAI_HMAC_PRECOMP,
2137 },
2138 },
2139 {
2140 .aead = {
2141 .base = {
2142 .cra_name = "echainiv(authenc(hmac(md5),"
2143 "cbc(des)))",
2144 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2145 "cbc-des-caam-qi",
2146 .cra_blocksize = DES_BLOCK_SIZE,
2147 },
2148 .setkey = aead_setkey,
2149 .setauthsize = aead_setauthsize,
2150 .encrypt = aead_encrypt,
2151 .decrypt = aead_decrypt,
2152 .ivsize = DES_BLOCK_SIZE,
2153 .maxauthsize = MD5_DIGEST_SIZE,
2154 },
2155 .caam = {
2156 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2157 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2158 OP_ALG_AAI_HMAC_PRECOMP,
2159 .geniv = true,
2160 }
2161 },
2162 {
2163 .aead = {
2164 .base = {
2165 .cra_name = "authenc(hmac(sha1),cbc(des))",
2166 .cra_driver_name = "authenc-hmac-sha1-"
2167 "cbc-des-caam-qi",
2168 .cra_blocksize = DES_BLOCK_SIZE,
2169 },
2170 .setkey = aead_setkey,
2171 .setauthsize = aead_setauthsize,
2172 .encrypt = aead_encrypt,
2173 .decrypt = aead_decrypt,
2174 .ivsize = DES_BLOCK_SIZE,
2175 .maxauthsize = SHA1_DIGEST_SIZE,
2176 },
2177 .caam = {
2178 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2179 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2180 OP_ALG_AAI_HMAC_PRECOMP,
2181 },
2182 },
2183 {
2184 .aead = {
2185 .base = {
2186 .cra_name = "echainiv(authenc(hmac(sha1),"
2187 "cbc(des)))",
2188 .cra_driver_name = "echainiv-authenc-"
2189 "hmac-sha1-cbc-des-caam-qi",
2190 .cra_blocksize = DES_BLOCK_SIZE,
2191 },
2192 .setkey = aead_setkey,
2193 .setauthsize = aead_setauthsize,
2194 .encrypt = aead_encrypt,
2195 .decrypt = aead_decrypt,
2196 .ivsize = DES_BLOCK_SIZE,
2197 .maxauthsize = SHA1_DIGEST_SIZE,
2198 },
2199 .caam = {
2200 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2201 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2202 OP_ALG_AAI_HMAC_PRECOMP,
2203 .geniv = true,
2204 }
2205 },
2206 {
2207 .aead = {
2208 .base = {
2209 .cra_name = "authenc(hmac(sha224),cbc(des))",
2210 .cra_driver_name = "authenc-hmac-sha224-"
2211 "cbc-des-caam-qi",
2212 .cra_blocksize = DES_BLOCK_SIZE,
2213 },
2214 .setkey = aead_setkey,
2215 .setauthsize = aead_setauthsize,
2216 .encrypt = aead_encrypt,
2217 .decrypt = aead_decrypt,
2218 .ivsize = DES_BLOCK_SIZE,
2219 .maxauthsize = SHA224_DIGEST_SIZE,
2220 },
2221 .caam = {
2222 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2223 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2224 OP_ALG_AAI_HMAC_PRECOMP,
2225 },
2226 },
2227 {
2228 .aead = {
2229 .base = {
2230 .cra_name = "echainiv(authenc(hmac(sha224),"
2231 "cbc(des)))",
2232 .cra_driver_name = "echainiv-authenc-"
2233 "hmac-sha224-cbc-des-"
2234 "caam-qi",
2235 .cra_blocksize = DES_BLOCK_SIZE,
2236 },
2237 .setkey = aead_setkey,
2238 .setauthsize = aead_setauthsize,
2239 .encrypt = aead_encrypt,
2240 .decrypt = aead_decrypt,
2241 .ivsize = DES_BLOCK_SIZE,
2242 .maxauthsize = SHA224_DIGEST_SIZE,
2243 },
2244 .caam = {
2245 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2246 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2247 OP_ALG_AAI_HMAC_PRECOMP,
2248 .geniv = true,
2249 }
2250 },
2251 {
2252 .aead = {
2253 .base = {
2254 .cra_name = "authenc(hmac(sha256),cbc(des))",
2255 .cra_driver_name = "authenc-hmac-sha256-"
2256 "cbc-des-caam-qi",
2257 .cra_blocksize = DES_BLOCK_SIZE,
2258 },
2259 .setkey = aead_setkey,
2260 .setauthsize = aead_setauthsize,
2261 .encrypt = aead_encrypt,
2262 .decrypt = aead_decrypt,
2263 .ivsize = DES_BLOCK_SIZE,
2264 .maxauthsize = SHA256_DIGEST_SIZE,
2265 },
2266 .caam = {
2267 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2268 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2269 OP_ALG_AAI_HMAC_PRECOMP,
2270 },
2271 },
2272 {
2273 .aead = {
2274 .base = {
2275 .cra_name = "echainiv(authenc(hmac(sha256),"
2276 "cbc(des)))",
2277 .cra_driver_name = "echainiv-authenc-"
2278 "hmac-sha256-cbc-des-"
2279 "caam-qi",
2280 .cra_blocksize = DES_BLOCK_SIZE,
2281 },
2282 .setkey = aead_setkey,
2283 .setauthsize = aead_setauthsize,
2284 .encrypt = aead_encrypt,
2285 .decrypt = aead_decrypt,
2286 .ivsize = DES_BLOCK_SIZE,
2287 .maxauthsize = SHA256_DIGEST_SIZE,
2288 },
2289 .caam = {
2290 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2291 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2292 OP_ALG_AAI_HMAC_PRECOMP,
2293 .geniv = true,
2294 },
2295 },
2296 {
2297 .aead = {
2298 .base = {
2299 .cra_name = "authenc(hmac(sha384),cbc(des))",
2300 .cra_driver_name = "authenc-hmac-sha384-"
2301 "cbc-des-caam-qi",
2302 .cra_blocksize = DES_BLOCK_SIZE,
2303 },
2304 .setkey = aead_setkey,
2305 .setauthsize = aead_setauthsize,
2306 .encrypt = aead_encrypt,
2307 .decrypt = aead_decrypt,
2308 .ivsize = DES_BLOCK_SIZE,
2309 .maxauthsize = SHA384_DIGEST_SIZE,
2310 },
2311 .caam = {
2312 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2313 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2314 OP_ALG_AAI_HMAC_PRECOMP,
2315 },
2316 },
2317 {
2318 .aead = {
2319 .base = {
2320 .cra_name = "echainiv(authenc(hmac(sha384),"
2321 "cbc(des)))",
2322 .cra_driver_name = "echainiv-authenc-"
2323 "hmac-sha384-cbc-des-"
2324 "caam-qi",
2325 .cra_blocksize = DES_BLOCK_SIZE,
2326 },
2327 .setkey = aead_setkey,
2328 .setauthsize = aead_setauthsize,
2329 .encrypt = aead_encrypt,
2330 .decrypt = aead_decrypt,
2331 .ivsize = DES_BLOCK_SIZE,
2332 .maxauthsize = SHA384_DIGEST_SIZE,
2333 },
2334 .caam = {
2335 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2336 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2337 OP_ALG_AAI_HMAC_PRECOMP,
2338 .geniv = true,
2339 }
2340 },
2341 {
2342 .aead = {
2343 .base = {
2344 .cra_name = "authenc(hmac(sha512),cbc(des))",
2345 .cra_driver_name = "authenc-hmac-sha512-"
2346 "cbc-des-caam-qi",
2347 .cra_blocksize = DES_BLOCK_SIZE,
2348 },
2349 .setkey = aead_setkey,
2350 .setauthsize = aead_setauthsize,
2351 .encrypt = aead_encrypt,
2352 .decrypt = aead_decrypt,
2353 .ivsize = DES_BLOCK_SIZE,
2354 .maxauthsize = SHA512_DIGEST_SIZE,
2355 },
2356 .caam = {
2357 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2358 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2359 OP_ALG_AAI_HMAC_PRECOMP,
2360 }
2361 },
2362 {
2363 .aead = {
2364 .base = {
2365 .cra_name = "echainiv(authenc(hmac(sha512),"
2366 "cbc(des)))",
2367 .cra_driver_name = "echainiv-authenc-"
2368 "hmac-sha512-cbc-des-"
2369 "caam-qi",
2370 .cra_blocksize = DES_BLOCK_SIZE,
2371 },
2372 .setkey = aead_setkey,
2373 .setauthsize = aead_setauthsize,
2374 .encrypt = aead_encrypt,
2375 .decrypt = aead_decrypt,
2376 .ivsize = DES_BLOCK_SIZE,
2377 .maxauthsize = SHA512_DIGEST_SIZE,
2378 },
2379 .caam = {
2380 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2381 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2382 OP_ALG_AAI_HMAC_PRECOMP,
2383 .geniv = true,
2384 }
2385 },
2386};
2387
2388static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
2389 bool uses_dkp)
2390{
2391 struct caam_drv_private *priv;
2392 struct device *dev;
2393
2394
2395
2396
2397
2398 ctx->jrdev = caam_jr_alloc();
2399 if (IS_ERR(ctx->jrdev)) {
2400 pr_err("Job Ring Device allocation for transform failed\n");
2401 return PTR_ERR(ctx->jrdev);
2402 }
2403
2404 dev = ctx->jrdev->parent;
2405 priv = dev_get_drvdata(dev);
2406 if (priv->era >= 6 && uses_dkp)
2407 ctx->dir = DMA_BIDIRECTIONAL;
2408 else
2409 ctx->dir = DMA_TO_DEVICE;
2410
2411 ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key),
2412 ctx->dir);
2413 if (dma_mapping_error(dev, ctx->key_dma)) {
2414 dev_err(dev, "unable to map key\n");
2415 caam_jr_free(ctx->jrdev);
2416 return -ENOMEM;
2417 }
2418
2419
2420 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
2421 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
2422
2423 ctx->qidev = dev;
2424
2425 spin_lock_init(&ctx->lock);
2426 ctx->drv_ctx[ENCRYPT] = NULL;
2427 ctx->drv_ctx[DECRYPT] = NULL;
2428
2429 return 0;
2430}
2431
2432static int caam_cra_init(struct crypto_skcipher *tfm)
2433{
2434 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
2435 struct caam_skcipher_alg *caam_alg =
2436 container_of(alg, typeof(*caam_alg), skcipher);
2437
2438 return caam_init_common(crypto_skcipher_ctx(tfm), &caam_alg->caam,
2439 false);
2440}
2441
2442static int caam_aead_init(struct crypto_aead *tfm)
2443{
2444 struct aead_alg *alg = crypto_aead_alg(tfm);
2445 struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg),
2446 aead);
2447 struct caam_ctx *ctx = crypto_aead_ctx(tfm);
2448
2449 return caam_init_common(ctx, &caam_alg->caam, !caam_alg->caam.nodkp);
2450}
2451
2452static void caam_exit_common(struct caam_ctx *ctx)
2453{
2454 caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]);
2455 caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]);
2456
2457 dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key),
2458 ctx->dir);
2459
2460 caam_jr_free(ctx->jrdev);
2461}
2462
2463static void caam_cra_exit(struct crypto_skcipher *tfm)
2464{
2465 caam_exit_common(crypto_skcipher_ctx(tfm));
2466}
2467
2468static void caam_aead_exit(struct crypto_aead *tfm)
2469{
2470 caam_exit_common(crypto_aead_ctx(tfm));
2471}
2472
2473void caam_qi_algapi_exit(void)
2474{
2475 int i;
2476
2477 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2478 struct caam_aead_alg *t_alg = driver_aeads + i;
2479
2480 if (t_alg->registered)
2481 crypto_unregister_aead(&t_alg->aead);
2482 }
2483
2484 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2485 struct caam_skcipher_alg *t_alg = driver_algs + i;
2486
2487 if (t_alg->registered)
2488 crypto_unregister_skcipher(&t_alg->skcipher);
2489 }
2490}
2491
2492static void caam_skcipher_alg_init(struct caam_skcipher_alg *t_alg)
2493{
2494 struct skcipher_alg *alg = &t_alg->skcipher;
2495
2496 alg->base.cra_module = THIS_MODULE;
2497 alg->base.cra_priority = CAAM_CRA_PRIORITY;
2498 alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2499 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2500
2501 alg->init = caam_cra_init;
2502 alg->exit = caam_cra_exit;
2503}
2504
2505static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
2506{
2507 struct aead_alg *alg = &t_alg->aead;
2508
2509 alg->base.cra_module = THIS_MODULE;
2510 alg->base.cra_priority = CAAM_CRA_PRIORITY;
2511 alg->base.cra_ctxsize = sizeof(struct caam_ctx);
2512 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2513
2514 alg->init = caam_aead_init;
2515 alg->exit = caam_aead_exit;
2516}
2517
2518int caam_qi_algapi_init(struct device *ctrldev)
2519{
2520 struct caam_drv_private *priv = dev_get_drvdata(ctrldev);
2521 int i = 0, err = 0;
2522 u32 aes_vid, aes_inst, des_inst, md_vid, md_inst;
2523 unsigned int md_limit = SHA512_DIGEST_SIZE;
2524 bool registered = false;
2525
2526 if (caam_dpaa2) {
2527 dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n");
2528 return -ENODEV;
2529 }
2530
2531
2532
2533
2534
2535 if (priv->era < 10) {
2536 u32 cha_vid, cha_inst;
2537
2538 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
2539 aes_vid = cha_vid & CHA_ID_LS_AES_MASK;
2540 md_vid = (cha_vid & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2541
2542 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
2543 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >>
2544 CHA_ID_LS_DES_SHIFT;
2545 aes_inst = cha_inst & CHA_ID_LS_AES_MASK;
2546 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
2547 } else {
2548 u32 aesa, mdha;
2549
2550 aesa = rd_reg32(&priv->ctrl->vreg.aesa);
2551 mdha = rd_reg32(&priv->ctrl->vreg.mdha);
2552
2553 aes_vid = (aesa & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2554 md_vid = (mdha & CHA_VER_VID_MASK) >> CHA_VER_VID_SHIFT;
2555
2556 des_inst = rd_reg32(&priv->ctrl->vreg.desa) & CHA_VER_NUM_MASK;
2557 aes_inst = aesa & CHA_VER_NUM_MASK;
2558 md_inst = mdha & CHA_VER_NUM_MASK;
2559 }
2560
2561
2562 if (md_inst && md_vid == CHA_VER_VID_MD_LP256)
2563 md_limit = SHA256_DIGEST_SIZE;
2564
2565 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
2566 struct caam_skcipher_alg *t_alg = driver_algs + i;
2567 u32 alg_sel = t_alg->caam.class1_alg_type & OP_ALG_ALGSEL_MASK;
2568
2569
2570 if (!des_inst &&
2571 ((alg_sel == OP_ALG_ALGSEL_3DES) ||
2572 (alg_sel == OP_ALG_ALGSEL_DES)))
2573 continue;
2574
2575
2576 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
2577 continue;
2578
2579 caam_skcipher_alg_init(t_alg);
2580
2581 err = crypto_register_skcipher(&t_alg->skcipher);
2582 if (err) {
2583 dev_warn(ctrldev, "%s alg registration failed\n",
2584 t_alg->skcipher.base.cra_driver_name);
2585 continue;
2586 }
2587
2588 t_alg->registered = true;
2589 registered = true;
2590 }
2591
2592 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
2593 struct caam_aead_alg *t_alg = driver_aeads + i;
2594 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
2595 OP_ALG_ALGSEL_MASK;
2596 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
2597 OP_ALG_ALGSEL_MASK;
2598 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
2599
2600
2601 if (!des_inst &&
2602 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
2603 (c1_alg_sel == OP_ALG_ALGSEL_DES)))
2604 continue;
2605
2606
2607 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
2608 continue;
2609
2610
2611
2612
2613
2614 if (aes_vid == CHA_VER_VID_AES_LP && alg_aai == OP_ALG_AAI_GCM)
2615 continue;
2616
2617
2618
2619
2620
2621 if (c2_alg_sel &&
2622 (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
2623 continue;
2624
2625 caam_aead_alg_init(t_alg);
2626
2627 err = crypto_register_aead(&t_alg->aead);
2628 if (err) {
2629 pr_warn("%s alg registration failed\n",
2630 t_alg->aead.base.cra_driver_name);
2631 continue;
2632 }
2633
2634 t_alg->registered = true;
2635 registered = true;
2636 }
2637
2638 if (registered)
2639 dev_info(ctrldev, "algorithms registered in /proc/crypto\n");
2640
2641 return err;
2642}
2643