1
2
3
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <crypto/algapi.h>
7#include <crypto/internal/aead.h>
8#include <crypto/authenc.h>
9#include <crypto/des.h>
10#include <linux/rtnetlink.h>
11#include "cc_driver.h"
12#include "cc_buffer_mgr.h"
13#include "cc_aead.h"
14#include "cc_request_mgr.h"
15#include "cc_hash.h"
16#include "cc_sram_mgr.h"
17
18#define template_aead template_u.aead
19
20#define MAX_AEAD_SETKEY_SEQ 12
21#define MAX_AEAD_PROCESS_SEQ 23
22
23#define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE)
24#define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE)
25
26#define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE
27
28struct cc_aead_handle {
29 cc_sram_addr_t sram_workspace_addr;
30 struct list_head aead_list;
31};
32
33struct cc_hmac_s {
34 u8 *padded_authkey;
35 u8 *ipad_opad;
36 dma_addr_t padded_authkey_dma_addr;
37 dma_addr_t ipad_opad_dma_addr;
38};
39
40struct cc_xcbc_s {
41 u8 *xcbc_keys;
42 dma_addr_t xcbc_keys_dma_addr;
43};
44
45struct cc_aead_ctx {
46 struct cc_drvdata *drvdata;
47 u8 ctr_nonce[MAX_NONCE_SIZE];
48 u8 *enckey;
49 dma_addr_t enckey_dma_addr;
50 union {
51 struct cc_hmac_s hmac;
52 struct cc_xcbc_s xcbc;
53 } auth_state;
54 unsigned int enc_keylen;
55 unsigned int auth_keylen;
56 unsigned int authsize;
57 unsigned int hash_len;
58 enum drv_cipher_mode cipher_mode;
59 enum cc_flow_mode flow_mode;
60 enum drv_hash_mode auth_mode;
61};
62
63static inline bool valid_assoclen(struct aead_request *req)
64{
65 return ((req->assoclen == 16) || (req->assoclen == 20));
66}
67
68static void cc_aead_exit(struct crypto_aead *tfm)
69{
70 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
71 struct device *dev = drvdata_to_dev(ctx->drvdata);
72
73 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm),
74 crypto_tfm_alg_name(&tfm->base));
75
76
77 if (ctx->enckey) {
78 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey,
79 ctx->enckey_dma_addr);
80 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n",
81 &ctx->enckey_dma_addr);
82 ctx->enckey_dma_addr = 0;
83 ctx->enckey = NULL;
84 }
85
86 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
87 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
88
89 if (xcbc->xcbc_keys) {
90 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,
91 xcbc->xcbc_keys,
92 xcbc->xcbc_keys_dma_addr);
93 }
94 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",
95 &xcbc->xcbc_keys_dma_addr);
96 xcbc->xcbc_keys_dma_addr = 0;
97 xcbc->xcbc_keys = NULL;
98 } else if (ctx->auth_mode != DRV_HASH_NULL) {
99 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
100
101 if (hmac->ipad_opad) {
102 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,
103 hmac->ipad_opad,
104 hmac->ipad_opad_dma_addr);
105 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",
106 &hmac->ipad_opad_dma_addr);
107 hmac->ipad_opad_dma_addr = 0;
108 hmac->ipad_opad = NULL;
109 }
110 if (hmac->padded_authkey) {
111 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,
112 hmac->padded_authkey,
113 hmac->padded_authkey_dma_addr);
114 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",
115 &hmac->padded_authkey_dma_addr);
116 hmac->padded_authkey_dma_addr = 0;
117 hmac->padded_authkey = NULL;
118 }
119 }
120}
121
122static unsigned int cc_get_aead_hash_len(struct crypto_aead *tfm)
123{
124 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
125
126 return cc_get_default_hash_len(ctx->drvdata);
127}
128
129static int cc_aead_init(struct crypto_aead *tfm)
130{
131 struct aead_alg *alg = crypto_aead_alg(tfm);
132 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
133 struct cc_crypto_alg *cc_alg =
134 container_of(alg, struct cc_crypto_alg, aead_alg);
135 struct device *dev = drvdata_to_dev(cc_alg->drvdata);
136
137 dev_dbg(dev, "Initializing context @%p for %s\n", ctx,
138 crypto_tfm_alg_name(&tfm->base));
139
140
141 ctx->cipher_mode = cc_alg->cipher_mode;
142 ctx->flow_mode = cc_alg->flow_mode;
143 ctx->auth_mode = cc_alg->auth_mode;
144 ctx->drvdata = cc_alg->drvdata;
145 crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx));
146
147
148 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
149 &ctx->enckey_dma_addr, GFP_KERNEL);
150 if (!ctx->enckey) {
151 dev_err(dev, "Failed allocating key buffer\n");
152 goto init_failed;
153 }
154 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n",
155 ctx->enckey);
156
157
158
159 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
160 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
161 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;
162
163
164
165 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,
166 &xcbc->xcbc_keys_dma_addr,
167 GFP_KERNEL);
168 if (!xcbc->xcbc_keys) {
169 dev_err(dev, "Failed allocating buffer for XCBC keys\n");
170 goto init_failed;
171 }
172 } else if (ctx->auth_mode != DRV_HASH_NULL) {
173 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
174 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;
175 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;
176
177
178 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,
179 &hmac->ipad_opad_dma_addr,
180 GFP_KERNEL);
181
182 if (!hmac->ipad_opad) {
183 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n");
184 goto init_failed;
185 }
186
187 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n",
188 hmac->ipad_opad);
189
190 hmac->padded_authkey = dma_alloc_coherent(dev,
191 MAX_HMAC_BLOCK_SIZE,
192 pkey_dma,
193 GFP_KERNEL);
194
195 if (!hmac->padded_authkey) {
196 dev_err(dev, "failed to allocate padded_authkey\n");
197 goto init_failed;
198 }
199 } else {
200 ctx->auth_state.hmac.ipad_opad = NULL;
201 ctx->auth_state.hmac.padded_authkey = NULL;
202 }
203 ctx->hash_len = cc_get_aead_hash_len(tfm);
204
205 return 0;
206
207init_failed:
208 cc_aead_exit(tfm);
209 return -ENOMEM;
210}
211
212static void cc_aead_complete(struct device *dev, void *cc_req, int err)
213{
214 struct aead_request *areq = (struct aead_request *)cc_req;
215 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
216 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);
217 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
218
219
220 if (err == -EINPROGRESS)
221 goto done;
222
223 cc_unmap_aead_request(dev, areq);
224
225
226 areq->iv = areq_ctx->backup_iv;
227
228 if (err)
229 goto done;
230
231 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
232 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
233 ctx->authsize) != 0) {
234 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",
235 ctx->authsize, ctx->cipher_mode);
236
237
238
239 cc_zero_sgl(areq->dst, areq_ctx->cryptlen);
240 err = -EBADMSG;
241 }
242 } else {
243 if (areq_ctx->is_icv_fragmented) {
244 u32 skip = areq->cryptlen + areq_ctx->dst_offset;
245
246 cc_copy_sg_portion(dev, areq_ctx->mac_buf,
247 areq_ctx->dst_sgl, skip,
248 (skip + ctx->authsize),
249 CC_SG_FROM_BUF);
250 }
251
252
253
254
255 if (areq_ctx->backup_giv) {
256 if (ctx->cipher_mode == DRV_CIPHER_CTR)
257 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
258 CTR_RFC3686_NONCE_SIZE,
259 CTR_RFC3686_IV_SIZE);
260 else if (ctx->cipher_mode == DRV_CIPHER_CCM)
261 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
262 CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE);
263 }
264 }
265done:
266 aead_request_complete(areq, err);
267}
268
269static unsigned int xcbc_setkey(struct cc_hw_desc *desc,
270 struct cc_aead_ctx *ctx)
271{
272
273 hw_desc_init(&desc[0]);
274
275
276
277
278 set_din_type(&desc[0], DMA_DLLI,
279 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen,
280 NS_BIT);
281 set_cipher_mode(&desc[0], DRV_CIPHER_ECB);
282 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT);
283 set_key_size_aes(&desc[0], ctx->auth_keylen);
284 set_flow_mode(&desc[0], S_DIN_to_AES);
285 set_setup_mode(&desc[0], SETUP_LOAD_KEY0);
286
287 hw_desc_init(&desc[1]);
288 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
289 set_flow_mode(&desc[1], DIN_AES_DOUT);
290 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr,
291 AES_KEYSIZE_128, NS_BIT, 0);
292
293 hw_desc_init(&desc[2]);
294 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
295 set_flow_mode(&desc[2], DIN_AES_DOUT);
296 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
297 + AES_KEYSIZE_128),
298 AES_KEYSIZE_128, NS_BIT, 0);
299
300 hw_desc_init(&desc[3]);
301 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
302 set_flow_mode(&desc[3], DIN_AES_DOUT);
303 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
304 + 2 * AES_KEYSIZE_128),
305 AES_KEYSIZE_128, NS_BIT, 0);
306
307 return 4;
308}
309
310static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx)
311{
312 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
313 unsigned int digest_ofs = 0;
314 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
315 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
316 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
317 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
318 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
319
320 unsigned int idx = 0;
321 int i;
322
323
324 for (i = 0; i < 2; i++) {
325
326 hw_desc_init(&desc[idx]);
327 set_cipher_mode(&desc[idx], hash_mode);
328 set_din_sram(&desc[idx],
329 cc_larval_digest_addr(ctx->drvdata,
330 ctx->auth_mode),
331 digest_size);
332 set_flow_mode(&desc[idx], S_DIN_to_HASH);
333 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
334 idx++;
335
336
337 hw_desc_init(&desc[idx]);
338 set_cipher_mode(&desc[idx], hash_mode);
339 set_din_const(&desc[idx], 0, ctx->hash_len);
340 set_flow_mode(&desc[idx], S_DIN_to_HASH);
341 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
342 idx++;
343
344
345 hw_desc_init(&desc[idx]);
346 set_xor_val(&desc[idx], hmac_pad_const[i]);
347 set_cipher_mode(&desc[idx], hash_mode);
348 set_flow_mode(&desc[idx], S_DIN_to_HASH);
349 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
350 idx++;
351
352
353 hw_desc_init(&desc[idx]);
354 set_din_type(&desc[idx], DMA_DLLI,
355 hmac->padded_authkey_dma_addr,
356 SHA256_BLOCK_SIZE, NS_BIT);
357 set_cipher_mode(&desc[idx], hash_mode);
358 set_xor_active(&desc[idx]);
359 set_flow_mode(&desc[idx], DIN_HASH);
360 idx++;
361
362
363 hw_desc_init(&desc[idx]);
364 set_cipher_mode(&desc[idx], hash_mode);
365 set_dout_dlli(&desc[idx],
366 (hmac->ipad_opad_dma_addr + digest_ofs),
367 digest_size, NS_BIT, 0);
368 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
369 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
370 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
371 idx++;
372
373 digest_ofs += digest_size;
374 }
375
376 return idx;
377}
378
379static int validate_keys_sizes(struct cc_aead_ctx *ctx)
380{
381 struct device *dev = drvdata_to_dev(ctx->drvdata);
382
383 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n",
384 ctx->enc_keylen, ctx->auth_keylen);
385
386 switch (ctx->auth_mode) {
387 case DRV_HASH_SHA1:
388 case DRV_HASH_SHA256:
389 break;
390 case DRV_HASH_XCBC_MAC:
391 if (ctx->auth_keylen != AES_KEYSIZE_128 &&
392 ctx->auth_keylen != AES_KEYSIZE_192 &&
393 ctx->auth_keylen != AES_KEYSIZE_256)
394 return -ENOTSUPP;
395 break;
396 case DRV_HASH_NULL:
397 if (ctx->auth_keylen > 0)
398 return -EINVAL;
399 break;
400 default:
401 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode);
402 return -EINVAL;
403 }
404
405 if (ctx->flow_mode == S_DIN_to_DES) {
406 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {
407 dev_err(dev, "Invalid cipher(3DES) key size: %u\n",
408 ctx->enc_keylen);
409 return -EINVAL;
410 }
411 } else {
412 if (ctx->enc_keylen != AES_KEYSIZE_128 &&
413 ctx->enc_keylen != AES_KEYSIZE_192 &&
414 ctx->enc_keylen != AES_KEYSIZE_256) {
415 dev_err(dev, "Invalid cipher(AES) key size: %u\n",
416 ctx->enc_keylen);
417 return -EINVAL;
418 }
419 }
420
421 return 0;
422}
423
424
425
426
427static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey,
428 unsigned int keylen)
429{
430 dma_addr_t key_dma_addr = 0;
431 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
432 struct device *dev = drvdata_to_dev(ctx->drvdata);
433 u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode);
434 struct cc_crypto_req cc_req = {};
435 unsigned int blocksize;
436 unsigned int digestsize;
437 unsigned int hashmode;
438 unsigned int idx = 0;
439 int rc = 0;
440 u8 *key = NULL;
441 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
442 dma_addr_t padded_authkey_dma_addr =
443 ctx->auth_state.hmac.padded_authkey_dma_addr;
444
445 switch (ctx->auth_mode) {
446 case DRV_HASH_SHA1:
447 blocksize = SHA1_BLOCK_SIZE;
448 digestsize = SHA1_DIGEST_SIZE;
449 hashmode = DRV_HASH_HW_SHA1;
450 break;
451 case DRV_HASH_SHA256:
452 default:
453 blocksize = SHA256_BLOCK_SIZE;
454 digestsize = SHA256_DIGEST_SIZE;
455 hashmode = DRV_HASH_HW_SHA256;
456 }
457
458 if (keylen != 0) {
459
460 key = kmemdup(authkey, keylen, GFP_KERNEL);
461 if (!key)
462 return -ENOMEM;
463
464 key_dma_addr = dma_map_single(dev, (void *)key, keylen,
465 DMA_TO_DEVICE);
466 if (dma_mapping_error(dev, key_dma_addr)) {
467 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
468 key, keylen);
469 kzfree(key);
470 return -ENOMEM;
471 }
472 if (keylen > blocksize) {
473
474 hw_desc_init(&desc[idx]);
475 set_cipher_mode(&desc[idx], hashmode);
476 set_din_sram(&desc[idx], larval_addr, digestsize);
477 set_flow_mode(&desc[idx], S_DIN_to_HASH);
478 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
479 idx++;
480
481
482 hw_desc_init(&desc[idx]);
483 set_cipher_mode(&desc[idx], hashmode);
484 set_din_const(&desc[idx], 0, ctx->hash_len);
485 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
486 set_flow_mode(&desc[idx], S_DIN_to_HASH);
487 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
488 idx++;
489
490 hw_desc_init(&desc[idx]);
491 set_din_type(&desc[idx], DMA_DLLI,
492 key_dma_addr, keylen, NS_BIT);
493 set_flow_mode(&desc[idx], DIN_HASH);
494 idx++;
495
496
497 hw_desc_init(&desc[idx]);
498 set_cipher_mode(&desc[idx], hashmode);
499 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
500 digestsize, NS_BIT, 0);
501 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
502 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
503 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
504 set_cipher_config0(&desc[idx],
505 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
506 idx++;
507
508 hw_desc_init(&desc[idx]);
509 set_din_const(&desc[idx], 0, (blocksize - digestsize));
510 set_flow_mode(&desc[idx], BYPASS);
511 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr +
512 digestsize), (blocksize - digestsize),
513 NS_BIT, 0);
514 idx++;
515 } else {
516 hw_desc_init(&desc[idx]);
517 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
518 keylen, NS_BIT);
519 set_flow_mode(&desc[idx], BYPASS);
520 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
521 keylen, NS_BIT, 0);
522 idx++;
523
524 if ((blocksize - keylen) != 0) {
525 hw_desc_init(&desc[idx]);
526 set_din_const(&desc[idx], 0,
527 (blocksize - keylen));
528 set_flow_mode(&desc[idx], BYPASS);
529 set_dout_dlli(&desc[idx],
530 (padded_authkey_dma_addr +
531 keylen),
532 (blocksize - keylen), NS_BIT, 0);
533 idx++;
534 }
535 }
536 } else {
537 hw_desc_init(&desc[idx]);
538 set_din_const(&desc[idx], 0, (blocksize - keylen));
539 set_flow_mode(&desc[idx], BYPASS);
540 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
541 blocksize, NS_BIT, 0);
542 idx++;
543 }
544
545 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
546 if (rc)
547 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
548
549 if (key_dma_addr)
550 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE);
551
552 kzfree(key);
553
554 return rc;
555}
556
557static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
558 unsigned int keylen)
559{
560 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
561 struct cc_crypto_req cc_req = {};
562 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
563 unsigned int seq_len = 0;
564 struct device *dev = drvdata_to_dev(ctx->drvdata);
565 const u8 *enckey, *authkey;
566 int rc;
567
568 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n",
569 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);
570
571
572
573 if (ctx->auth_mode != DRV_HASH_NULL) {
574 struct crypto_authenc_keys keys;
575
576 rc = crypto_authenc_extractkeys(&keys, key, keylen);
577 if (rc)
578 goto badkey;
579 enckey = keys.enckey;
580 authkey = keys.authkey;
581 ctx->enc_keylen = keys.enckeylen;
582 ctx->auth_keylen = keys.authkeylen;
583
584 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
585
586 rc = -EINVAL;
587 if (ctx->enc_keylen <
588 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE))
589 goto badkey;
590
591
592
593 memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen -
594 CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE);
595
596 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE;
597 }
598 } else {
599 enckey = key;
600 authkey = NULL;
601 ctx->enc_keylen = keylen;
602 ctx->auth_keylen = 0;
603 }
604
605 rc = validate_keys_sizes(ctx);
606 if (rc)
607 goto badkey;
608
609
610
611
612 memcpy(ctx->enckey, enckey, ctx->enc_keylen);
613 if (ctx->enc_keylen == 24)
614 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24);
615 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
616 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey,
617 ctx->auth_keylen);
618 } else if (ctx->auth_mode != DRV_HASH_NULL) {
619 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen);
620 if (rc)
621 goto badkey;
622 }
623
624
625
626 switch (ctx->auth_mode) {
627 case DRV_HASH_SHA1:
628 case DRV_HASH_SHA256:
629 seq_len = hmac_setkey(desc, ctx);
630 break;
631 case DRV_HASH_XCBC_MAC:
632 seq_len = xcbc_setkey(desc, ctx);
633 break;
634 case DRV_HASH_NULL:
635 break;
636 default:
637 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
638 rc = -ENOTSUPP;
639 goto badkey;
640 }
641
642
643
644 if (seq_len > 0) {
645 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len);
646 if (rc) {
647 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
648 goto setkey_error;
649 }
650 }
651
652
653 return rc;
654
655badkey:
656 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
657
658setkey_error:
659 return rc;
660}
661
662static int cc_des3_aead_setkey(struct crypto_aead *aead, const u8 *key,
663 unsigned int keylen)
664{
665 struct crypto_authenc_keys keys;
666 u32 flags;
667 int err;
668
669 err = crypto_authenc_extractkeys(&keys, key, keylen);
670 if (unlikely(err))
671 goto badkey;
672
673 err = -EINVAL;
674 if (keys.enckeylen != DES3_EDE_KEY_SIZE)
675 goto badkey;
676
677 flags = crypto_aead_get_flags(aead);
678 err = __des3_verify_key(&flags, keys.enckey);
679 if (unlikely(err)) {
680 crypto_aead_set_flags(aead, flags);
681 goto out;
682 }
683
684 err = cc_aead_setkey(aead, key, keylen);
685
686out:
687 memzero_explicit(&keys, sizeof(keys));
688 return err;
689
690badkey:
691 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
692 goto out;
693}
694
695static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
696 unsigned int keylen)
697{
698 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
699
700 if (keylen < 3)
701 return -EINVAL;
702
703 keylen -= 3;
704 memcpy(ctx->ctr_nonce, key + keylen, 3);
705
706 return cc_aead_setkey(tfm, key, keylen);
707}
708
709static int cc_aead_setauthsize(struct crypto_aead *authenc,
710 unsigned int authsize)
711{
712 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
713 struct device *dev = drvdata_to_dev(ctx->drvdata);
714
715
716 if (authsize == 0 ||
717 authsize > crypto_aead_maxauthsize(authenc)) {
718 return -ENOTSUPP;
719 }
720
721 ctx->authsize = authsize;
722 dev_dbg(dev, "authlen=%d\n", ctx->authsize);
723
724 return 0;
725}
726
727static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
728 unsigned int authsize)
729{
730 switch (authsize) {
731 case 8:
732 case 12:
733 case 16:
734 break;
735 default:
736 return -EINVAL;
737 }
738
739 return cc_aead_setauthsize(authenc, authsize);
740}
741
742static int cc_ccm_setauthsize(struct crypto_aead *authenc,
743 unsigned int authsize)
744{
745 switch (authsize) {
746 case 4:
747 case 6:
748 case 8:
749 case 10:
750 case 12:
751 case 14:
752 case 16:
753 break;
754 default:
755 return -EINVAL;
756 }
757
758 return cc_aead_setauthsize(authenc, authsize);
759}
760
761static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,
762 struct cc_hw_desc desc[], unsigned int *seq_size)
763{
764 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
765 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
766 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
767 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;
768 unsigned int idx = *seq_size;
769 struct device *dev = drvdata_to_dev(ctx->drvdata);
770
771 switch (assoc_dma_type) {
772 case CC_DMA_BUF_DLLI:
773 dev_dbg(dev, "ASSOC buffer type DLLI\n");
774 hw_desc_init(&desc[idx]);
775 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src),
776 areq_ctx->assoclen, NS_BIT);
777 set_flow_mode(&desc[idx], flow_mode);
778 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
779 areq_ctx->cryptlen > 0)
780 set_din_not_last_indication(&desc[idx]);
781 break;
782 case CC_DMA_BUF_MLLI:
783 dev_dbg(dev, "ASSOC buffer type MLLI\n");
784 hw_desc_init(&desc[idx]);
785 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr,
786 areq_ctx->assoc.mlli_nents, NS_BIT);
787 set_flow_mode(&desc[idx], flow_mode);
788 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
789 areq_ctx->cryptlen > 0)
790 set_din_not_last_indication(&desc[idx]);
791 break;
792 case CC_DMA_BUF_NULL:
793 default:
794 dev_err(dev, "Invalid ASSOC buffer type\n");
795 }
796
797 *seq_size = (++idx);
798}
799
800static void cc_proc_authen_desc(struct aead_request *areq,
801 unsigned int flow_mode,
802 struct cc_hw_desc desc[],
803 unsigned int *seq_size, int direct)
804{
805 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
806 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
807 unsigned int idx = *seq_size;
808 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
809 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
810 struct device *dev = drvdata_to_dev(ctx->drvdata);
811
812 switch (data_dma_type) {
813 case CC_DMA_BUF_DLLI:
814 {
815 struct scatterlist *cipher =
816 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
817 areq_ctx->dst_sgl : areq_ctx->src_sgl;
818
819 unsigned int offset =
820 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
821 areq_ctx->dst_offset : areq_ctx->src_offset;
822 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n");
823 hw_desc_init(&desc[idx]);
824 set_din_type(&desc[idx], DMA_DLLI,
825 (sg_dma_address(cipher) + offset),
826 areq_ctx->cryptlen, NS_BIT);
827 set_flow_mode(&desc[idx], flow_mode);
828 break;
829 }
830 case CC_DMA_BUF_MLLI:
831 {
832
833
834
835
836 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr;
837 u32 mlli_nents = areq_ctx->assoc.mlli_nents;
838
839 if (areq_ctx->is_single_pass) {
840 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
841 mlli_addr = areq_ctx->dst.sram_addr;
842 mlli_nents = areq_ctx->dst.mlli_nents;
843 } else {
844 mlli_addr = areq_ctx->src.sram_addr;
845 mlli_nents = areq_ctx->src.mlli_nents;
846 }
847 }
848
849 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n");
850 hw_desc_init(&desc[idx]);
851 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents,
852 NS_BIT);
853 set_flow_mode(&desc[idx], flow_mode);
854 break;
855 }
856 case CC_DMA_BUF_NULL:
857 default:
858 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n");
859 }
860
861 *seq_size = (++idx);
862}
863
864static void cc_proc_cipher_desc(struct aead_request *areq,
865 unsigned int flow_mode,
866 struct cc_hw_desc desc[],
867 unsigned int *seq_size)
868{
869 unsigned int idx = *seq_size;
870 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
871 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
872 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
873 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
874 struct device *dev = drvdata_to_dev(ctx->drvdata);
875
876 if (areq_ctx->cryptlen == 0)
877 return;
878
879 switch (data_dma_type) {
880 case CC_DMA_BUF_DLLI:
881 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n");
882 hw_desc_init(&desc[idx]);
883 set_din_type(&desc[idx], DMA_DLLI,
884 (sg_dma_address(areq_ctx->src_sgl) +
885 areq_ctx->src_offset), areq_ctx->cryptlen,
886 NS_BIT);
887 set_dout_dlli(&desc[idx],
888 (sg_dma_address(areq_ctx->dst_sgl) +
889 areq_ctx->dst_offset),
890 areq_ctx->cryptlen, NS_BIT, 0);
891 set_flow_mode(&desc[idx], flow_mode);
892 break;
893 case CC_DMA_BUF_MLLI:
894 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n");
895 hw_desc_init(&desc[idx]);
896 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr,
897 areq_ctx->src.mlli_nents, NS_BIT);
898 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr,
899 areq_ctx->dst.mlli_nents, NS_BIT, 0);
900 set_flow_mode(&desc[idx], flow_mode);
901 break;
902 case CC_DMA_BUF_NULL:
903 default:
904 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n");
905 }
906
907 *seq_size = (++idx);
908}
909
910static void cc_proc_digest_desc(struct aead_request *req,
911 struct cc_hw_desc desc[],
912 unsigned int *seq_size)
913{
914 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
915 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
916 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
917 unsigned int idx = *seq_size;
918 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
919 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
920 int direct = req_ctx->gen_ctx.op_type;
921
922
923 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
924 hw_desc_init(&desc[idx]);
925 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
926 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
927 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize,
928 NS_BIT, 1);
929 set_queue_last_ind(ctx->drvdata, &desc[idx]);
930 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
931 set_aes_not_hash_mode(&desc[idx]);
932 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
933 } else {
934 set_cipher_config0(&desc[idx],
935 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
936 set_cipher_mode(&desc[idx], hash_mode);
937 }
938 } else {
939
940 hw_desc_init(&desc[idx]);
941 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
942 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
943 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr,
944 ctx->authsize, NS_BIT, 1);
945 set_queue_last_ind(ctx->drvdata, &desc[idx]);
946 set_cipher_config0(&desc[idx],
947 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
948 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
949 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
950 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
951 set_aes_not_hash_mode(&desc[idx]);
952 } else {
953 set_cipher_mode(&desc[idx], hash_mode);
954 }
955 }
956
957 *seq_size = (++idx);
958}
959
960static void cc_set_cipher_desc(struct aead_request *req,
961 struct cc_hw_desc desc[],
962 unsigned int *seq_size)
963{
964 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
965 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
966 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
967 unsigned int hw_iv_size = req_ctx->hw_iv_size;
968 unsigned int idx = *seq_size;
969 int direct = req_ctx->gen_ctx.op_type;
970
971
972 hw_desc_init(&desc[idx]);
973 set_cipher_config0(&desc[idx], direct);
974 set_flow_mode(&desc[idx], ctx->flow_mode);
975 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr,
976 hw_iv_size, NS_BIT);
977 if (ctx->cipher_mode == DRV_CIPHER_CTR)
978 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
979 else
980 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
981 set_cipher_mode(&desc[idx], ctx->cipher_mode);
982 idx++;
983
984
985 hw_desc_init(&desc[idx]);
986 set_cipher_config0(&desc[idx], direct);
987 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
988 set_flow_mode(&desc[idx], ctx->flow_mode);
989 if (ctx->flow_mode == S_DIN_to_AES) {
990 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
991 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
992 ctx->enc_keylen), NS_BIT);
993 set_key_size_aes(&desc[idx], ctx->enc_keylen);
994 } else {
995 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
996 ctx->enc_keylen, NS_BIT);
997 set_key_size_des(&desc[idx], ctx->enc_keylen);
998 }
999 set_cipher_mode(&desc[idx], ctx->cipher_mode);
1000 idx++;
1001
1002 *seq_size = idx;
1003}
1004
1005static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],
1006 unsigned int *seq_size, unsigned int data_flow_mode)
1007{
1008 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1009 int direct = req_ctx->gen_ctx.op_type;
1010 unsigned int idx = *seq_size;
1011
1012 if (req_ctx->cryptlen == 0)
1013 return;
1014
1015 cc_set_cipher_desc(req, desc, &idx);
1016 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx);
1017 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1018
1019 hw_desc_init(&desc[idx]);
1020 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1021 set_dout_no_dma(&desc[idx], 0, 0, 1);
1022 idx++;
1023 }
1024
1025 *seq_size = idx;
1026}
1027
1028static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[],
1029 unsigned int *seq_size)
1030{
1031 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1032 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1033 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1034 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1035 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1036 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1037 unsigned int idx = *seq_size;
1038
1039
1040 hw_desc_init(&desc[idx]);
1041 set_cipher_mode(&desc[idx], hash_mode);
1042 set_din_type(&desc[idx], DMA_DLLI,
1043 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size,
1044 NS_BIT);
1045 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1046 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1047 idx++;
1048
1049
1050 hw_desc_init(&desc[idx]);
1051 set_cipher_mode(&desc[idx], hash_mode);
1052 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1053 ctx->hash_len);
1054 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1055 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1056 idx++;
1057
1058 *seq_size = idx;
1059}
1060
1061static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[],
1062 unsigned int *seq_size)
1063{
1064 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1065 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1066 unsigned int idx = *seq_size;
1067
1068
1069 hw_desc_init(&desc[idx]);
1070 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE);
1071 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1072 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1073 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1074 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1075 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1076 set_aes_not_hash_mode(&desc[idx]);
1077 idx++;
1078
1079
1080 hw_desc_init(&desc[idx]);
1081 set_din_type(&desc[idx], DMA_DLLI,
1082 ctx->auth_state.xcbc.xcbc_keys_dma_addr,
1083 AES_KEYSIZE_128, NS_BIT);
1084 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1085 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1086 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1087 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1088 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1089 set_aes_not_hash_mode(&desc[idx]);
1090 idx++;
1091
1092
1093 hw_desc_init(&desc[idx]);
1094 set_din_type(&desc[idx], DMA_DLLI,
1095 (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1096 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1097 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1098 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1099 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1100 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1101 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1102 set_aes_not_hash_mode(&desc[idx]);
1103 idx++;
1104
1105
1106 hw_desc_init(&desc[idx]);
1107 set_din_type(&desc[idx], DMA_DLLI,
1108 (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1109 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1110 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
1111 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1112 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1113 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1114 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1115 set_aes_not_hash_mode(&desc[idx]);
1116 idx++;
1117
1118 *seq_size = idx;
1119}
1120
1121static void cc_proc_header_desc(struct aead_request *req,
1122 struct cc_hw_desc desc[],
1123 unsigned int *seq_size)
1124{
1125 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1126 unsigned int idx = *seq_size;
1127
1128
1129 if (areq_ctx->assoclen > 0)
1130 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1131
1132
1133 *seq_size = idx;
1134}
1135
1136static void cc_proc_scheme_desc(struct aead_request *req,
1137 struct cc_hw_desc desc[],
1138 unsigned int *seq_size)
1139{
1140 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1141 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1142 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle;
1143 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1144 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1145 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1146 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1147 unsigned int idx = *seq_size;
1148
1149 hw_desc_init(&desc[idx]);
1150 set_cipher_mode(&desc[idx], hash_mode);
1151 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1152 ctx->hash_len);
1153 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1154 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
1155 set_cipher_do(&desc[idx], DO_PAD);
1156 idx++;
1157
1158
1159 hw_desc_init(&desc[idx]);
1160 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1161 digest_size);
1162 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1163 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1164 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1165 set_cipher_mode(&desc[idx], hash_mode);
1166 idx++;
1167
1168
1169 hw_desc_init(&desc[idx]);
1170 set_cipher_mode(&desc[idx], hash_mode);
1171 set_din_type(&desc[idx], DMA_DLLI,
1172 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size),
1173 digest_size, NS_BIT);
1174 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1175 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1176 idx++;
1177
1178
1179 hw_desc_init(&desc[idx]);
1180 set_cipher_mode(&desc[idx], hash_mode);
1181 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1182 ctx->hash_len);
1183 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1184 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1185 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1186 idx++;
1187
1188
1189 hw_desc_init(&desc[idx]);
1190 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr,
1191 digest_size);
1192 set_flow_mode(&desc[idx], DIN_HASH);
1193 idx++;
1194
1195 *seq_size = idx;
1196}
1197
1198static void cc_mlli_to_sram(struct aead_request *req,
1199 struct cc_hw_desc desc[], unsigned int *seq_size)
1200{
1201 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1202 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1203 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1204 struct device *dev = drvdata_to_dev(ctx->drvdata);
1205
1206 if ((req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
1207 req_ctx->data_buff_type == CC_DMA_BUF_MLLI ||
1208 !req_ctx->is_single_pass) && req_ctx->mlli_params.mlli_len) {
1209 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",
1210 (unsigned int)ctx->drvdata->mlli_sram_addr,
1211 req_ctx->mlli_params.mlli_len);
1212
1213 hw_desc_init(&desc[*seq_size]);
1214 set_din_type(&desc[*seq_size], DMA_DLLI,
1215 req_ctx->mlli_params.mlli_dma_addr,
1216 req_ctx->mlli_params.mlli_len, NS_BIT);
1217 set_dout_sram(&desc[*seq_size],
1218 ctx->drvdata->mlli_sram_addr,
1219 req_ctx->mlli_params.mlli_len);
1220 set_flow_mode(&desc[*seq_size], BYPASS);
1221 (*seq_size)++;
1222 }
1223}
1224
1225static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct,
1226 enum cc_flow_mode setup_flow_mode,
1227 bool is_single_pass)
1228{
1229 enum cc_flow_mode data_flow_mode;
1230
1231 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1232 if (setup_flow_mode == S_DIN_to_AES)
1233 data_flow_mode = is_single_pass ?
1234 AES_to_HASH_and_DOUT : DIN_AES_DOUT;
1235 else
1236 data_flow_mode = is_single_pass ?
1237 DES_to_HASH_and_DOUT : DIN_DES_DOUT;
1238 } else {
1239 if (setup_flow_mode == S_DIN_to_AES)
1240 data_flow_mode = is_single_pass ?
1241 AES_and_HASH : DIN_AES_DOUT;
1242 else
1243 data_flow_mode = is_single_pass ?
1244 DES_and_HASH : DIN_DES_DOUT;
1245 }
1246
1247 return data_flow_mode;
1248}
1249
1250static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1251 unsigned int *seq_size)
1252{
1253 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1254 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1255 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1256 int direct = req_ctx->gen_ctx.op_type;
1257 unsigned int data_flow_mode =
1258 cc_get_data_flow(direct, ctx->flow_mode,
1259 req_ctx->is_single_pass);
1260
1261 if (req_ctx->is_single_pass) {
1262
1263
1264
1265 cc_set_hmac_desc(req, desc, seq_size);
1266 cc_set_cipher_desc(req, desc, seq_size);
1267 cc_proc_header_desc(req, desc, seq_size);
1268 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1269 cc_proc_scheme_desc(req, desc, seq_size);
1270 cc_proc_digest_desc(req, desc, seq_size);
1271 return;
1272 }
1273
1274
1275
1276
1277
1278
1279 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1280
1281 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1282
1283 cc_set_hmac_desc(req, desc, seq_size);
1284 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1285 cc_proc_scheme_desc(req, desc, seq_size);
1286 cc_proc_digest_desc(req, desc, seq_size);
1287
1288 } else {
1289
1290 cc_set_hmac_desc(req, desc, seq_size);
1291 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1292 cc_proc_scheme_desc(req, desc, seq_size);
1293
1294 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1295
1296
1297
1298 cc_proc_digest_desc(req, desc, seq_size);
1299 }
1300}
1301
1302static void
1303cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1304 unsigned int *seq_size)
1305{
1306 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1307 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1308 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1309 int direct = req_ctx->gen_ctx.op_type;
1310 unsigned int data_flow_mode =
1311 cc_get_data_flow(direct, ctx->flow_mode,
1312 req_ctx->is_single_pass);
1313
1314 if (req_ctx->is_single_pass) {
1315
1316
1317
1318 cc_set_xcbc_desc(req, desc, seq_size);
1319 cc_set_cipher_desc(req, desc, seq_size);
1320 cc_proc_header_desc(req, desc, seq_size);
1321 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1322 cc_proc_digest_desc(req, desc, seq_size);
1323 return;
1324 }
1325
1326
1327
1328
1329
1330
1331 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1332
1333 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1334
1335 cc_set_xcbc_desc(req, desc, seq_size);
1336 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1337 cc_proc_digest_desc(req, desc, seq_size);
1338 } else {
1339
1340 cc_set_xcbc_desc(req, desc, seq_size);
1341 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1342
1343 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1344
1345
1346
1347 cc_proc_digest_desc(req, desc, seq_size);
1348 }
1349}
1350
1351static int validate_data_size(struct cc_aead_ctx *ctx,
1352 enum drv_crypto_direction direct,
1353 struct aead_request *req)
1354{
1355 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1356 struct device *dev = drvdata_to_dev(ctx->drvdata);
1357 unsigned int assoclen = areq_ctx->assoclen;
1358 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
1359 (req->cryptlen - ctx->authsize) : req->cryptlen;
1360
1361 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT &&
1362 req->cryptlen < ctx->authsize)
1363 goto data_size_err;
1364
1365 areq_ctx->is_single_pass = true;
1366
1367 switch (ctx->flow_mode) {
1368 case S_DIN_to_AES:
1369 if (ctx->cipher_mode == DRV_CIPHER_CBC &&
1370 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE))
1371 goto data_size_err;
1372 if (ctx->cipher_mode == DRV_CIPHER_CCM)
1373 break;
1374 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1375 if (areq_ctx->plaintext_authenticate_only)
1376 areq_ctx->is_single_pass = false;
1377 break;
1378 }
1379
1380 if (!IS_ALIGNED(assoclen, sizeof(u32)))
1381 areq_ctx->is_single_pass = false;
1382
1383 if (ctx->cipher_mode == DRV_CIPHER_CTR &&
1384 !IS_ALIGNED(cipherlen, sizeof(u32)))
1385 areq_ctx->is_single_pass = false;
1386
1387 break;
1388 case S_DIN_to_DES:
1389 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE))
1390 goto data_size_err;
1391 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE))
1392 areq_ctx->is_single_pass = false;
1393 break;
1394 default:
1395 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode);
1396 goto data_size_err;
1397 }
1398
1399 return 0;
1400
1401data_size_err:
1402 return -EINVAL;
1403}
1404
1405static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)
1406{
1407 unsigned int len = 0;
1408
1409 if (header_size == 0)
1410 return 0;
1411
1412 if (header_size < ((1UL << 16) - (1UL << 8))) {
1413 len = 2;
1414
1415 pa0_buff[0] = (header_size >> 8) & 0xFF;
1416 pa0_buff[1] = header_size & 0xFF;
1417 } else {
1418 len = 6;
1419
1420 pa0_buff[0] = 0xFF;
1421 pa0_buff[1] = 0xFE;
1422 pa0_buff[2] = (header_size >> 24) & 0xFF;
1423 pa0_buff[3] = (header_size >> 16) & 0xFF;
1424 pa0_buff[4] = (header_size >> 8) & 0xFF;
1425 pa0_buff[5] = header_size & 0xFF;
1426 }
1427
1428 return len;
1429}
1430
1431static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize)
1432{
1433 __be32 data;
1434
1435 memset(block, 0, csize);
1436 block += csize;
1437
1438 if (csize >= 4)
1439 csize = 4;
1440 else if (msglen > (1 << (8 * csize)))
1441 return -EOVERFLOW;
1442
1443 data = cpu_to_be32(msglen);
1444 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1445
1446 return 0;
1447}
1448
1449static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],
1450 unsigned int *seq_size)
1451{
1452 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1453 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1454 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1455 unsigned int idx = *seq_size;
1456 unsigned int cipher_flow_mode;
1457 dma_addr_t mac_result;
1458
1459 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1460 cipher_flow_mode = AES_to_HASH_and_DOUT;
1461 mac_result = req_ctx->mac_buf_dma_addr;
1462 } else {
1463 cipher_flow_mode = AES_and_HASH;
1464 mac_result = req_ctx->icv_dma_addr;
1465 }
1466
1467
1468 hw_desc_init(&desc[idx]);
1469 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1470 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1471 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
1472 ctx->enc_keylen), NS_BIT);
1473 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1474 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1475 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1476 set_flow_mode(&desc[idx], S_DIN_to_AES);
1477 idx++;
1478
1479
1480 hw_desc_init(&desc[idx]);
1481 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1482 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1483 set_din_type(&desc[idx], DMA_DLLI,
1484 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT);
1485 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1486 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1487 set_flow_mode(&desc[idx], S_DIN_to_AES);
1488 idx++;
1489
1490
1491 hw_desc_init(&desc[idx]);
1492 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1493 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1494 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
1495 ctx->enc_keylen), NS_BIT);
1496 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1497 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1498 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1499 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1500 set_aes_not_hash_mode(&desc[idx]);
1501 idx++;
1502
1503
1504 hw_desc_init(&desc[idx]);
1505 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1506 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1507 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1508 AES_BLOCK_SIZE, NS_BIT);
1509 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1510 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1511 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1512 set_aes_not_hash_mode(&desc[idx]);
1513 idx++;
1514
1515
1516 if (req_ctx->assoclen > 0) {
1517 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1518 } else {
1519 hw_desc_init(&desc[idx]);
1520 set_din_type(&desc[idx], DMA_DLLI,
1521 sg_dma_address(&req_ctx->ccm_adata_sg),
1522 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT);
1523 set_flow_mode(&desc[idx], DIN_HASH);
1524 idx++;
1525 }
1526
1527
1528 if (req_ctx->cryptlen)
1529 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx);
1530
1531
1532 hw_desc_init(&desc[idx]);
1533 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1534 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize,
1535 NS_BIT, 0);
1536 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1537 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1538 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1539 set_aes_not_hash_mode(&desc[idx]);
1540 idx++;
1541
1542
1543 hw_desc_init(&desc[idx]);
1544 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1545 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1546 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr,
1547 AES_BLOCK_SIZE, NS_BIT);
1548 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1549 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1550 set_flow_mode(&desc[idx], S_DIN_to_AES);
1551 idx++;
1552
1553 hw_desc_init(&desc[idx]);
1554 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1555 set_dout_no_dma(&desc[idx], 0, 0, 1);
1556 idx++;
1557
1558
1559 hw_desc_init(&desc[idx]);
1560 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1561 ctx->authsize, NS_BIT);
1562 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1563 set_queue_last_ind(ctx->drvdata, &desc[idx]);
1564 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1565 idx++;
1566
1567 *seq_size = idx;
1568 return 0;
1569}
1570
1571static int config_ccm_adata(struct aead_request *req)
1572{
1573 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1574 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1575 struct device *dev = drvdata_to_dev(ctx->drvdata);
1576 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1577
1578 unsigned int lp = req->iv[0];
1579
1580
1581
1582 unsigned int l = lp + 1;
1583 unsigned int m = ctx->authsize;
1584 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET;
1585 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET;
1586 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
1587 unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1588 DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1589 req->cryptlen :
1590 (req->cryptlen - ctx->authsize);
1591 int rc;
1592
1593 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1594 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3);
1595
1596
1597
1598 if (l < 2 || l > 8) {
1599 dev_err(dev, "illegal iv value %X\n", req->iv[0]);
1600 return -EINVAL;
1601 }
1602 memcpy(b0, req->iv, AES_BLOCK_SIZE);
1603
1604
1605
1606
1607 *b0 |= (8 * ((m - 2) / 2));
1608 if (req_ctx->assoclen > 0)
1609 *b0 |= 64;
1610
1611 rc = set_msg_len(b0 + 16 - l, cryptlen, l);
1612 if (rc) {
1613 dev_err(dev, "message len overflow detected");
1614 return rc;
1615 }
1616
1617
1618
1619 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen);
1620
1621 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1);
1622 req->iv[15] = 1;
1623
1624 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE);
1625 ctr_count_0[15] = 0;
1626
1627 return 0;
1628}
1629
1630static void cc_proc_rfc4309_ccm(struct aead_request *req)
1631{
1632 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1633 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1634 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1635
1636
1637 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);
1638
1639
1640
1641 areq_ctx->ctr_iv[0] = 3;
1642
1643
1644
1645
1646 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce,
1647 CCM_BLOCK_NONCE_SIZE);
1648 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv,
1649 CCM_BLOCK_IV_SIZE);
1650 req->iv = areq_ctx->ctr_iv;
1651 areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE;
1652}
1653
1654static void cc_set_ghash_desc(struct aead_request *req,
1655 struct cc_hw_desc desc[], unsigned int *seq_size)
1656{
1657 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1658 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1659 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1660 unsigned int idx = *seq_size;
1661
1662
1663 hw_desc_init(&desc[idx]);
1664 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1665 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1666 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1667 ctx->enc_keylen, NS_BIT);
1668 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1669 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1670 set_flow_mode(&desc[idx], S_DIN_to_AES);
1671 idx++;
1672
1673
1674 hw_desc_init(&desc[idx]);
1675 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1676 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE,
1677 NS_BIT, 0);
1678 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1679 idx++;
1680
1681
1682 hw_desc_init(&desc[idx]);
1683 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1684 set_dout_no_dma(&desc[idx], 0, 0, 1);
1685 idx++;
1686
1687
1688 hw_desc_init(&desc[idx]);
1689 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr,
1690 AES_BLOCK_SIZE, NS_BIT);
1691 set_dout_no_dma(&desc[idx], 0, 0, 1);
1692 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1693 set_aes_not_hash_mode(&desc[idx]);
1694 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1695 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1696 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1697 idx++;
1698
1699
1700
1701
1702
1703
1704 hw_desc_init(&desc[idx]);
1705 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1706 set_dout_no_dma(&desc[idx], 0, 0, 1);
1707 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1708 set_aes_not_hash_mode(&desc[idx]);
1709 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1710 set_cipher_do(&desc[idx], 1);
1711 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1712 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1713 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1714 idx++;
1715
1716
1717
1718
1719 hw_desc_init(&desc[idx]);
1720 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1721 set_dout_no_dma(&desc[idx], 0, 0, 1);
1722 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1723 set_aes_not_hash_mode(&desc[idx]);
1724 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1725 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1726 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1727 idx++;
1728
1729 *seq_size = idx;
1730}
1731
1732static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],
1733 unsigned int *seq_size)
1734{
1735 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1736 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1737 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1738 unsigned int idx = *seq_size;
1739
1740
1741 hw_desc_init(&desc[idx]);
1742 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1743 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1744 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1745 ctx->enc_keylen, NS_BIT);
1746 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1747 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1748 set_flow_mode(&desc[idx], S_DIN_to_AES);
1749 idx++;
1750
1751 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) {
1752
1753 hw_desc_init(&desc[idx]);
1754 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1755 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1756 set_din_type(&desc[idx], DMA_DLLI,
1757 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE,
1758 NS_BIT);
1759 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1760 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1761 set_flow_mode(&desc[idx], S_DIN_to_AES);
1762 idx++;
1763 }
1764
1765 *seq_size = idx;
1766}
1767
1768static void cc_proc_gcm_result(struct aead_request *req,
1769 struct cc_hw_desc desc[],
1770 unsigned int *seq_size)
1771{
1772 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1773 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1774 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1775 dma_addr_t mac_result;
1776 unsigned int idx = *seq_size;
1777
1778 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1779 mac_result = req_ctx->mac_buf_dma_addr;
1780 } else {
1781 mac_result = req_ctx->icv_dma_addr;
1782 }
1783
1784
1785 hw_desc_init(&desc[idx]);
1786 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr,
1787 AES_BLOCK_SIZE, NS_BIT);
1788 set_flow_mode(&desc[idx], DIN_HASH);
1789 idx++;
1790
1791
1792 hw_desc_init(&desc[idx]);
1793 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1794 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1795 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE,
1796 NS_BIT, 0);
1797 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1798 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1799 set_aes_not_hash_mode(&desc[idx]);
1800
1801 idx++;
1802
1803
1804 hw_desc_init(&desc[idx]);
1805 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1806 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1807 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr,
1808 AES_BLOCK_SIZE, NS_BIT);
1809 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1810 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1811 set_flow_mode(&desc[idx], S_DIN_to_AES);
1812 idx++;
1813
1814
1815 hw_desc_init(&desc[idx]);
1816 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1817 set_dout_no_dma(&desc[idx], 0, 0, 1);
1818 idx++;
1819
1820
1821 hw_desc_init(&desc[idx]);
1822 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1823 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1824 AES_BLOCK_SIZE, NS_BIT);
1825 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1826 set_queue_last_ind(ctx->drvdata, &desc[idx]);
1827 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1828 idx++;
1829
1830 *seq_size = idx;
1831}
1832
1833static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],
1834 unsigned int *seq_size)
1835{
1836 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1837 unsigned int cipher_flow_mode;
1838
1839 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1840 cipher_flow_mode = AES_and_HASH;
1841 } else {
1842 cipher_flow_mode = AES_to_HASH_and_DOUT;
1843 }
1844
1845
1846 if (req_ctx->plaintext_authenticate_only) {
1847 cc_proc_cipher_desc(req, BYPASS, desc, seq_size);
1848 cc_set_ghash_desc(req, desc, seq_size);
1849
1850 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1851 cc_set_gctr_desc(req, desc, seq_size);
1852 cc_proc_gcm_result(req, desc, seq_size);
1853 return 0;
1854 }
1855
1856
1857 cc_set_ghash_desc(req, desc, seq_size);
1858
1859 if (req_ctx->assoclen > 0)
1860 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1861 cc_set_gctr_desc(req, desc, seq_size);
1862
1863 if (req_ctx->cryptlen)
1864 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size);
1865 cc_proc_gcm_result(req, desc, seq_size);
1866
1867 return 0;
1868}
1869
1870static int config_gcm_context(struct aead_request *req)
1871{
1872 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1873 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1874 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1875 struct device *dev = drvdata_to_dev(ctx->drvdata);
1876
1877 unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1878 DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1879 req->cryptlen :
1880 (req->cryptlen - ctx->authsize);
1881 __be32 counter = cpu_to_be32(2);
1882
1883 dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n",
1884 __func__, cryptlen, req_ctx->assoclen, ctx->authsize);
1885
1886 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);
1887
1888 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1889
1890 memcpy(req->iv + 12, &counter, 4);
1891 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16);
1892
1893 counter = cpu_to_be32(1);
1894 memcpy(req->iv + 12, &counter, 4);
1895 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16);
1896
1897 if (!req_ctx->plaintext_authenticate_only) {
1898 __be64 temp64;
1899
1900 temp64 = cpu_to_be64(req_ctx->assoclen * 8);
1901 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1902 temp64 = cpu_to_be64(cryptlen * 8);
1903 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1904 } else {
1905
1906
1907
1908 __be64 temp64;
1909
1910 temp64 = cpu_to_be64((req_ctx->assoclen +
1911 GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8);
1912 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1913 temp64 = 0;
1914 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1915 }
1916
1917 return 0;
1918}
1919
1920static void cc_proc_rfc4_gcm(struct aead_request *req)
1921{
1922 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1923 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1924 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1925
1926 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,
1927 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);
1928 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv,
1929 GCM_BLOCK_RFC4_IV_SIZE);
1930 req->iv = areq_ctx->ctr_iv;
1931 areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE;
1932}
1933
1934static int cc_proc_aead(struct aead_request *req,
1935 enum drv_crypto_direction direct)
1936{
1937 int rc = 0;
1938 int seq_len = 0;
1939 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];
1940 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1941 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1942 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1943 struct device *dev = drvdata_to_dev(ctx->drvdata);
1944 struct cc_crypto_req cc_req = {};
1945
1946 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",
1947 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"),
1948 ctx, req, req->iv, sg_virt(req->src), req->src->offset,
1949 sg_virt(req->dst), req->dst->offset, req->cryptlen);
1950
1951
1952
1953
1954 if (validate_data_size(ctx, direct, req)) {
1955 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n",
1956 req->cryptlen, areq_ctx->assoclen);
1957 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);
1958 return -EINVAL;
1959 }
1960
1961
1962 cc_req.user_cb = (void *)cc_aead_complete;
1963 cc_req.user_arg = (void *)req;
1964
1965
1966 areq_ctx->gen_ctx.op_type = direct;
1967 areq_ctx->req_authsize = ctx->authsize;
1968 areq_ctx->cipher_mode = ctx->cipher_mode;
1969
1970
1971
1972 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1973
1974
1975
1976 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
1977 CTR_RFC3686_NONCE_SIZE);
1978 if (!areq_ctx->backup_giv)
1979 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE,
1980 req->iv, CTR_RFC3686_IV_SIZE);
1981
1982 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
1983 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1984
1985
1986 req->iv = areq_ctx->ctr_iv;
1987 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE;
1988 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) ||
1989 (ctx->cipher_mode == DRV_CIPHER_GCTR)) {
1990 areq_ctx->hw_iv_size = AES_BLOCK_SIZE;
1991 if (areq_ctx->ctr_iv != req->iv) {
1992 memcpy(areq_ctx->ctr_iv, req->iv,
1993 crypto_aead_ivsize(tfm));
1994 req->iv = areq_ctx->ctr_iv;
1995 }
1996 } else {
1997 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm);
1998 }
1999
2000 if (ctx->cipher_mode == DRV_CIPHER_CCM) {
2001 rc = config_ccm_adata(req);
2002 if (rc) {
2003 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!",
2004 rc);
2005 goto exit;
2006 }
2007 } else {
2008 areq_ctx->ccm_hdr_size = ccm_header_size_null;
2009 }
2010
2011 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
2012 rc = config_gcm_context(req);
2013 if (rc) {
2014 dev_dbg(dev, "config_gcm_context() returned with a failure %d!",
2015 rc);
2016 goto exit;
2017 }
2018 }
2019
2020 rc = cc_map_aead_request(ctx->drvdata, req);
2021 if (rc) {
2022 dev_err(dev, "map_request() failed\n");
2023 goto exit;
2024 }
2025
2026
2027 if (areq_ctx->backup_giv) {
2028
2029 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
2030 cc_req.ivgen_dma_addr[0] =
2031 areq_ctx->gen_ctx.iv_dma_addr +
2032 CTR_RFC3686_NONCE_SIZE;
2033 cc_req.ivgen_dma_addr_len = 1;
2034 } else if (ctx->cipher_mode == DRV_CIPHER_CCM) {
2035
2036
2037
2038
2039
2040 cc_req.ivgen_dma_addr[0] =
2041 areq_ctx->gen_ctx.iv_dma_addr +
2042 CCM_BLOCK_IV_OFFSET;
2043 cc_req.ivgen_dma_addr[1] =
2044 sg_dma_address(&areq_ctx->ccm_adata_sg) +
2045 CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET;
2046 cc_req.ivgen_dma_addr[2] =
2047 sg_dma_address(&areq_ctx->ccm_adata_sg) +
2048 CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET;
2049 cc_req.ivgen_dma_addr_len = 3;
2050 } else {
2051 cc_req.ivgen_dma_addr[0] =
2052 areq_ctx->gen_ctx.iv_dma_addr;
2053 cc_req.ivgen_dma_addr_len = 1;
2054 }
2055
2056
2057 cc_req.ivgen_size = crypto_aead_ivsize(tfm);
2058 }
2059
2060
2061
2062
2063 cc_mlli_to_sram(req, desc, &seq_len);
2064
2065
2066 switch (ctx->auth_mode) {
2067 case DRV_HASH_SHA1:
2068 case DRV_HASH_SHA256:
2069 cc_hmac_authenc(req, desc, &seq_len);
2070 break;
2071 case DRV_HASH_XCBC_MAC:
2072 cc_xcbc_authenc(req, desc, &seq_len);
2073 break;
2074 case DRV_HASH_NULL:
2075 if (ctx->cipher_mode == DRV_CIPHER_CCM)
2076 cc_ccm(req, desc, &seq_len);
2077 if (ctx->cipher_mode == DRV_CIPHER_GCTR)
2078 cc_gcm(req, desc, &seq_len);
2079 break;
2080 default:
2081 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
2082 cc_unmap_aead_request(dev, req);
2083 rc = -ENOTSUPP;
2084 goto exit;
2085 }
2086
2087
2088
2089 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base);
2090
2091 if (rc != -EINPROGRESS && rc != -EBUSY) {
2092 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
2093 cc_unmap_aead_request(dev, req);
2094 }
2095
2096exit:
2097 return rc;
2098}
2099
2100static int cc_aead_encrypt(struct aead_request *req)
2101{
2102 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2103 int rc;
2104
2105 memset(areq_ctx, 0, sizeof(*areq_ctx));
2106
2107
2108 areq_ctx->backup_iv = req->iv;
2109 areq_ctx->assoclen = req->assoclen;
2110 areq_ctx->backup_giv = NULL;
2111 areq_ctx->is_gcm4543 = false;
2112
2113 areq_ctx->plaintext_authenticate_only = false;
2114
2115 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2116 if (rc != -EINPROGRESS && rc != -EBUSY)
2117 req->iv = areq_ctx->backup_iv;
2118
2119 return rc;
2120}
2121
2122static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
2123{
2124
2125
2126 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2127 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2128 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2129 struct device *dev = drvdata_to_dev(ctx->drvdata);
2130 int rc = -EINVAL;
2131
2132 if (!valid_assoclen(req)) {
2133 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2134 goto out;
2135 }
2136
2137 memset(areq_ctx, 0, sizeof(*areq_ctx));
2138
2139
2140 areq_ctx->backup_iv = req->iv;
2141 areq_ctx->assoclen = req->assoclen;
2142 areq_ctx->backup_giv = NULL;
2143 areq_ctx->is_gcm4543 = true;
2144
2145 cc_proc_rfc4309_ccm(req);
2146
2147 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2148 if (rc != -EINPROGRESS && rc != -EBUSY)
2149 req->iv = areq_ctx->backup_iv;
2150out:
2151 return rc;
2152}
2153
2154static int cc_aead_decrypt(struct aead_request *req)
2155{
2156 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2157 int rc;
2158
2159 memset(areq_ctx, 0, sizeof(*areq_ctx));
2160
2161
2162 areq_ctx->backup_iv = req->iv;
2163 areq_ctx->assoclen = req->assoclen;
2164 areq_ctx->backup_giv = NULL;
2165 areq_ctx->is_gcm4543 = false;
2166
2167 areq_ctx->plaintext_authenticate_only = false;
2168
2169 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2170 if (rc != -EINPROGRESS && rc != -EBUSY)
2171 req->iv = areq_ctx->backup_iv;
2172
2173 return rc;
2174}
2175
2176static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
2177{
2178 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2179 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2180 struct device *dev = drvdata_to_dev(ctx->drvdata);
2181 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2182 int rc = -EINVAL;
2183
2184 if (!valid_assoclen(req)) {
2185 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2186 goto out;
2187 }
2188
2189 memset(areq_ctx, 0, sizeof(*areq_ctx));
2190
2191
2192 areq_ctx->backup_iv = req->iv;
2193 areq_ctx->assoclen = req->assoclen;
2194 areq_ctx->backup_giv = NULL;
2195
2196 areq_ctx->is_gcm4543 = true;
2197 cc_proc_rfc4309_ccm(req);
2198
2199 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2200 if (rc != -EINPROGRESS && rc != -EBUSY)
2201 req->iv = areq_ctx->backup_iv;
2202
2203out:
2204 return rc;
2205}
2206
2207static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2208 unsigned int keylen)
2209{
2210 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2211 struct device *dev = drvdata_to_dev(ctx->drvdata);
2212
2213 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);
2214
2215 if (keylen < 4)
2216 return -EINVAL;
2217
2218 keylen -= 4;
2219 memcpy(ctx->ctr_nonce, key + keylen, 4);
2220
2221 return cc_aead_setkey(tfm, key, keylen);
2222}
2223
2224static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2225 unsigned int keylen)
2226{
2227 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2228 struct device *dev = drvdata_to_dev(ctx->drvdata);
2229
2230 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);
2231
2232 if (keylen < 4)
2233 return -EINVAL;
2234
2235 keylen -= 4;
2236 memcpy(ctx->ctr_nonce, key + keylen, 4);
2237
2238 return cc_aead_setkey(tfm, key, keylen);
2239}
2240
2241static int cc_gcm_setauthsize(struct crypto_aead *authenc,
2242 unsigned int authsize)
2243{
2244 switch (authsize) {
2245 case 4:
2246 case 8:
2247 case 12:
2248 case 13:
2249 case 14:
2250 case 15:
2251 case 16:
2252 break;
2253 default:
2254 return -EINVAL;
2255 }
2256
2257 return cc_aead_setauthsize(authenc, authsize);
2258}
2259
2260static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
2261 unsigned int authsize)
2262{
2263 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2264 struct device *dev = drvdata_to_dev(ctx->drvdata);
2265
2266 dev_dbg(dev, "authsize %d\n", authsize);
2267
2268 switch (authsize) {
2269 case 8:
2270 case 12:
2271 case 16:
2272 break;
2273 default:
2274 return -EINVAL;
2275 }
2276
2277 return cc_aead_setauthsize(authenc, authsize);
2278}
2279
2280static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
2281 unsigned int authsize)
2282{
2283 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2284 struct device *dev = drvdata_to_dev(ctx->drvdata);
2285
2286 dev_dbg(dev, "authsize %d\n", authsize);
2287
2288 if (authsize != 16)
2289 return -EINVAL;
2290
2291 return cc_aead_setauthsize(authenc, authsize);
2292}
2293
2294static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
2295{
2296
2297
2298 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2299 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2300 struct device *dev = drvdata_to_dev(ctx->drvdata);
2301 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2302 int rc = -EINVAL;
2303
2304 if (!valid_assoclen(req)) {
2305 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2306 goto out;
2307 }
2308
2309 memset(areq_ctx, 0, sizeof(*areq_ctx));
2310
2311
2312 areq_ctx->backup_iv = req->iv;
2313 areq_ctx->assoclen = req->assoclen;
2314 areq_ctx->backup_giv = NULL;
2315
2316 areq_ctx->plaintext_authenticate_only = false;
2317
2318 cc_proc_rfc4_gcm(req);
2319 areq_ctx->is_gcm4543 = true;
2320
2321 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2322 if (rc != -EINPROGRESS && rc != -EBUSY)
2323 req->iv = areq_ctx->backup_iv;
2324out:
2325 return rc;
2326}
2327
2328static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
2329{
2330
2331
2332 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2333 int rc;
2334
2335 memset(areq_ctx, 0, sizeof(*areq_ctx));
2336
2337
2338 areq_ctx->plaintext_authenticate_only = true;
2339
2340
2341 areq_ctx->backup_iv = req->iv;
2342 areq_ctx->assoclen = req->assoclen;
2343 areq_ctx->backup_giv = NULL;
2344
2345 cc_proc_rfc4_gcm(req);
2346 areq_ctx->is_gcm4543 = true;
2347
2348 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2349 if (rc != -EINPROGRESS && rc != -EBUSY)
2350 req->iv = areq_ctx->backup_iv;
2351
2352 return rc;
2353}
2354
2355static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
2356{
2357
2358
2359 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2360 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2361 struct device *dev = drvdata_to_dev(ctx->drvdata);
2362 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2363 int rc = -EINVAL;
2364
2365 if (!valid_assoclen(req)) {
2366 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2367 goto out;
2368 }
2369
2370 memset(areq_ctx, 0, sizeof(*areq_ctx));
2371
2372
2373 areq_ctx->backup_iv = req->iv;
2374 areq_ctx->assoclen = req->assoclen;
2375 areq_ctx->backup_giv = NULL;
2376
2377 areq_ctx->plaintext_authenticate_only = false;
2378
2379 cc_proc_rfc4_gcm(req);
2380 areq_ctx->is_gcm4543 = true;
2381
2382 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2383 if (rc != -EINPROGRESS && rc != -EBUSY)
2384 req->iv = areq_ctx->backup_iv;
2385out:
2386 return rc;
2387}
2388
2389static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
2390{
2391
2392
2393 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2394 int rc;
2395
2396 memset(areq_ctx, 0, sizeof(*areq_ctx));
2397
2398
2399 areq_ctx->plaintext_authenticate_only = true;
2400
2401
2402 areq_ctx->backup_iv = req->iv;
2403 areq_ctx->assoclen = req->assoclen;
2404 areq_ctx->backup_giv = NULL;
2405
2406 cc_proc_rfc4_gcm(req);
2407 areq_ctx->is_gcm4543 = true;
2408
2409 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2410 if (rc != -EINPROGRESS && rc != -EBUSY)
2411 req->iv = areq_ctx->backup_iv;
2412
2413 return rc;
2414}
2415
2416
2417static struct cc_alg_template aead_algs[] = {
2418 {
2419 .name = "authenc(hmac(sha1),cbc(aes))",
2420 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree",
2421 .blocksize = AES_BLOCK_SIZE,
2422 .template_aead = {
2423 .setkey = cc_aead_setkey,
2424 .setauthsize = cc_aead_setauthsize,
2425 .encrypt = cc_aead_encrypt,
2426 .decrypt = cc_aead_decrypt,
2427 .init = cc_aead_init,
2428 .exit = cc_aead_exit,
2429 .ivsize = AES_BLOCK_SIZE,
2430 .maxauthsize = SHA1_DIGEST_SIZE,
2431 },
2432 .cipher_mode = DRV_CIPHER_CBC,
2433 .flow_mode = S_DIN_to_AES,
2434 .auth_mode = DRV_HASH_SHA1,
2435 .min_hw_rev = CC_HW_REV_630,
2436 .std_body = CC_STD_NIST,
2437 },
2438 {
2439 .name = "authenc(hmac(sha1),cbc(des3_ede))",
2440 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree",
2441 .blocksize = DES3_EDE_BLOCK_SIZE,
2442 .template_aead = {
2443 .setkey = cc_des3_aead_setkey,
2444 .setauthsize = cc_aead_setauthsize,
2445 .encrypt = cc_aead_encrypt,
2446 .decrypt = cc_aead_decrypt,
2447 .init = cc_aead_init,
2448 .exit = cc_aead_exit,
2449 .ivsize = DES3_EDE_BLOCK_SIZE,
2450 .maxauthsize = SHA1_DIGEST_SIZE,
2451 },
2452 .cipher_mode = DRV_CIPHER_CBC,
2453 .flow_mode = S_DIN_to_DES,
2454 .auth_mode = DRV_HASH_SHA1,
2455 .min_hw_rev = CC_HW_REV_630,
2456 .std_body = CC_STD_NIST,
2457 },
2458 {
2459 .name = "authenc(hmac(sha256),cbc(aes))",
2460 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree",
2461 .blocksize = AES_BLOCK_SIZE,
2462 .template_aead = {
2463 .setkey = cc_aead_setkey,
2464 .setauthsize = cc_aead_setauthsize,
2465 .encrypt = cc_aead_encrypt,
2466 .decrypt = cc_aead_decrypt,
2467 .init = cc_aead_init,
2468 .exit = cc_aead_exit,
2469 .ivsize = AES_BLOCK_SIZE,
2470 .maxauthsize = SHA256_DIGEST_SIZE,
2471 },
2472 .cipher_mode = DRV_CIPHER_CBC,
2473 .flow_mode = S_DIN_to_AES,
2474 .auth_mode = DRV_HASH_SHA256,
2475 .min_hw_rev = CC_HW_REV_630,
2476 .std_body = CC_STD_NIST,
2477 },
2478 {
2479 .name = "authenc(hmac(sha256),cbc(des3_ede))",
2480 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree",
2481 .blocksize = DES3_EDE_BLOCK_SIZE,
2482 .template_aead = {
2483 .setkey = cc_des3_aead_setkey,
2484 .setauthsize = cc_aead_setauthsize,
2485 .encrypt = cc_aead_encrypt,
2486 .decrypt = cc_aead_decrypt,
2487 .init = cc_aead_init,
2488 .exit = cc_aead_exit,
2489 .ivsize = DES3_EDE_BLOCK_SIZE,
2490 .maxauthsize = SHA256_DIGEST_SIZE,
2491 },
2492 .cipher_mode = DRV_CIPHER_CBC,
2493 .flow_mode = S_DIN_to_DES,
2494 .auth_mode = DRV_HASH_SHA256,
2495 .min_hw_rev = CC_HW_REV_630,
2496 .std_body = CC_STD_NIST,
2497 },
2498 {
2499 .name = "authenc(xcbc(aes),cbc(aes))",
2500 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree",
2501 .blocksize = AES_BLOCK_SIZE,
2502 .template_aead = {
2503 .setkey = cc_aead_setkey,
2504 .setauthsize = cc_aead_setauthsize,
2505 .encrypt = cc_aead_encrypt,
2506 .decrypt = cc_aead_decrypt,
2507 .init = cc_aead_init,
2508 .exit = cc_aead_exit,
2509 .ivsize = AES_BLOCK_SIZE,
2510 .maxauthsize = AES_BLOCK_SIZE,
2511 },
2512 .cipher_mode = DRV_CIPHER_CBC,
2513 .flow_mode = S_DIN_to_AES,
2514 .auth_mode = DRV_HASH_XCBC_MAC,
2515 .min_hw_rev = CC_HW_REV_630,
2516 .std_body = CC_STD_NIST,
2517 },
2518 {
2519 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2520 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",
2521 .blocksize = 1,
2522 .template_aead = {
2523 .setkey = cc_aead_setkey,
2524 .setauthsize = cc_aead_setauthsize,
2525 .encrypt = cc_aead_encrypt,
2526 .decrypt = cc_aead_decrypt,
2527 .init = cc_aead_init,
2528 .exit = cc_aead_exit,
2529 .ivsize = CTR_RFC3686_IV_SIZE,
2530 .maxauthsize = SHA1_DIGEST_SIZE,
2531 },
2532 .cipher_mode = DRV_CIPHER_CTR,
2533 .flow_mode = S_DIN_to_AES,
2534 .auth_mode = DRV_HASH_SHA1,
2535 .min_hw_rev = CC_HW_REV_630,
2536 .std_body = CC_STD_NIST,
2537 },
2538 {
2539 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2540 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",
2541 .blocksize = 1,
2542 .template_aead = {
2543 .setkey = cc_aead_setkey,
2544 .setauthsize = cc_aead_setauthsize,
2545 .encrypt = cc_aead_encrypt,
2546 .decrypt = cc_aead_decrypt,
2547 .init = cc_aead_init,
2548 .exit = cc_aead_exit,
2549 .ivsize = CTR_RFC3686_IV_SIZE,
2550 .maxauthsize = SHA256_DIGEST_SIZE,
2551 },
2552 .cipher_mode = DRV_CIPHER_CTR,
2553 .flow_mode = S_DIN_to_AES,
2554 .auth_mode = DRV_HASH_SHA256,
2555 .min_hw_rev = CC_HW_REV_630,
2556 .std_body = CC_STD_NIST,
2557 },
2558 {
2559 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",
2560 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",
2561 .blocksize = 1,
2562 .template_aead = {
2563 .setkey = cc_aead_setkey,
2564 .setauthsize = cc_aead_setauthsize,
2565 .encrypt = cc_aead_encrypt,
2566 .decrypt = cc_aead_decrypt,
2567 .init = cc_aead_init,
2568 .exit = cc_aead_exit,
2569 .ivsize = CTR_RFC3686_IV_SIZE,
2570 .maxauthsize = AES_BLOCK_SIZE,
2571 },
2572 .cipher_mode = DRV_CIPHER_CTR,
2573 .flow_mode = S_DIN_to_AES,
2574 .auth_mode = DRV_HASH_XCBC_MAC,
2575 .min_hw_rev = CC_HW_REV_630,
2576 .std_body = CC_STD_NIST,
2577 },
2578 {
2579 .name = "ccm(aes)",
2580 .driver_name = "ccm-aes-ccree",
2581 .blocksize = 1,
2582 .template_aead = {
2583 .setkey = cc_aead_setkey,
2584 .setauthsize = cc_ccm_setauthsize,
2585 .encrypt = cc_aead_encrypt,
2586 .decrypt = cc_aead_decrypt,
2587 .init = cc_aead_init,
2588 .exit = cc_aead_exit,
2589 .ivsize = AES_BLOCK_SIZE,
2590 .maxauthsize = AES_BLOCK_SIZE,
2591 },
2592 .cipher_mode = DRV_CIPHER_CCM,
2593 .flow_mode = S_DIN_to_AES,
2594 .auth_mode = DRV_HASH_NULL,
2595 .min_hw_rev = CC_HW_REV_630,
2596 .std_body = CC_STD_NIST,
2597 },
2598 {
2599 .name = "rfc4309(ccm(aes))",
2600 .driver_name = "rfc4309-ccm-aes-ccree",
2601 .blocksize = 1,
2602 .template_aead = {
2603 .setkey = cc_rfc4309_ccm_setkey,
2604 .setauthsize = cc_rfc4309_ccm_setauthsize,
2605 .encrypt = cc_rfc4309_ccm_encrypt,
2606 .decrypt = cc_rfc4309_ccm_decrypt,
2607 .init = cc_aead_init,
2608 .exit = cc_aead_exit,
2609 .ivsize = CCM_BLOCK_IV_SIZE,
2610 .maxauthsize = AES_BLOCK_SIZE,
2611 },
2612 .cipher_mode = DRV_CIPHER_CCM,
2613 .flow_mode = S_DIN_to_AES,
2614 .auth_mode = DRV_HASH_NULL,
2615 .min_hw_rev = CC_HW_REV_630,
2616 .std_body = CC_STD_NIST,
2617 },
2618 {
2619 .name = "gcm(aes)",
2620 .driver_name = "gcm-aes-ccree",
2621 .blocksize = 1,
2622 .template_aead = {
2623 .setkey = cc_aead_setkey,
2624 .setauthsize = cc_gcm_setauthsize,
2625 .encrypt = cc_aead_encrypt,
2626 .decrypt = cc_aead_decrypt,
2627 .init = cc_aead_init,
2628 .exit = cc_aead_exit,
2629 .ivsize = 12,
2630 .maxauthsize = AES_BLOCK_SIZE,
2631 },
2632 .cipher_mode = DRV_CIPHER_GCTR,
2633 .flow_mode = S_DIN_to_AES,
2634 .auth_mode = DRV_HASH_NULL,
2635 .min_hw_rev = CC_HW_REV_630,
2636 .std_body = CC_STD_NIST,
2637 },
2638 {
2639 .name = "rfc4106(gcm(aes))",
2640 .driver_name = "rfc4106-gcm-aes-ccree",
2641 .blocksize = 1,
2642 .template_aead = {
2643 .setkey = cc_rfc4106_gcm_setkey,
2644 .setauthsize = cc_rfc4106_gcm_setauthsize,
2645 .encrypt = cc_rfc4106_gcm_encrypt,
2646 .decrypt = cc_rfc4106_gcm_decrypt,
2647 .init = cc_aead_init,
2648 .exit = cc_aead_exit,
2649 .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2650 .maxauthsize = AES_BLOCK_SIZE,
2651 },
2652 .cipher_mode = DRV_CIPHER_GCTR,
2653 .flow_mode = S_DIN_to_AES,
2654 .auth_mode = DRV_HASH_NULL,
2655 .min_hw_rev = CC_HW_REV_630,
2656 .std_body = CC_STD_NIST,
2657 },
2658 {
2659 .name = "rfc4543(gcm(aes))",
2660 .driver_name = "rfc4543-gcm-aes-ccree",
2661 .blocksize = 1,
2662 .template_aead = {
2663 .setkey = cc_rfc4543_gcm_setkey,
2664 .setauthsize = cc_rfc4543_gcm_setauthsize,
2665 .encrypt = cc_rfc4543_gcm_encrypt,
2666 .decrypt = cc_rfc4543_gcm_decrypt,
2667 .init = cc_aead_init,
2668 .exit = cc_aead_exit,
2669 .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2670 .maxauthsize = AES_BLOCK_SIZE,
2671 },
2672 .cipher_mode = DRV_CIPHER_GCTR,
2673 .flow_mode = S_DIN_to_AES,
2674 .auth_mode = DRV_HASH_NULL,
2675 .min_hw_rev = CC_HW_REV_630,
2676 .std_body = CC_STD_NIST,
2677 },
2678};
2679
2680static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
2681 struct device *dev)
2682{
2683 struct cc_crypto_alg *t_alg;
2684 struct aead_alg *alg;
2685
2686 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
2687 if (!t_alg)
2688 return ERR_PTR(-ENOMEM);
2689
2690 alg = &tmpl->template_aead;
2691
2692 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
2693 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2694 tmpl->driver_name);
2695 alg->base.cra_module = THIS_MODULE;
2696 alg->base.cra_priority = CC_CRA_PRIO;
2697
2698 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);
2699 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2700 alg->init = cc_aead_init;
2701 alg->exit = cc_aead_exit;
2702
2703 t_alg->aead_alg = *alg;
2704
2705 t_alg->cipher_mode = tmpl->cipher_mode;
2706 t_alg->flow_mode = tmpl->flow_mode;
2707 t_alg->auth_mode = tmpl->auth_mode;
2708
2709 return t_alg;
2710}
2711
2712int cc_aead_free(struct cc_drvdata *drvdata)
2713{
2714 struct cc_crypto_alg *t_alg, *n;
2715 struct cc_aead_handle *aead_handle =
2716 (struct cc_aead_handle *)drvdata->aead_handle;
2717
2718 if (aead_handle) {
2719
2720 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list,
2721 entry) {
2722 crypto_unregister_aead(&t_alg->aead_alg);
2723 list_del(&t_alg->entry);
2724 kfree(t_alg);
2725 }
2726 kfree(aead_handle);
2727 drvdata->aead_handle = NULL;
2728 }
2729
2730 return 0;
2731}
2732
2733int cc_aead_alloc(struct cc_drvdata *drvdata)
2734{
2735 struct cc_aead_handle *aead_handle;
2736 struct cc_crypto_alg *t_alg;
2737 int rc = -ENOMEM;
2738 int alg;
2739 struct device *dev = drvdata_to_dev(drvdata);
2740
2741 aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL);
2742 if (!aead_handle) {
2743 rc = -ENOMEM;
2744 goto fail0;
2745 }
2746
2747 INIT_LIST_HEAD(&aead_handle->aead_list);
2748 drvdata->aead_handle = aead_handle;
2749
2750 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata,
2751 MAX_HMAC_DIGEST_SIZE);
2752
2753 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) {
2754 dev_err(dev, "SRAM pool exhausted\n");
2755 rc = -ENOMEM;
2756 goto fail1;
2757 }
2758
2759
2760 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) {
2761 if ((aead_algs[alg].min_hw_rev > drvdata->hw_rev) ||
2762 !(drvdata->std_bodies & aead_algs[alg].std_body))
2763 continue;
2764
2765 t_alg = cc_create_aead_alg(&aead_algs[alg], dev);
2766 if (IS_ERR(t_alg)) {
2767 rc = PTR_ERR(t_alg);
2768 dev_err(dev, "%s alg allocation failed\n",
2769 aead_algs[alg].driver_name);
2770 goto fail1;
2771 }
2772 t_alg->drvdata = drvdata;
2773 rc = crypto_register_aead(&t_alg->aead_alg);
2774 if (rc) {
2775 dev_err(dev, "%s alg registration failed\n",
2776 t_alg->aead_alg.base.cra_driver_name);
2777 goto fail2;
2778 } else {
2779 list_add_tail(&t_alg->entry, &aead_handle->aead_list);
2780 dev_dbg(dev, "Registered %s\n",
2781 t_alg->aead_alg.base.cra_driver_name);
2782 }
2783 }
2784
2785 return 0;
2786
2787fail2:
2788 kfree(t_alg);
2789fail1:
2790 cc_aead_free(drvdata);
2791fail0:
2792 return rc;
2793}
2794
