1
2
3
4
5
6
7
8#include <asm/hwcap.h>
9#include <asm/neon.h>
10#include <asm/simd.h>
11#include <asm/unaligned.h>
12#include <crypto/aes.h>
13#include <crypto/ctr.h>
14#include <crypto/internal/simd.h>
15#include <crypto/internal/skcipher.h>
16#include <crypto/scatterwalk.h>
17#include <linux/cpufeature.h>
18#include <linux/module.h>
19#include <crypto/xts.h>
20
21MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
22MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
23MODULE_LICENSE("GPL v2");
24
25
26asmlinkage u32 ce_aes_sub(u32 input);
27asmlinkage void ce_aes_invert(void *dst, void *src);
28
29asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
30 int rounds, int blocks);
31asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
32 int rounds, int blocks);
33
34asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
35 int rounds, int blocks, u8 iv[]);
36asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
37 int rounds, int blocks, u8 iv[]);
38asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
39 int rounds, int bytes, u8 const iv[]);
40asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
41 int rounds, int bytes, u8 const iv[]);
42
43asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
44 int rounds, int blocks, u8 ctr[]);
45
46asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
47 int rounds, int bytes, u8 iv[],
48 u32 const rk2[], int first);
49asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
50 int rounds, int bytes, u8 iv[],
51 u32 const rk2[], int first);
52
53struct aes_block {
54 u8 b[AES_BLOCK_SIZE];
55};
56
57static int num_rounds(struct crypto_aes_ctx *ctx)
58{
59
60
61
62
63
64
65
66 return 6 + ctx->key_length / 4;
67}
68
69static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
70 unsigned int key_len)
71{
72
73
74
75 static u8 const rcon[] = {
76 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
77 };
78
79 u32 kwords = key_len / sizeof(u32);
80 struct aes_block *key_enc, *key_dec;
81 int i, j;
82
83 if (key_len != AES_KEYSIZE_128 &&
84 key_len != AES_KEYSIZE_192 &&
85 key_len != AES_KEYSIZE_256)
86 return -EINVAL;
87
88 ctx->key_length = key_len;
89 for (i = 0; i < kwords; i++)
90 ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
91
92 kernel_neon_begin();
93 for (i = 0; i < sizeof(rcon); i++) {
94 u32 *rki = ctx->key_enc + (i * kwords);
95 u32 *rko = rki + kwords;
96
97 rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8);
98 rko[0] = rko[0] ^ rki[0] ^ rcon[i];
99 rko[1] = rko[0] ^ rki[1];
100 rko[2] = rko[1] ^ rki[2];
101 rko[3] = rko[2] ^ rki[3];
102
103 if (key_len == AES_KEYSIZE_192) {
104 if (i >= 7)
105 break;
106 rko[4] = rko[3] ^ rki[4];
107 rko[5] = rko[4] ^ rki[5];
108 } else if (key_len == AES_KEYSIZE_256) {
109 if (i >= 6)
110 break;
111 rko[4] = ce_aes_sub(rko[3]) ^ rki[4];
112 rko[5] = rko[4] ^ rki[5];
113 rko[6] = rko[5] ^ rki[6];
114 rko[7] = rko[6] ^ rki[7];
115 }
116 }
117
118
119
120
121
122
123
124 key_enc = (struct aes_block *)ctx->key_enc;
125 key_dec = (struct aes_block *)ctx->key_dec;
126 j = num_rounds(ctx);
127
128 key_dec[0] = key_enc[j];
129 for (i = 1, j--; j > 0; i++, j--)
130 ce_aes_invert(key_dec + i, key_enc + j);
131 key_dec[i] = key_enc[0];
132
133 kernel_neon_end();
134 return 0;
135}
136
137static int ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
138 unsigned int key_len)
139{
140 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
141
142 return ce_aes_expandkey(ctx, in_key, key_len);
143}
144
145struct crypto_aes_xts_ctx {
146 struct crypto_aes_ctx key1;
147 struct crypto_aes_ctx __aligned(8) key2;
148};
149
150static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
151 unsigned int key_len)
152{
153 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
154 int ret;
155
156 ret = xts_verify_key(tfm, in_key, key_len);
157 if (ret)
158 return ret;
159
160 ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2);
161 if (!ret)
162 ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2],
163 key_len / 2);
164 return ret;
165}
166
167static int ecb_encrypt(struct skcipher_request *req)
168{
169 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
170 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
171 struct skcipher_walk walk;
172 unsigned int blocks;
173 int err;
174
175 err = skcipher_walk_virt(&walk, req, false);
176
177 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
178 kernel_neon_begin();
179 ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
180 ctx->key_enc, num_rounds(ctx), blocks);
181 kernel_neon_end();
182 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
183 }
184 return err;
185}
186
187static int ecb_decrypt(struct skcipher_request *req)
188{
189 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
190 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
191 struct skcipher_walk walk;
192 unsigned int blocks;
193 int err;
194
195 err = skcipher_walk_virt(&walk, req, false);
196
197 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
198 kernel_neon_begin();
199 ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
200 ctx->key_dec, num_rounds(ctx), blocks);
201 kernel_neon_end();
202 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
203 }
204 return err;
205}
206
207static int cbc_encrypt_walk(struct skcipher_request *req,
208 struct skcipher_walk *walk)
209{
210 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
211 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
212 unsigned int blocks;
213 int err = 0;
214
215 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
216 kernel_neon_begin();
217 ce_aes_cbc_encrypt(walk->dst.virt.addr, walk->src.virt.addr,
218 ctx->key_enc, num_rounds(ctx), blocks,
219 walk->iv);
220 kernel_neon_end();
221 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
222 }
223 return err;
224}
225
226static int cbc_encrypt(struct skcipher_request *req)
227{
228 struct skcipher_walk walk;
229 int err;
230
231 err = skcipher_walk_virt(&walk, req, false);
232 if (err)
233 return err;
234 return cbc_encrypt_walk(req, &walk);
235}
236
237static int cbc_decrypt_walk(struct skcipher_request *req,
238 struct skcipher_walk *walk)
239{
240 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
241 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
242 unsigned int blocks;
243 int err = 0;
244
245 while ((blocks = (walk->nbytes / AES_BLOCK_SIZE))) {
246 kernel_neon_begin();
247 ce_aes_cbc_decrypt(walk->dst.virt.addr, walk->src.virt.addr,
248 ctx->key_dec, num_rounds(ctx), blocks,
249 walk->iv);
250 kernel_neon_end();
251 err = skcipher_walk_done(walk, walk->nbytes % AES_BLOCK_SIZE);
252 }
253 return err;
254}
255
256static int cbc_decrypt(struct skcipher_request *req)
257{
258 struct skcipher_walk walk;
259 int err;
260
261 err = skcipher_walk_virt(&walk, req, false);
262 if (err)
263 return err;
264 return cbc_decrypt_walk(req, &walk);
265}
266
267static int cts_cbc_encrypt(struct skcipher_request *req)
268{
269 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
270 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
271 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
272 struct scatterlist *src = req->src, *dst = req->dst;
273 struct scatterlist sg_src[2], sg_dst[2];
274 struct skcipher_request subreq;
275 struct skcipher_walk walk;
276 int err;
277
278 skcipher_request_set_tfm(&subreq, tfm);
279 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
280 NULL, NULL);
281
282 if (req->cryptlen <= AES_BLOCK_SIZE) {
283 if (req->cryptlen < AES_BLOCK_SIZE)
284 return -EINVAL;
285 cbc_blocks = 1;
286 }
287
288 if (cbc_blocks > 0) {
289 skcipher_request_set_crypt(&subreq, req->src, req->dst,
290 cbc_blocks * AES_BLOCK_SIZE,
291 req->iv);
292
293 err = skcipher_walk_virt(&walk, &subreq, false) ?:
294 cbc_encrypt_walk(&subreq, &walk);
295 if (err)
296 return err;
297
298 if (req->cryptlen == AES_BLOCK_SIZE)
299 return 0;
300
301 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
302 if (req->dst != req->src)
303 dst = scatterwalk_ffwd(sg_dst, req->dst,
304 subreq.cryptlen);
305 }
306
307
308 skcipher_request_set_crypt(&subreq, src, dst,
309 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
310 req->iv);
311
312 err = skcipher_walk_virt(&walk, &subreq, false);
313 if (err)
314 return err;
315
316 kernel_neon_begin();
317 ce_aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
318 ctx->key_enc, num_rounds(ctx), walk.nbytes,
319 walk.iv);
320 kernel_neon_end();
321
322 return skcipher_walk_done(&walk, 0);
323}
324
325static int cts_cbc_decrypt(struct skcipher_request *req)
326{
327 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
328 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
329 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
330 struct scatterlist *src = req->src, *dst = req->dst;
331 struct scatterlist sg_src[2], sg_dst[2];
332 struct skcipher_request subreq;
333 struct skcipher_walk walk;
334 int err;
335
336 skcipher_request_set_tfm(&subreq, tfm);
337 skcipher_request_set_callback(&subreq, skcipher_request_flags(req),
338 NULL, NULL);
339
340 if (req->cryptlen <= AES_BLOCK_SIZE) {
341 if (req->cryptlen < AES_BLOCK_SIZE)
342 return -EINVAL;
343 cbc_blocks = 1;
344 }
345
346 if (cbc_blocks > 0) {
347 skcipher_request_set_crypt(&subreq, req->src, req->dst,
348 cbc_blocks * AES_BLOCK_SIZE,
349 req->iv);
350
351 err = skcipher_walk_virt(&walk, &subreq, false) ?:
352 cbc_decrypt_walk(&subreq, &walk);
353 if (err)
354 return err;
355
356 if (req->cryptlen == AES_BLOCK_SIZE)
357 return 0;
358
359 dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen);
360 if (req->dst != req->src)
361 dst = scatterwalk_ffwd(sg_dst, req->dst,
362 subreq.cryptlen);
363 }
364
365
366 skcipher_request_set_crypt(&subreq, src, dst,
367 req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
368 req->iv);
369
370 err = skcipher_walk_virt(&walk, &subreq, false);
371 if (err)
372 return err;
373
374 kernel_neon_begin();
375 ce_aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
376 ctx->key_dec, num_rounds(ctx), walk.nbytes,
377 walk.iv);
378 kernel_neon_end();
379
380 return skcipher_walk_done(&walk, 0);
381}
382
383static int ctr_encrypt(struct skcipher_request *req)
384{
385 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
386 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
387 struct skcipher_walk walk;
388 int err, blocks;
389
390 err = skcipher_walk_virt(&walk, req, false);
391
392 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
393 kernel_neon_begin();
394 ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
395 ctx->key_enc, num_rounds(ctx), blocks,
396 walk.iv);
397 kernel_neon_end();
398 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
399 }
400 if (walk.nbytes) {
401 u8 __aligned(8) tail[AES_BLOCK_SIZE];
402 unsigned int nbytes = walk.nbytes;
403 u8 *tdst = walk.dst.virt.addr;
404 u8 *tsrc = walk.src.virt.addr;
405
406
407
408
409 blocks = -1;
410
411 kernel_neon_begin();
412 ce_aes_ctr_encrypt(tail, NULL, ctx->key_enc, num_rounds(ctx),
413 blocks, walk.iv);
414 kernel_neon_end();
415 crypto_xor_cpy(tdst, tsrc, tail, nbytes);
416 err = skcipher_walk_done(&walk, 0);
417 }
418 return err;
419}
420
421static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
422{
423 struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
424 unsigned long flags;
425
426
427
428
429
430
431 local_irq_save(flags);
432 aes_encrypt(ctx, dst, src);
433 local_irq_restore(flags);
434}
435
436static int ctr_encrypt_sync(struct skcipher_request *req)
437{
438 if (!crypto_simd_usable())
439 return crypto_ctr_encrypt_walk(req, ctr_encrypt_one);
440
441 return ctr_encrypt(req);
442}
443
444static int xts_encrypt(struct skcipher_request *req)
445{
446 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
447 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
448 int err, first, rounds = num_rounds(&ctx->key1);
449 int tail = req->cryptlen % AES_BLOCK_SIZE;
450 struct scatterlist sg_src[2], sg_dst[2];
451 struct skcipher_request subreq;
452 struct scatterlist *src, *dst;
453 struct skcipher_walk walk;
454
455 if (req->cryptlen < AES_BLOCK_SIZE)
456 return -EINVAL;
457
458 err = skcipher_walk_virt(&walk, req, false);
459
460 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
461 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
462 AES_BLOCK_SIZE) - 2;
463
464 skcipher_walk_abort(&walk);
465
466 skcipher_request_set_tfm(&subreq, tfm);
467 skcipher_request_set_callback(&subreq,
468 skcipher_request_flags(req),
469 NULL, NULL);
470 skcipher_request_set_crypt(&subreq, req->src, req->dst,
471 xts_blocks * AES_BLOCK_SIZE,
472 req->iv);
473 req = &subreq;
474 err = skcipher_walk_virt(&walk, req, false);
475 } else {
476 tail = 0;
477 }
478
479 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
480 int nbytes = walk.nbytes;
481
482 if (walk.nbytes < walk.total)
483 nbytes &= ~(AES_BLOCK_SIZE - 1);
484
485 kernel_neon_begin();
486 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
487 ctx->key1.key_enc, rounds, nbytes, walk.iv,
488 ctx->key2.key_enc, first);
489 kernel_neon_end();
490 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
491 }
492
493 if (err || likely(!tail))
494 return err;
495
496 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
497 if (req->dst != req->src)
498 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
499
500 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
501 req->iv);
502
503 err = skcipher_walk_virt(&walk, req, false);
504 if (err)
505 return err;
506
507 kernel_neon_begin();
508 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
509 ctx->key1.key_enc, rounds, walk.nbytes, walk.iv,
510 ctx->key2.key_enc, first);
511 kernel_neon_end();
512
513 return skcipher_walk_done(&walk, 0);
514}
515
516static int xts_decrypt(struct skcipher_request *req)
517{
518 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
519 struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
520 int err, first, rounds = num_rounds(&ctx->key1);
521 int tail = req->cryptlen % AES_BLOCK_SIZE;
522 struct scatterlist sg_src[2], sg_dst[2];
523 struct skcipher_request subreq;
524 struct scatterlist *src, *dst;
525 struct skcipher_walk walk;
526
527 if (req->cryptlen < AES_BLOCK_SIZE)
528 return -EINVAL;
529
530 err = skcipher_walk_virt(&walk, req, false);
531
532 if (unlikely(tail > 0 && walk.nbytes < walk.total)) {
533 int xts_blocks = DIV_ROUND_UP(req->cryptlen,
534 AES_BLOCK_SIZE) - 2;
535
536 skcipher_walk_abort(&walk);
537
538 skcipher_request_set_tfm(&subreq, tfm);
539 skcipher_request_set_callback(&subreq,
540 skcipher_request_flags(req),
541 NULL, NULL);
542 skcipher_request_set_crypt(&subreq, req->src, req->dst,
543 xts_blocks * AES_BLOCK_SIZE,
544 req->iv);
545 req = &subreq;
546 err = skcipher_walk_virt(&walk, req, false);
547 } else {
548 tail = 0;
549 }
550
551 for (first = 1; walk.nbytes >= AES_BLOCK_SIZE; first = 0) {
552 int nbytes = walk.nbytes;
553
554 if (walk.nbytes < walk.total)
555 nbytes &= ~(AES_BLOCK_SIZE - 1);
556
557 kernel_neon_begin();
558 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
559 ctx->key1.key_dec, rounds, nbytes, walk.iv,
560 ctx->key2.key_enc, first);
561 kernel_neon_end();
562 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
563 }
564
565 if (err || likely(!tail))
566 return err;
567
568 dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
569 if (req->dst != req->src)
570 dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
571
572 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
573 req->iv);
574
575 err = skcipher_walk_virt(&walk, req, false);
576 if (err)
577 return err;
578
579 kernel_neon_begin();
580 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
581 ctx->key1.key_dec, rounds, walk.nbytes, walk.iv,
582 ctx->key2.key_enc, first);
583 kernel_neon_end();
584
585 return skcipher_walk_done(&walk, 0);
586}
587
588static struct skcipher_alg aes_algs[] = { {
589 .base.cra_name = "__ecb(aes)",
590 .base.cra_driver_name = "__ecb-aes-ce",
591 .base.cra_priority = 300,
592 .base.cra_flags = CRYPTO_ALG_INTERNAL,
593 .base.cra_blocksize = AES_BLOCK_SIZE,
594 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
595 .base.cra_module = THIS_MODULE,
596
597 .min_keysize = AES_MIN_KEY_SIZE,
598 .max_keysize = AES_MAX_KEY_SIZE,
599 .setkey = ce_aes_setkey,
600 .encrypt = ecb_encrypt,
601 .decrypt = ecb_decrypt,
602}, {
603 .base.cra_name = "__cbc(aes)",
604 .base.cra_driver_name = "__cbc-aes-ce",
605 .base.cra_priority = 300,
606 .base.cra_flags = CRYPTO_ALG_INTERNAL,
607 .base.cra_blocksize = AES_BLOCK_SIZE,
608 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
609 .base.cra_module = THIS_MODULE,
610
611 .min_keysize = AES_MIN_KEY_SIZE,
612 .max_keysize = AES_MAX_KEY_SIZE,
613 .ivsize = AES_BLOCK_SIZE,
614 .setkey = ce_aes_setkey,
615 .encrypt = cbc_encrypt,
616 .decrypt = cbc_decrypt,
617}, {
618 .base.cra_name = "__cts(cbc(aes))",
619 .base.cra_driver_name = "__cts-cbc-aes-ce",
620 .base.cra_priority = 300,
621 .base.cra_flags = CRYPTO_ALG_INTERNAL,
622 .base.cra_blocksize = AES_BLOCK_SIZE,
623 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
624 .base.cra_module = THIS_MODULE,
625
626 .min_keysize = AES_MIN_KEY_SIZE,
627 .max_keysize = AES_MAX_KEY_SIZE,
628 .ivsize = AES_BLOCK_SIZE,
629 .walksize = 2 * AES_BLOCK_SIZE,
630 .setkey = ce_aes_setkey,
631 .encrypt = cts_cbc_encrypt,
632 .decrypt = cts_cbc_decrypt,
633}, {
634 .base.cra_name = "__ctr(aes)",
635 .base.cra_driver_name = "__ctr-aes-ce",
636 .base.cra_priority = 300,
637 .base.cra_flags = CRYPTO_ALG_INTERNAL,
638 .base.cra_blocksize = 1,
639 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
640 .base.cra_module = THIS_MODULE,
641
642 .min_keysize = AES_MIN_KEY_SIZE,
643 .max_keysize = AES_MAX_KEY_SIZE,
644 .ivsize = AES_BLOCK_SIZE,
645 .chunksize = AES_BLOCK_SIZE,
646 .setkey = ce_aes_setkey,
647 .encrypt = ctr_encrypt,
648 .decrypt = ctr_encrypt,
649}, {
650 .base.cra_name = "ctr(aes)",
651 .base.cra_driver_name = "ctr-aes-ce-sync",
652 .base.cra_priority = 300 - 1,
653 .base.cra_blocksize = 1,
654 .base.cra_ctxsize = sizeof(struct crypto_aes_ctx),
655 .base.cra_module = THIS_MODULE,
656
657 .min_keysize = AES_MIN_KEY_SIZE,
658 .max_keysize = AES_MAX_KEY_SIZE,
659 .ivsize = AES_BLOCK_SIZE,
660 .chunksize = AES_BLOCK_SIZE,
661 .setkey = ce_aes_setkey,
662 .encrypt = ctr_encrypt_sync,
663 .decrypt = ctr_encrypt_sync,
664}, {
665 .base.cra_name = "__xts(aes)",
666 .base.cra_driver_name = "__xts-aes-ce",
667 .base.cra_priority = 300,
668 .base.cra_flags = CRYPTO_ALG_INTERNAL,
669 .base.cra_blocksize = AES_BLOCK_SIZE,
670 .base.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
671 .base.cra_module = THIS_MODULE,
672
673 .min_keysize = 2 * AES_MIN_KEY_SIZE,
674 .max_keysize = 2 * AES_MAX_KEY_SIZE,
675 .ivsize = AES_BLOCK_SIZE,
676 .walksize = 2 * AES_BLOCK_SIZE,
677 .setkey = xts_set_key,
678 .encrypt = xts_encrypt,
679 .decrypt = xts_decrypt,
680} };
681
682static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
683
684static void aes_exit(void)
685{
686 int i;
687
688 for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
689 simd_skcipher_free(aes_simd_algs[i]);
690
691 crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
692}
693
694static int __init aes_init(void)
695{
696 struct simd_skcipher_alg *simd;
697 const char *basename;
698 const char *algname;
699 const char *drvname;
700 int err;
701 int i;
702
703 err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
704 if (err)
705 return err;
706
707 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
708 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
709 continue;
710
711 algname = aes_algs[i].base.cra_name + 2;
712 drvname = aes_algs[i].base.cra_driver_name + 2;
713 basename = aes_algs[i].base.cra_driver_name;
714 simd = simd_skcipher_create_compat(algname, drvname, basename);
715 err = PTR_ERR(simd);
716 if (IS_ERR(simd))
717 goto unregister_simds;
718
719 aes_simd_algs[i] = simd;
720 }
721
722 return 0;
723
724unregister_simds:
725 aes_exit();
726 return err;
727}
728
729module_cpu_feature_match(AES, aes_init);
730module_exit(aes_exit);
731
