1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/module.h>
25#include <linux/hardirq.h>
26#include <linux/types.h>
27#include <linux/crypto.h>
28#include <linux/err.h>
29#include <crypto/algapi.h>
30#include <crypto/cast6.h>
31#include <crypto/cryptd.h>
32#include <crypto/b128ops.h>
33#include <crypto/ctr.h>
34#include <crypto/lrw.h>
35#include <crypto/xts.h>
36#include <asm/xcr.h>
37#include <asm/xsave.h>
38#include <asm/crypto/ablk_helper.h>
39#include <asm/crypto/glue_helper.h>
40
41#define CAST6_PARALLEL_BLOCKS 8
42
43asmlinkage void cast6_ecb_enc_8way(struct cast6_ctx *ctx, u8 *dst,
44 const u8 *src);
45asmlinkage void cast6_ecb_dec_8way(struct cast6_ctx *ctx, u8 *dst,
46 const u8 *src);
47
48asmlinkage void cast6_cbc_dec_8way(struct cast6_ctx *ctx, u8 *dst,
49 const u8 *src);
50asmlinkage void cast6_ctr_8way(struct cast6_ctx *ctx, u8 *dst, const u8 *src,
51 le128 *iv);
52
53static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
54{
55 be128 ctrblk;
56
57 le128_to_be128(&ctrblk, iv);
58 le128_inc(iv);
59
60 __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
61 u128_xor(dst, src, (u128 *)&ctrblk);
62}
63
64static const struct common_glue_ctx cast6_enc = {
65 .num_funcs = 2,
66 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
67
68 .funcs = { {
69 .num_blocks = CAST6_PARALLEL_BLOCKS,
70 .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_enc_8way) }
71 }, {
72 .num_blocks = 1,
73 .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
74 } }
75};
76
77static const struct common_glue_ctx cast6_ctr = {
78 .num_funcs = 2,
79 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
80
81 .funcs = { {
82 .num_blocks = CAST6_PARALLEL_BLOCKS,
83 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_ctr_8way) }
84 }, {
85 .num_blocks = 1,
86 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
87 } }
88};
89
90static const struct common_glue_ctx cast6_dec = {
91 .num_funcs = 2,
92 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
93
94 .funcs = { {
95 .num_blocks = CAST6_PARALLEL_BLOCKS,
96 .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_ecb_dec_8way) }
97 }, {
98 .num_blocks = 1,
99 .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
100 } }
101};
102
103static const struct common_glue_ctx cast6_dec_cbc = {
104 .num_funcs = 2,
105 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
106
107 .funcs = { {
108 .num_blocks = CAST6_PARALLEL_BLOCKS,
109 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_cbc_dec_8way) }
110 }, {
111 .num_blocks = 1,
112 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
113 } }
114};
115
116static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
117 struct scatterlist *src, unsigned int nbytes)
118{
119 return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
120}
121
122static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
123 struct scatterlist *src, unsigned int nbytes)
124{
125 return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
126}
127
128static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
129 struct scatterlist *src, unsigned int nbytes)
130{
131 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
132 dst, src, nbytes);
133}
134
135static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
136 struct scatterlist *src, unsigned int nbytes)
137{
138 return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
139 nbytes);
140}
141
142static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
143 struct scatterlist *src, unsigned int nbytes)
144{
145 return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
146}
147
148static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
149{
150 return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
151 NULL, fpu_enabled, nbytes);
152}
153
154static inline void cast6_fpu_end(bool fpu_enabled)
155{
156 glue_fpu_end(fpu_enabled);
157}
158
159struct crypt_priv {
160 struct cast6_ctx *ctx;
161 bool fpu_enabled;
162};
163
164static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
165{
166 const unsigned int bsize = CAST6_BLOCK_SIZE;
167 struct crypt_priv *ctx = priv;
168 int i;
169
170 ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
171
172 if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
173 cast6_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
174 return;
175 }
176
177 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
178 __cast6_encrypt(ctx->ctx, srcdst, srcdst);
179}
180
181static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
182{
183 const unsigned int bsize = CAST6_BLOCK_SIZE;
184 struct crypt_priv *ctx = priv;
185 int i;
186
187 ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
188
189 if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
190 cast6_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
191 return;
192 }
193
194 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
195 __cast6_decrypt(ctx->ctx, srcdst, srcdst);
196}
197
198struct cast6_lrw_ctx {
199 struct lrw_table_ctx lrw_table;
200 struct cast6_ctx cast6_ctx;
201};
202
203static int lrw_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
204 unsigned int keylen)
205{
206 struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
207 int err;
208
209 err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
210 &tfm->crt_flags);
211 if (err)
212 return err;
213
214 return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
215}
216
217static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
218 struct scatterlist *src, unsigned int nbytes)
219{
220 struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
221 be128 buf[CAST6_PARALLEL_BLOCKS];
222 struct crypt_priv crypt_ctx = {
223 .ctx = &ctx->cast6_ctx,
224 .fpu_enabled = false,
225 };
226 struct lrw_crypt_req req = {
227 .tbuf = buf,
228 .tbuflen = sizeof(buf),
229
230 .table_ctx = &ctx->lrw_table,
231 .crypt_ctx = &crypt_ctx,
232 .crypt_fn = encrypt_callback,
233 };
234 int ret;
235
236 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
237 ret = lrw_crypt(desc, dst, src, nbytes, &req);
238 cast6_fpu_end(crypt_ctx.fpu_enabled);
239
240 return ret;
241}
242
243static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
244 struct scatterlist *src, unsigned int nbytes)
245{
246 struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
247 be128 buf[CAST6_PARALLEL_BLOCKS];
248 struct crypt_priv crypt_ctx = {
249 .ctx = &ctx->cast6_ctx,
250 .fpu_enabled = false,
251 };
252 struct lrw_crypt_req req = {
253 .tbuf = buf,
254 .tbuflen = sizeof(buf),
255
256 .table_ctx = &ctx->lrw_table,
257 .crypt_ctx = &crypt_ctx,
258 .crypt_fn = decrypt_callback,
259 };
260 int ret;
261
262 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
263 ret = lrw_crypt(desc, dst, src, nbytes, &req);
264 cast6_fpu_end(crypt_ctx.fpu_enabled);
265
266 return ret;
267}
268
269static void lrw_exit_tfm(struct crypto_tfm *tfm)
270{
271 struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
272
273 lrw_free_table(&ctx->lrw_table);
274}
275
276struct cast6_xts_ctx {
277 struct cast6_ctx tweak_ctx;
278 struct cast6_ctx crypt_ctx;
279};
280
281static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
282 unsigned int keylen)
283{
284 struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
285 u32 *flags = &tfm->crt_flags;
286 int err;
287
288
289
290
291 if (keylen % 2) {
292 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
293 return -EINVAL;
294 }
295
296
297 err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
298 if (err)
299 return err;
300
301
302 return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
303 flags);
304}
305
306static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
307 struct scatterlist *src, unsigned int nbytes)
308{
309 struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
310 be128 buf[CAST6_PARALLEL_BLOCKS];
311 struct crypt_priv crypt_ctx = {
312 .ctx = &ctx->crypt_ctx,
313 .fpu_enabled = false,
314 };
315 struct xts_crypt_req req = {
316 .tbuf = buf,
317 .tbuflen = sizeof(buf),
318
319 .tweak_ctx = &ctx->tweak_ctx,
320 .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt),
321 .crypt_ctx = &crypt_ctx,
322 .crypt_fn = encrypt_callback,
323 };
324 int ret;
325
326 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
327 ret = xts_crypt(desc, dst, src, nbytes, &req);
328 cast6_fpu_end(crypt_ctx.fpu_enabled);
329
330 return ret;
331}
332
333static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
334 struct scatterlist *src, unsigned int nbytes)
335{
336 struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
337 be128 buf[CAST6_PARALLEL_BLOCKS];
338 struct crypt_priv crypt_ctx = {
339 .ctx = &ctx->crypt_ctx,
340 .fpu_enabled = false,
341 };
342 struct xts_crypt_req req = {
343 .tbuf = buf,
344 .tbuflen = sizeof(buf),
345
346 .tweak_ctx = &ctx->tweak_ctx,
347 .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt),
348 .crypt_ctx = &crypt_ctx,
349 .crypt_fn = decrypt_callback,
350 };
351 int ret;
352
353 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
354 ret = xts_crypt(desc, dst, src, nbytes, &req);
355 cast6_fpu_end(crypt_ctx.fpu_enabled);
356
357 return ret;
358}
359
360static struct crypto_alg cast6_algs[10] = { {
361 .cra_name = "__ecb-cast6-avx",
362 .cra_driver_name = "__driver-ecb-cast6-avx",
363 .cra_priority = 0,
364 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
365 .cra_blocksize = CAST6_BLOCK_SIZE,
366 .cra_ctxsize = sizeof(struct cast6_ctx),
367 .cra_alignmask = 0,
368 .cra_type = &crypto_blkcipher_type,
369 .cra_module = THIS_MODULE,
370 .cra_u = {
371 .blkcipher = {
372 .min_keysize = CAST6_MIN_KEY_SIZE,
373 .max_keysize = CAST6_MAX_KEY_SIZE,
374 .setkey = cast6_setkey,
375 .encrypt = ecb_encrypt,
376 .decrypt = ecb_decrypt,
377 },
378 },
379}, {
380 .cra_name = "__cbc-cast6-avx",
381 .cra_driver_name = "__driver-cbc-cast6-avx",
382 .cra_priority = 0,
383 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
384 .cra_blocksize = CAST6_BLOCK_SIZE,
385 .cra_ctxsize = sizeof(struct cast6_ctx),
386 .cra_alignmask = 0,
387 .cra_type = &crypto_blkcipher_type,
388 .cra_module = THIS_MODULE,
389 .cra_u = {
390 .blkcipher = {
391 .min_keysize = CAST6_MIN_KEY_SIZE,
392 .max_keysize = CAST6_MAX_KEY_SIZE,
393 .setkey = cast6_setkey,
394 .encrypt = cbc_encrypt,
395 .decrypt = cbc_decrypt,
396 },
397 },
398}, {
399 .cra_name = "__ctr-cast6-avx",
400 .cra_driver_name = "__driver-ctr-cast6-avx",
401 .cra_priority = 0,
402 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
403 .cra_blocksize = 1,
404 .cra_ctxsize = sizeof(struct cast6_ctx),
405 .cra_alignmask = 0,
406 .cra_type = &crypto_blkcipher_type,
407 .cra_module = THIS_MODULE,
408 .cra_u = {
409 .blkcipher = {
410 .min_keysize = CAST6_MIN_KEY_SIZE,
411 .max_keysize = CAST6_MAX_KEY_SIZE,
412 .ivsize = CAST6_BLOCK_SIZE,
413 .setkey = cast6_setkey,
414 .encrypt = ctr_crypt,
415 .decrypt = ctr_crypt,
416 },
417 },
418}, {
419 .cra_name = "__lrw-cast6-avx",
420 .cra_driver_name = "__driver-lrw-cast6-avx",
421 .cra_priority = 0,
422 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
423 .cra_blocksize = CAST6_BLOCK_SIZE,
424 .cra_ctxsize = sizeof(struct cast6_lrw_ctx),
425 .cra_alignmask = 0,
426 .cra_type = &crypto_blkcipher_type,
427 .cra_module = THIS_MODULE,
428 .cra_exit = lrw_exit_tfm,
429 .cra_u = {
430 .blkcipher = {
431 .min_keysize = CAST6_MIN_KEY_SIZE +
432 CAST6_BLOCK_SIZE,
433 .max_keysize = CAST6_MAX_KEY_SIZE +
434 CAST6_BLOCK_SIZE,
435 .ivsize = CAST6_BLOCK_SIZE,
436 .setkey = lrw_cast6_setkey,
437 .encrypt = lrw_encrypt,
438 .decrypt = lrw_decrypt,
439 },
440 },
441}, {
442 .cra_name = "__xts-cast6-avx",
443 .cra_driver_name = "__driver-xts-cast6-avx",
444 .cra_priority = 0,
445 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
446 .cra_blocksize = CAST6_BLOCK_SIZE,
447 .cra_ctxsize = sizeof(struct cast6_xts_ctx),
448 .cra_alignmask = 0,
449 .cra_type = &crypto_blkcipher_type,
450 .cra_module = THIS_MODULE,
451 .cra_u = {
452 .blkcipher = {
453 .min_keysize = CAST6_MIN_KEY_SIZE * 2,
454 .max_keysize = CAST6_MAX_KEY_SIZE * 2,
455 .ivsize = CAST6_BLOCK_SIZE,
456 .setkey = xts_cast6_setkey,
457 .encrypt = xts_encrypt,
458 .decrypt = xts_decrypt,
459 },
460 },
461}, {
462 .cra_name = "ecb(cast6)",
463 .cra_driver_name = "ecb-cast6-avx",
464 .cra_priority = 200,
465 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
466 .cra_blocksize = CAST6_BLOCK_SIZE,
467 .cra_ctxsize = sizeof(struct async_helper_ctx),
468 .cra_alignmask = 0,
469 .cra_type = &crypto_ablkcipher_type,
470 .cra_module = THIS_MODULE,
471 .cra_init = ablk_init,
472 .cra_exit = ablk_exit,
473 .cra_u = {
474 .ablkcipher = {
475 .min_keysize = CAST6_MIN_KEY_SIZE,
476 .max_keysize = CAST6_MAX_KEY_SIZE,
477 .setkey = ablk_set_key,
478 .encrypt = ablk_encrypt,
479 .decrypt = ablk_decrypt,
480 },
481 },
482}, {
483 .cra_name = "cbc(cast6)",
484 .cra_driver_name = "cbc-cast6-avx",
485 .cra_priority = 200,
486 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
487 .cra_blocksize = CAST6_BLOCK_SIZE,
488 .cra_ctxsize = sizeof(struct async_helper_ctx),
489 .cra_alignmask = 0,
490 .cra_type = &crypto_ablkcipher_type,
491 .cra_module = THIS_MODULE,
492 .cra_init = ablk_init,
493 .cra_exit = ablk_exit,
494 .cra_u = {
495 .ablkcipher = {
496 .min_keysize = CAST6_MIN_KEY_SIZE,
497 .max_keysize = CAST6_MAX_KEY_SIZE,
498 .ivsize = CAST6_BLOCK_SIZE,
499 .setkey = ablk_set_key,
500 .encrypt = __ablk_encrypt,
501 .decrypt = ablk_decrypt,
502 },
503 },
504}, {
505 .cra_name = "ctr(cast6)",
506 .cra_driver_name = "ctr-cast6-avx",
507 .cra_priority = 200,
508 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
509 .cra_blocksize = 1,
510 .cra_ctxsize = sizeof(struct async_helper_ctx),
511 .cra_alignmask = 0,
512 .cra_type = &crypto_ablkcipher_type,
513 .cra_module = THIS_MODULE,
514 .cra_init = ablk_init,
515 .cra_exit = ablk_exit,
516 .cra_u = {
517 .ablkcipher = {
518 .min_keysize = CAST6_MIN_KEY_SIZE,
519 .max_keysize = CAST6_MAX_KEY_SIZE,
520 .ivsize = CAST6_BLOCK_SIZE,
521 .setkey = ablk_set_key,
522 .encrypt = ablk_encrypt,
523 .decrypt = ablk_encrypt,
524 .geniv = "chainiv",
525 },
526 },
527}, {
528 .cra_name = "lrw(cast6)",
529 .cra_driver_name = "lrw-cast6-avx",
530 .cra_priority = 200,
531 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
532 .cra_blocksize = CAST6_BLOCK_SIZE,
533 .cra_ctxsize = sizeof(struct async_helper_ctx),
534 .cra_alignmask = 0,
535 .cra_type = &crypto_ablkcipher_type,
536 .cra_module = THIS_MODULE,
537 .cra_init = ablk_init,
538 .cra_exit = ablk_exit,
539 .cra_u = {
540 .ablkcipher = {
541 .min_keysize = CAST6_MIN_KEY_SIZE +
542 CAST6_BLOCK_SIZE,
543 .max_keysize = CAST6_MAX_KEY_SIZE +
544 CAST6_BLOCK_SIZE,
545 .ivsize = CAST6_BLOCK_SIZE,
546 .setkey = ablk_set_key,
547 .encrypt = ablk_encrypt,
548 .decrypt = ablk_decrypt,
549 },
550 },
551}, {
552 .cra_name = "xts(cast6)",
553 .cra_driver_name = "xts-cast6-avx",
554 .cra_priority = 200,
555 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
556 .cra_blocksize = CAST6_BLOCK_SIZE,
557 .cra_ctxsize = sizeof(struct async_helper_ctx),
558 .cra_alignmask = 0,
559 .cra_type = &crypto_ablkcipher_type,
560 .cra_module = THIS_MODULE,
561 .cra_init = ablk_init,
562 .cra_exit = ablk_exit,
563 .cra_u = {
564 .ablkcipher = {
565 .min_keysize = CAST6_MIN_KEY_SIZE * 2,
566 .max_keysize = CAST6_MAX_KEY_SIZE * 2,
567 .ivsize = CAST6_BLOCK_SIZE,
568 .setkey = ablk_set_key,
569 .encrypt = ablk_encrypt,
570 .decrypt = ablk_decrypt,
571 },
572 },
573} };
574
575static int __init cast6_init(void)
576{
577 u64 xcr0;
578
579 if (!cpu_has_avx || !cpu_has_osxsave) {
580 pr_info("AVX instructions are not detected.\n");
581 return -ENODEV;
582 }
583
584 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
585 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
586 pr_info("AVX detected but unusable.\n");
587 return -ENODEV;
588 }
589
590 return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
591}
592
593static void __exit cast6_exit(void)
594{
595 crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
596}
597
598module_init(cast6_init);
599module_exit(cast6_exit);
600
601MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
602MODULE_LICENSE("GPL");
603MODULE_ALIAS("cast6");
604
