1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27#include <linux/hardirq.h>
28#include <linux/types.h>
29#include <linux/crypto.h>
30#include <linux/err.h>
31#include <crypto/ablk_helper.h>
32#include <crypto/algapi.h>
33#include <crypto/twofish.h>
34#include <crypto/cryptd.h>
35#include <crypto/b128ops.h>
36#include <crypto/ctr.h>
37#include <crypto/lrw.h>
38#include <crypto/xts.h>
39#include <asm/i387.h>
40#include <asm/xcr.h>
41#include <asm/xsave.h>
42#include <asm/crypto/twofish.h>
43#include <asm/crypto/glue_helper.h>
44#include <crypto/scatterwalk.h>
45#include <linux/workqueue.h>
46#include <linux/spinlock.h>
47
48#define TWOFISH_PARALLEL_BLOCKS 8
49
50
51asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
52 const u8 *src);
53asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
54 const u8 *src);
55
56asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
57 const u8 *src);
58asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
59 const u8 *src, le128 *iv);
60
61asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
62 const u8 *src, le128 *iv);
63asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
64 const u8 *src, le128 *iv);
65
66static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
67 const u8 *src)
68{
69 __twofish_enc_blk_3way(ctx, dst, src, false);
70}
71
72static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
73{
74 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
75 GLUE_FUNC_CAST(twofish_enc_blk));
76}
77
78static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
79{
80 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
81 GLUE_FUNC_CAST(twofish_dec_blk));
82}
83
84
85static const struct common_glue_ctx twofish_enc = {
86 .num_funcs = 3,
87 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
88
89 .funcs = { {
90 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
91 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
92 }, {
93 .num_blocks = 3,
94 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
95 }, {
96 .num_blocks = 1,
97 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
98 } }
99};
100
101static const struct common_glue_ctx twofish_ctr = {
102 .num_funcs = 3,
103 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
104
105 .funcs = { {
106 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
107 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
108 }, {
109 .num_blocks = 3,
110 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
111 }, {
112 .num_blocks = 1,
113 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
114 } }
115};
116
117static const struct common_glue_ctx twofish_enc_xts = {
118 .num_funcs = 2,
119 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
120
121 .funcs = { {
122 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
123 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
124 }, {
125 .num_blocks = 1,
126 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
127 } }
128};
129
130static const struct common_glue_ctx twofish_dec = {
131 .num_funcs = 3,
132 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
133
134 .funcs = { {
135 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
136 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
137 }, {
138 .num_blocks = 3,
139 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
140 }, {
141 .num_blocks = 1,
142 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
143 } }
144};
145
146static const struct common_glue_ctx twofish_dec_cbc = {
147 .num_funcs = 3,
148 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
149
150 .funcs = { {
151 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
152 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
153 }, {
154 .num_blocks = 3,
155 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
156 }, {
157 .num_blocks = 1,
158 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
159 } }
160};
161
162static const struct common_glue_ctx twofish_dec_xts = {
163 .num_funcs = 2,
164 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
165
166 .funcs = { {
167 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
168 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
169 }, {
170 .num_blocks = 1,
171 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
172 } }
173};
174
175static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
176 struct scatterlist *src, unsigned int nbytes)
177{
178 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
179}
180
181static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
182 struct scatterlist *src, unsigned int nbytes)
183{
184 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
185}
186
187static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
188 struct scatterlist *src, unsigned int nbytes)
189{
190 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
191 dst, src, nbytes);
192}
193
194static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
195 struct scatterlist *src, unsigned int nbytes)
196{
197 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
198 nbytes);
199}
200
201static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
202 struct scatterlist *src, unsigned int nbytes)
203{
204 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
205}
206
207static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
208{
209 return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
210 fpu_enabled, nbytes);
211}
212
213static inline void twofish_fpu_end(bool fpu_enabled)
214{
215 glue_fpu_end(fpu_enabled);
216}
217
218struct crypt_priv {
219 struct twofish_ctx *ctx;
220 bool fpu_enabled;
221};
222
223static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
224{
225 const unsigned int bsize = TF_BLOCK_SIZE;
226 struct crypt_priv *ctx = priv;
227 int i;
228
229 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
230
231 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
232 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
233 return;
234 }
235
236 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
237 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
238
239 nbytes %= bsize * 3;
240
241 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
242 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
243}
244
245static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
246{
247 const unsigned int bsize = TF_BLOCK_SIZE;
248 struct crypt_priv *ctx = priv;
249 int i;
250
251 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
252
253 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
254 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
255 return;
256 }
257
258 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
259 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
260
261 nbytes %= bsize * 3;
262
263 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
264 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
265}
266
267static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
268 struct scatterlist *src, unsigned int nbytes)
269{
270 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
271 be128 buf[TWOFISH_PARALLEL_BLOCKS];
272 struct crypt_priv crypt_ctx = {
273 .ctx = &ctx->twofish_ctx,
274 .fpu_enabled = false,
275 };
276 struct lrw_crypt_req req = {
277 .tbuf = buf,
278 .tbuflen = sizeof(buf),
279
280 .table_ctx = &ctx->lrw_table,
281 .crypt_ctx = &crypt_ctx,
282 .crypt_fn = encrypt_callback,
283 };
284 int ret;
285
286 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
287 ret = lrw_crypt(desc, dst, src, nbytes, &req);
288 twofish_fpu_end(crypt_ctx.fpu_enabled);
289
290 return ret;
291}
292
293static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
294 struct scatterlist *src, unsigned int nbytes)
295{
296 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
297 be128 buf[TWOFISH_PARALLEL_BLOCKS];
298 struct crypt_priv crypt_ctx = {
299 .ctx = &ctx->twofish_ctx,
300 .fpu_enabled = false,
301 };
302 struct lrw_crypt_req req = {
303 .tbuf = buf,
304 .tbuflen = sizeof(buf),
305
306 .table_ctx = &ctx->lrw_table,
307 .crypt_ctx = &crypt_ctx,
308 .crypt_fn = decrypt_callback,
309 };
310 int ret;
311
312 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
313 ret = lrw_crypt(desc, dst, src, nbytes, &req);
314 twofish_fpu_end(crypt_ctx.fpu_enabled);
315
316 return ret;
317}
318
319static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
320 struct scatterlist *src, unsigned int nbytes)
321{
322 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
323
324 return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
325 XTS_TWEAK_CAST(twofish_enc_blk),
326 &ctx->tweak_ctx, &ctx->crypt_ctx);
327}
328
329static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
330 struct scatterlist *src, unsigned int nbytes)
331{
332 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
333
334 return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
335 XTS_TWEAK_CAST(twofish_enc_blk),
336 &ctx->tweak_ctx, &ctx->crypt_ctx);
337}
338
339static struct crypto_alg twofish_algs[10] = { {
340 .cra_name = "__ecb-twofish-avx",
341 .cra_driver_name = "__driver-ecb-twofish-avx",
342 .cra_priority = 0,
343 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
344 .cra_blocksize = TF_BLOCK_SIZE,
345 .cra_ctxsize = sizeof(struct twofish_ctx),
346 .cra_alignmask = 0,
347 .cra_type = &crypto_blkcipher_type,
348 .cra_module = THIS_MODULE,
349 .cra_u = {
350 .blkcipher = {
351 .min_keysize = TF_MIN_KEY_SIZE,
352 .max_keysize = TF_MAX_KEY_SIZE,
353 .setkey = twofish_setkey,
354 .encrypt = ecb_encrypt,
355 .decrypt = ecb_decrypt,
356 },
357 },
358}, {
359 .cra_name = "__cbc-twofish-avx",
360 .cra_driver_name = "__driver-cbc-twofish-avx",
361 .cra_priority = 0,
362 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
363 .cra_blocksize = TF_BLOCK_SIZE,
364 .cra_ctxsize = sizeof(struct twofish_ctx),
365 .cra_alignmask = 0,
366 .cra_type = &crypto_blkcipher_type,
367 .cra_module = THIS_MODULE,
368 .cra_u = {
369 .blkcipher = {
370 .min_keysize = TF_MIN_KEY_SIZE,
371 .max_keysize = TF_MAX_KEY_SIZE,
372 .setkey = twofish_setkey,
373 .encrypt = cbc_encrypt,
374 .decrypt = cbc_decrypt,
375 },
376 },
377}, {
378 .cra_name = "__ctr-twofish-avx",
379 .cra_driver_name = "__driver-ctr-twofish-avx",
380 .cra_priority = 0,
381 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
382 .cra_blocksize = 1,
383 .cra_ctxsize = sizeof(struct twofish_ctx),
384 .cra_alignmask = 0,
385 .cra_type = &crypto_blkcipher_type,
386 .cra_module = THIS_MODULE,
387 .cra_u = {
388 .blkcipher = {
389 .min_keysize = TF_MIN_KEY_SIZE,
390 .max_keysize = TF_MAX_KEY_SIZE,
391 .ivsize = TF_BLOCK_SIZE,
392 .setkey = twofish_setkey,
393 .encrypt = ctr_crypt,
394 .decrypt = ctr_crypt,
395 },
396 },
397}, {
398 .cra_name = "__lrw-twofish-avx",
399 .cra_driver_name = "__driver-lrw-twofish-avx",
400 .cra_priority = 0,
401 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
402 .cra_blocksize = TF_BLOCK_SIZE,
403 .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
404 .cra_alignmask = 0,
405 .cra_type = &crypto_blkcipher_type,
406 .cra_module = THIS_MODULE,
407 .cra_exit = lrw_twofish_exit_tfm,
408 .cra_u = {
409 .blkcipher = {
410 .min_keysize = TF_MIN_KEY_SIZE +
411 TF_BLOCK_SIZE,
412 .max_keysize = TF_MAX_KEY_SIZE +
413 TF_BLOCK_SIZE,
414 .ivsize = TF_BLOCK_SIZE,
415 .setkey = lrw_twofish_setkey,
416 .encrypt = lrw_encrypt,
417 .decrypt = lrw_decrypt,
418 },
419 },
420}, {
421 .cra_name = "__xts-twofish-avx",
422 .cra_driver_name = "__driver-xts-twofish-avx",
423 .cra_priority = 0,
424 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
425 .cra_blocksize = TF_BLOCK_SIZE,
426 .cra_ctxsize = sizeof(struct twofish_xts_ctx),
427 .cra_alignmask = 0,
428 .cra_type = &crypto_blkcipher_type,
429 .cra_module = THIS_MODULE,
430 .cra_u = {
431 .blkcipher = {
432 .min_keysize = TF_MIN_KEY_SIZE * 2,
433 .max_keysize = TF_MAX_KEY_SIZE * 2,
434 .ivsize = TF_BLOCK_SIZE,
435 .setkey = xts_twofish_setkey,
436 .encrypt = xts_encrypt,
437 .decrypt = xts_decrypt,
438 },
439 },
440}, {
441 .cra_name = "ecb(twofish)",
442 .cra_driver_name = "ecb-twofish-avx",
443 .cra_priority = 400,
444 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
445 .cra_blocksize = TF_BLOCK_SIZE,
446 .cra_ctxsize = sizeof(struct async_helper_ctx),
447 .cra_alignmask = 0,
448 .cra_type = &crypto_ablkcipher_type,
449 .cra_module = THIS_MODULE,
450 .cra_init = ablk_init,
451 .cra_exit = ablk_exit,
452 .cra_u = {
453 .ablkcipher = {
454 .min_keysize = TF_MIN_KEY_SIZE,
455 .max_keysize = TF_MAX_KEY_SIZE,
456 .setkey = ablk_set_key,
457 .encrypt = ablk_encrypt,
458 .decrypt = ablk_decrypt,
459 },
460 },
461}, {
462 .cra_name = "cbc(twofish)",
463 .cra_driver_name = "cbc-twofish-avx",
464 .cra_priority = 400,
465 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
466 .cra_blocksize = TF_BLOCK_SIZE,
467 .cra_ctxsize = sizeof(struct async_helper_ctx),
468 .cra_alignmask = 0,
469 .cra_type = &crypto_ablkcipher_type,
470 .cra_module = THIS_MODULE,
471 .cra_init = ablk_init,
472 .cra_exit = ablk_exit,
473 .cra_u = {
474 .ablkcipher = {
475 .min_keysize = TF_MIN_KEY_SIZE,
476 .max_keysize = TF_MAX_KEY_SIZE,
477 .ivsize = TF_BLOCK_SIZE,
478 .setkey = ablk_set_key,
479 .encrypt = __ablk_encrypt,
480 .decrypt = ablk_decrypt,
481 },
482 },
483}, {
484 .cra_name = "ctr(twofish)",
485 .cra_driver_name = "ctr-twofish-avx",
486 .cra_priority = 400,
487 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
488 .cra_blocksize = 1,
489 .cra_ctxsize = sizeof(struct async_helper_ctx),
490 .cra_alignmask = 0,
491 .cra_type = &crypto_ablkcipher_type,
492 .cra_module = THIS_MODULE,
493 .cra_init = ablk_init,
494 .cra_exit = ablk_exit,
495 .cra_u = {
496 .ablkcipher = {
497 .min_keysize = TF_MIN_KEY_SIZE,
498 .max_keysize = TF_MAX_KEY_SIZE,
499 .ivsize = TF_BLOCK_SIZE,
500 .setkey = ablk_set_key,
501 .encrypt = ablk_encrypt,
502 .decrypt = ablk_encrypt,
503 .geniv = "chainiv",
504 },
505 },
506}, {
507 .cra_name = "lrw(twofish)",
508 .cra_driver_name = "lrw-twofish-avx",
509 .cra_priority = 400,
510 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
511 .cra_blocksize = TF_BLOCK_SIZE,
512 .cra_ctxsize = sizeof(struct async_helper_ctx),
513 .cra_alignmask = 0,
514 .cra_type = &crypto_ablkcipher_type,
515 .cra_module = THIS_MODULE,
516 .cra_init = ablk_init,
517 .cra_exit = ablk_exit,
518 .cra_u = {
519 .ablkcipher = {
520 .min_keysize = TF_MIN_KEY_SIZE +
521 TF_BLOCK_SIZE,
522 .max_keysize = TF_MAX_KEY_SIZE +
523 TF_BLOCK_SIZE,
524 .ivsize = TF_BLOCK_SIZE,
525 .setkey = ablk_set_key,
526 .encrypt = ablk_encrypt,
527 .decrypt = ablk_decrypt,
528 },
529 },
530}, {
531 .cra_name = "xts(twofish)",
532 .cra_driver_name = "xts-twofish-avx",
533 .cra_priority = 400,
534 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
535 .cra_blocksize = TF_BLOCK_SIZE,
536 .cra_ctxsize = sizeof(struct async_helper_ctx),
537 .cra_alignmask = 0,
538 .cra_type = &crypto_ablkcipher_type,
539 .cra_module = THIS_MODULE,
540 .cra_init = ablk_init,
541 .cra_exit = ablk_exit,
542 .cra_u = {
543 .ablkcipher = {
544 .min_keysize = TF_MIN_KEY_SIZE * 2,
545 .max_keysize = TF_MAX_KEY_SIZE * 2,
546 .ivsize = TF_BLOCK_SIZE,
547 .setkey = ablk_set_key,
548 .encrypt = ablk_encrypt,
549 .decrypt = ablk_decrypt,
550 },
551 },
552} };
553
554static int __init twofish_init(void)
555{
556 u64 xcr0;
557
558 if (!cpu_has_avx || !cpu_has_osxsave) {
559 printk(KERN_INFO "AVX instructions are not detected.\n");
560 return -ENODEV;
561 }
562
563 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
564 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
565 printk(KERN_INFO "AVX detected but unusable.\n");
566 return -ENODEV;
567 }
568
569 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
570}
571
572static void __exit twofish_exit(void)
573{
574 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
575}
576
577module_init(twofish_init);
578module_exit(twofish_exit);
579
580MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
581MODULE_LICENSE("GPL");
582MODULE_ALIAS("twofish");
583
