1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27#include <linux/hardirq.h>
28#include <linux/types.h>
29#include <linux/crypto.h>
30#include <linux/err.h>
31#include <crypto/ablk_helper.h>
32#include <crypto/algapi.h>
33#include <crypto/twofish.h>
34#include <crypto/cryptd.h>
35#include <crypto/b128ops.h>
36#include <crypto/ctr.h>
37#include <crypto/lrw.h>
38#include <crypto/xts.h>
39#include <asm/fpu/api.h>
40#include <asm/crypto/twofish.h>
41#include <asm/crypto/glue_helper.h>
42#include <crypto/scatterwalk.h>
43#include <linux/workqueue.h>
44#include <linux/spinlock.h>
45
46#define TWOFISH_PARALLEL_BLOCKS 8
47
48
49asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
50 const u8 *src);
51asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
52 const u8 *src);
53
54asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
55 const u8 *src);
56asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
57 const u8 *src, le128 *iv);
58
59asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
60 const u8 *src, le128 *iv);
61asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
62 const u8 *src, le128 *iv);
63
64static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
65 const u8 *src)
66{
67 __twofish_enc_blk_3way(ctx, dst, src, false);
68}
69
70static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
71{
72 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
73 GLUE_FUNC_CAST(twofish_enc_blk));
74}
75
76static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
77{
78 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
79 GLUE_FUNC_CAST(twofish_dec_blk));
80}
81
82
83static const struct common_glue_ctx twofish_enc = {
84 .num_funcs = 3,
85 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
86
87 .funcs = { {
88 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
89 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
90 }, {
91 .num_blocks = 3,
92 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
93 }, {
94 .num_blocks = 1,
95 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
96 } }
97};
98
99static const struct common_glue_ctx twofish_ctr = {
100 .num_funcs = 3,
101 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
102
103 .funcs = { {
104 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
105 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
106 }, {
107 .num_blocks = 3,
108 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
109 }, {
110 .num_blocks = 1,
111 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
112 } }
113};
114
115static const struct common_glue_ctx twofish_enc_xts = {
116 .num_funcs = 2,
117 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
118
119 .funcs = { {
120 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
121 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
122 }, {
123 .num_blocks = 1,
124 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
125 } }
126};
127
128static const struct common_glue_ctx twofish_dec = {
129 .num_funcs = 3,
130 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
131
132 .funcs = { {
133 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
134 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
135 }, {
136 .num_blocks = 3,
137 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
138 }, {
139 .num_blocks = 1,
140 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
141 } }
142};
143
144static const struct common_glue_ctx twofish_dec_cbc = {
145 .num_funcs = 3,
146 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
147
148 .funcs = { {
149 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
150 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
151 }, {
152 .num_blocks = 3,
153 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
154 }, {
155 .num_blocks = 1,
156 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
157 } }
158};
159
160static const struct common_glue_ctx twofish_dec_xts = {
161 .num_funcs = 2,
162 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
163
164 .funcs = { {
165 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
166 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
167 }, {
168 .num_blocks = 1,
169 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
170 } }
171};
172
173static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
174 struct scatterlist *src, unsigned int nbytes)
175{
176 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
177}
178
179static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
180 struct scatterlist *src, unsigned int nbytes)
181{
182 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
183}
184
185static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
186 struct scatterlist *src, unsigned int nbytes)
187{
188 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
189 dst, src, nbytes);
190}
191
192static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
193 struct scatterlist *src, unsigned int nbytes)
194{
195 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
196 nbytes);
197}
198
199static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
200 struct scatterlist *src, unsigned int nbytes)
201{
202 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
203}
204
205static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
206{
207 return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
208 fpu_enabled, nbytes);
209}
210
211static inline void twofish_fpu_end(bool fpu_enabled)
212{
213 glue_fpu_end(fpu_enabled);
214}
215
216struct crypt_priv {
217 struct twofish_ctx *ctx;
218 bool fpu_enabled;
219};
220
221static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
222{
223 const unsigned int bsize = TF_BLOCK_SIZE;
224 struct crypt_priv *ctx = priv;
225 int i;
226
227 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
228
229 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
230 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
231 return;
232 }
233
234 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
235 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
236
237 nbytes %= bsize * 3;
238
239 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
240 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
241}
242
243static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
244{
245 const unsigned int bsize = TF_BLOCK_SIZE;
246 struct crypt_priv *ctx = priv;
247 int i;
248
249 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
250
251 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
252 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
253 return;
254 }
255
256 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
257 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
258
259 nbytes %= bsize * 3;
260
261 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
262 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
263}
264
265static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
266 struct scatterlist *src, unsigned int nbytes)
267{
268 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
269 be128 buf[TWOFISH_PARALLEL_BLOCKS];
270 struct crypt_priv crypt_ctx = {
271 .ctx = &ctx->twofish_ctx,
272 .fpu_enabled = false,
273 };
274 struct lrw_crypt_req req = {
275 .tbuf = buf,
276 .tbuflen = sizeof(buf),
277
278 .table_ctx = &ctx->lrw_table,
279 .crypt_ctx = &crypt_ctx,
280 .crypt_fn = encrypt_callback,
281 };
282 int ret;
283
284 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
285 ret = lrw_crypt(desc, dst, src, nbytes, &req);
286 twofish_fpu_end(crypt_ctx.fpu_enabled);
287
288 return ret;
289}
290
291static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
292 struct scatterlist *src, unsigned int nbytes)
293{
294 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
295 be128 buf[TWOFISH_PARALLEL_BLOCKS];
296 struct crypt_priv crypt_ctx = {
297 .ctx = &ctx->twofish_ctx,
298 .fpu_enabled = false,
299 };
300 struct lrw_crypt_req req = {
301 .tbuf = buf,
302 .tbuflen = sizeof(buf),
303
304 .table_ctx = &ctx->lrw_table,
305 .crypt_ctx = &crypt_ctx,
306 .crypt_fn = decrypt_callback,
307 };
308 int ret;
309
310 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
311 ret = lrw_crypt(desc, dst, src, nbytes, &req);
312 twofish_fpu_end(crypt_ctx.fpu_enabled);
313
314 return ret;
315}
316
317static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
318 struct scatterlist *src, unsigned int nbytes)
319{
320 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
321
322 return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
323 XTS_TWEAK_CAST(twofish_enc_blk),
324 &ctx->tweak_ctx, &ctx->crypt_ctx);
325}
326
327static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
328 struct scatterlist *src, unsigned int nbytes)
329{
330 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
331
332 return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
333 XTS_TWEAK_CAST(twofish_enc_blk),
334 &ctx->tweak_ctx, &ctx->crypt_ctx);
335}
336
337static struct crypto_alg twofish_algs[10] = { {
338 .cra_name = "__ecb-twofish-avx",
339 .cra_driver_name = "__driver-ecb-twofish-avx",
340 .cra_priority = 0,
341 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
342 CRYPTO_ALG_INTERNAL,
343 .cra_blocksize = TF_BLOCK_SIZE,
344 .cra_ctxsize = sizeof(struct twofish_ctx),
345 .cra_alignmask = 0,
346 .cra_type = &crypto_blkcipher_type,
347 .cra_module = THIS_MODULE,
348 .cra_u = {
349 .blkcipher = {
350 .min_keysize = TF_MIN_KEY_SIZE,
351 .max_keysize = TF_MAX_KEY_SIZE,
352 .setkey = twofish_setkey,
353 .encrypt = ecb_encrypt,
354 .decrypt = ecb_decrypt,
355 },
356 },
357}, {
358 .cra_name = "__cbc-twofish-avx",
359 .cra_driver_name = "__driver-cbc-twofish-avx",
360 .cra_priority = 0,
361 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
362 CRYPTO_ALG_INTERNAL,
363 .cra_blocksize = TF_BLOCK_SIZE,
364 .cra_ctxsize = sizeof(struct twofish_ctx),
365 .cra_alignmask = 0,
366 .cra_type = &crypto_blkcipher_type,
367 .cra_module = THIS_MODULE,
368 .cra_u = {
369 .blkcipher = {
370 .min_keysize = TF_MIN_KEY_SIZE,
371 .max_keysize = TF_MAX_KEY_SIZE,
372 .setkey = twofish_setkey,
373 .encrypt = cbc_encrypt,
374 .decrypt = cbc_decrypt,
375 },
376 },
377}, {
378 .cra_name = "__ctr-twofish-avx",
379 .cra_driver_name = "__driver-ctr-twofish-avx",
380 .cra_priority = 0,
381 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
382 CRYPTO_ALG_INTERNAL,
383 .cra_blocksize = 1,
384 .cra_ctxsize = sizeof(struct twofish_ctx),
385 .cra_alignmask = 0,
386 .cra_type = &crypto_blkcipher_type,
387 .cra_module = THIS_MODULE,
388 .cra_u = {
389 .blkcipher = {
390 .min_keysize = TF_MIN_KEY_SIZE,
391 .max_keysize = TF_MAX_KEY_SIZE,
392 .ivsize = TF_BLOCK_SIZE,
393 .setkey = twofish_setkey,
394 .encrypt = ctr_crypt,
395 .decrypt = ctr_crypt,
396 },
397 },
398}, {
399 .cra_name = "__lrw-twofish-avx",
400 .cra_driver_name = "__driver-lrw-twofish-avx",
401 .cra_priority = 0,
402 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
403 CRYPTO_ALG_INTERNAL,
404 .cra_blocksize = TF_BLOCK_SIZE,
405 .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
406 .cra_alignmask = 0,
407 .cra_type = &crypto_blkcipher_type,
408 .cra_module = THIS_MODULE,
409 .cra_exit = lrw_twofish_exit_tfm,
410 .cra_u = {
411 .blkcipher = {
412 .min_keysize = TF_MIN_KEY_SIZE +
413 TF_BLOCK_SIZE,
414 .max_keysize = TF_MAX_KEY_SIZE +
415 TF_BLOCK_SIZE,
416 .ivsize = TF_BLOCK_SIZE,
417 .setkey = lrw_twofish_setkey,
418 .encrypt = lrw_encrypt,
419 .decrypt = lrw_decrypt,
420 },
421 },
422}, {
423 .cra_name = "__xts-twofish-avx",
424 .cra_driver_name = "__driver-xts-twofish-avx",
425 .cra_priority = 0,
426 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
427 CRYPTO_ALG_INTERNAL,
428 .cra_blocksize = TF_BLOCK_SIZE,
429 .cra_ctxsize = sizeof(struct twofish_xts_ctx),
430 .cra_alignmask = 0,
431 .cra_type = &crypto_blkcipher_type,
432 .cra_module = THIS_MODULE,
433 .cra_u = {
434 .blkcipher = {
435 .min_keysize = TF_MIN_KEY_SIZE * 2,
436 .max_keysize = TF_MAX_KEY_SIZE * 2,
437 .ivsize = TF_BLOCK_SIZE,
438 .setkey = xts_twofish_setkey,
439 .encrypt = xts_encrypt,
440 .decrypt = xts_decrypt,
441 },
442 },
443}, {
444 .cra_name = "ecb(twofish)",
445 .cra_driver_name = "ecb-twofish-avx",
446 .cra_priority = 400,
447 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
448 .cra_blocksize = TF_BLOCK_SIZE,
449 .cra_ctxsize = sizeof(struct async_helper_ctx),
450 .cra_alignmask = 0,
451 .cra_type = &crypto_ablkcipher_type,
452 .cra_module = THIS_MODULE,
453 .cra_init = ablk_init,
454 .cra_exit = ablk_exit,
455 .cra_u = {
456 .ablkcipher = {
457 .min_keysize = TF_MIN_KEY_SIZE,
458 .max_keysize = TF_MAX_KEY_SIZE,
459 .setkey = ablk_set_key,
460 .encrypt = ablk_encrypt,
461 .decrypt = ablk_decrypt,
462 },
463 },
464}, {
465 .cra_name = "cbc(twofish)",
466 .cra_driver_name = "cbc-twofish-avx",
467 .cra_priority = 400,
468 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
469 .cra_blocksize = TF_BLOCK_SIZE,
470 .cra_ctxsize = sizeof(struct async_helper_ctx),
471 .cra_alignmask = 0,
472 .cra_type = &crypto_ablkcipher_type,
473 .cra_module = THIS_MODULE,
474 .cra_init = ablk_init,
475 .cra_exit = ablk_exit,
476 .cra_u = {
477 .ablkcipher = {
478 .min_keysize = TF_MIN_KEY_SIZE,
479 .max_keysize = TF_MAX_KEY_SIZE,
480 .ivsize = TF_BLOCK_SIZE,
481 .setkey = ablk_set_key,
482 .encrypt = __ablk_encrypt,
483 .decrypt = ablk_decrypt,
484 },
485 },
486}, {
487 .cra_name = "ctr(twofish)",
488 .cra_driver_name = "ctr-twofish-avx",
489 .cra_priority = 400,
490 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
491 .cra_blocksize = 1,
492 .cra_ctxsize = sizeof(struct async_helper_ctx),
493 .cra_alignmask = 0,
494 .cra_type = &crypto_ablkcipher_type,
495 .cra_module = THIS_MODULE,
496 .cra_init = ablk_init,
497 .cra_exit = ablk_exit,
498 .cra_u = {
499 .ablkcipher = {
500 .min_keysize = TF_MIN_KEY_SIZE,
501 .max_keysize = TF_MAX_KEY_SIZE,
502 .ivsize = TF_BLOCK_SIZE,
503 .setkey = ablk_set_key,
504 .encrypt = ablk_encrypt,
505 .decrypt = ablk_encrypt,
506 .geniv = "chainiv",
507 },
508 },
509}, {
510 .cra_name = "lrw(twofish)",
511 .cra_driver_name = "lrw-twofish-avx",
512 .cra_priority = 400,
513 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
514 .cra_blocksize = TF_BLOCK_SIZE,
515 .cra_ctxsize = sizeof(struct async_helper_ctx),
516 .cra_alignmask = 0,
517 .cra_type = &crypto_ablkcipher_type,
518 .cra_module = THIS_MODULE,
519 .cra_init = ablk_init,
520 .cra_exit = ablk_exit,
521 .cra_u = {
522 .ablkcipher = {
523 .min_keysize = TF_MIN_KEY_SIZE +
524 TF_BLOCK_SIZE,
525 .max_keysize = TF_MAX_KEY_SIZE +
526 TF_BLOCK_SIZE,
527 .ivsize = TF_BLOCK_SIZE,
528 .setkey = ablk_set_key,
529 .encrypt = ablk_encrypt,
530 .decrypt = ablk_decrypt,
531 },
532 },
533}, {
534 .cra_name = "xts(twofish)",
535 .cra_driver_name = "xts-twofish-avx",
536 .cra_priority = 400,
537 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
538 .cra_blocksize = TF_BLOCK_SIZE,
539 .cra_ctxsize = sizeof(struct async_helper_ctx),
540 .cra_alignmask = 0,
541 .cra_type = &crypto_ablkcipher_type,
542 .cra_module = THIS_MODULE,
543 .cra_init = ablk_init,
544 .cra_exit = ablk_exit,
545 .cra_u = {
546 .ablkcipher = {
547 .min_keysize = TF_MIN_KEY_SIZE * 2,
548 .max_keysize = TF_MAX_KEY_SIZE * 2,
549 .ivsize = TF_BLOCK_SIZE,
550 .setkey = ablk_set_key,
551 .encrypt = ablk_encrypt,
552 .decrypt = ablk_decrypt,
553 },
554 },
555} };
556
557static int __init twofish_init(void)
558{
559 const char *feature_name;
560
561 if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, &feature_name)) {
562 pr_info("CPU feature '%s' is not supported.\n", feature_name);
563 return -ENODEV;
564 }
565
566 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
567}
568
569static void __exit twofish_exit(void)
570{
571 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
572}
573
574module_init(twofish_init);
575module_exit(twofish_exit);
576
577MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
578MODULE_LICENSE("GPL");
579MODULE_ALIAS_CRYPTO("twofish");
580
