1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/module.h>
14#include <linux/types.h>
15#include <linux/crypto.h>
16#include <linux/err.h>
17#include <crypto/algapi.h>
18#include <crypto/ctr.h>
19#include <crypto/lrw.h>
20#include <crypto/xts.h>
21#include <crypto/serpent.h>
22#include <asm/xcr.h>
23#include <asm/xsave.h>
24#include <asm/crypto/serpent-avx.h>
25#include <asm/crypto/ablk_helper.h>
26#include <asm/crypto/glue_helper.h>
27
28#define SERPENT_AVX2_PARALLEL_BLOCKS 16
29
30
31asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
32 const u8 *src);
33asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
34 const u8 *src);
35asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);
36
37asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
38 le128 *iv);
39asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
40 const u8 *src, le128 *iv);
41asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
42 const u8 *src, le128 *iv);
43
44static const struct common_glue_ctx serpent_enc = {
45 .num_funcs = 3,
46 .fpu_blocks_limit = 8,
47
48 .funcs = { {
49 .num_blocks = 16,
50 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
51 }, {
52 .num_blocks = 8,
53 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
54 }, {
55 .num_blocks = 1,
56 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
57 } }
58};
59
60static const struct common_glue_ctx serpent_ctr = {
61 .num_funcs = 3,
62 .fpu_blocks_limit = 8,
63
64 .funcs = { {
65 .num_blocks = 16,
66 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
67 }, {
68 .num_blocks = 8,
69 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
70 }, {
71 .num_blocks = 1,
72 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
73 } }
74};
75
76static const struct common_glue_ctx serpent_enc_xts = {
77 .num_funcs = 3,
78 .fpu_blocks_limit = 8,
79
80 .funcs = { {
81 .num_blocks = 16,
82 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
83 }, {
84 .num_blocks = 8,
85 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
86 }, {
87 .num_blocks = 1,
88 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
89 } }
90};
91
92static const struct common_glue_ctx serpent_dec = {
93 .num_funcs = 3,
94 .fpu_blocks_limit = 8,
95
96 .funcs = { {
97 .num_blocks = 16,
98 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
99 }, {
100 .num_blocks = 8,
101 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
102 }, {
103 .num_blocks = 1,
104 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
105 } }
106};
107
108static const struct common_glue_ctx serpent_dec_cbc = {
109 .num_funcs = 3,
110 .fpu_blocks_limit = 8,
111
112 .funcs = { {
113 .num_blocks = 16,
114 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
115 }, {
116 .num_blocks = 8,
117 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
118 }, {
119 .num_blocks = 1,
120 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
121 } }
122};
123
124static const struct common_glue_ctx serpent_dec_xts = {
125 .num_funcs = 3,
126 .fpu_blocks_limit = 8,
127
128 .funcs = { {
129 .num_blocks = 16,
130 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
131 }, {
132 .num_blocks = 8,
133 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
134 }, {
135 .num_blocks = 1,
136 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
137 } }
138};
139
140static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
141 struct scatterlist *src, unsigned int nbytes)
142{
143 return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
144}
145
146static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
147 struct scatterlist *src, unsigned int nbytes)
148{
149 return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
150}
151
152static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
153 struct scatterlist *src, unsigned int nbytes)
154{
155 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
156 dst, src, nbytes);
157}
158
159static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
160 struct scatterlist *src, unsigned int nbytes)
161{
162 return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
163 nbytes);
164}
165
166static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
167 struct scatterlist *src, unsigned int nbytes)
168{
169 return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
170}
171
172static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
173{
174
175 return glue_fpu_begin(SERPENT_BLOCK_SIZE, 8, NULL, fpu_enabled, nbytes);
176}
177
178static inline void serpent_fpu_end(bool fpu_enabled)
179{
180 glue_fpu_end(fpu_enabled);
181}
182
183struct crypt_priv {
184 struct serpent_ctx *ctx;
185 bool fpu_enabled;
186};
187
188static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
189{
190 const unsigned int bsize = SERPENT_BLOCK_SIZE;
191 struct crypt_priv *ctx = priv;
192 int i;
193
194 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
195
196 if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) {
197 serpent_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
198 srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
199 nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
200 }
201
202 while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) {
203 serpent_ecb_enc_8way_avx(ctx->ctx, srcdst, srcdst);
204 srcdst += bsize * SERPENT_PARALLEL_BLOCKS;
205 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
206 }
207
208 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
209 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
210}
211
212static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
213{
214 const unsigned int bsize = SERPENT_BLOCK_SIZE;
215 struct crypt_priv *ctx = priv;
216 int i;
217
218 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
219
220 if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) {
221 serpent_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
222 srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
223 nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
224 }
225
226 while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) {
227 serpent_ecb_dec_8way_avx(ctx->ctx, srcdst, srcdst);
228 srcdst += bsize * SERPENT_PARALLEL_BLOCKS;
229 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
230 }
231
232 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
233 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
234}
235
236static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
237 struct scatterlist *src, unsigned int nbytes)
238{
239 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
240 be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS];
241 struct crypt_priv crypt_ctx = {
242 .ctx = &ctx->serpent_ctx,
243 .fpu_enabled = false,
244 };
245 struct lrw_crypt_req req = {
246 .tbuf = buf,
247 .tbuflen = sizeof(buf),
248
249 .table_ctx = &ctx->lrw_table,
250 .crypt_ctx = &crypt_ctx,
251 .crypt_fn = encrypt_callback,
252 };
253 int ret;
254
255 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
256 ret = lrw_crypt(desc, dst, src, nbytes, &req);
257 serpent_fpu_end(crypt_ctx.fpu_enabled);
258
259 return ret;
260}
261
262static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
263 struct scatterlist *src, unsigned int nbytes)
264{
265 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
266 be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS];
267 struct crypt_priv crypt_ctx = {
268 .ctx = &ctx->serpent_ctx,
269 .fpu_enabled = false,
270 };
271 struct lrw_crypt_req req = {
272 .tbuf = buf,
273 .tbuflen = sizeof(buf),
274
275 .table_ctx = &ctx->lrw_table,
276 .crypt_ctx = &crypt_ctx,
277 .crypt_fn = decrypt_callback,
278 };
279 int ret;
280
281 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
282 ret = lrw_crypt(desc, dst, src, nbytes, &req);
283 serpent_fpu_end(crypt_ctx.fpu_enabled);
284
285 return ret;
286}
287
288static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
289 struct scatterlist *src, unsigned int nbytes)
290{
291 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
292
293 return glue_xts_crypt_128bit(&serpent_enc_xts, desc, dst, src, nbytes,
294 XTS_TWEAK_CAST(__serpent_encrypt),
295 &ctx->tweak_ctx, &ctx->crypt_ctx);
296}
297
298static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
299 struct scatterlist *src, unsigned int nbytes)
300{
301 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
302
303 return glue_xts_crypt_128bit(&serpent_dec_xts, desc, dst, src, nbytes,
304 XTS_TWEAK_CAST(__serpent_encrypt),
305 &ctx->tweak_ctx, &ctx->crypt_ctx);
306}
307
308static struct crypto_alg srp_algs[10] = { {
309 .cra_name = "__ecb-serpent-avx2",
310 .cra_driver_name = "__driver-ecb-serpent-avx2",
311 .cra_priority = 0,
312 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
313 .cra_blocksize = SERPENT_BLOCK_SIZE,
314 .cra_ctxsize = sizeof(struct serpent_ctx),
315 .cra_alignmask = 0,
316 .cra_type = &crypto_blkcipher_type,
317 .cra_module = THIS_MODULE,
318 .cra_list = LIST_HEAD_INIT(srp_algs[0].cra_list),
319 .cra_u = {
320 .blkcipher = {
321 .min_keysize = SERPENT_MIN_KEY_SIZE,
322 .max_keysize = SERPENT_MAX_KEY_SIZE,
323 .setkey = serpent_setkey,
324 .encrypt = ecb_encrypt,
325 .decrypt = ecb_decrypt,
326 },
327 },
328}, {
329 .cra_name = "__cbc-serpent-avx2",
330 .cra_driver_name = "__driver-cbc-serpent-avx2",
331 .cra_priority = 0,
332 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
333 .cra_blocksize = SERPENT_BLOCK_SIZE,
334 .cra_ctxsize = sizeof(struct serpent_ctx),
335 .cra_alignmask = 0,
336 .cra_type = &crypto_blkcipher_type,
337 .cra_module = THIS_MODULE,
338 .cra_list = LIST_HEAD_INIT(srp_algs[1].cra_list),
339 .cra_u = {
340 .blkcipher = {
341 .min_keysize = SERPENT_MIN_KEY_SIZE,
342 .max_keysize = SERPENT_MAX_KEY_SIZE,
343 .setkey = serpent_setkey,
344 .encrypt = cbc_encrypt,
345 .decrypt = cbc_decrypt,
346 },
347 },
348}, {
349 .cra_name = "__ctr-serpent-avx2",
350 .cra_driver_name = "__driver-ctr-serpent-avx2",
351 .cra_priority = 0,
352 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
353 .cra_blocksize = 1,
354 .cra_ctxsize = sizeof(struct serpent_ctx),
355 .cra_alignmask = 0,
356 .cra_type = &crypto_blkcipher_type,
357 .cra_module = THIS_MODULE,
358 .cra_list = LIST_HEAD_INIT(srp_algs[2].cra_list),
359 .cra_u = {
360 .blkcipher = {
361 .min_keysize = SERPENT_MIN_KEY_SIZE,
362 .max_keysize = SERPENT_MAX_KEY_SIZE,
363 .ivsize = SERPENT_BLOCK_SIZE,
364 .setkey = serpent_setkey,
365 .encrypt = ctr_crypt,
366 .decrypt = ctr_crypt,
367 },
368 },
369}, {
370 .cra_name = "__lrw-serpent-avx2",
371 .cra_driver_name = "__driver-lrw-serpent-avx2",
372 .cra_priority = 0,
373 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
374 .cra_blocksize = SERPENT_BLOCK_SIZE,
375 .cra_ctxsize = sizeof(struct serpent_lrw_ctx),
376 .cra_alignmask = 0,
377 .cra_type = &crypto_blkcipher_type,
378 .cra_module = THIS_MODULE,
379 .cra_list = LIST_HEAD_INIT(srp_algs[3].cra_list),
380 .cra_exit = lrw_serpent_exit_tfm,
381 .cra_u = {
382 .blkcipher = {
383 .min_keysize = SERPENT_MIN_KEY_SIZE +
384 SERPENT_BLOCK_SIZE,
385 .max_keysize = SERPENT_MAX_KEY_SIZE +
386 SERPENT_BLOCK_SIZE,
387 .ivsize = SERPENT_BLOCK_SIZE,
388 .setkey = lrw_serpent_setkey,
389 .encrypt = lrw_encrypt,
390 .decrypt = lrw_decrypt,
391 },
392 },
393}, {
394 .cra_name = "__xts-serpent-avx2",
395 .cra_driver_name = "__driver-xts-serpent-avx2",
396 .cra_priority = 0,
397 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
398 .cra_blocksize = SERPENT_BLOCK_SIZE,
399 .cra_ctxsize = sizeof(struct serpent_xts_ctx),
400 .cra_alignmask = 0,
401 .cra_type = &crypto_blkcipher_type,
402 .cra_module = THIS_MODULE,
403 .cra_list = LIST_HEAD_INIT(srp_algs[4].cra_list),
404 .cra_u = {
405 .blkcipher = {
406 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
407 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
408 .ivsize = SERPENT_BLOCK_SIZE,
409 .setkey = xts_serpent_setkey,
410 .encrypt = xts_encrypt,
411 .decrypt = xts_decrypt,
412 },
413 },
414}, {
415 .cra_name = "ecb(serpent)",
416 .cra_driver_name = "ecb-serpent-avx2",
417 .cra_priority = 600,
418 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
419 .cra_blocksize = SERPENT_BLOCK_SIZE,
420 .cra_ctxsize = sizeof(struct async_helper_ctx),
421 .cra_alignmask = 0,
422 .cra_type = &crypto_ablkcipher_type,
423 .cra_module = THIS_MODULE,
424 .cra_list = LIST_HEAD_INIT(srp_algs[5].cra_list),
425 .cra_init = ablk_init,
426 .cra_exit = ablk_exit,
427 .cra_u = {
428 .ablkcipher = {
429 .min_keysize = SERPENT_MIN_KEY_SIZE,
430 .max_keysize = SERPENT_MAX_KEY_SIZE,
431 .setkey = ablk_set_key,
432 .encrypt = ablk_encrypt,
433 .decrypt = ablk_decrypt,
434 },
435 },
436}, {
437 .cra_name = "cbc(serpent)",
438 .cra_driver_name = "cbc-serpent-avx2",
439 .cra_priority = 600,
440 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
441 .cra_blocksize = SERPENT_BLOCK_SIZE,
442 .cra_ctxsize = sizeof(struct async_helper_ctx),
443 .cra_alignmask = 0,
444 .cra_type = &crypto_ablkcipher_type,
445 .cra_module = THIS_MODULE,
446 .cra_list = LIST_HEAD_INIT(srp_algs[6].cra_list),
447 .cra_init = ablk_init,
448 .cra_exit = ablk_exit,
449 .cra_u = {
450 .ablkcipher = {
451 .min_keysize = SERPENT_MIN_KEY_SIZE,
452 .max_keysize = SERPENT_MAX_KEY_SIZE,
453 .ivsize = SERPENT_BLOCK_SIZE,
454 .setkey = ablk_set_key,
455 .encrypt = __ablk_encrypt,
456 .decrypt = ablk_decrypt,
457 },
458 },
459}, {
460 .cra_name = "ctr(serpent)",
461 .cra_driver_name = "ctr-serpent-avx2",
462 .cra_priority = 600,
463 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
464 .cra_blocksize = 1,
465 .cra_ctxsize = sizeof(struct async_helper_ctx),
466 .cra_alignmask = 0,
467 .cra_type = &crypto_ablkcipher_type,
468 .cra_module = THIS_MODULE,
469 .cra_list = LIST_HEAD_INIT(srp_algs[7].cra_list),
470 .cra_init = ablk_init,
471 .cra_exit = ablk_exit,
472 .cra_u = {
473 .ablkcipher = {
474 .min_keysize = SERPENT_MIN_KEY_SIZE,
475 .max_keysize = SERPENT_MAX_KEY_SIZE,
476 .ivsize = SERPENT_BLOCK_SIZE,
477 .setkey = ablk_set_key,
478 .encrypt = ablk_encrypt,
479 .decrypt = ablk_encrypt,
480 .geniv = "chainiv",
481 },
482 },
483}, {
484 .cra_name = "lrw(serpent)",
485 .cra_driver_name = "lrw-serpent-avx2",
486 .cra_priority = 600,
487 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
488 .cra_blocksize = SERPENT_BLOCK_SIZE,
489 .cra_ctxsize = sizeof(struct async_helper_ctx),
490 .cra_alignmask = 0,
491 .cra_type = &crypto_ablkcipher_type,
492 .cra_module = THIS_MODULE,
493 .cra_list = LIST_HEAD_INIT(srp_algs[8].cra_list),
494 .cra_init = ablk_init,
495 .cra_exit = ablk_exit,
496 .cra_u = {
497 .ablkcipher = {
498 .min_keysize = SERPENT_MIN_KEY_SIZE +
499 SERPENT_BLOCK_SIZE,
500 .max_keysize = SERPENT_MAX_KEY_SIZE +
501 SERPENT_BLOCK_SIZE,
502 .ivsize = SERPENT_BLOCK_SIZE,
503 .setkey = ablk_set_key,
504 .encrypt = ablk_encrypt,
505 .decrypt = ablk_decrypt,
506 },
507 },
508}, {
509 .cra_name = "xts(serpent)",
510 .cra_driver_name = "xts-serpent-avx2",
511 .cra_priority = 600,
512 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
513 .cra_blocksize = SERPENT_BLOCK_SIZE,
514 .cra_ctxsize = sizeof(struct async_helper_ctx),
515 .cra_alignmask = 0,
516 .cra_type = &crypto_ablkcipher_type,
517 .cra_module = THIS_MODULE,
518 .cra_list = LIST_HEAD_INIT(srp_algs[9].cra_list),
519 .cra_init = ablk_init,
520 .cra_exit = ablk_exit,
521 .cra_u = {
522 .ablkcipher = {
523 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
524 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
525 .ivsize = SERPENT_BLOCK_SIZE,
526 .setkey = ablk_set_key,
527 .encrypt = ablk_encrypt,
528 .decrypt = ablk_decrypt,
529 },
530 },
531} };
532
533static int __init init(void)
534{
535 u64 xcr0;
536
537 if (!cpu_has_avx2 || !cpu_has_osxsave) {
538 pr_info("AVX2 instructions are not detected.\n");
539 return -ENODEV;
540 }
541
542 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
543 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
544 pr_info("AVX detected but unusable.\n");
545 return -ENODEV;
546 }
547
548 return crypto_register_algs(srp_algs, ARRAY_SIZE(srp_algs));
549}
550
551static void __exit fini(void)
552{
553 crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs));
554}
555
556module_init(init);
557module_exit(fini);
558
559MODULE_LICENSE("GPL");
560MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
561MODULE_ALIAS("serpent");
562MODULE_ALIAS("serpent-asm");
563
