1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <linux/module.h>
25#include <linux/hardirq.h>
26#include <linux/types.h>
27#include <linux/crypto.h>
28#include <linux/err.h>
29#include <crypto/ablk_helper.h>
30#include <crypto/algapi.h>
31#include <crypto/cast5.h>
32#include <crypto/cryptd.h>
33#include <crypto/ctr.h>
34#include <asm/fpu/api.h>
35#include <asm/crypto/glue_helper.h>
36
37#define CAST5_PARALLEL_BLOCKS 16
38
39asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst,
40 const u8 *src);
41asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst,
42 const u8 *src);
43asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst,
44 const u8 *src);
45asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src,
46 __be64 *iv);
47
48static inline bool cast5_fpu_begin(bool fpu_enabled, unsigned int nbytes)
49{
50 return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS,
51 NULL, fpu_enabled, nbytes);
52}
53
54static inline void cast5_fpu_end(bool fpu_enabled)
55{
56 return glue_fpu_end(fpu_enabled);
57}
58
59static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
60 bool enc)
61{
62 bool fpu_enabled = false;
63 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
64 const unsigned int bsize = CAST5_BLOCK_SIZE;
65 unsigned int nbytes;
66 void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
67 int err;
68
69 fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way;
70
71 err = blkcipher_walk_virt(desc, walk);
72 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
73
74 while ((nbytes = walk->nbytes)) {
75 u8 *wsrc = walk->src.virt.addr;
76 u8 *wdst = walk->dst.virt.addr;
77
78 fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
79
80
81 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
82 do {
83 fn(ctx, wdst, wsrc);
84
85 wsrc += bsize * CAST5_PARALLEL_BLOCKS;
86 wdst += bsize * CAST5_PARALLEL_BLOCKS;
87 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
88 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
89
90 if (nbytes < bsize)
91 goto done;
92 }
93
94 fn = (enc) ? __cast5_encrypt : __cast5_decrypt;
95
96
97 do {
98 fn(ctx, wdst, wsrc);
99
100 wsrc += bsize;
101 wdst += bsize;
102 nbytes -= bsize;
103 } while (nbytes >= bsize);
104
105done:
106 err = blkcipher_walk_done(desc, walk, nbytes);
107 }
108
109 cast5_fpu_end(fpu_enabled);
110 return err;
111}
112
113static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
114 struct scatterlist *src, unsigned int nbytes)
115{
116 struct blkcipher_walk walk;
117
118 blkcipher_walk_init(&walk, dst, src, nbytes);
119 return ecb_crypt(desc, &walk, true);
120}
121
122static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
123 struct scatterlist *src, unsigned int nbytes)
124{
125 struct blkcipher_walk walk;
126
127 blkcipher_walk_init(&walk, dst, src, nbytes);
128 return ecb_crypt(desc, &walk, false);
129}
130
131static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
132 struct blkcipher_walk *walk)
133{
134 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
135 const unsigned int bsize = CAST5_BLOCK_SIZE;
136 unsigned int nbytes = walk->nbytes;
137 u64 *src = (u64 *)walk->src.virt.addr;
138 u64 *dst = (u64 *)walk->dst.virt.addr;
139 u64 *iv = (u64 *)walk->iv;
140
141 do {
142 *dst = *src ^ *iv;
143 __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst);
144 iv = dst;
145
146 src += 1;
147 dst += 1;
148 nbytes -= bsize;
149 } while (nbytes >= bsize);
150
151 *(u64 *)walk->iv = *iv;
152 return nbytes;
153}
154
155static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
156 struct scatterlist *src, unsigned int nbytes)
157{
158 struct blkcipher_walk walk;
159 int err;
160
161 blkcipher_walk_init(&walk, dst, src, nbytes);
162 err = blkcipher_walk_virt(desc, &walk);
163
164 while ((nbytes = walk.nbytes)) {
165 nbytes = __cbc_encrypt(desc, &walk);
166 err = blkcipher_walk_done(desc, &walk, nbytes);
167 }
168
169 return err;
170}
171
172static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
173 struct blkcipher_walk *walk)
174{
175 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
176 const unsigned int bsize = CAST5_BLOCK_SIZE;
177 unsigned int nbytes = walk->nbytes;
178 u64 *src = (u64 *)walk->src.virt.addr;
179 u64 *dst = (u64 *)walk->dst.virt.addr;
180 u64 last_iv;
181
182
183 src += nbytes / bsize - 1;
184 dst += nbytes / bsize - 1;
185
186 last_iv = *src;
187
188
189 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
190 do {
191 nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1);
192 src -= CAST5_PARALLEL_BLOCKS - 1;
193 dst -= CAST5_PARALLEL_BLOCKS - 1;
194
195 cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src);
196
197 nbytes -= bsize;
198 if (nbytes < bsize)
199 goto done;
200
201 *dst ^= *(src - 1);
202 src -= 1;
203 dst -= 1;
204 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
205 }
206
207
208 for (;;) {
209 __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src);
210
211 nbytes -= bsize;
212 if (nbytes < bsize)
213 break;
214
215 *dst ^= *(src - 1);
216 src -= 1;
217 dst -= 1;
218 }
219
220done:
221 *dst ^= *(u64 *)walk->iv;
222 *(u64 *)walk->iv = last_iv;
223
224 return nbytes;
225}
226
227static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
228 struct scatterlist *src, unsigned int nbytes)
229{
230 bool fpu_enabled = false;
231 struct blkcipher_walk walk;
232 int err;
233
234 blkcipher_walk_init(&walk, dst, src, nbytes);
235 err = blkcipher_walk_virt(desc, &walk);
236 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
237
238 while ((nbytes = walk.nbytes)) {
239 fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
240 nbytes = __cbc_decrypt(desc, &walk);
241 err = blkcipher_walk_done(desc, &walk, nbytes);
242 }
243
244 cast5_fpu_end(fpu_enabled);
245 return err;
246}
247
248static void ctr_crypt_final(struct blkcipher_desc *desc,
249 struct blkcipher_walk *walk)
250{
251 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
252 u8 *ctrblk = walk->iv;
253 u8 keystream[CAST5_BLOCK_SIZE];
254 u8 *src = walk->src.virt.addr;
255 u8 *dst = walk->dst.virt.addr;
256 unsigned int nbytes = walk->nbytes;
257
258 __cast5_encrypt(ctx, keystream, ctrblk);
259 crypto_xor(keystream, src, nbytes);
260 memcpy(dst, keystream, nbytes);
261
262 crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
263}
264
265static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
266 struct blkcipher_walk *walk)
267{
268 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
269 const unsigned int bsize = CAST5_BLOCK_SIZE;
270 unsigned int nbytes = walk->nbytes;
271 u64 *src = (u64 *)walk->src.virt.addr;
272 u64 *dst = (u64 *)walk->dst.virt.addr;
273
274
275 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
276 do {
277 cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src,
278 (__be64 *)walk->iv);
279
280 src += CAST5_PARALLEL_BLOCKS;
281 dst += CAST5_PARALLEL_BLOCKS;
282 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
283 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
284
285 if (nbytes < bsize)
286 goto done;
287 }
288
289
290 do {
291 u64 ctrblk;
292
293 if (dst != src)
294 *dst = *src;
295
296 ctrblk = *(u64 *)walk->iv;
297 be64_add_cpu((__be64 *)walk->iv, 1);
298
299 __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
300 *dst ^= ctrblk;
301
302 src += 1;
303 dst += 1;
304 nbytes -= bsize;
305 } while (nbytes >= bsize);
306
307done:
308 return nbytes;
309}
310
311static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
312 struct scatterlist *src, unsigned int nbytes)
313{
314 bool fpu_enabled = false;
315 struct blkcipher_walk walk;
316 int err;
317
318 blkcipher_walk_init(&walk, dst, src, nbytes);
319 err = blkcipher_walk_virt_block(desc, &walk, CAST5_BLOCK_SIZE);
320 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
321
322 while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
323 fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
324 nbytes = __ctr_crypt(desc, &walk);
325 err = blkcipher_walk_done(desc, &walk, nbytes);
326 }
327
328 cast5_fpu_end(fpu_enabled);
329
330 if (walk.nbytes) {
331 ctr_crypt_final(desc, &walk);
332 err = blkcipher_walk_done(desc, &walk, 0);
333 }
334
335 return err;
336}
337
338
339static struct crypto_alg cast5_algs[6] = { {
340 .cra_name = "__ecb-cast5-avx",
341 .cra_driver_name = "__driver-ecb-cast5-avx",
342 .cra_priority = 0,
343 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
344 CRYPTO_ALG_INTERNAL,
345 .cra_blocksize = CAST5_BLOCK_SIZE,
346 .cra_ctxsize = sizeof(struct cast5_ctx),
347 .cra_alignmask = 0,
348 .cra_type = &crypto_blkcipher_type,
349 .cra_module = THIS_MODULE,
350 .cra_u = {
351 .blkcipher = {
352 .min_keysize = CAST5_MIN_KEY_SIZE,
353 .max_keysize = CAST5_MAX_KEY_SIZE,
354 .setkey = cast5_setkey,
355 .encrypt = ecb_encrypt,
356 .decrypt = ecb_decrypt,
357 },
358 },
359}, {
360 .cra_name = "__cbc-cast5-avx",
361 .cra_driver_name = "__driver-cbc-cast5-avx",
362 .cra_priority = 0,
363 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
364 CRYPTO_ALG_INTERNAL,
365 .cra_blocksize = CAST5_BLOCK_SIZE,
366 .cra_ctxsize = sizeof(struct cast5_ctx),
367 .cra_alignmask = 0,
368 .cra_type = &crypto_blkcipher_type,
369 .cra_module = THIS_MODULE,
370 .cra_u = {
371 .blkcipher = {
372 .min_keysize = CAST5_MIN_KEY_SIZE,
373 .max_keysize = CAST5_MAX_KEY_SIZE,
374 .setkey = cast5_setkey,
375 .encrypt = cbc_encrypt,
376 .decrypt = cbc_decrypt,
377 },
378 },
379}, {
380 .cra_name = "__ctr-cast5-avx",
381 .cra_driver_name = "__driver-ctr-cast5-avx",
382 .cra_priority = 0,
383 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
384 CRYPTO_ALG_INTERNAL,
385 .cra_blocksize = 1,
386 .cra_ctxsize = sizeof(struct cast5_ctx),
387 .cra_alignmask = 0,
388 .cra_type = &crypto_blkcipher_type,
389 .cra_module = THIS_MODULE,
390 .cra_u = {
391 .blkcipher = {
392 .min_keysize = CAST5_MIN_KEY_SIZE,
393 .max_keysize = CAST5_MAX_KEY_SIZE,
394 .ivsize = CAST5_BLOCK_SIZE,
395 .setkey = cast5_setkey,
396 .encrypt = ctr_crypt,
397 .decrypt = ctr_crypt,
398 },
399 },
400}, {
401 .cra_name = "ecb(cast5)",
402 .cra_driver_name = "ecb-cast5-avx",
403 .cra_priority = 200,
404 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
405 .cra_blocksize = CAST5_BLOCK_SIZE,
406 .cra_ctxsize = sizeof(struct async_helper_ctx),
407 .cra_alignmask = 0,
408 .cra_type = &crypto_ablkcipher_type,
409 .cra_module = THIS_MODULE,
410 .cra_init = ablk_init,
411 .cra_exit = ablk_exit,
412 .cra_u = {
413 .ablkcipher = {
414 .min_keysize = CAST5_MIN_KEY_SIZE,
415 .max_keysize = CAST5_MAX_KEY_SIZE,
416 .setkey = ablk_set_key,
417 .encrypt = ablk_encrypt,
418 .decrypt = ablk_decrypt,
419 },
420 },
421}, {
422 .cra_name = "cbc(cast5)",
423 .cra_driver_name = "cbc-cast5-avx",
424 .cra_priority = 200,
425 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
426 .cra_blocksize = CAST5_BLOCK_SIZE,
427 .cra_ctxsize = sizeof(struct async_helper_ctx),
428 .cra_alignmask = 0,
429 .cra_type = &crypto_ablkcipher_type,
430 .cra_module = THIS_MODULE,
431 .cra_init = ablk_init,
432 .cra_exit = ablk_exit,
433 .cra_u = {
434 .ablkcipher = {
435 .min_keysize = CAST5_MIN_KEY_SIZE,
436 .max_keysize = CAST5_MAX_KEY_SIZE,
437 .ivsize = CAST5_BLOCK_SIZE,
438 .setkey = ablk_set_key,
439 .encrypt = __ablk_encrypt,
440 .decrypt = ablk_decrypt,
441 },
442 },
443}, {
444 .cra_name = "ctr(cast5)",
445 .cra_driver_name = "ctr-cast5-avx",
446 .cra_priority = 200,
447 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
448 .cra_blocksize = 1,
449 .cra_ctxsize = sizeof(struct async_helper_ctx),
450 .cra_alignmask = 0,
451 .cra_type = &crypto_ablkcipher_type,
452 .cra_module = THIS_MODULE,
453 .cra_init = ablk_init,
454 .cra_exit = ablk_exit,
455 .cra_u = {
456 .ablkcipher = {
457 .min_keysize = CAST5_MIN_KEY_SIZE,
458 .max_keysize = CAST5_MAX_KEY_SIZE,
459 .ivsize = CAST5_BLOCK_SIZE,
460 .setkey = ablk_set_key,
461 .encrypt = ablk_encrypt,
462 .decrypt = ablk_encrypt,
463 .geniv = "chainiv",
464 },
465 },
466} };
467
468static int __init cast5_init(void)
469{
470 const char *feature_name;
471
472 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
473 &feature_name)) {
474 pr_info("CPU feature '%s' is not supported.\n", feature_name);
475 return -ENODEV;
476 }
477
478 return crypto_register_algs(cast5_algs, ARRAY_SIZE(cast5_algs));
479}
480
481static void __exit cast5_exit(void)
482{
483 crypto_unregister_algs(cast5_algs, ARRAY_SIZE(cast5_algs));
484}
485
486module_init(cast5_init);
487module_exit(cast5_exit);
488
489MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
490MODULE_LICENSE("GPL");
491MODULE_ALIAS_CRYPTO("cast5");
492
