1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include <asm/crypto/glue_helper.h>
25#include <crypto/algapi.h>
26#include <crypto/cast5.h>
27#include <crypto/internal/simd.h>
28#include <linux/crypto.h>
29#include <linux/err.h>
30#include <linux/module.h>
31#include <linux/types.h>
32
33#define CAST5_PARALLEL_BLOCKS 16
34
35asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst,
36 const u8 *src);
37asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst,
38 const u8 *src);
39asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst,
40 const u8 *src);
41asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src,
42 __be64 *iv);
43
44static int cast5_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key,
45 unsigned int keylen)
46{
47 return cast5_setkey(&tfm->base, key, keylen);
48}
49
50static inline bool cast5_fpu_begin(bool fpu_enabled, struct skcipher_walk *walk,
51 unsigned int nbytes)
52{
53 return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS,
54 walk, fpu_enabled, nbytes);
55}
56
57static inline void cast5_fpu_end(bool fpu_enabled)
58{
59 return glue_fpu_end(fpu_enabled);
60}
61
62static int ecb_crypt(struct skcipher_request *req, bool enc)
63{
64 bool fpu_enabled = false;
65 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
66 struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
67 struct skcipher_walk walk;
68 const unsigned int bsize = CAST5_BLOCK_SIZE;
69 unsigned int nbytes;
70 void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
71 int err;
72
73 err = skcipher_walk_virt(&walk, req, false);
74
75 while ((nbytes = walk.nbytes)) {
76 u8 *wsrc = walk.src.virt.addr;
77 u8 *wdst = walk.dst.virt.addr;
78
79 fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes);
80
81
82 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
83 fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way;
84 do {
85 fn(ctx, wdst, wsrc);
86
87 wsrc += bsize * CAST5_PARALLEL_BLOCKS;
88 wdst += bsize * CAST5_PARALLEL_BLOCKS;
89 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
90 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
91
92 if (nbytes < bsize)
93 goto done;
94 }
95
96 fn = (enc) ? __cast5_encrypt : __cast5_decrypt;
97
98
99 do {
100 fn(ctx, wdst, wsrc);
101
102 wsrc += bsize;
103 wdst += bsize;
104 nbytes -= bsize;
105 } while (nbytes >= bsize);
106
107done:
108 err = skcipher_walk_done(&walk, nbytes);
109 }
110
111 cast5_fpu_end(fpu_enabled);
112 return err;
113}
114
115static int ecb_encrypt(struct skcipher_request *req)
116{
117 return ecb_crypt(req, true);
118}
119
120static int ecb_decrypt(struct skcipher_request *req)
121{
122 return ecb_crypt(req, false);
123}
124
125static int cbc_encrypt(struct skcipher_request *req)
126{
127 const unsigned int bsize = CAST5_BLOCK_SIZE;
128 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
129 struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
130 struct skcipher_walk walk;
131 unsigned int nbytes;
132 int err;
133
134 err = skcipher_walk_virt(&walk, req, false);
135
136 while ((nbytes = walk.nbytes)) {
137 u64 *src = (u64 *)walk.src.virt.addr;
138 u64 *dst = (u64 *)walk.dst.virt.addr;
139 u64 *iv = (u64 *)walk.iv;
140
141 do {
142 *dst = *src ^ *iv;
143 __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst);
144 iv = dst;
145 src++;
146 dst++;
147 nbytes -= bsize;
148 } while (nbytes >= bsize);
149
150 *(u64 *)walk.iv = *iv;
151 err = skcipher_walk_done(&walk, nbytes);
152 }
153
154 return err;
155}
156
157static unsigned int __cbc_decrypt(struct cast5_ctx *ctx,
158 struct skcipher_walk *walk)
159{
160 const unsigned int bsize = CAST5_BLOCK_SIZE;
161 unsigned int nbytes = walk->nbytes;
162 u64 *src = (u64 *)walk->src.virt.addr;
163 u64 *dst = (u64 *)walk->dst.virt.addr;
164 u64 last_iv;
165
166
167 src += nbytes / bsize - 1;
168 dst += nbytes / bsize - 1;
169
170 last_iv = *src;
171
172
173 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
174 do {
175 nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1);
176 src -= CAST5_PARALLEL_BLOCKS - 1;
177 dst -= CAST5_PARALLEL_BLOCKS - 1;
178
179 cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src);
180
181 nbytes -= bsize;
182 if (nbytes < bsize)
183 goto done;
184
185 *dst ^= *(src - 1);
186 src -= 1;
187 dst -= 1;
188 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
189 }
190
191
192 for (;;) {
193 __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src);
194
195 nbytes -= bsize;
196 if (nbytes < bsize)
197 break;
198
199 *dst ^= *(src - 1);
200 src -= 1;
201 dst -= 1;
202 }
203
204done:
205 *dst ^= *(u64 *)walk->iv;
206 *(u64 *)walk->iv = last_iv;
207
208 return nbytes;
209}
210
211static int cbc_decrypt(struct skcipher_request *req)
212{
213 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
214 struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
215 bool fpu_enabled = false;
216 struct skcipher_walk walk;
217 unsigned int nbytes;
218 int err;
219
220 err = skcipher_walk_virt(&walk, req, false);
221
222 while ((nbytes = walk.nbytes)) {
223 fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes);
224 nbytes = __cbc_decrypt(ctx, &walk);
225 err = skcipher_walk_done(&walk, nbytes);
226 }
227
228 cast5_fpu_end(fpu_enabled);
229 return err;
230}
231
232static void ctr_crypt_final(struct skcipher_walk *walk, struct cast5_ctx *ctx)
233{
234 u8 *ctrblk = walk->iv;
235 u8 keystream[CAST5_BLOCK_SIZE];
236 u8 *src = walk->src.virt.addr;
237 u8 *dst = walk->dst.virt.addr;
238 unsigned int nbytes = walk->nbytes;
239
240 __cast5_encrypt(ctx, keystream, ctrblk);
241 crypto_xor_cpy(dst, keystream, src, nbytes);
242
243 crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
244}
245
246static unsigned int __ctr_crypt(struct skcipher_walk *walk,
247 struct cast5_ctx *ctx)
248{
249 const unsigned int bsize = CAST5_BLOCK_SIZE;
250 unsigned int nbytes = walk->nbytes;
251 u64 *src = (u64 *)walk->src.virt.addr;
252 u64 *dst = (u64 *)walk->dst.virt.addr;
253
254
255 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
256 do {
257 cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src,
258 (__be64 *)walk->iv);
259
260 src += CAST5_PARALLEL_BLOCKS;
261 dst += CAST5_PARALLEL_BLOCKS;
262 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
263 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
264
265 if (nbytes < bsize)
266 goto done;
267 }
268
269
270 do {
271 u64 ctrblk;
272
273 if (dst != src)
274 *dst = *src;
275
276 ctrblk = *(u64 *)walk->iv;
277 be64_add_cpu((__be64 *)walk->iv, 1);
278
279 __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
280 *dst ^= ctrblk;
281
282 src += 1;
283 dst += 1;
284 nbytes -= bsize;
285 } while (nbytes >= bsize);
286
287done:
288 return nbytes;
289}
290
291static int ctr_crypt(struct skcipher_request *req)
292{
293 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
294 struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
295 bool fpu_enabled = false;
296 struct skcipher_walk walk;
297 unsigned int nbytes;
298 int err;
299
300 err = skcipher_walk_virt(&walk, req, false);
301
302 while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
303 fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes);
304 nbytes = __ctr_crypt(&walk, ctx);
305 err = skcipher_walk_done(&walk, nbytes);
306 }
307
308 cast5_fpu_end(fpu_enabled);
309
310 if (walk.nbytes) {
311 ctr_crypt_final(&walk, ctx);
312 err = skcipher_walk_done(&walk, 0);
313 }
314
315 return err;
316}
317
318static struct skcipher_alg cast5_algs[] = {
319 {
320 .base.cra_name = "__ecb(cast5)",
321 .base.cra_driver_name = "__ecb-cast5-avx",
322 .base.cra_priority = 200,
323 .base.cra_flags = CRYPTO_ALG_INTERNAL,
324 .base.cra_blocksize = CAST5_BLOCK_SIZE,
325 .base.cra_ctxsize = sizeof(struct cast5_ctx),
326 .base.cra_module = THIS_MODULE,
327 .min_keysize = CAST5_MIN_KEY_SIZE,
328 .max_keysize = CAST5_MAX_KEY_SIZE,
329 .setkey = cast5_setkey_skcipher,
330 .encrypt = ecb_encrypt,
331 .decrypt = ecb_decrypt,
332 }, {
333 .base.cra_name = "__cbc(cast5)",
334 .base.cra_driver_name = "__cbc-cast5-avx",
335 .base.cra_priority = 200,
336 .base.cra_flags = CRYPTO_ALG_INTERNAL,
337 .base.cra_blocksize = CAST5_BLOCK_SIZE,
338 .base.cra_ctxsize = sizeof(struct cast5_ctx),
339 .base.cra_module = THIS_MODULE,
340 .min_keysize = CAST5_MIN_KEY_SIZE,
341 .max_keysize = CAST5_MAX_KEY_SIZE,
342 .ivsize = CAST5_BLOCK_SIZE,
343 .setkey = cast5_setkey_skcipher,
344 .encrypt = cbc_encrypt,
345 .decrypt = cbc_decrypt,
346 }, {
347 .base.cra_name = "__ctr(cast5)",
348 .base.cra_driver_name = "__ctr-cast5-avx",
349 .base.cra_priority = 200,
350 .base.cra_flags = CRYPTO_ALG_INTERNAL,
351 .base.cra_blocksize = 1,
352 .base.cra_ctxsize = sizeof(struct cast5_ctx),
353 .base.cra_module = THIS_MODULE,
354 .min_keysize = CAST5_MIN_KEY_SIZE,
355 .max_keysize = CAST5_MAX_KEY_SIZE,
356 .ivsize = CAST5_BLOCK_SIZE,
357 .chunksize = CAST5_BLOCK_SIZE,
358 .setkey = cast5_setkey_skcipher,
359 .encrypt = ctr_crypt,
360 .decrypt = ctr_crypt,
361 }
362};
363
364static struct simd_skcipher_alg *cast5_simd_algs[ARRAY_SIZE(cast5_algs)];
365
366static int __init cast5_init(void)
367{
368 const char *feature_name;
369
370 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
371 &feature_name)) {
372 pr_info("CPU feature '%s' is not supported.\n", feature_name);
373 return -ENODEV;
374 }
375
376 return simd_register_skciphers_compat(cast5_algs,
377 ARRAY_SIZE(cast5_algs),
378 cast5_simd_algs);
379}
380
381static void __exit cast5_exit(void)
382{
383 simd_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs),
384 cast5_simd_algs);
385}
386
387module_init(cast5_init);
388module_exit(cast5_exit);
389
390MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
391MODULE_LICENSE("GPL");
392MODULE_ALIAS_CRYPTO("cast5");
393
