1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27#include <linux/types.h>
28#include <linux/crypto.h>
29#include <linux/err.h>
30#include <crypto/algapi.h>
31#include <crypto/internal/simd.h>
32#include <crypto/twofish.h>
33#include <crypto/xts.h>
34#include <asm/crypto/glue_helper.h>
35#include <asm/crypto/twofish.h>
36
37#define TWOFISH_PARALLEL_BLOCKS 8
38
39
40asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
41 const u8 *src);
42asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
43 const u8 *src);
44
45asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
46 const u8 *src);
47asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
48 const u8 *src, le128 *iv);
49
50asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
51 const u8 *src, le128 *iv);
52asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
53 const u8 *src, le128 *iv);
54
55static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
56 const u8 *key, unsigned int keylen)
57{
58 return twofish_setkey(&tfm->base, key, keylen);
59}
60
61static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
62 const u8 *src)
63{
64 __twofish_enc_blk_3way(ctx, dst, src, false);
65}
66
67static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
68{
69 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
70 GLUE_FUNC_CAST(twofish_enc_blk));
71}
72
73static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
74{
75 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
76 GLUE_FUNC_CAST(twofish_dec_blk));
77}
78
79struct twofish_xts_ctx {
80 struct twofish_ctx tweak_ctx;
81 struct twofish_ctx crypt_ctx;
82};
83
84static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
85 unsigned int keylen)
86{
87 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
88 u32 *flags = &tfm->base.crt_flags;
89 int err;
90
91 err = xts_verify_key(tfm, key, keylen);
92 if (err)
93 return err;
94
95
96 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
97 if (err)
98 return err;
99
100
101 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
102 flags);
103}
104
105static const struct common_glue_ctx twofish_enc = {
106 .num_funcs = 3,
107 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
108
109 .funcs = { {
110 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
111 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
112 }, {
113 .num_blocks = 3,
114 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
115 }, {
116 .num_blocks = 1,
117 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
118 } }
119};
120
121static const struct common_glue_ctx twofish_ctr = {
122 .num_funcs = 3,
123 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
124
125 .funcs = { {
126 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
127 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
128 }, {
129 .num_blocks = 3,
130 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
131 }, {
132 .num_blocks = 1,
133 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
134 } }
135};
136
137static const struct common_glue_ctx twofish_enc_xts = {
138 .num_funcs = 2,
139 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
140
141 .funcs = { {
142 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
143 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
144 }, {
145 .num_blocks = 1,
146 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
147 } }
148};
149
150static const struct common_glue_ctx twofish_dec = {
151 .num_funcs = 3,
152 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
153
154 .funcs = { {
155 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
156 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
157 }, {
158 .num_blocks = 3,
159 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
160 }, {
161 .num_blocks = 1,
162 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
163 } }
164};
165
166static const struct common_glue_ctx twofish_dec_cbc = {
167 .num_funcs = 3,
168 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
169
170 .funcs = { {
171 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
172 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
173 }, {
174 .num_blocks = 3,
175 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
176 }, {
177 .num_blocks = 1,
178 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
179 } }
180};
181
182static const struct common_glue_ctx twofish_dec_xts = {
183 .num_funcs = 2,
184 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
185
186 .funcs = { {
187 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
188 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
189 }, {
190 .num_blocks = 1,
191 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
192 } }
193};
194
195static int ecb_encrypt(struct skcipher_request *req)
196{
197 return glue_ecb_req_128bit(&twofish_enc, req);
198}
199
200static int ecb_decrypt(struct skcipher_request *req)
201{
202 return glue_ecb_req_128bit(&twofish_dec, req);
203}
204
205static int cbc_encrypt(struct skcipher_request *req)
206{
207 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
208 req);
209}
210
211static int cbc_decrypt(struct skcipher_request *req)
212{
213 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
214}
215
216static int ctr_crypt(struct skcipher_request *req)
217{
218 return glue_ctr_req_128bit(&twofish_ctr, req);
219}
220
221static int xts_encrypt(struct skcipher_request *req)
222{
223 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
224 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
225
226 return glue_xts_req_128bit(&twofish_enc_xts, req,
227 XTS_TWEAK_CAST(twofish_enc_blk),
228 &ctx->tweak_ctx, &ctx->crypt_ctx);
229}
230
231static int xts_decrypt(struct skcipher_request *req)
232{
233 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
234 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
235
236 return glue_xts_req_128bit(&twofish_dec_xts, req,
237 XTS_TWEAK_CAST(twofish_enc_blk),
238 &ctx->tweak_ctx, &ctx->crypt_ctx);
239}
240
241static struct skcipher_alg twofish_algs[] = {
242 {
243 .base.cra_name = "__ecb(twofish)",
244 .base.cra_driver_name = "__ecb-twofish-avx",
245 .base.cra_priority = 400,
246 .base.cra_flags = CRYPTO_ALG_INTERNAL,
247 .base.cra_blocksize = TF_BLOCK_SIZE,
248 .base.cra_ctxsize = sizeof(struct twofish_ctx),
249 .base.cra_module = THIS_MODULE,
250 .min_keysize = TF_MIN_KEY_SIZE,
251 .max_keysize = TF_MAX_KEY_SIZE,
252 .setkey = twofish_setkey_skcipher,
253 .encrypt = ecb_encrypt,
254 .decrypt = ecb_decrypt,
255 }, {
256 .base.cra_name = "__cbc(twofish)",
257 .base.cra_driver_name = "__cbc-twofish-avx",
258 .base.cra_priority = 400,
259 .base.cra_flags = CRYPTO_ALG_INTERNAL,
260 .base.cra_blocksize = TF_BLOCK_SIZE,
261 .base.cra_ctxsize = sizeof(struct twofish_ctx),
262 .base.cra_module = THIS_MODULE,
263 .min_keysize = TF_MIN_KEY_SIZE,
264 .max_keysize = TF_MAX_KEY_SIZE,
265 .ivsize = TF_BLOCK_SIZE,
266 .setkey = twofish_setkey_skcipher,
267 .encrypt = cbc_encrypt,
268 .decrypt = cbc_decrypt,
269 }, {
270 .base.cra_name = "__ctr(twofish)",
271 .base.cra_driver_name = "__ctr-twofish-avx",
272 .base.cra_priority = 400,
273 .base.cra_flags = CRYPTO_ALG_INTERNAL,
274 .base.cra_blocksize = 1,
275 .base.cra_ctxsize = sizeof(struct twofish_ctx),
276 .base.cra_module = THIS_MODULE,
277 .min_keysize = TF_MIN_KEY_SIZE,
278 .max_keysize = TF_MAX_KEY_SIZE,
279 .ivsize = TF_BLOCK_SIZE,
280 .chunksize = TF_BLOCK_SIZE,
281 .setkey = twofish_setkey_skcipher,
282 .encrypt = ctr_crypt,
283 .decrypt = ctr_crypt,
284 }, {
285 .base.cra_name = "__xts(twofish)",
286 .base.cra_driver_name = "__xts-twofish-avx",
287 .base.cra_priority = 400,
288 .base.cra_flags = CRYPTO_ALG_INTERNAL,
289 .base.cra_blocksize = TF_BLOCK_SIZE,
290 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx),
291 .base.cra_module = THIS_MODULE,
292 .min_keysize = 2 * TF_MIN_KEY_SIZE,
293 .max_keysize = 2 * TF_MAX_KEY_SIZE,
294 .ivsize = TF_BLOCK_SIZE,
295 .setkey = xts_twofish_setkey,
296 .encrypt = xts_encrypt,
297 .decrypt = xts_decrypt,
298 },
299};
300
301static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
302
303static int __init twofish_init(void)
304{
305 const char *feature_name;
306
307 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
308 pr_info("CPU feature '%s' is not supported.\n", feature_name);
309 return -ENODEV;
310 }
311
312 return simd_register_skciphers_compat(twofish_algs,
313 ARRAY_SIZE(twofish_algs),
314 twofish_simd_algs);
315}
316
317static void __exit twofish_exit(void)
318{
319 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
320 twofish_simd_algs);
321}
322
323module_init(twofish_init);
324module_exit(twofish_exit);
325
326MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
327MODULE_LICENSE("GPL");
328MODULE_ALIAS_CRYPTO("twofish");
329
