1
2
3
4
5
6
7
8
9
10
11#include <linux/module.h>
12#include <linux/types.h>
13#include <linux/crypto.h>
14#include <linux/err.h>
15#include <crypto/algapi.h>
16#include <crypto/internal/simd.h>
17#include <crypto/twofish.h>
18#include <crypto/xts.h>
19#include <asm/crypto/glue_helper.h>
20#include <asm/crypto/twofish.h>
21
22#define TWOFISH_PARALLEL_BLOCKS 8
23
24
25asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
26 const u8 *src);
27asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
28 const u8 *src);
29
30asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
31 const u8 *src);
32asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
33 const u8 *src, le128 *iv);
34
35asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
36 const u8 *src, le128 *iv);
37asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
38 const u8 *src, le128 *iv);
39
40static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
41 const u8 *key, unsigned int keylen)
42{
43 return twofish_setkey(&tfm->base, key, keylen);
44}
45
46static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
47 const u8 *src)
48{
49 __twofish_enc_blk_3way(ctx, dst, src, false);
50}
51
52static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
53{
54 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
55 GLUE_FUNC_CAST(twofish_enc_blk));
56}
57
58static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
59{
60 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
61 GLUE_FUNC_CAST(twofish_dec_blk));
62}
63
64struct twofish_xts_ctx {
65 struct twofish_ctx tweak_ctx;
66 struct twofish_ctx crypt_ctx;
67};
68
69static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
70 unsigned int keylen)
71{
72 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
73 u32 *flags = &tfm->base.crt_flags;
74 int err;
75
76 err = xts_verify_key(tfm, key, keylen);
77 if (err)
78 return err;
79
80
81 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
82 if (err)
83 return err;
84
85
86 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
87 flags);
88}
89
90static const struct common_glue_ctx twofish_enc = {
91 .num_funcs = 3,
92 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
93
94 .funcs = { {
95 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
96 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
97 }, {
98 .num_blocks = 3,
99 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
100 }, {
101 .num_blocks = 1,
102 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
103 } }
104};
105
106static const struct common_glue_ctx twofish_ctr = {
107 .num_funcs = 3,
108 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
109
110 .funcs = { {
111 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
112 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
113 }, {
114 .num_blocks = 3,
115 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
116 }, {
117 .num_blocks = 1,
118 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
119 } }
120};
121
122static const struct common_glue_ctx twofish_enc_xts = {
123 .num_funcs = 2,
124 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
125
126 .funcs = { {
127 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
128 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
129 }, {
130 .num_blocks = 1,
131 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
132 } }
133};
134
135static const struct common_glue_ctx twofish_dec = {
136 .num_funcs = 3,
137 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
138
139 .funcs = { {
140 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
141 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
142 }, {
143 .num_blocks = 3,
144 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
145 }, {
146 .num_blocks = 1,
147 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
148 } }
149};
150
151static const struct common_glue_ctx twofish_dec_cbc = {
152 .num_funcs = 3,
153 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
154
155 .funcs = { {
156 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
157 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
158 }, {
159 .num_blocks = 3,
160 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
161 }, {
162 .num_blocks = 1,
163 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
164 } }
165};
166
167static const struct common_glue_ctx twofish_dec_xts = {
168 .num_funcs = 2,
169 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
170
171 .funcs = { {
172 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
173 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
174 }, {
175 .num_blocks = 1,
176 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
177 } }
178};
179
180static int ecb_encrypt(struct skcipher_request *req)
181{
182 return glue_ecb_req_128bit(&twofish_enc, req);
183}
184
185static int ecb_decrypt(struct skcipher_request *req)
186{
187 return glue_ecb_req_128bit(&twofish_dec, req);
188}
189
190static int cbc_encrypt(struct skcipher_request *req)
191{
192 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
193 req);
194}
195
196static int cbc_decrypt(struct skcipher_request *req)
197{
198 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
199}
200
201static int ctr_crypt(struct skcipher_request *req)
202{
203 return glue_ctr_req_128bit(&twofish_ctr, req);
204}
205
206static int xts_encrypt(struct skcipher_request *req)
207{
208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
209 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
210
211 return glue_xts_req_128bit(&twofish_enc_xts, req,
212 XTS_TWEAK_CAST(twofish_enc_blk),
213 &ctx->tweak_ctx, &ctx->crypt_ctx, false);
214}
215
216static int xts_decrypt(struct skcipher_request *req)
217{
218 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
219 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
220
221 return glue_xts_req_128bit(&twofish_dec_xts, req,
222 XTS_TWEAK_CAST(twofish_enc_blk),
223 &ctx->tweak_ctx, &ctx->crypt_ctx, true);
224}
225
226static struct skcipher_alg twofish_algs[] = {
227 {
228 .base.cra_name = "__ecb(twofish)",
229 .base.cra_driver_name = "__ecb-twofish-avx",
230 .base.cra_priority = 400,
231 .base.cra_flags = CRYPTO_ALG_INTERNAL,
232 .base.cra_blocksize = TF_BLOCK_SIZE,
233 .base.cra_ctxsize = sizeof(struct twofish_ctx),
234 .base.cra_module = THIS_MODULE,
235 .min_keysize = TF_MIN_KEY_SIZE,
236 .max_keysize = TF_MAX_KEY_SIZE,
237 .setkey = twofish_setkey_skcipher,
238 .encrypt = ecb_encrypt,
239 .decrypt = ecb_decrypt,
240 }, {
241 .base.cra_name = "__cbc(twofish)",
242 .base.cra_driver_name = "__cbc-twofish-avx",
243 .base.cra_priority = 400,
244 .base.cra_flags = CRYPTO_ALG_INTERNAL,
245 .base.cra_blocksize = TF_BLOCK_SIZE,
246 .base.cra_ctxsize = sizeof(struct twofish_ctx),
247 .base.cra_module = THIS_MODULE,
248 .min_keysize = TF_MIN_KEY_SIZE,
249 .max_keysize = TF_MAX_KEY_SIZE,
250 .ivsize = TF_BLOCK_SIZE,
251 .setkey = twofish_setkey_skcipher,
252 .encrypt = cbc_encrypt,
253 .decrypt = cbc_decrypt,
254 }, {
255 .base.cra_name = "__ctr(twofish)",
256 .base.cra_driver_name = "__ctr-twofish-avx",
257 .base.cra_priority = 400,
258 .base.cra_flags = CRYPTO_ALG_INTERNAL,
259 .base.cra_blocksize = 1,
260 .base.cra_ctxsize = sizeof(struct twofish_ctx),
261 .base.cra_module = THIS_MODULE,
262 .min_keysize = TF_MIN_KEY_SIZE,
263 .max_keysize = TF_MAX_KEY_SIZE,
264 .ivsize = TF_BLOCK_SIZE,
265 .chunksize = TF_BLOCK_SIZE,
266 .setkey = twofish_setkey_skcipher,
267 .encrypt = ctr_crypt,
268 .decrypt = ctr_crypt,
269 }, {
270 .base.cra_name = "__xts(twofish)",
271 .base.cra_driver_name = "__xts-twofish-avx",
272 .base.cra_priority = 400,
273 .base.cra_flags = CRYPTO_ALG_INTERNAL,
274 .base.cra_blocksize = TF_BLOCK_SIZE,
275 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx),
276 .base.cra_module = THIS_MODULE,
277 .min_keysize = 2 * TF_MIN_KEY_SIZE,
278 .max_keysize = 2 * TF_MAX_KEY_SIZE,
279 .ivsize = TF_BLOCK_SIZE,
280 .setkey = xts_twofish_setkey,
281 .encrypt = xts_encrypt,
282 .decrypt = xts_decrypt,
283 },
284};
285
286static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
287
288static int __init twofish_init(void)
289{
290 const char *feature_name;
291
292 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
293 pr_info("CPU feature '%s' is not supported.\n", feature_name);
294 return -ENODEV;
295 }
296
297 return simd_register_skciphers_compat(twofish_algs,
298 ARRAY_SIZE(twofish_algs),
299 twofish_simd_algs);
300}
301
302static void __exit twofish_exit(void)
303{
304 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
305 twofish_simd_algs);
306}
307
308module_init(twofish_init);
309module_exit(twofish_exit);
310
311MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
312MODULE_LICENSE("GPL");
313MODULE_ALIAS_CRYPTO("twofish");
314
