1
2
3
4
5
6
7
8
9
10
11#include <linux/module.h>
12#include <linux/types.h>
13#include <linux/crypto.h>
14#include <linux/err.h>
15#include <crypto/algapi.h>
16#include <crypto/internal/simd.h>
17#include <crypto/twofish.h>
18#include <crypto/xts.h>
19#include <asm/crypto/glue_helper.h>
20#include <asm/crypto/twofish.h>
21
22#define TWOFISH_PARALLEL_BLOCKS 8
23
24
25asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src);
26asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src);
27
28asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src);
29asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src,
30 le128 *iv);
31
32asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src,
33 le128 *iv);
34asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src,
35 le128 *iv);
36
37static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
38 const u8 *key, unsigned int keylen)
39{
40 return twofish_setkey(&tfm->base, key, keylen);
41}
42
43static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src)
44{
45 __twofish_enc_blk_3way(ctx, dst, src, false);
46}
47
48static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
49{
50 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk);
51}
52
53static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
54{
55 glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk);
56}
57
58struct twofish_xts_ctx {
59 struct twofish_ctx tweak_ctx;
60 struct twofish_ctx crypt_ctx;
61};
62
63static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key,
64 unsigned int keylen)
65{
66 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
67 int err;
68
69 err = xts_verify_key(tfm, key, keylen);
70 if (err)
71 return err;
72
73
74 err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2);
75 if (err)
76 return err;
77
78
79 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
80}
81
82static const struct common_glue_ctx twofish_enc = {
83 .num_funcs = 3,
84 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
85
86 .funcs = { {
87 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
88 .fn_u = { .ecb = twofish_ecb_enc_8way }
89 }, {
90 .num_blocks = 3,
91 .fn_u = { .ecb = twofish_enc_blk_3way }
92 }, {
93 .num_blocks = 1,
94 .fn_u = { .ecb = twofish_enc_blk }
95 } }
96};
97
98static const struct common_glue_ctx twofish_ctr = {
99 .num_funcs = 3,
100 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
101
102 .funcs = { {
103 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
104 .fn_u = { .ctr = twofish_ctr_8way }
105 }, {
106 .num_blocks = 3,
107 .fn_u = { .ctr = twofish_enc_blk_ctr_3way }
108 }, {
109 .num_blocks = 1,
110 .fn_u = { .ctr = twofish_enc_blk_ctr }
111 } }
112};
113
114static const struct common_glue_ctx twofish_enc_xts = {
115 .num_funcs = 2,
116 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
117
118 .funcs = { {
119 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
120 .fn_u = { .xts = twofish_xts_enc_8way }
121 }, {
122 .num_blocks = 1,
123 .fn_u = { .xts = twofish_xts_enc }
124 } }
125};
126
127static const struct common_glue_ctx twofish_dec = {
128 .num_funcs = 3,
129 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
130
131 .funcs = { {
132 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
133 .fn_u = { .ecb = twofish_ecb_dec_8way }
134 }, {
135 .num_blocks = 3,
136 .fn_u = { .ecb = twofish_dec_blk_3way }
137 }, {
138 .num_blocks = 1,
139 .fn_u = { .ecb = twofish_dec_blk }
140 } }
141};
142
143static const struct common_glue_ctx twofish_dec_cbc = {
144 .num_funcs = 3,
145 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
146
147 .funcs = { {
148 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
149 .fn_u = { .cbc = twofish_cbc_dec_8way }
150 }, {
151 .num_blocks = 3,
152 .fn_u = { .cbc = twofish_dec_blk_cbc_3way }
153 }, {
154 .num_blocks = 1,
155 .fn_u = { .cbc = twofish_dec_blk }
156 } }
157};
158
159static const struct common_glue_ctx twofish_dec_xts = {
160 .num_funcs = 2,
161 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
162
163 .funcs = { {
164 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
165 .fn_u = { .xts = twofish_xts_dec_8way }
166 }, {
167 .num_blocks = 1,
168 .fn_u = { .xts = twofish_xts_dec }
169 } }
170};
171
172static int ecb_encrypt(struct skcipher_request *req)
173{
174 return glue_ecb_req_128bit(&twofish_enc, req);
175}
176
177static int ecb_decrypt(struct skcipher_request *req)
178{
179 return glue_ecb_req_128bit(&twofish_dec, req);
180}
181
182static int cbc_encrypt(struct skcipher_request *req)
183{
184 return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req);
185}
186
187static int cbc_decrypt(struct skcipher_request *req)
188{
189 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
190}
191
192static int ctr_crypt(struct skcipher_request *req)
193{
194 return glue_ctr_req_128bit(&twofish_ctr, req);
195}
196
197static int xts_encrypt(struct skcipher_request *req)
198{
199 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
200 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
201
202 return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk,
203 &ctx->tweak_ctx, &ctx->crypt_ctx, false);
204}
205
206static int xts_decrypt(struct skcipher_request *req)
207{
208 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
209 struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
210
211 return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk,
212 &ctx->tweak_ctx, &ctx->crypt_ctx, true);
213}
214
215static struct skcipher_alg twofish_algs[] = {
216 {
217 .base.cra_name = "__ecb(twofish)",
218 .base.cra_driver_name = "__ecb-twofish-avx",
219 .base.cra_priority = 400,
220 .base.cra_flags = CRYPTO_ALG_INTERNAL,
221 .base.cra_blocksize = TF_BLOCK_SIZE,
222 .base.cra_ctxsize = sizeof(struct twofish_ctx),
223 .base.cra_module = THIS_MODULE,
224 .min_keysize = TF_MIN_KEY_SIZE,
225 .max_keysize = TF_MAX_KEY_SIZE,
226 .setkey = twofish_setkey_skcipher,
227 .encrypt = ecb_encrypt,
228 .decrypt = ecb_decrypt,
229 }, {
230 .base.cra_name = "__cbc(twofish)",
231 .base.cra_driver_name = "__cbc-twofish-avx",
232 .base.cra_priority = 400,
233 .base.cra_flags = CRYPTO_ALG_INTERNAL,
234 .base.cra_blocksize = TF_BLOCK_SIZE,
235 .base.cra_ctxsize = sizeof(struct twofish_ctx),
236 .base.cra_module = THIS_MODULE,
237 .min_keysize = TF_MIN_KEY_SIZE,
238 .max_keysize = TF_MAX_KEY_SIZE,
239 .ivsize = TF_BLOCK_SIZE,
240 .setkey = twofish_setkey_skcipher,
241 .encrypt = cbc_encrypt,
242 .decrypt = cbc_decrypt,
243 }, {
244 .base.cra_name = "__ctr(twofish)",
245 .base.cra_driver_name = "__ctr-twofish-avx",
246 .base.cra_priority = 400,
247 .base.cra_flags = CRYPTO_ALG_INTERNAL,
248 .base.cra_blocksize = 1,
249 .base.cra_ctxsize = sizeof(struct twofish_ctx),
250 .base.cra_module = THIS_MODULE,
251 .min_keysize = TF_MIN_KEY_SIZE,
252 .max_keysize = TF_MAX_KEY_SIZE,
253 .ivsize = TF_BLOCK_SIZE,
254 .chunksize = TF_BLOCK_SIZE,
255 .setkey = twofish_setkey_skcipher,
256 .encrypt = ctr_crypt,
257 .decrypt = ctr_crypt,
258 }, {
259 .base.cra_name = "__xts(twofish)",
260 .base.cra_driver_name = "__xts-twofish-avx",
261 .base.cra_priority = 400,
262 .base.cra_flags = CRYPTO_ALG_INTERNAL,
263 .base.cra_blocksize = TF_BLOCK_SIZE,
264 .base.cra_ctxsize = sizeof(struct twofish_xts_ctx),
265 .base.cra_module = THIS_MODULE,
266 .min_keysize = 2 * TF_MIN_KEY_SIZE,
267 .max_keysize = 2 * TF_MAX_KEY_SIZE,
268 .ivsize = TF_BLOCK_SIZE,
269 .setkey = xts_twofish_setkey,
270 .encrypt = xts_encrypt,
271 .decrypt = xts_decrypt,
272 },
273};
274
275static struct simd_skcipher_alg *twofish_simd_algs[ARRAY_SIZE(twofish_algs)];
276
277static int __init twofish_init(void)
278{
279 const char *feature_name;
280
281 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
282 pr_info("CPU feature '%s' is not supported.\n", feature_name);
283 return -ENODEV;
284 }
285
286 return simd_register_skciphers_compat(twofish_algs,
287 ARRAY_SIZE(twofish_algs),
288 twofish_simd_algs);
289}
290
291static void __exit twofish_exit(void)
292{
293 simd_unregister_skciphers(twofish_algs, ARRAY_SIZE(twofish_algs),
294 twofish_simd_algs);
295}
296
297module_init(twofish_init);
298module_exit(twofish_exit);
299
300MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
301MODULE_LICENSE("GPL");
302MODULE_ALIAS_CRYPTO("twofish");
303
