1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <asm/crypto/glue_helper.h>
24#include <asm/crypto/twofish.h>
25#include <crypto/algapi.h>
26#include <crypto/b128ops.h>
27#include <crypto/internal/skcipher.h>
28#include <crypto/twofish.h>
29#include <linux/crypto.h>
30#include <linux/init.h>
31#include <linux/module.h>
32#include <linux/types.h>
33
34EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
35EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
36
37static int twofish_setkey_skcipher(struct crypto_skcipher *tfm,
38 const u8 *key, unsigned int keylen)
39{
40 return twofish_setkey(&tfm->base, key, keylen);
41}
42
43static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
44 const u8 *src)
45{
46 __twofish_enc_blk_3way(ctx, dst, src, false);
47}
48
49static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
50 const u8 *src)
51{
52 __twofish_enc_blk_3way(ctx, dst, src, true);
53}
54
55void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
56{
57 u128 ivs[2];
58
59 ivs[0] = src[0];
60 ivs[1] = src[1];
61
62 twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
63
64 u128_xor(&dst[1], &dst[1], &ivs[0]);
65 u128_xor(&dst[2], &dst[2], &ivs[1]);
66}
67EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
68
69void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
70{
71 be128 ctrblk;
72
73 if (dst != src)
74 *dst = *src;
75
76 le128_to_be128(&ctrblk, iv);
77 le128_inc(iv);
78
79 twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
80 u128_xor(dst, dst, (u128 *)&ctrblk);
81}
82EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);
83
84void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
85 le128 *iv)
86{
87 be128 ctrblks[3];
88
89 if (dst != src) {
90 dst[0] = src[0];
91 dst[1] = src[1];
92 dst[2] = src[2];
93 }
94
95 le128_to_be128(&ctrblks[0], iv);
96 le128_inc(iv);
97 le128_to_be128(&ctrblks[1], iv);
98 le128_inc(iv);
99 le128_to_be128(&ctrblks[2], iv);
100 le128_inc(iv);
101
102 twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
103}
104EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);
105
106static const struct common_glue_ctx twofish_enc = {
107 .num_funcs = 2,
108 .fpu_blocks_limit = -1,
109
110 .funcs = { {
111 .num_blocks = 3,
112 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
113 }, {
114 .num_blocks = 1,
115 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
116 } }
117};
118
119static const struct common_glue_ctx twofish_ctr = {
120 .num_funcs = 2,
121 .fpu_blocks_limit = -1,
122
123 .funcs = { {
124 .num_blocks = 3,
125 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) }
126 }, {
127 .num_blocks = 1,
128 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) }
129 } }
130};
131
132static const struct common_glue_ctx twofish_dec = {
133 .num_funcs = 2,
134 .fpu_blocks_limit = -1,
135
136 .funcs = { {
137 .num_blocks = 3,
138 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
139 }, {
140 .num_blocks = 1,
141 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
142 } }
143};
144
145static const struct common_glue_ctx twofish_dec_cbc = {
146 .num_funcs = 2,
147 .fpu_blocks_limit = -1,
148
149 .funcs = { {
150 .num_blocks = 3,
151 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
152 }, {
153 .num_blocks = 1,
154 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
155 } }
156};
157
158static int ecb_encrypt(struct skcipher_request *req)
159{
160 return glue_ecb_req_128bit(&twofish_enc, req);
161}
162
163static int ecb_decrypt(struct skcipher_request *req)
164{
165 return glue_ecb_req_128bit(&twofish_dec, req);
166}
167
168static int cbc_encrypt(struct skcipher_request *req)
169{
170 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(twofish_enc_blk),
171 req);
172}
173
174static int cbc_decrypt(struct skcipher_request *req)
175{
176 return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req);
177}
178
179static int ctr_crypt(struct skcipher_request *req)
180{
181 return glue_ctr_req_128bit(&twofish_ctr, req);
182}
183
184static struct skcipher_alg tf_skciphers[] = {
185 {
186 .base.cra_name = "ecb(twofish)",
187 .base.cra_driver_name = "ecb-twofish-3way",
188 .base.cra_priority = 300,
189 .base.cra_blocksize = TF_BLOCK_SIZE,
190 .base.cra_ctxsize = sizeof(struct twofish_ctx),
191 .base.cra_module = THIS_MODULE,
192 .min_keysize = TF_MIN_KEY_SIZE,
193 .max_keysize = TF_MAX_KEY_SIZE,
194 .setkey = twofish_setkey_skcipher,
195 .encrypt = ecb_encrypt,
196 .decrypt = ecb_decrypt,
197 }, {
198 .base.cra_name = "cbc(twofish)",
199 .base.cra_driver_name = "cbc-twofish-3way",
200 .base.cra_priority = 300,
201 .base.cra_blocksize = TF_BLOCK_SIZE,
202 .base.cra_ctxsize = sizeof(struct twofish_ctx),
203 .base.cra_module = THIS_MODULE,
204 .min_keysize = TF_MIN_KEY_SIZE,
205 .max_keysize = TF_MAX_KEY_SIZE,
206 .ivsize = TF_BLOCK_SIZE,
207 .setkey = twofish_setkey_skcipher,
208 .encrypt = cbc_encrypt,
209 .decrypt = cbc_decrypt,
210 }, {
211 .base.cra_name = "ctr(twofish)",
212 .base.cra_driver_name = "ctr-twofish-3way",
213 .base.cra_priority = 300,
214 .base.cra_blocksize = 1,
215 .base.cra_ctxsize = sizeof(struct twofish_ctx),
216 .base.cra_module = THIS_MODULE,
217 .min_keysize = TF_MIN_KEY_SIZE,
218 .max_keysize = TF_MAX_KEY_SIZE,
219 .ivsize = TF_BLOCK_SIZE,
220 .chunksize = TF_BLOCK_SIZE,
221 .setkey = twofish_setkey_skcipher,
222 .encrypt = ctr_crypt,
223 .decrypt = ctr_crypt,
224 },
225};
226
227static bool is_blacklisted_cpu(void)
228{
229 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
230 return false;
231
232 if (boot_cpu_data.x86 == 0x06 &&
233 (boot_cpu_data.x86_model == 0x1c ||
234 boot_cpu_data.x86_model == 0x26 ||
235 boot_cpu_data.x86_model == 0x36)) {
236
237
238
239
240
241
242
243
244
245 return true;
246 }
247
248 if (boot_cpu_data.x86 == 0x0f) {
249
250
251
252
253
254
255 return true;
256 }
257
258 return false;
259}
260
261static int force;
262module_param(force, int, 0);
263MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
264
265static int __init init(void)
266{
267 if (!force && is_blacklisted_cpu()) {
268 printk(KERN_INFO
269 "twofish-x86_64-3way: performance on this CPU "
270 "would be suboptimal: disabling "
271 "twofish-x86_64-3way.\n");
272 return -ENODEV;
273 }
274
275 return crypto_register_skciphers(tf_skciphers,
276 ARRAY_SIZE(tf_skciphers));
277}
278
279static void __exit fini(void)
280{
281 crypto_unregister_skciphers(tf_skciphers, ARRAY_SIZE(tf_skciphers));
282}
283
284module_init(init);
285module_exit(fini);
286
287MODULE_LICENSE("GPL");
288MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
289MODULE_ALIAS_CRYPTO("twofish");
290MODULE_ALIAS_CRYPTO("twofish-asm");
291