1
2
3
4
5
6
7
8
9
10
11
12
13#include <asm/crypto/camellia.h>
14#include <asm/crypto/glue_helper.h>
15#include <crypto/algapi.h>
16#include <crypto/internal/simd.h>
17#include <crypto/xts.h>
18#include <linux/crypto.h>
19#include <linux/err.h>
20#include <linux/module.h>
21#include <linux/types.h>
22
23#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
24
25
26asmlinkage void camellia_ecb_enc_16way(struct camellia_ctx *ctx, u8 *dst,
27 const u8 *src);
28EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way);
29
30asmlinkage void camellia_ecb_dec_16way(struct camellia_ctx *ctx, u8 *dst,
31 const u8 *src);
32EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way);
33
34asmlinkage void camellia_cbc_dec_16way(struct camellia_ctx *ctx, u8 *dst,
35 const u8 *src);
36EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way);
37
38asmlinkage void camellia_ctr_16way(struct camellia_ctx *ctx, u8 *dst,
39 const u8 *src, le128 *iv);
40EXPORT_SYMBOL_GPL(camellia_ctr_16way);
41
42asmlinkage void camellia_xts_enc_16way(struct camellia_ctx *ctx, u8 *dst,
43 const u8 *src, le128 *iv);
44EXPORT_SYMBOL_GPL(camellia_xts_enc_16way);
45
46asmlinkage void camellia_xts_dec_16way(struct camellia_ctx *ctx, u8 *dst,
47 const u8 *src, le128 *iv);
48EXPORT_SYMBOL_GPL(camellia_xts_dec_16way);
49
50void camellia_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
51{
52 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
53 GLUE_FUNC_CAST(camellia_enc_blk));
54}
55EXPORT_SYMBOL_GPL(camellia_xts_enc);
56
57void camellia_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
58{
59 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
60 GLUE_FUNC_CAST(camellia_dec_blk));
61}
62EXPORT_SYMBOL_GPL(camellia_xts_dec);
63
64static const struct common_glue_ctx camellia_enc = {
65 .num_funcs = 3,
66 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
67
68 .funcs = { {
69 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
70 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
71 }, {
72 .num_blocks = 2,
73 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
74 }, {
75 .num_blocks = 1,
76 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
77 } }
78};
79
80static const struct common_glue_ctx camellia_ctr = {
81 .num_funcs = 3,
82 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
83
84 .funcs = { {
85 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
86 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
87 }, {
88 .num_blocks = 2,
89 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
90 }, {
91 .num_blocks = 1,
92 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
93 } }
94};
95
96static const struct common_glue_ctx camellia_enc_xts = {
97 .num_funcs = 2,
98 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
99
100 .funcs = { {
101 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
102 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
103 }, {
104 .num_blocks = 1,
105 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
106 } }
107};
108
109static const struct common_glue_ctx camellia_dec = {
110 .num_funcs = 3,
111 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
112
113 .funcs = { {
114 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
115 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
116 }, {
117 .num_blocks = 2,
118 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
119 }, {
120 .num_blocks = 1,
121 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
122 } }
123};
124
125static const struct common_glue_ctx camellia_dec_cbc = {
126 .num_funcs = 3,
127 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
128
129 .funcs = { {
130 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
131 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
132 }, {
133 .num_blocks = 2,
134 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
135 }, {
136 .num_blocks = 1,
137 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
138 } }
139};
140
141static const struct common_glue_ctx camellia_dec_xts = {
142 .num_funcs = 2,
143 .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
144
145 .funcs = { {
146 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
147 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
148 }, {
149 .num_blocks = 1,
150 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
151 } }
152};
153
154static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
155 unsigned int keylen)
156{
157 return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen,
158 &tfm->base.crt_flags);
159}
160
161static int ecb_encrypt(struct skcipher_request *req)
162{
163 return glue_ecb_req_128bit(&camellia_enc, req);
164}
165
166static int ecb_decrypt(struct skcipher_request *req)
167{
168 return glue_ecb_req_128bit(&camellia_dec, req);
169}
170
171static int cbc_encrypt(struct skcipher_request *req)
172{
173 return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(camellia_enc_blk),
174 req);
175}
176
177static int cbc_decrypt(struct skcipher_request *req)
178{
179 return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
180}
181
182static int ctr_crypt(struct skcipher_request *req)
183{
184 return glue_ctr_req_128bit(&camellia_ctr, req);
185}
186
187int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
188 unsigned int keylen)
189{
190 struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
191 u32 *flags = &tfm->base.crt_flags;
192 int err;
193
194 err = xts_verify_key(tfm, key, keylen);
195 if (err)
196 return err;
197
198
199 err = __camellia_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
200 if (err)
201 return err;
202
203
204 return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
205 flags);
206}
207EXPORT_SYMBOL_GPL(xts_camellia_setkey);
208
209static int xts_encrypt(struct skcipher_request *req)
210{
211 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
212 struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
213
214 return glue_xts_req_128bit(&camellia_enc_xts, req,
215 XTS_TWEAK_CAST(camellia_enc_blk),
216 &ctx->tweak_ctx, &ctx->crypt_ctx);
217}
218
219static int xts_decrypt(struct skcipher_request *req)
220{
221 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
222 struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
223
224 return glue_xts_req_128bit(&camellia_dec_xts, req,
225 XTS_TWEAK_CAST(camellia_enc_blk),
226 &ctx->tweak_ctx, &ctx->crypt_ctx);
227}
228
229static struct skcipher_alg camellia_algs[] = {
230 {
231 .base.cra_name = "__ecb(camellia)",
232 .base.cra_driver_name = "__ecb-camellia-aesni",
233 .base.cra_priority = 400,
234 .base.cra_flags = CRYPTO_ALG_INTERNAL,
235 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE,
236 .base.cra_ctxsize = sizeof(struct camellia_ctx),
237 .base.cra_module = THIS_MODULE,
238 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
239 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
240 .setkey = camellia_setkey,
241 .encrypt = ecb_encrypt,
242 .decrypt = ecb_decrypt,
243 }, {
244 .base.cra_name = "__cbc(camellia)",
245 .base.cra_driver_name = "__cbc-camellia-aesni",
246 .base.cra_priority = 400,
247 .base.cra_flags = CRYPTO_ALG_INTERNAL,
248 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE,
249 .base.cra_ctxsize = sizeof(struct camellia_ctx),
250 .base.cra_module = THIS_MODULE,
251 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
252 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
253 .ivsize = CAMELLIA_BLOCK_SIZE,
254 .setkey = camellia_setkey,
255 .encrypt = cbc_encrypt,
256 .decrypt = cbc_decrypt,
257 }, {
258 .base.cra_name = "__ctr(camellia)",
259 .base.cra_driver_name = "__ctr-camellia-aesni",
260 .base.cra_priority = 400,
261 .base.cra_flags = CRYPTO_ALG_INTERNAL,
262 .base.cra_blocksize = 1,
263 .base.cra_ctxsize = sizeof(struct camellia_ctx),
264 .base.cra_module = THIS_MODULE,
265 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
266 .max_keysize = CAMELLIA_MAX_KEY_SIZE,
267 .ivsize = CAMELLIA_BLOCK_SIZE,
268 .chunksize = CAMELLIA_BLOCK_SIZE,
269 .setkey = camellia_setkey,
270 .encrypt = ctr_crypt,
271 .decrypt = ctr_crypt,
272 }, {
273 .base.cra_name = "__xts(camellia)",
274 .base.cra_driver_name = "__xts-camellia-aesni",
275 .base.cra_priority = 400,
276 .base.cra_flags = CRYPTO_ALG_INTERNAL,
277 .base.cra_blocksize = CAMELLIA_BLOCK_SIZE,
278 .base.cra_ctxsize = sizeof(struct camellia_xts_ctx),
279 .base.cra_module = THIS_MODULE,
280 .min_keysize = 2 * CAMELLIA_MIN_KEY_SIZE,
281 .max_keysize = 2 * CAMELLIA_MAX_KEY_SIZE,
282 .ivsize = CAMELLIA_BLOCK_SIZE,
283 .setkey = xts_camellia_setkey,
284 .encrypt = xts_encrypt,
285 .decrypt = xts_decrypt,
286 },
287};
288
289static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
290
291static int __init camellia_aesni_init(void)
292{
293 const char *feature_name;
294
295 if (!boot_cpu_has(X86_FEATURE_AVX) ||
296 !boot_cpu_has(X86_FEATURE_AES) ||
297 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
298 pr_info("AVX or AES-NI instructions are not detected.\n");
299 return -ENODEV;
300 }
301
302 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
303 &feature_name)) {
304 pr_info("CPU feature '%s' is not supported.\n", feature_name);
305 return -ENODEV;
306 }
307
308 return simd_register_skciphers_compat(camellia_algs,
309 ARRAY_SIZE(camellia_algs),
310 camellia_simd_algs);
311}
312
313static void __exit camellia_aesni_fini(void)
314{
315 simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
316 camellia_simd_algs);
317}
318
319module_init(camellia_aesni_init);
320module_exit(camellia_aesni_fini);
321
322MODULE_LICENSE("GPL");
323MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized");
324MODULE_ALIAS_CRYPTO("camellia");
325MODULE_ALIAS_CRYPTO("camellia-asm");
326
