1
2
3
4
5
6
7
8
9
10#include <crypto/algapi.h>
11#include <crypto/internal/chacha.h>
12#include <crypto/internal/simd.h>
13#include <crypto/internal/skcipher.h>
14#include <linux/jump_label.h>
15#include <linux/kernel.h>
16#include <linux/module.h>
17
18#include <asm/cputype.h>
19#include <asm/hwcap.h>
20#include <asm/neon.h>
21#include <asm/simd.h>
22
23asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
24 int nrounds);
25asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
26 int nrounds, unsigned int nbytes);
27asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds);
28asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
29
30asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
31 const u32 *state, int nrounds);
32
33static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);
34
35static inline bool neon_usable(void)
36{
37 return static_branch_likely(&use_neon) && crypto_simd_usable();
38}
39
40static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
41 unsigned int bytes, int nrounds)
42{
43 u8 buf[CHACHA_BLOCK_SIZE];
44
45 while (bytes > CHACHA_BLOCK_SIZE) {
46 unsigned int l = min(bytes, CHACHA_BLOCK_SIZE * 4U);
47
48 chacha_4block_xor_neon(state, dst, src, nrounds, l);
49 bytes -= l;
50 src += l;
51 dst += l;
52 state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE);
53 }
54 if (bytes) {
55 const u8 *s = src;
56 u8 *d = dst;
57
58 if (bytes != CHACHA_BLOCK_SIZE)
59 s = d = memcpy(buf, src, bytes);
60 chacha_block_xor_neon(state, d, s, nrounds);
61 if (d != dst)
62 memcpy(dst, buf, bytes);
63 state[12]++;
64 }
65}
66
67void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
68{
69 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
70 hchacha_block_arm(state, stream, nrounds);
71 } else {
72 kernel_neon_begin();
73 hchacha_block_neon(state, stream, nrounds);
74 kernel_neon_end();
75 }
76}
77EXPORT_SYMBOL(hchacha_block_arch);
78
79void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
80{
81 chacha_init_generic(state, key, iv);
82}
83EXPORT_SYMBOL(chacha_init_arch);
84
85void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
86 int nrounds)
87{
88 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||
89 bytes <= CHACHA_BLOCK_SIZE) {
90 chacha_doarm(dst, src, bytes, state, nrounds);
91 state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
92 return;
93 }
94
95 do {
96 unsigned int todo = min_t(unsigned int, bytes, SZ_4K);
97
98 kernel_neon_begin();
99 chacha_doneon(state, dst, src, todo, nrounds);
100 kernel_neon_end();
101
102 bytes -= todo;
103 src += todo;
104 dst += todo;
105 } while (bytes);
106}
107EXPORT_SYMBOL(chacha_crypt_arch);
108
109static int chacha_stream_xor(struct skcipher_request *req,
110 const struct chacha_ctx *ctx, const u8 *iv,
111 bool neon)
112{
113 struct skcipher_walk walk;
114 u32 state[16];
115 int err;
116
117 err = skcipher_walk_virt(&walk, req, false);
118
119 chacha_init_generic(state, ctx->key, iv);
120
121 while (walk.nbytes > 0) {
122 unsigned int nbytes = walk.nbytes;
123
124 if (nbytes < walk.total)
125 nbytes = round_down(nbytes, walk.stride);
126
127 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
128 chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr,
129 nbytes, state, ctx->nrounds);
130 state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE);
131 } else {
132 kernel_neon_begin();
133 chacha_doneon(state, walk.dst.virt.addr,
134 walk.src.virt.addr, nbytes, ctx->nrounds);
135 kernel_neon_end();
136 }
137 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
138 }
139
140 return err;
141}
142
143static int do_chacha(struct skcipher_request *req, bool neon)
144{
145 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
146 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
147
148 return chacha_stream_xor(req, ctx, req->iv, neon);
149}
150
151static int chacha_arm(struct skcipher_request *req)
152{
153 return do_chacha(req, false);
154}
155
156static int chacha_neon(struct skcipher_request *req)
157{
158 return do_chacha(req, neon_usable());
159}
160
161static int do_xchacha(struct skcipher_request *req, bool neon)
162{
163 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
164 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
165 struct chacha_ctx subctx;
166 u32 state[16];
167 u8 real_iv[16];
168
169 chacha_init_generic(state, ctx->key, req->iv);
170
171 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
172 hchacha_block_arm(state, subctx.key, ctx->nrounds);
173 } else {
174 kernel_neon_begin();
175 hchacha_block_neon(state, subctx.key, ctx->nrounds);
176 kernel_neon_end();
177 }
178 subctx.nrounds = ctx->nrounds;
179
180 memcpy(&real_iv[0], req->iv + 24, 8);
181 memcpy(&real_iv[8], req->iv + 16, 8);
182 return chacha_stream_xor(req, &subctx, real_iv, neon);
183}
184
185static int xchacha_arm(struct skcipher_request *req)
186{
187 return do_xchacha(req, false);
188}
189
190static int xchacha_neon(struct skcipher_request *req)
191{
192 return do_xchacha(req, neon_usable());
193}
194
195static struct skcipher_alg arm_algs[] = {
196 {
197 .base.cra_name = "chacha20",
198 .base.cra_driver_name = "chacha20-arm",
199 .base.cra_priority = 200,
200 .base.cra_blocksize = 1,
201 .base.cra_ctxsize = sizeof(struct chacha_ctx),
202 .base.cra_module = THIS_MODULE,
203
204 .min_keysize = CHACHA_KEY_SIZE,
205 .max_keysize = CHACHA_KEY_SIZE,
206 .ivsize = CHACHA_IV_SIZE,
207 .chunksize = CHACHA_BLOCK_SIZE,
208 .setkey = chacha20_setkey,
209 .encrypt = chacha_arm,
210 .decrypt = chacha_arm,
211 }, {
212 .base.cra_name = "xchacha20",
213 .base.cra_driver_name = "xchacha20-arm",
214 .base.cra_priority = 200,
215 .base.cra_blocksize = 1,
216 .base.cra_ctxsize = sizeof(struct chacha_ctx),
217 .base.cra_module = THIS_MODULE,
218
219 .min_keysize = CHACHA_KEY_SIZE,
220 .max_keysize = CHACHA_KEY_SIZE,
221 .ivsize = XCHACHA_IV_SIZE,
222 .chunksize = CHACHA_BLOCK_SIZE,
223 .setkey = chacha20_setkey,
224 .encrypt = xchacha_arm,
225 .decrypt = xchacha_arm,
226 }, {
227 .base.cra_name = "xchacha12",
228 .base.cra_driver_name = "xchacha12-arm",
229 .base.cra_priority = 200,
230 .base.cra_blocksize = 1,
231 .base.cra_ctxsize = sizeof(struct chacha_ctx),
232 .base.cra_module = THIS_MODULE,
233
234 .min_keysize = CHACHA_KEY_SIZE,
235 .max_keysize = CHACHA_KEY_SIZE,
236 .ivsize = XCHACHA_IV_SIZE,
237 .chunksize = CHACHA_BLOCK_SIZE,
238 .setkey = chacha12_setkey,
239 .encrypt = xchacha_arm,
240 .decrypt = xchacha_arm,
241 },
242};
243
244static struct skcipher_alg neon_algs[] = {
245 {
246 .base.cra_name = "chacha20",
247 .base.cra_driver_name = "chacha20-neon",
248 .base.cra_priority = 300,
249 .base.cra_blocksize = 1,
250 .base.cra_ctxsize = sizeof(struct chacha_ctx),
251 .base.cra_module = THIS_MODULE,
252
253 .min_keysize = CHACHA_KEY_SIZE,
254 .max_keysize = CHACHA_KEY_SIZE,
255 .ivsize = CHACHA_IV_SIZE,
256 .chunksize = CHACHA_BLOCK_SIZE,
257 .walksize = 4 * CHACHA_BLOCK_SIZE,
258 .setkey = chacha20_setkey,
259 .encrypt = chacha_neon,
260 .decrypt = chacha_neon,
261 }, {
262 .base.cra_name = "xchacha20",
263 .base.cra_driver_name = "xchacha20-neon",
264 .base.cra_priority = 300,
265 .base.cra_blocksize = 1,
266 .base.cra_ctxsize = sizeof(struct chacha_ctx),
267 .base.cra_module = THIS_MODULE,
268
269 .min_keysize = CHACHA_KEY_SIZE,
270 .max_keysize = CHACHA_KEY_SIZE,
271 .ivsize = XCHACHA_IV_SIZE,
272 .chunksize = CHACHA_BLOCK_SIZE,
273 .walksize = 4 * CHACHA_BLOCK_SIZE,
274 .setkey = chacha20_setkey,
275 .encrypt = xchacha_neon,
276 .decrypt = xchacha_neon,
277 }, {
278 .base.cra_name = "xchacha12",
279 .base.cra_driver_name = "xchacha12-neon",
280 .base.cra_priority = 300,
281 .base.cra_blocksize = 1,
282 .base.cra_ctxsize = sizeof(struct chacha_ctx),
283 .base.cra_module = THIS_MODULE,
284
285 .min_keysize = CHACHA_KEY_SIZE,
286 .max_keysize = CHACHA_KEY_SIZE,
287 .ivsize = XCHACHA_IV_SIZE,
288 .chunksize = CHACHA_BLOCK_SIZE,
289 .walksize = 4 * CHACHA_BLOCK_SIZE,
290 .setkey = chacha12_setkey,
291 .encrypt = xchacha_neon,
292 .decrypt = xchacha_neon,
293 }
294};
295
296static int __init chacha_simd_mod_init(void)
297{
298 int err = 0;
299
300 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
301 err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
302 if (err)
303 return err;
304 }
305
306 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
307 int i;
308
309 switch (read_cpuid_part()) {
310 case ARM_CPU_PART_CORTEX_A7:
311 case ARM_CPU_PART_CORTEX_A5:
312
313
314
315
316
317 for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
318 neon_algs[i].base.cra_priority = 0;
319 break;
320 default:
321 static_branch_enable(&use_neon);
322 }
323
324 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
325 err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
326 if (err)
327 crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
328 }
329 }
330 return err;
331}
332
333static void __exit chacha_simd_mod_fini(void)
334{
335 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
336 crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
337 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON))
338 crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
339 }
340}
341
342module_init(chacha_simd_mod_init);
343module_exit(chacha_simd_mod_fini);
344
345MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)");
346MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
347MODULE_LICENSE("GPL v2");
348MODULE_ALIAS_CRYPTO("chacha20");
349MODULE_ALIAS_CRYPTO("chacha20-arm");
350MODULE_ALIAS_CRYPTO("xchacha20");
351MODULE_ALIAS_CRYPTO("xchacha20-arm");
352MODULE_ALIAS_CRYPTO("xchacha12");
353MODULE_ALIAS_CRYPTO("xchacha12-arm");
354#ifdef CONFIG_KERNEL_MODE_NEON
355MODULE_ALIAS_CRYPTO("chacha20-neon");
356MODULE_ALIAS_CRYPTO("xchacha20-neon");
357MODULE_ALIAS_CRYPTO("xchacha12-neon");
358#endif
359
