1
2
3
4
5
6
7
8
9
10#include <crypto/algapi.h>
11#include <crypto/internal/chacha.h>
12#include <crypto/internal/simd.h>
13#include <crypto/internal/skcipher.h>
14#include <linux/jump_label.h>
15#include <linux/kernel.h>
16#include <linux/module.h>
17
18#include <asm/cputype.h>
19#include <asm/hwcap.h>
20#include <asm/neon.h>
21#include <asm/simd.h>
22
23asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
24 int nrounds);
25asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
26 int nrounds);
27asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds);
28asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
29
30asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
31 const u32 *state, int nrounds);
32
33static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);
34
35static inline bool neon_usable(void)
36{
37 return static_branch_likely(&use_neon) && crypto_simd_usable();
38}
39
40static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
41 unsigned int bytes, int nrounds)
42{
43 u8 buf[CHACHA_BLOCK_SIZE];
44
45 while (bytes >= CHACHA_BLOCK_SIZE * 4) {
46 chacha_4block_xor_neon(state, dst, src, nrounds);
47 bytes -= CHACHA_BLOCK_SIZE * 4;
48 src += CHACHA_BLOCK_SIZE * 4;
49 dst += CHACHA_BLOCK_SIZE * 4;
50 state[12] += 4;
51 }
52 while (bytes >= CHACHA_BLOCK_SIZE) {
53 chacha_block_xor_neon(state, dst, src, nrounds);
54 bytes -= CHACHA_BLOCK_SIZE;
55 src += CHACHA_BLOCK_SIZE;
56 dst += CHACHA_BLOCK_SIZE;
57 state[12]++;
58 }
59 if (bytes) {
60 memcpy(buf, src, bytes);
61 chacha_block_xor_neon(state, buf, buf, nrounds);
62 memcpy(dst, buf, bytes);
63 }
64}
65
66void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
67{
68 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
69 hchacha_block_arm(state, stream, nrounds);
70 } else {
71 kernel_neon_begin();
72 hchacha_block_neon(state, stream, nrounds);
73 kernel_neon_end();
74 }
75}
76EXPORT_SYMBOL(hchacha_block_arch);
77
78void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
79{
80 chacha_init_generic(state, key, iv);
81}
82EXPORT_SYMBOL(chacha_init_arch);
83
84void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
85 int nrounds)
86{
87 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||
88 bytes <= CHACHA_BLOCK_SIZE) {
89 chacha_doarm(dst, src, bytes, state, nrounds);
90 state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
91 return;
92 }
93
94 kernel_neon_begin();
95 chacha_doneon(state, dst, src, bytes, nrounds);
96 kernel_neon_end();
97}
98EXPORT_SYMBOL(chacha_crypt_arch);
99
100static int chacha_stream_xor(struct skcipher_request *req,
101 const struct chacha_ctx *ctx, const u8 *iv,
102 bool neon)
103{
104 struct skcipher_walk walk;
105 u32 state[16];
106 int err;
107
108 err = skcipher_walk_virt(&walk, req, false);
109
110 chacha_init_generic(state, ctx->key, iv);
111
112 while (walk.nbytes > 0) {
113 unsigned int nbytes = walk.nbytes;
114
115 if (nbytes < walk.total)
116 nbytes = round_down(nbytes, walk.stride);
117
118 if (!neon) {
119 chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr,
120 nbytes, state, ctx->nrounds);
121 state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE);
122 } else {
123 kernel_neon_begin();
124 chacha_doneon(state, walk.dst.virt.addr,
125 walk.src.virt.addr, nbytes, ctx->nrounds);
126 kernel_neon_end();
127 }
128 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
129 }
130
131 return err;
132}
133
134static int do_chacha(struct skcipher_request *req, bool neon)
135{
136 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
137 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
138
139 return chacha_stream_xor(req, ctx, req->iv, neon);
140}
141
142static int chacha_arm(struct skcipher_request *req)
143{
144 return do_chacha(req, false);
145}
146
147static int chacha_neon(struct skcipher_request *req)
148{
149 return do_chacha(req, neon_usable());
150}
151
152static int do_xchacha(struct skcipher_request *req, bool neon)
153{
154 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
155 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
156 struct chacha_ctx subctx;
157 u32 state[16];
158 u8 real_iv[16];
159
160 chacha_init_generic(state, ctx->key, req->iv);
161
162 if (!neon) {
163 hchacha_block_arm(state, subctx.key, ctx->nrounds);
164 } else {
165 kernel_neon_begin();
166 hchacha_block_neon(state, subctx.key, ctx->nrounds);
167 kernel_neon_end();
168 }
169 subctx.nrounds = ctx->nrounds;
170
171 memcpy(&real_iv[0], req->iv + 24, 8);
172 memcpy(&real_iv[8], req->iv + 16, 8);
173 return chacha_stream_xor(req, &subctx, real_iv, neon);
174}
175
176static int xchacha_arm(struct skcipher_request *req)
177{
178 return do_xchacha(req, false);
179}
180
181static int xchacha_neon(struct skcipher_request *req)
182{
183 return do_xchacha(req, neon_usable());
184}
185
186static struct skcipher_alg arm_algs[] = {
187 {
188 .base.cra_name = "chacha20",
189 .base.cra_driver_name = "chacha20-arm",
190 .base.cra_priority = 200,
191 .base.cra_blocksize = 1,
192 .base.cra_ctxsize = sizeof(struct chacha_ctx),
193 .base.cra_module = THIS_MODULE,
194
195 .min_keysize = CHACHA_KEY_SIZE,
196 .max_keysize = CHACHA_KEY_SIZE,
197 .ivsize = CHACHA_IV_SIZE,
198 .chunksize = CHACHA_BLOCK_SIZE,
199 .setkey = chacha20_setkey,
200 .encrypt = chacha_arm,
201 .decrypt = chacha_arm,
202 }, {
203 .base.cra_name = "xchacha20",
204 .base.cra_driver_name = "xchacha20-arm",
205 .base.cra_priority = 200,
206 .base.cra_blocksize = 1,
207 .base.cra_ctxsize = sizeof(struct chacha_ctx),
208 .base.cra_module = THIS_MODULE,
209
210 .min_keysize = CHACHA_KEY_SIZE,
211 .max_keysize = CHACHA_KEY_SIZE,
212 .ivsize = XCHACHA_IV_SIZE,
213 .chunksize = CHACHA_BLOCK_SIZE,
214 .setkey = chacha20_setkey,
215 .encrypt = xchacha_arm,
216 .decrypt = xchacha_arm,
217 }, {
218 .base.cra_name = "xchacha12",
219 .base.cra_driver_name = "xchacha12-arm",
220 .base.cra_priority = 200,
221 .base.cra_blocksize = 1,
222 .base.cra_ctxsize = sizeof(struct chacha_ctx),
223 .base.cra_module = THIS_MODULE,
224
225 .min_keysize = CHACHA_KEY_SIZE,
226 .max_keysize = CHACHA_KEY_SIZE,
227 .ivsize = XCHACHA_IV_SIZE,
228 .chunksize = CHACHA_BLOCK_SIZE,
229 .setkey = chacha12_setkey,
230 .encrypt = xchacha_arm,
231 .decrypt = xchacha_arm,
232 },
233};
234
235static struct skcipher_alg neon_algs[] = {
236 {
237 .base.cra_name = "chacha20",
238 .base.cra_driver_name = "chacha20-neon",
239 .base.cra_priority = 300,
240 .base.cra_blocksize = 1,
241 .base.cra_ctxsize = sizeof(struct chacha_ctx),
242 .base.cra_module = THIS_MODULE,
243
244 .min_keysize = CHACHA_KEY_SIZE,
245 .max_keysize = CHACHA_KEY_SIZE,
246 .ivsize = CHACHA_IV_SIZE,
247 .chunksize = CHACHA_BLOCK_SIZE,
248 .walksize = 4 * CHACHA_BLOCK_SIZE,
249 .setkey = chacha20_setkey,
250 .encrypt = chacha_neon,
251 .decrypt = chacha_neon,
252 }, {
253 .base.cra_name = "xchacha20",
254 .base.cra_driver_name = "xchacha20-neon",
255 .base.cra_priority = 300,
256 .base.cra_blocksize = 1,
257 .base.cra_ctxsize = sizeof(struct chacha_ctx),
258 .base.cra_module = THIS_MODULE,
259
260 .min_keysize = CHACHA_KEY_SIZE,
261 .max_keysize = CHACHA_KEY_SIZE,
262 .ivsize = XCHACHA_IV_SIZE,
263 .chunksize = CHACHA_BLOCK_SIZE,
264 .walksize = 4 * CHACHA_BLOCK_SIZE,
265 .setkey = chacha20_setkey,
266 .encrypt = xchacha_neon,
267 .decrypt = xchacha_neon,
268 }, {
269 .base.cra_name = "xchacha12",
270 .base.cra_driver_name = "xchacha12-neon",
271 .base.cra_priority = 300,
272 .base.cra_blocksize = 1,
273 .base.cra_ctxsize = sizeof(struct chacha_ctx),
274 .base.cra_module = THIS_MODULE,
275
276 .min_keysize = CHACHA_KEY_SIZE,
277 .max_keysize = CHACHA_KEY_SIZE,
278 .ivsize = XCHACHA_IV_SIZE,
279 .chunksize = CHACHA_BLOCK_SIZE,
280 .walksize = 4 * CHACHA_BLOCK_SIZE,
281 .setkey = chacha12_setkey,
282 .encrypt = xchacha_neon,
283 .decrypt = xchacha_neon,
284 }
285};
286
287static int __init chacha_simd_mod_init(void)
288{
289 int err = 0;
290
291 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
292 err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
293 if (err)
294 return err;
295 }
296
297 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
298 int i;
299
300 switch (read_cpuid_part()) {
301 case ARM_CPU_PART_CORTEX_A7:
302 case ARM_CPU_PART_CORTEX_A5:
303
304
305
306
307
308 for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
309 neon_algs[i].base.cra_priority = 0;
310 break;
311 default:
312 static_branch_enable(&use_neon);
313 }
314
315 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
316 err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
317 if (err)
318 crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
319 }
320 }
321 return err;
322}
323
324static void __exit chacha_simd_mod_fini(void)
325{
326 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
327 crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
328 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON))
329 crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
330 }
331}
332
333module_init(chacha_simd_mod_init);
334module_exit(chacha_simd_mod_fini);
335
336MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)");
337MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
338MODULE_LICENSE("GPL v2");
339MODULE_ALIAS_CRYPTO("chacha20");
340MODULE_ALIAS_CRYPTO("chacha20-arm");
341MODULE_ALIAS_CRYPTO("xchacha20");
342MODULE_ALIAS_CRYPTO("xchacha20-arm");
343MODULE_ALIAS_CRYPTO("xchacha12");
344MODULE_ALIAS_CRYPTO("xchacha12-arm");
345#ifdef CONFIG_KERNEL_MODE_NEON
346MODULE_ALIAS_CRYPTO("chacha20-neon");
347MODULE_ALIAS_CRYPTO("xchacha20-neon");
348MODULE_ALIAS_CRYPTO("xchacha12-neon");
349#endif
350
