1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#include <crypto/algapi.h>
23#include <crypto/chacha.h>
24#include <crypto/internal/simd.h>
25#include <crypto/internal/skcipher.h>
26#include <linux/kernel.h>
27#include <linux/module.h>
28
29#include <asm/hwcap.h>
30#include <asm/neon.h>
31#include <asm/simd.h>
32
33asmlinkage void chacha_block_xor_neon(u32 *state, u8 *dst, const u8 *src,
34 int nrounds);
35asmlinkage void chacha_4block_xor_neon(u32 *state, u8 *dst, const u8 *src,
36 int nrounds, int bytes);
37asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
38
39static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
40 int bytes, int nrounds)
41{
42 while (bytes > 0) {
43 int l = min(bytes, CHACHA_BLOCK_SIZE * 5);
44
45 if (l <= CHACHA_BLOCK_SIZE) {
46 u8 buf[CHACHA_BLOCK_SIZE];
47
48 memcpy(buf, src, l);
49 chacha_block_xor_neon(state, buf, buf, nrounds);
50 memcpy(dst, buf, l);
51 state[12] += 1;
52 break;
53 }
54 chacha_4block_xor_neon(state, dst, src, nrounds, l);
55 bytes -= CHACHA_BLOCK_SIZE * 5;
56 src += CHACHA_BLOCK_SIZE * 5;
57 dst += CHACHA_BLOCK_SIZE * 5;
58 state[12] += 5;
59 }
60}
61
62static int chacha_neon_stream_xor(struct skcipher_request *req,
63 const struct chacha_ctx *ctx, const u8 *iv)
64{
65 struct skcipher_walk walk;
66 u32 state[16];
67 int err;
68
69 err = skcipher_walk_virt(&walk, req, false);
70
71 crypto_chacha_init(state, ctx, iv);
72
73 while (walk.nbytes > 0) {
74 unsigned int nbytes = walk.nbytes;
75
76 if (nbytes < walk.total)
77 nbytes = rounddown(nbytes, walk.stride);
78
79 kernel_neon_begin();
80 chacha_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
81 nbytes, ctx->nrounds);
82 kernel_neon_end();
83 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
84 }
85
86 return err;
87}
88
89static int chacha_neon(struct skcipher_request *req)
90{
91 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
92 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
93
94 if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
95 return crypto_chacha_crypt(req);
96
97 return chacha_neon_stream_xor(req, ctx, req->iv);
98}
99
100static int xchacha_neon(struct skcipher_request *req)
101{
102 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
103 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
104 struct chacha_ctx subctx;
105 u32 state[16];
106 u8 real_iv[16];
107
108 if (req->cryptlen <= CHACHA_BLOCK_SIZE || !crypto_simd_usable())
109 return crypto_xchacha_crypt(req);
110
111 crypto_chacha_init(state, ctx, req->iv);
112
113 kernel_neon_begin();
114 hchacha_block_neon(state, subctx.key, ctx->nrounds);
115 kernel_neon_end();
116 subctx.nrounds = ctx->nrounds;
117
118 memcpy(&real_iv[0], req->iv + 24, 8);
119 memcpy(&real_iv[8], req->iv + 16, 8);
120 return chacha_neon_stream_xor(req, &subctx, real_iv);
121}
122
123static struct skcipher_alg algs[] = {
124 {
125 .base.cra_name = "chacha20",
126 .base.cra_driver_name = "chacha20-neon",
127 .base.cra_priority = 300,
128 .base.cra_blocksize = 1,
129 .base.cra_ctxsize = sizeof(struct chacha_ctx),
130 .base.cra_module = THIS_MODULE,
131
132 .min_keysize = CHACHA_KEY_SIZE,
133 .max_keysize = CHACHA_KEY_SIZE,
134 .ivsize = CHACHA_IV_SIZE,
135 .chunksize = CHACHA_BLOCK_SIZE,
136 .walksize = 5 * CHACHA_BLOCK_SIZE,
137 .setkey = crypto_chacha20_setkey,
138 .encrypt = chacha_neon,
139 .decrypt = chacha_neon,
140 }, {
141 .base.cra_name = "xchacha20",
142 .base.cra_driver_name = "xchacha20-neon",
143 .base.cra_priority = 300,
144 .base.cra_blocksize = 1,
145 .base.cra_ctxsize = sizeof(struct chacha_ctx),
146 .base.cra_module = THIS_MODULE,
147
148 .min_keysize = CHACHA_KEY_SIZE,
149 .max_keysize = CHACHA_KEY_SIZE,
150 .ivsize = XCHACHA_IV_SIZE,
151 .chunksize = CHACHA_BLOCK_SIZE,
152 .walksize = 5 * CHACHA_BLOCK_SIZE,
153 .setkey = crypto_chacha20_setkey,
154 .encrypt = xchacha_neon,
155 .decrypt = xchacha_neon,
156 }, {
157 .base.cra_name = "xchacha12",
158 .base.cra_driver_name = "xchacha12-neon",
159 .base.cra_priority = 300,
160 .base.cra_blocksize = 1,
161 .base.cra_ctxsize = sizeof(struct chacha_ctx),
162 .base.cra_module = THIS_MODULE,
163
164 .min_keysize = CHACHA_KEY_SIZE,
165 .max_keysize = CHACHA_KEY_SIZE,
166 .ivsize = XCHACHA_IV_SIZE,
167 .chunksize = CHACHA_BLOCK_SIZE,
168 .walksize = 5 * CHACHA_BLOCK_SIZE,
169 .setkey = crypto_chacha12_setkey,
170 .encrypt = xchacha_neon,
171 .decrypt = xchacha_neon,
172 }
173};
174
175static int __init chacha_simd_mod_init(void)
176{
177 if (!cpu_have_named_feature(ASIMD))
178 return -ENODEV;
179
180 return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
181}
182
183static void __exit chacha_simd_mod_fini(void)
184{
185 crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
186}
187
188module_init(chacha_simd_mod_init);
189module_exit(chacha_simd_mod_fini);
190
191MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)");
192MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
193MODULE_LICENSE("GPL v2");
194MODULE_ALIAS_CRYPTO("chacha20");
195MODULE_ALIAS_CRYPTO("chacha20-neon");
196MODULE_ALIAS_CRYPTO("xchacha20");
197MODULE_ALIAS_CRYPTO("xchacha20-neon");
198MODULE_ALIAS_CRYPTO("xchacha12");
199MODULE_ALIAS_CRYPTO("xchacha12-neon");
200
