1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
23
24#include <crypto/internal/hash.h>
25#include <linux/init.h>
26#include <linux/module.h>
27#include <linux/mm.h>
28#include <linux/cryptohash.h>
29#include <linux/types.h>
30#include <crypto/sha.h>
31#include <crypto/sha1_base.h>
32#include <asm/fpu/api.h>
33
34typedef void (sha1_transform_fn)(u32 *digest, const char *data,
35 unsigned int rounds);
36
37static int sha1_update(struct shash_desc *desc, const u8 *data,
38 unsigned int len, sha1_transform_fn *sha1_xform)
39{
40 struct sha1_state *sctx = shash_desc_ctx(desc);
41
42 if (!irq_fpu_usable() ||
43 (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE)
44 return crypto_sha1_update(desc, data, len);
45
46
47 BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
48
49 kernel_fpu_begin();
50 sha1_base_do_update(desc, data, len,
51 (sha1_block_fn *)sha1_xform);
52 kernel_fpu_end();
53
54 return 0;
55}
56
57static int sha1_finup(struct shash_desc *desc, const u8 *data,
58 unsigned int len, u8 *out, sha1_transform_fn *sha1_xform)
59{
60 if (!irq_fpu_usable())
61 return crypto_sha1_finup(desc, data, len, out);
62
63 kernel_fpu_begin();
64 if (len)
65 sha1_base_do_update(desc, data, len,
66 (sha1_block_fn *)sha1_xform);
67 sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_xform);
68 kernel_fpu_end();
69
70 return sha1_base_finish(desc, out);
71}
72
73asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
74 unsigned int rounds);
75
76static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
77 unsigned int len)
78{
79 return sha1_update(desc, data, len,
80 (sha1_transform_fn *) sha1_transform_ssse3);
81}
82
83static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data,
84 unsigned int len, u8 *out)
85{
86 return sha1_finup(desc, data, len, out,
87 (sha1_transform_fn *) sha1_transform_ssse3);
88}
89
90
91static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
92{
93 return sha1_ssse3_finup(desc, NULL, 0, out);
94}
95
96static struct shash_alg sha1_ssse3_alg = {
97 .digestsize = SHA1_DIGEST_SIZE,
98 .init = sha1_base_init,
99 .update = sha1_ssse3_update,
100 .final = sha1_ssse3_final,
101 .finup = sha1_ssse3_finup,
102 .descsize = sizeof(struct sha1_state),
103 .base = {
104 .cra_name = "sha1",
105 .cra_driver_name = "sha1-ssse3",
106 .cra_priority = 150,
107 .cra_blocksize = SHA1_BLOCK_SIZE,
108 .cra_module = THIS_MODULE,
109 }
110};
111
112static int register_sha1_ssse3(void)
113{
114 if (boot_cpu_has(X86_FEATURE_SSSE3))
115 return crypto_register_shash(&sha1_ssse3_alg);
116 return 0;
117}
118
119static void unregister_sha1_ssse3(void)
120{
121 if (boot_cpu_has(X86_FEATURE_SSSE3))
122 crypto_unregister_shash(&sha1_ssse3_alg);
123}
124
125#ifdef CONFIG_AS_AVX
126asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
127 unsigned int rounds);
128
129static int sha1_avx_update(struct shash_desc *desc, const u8 *data,
130 unsigned int len)
131{
132 return sha1_update(desc, data, len,
133 (sha1_transform_fn *) sha1_transform_avx);
134}
135
136static int sha1_avx_finup(struct shash_desc *desc, const u8 *data,
137 unsigned int len, u8 *out)
138{
139 return sha1_finup(desc, data, len, out,
140 (sha1_transform_fn *) sha1_transform_avx);
141}
142
143static int sha1_avx_final(struct shash_desc *desc, u8 *out)
144{
145 return sha1_avx_finup(desc, NULL, 0, out);
146}
147
148static struct shash_alg sha1_avx_alg = {
149 .digestsize = SHA1_DIGEST_SIZE,
150 .init = sha1_base_init,
151 .update = sha1_avx_update,
152 .final = sha1_avx_final,
153 .finup = sha1_avx_finup,
154 .descsize = sizeof(struct sha1_state),
155 .base = {
156 .cra_name = "sha1",
157 .cra_driver_name = "sha1-avx",
158 .cra_priority = 160,
159 .cra_blocksize = SHA1_BLOCK_SIZE,
160 .cra_module = THIS_MODULE,
161 }
162};
163
164static bool avx_usable(void)
165{
166 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
167 if (boot_cpu_has(X86_FEATURE_AVX))
168 pr_info("AVX detected but unusable.\n");
169 return false;
170 }
171
172 return true;
173}
174
175static int register_sha1_avx(void)
176{
177 if (avx_usable())
178 return crypto_register_shash(&sha1_avx_alg);
179 return 0;
180}
181
182static void unregister_sha1_avx(void)
183{
184 if (avx_usable())
185 crypto_unregister_shash(&sha1_avx_alg);
186}
187
188#else
189static inline int register_sha1_avx(void) { return 0; }
190static inline void unregister_sha1_avx(void) { }
191#endif
192
193
194#if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
195#define SHA1_AVX2_BLOCK_OPTSIZE 4
196
197asmlinkage void sha1_transform_avx2(u32 *digest, const char *data,
198 unsigned int rounds);
199
200static bool avx2_usable(void)
201{
202 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2)
203 && boot_cpu_has(X86_FEATURE_BMI1)
204 && boot_cpu_has(X86_FEATURE_BMI2))
205 return true;
206
207 return false;
208}
209
210static void sha1_apply_transform_avx2(u32 *digest, const char *data,
211 unsigned int rounds)
212{
213
214 if (rounds >= SHA1_AVX2_BLOCK_OPTSIZE)
215 sha1_transform_avx2(digest, data, rounds);
216 else
217 sha1_transform_avx(digest, data, rounds);
218}
219
220static int sha1_avx2_update(struct shash_desc *desc, const u8 *data,
221 unsigned int len)
222{
223 return sha1_update(desc, data, len,
224 (sha1_transform_fn *) sha1_apply_transform_avx2);
225}
226
227static int sha1_avx2_finup(struct shash_desc *desc, const u8 *data,
228 unsigned int len, u8 *out)
229{
230 return sha1_finup(desc, data, len, out,
231 (sha1_transform_fn *) sha1_apply_transform_avx2);
232}
233
234static int sha1_avx2_final(struct shash_desc *desc, u8 *out)
235{
236 return sha1_avx2_finup(desc, NULL, 0, out);
237}
238
239static struct shash_alg sha1_avx2_alg = {
240 .digestsize = SHA1_DIGEST_SIZE,
241 .init = sha1_base_init,
242 .update = sha1_avx2_update,
243 .final = sha1_avx2_final,
244 .finup = sha1_avx2_finup,
245 .descsize = sizeof(struct sha1_state),
246 .base = {
247 .cra_name = "sha1",
248 .cra_driver_name = "sha1-avx2",
249 .cra_priority = 170,
250 .cra_blocksize = SHA1_BLOCK_SIZE,
251 .cra_module = THIS_MODULE,
252 }
253};
254
255static int register_sha1_avx2(void)
256{
257 if (avx2_usable())
258 return crypto_register_shash(&sha1_avx2_alg);
259 return 0;
260}
261
262static void unregister_sha1_avx2(void)
263{
264 if (avx2_usable())
265 crypto_unregister_shash(&sha1_avx2_alg);
266}
267
268#else
269static inline int register_sha1_avx2(void) { return 0; }
270static inline void unregister_sha1_avx2(void) { }
271#endif
272
273#ifdef CONFIG_AS_SHA1_NI
274asmlinkage void sha1_ni_transform(u32 *digest, const char *data,
275 unsigned int rounds);
276
277static int sha1_ni_update(struct shash_desc *desc, const u8 *data,
278 unsigned int len)
279{
280 return sha1_update(desc, data, len,
281 (sha1_transform_fn *) sha1_ni_transform);
282}
283
284static int sha1_ni_finup(struct shash_desc *desc, const u8 *data,
285 unsigned int len, u8 *out)
286{
287 return sha1_finup(desc, data, len, out,
288 (sha1_transform_fn *) sha1_ni_transform);
289}
290
291static int sha1_ni_final(struct shash_desc *desc, u8 *out)
292{
293 return sha1_ni_finup(desc, NULL, 0, out);
294}
295
296static struct shash_alg sha1_ni_alg = {
297 .digestsize = SHA1_DIGEST_SIZE,
298 .init = sha1_base_init,
299 .update = sha1_ni_update,
300 .final = sha1_ni_final,
301 .finup = sha1_ni_finup,
302 .descsize = sizeof(struct sha1_state),
303 .base = {
304 .cra_name = "sha1",
305 .cra_driver_name = "sha1-ni",
306 .cra_priority = 250,
307 .cra_blocksize = SHA1_BLOCK_SIZE,
308 .cra_module = THIS_MODULE,
309 }
310};
311
312static int register_sha1_ni(void)
313{
314 if (boot_cpu_has(X86_FEATURE_SHA_NI))
315 return crypto_register_shash(&sha1_ni_alg);
316 return 0;
317}
318
319static void unregister_sha1_ni(void)
320{
321 if (boot_cpu_has(X86_FEATURE_SHA_NI))
322 crypto_unregister_shash(&sha1_ni_alg);
323}
324
325#else
326static inline int register_sha1_ni(void) { return 0; }
327static inline void unregister_sha1_ni(void) { }
328#endif
329
330static int __init sha1_ssse3_mod_init(void)
331{
332 if (register_sha1_ssse3())
333 goto fail;
334
335 if (register_sha1_avx()) {
336 unregister_sha1_ssse3();
337 goto fail;
338 }
339
340 if (register_sha1_avx2()) {
341 unregister_sha1_avx();
342 unregister_sha1_ssse3();
343 goto fail;
344 }
345
346 if (register_sha1_ni()) {
347 unregister_sha1_avx2();
348 unregister_sha1_avx();
349 unregister_sha1_ssse3();
350 goto fail;
351 }
352
353 return 0;
354fail:
355 return -ENODEV;
356}
357
358static void __exit sha1_ssse3_mod_fini(void)
359{
360 unregister_sha1_ni();
361 unregister_sha1_avx2();
362 unregister_sha1_avx();
363 unregister_sha1_ssse3();
364}
365
366module_init(sha1_ssse3_mod_init);
367module_exit(sha1_ssse3_mod_fini);
368
369MODULE_LICENSE("GPL");
370MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
371
372MODULE_ALIAS_CRYPTO("sha1");
373MODULE_ALIAS_CRYPTO("sha1-ssse3");
374MODULE_ALIAS_CRYPTO("sha1-avx");
375MODULE_ALIAS_CRYPTO("sha1-avx2");
376#ifdef CONFIG_AS_SHA1_NI
377MODULE_ALIAS_CRYPTO("sha1-ni");
378#endif
379
