1
2
3
4
5
6
7
8#include <linux/cpufeature.h>
9#include <linux/crc-t10dif.h>
10#include <linux/init.h>
11#include <linux/kernel.h>
12#include <linux/module.h>
13#include <linux/string.h>
14
15#include <crypto/internal/hash.h>
16#include <crypto/internal/simd.h>
17
18#include <asm/neon.h>
19#include <asm/simd.h>
20
21#define CRC_T10DIF_PMULL_CHUNK_SIZE 16U
22
23asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len);
24asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
25
26static int crct10dif_init(struct shash_desc *desc)
27{
28 u16 *crc = shash_desc_ctx(desc);
29
30 *crc = 0;
31 return 0;
32}
33
34static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data,
35 unsigned int length)
36{
37 u16 *crc = shash_desc_ctx(desc);
38
39 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
40 kernel_neon_begin();
41 *crc = crc_t10dif_pmull_p8(*crc, data, length);
42 kernel_neon_end();
43 } else {
44 *crc = crc_t10dif_generic(*crc, data, length);
45 }
46
47 return 0;
48}
49
50static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data,
51 unsigned int length)
52{
53 u16 *crc = shash_desc_ctx(desc);
54
55 if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
56 kernel_neon_begin();
57 *crc = crc_t10dif_pmull_p64(*crc, data, length);
58 kernel_neon_end();
59 } else {
60 *crc = crc_t10dif_generic(*crc, data, length);
61 }
62
63 return 0;
64}
65
66static int crct10dif_final(struct shash_desc *desc, u8 *out)
67{
68 u16 *crc = shash_desc_ctx(desc);
69
70 *(u16 *)out = *crc;
71 return 0;
72}
73
74static struct shash_alg crc_t10dif_alg[] = {{
75 .digestsize = CRC_T10DIF_DIGEST_SIZE,
76 .init = crct10dif_init,
77 .update = crct10dif_update_pmull_p8,
78 .final = crct10dif_final,
79 .descsize = CRC_T10DIF_DIGEST_SIZE,
80
81 .base.cra_name = "crct10dif",
82 .base.cra_driver_name = "crct10dif-arm64-neon",
83 .base.cra_priority = 100,
84 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
85 .base.cra_module = THIS_MODULE,
86}, {
87 .digestsize = CRC_T10DIF_DIGEST_SIZE,
88 .init = crct10dif_init,
89 .update = crct10dif_update_pmull_p64,
90 .final = crct10dif_final,
91 .descsize = CRC_T10DIF_DIGEST_SIZE,
92
93 .base.cra_name = "crct10dif",
94 .base.cra_driver_name = "crct10dif-arm64-ce",
95 .base.cra_priority = 200,
96 .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE,
97 .base.cra_module = THIS_MODULE,
98}};
99
100static int __init crc_t10dif_mod_init(void)
101{
102 if (cpu_have_named_feature(PMULL))
103 return crypto_register_shashes(crc_t10dif_alg,
104 ARRAY_SIZE(crc_t10dif_alg));
105 else
106
107 return crypto_register_shash(crc_t10dif_alg);
108}
109
110static void __exit crc_t10dif_mod_exit(void)
111{
112 if (cpu_have_named_feature(PMULL))
113 crypto_unregister_shashes(crc_t10dif_alg,
114 ARRAY_SIZE(crc_t10dif_alg));
115 else
116 crypto_unregister_shash(crc_t10dif_alg);
117}
118
119module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
120module_exit(crc_t10dif_mod_exit);
121
122MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
123MODULE_LICENSE("GPL v2");
124MODULE_ALIAS_CRYPTO("crct10dif");
125MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");
126
