1
2
3
4
5
6
7
8#include <linux/module.h>
9#include <linux/types.h>
10#include <linux/crypto.h>
11#include <linux/err.h>
12#include <crypto/algapi.h>
13#include <crypto/internal/simd.h>
14#include <crypto/serpent.h>
15#include <crypto/xts.h>
16#include <asm/crypto/glue_helper.h>
17#include <asm/crypto/serpent-avx.h>
18
19#define SERPENT_AVX2_PARALLEL_BLOCKS 16
20
21
22asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src);
23asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src);
24asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src);
25
26asmlinkage void serpent_ctr_16way(const void *ctx, u8 *dst, const u8 *src,
27 le128 *iv);
28asmlinkage void serpent_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src,
29 le128 *iv);
30asmlinkage void serpent_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src,
31 le128 *iv);
32
33static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
34 const u8 *key, unsigned int keylen)
35{
36 return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
37}
38
39static const struct common_glue_ctx serpent_enc = {
40 .num_funcs = 3,
41 .fpu_blocks_limit = 8,
42
43 .funcs = { {
44 .num_blocks = 16,
45 .fn_u = { .ecb = serpent_ecb_enc_16way }
46 }, {
47 .num_blocks = 8,
48 .fn_u = { .ecb = serpent_ecb_enc_8way_avx }
49 }, {
50 .num_blocks = 1,
51 .fn_u = { .ecb = __serpent_encrypt }
52 } }
53};
54
55static const struct common_glue_ctx serpent_ctr = {
56 .num_funcs = 3,
57 .fpu_blocks_limit = 8,
58
59 .funcs = { {
60 .num_blocks = 16,
61 .fn_u = { .ctr = serpent_ctr_16way }
62 }, {
63 .num_blocks = 8,
64 .fn_u = { .ctr = serpent_ctr_8way_avx }
65 }, {
66 .num_blocks = 1,
67 .fn_u = { .ctr = __serpent_crypt_ctr }
68 } }
69};
70
71static const struct common_glue_ctx serpent_enc_xts = {
72 .num_funcs = 3,
73 .fpu_blocks_limit = 8,
74
75 .funcs = { {
76 .num_blocks = 16,
77 .fn_u = { .xts = serpent_xts_enc_16way }
78 }, {
79 .num_blocks = 8,
80 .fn_u = { .xts = serpent_xts_enc_8way_avx }
81 }, {
82 .num_blocks = 1,
83 .fn_u = { .xts = serpent_xts_enc }
84 } }
85};
86
87static const struct common_glue_ctx serpent_dec = {
88 .num_funcs = 3,
89 .fpu_blocks_limit = 8,
90
91 .funcs = { {
92 .num_blocks = 16,
93 .fn_u = { .ecb = serpent_ecb_dec_16way }
94 }, {
95 .num_blocks = 8,
96 .fn_u = { .ecb = serpent_ecb_dec_8way_avx }
97 }, {
98 .num_blocks = 1,
99 .fn_u = { .ecb = __serpent_decrypt }
100 } }
101};
102
103static const struct common_glue_ctx serpent_dec_cbc = {
104 .num_funcs = 3,
105 .fpu_blocks_limit = 8,
106
107 .funcs = { {
108 .num_blocks = 16,
109 .fn_u = { .cbc = serpent_cbc_dec_16way }
110 }, {
111 .num_blocks = 8,
112 .fn_u = { .cbc = serpent_cbc_dec_8way_avx }
113 }, {
114 .num_blocks = 1,
115 .fn_u = { .cbc = __serpent_decrypt }
116 } }
117};
118
119static const struct common_glue_ctx serpent_dec_xts = {
120 .num_funcs = 3,
121 .fpu_blocks_limit = 8,
122
123 .funcs = { {
124 .num_blocks = 16,
125 .fn_u = { .xts = serpent_xts_dec_16way }
126 }, {
127 .num_blocks = 8,
128 .fn_u = { .xts = serpent_xts_dec_8way_avx }
129 }, {
130 .num_blocks = 1,
131 .fn_u = { .xts = serpent_xts_dec }
132 } }
133};
134
135static int ecb_encrypt(struct skcipher_request *req)
136{
137 return glue_ecb_req_128bit(&serpent_enc, req);
138}
139
140static int ecb_decrypt(struct skcipher_request *req)
141{
142 return glue_ecb_req_128bit(&serpent_dec, req);
143}
144
145static int cbc_encrypt(struct skcipher_request *req)
146{
147 return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req);
148}
149
150static int cbc_decrypt(struct skcipher_request *req)
151{
152 return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
153}
154
155static int ctr_crypt(struct skcipher_request *req)
156{
157 return glue_ctr_req_128bit(&serpent_ctr, req);
158}
159
160static int xts_encrypt(struct skcipher_request *req)
161{
162 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
163 struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
164
165 return glue_xts_req_128bit(&serpent_enc_xts, req,
166 __serpent_encrypt, &ctx->tweak_ctx,
167 &ctx->crypt_ctx, false);
168}
169
170static int xts_decrypt(struct skcipher_request *req)
171{
172 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
173 struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
174
175 return glue_xts_req_128bit(&serpent_dec_xts, req,
176 __serpent_encrypt, &ctx->tweak_ctx,
177 &ctx->crypt_ctx, true);
178}
179
180static struct skcipher_alg serpent_algs[] = {
181 {
182 .base.cra_name = "__ecb(serpent)",
183 .base.cra_driver_name = "__ecb-serpent-avx2",
184 .base.cra_priority = 600,
185 .base.cra_flags = CRYPTO_ALG_INTERNAL,
186 .base.cra_blocksize = SERPENT_BLOCK_SIZE,
187 .base.cra_ctxsize = sizeof(struct serpent_ctx),
188 .base.cra_module = THIS_MODULE,
189 .min_keysize = SERPENT_MIN_KEY_SIZE,
190 .max_keysize = SERPENT_MAX_KEY_SIZE,
191 .setkey = serpent_setkey_skcipher,
192 .encrypt = ecb_encrypt,
193 .decrypt = ecb_decrypt,
194 }, {
195 .base.cra_name = "__cbc(serpent)",
196 .base.cra_driver_name = "__cbc-serpent-avx2",
197 .base.cra_priority = 600,
198 .base.cra_flags = CRYPTO_ALG_INTERNAL,
199 .base.cra_blocksize = SERPENT_BLOCK_SIZE,
200 .base.cra_ctxsize = sizeof(struct serpent_ctx),
201 .base.cra_module = THIS_MODULE,
202 .min_keysize = SERPENT_MIN_KEY_SIZE,
203 .max_keysize = SERPENT_MAX_KEY_SIZE,
204 .ivsize = SERPENT_BLOCK_SIZE,
205 .setkey = serpent_setkey_skcipher,
206 .encrypt = cbc_encrypt,
207 .decrypt = cbc_decrypt,
208 }, {
209 .base.cra_name = "__ctr(serpent)",
210 .base.cra_driver_name = "__ctr-serpent-avx2",
211 .base.cra_priority = 600,
212 .base.cra_flags = CRYPTO_ALG_INTERNAL,
213 .base.cra_blocksize = 1,
214 .base.cra_ctxsize = sizeof(struct serpent_ctx),
215 .base.cra_module = THIS_MODULE,
216 .min_keysize = SERPENT_MIN_KEY_SIZE,
217 .max_keysize = SERPENT_MAX_KEY_SIZE,
218 .ivsize = SERPENT_BLOCK_SIZE,
219 .chunksize = SERPENT_BLOCK_SIZE,
220 .setkey = serpent_setkey_skcipher,
221 .encrypt = ctr_crypt,
222 .decrypt = ctr_crypt,
223 }, {
224 .base.cra_name = "__xts(serpent)",
225 .base.cra_driver_name = "__xts-serpent-avx2",
226 .base.cra_priority = 600,
227 .base.cra_flags = CRYPTO_ALG_INTERNAL,
228 .base.cra_blocksize = SERPENT_BLOCK_SIZE,
229 .base.cra_ctxsize = sizeof(struct serpent_xts_ctx),
230 .base.cra_module = THIS_MODULE,
231 .min_keysize = 2 * SERPENT_MIN_KEY_SIZE,
232 .max_keysize = 2 * SERPENT_MAX_KEY_SIZE,
233 .ivsize = SERPENT_BLOCK_SIZE,
234 .setkey = xts_serpent_setkey,
235 .encrypt = xts_encrypt,
236 .decrypt = xts_decrypt,
237 },
238};
239
240static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
241
242static int __init init(void)
243{
244 const char *feature_name;
245
246 if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
247 pr_info("AVX2 instructions are not detected.\n");
248 return -ENODEV;
249 }
250 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
251 &feature_name)) {
252 pr_info("CPU feature '%s' is not supported.\n", feature_name);
253 return -ENODEV;
254 }
255
256 return simd_register_skciphers_compat(serpent_algs,
257 ARRAY_SIZE(serpent_algs),
258 serpent_simd_algs);
259}
260
261static void __exit fini(void)
262{
263 simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
264 serpent_simd_algs);
265}
266
267module_init(init);
268module_exit(fini);
269
270MODULE_LICENSE("GPL");
271MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
272MODULE_ALIAS_CRYPTO("serpent");
273MODULE_ALIAS_CRYPTO("serpent-asm");
274
