1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/module.h>
14#include <crypto/b128ops.h>
15#include <crypto/gf128mul.h>
16#include <crypto/internal/skcipher.h>
17#include <crypto/scatterwalk.h>
18#include <crypto/xts.h>
19#include <asm/crypto/glue_helper.h>
20
21int glue_ecb_req_128bit(const struct common_glue_ctx *gctx,
22 struct skcipher_request *req)
23{
24 void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
25 const unsigned int bsize = 128 / 8;
26 struct skcipher_walk walk;
27 bool fpu_enabled = false;
28 unsigned int nbytes;
29 int err;
30
31 err = skcipher_walk_virt(&walk, req, false);
32
33 while ((nbytes = walk.nbytes)) {
34 const u8 *src = walk.src.virt.addr;
35 u8 *dst = walk.dst.virt.addr;
36 unsigned int func_bytes;
37 unsigned int i;
38
39 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
40 &walk, fpu_enabled, nbytes);
41 for (i = 0; i < gctx->num_funcs; i++) {
42 func_bytes = bsize * gctx->funcs[i].num_blocks;
43
44 if (nbytes < func_bytes)
45 continue;
46
47
48 do {
49 gctx->funcs[i].fn_u.ecb(ctx, dst, src);
50 src += func_bytes;
51 dst += func_bytes;
52 nbytes -= func_bytes;
53 } while (nbytes >= func_bytes);
54
55 if (nbytes < bsize)
56 break;
57 }
58 err = skcipher_walk_done(&walk, nbytes);
59 }
60
61 glue_fpu_end(fpu_enabled);
62 return err;
63}
64EXPORT_SYMBOL_GPL(glue_ecb_req_128bit);
65
66int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn,
67 struct skcipher_request *req)
68{
69 void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
70 const unsigned int bsize = 128 / 8;
71 struct skcipher_walk walk;
72 unsigned int nbytes;
73 int err;
74
75 err = skcipher_walk_virt(&walk, req, false);
76
77 while ((nbytes = walk.nbytes)) {
78 const u128 *src = (u128 *)walk.src.virt.addr;
79 u128 *dst = (u128 *)walk.dst.virt.addr;
80 u128 *iv = (u128 *)walk.iv;
81
82 do {
83 u128_xor(dst, src, iv);
84 fn(ctx, (u8 *)dst, (u8 *)dst);
85 iv = dst;
86 src++;
87 dst++;
88 nbytes -= bsize;
89 } while (nbytes >= bsize);
90
91 *(u128 *)walk.iv = *iv;
92 err = skcipher_walk_done(&walk, nbytes);
93 }
94 return err;
95}
96EXPORT_SYMBOL_GPL(glue_cbc_encrypt_req_128bit);
97
98int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx,
99 struct skcipher_request *req)
100{
101 void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
102 const unsigned int bsize = 128 / 8;
103 struct skcipher_walk walk;
104 bool fpu_enabled = false;
105 unsigned int nbytes;
106 int err;
107
108 err = skcipher_walk_virt(&walk, req, false);
109
110 while ((nbytes = walk.nbytes)) {
111 const u128 *src = walk.src.virt.addr;
112 u128 *dst = walk.dst.virt.addr;
113 unsigned int func_bytes, num_blocks;
114 unsigned int i;
115 u128 last_iv;
116
117 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
118 &walk, fpu_enabled, nbytes);
119
120 src += nbytes / bsize - 1;
121 dst += nbytes / bsize - 1;
122
123 last_iv = *src;
124
125 for (i = 0; i < gctx->num_funcs; i++) {
126 num_blocks = gctx->funcs[i].num_blocks;
127 func_bytes = bsize * num_blocks;
128
129 if (nbytes < func_bytes)
130 continue;
131
132
133 do {
134 src -= num_blocks - 1;
135 dst -= num_blocks - 1;
136
137 gctx->funcs[i].fn_u.cbc(ctx, (u8 *)dst,
138 (const u8 *)src);
139
140 nbytes -= func_bytes;
141 if (nbytes < bsize)
142 goto done;
143
144 u128_xor(dst, dst, --src);
145 dst--;
146 } while (nbytes >= func_bytes);
147 }
148done:
149 u128_xor(dst, dst, (u128 *)walk.iv);
150 *(u128 *)walk.iv = last_iv;
151 err = skcipher_walk_done(&walk, nbytes);
152 }
153
154 glue_fpu_end(fpu_enabled);
155 return err;
156}
157EXPORT_SYMBOL_GPL(glue_cbc_decrypt_req_128bit);
158
159int glue_ctr_req_128bit(const struct common_glue_ctx *gctx,
160 struct skcipher_request *req)
161{
162 void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
163 const unsigned int bsize = 128 / 8;
164 struct skcipher_walk walk;
165 bool fpu_enabled = false;
166 unsigned int nbytes;
167 int err;
168
169 err = skcipher_walk_virt(&walk, req, false);
170
171 while ((nbytes = walk.nbytes) >= bsize) {
172 const u128 *src = walk.src.virt.addr;
173 u128 *dst = walk.dst.virt.addr;
174 unsigned int func_bytes, num_blocks;
175 unsigned int i;
176 le128 ctrblk;
177
178 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
179 &walk, fpu_enabled, nbytes);
180
181 be128_to_le128(&ctrblk, (be128 *)walk.iv);
182
183 for (i = 0; i < gctx->num_funcs; i++) {
184 num_blocks = gctx->funcs[i].num_blocks;
185 func_bytes = bsize * num_blocks;
186
187 if (nbytes < func_bytes)
188 continue;
189
190
191 do {
192 gctx->funcs[i].fn_u.ctr(ctx, (u8 *)dst,
193 (const u8 *)src,
194 &ctrblk);
195 src += num_blocks;
196 dst += num_blocks;
197 nbytes -= func_bytes;
198 } while (nbytes >= func_bytes);
199
200 if (nbytes < bsize)
201 break;
202 }
203
204 le128_to_be128((be128 *)walk.iv, &ctrblk);
205 err = skcipher_walk_done(&walk, nbytes);
206 }
207
208 glue_fpu_end(fpu_enabled);
209
210 if (nbytes) {
211 le128 ctrblk;
212 u128 tmp;
213
214 be128_to_le128(&ctrblk, (be128 *)walk.iv);
215 memcpy(&tmp, walk.src.virt.addr, nbytes);
216 gctx->funcs[gctx->num_funcs - 1].fn_u.ctr(ctx, (u8 *)&tmp,
217 (const u8 *)&tmp,
218 &ctrblk);
219 memcpy(walk.dst.virt.addr, &tmp, nbytes);
220 le128_to_be128((be128 *)walk.iv, &ctrblk);
221
222 err = skcipher_walk_done(&walk, 0);
223 }
224
225 return err;
226}
227EXPORT_SYMBOL_GPL(glue_ctr_req_128bit);
228
229static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx,
230 void *ctx,
231 struct skcipher_walk *walk)
232{
233 const unsigned int bsize = 128 / 8;
234 unsigned int nbytes = walk->nbytes;
235 u128 *src = walk->src.virt.addr;
236 u128 *dst = walk->dst.virt.addr;
237 unsigned int num_blocks, func_bytes;
238 unsigned int i;
239
240
241 for (i = 0; i < gctx->num_funcs; i++) {
242 num_blocks = gctx->funcs[i].num_blocks;
243 func_bytes = bsize * num_blocks;
244
245 if (nbytes >= func_bytes) {
246 do {
247 gctx->funcs[i].fn_u.xts(ctx, (u8 *)dst,
248 (const u8 *)src,
249 walk->iv);
250
251 src += num_blocks;
252 dst += num_blocks;
253 nbytes -= func_bytes;
254 } while (nbytes >= func_bytes);
255
256 if (nbytes < bsize)
257 goto done;
258 }
259 }
260
261done:
262 return nbytes;
263}
264
265int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
266 struct skcipher_request *req,
267 common_glue_func_t tweak_fn, void *tweak_ctx,
268 void *crypt_ctx, bool decrypt)
269{
270 const bool cts = (req->cryptlen % XTS_BLOCK_SIZE);
271 const unsigned int bsize = 128 / 8;
272 struct skcipher_request subreq;
273 struct skcipher_walk walk;
274 bool fpu_enabled = false;
275 unsigned int nbytes, tail;
276 int err;
277
278 if (req->cryptlen < XTS_BLOCK_SIZE)
279 return -EINVAL;
280
281 if (unlikely(cts)) {
282 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
283
284 tail = req->cryptlen % XTS_BLOCK_SIZE + XTS_BLOCK_SIZE;
285
286 skcipher_request_set_tfm(&subreq, tfm);
287 skcipher_request_set_callback(&subreq,
288 crypto_skcipher_get_flags(tfm),
289 NULL, NULL);
290 skcipher_request_set_crypt(&subreq, req->src, req->dst,
291 req->cryptlen - tail, req->iv);
292 req = &subreq;
293 }
294
295 err = skcipher_walk_virt(&walk, req, false);
296 nbytes = walk.nbytes;
297 if (err)
298 return err;
299
300
301 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
302 &walk, fpu_enabled,
303 nbytes < bsize ? bsize : nbytes);
304
305
306 tweak_fn(tweak_ctx, walk.iv, walk.iv);
307
308 while (nbytes) {
309 nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk);
310
311 err = skcipher_walk_done(&walk, nbytes);
312 nbytes = walk.nbytes;
313 }
314
315 if (unlikely(cts)) {
316 u8 *next_tweak, *final_tweak = req->iv;
317 struct scatterlist *src, *dst;
318 struct scatterlist s[2], d[2];
319 le128 b[2];
320
321 dst = src = scatterwalk_ffwd(s, req->src, req->cryptlen);
322 if (req->dst != req->src)
323 dst = scatterwalk_ffwd(d, req->dst, req->cryptlen);
324
325 if (decrypt) {
326 next_tweak = memcpy(b, req->iv, XTS_BLOCK_SIZE);
327 gf128mul_x_ble(b, b);
328 } else {
329 next_tweak = req->iv;
330 }
331
332 skcipher_request_set_crypt(&subreq, src, dst, XTS_BLOCK_SIZE,
333 next_tweak);
334
335 err = skcipher_walk_virt(&walk, req, false) ?:
336 skcipher_walk_done(&walk,
337 __glue_xts_req_128bit(gctx, crypt_ctx, &walk));
338 if (err)
339 goto out;
340
341 scatterwalk_map_and_copy(b, dst, 0, XTS_BLOCK_SIZE, 0);
342 memcpy(b + 1, b, tail - XTS_BLOCK_SIZE);
343 scatterwalk_map_and_copy(b, src, XTS_BLOCK_SIZE,
344 tail - XTS_BLOCK_SIZE, 0);
345 scatterwalk_map_and_copy(b, dst, 0, tail, 1);
346
347 skcipher_request_set_crypt(&subreq, dst, dst, XTS_BLOCK_SIZE,
348 final_tweak);
349
350 err = skcipher_walk_virt(&walk, req, false) ?:
351 skcipher_walk_done(&walk,
352 __glue_xts_req_128bit(gctx, crypt_ctx, &walk));
353 }
354
355out:
356 glue_fpu_end(fpu_enabled);
357
358 return err;
359}
360EXPORT_SYMBOL_GPL(glue_xts_req_128bit);
361
362void glue_xts_crypt_128bit_one(const void *ctx, u8 *dst, const u8 *src,
363 le128 *iv, common_glue_func_t fn)
364{
365 le128 ivblk = *iv;
366
367
368 gf128mul_x_ble(iv, &ivblk);
369
370
371 u128_xor((u128 *)dst, (const u128 *)src, (u128 *)&ivblk);
372
373
374 fn(ctx, dst, dst);
375
376
377 u128_xor((u128 *)dst, (u128 *)dst, (u128 *)&ivblk);
378}
379EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);
380
381MODULE_LICENSE("GPL");
382
