1
2
3
4
5
6
7
8#include <crypto/gf128mul.h>
9#include <crypto/internal/aead.h>
10#include <crypto/internal/skcipher.h>
11#include <crypto/internal/hash.h>
12#include <crypto/null.h>
13#include <crypto/scatterwalk.h>
14#include <crypto/gcm.h>
15#include <crypto/hash.h>
16#include "internal.h"
17#include <linux/err.h>
18#include <linux/init.h>
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/slab.h>
22
23struct gcm_instance_ctx {
24 struct crypto_skcipher_spawn ctr;
25 struct crypto_ahash_spawn ghash;
26};
27
28struct crypto_gcm_ctx {
29 struct crypto_skcipher *ctr;
30 struct crypto_ahash *ghash;
31};
32
33struct crypto_rfc4106_ctx {
34 struct crypto_aead *child;
35 u8 nonce[4];
36};
37
38struct crypto_rfc4106_req_ctx {
39 struct scatterlist src[3];
40 struct scatterlist dst[3];
41 struct aead_request subreq;
42};
43
44struct crypto_rfc4543_instance_ctx {
45 struct crypto_aead_spawn aead;
46};
47
48struct crypto_rfc4543_ctx {
49 struct crypto_aead *child;
50 struct crypto_sync_skcipher *null;
51 u8 nonce[4];
52};
53
54struct crypto_rfc4543_req_ctx {
55 struct aead_request subreq;
56};
57
58struct crypto_gcm_ghash_ctx {
59 unsigned int cryptlen;
60 struct scatterlist *src;
61 int (*complete)(struct aead_request *req, u32 flags);
62};
63
64struct crypto_gcm_req_priv_ctx {
65 u8 iv[16];
66 u8 auth_tag[16];
67 u8 iauth_tag[16];
68 struct scatterlist src[3];
69 struct scatterlist dst[3];
70 struct scatterlist sg;
71 struct crypto_gcm_ghash_ctx ghash_ctx;
72 union {
73 struct ahash_request ahreq;
74 struct skcipher_request skreq;
75 } u;
76};
77
78static struct {
79 u8 buf[16];
80 struct scatterlist sg;
81} *gcm_zeroes;
82
83static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
84
85static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
86 struct aead_request *req)
87{
88 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
89
90 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
91}
92
93static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
94 unsigned int keylen)
95{
96 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
97 struct crypto_ahash *ghash = ctx->ghash;
98 struct crypto_skcipher *ctr = ctx->ctr;
99 struct {
100 be128 hash;
101 u8 iv[16];
102
103 struct crypto_wait wait;
104
105 struct scatterlist sg[1];
106 struct skcipher_request req;
107 } *data;
108 int err;
109
110 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
111 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
112 CRYPTO_TFM_REQ_MASK);
113 err = crypto_skcipher_setkey(ctr, key, keylen);
114 crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
115 CRYPTO_TFM_RES_MASK);
116 if (err)
117 return err;
118
119 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
120 GFP_KERNEL);
121 if (!data)
122 return -ENOMEM;
123
124 crypto_init_wait(&data->wait);
125 sg_init_one(data->sg, &data->hash, sizeof(data->hash));
126 skcipher_request_set_tfm(&data->req, ctr);
127 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
128 CRYPTO_TFM_REQ_MAY_BACKLOG,
129 crypto_req_done,
130 &data->wait);
131 skcipher_request_set_crypt(&data->req, data->sg, data->sg,
132 sizeof(data->hash), data->iv);
133
134 err = crypto_wait_req(crypto_skcipher_encrypt(&data->req),
135 &data->wait);
136
137 if (err)
138 goto out;
139
140 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);
141 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &
142 CRYPTO_TFM_REQ_MASK);
143 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));
144 crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) &
145 CRYPTO_TFM_RES_MASK);
146
147out:
148 kzfree(data);
149 return err;
150}
151
152static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
153 unsigned int authsize)
154{
155 switch (authsize) {
156 case 4:
157 case 8:
158 case 12:
159 case 13:
160 case 14:
161 case 15:
162 case 16:
163 break;
164 default:
165 return -EINVAL;
166 }
167
168 return 0;
169}
170
171static void crypto_gcm_init_common(struct aead_request *req)
172{
173 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
174 __be32 counter = cpu_to_be32(1);
175 struct scatterlist *sg;
176
177 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
178 memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE);
179 memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4);
180
181 sg_init_table(pctx->src, 3);
182 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
183 sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);
184 if (sg != pctx->src + 1)
185 sg_chain(pctx->src, 2, sg);
186
187 if (req->src != req->dst) {
188 sg_init_table(pctx->dst, 3);
189 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
190 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);
191 if (sg != pctx->dst + 1)
192 sg_chain(pctx->dst, 2, sg);
193 }
194}
195
196static void crypto_gcm_init_crypt(struct aead_request *req,
197 unsigned int cryptlen)
198{
199 struct crypto_aead *aead = crypto_aead_reqtfm(req);
200 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
201 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
202 struct skcipher_request *skreq = &pctx->u.skreq;
203 struct scatterlist *dst;
204
205 dst = req->src == req->dst ? pctx->src : pctx->dst;
206
207 skcipher_request_set_tfm(skreq, ctx->ctr);
208 skcipher_request_set_crypt(skreq, pctx->src, dst,
209 cryptlen + sizeof(pctx->auth_tag),
210 pctx->iv);
211}
212
213static inline unsigned int gcm_remain(unsigned int len)
214{
215 len &= 0xfU;
216 return len ? 16 - len : 0;
217}
218
219static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
220
221static int gcm_hash_update(struct aead_request *req,
222 crypto_completion_t compl,
223 struct scatterlist *src,
224 unsigned int len, u32 flags)
225{
226 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
227 struct ahash_request *ahreq = &pctx->u.ahreq;
228
229 ahash_request_set_callback(ahreq, flags, compl, req);
230 ahash_request_set_crypt(ahreq, src, NULL, len);
231
232 return crypto_ahash_update(ahreq);
233}
234
235static int gcm_hash_remain(struct aead_request *req,
236 unsigned int remain,
237 crypto_completion_t compl, u32 flags)
238{
239 return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);
240}
241
242static int gcm_hash_len(struct aead_request *req, u32 flags)
243{
244 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
245 struct ahash_request *ahreq = &pctx->u.ahreq;
246 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
247 be128 lengths;
248
249 lengths.a = cpu_to_be64(req->assoclen * 8);
250 lengths.b = cpu_to_be64(gctx->cryptlen * 8);
251 memcpy(pctx->iauth_tag, &lengths, 16);
252 sg_init_one(&pctx->sg, pctx->iauth_tag, 16);
253 ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);
254 ahash_request_set_crypt(ahreq, &pctx->sg,
255 pctx->iauth_tag, sizeof(lengths));
256
257 return crypto_ahash_finup(ahreq);
258}
259
260static int gcm_hash_len_continue(struct aead_request *req, u32 flags)
261{
262 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
263 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
264
265 return gctx->complete(req, flags);
266}
267
268static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
269{
270 struct aead_request *req = areq->data;
271
272 if (err)
273 goto out;
274
275 err = gcm_hash_len_continue(req, 0);
276 if (err == -EINPROGRESS)
277 return;
278
279out:
280 aead_request_complete(req, err);
281}
282
283static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)
284{
285 return gcm_hash_len(req, flags) ?:
286 gcm_hash_len_continue(req, flags);
287}
288
289static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
290 int err)
291{
292 struct aead_request *req = areq->data;
293
294 if (err)
295 goto out;
296
297 err = gcm_hash_crypt_remain_continue(req, 0);
298 if (err == -EINPROGRESS)
299 return;
300
301out:
302 aead_request_complete(req, err);
303}
304
305static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)
306{
307 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
308 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
309 unsigned int remain;
310
311 remain = gcm_remain(gctx->cryptlen);
312 if (remain)
313 return gcm_hash_remain(req, remain,
314 gcm_hash_crypt_remain_done, flags) ?:
315 gcm_hash_crypt_remain_continue(req, flags);
316
317 return gcm_hash_crypt_remain_continue(req, flags);
318}
319
320static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
321{
322 struct aead_request *req = areq->data;
323
324 if (err)
325 goto out;
326
327 err = gcm_hash_crypt_continue(req, 0);
328 if (err == -EINPROGRESS)
329 return;
330
331out:
332 aead_request_complete(req, err);
333}
334
335static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)
336{
337 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
338 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
339
340 if (gctx->cryptlen)
341 return gcm_hash_update(req, gcm_hash_crypt_done,
342 gctx->src, gctx->cryptlen, flags) ?:
343 gcm_hash_crypt_continue(req, flags);
344
345 return gcm_hash_crypt_remain_continue(req, flags);
346}
347
348static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
349 int err)
350{
351 struct aead_request *req = areq->data;
352
353 if (err)
354 goto out;
355
356 err = gcm_hash_assoc_remain_continue(req, 0);
357 if (err == -EINPROGRESS)
358 return;
359
360out:
361 aead_request_complete(req, err);
362}
363
364static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)
365{
366 unsigned int remain;
367
368 remain = gcm_remain(req->assoclen);
369 if (remain)
370 return gcm_hash_remain(req, remain,
371 gcm_hash_assoc_remain_done, flags) ?:
372 gcm_hash_assoc_remain_continue(req, flags);
373
374 return gcm_hash_assoc_remain_continue(req, flags);
375}
376
377static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
378{
379 struct aead_request *req = areq->data;
380
381 if (err)
382 goto out;
383
384 err = gcm_hash_assoc_continue(req, 0);
385 if (err == -EINPROGRESS)
386 return;
387
388out:
389 aead_request_complete(req, err);
390}
391
392static int gcm_hash_init_continue(struct aead_request *req, u32 flags)
393{
394 if (req->assoclen)
395 return gcm_hash_update(req, gcm_hash_assoc_done,
396 req->src, req->assoclen, flags) ?:
397 gcm_hash_assoc_continue(req, flags);
398
399 return gcm_hash_assoc_remain_continue(req, flags);
400}
401
402static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
403{
404 struct aead_request *req = areq->data;
405
406 if (err)
407 goto out;
408
409 err = gcm_hash_init_continue(req, 0);
410 if (err == -EINPROGRESS)
411 return;
412
413out:
414 aead_request_complete(req, err);
415}
416
417static int gcm_hash(struct aead_request *req, u32 flags)
418{
419 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
420 struct ahash_request *ahreq = &pctx->u.ahreq;
421 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
422
423 ahash_request_set_tfm(ahreq, ctx->ghash);
424
425 ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);
426 return crypto_ahash_init(ahreq) ?:
427 gcm_hash_init_continue(req, flags);
428}
429
430static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)
431{
432 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
433 struct crypto_aead *aead = crypto_aead_reqtfm(req);
434 u8 *auth_tag = pctx->auth_tag;
435
436 crypto_xor(auth_tag, pctx->iauth_tag, 16);
437 scatterwalk_map_and_copy(auth_tag, req->dst,
438 req->assoclen + req->cryptlen,
439 crypto_aead_authsize(aead), 1);
440 return 0;
441}
442
443static int gcm_encrypt_continue(struct aead_request *req, u32 flags)
444{
445 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
446 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
447
448 gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);
449 gctx->cryptlen = req->cryptlen;
450 gctx->complete = gcm_enc_copy_hash;
451
452 return gcm_hash(req, flags);
453}
454
455static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
456{
457 struct aead_request *req = areq->data;
458
459 if (err)
460 goto out;
461
462 err = gcm_encrypt_continue(req, 0);
463 if (err == -EINPROGRESS)
464 return;
465
466out:
467 aead_request_complete(req, err);
468}
469
470static int crypto_gcm_encrypt(struct aead_request *req)
471{
472 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
473 struct skcipher_request *skreq = &pctx->u.skreq;
474 u32 flags = aead_request_flags(req);
475
476 crypto_gcm_init_common(req);
477 crypto_gcm_init_crypt(req, req->cryptlen);
478 skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);
479
480 return crypto_skcipher_encrypt(skreq) ?:
481 gcm_encrypt_continue(req, flags);
482}
483
484static int crypto_gcm_verify(struct aead_request *req)
485{
486 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
487 struct crypto_aead *aead = crypto_aead_reqtfm(req);
488 u8 *auth_tag = pctx->auth_tag;
489 u8 *iauth_tag = pctx->iauth_tag;
490 unsigned int authsize = crypto_aead_authsize(aead);
491 unsigned int cryptlen = req->cryptlen - authsize;
492
493 crypto_xor(auth_tag, iauth_tag, 16);
494 scatterwalk_map_and_copy(iauth_tag, req->src,
495 req->assoclen + cryptlen, authsize, 0);
496 return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
497}
498
499static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
500{
501 struct aead_request *req = areq->data;
502
503 if (!err)
504 err = crypto_gcm_verify(req);
505
506 aead_request_complete(req, err);
507}
508
509static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)
510{
511 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
512 struct skcipher_request *skreq = &pctx->u.skreq;
513 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
514
515 crypto_gcm_init_crypt(req, gctx->cryptlen);
516 skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);
517 return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);
518}
519
520static int crypto_gcm_decrypt(struct aead_request *req)
521{
522 struct crypto_aead *aead = crypto_aead_reqtfm(req);
523 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
524 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
525 unsigned int authsize = crypto_aead_authsize(aead);
526 unsigned int cryptlen = req->cryptlen;
527 u32 flags = aead_request_flags(req);
528
529 cryptlen -= authsize;
530
531 crypto_gcm_init_common(req);
532
533 gctx->src = sg_next(pctx->src);
534 gctx->cryptlen = cryptlen;
535 gctx->complete = gcm_dec_hash_continue;
536
537 return gcm_hash(req, flags);
538}
539
540static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
541{
542 struct aead_instance *inst = aead_alg_instance(tfm);
543 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);
544 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
545 struct crypto_skcipher *ctr;
546 struct crypto_ahash *ghash;
547 unsigned long align;
548 int err;
549
550 ghash = crypto_spawn_ahash(&ictx->ghash);
551 if (IS_ERR(ghash))
552 return PTR_ERR(ghash);
553
554 ctr = crypto_spawn_skcipher(&ictx->ctr);
555 err = PTR_ERR(ctr);
556 if (IS_ERR(ctr))
557 goto err_free_hash;
558
559 ctx->ctr = ctr;
560 ctx->ghash = ghash;
561
562 align = crypto_aead_alignmask(tfm);
563 align &= ~(crypto_tfm_ctx_alignment() - 1);
564 crypto_aead_set_reqsize(tfm,
565 align + offsetof(struct crypto_gcm_req_priv_ctx, u) +
566 max(sizeof(struct skcipher_request) +
567 crypto_skcipher_reqsize(ctr),
568 sizeof(struct ahash_request) +
569 crypto_ahash_reqsize(ghash)));
570
571 return 0;
572
573err_free_hash:
574 crypto_free_ahash(ghash);
575 return err;
576}
577
578static void crypto_gcm_exit_tfm(struct crypto_aead *tfm)
579{
580 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
581
582 crypto_free_ahash(ctx->ghash);
583 crypto_free_skcipher(ctx->ctr);
584}
585
586static void crypto_gcm_free(struct aead_instance *inst)
587{
588 struct gcm_instance_ctx *ctx = aead_instance_ctx(inst);
589
590 crypto_drop_skcipher(&ctx->ctr);
591 crypto_drop_ahash(&ctx->ghash);
592 kfree(inst);
593}
594
595static int crypto_gcm_create_common(struct crypto_template *tmpl,
596 struct rtattr **tb,
597 const char *ctr_name,
598 const char *ghash_name)
599{
600 struct crypto_attr_type *algt;
601 struct aead_instance *inst;
602 struct skcipher_alg *ctr;
603 struct crypto_alg *ghash_alg;
604 struct hash_alg_common *ghash;
605 struct gcm_instance_ctx *ctx;
606 int err;
607
608 algt = crypto_get_attr_type(tb);
609 if (IS_ERR(algt))
610 return PTR_ERR(algt);
611
612 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
613 return -EINVAL;
614
615 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
616 CRYPTO_ALG_TYPE_HASH,
617 CRYPTO_ALG_TYPE_AHASH_MASK |
618 crypto_requires_sync(algt->type,
619 algt->mask));
620 if (IS_ERR(ghash_alg))
621 return PTR_ERR(ghash_alg);
622
623 ghash = __crypto_hash_alg_common(ghash_alg);
624
625 err = -ENOMEM;
626 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
627 if (!inst)
628 goto out_put_ghash;
629
630 ctx = aead_instance_ctx(inst);
631 err = crypto_init_ahash_spawn(&ctx->ghash, ghash,
632 aead_crypto_instance(inst));
633 if (err)
634 goto err_free_inst;
635
636 err = -EINVAL;
637 if (strcmp(ghash->base.cra_name, "ghash") != 0 ||
638 ghash->digestsize != 16)
639 goto err_drop_ghash;
640
641 crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst));
642 err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
643 crypto_requires_sync(algt->type,
644 algt->mask));
645 if (err)
646 goto err_drop_ghash;
647
648 ctr = crypto_spawn_skcipher_alg(&ctx->ctr);
649
650
651 err = -EINVAL;
652 if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 ||
653 crypto_skcipher_alg_ivsize(ctr) != 16 ||
654 ctr->base.cra_blocksize != 1)
655 goto out_put_ctr;
656
657 err = -ENAMETOOLONG;
658 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
659 "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME)
660 goto out_put_ctr;
661
662 if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
663 "gcm_base(%s,%s)", ctr->base.cra_driver_name,
664 ghash_alg->cra_driver_name) >=
665 CRYPTO_MAX_ALG_NAME)
666 goto out_put_ctr;
667
668 inst->alg.base.cra_flags = (ghash->base.cra_flags |
669 ctr->base.cra_flags) & CRYPTO_ALG_ASYNC;
670 inst->alg.base.cra_priority = (ghash->base.cra_priority +
671 ctr->base.cra_priority) / 2;
672 inst->alg.base.cra_blocksize = 1;
673 inst->alg.base.cra_alignmask = ghash->base.cra_alignmask |
674 ctr->base.cra_alignmask;
675 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
676 inst->alg.ivsize = GCM_AES_IV_SIZE;
677 inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
678 inst->alg.maxauthsize = 16;
679 inst->alg.init = crypto_gcm_init_tfm;
680 inst->alg.exit = crypto_gcm_exit_tfm;
681 inst->alg.setkey = crypto_gcm_setkey;
682 inst->alg.setauthsize = crypto_gcm_setauthsize;
683 inst->alg.encrypt = crypto_gcm_encrypt;
684 inst->alg.decrypt = crypto_gcm_decrypt;
685
686 inst->free = crypto_gcm_free;
687
688 err = aead_register_instance(tmpl, inst);
689 if (err)
690 goto out_put_ctr;
691
692out_put_ghash:
693 crypto_mod_put(ghash_alg);
694 return err;
695
696out_put_ctr:
697 crypto_drop_skcipher(&ctx->ctr);
698err_drop_ghash:
699 crypto_drop_ahash(&ctx->ghash);
700err_free_inst:
701 kfree(inst);
702 goto out_put_ghash;
703}
704
705static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb)
706{
707 const char *cipher_name;
708 char ctr_name[CRYPTO_MAX_ALG_NAME];
709
710 cipher_name = crypto_attr_alg_name(tb[1]);
711 if (IS_ERR(cipher_name))
712 return PTR_ERR(cipher_name);
713
714 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
715 CRYPTO_MAX_ALG_NAME)
716 return -ENAMETOOLONG;
717
718 return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash");
719}
720
721static int crypto_gcm_base_create(struct crypto_template *tmpl,
722 struct rtattr **tb)
723{
724 const char *ctr_name;
725 const char *ghash_name;
726
727 ctr_name = crypto_attr_alg_name(tb[1]);
728 if (IS_ERR(ctr_name))
729 return PTR_ERR(ctr_name);
730
731 ghash_name = crypto_attr_alg_name(tb[2]);
732 if (IS_ERR(ghash_name))
733 return PTR_ERR(ghash_name);
734
735 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name);
736}
737
738static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
739 unsigned int keylen)
740{
741 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
742 struct crypto_aead *child = ctx->child;
743 int err;
744
745 if (keylen < 4)
746 return -EINVAL;
747
748 keylen -= 4;
749 memcpy(ctx->nonce, key + keylen, 4);
750
751 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
752 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
753 CRYPTO_TFM_REQ_MASK);
754 err = crypto_aead_setkey(child, key, keylen);
755 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
756 CRYPTO_TFM_RES_MASK);
757
758 return err;
759}
760
761static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
762 unsigned int authsize)
763{
764 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
765
766 switch (authsize) {
767 case 8:
768 case 12:
769 case 16:
770 break;
771 default:
772 return -EINVAL;
773 }
774
775 return crypto_aead_setauthsize(ctx->child, authsize);
776}
777
778static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
779{
780 struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);
781 struct crypto_aead *aead = crypto_aead_reqtfm(req);
782 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);
783 struct aead_request *subreq = &rctx->subreq;
784 struct crypto_aead *child = ctx->child;
785 struct scatterlist *sg;
786 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
787 crypto_aead_alignmask(child) + 1);
788
789 scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0);
790
791 memcpy(iv, ctx->nonce, 4);
792 memcpy(iv + 4, req->iv, 8);
793
794 sg_init_table(rctx->src, 3);
795 sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8);
796 sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);
797 if (sg != rctx->src + 1)
798 sg_chain(rctx->src, 2, sg);
799
800 if (req->src != req->dst) {
801 sg_init_table(rctx->dst, 3);
802 sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8);
803 sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);
804 if (sg != rctx->dst + 1)
805 sg_chain(rctx->dst, 2, sg);
806 }
807
808 aead_request_set_tfm(subreq, child);
809 aead_request_set_callback(subreq, req->base.flags, req->base.complete,
810 req->base.data);
811 aead_request_set_crypt(subreq, rctx->src,
812 req->src == req->dst ? rctx->src : rctx->dst,
813 req->cryptlen, iv);
814 aead_request_set_ad(subreq, req->assoclen - 8);
815
816 return subreq;
817}
818
819static int crypto_rfc4106_encrypt(struct aead_request *req)
820{
821 if (req->assoclen != 16 && req->assoclen != 20)
822 return -EINVAL;
823
824 req = crypto_rfc4106_crypt(req);
825
826 return crypto_aead_encrypt(req);
827}
828
829static int crypto_rfc4106_decrypt(struct aead_request *req)
830{
831 if (req->assoclen != 16 && req->assoclen != 20)
832 return -EINVAL;
833
834 req = crypto_rfc4106_crypt(req);
835
836 return crypto_aead_decrypt(req);
837}
838
839static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm)
840{
841 struct aead_instance *inst = aead_alg_instance(tfm);
842 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst);
843 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
844 struct crypto_aead *aead;
845 unsigned long align;
846
847 aead = crypto_spawn_aead(spawn);
848 if (IS_ERR(aead))
849 return PTR_ERR(aead);
850
851 ctx->child = aead;
852
853 align = crypto_aead_alignmask(aead);
854 align &= ~(crypto_tfm_ctx_alignment() - 1);
855 crypto_aead_set_reqsize(
856 tfm,
857 sizeof(struct crypto_rfc4106_req_ctx) +
858 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
859 align + 24);
860
861 return 0;
862}
863
864static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm)
865{
866 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
867
868 crypto_free_aead(ctx->child);
869}
870
871static void crypto_rfc4106_free(struct aead_instance *inst)
872{
873 crypto_drop_aead(aead_instance_ctx(inst));
874 kfree(inst);
875}
876
877static int crypto_rfc4106_create(struct crypto_template *tmpl,
878 struct rtattr **tb)
879{
880 struct crypto_attr_type *algt;
881 struct aead_instance *inst;
882 struct crypto_aead_spawn *spawn;
883 struct aead_alg *alg;
884 const char *ccm_name;
885 int err;
886
887 algt = crypto_get_attr_type(tb);
888 if (IS_ERR(algt))
889 return PTR_ERR(algt);
890
891 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
892 return -EINVAL;
893
894 ccm_name = crypto_attr_alg_name(tb[1]);
895 if (IS_ERR(ccm_name))
896 return PTR_ERR(ccm_name);
897
898 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
899 if (!inst)
900 return -ENOMEM;
901
902 spawn = aead_instance_ctx(inst);
903 crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
904 err = crypto_grab_aead(spawn, ccm_name, 0,
905 crypto_requires_sync(algt->type, algt->mask));
906 if (err)
907 goto out_free_inst;
908
909 alg = crypto_spawn_aead_alg(spawn);
910
911 err = -EINVAL;
912
913
914 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE)
915 goto out_drop_alg;
916
917
918 if (alg->base.cra_blocksize != 1)
919 goto out_drop_alg;
920
921 err = -ENAMETOOLONG;
922 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
923 "rfc4106(%s)", alg->base.cra_name) >=
924 CRYPTO_MAX_ALG_NAME ||
925 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
926 "rfc4106(%s)", alg->base.cra_driver_name) >=
927 CRYPTO_MAX_ALG_NAME)
928 goto out_drop_alg;
929
930 inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
931 inst->alg.base.cra_priority = alg->base.cra_priority;
932 inst->alg.base.cra_blocksize = 1;
933 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
934
935 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
936
937 inst->alg.ivsize = GCM_RFC4106_IV_SIZE;
938 inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
939 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
940
941 inst->alg.init = crypto_rfc4106_init_tfm;
942 inst->alg.exit = crypto_rfc4106_exit_tfm;
943
944 inst->alg.setkey = crypto_rfc4106_setkey;
945 inst->alg.setauthsize = crypto_rfc4106_setauthsize;
946 inst->alg.encrypt = crypto_rfc4106_encrypt;
947 inst->alg.decrypt = crypto_rfc4106_decrypt;
948
949 inst->free = crypto_rfc4106_free;
950
951 err = aead_register_instance(tmpl, inst);
952 if (err)
953 goto out_drop_alg;
954
955out:
956 return err;
957
958out_drop_alg:
959 crypto_drop_aead(spawn);
960out_free_inst:
961 kfree(inst);
962 goto out;
963}
964
965static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key,
966 unsigned int keylen)
967{
968 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
969 struct crypto_aead *child = ctx->child;
970 int err;
971
972 if (keylen < 4)
973 return -EINVAL;
974
975 keylen -= 4;
976 memcpy(ctx->nonce, key + keylen, 4);
977
978 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
979 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
980 CRYPTO_TFM_REQ_MASK);
981 err = crypto_aead_setkey(child, key, keylen);
982 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
983 CRYPTO_TFM_RES_MASK);
984
985 return err;
986}
987
988static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,
989 unsigned int authsize)
990{
991 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
992
993 if (authsize != 16)
994 return -EINVAL;
995
996 return crypto_aead_setauthsize(ctx->child, authsize);
997}
998
999static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)
1000{
1001 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1002 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
1003 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);
1004 struct aead_request *subreq = &rctx->subreq;
1005 unsigned int authsize = crypto_aead_authsize(aead);
1006 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
1007 crypto_aead_alignmask(ctx->child) + 1);
1008 int err;
1009
1010 if (req->src != req->dst) {
1011 err = crypto_rfc4543_copy_src_to_dst(req, enc);
1012 if (err)
1013 return err;
1014 }
1015
1016 memcpy(iv, ctx->nonce, 4);
1017 memcpy(iv + 4, req->iv, 8);
1018
1019 aead_request_set_tfm(subreq, ctx->child);
1020 aead_request_set_callback(subreq, req->base.flags,
1021 req->base.complete, req->base.data);
1022 aead_request_set_crypt(subreq, req->src, req->dst,
1023 enc ? 0 : authsize, iv);
1024 aead_request_set_ad(subreq, req->assoclen + req->cryptlen -
1025 subreq->cryptlen);
1026
1027 return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);
1028}
1029
1030static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
1031{
1032 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1033 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
1034 unsigned int authsize = crypto_aead_authsize(aead);
1035 unsigned int nbytes = req->assoclen + req->cryptlen -
1036 (enc ? 0 : authsize);
1037 SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);
1038
1039 skcipher_request_set_sync_tfm(nreq, ctx->null);
1040 skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);
1041 skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);
1042
1043 return crypto_skcipher_encrypt(nreq);
1044}
1045
1046static int crypto_rfc4543_encrypt(struct aead_request *req)
1047{
1048 return crypto_rfc4543_crypt(req, true);
1049}
1050
1051static int crypto_rfc4543_decrypt(struct aead_request *req)
1052{
1053 return crypto_rfc4543_crypt(req, false);
1054}
1055
1056static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
1057{
1058 struct aead_instance *inst = aead_alg_instance(tfm);
1059 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst);
1060 struct crypto_aead_spawn *spawn = &ictx->aead;
1061 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1062 struct crypto_aead *aead;
1063 struct crypto_sync_skcipher *null;
1064 unsigned long align;
1065 int err = 0;
1066
1067 aead = crypto_spawn_aead(spawn);
1068 if (IS_ERR(aead))
1069 return PTR_ERR(aead);
1070
1071 null = crypto_get_default_null_skcipher();
1072 err = PTR_ERR(null);
1073 if (IS_ERR(null))
1074 goto err_free_aead;
1075
1076 ctx->child = aead;
1077 ctx->null = null;
1078
1079 align = crypto_aead_alignmask(aead);
1080 align &= ~(crypto_tfm_ctx_alignment() - 1);
1081 crypto_aead_set_reqsize(
1082 tfm,
1083 sizeof(struct crypto_rfc4543_req_ctx) +
1084 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
1085 align + GCM_AES_IV_SIZE);
1086
1087 return 0;
1088
1089err_free_aead:
1090 crypto_free_aead(aead);
1091 return err;
1092}
1093
1094static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)
1095{
1096 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1097
1098 crypto_free_aead(ctx->child);
1099 crypto_put_default_null_skcipher();
1100}
1101
1102static void crypto_rfc4543_free(struct aead_instance *inst)
1103{
1104 struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst);
1105
1106 crypto_drop_aead(&ctx->aead);
1107
1108 kfree(inst);
1109}
1110
1111static int crypto_rfc4543_create(struct crypto_template *tmpl,
1112 struct rtattr **tb)
1113{
1114 struct crypto_attr_type *algt;
1115 struct aead_instance *inst;
1116 struct crypto_aead_spawn *spawn;
1117 struct aead_alg *alg;
1118 struct crypto_rfc4543_instance_ctx *ctx;
1119 const char *ccm_name;
1120 int err;
1121
1122 algt = crypto_get_attr_type(tb);
1123 if (IS_ERR(algt))
1124 return PTR_ERR(algt);
1125
1126 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
1127 return -EINVAL;
1128
1129 ccm_name = crypto_attr_alg_name(tb[1]);
1130 if (IS_ERR(ccm_name))
1131 return PTR_ERR(ccm_name);
1132
1133 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
1134 if (!inst)
1135 return -ENOMEM;
1136
1137 ctx = aead_instance_ctx(inst);
1138 spawn = &ctx->aead;
1139 crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
1140 err = crypto_grab_aead(spawn, ccm_name, 0,
1141 crypto_requires_sync(algt->type, algt->mask));
1142 if (err)
1143 goto out_free_inst;
1144
1145 alg = crypto_spawn_aead_alg(spawn);
1146
1147 err = -EINVAL;
1148
1149
1150 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE)
1151 goto out_drop_alg;
1152
1153
1154 if (alg->base.cra_blocksize != 1)
1155 goto out_drop_alg;
1156
1157 err = -ENAMETOOLONG;
1158 if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
1159 "rfc4543(%s)", alg->base.cra_name) >=
1160 CRYPTO_MAX_ALG_NAME ||
1161 snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1162 "rfc4543(%s)", alg->base.cra_driver_name) >=
1163 CRYPTO_MAX_ALG_NAME)
1164 goto out_drop_alg;
1165
1166 inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
1167 inst->alg.base.cra_priority = alg->base.cra_priority;
1168 inst->alg.base.cra_blocksize = 1;
1169 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
1170
1171 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);
1172
1173 inst->alg.ivsize = GCM_RFC4543_IV_SIZE;
1174 inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
1175 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
1176
1177 inst->alg.init = crypto_rfc4543_init_tfm;
1178 inst->alg.exit = crypto_rfc4543_exit_tfm;
1179
1180 inst->alg.setkey = crypto_rfc4543_setkey;
1181 inst->alg.setauthsize = crypto_rfc4543_setauthsize;
1182 inst->alg.encrypt = crypto_rfc4543_encrypt;
1183 inst->alg.decrypt = crypto_rfc4543_decrypt;
1184
1185 inst->free = crypto_rfc4543_free,
1186
1187 err = aead_register_instance(tmpl, inst);
1188 if (err)
1189 goto out_drop_alg;
1190
1191out:
1192 return err;
1193
1194out_drop_alg:
1195 crypto_drop_aead(spawn);
1196out_free_inst:
1197 kfree(inst);
1198 goto out;
1199}
1200
1201static struct crypto_template crypto_gcm_tmpls[] = {
1202 {
1203 .name = "gcm_base",
1204 .create = crypto_gcm_base_create,
1205 .module = THIS_MODULE,
1206 }, {
1207 .name = "gcm",
1208 .create = crypto_gcm_create,
1209 .module = THIS_MODULE,
1210 }, {
1211 .name = "rfc4106",
1212 .create = crypto_rfc4106_create,
1213 .module = THIS_MODULE,
1214 }, {
1215 .name = "rfc4543",
1216 .create = crypto_rfc4543_create,
1217 .module = THIS_MODULE,
1218 },
1219};
1220
1221static int __init crypto_gcm_module_init(void)
1222{
1223 int err;
1224
1225 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL);
1226 if (!gcm_zeroes)
1227 return -ENOMEM;
1228
1229 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf));
1230
1231 err = crypto_register_templates(crypto_gcm_tmpls,
1232 ARRAY_SIZE(crypto_gcm_tmpls));
1233 if (err)
1234 kfree(gcm_zeroes);
1235
1236 return err;
1237}
1238
1239static void __exit crypto_gcm_module_exit(void)
1240{
1241 kfree(gcm_zeroes);
1242 crypto_unregister_templates(crypto_gcm_tmpls,
1243 ARRAY_SIZE(crypto_gcm_tmpls));
1244}
1245
1246subsys_initcall(crypto_gcm_module_init);
1247module_exit(crypto_gcm_module_exit);
1248
1249MODULE_LICENSE("GPL");
1250MODULE_DESCRIPTION("Galois/Counter Mode");
1251MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
1252MODULE_ALIAS_CRYPTO("gcm_base");
1253MODULE_ALIAS_CRYPTO("rfc4106");
1254MODULE_ALIAS_CRYPTO("rfc4543");
1255MODULE_ALIAS_CRYPTO("gcm");
1256
