1
2
3
4
5
6
7
8
9
10
11#include <crypto/gf128mul.h>
12#include <crypto/internal/aead.h>
13#include <crypto/internal/skcipher.h>
14#include <crypto/internal/hash.h>
15#include <crypto/scatterwalk.h>
16#include <crypto/hash.h>
17#include "internal.h"
18#include <linux/completion.h>
19#include <linux/err.h>
20#include <linux/init.h>
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/slab.h>
24
25struct gcm_instance_ctx {
26 struct crypto_skcipher_spawn ctr;
27 struct crypto_ahash_spawn ghash;
28};
29
30struct crypto_gcm_ctx {
31 struct crypto_ablkcipher *ctr;
32 struct crypto_ahash *ghash;
33};
34
35struct crypto_rfc4106_ctx {
36 struct crypto_aead *child;
37 u8 nonce[4];
38};
39
40struct crypto_gcm_ghash_ctx {
41 unsigned int cryptlen;
42 struct scatterlist *src;
43 void (*complete)(struct aead_request *req, int err);
44};
45
46struct crypto_gcm_req_priv_ctx {
47 u8 auth_tag[16];
48 u8 iauth_tag[16];
49 struct scatterlist src[2];
50 struct scatterlist dst[2];
51 struct crypto_gcm_ghash_ctx ghash_ctx;
52 union {
53 struct ahash_request ahreq;
54 struct ablkcipher_request abreq;
55 } u;
56};
57
58struct crypto_gcm_setkey_result {
59 int err;
60 struct completion completion;
61};
62
63static void *gcm_zeroes;
64
65static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
66 struct aead_request *req)
67{
68 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
69
70 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
71}
72
73static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
74{
75 struct crypto_gcm_setkey_result *result = req->data;
76
77 if (err == -EINPROGRESS)
78 return;
79
80 result->err = err;
81 complete(&result->completion);
82}
83
84static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
85 unsigned int keylen)
86{
87 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
88 struct crypto_ahash *ghash = ctx->ghash;
89 struct crypto_ablkcipher *ctr = ctx->ctr;
90 struct {
91 be128 hash;
92 u8 iv[8];
93
94 struct crypto_gcm_setkey_result result;
95
96 struct scatterlist sg[1];
97 struct ablkcipher_request req;
98 } *data;
99 int err;
100
101 crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
102 crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
103 CRYPTO_TFM_REQ_MASK);
104
105 err = crypto_ablkcipher_setkey(ctr, key, keylen);
106 if (err)
107 return err;
108
109 crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
110 CRYPTO_TFM_RES_MASK);
111
112 data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
113 GFP_KERNEL);
114 if (!data)
115 return -ENOMEM;
116
117 init_completion(&data->result.completion);
118 sg_init_one(data->sg, &data->hash, sizeof(data->hash));
119 ablkcipher_request_set_tfm(&data->req, ctr);
120 ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
121 CRYPTO_TFM_REQ_MAY_BACKLOG,
122 crypto_gcm_setkey_done,
123 &data->result);
124 ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
125 sizeof(data->hash), data->iv);
126
127 err = crypto_ablkcipher_encrypt(&data->req);
128 if (err == -EINPROGRESS || err == -EBUSY) {
129 err = wait_for_completion_interruptible(
130 &data->result.completion);
131 if (!err)
132 err = data->result.err;
133 }
134
135 if (err)
136 goto out;
137
138 crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);
139 crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &
140 CRYPTO_TFM_REQ_MASK);
141 err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));
142 crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) &
143 CRYPTO_TFM_RES_MASK);
144
145out:
146 kfree(data);
147 return err;
148}
149
150static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
151 unsigned int authsize)
152{
153 switch (authsize) {
154 case 4:
155 case 8:
156 case 12:
157 case 13:
158 case 14:
159 case 15:
160 case 16:
161 break;
162 default:
163 return -EINVAL;
164 }
165
166 return 0;
167}
168
169static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
170 struct aead_request *req,
171 unsigned int cryptlen)
172{
173 struct crypto_aead *aead = crypto_aead_reqtfm(req);
174 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
175 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
176 struct scatterlist *dst;
177 __be32 counter = cpu_to_be32(1);
178
179 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
180 memcpy(req->iv + 12, &counter, 4);
181
182 sg_init_table(pctx->src, 2);
183 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
184 scatterwalk_sg_chain(pctx->src, 2, req->src);
185
186 dst = pctx->src;
187 if (req->src != req->dst) {
188 sg_init_table(pctx->dst, 2);
189 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
190 scatterwalk_sg_chain(pctx->dst, 2, req->dst);
191 dst = pctx->dst;
192 }
193
194 ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
195 ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
196 cryptlen + sizeof(pctx->auth_tag),
197 req->iv);
198}
199
200static inline unsigned int gcm_remain(unsigned int len)
201{
202 len &= 0xfU;
203 return len ? 16 - len : 0;
204}
205
206static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
207static void gcm_hash_final_done(struct crypto_async_request *areq, int err);
208
209static int gcm_hash_update(struct aead_request *req,
210 struct crypto_gcm_req_priv_ctx *pctx,
211 crypto_completion_t complete,
212 struct scatterlist *src,
213 unsigned int len)
214{
215 struct ahash_request *ahreq = &pctx->u.ahreq;
216
217 ahash_request_set_callback(ahreq, aead_request_flags(req),
218 complete, req);
219 ahash_request_set_crypt(ahreq, src, NULL, len);
220
221 return crypto_ahash_update(ahreq);
222}
223
224static int gcm_hash_remain(struct aead_request *req,
225 struct crypto_gcm_req_priv_ctx *pctx,
226 unsigned int remain,
227 crypto_completion_t complete)
228{
229 struct ahash_request *ahreq = &pctx->u.ahreq;
230
231 ahash_request_set_callback(ahreq, aead_request_flags(req),
232 complete, req);
233 sg_init_one(pctx->src, gcm_zeroes, remain);
234 ahash_request_set_crypt(ahreq, pctx->src, NULL, remain);
235
236 return crypto_ahash_update(ahreq);
237}
238
239static int gcm_hash_len(struct aead_request *req,
240 struct crypto_gcm_req_priv_ctx *pctx)
241{
242 struct ahash_request *ahreq = &pctx->u.ahreq;
243 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
244 u128 lengths;
245
246 lengths.a = cpu_to_be64(req->assoclen * 8);
247 lengths.b = cpu_to_be64(gctx->cryptlen * 8);
248 memcpy(pctx->iauth_tag, &lengths, 16);
249 sg_init_one(pctx->src, pctx->iauth_tag, 16);
250 ahash_request_set_callback(ahreq, aead_request_flags(req),
251 gcm_hash_len_done, req);
252 ahash_request_set_crypt(ahreq, pctx->src,
253 NULL, sizeof(lengths));
254
255 return crypto_ahash_update(ahreq);
256}
257
258static int gcm_hash_final(struct aead_request *req,
259 struct crypto_gcm_req_priv_ctx *pctx)
260{
261 struct ahash_request *ahreq = &pctx->u.ahreq;
262
263 ahash_request_set_callback(ahreq, aead_request_flags(req),
264 gcm_hash_final_done, req);
265 ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0);
266
267 return crypto_ahash_final(ahreq);
268}
269
270static void __gcm_hash_final_done(struct aead_request *req, int err)
271{
272 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
273 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
274
275 if (!err)
276 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
277
278 gctx->complete(req, err);
279}
280
281static void gcm_hash_final_done(struct crypto_async_request *areq, int err)
282{
283 struct aead_request *req = areq->data;
284
285 __gcm_hash_final_done(req, err);
286}
287
288static void __gcm_hash_len_done(struct aead_request *req, int err)
289{
290 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
291
292 if (!err) {
293 err = gcm_hash_final(req, pctx);
294 if (err == -EINPROGRESS || err == -EBUSY)
295 return;
296 }
297
298 __gcm_hash_final_done(req, err);
299}
300
301static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
302{
303 struct aead_request *req = areq->data;
304
305 __gcm_hash_len_done(req, err);
306}
307
308static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err)
309{
310 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
311
312 if (!err) {
313 err = gcm_hash_len(req, pctx);
314 if (err == -EINPROGRESS || err == -EBUSY)
315 return;
316 }
317
318 __gcm_hash_len_done(req, err);
319}
320
321static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
322 int err)
323{
324 struct aead_request *req = areq->data;
325
326 __gcm_hash_crypt_remain_done(req, err);
327}
328
329static void __gcm_hash_crypt_done(struct aead_request *req, int err)
330{
331 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
332 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
333 unsigned int remain;
334
335 if (!err) {
336 remain = gcm_remain(gctx->cryptlen);
337 BUG_ON(!remain);
338 err = gcm_hash_remain(req, pctx, remain,
339 gcm_hash_crypt_remain_done);
340 if (err == -EINPROGRESS || err == -EBUSY)
341 return;
342 }
343
344 __gcm_hash_crypt_remain_done(req, err);
345}
346
347static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
348{
349 struct aead_request *req = areq->data;
350
351 __gcm_hash_crypt_done(req, err);
352}
353
354static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err)
355{
356 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
357 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
358 crypto_completion_t complete;
359 unsigned int remain = 0;
360
361 if (!err && gctx->cryptlen) {
362 remain = gcm_remain(gctx->cryptlen);
363 complete = remain ? gcm_hash_crypt_done :
364 gcm_hash_crypt_remain_done;
365 err = gcm_hash_update(req, pctx, complete,
366 gctx->src, gctx->cryptlen);
367 if (err == -EINPROGRESS || err == -EBUSY)
368 return;
369 }
370
371 if (remain)
372 __gcm_hash_crypt_done(req, err);
373 else
374 __gcm_hash_crypt_remain_done(req, err);
375}
376
377static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
378 int err)
379{
380 struct aead_request *req = areq->data;
381
382 __gcm_hash_assoc_remain_done(req, err);
383}
384
385static void __gcm_hash_assoc_done(struct aead_request *req, int err)
386{
387 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
388 unsigned int remain;
389
390 if (!err) {
391 remain = gcm_remain(req->assoclen);
392 BUG_ON(!remain);
393 err = gcm_hash_remain(req, pctx, remain,
394 gcm_hash_assoc_remain_done);
395 if (err == -EINPROGRESS || err == -EBUSY)
396 return;
397 }
398
399 __gcm_hash_assoc_remain_done(req, err);
400}
401
402static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
403{
404 struct aead_request *req = areq->data;
405
406 __gcm_hash_assoc_done(req, err);
407}
408
409static void __gcm_hash_init_done(struct aead_request *req, int err)
410{
411 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
412 crypto_completion_t complete;
413 unsigned int remain = 0;
414
415 if (!err && req->assoclen) {
416 remain = gcm_remain(req->assoclen);
417 complete = remain ? gcm_hash_assoc_done :
418 gcm_hash_assoc_remain_done;
419 err = gcm_hash_update(req, pctx, complete,
420 req->assoc, req->assoclen);
421 if (err == -EINPROGRESS || err == -EBUSY)
422 return;
423 }
424
425 if (remain)
426 __gcm_hash_assoc_done(req, err);
427 else
428 __gcm_hash_assoc_remain_done(req, err);
429}
430
431static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
432{
433 struct aead_request *req = areq->data;
434
435 __gcm_hash_init_done(req, err);
436}
437
438static int gcm_hash(struct aead_request *req,
439 struct crypto_gcm_req_priv_ctx *pctx)
440{
441 struct ahash_request *ahreq = &pctx->u.ahreq;
442 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
443 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
444 unsigned int remain;
445 crypto_completion_t complete;
446 int err;
447
448 ahash_request_set_tfm(ahreq, ctx->ghash);
449
450 ahash_request_set_callback(ahreq, aead_request_flags(req),
451 gcm_hash_init_done, req);
452 err = crypto_ahash_init(ahreq);
453 if (err)
454 return err;
455 remain = gcm_remain(req->assoclen);
456 complete = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done;
457 err = gcm_hash_update(req, pctx, complete, req->assoc, req->assoclen);
458 if (err)
459 return err;
460 if (remain) {
461 err = gcm_hash_remain(req, pctx, remain,
462 gcm_hash_assoc_remain_done);
463 if (err)
464 return err;
465 }
466 remain = gcm_remain(gctx->cryptlen);
467 complete = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done;
468 err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen);
469 if (err)
470 return err;
471 if (remain) {
472 err = gcm_hash_remain(req, pctx, remain,
473 gcm_hash_crypt_remain_done);
474 if (err)
475 return err;
476 }
477 err = gcm_hash_len(req, pctx);
478 if (err)
479 return err;
480 err = gcm_hash_final(req, pctx);
481 if (err)
482 return err;
483
484 return 0;
485}
486
487static void gcm_enc_copy_hash(struct aead_request *req,
488 struct crypto_gcm_req_priv_ctx *pctx)
489{
490 struct crypto_aead *aead = crypto_aead_reqtfm(req);
491 u8 *auth_tag = pctx->auth_tag;
492
493 scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
494 crypto_aead_authsize(aead), 1);
495}
496
497static void gcm_enc_hash_done(struct aead_request *req, int err)
498{
499 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
500
501 if (!err)
502 gcm_enc_copy_hash(req, pctx);
503
504 aead_request_complete(req, err);
505}
506
507static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
508{
509 struct aead_request *req = areq->data;
510 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
511
512 if (!err) {
513 err = gcm_hash(req, pctx);
514 if (err == -EINPROGRESS || err == -EBUSY)
515 return;
516 else if (!err) {
517 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
518 gcm_enc_copy_hash(req, pctx);
519 }
520 }
521
522 aead_request_complete(req, err);
523}
524
525static int crypto_gcm_encrypt(struct aead_request *req)
526{
527 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
528 struct ablkcipher_request *abreq = &pctx->u.abreq;
529 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
530 int err;
531
532 crypto_gcm_init_crypt(abreq, req, req->cryptlen);
533 ablkcipher_request_set_callback(abreq, aead_request_flags(req),
534 gcm_encrypt_done, req);
535
536 gctx->src = req->dst;
537 gctx->cryptlen = req->cryptlen;
538 gctx->complete = gcm_enc_hash_done;
539
540 err = crypto_ablkcipher_encrypt(abreq);
541 if (err)
542 return err;
543
544 err = gcm_hash(req, pctx);
545 if (err)
546 return err;
547
548 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
549 gcm_enc_copy_hash(req, pctx);
550
551 return 0;
552}
553
554static int crypto_gcm_verify(struct aead_request *req,
555 struct crypto_gcm_req_priv_ctx *pctx)
556{
557 struct crypto_aead *aead = crypto_aead_reqtfm(req);
558 u8 *auth_tag = pctx->auth_tag;
559 u8 *iauth_tag = pctx->iauth_tag;
560 unsigned int authsize = crypto_aead_authsize(aead);
561 unsigned int cryptlen = req->cryptlen - authsize;
562
563 crypto_xor(auth_tag, iauth_tag, 16);
564 scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
565 return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
566}
567
568static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
569{
570 struct aead_request *req = areq->data;
571 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
572
573 if (!err)
574 err = crypto_gcm_verify(req, pctx);
575
576 aead_request_complete(req, err);
577}
578
579static void gcm_dec_hash_done(struct aead_request *req, int err)
580{
581 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
582 struct ablkcipher_request *abreq = &pctx->u.abreq;
583 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
584
585 if (!err) {
586 ablkcipher_request_set_callback(abreq, aead_request_flags(req),
587 gcm_decrypt_done, req);
588 crypto_gcm_init_crypt(abreq, req, gctx->cryptlen);
589 err = crypto_ablkcipher_decrypt(abreq);
590 if (err == -EINPROGRESS || err == -EBUSY)
591 return;
592 else if (!err)
593 err = crypto_gcm_verify(req, pctx);
594 }
595
596 aead_request_complete(req, err);
597}
598
599static int crypto_gcm_decrypt(struct aead_request *req)
600{
601 struct crypto_aead *aead = crypto_aead_reqtfm(req);
602 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
603 struct ablkcipher_request *abreq = &pctx->u.abreq;
604 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
605 unsigned int authsize = crypto_aead_authsize(aead);
606 unsigned int cryptlen = req->cryptlen;
607 int err;
608
609 if (cryptlen < authsize)
610 return -EINVAL;
611 cryptlen -= authsize;
612
613 gctx->src = req->src;
614 gctx->cryptlen = cryptlen;
615 gctx->complete = gcm_dec_hash_done;
616
617 err = gcm_hash(req, pctx);
618 if (err)
619 return err;
620
621 ablkcipher_request_set_callback(abreq, aead_request_flags(req),
622 gcm_decrypt_done, req);
623 crypto_gcm_init_crypt(abreq, req, cryptlen);
624 err = crypto_ablkcipher_decrypt(abreq);
625 if (err)
626 return err;
627
628 return crypto_gcm_verify(req, pctx);
629}
630
631static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
632{
633 struct crypto_instance *inst = (void *)tfm->__crt_alg;
634 struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
635 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
636 struct crypto_ablkcipher *ctr;
637 struct crypto_ahash *ghash;
638 unsigned long align;
639 int err;
640
641 ghash = crypto_spawn_ahash(&ictx->ghash);
642 if (IS_ERR(ghash))
643 return PTR_ERR(ghash);
644
645 ctr = crypto_spawn_skcipher(&ictx->ctr);
646 err = PTR_ERR(ctr);
647 if (IS_ERR(ctr))
648 goto err_free_hash;
649
650 ctx->ctr = ctr;
651 ctx->ghash = ghash;
652
653 align = crypto_tfm_alg_alignmask(tfm);
654 align &= ~(crypto_tfm_ctx_alignment() - 1);
655 tfm->crt_aead.reqsize = align +
656 offsetof(struct crypto_gcm_req_priv_ctx, u) +
657 max(sizeof(struct ablkcipher_request) +
658 crypto_ablkcipher_reqsize(ctr),
659 sizeof(struct ahash_request) +
660 crypto_ahash_reqsize(ghash));
661
662 return 0;
663
664err_free_hash:
665 crypto_free_ahash(ghash);
666 return err;
667}
668
669static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
670{
671 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
672
673 crypto_free_ahash(ctx->ghash);
674 crypto_free_ablkcipher(ctx->ctr);
675}
676
677static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb,
678 const char *full_name,
679 const char *ctr_name,
680 const char *ghash_name)
681{
682 struct crypto_attr_type *algt;
683 struct crypto_instance *inst;
684 struct crypto_alg *ctr;
685 struct crypto_alg *ghash_alg;
686 struct ahash_alg *ghash_ahash_alg;
687 struct gcm_instance_ctx *ctx;
688 int err;
689
690 algt = crypto_get_attr_type(tb);
691 err = PTR_ERR(algt);
692 if (IS_ERR(algt))
693 return ERR_PTR(err);
694
695 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
696 return ERR_PTR(-EINVAL);
697
698 ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
699 CRYPTO_ALG_TYPE_HASH,
700 CRYPTO_ALG_TYPE_AHASH_MASK);
701 err = PTR_ERR(ghash_alg);
702 if (IS_ERR(ghash_alg))
703 return ERR_PTR(err);
704
705 err = -ENOMEM;
706 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
707 if (!inst)
708 goto out_put_ghash;
709
710 ctx = crypto_instance_ctx(inst);
711 ghash_ahash_alg = container_of(ghash_alg, struct ahash_alg, halg.base);
712 err = crypto_init_ahash_spawn(&ctx->ghash, &ghash_ahash_alg->halg,
713 inst);
714 if (err)
715 goto err_free_inst;
716
717 crypto_set_skcipher_spawn(&ctx->ctr, inst);
718 err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
719 crypto_requires_sync(algt->type,
720 algt->mask));
721 if (err)
722 goto err_drop_ghash;
723
724 ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
725
726
727 if (ctr->cra_ablkcipher.ivsize != 16)
728 goto out_put_ctr;
729
730
731 err = -EINVAL;
732 if (ctr->cra_blocksize != 1)
733 goto out_put_ctr;
734
735 err = -ENAMETOOLONG;
736 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
737 "gcm_base(%s,%s)", ctr->cra_driver_name,
738 ghash_alg->cra_driver_name) >=
739 CRYPTO_MAX_ALG_NAME)
740 goto out_put_ctr;
741
742 memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
743
744 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
745 inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC;
746 inst->alg.cra_priority = ctr->cra_priority;
747 inst->alg.cra_blocksize = 1;
748 inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1);
749 inst->alg.cra_type = &crypto_aead_type;
750 inst->alg.cra_aead.ivsize = 16;
751 inst->alg.cra_aead.maxauthsize = 16;
752 inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
753 inst->alg.cra_init = crypto_gcm_init_tfm;
754 inst->alg.cra_exit = crypto_gcm_exit_tfm;
755 inst->alg.cra_aead.setkey = crypto_gcm_setkey;
756 inst->alg.cra_aead.setauthsize = crypto_gcm_setauthsize;
757 inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
758 inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;
759
760out:
761 crypto_mod_put(ghash_alg);
762 return inst;
763
764out_put_ctr:
765 crypto_drop_skcipher(&ctx->ctr);
766err_drop_ghash:
767 crypto_drop_ahash(&ctx->ghash);
768err_free_inst:
769 kfree(inst);
770out_put_ghash:
771 inst = ERR_PTR(err);
772 goto out;
773}
774
775static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
776{
777 int err;
778 const char *cipher_name;
779 char ctr_name[CRYPTO_MAX_ALG_NAME];
780 char full_name[CRYPTO_MAX_ALG_NAME];
781
782 cipher_name = crypto_attr_alg_name(tb[1]);
783 err = PTR_ERR(cipher_name);
784 if (IS_ERR(cipher_name))
785 return ERR_PTR(err);
786
787 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
788 CRYPTO_MAX_ALG_NAME)
789 return ERR_PTR(-ENAMETOOLONG);
790
791 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
792 CRYPTO_MAX_ALG_NAME)
793 return ERR_PTR(-ENAMETOOLONG);
794
795 return crypto_gcm_alloc_common(tb, full_name, ctr_name, "ghash");
796}
797
798static void crypto_gcm_free(struct crypto_instance *inst)
799{
800 struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
801
802 crypto_drop_skcipher(&ctx->ctr);
803 crypto_drop_ahash(&ctx->ghash);
804 kfree(inst);
805}
806
807static struct crypto_template crypto_gcm_tmpl = {
808 .name = "gcm",
809 .alloc = crypto_gcm_alloc,
810 .free = crypto_gcm_free,
811 .module = THIS_MODULE,
812};
813
814static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb)
815{
816 int err;
817 const char *ctr_name;
818 const char *ghash_name;
819 char full_name[CRYPTO_MAX_ALG_NAME];
820
821 ctr_name = crypto_attr_alg_name(tb[1]);
822 err = PTR_ERR(ctr_name);
823 if (IS_ERR(ctr_name))
824 return ERR_PTR(err);
825
826 ghash_name = crypto_attr_alg_name(tb[2]);
827 err = PTR_ERR(ghash_name);
828 if (IS_ERR(ghash_name))
829 return ERR_PTR(err);
830
831 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)",
832 ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME)
833 return ERR_PTR(-ENAMETOOLONG);
834
835 return crypto_gcm_alloc_common(tb, full_name, ctr_name, ghash_name);
836}
837
838static struct crypto_template crypto_gcm_base_tmpl = {
839 .name = "gcm_base",
840 .alloc = crypto_gcm_base_alloc,
841 .free = crypto_gcm_free,
842 .module = THIS_MODULE,
843};
844
845static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
846 unsigned int keylen)
847{
848 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
849 struct crypto_aead *child = ctx->child;
850 int err;
851
852 if (keylen < 4)
853 return -EINVAL;
854
855 keylen -= 4;
856 memcpy(ctx->nonce, key + keylen, 4);
857
858 crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
859 crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
860 CRYPTO_TFM_REQ_MASK);
861 err = crypto_aead_setkey(child, key, keylen);
862 crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
863 CRYPTO_TFM_RES_MASK);
864
865 return err;
866}
867
868static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
869 unsigned int authsize)
870{
871 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
872
873 switch (authsize) {
874 case 8:
875 case 12:
876 case 16:
877 break;
878 default:
879 return -EINVAL;
880 }
881
882 return crypto_aead_setauthsize(ctx->child, authsize);
883}
884
885static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
886{
887 struct aead_request *subreq = aead_request_ctx(req);
888 struct crypto_aead *aead = crypto_aead_reqtfm(req);
889 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);
890 struct crypto_aead *child = ctx->child;
891 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
892 crypto_aead_alignmask(child) + 1);
893
894 memcpy(iv, ctx->nonce, 4);
895 memcpy(iv + 4, req->iv, 8);
896
897 aead_request_set_tfm(subreq, child);
898 aead_request_set_callback(subreq, req->base.flags, req->base.complete,
899 req->base.data);
900 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv);
901 aead_request_set_assoc(subreq, req->assoc, req->assoclen);
902
903 return subreq;
904}
905
906static int crypto_rfc4106_encrypt(struct aead_request *req)
907{
908 req = crypto_rfc4106_crypt(req);
909
910 return crypto_aead_encrypt(req);
911}
912
913static int crypto_rfc4106_decrypt(struct aead_request *req)
914{
915 req = crypto_rfc4106_crypt(req);
916
917 return crypto_aead_decrypt(req);
918}
919
920static int crypto_rfc4106_init_tfm(struct crypto_tfm *tfm)
921{
922 struct crypto_instance *inst = (void *)tfm->__crt_alg;
923 struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst);
924 struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm);
925 struct crypto_aead *aead;
926 unsigned long align;
927
928 aead = crypto_spawn_aead(spawn);
929 if (IS_ERR(aead))
930 return PTR_ERR(aead);
931
932 ctx->child = aead;
933
934 align = crypto_aead_alignmask(aead);
935 align &= ~(crypto_tfm_ctx_alignment() - 1);
936 tfm->crt_aead.reqsize = sizeof(struct aead_request) +
937 ALIGN(crypto_aead_reqsize(aead),
938 crypto_tfm_ctx_alignment()) +
939 align + 16;
940
941 return 0;
942}
943
944static void crypto_rfc4106_exit_tfm(struct crypto_tfm *tfm)
945{
946 struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm);
947
948 crypto_free_aead(ctx->child);
949}
950
951static struct crypto_instance *crypto_rfc4106_alloc(struct rtattr **tb)
952{
953 struct crypto_attr_type *algt;
954 struct crypto_instance *inst;
955 struct crypto_aead_spawn *spawn;
956 struct crypto_alg *alg;
957 const char *ccm_name;
958 int err;
959
960 algt = crypto_get_attr_type(tb);
961 err = PTR_ERR(algt);
962 if (IS_ERR(algt))
963 return ERR_PTR(err);
964
965 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
966 return ERR_PTR(-EINVAL);
967
968 ccm_name = crypto_attr_alg_name(tb[1]);
969 err = PTR_ERR(ccm_name);
970 if (IS_ERR(ccm_name))
971 return ERR_PTR(err);
972
973 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
974 if (!inst)
975 return ERR_PTR(-ENOMEM);
976
977 spawn = crypto_instance_ctx(inst);
978 crypto_set_aead_spawn(spawn, inst);
979 err = crypto_grab_aead(spawn, ccm_name, 0,
980 crypto_requires_sync(algt->type, algt->mask));
981 if (err)
982 goto out_free_inst;
983
984 alg = crypto_aead_spawn_alg(spawn);
985
986 err = -EINVAL;
987
988
989 if (alg->cra_aead.ivsize != 16)
990 goto out_drop_alg;
991
992
993 if (alg->cra_blocksize != 1)
994 goto out_drop_alg;
995
996 err = -ENAMETOOLONG;
997 if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
998 "rfc4106(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME ||
999 snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1000 "rfc4106(%s)", alg->cra_driver_name) >=
1001 CRYPTO_MAX_ALG_NAME)
1002 goto out_drop_alg;
1003
1004 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
1005 inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC;
1006 inst->alg.cra_priority = alg->cra_priority;
1007 inst->alg.cra_blocksize = 1;
1008 inst->alg.cra_alignmask = alg->cra_alignmask;
1009 inst->alg.cra_type = &crypto_nivaead_type;
1010
1011 inst->alg.cra_aead.ivsize = 8;
1012 inst->alg.cra_aead.maxauthsize = 16;
1013
1014 inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
1015
1016 inst->alg.cra_init = crypto_rfc4106_init_tfm;
1017 inst->alg.cra_exit = crypto_rfc4106_exit_tfm;
1018
1019 inst->alg.cra_aead.setkey = crypto_rfc4106_setkey;
1020 inst->alg.cra_aead.setauthsize = crypto_rfc4106_setauthsize;
1021 inst->alg.cra_aead.encrypt = crypto_rfc4106_encrypt;
1022 inst->alg.cra_aead.decrypt = crypto_rfc4106_decrypt;
1023
1024 inst->alg.cra_aead.geniv = "seqiv";
1025
1026out:
1027 return inst;
1028
1029out_drop_alg:
1030 crypto_drop_aead(spawn);
1031out_free_inst:
1032 kfree(inst);
1033 inst = ERR_PTR(err);
1034 goto out;
1035}
1036
1037static void crypto_rfc4106_free(struct crypto_instance *inst)
1038{
1039 crypto_drop_spawn(crypto_instance_ctx(inst));
1040 kfree(inst);
1041}
1042
1043static struct crypto_template crypto_rfc4106_tmpl = {
1044 .name = "rfc4106",
1045 .alloc = crypto_rfc4106_alloc,
1046 .free = crypto_rfc4106_free,
1047 .module = THIS_MODULE,
1048};
1049
1050static int __init crypto_gcm_module_init(void)
1051{
1052 int err;
1053
1054 gcm_zeroes = kzalloc(16, GFP_KERNEL);
1055 if (!gcm_zeroes)
1056 return -ENOMEM;
1057
1058 err = crypto_register_template(&crypto_gcm_base_tmpl);
1059 if (err)
1060 goto out;
1061
1062 err = crypto_register_template(&crypto_gcm_tmpl);
1063 if (err)
1064 goto out_undo_base;
1065
1066 err = crypto_register_template(&crypto_rfc4106_tmpl);
1067 if (err)
1068 goto out_undo_gcm;
1069
1070 return 0;
1071
1072out_undo_gcm:
1073 crypto_unregister_template(&crypto_gcm_tmpl);
1074out_undo_base:
1075 crypto_unregister_template(&crypto_gcm_base_tmpl);
1076out:
1077 kfree(gcm_zeroes);
1078 return err;
1079}
1080
1081static void __exit crypto_gcm_module_exit(void)
1082{
1083 kfree(gcm_zeroes);
1084 crypto_unregister_template(&crypto_rfc4106_tmpl);
1085 crypto_unregister_template(&crypto_gcm_tmpl);
1086 crypto_unregister_template(&crypto_gcm_base_tmpl);
1087}
1088
1089module_init(crypto_gcm_module_init);
1090module_exit(crypto_gcm_module_exit);
1091
1092MODULE_LICENSE("GPL");
1093MODULE_DESCRIPTION("Galois/Counter Mode");
1094MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
1095MODULE_ALIAS("gcm_base");
1096MODULE_ALIAS("rfc4106");
1097
