1
2
3
4
5
6
7
8
9
10#include <linux/completion.h>
11#include <linux/err.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/kernel.h>
15#include <linux/key.h>
16#include <linux/bio.h>
17#include <linux/blkdev.h>
18#include <linux/mempool.h>
19#include <linux/slab.h>
20#include <linux/crypto.h>
21#include <linux/workqueue.h>
22#include <linux/kthread.h>
23#include <linux/backing-dev.h>
24#include <linux/atomic.h>
25#include <linux/scatterlist.h>
26#include <linux/rbtree.h>
27#include <linux/ctype.h>
28#include <asm/page.h>
29#include <asm/unaligned.h>
30#include <crypto/hash.h>
31#include <crypto/md5.h>
32#include <crypto/algapi.h>
33#include <crypto/skcipher.h>
34#include <crypto/aead.h>
35#include <crypto/authenc.h>
36#include <linux/rtnetlink.h>
37#include <keys/user-type.h>
38
39#include <linux/device-mapper.h>
40
41#define DM_MSG_PREFIX "crypt"
42
43
44
45
46struct convert_context {
47 struct completion restart;
48 struct bio *bio_in;
49 struct bio *bio_out;
50 struct bvec_iter iter_in;
51 struct bvec_iter iter_out;
52 sector_t cc_sector;
53 atomic_t cc_pending;
54 union {
55 struct skcipher_request *req;
56 struct aead_request *req_aead;
57 } r;
58
59};
60
61
62
63
64struct dm_crypt_io {
65 struct crypt_config *cc;
66 struct bio *base_bio;
67 u8 *integrity_metadata;
68 bool integrity_metadata_from_pool;
69 struct work_struct work;
70
71 struct convert_context ctx;
72
73 atomic_t io_pending;
74 blk_status_t error;
75 sector_t sector;
76
77 struct rb_node rb_node;
78} CRYPTO_MINALIGN_ATTR;
79
80struct dm_crypt_request {
81 struct convert_context *ctx;
82 struct scatterlist sg_in[4];
83 struct scatterlist sg_out[4];
84 sector_t iv_sector;
85};
86
87struct crypt_config;
88
89struct crypt_iv_operations {
90 int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
91 const char *opts);
92 void (*dtr)(struct crypt_config *cc);
93 int (*init)(struct crypt_config *cc);
94 int (*wipe)(struct crypt_config *cc);
95 int (*generator)(struct crypt_config *cc, u8 *iv,
96 struct dm_crypt_request *dmreq);
97 int (*post)(struct crypt_config *cc, u8 *iv,
98 struct dm_crypt_request *dmreq);
99};
100
101struct iv_essiv_private {
102 struct crypto_ahash *hash_tfm;
103 u8 *salt;
104};
105
106struct iv_benbi_private {
107 int shift;
108};
109
110#define LMK_SEED_SIZE 64
111struct iv_lmk_private {
112 struct crypto_shash *hash_tfm;
113 u8 *seed;
114};
115
116#define TCW_WHITENING_SIZE 16
117struct iv_tcw_private {
118 struct crypto_shash *crc32_tfm;
119 u8 *iv_seed;
120 u8 *whitening;
121};
122
123
124
125
126
127enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
128 DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD };
129
130enum cipher_flags {
131 CRYPT_MODE_INTEGRITY_AEAD,
132 CRYPT_IV_LARGE_SECTORS,
133};
134
135
136
137
138struct crypt_config {
139 struct dm_dev *dev;
140 sector_t start;
141
142
143
144
145
146 mempool_t *req_pool;
147 mempool_t *page_pool;
148 mempool_t *tag_pool;
149 unsigned tag_pool_max_sectors;
150
151 struct percpu_counter n_allocated_pages;
152
153 struct bio_set *bs;
154 struct mutex bio_alloc_lock;
155
156 struct workqueue_struct *io_queue;
157 struct workqueue_struct *crypt_queue;
158
159 struct task_struct *write_thread;
160 wait_queue_head_t write_thread_wait;
161 struct rb_root write_tree;
162
163 char *cipher;
164 char *cipher_string;
165 char *cipher_auth;
166 char *key_string;
167
168 const struct crypt_iv_operations *iv_gen_ops;
169 union {
170 struct iv_essiv_private essiv;
171 struct iv_benbi_private benbi;
172 struct iv_lmk_private lmk;
173 struct iv_tcw_private tcw;
174 } iv_gen_private;
175 sector_t iv_offset;
176 unsigned int iv_size;
177 unsigned short int sector_size;
178 unsigned char sector_shift;
179
180
181 void *iv_private;
182 union {
183 struct crypto_skcipher **tfms;
184 struct crypto_aead **tfms_aead;
185 } cipher_tfm;
186 unsigned tfms_count;
187 unsigned long cipher_flags;
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202 unsigned int dmreq_start;
203
204 unsigned int per_bio_data_size;
205
206 unsigned long flags;
207 unsigned int key_size;
208 unsigned int key_parts;
209 unsigned int key_extra_size;
210 unsigned int key_mac_size;
211
212 unsigned int integrity_tag_size;
213 unsigned int integrity_iv_size;
214 unsigned int on_disk_tag_size;
215
216 u8 *authenc_key;
217 u8 key[0];
218};
219
220#define MIN_IOS 64
221#define MAX_TAG_SIZE 480
222#define POOL_ENTRY_SIZE 512
223
224static DEFINE_SPINLOCK(dm_crypt_clients_lock);
225static unsigned dm_crypt_clients_n = 0;
226static volatile unsigned long dm_crypt_pages_per_client;
227#define DM_CRYPT_MEMORY_PERCENT 2
228#define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_PAGES * 16)
229
230static void clone_init(struct dm_crypt_io *, struct bio *);
231static void kcryptd_queue_crypt(struct dm_crypt_io *io);
232static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
233 struct scatterlist *sg);
234
235
236
237
238static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
239{
240 return cc->cipher_tfm.tfms[0];
241}
242
243static struct crypto_aead *any_tfm_aead(struct crypt_config *cc)
244{
245 return cc->cipher_tfm.tfms_aead[0];
246}
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
299 struct dm_crypt_request *dmreq)
300{
301 memset(iv, 0, cc->iv_size);
302 *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
303
304 return 0;
305}
306
307static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
308 struct dm_crypt_request *dmreq)
309{
310 memset(iv, 0, cc->iv_size);
311 *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
312
313 return 0;
314}
315
316static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv,
317 struct dm_crypt_request *dmreq)
318{
319 memset(iv, 0, cc->iv_size);
320
321 *(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector);
322
323 return 0;
324}
325
326
327static int crypt_iv_essiv_init(struct crypt_config *cc)
328{
329 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
330 AHASH_REQUEST_ON_STACK(req, essiv->hash_tfm);
331 struct scatterlist sg;
332 struct crypto_cipher *essiv_tfm;
333 int err;
334
335 sg_init_one(&sg, cc->key, cc->key_size);
336 ahash_request_set_tfm(req, essiv->hash_tfm);
337 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
338 ahash_request_set_crypt(req, &sg, essiv->salt, cc->key_size);
339
340 err = crypto_ahash_digest(req);
341 ahash_request_zero(req);
342 if (err)
343 return err;
344
345 essiv_tfm = cc->iv_private;
346
347 err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
348 crypto_ahash_digestsize(essiv->hash_tfm));
349 if (err)
350 return err;
351
352 return 0;
353}
354
355
356static int crypt_iv_essiv_wipe(struct crypt_config *cc)
357{
358 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
359 unsigned salt_size = crypto_ahash_digestsize(essiv->hash_tfm);
360 struct crypto_cipher *essiv_tfm;
361 int r, err = 0;
362
363 memset(essiv->salt, 0, salt_size);
364
365 essiv_tfm = cc->iv_private;
366 r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
367 if (r)
368 err = r;
369
370 return err;
371}
372
373
374static struct crypto_cipher *alloc_essiv_cipher(struct crypt_config *cc,
375 struct dm_target *ti,
376 const u8 *salt,
377 unsigned int saltsize)
378{
379 struct crypto_cipher *essiv_tfm;
380 int err;
381
382
383 essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
384 if (IS_ERR(essiv_tfm)) {
385 ti->error = "Error allocating crypto tfm for ESSIV";
386 return essiv_tfm;
387 }
388
389 if (crypto_cipher_blocksize(essiv_tfm) != cc->iv_size) {
390 ti->error = "Block size of ESSIV cipher does "
391 "not match IV size of block cipher";
392 crypto_free_cipher(essiv_tfm);
393 return ERR_PTR(-EINVAL);
394 }
395
396 err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
397 if (err) {
398 ti->error = "Failed to set key for ESSIV cipher";
399 crypto_free_cipher(essiv_tfm);
400 return ERR_PTR(err);
401 }
402
403 return essiv_tfm;
404}
405
406static void crypt_iv_essiv_dtr(struct crypt_config *cc)
407{
408 struct crypto_cipher *essiv_tfm;
409 struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
410
411 crypto_free_ahash(essiv->hash_tfm);
412 essiv->hash_tfm = NULL;
413
414 kzfree(essiv->salt);
415 essiv->salt = NULL;
416
417 essiv_tfm = cc->iv_private;
418
419 if (essiv_tfm)
420 crypto_free_cipher(essiv_tfm);
421
422 cc->iv_private = NULL;
423}
424
425static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
426 const char *opts)
427{
428 struct crypto_cipher *essiv_tfm = NULL;
429 struct crypto_ahash *hash_tfm = NULL;
430 u8 *salt = NULL;
431 int err;
432
433 if (!opts) {
434 ti->error = "Digest algorithm missing for ESSIV mode";
435 return -EINVAL;
436 }
437
438
439 hash_tfm = crypto_alloc_ahash(opts, 0, CRYPTO_ALG_ASYNC);
440 if (IS_ERR(hash_tfm)) {
441 ti->error = "Error initializing ESSIV hash";
442 err = PTR_ERR(hash_tfm);
443 goto bad;
444 }
445
446 salt = kzalloc(crypto_ahash_digestsize(hash_tfm), GFP_KERNEL);
447 if (!salt) {
448 ti->error = "Error kmallocing salt storage in ESSIV";
449 err = -ENOMEM;
450 goto bad;
451 }
452
453 cc->iv_gen_private.essiv.salt = salt;
454 cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
455
456 essiv_tfm = alloc_essiv_cipher(cc, ti, salt,
457 crypto_ahash_digestsize(hash_tfm));
458 if (IS_ERR(essiv_tfm)) {
459 crypt_iv_essiv_dtr(cc);
460 return PTR_ERR(essiv_tfm);
461 }
462 cc->iv_private = essiv_tfm;
463
464 return 0;
465
466bad:
467 if (hash_tfm && !IS_ERR(hash_tfm))
468 crypto_free_ahash(hash_tfm);
469 kfree(salt);
470 return err;
471}
472
473static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
474 struct dm_crypt_request *dmreq)
475{
476 struct crypto_cipher *essiv_tfm = cc->iv_private;
477
478 memset(iv, 0, cc->iv_size);
479 *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
480 crypto_cipher_encrypt_one(essiv_tfm, iv, iv);
481
482 return 0;
483}
484
485static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
486 const char *opts)
487{
488 unsigned bs = crypto_skcipher_blocksize(any_tfm(cc));
489 int log = ilog2(bs);
490
491
492
493
494 if (1 << log != bs) {
495 ti->error = "cypher blocksize is not a power of 2";
496 return -EINVAL;
497 }
498
499 if (log > 9) {
500 ti->error = "cypher blocksize is > 512";
501 return -EINVAL;
502 }
503
504 cc->iv_gen_private.benbi.shift = 9 - log;
505
506 return 0;
507}
508
509static void crypt_iv_benbi_dtr(struct crypt_config *cc)
510{
511}
512
513static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
514 struct dm_crypt_request *dmreq)
515{
516 __be64 val;
517
518 memset(iv, 0, cc->iv_size - sizeof(u64));
519
520 val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
521 put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
522
523 return 0;
524}
525
526static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
527 struct dm_crypt_request *dmreq)
528{
529 memset(iv, 0, cc->iv_size);
530
531 return 0;
532}
533
534static void crypt_iv_lmk_dtr(struct crypt_config *cc)
535{
536 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
537
538 if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))
539 crypto_free_shash(lmk->hash_tfm);
540 lmk->hash_tfm = NULL;
541
542 kzfree(lmk->seed);
543 lmk->seed = NULL;
544}
545
546static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
547 const char *opts)
548{
549 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
550
551 if (cc->sector_size != (1 << SECTOR_SHIFT)) {
552 ti->error = "Unsupported sector size for LMK";
553 return -EINVAL;
554 }
555
556 lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0);
557 if (IS_ERR(lmk->hash_tfm)) {
558 ti->error = "Error initializing LMK hash";
559 return PTR_ERR(lmk->hash_tfm);
560 }
561
562
563 if (cc->key_parts == cc->tfms_count) {
564 lmk->seed = NULL;
565 return 0;
566 }
567
568 lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
569 if (!lmk->seed) {
570 crypt_iv_lmk_dtr(cc);
571 ti->error = "Error kmallocing seed storage in LMK";
572 return -ENOMEM;
573 }
574
575 return 0;
576}
577
578static int crypt_iv_lmk_init(struct crypt_config *cc)
579{
580 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
581 int subkey_size = cc->key_size / cc->key_parts;
582
583
584 if (lmk->seed)
585 memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
586 crypto_shash_digestsize(lmk->hash_tfm));
587
588 return 0;
589}
590
591static int crypt_iv_lmk_wipe(struct crypt_config *cc)
592{
593 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
594
595 if (lmk->seed)
596 memset(lmk->seed, 0, LMK_SEED_SIZE);
597
598 return 0;
599}
600
601static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
602 struct dm_crypt_request *dmreq,
603 u8 *data)
604{
605 struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
606 SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
607 struct md5_state md5state;
608 __le32 buf[4];
609 int i, r;
610
611 desc->tfm = lmk->hash_tfm;
612 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
613
614 r = crypto_shash_init(desc);
615 if (r)
616 return r;
617
618 if (lmk->seed) {
619 r = crypto_shash_update(desc, lmk->seed, LMK_SEED_SIZE);
620 if (r)
621 return r;
622 }
623
624
625 r = crypto_shash_update(desc, data + 16, 16 * 31);
626 if (r)
627 return r;
628
629
630 buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
631 buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
632 buf[2] = cpu_to_le32(4024);
633 buf[3] = 0;
634 r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf));
635 if (r)
636 return r;
637
638
639 r = crypto_shash_export(desc, &md5state);
640 if (r)
641 return r;
642
643 for (i = 0; i < MD5_HASH_WORDS; i++)
644 __cpu_to_le32s(&md5state.hash[i]);
645 memcpy(iv, &md5state.hash, cc->iv_size);
646
647 return 0;
648}
649
650static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
651 struct dm_crypt_request *dmreq)
652{
653 struct scatterlist *sg;
654 u8 *src;
655 int r = 0;
656
657 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
658 sg = crypt_get_sg_data(cc, dmreq->sg_in);
659 src = kmap_atomic(sg_page(sg));
660 r = crypt_iv_lmk_one(cc, iv, dmreq, src + sg->offset);
661 kunmap_atomic(src);
662 } else
663 memset(iv, 0, cc->iv_size);
664
665 return r;
666}
667
668static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
669 struct dm_crypt_request *dmreq)
670{
671 struct scatterlist *sg;
672 u8 *dst;
673 int r;
674
675 if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
676 return 0;
677
678 sg = crypt_get_sg_data(cc, dmreq->sg_out);
679 dst = kmap_atomic(sg_page(sg));
680 r = crypt_iv_lmk_one(cc, iv, dmreq, dst + sg->offset);
681
682
683 if (!r)
684 crypto_xor(dst + sg->offset, iv, cc->iv_size);
685
686 kunmap_atomic(dst);
687 return r;
688}
689
690static void crypt_iv_tcw_dtr(struct crypt_config *cc)
691{
692 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
693
694 kzfree(tcw->iv_seed);
695 tcw->iv_seed = NULL;
696 kzfree(tcw->whitening);
697 tcw->whitening = NULL;
698
699 if (tcw->crc32_tfm && !IS_ERR(tcw->crc32_tfm))
700 crypto_free_shash(tcw->crc32_tfm);
701 tcw->crc32_tfm = NULL;
702}
703
704static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
705 const char *opts)
706{
707 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
708
709 if (cc->sector_size != (1 << SECTOR_SHIFT)) {
710 ti->error = "Unsupported sector size for TCW";
711 return -EINVAL;
712 }
713
714 if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
715 ti->error = "Wrong key size for TCW";
716 return -EINVAL;
717 }
718
719 tcw->crc32_tfm = crypto_alloc_shash("crc32", 0, 0);
720 if (IS_ERR(tcw->crc32_tfm)) {
721 ti->error = "Error initializing CRC32 in TCW";
722 return PTR_ERR(tcw->crc32_tfm);
723 }
724
725 tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);
726 tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
727 if (!tcw->iv_seed || !tcw->whitening) {
728 crypt_iv_tcw_dtr(cc);
729 ti->error = "Error allocating seed storage in TCW";
730 return -ENOMEM;
731 }
732
733 return 0;
734}
735
736static int crypt_iv_tcw_init(struct crypt_config *cc)
737{
738 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
739 int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;
740
741 memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
742 memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
743 TCW_WHITENING_SIZE);
744
745 return 0;
746}
747
748static int crypt_iv_tcw_wipe(struct crypt_config *cc)
749{
750 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
751
752 memset(tcw->iv_seed, 0, cc->iv_size);
753 memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
754
755 return 0;
756}
757
758static int crypt_iv_tcw_whitening(struct crypt_config *cc,
759 struct dm_crypt_request *dmreq,
760 u8 *data)
761{
762 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
763 __le64 sector = cpu_to_le64(dmreq->iv_sector);
764 u8 buf[TCW_WHITENING_SIZE];
765 SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
766 int i, r;
767
768
769 crypto_xor_cpy(buf, tcw->whitening, (u8 *)§or, 8);
770 crypto_xor_cpy(&buf[8], tcw->whitening + 8, (u8 *)§or, 8);
771
772
773 desc->tfm = tcw->crc32_tfm;
774 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
775 for (i = 0; i < 4; i++) {
776 r = crypto_shash_init(desc);
777 if (r)
778 goto out;
779 r = crypto_shash_update(desc, &buf[i * 4], 4);
780 if (r)
781 goto out;
782 r = crypto_shash_final(desc, &buf[i * 4]);
783 if (r)
784 goto out;
785 }
786 crypto_xor(&buf[0], &buf[12], 4);
787 crypto_xor(&buf[4], &buf[8], 4);
788
789
790 for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
791 crypto_xor(data + i * 8, buf, 8);
792out:
793 memzero_explicit(buf, sizeof(buf));
794 return r;
795}
796
797static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
798 struct dm_crypt_request *dmreq)
799{
800 struct scatterlist *sg;
801 struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
802 __le64 sector = cpu_to_le64(dmreq->iv_sector);
803 u8 *src;
804 int r = 0;
805
806
807 if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
808 sg = crypt_get_sg_data(cc, dmreq->sg_in);
809 src = kmap_atomic(sg_page(sg));
810 r = crypt_iv_tcw_whitening(cc, dmreq, src + sg->offset);
811 kunmap_atomic(src);
812 }
813
814
815 crypto_xor_cpy(iv, tcw->iv_seed, (u8 *)§or, 8);
816 if (cc->iv_size > 8)
817 crypto_xor_cpy(&iv[8], tcw->iv_seed + 8, (u8 *)§or,
818 cc->iv_size - 8);
819
820 return r;
821}
822
823static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
824 struct dm_crypt_request *dmreq)
825{
826 struct scatterlist *sg;
827 u8 *dst;
828 int r;
829
830 if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
831 return 0;
832
833
834 sg = crypt_get_sg_data(cc, dmreq->sg_out);
835 dst = kmap_atomic(sg_page(sg));
836 r = crypt_iv_tcw_whitening(cc, dmreq, dst + sg->offset);
837 kunmap_atomic(dst);
838
839 return r;
840}
841
842static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv,
843 struct dm_crypt_request *dmreq)
844{
845
846 get_random_bytes(iv, cc->iv_size);
847 return 0;
848}
849
850static const struct crypt_iv_operations crypt_iv_plain_ops = {
851 .generator = crypt_iv_plain_gen
852};
853
854static const struct crypt_iv_operations crypt_iv_plain64_ops = {
855 .generator = crypt_iv_plain64_gen
856};
857
858static const struct crypt_iv_operations crypt_iv_plain64be_ops = {
859 .generator = crypt_iv_plain64be_gen
860};
861
862static const struct crypt_iv_operations crypt_iv_essiv_ops = {
863 .ctr = crypt_iv_essiv_ctr,
864 .dtr = crypt_iv_essiv_dtr,
865 .init = crypt_iv_essiv_init,
866 .wipe = crypt_iv_essiv_wipe,
867 .generator = crypt_iv_essiv_gen
868};
869
870static const struct crypt_iv_operations crypt_iv_benbi_ops = {
871 .ctr = crypt_iv_benbi_ctr,
872 .dtr = crypt_iv_benbi_dtr,
873 .generator = crypt_iv_benbi_gen
874};
875
876static const struct crypt_iv_operations crypt_iv_null_ops = {
877 .generator = crypt_iv_null_gen
878};
879
880static const struct crypt_iv_operations crypt_iv_lmk_ops = {
881 .ctr = crypt_iv_lmk_ctr,
882 .dtr = crypt_iv_lmk_dtr,
883 .init = crypt_iv_lmk_init,
884 .wipe = crypt_iv_lmk_wipe,
885 .generator = crypt_iv_lmk_gen,
886 .post = crypt_iv_lmk_post
887};
888
889static const struct crypt_iv_operations crypt_iv_tcw_ops = {
890 .ctr = crypt_iv_tcw_ctr,
891 .dtr = crypt_iv_tcw_dtr,
892 .init = crypt_iv_tcw_init,
893 .wipe = crypt_iv_tcw_wipe,
894 .generator = crypt_iv_tcw_gen,
895 .post = crypt_iv_tcw_post
896};
897
898static struct crypt_iv_operations crypt_iv_random_ops = {
899 .generator = crypt_iv_random_gen
900};
901
902
903
904
905static bool crypt_integrity_aead(struct crypt_config *cc)
906{
907 return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
908}
909
910static bool crypt_integrity_hmac(struct crypt_config *cc)
911{
912 return crypt_integrity_aead(cc) && cc->key_mac_size;
913}
914
915
916static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
917 struct scatterlist *sg)
918{
919 if (unlikely(crypt_integrity_aead(cc)))
920 return &sg[2];
921
922 return sg;
923}
924
925static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
926{
927 struct bio_integrity_payload *bip;
928 unsigned int tag_len;
929 int ret;
930
931 if (!bio_sectors(bio) || !io->cc->on_disk_tag_size)
932 return 0;
933
934 bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
935 if (IS_ERR(bip))
936 return PTR_ERR(bip);
937
938 tag_len = io->cc->on_disk_tag_size * bio_sectors(bio);
939
940 bip->bip_iter.bi_size = tag_len;
941 bip->bip_iter.bi_sector = io->cc->start + io->sector;
942
943 ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata),
944 tag_len, offset_in_page(io->integrity_metadata));
945 if (unlikely(ret != tag_len))
946 return -ENOMEM;
947
948 return 0;
949}
950
951static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)
952{
953#ifdef CONFIG_BLK_DEV_INTEGRITY
954 struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk);
955
956
957 if (!bi || strcasecmp(bi->profile->name, "DM-DIF-EXT-TAG")) {
958 ti->error = "Integrity profile not supported.";
959 return -EINVAL;
960 }
961
962 if (bi->tag_size != cc->on_disk_tag_size ||
963 bi->tuple_size != cc->on_disk_tag_size) {
964 ti->error = "Integrity profile tag size mismatch.";
965 return -EINVAL;
966 }
967 if (1 << bi->interval_exp != cc->sector_size) {
968 ti->error = "Integrity profile sector size mismatch.";
969 return -EINVAL;
970 }
971
972 if (crypt_integrity_aead(cc)) {
973 cc->integrity_tag_size = cc->on_disk_tag_size - cc->integrity_iv_size;
974 DMINFO("Integrity AEAD, tag size %u, IV size %u.",
975 cc->integrity_tag_size, cc->integrity_iv_size);
976
977 if (crypto_aead_setauthsize(any_tfm_aead(cc), cc->integrity_tag_size)) {
978 ti->error = "Integrity AEAD auth tag size is not supported.";
979 return -EINVAL;
980 }
981 } else if (cc->integrity_iv_size)
982 DMINFO("Additional per-sector space %u bytes for IV.",
983 cc->integrity_iv_size);
984
985 if ((cc->integrity_tag_size + cc->integrity_iv_size) != bi->tag_size) {
986 ti->error = "Not enough space for integrity tag in the profile.";
987 return -EINVAL;
988 }
989
990 return 0;
991#else
992 ti->error = "Integrity profile not supported.";
993 return -EINVAL;
994#endif
995}
996
997static void crypt_convert_init(struct crypt_config *cc,
998 struct convert_context *ctx,
999 struct bio *bio_out, struct bio *bio_in,
1000 sector_t sector)
1001{
1002 ctx->bio_in = bio_in;
1003 ctx->bio_out = bio_out;
1004 if (bio_in)
1005 ctx->iter_in = bio_in->bi_iter;
1006 if (bio_out)
1007 ctx->iter_out = bio_out->bi_iter;
1008 ctx->cc_sector = sector + cc->iv_offset;
1009 init_completion(&ctx->restart);
1010}
1011
1012static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
1013 void *req)
1014{
1015 return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
1016}
1017
1018static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq)
1019{
1020 return (void *)((char *)dmreq - cc->dmreq_start);
1021}
1022
1023static u8 *iv_of_dmreq(struct crypt_config *cc,
1024 struct dm_crypt_request *dmreq)
1025{
1026 if (crypt_integrity_aead(cc))
1027 return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1028 crypto_aead_alignmask(any_tfm_aead(cc)) + 1);
1029 else
1030 return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1031 crypto_skcipher_alignmask(any_tfm(cc)) + 1);
1032}
1033
1034static u8 *org_iv_of_dmreq(struct crypt_config *cc,
1035 struct dm_crypt_request *dmreq)
1036{
1037 return iv_of_dmreq(cc, dmreq) + cc->iv_size;
1038}
1039
1040static uint64_t *org_sector_of_dmreq(struct crypt_config *cc,
1041 struct dm_crypt_request *dmreq)
1042{
1043 u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size;
1044 return (uint64_t*) ptr;
1045}
1046
1047static unsigned int *org_tag_of_dmreq(struct crypt_config *cc,
1048 struct dm_crypt_request *dmreq)
1049{
1050 u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size +
1051 cc->iv_size + sizeof(uint64_t);
1052 return (unsigned int*)ptr;
1053}
1054
1055static void *tag_from_dmreq(struct crypt_config *cc,
1056 struct dm_crypt_request *dmreq)
1057{
1058 struct convert_context *ctx = dmreq->ctx;
1059 struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
1060
1061 return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *
1062 cc->on_disk_tag_size];
1063}
1064
1065static void *iv_tag_from_dmreq(struct crypt_config *cc,
1066 struct dm_crypt_request *dmreq)
1067{
1068 return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;
1069}
1070
1071static int crypt_convert_block_aead(struct crypt_config *cc,
1072 struct convert_context *ctx,
1073 struct aead_request *req,
1074 unsigned int tag_offset)
1075{
1076 struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1077 struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1078 struct dm_crypt_request *dmreq;
1079 u8 *iv, *org_iv, *tag_iv, *tag;
1080 uint64_t *sector;
1081 int r = 0;
1082
1083 BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size);
1084
1085
1086 if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
1087 return -EIO;
1088
1089 dmreq = dmreq_of_req(cc, req);
1090 dmreq->iv_sector = ctx->cc_sector;
1091 if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1092 dmreq->iv_sector >>= cc->sector_shift;
1093 dmreq->ctx = ctx;
1094
1095 *org_tag_of_dmreq(cc, dmreq) = tag_offset;
1096
1097 sector = org_sector_of_dmreq(cc, dmreq);
1098 *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1099
1100 iv = iv_of_dmreq(cc, dmreq);
1101 org_iv = org_iv_of_dmreq(cc, dmreq);
1102 tag = tag_from_dmreq(cc, dmreq);
1103 tag_iv = iv_tag_from_dmreq(cc, dmreq);
1104
1105
1106
1107
1108
1109
1110 sg_init_table(dmreq->sg_in, 4);
1111 sg_set_buf(&dmreq->sg_in[0], sector, sizeof(uint64_t));
1112 sg_set_buf(&dmreq->sg_in[1], org_iv, cc->iv_size);
1113 sg_set_page(&dmreq->sg_in[2], bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
1114 sg_set_buf(&dmreq->sg_in[3], tag, cc->integrity_tag_size);
1115
1116 sg_init_table(dmreq->sg_out, 4);
1117 sg_set_buf(&dmreq->sg_out[0], sector, sizeof(uint64_t));
1118 sg_set_buf(&dmreq->sg_out[1], org_iv, cc->iv_size);
1119 sg_set_page(&dmreq->sg_out[2], bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
1120 sg_set_buf(&dmreq->sg_out[3], tag, cc->integrity_tag_size);
1121
1122 if (cc->iv_gen_ops) {
1123
1124 if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1125 memcpy(org_iv, tag_iv, cc->iv_size);
1126 } else {
1127 r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1128 if (r < 0)
1129 return r;
1130
1131 if (cc->integrity_iv_size)
1132 memcpy(tag_iv, org_iv, cc->iv_size);
1133 }
1134
1135 memcpy(iv, org_iv, cc->iv_size);
1136 }
1137
1138 aead_request_set_ad(req, sizeof(uint64_t) + cc->iv_size);
1139 if (bio_data_dir(ctx->bio_in) == WRITE) {
1140 aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
1141 cc->sector_size, iv);
1142 r = crypto_aead_encrypt(req);
1143 if (cc->integrity_tag_size + cc->integrity_iv_size != cc->on_disk_tag_size)
1144 memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0,
1145 cc->on_disk_tag_size - (cc->integrity_tag_size + cc->integrity_iv_size));
1146 } else {
1147 aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
1148 cc->sector_size + cc->integrity_tag_size, iv);
1149 r = crypto_aead_decrypt(req);
1150 }
1151
1152 if (r == -EBADMSG)
1153 DMERR_LIMIT("INTEGRITY AEAD ERROR, sector %llu",
1154 (unsigned long long)le64_to_cpu(*sector));
1155
1156 if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1157 r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1158
1159 bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
1160 bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
1161
1162 return r;
1163}
1164
1165static int crypt_convert_block_skcipher(struct crypt_config *cc,
1166 struct convert_context *ctx,
1167 struct skcipher_request *req,
1168 unsigned int tag_offset)
1169{
1170 struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1171 struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1172 struct scatterlist *sg_in, *sg_out;
1173 struct dm_crypt_request *dmreq;
1174 u8 *iv, *org_iv, *tag_iv;
1175 uint64_t *sector;
1176 int r = 0;
1177
1178
1179 if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
1180 return -EIO;
1181
1182 dmreq = dmreq_of_req(cc, req);
1183 dmreq->iv_sector = ctx->cc_sector;
1184 if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1185 dmreq->iv_sector >>= cc->sector_shift;
1186 dmreq->ctx = ctx;
1187
1188 *org_tag_of_dmreq(cc, dmreq) = tag_offset;
1189
1190 iv = iv_of_dmreq(cc, dmreq);
1191 org_iv = org_iv_of_dmreq(cc, dmreq);
1192 tag_iv = iv_tag_from_dmreq(cc, dmreq);
1193
1194 sector = org_sector_of_dmreq(cc, dmreq);
1195 *sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1196
1197
1198 sg_in = &dmreq->sg_in[0];
1199 sg_out = &dmreq->sg_out[0];
1200
1201 sg_init_table(sg_in, 1);
1202 sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
1203
1204 sg_init_table(sg_out, 1);
1205 sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
1206
1207 if (cc->iv_gen_ops) {
1208
1209 if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1210 memcpy(org_iv, tag_iv, cc->integrity_iv_size);
1211 } else {
1212 r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1213 if (r < 0)
1214 return r;
1215
1216 if (cc->integrity_iv_size)
1217 memcpy(tag_iv, org_iv, cc->integrity_iv_size);
1218 }
1219
1220 memcpy(iv, org_iv, cc->iv_size);
1221 }
1222
1223 skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv);
1224
1225 if (bio_data_dir(ctx->bio_in) == WRITE)
1226 r = crypto_skcipher_encrypt(req);
1227 else
1228 r = crypto_skcipher_decrypt(req);
1229
1230 if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1231 r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1232
1233 bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
1234 bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
1235
1236 return r;
1237}
1238
1239static void kcryptd_async_done(struct crypto_async_request *async_req,
1240 int error);
1241
1242static void crypt_alloc_req_skcipher(struct crypt_config *cc,
1243 struct convert_context *ctx)
1244{
1245 unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1);
1246
1247 if (!ctx->r.req)
1248 ctx->r.req = mempool_alloc(cc->req_pool, GFP_NOIO);
1249
1250 skcipher_request_set_tfm(ctx->r.req, cc->cipher_tfm.tfms[key_index]);
1251
1252
1253
1254
1255
1256 skcipher_request_set_callback(ctx->r.req,
1257 CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
1258 kcryptd_async_done, dmreq_of_req(cc, ctx->r.req));
1259}
1260
1261static void crypt_alloc_req_aead(struct crypt_config *cc,
1262 struct convert_context *ctx)
1263{
1264 if (!ctx->r.req_aead)
1265 ctx->r.req_aead = mempool_alloc(cc->req_pool, GFP_NOIO);
1266
1267 aead_request_set_tfm(ctx->r.req_aead, cc->cipher_tfm.tfms_aead[0]);
1268
1269
1270
1271
1272
1273 aead_request_set_callback(ctx->r.req_aead,
1274 CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
1275 kcryptd_async_done, dmreq_of_req(cc, ctx->r.req_aead));
1276}
1277
1278static void crypt_alloc_req(struct crypt_config *cc,
1279 struct convert_context *ctx)
1280{
1281 if (crypt_integrity_aead(cc))
1282 crypt_alloc_req_aead(cc, ctx);
1283 else
1284 crypt_alloc_req_skcipher(cc, ctx);
1285}
1286
1287static void crypt_free_req_skcipher(struct crypt_config *cc,
1288 struct skcipher_request *req, struct bio *base_bio)
1289{
1290 struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
1291
1292 if ((struct skcipher_request *)(io + 1) != req)
1293 mempool_free(req, cc->req_pool);
1294}
1295
1296static void crypt_free_req_aead(struct crypt_config *cc,
1297 struct aead_request *req, struct bio *base_bio)
1298{
1299 struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
1300
1301 if ((struct aead_request *)(io + 1) != req)
1302 mempool_free(req, cc->req_pool);
1303}
1304
1305static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio)
1306{
1307 if (crypt_integrity_aead(cc))
1308 crypt_free_req_aead(cc, req, base_bio);
1309 else
1310 crypt_free_req_skcipher(cc, req, base_bio);
1311}
1312
1313
1314
1315
1316static blk_status_t crypt_convert(struct crypt_config *cc,
1317 struct convert_context *ctx)
1318{
1319 unsigned int tag_offset = 0;
1320 unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
1321 int r;
1322
1323 atomic_set(&ctx->cc_pending, 1);
1324
1325 while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
1326
1327 crypt_alloc_req(cc, ctx);
1328 atomic_inc(&ctx->cc_pending);
1329
1330 if (crypt_integrity_aead(cc))
1331 r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, tag_offset);
1332 else
1333 r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, tag_offset);
1334
1335 switch (r) {
1336
1337
1338
1339
1340 case -EBUSY:
1341 wait_for_completion(&ctx->restart);
1342 reinit_completion(&ctx->restart);
1343
1344
1345
1346
1347
1348 case -EINPROGRESS:
1349 ctx->r.req = NULL;
1350 ctx->cc_sector += sector_step;
1351 tag_offset++;
1352 continue;
1353
1354
1355
1356 case 0:
1357 atomic_dec(&ctx->cc_pending);
1358 ctx->cc_sector += sector_step;
1359 tag_offset++;
1360 cond_resched();
1361 continue;
1362
1363
1364
1365 case -EBADMSG:
1366 atomic_dec(&ctx->cc_pending);
1367 return BLK_STS_PROTECTION;
1368
1369
1370
1371 default:
1372 atomic_dec(&ctx->cc_pending);
1373 return BLK_STS_IOERR;
1374 }
1375 }
1376
1377 return 0;
1378}
1379
1380static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size)
1400{
1401 struct crypt_config *cc = io->cc;
1402 struct bio *clone;
1403 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1404 gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
1405 unsigned i, len, remaining_size;
1406 struct page *page;
1407
1408retry:
1409 if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
1410 mutex_lock(&cc->bio_alloc_lock);
1411
1412 clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs);
1413 if (!clone)
1414 goto out;
1415
1416 clone_init(io, clone);
1417
1418 remaining_size = size;
1419
1420 for (i = 0; i < nr_iovecs; i++) {
1421 page = mempool_alloc(cc->page_pool, gfp_mask);
1422 if (!page) {
1423 crypt_free_buffer_pages(cc, clone);
1424 bio_put(clone);
1425 gfp_mask |= __GFP_DIRECT_RECLAIM;
1426 goto retry;
1427 }
1428
1429 len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size;
1430
1431 bio_add_page(clone, page, len, 0);
1432
1433 remaining_size -= len;
1434 }
1435
1436
1437 if (dm_crypt_integrity_io_alloc(io, clone)) {
1438 crypt_free_buffer_pages(cc, clone);
1439 bio_put(clone);
1440 clone = NULL;
1441 }
1442out:
1443 if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
1444 mutex_unlock(&cc->bio_alloc_lock);
1445
1446 return clone;
1447}
1448
1449static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
1450{
1451 unsigned int i;
1452 struct bio_vec *bv;
1453
1454 bio_for_each_segment_all(bv, clone, i) {
1455 BUG_ON(!bv->bv_page);
1456 mempool_free(bv->bv_page, cc->page_pool);
1457 }
1458}
1459
1460static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
1461 struct bio *bio, sector_t sector)
1462{
1463 io->cc = cc;
1464 io->base_bio = bio;
1465 io->sector = sector;
1466 io->error = 0;
1467 io->ctx.r.req = NULL;
1468 io->integrity_metadata = NULL;
1469 io->integrity_metadata_from_pool = false;
1470 atomic_set(&io->io_pending, 0);
1471}
1472
1473static void crypt_inc_pending(struct dm_crypt_io *io)
1474{
1475 atomic_inc(&io->io_pending);
1476}
1477
1478
1479
1480
1481
1482static void crypt_dec_pending(struct dm_crypt_io *io)
1483{
1484 struct crypt_config *cc = io->cc;
1485 struct bio *base_bio = io->base_bio;
1486 blk_status_t error = io->error;
1487
1488 if (!atomic_dec_and_test(&io->io_pending))
1489 return;
1490
1491 if (io->ctx.r.req)
1492 crypt_free_req(cc, io->ctx.r.req, base_bio);
1493
1494 if (unlikely(io->integrity_metadata_from_pool))
1495 mempool_free(io->integrity_metadata, io->cc->tag_pool);
1496 else
1497 kfree(io->integrity_metadata);
1498
1499 base_bio->bi_status = error;
1500 bio_endio(base_bio);
1501}
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520static void crypt_endio(struct bio *clone)
1521{
1522 struct dm_crypt_io *io = clone->bi_private;
1523 struct crypt_config *cc = io->cc;
1524 unsigned rw = bio_data_dir(clone);
1525 blk_status_t error;
1526
1527
1528
1529
1530 if (rw == WRITE)
1531 crypt_free_buffer_pages(cc, clone);
1532
1533 error = clone->bi_status;
1534 bio_put(clone);
1535
1536 if (rw == READ && !error) {
1537 kcryptd_queue_crypt(io);
1538 return;
1539 }
1540
1541 if (unlikely(error))
1542 io->error = error;
1543
1544 crypt_dec_pending(io);
1545}
1546
1547static void clone_init(struct dm_crypt_io *io, struct bio *clone)
1548{
1549 struct crypt_config *cc = io->cc;
1550
1551 clone->bi_private = io;
1552 clone->bi_end_io = crypt_endio;
1553 bio_set_dev(clone, cc->dev->bdev);
1554 clone->bi_opf = io->base_bio->bi_opf;
1555}
1556
1557static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
1558{
1559 struct crypt_config *cc = io->cc;
1560 struct bio *clone;
1561
1562
1563
1564
1565
1566
1567
1568 clone = bio_clone_fast(io->base_bio, gfp, cc->bs);
1569 if (!clone)
1570 return 1;
1571
1572 crypt_inc_pending(io);
1573
1574 clone_init(io, clone);
1575 clone->bi_iter.bi_sector = cc->start + io->sector;
1576
1577 if (dm_crypt_integrity_io_alloc(io, clone)) {
1578 crypt_dec_pending(io);
1579 bio_put(clone);
1580 return 1;
1581 }
1582
1583 generic_make_request(clone);
1584 return 0;
1585}
1586
1587static void kcryptd_io_read_work(struct work_struct *work)
1588{
1589 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1590
1591 crypt_inc_pending(io);
1592 if (kcryptd_io_read(io, GFP_NOIO))
1593 io->error = BLK_STS_RESOURCE;
1594 crypt_dec_pending(io);
1595}
1596
1597static void kcryptd_queue_read(struct dm_crypt_io *io)
1598{
1599 struct crypt_config *cc = io->cc;
1600
1601 INIT_WORK(&io->work, kcryptd_io_read_work);
1602 queue_work(cc->io_queue, &io->work);
1603}
1604
1605static void kcryptd_io_write(struct dm_crypt_io *io)
1606{
1607 struct bio *clone = io->ctx.bio_out;
1608
1609 generic_make_request(clone);
1610}
1611
1612#define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
1613
1614static int dmcrypt_write(void *data)
1615{
1616 struct crypt_config *cc = data;
1617 struct dm_crypt_io *io;
1618
1619 while (1) {
1620 struct rb_root write_tree;
1621 struct blk_plug plug;
1622
1623 DECLARE_WAITQUEUE(wait, current);
1624
1625 spin_lock_irq(&cc->write_thread_wait.lock);
1626continue_locked:
1627
1628 if (!RB_EMPTY_ROOT(&cc->write_tree))
1629 goto pop_from_list;
1630
1631 set_current_state(TASK_INTERRUPTIBLE);
1632 __add_wait_queue(&cc->write_thread_wait, &wait);
1633
1634 spin_unlock_irq(&cc->write_thread_wait.lock);
1635
1636 if (unlikely(kthread_should_stop())) {
1637 set_current_state(TASK_RUNNING);
1638 remove_wait_queue(&cc->write_thread_wait, &wait);
1639 break;
1640 }
1641
1642 schedule();
1643
1644 set_current_state(TASK_RUNNING);
1645 spin_lock_irq(&cc->write_thread_wait.lock);
1646 __remove_wait_queue(&cc->write_thread_wait, &wait);
1647 goto continue_locked;
1648
1649pop_from_list:
1650 write_tree = cc->write_tree;
1651 cc->write_tree = RB_ROOT;
1652 spin_unlock_irq(&cc->write_thread_wait.lock);
1653
1654 BUG_ON(rb_parent(write_tree.rb_node));
1655
1656
1657
1658
1659
1660 blk_start_plug(&plug);
1661 do {
1662 io = crypt_io_from_node(rb_first(&write_tree));
1663 rb_erase(&io->rb_node, &write_tree);
1664 kcryptd_io_write(io);
1665 } while (!RB_EMPTY_ROOT(&write_tree));
1666 blk_finish_plug(&plug);
1667 }
1668 return 0;
1669}
1670
1671static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
1672{
1673 struct bio *clone = io->ctx.bio_out;
1674 struct crypt_config *cc = io->cc;
1675 unsigned long flags;
1676 sector_t sector;
1677 struct rb_node **rbp, *parent;
1678
1679 if (unlikely(io->error)) {
1680 crypt_free_buffer_pages(cc, clone);
1681 bio_put(clone);
1682 crypt_dec_pending(io);
1683 return;
1684 }
1685
1686
1687 BUG_ON(io->ctx.iter_out.bi_size);
1688
1689 clone->bi_iter.bi_sector = cc->start + io->sector;
1690
1691 if (likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) {
1692 generic_make_request(clone);
1693 return;
1694 }
1695
1696 spin_lock_irqsave(&cc->write_thread_wait.lock, flags);
1697 rbp = &cc->write_tree.rb_node;
1698 parent = NULL;
1699 sector = io->sector;
1700 while (*rbp) {
1701 parent = *rbp;
1702 if (sector < crypt_io_from_node(parent)->sector)
1703 rbp = &(*rbp)->rb_left;
1704 else
1705 rbp = &(*rbp)->rb_right;
1706 }
1707 rb_link_node(&io->rb_node, parent, rbp);
1708 rb_insert_color(&io->rb_node, &cc->write_tree);
1709
1710 wake_up_locked(&cc->write_thread_wait);
1711 spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags);
1712}
1713
1714static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
1715{
1716 struct crypt_config *cc = io->cc;
1717 struct bio *clone;
1718 int crypt_finished;
1719 sector_t sector = io->sector;
1720 blk_status_t r;
1721
1722
1723
1724
1725 crypt_inc_pending(io);
1726 crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector);
1727
1728 clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
1729 if (unlikely(!clone)) {
1730 io->error = BLK_STS_IOERR;
1731 goto dec;
1732 }
1733
1734 io->ctx.bio_out = clone;
1735 io->ctx.iter_out = clone->bi_iter;
1736
1737 sector += bio_sectors(clone);
1738
1739 crypt_inc_pending(io);
1740 r = crypt_convert(cc, &io->ctx);
1741 if (r)
1742 io->error = r;
1743 crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending);
1744
1745
1746 if (crypt_finished) {
1747 kcryptd_crypt_write_io_submit(io, 0);
1748 io->sector = sector;
1749 }
1750
1751dec:
1752 crypt_dec_pending(io);
1753}
1754
1755static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
1756{
1757 crypt_dec_pending(io);
1758}
1759
1760static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
1761{
1762 struct crypt_config *cc = io->cc;
1763 blk_status_t r;
1764
1765 crypt_inc_pending(io);
1766
1767 crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
1768 io->sector);
1769
1770 r = crypt_convert(cc, &io->ctx);
1771 if (r)
1772 io->error = r;
1773
1774 if (atomic_dec_and_test(&io->ctx.cc_pending))
1775 kcryptd_crypt_read_done(io);
1776
1777 crypt_dec_pending(io);
1778}
1779
1780static void kcryptd_async_done(struct crypto_async_request *async_req,
1781 int error)
1782{
1783 struct dm_crypt_request *dmreq = async_req->data;
1784 struct convert_context *ctx = dmreq->ctx;
1785 struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
1786 struct crypt_config *cc = io->cc;
1787
1788
1789
1790
1791
1792
1793 if (error == -EINPROGRESS) {
1794 complete(&ctx->restart);
1795 return;
1796 }
1797
1798 if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
1799 error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq);
1800
1801 if (error == -EBADMSG) {
1802 DMERR_LIMIT("INTEGRITY AEAD ERROR, sector %llu",
1803 (unsigned long long)le64_to_cpu(*org_sector_of_dmreq(cc, dmreq)));
1804 io->error = BLK_STS_PROTECTION;
1805 } else if (error < 0)
1806 io->error = BLK_STS_IOERR;
1807
1808 crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);
1809
1810 if (!atomic_dec_and_test(&ctx->cc_pending))
1811 return;
1812
1813 if (bio_data_dir(io->base_bio) == READ)
1814 kcryptd_crypt_read_done(io);
1815 else
1816 kcryptd_crypt_write_io_submit(io, 1);
1817}
1818
1819static void kcryptd_crypt(struct work_struct *work)
1820{
1821 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1822
1823 if (bio_data_dir(io->base_bio) == READ)
1824 kcryptd_crypt_read_convert(io);
1825 else
1826 kcryptd_crypt_write_convert(io);
1827}
1828
1829static void kcryptd_queue_crypt(struct dm_crypt_io *io)
1830{
1831 struct crypt_config *cc = io->cc;
1832
1833 INIT_WORK(&io->work, kcryptd_crypt);
1834 queue_work(cc->crypt_queue, &io->work);
1835}
1836
1837static void crypt_free_tfms_aead(struct crypt_config *cc)
1838{
1839 if (!cc->cipher_tfm.tfms_aead)
1840 return;
1841
1842 if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(cc->cipher_tfm.tfms_aead[0])) {
1843 crypto_free_aead(cc->cipher_tfm.tfms_aead[0]);
1844 cc->cipher_tfm.tfms_aead[0] = NULL;
1845 }
1846
1847 kfree(cc->cipher_tfm.tfms_aead);
1848 cc->cipher_tfm.tfms_aead = NULL;
1849}
1850
1851static void crypt_free_tfms_skcipher(struct crypt_config *cc)
1852{
1853 unsigned i;
1854
1855 if (!cc->cipher_tfm.tfms)
1856 return;
1857
1858 for (i = 0; i < cc->tfms_count; i++)
1859 if (cc->cipher_tfm.tfms[i] && !IS_ERR(cc->cipher_tfm.tfms[i])) {
1860 crypto_free_skcipher(cc->cipher_tfm.tfms[i]);
1861 cc->cipher_tfm.tfms[i] = NULL;
1862 }
1863
1864 kfree(cc->cipher_tfm.tfms);
1865 cc->cipher_tfm.tfms = NULL;
1866}
1867
1868static void crypt_free_tfms(struct crypt_config *cc)
1869{
1870 if (crypt_integrity_aead(cc))
1871 crypt_free_tfms_aead(cc);
1872 else
1873 crypt_free_tfms_skcipher(cc);
1874}
1875
1876static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode)
1877{
1878 unsigned i;
1879 int err;
1880
1881 cc->cipher_tfm.tfms = kzalloc(cc->tfms_count *
1882 sizeof(struct crypto_skcipher *), GFP_KERNEL);
1883 if (!cc->cipher_tfm.tfms)
1884 return -ENOMEM;
1885
1886 for (i = 0; i < cc->tfms_count; i++) {
1887 cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(ciphermode, 0, 0);
1888 if (IS_ERR(cc->cipher_tfm.tfms[i])) {
1889 err = PTR_ERR(cc->cipher_tfm.tfms[i]);
1890 crypt_free_tfms(cc);
1891 return err;
1892 }
1893 }
1894
1895 return 0;
1896}
1897
1898static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode)
1899{
1900 int err;
1901
1902 cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL);
1903 if (!cc->cipher_tfm.tfms)
1904 return -ENOMEM;
1905
1906 cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(ciphermode, 0, 0);
1907 if (IS_ERR(cc->cipher_tfm.tfms_aead[0])) {
1908 err = PTR_ERR(cc->cipher_tfm.tfms_aead[0]);
1909 crypt_free_tfms(cc);
1910 return err;
1911 }
1912
1913 return 0;
1914}
1915
1916static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
1917{
1918 if (crypt_integrity_aead(cc))
1919 return crypt_alloc_tfms_aead(cc, ciphermode);
1920 else
1921 return crypt_alloc_tfms_skcipher(cc, ciphermode);
1922}
1923
1924static unsigned crypt_subkey_size(struct crypt_config *cc)
1925{
1926 return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);
1927}
1928
1929static unsigned crypt_authenckey_size(struct crypt_config *cc)
1930{
1931 return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param));
1932}
1933
1934
1935
1936
1937
1938
1939static void crypt_copy_authenckey(char *p, const void *key,
1940 unsigned enckeylen, unsigned authkeylen)
1941{
1942 struct crypto_authenc_key_param *param;
1943 struct rtattr *rta;
1944
1945 rta = (struct rtattr *)p;
1946 param = RTA_DATA(rta);
1947 param->enckeylen = cpu_to_be32(enckeylen);
1948 rta->rta_len = RTA_LENGTH(sizeof(*param));
1949 rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
1950 p += RTA_SPACE(sizeof(*param));
1951 memcpy(p, key + enckeylen, authkeylen);
1952 p += authkeylen;
1953 memcpy(p, key, enckeylen);
1954}
1955
1956static int crypt_setkey(struct crypt_config *cc)
1957{
1958 unsigned subkey_size;
1959 int err = 0, i, r;
1960
1961
1962 subkey_size = crypt_subkey_size(cc);
1963
1964 if (crypt_integrity_hmac(cc)) {
1965 if (subkey_size < cc->key_mac_size)
1966 return -EINVAL;
1967
1968 crypt_copy_authenckey(cc->authenc_key, cc->key,
1969 subkey_size - cc->key_mac_size,
1970 cc->key_mac_size);
1971 }
1972
1973 for (i = 0; i < cc->tfms_count; i++) {
1974 if (crypt_integrity_hmac(cc))
1975 r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],
1976 cc->authenc_key, crypt_authenckey_size(cc));
1977 else if (crypt_integrity_aead(cc))
1978 r = crypto_aead_setkey(cc->cipher_tfm.tfms_aead[i],
1979 cc->key + (i * subkey_size),
1980 subkey_size);
1981 else
1982 r = crypto_skcipher_setkey(cc->cipher_tfm.tfms[i],
1983 cc->key + (i * subkey_size),
1984 subkey_size);
1985 if (r)
1986 err = r;
1987 }
1988
1989 if (crypt_integrity_hmac(cc))
1990 memzero_explicit(cc->authenc_key, crypt_authenckey_size(cc));
1991
1992 return err;
1993}
1994
1995#ifdef CONFIG_KEYS
1996
1997static bool contains_whitespace(const char *str)
1998{
1999 while (*str)
2000 if (isspace(*str++))
2001 return true;
2002 return false;
2003}
2004
2005static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2006{
2007 char *new_key_string, *key_desc;
2008 int ret;
2009 struct key *key;
2010 const struct user_key_payload *ukp;
2011
2012
2013
2014
2015
2016 if (contains_whitespace(key_string)) {
2017 DMERR("whitespace chars not allowed in key string");
2018 return -EINVAL;
2019 }
2020
2021
2022 key_desc = strpbrk(key_string, ":");
2023 if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
2024 return -EINVAL;
2025
2026 if (strncmp(key_string, "logon:", key_desc - key_string + 1) &&
2027 strncmp(key_string, "user:", key_desc - key_string + 1))
2028 return -EINVAL;
2029
2030 new_key_string = kstrdup(key_string, GFP_KERNEL);
2031 if (!new_key_string)
2032 return -ENOMEM;
2033
2034 key = request_key(key_string[0] == 'l' ? &key_type_logon : &key_type_user,
2035 key_desc + 1, NULL);
2036 if (IS_ERR(key)) {
2037 kzfree(new_key_string);
2038 return PTR_ERR(key);
2039 }
2040
2041 down_read(&key->sem);
2042
2043 ukp = user_key_payload_locked(key);
2044 if (!ukp) {
2045 up_read(&key->sem);
2046 key_put(key);
2047 kzfree(new_key_string);
2048 return -EKEYREVOKED;
2049 }
2050
2051 if (cc->key_size != ukp->datalen) {
2052 up_read(&key->sem);
2053 key_put(key);
2054 kzfree(new_key_string);
2055 return -EINVAL;
2056 }
2057
2058 memcpy(cc->key, ukp->data, cc->key_size);
2059
2060 up_read(&key->sem);
2061 key_put(key);
2062
2063
2064 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2065
2066 ret = crypt_setkey(cc);
2067
2068 if (!ret) {
2069 set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2070 kzfree(cc->key_string);
2071 cc->key_string = new_key_string;
2072 } else
2073 kzfree(new_key_string);
2074
2075 return ret;
2076}
2077
2078static int get_key_size(char **key_string)
2079{
2080 char *colon, dummy;
2081 int ret;
2082
2083 if (*key_string[0] != ':')
2084 return strlen(*key_string) >> 1;
2085
2086
2087 colon = strpbrk(*key_string + 1, ":");
2088 if (!colon)
2089 return -EINVAL;
2090
2091 if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
2092 return -EINVAL;
2093
2094 *key_string = colon;
2095
2096
2097
2098 return ret;
2099}
2100
2101#else
2102
2103static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2104{
2105 return -EINVAL;
2106}
2107
2108static int get_key_size(char **key_string)
2109{
2110 return (*key_string[0] == ':') ? -EINVAL : strlen(*key_string) >> 1;
2111}
2112
2113#endif
2114
2115static int crypt_set_key(struct crypt_config *cc, char *key)
2116{
2117 int r = -EINVAL;
2118 int key_string_len = strlen(key);
2119
2120
2121 if (!cc->key_size && strcmp(key, "-"))
2122 goto out;
2123
2124
2125 if (key[0] == ':') {
2126 r = crypt_set_keyring_key(cc, key + 1);
2127 goto out;
2128 }
2129
2130
2131 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2132
2133
2134 kzfree(cc->key_string);
2135 cc->key_string = NULL;
2136
2137
2138 if (cc->key_size && hex2bin(cc->key, key, cc->key_size) < 0)
2139 goto out;
2140
2141 r = crypt_setkey(cc);
2142 if (!r)
2143 set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2144
2145out:
2146
2147 memset(key, '0', key_string_len);
2148
2149 return r;
2150}
2151
2152static int crypt_wipe_key(struct crypt_config *cc)
2153{
2154 int r;
2155
2156 clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
2157 get_random_bytes(&cc->key, cc->key_size);
2158 kzfree(cc->key_string);
2159 cc->key_string = NULL;
2160 r = crypt_setkey(cc);
2161 memset(&cc->key, 0, cc->key_size * sizeof(u8));
2162
2163 return r;
2164}
2165
2166static void crypt_calculate_pages_per_client(void)
2167{
2168 unsigned long pages = (totalram_pages - totalhigh_pages) * DM_CRYPT_MEMORY_PERCENT / 100;
2169
2170 if (!dm_crypt_clients_n)
2171 return;
2172
2173 pages /= dm_crypt_clients_n;
2174 if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT)
2175 pages = DM_CRYPT_MIN_PAGES_PER_CLIENT;
2176 dm_crypt_pages_per_client = pages;
2177}
2178
2179static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data)
2180{
2181 struct crypt_config *cc = pool_data;
2182 struct page *page;
2183
2184 if (unlikely(percpu_counter_compare(&cc->n_allocated_pages, dm_crypt_pages_per_client) >= 0) &&
2185 likely(gfp_mask & __GFP_NORETRY))
2186 return NULL;
2187
2188 page = alloc_page(gfp_mask);
2189 if (likely(page != NULL))
2190 percpu_counter_add(&cc->n_allocated_pages, 1);
2191
2192 return page;
2193}
2194
2195static void crypt_page_free(void *page, void *pool_data)
2196{
2197 struct crypt_config *cc = pool_data;
2198
2199 __free_page(page);
2200 percpu_counter_sub(&cc->n_allocated_pages, 1);
2201}
2202
2203static void crypt_dtr(struct dm_target *ti)
2204{
2205 struct crypt_config *cc = ti->private;
2206
2207 ti->private = NULL;
2208
2209 if (!cc)
2210 return;
2211
2212 if (cc->write_thread)
2213 kthread_stop(cc->write_thread);
2214
2215 if (cc->io_queue)
2216 destroy_workqueue(cc->io_queue);
2217 if (cc->crypt_queue)
2218 destroy_workqueue(cc->crypt_queue);
2219
2220 crypt_free_tfms(cc);
2221
2222 if (cc->bs)
2223 bioset_free(cc->bs);
2224
2225 mempool_destroy(cc->page_pool);
2226 mempool_destroy(cc->req_pool);
2227 mempool_destroy(cc->tag_pool);
2228
2229 if (cc->page_pool)
2230 WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);
2231 percpu_counter_destroy(&cc->n_allocated_pages);
2232
2233 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
2234 cc->iv_gen_ops->dtr(cc);
2235
2236 if (cc->dev)
2237 dm_put_device(ti, cc->dev);
2238
2239 kzfree(cc->cipher);
2240 kzfree(cc->cipher_string);
2241 kzfree(cc->key_string);
2242 kzfree(cc->cipher_auth);
2243 kzfree(cc->authenc_key);
2244
2245 mutex_destroy(&cc->bio_alloc_lock);
2246
2247
2248 kzfree(cc);
2249
2250 spin_lock(&dm_crypt_clients_lock);
2251 WARN_ON(!dm_crypt_clients_n);
2252 dm_crypt_clients_n--;
2253 crypt_calculate_pages_per_client();
2254 spin_unlock(&dm_crypt_clients_lock);
2255}
2256
2257static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
2258{
2259 struct crypt_config *cc = ti->private;
2260
2261 if (crypt_integrity_aead(cc))
2262 cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
2263 else
2264 cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
2265
2266 if (cc->iv_size)
2267
2268 cc->iv_size = max(cc->iv_size,
2269 (unsigned int)(sizeof(u64) / sizeof(u8)));
2270 else if (ivmode) {
2271 DMWARN("Selected cipher does not support IVs");
2272 ivmode = NULL;
2273 }
2274
2275
2276 if (ivmode == NULL)
2277 cc->iv_gen_ops = NULL;
2278 else if (strcmp(ivmode, "plain") == 0)
2279 cc->iv_gen_ops = &crypt_iv_plain_ops;
2280 else if (strcmp(ivmode, "plain64") == 0)
2281 cc->iv_gen_ops = &crypt_iv_plain64_ops;
2282 else if (strcmp(ivmode, "plain64be") == 0)
2283 cc->iv_gen_ops = &crypt_iv_plain64be_ops;
2284 else if (strcmp(ivmode, "essiv") == 0)
2285 cc->iv_gen_ops = &crypt_iv_essiv_ops;
2286 else if (strcmp(ivmode, "benbi") == 0)
2287 cc->iv_gen_ops = &crypt_iv_benbi_ops;
2288 else if (strcmp(ivmode, "null") == 0)
2289 cc->iv_gen_ops = &crypt_iv_null_ops;
2290 else if (strcmp(ivmode, "lmk") == 0) {
2291 cc->iv_gen_ops = &crypt_iv_lmk_ops;
2292
2293
2294
2295
2296
2297
2298 if (cc->key_size % cc->key_parts) {
2299 cc->key_parts++;
2300 cc->key_extra_size = cc->key_size / cc->key_parts;
2301 }
2302 } else if (strcmp(ivmode, "tcw") == 0) {
2303 cc->iv_gen_ops = &crypt_iv_tcw_ops;
2304 cc->key_parts += 2;
2305 cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
2306 } else if (strcmp(ivmode, "random") == 0) {
2307 cc->iv_gen_ops = &crypt_iv_random_ops;
2308
2309 cc->integrity_iv_size = cc->iv_size;
2310 } else {
2311 ti->error = "Invalid IV mode";
2312 return -EINVAL;
2313 }
2314
2315 return 0;
2316}
2317
2318
2319
2320
2321
2322
2323static int crypt_ctr_blkdev_cipher(struct crypt_config *cc)
2324{
2325 const char *alg_name = NULL;
2326 char *start, *end;
2327
2328 if (crypt_integrity_aead(cc)) {
2329 alg_name = crypto_tfm_alg_name(crypto_aead_tfm(any_tfm_aead(cc)));
2330 if (!alg_name)
2331 return -EINVAL;
2332 if (crypt_integrity_hmac(cc)) {
2333 alg_name = strchr(alg_name, ',');
2334 if (!alg_name)
2335 return -EINVAL;
2336 }
2337 alg_name++;
2338 } else {
2339 alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(any_tfm(cc)));
2340 if (!alg_name)
2341 return -EINVAL;
2342 }
2343
2344 start = strchr(alg_name, '(');
2345 end = strchr(alg_name, ')');
2346
2347 if (!start && !end) {
2348 cc->cipher = kstrdup(alg_name, GFP_KERNEL);
2349 return cc->cipher ? 0 : -ENOMEM;
2350 }
2351
2352 if (!start || !end || ++start >= end)
2353 return -EINVAL;
2354
2355 cc->cipher = kzalloc(end - start + 1, GFP_KERNEL);
2356 if (!cc->cipher)
2357 return -ENOMEM;
2358
2359 strncpy(cc->cipher, start, end - start);
2360
2361 return 0;
2362}
2363
2364
2365
2366
2367
2368
2369static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api)
2370{
2371 char *start, *end, *mac_alg = NULL;
2372 struct crypto_ahash *mac;
2373
2374 if (!strstarts(cipher_api, "authenc("))
2375 return 0;
2376
2377 start = strchr(cipher_api, '(');
2378 end = strchr(cipher_api, ',');
2379 if (!start || !end || ++start > end)
2380 return -EINVAL;
2381
2382 mac_alg = kzalloc(end - start + 1, GFP_KERNEL);
2383 if (!mac_alg)
2384 return -ENOMEM;
2385 strncpy(mac_alg, start, end - start);
2386
2387 mac = crypto_alloc_ahash(mac_alg, 0, 0);
2388 kfree(mac_alg);
2389
2390 if (IS_ERR(mac))
2391 return PTR_ERR(mac);
2392
2393 cc->key_mac_size = crypto_ahash_digestsize(mac);
2394 crypto_free_ahash(mac);
2395
2396 cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL);
2397 if (!cc->authenc_key)
2398 return -ENOMEM;
2399
2400 return 0;
2401}
2402
2403static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key,
2404 char **ivmode, char **ivopts)
2405{
2406 struct crypt_config *cc = ti->private;
2407 char *tmp, *cipher_api;
2408 int ret = -EINVAL;
2409
2410 cc->tfms_count = 1;
2411
2412
2413
2414
2415
2416 tmp = &cipher_in[strlen("capi:")];
2417 cipher_api = strsep(&tmp, "-");
2418 *ivmode = strsep(&tmp, ":");
2419 *ivopts = tmp;
2420
2421 if (*ivmode && !strcmp(*ivmode, "lmk"))
2422 cc->tfms_count = 64;
2423
2424 cc->key_parts = cc->tfms_count;
2425
2426
2427 ret = crypt_alloc_tfms(cc, cipher_api);
2428 if (ret < 0) {
2429 ti->error = "Error allocating crypto tfm";
2430 return ret;
2431 }
2432
2433
2434 if (crypt_integrity_aead(cc)) {
2435 ret = crypt_ctr_auth_cipher(cc, cipher_api);
2436 if (ret < 0) {
2437 ti->error = "Invalid AEAD cipher spec";
2438 return -ENOMEM;
2439 }
2440 cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
2441 } else
2442 cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
2443
2444 ret = crypt_ctr_blkdev_cipher(cc);
2445 if (ret < 0) {
2446 ti->error = "Cannot allocate cipher string";
2447 return -ENOMEM;
2448 }
2449
2450 return 0;
2451}
2452
2453static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key,
2454 char **ivmode, char **ivopts)
2455{
2456 struct crypt_config *cc = ti->private;
2457 char *tmp, *cipher, *chainmode, *keycount;
2458 char *cipher_api = NULL;
2459 int ret = -EINVAL;
2460 char dummy;
2461
2462 if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) {
2463 ti->error = "Bad cipher specification";
2464 return -EINVAL;
2465 }
2466
2467
2468
2469
2470
2471 tmp = cipher_in;
2472 keycount = strsep(&tmp, "-");
2473 cipher = strsep(&keycount, ":");
2474
2475 if (!keycount)
2476 cc->tfms_count = 1;
2477 else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||
2478 !is_power_of_2(cc->tfms_count)) {
2479 ti->error = "Bad cipher key count specification";
2480 return -EINVAL;
2481 }
2482 cc->key_parts = cc->tfms_count;
2483
2484 cc->cipher = kstrdup(cipher, GFP_KERNEL);
2485 if (!cc->cipher)
2486 goto bad_mem;
2487
2488 chainmode = strsep(&tmp, "-");
2489 *ivopts = strsep(&tmp, "-");
2490 *ivmode = strsep(&*ivopts, ":");
2491
2492 if (tmp)
2493 DMWARN("Ignoring unexpected additional cipher options");
2494
2495
2496
2497
2498
2499 if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) {
2500 chainmode = "cbc";
2501 *ivmode = "plain";
2502 }
2503
2504 if (strcmp(chainmode, "ecb") && !*ivmode) {
2505 ti->error = "IV mechanism required";
2506 return -EINVAL;
2507 }
2508
2509 cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
2510 if (!cipher_api)
2511 goto bad_mem;
2512
2513 ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
2514 "%s(%s)", chainmode, cipher);
2515 if (ret < 0) {
2516 kfree(cipher_api);
2517 goto bad_mem;
2518 }
2519
2520
2521 ret = crypt_alloc_tfms(cc, cipher_api);
2522 if (ret < 0) {
2523 ti->error = "Error allocating crypto tfm";
2524 kfree(cipher_api);
2525 return ret;
2526 }
2527 kfree(cipher_api);
2528
2529 return 0;
2530bad_mem:
2531 ti->error = "Cannot allocate cipher strings";
2532 return -ENOMEM;
2533}
2534
2535static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
2536{
2537 struct crypt_config *cc = ti->private;
2538 char *ivmode = NULL, *ivopts = NULL;
2539 int ret;
2540
2541 cc->cipher_string = kstrdup(cipher_in, GFP_KERNEL);
2542 if (!cc->cipher_string) {
2543 ti->error = "Cannot allocate cipher strings";
2544 return -ENOMEM;
2545 }
2546
2547 if (strstarts(cipher_in, "capi:"))
2548 ret = crypt_ctr_cipher_new(ti, cipher_in, key, &ivmode, &ivopts);
2549 else
2550 ret = crypt_ctr_cipher_old(ti, cipher_in, key, &ivmode, &ivopts);
2551 if (ret)
2552 return ret;
2553
2554
2555 ret = crypt_ctr_ivmode(ti, ivmode);
2556 if (ret < 0)
2557 return ret;
2558
2559
2560 ret = crypt_set_key(cc, key);
2561 if (ret < 0) {
2562 ti->error = "Error decoding and setting key";
2563 return ret;
2564 }
2565
2566
2567 if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
2568 ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
2569 if (ret < 0) {
2570 ti->error = "Error creating IV";
2571 return ret;
2572 }
2573 }
2574
2575
2576 if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
2577 ret = cc->iv_gen_ops->init(cc);
2578 if (ret < 0) {
2579 ti->error = "Error initialising IV";
2580 return ret;
2581 }
2582 }
2583
2584
2585 if (cc->key_string)
2586 memset(cc->key, 0, cc->key_size * sizeof(u8));
2587
2588 return ret;
2589}
2590
2591static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv)
2592{
2593 struct crypt_config *cc = ti->private;
2594 struct dm_arg_set as;
2595 static const struct dm_arg _args[] = {
2596 {0, 6, "Invalid number of feature args"},
2597 };
2598 unsigned int opt_params, val;
2599 const char *opt_string, *sval;
2600 char dummy;
2601 int ret;
2602
2603
2604 as.argc = argc;
2605 as.argv = argv;
2606
2607 ret = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
2608 if (ret)
2609 return ret;
2610
2611 while (opt_params--) {
2612 opt_string = dm_shift_arg(&as);
2613 if (!opt_string) {
2614 ti->error = "Not enough feature arguments";
2615 return -EINVAL;
2616 }
2617
2618 if (!strcasecmp(opt_string, "allow_discards"))
2619 ti->num_discard_bios = 1;
2620
2621 else if (!strcasecmp(opt_string, "same_cpu_crypt"))
2622 set_bit(DM_CRYPT_SAME_CPU, &cc->flags);
2623
2624 else if (!strcasecmp(opt_string, "submit_from_crypt_cpus"))
2625 set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
2626 else if (sscanf(opt_string, "integrity:%u:", &val) == 1) {
2627 if (val == 0 || val > MAX_TAG_SIZE) {
2628 ti->error = "Invalid integrity arguments";
2629 return -EINVAL;
2630 }
2631 cc->on_disk_tag_size = val;
2632 sval = strchr(opt_string + strlen("integrity:"), ':') + 1;
2633 if (!strcasecmp(sval, "aead")) {
2634 set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
2635 } else if (strcasecmp(sval, "none")) {
2636 ti->error = "Unknown integrity profile";
2637 return -EINVAL;
2638 }
2639
2640 cc->cipher_auth = kstrdup(sval, GFP_KERNEL);
2641 if (!cc->cipher_auth)
2642 return -ENOMEM;
2643 } else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) {
2644 if (cc->sector_size < (1 << SECTOR_SHIFT) ||
2645 cc->sector_size > 4096 ||
2646 (cc->sector_size & (cc->sector_size - 1))) {
2647 ti->error = "Invalid feature value for sector_size";
2648 return -EINVAL;
2649 }
2650 if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) {
2651 ti->error = "Device size is not multiple of sector_size feature";
2652 return -EINVAL;
2653 }
2654 cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT;
2655 } else if (!strcasecmp(opt_string, "iv_large_sectors"))
2656 set_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
2657 else {
2658 ti->error = "Invalid feature arguments";
2659 return -EINVAL;
2660 }
2661 }
2662
2663 return 0;
2664}
2665
2666
2667
2668
2669
2670static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
2671{
2672 struct crypt_config *cc;
2673 int key_size;
2674 unsigned int align_mask;
2675 unsigned long long tmpll;
2676 int ret;
2677 size_t iv_size_padding, additional_req_size;
2678 char dummy;
2679
2680 if (argc < 5) {
2681 ti->error = "Not enough arguments";
2682 return -EINVAL;
2683 }
2684
2685 key_size = get_key_size(&argv[1]);
2686 if (key_size < 0) {
2687 ti->error = "Cannot parse key size";
2688 return -EINVAL;
2689 }
2690
2691 cc = kzalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
2692 if (!cc) {
2693 ti->error = "Cannot allocate encryption context";
2694 return -ENOMEM;
2695 }
2696 cc->key_size = key_size;
2697 cc->sector_size = (1 << SECTOR_SHIFT);
2698 cc->sector_shift = 0;
2699
2700 ti->private = cc;
2701
2702 spin_lock(&dm_crypt_clients_lock);
2703 dm_crypt_clients_n++;
2704 crypt_calculate_pages_per_client();
2705 spin_unlock(&dm_crypt_clients_lock);
2706
2707 ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL);
2708 if (ret < 0)
2709 goto bad;
2710
2711
2712 if (argc > 5) {
2713 ret = crypt_ctr_optional(ti, argc - 5, &argv[5]);
2714 if (ret)
2715 goto bad;
2716 }
2717
2718 ret = crypt_ctr_cipher(ti, argv[0], argv[1]);
2719 if (ret < 0)
2720 goto bad;
2721
2722 if (crypt_integrity_aead(cc)) {
2723 cc->dmreq_start = sizeof(struct aead_request);
2724 cc->dmreq_start += crypto_aead_reqsize(any_tfm_aead(cc));
2725 align_mask = crypto_aead_alignmask(any_tfm_aead(cc));
2726 } else {
2727 cc->dmreq_start = sizeof(struct skcipher_request);
2728 cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc));
2729 align_mask = crypto_skcipher_alignmask(any_tfm(cc));
2730 }
2731 cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));
2732
2733 if (align_mask < CRYPTO_MINALIGN) {
2734
2735 iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))
2736 & align_mask;
2737 } else {
2738
2739
2740
2741
2742
2743 iv_size_padding = align_mask;
2744 }
2745
2746 ret = -ENOMEM;
2747
2748
2749 additional_req_size = sizeof(struct dm_crypt_request) +
2750 iv_size_padding + cc->iv_size +
2751 cc->iv_size +
2752 sizeof(uint64_t) +
2753 sizeof(unsigned int);
2754
2755 cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + additional_req_size);
2756 if (!cc->req_pool) {
2757 ti->error = "Cannot allocate crypt request mempool";
2758 goto bad;
2759 }
2760
2761 cc->per_bio_data_size = ti->per_io_data_size =
2762 ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,
2763 ARCH_KMALLOC_MINALIGN);
2764
2765 cc->page_pool = mempool_create(BIO_MAX_PAGES, crypt_page_alloc, crypt_page_free, cc);
2766 if (!cc->page_pool) {
2767 ti->error = "Cannot allocate page mempool";
2768 goto bad;
2769 }
2770
2771 cc->bs = bioset_create(MIN_IOS, 0, BIOSET_NEED_BVECS);
2772 if (!cc->bs) {
2773 ti->error = "Cannot allocate crypt bioset";
2774 goto bad;
2775 }
2776
2777 mutex_init(&cc->bio_alloc_lock);
2778
2779 ret = -EINVAL;
2780 if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) ||
2781 (tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) {
2782 ti->error = "Invalid iv_offset sector";
2783 goto bad;
2784 }
2785 cc->iv_offset = tmpll;
2786
2787 ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), &cc->dev);
2788 if (ret) {
2789 ti->error = "Device lookup failed";
2790 goto bad;
2791 }
2792
2793 ret = -EINVAL;
2794 if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1) {
2795 ti->error = "Invalid device sector";
2796 goto bad;
2797 }
2798 cc->start = tmpll;
2799
2800 if (crypt_integrity_aead(cc) || cc->integrity_iv_size) {
2801 ret = crypt_integrity_ctr(cc, ti);
2802 if (ret)
2803 goto bad;
2804
2805 cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->on_disk_tag_size;
2806 if (!cc->tag_pool_max_sectors)
2807 cc->tag_pool_max_sectors = 1;
2808
2809 cc->tag_pool = mempool_create_kmalloc_pool(MIN_IOS,
2810 cc->tag_pool_max_sectors * cc->on_disk_tag_size);
2811 if (!cc->tag_pool) {
2812 ti->error = "Cannot allocate integrity tags mempool";
2813 ret = -ENOMEM;
2814 goto bad;
2815 }
2816
2817 cc->tag_pool_max_sectors <<= cc->sector_shift;
2818 }
2819
2820 ret = -ENOMEM;
2821 cc->io_queue = alloc_workqueue("kcryptd_io", WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);
2822 if (!cc->io_queue) {
2823 ti->error = "Couldn't create kcryptd io queue";
2824 goto bad;
2825 }
2826
2827 if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
2828 cc->crypt_queue = alloc_workqueue("kcryptd", WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1);
2829 else
2830 cc->crypt_queue = alloc_workqueue("kcryptd",
2831 WQ_HIGHPRI | WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND,
2832 num_online_cpus());
2833 if (!cc->crypt_queue) {
2834 ti->error = "Couldn't create kcryptd queue";
2835 goto bad;
2836 }
2837
2838 init_waitqueue_head(&cc->write_thread_wait);
2839 cc->write_tree = RB_ROOT;
2840
2841 cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write");
2842 if (IS_ERR(cc->write_thread)) {
2843 ret = PTR_ERR(cc->write_thread);
2844 cc->write_thread = NULL;
2845 ti->error = "Couldn't spawn write thread";
2846 goto bad;
2847 }
2848 wake_up_process(cc->write_thread);
2849
2850 ti->num_flush_bios = 1;
2851
2852 return 0;
2853
2854bad:
2855 crypt_dtr(ti);
2856 return ret;
2857}
2858
2859static int crypt_map(struct dm_target *ti, struct bio *bio)
2860{
2861 struct dm_crypt_io *io;
2862 struct crypt_config *cc = ti->private;
2863
2864
2865
2866
2867
2868
2869 if (unlikely(bio->bi_opf & REQ_PREFLUSH ||
2870 bio_op(bio) == REQ_OP_DISCARD)) {
2871 bio_set_dev(bio, cc->dev->bdev);
2872 if (bio_sectors(bio))
2873 bio->bi_iter.bi_sector = cc->start +
2874 dm_target_offset(ti, bio->bi_iter.bi_sector);
2875 return DM_MAPIO_REMAPPED;
2876 }
2877
2878
2879
2880
2881 if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_PAGES << PAGE_SHIFT)) &&
2882 (bio_data_dir(bio) == WRITE || cc->on_disk_tag_size))
2883 dm_accept_partial_bio(bio, ((BIO_MAX_PAGES << PAGE_SHIFT) >> SECTOR_SHIFT));
2884
2885
2886
2887
2888
2889 if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0))
2890 return DM_MAPIO_KILL;
2891
2892 if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1)))
2893 return DM_MAPIO_KILL;
2894
2895 io = dm_per_bio_data(bio, cc->per_bio_data_size);
2896 crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
2897
2898 if (cc->on_disk_tag_size) {
2899 unsigned tag_len = cc->on_disk_tag_size * (bio_sectors(bio) >> cc->sector_shift);
2900
2901 if (unlikely(tag_len > KMALLOC_MAX_SIZE) ||
2902 unlikely(!(io->integrity_metadata = kmalloc(tag_len,
2903 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN)))) {
2904 if (bio_sectors(bio) > cc->tag_pool_max_sectors)
2905 dm_accept_partial_bio(bio, cc->tag_pool_max_sectors);
2906 io->integrity_metadata = mempool_alloc(cc->tag_pool, GFP_NOIO);
2907 io->integrity_metadata_from_pool = true;
2908 }
2909 }
2910
2911 if (crypt_integrity_aead(cc))
2912 io->ctx.r.req_aead = (struct aead_request *)(io + 1);
2913 else
2914 io->ctx.r.req = (struct skcipher_request *)(io + 1);
2915
2916 if (bio_data_dir(io->base_bio) == READ) {
2917 if (kcryptd_io_read(io, GFP_NOWAIT))
2918 kcryptd_queue_read(io);
2919 } else
2920 kcryptd_queue_crypt(io);
2921
2922 return DM_MAPIO_SUBMITTED;
2923}
2924
2925static void crypt_status(struct dm_target *ti, status_type_t type,
2926 unsigned status_flags, char *result, unsigned maxlen)
2927{
2928 struct crypt_config *cc = ti->private;
2929 unsigned i, sz = 0;
2930 int num_feature_args = 0;
2931
2932 switch (type) {
2933 case STATUSTYPE_INFO:
2934 result[0] = '\0';
2935 break;
2936
2937 case STATUSTYPE_TABLE:
2938 DMEMIT("%s ", cc->cipher_string);
2939
2940 if (cc->key_size > 0) {
2941 if (cc->key_string)
2942 DMEMIT(":%u:%s", cc->key_size, cc->key_string);
2943 else
2944 for (i = 0; i < cc->key_size; i++)
2945 DMEMIT("%02x", cc->key[i]);
2946 } else
2947 DMEMIT("-");
2948
2949 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
2950 cc->dev->name, (unsigned long long)cc->start);
2951
2952 num_feature_args += !!ti->num_discard_bios;
2953 num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags);
2954 num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
2955 num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT);
2956 num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
2957 if (cc->on_disk_tag_size)
2958 num_feature_args++;
2959 if (num_feature_args) {
2960 DMEMIT(" %d", num_feature_args);
2961 if (ti->num_discard_bios)
2962 DMEMIT(" allow_discards");
2963 if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
2964 DMEMIT(" same_cpu_crypt");
2965 if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags))
2966 DMEMIT(" submit_from_crypt_cpus");
2967 if (cc->on_disk_tag_size)
2968 DMEMIT(" integrity:%u:%s", cc->on_disk_tag_size, cc->cipher_auth);
2969 if (cc->sector_size != (1 << SECTOR_SHIFT))
2970 DMEMIT(" sector_size:%d", cc->sector_size);
2971 if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
2972 DMEMIT(" iv_large_sectors");
2973 }
2974
2975 break;
2976 }
2977}
2978
2979static void crypt_postsuspend(struct dm_target *ti)
2980{
2981 struct crypt_config *cc = ti->private;
2982
2983 set_bit(DM_CRYPT_SUSPENDED, &cc->flags);
2984}
2985
2986static int crypt_preresume(struct dm_target *ti)
2987{
2988 struct crypt_config *cc = ti->private;
2989
2990 if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
2991 DMERR("aborting resume - crypt key is not set.");
2992 return -EAGAIN;
2993 }
2994
2995 return 0;
2996}
2997
2998static void crypt_resume(struct dm_target *ti)
2999{
3000 struct crypt_config *cc = ti->private;
3001
3002 clear_bit(DM_CRYPT_SUSPENDED, &cc->flags);
3003}
3004
3005
3006
3007
3008
3009static int crypt_message(struct dm_target *ti, unsigned argc, char **argv,
3010 char *result, unsigned maxlen)
3011{
3012 struct crypt_config *cc = ti->private;
3013 int key_size, ret = -EINVAL;
3014
3015 if (argc < 2)
3016 goto error;
3017
3018 if (!strcasecmp(argv[0], "key")) {
3019 if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
3020 DMWARN("not suspended during key manipulation.");
3021 return -EINVAL;
3022 }
3023 if (argc == 3 && !strcasecmp(argv[1], "set")) {
3024
3025 key_size = get_key_size(&argv[2]);
3026 if (key_size < 0 || cc->key_size != key_size) {
3027 memset(argv[2], '0', strlen(argv[2]));
3028 return -EINVAL;
3029 }
3030
3031 ret = crypt_set_key(cc, argv[2]);
3032 if (ret)
3033 return ret;
3034 if (cc->iv_gen_ops && cc->iv_gen_ops->init)
3035 ret = cc->iv_gen_ops->init(cc);
3036
3037 if (cc->key_string)
3038 memset(cc->key, 0, cc->key_size * sizeof(u8));
3039 return ret;
3040 }
3041 if (argc == 2 && !strcasecmp(argv[1], "wipe")) {
3042 if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
3043 ret = cc->iv_gen_ops->wipe(cc);
3044 if (ret)
3045 return ret;
3046 }
3047 return crypt_wipe_key(cc);
3048 }
3049 }
3050
3051error:
3052 DMWARN("unrecognised message received.");
3053 return -EINVAL;
3054}
3055
3056static int crypt_iterate_devices(struct dm_target *ti,
3057 iterate_devices_callout_fn fn, void *data)
3058{
3059 struct crypt_config *cc = ti->private;
3060
3061 return fn(ti, cc->dev, cc->start, ti->len, data);
3062}
3063
3064static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
3065{
3066 struct crypt_config *cc = ti->private;
3067
3068
3069
3070
3071
3072
3073
3074 limits->max_segment_size = PAGE_SIZE;
3075
3076 if (cc->sector_size != (1 << SECTOR_SHIFT)) {
3077 limits->logical_block_size = cc->sector_size;
3078 limits->physical_block_size = cc->sector_size;
3079 blk_limits_io_min(limits, cc->sector_size);
3080 }
3081}
3082
3083static struct target_type crypt_target = {
3084 .name = "crypt",
3085 .version = {1, 18, 1},
3086 .module = THIS_MODULE,
3087 .ctr = crypt_ctr,
3088 .dtr = crypt_dtr,
3089 .map = crypt_map,
3090 .status = crypt_status,
3091 .postsuspend = crypt_postsuspend,
3092 .preresume = crypt_preresume,
3093 .resume = crypt_resume,
3094 .message = crypt_message,
3095 .iterate_devices = crypt_iterate_devices,
3096 .io_hints = crypt_io_hints,
3097};
3098
3099static int __init dm_crypt_init(void)
3100{
3101 int r;
3102
3103 r = dm_register_target(&crypt_target);
3104 if (r < 0)
3105 DMERR("register failed %d", r);
3106
3107 return r;
3108}
3109
3110static void __exit dm_crypt_exit(void)
3111{
3112 dm_unregister_target(&crypt_target);
3113}
3114
3115module_init(dm_crypt_init);
3116module_exit(dm_crypt_exit);
3117
3118MODULE_AUTHOR("Jana Saout <jana@saout.de>");
3119MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
3120MODULE_LICENSE("GPL");
3121