1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include "dm-verity.h"
18
19#include <linux/module.h>
20#include <linux/reboot.h>
21
22#define DM_MSG_PREFIX "verity"
23
24#define DM_VERITY_ENV_LENGTH 42
25#define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR"
26
27#define DM_VERITY_MEMPOOL_SIZE 4
28#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
29
30#define DM_VERITY_MAX_CORRUPTED_ERRS 100
31
32#define DM_VERITY_OPT_LOGGING "ignore_corruption"
33#define DM_VERITY_OPT_RESTART "restart_on_corruption"
34
35#define DM_VERITY_OPTS_MAX 1
36
37static unsigned dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
38
39module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, S_IRUGO | S_IWUSR);
40
41struct dm_verity_prefetch_work {
42 struct work_struct work;
43 struct dm_verity *v;
44 sector_t block;
45 unsigned n_blocks;
46};
47
48
49
50
51
52
53
54
55
56
57
58
59
60struct buffer_aux {
61 int hash_verified;
62};
63
64
65
66
67static void dm_bufio_alloc_callback(struct dm_buffer *buf)
68{
69 struct buffer_aux *aux = dm_bufio_get_aux_data(buf);
70
71 aux->hash_verified = 0;
72}
73
74
75
76
77static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
78{
79 return v->data_start + dm_target_offset(v->ti, bi_sector);
80}
81
82
83
84
85
86
87
88static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
89 int level)
90{
91 return block >> (level * v->hash_per_block_bits);
92}
93
94
95
96
97static int verity_hash_init(struct dm_verity *v, struct shash_desc *desc)
98{
99 int r;
100
101 desc->tfm = v->tfm;
102 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
103
104 r = crypto_shash_init(desc);
105
106 if (unlikely(r < 0)) {
107 DMERR("crypto_shash_init failed: %d", r);
108 return r;
109 }
110
111 if (likely(v->version >= 1)) {
112 r = crypto_shash_update(desc, v->salt, v->salt_size);
113
114 if (unlikely(r < 0)) {
115 DMERR("crypto_shash_update failed: %d", r);
116 return r;
117 }
118 }
119
120 return 0;
121}
122
123static int verity_hash_update(struct dm_verity *v, struct shash_desc *desc,
124 const u8 *data, size_t len)
125{
126 int r = crypto_shash_update(desc, data, len);
127
128 if (unlikely(r < 0))
129 DMERR("crypto_shash_update failed: %d", r);
130
131 return r;
132}
133
134static int verity_hash_final(struct dm_verity *v, struct shash_desc *desc,
135 u8 *digest)
136{
137 int r;
138
139 if (unlikely(!v->version)) {
140 r = crypto_shash_update(desc, v->salt, v->salt_size);
141
142 if (r < 0) {
143 DMERR("crypto_shash_update failed: %d", r);
144 return r;
145 }
146 }
147
148 r = crypto_shash_final(desc, digest);
149
150 if (unlikely(r < 0))
151 DMERR("crypto_shash_final failed: %d", r);
152
153 return r;
154}
155
156int verity_hash(struct dm_verity *v, struct shash_desc *desc,
157 const u8 *data, size_t len, u8 *digest)
158{
159 int r;
160
161 r = verity_hash_init(v, desc);
162 if (unlikely(r < 0))
163 return r;
164
165 r = verity_hash_update(v, desc, data, len);
166 if (unlikely(r < 0))
167 return r;
168
169 return verity_hash_final(v, desc, digest);
170}
171
172static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
173 sector_t *hash_block, unsigned *offset)
174{
175 sector_t position = verity_position_at_level(v, block, level);
176 unsigned idx;
177
178 *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
179
180 if (!offset)
181 return;
182
183 idx = position & ((1 << v->hash_per_block_bits) - 1);
184 if (!v->version)
185 *offset = idx * v->digest_size;
186 else
187 *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);
188}
189
190
191
192
193static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,
194 unsigned long long block)
195{
196 char verity_env[DM_VERITY_ENV_LENGTH];
197 char *envp[] = { verity_env, NULL };
198 const char *type_str = "";
199 struct mapped_device *md = dm_table_get_md(v->ti->table);
200
201
202 v->hash_failed = 1;
203
204 if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)
205 goto out;
206
207 v->corrupted_errs++;
208
209 switch (type) {
210 case DM_VERITY_BLOCK_TYPE_DATA:
211 type_str = "data";
212 break;
213 case DM_VERITY_BLOCK_TYPE_METADATA:
214 type_str = "metadata";
215 break;
216 default:
217 BUG();
218 }
219
220 DMERR("%s: %s block %llu is corrupted", v->data_dev->name, type_str,
221 block);
222
223 if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS)
224 DMERR("%s: reached maximum errors", v->data_dev->name);
225
226 snprintf(verity_env, DM_VERITY_ENV_LENGTH, "%s=%d,%llu",
227 DM_VERITY_ENV_VAR_NAME, type, block);
228
229 kobject_uevent_env(&disk_to_dev(dm_disk(md))->kobj, KOBJ_CHANGE, envp);
230
231out:
232 if (v->mode == DM_VERITY_MODE_LOGGING)
233 return 0;
234
235 if (v->mode == DM_VERITY_MODE_RESTART)
236 kernel_restart("dm-verity device corrupted");
237
238 return 1;
239}
240
241
242
243
244
245
246
247
248
249
250
251
252static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
253 sector_t block, int level, bool skip_unverified,
254 u8 *want_digest)
255{
256 struct dm_buffer *buf;
257 struct buffer_aux *aux;
258 u8 *data;
259 int r;
260 sector_t hash_block;
261 unsigned offset;
262
263 verity_hash_at_level(v, block, level, &hash_block, &offset);
264
265 data = dm_bufio_read(v->bufio, hash_block, &buf);
266 if (IS_ERR(data))
267 return PTR_ERR(data);
268
269 aux = dm_bufio_get_aux_data(buf);
270
271 if (!aux->hash_verified) {
272 if (skip_unverified) {
273 r = 1;
274 goto release_ret_r;
275 }
276
277 r = verity_hash(v, verity_io_hash_desc(v, io),
278 data, 1 << v->hash_dev_block_bits,
279 verity_io_real_digest(v, io));
280 if (unlikely(r < 0))
281 goto release_ret_r;
282
283 if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
284 v->digest_size) == 0))
285 aux->hash_verified = 1;
286 else if (verity_handle_err(v,
287 DM_VERITY_BLOCK_TYPE_METADATA,
288 hash_block)) {
289 r = -EIO;
290 goto release_ret_r;
291 }
292 }
293
294 data += offset;
295 memcpy(want_digest, data, v->digest_size);
296 r = 0;
297
298release_ret_r:
299 dm_bufio_release(buf);
300 return r;
301}
302
303
304
305
306
307int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
308 sector_t block, u8 *digest)
309{
310 int i;
311 int r;
312
313 if (likely(v->levels)) {
314
315
316
317
318
319
320
321 r = verity_verify_level(v, io, block, 0, true, digest);
322 if (likely(r <= 0))
323 return r;
324 }
325
326 memcpy(digest, v->root_digest, v->digest_size);
327
328 for (i = v->levels - 1; i >= 0; i--) {
329 r = verity_verify_level(v, io, block, i, false, digest);
330 if (unlikely(r))
331 return r;
332 }
333
334 return 0;
335}
336
337
338
339
340static int verity_verify_io(struct dm_verity_io *io)
341{
342 struct dm_verity *v = io->v;
343 unsigned b;
344 unsigned vector = 0, offset = 0;
345
346 for (b = 0; b < io->n_blocks; b++) {
347 int r;
348 unsigned todo;
349 struct shash_desc *desc = verity_io_hash_desc(v, io);
350
351 r = verity_hash_for_block(v, io, io->block + b,
352 verity_io_want_digest(v, io));
353 if (unlikely(r < 0))
354 return r;
355
356 r = verity_hash_init(v, desc);
357 if (unlikely(r < 0))
358 return r;
359
360 todo = 1 << v->data_dev_block_bits;
361 do {
362 struct bio_vec *bv;
363 u8 *page;
364 unsigned len;
365
366 BUG_ON(vector >= io->io_vec_size);
367 bv = &io->io_vec[vector];
368 page = kmap_atomic(bv->bv_page);
369 len = bv->bv_len - offset;
370 if (likely(len >= todo))
371 len = todo;
372 r = verity_hash_update(v, desc,
373 page + bv->bv_offset + offset, len);
374 kunmap_atomic(page);
375
376 if (unlikely(r < 0))
377 return r;
378
379 offset += len;
380 if (likely(offset == bv->bv_len)) {
381 offset = 0;
382 vector++;
383 }
384 todo -= len;
385 } while (todo);
386
387 r = verity_hash_final(v, desc, verity_io_real_digest(v, io));
388 if (unlikely(r < 0))
389 return r;
390
391 if (likely(memcmp(verity_io_real_digest(v, io),
392 verity_io_want_digest(v, io), v->digest_size) == 0))
393 continue;
394 else if (verity_handle_err(v, DM_VERITY_BLOCK_TYPE_DATA,
395 io->block + b))
396 return -EIO;
397 }
398 BUG_ON(vector != io->io_vec_size);
399 BUG_ON(offset);
400
401 return 0;
402}
403
404
405
406
407static void verity_finish_io(struct dm_verity_io *io, int error)
408{
409 struct dm_verity *v = io->v;
410 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size);
411
412 bio->bi_end_io = io->orig_bi_end_io;
413
414 if (io->io_vec != io->io_vec_inline)
415 mempool_free(io->io_vec, v->vec_mempool);
416
417 bio_endio(bio, error);
418}
419
420static void verity_work(struct work_struct *w)
421{
422 struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
423
424 verity_finish_io(io, verity_verify_io(io));
425}
426
427static void verity_end_io(struct bio *bio, int error)
428{
429 struct dm_verity_io *io = bio->bi_private;
430
431 if (error) {
432 verity_finish_io(io, error);
433 return;
434 }
435
436 INIT_WORK(&io->work, verity_work);
437 queue_work(io->v->verify_wq, &io->work);
438}
439
440
441
442
443
444
445static void verity_prefetch_io(struct work_struct *work)
446{
447 struct dm_verity_prefetch_work *pw =
448 container_of(work, struct dm_verity_prefetch_work, work);
449 struct dm_verity *v = pw->v;
450 int i;
451
452 for (i = v->levels - 2; i >= 0; i--) {
453 sector_t hash_block_start;
454 sector_t hash_block_end;
455 verity_hash_at_level(v, pw->block, i, &hash_block_start, NULL);
456 verity_hash_at_level(v, pw->block + pw->n_blocks - 1, i, &hash_block_end, NULL);
457 if (!i) {
458 unsigned cluster = ACCESS_ONCE(dm_verity_prefetch_cluster);
459
460 cluster >>= v->data_dev_block_bits;
461 if (unlikely(!cluster))
462 goto no_prefetch_cluster;
463
464 if (unlikely(cluster & (cluster - 1)))
465 cluster = 1 << __fls(cluster);
466
467 hash_block_start &= ~(sector_t)(cluster - 1);
468 hash_block_end |= cluster - 1;
469 if (unlikely(hash_block_end >= v->hash_blocks))
470 hash_block_end = v->hash_blocks - 1;
471 }
472no_prefetch_cluster:
473 dm_bufio_prefetch(v->bufio, hash_block_start,
474 hash_block_end - hash_block_start + 1);
475 }
476
477 kfree(pw);
478}
479
480static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io)
481{
482 struct dm_verity_prefetch_work *pw;
483
484 pw = kmalloc(sizeof(struct dm_verity_prefetch_work),
485 GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
486
487 if (!pw)
488 return;
489
490 INIT_WORK(&pw->work, verity_prefetch_io);
491 pw->v = v;
492 pw->block = io->block;
493 pw->n_blocks = io->n_blocks;
494 queue_work(v->verify_wq, &pw->work);
495}
496
497
498
499
500
501static int verity_map(struct dm_target *ti, struct bio *bio)
502{
503 struct dm_verity *v = ti->private;
504 struct dm_verity_io *io;
505
506 bio->bi_bdev = v->data_dev->bdev;
507 bio->bi_sector = verity_map_sector(v, bio->bi_sector);
508
509 if (((unsigned)bio->bi_sector | bio_sectors(bio)) &
510 ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
511 DMERR_LIMIT("unaligned io");
512 return -EIO;
513 }
514
515 if (bio_end_sector(bio) >>
516 (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
517 DMERR_LIMIT("io out of range");
518 return -EIO;
519 }
520
521 if (bio_data_dir(bio) == WRITE)
522 return -EIO;
523
524 io = dm_per_bio_data(bio, ti->per_io_data_size);
525 io->v = v;
526 io->orig_bi_end_io = bio->bi_end_io;
527 io->block = bio->bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
528 io->n_blocks = bio->bi_size >> v->data_dev_block_bits;
529
530 bio->bi_end_io = verity_end_io;
531 bio->bi_private = io;
532 io->io_vec_size = bio_segments(bio);
533 if (io->io_vec_size < DM_VERITY_IO_VEC_INLINE)
534 io->io_vec = io->io_vec_inline;
535 else
536 io->io_vec = mempool_alloc(v->vec_mempool, GFP_NOIO);
537 memcpy(io->io_vec, bio_iovec(bio),
538 io->io_vec_size * sizeof(struct bio_vec));
539
540 verity_submit_prefetch(v, io);
541
542 generic_make_request(bio);
543
544 return DM_MAPIO_SUBMITTED;
545}
546
547
548
549
550static void verity_status(struct dm_target *ti, status_type_t type,
551 unsigned status_flags, char *result, unsigned maxlen)
552{
553 struct dm_verity *v = ti->private;
554 unsigned sz = 0;
555 unsigned x;
556
557 switch (type) {
558 case STATUSTYPE_INFO:
559 DMEMIT("%c", v->hash_failed ? 'C' : 'V');
560 break;
561 case STATUSTYPE_TABLE:
562 DMEMIT("%u %s %s %u %u %llu %llu %s ",
563 v->version,
564 v->data_dev->name,
565 v->hash_dev->name,
566 1 << v->data_dev_block_bits,
567 1 << v->hash_dev_block_bits,
568 (unsigned long long)v->data_blocks,
569 (unsigned long long)v->hash_start,
570 v->alg_name
571 );
572 for (x = 0; x < v->digest_size; x++)
573 DMEMIT("%02x", v->root_digest[x]);
574 DMEMIT(" ");
575 if (!v->salt_size)
576 DMEMIT("-");
577 else
578 for (x = 0; x < v->salt_size; x++)
579 DMEMIT("%02x", v->salt[x]);
580 if (v->mode != DM_VERITY_MODE_EIO) {
581 DMEMIT(" 1 ");
582 switch (v->mode) {
583 case DM_VERITY_MODE_LOGGING:
584 DMEMIT(DM_VERITY_OPT_LOGGING);
585 break;
586 case DM_VERITY_MODE_RESTART:
587 DMEMIT(DM_VERITY_OPT_RESTART);
588 break;
589 default:
590 BUG();
591 }
592 }
593 break;
594 }
595}
596
597static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
598{
599 struct dm_verity *v = ti->private;
600
601 *bdev = v->data_dev->bdev;
602
603 if (v->data_start ||
604 ti->len != i_size_read(v->data_dev->bdev->bd_inode) >> SECTOR_SHIFT)
605 return 1;
606 return 0;
607}
608
609static int verity_merge(struct dm_target *ti, struct bvec_merge_data *bvm,
610 struct bio_vec *biovec, int max_size)
611{
612 struct dm_verity *v = ti->private;
613 struct request_queue *q = bdev_get_queue(v->data_dev->bdev);
614
615 if (!q->merge_bvec_fn)
616 return max_size;
617
618 bvm->bi_bdev = v->data_dev->bdev;
619 bvm->bi_sector = verity_map_sector(v, bvm->bi_sector);
620
621 return min(max_size, q->merge_bvec_fn(q, bvm, biovec));
622}
623
624static int verity_iterate_devices(struct dm_target *ti,
625 iterate_devices_callout_fn fn, void *data)
626{
627 struct dm_verity *v = ti->private;
628
629 return fn(ti, v->data_dev, v->data_start, ti->len, data);
630}
631
632static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
633{
634 struct dm_verity *v = ti->private;
635
636 if (limits->logical_block_size < 1 << v->data_dev_block_bits)
637 limits->logical_block_size = 1 << v->data_dev_block_bits;
638
639 if (limits->physical_block_size < 1 << v->data_dev_block_bits)
640 limits->physical_block_size = 1 << v->data_dev_block_bits;
641
642 blk_limits_io_min(limits, limits->logical_block_size);
643}
644
645static void verity_dtr(struct dm_target *ti)
646{
647 struct dm_verity *v = ti->private;
648
649 if (v->verify_wq)
650 destroy_workqueue(v->verify_wq);
651
652 if (v->vec_mempool)
653 mempool_destroy(v->vec_mempool);
654
655 if (v->bufio)
656 dm_bufio_client_destroy(v->bufio);
657
658 kfree(v->salt);
659 kfree(v->root_digest);
660
661 if (v->tfm)
662 crypto_free_shash(v->tfm);
663
664 kfree(v->alg_name);
665
666 if (v->hash_dev)
667 dm_put_device(ti, v->hash_dev);
668
669 if (v->data_dev)
670 dm_put_device(ti, v->data_dev);
671
672 kfree(v);
673}
674
675static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v)
676{
677 int r;
678 unsigned argc;
679 struct dm_target *ti = v->ti;
680 const char *arg_name;
681
682 static const struct dm_arg _args[] = {
683 {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
684 };
685
686 r = dm_read_arg_group(_args, as, &argc, &ti->error);
687 if (r)
688 return -EINVAL;
689
690 if (!argc)
691 return 0;
692
693 do {
694 arg_name = dm_shift_arg(as);
695 argc--;
696
697 if (!strcasecmp(arg_name, DM_VERITY_OPT_LOGGING)) {
698 v->mode = DM_VERITY_MODE_LOGGING;
699 continue;
700
701 } else if (!strcasecmp(arg_name, DM_VERITY_OPT_RESTART)) {
702 v->mode = DM_VERITY_MODE_RESTART;
703 continue;
704 }
705
706 ti->error = "Unrecognized verity feature request";
707 return -EINVAL;
708 } while (argc && !r);
709
710 return r;
711}
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv)
728{
729 struct dm_verity *v;
730 struct dm_arg_set as;
731 unsigned int num;
732 unsigned long long num_ll;
733 int r;
734 int i;
735 sector_t hash_position;
736 char dummy;
737
738 v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
739 if (!v) {
740 ti->error = "Cannot allocate verity structure";
741 return -ENOMEM;
742 }
743 ti->private = v;
744 v->ti = ti;
745
746 if ((dm_table_get_mode(ti->table) & ~FMODE_READ)) {
747 ti->error = "Device must be readonly";
748 r = -EINVAL;
749 goto bad;
750 }
751
752 if (argc < 10) {
753 ti->error = "Not enough arguments";
754 r = -EINVAL;
755 goto bad;
756 }
757
758 if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 ||
759 num > 1) {
760 ti->error = "Invalid version";
761 r = -EINVAL;
762 goto bad;
763 }
764 v->version = num;
765
766 r = dm_get_device(ti, argv[1], FMODE_READ, &v->data_dev);
767 if (r) {
768 ti->error = "Data device lookup failed";
769 goto bad;
770 }
771
772 r = dm_get_device(ti, argv[2], FMODE_READ, &v->hash_dev);
773 if (r) {
774 ti->error = "Hash device lookup failed";
775 goto bad;
776 }
777
778 if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||
779 !num || (num & (num - 1)) ||
780 num < bdev_logical_block_size(v->data_dev->bdev) ||
781 num > PAGE_SIZE) {
782 ti->error = "Invalid data device block size";
783 r = -EINVAL;
784 goto bad;
785 }
786 v->data_dev_block_bits = __ffs(num);
787
788 if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
789 !num || (num & (num - 1)) ||
790 num < bdev_logical_block_size(v->hash_dev->bdev) ||
791 num > INT_MAX) {
792 ti->error = "Invalid hash device block size";
793 r = -EINVAL;
794 goto bad;
795 }
796 v->hash_dev_block_bits = __ffs(num);
797
798 if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||
799 (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
800 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {
801 ti->error = "Invalid data blocks";
802 r = -EINVAL;
803 goto bad;
804 }
805 v->data_blocks = num_ll;
806
807 if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {
808 ti->error = "Data device is too small";
809 r = -EINVAL;
810 goto bad;
811 }
812
813 if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||
814 (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))
815 >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {
816 ti->error = "Invalid hash start";
817 r = -EINVAL;
818 goto bad;
819 }
820 v->hash_start = num_ll;
821
822 v->alg_name = kstrdup(argv[7], GFP_KERNEL);
823 if (!v->alg_name) {
824 ti->error = "Cannot allocate algorithm name";
825 r = -ENOMEM;
826 goto bad;
827 }
828
829 v->tfm = crypto_alloc_shash(v->alg_name, 0, 0);
830 if (IS_ERR(v->tfm)) {
831 ti->error = "Cannot initialize hash function";
832 r = PTR_ERR(v->tfm);
833 v->tfm = NULL;
834 goto bad;
835 }
836 v->digest_size = crypto_shash_digestsize(v->tfm);
837 if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
838 ti->error = "Digest size too big";
839 r = -EINVAL;
840 goto bad;
841 }
842 v->shash_descsize =
843 sizeof(struct shash_desc) + crypto_shash_descsize(v->tfm);
844
845 v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
846 if (!v->root_digest) {
847 ti->error = "Cannot allocate root digest";
848 r = -ENOMEM;
849 goto bad;
850 }
851 if (strlen(argv[8]) != v->digest_size * 2 ||
852 hex2bin(v->root_digest, argv[8], v->digest_size)) {
853 ti->error = "Invalid root digest";
854 r = -EINVAL;
855 goto bad;
856 }
857
858 if (strcmp(argv[9], "-")) {
859 v->salt_size = strlen(argv[9]) / 2;
860 v->salt = kmalloc(v->salt_size, GFP_KERNEL);
861 if (!v->salt) {
862 ti->error = "Cannot allocate salt";
863 r = -ENOMEM;
864 goto bad;
865 }
866 if (strlen(argv[9]) != v->salt_size * 2 ||
867 hex2bin(v->salt, argv[9], v->salt_size)) {
868 ti->error = "Invalid salt";
869 r = -EINVAL;
870 goto bad;
871 }
872 }
873
874 argv += 10;
875 argc -= 10;
876
877
878 if (argc) {
879 as.argc = argc;
880 as.argv = argv;
881
882 r = verity_parse_opt_args(&as, v);
883 if (r < 0)
884 goto bad;
885 }
886
887 v->hash_per_block_bits =
888 __fls((1 << v->hash_dev_block_bits) / v->digest_size);
889
890 v->levels = 0;
891 if (v->data_blocks)
892 while (v->hash_per_block_bits * v->levels < 64 &&
893 (unsigned long long)(v->data_blocks - 1) >>
894 (v->hash_per_block_bits * v->levels))
895 v->levels++;
896
897 if (v->levels > DM_VERITY_MAX_LEVELS) {
898 ti->error = "Too many tree levels";
899 r = -E2BIG;
900 goto bad;
901 }
902
903 hash_position = v->hash_start;
904 for (i = v->levels - 1; i >= 0; i--) {
905 sector_t s;
906 v->hash_level_block[i] = hash_position;
907 s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1)
908 >> ((i + 1) * v->hash_per_block_bits);
909 if (hash_position + s < hash_position) {
910 ti->error = "Hash device offset overflow";
911 r = -E2BIG;
912 goto bad;
913 }
914 hash_position += s;
915 }
916 v->hash_blocks = hash_position;
917
918 v->bufio = dm_bufio_client_create(v->hash_dev->bdev,
919 1 << v->hash_dev_block_bits, 1, sizeof(struct buffer_aux),
920 dm_bufio_alloc_callback, NULL);
921 if (IS_ERR(v->bufio)) {
922 ti->error = "Cannot initialize dm-bufio";
923 r = PTR_ERR(v->bufio);
924 v->bufio = NULL;
925 goto bad;
926 }
927
928 if (dm_bufio_get_device_size(v->bufio) < v->hash_blocks) {
929 ti->error = "Hash device is too small";
930 r = -E2BIG;
931 goto bad;
932 }
933
934 ti->per_io_data_size = roundup(sizeof(struct dm_verity_io) + v->shash_descsize + v->digest_size * 2, __alignof__(struct dm_verity_io));
935
936 v->vec_mempool = mempool_create_kmalloc_pool(DM_VERITY_MEMPOOL_SIZE,
937 BIO_MAX_PAGES * sizeof(struct bio_vec));
938 if (!v->vec_mempool) {
939 ti->error = "Cannot allocate vector mempool";
940 r = -ENOMEM;
941 goto bad;
942 }
943
944
945 v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus());
946 if (!v->verify_wq) {
947 ti->error = "Cannot allocate workqueue";
948 r = -ENOMEM;
949 goto bad;
950 }
951
952 return 0;
953
954bad:
955 verity_dtr(ti);
956
957 return r;
958}
959
960static struct target_type verity_target = {
961 .name = "verity",
962 .version = {1, 2, 0},
963 .module = THIS_MODULE,
964 .ctr = verity_ctr,
965 .dtr = verity_dtr,
966 .map = verity_map,
967 .status = verity_status,
968 .prepare_ioctl = verity_prepare_ioctl,
969 .merge = verity_merge,
970 .iterate_devices = verity_iterate_devices,
971 .io_hints = verity_io_hints,
972};
973
974static int __init dm_verity_init(void)
975{
976 int r;
977
978 r = dm_register_target(&verity_target);
979 if (r < 0)
980 DMERR("register failed %d", r);
981
982 return r;
983}
984
985static void __exit dm_verity_exit(void)
986{
987 dm_unregister_target(&verity_target);
988}
989
990module_init(dm_verity_init);
991module_exit(dm_verity_exit);
992
993MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
994MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
995MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
996MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
997MODULE_LICENSE("GPL");
998