1
2
3
4
5
6#include <linux/bio.h>
7#include <linux/slab.h>
8#include <linux/pagemap.h>
9#include <linux/highmem.h>
10#include <linux/sched/mm.h>
11#include <crypto/hash.h>
12#include "misc.h"
13#include "ctree.h"
14#include "disk-io.h"
15#include "transaction.h"
16#include "volumes.h"
17#include "print-tree.h"
18#include "compression.h"
19
20#define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \
21 sizeof(struct btrfs_item) * 2) / \
22 size) - 1))
23
24#define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \
25 PAGE_SIZE))
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44void btrfs_inode_safe_disk_i_size_write(struct btrfs_inode *inode, u64 new_i_size)
45{
46 struct btrfs_fs_info *fs_info = inode->root->fs_info;
47 u64 start, end, i_size;
48 int ret;
49
50 i_size = new_i_size ?: i_size_read(&inode->vfs_inode);
51 if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
52 inode->disk_i_size = i_size;
53 return;
54 }
55
56 spin_lock(&inode->lock);
57 ret = find_contiguous_extent_bit(&inode->file_extent_tree, 0, &start,
58 &end, EXTENT_DIRTY);
59 if (!ret && start == 0)
60 i_size = min(i_size, end + 1);
61 else
62 i_size = 0;
63 inode->disk_i_size = i_size;
64 spin_unlock(&inode->lock);
65}
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start,
82 u64 len)
83{
84 if (len == 0)
85 return 0;
86
87 ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize));
88
89 if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES))
90 return 0;
91 return set_extent_bits(&inode->file_extent_tree, start, start + len - 1,
92 EXTENT_DIRTY);
93}
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start,
110 u64 len)
111{
112 if (len == 0)
113 return 0;
114
115 ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize) ||
116 len == (u64)-1);
117
118 if (btrfs_fs_incompat(inode->root->fs_info, NO_HOLES))
119 return 0;
120 return clear_extent_bit(&inode->file_extent_tree, start,
121 start + len - 1, EXTENT_DIRTY, 0, 0, NULL);
122}
123
124static inline u32 max_ordered_sum_bytes(struct btrfs_fs_info *fs_info,
125 u16 csum_size)
126{
127 u32 ncsums = (PAGE_SIZE - sizeof(struct btrfs_ordered_sum)) / csum_size;
128
129 return ncsums * fs_info->sectorsize;
130}
131
132int btrfs_insert_file_extent(struct btrfs_trans_handle *trans,
133 struct btrfs_root *root,
134 u64 objectid, u64 pos,
135 u64 disk_offset, u64 disk_num_bytes,
136 u64 num_bytes, u64 offset, u64 ram_bytes,
137 u8 compression, u8 encryption, u16 other_encoding)
138{
139 int ret = 0;
140 struct btrfs_file_extent_item *item;
141 struct btrfs_key file_key;
142 struct btrfs_path *path;
143 struct extent_buffer *leaf;
144
145 path = btrfs_alloc_path();
146 if (!path)
147 return -ENOMEM;
148 file_key.objectid = objectid;
149 file_key.offset = pos;
150 file_key.type = BTRFS_EXTENT_DATA_KEY;
151
152 ret = btrfs_insert_empty_item(trans, root, path, &file_key,
153 sizeof(*item));
154 if (ret < 0)
155 goto out;
156 BUG_ON(ret);
157 leaf = path->nodes[0];
158 item = btrfs_item_ptr(leaf, path->slots[0],
159 struct btrfs_file_extent_item);
160 btrfs_set_file_extent_disk_bytenr(leaf, item, disk_offset);
161 btrfs_set_file_extent_disk_num_bytes(leaf, item, disk_num_bytes);
162 btrfs_set_file_extent_offset(leaf, item, offset);
163 btrfs_set_file_extent_num_bytes(leaf, item, num_bytes);
164 btrfs_set_file_extent_ram_bytes(leaf, item, ram_bytes);
165 btrfs_set_file_extent_generation(leaf, item, trans->transid);
166 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
167 btrfs_set_file_extent_compression(leaf, item, compression);
168 btrfs_set_file_extent_encryption(leaf, item, encryption);
169 btrfs_set_file_extent_other_encoding(leaf, item, other_encoding);
170
171 btrfs_mark_buffer_dirty(leaf);
172out:
173 btrfs_free_path(path);
174 return ret;
175}
176
177static struct btrfs_csum_item *
178btrfs_lookup_csum(struct btrfs_trans_handle *trans,
179 struct btrfs_root *root,
180 struct btrfs_path *path,
181 u64 bytenr, int cow)
182{
183 struct btrfs_fs_info *fs_info = root->fs_info;
184 int ret;
185 struct btrfs_key file_key;
186 struct btrfs_key found_key;
187 struct btrfs_csum_item *item;
188 struct extent_buffer *leaf;
189 u64 csum_offset = 0;
190 const u32 csum_size = fs_info->csum_size;
191 int csums_in_item;
192
193 file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
194 file_key.offset = bytenr;
195 file_key.type = BTRFS_EXTENT_CSUM_KEY;
196 ret = btrfs_search_slot(trans, root, &file_key, path, 0, cow);
197 if (ret < 0)
198 goto fail;
199 leaf = path->nodes[0];
200 if (ret > 0) {
201 ret = 1;
202 if (path->slots[0] == 0)
203 goto fail;
204 path->slots[0]--;
205 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
206 if (found_key.type != BTRFS_EXTENT_CSUM_KEY)
207 goto fail;
208
209 csum_offset = (bytenr - found_key.offset) >>
210 fs_info->sectorsize_bits;
211 csums_in_item = btrfs_item_size_nr(leaf, path->slots[0]);
212 csums_in_item /= csum_size;
213
214 if (csum_offset == csums_in_item) {
215 ret = -EFBIG;
216 goto fail;
217 } else if (csum_offset > csums_in_item) {
218 goto fail;
219 }
220 }
221 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
222 item = (struct btrfs_csum_item *)((unsigned char *)item +
223 csum_offset * csum_size);
224 return item;
225fail:
226 if (ret > 0)
227 ret = -ENOENT;
228 return ERR_PTR(ret);
229}
230
231int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans,
232 struct btrfs_root *root,
233 struct btrfs_path *path, u64 objectid,
234 u64 offset, int mod)
235{
236 int ret;
237 struct btrfs_key file_key;
238 int ins_len = mod < 0 ? -1 : 0;
239 int cow = mod != 0;
240
241 file_key.objectid = objectid;
242 file_key.offset = offset;
243 file_key.type = BTRFS_EXTENT_DATA_KEY;
244 ret = btrfs_search_slot(trans, root, &file_key, path, ins_len, cow);
245 return ret;
246}
247
248
249
250
251
252
253
254
255
256
257static int search_csum_tree(struct btrfs_fs_info *fs_info,
258 struct btrfs_path *path, u64 disk_bytenr,
259 u64 len, u8 *dst)
260{
261 struct btrfs_csum_item *item = NULL;
262 struct btrfs_key key;
263 const u32 sectorsize = fs_info->sectorsize;
264 const u32 csum_size = fs_info->csum_size;
265 u32 itemsize;
266 int ret;
267 u64 csum_start;
268 u64 csum_len;
269
270 ASSERT(IS_ALIGNED(disk_bytenr, sectorsize) &&
271 IS_ALIGNED(len, sectorsize));
272
273
274 if (path->nodes[0]) {
275 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
276 struct btrfs_csum_item);
277 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
278 itemsize = btrfs_item_size_nr(path->nodes[0], path->slots[0]);
279
280 csum_start = key.offset;
281 csum_len = (itemsize / csum_size) * sectorsize;
282
283 if (in_range(disk_bytenr, csum_start, csum_len))
284 goto found;
285 }
286
287
288 btrfs_release_path(path);
289 item = btrfs_lookup_csum(NULL, fs_info->csum_root, path, disk_bytenr, 0);
290 if (IS_ERR(item)) {
291 ret = PTR_ERR(item);
292 goto out;
293 }
294 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
295 itemsize = btrfs_item_size_nr(path->nodes[0], path->slots[0]);
296
297 csum_start = key.offset;
298 csum_len = (itemsize / csum_size) * sectorsize;
299 ASSERT(in_range(disk_bytenr, csum_start, csum_len));
300
301found:
302 ret = (min(csum_start + csum_len, disk_bytenr + len) -
303 disk_bytenr) >> fs_info->sectorsize_bits;
304 read_extent_buffer(path->nodes[0], dst, (unsigned long)item,
305 ret * csum_size);
306out:
307 if (ret == -ENOENT)
308 ret = 0;
309 return ret;
310}
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325static int search_file_offset_in_bio(struct bio *bio, struct inode *inode,
326 u64 disk_bytenr, u64 *file_offset_ret)
327{
328 struct bvec_iter iter;
329 struct bio_vec bvec;
330 u64 cur = bio->bi_iter.bi_sector << SECTOR_SHIFT;
331 int ret = 0;
332
333 bio_for_each_segment(bvec, bio, iter) {
334 struct page *page = bvec.bv_page;
335
336 if (cur > disk_bytenr)
337 break;
338 if (cur + bvec.bv_len <= disk_bytenr) {
339 cur += bvec.bv_len;
340 continue;
341 }
342 ASSERT(in_range(disk_bytenr, cur, bvec.bv_len));
343 if (page->mapping && page->mapping->host &&
344 page->mapping->host == inode) {
345 ret = 1;
346 *file_offset_ret = page_offset(page) + bvec.bv_offset +
347 disk_bytenr - cur;
348 break;
349 }
350 }
351 return ret;
352}
353
354
355
356
357
358
359
360
361
362
363
364
365
366blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, u8 *dst)
367{
368 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
369 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
370 struct btrfs_path *path;
371 const u32 sectorsize = fs_info->sectorsize;
372 const u32 csum_size = fs_info->csum_size;
373 u32 orig_len = bio->bi_iter.bi_size;
374 u64 orig_disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT;
375 u64 cur_disk_bytenr;
376 u8 *csum;
377 const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits;
378 int count = 0;
379
380 if (!fs_info->csum_root || (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM))
381 return BLK_STS_OK;
382
383
384
385
386
387
388
389
390
391
392
393
394
395 ASSERT(bio_op(bio) == REQ_OP_READ);
396 path = btrfs_alloc_path();
397 if (!path)
398 return BLK_STS_RESOURCE;
399
400 if (!dst) {
401 struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio);
402
403 if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) {
404 btrfs_bio->csum = kmalloc_array(nblocks, csum_size,
405 GFP_NOFS);
406 if (!btrfs_bio->csum) {
407 btrfs_free_path(path);
408 return BLK_STS_RESOURCE;
409 }
410 } else {
411 btrfs_bio->csum = btrfs_bio->csum_inline;
412 }
413 csum = btrfs_bio->csum;
414 } else {
415 csum = dst;
416 }
417
418
419
420
421
422 if (nblocks > fs_info->csums_per_leaf)
423 path->reada = READA_FORWARD;
424
425
426
427
428
429
430
431 if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
432 path->search_commit_root = 1;
433 path->skip_locking = 1;
434 }
435
436 for (cur_disk_bytenr = orig_disk_bytenr;
437 cur_disk_bytenr < orig_disk_bytenr + orig_len;
438 cur_disk_bytenr += (count * sectorsize)) {
439 u64 search_len = orig_disk_bytenr + orig_len - cur_disk_bytenr;
440 unsigned int sector_offset;
441 u8 *csum_dst;
442
443
444
445
446
447
448
449
450
451 ASSERT(cur_disk_bytenr - orig_disk_bytenr < UINT_MAX);
452 sector_offset = (cur_disk_bytenr - orig_disk_bytenr) >>
453 fs_info->sectorsize_bits;
454 csum_dst = csum + sector_offset * csum_size;
455
456 count = search_csum_tree(fs_info, path, cur_disk_bytenr,
457 search_len, csum_dst);
458 if (count <= 0) {
459
460
461
462
463
464
465 memset(csum_dst, 0, csum_size);
466 count = 1;
467
468
469
470
471
472
473 if (BTRFS_I(inode)->root->root_key.objectid ==
474 BTRFS_DATA_RELOC_TREE_OBJECTID) {
475 u64 file_offset;
476 int ret;
477
478 ret = search_file_offset_in_bio(bio, inode,
479 cur_disk_bytenr, &file_offset);
480 if (ret)
481 set_extent_bits(io_tree, file_offset,
482 file_offset + sectorsize - 1,
483 EXTENT_NODATASUM);
484 } else {
485 btrfs_warn_rl(fs_info,
486 "csum hole found for disk bytenr range [%llu, %llu)",
487 cur_disk_bytenr, cur_disk_bytenr + sectorsize);
488 }
489 }
490 }
491
492 btrfs_free_path(path);
493 return BLK_STS_OK;
494}
495
496int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end,
497 struct list_head *list, int search_commit)
498{
499 struct btrfs_fs_info *fs_info = root->fs_info;
500 struct btrfs_key key;
501 struct btrfs_path *path;
502 struct extent_buffer *leaf;
503 struct btrfs_ordered_sum *sums;
504 struct btrfs_csum_item *item;
505 LIST_HEAD(tmplist);
506 unsigned long offset;
507 int ret;
508 size_t size;
509 u64 csum_end;
510 const u32 csum_size = fs_info->csum_size;
511
512 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
513 IS_ALIGNED(end + 1, fs_info->sectorsize));
514
515 path = btrfs_alloc_path();
516 if (!path)
517 return -ENOMEM;
518
519 if (search_commit) {
520 path->skip_locking = 1;
521 path->reada = READA_FORWARD;
522 path->search_commit_root = 1;
523 }
524
525 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
526 key.offset = start;
527 key.type = BTRFS_EXTENT_CSUM_KEY;
528
529 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
530 if (ret < 0)
531 goto fail;
532 if (ret > 0 && path->slots[0] > 0) {
533 leaf = path->nodes[0];
534 btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
535 if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID &&
536 key.type == BTRFS_EXTENT_CSUM_KEY) {
537 offset = (start - key.offset) >> fs_info->sectorsize_bits;
538 if (offset * csum_size <
539 btrfs_item_size_nr(leaf, path->slots[0] - 1))
540 path->slots[0]--;
541 }
542 }
543
544 while (start <= end) {
545 leaf = path->nodes[0];
546 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
547 ret = btrfs_next_leaf(root, path);
548 if (ret < 0)
549 goto fail;
550 if (ret > 0)
551 break;
552 leaf = path->nodes[0];
553 }
554
555 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
556 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
557 key.type != BTRFS_EXTENT_CSUM_KEY ||
558 key.offset > end)
559 break;
560
561 if (key.offset > start)
562 start = key.offset;
563
564 size = btrfs_item_size_nr(leaf, path->slots[0]);
565 csum_end = key.offset + (size / csum_size) * fs_info->sectorsize;
566 if (csum_end <= start) {
567 path->slots[0]++;
568 continue;
569 }
570
571 csum_end = min(csum_end, end + 1);
572 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
573 struct btrfs_csum_item);
574 while (start < csum_end) {
575 size = min_t(size_t, csum_end - start,
576 max_ordered_sum_bytes(fs_info, csum_size));
577 sums = kzalloc(btrfs_ordered_sum_size(fs_info, size),
578 GFP_NOFS);
579 if (!sums) {
580 ret = -ENOMEM;
581 goto fail;
582 }
583
584 sums->bytenr = start;
585 sums->len = (int)size;
586
587 offset = (start - key.offset) >> fs_info->sectorsize_bits;
588 offset *= csum_size;
589 size >>= fs_info->sectorsize_bits;
590
591 read_extent_buffer(path->nodes[0],
592 sums->sums,
593 ((unsigned long)item) + offset,
594 csum_size * size);
595
596 start += fs_info->sectorsize * size;
597 list_add_tail(&sums->list, &tmplist);
598 }
599 path->slots[0]++;
600 }
601 ret = 0;
602fail:
603 while (ret < 0 && !list_empty(&tmplist)) {
604 sums = list_entry(tmplist.next, struct btrfs_ordered_sum, list);
605 list_del(&sums->list);
606 kfree(sums);
607 }
608 list_splice_tail(&tmplist, list);
609
610 btrfs_free_path(path);
611 return ret;
612}
613
614
615
616
617
618
619
620
621
622
623
624blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio,
625 u64 file_start, int contig)
626{
627 struct btrfs_fs_info *fs_info = inode->root->fs_info;
628 SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
629 struct btrfs_ordered_sum *sums;
630 struct btrfs_ordered_extent *ordered = NULL;
631 char *data;
632 struct bvec_iter iter;
633 struct bio_vec bvec;
634 int index;
635 int nr_sectors;
636 unsigned long total_bytes = 0;
637 unsigned long this_sum_bytes = 0;
638 int i;
639 u64 offset;
640 unsigned nofs_flag;
641
642 nofs_flag = memalloc_nofs_save();
643 sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size),
644 GFP_KERNEL);
645 memalloc_nofs_restore(nofs_flag);
646
647 if (!sums)
648 return BLK_STS_RESOURCE;
649
650 sums->len = bio->bi_iter.bi_size;
651 INIT_LIST_HEAD(&sums->list);
652
653 if (contig)
654 offset = file_start;
655 else
656 offset = 0;
657
658 sums->bytenr = bio->bi_iter.bi_sector << 9;
659 index = 0;
660
661 shash->tfm = fs_info->csum_shash;
662
663 bio_for_each_segment(bvec, bio, iter) {
664 if (!contig)
665 offset = page_offset(bvec.bv_page) + bvec.bv_offset;
666
667 if (!ordered) {
668 ordered = btrfs_lookup_ordered_extent(inode, offset);
669 BUG_ON(!ordered);
670 }
671
672 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info,
673 bvec.bv_len + fs_info->sectorsize
674 - 1);
675
676 for (i = 0; i < nr_sectors; i++) {
677 if (offset >= ordered->file_offset + ordered->num_bytes ||
678 offset < ordered->file_offset) {
679 unsigned long bytes_left;
680
681 sums->len = this_sum_bytes;
682 this_sum_bytes = 0;
683 btrfs_add_ordered_sum(ordered, sums);
684 btrfs_put_ordered_extent(ordered);
685
686 bytes_left = bio->bi_iter.bi_size - total_bytes;
687
688 nofs_flag = memalloc_nofs_save();
689 sums = kvzalloc(btrfs_ordered_sum_size(fs_info,
690 bytes_left), GFP_KERNEL);
691 memalloc_nofs_restore(nofs_flag);
692 BUG_ON(!sums);
693 sums->len = bytes_left;
694 ordered = btrfs_lookup_ordered_extent(inode,
695 offset);
696 ASSERT(ordered);
697 sums->bytenr = (bio->bi_iter.bi_sector << 9)
698 + total_bytes;
699 index = 0;
700 }
701
702 data = kmap_atomic(bvec.bv_page);
703 crypto_shash_digest(shash, data + bvec.bv_offset
704 + (i * fs_info->sectorsize),
705 fs_info->sectorsize,
706 sums->sums + index);
707 kunmap_atomic(data);
708 index += fs_info->csum_size;
709 offset += fs_info->sectorsize;
710 this_sum_bytes += fs_info->sectorsize;
711 total_bytes += fs_info->sectorsize;
712 }
713
714 }
715 this_sum_bytes = 0;
716 btrfs_add_ordered_sum(ordered, sums);
717 btrfs_put_ordered_extent(ordered);
718 return 0;
719}
720
721
722
723
724
725
726
727
728
729
730
731
732static noinline void truncate_one_csum(struct btrfs_fs_info *fs_info,
733 struct btrfs_path *path,
734 struct btrfs_key *key,
735 u64 bytenr, u64 len)
736{
737 struct extent_buffer *leaf;
738 const u32 csum_size = fs_info->csum_size;
739 u64 csum_end;
740 u64 end_byte = bytenr + len;
741 u32 blocksize_bits = fs_info->sectorsize_bits;
742
743 leaf = path->nodes[0];
744 csum_end = btrfs_item_size_nr(leaf, path->slots[0]) / csum_size;
745 csum_end <<= blocksize_bits;
746 csum_end += key->offset;
747
748 if (key->offset < bytenr && csum_end <= end_byte) {
749
750
751
752
753
754
755 u32 new_size = (bytenr - key->offset) >> blocksize_bits;
756 new_size *= csum_size;
757 btrfs_truncate_item(path, new_size, 1);
758 } else if (key->offset >= bytenr && csum_end > end_byte &&
759 end_byte > key->offset) {
760
761
762
763
764
765
766 u32 new_size = (csum_end - end_byte) >> blocksize_bits;
767 new_size *= csum_size;
768
769 btrfs_truncate_item(path, new_size, 0);
770
771 key->offset = end_byte;
772 btrfs_set_item_key_safe(fs_info, path, key);
773 } else {
774 BUG();
775 }
776}
777
778
779
780
781
782int btrfs_del_csums(struct btrfs_trans_handle *trans,
783 struct btrfs_root *root, u64 bytenr, u64 len)
784{
785 struct btrfs_fs_info *fs_info = trans->fs_info;
786 struct btrfs_path *path;
787 struct btrfs_key key;
788 u64 end_byte = bytenr + len;
789 u64 csum_end;
790 struct extent_buffer *leaf;
791 int ret = 0;
792 const u32 csum_size = fs_info->csum_size;
793 u32 blocksize_bits = fs_info->sectorsize_bits;
794
795 ASSERT(root == fs_info->csum_root ||
796 root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID);
797
798 path = btrfs_alloc_path();
799 if (!path)
800 return -ENOMEM;
801
802 while (1) {
803 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
804 key.offset = end_byte - 1;
805 key.type = BTRFS_EXTENT_CSUM_KEY;
806
807 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
808 if (ret > 0) {
809 ret = 0;
810 if (path->slots[0] == 0)
811 break;
812 path->slots[0]--;
813 } else if (ret < 0) {
814 break;
815 }
816
817 leaf = path->nodes[0];
818 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
819
820 if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
821 key.type != BTRFS_EXTENT_CSUM_KEY) {
822 break;
823 }
824
825 if (key.offset >= end_byte)
826 break;
827
828 csum_end = btrfs_item_size_nr(leaf, path->slots[0]) / csum_size;
829 csum_end <<= blocksize_bits;
830 csum_end += key.offset;
831
832
833 if (csum_end <= bytenr)
834 break;
835
836
837 if (key.offset >= bytenr && csum_end <= end_byte) {
838 int del_nr = 1;
839
840
841
842
843
844
845 if (key.offset > bytenr && path->slots[0] > 0) {
846 int slot = path->slots[0] - 1;
847
848 while (slot >= 0) {
849 struct btrfs_key pk;
850
851 btrfs_item_key_to_cpu(leaf, &pk, slot);
852 if (pk.offset < bytenr ||
853 pk.type != BTRFS_EXTENT_CSUM_KEY ||
854 pk.objectid !=
855 BTRFS_EXTENT_CSUM_OBJECTID)
856 break;
857 path->slots[0] = slot;
858 del_nr++;
859 key.offset = pk.offset;
860 slot--;
861 }
862 }
863 ret = btrfs_del_items(trans, root, path,
864 path->slots[0], del_nr);
865 if (ret)
866 break;
867 if (key.offset == bytenr)
868 break;
869 } else if (key.offset < bytenr && csum_end > end_byte) {
870 unsigned long offset;
871 unsigned long shift_len;
872 unsigned long item_offset;
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891 offset = (bytenr - key.offset) >> blocksize_bits;
892 offset *= csum_size;
893
894 shift_len = (len >> blocksize_bits) * csum_size;
895
896 item_offset = btrfs_item_ptr_offset(leaf,
897 path->slots[0]);
898
899 memzero_extent_buffer(leaf, item_offset + offset,
900 shift_len);
901 key.offset = bytenr;
902
903
904
905
906
907 ret = btrfs_split_item(trans, root, path, &key, offset);
908 if (ret && ret != -EAGAIN) {
909 btrfs_abort_transaction(trans, ret);
910 break;
911 }
912 ret = 0;
913
914 key.offset = end_byte - 1;
915 } else {
916 truncate_one_csum(fs_info, path, &key, bytenr, len);
917 if (key.offset < bytenr)
918 break;
919 }
920 btrfs_release_path(path);
921 }
922 btrfs_free_path(path);
923 return ret;
924}
925
926static int find_next_csum_offset(struct btrfs_root *root,
927 struct btrfs_path *path,
928 u64 *next_offset)
929{
930 const u32 nritems = btrfs_header_nritems(path->nodes[0]);
931 struct btrfs_key found_key;
932 int slot = path->slots[0] + 1;
933 int ret;
934
935 if (nritems == 0 || slot >= nritems) {
936 ret = btrfs_next_leaf(root, path);
937 if (ret < 0) {
938 return ret;
939 } else if (ret > 0) {
940 *next_offset = (u64)-1;
941 return 0;
942 }
943 slot = path->slots[0];
944 }
945
946 btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot);
947
948 if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
949 found_key.type != BTRFS_EXTENT_CSUM_KEY)
950 *next_offset = (u64)-1;
951 else
952 *next_offset = found_key.offset;
953
954 return 0;
955}
956
957int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans,
958 struct btrfs_root *root,
959 struct btrfs_ordered_sum *sums)
960{
961 struct btrfs_fs_info *fs_info = root->fs_info;
962 struct btrfs_key file_key;
963 struct btrfs_key found_key;
964 struct btrfs_path *path;
965 struct btrfs_csum_item *item;
966 struct btrfs_csum_item *item_end;
967 struct extent_buffer *leaf = NULL;
968 u64 next_offset;
969 u64 total_bytes = 0;
970 u64 csum_offset;
971 u64 bytenr;
972 u32 ins_size;
973 int index = 0;
974 int found_next;
975 int ret;
976 const u32 csum_size = fs_info->csum_size;
977
978 path = btrfs_alloc_path();
979 if (!path)
980 return -ENOMEM;
981again:
982 next_offset = (u64)-1;
983 found_next = 0;
984 bytenr = sums->bytenr + total_bytes;
985 file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
986 file_key.offset = bytenr;
987 file_key.type = BTRFS_EXTENT_CSUM_KEY;
988
989 item = btrfs_lookup_csum(trans, root, path, bytenr, 1);
990 if (!IS_ERR(item)) {
991 ret = 0;
992 leaf = path->nodes[0];
993 item_end = btrfs_item_ptr(leaf, path->slots[0],
994 struct btrfs_csum_item);
995 item_end = (struct btrfs_csum_item *)((char *)item_end +
996 btrfs_item_size_nr(leaf, path->slots[0]));
997 goto found;
998 }
999 ret = PTR_ERR(item);
1000 if (ret != -EFBIG && ret != -ENOENT)
1001 goto out;
1002
1003 if (ret == -EFBIG) {
1004 u32 item_size;
1005
1006 leaf = path->nodes[0];
1007 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1008 if ((item_size / csum_size) >=
1009 MAX_CSUM_ITEMS(fs_info, csum_size)) {
1010
1011 goto insert;
1012 }
1013 } else {
1014
1015 ret = find_next_csum_offset(root, path, &next_offset);
1016 if (ret < 0)
1017 goto out;
1018 found_next = 1;
1019 goto insert;
1020 }
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032 if (btrfs_leaf_free_space(leaf) >= csum_size) {
1033 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1034 csum_offset = (bytenr - found_key.offset) >>
1035 fs_info->sectorsize_bits;
1036 goto extend_csum;
1037 }
1038
1039 btrfs_release_path(path);
1040 path->search_for_extension = 1;
1041 ret = btrfs_search_slot(trans, root, &file_key, path,
1042 csum_size, 1);
1043 path->search_for_extension = 0;
1044 if (ret < 0)
1045 goto out;
1046
1047 if (ret > 0) {
1048 if (path->slots[0] == 0)
1049 goto insert;
1050 path->slots[0]--;
1051 }
1052
1053 leaf = path->nodes[0];
1054 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1055 csum_offset = (bytenr - found_key.offset) >> fs_info->sectorsize_bits;
1056
1057 if (found_key.type != BTRFS_EXTENT_CSUM_KEY ||
1058 found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID ||
1059 csum_offset >= MAX_CSUM_ITEMS(fs_info, csum_size)) {
1060 goto insert;
1061 }
1062
1063extend_csum:
1064 if (csum_offset == btrfs_item_size_nr(leaf, path->slots[0]) /
1065 csum_size) {
1066 int extend_nr;
1067 u64 tmp;
1068 u32 diff;
1069
1070 tmp = sums->len - total_bytes;
1071 tmp >>= fs_info->sectorsize_bits;
1072 WARN_ON(tmp < 1);
1073 extend_nr = max_t(int, 1, tmp);
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096 if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
1097 if (path->slots[0] + 1 >=
1098 btrfs_header_nritems(path->nodes[0])) {
1099 ret = find_next_csum_offset(root, path, &next_offset);
1100 if (ret < 0)
1101 goto out;
1102 found_next = 1;
1103 goto insert;
1104 }
1105
1106 ret = find_next_csum_offset(root, path, &next_offset);
1107 if (ret < 0)
1108 goto out;
1109
1110 tmp = (next_offset - bytenr) >> fs_info->sectorsize_bits;
1111 if (tmp <= INT_MAX)
1112 extend_nr = min_t(int, extend_nr, tmp);
1113 }
1114
1115 diff = (csum_offset + extend_nr) * csum_size;
1116 diff = min(diff,
1117 MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size);
1118
1119 diff = diff - btrfs_item_size_nr(leaf, path->slots[0]);
1120 diff = min_t(u32, btrfs_leaf_free_space(leaf), diff);
1121 diff /= csum_size;
1122 diff *= csum_size;
1123
1124 btrfs_extend_item(path, diff);
1125 ret = 0;
1126 goto csum;
1127 }
1128
1129insert:
1130 btrfs_release_path(path);
1131 csum_offset = 0;
1132 if (found_next) {
1133 u64 tmp;
1134
1135 tmp = sums->len - total_bytes;
1136 tmp >>= fs_info->sectorsize_bits;
1137 tmp = min(tmp, (next_offset - file_key.offset) >>
1138 fs_info->sectorsize_bits);
1139
1140 tmp = max_t(u64, 1, tmp);
1141 tmp = min_t(u64, tmp, MAX_CSUM_ITEMS(fs_info, csum_size));
1142 ins_size = csum_size * tmp;
1143 } else {
1144 ins_size = csum_size;
1145 }
1146 ret = btrfs_insert_empty_item(trans, root, path, &file_key,
1147 ins_size);
1148 if (ret < 0)
1149 goto out;
1150 if (WARN_ON(ret != 0))
1151 goto out;
1152 leaf = path->nodes[0];
1153csum:
1154 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item);
1155 item_end = (struct btrfs_csum_item *)((unsigned char *)item +
1156 btrfs_item_size_nr(leaf, path->slots[0]));
1157 item = (struct btrfs_csum_item *)((unsigned char *)item +
1158 csum_offset * csum_size);
1159found:
1160 ins_size = (u32)(sums->len - total_bytes) >> fs_info->sectorsize_bits;
1161 ins_size *= csum_size;
1162 ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item,
1163 ins_size);
1164 write_extent_buffer(leaf, sums->sums + index, (unsigned long)item,
1165 ins_size);
1166
1167 index += ins_size;
1168 ins_size /= csum_size;
1169 total_bytes += ins_size * fs_info->sectorsize;
1170
1171 btrfs_mark_buffer_dirty(path->nodes[0]);
1172 if (total_bytes < sums->len) {
1173 btrfs_release_path(path);
1174 cond_resched();
1175 goto again;
1176 }
1177out:
1178 btrfs_free_path(path);
1179 return ret;
1180}
1181
1182void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode,
1183 const struct btrfs_path *path,
1184 struct btrfs_file_extent_item *fi,
1185 const bool new_inline,
1186 struct extent_map *em)
1187{
1188 struct btrfs_fs_info *fs_info = inode->root->fs_info;
1189 struct btrfs_root *root = inode->root;
1190 struct extent_buffer *leaf = path->nodes[0];
1191 const int slot = path->slots[0];
1192 struct btrfs_key key;
1193 u64 extent_start, extent_end;
1194 u64 bytenr;
1195 u8 type = btrfs_file_extent_type(leaf, fi);
1196 int compress_type = btrfs_file_extent_compression(leaf, fi);
1197
1198 btrfs_item_key_to_cpu(leaf, &key, slot);
1199 extent_start = key.offset;
1200 extent_end = btrfs_file_extent_end(path);
1201 em->ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1202 if (type == BTRFS_FILE_EXTENT_REG ||
1203 type == BTRFS_FILE_EXTENT_PREALLOC) {
1204 em->start = extent_start;
1205 em->len = extent_end - extent_start;
1206 em->orig_start = extent_start -
1207 btrfs_file_extent_offset(leaf, fi);
1208 em->orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
1209 bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1210 if (bytenr == 0) {
1211 em->block_start = EXTENT_MAP_HOLE;
1212 return;
1213 }
1214 if (compress_type != BTRFS_COMPRESS_NONE) {
1215 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
1216 em->compress_type = compress_type;
1217 em->block_start = bytenr;
1218 em->block_len = em->orig_block_len;
1219 } else {
1220 bytenr += btrfs_file_extent_offset(leaf, fi);
1221 em->block_start = bytenr;
1222 em->block_len = em->len;
1223 if (type == BTRFS_FILE_EXTENT_PREALLOC)
1224 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
1225 }
1226 } else if (type == BTRFS_FILE_EXTENT_INLINE) {
1227 em->block_start = EXTENT_MAP_INLINE;
1228 em->start = extent_start;
1229 em->len = extent_end - extent_start;
1230
1231
1232
1233
1234 em->orig_start = EXTENT_MAP_HOLE;
1235 em->block_len = (u64)-1;
1236 if (!new_inline && compress_type != BTRFS_COMPRESS_NONE) {
1237 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
1238 em->compress_type = compress_type;
1239 }
1240 } else {
1241 btrfs_err(fs_info,
1242 "unknown file extent item type %d, inode %llu, offset %llu, "
1243 "root %llu", type, btrfs_ino(inode), extent_start,
1244 root->root_key.objectid);
1245 }
1246}
1247
1248
1249
1250
1251
1252
1253u64 btrfs_file_extent_end(const struct btrfs_path *path)
1254{
1255 const struct extent_buffer *leaf = path->nodes[0];
1256 const int slot = path->slots[0];
1257 struct btrfs_file_extent_item *fi;
1258 struct btrfs_key key;
1259 u64 end;
1260
1261 btrfs_item_key_to_cpu(leaf, &key, slot);
1262 ASSERT(key.type == BTRFS_EXTENT_DATA_KEY);
1263 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
1264
1265 if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) {
1266 end = btrfs_file_extent_ram_bytes(leaf, fi);
1267 end = ALIGN(key.offset + end, leaf->fs_info->sectorsize);
1268 } else {
1269 end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1270 }
1271
1272 return end;
1273}
1274