1#include <linux/bitops.h>
2#include <linux/slab.h>
3#include <linux/bio.h>
4#include <linux/mm.h>
5#include <linux/pagemap.h>
6#include <linux/page-flags.h>
7#include <linux/spinlock.h>
8#include <linux/blkdev.h>
9#include <linux/swap.h>
10#include <linux/writeback.h>
11#include <linux/pagevec.h>
12#include <linux/prefetch.h>
13#include <linux/cleancache.h>
14#include "extent_io.h"
15#include "extent_map.h"
16#include "compat.h"
17#include "ctree.h"
18#include "btrfs_inode.h"
19#include "volumes.h"
20#include "check-integrity.h"
21#include "locking.h"
22#include "rcu-string.h"
23
24static struct kmem_cache *extent_state_cache;
25static struct kmem_cache *extent_buffer_cache;
26static struct bio_set *btrfs_bioset;
27
28#ifdef CONFIG_BTRFS_DEBUG
29static LIST_HEAD(buffers);
30static LIST_HEAD(states);
31
32static DEFINE_SPINLOCK(leak_lock);
33
34static inline
35void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
36{
37 unsigned long flags;
38
39 spin_lock_irqsave(&leak_lock, flags);
40 list_add(new, head);
41 spin_unlock_irqrestore(&leak_lock, flags);
42}
43
44static inline
45void btrfs_leak_debug_del(struct list_head *entry)
46{
47 unsigned long flags;
48
49 spin_lock_irqsave(&leak_lock, flags);
50 list_del(entry);
51 spin_unlock_irqrestore(&leak_lock, flags);
52}
53
54static inline
55void btrfs_leak_debug_check(void)
56{
57 struct extent_state *state;
58 struct extent_buffer *eb;
59
60 while (!list_empty(&states)) {
61 state = list_entry(states.next, struct extent_state, leak_list);
62 printk(KERN_ERR "btrfs state leak: start %llu end %llu "
63 "state %lu in tree %p refs %d\n",
64 state->start, state->end, state->state, state->tree,
65 atomic_read(&state->refs));
66 list_del(&state->leak_list);
67 kmem_cache_free(extent_state_cache, state);
68 }
69
70 while (!list_empty(&buffers)) {
71 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
72 printk(KERN_ERR "btrfs buffer leak start %llu len %lu "
73 "refs %d\n",
74 eb->start, eb->len, atomic_read(&eb->refs));
75 list_del(&eb->leak_list);
76 kmem_cache_free(extent_buffer_cache, eb);
77 }
78}
79
80#define btrfs_debug_check_extent_io_range(inode, start, end) \
81 __btrfs_debug_check_extent_io_range(__func__, (inode), (start), (end))
82static inline void __btrfs_debug_check_extent_io_range(const char *caller,
83 struct inode *inode, u64 start, u64 end)
84{
85 u64 isize = i_size_read(inode);
86
87 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
88 printk_ratelimited(KERN_DEBUG
89 "btrfs: %s: ino %llu isize %llu odd range [%llu,%llu]\n",
90 caller, btrfs_ino(inode), isize, start, end);
91 }
92}
93#else
94#define btrfs_leak_debug_add(new, head) do {} while (0)
95#define btrfs_leak_debug_del(entry) do {} while (0)
96#define btrfs_leak_debug_check() do {} while (0)
97#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0)
98#endif
99
100#define BUFFER_LRU_MAX 64
101
102struct tree_entry {
103 u64 start;
104 u64 end;
105 struct rb_node rb_node;
106};
107
108struct extent_page_data {
109 struct bio *bio;
110 struct extent_io_tree *tree;
111 get_extent_t *get_extent;
112 unsigned long bio_flags;
113
114
115
116
117 unsigned int extent_locked:1;
118
119
120 unsigned int sync_io:1;
121};
122
123static noinline void flush_write_bio(void *data);
124static inline struct btrfs_fs_info *
125tree_fs_info(struct extent_io_tree *tree)
126{
127 return btrfs_sb(tree->mapping->host->i_sb);
128}
129
130int __init extent_io_init(void)
131{
132 extent_state_cache = kmem_cache_create("btrfs_extent_state",
133 sizeof(struct extent_state), 0,
134 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
135 if (!extent_state_cache)
136 return -ENOMEM;
137
138 extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
139 sizeof(struct extent_buffer), 0,
140 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
141 if (!extent_buffer_cache)
142 goto free_state_cache;
143
144 btrfs_bioset = bioset_create(BIO_POOL_SIZE,
145 offsetof(struct btrfs_io_bio, bio));
146 if (!btrfs_bioset)
147 goto free_buffer_cache;
148
149 if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
150 goto free_bioset;
151
152 return 0;
153
154free_bioset:
155 bioset_free(btrfs_bioset);
156 btrfs_bioset = NULL;
157
158free_buffer_cache:
159 kmem_cache_destroy(extent_buffer_cache);
160 extent_buffer_cache = NULL;
161
162free_state_cache:
163 kmem_cache_destroy(extent_state_cache);
164 extent_state_cache = NULL;
165 return -ENOMEM;
166}
167
168void extent_io_exit(void)
169{
170 btrfs_leak_debug_check();
171
172
173
174
175
176 rcu_barrier();
177 if (extent_state_cache)
178 kmem_cache_destroy(extent_state_cache);
179 if (extent_buffer_cache)
180 kmem_cache_destroy(extent_buffer_cache);
181 if (btrfs_bioset)
182 bioset_free(btrfs_bioset);
183}
184
185void extent_io_tree_init(struct extent_io_tree *tree,
186 struct address_space *mapping)
187{
188 tree->state = RB_ROOT;
189 INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
190 tree->ops = NULL;
191 tree->dirty_bytes = 0;
192 spin_lock_init(&tree->lock);
193 spin_lock_init(&tree->buffer_lock);
194 tree->mapping = mapping;
195}
196
197static struct extent_state *alloc_extent_state(gfp_t mask)
198{
199 struct extent_state *state;
200
201 state = kmem_cache_alloc(extent_state_cache, mask);
202 if (!state)
203 return state;
204 state->state = 0;
205 state->private = 0;
206 state->tree = NULL;
207 btrfs_leak_debug_add(&state->leak_list, &states);
208 atomic_set(&state->refs, 1);
209 init_waitqueue_head(&state->wq);
210 trace_alloc_extent_state(state, mask, _RET_IP_);
211 return state;
212}
213
214void free_extent_state(struct extent_state *state)
215{
216 if (!state)
217 return;
218 if (atomic_dec_and_test(&state->refs)) {
219 WARN_ON(state->tree);
220 btrfs_leak_debug_del(&state->leak_list);
221 trace_free_extent_state(state, _RET_IP_);
222 kmem_cache_free(extent_state_cache, state);
223 }
224}
225
226static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
227 struct rb_node *node)
228{
229 struct rb_node **p = &root->rb_node;
230 struct rb_node *parent = NULL;
231 struct tree_entry *entry;
232
233 while (*p) {
234 parent = *p;
235 entry = rb_entry(parent, struct tree_entry, rb_node);
236
237 if (offset < entry->start)
238 p = &(*p)->rb_left;
239 else if (offset > entry->end)
240 p = &(*p)->rb_right;
241 else
242 return parent;
243 }
244
245 rb_link_node(node, parent, p);
246 rb_insert_color(node, root);
247 return NULL;
248}
249
250static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
251 struct rb_node **prev_ret,
252 struct rb_node **next_ret)
253{
254 struct rb_root *root = &tree->state;
255 struct rb_node *n = root->rb_node;
256 struct rb_node *prev = NULL;
257 struct rb_node *orig_prev = NULL;
258 struct tree_entry *entry;
259 struct tree_entry *prev_entry = NULL;
260
261 while (n) {
262 entry = rb_entry(n, struct tree_entry, rb_node);
263 prev = n;
264 prev_entry = entry;
265
266 if (offset < entry->start)
267 n = n->rb_left;
268 else if (offset > entry->end)
269 n = n->rb_right;
270 else
271 return n;
272 }
273
274 if (prev_ret) {
275 orig_prev = prev;
276 while (prev && offset > prev_entry->end) {
277 prev = rb_next(prev);
278 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
279 }
280 *prev_ret = prev;
281 prev = orig_prev;
282 }
283
284 if (next_ret) {
285 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
286 while (prev && offset < prev_entry->start) {
287 prev = rb_prev(prev);
288 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
289 }
290 *next_ret = prev;
291 }
292 return NULL;
293}
294
295static inline struct rb_node *tree_search(struct extent_io_tree *tree,
296 u64 offset)
297{
298 struct rb_node *prev = NULL;
299 struct rb_node *ret;
300
301 ret = __etree_search(tree, offset, &prev, NULL);
302 if (!ret)
303 return prev;
304 return ret;
305}
306
307static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
308 struct extent_state *other)
309{
310 if (tree->ops && tree->ops->merge_extent_hook)
311 tree->ops->merge_extent_hook(tree->mapping->host, new,
312 other);
313}
314
315
316
317
318
319
320
321
322
323
324static void merge_state(struct extent_io_tree *tree,
325 struct extent_state *state)
326{
327 struct extent_state *other;
328 struct rb_node *other_node;
329
330 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
331 return;
332
333 other_node = rb_prev(&state->rb_node);
334 if (other_node) {
335 other = rb_entry(other_node, struct extent_state, rb_node);
336 if (other->end == state->start - 1 &&
337 other->state == state->state) {
338 merge_cb(tree, state, other);
339 state->start = other->start;
340 other->tree = NULL;
341 rb_erase(&other->rb_node, &tree->state);
342 free_extent_state(other);
343 }
344 }
345 other_node = rb_next(&state->rb_node);
346 if (other_node) {
347 other = rb_entry(other_node, struct extent_state, rb_node);
348 if (other->start == state->end + 1 &&
349 other->state == state->state) {
350 merge_cb(tree, state, other);
351 state->end = other->end;
352 other->tree = NULL;
353 rb_erase(&other->rb_node, &tree->state);
354 free_extent_state(other);
355 }
356 }
357}
358
359static void set_state_cb(struct extent_io_tree *tree,
360 struct extent_state *state, unsigned long *bits)
361{
362 if (tree->ops && tree->ops->set_bit_hook)
363 tree->ops->set_bit_hook(tree->mapping->host, state, bits);
364}
365
366static void clear_state_cb(struct extent_io_tree *tree,
367 struct extent_state *state, unsigned long *bits)
368{
369 if (tree->ops && tree->ops->clear_bit_hook)
370 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
371}
372
373static void set_state_bits(struct extent_io_tree *tree,
374 struct extent_state *state, unsigned long *bits);
375
376
377
378
379
380
381
382
383
384
385
386static int insert_state(struct extent_io_tree *tree,
387 struct extent_state *state, u64 start, u64 end,
388 unsigned long *bits)
389{
390 struct rb_node *node;
391
392 if (end < start)
393 WARN(1, KERN_ERR "btrfs end < start %llu %llu\n",
394 end, start);
395 state->start = start;
396 state->end = end;
397
398 set_state_bits(tree, state, bits);
399
400 node = tree_insert(&tree->state, end, &state->rb_node);
401 if (node) {
402 struct extent_state *found;
403 found = rb_entry(node, struct extent_state, rb_node);
404 printk(KERN_ERR "btrfs found node %llu %llu on insert of "
405 "%llu %llu\n",
406 found->start, found->end, start, end);
407 return -EEXIST;
408 }
409 state->tree = tree;
410 merge_state(tree, state);
411 return 0;
412}
413
414static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
415 u64 split)
416{
417 if (tree->ops && tree->ops->split_extent_hook)
418 tree->ops->split_extent_hook(tree->mapping->host, orig, split);
419}
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
436 struct extent_state *prealloc, u64 split)
437{
438 struct rb_node *node;
439
440 split_cb(tree, orig, split);
441
442 prealloc->start = orig->start;
443 prealloc->end = split - 1;
444 prealloc->state = orig->state;
445 orig->start = split;
446
447 node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
448 if (node) {
449 free_extent_state(prealloc);
450 return -EEXIST;
451 }
452 prealloc->tree = tree;
453 return 0;
454}
455
456static struct extent_state *next_state(struct extent_state *state)
457{
458 struct rb_node *next = rb_next(&state->rb_node);
459 if (next)
460 return rb_entry(next, struct extent_state, rb_node);
461 else
462 return NULL;
463}
464
465
466
467
468
469
470
471
472static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
473 struct extent_state *state,
474 unsigned long *bits, int wake)
475{
476 struct extent_state *next;
477 unsigned long bits_to_clear = *bits & ~EXTENT_CTLBITS;
478
479 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
480 u64 range = state->end - state->start + 1;
481 WARN_ON(range > tree->dirty_bytes);
482 tree->dirty_bytes -= range;
483 }
484 clear_state_cb(tree, state, bits);
485 state->state &= ~bits_to_clear;
486 if (wake)
487 wake_up(&state->wq);
488 if (state->state == 0) {
489 next = next_state(state);
490 if (state->tree) {
491 rb_erase(&state->rb_node, &tree->state);
492 state->tree = NULL;
493 free_extent_state(state);
494 } else {
495 WARN_ON(1);
496 }
497 } else {
498 merge_state(tree, state);
499 next = next_state(state);
500 }
501 return next;
502}
503
504static struct extent_state *
505alloc_extent_state_atomic(struct extent_state *prealloc)
506{
507 if (!prealloc)
508 prealloc = alloc_extent_state(GFP_ATOMIC);
509
510 return prealloc;
511}
512
513static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
514{
515 btrfs_panic(tree_fs_info(tree), err, "Locking error: "
516 "Extent tree was modified by another "
517 "thread while locked.");
518}
519
520
521
522
523
524
525
526
527
528
529
530
531
532int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
533 unsigned long bits, int wake, int delete,
534 struct extent_state **cached_state,
535 gfp_t mask)
536{
537 struct extent_state *state;
538 struct extent_state *cached;
539 struct extent_state *prealloc = NULL;
540 struct rb_node *node;
541 u64 last_end;
542 int err;
543 int clear = 0;
544
545 btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
546
547 if (bits & EXTENT_DELALLOC)
548 bits |= EXTENT_NORESERVE;
549
550 if (delete)
551 bits |= ~EXTENT_CTLBITS;
552 bits |= EXTENT_FIRST_DELALLOC;
553
554 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
555 clear = 1;
556again:
557 if (!prealloc && (mask & __GFP_WAIT)) {
558 prealloc = alloc_extent_state(mask);
559 if (!prealloc)
560 return -ENOMEM;
561 }
562
563 spin_lock(&tree->lock);
564 if (cached_state) {
565 cached = *cached_state;
566
567 if (clear) {
568 *cached_state = NULL;
569 cached_state = NULL;
570 }
571
572 if (cached && cached->tree && cached->start <= start &&
573 cached->end > start) {
574 if (clear)
575 atomic_dec(&cached->refs);
576 state = cached;
577 goto hit_next;
578 }
579 if (clear)
580 free_extent_state(cached);
581 }
582
583
584
585
586 node = tree_search(tree, start);
587 if (!node)
588 goto out;
589 state = rb_entry(node, struct extent_state, rb_node);
590hit_next:
591 if (state->start > end)
592 goto out;
593 WARN_ON(state->end < start);
594 last_end = state->end;
595
596
597 if (!(state->state & bits)) {
598 state = next_state(state);
599 goto next;
600 }
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618 if (state->start < start) {
619 prealloc = alloc_extent_state_atomic(prealloc);
620 BUG_ON(!prealloc);
621 err = split_state(tree, state, prealloc, start);
622 if (err)
623 extent_io_tree_panic(tree, err);
624
625 prealloc = NULL;
626 if (err)
627 goto out;
628 if (state->end <= end) {
629 state = clear_state_bit(tree, state, &bits, wake);
630 goto next;
631 }
632 goto search_again;
633 }
634
635
636
637
638
639
640 if (state->start <= end && state->end > end) {
641 prealloc = alloc_extent_state_atomic(prealloc);
642 BUG_ON(!prealloc);
643 err = split_state(tree, state, prealloc, end + 1);
644 if (err)
645 extent_io_tree_panic(tree, err);
646
647 if (wake)
648 wake_up(&state->wq);
649
650 clear_state_bit(tree, prealloc, &bits, wake);
651
652 prealloc = NULL;
653 goto out;
654 }
655
656 state = clear_state_bit(tree, state, &bits, wake);
657next:
658 if (last_end == (u64)-1)
659 goto out;
660 start = last_end + 1;
661 if (start <= end && state && !need_resched())
662 goto hit_next;
663 goto search_again;
664
665out:
666 spin_unlock(&tree->lock);
667 if (prealloc)
668 free_extent_state(prealloc);
669
670 return 0;
671
672search_again:
673 if (start > end)
674 goto out;
675 spin_unlock(&tree->lock);
676 if (mask & __GFP_WAIT)
677 cond_resched();
678 goto again;
679}
680
681static void wait_on_state(struct extent_io_tree *tree,
682 struct extent_state *state)
683 __releases(tree->lock)
684 __acquires(tree->lock)
685{
686 DEFINE_WAIT(wait);
687 prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
688 spin_unlock(&tree->lock);
689 schedule();
690 spin_lock(&tree->lock);
691 finish_wait(&state->wq, &wait);
692}
693
694
695
696
697
698
699static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
700 unsigned long bits)
701{
702 struct extent_state *state;
703 struct rb_node *node;
704
705 btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
706
707 spin_lock(&tree->lock);
708again:
709 while (1) {
710
711
712
713
714 node = tree_search(tree, start);
715 if (!node)
716 break;
717
718 state = rb_entry(node, struct extent_state, rb_node);
719
720 if (state->start > end)
721 goto out;
722
723 if (state->state & bits) {
724 start = state->start;
725 atomic_inc(&state->refs);
726 wait_on_state(tree, state);
727 free_extent_state(state);
728 goto again;
729 }
730 start = state->end + 1;
731
732 if (start > end)
733 break;
734
735 cond_resched_lock(&tree->lock);
736 }
737out:
738 spin_unlock(&tree->lock);
739}
740
741static void set_state_bits(struct extent_io_tree *tree,
742 struct extent_state *state,
743 unsigned long *bits)
744{
745 unsigned long bits_to_set = *bits & ~EXTENT_CTLBITS;
746
747 set_state_cb(tree, state, bits);
748 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
749 u64 range = state->end - state->start + 1;
750 tree->dirty_bytes += range;
751 }
752 state->state |= bits_to_set;
753}
754
755static void cache_state(struct extent_state *state,
756 struct extent_state **cached_ptr)
757{
758 if (cached_ptr && !(*cached_ptr)) {
759 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
760 *cached_ptr = state;
761 atomic_inc(&state->refs);
762 }
763 }
764}
765
766
767
768
769
770
771
772
773
774
775
776
777static int __must_check
778__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
779 unsigned long bits, unsigned long exclusive_bits,
780 u64 *failed_start, struct extent_state **cached_state,
781 gfp_t mask)
782{
783 struct extent_state *state;
784 struct extent_state *prealloc = NULL;
785 struct rb_node *node;
786 int err = 0;
787 u64 last_start;
788 u64 last_end;
789
790 btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
791
792 bits |= EXTENT_FIRST_DELALLOC;
793again:
794 if (!prealloc && (mask & __GFP_WAIT)) {
795 prealloc = alloc_extent_state(mask);
796 BUG_ON(!prealloc);
797 }
798
799 spin_lock(&tree->lock);
800 if (cached_state && *cached_state) {
801 state = *cached_state;
802 if (state->start <= start && state->end > start &&
803 state->tree) {
804 node = &state->rb_node;
805 goto hit_next;
806 }
807 }
808
809
810
811
812 node = tree_search(tree, start);
813 if (!node) {
814 prealloc = alloc_extent_state_atomic(prealloc);
815 BUG_ON(!prealloc);
816 err = insert_state(tree, prealloc, start, end, &bits);
817 if (err)
818 extent_io_tree_panic(tree, err);
819
820 prealloc = NULL;
821 goto out;
822 }
823 state = rb_entry(node, struct extent_state, rb_node);
824hit_next:
825 last_start = state->start;
826 last_end = state->end;
827
828
829
830
831
832
833
834 if (state->start == start && state->end <= end) {
835 if (state->state & exclusive_bits) {
836 *failed_start = state->start;
837 err = -EEXIST;
838 goto out;
839 }
840
841 set_state_bits(tree, state, &bits);
842 cache_state(state, cached_state);
843 merge_state(tree, state);
844 if (last_end == (u64)-1)
845 goto out;
846 start = last_end + 1;
847 state = next_state(state);
848 if (start < end && state && state->start == start &&
849 !need_resched())
850 goto hit_next;
851 goto search_again;
852 }
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870 if (state->start < start) {
871 if (state->state & exclusive_bits) {
872 *failed_start = start;
873 err = -EEXIST;
874 goto out;
875 }
876
877 prealloc = alloc_extent_state_atomic(prealloc);
878 BUG_ON(!prealloc);
879 err = split_state(tree, state, prealloc, start);
880 if (err)
881 extent_io_tree_panic(tree, err);
882
883 prealloc = NULL;
884 if (err)
885 goto out;
886 if (state->end <= end) {
887 set_state_bits(tree, state, &bits);
888 cache_state(state, cached_state);
889 merge_state(tree, state);
890 if (last_end == (u64)-1)
891 goto out;
892 start = last_end + 1;
893 state = next_state(state);
894 if (start < end && state && state->start == start &&
895 !need_resched())
896 goto hit_next;
897 }
898 goto search_again;
899 }
900
901
902
903
904
905
906
907 if (state->start > start) {
908 u64 this_end;
909 if (end < last_start)
910 this_end = end;
911 else
912 this_end = last_start - 1;
913
914 prealloc = alloc_extent_state_atomic(prealloc);
915 BUG_ON(!prealloc);
916
917
918
919
920
921 err = insert_state(tree, prealloc, start, this_end,
922 &bits);
923 if (err)
924 extent_io_tree_panic(tree, err);
925
926 cache_state(prealloc, cached_state);
927 prealloc = NULL;
928 start = this_end + 1;
929 goto search_again;
930 }
931
932
933
934
935
936
937 if (state->start <= end && state->end > end) {
938 if (state->state & exclusive_bits) {
939 *failed_start = start;
940 err = -EEXIST;
941 goto out;
942 }
943
944 prealloc = alloc_extent_state_atomic(prealloc);
945 BUG_ON(!prealloc);
946 err = split_state(tree, state, prealloc, end + 1);
947 if (err)
948 extent_io_tree_panic(tree, err);
949
950 set_state_bits(tree, prealloc, &bits);
951 cache_state(prealloc, cached_state);
952 merge_state(tree, prealloc);
953 prealloc = NULL;
954 goto out;
955 }
956
957 goto search_again;
958
959out:
960 spin_unlock(&tree->lock);
961 if (prealloc)
962 free_extent_state(prealloc);
963
964 return err;
965
966search_again:
967 if (start > end)
968 goto out;
969 spin_unlock(&tree->lock);
970 if (mask & __GFP_WAIT)
971 cond_resched();
972 goto again;
973}
974
975int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
976 unsigned long bits, u64 * failed_start,
977 struct extent_state **cached_state, gfp_t mask)
978{
979 return __set_extent_bit(tree, start, end, bits, 0, failed_start,
980 cached_state, mask);
981}
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1002 unsigned long bits, unsigned long clear_bits,
1003 struct extent_state **cached_state, gfp_t mask)
1004{
1005 struct extent_state *state;
1006 struct extent_state *prealloc = NULL;
1007 struct rb_node *node;
1008 int err = 0;
1009 u64 last_start;
1010 u64 last_end;
1011
1012 btrfs_debug_check_extent_io_range(tree->mapping->host, start, end);
1013
1014again:
1015 if (!prealloc && (mask & __GFP_WAIT)) {
1016 prealloc = alloc_extent_state(mask);
1017 if (!prealloc)
1018 return -ENOMEM;
1019 }
1020
1021 spin_lock(&tree->lock);
1022 if (cached_state && *cached_state) {
1023 state = *cached_state;
1024 if (state->start <= start && state->end > start &&
1025 state->tree) {
1026 node = &state->rb_node;
1027 goto hit_next;
1028 }
1029 }
1030
1031
1032
1033
1034
1035 node = tree_search(tree, start);
1036 if (!node) {
1037 prealloc = alloc_extent_state_atomic(prealloc);
1038 if (!prealloc) {
1039 err = -ENOMEM;
1040 goto out;
1041 }
1042 err = insert_state(tree, prealloc, start, end, &bits);
1043 prealloc = NULL;
1044 if (err)
1045 extent_io_tree_panic(tree, err);
1046 goto out;
1047 }
1048 state = rb_entry(node, struct extent_state, rb_node);
1049hit_next:
1050 last_start = state->start;
1051 last_end = state->end;
1052
1053
1054
1055
1056
1057
1058
1059 if (state->start == start && state->end <= end) {
1060 set_state_bits(tree, state, &bits);
1061 cache_state(state, cached_state);
1062 state = clear_state_bit(tree, state, &clear_bits, 0);
1063 if (last_end == (u64)-1)
1064 goto out;
1065 start = last_end + 1;
1066 if (start < end && state && state->start == start &&
1067 !need_resched())
1068 goto hit_next;
1069 goto search_again;
1070 }
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088 if (state->start < start) {
1089 prealloc = alloc_extent_state_atomic(prealloc);
1090 if (!prealloc) {
1091 err = -ENOMEM;
1092 goto out;
1093 }
1094 err = split_state(tree, state, prealloc, start);
1095 if (err)
1096 extent_io_tree_panic(tree, err);
1097 prealloc = NULL;
1098 if (err)
1099 goto out;
1100 if (state->end <= end) {
1101 set_state_bits(tree, state, &bits);
1102 cache_state(state, cached_state);
1103 state = clear_state_bit(tree, state, &clear_bits, 0);
1104 if (last_end == (u64)-1)
1105 goto out;
1106 start = last_end + 1;
1107 if (start < end && state && state->start == start &&
1108 !need_resched())
1109 goto hit_next;
1110 }
1111 goto search_again;
1112 }
1113
1114
1115
1116
1117
1118
1119
1120 if (state->start > start) {
1121 u64 this_end;
1122 if (end < last_start)
1123 this_end = end;
1124 else
1125 this_end = last_start - 1;
1126
1127 prealloc = alloc_extent_state_atomic(prealloc);
1128 if (!prealloc) {
1129 err = -ENOMEM;
1130 goto out;
1131 }
1132
1133
1134
1135
1136
1137 err = insert_state(tree, prealloc, start, this_end,
1138 &bits);
1139 if (err)
1140 extent_io_tree_panic(tree, err);
1141 cache_state(prealloc, cached_state);
1142 prealloc = NULL;
1143 start = this_end + 1;
1144 goto search_again;
1145 }
1146
1147
1148
1149
1150
1151
1152 if (state->start <= end && state->end > end) {
1153 prealloc = alloc_extent_state_atomic(prealloc);
1154 if (!prealloc) {
1155 err = -ENOMEM;
1156 goto out;
1157 }
1158
1159 err = split_state(tree, state, prealloc, end + 1);
1160 if (err)
1161 extent_io_tree_panic(tree, err);
1162
1163 set_state_bits(tree, prealloc, &bits);
1164 cache_state(prealloc, cached_state);
1165 clear_state_bit(tree, prealloc, &clear_bits, 0);
1166 prealloc = NULL;
1167 goto out;
1168 }
1169
1170 goto search_again;
1171
1172out:
1173 spin_unlock(&tree->lock);
1174 if (prealloc)
1175 free_extent_state(prealloc);
1176
1177 return err;
1178
1179search_again:
1180 if (start > end)
1181 goto out;
1182 spin_unlock(&tree->lock);
1183 if (mask & __GFP_WAIT)
1184 cond_resched();
1185 goto again;
1186}
1187
1188
1189int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1190 gfp_t mask)
1191{
1192 return set_extent_bit(tree, start, end, EXTENT_DIRTY, NULL,
1193 NULL, mask);
1194}
1195
1196int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1197 unsigned long bits, gfp_t mask)
1198{
1199 return set_extent_bit(tree, start, end, bits, NULL,
1200 NULL, mask);
1201}
1202
1203int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1204 unsigned long bits, gfp_t mask)
1205{
1206 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
1207}
1208
1209int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
1210 struct extent_state **cached_state, gfp_t mask)
1211{
1212 return set_extent_bit(tree, start, end,
1213 EXTENT_DELALLOC | EXTENT_UPTODATE,
1214 NULL, cached_state, mask);
1215}
1216
1217int set_extent_defrag(struct extent_io_tree *tree, u64 start, u64 end,
1218 struct extent_state **cached_state, gfp_t mask)
1219{
1220 return set_extent_bit(tree, start, end,
1221 EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DEFRAG,
1222 NULL, cached_state, mask);
1223}
1224
1225int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
1226 gfp_t mask)
1227{
1228 return clear_extent_bit(tree, start, end,
1229 EXTENT_DIRTY | EXTENT_DELALLOC |
1230 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
1231}
1232
1233int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
1234 gfp_t mask)
1235{
1236 return set_extent_bit(tree, start, end, EXTENT_NEW, NULL,
1237 NULL, mask);
1238}
1239
1240int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1241 struct extent_state **cached_state, gfp_t mask)
1242{
1243 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, NULL,
1244 cached_state, mask);
1245}
1246
1247int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
1248 struct extent_state **cached_state, gfp_t mask)
1249{
1250 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
1251 cached_state, mask);
1252}
1253
1254
1255
1256
1257
1258int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1259 unsigned long bits, struct extent_state **cached_state)
1260{
1261 int err;
1262 u64 failed_start;
1263 while (1) {
1264 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
1265 EXTENT_LOCKED, &failed_start,
1266 cached_state, GFP_NOFS);
1267 if (err == -EEXIST) {
1268 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1269 start = failed_start;
1270 } else
1271 break;
1272 WARN_ON(start > end);
1273 }
1274 return err;
1275}
1276
1277int lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1278{
1279 return lock_extent_bits(tree, start, end, 0, NULL);
1280}
1281
1282int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1283{
1284 int err;
1285 u64 failed_start;
1286
1287 err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1288 &failed_start, NULL, GFP_NOFS);
1289 if (err == -EEXIST) {
1290 if (failed_start > start)
1291 clear_extent_bit(tree, start, failed_start - 1,
1292 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
1293 return 0;
1294 }
1295 return 1;
1296}
1297
1298int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
1299 struct extent_state **cached, gfp_t mask)
1300{
1301 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1302 mask);
1303}
1304
1305int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1306{
1307 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
1308 GFP_NOFS);
1309}
1310
1311int extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1312{
1313 unsigned long index = start >> PAGE_CACHE_SHIFT;
1314 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1315 struct page *page;
1316
1317 while (index <= end_index) {
1318 page = find_get_page(inode->i_mapping, index);
1319 BUG_ON(!page);
1320 clear_page_dirty_for_io(page);
1321 page_cache_release(page);
1322 index++;
1323 }
1324 return 0;
1325}
1326
1327int extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1328{
1329 unsigned long index = start >> PAGE_CACHE_SHIFT;
1330 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1331 struct page *page;
1332
1333 while (index <= end_index) {
1334 page = find_get_page(inode->i_mapping, index);
1335 BUG_ON(!page);
1336 account_page_redirty(page);
1337 __set_page_dirty_nobuffers(page);
1338 page_cache_release(page);
1339 index++;
1340 }
1341 return 0;
1342}
1343
1344
1345
1346
1347static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
1348{
1349 unsigned long index = start >> PAGE_CACHE_SHIFT;
1350 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1351 struct page *page;
1352
1353 while (index <= end_index) {
1354 page = find_get_page(tree->mapping, index);
1355 BUG_ON(!page);
1356 set_page_writeback(page);
1357 page_cache_release(page);
1358 index++;
1359 }
1360 return 0;
1361}
1362
1363
1364
1365
1366
1367static struct extent_state *
1368find_first_extent_bit_state(struct extent_io_tree *tree,
1369 u64 start, unsigned long bits)
1370{
1371 struct rb_node *node;
1372 struct extent_state *state;
1373
1374
1375
1376
1377
1378 node = tree_search(tree, start);
1379 if (!node)
1380 goto out;
1381
1382 while (1) {
1383 state = rb_entry(node, struct extent_state, rb_node);
1384 if (state->end >= start && (state->state & bits))
1385 return state;
1386
1387 node = rb_next(node);
1388 if (!node)
1389 break;
1390 }
1391out:
1392 return NULL;
1393}
1394
1395
1396
1397
1398
1399
1400
1401
1402int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1403 u64 *start_ret, u64 *end_ret, unsigned long bits,
1404 struct extent_state **cached_state)
1405{
1406 struct extent_state *state;
1407 struct rb_node *n;
1408 int ret = 1;
1409
1410 spin_lock(&tree->lock);
1411 if (cached_state && *cached_state) {
1412 state = *cached_state;
1413 if (state->end == start - 1 && state->tree) {
1414 n = rb_next(&state->rb_node);
1415 while (n) {
1416 state = rb_entry(n, struct extent_state,
1417 rb_node);
1418 if (state->state & bits)
1419 goto got_it;
1420 n = rb_next(n);
1421 }
1422 free_extent_state(*cached_state);
1423 *cached_state = NULL;
1424 goto out;
1425 }
1426 free_extent_state(*cached_state);
1427 *cached_state = NULL;
1428 }
1429
1430 state = find_first_extent_bit_state(tree, start, bits);
1431got_it:
1432 if (state) {
1433 cache_state(state, cached_state);
1434 *start_ret = state->start;
1435 *end_ret = state->end;
1436 ret = 0;
1437 }
1438out:
1439 spin_unlock(&tree->lock);
1440 return ret;
1441}
1442
1443
1444
1445
1446
1447
1448
1449static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1450 u64 *start, u64 *end, u64 max_bytes,
1451 struct extent_state **cached_state)
1452{
1453 struct rb_node *node;
1454 struct extent_state *state;
1455 u64 cur_start = *start;
1456 u64 found = 0;
1457 u64 total_bytes = 0;
1458
1459 spin_lock(&tree->lock);
1460
1461
1462
1463
1464
1465 node = tree_search(tree, cur_start);
1466 if (!node) {
1467 if (!found)
1468 *end = (u64)-1;
1469 goto out;
1470 }
1471
1472 while (1) {
1473 state = rb_entry(node, struct extent_state, rb_node);
1474 if (found && (state->start != cur_start ||
1475 (state->state & EXTENT_BOUNDARY))) {
1476 goto out;
1477 }
1478 if (!(state->state & EXTENT_DELALLOC)) {
1479 if (!found)
1480 *end = state->end;
1481 goto out;
1482 }
1483 if (!found) {
1484 *start = state->start;
1485 *cached_state = state;
1486 atomic_inc(&state->refs);
1487 }
1488 found++;
1489 *end = state->end;
1490 cur_start = state->end + 1;
1491 node = rb_next(node);
1492 total_bytes += state->end - state->start + 1;
1493 if (total_bytes >= max_bytes)
1494 break;
1495 if (!node)
1496 break;
1497 }
1498out:
1499 spin_unlock(&tree->lock);
1500 return found;
1501}
1502
1503static noinline void __unlock_for_delalloc(struct inode *inode,
1504 struct page *locked_page,
1505 u64 start, u64 end)
1506{
1507 int ret;
1508 struct page *pages[16];
1509 unsigned long index = start >> PAGE_CACHE_SHIFT;
1510 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1511 unsigned long nr_pages = end_index - index + 1;
1512 int i;
1513
1514 if (index == locked_page->index && end_index == index)
1515 return;
1516
1517 while (nr_pages > 0) {
1518 ret = find_get_pages_contig(inode->i_mapping, index,
1519 min_t(unsigned long, nr_pages,
1520 ARRAY_SIZE(pages)), pages);
1521 for (i = 0; i < ret; i++) {
1522 if (pages[i] != locked_page)
1523 unlock_page(pages[i]);
1524 page_cache_release(pages[i]);
1525 }
1526 nr_pages -= ret;
1527 index += ret;
1528 cond_resched();
1529 }
1530}
1531
1532static noinline int lock_delalloc_pages(struct inode *inode,
1533 struct page *locked_page,
1534 u64 delalloc_start,
1535 u64 delalloc_end)
1536{
1537 unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1538 unsigned long start_index = index;
1539 unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1540 unsigned long pages_locked = 0;
1541 struct page *pages[16];
1542 unsigned long nrpages;
1543 int ret;
1544 int i;
1545
1546
1547 if (index == locked_page->index && index == end_index)
1548 return 0;
1549
1550
1551 nrpages = end_index - index + 1;
1552 while (nrpages > 0) {
1553 ret = find_get_pages_contig(inode->i_mapping, index,
1554 min_t(unsigned long,
1555 nrpages, ARRAY_SIZE(pages)), pages);
1556 if (ret == 0) {
1557 ret = -EAGAIN;
1558 goto done;
1559 }
1560
1561 for (i = 0; i < ret; i++) {
1562
1563
1564
1565
1566 if (pages[i] != locked_page) {
1567 lock_page(pages[i]);
1568 if (!PageDirty(pages[i]) ||
1569 pages[i]->mapping != inode->i_mapping) {
1570 ret = -EAGAIN;
1571 unlock_page(pages[i]);
1572 page_cache_release(pages[i]);
1573 goto done;
1574 }
1575 }
1576 page_cache_release(pages[i]);
1577 pages_locked++;
1578 }
1579 nrpages -= ret;
1580 index += ret;
1581 cond_resched();
1582 }
1583 ret = 0;
1584done:
1585 if (ret && pages_locked) {
1586 __unlock_for_delalloc(inode, locked_page,
1587 delalloc_start,
1588 ((u64)(start_index + pages_locked - 1)) <<
1589 PAGE_CACHE_SHIFT);
1590 }
1591 return ret;
1592}
1593
1594
1595
1596
1597
1598
1599
1600static noinline u64 find_lock_delalloc_range(struct inode *inode,
1601 struct extent_io_tree *tree,
1602 struct page *locked_page,
1603 u64 *start, u64 *end,
1604 u64 max_bytes)
1605{
1606 u64 delalloc_start;
1607 u64 delalloc_end;
1608 u64 found;
1609 struct extent_state *cached_state = NULL;
1610 int ret;
1611 int loops = 0;
1612
1613again:
1614
1615 delalloc_start = *start;
1616 delalloc_end = 0;
1617 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1618 max_bytes, &cached_state);
1619 if (!found || delalloc_end <= *start) {
1620 *start = delalloc_start;
1621 *end = delalloc_end;
1622 free_extent_state(cached_state);
1623 return 0;
1624 }
1625
1626
1627
1628
1629
1630
1631 if (delalloc_start < *start)
1632 delalloc_start = *start;
1633
1634
1635
1636
1637 if (delalloc_end + 1 - delalloc_start > max_bytes)
1638 delalloc_end = delalloc_start + max_bytes - 1;
1639
1640
1641 ret = lock_delalloc_pages(inode, locked_page,
1642 delalloc_start, delalloc_end);
1643 if (ret == -EAGAIN) {
1644
1645
1646
1647 free_extent_state(cached_state);
1648 if (!loops) {
1649 max_bytes = PAGE_CACHE_SIZE;
1650 loops = 1;
1651 goto again;
1652 } else {
1653 found = 0;
1654 goto out_failed;
1655 }
1656 }
1657 BUG_ON(ret);
1658
1659
1660 lock_extent_bits(tree, delalloc_start, delalloc_end, 0, &cached_state);
1661
1662
1663 ret = test_range_bit(tree, delalloc_start, delalloc_end,
1664 EXTENT_DELALLOC, 1, cached_state);
1665 if (!ret) {
1666 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1667 &cached_state, GFP_NOFS);
1668 __unlock_for_delalloc(inode, locked_page,
1669 delalloc_start, delalloc_end);
1670 cond_resched();
1671 goto again;
1672 }
1673 free_extent_state(cached_state);
1674 *start = delalloc_start;
1675 *end = delalloc_end;
1676out_failed:
1677 return found;
1678}
1679
1680int extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1681 struct page *locked_page,
1682 unsigned long clear_bits,
1683 unsigned long page_ops)
1684{
1685 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1686 int ret;
1687 struct page *pages[16];
1688 unsigned long index = start >> PAGE_CACHE_SHIFT;
1689 unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1690 unsigned long nr_pages = end_index - index + 1;
1691 int i;
1692
1693 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
1694 if (page_ops == 0)
1695 return 0;
1696
1697 while (nr_pages > 0) {
1698 ret = find_get_pages_contig(inode->i_mapping, index,
1699 min_t(unsigned long,
1700 nr_pages, ARRAY_SIZE(pages)), pages);
1701 for (i = 0; i < ret; i++) {
1702
1703 if (page_ops & PAGE_SET_PRIVATE2)
1704 SetPagePrivate2(pages[i]);
1705
1706 if (pages[i] == locked_page) {
1707 page_cache_release(pages[i]);
1708 continue;
1709 }
1710 if (page_ops & PAGE_CLEAR_DIRTY)
1711 clear_page_dirty_for_io(pages[i]);
1712 if (page_ops & PAGE_SET_WRITEBACK)
1713 set_page_writeback(pages[i]);
1714 if (page_ops & PAGE_END_WRITEBACK)
1715 end_page_writeback(pages[i]);
1716 if (page_ops & PAGE_UNLOCK)
1717 unlock_page(pages[i]);
1718 page_cache_release(pages[i]);
1719 }
1720 nr_pages -= ret;
1721 index += ret;
1722 cond_resched();
1723 }
1724 return 0;
1725}
1726
1727
1728
1729
1730
1731
1732u64 count_range_bits(struct extent_io_tree *tree,
1733 u64 *start, u64 search_end, u64 max_bytes,
1734 unsigned long bits, int contig)
1735{
1736 struct rb_node *node;
1737 struct extent_state *state;
1738 u64 cur_start = *start;
1739 u64 total_bytes = 0;
1740 u64 last = 0;
1741 int found = 0;
1742
1743 if (search_end <= cur_start) {
1744 WARN_ON(1);
1745 return 0;
1746 }
1747
1748 spin_lock(&tree->lock);
1749 if (cur_start == 0 && bits == EXTENT_DIRTY) {
1750 total_bytes = tree->dirty_bytes;
1751 goto out;
1752 }
1753
1754
1755
1756
1757 node = tree_search(tree, cur_start);
1758 if (!node)
1759 goto out;
1760
1761 while (1) {
1762 state = rb_entry(node, struct extent_state, rb_node);
1763 if (state->start > search_end)
1764 break;
1765 if (contig && found && state->start > last + 1)
1766 break;
1767 if (state->end >= cur_start && (state->state & bits) == bits) {
1768 total_bytes += min(search_end, state->end) + 1 -
1769 max(cur_start, state->start);
1770 if (total_bytes >= max_bytes)
1771 break;
1772 if (!found) {
1773 *start = max(cur_start, state->start);
1774 found = 1;
1775 }
1776 last = state->end;
1777 } else if (contig && found) {
1778 break;
1779 }
1780 node = rb_next(node);
1781 if (!node)
1782 break;
1783 }
1784out:
1785 spin_unlock(&tree->lock);
1786 return total_bytes;
1787}
1788
1789
1790
1791
1792
1793static int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1794{
1795 struct rb_node *node;
1796 struct extent_state *state;
1797 int ret = 0;
1798
1799 spin_lock(&tree->lock);
1800
1801
1802
1803
1804 node = tree_search(tree, start);
1805 if (!node) {
1806 ret = -ENOENT;
1807 goto out;
1808 }
1809 state = rb_entry(node, struct extent_state, rb_node);
1810 if (state->start != start) {
1811 ret = -ENOENT;
1812 goto out;
1813 }
1814 state->private = private;
1815out:
1816 spin_unlock(&tree->lock);
1817 return ret;
1818}
1819
1820int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1821{
1822 struct rb_node *node;
1823 struct extent_state *state;
1824 int ret = 0;
1825
1826 spin_lock(&tree->lock);
1827
1828
1829
1830
1831 node = tree_search(tree, start);
1832 if (!node) {
1833 ret = -ENOENT;
1834 goto out;
1835 }
1836 state = rb_entry(node, struct extent_state, rb_node);
1837 if (state->start != start) {
1838 ret = -ENOENT;
1839 goto out;
1840 }
1841 *private = state->private;
1842out:
1843 spin_unlock(&tree->lock);
1844 return ret;
1845}
1846
1847
1848
1849
1850
1851
1852
1853int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1854 unsigned long bits, int filled, struct extent_state *cached)
1855{
1856 struct extent_state *state = NULL;
1857 struct rb_node *node;
1858 int bitset = 0;
1859
1860 spin_lock(&tree->lock);
1861 if (cached && cached->tree && cached->start <= start &&
1862 cached->end > start)
1863 node = &cached->rb_node;
1864 else
1865 node = tree_search(tree, start);
1866 while (node && start <= end) {
1867 state = rb_entry(node, struct extent_state, rb_node);
1868
1869 if (filled && state->start > start) {
1870 bitset = 0;
1871 break;
1872 }
1873
1874 if (state->start > end)
1875 break;
1876
1877 if (state->state & bits) {
1878 bitset = 1;
1879 if (!filled)
1880 break;
1881 } else if (filled) {
1882 bitset = 0;
1883 break;
1884 }
1885
1886 if (state->end == (u64)-1)
1887 break;
1888
1889 start = state->end + 1;
1890 if (start > end)
1891 break;
1892 node = rb_next(node);
1893 if (!node) {
1894 if (filled)
1895 bitset = 0;
1896 break;
1897 }
1898 }
1899 spin_unlock(&tree->lock);
1900 return bitset;
1901}
1902
1903
1904
1905
1906
1907static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1908{
1909 u64 start = page_offset(page);
1910 u64 end = start + PAGE_CACHE_SIZE - 1;
1911 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1912 SetPageUptodate(page);
1913}
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923struct io_failure_record {
1924 struct page *page;
1925 u64 start;
1926 u64 len;
1927 u64 logical;
1928 unsigned long bio_flags;
1929 int this_mirror;
1930 int failed_mirror;
1931 int in_validation;
1932};
1933
1934static int free_io_failure(struct inode *inode, struct io_failure_record *rec,
1935 int did_repair)
1936{
1937 int ret;
1938 int err = 0;
1939 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1940
1941 set_state_private(failure_tree, rec->start, 0);
1942 ret = clear_extent_bits(failure_tree, rec->start,
1943 rec->start + rec->len - 1,
1944 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1945 if (ret)
1946 err = ret;
1947
1948 ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1949 rec->start + rec->len - 1,
1950 EXTENT_DAMAGED, GFP_NOFS);
1951 if (ret && !err)
1952 err = ret;
1953
1954 kfree(rec);
1955 return err;
1956}
1957
1958static void repair_io_failure_callback(struct bio *bio, int err)
1959{
1960 complete(bio->bi_private);
1961}
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973int repair_io_failure(struct btrfs_fs_info *fs_info, u64 start,
1974 u64 length, u64 logical, struct page *page,
1975 int mirror_num)
1976{
1977 struct bio *bio;
1978 struct btrfs_device *dev;
1979 DECLARE_COMPLETION_ONSTACK(compl);
1980 u64 map_length = 0;
1981 u64 sector;
1982 struct btrfs_bio *bbio = NULL;
1983 struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
1984 int ret;
1985
1986 BUG_ON(!mirror_num);
1987
1988
1989 if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
1990 return 0;
1991
1992 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1993 if (!bio)
1994 return -EIO;
1995 bio->bi_private = &compl;
1996 bio->bi_end_io = repair_io_failure_callback;
1997 bio->bi_size = 0;
1998 map_length = length;
1999
2000 ret = btrfs_map_block(fs_info, WRITE, logical,
2001 &map_length, &bbio, mirror_num);
2002 if (ret) {
2003 bio_put(bio);
2004 return -EIO;
2005 }
2006 BUG_ON(mirror_num != bbio->mirror_num);
2007 sector = bbio->stripes[mirror_num-1].physical >> 9;
2008 bio->bi_sector = sector;
2009 dev = bbio->stripes[mirror_num-1].dev;
2010 kfree(bbio);
2011 if (!dev || !dev->bdev || !dev->writeable) {
2012 bio_put(bio);
2013 return -EIO;
2014 }
2015 bio->bi_bdev = dev->bdev;
2016 bio_add_page(bio, page, length, start - page_offset(page));
2017 btrfsic_submit_bio(WRITE_SYNC, bio);
2018 wait_for_completion(&compl);
2019
2020 if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) {
2021
2022 bio_put(bio);
2023 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2024 return -EIO;
2025 }
2026
2027 printk_ratelimited_in_rcu(KERN_INFO "btrfs read error corrected: ino %lu off %llu "
2028 "(dev %s sector %llu)\n", page->mapping->host->i_ino,
2029 start, rcu_str_deref(dev->name), sector);
2030
2031 bio_put(bio);
2032 return 0;
2033}
2034
2035int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2036 int mirror_num)
2037{
2038 u64 start = eb->start;
2039 unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
2040 int ret = 0;
2041
2042 for (i = 0; i < num_pages; i++) {
2043 struct page *p = extent_buffer_page(eb, i);
2044 ret = repair_io_failure(root->fs_info, start, PAGE_CACHE_SIZE,
2045 start, p, mirror_num);
2046 if (ret)
2047 break;
2048 start += PAGE_CACHE_SIZE;
2049 }
2050
2051 return ret;
2052}
2053
2054
2055
2056
2057
2058static int clean_io_failure(u64 start, struct page *page)
2059{
2060 u64 private;
2061 u64 private_failure;
2062 struct io_failure_record *failrec;
2063 struct btrfs_fs_info *fs_info;
2064 struct extent_state *state;
2065 int num_copies;
2066 int did_repair = 0;
2067 int ret;
2068 struct inode *inode = page->mapping->host;
2069
2070 private = 0;
2071 ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2072 (u64)-1, 1, EXTENT_DIRTY, 0);
2073 if (!ret)
2074 return 0;
2075
2076 ret = get_state_private(&BTRFS_I(inode)->io_failure_tree, start,
2077 &private_failure);
2078 if (ret)
2079 return 0;
2080
2081 failrec = (struct io_failure_record *)(unsigned long) private_failure;
2082 BUG_ON(!failrec->this_mirror);
2083
2084 if (failrec->in_validation) {
2085
2086 pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2087 failrec->start);
2088 did_repair = 1;
2089 goto out;
2090 }
2091
2092 spin_lock(&BTRFS_I(inode)->io_tree.lock);
2093 state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2094 failrec->start,
2095 EXTENT_LOCKED);
2096 spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2097
2098 if (state && state->start <= failrec->start &&
2099 state->end >= failrec->start + failrec->len - 1) {
2100 fs_info = BTRFS_I(inode)->root->fs_info;
2101 num_copies = btrfs_num_copies(fs_info, failrec->logical,
2102 failrec->len);
2103 if (num_copies > 1) {
2104 ret = repair_io_failure(fs_info, start, failrec->len,
2105 failrec->logical, page,
2106 failrec->failed_mirror);
2107 did_repair = !ret;
2108 }
2109 ret = 0;
2110 }
2111
2112out:
2113 if (!ret)
2114 ret = free_io_failure(inode, failrec, did_repair);
2115
2116 return ret;
2117}
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2128 struct page *page, u64 start, u64 end,
2129 int failed_mirror)
2130{
2131 struct io_failure_record *failrec = NULL;
2132 u64 private;
2133 struct extent_map *em;
2134 struct inode *inode = page->mapping->host;
2135 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2136 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2137 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2138 struct bio *bio;
2139 struct btrfs_io_bio *btrfs_failed_bio;
2140 struct btrfs_io_bio *btrfs_bio;
2141 int num_copies;
2142 int ret;
2143 int read_mode;
2144 u64 logical;
2145
2146 BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2147
2148 ret = get_state_private(failure_tree, start, &private);
2149 if (ret) {
2150 failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2151 if (!failrec)
2152 return -ENOMEM;
2153 failrec->start = start;
2154 failrec->len = end - start + 1;
2155 failrec->this_mirror = 0;
2156 failrec->bio_flags = 0;
2157 failrec->in_validation = 0;
2158
2159 read_lock(&em_tree->lock);
2160 em = lookup_extent_mapping(em_tree, start, failrec->len);
2161 if (!em) {
2162 read_unlock(&em_tree->lock);
2163 kfree(failrec);
2164 return -EIO;
2165 }
2166
2167 if (em->start > start || em->start + em->len < start) {
2168 free_extent_map(em);
2169 em = NULL;
2170 }
2171 read_unlock(&em_tree->lock);
2172
2173 if (!em) {
2174 kfree(failrec);
2175 return -EIO;
2176 }
2177 logical = start - em->start;
2178 logical = em->block_start + logical;
2179 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2180 logical = em->block_start;
2181 failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2182 extent_set_compress_type(&failrec->bio_flags,
2183 em->compress_type);
2184 }
2185 pr_debug("bio_readpage_error: (new) logical=%llu, start=%llu, "
2186 "len=%llu\n", logical, start, failrec->len);
2187 failrec->logical = logical;
2188 free_extent_map(em);
2189
2190
2191 ret = set_extent_bits(failure_tree, start, end,
2192 EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2193 if (ret >= 0)
2194 ret = set_state_private(failure_tree, start,
2195 (u64)(unsigned long)failrec);
2196
2197 if (ret >= 0)
2198 ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2199 GFP_NOFS);
2200 if (ret < 0) {
2201 kfree(failrec);
2202 return ret;
2203 }
2204 } else {
2205 failrec = (struct io_failure_record *)(unsigned long)private;
2206 pr_debug("bio_readpage_error: (found) logical=%llu, "
2207 "start=%llu, len=%llu, validation=%d\n",
2208 failrec->logical, failrec->start, failrec->len,
2209 failrec->in_validation);
2210
2211
2212
2213
2214
2215 }
2216 num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2217 failrec->logical, failrec->len);
2218 if (num_copies == 1) {
2219
2220
2221
2222
2223
2224 pr_debug("bio_readpage_error: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
2225 num_copies, failrec->this_mirror, failed_mirror);
2226 free_io_failure(inode, failrec, 0);
2227 return -EIO;
2228 }
2229
2230
2231
2232
2233
2234
2235 if (failed_bio->bi_vcnt > 1) {
2236
2237
2238
2239
2240
2241
2242
2243
2244 BUG_ON(failrec->in_validation);
2245 failrec->in_validation = 1;
2246 failrec->this_mirror = failed_mirror;
2247 read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2248 } else {
2249
2250
2251
2252
2253
2254 if (failrec->in_validation) {
2255 BUG_ON(failrec->this_mirror != failed_mirror);
2256 failrec->in_validation = 0;
2257 failrec->this_mirror = 0;
2258 }
2259 failrec->failed_mirror = failed_mirror;
2260 failrec->this_mirror++;
2261 if (failrec->this_mirror == failed_mirror)
2262 failrec->this_mirror++;
2263 read_mode = READ_SYNC;
2264 }
2265
2266 if (failrec->this_mirror > num_copies) {
2267 pr_debug("bio_readpage_error: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
2268 num_copies, failrec->this_mirror, failed_mirror);
2269 free_io_failure(inode, failrec, 0);
2270 return -EIO;
2271 }
2272
2273 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
2274 if (!bio) {
2275 free_io_failure(inode, failrec, 0);
2276 return -EIO;
2277 }
2278 bio->bi_end_io = failed_bio->bi_end_io;
2279 bio->bi_sector = failrec->logical >> 9;
2280 bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2281 bio->bi_size = 0;
2282
2283 btrfs_failed_bio = btrfs_io_bio(failed_bio);
2284 if (btrfs_failed_bio->csum) {
2285 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2286 u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2287
2288 btrfs_bio = btrfs_io_bio(bio);
2289 btrfs_bio->csum = btrfs_bio->csum_inline;
2290 phy_offset >>= inode->i_sb->s_blocksize_bits;
2291 phy_offset *= csum_size;
2292 memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + phy_offset,
2293 csum_size);
2294 }
2295
2296 bio_add_page(bio, page, failrec->len, start - page_offset(page));
2297
2298 pr_debug("bio_readpage_error: submitting new read[%#x] to "
2299 "this_mirror=%d, num_copies=%d, in_validation=%d\n", read_mode,
2300 failrec->this_mirror, num_copies, failrec->in_validation);
2301
2302 ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2303 failrec->this_mirror,
2304 failrec->bio_flags, 0);
2305 return ret;
2306}
2307
2308
2309
2310int end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2311{
2312 int uptodate = (err == 0);
2313 struct extent_io_tree *tree;
2314 int ret;
2315
2316 tree = &BTRFS_I(page->mapping->host)->io_tree;
2317
2318 if (tree->ops && tree->ops->writepage_end_io_hook) {
2319 ret = tree->ops->writepage_end_io_hook(page, start,
2320 end, NULL, uptodate);
2321 if (ret)
2322 uptodate = 0;
2323 }
2324
2325 if (!uptodate) {
2326 ClearPageUptodate(page);
2327 SetPageError(page);
2328 }
2329 return 0;
2330}
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341static void end_bio_extent_writepage(struct bio *bio, int err)
2342{
2343 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2344 struct extent_io_tree *tree;
2345 u64 start;
2346 u64 end;
2347
2348 do {
2349 struct page *page = bvec->bv_page;
2350 tree = &BTRFS_I(page->mapping->host)->io_tree;
2351
2352
2353
2354
2355
2356
2357 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE)
2358 printk("%s page write in btrfs with offset %u and length %u\n",
2359 bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE
2360 ? KERN_ERR "partial" : KERN_INFO "incomplete",
2361 bvec->bv_offset, bvec->bv_len);
2362
2363 start = page_offset(page);
2364 end = start + bvec->bv_offset + bvec->bv_len - 1;
2365
2366 if (--bvec >= bio->bi_io_vec)
2367 prefetchw(&bvec->bv_page->flags);
2368
2369 if (end_extent_writepage(page, err, start, end))
2370 continue;
2371
2372 end_page_writeback(page);
2373 } while (bvec >= bio->bi_io_vec);
2374
2375 bio_put(bio);
2376}
2377
2378static void
2379endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2380 int uptodate)
2381{
2382 struct extent_state *cached = NULL;
2383 u64 end = start + len - 1;
2384
2385 if (uptodate && tree->track_uptodate)
2386 set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2387 unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2388}
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401static void end_bio_extent_readpage(struct bio *bio, int err)
2402{
2403 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
2404 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
2405 struct bio_vec *bvec = bio->bi_io_vec;
2406 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2407 struct extent_io_tree *tree;
2408 u64 offset = 0;
2409 u64 start;
2410 u64 end;
2411 u64 len;
2412 u64 extent_start = 0;
2413 u64 extent_len = 0;
2414 int mirror;
2415 int ret;
2416
2417 if (err)
2418 uptodate = 0;
2419
2420 do {
2421 struct page *page = bvec->bv_page;
2422 struct inode *inode = page->mapping->host;
2423
2424 pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
2425 "mirror=%lu\n", (u64)bio->bi_sector, err,
2426 io_bio->mirror_num);
2427 tree = &BTRFS_I(inode)->io_tree;
2428
2429
2430
2431
2432
2433
2434 if (bvec->bv_offset || bvec->bv_len != PAGE_CACHE_SIZE)
2435 printk("%s page read in btrfs with offset %u and length %u\n",
2436 bvec->bv_offset + bvec->bv_len != PAGE_CACHE_SIZE
2437 ? KERN_ERR "partial" : KERN_INFO "incomplete",
2438 bvec->bv_offset, bvec->bv_len);
2439
2440 start = page_offset(page);
2441 end = start + bvec->bv_offset + bvec->bv_len - 1;
2442 len = bvec->bv_len;
2443
2444 if (++bvec <= bvec_end)
2445 prefetchw(&bvec->bv_page->flags);
2446
2447 mirror = io_bio->mirror_num;
2448 if (likely(uptodate && tree->ops &&
2449 tree->ops->readpage_end_io_hook)) {
2450 ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2451 page, start, end,
2452 mirror);
2453 if (ret)
2454 uptodate = 0;
2455 else
2456 clean_io_failure(start, page);
2457 }
2458
2459 if (likely(uptodate))
2460 goto readpage_ok;
2461
2462 if (tree->ops && tree->ops->readpage_io_failed_hook) {
2463 ret = tree->ops->readpage_io_failed_hook(page, mirror);
2464 if (!ret && !err &&
2465 test_bit(BIO_UPTODATE, &bio->bi_flags))
2466 uptodate = 1;
2467 } else {
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478 ret = bio_readpage_error(bio, offset, page, start, end,
2479 mirror);
2480 if (ret == 0) {
2481 uptodate =
2482 test_bit(BIO_UPTODATE, &bio->bi_flags);
2483 if (err)
2484 uptodate = 0;
2485 continue;
2486 }
2487 }
2488readpage_ok:
2489 if (likely(uptodate)) {
2490 loff_t i_size = i_size_read(inode);
2491 pgoff_t end_index = i_size >> PAGE_CACHE_SHIFT;
2492 unsigned offset;
2493
2494
2495 offset = i_size & (PAGE_CACHE_SIZE-1);
2496 if (page->index == end_index && offset)
2497 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
2498 SetPageUptodate(page);
2499 } else {
2500 ClearPageUptodate(page);
2501 SetPageError(page);
2502 }
2503 unlock_page(page);
2504 offset += len;
2505
2506 if (unlikely(!uptodate)) {
2507 if (extent_len) {
2508 endio_readpage_release_extent(tree,
2509 extent_start,
2510 extent_len, 1);
2511 extent_start = 0;
2512 extent_len = 0;
2513 }
2514 endio_readpage_release_extent(tree, start,
2515 end - start + 1, 0);
2516 } else if (!extent_len) {
2517 extent_start = start;
2518 extent_len = end + 1 - start;
2519 } else if (extent_start + extent_len == start) {
2520 extent_len += end + 1 - start;
2521 } else {
2522 endio_readpage_release_extent(tree, extent_start,
2523 extent_len, uptodate);
2524 extent_start = start;
2525 extent_len = end + 1 - start;
2526 }
2527 } while (bvec <= bvec_end);
2528
2529 if (extent_len)
2530 endio_readpage_release_extent(tree, extent_start, extent_len,
2531 uptodate);
2532 if (io_bio->end_io)
2533 io_bio->end_io(io_bio, err);
2534 bio_put(bio);
2535}
2536
2537
2538
2539
2540
2541struct bio *
2542btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2543 gfp_t gfp_flags)
2544{
2545 struct btrfs_io_bio *btrfs_bio;
2546 struct bio *bio;
2547
2548 bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
2549
2550 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
2551 while (!bio && (nr_vecs /= 2)) {
2552 bio = bio_alloc_bioset(gfp_flags,
2553 nr_vecs, btrfs_bioset);
2554 }
2555 }
2556
2557 if (bio) {
2558 bio->bi_size = 0;
2559 bio->bi_bdev = bdev;
2560 bio->bi_sector = first_sector;
2561 btrfs_bio = btrfs_io_bio(bio);
2562 btrfs_bio->csum = NULL;
2563 btrfs_bio->csum_allocated = NULL;
2564 btrfs_bio->end_io = NULL;
2565 }
2566 return bio;
2567}
2568
2569struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
2570{
2571 return bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2572}
2573
2574
2575
2576struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2577{
2578 struct btrfs_io_bio *btrfs_bio;
2579 struct bio *bio;
2580
2581 bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2582 if (bio) {
2583 btrfs_bio = btrfs_io_bio(bio);
2584 btrfs_bio->csum = NULL;
2585 btrfs_bio->csum_allocated = NULL;
2586 btrfs_bio->end_io = NULL;
2587 }
2588 return bio;
2589}
2590
2591
2592static int __must_check submit_one_bio(int rw, struct bio *bio,
2593 int mirror_num, unsigned long bio_flags)
2594{
2595 int ret = 0;
2596 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2597 struct page *page = bvec->bv_page;
2598 struct extent_io_tree *tree = bio->bi_private;
2599 u64 start;
2600
2601 start = page_offset(page) + bvec->bv_offset;
2602
2603 bio->bi_private = NULL;
2604
2605 bio_get(bio);
2606
2607 if (tree->ops && tree->ops->submit_bio_hook)
2608 ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
2609 mirror_num, bio_flags, start);
2610 else
2611 btrfsic_submit_bio(rw, bio);
2612
2613 if (bio_flagged(bio, BIO_EOPNOTSUPP))
2614 ret = -EOPNOTSUPP;
2615 bio_put(bio);
2616 return ret;
2617}
2618
2619static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
2620 unsigned long offset, size_t size, struct bio *bio,
2621 unsigned long bio_flags)
2622{
2623 int ret = 0;
2624 if (tree->ops && tree->ops->merge_bio_hook)
2625 ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
2626 bio_flags);
2627 BUG_ON(ret < 0);
2628 return ret;
2629
2630}
2631
2632static int submit_extent_page(int rw, struct extent_io_tree *tree,
2633 struct page *page, sector_t sector,
2634 size_t size, unsigned long offset,
2635 struct block_device *bdev,
2636 struct bio **bio_ret,
2637 unsigned long max_pages,
2638 bio_end_io_t end_io_func,
2639 int mirror_num,
2640 unsigned long prev_bio_flags,
2641 unsigned long bio_flags)
2642{
2643 int ret = 0;
2644 struct bio *bio;
2645 int nr;
2646 int contig = 0;
2647 int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
2648 int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
2649 size_t page_size = min_t(size_t, size, PAGE_CACHE_SIZE);
2650
2651 if (bio_ret && *bio_ret) {
2652 bio = *bio_ret;
2653 if (old_compressed)
2654 contig = bio->bi_sector == sector;
2655 else
2656 contig = bio_end_sector(bio) == sector;
2657
2658 if (prev_bio_flags != bio_flags || !contig ||
2659 merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
2660 bio_add_page(bio, page, page_size, offset) < page_size) {
2661 ret = submit_one_bio(rw, bio, mirror_num,
2662 prev_bio_flags);
2663 if (ret < 0)
2664 return ret;
2665 bio = NULL;
2666 } else {
2667 return 0;
2668 }
2669 }
2670 if (this_compressed)
2671 nr = BIO_MAX_PAGES;
2672 else
2673 nr = bio_get_nr_vecs(bdev);
2674
2675 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
2676 if (!bio)
2677 return -ENOMEM;
2678
2679 bio_add_page(bio, page, page_size, offset);
2680 bio->bi_end_io = end_io_func;
2681 bio->bi_private = tree;
2682
2683 if (bio_ret)
2684 *bio_ret = bio;
2685 else
2686 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
2687
2688 return ret;
2689}
2690
2691static void attach_extent_buffer_page(struct extent_buffer *eb,
2692 struct page *page)
2693{
2694 if (!PagePrivate(page)) {
2695 SetPagePrivate(page);
2696 page_cache_get(page);
2697 set_page_private(page, (unsigned long)eb);
2698 } else {
2699 WARN_ON(page->private != (unsigned long)eb);
2700 }
2701}
2702
2703void set_page_extent_mapped(struct page *page)
2704{
2705 if (!PagePrivate(page)) {
2706 SetPagePrivate(page);
2707 page_cache_get(page);
2708 set_page_private(page, EXTENT_PAGE_PRIVATE);
2709 }
2710}
2711
2712static struct extent_map *
2713__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2714 u64 start, u64 len, get_extent_t *get_extent,
2715 struct extent_map **em_cached)
2716{
2717 struct extent_map *em;
2718
2719 if (em_cached && *em_cached) {
2720 em = *em_cached;
2721 if (em->in_tree && start >= em->start &&
2722 start < extent_map_end(em)) {
2723 atomic_inc(&em->refs);
2724 return em;
2725 }
2726
2727 free_extent_map(em);
2728 *em_cached = NULL;
2729 }
2730
2731 em = get_extent(inode, page, pg_offset, start, len, 0);
2732 if (em_cached && !IS_ERR_OR_NULL(em)) {
2733 BUG_ON(*em_cached);
2734 atomic_inc(&em->refs);
2735 *em_cached = em;
2736 }
2737 return em;
2738}
2739
2740
2741
2742
2743
2744
2745static int __do_readpage(struct extent_io_tree *tree,
2746 struct page *page,
2747 get_extent_t *get_extent,
2748 struct extent_map **em_cached,
2749 struct bio **bio, int mirror_num,
2750 unsigned long *bio_flags, int rw)
2751{
2752 struct inode *inode = page->mapping->host;
2753 u64 start = page_offset(page);
2754 u64 page_end = start + PAGE_CACHE_SIZE - 1;
2755 u64 end;
2756 u64 cur = start;
2757 u64 extent_offset;
2758 u64 last_byte = i_size_read(inode);
2759 u64 block_start;
2760 u64 cur_end;
2761 sector_t sector;
2762 struct extent_map *em;
2763 struct block_device *bdev;
2764 int ret;
2765 int nr = 0;
2766 int parent_locked = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
2767 size_t pg_offset = 0;
2768 size_t iosize;
2769 size_t disk_io_size;
2770 size_t blocksize = inode->i_sb->s_blocksize;
2771 unsigned long this_bio_flag = *bio_flags & EXTENT_BIO_PARENT_LOCKED;
2772
2773 set_page_extent_mapped(page);
2774
2775 end = page_end;
2776 if (!PageUptodate(page)) {
2777 if (cleancache_get_page(page) == 0) {
2778 BUG_ON(blocksize != PAGE_SIZE);
2779 unlock_extent(tree, start, end);
2780 goto out;
2781 }
2782 }
2783
2784 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2785 char *userpage;
2786 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
2787
2788 if (zero_offset) {
2789 iosize = PAGE_CACHE_SIZE - zero_offset;
2790 userpage = kmap_atomic(page);
2791 memset(userpage + zero_offset, 0, iosize);
2792 flush_dcache_page(page);
2793 kunmap_atomic(userpage);
2794 }
2795 }
2796 while (cur <= end) {
2797 unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2798
2799 if (cur >= last_byte) {
2800 char *userpage;
2801 struct extent_state *cached = NULL;
2802
2803 iosize = PAGE_CACHE_SIZE - pg_offset;
2804 userpage = kmap_atomic(page);
2805 memset(userpage + pg_offset, 0, iosize);
2806 flush_dcache_page(page);
2807 kunmap_atomic(userpage);
2808 set_extent_uptodate(tree, cur, cur + iosize - 1,
2809 &cached, GFP_NOFS);
2810 if (!parent_locked)
2811 unlock_extent_cached(tree, cur,
2812 cur + iosize - 1,
2813 &cached, GFP_NOFS);
2814 break;
2815 }
2816 em = __get_extent_map(inode, page, pg_offset, cur,
2817 end - cur + 1, get_extent, em_cached);
2818 if (IS_ERR_OR_NULL(em)) {
2819 SetPageError(page);
2820 if (!parent_locked)
2821 unlock_extent(tree, cur, end);
2822 break;
2823 }
2824 extent_offset = cur - em->start;
2825 BUG_ON(extent_map_end(em) <= cur);
2826 BUG_ON(end < cur);
2827
2828 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2829 this_bio_flag |= EXTENT_BIO_COMPRESSED;
2830 extent_set_compress_type(&this_bio_flag,
2831 em->compress_type);
2832 }
2833
2834 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2835 cur_end = min(extent_map_end(em) - 1, end);
2836 iosize = ALIGN(iosize, blocksize);
2837 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2838 disk_io_size = em->block_len;
2839 sector = em->block_start >> 9;
2840 } else {
2841 sector = (em->block_start + extent_offset) >> 9;
2842 disk_io_size = iosize;
2843 }
2844 bdev = em->bdev;
2845 block_start = em->block_start;
2846 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2847 block_start = EXTENT_MAP_HOLE;
2848 free_extent_map(em);
2849 em = NULL;
2850
2851
2852 if (block_start == EXTENT_MAP_HOLE) {
2853 char *userpage;
2854 struct extent_state *cached = NULL;
2855
2856 userpage = kmap_atomic(page);
2857 memset(userpage + pg_offset, 0, iosize);
2858 flush_dcache_page(page);
2859 kunmap_atomic(userpage);
2860
2861 set_extent_uptodate(tree, cur, cur + iosize - 1,
2862 &cached, GFP_NOFS);
2863 unlock_extent_cached(tree, cur, cur + iosize - 1,
2864 &cached, GFP_NOFS);
2865 cur = cur + iosize;
2866 pg_offset += iosize;
2867 continue;
2868 }
2869
2870 if (test_range_bit(tree, cur, cur_end,
2871 EXTENT_UPTODATE, 1, NULL)) {
2872 check_page_uptodate(tree, page);
2873 if (!parent_locked)
2874 unlock_extent(tree, cur, cur + iosize - 1);
2875 cur = cur + iosize;
2876 pg_offset += iosize;
2877 continue;
2878 }
2879
2880
2881
2882 if (block_start == EXTENT_MAP_INLINE) {
2883 SetPageError(page);
2884 if (!parent_locked)
2885 unlock_extent(tree, cur, cur + iosize - 1);
2886 cur = cur + iosize;
2887 pg_offset += iosize;
2888 continue;
2889 }
2890
2891 pnr -= page->index;
2892 ret = submit_extent_page(rw, tree, page,
2893 sector, disk_io_size, pg_offset,
2894 bdev, bio, pnr,
2895 end_bio_extent_readpage, mirror_num,
2896 *bio_flags,
2897 this_bio_flag);
2898 if (!ret) {
2899 nr++;
2900 *bio_flags = this_bio_flag;
2901 } else {
2902 SetPageError(page);
2903 if (!parent_locked)
2904 unlock_extent(tree, cur, cur + iosize - 1);
2905 }
2906 cur = cur + iosize;
2907 pg_offset += iosize;
2908 }
2909out:
2910 if (!nr) {
2911 if (!PageError(page))
2912 SetPageUptodate(page);
2913 unlock_page(page);
2914 }
2915 return 0;
2916}
2917
2918static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
2919 struct page *pages[], int nr_pages,
2920 u64 start, u64 end,
2921 get_extent_t *get_extent,
2922 struct extent_map **em_cached,
2923 struct bio **bio, int mirror_num,
2924 unsigned long *bio_flags, int rw)
2925{
2926 struct inode *inode;
2927 struct btrfs_ordered_extent *ordered;
2928 int index;
2929
2930 inode = pages[0]->mapping->host;
2931 while (1) {
2932 lock_extent(tree, start, end);
2933 ordered = btrfs_lookup_ordered_range(inode, start,
2934 end - start + 1);
2935 if (!ordered)
2936 break;
2937 unlock_extent(tree, start, end);
2938 btrfs_start_ordered_extent(inode, ordered, 1);
2939 btrfs_put_ordered_extent(ordered);
2940 }
2941
2942 for (index = 0; index < nr_pages; index++) {
2943 __do_readpage(tree, pages[index], get_extent, em_cached, bio,
2944 mirror_num, bio_flags, rw);
2945 page_cache_release(pages[index]);
2946 }
2947}
2948
2949static void __extent_readpages(struct extent_io_tree *tree,
2950 struct page *pages[],
2951 int nr_pages, get_extent_t *get_extent,
2952 struct extent_map **em_cached,
2953 struct bio **bio, int mirror_num,
2954 unsigned long *bio_flags, int rw)
2955{
2956 u64 start = 0;
2957 u64 end = 0;
2958 u64 page_start;
2959 int index;
2960 int first_index = 0;
2961
2962 for (index = 0; index < nr_pages; index++) {
2963 page_start = page_offset(pages[index]);
2964 if (!end) {
2965 start = page_start;
2966 end = start + PAGE_CACHE_SIZE - 1;
2967 first_index = index;
2968 } else if (end + 1 == page_start) {
2969 end += PAGE_CACHE_SIZE;
2970 } else {
2971 __do_contiguous_readpages(tree, &pages[first_index],
2972 index - first_index, start,
2973 end, get_extent, em_cached,
2974 bio, mirror_num, bio_flags,
2975 rw);
2976 start = page_start;
2977 end = start + PAGE_CACHE_SIZE - 1;
2978 first_index = index;
2979 }
2980 }
2981
2982 if (end)
2983 __do_contiguous_readpages(tree, &pages[first_index],
2984 index - first_index, start,
2985 end, get_extent, em_cached, bio,
2986 mirror_num, bio_flags, rw);
2987}
2988
2989static int __extent_read_full_page(struct extent_io_tree *tree,
2990 struct page *page,
2991 get_extent_t *get_extent,
2992 struct bio **bio, int mirror_num,
2993 unsigned long *bio_flags, int rw)
2994{
2995 struct inode *inode = page->mapping->host;
2996 struct btrfs_ordered_extent *ordered;
2997 u64 start = page_offset(page);
2998 u64 end = start + PAGE_CACHE_SIZE - 1;
2999 int ret;
3000
3001 while (1) {
3002 lock_extent(tree, start, end);
3003 ordered = btrfs_lookup_ordered_extent(inode, start);
3004 if (!ordered)
3005 break;
3006 unlock_extent(tree, start, end);
3007 btrfs_start_ordered_extent(inode, ordered, 1);
3008 btrfs_put_ordered_extent(ordered);
3009 }
3010
3011 ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
3012 bio_flags, rw);
3013 return ret;
3014}
3015
3016int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
3017 get_extent_t *get_extent, int mirror_num)
3018{
3019 struct bio *bio = NULL;
3020 unsigned long bio_flags = 0;
3021 int ret;
3022
3023 ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
3024 &bio_flags, READ);
3025 if (bio)
3026 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3027 return ret;
3028}
3029
3030int extent_read_full_page_nolock(struct extent_io_tree *tree, struct page *page,
3031 get_extent_t *get_extent, int mirror_num)
3032{
3033 struct bio *bio = NULL;
3034 unsigned long bio_flags = EXTENT_BIO_PARENT_LOCKED;
3035 int ret;
3036
3037 ret = __do_readpage(tree, page, get_extent, NULL, &bio, mirror_num,
3038 &bio_flags, READ);
3039 if (bio)
3040 ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3041 return ret;
3042}
3043
3044static noinline void update_nr_written(struct page *page,
3045 struct writeback_control *wbc,
3046 unsigned long nr_written)
3047{
3048 wbc->nr_to_write -= nr_written;
3049 if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
3050 wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
3051 page->mapping->writeback_index = page->index + nr_written;
3052}
3053
3054
3055
3056
3057
3058
3059
3060static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3061 void *data)
3062{
3063 struct inode *inode = page->mapping->host;
3064 struct extent_page_data *epd = data;
3065 struct extent_io_tree *tree = epd->tree;
3066 u64 start = page_offset(page);
3067 u64 delalloc_start;
3068 u64 page_end = start + PAGE_CACHE_SIZE - 1;
3069 u64 end;
3070 u64 cur = start;
3071 u64 extent_offset;
3072 u64 last_byte = i_size_read(inode);
3073 u64 block_start;
3074 u64 iosize;
3075 sector_t sector;
3076 struct extent_state *cached_state = NULL;
3077 struct extent_map *em;
3078 struct block_device *bdev;
3079 int ret;
3080 int nr = 0;
3081 size_t pg_offset = 0;
3082 size_t blocksize;
3083 loff_t i_size = i_size_read(inode);
3084 unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
3085 u64 nr_delalloc;
3086 u64 delalloc_end;
3087 int page_started;
3088 int compressed;
3089 int write_flags;
3090 unsigned long nr_written = 0;
3091 bool fill_delalloc = true;
3092
3093 if (wbc->sync_mode == WB_SYNC_ALL)
3094 write_flags = WRITE_SYNC;
3095 else
3096 write_flags = WRITE;
3097
3098 trace___extent_writepage(page, inode, wbc);
3099
3100 WARN_ON(!PageLocked(page));
3101
3102 ClearPageError(page);
3103
3104 pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
3105 if (page->index > end_index ||
3106 (page->index == end_index && !pg_offset)) {
3107 page->mapping->a_ops->invalidatepage(page, 0, PAGE_CACHE_SIZE);
3108 unlock_page(page);
3109 return 0;
3110 }
3111
3112 if (page->index == end_index) {
3113 char *userpage;
3114
3115 userpage = kmap_atomic(page);
3116 memset(userpage + pg_offset, 0,
3117 PAGE_CACHE_SIZE - pg_offset);
3118 kunmap_atomic(userpage);
3119 flush_dcache_page(page);
3120 }
3121 pg_offset = 0;
3122
3123 set_page_extent_mapped(page);
3124
3125 if (!tree->ops || !tree->ops->fill_delalloc)
3126 fill_delalloc = false;
3127
3128 delalloc_start = start;
3129 delalloc_end = 0;
3130 page_started = 0;
3131 if (!epd->extent_locked && fill_delalloc) {
3132 u64 delalloc_to_write = 0;
3133
3134
3135
3136
3137 update_nr_written(page, wbc, 0);
3138
3139 while (delalloc_end < page_end) {
3140 nr_delalloc = find_lock_delalloc_range(inode, tree,
3141 page,
3142 &delalloc_start,
3143 &delalloc_end,
3144 128 * 1024 * 1024);
3145 if (nr_delalloc == 0) {
3146 delalloc_start = delalloc_end + 1;
3147 continue;
3148 }
3149 ret = tree->ops->fill_delalloc(inode, page,
3150 delalloc_start,
3151 delalloc_end,
3152 &page_started,
3153 &nr_written);
3154
3155 if (ret) {
3156 SetPageError(page);
3157 goto done;
3158 }
3159
3160
3161
3162
3163
3164 delalloc_to_write += (delalloc_end - delalloc_start +
3165 PAGE_CACHE_SIZE) >>
3166 PAGE_CACHE_SHIFT;
3167 delalloc_start = delalloc_end + 1;
3168 }
3169 if (wbc->nr_to_write < delalloc_to_write) {
3170 int thresh = 8192;
3171
3172 if (delalloc_to_write < thresh * 2)
3173 thresh = delalloc_to_write;
3174 wbc->nr_to_write = min_t(u64, delalloc_to_write,
3175 thresh);
3176 }
3177
3178
3179
3180
3181 if (page_started) {
3182 ret = 0;
3183
3184
3185
3186
3187
3188 wbc->nr_to_write -= nr_written;
3189 goto done_unlocked;
3190 }
3191 }
3192 if (tree->ops && tree->ops->writepage_start_hook) {
3193 ret = tree->ops->writepage_start_hook(page, start,
3194 page_end);
3195 if (ret) {
3196
3197 if (ret == -EBUSY)
3198 wbc->pages_skipped++;
3199 else
3200 redirty_page_for_writepage(wbc, page);
3201 update_nr_written(page, wbc, nr_written);
3202 unlock_page(page);
3203 ret = 0;
3204 goto done_unlocked;
3205 }
3206 }
3207
3208
3209
3210
3211
3212 update_nr_written(page, wbc, nr_written + 1);
3213
3214 end = page_end;
3215 if (last_byte <= start) {
3216 if (tree->ops && tree->ops->writepage_end_io_hook)
3217 tree->ops->writepage_end_io_hook(page, start,
3218 page_end, NULL, 1);
3219 goto done;
3220 }
3221
3222 blocksize = inode->i_sb->s_blocksize;
3223
3224 while (cur <= end) {
3225 if (cur >= last_byte) {
3226 if (tree->ops && tree->ops->writepage_end_io_hook)
3227 tree->ops->writepage_end_io_hook(page, cur,
3228 page_end, NULL, 1);
3229 break;
3230 }
3231 em = epd->get_extent(inode, page, pg_offset, cur,
3232 end - cur + 1, 1);
3233 if (IS_ERR_OR_NULL(em)) {
3234 SetPageError(page);
3235 break;
3236 }
3237
3238 extent_offset = cur - em->start;
3239 BUG_ON(extent_map_end(em) <= cur);
3240 BUG_ON(end < cur);
3241 iosize = min(extent_map_end(em) - cur, end - cur + 1);
3242 iosize = ALIGN(iosize, blocksize);
3243 sector = (em->block_start + extent_offset) >> 9;
3244 bdev = em->bdev;
3245 block_start = em->block_start;
3246 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
3247 free_extent_map(em);
3248 em = NULL;
3249
3250
3251
3252
3253
3254 if (compressed || block_start == EXTENT_MAP_HOLE ||
3255 block_start == EXTENT_MAP_INLINE) {
3256
3257
3258
3259
3260 if (!compressed && tree->ops &&
3261 tree->ops->writepage_end_io_hook)
3262 tree->ops->writepage_end_io_hook(page, cur,
3263 cur + iosize - 1,
3264 NULL, 1);
3265 else if (compressed) {
3266
3267
3268
3269
3270 nr++;
3271 }
3272
3273 cur += iosize;
3274 pg_offset += iosize;
3275 continue;
3276 }
3277
3278 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
3279 EXTENT_DIRTY, 0, NULL)) {
3280 cur = cur + iosize;
3281 pg_offset += iosize;
3282 continue;
3283 }
3284
3285 if (tree->ops && tree->ops->writepage_io_hook) {
3286 ret = tree->ops->writepage_io_hook(page, cur,
3287 cur + iosize - 1);
3288 } else {
3289 ret = 0;
3290 }
3291 if (ret) {
3292 SetPageError(page);
3293 } else {
3294 unsigned long max_nr = end_index + 1;
3295
3296 set_range_writeback(tree, cur, cur + iosize - 1);
3297 if (!PageWriteback(page)) {
3298 printk(KERN_ERR "btrfs warning page %lu not "
3299 "writeback, cur %llu end %llu\n",
3300 page->index, cur, end);
3301 }
3302
3303 ret = submit_extent_page(write_flags, tree, page,
3304 sector, iosize, pg_offset,
3305 bdev, &epd->bio, max_nr,
3306 end_bio_extent_writepage,
3307 0, 0, 0);
3308 if (ret)
3309 SetPageError(page);
3310 }
3311 cur = cur + iosize;
3312 pg_offset += iosize;
3313 nr++;
3314 }
3315done:
3316 if (nr == 0) {
3317
3318 set_page_writeback(page);
3319 end_page_writeback(page);
3320 }
3321 unlock_page(page);
3322
3323done_unlocked:
3324
3325
3326 free_extent_state(cached_state);
3327 return 0;
3328}
3329
3330static int eb_wait(void *word)
3331{
3332 io_schedule();
3333 return 0;
3334}
3335
3336void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3337{
3338 wait_on_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK, eb_wait,
3339 TASK_UNINTERRUPTIBLE);
3340}
3341
3342static int lock_extent_buffer_for_io(struct extent_buffer *eb,
3343 struct btrfs_fs_info *fs_info,
3344 struct extent_page_data *epd)
3345{
3346 unsigned long i, num_pages;
3347 int flush = 0;
3348 int ret = 0;
3349
3350 if (!btrfs_try_tree_write_lock(eb)) {
3351 flush = 1;
3352 flush_write_bio(epd);
3353 btrfs_tree_lock(eb);
3354 }
3355
3356 if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3357 btrfs_tree_unlock(eb);
3358 if (!epd->sync_io)
3359 return 0;
3360 if (!flush) {
3361 flush_write_bio(epd);
3362 flush = 1;
3363 }
3364 while (1) {
3365 wait_on_extent_buffer_writeback(eb);
3366 btrfs_tree_lock(eb);
3367 if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3368 break;
3369 btrfs_tree_unlock(eb);
3370 }
3371 }
3372
3373
3374
3375
3376
3377
3378 spin_lock(&eb->refs_lock);
3379 if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3380 set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3381 spin_unlock(&eb->refs_lock);
3382 btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3383 __percpu_counter_add(&fs_info->dirty_metadata_bytes,
3384 -eb->len,
3385 fs_info->dirty_metadata_batch);
3386 ret = 1;
3387 } else {
3388 spin_unlock(&eb->refs_lock);
3389 }
3390
3391 btrfs_tree_unlock(eb);
3392
3393 if (!ret)
3394 return ret;
3395
3396 num_pages = num_extent_pages(eb->start, eb->len);
3397 for (i = 0; i < num_pages; i++) {
3398 struct page *p = extent_buffer_page(eb, i);
3399
3400 if (!trylock_page(p)) {
3401 if (!flush) {
3402 flush_write_bio(epd);
3403 flush = 1;
3404 }
3405 lock_page(p);
3406 }
3407 }
3408
3409 return ret;
3410}
3411
3412static void end_extent_buffer_writeback(struct extent_buffer *eb)
3413{
3414 clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3415 smp_mb__after_clear_bit();
3416 wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3417}
3418
3419static void end_bio_extent_buffer_writepage(struct bio *bio, int err)
3420{
3421 int uptodate = err == 0;
3422 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
3423 struct extent_buffer *eb;
3424 int done;
3425
3426 do {
3427 struct page *page = bvec->bv_page;
3428
3429 bvec--;
3430 eb = (struct extent_buffer *)page->private;
3431 BUG_ON(!eb);
3432 done = atomic_dec_and_test(&eb->io_pages);
3433
3434 if (!uptodate || test_bit(EXTENT_BUFFER_IOERR, &eb->bflags)) {
3435 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3436 ClearPageUptodate(page);
3437 SetPageError(page);
3438 }
3439
3440 end_page_writeback(page);
3441
3442 if (!done)
3443 continue;
3444
3445 end_extent_buffer_writeback(eb);
3446 } while (bvec >= bio->bi_io_vec);
3447
3448 bio_put(bio);
3449
3450}
3451
3452static int write_one_eb(struct extent_buffer *eb,
3453 struct btrfs_fs_info *fs_info,
3454 struct writeback_control *wbc,
3455 struct extent_page_data *epd)
3456{
3457 struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3458 u64 offset = eb->start;
3459 unsigned long i, num_pages;
3460 unsigned long bio_flags = 0;
3461 int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
3462 int ret = 0;
3463
3464 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3465 num_pages = num_extent_pages(eb->start, eb->len);
3466 atomic_set(&eb->io_pages, num_pages);
3467 if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3468 bio_flags = EXTENT_BIO_TREE_LOG;
3469
3470 for (i = 0; i < num_pages; i++) {
3471 struct page *p = extent_buffer_page(eb, i);
3472
3473 clear_page_dirty_for_io(p);
3474 set_page_writeback(p);
3475 ret = submit_extent_page(rw, eb->tree, p, offset >> 9,
3476 PAGE_CACHE_SIZE, 0, bdev, &epd->bio,
3477 -1, end_bio_extent_buffer_writepage,
3478 0, epd->bio_flags, bio_flags);
3479 epd->bio_flags = bio_flags;
3480 if (ret) {
3481 set_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
3482 SetPageError(p);
3483 if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3484 end_extent_buffer_writeback(eb);
3485 ret = -EIO;
3486 break;
3487 }
3488 offset += PAGE_CACHE_SIZE;
3489 update_nr_written(p, wbc, 1);
3490 unlock_page(p);
3491 }
3492
3493 if (unlikely(ret)) {
3494 for (; i < num_pages; i++) {
3495 struct page *p = extent_buffer_page(eb, i);
3496 unlock_page(p);
3497 }
3498 }
3499
3500 return ret;
3501}
3502
3503int btree_write_cache_pages(struct address_space *mapping,
3504 struct writeback_control *wbc)
3505{
3506 struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3507 struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3508 struct extent_buffer *eb, *prev_eb = NULL;
3509 struct extent_page_data epd = {
3510 .bio = NULL,
3511 .tree = tree,
3512 .extent_locked = 0,
3513 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3514 .bio_flags = 0,
3515 };
3516 int ret = 0;
3517 int done = 0;
3518 int nr_to_write_done = 0;
3519 struct pagevec pvec;
3520 int nr_pages;
3521 pgoff_t index;
3522 pgoff_t end;
3523 int scanned = 0;
3524 int tag;
3525
3526 pagevec_init(&pvec, 0);
3527 if (wbc->range_cyclic) {
3528 index = mapping->writeback_index;
3529 end = -1;
3530 } else {
3531 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3532 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3533 scanned = 1;
3534 }
3535 if (wbc->sync_mode == WB_SYNC_ALL)
3536 tag = PAGECACHE_TAG_TOWRITE;
3537 else
3538 tag = PAGECACHE_TAG_DIRTY;
3539retry:
3540 if (wbc->sync_mode == WB_SYNC_ALL)
3541 tag_pages_for_writeback(mapping, index, end);
3542 while (!done && !nr_to_write_done && (index <= end) &&
3543 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3544 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3545 unsigned i;
3546
3547 scanned = 1;
3548 for (i = 0; i < nr_pages; i++) {
3549 struct page *page = pvec.pages[i];
3550
3551 if (!PagePrivate(page))
3552 continue;
3553
3554 if (!wbc->range_cyclic && page->index > end) {
3555 done = 1;
3556 break;
3557 }
3558
3559 spin_lock(&mapping->private_lock);
3560 if (!PagePrivate(page)) {
3561 spin_unlock(&mapping->private_lock);
3562 continue;
3563 }
3564
3565 eb = (struct extent_buffer *)page->private;
3566
3567
3568
3569
3570
3571
3572 if (!eb) {
3573 spin_unlock(&mapping->private_lock);
3574 WARN_ON(1);
3575 continue;
3576 }
3577
3578 if (eb == prev_eb) {
3579 spin_unlock(&mapping->private_lock);
3580 continue;
3581 }
3582
3583 ret = atomic_inc_not_zero(&eb->refs);
3584 spin_unlock(&mapping->private_lock);
3585 if (!ret)
3586 continue;
3587
3588 prev_eb = eb;
3589 ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3590 if (!ret) {
3591 free_extent_buffer(eb);
3592 continue;
3593 }
3594
3595 ret = write_one_eb(eb, fs_info, wbc, &epd);
3596 if (ret) {
3597 done = 1;
3598 free_extent_buffer(eb);
3599 break;
3600 }
3601 free_extent_buffer(eb);
3602
3603
3604
3605
3606
3607
3608 nr_to_write_done = wbc->nr_to_write <= 0;
3609 }
3610 pagevec_release(&pvec);
3611 cond_resched();
3612 }
3613 if (!scanned && !done) {
3614
3615
3616
3617
3618 scanned = 1;
3619 index = 0;
3620 goto retry;
3621 }
3622 flush_write_bio(&epd);
3623 return ret;
3624}
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641static int extent_write_cache_pages(struct extent_io_tree *tree,
3642 struct address_space *mapping,
3643 struct writeback_control *wbc,
3644 writepage_t writepage, void *data,
3645 void (*flush_fn)(void *))
3646{
3647 struct inode *inode = mapping->host;
3648 int ret = 0;
3649 int done = 0;
3650 int nr_to_write_done = 0;
3651 struct pagevec pvec;
3652 int nr_pages;
3653 pgoff_t index;
3654 pgoff_t end;
3655 int scanned = 0;
3656 int tag;
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667 if (!igrab(inode))
3668 return 0;
3669
3670 pagevec_init(&pvec, 0);
3671 if (wbc->range_cyclic) {
3672 index = mapping->writeback_index;
3673 end = -1;
3674 } else {
3675 index = wbc->range_start >> PAGE_CACHE_SHIFT;
3676 end = wbc->range_end >> PAGE_CACHE_SHIFT;
3677 scanned = 1;
3678 }
3679 if (wbc->sync_mode == WB_SYNC_ALL)
3680 tag = PAGECACHE_TAG_TOWRITE;
3681 else
3682 tag = PAGECACHE_TAG_DIRTY;
3683retry:
3684 if (wbc->sync_mode == WB_SYNC_ALL)
3685 tag_pages_for_writeback(mapping, index, end);
3686 while (!done && !nr_to_write_done && (index <= end) &&
3687 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3688 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3689 unsigned i;
3690
3691 scanned = 1;
3692 for (i = 0; i < nr_pages; i++) {
3693 struct page *page = pvec.pages[i];
3694
3695
3696
3697
3698
3699
3700
3701
3702 if (!trylock_page(page)) {
3703 flush_fn(data);
3704 lock_page(page);
3705 }
3706
3707 if (unlikely(page->mapping != mapping)) {
3708 unlock_page(page);
3709 continue;
3710 }
3711
3712 if (!wbc->range_cyclic && page->index > end) {
3713 done = 1;
3714 unlock_page(page);
3715 continue;
3716 }
3717
3718 if (wbc->sync_mode != WB_SYNC_NONE) {
3719 if (PageWriteback(page))
3720 flush_fn(data);
3721 wait_on_page_writeback(page);
3722 }
3723
3724 if (PageWriteback(page) ||
3725 !clear_page_dirty_for_io(page)) {
3726 unlock_page(page);
3727 continue;
3728 }
3729
3730 ret = (*writepage)(page, wbc, data);
3731
3732 if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
3733 unlock_page(page);
3734 ret = 0;
3735 }
3736 if (ret)
3737 done = 1;
3738
3739
3740
3741
3742
3743
3744 nr_to_write_done = wbc->nr_to_write <= 0;
3745 }
3746 pagevec_release(&pvec);
3747 cond_resched();
3748 }
3749 if (!scanned && !done) {
3750
3751
3752
3753
3754 scanned = 1;
3755 index = 0;
3756 goto retry;
3757 }
3758 btrfs_add_delayed_iput(inode);
3759 return ret;
3760}
3761
3762static void flush_epd_write_bio(struct extent_page_data *epd)
3763{
3764 if (epd->bio) {
3765 int rw = WRITE;
3766 int ret;
3767
3768 if (epd->sync_io)
3769 rw = WRITE_SYNC;
3770
3771 ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
3772 BUG_ON(ret < 0);
3773 epd->bio = NULL;
3774 }
3775}
3776
3777static noinline void flush_write_bio(void *data)
3778{
3779 struct extent_page_data *epd = data;
3780 flush_epd_write_bio(epd);
3781}
3782
3783int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
3784 get_extent_t *get_extent,
3785 struct writeback_control *wbc)
3786{
3787 int ret;
3788 struct extent_page_data epd = {
3789 .bio = NULL,
3790 .tree = tree,
3791 .get_extent = get_extent,
3792 .extent_locked = 0,
3793 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3794 .bio_flags = 0,
3795 };
3796
3797 ret = __extent_writepage(page, wbc, &epd);
3798
3799 flush_epd_write_bio(&epd);
3800 return ret;
3801}
3802
3803int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
3804 u64 start, u64 end, get_extent_t *get_extent,
3805 int mode)
3806{
3807 int ret = 0;
3808 struct address_space *mapping = inode->i_mapping;
3809 struct page *page;
3810 unsigned long nr_pages = (end - start + PAGE_CACHE_SIZE) >>
3811 PAGE_CACHE_SHIFT;
3812
3813 struct extent_page_data epd = {
3814 .bio = NULL,
3815 .tree = tree,
3816 .get_extent = get_extent,
3817 .extent_locked = 1,
3818 .sync_io = mode == WB_SYNC_ALL,
3819 .bio_flags = 0,
3820 };
3821 struct writeback_control wbc_writepages = {
3822 .sync_mode = mode,
3823 .nr_to_write = nr_pages * 2,
3824 .range_start = start,
3825 .range_end = end + 1,
3826 };
3827
3828 while (start <= end) {
3829 page = find_get_page(mapping, start >> PAGE_CACHE_SHIFT);
3830 if (clear_page_dirty_for_io(page))
3831 ret = __extent_writepage(page, &wbc_writepages, &epd);
3832 else {
3833 if (tree->ops && tree->ops->writepage_end_io_hook)
3834 tree->ops->writepage_end_io_hook(page, start,
3835 start + PAGE_CACHE_SIZE - 1,
3836 NULL, 1);
3837 unlock_page(page);
3838 }
3839 page_cache_release(page);
3840 start += PAGE_CACHE_SIZE;
3841 }
3842
3843 flush_epd_write_bio(&epd);
3844 return ret;
3845}
3846
3847int extent_writepages(struct extent_io_tree *tree,
3848 struct address_space *mapping,
3849 get_extent_t *get_extent,
3850 struct writeback_control *wbc)
3851{
3852 int ret = 0;
3853 struct extent_page_data epd = {
3854 .bio = NULL,
3855 .tree = tree,
3856 .get_extent = get_extent,
3857 .extent_locked = 0,
3858 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
3859 .bio_flags = 0,
3860 };
3861
3862 ret = extent_write_cache_pages(tree, mapping, wbc,
3863 __extent_writepage, &epd,
3864 flush_write_bio);
3865 flush_epd_write_bio(&epd);
3866 return ret;
3867}
3868
3869int extent_readpages(struct extent_io_tree *tree,
3870 struct address_space *mapping,
3871 struct list_head *pages, unsigned nr_pages,
3872 get_extent_t get_extent)
3873{
3874 struct bio *bio = NULL;
3875 unsigned page_idx;
3876 unsigned long bio_flags = 0;
3877 struct page *pagepool[16];
3878 struct page *page;
3879 struct extent_map *em_cached = NULL;
3880 int nr = 0;
3881
3882 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
3883 page = list_entry(pages->prev, struct page, lru);
3884
3885 prefetchw(&page->flags);
3886 list_del(&page->lru);
3887 if (add_to_page_cache_lru(page, mapping,
3888 page->index, GFP_NOFS)) {
3889 page_cache_release(page);
3890 continue;
3891 }
3892
3893 pagepool[nr++] = page;
3894 if (nr < ARRAY_SIZE(pagepool))
3895 continue;
3896 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
3897 &bio, 0, &bio_flags, READ);
3898 nr = 0;
3899 }
3900 if (nr)
3901 __extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
3902 &bio, 0, &bio_flags, READ);
3903
3904 if (em_cached)
3905 free_extent_map(em_cached);
3906
3907 BUG_ON(!list_empty(pages));
3908 if (bio)
3909 return submit_one_bio(READ, bio, 0, bio_flags);
3910 return 0;
3911}
3912
3913
3914
3915
3916
3917
3918int extent_invalidatepage(struct extent_io_tree *tree,
3919 struct page *page, unsigned long offset)
3920{
3921 struct extent_state *cached_state = NULL;
3922 u64 start = page_offset(page);
3923 u64 end = start + PAGE_CACHE_SIZE - 1;
3924 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
3925
3926 start += ALIGN(offset, blocksize);
3927 if (start > end)
3928 return 0;
3929
3930 lock_extent_bits(tree, start, end, 0, &cached_state);
3931 wait_on_page_writeback(page);
3932 clear_extent_bit(tree, start, end,
3933 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
3934 EXTENT_DO_ACCOUNTING,
3935 1, 1, &cached_state, GFP_NOFS);
3936 return 0;
3937}
3938
3939
3940
3941
3942
3943
3944static int try_release_extent_state(struct extent_map_tree *map,
3945 struct extent_io_tree *tree,
3946 struct page *page, gfp_t mask)
3947{
3948 u64 start = page_offset(page);
3949 u64 end = start + PAGE_CACHE_SIZE - 1;
3950 int ret = 1;
3951
3952 if (test_range_bit(tree, start, end,
3953 EXTENT_IOBITS, 0, NULL))
3954 ret = 0;
3955 else {
3956 if ((mask & GFP_NOFS) == GFP_NOFS)
3957 mask = GFP_NOFS;
3958
3959
3960
3961
3962 ret = clear_extent_bit(tree, start, end,
3963 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
3964 0, 0, NULL, mask);
3965
3966
3967
3968
3969 if (ret < 0)
3970 ret = 0;
3971 else
3972 ret = 1;
3973 }
3974 return ret;
3975}
3976
3977
3978
3979
3980
3981
3982int try_release_extent_mapping(struct extent_map_tree *map,
3983 struct extent_io_tree *tree, struct page *page,
3984 gfp_t mask)
3985{
3986 struct extent_map *em;
3987 u64 start = page_offset(page);
3988 u64 end = start + PAGE_CACHE_SIZE - 1;
3989
3990 if ((mask & __GFP_WAIT) &&
3991 page->mapping->host->i_size > 16 * 1024 * 1024) {
3992 u64 len;
3993 while (start <= end) {
3994 len = end - start + 1;
3995 write_lock(&map->lock);
3996 em = lookup_extent_mapping(map, start, len);
3997 if (!em) {
3998 write_unlock(&map->lock);
3999 break;
4000 }
4001 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4002 em->start != start) {
4003 write_unlock(&map->lock);
4004 free_extent_map(em);
4005 break;
4006 }
4007 if (!test_range_bit(tree, em->start,
4008 extent_map_end(em) - 1,
4009 EXTENT_LOCKED | EXTENT_WRITEBACK,
4010 0, NULL)) {
4011 remove_extent_mapping(map, em);
4012
4013 free_extent_map(em);
4014 }
4015 start = extent_map_end(em);
4016 write_unlock(&map->lock);
4017
4018
4019 free_extent_map(em);
4020 }
4021 }
4022 return try_release_extent_state(map, tree, page, mask);
4023}
4024
4025
4026
4027
4028
4029static struct extent_map *get_extent_skip_holes(struct inode *inode,
4030 u64 offset,
4031 u64 last,
4032 get_extent_t *get_extent)
4033{
4034 u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4035 struct extent_map *em;
4036 u64 len;
4037
4038 if (offset >= last)
4039 return NULL;
4040
4041 while(1) {
4042 len = last - offset;
4043 if (len == 0)
4044 break;
4045 len = ALIGN(len, sectorsize);
4046 em = get_extent(inode, NULL, 0, offset, len, 0);
4047 if (IS_ERR_OR_NULL(em))
4048 return em;
4049
4050
4051 if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4052 em->block_start != EXTENT_MAP_HOLE) {
4053 return em;
4054 }
4055
4056
4057 offset = extent_map_end(em);
4058 free_extent_map(em);
4059 if (offset >= last)
4060 break;
4061 }
4062 return NULL;
4063}
4064
4065int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4066 __u64 start, __u64 len, get_extent_t *get_extent)
4067{
4068 int ret = 0;
4069 u64 off = start;
4070 u64 max = start + len;
4071 u32 flags = 0;
4072 u32 found_type;
4073 u64 last;
4074 u64 last_for_get_extent = 0;
4075 u64 disko = 0;
4076 u64 isize = i_size_read(inode);
4077 struct btrfs_key found_key;
4078 struct extent_map *em = NULL;
4079 struct extent_state *cached_state = NULL;
4080 struct btrfs_path *path;
4081 struct btrfs_file_extent_item *item;
4082 int end = 0;
4083 u64 em_start = 0;
4084 u64 em_len = 0;
4085 u64 em_end = 0;
4086 unsigned long emflags;
4087
4088 if (len == 0)
4089 return -EINVAL;
4090
4091 path = btrfs_alloc_path();
4092 if (!path)
4093 return -ENOMEM;
4094 path->leave_spinning = 1;
4095
4096 start = ALIGN(start, BTRFS_I(inode)->root->sectorsize);
4097 len = ALIGN(len, BTRFS_I(inode)->root->sectorsize);
4098
4099
4100
4101
4102
4103 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
4104 path, btrfs_ino(inode), -1, 0);
4105 if (ret < 0) {
4106 btrfs_free_path(path);
4107 return ret;
4108 }
4109 WARN_ON(!ret);
4110 path->slots[0]--;
4111 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
4112 struct btrfs_file_extent_item);
4113 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
4114 found_type = btrfs_key_type(&found_key);
4115
4116
4117 if (found_key.objectid != btrfs_ino(inode) ||
4118 found_type != BTRFS_EXTENT_DATA_KEY) {
4119
4120 last = (u64)-1;
4121 last_for_get_extent = isize;
4122 } else {
4123
4124
4125
4126
4127
4128 last = found_key.offset;
4129 last_for_get_extent = last + 1;
4130 }
4131 btrfs_free_path(path);
4132
4133
4134
4135
4136
4137
4138 if (last < isize) {
4139 last = (u64)-1;
4140 last_for_get_extent = isize;
4141 }
4142
4143 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1, 0,
4144 &cached_state);
4145
4146 em = get_extent_skip_holes(inode, start, last_for_get_extent,
4147 get_extent);
4148 if (!em)
4149 goto out;
4150 if (IS_ERR(em)) {
4151 ret = PTR_ERR(em);
4152 goto out;
4153 }
4154
4155 while (!end) {
4156 u64 offset_in_extent = 0;
4157
4158
4159 if (em->start >= max || extent_map_end(em) < off)
4160 break;
4161
4162
4163
4164
4165
4166
4167
4168 em_start = max(em->start, off);
4169
4170
4171
4172
4173
4174
4175
4176 if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4177 offset_in_extent = em_start - em->start;
4178 em_end = extent_map_end(em);
4179 em_len = em_end - em_start;
4180 emflags = em->flags;
4181 disko = 0;
4182 flags = 0;
4183
4184
4185
4186
4187 off = extent_map_end(em);
4188 if (off >= max)
4189 end = 1;
4190
4191 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
4192 end = 1;
4193 flags |= FIEMAP_EXTENT_LAST;
4194 } else if (em->block_start == EXTENT_MAP_INLINE) {
4195 flags |= (FIEMAP_EXTENT_DATA_INLINE |
4196 FIEMAP_EXTENT_NOT_ALIGNED);
4197 } else if (em->block_start == EXTENT_MAP_DELALLOC) {
4198 flags |= (FIEMAP_EXTENT_DELALLOC |
4199 FIEMAP_EXTENT_UNKNOWN);
4200 } else {
4201 disko = em->block_start + offset_in_extent;
4202 }
4203 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4204 flags |= FIEMAP_EXTENT_ENCODED;
4205
4206 free_extent_map(em);
4207 em = NULL;
4208 if ((em_start >= last) || em_len == (u64)-1 ||
4209 (last == (u64)-1 && isize <= em_end)) {
4210 flags |= FIEMAP_EXTENT_LAST;
4211 end = 1;
4212 }
4213
4214
4215 em = get_extent_skip_holes(inode, off, last_for_get_extent,
4216 get_extent);
4217 if (IS_ERR(em)) {
4218 ret = PTR_ERR(em);
4219 goto out;
4220 }
4221 if (!em) {
4222 flags |= FIEMAP_EXTENT_LAST;
4223 end = 1;
4224 }
4225 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4226 em_len, flags);
4227 if (ret)
4228 goto out_free;
4229 }
4230out_free:
4231 free_extent_map(em);
4232out:
4233 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
4234 &cached_state, GFP_NOFS);
4235 return ret;
4236}
4237
4238static void __free_extent_buffer(struct extent_buffer *eb)
4239{
4240 btrfs_leak_debug_del(&eb->leak_list);
4241 kmem_cache_free(extent_buffer_cache, eb);
4242}
4243
4244static int extent_buffer_under_io(struct extent_buffer *eb)
4245{
4246 return (atomic_read(&eb->io_pages) ||
4247 test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4248 test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4249}
4250
4251
4252
4253
4254static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
4255 unsigned long start_idx)
4256{
4257 unsigned long index;
4258 unsigned long num_pages;
4259 struct page *page;
4260 int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4261
4262 BUG_ON(extent_buffer_under_io(eb));
4263
4264 num_pages = num_extent_pages(eb->start, eb->len);
4265 index = start_idx + num_pages;
4266 if (start_idx >= index)
4267 return;
4268
4269 do {
4270 index--;
4271 page = extent_buffer_page(eb, index);
4272 if (page && mapped) {
4273 spin_lock(&page->mapping->private_lock);
4274
4275
4276
4277
4278
4279
4280
4281 if (PagePrivate(page) &&
4282 page->private == (unsigned long)eb) {
4283 BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4284 BUG_ON(PageDirty(page));
4285 BUG_ON(PageWriteback(page));
4286
4287
4288
4289
4290 ClearPagePrivate(page);
4291 set_page_private(page, 0);
4292
4293 page_cache_release(page);
4294 }
4295 spin_unlock(&page->mapping->private_lock);
4296
4297 }
4298 if (page) {
4299
4300 page_cache_release(page);
4301 }
4302 } while (index != start_idx);
4303}
4304
4305
4306
4307
4308static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4309{
4310 btrfs_release_extent_buffer_page(eb, 0);
4311 __free_extent_buffer(eb);
4312}
4313
4314static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
4315 u64 start,
4316 unsigned long len,
4317 gfp_t mask)
4318{
4319 struct extent_buffer *eb = NULL;
4320
4321 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
4322 if (eb == NULL)
4323 return NULL;
4324 eb->start = start;
4325 eb->len = len;
4326 eb->tree = tree;
4327 eb->bflags = 0;
4328 rwlock_init(&eb->lock);
4329 atomic_set(&eb->write_locks, 0);
4330 atomic_set(&eb->read_locks, 0);
4331 atomic_set(&eb->blocking_readers, 0);
4332 atomic_set(&eb->blocking_writers, 0);
4333 atomic_set(&eb->spinning_readers, 0);
4334 atomic_set(&eb->spinning_writers, 0);
4335 eb->lock_nested = 0;
4336 init_waitqueue_head(&eb->write_lock_wq);
4337 init_waitqueue_head(&eb->read_lock_wq);
4338
4339 btrfs_leak_debug_add(&eb->leak_list, &buffers);
4340
4341 spin_lock_init(&eb->refs_lock);
4342 atomic_set(&eb->refs, 1);
4343 atomic_set(&eb->io_pages, 0);
4344
4345
4346
4347
4348 BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4349 > MAX_INLINE_EXTENT_BUFFER_SIZE);
4350 BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4351
4352 return eb;
4353}
4354
4355struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4356{
4357 unsigned long i;
4358 struct page *p;
4359 struct extent_buffer *new;
4360 unsigned long num_pages = num_extent_pages(src->start, src->len);
4361
4362 new = __alloc_extent_buffer(NULL, src->start, src->len, GFP_NOFS);
4363 if (new == NULL)
4364 return NULL;
4365
4366 for (i = 0; i < num_pages; i++) {
4367 p = alloc_page(GFP_NOFS);
4368 if (!p) {
4369 btrfs_release_extent_buffer(new);
4370 return NULL;
4371 }
4372 attach_extent_buffer_page(new, p);
4373 WARN_ON(PageDirty(p));
4374 SetPageUptodate(p);
4375 new->pages[i] = p;
4376 }
4377
4378 copy_extent_buffer(new, src, 0, 0, src->len);
4379 set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4380 set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4381
4382 return new;
4383}
4384
4385struct extent_buffer *alloc_dummy_extent_buffer(u64 start, unsigned long len)
4386{
4387 struct extent_buffer *eb;
4388 unsigned long num_pages = num_extent_pages(0, len);
4389 unsigned long i;
4390
4391 eb = __alloc_extent_buffer(NULL, start, len, GFP_NOFS);
4392 if (!eb)
4393 return NULL;
4394
4395 for (i = 0; i < num_pages; i++) {
4396 eb->pages[i] = alloc_page(GFP_NOFS);
4397 if (!eb->pages[i])
4398 goto err;
4399 }
4400 set_extent_buffer_uptodate(eb);
4401 btrfs_set_header_nritems(eb, 0);
4402 set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4403
4404 return eb;
4405err:
4406 for (; i > 0; i--)
4407 __free_page(eb->pages[i - 1]);
4408 __free_extent_buffer(eb);
4409 return NULL;
4410}
4411
4412static void check_buffer_tree_ref(struct extent_buffer *eb)
4413{
4414 int refs;
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435 refs = atomic_read(&eb->refs);
4436 if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4437 return;
4438
4439 spin_lock(&eb->refs_lock);
4440 if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4441 atomic_inc(&eb->refs);
4442 spin_unlock(&eb->refs_lock);
4443}
4444
4445static void mark_extent_buffer_accessed(struct extent_buffer *eb)
4446{
4447 unsigned long num_pages, i;
4448
4449 check_buffer_tree_ref(eb);
4450
4451 num_pages = num_extent_pages(eb->start, eb->len);
4452 for (i = 0; i < num_pages; i++) {
4453 struct page *p = extent_buffer_page(eb, i);
4454 mark_page_accessed(p);
4455 }
4456}
4457
4458struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
4459 u64 start, unsigned long len)
4460{
4461 unsigned long num_pages = num_extent_pages(start, len);
4462 unsigned long i;
4463 unsigned long index = start >> PAGE_CACHE_SHIFT;
4464 struct extent_buffer *eb;
4465 struct extent_buffer *exists = NULL;
4466 struct page *p;
4467 struct address_space *mapping = tree->mapping;
4468 int uptodate = 1;
4469 int ret;
4470
4471 rcu_read_lock();
4472 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4473 if (eb && atomic_inc_not_zero(&eb->refs)) {
4474 rcu_read_unlock();
4475 mark_extent_buffer_accessed(eb);
4476 return eb;
4477 }
4478 rcu_read_unlock();
4479
4480 eb = __alloc_extent_buffer(tree, start, len, GFP_NOFS);
4481 if (!eb)
4482 return NULL;
4483
4484 for (i = 0; i < num_pages; i++, index++) {
4485 p = find_or_create_page(mapping, index, GFP_NOFS);
4486 if (!p)
4487 goto free_eb;
4488
4489 spin_lock(&mapping->private_lock);
4490 if (PagePrivate(p)) {
4491
4492
4493
4494
4495
4496
4497
4498 exists = (struct extent_buffer *)p->private;
4499 if (atomic_inc_not_zero(&exists->refs)) {
4500 spin_unlock(&mapping->private_lock);
4501 unlock_page(p);
4502 page_cache_release(p);
4503 mark_extent_buffer_accessed(exists);
4504 goto free_eb;
4505 }
4506
4507
4508
4509
4510
4511 ClearPagePrivate(p);
4512 WARN_ON(PageDirty(p));
4513 page_cache_release(p);
4514 }
4515 attach_extent_buffer_page(eb, p);
4516 spin_unlock(&mapping->private_lock);
4517 WARN_ON(PageDirty(p));
4518 mark_page_accessed(p);
4519 eb->pages[i] = p;
4520 if (!PageUptodate(p))
4521 uptodate = 0;
4522
4523
4524
4525
4526
4527 }
4528 if (uptodate)
4529 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4530again:
4531 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4532 if (ret)
4533 goto free_eb;
4534
4535 spin_lock(&tree->buffer_lock);
4536 ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
4537 if (ret == -EEXIST) {
4538 exists = radix_tree_lookup(&tree->buffer,
4539 start >> PAGE_CACHE_SHIFT);
4540 if (!atomic_inc_not_zero(&exists->refs)) {
4541 spin_unlock(&tree->buffer_lock);
4542 radix_tree_preload_end();
4543 exists = NULL;
4544 goto again;
4545 }
4546 spin_unlock(&tree->buffer_lock);
4547 radix_tree_preload_end();
4548 mark_extent_buffer_accessed(exists);
4549 goto free_eb;
4550 }
4551
4552 check_buffer_tree_ref(eb);
4553 spin_unlock(&tree->buffer_lock);
4554 radix_tree_preload_end();
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565 SetPageChecked(eb->pages[0]);
4566 for (i = 1; i < num_pages; i++) {
4567 p = extent_buffer_page(eb, i);
4568 ClearPageChecked(p);
4569 unlock_page(p);
4570 }
4571 unlock_page(eb->pages[0]);
4572 return eb;
4573
4574free_eb:
4575 for (i = 0; i < num_pages; i++) {
4576 if (eb->pages[i])
4577 unlock_page(eb->pages[i]);
4578 }
4579
4580 WARN_ON(!atomic_dec_and_test(&eb->refs));
4581 btrfs_release_extent_buffer(eb);
4582 return exists;
4583}
4584
4585struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
4586 u64 start, unsigned long len)
4587{
4588 struct extent_buffer *eb;
4589
4590 rcu_read_lock();
4591 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
4592 if (eb && atomic_inc_not_zero(&eb->refs)) {
4593 rcu_read_unlock();
4594 mark_extent_buffer_accessed(eb);
4595 return eb;
4596 }
4597 rcu_read_unlock();
4598
4599 return NULL;
4600}
4601
4602static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4603{
4604 struct extent_buffer *eb =
4605 container_of(head, struct extent_buffer, rcu_head);
4606
4607 __free_extent_buffer(eb);
4608}
4609
4610
4611static int release_extent_buffer(struct extent_buffer *eb)
4612{
4613 WARN_ON(atomic_read(&eb->refs) == 0);
4614 if (atomic_dec_and_test(&eb->refs)) {
4615 if (test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags)) {
4616 spin_unlock(&eb->refs_lock);
4617 } else {
4618 struct extent_io_tree *tree = eb->tree;
4619
4620 spin_unlock(&eb->refs_lock);
4621
4622 spin_lock(&tree->buffer_lock);
4623 radix_tree_delete(&tree->buffer,
4624 eb->start >> PAGE_CACHE_SHIFT);
4625 spin_unlock(&tree->buffer_lock);
4626 }
4627
4628
4629 btrfs_release_extent_buffer_page(eb, 0);
4630 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
4631 return 1;
4632 }
4633 spin_unlock(&eb->refs_lock);
4634
4635 return 0;
4636}
4637
4638void free_extent_buffer(struct extent_buffer *eb)
4639{
4640 int refs;
4641 int old;
4642 if (!eb)
4643 return;
4644
4645 while (1) {
4646 refs = atomic_read(&eb->refs);
4647 if (refs <= 3)
4648 break;
4649 old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
4650 if (old == refs)
4651 return;
4652 }
4653
4654 spin_lock(&eb->refs_lock);
4655 if (atomic_read(&eb->refs) == 2 &&
4656 test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
4657 atomic_dec(&eb->refs);
4658
4659 if (atomic_read(&eb->refs) == 2 &&
4660 test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
4661 !extent_buffer_under_io(eb) &&
4662 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4663 atomic_dec(&eb->refs);
4664
4665
4666
4667
4668
4669 release_extent_buffer(eb);
4670}
4671
4672void free_extent_buffer_stale(struct extent_buffer *eb)
4673{
4674 if (!eb)
4675 return;
4676
4677 spin_lock(&eb->refs_lock);
4678 set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
4679
4680 if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
4681 test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4682 atomic_dec(&eb->refs);
4683 release_extent_buffer(eb);
4684}
4685
4686void clear_extent_buffer_dirty(struct extent_buffer *eb)
4687{
4688 unsigned long i;
4689 unsigned long num_pages;
4690 struct page *page;
4691
4692 num_pages = num_extent_pages(eb->start, eb->len);
4693
4694 for (i = 0; i < num_pages; i++) {
4695 page = extent_buffer_page(eb, i);
4696 if (!PageDirty(page))
4697 continue;
4698
4699 lock_page(page);
4700 WARN_ON(!PagePrivate(page));
4701
4702 clear_page_dirty_for_io(page);
4703 spin_lock_irq(&page->mapping->tree_lock);
4704 if (!PageDirty(page)) {
4705 radix_tree_tag_clear(&page->mapping->page_tree,
4706 page_index(page),
4707 PAGECACHE_TAG_DIRTY);
4708 }
4709 spin_unlock_irq(&page->mapping->tree_lock);
4710 ClearPageError(page);
4711 unlock_page(page);
4712 }
4713 WARN_ON(atomic_read(&eb->refs) == 0);
4714}
4715
4716int set_extent_buffer_dirty(struct extent_buffer *eb)
4717{
4718 unsigned long i;
4719 unsigned long num_pages;
4720 int was_dirty = 0;
4721
4722 check_buffer_tree_ref(eb);
4723
4724 was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
4725
4726 num_pages = num_extent_pages(eb->start, eb->len);
4727 WARN_ON(atomic_read(&eb->refs) == 0);
4728 WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
4729
4730 for (i = 0; i < num_pages; i++)
4731 set_page_dirty(extent_buffer_page(eb, i));
4732 return was_dirty;
4733}
4734
4735int clear_extent_buffer_uptodate(struct extent_buffer *eb)
4736{
4737 unsigned long i;
4738 struct page *page;
4739 unsigned long num_pages;
4740
4741 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4742 num_pages = num_extent_pages(eb->start, eb->len);
4743 for (i = 0; i < num_pages; i++) {
4744 page = extent_buffer_page(eb, i);
4745 if (page)
4746 ClearPageUptodate(page);
4747 }
4748 return 0;
4749}
4750
4751int set_extent_buffer_uptodate(struct extent_buffer *eb)
4752{
4753 unsigned long i;
4754 struct page *page;
4755 unsigned long num_pages;
4756
4757 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4758 num_pages = num_extent_pages(eb->start, eb->len);
4759 for (i = 0; i < num_pages; i++) {
4760 page = extent_buffer_page(eb, i);
4761 SetPageUptodate(page);
4762 }
4763 return 0;
4764}
4765
4766int extent_buffer_uptodate(struct extent_buffer *eb)
4767{
4768 return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4769}
4770
4771int read_extent_buffer_pages(struct extent_io_tree *tree,
4772 struct extent_buffer *eb, u64 start, int wait,
4773 get_extent_t *get_extent, int mirror_num)
4774{
4775 unsigned long i;
4776 unsigned long start_i;
4777 struct page *page;
4778 int err;
4779 int ret = 0;
4780 int locked_pages = 0;
4781 int all_uptodate = 1;
4782 unsigned long num_pages;
4783 unsigned long num_reads = 0;
4784 struct bio *bio = NULL;
4785 unsigned long bio_flags = 0;
4786
4787 if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
4788 return 0;
4789
4790 if (start) {
4791 WARN_ON(start < eb->start);
4792 start_i = (start >> PAGE_CACHE_SHIFT) -
4793 (eb->start >> PAGE_CACHE_SHIFT);
4794 } else {
4795 start_i = 0;
4796 }
4797
4798 num_pages = num_extent_pages(eb->start, eb->len);
4799 for (i = start_i; i < num_pages; i++) {
4800 page = extent_buffer_page(eb, i);
4801 if (wait == WAIT_NONE) {
4802 if (!trylock_page(page))
4803 goto unlock_exit;
4804 } else {
4805 lock_page(page);
4806 }
4807 locked_pages++;
4808 if (!PageUptodate(page)) {
4809 num_reads++;
4810 all_uptodate = 0;
4811 }
4812 }
4813 if (all_uptodate) {
4814 if (start_i == 0)
4815 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
4816 goto unlock_exit;
4817 }
4818
4819 clear_bit(EXTENT_BUFFER_IOERR, &eb->bflags);
4820 eb->read_mirror = 0;
4821 atomic_set(&eb->io_pages, num_reads);
4822 for (i = start_i; i < num_pages; i++) {
4823 page = extent_buffer_page(eb, i);
4824 if (!PageUptodate(page)) {
4825 ClearPageError(page);
4826 err = __extent_read_full_page(tree, page,
4827 get_extent, &bio,
4828 mirror_num, &bio_flags,
4829 READ | REQ_META);
4830 if (err)
4831 ret = err;
4832 } else {
4833 unlock_page(page);
4834 }
4835 }
4836
4837 if (bio) {
4838 err = submit_one_bio(READ | REQ_META, bio, mirror_num,
4839 bio_flags);
4840 if (err)
4841 return err;
4842 }
4843
4844 if (ret || wait != WAIT_COMPLETE)
4845 return ret;
4846
4847 for (i = start_i; i < num_pages; i++) {
4848 page = extent_buffer_page(eb, i);
4849 wait_on_page_locked(page);
4850 if (!PageUptodate(page))
4851 ret = -EIO;
4852 }
4853
4854 return ret;
4855
4856unlock_exit:
4857 i = start_i;
4858 while (locked_pages > 0) {
4859 page = extent_buffer_page(eb, i);
4860 i++;
4861 unlock_page(page);
4862 locked_pages--;
4863 }
4864 return ret;
4865}
4866
4867void read_extent_buffer(struct extent_buffer *eb, void *dstv,
4868 unsigned long start,
4869 unsigned long len)
4870{
4871 size_t cur;
4872 size_t offset;
4873 struct page *page;
4874 char *kaddr;
4875 char *dst = (char *)dstv;
4876 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4877 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4878
4879 WARN_ON(start > eb->len);
4880 WARN_ON(start + len > eb->start + eb->len);
4881
4882 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
4883
4884 while (len > 0) {
4885 page = extent_buffer_page(eb, i);
4886
4887 cur = min(len, (PAGE_CACHE_SIZE - offset));
4888 kaddr = page_address(page);
4889 memcpy(dst, kaddr + offset, cur);
4890
4891 dst += cur;
4892 len -= cur;
4893 offset = 0;
4894 i++;
4895 }
4896}
4897
4898int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
4899 unsigned long min_len, char **map,
4900 unsigned long *map_start,
4901 unsigned long *map_len)
4902{
4903 size_t offset = start & (PAGE_CACHE_SIZE - 1);
4904 char *kaddr;
4905 struct page *p;
4906 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4907 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4908 unsigned long end_i = (start_offset + start + min_len - 1) >>
4909 PAGE_CACHE_SHIFT;
4910
4911 if (i != end_i)
4912 return -EINVAL;
4913
4914 if (i == 0) {
4915 offset = start_offset;
4916 *map_start = 0;
4917 } else {
4918 offset = 0;
4919 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
4920 }
4921
4922 if (start + min_len > eb->len) {
4923 WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
4924 "wanted %lu %lu\n",
4925 eb->start, eb->len, start, min_len);
4926 return -EINVAL;
4927 }
4928
4929 p = extent_buffer_page(eb, i);
4930 kaddr = page_address(p);
4931 *map = kaddr + offset;
4932 *map_len = PAGE_CACHE_SIZE - offset;
4933 return 0;
4934}
4935
4936int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
4937 unsigned long start,
4938 unsigned long len)
4939{
4940 size_t cur;
4941 size_t offset;
4942 struct page *page;
4943 char *kaddr;
4944 char *ptr = (char *)ptrv;
4945 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4946 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4947 int ret = 0;
4948
4949 WARN_ON(start > eb->len);
4950 WARN_ON(start + len > eb->start + eb->len);
4951
4952 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
4953
4954 while (len > 0) {
4955 page = extent_buffer_page(eb, i);
4956
4957 cur = min(len, (PAGE_CACHE_SIZE - offset));
4958
4959 kaddr = page_address(page);
4960 ret = memcmp(ptr, kaddr + offset, cur);
4961 if (ret)
4962 break;
4963
4964 ptr += cur;
4965 len -= cur;
4966 offset = 0;
4967 i++;
4968 }
4969 return ret;
4970}
4971
4972void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
4973 unsigned long start, unsigned long len)
4974{
4975 size_t cur;
4976 size_t offset;
4977 struct page *page;
4978 char *kaddr;
4979 char *src = (char *)srcv;
4980 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
4981 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
4982
4983 WARN_ON(start > eb->len);
4984 WARN_ON(start + len > eb->start + eb->len);
4985
4986 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
4987
4988 while (len > 0) {
4989 page = extent_buffer_page(eb, i);
4990 WARN_ON(!PageUptodate(page));
4991
4992 cur = min(len, PAGE_CACHE_SIZE - offset);
4993 kaddr = page_address(page);
4994 memcpy(kaddr + offset, src, cur);
4995
4996 src += cur;
4997 len -= cur;
4998 offset = 0;
4999 i++;
5000 }
5001}
5002
5003void memset_extent_buffer(struct extent_buffer *eb, char c,
5004 unsigned long start, unsigned long len)
5005{
5006 size_t cur;
5007 size_t offset;
5008 struct page *page;
5009 char *kaddr;
5010 size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
5011 unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
5012
5013 WARN_ON(start > eb->len);
5014 WARN_ON(start + len > eb->start + eb->len);
5015
5016 offset = (start_offset + start) & (PAGE_CACHE_SIZE - 1);
5017
5018 while (len > 0) {
5019 page = extent_buffer_page(eb, i);
5020 WARN_ON(!PageUptodate(page));
5021
5022 cur = min(len, PAGE_CACHE_SIZE - offset);
5023 kaddr = page_address(page);
5024 memset(kaddr + offset, c, cur);
5025
5026 len -= cur;
5027 offset = 0;
5028 i++;
5029 }
5030}
5031
5032void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
5033 unsigned long dst_offset, unsigned long src_offset,
5034 unsigned long len)
5035{
5036 u64 dst_len = dst->len;
5037 size_t cur;
5038 size_t offset;
5039 struct page *page;
5040 char *kaddr;
5041 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5042 unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5043
5044 WARN_ON(src->len != dst_len);
5045
5046 offset = (start_offset + dst_offset) &
5047 (PAGE_CACHE_SIZE - 1);
5048
5049 while (len > 0) {
5050 page = extent_buffer_page(dst, i);
5051 WARN_ON(!PageUptodate(page));
5052
5053 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
5054
5055 kaddr = page_address(page);
5056 read_extent_buffer(src, kaddr + offset, src_offset, cur);
5057
5058 src_offset += cur;
5059 len -= cur;
5060 offset = 0;
5061 i++;
5062 }
5063}
5064
5065static void move_pages(struct page *dst_page, struct page *src_page,
5066 unsigned long dst_off, unsigned long src_off,
5067 unsigned long len)
5068{
5069 char *dst_kaddr = page_address(dst_page);
5070 if (dst_page == src_page) {
5071 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
5072 } else {
5073 char *src_kaddr = page_address(src_page);
5074 char *p = dst_kaddr + dst_off + len;
5075 char *s = src_kaddr + src_off + len;
5076
5077 while (len--)
5078 *--p = *--s;
5079 }
5080}
5081
5082static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5083{
5084 unsigned long distance = (src > dst) ? src - dst : dst - src;
5085 return distance < len;
5086}
5087
5088static void copy_pages(struct page *dst_page, struct page *src_page,
5089 unsigned long dst_off, unsigned long src_off,
5090 unsigned long len)
5091{
5092 char *dst_kaddr = page_address(dst_page);
5093 char *src_kaddr;
5094 int must_memmove = 0;
5095
5096 if (dst_page != src_page) {
5097 src_kaddr = page_address(src_page);
5098 } else {
5099 src_kaddr = dst_kaddr;
5100 if (areas_overlap(src_off, dst_off, len))
5101 must_memmove = 1;
5102 }
5103
5104 if (must_memmove)
5105 memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5106 else
5107 memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
5108}
5109
5110void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5111 unsigned long src_offset, unsigned long len)
5112{
5113 size_t cur;
5114 size_t dst_off_in_page;
5115 size_t src_off_in_page;
5116 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5117 unsigned long dst_i;
5118 unsigned long src_i;
5119
5120 if (src_offset + len > dst->len) {
5121 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
5122 "len %lu dst len %lu\n", src_offset, len, dst->len);
5123 BUG_ON(1);
5124 }
5125 if (dst_offset + len > dst->len) {
5126 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
5127 "len %lu dst len %lu\n", dst_offset, len, dst->len);
5128 BUG_ON(1);
5129 }
5130
5131 while (len > 0) {
5132 dst_off_in_page = (start_offset + dst_offset) &
5133 (PAGE_CACHE_SIZE - 1);
5134 src_off_in_page = (start_offset + src_offset) &
5135 (PAGE_CACHE_SIZE - 1);
5136
5137 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
5138 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
5139
5140 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
5141 src_off_in_page));
5142 cur = min_t(unsigned long, cur,
5143 (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
5144
5145 copy_pages(extent_buffer_page(dst, dst_i),
5146 extent_buffer_page(dst, src_i),
5147 dst_off_in_page, src_off_in_page, cur);
5148
5149 src_offset += cur;
5150 dst_offset += cur;
5151 len -= cur;
5152 }
5153}
5154
5155void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5156 unsigned long src_offset, unsigned long len)
5157{
5158 size_t cur;
5159 size_t dst_off_in_page;
5160 size_t src_off_in_page;
5161 unsigned long dst_end = dst_offset + len - 1;
5162 unsigned long src_end = src_offset + len - 1;
5163 size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
5164 unsigned long dst_i;
5165 unsigned long src_i;
5166
5167 if (src_offset + len > dst->len) {
5168 printk(KERN_ERR "btrfs memmove bogus src_offset %lu move "
5169 "len %lu len %lu\n", src_offset, len, dst->len);
5170 BUG_ON(1);
5171 }
5172 if (dst_offset + len > dst->len) {
5173 printk(KERN_ERR "btrfs memmove bogus dst_offset %lu move "
5174 "len %lu len %lu\n", dst_offset, len, dst->len);
5175 BUG_ON(1);
5176 }
5177 if (dst_offset < src_offset) {
5178 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5179 return;
5180 }
5181 while (len > 0) {
5182 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
5183 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
5184
5185 dst_off_in_page = (start_offset + dst_end) &
5186 (PAGE_CACHE_SIZE - 1);
5187 src_off_in_page = (start_offset + src_end) &
5188 (PAGE_CACHE_SIZE - 1);
5189
5190 cur = min_t(unsigned long, len, src_off_in_page + 1);
5191 cur = min(cur, dst_off_in_page + 1);
5192 move_pages(extent_buffer_page(dst, dst_i),
5193 extent_buffer_page(dst, src_i),
5194 dst_off_in_page - cur + 1,
5195 src_off_in_page - cur + 1, cur);
5196
5197 dst_end -= cur;
5198 src_end -= cur;
5199 len -= cur;
5200 }
5201}
5202
5203int try_release_extent_buffer(struct page *page)
5204{
5205 struct extent_buffer *eb;
5206
5207
5208
5209
5210
5211 spin_lock(&page->mapping->private_lock);
5212 if (!PagePrivate(page)) {
5213 spin_unlock(&page->mapping->private_lock);
5214 return 1;
5215 }
5216
5217 eb = (struct extent_buffer *)page->private;
5218 BUG_ON(!eb);
5219
5220
5221
5222
5223
5224
5225 spin_lock(&eb->refs_lock);
5226 if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
5227 spin_unlock(&eb->refs_lock);
5228 spin_unlock(&page->mapping->private_lock);
5229 return 0;
5230 }
5231 spin_unlock(&page->mapping->private_lock);
5232
5233
5234
5235
5236
5237 if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5238 spin_unlock(&eb->refs_lock);
5239 return 0;
5240 }
5241
5242 return release_extent_buffer(eb);
5243}
5244