1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/sched.h>
19#include <linux/pagemap.h>
20#include <linux/writeback.h>
21#include <linux/blkdev.h>
22#include <linux/sort.h>
23#include <linux/rcupdate.h>
24#include <linux/kthread.h>
25#include <linux/slab.h>
26#include <linux/ratelimit.h>
27#include <linux/percpu_counter.h>
28#include "compat.h"
29#include "hash.h"
30#include "ctree.h"
31#include "disk-io.h"
32#include "print-tree.h"
33#include "transaction.h"
34#include "volumes.h"
35#include "raid56.h"
36#include "locking.h"
37#include "free-space-cache.h"
38#include "math.h"
39
40#undef SCRAMBLE_DELAYED_REFS
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56enum {
57 CHUNK_ALLOC_NO_FORCE = 0,
58 CHUNK_ALLOC_LIMITED = 1,
59 CHUNK_ALLOC_FORCE = 2,
60};
61
62
63
64
65
66
67
68
69
70
71enum {
72 RESERVE_FREE = 0,
73 RESERVE_ALLOC = 1,
74 RESERVE_ALLOC_NO_ACCOUNT = 2,
75};
76
77static int update_block_group(struct btrfs_root *root,
78 u64 bytenr, u64 num_bytes, int alloc);
79static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
80 struct btrfs_root *root,
81 u64 bytenr, u64 num_bytes, u64 parent,
82 u64 root_objectid, u64 owner_objectid,
83 u64 owner_offset, int refs_to_drop,
84 struct btrfs_delayed_extent_op *extra_op);
85static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
86 struct extent_buffer *leaf,
87 struct btrfs_extent_item *ei);
88static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
89 struct btrfs_root *root,
90 u64 parent, u64 root_objectid,
91 u64 flags, u64 owner, u64 offset,
92 struct btrfs_key *ins, int ref_mod);
93static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
94 struct btrfs_root *root,
95 u64 parent, u64 root_objectid,
96 u64 flags, struct btrfs_disk_key *key,
97 int level, struct btrfs_key *ins);
98static int do_chunk_alloc(struct btrfs_trans_handle *trans,
99 struct btrfs_root *extent_root, u64 flags,
100 int force);
101static int find_next_key(struct btrfs_path *path, int level,
102 struct btrfs_key *key);
103static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
104 int dump_block_groups);
105static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
106 u64 num_bytes, int reserve);
107static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
108 u64 num_bytes);
109int btrfs_pin_extent(struct btrfs_root *root,
110 u64 bytenr, u64 num_bytes, int reserved);
111
112static noinline int
113block_group_cache_done(struct btrfs_block_group_cache *cache)
114{
115 smp_mb();
116 return cache->cached == BTRFS_CACHE_FINISHED ||
117 cache->cached == BTRFS_CACHE_ERROR;
118}
119
120static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
121{
122 return (cache->flags & bits) == bits;
123}
124
125static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
126{
127 atomic_inc(&cache->count);
128}
129
130void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
131{
132 if (atomic_dec_and_test(&cache->count)) {
133 WARN_ON(cache->pinned > 0);
134 WARN_ON(cache->reserved > 0);
135 kfree(cache->free_space_ctl);
136 kfree(cache);
137 }
138}
139
140
141
142
143
144static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
145 struct btrfs_block_group_cache *block_group)
146{
147 struct rb_node **p;
148 struct rb_node *parent = NULL;
149 struct btrfs_block_group_cache *cache;
150
151 spin_lock(&info->block_group_cache_lock);
152 p = &info->block_group_cache_tree.rb_node;
153
154 while (*p) {
155 parent = *p;
156 cache = rb_entry(parent, struct btrfs_block_group_cache,
157 cache_node);
158 if (block_group->key.objectid < cache->key.objectid) {
159 p = &(*p)->rb_left;
160 } else if (block_group->key.objectid > cache->key.objectid) {
161 p = &(*p)->rb_right;
162 } else {
163 spin_unlock(&info->block_group_cache_lock);
164 return -EEXIST;
165 }
166 }
167
168 rb_link_node(&block_group->cache_node, parent, p);
169 rb_insert_color(&block_group->cache_node,
170 &info->block_group_cache_tree);
171
172 if (info->first_logical_byte > block_group->key.objectid)
173 info->first_logical_byte = block_group->key.objectid;
174
175 spin_unlock(&info->block_group_cache_lock);
176
177 return 0;
178}
179
180
181
182
183
184static struct btrfs_block_group_cache *
185block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
186 int contains)
187{
188 struct btrfs_block_group_cache *cache, *ret = NULL;
189 struct rb_node *n;
190 u64 end, start;
191
192 spin_lock(&info->block_group_cache_lock);
193 n = info->block_group_cache_tree.rb_node;
194
195 while (n) {
196 cache = rb_entry(n, struct btrfs_block_group_cache,
197 cache_node);
198 end = cache->key.objectid + cache->key.offset - 1;
199 start = cache->key.objectid;
200
201 if (bytenr < start) {
202 if (!contains && (!ret || start < ret->key.objectid))
203 ret = cache;
204 n = n->rb_left;
205 } else if (bytenr > start) {
206 if (contains && bytenr <= end) {
207 ret = cache;
208 break;
209 }
210 n = n->rb_right;
211 } else {
212 ret = cache;
213 break;
214 }
215 }
216 if (ret) {
217 btrfs_get_block_group(ret);
218 if (bytenr == 0 && info->first_logical_byte > ret->key.objectid)
219 info->first_logical_byte = ret->key.objectid;
220 }
221 spin_unlock(&info->block_group_cache_lock);
222
223 return ret;
224}
225
226static int add_excluded_extent(struct btrfs_root *root,
227 u64 start, u64 num_bytes)
228{
229 u64 end = start + num_bytes - 1;
230 set_extent_bits(&root->fs_info->freed_extents[0],
231 start, end, EXTENT_UPTODATE, GFP_NOFS);
232 set_extent_bits(&root->fs_info->freed_extents[1],
233 start, end, EXTENT_UPTODATE, GFP_NOFS);
234 return 0;
235}
236
237static void free_excluded_extents(struct btrfs_root *root,
238 struct btrfs_block_group_cache *cache)
239{
240 u64 start, end;
241
242 start = cache->key.objectid;
243 end = start + cache->key.offset - 1;
244
245 clear_extent_bits(&root->fs_info->freed_extents[0],
246 start, end, EXTENT_UPTODATE, GFP_NOFS);
247 clear_extent_bits(&root->fs_info->freed_extents[1],
248 start, end, EXTENT_UPTODATE, GFP_NOFS);
249}
250
251static int exclude_super_stripes(struct btrfs_root *root,
252 struct btrfs_block_group_cache *cache)
253{
254 u64 bytenr;
255 u64 *logical;
256 int stripe_len;
257 int i, nr, ret;
258
259 if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
260 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
261 cache->bytes_super += stripe_len;
262 ret = add_excluded_extent(root, cache->key.objectid,
263 stripe_len);
264 if (ret)
265 return ret;
266 }
267
268 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
269 bytenr = btrfs_sb_offset(i);
270 ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
271 cache->key.objectid, bytenr,
272 0, &logical, &nr, &stripe_len);
273 if (ret)
274 return ret;
275
276 while (nr--) {
277 u64 start, len;
278
279 if (logical[nr] > cache->key.objectid +
280 cache->key.offset)
281 continue;
282
283 if (logical[nr] + stripe_len <= cache->key.objectid)
284 continue;
285
286 start = logical[nr];
287 if (start < cache->key.objectid) {
288 start = cache->key.objectid;
289 len = (logical[nr] + stripe_len) - start;
290 } else {
291 len = min_t(u64, stripe_len,
292 cache->key.objectid +
293 cache->key.offset - start);
294 }
295
296 cache->bytes_super += len;
297 ret = add_excluded_extent(root, start, len);
298 if (ret) {
299 kfree(logical);
300 return ret;
301 }
302 }
303
304 kfree(logical);
305 }
306 return 0;
307}
308
309static struct btrfs_caching_control *
310get_caching_control(struct btrfs_block_group_cache *cache)
311{
312 struct btrfs_caching_control *ctl;
313
314 spin_lock(&cache->lock);
315 if (cache->cached != BTRFS_CACHE_STARTED) {
316 spin_unlock(&cache->lock);
317 return NULL;
318 }
319
320
321 if (!cache->caching_ctl) {
322 spin_unlock(&cache->lock);
323 return NULL;
324 }
325
326 ctl = cache->caching_ctl;
327 atomic_inc(&ctl->count);
328 spin_unlock(&cache->lock);
329 return ctl;
330}
331
332static void put_caching_control(struct btrfs_caching_control *ctl)
333{
334 if (atomic_dec_and_test(&ctl->count))
335 kfree(ctl);
336}
337
338
339
340
341
342
343static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
344 struct btrfs_fs_info *info, u64 start, u64 end)
345{
346 u64 extent_start, extent_end, size, total_added = 0;
347 int ret;
348
349 while (start < end) {
350 ret = find_first_extent_bit(info->pinned_extents, start,
351 &extent_start, &extent_end,
352 EXTENT_DIRTY | EXTENT_UPTODATE,
353 NULL);
354 if (ret)
355 break;
356
357 if (extent_start <= start) {
358 start = extent_end + 1;
359 } else if (extent_start > start && extent_start < end) {
360 size = extent_start - start;
361 total_added += size;
362 ret = btrfs_add_free_space(block_group, start,
363 size);
364 BUG_ON(ret);
365 start = extent_end + 1;
366 } else {
367 break;
368 }
369 }
370
371 if (start < end) {
372 size = end - start;
373 total_added += size;
374 ret = btrfs_add_free_space(block_group, start, size);
375 BUG_ON(ret);
376 }
377
378 return total_added;
379}
380
381static noinline void caching_thread(struct btrfs_work *work)
382{
383 struct btrfs_block_group_cache *block_group;
384 struct btrfs_fs_info *fs_info;
385 struct btrfs_caching_control *caching_ctl;
386 struct btrfs_root *extent_root;
387 struct btrfs_path *path;
388 struct extent_buffer *leaf;
389 struct btrfs_key key;
390 u64 total_found = 0;
391 u64 last = 0;
392 u32 nritems;
393 int ret = -ENOMEM;
394
395 caching_ctl = container_of(work, struct btrfs_caching_control, work);
396 block_group = caching_ctl->block_group;
397 fs_info = block_group->fs_info;
398 extent_root = fs_info->extent_root;
399
400 path = btrfs_alloc_path();
401 if (!path)
402 goto out;
403
404 last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
405
406
407
408
409
410
411
412 path->skip_locking = 1;
413 path->search_commit_root = 1;
414 path->reada = 1;
415
416 key.objectid = last;
417 key.offset = 0;
418 key.type = BTRFS_EXTENT_ITEM_KEY;
419again:
420 mutex_lock(&caching_ctl->mutex);
421
422 down_read(&fs_info->extent_commit_sem);
423
424next:
425 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
426 if (ret < 0)
427 goto err;
428
429 leaf = path->nodes[0];
430 nritems = btrfs_header_nritems(leaf);
431
432 while (1) {
433 if (btrfs_fs_closing(fs_info) > 1) {
434 last = (u64)-1;
435 break;
436 }
437
438 if (path->slots[0] < nritems) {
439 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
440 } else {
441 ret = find_next_key(path, 0, &key);
442 if (ret)
443 break;
444
445 if (need_resched()) {
446 caching_ctl->progress = last;
447 btrfs_release_path(path);
448 up_read(&fs_info->extent_commit_sem);
449 mutex_unlock(&caching_ctl->mutex);
450 cond_resched();
451 goto again;
452 }
453
454 ret = btrfs_next_leaf(extent_root, path);
455 if (ret < 0)
456 goto err;
457 if (ret)
458 break;
459 leaf = path->nodes[0];
460 nritems = btrfs_header_nritems(leaf);
461 continue;
462 }
463
464 if (key.objectid < last) {
465 key.objectid = last;
466 key.offset = 0;
467 key.type = BTRFS_EXTENT_ITEM_KEY;
468
469 caching_ctl->progress = last;
470 btrfs_release_path(path);
471 goto next;
472 }
473
474 if (key.objectid < block_group->key.objectid) {
475 path->slots[0]++;
476 continue;
477 }
478
479 if (key.objectid >= block_group->key.objectid +
480 block_group->key.offset)
481 break;
482
483 if (key.type == BTRFS_EXTENT_ITEM_KEY ||
484 key.type == BTRFS_METADATA_ITEM_KEY) {
485 total_found += add_new_free_space(block_group,
486 fs_info, last,
487 key.objectid);
488 if (key.type == BTRFS_METADATA_ITEM_KEY)
489 last = key.objectid +
490 fs_info->tree_root->leafsize;
491 else
492 last = key.objectid + key.offset;
493
494 if (total_found > (1024 * 1024 * 2)) {
495 total_found = 0;
496 wake_up(&caching_ctl->wait);
497 }
498 }
499 path->slots[0]++;
500 }
501 ret = 0;
502
503 total_found += add_new_free_space(block_group, fs_info, last,
504 block_group->key.objectid +
505 block_group->key.offset);
506 caching_ctl->progress = (u64)-1;
507
508 spin_lock(&block_group->lock);
509 block_group->caching_ctl = NULL;
510 block_group->cached = BTRFS_CACHE_FINISHED;
511 spin_unlock(&block_group->lock);
512
513err:
514 btrfs_free_path(path);
515 up_read(&fs_info->extent_commit_sem);
516
517 free_excluded_extents(extent_root, block_group);
518
519 mutex_unlock(&caching_ctl->mutex);
520out:
521 if (ret) {
522 spin_lock(&block_group->lock);
523 block_group->caching_ctl = NULL;
524 block_group->cached = BTRFS_CACHE_ERROR;
525 spin_unlock(&block_group->lock);
526 }
527 wake_up(&caching_ctl->wait);
528
529 put_caching_control(caching_ctl);
530 btrfs_put_block_group(block_group);
531}
532
533static int cache_block_group(struct btrfs_block_group_cache *cache,
534 int load_cache_only)
535{
536 DEFINE_WAIT(wait);
537 struct btrfs_fs_info *fs_info = cache->fs_info;
538 struct btrfs_caching_control *caching_ctl;
539 int ret = 0;
540
541 caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
542 if (!caching_ctl)
543 return -ENOMEM;
544
545 INIT_LIST_HEAD(&caching_ctl->list);
546 mutex_init(&caching_ctl->mutex);
547 init_waitqueue_head(&caching_ctl->wait);
548 caching_ctl->block_group = cache;
549 caching_ctl->progress = cache->key.objectid;
550 atomic_set(&caching_ctl->count, 1);
551 caching_ctl->work.func = caching_thread;
552
553 spin_lock(&cache->lock);
554
555
556
557
558
559
560
561
562
563
564
565
566 while (cache->cached == BTRFS_CACHE_FAST) {
567 struct btrfs_caching_control *ctl;
568
569 ctl = cache->caching_ctl;
570 atomic_inc(&ctl->count);
571 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
572 spin_unlock(&cache->lock);
573
574 schedule();
575
576 finish_wait(&ctl->wait, &wait);
577 put_caching_control(ctl);
578 spin_lock(&cache->lock);
579 }
580
581 if (cache->cached != BTRFS_CACHE_NO) {
582 spin_unlock(&cache->lock);
583 kfree(caching_ctl);
584 return 0;
585 }
586 WARN_ON(cache->caching_ctl);
587 cache->caching_ctl = caching_ctl;
588 cache->cached = BTRFS_CACHE_FAST;
589 spin_unlock(&cache->lock);
590
591 if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
592 ret = load_free_space_cache(fs_info, cache);
593
594 spin_lock(&cache->lock);
595 if (ret == 1) {
596 cache->caching_ctl = NULL;
597 cache->cached = BTRFS_CACHE_FINISHED;
598 cache->last_byte_to_unpin = (u64)-1;
599 } else {
600 if (load_cache_only) {
601 cache->caching_ctl = NULL;
602 cache->cached = BTRFS_CACHE_NO;
603 } else {
604 cache->cached = BTRFS_CACHE_STARTED;
605 }
606 }
607 spin_unlock(&cache->lock);
608 wake_up(&caching_ctl->wait);
609 if (ret == 1) {
610 put_caching_control(caching_ctl);
611 free_excluded_extents(fs_info->extent_root, cache);
612 return 0;
613 }
614 } else {
615
616
617
618
619 spin_lock(&cache->lock);
620 if (load_cache_only) {
621 cache->caching_ctl = NULL;
622 cache->cached = BTRFS_CACHE_NO;
623 } else {
624 cache->cached = BTRFS_CACHE_STARTED;
625 }
626 spin_unlock(&cache->lock);
627 wake_up(&caching_ctl->wait);
628 }
629
630 if (load_cache_only) {
631 put_caching_control(caching_ctl);
632 return 0;
633 }
634
635 down_write(&fs_info->extent_commit_sem);
636 atomic_inc(&caching_ctl->count);
637 list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
638 up_write(&fs_info->extent_commit_sem);
639
640 btrfs_get_block_group(cache);
641
642 btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
643
644 return ret;
645}
646
647
648
649
650static struct btrfs_block_group_cache *
651btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
652{
653 struct btrfs_block_group_cache *cache;
654
655 cache = block_group_cache_tree_search(info, bytenr, 0);
656
657 return cache;
658}
659
660
661
662
663struct btrfs_block_group_cache *btrfs_lookup_block_group(
664 struct btrfs_fs_info *info,
665 u64 bytenr)
666{
667 struct btrfs_block_group_cache *cache;
668
669 cache = block_group_cache_tree_search(info, bytenr, 1);
670
671 return cache;
672}
673
674static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
675 u64 flags)
676{
677 struct list_head *head = &info->space_info;
678 struct btrfs_space_info *found;
679
680 flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
681
682 rcu_read_lock();
683 list_for_each_entry_rcu(found, head, list) {
684 if (found->flags & flags) {
685 rcu_read_unlock();
686 return found;
687 }
688 }
689 rcu_read_unlock();
690 return NULL;
691}
692
693
694
695
696
697void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
698{
699 struct list_head *head = &info->space_info;
700 struct btrfs_space_info *found;
701
702 rcu_read_lock();
703 list_for_each_entry_rcu(found, head, list)
704 found->full = 0;
705 rcu_read_unlock();
706}
707
708
709int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
710{
711 int ret;
712 struct btrfs_key key;
713 struct btrfs_path *path;
714
715 path = btrfs_alloc_path();
716 if (!path)
717 return -ENOMEM;
718
719 key.objectid = start;
720 key.offset = len;
721 key.type = BTRFS_EXTENT_ITEM_KEY;
722 ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
723 0, 0);
724 if (ret > 0) {
725 btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
726 if (key.objectid == start &&
727 key.type == BTRFS_METADATA_ITEM_KEY)
728 ret = 0;
729 }
730 btrfs_free_path(path);
731 return ret;
732}
733
734
735
736
737
738
739
740
741
742
743int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
744 struct btrfs_root *root, u64 bytenr,
745 u64 offset, int metadata, u64 *refs, u64 *flags)
746{
747 struct btrfs_delayed_ref_head *head;
748 struct btrfs_delayed_ref_root *delayed_refs;
749 struct btrfs_path *path;
750 struct btrfs_extent_item *ei;
751 struct extent_buffer *leaf;
752 struct btrfs_key key;
753 u32 item_size;
754 u64 num_refs;
755 u64 extent_flags;
756 int ret;
757
758
759
760
761
762 if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA)) {
763 offset = root->leafsize;
764 metadata = 0;
765 }
766
767 path = btrfs_alloc_path();
768 if (!path)
769 return -ENOMEM;
770
771 if (metadata) {
772 key.objectid = bytenr;
773 key.type = BTRFS_METADATA_ITEM_KEY;
774 key.offset = offset;
775 } else {
776 key.objectid = bytenr;
777 key.type = BTRFS_EXTENT_ITEM_KEY;
778 key.offset = offset;
779 }
780
781 if (!trans) {
782 path->skip_locking = 1;
783 path->search_commit_root = 1;
784 }
785again:
786 ret = btrfs_search_slot(trans, root->fs_info->extent_root,
787 &key, path, 0, 0);
788 if (ret < 0)
789 goto out_free;
790
791 if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) {
792 metadata = 0;
793 if (path->slots[0]) {
794 path->slots[0]--;
795 btrfs_item_key_to_cpu(path->nodes[0], &key,
796 path->slots[0]);
797 if (key.objectid == bytenr &&
798 key.type == BTRFS_EXTENT_ITEM_KEY &&
799 key.offset == root->leafsize)
800 ret = 0;
801 }
802 if (ret) {
803 key.objectid = bytenr;
804 key.type = BTRFS_EXTENT_ITEM_KEY;
805 key.offset = root->leafsize;
806 btrfs_release_path(path);
807 goto again;
808 }
809 }
810
811 if (ret == 0) {
812 leaf = path->nodes[0];
813 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
814 if (item_size >= sizeof(*ei)) {
815 ei = btrfs_item_ptr(leaf, path->slots[0],
816 struct btrfs_extent_item);
817 num_refs = btrfs_extent_refs(leaf, ei);
818 extent_flags = btrfs_extent_flags(leaf, ei);
819 } else {
820#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
821 struct btrfs_extent_item_v0 *ei0;
822 BUG_ON(item_size != sizeof(*ei0));
823 ei0 = btrfs_item_ptr(leaf, path->slots[0],
824 struct btrfs_extent_item_v0);
825 num_refs = btrfs_extent_refs_v0(leaf, ei0);
826
827 extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
828#else
829 BUG();
830#endif
831 }
832 BUG_ON(num_refs == 0);
833 } else {
834 num_refs = 0;
835 extent_flags = 0;
836 ret = 0;
837 }
838
839 if (!trans)
840 goto out;
841
842 delayed_refs = &trans->transaction->delayed_refs;
843 spin_lock(&delayed_refs->lock);
844 head = btrfs_find_delayed_ref_head(trans, bytenr);
845 if (head) {
846 if (!mutex_trylock(&head->mutex)) {
847 atomic_inc(&head->node.refs);
848 spin_unlock(&delayed_refs->lock);
849
850 btrfs_release_path(path);
851
852
853
854
855
856 mutex_lock(&head->mutex);
857 mutex_unlock(&head->mutex);
858 btrfs_put_delayed_ref(&head->node);
859 goto again;
860 }
861 if (head->extent_op && head->extent_op->update_flags)
862 extent_flags |= head->extent_op->flags_to_set;
863 else
864 BUG_ON(num_refs == 0);
865
866 num_refs += head->node.ref_mod;
867 mutex_unlock(&head->mutex);
868 }
869 spin_unlock(&delayed_refs->lock);
870out:
871 WARN_ON(num_refs == 0);
872 if (refs)
873 *refs = num_refs;
874 if (flags)
875 *flags = extent_flags;
876out_free:
877 btrfs_free_path(path);
878 return ret;
879}
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
988static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
989 struct btrfs_root *root,
990 struct btrfs_path *path,
991 u64 owner, u32 extra_size)
992{
993 struct btrfs_extent_item *item;
994 struct btrfs_extent_item_v0 *ei0;
995 struct btrfs_extent_ref_v0 *ref0;
996 struct btrfs_tree_block_info *bi;
997 struct extent_buffer *leaf;
998 struct btrfs_key key;
999 struct btrfs_key found_key;
1000 u32 new_size = sizeof(*item);
1001 u64 refs;
1002 int ret;
1003
1004 leaf = path->nodes[0];
1005 BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));
1006
1007 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1008 ei0 = btrfs_item_ptr(leaf, path->slots[0],
1009 struct btrfs_extent_item_v0);
1010 refs = btrfs_extent_refs_v0(leaf, ei0);
1011
1012 if (owner == (u64)-1) {
1013 while (1) {
1014 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1015 ret = btrfs_next_leaf(root, path);
1016 if (ret < 0)
1017 return ret;
1018 BUG_ON(ret > 0);
1019 leaf = path->nodes[0];
1020 }
1021 btrfs_item_key_to_cpu(leaf, &found_key,
1022 path->slots[0]);
1023 BUG_ON(key.objectid != found_key.objectid);
1024 if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
1025 path->slots[0]++;
1026 continue;
1027 }
1028 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1029 struct btrfs_extent_ref_v0);
1030 owner = btrfs_ref_objectid_v0(leaf, ref0);
1031 break;
1032 }
1033 }
1034 btrfs_release_path(path);
1035
1036 if (owner < BTRFS_FIRST_FREE_OBJECTID)
1037 new_size += sizeof(*bi);
1038
1039 new_size -= sizeof(*ei0);
1040 ret = btrfs_search_slot(trans, root, &key, path,
1041 new_size + extra_size, 1);
1042 if (ret < 0)
1043 return ret;
1044 BUG_ON(ret);
1045
1046 btrfs_extend_item(root, path, new_size);
1047
1048 leaf = path->nodes[0];
1049 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1050 btrfs_set_extent_refs(leaf, item, refs);
1051
1052 btrfs_set_extent_generation(leaf, item, 0);
1053 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1054 btrfs_set_extent_flags(leaf, item,
1055 BTRFS_EXTENT_FLAG_TREE_BLOCK |
1056 BTRFS_BLOCK_FLAG_FULL_BACKREF);
1057 bi = (struct btrfs_tree_block_info *)(item + 1);
1058
1059 memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
1060 btrfs_set_tree_block_level(leaf, bi, (int)owner);
1061 } else {
1062 btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
1063 }
1064 btrfs_mark_buffer_dirty(leaf);
1065 return 0;
1066}
1067#endif
1068
1069static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
1070{
1071 u32 high_crc = ~(u32)0;
1072 u32 low_crc = ~(u32)0;
1073 __le64 lenum;
1074
1075 lenum = cpu_to_le64(root_objectid);
1076 high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1077 lenum = cpu_to_le64(owner);
1078 low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1079 lenum = cpu_to_le64(offset);
1080 low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1081
1082 return ((u64)high_crc << 31) ^ (u64)low_crc;
1083}
1084
1085static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
1086 struct btrfs_extent_data_ref *ref)
1087{
1088 return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
1089 btrfs_extent_data_ref_objectid(leaf, ref),
1090 btrfs_extent_data_ref_offset(leaf, ref));
1091}
1092
1093static int match_extent_data_ref(struct extent_buffer *leaf,
1094 struct btrfs_extent_data_ref *ref,
1095 u64 root_objectid, u64 owner, u64 offset)
1096{
1097 if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
1098 btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
1099 btrfs_extent_data_ref_offset(leaf, ref) != offset)
1100 return 0;
1101 return 1;
1102}
1103
1104static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
1105 struct btrfs_root *root,
1106 struct btrfs_path *path,
1107 u64 bytenr, u64 parent,
1108 u64 root_objectid,
1109 u64 owner, u64 offset)
1110{
1111 struct btrfs_key key;
1112 struct btrfs_extent_data_ref *ref;
1113 struct extent_buffer *leaf;
1114 u32 nritems;
1115 int ret;
1116 int recow;
1117 int err = -ENOENT;
1118
1119 key.objectid = bytenr;
1120 if (parent) {
1121 key.type = BTRFS_SHARED_DATA_REF_KEY;
1122 key.offset = parent;
1123 } else {
1124 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1125 key.offset = hash_extent_data_ref(root_objectid,
1126 owner, offset);
1127 }
1128again:
1129 recow = 0;
1130 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1131 if (ret < 0) {
1132 err = ret;
1133 goto fail;
1134 }
1135
1136 if (parent) {
1137 if (!ret)
1138 return 0;
1139#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1140 key.type = BTRFS_EXTENT_REF_V0_KEY;
1141 btrfs_release_path(path);
1142 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1143 if (ret < 0) {
1144 err = ret;
1145 goto fail;
1146 }
1147 if (!ret)
1148 return 0;
1149#endif
1150 goto fail;
1151 }
1152
1153 leaf = path->nodes[0];
1154 nritems = btrfs_header_nritems(leaf);
1155 while (1) {
1156 if (path->slots[0] >= nritems) {
1157 ret = btrfs_next_leaf(root, path);
1158 if (ret < 0)
1159 err = ret;
1160 if (ret)
1161 goto fail;
1162
1163 leaf = path->nodes[0];
1164 nritems = btrfs_header_nritems(leaf);
1165 recow = 1;
1166 }
1167
1168 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1169 if (key.objectid != bytenr ||
1170 key.type != BTRFS_EXTENT_DATA_REF_KEY)
1171 goto fail;
1172
1173 ref = btrfs_item_ptr(leaf, path->slots[0],
1174 struct btrfs_extent_data_ref);
1175
1176 if (match_extent_data_ref(leaf, ref, root_objectid,
1177 owner, offset)) {
1178 if (recow) {
1179 btrfs_release_path(path);
1180 goto again;
1181 }
1182 err = 0;
1183 break;
1184 }
1185 path->slots[0]++;
1186 }
1187fail:
1188 return err;
1189}
1190
1191static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
1192 struct btrfs_root *root,
1193 struct btrfs_path *path,
1194 u64 bytenr, u64 parent,
1195 u64 root_objectid, u64 owner,
1196 u64 offset, int refs_to_add)
1197{
1198 struct btrfs_key key;
1199 struct extent_buffer *leaf;
1200 u32 size;
1201 u32 num_refs;
1202 int ret;
1203
1204 key.objectid = bytenr;
1205 if (parent) {
1206 key.type = BTRFS_SHARED_DATA_REF_KEY;
1207 key.offset = parent;
1208 size = sizeof(struct btrfs_shared_data_ref);
1209 } else {
1210 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1211 key.offset = hash_extent_data_ref(root_objectid,
1212 owner, offset);
1213 size = sizeof(struct btrfs_extent_data_ref);
1214 }
1215
1216 ret = btrfs_insert_empty_item(trans, root, path, &key, size);
1217 if (ret && ret != -EEXIST)
1218 goto fail;
1219
1220 leaf = path->nodes[0];
1221 if (parent) {
1222 struct btrfs_shared_data_ref *ref;
1223 ref = btrfs_item_ptr(leaf, path->slots[0],
1224 struct btrfs_shared_data_ref);
1225 if (ret == 0) {
1226 btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
1227 } else {
1228 num_refs = btrfs_shared_data_ref_count(leaf, ref);
1229 num_refs += refs_to_add;
1230 btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
1231 }
1232 } else {
1233 struct btrfs_extent_data_ref *ref;
1234 while (ret == -EEXIST) {
1235 ref = btrfs_item_ptr(leaf, path->slots[0],
1236 struct btrfs_extent_data_ref);
1237 if (match_extent_data_ref(leaf, ref, root_objectid,
1238 owner, offset))
1239 break;
1240 btrfs_release_path(path);
1241 key.offset++;
1242 ret = btrfs_insert_empty_item(trans, root, path, &key,
1243 size);
1244 if (ret && ret != -EEXIST)
1245 goto fail;
1246
1247 leaf = path->nodes[0];
1248 }
1249 ref = btrfs_item_ptr(leaf, path->slots[0],
1250 struct btrfs_extent_data_ref);
1251 if (ret == 0) {
1252 btrfs_set_extent_data_ref_root(leaf, ref,
1253 root_objectid);
1254 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
1255 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
1256 btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
1257 } else {
1258 num_refs = btrfs_extent_data_ref_count(leaf, ref);
1259 num_refs += refs_to_add;
1260 btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
1261 }
1262 }
1263 btrfs_mark_buffer_dirty(leaf);
1264 ret = 0;
1265fail:
1266 btrfs_release_path(path);
1267 return ret;
1268}
1269
1270static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
1271 struct btrfs_root *root,
1272 struct btrfs_path *path,
1273 int refs_to_drop)
1274{
1275 struct btrfs_key key;
1276 struct btrfs_extent_data_ref *ref1 = NULL;
1277 struct btrfs_shared_data_ref *ref2 = NULL;
1278 struct extent_buffer *leaf;
1279 u32 num_refs = 0;
1280 int ret = 0;
1281
1282 leaf = path->nodes[0];
1283 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1284
1285 if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1286 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1287 struct btrfs_extent_data_ref);
1288 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1289 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1290 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1291 struct btrfs_shared_data_ref);
1292 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1293#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1294 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
1295 struct btrfs_extent_ref_v0 *ref0;
1296 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1297 struct btrfs_extent_ref_v0);
1298 num_refs = btrfs_ref_count_v0(leaf, ref0);
1299#endif
1300 } else {
1301 BUG();
1302 }
1303
1304 BUG_ON(num_refs < refs_to_drop);
1305 num_refs -= refs_to_drop;
1306
1307 if (num_refs == 0) {
1308 ret = btrfs_del_item(trans, root, path);
1309 } else {
1310 if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
1311 btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
1312 else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
1313 btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
1314#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1315 else {
1316 struct btrfs_extent_ref_v0 *ref0;
1317 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1318 struct btrfs_extent_ref_v0);
1319 btrfs_set_ref_count_v0(leaf, ref0, num_refs);
1320 }
1321#endif
1322 btrfs_mark_buffer_dirty(leaf);
1323 }
1324 return ret;
1325}
1326
1327static noinline u32 extent_data_ref_count(struct btrfs_root *root,
1328 struct btrfs_path *path,
1329 struct btrfs_extent_inline_ref *iref)
1330{
1331 struct btrfs_key key;
1332 struct extent_buffer *leaf;
1333 struct btrfs_extent_data_ref *ref1;
1334 struct btrfs_shared_data_ref *ref2;
1335 u32 num_refs = 0;
1336
1337 leaf = path->nodes[0];
1338 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1339 if (iref) {
1340 if (btrfs_extent_inline_ref_type(leaf, iref) ==
1341 BTRFS_EXTENT_DATA_REF_KEY) {
1342 ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
1343 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1344 } else {
1345 ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
1346 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1347 }
1348 } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1349 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1350 struct btrfs_extent_data_ref);
1351 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1352 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1353 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1354 struct btrfs_shared_data_ref);
1355 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1356#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1357 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
1358 struct btrfs_extent_ref_v0 *ref0;
1359 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1360 struct btrfs_extent_ref_v0);
1361 num_refs = btrfs_ref_count_v0(leaf, ref0);
1362#endif
1363 } else {
1364 WARN_ON(1);
1365 }
1366 return num_refs;
1367}
1368
1369static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
1370 struct btrfs_root *root,
1371 struct btrfs_path *path,
1372 u64 bytenr, u64 parent,
1373 u64 root_objectid)
1374{
1375 struct btrfs_key key;
1376 int ret;
1377
1378 key.objectid = bytenr;
1379 if (parent) {
1380 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1381 key.offset = parent;
1382 } else {
1383 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1384 key.offset = root_objectid;
1385 }
1386
1387 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1388 if (ret > 0)
1389 ret = -ENOENT;
1390#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1391 if (ret == -ENOENT && parent) {
1392 btrfs_release_path(path);
1393 key.type = BTRFS_EXTENT_REF_V0_KEY;
1394 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1395 if (ret > 0)
1396 ret = -ENOENT;
1397 }
1398#endif
1399 return ret;
1400}
1401
1402static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
1403 struct btrfs_root *root,
1404 struct btrfs_path *path,
1405 u64 bytenr, u64 parent,
1406 u64 root_objectid)
1407{
1408 struct btrfs_key key;
1409 int ret;
1410
1411 key.objectid = bytenr;
1412 if (parent) {
1413 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1414 key.offset = parent;
1415 } else {
1416 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1417 key.offset = root_objectid;
1418 }
1419
1420 ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1421 btrfs_release_path(path);
1422 return ret;
1423}
1424
1425static inline int extent_ref_type(u64 parent, u64 owner)
1426{
1427 int type;
1428 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1429 if (parent > 0)
1430 type = BTRFS_SHARED_BLOCK_REF_KEY;
1431 else
1432 type = BTRFS_TREE_BLOCK_REF_KEY;
1433 } else {
1434 if (parent > 0)
1435 type = BTRFS_SHARED_DATA_REF_KEY;
1436 else
1437 type = BTRFS_EXTENT_DATA_REF_KEY;
1438 }
1439 return type;
1440}
1441
1442static int find_next_key(struct btrfs_path *path, int level,
1443 struct btrfs_key *key)
1444
1445{
1446 for (; level < BTRFS_MAX_LEVEL; level++) {
1447 if (!path->nodes[level])
1448 break;
1449 if (path->slots[level] + 1 >=
1450 btrfs_header_nritems(path->nodes[level]))
1451 continue;
1452 if (level == 0)
1453 btrfs_item_key_to_cpu(path->nodes[level], key,
1454 path->slots[level] + 1);
1455 else
1456 btrfs_node_key_to_cpu(path->nodes[level], key,
1457 path->slots[level] + 1);
1458 return 0;
1459 }
1460 return 1;
1461}
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476static noinline_for_stack
1477int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
1478 struct btrfs_root *root,
1479 struct btrfs_path *path,
1480 struct btrfs_extent_inline_ref **ref_ret,
1481 u64 bytenr, u64 num_bytes,
1482 u64 parent, u64 root_objectid,
1483 u64 owner, u64 offset, int insert)
1484{
1485 struct btrfs_key key;
1486 struct extent_buffer *leaf;
1487 struct btrfs_extent_item *ei;
1488 struct btrfs_extent_inline_ref *iref;
1489 u64 flags;
1490 u64 item_size;
1491 unsigned long ptr;
1492 unsigned long end;
1493 int extra_size;
1494 int type;
1495 int want;
1496 int ret;
1497 int err = 0;
1498 bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
1499 SKINNY_METADATA);
1500
1501 key.objectid = bytenr;
1502 key.type = BTRFS_EXTENT_ITEM_KEY;
1503 key.offset = num_bytes;
1504
1505 want = extent_ref_type(parent, owner);
1506 if (insert) {
1507 extra_size = btrfs_extent_inline_ref_size(want);
1508 path->keep_locks = 1;
1509 } else
1510 extra_size = -1;
1511
1512
1513
1514
1515
1516 if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) {
1517 key.type = BTRFS_METADATA_ITEM_KEY;
1518 key.offset = owner;
1519 }
1520
1521again:
1522 ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1523 if (ret < 0) {
1524 err = ret;
1525 goto out;
1526 }
1527
1528
1529
1530
1531
1532 if (ret > 0 && skinny_metadata) {
1533 skinny_metadata = false;
1534 if (path->slots[0]) {
1535 path->slots[0]--;
1536 btrfs_item_key_to_cpu(path->nodes[0], &key,
1537 path->slots[0]);
1538 if (key.objectid == bytenr &&
1539 key.type == BTRFS_EXTENT_ITEM_KEY &&
1540 key.offset == num_bytes)
1541 ret = 0;
1542 }
1543 if (ret) {
1544 key.type = BTRFS_EXTENT_ITEM_KEY;
1545 key.offset = num_bytes;
1546 btrfs_release_path(path);
1547 goto again;
1548 }
1549 }
1550
1551 if (ret && !insert) {
1552 err = -ENOENT;
1553 goto out;
1554 } else if (ret) {
1555 err = -EIO;
1556 WARN_ON(1);
1557 goto out;
1558 }
1559
1560 leaf = path->nodes[0];
1561 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1562#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1563 if (item_size < sizeof(*ei)) {
1564 if (!insert) {
1565 err = -ENOENT;
1566 goto out;
1567 }
1568 ret = convert_extent_item_v0(trans, root, path, owner,
1569 extra_size);
1570 if (ret < 0) {
1571 err = ret;
1572 goto out;
1573 }
1574 leaf = path->nodes[0];
1575 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1576 }
1577#endif
1578 BUG_ON(item_size < sizeof(*ei));
1579
1580 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1581 flags = btrfs_extent_flags(leaf, ei);
1582
1583 ptr = (unsigned long)(ei + 1);
1584 end = (unsigned long)ei + item_size;
1585
1586 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) {
1587 ptr += sizeof(struct btrfs_tree_block_info);
1588 BUG_ON(ptr > end);
1589 }
1590
1591 err = -ENOENT;
1592 while (1) {
1593 if (ptr >= end) {
1594 WARN_ON(ptr > end);
1595 break;
1596 }
1597 iref = (struct btrfs_extent_inline_ref *)ptr;
1598 type = btrfs_extent_inline_ref_type(leaf, iref);
1599 if (want < type)
1600 break;
1601 if (want > type) {
1602 ptr += btrfs_extent_inline_ref_size(type);
1603 continue;
1604 }
1605
1606 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1607 struct btrfs_extent_data_ref *dref;
1608 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1609 if (match_extent_data_ref(leaf, dref, root_objectid,
1610 owner, offset)) {
1611 err = 0;
1612 break;
1613 }
1614 if (hash_extent_data_ref_item(leaf, dref) <
1615 hash_extent_data_ref(root_objectid, owner, offset))
1616 break;
1617 } else {
1618 u64 ref_offset;
1619 ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
1620 if (parent > 0) {
1621 if (parent == ref_offset) {
1622 err = 0;
1623 break;
1624 }
1625 if (ref_offset < parent)
1626 break;
1627 } else {
1628 if (root_objectid == ref_offset) {
1629 err = 0;
1630 break;
1631 }
1632 if (ref_offset < root_objectid)
1633 break;
1634 }
1635 }
1636 ptr += btrfs_extent_inline_ref_size(type);
1637 }
1638 if (err == -ENOENT && insert) {
1639 if (item_size + extra_size >=
1640 BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
1641 err = -EAGAIN;
1642 goto out;
1643 }
1644
1645
1646
1647
1648
1649
1650 if (find_next_key(path, 0, &key) == 0 &&
1651 key.objectid == bytenr &&
1652 key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1653 err = -EAGAIN;
1654 goto out;
1655 }
1656 }
1657 *ref_ret = (struct btrfs_extent_inline_ref *)ptr;
1658out:
1659 if (insert) {
1660 path->keep_locks = 0;
1661 btrfs_unlock_up_safe(path, 1);
1662 }
1663 return err;
1664}
1665
1666
1667
1668
1669static noinline_for_stack
1670void setup_inline_extent_backref(struct btrfs_root *root,
1671 struct btrfs_path *path,
1672 struct btrfs_extent_inline_ref *iref,
1673 u64 parent, u64 root_objectid,
1674 u64 owner, u64 offset, int refs_to_add,
1675 struct btrfs_delayed_extent_op *extent_op)
1676{
1677 struct extent_buffer *leaf;
1678 struct btrfs_extent_item *ei;
1679 unsigned long ptr;
1680 unsigned long end;
1681 unsigned long item_offset;
1682 u64 refs;
1683 int size;
1684 int type;
1685
1686 leaf = path->nodes[0];
1687 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1688 item_offset = (unsigned long)iref - (unsigned long)ei;
1689
1690 type = extent_ref_type(parent, owner);
1691 size = btrfs_extent_inline_ref_size(type);
1692
1693 btrfs_extend_item(root, path, size);
1694
1695 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1696 refs = btrfs_extent_refs(leaf, ei);
1697 refs += refs_to_add;
1698 btrfs_set_extent_refs(leaf, ei, refs);
1699 if (extent_op)
1700 __run_delayed_extent_op(extent_op, leaf, ei);
1701
1702 ptr = (unsigned long)ei + item_offset;
1703 end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
1704 if (ptr < end - size)
1705 memmove_extent_buffer(leaf, ptr + size, ptr,
1706 end - size - ptr);
1707
1708 iref = (struct btrfs_extent_inline_ref *)ptr;
1709 btrfs_set_extent_inline_ref_type(leaf, iref, type);
1710 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1711 struct btrfs_extent_data_ref *dref;
1712 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1713 btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
1714 btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
1715 btrfs_set_extent_data_ref_offset(leaf, dref, offset);
1716 btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
1717 } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1718 struct btrfs_shared_data_ref *sref;
1719 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1720 btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
1721 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1722 } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
1723 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1724 } else {
1725 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
1726 }
1727 btrfs_mark_buffer_dirty(leaf);
1728}
1729
1730static int lookup_extent_backref(struct btrfs_trans_handle *trans,
1731 struct btrfs_root *root,
1732 struct btrfs_path *path,
1733 struct btrfs_extent_inline_ref **ref_ret,
1734 u64 bytenr, u64 num_bytes, u64 parent,
1735 u64 root_objectid, u64 owner, u64 offset)
1736{
1737 int ret;
1738
1739 ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
1740 bytenr, num_bytes, parent,
1741 root_objectid, owner, offset, 0);
1742 if (ret != -ENOENT)
1743 return ret;
1744
1745 btrfs_release_path(path);
1746 *ref_ret = NULL;
1747
1748 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1749 ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
1750 root_objectid);
1751 } else {
1752 ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
1753 root_objectid, owner, offset);
1754 }
1755 return ret;
1756}
1757
1758
1759
1760
1761static noinline_for_stack
1762void update_inline_extent_backref(struct btrfs_root *root,
1763 struct btrfs_path *path,
1764 struct btrfs_extent_inline_ref *iref,
1765 int refs_to_mod,
1766 struct btrfs_delayed_extent_op *extent_op)
1767{
1768 struct extent_buffer *leaf;
1769 struct btrfs_extent_item *ei;
1770 struct btrfs_extent_data_ref *dref = NULL;
1771 struct btrfs_shared_data_ref *sref = NULL;
1772 unsigned long ptr;
1773 unsigned long end;
1774 u32 item_size;
1775 int size;
1776 int type;
1777 u64 refs;
1778
1779 leaf = path->nodes[0];
1780 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1781 refs = btrfs_extent_refs(leaf, ei);
1782 WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
1783 refs += refs_to_mod;
1784 btrfs_set_extent_refs(leaf, ei, refs);
1785 if (extent_op)
1786 __run_delayed_extent_op(extent_op, leaf, ei);
1787
1788 type = btrfs_extent_inline_ref_type(leaf, iref);
1789
1790 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1791 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1792 refs = btrfs_extent_data_ref_count(leaf, dref);
1793 } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1794 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1795 refs = btrfs_shared_data_ref_count(leaf, sref);
1796 } else {
1797 refs = 1;
1798 BUG_ON(refs_to_mod != -1);
1799 }
1800
1801 BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
1802 refs += refs_to_mod;
1803
1804 if (refs > 0) {
1805 if (type == BTRFS_EXTENT_DATA_REF_KEY)
1806 btrfs_set_extent_data_ref_count(leaf, dref, refs);
1807 else
1808 btrfs_set_shared_data_ref_count(leaf, sref, refs);
1809 } else {
1810 size = btrfs_extent_inline_ref_size(type);
1811 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1812 ptr = (unsigned long)iref;
1813 end = (unsigned long)ei + item_size;
1814 if (ptr + size < end)
1815 memmove_extent_buffer(leaf, ptr, ptr + size,
1816 end - ptr - size);
1817 item_size -= size;
1818 btrfs_truncate_item(root, path, item_size, 1);
1819 }
1820 btrfs_mark_buffer_dirty(leaf);
1821}
1822
1823static noinline_for_stack
1824int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
1825 struct btrfs_root *root,
1826 struct btrfs_path *path,
1827 u64 bytenr, u64 num_bytes, u64 parent,
1828 u64 root_objectid, u64 owner,
1829 u64 offset, int refs_to_add,
1830 struct btrfs_delayed_extent_op *extent_op)
1831{
1832 struct btrfs_extent_inline_ref *iref;
1833 int ret;
1834
1835 ret = lookup_inline_extent_backref(trans, root, path, &iref,
1836 bytenr, num_bytes, parent,
1837 root_objectid, owner, offset, 1);
1838 if (ret == 0) {
1839 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1840 update_inline_extent_backref(root, path, iref,
1841 refs_to_add, extent_op);
1842 } else if (ret == -ENOENT) {
1843 setup_inline_extent_backref(root, path, iref, parent,
1844 root_objectid, owner, offset,
1845 refs_to_add, extent_op);
1846 ret = 0;
1847 }
1848 return ret;
1849}
1850
1851static int insert_extent_backref(struct btrfs_trans_handle *trans,
1852 struct btrfs_root *root,
1853 struct btrfs_path *path,
1854 u64 bytenr, u64 parent, u64 root_objectid,
1855 u64 owner, u64 offset, int refs_to_add)
1856{
1857 int ret;
1858 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1859 BUG_ON(refs_to_add != 1);
1860 ret = insert_tree_block_ref(trans, root, path, bytenr,
1861 parent, root_objectid);
1862 } else {
1863 ret = insert_extent_data_ref(trans, root, path, bytenr,
1864 parent, root_objectid,
1865 owner, offset, refs_to_add);
1866 }
1867 return ret;
1868}
1869
1870static int remove_extent_backref(struct btrfs_trans_handle *trans,
1871 struct btrfs_root *root,
1872 struct btrfs_path *path,
1873 struct btrfs_extent_inline_ref *iref,
1874 int refs_to_drop, int is_data)
1875{
1876 int ret = 0;
1877
1878 BUG_ON(!is_data && refs_to_drop != 1);
1879 if (iref) {
1880 update_inline_extent_backref(root, path, iref,
1881 -refs_to_drop, NULL);
1882 } else if (is_data) {
1883 ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
1884 } else {
1885 ret = btrfs_del_item(trans, root, path);
1886 }
1887 return ret;
1888}
1889
1890static int btrfs_issue_discard(struct block_device *bdev,
1891 u64 start, u64 len)
1892{
1893 return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1894}
1895
1896static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1897 u64 num_bytes, u64 *actual_bytes)
1898{
1899 int ret;
1900 u64 discarded_bytes = 0;
1901 struct btrfs_bio *bbio = NULL;
1902
1903
1904
1905 ret = btrfs_map_block(root->fs_info, REQ_DISCARD,
1906 bytenr, &num_bytes, &bbio, 0);
1907
1908 if (!ret) {
1909 struct btrfs_bio_stripe *stripe = bbio->stripes;
1910 int i;
1911
1912
1913 for (i = 0; i < bbio->num_stripes; i++, stripe++) {
1914 if (!stripe->dev->can_discard)
1915 continue;
1916
1917 ret = btrfs_issue_discard(stripe->dev->bdev,
1918 stripe->physical,
1919 stripe->length);
1920 if (!ret)
1921 discarded_bytes += stripe->length;
1922 else if (ret != -EOPNOTSUPP)
1923 break;
1924
1925
1926
1927
1928
1929
1930 ret = 0;
1931 }
1932 kfree(bbio);
1933 }
1934
1935 if (actual_bytes)
1936 *actual_bytes = discarded_bytes;
1937
1938
1939 if (ret == -EOPNOTSUPP)
1940 ret = 0;
1941 return ret;
1942}
1943
1944
1945int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
1946 struct btrfs_root *root,
1947 u64 bytenr, u64 num_bytes, u64 parent,
1948 u64 root_objectid, u64 owner, u64 offset, int for_cow)
1949{
1950 int ret;
1951 struct btrfs_fs_info *fs_info = root->fs_info;
1952
1953 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
1954 root_objectid == BTRFS_TREE_LOG_OBJECTID);
1955
1956 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1957 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
1958 num_bytes,
1959 parent, root_objectid, (int)owner,
1960 BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1961 } else {
1962 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
1963 num_bytes,
1964 parent, root_objectid, owner, offset,
1965 BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1966 }
1967 return ret;
1968}
1969
1970static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
1971 struct btrfs_root *root,
1972 u64 bytenr, u64 num_bytes,
1973 u64 parent, u64 root_objectid,
1974 u64 owner, u64 offset, int refs_to_add,
1975 struct btrfs_delayed_extent_op *extent_op)
1976{
1977 struct btrfs_path *path;
1978 struct extent_buffer *leaf;
1979 struct btrfs_extent_item *item;
1980 u64 refs;
1981 int ret;
1982 int err = 0;
1983
1984 path = btrfs_alloc_path();
1985 if (!path)
1986 return -ENOMEM;
1987
1988 path->reada = 1;
1989 path->leave_spinning = 1;
1990
1991 ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
1992 path, bytenr, num_bytes, parent,
1993 root_objectid, owner, offset,
1994 refs_to_add, extent_op);
1995 if (ret == 0)
1996 goto out;
1997
1998 if (ret != -EAGAIN) {
1999 err = ret;
2000 goto out;
2001 }
2002
2003 leaf = path->nodes[0];
2004 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2005 refs = btrfs_extent_refs(leaf, item);
2006 btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
2007 if (extent_op)
2008 __run_delayed_extent_op(extent_op, leaf, item);
2009
2010 btrfs_mark_buffer_dirty(leaf);
2011 btrfs_release_path(path);
2012
2013 path->reada = 1;
2014 path->leave_spinning = 1;
2015
2016
2017 ret = insert_extent_backref(trans, root->fs_info->extent_root,
2018 path, bytenr, parent, root_objectid,
2019 owner, offset, refs_to_add);
2020 if (ret)
2021 btrfs_abort_transaction(trans, root, ret);
2022out:
2023 btrfs_free_path(path);
2024 return err;
2025}
2026
2027static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
2028 struct btrfs_root *root,
2029 struct btrfs_delayed_ref_node *node,
2030 struct btrfs_delayed_extent_op *extent_op,
2031 int insert_reserved)
2032{
2033 int ret = 0;
2034 struct btrfs_delayed_data_ref *ref;
2035 struct btrfs_key ins;
2036 u64 parent = 0;
2037 u64 ref_root = 0;
2038 u64 flags = 0;
2039
2040 ins.objectid = node->bytenr;
2041 ins.offset = node->num_bytes;
2042 ins.type = BTRFS_EXTENT_ITEM_KEY;
2043
2044 ref = btrfs_delayed_node_to_data_ref(node);
2045 trace_run_delayed_data_ref(node, ref, node->action);
2046
2047 if (node->type == BTRFS_SHARED_DATA_REF_KEY)
2048 parent = ref->parent;
2049 else
2050 ref_root = ref->root;
2051
2052 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2053 if (extent_op)
2054 flags |= extent_op->flags_to_set;
2055 ret = alloc_reserved_file_extent(trans, root,
2056 parent, ref_root, flags,
2057 ref->objectid, ref->offset,
2058 &ins, node->ref_mod);
2059 } else if (node->action == BTRFS_ADD_DELAYED_REF) {
2060 ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
2061 node->num_bytes, parent,
2062 ref_root, ref->objectid,
2063 ref->offset, node->ref_mod,
2064 extent_op);
2065 } else if (node->action == BTRFS_DROP_DELAYED_REF) {
2066 ret = __btrfs_free_extent(trans, root, node->bytenr,
2067 node->num_bytes, parent,
2068 ref_root, ref->objectid,
2069 ref->offset, node->ref_mod,
2070 extent_op);
2071 } else {
2072 BUG();
2073 }
2074 return ret;
2075}
2076
2077static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
2078 struct extent_buffer *leaf,
2079 struct btrfs_extent_item *ei)
2080{
2081 u64 flags = btrfs_extent_flags(leaf, ei);
2082 if (extent_op->update_flags) {
2083 flags |= extent_op->flags_to_set;
2084 btrfs_set_extent_flags(leaf, ei, flags);
2085 }
2086
2087 if (extent_op->update_key) {
2088 struct btrfs_tree_block_info *bi;
2089 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
2090 bi = (struct btrfs_tree_block_info *)(ei + 1);
2091 btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
2092 }
2093}
2094
2095static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
2096 struct btrfs_root *root,
2097 struct btrfs_delayed_ref_node *node,
2098 struct btrfs_delayed_extent_op *extent_op)
2099{
2100 struct btrfs_key key;
2101 struct btrfs_path *path;
2102 struct btrfs_extent_item *ei;
2103 struct extent_buffer *leaf;
2104 u32 item_size;
2105 int ret;
2106 int err = 0;
2107 int metadata = !extent_op->is_data;
2108
2109 if (trans->aborted)
2110 return 0;
2111
2112 if (metadata && !btrfs_fs_incompat(root->fs_info, SKINNY_METADATA))
2113 metadata = 0;
2114
2115 path = btrfs_alloc_path();
2116 if (!path)
2117 return -ENOMEM;
2118
2119 key.objectid = node->bytenr;
2120
2121 if (metadata) {
2122 key.type = BTRFS_METADATA_ITEM_KEY;
2123 key.offset = extent_op->level;
2124 } else {
2125 key.type = BTRFS_EXTENT_ITEM_KEY;
2126 key.offset = node->num_bytes;
2127 }
2128
2129again:
2130 path->reada = 1;
2131 path->leave_spinning = 1;
2132 ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
2133 path, 0, 1);
2134 if (ret < 0) {
2135 err = ret;
2136 goto out;
2137 }
2138 if (ret > 0) {
2139 if (metadata) {
2140 btrfs_release_path(path);
2141 metadata = 0;
2142
2143 key.offset = node->num_bytes;
2144 key.type = BTRFS_EXTENT_ITEM_KEY;
2145 goto again;
2146 }
2147 err = -EIO;
2148 goto out;
2149 }
2150
2151 leaf = path->nodes[0];
2152 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2153#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
2154 if (item_size < sizeof(*ei)) {
2155 ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
2156 path, (u64)-1, 0);
2157 if (ret < 0) {
2158 err = ret;
2159 goto out;
2160 }
2161 leaf = path->nodes[0];
2162 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2163 }
2164#endif
2165 BUG_ON(item_size < sizeof(*ei));
2166 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2167 __run_delayed_extent_op(extent_op, leaf, ei);
2168
2169 btrfs_mark_buffer_dirty(leaf);
2170out:
2171 btrfs_free_path(path);
2172 return err;
2173}
2174
2175static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
2176 struct btrfs_root *root,
2177 struct btrfs_delayed_ref_node *node,
2178 struct btrfs_delayed_extent_op *extent_op,
2179 int insert_reserved)
2180{
2181 int ret = 0;
2182 struct btrfs_delayed_tree_ref *ref;
2183 struct btrfs_key ins;
2184 u64 parent = 0;
2185 u64 ref_root = 0;
2186 bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
2187 SKINNY_METADATA);
2188
2189 ref = btrfs_delayed_node_to_tree_ref(node);
2190 trace_run_delayed_tree_ref(node, ref, node->action);
2191
2192 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2193 parent = ref->parent;
2194 else
2195 ref_root = ref->root;
2196
2197 ins.objectid = node->bytenr;
2198 if (skinny_metadata) {
2199 ins.offset = ref->level;
2200 ins.type = BTRFS_METADATA_ITEM_KEY;
2201 } else {
2202 ins.offset = node->num_bytes;
2203 ins.type = BTRFS_EXTENT_ITEM_KEY;
2204 }
2205
2206 BUG_ON(node->ref_mod != 1);
2207 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2208 BUG_ON(!extent_op || !extent_op->update_flags);
2209 ret = alloc_reserved_tree_block(trans, root,
2210 parent, ref_root,
2211 extent_op->flags_to_set,
2212 &extent_op->key,
2213 ref->level, &ins);
2214 } else if (node->action == BTRFS_ADD_DELAYED_REF) {
2215 ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
2216 node->num_bytes, parent, ref_root,
2217 ref->level, 0, 1, extent_op);
2218 } else if (node->action == BTRFS_DROP_DELAYED_REF) {
2219 ret = __btrfs_free_extent(trans, root, node->bytenr,
2220 node->num_bytes, parent, ref_root,
2221 ref->level, 0, 1, extent_op);
2222 } else {
2223 BUG();
2224 }
2225 return ret;
2226}
2227
2228
2229static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
2230 struct btrfs_root *root,
2231 struct btrfs_delayed_ref_node *node,
2232 struct btrfs_delayed_extent_op *extent_op,
2233 int insert_reserved)
2234{
2235 int ret = 0;
2236
2237 if (trans->aborted)
2238 return 0;
2239
2240 if (btrfs_delayed_ref_is_head(node)) {
2241 struct btrfs_delayed_ref_head *head;
2242
2243
2244
2245
2246
2247
2248 BUG_ON(extent_op);
2249 head = btrfs_delayed_node_to_head(node);
2250 trace_run_delayed_ref_head(node, head, node->action);
2251
2252 if (insert_reserved) {
2253 btrfs_pin_extent(root, node->bytenr,
2254 node->num_bytes, 1);
2255 if (head->is_data) {
2256 ret = btrfs_del_csums(trans, root,
2257 node->bytenr,
2258 node->num_bytes);
2259 }
2260 }
2261 return ret;
2262 }
2263
2264 if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
2265 node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2266 ret = run_delayed_tree_ref(trans, root, node, extent_op,
2267 insert_reserved);
2268 else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
2269 node->type == BTRFS_SHARED_DATA_REF_KEY)
2270 ret = run_delayed_data_ref(trans, root, node, extent_op,
2271 insert_reserved);
2272 else
2273 BUG();
2274 return ret;
2275}
2276
2277static noinline struct btrfs_delayed_ref_node *
2278select_delayed_ref(struct btrfs_delayed_ref_head *head)
2279{
2280 struct rb_node *node;
2281 struct btrfs_delayed_ref_node *ref;
2282 int action = BTRFS_ADD_DELAYED_REF;
2283again:
2284
2285
2286
2287
2288
2289 node = rb_prev(&head->node.rb_node);
2290 while (1) {
2291 if (!node)
2292 break;
2293 ref = rb_entry(node, struct btrfs_delayed_ref_node,
2294 rb_node);
2295 if (ref->bytenr != head->node.bytenr)
2296 break;
2297 if (ref->action == action)
2298 return ref;
2299 node = rb_prev(node);
2300 }
2301 if (action == BTRFS_ADD_DELAYED_REF) {
2302 action = BTRFS_DROP_DELAYED_REF;
2303 goto again;
2304 }
2305 return NULL;
2306}
2307
2308
2309
2310
2311
2312static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
2313 struct btrfs_root *root,
2314 struct list_head *cluster)
2315{
2316 struct btrfs_delayed_ref_root *delayed_refs;
2317 struct btrfs_delayed_ref_node *ref;
2318 struct btrfs_delayed_ref_head *locked_ref = NULL;
2319 struct btrfs_delayed_extent_op *extent_op;
2320 struct btrfs_fs_info *fs_info = root->fs_info;
2321 int ret;
2322 int count = 0;
2323 int must_insert_reserved = 0;
2324
2325 delayed_refs = &trans->transaction->delayed_refs;
2326 while (1) {
2327 if (!locked_ref) {
2328
2329 if (list_empty(cluster))
2330 break;
2331
2332 locked_ref = list_entry(cluster->next,
2333 struct btrfs_delayed_ref_head, cluster);
2334
2335
2336
2337 ret = btrfs_delayed_ref_lock(trans, locked_ref);
2338
2339
2340
2341
2342
2343
2344
2345 if (ret == -EAGAIN) {
2346 locked_ref = NULL;
2347 count++;
2348 continue;
2349 }
2350 }
2351
2352
2353
2354
2355
2356
2357
2358
2359 btrfs_merge_delayed_refs(trans, fs_info, delayed_refs,
2360 locked_ref);
2361
2362
2363
2364
2365
2366 ref = select_delayed_ref(locked_ref);
2367
2368 if (ref && ref->seq &&
2369 btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) {
2370
2371
2372
2373
2374 list_del_init(&locked_ref->cluster);
2375 btrfs_delayed_ref_unlock(locked_ref);
2376 locked_ref = NULL;
2377 delayed_refs->num_heads_ready++;
2378 spin_unlock(&delayed_refs->lock);
2379 cond_resched();
2380 spin_lock(&delayed_refs->lock);
2381 continue;
2382 }
2383
2384
2385
2386
2387
2388 must_insert_reserved = locked_ref->must_insert_reserved;
2389 locked_ref->must_insert_reserved = 0;
2390
2391 extent_op = locked_ref->extent_op;
2392 locked_ref->extent_op = NULL;
2393
2394 if (!ref) {
2395
2396
2397
2398
2399 ref = &locked_ref->node;
2400
2401 if (extent_op && must_insert_reserved) {
2402 btrfs_free_delayed_extent_op(extent_op);
2403 extent_op = NULL;
2404 }
2405
2406 if (extent_op) {
2407 spin_unlock(&delayed_refs->lock);
2408
2409 ret = run_delayed_extent_op(trans, root,
2410 ref, extent_op);
2411 btrfs_free_delayed_extent_op(extent_op);
2412
2413 if (ret) {
2414 btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret);
2415 spin_lock(&delayed_refs->lock);
2416 btrfs_delayed_ref_unlock(locked_ref);
2417 return ret;
2418 }
2419
2420 goto next;
2421 }
2422 }
2423
2424 ref->in_tree = 0;
2425 rb_erase(&ref->rb_node, &delayed_refs->root);
2426 delayed_refs->num_entries--;
2427 if (!btrfs_delayed_ref_is_head(ref)) {
2428
2429
2430
2431
2432 switch (ref->action) {
2433 case BTRFS_ADD_DELAYED_REF:
2434 case BTRFS_ADD_DELAYED_EXTENT:
2435 locked_ref->node.ref_mod -= ref->ref_mod;
2436 break;
2437 case BTRFS_DROP_DELAYED_REF:
2438 locked_ref->node.ref_mod += ref->ref_mod;
2439 break;
2440 default:
2441 WARN_ON(1);
2442 }
2443 } else {
2444 list_del_init(&locked_ref->cluster);
2445 }
2446 spin_unlock(&delayed_refs->lock);
2447
2448 ret = run_one_delayed_ref(trans, root, ref, extent_op,
2449 must_insert_reserved);
2450
2451 btrfs_free_delayed_extent_op(extent_op);
2452 if (ret) {
2453 btrfs_delayed_ref_unlock(locked_ref);
2454 btrfs_put_delayed_ref(ref);
2455 btrfs_debug(fs_info, "run_one_delayed_ref returned %d", ret);
2456 spin_lock(&delayed_refs->lock);
2457 return ret;
2458 }
2459
2460
2461
2462
2463
2464
2465
2466 if (btrfs_delayed_ref_is_head(ref)) {
2467 btrfs_delayed_ref_unlock(locked_ref);
2468 locked_ref = NULL;
2469 }
2470 btrfs_put_delayed_ref(ref);
2471 count++;
2472next:
2473 cond_resched();
2474 spin_lock(&delayed_refs->lock);
2475 }
2476 return count;
2477}
2478
2479#ifdef SCRAMBLE_DELAYED_REFS
2480
2481
2482
2483
2484
2485static u64 find_middle(struct rb_root *root)
2486{
2487 struct rb_node *n = root->rb_node;
2488 struct btrfs_delayed_ref_node *entry;
2489 int alt = 1;
2490 u64 middle;
2491 u64 first = 0, last = 0;
2492
2493 n = rb_first(root);
2494 if (n) {
2495 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2496 first = entry->bytenr;
2497 }
2498 n = rb_last(root);
2499 if (n) {
2500 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2501 last = entry->bytenr;
2502 }
2503 n = root->rb_node;
2504
2505 while (n) {
2506 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2507 WARN_ON(!entry->in_tree);
2508
2509 middle = entry->bytenr;
2510
2511 if (alt)
2512 n = n->rb_left;
2513 else
2514 n = n->rb_right;
2515
2516 alt = 1 - alt;
2517 }
2518 return middle;
2519}
2520#endif
2521
2522int btrfs_delayed_refs_qgroup_accounting(struct btrfs_trans_handle *trans,
2523 struct btrfs_fs_info *fs_info)
2524{
2525 struct qgroup_update *qgroup_update;
2526 int ret = 0;
2527
2528 if (list_empty(&trans->qgroup_ref_list) !=
2529 !trans->delayed_ref_elem.seq) {
2530
2531 btrfs_err(fs_info,
2532 "qgroup accounting update error, list is%s empty, seq is %#x.%x",
2533 list_empty(&trans->qgroup_ref_list) ? "" : " not",
2534 (u32)(trans->delayed_ref_elem.seq >> 32),
2535 (u32)trans->delayed_ref_elem.seq);
2536 BUG();
2537 }
2538
2539 if (!trans->delayed_ref_elem.seq)
2540 return 0;
2541
2542 while (!list_empty(&trans->qgroup_ref_list)) {
2543 qgroup_update = list_first_entry(&trans->qgroup_ref_list,
2544 struct qgroup_update, list);
2545 list_del(&qgroup_update->list);
2546 if (!ret)
2547 ret = btrfs_qgroup_account_ref(
2548 trans, fs_info, qgroup_update->node,
2549 qgroup_update->extent_op);
2550 kfree(qgroup_update);
2551 }
2552
2553 btrfs_put_tree_mod_seq(fs_info, &trans->delayed_ref_elem);
2554
2555 return ret;
2556}
2557
2558static int refs_newer(struct btrfs_delayed_ref_root *delayed_refs, int seq,
2559 int count)
2560{
2561 int val = atomic_read(&delayed_refs->ref_seq);
2562
2563 if (val < seq || val >= seq + count)
2564 return 1;
2565 return 0;
2566}
2567
2568static inline u64 heads_to_leaves(struct btrfs_root *root, u64 heads)
2569{
2570 u64 num_bytes;
2571
2572 num_bytes = heads * (sizeof(struct btrfs_extent_item) +
2573 sizeof(struct btrfs_extent_inline_ref));
2574 if (!btrfs_fs_incompat(root->fs_info, SKINNY_METADATA))
2575 num_bytes += heads * sizeof(struct btrfs_tree_block_info);
2576
2577
2578
2579
2580
2581 return div64_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(root));
2582}
2583
2584int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans,
2585 struct btrfs_root *root)
2586{
2587 struct btrfs_block_rsv *global_rsv;
2588 u64 num_heads = trans->transaction->delayed_refs.num_heads_ready;
2589 u64 num_bytes;
2590 int ret = 0;
2591
2592 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
2593 num_heads = heads_to_leaves(root, num_heads);
2594 if (num_heads > 1)
2595 num_bytes += (num_heads - 1) * root->leafsize;
2596 num_bytes <<= 1;
2597 global_rsv = &root->fs_info->global_block_rsv;
2598
2599
2600
2601
2602
2603 if (global_rsv->space_info->full)
2604 num_bytes <<= 1;
2605
2606 spin_lock(&global_rsv->lock);
2607 if (global_rsv->reserved <= num_bytes)
2608 ret = 1;
2609 spin_unlock(&global_rsv->lock);
2610 return ret;
2611}
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
2624 struct btrfs_root *root, unsigned long count)
2625{
2626 struct rb_node *node;
2627 struct btrfs_delayed_ref_root *delayed_refs;
2628 struct btrfs_delayed_ref_node *ref;
2629 struct list_head cluster;
2630 int ret;
2631 u64 delayed_start;
2632 int run_all = count == (unsigned long)-1;
2633 int run_most = 0;
2634 int loops;
2635
2636
2637 if (trans->aborted)
2638 return 0;
2639
2640 if (root == root->fs_info->extent_root)
2641 root = root->fs_info->tree_root;
2642
2643 btrfs_delayed_refs_qgroup_accounting(trans, root->fs_info);
2644
2645 delayed_refs = &trans->transaction->delayed_refs;
2646 INIT_LIST_HEAD(&cluster);
2647 if (count == 0) {
2648 count = delayed_refs->num_entries * 2;
2649 run_most = 1;
2650 }
2651
2652 if (!run_all && !run_most) {
2653 int old;
2654 int seq = atomic_read(&delayed_refs->ref_seq);
2655
2656progress:
2657 old = atomic_cmpxchg(&delayed_refs->procs_running_refs, 0, 1);
2658 if (old) {
2659 DEFINE_WAIT(__wait);
2660 if (delayed_refs->flushing ||
2661 !btrfs_should_throttle_delayed_refs(trans, root))
2662 return 0;
2663
2664 prepare_to_wait(&delayed_refs->wait, &__wait,
2665 TASK_UNINTERRUPTIBLE);
2666
2667 old = atomic_cmpxchg(&delayed_refs->procs_running_refs, 0, 1);
2668 if (old) {
2669 schedule();
2670 finish_wait(&delayed_refs->wait, &__wait);
2671
2672 if (!refs_newer(delayed_refs, seq, 256))
2673 goto progress;
2674 else
2675 return 0;
2676 } else {
2677 finish_wait(&delayed_refs->wait, &__wait);
2678 goto again;
2679 }
2680 }
2681
2682 } else {
2683 atomic_inc(&delayed_refs->procs_running_refs);
2684 }
2685
2686again:
2687 loops = 0;
2688 spin_lock(&delayed_refs->lock);
2689
2690#ifdef SCRAMBLE_DELAYED_REFS
2691 delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
2692#endif
2693
2694 while (1) {
2695 if (!(run_all || run_most) &&
2696 !btrfs_should_throttle_delayed_refs(trans, root))
2697 break;
2698
2699
2700
2701
2702
2703
2704
2705 delayed_start = delayed_refs->run_delayed_start;
2706 ret = btrfs_find_ref_cluster(trans, &cluster,
2707 delayed_refs->run_delayed_start);
2708 if (ret)
2709 break;
2710
2711 ret = run_clustered_refs(trans, root, &cluster);
2712 if (ret < 0) {
2713 btrfs_release_ref_cluster(&cluster);
2714 spin_unlock(&delayed_refs->lock);
2715 btrfs_abort_transaction(trans, root, ret);
2716 atomic_dec(&delayed_refs->procs_running_refs);
2717 wake_up(&delayed_refs->wait);
2718 return ret;
2719 }
2720
2721 atomic_add(ret, &delayed_refs->ref_seq);
2722
2723 count -= min_t(unsigned long, ret, count);
2724
2725 if (count == 0)
2726 break;
2727
2728 if (delayed_start >= delayed_refs->run_delayed_start) {
2729 if (loops == 0) {
2730
2731
2732
2733
2734
2735
2736 loops = 1;
2737 } else {
2738
2739
2740
2741 BUG_ON(run_all);
2742 break;
2743 }
2744 }
2745 if (ret) {
2746
2747 loops = 0;
2748 }
2749 }
2750
2751 if (run_all) {
2752 if (!list_empty(&trans->new_bgs)) {
2753 spin_unlock(&delayed_refs->lock);
2754 btrfs_create_pending_block_groups(trans, root);
2755 spin_lock(&delayed_refs->lock);
2756 }
2757
2758 node = rb_first(&delayed_refs->root);
2759 if (!node)
2760 goto out;
2761 count = (unsigned long)-1;
2762
2763 while (node) {
2764 ref = rb_entry(node, struct btrfs_delayed_ref_node,
2765 rb_node);
2766 if (btrfs_delayed_ref_is_head(ref)) {
2767 struct btrfs_delayed_ref_head *head;
2768
2769 head = btrfs_delayed_node_to_head(ref);
2770 atomic_inc(&ref->refs);
2771
2772 spin_unlock(&delayed_refs->lock);
2773
2774
2775
2776
2777 mutex_lock(&head->mutex);
2778 mutex_unlock(&head->mutex);
2779
2780 btrfs_put_delayed_ref(ref);
2781 cond_resched();
2782 goto again;
2783 }
2784 node = rb_next(node);
2785 }
2786 spin_unlock(&delayed_refs->lock);
2787 schedule_timeout(1);
2788 goto again;
2789 }
2790out:
2791 atomic_dec(&delayed_refs->procs_running_refs);
2792 smp_mb();
2793 if (waitqueue_active(&delayed_refs->wait))
2794 wake_up(&delayed_refs->wait);
2795
2796 spin_unlock(&delayed_refs->lock);
2797 assert_qgroups_uptodate(trans);
2798 return 0;
2799}
2800
2801int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
2802 struct btrfs_root *root,
2803 u64 bytenr, u64 num_bytes, u64 flags,
2804 int level, int is_data)
2805{
2806 struct btrfs_delayed_extent_op *extent_op;
2807 int ret;
2808
2809 extent_op = btrfs_alloc_delayed_extent_op();
2810 if (!extent_op)
2811 return -ENOMEM;
2812
2813 extent_op->flags_to_set = flags;
2814 extent_op->update_flags = 1;
2815 extent_op->update_key = 0;
2816 extent_op->is_data = is_data ? 1 : 0;
2817 extent_op->level = level;
2818
2819 ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
2820 num_bytes, extent_op);
2821 if (ret)
2822 btrfs_free_delayed_extent_op(extent_op);
2823 return ret;
2824}
2825
2826static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
2827 struct btrfs_root *root,
2828 struct btrfs_path *path,
2829 u64 objectid, u64 offset, u64 bytenr)
2830{
2831 struct btrfs_delayed_ref_head *head;
2832 struct btrfs_delayed_ref_node *ref;
2833 struct btrfs_delayed_data_ref *data_ref;
2834 struct btrfs_delayed_ref_root *delayed_refs;
2835 struct rb_node *node;
2836 int ret = 0;
2837
2838 ret = -ENOENT;
2839 delayed_refs = &trans->transaction->delayed_refs;
2840 spin_lock(&delayed_refs->lock);
2841 head = btrfs_find_delayed_ref_head(trans, bytenr);
2842 if (!head)
2843 goto out;
2844
2845 if (!mutex_trylock(&head->mutex)) {
2846 atomic_inc(&head->node.refs);
2847 spin_unlock(&delayed_refs->lock);
2848
2849 btrfs_release_path(path);
2850
2851
2852
2853
2854
2855 mutex_lock(&head->mutex);
2856 mutex_unlock(&head->mutex);
2857 btrfs_put_delayed_ref(&head->node);
2858 return -EAGAIN;
2859 }
2860
2861 node = rb_prev(&head->node.rb_node);
2862 if (!node)
2863 goto out_unlock;
2864
2865 ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2866
2867 if (ref->bytenr != bytenr)
2868 goto out_unlock;
2869
2870 ret = 1;
2871 if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
2872 goto out_unlock;
2873
2874 data_ref = btrfs_delayed_node_to_data_ref(ref);
2875
2876 node = rb_prev(node);
2877 if (node) {
2878 int seq = ref->seq;
2879
2880 ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2881 if (ref->bytenr == bytenr && ref->seq == seq)
2882 goto out_unlock;
2883 }
2884
2885 if (data_ref->root != root->root_key.objectid ||
2886 data_ref->objectid != objectid || data_ref->offset != offset)
2887 goto out_unlock;
2888
2889 ret = 0;
2890out_unlock:
2891 mutex_unlock(&head->mutex);
2892out:
2893 spin_unlock(&delayed_refs->lock);
2894 return ret;
2895}
2896
2897static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
2898 struct btrfs_root *root,
2899 struct btrfs_path *path,
2900 u64 objectid, u64 offset, u64 bytenr)
2901{
2902 struct btrfs_root *extent_root = root->fs_info->extent_root;
2903 struct extent_buffer *leaf;
2904 struct btrfs_extent_data_ref *ref;
2905 struct btrfs_extent_inline_ref *iref;
2906 struct btrfs_extent_item *ei;
2907 struct btrfs_key key;
2908 u32 item_size;
2909 int ret;
2910
2911 key.objectid = bytenr;
2912 key.offset = (u64)-1;
2913 key.type = BTRFS_EXTENT_ITEM_KEY;
2914
2915 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
2916 if (ret < 0)
2917 goto out;
2918 BUG_ON(ret == 0);
2919
2920 ret = -ENOENT;
2921 if (path->slots[0] == 0)
2922 goto out;
2923
2924 path->slots[0]--;
2925 leaf = path->nodes[0];
2926 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2927
2928 if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2929 goto out;
2930
2931 ret = 1;
2932 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2933#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
2934 if (item_size < sizeof(*ei)) {
2935 WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
2936 goto out;
2937 }
2938#endif
2939 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2940
2941 if (item_size != sizeof(*ei) +
2942 btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
2943 goto out;
2944
2945 if (btrfs_extent_generation(leaf, ei) <=
2946 btrfs_root_last_snapshot(&root->root_item))
2947 goto out;
2948
2949 iref = (struct btrfs_extent_inline_ref *)(ei + 1);
2950 if (btrfs_extent_inline_ref_type(leaf, iref) !=
2951 BTRFS_EXTENT_DATA_REF_KEY)
2952 goto out;
2953
2954 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
2955 if (btrfs_extent_refs(leaf, ei) !=
2956 btrfs_extent_data_ref_count(leaf, ref) ||
2957 btrfs_extent_data_ref_root(leaf, ref) !=
2958 root->root_key.objectid ||
2959 btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
2960 btrfs_extent_data_ref_offset(leaf, ref) != offset)
2961 goto out;
2962
2963 ret = 0;
2964out:
2965 return ret;
2966}
2967
2968int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
2969 struct btrfs_root *root,
2970 u64 objectid, u64 offset, u64 bytenr)
2971{
2972 struct btrfs_path *path;
2973 int ret;
2974 int ret2;
2975
2976 path = btrfs_alloc_path();
2977 if (!path)
2978 return -ENOENT;
2979
2980 do {
2981 ret = check_committed_ref(trans, root, path, objectid,
2982 offset, bytenr);
2983 if (ret && ret != -ENOENT)
2984 goto out;
2985
2986 ret2 = check_delayed_ref(trans, root, path, objectid,
2987 offset, bytenr);
2988 } while (ret2 == -EAGAIN);
2989
2990 if (ret2 && ret2 != -ENOENT) {
2991 ret = ret2;
2992 goto out;
2993 }
2994
2995 if (ret != -ENOENT || ret2 != -ENOENT)
2996 ret = 0;
2997out:
2998 btrfs_free_path(path);
2999 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
3000 WARN_ON(ret > 0);
3001 return ret;
3002}
3003
3004static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
3005 struct btrfs_root *root,
3006 struct extent_buffer *buf,
3007 int full_backref, int inc, int for_cow)
3008{
3009 u64 bytenr;
3010 u64 num_bytes;
3011 u64 parent;
3012 u64 ref_root;
3013 u32 nritems;
3014 struct btrfs_key key;
3015 struct btrfs_file_extent_item *fi;
3016 int i;
3017 int level;
3018 int ret = 0;
3019 int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
3020 u64, u64, u64, u64, u64, u64, int);
3021
3022 ref_root = btrfs_header_owner(buf);
3023 nritems = btrfs_header_nritems(buf);
3024 level = btrfs_header_level(buf);
3025
3026 if (!root->ref_cows && level == 0)
3027 return 0;
3028
3029 if (inc)
3030 process_func = btrfs_inc_extent_ref;
3031 else
3032 process_func = btrfs_free_extent;
3033
3034 if (full_backref)
3035 parent = buf->start;
3036 else
3037 parent = 0;
3038
3039 for (i = 0; i < nritems; i++) {
3040 if (level == 0) {
3041 btrfs_item_key_to_cpu(buf, &key, i);
3042 if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
3043 continue;
3044 fi = btrfs_item_ptr(buf, i,
3045 struct btrfs_file_extent_item);
3046 if (btrfs_file_extent_type(buf, fi) ==
3047 BTRFS_FILE_EXTENT_INLINE)
3048 continue;
3049 bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
3050 if (bytenr == 0)
3051 continue;
3052
3053 num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
3054 key.offset -= btrfs_file_extent_offset(buf, fi);
3055 ret = process_func(trans, root, bytenr, num_bytes,
3056 parent, ref_root, key.objectid,
3057 key.offset, for_cow);
3058 if (ret)
3059 goto fail;
3060 } else {
3061 bytenr = btrfs_node_blockptr(buf, i);
3062 num_bytes = btrfs_level_size(root, level - 1);
3063 ret = process_func(trans, root, bytenr, num_bytes,
3064 parent, ref_root, level - 1, 0,
3065 for_cow);
3066 if (ret)
3067 goto fail;
3068 }
3069 }
3070 return 0;
3071fail:
3072 return ret;
3073}
3074
3075int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3076 struct extent_buffer *buf, int full_backref, int for_cow)
3077{
3078 return __btrfs_mod_ref(trans, root, buf, full_backref, 1, for_cow);
3079}
3080
3081int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3082 struct extent_buffer *buf, int full_backref, int for_cow)
3083{
3084 return __btrfs_mod_ref(trans, root, buf, full_backref, 0, for_cow);
3085}
3086
3087static int write_one_cache_group(struct btrfs_trans_handle *trans,
3088 struct btrfs_root *root,
3089 struct btrfs_path *path,
3090 struct btrfs_block_group_cache *cache)
3091{
3092 int ret;
3093 struct btrfs_root *extent_root = root->fs_info->extent_root;
3094 unsigned long bi;
3095 struct extent_buffer *leaf;
3096
3097 ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
3098 if (ret < 0)
3099 goto fail;
3100 BUG_ON(ret);
3101
3102 leaf = path->nodes[0];
3103 bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
3104 write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
3105 btrfs_mark_buffer_dirty(leaf);
3106 btrfs_release_path(path);
3107fail:
3108 if (ret) {
3109 btrfs_abort_transaction(trans, root, ret);
3110 return ret;
3111 }
3112 return 0;
3113
3114}
3115
3116static struct btrfs_block_group_cache *
3117next_block_group(struct btrfs_root *root,
3118 struct btrfs_block_group_cache *cache)
3119{
3120 struct rb_node *node;
3121 spin_lock(&root->fs_info->block_group_cache_lock);
3122 node = rb_next(&cache->cache_node);
3123 btrfs_put_block_group(cache);
3124 if (node) {
3125 cache = rb_entry(node, struct btrfs_block_group_cache,
3126 cache_node);
3127 btrfs_get_block_group(cache);
3128 } else
3129 cache = NULL;
3130 spin_unlock(&root->fs_info->block_group_cache_lock);
3131 return cache;
3132}
3133
3134static int cache_save_setup(struct btrfs_block_group_cache *block_group,
3135 struct btrfs_trans_handle *trans,
3136 struct btrfs_path *path)
3137{
3138 struct btrfs_root *root = block_group->fs_info->tree_root;
3139 struct inode *inode = NULL;
3140 u64 alloc_hint = 0;
3141 int dcs = BTRFS_DC_ERROR;
3142 int num_pages = 0;
3143 int retries = 0;
3144 int ret = 0;
3145
3146
3147
3148
3149
3150 if (block_group->key.offset < (100 * 1024 * 1024)) {
3151 spin_lock(&block_group->lock);
3152 block_group->disk_cache_state = BTRFS_DC_WRITTEN;
3153 spin_unlock(&block_group->lock);
3154 return 0;
3155 }
3156
3157again:
3158 inode = lookup_free_space_inode(root, block_group, path);
3159 if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
3160 ret = PTR_ERR(inode);
3161 btrfs_release_path(path);
3162 goto out;
3163 }
3164
3165 if (IS_ERR(inode)) {
3166 BUG_ON(retries);
3167 retries++;
3168
3169 if (block_group->ro)
3170 goto out_free;
3171
3172 ret = create_free_space_inode(root, trans, block_group, path);
3173 if (ret)
3174 goto out_free;
3175 goto again;
3176 }
3177
3178
3179 if (block_group->cache_generation == trans->transid &&
3180 i_size_read(inode)) {
3181 dcs = BTRFS_DC_SETUP;
3182 goto out_put;
3183 }
3184
3185
3186
3187
3188
3189
3190 BTRFS_I(inode)->generation = 0;
3191 ret = btrfs_update_inode(trans, root, inode);
3192 WARN_ON(ret);
3193
3194 if (i_size_read(inode) > 0) {
3195 ret = btrfs_check_trunc_cache_free_space(root,
3196 &root->fs_info->global_block_rsv);
3197 if (ret)
3198 goto out_put;
3199
3200 ret = btrfs_truncate_free_space_cache(root, trans, path,
3201 inode);
3202 if (ret)
3203 goto out_put;
3204 }
3205
3206 spin_lock(&block_group->lock);
3207 if (block_group->cached != BTRFS_CACHE_FINISHED ||
3208 !btrfs_test_opt(root, SPACE_CACHE)) {
3209
3210
3211
3212
3213
3214 dcs = BTRFS_DC_WRITTEN;
3215 spin_unlock(&block_group->lock);
3216 goto out_put;
3217 }
3218 spin_unlock(&block_group->lock);
3219
3220
3221
3222
3223
3224
3225
3226 num_pages = (int)div64_u64(block_group->key.offset, 256 * 1024 * 1024);
3227 if (!num_pages)
3228 num_pages = 1;
3229
3230 num_pages *= 16;
3231 num_pages *= PAGE_CACHE_SIZE;
3232
3233 ret = btrfs_check_data_free_space(inode, num_pages);
3234 if (ret)
3235 goto out_put;
3236
3237 ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
3238 num_pages, num_pages,
3239 &alloc_hint);
3240 if (!ret)
3241 dcs = BTRFS_DC_SETUP;
3242 btrfs_free_reserved_data_space(inode, num_pages);
3243
3244out_put:
3245 iput(inode);
3246out_free:
3247 btrfs_release_path(path);
3248out:
3249 spin_lock(&block_group->lock);
3250 if (!ret && dcs == BTRFS_DC_SETUP)
3251 block_group->cache_generation = trans->transid;
3252 block_group->disk_cache_state = dcs;
3253 spin_unlock(&block_group->lock);
3254
3255 return ret;
3256}
3257
3258int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
3259 struct btrfs_root *root)
3260{
3261 struct btrfs_block_group_cache *cache;
3262 int err = 0;
3263 struct btrfs_path *path;
3264 u64 last = 0;
3265
3266 path = btrfs_alloc_path();
3267 if (!path)
3268 return -ENOMEM;
3269
3270again:
3271 while (1) {
3272 cache = btrfs_lookup_first_block_group(root->fs_info, last);
3273 while (cache) {
3274 if (cache->disk_cache_state == BTRFS_DC_CLEAR)
3275 break;
3276 cache = next_block_group(root, cache);
3277 }
3278 if (!cache) {
3279 if (last == 0)
3280 break;
3281 last = 0;
3282 continue;
3283 }
3284 err = cache_save_setup(cache, trans, path);
3285 last = cache->key.objectid + cache->key.offset;
3286 btrfs_put_block_group(cache);
3287 }
3288
3289 while (1) {
3290 if (last == 0) {
3291 err = btrfs_run_delayed_refs(trans, root,
3292 (unsigned long)-1);
3293 if (err)
3294 goto out;
3295 }
3296
3297 cache = btrfs_lookup_first_block_group(root->fs_info, last);
3298 while (cache) {
3299 if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
3300 btrfs_put_block_group(cache);
3301 goto again;
3302 }
3303
3304 if (cache->dirty)
3305 break;
3306 cache = next_block_group(root, cache);
3307 }
3308 if (!cache) {
3309 if (last == 0)
3310 break;
3311 last = 0;
3312 continue;
3313 }
3314
3315 if (cache->disk_cache_state == BTRFS_DC_SETUP)
3316 cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
3317 cache->dirty = 0;
3318 last = cache->key.objectid + cache->key.offset;
3319
3320 err = write_one_cache_group(trans, root, path, cache);
3321 if (err)
3322 goto out;
3323
3324 btrfs_put_block_group(cache);
3325 }
3326
3327 while (1) {
3328
3329
3330
3331
3332
3333 if (last == 0) {
3334 err = btrfs_run_delayed_refs(trans, root,
3335 (unsigned long)-1);
3336 if (err)
3337 goto out;
3338 }
3339
3340 cache = btrfs_lookup_first_block_group(root->fs_info, last);
3341 while (cache) {
3342
3343
3344
3345
3346
3347 if (cache->dirty) {
3348 btrfs_put_block_group(cache);
3349 goto again;
3350 }
3351 if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
3352 break;
3353 cache = next_block_group(root, cache);
3354 }
3355 if (!cache) {
3356 if (last == 0)
3357 break;
3358 last = 0;
3359 continue;
3360 }
3361
3362 err = btrfs_write_out_cache(root, trans, cache, path);
3363
3364
3365
3366
3367
3368 if (!err && cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
3369 cache->disk_cache_state = BTRFS_DC_WRITTEN;
3370 last = cache->key.objectid + cache->key.offset;
3371 btrfs_put_block_group(cache);
3372 }
3373out:
3374
3375 btrfs_free_path(path);
3376 return err;
3377}
3378
3379int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
3380{
3381 struct btrfs_block_group_cache *block_group;
3382 int readonly = 0;
3383
3384 block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
3385 if (!block_group || block_group->ro)
3386 readonly = 1;
3387 if (block_group)
3388 btrfs_put_block_group(block_group);
3389 return readonly;
3390}
3391
3392static int update_space_info(struct btrfs_fs_info *info, u64 flags,
3393 u64 total_bytes, u64 bytes_used,
3394 struct btrfs_space_info **space_info)
3395{
3396 struct btrfs_space_info *found;
3397 int i;
3398 int factor;
3399 int ret;
3400
3401 if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
3402 BTRFS_BLOCK_GROUP_RAID10))
3403 factor = 2;
3404 else
3405 factor = 1;
3406
3407 found = __find_space_info(info, flags);
3408 if (found) {
3409 spin_lock(&found->lock);
3410 found->total_bytes += total_bytes;
3411 found->disk_total += total_bytes * factor;
3412 found->bytes_used += bytes_used;
3413 found->disk_used += bytes_used * factor;
3414 found->full = 0;
3415 spin_unlock(&found->lock);
3416 *space_info = found;
3417 return 0;
3418 }
3419 found = kzalloc(sizeof(*found), GFP_NOFS);
3420 if (!found)
3421 return -ENOMEM;
3422
3423 ret = percpu_counter_init(&found->total_bytes_pinned, 0);
3424 if (ret) {
3425 kfree(found);
3426 return ret;
3427 }
3428
3429 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
3430 INIT_LIST_HEAD(&found->block_groups[i]);
3431 init_rwsem(&found->groups_sem);
3432 spin_lock_init(&found->lock);
3433 found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3434 found->total_bytes = total_bytes;
3435 found->disk_total = total_bytes * factor;
3436 found->bytes_used = bytes_used;
3437 found->disk_used = bytes_used * factor;
3438 found->bytes_pinned = 0;
3439 found->bytes_reserved = 0;
3440 found->bytes_readonly = 0;
3441 found->bytes_may_use = 0;
3442 found->full = 0;
3443 found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3444 found->chunk_alloc = 0;
3445 found->flush = 0;
3446 init_waitqueue_head(&found->wait);
3447 *space_info = found;
3448 list_add_rcu(&found->list, &info->space_info);
3449 if (flags & BTRFS_BLOCK_GROUP_DATA)
3450 info->data_sinfo = found;
3451 return 0;
3452}
3453
3454static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
3455{
3456 u64 extra_flags = chunk_to_extended(flags) &
3457 BTRFS_EXTENDED_PROFILE_MASK;
3458
3459 write_seqlock(&fs_info->profiles_lock);
3460 if (flags & BTRFS_BLOCK_GROUP_DATA)
3461 fs_info->avail_data_alloc_bits |= extra_flags;
3462 if (flags & BTRFS_BLOCK_GROUP_METADATA)
3463 fs_info->avail_metadata_alloc_bits |= extra_flags;
3464 if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3465 fs_info->avail_system_alloc_bits |= extra_flags;
3466 write_sequnlock(&fs_info->profiles_lock);
3467}
3468
3469
3470
3471
3472
3473
3474
3475static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
3476{
3477 struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3478 u64 target = 0;
3479
3480 if (!bctl)
3481 return 0;
3482
3483 if (flags & BTRFS_BLOCK_GROUP_DATA &&
3484 bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3485 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
3486 } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
3487 bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3488 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
3489 } else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
3490 bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3491 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
3492 }
3493
3494 return target;
3495}
3496
3497
3498
3499
3500
3501
3502
3503
3504static u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3505{
3506
3507
3508
3509
3510
3511 u64 num_devices = root->fs_info->fs_devices->rw_devices +
3512 root->fs_info->fs_devices->missing_devices;
3513 u64 target;
3514 u64 tmp;
3515
3516
3517
3518
3519
3520 spin_lock(&root->fs_info->balance_lock);
3521 target = get_restripe_target(root->fs_info, flags);
3522 if (target) {
3523
3524 if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
3525 spin_unlock(&root->fs_info->balance_lock);
3526 return extended_to_chunk(target);
3527 }
3528 }
3529 spin_unlock(&root->fs_info->balance_lock);
3530
3531
3532 if (num_devices == 1)
3533 flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0 |
3534 BTRFS_BLOCK_GROUP_RAID5);
3535 if (num_devices < 3)
3536 flags &= ~BTRFS_BLOCK_GROUP_RAID6;
3537 if (num_devices < 4)
3538 flags &= ~BTRFS_BLOCK_GROUP_RAID10;
3539
3540 tmp = flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID0 |
3541 BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID5 |
3542 BTRFS_BLOCK_GROUP_RAID6 | BTRFS_BLOCK_GROUP_RAID10);
3543 flags &= ~tmp;
3544
3545 if (tmp & BTRFS_BLOCK_GROUP_RAID6)
3546 tmp = BTRFS_BLOCK_GROUP_RAID6;
3547 else if (tmp & BTRFS_BLOCK_GROUP_RAID5)
3548 tmp = BTRFS_BLOCK_GROUP_RAID5;
3549 else if (tmp & BTRFS_BLOCK_GROUP_RAID10)
3550 tmp = BTRFS_BLOCK_GROUP_RAID10;
3551 else if (tmp & BTRFS_BLOCK_GROUP_RAID1)
3552 tmp = BTRFS_BLOCK_GROUP_RAID1;
3553 else if (tmp & BTRFS_BLOCK_GROUP_RAID0)
3554 tmp = BTRFS_BLOCK_GROUP_RAID0;
3555
3556 return extended_to_chunk(flags | tmp);
3557}
3558
3559static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
3560{
3561 unsigned seq;
3562
3563 do {
3564 seq = read_seqbegin(&root->fs_info->profiles_lock);
3565
3566 if (flags & BTRFS_BLOCK_GROUP_DATA)
3567 flags |= root->fs_info->avail_data_alloc_bits;
3568 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3569 flags |= root->fs_info->avail_system_alloc_bits;
3570 else if (flags & BTRFS_BLOCK_GROUP_METADATA)
3571 flags |= root->fs_info->avail_metadata_alloc_bits;
3572 } while (read_seqretry(&root->fs_info->profiles_lock, seq));
3573
3574 return btrfs_reduce_alloc_profile(root, flags);
3575}
3576
3577u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
3578{
3579 u64 flags;
3580 u64 ret;
3581
3582 if (data)
3583 flags = BTRFS_BLOCK_GROUP_DATA;
3584 else if (root == root->fs_info->chunk_root)
3585 flags = BTRFS_BLOCK_GROUP_SYSTEM;
3586 else
3587 flags = BTRFS_BLOCK_GROUP_METADATA;
3588
3589 ret = get_alloc_profile(root, flags);
3590 return ret;
3591}
3592
3593
3594
3595
3596
3597int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
3598{
3599 struct btrfs_space_info *data_sinfo;
3600 struct btrfs_root *root = BTRFS_I(inode)->root;
3601 struct btrfs_fs_info *fs_info = root->fs_info;
3602 u64 used;
3603 int ret = 0, committed = 0, alloc_chunk = 1;
3604
3605
3606 bytes = ALIGN(bytes, root->sectorsize);
3607
3608 if (root == root->fs_info->tree_root ||
3609 BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3610 alloc_chunk = 0;
3611 committed = 1;
3612 }
3613
3614 data_sinfo = fs_info->data_sinfo;
3615 if (!data_sinfo)
3616 goto alloc;
3617
3618again:
3619
3620 spin_lock(&data_sinfo->lock);
3621 used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
3622 data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
3623 data_sinfo->bytes_may_use;
3624
3625 if (used + bytes > data_sinfo->total_bytes) {
3626 struct btrfs_trans_handle *trans;
3627
3628
3629
3630
3631
3632 if (!data_sinfo->full && alloc_chunk) {
3633 u64 alloc_target;
3634
3635 data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
3636 spin_unlock(&data_sinfo->lock);
3637alloc:
3638 alloc_target = btrfs_get_alloc_profile(root, 1);
3639 trans = btrfs_join_transaction(root);
3640 if (IS_ERR(trans))
3641 return PTR_ERR(trans);
3642
3643 ret = do_chunk_alloc(trans, root->fs_info->extent_root,
3644 alloc_target,
3645 CHUNK_ALLOC_NO_FORCE);
3646 btrfs_end_transaction(trans, root);
3647 if (ret < 0) {
3648 if (ret != -ENOSPC)
3649 return ret;
3650 else
3651 goto commit_trans;
3652 }
3653
3654 if (!data_sinfo)
3655 data_sinfo = fs_info->data_sinfo;
3656
3657 goto again;
3658 }
3659
3660
3661
3662
3663
3664 if (percpu_counter_compare(&data_sinfo->total_bytes_pinned,
3665 bytes) < 0)
3666 committed = 1;
3667 spin_unlock(&data_sinfo->lock);
3668
3669
3670commit_trans:
3671 if (!committed &&
3672 !atomic_read(&root->fs_info->open_ioctl_trans)) {
3673 committed = 1;
3674
3675 trans = btrfs_join_transaction(root);
3676 if (IS_ERR(trans))
3677 return PTR_ERR(trans);
3678 ret = btrfs_commit_transaction(trans, root);
3679 if (ret)
3680 return ret;
3681 goto again;
3682 }
3683
3684 return -ENOSPC;
3685 }
3686 data_sinfo->bytes_may_use += bytes;
3687 trace_btrfs_space_reservation(root->fs_info, "space_info",
3688 data_sinfo->flags, bytes, 1);
3689 spin_unlock(&data_sinfo->lock);
3690
3691 return 0;
3692}
3693
3694
3695
3696
3697void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3698{
3699 struct btrfs_root *root = BTRFS_I(inode)->root;
3700 struct btrfs_space_info *data_sinfo;
3701
3702
3703 bytes = ALIGN(bytes, root->sectorsize);
3704
3705 data_sinfo = root->fs_info->data_sinfo;
3706 spin_lock(&data_sinfo->lock);
3707 WARN_ON(data_sinfo->bytes_may_use < bytes);
3708 data_sinfo->bytes_may_use -= bytes;
3709 trace_btrfs_space_reservation(root->fs_info, "space_info",
3710 data_sinfo->flags, bytes, 0);
3711 spin_unlock(&data_sinfo->lock);
3712}
3713
3714static void force_metadata_allocation(struct btrfs_fs_info *info)
3715{
3716 struct list_head *head = &info->space_info;
3717 struct btrfs_space_info *found;
3718
3719 rcu_read_lock();
3720 list_for_each_entry_rcu(found, head, list) {
3721 if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3722 found->force_alloc = CHUNK_ALLOC_FORCE;
3723 }
3724 rcu_read_unlock();
3725}
3726
3727static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global)
3728{
3729 return (global->size << 1);
3730}
3731
3732static int should_alloc_chunk(struct btrfs_root *root,
3733 struct btrfs_space_info *sinfo, int force)
3734{
3735 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3736 u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3737 u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3738 u64 thresh;
3739
3740 if (force == CHUNK_ALLOC_FORCE)
3741 return 1;
3742
3743
3744
3745
3746
3747
3748 if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)
3749 num_allocated += calc_global_rsv_need_space(global_rsv);
3750
3751
3752
3753
3754
3755 if (force == CHUNK_ALLOC_LIMITED) {
3756 thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3757 thresh = max_t(u64, 64 * 1024 * 1024,
3758 div_factor_fine(thresh, 1));
3759
3760 if (num_bytes - num_allocated < thresh)
3761 return 1;
3762 }
3763
3764 if (num_allocated + 2 * 1024 * 1024 < div_factor(num_bytes, 8))
3765 return 0;
3766 return 1;
3767}
3768
3769static u64 get_system_chunk_thresh(struct btrfs_root *root, u64 type)
3770{
3771 u64 num_dev;
3772
3773 if (type & (BTRFS_BLOCK_GROUP_RAID10 |
3774 BTRFS_BLOCK_GROUP_RAID0 |
3775 BTRFS_BLOCK_GROUP_RAID5 |
3776 BTRFS_BLOCK_GROUP_RAID6))
3777 num_dev = root->fs_info->fs_devices->rw_devices;
3778 else if (type & BTRFS_BLOCK_GROUP_RAID1)
3779 num_dev = 2;
3780 else
3781 num_dev = 1;
3782
3783
3784 return btrfs_calc_trans_metadata_size(root, num_dev + 1);
3785}
3786
3787static void check_system_chunk(struct btrfs_trans_handle *trans,
3788 struct btrfs_root *root, u64 type)
3789{
3790 struct btrfs_space_info *info;
3791 u64 left;
3792 u64 thresh;
3793
3794 info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
3795 spin_lock(&info->lock);
3796 left = info->total_bytes - info->bytes_used - info->bytes_pinned -
3797 info->bytes_reserved - info->bytes_readonly;
3798 spin_unlock(&info->lock);
3799
3800 thresh = get_system_chunk_thresh(root, type);
3801 if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
3802 btrfs_info(root->fs_info, "left=%llu, need=%llu, flags=%llu",
3803 left, thresh, type);
3804 dump_space_info(info, 0, 0);
3805 }
3806
3807 if (left < thresh) {
3808 u64 flags;
3809
3810 flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
3811 btrfs_alloc_chunk(trans, root, flags);
3812 }
3813}
3814
3815static int do_chunk_alloc(struct btrfs_trans_handle *trans,
3816 struct btrfs_root *extent_root, u64 flags, int force)
3817{
3818 struct btrfs_space_info *space_info;
3819 struct btrfs_fs_info *fs_info = extent_root->fs_info;
3820 int wait_for_alloc = 0;
3821 int ret = 0;
3822
3823
3824 if (trans->allocating_chunk)
3825 return -ENOSPC;
3826
3827 space_info = __find_space_info(extent_root->fs_info, flags);
3828 if (!space_info) {
3829 ret = update_space_info(extent_root->fs_info, flags,
3830 0, 0, &space_info);
3831 BUG_ON(ret);
3832 }
3833 BUG_ON(!space_info);
3834
3835again:
3836 spin_lock(&space_info->lock);
3837 if (force < space_info->force_alloc)
3838 force = space_info->force_alloc;
3839 if (space_info->full) {
3840 if (should_alloc_chunk(extent_root, space_info, force))
3841 ret = -ENOSPC;
3842 else
3843 ret = 0;
3844 spin_unlock(&space_info->lock);
3845 return ret;
3846 }
3847
3848 if (!should_alloc_chunk(extent_root, space_info, force)) {
3849 spin_unlock(&space_info->lock);
3850 return 0;
3851 } else if (space_info->chunk_alloc) {
3852 wait_for_alloc = 1;
3853 } else {
3854 space_info->chunk_alloc = 1;
3855 }
3856
3857 spin_unlock(&space_info->lock);
3858
3859 mutex_lock(&fs_info->chunk_mutex);
3860
3861
3862
3863
3864
3865
3866
3867 if (wait_for_alloc) {
3868 mutex_unlock(&fs_info->chunk_mutex);
3869 wait_for_alloc = 0;
3870 goto again;
3871 }
3872
3873 trans->allocating_chunk = true;
3874
3875
3876
3877
3878
3879 if (btrfs_mixed_space_info(space_info))
3880 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
3881
3882
3883
3884
3885
3886
3887 if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3888 fs_info->data_chunk_allocations++;
3889 if (!(fs_info->data_chunk_allocations %
3890 fs_info->metadata_ratio))
3891 force_metadata_allocation(fs_info);
3892 }
3893
3894
3895
3896
3897
3898 check_system_chunk(trans, extent_root, flags);
3899
3900 ret = btrfs_alloc_chunk(trans, extent_root, flags);
3901 trans->allocating_chunk = false;
3902
3903 spin_lock(&space_info->lock);
3904 if (ret < 0 && ret != -ENOSPC)
3905 goto out;
3906 if (ret)
3907 space_info->full = 1;
3908 else
3909 ret = 1;
3910
3911 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3912out:
3913 space_info->chunk_alloc = 0;
3914 spin_unlock(&space_info->lock);
3915 mutex_unlock(&fs_info->chunk_mutex);
3916 return ret;
3917}
3918
3919static int can_overcommit(struct btrfs_root *root,
3920 struct btrfs_space_info *space_info, u64 bytes,
3921 enum btrfs_reserve_flush_enum flush)
3922{
3923 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3924 u64 profile = btrfs_get_alloc_profile(root, 0);
3925 u64 space_size;
3926 u64 avail;
3927 u64 used;
3928
3929 used = space_info->bytes_used + space_info->bytes_reserved +
3930 space_info->bytes_pinned + space_info->bytes_readonly;
3931
3932
3933
3934
3935
3936
3937
3938 spin_lock(&global_rsv->lock);
3939 space_size = calc_global_rsv_need_space(global_rsv);
3940 spin_unlock(&global_rsv->lock);
3941 if (used + space_size >= space_info->total_bytes)
3942 return 0;
3943
3944 used += space_info->bytes_may_use;
3945
3946 spin_lock(&root->fs_info->free_chunk_lock);
3947 avail = root->fs_info->free_chunk_space;
3948 spin_unlock(&root->fs_info->free_chunk_lock);
3949
3950
3951
3952
3953
3954
3955
3956 if (profile & (BTRFS_BLOCK_GROUP_DUP |
3957 BTRFS_BLOCK_GROUP_RAID1 |
3958 BTRFS_BLOCK_GROUP_RAID10))
3959 avail >>= 1;
3960
3961
3962
3963
3964
3965
3966 if (flush == BTRFS_RESERVE_FLUSH_ALL)
3967 avail >>= 3;
3968 else
3969 avail >>= 1;
3970
3971 if (used + bytes < space_info->total_bytes + avail)
3972 return 1;
3973 return 0;
3974}
3975
3976static void btrfs_writeback_inodes_sb_nr(struct btrfs_root *root,
3977 unsigned long nr_pages)
3978{
3979 struct super_block *sb = root->fs_info->sb;
3980
3981 if (down_read_trylock(&sb->s_umount)) {
3982 writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE);
3983 up_read(&sb->s_umount);
3984 } else {
3985
3986
3987
3988
3989
3990
3991
3992 btrfs_start_all_delalloc_inodes(root->fs_info, 0);
3993 if (!current->journal_info)
3994 btrfs_wait_all_ordered_extents(root->fs_info);
3995 }
3996}
3997
3998
3999
4000
4001static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig,
4002 bool wait_ordered)
4003{
4004 struct btrfs_block_rsv *block_rsv;
4005 struct btrfs_space_info *space_info;
4006 struct btrfs_trans_handle *trans;
4007 u64 delalloc_bytes;
4008 u64 max_reclaim;
4009 long time_left;
4010 unsigned long nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
4011 int loops = 0;
4012 enum btrfs_reserve_flush_enum flush;
4013
4014 trans = (struct btrfs_trans_handle *)current->journal_info;
4015 block_rsv = &root->fs_info->delalloc_block_rsv;
4016 space_info = block_rsv->space_info;
4017
4018 smp_mb();
4019 delalloc_bytes = percpu_counter_sum_positive(
4020 &root->fs_info->delalloc_bytes);
4021 if (delalloc_bytes == 0) {
4022 if (trans)
4023 return;
4024 btrfs_wait_all_ordered_extents(root->fs_info);
4025 return;
4026 }
4027
4028 while (delalloc_bytes && loops < 3) {
4029 max_reclaim = min(delalloc_bytes, to_reclaim);
4030 nr_pages = max_reclaim >> PAGE_CACHE_SHIFT;
4031 btrfs_writeback_inodes_sb_nr(root, nr_pages);
4032
4033
4034
4035
4036 wait_event(root->fs_info->async_submit_wait,
4037 !atomic_read(&root->fs_info->async_delalloc_pages));
4038
4039 if (!trans)
4040 flush = BTRFS_RESERVE_FLUSH_ALL;
4041 else
4042 flush = BTRFS_RESERVE_NO_FLUSH;
4043 spin_lock(&space_info->lock);
4044 if (can_overcommit(root, space_info, orig, flush)) {
4045 spin_unlock(&space_info->lock);
4046 break;
4047 }
4048 spin_unlock(&space_info->lock);
4049
4050 loops++;
4051 if (wait_ordered && !trans) {
4052 btrfs_wait_all_ordered_extents(root->fs_info);
4053 } else {
4054 time_left = schedule_timeout_killable(1);
4055 if (time_left)
4056 break;
4057 }
4058 smp_mb();
4059 delalloc_bytes = percpu_counter_sum_positive(
4060 &root->fs_info->delalloc_bytes);
4061 }
4062}
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074static int may_commit_transaction(struct btrfs_root *root,
4075 struct btrfs_space_info *space_info,
4076 u64 bytes, int force)
4077{
4078 struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
4079 struct btrfs_trans_handle *trans;
4080
4081 trans = (struct btrfs_trans_handle *)current->journal_info;
4082 if (trans)
4083 return -EAGAIN;
4084
4085 if (force)
4086 goto commit;
4087
4088
4089 spin_lock(&space_info->lock);
4090 if (percpu_counter_compare(&space_info->total_bytes_pinned,
4091 bytes) >= 0) {
4092 spin_unlock(&space_info->lock);
4093 goto commit;
4094 }
4095 spin_unlock(&space_info->lock);
4096
4097
4098
4099
4100
4101 if (space_info != delayed_rsv->space_info)
4102 return -ENOSPC;
4103
4104 spin_lock(&space_info->lock);
4105 spin_lock(&delayed_rsv->lock);
4106 if (percpu_counter_compare(&space_info->total_bytes_pinned,
4107 bytes - delayed_rsv->size) >= 0) {
4108 spin_unlock(&delayed_rsv->lock);
4109 spin_unlock(&space_info->lock);
4110 return -ENOSPC;
4111 }
4112 spin_unlock(&delayed_rsv->lock);
4113 spin_unlock(&space_info->lock);
4114
4115commit:
4116 trans = btrfs_join_transaction(root);
4117 if (IS_ERR(trans))
4118 return -ENOSPC;
4119
4120 return btrfs_commit_transaction(trans, root);
4121}
4122
4123enum flush_state {
4124 FLUSH_DELAYED_ITEMS_NR = 1,
4125 FLUSH_DELAYED_ITEMS = 2,
4126 FLUSH_DELALLOC = 3,
4127 FLUSH_DELALLOC_WAIT = 4,
4128 ALLOC_CHUNK = 5,
4129 COMMIT_TRANS = 6,
4130};
4131
4132static int flush_space(struct btrfs_root *root,
4133 struct btrfs_space_info *space_info, u64 num_bytes,
4134 u64 orig_bytes, int state)
4135{
4136 struct btrfs_trans_handle *trans;
4137 int nr;
4138 int ret = 0;
4139
4140 switch (state) {
4141 case FLUSH_DELAYED_ITEMS_NR:
4142 case FLUSH_DELAYED_ITEMS:
4143 if (state == FLUSH_DELAYED_ITEMS_NR) {
4144 u64 bytes = btrfs_calc_trans_metadata_size(root, 1);
4145
4146 nr = (int)div64_u64(num_bytes, bytes);
4147 if (!nr)
4148 nr = 1;
4149 nr *= 2;
4150 } else {
4151 nr = -1;
4152 }
4153 trans = btrfs_join_transaction(root);
4154 if (IS_ERR(trans)) {
4155 ret = PTR_ERR(trans);
4156 break;
4157 }
4158 ret = btrfs_run_delayed_items_nr(trans, root, nr);
4159 btrfs_end_transaction(trans, root);
4160 break;
4161 case FLUSH_DELALLOC:
4162 case FLUSH_DELALLOC_WAIT:
4163 shrink_delalloc(root, num_bytes, orig_bytes,
4164 state == FLUSH_DELALLOC_WAIT);
4165 break;
4166 case ALLOC_CHUNK:
4167 trans = btrfs_join_transaction(root);
4168 if (IS_ERR(trans)) {
4169 ret = PTR_ERR(trans);
4170 break;
4171 }
4172 ret = do_chunk_alloc(trans, root->fs_info->extent_root,
4173 btrfs_get_alloc_profile(root, 0),
4174 CHUNK_ALLOC_NO_FORCE);
4175 btrfs_end_transaction(trans, root);
4176 if (ret == -ENOSPC)
4177 ret = 0;
4178 break;
4179 case COMMIT_TRANS:
4180 ret = may_commit_transaction(root, space_info, orig_bytes, 0);
4181 break;
4182 default:
4183 ret = -ENOSPC;
4184 break;
4185 }
4186
4187 return ret;
4188}
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203static int reserve_metadata_bytes(struct btrfs_root *root,
4204 struct btrfs_block_rsv *block_rsv,
4205 u64 orig_bytes,
4206 enum btrfs_reserve_flush_enum flush)
4207{
4208 struct btrfs_space_info *space_info = block_rsv->space_info;
4209 u64 used;
4210 u64 num_bytes = orig_bytes;
4211 int flush_state = FLUSH_DELAYED_ITEMS_NR;
4212 int ret = 0;
4213 bool flushing = false;
4214
4215again:
4216 ret = 0;
4217 spin_lock(&space_info->lock);
4218
4219
4220
4221
4222 while (flush == BTRFS_RESERVE_FLUSH_ALL && !flushing &&
4223 space_info->flush) {
4224 spin_unlock(&space_info->lock);
4225
4226
4227
4228
4229
4230
4231 if (current->journal_info)
4232 return -EAGAIN;
4233 ret = wait_event_killable(space_info->wait, !space_info->flush);
4234
4235 if (ret)
4236 return -EINTR;
4237
4238 spin_lock(&space_info->lock);
4239 }
4240
4241 ret = -ENOSPC;
4242 used = space_info->bytes_used + space_info->bytes_reserved +
4243 space_info->bytes_pinned + space_info->bytes_readonly +
4244 space_info->bytes_may_use;
4245
4246
4247
4248
4249
4250
4251
4252
4253 if (used <= space_info->total_bytes) {
4254 if (used + orig_bytes <= space_info->total_bytes) {
4255 space_info->bytes_may_use += orig_bytes;
4256 trace_btrfs_space_reservation(root->fs_info,
4257 "space_info", space_info->flags, orig_bytes, 1);
4258 ret = 0;
4259 } else {
4260
4261
4262
4263
4264
4265 num_bytes = orig_bytes;
4266 }
4267 } else {
4268
4269
4270
4271
4272
4273 num_bytes = used - space_info->total_bytes +
4274 (orig_bytes * 2);
4275 }
4276
4277 if (ret && can_overcommit(root, space_info, orig_bytes, flush)) {
4278 space_info->bytes_may_use += orig_bytes;
4279 trace_btrfs_space_reservation(root->fs_info, "space_info",
4280 space_info->flags, orig_bytes,
4281 1);
4282 ret = 0;
4283 }
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293 if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
4294 flushing = true;
4295 space_info->flush = 1;
4296 }
4297
4298 spin_unlock(&space_info->lock);
4299
4300 if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
4301 goto out;
4302
4303 ret = flush_space(root, space_info, num_bytes, orig_bytes,
4304 flush_state);
4305 flush_state++;
4306
4307
4308
4309
4310
4311 if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
4312 (flush_state == FLUSH_DELALLOC ||
4313 flush_state == FLUSH_DELALLOC_WAIT))
4314 flush_state = ALLOC_CHUNK;
4315
4316 if (!ret)
4317 goto again;
4318 else if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
4319 flush_state < COMMIT_TRANS)
4320 goto again;
4321 else if (flush == BTRFS_RESERVE_FLUSH_ALL &&
4322 flush_state <= COMMIT_TRANS)
4323 goto again;
4324
4325out:
4326 if (ret == -ENOSPC &&
4327 unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) {
4328 struct btrfs_block_rsv *global_rsv =
4329 &root->fs_info->global_block_rsv;
4330
4331 if (block_rsv != global_rsv &&
4332 !block_rsv_use_bytes(global_rsv, orig_bytes))
4333 ret = 0;
4334 }
4335 if (flushing) {
4336 spin_lock(&space_info->lock);
4337 space_info->flush = 0;
4338 wake_up_all(&space_info->wait);
4339 spin_unlock(&space_info->lock);
4340 }
4341 return ret;
4342}
4343
4344static struct btrfs_block_rsv *get_block_rsv(
4345 const struct btrfs_trans_handle *trans,
4346 const struct btrfs_root *root)
4347{
4348 struct btrfs_block_rsv *block_rsv = NULL;
4349
4350 if (root->ref_cows)
4351 block_rsv = trans->block_rsv;
4352
4353 if (root == root->fs_info->csum_root && trans->adding_csums)
4354 block_rsv = trans->block_rsv;
4355
4356 if (root == root->fs_info->uuid_root)
4357 block_rsv = trans->block_rsv;
4358
4359 if (!block_rsv)
4360 block_rsv = root->block_rsv;
4361
4362 if (!block_rsv)
4363 block_rsv = &root->fs_info->empty_block_rsv;
4364
4365 return block_rsv;
4366}
4367
4368static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
4369 u64 num_bytes)
4370{
4371 int ret = -ENOSPC;
4372 spin_lock(&block_rsv->lock);
4373 if (block_rsv->reserved >= num_bytes) {
4374 block_rsv->reserved -= num_bytes;
4375 if (block_rsv->reserved < block_rsv->size)
4376 block_rsv->full = 0;
4377 ret = 0;
4378 }
4379 spin_unlock(&block_rsv->lock);
4380 return ret;
4381}
4382
4383static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
4384 u64 num_bytes, int update_size)
4385{
4386 spin_lock(&block_rsv->lock);
4387 block_rsv->reserved += num_bytes;
4388 if (update_size)
4389 block_rsv->size += num_bytes;
4390 else if (block_rsv->reserved >= block_rsv->size)
4391 block_rsv->full = 1;
4392 spin_unlock(&block_rsv->lock);
4393}
4394
4395int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info,
4396 struct btrfs_block_rsv *dest, u64 num_bytes,
4397 int min_factor)
4398{
4399 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
4400 u64 min_bytes;
4401
4402 if (global_rsv->space_info != dest->space_info)
4403 return -ENOSPC;
4404
4405 spin_lock(&global_rsv->lock);
4406 min_bytes = div_factor(global_rsv->size, min_factor);
4407 if (global_rsv->reserved < min_bytes + num_bytes) {
4408 spin_unlock(&global_rsv->lock);
4409 return -ENOSPC;
4410 }
4411 global_rsv->reserved -= num_bytes;
4412 if (global_rsv->reserved < global_rsv->size)
4413 global_rsv->full = 0;
4414 spin_unlock(&global_rsv->lock);
4415
4416 block_rsv_add_bytes(dest, num_bytes, 1);
4417 return 0;
4418}
4419
4420static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
4421 struct btrfs_block_rsv *block_rsv,
4422 struct btrfs_block_rsv *dest, u64 num_bytes)
4423{
4424 struct btrfs_space_info *space_info = block_rsv->space_info;
4425
4426 spin_lock(&block_rsv->lock);
4427 if (num_bytes == (u64)-1)
4428 num_bytes = block_rsv->size;
4429 block_rsv->size -= num_bytes;
4430 if (block_rsv->reserved >= block_rsv->size) {
4431 num_bytes = block_rsv->reserved - block_rsv->size;
4432 block_rsv->reserved = block_rsv->size;
4433 block_rsv->full = 1;
4434 } else {
4435 num_bytes = 0;
4436 }
4437 spin_unlock(&block_rsv->lock);
4438
4439 if (num_bytes > 0) {
4440 if (dest) {
4441 spin_lock(&dest->lock);
4442 if (!dest->full) {
4443 u64 bytes_to_add;
4444
4445 bytes_to_add = dest->size - dest->reserved;
4446 bytes_to_add = min(num_bytes, bytes_to_add);
4447 dest->reserved += bytes_to_add;
4448 if (dest->reserved >= dest->size)
4449 dest->full = 1;
4450 num_bytes -= bytes_to_add;
4451 }
4452 spin_unlock(&dest->lock);
4453 }
4454 if (num_bytes) {
4455 spin_lock(&space_info->lock);
4456 space_info->bytes_may_use -= num_bytes;
4457 trace_btrfs_space_reservation(fs_info, "space_info",
4458 space_info->flags, num_bytes, 0);
4459 spin_unlock(&space_info->lock);
4460 }
4461 }
4462}
4463
4464static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
4465 struct btrfs_block_rsv *dst, u64 num_bytes)
4466{
4467 int ret;
4468
4469 ret = block_rsv_use_bytes(src, num_bytes);
4470 if (ret)
4471 return ret;
4472
4473 block_rsv_add_bytes(dst, num_bytes, 1);
4474 return 0;
4475}
4476
4477void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
4478{
4479 memset(rsv, 0, sizeof(*rsv));
4480 spin_lock_init(&rsv->lock);
4481 rsv->type = type;
4482}
4483
4484struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root,
4485 unsigned short type)
4486{
4487 struct btrfs_block_rsv *block_rsv;
4488 struct btrfs_fs_info *fs_info = root->fs_info;
4489
4490 block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
4491 if (!block_rsv)
4492 return NULL;
4493
4494 btrfs_init_block_rsv(block_rsv, type);
4495 block_rsv->space_info = __find_space_info(fs_info,
4496 BTRFS_BLOCK_GROUP_METADATA);
4497 return block_rsv;
4498}
4499
4500void btrfs_free_block_rsv(struct btrfs_root *root,
4501 struct btrfs_block_rsv *rsv)
4502{
4503 if (!rsv)
4504 return;
4505 btrfs_block_rsv_release(root, rsv, (u64)-1);
4506 kfree(rsv);
4507}
4508
4509int btrfs_block_rsv_add(struct btrfs_root *root,
4510 struct btrfs_block_rsv *block_rsv, u64 num_bytes,
4511 enum btrfs_reserve_flush_enum flush)
4512{
4513 int ret;
4514
4515 if (num_bytes == 0)
4516 return 0;
4517
4518 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4519 if (!ret) {
4520 block_rsv_add_bytes(block_rsv, num_bytes, 1);
4521 return 0;
4522 }
4523
4524 return ret;
4525}
4526
4527int btrfs_block_rsv_check(struct btrfs_root *root,
4528 struct btrfs_block_rsv *block_rsv, int min_factor)
4529{
4530 u64 num_bytes = 0;
4531 int ret = -ENOSPC;
4532
4533 if (!block_rsv)
4534 return 0;
4535
4536 spin_lock(&block_rsv->lock);
4537 num_bytes = div_factor(block_rsv->size, min_factor);
4538 if (block_rsv->reserved >= num_bytes)
4539 ret = 0;
4540 spin_unlock(&block_rsv->lock);
4541
4542 return ret;
4543}
4544
4545int btrfs_block_rsv_refill(struct btrfs_root *root,
4546 struct btrfs_block_rsv *block_rsv, u64 min_reserved,
4547 enum btrfs_reserve_flush_enum flush)
4548{
4549 u64 num_bytes = 0;
4550 int ret = -ENOSPC;
4551
4552 if (!block_rsv)
4553 return 0;
4554
4555 spin_lock(&block_rsv->lock);
4556 num_bytes = min_reserved;
4557 if (block_rsv->reserved >= num_bytes)
4558 ret = 0;
4559 else
4560 num_bytes -= block_rsv->reserved;
4561 spin_unlock(&block_rsv->lock);
4562
4563 if (!ret)
4564 return 0;
4565
4566 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4567 if (!ret) {
4568 block_rsv_add_bytes(block_rsv, num_bytes, 0);
4569 return 0;
4570 }
4571
4572 return ret;
4573}
4574
4575int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
4576 struct btrfs_block_rsv *dst_rsv,
4577 u64 num_bytes)
4578{
4579 return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4580}
4581
4582void btrfs_block_rsv_release(struct btrfs_root *root,
4583 struct btrfs_block_rsv *block_rsv,
4584 u64 num_bytes)
4585{
4586 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
4587 if (global_rsv->full || global_rsv == block_rsv ||
4588 block_rsv->space_info != global_rsv->space_info)
4589 global_rsv = NULL;
4590 block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
4591 num_bytes);
4592}
4593
4594
4595
4596
4597
4598
4599static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
4600{
4601 struct btrfs_space_info *sinfo;
4602 u64 num_bytes;
4603 u64 meta_used;
4604 u64 data_used;
4605 int csum_size = btrfs_super_csum_size(fs_info->super_copy);
4606
4607 sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
4608 spin_lock(&sinfo->lock);
4609 data_used = sinfo->bytes_used;
4610 spin_unlock(&sinfo->lock);
4611
4612 sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4613 spin_lock(&sinfo->lock);
4614 if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
4615 data_used = 0;
4616 meta_used = sinfo->bytes_used;
4617 spin_unlock(&sinfo->lock);
4618
4619 num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
4620 csum_size * 2;
4621 num_bytes += div64_u64(data_used + meta_used, 50);
4622
4623 if (num_bytes * 3 > meta_used)
4624 num_bytes = div64_u64(meta_used, 3);
4625
4626 return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
4627}
4628
4629static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
4630{
4631 struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
4632 struct btrfs_space_info *sinfo = block_rsv->space_info;
4633 u64 num_bytes;
4634
4635 num_bytes = calc_global_metadata_size(fs_info);
4636
4637 spin_lock(&sinfo->lock);
4638 spin_lock(&block_rsv->lock);
4639
4640 block_rsv->size = min_t(u64, num_bytes, 512 * 1024 * 1024);
4641
4642 num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
4643 sinfo->bytes_reserved + sinfo->bytes_readonly +
4644 sinfo->bytes_may_use;
4645
4646 if (sinfo->total_bytes > num_bytes) {
4647 num_bytes = sinfo->total_bytes - num_bytes;
4648 block_rsv->reserved += num_bytes;
4649 sinfo->bytes_may_use += num_bytes;
4650 trace_btrfs_space_reservation(fs_info, "space_info",
4651 sinfo->flags, num_bytes, 1);
4652 }
4653
4654 if (block_rsv->reserved >= block_rsv->size) {
4655 num_bytes = block_rsv->reserved - block_rsv->size;
4656 sinfo->bytes_may_use -= num_bytes;
4657 trace_btrfs_space_reservation(fs_info, "space_info",
4658 sinfo->flags, num_bytes, 0);
4659 block_rsv->reserved = block_rsv->size;
4660 block_rsv->full = 1;
4661 }
4662
4663 spin_unlock(&block_rsv->lock);
4664 spin_unlock(&sinfo->lock);
4665}
4666
4667static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
4668{
4669 struct btrfs_space_info *space_info;
4670
4671 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
4672 fs_info->chunk_block_rsv.space_info = space_info;
4673
4674 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4675 fs_info->global_block_rsv.space_info = space_info;
4676 fs_info->delalloc_block_rsv.space_info = space_info;
4677 fs_info->trans_block_rsv.space_info = space_info;
4678 fs_info->empty_block_rsv.space_info = space_info;
4679 fs_info->delayed_block_rsv.space_info = space_info;
4680
4681 fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
4682 fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
4683 fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
4684 fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
4685 if (fs_info->quota_root)
4686 fs_info->quota_root->block_rsv = &fs_info->global_block_rsv;
4687 fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
4688
4689 update_global_block_rsv(fs_info);
4690}
4691
4692static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
4693{
4694 block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
4695 (u64)-1);
4696 WARN_ON(fs_info->delalloc_block_rsv.size > 0);
4697 WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
4698 WARN_ON(fs_info->trans_block_rsv.size > 0);
4699 WARN_ON(fs_info->trans_block_rsv.reserved > 0);
4700 WARN_ON(fs_info->chunk_block_rsv.size > 0);
4701 WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
4702 WARN_ON(fs_info->delayed_block_rsv.size > 0);
4703 WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
4704}
4705
4706void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
4707 struct btrfs_root *root)
4708{
4709 if (!trans->block_rsv)
4710 return;
4711
4712 if (!trans->bytes_reserved)
4713 return;
4714
4715 trace_btrfs_space_reservation(root->fs_info, "transaction",
4716 trans->transid, trans->bytes_reserved, 0);
4717 btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
4718 trans->bytes_reserved = 0;
4719}
4720
4721
4722int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
4723 struct inode *inode)
4724{
4725 struct btrfs_root *root = BTRFS_I(inode)->root;
4726 struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
4727 struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;
4728
4729
4730
4731
4732
4733
4734 u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
4735 trace_btrfs_space_reservation(root->fs_info, "orphan",
4736 btrfs_ino(inode), num_bytes, 1);
4737 return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4738}
4739
4740void btrfs_orphan_release_metadata(struct inode *inode)
4741{
4742 struct btrfs_root *root = BTRFS_I(inode)->root;
4743 u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
4744 trace_btrfs_space_reservation(root->fs_info, "orphan",
4745 btrfs_ino(inode), num_bytes, 0);
4746 btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
4747}
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763int btrfs_subvolume_reserve_metadata(struct btrfs_root *root,
4764 struct btrfs_block_rsv *rsv,
4765 int items,
4766 u64 *qgroup_reserved,
4767 bool use_global_rsv)
4768{
4769 u64 num_bytes;
4770 int ret;
4771 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
4772
4773 if (root->fs_info->quota_enabled) {
4774
4775 num_bytes = 3 * root->leafsize;
4776 ret = btrfs_qgroup_reserve(root, num_bytes);
4777 if (ret)
4778 return ret;
4779 } else {
4780 num_bytes = 0;
4781 }
4782
4783 *qgroup_reserved = num_bytes;
4784
4785 num_bytes = btrfs_calc_trans_metadata_size(root, items);
4786 rsv->space_info = __find_space_info(root->fs_info,
4787 BTRFS_BLOCK_GROUP_METADATA);
4788 ret = btrfs_block_rsv_add(root, rsv, num_bytes,
4789 BTRFS_RESERVE_FLUSH_ALL);
4790
4791 if (ret == -ENOSPC && use_global_rsv)
4792 ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes);
4793
4794 if (ret) {
4795 if (*qgroup_reserved)
4796 btrfs_qgroup_free(root, *qgroup_reserved);
4797 }
4798
4799 return ret;
4800}
4801
4802void btrfs_subvolume_release_metadata(struct btrfs_root *root,
4803 struct btrfs_block_rsv *rsv,
4804 u64 qgroup_reserved)
4805{
4806 btrfs_block_rsv_release(root, rsv, (u64)-1);
4807 if (qgroup_reserved)
4808 btrfs_qgroup_free(root, qgroup_reserved);
4809}
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820static unsigned drop_outstanding_extent(struct inode *inode)
4821{
4822 unsigned drop_inode_space = 0;
4823 unsigned dropped_extents = 0;
4824
4825 BUG_ON(!BTRFS_I(inode)->outstanding_extents);
4826 BTRFS_I(inode)->outstanding_extents--;
4827
4828 if (BTRFS_I(inode)->outstanding_extents == 0 &&
4829 test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
4830 &BTRFS_I(inode)->runtime_flags))
4831 drop_inode_space = 1;
4832
4833
4834
4835
4836
4837 if (BTRFS_I(inode)->outstanding_extents >=
4838 BTRFS_I(inode)->reserved_extents)
4839 return drop_inode_space;
4840
4841 dropped_extents = BTRFS_I(inode)->reserved_extents -
4842 BTRFS_I(inode)->outstanding_extents;
4843 BTRFS_I(inode)->reserved_extents -= dropped_extents;
4844 return dropped_extents + drop_inode_space;
4845}
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
4866 int reserve)
4867{
4868 struct btrfs_root *root = BTRFS_I(inode)->root;
4869 u64 csum_size;
4870 int num_csums_per_leaf;
4871 int num_csums;
4872 int old_csums;
4873
4874 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
4875 BTRFS_I(inode)->csum_bytes == 0)
4876 return 0;
4877
4878 old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
4879 if (reserve)
4880 BTRFS_I(inode)->csum_bytes += num_bytes;
4881 else
4882 BTRFS_I(inode)->csum_bytes -= num_bytes;
4883 csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
4884 num_csums_per_leaf = (int)div64_u64(csum_size,
4885 sizeof(struct btrfs_csum_item) +
4886 sizeof(struct btrfs_disk_key));
4887 num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
4888 num_csums = num_csums + num_csums_per_leaf - 1;
4889 num_csums = num_csums / num_csums_per_leaf;
4890
4891 old_csums = old_csums + num_csums_per_leaf - 1;
4892 old_csums = old_csums / num_csums_per_leaf;
4893
4894
4895 if (old_csums == num_csums)
4896 return 0;
4897
4898 if (reserve)
4899 return btrfs_calc_trans_metadata_size(root,
4900 num_csums - old_csums);
4901
4902 return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
4903}
4904
4905int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
4906{
4907 struct btrfs_root *root = BTRFS_I(inode)->root;
4908 struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4909 u64 to_reserve = 0;
4910 u64 csum_bytes;
4911 unsigned nr_extents = 0;
4912 int extra_reserve = 0;
4913 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
4914 int ret = 0;
4915 bool delalloc_lock = true;
4916 u64 to_free = 0;
4917 unsigned dropped;
4918
4919
4920
4921
4922
4923
4924 if (btrfs_is_free_space_inode(inode)) {
4925 flush = BTRFS_RESERVE_NO_FLUSH;
4926 delalloc_lock = false;
4927 }
4928
4929 if (flush != BTRFS_RESERVE_NO_FLUSH &&
4930 btrfs_transaction_in_commit(root->fs_info))
4931 schedule_timeout(1);
4932
4933 if (delalloc_lock)
4934 mutex_lock(&BTRFS_I(inode)->delalloc_mutex);
4935
4936 num_bytes = ALIGN(num_bytes, root->sectorsize);
4937
4938 spin_lock(&BTRFS_I(inode)->lock);
4939 BTRFS_I(inode)->outstanding_extents++;
4940
4941 if (BTRFS_I(inode)->outstanding_extents >
4942 BTRFS_I(inode)->reserved_extents)
4943 nr_extents = BTRFS_I(inode)->outstanding_extents -
4944 BTRFS_I(inode)->reserved_extents;
4945
4946
4947
4948
4949
4950 if (!test_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
4951 &BTRFS_I(inode)->runtime_flags)) {
4952 nr_extents++;
4953 extra_reserve = 1;
4954 }
4955
4956 to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4957 to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
4958 csum_bytes = BTRFS_I(inode)->csum_bytes;
4959 spin_unlock(&BTRFS_I(inode)->lock);
4960
4961 if (root->fs_info->quota_enabled) {
4962 ret = btrfs_qgroup_reserve(root, num_bytes +
4963 nr_extents * root->leafsize);
4964 if (ret)
4965 goto out_fail;
4966 }
4967
4968 ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush);
4969 if (unlikely(ret)) {
4970 if (root->fs_info->quota_enabled)
4971 btrfs_qgroup_free(root, num_bytes +
4972 nr_extents * root->leafsize);
4973 goto out_fail;
4974 }
4975
4976 spin_lock(&BTRFS_I(inode)->lock);
4977 if (extra_reserve) {
4978 set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
4979 &BTRFS_I(inode)->runtime_flags);
4980 nr_extents--;
4981 }
4982 BTRFS_I(inode)->reserved_extents += nr_extents;
4983 spin_unlock(&BTRFS_I(inode)->lock);
4984
4985 if (delalloc_lock)
4986 mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4987
4988 if (to_reserve)
4989 trace_btrfs_space_reservation(root->fs_info,"delalloc",
4990 btrfs_ino(inode), to_reserve, 1);
4991 block_rsv_add_bytes(block_rsv, to_reserve, 1);
4992
4993 return 0;
4994
4995out_fail:
4996 spin_lock(&BTRFS_I(inode)->lock);
4997 dropped = drop_outstanding_extent(inode);
4998
4999
5000
5001
5002
5003 if (BTRFS_I(inode)->csum_bytes == csum_bytes) {
5004 calc_csum_metadata_size(inode, num_bytes, 0);
5005 } else {
5006 u64 orig_csum_bytes = BTRFS_I(inode)->csum_bytes;
5007 u64 bytes;
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017 bytes = csum_bytes - BTRFS_I(inode)->csum_bytes;
5018 BTRFS_I(inode)->csum_bytes = csum_bytes;
5019 to_free = calc_csum_metadata_size(inode, bytes, 0);
5020
5021
5022
5023
5024
5025
5026
5027 BTRFS_I(inode)->csum_bytes = csum_bytes - num_bytes;
5028 bytes = csum_bytes - orig_csum_bytes;
5029 bytes = calc_csum_metadata_size(inode, bytes, 0);
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039 BTRFS_I(inode)->csum_bytes = orig_csum_bytes - num_bytes;
5040 if (bytes > to_free)
5041 to_free = bytes - to_free;
5042 else
5043 to_free = 0;
5044 }
5045 spin_unlock(&BTRFS_I(inode)->lock);
5046 if (dropped)
5047 to_free += btrfs_calc_trans_metadata_size(root, dropped);
5048
5049 if (to_free) {
5050 btrfs_block_rsv_release(root, block_rsv, to_free);
5051 trace_btrfs_space_reservation(root->fs_info, "delalloc",
5052 btrfs_ino(inode), to_free, 0);
5053 }
5054 if (delalloc_lock)
5055 mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
5056 return ret;
5057}
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
5069{
5070 struct btrfs_root *root = BTRFS_I(inode)->root;
5071 u64 to_free = 0;
5072 unsigned dropped;
5073
5074 num_bytes = ALIGN(num_bytes, root->sectorsize);
5075 spin_lock(&BTRFS_I(inode)->lock);
5076 dropped = drop_outstanding_extent(inode);
5077
5078 if (num_bytes)
5079 to_free = calc_csum_metadata_size(inode, num_bytes, 0);
5080 spin_unlock(&BTRFS_I(inode)->lock);
5081 if (dropped > 0)
5082 to_free += btrfs_calc_trans_metadata_size(root, dropped);
5083
5084 trace_btrfs_space_reservation(root->fs_info, "delalloc",
5085 btrfs_ino(inode), to_free, 0);
5086 if (root->fs_info->quota_enabled) {
5087 btrfs_qgroup_free(root, num_bytes +
5088 dropped * root->leafsize);
5089 }
5090
5091 btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
5092 to_free);
5093}
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
5111{
5112 int ret;
5113
5114 ret = btrfs_check_data_free_space(inode, num_bytes);
5115 if (ret)
5116 return ret;
5117
5118 ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
5119 if (ret) {
5120 btrfs_free_reserved_data_space(inode, num_bytes);
5121 return ret;
5122 }
5123
5124 return 0;
5125}
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
5141{
5142 btrfs_delalloc_release_metadata(inode, num_bytes);
5143 btrfs_free_reserved_data_space(inode, num_bytes);
5144}
5145
5146static int update_block_group(struct btrfs_root *root,
5147 u64 bytenr, u64 num_bytes, int alloc)
5148{
5149 struct btrfs_block_group_cache *cache = NULL;
5150 struct btrfs_fs_info *info = root->fs_info;
5151 u64 total = num_bytes;
5152 u64 old_val;
5153 u64 byte_in_group;
5154 int factor;
5155
5156
5157 spin_lock(&info->delalloc_root_lock);
5158 old_val = btrfs_super_bytes_used(info->super_copy);
5159 if (alloc)
5160 old_val += num_bytes;
5161 else
5162 old_val -= num_bytes;
5163 btrfs_set_super_bytes_used(info->super_copy, old_val);
5164 spin_unlock(&info->delalloc_root_lock);
5165
5166 while (total) {
5167 cache = btrfs_lookup_block_group(info, bytenr);
5168 if (!cache)
5169 return -ENOENT;
5170 if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
5171 BTRFS_BLOCK_GROUP_RAID1 |
5172 BTRFS_BLOCK_GROUP_RAID10))
5173 factor = 2;
5174 else
5175 factor = 1;
5176
5177
5178
5179
5180
5181
5182 if (!alloc && cache->cached == BTRFS_CACHE_NO)
5183 cache_block_group(cache, 1);
5184
5185 byte_in_group = bytenr - cache->key.objectid;
5186 WARN_ON(byte_in_group > cache->key.offset);
5187
5188 spin_lock(&cache->space_info->lock);
5189 spin_lock(&cache->lock);
5190
5191 if (btrfs_test_opt(root, SPACE_CACHE) &&
5192 cache->disk_cache_state < BTRFS_DC_CLEAR)
5193 cache->disk_cache_state = BTRFS_DC_CLEAR;
5194
5195 cache->dirty = 1;
5196 old_val = btrfs_block_group_used(&cache->item);
5197 num_bytes = min(total, cache->key.offset - byte_in_group);
5198 if (alloc) {
5199 old_val += num_bytes;
5200 btrfs_set_block_group_used(&cache->item, old_val);
5201 cache->reserved -= num_bytes;
5202 cache->space_info->bytes_reserved -= num_bytes;
5203 cache->space_info->bytes_used += num_bytes;
5204 cache->space_info->disk_used += num_bytes * factor;
5205 spin_unlock(&cache->lock);
5206 spin_unlock(&cache->space_info->lock);
5207 } else {
5208 old_val -= num_bytes;
5209 btrfs_set_block_group_used(&cache->item, old_val);
5210 cache->pinned += num_bytes;
5211 cache->space_info->bytes_pinned += num_bytes;
5212 cache->space_info->bytes_used -= num_bytes;
5213 cache->space_info->disk_used -= num_bytes * factor;
5214 spin_unlock(&cache->lock);
5215 spin_unlock(&cache->space_info->lock);
5216
5217 set_extent_dirty(info->pinned_extents,
5218 bytenr, bytenr + num_bytes - 1,
5219 GFP_NOFS | __GFP_NOFAIL);
5220 }
5221 btrfs_put_block_group(cache);
5222 total -= num_bytes;
5223 bytenr += num_bytes;
5224 }
5225 return 0;
5226}
5227
5228static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
5229{
5230 struct btrfs_block_group_cache *cache;
5231 u64 bytenr;
5232
5233 spin_lock(&root->fs_info->block_group_cache_lock);
5234 bytenr = root->fs_info->first_logical_byte;
5235 spin_unlock(&root->fs_info->block_group_cache_lock);
5236
5237 if (bytenr < (u64)-1)
5238 return bytenr;
5239
5240 cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
5241 if (!cache)
5242 return 0;
5243
5244 bytenr = cache->key.objectid;
5245 btrfs_put_block_group(cache);
5246
5247 return bytenr;
5248}
5249
5250static int pin_down_extent(struct btrfs_root *root,
5251 struct btrfs_block_group_cache *cache,
5252 u64 bytenr, u64 num_bytes, int reserved)
5253{
5254 spin_lock(&cache->space_info->lock);
5255 spin_lock(&cache->lock);
5256 cache->pinned += num_bytes;
5257 cache->space_info->bytes_pinned += num_bytes;
5258 if (reserved) {
5259 cache->reserved -= num_bytes;
5260 cache->space_info->bytes_reserved -= num_bytes;
5261 }
5262 spin_unlock(&cache->lock);
5263 spin_unlock(&cache->space_info->lock);
5264
5265 set_extent_dirty(root->fs_info->pinned_extents, bytenr,
5266 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
5267 return 0;
5268}
5269
5270
5271
5272
5273int btrfs_pin_extent(struct btrfs_root *root,
5274 u64 bytenr, u64 num_bytes, int reserved)
5275{
5276 struct btrfs_block_group_cache *cache;
5277
5278 cache = btrfs_lookup_block_group(root->fs_info, bytenr);
5279 BUG_ON(!cache);
5280
5281 pin_down_extent(root, cache, bytenr, num_bytes, reserved);
5282
5283 btrfs_put_block_group(cache);
5284 return 0;
5285}
5286
5287
5288
5289
5290int btrfs_pin_extent_for_log_replay(struct btrfs_root *root,
5291 u64 bytenr, u64 num_bytes)
5292{
5293 struct btrfs_block_group_cache *cache;
5294 int ret;
5295
5296 cache = btrfs_lookup_block_group(root->fs_info, bytenr);
5297 if (!cache)
5298 return -EINVAL;
5299
5300
5301
5302
5303
5304
5305
5306 cache_block_group(cache, 1);
5307
5308 pin_down_extent(root, cache, bytenr, num_bytes, 0);
5309
5310
5311 ret = btrfs_remove_free_space(cache, bytenr, num_bytes);
5312 btrfs_put_block_group(cache);
5313 return ret;
5314}
5315
5316static int __exclude_logged_extent(struct btrfs_root *root, u64 start, u64 num_bytes)
5317{
5318 int ret;
5319 struct btrfs_block_group_cache *block_group;
5320 struct btrfs_caching_control *caching_ctl;
5321
5322 block_group = btrfs_lookup_block_group(root->fs_info, start);
5323 if (!block_group)
5324 return -EINVAL;
5325
5326 cache_block_group(block_group, 0);
5327 caching_ctl = get_caching_control(block_group);
5328
5329 if (!caching_ctl) {
5330
5331 BUG_ON(!block_group_cache_done(block_group));
5332 ret = btrfs_remove_free_space(block_group, start, num_bytes);
5333 } else {
5334 mutex_lock(&caching_ctl->mutex);
5335
5336 if (start >= caching_ctl->progress) {
5337 ret = add_excluded_extent(root, start, num_bytes);
5338 } else if (start + num_bytes <= caching_ctl->progress) {
5339 ret = btrfs_remove_free_space(block_group,
5340 start, num_bytes);
5341 } else {
5342 num_bytes = caching_ctl->progress - start;
5343 ret = btrfs_remove_free_space(block_group,
5344 start, num_bytes);
5345 if (ret)
5346 goto out_lock;
5347
5348 num_bytes = (start + num_bytes) -
5349 caching_ctl->progress;
5350 start = caching_ctl->progress;
5351 ret = add_excluded_extent(root, start, num_bytes);
5352 }
5353out_lock:
5354 mutex_unlock(&caching_ctl->mutex);
5355 put_caching_control(caching_ctl);
5356 }
5357 btrfs_put_block_group(block_group);
5358 return ret;
5359}
5360
5361int btrfs_exclude_logged_extents(struct btrfs_root *log,
5362 struct extent_buffer *eb)
5363{
5364 struct btrfs_file_extent_item *item;
5365 struct btrfs_key key;
5366 int found_type;
5367 int i;
5368
5369 if (!btrfs_fs_incompat(log->fs_info, MIXED_GROUPS))
5370 return 0;
5371
5372 for (i = 0; i < btrfs_header_nritems(eb); i++) {
5373 btrfs_item_key_to_cpu(eb, &key, i);
5374 if (key.type != BTRFS_EXTENT_DATA_KEY)
5375 continue;
5376 item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
5377 found_type = btrfs_file_extent_type(eb, item);
5378 if (found_type == BTRFS_FILE_EXTENT_INLINE)
5379 continue;
5380 if (btrfs_file_extent_disk_bytenr(eb, item) == 0)
5381 continue;
5382 key.objectid = btrfs_file_extent_disk_bytenr(eb, item);
5383 key.offset = btrfs_file_extent_disk_num_bytes(eb, item);
5384 __exclude_logged_extent(log, key.objectid, key.offset);
5385 }
5386
5387 return 0;
5388}
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
5413 u64 num_bytes, int reserve)
5414{
5415 struct btrfs_space_info *space_info = cache->space_info;
5416 int ret = 0;
5417
5418 spin_lock(&space_info->lock);
5419 spin_lock(&cache->lock);
5420 if (reserve != RESERVE_FREE) {
5421 if (cache->ro) {
5422 ret = -EAGAIN;
5423 } else {
5424 cache->reserved += num_bytes;
5425 space_info->bytes_reserved += num_bytes;
5426 if (reserve == RESERVE_ALLOC) {
5427 trace_btrfs_space_reservation(cache->fs_info,
5428 "space_info", space_info->flags,
5429 num_bytes, 0);
5430 space_info->bytes_may_use -= num_bytes;
5431 }
5432 }
5433 } else {
5434 if (cache->ro)
5435 space_info->bytes_readonly += num_bytes;
5436 cache->reserved -= num_bytes;
5437 space_info->bytes_reserved -= num_bytes;
5438 }
5439 spin_unlock(&cache->lock);
5440 spin_unlock(&space_info->lock);
5441 return ret;
5442}
5443
5444void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
5445 struct btrfs_root *root)
5446{
5447 struct btrfs_fs_info *fs_info = root->fs_info;
5448 struct btrfs_caching_control *next;
5449 struct btrfs_caching_control *caching_ctl;
5450 struct btrfs_block_group_cache *cache;
5451 struct btrfs_space_info *space_info;
5452
5453 down_write(&fs_info->extent_commit_sem);
5454
5455 list_for_each_entry_safe(caching_ctl, next,
5456 &fs_info->caching_block_groups, list) {
5457 cache = caching_ctl->block_group;
5458 if (block_group_cache_done(cache)) {
5459 cache->last_byte_to_unpin = (u64)-1;
5460 list_del_init(&caching_ctl->list);
5461 put_caching_control(caching_ctl);
5462 } else {
5463 cache->last_byte_to_unpin = caching_ctl->progress;
5464 }
5465 }
5466
5467 if (fs_info->pinned_extents == &fs_info->freed_extents[0])
5468 fs_info->pinned_extents = &fs_info->freed_extents[1];
5469 else
5470 fs_info->pinned_extents = &fs_info->freed_extents[0];
5471
5472 up_write(&fs_info->extent_commit_sem);
5473
5474 list_for_each_entry_rcu(space_info, &fs_info->space_info, list)
5475 percpu_counter_set(&space_info->total_bytes_pinned, 0);
5476
5477 update_global_block_rsv(fs_info);
5478}
5479
5480static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
5481{
5482 struct btrfs_fs_info *fs_info = root->fs_info;
5483 struct btrfs_block_group_cache *cache = NULL;
5484 struct btrfs_space_info *space_info;
5485 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5486 u64 len;
5487 bool readonly;
5488
5489 while (start <= end) {
5490 readonly = false;
5491 if (!cache ||
5492 start >= cache->key.objectid + cache->key.offset) {
5493 if (cache)
5494 btrfs_put_block_group(cache);
5495 cache = btrfs_lookup_block_group(fs_info, start);
5496 BUG_ON(!cache);
5497 }
5498
5499 len = cache->key.objectid + cache->key.offset - start;
5500 len = min(len, end + 1 - start);
5501
5502 if (start < cache->last_byte_to_unpin) {
5503 len = min(len, cache->last_byte_to_unpin - start);
5504 btrfs_add_free_space(cache, start, len);
5505 }
5506
5507 start += len;
5508 space_info = cache->space_info;
5509
5510 spin_lock(&space_info->lock);
5511 spin_lock(&cache->lock);
5512 cache->pinned -= len;
5513 space_info->bytes_pinned -= len;
5514 if (cache->ro) {
5515 space_info->bytes_readonly += len;
5516 readonly = true;
5517 }
5518 spin_unlock(&cache->lock);
5519 if (!readonly && global_rsv->space_info == space_info) {
5520 spin_lock(&global_rsv->lock);
5521 if (!global_rsv->full) {
5522 len = min(len, global_rsv->size -
5523 global_rsv->reserved);
5524 global_rsv->reserved += len;
5525 space_info->bytes_may_use += len;
5526 if (global_rsv->reserved >= global_rsv->size)
5527 global_rsv->full = 1;
5528 }
5529 spin_unlock(&global_rsv->lock);
5530 }
5531 spin_unlock(&space_info->lock);
5532 }
5533
5534 if (cache)
5535 btrfs_put_block_group(cache);
5536 return 0;
5537}
5538
5539int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
5540 struct btrfs_root *root)
5541{
5542 struct btrfs_fs_info *fs_info = root->fs_info;
5543 struct extent_io_tree *unpin;
5544 u64 start;
5545 u64 end;
5546 int ret;
5547
5548 if (trans->aborted)
5549 return 0;
5550
5551 if (fs_info->pinned_extents == &fs_info->freed_extents[0])
5552 unpin = &fs_info->freed_extents[1];
5553 else
5554 unpin = &fs_info->freed_extents[0];
5555
5556 while (1) {
5557 ret = find_first_extent_bit(unpin, 0, &start, &end,
5558 EXTENT_DIRTY, NULL);
5559 if (ret)
5560 break;
5561
5562 if (btrfs_test_opt(root, DISCARD))
5563 ret = btrfs_discard_extent(root, start,
5564 end + 1 - start, NULL);
5565
5566 clear_extent_dirty(unpin, start, end, GFP_NOFS);
5567 unpin_extent_range(root, start, end);
5568 cond_resched();
5569 }
5570
5571 return 0;
5572}
5573
5574static void add_pinned_bytes(struct btrfs_fs_info *fs_info, u64 num_bytes,
5575 u64 owner, u64 root_objectid)
5576{
5577 struct btrfs_space_info *space_info;
5578 u64 flags;
5579
5580 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
5581 if (root_objectid == BTRFS_CHUNK_TREE_OBJECTID)
5582 flags = BTRFS_BLOCK_GROUP_SYSTEM;
5583 else
5584 flags = BTRFS_BLOCK_GROUP_METADATA;
5585 } else {
5586 flags = BTRFS_BLOCK_GROUP_DATA;
5587 }
5588
5589 space_info = __find_space_info(fs_info, flags);
5590 BUG_ON(!space_info);
5591 percpu_counter_add(&space_info->total_bytes_pinned, num_bytes);
5592}
5593
5594
5595static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
5596 struct btrfs_root *root,
5597 u64 bytenr, u64 num_bytes, u64 parent,
5598 u64 root_objectid, u64 owner_objectid,
5599 u64 owner_offset, int refs_to_drop,
5600 struct btrfs_delayed_extent_op *extent_op)
5601{
5602 struct btrfs_key key;
5603 struct btrfs_path *path;
5604 struct btrfs_fs_info *info = root->fs_info;
5605 struct btrfs_root *extent_root = info->extent_root;
5606 struct extent_buffer *leaf;
5607 struct btrfs_extent_item *ei;
5608 struct btrfs_extent_inline_ref *iref;
5609 int ret;
5610 int is_data;
5611 int extent_slot = 0;
5612 int found_extent = 0;
5613 int num_to_del = 1;
5614 u32 item_size;
5615 u64 refs;
5616 bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
5617 SKINNY_METADATA);
5618
5619 path = btrfs_alloc_path();
5620 if (!path)
5621 return -ENOMEM;
5622
5623 path->reada = 1;
5624 path->leave_spinning = 1;
5625
5626 is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
5627 BUG_ON(!is_data && refs_to_drop != 1);
5628
5629 if (is_data)
5630 skinny_metadata = 0;
5631
5632 ret = lookup_extent_backref(trans, extent_root, path, &iref,
5633 bytenr, num_bytes, parent,
5634 root_objectid, owner_objectid,
5635 owner_offset);
5636 if (ret == 0) {
5637 extent_slot = path->slots[0];
5638 while (extent_slot >= 0) {
5639 btrfs_item_key_to_cpu(path->nodes[0], &key,
5640 extent_slot);
5641 if (key.objectid != bytenr)
5642 break;
5643 if (key.type == BTRFS_EXTENT_ITEM_KEY &&
5644 key.offset == num_bytes) {
5645 found_extent = 1;
5646 break;
5647 }
5648 if (key.type == BTRFS_METADATA_ITEM_KEY &&
5649 key.offset == owner_objectid) {
5650 found_extent = 1;
5651 break;
5652 }
5653 if (path->slots[0] - extent_slot > 5)
5654 break;
5655 extent_slot--;
5656 }
5657#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
5658 item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
5659 if (found_extent && item_size < sizeof(*ei))
5660 found_extent = 0;
5661#endif
5662 if (!found_extent) {
5663 BUG_ON(iref);
5664 ret = remove_extent_backref(trans, extent_root, path,
5665 NULL, refs_to_drop,
5666 is_data);
5667 if (ret) {
5668 btrfs_abort_transaction(trans, extent_root, ret);
5669 goto out;
5670 }
5671 btrfs_release_path(path);
5672 path->leave_spinning = 1;
5673
5674 key.objectid = bytenr;
5675 key.type = BTRFS_EXTENT_ITEM_KEY;
5676 key.offset = num_bytes;
5677
5678 if (!is_data && skinny_metadata) {
5679 key.type = BTRFS_METADATA_ITEM_KEY;
5680 key.offset = owner_objectid;
5681 }
5682
5683 ret = btrfs_search_slot(trans, extent_root,
5684 &key, path, -1, 1);
5685 if (ret > 0 && skinny_metadata && path->slots[0]) {
5686
5687
5688
5689
5690 path->slots[0]--;
5691 btrfs_item_key_to_cpu(path->nodes[0], &key,
5692 path->slots[0]);
5693 if (key.objectid == bytenr &&
5694 key.type == BTRFS_EXTENT_ITEM_KEY &&
5695 key.offset == num_bytes)
5696 ret = 0;
5697 }
5698
5699 if (ret > 0 && skinny_metadata) {
5700 skinny_metadata = false;
5701 key.type = BTRFS_EXTENT_ITEM_KEY;
5702 key.offset = num_bytes;
5703 btrfs_release_path(path);
5704 ret = btrfs_search_slot(trans, extent_root,
5705 &key, path, -1, 1);
5706 }
5707
5708 if (ret) {
5709 btrfs_err(info, "umm, got %d back from search, was looking for %llu",
5710 ret, bytenr);
5711 if (ret > 0)
5712 btrfs_print_leaf(extent_root,
5713 path->nodes[0]);
5714 }
5715 if (ret < 0) {
5716 btrfs_abort_transaction(trans, extent_root, ret);
5717 goto out;
5718 }
5719 extent_slot = path->slots[0];
5720 }
5721 } else if (ret == -ENOENT) {
5722 btrfs_print_leaf(extent_root, path->nodes[0]);
5723 WARN_ON(1);
5724 btrfs_err(info,
5725 "unable to find ref byte nr %llu parent %llu root %llu owner %llu offset %llu",
5726 bytenr, parent, root_objectid, owner_objectid,
5727 owner_offset);
5728 } else {
5729 btrfs_abort_transaction(trans, extent_root, ret);
5730 goto out;
5731 }
5732
5733 leaf = path->nodes[0];
5734 item_size = btrfs_item_size_nr(leaf, extent_slot);
5735#ifdef BTRFS_COMPAT_EXTENT_TREE_V0
5736 if (item_size < sizeof(*ei)) {
5737 BUG_ON(found_extent || extent_slot != path->slots[0]);
5738 ret = convert_extent_item_v0(trans, extent_root, path,
5739 owner_objectid, 0);
5740 if (ret < 0) {
5741 btrfs_abort_transaction(trans, extent_root, ret);
5742 goto out;
5743 }
5744
5745 btrfs_release_path(path);
5746 path->leave_spinning = 1;
5747
5748 key.objectid = bytenr;
5749 key.type = BTRFS_EXTENT_ITEM_KEY;
5750 key.offset = num_bytes;
5751
5752 ret = btrfs_search_slot(trans, extent_root, &key, path,
5753 -1, 1);
5754 if (ret) {
5755 btrfs_err(info, "umm, got %d back from search, was looking for %llu",
5756 ret, bytenr);
5757 btrfs_print_leaf(extent_root, path->nodes[0]);
5758 }
5759 if (ret < 0) {
5760 btrfs_abort_transaction(trans, extent_root, ret);
5761 goto out;
5762 }
5763
5764 extent_slot = path->slots[0];
5765 leaf = path->nodes[0];
5766 item_size = btrfs_item_size_nr(leaf, extent_slot);
5767 }
5768#endif
5769 BUG_ON(item_size < sizeof(*ei));
5770 ei = btrfs_item_ptr(leaf, extent_slot,
5771 struct btrfs_extent_item);
5772 if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID &&
5773 key.type == BTRFS_EXTENT_ITEM_KEY) {
5774 struct btrfs_tree_block_info *bi;
5775 BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
5776 bi = (struct btrfs_tree_block_info *)(ei + 1);
5777 WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
5778 }
5779
5780 refs = btrfs_extent_refs(leaf, ei);
5781 if (refs < refs_to_drop) {
5782 btrfs_err(info, "trying to drop %d refs but we only have %Lu "
5783 "for bytenr %Lu\n", refs_to_drop, refs, bytenr);
5784 ret = -EINVAL;
5785 btrfs_abort_transaction(trans, extent_root, ret);
5786 goto out;
5787 }
5788 refs -= refs_to_drop;
5789
5790 if (refs > 0) {
5791 if (extent_op)
5792 __run_delayed_extent_op(extent_op, leaf, ei);
5793
5794
5795
5796
5797 if (iref) {
5798 BUG_ON(!found_extent);
5799 } else {
5800 btrfs_set_extent_refs(leaf, ei, refs);
5801 btrfs_mark_buffer_dirty(leaf);
5802 }
5803 if (found_extent) {
5804 ret = remove_extent_backref(trans, extent_root, path,
5805 iref, refs_to_drop,
5806 is_data);
5807 if (ret) {
5808 btrfs_abort_transaction(trans, extent_root, ret);
5809 goto out;
5810 }
5811 }
5812 add_pinned_bytes(root->fs_info, -num_bytes, owner_objectid,
5813 root_objectid);
5814 } else {
5815 if (found_extent) {
5816 BUG_ON(is_data && refs_to_drop !=
5817 extent_data_ref_count(root, path, iref));
5818 if (iref) {
5819 BUG_ON(path->slots[0] != extent_slot);
5820 } else {
5821 BUG_ON(path->slots[0] != extent_slot + 1);
5822 path->slots[0] = extent_slot;
5823 num_to_del = 2;
5824 }
5825 }
5826
5827 ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
5828 num_to_del);
5829 if (ret) {
5830 btrfs_abort_transaction(trans, extent_root, ret);
5831 goto out;
5832 }
5833 btrfs_release_path(path);
5834
5835 if (is_data) {
5836 ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
5837 if (ret) {
5838 btrfs_abort_transaction(trans, extent_root, ret);
5839 goto out;
5840 }
5841 }
5842
5843 ret = update_block_group(root, bytenr, num_bytes, 0);
5844 if (ret) {
5845 btrfs_abort_transaction(trans, extent_root, ret);
5846 goto out;
5847 }
5848 }
5849out:
5850 btrfs_free_path(path);
5851 return ret;
5852}
5853
5854
5855
5856
5857
5858
5859
5860static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
5861 struct btrfs_root *root, u64 bytenr)
5862{
5863 struct btrfs_delayed_ref_head *head;
5864 struct btrfs_delayed_ref_root *delayed_refs;
5865 struct btrfs_delayed_ref_node *ref;
5866 struct rb_node *node;
5867 int ret = 0;
5868
5869 delayed_refs = &trans->transaction->delayed_refs;
5870 spin_lock(&delayed_refs->lock);
5871 head = btrfs_find_delayed_ref_head(trans, bytenr);
5872 if (!head)
5873 goto out;
5874
5875 node = rb_prev(&head->node.rb_node);
5876 if (!node)
5877 goto out;
5878
5879 ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
5880
5881
5882 if (ref->bytenr == bytenr)
5883 goto out;
5884
5885 if (head->extent_op) {
5886 if (!head->must_insert_reserved)
5887 goto out;
5888 btrfs_free_delayed_extent_op(head->extent_op);
5889 head->extent_op = NULL;
5890 }
5891
5892
5893
5894
5895
5896 if (!mutex_trylock(&head->mutex))
5897 goto out;
5898
5899
5900
5901
5902
5903 head->node.in_tree = 0;
5904 rb_erase(&head->node.rb_node, &delayed_refs->root);
5905
5906 delayed_refs->num_entries--;
5907
5908
5909
5910
5911
5912 delayed_refs->num_heads--;
5913 if (list_empty(&head->cluster))
5914 delayed_refs->num_heads_ready--;
5915
5916 list_del_init(&head->cluster);
5917 spin_unlock(&delayed_refs->lock);
5918
5919 BUG_ON(head->extent_op);
5920 if (head->must_insert_reserved)
5921 ret = 1;
5922
5923 mutex_unlock(&head->mutex);
5924 btrfs_put_delayed_ref(&head->node);
5925 return ret;
5926out:
5927 spin_unlock(&delayed_refs->lock);
5928 return 0;
5929}
5930
5931void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
5932 struct btrfs_root *root,
5933 struct extent_buffer *buf,
5934 u64 parent, int last_ref)
5935{
5936 struct btrfs_block_group_cache *cache = NULL;
5937 int pin = 1;
5938 int ret;
5939
5940 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
5941 ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
5942 buf->start, buf->len,
5943 parent, root->root_key.objectid,
5944 btrfs_header_level(buf),
5945 BTRFS_DROP_DELAYED_REF, NULL, 0);
5946 BUG_ON(ret);
5947 }
5948
5949 if (!last_ref)
5950 return;
5951
5952 cache = btrfs_lookup_block_group(root->fs_info, buf->start);
5953
5954 if (btrfs_header_generation(buf) == trans->transid) {
5955 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
5956 ret = check_ref_cleanup(trans, root, buf->start);
5957 if (!ret)
5958 goto out;
5959 }
5960
5961 if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
5962 pin_down_extent(root, cache, buf->start, buf->len, 1);
5963 goto out;
5964 }
5965
5966 WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));
5967
5968 btrfs_add_free_space(cache, buf->start, buf->len);
5969 btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
5970 pin = 0;
5971 }
5972out:
5973 if (pin)
5974 add_pinned_bytes(root->fs_info, buf->len,
5975 btrfs_header_level(buf),
5976 root->root_key.objectid);
5977
5978
5979
5980
5981
5982 clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
5983 btrfs_put_block_group(cache);
5984}
5985
5986
5987int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
5988 u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
5989 u64 owner, u64 offset, int for_cow)
5990{
5991 int ret;
5992 struct btrfs_fs_info *fs_info = root->fs_info;
5993
5994 add_pinned_bytes(root->fs_info, num_bytes, owner, root_objectid);
5995
5996
5997
5998
5999
6000 if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
6001 WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
6002
6003 btrfs_pin_extent(root, bytenr, num_bytes, 1);
6004 ret = 0;
6005 } else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
6006 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
6007 num_bytes,
6008 parent, root_objectid, (int)owner,
6009 BTRFS_DROP_DELAYED_REF, NULL, for_cow);
6010 } else {
6011 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
6012 num_bytes,
6013 parent, root_objectid, owner,
6014 offset, BTRFS_DROP_DELAYED_REF,
6015 NULL, for_cow);
6016 }
6017 return ret;
6018}
6019
6020static u64 stripe_align(struct btrfs_root *root,
6021 struct btrfs_block_group_cache *cache,
6022 u64 val, u64 num_bytes)
6023{
6024 u64 ret = ALIGN(val, root->stripesize);
6025 return ret;
6026}
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042static noinline void
6043wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
6044 u64 num_bytes)
6045{
6046 struct btrfs_caching_control *caching_ctl;
6047
6048 caching_ctl = get_caching_control(cache);
6049 if (!caching_ctl)
6050 return;
6051
6052 wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
6053 (cache->free_space_ctl->free_space >= num_bytes));
6054
6055 put_caching_control(caching_ctl);
6056}
6057
6058static noinline int
6059wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
6060{
6061 struct btrfs_caching_control *caching_ctl;
6062 int ret = 0;
6063
6064 caching_ctl = get_caching_control(cache);
6065 if (!caching_ctl)
6066 return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0;
6067
6068 wait_event(caching_ctl->wait, block_group_cache_done(cache));
6069 if (cache->cached == BTRFS_CACHE_ERROR)
6070 ret = -EIO;
6071 put_caching_control(caching_ctl);
6072 return ret;
6073}
6074
6075int __get_raid_index(u64 flags)
6076{
6077 if (flags & BTRFS_BLOCK_GROUP_RAID10)
6078 return BTRFS_RAID_RAID10;
6079 else if (flags & BTRFS_BLOCK_GROUP_RAID1)
6080 return BTRFS_RAID_RAID1;
6081 else if (flags & BTRFS_BLOCK_GROUP_DUP)
6082 return BTRFS_RAID_DUP;
6083 else if (flags & BTRFS_BLOCK_GROUP_RAID0)
6084 return BTRFS_RAID_RAID0;
6085 else if (flags & BTRFS_BLOCK_GROUP_RAID5)
6086 return BTRFS_RAID_RAID5;
6087 else if (flags & BTRFS_BLOCK_GROUP_RAID6)
6088 return BTRFS_RAID_RAID6;
6089
6090 return BTRFS_RAID_SINGLE;
6091}
6092
6093static int get_block_group_index(struct btrfs_block_group_cache *cache)
6094{
6095 return __get_raid_index(cache->flags);
6096}
6097
6098enum btrfs_loop_type {
6099 LOOP_CACHING_NOWAIT = 0,
6100 LOOP_CACHING_WAIT = 1,
6101 LOOP_ALLOC_CHUNK = 2,
6102 LOOP_NO_EMPTY_SIZE = 3,
6103};
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116static noinline int find_free_extent(struct btrfs_root *orig_root,
6117 u64 num_bytes, u64 empty_size,
6118 u64 hint_byte, struct btrfs_key *ins,
6119 u64 flags)
6120{
6121 int ret = 0;
6122 struct btrfs_root *root = orig_root->fs_info->extent_root;
6123 struct btrfs_free_cluster *last_ptr = NULL;
6124 struct btrfs_block_group_cache *block_group = NULL;
6125 struct btrfs_block_group_cache *used_block_group;
6126 u64 search_start = 0;
6127 u64 max_extent_size = 0;
6128 int empty_cluster = 2 * 1024 * 1024;
6129 struct btrfs_space_info *space_info;
6130 int loop = 0;
6131 int index = __get_raid_index(flags);
6132 int alloc_type = (flags & BTRFS_BLOCK_GROUP_DATA) ?
6133 RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
6134 bool found_uncached_bg = false;
6135 bool failed_cluster_refill = false;
6136 bool failed_alloc = false;
6137 bool use_cluster = true;
6138 bool have_caching_bg = false;
6139
6140 WARN_ON(num_bytes < root->sectorsize);
6141 btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
6142 ins->objectid = 0;
6143 ins->offset = 0;
6144
6145 trace_find_free_extent(orig_root, num_bytes, empty_size, flags);
6146
6147 space_info = __find_space_info(root->fs_info, flags);
6148 if (!space_info) {
6149 btrfs_err(root->fs_info, "No space info for %llu", flags);
6150 return -ENOSPC;
6151 }
6152
6153
6154
6155
6156
6157 if (btrfs_mixed_space_info(space_info))
6158 use_cluster = false;
6159
6160 if (flags & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
6161 last_ptr = &root->fs_info->meta_alloc_cluster;
6162 if (!btrfs_test_opt(root, SSD))
6163 empty_cluster = 64 * 1024;
6164 }
6165
6166 if ((flags & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
6167 btrfs_test_opt(root, SSD)) {
6168 last_ptr = &root->fs_info->data_alloc_cluster;
6169 }
6170
6171 if (last_ptr) {
6172 spin_lock(&last_ptr->lock);
6173 if (last_ptr->block_group)
6174 hint_byte = last_ptr->window_start;
6175 spin_unlock(&last_ptr->lock);
6176 }
6177
6178 search_start = max(search_start, first_logical_byte(root, 0));
6179 search_start = max(search_start, hint_byte);
6180
6181 if (!last_ptr)
6182 empty_cluster = 0;
6183
6184 if (search_start == hint_byte) {
6185 block_group = btrfs_lookup_block_group(root->fs_info,
6186 search_start);
6187 used_block_group = block_group;
6188
6189
6190
6191
6192
6193
6194
6195 if (block_group && block_group_bits(block_group, flags) &&
6196 block_group->cached != BTRFS_CACHE_NO) {
6197 down_read(&space_info->groups_sem);
6198 if (list_empty(&block_group->list) ||
6199 block_group->ro) {
6200
6201
6202
6203
6204
6205
6206 btrfs_put_block_group(block_group);
6207 up_read(&space_info->groups_sem);
6208 } else {
6209 index = get_block_group_index(block_group);
6210 goto have_block_group;
6211 }
6212 } else if (block_group) {
6213 btrfs_put_block_group(block_group);
6214 }
6215 }
6216search:
6217 have_caching_bg = false;
6218 down_read(&space_info->groups_sem);
6219 list_for_each_entry(block_group, &space_info->block_groups[index],
6220 list) {
6221 u64 offset;
6222 int cached;
6223
6224 used_block_group = block_group;
6225 btrfs_get_block_group(block_group);
6226 search_start = block_group->key.objectid;
6227
6228
6229
6230
6231
6232
6233 if (!block_group_bits(block_group, flags)) {
6234 u64 extra = BTRFS_BLOCK_GROUP_DUP |
6235 BTRFS_BLOCK_GROUP_RAID1 |
6236 BTRFS_BLOCK_GROUP_RAID5 |
6237 BTRFS_BLOCK_GROUP_RAID6 |
6238 BTRFS_BLOCK_GROUP_RAID10;
6239
6240
6241
6242
6243
6244
6245 if ((flags & extra) && !(block_group->flags & extra))
6246 goto loop;
6247 }
6248
6249have_block_group:
6250 cached = block_group_cache_done(block_group);
6251 if (unlikely(!cached)) {
6252 found_uncached_bg = true;
6253 ret = cache_block_group(block_group, 0);
6254 BUG_ON(ret < 0);
6255 ret = 0;
6256 }
6257
6258 if (unlikely(block_group->cached == BTRFS_CACHE_ERROR))
6259 goto loop;
6260 if (unlikely(block_group->ro))
6261 goto loop;
6262
6263
6264
6265
6266
6267 if (last_ptr) {
6268 unsigned long aligned_cluster;
6269
6270
6271
6272
6273 spin_lock(&last_ptr->refill_lock);
6274 used_block_group = last_ptr->block_group;
6275 if (used_block_group != block_group &&
6276 (!used_block_group ||
6277 used_block_group->ro ||
6278 !block_group_bits(used_block_group, flags))) {
6279 used_block_group = block_group;
6280 goto refill_cluster;
6281 }
6282
6283 if (used_block_group != block_group)
6284 btrfs_get_block_group(used_block_group);
6285
6286 offset = btrfs_alloc_from_cluster(used_block_group,
6287 last_ptr,
6288 num_bytes,
6289 used_block_group->key.objectid,
6290 &max_extent_size);
6291 if (offset) {
6292
6293 spin_unlock(&last_ptr->refill_lock);
6294 trace_btrfs_reserve_extent_cluster(root,
6295 block_group, search_start, num_bytes);
6296 goto checks;
6297 }
6298
6299 WARN_ON(last_ptr->block_group != used_block_group);
6300 if (used_block_group != block_group) {
6301 btrfs_put_block_group(used_block_group);
6302 used_block_group = block_group;
6303 }
6304refill_cluster:
6305 BUG_ON(used_block_group != block_group);
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321 if (loop >= LOOP_NO_EMPTY_SIZE &&
6322 last_ptr->block_group != block_group) {
6323 spin_unlock(&last_ptr->refill_lock);
6324 goto unclustered_alloc;
6325 }
6326
6327
6328
6329
6330
6331 btrfs_return_cluster_to_free_space(NULL, last_ptr);
6332
6333 if (loop >= LOOP_NO_EMPTY_SIZE) {
6334 spin_unlock(&last_ptr->refill_lock);
6335 goto unclustered_alloc;
6336 }
6337
6338 aligned_cluster = max_t(unsigned long,
6339 empty_cluster + empty_size,
6340 block_group->full_stripe_len);
6341
6342
6343 ret = btrfs_find_space_cluster(root, block_group,
6344 last_ptr, search_start,
6345 num_bytes,
6346 aligned_cluster);
6347 if (ret == 0) {
6348
6349
6350
6351
6352 offset = btrfs_alloc_from_cluster(block_group,
6353 last_ptr,
6354 num_bytes,
6355 search_start,
6356 &max_extent_size);
6357 if (offset) {
6358
6359 spin_unlock(&last_ptr->refill_lock);
6360 trace_btrfs_reserve_extent_cluster(root,
6361 block_group, search_start,
6362 num_bytes);
6363 goto checks;
6364 }
6365 } else if (!cached && loop > LOOP_CACHING_NOWAIT
6366 && !failed_cluster_refill) {
6367 spin_unlock(&last_ptr->refill_lock);
6368
6369 failed_cluster_refill = true;
6370 wait_block_group_cache_progress(block_group,
6371 num_bytes + empty_cluster + empty_size);
6372 goto have_block_group;
6373 }
6374
6375
6376
6377
6378
6379
6380
6381 btrfs_return_cluster_to_free_space(NULL, last_ptr);
6382 spin_unlock(&last_ptr->refill_lock);
6383 goto loop;
6384 }
6385
6386unclustered_alloc:
6387 spin_lock(&block_group->free_space_ctl->tree_lock);
6388 if (cached &&
6389 block_group->free_space_ctl->free_space <
6390 num_bytes + empty_cluster + empty_size) {
6391 if (block_group->free_space_ctl->free_space >
6392 max_extent_size)
6393 max_extent_size =
6394 block_group->free_space_ctl->free_space;
6395 spin_unlock(&block_group->free_space_ctl->tree_lock);
6396 goto loop;
6397 }
6398 spin_unlock(&block_group->free_space_ctl->tree_lock);
6399
6400 offset = btrfs_find_space_for_alloc(block_group, search_start,
6401 num_bytes, empty_size,
6402 &max_extent_size);
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412 if (!offset && !failed_alloc && !cached &&
6413 loop > LOOP_CACHING_NOWAIT) {
6414 wait_block_group_cache_progress(block_group,
6415 num_bytes + empty_size);
6416 failed_alloc = true;
6417 goto have_block_group;
6418 } else if (!offset) {
6419 if (!cached)
6420 have_caching_bg = true;
6421 goto loop;
6422 }
6423checks:
6424 search_start = stripe_align(root, used_block_group,
6425 offset, num_bytes);
6426
6427
6428 if (search_start + num_bytes >
6429 used_block_group->key.objectid + used_block_group->key.offset) {
6430 btrfs_add_free_space(used_block_group, offset, num_bytes);
6431 goto loop;
6432 }
6433
6434 if (offset < search_start)
6435 btrfs_add_free_space(used_block_group, offset,
6436 search_start - offset);
6437 BUG_ON(offset > search_start);
6438
6439 ret = btrfs_update_reserved_bytes(used_block_group, num_bytes,
6440 alloc_type);
6441 if (ret == -EAGAIN) {
6442 btrfs_add_free_space(used_block_group, offset, num_bytes);
6443 goto loop;
6444 }
6445
6446
6447 ins->objectid = search_start;
6448 ins->offset = num_bytes;
6449
6450 trace_btrfs_reserve_extent(orig_root, block_group,
6451 search_start, num_bytes);
6452 if (used_block_group != block_group)
6453 btrfs_put_block_group(used_block_group);
6454 btrfs_put_block_group(block_group);
6455 break;
6456loop:
6457 failed_cluster_refill = false;
6458 failed_alloc = false;
6459 BUG_ON(index != get_block_group_index(block_group));
6460 if (used_block_group != block_group)
6461 btrfs_put_block_group(used_block_group);
6462 btrfs_put_block_group(block_group);
6463 }
6464 up_read(&space_info->groups_sem);
6465
6466 if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
6467 goto search;
6468
6469 if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
6470 goto search;
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480 if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
6481 index = 0;
6482 loop++;
6483 if (loop == LOOP_ALLOC_CHUNK) {
6484 struct btrfs_trans_handle *trans;
6485
6486 trans = btrfs_join_transaction(root);
6487 if (IS_ERR(trans)) {
6488 ret = PTR_ERR(trans);
6489 goto out;
6490 }
6491
6492 ret = do_chunk_alloc(trans, root, flags,
6493 CHUNK_ALLOC_FORCE);
6494
6495
6496
6497
6498 if (ret < 0 && ret != -ENOSPC)
6499 btrfs_abort_transaction(trans,
6500 root, ret);
6501 else
6502 ret = 0;
6503 btrfs_end_transaction(trans, root);
6504 if (ret)
6505 goto out;
6506 }
6507
6508 if (loop == LOOP_NO_EMPTY_SIZE) {
6509 empty_size = 0;
6510 empty_cluster = 0;
6511 }
6512
6513 goto search;
6514 } else if (!ins->objectid) {
6515 ret = -ENOSPC;
6516 } else if (ins->objectid) {
6517 ret = 0;
6518 }
6519out:
6520 if (ret == -ENOSPC)
6521 ins->offset = max_extent_size;
6522 return ret;
6523}
6524
6525static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
6526 int dump_block_groups)
6527{
6528 struct btrfs_block_group_cache *cache;
6529 int index = 0;
6530
6531 spin_lock(&info->lock);
6532 printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
6533 info->flags,
6534 info->total_bytes - info->bytes_used - info->bytes_pinned -
6535 info->bytes_reserved - info->bytes_readonly,
6536 (info->full) ? "" : "not ");
6537 printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
6538 "reserved=%llu, may_use=%llu, readonly=%llu\n",
6539 info->total_bytes, info->bytes_used, info->bytes_pinned,
6540 info->bytes_reserved, info->bytes_may_use,
6541 info->bytes_readonly);
6542 spin_unlock(&info->lock);
6543
6544 if (!dump_block_groups)
6545 return;
6546
6547 down_read(&info->groups_sem);
6548again:
6549 list_for_each_entry(cache, &info->block_groups[index], list) {
6550 spin_lock(&cache->lock);
6551 printk(KERN_INFO "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s\n",
6552 cache->key.objectid, cache->key.offset,
6553 btrfs_block_group_used(&cache->item), cache->pinned,
6554 cache->reserved, cache->ro ? "[readonly]" : "");
6555 btrfs_dump_free_space(cache, bytes);
6556 spin_unlock(&cache->lock);
6557 }
6558 if (++index < BTRFS_NR_RAID_TYPES)
6559 goto again;
6560 up_read(&info->groups_sem);
6561}
6562
6563int btrfs_reserve_extent(struct btrfs_root *root,
6564 u64 num_bytes, u64 min_alloc_size,
6565 u64 empty_size, u64 hint_byte,
6566 struct btrfs_key *ins, int is_data)
6567{
6568 bool final_tried = false;
6569 u64 flags;
6570 int ret;
6571
6572 flags = btrfs_get_alloc_profile(root, is_data);
6573again:
6574 WARN_ON(num_bytes < root->sectorsize);
6575 ret = find_free_extent(root, num_bytes, empty_size, hint_byte, ins,
6576 flags);
6577
6578 if (ret == -ENOSPC) {
6579 if (!final_tried && ins->offset) {
6580 num_bytes = min(num_bytes >> 1, ins->offset);
6581 num_bytes = round_down(num_bytes, root->sectorsize);
6582 num_bytes = max(num_bytes, min_alloc_size);
6583 if (num_bytes == min_alloc_size)
6584 final_tried = true;
6585 goto again;
6586 } else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
6587 struct btrfs_space_info *sinfo;
6588
6589 sinfo = __find_space_info(root->fs_info, flags);
6590 btrfs_err(root->fs_info, "allocation failed flags %llu, wanted %llu",
6591 flags, num_bytes);
6592 if (sinfo)
6593 dump_space_info(sinfo, num_bytes, 1);
6594 }
6595 }
6596
6597 trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);
6598
6599 return ret;
6600}
6601
6602static int __btrfs_free_reserved_extent(struct btrfs_root *root,
6603 u64 start, u64 len, int pin)
6604{
6605 struct btrfs_block_group_cache *cache;
6606 int ret = 0;
6607
6608 cache = btrfs_lookup_block_group(root->fs_info, start);
6609 if (!cache) {
6610 btrfs_err(root->fs_info, "Unable to find block group for %llu",
6611 start);
6612 return -ENOSPC;
6613 }
6614
6615 if (btrfs_test_opt(root, DISCARD))
6616 ret = btrfs_discard_extent(root, start, len, NULL);
6617
6618 if (pin)
6619 pin_down_extent(root, cache, start, len, 1);
6620 else {
6621 btrfs_add_free_space(cache, start, len);
6622 btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
6623 }
6624 btrfs_put_block_group(cache);
6625
6626 trace_btrfs_reserved_extent_free(root, start, len);
6627
6628 return ret;
6629}
6630
6631int btrfs_free_reserved_extent(struct btrfs_root *root,
6632 u64 start, u64 len)
6633{
6634 return __btrfs_free_reserved_extent(root, start, len, 0);
6635}
6636
6637int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
6638 u64 start, u64 len)
6639{
6640 return __btrfs_free_reserved_extent(root, start, len, 1);
6641}
6642
6643static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
6644 struct btrfs_root *root,
6645 u64 parent, u64 root_objectid,
6646 u64 flags, u64 owner, u64 offset,
6647 struct btrfs_key *ins, int ref_mod)
6648{
6649 int ret;
6650 struct btrfs_fs_info *fs_info = root->fs_info;
6651 struct btrfs_extent_item *extent_item;
6652 struct btrfs_extent_inline_ref *iref;
6653 struct btrfs_path *path;
6654 struct extent_buffer *leaf;
6655 int type;
6656 u32 size;
6657
6658 if (parent > 0)
6659 type = BTRFS_SHARED_DATA_REF_KEY;
6660 else
6661 type = BTRFS_EXTENT_DATA_REF_KEY;
6662
6663 size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
6664
6665 path = btrfs_alloc_path();
6666 if (!path)
6667 return -ENOMEM;
6668
6669 path->leave_spinning = 1;
6670 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
6671 ins, size);
6672 if (ret) {
6673 btrfs_free_path(path);
6674 return ret;
6675 }
6676
6677 leaf = path->nodes[0];
6678 extent_item = btrfs_item_ptr(leaf, path->slots[0],
6679 struct btrfs_extent_item);
6680 btrfs_set_extent_refs(leaf, extent_item, ref_mod);
6681 btrfs_set_extent_generation(leaf, extent_item, trans->transid);
6682 btrfs_set_extent_flags(leaf, extent_item,
6683 flags | BTRFS_EXTENT_FLAG_DATA);
6684
6685 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
6686 btrfs_set_extent_inline_ref_type(leaf, iref, type);
6687 if (parent > 0) {
6688 struct btrfs_shared_data_ref *ref;
6689 ref = (struct btrfs_shared_data_ref *)(iref + 1);
6690 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
6691 btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
6692 } else {
6693 struct btrfs_extent_data_ref *ref;
6694 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
6695 btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
6696 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
6697 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
6698 btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
6699 }
6700
6701 btrfs_mark_buffer_dirty(path->nodes[0]);
6702 btrfs_free_path(path);
6703
6704 ret = update_block_group(root, ins->objectid, ins->offset, 1);
6705 if (ret) {
6706 btrfs_err(fs_info, "update block group failed for %llu %llu",
6707 ins->objectid, ins->offset);
6708 BUG();
6709 }
6710 return ret;
6711}
6712
6713static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
6714 struct btrfs_root *root,
6715 u64 parent, u64 root_objectid,
6716 u64 flags, struct btrfs_disk_key *key,
6717 int level, struct btrfs_key *ins)
6718{
6719 int ret;
6720 struct btrfs_fs_info *fs_info = root->fs_info;
6721 struct btrfs_extent_item *extent_item;
6722 struct btrfs_tree_block_info *block_info;
6723 struct btrfs_extent_inline_ref *iref;
6724 struct btrfs_path *path;
6725 struct extent_buffer *leaf;
6726 u32 size = sizeof(*extent_item) + sizeof(*iref);
6727 bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
6728 SKINNY_METADATA);
6729
6730 if (!skinny_metadata)
6731 size += sizeof(*block_info);
6732
6733 path = btrfs_alloc_path();
6734 if (!path)
6735 return -ENOMEM;
6736
6737 path->leave_spinning = 1;
6738 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
6739 ins, size);
6740 if (ret) {
6741 btrfs_free_path(path);
6742 return ret;
6743 }
6744
6745 leaf = path->nodes[0];
6746 extent_item = btrfs_item_ptr(leaf, path->slots[0],
6747 struct btrfs_extent_item);
6748 btrfs_set_extent_refs(leaf, extent_item, 1);
6749 btrfs_set_extent_generation(leaf, extent_item, trans->transid);
6750 btrfs_set_extent_flags(leaf, extent_item,
6751 flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
6752
6753 if (skinny_metadata) {
6754 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
6755 } else {
6756 block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
6757 btrfs_set_tree_block_key(leaf, block_info, key);
6758 btrfs_set_tree_block_level(leaf, block_info, level);
6759 iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
6760 }
6761
6762 if (parent > 0) {
6763 BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
6764 btrfs_set_extent_inline_ref_type(leaf, iref,
6765 BTRFS_SHARED_BLOCK_REF_KEY);
6766 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
6767 } else {
6768 btrfs_set_extent_inline_ref_type(leaf, iref,
6769 BTRFS_TREE_BLOCK_REF_KEY);
6770 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
6771 }
6772
6773 btrfs_mark_buffer_dirty(leaf);
6774 btrfs_free_path(path);
6775
6776 ret = update_block_group(root, ins->objectid, root->leafsize, 1);
6777 if (ret) {
6778 btrfs_err(fs_info, "update block group failed for %llu %llu",
6779 ins->objectid, ins->offset);
6780 BUG();
6781 }
6782 return ret;
6783}
6784
6785int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
6786 struct btrfs_root *root,
6787 u64 root_objectid, u64 owner,
6788 u64 offset, struct btrfs_key *ins)
6789{
6790 int ret;
6791
6792 BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);
6793
6794 ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
6795 ins->offset, 0,
6796 root_objectid, owner, offset,
6797 BTRFS_ADD_DELAYED_EXTENT, NULL, 0);
6798 return ret;
6799}
6800
6801
6802
6803
6804
6805
6806int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
6807 struct btrfs_root *root,
6808 u64 root_objectid, u64 owner, u64 offset,
6809 struct btrfs_key *ins)
6810{
6811 int ret;
6812 struct btrfs_block_group_cache *block_group;
6813
6814
6815
6816
6817
6818 if (!btrfs_fs_incompat(root->fs_info, MIXED_GROUPS)) {
6819 ret = __exclude_logged_extent(root, ins->objectid, ins->offset);
6820 if (ret)
6821 return ret;
6822 }
6823
6824 block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
6825 if (!block_group)
6826 return -EINVAL;
6827
6828 ret = btrfs_update_reserved_bytes(block_group, ins->offset,
6829 RESERVE_ALLOC_NO_ACCOUNT);
6830 BUG_ON(ret);
6831 ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
6832 0, owner, offset, ins, 1);
6833 btrfs_put_block_group(block_group);
6834 return ret;
6835}
6836
6837static struct extent_buffer *
6838btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root,
6839 u64 bytenr, u32 blocksize, int level)
6840{
6841 struct extent_buffer *buf;
6842
6843 buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
6844 if (!buf)
6845 return ERR_PTR(-ENOMEM);
6846 btrfs_set_header_generation(buf, trans->transid);
6847 btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
6848 btrfs_tree_lock(buf);
6849 clean_tree_block(trans, root, buf);
6850 clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
6851
6852 btrfs_set_lock_blocking(buf);
6853 btrfs_set_buffer_uptodate(buf);
6854
6855 if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
6856
6857
6858
6859
6860 if (root->log_transid % 2 == 0)
6861 set_extent_dirty(&root->dirty_log_pages, buf->start,
6862 buf->start + buf->len - 1, GFP_NOFS);
6863 else
6864 set_extent_new(&root->dirty_log_pages, buf->start,
6865 buf->start + buf->len - 1, GFP_NOFS);
6866 } else {
6867 set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
6868 buf->start + buf->len - 1, GFP_NOFS);
6869 }
6870 trans->blocks_used++;
6871
6872 return buf;
6873}
6874
6875static struct btrfs_block_rsv *
6876use_block_rsv(struct btrfs_trans_handle *trans,
6877 struct btrfs_root *root, u32 blocksize)
6878{
6879 struct btrfs_block_rsv *block_rsv;
6880 struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
6881 int ret;
6882 bool global_updated = false;
6883
6884 block_rsv = get_block_rsv(trans, root);
6885
6886 if (unlikely(block_rsv->size == 0))
6887 goto try_reserve;
6888again:
6889 ret = block_rsv_use_bytes(block_rsv, blocksize);
6890 if (!ret)
6891 return block_rsv;
6892
6893 if (block_rsv->failfast)
6894 return ERR_PTR(ret);
6895
6896 if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) {
6897 global_updated = true;
6898 update_global_block_rsv(root->fs_info);
6899 goto again;
6900 }
6901
6902 if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
6903 static DEFINE_RATELIMIT_STATE(_rs,
6904 DEFAULT_RATELIMIT_INTERVAL * 10,
6905 1);
6906 if (__ratelimit(&_rs))
6907 WARN(1, KERN_DEBUG
6908 "btrfs: block rsv returned %d\n", ret);
6909 }
6910try_reserve:
6911 ret = reserve_metadata_bytes(root, block_rsv, blocksize,
6912 BTRFS_RESERVE_NO_FLUSH);
6913 if (!ret)
6914 return block_rsv;
6915
6916
6917
6918
6919
6920 if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL &&
6921 block_rsv->space_info == global_rsv->space_info) {
6922 ret = block_rsv_use_bytes(global_rsv, blocksize);
6923 if (!ret)
6924 return global_rsv;
6925 }
6926 return ERR_PTR(ret);
6927}
6928
6929static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
6930 struct btrfs_block_rsv *block_rsv, u32 blocksize)
6931{
6932 block_rsv_add_bytes(block_rsv, blocksize, 0);
6933 block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
6934}
6935
6936
6937
6938
6939
6940
6941
6942
6943struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
6944 struct btrfs_root *root, u32 blocksize,
6945 u64 parent, u64 root_objectid,
6946 struct btrfs_disk_key *key, int level,
6947 u64 hint, u64 empty_size)
6948{
6949 struct btrfs_key ins;
6950 struct btrfs_block_rsv *block_rsv;
6951 struct extent_buffer *buf;
6952 u64 flags = 0;
6953 int ret;
6954 bool skinny_metadata = btrfs_fs_incompat(root->fs_info,
6955 SKINNY_METADATA);
6956
6957 block_rsv = use_block_rsv(trans, root, blocksize);
6958 if (IS_ERR(block_rsv))
6959 return ERR_CAST(block_rsv);
6960
6961 ret = btrfs_reserve_extent(root, blocksize, blocksize,
6962 empty_size, hint, &ins, 0);
6963 if (ret) {
6964 unuse_block_rsv(root->fs_info, block_rsv, blocksize);
6965 return ERR_PTR(ret);
6966 }
6967
6968 buf = btrfs_init_new_buffer(trans, root, ins.objectid,
6969 blocksize, level);
6970 BUG_ON(IS_ERR(buf));
6971
6972 if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
6973 if (parent == 0)
6974 parent = ins.objectid;
6975 flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
6976 } else
6977 BUG_ON(parent > 0);
6978
6979 if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
6980 struct btrfs_delayed_extent_op *extent_op;
6981 extent_op = btrfs_alloc_delayed_extent_op();
6982 BUG_ON(!extent_op);
6983 if (key)
6984 memcpy(&extent_op->key, key, sizeof(extent_op->key));
6985 else
6986 memset(&extent_op->key, 0, sizeof(extent_op->key));
6987 extent_op->flags_to_set = flags;
6988 if (skinny_metadata)
6989 extent_op->update_key = 0;
6990 else
6991 extent_op->update_key = 1;
6992 extent_op->update_flags = 1;
6993 extent_op->is_data = 0;
6994 extent_op->level = level;
6995
6996 ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
6997 ins.objectid,
6998 ins.offset, parent, root_objectid,
6999 level, BTRFS_ADD_DELAYED_EXTENT,
7000 extent_op, 0);
7001 BUG_ON(ret);
7002 }
7003 return buf;
7004}
7005
7006struct walk_control {
7007 u64 refs[BTRFS_MAX_LEVEL];
7008 u64 flags[BTRFS_MAX_LEVEL];
7009 struct btrfs_key update_progress;
7010 int stage;
7011 int level;
7012 int shared_level;
7013 int update_ref;
7014 int keep_locks;
7015 int reada_slot;
7016 int reada_count;
7017 int for_reloc;
7018};
7019
7020#define DROP_REFERENCE 1
7021#define UPDATE_BACKREF 2
7022
7023static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
7024 struct btrfs_root *root,
7025 struct walk_control *wc,
7026 struct btrfs_path *path)
7027{
7028 u64 bytenr;
7029 u64 generation;
7030 u64 refs;
7031 u64 flags;
7032 u32 nritems;
7033 u32 blocksize;
7034 struct btrfs_key key;
7035 struct extent_buffer *eb;
7036 int ret;
7037 int slot;
7038 int nread = 0;
7039
7040 if (path->slots[wc->level] < wc->reada_slot) {
7041 wc->reada_count = wc->reada_count * 2 / 3;
7042 wc->reada_count = max(wc->reada_count, 2);
7043 } else {
7044 wc->reada_count = wc->reada_count * 3 / 2;
7045 wc->reada_count = min_t(int, wc->reada_count,
7046 BTRFS_NODEPTRS_PER_BLOCK(root));
7047 }
7048
7049 eb = path->nodes[wc->level];
7050 nritems = btrfs_header_nritems(eb);
7051 blocksize = btrfs_level_size(root, wc->level - 1);
7052
7053 for (slot = path->slots[wc->level]; slot < nritems; slot++) {
7054 if (nread >= wc->reada_count)
7055 break;
7056
7057 cond_resched();
7058 bytenr = btrfs_node_blockptr(eb, slot);
7059 generation = btrfs_node_ptr_generation(eb, slot);
7060
7061 if (slot == path->slots[wc->level])
7062 goto reada;
7063
7064 if (wc->stage == UPDATE_BACKREF &&
7065 generation <= root->root_key.offset)
7066 continue;
7067
7068
7069 ret = btrfs_lookup_extent_info(trans, root, bytenr,
7070 wc->level - 1, 1, &refs,
7071 &flags);
7072
7073 if (ret < 0)
7074 continue;
7075 BUG_ON(refs == 0);
7076
7077 if (wc->stage == DROP_REFERENCE) {
7078 if (refs == 1)
7079 goto reada;
7080
7081 if (wc->level == 1 &&
7082 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
7083 continue;
7084 if (!wc->update_ref ||
7085 generation <= root->root_key.offset)
7086 continue;
7087 btrfs_node_key_to_cpu(eb, &key, slot);
7088 ret = btrfs_comp_cpu_keys(&key,
7089 &wc->update_progress);
7090 if (ret < 0)
7091 continue;
7092 } else {
7093 if (wc->level == 1 &&
7094 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
7095 continue;
7096 }
7097reada:
7098 ret = readahead_tree_block(root, bytenr, blocksize,
7099 generation);
7100 if (ret)
7101 break;
7102 nread++;
7103 }
7104 wc->reada_slot = slot;
7105}
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
7116 struct btrfs_root *root,
7117 struct btrfs_path *path,
7118 struct walk_control *wc, int lookup_info)
7119{
7120 int level = wc->level;
7121 struct extent_buffer *eb = path->nodes[level];
7122 u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
7123 int ret;
7124
7125 if (wc->stage == UPDATE_BACKREF &&
7126 btrfs_header_owner(eb) != root->root_key.objectid)
7127 return 1;
7128
7129
7130
7131
7132
7133 if (lookup_info &&
7134 ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
7135 (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
7136 BUG_ON(!path->locks[level]);
7137 ret = btrfs_lookup_extent_info(trans, root,
7138 eb->start, level, 1,
7139 &wc->refs[level],
7140 &wc->flags[level]);
7141 BUG_ON(ret == -ENOMEM);
7142 if (ret)
7143 return ret;
7144 BUG_ON(wc->refs[level] == 0);
7145 }
7146
7147 if (wc->stage == DROP_REFERENCE) {
7148 if (wc->refs[level] > 1)
7149 return 1;
7150
7151 if (path->locks[level] && !wc->keep_locks) {
7152 btrfs_tree_unlock_rw(eb, path->locks[level]);
7153 path->locks[level] = 0;
7154 }
7155 return 0;
7156 }
7157
7158
7159 if (!(wc->flags[level] & flag)) {
7160 BUG_ON(!path->locks[level]);
7161 ret = btrfs_inc_ref(trans, root, eb, 1, wc->for_reloc);
7162 BUG_ON(ret);
7163 ret = btrfs_dec_ref(trans, root, eb, 0, wc->for_reloc);
7164 BUG_ON(ret);
7165 ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
7166 eb->len, flag,
7167 btrfs_header_level(eb), 0);
7168 BUG_ON(ret);
7169 wc->flags[level] |= flag;
7170 }
7171
7172
7173
7174
7175
7176 if (path->locks[level] && level > 0) {
7177 btrfs_tree_unlock_rw(eb, path->locks[level]);
7178 path->locks[level] = 0;
7179 }
7180 return 0;
7181}
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
7192
7193
7194
7195
7196static noinline int do_walk_down(struct btrfs_trans_handle *trans,
7197 struct btrfs_root *root,
7198 struct btrfs_path *path,
7199 struct walk_control *wc, int *lookup_info)
7200{
7201 u64 bytenr;
7202 u64 generation;
7203 u64 parent;
7204 u32 blocksize;
7205 struct btrfs_key key;
7206 struct extent_buffer *next;
7207 int level = wc->level;
7208 int reada = 0;
7209 int ret = 0;
7210
7211 generation = btrfs_node_ptr_generation(path->nodes[level],
7212 path->slots[level]);
7213
7214
7215
7216
7217
7218 if (wc->stage == UPDATE_BACKREF &&
7219 generation <= root->root_key.offset) {
7220 *lookup_info = 1;
7221 return 1;
7222 }
7223
7224 bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
7225 blocksize = btrfs_level_size(root, level - 1);
7226
7227 next = btrfs_find_tree_block(root, bytenr, blocksize);
7228 if (!next) {
7229 next = btrfs_find_create_tree_block(root, bytenr, blocksize);
7230 if (!next)
7231 return -ENOMEM;
7232 btrfs_set_buffer_lockdep_class(root->root_key.objectid, next,
7233 level - 1);
7234 reada = 1;
7235 }
7236 btrfs_tree_lock(next);
7237 btrfs_set_lock_blocking(next);
7238
7239 ret = btrfs_lookup_extent_info(trans, root, bytenr, level - 1, 1,
7240 &wc->refs[level - 1],
7241 &wc->flags[level - 1]);
7242 if (ret < 0) {
7243 btrfs_tree_unlock(next);
7244 return ret;
7245 }
7246
7247 if (unlikely(wc->refs[level - 1] == 0)) {
7248 btrfs_err(root->fs_info, "Missing references.");
7249 BUG();
7250 }
7251 *lookup_info = 0;
7252
7253 if (wc->stage == DROP_REFERENCE) {
7254 if (wc->refs[level - 1] > 1) {
7255 if (level == 1 &&
7256 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
7257 goto skip;
7258
7259 if (!wc->update_ref ||
7260 generation <= root->root_key.offset)
7261 goto skip;
7262
7263 btrfs_node_key_to_cpu(path->nodes[level], &key,
7264 path->slots[level]);
7265 ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
7266 if (ret < 0)
7267 goto skip;
7268
7269 wc->stage = UPDATE_BACKREF;
7270 wc->shared_level = level - 1;
7271 }
7272 } else {
7273 if (level == 1 &&
7274 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
7275 goto skip;
7276 }
7277
7278 if (!btrfs_buffer_uptodate(next, generation, 0)) {
7279 btrfs_tree_unlock(next);
7280 free_extent_buffer(next);
7281 next = NULL;
7282 *lookup_info = 1;
7283 }
7284
7285 if (!next) {
7286 if (reada && level == 1)
7287 reada_walk_down(trans, root, wc, path);
7288 next = read_tree_block(root, bytenr, blocksize, generation);
7289 if (!next || !extent_buffer_uptodate(next)) {
7290 free_extent_buffer(next);
7291 return -EIO;
7292 }
7293 btrfs_tree_lock(next);
7294 btrfs_set_lock_blocking(next);
7295 }
7296
7297 level--;
7298 BUG_ON(level != btrfs_header_level(next));
7299 path->nodes[level] = next;
7300 path->slots[level] = 0;
7301 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7302 wc->level = level;
7303 if (wc->level == 1)
7304 wc->reada_slot = 0;
7305 return 0;
7306skip:
7307 wc->refs[level - 1] = 0;
7308 wc->flags[level - 1] = 0;
7309 if (wc->stage == DROP_REFERENCE) {
7310 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
7311 parent = path->nodes[level]->start;
7312 } else {
7313 BUG_ON(root->root_key.objectid !=
7314 btrfs_header_owner(path->nodes[level]));
7315 parent = 0;
7316 }
7317
7318 ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
7319 root->root_key.objectid, level - 1, 0, 0);
7320 BUG_ON(ret);
7321 }
7322 btrfs_tree_unlock(next);
7323 free_extent_buffer(next);
7324 *lookup_info = 1;
7325 return 1;
7326}
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
7341 struct btrfs_root *root,
7342 struct btrfs_path *path,
7343 struct walk_control *wc)
7344{
7345 int ret;
7346 int level = wc->level;
7347 struct extent_buffer *eb = path->nodes[level];
7348 u64 parent = 0;
7349
7350 if (wc->stage == UPDATE_BACKREF) {
7351 BUG_ON(wc->shared_level < level);
7352 if (level < wc->shared_level)
7353 goto out;
7354
7355 ret = find_next_key(path, level + 1, &wc->update_progress);
7356 if (ret > 0)
7357 wc->update_ref = 0;
7358
7359 wc->stage = DROP_REFERENCE;
7360 wc->shared_level = -1;
7361 path->slots[level] = 0;
7362
7363
7364
7365
7366
7367
7368 if (!path->locks[level]) {
7369 BUG_ON(level == 0);
7370 btrfs_tree_lock(eb);
7371 btrfs_set_lock_blocking(eb);
7372 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7373
7374 ret = btrfs_lookup_extent_info(trans, root,
7375 eb->start, level, 1,
7376 &wc->refs[level],
7377 &wc->flags[level]);
7378 if (ret < 0) {
7379 btrfs_tree_unlock_rw(eb, path->locks[level]);
7380 path->locks[level] = 0;
7381 return ret;
7382 }
7383 BUG_ON(wc->refs[level] == 0);
7384 if (wc->refs[level] == 1) {
7385 btrfs_tree_unlock_rw(eb, path->locks[level]);
7386 path->locks[level] = 0;
7387 return 1;
7388 }
7389 }
7390 }
7391
7392
7393 BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
7394
7395 if (wc->refs[level] == 1) {
7396 if (level == 0) {
7397 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
7398 ret = btrfs_dec_ref(trans, root, eb, 1,
7399 wc->for_reloc);
7400 else
7401 ret = btrfs_dec_ref(trans, root, eb, 0,
7402 wc->for_reloc);
7403 BUG_ON(ret);
7404 }
7405
7406 if (!path->locks[level] &&
7407 btrfs_header_generation(eb) == trans->transid) {
7408 btrfs_tree_lock(eb);
7409 btrfs_set_lock_blocking(eb);
7410 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7411 }
7412 clean_tree_block(trans, root, eb);
7413 }
7414
7415 if (eb == root->node) {
7416 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
7417 parent = eb->start;
7418 else
7419 BUG_ON(root->root_key.objectid !=
7420 btrfs_header_owner(eb));
7421 } else {
7422 if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
7423 parent = path->nodes[level + 1]->start;
7424 else
7425 BUG_ON(root->root_key.objectid !=
7426 btrfs_header_owner(path->nodes[level + 1]));
7427 }
7428
7429 btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
7430out:
7431 wc->refs[level] = 0;
7432 wc->flags[level] = 0;
7433 return 0;
7434}
7435
7436static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
7437 struct btrfs_root *root,
7438 struct btrfs_path *path,
7439 struct walk_control *wc)
7440{
7441 int level = wc->level;
7442 int lookup_info = 1;
7443 int ret;
7444
7445 while (level >= 0) {
7446 ret = walk_down_proc(trans, root, path, wc, lookup_info);
7447 if (ret > 0)
7448 break;
7449
7450 if (level == 0)
7451 break;
7452
7453 if (path->slots[level] >=
7454 btrfs_header_nritems(path->nodes[level]))
7455 break;
7456
7457 ret = do_walk_down(trans, root, path, wc, &lookup_info);
7458 if (ret > 0) {
7459 path->slots[level]++;
7460 continue;
7461 } else if (ret < 0)
7462 return ret;
7463 level = wc->level;
7464 }
7465 return 0;
7466}
7467
7468static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
7469 struct btrfs_root *root,
7470 struct btrfs_path *path,
7471 struct walk_control *wc, int max_level)
7472{
7473 int level = wc->level;
7474 int ret;
7475
7476 path->slots[level] = btrfs_header_nritems(path->nodes[level]);
7477 while (level < max_level && path->nodes[level]) {
7478 wc->level = level;
7479 if (path->slots[level] + 1 <
7480 btrfs_header_nritems(path->nodes[level])) {
7481 path->slots[level]++;
7482 return 0;
7483 } else {
7484 ret = walk_up_proc(trans, root, path, wc);
7485 if (ret > 0)
7486 return 0;
7487
7488 if (path->locks[level]) {
7489 btrfs_tree_unlock_rw(path->nodes[level],
7490 path->locks[level]);
7491 path->locks[level] = 0;
7492 }
7493 free_extent_buffer(path->nodes[level]);
7494 path->nodes[level] = NULL;
7495 level++;
7496 }
7497 }
7498 return 1;
7499}
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514int btrfs_drop_snapshot(struct btrfs_root *root,
7515 struct btrfs_block_rsv *block_rsv, int update_ref,
7516 int for_reloc)
7517{
7518 struct btrfs_path *path;
7519 struct btrfs_trans_handle *trans;
7520 struct btrfs_root *tree_root = root->fs_info->tree_root;
7521 struct btrfs_root_item *root_item = &root->root_item;
7522 struct walk_control *wc;
7523 struct btrfs_key key;
7524 int err = 0;
7525 int ret;
7526 int level;
7527 bool root_dropped = false;
7528
7529 path = btrfs_alloc_path();
7530 if (!path) {
7531 err = -ENOMEM;
7532 goto out;
7533 }
7534
7535 wc = kzalloc(sizeof(*wc), GFP_NOFS);
7536 if (!wc) {
7537 btrfs_free_path(path);
7538 err = -ENOMEM;
7539 goto out;
7540 }
7541
7542 trans = btrfs_start_transaction(tree_root, 0);
7543 if (IS_ERR(trans)) {
7544 err = PTR_ERR(trans);
7545 goto out_free;
7546 }
7547
7548 if (block_rsv)
7549 trans->block_rsv = block_rsv;
7550
7551 if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
7552 level = btrfs_header_level(root->node);
7553 path->nodes[level] = btrfs_lock_root_node(root);
7554 btrfs_set_lock_blocking(path->nodes[level]);
7555 path->slots[level] = 0;
7556 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7557 memset(&wc->update_progress, 0,
7558 sizeof(wc->update_progress));
7559 } else {
7560 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
7561 memcpy(&wc->update_progress, &key,
7562 sizeof(wc->update_progress));
7563
7564 level = root_item->drop_level;
7565 BUG_ON(level == 0);
7566 path->lowest_level = level;
7567 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
7568 path->lowest_level = 0;
7569 if (ret < 0) {
7570 err = ret;
7571 goto out_end_trans;
7572 }
7573 WARN_ON(ret > 0);
7574
7575
7576
7577
7578
7579 btrfs_unlock_up_safe(path, 0);
7580
7581 level = btrfs_header_level(root->node);
7582 while (1) {
7583 btrfs_tree_lock(path->nodes[level]);
7584 btrfs_set_lock_blocking(path->nodes[level]);
7585 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7586
7587 ret = btrfs_lookup_extent_info(trans, root,
7588 path->nodes[level]->start,
7589 level, 1, &wc->refs[level],
7590 &wc->flags[level]);
7591 if (ret < 0) {
7592 err = ret;
7593 goto out_end_trans;
7594 }
7595 BUG_ON(wc->refs[level] == 0);
7596
7597 if (level == root_item->drop_level)
7598 break;
7599
7600 btrfs_tree_unlock(path->nodes[level]);
7601 path->locks[level] = 0;
7602 WARN_ON(wc->refs[level] != 1);
7603 level--;
7604 }
7605 }
7606
7607 wc->level = level;
7608 wc->shared_level = -1;
7609 wc->stage = DROP_REFERENCE;
7610 wc->update_ref = update_ref;
7611 wc->keep_locks = 0;
7612 wc->for_reloc = for_reloc;
7613 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
7614
7615 while (1) {
7616
7617 ret = walk_down_tree(trans, root, path, wc);
7618 if (ret < 0) {
7619 err = ret;
7620 break;
7621 }
7622
7623 ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
7624 if (ret < 0) {
7625 err = ret;
7626 break;
7627 }
7628
7629 if (ret > 0) {
7630 BUG_ON(wc->stage != DROP_REFERENCE);
7631 break;
7632 }
7633
7634 if (wc->stage == DROP_REFERENCE) {
7635 level = wc->level;
7636 btrfs_node_key(path->nodes[level],
7637 &root_item->drop_progress,
7638 path->slots[level]);
7639 root_item->drop_level = level;
7640 }
7641
7642 BUG_ON(wc->level == 0);
7643 if (btrfs_should_end_transaction(trans, tree_root) ||
7644 (!for_reloc && btrfs_need_cleaner_sleep(root))) {
7645 ret = btrfs_update_root(trans, tree_root,
7646 &root->root_key,
7647 root_item);
7648 if (ret) {
7649 btrfs_abort_transaction(trans, tree_root, ret);
7650 err = ret;
7651 goto out_end_trans;
7652 }
7653
7654 btrfs_end_transaction_throttle(trans, tree_root);
7655 if (!for_reloc && btrfs_need_cleaner_sleep(root)) {
7656 pr_debug("btrfs: drop snapshot early exit\n");
7657 err = -EAGAIN;
7658 goto out_free;
7659 }
7660
7661 trans = btrfs_start_transaction(tree_root, 0);
7662 if (IS_ERR(trans)) {
7663 err = PTR_ERR(trans);
7664 goto out_free;
7665 }
7666 if (block_rsv)
7667 trans->block_rsv = block_rsv;
7668 }
7669 }
7670 btrfs_release_path(path);
7671 if (err)
7672 goto out_end_trans;
7673
7674 ret = btrfs_del_root(trans, tree_root, &root->root_key);
7675 if (ret) {
7676 btrfs_abort_transaction(trans, tree_root, ret);
7677 goto out_end_trans;
7678 }
7679
7680 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
7681 ret = btrfs_find_root(tree_root, &root->root_key, path,
7682 NULL, NULL);
7683 if (ret < 0) {
7684 btrfs_abort_transaction(trans, tree_root, ret);
7685 err = ret;
7686 goto out_end_trans;
7687 } else if (ret > 0) {
7688
7689
7690
7691
7692
7693 btrfs_del_orphan_item(trans, tree_root,
7694 root->root_key.objectid);
7695 }
7696 }
7697
7698 if (root->in_radix) {
7699 btrfs_drop_and_free_fs_root(tree_root->fs_info, root);
7700 } else {
7701 free_extent_buffer(root->node);
7702 free_extent_buffer(root->commit_root);
7703 btrfs_put_fs_root(root);
7704 }
7705 root_dropped = true;
7706out_end_trans:
7707 btrfs_end_transaction_throttle(trans, tree_root);
7708out_free:
7709 kfree(wc);
7710 btrfs_free_path(path);
7711out:
7712
7713
7714
7715
7716
7717
7718
7719 if (!for_reloc && root_dropped == false)
7720 btrfs_add_dead_root(root);
7721 if (err)
7722 btrfs_std_error(root->fs_info, err);
7723 return err;
7724}
7725
7726
7727
7728
7729
7730
7731
7732int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
7733 struct btrfs_root *root,
7734 struct extent_buffer *node,
7735 struct extent_buffer *parent)
7736{
7737 struct btrfs_path *path;
7738 struct walk_control *wc;
7739 int level;
7740 int parent_level;
7741 int ret = 0;
7742 int wret;
7743
7744 BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
7745
7746 path = btrfs_alloc_path();
7747 if (!path)
7748 return -ENOMEM;
7749
7750 wc = kzalloc(sizeof(*wc), GFP_NOFS);
7751 if (!wc) {
7752 btrfs_free_path(path);
7753 return -ENOMEM;
7754 }
7755
7756 btrfs_assert_tree_locked(parent);
7757 parent_level = btrfs_header_level(parent);
7758 extent_buffer_get(parent);
7759 path->nodes[parent_level] = parent;
7760 path->slots[parent_level] = btrfs_header_nritems(parent);
7761
7762 btrfs_assert_tree_locked(node);
7763 level = btrfs_header_level(node);
7764 path->nodes[level] = node;
7765 path->slots[level] = 0;
7766 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7767
7768 wc->refs[parent_level] = 1;
7769 wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
7770 wc->level = level;
7771 wc->shared_level = -1;
7772 wc->stage = DROP_REFERENCE;
7773 wc->update_ref = 0;
7774 wc->keep_locks = 1;
7775 wc->for_reloc = 1;
7776 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
7777
7778 while (1) {
7779 wret = walk_down_tree(trans, root, path, wc);
7780 if (wret < 0) {
7781 ret = wret;
7782 break;
7783 }
7784
7785 wret = walk_up_tree(trans, root, path, wc, parent_level);
7786 if (wret < 0)
7787 ret = wret;
7788 if (wret != 0)
7789 break;
7790 }
7791
7792 kfree(wc);
7793 btrfs_free_path(path);
7794 return ret;
7795}
7796
7797static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
7798{
7799 u64 num_devices;
7800 u64 stripped;
7801
7802
7803
7804
7805
7806 stripped = get_restripe_target(root->fs_info, flags);
7807 if (stripped)
7808 return extended_to_chunk(stripped);
7809
7810
7811
7812
7813
7814
7815 num_devices = root->fs_info->fs_devices->rw_devices +
7816 root->fs_info->fs_devices->missing_devices;
7817
7818 stripped = BTRFS_BLOCK_GROUP_RAID0 |
7819 BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 |
7820 BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;
7821
7822 if (num_devices == 1) {
7823 stripped |= BTRFS_BLOCK_GROUP_DUP;
7824 stripped = flags & ~stripped;
7825
7826
7827 if (flags & BTRFS_BLOCK_GROUP_RAID0)
7828 return stripped;
7829
7830
7831 if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
7832 BTRFS_BLOCK_GROUP_RAID10))
7833 return stripped | BTRFS_BLOCK_GROUP_DUP;
7834 } else {
7835
7836 if (flags & stripped)
7837 return flags;
7838
7839 stripped |= BTRFS_BLOCK_GROUP_DUP;
7840 stripped = flags & ~stripped;
7841
7842
7843 if (flags & BTRFS_BLOCK_GROUP_DUP)
7844 return stripped | BTRFS_BLOCK_GROUP_RAID1;
7845
7846
7847 }
7848
7849 return flags;
7850}
7851
7852static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
7853{
7854 struct btrfs_space_info *sinfo = cache->space_info;
7855 u64 num_bytes;
7856 u64 min_allocable_bytes;
7857 int ret = -ENOSPC;
7858
7859
7860
7861
7862
7863
7864
7865 if ((sinfo->flags &
7866 (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
7867 !force)
7868 min_allocable_bytes = 1 * 1024 * 1024;
7869 else
7870 min_allocable_bytes = 0;
7871
7872 spin_lock(&sinfo->lock);
7873 spin_lock(&cache->lock);
7874
7875 if (cache->ro) {
7876 ret = 0;
7877 goto out;
7878 }
7879
7880 num_bytes = cache->key.offset - cache->reserved - cache->pinned -
7881 cache->bytes_super - btrfs_block_group_used(&cache->item);
7882
7883 if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
7884 sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
7885 min_allocable_bytes <= sinfo->total_bytes) {
7886 sinfo->bytes_readonly += num_bytes;
7887 cache->ro = 1;
7888 ret = 0;
7889 }
7890out:
7891 spin_unlock(&cache->lock);
7892 spin_unlock(&sinfo->lock);
7893 return ret;
7894}
7895
7896int btrfs_set_block_group_ro(struct btrfs_root *root,
7897 struct btrfs_block_group_cache *cache)
7898
7899{
7900 struct btrfs_trans_handle *trans;
7901 u64 alloc_flags;
7902 int ret;
7903
7904 BUG_ON(cache->ro);
7905
7906 trans = btrfs_join_transaction(root);
7907 if (IS_ERR(trans))
7908 return PTR_ERR(trans);
7909
7910 alloc_flags = update_block_group_flags(root, cache->flags);
7911 if (alloc_flags != cache->flags) {
7912 ret = do_chunk_alloc(trans, root, alloc_flags,
7913 CHUNK_ALLOC_FORCE);
7914 if (ret < 0)
7915 goto out;
7916 }
7917
7918 ret = set_block_group_ro(cache, 0);
7919 if (!ret)
7920 goto out;
7921 alloc_flags = get_alloc_profile(root, cache->space_info->flags);
7922 ret = do_chunk_alloc(trans, root, alloc_flags,
7923 CHUNK_ALLOC_FORCE);
7924 if (ret < 0)
7925 goto out;
7926 ret = set_block_group_ro(cache, 0);
7927out:
7928 btrfs_end_transaction(trans, root);
7929 return ret;
7930}
7931
7932int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
7933 struct btrfs_root *root, u64 type)
7934{
7935 u64 alloc_flags = get_alloc_profile(root, type);
7936 return do_chunk_alloc(trans, root, alloc_flags,
7937 CHUNK_ALLOC_FORCE);
7938}
7939
7940
7941
7942
7943
7944static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
7945{
7946 struct btrfs_block_group_cache *block_group;
7947 u64 free_bytes = 0;
7948 int factor;
7949
7950 list_for_each_entry(block_group, groups_list, list) {
7951 spin_lock(&block_group->lock);
7952
7953 if (!block_group->ro) {
7954 spin_unlock(&block_group->lock);
7955 continue;
7956 }
7957
7958 if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
7959 BTRFS_BLOCK_GROUP_RAID10 |
7960 BTRFS_BLOCK_GROUP_DUP))
7961 factor = 2;
7962 else
7963 factor = 1;
7964
7965 free_bytes += (block_group->key.offset -
7966 btrfs_block_group_used(&block_group->item)) *
7967 factor;
7968
7969 spin_unlock(&block_group->lock);
7970 }
7971
7972 return free_bytes;
7973}
7974
7975
7976
7977
7978
7979u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
7980{
7981 int i;
7982 u64 free_bytes = 0;
7983
7984 spin_lock(&sinfo->lock);
7985
7986 for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
7987 if (!list_empty(&sinfo->block_groups[i]))
7988 free_bytes += __btrfs_get_ro_block_group_free_space(
7989 &sinfo->block_groups[i]);
7990
7991 spin_unlock(&sinfo->lock);
7992
7993 return free_bytes;
7994}
7995
7996void btrfs_set_block_group_rw(struct btrfs_root *root,
7997 struct btrfs_block_group_cache *cache)
7998{
7999 struct btrfs_space_info *sinfo = cache->space_info;
8000 u64 num_bytes;
8001
8002 BUG_ON(!cache->ro);
8003
8004 spin_lock(&sinfo->lock);
8005 spin_lock(&cache->lock);
8006 num_bytes = cache->key.offset - cache->reserved - cache->pinned -
8007 cache->bytes_super - btrfs_block_group_used(&cache->item);
8008 sinfo->bytes_readonly -= num_bytes;
8009 cache->ro = 0;
8010 spin_unlock(&cache->lock);
8011 spin_unlock(&sinfo->lock);
8012}
8013
8014
8015
8016
8017
8018
8019
8020int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
8021{
8022 struct btrfs_block_group_cache *block_group;
8023 struct btrfs_space_info *space_info;
8024 struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
8025 struct btrfs_device *device;
8026 struct btrfs_trans_handle *trans;
8027 u64 min_free;
8028 u64 dev_min = 1;
8029 u64 dev_nr = 0;
8030 u64 target;
8031 int index;
8032 int full = 0;
8033 int ret = 0;
8034
8035 block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
8036
8037
8038 if (!block_group)
8039 return -1;
8040
8041 min_free = btrfs_block_group_used(&block_group->item);
8042
8043
8044 if (!min_free)
8045 goto out;
8046
8047 space_info = block_group->space_info;
8048 spin_lock(&space_info->lock);
8049
8050 full = space_info->full;
8051
8052
8053
8054
8055
8056
8057
8058
8059 if ((space_info->total_bytes != block_group->key.offset) &&
8060 (space_info->bytes_used + space_info->bytes_reserved +
8061 space_info->bytes_pinned + space_info->bytes_readonly +
8062 min_free < space_info->total_bytes)) {
8063 spin_unlock(&space_info->lock);
8064 goto out;
8065 }
8066 spin_unlock(&space_info->lock);
8067
8068
8069
8070
8071
8072
8073
8074
8075 ret = -1;
8076
8077
8078
8079
8080
8081
8082
8083
8084
8085 target = get_restripe_target(root->fs_info, block_group->flags);
8086 if (target) {
8087 index = __get_raid_index(extended_to_chunk(target));
8088 } else {
8089
8090
8091
8092
8093 if (full)
8094 goto out;
8095
8096 index = get_block_group_index(block_group);
8097 }
8098
8099 if (index == BTRFS_RAID_RAID10) {
8100 dev_min = 4;
8101
8102 min_free >>= 1;
8103 } else if (index == BTRFS_RAID_RAID1) {
8104 dev_min = 2;
8105 } else if (index == BTRFS_RAID_DUP) {
8106
8107 min_free <<= 1;
8108 } else if (index == BTRFS_RAID_RAID0) {
8109 dev_min = fs_devices->rw_devices;
8110 do_div(min_free, dev_min);
8111 }
8112
8113
8114 trans = btrfs_join_transaction(root);
8115 if (IS_ERR(trans)) {
8116 ret = PTR_ERR(trans);
8117 goto out;
8118 }
8119
8120 mutex_lock(&root->fs_info->chunk_mutex);
8121 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
8122 u64 dev_offset;
8123
8124
8125
8126
8127
8128 if (device->total_bytes > device->bytes_used + min_free &&
8129 !device->is_tgtdev_for_dev_replace) {
8130 ret = find_free_dev_extent(trans, device, min_free,
8131 &dev_offset, NULL);
8132 if (!ret)
8133 dev_nr++;
8134
8135 if (dev_nr >= dev_min)
8136 break;
8137
8138 ret = -1;
8139 }
8140 }
8141 mutex_unlock(&root->fs_info->chunk_mutex);
8142 btrfs_end_transaction(trans, root);
8143out:
8144 btrfs_put_block_group(block_group);
8145 return ret;
8146}
8147
8148static int find_first_block_group(struct btrfs_root *root,
8149 struct btrfs_path *path, struct btrfs_key *key)
8150{
8151 int ret = 0;
8152 struct btrfs_key found_key;
8153 struct extent_buffer *leaf;
8154 int slot;
8155
8156 ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
8157 if (ret < 0)
8158 goto out;
8159
8160 while (1) {
8161 slot = path->slots[0];
8162 leaf = path->nodes[0];
8163 if (slot >= btrfs_header_nritems(leaf)) {
8164 ret = btrfs_next_leaf(root, path);
8165 if (ret == 0)
8166 continue;
8167 if (ret < 0)
8168 goto out;
8169 break;
8170 }
8171 btrfs_item_key_to_cpu(leaf, &found_key, slot);
8172
8173 if (found_key.objectid >= key->objectid &&
8174 found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
8175 ret = 0;
8176 goto out;
8177 }
8178 path->slots[0]++;
8179 }
8180out:
8181 return ret;
8182}
8183
8184void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
8185{
8186 struct btrfs_block_group_cache *block_group;
8187 u64 last = 0;
8188
8189 while (1) {
8190 struct inode *inode;
8191
8192 block_group = btrfs_lookup_first_block_group(info, last);
8193 while (block_group) {
8194 spin_lock(&block_group->lock);
8195 if (block_group->iref)
8196 break;
8197 spin_unlock(&block_group->lock);
8198 block_group = next_block_group(info->tree_root,
8199 block_group);
8200 }
8201 if (!block_group) {
8202 if (last == 0)
8203 break;
8204 last = 0;
8205 continue;
8206 }
8207
8208 inode = block_group->inode;
8209 block_group->iref = 0;
8210 block_group->inode = NULL;
8211 spin_unlock(&block_group->lock);
8212 iput(inode);
8213 last = block_group->key.objectid + block_group->key.offset;
8214 btrfs_put_block_group(block_group);
8215 }
8216}
8217
8218int btrfs_free_block_groups(struct btrfs_fs_info *info)
8219{
8220 struct btrfs_block_group_cache *block_group;
8221 struct btrfs_space_info *space_info;
8222 struct btrfs_caching_control *caching_ctl;
8223 struct rb_node *n;
8224
8225 down_write(&info->extent_commit_sem);
8226 while (!list_empty(&info->caching_block_groups)) {
8227 caching_ctl = list_entry(info->caching_block_groups.next,
8228 struct btrfs_caching_control, list);
8229 list_del(&caching_ctl->list);
8230 put_caching_control(caching_ctl);
8231 }
8232 up_write(&info->extent_commit_sem);
8233
8234 spin_lock(&info->block_group_cache_lock);
8235 while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
8236 block_group = rb_entry(n, struct btrfs_block_group_cache,
8237 cache_node);
8238 rb_erase(&block_group->cache_node,
8239 &info->block_group_cache_tree);
8240 spin_unlock(&info->block_group_cache_lock);
8241
8242 down_write(&block_group->space_info->groups_sem);
8243 list_del(&block_group->list);
8244 up_write(&block_group->space_info->groups_sem);
8245
8246 if (block_group->cached == BTRFS_CACHE_STARTED)
8247 wait_block_group_cache_done(block_group);
8248
8249
8250
8251
8252
8253 if (block_group->cached == BTRFS_CACHE_NO ||
8254 block_group->cached == BTRFS_CACHE_ERROR)
8255 free_excluded_extents(info->extent_root, block_group);
8256
8257 btrfs_remove_free_space_cache(block_group);
8258 btrfs_put_block_group(block_group);
8259
8260 spin_lock(&info->block_group_cache_lock);
8261 }
8262 spin_unlock(&info->block_group_cache_lock);
8263
8264
8265
8266
8267
8268
8269
8270 synchronize_rcu();
8271
8272 release_global_block_rsv(info);
8273
8274 while(!list_empty(&info->space_info)) {
8275 space_info = list_entry(info->space_info.next,
8276 struct btrfs_space_info,
8277 list);
8278 if (btrfs_test_opt(info->tree_root, ENOSPC_DEBUG)) {
8279 if (space_info->bytes_pinned > 0 ||
8280 space_info->bytes_reserved > 0 ||
8281 space_info->bytes_may_use > 0) {
8282 WARN_ON(1);
8283 dump_space_info(space_info, 0, 0);
8284 }
8285 }
8286 percpu_counter_destroy(&space_info->total_bytes_pinned);
8287 list_del(&space_info->list);
8288 kfree(space_info);
8289 }
8290 return 0;
8291}
8292
8293static void __link_block_group(struct btrfs_space_info *space_info,
8294 struct btrfs_block_group_cache *cache)
8295{
8296 int index = get_block_group_index(cache);
8297
8298 down_write(&space_info->groups_sem);
8299 list_add_tail(&cache->list, &space_info->block_groups[index]);
8300 up_write(&space_info->groups_sem);
8301}
8302
8303int btrfs_read_block_groups(struct btrfs_root *root)
8304{
8305 struct btrfs_path *path;
8306 int ret;
8307 struct btrfs_block_group_cache *cache;
8308 struct btrfs_fs_info *info = root->fs_info;
8309 struct btrfs_space_info *space_info;
8310 struct btrfs_key key;
8311 struct btrfs_key found_key;
8312 struct extent_buffer *leaf;
8313 int need_clear = 0;
8314 u64 cache_gen;
8315
8316 root = info->extent_root;
8317 key.objectid = 0;
8318 key.offset = 0;
8319 btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
8320 path = btrfs_alloc_path();
8321 if (!path)
8322 return -ENOMEM;
8323 path->reada = 1;
8324
8325 cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
8326 if (btrfs_test_opt(root, SPACE_CACHE) &&
8327 btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
8328 need_clear = 1;
8329 if (btrfs_test_opt(root, CLEAR_CACHE))
8330 need_clear = 1;
8331
8332 while (1) {
8333 ret = find_first_block_group(root, path, &key);
8334 if (ret > 0)
8335 break;
8336 if (ret != 0)
8337 goto error;
8338 leaf = path->nodes[0];
8339 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
8340 cache = kzalloc(sizeof(*cache), GFP_NOFS);
8341 if (!cache) {
8342 ret = -ENOMEM;
8343 goto error;
8344 }
8345 cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
8346 GFP_NOFS);
8347 if (!cache->free_space_ctl) {
8348 kfree(cache);
8349 ret = -ENOMEM;
8350 goto error;
8351 }
8352
8353 atomic_set(&cache->count, 1);
8354 spin_lock_init(&cache->lock);
8355 cache->fs_info = info;
8356 INIT_LIST_HEAD(&cache->list);
8357 INIT_LIST_HEAD(&cache->cluster_list);
8358
8359 if (need_clear) {
8360
8361
8362
8363
8364
8365
8366
8367
8368
8369
8370 cache->disk_cache_state = BTRFS_DC_CLEAR;
8371 if (btrfs_test_opt(root, SPACE_CACHE))
8372 cache->dirty = 1;
8373 }
8374
8375 read_extent_buffer(leaf, &cache->item,
8376 btrfs_item_ptr_offset(leaf, path->slots[0]),
8377 sizeof(cache->item));
8378 memcpy(&cache->key, &found_key, sizeof(found_key));
8379
8380 key.objectid = found_key.objectid + found_key.offset;
8381 btrfs_release_path(path);
8382 cache->flags = btrfs_block_group_flags(&cache->item);
8383 cache->sectorsize = root->sectorsize;
8384 cache->full_stripe_len = btrfs_full_stripe_len(root,
8385 &root->fs_info->mapping_tree,
8386 found_key.objectid);
8387 btrfs_init_free_space_ctl(cache);
8388
8389
8390
8391
8392
8393
8394 ret = exclude_super_stripes(root, cache);
8395 if (ret) {
8396
8397
8398
8399
8400 free_excluded_extents(root, cache);
8401 kfree(cache->free_space_ctl);
8402 kfree(cache);
8403 goto error;
8404 }
8405
8406
8407
8408
8409
8410
8411
8412
8413 if (found_key.offset == btrfs_block_group_used(&cache->item)) {
8414 cache->last_byte_to_unpin = (u64)-1;
8415 cache->cached = BTRFS_CACHE_FINISHED;
8416 free_excluded_extents(root, cache);
8417 } else if (btrfs_block_group_used(&cache->item) == 0) {
8418 cache->last_byte_to_unpin = (u64)-1;
8419 cache->cached = BTRFS_CACHE_FINISHED;
8420 add_new_free_space(cache, root->fs_info,
8421 found_key.objectid,
8422 found_key.objectid +
8423 found_key.offset);
8424 free_excluded_extents(root, cache);
8425 }
8426
8427 ret = btrfs_add_block_group_cache(root->fs_info, cache);
8428 if (ret) {
8429 btrfs_remove_free_space_cache(cache);
8430 btrfs_put_block_group(cache);
8431 goto error;
8432 }
8433
8434 ret = update_space_info(info, cache->flags, found_key.offset,
8435 btrfs_block_group_used(&cache->item),
8436 &space_info);
8437 if (ret) {
8438 btrfs_remove_free_space_cache(cache);
8439 spin_lock(&info->block_group_cache_lock);
8440 rb_erase(&cache->cache_node,
8441 &info->block_group_cache_tree);
8442 spin_unlock(&info->block_group_cache_lock);
8443 btrfs_put_block_group(cache);
8444 goto error;
8445 }
8446
8447 cache->space_info = space_info;
8448 spin_lock(&cache->space_info->lock);
8449 cache->space_info->bytes_readonly += cache->bytes_super;
8450 spin_unlock(&cache->space_info->lock);
8451
8452 __link_block_group(space_info, cache);
8453
8454 set_avail_alloc_bits(root->fs_info, cache->flags);
8455 if (btrfs_chunk_readonly(root, cache->key.objectid))
8456 set_block_group_ro(cache, 1);
8457 }
8458
8459 list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
8460 if (!(get_alloc_profile(root, space_info->flags) &
8461 (BTRFS_BLOCK_GROUP_RAID10 |
8462 BTRFS_BLOCK_GROUP_RAID1 |
8463 BTRFS_BLOCK_GROUP_RAID5 |
8464 BTRFS_BLOCK_GROUP_RAID6 |
8465 BTRFS_BLOCK_GROUP_DUP)))
8466 continue;
8467
8468
8469
8470
8471 list_for_each_entry(cache,
8472 &space_info->block_groups[BTRFS_RAID_RAID0],
8473 list)
8474 set_block_group_ro(cache, 1);
8475 list_for_each_entry(cache,
8476 &space_info->block_groups[BTRFS_RAID_SINGLE],
8477 list)
8478 set_block_group_ro(cache, 1);
8479 }
8480
8481 init_global_block_rsv(info);
8482 ret = 0;
8483error:
8484 btrfs_free_path(path);
8485 return ret;
8486}
8487
8488void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans,
8489 struct btrfs_root *root)
8490{
8491 struct btrfs_block_group_cache *block_group, *tmp;
8492 struct btrfs_root *extent_root = root->fs_info->extent_root;
8493 struct btrfs_block_group_item item;
8494 struct btrfs_key key;
8495 int ret = 0;
8496
8497 list_for_each_entry_safe(block_group, tmp, &trans->new_bgs,
8498 new_bg_list) {
8499 list_del_init(&block_group->new_bg_list);
8500
8501 if (ret)
8502 continue;
8503
8504 spin_lock(&block_group->lock);
8505 memcpy(&item, &block_group->item, sizeof(item));
8506 memcpy(&key, &block_group->key, sizeof(key));
8507 spin_unlock(&block_group->lock);
8508
8509 ret = btrfs_insert_item(trans, extent_root, &key, &item,
8510 sizeof(item));
8511 if (ret)
8512 btrfs_abort_transaction(trans, extent_root, ret);
8513 ret = btrfs_finish_chunk_alloc(trans, extent_root,
8514 key.objectid, key.offset);
8515 if (ret)
8516 btrfs_abort_transaction(trans, extent_root, ret);
8517 }
8518}
8519
8520int btrfs_make_block_group(struct btrfs_trans_handle *trans,
8521 struct btrfs_root *root, u64 bytes_used,
8522 u64 type, u64 chunk_objectid, u64 chunk_offset,
8523 u64 size)
8524{
8525 int ret;
8526 struct btrfs_root *extent_root;
8527 struct btrfs_block_group_cache *cache;
8528
8529 extent_root = root->fs_info->extent_root;
8530
8531 root->fs_info->last_trans_log_full_commit = trans->transid;
8532
8533 cache = kzalloc(sizeof(*cache), GFP_NOFS);
8534 if (!cache)
8535 return -ENOMEM;
8536 cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
8537 GFP_NOFS);
8538 if (!cache->free_space_ctl) {
8539 kfree(cache);
8540 return -ENOMEM;
8541 }
8542
8543 cache->key.objectid = chunk_offset;
8544 cache->key.offset = size;
8545 cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
8546 cache->sectorsize = root->sectorsize;
8547 cache->fs_info = root->fs_info;
8548 cache->full_stripe_len = btrfs_full_stripe_len(root,
8549 &root->fs_info->mapping_tree,
8550 chunk_offset);
8551
8552 atomic_set(&cache->count, 1);
8553 spin_lock_init(&cache->lock);
8554 INIT_LIST_HEAD(&cache->list);
8555 INIT_LIST_HEAD(&cache->cluster_list);
8556 INIT_LIST_HEAD(&cache->new_bg_list);
8557
8558 btrfs_init_free_space_ctl(cache);
8559
8560 btrfs_set_block_group_used(&cache->item, bytes_used);
8561 btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
8562 cache->flags = type;
8563 btrfs_set_block_group_flags(&cache->item, type);
8564
8565 cache->last_byte_to_unpin = (u64)-1;
8566 cache->cached = BTRFS_CACHE_FINISHED;
8567 ret = exclude_super_stripes(root, cache);
8568 if (ret) {
8569
8570
8571
8572
8573 free_excluded_extents(root, cache);
8574 kfree(cache->free_space_ctl);
8575 kfree(cache);
8576 return ret;
8577 }
8578
8579 add_new_free_space(cache, root->fs_info, chunk_offset,
8580 chunk_offset + size);
8581
8582 free_excluded_extents(root, cache);
8583
8584 ret = btrfs_add_block_group_cache(root->fs_info, cache);
8585 if (ret) {
8586 btrfs_remove_free_space_cache(cache);
8587 btrfs_put_block_group(cache);
8588 return ret;
8589 }
8590
8591 ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
8592 &cache->space_info);
8593 if (ret) {
8594 btrfs_remove_free_space_cache(cache);
8595 spin_lock(&root->fs_info->block_group_cache_lock);
8596 rb_erase(&cache->cache_node,
8597 &root->fs_info->block_group_cache_tree);
8598 spin_unlock(&root->fs_info->block_group_cache_lock);
8599 btrfs_put_block_group(cache);
8600 return ret;
8601 }
8602 update_global_block_rsv(root->fs_info);
8603
8604 spin_lock(&cache->space_info->lock);
8605 cache->space_info->bytes_readonly += cache->bytes_super;
8606 spin_unlock(&cache->space_info->lock);
8607
8608 __link_block_group(cache->space_info, cache);
8609
8610 list_add_tail(&cache->new_bg_list, &trans->new_bgs);
8611
8612 set_avail_alloc_bits(extent_root->fs_info, type);
8613
8614 return 0;
8615}
8616
8617static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
8618{
8619 u64 extra_flags = chunk_to_extended(flags) &
8620 BTRFS_EXTENDED_PROFILE_MASK;
8621
8622 write_seqlock(&fs_info->profiles_lock);
8623 if (flags & BTRFS_BLOCK_GROUP_DATA)
8624 fs_info->avail_data_alloc_bits &= ~extra_flags;
8625 if (flags & BTRFS_BLOCK_GROUP_METADATA)
8626 fs_info->avail_metadata_alloc_bits &= ~extra_flags;
8627 if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
8628 fs_info->avail_system_alloc_bits &= ~extra_flags;
8629 write_sequnlock(&fs_info->profiles_lock);
8630}
8631
8632int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
8633 struct btrfs_root *root, u64 group_start)
8634{
8635 struct btrfs_path *path;
8636 struct btrfs_block_group_cache *block_group;
8637 struct btrfs_free_cluster *cluster;
8638 struct btrfs_root *tree_root = root->fs_info->tree_root;
8639 struct btrfs_key key;
8640 struct inode *inode;
8641 int ret;
8642 int index;
8643 int factor;
8644
8645 root = root->fs_info->extent_root;
8646
8647 block_group = btrfs_lookup_block_group(root->fs_info, group_start);
8648 BUG_ON(!block_group);
8649 BUG_ON(!block_group->ro);
8650
8651
8652
8653
8654
8655 free_excluded_extents(root, block_group);
8656
8657 memcpy(&key, &block_group->key, sizeof(key));
8658 index = get_block_group_index(block_group);
8659 if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
8660 BTRFS_BLOCK_GROUP_RAID1 |
8661 BTRFS_BLOCK_GROUP_RAID10))
8662 factor = 2;
8663 else
8664 factor = 1;
8665
8666
8667 cluster = &root->fs_info->data_alloc_cluster;
8668 spin_lock(&cluster->refill_lock);
8669 btrfs_return_cluster_to_free_space(block_group, cluster);
8670 spin_unlock(&cluster->refill_lock);
8671
8672
8673
8674
8675
8676 cluster = &root->fs_info->meta_alloc_cluster;
8677 spin_lock(&cluster->refill_lock);
8678 btrfs_return_cluster_to_free_space(block_group, cluster);
8679 spin_unlock(&cluster->refill_lock);
8680
8681 path = btrfs_alloc_path();
8682 if (!path) {
8683 ret = -ENOMEM;
8684 goto out;
8685 }
8686
8687 inode = lookup_free_space_inode(tree_root, block_group, path);
8688 if (!IS_ERR(inode)) {
8689 ret = btrfs_orphan_add(trans, inode);
8690 if (ret) {
8691 btrfs_add_delayed_iput(inode);
8692 goto out;
8693 }
8694 clear_nlink(inode);
8695
8696 spin_lock(&block_group->lock);
8697 if (block_group->iref) {
8698 block_group->iref = 0;
8699 block_group->inode = NULL;
8700 spin_unlock(&block_group->lock);
8701 iput(inode);
8702 } else {
8703 spin_unlock(&block_group->lock);
8704 }
8705
8706 btrfs_add_delayed_iput(inode);
8707 }
8708
8709 key.objectid = BTRFS_FREE_SPACE_OBJECTID;
8710 key.offset = block_group->key.objectid;
8711 key.type = 0;
8712
8713 ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
8714 if (ret < 0)
8715 goto out;
8716 if (ret > 0)
8717 btrfs_release_path(path);
8718 if (ret == 0) {
8719 ret = btrfs_del_item(trans, tree_root, path);
8720 if (ret)
8721 goto out;
8722 btrfs_release_path(path);
8723 }
8724
8725 spin_lock(&root->fs_info->block_group_cache_lock);
8726 rb_erase(&block_group->cache_node,
8727 &root->fs_info->block_group_cache_tree);
8728
8729 if (root->fs_info->first_logical_byte == block_group->key.objectid)
8730 root->fs_info->first_logical_byte = (u64)-1;
8731 spin_unlock(&root->fs_info->block_group_cache_lock);
8732
8733 down_write(&block_group->space_info->groups_sem);
8734
8735
8736
8737
8738 list_del_init(&block_group->list);
8739 if (list_empty(&block_group->space_info->block_groups[index]))
8740 clear_avail_alloc_bits(root->fs_info, block_group->flags);
8741 up_write(&block_group->space_info->groups_sem);
8742
8743 if (block_group->cached == BTRFS_CACHE_STARTED)
8744 wait_block_group_cache_done(block_group);
8745
8746 btrfs_remove_free_space_cache(block_group);
8747
8748 spin_lock(&block_group->space_info->lock);
8749 block_group->space_info->total_bytes -= block_group->key.offset;
8750 block_group->space_info->bytes_readonly -= block_group->key.offset;
8751 block_group->space_info->disk_total -= block_group->key.offset * factor;
8752 spin_unlock(&block_group->space_info->lock);
8753
8754 memcpy(&key, &block_group->key, sizeof(key));
8755
8756 btrfs_clear_space_info_full(root->fs_info);
8757
8758 btrfs_put_block_group(block_group);
8759 btrfs_put_block_group(block_group);
8760
8761 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
8762 if (ret > 0)
8763 ret = -EIO;
8764 if (ret < 0)
8765 goto out;
8766
8767 ret = btrfs_del_item(trans, root, path);
8768out:
8769 btrfs_free_path(path);
8770 return ret;
8771}
8772
8773int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
8774{
8775 struct btrfs_space_info *space_info;
8776 struct btrfs_super_block *disk_super;
8777 u64 features;
8778 u64 flags;
8779 int mixed = 0;
8780 int ret;
8781
8782 disk_super = fs_info->super_copy;
8783 if (!btrfs_super_root(disk_super))
8784 return 1;
8785
8786 features = btrfs_super_incompat_flags(disk_super);
8787 if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
8788 mixed = 1;
8789
8790 flags = BTRFS_BLOCK_GROUP_SYSTEM;
8791 ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8792 if (ret)
8793 goto out;
8794
8795 if (mixed) {
8796 flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
8797 ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8798 } else {
8799 flags = BTRFS_BLOCK_GROUP_METADATA;
8800 ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8801 if (ret)
8802 goto out;
8803
8804 flags = BTRFS_BLOCK_GROUP_DATA;
8805 ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8806 }
8807out:
8808 return ret;
8809}
8810
8811int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
8812{
8813 return unpin_extent_range(root, start, end);
8814}
8815
8816int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
8817 u64 num_bytes, u64 *actual_bytes)
8818{
8819 return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
8820}
8821
8822int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
8823{
8824 struct btrfs_fs_info *fs_info = root->fs_info;
8825 struct btrfs_block_group_cache *cache = NULL;
8826 u64 group_trimmed;
8827 u64 start;
8828 u64 end;
8829 u64 trimmed = 0;
8830 u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
8831 int ret = 0;
8832
8833
8834
8835
8836 if (range->len == total_bytes)
8837 cache = btrfs_lookup_first_block_group(fs_info, range->start);
8838 else
8839 cache = btrfs_lookup_block_group(fs_info, range->start);
8840
8841 while (cache) {
8842 if (cache->key.objectid >= (range->start + range->len)) {
8843 btrfs_put_block_group(cache);
8844 break;
8845 }
8846
8847 start = max(range->start, cache->key.objectid);
8848 end = min(range->start + range->len,
8849 cache->key.objectid + cache->key.offset);
8850
8851 if (end - start >= range->minlen) {
8852 if (!block_group_cache_done(cache)) {
8853 ret = cache_block_group(cache, 0);
8854 if (ret) {
8855 btrfs_put_block_group(cache);
8856 break;
8857 }
8858 ret = wait_block_group_cache_done(cache);
8859 if (ret) {
8860 btrfs_put_block_group(cache);
8861 break;
8862 }
8863 }
8864 ret = btrfs_trim_block_group(cache,
8865 &group_trimmed,
8866 start,
8867 end,
8868 range->minlen);
8869
8870 trimmed += group_trimmed;
8871 if (ret) {
8872 btrfs_put_block_group(cache);
8873 break;
8874 }
8875 }
8876
8877 cache = next_block_group(fs_info->tree_root, cache);
8878 }
8879
8880 range->len = trimmed;
8881 return ret;
8882}
8883