1
2
3
4
5
6#include <linux/fs.h>
7#include <linux/pagemap.h>
8#include <linux/time.h>
9#include <linux/init.h>
10#include <linux/string.h>
11#include <linux/backing-dev.h>
12#include <linux/falloc.h>
13#include <linux/writeback.h>
14#include <linux/compat.h>
15#include <linux/slab.h>
16#include <linux/btrfs.h>
17#include <linux/uio.h>
18#include <linux/iversion.h>
19#include <linux/fsverity.h>
20#include "ctree.h"
21#include "disk-io.h"
22#include "transaction.h"
23#include "btrfs_inode.h"
24#include "print-tree.h"
25#include "tree-log.h"
26#include "locking.h"
27#include "volumes.h"
28#include "qgroup.h"
29#include "compression.h"
30#include "delalloc-space.h"
31#include "reflink.h"
32#include "subpage.h"
33
34static struct kmem_cache *btrfs_inode_defrag_cachep;
35
36
37
38
39
40struct inode_defrag {
41 struct rb_node rb_node;
42
43 u64 ino;
44
45
46
47
48 u64 transid;
49
50
51 u64 root;
52
53
54 u64 last_offset;
55
56
57 int cycled;
58};
59
60static int __compare_inode_defrag(struct inode_defrag *defrag1,
61 struct inode_defrag *defrag2)
62{
63 if (defrag1->root > defrag2->root)
64 return 1;
65 else if (defrag1->root < defrag2->root)
66 return -1;
67 else if (defrag1->ino > defrag2->ino)
68 return 1;
69 else if (defrag1->ino < defrag2->ino)
70 return -1;
71 else
72 return 0;
73}
74
75
76
77
78
79
80
81
82
83
84static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
85 struct inode_defrag *defrag)
86{
87 struct btrfs_fs_info *fs_info = inode->root->fs_info;
88 struct inode_defrag *entry;
89 struct rb_node **p;
90 struct rb_node *parent = NULL;
91 int ret;
92
93 p = &fs_info->defrag_inodes.rb_node;
94 while (*p) {
95 parent = *p;
96 entry = rb_entry(parent, struct inode_defrag, rb_node);
97
98 ret = __compare_inode_defrag(defrag, entry);
99 if (ret < 0)
100 p = &parent->rb_left;
101 else if (ret > 0)
102 p = &parent->rb_right;
103 else {
104
105
106
107
108 if (defrag->transid < entry->transid)
109 entry->transid = defrag->transid;
110 if (defrag->last_offset > entry->last_offset)
111 entry->last_offset = defrag->last_offset;
112 return -EEXIST;
113 }
114 }
115 set_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags);
116 rb_link_node(&defrag->rb_node, parent, p);
117 rb_insert_color(&defrag->rb_node, &fs_info->defrag_inodes);
118 return 0;
119}
120
121static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
122{
123 if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
124 return 0;
125
126 if (btrfs_fs_closing(fs_info))
127 return 0;
128
129 return 1;
130}
131
132
133
134
135
136int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
137 struct btrfs_inode *inode)
138{
139 struct btrfs_root *root = inode->root;
140 struct btrfs_fs_info *fs_info = root->fs_info;
141 struct inode_defrag *defrag;
142 u64 transid;
143 int ret;
144
145 if (!__need_auto_defrag(fs_info))
146 return 0;
147
148 if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
149 return 0;
150
151 if (trans)
152 transid = trans->transid;
153 else
154 transid = inode->root->last_trans;
155
156 defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS);
157 if (!defrag)
158 return -ENOMEM;
159
160 defrag->ino = btrfs_ino(inode);
161 defrag->transid = transid;
162 defrag->root = root->root_key.objectid;
163
164 spin_lock(&fs_info->defrag_inodes_lock);
165 if (!test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) {
166
167
168
169
170
171 ret = __btrfs_add_inode_defrag(inode, defrag);
172 if (ret)
173 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
174 } else {
175 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
176 }
177 spin_unlock(&fs_info->defrag_inodes_lock);
178 return 0;
179}
180
181
182
183
184
185
186static void btrfs_requeue_inode_defrag(struct btrfs_inode *inode,
187 struct inode_defrag *defrag)
188{
189 struct btrfs_fs_info *fs_info = inode->root->fs_info;
190 int ret;
191
192 if (!__need_auto_defrag(fs_info))
193 goto out;
194
195
196
197
198
199 spin_lock(&fs_info->defrag_inodes_lock);
200 ret = __btrfs_add_inode_defrag(inode, defrag);
201 spin_unlock(&fs_info->defrag_inodes_lock);
202 if (ret)
203 goto out;
204 return;
205out:
206 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
207}
208
209
210
211
212
213static struct inode_defrag *
214btrfs_pick_defrag_inode(struct btrfs_fs_info *fs_info, u64 root, u64 ino)
215{
216 struct inode_defrag *entry = NULL;
217 struct inode_defrag tmp;
218 struct rb_node *p;
219 struct rb_node *parent = NULL;
220 int ret;
221
222 tmp.ino = ino;
223 tmp.root = root;
224
225 spin_lock(&fs_info->defrag_inodes_lock);
226 p = fs_info->defrag_inodes.rb_node;
227 while (p) {
228 parent = p;
229 entry = rb_entry(parent, struct inode_defrag, rb_node);
230
231 ret = __compare_inode_defrag(&tmp, entry);
232 if (ret < 0)
233 p = parent->rb_left;
234 else if (ret > 0)
235 p = parent->rb_right;
236 else
237 goto out;
238 }
239
240 if (parent && __compare_inode_defrag(&tmp, entry) > 0) {
241 parent = rb_next(parent);
242 if (parent)
243 entry = rb_entry(parent, struct inode_defrag, rb_node);
244 else
245 entry = NULL;
246 }
247out:
248 if (entry)
249 rb_erase(parent, &fs_info->defrag_inodes);
250 spin_unlock(&fs_info->defrag_inodes_lock);
251 return entry;
252}
253
254void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info)
255{
256 struct inode_defrag *defrag;
257 struct rb_node *node;
258
259 spin_lock(&fs_info->defrag_inodes_lock);
260 node = rb_first(&fs_info->defrag_inodes);
261 while (node) {
262 rb_erase(node, &fs_info->defrag_inodes);
263 defrag = rb_entry(node, struct inode_defrag, rb_node);
264 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
265
266 cond_resched_lock(&fs_info->defrag_inodes_lock);
267
268 node = rb_first(&fs_info->defrag_inodes);
269 }
270 spin_unlock(&fs_info->defrag_inodes_lock);
271}
272
273#define BTRFS_DEFRAG_BATCH 1024
274
275static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info,
276 struct inode_defrag *defrag)
277{
278 struct btrfs_root *inode_root;
279 struct inode *inode;
280 struct btrfs_ioctl_defrag_range_args range;
281 int num_defrag;
282 int ret;
283
284
285 inode_root = btrfs_get_fs_root(fs_info, defrag->root, true);
286 if (IS_ERR(inode_root)) {
287 ret = PTR_ERR(inode_root);
288 goto cleanup;
289 }
290
291 inode = btrfs_iget(fs_info->sb, defrag->ino, inode_root);
292 btrfs_put_root(inode_root);
293 if (IS_ERR(inode)) {
294 ret = PTR_ERR(inode);
295 goto cleanup;
296 }
297
298
299 clear_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
300 memset(&range, 0, sizeof(range));
301 range.len = (u64)-1;
302 range.start = defrag->last_offset;
303
304 sb_start_write(fs_info->sb);
305 num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
306 BTRFS_DEFRAG_BATCH);
307 sb_end_write(fs_info->sb);
308
309
310
311
312
313 if (num_defrag == BTRFS_DEFRAG_BATCH) {
314 defrag->last_offset = range.start;
315 btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
316 } else if (defrag->last_offset && !defrag->cycled) {
317
318
319
320
321
322 defrag->last_offset = 0;
323 defrag->cycled = 1;
324 btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
325 } else {
326 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
327 }
328
329 iput(inode);
330 return 0;
331cleanup:
332 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
333 return ret;
334}
335
336
337
338
339
340int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
341{
342 struct inode_defrag *defrag;
343 u64 first_ino = 0;
344 u64 root_objectid = 0;
345
346 atomic_inc(&fs_info->defrag_running);
347 while (1) {
348
349 if (test_bit(BTRFS_FS_STATE_REMOUNTING,
350 &fs_info->fs_state))
351 break;
352
353 if (!__need_auto_defrag(fs_info))
354 break;
355
356
357 defrag = btrfs_pick_defrag_inode(fs_info, root_objectid,
358 first_ino);
359 if (!defrag) {
360 if (root_objectid || first_ino) {
361 root_objectid = 0;
362 first_ino = 0;
363 continue;
364 } else {
365 break;
366 }
367 }
368
369 first_ino = defrag->ino + 1;
370 root_objectid = defrag->root;
371
372 __btrfs_run_defrag_inode(fs_info, defrag);
373 }
374 atomic_dec(&fs_info->defrag_running);
375
376
377
378
379
380 wake_up(&fs_info->transaction_wait);
381 return 0;
382}
383
384
385
386
387static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
388 struct page **prepared_pages,
389 struct iov_iter *i)
390{
391 size_t copied = 0;
392 size_t total_copied = 0;
393 int pg = 0;
394 int offset = offset_in_page(pos);
395
396 while (write_bytes > 0) {
397 size_t count = min_t(size_t,
398 PAGE_SIZE - offset, write_bytes);
399 struct page *page = prepared_pages[pg];
400
401
402
403 copied = copy_page_from_iter_atomic(page, offset, count, i);
404
405
406 flush_dcache_page(page);
407
408
409
410
411
412
413
414
415
416
417 if (unlikely(copied < count)) {
418 if (!PageUptodate(page)) {
419 iov_iter_revert(i, copied);
420 copied = 0;
421 }
422 if (!copied)
423 break;
424 }
425
426 write_bytes -= copied;
427 total_copied += copied;
428 offset += copied;
429 if (offset == PAGE_SIZE) {
430 pg++;
431 offset = 0;
432 }
433 }
434 return total_copied;
435}
436
437
438
439
440static void btrfs_drop_pages(struct btrfs_fs_info *fs_info,
441 struct page **pages, size_t num_pages,
442 u64 pos, u64 copied)
443{
444 size_t i;
445 u64 block_start = round_down(pos, fs_info->sectorsize);
446 u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start;
447
448 ASSERT(block_len <= U32_MAX);
449 for (i = 0; i < num_pages; i++) {
450
451
452
453
454
455
456 btrfs_page_clamp_clear_checked(fs_info, pages[i], block_start,
457 block_len);
458 unlock_page(pages[i]);
459 put_page(pages[i]);
460 }
461}
462
463
464
465
466
467
468
469
470int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages,
471 size_t num_pages, loff_t pos, size_t write_bytes,
472 struct extent_state **cached, bool noreserve)
473{
474 struct btrfs_fs_info *fs_info = inode->root->fs_info;
475 int err = 0;
476 int i;
477 u64 num_bytes;
478 u64 start_pos;
479 u64 end_of_last_block;
480 u64 end_pos = pos + write_bytes;
481 loff_t isize = i_size_read(&inode->vfs_inode);
482 unsigned int extra_bits = 0;
483
484 if (write_bytes == 0)
485 return 0;
486
487 if (noreserve)
488 extra_bits |= EXTENT_NORESERVE;
489
490 start_pos = round_down(pos, fs_info->sectorsize);
491 num_bytes = round_up(write_bytes + pos - start_pos,
492 fs_info->sectorsize);
493 ASSERT(num_bytes <= U32_MAX);
494
495 end_of_last_block = start_pos + num_bytes - 1;
496
497
498
499
500
501 clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block,
502 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
503 0, 0, cached);
504
505 err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
506 extra_bits, cached);
507 if (err)
508 return err;
509
510 for (i = 0; i < num_pages; i++) {
511 struct page *p = pages[i];
512
513 btrfs_page_clamp_set_uptodate(fs_info, p, start_pos, num_bytes);
514 btrfs_page_clamp_clear_checked(fs_info, p, start_pos, num_bytes);
515 btrfs_page_clamp_set_dirty(fs_info, p, start_pos, num_bytes);
516 }
517
518
519
520
521
522
523 if (end_pos > isize)
524 i_size_write(&inode->vfs_inode, end_pos);
525 return 0;
526}
527
528
529
530
531
532void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end,
533 int skip_pinned)
534{
535 struct extent_map *em;
536 struct extent_map *split = NULL;
537 struct extent_map *split2 = NULL;
538 struct extent_map_tree *em_tree = &inode->extent_tree;
539 u64 len = end - start + 1;
540 u64 gen;
541 int ret;
542 int testend = 1;
543 unsigned long flags;
544 int compressed = 0;
545 bool modified;
546
547 WARN_ON(end < start);
548 if (end == (u64)-1) {
549 len = (u64)-1;
550 testend = 0;
551 }
552 while (1) {
553 int no_splits = 0;
554
555 modified = false;
556 if (!split)
557 split = alloc_extent_map();
558 if (!split2)
559 split2 = alloc_extent_map();
560 if (!split || !split2)
561 no_splits = 1;
562
563 write_lock(&em_tree->lock);
564 em = lookup_extent_mapping(em_tree, start, len);
565 if (!em) {
566 write_unlock(&em_tree->lock);
567 break;
568 }
569 flags = em->flags;
570 gen = em->generation;
571 if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
572 if (testend && em->start + em->len >= start + len) {
573 free_extent_map(em);
574 write_unlock(&em_tree->lock);
575 break;
576 }
577 start = em->start + em->len;
578 if (testend)
579 len = start + len - (em->start + em->len);
580 free_extent_map(em);
581 write_unlock(&em_tree->lock);
582 continue;
583 }
584 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
585 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
586 clear_bit(EXTENT_FLAG_LOGGING, &flags);
587 modified = !list_empty(&em->list);
588 if (no_splits)
589 goto next;
590
591 if (em->start < start) {
592 split->start = em->start;
593 split->len = start - em->start;
594
595 if (em->block_start < EXTENT_MAP_LAST_BYTE) {
596 split->orig_start = em->orig_start;
597 split->block_start = em->block_start;
598
599 if (compressed)
600 split->block_len = em->block_len;
601 else
602 split->block_len = split->len;
603 split->orig_block_len = max(split->block_len,
604 em->orig_block_len);
605 split->ram_bytes = em->ram_bytes;
606 } else {
607 split->orig_start = split->start;
608 split->block_len = 0;
609 split->block_start = em->block_start;
610 split->orig_block_len = 0;
611 split->ram_bytes = split->len;
612 }
613
614 split->generation = gen;
615 split->flags = flags;
616 split->compress_type = em->compress_type;
617 replace_extent_mapping(em_tree, em, split, modified);
618 free_extent_map(split);
619 split = split2;
620 split2 = NULL;
621 }
622 if (testend && em->start + em->len > start + len) {
623 u64 diff = start + len - em->start;
624
625 split->start = start + len;
626 split->len = em->start + em->len - (start + len);
627 split->flags = flags;
628 split->compress_type = em->compress_type;
629 split->generation = gen;
630
631 if (em->block_start < EXTENT_MAP_LAST_BYTE) {
632 split->orig_block_len = max(em->block_len,
633 em->orig_block_len);
634
635 split->ram_bytes = em->ram_bytes;
636 if (compressed) {
637 split->block_len = em->block_len;
638 split->block_start = em->block_start;
639 split->orig_start = em->orig_start;
640 } else {
641 split->block_len = split->len;
642 split->block_start = em->block_start
643 + diff;
644 split->orig_start = em->orig_start;
645 }
646 } else {
647 split->ram_bytes = split->len;
648 split->orig_start = split->start;
649 split->block_len = 0;
650 split->block_start = em->block_start;
651 split->orig_block_len = 0;
652 }
653
654 if (extent_map_in_tree(em)) {
655 replace_extent_mapping(em_tree, em, split,
656 modified);
657 } else {
658 ret = add_extent_mapping(em_tree, split,
659 modified);
660 ASSERT(ret == 0);
661 }
662 free_extent_map(split);
663 split = NULL;
664 }
665next:
666 if (extent_map_in_tree(em))
667 remove_extent_mapping(em_tree, em);
668 write_unlock(&em_tree->lock);
669
670
671 free_extent_map(em);
672
673 free_extent_map(em);
674 }
675 if (split)
676 free_extent_map(split);
677 if (split2)
678 free_extent_map(split2);
679}
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696int btrfs_drop_extents(struct btrfs_trans_handle *trans,
697 struct btrfs_root *root, struct btrfs_inode *inode,
698 struct btrfs_drop_extents_args *args)
699{
700 struct btrfs_fs_info *fs_info = root->fs_info;
701 struct extent_buffer *leaf;
702 struct btrfs_file_extent_item *fi;
703 struct btrfs_ref ref = { 0 };
704 struct btrfs_key key;
705 struct btrfs_key new_key;
706 u64 ino = btrfs_ino(inode);
707 u64 search_start = args->start;
708 u64 disk_bytenr = 0;
709 u64 num_bytes = 0;
710 u64 extent_offset = 0;
711 u64 extent_end = 0;
712 u64 last_end = args->start;
713 int del_nr = 0;
714 int del_slot = 0;
715 int extent_type;
716 int recow;
717 int ret;
718 int modify_tree = -1;
719 int update_refs;
720 int found = 0;
721 int leafs_visited = 0;
722 struct btrfs_path *path = args->path;
723
724 args->bytes_found = 0;
725 args->extent_inserted = false;
726
727
728 ASSERT(!(args->replace_extent && !args->path));
729
730 if (!path) {
731 path = btrfs_alloc_path();
732 if (!path) {
733 ret = -ENOMEM;
734 goto out;
735 }
736 }
737
738 if (args->drop_cache)
739 btrfs_drop_extent_cache(inode, args->start, args->end - 1, 0);
740
741 if (args->start >= inode->disk_i_size && !args->replace_extent)
742 modify_tree = 0;
743
744 update_refs = (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID);
745 while (1) {
746 recow = 0;
747 ret = btrfs_lookup_file_extent(trans, root, path, ino,
748 search_start, modify_tree);
749 if (ret < 0)
750 break;
751 if (ret > 0 && path->slots[0] > 0 && search_start == args->start) {
752 leaf = path->nodes[0];
753 btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
754 if (key.objectid == ino &&
755 key.type == BTRFS_EXTENT_DATA_KEY)
756 path->slots[0]--;
757 }
758 ret = 0;
759 leafs_visited++;
760next_slot:
761 leaf = path->nodes[0];
762 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
763 BUG_ON(del_nr > 0);
764 ret = btrfs_next_leaf(root, path);
765 if (ret < 0)
766 break;
767 if (ret > 0) {
768 ret = 0;
769 break;
770 }
771 leafs_visited++;
772 leaf = path->nodes[0];
773 recow = 1;
774 }
775
776 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
777
778 if (key.objectid > ino)
779 break;
780 if (WARN_ON_ONCE(key.objectid < ino) ||
781 key.type < BTRFS_EXTENT_DATA_KEY) {
782 ASSERT(del_nr == 0);
783 path->slots[0]++;
784 goto next_slot;
785 }
786 if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= args->end)
787 break;
788
789 fi = btrfs_item_ptr(leaf, path->slots[0],
790 struct btrfs_file_extent_item);
791 extent_type = btrfs_file_extent_type(leaf, fi);
792
793 if (extent_type == BTRFS_FILE_EXTENT_REG ||
794 extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
795 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
796 num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
797 extent_offset = btrfs_file_extent_offset(leaf, fi);
798 extent_end = key.offset +
799 btrfs_file_extent_num_bytes(leaf, fi);
800 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
801 extent_end = key.offset +
802 btrfs_file_extent_ram_bytes(leaf, fi);
803 } else {
804
805 BUG();
806 }
807
808
809
810
811
812
813
814
815
816
817 if (extent_end == key.offset && extent_end >= search_start) {
818 last_end = extent_end;
819 goto delete_extent_item;
820 }
821
822 if (extent_end <= search_start) {
823 path->slots[0]++;
824 goto next_slot;
825 }
826
827 found = 1;
828 search_start = max(key.offset, args->start);
829 if (recow || !modify_tree) {
830 modify_tree = -1;
831 btrfs_release_path(path);
832 continue;
833 }
834
835
836
837
838
839 if (args->start > key.offset && args->end < extent_end) {
840 BUG_ON(del_nr > 0);
841 if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
842 ret = -EOPNOTSUPP;
843 break;
844 }
845
846 memcpy(&new_key, &key, sizeof(new_key));
847 new_key.offset = args->start;
848 ret = btrfs_duplicate_item(trans, root, path,
849 &new_key);
850 if (ret == -EAGAIN) {
851 btrfs_release_path(path);
852 continue;
853 }
854 if (ret < 0)
855 break;
856
857 leaf = path->nodes[0];
858 fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
859 struct btrfs_file_extent_item);
860 btrfs_set_file_extent_num_bytes(leaf, fi,
861 args->start - key.offset);
862
863 fi = btrfs_item_ptr(leaf, path->slots[0],
864 struct btrfs_file_extent_item);
865
866 extent_offset += args->start - key.offset;
867 btrfs_set_file_extent_offset(leaf, fi, extent_offset);
868 btrfs_set_file_extent_num_bytes(leaf, fi,
869 extent_end - args->start);
870 btrfs_mark_buffer_dirty(leaf);
871
872 if (update_refs && disk_bytenr > 0) {
873 btrfs_init_generic_ref(&ref,
874 BTRFS_ADD_DELAYED_REF,
875 disk_bytenr, num_bytes, 0);
876 btrfs_init_data_ref(&ref,
877 root->root_key.objectid,
878 new_key.objectid,
879 args->start - extent_offset,
880 0, false);
881 ret = btrfs_inc_extent_ref(trans, &ref);
882 BUG_ON(ret);
883 }
884 key.offset = args->start;
885 }
886
887
888
889
890 last_end = extent_end;
891
892
893
894
895
896 if (args->start <= key.offset && args->end < extent_end) {
897 if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
898 ret = -EOPNOTSUPP;
899 break;
900 }
901
902 memcpy(&new_key, &key, sizeof(new_key));
903 new_key.offset = args->end;
904 btrfs_set_item_key_safe(fs_info, path, &new_key);
905
906 extent_offset += args->end - key.offset;
907 btrfs_set_file_extent_offset(leaf, fi, extent_offset);
908 btrfs_set_file_extent_num_bytes(leaf, fi,
909 extent_end - args->end);
910 btrfs_mark_buffer_dirty(leaf);
911 if (update_refs && disk_bytenr > 0)
912 args->bytes_found += args->end - key.offset;
913 break;
914 }
915
916 search_start = extent_end;
917
918
919
920
921 if (args->start > key.offset && args->end >= extent_end) {
922 BUG_ON(del_nr > 0);
923 if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
924 ret = -EOPNOTSUPP;
925 break;
926 }
927
928 btrfs_set_file_extent_num_bytes(leaf, fi,
929 args->start - key.offset);
930 btrfs_mark_buffer_dirty(leaf);
931 if (update_refs && disk_bytenr > 0)
932 args->bytes_found += extent_end - args->start;
933 if (args->end == extent_end)
934 break;
935
936 path->slots[0]++;
937 goto next_slot;
938 }
939
940
941
942
943
944 if (args->start <= key.offset && args->end >= extent_end) {
945delete_extent_item:
946 if (del_nr == 0) {
947 del_slot = path->slots[0];
948 del_nr = 1;
949 } else {
950 BUG_ON(del_slot + del_nr != path->slots[0]);
951 del_nr++;
952 }
953
954 if (update_refs &&
955 extent_type == BTRFS_FILE_EXTENT_INLINE) {
956 args->bytes_found += extent_end - key.offset;
957 extent_end = ALIGN(extent_end,
958 fs_info->sectorsize);
959 } else if (update_refs && disk_bytenr > 0) {
960 btrfs_init_generic_ref(&ref,
961 BTRFS_DROP_DELAYED_REF,
962 disk_bytenr, num_bytes, 0);
963 btrfs_init_data_ref(&ref,
964 root->root_key.objectid,
965 key.objectid,
966 key.offset - extent_offset, 0,
967 false);
968 ret = btrfs_free_extent(trans, &ref);
969 BUG_ON(ret);
970 args->bytes_found += extent_end - key.offset;
971 }
972
973 if (args->end == extent_end)
974 break;
975
976 if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
977 path->slots[0]++;
978 goto next_slot;
979 }
980
981 ret = btrfs_del_items(trans, root, path, del_slot,
982 del_nr);
983 if (ret) {
984 btrfs_abort_transaction(trans, ret);
985 break;
986 }
987
988 del_nr = 0;
989 del_slot = 0;
990
991 btrfs_release_path(path);
992 continue;
993 }
994
995 BUG();
996 }
997
998 if (!ret && del_nr > 0) {
999
1000
1001
1002
1003
1004
1005 path->slots[0] = del_slot;
1006 ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1007 if (ret)
1008 btrfs_abort_transaction(trans, ret);
1009 }
1010
1011 leaf = path->nodes[0];
1012
1013
1014
1015
1016
1017 if (!ret && args->replace_extent && leafs_visited == 1 &&
1018 path->locks[0] == BTRFS_WRITE_LOCK &&
1019 btrfs_leaf_free_space(leaf) >=
1020 sizeof(struct btrfs_item) + args->extent_item_size) {
1021
1022 key.objectid = ino;
1023 key.type = BTRFS_EXTENT_DATA_KEY;
1024 key.offset = args->start;
1025 if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {
1026 struct btrfs_key slot_key;
1027
1028 btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);
1029 if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
1030 path->slots[0]++;
1031 }
1032 btrfs_setup_item_for_insert(root, path, &key, args->extent_item_size);
1033 args->extent_inserted = true;
1034 }
1035
1036 if (!args->path)
1037 btrfs_free_path(path);
1038 else if (!args->extent_inserted)
1039 btrfs_release_path(path);
1040out:
1041 args->drop_end = found ? min(args->end, last_end) : args->end;
1042
1043 return ret;
1044}
1045
1046static int extent_mergeable(struct extent_buffer *leaf, int slot,
1047 u64 objectid, u64 bytenr, u64 orig_offset,
1048 u64 *start, u64 *end)
1049{
1050 struct btrfs_file_extent_item *fi;
1051 struct btrfs_key key;
1052 u64 extent_end;
1053
1054 if (slot < 0 || slot >= btrfs_header_nritems(leaf))
1055 return 0;
1056
1057 btrfs_item_key_to_cpu(leaf, &key, slot);
1058 if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
1059 return 0;
1060
1061 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
1062 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
1063 btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
1064 btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
1065 btrfs_file_extent_compression(leaf, fi) ||
1066 btrfs_file_extent_encryption(leaf, fi) ||
1067 btrfs_file_extent_other_encoding(leaf, fi))
1068 return 0;
1069
1070 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1071 if ((*start && *start != key.offset) || (*end && *end != extent_end))
1072 return 0;
1073
1074 *start = key.offset;
1075 *end = extent_end;
1076 return 1;
1077}
1078
1079
1080
1081
1082
1083
1084
1085
1086int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
1087 struct btrfs_inode *inode, u64 start, u64 end)
1088{
1089 struct btrfs_fs_info *fs_info = trans->fs_info;
1090 struct btrfs_root *root = inode->root;
1091 struct extent_buffer *leaf;
1092 struct btrfs_path *path;
1093 struct btrfs_file_extent_item *fi;
1094 struct btrfs_ref ref = { 0 };
1095 struct btrfs_key key;
1096 struct btrfs_key new_key;
1097 u64 bytenr;
1098 u64 num_bytes;
1099 u64 extent_end;
1100 u64 orig_offset;
1101 u64 other_start;
1102 u64 other_end;
1103 u64 split;
1104 int del_nr = 0;
1105 int del_slot = 0;
1106 int recow;
1107 int ret = 0;
1108 u64 ino = btrfs_ino(inode);
1109
1110 path = btrfs_alloc_path();
1111 if (!path)
1112 return -ENOMEM;
1113again:
1114 recow = 0;
1115 split = start;
1116 key.objectid = ino;
1117 key.type = BTRFS_EXTENT_DATA_KEY;
1118 key.offset = split;
1119
1120 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1121 if (ret < 0)
1122 goto out;
1123 if (ret > 0 && path->slots[0] > 0)
1124 path->slots[0]--;
1125
1126 leaf = path->nodes[0];
1127 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1128 if (key.objectid != ino ||
1129 key.type != BTRFS_EXTENT_DATA_KEY) {
1130 ret = -EINVAL;
1131 btrfs_abort_transaction(trans, ret);
1132 goto out;
1133 }
1134 fi = btrfs_item_ptr(leaf, path->slots[0],
1135 struct btrfs_file_extent_item);
1136 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC) {
1137 ret = -EINVAL;
1138 btrfs_abort_transaction(trans, ret);
1139 goto out;
1140 }
1141 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1142 if (key.offset > start || extent_end < end) {
1143 ret = -EINVAL;
1144 btrfs_abort_transaction(trans, ret);
1145 goto out;
1146 }
1147
1148 bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1149 num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
1150 orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
1151 memcpy(&new_key, &key, sizeof(new_key));
1152
1153 if (start == key.offset && end < extent_end) {
1154 other_start = 0;
1155 other_end = start;
1156 if (extent_mergeable(leaf, path->slots[0] - 1,
1157 ino, bytenr, orig_offset,
1158 &other_start, &other_end)) {
1159 new_key.offset = end;
1160 btrfs_set_item_key_safe(fs_info, path, &new_key);
1161 fi = btrfs_item_ptr(leaf, path->slots[0],
1162 struct btrfs_file_extent_item);
1163 btrfs_set_file_extent_generation(leaf, fi,
1164 trans->transid);
1165 btrfs_set_file_extent_num_bytes(leaf, fi,
1166 extent_end - end);
1167 btrfs_set_file_extent_offset(leaf, fi,
1168 end - orig_offset);
1169 fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1170 struct btrfs_file_extent_item);
1171 btrfs_set_file_extent_generation(leaf, fi,
1172 trans->transid);
1173 btrfs_set_file_extent_num_bytes(leaf, fi,
1174 end - other_start);
1175 btrfs_mark_buffer_dirty(leaf);
1176 goto out;
1177 }
1178 }
1179
1180 if (start > key.offset && end == extent_end) {
1181 other_start = end;
1182 other_end = 0;
1183 if (extent_mergeable(leaf, path->slots[0] + 1,
1184 ino, bytenr, orig_offset,
1185 &other_start, &other_end)) {
1186 fi = btrfs_item_ptr(leaf, path->slots[0],
1187 struct btrfs_file_extent_item);
1188 btrfs_set_file_extent_num_bytes(leaf, fi,
1189 start - key.offset);
1190 btrfs_set_file_extent_generation(leaf, fi,
1191 trans->transid);
1192 path->slots[0]++;
1193 new_key.offset = start;
1194 btrfs_set_item_key_safe(fs_info, path, &new_key);
1195
1196 fi = btrfs_item_ptr(leaf, path->slots[0],
1197 struct btrfs_file_extent_item);
1198 btrfs_set_file_extent_generation(leaf, fi,
1199 trans->transid);
1200 btrfs_set_file_extent_num_bytes(leaf, fi,
1201 other_end - start);
1202 btrfs_set_file_extent_offset(leaf, fi,
1203 start - orig_offset);
1204 btrfs_mark_buffer_dirty(leaf);
1205 goto out;
1206 }
1207 }
1208
1209 while (start > key.offset || end < extent_end) {
1210 if (key.offset == start)
1211 split = end;
1212
1213 new_key.offset = split;
1214 ret = btrfs_duplicate_item(trans, root, path, &new_key);
1215 if (ret == -EAGAIN) {
1216 btrfs_release_path(path);
1217 goto again;
1218 }
1219 if (ret < 0) {
1220 btrfs_abort_transaction(trans, ret);
1221 goto out;
1222 }
1223
1224 leaf = path->nodes[0];
1225 fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1226 struct btrfs_file_extent_item);
1227 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1228 btrfs_set_file_extent_num_bytes(leaf, fi,
1229 split - key.offset);
1230
1231 fi = btrfs_item_ptr(leaf, path->slots[0],
1232 struct btrfs_file_extent_item);
1233
1234 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1235 btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
1236 btrfs_set_file_extent_num_bytes(leaf, fi,
1237 extent_end - split);
1238 btrfs_mark_buffer_dirty(leaf);
1239
1240 btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF, bytenr,
1241 num_bytes, 0);
1242 btrfs_init_data_ref(&ref, root->root_key.objectid, ino,
1243 orig_offset, 0, false);
1244 ret = btrfs_inc_extent_ref(trans, &ref);
1245 if (ret) {
1246 btrfs_abort_transaction(trans, ret);
1247 goto out;
1248 }
1249
1250 if (split == start) {
1251 key.offset = start;
1252 } else {
1253 if (start != key.offset) {
1254 ret = -EINVAL;
1255 btrfs_abort_transaction(trans, ret);
1256 goto out;
1257 }
1258 path->slots[0]--;
1259 extent_end = end;
1260 }
1261 recow = 1;
1262 }
1263
1264 other_start = end;
1265 other_end = 0;
1266 btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF, bytenr,
1267 num_bytes, 0);
1268 btrfs_init_data_ref(&ref, root->root_key.objectid, ino, orig_offset,
1269 0, false);
1270 if (extent_mergeable(leaf, path->slots[0] + 1,
1271 ino, bytenr, orig_offset,
1272 &other_start, &other_end)) {
1273 if (recow) {
1274 btrfs_release_path(path);
1275 goto again;
1276 }
1277 extent_end = other_end;
1278 del_slot = path->slots[0] + 1;
1279 del_nr++;
1280 ret = btrfs_free_extent(trans, &ref);
1281 if (ret) {
1282 btrfs_abort_transaction(trans, ret);
1283 goto out;
1284 }
1285 }
1286 other_start = 0;
1287 other_end = start;
1288 if (extent_mergeable(leaf, path->slots[0] - 1,
1289 ino, bytenr, orig_offset,
1290 &other_start, &other_end)) {
1291 if (recow) {
1292 btrfs_release_path(path);
1293 goto again;
1294 }
1295 key.offset = other_start;
1296 del_slot = path->slots[0];
1297 del_nr++;
1298 ret = btrfs_free_extent(trans, &ref);
1299 if (ret) {
1300 btrfs_abort_transaction(trans, ret);
1301 goto out;
1302 }
1303 }
1304 if (del_nr == 0) {
1305 fi = btrfs_item_ptr(leaf, path->slots[0],
1306 struct btrfs_file_extent_item);
1307 btrfs_set_file_extent_type(leaf, fi,
1308 BTRFS_FILE_EXTENT_REG);
1309 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1310 btrfs_mark_buffer_dirty(leaf);
1311 } else {
1312 fi = btrfs_item_ptr(leaf, del_slot - 1,
1313 struct btrfs_file_extent_item);
1314 btrfs_set_file_extent_type(leaf, fi,
1315 BTRFS_FILE_EXTENT_REG);
1316 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1317 btrfs_set_file_extent_num_bytes(leaf, fi,
1318 extent_end - key.offset);
1319 btrfs_mark_buffer_dirty(leaf);
1320
1321 ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1322 if (ret < 0) {
1323 btrfs_abort_transaction(trans, ret);
1324 goto out;
1325 }
1326 }
1327out:
1328 btrfs_free_path(path);
1329 return ret;
1330}
1331
1332
1333
1334
1335
1336static int prepare_uptodate_page(struct inode *inode,
1337 struct page *page, u64 pos,
1338 bool force_uptodate)
1339{
1340 int ret = 0;
1341
1342 if (((pos & (PAGE_SIZE - 1)) || force_uptodate) &&
1343 !PageUptodate(page)) {
1344 ret = btrfs_readpage(NULL, page);
1345 if (ret)
1346 return ret;
1347 lock_page(page);
1348 if (!PageUptodate(page)) {
1349 unlock_page(page);
1350 return -EIO;
1351 }
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363 if (page->mapping != inode->i_mapping || !PagePrivate(page)) {
1364 unlock_page(page);
1365 return -EAGAIN;
1366 }
1367 }
1368 return 0;
1369}
1370
1371
1372
1373
1374static noinline int prepare_pages(struct inode *inode, struct page **pages,
1375 size_t num_pages, loff_t pos,
1376 size_t write_bytes, bool force_uptodate)
1377{
1378 int i;
1379 unsigned long index = pos >> PAGE_SHIFT;
1380 gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
1381 int err = 0;
1382 int faili;
1383
1384 for (i = 0; i < num_pages; i++) {
1385again:
1386 pages[i] = find_or_create_page(inode->i_mapping, index + i,
1387 mask | __GFP_WRITE);
1388 if (!pages[i]) {
1389 faili = i - 1;
1390 err = -ENOMEM;
1391 goto fail;
1392 }
1393
1394 err = set_page_extent_mapped(pages[i]);
1395 if (err < 0) {
1396 faili = i;
1397 goto fail;
1398 }
1399
1400 if (i == 0)
1401 err = prepare_uptodate_page(inode, pages[i], pos,
1402 force_uptodate);
1403 if (!err && i == num_pages - 1)
1404 err = prepare_uptodate_page(inode, pages[i],
1405 pos + write_bytes, false);
1406 if (err) {
1407 put_page(pages[i]);
1408 if (err == -EAGAIN) {
1409 err = 0;
1410 goto again;
1411 }
1412 faili = i - 1;
1413 goto fail;
1414 }
1415 wait_on_page_writeback(pages[i]);
1416 }
1417
1418 return 0;
1419fail:
1420 while (faili >= 0) {
1421 unlock_page(pages[faili]);
1422 put_page(pages[faili]);
1423 faili--;
1424 }
1425 return err;
1426
1427}
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439static noinline int
1440lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
1441 size_t num_pages, loff_t pos,
1442 size_t write_bytes,
1443 u64 *lockstart, u64 *lockend,
1444 struct extent_state **cached_state)
1445{
1446 struct btrfs_fs_info *fs_info = inode->root->fs_info;
1447 u64 start_pos;
1448 u64 last_pos;
1449 int i;
1450 int ret = 0;
1451
1452 start_pos = round_down(pos, fs_info->sectorsize);
1453 last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1;
1454
1455 if (start_pos < inode->vfs_inode.i_size) {
1456 struct btrfs_ordered_extent *ordered;
1457
1458 lock_extent_bits(&inode->io_tree, start_pos, last_pos,
1459 cached_state);
1460 ordered = btrfs_lookup_ordered_range(inode, start_pos,
1461 last_pos - start_pos + 1);
1462 if (ordered &&
1463 ordered->file_offset + ordered->num_bytes > start_pos &&
1464 ordered->file_offset <= last_pos) {
1465 unlock_extent_cached(&inode->io_tree, start_pos,
1466 last_pos, cached_state);
1467 for (i = 0; i < num_pages; i++) {
1468 unlock_page(pages[i]);
1469 put_page(pages[i]);
1470 }
1471 btrfs_start_ordered_extent(ordered, 1);
1472 btrfs_put_ordered_extent(ordered);
1473 return -EAGAIN;
1474 }
1475 if (ordered)
1476 btrfs_put_ordered_extent(ordered);
1477
1478 *lockstart = start_pos;
1479 *lockend = last_pos;
1480 ret = 1;
1481 }
1482
1483
1484
1485
1486
1487 for (i = 0; i < num_pages; i++)
1488 WARN_ON(!PageLocked(pages[i]));
1489
1490 return ret;
1491}
1492
1493static int check_can_nocow(struct btrfs_inode *inode, loff_t pos,
1494 size_t *write_bytes, bool nowait)
1495{
1496 struct btrfs_fs_info *fs_info = inode->root->fs_info;
1497 struct btrfs_root *root = inode->root;
1498 u64 lockstart, lockend;
1499 u64 num_bytes;
1500 int ret;
1501
1502 if (!(inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))
1503 return 0;
1504
1505 if (!nowait && !btrfs_drew_try_write_lock(&root->snapshot_lock))
1506 return -EAGAIN;
1507
1508 lockstart = round_down(pos, fs_info->sectorsize);
1509 lockend = round_up(pos + *write_bytes,
1510 fs_info->sectorsize) - 1;
1511 num_bytes = lockend - lockstart + 1;
1512
1513 if (nowait) {
1514 struct btrfs_ordered_extent *ordered;
1515
1516 if (!try_lock_extent(&inode->io_tree, lockstart, lockend))
1517 return -EAGAIN;
1518
1519 ordered = btrfs_lookup_ordered_range(inode, lockstart,
1520 num_bytes);
1521 if (ordered) {
1522 btrfs_put_ordered_extent(ordered);
1523 ret = -EAGAIN;
1524 goto out_unlock;
1525 }
1526 } else {
1527 btrfs_lock_and_flush_ordered_range(inode, lockstart,
1528 lockend, NULL);
1529 }
1530
1531 ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes,
1532 NULL, NULL, NULL, false);
1533 if (ret <= 0) {
1534 ret = 0;
1535 if (!nowait)
1536 btrfs_drew_write_unlock(&root->snapshot_lock);
1537 } else {
1538 *write_bytes = min_t(size_t, *write_bytes ,
1539 num_bytes - pos + lockstart);
1540 }
1541out_unlock:
1542 unlock_extent(&inode->io_tree, lockstart, lockend);
1543
1544 return ret;
1545}
1546
1547static int check_nocow_nolock(struct btrfs_inode *inode, loff_t pos,
1548 size_t *write_bytes)
1549{
1550 return check_can_nocow(inode, pos, write_bytes, true);
1551}
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos,
1573 size_t *write_bytes)
1574{
1575 return check_can_nocow(inode, pos, write_bytes, false);
1576}
1577
1578void btrfs_check_nocow_unlock(struct btrfs_inode *inode)
1579{
1580 btrfs_drew_write_unlock(&inode->root->snapshot_lock);
1581}
1582
1583static void update_time_for_write(struct inode *inode)
1584{
1585 struct timespec64 now;
1586
1587 if (IS_NOCMTIME(inode))
1588 return;
1589
1590 now = current_time(inode);
1591 if (!timespec64_equal(&inode->i_mtime, &now))
1592 inode->i_mtime = now;
1593
1594 if (!timespec64_equal(&inode->i_ctime, &now))
1595 inode->i_ctime = now;
1596
1597 if (IS_I_VERSION(inode))
1598 inode_inc_iversion(inode);
1599}
1600
1601static int btrfs_write_check(struct kiocb *iocb, struct iov_iter *from,
1602 size_t count)
1603{
1604 struct file *file = iocb->ki_filp;
1605 struct inode *inode = file_inode(file);
1606 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1607 loff_t pos = iocb->ki_pos;
1608 int ret;
1609 loff_t oldsize;
1610 loff_t start_pos;
1611
1612 if (iocb->ki_flags & IOCB_NOWAIT) {
1613 size_t nocow_bytes = count;
1614
1615
1616 if (check_nocow_nolock(BTRFS_I(inode), pos, &nocow_bytes) <= 0)
1617 return -EAGAIN;
1618
1619
1620
1621
1622
1623 if (nocow_bytes < count)
1624 return -EAGAIN;
1625 }
1626
1627 current->backing_dev_info = inode_to_bdi(inode);
1628 ret = file_remove_privs(file);
1629 if (ret)
1630 return ret;
1631
1632
1633
1634
1635
1636
1637
1638 update_time_for_write(inode);
1639
1640 start_pos = round_down(pos, fs_info->sectorsize);
1641 oldsize = i_size_read(inode);
1642 if (start_pos > oldsize) {
1643
1644 loff_t end_pos = round_up(pos + count, fs_info->sectorsize);
1645
1646 ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, end_pos);
1647 if (ret) {
1648 current->backing_dev_info = NULL;
1649 return ret;
1650 }
1651 }
1652
1653 return 0;
1654}
1655
1656static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb,
1657 struct iov_iter *i)
1658{
1659 struct file *file = iocb->ki_filp;
1660 loff_t pos;
1661 struct inode *inode = file_inode(file);
1662 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1663 struct page **pages = NULL;
1664 struct extent_changeset *data_reserved = NULL;
1665 u64 release_bytes = 0;
1666 u64 lockstart;
1667 u64 lockend;
1668 size_t num_written = 0;
1669 int nrptrs;
1670 ssize_t ret;
1671 bool only_release_metadata = false;
1672 bool force_page_uptodate = false;
1673 loff_t old_isize = i_size_read(inode);
1674 unsigned int ilock_flags = 0;
1675
1676 if (iocb->ki_flags & IOCB_NOWAIT)
1677 ilock_flags |= BTRFS_ILOCK_TRY;
1678
1679 ret = btrfs_inode_lock(inode, ilock_flags);
1680 if (ret < 0)
1681 return ret;
1682
1683 ret = generic_write_checks(iocb, i);
1684 if (ret <= 0)
1685 goto out;
1686
1687 ret = btrfs_write_check(iocb, i, ret);
1688 if (ret < 0)
1689 goto out;
1690
1691 pos = iocb->ki_pos;
1692 nrptrs = min(DIV_ROUND_UP(iov_iter_count(i), PAGE_SIZE),
1693 PAGE_SIZE / (sizeof(struct page *)));
1694 nrptrs = min(nrptrs, current->nr_dirtied_pause - current->nr_dirtied);
1695 nrptrs = max(nrptrs, 8);
1696 pages = kmalloc_array(nrptrs, sizeof(struct page *), GFP_KERNEL);
1697 if (!pages) {
1698 ret = -ENOMEM;
1699 goto out;
1700 }
1701
1702 while (iov_iter_count(i) > 0) {
1703 struct extent_state *cached_state = NULL;
1704 size_t offset = offset_in_page(pos);
1705 size_t sector_offset;
1706 size_t write_bytes = min(iov_iter_count(i),
1707 nrptrs * (size_t)PAGE_SIZE -
1708 offset);
1709 size_t num_pages;
1710 size_t reserve_bytes;
1711 size_t dirty_pages;
1712 size_t copied;
1713 size_t dirty_sectors;
1714 size_t num_sectors;
1715 int extents_locked;
1716
1717
1718
1719
1720
1721 if (unlikely(fault_in_iov_iter_readable(i, write_bytes))) {
1722 ret = -EFAULT;
1723 break;
1724 }
1725
1726 only_release_metadata = false;
1727 sector_offset = pos & (fs_info->sectorsize - 1);
1728
1729 extent_changeset_release(data_reserved);
1730 ret = btrfs_check_data_free_space(BTRFS_I(inode),
1731 &data_reserved, pos,
1732 write_bytes);
1733 if (ret < 0) {
1734
1735
1736
1737
1738
1739 if (btrfs_check_nocow_lock(BTRFS_I(inode), pos,
1740 &write_bytes) > 0)
1741 only_release_metadata = true;
1742 else
1743 break;
1744 }
1745
1746 num_pages = DIV_ROUND_UP(write_bytes + offset, PAGE_SIZE);
1747 WARN_ON(num_pages > nrptrs);
1748 reserve_bytes = round_up(write_bytes + sector_offset,
1749 fs_info->sectorsize);
1750 WARN_ON(reserve_bytes == 0);
1751 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode),
1752 reserve_bytes);
1753 if (ret) {
1754 if (!only_release_metadata)
1755 btrfs_free_reserved_data_space(BTRFS_I(inode),
1756 data_reserved, pos,
1757 write_bytes);
1758 else
1759 btrfs_check_nocow_unlock(BTRFS_I(inode));
1760 break;
1761 }
1762
1763 release_bytes = reserve_bytes;
1764again:
1765
1766
1767
1768
1769
1770 ret = prepare_pages(inode, pages, num_pages,
1771 pos, write_bytes,
1772 force_page_uptodate);
1773 if (ret) {
1774 btrfs_delalloc_release_extents(BTRFS_I(inode),
1775 reserve_bytes);
1776 break;
1777 }
1778
1779 extents_locked = lock_and_cleanup_extent_if_need(
1780 BTRFS_I(inode), pages,
1781 num_pages, pos, write_bytes, &lockstart,
1782 &lockend, &cached_state);
1783 if (extents_locked < 0) {
1784 if (extents_locked == -EAGAIN)
1785 goto again;
1786 btrfs_delalloc_release_extents(BTRFS_I(inode),
1787 reserve_bytes);
1788 ret = extents_locked;
1789 break;
1790 }
1791
1792 copied = btrfs_copy_from_user(pos, write_bytes, pages, i);
1793
1794 num_sectors = BTRFS_BYTES_TO_BLKS(fs_info, reserve_bytes);
1795 dirty_sectors = round_up(copied + sector_offset,
1796 fs_info->sectorsize);
1797 dirty_sectors = BTRFS_BYTES_TO_BLKS(fs_info, dirty_sectors);
1798
1799
1800
1801
1802
1803 if (copied < write_bytes)
1804 nrptrs = 1;
1805
1806 if (copied == 0) {
1807 force_page_uptodate = true;
1808 dirty_sectors = 0;
1809 dirty_pages = 0;
1810 } else {
1811 force_page_uptodate = false;
1812 dirty_pages = DIV_ROUND_UP(copied + offset,
1813 PAGE_SIZE);
1814 }
1815
1816 if (num_sectors > dirty_sectors) {
1817
1818 release_bytes -= dirty_sectors << fs_info->sectorsize_bits;
1819 if (only_release_metadata) {
1820 btrfs_delalloc_release_metadata(BTRFS_I(inode),
1821 release_bytes, true);
1822 } else {
1823 u64 __pos;
1824
1825 __pos = round_down(pos,
1826 fs_info->sectorsize) +
1827 (dirty_pages << PAGE_SHIFT);
1828 btrfs_delalloc_release_space(BTRFS_I(inode),
1829 data_reserved, __pos,
1830 release_bytes, true);
1831 }
1832 }
1833
1834 release_bytes = round_up(copied + sector_offset,
1835 fs_info->sectorsize);
1836
1837 ret = btrfs_dirty_pages(BTRFS_I(inode), pages,
1838 dirty_pages, pos, copied,
1839 &cached_state, only_release_metadata);
1840
1841
1842
1843
1844
1845
1846
1847
1848 if (extents_locked)
1849 unlock_extent_cached(&BTRFS_I(inode)->io_tree,
1850 lockstart, lockend, &cached_state);
1851 else
1852 free_extent_state(cached_state);
1853
1854 btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes);
1855 if (ret) {
1856 btrfs_drop_pages(fs_info, pages, num_pages, pos, copied);
1857 break;
1858 }
1859
1860 release_bytes = 0;
1861 if (only_release_metadata)
1862 btrfs_check_nocow_unlock(BTRFS_I(inode));
1863
1864 btrfs_drop_pages(fs_info, pages, num_pages, pos, copied);
1865
1866 cond_resched();
1867
1868 balance_dirty_pages_ratelimited(inode->i_mapping);
1869
1870 pos += copied;
1871 num_written += copied;
1872 }
1873
1874 kfree(pages);
1875
1876 if (release_bytes) {
1877 if (only_release_metadata) {
1878 btrfs_check_nocow_unlock(BTRFS_I(inode));
1879 btrfs_delalloc_release_metadata(BTRFS_I(inode),
1880 release_bytes, true);
1881 } else {
1882 btrfs_delalloc_release_space(BTRFS_I(inode),
1883 data_reserved,
1884 round_down(pos, fs_info->sectorsize),
1885 release_bytes, true);
1886 }
1887 }
1888
1889 extent_changeset_free(data_reserved);
1890 if (num_written > 0) {
1891 pagecache_isize_extended(inode, old_isize, iocb->ki_pos);
1892 iocb->ki_pos += num_written;
1893 }
1894out:
1895 btrfs_inode_unlock(inode, ilock_flags);
1896 return num_written ? num_written : ret;
1897}
1898
1899static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
1900 const struct iov_iter *iter, loff_t offset)
1901{
1902 const u32 blocksize_mask = fs_info->sectorsize - 1;
1903
1904 if (offset & blocksize_mask)
1905 return -EINVAL;
1906
1907 if (iov_iter_alignment(iter) & blocksize_mask)
1908 return -EINVAL;
1909
1910 return 0;
1911}
1912
1913static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
1914{
1915 const bool is_sync_write = (iocb->ki_flags & IOCB_DSYNC);
1916 struct file *file = iocb->ki_filp;
1917 struct inode *inode = file_inode(file);
1918 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1919 loff_t pos;
1920 ssize_t written = 0;
1921 ssize_t written_buffered;
1922 size_t prev_left = 0;
1923 loff_t endbyte;
1924 ssize_t err;
1925 unsigned int ilock_flags = 0;
1926
1927 if (iocb->ki_flags & IOCB_NOWAIT)
1928 ilock_flags |= BTRFS_ILOCK_TRY;
1929
1930
1931 if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode))
1932 ilock_flags |= BTRFS_ILOCK_SHARED;
1933
1934relock:
1935 err = btrfs_inode_lock(inode, ilock_flags);
1936 if (err < 0)
1937 return err;
1938
1939 err = generic_write_checks(iocb, from);
1940 if (err <= 0) {
1941 btrfs_inode_unlock(inode, ilock_flags);
1942 return err;
1943 }
1944
1945 err = btrfs_write_check(iocb, from, err);
1946 if (err < 0) {
1947 btrfs_inode_unlock(inode, ilock_flags);
1948 goto out;
1949 }
1950
1951 pos = iocb->ki_pos;
1952
1953
1954
1955
1956 if ((ilock_flags & BTRFS_ILOCK_SHARED) &&
1957 pos + iov_iter_count(from) > i_size_read(inode)) {
1958 btrfs_inode_unlock(inode, ilock_flags);
1959 ilock_flags &= ~BTRFS_ILOCK_SHARED;
1960 goto relock;
1961 }
1962
1963 if (check_direct_IO(fs_info, from, pos)) {
1964 btrfs_inode_unlock(inode, ilock_flags);
1965 goto buffered;
1966 }
1967
1968
1969
1970
1971
1972
1973
1974 if (is_sync_write)
1975 iocb->ki_flags &= ~IOCB_DSYNC;
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995again:
1996 from->nofault = true;
1997 err = iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
1998 IOMAP_DIO_PARTIAL, written);
1999 from->nofault = false;
2000
2001
2002 if (err > 0)
2003 written = err;
2004
2005 if (iov_iter_count(from) > 0 && (err == -EFAULT || err > 0)) {
2006 const size_t left = iov_iter_count(from);
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021 if (left == prev_left) {
2022 err = -ENOTBLK;
2023 } else {
2024 fault_in_iov_iter_readable(from, left);
2025 prev_left = left;
2026 goto again;
2027 }
2028 }
2029
2030 btrfs_inode_unlock(inode, ilock_flags);
2031
2032
2033
2034
2035
2036 if (is_sync_write)
2037 iocb->ki_flags |= IOCB_DSYNC;
2038
2039
2040 if ((err < 0 && err != -ENOTBLK) || !iov_iter_count(from))
2041 goto out;
2042
2043buffered:
2044 pos = iocb->ki_pos;
2045 written_buffered = btrfs_buffered_write(iocb, from);
2046 if (written_buffered < 0) {
2047 err = written_buffered;
2048 goto out;
2049 }
2050
2051
2052
2053
2054 endbyte = pos + written_buffered - 1;
2055 err = btrfs_fdatawrite_range(inode, pos, endbyte);
2056 if (err)
2057 goto out;
2058 err = filemap_fdatawait_range(inode->i_mapping, pos, endbyte);
2059 if (err)
2060 goto out;
2061 written += written_buffered;
2062 iocb->ki_pos = pos + written_buffered;
2063 invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT,
2064 endbyte >> PAGE_SHIFT);
2065out:
2066 return err < 0 ? err : written;
2067}
2068
2069static ssize_t btrfs_file_write_iter(struct kiocb *iocb,
2070 struct iov_iter *from)
2071{
2072 struct file *file = iocb->ki_filp;
2073 struct btrfs_inode *inode = BTRFS_I(file_inode(file));
2074 ssize_t num_written = 0;
2075 const bool sync = iocb->ki_flags & IOCB_DSYNC;
2076
2077
2078
2079
2080
2081
2082 if (BTRFS_FS_ERROR(inode->root->fs_info))
2083 return -EROFS;
2084
2085 if (!(iocb->ki_flags & IOCB_DIRECT) &&
2086 (iocb->ki_flags & IOCB_NOWAIT))
2087 return -EOPNOTSUPP;
2088
2089 if (sync)
2090 atomic_inc(&inode->sync_writers);
2091
2092 if (iocb->ki_flags & IOCB_DIRECT)
2093 num_written = btrfs_direct_write(iocb, from);
2094 else
2095 num_written = btrfs_buffered_write(iocb, from);
2096
2097 btrfs_set_inode_last_sub_trans(inode);
2098
2099 if (num_written > 0)
2100 num_written = generic_write_sync(iocb, num_written);
2101
2102 if (sync)
2103 atomic_dec(&inode->sync_writers);
2104
2105 current->backing_dev_info = NULL;
2106 return num_written;
2107}
2108
2109int btrfs_release_file(struct inode *inode, struct file *filp)
2110{
2111 struct btrfs_file_private *private = filp->private_data;
2112
2113 if (private && private->filldir_buf)
2114 kfree(private->filldir_buf);
2115 kfree(private);
2116 filp->private_data = NULL;
2117
2118
2119
2120
2121
2122
2123
2124 if (test_and_clear_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
2125 &BTRFS_I(inode)->runtime_flags))
2126 filemap_flush(inode->i_mapping);
2127 return 0;
2128}
2129
2130static int start_ordered_ops(struct inode *inode, loff_t start, loff_t end)
2131{
2132 int ret;
2133 struct blk_plug plug;
2134
2135
2136
2137
2138
2139
2140
2141 blk_start_plug(&plug);
2142 atomic_inc(&BTRFS_I(inode)->sync_writers);
2143 ret = btrfs_fdatawrite_range(inode, start, end);
2144 atomic_dec(&BTRFS_I(inode)->sync_writers);
2145 blk_finish_plug(&plug);
2146
2147 return ret;
2148}
2149
2150static inline bool skip_inode_logging(const struct btrfs_log_ctx *ctx)
2151{
2152 struct btrfs_inode *inode = BTRFS_I(ctx->inode);
2153 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2154
2155 if (btrfs_inode_in_log(inode, fs_info->generation) &&
2156 list_empty(&ctx->ordered_extents))
2157 return true;
2158
2159
2160
2161
2162
2163
2164
2165
2166 if (inode->last_trans <= fs_info->last_trans_committed &&
2167 (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) ||
2168 list_empty(&ctx->ordered_extents)))
2169 return true;
2170
2171 return false;
2172}
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
2186{
2187 struct dentry *dentry = file_dentry(file);
2188 struct inode *inode = d_inode(dentry);
2189 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2190 struct btrfs_root *root = BTRFS_I(inode)->root;
2191 struct btrfs_trans_handle *trans;
2192 struct btrfs_log_ctx ctx;
2193 int ret = 0, err;
2194 u64 len;
2195 bool full_sync;
2196
2197 trace_btrfs_sync_file(file, datasync);
2198
2199 btrfs_init_log_ctx(&ctx, inode);
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209 start = 0;
2210 end = LLONG_MAX;
2211 len = (u64)LLONG_MAX + 1;
2212
2213
2214
2215
2216
2217
2218
2219 ret = start_ordered_ops(inode, start, end);
2220 if (ret)
2221 goto out;
2222
2223 btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
2224
2225 atomic_inc(&root->log_batch);
2226
2227
2228
2229
2230
2231
2232 full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2233 &BTRFS_I(inode)->runtime_flags);
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253 ret = start_ordered_ops(inode, start, end);
2254 if (ret) {
2255 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2256 goto out;
2257 }
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274 if (full_sync || btrfs_is_zoned(fs_info)) {
2275 ret = btrfs_wait_ordered_range(inode, start, len);
2276 } else {
2277
2278
2279
2280
2281
2282 btrfs_get_ordered_extents_for_logging(BTRFS_I(inode),
2283 &ctx.ordered_extents);
2284 ret = filemap_fdatawait_range(inode->i_mapping, start, end);
2285 }
2286
2287 if (ret)
2288 goto out_release_extents;
2289
2290 atomic_inc(&root->log_batch);
2291
2292 smp_mb();
2293 if (skip_inode_logging(&ctx)) {
2294
2295
2296
2297
2298
2299 clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2300 &BTRFS_I(inode)->runtime_flags);
2301
2302
2303
2304
2305
2306
2307
2308 ret = filemap_check_wb_err(inode->i_mapping, file->f_wb_err);
2309 goto out_release_extents;
2310 }
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323 trans = btrfs_start_transaction(root, 0);
2324 if (IS_ERR(trans)) {
2325 ret = PTR_ERR(trans);
2326 goto out_release_extents;
2327 }
2328 trans->in_fsync = true;
2329
2330 ret = btrfs_log_dentry_safe(trans, dentry, &ctx);
2331 btrfs_release_log_ctx_extents(&ctx);
2332 if (ret < 0) {
2333
2334 ret = 1;
2335 }
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2348
2349 if (ret != BTRFS_NO_LOG_SYNC) {
2350 if (!ret) {
2351 ret = btrfs_sync_log(trans, root, &ctx);
2352 if (!ret) {
2353 ret = btrfs_end_transaction(trans);
2354 goto out;
2355 }
2356 }
2357 if (!full_sync) {
2358 ret = btrfs_wait_ordered_range(inode, start, len);
2359 if (ret) {
2360 btrfs_end_transaction(trans);
2361 goto out;
2362 }
2363 }
2364 ret = btrfs_commit_transaction(trans);
2365 } else {
2366 ret = btrfs_end_transaction(trans);
2367 }
2368out:
2369 ASSERT(list_empty(&ctx.list));
2370 err = file_check_and_advance_wb_err(file);
2371 if (!ret)
2372 ret = err;
2373 return ret > 0 ? -EIO : ret;
2374
2375out_release_extents:
2376 btrfs_release_log_ctx_extents(&ctx);
2377 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2378 goto out;
2379}
2380
2381static const struct vm_operations_struct btrfs_file_vm_ops = {
2382 .fault = filemap_fault,
2383 .map_pages = filemap_map_pages,
2384 .page_mkwrite = btrfs_page_mkwrite,
2385};
2386
2387static int btrfs_file_mmap(struct file *filp, struct vm_area_struct *vma)
2388{
2389 struct address_space *mapping = filp->f_mapping;
2390
2391 if (!mapping->a_ops->readpage)
2392 return -ENOEXEC;
2393
2394 file_accessed(filp);
2395 vma->vm_ops = &btrfs_file_vm_ops;
2396
2397 return 0;
2398}
2399
2400static int hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
2401 int slot, u64 start, u64 end)
2402{
2403 struct btrfs_file_extent_item *fi;
2404 struct btrfs_key key;
2405
2406 if (slot < 0 || slot >= btrfs_header_nritems(leaf))
2407 return 0;
2408
2409 btrfs_item_key_to_cpu(leaf, &key, slot);
2410 if (key.objectid != btrfs_ino(inode) ||
2411 key.type != BTRFS_EXTENT_DATA_KEY)
2412 return 0;
2413
2414 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2415
2416 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2417 return 0;
2418
2419 if (btrfs_file_extent_disk_bytenr(leaf, fi))
2420 return 0;
2421
2422 if (key.offset == end)
2423 return 1;
2424 if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
2425 return 1;
2426 return 0;
2427}
2428
2429static int fill_holes(struct btrfs_trans_handle *trans,
2430 struct btrfs_inode *inode,
2431 struct btrfs_path *path, u64 offset, u64 end)
2432{
2433 struct btrfs_fs_info *fs_info = trans->fs_info;
2434 struct btrfs_root *root = inode->root;
2435 struct extent_buffer *leaf;
2436 struct btrfs_file_extent_item *fi;
2437 struct extent_map *hole_em;
2438 struct extent_map_tree *em_tree = &inode->extent_tree;
2439 struct btrfs_key key;
2440 int ret;
2441
2442 if (btrfs_fs_incompat(fs_info, NO_HOLES))
2443 goto out;
2444
2445 key.objectid = btrfs_ino(inode);
2446 key.type = BTRFS_EXTENT_DATA_KEY;
2447 key.offset = offset;
2448
2449 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2450 if (ret <= 0) {
2451
2452
2453
2454
2455 if (ret == 0)
2456 ret = -EINVAL;
2457 return ret;
2458 }
2459
2460 leaf = path->nodes[0];
2461 if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
2462 u64 num_bytes;
2463
2464 path->slots[0]--;
2465 fi = btrfs_item_ptr(leaf, path->slots[0],
2466 struct btrfs_file_extent_item);
2467 num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
2468 end - offset;
2469 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2470 btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2471 btrfs_set_file_extent_offset(leaf, fi, 0);
2472 btrfs_mark_buffer_dirty(leaf);
2473 goto out;
2474 }
2475
2476 if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) {
2477 u64 num_bytes;
2478
2479 key.offset = offset;
2480 btrfs_set_item_key_safe(fs_info, path, &key);
2481 fi = btrfs_item_ptr(leaf, path->slots[0],
2482 struct btrfs_file_extent_item);
2483 num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end -
2484 offset;
2485 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2486 btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2487 btrfs_set_file_extent_offset(leaf, fi, 0);
2488 btrfs_mark_buffer_dirty(leaf);
2489 goto out;
2490 }
2491 btrfs_release_path(path);
2492
2493 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode),
2494 offset, 0, 0, end - offset, 0, end - offset, 0, 0, 0);
2495 if (ret)
2496 return ret;
2497
2498out:
2499 btrfs_release_path(path);
2500
2501 hole_em = alloc_extent_map();
2502 if (!hole_em) {
2503 btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2504 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
2505 } else {
2506 hole_em->start = offset;
2507 hole_em->len = end - offset;
2508 hole_em->ram_bytes = hole_em->len;
2509 hole_em->orig_start = offset;
2510
2511 hole_em->block_start = EXTENT_MAP_HOLE;
2512 hole_em->block_len = 0;
2513 hole_em->orig_block_len = 0;
2514 hole_em->compress_type = BTRFS_COMPRESS_NONE;
2515 hole_em->generation = trans->transid;
2516
2517 do {
2518 btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2519 write_lock(&em_tree->lock);
2520 ret = add_extent_mapping(em_tree, hole_em, 1);
2521 write_unlock(&em_tree->lock);
2522 } while (ret == -EEXIST);
2523 free_extent_map(hole_em);
2524 if (ret)
2525 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2526 &inode->runtime_flags);
2527 }
2528
2529 return 0;
2530}
2531
2532
2533
2534
2535
2536
2537
2538static int find_first_non_hole(struct btrfs_inode *inode, u64 *start, u64 *len)
2539{
2540 struct btrfs_fs_info *fs_info = inode->root->fs_info;
2541 struct extent_map *em;
2542 int ret = 0;
2543
2544 em = btrfs_get_extent(inode, NULL, 0,
2545 round_down(*start, fs_info->sectorsize),
2546 round_up(*len, fs_info->sectorsize));
2547 if (IS_ERR(em))
2548 return PTR_ERR(em);
2549
2550
2551 if (em->block_start == EXTENT_MAP_HOLE) {
2552 ret = 1;
2553 *len = em->start + em->len > *start + *len ?
2554 0 : *start + *len - em->start - em->len;
2555 *start = em->start + em->len;
2556 }
2557 free_extent_map(em);
2558 return ret;
2559}
2560
2561static int btrfs_punch_hole_lock_range(struct inode *inode,
2562 const u64 lockstart,
2563 const u64 lockend,
2564 struct extent_state **cached_state)
2565{
2566
2567
2568
2569
2570
2571
2572
2573
2574 const u64 page_lockstart = round_up(lockstart, PAGE_SIZE);
2575 const u64 page_lockend = round_down(lockend + 1, PAGE_SIZE) - 1;
2576
2577 while (1) {
2578 struct btrfs_ordered_extent *ordered;
2579 int ret;
2580
2581 truncate_pagecache_range(inode, lockstart, lockend);
2582
2583 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2584 cached_state);
2585 ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode),
2586 lockend);
2587
2588
2589
2590
2591
2592
2593 if ((!ordered ||
2594 (ordered->file_offset + ordered->num_bytes <= lockstart ||
2595 ordered->file_offset > lockend)) &&
2596 !filemap_range_has_page(inode->i_mapping,
2597 page_lockstart, page_lockend)) {
2598 if (ordered)
2599 btrfs_put_ordered_extent(ordered);
2600 break;
2601 }
2602 if (ordered)
2603 btrfs_put_ordered_extent(ordered);
2604 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
2605 lockend, cached_state);
2606 ret = btrfs_wait_ordered_range(inode, lockstart,
2607 lockend - lockstart + 1);
2608 if (ret)
2609 return ret;
2610 }
2611 return 0;
2612}
2613
2614static int btrfs_insert_replace_extent(struct btrfs_trans_handle *trans,
2615 struct btrfs_inode *inode,
2616 struct btrfs_path *path,
2617 struct btrfs_replace_extent_info *extent_info,
2618 const u64 replace_len,
2619 const u64 bytes_to_drop)
2620{
2621 struct btrfs_fs_info *fs_info = trans->fs_info;
2622 struct btrfs_root *root = inode->root;
2623 struct btrfs_file_extent_item *extent;
2624 struct extent_buffer *leaf;
2625 struct btrfs_key key;
2626 int slot;
2627 struct btrfs_ref ref = { 0 };
2628 int ret;
2629
2630 if (replace_len == 0)
2631 return 0;
2632
2633 if (extent_info->disk_offset == 0 &&
2634 btrfs_fs_incompat(fs_info, NO_HOLES)) {
2635 btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
2636 return 0;
2637 }
2638
2639 key.objectid = btrfs_ino(inode);
2640 key.type = BTRFS_EXTENT_DATA_KEY;
2641 key.offset = extent_info->file_offset;
2642 ret = btrfs_insert_empty_item(trans, root, path, &key,
2643 sizeof(struct btrfs_file_extent_item));
2644 if (ret)
2645 return ret;
2646 leaf = path->nodes[0];
2647 slot = path->slots[0];
2648 write_extent_buffer(leaf, extent_info->extent_buf,
2649 btrfs_item_ptr_offset(leaf, slot),
2650 sizeof(struct btrfs_file_extent_item));
2651 extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2652 ASSERT(btrfs_file_extent_type(leaf, extent) != BTRFS_FILE_EXTENT_INLINE);
2653 btrfs_set_file_extent_offset(leaf, extent, extent_info->data_offset);
2654 btrfs_set_file_extent_num_bytes(leaf, extent, replace_len);
2655 if (extent_info->is_new_extent)
2656 btrfs_set_file_extent_generation(leaf, extent, trans->transid);
2657 btrfs_mark_buffer_dirty(leaf);
2658 btrfs_release_path(path);
2659
2660 ret = btrfs_inode_set_file_extent_range(inode, extent_info->file_offset,
2661 replace_len);
2662 if (ret)
2663 return ret;
2664
2665
2666 if (extent_info->disk_offset == 0) {
2667 btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
2668 return 0;
2669 }
2670
2671 btrfs_update_inode_bytes(inode, replace_len, bytes_to_drop);
2672
2673 if (extent_info->is_new_extent && extent_info->insertions == 0) {
2674 key.objectid = extent_info->disk_offset;
2675 key.type = BTRFS_EXTENT_ITEM_KEY;
2676 key.offset = extent_info->disk_len;
2677 ret = btrfs_alloc_reserved_file_extent(trans, root,
2678 btrfs_ino(inode),
2679 extent_info->file_offset,
2680 extent_info->qgroup_reserved,
2681 &key);
2682 } else {
2683 u64 ref_offset;
2684
2685 btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF,
2686 extent_info->disk_offset,
2687 extent_info->disk_len, 0);
2688 ref_offset = extent_info->file_offset - extent_info->data_offset;
2689 btrfs_init_data_ref(&ref, root->root_key.objectid,
2690 btrfs_ino(inode), ref_offset, 0, false);
2691 ret = btrfs_inc_extent_ref(trans, &ref);
2692 }
2693
2694 extent_info->insertions++;
2695
2696 return ret;
2697}
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708int btrfs_replace_file_extents(struct btrfs_inode *inode,
2709 struct btrfs_path *path, const u64 start,
2710 const u64 end,
2711 struct btrfs_replace_extent_info *extent_info,
2712 struct btrfs_trans_handle **trans_out)
2713{
2714 struct btrfs_drop_extents_args drop_args = { 0 };
2715 struct btrfs_root *root = inode->root;
2716 struct btrfs_fs_info *fs_info = root->fs_info;
2717 u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1);
2718 u64 ino_size = round_up(inode->vfs_inode.i_size, fs_info->sectorsize);
2719 struct btrfs_trans_handle *trans = NULL;
2720 struct btrfs_block_rsv *rsv;
2721 unsigned int rsv_count;
2722 u64 cur_offset;
2723 u64 len = end - start;
2724 int ret = 0;
2725
2726 if (end <= start)
2727 return -EINVAL;
2728
2729 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
2730 if (!rsv) {
2731 ret = -ENOMEM;
2732 goto out;
2733 }
2734 rsv->size = btrfs_calc_insert_metadata_size(fs_info, 1);
2735 rsv->failfast = 1;
2736
2737
2738
2739
2740
2741
2742
2743 if (!btrfs_fs_incompat(fs_info, NO_HOLES) || extent_info)
2744 rsv_count = 3;
2745 else
2746 rsv_count = 2;
2747
2748 trans = btrfs_start_transaction(root, rsv_count);
2749 if (IS_ERR(trans)) {
2750 ret = PTR_ERR(trans);
2751 trans = NULL;
2752 goto out_free;
2753 }
2754
2755 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
2756 min_size, false);
2757 BUG_ON(ret);
2758 trans->block_rsv = rsv;
2759
2760 cur_offset = start;
2761 drop_args.path = path;
2762 drop_args.end = end + 1;
2763 drop_args.drop_cache = true;
2764 while (cur_offset < end) {
2765 drop_args.start = cur_offset;
2766 ret = btrfs_drop_extents(trans, root, inode, &drop_args);
2767
2768 if (!extent_info)
2769 btrfs_update_inode_bytes(inode, 0,
2770 drop_args.bytes_found);
2771 if (ret != -ENOSPC) {
2772
2773
2774
2775
2776
2777
2778
2779
2780 if (ret &&
2781 (ret != -EOPNOTSUPP ||
2782 (extent_info && extent_info->is_new_extent)))
2783 btrfs_abort_transaction(trans, ret);
2784 break;
2785 }
2786
2787 trans->block_rsv = &fs_info->trans_block_rsv;
2788
2789 if (!extent_info && cur_offset < drop_args.drop_end &&
2790 cur_offset < ino_size) {
2791 ret = fill_holes(trans, inode, path, cur_offset,
2792 drop_args.drop_end);
2793 if (ret) {
2794
2795
2796
2797
2798
2799
2800 btrfs_abort_transaction(trans, ret);
2801 break;
2802 }
2803 } else if (!extent_info && cur_offset < drop_args.drop_end) {
2804
2805
2806
2807
2808
2809
2810 ret = btrfs_inode_clear_file_extent_range(inode,
2811 cur_offset,
2812 drop_args.drop_end - cur_offset);
2813 if (ret) {
2814
2815
2816
2817
2818
2819 btrfs_abort_transaction(trans, ret);
2820 break;
2821 }
2822 }
2823
2824 if (extent_info &&
2825 drop_args.drop_end > extent_info->file_offset) {
2826 u64 replace_len = drop_args.drop_end -
2827 extent_info->file_offset;
2828
2829 ret = btrfs_insert_replace_extent(trans, inode, path,
2830 extent_info, replace_len,
2831 drop_args.bytes_found);
2832 if (ret) {
2833 btrfs_abort_transaction(trans, ret);
2834 break;
2835 }
2836 extent_info->data_len -= replace_len;
2837 extent_info->data_offset += replace_len;
2838 extent_info->file_offset += replace_len;
2839 }
2840
2841 ret = btrfs_update_inode(trans, root, inode);
2842 if (ret)
2843 break;
2844
2845 btrfs_end_transaction(trans);
2846 btrfs_btree_balance_dirty(fs_info);
2847
2848 trans = btrfs_start_transaction(root, rsv_count);
2849 if (IS_ERR(trans)) {
2850 ret = PTR_ERR(trans);
2851 trans = NULL;
2852 break;
2853 }
2854
2855 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
2856 rsv, min_size, false);
2857 BUG_ON(ret);
2858 trans->block_rsv = rsv;
2859
2860 cur_offset = drop_args.drop_end;
2861 len = end - cur_offset;
2862 if (!extent_info && len) {
2863 ret = find_first_non_hole(inode, &cur_offset, &len);
2864 if (unlikely(ret < 0))
2865 break;
2866 if (ret && !len) {
2867 ret = 0;
2868 break;
2869 }
2870 }
2871 }
2872
2873
2874
2875
2876
2877
2878
2879 if (extent_info && !extent_info->is_new_extent)
2880 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
2881
2882 if (ret)
2883 goto out_trans;
2884
2885 trans->block_rsv = &fs_info->trans_block_rsv;
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897 if (drop_args.drop_end <= end)
2898 drop_args.drop_end = end + 1;
2899
2900
2901
2902
2903
2904 if (!extent_info && cur_offset < ino_size &&
2905 cur_offset < drop_args.drop_end) {
2906 ret = fill_holes(trans, inode, path, cur_offset,
2907 drop_args.drop_end);
2908 if (ret) {
2909
2910 btrfs_abort_transaction(trans, ret);
2911 goto out_trans;
2912 }
2913 } else if (!extent_info && cur_offset < drop_args.drop_end) {
2914
2915 ret = btrfs_inode_clear_file_extent_range(inode, cur_offset,
2916 drop_args.drop_end - cur_offset);
2917 if (ret) {
2918 btrfs_abort_transaction(trans, ret);
2919 goto out_trans;
2920 }
2921
2922 }
2923 if (extent_info) {
2924 ret = btrfs_insert_replace_extent(trans, inode, path,
2925 extent_info, extent_info->data_len,
2926 drop_args.bytes_found);
2927 if (ret) {
2928 btrfs_abort_transaction(trans, ret);
2929 goto out_trans;
2930 }
2931 }
2932
2933out_trans:
2934 if (!trans)
2935 goto out_free;
2936
2937 trans->block_rsv = &fs_info->trans_block_rsv;
2938 if (ret)
2939 btrfs_end_transaction(trans);
2940 else
2941 *trans_out = trans;
2942out_free:
2943 btrfs_free_block_rsv(fs_info, rsv);
2944out:
2945 return ret;
2946}
2947
2948static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
2949{
2950 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2951 struct btrfs_root *root = BTRFS_I(inode)->root;
2952 struct extent_state *cached_state = NULL;
2953 struct btrfs_path *path;
2954 struct btrfs_trans_handle *trans = NULL;
2955 u64 lockstart;
2956 u64 lockend;
2957 u64 tail_start;
2958 u64 tail_len;
2959 u64 orig_start = offset;
2960 int ret = 0;
2961 bool same_block;
2962 u64 ino_size;
2963 bool truncated_block = false;
2964 bool updated_inode = false;
2965
2966 ret = btrfs_wait_ordered_range(inode, offset, len);
2967 if (ret)
2968 return ret;
2969
2970 btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
2971 ino_size = round_up(inode->i_size, fs_info->sectorsize);
2972 ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
2973 if (ret < 0)
2974 goto out_only_mutex;
2975 if (ret && !len) {
2976
2977 ret = 0;
2978 goto out_only_mutex;
2979 }
2980
2981 lockstart = round_up(offset, btrfs_inode_sectorsize(BTRFS_I(inode)));
2982 lockend = round_down(offset + len,
2983 btrfs_inode_sectorsize(BTRFS_I(inode))) - 1;
2984 same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset))
2985 == (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1));
2986
2987
2988
2989
2990
2991
2992
2993
2994 if (same_block && len < fs_info->sectorsize) {
2995 if (offset < ino_size) {
2996 truncated_block = true;
2997 ret = btrfs_truncate_block(BTRFS_I(inode), offset, len,
2998 0);
2999 } else {
3000 ret = 0;
3001 }
3002 goto out_only_mutex;
3003 }
3004
3005
3006 if (offset < ino_size) {
3007 truncated_block = true;
3008 ret = btrfs_truncate_block(BTRFS_I(inode), offset, 0, 0);
3009 if (ret) {
3010 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3011 return ret;
3012 }
3013 }
3014
3015
3016
3017
3018
3019 if (offset == orig_start) {
3020
3021 len = offset + len - lockstart;
3022 offset = lockstart;
3023 ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
3024 if (ret < 0)
3025 goto out_only_mutex;
3026 if (ret && !len) {
3027 ret = 0;
3028 goto out_only_mutex;
3029 }
3030 lockstart = offset;
3031 }
3032
3033
3034 tail_start = lockend + 1;
3035 tail_len = offset + len - tail_start;
3036 if (tail_len) {
3037 ret = find_first_non_hole(BTRFS_I(inode), &tail_start, &tail_len);
3038 if (unlikely(ret < 0))
3039 goto out_only_mutex;
3040 if (!ret) {
3041
3042 if (tail_start + tail_len < ino_size) {
3043 truncated_block = true;
3044 ret = btrfs_truncate_block(BTRFS_I(inode),
3045 tail_start + tail_len,
3046 0, 1);
3047 if (ret)
3048 goto out_only_mutex;
3049 }
3050 }
3051 }
3052
3053 if (lockend < lockstart) {
3054 ret = 0;
3055 goto out_only_mutex;
3056 }
3057
3058 ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3059 &cached_state);
3060 if (ret)
3061 goto out_only_mutex;
3062
3063 path = btrfs_alloc_path();
3064 if (!path) {
3065 ret = -ENOMEM;
3066 goto out;
3067 }
3068
3069 ret = btrfs_replace_file_extents(BTRFS_I(inode), path, lockstart,
3070 lockend, NULL, &trans);
3071 btrfs_free_path(path);
3072 if (ret)
3073 goto out;
3074
3075 ASSERT(trans != NULL);
3076 inode_inc_iversion(inode);
3077 inode->i_mtime = inode->i_ctime = current_time(inode);
3078 ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3079 updated_inode = true;
3080 btrfs_end_transaction(trans);
3081 btrfs_btree_balance_dirty(fs_info);
3082out:
3083 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
3084 &cached_state);
3085out_only_mutex:
3086 if (!updated_inode && truncated_block && !ret) {
3087
3088
3089
3090
3091
3092
3093
3094 struct timespec64 now = current_time(inode);
3095
3096 inode_inc_iversion(inode);
3097 inode->i_mtime = now;
3098 inode->i_ctime = now;
3099 trans = btrfs_start_transaction(root, 1);
3100 if (IS_ERR(trans)) {
3101 ret = PTR_ERR(trans);
3102 } else {
3103 int ret2;
3104
3105 ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3106 ret2 = btrfs_end_transaction(trans);
3107 if (!ret)
3108 ret = ret2;
3109 }
3110 }
3111 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3112 return ret;
3113}
3114
3115
3116struct falloc_range {
3117 struct list_head list;
3118 u64 start;
3119 u64 len;
3120};
3121
3122
3123
3124
3125
3126
3127
3128static int add_falloc_range(struct list_head *head, u64 start, u64 len)
3129{
3130 struct falloc_range *range = NULL;
3131
3132 if (!list_empty(head)) {
3133
3134
3135
3136
3137 range = list_last_entry(head, struct falloc_range, list);
3138 if (range->start + range->len == start) {
3139 range->len += len;
3140 return 0;
3141 }
3142 }
3143
3144 range = kmalloc(sizeof(*range), GFP_KERNEL);
3145 if (!range)
3146 return -ENOMEM;
3147 range->start = start;
3148 range->len = len;
3149 list_add_tail(&range->list, head);
3150 return 0;
3151}
3152
3153static int btrfs_fallocate_update_isize(struct inode *inode,
3154 const u64 end,
3155 const int mode)
3156{
3157 struct btrfs_trans_handle *trans;
3158 struct btrfs_root *root = BTRFS_I(inode)->root;
3159 int ret;
3160 int ret2;
3161
3162 if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode))
3163 return 0;
3164
3165 trans = btrfs_start_transaction(root, 1);
3166 if (IS_ERR(trans))
3167 return PTR_ERR(trans);
3168
3169 inode->i_ctime = current_time(inode);
3170 i_size_write(inode, end);
3171 btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
3172 ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3173 ret2 = btrfs_end_transaction(trans);
3174
3175 return ret ? ret : ret2;
3176}
3177
3178enum {
3179 RANGE_BOUNDARY_WRITTEN_EXTENT,
3180 RANGE_BOUNDARY_PREALLOC_EXTENT,
3181 RANGE_BOUNDARY_HOLE,
3182};
3183
3184static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode,
3185 u64 offset)
3186{
3187 const u64 sectorsize = btrfs_inode_sectorsize(inode);
3188 struct extent_map *em;
3189 int ret;
3190
3191 offset = round_down(offset, sectorsize);
3192 em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize);
3193 if (IS_ERR(em))
3194 return PTR_ERR(em);
3195
3196 if (em->block_start == EXTENT_MAP_HOLE)
3197 ret = RANGE_BOUNDARY_HOLE;
3198 else if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
3199 ret = RANGE_BOUNDARY_PREALLOC_EXTENT;
3200 else
3201 ret = RANGE_BOUNDARY_WRITTEN_EXTENT;
3202
3203 free_extent_map(em);
3204 return ret;
3205}
3206
3207static int btrfs_zero_range(struct inode *inode,
3208 loff_t offset,
3209 loff_t len,
3210 const int mode)
3211{
3212 struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
3213 struct extent_map *em;
3214 struct extent_changeset *data_reserved = NULL;
3215 int ret;
3216 u64 alloc_hint = 0;
3217 const u64 sectorsize = btrfs_inode_sectorsize(BTRFS_I(inode));
3218 u64 alloc_start = round_down(offset, sectorsize);
3219 u64 alloc_end = round_up(offset + len, sectorsize);
3220 u64 bytes_to_reserve = 0;
3221 bool space_reserved = false;
3222
3223 inode_dio_wait(inode);
3224
3225 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, alloc_start,
3226 alloc_end - alloc_start);
3227 if (IS_ERR(em)) {
3228 ret = PTR_ERR(em);
3229 goto out;
3230 }
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240 if (em->start <= alloc_start &&
3241 test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3242 const u64 em_end = em->start + em->len;
3243
3244 if (em_end >= offset + len) {
3245
3246
3247
3248
3249
3250 free_extent_map(em);
3251 ret = btrfs_fallocate_update_isize(inode, offset + len,
3252 mode);
3253 goto out;
3254 }
3255
3256
3257
3258
3259 alloc_start = em_end;
3260 ASSERT(IS_ALIGNED(alloc_start, sectorsize));
3261 len = offset + len - alloc_start;
3262 offset = alloc_start;
3263 alloc_hint = em->block_start + em->len;
3264 }
3265 free_extent_map(em);
3266
3267 if (BTRFS_BYTES_TO_BLKS(fs_info, offset) ==
3268 BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)) {
3269 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, alloc_start,
3270 sectorsize);
3271 if (IS_ERR(em)) {
3272 ret = PTR_ERR(em);
3273 goto out;
3274 }
3275
3276 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3277 free_extent_map(em);
3278 ret = btrfs_fallocate_update_isize(inode, offset + len,
3279 mode);
3280 goto out;
3281 }
3282 if (len < sectorsize && em->block_start != EXTENT_MAP_HOLE) {
3283 free_extent_map(em);
3284 ret = btrfs_truncate_block(BTRFS_I(inode), offset, len,
3285 0);
3286 if (!ret)
3287 ret = btrfs_fallocate_update_isize(inode,
3288 offset + len,
3289 mode);
3290 return ret;
3291 }
3292 free_extent_map(em);
3293 alloc_start = round_down(offset, sectorsize);
3294 alloc_end = alloc_start + sectorsize;
3295 goto reserve_space;
3296 }
3297
3298 alloc_start = round_up(offset, sectorsize);
3299 alloc_end = round_down(offset + len, sectorsize);
3300
3301
3302
3303
3304
3305
3306
3307 if (!IS_ALIGNED(offset, sectorsize)) {
3308 ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
3309 offset);
3310 if (ret < 0)
3311 goto out;
3312 if (ret == RANGE_BOUNDARY_HOLE) {
3313 alloc_start = round_down(offset, sectorsize);
3314 ret = 0;
3315 } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3316 ret = btrfs_truncate_block(BTRFS_I(inode), offset, 0, 0);
3317 if (ret)
3318 goto out;
3319 } else {
3320 ret = 0;
3321 }
3322 }
3323
3324 if (!IS_ALIGNED(offset + len, sectorsize)) {
3325 ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
3326 offset + len);
3327 if (ret < 0)
3328 goto out;
3329 if (ret == RANGE_BOUNDARY_HOLE) {
3330 alloc_end = round_up(offset + len, sectorsize);
3331 ret = 0;
3332 } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3333 ret = btrfs_truncate_block(BTRFS_I(inode), offset + len,
3334 0, 1);
3335 if (ret)
3336 goto out;
3337 } else {
3338 ret = 0;
3339 }
3340 }
3341
3342reserve_space:
3343 if (alloc_start < alloc_end) {
3344 struct extent_state *cached_state = NULL;
3345 const u64 lockstart = alloc_start;
3346 const u64 lockend = alloc_end - 1;
3347
3348 bytes_to_reserve = alloc_end - alloc_start;
3349 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3350 bytes_to_reserve);
3351 if (ret < 0)
3352 goto out;
3353 space_reserved = true;
3354 ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3355 &cached_state);
3356 if (ret)
3357 goto out;
3358 ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved,
3359 alloc_start, bytes_to_reserve);
3360 if (ret) {
3361 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3362 lockend, &cached_state);
3363 goto out;
3364 }
3365 ret = btrfs_prealloc_file_range(inode, mode, alloc_start,
3366 alloc_end - alloc_start,
3367 i_blocksize(inode),
3368 offset + len, &alloc_hint);
3369 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3370 lockend, &cached_state);
3371
3372 if (ret) {
3373 space_reserved = false;
3374 goto out;
3375 }
3376 }
3377 ret = btrfs_fallocate_update_isize(inode, offset + len, mode);
3378 out:
3379 if (ret && space_reserved)
3380 btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved,
3381 alloc_start, bytes_to_reserve);
3382 extent_changeset_free(data_reserved);
3383
3384 return ret;
3385}
3386
3387static long btrfs_fallocate(struct file *file, int mode,
3388 loff_t offset, loff_t len)
3389{
3390 struct inode *inode = file_inode(file);
3391 struct extent_state *cached_state = NULL;
3392 struct extent_changeset *data_reserved = NULL;
3393 struct falloc_range *range;
3394 struct falloc_range *tmp;
3395 struct list_head reserve_list;
3396 u64 cur_offset;
3397 u64 last_byte;
3398 u64 alloc_start;
3399 u64 alloc_end;
3400 u64 alloc_hint = 0;
3401 u64 locked_end;
3402 u64 actual_end = 0;
3403 struct extent_map *em;
3404 int blocksize = btrfs_inode_sectorsize(BTRFS_I(inode));
3405 int ret;
3406
3407
3408 if (btrfs_is_zoned(btrfs_sb(inode->i_sb)))
3409 return -EOPNOTSUPP;
3410
3411 alloc_start = round_down(offset, blocksize);
3412 alloc_end = round_up(offset + len, blocksize);
3413 cur_offset = alloc_start;
3414
3415
3416 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
3417 FALLOC_FL_ZERO_RANGE))
3418 return -EOPNOTSUPP;
3419
3420 if (mode & FALLOC_FL_PUNCH_HOLE)
3421 return btrfs_punch_hole(inode, offset, len);
3422
3423
3424
3425
3426
3427
3428 if (!(mode & FALLOC_FL_ZERO_RANGE)) {
3429 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3430 alloc_end - alloc_start);
3431 if (ret < 0)
3432 return ret;
3433 }
3434
3435 btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
3436
3437 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) {
3438 ret = inode_newsize_ok(inode, offset + len);
3439 if (ret)
3440 goto out;
3441 }
3442
3443
3444
3445
3446
3447
3448
3449
3450 if (alloc_start > inode->i_size) {
3451 ret = btrfs_cont_expand(BTRFS_I(inode), i_size_read(inode),
3452 alloc_start);
3453 if (ret)
3454 goto out;
3455 } else if (offset + len > inode->i_size) {
3456
3457
3458
3459
3460
3461 ret = btrfs_truncate_block(BTRFS_I(inode), inode->i_size, 0, 0);
3462 if (ret)
3463 goto out;
3464 }
3465
3466
3467
3468
3469
3470 ret = btrfs_wait_ordered_range(inode, alloc_start,
3471 alloc_end - alloc_start);
3472 if (ret)
3473 goto out;
3474
3475 if (mode & FALLOC_FL_ZERO_RANGE) {
3476 ret = btrfs_zero_range(inode, offset, len, mode);
3477 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3478 return ret;
3479 }
3480
3481 locked_end = alloc_end - 1;
3482 while (1) {
3483 struct btrfs_ordered_extent *ordered;
3484
3485
3486
3487
3488 lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
3489 locked_end, &cached_state);
3490 ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode),
3491 locked_end);
3492
3493 if (ordered &&
3494 ordered->file_offset + ordered->num_bytes > alloc_start &&
3495 ordered->file_offset < alloc_end) {
3496 btrfs_put_ordered_extent(ordered);
3497 unlock_extent_cached(&BTRFS_I(inode)->io_tree,
3498 alloc_start, locked_end,
3499 &cached_state);
3500
3501
3502
3503
3504 ret = btrfs_wait_ordered_range(inode, alloc_start,
3505 alloc_end - alloc_start);
3506 if (ret)
3507 goto out;
3508 } else {
3509 if (ordered)
3510 btrfs_put_ordered_extent(ordered);
3511 break;
3512 }
3513 }
3514
3515
3516 INIT_LIST_HEAD(&reserve_list);
3517 while (cur_offset < alloc_end) {
3518 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
3519 alloc_end - cur_offset);
3520 if (IS_ERR(em)) {
3521 ret = PTR_ERR(em);
3522 break;
3523 }
3524 last_byte = min(extent_map_end(em), alloc_end);
3525 actual_end = min_t(u64, extent_map_end(em), offset + len);
3526 last_byte = ALIGN(last_byte, blocksize);
3527 if (em->block_start == EXTENT_MAP_HOLE ||
3528 (cur_offset >= inode->i_size &&
3529 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
3530 ret = add_falloc_range(&reserve_list, cur_offset,
3531 last_byte - cur_offset);
3532 if (ret < 0) {
3533 free_extent_map(em);
3534 break;
3535 }
3536 ret = btrfs_qgroup_reserve_data(BTRFS_I(inode),
3537 &data_reserved, cur_offset,
3538 last_byte - cur_offset);
3539 if (ret < 0) {
3540 cur_offset = last_byte;
3541 free_extent_map(em);
3542 break;
3543 }
3544 } else {
3545
3546
3547
3548
3549
3550 btrfs_free_reserved_data_space(BTRFS_I(inode),
3551 data_reserved, cur_offset,
3552 last_byte - cur_offset);
3553 }
3554 free_extent_map(em);
3555 cur_offset = last_byte;
3556 }
3557
3558
3559
3560
3561
3562 list_for_each_entry_safe(range, tmp, &reserve_list, list) {
3563 if (!ret)
3564 ret = btrfs_prealloc_file_range(inode, mode,
3565 range->start,
3566 range->len, i_blocksize(inode),
3567 offset + len, &alloc_hint);
3568 else
3569 btrfs_free_reserved_data_space(BTRFS_I(inode),
3570 data_reserved, range->start,
3571 range->len);
3572 list_del(&range->list);
3573 kfree(range);
3574 }
3575 if (ret < 0)
3576 goto out_unlock;
3577
3578
3579
3580
3581
3582 ret = btrfs_fallocate_update_isize(inode, actual_end, mode);
3583out_unlock:
3584 unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
3585 &cached_state);
3586out:
3587 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3588
3589 if (ret != 0 && !(mode & FALLOC_FL_ZERO_RANGE))
3590 btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved,
3591 cur_offset, alloc_end - cur_offset);
3592 extent_changeset_free(data_reserved);
3593 return ret;
3594}
3595
3596static loff_t find_desired_extent(struct btrfs_inode *inode, loff_t offset,
3597 int whence)
3598{
3599 struct btrfs_fs_info *fs_info = inode->root->fs_info;
3600 struct extent_map *em = NULL;
3601 struct extent_state *cached_state = NULL;
3602 loff_t i_size = inode->vfs_inode.i_size;
3603 u64 lockstart;
3604 u64 lockend;
3605 u64 start;
3606 u64 len;
3607 int ret = 0;
3608
3609 if (i_size == 0 || offset >= i_size)
3610 return -ENXIO;
3611
3612
3613
3614
3615
3616 start = max_t(loff_t, 0, offset);
3617
3618 lockstart = round_down(start, fs_info->sectorsize);
3619 lockend = round_up(i_size, fs_info->sectorsize);
3620 if (lockend <= lockstart)
3621 lockend = lockstart + fs_info->sectorsize;
3622 lockend--;
3623 len = lockend - lockstart + 1;
3624
3625 lock_extent_bits(&inode->io_tree, lockstart, lockend, &cached_state);
3626
3627 while (start < i_size) {
3628 em = btrfs_get_extent_fiemap(inode, start, len);
3629 if (IS_ERR(em)) {
3630 ret = PTR_ERR(em);
3631 em = NULL;
3632 break;
3633 }
3634
3635 if (whence == SEEK_HOLE &&
3636 (em->block_start == EXTENT_MAP_HOLE ||
3637 test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3638 break;
3639 else if (whence == SEEK_DATA &&
3640 (em->block_start != EXTENT_MAP_HOLE &&
3641 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3642 break;
3643
3644 start = em->start + em->len;
3645 free_extent_map(em);
3646 em = NULL;
3647 cond_resched();
3648 }
3649 free_extent_map(em);
3650 unlock_extent_cached(&inode->io_tree, lockstart, lockend,
3651 &cached_state);
3652 if (ret) {
3653 offset = ret;
3654 } else {
3655 if (whence == SEEK_DATA && start >= i_size)
3656 offset = -ENXIO;
3657 else
3658 offset = min_t(loff_t, start, i_size);
3659 }
3660
3661 return offset;
3662}
3663
3664static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence)
3665{
3666 struct inode *inode = file->f_mapping->host;
3667
3668 switch (whence) {
3669 default:
3670 return generic_file_llseek(file, offset, whence);
3671 case SEEK_DATA:
3672 case SEEK_HOLE:
3673 btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
3674 offset = find_desired_extent(BTRFS_I(inode), offset, whence);
3675 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3676 break;
3677 }
3678
3679 if (offset < 0)
3680 return offset;
3681
3682 return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
3683}
3684
3685static int btrfs_file_open(struct inode *inode, struct file *filp)
3686{
3687 int ret;
3688
3689 filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC;
3690
3691 ret = fsverity_file_open(inode, filp);
3692 if (ret)
3693 return ret;
3694 return generic_file_open(inode, filp);
3695}
3696
3697static int check_direct_read(struct btrfs_fs_info *fs_info,
3698 const struct iov_iter *iter, loff_t offset)
3699{
3700 int ret;
3701 int i, seg;
3702
3703 ret = check_direct_IO(fs_info, iter, offset);
3704 if (ret < 0)
3705 return ret;
3706
3707 if (!iter_is_iovec(iter))
3708 return 0;
3709
3710 for (seg = 0; seg < iter->nr_segs; seg++)
3711 for (i = seg + 1; i < iter->nr_segs; i++)
3712 if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
3713 return -EINVAL;
3714 return 0;
3715}
3716
3717static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to)
3718{
3719 struct inode *inode = file_inode(iocb->ki_filp);
3720 size_t prev_left = 0;
3721 ssize_t read = 0;
3722 ssize_t ret;
3723
3724 if (fsverity_active(inode))
3725 return 0;
3726
3727 if (check_direct_read(btrfs_sb(inode->i_sb), to, iocb->ki_pos))
3728 return 0;
3729
3730 btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
3731again:
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747 pagefault_disable();
3748 to->nofault = true;
3749 ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
3750 IOMAP_DIO_PARTIAL, read);
3751 to->nofault = false;
3752 pagefault_enable();
3753
3754
3755 if (ret > 0)
3756 read = ret;
3757
3758 if (iov_iter_count(to) > 0 && (ret == -EFAULT || ret > 0)) {
3759 const size_t left = iov_iter_count(to);
3760
3761 if (left == prev_left) {
3762
3763
3764
3765
3766
3767
3768 ret = read;
3769 } else {
3770
3771
3772
3773
3774
3775 fault_in_iov_iter_writeable(to, left);
3776 prev_left = left;
3777 goto again;
3778 }
3779 }
3780 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3781 return ret < 0 ? ret : read;
3782}
3783
3784static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
3785{
3786 ssize_t ret = 0;
3787
3788 if (iocb->ki_flags & IOCB_DIRECT) {
3789 ret = btrfs_direct_read(iocb, to);
3790 if (ret < 0 || !iov_iter_count(to) ||
3791 iocb->ki_pos >= i_size_read(file_inode(iocb->ki_filp)))
3792 return ret;
3793 }
3794
3795 return filemap_read(iocb, to, ret);
3796}
3797
3798const struct file_operations btrfs_file_operations = {
3799 .llseek = btrfs_file_llseek,
3800 .read_iter = btrfs_file_read_iter,
3801 .splice_read = generic_file_splice_read,
3802 .write_iter = btrfs_file_write_iter,
3803 .splice_write = iter_file_splice_write,
3804 .mmap = btrfs_file_mmap,
3805 .open = btrfs_file_open,
3806 .release = btrfs_release_file,
3807 .fsync = btrfs_sync_file,
3808 .fallocate = btrfs_fallocate,
3809 .unlocked_ioctl = btrfs_ioctl,
3810#ifdef CONFIG_COMPAT
3811 .compat_ioctl = btrfs_compat_ioctl,
3812#endif
3813 .remap_file_range = btrfs_remap_file_range,
3814};
3815
3816void __cold btrfs_auto_defrag_exit(void)
3817{
3818 kmem_cache_destroy(btrfs_inode_defrag_cachep);
3819}
3820
3821int __init btrfs_auto_defrag_init(void)
3822{
3823 btrfs_inode_defrag_cachep = kmem_cache_create("btrfs_inode_defrag",
3824 sizeof(struct inode_defrag), 0,
3825 SLAB_MEM_SPREAD,
3826 NULL);
3827 if (!btrfs_inode_defrag_cachep)
3828 return -ENOMEM;
3829
3830 return 0;
3831}
3832
3833int btrfs_fdatawrite_range(struct inode *inode, loff_t start, loff_t end)
3834{
3835 int ret;
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851 ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3852 if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
3853 &BTRFS_I(inode)->runtime_flags))
3854 ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3855
3856 return ret;
3857}
3858