1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/fs.h>
17#include <linux/quotaops.h>
18#include <linux/slab.h>
19#include "ext4_jbd2.h"
20#include "ext4.h"
21#include "ext4_extents.h"
22
23
24
25
26
27
28
29
30
31
32
33static inline int
34get_ext_path(struct inode *inode, ext4_lblk_t lblock,
35 struct ext4_ext_path **orig_path)
36{
37 int ret = 0;
38 struct ext4_ext_path *path;
39
40 path = ext4_ext_find_extent(inode, lblock, *orig_path, EXT4_EX_NOCACHE);
41 if (IS_ERR(path))
42 ret = PTR_ERR(path);
43 else if (path[ext_depth(inode)].p_ext == NULL)
44 ret = -ENODATA;
45 else
46 *orig_path = path;
47
48 return ret;
49}
50
51
52
53
54
55
56
57static void
58copy_extent_status(struct ext4_extent *src, struct ext4_extent *dest)
59{
60 if (ext4_ext_is_uninitialized(src))
61 ext4_ext_mark_uninitialized(dest);
62 else
63 dest->ee_len = cpu_to_le16(ext4_ext_get_actual_len(dest));
64}
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79static int
80mext_next_extent(struct inode *inode, struct ext4_ext_path *path,
81 struct ext4_extent **extent)
82{
83 struct ext4_extent_header *eh;
84 int ppos, leaf_ppos = path->p_depth;
85
86 ppos = leaf_ppos;
87 if (EXT_LAST_EXTENT(path[ppos].p_hdr) > path[ppos].p_ext) {
88
89 *extent = ++path[ppos].p_ext;
90 path[ppos].p_block = ext4_ext_pblock(path[ppos].p_ext);
91 return 0;
92 }
93
94 while (--ppos >= 0) {
95 if (EXT_LAST_INDEX(path[ppos].p_hdr) >
96 path[ppos].p_idx) {
97 int cur_ppos = ppos;
98
99
100 path[ppos].p_idx++;
101 path[ppos].p_block = ext4_idx_pblock(path[ppos].p_idx);
102 if (path[ppos+1].p_bh)
103 brelse(path[ppos+1].p_bh);
104 path[ppos+1].p_bh =
105 sb_bread(inode->i_sb, path[ppos].p_block);
106 if (!path[ppos+1].p_bh)
107 return -EIO;
108 path[ppos+1].p_hdr =
109 ext_block_hdr(path[ppos+1].p_bh);
110
111
112 while (++cur_ppos < leaf_ppos) {
113 path[cur_ppos].p_idx =
114 EXT_FIRST_INDEX(path[cur_ppos].p_hdr);
115 path[cur_ppos].p_block =
116 ext4_idx_pblock(path[cur_ppos].p_idx);
117 if (path[cur_ppos+1].p_bh)
118 brelse(path[cur_ppos+1].p_bh);
119 path[cur_ppos+1].p_bh = sb_bread(inode->i_sb,
120 path[cur_ppos].p_block);
121 if (!path[cur_ppos+1].p_bh)
122 return -EIO;
123 path[cur_ppos+1].p_hdr =
124 ext_block_hdr(path[cur_ppos+1].p_bh);
125 }
126
127 path[leaf_ppos].p_ext = *extent = NULL;
128
129 eh = path[leaf_ppos].p_hdr;
130 if (le16_to_cpu(eh->eh_entries) == 0)
131
132 return -ENODATA;
133
134
135 path[leaf_ppos].p_ext = *extent =
136 EXT_FIRST_EXTENT(path[leaf_ppos].p_hdr);
137 path[leaf_ppos].p_block =
138 ext4_ext_pblock(path[leaf_ppos].p_ext);
139 return 0;
140 }
141 }
142
143 return 1;
144}
145
146
147
148
149
150
151
152void
153ext4_double_down_write_data_sem(struct inode *first, struct inode *second)
154{
155 if (first < second) {
156 down_write(&EXT4_I(first)->i_data_sem);
157 down_write_nested(&EXT4_I(second)->i_data_sem, SINGLE_DEPTH_NESTING);
158 } else {
159 down_write(&EXT4_I(second)->i_data_sem);
160 down_write_nested(&EXT4_I(first)->i_data_sem, SINGLE_DEPTH_NESTING);
161
162 }
163}
164
165
166
167
168
169
170
171
172void
173ext4_double_up_write_data_sem(struct inode *orig_inode,
174 struct inode *donor_inode)
175{
176 up_write(&EXT4_I(orig_inode)->i_data_sem);
177 up_write(&EXT4_I(donor_inode)->i_data_sem);
178}
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194static int
195mext_insert_across_blocks(handle_t *handle, struct inode *orig_inode,
196 struct ext4_extent *o_start, struct ext4_extent *o_end,
197 struct ext4_extent *start_ext, struct ext4_extent *new_ext,
198 struct ext4_extent *end_ext)
199{
200 struct ext4_ext_path *orig_path = NULL;
201 ext4_lblk_t eblock = 0;
202 int new_flag = 0;
203 int end_flag = 0;
204 int err = 0;
205
206 if (start_ext->ee_len && new_ext->ee_len && end_ext->ee_len) {
207 if (o_start == o_end) {
208
209
210
211
212
213 end_flag = 1;
214 } else {
215
216
217
218
219
220 o_end->ee_block = end_ext->ee_block;
221 o_end->ee_len = end_ext->ee_len;
222 ext4_ext_store_pblock(o_end, ext4_ext_pblock(end_ext));
223 }
224
225 o_start->ee_len = start_ext->ee_len;
226 eblock = le32_to_cpu(start_ext->ee_block);
227 new_flag = 1;
228
229 } else if (start_ext->ee_len && new_ext->ee_len &&
230 !end_ext->ee_len && o_start == o_end) {
231
232
233
234
235
236 o_start->ee_len = start_ext->ee_len;
237 eblock = le32_to_cpu(start_ext->ee_block);
238 new_flag = 1;
239
240 } else if (!start_ext->ee_len && new_ext->ee_len &&
241 end_ext->ee_len && o_start == o_end) {
242
243
244
245
246
247 o_end->ee_block = end_ext->ee_block;
248 o_end->ee_len = end_ext->ee_len;
249 ext4_ext_store_pblock(o_end, ext4_ext_pblock(end_ext));
250
251
252
253
254
255 if (new_ext->ee_block)
256 eblock = le32_to_cpu(new_ext->ee_block);
257
258 new_flag = 1;
259 } else {
260 ext4_debug("ext4 move extent: Unexpected insert case\n");
261 return -EIO;
262 }
263
264 if (new_flag) {
265 err = get_ext_path(orig_inode, eblock, &orig_path);
266 if (err)
267 goto out;
268
269 if (ext4_ext_insert_extent(handle, orig_inode,
270 orig_path, new_ext, 0))
271 goto out;
272 }
273
274 if (end_flag) {
275 err = get_ext_path(orig_inode,
276 le32_to_cpu(end_ext->ee_block) - 1, &orig_path);
277 if (err)
278 goto out;
279
280 if (ext4_ext_insert_extent(handle, orig_inode,
281 orig_path, end_ext, 0))
282 goto out;
283 }
284out:
285 if (orig_path) {
286 ext4_ext_drop_refs(orig_path);
287 kfree(orig_path);
288 }
289
290 return err;
291
292}
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308static void
309mext_insert_inside_block(struct ext4_extent *o_start,
310 struct ext4_extent *o_end,
311 struct ext4_extent *start_ext,
312 struct ext4_extent *new_ext,
313 struct ext4_extent *end_ext,
314 struct ext4_extent_header *eh,
315 int range_to_move)
316{
317 int i = 0;
318 unsigned long len;
319
320
321 if (range_to_move && o_end < EXT_LAST_EXTENT(eh)) {
322 len = (unsigned long)(EXT_LAST_EXTENT(eh) + 1) -
323 (unsigned long)(o_end + 1);
324 memmove(o_end + 1 + range_to_move, o_end + 1, len);
325 }
326
327
328 if (start_ext->ee_len)
329 o_start[i++].ee_len = start_ext->ee_len;
330
331
332 if (new_ext->ee_len) {
333 o_start[i] = *new_ext;
334 ext4_ext_store_pblock(&o_start[i++], ext4_ext_pblock(new_ext));
335 }
336
337
338 if (end_ext->ee_len)
339 o_start[i] = *end_ext;
340
341
342 le16_add_cpu(&eh->eh_entries, range_to_move);
343}
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362static int
363mext_insert_extents(handle_t *handle, struct inode *orig_inode,
364 struct ext4_ext_path *orig_path,
365 struct ext4_extent *o_start,
366 struct ext4_extent *o_end,
367 struct ext4_extent *start_ext,
368 struct ext4_extent *new_ext,
369 struct ext4_extent *end_ext)
370{
371 struct ext4_extent_header *eh;
372 unsigned long need_slots, slots_range;
373 int range_to_move, depth, ret;
374
375
376
377
378
379 need_slots = (start_ext->ee_len ? 1 : 0) + (end_ext->ee_len ? 1 : 0) +
380 (new_ext->ee_len ? 1 : 0);
381
382
383 slots_range = ((unsigned long)(o_end + 1) - (unsigned long)o_start + 1)
384 / sizeof(struct ext4_extent);
385
386
387 range_to_move = need_slots - slots_range;
388 depth = orig_path->p_depth;
389 orig_path += depth;
390 eh = orig_path->p_hdr;
391
392 if (depth) {
393
394 ret = ext4_journal_get_write_access(handle, orig_path->p_bh);
395 if (ret)
396 return ret;
397 }
398
399
400 if (range_to_move > 0 &&
401 (range_to_move > le16_to_cpu(eh->eh_max)
402 - le16_to_cpu(eh->eh_entries))) {
403
404 ret = mext_insert_across_blocks(handle, orig_inode, o_start,
405 o_end, start_ext, new_ext, end_ext);
406 if (ret < 0)
407 return ret;
408 } else
409 mext_insert_inside_block(o_start, o_end, start_ext, new_ext,
410 end_ext, eh, range_to_move);
411
412 return ext4_ext_dirty(handle, orig_inode, orig_path);
413}
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432static int
433mext_leaf_block(handle_t *handle, struct inode *orig_inode,
434 struct ext4_ext_path *orig_path, struct ext4_extent *dext,
435 ext4_lblk_t *from)
436{
437 struct ext4_extent *oext, *o_start, *o_end, *prev_ext;
438 struct ext4_extent new_ext, start_ext, end_ext;
439 ext4_lblk_t new_ext_end;
440 int oext_alen, new_ext_alen, end_ext_alen;
441 int depth = ext_depth(orig_inode);
442 int ret;
443
444 start_ext.ee_block = end_ext.ee_block = 0;
445 o_start = o_end = oext = orig_path[depth].p_ext;
446 oext_alen = ext4_ext_get_actual_len(oext);
447 start_ext.ee_len = end_ext.ee_len = 0;
448
449 new_ext.ee_block = cpu_to_le32(*from);
450 ext4_ext_store_pblock(&new_ext, ext4_ext_pblock(dext));
451 new_ext.ee_len = dext->ee_len;
452 new_ext_alen = ext4_ext_get_actual_len(&new_ext);
453 new_ext_end = le32_to_cpu(new_ext.ee_block) + new_ext_alen - 1;
454
455
456
457
458
459
460
461 if (le32_to_cpu(oext->ee_block) < le32_to_cpu(new_ext.ee_block) &&
462 le32_to_cpu(new_ext.ee_block) <
463 le32_to_cpu(oext->ee_block) + oext_alen) {
464 start_ext.ee_len = cpu_to_le16(le32_to_cpu(new_ext.ee_block) -
465 le32_to_cpu(oext->ee_block));
466 start_ext.ee_block = oext->ee_block;
467 copy_extent_status(oext, &start_ext);
468 } else if (oext > EXT_FIRST_EXTENT(orig_path[depth].p_hdr)) {
469 prev_ext = oext - 1;
470
471
472
473
474 if (ext4_can_extents_be_merged(orig_inode, prev_ext,
475 &new_ext)) {
476 o_start = prev_ext;
477 start_ext.ee_len = cpu_to_le16(
478 ext4_ext_get_actual_len(prev_ext) +
479 new_ext_alen);
480 start_ext.ee_block = oext->ee_block;
481 copy_extent_status(prev_ext, &start_ext);
482 new_ext.ee_len = 0;
483 }
484 }
485
486
487
488
489
490
491 if (le32_to_cpu(oext->ee_block) + oext_alen - 1 < new_ext_end) {
492 EXT4_ERROR_INODE(orig_inode,
493 "new_ext_end(%u) should be less than or equal to "
494 "oext->ee_block(%u) + oext_alen(%d) - 1",
495 new_ext_end, le32_to_cpu(oext->ee_block),
496 oext_alen);
497 ret = -EIO;
498 goto out;
499 }
500
501
502
503
504
505
506
507 if (le32_to_cpu(oext->ee_block) <= new_ext_end &&
508 new_ext_end < le32_to_cpu(oext->ee_block) + oext_alen - 1) {
509 end_ext.ee_len =
510 cpu_to_le16(le32_to_cpu(oext->ee_block) +
511 oext_alen - 1 - new_ext_end);
512 copy_extent_status(oext, &end_ext);
513 end_ext_alen = ext4_ext_get_actual_len(&end_ext);
514 ext4_ext_store_pblock(&end_ext,
515 (ext4_ext_pblock(o_end) + oext_alen - end_ext_alen));
516 end_ext.ee_block =
517 cpu_to_le32(le32_to_cpu(o_end->ee_block) +
518 oext_alen - end_ext_alen);
519 }
520
521 ret = mext_insert_extents(handle, orig_inode, orig_path, o_start,
522 o_end, &start_ext, &new_ext, &end_ext);
523out:
524 return ret;
525}
526
527
528
529
530
531
532
533
534
535
536
537
538static int
539mext_calc_swap_extents(struct ext4_extent *tmp_dext,
540 struct ext4_extent *tmp_oext,
541 ext4_lblk_t orig_off, ext4_lblk_t donor_off,
542 ext4_lblk_t max_count)
543{
544 ext4_lblk_t diff, orig_diff;
545 struct ext4_extent dext_old, oext_old;
546
547 BUG_ON(orig_off != donor_off);
548
549
550 if (orig_off < le32_to_cpu(tmp_oext->ee_block) ||
551 le32_to_cpu(tmp_oext->ee_block) +
552 ext4_ext_get_actual_len(tmp_oext) - 1 < orig_off)
553 return -ENODATA;
554
555 if (orig_off < le32_to_cpu(tmp_dext->ee_block) ||
556 le32_to_cpu(tmp_dext->ee_block) +
557 ext4_ext_get_actual_len(tmp_dext) - 1 < orig_off)
558 return -ENODATA;
559
560 dext_old = *tmp_dext;
561 oext_old = *tmp_oext;
562
563
564 diff = donor_off - le32_to_cpu(tmp_dext->ee_block);
565
566 ext4_ext_store_pblock(tmp_dext, ext4_ext_pblock(tmp_dext) + diff);
567 le32_add_cpu(&tmp_dext->ee_block, diff);
568 le16_add_cpu(&tmp_dext->ee_len, -diff);
569
570 if (max_count < ext4_ext_get_actual_len(tmp_dext))
571 tmp_dext->ee_len = cpu_to_le16(max_count);
572
573 orig_diff = orig_off - le32_to_cpu(tmp_oext->ee_block);
574 ext4_ext_store_pblock(tmp_oext, ext4_ext_pblock(tmp_oext) + orig_diff);
575
576
577 if (ext4_ext_get_actual_len(tmp_dext) >
578 ext4_ext_get_actual_len(tmp_oext) - orig_diff)
579 tmp_dext->ee_len = cpu_to_le16(le16_to_cpu(tmp_oext->ee_len) -
580 orig_diff);
581
582 tmp_oext->ee_len = cpu_to_le16(ext4_ext_get_actual_len(tmp_dext));
583
584 copy_extent_status(&oext_old, tmp_dext);
585 copy_extent_status(&dext_old, tmp_oext);
586
587 return 0;
588}
589
590
591
592
593
594
595
596
597
598
599
600
601static int
602mext_check_coverage(struct inode *inode, ext4_lblk_t from, ext4_lblk_t count,
603 int uninit, int *err)
604{
605 struct ext4_ext_path *path = NULL;
606 struct ext4_extent *ext;
607 int ret = 0;
608 ext4_lblk_t last = from + count;
609 while (from < last) {
610 *err = get_ext_path(inode, from, &path);
611 if (*err)
612 goto out;
613 ext = path[ext_depth(inode)].p_ext;
614 if (uninit != ext4_ext_is_uninitialized(ext))
615 goto out;
616 from += ext4_ext_get_actual_len(ext);
617 ext4_ext_drop_refs(path);
618 }
619 ret = 1;
620out:
621 if (path) {
622 ext4_ext_drop_refs(path);
623 kfree(path);
624 }
625 return ret;
626}
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649static int
650mext_replace_branches(handle_t *handle, struct inode *orig_inode,
651 struct inode *donor_inode, ext4_lblk_t from,
652 ext4_lblk_t count, int *err)
653{
654 struct ext4_ext_path *orig_path = NULL;
655 struct ext4_ext_path *donor_path = NULL;
656 struct ext4_extent *oext, *dext;
657 struct ext4_extent tmp_dext, tmp_oext;
658 ext4_lblk_t orig_off = from, donor_off = from;
659 int depth;
660 int replaced_count = 0;
661 int dext_alen;
662
663 *err = ext4_es_remove_extent(orig_inode, from, count);
664 if (*err)
665 goto out;
666
667 *err = ext4_es_remove_extent(donor_inode, from, count);
668 if (*err)
669 goto out;
670
671
672 *err = get_ext_path(orig_inode, orig_off, &orig_path);
673 if (*err)
674 goto out;
675
676
677 *err = get_ext_path(donor_inode, donor_off, &donor_path);
678 if (*err)
679 goto out;
680 depth = ext_depth(orig_inode);
681 oext = orig_path[depth].p_ext;
682 tmp_oext = *oext;
683
684 depth = ext_depth(donor_inode);
685 dext = donor_path[depth].p_ext;
686 if (unlikely(!dext))
687 goto missing_donor_extent;
688 tmp_dext = *dext;
689
690 *err = mext_calc_swap_extents(&tmp_dext, &tmp_oext, orig_off,
691 donor_off, count);
692 if (*err)
693 goto out;
694
695
696 while (1) {
697
698 if (unlikely(!dext)) {
699 missing_donor_extent:
700 EXT4_ERROR_INODE(donor_inode,
701 "The extent for donor must be found");
702 *err = -EIO;
703 goto out;
704 } else if (donor_off != le32_to_cpu(tmp_dext.ee_block)) {
705 EXT4_ERROR_INODE(donor_inode,
706 "Donor offset(%u) and the first block of donor "
707 "extent(%u) should be equal",
708 donor_off,
709 le32_to_cpu(tmp_dext.ee_block));
710 *err = -EIO;
711 goto out;
712 }
713
714
715 *err = mext_leaf_block(handle, orig_inode,
716 orig_path, &tmp_dext, &orig_off);
717 if (*err)
718 goto out;
719
720
721 *err = mext_leaf_block(handle, donor_inode,
722 donor_path, &tmp_oext, &donor_off);
723 if (*err)
724 goto out;
725
726 dext_alen = ext4_ext_get_actual_len(&tmp_dext);
727 replaced_count += dext_alen;
728 donor_off += dext_alen;
729 orig_off += dext_alen;
730
731 BUG_ON(replaced_count > count);
732
733 if (replaced_count >= count)
734 break;
735
736 if (orig_path)
737 ext4_ext_drop_refs(orig_path);
738 *err = get_ext_path(orig_inode, orig_off, &orig_path);
739 if (*err)
740 goto out;
741 depth = ext_depth(orig_inode);
742 oext = orig_path[depth].p_ext;
743 tmp_oext = *oext;
744
745 if (donor_path)
746 ext4_ext_drop_refs(donor_path);
747 *err = get_ext_path(donor_inode, donor_off, &donor_path);
748 if (*err)
749 goto out;
750 depth = ext_depth(donor_inode);
751 dext = donor_path[depth].p_ext;
752 tmp_dext = *dext;
753
754 *err = mext_calc_swap_extents(&tmp_dext, &tmp_oext, orig_off,
755 donor_off, count - replaced_count);
756 if (*err)
757 goto out;
758 }
759
760out:
761 if (orig_path) {
762 ext4_ext_drop_refs(orig_path);
763 kfree(orig_path);
764 }
765 if (donor_path) {
766 ext4_ext_drop_refs(donor_path);
767 kfree(donor_path);
768 }
769
770 return replaced_count;
771}
772
773
774
775
776
777
778
779
780
781
782
783static int
784mext_page_double_lock(struct inode *inode1, struct inode *inode2,
785 pgoff_t index, struct page *page[2])
786{
787 struct address_space *mapping[2];
788 unsigned fl = AOP_FLAG_NOFS;
789
790 BUG_ON(!inode1 || !inode2);
791 if (inode1 < inode2) {
792 mapping[0] = inode1->i_mapping;
793 mapping[1] = inode2->i_mapping;
794 } else {
795 mapping[0] = inode2->i_mapping;
796 mapping[1] = inode1->i_mapping;
797 }
798
799 page[0] = grab_cache_page_write_begin(mapping[0], index, fl);
800 if (!page[0])
801 return -ENOMEM;
802
803 page[1] = grab_cache_page_write_begin(mapping[1], index, fl);
804 if (!page[1]) {
805 unlock_page(page[0]);
806 page_cache_release(page[0]);
807 return -ENOMEM;
808 }
809
810
811
812
813
814 wait_on_page_writeback(page[0]);
815 wait_on_page_writeback(page[1]);
816 if (inode1 > inode2) {
817 struct page *tmp;
818 tmp = page[0];
819 page[0] = page[1];
820 page[1] = tmp;
821 }
822 return 0;
823}
824
825
826static int
827mext_page_mkuptodate(struct page *page, unsigned from, unsigned to)
828{
829 struct inode *inode = page->mapping->host;
830 sector_t block;
831 struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
832 unsigned int blocksize, block_start, block_end;
833 int i, err, nr = 0, partial = 0;
834 BUG_ON(!PageLocked(page));
835 BUG_ON(PageWriteback(page));
836
837 if (PageUptodate(page))
838 return 0;
839
840 blocksize = 1 << inode->i_blkbits;
841 if (!page_has_buffers(page))
842 create_empty_buffers(page, blocksize, 0);
843
844 head = page_buffers(page);
845 block = (sector_t)page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits);
846 for (bh = head, block_start = 0; bh != head || !block_start;
847 block++, block_start = block_end, bh = bh->b_this_page) {
848 block_end = block_start + blocksize;
849 if (block_end <= from || block_start >= to) {
850 if (!buffer_uptodate(bh))
851 partial = 1;
852 continue;
853 }
854 if (buffer_uptodate(bh))
855 continue;
856 if (!buffer_mapped(bh)) {
857 err = ext4_get_block(inode, block, bh, 0);
858 if (err) {
859 SetPageError(page);
860 return err;
861 }
862 if (!buffer_mapped(bh)) {
863 zero_user(page, block_start, blocksize);
864 if (!err)
865 set_buffer_uptodate(bh);
866 continue;
867 }
868 }
869 BUG_ON(nr >= MAX_BUF_PER_PAGE);
870 arr[nr++] = bh;
871 }
872
873 if (!nr)
874 goto out;
875
876 for (i = 0; i < nr; i++) {
877 bh = arr[i];
878 if (!bh_uptodate_or_lock(bh)) {
879 err = bh_submit_read(bh);
880 if (err)
881 return err;
882 }
883 }
884out:
885 if (!partial)
886 SetPageUptodate(page);
887 return 0;
888}
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906static int
907move_extent_per_page(struct file *o_filp, struct inode *donor_inode,
908 pgoff_t orig_page_offset, int data_offset_in_page,
909 int block_len_in_page, int uninit, int *err)
910{
911 struct inode *orig_inode = file_inode(o_filp);
912 struct page *pagep[2] = {NULL, NULL};
913 handle_t *handle;
914 ext4_lblk_t orig_blk_offset;
915 unsigned long blocksize = orig_inode->i_sb->s_blocksize;
916 unsigned int w_flags = 0;
917 unsigned int tmp_data_size, data_size, replaced_size;
918 int err2, jblocks, retries = 0;
919 int replaced_count = 0;
920 int from = data_offset_in_page << orig_inode->i_blkbits;
921 int blocks_per_page = PAGE_CACHE_SIZE >> orig_inode->i_blkbits;
922
923
924
925
926
927again:
928 *err = 0;
929 jblocks = ext4_writepage_trans_blocks(orig_inode) * 2;
930 handle = ext4_journal_start(orig_inode, EXT4_HT_MOVE_EXTENTS, jblocks);
931 if (IS_ERR(handle)) {
932 *err = PTR_ERR(handle);
933 return 0;
934 }
935
936 if (segment_eq(get_fs(), KERNEL_DS))
937 w_flags |= AOP_FLAG_UNINTERRUPTIBLE;
938
939 orig_blk_offset = orig_page_offset * blocks_per_page +
940 data_offset_in_page;
941
942
943 if ((orig_blk_offset + block_len_in_page - 1) ==
944 ((orig_inode->i_size - 1) >> orig_inode->i_blkbits)) {
945
946 tmp_data_size = orig_inode->i_size & (blocksize - 1);
947
948
949
950
951 if (tmp_data_size == 0)
952 tmp_data_size = blocksize;
953
954 data_size = tmp_data_size +
955 ((block_len_in_page - 1) << orig_inode->i_blkbits);
956 } else
957 data_size = block_len_in_page << orig_inode->i_blkbits;
958
959 replaced_size = data_size;
960
961 *err = mext_page_double_lock(orig_inode, donor_inode, orig_page_offset,
962 pagep);
963 if (unlikely(*err < 0))
964 goto stop_journal;
965
966
967
968
969
970
971
972 if (uninit) {
973 ext4_double_down_write_data_sem(orig_inode, donor_inode);
974
975
976 uninit = mext_check_coverage(orig_inode, orig_blk_offset,
977 block_len_in_page, 1, err);
978 if (*err)
979 goto drop_data_sem;
980
981 uninit &= mext_check_coverage(donor_inode, orig_blk_offset,
982 block_len_in_page, 1, err);
983 if (*err)
984 goto drop_data_sem;
985
986 if (!uninit) {
987 ext4_double_up_write_data_sem(orig_inode, donor_inode);
988 goto data_copy;
989 }
990 if ((page_has_private(pagep[0]) &&
991 !try_to_release_page(pagep[0], 0)) ||
992 (page_has_private(pagep[1]) &&
993 !try_to_release_page(pagep[1], 0))) {
994 *err = -EBUSY;
995 goto drop_data_sem;
996 }
997 replaced_count = mext_replace_branches(handle, orig_inode,
998 donor_inode, orig_blk_offset,
999 block_len_in_page, err);
1000 drop_data_sem:
1001 ext4_double_up_write_data_sem(orig_inode, donor_inode);
1002 goto unlock_pages;
1003 }
1004data_copy:
1005 *err = mext_page_mkuptodate(pagep[0], from, from + replaced_size);
1006 if (*err)
1007 goto unlock_pages;
1008
1009
1010
1011 if ((page_has_private(pagep[0]) && !try_to_release_page(pagep[0], 0)) ||
1012 (page_has_private(pagep[1]) && !try_to_release_page(pagep[1], 0))) {
1013 *err = -EBUSY;
1014 goto unlock_pages;
1015 }
1016
1017 replaced_count = mext_replace_branches(handle, orig_inode, donor_inode,
1018 orig_blk_offset,
1019 block_len_in_page, err);
1020 if (*err) {
1021 if (replaced_count) {
1022 block_len_in_page = replaced_count;
1023 replaced_size =
1024 block_len_in_page << orig_inode->i_blkbits;
1025 } else
1026 goto unlock_pages;
1027 }
1028
1029
1030 *err = __block_write_begin(pagep[0], from, replaced_size,
1031 ext4_get_block);
1032 if (!*err)
1033 *err = block_commit_write(pagep[0], from, from + replaced_size);
1034
1035 if (unlikely(*err < 0))
1036 goto repair_branches;
1037
1038
1039
1040 *err = ext4_jbd2_file_inode(handle, orig_inode);
1041
1042unlock_pages:
1043 unlock_page(pagep[0]);
1044 page_cache_release(pagep[0]);
1045 unlock_page(pagep[1]);
1046 page_cache_release(pagep[1]);
1047stop_journal:
1048 ext4_journal_stop(handle);
1049
1050
1051 if (*err == -EBUSY && ext4_should_retry_alloc(orig_inode->i_sb,
1052 &retries))
1053 goto again;
1054 return replaced_count;
1055
1056repair_branches:
1057
1058
1059
1060
1061
1062 ext4_double_down_write_data_sem(orig_inode, donor_inode);
1063 replaced_count = mext_replace_branches(handle, donor_inode, orig_inode,
1064 orig_blk_offset,
1065 block_len_in_page, &err2);
1066 ext4_double_up_write_data_sem(orig_inode, donor_inode);
1067 if (replaced_count != block_len_in_page) {
1068 EXT4_ERROR_INODE_BLOCK(orig_inode, (sector_t)(orig_blk_offset),
1069 "Unable to copy data block,"
1070 " data will be lost.");
1071 *err = -EIO;
1072 }
1073 replaced_count = 0;
1074 goto unlock_pages;
1075}
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090static int
1091mext_check_arguments(struct inode *orig_inode,
1092 struct inode *donor_inode, __u64 orig_start,
1093 __u64 donor_start, __u64 *len)
1094{
1095 ext4_lblk_t orig_blocks, donor_blocks;
1096 unsigned int blkbits = orig_inode->i_blkbits;
1097 unsigned int blocksize = 1 << blkbits;
1098
1099 if (donor_inode->i_mode & (S_ISUID|S_ISGID)) {
1100 ext4_debug("ext4 move extent: suid or sgid is set"
1101 " to donor file [ino:orig %lu, donor %lu]\n",
1102 orig_inode->i_ino, donor_inode->i_ino);
1103 return -EINVAL;
1104 }
1105
1106 if (IS_IMMUTABLE(donor_inode) || IS_APPEND(donor_inode))
1107 return -EPERM;
1108
1109
1110 if (IS_SWAPFILE(orig_inode) || IS_SWAPFILE(donor_inode)) {
1111 ext4_debug("ext4 move extent: The argument files should "
1112 "not be swapfile [ino:orig %lu, donor %lu]\n",
1113 orig_inode->i_ino, donor_inode->i_ino);
1114 return -EINVAL;
1115 }
1116
1117
1118 if (!(ext4_test_inode_flag(orig_inode, EXT4_INODE_EXTENTS))) {
1119 ext4_debug("ext4 move extent: orig file is not extents "
1120 "based file [ino:orig %lu]\n", orig_inode->i_ino);
1121 return -EOPNOTSUPP;
1122 } else if (!(ext4_test_inode_flag(donor_inode, EXT4_INODE_EXTENTS))) {
1123 ext4_debug("ext4 move extent: donor file is not extents "
1124 "based file [ino:donor %lu]\n", donor_inode->i_ino);
1125 return -EOPNOTSUPP;
1126 }
1127
1128 if ((!orig_inode->i_size) || (!donor_inode->i_size)) {
1129 ext4_debug("ext4 move extent: File size is 0 byte\n");
1130 return -EINVAL;
1131 }
1132
1133
1134 if (orig_start != donor_start) {
1135 ext4_debug("ext4 move extent: orig and donor's start "
1136 "offset are not same [ino:orig %lu, donor %lu]\n",
1137 orig_inode->i_ino, donor_inode->i_ino);
1138 return -EINVAL;
1139 }
1140
1141 if ((orig_start >= EXT_MAX_BLOCKS) ||
1142 (*len > EXT_MAX_BLOCKS) ||
1143 (orig_start + *len >= EXT_MAX_BLOCKS)) {
1144 ext4_debug("ext4 move extent: Can't handle over [%u] blocks "
1145 "[ino:orig %lu, donor %lu]\n", EXT_MAX_BLOCKS,
1146 orig_inode->i_ino, donor_inode->i_ino);
1147 return -EINVAL;
1148 }
1149
1150 if (orig_inode->i_size > donor_inode->i_size) {
1151 donor_blocks = (donor_inode->i_size + blocksize - 1) >> blkbits;
1152
1153 if (orig_start >= donor_blocks) {
1154 ext4_debug("ext4 move extent: orig start offset "
1155 "[%llu] should be less than donor file blocks "
1156 "[%u] [ino:orig %lu, donor %lu]\n",
1157 orig_start, donor_blocks,
1158 orig_inode->i_ino, donor_inode->i_ino);
1159 return -EINVAL;
1160 }
1161
1162
1163 if (orig_start + *len > donor_blocks) {
1164 ext4_debug("ext4 move extent: End offset [%llu] should "
1165 "be less than donor file blocks [%u]."
1166 "So adjust length from %llu to %llu "
1167 "[ino:orig %lu, donor %lu]\n",
1168 orig_start + *len, donor_blocks,
1169 *len, donor_blocks - orig_start,
1170 orig_inode->i_ino, donor_inode->i_ino);
1171 *len = donor_blocks - orig_start;
1172 }
1173 } else {
1174 orig_blocks = (orig_inode->i_size + blocksize - 1) >> blkbits;
1175 if (orig_start >= orig_blocks) {
1176 ext4_debug("ext4 move extent: start offset [%llu] "
1177 "should be less than original file blocks "
1178 "[%u] [ino:orig %lu, donor %lu]\n",
1179 orig_start, orig_blocks,
1180 orig_inode->i_ino, donor_inode->i_ino);
1181 return -EINVAL;
1182 }
1183
1184 if (orig_start + *len > orig_blocks) {
1185 ext4_debug("ext4 move extent: Adjust length "
1186 "from %llu to %llu. Because it should be "
1187 "less than original file blocks "
1188 "[ino:orig %lu, donor %lu]\n",
1189 *len, orig_blocks - orig_start,
1190 orig_inode->i_ino, donor_inode->i_ino);
1191 *len = orig_blocks - orig_start;
1192 }
1193 }
1194
1195 if (!*len) {
1196 ext4_debug("ext4 move extent: len should not be 0 "
1197 "[ino:orig %lu, donor %lu]\n", orig_inode->i_ino,
1198 donor_inode->i_ino);
1199 return -EINVAL;
1200 }
1201
1202 return 0;
1203}
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213void
1214ext4_inode_double_lock(struct inode *inode1, struct inode *inode2)
1215{
1216 BUG_ON(inode1 == inode2);
1217 if (inode1 < inode2) {
1218 mutex_lock_nested(&inode1->i_mutex, I_MUTEX_PARENT);
1219 mutex_lock_nested(&inode2->i_mutex, I_MUTEX_CHILD);
1220 } else {
1221 mutex_lock_nested(&inode2->i_mutex, I_MUTEX_PARENT);
1222 mutex_lock_nested(&inode1->i_mutex, I_MUTEX_CHILD);
1223 }
1224}
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234void
1235ext4_inode_double_unlock(struct inode *inode1, struct inode *inode2)
1236{
1237 mutex_unlock(&inode1->i_mutex);
1238 mutex_unlock(&inode2->i_mutex);
1239}
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282int
1283ext4_move_extents(struct file *o_filp, struct file *d_filp,
1284 __u64 orig_start, __u64 donor_start, __u64 len,
1285 __u64 *moved_len)
1286{
1287 struct inode *orig_inode = file_inode(o_filp);
1288 struct inode *donor_inode = file_inode(d_filp);
1289 struct ext4_ext_path *orig_path = NULL, *holecheck_path = NULL;
1290 struct ext4_extent *ext_prev, *ext_cur, *ext_dummy;
1291 ext4_lblk_t block_start = orig_start;
1292 ext4_lblk_t block_end, seq_start, add_blocks, file_end, seq_blocks = 0;
1293 ext4_lblk_t rest_blocks;
1294 pgoff_t orig_page_offset = 0, seq_end_page;
1295 int ret, depth, last_extent = 0;
1296 int blocks_per_page = PAGE_CACHE_SIZE >> orig_inode->i_blkbits;
1297 int data_offset_in_page;
1298 int block_len_in_page;
1299 int uninit;
1300
1301 if (orig_inode->i_sb != donor_inode->i_sb) {
1302 ext4_debug("ext4 move extent: The argument files "
1303 "should be in same FS [ino:orig %lu, donor %lu]\n",
1304 orig_inode->i_ino, donor_inode->i_ino);
1305 return -EINVAL;
1306 }
1307
1308
1309 if (orig_inode == donor_inode) {
1310 ext4_debug("ext4 move extent: The argument files should not "
1311 "be same inode [ino:orig %lu, donor %lu]\n",
1312 orig_inode->i_ino, donor_inode->i_ino);
1313 return -EINVAL;
1314 }
1315
1316
1317 if (!S_ISREG(orig_inode->i_mode) || !S_ISREG(donor_inode->i_mode)) {
1318 ext4_debug("ext4 move extent: The argument files should be "
1319 "regular file [ino:orig %lu, donor %lu]\n",
1320 orig_inode->i_ino, donor_inode->i_ino);
1321 return -EINVAL;
1322 }
1323
1324
1325 if (ext4_should_journal_data(orig_inode) ||
1326 ext4_should_journal_data(donor_inode)) {
1327 return -EINVAL;
1328 }
1329
1330 ext4_inode_double_lock(orig_inode, donor_inode);
1331
1332
1333 ext4_inode_block_unlocked_dio(orig_inode);
1334 ext4_inode_block_unlocked_dio(donor_inode);
1335 inode_dio_wait(orig_inode);
1336 inode_dio_wait(donor_inode);
1337
1338
1339 ext4_double_down_write_data_sem(orig_inode, donor_inode);
1340
1341 ret = mext_check_arguments(orig_inode, donor_inode, orig_start,
1342 donor_start, &len);
1343 if (ret)
1344 goto out;
1345
1346 file_end = (i_size_read(orig_inode) - 1) >> orig_inode->i_blkbits;
1347 block_end = block_start + len - 1;
1348 if (file_end < block_end)
1349 len -= block_end - file_end;
1350
1351 ret = get_ext_path(orig_inode, block_start, &orig_path);
1352 if (ret)
1353 goto out;
1354
1355
1356 ret = get_ext_path(orig_inode, block_start, &holecheck_path);
1357 if (ret)
1358 goto out;
1359
1360 depth = ext_depth(orig_inode);
1361 ext_cur = holecheck_path[depth].p_ext;
1362
1363
1364
1365
1366
1367 if (le32_to_cpu(ext_cur->ee_block) +
1368 ext4_ext_get_actual_len(ext_cur) - 1 < block_start) {
1369
1370
1371
1372
1373 last_extent = mext_next_extent(orig_inode,
1374 holecheck_path, &ext_cur);
1375 if (last_extent < 0) {
1376 ret = last_extent;
1377 goto out;
1378 }
1379 last_extent = mext_next_extent(orig_inode, orig_path,
1380 &ext_dummy);
1381 if (last_extent < 0) {
1382 ret = last_extent;
1383 goto out;
1384 }
1385 seq_start = le32_to_cpu(ext_cur->ee_block);
1386 } else if (le32_to_cpu(ext_cur->ee_block) > block_start)
1387
1388 seq_start = le32_to_cpu(ext_cur->ee_block);
1389 else
1390 seq_start = block_start;
1391
1392
1393 if (le32_to_cpu(ext_cur->ee_block) > block_end) {
1394 ext4_debug("ext4 move extent: The specified range of file "
1395 "may be the hole\n");
1396 ret = -EINVAL;
1397 goto out;
1398 }
1399
1400
1401 add_blocks = min(le32_to_cpu(ext_cur->ee_block) +
1402 ext4_ext_get_actual_len(ext_cur), block_end + 1) -
1403 max(le32_to_cpu(ext_cur->ee_block), block_start);
1404
1405 while (!last_extent && le32_to_cpu(ext_cur->ee_block) <= block_end) {
1406 seq_blocks += add_blocks;
1407
1408
1409 if (seq_start + seq_blocks - 1 > block_end)
1410 seq_blocks = block_end - seq_start + 1;
1411
1412 ext_prev = ext_cur;
1413 last_extent = mext_next_extent(orig_inode, holecheck_path,
1414 &ext_cur);
1415 if (last_extent < 0) {
1416 ret = last_extent;
1417 break;
1418 }
1419 add_blocks = ext4_ext_get_actual_len(ext_cur);
1420
1421
1422
1423
1424
1425 if (ext4_can_extents_be_merged(orig_inode,
1426 ext_prev, ext_cur) &&
1427 block_end >= le32_to_cpu(ext_cur->ee_block) &&
1428 !last_extent)
1429 continue;
1430
1431
1432 uninit = ext4_ext_is_uninitialized(ext_prev);
1433
1434 data_offset_in_page = seq_start % blocks_per_page;
1435
1436
1437
1438
1439
1440 if (data_offset_in_page + seq_blocks > blocks_per_page) {
1441
1442 block_len_in_page =
1443 blocks_per_page - data_offset_in_page;
1444 } else {
1445
1446 block_len_in_page = seq_blocks;
1447 }
1448
1449 orig_page_offset = seq_start >>
1450 (PAGE_CACHE_SHIFT - orig_inode->i_blkbits);
1451 seq_end_page = (seq_start + seq_blocks - 1) >>
1452 (PAGE_CACHE_SHIFT - orig_inode->i_blkbits);
1453 seq_start = le32_to_cpu(ext_cur->ee_block);
1454 rest_blocks = seq_blocks;
1455
1456
1457
1458
1459
1460
1461
1462
1463 ext4_double_up_write_data_sem(orig_inode, donor_inode);
1464
1465 while (orig_page_offset <= seq_end_page) {
1466
1467
1468 block_len_in_page = move_extent_per_page(
1469 o_filp, donor_inode,
1470 orig_page_offset,
1471 data_offset_in_page,
1472 block_len_in_page, uninit,
1473 &ret);
1474
1475
1476 *moved_len += block_len_in_page;
1477 if (ret < 0)
1478 break;
1479 if (*moved_len > len) {
1480 EXT4_ERROR_INODE(orig_inode,
1481 "We replaced blocks too much! "
1482 "sum of replaced: %llu requested: %llu",
1483 *moved_len, len);
1484 ret = -EIO;
1485 break;
1486 }
1487
1488 orig_page_offset++;
1489 data_offset_in_page = 0;
1490 rest_blocks -= block_len_in_page;
1491 if (rest_blocks > blocks_per_page)
1492 block_len_in_page = blocks_per_page;
1493 else
1494 block_len_in_page = rest_blocks;
1495 }
1496
1497 ext4_double_down_write_data_sem(orig_inode, donor_inode);
1498 if (ret < 0)
1499 break;
1500
1501
1502 if (holecheck_path)
1503 ext4_ext_drop_refs(holecheck_path);
1504 ret = get_ext_path(orig_inode, seq_start, &holecheck_path);
1505 if (ret)
1506 break;
1507 depth = holecheck_path->p_depth;
1508
1509
1510 if (orig_path)
1511 ext4_ext_drop_refs(orig_path);
1512 ret = get_ext_path(orig_inode, seq_start, &orig_path);
1513 if (ret)
1514 break;
1515
1516 ext_cur = holecheck_path[depth].p_ext;
1517 add_blocks = ext4_ext_get_actual_len(ext_cur);
1518 seq_blocks = 0;
1519
1520 }
1521out:
1522 if (*moved_len) {
1523 ext4_discard_preallocations(orig_inode);
1524 ext4_discard_preallocations(donor_inode);
1525 }
1526
1527 if (orig_path) {
1528 ext4_ext_drop_refs(orig_path);
1529 kfree(orig_path);
1530 }
1531 if (holecheck_path) {
1532 ext4_ext_drop_refs(holecheck_path);
1533 kfree(holecheck_path);
1534 }
1535 ext4_double_up_write_data_sem(orig_inode, donor_inode);
1536 ext4_inode_resume_unlocked_dio(orig_inode);
1537 ext4_inode_resume_unlocked_dio(donor_inode);
1538 ext4_inode_double_unlock(orig_inode, donor_inode);
1539
1540 return ret;
1541}
1542