1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#include <linux/fs.h>
33#include <linux/time.h>
34#include <linux/jbd2.h>
35#include <linux/highuid.h>
36#include <linux/pagemap.h>
37#include <linux/quotaops.h>
38#include <linux/string.h>
39#include <linux/slab.h>
40#include <linux/uaccess.h>
41#include <linux/fiemap.h>
42#include <linux/backing-dev.h>
43#include "ext4_jbd2.h"
44#include "ext4_extents.h"
45#include "xattr.h"
46
47#include <trace/events/ext4.h>
48
49
50
51
52#define EXT4_EXT_MAY_ZEROOUT 0x1
53
54#define EXT4_EXT_MARK_UNWRIT1 0x2
55#define EXT4_EXT_MARK_UNWRIT2 0x4
56
57#define EXT4_EXT_DATA_VALID1 0x8
58#define EXT4_EXT_DATA_VALID2 0x10
59
60static __le32 ext4_extent_block_csum(struct inode *inode,
61 struct ext4_extent_header *eh)
62{
63 struct ext4_inode_info *ei = EXT4_I(inode);
64 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
65 __u32 csum;
66
67 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)eh,
68 EXT4_EXTENT_TAIL_OFFSET(eh));
69 return cpu_to_le32(csum);
70}
71
72static int ext4_extent_block_csum_verify(struct inode *inode,
73 struct ext4_extent_header *eh)
74{
75 struct ext4_extent_tail *et;
76
77 if (!ext4_has_metadata_csum(inode->i_sb))
78 return 1;
79
80 et = find_ext4_extent_tail(eh);
81 if (et->et_checksum != ext4_extent_block_csum(inode, eh))
82 return 0;
83 return 1;
84}
85
86static void ext4_extent_block_csum_set(struct inode *inode,
87 struct ext4_extent_header *eh)
88{
89 struct ext4_extent_tail *et;
90
91 if (!ext4_has_metadata_csum(inode->i_sb))
92 return;
93
94 et = find_ext4_extent_tail(eh);
95 et->et_checksum = ext4_extent_block_csum(inode, eh);
96}
97
98static int ext4_split_extent(handle_t *handle,
99 struct inode *inode,
100 struct ext4_ext_path **ppath,
101 struct ext4_map_blocks *map,
102 int split_flag,
103 int flags);
104
105static int ext4_split_extent_at(handle_t *handle,
106 struct inode *inode,
107 struct ext4_ext_path **ppath,
108 ext4_lblk_t split,
109 int split_flag,
110 int flags);
111
112static int ext4_find_delayed_extent(struct inode *inode,
113 struct extent_status *newes);
114
115static int ext4_ext_truncate_extend_restart(handle_t *handle,
116 struct inode *inode,
117 int needed)
118{
119 int err;
120
121 if (!ext4_handle_valid(handle))
122 return 0;
123 if (handle->h_buffer_credits >= needed)
124 return 0;
125
126
127
128
129 needed += 3;
130 err = ext4_journal_extend(handle, needed - handle->h_buffer_credits);
131 if (err <= 0)
132 return err;
133 err = ext4_truncate_restart_trans(handle, inode, needed);
134 if (err == 0)
135 err = -EAGAIN;
136
137 return err;
138}
139
140
141
142
143
144
145static int ext4_ext_get_access(handle_t *handle, struct inode *inode,
146 struct ext4_ext_path *path)
147{
148 if (path->p_bh) {
149
150 BUFFER_TRACE(path->p_bh, "get_write_access");
151 return ext4_journal_get_write_access(handle, path->p_bh);
152 }
153
154
155 return 0;
156}
157
158
159
160
161
162
163
164int __ext4_ext_dirty(const char *where, unsigned int line, handle_t *handle,
165 struct inode *inode, struct ext4_ext_path *path)
166{
167 int err;
168
169 WARN_ON(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem));
170 if (path->p_bh) {
171 ext4_extent_block_csum_set(inode, ext_block_hdr(path->p_bh));
172
173 err = __ext4_handle_dirty_metadata(where, line, handle,
174 inode, path->p_bh);
175 } else {
176
177 err = ext4_mark_inode_dirty(handle, inode);
178 }
179 return err;
180}
181
182static ext4_fsblk_t ext4_ext_find_goal(struct inode *inode,
183 struct ext4_ext_path *path,
184 ext4_lblk_t block)
185{
186 if (path) {
187 int depth = path->p_depth;
188 struct ext4_extent *ex;
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207 ex = path[depth].p_ext;
208 if (ex) {
209 ext4_fsblk_t ext_pblk = ext4_ext_pblock(ex);
210 ext4_lblk_t ext_block = le32_to_cpu(ex->ee_block);
211
212 if (block > ext_block)
213 return ext_pblk + (block - ext_block);
214 else
215 return ext_pblk - (ext_block - block);
216 }
217
218
219
220 if (path[depth].p_bh)
221 return path[depth].p_bh->b_blocknr;
222 }
223
224
225 return ext4_inode_to_goal_block(inode);
226}
227
228
229
230
231static ext4_fsblk_t
232ext4_ext_new_meta_block(handle_t *handle, struct inode *inode,
233 struct ext4_ext_path *path,
234 struct ext4_extent *ex, int *err, unsigned int flags)
235{
236 ext4_fsblk_t goal, newblock;
237
238 goal = ext4_ext_find_goal(inode, path, le32_to_cpu(ex->ee_block));
239 newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
240 NULL, err);
241 return newblock;
242}
243
244static inline int ext4_ext_space_block(struct inode *inode, int check)
245{
246 int size;
247
248 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
249 / sizeof(struct ext4_extent);
250#ifdef AGGRESSIVE_TEST
251 if (!check && size > 6)
252 size = 6;
253#endif
254 return size;
255}
256
257static inline int ext4_ext_space_block_idx(struct inode *inode, int check)
258{
259 int size;
260
261 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
262 / sizeof(struct ext4_extent_idx);
263#ifdef AGGRESSIVE_TEST
264 if (!check && size > 5)
265 size = 5;
266#endif
267 return size;
268}
269
270static inline int ext4_ext_space_root(struct inode *inode, int check)
271{
272 int size;
273
274 size = sizeof(EXT4_I(inode)->i_data);
275 size -= sizeof(struct ext4_extent_header);
276 size /= sizeof(struct ext4_extent);
277#ifdef AGGRESSIVE_TEST
278 if (!check && size > 3)
279 size = 3;
280#endif
281 return size;
282}
283
284static inline int ext4_ext_space_root_idx(struct inode *inode, int check)
285{
286 int size;
287
288 size = sizeof(EXT4_I(inode)->i_data);
289 size -= sizeof(struct ext4_extent_header);
290 size /= sizeof(struct ext4_extent_idx);
291#ifdef AGGRESSIVE_TEST
292 if (!check && size > 4)
293 size = 4;
294#endif
295 return size;
296}
297
298static inline int
299ext4_force_split_extent_at(handle_t *handle, struct inode *inode,
300 struct ext4_ext_path **ppath, ext4_lblk_t lblk,
301 int nofail)
302{
303 struct ext4_ext_path *path = *ppath;
304 int unwritten = ext4_ext_is_unwritten(path[path->p_depth].p_ext);
305
306 return ext4_split_extent_at(handle, inode, ppath, lblk, unwritten ?
307 EXT4_EXT_MARK_UNWRIT1|EXT4_EXT_MARK_UNWRIT2 : 0,
308 EXT4_EX_NOCACHE | EXT4_GET_BLOCKS_PRE_IO |
309 (nofail ? EXT4_GET_BLOCKS_METADATA_NOFAIL:0));
310}
311
312
313
314
315
316
317int ext4_ext_calc_metadata_amount(struct inode *inode, ext4_lblk_t lblock)
318{
319 struct ext4_inode_info *ei = EXT4_I(inode);
320 int idxs;
321
322 idxs = ((inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
323 / sizeof(struct ext4_extent_idx));
324
325
326
327
328
329
330
331
332
333 if (ei->i_da_metadata_calc_len &&
334 ei->i_da_metadata_calc_last_lblock+1 == lblock) {
335 int num = 0;
336
337 if ((ei->i_da_metadata_calc_len % idxs) == 0)
338 num++;
339 if ((ei->i_da_metadata_calc_len % (idxs*idxs)) == 0)
340 num++;
341 if ((ei->i_da_metadata_calc_len % (idxs*idxs*idxs)) == 0) {
342 num++;
343 ei->i_da_metadata_calc_len = 0;
344 } else
345 ei->i_da_metadata_calc_len++;
346 ei->i_da_metadata_calc_last_lblock++;
347 return num;
348 }
349
350
351
352
353
354 ei->i_da_metadata_calc_len = 1;
355 ei->i_da_metadata_calc_last_lblock = lblock;
356 return ext_depth(inode) + 1;
357}
358
359static int
360ext4_ext_max_entries(struct inode *inode, int depth)
361{
362 int max;
363
364 if (depth == ext_depth(inode)) {
365 if (depth == 0)
366 max = ext4_ext_space_root(inode, 1);
367 else
368 max = ext4_ext_space_root_idx(inode, 1);
369 } else {
370 if (depth == 0)
371 max = ext4_ext_space_block(inode, 1);
372 else
373 max = ext4_ext_space_block_idx(inode, 1);
374 }
375
376 return max;
377}
378
379static int ext4_valid_extent(struct inode *inode, struct ext4_extent *ext)
380{
381 ext4_fsblk_t block = ext4_ext_pblock(ext);
382 int len = ext4_ext_get_actual_len(ext);
383 ext4_lblk_t lblock = le32_to_cpu(ext->ee_block);
384
385
386
387
388
389
390 if (lblock + len <= lblock)
391 return 0;
392 return ext4_data_block_valid(EXT4_SB(inode->i_sb), block, len);
393}
394
395static int ext4_valid_extent_idx(struct inode *inode,
396 struct ext4_extent_idx *ext_idx)
397{
398 ext4_fsblk_t block = ext4_idx_pblock(ext_idx);
399
400 return ext4_data_block_valid(EXT4_SB(inode->i_sb), block, 1);
401}
402
403static int ext4_valid_extent_entries(struct inode *inode,
404 struct ext4_extent_header *eh,
405 int depth)
406{
407 unsigned short entries;
408 if (eh->eh_entries == 0)
409 return 1;
410
411 entries = le16_to_cpu(eh->eh_entries);
412
413 if (depth == 0) {
414
415 struct ext4_extent *ext = EXT_FIRST_EXTENT(eh);
416 struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es;
417 ext4_fsblk_t pblock = 0;
418 ext4_lblk_t lblock = 0;
419 ext4_lblk_t prev = 0;
420 int len = 0;
421 while (entries) {
422 if (!ext4_valid_extent(inode, ext))
423 return 0;
424
425
426 lblock = le32_to_cpu(ext->ee_block);
427 len = ext4_ext_get_actual_len(ext);
428 if ((lblock <= prev) && prev) {
429 pblock = ext4_ext_pblock(ext);
430 es->s_last_error_block = cpu_to_le64(pblock);
431 return 0;
432 }
433 ext++;
434 entries--;
435 prev = lblock + len - 1;
436 }
437 } else {
438 struct ext4_extent_idx *ext_idx = EXT_FIRST_INDEX(eh);
439 while (entries) {
440 if (!ext4_valid_extent_idx(inode, ext_idx))
441 return 0;
442 ext_idx++;
443 entries--;
444 }
445 }
446 return 1;
447}
448
449static int __ext4_ext_check(const char *function, unsigned int line,
450 struct inode *inode, struct ext4_extent_header *eh,
451 int depth, ext4_fsblk_t pblk)
452{
453 const char *error_msg;
454 int max = 0, err = -EFSCORRUPTED;
455
456 if (unlikely(eh->eh_magic != EXT4_EXT_MAGIC)) {
457 error_msg = "invalid magic";
458 goto corrupted;
459 }
460 if (unlikely(le16_to_cpu(eh->eh_depth) != depth)) {
461 error_msg = "unexpected eh_depth";
462 goto corrupted;
463 }
464 if (unlikely(eh->eh_max == 0)) {
465 error_msg = "invalid eh_max";
466 goto corrupted;
467 }
468 max = ext4_ext_max_entries(inode, depth);
469 if (unlikely(le16_to_cpu(eh->eh_max) > max)) {
470 error_msg = "too large eh_max";
471 goto corrupted;
472 }
473 if (unlikely(le16_to_cpu(eh->eh_entries) > le16_to_cpu(eh->eh_max))) {
474 error_msg = "invalid eh_entries";
475 goto corrupted;
476 }
477 if (!ext4_valid_extent_entries(inode, eh, depth)) {
478 error_msg = "invalid extent entries";
479 goto corrupted;
480 }
481 if (unlikely(depth > 32)) {
482 error_msg = "too large eh_depth";
483 goto corrupted;
484 }
485
486 if (ext_depth(inode) != depth &&
487 !ext4_extent_block_csum_verify(inode, eh)) {
488 error_msg = "extent tree corrupted";
489 err = -EFSBADCRC;
490 goto corrupted;
491 }
492 return 0;
493
494corrupted:
495 ext4_error_inode(inode, function, line, 0,
496 "pblk %llu bad header/extent: %s - magic %x, "
497 "entries %u, max %u(%u), depth %u(%u)",
498 (unsigned long long) pblk, error_msg,
499 le16_to_cpu(eh->eh_magic),
500 le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max),
501 max, le16_to_cpu(eh->eh_depth), depth);
502 return err;
503}
504
505#define ext4_ext_check(inode, eh, depth, pblk) \
506 __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk))
507
508int ext4_ext_check_inode(struct inode *inode)
509{
510 return ext4_ext_check(inode, ext_inode_hdr(inode), ext_depth(inode), 0);
511}
512
513static struct buffer_head *
514__read_extent_tree_block(const char *function, unsigned int line,
515 struct inode *inode, ext4_fsblk_t pblk, int depth,
516 int flags)
517{
518 struct buffer_head *bh;
519 int err;
520
521 bh = sb_getblk_gfp(inode->i_sb, pblk, __GFP_MOVABLE | GFP_NOFS);
522 if (unlikely(!bh))
523 return ERR_PTR(-ENOMEM);
524
525 if (!bh_uptodate_or_lock(bh)) {
526 trace_ext4_ext_load_extent(inode, pblk, _RET_IP_);
527 err = bh_submit_read(bh);
528 if (err < 0)
529 goto errout;
530 }
531 if (buffer_verified(bh) && !(flags & EXT4_EX_FORCE_CACHE))
532 return bh;
533 err = __ext4_ext_check(function, line, inode,
534 ext_block_hdr(bh), depth, pblk);
535 if (err)
536 goto errout;
537 set_buffer_verified(bh);
538
539
540
541 if (!(flags & EXT4_EX_NOCACHE) && depth == 0) {
542 struct ext4_extent_header *eh = ext_block_hdr(bh);
543 struct ext4_extent *ex = EXT_FIRST_EXTENT(eh);
544 ext4_lblk_t prev = 0;
545 int i;
546
547 for (i = le16_to_cpu(eh->eh_entries); i > 0; i--, ex++) {
548 unsigned int status = EXTENT_STATUS_WRITTEN;
549 ext4_lblk_t lblk = le32_to_cpu(ex->ee_block);
550 int len = ext4_ext_get_actual_len(ex);
551
552 if (prev && (prev != lblk))
553 ext4_es_cache_extent(inode, prev,
554 lblk - prev, ~0,
555 EXTENT_STATUS_HOLE);
556
557 if (ext4_ext_is_unwritten(ex))
558 status = EXTENT_STATUS_UNWRITTEN;
559 ext4_es_cache_extent(inode, lblk, len,
560 ext4_ext_pblock(ex), status);
561 prev = lblk + len;
562 }
563 }
564 return bh;
565errout:
566 put_bh(bh);
567 return ERR_PTR(err);
568
569}
570
571#define read_extent_tree_block(inode, pblk, depth, flags) \
572 __read_extent_tree_block(__func__, __LINE__, (inode), (pblk), \
573 (depth), (flags))
574
575
576
577
578
579int ext4_ext_precache(struct inode *inode)
580{
581 struct ext4_inode_info *ei = EXT4_I(inode);
582 struct ext4_ext_path *path = NULL;
583 struct buffer_head *bh;
584 int i = 0, depth, ret = 0;
585
586 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
587 return 0;
588
589 down_read(&ei->i_data_sem);
590 depth = ext_depth(inode);
591
592 path = kzalloc(sizeof(struct ext4_ext_path) * (depth + 1),
593 GFP_NOFS);
594 if (path == NULL) {
595 up_read(&ei->i_data_sem);
596 return -ENOMEM;
597 }
598
599
600 if (depth == 0)
601 goto out;
602 path[0].p_hdr = ext_inode_hdr(inode);
603 ret = ext4_ext_check(inode, path[0].p_hdr, depth, 0);
604 if (ret)
605 goto out;
606 path[0].p_idx = EXT_FIRST_INDEX(path[0].p_hdr);
607 while (i >= 0) {
608
609
610
611
612 if ((i == depth) ||
613 path[i].p_idx > EXT_LAST_INDEX(path[i].p_hdr)) {
614 brelse(path[i].p_bh);
615 path[i].p_bh = NULL;
616 i--;
617 continue;
618 }
619 bh = read_extent_tree_block(inode,
620 ext4_idx_pblock(path[i].p_idx++),
621 depth - i - 1,
622 EXT4_EX_FORCE_CACHE);
623 if (IS_ERR(bh)) {
624 ret = PTR_ERR(bh);
625 break;
626 }
627 i++;
628 path[i].p_bh = bh;
629 path[i].p_hdr = ext_block_hdr(bh);
630 path[i].p_idx = EXT_FIRST_INDEX(path[i].p_hdr);
631 }
632 ext4_set_inode_state(inode, EXT4_STATE_EXT_PRECACHED);
633out:
634 up_read(&ei->i_data_sem);
635 ext4_ext_drop_refs(path);
636 kfree(path);
637 return ret;
638}
639
640#ifdef EXT_DEBUG
641static void ext4_ext_show_path(struct inode *inode, struct ext4_ext_path *path)
642{
643 int k, l = path->p_depth;
644
645 ext_debug("path:");
646 for (k = 0; k <= l; k++, path++) {
647 if (path->p_idx) {
648 ext_debug(" %d->%llu", le32_to_cpu(path->p_idx->ei_block),
649 ext4_idx_pblock(path->p_idx));
650 } else if (path->p_ext) {
651 ext_debug(" %d:[%d]%d:%llu ",
652 le32_to_cpu(path->p_ext->ee_block),
653 ext4_ext_is_unwritten(path->p_ext),
654 ext4_ext_get_actual_len(path->p_ext),
655 ext4_ext_pblock(path->p_ext));
656 } else
657 ext_debug(" []");
658 }
659 ext_debug("\n");
660}
661
662static void ext4_ext_show_leaf(struct inode *inode, struct ext4_ext_path *path)
663{
664 int depth = ext_depth(inode);
665 struct ext4_extent_header *eh;
666 struct ext4_extent *ex;
667 int i;
668
669 if (!path)
670 return;
671
672 eh = path[depth].p_hdr;
673 ex = EXT_FIRST_EXTENT(eh);
674
675 ext_debug("Displaying leaf extents for inode %lu\n", inode->i_ino);
676
677 for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ex++) {
678 ext_debug("%d:[%d]%d:%llu ", le32_to_cpu(ex->ee_block),
679 ext4_ext_is_unwritten(ex),
680 ext4_ext_get_actual_len(ex), ext4_ext_pblock(ex));
681 }
682 ext_debug("\n");
683}
684
685static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path,
686 ext4_fsblk_t newblock, int level)
687{
688 int depth = ext_depth(inode);
689 struct ext4_extent *ex;
690
691 if (depth != level) {
692 struct ext4_extent_idx *idx;
693 idx = path[level].p_idx;
694 while (idx <= EXT_MAX_INDEX(path[level].p_hdr)) {
695 ext_debug("%d: move %d:%llu in new index %llu\n", level,
696 le32_to_cpu(idx->ei_block),
697 ext4_idx_pblock(idx),
698 newblock);
699 idx++;
700 }
701
702 return;
703 }
704
705 ex = path[depth].p_ext;
706 while (ex <= EXT_MAX_EXTENT(path[depth].p_hdr)) {
707 ext_debug("move %d:%llu:[%d]%d in new leaf %llu\n",
708 le32_to_cpu(ex->ee_block),
709 ext4_ext_pblock(ex),
710 ext4_ext_is_unwritten(ex),
711 ext4_ext_get_actual_len(ex),
712 newblock);
713 ex++;
714 }
715}
716
717#else
718#define ext4_ext_show_path(inode, path)
719#define ext4_ext_show_leaf(inode, path)
720#define ext4_ext_show_move(inode, path, newblock, level)
721#endif
722
723void ext4_ext_drop_refs(struct ext4_ext_path *path)
724{
725 int depth, i;
726
727 if (!path)
728 return;
729 depth = path->p_depth;
730 for (i = 0; i <= depth; i++, path++)
731 if (path->p_bh) {
732 brelse(path->p_bh);
733 path->p_bh = NULL;
734 }
735}
736
737
738
739
740
741
742static void
743ext4_ext_binsearch_idx(struct inode *inode,
744 struct ext4_ext_path *path, ext4_lblk_t block)
745{
746 struct ext4_extent_header *eh = path->p_hdr;
747 struct ext4_extent_idx *r, *l, *m;
748
749
750 ext_debug("binsearch for %u(idx): ", block);
751
752 l = EXT_FIRST_INDEX(eh) + 1;
753 r = EXT_LAST_INDEX(eh);
754 while (l <= r) {
755 m = l + (r - l) / 2;
756 if (block < le32_to_cpu(m->ei_block))
757 r = m - 1;
758 else
759 l = m + 1;
760 ext_debug("%p(%u):%p(%u):%p(%u) ", l, le32_to_cpu(l->ei_block),
761 m, le32_to_cpu(m->ei_block),
762 r, le32_to_cpu(r->ei_block));
763 }
764
765 path->p_idx = l - 1;
766 ext_debug(" -> %u->%lld ", le32_to_cpu(path->p_idx->ei_block),
767 ext4_idx_pblock(path->p_idx));
768
769#ifdef CHECK_BINSEARCH
770 {
771 struct ext4_extent_idx *chix, *ix;
772 int k;
773
774 chix = ix = EXT_FIRST_INDEX(eh);
775 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ix++) {
776 if (k != 0 &&
777 le32_to_cpu(ix->ei_block) <= le32_to_cpu(ix[-1].ei_block)) {
778 printk(KERN_DEBUG "k=%d, ix=0x%p, "
779 "first=0x%p\n", k,
780 ix, EXT_FIRST_INDEX(eh));
781 printk(KERN_DEBUG "%u <= %u\n",
782 le32_to_cpu(ix->ei_block),
783 le32_to_cpu(ix[-1].ei_block));
784 }
785 BUG_ON(k && le32_to_cpu(ix->ei_block)
786 <= le32_to_cpu(ix[-1].ei_block));
787 if (block < le32_to_cpu(ix->ei_block))
788 break;
789 chix = ix;
790 }
791 BUG_ON(chix != path->p_idx);
792 }
793#endif
794
795}
796
797
798
799
800
801
802static void
803ext4_ext_binsearch(struct inode *inode,
804 struct ext4_ext_path *path, ext4_lblk_t block)
805{
806 struct ext4_extent_header *eh = path->p_hdr;
807 struct ext4_extent *r, *l, *m;
808
809 if (eh->eh_entries == 0) {
810
811
812
813
814 return;
815 }
816
817 ext_debug("binsearch for %u: ", block);
818
819 l = EXT_FIRST_EXTENT(eh) + 1;
820 r = EXT_LAST_EXTENT(eh);
821
822 while (l <= r) {
823 m = l + (r - l) / 2;
824 if (block < le32_to_cpu(m->ee_block))
825 r = m - 1;
826 else
827 l = m + 1;
828 ext_debug("%p(%u):%p(%u):%p(%u) ", l, le32_to_cpu(l->ee_block),
829 m, le32_to_cpu(m->ee_block),
830 r, le32_to_cpu(r->ee_block));
831 }
832
833 path->p_ext = l - 1;
834 ext_debug(" -> %d:%llu:[%d]%d ",
835 le32_to_cpu(path->p_ext->ee_block),
836 ext4_ext_pblock(path->p_ext),
837 ext4_ext_is_unwritten(path->p_ext),
838 ext4_ext_get_actual_len(path->p_ext));
839
840#ifdef CHECK_BINSEARCH
841 {
842 struct ext4_extent *chex, *ex;
843 int k;
844
845 chex = ex = EXT_FIRST_EXTENT(eh);
846 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ex++) {
847 BUG_ON(k && le32_to_cpu(ex->ee_block)
848 <= le32_to_cpu(ex[-1].ee_block));
849 if (block < le32_to_cpu(ex->ee_block))
850 break;
851 chex = ex;
852 }
853 BUG_ON(chex != path->p_ext);
854 }
855#endif
856
857}
858
859int ext4_ext_tree_init(handle_t *handle, struct inode *inode)
860{
861 struct ext4_extent_header *eh;
862
863 eh = ext_inode_hdr(inode);
864 eh->eh_depth = 0;
865 eh->eh_entries = 0;
866 eh->eh_magic = EXT4_EXT_MAGIC;
867 eh->eh_max = cpu_to_le16(ext4_ext_space_root(inode, 0));
868 ext4_mark_inode_dirty(handle, inode);
869 return 0;
870}
871
872struct ext4_ext_path *
873ext4_find_extent(struct inode *inode, ext4_lblk_t block,
874 struct ext4_ext_path **orig_path, int flags)
875{
876 struct ext4_extent_header *eh;
877 struct buffer_head *bh;
878 struct ext4_ext_path *path = orig_path ? *orig_path : NULL;
879 short int depth, i, ppos = 0;
880 int ret;
881
882 eh = ext_inode_hdr(inode);
883 depth = ext_depth(inode);
884
885 if (path) {
886 ext4_ext_drop_refs(path);
887 if (depth > path[0].p_maxdepth) {
888 kfree(path);
889 *orig_path = path = NULL;
890 }
891 }
892 if (!path) {
893
894 path = kzalloc(sizeof(struct ext4_ext_path) * (depth + 2),
895 GFP_NOFS);
896 if (unlikely(!path))
897 return ERR_PTR(-ENOMEM);
898 path[0].p_maxdepth = depth + 1;
899 }
900 path[0].p_hdr = eh;
901 path[0].p_bh = NULL;
902
903 i = depth;
904
905 while (i) {
906 ext_debug("depth %d: num %d, max %d\n",
907 ppos, le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
908
909 ext4_ext_binsearch_idx(inode, path + ppos, block);
910 path[ppos].p_block = ext4_idx_pblock(path[ppos].p_idx);
911 path[ppos].p_depth = i;
912 path[ppos].p_ext = NULL;
913
914 bh = read_extent_tree_block(inode, path[ppos].p_block, --i,
915 flags);
916 if (IS_ERR(bh)) {
917 ret = PTR_ERR(bh);
918 goto err;
919 }
920
921 eh = ext_block_hdr(bh);
922 ppos++;
923 path[ppos].p_bh = bh;
924 path[ppos].p_hdr = eh;
925 }
926
927 path[ppos].p_depth = i;
928 path[ppos].p_ext = NULL;
929 path[ppos].p_idx = NULL;
930
931
932 ext4_ext_binsearch(inode, path + ppos, block);
933
934 if (path[ppos].p_ext)
935 path[ppos].p_block = ext4_ext_pblock(path[ppos].p_ext);
936
937 ext4_ext_show_path(inode, path);
938
939 return path;
940
941err:
942 ext4_ext_drop_refs(path);
943 kfree(path);
944 if (orig_path)
945 *orig_path = NULL;
946 return ERR_PTR(ret);
947}
948
949
950
951
952
953
954static int ext4_ext_insert_index(handle_t *handle, struct inode *inode,
955 struct ext4_ext_path *curp,
956 int logical, ext4_fsblk_t ptr)
957{
958 struct ext4_extent_idx *ix;
959 int len, err;
960
961 err = ext4_ext_get_access(handle, inode, curp);
962 if (err)
963 return err;
964
965 if (unlikely(logical == le32_to_cpu(curp->p_idx->ei_block))) {
966 EXT4_ERROR_INODE(inode,
967 "logical %d == ei_block %d!",
968 logical, le32_to_cpu(curp->p_idx->ei_block));
969 return -EFSCORRUPTED;
970 }
971
972 if (unlikely(le16_to_cpu(curp->p_hdr->eh_entries)
973 >= le16_to_cpu(curp->p_hdr->eh_max))) {
974 EXT4_ERROR_INODE(inode,
975 "eh_entries %d >= eh_max %d!",
976 le16_to_cpu(curp->p_hdr->eh_entries),
977 le16_to_cpu(curp->p_hdr->eh_max));
978 return -EFSCORRUPTED;
979 }
980
981 if (logical > le32_to_cpu(curp->p_idx->ei_block)) {
982
983 ext_debug("insert new index %d after: %llu\n", logical, ptr);
984 ix = curp->p_idx + 1;
985 } else {
986
987 ext_debug("insert new index %d before: %llu\n", logical, ptr);
988 ix = curp->p_idx;
989 }
990
991 len = EXT_LAST_INDEX(curp->p_hdr) - ix + 1;
992 BUG_ON(len < 0);
993 if (len > 0) {
994 ext_debug("insert new index %d: "
995 "move %d indices from 0x%p to 0x%p\n",
996 logical, len, ix, ix + 1);
997 memmove(ix + 1, ix, len * sizeof(struct ext4_extent_idx));
998 }
999
1000 if (unlikely(ix > EXT_MAX_INDEX(curp->p_hdr))) {
1001 EXT4_ERROR_INODE(inode, "ix > EXT_MAX_INDEX!");
1002 return -EFSCORRUPTED;
1003 }
1004
1005 ix->ei_block = cpu_to_le32(logical);
1006 ext4_idx_store_pblock(ix, ptr);
1007 le16_add_cpu(&curp->p_hdr->eh_entries, 1);
1008
1009 if (unlikely(ix > EXT_LAST_INDEX(curp->p_hdr))) {
1010 EXT4_ERROR_INODE(inode, "ix > EXT_LAST_INDEX!");
1011 return -EFSCORRUPTED;
1012 }
1013
1014 err = ext4_ext_dirty(handle, inode, curp);
1015 ext4_std_error(inode->i_sb, err);
1016
1017 return err;
1018}
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030static int ext4_ext_split(handle_t *handle, struct inode *inode,
1031 unsigned int flags,
1032 struct ext4_ext_path *path,
1033 struct ext4_extent *newext, int at)
1034{
1035 struct buffer_head *bh = NULL;
1036 int depth = ext_depth(inode);
1037 struct ext4_extent_header *neh;
1038 struct ext4_extent_idx *fidx;
1039 int i = at, k, m, a;
1040 ext4_fsblk_t newblock, oldblock;
1041 __le32 border;
1042 ext4_fsblk_t *ablocks = NULL;
1043 int err = 0;
1044
1045
1046
1047
1048
1049
1050 if (unlikely(path[depth].p_ext > EXT_MAX_EXTENT(path[depth].p_hdr))) {
1051 EXT4_ERROR_INODE(inode, "p_ext > EXT_MAX_EXTENT!");
1052 return -EFSCORRUPTED;
1053 }
1054 if (path[depth].p_ext != EXT_MAX_EXTENT(path[depth].p_hdr)) {
1055 border = path[depth].p_ext[1].ee_block;
1056 ext_debug("leaf will be split."
1057 " next leaf starts at %d\n",
1058 le32_to_cpu(border));
1059 } else {
1060 border = newext->ee_block;
1061 ext_debug("leaf will be added."
1062 " next leaf starts at %d\n",
1063 le32_to_cpu(border));
1064 }
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078 ablocks = kzalloc(sizeof(ext4_fsblk_t) * depth, GFP_NOFS);
1079 if (!ablocks)
1080 return -ENOMEM;
1081
1082
1083 ext_debug("allocate %d blocks for indexes/leaf\n", depth - at);
1084 for (a = 0; a < depth - at; a++) {
1085 newblock = ext4_ext_new_meta_block(handle, inode, path,
1086 newext, &err, flags);
1087 if (newblock == 0)
1088 goto cleanup;
1089 ablocks[a] = newblock;
1090 }
1091
1092
1093 newblock = ablocks[--a];
1094 if (unlikely(newblock == 0)) {
1095 EXT4_ERROR_INODE(inode, "newblock == 0!");
1096 err = -EFSCORRUPTED;
1097 goto cleanup;
1098 }
1099 bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS);
1100 if (unlikely(!bh)) {
1101 err = -ENOMEM;
1102 goto cleanup;
1103 }
1104 lock_buffer(bh);
1105
1106 err = ext4_journal_get_create_access(handle, bh);
1107 if (err)
1108 goto cleanup;
1109
1110 neh = ext_block_hdr(bh);
1111 neh->eh_entries = 0;
1112 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
1113 neh->eh_magic = EXT4_EXT_MAGIC;
1114 neh->eh_depth = 0;
1115
1116
1117 if (unlikely(path[depth].p_hdr->eh_entries !=
1118 path[depth].p_hdr->eh_max)) {
1119 EXT4_ERROR_INODE(inode, "eh_entries %d != eh_max %d!",
1120 path[depth].p_hdr->eh_entries,
1121 path[depth].p_hdr->eh_max);
1122 err = -EFSCORRUPTED;
1123 goto cleanup;
1124 }
1125
1126 m = EXT_MAX_EXTENT(path[depth].p_hdr) - path[depth].p_ext++;
1127 ext4_ext_show_move(inode, path, newblock, depth);
1128 if (m) {
1129 struct ext4_extent *ex;
1130 ex = EXT_FIRST_EXTENT(neh);
1131 memmove(ex, path[depth].p_ext, sizeof(struct ext4_extent) * m);
1132 le16_add_cpu(&neh->eh_entries, m);
1133 }
1134
1135 ext4_extent_block_csum_set(inode, neh);
1136 set_buffer_uptodate(bh);
1137 unlock_buffer(bh);
1138
1139 err = ext4_handle_dirty_metadata(handle, inode, bh);
1140 if (err)
1141 goto cleanup;
1142 brelse(bh);
1143 bh = NULL;
1144
1145
1146 if (m) {
1147 err = ext4_ext_get_access(handle, inode, path + depth);
1148 if (err)
1149 goto cleanup;
1150 le16_add_cpu(&path[depth].p_hdr->eh_entries, -m);
1151 err = ext4_ext_dirty(handle, inode, path + depth);
1152 if (err)
1153 goto cleanup;
1154
1155 }
1156
1157
1158 k = depth - at - 1;
1159 if (unlikely(k < 0)) {
1160 EXT4_ERROR_INODE(inode, "k %d < 0!", k);
1161 err = -EFSCORRUPTED;
1162 goto cleanup;
1163 }
1164 if (k)
1165 ext_debug("create %d intermediate indices\n", k);
1166
1167
1168 i = depth - 1;
1169 while (k--) {
1170 oldblock = newblock;
1171 newblock = ablocks[--a];
1172 bh = sb_getblk(inode->i_sb, newblock);
1173 if (unlikely(!bh)) {
1174 err = -ENOMEM;
1175 goto cleanup;
1176 }
1177 lock_buffer(bh);
1178
1179 err = ext4_journal_get_create_access(handle, bh);
1180 if (err)
1181 goto cleanup;
1182
1183 neh = ext_block_hdr(bh);
1184 neh->eh_entries = cpu_to_le16(1);
1185 neh->eh_magic = EXT4_EXT_MAGIC;
1186 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
1187 neh->eh_depth = cpu_to_le16(depth - i);
1188 fidx = EXT_FIRST_INDEX(neh);
1189 fidx->ei_block = border;
1190 ext4_idx_store_pblock(fidx, oldblock);
1191
1192 ext_debug("int.index at %d (block %llu): %u -> %llu\n",
1193 i, newblock, le32_to_cpu(border), oldblock);
1194
1195
1196 if (unlikely(EXT_MAX_INDEX(path[i].p_hdr) !=
1197 EXT_LAST_INDEX(path[i].p_hdr))) {
1198 EXT4_ERROR_INODE(inode,
1199 "EXT_MAX_INDEX != EXT_LAST_INDEX ee_block %d!",
1200 le32_to_cpu(path[i].p_ext->ee_block));
1201 err = -EFSCORRUPTED;
1202 goto cleanup;
1203 }
1204
1205 m = EXT_MAX_INDEX(path[i].p_hdr) - path[i].p_idx++;
1206 ext_debug("cur 0x%p, last 0x%p\n", path[i].p_idx,
1207 EXT_MAX_INDEX(path[i].p_hdr));
1208 ext4_ext_show_move(inode, path, newblock, i);
1209 if (m) {
1210 memmove(++fidx, path[i].p_idx,
1211 sizeof(struct ext4_extent_idx) * m);
1212 le16_add_cpu(&neh->eh_entries, m);
1213 }
1214 ext4_extent_block_csum_set(inode, neh);
1215 set_buffer_uptodate(bh);
1216 unlock_buffer(bh);
1217
1218 err = ext4_handle_dirty_metadata(handle, inode, bh);
1219 if (err)
1220 goto cleanup;
1221 brelse(bh);
1222 bh = NULL;
1223
1224
1225 if (m) {
1226 err = ext4_ext_get_access(handle, inode, path + i);
1227 if (err)
1228 goto cleanup;
1229 le16_add_cpu(&path[i].p_hdr->eh_entries, -m);
1230 err = ext4_ext_dirty(handle, inode, path + i);
1231 if (err)
1232 goto cleanup;
1233 }
1234
1235 i--;
1236 }
1237
1238
1239 err = ext4_ext_insert_index(handle, inode, path + at,
1240 le32_to_cpu(border), newblock);
1241
1242cleanup:
1243 if (bh) {
1244 if (buffer_locked(bh))
1245 unlock_buffer(bh);
1246 brelse(bh);
1247 }
1248
1249 if (err) {
1250
1251 for (i = 0; i < depth; i++) {
1252 if (!ablocks[i])
1253 continue;
1254 ext4_free_blocks(handle, inode, NULL, ablocks[i], 1,
1255 EXT4_FREE_BLOCKS_METADATA);
1256 }
1257 }
1258 kfree(ablocks);
1259
1260 return err;
1261}
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271static int ext4_ext_grow_indepth(handle_t *handle, struct inode *inode,
1272 unsigned int flags)
1273{
1274 struct ext4_extent_header *neh;
1275 struct buffer_head *bh;
1276 ext4_fsblk_t newblock, goal = 0;
1277 struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es;
1278 int err = 0;
1279
1280
1281 if (ext_depth(inode))
1282 goal = ext4_idx_pblock(EXT_FIRST_INDEX(ext_inode_hdr(inode)));
1283 if (goal > le32_to_cpu(es->s_first_data_block)) {
1284 flags |= EXT4_MB_HINT_TRY_GOAL;
1285 goal--;
1286 } else
1287 goal = ext4_inode_to_goal_block(inode);
1288 newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
1289 NULL, &err);
1290 if (newblock == 0)
1291 return err;
1292
1293 bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS);
1294 if (unlikely(!bh))
1295 return -ENOMEM;
1296 lock_buffer(bh);
1297
1298 err = ext4_journal_get_create_access(handle, bh);
1299 if (err) {
1300 unlock_buffer(bh);
1301 goto out;
1302 }
1303
1304
1305 memmove(bh->b_data, EXT4_I(inode)->i_data,
1306 sizeof(EXT4_I(inode)->i_data));
1307
1308
1309 neh = ext_block_hdr(bh);
1310
1311
1312 if (ext_depth(inode))
1313 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
1314 else
1315 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
1316 neh->eh_magic = EXT4_EXT_MAGIC;
1317 ext4_extent_block_csum_set(inode, neh);
1318 set_buffer_uptodate(bh);
1319 unlock_buffer(bh);
1320
1321 err = ext4_handle_dirty_metadata(handle, inode, bh);
1322 if (err)
1323 goto out;
1324
1325
1326 neh = ext_inode_hdr(inode);
1327 neh->eh_entries = cpu_to_le16(1);
1328 ext4_idx_store_pblock(EXT_FIRST_INDEX(neh), newblock);
1329 if (neh->eh_depth == 0) {
1330
1331 neh->eh_max = cpu_to_le16(ext4_ext_space_root_idx(inode, 0));
1332 EXT_FIRST_INDEX(neh)->ei_block =
1333 EXT_FIRST_EXTENT(neh)->ee_block;
1334 }
1335 ext_debug("new root: num %d(%d), lblock %d, ptr %llu\n",
1336 le16_to_cpu(neh->eh_entries), le16_to_cpu(neh->eh_max),
1337 le32_to_cpu(EXT_FIRST_INDEX(neh)->ei_block),
1338 ext4_idx_pblock(EXT_FIRST_INDEX(neh)));
1339
1340 le16_add_cpu(&neh->eh_depth, 1);
1341 ext4_mark_inode_dirty(handle, inode);
1342out:
1343 brelse(bh);
1344
1345 return err;
1346}
1347
1348
1349
1350
1351
1352
1353static int ext4_ext_create_new_leaf(handle_t *handle, struct inode *inode,
1354 unsigned int mb_flags,
1355 unsigned int gb_flags,
1356 struct ext4_ext_path **ppath,
1357 struct ext4_extent *newext)
1358{
1359 struct ext4_ext_path *path = *ppath;
1360 struct ext4_ext_path *curp;
1361 int depth, i, err = 0;
1362
1363repeat:
1364 i = depth = ext_depth(inode);
1365
1366
1367 curp = path + depth;
1368 while (i > 0 && !EXT_HAS_FREE_INDEX(curp)) {
1369 i--;
1370 curp--;
1371 }
1372
1373
1374
1375 if (EXT_HAS_FREE_INDEX(curp)) {
1376
1377
1378 err = ext4_ext_split(handle, inode, mb_flags, path, newext, i);
1379 if (err)
1380 goto out;
1381
1382
1383 path = ext4_find_extent(inode,
1384 (ext4_lblk_t)le32_to_cpu(newext->ee_block),
1385 ppath, gb_flags);
1386 if (IS_ERR(path))
1387 err = PTR_ERR(path);
1388 } else {
1389
1390 err = ext4_ext_grow_indepth(handle, inode, mb_flags);
1391 if (err)
1392 goto out;
1393
1394
1395 path = ext4_find_extent(inode,
1396 (ext4_lblk_t)le32_to_cpu(newext->ee_block),
1397 ppath, gb_flags);
1398 if (IS_ERR(path)) {
1399 err = PTR_ERR(path);
1400 goto out;
1401 }
1402
1403
1404
1405
1406
1407 depth = ext_depth(inode);
1408 if (path[depth].p_hdr->eh_entries == path[depth].p_hdr->eh_max) {
1409
1410 goto repeat;
1411 }
1412 }
1413
1414out:
1415 return err;
1416}
1417
1418
1419
1420
1421
1422
1423
1424
1425static int ext4_ext_search_left(struct inode *inode,
1426 struct ext4_ext_path *path,
1427 ext4_lblk_t *logical, ext4_fsblk_t *phys)
1428{
1429 struct ext4_extent_idx *ix;
1430 struct ext4_extent *ex;
1431 int depth, ee_len;
1432
1433 if (unlikely(path == NULL)) {
1434 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
1435 return -EFSCORRUPTED;
1436 }
1437 depth = path->p_depth;
1438 *phys = 0;
1439
1440 if (depth == 0 && path->p_ext == NULL)
1441 return 0;
1442
1443
1444
1445
1446
1447 ex = path[depth].p_ext;
1448 ee_len = ext4_ext_get_actual_len(ex);
1449 if (*logical < le32_to_cpu(ex->ee_block)) {
1450 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
1451 EXT4_ERROR_INODE(inode,
1452 "EXT_FIRST_EXTENT != ex *logical %d ee_block %d!",
1453 *logical, le32_to_cpu(ex->ee_block));
1454 return -EFSCORRUPTED;
1455 }
1456 while (--depth >= 0) {
1457 ix = path[depth].p_idx;
1458 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
1459 EXT4_ERROR_INODE(inode,
1460 "ix (%d) != EXT_FIRST_INDEX (%d) (depth %d)!",
1461 ix != NULL ? le32_to_cpu(ix->ei_block) : 0,
1462 EXT_FIRST_INDEX(path[depth].p_hdr) != NULL ?
1463 le32_to_cpu(EXT_FIRST_INDEX(path[depth].p_hdr)->ei_block) : 0,
1464 depth);
1465 return -EFSCORRUPTED;
1466 }
1467 }
1468 return 0;
1469 }
1470
1471 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
1472 EXT4_ERROR_INODE(inode,
1473 "logical %d < ee_block %d + ee_len %d!",
1474 *logical, le32_to_cpu(ex->ee_block), ee_len);
1475 return -EFSCORRUPTED;
1476 }
1477
1478 *logical = le32_to_cpu(ex->ee_block) + ee_len - 1;
1479 *phys = ext4_ext_pblock(ex) + ee_len - 1;
1480 return 0;
1481}
1482
1483
1484
1485
1486
1487
1488
1489
1490static int ext4_ext_search_right(struct inode *inode,
1491 struct ext4_ext_path *path,
1492 ext4_lblk_t *logical, ext4_fsblk_t *phys,
1493 struct ext4_extent **ret_ex)
1494{
1495 struct buffer_head *bh = NULL;
1496 struct ext4_extent_header *eh;
1497 struct ext4_extent_idx *ix;
1498 struct ext4_extent *ex;
1499 ext4_fsblk_t block;
1500 int depth;
1501 int ee_len;
1502
1503 if (unlikely(path == NULL)) {
1504 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
1505 return -EFSCORRUPTED;
1506 }
1507 depth = path->p_depth;
1508 *phys = 0;
1509
1510 if (depth == 0 && path->p_ext == NULL)
1511 return 0;
1512
1513
1514
1515
1516
1517 ex = path[depth].p_ext;
1518 ee_len = ext4_ext_get_actual_len(ex);
1519 if (*logical < le32_to_cpu(ex->ee_block)) {
1520 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
1521 EXT4_ERROR_INODE(inode,
1522 "first_extent(path[%d].p_hdr) != ex",
1523 depth);
1524 return -EFSCORRUPTED;
1525 }
1526 while (--depth >= 0) {
1527 ix = path[depth].p_idx;
1528 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
1529 EXT4_ERROR_INODE(inode,
1530 "ix != EXT_FIRST_INDEX *logical %d!",
1531 *logical);
1532 return -EFSCORRUPTED;
1533 }
1534 }
1535 goto found_extent;
1536 }
1537
1538 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
1539 EXT4_ERROR_INODE(inode,
1540 "logical %d < ee_block %d + ee_len %d!",
1541 *logical, le32_to_cpu(ex->ee_block), ee_len);
1542 return -EFSCORRUPTED;
1543 }
1544
1545 if (ex != EXT_LAST_EXTENT(path[depth].p_hdr)) {
1546
1547 ex++;
1548 goto found_extent;
1549 }
1550
1551
1552 while (--depth >= 0) {
1553 ix = path[depth].p_idx;
1554 if (ix != EXT_LAST_INDEX(path[depth].p_hdr))
1555 goto got_index;
1556 }
1557
1558
1559 return 0;
1560
1561got_index:
1562
1563
1564
1565 ix++;
1566 block = ext4_idx_pblock(ix);
1567 while (++depth < path->p_depth) {
1568
1569 bh = read_extent_tree_block(inode, block,
1570 path->p_depth - depth, 0);
1571 if (IS_ERR(bh))
1572 return PTR_ERR(bh);
1573 eh = ext_block_hdr(bh);
1574 ix = EXT_FIRST_INDEX(eh);
1575 block = ext4_idx_pblock(ix);
1576 put_bh(bh);
1577 }
1578
1579 bh = read_extent_tree_block(inode, block, path->p_depth - depth, 0);
1580 if (IS_ERR(bh))
1581 return PTR_ERR(bh);
1582 eh = ext_block_hdr(bh);
1583 ex = EXT_FIRST_EXTENT(eh);
1584found_extent:
1585 *logical = le32_to_cpu(ex->ee_block);
1586 *phys = ext4_ext_pblock(ex);
1587 *ret_ex = ex;
1588 if (bh)
1589 put_bh(bh);
1590 return 0;
1591}
1592
1593
1594
1595
1596
1597
1598
1599
1600ext4_lblk_t
1601ext4_ext_next_allocated_block(struct ext4_ext_path *path)
1602{
1603 int depth;
1604
1605 BUG_ON(path == NULL);
1606 depth = path->p_depth;
1607
1608 if (depth == 0 && path->p_ext == NULL)
1609 return EXT_MAX_BLOCKS;
1610
1611 while (depth >= 0) {
1612 if (depth == path->p_depth) {
1613
1614 if (path[depth].p_ext &&
1615 path[depth].p_ext !=
1616 EXT_LAST_EXTENT(path[depth].p_hdr))
1617 return le32_to_cpu(path[depth].p_ext[1].ee_block);
1618 } else {
1619
1620 if (path[depth].p_idx !=
1621 EXT_LAST_INDEX(path[depth].p_hdr))
1622 return le32_to_cpu(path[depth].p_idx[1].ei_block);
1623 }
1624 depth--;
1625 }
1626
1627 return EXT_MAX_BLOCKS;
1628}
1629
1630
1631
1632
1633
1634static ext4_lblk_t ext4_ext_next_leaf_block(struct ext4_ext_path *path)
1635{
1636 int depth;
1637
1638 BUG_ON(path == NULL);
1639 depth = path->p_depth;
1640
1641
1642 if (depth == 0)
1643 return EXT_MAX_BLOCKS;
1644
1645
1646 depth--;
1647
1648 while (depth >= 0) {
1649 if (path[depth].p_idx !=
1650 EXT_LAST_INDEX(path[depth].p_hdr))
1651 return (ext4_lblk_t)
1652 le32_to_cpu(path[depth].p_idx[1].ei_block);
1653 depth--;
1654 }
1655
1656 return EXT_MAX_BLOCKS;
1657}
1658
1659
1660
1661
1662
1663
1664
1665static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode,
1666 struct ext4_ext_path *path)
1667{
1668 struct ext4_extent_header *eh;
1669 int depth = ext_depth(inode);
1670 struct ext4_extent *ex;
1671 __le32 border;
1672 int k, err = 0;
1673
1674 eh = path[depth].p_hdr;
1675 ex = path[depth].p_ext;
1676
1677 if (unlikely(ex == NULL || eh == NULL)) {
1678 EXT4_ERROR_INODE(inode,
1679 "ex %p == NULL or eh %p == NULL", ex, eh);
1680 return -EFSCORRUPTED;
1681 }
1682
1683 if (depth == 0) {
1684
1685 return 0;
1686 }
1687
1688 if (ex != EXT_FIRST_EXTENT(eh)) {
1689
1690 return 0;
1691 }
1692
1693
1694
1695
1696 k = depth - 1;
1697 border = path[depth].p_ext->ee_block;
1698 err = ext4_ext_get_access(handle, inode, path + k);
1699 if (err)
1700 return err;
1701 path[k].p_idx->ei_block = border;
1702 err = ext4_ext_dirty(handle, inode, path + k);
1703 if (err)
1704 return err;
1705
1706 while (k--) {
1707
1708 if (path[k+1].p_idx != EXT_FIRST_INDEX(path[k+1].p_hdr))
1709 break;
1710 err = ext4_ext_get_access(handle, inode, path + k);
1711 if (err)
1712 break;
1713 path[k].p_idx->ei_block = border;
1714 err = ext4_ext_dirty(handle, inode, path + k);
1715 if (err)
1716 break;
1717 }
1718
1719 return err;
1720}
1721
1722int
1723ext4_can_extents_be_merged(struct inode *inode, struct ext4_extent *ex1,
1724 struct ext4_extent *ex2)
1725{
1726 unsigned short ext1_ee_len, ext2_ee_len;
1727
1728 if (ext4_ext_is_unwritten(ex1) != ext4_ext_is_unwritten(ex2))
1729 return 0;
1730
1731 ext1_ee_len = ext4_ext_get_actual_len(ex1);
1732 ext2_ee_len = ext4_ext_get_actual_len(ex2);
1733
1734 if (le32_to_cpu(ex1->ee_block) + ext1_ee_len !=
1735 le32_to_cpu(ex2->ee_block))
1736 return 0;
1737
1738
1739
1740
1741
1742
1743 if (ext1_ee_len + ext2_ee_len > EXT_INIT_MAX_LEN)
1744 return 0;
1745
1746
1747
1748
1749
1750
1751 if (ext4_ext_is_unwritten(ex1) &&
1752 (ext4_test_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN) ||
1753 atomic_read(&EXT4_I(inode)->i_unwritten) ||
1754 (ext1_ee_len + ext2_ee_len > EXT_UNWRITTEN_MAX_LEN)))
1755 return 0;
1756#ifdef AGGRESSIVE_TEST
1757 if (ext1_ee_len >= 4)
1758 return 0;
1759#endif
1760
1761 if (ext4_ext_pblock(ex1) + ext1_ee_len == ext4_ext_pblock(ex2))
1762 return 1;
1763 return 0;
1764}
1765
1766
1767
1768
1769
1770
1771
1772
1773static int ext4_ext_try_to_merge_right(struct inode *inode,
1774 struct ext4_ext_path *path,
1775 struct ext4_extent *ex)
1776{
1777 struct ext4_extent_header *eh;
1778 unsigned int depth, len;
1779 int merge_done = 0, unwritten;
1780
1781 depth = ext_depth(inode);
1782 BUG_ON(path[depth].p_hdr == NULL);
1783 eh = path[depth].p_hdr;
1784
1785 while (ex < EXT_LAST_EXTENT(eh)) {
1786 if (!ext4_can_extents_be_merged(inode, ex, ex + 1))
1787 break;
1788
1789 unwritten = ext4_ext_is_unwritten(ex);
1790 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
1791 + ext4_ext_get_actual_len(ex + 1));
1792 if (unwritten)
1793 ext4_ext_mark_unwritten(ex);
1794
1795 if (ex + 1 < EXT_LAST_EXTENT(eh)) {
1796 len = (EXT_LAST_EXTENT(eh) - ex - 1)
1797 * sizeof(struct ext4_extent);
1798 memmove(ex + 1, ex + 2, len);
1799 }
1800 le16_add_cpu(&eh->eh_entries, -1);
1801 merge_done = 1;
1802 WARN_ON(eh->eh_entries == 0);
1803 if (!eh->eh_entries)
1804 EXT4_ERROR_INODE(inode, "eh->eh_entries = 0!");
1805 }
1806
1807 return merge_done;
1808}
1809
1810
1811
1812
1813
1814static void ext4_ext_try_to_merge_up(handle_t *handle,
1815 struct inode *inode,
1816 struct ext4_ext_path *path)
1817{
1818 size_t s;
1819 unsigned max_root = ext4_ext_space_root(inode, 0);
1820 ext4_fsblk_t blk;
1821
1822 if ((path[0].p_depth != 1) ||
1823 (le16_to_cpu(path[0].p_hdr->eh_entries) != 1) ||
1824 (le16_to_cpu(path[1].p_hdr->eh_entries) > max_root))
1825 return;
1826
1827
1828
1829
1830
1831
1832 if (ext4_journal_extend(handle, 2))
1833 return;
1834
1835
1836
1837
1838 blk = ext4_idx_pblock(path[0].p_idx);
1839 s = le16_to_cpu(path[1].p_hdr->eh_entries) *
1840 sizeof(struct ext4_extent_idx);
1841 s += sizeof(struct ext4_extent_header);
1842
1843 path[1].p_maxdepth = path[0].p_maxdepth;
1844 memcpy(path[0].p_hdr, path[1].p_hdr, s);
1845 path[0].p_depth = 0;
1846 path[0].p_ext = EXT_FIRST_EXTENT(path[0].p_hdr) +
1847 (path[1].p_ext - EXT_FIRST_EXTENT(path[1].p_hdr));
1848 path[0].p_hdr->eh_max = cpu_to_le16(max_root);
1849
1850 brelse(path[1].p_bh);
1851 ext4_free_blocks(handle, inode, NULL, blk, 1,
1852 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
1853}
1854
1855
1856
1857
1858
1859static void ext4_ext_try_to_merge(handle_t *handle,
1860 struct inode *inode,
1861 struct ext4_ext_path *path,
1862 struct ext4_extent *ex) {
1863 struct ext4_extent_header *eh;
1864 unsigned int depth;
1865 int merge_done = 0;
1866
1867 depth = ext_depth(inode);
1868 BUG_ON(path[depth].p_hdr == NULL);
1869 eh = path[depth].p_hdr;
1870
1871 if (ex > EXT_FIRST_EXTENT(eh))
1872 merge_done = ext4_ext_try_to_merge_right(inode, path, ex - 1);
1873
1874 if (!merge_done)
1875 (void) ext4_ext_try_to_merge_right(inode, path, ex);
1876
1877 ext4_ext_try_to_merge_up(handle, inode, path);
1878}
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888static unsigned int ext4_ext_check_overlap(struct ext4_sb_info *sbi,
1889 struct inode *inode,
1890 struct ext4_extent *newext,
1891 struct ext4_ext_path *path)
1892{
1893 ext4_lblk_t b1, b2;
1894 unsigned int depth, len1;
1895 unsigned int ret = 0;
1896
1897 b1 = le32_to_cpu(newext->ee_block);
1898 len1 = ext4_ext_get_actual_len(newext);
1899 depth = ext_depth(inode);
1900 if (!path[depth].p_ext)
1901 goto out;
1902 b2 = EXT4_LBLK_CMASK(sbi, le32_to_cpu(path[depth].p_ext->ee_block));
1903
1904
1905
1906
1907
1908 if (b2 < b1) {
1909 b2 = ext4_ext_next_allocated_block(path);
1910 if (b2 == EXT_MAX_BLOCKS)
1911 goto out;
1912 b2 = EXT4_LBLK_CMASK(sbi, b2);
1913 }
1914
1915
1916 if (b1 + len1 < b1) {
1917 len1 = EXT_MAX_BLOCKS - b1;
1918 newext->ee_len = cpu_to_le16(len1);
1919 ret = 1;
1920 }
1921
1922
1923 if (b1 + len1 > b2) {
1924 newext->ee_len = cpu_to_le16(b2 - b1);
1925 ret = 1;
1926 }
1927out:
1928 return ret;
1929}
1930
1931
1932
1933
1934
1935
1936
1937int ext4_ext_insert_extent(handle_t *handle, struct inode *inode,
1938 struct ext4_ext_path **ppath,
1939 struct ext4_extent *newext, int gb_flags)
1940{
1941 struct ext4_ext_path *path = *ppath;
1942 struct ext4_extent_header *eh;
1943 struct ext4_extent *ex, *fex;
1944 struct ext4_extent *nearex;
1945 struct ext4_ext_path *npath = NULL;
1946 int depth, len, err;
1947 ext4_lblk_t next;
1948 int mb_flags = 0, unwritten;
1949
1950 if (gb_flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
1951 mb_flags |= EXT4_MB_DELALLOC_RESERVED;
1952 if (unlikely(ext4_ext_get_actual_len(newext) == 0)) {
1953 EXT4_ERROR_INODE(inode, "ext4_ext_get_actual_len(newext) == 0");
1954 return -EFSCORRUPTED;
1955 }
1956 depth = ext_depth(inode);
1957 ex = path[depth].p_ext;
1958 eh = path[depth].p_hdr;
1959 if (unlikely(path[depth].p_hdr == NULL)) {
1960 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
1961 return -EFSCORRUPTED;
1962 }
1963
1964
1965 if (ex && !(gb_flags & EXT4_GET_BLOCKS_PRE_IO)) {
1966
1967
1968
1969
1970
1971
1972
1973
1974 if (ex < EXT_LAST_EXTENT(eh) &&
1975 (le32_to_cpu(ex->ee_block) +
1976 ext4_ext_get_actual_len(ex) <
1977 le32_to_cpu(newext->ee_block))) {
1978 ex += 1;
1979 goto prepend;
1980 } else if ((ex > EXT_FIRST_EXTENT(eh)) &&
1981 (le32_to_cpu(newext->ee_block) +
1982 ext4_ext_get_actual_len(newext) <
1983 le32_to_cpu(ex->ee_block)))
1984 ex -= 1;
1985
1986
1987 if (ext4_can_extents_be_merged(inode, ex, newext)) {
1988 ext_debug("append [%d]%d block to %u:[%d]%d"
1989 "(from %llu)\n",
1990 ext4_ext_is_unwritten(newext),
1991 ext4_ext_get_actual_len(newext),
1992 le32_to_cpu(ex->ee_block),
1993 ext4_ext_is_unwritten(ex),
1994 ext4_ext_get_actual_len(ex),
1995 ext4_ext_pblock(ex));
1996 err = ext4_ext_get_access(handle, inode,
1997 path + depth);
1998 if (err)
1999 return err;
2000 unwritten = ext4_ext_is_unwritten(ex);
2001 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
2002 + ext4_ext_get_actual_len(newext));
2003 if (unwritten)
2004 ext4_ext_mark_unwritten(ex);
2005 eh = path[depth].p_hdr;
2006 nearex = ex;
2007 goto merge;
2008 }
2009
2010prepend:
2011
2012 if (ext4_can_extents_be_merged(inode, newext, ex)) {
2013 ext_debug("prepend %u[%d]%d block to %u:[%d]%d"
2014 "(from %llu)\n",
2015 le32_to_cpu(newext->ee_block),
2016 ext4_ext_is_unwritten(newext),
2017 ext4_ext_get_actual_len(newext),
2018 le32_to_cpu(ex->ee_block),
2019 ext4_ext_is_unwritten(ex),
2020 ext4_ext_get_actual_len(ex),
2021 ext4_ext_pblock(ex));
2022 err = ext4_ext_get_access(handle, inode,
2023 path + depth);
2024 if (err)
2025 return err;
2026
2027 unwritten = ext4_ext_is_unwritten(ex);
2028 ex->ee_block = newext->ee_block;
2029 ext4_ext_store_pblock(ex, ext4_ext_pblock(newext));
2030 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
2031 + ext4_ext_get_actual_len(newext));
2032 if (unwritten)
2033 ext4_ext_mark_unwritten(ex);
2034 eh = path[depth].p_hdr;
2035 nearex = ex;
2036 goto merge;
2037 }
2038 }
2039
2040 depth = ext_depth(inode);
2041 eh = path[depth].p_hdr;
2042 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max))
2043 goto has_space;
2044
2045
2046 fex = EXT_LAST_EXTENT(eh);
2047 next = EXT_MAX_BLOCKS;
2048 if (le32_to_cpu(newext->ee_block) > le32_to_cpu(fex->ee_block))
2049 next = ext4_ext_next_leaf_block(path);
2050 if (next != EXT_MAX_BLOCKS) {
2051 ext_debug("next leaf block - %u\n", next);
2052 BUG_ON(npath != NULL);
2053 npath = ext4_find_extent(inode, next, NULL, 0);
2054 if (IS_ERR(npath))
2055 return PTR_ERR(npath);
2056 BUG_ON(npath->p_depth != path->p_depth);
2057 eh = npath[depth].p_hdr;
2058 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) {
2059 ext_debug("next leaf isn't full(%d)\n",
2060 le16_to_cpu(eh->eh_entries));
2061 path = npath;
2062 goto has_space;
2063 }
2064 ext_debug("next leaf has no free space(%d,%d)\n",
2065 le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
2066 }
2067
2068
2069
2070
2071
2072 if (gb_flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
2073 mb_flags |= EXT4_MB_USE_RESERVED;
2074 err = ext4_ext_create_new_leaf(handle, inode, mb_flags, gb_flags,
2075 ppath, newext);
2076 if (err)
2077 goto cleanup;
2078 depth = ext_depth(inode);
2079 eh = path[depth].p_hdr;
2080
2081has_space:
2082 nearex = path[depth].p_ext;
2083
2084 err = ext4_ext_get_access(handle, inode, path + depth);
2085 if (err)
2086 goto cleanup;
2087
2088 if (!nearex) {
2089
2090 ext_debug("first extent in the leaf: %u:%llu:[%d]%d\n",
2091 le32_to_cpu(newext->ee_block),
2092 ext4_ext_pblock(newext),
2093 ext4_ext_is_unwritten(newext),
2094 ext4_ext_get_actual_len(newext));
2095 nearex = EXT_FIRST_EXTENT(eh);
2096 } else {
2097 if (le32_to_cpu(newext->ee_block)
2098 > le32_to_cpu(nearex->ee_block)) {
2099
2100 ext_debug("insert %u:%llu:[%d]%d before: "
2101 "nearest %p\n",
2102 le32_to_cpu(newext->ee_block),
2103 ext4_ext_pblock(newext),
2104 ext4_ext_is_unwritten(newext),
2105 ext4_ext_get_actual_len(newext),
2106 nearex);
2107 nearex++;
2108 } else {
2109
2110 BUG_ON(newext->ee_block == nearex->ee_block);
2111 ext_debug("insert %u:%llu:[%d]%d after: "
2112 "nearest %p\n",
2113 le32_to_cpu(newext->ee_block),
2114 ext4_ext_pblock(newext),
2115 ext4_ext_is_unwritten(newext),
2116 ext4_ext_get_actual_len(newext),
2117 nearex);
2118 }
2119 len = EXT_LAST_EXTENT(eh) - nearex + 1;
2120 if (len > 0) {
2121 ext_debug("insert %u:%llu:[%d]%d: "
2122 "move %d extents from 0x%p to 0x%p\n",
2123 le32_to_cpu(newext->ee_block),
2124 ext4_ext_pblock(newext),
2125 ext4_ext_is_unwritten(newext),
2126 ext4_ext_get_actual_len(newext),
2127 len, nearex, nearex + 1);
2128 memmove(nearex + 1, nearex,
2129 len * sizeof(struct ext4_extent));
2130 }
2131 }
2132
2133 le16_add_cpu(&eh->eh_entries, 1);
2134 path[depth].p_ext = nearex;
2135 nearex->ee_block = newext->ee_block;
2136 ext4_ext_store_pblock(nearex, ext4_ext_pblock(newext));
2137 nearex->ee_len = newext->ee_len;
2138
2139merge:
2140
2141 if (!(gb_flags & EXT4_GET_BLOCKS_PRE_IO))
2142 ext4_ext_try_to_merge(handle, inode, path, nearex);
2143
2144
2145
2146 err = ext4_ext_correct_indexes(handle, inode, path);
2147 if (err)
2148 goto cleanup;
2149
2150 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
2151
2152cleanup:
2153 ext4_ext_drop_refs(npath);
2154 kfree(npath);
2155 return err;
2156}
2157
2158static int ext4_fill_fiemap_extents(struct inode *inode,
2159 ext4_lblk_t block, ext4_lblk_t num,
2160 struct fiemap_extent_info *fieinfo)
2161{
2162 struct ext4_ext_path *path = NULL;
2163 struct ext4_extent *ex;
2164 struct extent_status es;
2165 ext4_lblk_t next, next_del, start = 0, end = 0;
2166 ext4_lblk_t last = block + num;
2167 int exists, depth = 0, err = 0;
2168 unsigned int flags = 0;
2169 unsigned char blksize_bits = inode->i_sb->s_blocksize_bits;
2170
2171 while (block < last && block != EXT_MAX_BLOCKS) {
2172 num = last - block;
2173
2174 down_read(&EXT4_I(inode)->i_data_sem);
2175
2176 path = ext4_find_extent(inode, block, &path, 0);
2177 if (IS_ERR(path)) {
2178 up_read(&EXT4_I(inode)->i_data_sem);
2179 err = PTR_ERR(path);
2180 path = NULL;
2181 break;
2182 }
2183
2184 depth = ext_depth(inode);
2185 if (unlikely(path[depth].p_hdr == NULL)) {
2186 up_read(&EXT4_I(inode)->i_data_sem);
2187 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
2188 err = -EFSCORRUPTED;
2189 break;
2190 }
2191 ex = path[depth].p_ext;
2192 next = ext4_ext_next_allocated_block(path);
2193
2194 flags = 0;
2195 exists = 0;
2196 if (!ex) {
2197
2198
2199 start = block;
2200 end = block + num;
2201 } else if (le32_to_cpu(ex->ee_block) > block) {
2202
2203 start = block;
2204 end = le32_to_cpu(ex->ee_block);
2205 if (block + num < end)
2206 end = block + num;
2207 } else if (block >= le32_to_cpu(ex->ee_block)
2208 + ext4_ext_get_actual_len(ex)) {
2209
2210 start = block;
2211 end = block + num;
2212 if (end >= next)
2213 end = next;
2214 } else if (block >= le32_to_cpu(ex->ee_block)) {
2215
2216
2217
2218
2219 start = block;
2220 end = le32_to_cpu(ex->ee_block)
2221 + ext4_ext_get_actual_len(ex);
2222 if (block + num < end)
2223 end = block + num;
2224 exists = 1;
2225 } else {
2226 BUG();
2227 }
2228 BUG_ON(end <= start);
2229
2230 if (!exists) {
2231 es.es_lblk = start;
2232 es.es_len = end - start;
2233 es.es_pblk = 0;
2234 } else {
2235 es.es_lblk = le32_to_cpu(ex->ee_block);
2236 es.es_len = ext4_ext_get_actual_len(ex);
2237 es.es_pblk = ext4_ext_pblock(ex);
2238 if (ext4_ext_is_unwritten(ex))
2239 flags |= FIEMAP_EXTENT_UNWRITTEN;
2240 }
2241
2242
2243
2244
2245
2246
2247 next_del = ext4_find_delayed_extent(inode, &es);
2248 if (!exists && next_del) {
2249 exists = 1;
2250 flags |= (FIEMAP_EXTENT_DELALLOC |
2251 FIEMAP_EXTENT_UNKNOWN);
2252 }
2253 up_read(&EXT4_I(inode)->i_data_sem);
2254
2255 if (unlikely(es.es_len == 0)) {
2256 EXT4_ERROR_INODE(inode, "es.es_len == 0");
2257 err = -EFSCORRUPTED;
2258 break;
2259 }
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272 if (next == next_del && next == EXT_MAX_BLOCKS) {
2273 flags |= FIEMAP_EXTENT_LAST;
2274 if (unlikely(next_del != EXT_MAX_BLOCKS ||
2275 next != EXT_MAX_BLOCKS)) {
2276 EXT4_ERROR_INODE(inode,
2277 "next extent == %u, next "
2278 "delalloc extent = %u",
2279 next, next_del);
2280 err = -EFSCORRUPTED;
2281 break;
2282 }
2283 }
2284
2285 if (exists) {
2286 err = fiemap_fill_next_extent(fieinfo,
2287 (__u64)es.es_lblk << blksize_bits,
2288 (__u64)es.es_pblk << blksize_bits,
2289 (__u64)es.es_len << blksize_bits,
2290 flags);
2291 if (err < 0)
2292 break;
2293 if (err == 1) {
2294 err = 0;
2295 break;
2296 }
2297 }
2298
2299 block = es.es_lblk + es.es_len;
2300 }
2301
2302 ext4_ext_drop_refs(path);
2303 kfree(path);
2304 return err;
2305}
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320static ext4_lblk_t ext4_ext_determine_hole(struct inode *inode,
2321 struct ext4_ext_path *path,
2322 ext4_lblk_t *lblk)
2323{
2324 int depth = ext_depth(inode);
2325 struct ext4_extent *ex;
2326 ext4_lblk_t len;
2327
2328 ex = path[depth].p_ext;
2329 if (ex == NULL) {
2330
2331 *lblk = 0;
2332 len = EXT_MAX_BLOCKS;
2333 } else if (*lblk < le32_to_cpu(ex->ee_block)) {
2334 len = le32_to_cpu(ex->ee_block) - *lblk;
2335 } else if (*lblk >= le32_to_cpu(ex->ee_block)
2336 + ext4_ext_get_actual_len(ex)) {
2337 ext4_lblk_t next;
2338
2339 *lblk = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex);
2340 next = ext4_ext_next_allocated_block(path);
2341 BUG_ON(next == *lblk);
2342 len = next - *lblk;
2343 } else {
2344 BUG();
2345 }
2346 return len;
2347}
2348
2349
2350
2351
2352
2353
2354static void
2355ext4_ext_put_gap_in_cache(struct inode *inode, ext4_lblk_t hole_start,
2356 ext4_lblk_t hole_len)
2357{
2358 struct extent_status es;
2359
2360 ext4_es_find_delayed_extent_range(inode, hole_start,
2361 hole_start + hole_len - 1, &es);
2362 if (es.es_len) {
2363
2364 if (es.es_lblk <= hole_start)
2365 return;
2366 hole_len = min(es.es_lblk - hole_start, hole_len);
2367 }
2368 ext_debug(" -> %u:%u\n", hole_start, hole_len);
2369 ext4_es_insert_extent(inode, hole_start, hole_len, ~0,
2370 EXTENT_STATUS_HOLE);
2371}
2372
2373
2374
2375
2376
2377static int ext4_ext_rm_idx(handle_t *handle, struct inode *inode,
2378 struct ext4_ext_path *path, int depth)
2379{
2380 int err;
2381 ext4_fsblk_t leaf;
2382
2383
2384 depth--;
2385 path = path + depth;
2386 leaf = ext4_idx_pblock(path->p_idx);
2387 if (unlikely(path->p_hdr->eh_entries == 0)) {
2388 EXT4_ERROR_INODE(inode, "path->p_hdr->eh_entries == 0");
2389 return -EFSCORRUPTED;
2390 }
2391 err = ext4_ext_get_access(handle, inode, path);
2392 if (err)
2393 return err;
2394
2395 if (path->p_idx != EXT_LAST_INDEX(path->p_hdr)) {
2396 int len = EXT_LAST_INDEX(path->p_hdr) - path->p_idx;
2397 len *= sizeof(struct ext4_extent_idx);
2398 memmove(path->p_idx, path->p_idx + 1, len);
2399 }
2400
2401 le16_add_cpu(&path->p_hdr->eh_entries, -1);
2402 err = ext4_ext_dirty(handle, inode, path);
2403 if (err)
2404 return err;
2405 ext_debug("index is empty, remove it, free block %llu\n", leaf);
2406 trace_ext4_ext_rm_idx(inode, leaf);
2407
2408 ext4_free_blocks(handle, inode, NULL, leaf, 1,
2409 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
2410
2411 while (--depth >= 0) {
2412 if (path->p_idx != EXT_FIRST_INDEX(path->p_hdr))
2413 break;
2414 path--;
2415 err = ext4_ext_get_access(handle, inode, path);
2416 if (err)
2417 break;
2418 path->p_idx->ei_block = (path+1)->p_idx->ei_block;
2419 err = ext4_ext_dirty(handle, inode, path);
2420 if (err)
2421 break;
2422 }
2423 return err;
2424}
2425
2426
2427
2428
2429
2430
2431
2432
2433int ext4_ext_calc_credits_for_single_extent(struct inode *inode, int nrblocks,
2434 struct ext4_ext_path *path)
2435{
2436 if (path) {
2437 int depth = ext_depth(inode);
2438 int ret = 0;
2439
2440
2441 if (le16_to_cpu(path[depth].p_hdr->eh_entries)
2442 < le16_to_cpu(path[depth].p_hdr->eh_max)) {
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453 ret = 2 + EXT4_META_TRANS_BLOCKS(inode->i_sb);
2454 return ret;
2455 }
2456 }
2457
2458 return ext4_chunk_trans_blocks(inode, nrblocks);
2459}
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470int ext4_ext_index_trans_blocks(struct inode *inode, int extents)
2471{
2472 int index;
2473 int depth;
2474
2475
2476 if (ext4_has_inline_data(inode))
2477 return 1;
2478
2479 depth = ext_depth(inode);
2480
2481 if (extents <= 1)
2482 index = depth * 2;
2483 else
2484 index = depth * 3;
2485
2486 return index;
2487}
2488
2489static inline int get_default_free_blocks_flags(struct inode *inode)
2490{
2491 if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode) ||
2492 ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE))
2493 return EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET;
2494 else if (ext4_should_journal_data(inode))
2495 return EXT4_FREE_BLOCKS_FORGET;
2496 return 0;
2497}
2498
2499static int ext4_remove_blocks(handle_t *handle, struct inode *inode,
2500 struct ext4_extent *ex,
2501 long long *partial_cluster,
2502 ext4_lblk_t from, ext4_lblk_t to)
2503{
2504 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2505 unsigned short ee_len = ext4_ext_get_actual_len(ex);
2506 ext4_fsblk_t pblk;
2507 int flags = get_default_free_blocks_flags(inode);
2508
2509
2510
2511
2512
2513
2514
2515
2516 flags |= EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER;
2517
2518 trace_ext4_remove_blocks(inode, ex, from, to, *partial_cluster);
2519
2520
2521
2522
2523
2524 pblk = ext4_ext_pblock(ex) + ee_len - 1;
2525 if (*partial_cluster > 0 &&
2526 *partial_cluster != (long long) EXT4_B2C(sbi, pblk)) {
2527 ext4_free_blocks(handle, inode, NULL,
2528 EXT4_C2B(sbi, *partial_cluster),
2529 sbi->s_cluster_ratio, flags);
2530 *partial_cluster = 0;
2531 }
2532
2533#ifdef EXTENTS_STATS
2534 {
2535 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2536 spin_lock(&sbi->s_ext_stats_lock);
2537 sbi->s_ext_blocks += ee_len;
2538 sbi->s_ext_extents++;
2539 if (ee_len < sbi->s_ext_min)
2540 sbi->s_ext_min = ee_len;
2541 if (ee_len > sbi->s_ext_max)
2542 sbi->s_ext_max = ee_len;
2543 if (ext_depth(inode) > sbi->s_depth_max)
2544 sbi->s_depth_max = ext_depth(inode);
2545 spin_unlock(&sbi->s_ext_stats_lock);
2546 }
2547#endif
2548 if (from >= le32_to_cpu(ex->ee_block)
2549 && to == le32_to_cpu(ex->ee_block) + ee_len - 1) {
2550
2551 ext4_lblk_t num;
2552 long long first_cluster;
2553
2554 num = le32_to_cpu(ex->ee_block) + ee_len - from;
2555 pblk = ext4_ext_pblock(ex) + ee_len - num;
2556
2557
2558
2559
2560
2561 if (*partial_cluster < 0 &&
2562 *partial_cluster == -(long long) EXT4_B2C(sbi, pblk+num-1))
2563 flags |= EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER;
2564
2565 ext_debug("free last %u blocks starting %llu partial %lld\n",
2566 num, pblk, *partial_cluster);
2567 ext4_free_blocks(handle, inode, NULL, pblk, num, flags);
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584 if (EXT4_PBLK_COFF(sbi, pblk) && ee_len == num) {
2585 first_cluster = (long long) EXT4_B2C(sbi, pblk);
2586 if (first_cluster != -*partial_cluster)
2587 *partial_cluster = first_cluster;
2588 } else {
2589 *partial_cluster = 0;
2590 }
2591 } else
2592 ext4_error(sbi->s_sb, "strange request: removal(2) "
2593 "%u-%u from %u:%u",
2594 from, to, le32_to_cpu(ex->ee_block), ee_len);
2595 return 0;
2596}
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614static int
2615ext4_ext_rm_leaf(handle_t *handle, struct inode *inode,
2616 struct ext4_ext_path *path,
2617 long long *partial_cluster,
2618 ext4_lblk_t start, ext4_lblk_t end)
2619{
2620 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2621 int err = 0, correct_index = 0;
2622 int depth = ext_depth(inode), credits;
2623 struct ext4_extent_header *eh;
2624 ext4_lblk_t a, b;
2625 unsigned num;
2626 ext4_lblk_t ex_ee_block;
2627 unsigned short ex_ee_len;
2628 unsigned unwritten = 0;
2629 struct ext4_extent *ex;
2630 ext4_fsblk_t pblk;
2631
2632
2633 ext_debug("truncate since %u in leaf to %u\n", start, end);
2634 if (!path[depth].p_hdr)
2635 path[depth].p_hdr = ext_block_hdr(path[depth].p_bh);
2636 eh = path[depth].p_hdr;
2637 if (unlikely(path[depth].p_hdr == NULL)) {
2638 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
2639 return -EFSCORRUPTED;
2640 }
2641
2642 ex = path[depth].p_ext;
2643 if (!ex)
2644 ex = EXT_LAST_EXTENT(eh);
2645
2646 ex_ee_block = le32_to_cpu(ex->ee_block);
2647 ex_ee_len = ext4_ext_get_actual_len(ex);
2648
2649 trace_ext4_ext_rm_leaf(inode, start, ex, *partial_cluster);
2650
2651 while (ex >= EXT_FIRST_EXTENT(eh) &&
2652 ex_ee_block + ex_ee_len > start) {
2653
2654 if (ext4_ext_is_unwritten(ex))
2655 unwritten = 1;
2656 else
2657 unwritten = 0;
2658
2659 ext_debug("remove ext %u:[%d]%d\n", ex_ee_block,
2660 unwritten, ex_ee_len);
2661 path[depth].p_ext = ex;
2662
2663 a = ex_ee_block > start ? ex_ee_block : start;
2664 b = ex_ee_block+ex_ee_len - 1 < end ?
2665 ex_ee_block+ex_ee_len - 1 : end;
2666
2667 ext_debug(" border %u:%u\n", a, b);
2668
2669
2670 if (end < ex_ee_block) {
2671
2672
2673
2674
2675
2676
2677
2678 if (sbi->s_cluster_ratio > 1) {
2679 pblk = ext4_ext_pblock(ex);
2680 *partial_cluster =
2681 -(long long) EXT4_B2C(sbi, pblk);
2682 }
2683 ex--;
2684 ex_ee_block = le32_to_cpu(ex->ee_block);
2685 ex_ee_len = ext4_ext_get_actual_len(ex);
2686 continue;
2687 } else if (b != ex_ee_block + ex_ee_len - 1) {
2688 EXT4_ERROR_INODE(inode,
2689 "can not handle truncate %u:%u "
2690 "on extent %u:%u",
2691 start, end, ex_ee_block,
2692 ex_ee_block + ex_ee_len - 1);
2693 err = -EFSCORRUPTED;
2694 goto out;
2695 } else if (a != ex_ee_block) {
2696
2697 num = a - ex_ee_block;
2698 } else {
2699
2700 num = 0;
2701 }
2702
2703
2704
2705
2706
2707
2708 credits = 7 + 2*(ex_ee_len/EXT4_BLOCKS_PER_GROUP(inode->i_sb));
2709 if (ex == EXT_FIRST_EXTENT(eh)) {
2710 correct_index = 1;
2711 credits += (ext_depth(inode)) + 1;
2712 }
2713 credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb);
2714
2715 err = ext4_ext_truncate_extend_restart(handle, inode, credits);
2716 if (err)
2717 goto out;
2718
2719 err = ext4_ext_get_access(handle, inode, path + depth);
2720 if (err)
2721 goto out;
2722
2723 err = ext4_remove_blocks(handle, inode, ex, partial_cluster,
2724 a, b);
2725 if (err)
2726 goto out;
2727
2728 if (num == 0)
2729
2730 ext4_ext_store_pblock(ex, 0);
2731
2732 ex->ee_len = cpu_to_le16(num);
2733
2734
2735
2736
2737 if (unwritten && num)
2738 ext4_ext_mark_unwritten(ex);
2739
2740
2741
2742
2743 if (num == 0) {
2744 if (end != EXT_MAX_BLOCKS - 1) {
2745
2746
2747
2748
2749
2750 memmove(ex, ex+1, (EXT_LAST_EXTENT(eh) - ex) *
2751 sizeof(struct ext4_extent));
2752
2753
2754 memset(EXT_LAST_EXTENT(eh), 0,
2755 sizeof(struct ext4_extent));
2756 }
2757 le16_add_cpu(&eh->eh_entries, -1);
2758 }
2759
2760 err = ext4_ext_dirty(handle, inode, path + depth);
2761 if (err)
2762 goto out;
2763
2764 ext_debug("new extent: %u:%u:%llu\n", ex_ee_block, num,
2765 ext4_ext_pblock(ex));
2766 ex--;
2767 ex_ee_block = le32_to_cpu(ex->ee_block);
2768 ex_ee_len = ext4_ext_get_actual_len(ex);
2769 }
2770
2771 if (correct_index && eh->eh_entries)
2772 err = ext4_ext_correct_indexes(handle, inode, path);
2773
2774
2775
2776
2777
2778
2779
2780
2781 if (*partial_cluster > 0 && ex >= EXT_FIRST_EXTENT(eh)) {
2782 pblk = ext4_ext_pblock(ex) + ex_ee_len - 1;
2783 if (*partial_cluster != (long long) EXT4_B2C(sbi, pblk)) {
2784 ext4_free_blocks(handle, inode, NULL,
2785 EXT4_C2B(sbi, *partial_cluster),
2786 sbi->s_cluster_ratio,
2787 get_default_free_blocks_flags(inode));
2788 }
2789 *partial_cluster = 0;
2790 }
2791
2792
2793
2794 if (err == 0 && eh->eh_entries == 0 && path[depth].p_bh != NULL)
2795 err = ext4_ext_rm_idx(handle, inode, path, depth);
2796
2797out:
2798 return err;
2799}
2800
2801
2802
2803
2804
2805static int
2806ext4_ext_more_to_rm(struct ext4_ext_path *path)
2807{
2808 BUG_ON(path->p_idx == NULL);
2809
2810 if (path->p_idx < EXT_FIRST_INDEX(path->p_hdr))
2811 return 0;
2812
2813
2814
2815
2816
2817 if (le16_to_cpu(path->p_hdr->eh_entries) == path->p_block)
2818 return 0;
2819 return 1;
2820}
2821
2822int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start,
2823 ext4_lblk_t end)
2824{
2825 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2826 int depth = ext_depth(inode);
2827 struct ext4_ext_path *path = NULL;
2828 long long partial_cluster = 0;
2829 handle_t *handle;
2830 int i = 0, err = 0;
2831
2832 ext_debug("truncate since %u to %u\n", start, end);
2833
2834
2835 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, depth + 1);
2836 if (IS_ERR(handle))
2837 return PTR_ERR(handle);
2838
2839again:
2840 trace_ext4_ext_remove_space(inode, start, end, depth);
2841
2842
2843
2844
2845
2846
2847
2848
2849 if (end < EXT_MAX_BLOCKS - 1) {
2850 struct ext4_extent *ex;
2851 ext4_lblk_t ee_block, ex_end, lblk;
2852 ext4_fsblk_t pblk;
2853
2854
2855 path = ext4_find_extent(inode, end, NULL, EXT4_EX_NOCACHE);
2856 if (IS_ERR(path)) {
2857 ext4_journal_stop(handle);
2858 return PTR_ERR(path);
2859 }
2860 depth = ext_depth(inode);
2861
2862 ex = path[depth].p_ext;
2863 if (!ex) {
2864 if (depth) {
2865 EXT4_ERROR_INODE(inode,
2866 "path[%d].p_hdr == NULL",
2867 depth);
2868 err = -EFSCORRUPTED;
2869 }
2870 goto out;
2871 }
2872
2873 ee_block = le32_to_cpu(ex->ee_block);
2874 ex_end = ee_block + ext4_ext_get_actual_len(ex) - 1;
2875
2876
2877
2878
2879
2880
2881
2882 if (end >= ee_block && end < ex_end) {
2883
2884
2885
2886
2887
2888
2889 if (sbi->s_cluster_ratio > 1) {
2890 pblk = ext4_ext_pblock(ex) + end - ee_block + 2;
2891 partial_cluster =
2892 -(long long) EXT4_B2C(sbi, pblk);
2893 }
2894
2895
2896
2897
2898
2899
2900
2901 err = ext4_force_split_extent_at(handle, inode, &path,
2902 end + 1, 1);
2903 if (err < 0)
2904 goto out;
2905
2906 } else if (sbi->s_cluster_ratio > 1 && end >= ex_end) {
2907
2908
2909
2910
2911
2912
2913
2914
2915 lblk = ex_end + 1;
2916 err = ext4_ext_search_right(inode, path, &lblk, &pblk,
2917 &ex);
2918 if (err)
2919 goto out;
2920 if (pblk)
2921 partial_cluster =
2922 -(long long) EXT4_B2C(sbi, pblk);
2923 }
2924 }
2925
2926
2927
2928
2929 depth = ext_depth(inode);
2930 if (path) {
2931 int k = i = depth;
2932 while (--k > 0)
2933 path[k].p_block =
2934 le16_to_cpu(path[k].p_hdr->eh_entries)+1;
2935 } else {
2936 path = kzalloc(sizeof(struct ext4_ext_path) * (depth + 1),
2937 GFP_NOFS);
2938 if (path == NULL) {
2939 ext4_journal_stop(handle);
2940 return -ENOMEM;
2941 }
2942 path[0].p_maxdepth = path[0].p_depth = depth;
2943 path[0].p_hdr = ext_inode_hdr(inode);
2944 i = 0;
2945
2946 if (ext4_ext_check(inode, path[0].p_hdr, depth, 0)) {
2947 err = -EFSCORRUPTED;
2948 goto out;
2949 }
2950 }
2951 err = 0;
2952
2953 while (i >= 0 && err == 0) {
2954 if (i == depth) {
2955
2956 err = ext4_ext_rm_leaf(handle, inode, path,
2957 &partial_cluster, start,
2958 end);
2959
2960 brelse(path[i].p_bh);
2961 path[i].p_bh = NULL;
2962 i--;
2963 continue;
2964 }
2965
2966
2967 if (!path[i].p_hdr) {
2968 ext_debug("initialize header\n");
2969 path[i].p_hdr = ext_block_hdr(path[i].p_bh);
2970 }
2971
2972 if (!path[i].p_idx) {
2973
2974 path[i].p_idx = EXT_LAST_INDEX(path[i].p_hdr);
2975 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries)+1;
2976 ext_debug("init index ptr: hdr 0x%p, num %d\n",
2977 path[i].p_hdr,
2978 le16_to_cpu(path[i].p_hdr->eh_entries));
2979 } else {
2980
2981 path[i].p_idx--;
2982 }
2983
2984 ext_debug("level %d - index, first 0x%p, cur 0x%p\n",
2985 i, EXT_FIRST_INDEX(path[i].p_hdr),
2986 path[i].p_idx);
2987 if (ext4_ext_more_to_rm(path + i)) {
2988 struct buffer_head *bh;
2989
2990 ext_debug("move to level %d (block %llu)\n",
2991 i + 1, ext4_idx_pblock(path[i].p_idx));
2992 memset(path + i + 1, 0, sizeof(*path));
2993 bh = read_extent_tree_block(inode,
2994 ext4_idx_pblock(path[i].p_idx), depth - i - 1,
2995 EXT4_EX_NOCACHE);
2996 if (IS_ERR(bh)) {
2997
2998 err = PTR_ERR(bh);
2999 break;
3000 }
3001
3002
3003 cond_resched();
3004 if (WARN_ON(i + 1 > depth)) {
3005 err = -EFSCORRUPTED;
3006 break;
3007 }
3008 path[i + 1].p_bh = bh;
3009
3010
3011
3012 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries);
3013 i++;
3014 } else {
3015
3016 if (path[i].p_hdr->eh_entries == 0 && i > 0) {
3017
3018
3019
3020 err = ext4_ext_rm_idx(handle, inode, path, i);
3021 }
3022
3023 brelse(path[i].p_bh);
3024 path[i].p_bh = NULL;
3025 i--;
3026 ext_debug("return to level %d\n", i);
3027 }
3028 }
3029
3030 trace_ext4_ext_remove_space_done(inode, start, end, depth,
3031 partial_cluster, path->p_hdr->eh_entries);
3032
3033
3034
3035
3036
3037
3038
3039 if (partial_cluster > 0 && err == 0) {
3040
3041 ext4_free_blocks(handle, inode, NULL,
3042 EXT4_C2B(sbi, partial_cluster),
3043 sbi->s_cluster_ratio,
3044 get_default_free_blocks_flags(inode));
3045 }
3046
3047
3048 if (path->p_hdr->eh_entries == 0) {
3049
3050
3051
3052
3053 err = ext4_ext_get_access(handle, inode, path);
3054 if (err == 0) {
3055 ext_inode_hdr(inode)->eh_depth = 0;
3056 ext_inode_hdr(inode)->eh_max =
3057 cpu_to_le16(ext4_ext_space_root(inode, 0));
3058 err = ext4_ext_dirty(handle, inode, path);
3059 }
3060 }
3061out:
3062 ext4_ext_drop_refs(path);
3063 kfree(path);
3064 path = NULL;
3065 if (err == -EAGAIN)
3066 goto again;
3067 ext4_journal_stop(handle);
3068
3069 return err;
3070}
3071
3072
3073
3074
3075void ext4_ext_init(struct super_block *sb)
3076{
3077
3078
3079
3080
3081 if (ext4_has_feature_extents(sb)) {
3082#if defined(AGGRESSIVE_TEST) || defined(CHECK_BINSEARCH) || defined(EXTENTS_STATS)
3083 printk(KERN_INFO "EXT4-fs: file extents enabled"
3084#ifdef AGGRESSIVE_TEST
3085 ", aggressive tests"
3086#endif
3087#ifdef CHECK_BINSEARCH
3088 ", check binsearch"
3089#endif
3090#ifdef EXTENTS_STATS
3091 ", stats"
3092#endif
3093 "\n");
3094#endif
3095#ifdef EXTENTS_STATS
3096 spin_lock_init(&EXT4_SB(sb)->s_ext_stats_lock);
3097 EXT4_SB(sb)->s_ext_min = 1 << 30;
3098 EXT4_SB(sb)->s_ext_max = 0;
3099#endif
3100 }
3101}
3102
3103
3104
3105
3106void ext4_ext_release(struct super_block *sb)
3107{
3108 if (!ext4_has_feature_extents(sb))
3109 return;
3110
3111#ifdef EXTENTS_STATS
3112 if (EXT4_SB(sb)->s_ext_blocks && EXT4_SB(sb)->s_ext_extents) {
3113 struct ext4_sb_info *sbi = EXT4_SB(sb);
3114 printk(KERN_ERR "EXT4-fs: %lu blocks in %lu extents (%lu ave)\n",
3115 sbi->s_ext_blocks, sbi->s_ext_extents,
3116 sbi->s_ext_blocks / sbi->s_ext_extents);
3117 printk(KERN_ERR "EXT4-fs: extents: %lu min, %lu max, max depth %lu\n",
3118 sbi->s_ext_min, sbi->s_ext_max, sbi->s_depth_max);
3119 }
3120#endif
3121}
3122
3123static int ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex)
3124{
3125 ext4_lblk_t ee_block;
3126 ext4_fsblk_t ee_pblock;
3127 unsigned int ee_len;
3128
3129 ee_block = le32_to_cpu(ex->ee_block);
3130 ee_len = ext4_ext_get_actual_len(ex);
3131 ee_pblock = ext4_ext_pblock(ex);
3132
3133 if (ee_len == 0)
3134 return 0;
3135
3136 return ext4_es_insert_extent(inode, ee_block, ee_len, ee_pblock,
3137 EXTENT_STATUS_WRITTEN);
3138}
3139
3140
3141static int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex)
3142{
3143 ext4_fsblk_t ee_pblock;
3144 unsigned int ee_len;
3145
3146 ee_len = ext4_ext_get_actual_len(ex);
3147 ee_pblock = ext4_ext_pblock(ex);
3148 return ext4_issue_zeroout(inode, le32_to_cpu(ex->ee_block), ee_pblock,
3149 ee_len);
3150}
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173static int ext4_split_extent_at(handle_t *handle,
3174 struct inode *inode,
3175 struct ext4_ext_path **ppath,
3176 ext4_lblk_t split,
3177 int split_flag,
3178 int flags)
3179{
3180 struct ext4_ext_path *path = *ppath;
3181 ext4_fsblk_t newblock;
3182 ext4_lblk_t ee_block;
3183 struct ext4_extent *ex, newex, orig_ex, zero_ex;
3184 struct ext4_extent *ex2 = NULL;
3185 unsigned int ee_len, depth;
3186 int err = 0;
3187
3188 BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) ==
3189 (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2));
3190
3191 ext_debug("ext4_split_extents_at: inode %lu, logical"
3192 "block %llu\n", inode->i_ino, (unsigned long long)split);
3193
3194 ext4_ext_show_leaf(inode, path);
3195
3196 depth = ext_depth(inode);
3197 ex = path[depth].p_ext;
3198 ee_block = le32_to_cpu(ex->ee_block);
3199 ee_len = ext4_ext_get_actual_len(ex);
3200 newblock = split - ee_block + ext4_ext_pblock(ex);
3201
3202 BUG_ON(split < ee_block || split >= (ee_block + ee_len));
3203 BUG_ON(!ext4_ext_is_unwritten(ex) &&
3204 split_flag & (EXT4_EXT_MAY_ZEROOUT |
3205 EXT4_EXT_MARK_UNWRIT1 |
3206 EXT4_EXT_MARK_UNWRIT2));
3207
3208 err = ext4_ext_get_access(handle, inode, path + depth);
3209 if (err)
3210 goto out;
3211
3212 if (split == ee_block) {
3213
3214
3215
3216
3217
3218 if (split_flag & EXT4_EXT_MARK_UNWRIT2)
3219 ext4_ext_mark_unwritten(ex);
3220 else
3221 ext4_ext_mark_initialized(ex);
3222
3223 if (!(flags & EXT4_GET_BLOCKS_PRE_IO))
3224 ext4_ext_try_to_merge(handle, inode, path, ex);
3225
3226 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3227 goto out;
3228 }
3229
3230
3231 memcpy(&orig_ex, ex, sizeof(orig_ex));
3232 ex->ee_len = cpu_to_le16(split - ee_block);
3233 if (split_flag & EXT4_EXT_MARK_UNWRIT1)
3234 ext4_ext_mark_unwritten(ex);
3235
3236
3237
3238
3239
3240 err = ext4_ext_dirty(handle, inode, path + depth);
3241 if (err)
3242 goto fix_extent_len;
3243
3244 ex2 = &newex;
3245 ex2->ee_block = cpu_to_le32(split);
3246 ex2->ee_len = cpu_to_le16(ee_len - (split - ee_block));
3247 ext4_ext_store_pblock(ex2, newblock);
3248 if (split_flag & EXT4_EXT_MARK_UNWRIT2)
3249 ext4_ext_mark_unwritten(ex2);
3250
3251 err = ext4_ext_insert_extent(handle, inode, ppath, &newex, flags);
3252 if (err == -ENOSPC && (EXT4_EXT_MAY_ZEROOUT & split_flag)) {
3253 if (split_flag & (EXT4_EXT_DATA_VALID1|EXT4_EXT_DATA_VALID2)) {
3254 if (split_flag & EXT4_EXT_DATA_VALID1) {
3255 err = ext4_ext_zeroout(inode, ex2);
3256 zero_ex.ee_block = ex2->ee_block;
3257 zero_ex.ee_len = cpu_to_le16(
3258 ext4_ext_get_actual_len(ex2));
3259 ext4_ext_store_pblock(&zero_ex,
3260 ext4_ext_pblock(ex2));
3261 } else {
3262 err = ext4_ext_zeroout(inode, ex);
3263 zero_ex.ee_block = ex->ee_block;
3264 zero_ex.ee_len = cpu_to_le16(
3265 ext4_ext_get_actual_len(ex));
3266 ext4_ext_store_pblock(&zero_ex,
3267 ext4_ext_pblock(ex));
3268 }
3269 } else {
3270 err = ext4_ext_zeroout(inode, &orig_ex);
3271 zero_ex.ee_block = orig_ex.ee_block;
3272 zero_ex.ee_len = cpu_to_le16(
3273 ext4_ext_get_actual_len(&orig_ex));
3274 ext4_ext_store_pblock(&zero_ex,
3275 ext4_ext_pblock(&orig_ex));
3276 }
3277
3278 if (err)
3279 goto fix_extent_len;
3280
3281 ex->ee_len = cpu_to_le16(ee_len);
3282 ext4_ext_try_to_merge(handle, inode, path, ex);
3283 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3284 if (err)
3285 goto fix_extent_len;
3286
3287
3288 err = ext4_zeroout_es(inode, &zero_ex);
3289
3290 goto out;
3291 } else if (err)
3292 goto fix_extent_len;
3293
3294out:
3295 ext4_ext_show_leaf(inode, path);
3296 return err;
3297
3298fix_extent_len:
3299 ex->ee_len = orig_ex.ee_len;
3300 ext4_ext_dirty(handle, inode, path + path->p_depth);
3301 return err;
3302}
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315static int ext4_split_extent(handle_t *handle,
3316 struct inode *inode,
3317 struct ext4_ext_path **ppath,
3318 struct ext4_map_blocks *map,
3319 int split_flag,
3320 int flags)
3321{
3322 struct ext4_ext_path *path = *ppath;
3323 ext4_lblk_t ee_block;
3324 struct ext4_extent *ex;
3325 unsigned int ee_len, depth;
3326 int err = 0;
3327 int unwritten;
3328 int split_flag1, flags1;
3329 int allocated = map->m_len;
3330
3331 depth = ext_depth(inode);
3332 ex = path[depth].p_ext;
3333 ee_block = le32_to_cpu(ex->ee_block);
3334 ee_len = ext4_ext_get_actual_len(ex);
3335 unwritten = ext4_ext_is_unwritten(ex);
3336
3337 if (map->m_lblk + map->m_len < ee_block + ee_len) {
3338 split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT;
3339 flags1 = flags | EXT4_GET_BLOCKS_PRE_IO;
3340 if (unwritten)
3341 split_flag1 |= EXT4_EXT_MARK_UNWRIT1 |
3342 EXT4_EXT_MARK_UNWRIT2;
3343 if (split_flag & EXT4_EXT_DATA_VALID2)
3344 split_flag1 |= EXT4_EXT_DATA_VALID1;
3345 err = ext4_split_extent_at(handle, inode, ppath,
3346 map->m_lblk + map->m_len, split_flag1, flags1);
3347 if (err)
3348 goto out;
3349 } else {
3350 allocated = ee_len - (map->m_lblk - ee_block);
3351 }
3352
3353
3354
3355
3356 path = ext4_find_extent(inode, map->m_lblk, ppath, 0);
3357 if (IS_ERR(path))
3358 return PTR_ERR(path);
3359 depth = ext_depth(inode);
3360 ex = path[depth].p_ext;
3361 if (!ex) {
3362 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
3363 (unsigned long) map->m_lblk);
3364 return -EFSCORRUPTED;
3365 }
3366 unwritten = ext4_ext_is_unwritten(ex);
3367 split_flag1 = 0;
3368
3369 if (map->m_lblk >= ee_block) {
3370 split_flag1 = split_flag & EXT4_EXT_DATA_VALID2;
3371 if (unwritten) {
3372 split_flag1 |= EXT4_EXT_MARK_UNWRIT1;
3373 split_flag1 |= split_flag & (EXT4_EXT_MAY_ZEROOUT |
3374 EXT4_EXT_MARK_UNWRIT2);
3375 }
3376 err = ext4_split_extent_at(handle, inode, ppath,
3377 map->m_lblk, split_flag1, flags);
3378 if (err)
3379 goto out;
3380 }
3381
3382 ext4_ext_show_leaf(inode, path);
3383out:
3384 return err ? err : allocated;
3385}
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407static int ext4_ext_convert_to_initialized(handle_t *handle,
3408 struct inode *inode,
3409 struct ext4_map_blocks *map,
3410 struct ext4_ext_path **ppath,
3411 int flags)
3412{
3413 struct ext4_ext_path *path = *ppath;
3414 struct ext4_sb_info *sbi;
3415 struct ext4_extent_header *eh;
3416 struct ext4_map_blocks split_map;
3417 struct ext4_extent zero_ex1, zero_ex2;
3418 struct ext4_extent *ex, *abut_ex;
3419 ext4_lblk_t ee_block, eof_block;
3420 unsigned int ee_len, depth, map_len = map->m_len;
3421 int allocated = 0, max_zeroout = 0;
3422 int err = 0;
3423 int split_flag = EXT4_EXT_DATA_VALID2;
3424
3425 ext_debug("ext4_ext_convert_to_initialized: inode %lu, logical"
3426 "block %llu, max_blocks %u\n", inode->i_ino,
3427 (unsigned long long)map->m_lblk, map_len);
3428
3429 sbi = EXT4_SB(inode->i_sb);
3430 eof_block = (inode->i_size + inode->i_sb->s_blocksize - 1) >>
3431 inode->i_sb->s_blocksize_bits;
3432 if (eof_block < map->m_lblk + map_len)
3433 eof_block = map->m_lblk + map_len;
3434
3435 depth = ext_depth(inode);
3436 eh = path[depth].p_hdr;
3437 ex = path[depth].p_ext;
3438 ee_block = le32_to_cpu(ex->ee_block);
3439 ee_len = ext4_ext_get_actual_len(ex);
3440 zero_ex1.ee_len = 0;
3441 zero_ex2.ee_len = 0;
3442
3443 trace_ext4_ext_convert_to_initialized_enter(inode, map, ex);
3444
3445
3446 BUG_ON(!ext4_ext_is_unwritten(ex));
3447 BUG_ON(!in_range(map->m_lblk, ee_block, ee_len));
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464 if ((map->m_lblk == ee_block) &&
3465
3466 (map_len < ee_len) &&
3467 (ex > EXT_FIRST_EXTENT(eh))) {
3468 ext4_lblk_t prev_lblk;
3469 ext4_fsblk_t prev_pblk, ee_pblk;
3470 unsigned int prev_len;
3471
3472 abut_ex = ex - 1;
3473 prev_lblk = le32_to_cpu(abut_ex->ee_block);
3474 prev_len = ext4_ext_get_actual_len(abut_ex);
3475 prev_pblk = ext4_ext_pblock(abut_ex);
3476 ee_pblk = ext4_ext_pblock(ex);
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487 if ((!ext4_ext_is_unwritten(abut_ex)) &&
3488 ((prev_lblk + prev_len) == ee_block) &&
3489 ((prev_pblk + prev_len) == ee_pblk) &&
3490 (prev_len < (EXT_INIT_MAX_LEN - map_len))) {
3491 err = ext4_ext_get_access(handle, inode, path + depth);
3492 if (err)
3493 goto out;
3494
3495 trace_ext4_ext_convert_to_initialized_fastpath(inode,
3496 map, ex, abut_ex);
3497
3498
3499 ex->ee_block = cpu_to_le32(ee_block + map_len);
3500 ext4_ext_store_pblock(ex, ee_pblk + map_len);
3501 ex->ee_len = cpu_to_le16(ee_len - map_len);
3502 ext4_ext_mark_unwritten(ex);
3503
3504
3505 abut_ex->ee_len = cpu_to_le16(prev_len + map_len);
3506
3507
3508 allocated = map_len;
3509 }
3510 } else if (((map->m_lblk + map_len) == (ee_block + ee_len)) &&
3511 (map_len < ee_len) &&
3512 ex < EXT_LAST_EXTENT(eh)) {
3513
3514 ext4_lblk_t next_lblk;
3515 ext4_fsblk_t next_pblk, ee_pblk;
3516 unsigned int next_len;
3517
3518 abut_ex = ex + 1;
3519 next_lblk = le32_to_cpu(abut_ex->ee_block);
3520 next_len = ext4_ext_get_actual_len(abut_ex);
3521 next_pblk = ext4_ext_pblock(abut_ex);
3522 ee_pblk = ext4_ext_pblock(ex);
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533 if ((!ext4_ext_is_unwritten(abut_ex)) &&
3534 ((map->m_lblk + map_len) == next_lblk) &&
3535 ((ee_pblk + ee_len) == next_pblk) &&
3536 (next_len < (EXT_INIT_MAX_LEN - map_len))) {
3537 err = ext4_ext_get_access(handle, inode, path + depth);
3538 if (err)
3539 goto out;
3540
3541 trace_ext4_ext_convert_to_initialized_fastpath(inode,
3542 map, ex, abut_ex);
3543
3544
3545 abut_ex->ee_block = cpu_to_le32(next_lblk - map_len);
3546 ext4_ext_store_pblock(abut_ex, next_pblk - map_len);
3547 ex->ee_len = cpu_to_le16(ee_len - map_len);
3548 ext4_ext_mark_unwritten(ex);
3549
3550
3551 abut_ex->ee_len = cpu_to_le16(next_len + map_len);
3552
3553
3554 allocated = map_len;
3555 }
3556 }
3557 if (allocated) {
3558
3559 ext4_ext_dirty(handle, inode, path + depth);
3560
3561
3562 path[depth].p_ext = abut_ex;
3563 goto out;
3564 } else
3565 allocated = ee_len - (map->m_lblk - ee_block);
3566
3567 WARN_ON(map->m_lblk < ee_block);
3568
3569
3570
3571
3572 split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0;
3573
3574 if (EXT4_EXT_MAY_ZEROOUT & split_flag)
3575 max_zeroout = sbi->s_extent_max_zeroout_kb >>
3576 (inode->i_sb->s_blocksize_bits - 10);
3577
3578 if (ext4_encrypted_inode(inode))
3579 max_zeroout = 0;
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592 split_map.m_lblk = map->m_lblk;
3593 split_map.m_len = map->m_len;
3594
3595 if (max_zeroout && (allocated > split_map.m_len)) {
3596 if (allocated <= max_zeroout) {
3597
3598 zero_ex1.ee_block =
3599 cpu_to_le32(split_map.m_lblk +
3600 split_map.m_len);
3601 zero_ex1.ee_len =
3602 cpu_to_le16(allocated - split_map.m_len);
3603 ext4_ext_store_pblock(&zero_ex1,
3604 ext4_ext_pblock(ex) + split_map.m_lblk +
3605 split_map.m_len - ee_block);
3606 err = ext4_ext_zeroout(inode, &zero_ex1);
3607 if (err)
3608 goto out;
3609 split_map.m_len = allocated;
3610 }
3611 if (split_map.m_lblk - ee_block + split_map.m_len <
3612 max_zeroout) {
3613
3614 if (split_map.m_lblk != ee_block) {
3615 zero_ex2.ee_block = ex->ee_block;
3616 zero_ex2.ee_len = cpu_to_le16(split_map.m_lblk -
3617 ee_block);
3618 ext4_ext_store_pblock(&zero_ex2,
3619 ext4_ext_pblock(ex));
3620 err = ext4_ext_zeroout(inode, &zero_ex2);
3621 if (err)
3622 goto out;
3623 }
3624
3625 split_map.m_len += split_map.m_lblk - ee_block;
3626 split_map.m_lblk = ee_block;
3627 allocated = map->m_len;
3628 }
3629 }
3630
3631 err = ext4_split_extent(handle, inode, ppath, &split_map, split_flag,
3632 flags);
3633 if (err > 0)
3634 err = 0;
3635out:
3636
3637 if (!err) {
3638 err = ext4_zeroout_es(inode, &zero_ex1);
3639 if (!err)
3640 err = ext4_zeroout_es(inode, &zero_ex2);
3641 }
3642 return err ? err : allocated;
3643}
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669static int ext4_split_convert_extents(handle_t *handle,
3670 struct inode *inode,
3671 struct ext4_map_blocks *map,
3672 struct ext4_ext_path **ppath,
3673 int flags)
3674{
3675 struct ext4_ext_path *path = *ppath;
3676 ext4_lblk_t eof_block;
3677 ext4_lblk_t ee_block;
3678 struct ext4_extent *ex;
3679 unsigned int ee_len;
3680 int split_flag = 0, depth;
3681
3682 ext_debug("%s: inode %lu, logical block %llu, max_blocks %u\n",
3683 __func__, inode->i_ino,
3684 (unsigned long long)map->m_lblk, map->m_len);
3685
3686 eof_block = (inode->i_size + inode->i_sb->s_blocksize - 1) >>
3687 inode->i_sb->s_blocksize_bits;
3688 if (eof_block < map->m_lblk + map->m_len)
3689 eof_block = map->m_lblk + map->m_len;
3690
3691
3692
3693
3694 depth = ext_depth(inode);
3695 ex = path[depth].p_ext;
3696 ee_block = le32_to_cpu(ex->ee_block);
3697 ee_len = ext4_ext_get_actual_len(ex);
3698
3699
3700 if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) {
3701 split_flag |= EXT4_EXT_DATA_VALID1;
3702
3703 } else if (flags & EXT4_GET_BLOCKS_CONVERT) {
3704 split_flag |= ee_block + ee_len <= eof_block ?
3705 EXT4_EXT_MAY_ZEROOUT : 0;
3706 split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2);
3707 }
3708 flags |= EXT4_GET_BLOCKS_PRE_IO;
3709 return ext4_split_extent(handle, inode, ppath, map, split_flag, flags);
3710}
3711
3712static int ext4_convert_unwritten_extents_endio(handle_t *handle,
3713 struct inode *inode,
3714 struct ext4_map_blocks *map,
3715 struct ext4_ext_path **ppath)
3716{
3717 struct ext4_ext_path *path = *ppath;
3718 struct ext4_extent *ex;
3719 ext4_lblk_t ee_block;
3720 unsigned int ee_len;
3721 int depth;
3722 int err = 0;
3723
3724 depth = ext_depth(inode);
3725 ex = path[depth].p_ext;
3726 ee_block = le32_to_cpu(ex->ee_block);
3727 ee_len = ext4_ext_get_actual_len(ex);
3728
3729 ext_debug("ext4_convert_unwritten_extents_endio: inode %lu, logical"
3730 "block %llu, max_blocks %u\n", inode->i_ino,
3731 (unsigned long long)ee_block, ee_len);
3732
3733
3734
3735
3736
3737
3738
3739 if (ee_block != map->m_lblk || ee_len > map->m_len) {
3740#ifdef EXT4_DEBUG
3741 ext4_warning("Inode (%ld) finished: extent logical block %llu,"
3742 " len %u; IO logical block %llu, len %u",
3743 inode->i_ino, (unsigned long long)ee_block, ee_len,
3744 (unsigned long long)map->m_lblk, map->m_len);
3745#endif
3746 err = ext4_split_convert_extents(handle, inode, map, ppath,
3747 EXT4_GET_BLOCKS_CONVERT);
3748 if (err < 0)
3749 return err;
3750 path = ext4_find_extent(inode, map->m_lblk, ppath, 0);
3751 if (IS_ERR(path))
3752 return PTR_ERR(path);
3753 depth = ext_depth(inode);
3754 ex = path[depth].p_ext;
3755 }
3756
3757 err = ext4_ext_get_access(handle, inode, path + depth);
3758 if (err)
3759 goto out;
3760
3761 ext4_ext_mark_initialized(ex);
3762
3763
3764
3765
3766 ext4_ext_try_to_merge(handle, inode, path, ex);
3767
3768
3769 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3770out:
3771 ext4_ext_show_leaf(inode, path);
3772 return err;
3773}
3774
3775
3776
3777
3778static int check_eofblocks_fl(handle_t *handle, struct inode *inode,
3779 ext4_lblk_t lblk,
3780 struct ext4_ext_path *path,
3781 unsigned int len)
3782{
3783 int i, depth;
3784 struct ext4_extent_header *eh;
3785 struct ext4_extent *last_ex;
3786
3787 if (!ext4_test_inode_flag(inode, EXT4_INODE_EOFBLOCKS))
3788 return 0;
3789
3790 depth = ext_depth(inode);
3791 eh = path[depth].p_hdr;
3792
3793
3794
3795
3796
3797
3798 if (unlikely(!eh->eh_entries))
3799 goto out;
3800 last_ex = EXT_LAST_EXTENT(eh);
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810 if (lblk + len < le32_to_cpu(last_ex->ee_block) +
3811 ext4_ext_get_actual_len(last_ex))
3812 return 0;
3813
3814
3815
3816
3817
3818
3819
3820 for (i = depth-1; i >= 0; i--)
3821 if (path[i].p_idx != EXT_LAST_INDEX(path[i].p_hdr))
3822 return 0;
3823out:
3824 ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
3825 return ext4_mark_inode_dirty(handle, inode);
3826}
3827
3828
3829
3830
3831
3832
3833int ext4_find_delalloc_range(struct inode *inode,
3834 ext4_lblk_t lblk_start,
3835 ext4_lblk_t lblk_end)
3836{
3837 struct extent_status es;
3838
3839 ext4_es_find_delayed_extent_range(inode, lblk_start, lblk_end, &es);
3840 if (es.es_len == 0)
3841 return 0;
3842 else if (es.es_lblk <= lblk_start &&
3843 lblk_start < es.es_lblk + es.es_len)
3844 return 1;
3845 else if (lblk_start <= es.es_lblk && es.es_lblk <= lblk_end)
3846 return 1;
3847 else
3848 return 0;
3849}
3850
3851int ext4_find_delalloc_cluster(struct inode *inode, ext4_lblk_t lblk)
3852{
3853 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3854 ext4_lblk_t lblk_start, lblk_end;
3855 lblk_start = EXT4_LBLK_CMASK(sbi, lblk);
3856 lblk_end = lblk_start + sbi->s_cluster_ratio - 1;
3857
3858 return ext4_find_delalloc_range(inode, lblk_start, lblk_end);
3859}
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896static unsigned int
3897get_reserved_cluster_alloc(struct inode *inode, ext4_lblk_t lblk_start,
3898 unsigned int num_blks)
3899{
3900 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3901 ext4_lblk_t alloc_cluster_start, alloc_cluster_end;
3902 ext4_lblk_t lblk_from, lblk_to, c_offset;
3903 unsigned int allocated_clusters = 0;
3904
3905 alloc_cluster_start = EXT4_B2C(sbi, lblk_start);
3906 alloc_cluster_end = EXT4_B2C(sbi, lblk_start + num_blks - 1);
3907
3908
3909 allocated_clusters = alloc_cluster_end - alloc_cluster_start + 1;
3910
3911 trace_ext4_get_reserved_cluster_alloc(inode, lblk_start, num_blks);
3912
3913
3914 c_offset = EXT4_LBLK_COFF(sbi, lblk_start);
3915 if (c_offset) {
3916 lblk_from = EXT4_LBLK_CMASK(sbi, lblk_start);
3917 lblk_to = lblk_from + c_offset - 1;
3918
3919 if (ext4_find_delalloc_range(inode, lblk_from, lblk_to))
3920 allocated_clusters--;
3921 }
3922
3923
3924 c_offset = EXT4_LBLK_COFF(sbi, lblk_start + num_blks);
3925 if (allocated_clusters && c_offset) {
3926 lblk_from = lblk_start + num_blks;
3927 lblk_to = lblk_from + (sbi->s_cluster_ratio - c_offset) - 1;
3928
3929 if (ext4_find_delalloc_range(inode, lblk_from, lblk_to))
3930 allocated_clusters--;
3931 }
3932
3933 return allocated_clusters;
3934}
3935
3936static int
3937convert_initialized_extent(handle_t *handle, struct inode *inode,
3938 struct ext4_map_blocks *map,
3939 struct ext4_ext_path **ppath,
3940 unsigned int allocated)
3941{
3942 struct ext4_ext_path *path = *ppath;
3943 struct ext4_extent *ex;
3944 ext4_lblk_t ee_block;
3945 unsigned int ee_len;
3946 int depth;
3947 int err = 0;
3948
3949
3950
3951
3952
3953 if (map->m_len > EXT_UNWRITTEN_MAX_LEN)
3954 map->m_len = EXT_UNWRITTEN_MAX_LEN / 2;
3955
3956 depth = ext_depth(inode);
3957 ex = path[depth].p_ext;
3958 ee_block = le32_to_cpu(ex->ee_block);
3959 ee_len = ext4_ext_get_actual_len(ex);
3960
3961 ext_debug("%s: inode %lu, logical"
3962 "block %llu, max_blocks %u\n", __func__, inode->i_ino,
3963 (unsigned long long)ee_block, ee_len);
3964
3965 if (ee_block != map->m_lblk || ee_len > map->m_len) {
3966 err = ext4_split_convert_extents(handle, inode, map, ppath,
3967 EXT4_GET_BLOCKS_CONVERT_UNWRITTEN);
3968 if (err < 0)
3969 return err;
3970 path = ext4_find_extent(inode, map->m_lblk, ppath, 0);
3971 if (IS_ERR(path))
3972 return PTR_ERR(path);
3973 depth = ext_depth(inode);
3974 ex = path[depth].p_ext;
3975 if (!ex) {
3976 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
3977 (unsigned long) map->m_lblk);
3978 return -EFSCORRUPTED;
3979 }
3980 }
3981
3982 err = ext4_ext_get_access(handle, inode, path + depth);
3983 if (err)
3984 return err;
3985
3986 ext4_ext_mark_unwritten(ex);
3987
3988
3989
3990
3991 ext4_ext_try_to_merge(handle, inode, path, ex);
3992
3993
3994 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3995 if (err)
3996 return err;
3997 ext4_ext_show_leaf(inode, path);
3998
3999 ext4_update_inode_fsync_trans(handle, inode, 1);
4000 err = check_eofblocks_fl(handle, inode, map->m_lblk, path, map->m_len);
4001 if (err)
4002 return err;
4003 map->m_flags |= EXT4_MAP_UNWRITTEN;
4004 if (allocated > map->m_len)
4005 allocated = map->m_len;
4006 map->m_len = allocated;
4007 return allocated;
4008}
4009
4010static int
4011ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode,
4012 struct ext4_map_blocks *map,
4013 struct ext4_ext_path **ppath, int flags,
4014 unsigned int allocated, ext4_fsblk_t newblock)
4015{
4016 struct ext4_ext_path *path = *ppath;
4017 int ret = 0;
4018 int err = 0;
4019
4020 ext_debug("ext4_ext_handle_unwritten_extents: inode %lu, logical "
4021 "block %llu, max_blocks %u, flags %x, allocated %u\n",
4022 inode->i_ino, (unsigned long long)map->m_lblk, map->m_len,
4023 flags, allocated);
4024 ext4_ext_show_leaf(inode, path);
4025
4026
4027
4028
4029
4030 flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL;
4031
4032 trace_ext4_ext_handle_unwritten_extents(inode, map, flags,
4033 allocated, newblock);
4034
4035
4036 if (flags & EXT4_GET_BLOCKS_PRE_IO) {
4037 ret = ext4_split_convert_extents(handle, inode, map, ppath,
4038 flags | EXT4_GET_BLOCKS_CONVERT);
4039 if (ret <= 0)
4040 goto out;
4041 map->m_flags |= EXT4_MAP_UNWRITTEN;
4042 goto out;
4043 }
4044
4045 if (flags & EXT4_GET_BLOCKS_CONVERT) {
4046 if (flags & EXT4_GET_BLOCKS_ZERO) {
4047 if (allocated > map->m_len)
4048 allocated = map->m_len;
4049 err = ext4_issue_zeroout(inode, map->m_lblk, newblock,
4050 allocated);
4051 if (err < 0)
4052 goto out2;
4053 }
4054 ret = ext4_convert_unwritten_extents_endio(handle, inode, map,
4055 ppath);
4056 if (ret >= 0) {
4057 ext4_update_inode_fsync_trans(handle, inode, 1);
4058 err = check_eofblocks_fl(handle, inode, map->m_lblk,
4059 path, map->m_len);
4060 } else
4061 err = ret;
4062 map->m_flags |= EXT4_MAP_MAPPED;
4063 map->m_pblk = newblock;
4064 if (allocated > map->m_len)
4065 allocated = map->m_len;
4066 map->m_len = allocated;
4067 goto out2;
4068 }
4069
4070
4071
4072
4073
4074 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) {
4075 map->m_flags |= EXT4_MAP_UNWRITTEN;
4076 goto map_out;
4077 }
4078
4079
4080 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
4081
4082
4083
4084
4085
4086
4087
4088 map->m_flags |= EXT4_MAP_UNWRITTEN;
4089 goto out1;
4090 }
4091
4092
4093 ret = ext4_ext_convert_to_initialized(handle, inode, map, ppath, flags);
4094 if (ret >= 0)
4095 ext4_update_inode_fsync_trans(handle, inode, 1);
4096out:
4097 if (ret <= 0) {
4098 err = ret;
4099 goto out2;
4100 } else
4101 allocated = ret;
4102 map->m_flags |= EXT4_MAP_NEW;
4103
4104
4105
4106
4107
4108
4109
4110 if (allocated > map->m_len) {
4111 clean_bdev_aliases(inode->i_sb->s_bdev, newblock + map->m_len,
4112 allocated - map->m_len);
4113 allocated = map->m_len;
4114 }
4115 map->m_len = allocated;
4116
4117
4118
4119
4120
4121
4122
4123
4124 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) {
4125 unsigned int reserved_clusters;
4126 reserved_clusters = get_reserved_cluster_alloc(inode,
4127 map->m_lblk, map->m_len);
4128 if (reserved_clusters)
4129 ext4_da_update_reserve_space(inode,
4130 reserved_clusters,
4131 0);
4132 }
4133
4134map_out:
4135 map->m_flags |= EXT4_MAP_MAPPED;
4136 if ((flags & EXT4_GET_BLOCKS_KEEP_SIZE) == 0) {
4137 err = check_eofblocks_fl(handle, inode, map->m_lblk, path,
4138 map->m_len);
4139 if (err < 0)
4140 goto out2;
4141 }
4142out1:
4143 if (allocated > map->m_len)
4144 allocated = map->m_len;
4145 ext4_ext_show_leaf(inode, path);
4146 map->m_pblk = newblock;
4147 map->m_len = allocated;
4148out2:
4149 return err ? err : allocated;
4150}
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193static int get_implied_cluster_alloc(struct super_block *sb,
4194 struct ext4_map_blocks *map,
4195 struct ext4_extent *ex,
4196 struct ext4_ext_path *path)
4197{
4198 struct ext4_sb_info *sbi = EXT4_SB(sb);
4199 ext4_lblk_t c_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
4200 ext4_lblk_t ex_cluster_start, ex_cluster_end;
4201 ext4_lblk_t rr_cluster_start;
4202 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
4203 ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
4204 unsigned short ee_len = ext4_ext_get_actual_len(ex);
4205
4206
4207 ex_cluster_start = EXT4_B2C(sbi, ee_block);
4208 ex_cluster_end = EXT4_B2C(sbi, ee_block + ee_len - 1);
4209
4210
4211 rr_cluster_start = EXT4_B2C(sbi, map->m_lblk);
4212
4213 if ((rr_cluster_start == ex_cluster_end) ||
4214 (rr_cluster_start == ex_cluster_start)) {
4215 if (rr_cluster_start == ex_cluster_end)
4216 ee_start += ee_len - 1;
4217 map->m_pblk = EXT4_PBLK_CMASK(sbi, ee_start) + c_offset;
4218 map->m_len = min(map->m_len,
4219 (unsigned) sbi->s_cluster_ratio - c_offset);
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229 if (map->m_lblk < ee_block)
4230 map->m_len = min(map->m_len, ee_block - map->m_lblk);
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241 if (map->m_lblk > ee_block) {
4242 ext4_lblk_t next = ext4_ext_next_allocated_block(path);
4243 map->m_len = min(map->m_len, next - map->m_lblk);
4244 }
4245
4246 trace_ext4_get_implied_cluster_alloc_exit(sb, map, 1);
4247 return 1;
4248 }
4249
4250 trace_ext4_get_implied_cluster_alloc_exit(sb, map, 0);
4251 return 0;
4252}
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273int ext4_ext_map_blocks(handle_t *handle, struct inode *inode,
4274 struct ext4_map_blocks *map, int flags)
4275{
4276 struct ext4_ext_path *path = NULL;
4277 struct ext4_extent newex, *ex, *ex2;
4278 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4279 ext4_fsblk_t newblock = 0;
4280 int free_on_err = 0, err = 0, depth, ret;
4281 unsigned int allocated = 0, offset = 0;
4282 unsigned int allocated_clusters = 0;
4283 struct ext4_allocation_request ar;
4284 ext4_lblk_t cluster_offset;
4285 bool map_from_cluster = false;
4286
4287 ext_debug("blocks %u/%u requested for inode %lu\n",
4288 map->m_lblk, map->m_len, inode->i_ino);
4289 trace_ext4_ext_map_blocks_enter(inode, map->m_lblk, map->m_len, flags);
4290
4291
4292 path = ext4_find_extent(inode, map->m_lblk, NULL, 0);
4293 if (IS_ERR(path)) {
4294 err = PTR_ERR(path);
4295 path = NULL;
4296 goto out2;
4297 }
4298
4299 depth = ext_depth(inode);
4300
4301
4302
4303
4304
4305
4306 if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
4307 EXT4_ERROR_INODE(inode, "bad extent address "
4308 "lblock: %lu, depth: %d pblock %lld",
4309 (unsigned long) map->m_lblk, depth,
4310 path[depth].p_block);
4311 err = -EFSCORRUPTED;
4312 goto out2;
4313 }
4314
4315 ex = path[depth].p_ext;
4316 if (ex) {
4317 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
4318 ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
4319 unsigned short ee_len;
4320
4321
4322
4323
4324
4325
4326 ee_len = ext4_ext_get_actual_len(ex);
4327
4328 trace_ext4_ext_show_extent(inode, ee_block, ee_start, ee_len);
4329
4330
4331 if (in_range(map->m_lblk, ee_block, ee_len)) {
4332 newblock = map->m_lblk - ee_block + ee_start;
4333
4334 allocated = ee_len - (map->m_lblk - ee_block);
4335 ext_debug("%u fit into %u:%d -> %llu\n", map->m_lblk,
4336 ee_block, ee_len, newblock);
4337
4338
4339
4340
4341
4342 if ((!ext4_ext_is_unwritten(ex)) &&
4343 (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) {
4344 allocated = convert_initialized_extent(
4345 handle, inode, map, &path,
4346 allocated);
4347 goto out2;
4348 } else if (!ext4_ext_is_unwritten(ex))
4349 goto out;
4350
4351 ret = ext4_ext_handle_unwritten_extents(
4352 handle, inode, map, &path, flags,
4353 allocated, newblock);
4354 if (ret < 0)
4355 err = ret;
4356 else
4357 allocated = ret;
4358 goto out2;
4359 }
4360 }
4361
4362
4363
4364
4365
4366 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
4367 ext4_lblk_t hole_start, hole_len;
4368
4369 hole_start = map->m_lblk;
4370 hole_len = ext4_ext_determine_hole(inode, path, &hole_start);
4371
4372
4373
4374
4375 ext4_ext_put_gap_in_cache(inode, hole_start, hole_len);
4376
4377
4378 if (hole_start != map->m_lblk)
4379 hole_len -= map->m_lblk - hole_start;
4380 map->m_pblk = 0;
4381 map->m_len = min_t(unsigned int, map->m_len, hole_len);
4382
4383 goto out2;
4384 }
4385
4386
4387
4388
4389 newex.ee_block = cpu_to_le32(map->m_lblk);
4390 cluster_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
4391
4392
4393
4394
4395
4396 if (cluster_offset && ex &&
4397 get_implied_cluster_alloc(inode->i_sb, map, ex, path)) {
4398 ar.len = allocated = map->m_len;
4399 newblock = map->m_pblk;
4400 map_from_cluster = true;
4401 goto got_allocated_blocks;
4402 }
4403
4404
4405 ar.lleft = map->m_lblk;
4406 err = ext4_ext_search_left(inode, path, &ar.lleft, &ar.pleft);
4407 if (err)
4408 goto out2;
4409 ar.lright = map->m_lblk;
4410 ex2 = NULL;
4411 err = ext4_ext_search_right(inode, path, &ar.lright, &ar.pright, &ex2);
4412 if (err)
4413 goto out2;
4414
4415
4416
4417 if ((sbi->s_cluster_ratio > 1) && ex2 &&
4418 get_implied_cluster_alloc(inode->i_sb, map, ex2, path)) {
4419 ar.len = allocated = map->m_len;
4420 newblock = map->m_pblk;
4421 map_from_cluster = true;
4422 goto got_allocated_blocks;
4423 }
4424
4425
4426
4427
4428
4429
4430
4431 if (map->m_len > EXT_INIT_MAX_LEN &&
4432 !(flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
4433 map->m_len = EXT_INIT_MAX_LEN;
4434 else if (map->m_len > EXT_UNWRITTEN_MAX_LEN &&
4435 (flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
4436 map->m_len = EXT_UNWRITTEN_MAX_LEN;
4437
4438
4439 newex.ee_len = cpu_to_le16(map->m_len);
4440 err = ext4_ext_check_overlap(sbi, inode, &newex, path);
4441 if (err)
4442 allocated = ext4_ext_get_actual_len(&newex);
4443 else
4444 allocated = map->m_len;
4445
4446
4447 ar.inode = inode;
4448 ar.goal = ext4_ext_find_goal(inode, path, map->m_lblk);
4449 ar.logical = map->m_lblk;
4450
4451
4452
4453
4454
4455
4456
4457
4458 offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
4459 ar.len = EXT4_NUM_B2C(sbi, offset+allocated);
4460 ar.goal -= offset;
4461 ar.logical -= offset;
4462 if (S_ISREG(inode->i_mode))
4463 ar.flags = EXT4_MB_HINT_DATA;
4464 else
4465
4466 ar.flags = 0;
4467 if (flags & EXT4_GET_BLOCKS_NO_NORMALIZE)
4468 ar.flags |= EXT4_MB_HINT_NOPREALLOC;
4469 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
4470 ar.flags |= EXT4_MB_DELALLOC_RESERVED;
4471 if (flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
4472 ar.flags |= EXT4_MB_USE_RESERVED;
4473 newblock = ext4_mb_new_blocks(handle, &ar, &err);
4474 if (!newblock)
4475 goto out2;
4476 ext_debug("allocate new block: goal %llu, found %llu/%u\n",
4477 ar.goal, newblock, allocated);
4478 free_on_err = 1;
4479 allocated_clusters = ar.len;
4480 ar.len = EXT4_C2B(sbi, ar.len) - offset;
4481 if (ar.len > allocated)
4482 ar.len = allocated;
4483
4484got_allocated_blocks:
4485
4486 ext4_ext_store_pblock(&newex, newblock + offset);
4487 newex.ee_len = cpu_to_le16(ar.len);
4488
4489 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT){
4490 ext4_ext_mark_unwritten(&newex);
4491 map->m_flags |= EXT4_MAP_UNWRITTEN;
4492 }
4493
4494 err = 0;
4495 if ((flags & EXT4_GET_BLOCKS_KEEP_SIZE) == 0)
4496 err = check_eofblocks_fl(handle, inode, map->m_lblk,
4497 path, ar.len);
4498 if (!err)
4499 err = ext4_ext_insert_extent(handle, inode, &path,
4500 &newex, flags);
4501
4502 if (err && free_on_err) {
4503 int fb_flags = flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE ?
4504 EXT4_FREE_BLOCKS_NO_QUOT_UPDATE : 0;
4505
4506
4507
4508 ext4_discard_preallocations(inode);
4509 ext4_free_blocks(handle, inode, NULL, newblock,
4510 EXT4_C2B(sbi, allocated_clusters), fb_flags);
4511 goto out2;
4512 }
4513
4514
4515 newblock = ext4_ext_pblock(&newex);
4516 allocated = ext4_ext_get_actual_len(&newex);
4517 if (allocated > map->m_len)
4518 allocated = map->m_len;
4519 map->m_flags |= EXT4_MAP_NEW;
4520
4521
4522
4523
4524
4525 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) {
4526 unsigned int reserved_clusters;
4527
4528
4529
4530 reserved_clusters = get_reserved_cluster_alloc(inode,
4531 map->m_lblk, allocated);
4532 if (!map_from_cluster) {
4533 BUG_ON(allocated_clusters < reserved_clusters);
4534 if (reserved_clusters < allocated_clusters) {
4535 struct ext4_inode_info *ei = EXT4_I(inode);
4536 int reservation = allocated_clusters -
4537 reserved_clusters;
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578 dquot_reserve_block(inode,
4579 EXT4_C2B(sbi, reservation));
4580 spin_lock(&ei->i_block_reservation_lock);
4581 ei->i_reserved_data_blocks += reservation;
4582 spin_unlock(&ei->i_block_reservation_lock);
4583 }
4584
4585
4586
4587
4588
4589
4590
4591 ext4_da_update_reserve_space(inode, allocated_clusters,
4592 1);
4593 }
4594 }
4595
4596
4597
4598
4599
4600 if ((flags & EXT4_GET_BLOCKS_UNWRIT_EXT) == 0)
4601 ext4_update_inode_fsync_trans(handle, inode, 1);
4602 else
4603 ext4_update_inode_fsync_trans(handle, inode, 0);
4604out:
4605 if (allocated > map->m_len)
4606 allocated = map->m_len;
4607 ext4_ext_show_leaf(inode, path);
4608 map->m_flags |= EXT4_MAP_MAPPED;
4609 map->m_pblk = newblock;
4610 map->m_len = allocated;
4611out2:
4612 ext4_ext_drop_refs(path);
4613 kfree(path);
4614
4615 trace_ext4_ext_map_blocks_exit(inode, flags, map,
4616 err ? err : allocated);
4617 return err ? err : allocated;
4618}
4619
4620int ext4_ext_truncate(handle_t *handle, struct inode *inode)
4621{
4622 struct super_block *sb = inode->i_sb;
4623 ext4_lblk_t last_block;
4624 int err = 0;
4625
4626
4627
4628
4629
4630
4631
4632
4633 EXT4_I(inode)->i_disksize = inode->i_size;
4634 err = ext4_mark_inode_dirty(handle, inode);
4635 if (err)
4636 return err;
4637
4638 last_block = (inode->i_size + sb->s_blocksize - 1)
4639 >> EXT4_BLOCK_SIZE_BITS(sb);
4640retry:
4641 err = ext4_es_remove_extent(inode, last_block,
4642 EXT_MAX_BLOCKS - last_block);
4643 if (err == -ENOMEM) {
4644 cond_resched();
4645 congestion_wait(BLK_RW_ASYNC, HZ/50);
4646 goto retry;
4647 }
4648 if (err)
4649 return err;
4650 return ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1);
4651}
4652
4653static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset,
4654 ext4_lblk_t len, loff_t new_size,
4655 int flags)
4656{
4657 struct inode *inode = file_inode(file);
4658 handle_t *handle;
4659 int ret = 0;
4660 int ret2 = 0;
4661 int retries = 0;
4662 int depth = 0;
4663 struct ext4_map_blocks map;
4664 unsigned int credits;
4665 loff_t epos;
4666
4667 BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS));
4668 map.m_lblk = offset;
4669 map.m_len = len;
4670
4671
4672
4673
4674
4675 if (len <= EXT_UNWRITTEN_MAX_LEN)
4676 flags |= EXT4_GET_BLOCKS_NO_NORMALIZE;
4677
4678
4679
4680
4681 credits = ext4_chunk_trans_blocks(inode, len);
4682 depth = ext_depth(inode);
4683
4684retry:
4685 while (ret >= 0 && len) {
4686
4687
4688
4689 if (depth != ext_depth(inode)) {
4690 credits = ext4_chunk_trans_blocks(inode, len);
4691 depth = ext_depth(inode);
4692 }
4693
4694 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
4695 credits);
4696 if (IS_ERR(handle)) {
4697 ret = PTR_ERR(handle);
4698 break;
4699 }
4700 ret = ext4_map_blocks(handle, inode, &map, flags);
4701 if (ret <= 0) {
4702 ext4_debug("inode #%lu: block %u: len %u: "
4703 "ext4_ext_map_blocks returned %d",
4704 inode->i_ino, map.m_lblk,
4705 map.m_len, ret);
4706 ext4_mark_inode_dirty(handle, inode);
4707 ret2 = ext4_journal_stop(handle);
4708 break;
4709 }
4710 map.m_lblk += ret;
4711 map.m_len = len = len - ret;
4712 epos = (loff_t)map.m_lblk << inode->i_blkbits;
4713 inode->i_ctime = current_time(inode);
4714 if (new_size) {
4715 if (epos > new_size)
4716 epos = new_size;
4717 if (ext4_update_inode_size(inode, epos) & 0x1)
4718 inode->i_mtime = inode->i_ctime;
4719 } else {
4720 if (epos > inode->i_size)
4721 ext4_set_inode_flag(inode,
4722 EXT4_INODE_EOFBLOCKS);
4723 }
4724 ext4_mark_inode_dirty(handle, inode);
4725 ret2 = ext4_journal_stop(handle);
4726 if (ret2)
4727 break;
4728 }
4729 if (ret == -ENOSPC &&
4730 ext4_should_retry_alloc(inode->i_sb, &retries)) {
4731 ret = 0;
4732 goto retry;
4733 }
4734
4735 return ret > 0 ? ret2 : ret;
4736}
4737
4738static long ext4_zero_range(struct file *file, loff_t offset,
4739 loff_t len, int mode)
4740{
4741 struct inode *inode = file_inode(file);
4742 handle_t *handle = NULL;
4743 unsigned int max_blocks;
4744 loff_t new_size = 0;
4745 int ret = 0;
4746 int flags;
4747 int credits;
4748 int partial_begin, partial_end;
4749 loff_t start, end;
4750 ext4_lblk_t lblk;
4751 unsigned int blkbits = inode->i_blkbits;
4752
4753 trace_ext4_zero_range(inode, offset, len, mode);
4754
4755 if (!S_ISREG(inode->i_mode))
4756 return -EINVAL;
4757
4758
4759 if (ext4_should_journal_data(inode)) {
4760 ret = ext4_force_commit(inode->i_sb);
4761 if (ret)
4762 return ret;
4763 }
4764
4765
4766
4767
4768
4769
4770
4771 start = round_up(offset, 1 << blkbits);
4772 end = round_down((offset + len), 1 << blkbits);
4773
4774 if (start < offset || end > offset + len)
4775 return -EINVAL;
4776 partial_begin = offset & ((1 << blkbits) - 1);
4777 partial_end = (offset + len) & ((1 << blkbits) - 1);
4778
4779 lblk = start >> blkbits;
4780 max_blocks = (end >> blkbits);
4781 if (max_blocks < lblk)
4782 max_blocks = 0;
4783 else
4784 max_blocks -= lblk;
4785
4786 inode_lock(inode);
4787
4788
4789
4790
4791 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
4792 ret = -EOPNOTSUPP;
4793 goto out_mutex;
4794 }
4795
4796 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
4797 offset + len > i_size_read(inode)) {
4798 new_size = offset + len;
4799 ret = inode_newsize_ok(inode, new_size);
4800 if (ret)
4801 goto out_mutex;
4802 }
4803
4804 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
4805 if (mode & FALLOC_FL_KEEP_SIZE)
4806 flags |= EXT4_GET_BLOCKS_KEEP_SIZE;
4807
4808
4809 ext4_inode_block_unlocked_dio(inode);
4810 inode_dio_wait(inode);
4811
4812
4813 if (partial_begin || partial_end) {
4814 ret = ext4_alloc_file_blocks(file,
4815 round_down(offset, 1 << blkbits) >> blkbits,
4816 (round_up((offset + len), 1 << blkbits) -
4817 round_down(offset, 1 << blkbits)) >> blkbits,
4818 new_size, flags);
4819 if (ret)
4820 goto out_dio;
4821
4822 }
4823
4824
4825 if (max_blocks > 0) {
4826 flags |= (EXT4_GET_BLOCKS_CONVERT_UNWRITTEN |
4827 EXT4_EX_NOCACHE);
4828
4829
4830
4831
4832
4833 down_write(&EXT4_I(inode)->i_mmap_sem);
4834 ret = ext4_update_disksize_before_punch(inode, offset, len);
4835 if (ret) {
4836 up_write(&EXT4_I(inode)->i_mmap_sem);
4837 goto out_dio;
4838 }
4839
4840 truncate_pagecache_range(inode, start, end - 1);
4841 inode->i_mtime = inode->i_ctime = current_time(inode);
4842
4843 ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size,
4844 flags);
4845 up_write(&EXT4_I(inode)->i_mmap_sem);
4846 if (ret)
4847 goto out_dio;
4848 }
4849 if (!partial_begin && !partial_end)
4850 goto out_dio;
4851
4852
4853
4854
4855
4856 credits = (2 * ext4_ext_index_trans_blocks(inode, 2)) + 1;
4857 if (ext4_should_journal_data(inode))
4858 credits += 2;
4859 handle = ext4_journal_start(inode, EXT4_HT_MISC, credits);
4860 if (IS_ERR(handle)) {
4861 ret = PTR_ERR(handle);
4862 ext4_std_error(inode->i_sb, ret);
4863 goto out_dio;
4864 }
4865
4866 inode->i_mtime = inode->i_ctime = current_time(inode);
4867 if (new_size) {
4868 ext4_update_inode_size(inode, new_size);
4869 } else {
4870
4871
4872
4873
4874 if ((offset + len) > i_size_read(inode))
4875 ext4_set_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
4876 }
4877 ext4_mark_inode_dirty(handle, inode);
4878
4879
4880 ret = ext4_zero_partial_blocks(handle, inode, offset, len);
4881 if (ret >= 0)
4882 ext4_update_inode_fsync_trans(handle, inode, 1);
4883
4884 if (file->f_flags & O_SYNC)
4885 ext4_handle_sync(handle);
4886
4887 ext4_journal_stop(handle);
4888out_dio:
4889 ext4_inode_resume_unlocked_dio(inode);
4890out_mutex:
4891 inode_unlock(inode);
4892 return ret;
4893}
4894
4895
4896
4897
4898
4899
4900
4901
4902long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
4903{
4904 struct inode *inode = file_inode(file);
4905 loff_t new_size = 0;
4906 unsigned int max_blocks;
4907 int ret = 0;
4908 int flags;
4909 ext4_lblk_t lblk;
4910 unsigned int blkbits = inode->i_blkbits;
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922 if (ext4_encrypted_inode(inode) &&
4923 (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE |
4924 FALLOC_FL_ZERO_RANGE)))
4925 return -EOPNOTSUPP;
4926
4927
4928 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
4929 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE |
4930 FALLOC_FL_INSERT_RANGE))
4931 return -EOPNOTSUPP;
4932
4933 if (mode & FALLOC_FL_PUNCH_HOLE)
4934 return ext4_punch_hole(inode, offset, len);
4935
4936 ret = ext4_convert_inline_data(inode);
4937 if (ret)
4938 return ret;
4939
4940 if (mode & FALLOC_FL_COLLAPSE_RANGE)
4941 return ext4_collapse_range(inode, offset, len);
4942
4943 if (mode & FALLOC_FL_INSERT_RANGE)
4944 return ext4_insert_range(inode, offset, len);
4945
4946 if (mode & FALLOC_FL_ZERO_RANGE)
4947 return ext4_zero_range(file, offset, len, mode);
4948
4949 trace_ext4_fallocate_enter(inode, offset, len, mode);
4950 lblk = offset >> blkbits;
4951
4952 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
4953 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
4954 if (mode & FALLOC_FL_KEEP_SIZE)
4955 flags |= EXT4_GET_BLOCKS_KEEP_SIZE;
4956
4957 inode_lock(inode);
4958
4959
4960
4961
4962 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
4963 ret = -EOPNOTSUPP;
4964 goto out;
4965 }
4966
4967 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
4968 offset + len > i_size_read(inode)) {
4969 new_size = offset + len;
4970 ret = inode_newsize_ok(inode, new_size);
4971 if (ret)
4972 goto out;
4973 }
4974
4975
4976 ext4_inode_block_unlocked_dio(inode);
4977 inode_dio_wait(inode);
4978
4979 ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size, flags);
4980 ext4_inode_resume_unlocked_dio(inode);
4981 if (ret)
4982 goto out;
4983
4984 if (file->f_flags & O_SYNC && EXT4_SB(inode->i_sb)->s_journal) {
4985 ret = jbd2_complete_transaction(EXT4_SB(inode->i_sb)->s_journal,
4986 EXT4_I(inode)->i_sync_tid);
4987 }
4988out:
4989 inode_unlock(inode);
4990 trace_ext4_fallocate_exit(inode, offset, max_blocks, ret);
4991 return ret;
4992}
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
5005 loff_t offset, ssize_t len)
5006{
5007 unsigned int max_blocks;
5008 int ret = 0;
5009 int ret2 = 0;
5010 struct ext4_map_blocks map;
5011 unsigned int credits, blkbits = inode->i_blkbits;
5012
5013 map.m_lblk = offset >> blkbits;
5014 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
5015
5016
5017
5018
5019
5020
5021 if (handle) {
5022 handle = ext4_journal_start_reserved(handle,
5023 EXT4_HT_EXT_CONVERT);
5024 if (IS_ERR(handle))
5025 return PTR_ERR(handle);
5026 credits = 0;
5027 } else {
5028
5029
5030
5031 credits = ext4_chunk_trans_blocks(inode, max_blocks);
5032 }
5033 while (ret >= 0 && ret < max_blocks) {
5034 map.m_lblk += ret;
5035 map.m_len = (max_blocks -= ret);
5036 if (credits) {
5037 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
5038 credits);
5039 if (IS_ERR(handle)) {
5040 ret = PTR_ERR(handle);
5041 break;
5042 }
5043 }
5044 ret = ext4_map_blocks(handle, inode, &map,
5045 EXT4_GET_BLOCKS_IO_CONVERT_EXT);
5046 if (ret <= 0)
5047 ext4_warning(inode->i_sb,
5048 "inode #%lu: block %u: len %u: "
5049 "ext4_ext_map_blocks returned %d",
5050 inode->i_ino, map.m_lblk,
5051 map.m_len, ret);
5052 ext4_mark_inode_dirty(handle, inode);
5053 if (credits)
5054 ret2 = ext4_journal_stop(handle);
5055 if (ret <= 0 || ret2)
5056 break;
5057 }
5058 if (!credits)
5059 ret2 = ext4_journal_stop(handle);
5060 return ret > 0 ? ret2 : ret;
5061}
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072static int ext4_find_delayed_extent(struct inode *inode,
5073 struct extent_status *newes)
5074{
5075 struct extent_status es;
5076 ext4_lblk_t block, next_del;
5077
5078 if (newes->es_pblk == 0) {
5079 ext4_es_find_delayed_extent_range(inode, newes->es_lblk,
5080 newes->es_lblk + newes->es_len - 1, &es);
5081
5082
5083
5084
5085
5086 if (es.es_len == 0)
5087
5088 return 0;
5089
5090 if (es.es_lblk > newes->es_lblk) {
5091
5092 newes->es_len = min(es.es_lblk - newes->es_lblk,
5093 newes->es_len);
5094 return 0;
5095 }
5096
5097 newes->es_len = es.es_lblk + es.es_len - newes->es_lblk;
5098 }
5099
5100 block = newes->es_lblk + newes->es_len;
5101 ext4_es_find_delayed_extent_range(inode, block, EXT_MAX_BLOCKS, &es);
5102 if (es.es_len == 0)
5103 next_del = EXT_MAX_BLOCKS;
5104 else
5105 next_del = es.es_lblk;
5106
5107 return next_del;
5108}
5109
5110#define EXT4_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC|FIEMAP_FLAG_XATTR)
5111
5112static int ext4_xattr_fiemap(struct inode *inode,
5113 struct fiemap_extent_info *fieinfo)
5114{
5115 __u64 physical = 0;
5116 __u64 length;
5117 __u32 flags = FIEMAP_EXTENT_LAST;
5118 int blockbits = inode->i_sb->s_blocksize_bits;
5119 int error = 0;
5120
5121
5122 if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
5123 struct ext4_iloc iloc;
5124 int offset;
5125
5126 error = ext4_get_inode_loc(inode, &iloc);
5127 if (error)
5128 return error;
5129 physical = (__u64)iloc.bh->b_blocknr << blockbits;
5130 offset = EXT4_GOOD_OLD_INODE_SIZE +
5131 EXT4_I(inode)->i_extra_isize;
5132 physical += offset;
5133 length = EXT4_SB(inode->i_sb)->s_inode_size - offset;
5134 flags |= FIEMAP_EXTENT_DATA_INLINE;
5135 brelse(iloc.bh);
5136 } else {
5137 physical = (__u64)EXT4_I(inode)->i_file_acl << blockbits;
5138 length = inode->i_sb->s_blocksize;
5139 }
5140
5141 if (physical)
5142 error = fiemap_fill_next_extent(fieinfo, 0, physical,
5143 length, flags);
5144 return (error < 0 ? error : 0);
5145}
5146
5147int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
5148 __u64 start, __u64 len)
5149{
5150 ext4_lblk_t start_blk;
5151 int error = 0;
5152
5153 if (ext4_has_inline_data(inode)) {
5154 int has_inline = 1;
5155
5156 error = ext4_inline_data_fiemap(inode, fieinfo, &has_inline,
5157 start, len);
5158
5159 if (has_inline)
5160 return error;
5161 }
5162
5163 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
5164 error = ext4_ext_precache(inode);
5165 if (error)
5166 return error;
5167 }
5168
5169
5170 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
5171 return generic_block_fiemap(inode, fieinfo, start, len,
5172 ext4_get_block);
5173
5174 if (fiemap_check_flags(fieinfo, EXT4_FIEMAP_FLAGS))
5175 return -EBADR;
5176
5177 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
5178 error = ext4_xattr_fiemap(inode, fieinfo);
5179 } else {
5180 ext4_lblk_t len_blks;
5181 __u64 last_blk;
5182
5183 start_blk = start >> inode->i_sb->s_blocksize_bits;
5184 last_blk = (start + len - 1) >> inode->i_sb->s_blocksize_bits;
5185 if (last_blk >= EXT_MAX_BLOCKS)
5186 last_blk = EXT_MAX_BLOCKS-1;
5187 len_blks = ((ext4_lblk_t) last_blk) - start_blk + 1;
5188
5189
5190
5191
5192
5193 error = ext4_fill_fiemap_extents(inode, start_blk,
5194 len_blks, fieinfo);
5195 }
5196 return error;
5197}
5198
5199
5200
5201
5202
5203
5204
5205static int
5206ext4_access_path(handle_t *handle, struct inode *inode,
5207 struct ext4_ext_path *path)
5208{
5209 int credits, err;
5210
5211 if (!ext4_handle_valid(handle))
5212 return 0;
5213
5214
5215
5216
5217
5218
5219
5220 if (handle->h_buffer_credits < 7) {
5221 credits = ext4_writepage_trans_blocks(inode);
5222 err = ext4_ext_truncate_extend_restart(handle, inode, credits);
5223
5224 if (err && err != -EAGAIN)
5225 return err;
5226 }
5227
5228 err = ext4_ext_get_access(handle, inode, path);
5229 return err;
5230}
5231
5232
5233
5234
5235
5236
5237
5238static int
5239ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift,
5240 struct inode *inode, handle_t *handle,
5241 enum SHIFT_DIRECTION SHIFT)
5242{
5243 int depth, err = 0;
5244 struct ext4_extent *ex_start, *ex_last;
5245 bool update = 0;
5246 depth = path->p_depth;
5247
5248 while (depth >= 0) {
5249 if (depth == path->p_depth) {
5250 ex_start = path[depth].p_ext;
5251 if (!ex_start)
5252 return -EFSCORRUPTED;
5253
5254 ex_last = EXT_LAST_EXTENT(path[depth].p_hdr);
5255
5256 err = ext4_access_path(handle, inode, path + depth);
5257 if (err)
5258 goto out;
5259
5260 if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr))
5261 update = 1;
5262
5263 while (ex_start <= ex_last) {
5264 if (SHIFT == SHIFT_LEFT) {
5265 le32_add_cpu(&ex_start->ee_block,
5266 -shift);
5267
5268 if ((ex_start >
5269 EXT_FIRST_EXTENT(path[depth].p_hdr))
5270 &&
5271 ext4_ext_try_to_merge_right(inode,
5272 path, ex_start - 1))
5273 ex_last--;
5274 else
5275 ex_start++;
5276 } else {
5277 le32_add_cpu(&ex_last->ee_block, shift);
5278 ext4_ext_try_to_merge_right(inode, path,
5279 ex_last);
5280 ex_last--;
5281 }
5282 }
5283 err = ext4_ext_dirty(handle, inode, path + depth);
5284 if (err)
5285 goto out;
5286
5287 if (--depth < 0 || !update)
5288 break;
5289 }
5290
5291
5292 err = ext4_access_path(handle, inode, path + depth);
5293 if (err)
5294 goto out;
5295
5296 if (SHIFT == SHIFT_LEFT)
5297 le32_add_cpu(&path[depth].p_idx->ei_block, -shift);
5298 else
5299 le32_add_cpu(&path[depth].p_idx->ei_block, shift);
5300 err = ext4_ext_dirty(handle, inode, path + depth);
5301 if (err)
5302 goto out;
5303
5304
5305 if (path[depth].p_idx != EXT_FIRST_INDEX(path[depth].p_hdr))
5306 break;
5307
5308 depth--;
5309 }
5310
5311out:
5312 return err;
5313}
5314
5315
5316
5317
5318
5319
5320
5321
5322static int
5323ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
5324 ext4_lblk_t start, ext4_lblk_t shift,
5325 enum SHIFT_DIRECTION SHIFT)
5326{
5327 struct ext4_ext_path *path;
5328 int ret = 0, depth;
5329 struct ext4_extent *extent;
5330 ext4_lblk_t stop, *iterator, ex_start, ex_end;
5331
5332
5333 path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
5334 EXT4_EX_NOCACHE);
5335 if (IS_ERR(path))
5336 return PTR_ERR(path);
5337
5338 depth = path->p_depth;
5339 extent = path[depth].p_ext;
5340 if (!extent)
5341 goto out;
5342
5343 stop = le32_to_cpu(extent->ee_block);
5344
5345
5346
5347
5348
5349 if (SHIFT == SHIFT_LEFT) {
5350 path = ext4_find_extent(inode, start - 1, &path,
5351 EXT4_EX_NOCACHE);
5352 if (IS_ERR(path))
5353 return PTR_ERR(path);
5354 depth = path->p_depth;
5355 extent = path[depth].p_ext;
5356 if (extent) {
5357 ex_start = le32_to_cpu(extent->ee_block);
5358 ex_end = le32_to_cpu(extent->ee_block) +
5359 ext4_ext_get_actual_len(extent);
5360 } else {
5361 ex_start = 0;
5362 ex_end = 0;
5363 }
5364
5365 if ((start == ex_start && shift > ex_start) ||
5366 (shift > start - ex_end)) {
5367 ext4_ext_drop_refs(path);
5368 kfree(path);
5369 return -EINVAL;
5370 }
5371 }
5372
5373
5374
5375
5376
5377
5378 if (SHIFT == SHIFT_LEFT)
5379 iterator = &start;
5380 else
5381 iterator = &stop;
5382
5383
5384
5385
5386
5387
5388 while (iterator && start <= stop) {
5389 path = ext4_find_extent(inode, *iterator, &path,
5390 EXT4_EX_NOCACHE);
5391 if (IS_ERR(path))
5392 return PTR_ERR(path);
5393 depth = path->p_depth;
5394 extent = path[depth].p_ext;
5395 if (!extent) {
5396 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
5397 (unsigned long) *iterator);
5398 return -EFSCORRUPTED;
5399 }
5400 if (SHIFT == SHIFT_LEFT && *iterator >
5401 le32_to_cpu(extent->ee_block)) {
5402
5403 if (extent < EXT_LAST_EXTENT(path[depth].p_hdr)) {
5404 path[depth].p_ext++;
5405 } else {
5406 *iterator = ext4_ext_next_allocated_block(path);
5407 continue;
5408 }
5409 }
5410
5411 if (SHIFT == SHIFT_LEFT) {
5412 extent = EXT_LAST_EXTENT(path[depth].p_hdr);
5413 *iterator = le32_to_cpu(extent->ee_block) +
5414 ext4_ext_get_actual_len(extent);
5415 } else {
5416 extent = EXT_FIRST_EXTENT(path[depth].p_hdr);
5417 if (le32_to_cpu(extent->ee_block) > 0)
5418 *iterator = le32_to_cpu(extent->ee_block) - 1;
5419 else
5420
5421 iterator = NULL;
5422
5423 while (le32_to_cpu(extent->ee_block) < start)
5424 extent++;
5425 path[depth].p_ext = extent;
5426 }
5427 ret = ext4_ext_shift_path_extents(path, shift, inode,
5428 handle, SHIFT);
5429 if (ret)
5430 break;
5431 }
5432out:
5433 ext4_ext_drop_refs(path);
5434 kfree(path);
5435 return ret;
5436}
5437
5438
5439
5440
5441
5442
5443int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len)
5444{
5445 struct super_block *sb = inode->i_sb;
5446 ext4_lblk_t punch_start, punch_stop;
5447 handle_t *handle;
5448 unsigned int credits;
5449 loff_t new_size, ioffset;
5450 int ret;
5451
5452
5453
5454
5455
5456
5457 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5458 return -EOPNOTSUPP;
5459
5460
5461 if (offset & (EXT4_CLUSTER_SIZE(sb) - 1) ||
5462 len & (EXT4_CLUSTER_SIZE(sb) - 1))
5463 return -EINVAL;
5464
5465 if (!S_ISREG(inode->i_mode))
5466 return -EINVAL;
5467
5468 trace_ext4_collapse_range(inode, offset, len);
5469
5470 punch_start = offset >> EXT4_BLOCK_SIZE_BITS(sb);
5471 punch_stop = (offset + len) >> EXT4_BLOCK_SIZE_BITS(sb);
5472
5473
5474 if (ext4_should_journal_data(inode)) {
5475 ret = ext4_force_commit(inode->i_sb);
5476 if (ret)
5477 return ret;
5478 }
5479
5480 inode_lock(inode);
5481
5482
5483
5484
5485 if (offset + len >= i_size_read(inode)) {
5486 ret = -EINVAL;
5487 goto out_mutex;
5488 }
5489
5490
5491 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
5492 ret = -EOPNOTSUPP;
5493 goto out_mutex;
5494 }
5495
5496
5497 ext4_inode_block_unlocked_dio(inode);
5498 inode_dio_wait(inode);
5499
5500
5501
5502
5503
5504 down_write(&EXT4_I(inode)->i_mmap_sem);
5505
5506
5507
5508
5509 ioffset = round_down(offset, PAGE_SIZE);
5510
5511
5512
5513
5514 ret = filemap_write_and_wait_range(inode->i_mapping, ioffset, offset);
5515 if (ret)
5516 goto out_mmap;
5517
5518
5519
5520
5521
5522 ret = filemap_write_and_wait_range(inode->i_mapping, offset + len,
5523 LLONG_MAX);
5524 if (ret)
5525 goto out_mmap;
5526 truncate_pagecache(inode, ioffset);
5527
5528 credits = ext4_writepage_trans_blocks(inode);
5529 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5530 if (IS_ERR(handle)) {
5531 ret = PTR_ERR(handle);
5532 goto out_mmap;
5533 }
5534
5535 down_write(&EXT4_I(inode)->i_data_sem);
5536 ext4_discard_preallocations(inode);
5537
5538 ret = ext4_es_remove_extent(inode, punch_start,
5539 EXT_MAX_BLOCKS - punch_start);
5540 if (ret) {
5541 up_write(&EXT4_I(inode)->i_data_sem);
5542 goto out_stop;
5543 }
5544
5545 ret = ext4_ext_remove_space(inode, punch_start, punch_stop - 1);
5546 if (ret) {
5547 up_write(&EXT4_I(inode)->i_data_sem);
5548 goto out_stop;
5549 }
5550 ext4_discard_preallocations(inode);
5551
5552 ret = ext4_ext_shift_extents(inode, handle, punch_stop,
5553 punch_stop - punch_start, SHIFT_LEFT);
5554 if (ret) {
5555 up_write(&EXT4_I(inode)->i_data_sem);
5556 goto out_stop;
5557 }
5558
5559 new_size = i_size_read(inode) - len;
5560 i_size_write(inode, new_size);
5561 EXT4_I(inode)->i_disksize = new_size;
5562
5563 up_write(&EXT4_I(inode)->i_data_sem);
5564 if (IS_SYNC(inode))
5565 ext4_handle_sync(handle);
5566 inode->i_mtime = inode->i_ctime = current_time(inode);
5567 ext4_mark_inode_dirty(handle, inode);
5568 ext4_update_inode_fsync_trans(handle, inode, 1);
5569
5570out_stop:
5571 ext4_journal_stop(handle);
5572out_mmap:
5573 up_write(&EXT4_I(inode)->i_mmap_sem);
5574 ext4_inode_resume_unlocked_dio(inode);
5575out_mutex:
5576 inode_unlock(inode);
5577 return ret;
5578}
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len)
5589{
5590 struct super_block *sb = inode->i_sb;
5591 handle_t *handle;
5592 struct ext4_ext_path *path;
5593 struct ext4_extent *extent;
5594 ext4_lblk_t offset_lblk, len_lblk, ee_start_lblk = 0;
5595 unsigned int credits, ee_len;
5596 int ret = 0, depth, split_flag = 0;
5597 loff_t ioffset;
5598
5599
5600
5601
5602
5603
5604 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5605 return -EOPNOTSUPP;
5606
5607
5608 if (offset & (EXT4_CLUSTER_SIZE(sb) - 1) ||
5609 len & (EXT4_CLUSTER_SIZE(sb) - 1))
5610 return -EINVAL;
5611
5612 if (!S_ISREG(inode->i_mode))
5613 return -EOPNOTSUPP;
5614
5615 trace_ext4_insert_range(inode, offset, len);
5616
5617 offset_lblk = offset >> EXT4_BLOCK_SIZE_BITS(sb);
5618 len_lblk = len >> EXT4_BLOCK_SIZE_BITS(sb);
5619
5620
5621 if (ext4_should_journal_data(inode)) {
5622 ret = ext4_force_commit(inode->i_sb);
5623 if (ret)
5624 return ret;
5625 }
5626
5627 inode_lock(inode);
5628
5629 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
5630 ret = -EOPNOTSUPP;
5631 goto out_mutex;
5632 }
5633
5634
5635 if (inode->i_size + len > inode->i_sb->s_maxbytes) {
5636 ret = -EFBIG;
5637 goto out_mutex;
5638 }
5639
5640
5641 if (offset >= i_size_read(inode)) {
5642 ret = -EINVAL;
5643 goto out_mutex;
5644 }
5645
5646
5647 ext4_inode_block_unlocked_dio(inode);
5648 inode_dio_wait(inode);
5649
5650
5651
5652
5653
5654 down_write(&EXT4_I(inode)->i_mmap_sem);
5655
5656
5657
5658
5659 ioffset = round_down(offset, PAGE_SIZE);
5660
5661 ret = filemap_write_and_wait_range(inode->i_mapping, ioffset,
5662 LLONG_MAX);
5663 if (ret)
5664 goto out_mmap;
5665 truncate_pagecache(inode, ioffset);
5666
5667 credits = ext4_writepage_trans_blocks(inode);
5668 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5669 if (IS_ERR(handle)) {
5670 ret = PTR_ERR(handle);
5671 goto out_mmap;
5672 }
5673
5674
5675 inode->i_size += len;
5676 EXT4_I(inode)->i_disksize += len;
5677 inode->i_mtime = inode->i_ctime = current_time(inode);
5678 ret = ext4_mark_inode_dirty(handle, inode);
5679 if (ret)
5680 goto out_stop;
5681
5682 down_write(&EXT4_I(inode)->i_data_sem);
5683 ext4_discard_preallocations(inode);
5684
5685 path = ext4_find_extent(inode, offset_lblk, NULL, 0);
5686 if (IS_ERR(path)) {
5687 up_write(&EXT4_I(inode)->i_data_sem);
5688 goto out_stop;
5689 }
5690
5691 depth = ext_depth(inode);
5692 extent = path[depth].p_ext;
5693 if (extent) {
5694 ee_start_lblk = le32_to_cpu(extent->ee_block);
5695 ee_len = ext4_ext_get_actual_len(extent);
5696
5697
5698
5699
5700
5701 if ((offset_lblk > ee_start_lblk) &&
5702 (offset_lblk < (ee_start_lblk + ee_len))) {
5703 if (ext4_ext_is_unwritten(extent))
5704 split_flag = EXT4_EXT_MARK_UNWRIT1 |
5705 EXT4_EXT_MARK_UNWRIT2;
5706 ret = ext4_split_extent_at(handle, inode, &path,
5707 offset_lblk, split_flag,
5708 EXT4_EX_NOCACHE |
5709 EXT4_GET_BLOCKS_PRE_IO |
5710 EXT4_GET_BLOCKS_METADATA_NOFAIL);
5711 }
5712
5713 ext4_ext_drop_refs(path);
5714 kfree(path);
5715 if (ret < 0) {
5716 up_write(&EXT4_I(inode)->i_data_sem);
5717 goto out_stop;
5718 }
5719 } else {
5720 ext4_ext_drop_refs(path);
5721 kfree(path);
5722 }
5723
5724 ret = ext4_es_remove_extent(inode, offset_lblk,
5725 EXT_MAX_BLOCKS - offset_lblk);
5726 if (ret) {
5727 up_write(&EXT4_I(inode)->i_data_sem);
5728 goto out_stop;
5729 }
5730
5731
5732
5733
5734
5735 ret = ext4_ext_shift_extents(inode, handle,
5736 ee_start_lblk > offset_lblk ? ee_start_lblk : offset_lblk,
5737 len_lblk, SHIFT_RIGHT);
5738
5739 up_write(&EXT4_I(inode)->i_data_sem);
5740 if (IS_SYNC(inode))
5741 ext4_handle_sync(handle);
5742 if (ret >= 0)
5743 ext4_update_inode_fsync_trans(handle, inode, 1);
5744
5745out_stop:
5746 ext4_journal_stop(handle);
5747out_mmap:
5748 up_write(&EXT4_I(inode)->i_mmap_sem);
5749 ext4_inode_resume_unlocked_dio(inode);
5750out_mutex:
5751 inode_unlock(inode);
5752 return ret;
5753}
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775int
5776ext4_swap_extents(handle_t *handle, struct inode *inode1,
5777 struct inode *inode2, ext4_lblk_t lblk1, ext4_lblk_t lblk2,
5778 ext4_lblk_t count, int unwritten, int *erp)
5779{
5780 struct ext4_ext_path *path1 = NULL;
5781 struct ext4_ext_path *path2 = NULL;
5782 int replaced_count = 0;
5783
5784 BUG_ON(!rwsem_is_locked(&EXT4_I(inode1)->i_data_sem));
5785 BUG_ON(!rwsem_is_locked(&EXT4_I(inode2)->i_data_sem));
5786 BUG_ON(!inode_is_locked(inode1));
5787 BUG_ON(!inode_is_locked(inode2));
5788
5789 *erp = ext4_es_remove_extent(inode1, lblk1, count);
5790 if (unlikely(*erp))
5791 return 0;
5792 *erp = ext4_es_remove_extent(inode2, lblk2, count);
5793 if (unlikely(*erp))
5794 return 0;
5795
5796 while (count) {
5797 struct ext4_extent *ex1, *ex2, tmp_ex;
5798 ext4_lblk_t e1_blk, e2_blk;
5799 int e1_len, e2_len, len;
5800 int split = 0;
5801
5802 path1 = ext4_find_extent(inode1, lblk1, NULL, EXT4_EX_NOCACHE);
5803 if (IS_ERR(path1)) {
5804 *erp = PTR_ERR(path1);
5805 path1 = NULL;
5806 finish:
5807 count = 0;
5808 goto repeat;
5809 }
5810 path2 = ext4_find_extent(inode2, lblk2, NULL, EXT4_EX_NOCACHE);
5811 if (IS_ERR(path2)) {
5812 *erp = PTR_ERR(path2);
5813 path2 = NULL;
5814 goto finish;
5815 }
5816 ex1 = path1[path1->p_depth].p_ext;
5817 ex2 = path2[path2->p_depth].p_ext;
5818
5819 if (unlikely(!ex2 || !ex1))
5820 goto finish;
5821
5822 e1_blk = le32_to_cpu(ex1->ee_block);
5823 e2_blk = le32_to_cpu(ex2->ee_block);
5824 e1_len = ext4_ext_get_actual_len(ex1);
5825 e2_len = ext4_ext_get_actual_len(ex2);
5826
5827
5828 if (!in_range(lblk1, e1_blk, e1_len) ||
5829 !in_range(lblk2, e2_blk, e2_len)) {
5830 ext4_lblk_t next1, next2;
5831
5832
5833 next1 = ext4_ext_next_allocated_block(path1);
5834 next2 = ext4_ext_next_allocated_block(path2);
5835
5836 if (e1_blk > lblk1)
5837 next1 = e1_blk;
5838 if (e2_blk > lblk2)
5839 next2 = e2_blk;
5840
5841 if (next1 == EXT_MAX_BLOCKS || next2 == EXT_MAX_BLOCKS)
5842 goto finish;
5843
5844 len = next1 - lblk1;
5845 if (len < next2 - lblk2)
5846 len = next2 - lblk2;
5847 if (len > count)
5848 len = count;
5849 lblk1 += len;
5850 lblk2 += len;
5851 count -= len;
5852 goto repeat;
5853 }
5854
5855
5856 if (e1_blk < lblk1) {
5857 split = 1;
5858 *erp = ext4_force_split_extent_at(handle, inode1,
5859 &path1, lblk1, 0);
5860 if (unlikely(*erp))
5861 goto finish;
5862 }
5863 if (e2_blk < lblk2) {
5864 split = 1;
5865 *erp = ext4_force_split_extent_at(handle, inode2,
5866 &path2, lblk2, 0);
5867 if (unlikely(*erp))
5868 goto finish;
5869 }
5870
5871
5872 if (split)
5873 goto repeat;
5874
5875
5876 len = count;
5877 if (len > e1_blk + e1_len - lblk1)
5878 len = e1_blk + e1_len - lblk1;
5879 if (len > e2_blk + e2_len - lblk2)
5880 len = e2_blk + e2_len - lblk2;
5881
5882 if (len != e1_len) {
5883 split = 1;
5884 *erp = ext4_force_split_extent_at(handle, inode1,
5885 &path1, lblk1 + len, 0);
5886 if (unlikely(*erp))
5887 goto finish;
5888 }
5889 if (len != e2_len) {
5890 split = 1;
5891 *erp = ext4_force_split_extent_at(handle, inode2,
5892 &path2, lblk2 + len, 0);
5893 if (*erp)
5894 goto finish;
5895 }
5896
5897
5898 if (split)
5899 goto repeat;
5900
5901 BUG_ON(e2_len != e1_len);
5902 *erp = ext4_ext_get_access(handle, inode1, path1 + path1->p_depth);
5903 if (unlikely(*erp))
5904 goto finish;
5905 *erp = ext4_ext_get_access(handle, inode2, path2 + path2->p_depth);
5906 if (unlikely(*erp))
5907 goto finish;
5908
5909
5910 tmp_ex = *ex1;
5911 ext4_ext_store_pblock(ex1, ext4_ext_pblock(ex2));
5912 ext4_ext_store_pblock(ex2, ext4_ext_pblock(&tmp_ex));
5913 ex1->ee_len = cpu_to_le16(e2_len);
5914 ex2->ee_len = cpu_to_le16(e1_len);
5915 if (unwritten)
5916 ext4_ext_mark_unwritten(ex2);
5917 if (ext4_ext_is_unwritten(&tmp_ex))
5918 ext4_ext_mark_unwritten(ex1);
5919
5920 ext4_ext_try_to_merge(handle, inode2, path2, ex2);
5921 ext4_ext_try_to_merge(handle, inode1, path1, ex1);
5922 *erp = ext4_ext_dirty(handle, inode2, path2 +
5923 path2->p_depth);
5924 if (unlikely(*erp))
5925 goto finish;
5926 *erp = ext4_ext_dirty(handle, inode1, path1 +
5927 path1->p_depth);
5928
5929
5930
5931
5932
5933
5934 if (unlikely(*erp))
5935 goto finish;
5936 lblk1 += len;
5937 lblk2 += len;
5938 replaced_count += len;
5939 count -= len;
5940
5941 repeat:
5942 ext4_ext_drop_refs(path1);
5943 kfree(path1);
5944 ext4_ext_drop_refs(path2);
5945 kfree(path2);
5946 path1 = path2 = NULL;
5947 }
5948 return replaced_count;
5949}
5950