1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/fs.h>
21#include <linux/time.h>
22#include <linux/jbd2.h>
23#include <linux/highuid.h>
24#include <linux/pagemap.h>
25#include <linux/quotaops.h>
26#include <linux/string.h>
27#include <linux/slab.h>
28#include <linux/uaccess.h>
29#include <linux/fiemap.h>
30#include <linux/backing-dev.h>
31#include <linux/iomap.h>
32#include "ext4_jbd2.h"
33#include "ext4_extents.h"
34#include "xattr.h"
35
36#include <trace/events/ext4.h>
37
38
39
40
41#define EXT4_EXT_MAY_ZEROOUT 0x1
42
43#define EXT4_EXT_MARK_UNWRIT1 0x2
44#define EXT4_EXT_MARK_UNWRIT2 0x4
45
46#define EXT4_EXT_DATA_VALID1 0x8
47#define EXT4_EXT_DATA_VALID2 0x10
48
49static __le32 ext4_extent_block_csum(struct inode *inode,
50 struct ext4_extent_header *eh)
51{
52 struct ext4_inode_info *ei = EXT4_I(inode);
53 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
54 __u32 csum;
55
56 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)eh,
57 EXT4_EXTENT_TAIL_OFFSET(eh));
58 return cpu_to_le32(csum);
59}
60
61static int ext4_extent_block_csum_verify(struct inode *inode,
62 struct ext4_extent_header *eh)
63{
64 struct ext4_extent_tail *et;
65
66 if (!ext4_has_metadata_csum(inode->i_sb))
67 return 1;
68
69 et = find_ext4_extent_tail(eh);
70 if (et->et_checksum != ext4_extent_block_csum(inode, eh))
71 return 0;
72 return 1;
73}
74
75static void ext4_extent_block_csum_set(struct inode *inode,
76 struct ext4_extent_header *eh)
77{
78 struct ext4_extent_tail *et;
79
80 if (!ext4_has_metadata_csum(inode->i_sb))
81 return;
82
83 et = find_ext4_extent_tail(eh);
84 et->et_checksum = ext4_extent_block_csum(inode, eh);
85}
86
87static int ext4_split_extent_at(handle_t *handle,
88 struct inode *inode,
89 struct ext4_ext_path **ppath,
90 ext4_lblk_t split,
91 int split_flag,
92 int flags);
93
94static int ext4_ext_trunc_restart_fn(struct inode *inode, int *dropped)
95{
96
97
98
99
100
101
102 BUG_ON(EXT4_JOURNAL(inode) == NULL);
103 ext4_discard_preallocations(inode, 0);
104 up_write(&EXT4_I(inode)->i_data_sem);
105 *dropped = 1;
106 return 0;
107}
108
109
110
111
112
113
114
115
116
117int ext4_datasem_ensure_credits(handle_t *handle, struct inode *inode,
118 int check_cred, int restart_cred,
119 int revoke_cred)
120{
121 int ret;
122 int dropped = 0;
123
124 ret = ext4_journal_ensure_credits_fn(handle, check_cred, restart_cred,
125 revoke_cred, ext4_ext_trunc_restart_fn(inode, &dropped));
126 if (dropped)
127 down_write(&EXT4_I(inode)->i_data_sem);
128 return ret;
129}
130
131
132
133
134
135
136static int ext4_ext_get_access(handle_t *handle, struct inode *inode,
137 struct ext4_ext_path *path)
138{
139 if (path->p_bh) {
140
141 BUFFER_TRACE(path->p_bh, "get_write_access");
142 return ext4_journal_get_write_access(handle, path->p_bh);
143 }
144
145
146 return 0;
147}
148
149
150
151
152
153
154
155static int __ext4_ext_dirty(const char *where, unsigned int line,
156 handle_t *handle, struct inode *inode,
157 struct ext4_ext_path *path)
158{
159 int err;
160
161 WARN_ON(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem));
162 if (path->p_bh) {
163 ext4_extent_block_csum_set(inode, ext_block_hdr(path->p_bh));
164
165 err = __ext4_handle_dirty_metadata(where, line, handle,
166 inode, path->p_bh);
167 } else {
168
169 err = ext4_mark_inode_dirty(handle, inode);
170 }
171 return err;
172}
173
174#define ext4_ext_dirty(handle, inode, path) \
175 __ext4_ext_dirty(__func__, __LINE__, (handle), (inode), (path))
176
177static ext4_fsblk_t ext4_ext_find_goal(struct inode *inode,
178 struct ext4_ext_path *path,
179 ext4_lblk_t block)
180{
181 if (path) {
182 int depth = path->p_depth;
183 struct ext4_extent *ex;
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202 ex = path[depth].p_ext;
203 if (ex) {
204 ext4_fsblk_t ext_pblk = ext4_ext_pblock(ex);
205 ext4_lblk_t ext_block = le32_to_cpu(ex->ee_block);
206
207 if (block > ext_block)
208 return ext_pblk + (block - ext_block);
209 else
210 return ext_pblk - (ext_block - block);
211 }
212
213
214
215 if (path[depth].p_bh)
216 return path[depth].p_bh->b_blocknr;
217 }
218
219
220 return ext4_inode_to_goal_block(inode);
221}
222
223
224
225
226static ext4_fsblk_t
227ext4_ext_new_meta_block(handle_t *handle, struct inode *inode,
228 struct ext4_ext_path *path,
229 struct ext4_extent *ex, int *err, unsigned int flags)
230{
231 ext4_fsblk_t goal, newblock;
232
233 goal = ext4_ext_find_goal(inode, path, le32_to_cpu(ex->ee_block));
234 newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
235 NULL, err);
236 return newblock;
237}
238
239static inline int ext4_ext_space_block(struct inode *inode, int check)
240{
241 int size;
242
243 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
244 / sizeof(struct ext4_extent);
245#ifdef AGGRESSIVE_TEST
246 if (!check && size > 6)
247 size = 6;
248#endif
249 return size;
250}
251
252static inline int ext4_ext_space_block_idx(struct inode *inode, int check)
253{
254 int size;
255
256 size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header))
257 / sizeof(struct ext4_extent_idx);
258#ifdef AGGRESSIVE_TEST
259 if (!check && size > 5)
260 size = 5;
261#endif
262 return size;
263}
264
265static inline int ext4_ext_space_root(struct inode *inode, int check)
266{
267 int size;
268
269 size = sizeof(EXT4_I(inode)->i_data);
270 size -= sizeof(struct ext4_extent_header);
271 size /= sizeof(struct ext4_extent);
272#ifdef AGGRESSIVE_TEST
273 if (!check && size > 3)
274 size = 3;
275#endif
276 return size;
277}
278
279static inline int ext4_ext_space_root_idx(struct inode *inode, int check)
280{
281 int size;
282
283 size = sizeof(EXT4_I(inode)->i_data);
284 size -= sizeof(struct ext4_extent_header);
285 size /= sizeof(struct ext4_extent_idx);
286#ifdef AGGRESSIVE_TEST
287 if (!check && size > 4)
288 size = 4;
289#endif
290 return size;
291}
292
293static inline int
294ext4_force_split_extent_at(handle_t *handle, struct inode *inode,
295 struct ext4_ext_path **ppath, ext4_lblk_t lblk,
296 int nofail)
297{
298 struct ext4_ext_path *path = *ppath;
299 int unwritten = ext4_ext_is_unwritten(path[path->p_depth].p_ext);
300 int flags = EXT4_EX_NOCACHE | EXT4_GET_BLOCKS_PRE_IO;
301
302 if (nofail)
303 flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL | EXT4_EX_NOFAIL;
304
305 return ext4_split_extent_at(handle, inode, ppath, lblk, unwritten ?
306 EXT4_EXT_MARK_UNWRIT1|EXT4_EXT_MARK_UNWRIT2 : 0,
307 flags);
308}
309
310static int
311ext4_ext_max_entries(struct inode *inode, int depth)
312{
313 int max;
314
315 if (depth == ext_depth(inode)) {
316 if (depth == 0)
317 max = ext4_ext_space_root(inode, 1);
318 else
319 max = ext4_ext_space_root_idx(inode, 1);
320 } else {
321 if (depth == 0)
322 max = ext4_ext_space_block(inode, 1);
323 else
324 max = ext4_ext_space_block_idx(inode, 1);
325 }
326
327 return max;
328}
329
330static int ext4_valid_extent(struct inode *inode, struct ext4_extent *ext)
331{
332 ext4_fsblk_t block = ext4_ext_pblock(ext);
333 int len = ext4_ext_get_actual_len(ext);
334 ext4_lblk_t lblock = le32_to_cpu(ext->ee_block);
335
336
337
338
339
340
341 if (lblock + len <= lblock)
342 return 0;
343 return ext4_inode_block_valid(inode, block, len);
344}
345
346static int ext4_valid_extent_idx(struct inode *inode,
347 struct ext4_extent_idx *ext_idx)
348{
349 ext4_fsblk_t block = ext4_idx_pblock(ext_idx);
350
351 return ext4_inode_block_valid(inode, block, 1);
352}
353
354static int ext4_valid_extent_entries(struct inode *inode,
355 struct ext4_extent_header *eh,
356 ext4_fsblk_t *pblk, int depth)
357{
358 unsigned short entries;
359 if (eh->eh_entries == 0)
360 return 1;
361
362 entries = le16_to_cpu(eh->eh_entries);
363
364 if (depth == 0) {
365
366 struct ext4_extent *ext = EXT_FIRST_EXTENT(eh);
367 ext4_lblk_t lblock = 0;
368 ext4_lblk_t prev = 0;
369 int len = 0;
370 while (entries) {
371 if (!ext4_valid_extent(inode, ext))
372 return 0;
373
374
375 lblock = le32_to_cpu(ext->ee_block);
376 len = ext4_ext_get_actual_len(ext);
377 if ((lblock <= prev) && prev) {
378 *pblk = ext4_ext_pblock(ext);
379 return 0;
380 }
381 ext++;
382 entries--;
383 prev = lblock + len - 1;
384 }
385 } else {
386 struct ext4_extent_idx *ext_idx = EXT_FIRST_INDEX(eh);
387 while (entries) {
388 if (!ext4_valid_extent_idx(inode, ext_idx))
389 return 0;
390 ext_idx++;
391 entries--;
392 }
393 }
394 return 1;
395}
396
397static int __ext4_ext_check(const char *function, unsigned int line,
398 struct inode *inode, struct ext4_extent_header *eh,
399 int depth, ext4_fsblk_t pblk)
400{
401 const char *error_msg;
402 int max = 0, err = -EFSCORRUPTED;
403
404 if (unlikely(eh->eh_magic != EXT4_EXT_MAGIC)) {
405 error_msg = "invalid magic";
406 goto corrupted;
407 }
408 if (unlikely(le16_to_cpu(eh->eh_depth) != depth)) {
409 error_msg = "unexpected eh_depth";
410 goto corrupted;
411 }
412 if (unlikely(eh->eh_max == 0)) {
413 error_msg = "invalid eh_max";
414 goto corrupted;
415 }
416 max = ext4_ext_max_entries(inode, depth);
417 if (unlikely(le16_to_cpu(eh->eh_max) > max)) {
418 error_msg = "too large eh_max";
419 goto corrupted;
420 }
421 if (unlikely(le16_to_cpu(eh->eh_entries) > le16_to_cpu(eh->eh_max))) {
422 error_msg = "invalid eh_entries";
423 goto corrupted;
424 }
425 if (!ext4_valid_extent_entries(inode, eh, &pblk, depth)) {
426 error_msg = "invalid extent entries";
427 goto corrupted;
428 }
429 if (unlikely(depth > 32)) {
430 error_msg = "too large eh_depth";
431 goto corrupted;
432 }
433
434 if (ext_depth(inode) != depth &&
435 !ext4_extent_block_csum_verify(inode, eh)) {
436 error_msg = "extent tree corrupted";
437 err = -EFSBADCRC;
438 goto corrupted;
439 }
440 return 0;
441
442corrupted:
443 ext4_error_inode_err(inode, function, line, 0, -err,
444 "pblk %llu bad header/extent: %s - magic %x, "
445 "entries %u, max %u(%u), depth %u(%u)",
446 (unsigned long long) pblk, error_msg,
447 le16_to_cpu(eh->eh_magic),
448 le16_to_cpu(eh->eh_entries),
449 le16_to_cpu(eh->eh_max),
450 max, le16_to_cpu(eh->eh_depth), depth);
451 return err;
452}
453
454#define ext4_ext_check(inode, eh, depth, pblk) \
455 __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk))
456
457int ext4_ext_check_inode(struct inode *inode)
458{
459 return ext4_ext_check(inode, ext_inode_hdr(inode), ext_depth(inode), 0);
460}
461
462static void ext4_cache_extents(struct inode *inode,
463 struct ext4_extent_header *eh)
464{
465 struct ext4_extent *ex = EXT_FIRST_EXTENT(eh);
466 ext4_lblk_t prev = 0;
467 int i;
468
469 for (i = le16_to_cpu(eh->eh_entries); i > 0; i--, ex++) {
470 unsigned int status = EXTENT_STATUS_WRITTEN;
471 ext4_lblk_t lblk = le32_to_cpu(ex->ee_block);
472 int len = ext4_ext_get_actual_len(ex);
473
474 if (prev && (prev != lblk))
475 ext4_es_cache_extent(inode, prev, lblk - prev, ~0,
476 EXTENT_STATUS_HOLE);
477
478 if (ext4_ext_is_unwritten(ex))
479 status = EXTENT_STATUS_UNWRITTEN;
480 ext4_es_cache_extent(inode, lblk, len,
481 ext4_ext_pblock(ex), status);
482 prev = lblk + len;
483 }
484}
485
486static struct buffer_head *
487__read_extent_tree_block(const char *function, unsigned int line,
488 struct inode *inode, ext4_fsblk_t pblk, int depth,
489 int flags)
490{
491 struct buffer_head *bh;
492 int err;
493 gfp_t gfp_flags = __GFP_MOVABLE | GFP_NOFS;
494
495 if (flags & EXT4_EX_NOFAIL)
496 gfp_flags |= __GFP_NOFAIL;
497
498 bh = sb_getblk_gfp(inode->i_sb, pblk, gfp_flags);
499 if (unlikely(!bh))
500 return ERR_PTR(-ENOMEM);
501
502 if (!bh_uptodate_or_lock(bh)) {
503 trace_ext4_ext_load_extent(inode, pblk, _RET_IP_);
504 err = bh_submit_read(bh);
505 if (err < 0)
506 goto errout;
507 }
508 if (buffer_verified(bh) && !(flags & EXT4_EX_FORCE_CACHE))
509 return bh;
510 err = __ext4_ext_check(function, line, inode,
511 ext_block_hdr(bh), depth, pblk);
512 if (err)
513 goto errout;
514 set_buffer_verified(bh);
515
516
517
518 if (!(flags & EXT4_EX_NOCACHE) && depth == 0) {
519 struct ext4_extent_header *eh = ext_block_hdr(bh);
520 ext4_cache_extents(inode, eh);
521 }
522 return bh;
523errout:
524 put_bh(bh);
525 return ERR_PTR(err);
526
527}
528
529#define read_extent_tree_block(inode, pblk, depth, flags) \
530 __read_extent_tree_block(__func__, __LINE__, (inode), (pblk), \
531 (depth), (flags))
532
533
534
535
536
537int ext4_ext_precache(struct inode *inode)
538{
539 struct ext4_inode_info *ei = EXT4_I(inode);
540 struct ext4_ext_path *path = NULL;
541 struct buffer_head *bh;
542 int i = 0, depth, ret = 0;
543
544 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
545 return 0;
546
547 down_read(&ei->i_data_sem);
548 depth = ext_depth(inode);
549
550
551 if (!depth) {
552 up_read(&ei->i_data_sem);
553 return ret;
554 }
555
556 path = kcalloc(depth + 1, sizeof(struct ext4_ext_path),
557 GFP_NOFS);
558 if (path == NULL) {
559 up_read(&ei->i_data_sem);
560 return -ENOMEM;
561 }
562
563 path[0].p_hdr = ext_inode_hdr(inode);
564 ret = ext4_ext_check(inode, path[0].p_hdr, depth, 0);
565 if (ret)
566 goto out;
567 path[0].p_idx = EXT_FIRST_INDEX(path[0].p_hdr);
568 while (i >= 0) {
569
570
571
572
573 if ((i == depth) ||
574 path[i].p_idx > EXT_LAST_INDEX(path[i].p_hdr)) {
575 brelse(path[i].p_bh);
576 path[i].p_bh = NULL;
577 i--;
578 continue;
579 }
580 bh = read_extent_tree_block(inode,
581 ext4_idx_pblock(path[i].p_idx++),
582 depth - i - 1,
583 EXT4_EX_FORCE_CACHE);
584 if (IS_ERR(bh)) {
585 ret = PTR_ERR(bh);
586 break;
587 }
588 i++;
589 path[i].p_bh = bh;
590 path[i].p_hdr = ext_block_hdr(bh);
591 path[i].p_idx = EXT_FIRST_INDEX(path[i].p_hdr);
592 }
593 ext4_set_inode_state(inode, EXT4_STATE_EXT_PRECACHED);
594out:
595 up_read(&ei->i_data_sem);
596 ext4_ext_drop_refs(path);
597 kfree(path);
598 return ret;
599}
600
601#ifdef EXT_DEBUG
602static void ext4_ext_show_path(struct inode *inode, struct ext4_ext_path *path)
603{
604 int k, l = path->p_depth;
605
606 ext_debug(inode, "path:");
607 for (k = 0; k <= l; k++, path++) {
608 if (path->p_idx) {
609 ext_debug(inode, " %d->%llu",
610 le32_to_cpu(path->p_idx->ei_block),
611 ext4_idx_pblock(path->p_idx));
612 } else if (path->p_ext) {
613 ext_debug(inode, " %d:[%d]%d:%llu ",
614 le32_to_cpu(path->p_ext->ee_block),
615 ext4_ext_is_unwritten(path->p_ext),
616 ext4_ext_get_actual_len(path->p_ext),
617 ext4_ext_pblock(path->p_ext));
618 } else
619 ext_debug(inode, " []");
620 }
621 ext_debug(inode, "\n");
622}
623
624static void ext4_ext_show_leaf(struct inode *inode, struct ext4_ext_path *path)
625{
626 int depth = ext_depth(inode);
627 struct ext4_extent_header *eh;
628 struct ext4_extent *ex;
629 int i;
630
631 if (!path)
632 return;
633
634 eh = path[depth].p_hdr;
635 ex = EXT_FIRST_EXTENT(eh);
636
637 ext_debug(inode, "Displaying leaf extents\n");
638
639 for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ex++) {
640 ext_debug(inode, "%d:[%d]%d:%llu ", le32_to_cpu(ex->ee_block),
641 ext4_ext_is_unwritten(ex),
642 ext4_ext_get_actual_len(ex), ext4_ext_pblock(ex));
643 }
644 ext_debug(inode, "\n");
645}
646
647static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path,
648 ext4_fsblk_t newblock, int level)
649{
650 int depth = ext_depth(inode);
651 struct ext4_extent *ex;
652
653 if (depth != level) {
654 struct ext4_extent_idx *idx;
655 idx = path[level].p_idx;
656 while (idx <= EXT_MAX_INDEX(path[level].p_hdr)) {
657 ext_debug(inode, "%d: move %d:%llu in new index %llu\n",
658 level, le32_to_cpu(idx->ei_block),
659 ext4_idx_pblock(idx), newblock);
660 idx++;
661 }
662
663 return;
664 }
665
666 ex = path[depth].p_ext;
667 while (ex <= EXT_MAX_EXTENT(path[depth].p_hdr)) {
668 ext_debug(inode, "move %d:%llu:[%d]%d in new leaf %llu\n",
669 le32_to_cpu(ex->ee_block),
670 ext4_ext_pblock(ex),
671 ext4_ext_is_unwritten(ex),
672 ext4_ext_get_actual_len(ex),
673 newblock);
674 ex++;
675 }
676}
677
678#else
679#define ext4_ext_show_path(inode, path)
680#define ext4_ext_show_leaf(inode, path)
681#define ext4_ext_show_move(inode, path, newblock, level)
682#endif
683
684void ext4_ext_drop_refs(struct ext4_ext_path *path)
685{
686 int depth, i;
687
688 if (!path)
689 return;
690 depth = path->p_depth;
691 for (i = 0; i <= depth; i++, path++) {
692 brelse(path->p_bh);
693 path->p_bh = NULL;
694 }
695}
696
697
698
699
700
701
702static void
703ext4_ext_binsearch_idx(struct inode *inode,
704 struct ext4_ext_path *path, ext4_lblk_t block)
705{
706 struct ext4_extent_header *eh = path->p_hdr;
707 struct ext4_extent_idx *r, *l, *m;
708
709
710 ext_debug(inode, "binsearch for %u(idx): ", block);
711
712 l = EXT_FIRST_INDEX(eh) + 1;
713 r = EXT_LAST_INDEX(eh);
714 while (l <= r) {
715 m = l + (r - l) / 2;
716 if (block < le32_to_cpu(m->ei_block))
717 r = m - 1;
718 else
719 l = m + 1;
720 ext_debug(inode, "%p(%u):%p(%u):%p(%u) ", l,
721 le32_to_cpu(l->ei_block), m, le32_to_cpu(m->ei_block),
722 r, le32_to_cpu(r->ei_block));
723 }
724
725 path->p_idx = l - 1;
726 ext_debug(inode, " -> %u->%lld ", le32_to_cpu(path->p_idx->ei_block),
727 ext4_idx_pblock(path->p_idx));
728
729#ifdef CHECK_BINSEARCH
730 {
731 struct ext4_extent_idx *chix, *ix;
732 int k;
733
734 chix = ix = EXT_FIRST_INDEX(eh);
735 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ix++) {
736 if (k != 0 && le32_to_cpu(ix->ei_block) <=
737 le32_to_cpu(ix[-1].ei_block)) {
738 printk(KERN_DEBUG "k=%d, ix=0x%p, "
739 "first=0x%p\n", k,
740 ix, EXT_FIRST_INDEX(eh));
741 printk(KERN_DEBUG "%u <= %u\n",
742 le32_to_cpu(ix->ei_block),
743 le32_to_cpu(ix[-1].ei_block));
744 }
745 BUG_ON(k && le32_to_cpu(ix->ei_block)
746 <= le32_to_cpu(ix[-1].ei_block));
747 if (block < le32_to_cpu(ix->ei_block))
748 break;
749 chix = ix;
750 }
751 BUG_ON(chix != path->p_idx);
752 }
753#endif
754
755}
756
757
758
759
760
761
762static void
763ext4_ext_binsearch(struct inode *inode,
764 struct ext4_ext_path *path, ext4_lblk_t block)
765{
766 struct ext4_extent_header *eh = path->p_hdr;
767 struct ext4_extent *r, *l, *m;
768
769 if (eh->eh_entries == 0) {
770
771
772
773
774 return;
775 }
776
777 ext_debug(inode, "binsearch for %u: ", block);
778
779 l = EXT_FIRST_EXTENT(eh) + 1;
780 r = EXT_LAST_EXTENT(eh);
781
782 while (l <= r) {
783 m = l + (r - l) / 2;
784 if (block < le32_to_cpu(m->ee_block))
785 r = m - 1;
786 else
787 l = m + 1;
788 ext_debug(inode, "%p(%u):%p(%u):%p(%u) ", l,
789 le32_to_cpu(l->ee_block), m, le32_to_cpu(m->ee_block),
790 r, le32_to_cpu(r->ee_block));
791 }
792
793 path->p_ext = l - 1;
794 ext_debug(inode, " -> %d:%llu:[%d]%d ",
795 le32_to_cpu(path->p_ext->ee_block),
796 ext4_ext_pblock(path->p_ext),
797 ext4_ext_is_unwritten(path->p_ext),
798 ext4_ext_get_actual_len(path->p_ext));
799
800#ifdef CHECK_BINSEARCH
801 {
802 struct ext4_extent *chex, *ex;
803 int k;
804
805 chex = ex = EXT_FIRST_EXTENT(eh);
806 for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ex++) {
807 BUG_ON(k && le32_to_cpu(ex->ee_block)
808 <= le32_to_cpu(ex[-1].ee_block));
809 if (block < le32_to_cpu(ex->ee_block))
810 break;
811 chex = ex;
812 }
813 BUG_ON(chex != path->p_ext);
814 }
815#endif
816
817}
818
819void ext4_ext_tree_init(handle_t *handle, struct inode *inode)
820{
821 struct ext4_extent_header *eh;
822
823 eh = ext_inode_hdr(inode);
824 eh->eh_depth = 0;
825 eh->eh_entries = 0;
826 eh->eh_magic = EXT4_EXT_MAGIC;
827 eh->eh_max = cpu_to_le16(ext4_ext_space_root(inode, 0));
828 ext4_mark_inode_dirty(handle, inode);
829}
830
831struct ext4_ext_path *
832ext4_find_extent(struct inode *inode, ext4_lblk_t block,
833 struct ext4_ext_path **orig_path, int flags)
834{
835 struct ext4_extent_header *eh;
836 struct buffer_head *bh;
837 struct ext4_ext_path *path = orig_path ? *orig_path : NULL;
838 short int depth, i, ppos = 0;
839 int ret;
840 gfp_t gfp_flags = GFP_NOFS;
841
842 if (flags & EXT4_EX_NOFAIL)
843 gfp_flags |= __GFP_NOFAIL;
844
845 eh = ext_inode_hdr(inode);
846 depth = ext_depth(inode);
847 if (depth < 0 || depth > EXT4_MAX_EXTENT_DEPTH) {
848 EXT4_ERROR_INODE(inode, "inode has invalid extent depth: %d",
849 depth);
850 ret = -EFSCORRUPTED;
851 goto err;
852 }
853
854 if (path) {
855 ext4_ext_drop_refs(path);
856 if (depth > path[0].p_maxdepth) {
857 kfree(path);
858 *orig_path = path = NULL;
859 }
860 }
861 if (!path) {
862
863 path = kcalloc(depth + 2, sizeof(struct ext4_ext_path),
864 gfp_flags);
865 if (unlikely(!path))
866 return ERR_PTR(-ENOMEM);
867 path[0].p_maxdepth = depth + 1;
868 }
869 path[0].p_hdr = eh;
870 path[0].p_bh = NULL;
871
872 i = depth;
873 if (!(flags & EXT4_EX_NOCACHE) && depth == 0)
874 ext4_cache_extents(inode, eh);
875
876 while (i) {
877 ext_debug(inode, "depth %d: num %d, max %d\n",
878 ppos, le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
879
880 ext4_ext_binsearch_idx(inode, path + ppos, block);
881 path[ppos].p_block = ext4_idx_pblock(path[ppos].p_idx);
882 path[ppos].p_depth = i;
883 path[ppos].p_ext = NULL;
884
885 bh = read_extent_tree_block(inode, path[ppos].p_block, --i,
886 flags);
887 if (IS_ERR(bh)) {
888 ret = PTR_ERR(bh);
889 goto err;
890 }
891
892 eh = ext_block_hdr(bh);
893 ppos++;
894 path[ppos].p_bh = bh;
895 path[ppos].p_hdr = eh;
896 }
897
898 path[ppos].p_depth = i;
899 path[ppos].p_ext = NULL;
900 path[ppos].p_idx = NULL;
901
902
903 ext4_ext_binsearch(inode, path + ppos, block);
904
905 if (path[ppos].p_ext)
906 path[ppos].p_block = ext4_ext_pblock(path[ppos].p_ext);
907
908 ext4_ext_show_path(inode, path);
909
910 return path;
911
912err:
913 ext4_ext_drop_refs(path);
914 kfree(path);
915 if (orig_path)
916 *orig_path = NULL;
917 return ERR_PTR(ret);
918}
919
920
921
922
923
924
925static int ext4_ext_insert_index(handle_t *handle, struct inode *inode,
926 struct ext4_ext_path *curp,
927 int logical, ext4_fsblk_t ptr)
928{
929 struct ext4_extent_idx *ix;
930 int len, err;
931
932 err = ext4_ext_get_access(handle, inode, curp);
933 if (err)
934 return err;
935
936 if (unlikely(logical == le32_to_cpu(curp->p_idx->ei_block))) {
937 EXT4_ERROR_INODE(inode,
938 "logical %d == ei_block %d!",
939 logical, le32_to_cpu(curp->p_idx->ei_block));
940 return -EFSCORRUPTED;
941 }
942
943 if (unlikely(le16_to_cpu(curp->p_hdr->eh_entries)
944 >= le16_to_cpu(curp->p_hdr->eh_max))) {
945 EXT4_ERROR_INODE(inode,
946 "eh_entries %d >= eh_max %d!",
947 le16_to_cpu(curp->p_hdr->eh_entries),
948 le16_to_cpu(curp->p_hdr->eh_max));
949 return -EFSCORRUPTED;
950 }
951
952 if (logical > le32_to_cpu(curp->p_idx->ei_block)) {
953
954 ext_debug(inode, "insert new index %d after: %llu\n",
955 logical, ptr);
956 ix = curp->p_idx + 1;
957 } else {
958
959 ext_debug(inode, "insert new index %d before: %llu\n",
960 logical, ptr);
961 ix = curp->p_idx;
962 }
963
964 len = EXT_LAST_INDEX(curp->p_hdr) - ix + 1;
965 BUG_ON(len < 0);
966 if (len > 0) {
967 ext_debug(inode, "insert new index %d: "
968 "move %d indices from 0x%p to 0x%p\n",
969 logical, len, ix, ix + 1);
970 memmove(ix + 1, ix, len * sizeof(struct ext4_extent_idx));
971 }
972
973 if (unlikely(ix > EXT_MAX_INDEX(curp->p_hdr))) {
974 EXT4_ERROR_INODE(inode, "ix > EXT_MAX_INDEX!");
975 return -EFSCORRUPTED;
976 }
977
978 ix->ei_block = cpu_to_le32(logical);
979 ext4_idx_store_pblock(ix, ptr);
980 le16_add_cpu(&curp->p_hdr->eh_entries, 1);
981
982 if (unlikely(ix > EXT_LAST_INDEX(curp->p_hdr))) {
983 EXT4_ERROR_INODE(inode, "ix > EXT_LAST_INDEX!");
984 return -EFSCORRUPTED;
985 }
986
987 err = ext4_ext_dirty(handle, inode, curp);
988 ext4_std_error(inode->i_sb, err);
989
990 return err;
991}
992
993
994
995
996
997
998
999
1000
1001
1002
1003static int ext4_ext_split(handle_t *handle, struct inode *inode,
1004 unsigned int flags,
1005 struct ext4_ext_path *path,
1006 struct ext4_extent *newext, int at)
1007{
1008 struct buffer_head *bh = NULL;
1009 int depth = ext_depth(inode);
1010 struct ext4_extent_header *neh;
1011 struct ext4_extent_idx *fidx;
1012 int i = at, k, m, a;
1013 ext4_fsblk_t newblock, oldblock;
1014 __le32 border;
1015 ext4_fsblk_t *ablocks = NULL;
1016 gfp_t gfp_flags = GFP_NOFS;
1017 int err = 0;
1018 size_t ext_size = 0;
1019
1020 if (flags & EXT4_EX_NOFAIL)
1021 gfp_flags |= __GFP_NOFAIL;
1022
1023
1024
1025
1026
1027
1028 if (unlikely(path[depth].p_ext > EXT_MAX_EXTENT(path[depth].p_hdr))) {
1029 EXT4_ERROR_INODE(inode, "p_ext > EXT_MAX_EXTENT!");
1030 return -EFSCORRUPTED;
1031 }
1032 if (path[depth].p_ext != EXT_MAX_EXTENT(path[depth].p_hdr)) {
1033 border = path[depth].p_ext[1].ee_block;
1034 ext_debug(inode, "leaf will be split."
1035 " next leaf starts at %d\n",
1036 le32_to_cpu(border));
1037 } else {
1038 border = newext->ee_block;
1039 ext_debug(inode, "leaf will be added."
1040 " next leaf starts at %d\n",
1041 le32_to_cpu(border));
1042 }
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056 ablocks = kcalloc(depth, sizeof(ext4_fsblk_t), gfp_flags);
1057 if (!ablocks)
1058 return -ENOMEM;
1059
1060
1061 ext_debug(inode, "allocate %d blocks for indexes/leaf\n", depth - at);
1062 for (a = 0; a < depth - at; a++) {
1063 newblock = ext4_ext_new_meta_block(handle, inode, path,
1064 newext, &err, flags);
1065 if (newblock == 0)
1066 goto cleanup;
1067 ablocks[a] = newblock;
1068 }
1069
1070
1071 newblock = ablocks[--a];
1072 if (unlikely(newblock == 0)) {
1073 EXT4_ERROR_INODE(inode, "newblock == 0!");
1074 err = -EFSCORRUPTED;
1075 goto cleanup;
1076 }
1077 bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS);
1078 if (unlikely(!bh)) {
1079 err = -ENOMEM;
1080 goto cleanup;
1081 }
1082 lock_buffer(bh);
1083
1084 err = ext4_journal_get_create_access(handle, bh);
1085 if (err)
1086 goto cleanup;
1087
1088 neh = ext_block_hdr(bh);
1089 neh->eh_entries = 0;
1090 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
1091 neh->eh_magic = EXT4_EXT_MAGIC;
1092 neh->eh_depth = 0;
1093
1094
1095 if (unlikely(path[depth].p_hdr->eh_entries !=
1096 path[depth].p_hdr->eh_max)) {
1097 EXT4_ERROR_INODE(inode, "eh_entries %d != eh_max %d!",
1098 path[depth].p_hdr->eh_entries,
1099 path[depth].p_hdr->eh_max);
1100 err = -EFSCORRUPTED;
1101 goto cleanup;
1102 }
1103
1104 m = EXT_MAX_EXTENT(path[depth].p_hdr) - path[depth].p_ext++;
1105 ext4_ext_show_move(inode, path, newblock, depth);
1106 if (m) {
1107 struct ext4_extent *ex;
1108 ex = EXT_FIRST_EXTENT(neh);
1109 memmove(ex, path[depth].p_ext, sizeof(struct ext4_extent) * m);
1110 le16_add_cpu(&neh->eh_entries, m);
1111 }
1112
1113
1114 ext_size = sizeof(struct ext4_extent_header) +
1115 sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries);
1116 memset(bh->b_data + ext_size, 0, inode->i_sb->s_blocksize - ext_size);
1117 ext4_extent_block_csum_set(inode, neh);
1118 set_buffer_uptodate(bh);
1119 unlock_buffer(bh);
1120
1121 err = ext4_handle_dirty_metadata(handle, inode, bh);
1122 if (err)
1123 goto cleanup;
1124 brelse(bh);
1125 bh = NULL;
1126
1127
1128 if (m) {
1129 err = ext4_ext_get_access(handle, inode, path + depth);
1130 if (err)
1131 goto cleanup;
1132 le16_add_cpu(&path[depth].p_hdr->eh_entries, -m);
1133 err = ext4_ext_dirty(handle, inode, path + depth);
1134 if (err)
1135 goto cleanup;
1136
1137 }
1138
1139
1140 k = depth - at - 1;
1141 if (unlikely(k < 0)) {
1142 EXT4_ERROR_INODE(inode, "k %d < 0!", k);
1143 err = -EFSCORRUPTED;
1144 goto cleanup;
1145 }
1146 if (k)
1147 ext_debug(inode, "create %d intermediate indices\n", k);
1148
1149
1150 i = depth - 1;
1151 while (k--) {
1152 oldblock = newblock;
1153 newblock = ablocks[--a];
1154 bh = sb_getblk(inode->i_sb, newblock);
1155 if (unlikely(!bh)) {
1156 err = -ENOMEM;
1157 goto cleanup;
1158 }
1159 lock_buffer(bh);
1160
1161 err = ext4_journal_get_create_access(handle, bh);
1162 if (err)
1163 goto cleanup;
1164
1165 neh = ext_block_hdr(bh);
1166 neh->eh_entries = cpu_to_le16(1);
1167 neh->eh_magic = EXT4_EXT_MAGIC;
1168 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
1169 neh->eh_depth = cpu_to_le16(depth - i);
1170 fidx = EXT_FIRST_INDEX(neh);
1171 fidx->ei_block = border;
1172 ext4_idx_store_pblock(fidx, oldblock);
1173
1174 ext_debug(inode, "int.index at %d (block %llu): %u -> %llu\n",
1175 i, newblock, le32_to_cpu(border), oldblock);
1176
1177
1178 if (unlikely(EXT_MAX_INDEX(path[i].p_hdr) !=
1179 EXT_LAST_INDEX(path[i].p_hdr))) {
1180 EXT4_ERROR_INODE(inode,
1181 "EXT_MAX_INDEX != EXT_LAST_INDEX ee_block %d!",
1182 le32_to_cpu(path[i].p_ext->ee_block));
1183 err = -EFSCORRUPTED;
1184 goto cleanup;
1185 }
1186
1187 m = EXT_MAX_INDEX(path[i].p_hdr) - path[i].p_idx++;
1188 ext_debug(inode, "cur 0x%p, last 0x%p\n", path[i].p_idx,
1189 EXT_MAX_INDEX(path[i].p_hdr));
1190 ext4_ext_show_move(inode, path, newblock, i);
1191 if (m) {
1192 memmove(++fidx, path[i].p_idx,
1193 sizeof(struct ext4_extent_idx) * m);
1194 le16_add_cpu(&neh->eh_entries, m);
1195 }
1196
1197 ext_size = sizeof(struct ext4_extent_header) +
1198 (sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries));
1199 memset(bh->b_data + ext_size, 0,
1200 inode->i_sb->s_blocksize - ext_size);
1201 ext4_extent_block_csum_set(inode, neh);
1202 set_buffer_uptodate(bh);
1203 unlock_buffer(bh);
1204
1205 err = ext4_handle_dirty_metadata(handle, inode, bh);
1206 if (err)
1207 goto cleanup;
1208 brelse(bh);
1209 bh = NULL;
1210
1211
1212 if (m) {
1213 err = ext4_ext_get_access(handle, inode, path + i);
1214 if (err)
1215 goto cleanup;
1216 le16_add_cpu(&path[i].p_hdr->eh_entries, -m);
1217 err = ext4_ext_dirty(handle, inode, path + i);
1218 if (err)
1219 goto cleanup;
1220 }
1221
1222 i--;
1223 }
1224
1225
1226 err = ext4_ext_insert_index(handle, inode, path + at,
1227 le32_to_cpu(border), newblock);
1228
1229cleanup:
1230 if (bh) {
1231 if (buffer_locked(bh))
1232 unlock_buffer(bh);
1233 brelse(bh);
1234 }
1235
1236 if (err) {
1237
1238 for (i = 0; i < depth; i++) {
1239 if (!ablocks[i])
1240 continue;
1241 ext4_free_blocks(handle, inode, NULL, ablocks[i], 1,
1242 EXT4_FREE_BLOCKS_METADATA);
1243 }
1244 }
1245 kfree(ablocks);
1246
1247 return err;
1248}
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258static int ext4_ext_grow_indepth(handle_t *handle, struct inode *inode,
1259 unsigned int flags)
1260{
1261 struct ext4_extent_header *neh;
1262 struct buffer_head *bh;
1263 ext4_fsblk_t newblock, goal = 0;
1264 struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es;
1265 int err = 0;
1266 size_t ext_size = 0;
1267
1268
1269 if (ext_depth(inode))
1270 goal = ext4_idx_pblock(EXT_FIRST_INDEX(ext_inode_hdr(inode)));
1271 if (goal > le32_to_cpu(es->s_first_data_block)) {
1272 flags |= EXT4_MB_HINT_TRY_GOAL;
1273 goal--;
1274 } else
1275 goal = ext4_inode_to_goal_block(inode);
1276 newblock = ext4_new_meta_blocks(handle, inode, goal, flags,
1277 NULL, &err);
1278 if (newblock == 0)
1279 return err;
1280
1281 bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS);
1282 if (unlikely(!bh))
1283 return -ENOMEM;
1284 lock_buffer(bh);
1285
1286 err = ext4_journal_get_create_access(handle, bh);
1287 if (err) {
1288 unlock_buffer(bh);
1289 goto out;
1290 }
1291
1292 ext_size = sizeof(EXT4_I(inode)->i_data);
1293
1294 memmove(bh->b_data, EXT4_I(inode)->i_data, ext_size);
1295
1296 memset(bh->b_data + ext_size, 0, inode->i_sb->s_blocksize - ext_size);
1297
1298
1299 neh = ext_block_hdr(bh);
1300
1301
1302 if (ext_depth(inode))
1303 neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0));
1304 else
1305 neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0));
1306 neh->eh_magic = EXT4_EXT_MAGIC;
1307 ext4_extent_block_csum_set(inode, neh);
1308 set_buffer_uptodate(bh);
1309 unlock_buffer(bh);
1310
1311 err = ext4_handle_dirty_metadata(handle, inode, bh);
1312 if (err)
1313 goto out;
1314
1315
1316 neh = ext_inode_hdr(inode);
1317 neh->eh_entries = cpu_to_le16(1);
1318 ext4_idx_store_pblock(EXT_FIRST_INDEX(neh), newblock);
1319 if (neh->eh_depth == 0) {
1320
1321 neh->eh_max = cpu_to_le16(ext4_ext_space_root_idx(inode, 0));
1322 EXT_FIRST_INDEX(neh)->ei_block =
1323 EXT_FIRST_EXTENT(neh)->ee_block;
1324 }
1325 ext_debug(inode, "new root: num %d(%d), lblock %d, ptr %llu\n",
1326 le16_to_cpu(neh->eh_entries), le16_to_cpu(neh->eh_max),
1327 le32_to_cpu(EXT_FIRST_INDEX(neh)->ei_block),
1328 ext4_idx_pblock(EXT_FIRST_INDEX(neh)));
1329
1330 le16_add_cpu(&neh->eh_depth, 1);
1331 err = ext4_mark_inode_dirty(handle, inode);
1332out:
1333 brelse(bh);
1334
1335 return err;
1336}
1337
1338
1339
1340
1341
1342
1343static int ext4_ext_create_new_leaf(handle_t *handle, struct inode *inode,
1344 unsigned int mb_flags,
1345 unsigned int gb_flags,
1346 struct ext4_ext_path **ppath,
1347 struct ext4_extent *newext)
1348{
1349 struct ext4_ext_path *path = *ppath;
1350 struct ext4_ext_path *curp;
1351 int depth, i, err = 0;
1352
1353repeat:
1354 i = depth = ext_depth(inode);
1355
1356
1357 curp = path + depth;
1358 while (i > 0 && !EXT_HAS_FREE_INDEX(curp)) {
1359 i--;
1360 curp--;
1361 }
1362
1363
1364
1365 if (EXT_HAS_FREE_INDEX(curp)) {
1366
1367
1368 err = ext4_ext_split(handle, inode, mb_flags, path, newext, i);
1369 if (err)
1370 goto out;
1371
1372
1373 path = ext4_find_extent(inode,
1374 (ext4_lblk_t)le32_to_cpu(newext->ee_block),
1375 ppath, gb_flags);
1376 if (IS_ERR(path))
1377 err = PTR_ERR(path);
1378 } else {
1379
1380 err = ext4_ext_grow_indepth(handle, inode, mb_flags);
1381 if (err)
1382 goto out;
1383
1384
1385 path = ext4_find_extent(inode,
1386 (ext4_lblk_t)le32_to_cpu(newext->ee_block),
1387 ppath, gb_flags);
1388 if (IS_ERR(path)) {
1389 err = PTR_ERR(path);
1390 goto out;
1391 }
1392
1393
1394
1395
1396
1397 depth = ext_depth(inode);
1398 if (path[depth].p_hdr->eh_entries == path[depth].p_hdr->eh_max) {
1399
1400 goto repeat;
1401 }
1402 }
1403
1404out:
1405 return err;
1406}
1407
1408
1409
1410
1411
1412
1413
1414
1415static int ext4_ext_search_left(struct inode *inode,
1416 struct ext4_ext_path *path,
1417 ext4_lblk_t *logical, ext4_fsblk_t *phys)
1418{
1419 struct ext4_extent_idx *ix;
1420 struct ext4_extent *ex;
1421 int depth, ee_len;
1422
1423 if (unlikely(path == NULL)) {
1424 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
1425 return -EFSCORRUPTED;
1426 }
1427 depth = path->p_depth;
1428 *phys = 0;
1429
1430 if (depth == 0 && path->p_ext == NULL)
1431 return 0;
1432
1433
1434
1435
1436
1437 ex = path[depth].p_ext;
1438 ee_len = ext4_ext_get_actual_len(ex);
1439 if (*logical < le32_to_cpu(ex->ee_block)) {
1440 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
1441 EXT4_ERROR_INODE(inode,
1442 "EXT_FIRST_EXTENT != ex *logical %d ee_block %d!",
1443 *logical, le32_to_cpu(ex->ee_block));
1444 return -EFSCORRUPTED;
1445 }
1446 while (--depth >= 0) {
1447 ix = path[depth].p_idx;
1448 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
1449 EXT4_ERROR_INODE(inode,
1450 "ix (%d) != EXT_FIRST_INDEX (%d) (depth %d)!",
1451 ix != NULL ? le32_to_cpu(ix->ei_block) : 0,
1452 EXT_FIRST_INDEX(path[depth].p_hdr) != NULL ?
1453 le32_to_cpu(EXT_FIRST_INDEX(path[depth].p_hdr)->ei_block) : 0,
1454 depth);
1455 return -EFSCORRUPTED;
1456 }
1457 }
1458 return 0;
1459 }
1460
1461 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
1462 EXT4_ERROR_INODE(inode,
1463 "logical %d < ee_block %d + ee_len %d!",
1464 *logical, le32_to_cpu(ex->ee_block), ee_len);
1465 return -EFSCORRUPTED;
1466 }
1467
1468 *logical = le32_to_cpu(ex->ee_block) + ee_len - 1;
1469 *phys = ext4_ext_pblock(ex) + ee_len - 1;
1470 return 0;
1471}
1472
1473
1474
1475
1476
1477
1478
1479
1480static int ext4_ext_search_right(struct inode *inode,
1481 struct ext4_ext_path *path,
1482 ext4_lblk_t *logical, ext4_fsblk_t *phys,
1483 struct ext4_extent **ret_ex)
1484{
1485 struct buffer_head *bh = NULL;
1486 struct ext4_extent_header *eh;
1487 struct ext4_extent_idx *ix;
1488 struct ext4_extent *ex;
1489 ext4_fsblk_t block;
1490 int depth;
1491 int ee_len;
1492
1493 if (unlikely(path == NULL)) {
1494 EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical);
1495 return -EFSCORRUPTED;
1496 }
1497 depth = path->p_depth;
1498 *phys = 0;
1499
1500 if (depth == 0 && path->p_ext == NULL)
1501 return 0;
1502
1503
1504
1505
1506
1507 ex = path[depth].p_ext;
1508 ee_len = ext4_ext_get_actual_len(ex);
1509 if (*logical < le32_to_cpu(ex->ee_block)) {
1510 if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) {
1511 EXT4_ERROR_INODE(inode,
1512 "first_extent(path[%d].p_hdr) != ex",
1513 depth);
1514 return -EFSCORRUPTED;
1515 }
1516 while (--depth >= 0) {
1517 ix = path[depth].p_idx;
1518 if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) {
1519 EXT4_ERROR_INODE(inode,
1520 "ix != EXT_FIRST_INDEX *logical %d!",
1521 *logical);
1522 return -EFSCORRUPTED;
1523 }
1524 }
1525 goto found_extent;
1526 }
1527
1528 if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) {
1529 EXT4_ERROR_INODE(inode,
1530 "logical %d < ee_block %d + ee_len %d!",
1531 *logical, le32_to_cpu(ex->ee_block), ee_len);
1532 return -EFSCORRUPTED;
1533 }
1534
1535 if (ex != EXT_LAST_EXTENT(path[depth].p_hdr)) {
1536
1537 ex++;
1538 goto found_extent;
1539 }
1540
1541
1542 while (--depth >= 0) {
1543 ix = path[depth].p_idx;
1544 if (ix != EXT_LAST_INDEX(path[depth].p_hdr))
1545 goto got_index;
1546 }
1547
1548
1549 return 0;
1550
1551got_index:
1552
1553
1554
1555 ix++;
1556 block = ext4_idx_pblock(ix);
1557 while (++depth < path->p_depth) {
1558
1559 bh = read_extent_tree_block(inode, block,
1560 path->p_depth - depth, 0);
1561 if (IS_ERR(bh))
1562 return PTR_ERR(bh);
1563 eh = ext_block_hdr(bh);
1564 ix = EXT_FIRST_INDEX(eh);
1565 block = ext4_idx_pblock(ix);
1566 put_bh(bh);
1567 }
1568
1569 bh = read_extent_tree_block(inode, block, path->p_depth - depth, 0);
1570 if (IS_ERR(bh))
1571 return PTR_ERR(bh);
1572 eh = ext_block_hdr(bh);
1573 ex = EXT_FIRST_EXTENT(eh);
1574found_extent:
1575 *logical = le32_to_cpu(ex->ee_block);
1576 *phys = ext4_ext_pblock(ex);
1577 *ret_ex = ex;
1578 if (bh)
1579 put_bh(bh);
1580 return 0;
1581}
1582
1583
1584
1585
1586
1587
1588
1589
1590ext4_lblk_t
1591ext4_ext_next_allocated_block(struct ext4_ext_path *path)
1592{
1593 int depth;
1594
1595 BUG_ON(path == NULL);
1596 depth = path->p_depth;
1597
1598 if (depth == 0 && path->p_ext == NULL)
1599 return EXT_MAX_BLOCKS;
1600
1601 while (depth >= 0) {
1602 struct ext4_ext_path *p = &path[depth];
1603
1604 if (depth == path->p_depth) {
1605
1606 if (p->p_ext && p->p_ext != EXT_LAST_EXTENT(p->p_hdr))
1607 return le32_to_cpu(p->p_ext[1].ee_block);
1608 } else {
1609
1610 if (p->p_idx != EXT_LAST_INDEX(p->p_hdr))
1611 return le32_to_cpu(p->p_idx[1].ei_block);
1612 }
1613 depth--;
1614 }
1615
1616 return EXT_MAX_BLOCKS;
1617}
1618
1619
1620
1621
1622
1623static ext4_lblk_t ext4_ext_next_leaf_block(struct ext4_ext_path *path)
1624{
1625 int depth;
1626
1627 BUG_ON(path == NULL);
1628 depth = path->p_depth;
1629
1630
1631 if (depth == 0)
1632 return EXT_MAX_BLOCKS;
1633
1634
1635 depth--;
1636
1637 while (depth >= 0) {
1638 if (path[depth].p_idx !=
1639 EXT_LAST_INDEX(path[depth].p_hdr))
1640 return (ext4_lblk_t)
1641 le32_to_cpu(path[depth].p_idx[1].ei_block);
1642 depth--;
1643 }
1644
1645 return EXT_MAX_BLOCKS;
1646}
1647
1648
1649
1650
1651
1652
1653
1654static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode,
1655 struct ext4_ext_path *path)
1656{
1657 struct ext4_extent_header *eh;
1658 int depth = ext_depth(inode);
1659 struct ext4_extent *ex;
1660 __le32 border;
1661 int k, err = 0;
1662
1663 eh = path[depth].p_hdr;
1664 ex = path[depth].p_ext;
1665
1666 if (unlikely(ex == NULL || eh == NULL)) {
1667 EXT4_ERROR_INODE(inode,
1668 "ex %p == NULL or eh %p == NULL", ex, eh);
1669 return -EFSCORRUPTED;
1670 }
1671
1672 if (depth == 0) {
1673
1674 return 0;
1675 }
1676
1677 if (ex != EXT_FIRST_EXTENT(eh)) {
1678
1679 return 0;
1680 }
1681
1682
1683
1684
1685 k = depth - 1;
1686 border = path[depth].p_ext->ee_block;
1687 err = ext4_ext_get_access(handle, inode, path + k);
1688 if (err)
1689 return err;
1690 path[k].p_idx->ei_block = border;
1691 err = ext4_ext_dirty(handle, inode, path + k);
1692 if (err)
1693 return err;
1694
1695 while (k--) {
1696
1697 if (path[k+1].p_idx != EXT_FIRST_INDEX(path[k+1].p_hdr))
1698 break;
1699 err = ext4_ext_get_access(handle, inode, path + k);
1700 if (err)
1701 break;
1702 path[k].p_idx->ei_block = border;
1703 err = ext4_ext_dirty(handle, inode, path + k);
1704 if (err)
1705 break;
1706 }
1707
1708 return err;
1709}
1710
1711static int ext4_can_extents_be_merged(struct inode *inode,
1712 struct ext4_extent *ex1,
1713 struct ext4_extent *ex2)
1714{
1715 unsigned short ext1_ee_len, ext2_ee_len;
1716
1717 if (ext4_ext_is_unwritten(ex1) != ext4_ext_is_unwritten(ex2))
1718 return 0;
1719
1720 ext1_ee_len = ext4_ext_get_actual_len(ex1);
1721 ext2_ee_len = ext4_ext_get_actual_len(ex2);
1722
1723 if (le32_to_cpu(ex1->ee_block) + ext1_ee_len !=
1724 le32_to_cpu(ex2->ee_block))
1725 return 0;
1726
1727 if (ext1_ee_len + ext2_ee_len > EXT_INIT_MAX_LEN)
1728 return 0;
1729
1730 if (ext4_ext_is_unwritten(ex1) &&
1731 ext1_ee_len + ext2_ee_len > EXT_UNWRITTEN_MAX_LEN)
1732 return 0;
1733#ifdef AGGRESSIVE_TEST
1734 if (ext1_ee_len >= 4)
1735 return 0;
1736#endif
1737
1738 if (ext4_ext_pblock(ex1) + ext1_ee_len == ext4_ext_pblock(ex2))
1739 return 1;
1740 return 0;
1741}
1742
1743
1744
1745
1746
1747
1748
1749
1750static int ext4_ext_try_to_merge_right(struct inode *inode,
1751 struct ext4_ext_path *path,
1752 struct ext4_extent *ex)
1753{
1754 struct ext4_extent_header *eh;
1755 unsigned int depth, len;
1756 int merge_done = 0, unwritten;
1757
1758 depth = ext_depth(inode);
1759 BUG_ON(path[depth].p_hdr == NULL);
1760 eh = path[depth].p_hdr;
1761
1762 while (ex < EXT_LAST_EXTENT(eh)) {
1763 if (!ext4_can_extents_be_merged(inode, ex, ex + 1))
1764 break;
1765
1766 unwritten = ext4_ext_is_unwritten(ex);
1767 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
1768 + ext4_ext_get_actual_len(ex + 1));
1769 if (unwritten)
1770 ext4_ext_mark_unwritten(ex);
1771
1772 if (ex + 1 < EXT_LAST_EXTENT(eh)) {
1773 len = (EXT_LAST_EXTENT(eh) - ex - 1)
1774 * sizeof(struct ext4_extent);
1775 memmove(ex + 1, ex + 2, len);
1776 }
1777 le16_add_cpu(&eh->eh_entries, -1);
1778 merge_done = 1;
1779 WARN_ON(eh->eh_entries == 0);
1780 if (!eh->eh_entries)
1781 EXT4_ERROR_INODE(inode, "eh->eh_entries = 0!");
1782 }
1783
1784 return merge_done;
1785}
1786
1787
1788
1789
1790
1791static void ext4_ext_try_to_merge_up(handle_t *handle,
1792 struct inode *inode,
1793 struct ext4_ext_path *path)
1794{
1795 size_t s;
1796 unsigned max_root = ext4_ext_space_root(inode, 0);
1797 ext4_fsblk_t blk;
1798
1799 if ((path[0].p_depth != 1) ||
1800 (le16_to_cpu(path[0].p_hdr->eh_entries) != 1) ||
1801 (le16_to_cpu(path[1].p_hdr->eh_entries) > max_root))
1802 return;
1803
1804
1805
1806
1807
1808
1809 if (ext4_journal_extend(handle, 2,
1810 ext4_free_metadata_revoke_credits(inode->i_sb, 1)))
1811 return;
1812
1813
1814
1815
1816 blk = ext4_idx_pblock(path[0].p_idx);
1817 s = le16_to_cpu(path[1].p_hdr->eh_entries) *
1818 sizeof(struct ext4_extent_idx);
1819 s += sizeof(struct ext4_extent_header);
1820
1821 path[1].p_maxdepth = path[0].p_maxdepth;
1822 memcpy(path[0].p_hdr, path[1].p_hdr, s);
1823 path[0].p_depth = 0;
1824 path[0].p_ext = EXT_FIRST_EXTENT(path[0].p_hdr) +
1825 (path[1].p_ext - EXT_FIRST_EXTENT(path[1].p_hdr));
1826 path[0].p_hdr->eh_max = cpu_to_le16(max_root);
1827
1828 brelse(path[1].p_bh);
1829 ext4_free_blocks(handle, inode, NULL, blk, 1,
1830 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
1831}
1832
1833
1834
1835
1836
1837static void ext4_ext_try_to_merge(handle_t *handle,
1838 struct inode *inode,
1839 struct ext4_ext_path *path,
1840 struct ext4_extent *ex)
1841{
1842 struct ext4_extent_header *eh;
1843 unsigned int depth;
1844 int merge_done = 0;
1845
1846 depth = ext_depth(inode);
1847 BUG_ON(path[depth].p_hdr == NULL);
1848 eh = path[depth].p_hdr;
1849
1850 if (ex > EXT_FIRST_EXTENT(eh))
1851 merge_done = ext4_ext_try_to_merge_right(inode, path, ex - 1);
1852
1853 if (!merge_done)
1854 (void) ext4_ext_try_to_merge_right(inode, path, ex);
1855
1856 ext4_ext_try_to_merge_up(handle, inode, path);
1857}
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867static unsigned int ext4_ext_check_overlap(struct ext4_sb_info *sbi,
1868 struct inode *inode,
1869 struct ext4_extent *newext,
1870 struct ext4_ext_path *path)
1871{
1872 ext4_lblk_t b1, b2;
1873 unsigned int depth, len1;
1874 unsigned int ret = 0;
1875
1876 b1 = le32_to_cpu(newext->ee_block);
1877 len1 = ext4_ext_get_actual_len(newext);
1878 depth = ext_depth(inode);
1879 if (!path[depth].p_ext)
1880 goto out;
1881 b2 = EXT4_LBLK_CMASK(sbi, le32_to_cpu(path[depth].p_ext->ee_block));
1882
1883
1884
1885
1886
1887 if (b2 < b1) {
1888 b2 = ext4_ext_next_allocated_block(path);
1889 if (b2 == EXT_MAX_BLOCKS)
1890 goto out;
1891 b2 = EXT4_LBLK_CMASK(sbi, b2);
1892 }
1893
1894
1895 if (b1 + len1 < b1) {
1896 len1 = EXT_MAX_BLOCKS - b1;
1897 newext->ee_len = cpu_to_le16(len1);
1898 ret = 1;
1899 }
1900
1901
1902 if (b1 + len1 > b2) {
1903 newext->ee_len = cpu_to_le16(b2 - b1);
1904 ret = 1;
1905 }
1906out:
1907 return ret;
1908}
1909
1910
1911
1912
1913
1914
1915
1916int ext4_ext_insert_extent(handle_t *handle, struct inode *inode,
1917 struct ext4_ext_path **ppath,
1918 struct ext4_extent *newext, int gb_flags)
1919{
1920 struct ext4_ext_path *path = *ppath;
1921 struct ext4_extent_header *eh;
1922 struct ext4_extent *ex, *fex;
1923 struct ext4_extent *nearex;
1924 struct ext4_ext_path *npath = NULL;
1925 int depth, len, err;
1926 ext4_lblk_t next;
1927 int mb_flags = 0, unwritten;
1928
1929 if (gb_flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
1930 mb_flags |= EXT4_MB_DELALLOC_RESERVED;
1931 if (unlikely(ext4_ext_get_actual_len(newext) == 0)) {
1932 EXT4_ERROR_INODE(inode, "ext4_ext_get_actual_len(newext) == 0");
1933 return -EFSCORRUPTED;
1934 }
1935 depth = ext_depth(inode);
1936 ex = path[depth].p_ext;
1937 eh = path[depth].p_hdr;
1938 if (unlikely(path[depth].p_hdr == NULL)) {
1939 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
1940 return -EFSCORRUPTED;
1941 }
1942
1943
1944 if (ex && !(gb_flags & EXT4_GET_BLOCKS_PRE_IO)) {
1945
1946
1947
1948
1949
1950
1951
1952
1953 if (ex < EXT_LAST_EXTENT(eh) &&
1954 (le32_to_cpu(ex->ee_block) +
1955 ext4_ext_get_actual_len(ex) <
1956 le32_to_cpu(newext->ee_block))) {
1957 ex += 1;
1958 goto prepend;
1959 } else if ((ex > EXT_FIRST_EXTENT(eh)) &&
1960 (le32_to_cpu(newext->ee_block) +
1961 ext4_ext_get_actual_len(newext) <
1962 le32_to_cpu(ex->ee_block)))
1963 ex -= 1;
1964
1965
1966 if (ext4_can_extents_be_merged(inode, ex, newext)) {
1967 ext_debug(inode, "append [%d]%d block to %u:[%d]%d"
1968 "(from %llu)\n",
1969 ext4_ext_is_unwritten(newext),
1970 ext4_ext_get_actual_len(newext),
1971 le32_to_cpu(ex->ee_block),
1972 ext4_ext_is_unwritten(ex),
1973 ext4_ext_get_actual_len(ex),
1974 ext4_ext_pblock(ex));
1975 err = ext4_ext_get_access(handle, inode,
1976 path + depth);
1977 if (err)
1978 return err;
1979 unwritten = ext4_ext_is_unwritten(ex);
1980 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
1981 + ext4_ext_get_actual_len(newext));
1982 if (unwritten)
1983 ext4_ext_mark_unwritten(ex);
1984 eh = path[depth].p_hdr;
1985 nearex = ex;
1986 goto merge;
1987 }
1988
1989prepend:
1990
1991 if (ext4_can_extents_be_merged(inode, newext, ex)) {
1992 ext_debug(inode, "prepend %u[%d]%d block to %u:[%d]%d"
1993 "(from %llu)\n",
1994 le32_to_cpu(newext->ee_block),
1995 ext4_ext_is_unwritten(newext),
1996 ext4_ext_get_actual_len(newext),
1997 le32_to_cpu(ex->ee_block),
1998 ext4_ext_is_unwritten(ex),
1999 ext4_ext_get_actual_len(ex),
2000 ext4_ext_pblock(ex));
2001 err = ext4_ext_get_access(handle, inode,
2002 path + depth);
2003 if (err)
2004 return err;
2005
2006 unwritten = ext4_ext_is_unwritten(ex);
2007 ex->ee_block = newext->ee_block;
2008 ext4_ext_store_pblock(ex, ext4_ext_pblock(newext));
2009 ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex)
2010 + ext4_ext_get_actual_len(newext));
2011 if (unwritten)
2012 ext4_ext_mark_unwritten(ex);
2013 eh = path[depth].p_hdr;
2014 nearex = ex;
2015 goto merge;
2016 }
2017 }
2018
2019 depth = ext_depth(inode);
2020 eh = path[depth].p_hdr;
2021 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max))
2022 goto has_space;
2023
2024
2025 fex = EXT_LAST_EXTENT(eh);
2026 next = EXT_MAX_BLOCKS;
2027 if (le32_to_cpu(newext->ee_block) > le32_to_cpu(fex->ee_block))
2028 next = ext4_ext_next_leaf_block(path);
2029 if (next != EXT_MAX_BLOCKS) {
2030 ext_debug(inode, "next leaf block - %u\n", next);
2031 BUG_ON(npath != NULL);
2032 npath = ext4_find_extent(inode, next, NULL, gb_flags);
2033 if (IS_ERR(npath))
2034 return PTR_ERR(npath);
2035 BUG_ON(npath->p_depth != path->p_depth);
2036 eh = npath[depth].p_hdr;
2037 if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) {
2038 ext_debug(inode, "next leaf isn't full(%d)\n",
2039 le16_to_cpu(eh->eh_entries));
2040 path = npath;
2041 goto has_space;
2042 }
2043 ext_debug(inode, "next leaf has no free space(%d,%d)\n",
2044 le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max));
2045 }
2046
2047
2048
2049
2050
2051 if (gb_flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
2052 mb_flags |= EXT4_MB_USE_RESERVED;
2053 err = ext4_ext_create_new_leaf(handle, inode, mb_flags, gb_flags,
2054 ppath, newext);
2055 if (err)
2056 goto cleanup;
2057 depth = ext_depth(inode);
2058 eh = path[depth].p_hdr;
2059
2060has_space:
2061 nearex = path[depth].p_ext;
2062
2063 err = ext4_ext_get_access(handle, inode, path + depth);
2064 if (err)
2065 goto cleanup;
2066
2067 if (!nearex) {
2068
2069 ext_debug(inode, "first extent in the leaf: %u:%llu:[%d]%d\n",
2070 le32_to_cpu(newext->ee_block),
2071 ext4_ext_pblock(newext),
2072 ext4_ext_is_unwritten(newext),
2073 ext4_ext_get_actual_len(newext));
2074 nearex = EXT_FIRST_EXTENT(eh);
2075 } else {
2076 if (le32_to_cpu(newext->ee_block)
2077 > le32_to_cpu(nearex->ee_block)) {
2078
2079 ext_debug(inode, "insert %u:%llu:[%d]%d before: "
2080 "nearest %p\n",
2081 le32_to_cpu(newext->ee_block),
2082 ext4_ext_pblock(newext),
2083 ext4_ext_is_unwritten(newext),
2084 ext4_ext_get_actual_len(newext),
2085 nearex);
2086 nearex++;
2087 } else {
2088
2089 BUG_ON(newext->ee_block == nearex->ee_block);
2090 ext_debug(inode, "insert %u:%llu:[%d]%d after: "
2091 "nearest %p\n",
2092 le32_to_cpu(newext->ee_block),
2093 ext4_ext_pblock(newext),
2094 ext4_ext_is_unwritten(newext),
2095 ext4_ext_get_actual_len(newext),
2096 nearex);
2097 }
2098 len = EXT_LAST_EXTENT(eh) - nearex + 1;
2099 if (len > 0) {
2100 ext_debug(inode, "insert %u:%llu:[%d]%d: "
2101 "move %d extents from 0x%p to 0x%p\n",
2102 le32_to_cpu(newext->ee_block),
2103 ext4_ext_pblock(newext),
2104 ext4_ext_is_unwritten(newext),
2105 ext4_ext_get_actual_len(newext),
2106 len, nearex, nearex + 1);
2107 memmove(nearex + 1, nearex,
2108 len * sizeof(struct ext4_extent));
2109 }
2110 }
2111
2112 le16_add_cpu(&eh->eh_entries, 1);
2113 path[depth].p_ext = nearex;
2114 nearex->ee_block = newext->ee_block;
2115 ext4_ext_store_pblock(nearex, ext4_ext_pblock(newext));
2116 nearex->ee_len = newext->ee_len;
2117
2118merge:
2119
2120 if (!(gb_flags & EXT4_GET_BLOCKS_PRE_IO))
2121 ext4_ext_try_to_merge(handle, inode, path, nearex);
2122
2123
2124
2125 err = ext4_ext_correct_indexes(handle, inode, path);
2126 if (err)
2127 goto cleanup;
2128
2129 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
2130
2131cleanup:
2132 ext4_ext_drop_refs(npath);
2133 kfree(npath);
2134 return err;
2135}
2136
2137static int ext4_fill_es_cache_info(struct inode *inode,
2138 ext4_lblk_t block, ext4_lblk_t num,
2139 struct fiemap_extent_info *fieinfo)
2140{
2141 ext4_lblk_t next, end = block + num - 1;
2142 struct extent_status es;
2143 unsigned char blksize_bits = inode->i_sb->s_blocksize_bits;
2144 unsigned int flags;
2145 int err;
2146
2147 while (block <= end) {
2148 next = 0;
2149 flags = 0;
2150 if (!ext4_es_lookup_extent(inode, block, &next, &es))
2151 break;
2152 if (ext4_es_is_unwritten(&es))
2153 flags |= FIEMAP_EXTENT_UNWRITTEN;
2154 if (ext4_es_is_delayed(&es))
2155 flags |= (FIEMAP_EXTENT_DELALLOC |
2156 FIEMAP_EXTENT_UNKNOWN);
2157 if (ext4_es_is_hole(&es))
2158 flags |= EXT4_FIEMAP_EXTENT_HOLE;
2159 if (next == 0)
2160 flags |= FIEMAP_EXTENT_LAST;
2161 if (flags & (FIEMAP_EXTENT_DELALLOC|
2162 EXT4_FIEMAP_EXTENT_HOLE))
2163 es.es_pblk = 0;
2164 else
2165 es.es_pblk = ext4_es_pblock(&es);
2166 err = fiemap_fill_next_extent(fieinfo,
2167 (__u64)es.es_lblk << blksize_bits,
2168 (__u64)es.es_pblk << blksize_bits,
2169 (__u64)es.es_len << blksize_bits,
2170 flags);
2171 if (next == 0)
2172 break;
2173 block = next;
2174 if (err < 0)
2175 return err;
2176 if (err == 1)
2177 return 0;
2178 }
2179 return 0;
2180}
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196static ext4_lblk_t ext4_ext_determine_hole(struct inode *inode,
2197 struct ext4_ext_path *path,
2198 ext4_lblk_t *lblk)
2199{
2200 int depth = ext_depth(inode);
2201 struct ext4_extent *ex;
2202 ext4_lblk_t len;
2203
2204 ex = path[depth].p_ext;
2205 if (ex == NULL) {
2206
2207 *lblk = 0;
2208 len = EXT_MAX_BLOCKS;
2209 } else if (*lblk < le32_to_cpu(ex->ee_block)) {
2210 len = le32_to_cpu(ex->ee_block) - *lblk;
2211 } else if (*lblk >= le32_to_cpu(ex->ee_block)
2212 + ext4_ext_get_actual_len(ex)) {
2213 ext4_lblk_t next;
2214
2215 *lblk = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex);
2216 next = ext4_ext_next_allocated_block(path);
2217 BUG_ON(next == *lblk);
2218 len = next - *lblk;
2219 } else {
2220 BUG();
2221 }
2222 return len;
2223}
2224
2225
2226
2227
2228
2229
2230static void
2231ext4_ext_put_gap_in_cache(struct inode *inode, ext4_lblk_t hole_start,
2232 ext4_lblk_t hole_len)
2233{
2234 struct extent_status es;
2235
2236 ext4_es_find_extent_range(inode, &ext4_es_is_delayed, hole_start,
2237 hole_start + hole_len - 1, &es);
2238 if (es.es_len) {
2239
2240 if (es.es_lblk <= hole_start)
2241 return;
2242 hole_len = min(es.es_lblk - hole_start, hole_len);
2243 }
2244 ext_debug(inode, " -> %u:%u\n", hole_start, hole_len);
2245 ext4_es_insert_extent(inode, hole_start, hole_len, ~0,
2246 EXTENT_STATUS_HOLE);
2247}
2248
2249
2250
2251
2252
2253static int ext4_ext_rm_idx(handle_t *handle, struct inode *inode,
2254 struct ext4_ext_path *path, int depth)
2255{
2256 int err;
2257 ext4_fsblk_t leaf;
2258
2259
2260 depth--;
2261 path = path + depth;
2262 leaf = ext4_idx_pblock(path->p_idx);
2263 if (unlikely(path->p_hdr->eh_entries == 0)) {
2264 EXT4_ERROR_INODE(inode, "path->p_hdr->eh_entries == 0");
2265 return -EFSCORRUPTED;
2266 }
2267 err = ext4_ext_get_access(handle, inode, path);
2268 if (err)
2269 return err;
2270
2271 if (path->p_idx != EXT_LAST_INDEX(path->p_hdr)) {
2272 int len = EXT_LAST_INDEX(path->p_hdr) - path->p_idx;
2273 len *= sizeof(struct ext4_extent_idx);
2274 memmove(path->p_idx, path->p_idx + 1, len);
2275 }
2276
2277 le16_add_cpu(&path->p_hdr->eh_entries, -1);
2278 err = ext4_ext_dirty(handle, inode, path);
2279 if (err)
2280 return err;
2281 ext_debug(inode, "index is empty, remove it, free block %llu\n", leaf);
2282 trace_ext4_ext_rm_idx(inode, leaf);
2283
2284 ext4_free_blocks(handle, inode, NULL, leaf, 1,
2285 EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET);
2286
2287 while (--depth >= 0) {
2288 if (path->p_idx != EXT_FIRST_INDEX(path->p_hdr))
2289 break;
2290 path--;
2291 err = ext4_ext_get_access(handle, inode, path);
2292 if (err)
2293 break;
2294 path->p_idx->ei_block = (path+1)->p_idx->ei_block;
2295 err = ext4_ext_dirty(handle, inode, path);
2296 if (err)
2297 break;
2298 }
2299 return err;
2300}
2301
2302
2303
2304
2305
2306
2307
2308
2309int ext4_ext_calc_credits_for_single_extent(struct inode *inode, int nrblocks,
2310 struct ext4_ext_path *path)
2311{
2312 if (path) {
2313 int depth = ext_depth(inode);
2314 int ret = 0;
2315
2316
2317 if (le16_to_cpu(path[depth].p_hdr->eh_entries)
2318 < le16_to_cpu(path[depth].p_hdr->eh_max)) {
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329 ret = 2 + EXT4_META_TRANS_BLOCKS(inode->i_sb);
2330 return ret;
2331 }
2332 }
2333
2334 return ext4_chunk_trans_blocks(inode, nrblocks);
2335}
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346int ext4_ext_index_trans_blocks(struct inode *inode, int extents)
2347{
2348 int index;
2349 int depth;
2350
2351
2352 if (ext4_has_inline_data(inode))
2353 return 1;
2354
2355 depth = ext_depth(inode);
2356
2357 if (extents <= 1)
2358 index = depth * 2;
2359 else
2360 index = depth * 3;
2361
2362 return index;
2363}
2364
2365static inline int get_default_free_blocks_flags(struct inode *inode)
2366{
2367 if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode) ||
2368 ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE))
2369 return EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET;
2370 else if (ext4_should_journal_data(inode))
2371 return EXT4_FREE_BLOCKS_FORGET;
2372 return 0;
2373}
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390static void ext4_rereserve_cluster(struct inode *inode, ext4_lblk_t lblk)
2391{
2392 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2393 struct ext4_inode_info *ei = EXT4_I(inode);
2394
2395 dquot_reclaim_block(inode, EXT4_C2B(sbi, 1));
2396
2397 spin_lock(&ei->i_block_reservation_lock);
2398 ei->i_reserved_data_blocks++;
2399 percpu_counter_add(&sbi->s_dirtyclusters_counter, 1);
2400 spin_unlock(&ei->i_block_reservation_lock);
2401
2402 percpu_counter_add(&sbi->s_freeclusters_counter, 1);
2403 ext4_remove_pending(inode, lblk);
2404}
2405
2406static int ext4_remove_blocks(handle_t *handle, struct inode *inode,
2407 struct ext4_extent *ex,
2408 struct partial_cluster *partial,
2409 ext4_lblk_t from, ext4_lblk_t to)
2410{
2411 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2412 unsigned short ee_len = ext4_ext_get_actual_len(ex);
2413 ext4_fsblk_t last_pblk, pblk;
2414 ext4_lblk_t num;
2415 int flags;
2416
2417
2418 if (from < le32_to_cpu(ex->ee_block) ||
2419 to != le32_to_cpu(ex->ee_block) + ee_len - 1) {
2420 ext4_error(sbi->s_sb,
2421 "strange request: removal(2) %u-%u from %u:%u",
2422 from, to, le32_to_cpu(ex->ee_block), ee_len);
2423 return 0;
2424 }
2425
2426#ifdef EXTENTS_STATS
2427 spin_lock(&sbi->s_ext_stats_lock);
2428 sbi->s_ext_blocks += ee_len;
2429 sbi->s_ext_extents++;
2430 if (ee_len < sbi->s_ext_min)
2431 sbi->s_ext_min = ee_len;
2432 if (ee_len > sbi->s_ext_max)
2433 sbi->s_ext_max = ee_len;
2434 if (ext_depth(inode) > sbi->s_depth_max)
2435 sbi->s_depth_max = ext_depth(inode);
2436 spin_unlock(&sbi->s_ext_stats_lock);
2437#endif
2438
2439 trace_ext4_remove_blocks(inode, ex, from, to, partial);
2440
2441
2442
2443
2444
2445 last_pblk = ext4_ext_pblock(ex) + ee_len - 1;
2446
2447 if (partial->state != initial &&
2448 partial->pclu != EXT4_B2C(sbi, last_pblk)) {
2449 if (partial->state == tofree) {
2450 flags = get_default_free_blocks_flags(inode);
2451 if (ext4_is_pending(inode, partial->lblk))
2452 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2453 ext4_free_blocks(handle, inode, NULL,
2454 EXT4_C2B(sbi, partial->pclu),
2455 sbi->s_cluster_ratio, flags);
2456 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2457 ext4_rereserve_cluster(inode, partial->lblk);
2458 }
2459 partial->state = initial;
2460 }
2461
2462 num = le32_to_cpu(ex->ee_block) + ee_len - from;
2463 pblk = ext4_ext_pblock(ex) + ee_len - num;
2464
2465
2466
2467
2468
2469
2470
2471 flags = get_default_free_blocks_flags(inode);
2472
2473
2474 if ((EXT4_LBLK_COFF(sbi, to) != sbi->s_cluster_ratio - 1) &&
2475 (EXT4_LBLK_CMASK(sbi, to) >= from) &&
2476 (partial->state != nofree)) {
2477 if (ext4_is_pending(inode, to))
2478 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2479 ext4_free_blocks(handle, inode, NULL,
2480 EXT4_PBLK_CMASK(sbi, last_pblk),
2481 sbi->s_cluster_ratio, flags);
2482 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2483 ext4_rereserve_cluster(inode, to);
2484 partial->state = initial;
2485 flags = get_default_free_blocks_flags(inode);
2486 }
2487
2488 flags |= EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER;
2489
2490
2491
2492
2493
2494
2495
2496 flags |= EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER;
2497 ext4_free_blocks(handle, inode, NULL, pblk, num, flags);
2498
2499
2500 if (partial->state != initial && partial->pclu != EXT4_B2C(sbi, pblk))
2501 partial->state = initial;
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513 if (EXT4_LBLK_COFF(sbi, from) && num == ee_len) {
2514 if (partial->state == initial) {
2515 partial->pclu = EXT4_B2C(sbi, pblk);
2516 partial->lblk = from;
2517 partial->state = tofree;
2518 }
2519 } else {
2520 partial->state = initial;
2521 }
2522
2523 return 0;
2524}
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541static int
2542ext4_ext_rm_leaf(handle_t *handle, struct inode *inode,
2543 struct ext4_ext_path *path,
2544 struct partial_cluster *partial,
2545 ext4_lblk_t start, ext4_lblk_t end)
2546{
2547 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2548 int err = 0, correct_index = 0;
2549 int depth = ext_depth(inode), credits, revoke_credits;
2550 struct ext4_extent_header *eh;
2551 ext4_lblk_t a, b;
2552 unsigned num;
2553 ext4_lblk_t ex_ee_block;
2554 unsigned short ex_ee_len;
2555 unsigned unwritten = 0;
2556 struct ext4_extent *ex;
2557 ext4_fsblk_t pblk;
2558
2559
2560 ext_debug(inode, "truncate since %u in leaf to %u\n", start, end);
2561 if (!path[depth].p_hdr)
2562 path[depth].p_hdr = ext_block_hdr(path[depth].p_bh);
2563 eh = path[depth].p_hdr;
2564 if (unlikely(path[depth].p_hdr == NULL)) {
2565 EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth);
2566 return -EFSCORRUPTED;
2567 }
2568
2569 ex = path[depth].p_ext;
2570 if (!ex)
2571 ex = EXT_LAST_EXTENT(eh);
2572
2573 ex_ee_block = le32_to_cpu(ex->ee_block);
2574 ex_ee_len = ext4_ext_get_actual_len(ex);
2575
2576 trace_ext4_ext_rm_leaf(inode, start, ex, partial);
2577
2578 while (ex >= EXT_FIRST_EXTENT(eh) &&
2579 ex_ee_block + ex_ee_len > start) {
2580
2581 if (ext4_ext_is_unwritten(ex))
2582 unwritten = 1;
2583 else
2584 unwritten = 0;
2585
2586 ext_debug(inode, "remove ext %u:[%d]%d\n", ex_ee_block,
2587 unwritten, ex_ee_len);
2588 path[depth].p_ext = ex;
2589
2590 a = ex_ee_block > start ? ex_ee_block : start;
2591 b = ex_ee_block+ex_ee_len - 1 < end ?
2592 ex_ee_block+ex_ee_len - 1 : end;
2593
2594 ext_debug(inode, " border %u:%u\n", a, b);
2595
2596
2597 if (end < ex_ee_block) {
2598
2599
2600
2601
2602
2603
2604
2605 if (sbi->s_cluster_ratio > 1) {
2606 pblk = ext4_ext_pblock(ex);
2607 partial->pclu = EXT4_B2C(sbi, pblk);
2608 partial->state = nofree;
2609 }
2610 ex--;
2611 ex_ee_block = le32_to_cpu(ex->ee_block);
2612 ex_ee_len = ext4_ext_get_actual_len(ex);
2613 continue;
2614 } else if (b != ex_ee_block + ex_ee_len - 1) {
2615 EXT4_ERROR_INODE(inode,
2616 "can not handle truncate %u:%u "
2617 "on extent %u:%u",
2618 start, end, ex_ee_block,
2619 ex_ee_block + ex_ee_len - 1);
2620 err = -EFSCORRUPTED;
2621 goto out;
2622 } else if (a != ex_ee_block) {
2623
2624 num = a - ex_ee_block;
2625 } else {
2626
2627 num = 0;
2628 }
2629
2630
2631
2632
2633
2634
2635 credits = 7 + 2*(ex_ee_len/EXT4_BLOCKS_PER_GROUP(inode->i_sb));
2636 if (ex == EXT_FIRST_EXTENT(eh)) {
2637 correct_index = 1;
2638 credits += (ext_depth(inode)) + 1;
2639 }
2640 credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb);
2641
2642
2643
2644
2645
2646 revoke_credits =
2647 ext4_free_metadata_revoke_credits(inode->i_sb,
2648 ext_depth(inode)) +
2649 ext4_free_data_revoke_credits(inode, b - a + 1);
2650
2651 err = ext4_datasem_ensure_credits(handle, inode, credits,
2652 credits, revoke_credits);
2653 if (err) {
2654 if (err > 0)
2655 err = -EAGAIN;
2656 goto out;
2657 }
2658
2659 err = ext4_ext_get_access(handle, inode, path + depth);
2660 if (err)
2661 goto out;
2662
2663 err = ext4_remove_blocks(handle, inode, ex, partial, a, b);
2664 if (err)
2665 goto out;
2666
2667 if (num == 0)
2668
2669 ext4_ext_store_pblock(ex, 0);
2670
2671 ex->ee_len = cpu_to_le16(num);
2672
2673
2674
2675
2676 if (unwritten && num)
2677 ext4_ext_mark_unwritten(ex);
2678
2679
2680
2681
2682 if (num == 0) {
2683 if (end != EXT_MAX_BLOCKS - 1) {
2684
2685
2686
2687
2688
2689 memmove(ex, ex+1, (EXT_LAST_EXTENT(eh) - ex) *
2690 sizeof(struct ext4_extent));
2691
2692
2693 memset(EXT_LAST_EXTENT(eh), 0,
2694 sizeof(struct ext4_extent));
2695 }
2696 le16_add_cpu(&eh->eh_entries, -1);
2697 }
2698
2699 err = ext4_ext_dirty(handle, inode, path + depth);
2700 if (err)
2701 goto out;
2702
2703 ext_debug(inode, "new extent: %u:%u:%llu\n", ex_ee_block, num,
2704 ext4_ext_pblock(ex));
2705 ex--;
2706 ex_ee_block = le32_to_cpu(ex->ee_block);
2707 ex_ee_len = ext4_ext_get_actual_len(ex);
2708 }
2709
2710 if (correct_index && eh->eh_entries)
2711 err = ext4_ext_correct_indexes(handle, inode, path);
2712
2713
2714
2715
2716
2717
2718
2719
2720 if (partial->state == tofree && ex >= EXT_FIRST_EXTENT(eh)) {
2721 pblk = ext4_ext_pblock(ex) + ex_ee_len - 1;
2722 if (partial->pclu != EXT4_B2C(sbi, pblk)) {
2723 int flags = get_default_free_blocks_flags(inode);
2724
2725 if (ext4_is_pending(inode, partial->lblk))
2726 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2727 ext4_free_blocks(handle, inode, NULL,
2728 EXT4_C2B(sbi, partial->pclu),
2729 sbi->s_cluster_ratio, flags);
2730 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
2731 ext4_rereserve_cluster(inode, partial->lblk);
2732 }
2733 partial->state = initial;
2734 }
2735
2736
2737
2738 if (err == 0 && eh->eh_entries == 0 && path[depth].p_bh != NULL)
2739 err = ext4_ext_rm_idx(handle, inode, path, depth);
2740
2741out:
2742 return err;
2743}
2744
2745
2746
2747
2748
2749static int
2750ext4_ext_more_to_rm(struct ext4_ext_path *path)
2751{
2752 BUG_ON(path->p_idx == NULL);
2753
2754 if (path->p_idx < EXT_FIRST_INDEX(path->p_hdr))
2755 return 0;
2756
2757
2758
2759
2760
2761 if (le16_to_cpu(path->p_hdr->eh_entries) == path->p_block)
2762 return 0;
2763 return 1;
2764}
2765
2766int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start,
2767 ext4_lblk_t end)
2768{
2769 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
2770 int depth = ext_depth(inode);
2771 struct ext4_ext_path *path = NULL;
2772 struct partial_cluster partial;
2773 handle_t *handle;
2774 int i = 0, err = 0;
2775
2776 partial.pclu = 0;
2777 partial.lblk = 0;
2778 partial.state = initial;
2779
2780 ext_debug(inode, "truncate since %u to %u\n", start, end);
2781
2782
2783 handle = ext4_journal_start_with_revoke(inode, EXT4_HT_TRUNCATE,
2784 depth + 1,
2785 ext4_free_metadata_revoke_credits(inode->i_sb, depth));
2786 if (IS_ERR(handle))
2787 return PTR_ERR(handle);
2788
2789again:
2790 trace_ext4_ext_remove_space(inode, start, end, depth);
2791
2792
2793
2794
2795
2796
2797
2798
2799 if (end < EXT_MAX_BLOCKS - 1) {
2800 struct ext4_extent *ex;
2801 ext4_lblk_t ee_block, ex_end, lblk;
2802 ext4_fsblk_t pblk;
2803
2804
2805 path = ext4_find_extent(inode, end, NULL,
2806 EXT4_EX_NOCACHE | EXT4_EX_NOFAIL);
2807 if (IS_ERR(path)) {
2808 ext4_journal_stop(handle);
2809 return PTR_ERR(path);
2810 }
2811 depth = ext_depth(inode);
2812
2813 ex = path[depth].p_ext;
2814 if (!ex) {
2815 if (depth) {
2816 EXT4_ERROR_INODE(inode,
2817 "path[%d].p_hdr == NULL",
2818 depth);
2819 err = -EFSCORRUPTED;
2820 }
2821 goto out;
2822 }
2823
2824 ee_block = le32_to_cpu(ex->ee_block);
2825 ex_end = ee_block + ext4_ext_get_actual_len(ex) - 1;
2826
2827
2828
2829
2830
2831
2832
2833 if (end >= ee_block && end < ex_end) {
2834
2835
2836
2837
2838
2839
2840 if (sbi->s_cluster_ratio > 1) {
2841 pblk = ext4_ext_pblock(ex) + end - ee_block + 1;
2842 partial.pclu = EXT4_B2C(sbi, pblk);
2843 partial.state = nofree;
2844 }
2845
2846
2847
2848
2849
2850
2851
2852 err = ext4_force_split_extent_at(handle, inode, &path,
2853 end + 1, 1);
2854 if (err < 0)
2855 goto out;
2856
2857 } else if (sbi->s_cluster_ratio > 1 && end >= ex_end &&
2858 partial.state == initial) {
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869 lblk = ex_end + 1;
2870 err = ext4_ext_search_right(inode, path, &lblk, &pblk,
2871 &ex);
2872 if (err)
2873 goto out;
2874 if (pblk) {
2875 partial.pclu = EXT4_B2C(sbi, pblk);
2876 partial.state = nofree;
2877 }
2878 }
2879 }
2880
2881
2882
2883
2884 depth = ext_depth(inode);
2885 if (path) {
2886 int k = i = depth;
2887 while (--k > 0)
2888 path[k].p_block =
2889 le16_to_cpu(path[k].p_hdr->eh_entries)+1;
2890 } else {
2891 path = kcalloc(depth + 1, sizeof(struct ext4_ext_path),
2892 GFP_NOFS | __GFP_NOFAIL);
2893 if (path == NULL) {
2894 ext4_journal_stop(handle);
2895 return -ENOMEM;
2896 }
2897 path[0].p_maxdepth = path[0].p_depth = depth;
2898 path[0].p_hdr = ext_inode_hdr(inode);
2899 i = 0;
2900
2901 if (ext4_ext_check(inode, path[0].p_hdr, depth, 0)) {
2902 err = -EFSCORRUPTED;
2903 goto out;
2904 }
2905 }
2906 err = 0;
2907
2908 while (i >= 0 && err == 0) {
2909 if (i == depth) {
2910
2911 err = ext4_ext_rm_leaf(handle, inode, path,
2912 &partial, start, end);
2913
2914 brelse(path[i].p_bh);
2915 path[i].p_bh = NULL;
2916 i--;
2917 continue;
2918 }
2919
2920
2921 if (!path[i].p_hdr) {
2922 ext_debug(inode, "initialize header\n");
2923 path[i].p_hdr = ext_block_hdr(path[i].p_bh);
2924 }
2925
2926 if (!path[i].p_idx) {
2927
2928 path[i].p_idx = EXT_LAST_INDEX(path[i].p_hdr);
2929 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries)+1;
2930 ext_debug(inode, "init index ptr: hdr 0x%p, num %d\n",
2931 path[i].p_hdr,
2932 le16_to_cpu(path[i].p_hdr->eh_entries));
2933 } else {
2934
2935 path[i].p_idx--;
2936 }
2937
2938 ext_debug(inode, "level %d - index, first 0x%p, cur 0x%p\n",
2939 i, EXT_FIRST_INDEX(path[i].p_hdr),
2940 path[i].p_idx);
2941 if (ext4_ext_more_to_rm(path + i)) {
2942 struct buffer_head *bh;
2943
2944 ext_debug(inode, "move to level %d (block %llu)\n",
2945 i + 1, ext4_idx_pblock(path[i].p_idx));
2946 memset(path + i + 1, 0, sizeof(*path));
2947 bh = read_extent_tree_block(inode,
2948 ext4_idx_pblock(path[i].p_idx), depth - i - 1,
2949 EXT4_EX_NOCACHE);
2950 if (IS_ERR(bh)) {
2951
2952 err = PTR_ERR(bh);
2953 break;
2954 }
2955
2956
2957 cond_resched();
2958 if (WARN_ON(i + 1 > depth)) {
2959 err = -EFSCORRUPTED;
2960 break;
2961 }
2962 path[i + 1].p_bh = bh;
2963
2964
2965
2966 path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries);
2967 i++;
2968 } else {
2969
2970 if (path[i].p_hdr->eh_entries == 0 && i > 0) {
2971
2972
2973
2974 err = ext4_ext_rm_idx(handle, inode, path, i);
2975 }
2976
2977 brelse(path[i].p_bh);
2978 path[i].p_bh = NULL;
2979 i--;
2980 ext_debug(inode, "return to level %d\n", i);
2981 }
2982 }
2983
2984 trace_ext4_ext_remove_space_done(inode, start, end, depth, &partial,
2985 path->p_hdr->eh_entries);
2986
2987
2988
2989
2990
2991 if (partial.state == tofree && err == 0) {
2992 int flags = get_default_free_blocks_flags(inode);
2993
2994 if (ext4_is_pending(inode, partial.lblk))
2995 flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER;
2996 ext4_free_blocks(handle, inode, NULL,
2997 EXT4_C2B(sbi, partial.pclu),
2998 sbi->s_cluster_ratio, flags);
2999 if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)
3000 ext4_rereserve_cluster(inode, partial.lblk);
3001 partial.state = initial;
3002 }
3003
3004
3005 if (path->p_hdr->eh_entries == 0) {
3006
3007
3008
3009
3010 err = ext4_ext_get_access(handle, inode, path);
3011 if (err == 0) {
3012 ext_inode_hdr(inode)->eh_depth = 0;
3013 ext_inode_hdr(inode)->eh_max =
3014 cpu_to_le16(ext4_ext_space_root(inode, 0));
3015 err = ext4_ext_dirty(handle, inode, path);
3016 }
3017 }
3018out:
3019 ext4_ext_drop_refs(path);
3020 kfree(path);
3021 path = NULL;
3022 if (err == -EAGAIN)
3023 goto again;
3024 ext4_journal_stop(handle);
3025
3026 return err;
3027}
3028
3029
3030
3031
3032void ext4_ext_init(struct super_block *sb)
3033{
3034
3035
3036
3037
3038 if (ext4_has_feature_extents(sb)) {
3039#if defined(AGGRESSIVE_TEST) || defined(CHECK_BINSEARCH) || defined(EXTENTS_STATS)
3040 printk(KERN_INFO "EXT4-fs: file extents enabled"
3041#ifdef AGGRESSIVE_TEST
3042 ", aggressive tests"
3043#endif
3044#ifdef CHECK_BINSEARCH
3045 ", check binsearch"
3046#endif
3047#ifdef EXTENTS_STATS
3048 ", stats"
3049#endif
3050 "\n");
3051#endif
3052#ifdef EXTENTS_STATS
3053 spin_lock_init(&EXT4_SB(sb)->s_ext_stats_lock);
3054 EXT4_SB(sb)->s_ext_min = 1 << 30;
3055 EXT4_SB(sb)->s_ext_max = 0;
3056#endif
3057 }
3058}
3059
3060
3061
3062
3063void ext4_ext_release(struct super_block *sb)
3064{
3065 if (!ext4_has_feature_extents(sb))
3066 return;
3067
3068#ifdef EXTENTS_STATS
3069 if (EXT4_SB(sb)->s_ext_blocks && EXT4_SB(sb)->s_ext_extents) {
3070 struct ext4_sb_info *sbi = EXT4_SB(sb);
3071 printk(KERN_ERR "EXT4-fs: %lu blocks in %lu extents (%lu ave)\n",
3072 sbi->s_ext_blocks, sbi->s_ext_extents,
3073 sbi->s_ext_blocks / sbi->s_ext_extents);
3074 printk(KERN_ERR "EXT4-fs: extents: %lu min, %lu max, max depth %lu\n",
3075 sbi->s_ext_min, sbi->s_ext_max, sbi->s_depth_max);
3076 }
3077#endif
3078}
3079
3080static int ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex)
3081{
3082 ext4_lblk_t ee_block;
3083 ext4_fsblk_t ee_pblock;
3084 unsigned int ee_len;
3085
3086 ee_block = le32_to_cpu(ex->ee_block);
3087 ee_len = ext4_ext_get_actual_len(ex);
3088 ee_pblock = ext4_ext_pblock(ex);
3089
3090 if (ee_len == 0)
3091 return 0;
3092
3093 return ext4_es_insert_extent(inode, ee_block, ee_len, ee_pblock,
3094 EXTENT_STATUS_WRITTEN);
3095}
3096
3097
3098static int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex)
3099{
3100 ext4_fsblk_t ee_pblock;
3101 unsigned int ee_len;
3102
3103 ee_len = ext4_ext_get_actual_len(ex);
3104 ee_pblock = ext4_ext_pblock(ex);
3105 return ext4_issue_zeroout(inode, le32_to_cpu(ex->ee_block), ee_pblock,
3106 ee_len);
3107}
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130static int ext4_split_extent_at(handle_t *handle,
3131 struct inode *inode,
3132 struct ext4_ext_path **ppath,
3133 ext4_lblk_t split,
3134 int split_flag,
3135 int flags)
3136{
3137 struct ext4_ext_path *path = *ppath;
3138 ext4_fsblk_t newblock;
3139 ext4_lblk_t ee_block;
3140 struct ext4_extent *ex, newex, orig_ex, zero_ex;
3141 struct ext4_extent *ex2 = NULL;
3142 unsigned int ee_len, depth;
3143 int err = 0;
3144
3145 BUG_ON((split_flag & (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2)) ==
3146 (EXT4_EXT_DATA_VALID1 | EXT4_EXT_DATA_VALID2));
3147
3148 ext_debug(inode, "logical block %llu\n", (unsigned long long)split);
3149
3150 ext4_ext_show_leaf(inode, path);
3151
3152 depth = ext_depth(inode);
3153 ex = path[depth].p_ext;
3154 ee_block = le32_to_cpu(ex->ee_block);
3155 ee_len = ext4_ext_get_actual_len(ex);
3156 newblock = split - ee_block + ext4_ext_pblock(ex);
3157
3158 BUG_ON(split < ee_block || split >= (ee_block + ee_len));
3159 BUG_ON(!ext4_ext_is_unwritten(ex) &&
3160 split_flag & (EXT4_EXT_MAY_ZEROOUT |
3161 EXT4_EXT_MARK_UNWRIT1 |
3162 EXT4_EXT_MARK_UNWRIT2));
3163
3164 err = ext4_ext_get_access(handle, inode, path + depth);
3165 if (err)
3166 goto out;
3167
3168 if (split == ee_block) {
3169
3170
3171
3172
3173
3174 if (split_flag & EXT4_EXT_MARK_UNWRIT2)
3175 ext4_ext_mark_unwritten(ex);
3176 else
3177 ext4_ext_mark_initialized(ex);
3178
3179 if (!(flags & EXT4_GET_BLOCKS_PRE_IO))
3180 ext4_ext_try_to_merge(handle, inode, path, ex);
3181
3182 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3183 goto out;
3184 }
3185
3186
3187 memcpy(&orig_ex, ex, sizeof(orig_ex));
3188 ex->ee_len = cpu_to_le16(split - ee_block);
3189 if (split_flag & EXT4_EXT_MARK_UNWRIT1)
3190 ext4_ext_mark_unwritten(ex);
3191
3192
3193
3194
3195
3196 err = ext4_ext_dirty(handle, inode, path + depth);
3197 if (err)
3198 goto fix_extent_len;
3199
3200 ex2 = &newex;
3201 ex2->ee_block = cpu_to_le32(split);
3202 ex2->ee_len = cpu_to_le16(ee_len - (split - ee_block));
3203 ext4_ext_store_pblock(ex2, newblock);
3204 if (split_flag & EXT4_EXT_MARK_UNWRIT2)
3205 ext4_ext_mark_unwritten(ex2);
3206
3207 err = ext4_ext_insert_extent(handle, inode, ppath, &newex, flags);
3208 if (err == -ENOSPC && (EXT4_EXT_MAY_ZEROOUT & split_flag)) {
3209 if (split_flag & (EXT4_EXT_DATA_VALID1|EXT4_EXT_DATA_VALID2)) {
3210 if (split_flag & EXT4_EXT_DATA_VALID1) {
3211 err = ext4_ext_zeroout(inode, ex2);
3212 zero_ex.ee_block = ex2->ee_block;
3213 zero_ex.ee_len = cpu_to_le16(
3214 ext4_ext_get_actual_len(ex2));
3215 ext4_ext_store_pblock(&zero_ex,
3216 ext4_ext_pblock(ex2));
3217 } else {
3218 err = ext4_ext_zeroout(inode, ex);
3219 zero_ex.ee_block = ex->ee_block;
3220 zero_ex.ee_len = cpu_to_le16(
3221 ext4_ext_get_actual_len(ex));
3222 ext4_ext_store_pblock(&zero_ex,
3223 ext4_ext_pblock(ex));
3224 }
3225 } else {
3226 err = ext4_ext_zeroout(inode, &orig_ex);
3227 zero_ex.ee_block = orig_ex.ee_block;
3228 zero_ex.ee_len = cpu_to_le16(
3229 ext4_ext_get_actual_len(&orig_ex));
3230 ext4_ext_store_pblock(&zero_ex,
3231 ext4_ext_pblock(&orig_ex));
3232 }
3233
3234 if (err)
3235 goto fix_extent_len;
3236
3237 ex->ee_len = cpu_to_le16(ee_len);
3238 ext4_ext_try_to_merge(handle, inode, path, ex);
3239 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3240 if (err)
3241 goto fix_extent_len;
3242
3243
3244 err = ext4_zeroout_es(inode, &zero_ex);
3245
3246 goto out;
3247 } else if (err)
3248 goto fix_extent_len;
3249
3250out:
3251 ext4_ext_show_leaf(inode, path);
3252 return err;
3253
3254fix_extent_len:
3255 ex->ee_len = orig_ex.ee_len;
3256
3257
3258
3259
3260 ext4_ext_dirty(handle, inode, path + path->p_depth);
3261 return err;
3262}
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275static int ext4_split_extent(handle_t *handle,
3276 struct inode *inode,
3277 struct ext4_ext_path **ppath,
3278 struct ext4_map_blocks *map,
3279 int split_flag,
3280 int flags)
3281{
3282 struct ext4_ext_path *path = *ppath;
3283 ext4_lblk_t ee_block;
3284 struct ext4_extent *ex;
3285 unsigned int ee_len, depth;
3286 int err = 0;
3287 int unwritten;
3288 int split_flag1, flags1;
3289 int allocated = map->m_len;
3290
3291 depth = ext_depth(inode);
3292 ex = path[depth].p_ext;
3293 ee_block = le32_to_cpu(ex->ee_block);
3294 ee_len = ext4_ext_get_actual_len(ex);
3295 unwritten = ext4_ext_is_unwritten(ex);
3296
3297 if (map->m_lblk + map->m_len < ee_block + ee_len) {
3298 split_flag1 = split_flag & EXT4_EXT_MAY_ZEROOUT;
3299 flags1 = flags | EXT4_GET_BLOCKS_PRE_IO;
3300 if (unwritten)
3301 split_flag1 |= EXT4_EXT_MARK_UNWRIT1 |
3302 EXT4_EXT_MARK_UNWRIT2;
3303 if (split_flag & EXT4_EXT_DATA_VALID2)
3304 split_flag1 |= EXT4_EXT_DATA_VALID1;
3305 err = ext4_split_extent_at(handle, inode, ppath,
3306 map->m_lblk + map->m_len, split_flag1, flags1);
3307 if (err)
3308 goto out;
3309 } else {
3310 allocated = ee_len - (map->m_lblk - ee_block);
3311 }
3312
3313
3314
3315
3316 path = ext4_find_extent(inode, map->m_lblk, ppath, flags);
3317 if (IS_ERR(path))
3318 return PTR_ERR(path);
3319 depth = ext_depth(inode);
3320 ex = path[depth].p_ext;
3321 if (!ex) {
3322 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
3323 (unsigned long) map->m_lblk);
3324 return -EFSCORRUPTED;
3325 }
3326 unwritten = ext4_ext_is_unwritten(ex);
3327 split_flag1 = 0;
3328
3329 if (map->m_lblk >= ee_block) {
3330 split_flag1 = split_flag & EXT4_EXT_DATA_VALID2;
3331 if (unwritten) {
3332 split_flag1 |= EXT4_EXT_MARK_UNWRIT1;
3333 split_flag1 |= split_flag & (EXT4_EXT_MAY_ZEROOUT |
3334 EXT4_EXT_MARK_UNWRIT2);
3335 }
3336 err = ext4_split_extent_at(handle, inode, ppath,
3337 map->m_lblk, split_flag1, flags);
3338 if (err)
3339 goto out;
3340 }
3341
3342 ext4_ext_show_leaf(inode, path);
3343out:
3344 return err ? err : allocated;
3345}
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367static int ext4_ext_convert_to_initialized(handle_t *handle,
3368 struct inode *inode,
3369 struct ext4_map_blocks *map,
3370 struct ext4_ext_path **ppath,
3371 int flags)
3372{
3373 struct ext4_ext_path *path = *ppath;
3374 struct ext4_sb_info *sbi;
3375 struct ext4_extent_header *eh;
3376 struct ext4_map_blocks split_map;
3377 struct ext4_extent zero_ex1, zero_ex2;
3378 struct ext4_extent *ex, *abut_ex;
3379 ext4_lblk_t ee_block, eof_block;
3380 unsigned int ee_len, depth, map_len = map->m_len;
3381 int allocated = 0, max_zeroout = 0;
3382 int err = 0;
3383 int split_flag = EXT4_EXT_DATA_VALID2;
3384
3385 ext_debug(inode, "logical block %llu, max_blocks %u\n",
3386 (unsigned long long)map->m_lblk, map_len);
3387
3388 sbi = EXT4_SB(inode->i_sb);
3389 eof_block = (EXT4_I(inode)->i_disksize + inode->i_sb->s_blocksize - 1)
3390 >> inode->i_sb->s_blocksize_bits;
3391 if (eof_block < map->m_lblk + map_len)
3392 eof_block = map->m_lblk + map_len;
3393
3394 depth = ext_depth(inode);
3395 eh = path[depth].p_hdr;
3396 ex = path[depth].p_ext;
3397 ee_block = le32_to_cpu(ex->ee_block);
3398 ee_len = ext4_ext_get_actual_len(ex);
3399 zero_ex1.ee_len = 0;
3400 zero_ex2.ee_len = 0;
3401
3402 trace_ext4_ext_convert_to_initialized_enter(inode, map, ex);
3403
3404
3405 BUG_ON(!ext4_ext_is_unwritten(ex));
3406 BUG_ON(!in_range(map->m_lblk, ee_block, ee_len));
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423 if ((map->m_lblk == ee_block) &&
3424
3425 (map_len < ee_len) &&
3426 (ex > EXT_FIRST_EXTENT(eh))) {
3427 ext4_lblk_t prev_lblk;
3428 ext4_fsblk_t prev_pblk, ee_pblk;
3429 unsigned int prev_len;
3430
3431 abut_ex = ex - 1;
3432 prev_lblk = le32_to_cpu(abut_ex->ee_block);
3433 prev_len = ext4_ext_get_actual_len(abut_ex);
3434 prev_pblk = ext4_ext_pblock(abut_ex);
3435 ee_pblk = ext4_ext_pblock(ex);
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446 if ((!ext4_ext_is_unwritten(abut_ex)) &&
3447 ((prev_lblk + prev_len) == ee_block) &&
3448 ((prev_pblk + prev_len) == ee_pblk) &&
3449 (prev_len < (EXT_INIT_MAX_LEN - map_len))) {
3450 err = ext4_ext_get_access(handle, inode, path + depth);
3451 if (err)
3452 goto out;
3453
3454 trace_ext4_ext_convert_to_initialized_fastpath(inode,
3455 map, ex, abut_ex);
3456
3457
3458 ex->ee_block = cpu_to_le32(ee_block + map_len);
3459 ext4_ext_store_pblock(ex, ee_pblk + map_len);
3460 ex->ee_len = cpu_to_le16(ee_len - map_len);
3461 ext4_ext_mark_unwritten(ex);
3462
3463
3464 abut_ex->ee_len = cpu_to_le16(prev_len + map_len);
3465
3466
3467 allocated = map_len;
3468 }
3469 } else if (((map->m_lblk + map_len) == (ee_block + ee_len)) &&
3470 (map_len < ee_len) &&
3471 ex < EXT_LAST_EXTENT(eh)) {
3472
3473 ext4_lblk_t next_lblk;
3474 ext4_fsblk_t next_pblk, ee_pblk;
3475 unsigned int next_len;
3476
3477 abut_ex = ex + 1;
3478 next_lblk = le32_to_cpu(abut_ex->ee_block);
3479 next_len = ext4_ext_get_actual_len(abut_ex);
3480 next_pblk = ext4_ext_pblock(abut_ex);
3481 ee_pblk = ext4_ext_pblock(ex);
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492 if ((!ext4_ext_is_unwritten(abut_ex)) &&
3493 ((map->m_lblk + map_len) == next_lblk) &&
3494 ((ee_pblk + ee_len) == next_pblk) &&
3495 (next_len < (EXT_INIT_MAX_LEN - map_len))) {
3496 err = ext4_ext_get_access(handle, inode, path + depth);
3497 if (err)
3498 goto out;
3499
3500 trace_ext4_ext_convert_to_initialized_fastpath(inode,
3501 map, ex, abut_ex);
3502
3503
3504 abut_ex->ee_block = cpu_to_le32(next_lblk - map_len);
3505 ext4_ext_store_pblock(abut_ex, next_pblk - map_len);
3506 ex->ee_len = cpu_to_le16(ee_len - map_len);
3507 ext4_ext_mark_unwritten(ex);
3508
3509
3510 abut_ex->ee_len = cpu_to_le16(next_len + map_len);
3511
3512
3513 allocated = map_len;
3514 }
3515 }
3516 if (allocated) {
3517
3518 err = ext4_ext_dirty(handle, inode, path + depth);
3519
3520
3521 path[depth].p_ext = abut_ex;
3522 goto out;
3523 } else
3524 allocated = ee_len - (map->m_lblk - ee_block);
3525
3526 WARN_ON(map->m_lblk < ee_block);
3527
3528
3529
3530
3531 split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0;
3532
3533 if (EXT4_EXT_MAY_ZEROOUT & split_flag)
3534 max_zeroout = sbi->s_extent_max_zeroout_kb >>
3535 (inode->i_sb->s_blocksize_bits - 10);
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548 split_map.m_lblk = map->m_lblk;
3549 split_map.m_len = map->m_len;
3550
3551 if (max_zeroout && (allocated > split_map.m_len)) {
3552 if (allocated <= max_zeroout) {
3553
3554 zero_ex1.ee_block =
3555 cpu_to_le32(split_map.m_lblk +
3556 split_map.m_len);
3557 zero_ex1.ee_len =
3558 cpu_to_le16(allocated - split_map.m_len);
3559 ext4_ext_store_pblock(&zero_ex1,
3560 ext4_ext_pblock(ex) + split_map.m_lblk +
3561 split_map.m_len - ee_block);
3562 err = ext4_ext_zeroout(inode, &zero_ex1);
3563 if (err)
3564 goto out;
3565 split_map.m_len = allocated;
3566 }
3567 if (split_map.m_lblk - ee_block + split_map.m_len <
3568 max_zeroout) {
3569
3570 if (split_map.m_lblk != ee_block) {
3571 zero_ex2.ee_block = ex->ee_block;
3572 zero_ex2.ee_len = cpu_to_le16(split_map.m_lblk -
3573 ee_block);
3574 ext4_ext_store_pblock(&zero_ex2,
3575 ext4_ext_pblock(ex));
3576 err = ext4_ext_zeroout(inode, &zero_ex2);
3577 if (err)
3578 goto out;
3579 }
3580
3581 split_map.m_len += split_map.m_lblk - ee_block;
3582 split_map.m_lblk = ee_block;
3583 allocated = map->m_len;
3584 }
3585 }
3586
3587 err = ext4_split_extent(handle, inode, ppath, &split_map, split_flag,
3588 flags);
3589 if (err > 0)
3590 err = 0;
3591out:
3592
3593 if (!err) {
3594 err = ext4_zeroout_es(inode, &zero_ex1);
3595 if (!err)
3596 err = ext4_zeroout_es(inode, &zero_ex2);
3597 }
3598 return err ? err : allocated;
3599}
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625static int ext4_split_convert_extents(handle_t *handle,
3626 struct inode *inode,
3627 struct ext4_map_blocks *map,
3628 struct ext4_ext_path **ppath,
3629 int flags)
3630{
3631 struct ext4_ext_path *path = *ppath;
3632 ext4_lblk_t eof_block;
3633 ext4_lblk_t ee_block;
3634 struct ext4_extent *ex;
3635 unsigned int ee_len;
3636 int split_flag = 0, depth;
3637
3638 ext_debug(inode, "logical block %llu, max_blocks %u\n",
3639 (unsigned long long)map->m_lblk, map->m_len);
3640
3641 eof_block = (EXT4_I(inode)->i_disksize + inode->i_sb->s_blocksize - 1)
3642 >> inode->i_sb->s_blocksize_bits;
3643 if (eof_block < map->m_lblk + map->m_len)
3644 eof_block = map->m_lblk + map->m_len;
3645
3646
3647
3648
3649 depth = ext_depth(inode);
3650 ex = path[depth].p_ext;
3651 ee_block = le32_to_cpu(ex->ee_block);
3652 ee_len = ext4_ext_get_actual_len(ex);
3653
3654
3655 if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) {
3656 split_flag |= EXT4_EXT_DATA_VALID1;
3657
3658 } else if (flags & EXT4_GET_BLOCKS_CONVERT) {
3659 split_flag |= ee_block + ee_len <= eof_block ?
3660 EXT4_EXT_MAY_ZEROOUT : 0;
3661 split_flag |= (EXT4_EXT_MARK_UNWRIT2 | EXT4_EXT_DATA_VALID2);
3662 }
3663 flags |= EXT4_GET_BLOCKS_PRE_IO;
3664 return ext4_split_extent(handle, inode, ppath, map, split_flag, flags);
3665}
3666
3667static int ext4_convert_unwritten_extents_endio(handle_t *handle,
3668 struct inode *inode,
3669 struct ext4_map_blocks *map,
3670 struct ext4_ext_path **ppath)
3671{
3672 struct ext4_ext_path *path = *ppath;
3673 struct ext4_extent *ex;
3674 ext4_lblk_t ee_block;
3675 unsigned int ee_len;
3676 int depth;
3677 int err = 0;
3678
3679 depth = ext_depth(inode);
3680 ex = path[depth].p_ext;
3681 ee_block = le32_to_cpu(ex->ee_block);
3682 ee_len = ext4_ext_get_actual_len(ex);
3683
3684 ext_debug(inode, "logical block %llu, max_blocks %u\n",
3685 (unsigned long long)ee_block, ee_len);
3686
3687
3688
3689
3690
3691
3692
3693 if (ee_block != map->m_lblk || ee_len > map->m_len) {
3694#ifdef CONFIG_EXT4_DEBUG
3695 ext4_warning(inode->i_sb, "Inode (%ld) finished: extent logical block %llu,"
3696 " len %u; IO logical block %llu, len %u",
3697 inode->i_ino, (unsigned long long)ee_block, ee_len,
3698 (unsigned long long)map->m_lblk, map->m_len);
3699#endif
3700 err = ext4_split_convert_extents(handle, inode, map, ppath,
3701 EXT4_GET_BLOCKS_CONVERT);
3702 if (err < 0)
3703 return err;
3704 path = ext4_find_extent(inode, map->m_lblk, ppath, 0);
3705 if (IS_ERR(path))
3706 return PTR_ERR(path);
3707 depth = ext_depth(inode);
3708 ex = path[depth].p_ext;
3709 }
3710
3711 err = ext4_ext_get_access(handle, inode, path + depth);
3712 if (err)
3713 goto out;
3714
3715 ext4_ext_mark_initialized(ex);
3716
3717
3718
3719
3720 ext4_ext_try_to_merge(handle, inode, path, ex);
3721
3722
3723 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3724out:
3725 ext4_ext_show_leaf(inode, path);
3726 return err;
3727}
3728
3729static int
3730convert_initialized_extent(handle_t *handle, struct inode *inode,
3731 struct ext4_map_blocks *map,
3732 struct ext4_ext_path **ppath,
3733 unsigned int *allocated)
3734{
3735 struct ext4_ext_path *path = *ppath;
3736 struct ext4_extent *ex;
3737 ext4_lblk_t ee_block;
3738 unsigned int ee_len;
3739 int depth;
3740 int err = 0;
3741
3742
3743
3744
3745
3746 if (map->m_len > EXT_UNWRITTEN_MAX_LEN)
3747 map->m_len = EXT_UNWRITTEN_MAX_LEN / 2;
3748
3749 depth = ext_depth(inode);
3750 ex = path[depth].p_ext;
3751 ee_block = le32_to_cpu(ex->ee_block);
3752 ee_len = ext4_ext_get_actual_len(ex);
3753
3754 ext_debug(inode, "logical block %llu, max_blocks %u\n",
3755 (unsigned long long)ee_block, ee_len);
3756
3757 if (ee_block != map->m_lblk || ee_len > map->m_len) {
3758 err = ext4_split_convert_extents(handle, inode, map, ppath,
3759 EXT4_GET_BLOCKS_CONVERT_UNWRITTEN);
3760 if (err < 0)
3761 return err;
3762 path = ext4_find_extent(inode, map->m_lblk, ppath, 0);
3763 if (IS_ERR(path))
3764 return PTR_ERR(path);
3765 depth = ext_depth(inode);
3766 ex = path[depth].p_ext;
3767 if (!ex) {
3768 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
3769 (unsigned long) map->m_lblk);
3770 return -EFSCORRUPTED;
3771 }
3772 }
3773
3774 err = ext4_ext_get_access(handle, inode, path + depth);
3775 if (err)
3776 return err;
3777
3778 ext4_ext_mark_unwritten(ex);
3779
3780
3781
3782
3783 ext4_ext_try_to_merge(handle, inode, path, ex);
3784
3785
3786 err = ext4_ext_dirty(handle, inode, path + path->p_depth);
3787 if (err)
3788 return err;
3789 ext4_ext_show_leaf(inode, path);
3790
3791 ext4_update_inode_fsync_trans(handle, inode, 1);
3792
3793 map->m_flags |= EXT4_MAP_UNWRITTEN;
3794 if (*allocated > map->m_len)
3795 *allocated = map->m_len;
3796 map->m_len = *allocated;
3797 return 0;
3798}
3799
3800static int
3801ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode,
3802 struct ext4_map_blocks *map,
3803 struct ext4_ext_path **ppath, int flags,
3804 unsigned int allocated, ext4_fsblk_t newblock)
3805{
3806 struct ext4_ext_path __maybe_unused *path = *ppath;
3807 int ret = 0;
3808 int err = 0;
3809
3810 ext_debug(inode, "logical block %llu, max_blocks %u, flags 0x%x, allocated %u\n",
3811 (unsigned long long)map->m_lblk, map->m_len, flags,
3812 allocated);
3813 ext4_ext_show_leaf(inode, path);
3814
3815
3816
3817
3818
3819 flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL;
3820
3821 trace_ext4_ext_handle_unwritten_extents(inode, map, flags,
3822 allocated, newblock);
3823
3824
3825 if (flags & EXT4_GET_BLOCKS_PRE_IO) {
3826 ret = ext4_split_convert_extents(handle, inode, map, ppath,
3827 flags | EXT4_GET_BLOCKS_CONVERT);
3828 if (ret < 0) {
3829 err = ret;
3830 goto out2;
3831 }
3832
3833
3834
3835
3836 if (unlikely(ret == 0)) {
3837 EXT4_ERROR_INODE(inode,
3838 "unexpected ret == 0, m_len = %u",
3839 map->m_len);
3840 err = -EFSCORRUPTED;
3841 goto out2;
3842 }
3843 map->m_flags |= EXT4_MAP_UNWRITTEN;
3844 goto out;
3845 }
3846
3847 if (flags & EXT4_GET_BLOCKS_CONVERT) {
3848 err = ext4_convert_unwritten_extents_endio(handle, inode, map,
3849 ppath);
3850 if (err < 0)
3851 goto out2;
3852 ext4_update_inode_fsync_trans(handle, inode, 1);
3853 goto map_out;
3854 }
3855
3856
3857
3858
3859
3860 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) {
3861 map->m_flags |= EXT4_MAP_UNWRITTEN;
3862 goto map_out;
3863 }
3864
3865
3866 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
3867
3868
3869
3870
3871
3872
3873
3874 map->m_flags |= EXT4_MAP_UNWRITTEN;
3875 goto out1;
3876 }
3877
3878
3879
3880
3881
3882
3883 ret = ext4_ext_convert_to_initialized(handle, inode, map, ppath, flags);
3884 if (ret < 0) {
3885 err = ret;
3886 goto out2;
3887 }
3888 ext4_update_inode_fsync_trans(handle, inode, 1);
3889
3890
3891
3892
3893 if (unlikely(ret == 0)) {
3894 EXT4_ERROR_INODE(inode, "unexpected ret == 0, m_len = %u",
3895 map->m_len);
3896 err = -EFSCORRUPTED;
3897 goto out2;
3898 }
3899
3900out:
3901 allocated = ret;
3902 map->m_flags |= EXT4_MAP_NEW;
3903map_out:
3904 map->m_flags |= EXT4_MAP_MAPPED;
3905out1:
3906 map->m_pblk = newblock;
3907 if (allocated > map->m_len)
3908 allocated = map->m_len;
3909 map->m_len = allocated;
3910 ext4_ext_show_leaf(inode, path);
3911out2:
3912 return err ? err : allocated;
3913}
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956static int get_implied_cluster_alloc(struct super_block *sb,
3957 struct ext4_map_blocks *map,
3958 struct ext4_extent *ex,
3959 struct ext4_ext_path *path)
3960{
3961 struct ext4_sb_info *sbi = EXT4_SB(sb);
3962 ext4_lblk_t c_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
3963 ext4_lblk_t ex_cluster_start, ex_cluster_end;
3964 ext4_lblk_t rr_cluster_start;
3965 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
3966 ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
3967 unsigned short ee_len = ext4_ext_get_actual_len(ex);
3968
3969
3970 ex_cluster_start = EXT4_B2C(sbi, ee_block);
3971 ex_cluster_end = EXT4_B2C(sbi, ee_block + ee_len - 1);
3972
3973
3974 rr_cluster_start = EXT4_B2C(sbi, map->m_lblk);
3975
3976 if ((rr_cluster_start == ex_cluster_end) ||
3977 (rr_cluster_start == ex_cluster_start)) {
3978 if (rr_cluster_start == ex_cluster_end)
3979 ee_start += ee_len - 1;
3980 map->m_pblk = EXT4_PBLK_CMASK(sbi, ee_start) + c_offset;
3981 map->m_len = min(map->m_len,
3982 (unsigned) sbi->s_cluster_ratio - c_offset);
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992 if (map->m_lblk < ee_block)
3993 map->m_len = min(map->m_len, ee_block - map->m_lblk);
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004 if (map->m_lblk > ee_block) {
4005 ext4_lblk_t next = ext4_ext_next_allocated_block(path);
4006 map->m_len = min(map->m_len, next - map->m_lblk);
4007 }
4008
4009 trace_ext4_get_implied_cluster_alloc_exit(sb, map, 1);
4010 return 1;
4011 }
4012
4013 trace_ext4_get_implied_cluster_alloc_exit(sb, map, 0);
4014 return 0;
4015}
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036int ext4_ext_map_blocks(handle_t *handle, struct inode *inode,
4037 struct ext4_map_blocks *map, int flags)
4038{
4039 struct ext4_ext_path *path = NULL;
4040 struct ext4_extent newex, *ex, *ex2;
4041 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4042 ext4_fsblk_t newblock = 0, pblk;
4043 int err = 0, depth, ret;
4044 unsigned int allocated = 0, offset = 0;
4045 unsigned int allocated_clusters = 0;
4046 struct ext4_allocation_request ar;
4047 ext4_lblk_t cluster_offset;
4048
4049 ext_debug(inode, "blocks %u/%u requested\n", map->m_lblk, map->m_len);
4050 trace_ext4_ext_map_blocks_enter(inode, map->m_lblk, map->m_len, flags);
4051
4052
4053 path = ext4_find_extent(inode, map->m_lblk, NULL, 0);
4054 if (IS_ERR(path)) {
4055 err = PTR_ERR(path);
4056 path = NULL;
4057 goto out;
4058 }
4059
4060 depth = ext_depth(inode);
4061
4062
4063
4064
4065
4066
4067 if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
4068 EXT4_ERROR_INODE(inode, "bad extent address "
4069 "lblock: %lu, depth: %d pblock %lld",
4070 (unsigned long) map->m_lblk, depth,
4071 path[depth].p_block);
4072 err = -EFSCORRUPTED;
4073 goto out;
4074 }
4075
4076 ex = path[depth].p_ext;
4077 if (ex) {
4078 ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block);
4079 ext4_fsblk_t ee_start = ext4_ext_pblock(ex);
4080 unsigned short ee_len;
4081
4082
4083
4084
4085
4086
4087 ee_len = ext4_ext_get_actual_len(ex);
4088
4089 trace_ext4_ext_show_extent(inode, ee_block, ee_start, ee_len);
4090
4091
4092 if (in_range(map->m_lblk, ee_block, ee_len)) {
4093 newblock = map->m_lblk - ee_block + ee_start;
4094
4095 allocated = ee_len - (map->m_lblk - ee_block);
4096 ext_debug(inode, "%u fit into %u:%d -> %llu\n",
4097 map->m_lblk, ee_block, ee_len, newblock);
4098
4099
4100
4101
4102
4103 if ((!ext4_ext_is_unwritten(ex)) &&
4104 (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) {
4105 err = convert_initialized_extent(handle,
4106 inode, map, &path, &allocated);
4107 goto out;
4108 } else if (!ext4_ext_is_unwritten(ex)) {
4109 map->m_flags |= EXT4_MAP_MAPPED;
4110 map->m_pblk = newblock;
4111 if (allocated > map->m_len)
4112 allocated = map->m_len;
4113 map->m_len = allocated;
4114 ext4_ext_show_leaf(inode, path);
4115 goto out;
4116 }
4117
4118 ret = ext4_ext_handle_unwritten_extents(
4119 handle, inode, map, &path, flags,
4120 allocated, newblock);
4121 if (ret < 0)
4122 err = ret;
4123 else
4124 allocated = ret;
4125 goto out;
4126 }
4127 }
4128
4129
4130
4131
4132
4133 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) {
4134 ext4_lblk_t hole_start, hole_len;
4135
4136 hole_start = map->m_lblk;
4137 hole_len = ext4_ext_determine_hole(inode, path, &hole_start);
4138
4139
4140
4141
4142 ext4_ext_put_gap_in_cache(inode, hole_start, hole_len);
4143
4144
4145 if (hole_start != map->m_lblk)
4146 hole_len -= map->m_lblk - hole_start;
4147 map->m_pblk = 0;
4148 map->m_len = min_t(unsigned int, map->m_len, hole_len);
4149
4150 goto out;
4151 }
4152
4153
4154
4155
4156 newex.ee_block = cpu_to_le32(map->m_lblk);
4157 cluster_offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
4158
4159
4160
4161
4162
4163 if (cluster_offset && ex &&
4164 get_implied_cluster_alloc(inode->i_sb, map, ex, path)) {
4165 ar.len = allocated = map->m_len;
4166 newblock = map->m_pblk;
4167 goto got_allocated_blocks;
4168 }
4169
4170
4171 ar.lleft = map->m_lblk;
4172 err = ext4_ext_search_left(inode, path, &ar.lleft, &ar.pleft);
4173 if (err)
4174 goto out;
4175 ar.lright = map->m_lblk;
4176 ex2 = NULL;
4177 err = ext4_ext_search_right(inode, path, &ar.lright, &ar.pright, &ex2);
4178 if (err)
4179 goto out;
4180
4181
4182
4183 if ((sbi->s_cluster_ratio > 1) && ex2 &&
4184 get_implied_cluster_alloc(inode->i_sb, map, ex2, path)) {
4185 ar.len = allocated = map->m_len;
4186 newblock = map->m_pblk;
4187 goto got_allocated_blocks;
4188 }
4189
4190
4191
4192
4193
4194
4195
4196 if (map->m_len > EXT_INIT_MAX_LEN &&
4197 !(flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
4198 map->m_len = EXT_INIT_MAX_LEN;
4199 else if (map->m_len > EXT_UNWRITTEN_MAX_LEN &&
4200 (flags & EXT4_GET_BLOCKS_UNWRIT_EXT))
4201 map->m_len = EXT_UNWRITTEN_MAX_LEN;
4202
4203
4204 newex.ee_len = cpu_to_le16(map->m_len);
4205 err = ext4_ext_check_overlap(sbi, inode, &newex, path);
4206 if (err)
4207 allocated = ext4_ext_get_actual_len(&newex);
4208 else
4209 allocated = map->m_len;
4210
4211
4212 ar.inode = inode;
4213 ar.goal = ext4_ext_find_goal(inode, path, map->m_lblk);
4214 ar.logical = map->m_lblk;
4215
4216
4217
4218
4219
4220
4221
4222
4223 offset = EXT4_LBLK_COFF(sbi, map->m_lblk);
4224 ar.len = EXT4_NUM_B2C(sbi, offset+allocated);
4225 ar.goal -= offset;
4226 ar.logical -= offset;
4227 if (S_ISREG(inode->i_mode))
4228 ar.flags = EXT4_MB_HINT_DATA;
4229 else
4230
4231 ar.flags = 0;
4232 if (flags & EXT4_GET_BLOCKS_NO_NORMALIZE)
4233 ar.flags |= EXT4_MB_HINT_NOPREALLOC;
4234 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
4235 ar.flags |= EXT4_MB_DELALLOC_RESERVED;
4236 if (flags & EXT4_GET_BLOCKS_METADATA_NOFAIL)
4237 ar.flags |= EXT4_MB_USE_RESERVED;
4238 newblock = ext4_mb_new_blocks(handle, &ar, &err);
4239 if (!newblock)
4240 goto out;
4241 allocated_clusters = ar.len;
4242 ar.len = EXT4_C2B(sbi, ar.len) - offset;
4243 ext_debug(inode, "allocate new block: goal %llu, found %llu/%u, requested %u\n",
4244 ar.goal, newblock, ar.len, allocated);
4245 if (ar.len > allocated)
4246 ar.len = allocated;
4247
4248got_allocated_blocks:
4249
4250 pblk = newblock + offset;
4251 ext4_ext_store_pblock(&newex, pblk);
4252 newex.ee_len = cpu_to_le16(ar.len);
4253
4254 if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) {
4255 ext4_ext_mark_unwritten(&newex);
4256 map->m_flags |= EXT4_MAP_UNWRITTEN;
4257 }
4258
4259 err = ext4_ext_insert_extent(handle, inode, &path, &newex, flags);
4260 if (err) {
4261 if (allocated_clusters) {
4262 int fb_flags = 0;
4263
4264
4265
4266
4267
4268
4269 ext4_discard_preallocations(inode, 0);
4270 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE)
4271 fb_flags = EXT4_FREE_BLOCKS_NO_QUOT_UPDATE;
4272 ext4_free_blocks(handle, inode, NULL, newblock,
4273 EXT4_C2B(sbi, allocated_clusters),
4274 fb_flags);
4275 }
4276 goto out;
4277 }
4278
4279
4280
4281
4282
4283
4284
4285 if (test_opt(inode->i_sb, DELALLOC) && allocated_clusters) {
4286 if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) {
4287
4288
4289
4290
4291 ext4_da_update_reserve_space(inode, allocated_clusters,
4292 1);
4293 } else {
4294 ext4_lblk_t lblk, len;
4295 unsigned int n;
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308 lblk = EXT4_LBLK_CMASK(sbi, map->m_lblk);
4309 len = allocated_clusters << sbi->s_cluster_bits;
4310 n = ext4_es_delayed_clu(inode, lblk, len);
4311 if (n > 0)
4312 ext4_da_update_reserve_space(inode, (int) n, 0);
4313 }
4314 }
4315
4316
4317
4318
4319
4320 if ((flags & EXT4_GET_BLOCKS_UNWRIT_EXT) == 0)
4321 ext4_update_inode_fsync_trans(handle, inode, 1);
4322 else
4323 ext4_update_inode_fsync_trans(handle, inode, 0);
4324
4325 map->m_flags |= (EXT4_MAP_NEW | EXT4_MAP_MAPPED);
4326 map->m_pblk = pblk;
4327 map->m_len = ar.len;
4328 allocated = map->m_len;
4329 ext4_ext_show_leaf(inode, path);
4330
4331out:
4332 ext4_ext_drop_refs(path);
4333 kfree(path);
4334
4335 trace_ext4_ext_map_blocks_exit(inode, flags, map,
4336 err ? err : allocated);
4337 return err ? err : allocated;
4338}
4339
4340int ext4_ext_truncate(handle_t *handle, struct inode *inode)
4341{
4342 struct super_block *sb = inode->i_sb;
4343 ext4_lblk_t last_block;
4344 int err = 0;
4345
4346
4347
4348
4349
4350
4351
4352
4353 EXT4_I(inode)->i_disksize = inode->i_size;
4354 err = ext4_mark_inode_dirty(handle, inode);
4355 if (err)
4356 return err;
4357
4358 last_block = (inode->i_size + sb->s_blocksize - 1)
4359 >> EXT4_BLOCK_SIZE_BITS(sb);
4360retry:
4361 err = ext4_es_remove_extent(inode, last_block,
4362 EXT_MAX_BLOCKS - last_block);
4363 if (err == -ENOMEM) {
4364 cond_resched();
4365 congestion_wait(BLK_RW_ASYNC, HZ/50);
4366 goto retry;
4367 }
4368 if (err)
4369 return err;
4370retry_remove_space:
4371 err = ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1);
4372 if (err == -ENOMEM) {
4373 cond_resched();
4374 congestion_wait(BLK_RW_ASYNC, HZ/50);
4375 goto retry_remove_space;
4376 }
4377 return err;
4378}
4379
4380static int ext4_alloc_file_blocks(struct file *file, ext4_lblk_t offset,
4381 ext4_lblk_t len, loff_t new_size,
4382 int flags)
4383{
4384 struct inode *inode = file_inode(file);
4385 handle_t *handle;
4386 int ret = 0;
4387 int ret2 = 0, ret3 = 0;
4388 int retries = 0;
4389 int depth = 0;
4390 struct ext4_map_blocks map;
4391 unsigned int credits;
4392 loff_t epos;
4393
4394 BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS));
4395 map.m_lblk = offset;
4396 map.m_len = len;
4397
4398
4399
4400
4401
4402 if (len <= EXT_UNWRITTEN_MAX_LEN)
4403 flags |= EXT4_GET_BLOCKS_NO_NORMALIZE;
4404
4405
4406
4407
4408 credits = ext4_chunk_trans_blocks(inode, len);
4409 depth = ext_depth(inode);
4410
4411retry:
4412 while (ret >= 0 && len) {
4413
4414
4415
4416 if (depth != ext_depth(inode)) {
4417 credits = ext4_chunk_trans_blocks(inode, len);
4418 depth = ext_depth(inode);
4419 }
4420
4421 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
4422 credits);
4423 if (IS_ERR(handle)) {
4424 ret = PTR_ERR(handle);
4425 break;
4426 }
4427 ret = ext4_map_blocks(handle, inode, &map, flags);
4428 if (ret <= 0) {
4429 ext4_debug("inode #%lu: block %u: len %u: "
4430 "ext4_ext_map_blocks returned %d",
4431 inode->i_ino, map.m_lblk,
4432 map.m_len, ret);
4433 ext4_mark_inode_dirty(handle, inode);
4434 ret2 = ext4_journal_stop(handle);
4435 break;
4436 }
4437 map.m_lblk += ret;
4438 map.m_len = len = len - ret;
4439 epos = (loff_t)map.m_lblk << inode->i_blkbits;
4440 inode->i_ctime = current_time(inode);
4441 if (new_size) {
4442 if (epos > new_size)
4443 epos = new_size;
4444 if (ext4_update_inode_size(inode, epos) & 0x1)
4445 inode->i_mtime = inode->i_ctime;
4446 }
4447 ret2 = ext4_mark_inode_dirty(handle, inode);
4448 ext4_update_inode_fsync_trans(handle, inode, 1);
4449 ret3 = ext4_journal_stop(handle);
4450 ret2 = ret3 ? ret3 : ret2;
4451 if (unlikely(ret2))
4452 break;
4453 }
4454 if (ret == -ENOSPC &&
4455 ext4_should_retry_alloc(inode->i_sb, &retries)) {
4456 ret = 0;
4457 goto retry;
4458 }
4459
4460 return ret > 0 ? ret2 : ret;
4461}
4462
4463static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len);
4464
4465static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len);
4466
4467static long ext4_zero_range(struct file *file, loff_t offset,
4468 loff_t len, int mode)
4469{
4470 struct inode *inode = file_inode(file);
4471 handle_t *handle = NULL;
4472 unsigned int max_blocks;
4473 loff_t new_size = 0;
4474 int ret = 0;
4475 int flags;
4476 int credits;
4477 int partial_begin, partial_end;
4478 loff_t start, end;
4479 ext4_lblk_t lblk;
4480 unsigned int blkbits = inode->i_blkbits;
4481
4482 trace_ext4_zero_range(inode, offset, len, mode);
4483
4484
4485 if (ext4_should_journal_data(inode)) {
4486 ret = ext4_force_commit(inode->i_sb);
4487 if (ret)
4488 return ret;
4489 }
4490
4491
4492
4493
4494
4495
4496
4497 start = round_up(offset, 1 << blkbits);
4498 end = round_down((offset + len), 1 << blkbits);
4499
4500 if (start < offset || end > offset + len)
4501 return -EINVAL;
4502 partial_begin = offset & ((1 << blkbits) - 1);
4503 partial_end = (offset + len) & ((1 << blkbits) - 1);
4504
4505 lblk = start >> blkbits;
4506 max_blocks = (end >> blkbits);
4507 if (max_blocks < lblk)
4508 max_blocks = 0;
4509 else
4510 max_blocks -= lblk;
4511
4512 inode_lock(inode);
4513
4514
4515
4516
4517 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
4518 ret = -EOPNOTSUPP;
4519 goto out_mutex;
4520 }
4521
4522 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
4523 (offset + len > inode->i_size ||
4524 offset + len > EXT4_I(inode)->i_disksize)) {
4525 new_size = offset + len;
4526 ret = inode_newsize_ok(inode, new_size);
4527 if (ret)
4528 goto out_mutex;
4529 }
4530
4531 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
4532
4533
4534 inode_dio_wait(inode);
4535
4536
4537 if (partial_begin || partial_end) {
4538 ret = ext4_alloc_file_blocks(file,
4539 round_down(offset, 1 << blkbits) >> blkbits,
4540 (round_up((offset + len), 1 << blkbits) -
4541 round_down(offset, 1 << blkbits)) >> blkbits,
4542 new_size, flags);
4543 if (ret)
4544 goto out_mutex;
4545
4546 }
4547
4548
4549 if (max_blocks > 0) {
4550 flags |= (EXT4_GET_BLOCKS_CONVERT_UNWRITTEN |
4551 EXT4_EX_NOCACHE);
4552
4553
4554
4555
4556
4557 down_write(&EXT4_I(inode)->i_mmap_sem);
4558
4559 ret = ext4_break_layouts(inode);
4560 if (ret) {
4561 up_write(&EXT4_I(inode)->i_mmap_sem);
4562 goto out_mutex;
4563 }
4564
4565 ret = ext4_update_disksize_before_punch(inode, offset, len);
4566 if (ret) {
4567 up_write(&EXT4_I(inode)->i_mmap_sem);
4568 goto out_mutex;
4569 }
4570
4571 truncate_pagecache_range(inode, start, end - 1);
4572 inode->i_mtime = inode->i_ctime = current_time(inode);
4573
4574 ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size,
4575 flags);
4576 up_write(&EXT4_I(inode)->i_mmap_sem);
4577 if (ret)
4578 goto out_mutex;
4579 }
4580 if (!partial_begin && !partial_end)
4581 goto out_mutex;
4582
4583
4584
4585
4586
4587 credits = (2 * ext4_ext_index_trans_blocks(inode, 2)) + 1;
4588 if (ext4_should_journal_data(inode))
4589 credits += 2;
4590 handle = ext4_journal_start(inode, EXT4_HT_MISC, credits);
4591 if (IS_ERR(handle)) {
4592 ret = PTR_ERR(handle);
4593 ext4_std_error(inode->i_sb, ret);
4594 goto out_mutex;
4595 }
4596
4597 inode->i_mtime = inode->i_ctime = current_time(inode);
4598 if (new_size)
4599 ext4_update_inode_size(inode, new_size);
4600 ret = ext4_mark_inode_dirty(handle, inode);
4601 if (unlikely(ret))
4602 goto out_handle;
4603
4604
4605 ret = ext4_zero_partial_blocks(handle, inode, offset, len);
4606 if (ret >= 0)
4607 ext4_update_inode_fsync_trans(handle, inode, 1);
4608
4609 if (file->f_flags & O_SYNC)
4610 ext4_handle_sync(handle);
4611
4612out_handle:
4613 ext4_journal_stop(handle);
4614out_mutex:
4615 inode_unlock(inode);
4616 return ret;
4617}
4618
4619
4620
4621
4622
4623
4624
4625
4626long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
4627{
4628 struct inode *inode = file_inode(file);
4629 loff_t new_size = 0;
4630 unsigned int max_blocks;
4631 int ret = 0;
4632 int flags;
4633 ext4_lblk_t lblk;
4634 unsigned int blkbits = inode->i_blkbits;
4635
4636
4637
4638
4639
4640
4641
4642 if (IS_ENCRYPTED(inode) &&
4643 (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE)))
4644 return -EOPNOTSUPP;
4645
4646
4647 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
4648 FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE |
4649 FALLOC_FL_INSERT_RANGE))
4650 return -EOPNOTSUPP;
4651
4652 if (mode & FALLOC_FL_PUNCH_HOLE)
4653 return ext4_punch_hole(inode, offset, len);
4654
4655 ret = ext4_convert_inline_data(inode);
4656 if (ret)
4657 return ret;
4658
4659 if (mode & FALLOC_FL_COLLAPSE_RANGE)
4660 return ext4_collapse_range(inode, offset, len);
4661
4662 if (mode & FALLOC_FL_INSERT_RANGE)
4663 return ext4_insert_range(inode, offset, len);
4664
4665 if (mode & FALLOC_FL_ZERO_RANGE)
4666 return ext4_zero_range(file, offset, len, mode);
4667
4668 trace_ext4_fallocate_enter(inode, offset, len, mode);
4669 lblk = offset >> blkbits;
4670
4671 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
4672 flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT;
4673
4674 inode_lock(inode);
4675
4676
4677
4678
4679 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
4680 ret = -EOPNOTSUPP;
4681 goto out;
4682 }
4683
4684 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
4685 (offset + len > inode->i_size ||
4686 offset + len > EXT4_I(inode)->i_disksize)) {
4687 new_size = offset + len;
4688 ret = inode_newsize_ok(inode, new_size);
4689 if (ret)
4690 goto out;
4691 }
4692
4693
4694 inode_dio_wait(inode);
4695
4696 ret = ext4_alloc_file_blocks(file, lblk, max_blocks, new_size, flags);
4697 if (ret)
4698 goto out;
4699
4700 if (file->f_flags & O_SYNC && EXT4_SB(inode->i_sb)->s_journal) {
4701 ret = jbd2_complete_transaction(EXT4_SB(inode->i_sb)->s_journal,
4702 EXT4_I(inode)->i_sync_tid);
4703 }
4704out:
4705 inode_unlock(inode);
4706 trace_ext4_fallocate_exit(inode, offset, max_blocks, ret);
4707 return ret;
4708}
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode,
4721 loff_t offset, ssize_t len)
4722{
4723 unsigned int max_blocks;
4724 int ret = 0, ret2 = 0, ret3 = 0;
4725 struct ext4_map_blocks map;
4726 unsigned int blkbits = inode->i_blkbits;
4727 unsigned int credits = 0;
4728
4729 map.m_lblk = offset >> blkbits;
4730 max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits);
4731
4732 if (!handle) {
4733
4734
4735
4736 credits = ext4_chunk_trans_blocks(inode, max_blocks);
4737 }
4738 while (ret >= 0 && ret < max_blocks) {
4739 map.m_lblk += ret;
4740 map.m_len = (max_blocks -= ret);
4741 if (credits) {
4742 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
4743 credits);
4744 if (IS_ERR(handle)) {
4745 ret = PTR_ERR(handle);
4746 break;
4747 }
4748 }
4749 ret = ext4_map_blocks(handle, inode, &map,
4750 EXT4_GET_BLOCKS_IO_CONVERT_EXT);
4751 if (ret <= 0)
4752 ext4_warning(inode->i_sb,
4753 "inode #%lu: block %u: len %u: "
4754 "ext4_ext_map_blocks returned %d",
4755 inode->i_ino, map.m_lblk,
4756 map.m_len, ret);
4757 ret2 = ext4_mark_inode_dirty(handle, inode);
4758 if (credits) {
4759 ret3 = ext4_journal_stop(handle);
4760 if (unlikely(ret3))
4761 ret2 = ret3;
4762 }
4763
4764 if (ret <= 0 || ret2)
4765 break;
4766 }
4767 return ret > 0 ? ret2 : ret;
4768}
4769
4770int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end)
4771{
4772 int ret, err = 0;
4773 struct ext4_io_end_vec *io_end_vec;
4774
4775
4776
4777
4778
4779
4780 if (handle) {
4781 handle = ext4_journal_start_reserved(handle,
4782 EXT4_HT_EXT_CONVERT);
4783 if (IS_ERR(handle))
4784 return PTR_ERR(handle);
4785 }
4786
4787 list_for_each_entry(io_end_vec, &io_end->list_vec, list) {
4788 ret = ext4_convert_unwritten_extents(handle, io_end->inode,
4789 io_end_vec->offset,
4790 io_end_vec->size);
4791 if (ret)
4792 break;
4793 }
4794
4795 if (handle)
4796 err = ext4_journal_stop(handle);
4797
4798 return ret < 0 ? ret : err;
4799}
4800
4801static int ext4_iomap_xattr_fiemap(struct inode *inode, struct iomap *iomap)
4802{
4803 __u64 physical = 0;
4804 __u64 length = 0;
4805 int blockbits = inode->i_sb->s_blocksize_bits;
4806 int error = 0;
4807 u16 iomap_type;
4808
4809
4810 if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
4811 struct ext4_iloc iloc;
4812 int offset;
4813
4814 error = ext4_get_inode_loc(inode, &iloc);
4815 if (error)
4816 return error;
4817 physical = (__u64)iloc.bh->b_blocknr << blockbits;
4818 offset = EXT4_GOOD_OLD_INODE_SIZE +
4819 EXT4_I(inode)->i_extra_isize;
4820 physical += offset;
4821 length = EXT4_SB(inode->i_sb)->s_inode_size - offset;
4822 brelse(iloc.bh);
4823 iomap_type = IOMAP_INLINE;
4824 } else if (EXT4_I(inode)->i_file_acl) {
4825 physical = (__u64)EXT4_I(inode)->i_file_acl << blockbits;
4826 length = inode->i_sb->s_blocksize;
4827 iomap_type = IOMAP_MAPPED;
4828 } else {
4829
4830 error = -ENOENT;
4831 goto out;
4832 }
4833
4834 iomap->addr = physical;
4835 iomap->offset = 0;
4836 iomap->length = length;
4837 iomap->type = iomap_type;
4838 iomap->flags = 0;
4839out:
4840 return error;
4841}
4842
4843static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset,
4844 loff_t length, unsigned flags,
4845 struct iomap *iomap, struct iomap *srcmap)
4846{
4847 int error;
4848
4849 error = ext4_iomap_xattr_fiemap(inode, iomap);
4850 if (error == 0 && (offset >= iomap->length))
4851 error = -ENOENT;
4852 return error;
4853}
4854
4855static const struct iomap_ops ext4_iomap_xattr_ops = {
4856 .iomap_begin = ext4_iomap_xattr_begin,
4857};
4858
4859static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len)
4860{
4861 u64 maxbytes;
4862
4863 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4864 maxbytes = inode->i_sb->s_maxbytes;
4865 else
4866 maxbytes = EXT4_SB(inode->i_sb)->s_bitmap_maxbytes;
4867
4868 if (*len == 0)
4869 return -EINVAL;
4870 if (start > maxbytes)
4871 return -EFBIG;
4872
4873
4874
4875
4876 if (*len > maxbytes || (maxbytes - *len) < start)
4877 *len = maxbytes - start;
4878 return 0;
4879}
4880
4881int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4882 u64 start, u64 len)
4883{
4884 int error = 0;
4885
4886 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
4887 error = ext4_ext_precache(inode);
4888 if (error)
4889 return error;
4890 fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE;
4891 }
4892
4893
4894
4895
4896
4897
4898 error = ext4_fiemap_check_ranges(inode, start, &len);
4899 if (error)
4900 return error;
4901
4902 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
4903 fieinfo->fi_flags &= ~FIEMAP_FLAG_XATTR;
4904 return iomap_fiemap(inode, fieinfo, start, len,
4905 &ext4_iomap_xattr_ops);
4906 }
4907
4908 return iomap_fiemap(inode, fieinfo, start, len, &ext4_iomap_report_ops);
4909}
4910
4911int ext4_get_es_cache(struct inode *inode, struct fiemap_extent_info *fieinfo,
4912 __u64 start, __u64 len)
4913{
4914 ext4_lblk_t start_blk, len_blks;
4915 __u64 last_blk;
4916 int error = 0;
4917
4918 if (ext4_has_inline_data(inode)) {
4919 int has_inline;
4920
4921 down_read(&EXT4_I(inode)->xattr_sem);
4922 has_inline = ext4_has_inline_data(inode);
4923 up_read(&EXT4_I(inode)->xattr_sem);
4924 if (has_inline)
4925 return 0;
4926 }
4927
4928 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
4929 error = ext4_ext_precache(inode);
4930 if (error)
4931 return error;
4932 fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE;
4933 }
4934
4935 error = fiemap_prep(inode, fieinfo, start, &len, 0);
4936 if (error)
4937 return error;
4938
4939 error = ext4_fiemap_check_ranges(inode, start, &len);
4940 if (error)
4941 return error;
4942
4943 start_blk = start >> inode->i_sb->s_blocksize_bits;
4944 last_blk = (start + len - 1) >> inode->i_sb->s_blocksize_bits;
4945 if (last_blk >= EXT_MAX_BLOCKS)
4946 last_blk = EXT_MAX_BLOCKS-1;
4947 len_blks = ((ext4_lblk_t) last_blk) - start_blk + 1;
4948
4949
4950
4951
4952
4953 return ext4_fill_es_cache_info(inode, start_blk, len_blks, fieinfo);
4954}
4955
4956
4957
4958
4959
4960
4961
4962static int
4963ext4_access_path(handle_t *handle, struct inode *inode,
4964 struct ext4_ext_path *path)
4965{
4966 int credits, err;
4967
4968 if (!ext4_handle_valid(handle))
4969 return 0;
4970
4971
4972
4973
4974
4975
4976
4977 credits = ext4_writepage_trans_blocks(inode);
4978 err = ext4_datasem_ensure_credits(handle, inode, 7, credits, 0);
4979 if (err < 0)
4980 return err;
4981
4982 err = ext4_ext_get_access(handle, inode, path);
4983 return err;
4984}
4985
4986
4987
4988
4989
4990
4991
4992static int
4993ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift,
4994 struct inode *inode, handle_t *handle,
4995 enum SHIFT_DIRECTION SHIFT)
4996{
4997 int depth, err = 0;
4998 struct ext4_extent *ex_start, *ex_last;
4999 bool update = false;
5000 depth = path->p_depth;
5001
5002 while (depth >= 0) {
5003 if (depth == path->p_depth) {
5004 ex_start = path[depth].p_ext;
5005 if (!ex_start)
5006 return -EFSCORRUPTED;
5007
5008 ex_last = EXT_LAST_EXTENT(path[depth].p_hdr);
5009
5010 err = ext4_access_path(handle, inode, path + depth);
5011 if (err)
5012 goto out;
5013
5014 if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr))
5015 update = true;
5016
5017 while (ex_start <= ex_last) {
5018 if (SHIFT == SHIFT_LEFT) {
5019 le32_add_cpu(&ex_start->ee_block,
5020 -shift);
5021
5022 if ((ex_start >
5023 EXT_FIRST_EXTENT(path[depth].p_hdr))
5024 &&
5025 ext4_ext_try_to_merge_right(inode,
5026 path, ex_start - 1))
5027 ex_last--;
5028 else
5029 ex_start++;
5030 } else {
5031 le32_add_cpu(&ex_last->ee_block, shift);
5032 ext4_ext_try_to_merge_right(inode, path,
5033 ex_last);
5034 ex_last--;
5035 }
5036 }
5037 err = ext4_ext_dirty(handle, inode, path + depth);
5038 if (err)
5039 goto out;
5040
5041 if (--depth < 0 || !update)
5042 break;
5043 }
5044
5045
5046 err = ext4_access_path(handle, inode, path + depth);
5047 if (err)
5048 goto out;
5049
5050 if (SHIFT == SHIFT_LEFT)
5051 le32_add_cpu(&path[depth].p_idx->ei_block, -shift);
5052 else
5053 le32_add_cpu(&path[depth].p_idx->ei_block, shift);
5054 err = ext4_ext_dirty(handle, inode, path + depth);
5055 if (err)
5056 goto out;
5057
5058
5059 if (path[depth].p_idx != EXT_FIRST_INDEX(path[depth].p_hdr))
5060 break;
5061
5062 depth--;
5063 }
5064
5065out:
5066 return err;
5067}
5068
5069
5070
5071
5072
5073
5074
5075
5076static int
5077ext4_ext_shift_extents(struct inode *inode, handle_t *handle,
5078 ext4_lblk_t start, ext4_lblk_t shift,
5079 enum SHIFT_DIRECTION SHIFT)
5080{
5081 struct ext4_ext_path *path;
5082 int ret = 0, depth;
5083 struct ext4_extent *extent;
5084 ext4_lblk_t stop, *iterator, ex_start, ex_end;
5085
5086
5087 path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL,
5088 EXT4_EX_NOCACHE);
5089 if (IS_ERR(path))
5090 return PTR_ERR(path);
5091
5092 depth = path->p_depth;
5093 extent = path[depth].p_ext;
5094 if (!extent)
5095 goto out;
5096
5097 stop = le32_to_cpu(extent->ee_block);
5098
5099
5100
5101
5102
5103
5104 if (SHIFT == SHIFT_LEFT) {
5105 path = ext4_find_extent(inode, start - 1, &path,
5106 EXT4_EX_NOCACHE);
5107 if (IS_ERR(path))
5108 return PTR_ERR(path);
5109 depth = path->p_depth;
5110 extent = path[depth].p_ext;
5111 if (extent) {
5112 ex_start = le32_to_cpu(extent->ee_block);
5113 ex_end = le32_to_cpu(extent->ee_block) +
5114 ext4_ext_get_actual_len(extent);
5115 } else {
5116 ex_start = 0;
5117 ex_end = 0;
5118 }
5119
5120 if ((start == ex_start && shift > ex_start) ||
5121 (shift > start - ex_end)) {
5122 ret = -EINVAL;
5123 goto out;
5124 }
5125 } else {
5126 if (shift > EXT_MAX_BLOCKS -
5127 (stop + ext4_ext_get_actual_len(extent))) {
5128 ret = -EINVAL;
5129 goto out;
5130 }
5131 }
5132
5133
5134
5135
5136
5137
5138 if (SHIFT == SHIFT_LEFT)
5139 iterator = &start;
5140 else
5141 iterator = &stop;
5142
5143
5144
5145
5146
5147
5148 while (iterator && start <= stop) {
5149 path = ext4_find_extent(inode, *iterator, &path,
5150 EXT4_EX_NOCACHE);
5151 if (IS_ERR(path))
5152 return PTR_ERR(path);
5153 depth = path->p_depth;
5154 extent = path[depth].p_ext;
5155 if (!extent) {
5156 EXT4_ERROR_INODE(inode, "unexpected hole at %lu",
5157 (unsigned long) *iterator);
5158 return -EFSCORRUPTED;
5159 }
5160 if (SHIFT == SHIFT_LEFT && *iterator >
5161 le32_to_cpu(extent->ee_block)) {
5162
5163 if (extent < EXT_LAST_EXTENT(path[depth].p_hdr)) {
5164 path[depth].p_ext++;
5165 } else {
5166 *iterator = ext4_ext_next_allocated_block(path);
5167 continue;
5168 }
5169 }
5170
5171 if (SHIFT == SHIFT_LEFT) {
5172 extent = EXT_LAST_EXTENT(path[depth].p_hdr);
5173 *iterator = le32_to_cpu(extent->ee_block) +
5174 ext4_ext_get_actual_len(extent);
5175 } else {
5176 extent = EXT_FIRST_EXTENT(path[depth].p_hdr);
5177 if (le32_to_cpu(extent->ee_block) > 0)
5178 *iterator = le32_to_cpu(extent->ee_block) - 1;
5179 else
5180
5181 iterator = NULL;
5182
5183 while (le32_to_cpu(extent->ee_block) < start)
5184 extent++;
5185 path[depth].p_ext = extent;
5186 }
5187 ret = ext4_ext_shift_path_extents(path, shift, inode,
5188 handle, SHIFT);
5189 if (ret)
5190 break;
5191 }
5192out:
5193 ext4_ext_drop_refs(path);
5194 kfree(path);
5195 return ret;
5196}
5197
5198
5199
5200
5201
5202
5203static int ext4_collapse_range(struct inode *inode, loff_t offset, loff_t len)
5204{
5205 struct super_block *sb = inode->i_sb;
5206 ext4_lblk_t punch_start, punch_stop;
5207 handle_t *handle;
5208 unsigned int credits;
5209 loff_t new_size, ioffset;
5210 int ret;
5211
5212
5213
5214
5215
5216
5217 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5218 return -EOPNOTSUPP;
5219
5220
5221 if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb)))
5222 return -EINVAL;
5223
5224 trace_ext4_collapse_range(inode, offset, len);
5225
5226 punch_start = offset >> EXT4_BLOCK_SIZE_BITS(sb);
5227 punch_stop = (offset + len) >> EXT4_BLOCK_SIZE_BITS(sb);
5228
5229
5230 if (ext4_should_journal_data(inode)) {
5231 ret = ext4_force_commit(inode->i_sb);
5232 if (ret)
5233 return ret;
5234 }
5235
5236 inode_lock(inode);
5237
5238
5239
5240
5241 if (offset + len >= inode->i_size) {
5242 ret = -EINVAL;
5243 goto out_mutex;
5244 }
5245
5246
5247 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
5248 ret = -EOPNOTSUPP;
5249 goto out_mutex;
5250 }
5251
5252
5253 inode_dio_wait(inode);
5254
5255
5256
5257
5258
5259 down_write(&EXT4_I(inode)->i_mmap_sem);
5260
5261 ret = ext4_break_layouts(inode);
5262 if (ret)
5263 goto out_mmap;
5264
5265
5266
5267
5268
5269 ioffset = round_down(offset, PAGE_SIZE);
5270
5271
5272
5273
5274 ret = filemap_write_and_wait_range(inode->i_mapping, ioffset, offset);
5275 if (ret)
5276 goto out_mmap;
5277
5278
5279
5280
5281
5282 ret = filemap_write_and_wait_range(inode->i_mapping, offset + len,
5283 LLONG_MAX);
5284 if (ret)
5285 goto out_mmap;
5286 truncate_pagecache(inode, ioffset);
5287
5288 credits = ext4_writepage_trans_blocks(inode);
5289 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5290 if (IS_ERR(handle)) {
5291 ret = PTR_ERR(handle);
5292 goto out_mmap;
5293 }
5294
5295 down_write(&EXT4_I(inode)->i_data_sem);
5296 ext4_discard_preallocations(inode, 0);
5297
5298 ret = ext4_es_remove_extent(inode, punch_start,
5299 EXT_MAX_BLOCKS - punch_start);
5300 if (ret) {
5301 up_write(&EXT4_I(inode)->i_data_sem);
5302 goto out_stop;
5303 }
5304
5305 ret = ext4_ext_remove_space(inode, punch_start, punch_stop - 1);
5306 if (ret) {
5307 up_write(&EXT4_I(inode)->i_data_sem);
5308 goto out_stop;
5309 }
5310 ext4_discard_preallocations(inode, 0);
5311
5312 ret = ext4_ext_shift_extents(inode, handle, punch_stop,
5313 punch_stop - punch_start, SHIFT_LEFT);
5314 if (ret) {
5315 up_write(&EXT4_I(inode)->i_data_sem);
5316 goto out_stop;
5317 }
5318
5319 new_size = inode->i_size - len;
5320 i_size_write(inode, new_size);
5321 EXT4_I(inode)->i_disksize = new_size;
5322
5323 up_write(&EXT4_I(inode)->i_data_sem);
5324 if (IS_SYNC(inode))
5325 ext4_handle_sync(handle);
5326 inode->i_mtime = inode->i_ctime = current_time(inode);
5327 ret = ext4_mark_inode_dirty(handle, inode);
5328 ext4_update_inode_fsync_trans(handle, inode, 1);
5329
5330out_stop:
5331 ext4_journal_stop(handle);
5332out_mmap:
5333 up_write(&EXT4_I(inode)->i_mmap_sem);
5334out_mutex:
5335 inode_unlock(inode);
5336 return ret;
5337}
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347static int ext4_insert_range(struct inode *inode, loff_t offset, loff_t len)
5348{
5349 struct super_block *sb = inode->i_sb;
5350 handle_t *handle;
5351 struct ext4_ext_path *path;
5352 struct ext4_extent *extent;
5353 ext4_lblk_t offset_lblk, len_lblk, ee_start_lblk = 0;
5354 unsigned int credits, ee_len;
5355 int ret = 0, depth, split_flag = 0;
5356 loff_t ioffset;
5357
5358
5359
5360
5361
5362
5363 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5364 return -EOPNOTSUPP;
5365
5366
5367 if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb)))
5368 return -EINVAL;
5369
5370 trace_ext4_insert_range(inode, offset, len);
5371
5372 offset_lblk = offset >> EXT4_BLOCK_SIZE_BITS(sb);
5373 len_lblk = len >> EXT4_BLOCK_SIZE_BITS(sb);
5374
5375
5376 if (ext4_should_journal_data(inode)) {
5377 ret = ext4_force_commit(inode->i_sb);
5378 if (ret)
5379 return ret;
5380 }
5381
5382 inode_lock(inode);
5383
5384 if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
5385 ret = -EOPNOTSUPP;
5386 goto out_mutex;
5387 }
5388
5389
5390 if (len > inode->i_sb->s_maxbytes - inode->i_size) {
5391 ret = -EFBIG;
5392 goto out_mutex;
5393 }
5394
5395
5396 if (offset >= inode->i_size) {
5397 ret = -EINVAL;
5398 goto out_mutex;
5399 }
5400
5401
5402 inode_dio_wait(inode);
5403
5404
5405
5406
5407
5408 down_write(&EXT4_I(inode)->i_mmap_sem);
5409
5410 ret = ext4_break_layouts(inode);
5411 if (ret)
5412 goto out_mmap;
5413
5414
5415
5416
5417
5418 ioffset = round_down(offset, PAGE_SIZE);
5419
5420 ret = filemap_write_and_wait_range(inode->i_mapping, ioffset,
5421 LLONG_MAX);
5422 if (ret)
5423 goto out_mmap;
5424 truncate_pagecache(inode, ioffset);
5425
5426 credits = ext4_writepage_trans_blocks(inode);
5427 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
5428 if (IS_ERR(handle)) {
5429 ret = PTR_ERR(handle);
5430 goto out_mmap;
5431 }
5432
5433
5434 inode->i_size += len;
5435 EXT4_I(inode)->i_disksize += len;
5436 inode->i_mtime = inode->i_ctime = current_time(inode);
5437 ret = ext4_mark_inode_dirty(handle, inode);
5438 if (ret)
5439 goto out_stop;
5440
5441 down_write(&EXT4_I(inode)->i_data_sem);
5442 ext4_discard_preallocations(inode, 0);
5443
5444 path = ext4_find_extent(inode, offset_lblk, NULL, 0);
5445 if (IS_ERR(path)) {
5446 up_write(&EXT4_I(inode)->i_data_sem);
5447 goto out_stop;
5448 }
5449
5450 depth = ext_depth(inode);
5451 extent = path[depth].p_ext;
5452 if (extent) {
5453 ee_start_lblk = le32_to_cpu(extent->ee_block);
5454 ee_len = ext4_ext_get_actual_len(extent);
5455
5456
5457
5458
5459
5460 if ((offset_lblk > ee_start_lblk) &&
5461 (offset_lblk < (ee_start_lblk + ee_len))) {
5462 if (ext4_ext_is_unwritten(extent))
5463 split_flag = EXT4_EXT_MARK_UNWRIT1 |
5464 EXT4_EXT_MARK_UNWRIT2;
5465 ret = ext4_split_extent_at(handle, inode, &path,
5466 offset_lblk, split_flag,
5467 EXT4_EX_NOCACHE |
5468 EXT4_GET_BLOCKS_PRE_IO |
5469 EXT4_GET_BLOCKS_METADATA_NOFAIL);
5470 }
5471
5472 ext4_ext_drop_refs(path);
5473 kfree(path);
5474 if (ret < 0) {
5475 up_write(&EXT4_I(inode)->i_data_sem);
5476 goto out_stop;
5477 }
5478 } else {
5479 ext4_ext_drop_refs(path);
5480 kfree(path);
5481 }
5482
5483 ret = ext4_es_remove_extent(inode, offset_lblk,
5484 EXT_MAX_BLOCKS - offset_lblk);
5485 if (ret) {
5486 up_write(&EXT4_I(inode)->i_data_sem);
5487 goto out_stop;
5488 }
5489
5490
5491
5492
5493
5494 ret = ext4_ext_shift_extents(inode, handle,
5495 ee_start_lblk > offset_lblk ? ee_start_lblk : offset_lblk,
5496 len_lblk, SHIFT_RIGHT);
5497
5498 up_write(&EXT4_I(inode)->i_data_sem);
5499 if (IS_SYNC(inode))
5500 ext4_handle_sync(handle);
5501 if (ret >= 0)
5502 ext4_update_inode_fsync_trans(handle, inode, 1);
5503
5504out_stop:
5505 ext4_journal_stop(handle);
5506out_mmap:
5507 up_write(&EXT4_I(inode)->i_mmap_sem);
5508out_mutex:
5509 inode_unlock(inode);
5510 return ret;
5511}
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533int
5534ext4_swap_extents(handle_t *handle, struct inode *inode1,
5535 struct inode *inode2, ext4_lblk_t lblk1, ext4_lblk_t lblk2,
5536 ext4_lblk_t count, int unwritten, int *erp)
5537{
5538 struct ext4_ext_path *path1 = NULL;
5539 struct ext4_ext_path *path2 = NULL;
5540 int replaced_count = 0;
5541
5542 BUG_ON(!rwsem_is_locked(&EXT4_I(inode1)->i_data_sem));
5543 BUG_ON(!rwsem_is_locked(&EXT4_I(inode2)->i_data_sem));
5544 BUG_ON(!inode_is_locked(inode1));
5545 BUG_ON(!inode_is_locked(inode2));
5546
5547 *erp = ext4_es_remove_extent(inode1, lblk1, count);
5548 if (unlikely(*erp))
5549 return 0;
5550 *erp = ext4_es_remove_extent(inode2, lblk2, count);
5551 if (unlikely(*erp))
5552 return 0;
5553
5554 while (count) {
5555 struct ext4_extent *ex1, *ex2, tmp_ex;
5556 ext4_lblk_t e1_blk, e2_blk;
5557 int e1_len, e2_len, len;
5558 int split = 0;
5559
5560 path1 = ext4_find_extent(inode1, lblk1, NULL, EXT4_EX_NOCACHE);
5561 if (IS_ERR(path1)) {
5562 *erp = PTR_ERR(path1);
5563 path1 = NULL;
5564 finish:
5565 count = 0;
5566 goto repeat;
5567 }
5568 path2 = ext4_find_extent(inode2, lblk2, NULL, EXT4_EX_NOCACHE);
5569 if (IS_ERR(path2)) {
5570 *erp = PTR_ERR(path2);
5571 path2 = NULL;
5572 goto finish;
5573 }
5574 ex1 = path1[path1->p_depth].p_ext;
5575 ex2 = path2[path2->p_depth].p_ext;
5576
5577 if (unlikely(!ex2 || !ex1))
5578 goto finish;
5579
5580 e1_blk = le32_to_cpu(ex1->ee_block);
5581 e2_blk = le32_to_cpu(ex2->ee_block);
5582 e1_len = ext4_ext_get_actual_len(ex1);
5583 e2_len = ext4_ext_get_actual_len(ex2);
5584
5585
5586 if (!in_range(lblk1, e1_blk, e1_len) ||
5587 !in_range(lblk2, e2_blk, e2_len)) {
5588 ext4_lblk_t next1, next2;
5589
5590
5591 next1 = ext4_ext_next_allocated_block(path1);
5592 next2 = ext4_ext_next_allocated_block(path2);
5593
5594 if (e1_blk > lblk1)
5595 next1 = e1_blk;
5596 if (e2_blk > lblk2)
5597 next2 = e2_blk;
5598
5599 if (next1 == EXT_MAX_BLOCKS || next2 == EXT_MAX_BLOCKS)
5600 goto finish;
5601
5602 len = next1 - lblk1;
5603 if (len < next2 - lblk2)
5604 len = next2 - lblk2;
5605 if (len > count)
5606 len = count;
5607 lblk1 += len;
5608 lblk2 += len;
5609 count -= len;
5610 goto repeat;
5611 }
5612
5613
5614 if (e1_blk < lblk1) {
5615 split = 1;
5616 *erp = ext4_force_split_extent_at(handle, inode1,
5617 &path1, lblk1, 0);
5618 if (unlikely(*erp))
5619 goto finish;
5620 }
5621 if (e2_blk < lblk2) {
5622 split = 1;
5623 *erp = ext4_force_split_extent_at(handle, inode2,
5624 &path2, lblk2, 0);
5625 if (unlikely(*erp))
5626 goto finish;
5627 }
5628
5629
5630 if (split)
5631 goto repeat;
5632
5633
5634 len = count;
5635 if (len > e1_blk + e1_len - lblk1)
5636 len = e1_blk + e1_len - lblk1;
5637 if (len > e2_blk + e2_len - lblk2)
5638 len = e2_blk + e2_len - lblk2;
5639
5640 if (len != e1_len) {
5641 split = 1;
5642 *erp = ext4_force_split_extent_at(handle, inode1,
5643 &path1, lblk1 + len, 0);
5644 if (unlikely(*erp))
5645 goto finish;
5646 }
5647 if (len != e2_len) {
5648 split = 1;
5649 *erp = ext4_force_split_extent_at(handle, inode2,
5650 &path2, lblk2 + len, 0);
5651 if (*erp)
5652 goto finish;
5653 }
5654
5655
5656 if (split)
5657 goto repeat;
5658
5659 BUG_ON(e2_len != e1_len);
5660 *erp = ext4_ext_get_access(handle, inode1, path1 + path1->p_depth);
5661 if (unlikely(*erp))
5662 goto finish;
5663 *erp = ext4_ext_get_access(handle, inode2, path2 + path2->p_depth);
5664 if (unlikely(*erp))
5665 goto finish;
5666
5667
5668 tmp_ex = *ex1;
5669 ext4_ext_store_pblock(ex1, ext4_ext_pblock(ex2));
5670 ext4_ext_store_pblock(ex2, ext4_ext_pblock(&tmp_ex));
5671 ex1->ee_len = cpu_to_le16(e2_len);
5672 ex2->ee_len = cpu_to_le16(e1_len);
5673 if (unwritten)
5674 ext4_ext_mark_unwritten(ex2);
5675 if (ext4_ext_is_unwritten(&tmp_ex))
5676 ext4_ext_mark_unwritten(ex1);
5677
5678 ext4_ext_try_to_merge(handle, inode2, path2, ex2);
5679 ext4_ext_try_to_merge(handle, inode1, path1, ex1);
5680 *erp = ext4_ext_dirty(handle, inode2, path2 +
5681 path2->p_depth);
5682 if (unlikely(*erp))
5683 goto finish;
5684 *erp = ext4_ext_dirty(handle, inode1, path1 +
5685 path1->p_depth);
5686
5687
5688
5689
5690
5691
5692 if (unlikely(*erp))
5693 goto finish;
5694 lblk1 += len;
5695 lblk2 += len;
5696 replaced_count += len;
5697 count -= len;
5698
5699 repeat:
5700 ext4_ext_drop_refs(path1);
5701 kfree(path1);
5702 ext4_ext_drop_refs(path2);
5703 kfree(path2);
5704 path1 = path2 = NULL;
5705 }
5706 return replaced_count;
5707}
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721int ext4_clu_mapped(struct inode *inode, ext4_lblk_t lclu)
5722{
5723 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5724 struct ext4_ext_path *path;
5725 int depth, mapped = 0, err = 0;
5726 struct ext4_extent *extent;
5727 ext4_lblk_t first_lblk, first_lclu, last_lclu;
5728
5729
5730 path = ext4_find_extent(inode, EXT4_C2B(sbi, lclu), NULL, 0);
5731 if (IS_ERR(path)) {
5732 err = PTR_ERR(path);
5733 path = NULL;
5734 goto out;
5735 }
5736
5737 depth = ext_depth(inode);
5738
5739
5740
5741
5742
5743
5744 if (unlikely(path[depth].p_ext == NULL && depth != 0)) {
5745 EXT4_ERROR_INODE(inode,
5746 "bad extent address - lblock: %lu, depth: %d, pblock: %lld",
5747 (unsigned long) EXT4_C2B(sbi, lclu),
5748 depth, path[depth].p_block);
5749 err = -EFSCORRUPTED;
5750 goto out;
5751 }
5752
5753 extent = path[depth].p_ext;
5754
5755
5756 if (extent == NULL)
5757 goto out;
5758
5759 first_lblk = le32_to_cpu(extent->ee_block);
5760 first_lclu = EXT4_B2C(sbi, first_lblk);
5761
5762
5763
5764
5765
5766
5767
5768 if (lclu >= first_lclu) {
5769 last_lclu = EXT4_B2C(sbi, first_lblk +
5770 ext4_ext_get_actual_len(extent) - 1);
5771 if (lclu <= last_lclu) {
5772 mapped = 1;
5773 } else {
5774 first_lblk = ext4_ext_next_allocated_block(path);
5775 first_lclu = EXT4_B2C(sbi, first_lblk);
5776 if (lclu == first_lclu)
5777 mapped = 1;
5778 }
5779 }
5780
5781out:
5782 ext4_ext_drop_refs(path);
5783 kfree(path);
5784
5785 return err ? err : mapped;
5786}
5787