1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#include <linux/fs.h>
23#include <linux/time.h>
24#include <linux/highuid.h>
25#include <linux/pagemap.h>
26#include <linux/dax.h>
27#include <linux/quotaops.h>
28#include <linux/string.h>
29#include <linux/buffer_head.h>
30#include <linux/writeback.h>
31#include <linux/pagevec.h>
32#include <linux/mpage.h>
33#include <linux/namei.h>
34#include <linux/uio.h>
35#include <linux/bio.h>
36#include <linux/workqueue.h>
37#include <linux/kernel.h>
38#include <linux/printk.h>
39#include <linux/slab.h>
40#include <linux/bitops.h>
41#include <linux/iomap.h>
42#include <linux/iversion.h>
43
44#include "ext4_jbd2.h"
45#include "xattr.h"
46#include "acl.h"
47#include "truncate.h"
48
49#include <trace/events/ext4.h>
50
51#define MPAGE_DA_EXTENT_TAIL 0x01
52
53static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
54 struct ext4_inode_info *ei)
55{
56 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
57 __u32 csum;
58 __u16 dummy_csum = 0;
59 int offset = offsetof(struct ext4_inode, i_checksum_lo);
60 unsigned int csum_size = sizeof(dummy_csum);
61
62 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);
63 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);
64 offset += csum_size;
65 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
66 EXT4_GOOD_OLD_INODE_SIZE - offset);
67
68 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
69 offset = offsetof(struct ext4_inode, i_checksum_hi);
70 csum = ext4_chksum(sbi, csum, (__u8 *)raw +
71 EXT4_GOOD_OLD_INODE_SIZE,
72 offset - EXT4_GOOD_OLD_INODE_SIZE);
73 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
74 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
75 csum_size);
76 offset += csum_size;
77 }
78 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
79 EXT4_INODE_SIZE(inode->i_sb) - offset);
80 }
81
82 return csum;
83}
84
85static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
86 struct ext4_inode_info *ei)
87{
88 __u32 provided, calculated;
89
90 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
91 cpu_to_le32(EXT4_OS_LINUX) ||
92 !ext4_has_metadata_csum(inode->i_sb))
93 return 1;
94
95 provided = le16_to_cpu(raw->i_checksum_lo);
96 calculated = ext4_inode_csum(inode, raw, ei);
97 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
98 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
99 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
100 else
101 calculated &= 0xFFFF;
102
103 return provided == calculated;
104}
105
106static void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
107 struct ext4_inode_info *ei)
108{
109 __u32 csum;
110
111 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
112 cpu_to_le32(EXT4_OS_LINUX) ||
113 !ext4_has_metadata_csum(inode->i_sb))
114 return;
115
116 csum = ext4_inode_csum(inode, raw, ei);
117 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
118 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
119 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
120 raw->i_checksum_hi = cpu_to_le16(csum >> 16);
121}
122
123static inline int ext4_begin_ordered_truncate(struct inode *inode,
124 loff_t new_size)
125{
126 trace_ext4_begin_ordered_truncate(inode, new_size);
127
128
129
130
131
132
133 if (!EXT4_I(inode)->jinode)
134 return 0;
135 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
136 EXT4_I(inode)->jinode,
137 new_size);
138}
139
140static void ext4_invalidatepage(struct page *page, unsigned int offset,
141 unsigned int length);
142static int __ext4_journalled_writepage(struct page *page, unsigned int len);
143static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh);
144static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
145 int pextents);
146
147
148
149
150
151int ext4_inode_is_fast_symlink(struct inode *inode)
152{
153 if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
154 int ea_blocks = EXT4_I(inode)->i_file_acl ?
155 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
156
157 if (ext4_has_inline_data(inode))
158 return 0;
159
160 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
161 }
162 return S_ISLNK(inode->i_mode) && inode->i_size &&
163 (inode->i_size < EXT4_N_BLOCKS * 4);
164}
165
166
167
168
169
170
171int ext4_truncate_restart_trans(handle_t *handle, struct inode *inode,
172 int nblocks)
173{
174 int ret;
175
176
177
178
179
180
181
182 BUG_ON(EXT4_JOURNAL(inode) == NULL);
183 jbd_debug(2, "restarting handle %p\n", handle);
184 up_write(&EXT4_I(inode)->i_data_sem);
185 ret = ext4_journal_restart(handle, nblocks);
186 down_write(&EXT4_I(inode)->i_data_sem);
187 ext4_discard_preallocations(inode);
188
189 return ret;
190}
191
192
193
194
195void ext4_evict_inode(struct inode *inode)
196{
197 handle_t *handle;
198 int err;
199 int extra_credits = 3;
200 struct ext4_xattr_inode_array *ea_inode_array = NULL;
201
202 trace_ext4_evict_inode(inode);
203
204 if (inode->i_nlink) {
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223 if (inode->i_ino != EXT4_JOURNAL_INO &&
224 ext4_should_journal_data(inode) &&
225 (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode)) &&
226 inode->i_data.nrpages) {
227 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
228 tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;
229
230 jbd2_complete_transaction(journal, commit_tid);
231 filemap_write_and_wait(&inode->i_data);
232 }
233 truncate_inode_pages_final(&inode->i_data);
234
235 goto no_delete;
236 }
237
238 if (is_bad_inode(inode))
239 goto no_delete;
240 dquot_initialize(inode);
241
242 if (ext4_should_order_data(inode))
243 ext4_begin_ordered_truncate(inode, 0);
244 truncate_inode_pages_final(&inode->i_data);
245
246
247
248
249
250 sb_start_intwrite(inode->i_sb);
251
252 if (!IS_NOQUOTA(inode))
253 extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
254
255 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
256 ext4_blocks_for_truncate(inode)+extra_credits);
257 if (IS_ERR(handle)) {
258 ext4_std_error(inode->i_sb, PTR_ERR(handle));
259
260
261
262
263
264 ext4_orphan_del(NULL, inode);
265 sb_end_intwrite(inode->i_sb);
266 goto no_delete;
267 }
268
269 if (IS_SYNC(inode))
270 ext4_handle_sync(handle);
271
272
273
274
275
276
277
278
279 if (ext4_inode_is_fast_symlink(inode))
280 memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
281 inode->i_size = 0;
282 err = ext4_mark_inode_dirty(handle, inode);
283 if (err) {
284 ext4_warning(inode->i_sb,
285 "couldn't mark inode dirty (err %d)", err);
286 goto stop_handle;
287 }
288 if (inode->i_blocks) {
289 err = ext4_truncate(inode);
290 if (err) {
291 ext4_error(inode->i_sb,
292 "couldn't truncate inode %lu (err %d)",
293 inode->i_ino, err);
294 goto stop_handle;
295 }
296 }
297
298
299 err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
300 extra_credits);
301 if (err) {
302 ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
303stop_handle:
304 ext4_journal_stop(handle);
305 ext4_orphan_del(NULL, inode);
306 sb_end_intwrite(inode->i_sb);
307 ext4_xattr_inode_array_free(ea_inode_array);
308 goto no_delete;
309 }
310
311
312
313
314
315
316
317
318
319 ext4_orphan_del(handle, inode);
320 EXT4_I(inode)->i_dtime = get_seconds();
321
322
323
324
325
326
327
328
329 if (ext4_mark_inode_dirty(handle, inode))
330
331 ext4_clear_inode(inode);
332 else
333 ext4_free_inode(handle, inode);
334 ext4_journal_stop(handle);
335 sb_end_intwrite(inode->i_sb);
336 ext4_xattr_inode_array_free(ea_inode_array);
337 return;
338no_delete:
339 ext4_clear_inode(inode);
340}
341
342#ifdef CONFIG_QUOTA
343qsize_t *ext4_get_reserved_space(struct inode *inode)
344{
345 return &EXT4_I(inode)->i_reserved_quota;
346}
347#endif
348
349
350
351
352
353void ext4_da_update_reserve_space(struct inode *inode,
354 int used, int quota_claim)
355{
356 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
357 struct ext4_inode_info *ei = EXT4_I(inode);
358
359 spin_lock(&ei->i_block_reservation_lock);
360 trace_ext4_da_update_reserve_space(inode, used, quota_claim);
361 if (unlikely(used > ei->i_reserved_data_blocks)) {
362 ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
363 "with only %d reserved data blocks",
364 __func__, inode->i_ino, used,
365 ei->i_reserved_data_blocks);
366 WARN_ON(1);
367 used = ei->i_reserved_data_blocks;
368 }
369
370
371 ei->i_reserved_data_blocks -= used;
372 percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
373
374 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
375
376
377 if (quota_claim)
378 dquot_claim_block(inode, EXT4_C2B(sbi, used));
379 else {
380
381
382
383
384
385 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
386 }
387
388
389
390
391
392
393 if ((ei->i_reserved_data_blocks == 0) &&
394 (atomic_read(&inode->i_writecount) == 0))
395 ext4_discard_preallocations(inode);
396}
397
398static int __check_block_validity(struct inode *inode, const char *func,
399 unsigned int line,
400 struct ext4_map_blocks *map)
401{
402 if (!ext4_data_block_valid(EXT4_SB(inode->i_sb), map->m_pblk,
403 map->m_len)) {
404 ext4_error_inode(inode, func, line, map->m_pblk,
405 "lblock %lu mapped to illegal pblock "
406 "(length %d)", (unsigned long) map->m_lblk,
407 map->m_len);
408 return -EFSCORRUPTED;
409 }
410 return 0;
411}
412
413int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
414 ext4_lblk_t len)
415{
416 int ret;
417
418 if (ext4_encrypted_inode(inode))
419 return fscrypt_zeroout_range(inode, lblk, pblk, len);
420
421 ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
422 if (ret > 0)
423 ret = 0;
424
425 return ret;
426}
427
428#define check_block_validity(inode, map) \
429 __check_block_validity((inode), __func__, __LINE__, (map))
430
431#ifdef ES_AGGRESSIVE_TEST
432static void ext4_map_blocks_es_recheck(handle_t *handle,
433 struct inode *inode,
434 struct ext4_map_blocks *es_map,
435 struct ext4_map_blocks *map,
436 int flags)
437{
438 int retval;
439
440 map->m_flags = 0;
441
442
443
444
445
446
447
448 down_read(&EXT4_I(inode)->i_data_sem);
449 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
450 retval = ext4_ext_map_blocks(handle, inode, map, flags &
451 EXT4_GET_BLOCKS_KEEP_SIZE);
452 } else {
453 retval = ext4_ind_map_blocks(handle, inode, map, flags &
454 EXT4_GET_BLOCKS_KEEP_SIZE);
455 }
456 up_read((&EXT4_I(inode)->i_data_sem));
457
458
459
460
461
462 if (es_map->m_lblk != map->m_lblk ||
463 es_map->m_flags != map->m_flags ||
464 es_map->m_pblk != map->m_pblk) {
465 printk("ES cache assertion failed for inode: %lu "
466 "es_cached ex [%d/%d/%llu/%x] != "
467 "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
468 inode->i_ino, es_map->m_lblk, es_map->m_len,
469 es_map->m_pblk, es_map->m_flags, map->m_lblk,
470 map->m_len, map->m_pblk, map->m_flags,
471 retval, flags);
472 }
473}
474#endif
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498int ext4_map_blocks(handle_t *handle, struct inode *inode,
499 struct ext4_map_blocks *map, int flags)
500{
501 struct extent_status es;
502 int retval;
503 int ret = 0;
504#ifdef ES_AGGRESSIVE_TEST
505 struct ext4_map_blocks orig_map;
506
507 memcpy(&orig_map, map, sizeof(*map));
508#endif
509
510 map->m_flags = 0;
511 ext_debug("ext4_map_blocks(): inode %lu, flag %d, max_blocks %u,"
512 "logical block %lu\n", inode->i_ino, flags, map->m_len,
513 (unsigned long) map->m_lblk);
514
515
516
517
518 if (unlikely(map->m_len > INT_MAX))
519 map->m_len = INT_MAX;
520
521
522 if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
523 return -EFSCORRUPTED;
524
525
526 if (ext4_es_lookup_extent(inode, map->m_lblk, &es)) {
527 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
528 map->m_pblk = ext4_es_pblock(&es) +
529 map->m_lblk - es.es_lblk;
530 map->m_flags |= ext4_es_is_written(&es) ?
531 EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
532 retval = es.es_len - (map->m_lblk - es.es_lblk);
533 if (retval > map->m_len)
534 retval = map->m_len;
535 map->m_len = retval;
536 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
537 map->m_pblk = 0;
538 retval = es.es_len - (map->m_lblk - es.es_lblk);
539 if (retval > map->m_len)
540 retval = map->m_len;
541 map->m_len = retval;
542 retval = 0;
543 } else {
544 BUG_ON(1);
545 }
546#ifdef ES_AGGRESSIVE_TEST
547 ext4_map_blocks_es_recheck(handle, inode, map,
548 &orig_map, flags);
549#endif
550 goto found;
551 }
552
553
554
555
556
557 down_read(&EXT4_I(inode)->i_data_sem);
558 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
559 retval = ext4_ext_map_blocks(handle, inode, map, flags &
560 EXT4_GET_BLOCKS_KEEP_SIZE);
561 } else {
562 retval = ext4_ind_map_blocks(handle, inode, map, flags &
563 EXT4_GET_BLOCKS_KEEP_SIZE);
564 }
565 if (retval > 0) {
566 unsigned int status;
567
568 if (unlikely(retval != map->m_len)) {
569 ext4_warning(inode->i_sb,
570 "ES len assertion failed for inode "
571 "%lu: retval %d != map->m_len %d",
572 inode->i_ino, retval, map->m_len);
573 WARN_ON(1);
574 }
575
576 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
577 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
578 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
579 !(status & EXTENT_STATUS_WRITTEN) &&
580 ext4_find_delalloc_range(inode, map->m_lblk,
581 map->m_lblk + map->m_len - 1))
582 status |= EXTENT_STATUS_DELAYED;
583 ret = ext4_es_insert_extent(inode, map->m_lblk,
584 map->m_len, map->m_pblk, status);
585 if (ret < 0)
586 retval = ret;
587 }
588 up_read((&EXT4_I(inode)->i_data_sem));
589
590found:
591 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
592 ret = check_block_validity(inode, map);
593 if (ret != 0)
594 return ret;
595 }
596
597
598 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
599 return retval;
600
601
602
603
604
605
606
607
608 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
609
610
611
612
613
614 if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
615 return retval;
616
617
618
619
620
621 map->m_flags &= ~EXT4_MAP_FLAGS;
622
623
624
625
626
627
628
629 down_write(&EXT4_I(inode)->i_data_sem);
630
631
632
633
634
635 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
636 retval = ext4_ext_map_blocks(handle, inode, map, flags);
637 } else {
638 retval = ext4_ind_map_blocks(handle, inode, map, flags);
639
640 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
641
642
643
644
645
646 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
647 }
648
649
650
651
652
653
654
655 if ((retval > 0) &&
656 (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE))
657 ext4_da_update_reserve_space(inode, retval, 1);
658 }
659
660 if (retval > 0) {
661 unsigned int status;
662
663 if (unlikely(retval != map->m_len)) {
664 ext4_warning(inode->i_sb,
665 "ES len assertion failed for inode "
666 "%lu: retval %d != map->m_len %d",
667 inode->i_ino, retval, map->m_len);
668 WARN_ON(1);
669 }
670
671
672
673
674
675
676
677
678 if (flags & EXT4_GET_BLOCKS_ZERO &&
679 map->m_flags & EXT4_MAP_MAPPED &&
680 map->m_flags & EXT4_MAP_NEW) {
681 clean_bdev_aliases(inode->i_sb->s_bdev, map->m_pblk,
682 map->m_len);
683 ret = ext4_issue_zeroout(inode, map->m_lblk,
684 map->m_pblk, map->m_len);
685 if (ret) {
686 retval = ret;
687 goto out_sem;
688 }
689 }
690
691
692
693
694
695 if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
696 ext4_es_lookup_extent(inode, map->m_lblk, &es)) {
697 if (ext4_es_is_written(&es))
698 goto out_sem;
699 }
700 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
701 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
702 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
703 !(status & EXTENT_STATUS_WRITTEN) &&
704 ext4_find_delalloc_range(inode, map->m_lblk,
705 map->m_lblk + map->m_len - 1))
706 status |= EXTENT_STATUS_DELAYED;
707 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
708 map->m_pblk, status);
709 if (ret < 0) {
710 retval = ret;
711 goto out_sem;
712 }
713 }
714
715out_sem:
716 up_write((&EXT4_I(inode)->i_data_sem));
717 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
718 ret = check_block_validity(inode, map);
719 if (ret != 0)
720 return ret;
721
722
723
724
725
726
727 if (map->m_flags & EXT4_MAP_NEW &&
728 !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
729 !(flags & EXT4_GET_BLOCKS_ZERO) &&
730 !ext4_is_quota_file(inode) &&
731 ext4_should_order_data(inode)) {
732 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
733 ret = ext4_jbd2_inode_add_wait(handle, inode);
734 else
735 ret = ext4_jbd2_inode_add_write(handle, inode);
736 if (ret)
737 return ret;
738 }
739 }
740 return retval;
741}
742
743
744
745
746
747static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
748{
749 unsigned long old_state;
750 unsigned long new_state;
751
752 flags &= EXT4_MAP_FLAGS;
753
754
755 if (!bh->b_page) {
756 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
757 return;
758 }
759
760
761
762
763
764 do {
765 old_state = READ_ONCE(bh->b_state);
766 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
767 } while (unlikely(
768 cmpxchg(&bh->b_state, old_state, new_state) != old_state));
769}
770
771static int _ext4_get_block(struct inode *inode, sector_t iblock,
772 struct buffer_head *bh, int flags)
773{
774 struct ext4_map_blocks map;
775 int ret = 0;
776
777 if (ext4_has_inline_data(inode))
778 return -ERANGE;
779
780 map.m_lblk = iblock;
781 map.m_len = bh->b_size >> inode->i_blkbits;
782
783 ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
784 flags);
785 if (ret > 0) {
786 map_bh(bh, inode->i_sb, map.m_pblk);
787 ext4_update_bh_state(bh, map.m_flags);
788 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
789 ret = 0;
790 } else if (ret == 0) {
791
792 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
793 }
794 return ret;
795}
796
797int ext4_get_block(struct inode *inode, sector_t iblock,
798 struct buffer_head *bh, int create)
799{
800 return _ext4_get_block(inode, iblock, bh,
801 create ? EXT4_GET_BLOCKS_CREATE : 0);
802}
803
804
805
806
807
808
809int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
810 struct buffer_head *bh_result, int create)
811{
812 ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
813 inode->i_ino, create);
814 return _ext4_get_block(inode, iblock, bh_result,
815 EXT4_GET_BLOCKS_IO_CREATE_EXT);
816}
817
818
819#define DIO_MAX_BLOCKS 4096
820
821
822
823
824
825
826static int ext4_get_block_trans(struct inode *inode, sector_t iblock,
827 struct buffer_head *bh_result, int flags)
828{
829 int dio_credits;
830 handle_t *handle;
831 int retries = 0;
832 int ret;
833
834
835 if (bh_result->b_size >> inode->i_blkbits > DIO_MAX_BLOCKS)
836 bh_result->b_size = DIO_MAX_BLOCKS << inode->i_blkbits;
837 dio_credits = ext4_chunk_trans_blocks(inode,
838 bh_result->b_size >> inode->i_blkbits);
839retry:
840 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
841 if (IS_ERR(handle))
842 return PTR_ERR(handle);
843
844 ret = _ext4_get_block(inode, iblock, bh_result, flags);
845 ext4_journal_stop(handle);
846
847 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
848 goto retry;
849 return ret;
850}
851
852
853int ext4_dio_get_block(struct inode *inode, sector_t iblock,
854 struct buffer_head *bh, int create)
855{
856
857 WARN_ON_ONCE(ext4_journal_current_handle());
858
859 if (!create)
860 return _ext4_get_block(inode, iblock, bh, 0);
861 return ext4_get_block_trans(inode, iblock, bh, EXT4_GET_BLOCKS_CREATE);
862}
863
864
865
866
867
868
869static int ext4_dio_get_block_unwritten_async(struct inode *inode,
870 sector_t iblock, struct buffer_head *bh_result, int create)
871{
872 int ret;
873
874
875 WARN_ON_ONCE(ext4_journal_current_handle());
876
877 ret = ext4_get_block_trans(inode, iblock, bh_result,
878 EXT4_GET_BLOCKS_IO_CREATE_EXT);
879
880
881
882
883
884
885
886
887 if (!ret && buffer_unwritten(bh_result)) {
888 if (!bh_result->b_private) {
889 ext4_io_end_t *io_end;
890
891 io_end = ext4_init_io_end(inode, GFP_KERNEL);
892 if (!io_end)
893 return -ENOMEM;
894 bh_result->b_private = io_end;
895 ext4_set_io_unwritten_flag(inode, io_end);
896 }
897 set_buffer_defer_completion(bh_result);
898 }
899
900 return ret;
901}
902
903
904
905
906
907
908static int ext4_dio_get_block_unwritten_sync(struct inode *inode,
909 sector_t iblock, struct buffer_head *bh_result, int create)
910{
911 int ret;
912
913
914 WARN_ON_ONCE(ext4_journal_current_handle());
915
916 ret = ext4_get_block_trans(inode, iblock, bh_result,
917 EXT4_GET_BLOCKS_IO_CREATE_EXT);
918
919
920
921
922
923
924 if (!ret && buffer_unwritten(bh_result))
925 ext4_set_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
926
927 return ret;
928}
929
930static int ext4_dio_get_block_overwrite(struct inode *inode, sector_t iblock,
931 struct buffer_head *bh_result, int create)
932{
933 int ret;
934
935 ext4_debug("ext4_dio_get_block_overwrite: inode %lu, create flag %d\n",
936 inode->i_ino, create);
937
938 WARN_ON_ONCE(ext4_journal_current_handle());
939
940 ret = _ext4_get_block(inode, iblock, bh_result, 0);
941
942
943
944
945 WARN_ON_ONCE(!buffer_mapped(bh_result) || buffer_unwritten(bh_result));
946
947 return ret;
948}
949
950
951
952
953
954struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
955 ext4_lblk_t block, int map_flags)
956{
957 struct ext4_map_blocks map;
958 struct buffer_head *bh;
959 int create = map_flags & EXT4_GET_BLOCKS_CREATE;
960 int err;
961
962 J_ASSERT(handle != NULL || create == 0);
963
964 map.m_lblk = block;
965 map.m_len = 1;
966 err = ext4_map_blocks(handle, inode, &map, map_flags);
967
968 if (err == 0)
969 return create ? ERR_PTR(-ENOSPC) : NULL;
970 if (err < 0)
971 return ERR_PTR(err);
972
973 bh = sb_getblk(inode->i_sb, map.m_pblk);
974 if (unlikely(!bh))
975 return ERR_PTR(-ENOMEM);
976 if (map.m_flags & EXT4_MAP_NEW) {
977 J_ASSERT(create != 0);
978 J_ASSERT(handle != NULL);
979
980
981
982
983
984
985
986
987 lock_buffer(bh);
988 BUFFER_TRACE(bh, "call get_create_access");
989 err = ext4_journal_get_create_access(handle, bh);
990 if (unlikely(err)) {
991 unlock_buffer(bh);
992 goto errout;
993 }
994 if (!buffer_uptodate(bh)) {
995 memset(bh->b_data, 0, inode->i_sb->s_blocksize);
996 set_buffer_uptodate(bh);
997 }
998 unlock_buffer(bh);
999 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
1000 err = ext4_handle_dirty_metadata(handle, inode, bh);
1001 if (unlikely(err))
1002 goto errout;
1003 } else
1004 BUFFER_TRACE(bh, "not a new buffer");
1005 return bh;
1006errout:
1007 brelse(bh);
1008 return ERR_PTR(err);
1009}
1010
1011struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
1012 ext4_lblk_t block, int map_flags)
1013{
1014 struct buffer_head *bh;
1015
1016 bh = ext4_getblk(handle, inode, block, map_flags);
1017 if (IS_ERR(bh))
1018 return bh;
1019 if (!bh || buffer_uptodate(bh))
1020 return bh;
1021 ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1, &bh);
1022 wait_on_buffer(bh);
1023 if (buffer_uptodate(bh))
1024 return bh;
1025 put_bh(bh);
1026 return ERR_PTR(-EIO);
1027}
1028
1029
1030int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
1031 bool wait, struct buffer_head **bhs)
1032{
1033 int i, err;
1034
1035 for (i = 0; i < bh_count; i++) {
1036 bhs[i] = ext4_getblk(NULL, inode, block + i, 0 );
1037 if (IS_ERR(bhs[i])) {
1038 err = PTR_ERR(bhs[i]);
1039 bh_count = i;
1040 goto out_brelse;
1041 }
1042 }
1043
1044 for (i = 0; i < bh_count; i++)
1045
1046 if (bhs[i] && !buffer_uptodate(bhs[i]))
1047 ll_rw_block(REQ_OP_READ, REQ_META | REQ_PRIO, 1,
1048 &bhs[i]);
1049
1050 if (!wait)
1051 return 0;
1052
1053 for (i = 0; i < bh_count; i++)
1054 if (bhs[i])
1055 wait_on_buffer(bhs[i]);
1056
1057 for (i = 0; i < bh_count; i++) {
1058 if (bhs[i] && !buffer_uptodate(bhs[i])) {
1059 err = -EIO;
1060 goto out_brelse;
1061 }
1062 }
1063 return 0;
1064
1065out_brelse:
1066 for (i = 0; i < bh_count; i++) {
1067 brelse(bhs[i]);
1068 bhs[i] = NULL;
1069 }
1070 return err;
1071}
1072
1073int ext4_walk_page_buffers(handle_t *handle,
1074 struct buffer_head *head,
1075 unsigned from,
1076 unsigned to,
1077 int *partial,
1078 int (*fn)(handle_t *handle,
1079 struct buffer_head *bh))
1080{
1081 struct buffer_head *bh;
1082 unsigned block_start, block_end;
1083 unsigned blocksize = head->b_size;
1084 int err, ret = 0;
1085 struct buffer_head *next;
1086
1087 for (bh = head, block_start = 0;
1088 ret == 0 && (bh != head || !block_start);
1089 block_start = block_end, bh = next) {
1090 next = bh->b_this_page;
1091 block_end = block_start + blocksize;
1092 if (block_end <= from || block_start >= to) {
1093 if (partial && !buffer_uptodate(bh))
1094 *partial = 1;
1095 continue;
1096 }
1097 err = (*fn)(handle, bh);
1098 if (!ret)
1099 ret = err;
1100 }
1101 return ret;
1102}
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128int do_journal_get_write_access(handle_t *handle,
1129 struct buffer_head *bh)
1130{
1131 int dirty = buffer_dirty(bh);
1132 int ret;
1133
1134 if (!buffer_mapped(bh) || buffer_freed(bh))
1135 return 0;
1136
1137
1138
1139
1140
1141
1142
1143
1144 if (dirty)
1145 clear_buffer_dirty(bh);
1146 BUFFER_TRACE(bh, "get write access");
1147 ret = ext4_journal_get_write_access(handle, bh);
1148 if (!ret && dirty)
1149 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1150 return ret;
1151}
1152
1153#ifdef CONFIG_EXT4_FS_ENCRYPTION
1154static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
1155 get_block_t *get_block)
1156{
1157 unsigned from = pos & (PAGE_SIZE - 1);
1158 unsigned to = from + len;
1159 struct inode *inode = page->mapping->host;
1160 unsigned block_start, block_end;
1161 sector_t block;
1162 int err = 0;
1163 unsigned blocksize = inode->i_sb->s_blocksize;
1164 unsigned bbits;
1165 struct buffer_head *bh, *head, *wait[2], **wait_bh = wait;
1166 bool decrypt = false;
1167
1168 BUG_ON(!PageLocked(page));
1169 BUG_ON(from > PAGE_SIZE);
1170 BUG_ON(to > PAGE_SIZE);
1171 BUG_ON(from > to);
1172
1173 if (!page_has_buffers(page))
1174 create_empty_buffers(page, blocksize, 0);
1175 head = page_buffers(page);
1176 bbits = ilog2(blocksize);
1177 block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1178
1179 for (bh = head, block_start = 0; bh != head || !block_start;
1180 block++, block_start = block_end, bh = bh->b_this_page) {
1181 block_end = block_start + blocksize;
1182 if (block_end <= from || block_start >= to) {
1183 if (PageUptodate(page)) {
1184 if (!buffer_uptodate(bh))
1185 set_buffer_uptodate(bh);
1186 }
1187 continue;
1188 }
1189 if (buffer_new(bh))
1190 clear_buffer_new(bh);
1191 if (!buffer_mapped(bh)) {
1192 WARN_ON(bh->b_size != blocksize);
1193 err = get_block(inode, block, bh, 1);
1194 if (err)
1195 break;
1196 if (buffer_new(bh)) {
1197 clean_bdev_bh_alias(bh);
1198 if (PageUptodate(page)) {
1199 clear_buffer_new(bh);
1200 set_buffer_uptodate(bh);
1201 mark_buffer_dirty(bh);
1202 continue;
1203 }
1204 if (block_end > to || block_start < from)
1205 zero_user_segments(page, to, block_end,
1206 block_start, from);
1207 continue;
1208 }
1209 }
1210 if (PageUptodate(page)) {
1211 if (!buffer_uptodate(bh))
1212 set_buffer_uptodate(bh);
1213 continue;
1214 }
1215 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1216 !buffer_unwritten(bh) &&
1217 (block_start < from || block_end > to)) {
1218 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
1219 *wait_bh++ = bh;
1220 decrypt = ext4_encrypted_inode(inode) &&
1221 S_ISREG(inode->i_mode);
1222 }
1223 }
1224
1225
1226
1227 while (wait_bh > wait) {
1228 wait_on_buffer(*--wait_bh);
1229 if (!buffer_uptodate(*wait_bh))
1230 err = -EIO;
1231 }
1232 if (unlikely(err))
1233 page_zero_new_buffers(page, from, to);
1234 else if (decrypt)
1235 err = fscrypt_decrypt_page(page->mapping->host, page,
1236 PAGE_SIZE, 0, page->index);
1237 return err;
1238}
1239#endif
1240
1241static int ext4_write_begin(struct file *file, struct address_space *mapping,
1242 loff_t pos, unsigned len, unsigned flags,
1243 struct page **pagep, void **fsdata)
1244{
1245 struct inode *inode = mapping->host;
1246 int ret, needed_blocks;
1247 handle_t *handle;
1248 int retries = 0;
1249 struct page *page;
1250 pgoff_t index;
1251 unsigned from, to;
1252
1253 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
1254 return -EIO;
1255
1256 trace_ext4_write_begin(inode, pos, len, flags);
1257
1258
1259
1260
1261 needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
1262 index = pos >> PAGE_SHIFT;
1263 from = pos & (PAGE_SIZE - 1);
1264 to = from + len;
1265
1266 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1267 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1268 flags, pagep);
1269 if (ret < 0)
1270 return ret;
1271 if (ret == 1)
1272 return 0;
1273 }
1274
1275
1276
1277
1278
1279
1280
1281
1282retry_grab:
1283 page = grab_cache_page_write_begin(mapping, index, flags);
1284 if (!page)
1285 return -ENOMEM;
1286 unlock_page(page);
1287
1288retry_journal:
1289 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
1290 if (IS_ERR(handle)) {
1291 put_page(page);
1292 return PTR_ERR(handle);
1293 }
1294
1295 lock_page(page);
1296 if (page->mapping != mapping) {
1297
1298 unlock_page(page);
1299 put_page(page);
1300 ext4_journal_stop(handle);
1301 goto retry_grab;
1302 }
1303
1304 wait_for_stable_page(page);
1305
1306#ifdef CONFIG_EXT4_FS_ENCRYPTION
1307 if (ext4_should_dioread_nolock(inode))
1308 ret = ext4_block_write_begin(page, pos, len,
1309 ext4_get_block_unwritten);
1310 else
1311 ret = ext4_block_write_begin(page, pos, len,
1312 ext4_get_block);
1313#else
1314 if (ext4_should_dioread_nolock(inode))
1315 ret = __block_write_begin(page, pos, len,
1316 ext4_get_block_unwritten);
1317 else
1318 ret = __block_write_begin(page, pos, len, ext4_get_block);
1319#endif
1320 if (!ret && ext4_should_journal_data(inode)) {
1321 ret = ext4_walk_page_buffers(handle, page_buffers(page),
1322 from, to, NULL,
1323 do_journal_get_write_access);
1324 }
1325
1326 if (ret) {
1327 unlock_page(page);
1328
1329
1330
1331
1332
1333
1334
1335
1336 if (pos + len > inode->i_size && ext4_can_truncate(inode))
1337 ext4_orphan_add(handle, inode);
1338
1339 ext4_journal_stop(handle);
1340 if (pos + len > inode->i_size) {
1341 ext4_truncate_failed_write(inode);
1342
1343
1344
1345
1346
1347
1348 if (inode->i_nlink)
1349 ext4_orphan_del(NULL, inode);
1350 }
1351
1352 if (ret == -ENOSPC &&
1353 ext4_should_retry_alloc(inode->i_sb, &retries))
1354 goto retry_journal;
1355 put_page(page);
1356 return ret;
1357 }
1358 *pagep = page;
1359 return ret;
1360}
1361
1362
1363static int write_end_fn(handle_t *handle, struct buffer_head *bh)
1364{
1365 int ret;
1366 if (!buffer_mapped(bh) || buffer_freed(bh))
1367 return 0;
1368 set_buffer_uptodate(bh);
1369 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1370 clear_buffer_meta(bh);
1371 clear_buffer_prio(bh);
1372 return ret;
1373}
1374
1375
1376
1377
1378
1379
1380
1381
1382static int ext4_write_end(struct file *file,
1383 struct address_space *mapping,
1384 loff_t pos, unsigned len, unsigned copied,
1385 struct page *page, void *fsdata)
1386{
1387 handle_t *handle = ext4_journal_current_handle();
1388 struct inode *inode = mapping->host;
1389 loff_t old_size = inode->i_size;
1390 int ret = 0, ret2;
1391 int i_size_changed = 0;
1392
1393 trace_ext4_write_end(inode, pos, len, copied);
1394 if (ext4_has_inline_data(inode)) {
1395 ret = ext4_write_inline_data_end(inode, pos, len,
1396 copied, page);
1397 if (ret < 0) {
1398 unlock_page(page);
1399 put_page(page);
1400 goto errout;
1401 }
1402 copied = ret;
1403 } else
1404 copied = block_write_end(file, mapping, pos,
1405 len, copied, page, fsdata);
1406
1407
1408
1409
1410 i_size_changed = ext4_update_inode_size(inode, pos + copied);
1411 unlock_page(page);
1412 put_page(page);
1413
1414 if (old_size < pos)
1415 pagecache_isize_extended(inode, old_size, pos);
1416
1417
1418
1419
1420
1421
1422 if (i_size_changed)
1423 ext4_mark_inode_dirty(handle, inode);
1424
1425 if (pos + len > inode->i_size && ext4_can_truncate(inode))
1426
1427
1428
1429
1430 ext4_orphan_add(handle, inode);
1431errout:
1432 ret2 = ext4_journal_stop(handle);
1433 if (!ret)
1434 ret = ret2;
1435
1436 if (pos + len > inode->i_size) {
1437 ext4_truncate_failed_write(inode);
1438
1439
1440
1441
1442
1443 if (inode->i_nlink)
1444 ext4_orphan_del(NULL, inode);
1445 }
1446
1447 return ret ? ret : copied;
1448}
1449
1450
1451
1452
1453
1454
1455static void ext4_journalled_zero_new_buffers(handle_t *handle,
1456 struct page *page,
1457 unsigned from, unsigned to)
1458{
1459 unsigned int block_start = 0, block_end;
1460 struct buffer_head *head, *bh;
1461
1462 bh = head = page_buffers(page);
1463 do {
1464 block_end = block_start + bh->b_size;
1465 if (buffer_new(bh)) {
1466 if (block_end > from && block_start < to) {
1467 if (!PageUptodate(page)) {
1468 unsigned start, size;
1469
1470 start = max(from, block_start);
1471 size = min(to, block_end) - start;
1472
1473 zero_user(page, start, size);
1474 write_end_fn(handle, bh);
1475 }
1476 clear_buffer_new(bh);
1477 }
1478 }
1479 block_start = block_end;
1480 bh = bh->b_this_page;
1481 } while (bh != head);
1482}
1483
1484static int ext4_journalled_write_end(struct file *file,
1485 struct address_space *mapping,
1486 loff_t pos, unsigned len, unsigned copied,
1487 struct page *page, void *fsdata)
1488{
1489 handle_t *handle = ext4_journal_current_handle();
1490 struct inode *inode = mapping->host;
1491 loff_t old_size = inode->i_size;
1492 int ret = 0, ret2;
1493 int partial = 0;
1494 unsigned from, to;
1495 int size_changed = 0;
1496
1497 trace_ext4_journalled_write_end(inode, pos, len, copied);
1498 from = pos & (PAGE_SIZE - 1);
1499 to = from + len;
1500
1501 BUG_ON(!ext4_handle_valid(handle));
1502
1503 if (ext4_has_inline_data(inode)) {
1504 ret = ext4_write_inline_data_end(inode, pos, len,
1505 copied, page);
1506 if (ret < 0) {
1507 unlock_page(page);
1508 put_page(page);
1509 goto errout;
1510 }
1511 copied = ret;
1512 } else if (unlikely(copied < len) && !PageUptodate(page)) {
1513 copied = 0;
1514 ext4_journalled_zero_new_buffers(handle, page, from, to);
1515 } else {
1516 if (unlikely(copied < len))
1517 ext4_journalled_zero_new_buffers(handle, page,
1518 from + copied, to);
1519 ret = ext4_walk_page_buffers(handle, page_buffers(page), from,
1520 from + copied, &partial,
1521 write_end_fn);
1522 if (!partial)
1523 SetPageUptodate(page);
1524 }
1525 size_changed = ext4_update_inode_size(inode, pos + copied);
1526 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1527 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1528 unlock_page(page);
1529 put_page(page);
1530
1531 if (old_size < pos)
1532 pagecache_isize_extended(inode, old_size, pos);
1533
1534 if (size_changed) {
1535 ret2 = ext4_mark_inode_dirty(handle, inode);
1536 if (!ret)
1537 ret = ret2;
1538 }
1539
1540 if (pos + len > inode->i_size && ext4_can_truncate(inode))
1541
1542
1543
1544
1545 ext4_orphan_add(handle, inode);
1546
1547errout:
1548 ret2 = ext4_journal_stop(handle);
1549 if (!ret)
1550 ret = ret2;
1551 if (pos + len > inode->i_size) {
1552 ext4_truncate_failed_write(inode);
1553
1554
1555
1556
1557
1558 if (inode->i_nlink)
1559 ext4_orphan_del(NULL, inode);
1560 }
1561
1562 return ret ? ret : copied;
1563}
1564
1565
1566
1567
1568static int ext4_da_reserve_space(struct inode *inode)
1569{
1570 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1571 struct ext4_inode_info *ei = EXT4_I(inode);
1572 int ret;
1573
1574
1575
1576
1577
1578
1579 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
1580 if (ret)
1581 return ret;
1582
1583 spin_lock(&ei->i_block_reservation_lock);
1584 if (ext4_claim_free_clusters(sbi, 1, 0)) {
1585 spin_unlock(&ei->i_block_reservation_lock);
1586 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
1587 return -ENOSPC;
1588 }
1589 ei->i_reserved_data_blocks++;
1590 trace_ext4_da_reserve_space(inode);
1591 spin_unlock(&ei->i_block_reservation_lock);
1592
1593 return 0;
1594}
1595
1596static void ext4_da_release_space(struct inode *inode, int to_free)
1597{
1598 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1599 struct ext4_inode_info *ei = EXT4_I(inode);
1600
1601 if (!to_free)
1602 return;
1603
1604 spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
1605
1606 trace_ext4_da_release_space(inode, to_free);
1607 if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1608
1609
1610
1611
1612
1613
1614 ext4_warning(inode->i_sb, "ext4_da_release_space: "
1615 "ino %lu, to_free %d with only %d reserved "
1616 "data blocks", inode->i_ino, to_free,
1617 ei->i_reserved_data_blocks);
1618 WARN_ON(1);
1619 to_free = ei->i_reserved_data_blocks;
1620 }
1621 ei->i_reserved_data_blocks -= to_free;
1622
1623
1624 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
1625
1626 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1627
1628 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1629}
1630
1631static void ext4_da_page_release_reservation(struct page *page,
1632 unsigned int offset,
1633 unsigned int length)
1634{
1635 int to_release = 0, contiguous_blks = 0;
1636 struct buffer_head *head, *bh;
1637 unsigned int curr_off = 0;
1638 struct inode *inode = page->mapping->host;
1639 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1640 unsigned int stop = offset + length;
1641 int num_clusters;
1642 ext4_fsblk_t lblk;
1643
1644 BUG_ON(stop > PAGE_SIZE || stop < length);
1645
1646 head = page_buffers(page);
1647 bh = head;
1648 do {
1649 unsigned int next_off = curr_off + bh->b_size;
1650
1651 if (next_off > stop)
1652 break;
1653
1654 if ((offset <= curr_off) && (buffer_delay(bh))) {
1655 to_release++;
1656 contiguous_blks++;
1657 clear_buffer_delay(bh);
1658 } else if (contiguous_blks) {
1659 lblk = page->index <<
1660 (PAGE_SHIFT - inode->i_blkbits);
1661 lblk += (curr_off >> inode->i_blkbits) -
1662 contiguous_blks;
1663 ext4_es_remove_extent(inode, lblk, contiguous_blks);
1664 contiguous_blks = 0;
1665 }
1666 curr_off = next_off;
1667 } while ((bh = bh->b_this_page) != head);
1668
1669 if (contiguous_blks) {
1670 lblk = page->index << (PAGE_SHIFT - inode->i_blkbits);
1671 lblk += (curr_off >> inode->i_blkbits) - contiguous_blks;
1672 ext4_es_remove_extent(inode, lblk, contiguous_blks);
1673 }
1674
1675
1676
1677 num_clusters = EXT4_NUM_B2C(sbi, to_release);
1678 while (num_clusters > 0) {
1679 lblk = (page->index << (PAGE_SHIFT - inode->i_blkbits)) +
1680 ((num_clusters - 1) << sbi->s_cluster_bits);
1681 if (sbi->s_cluster_ratio == 1 ||
1682 !ext4_find_delalloc_cluster(inode, lblk))
1683 ext4_da_release_space(inode, 1);
1684
1685 num_clusters--;
1686 }
1687}
1688
1689
1690
1691
1692
1693struct mpage_da_data {
1694 struct inode *inode;
1695 struct writeback_control *wbc;
1696
1697 pgoff_t first_page;
1698 pgoff_t next_page;
1699 pgoff_t last_page;
1700
1701
1702
1703
1704
1705 struct ext4_map_blocks map;
1706 struct ext4_io_submit io_submit;
1707 unsigned int do_map:1;
1708};
1709
1710static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1711 bool invalidate)
1712{
1713 int nr_pages, i;
1714 pgoff_t index, end;
1715 struct pagevec pvec;
1716 struct inode *inode = mpd->inode;
1717 struct address_space *mapping = inode->i_mapping;
1718
1719
1720 if (mpd->first_page >= mpd->next_page)
1721 return;
1722
1723 index = mpd->first_page;
1724 end = mpd->next_page - 1;
1725 if (invalidate) {
1726 ext4_lblk_t start, last;
1727 start = index << (PAGE_SHIFT - inode->i_blkbits);
1728 last = end << (PAGE_SHIFT - inode->i_blkbits);
1729 ext4_es_remove_extent(inode, start, last - start + 1);
1730 }
1731
1732 pagevec_init(&pvec);
1733 while (index <= end) {
1734 nr_pages = pagevec_lookup_range(&pvec, mapping, &index, end);
1735 if (nr_pages == 0)
1736 break;
1737 for (i = 0; i < nr_pages; i++) {
1738 struct page *page = pvec.pages[i];
1739
1740 BUG_ON(!PageLocked(page));
1741 BUG_ON(PageWriteback(page));
1742 if (invalidate) {
1743 if (page_mapped(page))
1744 clear_page_dirty_for_io(page);
1745 block_invalidatepage(page, 0, PAGE_SIZE);
1746 ClearPageUptodate(page);
1747 }
1748 unlock_page(page);
1749 }
1750 pagevec_release(&pvec);
1751 }
1752}
1753
1754static void ext4_print_free_blocks(struct inode *inode)
1755{
1756 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1757 struct super_block *sb = inode->i_sb;
1758 struct ext4_inode_info *ei = EXT4_I(inode);
1759
1760 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1761 EXT4_C2B(EXT4_SB(inode->i_sb),
1762 ext4_count_free_clusters(sb)));
1763 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1764 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1765 (long long) EXT4_C2B(EXT4_SB(sb),
1766 percpu_counter_sum(&sbi->s_freeclusters_counter)));
1767 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1768 (long long) EXT4_C2B(EXT4_SB(sb),
1769 percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1770 ext4_msg(sb, KERN_CRIT, "Block reservation details");
1771 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
1772 ei->i_reserved_data_blocks);
1773 return;
1774}
1775
1776static int ext4_bh_delay_or_unwritten(handle_t *handle, struct buffer_head *bh)
1777{
1778 return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
1779}
1780
1781
1782
1783
1784
1785
1786
1787static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
1788 struct ext4_map_blocks *map,
1789 struct buffer_head *bh)
1790{
1791 struct extent_status es;
1792 int retval;
1793 sector_t invalid_block = ~((sector_t) 0xffff);
1794#ifdef ES_AGGRESSIVE_TEST
1795 struct ext4_map_blocks orig_map;
1796
1797 memcpy(&orig_map, map, sizeof(*map));
1798#endif
1799
1800 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
1801 invalid_block = ~0;
1802
1803 map->m_flags = 0;
1804 ext_debug("ext4_da_map_blocks(): inode %lu, max_blocks %u,"
1805 "logical block %lu\n", inode->i_ino, map->m_len,
1806 (unsigned long) map->m_lblk);
1807
1808
1809 if (ext4_es_lookup_extent(inode, iblock, &es)) {
1810 if (ext4_es_is_hole(&es)) {
1811 retval = 0;
1812 down_read(&EXT4_I(inode)->i_data_sem);
1813 goto add_delayed;
1814 }
1815
1816
1817
1818
1819
1820 if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) {
1821 map_bh(bh, inode->i_sb, invalid_block);
1822 set_buffer_new(bh);
1823 set_buffer_delay(bh);
1824 return 0;
1825 }
1826
1827 map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk;
1828 retval = es.es_len - (iblock - es.es_lblk);
1829 if (retval > map->m_len)
1830 retval = map->m_len;
1831 map->m_len = retval;
1832 if (ext4_es_is_written(&es))
1833 map->m_flags |= EXT4_MAP_MAPPED;
1834 else if (ext4_es_is_unwritten(&es))
1835 map->m_flags |= EXT4_MAP_UNWRITTEN;
1836 else
1837 BUG_ON(1);
1838
1839#ifdef ES_AGGRESSIVE_TEST
1840 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1841#endif
1842 return retval;
1843 }
1844
1845
1846
1847
1848
1849 down_read(&EXT4_I(inode)->i_data_sem);
1850 if (ext4_has_inline_data(inode))
1851 retval = 0;
1852 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
1853 retval = ext4_ext_map_blocks(NULL, inode, map, 0);
1854 else
1855 retval = ext4_ind_map_blocks(NULL, inode, map, 0);
1856
1857add_delayed:
1858 if (retval == 0) {
1859 int ret;
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869 if (EXT4_SB(inode->i_sb)->s_cluster_ratio == 1 ||
1870 !ext4_find_delalloc_cluster(inode, map->m_lblk)) {
1871 ret = ext4_da_reserve_space(inode);
1872 if (ret) {
1873
1874 retval = ret;
1875 goto out_unlock;
1876 }
1877 }
1878
1879 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1880 ~0, EXTENT_STATUS_DELAYED);
1881 if (ret) {
1882 retval = ret;
1883 goto out_unlock;
1884 }
1885
1886 map_bh(bh, inode->i_sb, invalid_block);
1887 set_buffer_new(bh);
1888 set_buffer_delay(bh);
1889 } else if (retval > 0) {
1890 int ret;
1891 unsigned int status;
1892
1893 if (unlikely(retval != map->m_len)) {
1894 ext4_warning(inode->i_sb,
1895 "ES len assertion failed for inode "
1896 "%lu: retval %d != map->m_len %d",
1897 inode->i_ino, retval, map->m_len);
1898 WARN_ON(1);
1899 }
1900
1901 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
1902 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
1903 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1904 map->m_pblk, status);
1905 if (ret != 0)
1906 retval = ret;
1907 }
1908
1909out_unlock:
1910 up_read((&EXT4_I(inode)->i_data_sem));
1911
1912 return retval;
1913}
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1928 struct buffer_head *bh, int create)
1929{
1930 struct ext4_map_blocks map;
1931 int ret = 0;
1932
1933 BUG_ON(create == 0);
1934 BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
1935
1936 map.m_lblk = iblock;
1937 map.m_len = 1;
1938
1939
1940
1941
1942
1943
1944 ret = ext4_da_map_blocks(inode, iblock, &map, bh);
1945 if (ret <= 0)
1946 return ret;
1947
1948 map_bh(bh, inode->i_sb, map.m_pblk);
1949 ext4_update_bh_state(bh, map.m_flags);
1950
1951 if (buffer_unwritten(bh)) {
1952
1953
1954
1955
1956
1957
1958 set_buffer_new(bh);
1959 set_buffer_mapped(bh);
1960 }
1961 return 0;
1962}
1963
1964static int bget_one(handle_t *handle, struct buffer_head *bh)
1965{
1966 get_bh(bh);
1967 return 0;
1968}
1969
1970static int bput_one(handle_t *handle, struct buffer_head *bh)
1971{
1972 put_bh(bh);
1973 return 0;
1974}
1975
1976static int __ext4_journalled_writepage(struct page *page,
1977 unsigned int len)
1978{
1979 struct address_space *mapping = page->mapping;
1980 struct inode *inode = mapping->host;
1981 struct buffer_head *page_bufs = NULL;
1982 handle_t *handle = NULL;
1983 int ret = 0, err = 0;
1984 int inline_data = ext4_has_inline_data(inode);
1985 struct buffer_head *inode_bh = NULL;
1986
1987 ClearPageChecked(page);
1988
1989 if (inline_data) {
1990 BUG_ON(page->index != 0);
1991 BUG_ON(len > ext4_get_max_inline_size(inode));
1992 inode_bh = ext4_journalled_write_inline_data(inode, len, page);
1993 if (inode_bh == NULL)
1994 goto out;
1995 } else {
1996 page_bufs = page_buffers(page);
1997 if (!page_bufs) {
1998 BUG();
1999 goto out;
2000 }
2001 ext4_walk_page_buffers(handle, page_bufs, 0, len,
2002 NULL, bget_one);
2003 }
2004
2005
2006
2007
2008
2009 get_page(page);
2010 unlock_page(page);
2011
2012 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
2013 ext4_writepage_trans_blocks(inode));
2014 if (IS_ERR(handle)) {
2015 ret = PTR_ERR(handle);
2016 put_page(page);
2017 goto out_no_pagelock;
2018 }
2019 BUG_ON(!ext4_handle_valid(handle));
2020
2021 lock_page(page);
2022 put_page(page);
2023 if (page->mapping != mapping) {
2024
2025 ext4_journal_stop(handle);
2026 ret = 0;
2027 goto out;
2028 }
2029
2030 if (inline_data) {
2031 BUFFER_TRACE(inode_bh, "get write access");
2032 ret = ext4_journal_get_write_access(handle, inode_bh);
2033
2034 err = ext4_handle_dirty_metadata(handle, inode, inode_bh);
2035
2036 } else {
2037 ret = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
2038 do_journal_get_write_access);
2039
2040 err = ext4_walk_page_buffers(handle, page_bufs, 0, len, NULL,
2041 write_end_fn);
2042 }
2043 if (ret == 0)
2044 ret = err;
2045 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
2046 err = ext4_journal_stop(handle);
2047 if (!ret)
2048 ret = err;
2049
2050 if (!ext4_has_inline_data(inode))
2051 ext4_walk_page_buffers(NULL, page_bufs, 0, len,
2052 NULL, bput_one);
2053 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
2054out:
2055 unlock_page(page);
2056out_no_pagelock:
2057 brelse(inode_bh);
2058 return ret;
2059}
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102static int ext4_writepage(struct page *page,
2103 struct writeback_control *wbc)
2104{
2105 int ret = 0;
2106 loff_t size;
2107 unsigned int len;
2108 struct buffer_head *page_bufs = NULL;
2109 struct inode *inode = page->mapping->host;
2110 struct ext4_io_submit io_submit;
2111 bool keep_towrite = false;
2112
2113 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
2114 ext4_invalidatepage(page, 0, PAGE_SIZE);
2115 unlock_page(page);
2116 return -EIO;
2117 }
2118
2119 trace_ext4_writepage(page);
2120 size = i_size_read(inode);
2121 if (page->index == size >> PAGE_SHIFT)
2122 len = size & ~PAGE_MASK;
2123 else
2124 len = PAGE_SIZE;
2125
2126 page_bufs = page_buffers(page);
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144 if (ext4_walk_page_buffers(NULL, page_bufs, 0, len, NULL,
2145 ext4_bh_delay_or_unwritten)) {
2146 redirty_page_for_writepage(wbc, page);
2147 if ((current->flags & PF_MEMALLOC) ||
2148 (inode->i_sb->s_blocksize == PAGE_SIZE)) {
2149
2150
2151
2152
2153
2154 WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD))
2155 == PF_MEMALLOC);
2156 unlock_page(page);
2157 return 0;
2158 }
2159 keep_towrite = true;
2160 }
2161
2162 if (PageChecked(page) && ext4_should_journal_data(inode))
2163
2164
2165
2166
2167 return __ext4_journalled_writepage(page, len);
2168
2169 ext4_io_submit_init(&io_submit, wbc);
2170 io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
2171 if (!io_submit.io_end) {
2172 redirty_page_for_writepage(wbc, page);
2173 unlock_page(page);
2174 return -ENOMEM;
2175 }
2176 ret = ext4_bio_write_page(&io_submit, page, len, wbc, keep_towrite);
2177 ext4_io_submit(&io_submit);
2178
2179 ext4_put_io_end_defer(io_submit.io_end);
2180 return ret;
2181}
2182
2183static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page)
2184{
2185 int len;
2186 loff_t size;
2187 int err;
2188
2189 BUG_ON(page->index != mpd->first_page);
2190 clear_page_dirty_for_io(page);
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204 size = i_size_read(mpd->inode);
2205 if (page->index == size >> PAGE_SHIFT)
2206 len = size & ~PAGE_MASK;
2207 else
2208 len = PAGE_SIZE;
2209 err = ext4_bio_write_page(&mpd->io_submit, page, len, mpd->wbc, false);
2210 if (!err)
2211 mpd->wbc->nr_to_write--;
2212 mpd->first_page++;
2213
2214 return err;
2215}
2216
2217#define BH_FLAGS ((1 << BH_Unwritten) | (1 << BH_Delay))
2218
2219
2220
2221
2222
2223
2224#define MAX_WRITEPAGES_EXTENT_LEN 2048
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2241 struct buffer_head *bh)
2242{
2243 struct ext4_map_blocks *map = &mpd->map;
2244
2245
2246 if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2247 (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2248
2249 if (map->m_len == 0)
2250 return true;
2251 return false;
2252 }
2253
2254
2255 if (map->m_len == 0) {
2256
2257 if (!mpd->do_map)
2258 return false;
2259 map->m_lblk = lblk;
2260 map->m_len = 1;
2261 map->m_flags = bh->b_state & BH_FLAGS;
2262 return true;
2263 }
2264
2265
2266 if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2267 return false;
2268
2269
2270 if (lblk == map->m_lblk + map->m_len &&
2271 (bh->b_state & BH_FLAGS) == map->m_flags) {
2272 map->m_len++;
2273 return true;
2274 }
2275 return false;
2276}
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2295 struct buffer_head *head,
2296 struct buffer_head *bh,
2297 ext4_lblk_t lblk)
2298{
2299 struct inode *inode = mpd->inode;
2300 int err;
2301 ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
2302 >> inode->i_blkbits;
2303
2304 do {
2305 BUG_ON(buffer_locked(bh));
2306
2307 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
2308
2309 if (mpd->map.m_len)
2310 return 0;
2311
2312 if (!mpd->do_map)
2313 return 0;
2314
2315 break;
2316 }
2317 } while (lblk++, (bh = bh->b_this_page) != head);
2318
2319 if (mpd->map.m_len == 0) {
2320 err = mpage_submit_page(mpd, head->b_page);
2321 if (err < 0)
2322 return err;
2323 }
2324 return lblk < blocks;
2325}
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2342{
2343 struct pagevec pvec;
2344 int nr_pages, i;
2345 struct inode *inode = mpd->inode;
2346 struct buffer_head *head, *bh;
2347 int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
2348 pgoff_t start, end;
2349 ext4_lblk_t lblk;
2350 sector_t pblock;
2351 int err;
2352
2353 start = mpd->map.m_lblk >> bpp_bits;
2354 end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2355 lblk = start << bpp_bits;
2356 pblock = mpd->map.m_pblk;
2357
2358 pagevec_init(&pvec);
2359 while (start <= end) {
2360 nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping,
2361 &start, end);
2362 if (nr_pages == 0)
2363 break;
2364 for (i = 0; i < nr_pages; i++) {
2365 struct page *page = pvec.pages[i];
2366
2367 bh = head = page_buffers(page);
2368 do {
2369 if (lblk < mpd->map.m_lblk)
2370 continue;
2371 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2372
2373
2374
2375
2376 mpd->map.m_len = 0;
2377 mpd->map.m_flags = 0;
2378
2379
2380
2381
2382
2383
2384
2385 err = mpage_process_page_bufs(mpd, head,
2386 bh, lblk);
2387 pagevec_release(&pvec);
2388 if (err > 0)
2389 err = 0;
2390 return err;
2391 }
2392 if (buffer_delay(bh)) {
2393 clear_buffer_delay(bh);
2394 bh->b_blocknr = pblock++;
2395 }
2396 clear_buffer_unwritten(bh);
2397 } while (lblk++, (bh = bh->b_this_page) != head);
2398
2399
2400
2401
2402
2403
2404 mpd->io_submit.io_end->size += PAGE_SIZE;
2405
2406 err = mpage_submit_page(mpd, page);
2407 if (err < 0) {
2408 pagevec_release(&pvec);
2409 return err;
2410 }
2411 }
2412 pagevec_release(&pvec);
2413 }
2414
2415 mpd->map.m_len = 0;
2416 mpd->map.m_flags = 0;
2417 return 0;
2418}
2419
2420static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2421{
2422 struct inode *inode = mpd->inode;
2423 struct ext4_map_blocks *map = &mpd->map;
2424 int get_blocks_flags;
2425 int err, dioread_nolock;
2426
2427 trace_ext4_da_write_pages_extent(inode, map);
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443 get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
2444 EXT4_GET_BLOCKS_METADATA_NOFAIL |
2445 EXT4_GET_BLOCKS_IO_SUBMIT;
2446 dioread_nolock = ext4_should_dioread_nolock(inode);
2447 if (dioread_nolock)
2448 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2449 if (map->m_flags & (1 << BH_Delay))
2450 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2451
2452 err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2453 if (err < 0)
2454 return err;
2455 if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
2456 if (!mpd->io_submit.io_end->handle &&
2457 ext4_handle_valid(handle)) {
2458 mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2459 handle->h_rsv_handle = NULL;
2460 }
2461 ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
2462 }
2463
2464 BUG_ON(map->m_len == 0);
2465 if (map->m_flags & EXT4_MAP_NEW) {
2466 clean_bdev_aliases(inode->i_sb->s_bdev, map->m_pblk,
2467 map->m_len);
2468 }
2469 return 0;
2470}
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492static int mpage_map_and_submit_extent(handle_t *handle,
2493 struct mpage_da_data *mpd,
2494 bool *give_up_on_write)
2495{
2496 struct inode *inode = mpd->inode;
2497 struct ext4_map_blocks *map = &mpd->map;
2498 int err;
2499 loff_t disksize;
2500 int progress = 0;
2501
2502 mpd->io_submit.io_end->offset =
2503 ((loff_t)map->m_lblk) << inode->i_blkbits;
2504 do {
2505 err = mpage_map_one_extent(handle, mpd);
2506 if (err < 0) {
2507 struct super_block *sb = inode->i_sb;
2508
2509 if (ext4_forced_shutdown(EXT4_SB(sb)) ||
2510 EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)
2511 goto invalidate_dirty_pages;
2512
2513
2514
2515
2516
2517 if ((err == -ENOMEM) ||
2518 (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2519 if (progress)
2520 goto update_disksize;
2521 return err;
2522 }
2523 ext4_msg(sb, KERN_CRIT,
2524 "Delayed block allocation failed for "
2525 "inode %lu at logical offset %llu with"
2526 " max blocks %u with error %d",
2527 inode->i_ino,
2528 (unsigned long long)map->m_lblk,
2529 (unsigned)map->m_len, -err);
2530 ext4_msg(sb, KERN_CRIT,
2531 "This should not happen!! Data will "
2532 "be lost\n");
2533 if (err == -ENOSPC)
2534 ext4_print_free_blocks(inode);
2535 invalidate_dirty_pages:
2536 *give_up_on_write = true;
2537 return err;
2538 }
2539 progress = 1;
2540
2541
2542
2543
2544 err = mpage_map_and_submit_buffers(mpd);
2545 if (err < 0)
2546 goto update_disksize;
2547 } while (map->m_len);
2548
2549update_disksize:
2550
2551
2552
2553
2554 disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
2555 if (disksize > EXT4_I(inode)->i_disksize) {
2556 int err2;
2557 loff_t i_size;
2558
2559 down_write(&EXT4_I(inode)->i_data_sem);
2560 i_size = i_size_read(inode);
2561 if (disksize > i_size)
2562 disksize = i_size;
2563 if (disksize > EXT4_I(inode)->i_disksize)
2564 EXT4_I(inode)->i_disksize = disksize;
2565 up_write(&EXT4_I(inode)->i_data_sem);
2566 err2 = ext4_mark_inode_dirty(handle, inode);
2567 if (err2)
2568 ext4_error(inode->i_sb,
2569 "Failed to mark inode %lu dirty",
2570 inode->i_ino);
2571 if (!err)
2572 err = err2;
2573 }
2574 return err;
2575}
2576
2577
2578
2579
2580
2581
2582
2583
2584static int ext4_da_writepages_trans_blocks(struct inode *inode)
2585{
2586 int bpp = ext4_journal_blocks_per_page(inode);
2587
2588 return ext4_meta_trans_blocks(inode,
2589 MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);
2590}
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2611{
2612 struct address_space *mapping = mpd->inode->i_mapping;
2613 struct pagevec pvec;
2614 unsigned int nr_pages;
2615 long left = mpd->wbc->nr_to_write;
2616 pgoff_t index = mpd->first_page;
2617 pgoff_t end = mpd->last_page;
2618 int tag;
2619 int i, err = 0;
2620 int blkbits = mpd->inode->i_blkbits;
2621 ext4_lblk_t lblk;
2622 struct buffer_head *head;
2623
2624 if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
2625 tag = PAGECACHE_TAG_TOWRITE;
2626 else
2627 tag = PAGECACHE_TAG_DIRTY;
2628
2629 pagevec_init(&pvec);
2630 mpd->map.m_len = 0;
2631 mpd->next_page = index;
2632 while (index <= end) {
2633 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2634 tag);
2635 if (nr_pages == 0)
2636 goto out;
2637
2638 for (i = 0; i < nr_pages; i++) {
2639 struct page *page = pvec.pages[i];
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649 if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0)
2650 goto out;
2651
2652
2653 if (mpd->map.m_len > 0 && mpd->next_page != page->index)
2654 goto out;
2655
2656 lock_page(page);
2657
2658
2659
2660
2661
2662
2663
2664 if (!PageDirty(page) ||
2665 (PageWriteback(page) &&
2666 (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
2667 unlikely(page->mapping != mapping)) {
2668 unlock_page(page);
2669 continue;
2670 }
2671
2672 wait_on_page_writeback(page);
2673 BUG_ON(PageWriteback(page));
2674
2675 if (mpd->map.m_len == 0)
2676 mpd->first_page = page->index;
2677 mpd->next_page = page->index + 1;
2678
2679 lblk = ((ext4_lblk_t)page->index) <<
2680 (PAGE_SHIFT - blkbits);
2681 head = page_buffers(page);
2682 err = mpage_process_page_bufs(mpd, head, head, lblk);
2683 if (err <= 0)
2684 goto out;
2685 err = 0;
2686 left--;
2687 }
2688 pagevec_release(&pvec);
2689 cond_resched();
2690 }
2691 return 0;
2692out:
2693 pagevec_release(&pvec);
2694 return err;
2695}
2696
2697static int ext4_writepages(struct address_space *mapping,
2698 struct writeback_control *wbc)
2699{
2700 pgoff_t writeback_index = 0;
2701 long nr_to_write = wbc->nr_to_write;
2702 int range_whole = 0;
2703 int cycled = 1;
2704 handle_t *handle = NULL;
2705 struct mpage_da_data mpd;
2706 struct inode *inode = mapping->host;
2707 int needed_blocks, rsv_blocks = 0, ret = 0;
2708 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2709 bool done;
2710 struct blk_plug plug;
2711 bool give_up_on_write = false;
2712
2713 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2714 return -EIO;
2715
2716 percpu_down_read(&sbi->s_journal_flag_rwsem);
2717 trace_ext4_writepages(inode, wbc);
2718
2719
2720
2721
2722
2723
2724 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2725 goto out_writepages;
2726
2727 if (ext4_should_journal_data(inode)) {
2728 ret = generic_writepages(mapping, wbc);
2729 goto out_writepages;
2730 }
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742 if (unlikely(ext4_forced_shutdown(EXT4_SB(mapping->host->i_sb)) ||
2743 sbi->s_mount_flags & EXT4_MF_FS_ABORTED)) {
2744 ret = -EROFS;
2745 goto out_writepages;
2746 }
2747
2748 if (ext4_should_dioread_nolock(inode)) {
2749
2750
2751
2752
2753 rsv_blocks = 1 + (PAGE_SIZE >> inode->i_blkbits);
2754 }
2755
2756
2757
2758
2759
2760
2761 if (ext4_has_inline_data(inode)) {
2762
2763 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2764 if (IS_ERR(handle)) {
2765 ret = PTR_ERR(handle);
2766 goto out_writepages;
2767 }
2768 BUG_ON(ext4_test_inode_state(inode,
2769 EXT4_STATE_MAY_INLINE_DATA));
2770 ext4_destroy_inline_data(handle, inode);
2771 ext4_journal_stop(handle);
2772 }
2773
2774 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2775 range_whole = 1;
2776
2777 if (wbc->range_cyclic) {
2778 writeback_index = mapping->writeback_index;
2779 if (writeback_index)
2780 cycled = 0;
2781 mpd.first_page = writeback_index;
2782 mpd.last_page = -1;
2783 } else {
2784 mpd.first_page = wbc->range_start >> PAGE_SHIFT;
2785 mpd.last_page = wbc->range_end >> PAGE_SHIFT;
2786 }
2787
2788 mpd.inode = inode;
2789 mpd.wbc = wbc;
2790 ext4_io_submit_init(&mpd.io_submit, wbc);
2791retry:
2792 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2793 tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page);
2794 done = false;
2795 blk_start_plug(&plug);
2796
2797
2798
2799
2800
2801
2802
2803 mpd.do_map = 0;
2804 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2805 if (!mpd.io_submit.io_end) {
2806 ret = -ENOMEM;
2807 goto unplug;
2808 }
2809 ret = mpage_prepare_extent_to_map(&mpd);
2810
2811 ext4_io_submit(&mpd.io_submit);
2812 ext4_put_io_end_defer(mpd.io_submit.io_end);
2813 mpd.io_submit.io_end = NULL;
2814
2815 mpage_release_unused_pages(&mpd, false);
2816 if (ret < 0)
2817 goto unplug;
2818
2819 while (!done && mpd.first_page <= mpd.last_page) {
2820
2821 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2822 if (!mpd.io_submit.io_end) {
2823 ret = -ENOMEM;
2824 break;
2825 }
2826
2827
2828
2829
2830
2831
2832
2833
2834 BUG_ON(ext4_should_journal_data(inode));
2835 needed_blocks = ext4_da_writepages_trans_blocks(inode);
2836
2837
2838 handle = ext4_journal_start_with_reserve(inode,
2839 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
2840 if (IS_ERR(handle)) {
2841 ret = PTR_ERR(handle);
2842 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
2843 "%ld pages, ino %lu; err %d", __func__,
2844 wbc->nr_to_write, inode->i_ino, ret);
2845
2846 ext4_put_io_end(mpd.io_submit.io_end);
2847 mpd.io_submit.io_end = NULL;
2848 break;
2849 }
2850 mpd.do_map = 1;
2851
2852 trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc);
2853 ret = mpage_prepare_extent_to_map(&mpd);
2854 if (!ret) {
2855 if (mpd.map.m_len)
2856 ret = mpage_map_and_submit_extent(handle, &mpd,
2857 &give_up_on_write);
2858 else {
2859
2860
2861
2862
2863
2864
2865 done = true;
2866 }
2867 }
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878 if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2879 ext4_journal_stop(handle);
2880 handle = NULL;
2881 mpd.do_map = 0;
2882 }
2883
2884 ext4_io_submit(&mpd.io_submit);
2885
2886 mpage_release_unused_pages(&mpd, give_up_on_write);
2887
2888
2889
2890
2891
2892
2893
2894 if (handle) {
2895 ext4_put_io_end_defer(mpd.io_submit.io_end);
2896 ext4_journal_stop(handle);
2897 } else
2898 ext4_put_io_end(mpd.io_submit.io_end);
2899 mpd.io_submit.io_end = NULL;
2900
2901 if (ret == -ENOSPC && sbi->s_journal) {
2902
2903
2904
2905
2906
2907 jbd2_journal_force_commit_nested(sbi->s_journal);
2908 ret = 0;
2909 continue;
2910 }
2911
2912 if (ret)
2913 break;
2914 }
2915unplug:
2916 blk_finish_plug(&plug);
2917 if (!ret && !cycled && wbc->nr_to_write > 0) {
2918 cycled = 1;
2919 mpd.last_page = writeback_index - 1;
2920 mpd.first_page = 0;
2921 goto retry;
2922 }
2923
2924
2925 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2926
2927
2928
2929
2930 mapping->writeback_index = mpd.first_page;
2931
2932out_writepages:
2933 trace_ext4_writepages_result(inode, wbc, ret,
2934 nr_to_write - wbc->nr_to_write);
2935 percpu_up_read(&sbi->s_journal_flag_rwsem);
2936 return ret;
2937}
2938
2939static int ext4_dax_writepages(struct address_space *mapping,
2940 struct writeback_control *wbc)
2941{
2942 int ret;
2943 long nr_to_write = wbc->nr_to_write;
2944 struct inode *inode = mapping->host;
2945 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2946
2947 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2948 return -EIO;
2949
2950 percpu_down_read(&sbi->s_journal_flag_rwsem);
2951 trace_ext4_writepages(inode, wbc);
2952
2953 ret = dax_writeback_mapping_range(mapping, inode->i_sb->s_bdev, wbc);
2954 trace_ext4_writepages_result(inode, wbc, ret,
2955 nr_to_write - wbc->nr_to_write);
2956 percpu_up_read(&sbi->s_journal_flag_rwsem);
2957 return ret;
2958}
2959
2960static int ext4_nonda_switch(struct super_block *sb)
2961{
2962 s64 free_clusters, dirty_clusters;
2963 struct ext4_sb_info *sbi = EXT4_SB(sb);
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973 free_clusters =
2974 percpu_counter_read_positive(&sbi->s_freeclusters_counter);
2975 dirty_clusters =
2976 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
2977
2978
2979
2980 if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
2981 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
2982
2983 if (2 * free_clusters < 3 * dirty_clusters ||
2984 free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
2985
2986
2987
2988
2989 return 1;
2990 }
2991 return 0;
2992}
2993
2994
2995static int ext4_da_write_credits(struct inode *inode, loff_t pos, unsigned len)
2996{
2997 if (likely(ext4_has_feature_large_file(inode->i_sb)))
2998 return 1;
2999
3000 if (pos + len <= 0x7fffffffULL)
3001 return 1;
3002
3003
3004 return 2;
3005}
3006
3007static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
3008 loff_t pos, unsigned len, unsigned flags,
3009 struct page **pagep, void **fsdata)
3010{
3011 int ret, retries = 0;
3012 struct page *page;
3013 pgoff_t index;
3014 struct inode *inode = mapping->host;
3015 handle_t *handle;
3016
3017 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
3018 return -EIO;
3019
3020 index = pos >> PAGE_SHIFT;
3021
3022 if (ext4_nonda_switch(inode->i_sb) ||
3023 S_ISLNK(inode->i_mode)) {
3024 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
3025 return ext4_write_begin(file, mapping, pos,
3026 len, flags, pagep, fsdata);
3027 }
3028 *fsdata = (void *)0;
3029 trace_ext4_da_write_begin(inode, pos, len, flags);
3030
3031 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
3032 ret = ext4_da_write_inline_data_begin(mapping, inode,
3033 pos, len, flags,
3034 pagep, fsdata);
3035 if (ret < 0)
3036 return ret;
3037 if (ret == 1)
3038 return 0;
3039 }
3040
3041
3042
3043
3044
3045
3046
3047
3048retry_grab:
3049 page = grab_cache_page_write_begin(mapping, index, flags);
3050 if (!page)
3051 return -ENOMEM;
3052 unlock_page(page);
3053
3054
3055
3056
3057
3058
3059
3060retry_journal:
3061 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
3062 ext4_da_write_credits(inode, pos, len));
3063 if (IS_ERR(handle)) {
3064 put_page(page);
3065 return PTR_ERR(handle);
3066 }
3067
3068 lock_page(page);
3069 if (page->mapping != mapping) {
3070
3071 unlock_page(page);
3072 put_page(page);
3073 ext4_journal_stop(handle);
3074 goto retry_grab;
3075 }
3076
3077 wait_for_stable_page(page);
3078
3079#ifdef CONFIG_EXT4_FS_ENCRYPTION
3080 ret = ext4_block_write_begin(page, pos, len,
3081 ext4_da_get_block_prep);
3082#else
3083 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
3084#endif
3085 if (ret < 0) {
3086 unlock_page(page);
3087 ext4_journal_stop(handle);
3088
3089
3090
3091
3092
3093 if (pos + len > inode->i_size)
3094 ext4_truncate_failed_write(inode);
3095
3096 if (ret == -ENOSPC &&
3097 ext4_should_retry_alloc(inode->i_sb, &retries))
3098 goto retry_journal;
3099
3100 put_page(page);
3101 return ret;
3102 }
3103
3104 *pagep = page;
3105 return ret;
3106}
3107
3108
3109
3110
3111
3112static int ext4_da_should_update_i_disksize(struct page *page,
3113 unsigned long offset)
3114{
3115 struct buffer_head *bh;
3116 struct inode *inode = page->mapping->host;
3117 unsigned int idx;
3118 int i;
3119
3120 bh = page_buffers(page);
3121 idx = offset >> inode->i_blkbits;
3122
3123 for (i = 0; i < idx; i++)
3124 bh = bh->b_this_page;
3125
3126 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
3127 return 0;
3128 return 1;
3129}
3130
3131static int ext4_da_write_end(struct file *file,
3132 struct address_space *mapping,
3133 loff_t pos, unsigned len, unsigned copied,
3134 struct page *page, void *fsdata)
3135{
3136 struct inode *inode = mapping->host;
3137 int ret = 0, ret2;
3138 handle_t *handle = ext4_journal_current_handle();
3139 loff_t new_i_size;
3140 unsigned long start, end;
3141 int write_mode = (int)(unsigned long)fsdata;
3142
3143 if (write_mode == FALL_BACK_TO_NONDELALLOC)
3144 return ext4_write_end(file, mapping, pos,
3145 len, copied, page, fsdata);
3146
3147 trace_ext4_da_write_end(inode, pos, len, copied);
3148 start = pos & (PAGE_SIZE - 1);
3149 end = start + copied - 1;
3150
3151
3152
3153
3154
3155
3156 new_i_size = pos + copied;
3157 if (copied && new_i_size > EXT4_I(inode)->i_disksize) {
3158 if (ext4_has_inline_data(inode) ||
3159 ext4_da_should_update_i_disksize(page, end)) {
3160 ext4_update_i_disksize(inode, new_i_size);
3161
3162
3163
3164
3165 ext4_mark_inode_dirty(handle, inode);
3166 }
3167 }
3168
3169 if (write_mode != CONVERT_INLINE_DATA &&
3170 ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
3171 ext4_has_inline_data(inode))
3172 ret2 = ext4_da_write_inline_data_end(inode, pos, len, copied,
3173 page);
3174 else
3175 ret2 = generic_write_end(file, mapping, pos, len, copied,
3176 page, fsdata);
3177
3178 copied = ret2;
3179 if (ret2 < 0)
3180 ret = ret2;
3181 ret2 = ext4_journal_stop(handle);
3182 if (!ret)
3183 ret = ret2;
3184
3185 return ret ? ret : copied;
3186}
3187
3188static void ext4_da_invalidatepage(struct page *page, unsigned int offset,
3189 unsigned int length)
3190{
3191
3192
3193
3194 BUG_ON(!PageLocked(page));
3195 if (!page_has_buffers(page))
3196 goto out;
3197
3198 ext4_da_page_release_reservation(page, offset, length);
3199
3200out:
3201 ext4_invalidatepage(page, offset, length);
3202
3203 return;
3204}
3205
3206
3207
3208
3209int ext4_alloc_da_blocks(struct inode *inode)
3210{
3211 trace_ext4_alloc_da_blocks(inode);
3212
3213 if (!EXT4_I(inode)->i_reserved_data_blocks)
3214 return 0;
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247 return filemap_flush(inode->i_mapping);
3248}
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3265{
3266 struct inode *inode = mapping->host;
3267 journal_t *journal;
3268 int err;
3269
3270
3271
3272
3273 if (ext4_has_inline_data(inode))
3274 return 0;
3275
3276 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3277 test_opt(inode->i_sb, DELALLOC)) {
3278
3279
3280
3281
3282
3283 filemap_write_and_wait(mapping);
3284 }
3285
3286 if (EXT4_JOURNAL(inode) &&
3287 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306 ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
3307 journal = EXT4_JOURNAL(inode);
3308 jbd2_journal_lock_updates(journal);
3309 err = jbd2_journal_flush(journal);
3310 jbd2_journal_unlock_updates(journal);
3311
3312 if (err)
3313 return 0;
3314 }
3315
3316 return generic_block_bmap(mapping, block, ext4_get_block);
3317}
3318
3319static int ext4_readpage(struct file *file, struct page *page)
3320{
3321 int ret = -EAGAIN;
3322 struct inode *inode = page->mapping->host;
3323
3324 trace_ext4_readpage(page);
3325
3326 if (ext4_has_inline_data(inode))
3327 ret = ext4_readpage_inline(inode, page);
3328
3329 if (ret == -EAGAIN)
3330 return ext4_mpage_readpages(page->mapping, NULL, page, 1);
3331
3332 return ret;
3333}
3334
3335static int
3336ext4_readpages(struct file *file, struct address_space *mapping,
3337 struct list_head *pages, unsigned nr_pages)
3338{
3339 struct inode *inode = mapping->host;
3340
3341
3342 if (ext4_has_inline_data(inode))
3343 return 0;
3344
3345 return ext4_mpage_readpages(mapping, pages, NULL, nr_pages);
3346}
3347
3348static void ext4_invalidatepage(struct page *page, unsigned int offset,
3349 unsigned int length)
3350{
3351 trace_ext4_invalidatepage(page, offset, length);
3352
3353
3354 WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page)));
3355
3356 block_invalidatepage(page, offset, length);
3357}
3358
3359static int __ext4_journalled_invalidatepage(struct page *page,
3360 unsigned int offset,
3361 unsigned int length)
3362{
3363 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3364
3365 trace_ext4_journalled_invalidatepage(page, offset, length);
3366
3367
3368
3369
3370 if (offset == 0 && length == PAGE_SIZE)
3371 ClearPageChecked(page);
3372
3373 return jbd2_journal_invalidatepage(journal, page, offset, length);
3374}
3375
3376
3377static void ext4_journalled_invalidatepage(struct page *page,
3378 unsigned int offset,
3379 unsigned int length)
3380{
3381 WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0);
3382}
3383
3384static int ext4_releasepage(struct page *page, gfp_t wait)
3385{
3386 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3387
3388 trace_ext4_releasepage(page);
3389
3390
3391 if (PageChecked(page))
3392 return 0;
3393 if (journal)
3394 return jbd2_journal_try_to_free_buffers(journal, page, wait);
3395 else
3396 return try_to_free_buffers(page);
3397}
3398
3399static bool ext4_inode_datasync_dirty(struct inode *inode)
3400{
3401 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
3402
3403 if (journal)
3404 return !jbd2_transaction_committed(journal,
3405 EXT4_I(inode)->i_datasync_tid);
3406
3407 if (!list_empty(&inode->i_mapping->private_list))
3408 return true;
3409 return inode->i_state & I_DIRTY_DATASYNC;
3410}
3411
3412static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
3413 unsigned flags, struct iomap *iomap)
3414{
3415 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3416 unsigned int blkbits = inode->i_blkbits;
3417 unsigned long first_block = offset >> blkbits;
3418 unsigned long last_block = (offset + length - 1) >> blkbits;
3419 struct ext4_map_blocks map;
3420 bool delalloc = false;
3421 int ret;
3422
3423
3424 if (flags & IOMAP_REPORT) {
3425 if (ext4_has_inline_data(inode)) {
3426 ret = ext4_inline_data_iomap(inode, iomap);
3427 if (ret != -EAGAIN) {
3428 if (ret == 0 && offset >= iomap->length)
3429 ret = -ENOENT;
3430 return ret;
3431 }
3432 }
3433 } else {
3434 if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3435 return -ERANGE;
3436 }
3437
3438 map.m_lblk = first_block;
3439 map.m_len = last_block - first_block + 1;
3440
3441 if (flags & IOMAP_REPORT) {
3442 ret = ext4_map_blocks(NULL, inode, &map, 0);
3443 if (ret < 0)
3444 return ret;
3445
3446 if (ret == 0) {
3447 ext4_lblk_t end = map.m_lblk + map.m_len - 1;
3448 struct extent_status es;
3449
3450 ext4_es_find_delayed_extent_range(inode, map.m_lblk, end, &es);
3451
3452 if (!es.es_len || es.es_lblk > end) {
3453
3454 } else if (es.es_lblk > map.m_lblk) {
3455
3456 map.m_len = es.es_lblk - map.m_lblk;
3457 } else {
3458 ext4_lblk_t offs = 0;
3459
3460 if (es.es_lblk < map.m_lblk)
3461 offs = map.m_lblk - es.es_lblk;
3462 map.m_lblk = es.es_lblk + offs;
3463 map.m_len = es.es_len - offs;
3464 delalloc = true;
3465 }
3466 }
3467 } else if (flags & IOMAP_WRITE) {
3468 int dio_credits;
3469 handle_t *handle;
3470 int retries = 0;
3471
3472
3473 if (map.m_len > DIO_MAX_BLOCKS)
3474 map.m_len = DIO_MAX_BLOCKS;
3475 dio_credits = ext4_chunk_trans_blocks(inode, map.m_len);
3476retry:
3477
3478
3479
3480
3481
3482
3483 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS,
3484 dio_credits);
3485 if (IS_ERR(handle))
3486 return PTR_ERR(handle);
3487
3488 ret = ext4_map_blocks(handle, inode, &map,
3489 EXT4_GET_BLOCKS_CREATE_ZERO);
3490 if (ret < 0) {
3491 ext4_journal_stop(handle);
3492 if (ret == -ENOSPC &&
3493 ext4_should_retry_alloc(inode->i_sb, &retries))
3494 goto retry;
3495 return ret;
3496 }
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507 if (!(flags & IOMAP_FAULT) && first_block + map.m_len >
3508 (i_size_read(inode) + (1 << blkbits) - 1) >> blkbits) {
3509 int err;
3510
3511 err = ext4_orphan_add(handle, inode);
3512 if (err < 0) {
3513 ext4_journal_stop(handle);
3514 return err;
3515 }
3516 }
3517 ext4_journal_stop(handle);
3518 } else {
3519 ret = ext4_map_blocks(NULL, inode, &map, 0);
3520 if (ret < 0)
3521 return ret;
3522 }
3523
3524 iomap->flags = 0;
3525 if (ext4_inode_datasync_dirty(inode))
3526 iomap->flags |= IOMAP_F_DIRTY;
3527 iomap->bdev = inode->i_sb->s_bdev;
3528 iomap->dax_dev = sbi->s_daxdev;
3529 iomap->offset = (u64)first_block << blkbits;
3530 iomap->length = (u64)map.m_len << blkbits;
3531
3532 if (ret == 0) {
3533 iomap->type = delalloc ? IOMAP_DELALLOC : IOMAP_HOLE;
3534 iomap->addr = IOMAP_NULL_ADDR;
3535 } else {
3536 if (map.m_flags & EXT4_MAP_MAPPED) {
3537 iomap->type = IOMAP_MAPPED;
3538 } else if (map.m_flags & EXT4_MAP_UNWRITTEN) {
3539 iomap->type = IOMAP_UNWRITTEN;
3540 } else {
3541 WARN_ON_ONCE(1);
3542 return -EIO;
3543 }
3544 iomap->addr = (u64)map.m_pblk << blkbits;
3545 }
3546
3547 if (map.m_flags & EXT4_MAP_NEW)
3548 iomap->flags |= IOMAP_F_NEW;
3549
3550 return 0;
3551}
3552
3553static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
3554 ssize_t written, unsigned flags, struct iomap *iomap)
3555{
3556 int ret = 0;
3557 handle_t *handle;
3558 int blkbits = inode->i_blkbits;
3559 bool truncate = false;
3560
3561 if (!(flags & IOMAP_WRITE) || (flags & IOMAP_FAULT))
3562 return 0;
3563
3564 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
3565 if (IS_ERR(handle)) {
3566 ret = PTR_ERR(handle);
3567 goto orphan_del;
3568 }
3569 if (ext4_update_inode_size(inode, offset + written))
3570 ext4_mark_inode_dirty(handle, inode);
3571
3572
3573
3574 if (iomap->offset + iomap->length >
3575 ALIGN(inode->i_size, 1 << blkbits)) {
3576 ext4_lblk_t written_blk, end_blk;
3577
3578 written_blk = (offset + written) >> blkbits;
3579 end_blk = (offset + length) >> blkbits;
3580 if (written_blk < end_blk && ext4_can_truncate(inode))
3581 truncate = true;
3582 }
3583
3584
3585
3586
3587 if (!truncate && inode->i_nlink &&
3588 !list_empty(&EXT4_I(inode)->i_orphan))
3589 ext4_orphan_del(handle, inode);
3590 ext4_journal_stop(handle);
3591 if (truncate) {
3592 ext4_truncate_failed_write(inode);
3593orphan_del:
3594
3595
3596
3597
3598
3599 if (inode->i_nlink)
3600 ext4_orphan_del(NULL, inode);
3601 }
3602 return ret;
3603}
3604
3605const struct iomap_ops ext4_iomap_ops = {
3606 .iomap_begin = ext4_iomap_begin,
3607 .iomap_end = ext4_iomap_end,
3608};
3609
3610static int ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
3611 ssize_t size, void *private)
3612{
3613 ext4_io_end_t *io_end = private;
3614
3615
3616 if (!io_end)
3617 return 0;
3618
3619 ext_debug("ext4_end_io_dio(): io_end 0x%p "
3620 "for inode %lu, iocb 0x%p, offset %llu, size %zd\n",
3621 io_end, io_end->inode->i_ino, iocb, offset, size);
3622
3623
3624
3625
3626
3627 if (size <= 0) {
3628 ext4_clear_io_unwritten_flag(io_end);
3629 size = 0;
3630 }
3631 io_end->offset = offset;
3632 io_end->size = size;
3633 ext4_put_io_end(io_end);
3634
3635 return 0;
3636}
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659static ssize_t ext4_direct_IO_write(struct kiocb *iocb, struct iov_iter *iter)
3660{
3661 struct file *file = iocb->ki_filp;
3662 struct inode *inode = file->f_mapping->host;
3663 struct ext4_inode_info *ei = EXT4_I(inode);
3664 ssize_t ret;
3665 loff_t offset = iocb->ki_pos;
3666 size_t count = iov_iter_count(iter);
3667 int overwrite = 0;
3668 get_block_t *get_block_func = NULL;
3669 int dio_flags = 0;
3670 loff_t final_size = offset + count;
3671 int orphan = 0;
3672 handle_t *handle;
3673
3674 if (final_size > inode->i_size || final_size > ei->i_disksize) {
3675
3676 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
3677 if (IS_ERR(handle)) {
3678 ret = PTR_ERR(handle);
3679 goto out;
3680 }
3681 ret = ext4_orphan_add(handle, inode);
3682 if (ret) {
3683 ext4_journal_stop(handle);
3684 goto out;
3685 }
3686 orphan = 1;
3687 ext4_update_i_disksize(inode, inode->i_size);
3688 ext4_journal_stop(handle);
3689 }
3690
3691 BUG_ON(iocb->private == NULL);
3692
3693
3694
3695
3696
3697
3698 inode_dio_begin(inode);
3699
3700
3701 overwrite = *((int *)iocb->private);
3702
3703 if (overwrite)
3704 inode_unlock(inode);
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726 iocb->private = NULL;
3727 if (overwrite)
3728 get_block_func = ext4_dio_get_block_overwrite;
3729 else if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) ||
3730 round_down(offset, i_blocksize(inode)) >= inode->i_size) {
3731 get_block_func = ext4_dio_get_block;
3732 dio_flags = DIO_LOCKING | DIO_SKIP_HOLES;
3733 } else if (is_sync_kiocb(iocb)) {
3734 get_block_func = ext4_dio_get_block_unwritten_sync;
3735 dio_flags = DIO_LOCKING;
3736 } else {
3737 get_block_func = ext4_dio_get_block_unwritten_async;
3738 dio_flags = DIO_LOCKING;
3739 }
3740 ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, iter,
3741 get_block_func, ext4_end_io_dio, NULL,
3742 dio_flags);
3743
3744 if (ret > 0 && !overwrite && ext4_test_inode_state(inode,
3745 EXT4_STATE_DIO_UNWRITTEN)) {
3746 int err;
3747
3748
3749
3750
3751 err = ext4_convert_unwritten_extents(NULL, inode,
3752 offset, ret);
3753 if (err < 0)
3754 ret = err;
3755 ext4_clear_inode_state(inode, EXT4_STATE_DIO_UNWRITTEN);
3756 }
3757
3758 inode_dio_end(inode);
3759
3760 if (overwrite)
3761 inode_lock(inode);
3762
3763 if (ret < 0 && final_size > inode->i_size)
3764 ext4_truncate_failed_write(inode);
3765
3766
3767 if (orphan) {
3768 int err;
3769
3770
3771 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
3772 if (IS_ERR(handle)) {
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783 if (!ret)
3784 ret = PTR_ERR(handle);
3785 if (inode->i_nlink)
3786 ext4_orphan_del(NULL, inode);
3787
3788 goto out;
3789 }
3790 if (inode->i_nlink)
3791 ext4_orphan_del(handle, inode);
3792 if (ret > 0) {
3793 loff_t end = offset + ret;
3794 if (end > inode->i_size || end > ei->i_disksize) {
3795 ext4_update_i_disksize(inode, end);
3796 if (end > inode->i_size)
3797 i_size_write(inode, end);
3798
3799
3800
3801
3802
3803
3804
3805 ext4_mark_inode_dirty(handle, inode);
3806 }
3807 }
3808 err = ext4_journal_stop(handle);
3809 if (ret == 0)
3810 ret = err;
3811 }
3812out:
3813 return ret;
3814}
3815
3816static ssize_t ext4_direct_IO_read(struct kiocb *iocb, struct iov_iter *iter)
3817{
3818 struct address_space *mapping = iocb->ki_filp->f_mapping;
3819 struct inode *inode = mapping->host;
3820 size_t count = iov_iter_count(iter);
3821 ssize_t ret;
3822
3823
3824
3825
3826
3827
3828 inode_lock_shared(inode);
3829 ret = filemap_write_and_wait_range(mapping, iocb->ki_pos,
3830 iocb->ki_pos + count - 1);
3831 if (ret)
3832 goto out_unlock;
3833 ret = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev,
3834 iter, ext4_dio_get_block, NULL, NULL, 0);
3835out_unlock:
3836 inode_unlock_shared(inode);
3837 return ret;
3838}
3839
3840static ssize_t ext4_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
3841{
3842 struct file *file = iocb->ki_filp;
3843 struct inode *inode = file->f_mapping->host;
3844 size_t count = iov_iter_count(iter);
3845 loff_t offset = iocb->ki_pos;
3846 ssize_t ret;
3847
3848#ifdef CONFIG_EXT4_FS_ENCRYPTION
3849 if (ext4_encrypted_inode(inode) && S_ISREG(inode->i_mode))
3850 return 0;
3851#endif
3852
3853
3854
3855
3856 if (ext4_should_journal_data(inode))
3857 return 0;
3858
3859
3860 if (ext4_has_inline_data(inode))
3861 return 0;
3862
3863 trace_ext4_direct_IO_enter(inode, offset, count, iov_iter_rw(iter));
3864 if (iov_iter_rw(iter) == READ)
3865 ret = ext4_direct_IO_read(iocb, iter);
3866 else
3867 ret = ext4_direct_IO_write(iocb, iter);
3868 trace_ext4_direct_IO_exit(inode, offset, count, iov_iter_rw(iter), ret);
3869 return ret;
3870}
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885static int ext4_journalled_set_page_dirty(struct page *page)
3886{
3887 SetPageChecked(page);
3888 return __set_page_dirty_nobuffers(page);
3889}
3890
3891static int ext4_set_page_dirty(struct page *page)
3892{
3893 WARN_ON_ONCE(!PageLocked(page) && !PageDirty(page));
3894 WARN_ON_ONCE(!page_has_buffers(page));
3895 return __set_page_dirty_buffers(page);
3896}
3897
3898static const struct address_space_operations ext4_aops = {
3899 .readpage = ext4_readpage,
3900 .readpages = ext4_readpages,
3901 .writepage = ext4_writepage,
3902 .writepages = ext4_writepages,
3903 .write_begin = ext4_write_begin,
3904 .write_end = ext4_write_end,
3905 .set_page_dirty = ext4_set_page_dirty,
3906 .bmap = ext4_bmap,
3907 .invalidatepage = ext4_invalidatepage,
3908 .releasepage = ext4_releasepage,
3909 .direct_IO = ext4_direct_IO,
3910 .migratepage = buffer_migrate_page,
3911 .is_partially_uptodate = block_is_partially_uptodate,
3912 .error_remove_page = generic_error_remove_page,
3913};
3914
3915static const struct address_space_operations ext4_journalled_aops = {
3916 .readpage = ext4_readpage,
3917 .readpages = ext4_readpages,
3918 .writepage = ext4_writepage,
3919 .writepages = ext4_writepages,
3920 .write_begin = ext4_write_begin,
3921 .write_end = ext4_journalled_write_end,
3922 .set_page_dirty = ext4_journalled_set_page_dirty,
3923 .bmap = ext4_bmap,
3924 .invalidatepage = ext4_journalled_invalidatepage,
3925 .releasepage = ext4_releasepage,
3926 .direct_IO = ext4_direct_IO,
3927 .is_partially_uptodate = block_is_partially_uptodate,
3928 .error_remove_page = generic_error_remove_page,
3929};
3930
3931static const struct address_space_operations ext4_da_aops = {
3932 .readpage = ext4_readpage,
3933 .readpages = ext4_readpages,
3934 .writepage = ext4_writepage,
3935 .writepages = ext4_writepages,
3936 .write_begin = ext4_da_write_begin,
3937 .write_end = ext4_da_write_end,
3938 .set_page_dirty = ext4_set_page_dirty,
3939 .bmap = ext4_bmap,
3940 .invalidatepage = ext4_da_invalidatepage,
3941 .releasepage = ext4_releasepage,
3942 .direct_IO = ext4_direct_IO,
3943 .migratepage = buffer_migrate_page,
3944 .is_partially_uptodate = block_is_partially_uptodate,
3945 .error_remove_page = generic_error_remove_page,
3946};
3947
3948static const struct address_space_operations ext4_dax_aops = {
3949 .writepages = ext4_dax_writepages,
3950 .direct_IO = noop_direct_IO,
3951 .set_page_dirty = noop_set_page_dirty,
3952 .invalidatepage = noop_invalidatepage,
3953};
3954
3955void ext4_set_aops(struct inode *inode)
3956{
3957 switch (ext4_inode_journal_mode(inode)) {
3958 case EXT4_INODE_ORDERED_DATA_MODE:
3959 case EXT4_INODE_WRITEBACK_DATA_MODE:
3960 break;
3961 case EXT4_INODE_JOURNAL_DATA_MODE:
3962 inode->i_mapping->a_ops = &ext4_journalled_aops;
3963 return;
3964 default:
3965 BUG();
3966 }
3967 if (IS_DAX(inode))
3968 inode->i_mapping->a_ops = &ext4_dax_aops;
3969 else if (test_opt(inode->i_sb, DELALLOC))
3970 inode->i_mapping->a_ops = &ext4_da_aops;
3971 else
3972 inode->i_mapping->a_ops = &ext4_aops;
3973}
3974
3975static int __ext4_block_zero_page_range(handle_t *handle,
3976 struct address_space *mapping, loff_t from, loff_t length)
3977{
3978 ext4_fsblk_t index = from >> PAGE_SHIFT;
3979 unsigned offset = from & (PAGE_SIZE-1);
3980 unsigned blocksize, pos;
3981 ext4_lblk_t iblock;
3982 struct inode *inode = mapping->host;
3983 struct buffer_head *bh;
3984 struct page *page;
3985 int err = 0;
3986
3987 page = find_or_create_page(mapping, from >> PAGE_SHIFT,
3988 mapping_gfp_constraint(mapping, ~__GFP_FS));
3989 if (!page)
3990 return -ENOMEM;
3991
3992 blocksize = inode->i_sb->s_blocksize;
3993
3994 iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
3995
3996 if (!page_has_buffers(page))
3997 create_empty_buffers(page, blocksize, 0);
3998
3999
4000 bh = page_buffers(page);
4001 pos = blocksize;
4002 while (offset >= pos) {
4003 bh = bh->b_this_page;
4004 iblock++;
4005 pos += blocksize;
4006 }
4007 if (buffer_freed(bh)) {
4008 BUFFER_TRACE(bh, "freed: skip");
4009 goto unlock;
4010 }
4011 if (!buffer_mapped(bh)) {
4012 BUFFER_TRACE(bh, "unmapped");
4013 ext4_get_block(inode, iblock, bh, 0);
4014
4015 if (!buffer_mapped(bh)) {
4016 BUFFER_TRACE(bh, "still unmapped");
4017 goto unlock;
4018 }
4019 }
4020
4021
4022 if (PageUptodate(page))
4023 set_buffer_uptodate(bh);
4024
4025 if (!buffer_uptodate(bh)) {
4026 err = -EIO;
4027 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
4028 wait_on_buffer(bh);
4029
4030 if (!buffer_uptodate(bh))
4031 goto unlock;
4032 if (S_ISREG(inode->i_mode) &&
4033 ext4_encrypted_inode(inode)) {
4034
4035 BUG_ON(!fscrypt_has_encryption_key(inode));
4036 BUG_ON(blocksize != PAGE_SIZE);
4037 WARN_ON_ONCE(fscrypt_decrypt_page(page->mapping->host,
4038 page, PAGE_SIZE, 0, page->index));
4039 }
4040 }
4041 if (ext4_should_journal_data(inode)) {
4042 BUFFER_TRACE(bh, "get write access");
4043 err = ext4_journal_get_write_access(handle, bh);
4044 if (err)
4045 goto unlock;
4046 }
4047 zero_user(page, offset, length);
4048 BUFFER_TRACE(bh, "zeroed end of block");
4049
4050 if (ext4_should_journal_data(inode)) {
4051 err = ext4_handle_dirty_metadata(handle, inode, bh);
4052 } else {
4053 err = 0;
4054 mark_buffer_dirty(bh);
4055 if (ext4_should_order_data(inode))
4056 err = ext4_jbd2_inode_add_write(handle, inode);
4057 }
4058
4059unlock:
4060 unlock_page(page);
4061 put_page(page);
4062 return err;
4063}
4064
4065
4066
4067
4068
4069
4070
4071
4072static int ext4_block_zero_page_range(handle_t *handle,
4073 struct address_space *mapping, loff_t from, loff_t length)
4074{
4075 struct inode *inode = mapping->host;
4076 unsigned offset = from & (PAGE_SIZE-1);
4077 unsigned blocksize = inode->i_sb->s_blocksize;
4078 unsigned max = blocksize - (offset & (blocksize - 1));
4079
4080
4081
4082
4083
4084 if (length > max || length < 0)
4085 length = max;
4086
4087 if (IS_DAX(inode)) {
4088 return iomap_zero_range(inode, from, length, NULL,
4089 &ext4_iomap_ops);
4090 }
4091 return __ext4_block_zero_page_range(handle, mapping, from, length);
4092}
4093
4094
4095
4096
4097
4098
4099
4100static int ext4_block_truncate_page(handle_t *handle,
4101 struct address_space *mapping, loff_t from)
4102{
4103 unsigned offset = from & (PAGE_SIZE-1);
4104 unsigned length;
4105 unsigned blocksize;
4106 struct inode *inode = mapping->host;
4107
4108
4109 if (ext4_encrypted_inode(inode) && !fscrypt_has_encryption_key(inode))
4110 return 0;
4111
4112 blocksize = inode->i_sb->s_blocksize;
4113 length = blocksize - (offset & (blocksize - 1));
4114
4115 return ext4_block_zero_page_range(handle, mapping, from, length);
4116}
4117
4118int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
4119 loff_t lstart, loff_t length)
4120{
4121 struct super_block *sb = inode->i_sb;
4122 struct address_space *mapping = inode->i_mapping;
4123 unsigned partial_start, partial_end;
4124 ext4_fsblk_t start, end;
4125 loff_t byte_end = (lstart + length - 1);
4126 int err = 0;
4127
4128 partial_start = lstart & (sb->s_blocksize - 1);
4129 partial_end = byte_end & (sb->s_blocksize - 1);
4130
4131 start = lstart >> sb->s_blocksize_bits;
4132 end = byte_end >> sb->s_blocksize_bits;
4133
4134
4135 if (start == end &&
4136 (partial_start || (partial_end != sb->s_blocksize - 1))) {
4137 err = ext4_block_zero_page_range(handle, mapping,
4138 lstart, length);
4139 return err;
4140 }
4141
4142 if (partial_start) {
4143 err = ext4_block_zero_page_range(handle, mapping,
4144 lstart, sb->s_blocksize);
4145 if (err)
4146 return err;
4147 }
4148
4149 if (partial_end != sb->s_blocksize - 1)
4150 err = ext4_block_zero_page_range(handle, mapping,
4151 byte_end - partial_end,
4152 partial_end + 1);
4153 return err;
4154}
4155
4156int ext4_can_truncate(struct inode *inode)
4157{
4158 if (S_ISREG(inode->i_mode))
4159 return 1;
4160 if (S_ISDIR(inode->i_mode))
4161 return 1;
4162 if (S_ISLNK(inode->i_mode))
4163 return !ext4_inode_is_fast_symlink(inode);
4164 return 0;
4165}
4166
4167
4168
4169
4170
4171
4172
4173int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
4174 loff_t len)
4175{
4176 handle_t *handle;
4177 loff_t size = i_size_read(inode);
4178
4179 WARN_ON(!inode_is_locked(inode));
4180 if (offset > size || offset + len < size)
4181 return 0;
4182
4183 if (EXT4_I(inode)->i_disksize >= size)
4184 return 0;
4185
4186 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
4187 if (IS_ERR(handle))
4188 return PTR_ERR(handle);
4189 ext4_update_i_disksize(inode, size);
4190 ext4_mark_inode_dirty(handle, inode);
4191 ext4_journal_stop(handle);
4192
4193 return 0;
4194}
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
4208{
4209 struct super_block *sb = inode->i_sb;
4210 ext4_lblk_t first_block, stop_block;
4211 struct address_space *mapping = inode->i_mapping;
4212 loff_t first_block_offset, last_block_offset;
4213 handle_t *handle;
4214 unsigned int credits;
4215 int ret = 0;
4216
4217 if (!S_ISREG(inode->i_mode))
4218 return -EOPNOTSUPP;
4219
4220 trace_ext4_punch_hole(inode, offset, length, 0);
4221
4222
4223
4224
4225
4226 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
4227 ret = filemap_write_and_wait_range(mapping, offset,
4228 offset + length - 1);
4229 if (ret)
4230 return ret;
4231 }
4232
4233 inode_lock(inode);
4234
4235
4236 if (offset >= inode->i_size)
4237 goto out_mutex;
4238
4239
4240
4241
4242
4243 if (offset + length > inode->i_size) {
4244 length = inode->i_size +
4245 PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -
4246 offset;
4247 }
4248
4249 if (offset & (sb->s_blocksize - 1) ||
4250 (offset + length) & (sb->s_blocksize - 1)) {
4251
4252
4253
4254
4255 ret = ext4_inode_attach_jinode(inode);
4256 if (ret < 0)
4257 goto out_mutex;
4258
4259 }
4260
4261
4262 inode_dio_wait(inode);
4263
4264
4265
4266
4267
4268 down_write(&EXT4_I(inode)->i_mmap_sem);
4269 first_block_offset = round_up(offset, sb->s_blocksize);
4270 last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;
4271
4272
4273 if (last_block_offset > first_block_offset) {
4274 ret = ext4_update_disksize_before_punch(inode, offset, length);
4275 if (ret)
4276 goto out_dio;
4277 truncate_pagecache_range(inode, first_block_offset,
4278 last_block_offset);
4279 }
4280
4281 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4282 credits = ext4_writepage_trans_blocks(inode);
4283 else
4284 credits = ext4_blocks_for_truncate(inode);
4285 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4286 if (IS_ERR(handle)) {
4287 ret = PTR_ERR(handle);
4288 ext4_std_error(sb, ret);
4289 goto out_dio;
4290 }
4291
4292 ret = ext4_zero_partial_blocks(handle, inode, offset,
4293 length);
4294 if (ret)
4295 goto out_stop;
4296
4297 first_block = (offset + sb->s_blocksize - 1) >>
4298 EXT4_BLOCK_SIZE_BITS(sb);
4299 stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);
4300
4301
4302 if (first_block >= stop_block)
4303 goto out_stop;
4304
4305 down_write(&EXT4_I(inode)->i_data_sem);
4306 ext4_discard_preallocations(inode);
4307
4308 ret = ext4_es_remove_extent(inode, first_block,
4309 stop_block - first_block);
4310 if (ret) {
4311 up_write(&EXT4_I(inode)->i_data_sem);
4312 goto out_stop;
4313 }
4314
4315 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4316 ret = ext4_ext_remove_space(inode, first_block,
4317 stop_block - 1);
4318 else
4319 ret = ext4_ind_remove_space(handle, inode, first_block,
4320 stop_block);
4321
4322 up_write(&EXT4_I(inode)->i_data_sem);
4323 if (IS_SYNC(inode))
4324 ext4_handle_sync(handle);
4325
4326 inode->i_mtime = inode->i_ctime = current_time(inode);
4327 ext4_mark_inode_dirty(handle, inode);
4328 if (ret >= 0)
4329 ext4_update_inode_fsync_trans(handle, inode, 1);
4330out_stop:
4331 ext4_journal_stop(handle);
4332out_dio:
4333 up_write(&EXT4_I(inode)->i_mmap_sem);
4334out_mutex:
4335 inode_unlock(inode);
4336 return ret;
4337}
4338
4339int ext4_inode_attach_jinode(struct inode *inode)
4340{
4341 struct ext4_inode_info *ei = EXT4_I(inode);
4342 struct jbd2_inode *jinode;
4343
4344 if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
4345 return 0;
4346
4347 jinode = jbd2_alloc_inode(GFP_KERNEL);
4348 spin_lock(&inode->i_lock);
4349 if (!ei->jinode) {
4350 if (!jinode) {
4351 spin_unlock(&inode->i_lock);
4352 return -ENOMEM;
4353 }
4354 ei->jinode = jinode;
4355 jbd2_journal_init_jbd_inode(ei->jinode, inode);
4356 jinode = NULL;
4357 }
4358 spin_unlock(&inode->i_lock);
4359 if (unlikely(jinode != NULL))
4360 jbd2_free_inode(jinode);
4361 return 0;
4362}
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392int ext4_truncate(struct inode *inode)
4393{
4394 struct ext4_inode_info *ei = EXT4_I(inode);
4395 unsigned int credits;
4396 int err = 0;
4397 handle_t *handle;
4398 struct address_space *mapping = inode->i_mapping;
4399
4400
4401
4402
4403
4404
4405 if (!(inode->i_state & (I_NEW|I_FREEING)))
4406 WARN_ON(!inode_is_locked(inode));
4407 trace_ext4_truncate_enter(inode);
4408
4409 if (!ext4_can_truncate(inode))
4410 return 0;
4411
4412 ext4_clear_inode_flag(inode, EXT4_INODE_EOFBLOCKS);
4413
4414 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
4415 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
4416
4417 if (ext4_has_inline_data(inode)) {
4418 int has_inline = 1;
4419
4420 err = ext4_inline_data_truncate(inode, &has_inline);
4421 if (err)
4422 return err;
4423 if (has_inline)
4424 return 0;
4425 }
4426
4427
4428 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
4429 if (ext4_inode_attach_jinode(inode) < 0)
4430 return 0;
4431 }
4432
4433 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4434 credits = ext4_writepage_trans_blocks(inode);
4435 else
4436 credits = ext4_blocks_for_truncate(inode);
4437
4438 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4439 if (IS_ERR(handle))
4440 return PTR_ERR(handle);
4441
4442 if (inode->i_size & (inode->i_sb->s_blocksize - 1))
4443 ext4_block_truncate_page(handle, mapping, inode->i_size);
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454 err = ext4_orphan_add(handle, inode);
4455 if (err)
4456 goto out_stop;
4457
4458 down_write(&EXT4_I(inode)->i_data_sem);
4459
4460 ext4_discard_preallocations(inode);
4461
4462 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4463 err = ext4_ext_truncate(handle, inode);
4464 else
4465 ext4_ind_truncate(handle, inode);
4466
4467 up_write(&ei->i_data_sem);
4468 if (err)
4469 goto out_stop;
4470
4471 if (IS_SYNC(inode))
4472 ext4_handle_sync(handle);
4473
4474out_stop:
4475
4476
4477
4478
4479
4480
4481
4482 if (inode->i_nlink)
4483 ext4_orphan_del(handle, inode);
4484
4485 inode->i_mtime = inode->i_ctime = current_time(inode);
4486 ext4_mark_inode_dirty(handle, inode);
4487 ext4_journal_stop(handle);
4488
4489 trace_ext4_truncate_exit(inode);
4490 return err;
4491}
4492
4493
4494
4495
4496
4497
4498
4499static int __ext4_get_inode_loc(struct inode *inode,
4500 struct ext4_iloc *iloc, int in_mem)
4501{
4502 struct ext4_group_desc *gdp;
4503 struct buffer_head *bh;
4504 struct super_block *sb = inode->i_sb;
4505 ext4_fsblk_t block;
4506 int inodes_per_block, inode_offset;
4507
4508 iloc->bh = NULL;
4509 if (!ext4_valid_inum(sb, inode->i_ino))
4510 return -EFSCORRUPTED;
4511
4512 iloc->block_group = (inode->i_ino - 1) / EXT4_INODES_PER_GROUP(sb);
4513 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
4514 if (!gdp)
4515 return -EIO;
4516
4517
4518
4519
4520 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4521 inode_offset = ((inode->i_ino - 1) %
4522 EXT4_INODES_PER_GROUP(sb));
4523 block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block);
4524 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
4525
4526 bh = sb_getblk(sb, block);
4527 if (unlikely(!bh))
4528 return -ENOMEM;
4529 if (!buffer_uptodate(bh)) {
4530 lock_buffer(bh);
4531
4532
4533
4534
4535
4536
4537
4538 if (buffer_write_io_error(bh) && !buffer_uptodate(bh))
4539 set_buffer_uptodate(bh);
4540
4541 if (buffer_uptodate(bh)) {
4542
4543 unlock_buffer(bh);
4544 goto has_buffer;
4545 }
4546
4547
4548
4549
4550
4551
4552 if (in_mem) {
4553 struct buffer_head *bitmap_bh;
4554 int i, start;
4555
4556 start = inode_offset & ~(inodes_per_block - 1);
4557
4558
4559 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
4560 if (unlikely(!bitmap_bh))
4561 goto make_io;
4562
4563
4564
4565
4566
4567
4568 if (!buffer_uptodate(bitmap_bh)) {
4569 brelse(bitmap_bh);
4570 goto make_io;
4571 }
4572 for (i = start; i < start + inodes_per_block; i++) {
4573 if (i == inode_offset)
4574 continue;
4575 if (ext4_test_bit(i, bitmap_bh->b_data))
4576 break;
4577 }
4578 brelse(bitmap_bh);
4579 if (i == start + inodes_per_block) {
4580
4581 memset(bh->b_data, 0, bh->b_size);
4582 set_buffer_uptodate(bh);
4583 unlock_buffer(bh);
4584 goto has_buffer;
4585 }
4586 }
4587
4588make_io:
4589
4590
4591
4592
4593 if (EXT4_SB(sb)->s_inode_readahead_blks) {
4594 ext4_fsblk_t b, end, table;
4595 unsigned num;
4596 __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
4597
4598 table = ext4_inode_table(sb, gdp);
4599
4600 b = block & ~((ext4_fsblk_t) ra_blks - 1);
4601 if (table > b)
4602 b = table;
4603 end = b + ra_blks;
4604 num = EXT4_INODES_PER_GROUP(sb);
4605 if (ext4_has_group_desc_csum(sb))
4606 num -= ext4_itable_unused_count(sb, gdp);
4607 table += num / inodes_per_block;
4608 if (end > table)
4609 end = table;
4610 while (b <= end)
4611 sb_breadahead(sb, b++);
4612 }
4613
4614
4615
4616
4617
4618
4619 trace_ext4_load_inode(inode);
4620 get_bh(bh);
4621 bh->b_end_io = end_buffer_read_sync;
4622 submit_bh(REQ_OP_READ, REQ_META | REQ_PRIO, bh);
4623 wait_on_buffer(bh);
4624 if (!buffer_uptodate(bh)) {
4625 EXT4_ERROR_INODE_BLOCK(inode, block,
4626 "unable to read itable block");
4627 brelse(bh);
4628 return -EIO;
4629 }
4630 }
4631has_buffer:
4632 iloc->bh = bh;
4633 return 0;
4634}
4635
4636int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
4637{
4638
4639 return __ext4_get_inode_loc(inode, iloc,
4640 !ext4_test_inode_state(inode, EXT4_STATE_XATTR));
4641}
4642
4643static bool ext4_should_use_dax(struct inode *inode)
4644{
4645 if (!test_opt(inode->i_sb, DAX))
4646 return false;
4647 if (!S_ISREG(inode->i_mode))
4648 return false;
4649 if (ext4_should_journal_data(inode))
4650 return false;
4651 if (ext4_has_inline_data(inode))
4652 return false;
4653 if (ext4_encrypted_inode(inode))
4654 return false;
4655 return true;
4656}
4657
4658void ext4_set_inode_flags(struct inode *inode)
4659{
4660 unsigned int flags = EXT4_I(inode)->i_flags;
4661 unsigned int new_fl = 0;
4662
4663 if (flags & EXT4_SYNC_FL)
4664 new_fl |= S_SYNC;
4665 if (flags & EXT4_APPEND_FL)
4666 new_fl |= S_APPEND;
4667 if (flags & EXT4_IMMUTABLE_FL)
4668 new_fl |= S_IMMUTABLE;
4669 if (flags & EXT4_NOATIME_FL)
4670 new_fl |= S_NOATIME;
4671 if (flags & EXT4_DIRSYNC_FL)
4672 new_fl |= S_DIRSYNC;
4673 if (ext4_should_use_dax(inode))
4674 new_fl |= S_DAX;
4675 if (flags & EXT4_ENCRYPT_FL)
4676 new_fl |= S_ENCRYPTED;
4677 inode_set_flags(inode, new_fl,
4678 S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
4679 S_ENCRYPTED);
4680}
4681
4682static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
4683 struct ext4_inode_info *ei)
4684{
4685 blkcnt_t i_blocks ;
4686 struct inode *inode = &(ei->vfs_inode);
4687 struct super_block *sb = inode->i_sb;
4688
4689 if (ext4_has_feature_huge_file(sb)) {
4690
4691 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
4692 le32_to_cpu(raw_inode->i_blocks_lo);
4693 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
4694
4695 return i_blocks << (inode->i_blkbits - 9);
4696 } else {
4697 return i_blocks;
4698 }
4699 } else {
4700 return le32_to_cpu(raw_inode->i_blocks_lo);
4701 }
4702}
4703
4704static inline void ext4_iget_extra_inode(struct inode *inode,
4705 struct ext4_inode *raw_inode,
4706 struct ext4_inode_info *ei)
4707{
4708 __le32 *magic = (void *)raw_inode +
4709 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
4710 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize + sizeof(__le32) <=
4711 EXT4_INODE_SIZE(inode->i_sb) &&
4712 *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
4713 ext4_set_inode_state(inode, EXT4_STATE_XATTR);
4714 ext4_find_inline_data_nolock(inode);
4715 } else
4716 EXT4_I(inode)->i_inline_off = 0;
4717}
4718
4719int ext4_get_projid(struct inode *inode, kprojid_t *projid)
4720{
4721 if (!ext4_has_feature_project(inode->i_sb))
4722 return -EOPNOTSUPP;
4723 *projid = EXT4_I(inode)->i_projid;
4724 return 0;
4725}
4726
4727struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
4728{
4729 struct ext4_iloc iloc;
4730 struct ext4_inode *raw_inode;
4731 struct ext4_inode_info *ei;
4732 struct inode *inode;
4733 journal_t *journal = EXT4_SB(sb)->s_journal;
4734 long ret;
4735 loff_t size;
4736 int block;
4737 uid_t i_uid;
4738 gid_t i_gid;
4739 projid_t i_projid;
4740
4741 inode = iget_locked(sb, ino);
4742 if (!inode)
4743 return ERR_PTR(-ENOMEM);
4744 if (!(inode->i_state & I_NEW))
4745 return inode;
4746
4747 ei = EXT4_I(inode);
4748 iloc.bh = NULL;
4749
4750 ret = __ext4_get_inode_loc(inode, &iloc, 0);
4751 if (ret < 0)
4752 goto bad_inode;
4753 raw_inode = ext4_raw_inode(&iloc);
4754
4755 if ((ino == EXT4_ROOT_INO) && (raw_inode->i_links_count == 0)) {
4756 EXT4_ERROR_INODE(inode, "root inode unallocated");
4757 ret = -EFSCORRUPTED;
4758 goto bad_inode;
4759 }
4760
4761 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4762 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
4763 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
4764 EXT4_INODE_SIZE(inode->i_sb) ||
4765 (ei->i_extra_isize & 3)) {
4766 EXT4_ERROR_INODE(inode,
4767 "bad extra_isize %u (inode size %u)",
4768 ei->i_extra_isize,
4769 EXT4_INODE_SIZE(inode->i_sb));
4770 ret = -EFSCORRUPTED;
4771 goto bad_inode;
4772 }
4773 } else
4774 ei->i_extra_isize = 0;
4775
4776
4777 if (ext4_has_metadata_csum(sb)) {
4778 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4779 __u32 csum;
4780 __le32 inum = cpu_to_le32(inode->i_ino);
4781 __le32 gen = raw_inode->i_generation;
4782 csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum,
4783 sizeof(inum));
4784 ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen,
4785 sizeof(gen));
4786 }
4787
4788 if (!ext4_inode_csum_verify(inode, raw_inode, ei)) {
4789 EXT4_ERROR_INODE(inode, "checksum invalid");
4790 ret = -EFSBADCRC;
4791 goto bad_inode;
4792 }
4793
4794 inode->i_mode = le16_to_cpu(raw_inode->i_mode);
4795 i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
4796 i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
4797 if (ext4_has_feature_project(sb) &&
4798 EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4799 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4800 i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
4801 else
4802 i_projid = EXT4_DEF_PROJID;
4803
4804 if (!(test_opt(inode->i_sb, NO_UID32))) {
4805 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
4806 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
4807 }
4808 i_uid_write(inode, i_uid);
4809 i_gid_write(inode, i_gid);
4810 ei->i_projid = make_kprojid(&init_user_ns, i_projid);
4811 set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
4812
4813 ext4_clear_state_flags(ei);
4814 ei->i_inline_off = 0;
4815 ei->i_dir_start_lookup = 0;
4816 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
4817
4818
4819
4820
4821
4822 if (inode->i_nlink == 0) {
4823 if ((inode->i_mode == 0 ||
4824 !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
4825 ino != EXT4_BOOT_LOADER_INO) {
4826
4827 ret = -ESTALE;
4828 goto bad_inode;
4829 }
4830
4831
4832
4833
4834
4835
4836 }
4837 ei->i_flags = le32_to_cpu(raw_inode->i_flags);
4838 inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
4839 ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
4840 if (ext4_has_feature_64bit(sb))
4841 ei->i_file_acl |=
4842 ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
4843 inode->i_size = ext4_isize(sb, raw_inode);
4844 if ((size = i_size_read(inode)) < 0) {
4845 EXT4_ERROR_INODE(inode, "bad i_size value: %lld", size);
4846 ret = -EFSCORRUPTED;
4847 goto bad_inode;
4848 }
4849 ei->i_disksize = inode->i_size;
4850#ifdef CONFIG_QUOTA
4851 ei->i_reserved_quota = 0;
4852#endif
4853 inode->i_generation = le32_to_cpu(raw_inode->i_generation);
4854 ei->i_block_group = iloc.block_group;
4855 ei->i_last_alloc_group = ~0;
4856
4857
4858
4859
4860 for (block = 0; block < EXT4_N_BLOCKS; block++)
4861 ei->i_data[block] = raw_inode->i_block[block];
4862 INIT_LIST_HEAD(&ei->i_orphan);
4863
4864
4865
4866
4867
4868
4869
4870
4871 if (journal) {
4872 transaction_t *transaction;
4873 tid_t tid;
4874
4875 read_lock(&journal->j_state_lock);
4876 if (journal->j_running_transaction)
4877 transaction = journal->j_running_transaction;
4878 else
4879 transaction = journal->j_committing_transaction;
4880 if (transaction)
4881 tid = transaction->t_tid;
4882 else
4883 tid = journal->j_commit_sequence;
4884 read_unlock(&journal->j_state_lock);
4885 ei->i_sync_tid = tid;
4886 ei->i_datasync_tid = tid;
4887 }
4888
4889 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4890 if (ei->i_extra_isize == 0) {
4891
4892 BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
4893 ei->i_extra_isize = sizeof(struct ext4_inode) -
4894 EXT4_GOOD_OLD_INODE_SIZE;
4895 } else {
4896 ext4_iget_extra_inode(inode, raw_inode, ei);
4897 }
4898 }
4899
4900 EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode);
4901 EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode);
4902 EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode);
4903 EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
4904
4905 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
4906 u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
4907
4908 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4909 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4910 ivers |=
4911 (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
4912 }
4913 inode_set_iversion_queried(inode, ivers);
4914 }
4915
4916 ret = 0;
4917 if (ei->i_file_acl &&
4918 !ext4_data_block_valid(EXT4_SB(sb), ei->i_file_acl, 1)) {
4919 EXT4_ERROR_INODE(inode, "bad extended attribute block %llu",
4920 ei->i_file_acl);
4921 ret = -EFSCORRUPTED;
4922 goto bad_inode;
4923 } else if (!ext4_has_inline_data(inode)) {
4924 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
4925 if ((S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
4926 (S_ISLNK(inode->i_mode) &&
4927 !ext4_inode_is_fast_symlink(inode))))
4928
4929 ret = ext4_ext_check_inode(inode);
4930 } else if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
4931 (S_ISLNK(inode->i_mode) &&
4932 !ext4_inode_is_fast_symlink(inode))) {
4933
4934 ret = ext4_ind_check_inode(inode);
4935 }
4936 }
4937 if (ret)
4938 goto bad_inode;
4939
4940 if (S_ISREG(inode->i_mode)) {
4941 inode->i_op = &ext4_file_inode_operations;
4942 inode->i_fop = &ext4_file_operations;
4943 ext4_set_aops(inode);
4944 } else if (S_ISDIR(inode->i_mode)) {
4945 inode->i_op = &ext4_dir_inode_operations;
4946 inode->i_fop = &ext4_dir_operations;
4947 } else if (S_ISLNK(inode->i_mode)) {
4948 if (ext4_encrypted_inode(inode)) {
4949 inode->i_op = &ext4_encrypted_symlink_inode_operations;
4950 ext4_set_aops(inode);
4951 } else if (ext4_inode_is_fast_symlink(inode)) {
4952 inode->i_link = (char *)ei->i_data;
4953 inode->i_op = &ext4_fast_symlink_inode_operations;
4954 nd_terminate_link(ei->i_data, inode->i_size,
4955 sizeof(ei->i_data) - 1);
4956 } else {
4957 inode->i_op = &ext4_symlink_inode_operations;
4958 ext4_set_aops(inode);
4959 }
4960 inode_nohighmem(inode);
4961 } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
4962 S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
4963 inode->i_op = &ext4_special_inode_operations;
4964 if (raw_inode->i_block[0])
4965 init_special_inode(inode, inode->i_mode,
4966 old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
4967 else
4968 init_special_inode(inode, inode->i_mode,
4969 new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
4970 } else if (ino == EXT4_BOOT_LOADER_INO) {
4971 make_bad_inode(inode);
4972 } else {
4973 ret = -EFSCORRUPTED;
4974 EXT4_ERROR_INODE(inode, "bogus i_mode (%o)", inode->i_mode);
4975 goto bad_inode;
4976 }
4977 brelse(iloc.bh);
4978 ext4_set_inode_flags(inode);
4979
4980 unlock_new_inode(inode);
4981 return inode;
4982
4983bad_inode:
4984 brelse(iloc.bh);
4985 iget_failed(inode);
4986 return ERR_PTR(ret);
4987}
4988
4989struct inode *ext4_iget_normal(struct super_block *sb, unsigned long ino)
4990{
4991 if (ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO)
4992 return ERR_PTR(-EFSCORRUPTED);
4993 return ext4_iget(sb, ino);
4994}
4995
4996static int ext4_inode_blocks_set(handle_t *handle,
4997 struct ext4_inode *raw_inode,
4998 struct ext4_inode_info *ei)
4999{
5000 struct inode *inode = &(ei->vfs_inode);
5001 u64 i_blocks = inode->i_blocks;
5002 struct super_block *sb = inode->i_sb;
5003
5004 if (i_blocks <= ~0U) {
5005
5006
5007
5008
5009 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
5010 raw_inode->i_blocks_high = 0;
5011 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
5012 return 0;
5013 }
5014 if (!ext4_has_feature_huge_file(sb))
5015 return -EFBIG;
5016
5017 if (i_blocks <= 0xffffffffffffULL) {
5018
5019
5020
5021
5022 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
5023 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
5024 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
5025 } else {
5026 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
5027
5028 i_blocks = i_blocks >> (inode->i_blkbits - 9);
5029 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
5030 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
5031 }
5032 return 0;
5033}
5034
5035struct other_inode {
5036 unsigned long orig_ino;
5037 struct ext4_inode *raw_inode;
5038};
5039
5040static int other_inode_match(struct inode * inode, unsigned long ino,
5041 void *data)
5042{
5043 struct other_inode *oi = (struct other_inode *) data;
5044
5045 if ((inode->i_ino != ino) ||
5046 (inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW |
5047 I_DIRTY_INODE)) ||
5048 ((inode->i_state & I_DIRTY_TIME) == 0))
5049 return 0;
5050 spin_lock(&inode->i_lock);
5051 if (((inode->i_state & (I_FREEING | I_WILL_FREE | I_NEW |
5052 I_DIRTY_INODE)) == 0) &&
5053 (inode->i_state & I_DIRTY_TIME)) {
5054 struct ext4_inode_info *ei = EXT4_I(inode);
5055
5056 inode->i_state &= ~(I_DIRTY_TIME | I_DIRTY_TIME_EXPIRED);
5057 spin_unlock(&inode->i_lock);
5058
5059 spin_lock(&ei->i_raw_lock);
5060 EXT4_INODE_SET_XTIME(i_ctime, inode, oi->raw_inode);
5061 EXT4_INODE_SET_XTIME(i_mtime, inode, oi->raw_inode);
5062 EXT4_INODE_SET_XTIME(i_atime, inode, oi->raw_inode);
5063 ext4_inode_csum_set(inode, oi->raw_inode, ei);
5064 spin_unlock(&ei->i_raw_lock);
5065 trace_ext4_other_inode_update_time(inode, oi->orig_ino);
5066 return -1;
5067 }
5068 spin_unlock(&inode->i_lock);
5069 return -1;
5070}
5071
5072
5073
5074
5075
5076static void ext4_update_other_inodes_time(struct super_block *sb,
5077 unsigned long orig_ino, char *buf)
5078{
5079 struct other_inode oi;
5080 unsigned long ino;
5081 int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
5082 int inode_size = EXT4_INODE_SIZE(sb);
5083
5084 oi.orig_ino = orig_ino;
5085
5086
5087
5088
5089
5090 ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
5091 for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
5092 if (ino == orig_ino)
5093 continue;
5094 oi.raw_inode = (struct ext4_inode *) buf;
5095 (void) find_inode_nowait(sb, ino, other_inode_match, &oi);
5096 }
5097}
5098
5099
5100
5101
5102
5103
5104
5105
5106static int ext4_do_update_inode(handle_t *handle,
5107 struct inode *inode,
5108 struct ext4_iloc *iloc)
5109{
5110 struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
5111 struct ext4_inode_info *ei = EXT4_I(inode);
5112 struct buffer_head *bh = iloc->bh;
5113 struct super_block *sb = inode->i_sb;
5114 int err = 0, rc, block;
5115 int need_datasync = 0, set_large_file = 0;
5116 uid_t i_uid;
5117 gid_t i_gid;
5118 projid_t i_projid;
5119
5120 spin_lock(&ei->i_raw_lock);
5121
5122
5123
5124 if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
5125 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
5126
5127 raw_inode->i_mode = cpu_to_le16(inode->i_mode);
5128 i_uid = i_uid_read(inode);
5129 i_gid = i_gid_read(inode);
5130 i_projid = from_kprojid(&init_user_ns, ei->i_projid);
5131 if (!(test_opt(inode->i_sb, NO_UID32))) {
5132 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
5133 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
5134
5135
5136
5137
5138 if (ei->i_dtime && list_empty(&ei->i_orphan)) {
5139 raw_inode->i_uid_high = 0;
5140 raw_inode->i_gid_high = 0;
5141 } else {
5142 raw_inode->i_uid_high =
5143 cpu_to_le16(high_16_bits(i_uid));
5144 raw_inode->i_gid_high =
5145 cpu_to_le16(high_16_bits(i_gid));
5146 }
5147 } else {
5148 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
5149 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
5150 raw_inode->i_uid_high = 0;
5151 raw_inode->i_gid_high = 0;
5152 }
5153 raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
5154
5155 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
5156 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
5157 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
5158 EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
5159
5160 err = ext4_inode_blocks_set(handle, raw_inode, ei);
5161 if (err) {
5162 spin_unlock(&ei->i_raw_lock);
5163 goto out_brelse;
5164 }
5165 raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
5166 raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
5167 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
5168 raw_inode->i_file_acl_high =
5169 cpu_to_le16(ei->i_file_acl >> 32);
5170 raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
5171 if (ei->i_disksize != ext4_isize(inode->i_sb, raw_inode)) {
5172 ext4_isize_set(raw_inode, ei->i_disksize);
5173 need_datasync = 1;
5174 }
5175 if (ei->i_disksize > 0x7fffffffULL) {
5176 if (!ext4_has_feature_large_file(sb) ||
5177 EXT4_SB(sb)->s_es->s_rev_level ==
5178 cpu_to_le32(EXT4_GOOD_OLD_REV))
5179 set_large_file = 1;
5180 }
5181 raw_inode->i_generation = cpu_to_le32(inode->i_generation);
5182 if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
5183 if (old_valid_dev(inode->i_rdev)) {
5184 raw_inode->i_block[0] =
5185 cpu_to_le32(old_encode_dev(inode->i_rdev));
5186 raw_inode->i_block[1] = 0;
5187 } else {
5188 raw_inode->i_block[0] = 0;
5189 raw_inode->i_block[1] =
5190 cpu_to_le32(new_encode_dev(inode->i_rdev));
5191 raw_inode->i_block[2] = 0;
5192 }
5193 } else if (!ext4_has_inline_data(inode)) {
5194 for (block = 0; block < EXT4_N_BLOCKS; block++)
5195 raw_inode->i_block[block] = ei->i_data[block];
5196 }
5197
5198 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
5199 u64 ivers = inode_peek_iversion(inode);
5200
5201 raw_inode->i_disk_version = cpu_to_le32(ivers);
5202 if (ei->i_extra_isize) {
5203 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
5204 raw_inode->i_version_hi =
5205 cpu_to_le32(ivers >> 32);
5206 raw_inode->i_extra_isize =
5207 cpu_to_le16(ei->i_extra_isize);
5208 }
5209 }
5210
5211 BUG_ON(!ext4_has_feature_project(inode->i_sb) &&
5212 i_projid != EXT4_DEF_PROJID);
5213
5214 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
5215 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
5216 raw_inode->i_projid = cpu_to_le32(i_projid);
5217
5218 ext4_inode_csum_set(inode, raw_inode, ei);
5219 spin_unlock(&ei->i_raw_lock);
5220 if (inode->i_sb->s_flags & SB_LAZYTIME)
5221 ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
5222 bh->b_data);
5223
5224 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
5225 rc = ext4_handle_dirty_metadata(handle, NULL, bh);
5226 if (!err)
5227 err = rc;
5228 ext4_clear_inode_state(inode, EXT4_STATE_NEW);
5229 if (set_large_file) {
5230 BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
5231 err = ext4_journal_get_write_access(handle, EXT4_SB(sb)->s_sbh);
5232 if (err)
5233 goto out_brelse;
5234 ext4_update_dynamic_rev(sb);
5235 ext4_set_feature_large_file(sb);
5236 ext4_handle_sync(handle);
5237 err = ext4_handle_dirty_super(handle, sb);
5238 }
5239 ext4_update_inode_fsync_trans(handle, inode, need_datasync);
5240out_brelse:
5241 brelse(bh);
5242 ext4_std_error(inode->i_sb, err);
5243 return err;
5244}
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
5281{
5282 int err;
5283
5284 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC))
5285 return 0;
5286
5287 if (EXT4_SB(inode->i_sb)->s_journal) {
5288 if (ext4_journal_current_handle()) {
5289 jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n");
5290 dump_stack();
5291 return -EIO;
5292 }
5293
5294
5295
5296
5297
5298
5299 if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
5300 return 0;
5301
5302 err = ext4_force_commit(inode->i_sb);
5303 } else {
5304 struct ext4_iloc iloc;
5305
5306 err = __ext4_get_inode_loc(inode, &iloc, 0);
5307 if (err)
5308 return err;
5309
5310
5311
5312
5313 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
5314 sync_dirty_buffer(iloc.bh);
5315 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
5316 EXT4_ERROR_INODE_BLOCK(inode, iloc.bh->b_blocknr,
5317 "IO error syncing inode");
5318 err = -EIO;
5319 }
5320 brelse(iloc.bh);
5321 }
5322 return err;
5323}
5324
5325
5326
5327
5328
5329
5330static void ext4_wait_for_tail_page_commit(struct inode *inode)
5331{
5332 struct page *page;
5333 unsigned offset;
5334 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
5335 tid_t commit_tid = 0;
5336 int ret;
5337
5338 offset = inode->i_size & (PAGE_SIZE - 1);
5339
5340
5341
5342
5343
5344 if (offset > PAGE_SIZE - i_blocksize(inode))
5345 return;
5346 while (1) {
5347 page = find_lock_page(inode->i_mapping,
5348 inode->i_size >> PAGE_SHIFT);
5349 if (!page)
5350 return;
5351 ret = __ext4_journalled_invalidatepage(page, offset,
5352 PAGE_SIZE - offset);
5353 unlock_page(page);
5354 put_page(page);
5355 if (ret != -EBUSY)
5356 return;
5357 commit_tid = 0;
5358 read_lock(&journal->j_state_lock);
5359 if (journal->j_committing_transaction)
5360 commit_tid = journal->j_committing_transaction->t_tid;
5361 read_unlock(&journal->j_state_lock);
5362 if (commit_tid)
5363 jbd2_log_wait_commit(journal, commit_tid);
5364 }
5365}
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391int ext4_setattr(struct dentry *dentry, struct iattr *attr)
5392{
5393 struct inode *inode = d_inode(dentry);
5394 int error, rc = 0;
5395 int orphan = 0;
5396 const unsigned int ia_valid = attr->ia_valid;
5397
5398 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5399 return -EIO;
5400
5401 error = setattr_prepare(dentry, attr);
5402 if (error)
5403 return error;
5404
5405 error = fscrypt_prepare_setattr(dentry, attr);
5406 if (error)
5407 return error;
5408
5409 if (is_quota_modification(inode, attr)) {
5410 error = dquot_initialize(inode);
5411 if (error)
5412 return error;
5413 }
5414 if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) ||
5415 (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) {
5416 handle_t *handle;
5417
5418
5419
5420 handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
5421 (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
5422 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
5423 if (IS_ERR(handle)) {
5424 error = PTR_ERR(handle);
5425 goto err_out;
5426 }
5427
5428
5429
5430
5431 down_read(&EXT4_I(inode)->xattr_sem);
5432 error = dquot_transfer(inode, attr);
5433 up_read(&EXT4_I(inode)->xattr_sem);
5434
5435 if (error) {
5436 ext4_journal_stop(handle);
5437 return error;
5438 }
5439
5440
5441 if (attr->ia_valid & ATTR_UID)
5442 inode->i_uid = attr->ia_uid;
5443 if (attr->ia_valid & ATTR_GID)
5444 inode->i_gid = attr->ia_gid;
5445 error = ext4_mark_inode_dirty(handle, inode);
5446 ext4_journal_stop(handle);
5447 }
5448
5449 if (attr->ia_valid & ATTR_SIZE) {
5450 handle_t *handle;
5451 loff_t oldsize = inode->i_size;
5452 int shrink = (attr->ia_size <= inode->i_size);
5453
5454 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
5455 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5456
5457 if (attr->ia_size > sbi->s_bitmap_maxbytes)
5458 return -EFBIG;
5459 }
5460 if (!S_ISREG(inode->i_mode))
5461 return -EINVAL;
5462
5463 if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size)
5464 inode_inc_iversion(inode);
5465
5466 if (ext4_should_order_data(inode) &&
5467 (attr->ia_size < inode->i_size)) {
5468 error = ext4_begin_ordered_truncate(inode,
5469 attr->ia_size);
5470 if (error)
5471 goto err_out;
5472 }
5473 if (attr->ia_size != inode->i_size) {
5474 handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
5475 if (IS_ERR(handle)) {
5476 error = PTR_ERR(handle);
5477 goto err_out;
5478 }
5479 if (ext4_handle_valid(handle) && shrink) {
5480 error = ext4_orphan_add(handle, inode);
5481 orphan = 1;
5482 }
5483
5484
5485
5486
5487 if (!shrink) {
5488 inode->i_mtime = current_time(inode);
5489 inode->i_ctime = inode->i_mtime;
5490 }
5491 down_write(&EXT4_I(inode)->i_data_sem);
5492 EXT4_I(inode)->i_disksize = attr->ia_size;
5493 rc = ext4_mark_inode_dirty(handle, inode);
5494 if (!error)
5495 error = rc;
5496
5497
5498
5499
5500
5501 if (!error)
5502 i_size_write(inode, attr->ia_size);
5503 up_write(&EXT4_I(inode)->i_data_sem);
5504 ext4_journal_stop(handle);
5505 if (error) {
5506 if (orphan)
5507 ext4_orphan_del(NULL, inode);
5508 goto err_out;
5509 }
5510 }
5511 if (!shrink)
5512 pagecache_isize_extended(inode, oldsize, inode->i_size);
5513
5514
5515
5516
5517
5518
5519 if (orphan) {
5520 if (!ext4_should_journal_data(inode)) {
5521 inode_dio_wait(inode);
5522 } else
5523 ext4_wait_for_tail_page_commit(inode);
5524 }
5525 down_write(&EXT4_I(inode)->i_mmap_sem);
5526
5527
5528
5529
5530 truncate_pagecache(inode, inode->i_size);
5531 if (shrink) {
5532 rc = ext4_truncate(inode);
5533 if (rc)
5534 error = rc;
5535 }
5536 up_write(&EXT4_I(inode)->i_mmap_sem);
5537 }
5538
5539 if (!error) {
5540 setattr_copy(inode, attr);
5541 mark_inode_dirty(inode);
5542 }
5543
5544
5545
5546
5547
5548 if (orphan && inode->i_nlink)
5549 ext4_orphan_del(NULL, inode);
5550
5551 if (!error && (ia_valid & ATTR_MODE))
5552 rc = posix_acl_chmod(inode, inode->i_mode);
5553
5554err_out:
5555 ext4_std_error(inode->i_sb, error);
5556 if (!error)
5557 error = rc;
5558 return error;
5559}
5560
5561int ext4_getattr(const struct path *path, struct kstat *stat,
5562 u32 request_mask, unsigned int query_flags)
5563{
5564 struct inode *inode = d_inode(path->dentry);
5565 struct ext4_inode *raw_inode;
5566 struct ext4_inode_info *ei = EXT4_I(inode);
5567 unsigned int flags;
5568
5569 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
5570 stat->result_mask |= STATX_BTIME;
5571 stat->btime.tv_sec = ei->i_crtime.tv_sec;
5572 stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
5573 }
5574
5575 flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
5576 if (flags & EXT4_APPEND_FL)
5577 stat->attributes |= STATX_ATTR_APPEND;
5578 if (flags & EXT4_COMPR_FL)
5579 stat->attributes |= STATX_ATTR_COMPRESSED;
5580 if (flags & EXT4_ENCRYPT_FL)
5581 stat->attributes |= STATX_ATTR_ENCRYPTED;
5582 if (flags & EXT4_IMMUTABLE_FL)
5583 stat->attributes |= STATX_ATTR_IMMUTABLE;
5584 if (flags & EXT4_NODUMP_FL)
5585 stat->attributes |= STATX_ATTR_NODUMP;
5586
5587 stat->attributes_mask |= (STATX_ATTR_APPEND |
5588 STATX_ATTR_COMPRESSED |
5589 STATX_ATTR_ENCRYPTED |
5590 STATX_ATTR_IMMUTABLE |
5591 STATX_ATTR_NODUMP);
5592
5593 generic_fillattr(inode, stat);
5594 return 0;
5595}
5596
5597int ext4_file_getattr(const struct path *path, struct kstat *stat,
5598 u32 request_mask, unsigned int query_flags)
5599{
5600 struct inode *inode = d_inode(path->dentry);
5601 u64 delalloc_blocks;
5602
5603 ext4_getattr(path, stat, request_mask, query_flags);
5604
5605
5606
5607
5608
5609
5610
5611 if (unlikely(ext4_has_inline_data(inode)))
5612 stat->blocks += (stat->size + 511) >> 9;
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624 delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
5625 EXT4_I(inode)->i_reserved_data_blocks);
5626 stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
5627 return 0;
5628}
5629
5630static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
5631 int pextents)
5632{
5633 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
5634 return ext4_ind_trans_blocks(inode, lblocks);
5635 return ext4_ext_index_trans_blocks(inode, pextents);
5636}
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
5650 int pextents)
5651{
5652 ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
5653 int gdpblocks;
5654 int idxblocks;
5655 int ret = 0;
5656
5657
5658
5659
5660
5661 idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
5662
5663 ret = idxblocks;
5664
5665
5666
5667
5668
5669 groups = idxblocks + pextents;
5670 gdpblocks = groups;
5671 if (groups > ngroups)
5672 groups = ngroups;
5673 if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
5674 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
5675
5676
5677 ret += groups + gdpblocks;
5678
5679
5680 ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
5681
5682 return ret;
5683}
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695int ext4_writepage_trans_blocks(struct inode *inode)
5696{
5697 int bpp = ext4_journal_blocks_per_page(inode);
5698 int ret;
5699
5700 ret = ext4_meta_trans_blocks(inode, bpp, bpp);
5701
5702
5703 if (ext4_should_journal_data(inode))
5704 ret += bpp;
5705 return ret;
5706}
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
5718{
5719 return ext4_meta_trans_blocks(inode, nrblocks, 1);
5720}
5721
5722
5723
5724
5725
5726int ext4_mark_iloc_dirty(handle_t *handle,
5727 struct inode *inode, struct ext4_iloc *iloc)
5728{
5729 int err = 0;
5730
5731 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5732 return -EIO;
5733
5734 if (IS_I_VERSION(inode))
5735 inode_inc_iversion(inode);
5736
5737
5738 get_bh(iloc->bh);
5739
5740
5741 err = ext4_do_update_inode(handle, inode, iloc);
5742 put_bh(iloc->bh);
5743 return err;
5744}
5745
5746
5747
5748
5749
5750
5751int
5752ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
5753 struct ext4_iloc *iloc)
5754{
5755 int err;
5756
5757 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5758 return -EIO;
5759
5760 err = ext4_get_inode_loc(inode, iloc);
5761 if (!err) {
5762 BUFFER_TRACE(iloc->bh, "get_write_access");
5763 err = ext4_journal_get_write_access(handle, iloc->bh);
5764 if (err) {
5765 brelse(iloc->bh);
5766 iloc->bh = NULL;
5767 }
5768 }
5769 ext4_std_error(inode->i_sb, err);
5770 return err;
5771}
5772
5773static int __ext4_expand_extra_isize(struct inode *inode,
5774 unsigned int new_extra_isize,
5775 struct ext4_iloc *iloc,
5776 handle_t *handle, int *no_expand)
5777{
5778 struct ext4_inode *raw_inode;
5779 struct ext4_xattr_ibody_header *header;
5780 int error;
5781
5782 raw_inode = ext4_raw_inode(iloc);
5783
5784 header = IHDR(inode, raw_inode);
5785
5786
5787 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
5788 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
5789 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
5790 EXT4_I(inode)->i_extra_isize, 0,
5791 new_extra_isize - EXT4_I(inode)->i_extra_isize);
5792 EXT4_I(inode)->i_extra_isize = new_extra_isize;
5793 return 0;
5794 }
5795
5796
5797 error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
5798 raw_inode, handle);
5799 if (error) {
5800
5801
5802
5803 *no_expand = 1;
5804 }
5805
5806 return error;
5807}
5808
5809
5810
5811
5812
5813static int ext4_try_to_expand_extra_isize(struct inode *inode,
5814 unsigned int new_extra_isize,
5815 struct ext4_iloc iloc,
5816 handle_t *handle)
5817{
5818 int no_expand;
5819 int error;
5820
5821 if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
5822 return -EOVERFLOW;
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833 if (ext4_handle_valid(handle) &&
5834 jbd2_journal_extend(handle,
5835 EXT4_DATA_TRANS_BLOCKS(inode->i_sb)) != 0)
5836 return -ENOSPC;
5837
5838 if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
5839 return -EBUSY;
5840
5841 error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
5842 handle, &no_expand);
5843 ext4_write_unlock_xattr(inode, &no_expand);
5844
5845 return error;
5846}
5847
5848int ext4_expand_extra_isize(struct inode *inode,
5849 unsigned int new_extra_isize,
5850 struct ext4_iloc *iloc)
5851{
5852 handle_t *handle;
5853 int no_expand;
5854 int error, rc;
5855
5856 if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
5857 brelse(iloc->bh);
5858 return -EOVERFLOW;
5859 }
5860
5861 handle = ext4_journal_start(inode, EXT4_HT_INODE,
5862 EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
5863 if (IS_ERR(handle)) {
5864 error = PTR_ERR(handle);
5865 brelse(iloc->bh);
5866 return error;
5867 }
5868
5869 ext4_write_lock_xattr(inode, &no_expand);
5870
5871 BUFFER_TRACE(iloc.bh, "get_write_access");
5872 error = ext4_journal_get_write_access(handle, iloc->bh);
5873 if (error) {
5874 brelse(iloc->bh);
5875 goto out_stop;
5876 }
5877
5878 error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
5879 handle, &no_expand);
5880
5881 rc = ext4_mark_iloc_dirty(handle, inode, iloc);
5882 if (!error)
5883 error = rc;
5884
5885 ext4_write_unlock_xattr(inode, &no_expand);
5886out_stop:
5887 ext4_journal_stop(handle);
5888 return error;
5889}
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904int ext4_mark_inode_dirty(handle_t *handle, struct inode *inode)
5905{
5906 struct ext4_iloc iloc;
5907 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5908 int err;
5909
5910 might_sleep();
5911 trace_ext4_mark_inode_dirty(inode, _RET_IP_);
5912 err = ext4_reserve_inode_write(handle, inode, &iloc);
5913 if (err)
5914 return err;
5915
5916 if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
5917 ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
5918 iloc, handle);
5919
5920 return ext4_mark_iloc_dirty(handle, inode, &iloc);
5921}
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941void ext4_dirty_inode(struct inode *inode, int flags)
5942{
5943 handle_t *handle;
5944
5945 if (flags == I_DIRTY_TIME)
5946 return;
5947 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
5948 if (IS_ERR(handle))
5949 goto out;
5950
5951 ext4_mark_inode_dirty(handle, inode);
5952
5953 ext4_journal_stop(handle);
5954out:
5955 return;
5956}
5957
5958#if 0
5959
5960
5961
5962
5963
5964
5965
5966static int ext4_pin_inode(handle_t *handle, struct inode *inode)
5967{
5968 struct ext4_iloc iloc;
5969
5970 int err = 0;
5971 if (handle) {
5972 err = ext4_get_inode_loc(inode, &iloc);
5973 if (!err) {
5974 BUFFER_TRACE(iloc.bh, "get_write_access");
5975 err = jbd2_journal_get_write_access(handle, iloc.bh);
5976 if (!err)
5977 err = ext4_handle_dirty_metadata(handle,
5978 NULL,
5979 iloc.bh);
5980 brelse(iloc.bh);
5981 }
5982 }
5983 ext4_std_error(inode->i_sb, err);
5984 return err;
5985}
5986#endif
5987
5988int ext4_change_inode_journal_flag(struct inode *inode, int val)
5989{
5990 journal_t *journal;
5991 handle_t *handle;
5992 int err;
5993 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005 journal = EXT4_JOURNAL(inode);
6006 if (!journal)
6007 return 0;
6008 if (is_journal_aborted(journal))
6009 return -EROFS;
6010
6011
6012 inode_dio_wait(inode);
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022 if (val) {
6023 down_write(&EXT4_I(inode)->i_mmap_sem);
6024 err = filemap_write_and_wait(inode->i_mapping);
6025 if (err < 0) {
6026 up_write(&EXT4_I(inode)->i_mmap_sem);
6027 return err;
6028 }
6029 }
6030
6031 percpu_down_write(&sbi->s_journal_flag_rwsem);
6032 jbd2_journal_lock_updates(journal);
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042 if (val)
6043 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
6044 else {
6045 err = jbd2_journal_flush(journal);
6046 if (err < 0) {
6047 jbd2_journal_unlock_updates(journal);
6048 percpu_up_write(&sbi->s_journal_flag_rwsem);
6049 return err;
6050 }
6051 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
6052 }
6053 ext4_set_aops(inode);
6054
6055 jbd2_journal_unlock_updates(journal);
6056 percpu_up_write(&sbi->s_journal_flag_rwsem);
6057
6058 if (val)
6059 up_write(&EXT4_I(inode)->i_mmap_sem);
6060
6061
6062
6063 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
6064 if (IS_ERR(handle))
6065 return PTR_ERR(handle);
6066
6067 err = ext4_mark_inode_dirty(handle, inode);
6068 ext4_handle_sync(handle);
6069 ext4_journal_stop(handle);
6070 ext4_std_error(inode->i_sb, err);
6071
6072 return err;
6073}
6074
6075static int ext4_bh_unmapped(handle_t *handle, struct buffer_head *bh)
6076{
6077 return !buffer_mapped(bh);
6078}
6079
6080int ext4_page_mkwrite(struct vm_fault *vmf)
6081{
6082 struct vm_area_struct *vma = vmf->vma;
6083 struct page *page = vmf->page;
6084 loff_t size;
6085 unsigned long len;
6086 int ret;
6087 struct file *file = vma->vm_file;
6088 struct inode *inode = file_inode(file);
6089 struct address_space *mapping = inode->i_mapping;
6090 handle_t *handle;
6091 get_block_t *get_block;
6092 int retries = 0;
6093
6094 sb_start_pagefault(inode->i_sb);
6095 file_update_time(vma->vm_file);
6096
6097 down_read(&EXT4_I(inode)->i_mmap_sem);
6098
6099 ret = ext4_convert_inline_data(inode);
6100 if (ret)
6101 goto out_ret;
6102
6103
6104 if (test_opt(inode->i_sb, DELALLOC) &&
6105 !ext4_should_journal_data(inode) &&
6106 !ext4_nonda_switch(inode->i_sb)) {
6107 do {
6108 ret = block_page_mkwrite(vma, vmf,
6109 ext4_da_get_block_prep);
6110 } while (ret == -ENOSPC &&
6111 ext4_should_retry_alloc(inode->i_sb, &retries));
6112 goto out_ret;
6113 }
6114
6115 lock_page(page);
6116 size = i_size_read(inode);
6117
6118 if (page->mapping != mapping || page_offset(page) > size) {
6119 unlock_page(page);
6120 ret = VM_FAULT_NOPAGE;
6121 goto out;
6122 }
6123
6124 if (page->index == size >> PAGE_SHIFT)
6125 len = size & ~PAGE_MASK;
6126 else
6127 len = PAGE_SIZE;
6128
6129
6130
6131
6132 if (page_has_buffers(page)) {
6133 if (!ext4_walk_page_buffers(NULL, page_buffers(page),
6134 0, len, NULL,
6135 ext4_bh_unmapped)) {
6136
6137 wait_for_stable_page(page);
6138 ret = VM_FAULT_LOCKED;
6139 goto out;
6140 }
6141 }
6142 unlock_page(page);
6143
6144 if (ext4_should_dioread_nolock(inode))
6145 get_block = ext4_get_block_unwritten;
6146 else
6147 get_block = ext4_get_block;
6148retry_alloc:
6149 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
6150 ext4_writepage_trans_blocks(inode));
6151 if (IS_ERR(handle)) {
6152 ret = VM_FAULT_SIGBUS;
6153 goto out;
6154 }
6155 ret = block_page_mkwrite(vma, vmf, get_block);
6156 if (!ret && ext4_should_journal_data(inode)) {
6157 if (ext4_walk_page_buffers(handle, page_buffers(page), 0,
6158 PAGE_SIZE, NULL, do_journal_get_write_access)) {
6159 unlock_page(page);
6160 ret = VM_FAULT_SIGBUS;
6161 ext4_journal_stop(handle);
6162 goto out;
6163 }
6164 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
6165 }
6166 ext4_journal_stop(handle);
6167 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
6168 goto retry_alloc;
6169out_ret:
6170 ret = block_page_mkwrite_return(ret);
6171out:
6172 up_read(&EXT4_I(inode)->i_mmap_sem);
6173 sb_end_pagefault(inode->i_sb);
6174 return ret;
6175}
6176
6177int ext4_filemap_fault(struct vm_fault *vmf)
6178{
6179 struct inode *inode = file_inode(vmf->vma->vm_file);
6180 int err;
6181
6182 down_read(&EXT4_I(inode)->i_mmap_sem);
6183 err = filemap_fault(vmf);
6184 up_read(&EXT4_I(inode)->i_mmap_sem);
6185
6186 return err;
6187}
6188