1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#include <linux/fs.h>
23#include <linux/mount.h>
24#include <linux/time.h>
25#include <linux/highuid.h>
26#include <linux/pagemap.h>
27#include <linux/dax.h>
28#include <linux/quotaops.h>
29#include <linux/string.h>
30#include <linux/buffer_head.h>
31#include <linux/writeback.h>
32#include <linux/pagevec.h>
33#include <linux/mpage.h>
34#include <linux/namei.h>
35#include <linux/uio.h>
36#include <linux/bio.h>
37#include <linux/workqueue.h>
38#include <linux/kernel.h>
39#include <linux/printk.h>
40#include <linux/slab.h>
41#include <linux/bitops.h>
42#include <linux/iomap.h>
43#include <linux/iversion.h>
44
45#include "ext4_jbd2.h"
46#include "xattr.h"
47#include "acl.h"
48#include "truncate.h"
49
50#include <trace/events/ext4.h>
51
52static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
53 struct ext4_inode_info *ei)
54{
55 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
56 __u32 csum;
57 __u16 dummy_csum = 0;
58 int offset = offsetof(struct ext4_inode, i_checksum_lo);
59 unsigned int csum_size = sizeof(dummy_csum);
60
61 csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);
62 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);
63 offset += csum_size;
64 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
65 EXT4_GOOD_OLD_INODE_SIZE - offset);
66
67 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
68 offset = offsetof(struct ext4_inode, i_checksum_hi);
69 csum = ext4_chksum(sbi, csum, (__u8 *)raw +
70 EXT4_GOOD_OLD_INODE_SIZE,
71 offset - EXT4_GOOD_OLD_INODE_SIZE);
72 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
73 csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
74 csum_size);
75 offset += csum_size;
76 }
77 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
78 EXT4_INODE_SIZE(inode->i_sb) - offset);
79 }
80
81 return csum;
82}
83
84static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
85 struct ext4_inode_info *ei)
86{
87 __u32 provided, calculated;
88
89 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
90 cpu_to_le32(EXT4_OS_LINUX) ||
91 !ext4_has_metadata_csum(inode->i_sb))
92 return 1;
93
94 provided = le16_to_cpu(raw->i_checksum_lo);
95 calculated = ext4_inode_csum(inode, raw, ei);
96 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
97 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
98 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
99 else
100 calculated &= 0xFFFF;
101
102 return provided == calculated;
103}
104
105void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
106 struct ext4_inode_info *ei)
107{
108 __u32 csum;
109
110 if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
111 cpu_to_le32(EXT4_OS_LINUX) ||
112 !ext4_has_metadata_csum(inode->i_sb))
113 return;
114
115 csum = ext4_inode_csum(inode, raw, ei);
116 raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
117 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
118 EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
119 raw->i_checksum_hi = cpu_to_le16(csum >> 16);
120}
121
122static inline int ext4_begin_ordered_truncate(struct inode *inode,
123 loff_t new_size)
124{
125 trace_ext4_begin_ordered_truncate(inode, new_size);
126
127
128
129
130
131
132 if (!EXT4_I(inode)->jinode)
133 return 0;
134 return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
135 EXT4_I(inode)->jinode,
136 new_size);
137}
138
139static void ext4_invalidatepage(struct page *page, unsigned int offset,
140 unsigned int length);
141static int __ext4_journalled_writepage(struct page *page, unsigned int len);
142static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
143 int pextents);
144
145
146
147
148
149int ext4_inode_is_fast_symlink(struct inode *inode)
150{
151 if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
152 int ea_blocks = EXT4_I(inode)->i_file_acl ?
153 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
154
155 if (ext4_has_inline_data(inode))
156 return 0;
157
158 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
159 }
160 return S_ISLNK(inode->i_mode) && inode->i_size &&
161 (inode->i_size < EXT4_N_BLOCKS * 4);
162}
163
164
165
166
167void ext4_evict_inode(struct inode *inode)
168{
169 handle_t *handle;
170 int err;
171
172
173
174
175
176 int extra_credits = 6;
177 struct ext4_xattr_inode_array *ea_inode_array = NULL;
178 bool freeze_protected = false;
179
180 trace_ext4_evict_inode(inode);
181
182 if (inode->i_nlink) {
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201 if (inode->i_ino != EXT4_JOURNAL_INO &&
202 ext4_should_journal_data(inode) &&
203 (S_ISLNK(inode->i_mode) || S_ISREG(inode->i_mode)) &&
204 inode->i_data.nrpages) {
205 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
206 tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;
207
208 jbd2_complete_transaction(journal, commit_tid);
209 filemap_write_and_wait(&inode->i_data);
210 }
211 truncate_inode_pages_final(&inode->i_data);
212
213 goto no_delete;
214 }
215
216 if (is_bad_inode(inode))
217 goto no_delete;
218 dquot_initialize(inode);
219
220 if (ext4_should_order_data(inode))
221 ext4_begin_ordered_truncate(inode, 0);
222 truncate_inode_pages_final(&inode->i_data);
223
224
225
226
227
228
229 if (!list_empty_careful(&inode->i_io_list)) {
230 WARN_ON_ONCE(!ext4_should_journal_data(inode));
231 inode_io_list_del(inode);
232 }
233
234
235
236
237
238
239
240 if (!ext4_journal_current_handle()) {
241 sb_start_intwrite(inode->i_sb);
242 freeze_protected = true;
243 }
244
245 if (!IS_NOQUOTA(inode))
246 extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
247
248
249
250
251
252 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
253 ext4_blocks_for_truncate(inode) + extra_credits - 3);
254 if (IS_ERR(handle)) {
255 ext4_std_error(inode->i_sb, PTR_ERR(handle));
256
257
258
259
260
261 ext4_orphan_del(NULL, inode);
262 if (freeze_protected)
263 sb_end_intwrite(inode->i_sb);
264 goto no_delete;
265 }
266
267 if (IS_SYNC(inode))
268 ext4_handle_sync(handle);
269
270
271
272
273
274
275
276
277 if (ext4_inode_is_fast_symlink(inode))
278 memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
279 inode->i_size = 0;
280 err = ext4_mark_inode_dirty(handle, inode);
281 if (err) {
282 ext4_warning(inode->i_sb,
283 "couldn't mark inode dirty (err %d)", err);
284 goto stop_handle;
285 }
286 if (inode->i_blocks) {
287 err = ext4_truncate(inode);
288 if (err) {
289 ext4_error_err(inode->i_sb, -err,
290 "couldn't truncate inode %lu (err %d)",
291 inode->i_ino, err);
292 goto stop_handle;
293 }
294 }
295
296
297 err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
298 extra_credits);
299 if (err) {
300 ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
301stop_handle:
302 ext4_journal_stop(handle);
303 ext4_orphan_del(NULL, inode);
304 if (freeze_protected)
305 sb_end_intwrite(inode->i_sb);
306 ext4_xattr_inode_array_free(ea_inode_array);
307 goto no_delete;
308 }
309
310
311
312
313
314
315
316
317
318 ext4_orphan_del(handle, inode);
319 EXT4_I(inode)->i_dtime = (__u32)ktime_get_real_seconds();
320
321
322
323
324
325
326
327
328 if (ext4_mark_inode_dirty(handle, inode))
329
330 ext4_clear_inode(inode);
331 else
332 ext4_free_inode(handle, inode);
333 ext4_journal_stop(handle);
334 if (freeze_protected)
335 sb_end_intwrite(inode->i_sb);
336 ext4_xattr_inode_array_free(ea_inode_array);
337 return;
338no_delete:
339 if (!list_empty(&EXT4_I(inode)->i_fc_list))
340 ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM);
341 ext4_clear_inode(inode);
342}
343
344#ifdef CONFIG_QUOTA
345qsize_t *ext4_get_reserved_space(struct inode *inode)
346{
347 return &EXT4_I(inode)->i_reserved_quota;
348}
349#endif
350
351
352
353
354
355void ext4_da_update_reserve_space(struct inode *inode,
356 int used, int quota_claim)
357{
358 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
359 struct ext4_inode_info *ei = EXT4_I(inode);
360
361 spin_lock(&ei->i_block_reservation_lock);
362 trace_ext4_da_update_reserve_space(inode, used, quota_claim);
363 if (unlikely(used > ei->i_reserved_data_blocks)) {
364 ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
365 "with only %d reserved data blocks",
366 __func__, inode->i_ino, used,
367 ei->i_reserved_data_blocks);
368 WARN_ON(1);
369 used = ei->i_reserved_data_blocks;
370 }
371
372
373 ei->i_reserved_data_blocks -= used;
374 percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
375
376 spin_unlock(&ei->i_block_reservation_lock);
377
378
379 if (quota_claim)
380 dquot_claim_block(inode, EXT4_C2B(sbi, used));
381 else {
382
383
384
385
386
387 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
388 }
389
390
391
392
393
394
395 if ((ei->i_reserved_data_blocks == 0) &&
396 !inode_is_open_for_write(inode))
397 ext4_discard_preallocations(inode, 0);
398}
399
400static int __check_block_validity(struct inode *inode, const char *func,
401 unsigned int line,
402 struct ext4_map_blocks *map)
403{
404 if (ext4_has_feature_journal(inode->i_sb) &&
405 (inode->i_ino ==
406 le32_to_cpu(EXT4_SB(inode->i_sb)->s_es->s_journal_inum)))
407 return 0;
408 if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {
409 ext4_error_inode(inode, func, line, map->m_pblk,
410 "lblock %lu mapped to illegal pblock %llu "
411 "(length %d)", (unsigned long) map->m_lblk,
412 map->m_pblk, map->m_len);
413 return -EFSCORRUPTED;
414 }
415 return 0;
416}
417
418int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
419 ext4_lblk_t len)
420{
421 int ret;
422
423 if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
424 return fscrypt_zeroout_range(inode, lblk, pblk, len);
425
426 ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
427 if (ret > 0)
428 ret = 0;
429
430 return ret;
431}
432
433#define check_block_validity(inode, map) \
434 __check_block_validity((inode), __func__, __LINE__, (map))
435
436#ifdef ES_AGGRESSIVE_TEST
437static void ext4_map_blocks_es_recheck(handle_t *handle,
438 struct inode *inode,
439 struct ext4_map_blocks *es_map,
440 struct ext4_map_blocks *map,
441 int flags)
442{
443 int retval;
444
445 map->m_flags = 0;
446
447
448
449
450
451
452
453 down_read(&EXT4_I(inode)->i_data_sem);
454 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
455 retval = ext4_ext_map_blocks(handle, inode, map, 0);
456 } else {
457 retval = ext4_ind_map_blocks(handle, inode, map, 0);
458 }
459 up_read((&EXT4_I(inode)->i_data_sem));
460
461
462
463
464
465 if (es_map->m_lblk != map->m_lblk ||
466 es_map->m_flags != map->m_flags ||
467 es_map->m_pblk != map->m_pblk) {
468 printk("ES cache assertion failed for inode: %lu "
469 "es_cached ex [%d/%d/%llu/%x] != "
470 "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
471 inode->i_ino, es_map->m_lblk, es_map->m_len,
472 es_map->m_pblk, es_map->m_flags, map->m_lblk,
473 map->m_len, map->m_pblk, map->m_flags,
474 retval, flags);
475 }
476}
477#endif
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501int ext4_map_blocks(handle_t *handle, struct inode *inode,
502 struct ext4_map_blocks *map, int flags)
503{
504 struct extent_status es;
505 int retval;
506 int ret = 0;
507#ifdef ES_AGGRESSIVE_TEST
508 struct ext4_map_blocks orig_map;
509
510 memcpy(&orig_map, map, sizeof(*map));
511#endif
512
513 map->m_flags = 0;
514 ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",
515 flags, map->m_len, (unsigned long) map->m_lblk);
516
517
518
519
520 if (unlikely(map->m_len > INT_MAX))
521 map->m_len = INT_MAX;
522
523
524 if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
525 return -EFSCORRUPTED;
526
527
528 if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) &&
529 ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
530 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
531 map->m_pblk = ext4_es_pblock(&es) +
532 map->m_lblk - es.es_lblk;
533 map->m_flags |= ext4_es_is_written(&es) ?
534 EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
535 retval = es.es_len - (map->m_lblk - es.es_lblk);
536 if (retval > map->m_len)
537 retval = map->m_len;
538 map->m_len = retval;
539 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
540 map->m_pblk = 0;
541 retval = es.es_len - (map->m_lblk - es.es_lblk);
542 if (retval > map->m_len)
543 retval = map->m_len;
544 map->m_len = retval;
545 retval = 0;
546 } else {
547 BUG();
548 }
549#ifdef ES_AGGRESSIVE_TEST
550 ext4_map_blocks_es_recheck(handle, inode, map,
551 &orig_map, flags);
552#endif
553 goto found;
554 }
555
556
557
558
559
560 down_read(&EXT4_I(inode)->i_data_sem);
561 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
562 retval = ext4_ext_map_blocks(handle, inode, map, 0);
563 } else {
564 retval = ext4_ind_map_blocks(handle, inode, map, 0);
565 }
566 if (retval > 0) {
567 unsigned int status;
568
569 if (unlikely(retval != map->m_len)) {
570 ext4_warning(inode->i_sb,
571 "ES len assertion failed for inode "
572 "%lu: retval %d != map->m_len %d",
573 inode->i_ino, retval, map->m_len);
574 WARN_ON(1);
575 }
576
577 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
578 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
579 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
580 !(status & EXTENT_STATUS_WRITTEN) &&
581 ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
582 map->m_lblk + map->m_len - 1))
583 status |= EXTENT_STATUS_DELAYED;
584 ret = ext4_es_insert_extent(inode, map->m_lblk,
585 map->m_len, map->m_pblk, status);
586 if (ret < 0)
587 retval = ret;
588 }
589 up_read((&EXT4_I(inode)->i_data_sem));
590
591found:
592 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
593 ret = check_block_validity(inode, map);
594 if (ret != 0)
595 return ret;
596 }
597
598
599 if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
600 return retval;
601
602
603
604
605
606
607
608
609 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
610
611
612
613
614
615 if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
616 return retval;
617
618
619
620
621
622 map->m_flags &= ~EXT4_MAP_FLAGS;
623
624
625
626
627
628
629
630 down_write(&EXT4_I(inode)->i_data_sem);
631
632
633
634
635
636 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
637 retval = ext4_ext_map_blocks(handle, inode, map, flags);
638 } else {
639 retval = ext4_ind_map_blocks(handle, inode, map, flags);
640
641 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
642
643
644
645
646
647 ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
648 }
649
650
651
652
653
654
655
656 if ((retval > 0) &&
657 (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE))
658 ext4_da_update_reserve_space(inode, retval, 1);
659 }
660
661 if (retval > 0) {
662 unsigned int status;
663
664 if (unlikely(retval != map->m_len)) {
665 ext4_warning(inode->i_sb,
666 "ES len assertion failed for inode "
667 "%lu: retval %d != map->m_len %d",
668 inode->i_ino, retval, map->m_len);
669 WARN_ON(1);
670 }
671
672
673
674
675
676
677
678
679 if (flags & EXT4_GET_BLOCKS_ZERO &&
680 map->m_flags & EXT4_MAP_MAPPED &&
681 map->m_flags & EXT4_MAP_NEW) {
682 ret = ext4_issue_zeroout(inode, map->m_lblk,
683 map->m_pblk, map->m_len);
684 if (ret) {
685 retval = ret;
686 goto out_sem;
687 }
688 }
689
690
691
692
693
694 if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
695 ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
696 if (ext4_es_is_written(&es))
697 goto out_sem;
698 }
699 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
700 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
701 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
702 !(status & EXTENT_STATUS_WRITTEN) &&
703 ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
704 map->m_lblk + map->m_len - 1))
705 status |= EXTENT_STATUS_DELAYED;
706 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
707 map->m_pblk, status);
708 if (ret < 0) {
709 retval = ret;
710 goto out_sem;
711 }
712 }
713
714out_sem:
715 up_write((&EXT4_I(inode)->i_data_sem));
716 if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
717 ret = check_block_validity(inode, map);
718 if (ret != 0)
719 return ret;
720
721
722
723
724
725
726 if (map->m_flags & EXT4_MAP_NEW &&
727 !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
728 !(flags & EXT4_GET_BLOCKS_ZERO) &&
729 !ext4_is_quota_file(inode) &&
730 ext4_should_order_data(inode)) {
731 loff_t start_byte =
732 (loff_t)map->m_lblk << inode->i_blkbits;
733 loff_t length = (loff_t)map->m_len << inode->i_blkbits;
734
735 if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
736 ret = ext4_jbd2_inode_add_wait(handle, inode,
737 start_byte, length);
738 else
739 ret = ext4_jbd2_inode_add_write(handle, inode,
740 start_byte, length);
741 if (ret)
742 return ret;
743 }
744 ext4_fc_track_range(handle, inode, map->m_lblk,
745 map->m_lblk + map->m_len - 1);
746 }
747
748 if (retval < 0)
749 ext_debug(inode, "failed with err %d\n", retval);
750 return retval;
751}
752
753
754
755
756
757static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
758{
759 unsigned long old_state;
760 unsigned long new_state;
761
762 flags &= EXT4_MAP_FLAGS;
763
764
765 if (!bh->b_page) {
766 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
767 return;
768 }
769
770
771
772
773
774 do {
775 old_state = READ_ONCE(bh->b_state);
776 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
777 } while (unlikely(
778 cmpxchg(&bh->b_state, old_state, new_state) != old_state));
779}
780
781static int _ext4_get_block(struct inode *inode, sector_t iblock,
782 struct buffer_head *bh, int flags)
783{
784 struct ext4_map_blocks map;
785 int ret = 0;
786
787 if (ext4_has_inline_data(inode))
788 return -ERANGE;
789
790 map.m_lblk = iblock;
791 map.m_len = bh->b_size >> inode->i_blkbits;
792
793 ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
794 flags);
795 if (ret > 0) {
796 map_bh(bh, inode->i_sb, map.m_pblk);
797 ext4_update_bh_state(bh, map.m_flags);
798 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
799 ret = 0;
800 } else if (ret == 0) {
801
802 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
803 }
804 return ret;
805}
806
807int ext4_get_block(struct inode *inode, sector_t iblock,
808 struct buffer_head *bh, int create)
809{
810 return _ext4_get_block(inode, iblock, bh,
811 create ? EXT4_GET_BLOCKS_CREATE : 0);
812}
813
814
815
816
817
818
819int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
820 struct buffer_head *bh_result, int create)
821{
822 ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
823 inode->i_ino, create);
824 return _ext4_get_block(inode, iblock, bh_result,
825 EXT4_GET_BLOCKS_IO_CREATE_EXT);
826}
827
828
829#define DIO_MAX_BLOCKS 4096
830
831
832
833
834struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
835 ext4_lblk_t block, int map_flags)
836{
837 struct ext4_map_blocks map;
838 struct buffer_head *bh;
839 int create = map_flags & EXT4_GET_BLOCKS_CREATE;
840 int err;
841
842 ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
843 || handle != NULL || create == 0);
844
845 map.m_lblk = block;
846 map.m_len = 1;
847 err = ext4_map_blocks(handle, inode, &map, map_flags);
848
849 if (err == 0)
850 return create ? ERR_PTR(-ENOSPC) : NULL;
851 if (err < 0)
852 return ERR_PTR(err);
853
854 bh = sb_getblk(inode->i_sb, map.m_pblk);
855 if (unlikely(!bh))
856 return ERR_PTR(-ENOMEM);
857 if (map.m_flags & EXT4_MAP_NEW) {
858 ASSERT(create != 0);
859 ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
860 || (handle != NULL));
861
862
863
864
865
866
867
868
869 lock_buffer(bh);
870 BUFFER_TRACE(bh, "call get_create_access");
871 err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
872 EXT4_JTR_NONE);
873 if (unlikely(err)) {
874 unlock_buffer(bh);
875 goto errout;
876 }
877 if (!buffer_uptodate(bh)) {
878 memset(bh->b_data, 0, inode->i_sb->s_blocksize);
879 set_buffer_uptodate(bh);
880 }
881 unlock_buffer(bh);
882 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
883 err = ext4_handle_dirty_metadata(handle, inode, bh);
884 if (unlikely(err))
885 goto errout;
886 } else
887 BUFFER_TRACE(bh, "not a new buffer");
888 return bh;
889errout:
890 brelse(bh);
891 return ERR_PTR(err);
892}
893
894struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
895 ext4_lblk_t block, int map_flags)
896{
897 struct buffer_head *bh;
898 int ret;
899
900 bh = ext4_getblk(handle, inode, block, map_flags);
901 if (IS_ERR(bh))
902 return bh;
903 if (!bh || ext4_buffer_uptodate(bh))
904 return bh;
905
906 ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true);
907 if (ret) {
908 put_bh(bh);
909 return ERR_PTR(ret);
910 }
911 return bh;
912}
913
914
915int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
916 bool wait, struct buffer_head **bhs)
917{
918 int i, err;
919
920 for (i = 0; i < bh_count; i++) {
921 bhs[i] = ext4_getblk(NULL, inode, block + i, 0 );
922 if (IS_ERR(bhs[i])) {
923 err = PTR_ERR(bhs[i]);
924 bh_count = i;
925 goto out_brelse;
926 }
927 }
928
929 for (i = 0; i < bh_count; i++)
930
931 if (bhs[i] && !ext4_buffer_uptodate(bhs[i]))
932 ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false);
933
934 if (!wait)
935 return 0;
936
937 for (i = 0; i < bh_count; i++)
938 if (bhs[i])
939 wait_on_buffer(bhs[i]);
940
941 for (i = 0; i < bh_count; i++) {
942 if (bhs[i] && !buffer_uptodate(bhs[i])) {
943 err = -EIO;
944 goto out_brelse;
945 }
946 }
947 return 0;
948
949out_brelse:
950 for (i = 0; i < bh_count; i++) {
951 brelse(bhs[i]);
952 bhs[i] = NULL;
953 }
954 return err;
955}
956
957int ext4_walk_page_buffers(handle_t *handle, struct inode *inode,
958 struct buffer_head *head,
959 unsigned from,
960 unsigned to,
961 int *partial,
962 int (*fn)(handle_t *handle, struct inode *inode,
963 struct buffer_head *bh))
964{
965 struct buffer_head *bh;
966 unsigned block_start, block_end;
967 unsigned blocksize = head->b_size;
968 int err, ret = 0;
969 struct buffer_head *next;
970
971 for (bh = head, block_start = 0;
972 ret == 0 && (bh != head || !block_start);
973 block_start = block_end, bh = next) {
974 next = bh->b_this_page;
975 block_end = block_start + blocksize;
976 if (block_end <= from || block_start >= to) {
977 if (partial && !buffer_uptodate(bh))
978 *partial = 1;
979 continue;
980 }
981 err = (*fn)(handle, inode, bh);
982 if (!ret)
983 ret = err;
984 }
985 return ret;
986}
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012int do_journal_get_write_access(handle_t *handle, struct inode *inode,
1013 struct buffer_head *bh)
1014{
1015 int dirty = buffer_dirty(bh);
1016 int ret;
1017
1018 if (!buffer_mapped(bh) || buffer_freed(bh))
1019 return 0;
1020
1021
1022
1023
1024
1025
1026
1027
1028 if (dirty)
1029 clear_buffer_dirty(bh);
1030 BUFFER_TRACE(bh, "get write access");
1031 ret = ext4_journal_get_write_access(handle, inode->i_sb, bh,
1032 EXT4_JTR_NONE);
1033 if (!ret && dirty)
1034 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1035 return ret;
1036}
1037
1038#ifdef CONFIG_FS_ENCRYPTION
1039static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
1040 get_block_t *get_block)
1041{
1042 unsigned from = pos & (PAGE_SIZE - 1);
1043 unsigned to = from + len;
1044 struct inode *inode = page->mapping->host;
1045 unsigned block_start, block_end;
1046 sector_t block;
1047 int err = 0;
1048 unsigned blocksize = inode->i_sb->s_blocksize;
1049 unsigned bbits;
1050 struct buffer_head *bh, *head, *wait[2];
1051 int nr_wait = 0;
1052 int i;
1053
1054 BUG_ON(!PageLocked(page));
1055 BUG_ON(from > PAGE_SIZE);
1056 BUG_ON(to > PAGE_SIZE);
1057 BUG_ON(from > to);
1058
1059 if (!page_has_buffers(page))
1060 create_empty_buffers(page, blocksize, 0);
1061 head = page_buffers(page);
1062 bbits = ilog2(blocksize);
1063 block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1064
1065 for (bh = head, block_start = 0; bh != head || !block_start;
1066 block++, block_start = block_end, bh = bh->b_this_page) {
1067 block_end = block_start + blocksize;
1068 if (block_end <= from || block_start >= to) {
1069 if (PageUptodate(page)) {
1070 set_buffer_uptodate(bh);
1071 }
1072 continue;
1073 }
1074 if (buffer_new(bh))
1075 clear_buffer_new(bh);
1076 if (!buffer_mapped(bh)) {
1077 WARN_ON(bh->b_size != blocksize);
1078 err = get_block(inode, block, bh, 1);
1079 if (err)
1080 break;
1081 if (buffer_new(bh)) {
1082 if (PageUptodate(page)) {
1083 clear_buffer_new(bh);
1084 set_buffer_uptodate(bh);
1085 mark_buffer_dirty(bh);
1086 continue;
1087 }
1088 if (block_end > to || block_start < from)
1089 zero_user_segments(page, to, block_end,
1090 block_start, from);
1091 continue;
1092 }
1093 }
1094 if (PageUptodate(page)) {
1095 set_buffer_uptodate(bh);
1096 continue;
1097 }
1098 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1099 !buffer_unwritten(bh) &&
1100 (block_start < from || block_end > to)) {
1101 ext4_read_bh_lock(bh, 0, false);
1102 wait[nr_wait++] = bh;
1103 }
1104 }
1105
1106
1107
1108 for (i = 0; i < nr_wait; i++) {
1109 wait_on_buffer(wait[i]);
1110 if (!buffer_uptodate(wait[i]))
1111 err = -EIO;
1112 }
1113 if (unlikely(err)) {
1114 page_zero_new_buffers(page, from, to);
1115 } else if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
1116 for (i = 0; i < nr_wait; i++) {
1117 int err2;
1118
1119 err2 = fscrypt_decrypt_pagecache_blocks(page, blocksize,
1120 bh_offset(wait[i]));
1121 if (err2) {
1122 clear_buffer_uptodate(wait[i]);
1123 err = err2;
1124 }
1125 }
1126 }
1127
1128 return err;
1129}
1130#endif
1131
1132static int ext4_write_begin(struct file *file, struct address_space *mapping,
1133 loff_t pos, unsigned len, unsigned flags,
1134 struct page **pagep, void **fsdata)
1135{
1136 struct inode *inode = mapping->host;
1137 int ret, needed_blocks;
1138 handle_t *handle;
1139 int retries = 0;
1140 struct page *page;
1141 pgoff_t index;
1142 unsigned from, to;
1143
1144 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
1145 return -EIO;
1146
1147 trace_ext4_write_begin(inode, pos, len, flags);
1148
1149
1150
1151
1152 needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
1153 index = pos >> PAGE_SHIFT;
1154 from = pos & (PAGE_SIZE - 1);
1155 to = from + len;
1156
1157 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1158 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1159 flags, pagep);
1160 if (ret < 0)
1161 return ret;
1162 if (ret == 1)
1163 return 0;
1164 }
1165
1166
1167
1168
1169
1170
1171
1172
1173retry_grab:
1174 page = grab_cache_page_write_begin(mapping, index, flags);
1175 if (!page)
1176 return -ENOMEM;
1177 unlock_page(page);
1178
1179retry_journal:
1180 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
1181 if (IS_ERR(handle)) {
1182 put_page(page);
1183 return PTR_ERR(handle);
1184 }
1185
1186 lock_page(page);
1187 if (page->mapping != mapping) {
1188
1189 unlock_page(page);
1190 put_page(page);
1191 ext4_journal_stop(handle);
1192 goto retry_grab;
1193 }
1194
1195 wait_for_stable_page(page);
1196
1197#ifdef CONFIG_FS_ENCRYPTION
1198 if (ext4_should_dioread_nolock(inode))
1199 ret = ext4_block_write_begin(page, pos, len,
1200 ext4_get_block_unwritten);
1201 else
1202 ret = ext4_block_write_begin(page, pos, len,
1203 ext4_get_block);
1204#else
1205 if (ext4_should_dioread_nolock(inode))
1206 ret = __block_write_begin(page, pos, len,
1207 ext4_get_block_unwritten);
1208 else
1209 ret = __block_write_begin(page, pos, len, ext4_get_block);
1210#endif
1211 if (!ret && ext4_should_journal_data(inode)) {
1212 ret = ext4_walk_page_buffers(handle, inode,
1213 page_buffers(page), from, to, NULL,
1214 do_journal_get_write_access);
1215 }
1216
1217 if (ret) {
1218 bool extended = (pos + len > inode->i_size) &&
1219 !ext4_verity_in_progress(inode);
1220
1221 unlock_page(page);
1222
1223
1224
1225
1226
1227
1228
1229
1230 if (extended && ext4_can_truncate(inode))
1231 ext4_orphan_add(handle, inode);
1232
1233 ext4_journal_stop(handle);
1234 if (extended) {
1235 ext4_truncate_failed_write(inode);
1236
1237
1238
1239
1240
1241
1242 if (inode->i_nlink)
1243 ext4_orphan_del(NULL, inode);
1244 }
1245
1246 if (ret == -ENOSPC &&
1247 ext4_should_retry_alloc(inode->i_sb, &retries))
1248 goto retry_journal;
1249 put_page(page);
1250 return ret;
1251 }
1252 *pagep = page;
1253 return ret;
1254}
1255
1256
1257static int write_end_fn(handle_t *handle, struct inode *inode,
1258 struct buffer_head *bh)
1259{
1260 int ret;
1261 if (!buffer_mapped(bh) || buffer_freed(bh))
1262 return 0;
1263 set_buffer_uptodate(bh);
1264 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1265 clear_buffer_meta(bh);
1266 clear_buffer_prio(bh);
1267 return ret;
1268}
1269
1270
1271
1272
1273
1274
1275
1276
1277static int ext4_write_end(struct file *file,
1278 struct address_space *mapping,
1279 loff_t pos, unsigned len, unsigned copied,
1280 struct page *page, void *fsdata)
1281{
1282 handle_t *handle = ext4_journal_current_handle();
1283 struct inode *inode = mapping->host;
1284 loff_t old_size = inode->i_size;
1285 int ret = 0, ret2;
1286 int i_size_changed = 0;
1287 bool verity = ext4_verity_in_progress(inode);
1288
1289 trace_ext4_write_end(inode, pos, len, copied);
1290
1291 if (ext4_has_inline_data(inode))
1292 return ext4_write_inline_data_end(inode, pos, len, copied, page);
1293
1294 copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
1295
1296
1297
1298
1299
1300
1301
1302 if (!verity)
1303 i_size_changed = ext4_update_inode_size(inode, pos + copied);
1304 unlock_page(page);
1305 put_page(page);
1306
1307 if (old_size < pos && !verity)
1308 pagecache_isize_extended(inode, old_size, pos);
1309
1310
1311
1312
1313
1314
1315 if (i_size_changed)
1316 ret = ext4_mark_inode_dirty(handle, inode);
1317
1318 if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1319
1320
1321
1322
1323 ext4_orphan_add(handle, inode);
1324
1325 ret2 = ext4_journal_stop(handle);
1326 if (!ret)
1327 ret = ret2;
1328
1329 if (pos + len > inode->i_size && !verity) {
1330 ext4_truncate_failed_write(inode);
1331
1332
1333
1334
1335
1336 if (inode->i_nlink)
1337 ext4_orphan_del(NULL, inode);
1338 }
1339
1340 return ret ? ret : copied;
1341}
1342
1343
1344
1345
1346
1347
1348static void ext4_journalled_zero_new_buffers(handle_t *handle,
1349 struct inode *inode,
1350 struct page *page,
1351 unsigned from, unsigned to)
1352{
1353 unsigned int block_start = 0, block_end;
1354 struct buffer_head *head, *bh;
1355
1356 bh = head = page_buffers(page);
1357 do {
1358 block_end = block_start + bh->b_size;
1359 if (buffer_new(bh)) {
1360 if (block_end > from && block_start < to) {
1361 if (!PageUptodate(page)) {
1362 unsigned start, size;
1363
1364 start = max(from, block_start);
1365 size = min(to, block_end) - start;
1366
1367 zero_user(page, start, size);
1368 write_end_fn(handle, inode, bh);
1369 }
1370 clear_buffer_new(bh);
1371 }
1372 }
1373 block_start = block_end;
1374 bh = bh->b_this_page;
1375 } while (bh != head);
1376}
1377
1378static int ext4_journalled_write_end(struct file *file,
1379 struct address_space *mapping,
1380 loff_t pos, unsigned len, unsigned copied,
1381 struct page *page, void *fsdata)
1382{
1383 handle_t *handle = ext4_journal_current_handle();
1384 struct inode *inode = mapping->host;
1385 loff_t old_size = inode->i_size;
1386 int ret = 0, ret2;
1387 int partial = 0;
1388 unsigned from, to;
1389 int size_changed = 0;
1390 bool verity = ext4_verity_in_progress(inode);
1391
1392 trace_ext4_journalled_write_end(inode, pos, len, copied);
1393 from = pos & (PAGE_SIZE - 1);
1394 to = from + len;
1395
1396 BUG_ON(!ext4_handle_valid(handle));
1397
1398 if (ext4_has_inline_data(inode))
1399 return ext4_write_inline_data_end(inode, pos, len, copied, page);
1400
1401 if (unlikely(copied < len) && !PageUptodate(page)) {
1402 copied = 0;
1403 ext4_journalled_zero_new_buffers(handle, inode, page, from, to);
1404 } else {
1405 if (unlikely(copied < len))
1406 ext4_journalled_zero_new_buffers(handle, inode, page,
1407 from + copied, to);
1408 ret = ext4_walk_page_buffers(handle, inode, page_buffers(page),
1409 from, from + copied, &partial,
1410 write_end_fn);
1411 if (!partial)
1412 SetPageUptodate(page);
1413 }
1414 if (!verity)
1415 size_changed = ext4_update_inode_size(inode, pos + copied);
1416 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1417 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1418 unlock_page(page);
1419 put_page(page);
1420
1421 if (old_size < pos && !verity)
1422 pagecache_isize_extended(inode, old_size, pos);
1423
1424 if (size_changed) {
1425 ret2 = ext4_mark_inode_dirty(handle, inode);
1426 if (!ret)
1427 ret = ret2;
1428 }
1429
1430 if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1431
1432
1433
1434
1435 ext4_orphan_add(handle, inode);
1436
1437 ret2 = ext4_journal_stop(handle);
1438 if (!ret)
1439 ret = ret2;
1440 if (pos + len > inode->i_size && !verity) {
1441 ext4_truncate_failed_write(inode);
1442
1443
1444
1445
1446
1447 if (inode->i_nlink)
1448 ext4_orphan_del(NULL, inode);
1449 }
1450
1451 return ret ? ret : copied;
1452}
1453
1454
1455
1456
1457static int ext4_da_reserve_space(struct inode *inode)
1458{
1459 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1460 struct ext4_inode_info *ei = EXT4_I(inode);
1461 int ret;
1462
1463
1464
1465
1466
1467
1468 ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
1469 if (ret)
1470 return ret;
1471
1472 spin_lock(&ei->i_block_reservation_lock);
1473 if (ext4_claim_free_clusters(sbi, 1, 0)) {
1474 spin_unlock(&ei->i_block_reservation_lock);
1475 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
1476 return -ENOSPC;
1477 }
1478 ei->i_reserved_data_blocks++;
1479 trace_ext4_da_reserve_space(inode);
1480 spin_unlock(&ei->i_block_reservation_lock);
1481
1482 return 0;
1483}
1484
1485void ext4_da_release_space(struct inode *inode, int to_free)
1486{
1487 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1488 struct ext4_inode_info *ei = EXT4_I(inode);
1489
1490 if (!to_free)
1491 return;
1492
1493 spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
1494
1495 trace_ext4_da_release_space(inode, to_free);
1496 if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1497
1498
1499
1500
1501
1502
1503 ext4_warning(inode->i_sb, "ext4_da_release_space: "
1504 "ino %lu, to_free %d with only %d reserved "
1505 "data blocks", inode->i_ino, to_free,
1506 ei->i_reserved_data_blocks);
1507 WARN_ON(1);
1508 to_free = ei->i_reserved_data_blocks;
1509 }
1510 ei->i_reserved_data_blocks -= to_free;
1511
1512
1513 percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
1514
1515 spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1516
1517 dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1518}
1519
1520
1521
1522
1523
1524struct mpage_da_data {
1525 struct inode *inode;
1526 struct writeback_control *wbc;
1527
1528 pgoff_t first_page;
1529 pgoff_t next_page;
1530 pgoff_t last_page;
1531
1532
1533
1534
1535
1536 struct ext4_map_blocks map;
1537 struct ext4_io_submit io_submit;
1538 unsigned int do_map:1;
1539 unsigned int scanned_until_end:1;
1540};
1541
1542static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1543 bool invalidate)
1544{
1545 int nr_pages, i;
1546 pgoff_t index, end;
1547 struct pagevec pvec;
1548 struct inode *inode = mpd->inode;
1549 struct address_space *mapping = inode->i_mapping;
1550
1551
1552 if (mpd->first_page >= mpd->next_page)
1553 return;
1554
1555 mpd->scanned_until_end = 0;
1556 index = mpd->first_page;
1557 end = mpd->next_page - 1;
1558 if (invalidate) {
1559 ext4_lblk_t start, last;
1560 start = index << (PAGE_SHIFT - inode->i_blkbits);
1561 last = end << (PAGE_SHIFT - inode->i_blkbits);
1562 ext4_es_remove_extent(inode, start, last - start + 1);
1563 }
1564
1565 pagevec_init(&pvec);
1566 while (index <= end) {
1567 nr_pages = pagevec_lookup_range(&pvec, mapping, &index, end);
1568 if (nr_pages == 0)
1569 break;
1570 for (i = 0; i < nr_pages; i++) {
1571 struct page *page = pvec.pages[i];
1572
1573 BUG_ON(!PageLocked(page));
1574 BUG_ON(PageWriteback(page));
1575 if (invalidate) {
1576 if (page_mapped(page))
1577 clear_page_dirty_for_io(page);
1578 block_invalidatepage(page, 0, PAGE_SIZE);
1579 ClearPageUptodate(page);
1580 }
1581 unlock_page(page);
1582 }
1583 pagevec_release(&pvec);
1584 }
1585}
1586
1587static void ext4_print_free_blocks(struct inode *inode)
1588{
1589 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1590 struct super_block *sb = inode->i_sb;
1591 struct ext4_inode_info *ei = EXT4_I(inode);
1592
1593 ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1594 EXT4_C2B(EXT4_SB(inode->i_sb),
1595 ext4_count_free_clusters(sb)));
1596 ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1597 ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1598 (long long) EXT4_C2B(EXT4_SB(sb),
1599 percpu_counter_sum(&sbi->s_freeclusters_counter)));
1600 ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1601 (long long) EXT4_C2B(EXT4_SB(sb),
1602 percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1603 ext4_msg(sb, KERN_CRIT, "Block reservation details");
1604 ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
1605 ei->i_reserved_data_blocks);
1606 return;
1607}
1608
1609static int ext4_bh_delay_or_unwritten(handle_t *handle, struct inode *inode,
1610 struct buffer_head *bh)
1611{
1612 return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
1613}
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626static int ext4_insert_delayed_block(struct inode *inode, ext4_lblk_t lblk)
1627{
1628 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1629 int ret;
1630 bool allocated = false;
1631 bool reserved = false;
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644 if (sbi->s_cluster_ratio == 1) {
1645 ret = ext4_da_reserve_space(inode);
1646 if (ret != 0)
1647 goto errout;
1648 reserved = true;
1649 } else {
1650 if (!ext4_es_scan_clu(inode, &ext4_es_is_delonly, lblk)) {
1651 if (!ext4_es_scan_clu(inode,
1652 &ext4_es_is_mapped, lblk)) {
1653 ret = ext4_clu_mapped(inode,
1654 EXT4_B2C(sbi, lblk));
1655 if (ret < 0)
1656 goto errout;
1657 if (ret == 0) {
1658 ret = ext4_da_reserve_space(inode);
1659 if (ret != 0)
1660 goto errout;
1661 reserved = true;
1662 } else {
1663 allocated = true;
1664 }
1665 } else {
1666 allocated = true;
1667 }
1668 }
1669 }
1670
1671 ret = ext4_es_insert_delayed_block(inode, lblk, allocated);
1672 if (ret && reserved)
1673 ext4_da_release_space(inode, 1);
1674
1675errout:
1676 return ret;
1677}
1678
1679
1680
1681
1682
1683
1684
1685static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
1686 struct ext4_map_blocks *map,
1687 struct buffer_head *bh)
1688{
1689 struct extent_status es;
1690 int retval;
1691 sector_t invalid_block = ~((sector_t) 0xffff);
1692#ifdef ES_AGGRESSIVE_TEST
1693 struct ext4_map_blocks orig_map;
1694
1695 memcpy(&orig_map, map, sizeof(*map));
1696#endif
1697
1698 if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
1699 invalid_block = ~0;
1700
1701 map->m_flags = 0;
1702 ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,
1703 (unsigned long) map->m_lblk);
1704
1705
1706 if (ext4_es_lookup_extent(inode, iblock, NULL, &es)) {
1707 if (ext4_es_is_hole(&es)) {
1708 retval = 0;
1709 down_read(&EXT4_I(inode)->i_data_sem);
1710 goto add_delayed;
1711 }
1712
1713
1714
1715
1716
1717
1718
1719
1720 if (ext4_es_is_delonly(&es)) {
1721 BUG_ON(bh->b_blocknr != invalid_block);
1722 BUG_ON(!buffer_new(bh));
1723 BUG_ON(!buffer_delay(bh));
1724 return 0;
1725 }
1726
1727 map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk;
1728 retval = es.es_len - (iblock - es.es_lblk);
1729 if (retval > map->m_len)
1730 retval = map->m_len;
1731 map->m_len = retval;
1732 if (ext4_es_is_written(&es))
1733 map->m_flags |= EXT4_MAP_MAPPED;
1734 else if (ext4_es_is_unwritten(&es))
1735 map->m_flags |= EXT4_MAP_UNWRITTEN;
1736 else
1737 BUG();
1738
1739#ifdef ES_AGGRESSIVE_TEST
1740 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1741#endif
1742 return retval;
1743 }
1744
1745
1746
1747
1748
1749 down_read(&EXT4_I(inode)->i_data_sem);
1750 if (ext4_has_inline_data(inode))
1751 retval = 0;
1752 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
1753 retval = ext4_ext_map_blocks(NULL, inode, map, 0);
1754 else
1755 retval = ext4_ind_map_blocks(NULL, inode, map, 0);
1756
1757add_delayed:
1758 if (retval == 0) {
1759 int ret;
1760
1761
1762
1763
1764
1765
1766 ret = ext4_insert_delayed_block(inode, map->m_lblk);
1767 if (ret != 0) {
1768 retval = ret;
1769 goto out_unlock;
1770 }
1771
1772 map_bh(bh, inode->i_sb, invalid_block);
1773 set_buffer_new(bh);
1774 set_buffer_delay(bh);
1775 } else if (retval > 0) {
1776 int ret;
1777 unsigned int status;
1778
1779 if (unlikely(retval != map->m_len)) {
1780 ext4_warning(inode->i_sb,
1781 "ES len assertion failed for inode "
1782 "%lu: retval %d != map->m_len %d",
1783 inode->i_ino, retval, map->m_len);
1784 WARN_ON(1);
1785 }
1786
1787 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
1788 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
1789 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1790 map->m_pblk, status);
1791 if (ret != 0)
1792 retval = ret;
1793 }
1794
1795out_unlock:
1796 up_read((&EXT4_I(inode)->i_data_sem));
1797
1798 return retval;
1799}
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1814 struct buffer_head *bh, int create)
1815{
1816 struct ext4_map_blocks map;
1817 int ret = 0;
1818
1819 BUG_ON(create == 0);
1820 BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
1821
1822 map.m_lblk = iblock;
1823 map.m_len = 1;
1824
1825
1826
1827
1828
1829
1830 ret = ext4_da_map_blocks(inode, iblock, &map, bh);
1831 if (ret <= 0)
1832 return ret;
1833
1834 map_bh(bh, inode->i_sb, map.m_pblk);
1835 ext4_update_bh_state(bh, map.m_flags);
1836
1837 if (buffer_unwritten(bh)) {
1838
1839
1840
1841
1842
1843
1844 set_buffer_new(bh);
1845 set_buffer_mapped(bh);
1846 }
1847 return 0;
1848}
1849
1850static int bget_one(handle_t *handle, struct inode *inode,
1851 struct buffer_head *bh)
1852{
1853 get_bh(bh);
1854 return 0;
1855}
1856
1857static int bput_one(handle_t *handle, struct inode *inode,
1858 struct buffer_head *bh)
1859{
1860 put_bh(bh);
1861 return 0;
1862}
1863
1864static int __ext4_journalled_writepage(struct page *page,
1865 unsigned int len)
1866{
1867 struct address_space *mapping = page->mapping;
1868 struct inode *inode = mapping->host;
1869 struct buffer_head *page_bufs = NULL;
1870 handle_t *handle = NULL;
1871 int ret = 0, err = 0;
1872 int inline_data = ext4_has_inline_data(inode);
1873 struct buffer_head *inode_bh = NULL;
1874
1875 ClearPageChecked(page);
1876
1877 if (inline_data) {
1878 BUG_ON(page->index != 0);
1879 BUG_ON(len > ext4_get_max_inline_size(inode));
1880 inode_bh = ext4_journalled_write_inline_data(inode, len, page);
1881 if (inode_bh == NULL)
1882 goto out;
1883 } else {
1884 page_bufs = page_buffers(page);
1885 if (!page_bufs) {
1886 BUG();
1887 goto out;
1888 }
1889 ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
1890 NULL, bget_one);
1891 }
1892
1893
1894
1895
1896
1897 get_page(page);
1898 unlock_page(page);
1899
1900 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
1901 ext4_writepage_trans_blocks(inode));
1902 if (IS_ERR(handle)) {
1903 ret = PTR_ERR(handle);
1904 put_page(page);
1905 goto out_no_pagelock;
1906 }
1907 BUG_ON(!ext4_handle_valid(handle));
1908
1909 lock_page(page);
1910 put_page(page);
1911 if (page->mapping != mapping) {
1912
1913 ext4_journal_stop(handle);
1914 ret = 0;
1915 goto out;
1916 }
1917
1918 if (inline_data) {
1919 ret = ext4_mark_inode_dirty(handle, inode);
1920 } else {
1921 ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
1922 NULL, do_journal_get_write_access);
1923
1924 err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
1925 NULL, write_end_fn);
1926 }
1927 if (ret == 0)
1928 ret = err;
1929 err = ext4_jbd2_inode_add_write(handle, inode, page_offset(page), len);
1930 if (ret == 0)
1931 ret = err;
1932 EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1933 err = ext4_journal_stop(handle);
1934 if (!ret)
1935 ret = err;
1936
1937 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1938out:
1939 unlock_page(page);
1940out_no_pagelock:
1941 if (!inline_data && page_bufs)
1942 ext4_walk_page_buffers(NULL, inode, page_bufs, 0, len,
1943 NULL, bput_one);
1944 brelse(inode_bh);
1945 return ret;
1946}
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989static int ext4_writepage(struct page *page,
1990 struct writeback_control *wbc)
1991{
1992 int ret = 0;
1993 loff_t size;
1994 unsigned int len;
1995 struct buffer_head *page_bufs = NULL;
1996 struct inode *inode = page->mapping->host;
1997 struct ext4_io_submit io_submit;
1998 bool keep_towrite = false;
1999
2000 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
2001 inode->i_mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
2002 unlock_page(page);
2003 return -EIO;
2004 }
2005
2006 trace_ext4_writepage(page);
2007 size = i_size_read(inode);
2008 if (page->index == size >> PAGE_SHIFT &&
2009 !ext4_verity_in_progress(inode))
2010 len = size & ~PAGE_MASK;
2011 else
2012 len = PAGE_SIZE;
2013
2014 page_bufs = page_buffers(page);
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032 if (ext4_walk_page_buffers(NULL, inode, page_bufs, 0, len, NULL,
2033 ext4_bh_delay_or_unwritten)) {
2034 redirty_page_for_writepage(wbc, page);
2035 if ((current->flags & PF_MEMALLOC) ||
2036 (inode->i_sb->s_blocksize == PAGE_SIZE)) {
2037
2038
2039
2040
2041
2042 WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD))
2043 == PF_MEMALLOC);
2044 unlock_page(page);
2045 return 0;
2046 }
2047 keep_towrite = true;
2048 }
2049
2050 if (PageChecked(page) && ext4_should_journal_data(inode))
2051
2052
2053
2054
2055 return __ext4_journalled_writepage(page, len);
2056
2057 ext4_io_submit_init(&io_submit, wbc);
2058 io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
2059 if (!io_submit.io_end) {
2060 redirty_page_for_writepage(wbc, page);
2061 unlock_page(page);
2062 return -ENOMEM;
2063 }
2064 ret = ext4_bio_write_page(&io_submit, page, len, keep_towrite);
2065 ext4_io_submit(&io_submit);
2066
2067 ext4_put_io_end_defer(io_submit.io_end);
2068 return ret;
2069}
2070
2071static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page)
2072{
2073 int len;
2074 loff_t size;
2075 int err;
2076
2077 BUG_ON(page->index != mpd->first_page);
2078 clear_page_dirty_for_io(page);
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092 size = i_size_read(mpd->inode);
2093 if (page->index == size >> PAGE_SHIFT &&
2094 !ext4_verity_in_progress(mpd->inode))
2095 len = size & ~PAGE_MASK;
2096 else
2097 len = PAGE_SIZE;
2098 err = ext4_bio_write_page(&mpd->io_submit, page, len, false);
2099 if (!err)
2100 mpd->wbc->nr_to_write--;
2101 mpd->first_page++;
2102
2103 return err;
2104}
2105
2106#define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))
2107
2108
2109
2110
2111
2112
2113#define MAX_WRITEPAGES_EXTENT_LEN 2048
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2130 struct buffer_head *bh)
2131{
2132 struct ext4_map_blocks *map = &mpd->map;
2133
2134
2135 if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2136 (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2137
2138 if (map->m_len == 0)
2139 return true;
2140 return false;
2141 }
2142
2143
2144 if (map->m_len == 0) {
2145
2146 if (!mpd->do_map)
2147 return false;
2148 map->m_lblk = lblk;
2149 map->m_len = 1;
2150 map->m_flags = bh->b_state & BH_FLAGS;
2151 return true;
2152 }
2153
2154
2155 if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2156 return false;
2157
2158
2159 if (lblk == map->m_lblk + map->m_len &&
2160 (bh->b_state & BH_FLAGS) == map->m_flags) {
2161 map->m_len++;
2162 return true;
2163 }
2164 return false;
2165}
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2184 struct buffer_head *head,
2185 struct buffer_head *bh,
2186 ext4_lblk_t lblk)
2187{
2188 struct inode *inode = mpd->inode;
2189 int err;
2190 ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
2191 >> inode->i_blkbits;
2192
2193 if (ext4_verity_in_progress(inode))
2194 blocks = EXT_MAX_BLOCKS;
2195
2196 do {
2197 BUG_ON(buffer_locked(bh));
2198
2199 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
2200
2201 if (mpd->map.m_len)
2202 return 0;
2203
2204 if (!mpd->do_map)
2205 return 0;
2206
2207 break;
2208 }
2209 } while (lblk++, (bh = bh->b_this_page) != head);
2210
2211 if (mpd->map.m_len == 0) {
2212 err = mpage_submit_page(mpd, head->b_page);
2213 if (err < 0)
2214 return err;
2215 }
2216 if (lblk >= blocks) {
2217 mpd->scanned_until_end = 1;
2218 return 0;
2219 }
2220 return 1;
2221}
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238static int mpage_process_page(struct mpage_da_data *mpd, struct page *page,
2239 ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,
2240 bool *map_bh)
2241{
2242 struct buffer_head *head, *bh;
2243 ext4_io_end_t *io_end = mpd->io_submit.io_end;
2244 ext4_lblk_t lblk = *m_lblk;
2245 ext4_fsblk_t pblock = *m_pblk;
2246 int err = 0;
2247 int blkbits = mpd->inode->i_blkbits;
2248 ssize_t io_end_size = 0;
2249 struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);
2250
2251 bh = head = page_buffers(page);
2252 do {
2253 if (lblk < mpd->map.m_lblk)
2254 continue;
2255 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2256
2257
2258
2259
2260 mpd->map.m_len = 0;
2261 mpd->map.m_flags = 0;
2262 io_end_vec->size += io_end_size;
2263 io_end_size = 0;
2264
2265 err = mpage_process_page_bufs(mpd, head, bh, lblk);
2266 if (err > 0)
2267 err = 0;
2268 if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
2269 io_end_vec = ext4_alloc_io_end_vec(io_end);
2270 if (IS_ERR(io_end_vec)) {
2271 err = PTR_ERR(io_end_vec);
2272 goto out;
2273 }
2274 io_end_vec->offset = (loff_t)mpd->map.m_lblk << blkbits;
2275 }
2276 *map_bh = true;
2277 goto out;
2278 }
2279 if (buffer_delay(bh)) {
2280 clear_buffer_delay(bh);
2281 bh->b_blocknr = pblock++;
2282 }
2283 clear_buffer_unwritten(bh);
2284 io_end_size += (1 << blkbits);
2285 } while (lblk++, (bh = bh->b_this_page) != head);
2286
2287 io_end_vec->size += io_end_size;
2288 io_end_size = 0;
2289 *map_bh = false;
2290out:
2291 *m_lblk = lblk;
2292 *m_pblk = pblock;
2293 return err;
2294}
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2311{
2312 struct pagevec pvec;
2313 int nr_pages, i;
2314 struct inode *inode = mpd->inode;
2315 int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
2316 pgoff_t start, end;
2317 ext4_lblk_t lblk;
2318 ext4_fsblk_t pblock;
2319 int err;
2320 bool map_bh = false;
2321
2322 start = mpd->map.m_lblk >> bpp_bits;
2323 end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2324 lblk = start << bpp_bits;
2325 pblock = mpd->map.m_pblk;
2326
2327 pagevec_init(&pvec);
2328 while (start <= end) {
2329 nr_pages = pagevec_lookup_range(&pvec, inode->i_mapping,
2330 &start, end);
2331 if (nr_pages == 0)
2332 break;
2333 for (i = 0; i < nr_pages; i++) {
2334 struct page *page = pvec.pages[i];
2335
2336 err = mpage_process_page(mpd, page, &lblk, &pblock,
2337 &map_bh);
2338
2339
2340
2341
2342
2343 if (err < 0 || map_bh)
2344 goto out;
2345
2346 err = mpage_submit_page(mpd, page);
2347 if (err < 0)
2348 goto out;
2349 }
2350 pagevec_release(&pvec);
2351 }
2352
2353 mpd->map.m_len = 0;
2354 mpd->map.m_flags = 0;
2355 return 0;
2356out:
2357 pagevec_release(&pvec);
2358 return err;
2359}
2360
2361static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2362{
2363 struct inode *inode = mpd->inode;
2364 struct ext4_map_blocks *map = &mpd->map;
2365 int get_blocks_flags;
2366 int err, dioread_nolock;
2367
2368 trace_ext4_da_write_pages_extent(inode, map);
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384 get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
2385 EXT4_GET_BLOCKS_METADATA_NOFAIL |
2386 EXT4_GET_BLOCKS_IO_SUBMIT;
2387 dioread_nolock = ext4_should_dioread_nolock(inode);
2388 if (dioread_nolock)
2389 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2390 if (map->m_flags & BIT(BH_Delay))
2391 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2392
2393 err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2394 if (err < 0)
2395 return err;
2396 if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
2397 if (!mpd->io_submit.io_end->handle &&
2398 ext4_handle_valid(handle)) {
2399 mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2400 handle->h_rsv_handle = NULL;
2401 }
2402 ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
2403 }
2404
2405 BUG_ON(map->m_len == 0);
2406 return 0;
2407}
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429static int mpage_map_and_submit_extent(handle_t *handle,
2430 struct mpage_da_data *mpd,
2431 bool *give_up_on_write)
2432{
2433 struct inode *inode = mpd->inode;
2434 struct ext4_map_blocks *map = &mpd->map;
2435 int err;
2436 loff_t disksize;
2437 int progress = 0;
2438 ext4_io_end_t *io_end = mpd->io_submit.io_end;
2439 struct ext4_io_end_vec *io_end_vec;
2440
2441 io_end_vec = ext4_alloc_io_end_vec(io_end);
2442 if (IS_ERR(io_end_vec))
2443 return PTR_ERR(io_end_vec);
2444 io_end_vec->offset = ((loff_t)map->m_lblk) << inode->i_blkbits;
2445 do {
2446 err = mpage_map_one_extent(handle, mpd);
2447 if (err < 0) {
2448 struct super_block *sb = inode->i_sb;
2449
2450 if (ext4_forced_shutdown(EXT4_SB(sb)) ||
2451 ext4_test_mount_flag(sb, EXT4_MF_FS_ABORTED))
2452 goto invalidate_dirty_pages;
2453
2454
2455
2456
2457
2458 if ((err == -ENOMEM) ||
2459 (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2460 if (progress)
2461 goto update_disksize;
2462 return err;
2463 }
2464 ext4_msg(sb, KERN_CRIT,
2465 "Delayed block allocation failed for "
2466 "inode %lu at logical offset %llu with"
2467 " max blocks %u with error %d",
2468 inode->i_ino,
2469 (unsigned long long)map->m_lblk,
2470 (unsigned)map->m_len, -err);
2471 ext4_msg(sb, KERN_CRIT,
2472 "This should not happen!! Data will "
2473 "be lost\n");
2474 if (err == -ENOSPC)
2475 ext4_print_free_blocks(inode);
2476 invalidate_dirty_pages:
2477 *give_up_on_write = true;
2478 return err;
2479 }
2480 progress = 1;
2481
2482
2483
2484
2485 err = mpage_map_and_submit_buffers(mpd);
2486 if (err < 0)
2487 goto update_disksize;
2488 } while (map->m_len);
2489
2490update_disksize:
2491
2492
2493
2494
2495 disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
2496 if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
2497 int err2;
2498 loff_t i_size;
2499
2500 down_write(&EXT4_I(inode)->i_data_sem);
2501 i_size = i_size_read(inode);
2502 if (disksize > i_size)
2503 disksize = i_size;
2504 if (disksize > EXT4_I(inode)->i_disksize)
2505 EXT4_I(inode)->i_disksize = disksize;
2506 up_write(&EXT4_I(inode)->i_data_sem);
2507 err2 = ext4_mark_inode_dirty(handle, inode);
2508 if (err2) {
2509 ext4_error_err(inode->i_sb, -err2,
2510 "Failed to mark inode %lu dirty",
2511 inode->i_ino);
2512 }
2513 if (!err)
2514 err = err2;
2515 }
2516 return err;
2517}
2518
2519
2520
2521
2522
2523
2524
2525
2526static int ext4_da_writepages_trans_blocks(struct inode *inode)
2527{
2528 int bpp = ext4_journal_blocks_per_page(inode);
2529
2530 return ext4_meta_trans_blocks(inode,
2531 MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);
2532}
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2553{
2554 struct address_space *mapping = mpd->inode->i_mapping;
2555 struct pagevec pvec;
2556 unsigned int nr_pages;
2557 long left = mpd->wbc->nr_to_write;
2558 pgoff_t index = mpd->first_page;
2559 pgoff_t end = mpd->last_page;
2560 xa_mark_t tag;
2561 int i, err = 0;
2562 int blkbits = mpd->inode->i_blkbits;
2563 ext4_lblk_t lblk;
2564 struct buffer_head *head;
2565
2566 if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
2567 tag = PAGECACHE_TAG_TOWRITE;
2568 else
2569 tag = PAGECACHE_TAG_DIRTY;
2570
2571 pagevec_init(&pvec);
2572 mpd->map.m_len = 0;
2573 mpd->next_page = index;
2574 while (index <= end) {
2575 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2576 tag);
2577 if (nr_pages == 0)
2578 break;
2579
2580 for (i = 0; i < nr_pages; i++) {
2581 struct page *page = pvec.pages[i];
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591 if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0)
2592 goto out;
2593
2594
2595 if (mpd->map.m_len > 0 && mpd->next_page != page->index)
2596 goto out;
2597
2598 lock_page(page);
2599
2600
2601
2602
2603
2604
2605
2606 if (!PageDirty(page) ||
2607 (PageWriteback(page) &&
2608 (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
2609 unlikely(page->mapping != mapping)) {
2610 unlock_page(page);
2611 continue;
2612 }
2613
2614 wait_on_page_writeback(page);
2615 BUG_ON(PageWriteback(page));
2616
2617 if (mpd->map.m_len == 0)
2618 mpd->first_page = page->index;
2619 mpd->next_page = page->index + 1;
2620
2621 lblk = ((ext4_lblk_t)page->index) <<
2622 (PAGE_SHIFT - blkbits);
2623 head = page_buffers(page);
2624 err = mpage_process_page_bufs(mpd, head, head, lblk);
2625 if (err <= 0)
2626 goto out;
2627 err = 0;
2628 left--;
2629 }
2630 pagevec_release(&pvec);
2631 cond_resched();
2632 }
2633 mpd->scanned_until_end = 1;
2634 return 0;
2635out:
2636 pagevec_release(&pvec);
2637 return err;
2638}
2639
2640static int ext4_writepages(struct address_space *mapping,
2641 struct writeback_control *wbc)
2642{
2643 pgoff_t writeback_index = 0;
2644 long nr_to_write = wbc->nr_to_write;
2645 int range_whole = 0;
2646 int cycled = 1;
2647 handle_t *handle = NULL;
2648 struct mpage_da_data mpd;
2649 struct inode *inode = mapping->host;
2650 int needed_blocks, rsv_blocks = 0, ret = 0;
2651 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2652 struct blk_plug plug;
2653 bool give_up_on_write = false;
2654
2655 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2656 return -EIO;
2657
2658 percpu_down_read(&sbi->s_writepages_rwsem);
2659 trace_ext4_writepages(inode, wbc);
2660
2661
2662
2663
2664
2665
2666 if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2667 goto out_writepages;
2668
2669 if (ext4_should_journal_data(inode)) {
2670 ret = generic_writepages(mapping, wbc);
2671 goto out_writepages;
2672 }
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684 if (unlikely(ext4_forced_shutdown(EXT4_SB(mapping->host->i_sb)) ||
2685 ext4_test_mount_flag(inode->i_sb, EXT4_MF_FS_ABORTED))) {
2686 ret = -EROFS;
2687 goto out_writepages;
2688 }
2689
2690
2691
2692
2693
2694
2695 if (ext4_has_inline_data(inode)) {
2696
2697 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2698 if (IS_ERR(handle)) {
2699 ret = PTR_ERR(handle);
2700 goto out_writepages;
2701 }
2702 BUG_ON(ext4_test_inode_state(inode,
2703 EXT4_STATE_MAY_INLINE_DATA));
2704 ext4_destroy_inline_data(handle, inode);
2705 ext4_journal_stop(handle);
2706 }
2707
2708 if (ext4_should_dioread_nolock(inode)) {
2709
2710
2711
2712
2713 rsv_blocks = 1 + ext4_chunk_trans_blocks(inode,
2714 PAGE_SIZE >> inode->i_blkbits);
2715 }
2716
2717 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2718 range_whole = 1;
2719
2720 if (wbc->range_cyclic) {
2721 writeback_index = mapping->writeback_index;
2722 if (writeback_index)
2723 cycled = 0;
2724 mpd.first_page = writeback_index;
2725 mpd.last_page = -1;
2726 } else {
2727 mpd.first_page = wbc->range_start >> PAGE_SHIFT;
2728 mpd.last_page = wbc->range_end >> PAGE_SHIFT;
2729 }
2730
2731 mpd.inode = inode;
2732 mpd.wbc = wbc;
2733 ext4_io_submit_init(&mpd.io_submit, wbc);
2734retry:
2735 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2736 tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page);
2737 blk_start_plug(&plug);
2738
2739
2740
2741
2742
2743
2744
2745 mpd.do_map = 0;
2746 mpd.scanned_until_end = 0;
2747 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2748 if (!mpd.io_submit.io_end) {
2749 ret = -ENOMEM;
2750 goto unplug;
2751 }
2752 ret = mpage_prepare_extent_to_map(&mpd);
2753
2754 mpage_release_unused_pages(&mpd, false);
2755
2756 ext4_io_submit(&mpd.io_submit);
2757 ext4_put_io_end_defer(mpd.io_submit.io_end);
2758 mpd.io_submit.io_end = NULL;
2759 if (ret < 0)
2760 goto unplug;
2761
2762 while (!mpd.scanned_until_end && wbc->nr_to_write > 0) {
2763
2764 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2765 if (!mpd.io_submit.io_end) {
2766 ret = -ENOMEM;
2767 break;
2768 }
2769
2770
2771
2772
2773
2774
2775
2776
2777 BUG_ON(ext4_should_journal_data(inode));
2778 needed_blocks = ext4_da_writepages_trans_blocks(inode);
2779
2780
2781 handle = ext4_journal_start_with_reserve(inode,
2782 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
2783 if (IS_ERR(handle)) {
2784 ret = PTR_ERR(handle);
2785 ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
2786 "%ld pages, ino %lu; err %d", __func__,
2787 wbc->nr_to_write, inode->i_ino, ret);
2788
2789 ext4_put_io_end(mpd.io_submit.io_end);
2790 mpd.io_submit.io_end = NULL;
2791 break;
2792 }
2793 mpd.do_map = 1;
2794
2795 trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc);
2796 ret = mpage_prepare_extent_to_map(&mpd);
2797 if (!ret && mpd.map.m_len)
2798 ret = mpage_map_and_submit_extent(handle, &mpd,
2799 &give_up_on_write);
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810 if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2811 ext4_journal_stop(handle);
2812 handle = NULL;
2813 mpd.do_map = 0;
2814 }
2815
2816 mpage_release_unused_pages(&mpd, give_up_on_write);
2817
2818 ext4_io_submit(&mpd.io_submit);
2819
2820
2821
2822
2823
2824
2825
2826
2827 if (handle) {
2828 ext4_put_io_end_defer(mpd.io_submit.io_end);
2829 ext4_journal_stop(handle);
2830 } else
2831 ext4_put_io_end(mpd.io_submit.io_end);
2832 mpd.io_submit.io_end = NULL;
2833
2834 if (ret == -ENOSPC && sbi->s_journal) {
2835
2836
2837
2838
2839
2840 jbd2_journal_force_commit_nested(sbi->s_journal);
2841 ret = 0;
2842 continue;
2843 }
2844
2845 if (ret)
2846 break;
2847 }
2848unplug:
2849 blk_finish_plug(&plug);
2850 if (!ret && !cycled && wbc->nr_to_write > 0) {
2851 cycled = 1;
2852 mpd.last_page = writeback_index - 1;
2853 mpd.first_page = 0;
2854 goto retry;
2855 }
2856
2857
2858 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2859
2860
2861
2862
2863 mapping->writeback_index = mpd.first_page;
2864
2865out_writepages:
2866 trace_ext4_writepages_result(inode, wbc, ret,
2867 nr_to_write - wbc->nr_to_write);
2868 percpu_up_read(&sbi->s_writepages_rwsem);
2869 return ret;
2870}
2871
2872static int ext4_dax_writepages(struct address_space *mapping,
2873 struct writeback_control *wbc)
2874{
2875 int ret;
2876 long nr_to_write = wbc->nr_to_write;
2877 struct inode *inode = mapping->host;
2878 struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2879
2880 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2881 return -EIO;
2882
2883 percpu_down_read(&sbi->s_writepages_rwsem);
2884 trace_ext4_writepages(inode, wbc);
2885
2886 ret = dax_writeback_mapping_range(mapping, sbi->s_daxdev, wbc);
2887 trace_ext4_writepages_result(inode, wbc, ret,
2888 nr_to_write - wbc->nr_to_write);
2889 percpu_up_read(&sbi->s_writepages_rwsem);
2890 return ret;
2891}
2892
2893static int ext4_nonda_switch(struct super_block *sb)
2894{
2895 s64 free_clusters, dirty_clusters;
2896 struct ext4_sb_info *sbi = EXT4_SB(sb);
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906 free_clusters =
2907 percpu_counter_read_positive(&sbi->s_freeclusters_counter);
2908 dirty_clusters =
2909 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
2910
2911
2912
2913 if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
2914 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
2915
2916 if (2 * free_clusters < 3 * dirty_clusters ||
2917 free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
2918
2919
2920
2921
2922 return 1;
2923 }
2924 return 0;
2925}
2926
2927static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
2928 loff_t pos, unsigned len, unsigned flags,
2929 struct page **pagep, void **fsdata)
2930{
2931 int ret, retries = 0;
2932 struct page *page;
2933 pgoff_t index;
2934 struct inode *inode = mapping->host;
2935
2936 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2937 return -EIO;
2938
2939 index = pos >> PAGE_SHIFT;
2940
2941 if (ext4_nonda_switch(inode->i_sb) || S_ISLNK(inode->i_mode) ||
2942 ext4_verity_in_progress(inode)) {
2943 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
2944 return ext4_write_begin(file, mapping, pos,
2945 len, flags, pagep, fsdata);
2946 }
2947 *fsdata = (void *)0;
2948 trace_ext4_da_write_begin(inode, pos, len, flags);
2949
2950 if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
2951 ret = ext4_da_write_inline_data_begin(mapping, inode,
2952 pos, len, flags,
2953 pagep, fsdata);
2954 if (ret < 0)
2955 return ret;
2956 if (ret == 1)
2957 return 0;
2958 }
2959
2960retry:
2961 page = grab_cache_page_write_begin(mapping, index, flags);
2962 if (!page)
2963 return -ENOMEM;
2964
2965
2966 wait_for_stable_page(page);
2967
2968#ifdef CONFIG_FS_ENCRYPTION
2969 ret = ext4_block_write_begin(page, pos, len,
2970 ext4_da_get_block_prep);
2971#else
2972 ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
2973#endif
2974 if (ret < 0) {
2975 unlock_page(page);
2976 put_page(page);
2977
2978
2979
2980
2981
2982 if (pos + len > inode->i_size)
2983 ext4_truncate_failed_write(inode);
2984
2985 if (ret == -ENOSPC &&
2986 ext4_should_retry_alloc(inode->i_sb, &retries))
2987 goto retry;
2988 return ret;
2989 }
2990
2991 *pagep = page;
2992 return ret;
2993}
2994
2995
2996
2997
2998
2999static int ext4_da_should_update_i_disksize(struct page *page,
3000 unsigned long offset)
3001{
3002 struct buffer_head *bh;
3003 struct inode *inode = page->mapping->host;
3004 unsigned int idx;
3005 int i;
3006
3007 bh = page_buffers(page);
3008 idx = offset >> inode->i_blkbits;
3009
3010 for (i = 0; i < idx; i++)
3011 bh = bh->b_this_page;
3012
3013 if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
3014 return 0;
3015 return 1;
3016}
3017
3018static int ext4_da_write_end(struct file *file,
3019 struct address_space *mapping,
3020 loff_t pos, unsigned len, unsigned copied,
3021 struct page *page, void *fsdata)
3022{
3023 struct inode *inode = mapping->host;
3024 loff_t new_i_size;
3025 unsigned long start, end;
3026 int write_mode = (int)(unsigned long)fsdata;
3027
3028 if (write_mode == FALL_BACK_TO_NONDELALLOC)
3029 return ext4_write_end(file, mapping, pos,
3030 len, copied, page, fsdata);
3031
3032 trace_ext4_da_write_end(inode, pos, len, copied);
3033
3034 if (write_mode != CONVERT_INLINE_DATA &&
3035 ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
3036 ext4_has_inline_data(inode))
3037 return ext4_write_inline_data_end(inode, pos, len, copied, page);
3038
3039 start = pos & (PAGE_SIZE - 1);
3040 end = start + copied - 1;
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057 new_i_size = pos + copied;
3058 if (copied && new_i_size > inode->i_size &&
3059 ext4_da_should_update_i_disksize(page, end))
3060 ext4_update_i_disksize(inode, new_i_size);
3061
3062 return generic_write_end(file, mapping, pos, len, copied, page, fsdata);
3063}
3064
3065
3066
3067
3068int ext4_alloc_da_blocks(struct inode *inode)
3069{
3070 trace_ext4_alloc_da_blocks(inode);
3071
3072 if (!EXT4_I(inode)->i_reserved_data_blocks)
3073 return 0;
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106 return filemap_flush(inode->i_mapping);
3107}
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3124{
3125 struct inode *inode = mapping->host;
3126 journal_t *journal;
3127 int err;
3128
3129
3130
3131
3132 if (ext4_has_inline_data(inode))
3133 return 0;
3134
3135 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3136 test_opt(inode->i_sb, DELALLOC)) {
3137
3138
3139
3140
3141
3142 filemap_write_and_wait(mapping);
3143 }
3144
3145 if (EXT4_JOURNAL(inode) &&
3146 ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165 ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
3166 journal = EXT4_JOURNAL(inode);
3167 jbd2_journal_lock_updates(journal);
3168 err = jbd2_journal_flush(journal, 0);
3169 jbd2_journal_unlock_updates(journal);
3170
3171 if (err)
3172 return 0;
3173 }
3174
3175 return iomap_bmap(mapping, block, &ext4_iomap_ops);
3176}
3177
3178static int ext4_readpage(struct file *file, struct page *page)
3179{
3180 int ret = -EAGAIN;
3181 struct inode *inode = page->mapping->host;
3182
3183 trace_ext4_readpage(page);
3184
3185 if (ext4_has_inline_data(inode))
3186 ret = ext4_readpage_inline(inode, page);
3187
3188 if (ret == -EAGAIN)
3189 return ext4_mpage_readpages(inode, NULL, page);
3190
3191 return ret;
3192}
3193
3194static void ext4_readahead(struct readahead_control *rac)
3195{
3196 struct inode *inode = rac->mapping->host;
3197
3198
3199 if (ext4_has_inline_data(inode))
3200 return;
3201
3202 ext4_mpage_readpages(inode, rac, NULL);
3203}
3204
3205static void ext4_invalidatepage(struct page *page, unsigned int offset,
3206 unsigned int length)
3207{
3208 trace_ext4_invalidatepage(page, offset, length);
3209
3210
3211 WARN_ON(page_has_buffers(page) && buffer_jbd(page_buffers(page)));
3212
3213 block_invalidatepage(page, offset, length);
3214}
3215
3216static int __ext4_journalled_invalidatepage(struct page *page,
3217 unsigned int offset,
3218 unsigned int length)
3219{
3220 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3221
3222 trace_ext4_journalled_invalidatepage(page, offset, length);
3223
3224
3225
3226
3227 if (offset == 0 && length == PAGE_SIZE)
3228 ClearPageChecked(page);
3229
3230 return jbd2_journal_invalidatepage(journal, page, offset, length);
3231}
3232
3233
3234static void ext4_journalled_invalidatepage(struct page *page,
3235 unsigned int offset,
3236 unsigned int length)
3237{
3238 WARN_ON(__ext4_journalled_invalidatepage(page, offset, length) < 0);
3239}
3240
3241static int ext4_releasepage(struct page *page, gfp_t wait)
3242{
3243 journal_t *journal = EXT4_JOURNAL(page->mapping->host);
3244
3245 trace_ext4_releasepage(page);
3246
3247
3248 if (PageChecked(page))
3249 return 0;
3250 if (journal)
3251 return jbd2_journal_try_to_free_buffers(journal, page);
3252 else
3253 return try_to_free_buffers(page);
3254}
3255
3256static bool ext4_inode_datasync_dirty(struct inode *inode)
3257{
3258 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
3259
3260 if (journal) {
3261 if (jbd2_transaction_committed(journal,
3262 EXT4_I(inode)->i_datasync_tid))
3263 return false;
3264 if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT))
3265 return !list_empty(&EXT4_I(inode)->i_fc_list);
3266 return true;
3267 }
3268
3269
3270 if (!list_empty(&inode->i_mapping->private_list))
3271 return true;
3272 return inode->i_state & I_DIRTY_DATASYNC;
3273}
3274
3275static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,
3276 struct ext4_map_blocks *map, loff_t offset,
3277 loff_t length)
3278{
3279 u8 blkbits = inode->i_blkbits;
3280
3281
3282
3283
3284
3285
3286 iomap->flags = 0;
3287 if (ext4_inode_datasync_dirty(inode) ||
3288 offset + length > i_size_read(inode))
3289 iomap->flags |= IOMAP_F_DIRTY;
3290
3291 if (map->m_flags & EXT4_MAP_NEW)
3292 iomap->flags |= IOMAP_F_NEW;
3293
3294 iomap->bdev = inode->i_sb->s_bdev;
3295 iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;
3296 iomap->offset = (u64) map->m_lblk << blkbits;
3297 iomap->length = (u64) map->m_len << blkbits;
3298
3299 if ((map->m_flags & EXT4_MAP_MAPPED) &&
3300 !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3301 iomap->flags |= IOMAP_F_MERGED;
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312 if (map->m_flags & EXT4_MAP_UNWRITTEN) {
3313 iomap->type = IOMAP_UNWRITTEN;
3314 iomap->addr = (u64) map->m_pblk << blkbits;
3315 } else if (map->m_flags & EXT4_MAP_MAPPED) {
3316 iomap->type = IOMAP_MAPPED;
3317 iomap->addr = (u64) map->m_pblk << blkbits;
3318 } else {
3319 iomap->type = IOMAP_HOLE;
3320 iomap->addr = IOMAP_NULL_ADDR;
3321 }
3322}
3323
3324static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
3325 unsigned int flags)
3326{
3327 handle_t *handle;
3328 u8 blkbits = inode->i_blkbits;
3329 int ret, dio_credits, m_flags = 0, retries = 0;
3330
3331
3332
3333
3334
3335 if (map->m_len > DIO_MAX_BLOCKS)
3336 map->m_len = DIO_MAX_BLOCKS;
3337 dio_credits = ext4_chunk_trans_blocks(inode, map->m_len);
3338
3339retry:
3340
3341
3342
3343
3344
3345
3346 handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
3347 if (IS_ERR(handle))
3348 return PTR_ERR(handle);
3349
3350
3351
3352
3353
3354 WARN_ON(!IS_DAX(inode) && !(flags & IOMAP_DIRECT));
3355 if (IS_DAX(inode))
3356 m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3357
3358
3359
3360
3361
3362
3363 else if (((loff_t)map->m_lblk << blkbits) >= i_size_read(inode))
3364 m_flags = EXT4_GET_BLOCKS_CREATE;
3365 else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3366 m_flags = EXT4_GET_BLOCKS_IO_CREATE_EXT;
3367
3368 ret = ext4_map_blocks(handle, inode, map, m_flags);
3369
3370
3371
3372
3373
3374
3375 if (!m_flags && !ret)
3376 ret = -ENOTBLK;
3377
3378 ext4_journal_stop(handle);
3379 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
3380 goto retry;
3381
3382 return ret;
3383}
3384
3385
3386static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
3387 unsigned flags, struct iomap *iomap, struct iomap *srcmap)
3388{
3389 int ret;
3390 struct ext4_map_blocks map;
3391 u8 blkbits = inode->i_blkbits;
3392
3393 if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3394 return -EINVAL;
3395
3396 if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3397 return -ERANGE;
3398
3399
3400
3401
3402 map.m_lblk = offset >> blkbits;
3403 map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3404 EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3405
3406 if (flags & IOMAP_WRITE) {
3407
3408
3409
3410
3411
3412
3413 if (offset + length <= i_size_read(inode)) {
3414 ret = ext4_map_blocks(NULL, inode, &map, 0);
3415 if (ret > 0 && (map.m_flags & EXT4_MAP_MAPPED))
3416 goto out;
3417 }
3418 ret = ext4_iomap_alloc(inode, &map, flags);
3419 } else {
3420 ret = ext4_map_blocks(NULL, inode, &map, 0);
3421 }
3422
3423 if (ret < 0)
3424 return ret;
3425out:
3426 ext4_set_iomap(inode, iomap, &map, offset, length);
3427
3428 return 0;
3429}
3430
3431static int ext4_iomap_overwrite_begin(struct inode *inode, loff_t offset,
3432 loff_t length, unsigned flags, struct iomap *iomap,
3433 struct iomap *srcmap)
3434{
3435 int ret;
3436
3437
3438
3439
3440
3441 flags &= ~IOMAP_WRITE;
3442 ret = ext4_iomap_begin(inode, offset, length, flags, iomap, srcmap);
3443 WARN_ON_ONCE(iomap->type != IOMAP_MAPPED);
3444 return ret;
3445}
3446
3447static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
3448 ssize_t written, unsigned flags, struct iomap *iomap)
3449{
3450
3451
3452
3453
3454
3455
3456
3457 if (flags & (IOMAP_WRITE | IOMAP_DIRECT) && written == 0)
3458 return -ENOTBLK;
3459
3460 return 0;
3461}
3462
3463const struct iomap_ops ext4_iomap_ops = {
3464 .iomap_begin = ext4_iomap_begin,
3465 .iomap_end = ext4_iomap_end,
3466};
3467
3468const struct iomap_ops ext4_iomap_overwrite_ops = {
3469 .iomap_begin = ext4_iomap_overwrite_begin,
3470 .iomap_end = ext4_iomap_end,
3471};
3472
3473static bool ext4_iomap_is_delalloc(struct inode *inode,
3474 struct ext4_map_blocks *map)
3475{
3476 struct extent_status es;
3477 ext4_lblk_t offset = 0, end = map->m_lblk + map->m_len - 1;
3478
3479 ext4_es_find_extent_range(inode, &ext4_es_is_delayed,
3480 map->m_lblk, end, &es);
3481
3482 if (!es.es_len || es.es_lblk > end)
3483 return false;
3484
3485 if (es.es_lblk > map->m_lblk) {
3486 map->m_len = es.es_lblk - map->m_lblk;
3487 return false;
3488 }
3489
3490 offset = map->m_lblk - es.es_lblk;
3491 map->m_len = es.es_len - offset;
3492
3493 return true;
3494}
3495
3496static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
3497 loff_t length, unsigned int flags,
3498 struct iomap *iomap, struct iomap *srcmap)
3499{
3500 int ret;
3501 bool delalloc = false;
3502 struct ext4_map_blocks map;
3503 u8 blkbits = inode->i_blkbits;
3504
3505 if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3506 return -EINVAL;
3507
3508 if (ext4_has_inline_data(inode)) {
3509 ret = ext4_inline_data_iomap(inode, iomap);
3510 if (ret != -EAGAIN) {
3511 if (ret == 0 && offset >= iomap->length)
3512 ret = -ENOENT;
3513 return ret;
3514 }
3515 }
3516
3517
3518
3519
3520 map.m_lblk = offset >> blkbits;
3521 map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3522 EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3523
3524
3525
3526
3527
3528
3529
3530 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
3531 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3532
3533 if (offset >= sbi->s_bitmap_maxbytes) {
3534 map.m_flags = 0;
3535 goto set_iomap;
3536 }
3537 }
3538
3539 ret = ext4_map_blocks(NULL, inode, &map, 0);
3540 if (ret < 0)
3541 return ret;
3542 if (ret == 0)
3543 delalloc = ext4_iomap_is_delalloc(inode, &map);
3544
3545set_iomap:
3546 ext4_set_iomap(inode, iomap, &map, offset, length);
3547 if (delalloc && iomap->type == IOMAP_HOLE)
3548 iomap->type = IOMAP_DELALLOC;
3549
3550 return 0;
3551}
3552
3553const struct iomap_ops ext4_iomap_report_ops = {
3554 .iomap_begin = ext4_iomap_begin_report,
3555};
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570static int ext4_journalled_set_page_dirty(struct page *page)
3571{
3572 SetPageChecked(page);
3573 return __set_page_dirty_nobuffers(page);
3574}
3575
3576static int ext4_set_page_dirty(struct page *page)
3577{
3578 WARN_ON_ONCE(!PageLocked(page) && !PageDirty(page));
3579 WARN_ON_ONCE(!page_has_buffers(page));
3580 return __set_page_dirty_buffers(page);
3581}
3582
3583static int ext4_iomap_swap_activate(struct swap_info_struct *sis,
3584 struct file *file, sector_t *span)
3585{
3586 return iomap_swapfile_activate(sis, file, span,
3587 &ext4_iomap_report_ops);
3588}
3589
3590static const struct address_space_operations ext4_aops = {
3591 .readpage = ext4_readpage,
3592 .readahead = ext4_readahead,
3593 .writepage = ext4_writepage,
3594 .writepages = ext4_writepages,
3595 .write_begin = ext4_write_begin,
3596 .write_end = ext4_write_end,
3597 .set_page_dirty = ext4_set_page_dirty,
3598 .bmap = ext4_bmap,
3599 .invalidatepage = ext4_invalidatepage,
3600 .releasepage = ext4_releasepage,
3601 .direct_IO = noop_direct_IO,
3602 .migratepage = buffer_migrate_page,
3603 .is_partially_uptodate = block_is_partially_uptodate,
3604 .error_remove_page = generic_error_remove_page,
3605 .swap_activate = ext4_iomap_swap_activate,
3606};
3607
3608static const struct address_space_operations ext4_journalled_aops = {
3609 .readpage = ext4_readpage,
3610 .readahead = ext4_readahead,
3611 .writepage = ext4_writepage,
3612 .writepages = ext4_writepages,
3613 .write_begin = ext4_write_begin,
3614 .write_end = ext4_journalled_write_end,
3615 .set_page_dirty = ext4_journalled_set_page_dirty,
3616 .bmap = ext4_bmap,
3617 .invalidatepage = ext4_journalled_invalidatepage,
3618 .releasepage = ext4_releasepage,
3619 .direct_IO = noop_direct_IO,
3620 .is_partially_uptodate = block_is_partially_uptodate,
3621 .error_remove_page = generic_error_remove_page,
3622 .swap_activate = ext4_iomap_swap_activate,
3623};
3624
3625static const struct address_space_operations ext4_da_aops = {
3626 .readpage = ext4_readpage,
3627 .readahead = ext4_readahead,
3628 .writepage = ext4_writepage,
3629 .writepages = ext4_writepages,
3630 .write_begin = ext4_da_write_begin,
3631 .write_end = ext4_da_write_end,
3632 .set_page_dirty = ext4_set_page_dirty,
3633 .bmap = ext4_bmap,
3634 .invalidatepage = ext4_invalidatepage,
3635 .releasepage = ext4_releasepage,
3636 .direct_IO = noop_direct_IO,
3637 .migratepage = buffer_migrate_page,
3638 .is_partially_uptodate = block_is_partially_uptodate,
3639 .error_remove_page = generic_error_remove_page,
3640 .swap_activate = ext4_iomap_swap_activate,
3641};
3642
3643static const struct address_space_operations ext4_dax_aops = {
3644 .writepages = ext4_dax_writepages,
3645 .direct_IO = noop_direct_IO,
3646 .set_page_dirty = __set_page_dirty_no_writeback,
3647 .bmap = ext4_bmap,
3648 .invalidatepage = noop_invalidatepage,
3649 .swap_activate = ext4_iomap_swap_activate,
3650};
3651
3652void ext4_set_aops(struct inode *inode)
3653{
3654 switch (ext4_inode_journal_mode(inode)) {
3655 case EXT4_INODE_ORDERED_DATA_MODE:
3656 case EXT4_INODE_WRITEBACK_DATA_MODE:
3657 break;
3658 case EXT4_INODE_JOURNAL_DATA_MODE:
3659 inode->i_mapping->a_ops = &ext4_journalled_aops;
3660 return;
3661 default:
3662 BUG();
3663 }
3664 if (IS_DAX(inode))
3665 inode->i_mapping->a_ops = &ext4_dax_aops;
3666 else if (test_opt(inode->i_sb, DELALLOC))
3667 inode->i_mapping->a_ops = &ext4_da_aops;
3668 else
3669 inode->i_mapping->a_ops = &ext4_aops;
3670}
3671
3672static int __ext4_block_zero_page_range(handle_t *handle,
3673 struct address_space *mapping, loff_t from, loff_t length)
3674{
3675 ext4_fsblk_t index = from >> PAGE_SHIFT;
3676 unsigned offset = from & (PAGE_SIZE-1);
3677 unsigned blocksize, pos;
3678 ext4_lblk_t iblock;
3679 struct inode *inode = mapping->host;
3680 struct buffer_head *bh;
3681 struct page *page;
3682 int err = 0;
3683
3684 page = find_or_create_page(mapping, from >> PAGE_SHIFT,
3685 mapping_gfp_constraint(mapping, ~__GFP_FS));
3686 if (!page)
3687 return -ENOMEM;
3688
3689 blocksize = inode->i_sb->s_blocksize;
3690
3691 iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
3692
3693 if (!page_has_buffers(page))
3694 create_empty_buffers(page, blocksize, 0);
3695
3696
3697 bh = page_buffers(page);
3698 pos = blocksize;
3699 while (offset >= pos) {
3700 bh = bh->b_this_page;
3701 iblock++;
3702 pos += blocksize;
3703 }
3704 if (buffer_freed(bh)) {
3705 BUFFER_TRACE(bh, "freed: skip");
3706 goto unlock;
3707 }
3708 if (!buffer_mapped(bh)) {
3709 BUFFER_TRACE(bh, "unmapped");
3710 ext4_get_block(inode, iblock, bh, 0);
3711
3712 if (!buffer_mapped(bh)) {
3713 BUFFER_TRACE(bh, "still unmapped");
3714 goto unlock;
3715 }
3716 }
3717
3718
3719 if (PageUptodate(page))
3720 set_buffer_uptodate(bh);
3721
3722 if (!buffer_uptodate(bh)) {
3723 err = ext4_read_bh_lock(bh, 0, true);
3724 if (err)
3725 goto unlock;
3726 if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
3727
3728 BUG_ON(!fscrypt_has_encryption_key(inode));
3729 err = fscrypt_decrypt_pagecache_blocks(page, blocksize,
3730 bh_offset(bh));
3731 if (err) {
3732 clear_buffer_uptodate(bh);
3733 goto unlock;
3734 }
3735 }
3736 }
3737 if (ext4_should_journal_data(inode)) {
3738 BUFFER_TRACE(bh, "get write access");
3739 err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
3740 EXT4_JTR_NONE);
3741 if (err)
3742 goto unlock;
3743 }
3744 zero_user(page, offset, length);
3745 BUFFER_TRACE(bh, "zeroed end of block");
3746
3747 if (ext4_should_journal_data(inode)) {
3748 err = ext4_handle_dirty_metadata(handle, inode, bh);
3749 } else {
3750 err = 0;
3751 mark_buffer_dirty(bh);
3752 if (ext4_should_order_data(inode))
3753 err = ext4_jbd2_inode_add_write(handle, inode, from,
3754 length);
3755 }
3756
3757unlock:
3758 unlock_page(page);
3759 put_page(page);
3760 return err;
3761}
3762
3763
3764
3765
3766
3767
3768
3769
3770static int ext4_block_zero_page_range(handle_t *handle,
3771 struct address_space *mapping, loff_t from, loff_t length)
3772{
3773 struct inode *inode = mapping->host;
3774 unsigned offset = from & (PAGE_SIZE-1);
3775 unsigned blocksize = inode->i_sb->s_blocksize;
3776 unsigned max = blocksize - (offset & (blocksize - 1));
3777
3778
3779
3780
3781
3782 if (length > max || length < 0)
3783 length = max;
3784
3785 if (IS_DAX(inode)) {
3786 return iomap_zero_range(inode, from, length, NULL,
3787 &ext4_iomap_ops);
3788 }
3789 return __ext4_block_zero_page_range(handle, mapping, from, length);
3790}
3791
3792
3793
3794
3795
3796
3797
3798static int ext4_block_truncate_page(handle_t *handle,
3799 struct address_space *mapping, loff_t from)
3800{
3801 unsigned offset = from & (PAGE_SIZE-1);
3802 unsigned length;
3803 unsigned blocksize;
3804 struct inode *inode = mapping->host;
3805
3806
3807 if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
3808 return 0;
3809
3810 blocksize = inode->i_sb->s_blocksize;
3811 length = blocksize - (offset & (blocksize - 1));
3812
3813 return ext4_block_zero_page_range(handle, mapping, from, length);
3814}
3815
3816int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
3817 loff_t lstart, loff_t length)
3818{
3819 struct super_block *sb = inode->i_sb;
3820 struct address_space *mapping = inode->i_mapping;
3821 unsigned partial_start, partial_end;
3822 ext4_fsblk_t start, end;
3823 loff_t byte_end = (lstart + length - 1);
3824 int err = 0;
3825
3826 partial_start = lstart & (sb->s_blocksize - 1);
3827 partial_end = byte_end & (sb->s_blocksize - 1);
3828
3829 start = lstart >> sb->s_blocksize_bits;
3830 end = byte_end >> sb->s_blocksize_bits;
3831
3832
3833 if (start == end &&
3834 (partial_start || (partial_end != sb->s_blocksize - 1))) {
3835 err = ext4_block_zero_page_range(handle, mapping,
3836 lstart, length);
3837 return err;
3838 }
3839
3840 if (partial_start) {
3841 err = ext4_block_zero_page_range(handle, mapping,
3842 lstart, sb->s_blocksize);
3843 if (err)
3844 return err;
3845 }
3846
3847 if (partial_end != sb->s_blocksize - 1)
3848 err = ext4_block_zero_page_range(handle, mapping,
3849 byte_end - partial_end,
3850 partial_end + 1);
3851 return err;
3852}
3853
3854int ext4_can_truncate(struct inode *inode)
3855{
3856 if (S_ISREG(inode->i_mode))
3857 return 1;
3858 if (S_ISDIR(inode->i_mode))
3859 return 1;
3860 if (S_ISLNK(inode->i_mode))
3861 return !ext4_inode_is_fast_symlink(inode);
3862 return 0;
3863}
3864
3865
3866
3867
3868
3869
3870
3871int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
3872 loff_t len)
3873{
3874 handle_t *handle;
3875 int ret;
3876
3877 loff_t size = i_size_read(inode);
3878
3879 WARN_ON(!inode_is_locked(inode));
3880 if (offset > size || offset + len < size)
3881 return 0;
3882
3883 if (EXT4_I(inode)->i_disksize >= size)
3884 return 0;
3885
3886 handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
3887 if (IS_ERR(handle))
3888 return PTR_ERR(handle);
3889 ext4_update_i_disksize(inode, size);
3890 ret = ext4_mark_inode_dirty(handle, inode);
3891 ext4_journal_stop(handle);
3892
3893 return ret;
3894}
3895
3896static void ext4_wait_dax_page(struct inode *inode)
3897{
3898 filemap_invalidate_unlock(inode->i_mapping);
3899 schedule();
3900 filemap_invalidate_lock(inode->i_mapping);
3901}
3902
3903int ext4_break_layouts(struct inode *inode)
3904{
3905 struct page *page;
3906 int error;
3907
3908 if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)))
3909 return -EINVAL;
3910
3911 do {
3912 page = dax_layout_busy_page(inode->i_mapping);
3913 if (!page)
3914 return 0;
3915
3916 error = ___wait_var_event(&page->_refcount,
3917 atomic_read(&page->_refcount) == 1,
3918 TASK_INTERRUPTIBLE, 0, 0,
3919 ext4_wait_dax_page(inode));
3920 } while (error == 0);
3921
3922 return error;
3923}
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
3937{
3938 struct super_block *sb = inode->i_sb;
3939 ext4_lblk_t first_block, stop_block;
3940 struct address_space *mapping = inode->i_mapping;
3941 loff_t first_block_offset, last_block_offset;
3942 handle_t *handle;
3943 unsigned int credits;
3944 int ret = 0, ret2 = 0;
3945
3946 trace_ext4_punch_hole(inode, offset, length, 0);
3947
3948 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
3949 if (ext4_has_inline_data(inode)) {
3950 filemap_invalidate_lock(mapping);
3951 ret = ext4_convert_inline_data(inode);
3952 filemap_invalidate_unlock(mapping);
3953 if (ret)
3954 return ret;
3955 }
3956
3957
3958
3959
3960
3961 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
3962 ret = filemap_write_and_wait_range(mapping, offset,
3963 offset + length - 1);
3964 if (ret)
3965 return ret;
3966 }
3967
3968 inode_lock(inode);
3969
3970
3971 if (offset >= inode->i_size)
3972 goto out_mutex;
3973
3974
3975
3976
3977
3978 if (offset + length > inode->i_size) {
3979 length = inode->i_size +
3980 PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -
3981 offset;
3982 }
3983
3984 if (offset & (sb->s_blocksize - 1) ||
3985 (offset + length) & (sb->s_blocksize - 1)) {
3986
3987
3988
3989
3990 ret = ext4_inode_attach_jinode(inode);
3991 if (ret < 0)
3992 goto out_mutex;
3993
3994 }
3995
3996
3997 inode_dio_wait(inode);
3998
3999
4000
4001
4002
4003 filemap_invalidate_lock(mapping);
4004
4005 ret = ext4_break_layouts(inode);
4006 if (ret)
4007 goto out_dio;
4008
4009 first_block_offset = round_up(offset, sb->s_blocksize);
4010 last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;
4011
4012
4013 if (last_block_offset > first_block_offset) {
4014 ret = ext4_update_disksize_before_punch(inode, offset, length);
4015 if (ret)
4016 goto out_dio;
4017 truncate_pagecache_range(inode, first_block_offset,
4018 last_block_offset);
4019 }
4020
4021 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4022 credits = ext4_writepage_trans_blocks(inode);
4023 else
4024 credits = ext4_blocks_for_truncate(inode);
4025 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4026 if (IS_ERR(handle)) {
4027 ret = PTR_ERR(handle);
4028 ext4_std_error(sb, ret);
4029 goto out_dio;
4030 }
4031
4032 ret = ext4_zero_partial_blocks(handle, inode, offset,
4033 length);
4034 if (ret)
4035 goto out_stop;
4036
4037 first_block = (offset + sb->s_blocksize - 1) >>
4038 EXT4_BLOCK_SIZE_BITS(sb);
4039 stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);
4040
4041
4042 if (stop_block > first_block) {
4043
4044 down_write(&EXT4_I(inode)->i_data_sem);
4045 ext4_discard_preallocations(inode, 0);
4046
4047 ret = ext4_es_remove_extent(inode, first_block,
4048 stop_block - first_block);
4049 if (ret) {
4050 up_write(&EXT4_I(inode)->i_data_sem);
4051 goto out_stop;
4052 }
4053
4054 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4055 ret = ext4_ext_remove_space(inode, first_block,
4056 stop_block - 1);
4057 else
4058 ret = ext4_ind_remove_space(handle, inode, first_block,
4059 stop_block);
4060
4061 up_write(&EXT4_I(inode)->i_data_sem);
4062 }
4063 ext4_fc_track_range(handle, inode, first_block, stop_block);
4064 if (IS_SYNC(inode))
4065 ext4_handle_sync(handle);
4066
4067 inode->i_mtime = inode->i_ctime = current_time(inode);
4068 ret2 = ext4_mark_inode_dirty(handle, inode);
4069 if (unlikely(ret2))
4070 ret = ret2;
4071 if (ret >= 0)
4072 ext4_update_inode_fsync_trans(handle, inode, 1);
4073out_stop:
4074 ext4_journal_stop(handle);
4075out_dio:
4076 filemap_invalidate_unlock(mapping);
4077out_mutex:
4078 inode_unlock(inode);
4079 return ret;
4080}
4081
4082int ext4_inode_attach_jinode(struct inode *inode)
4083{
4084 struct ext4_inode_info *ei = EXT4_I(inode);
4085 struct jbd2_inode *jinode;
4086
4087 if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
4088 return 0;
4089
4090 jinode = jbd2_alloc_inode(GFP_KERNEL);
4091 spin_lock(&inode->i_lock);
4092 if (!ei->jinode) {
4093 if (!jinode) {
4094 spin_unlock(&inode->i_lock);
4095 return -ENOMEM;
4096 }
4097 ei->jinode = jinode;
4098 jbd2_journal_init_jbd_inode(ei->jinode, inode);
4099 jinode = NULL;
4100 }
4101 spin_unlock(&inode->i_lock);
4102 if (unlikely(jinode != NULL))
4103 jbd2_free_inode(jinode);
4104 return 0;
4105}
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135int ext4_truncate(struct inode *inode)
4136{
4137 struct ext4_inode_info *ei = EXT4_I(inode);
4138 unsigned int credits;
4139 int err = 0, err2;
4140 handle_t *handle;
4141 struct address_space *mapping = inode->i_mapping;
4142
4143
4144
4145
4146
4147
4148 if (!(inode->i_state & (I_NEW|I_FREEING)))
4149 WARN_ON(!inode_is_locked(inode));
4150 trace_ext4_truncate_enter(inode);
4151
4152 if (!ext4_can_truncate(inode))
4153 goto out_trace;
4154
4155 if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
4156 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
4157
4158 if (ext4_has_inline_data(inode)) {
4159 int has_inline = 1;
4160
4161 err = ext4_inline_data_truncate(inode, &has_inline);
4162 if (err || has_inline)
4163 goto out_trace;
4164 }
4165
4166
4167 if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
4168 if (ext4_inode_attach_jinode(inode) < 0)
4169 goto out_trace;
4170 }
4171
4172 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4173 credits = ext4_writepage_trans_blocks(inode);
4174 else
4175 credits = ext4_blocks_for_truncate(inode);
4176
4177 handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4178 if (IS_ERR(handle)) {
4179 err = PTR_ERR(handle);
4180 goto out_trace;
4181 }
4182
4183 if (inode->i_size & (inode->i_sb->s_blocksize - 1))
4184 ext4_block_truncate_page(handle, mapping, inode->i_size);
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195 err = ext4_orphan_add(handle, inode);
4196 if (err)
4197 goto out_stop;
4198
4199 down_write(&EXT4_I(inode)->i_data_sem);
4200
4201 ext4_discard_preallocations(inode, 0);
4202
4203 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4204 err = ext4_ext_truncate(handle, inode);
4205 else
4206 ext4_ind_truncate(handle, inode);
4207
4208 up_write(&ei->i_data_sem);
4209 if (err)
4210 goto out_stop;
4211
4212 if (IS_SYNC(inode))
4213 ext4_handle_sync(handle);
4214
4215out_stop:
4216
4217
4218
4219
4220
4221
4222
4223 if (inode->i_nlink)
4224 ext4_orphan_del(handle, inode);
4225
4226 inode->i_mtime = inode->i_ctime = current_time(inode);
4227 err2 = ext4_mark_inode_dirty(handle, inode);
4228 if (unlikely(err2 && !err))
4229 err = err2;
4230 ext4_journal_stop(handle);
4231
4232out_trace:
4233 trace_ext4_truncate_exit(inode);
4234 return err;
4235}
4236
4237
4238
4239
4240
4241
4242
4243static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,
4244 struct ext4_iloc *iloc, int in_mem,
4245 ext4_fsblk_t *ret_block)
4246{
4247 struct ext4_group_desc *gdp;
4248 struct buffer_head *bh;
4249 ext4_fsblk_t block;
4250 struct blk_plug plug;
4251 int inodes_per_block, inode_offset;
4252
4253 iloc->bh = NULL;
4254 if (ino < EXT4_ROOT_INO ||
4255 ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
4256 return -EFSCORRUPTED;
4257
4258 iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
4259 gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
4260 if (!gdp)
4261 return -EIO;
4262
4263
4264
4265
4266 inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4267 inode_offset = ((ino - 1) %
4268 EXT4_INODES_PER_GROUP(sb));
4269 block = ext4_inode_table(sb, gdp) + (inode_offset / inodes_per_block);
4270 iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
4271
4272 bh = sb_getblk(sb, block);
4273 if (unlikely(!bh))
4274 return -ENOMEM;
4275 if (ext4_buffer_uptodate(bh))
4276 goto has_buffer;
4277
4278 lock_buffer(bh);
4279 if (ext4_buffer_uptodate(bh)) {
4280
4281 unlock_buffer(bh);
4282 goto has_buffer;
4283 }
4284
4285
4286
4287
4288
4289
4290 if (in_mem) {
4291 struct buffer_head *bitmap_bh;
4292 int i, start;
4293
4294 start = inode_offset & ~(inodes_per_block - 1);
4295
4296
4297 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
4298 if (unlikely(!bitmap_bh))
4299 goto make_io;
4300
4301
4302
4303
4304
4305
4306 if (!buffer_uptodate(bitmap_bh)) {
4307 brelse(bitmap_bh);
4308 goto make_io;
4309 }
4310 for (i = start; i < start + inodes_per_block; i++) {
4311 if (i == inode_offset)
4312 continue;
4313 if (ext4_test_bit(i, bitmap_bh->b_data))
4314 break;
4315 }
4316 brelse(bitmap_bh);
4317 if (i == start + inodes_per_block) {
4318
4319 memset(bh->b_data, 0, bh->b_size);
4320 set_buffer_uptodate(bh);
4321 unlock_buffer(bh);
4322 goto has_buffer;
4323 }
4324 }
4325
4326make_io:
4327
4328
4329
4330
4331 blk_start_plug(&plug);
4332 if (EXT4_SB(sb)->s_inode_readahead_blks) {
4333 ext4_fsblk_t b, end, table;
4334 unsigned num;
4335 __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
4336
4337 table = ext4_inode_table(sb, gdp);
4338
4339 b = block & ~((ext4_fsblk_t) ra_blks - 1);
4340 if (table > b)
4341 b = table;
4342 end = b + ra_blks;
4343 num = EXT4_INODES_PER_GROUP(sb);
4344 if (ext4_has_group_desc_csum(sb))
4345 num -= ext4_itable_unused_count(sb, gdp);
4346 table += num / inodes_per_block;
4347 if (end > table)
4348 end = table;
4349 while (b <= end)
4350 ext4_sb_breadahead_unmovable(sb, b++);
4351 }
4352
4353
4354
4355
4356
4357
4358 trace_ext4_load_inode(sb, ino);
4359 ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL);
4360 blk_finish_plug(&plug);
4361 wait_on_buffer(bh);
4362 ext4_simulate_fail_bh(sb, bh, EXT4_SIM_INODE_EIO);
4363 if (!buffer_uptodate(bh)) {
4364 if (ret_block)
4365 *ret_block = block;
4366 brelse(bh);
4367 return -EIO;
4368 }
4369has_buffer:
4370 iloc->bh = bh;
4371 return 0;
4372}
4373
4374static int __ext4_get_inode_loc_noinmem(struct inode *inode,
4375 struct ext4_iloc *iloc)
4376{
4377 ext4_fsblk_t err_blk;
4378 int ret;
4379
4380 ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, iloc, 0,
4381 &err_blk);
4382
4383 if (ret == -EIO)
4384 ext4_error_inode_block(inode, err_blk, EIO,
4385 "unable to read itable block");
4386
4387 return ret;
4388}
4389
4390int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
4391{
4392 ext4_fsblk_t err_blk;
4393 int ret;
4394
4395
4396 ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, iloc,
4397 !ext4_test_inode_state(inode, EXT4_STATE_XATTR), &err_blk);
4398
4399 if (ret == -EIO)
4400 ext4_error_inode_block(inode, err_blk, EIO,
4401 "unable to read itable block");
4402
4403 return ret;
4404}
4405
4406
4407int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,
4408 struct ext4_iloc *iloc)
4409{
4410 return __ext4_get_inode_loc(sb, ino, iloc, 0, NULL);
4411}
4412
4413static bool ext4_should_enable_dax(struct inode *inode)
4414{
4415 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4416
4417 if (test_opt2(inode->i_sb, DAX_NEVER))
4418 return false;
4419 if (!S_ISREG(inode->i_mode))
4420 return false;
4421 if (ext4_should_journal_data(inode))
4422 return false;
4423 if (ext4_has_inline_data(inode))
4424 return false;
4425 if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT))
4426 return false;
4427 if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY))
4428 return false;
4429 if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))
4430 return false;
4431 if (test_opt(inode->i_sb, DAX_ALWAYS))
4432 return true;
4433
4434 return ext4_test_inode_flag(inode, EXT4_INODE_DAX);
4435}
4436
4437void ext4_set_inode_flags(struct inode *inode, bool init)
4438{
4439 unsigned int flags = EXT4_I(inode)->i_flags;
4440 unsigned int new_fl = 0;
4441
4442 WARN_ON_ONCE(IS_DAX(inode) && init);
4443
4444 if (flags & EXT4_SYNC_FL)
4445 new_fl |= S_SYNC;
4446 if (flags & EXT4_APPEND_FL)
4447 new_fl |= S_APPEND;
4448 if (flags & EXT4_IMMUTABLE_FL)
4449 new_fl |= S_IMMUTABLE;
4450 if (flags & EXT4_NOATIME_FL)
4451 new_fl |= S_NOATIME;
4452 if (flags & EXT4_DIRSYNC_FL)
4453 new_fl |= S_DIRSYNC;
4454
4455
4456
4457 new_fl |= (inode->i_flags & S_DAX);
4458 if (init && ext4_should_enable_dax(inode))
4459 new_fl |= S_DAX;
4460
4461 if (flags & EXT4_ENCRYPT_FL)
4462 new_fl |= S_ENCRYPTED;
4463 if (flags & EXT4_CASEFOLD_FL)
4464 new_fl |= S_CASEFOLD;
4465 if (flags & EXT4_VERITY_FL)
4466 new_fl |= S_VERITY;
4467 inode_set_flags(inode, new_fl,
4468 S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
4469 S_ENCRYPTED|S_CASEFOLD|S_VERITY);
4470}
4471
4472static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
4473 struct ext4_inode_info *ei)
4474{
4475 blkcnt_t i_blocks ;
4476 struct inode *inode = &(ei->vfs_inode);
4477 struct super_block *sb = inode->i_sb;
4478
4479 if (ext4_has_feature_huge_file(sb)) {
4480
4481 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
4482 le32_to_cpu(raw_inode->i_blocks_lo);
4483 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
4484
4485 return i_blocks << (inode->i_blkbits - 9);
4486 } else {
4487 return i_blocks;
4488 }
4489 } else {
4490 return le32_to_cpu(raw_inode->i_blocks_lo);
4491 }
4492}
4493
4494static inline int ext4_iget_extra_inode(struct inode *inode,
4495 struct ext4_inode *raw_inode,
4496 struct ext4_inode_info *ei)
4497{
4498 __le32 *magic = (void *)raw_inode +
4499 EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
4500
4501 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize + sizeof(__le32) <=
4502 EXT4_INODE_SIZE(inode->i_sb) &&
4503 *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
4504 ext4_set_inode_state(inode, EXT4_STATE_XATTR);
4505 return ext4_find_inline_data_nolock(inode);
4506 } else
4507 EXT4_I(inode)->i_inline_off = 0;
4508 return 0;
4509}
4510
4511int ext4_get_projid(struct inode *inode, kprojid_t *projid)
4512{
4513 if (!ext4_has_feature_project(inode->i_sb))
4514 return -EOPNOTSUPP;
4515 *projid = EXT4_I(inode)->i_projid;
4516 return 0;
4517}
4518
4519
4520
4521
4522
4523
4524static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
4525{
4526 if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4527 inode_set_iversion_raw(inode, val);
4528 else
4529 inode_set_iversion_queried(inode, val);
4530}
4531static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
4532{
4533 if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4534 return inode_peek_iversion_raw(inode);
4535 else
4536 return inode_peek_iversion(inode);
4537}
4538
4539struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
4540 ext4_iget_flags flags, const char *function,
4541 unsigned int line)
4542{
4543 struct ext4_iloc iloc;
4544 struct ext4_inode *raw_inode;
4545 struct ext4_inode_info *ei;
4546 struct ext4_super_block *es = EXT4_SB(sb)->s_es;
4547 struct inode *inode;
4548 journal_t *journal = EXT4_SB(sb)->s_journal;
4549 long ret;
4550 loff_t size;
4551 int block;
4552 uid_t i_uid;
4553 gid_t i_gid;
4554 projid_t i_projid;
4555
4556 if ((!(flags & EXT4_IGET_SPECIAL) &&
4557 ((ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO) ||
4558 ino == le32_to_cpu(es->s_usr_quota_inum) ||
4559 ino == le32_to_cpu(es->s_grp_quota_inum) ||
4560 ino == le32_to_cpu(es->s_prj_quota_inum) ||
4561 ino == le32_to_cpu(es->s_orphan_file_inum))) ||
4562 (ino < EXT4_ROOT_INO) ||
4563 (ino > le32_to_cpu(es->s_inodes_count))) {
4564 if (flags & EXT4_IGET_HANDLE)
4565 return ERR_PTR(-ESTALE);
4566 __ext4_error(sb, function, line, false, EFSCORRUPTED, 0,
4567 "inode #%lu: comm %s: iget: illegal inode #",
4568 ino, current->comm);
4569 return ERR_PTR(-EFSCORRUPTED);
4570 }
4571
4572 inode = iget_locked(sb, ino);
4573 if (!inode)
4574 return ERR_PTR(-ENOMEM);
4575 if (!(inode->i_state & I_NEW))
4576 return inode;
4577
4578 ei = EXT4_I(inode);
4579 iloc.bh = NULL;
4580
4581 ret = __ext4_get_inode_loc_noinmem(inode, &iloc);
4582 if (ret < 0)
4583 goto bad_inode;
4584 raw_inode = ext4_raw_inode(&iloc);
4585
4586 if ((ino == EXT4_ROOT_INO) && (raw_inode->i_links_count == 0)) {
4587 ext4_error_inode(inode, function, line, 0,
4588 "iget: root inode unallocated");
4589 ret = -EFSCORRUPTED;
4590 goto bad_inode;
4591 }
4592
4593 if ((flags & EXT4_IGET_HANDLE) &&
4594 (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
4595 ret = -ESTALE;
4596 goto bad_inode;
4597 }
4598
4599 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4600 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
4601 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
4602 EXT4_INODE_SIZE(inode->i_sb) ||
4603 (ei->i_extra_isize & 3)) {
4604 ext4_error_inode(inode, function, line, 0,
4605 "iget: bad extra_isize %u "
4606 "(inode size %u)",
4607 ei->i_extra_isize,
4608 EXT4_INODE_SIZE(inode->i_sb));
4609 ret = -EFSCORRUPTED;
4610 goto bad_inode;
4611 }
4612 } else
4613 ei->i_extra_isize = 0;
4614
4615
4616 if (ext4_has_metadata_csum(sb)) {
4617 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4618 __u32 csum;
4619 __le32 inum = cpu_to_le32(inode->i_ino);
4620 __le32 gen = raw_inode->i_generation;
4621 csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum,
4622 sizeof(inum));
4623 ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen,
4624 sizeof(gen));
4625 }
4626
4627 if ((!ext4_inode_csum_verify(inode, raw_inode, ei) ||
4628 ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&
4629 (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {
4630 ext4_error_inode_err(inode, function, line, 0,
4631 EFSBADCRC, "iget: checksum invalid");
4632 ret = -EFSBADCRC;
4633 goto bad_inode;
4634 }
4635
4636 inode->i_mode = le16_to_cpu(raw_inode->i_mode);
4637 i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
4638 i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
4639 if (ext4_has_feature_project(sb) &&
4640 EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4641 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4642 i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
4643 else
4644 i_projid = EXT4_DEF_PROJID;
4645
4646 if (!(test_opt(inode->i_sb, NO_UID32))) {
4647 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
4648 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
4649 }
4650 i_uid_write(inode, i_uid);
4651 i_gid_write(inode, i_gid);
4652 ei->i_projid = make_kprojid(&init_user_ns, i_projid);
4653 set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
4654
4655 ext4_clear_state_flags(ei);
4656 ei->i_inline_off = 0;
4657 ei->i_dir_start_lookup = 0;
4658 ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
4659
4660
4661
4662
4663
4664 if (inode->i_nlink == 0) {
4665 if ((inode->i_mode == 0 ||
4666 !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
4667 ino != EXT4_BOOT_LOADER_INO) {
4668
4669 ret = -ESTALE;
4670 goto bad_inode;
4671 }
4672
4673
4674
4675
4676
4677
4678 }
4679 ei->i_flags = le32_to_cpu(raw_inode->i_flags);
4680 ext4_set_inode_flags(inode, true);
4681 inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
4682 ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
4683 if (ext4_has_feature_64bit(sb))
4684 ei->i_file_acl |=
4685 ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
4686 inode->i_size = ext4_isize(sb, raw_inode);
4687 if ((size = i_size_read(inode)) < 0) {
4688 ext4_error_inode(inode, function, line, 0,
4689 "iget: bad i_size value: %lld", size);
4690 ret = -EFSCORRUPTED;
4691 goto bad_inode;
4692 }
4693
4694
4695
4696
4697
4698 if (!ext4_has_feature_dir_index(sb) && ext4_has_metadata_csum(sb) &&
4699 ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {
4700 ext4_error_inode(inode, function, line, 0,
4701 "iget: Dir with htree data on filesystem without dir_index feature.");
4702 ret = -EFSCORRUPTED;
4703 goto bad_inode;
4704 }
4705 ei->i_disksize = inode->i_size;
4706#ifdef CONFIG_QUOTA
4707 ei->i_reserved_quota = 0;
4708#endif
4709 inode->i_generation = le32_to_cpu(raw_inode->i_generation);
4710 ei->i_block_group = iloc.block_group;
4711 ei->i_last_alloc_group = ~0;
4712
4713
4714
4715
4716 for (block = 0; block < EXT4_N_BLOCKS; block++)
4717 ei->i_data[block] = raw_inode->i_block[block];
4718 INIT_LIST_HEAD(&ei->i_orphan);
4719 ext4_fc_init_inode(&ei->vfs_inode);
4720
4721
4722
4723
4724
4725
4726
4727
4728 if (journal) {
4729 transaction_t *transaction;
4730 tid_t tid;
4731
4732 read_lock(&journal->j_state_lock);
4733 if (journal->j_running_transaction)
4734 transaction = journal->j_running_transaction;
4735 else
4736 transaction = journal->j_committing_transaction;
4737 if (transaction)
4738 tid = transaction->t_tid;
4739 else
4740 tid = journal->j_commit_sequence;
4741 read_unlock(&journal->j_state_lock);
4742 ei->i_sync_tid = tid;
4743 ei->i_datasync_tid = tid;
4744 }
4745
4746 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4747 if (ei->i_extra_isize == 0) {
4748
4749 BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
4750 ei->i_extra_isize = sizeof(struct ext4_inode) -
4751 EXT4_GOOD_OLD_INODE_SIZE;
4752 } else {
4753 ret = ext4_iget_extra_inode(inode, raw_inode, ei);
4754 if (ret)
4755 goto bad_inode;
4756 }
4757 }
4758
4759 EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode);
4760 EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode);
4761 EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode);
4762 EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
4763
4764 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
4765 u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
4766
4767 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4768 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4769 ivers |=
4770 (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
4771 }
4772 ext4_inode_set_iversion_queried(inode, ivers);
4773 }
4774
4775 ret = 0;
4776 if (ei->i_file_acl &&
4777 !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {
4778 ext4_error_inode(inode, function, line, 0,
4779 "iget: bad extended attribute block %llu",
4780 ei->i_file_acl);
4781 ret = -EFSCORRUPTED;
4782 goto bad_inode;
4783 } else if (!ext4_has_inline_data(inode)) {
4784
4785 if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
4786 (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
4787 (S_ISLNK(inode->i_mode) &&
4788 !ext4_inode_is_fast_symlink(inode)))) {
4789 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4790 ret = ext4_ext_check_inode(inode);
4791 else
4792 ret = ext4_ind_check_inode(inode);
4793 }
4794 }
4795 if (ret)
4796 goto bad_inode;
4797
4798 if (S_ISREG(inode->i_mode)) {
4799 inode->i_op = &ext4_file_inode_operations;
4800 inode->i_fop = &ext4_file_operations;
4801 ext4_set_aops(inode);
4802 } else if (S_ISDIR(inode->i_mode)) {
4803 inode->i_op = &ext4_dir_inode_operations;
4804 inode->i_fop = &ext4_dir_operations;
4805 } else if (S_ISLNK(inode->i_mode)) {
4806
4807 if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
4808 ext4_error_inode(inode, function, line, 0,
4809 "iget: immutable or append flags "
4810 "not allowed on symlinks");
4811 ret = -EFSCORRUPTED;
4812 goto bad_inode;
4813 }
4814 if (IS_ENCRYPTED(inode)) {
4815 inode->i_op = &ext4_encrypted_symlink_inode_operations;
4816 ext4_set_aops(inode);
4817 } else if (ext4_inode_is_fast_symlink(inode)) {
4818 inode->i_link = (char *)ei->i_data;
4819 inode->i_op = &ext4_fast_symlink_inode_operations;
4820 nd_terminate_link(ei->i_data, inode->i_size,
4821 sizeof(ei->i_data) - 1);
4822 } else {
4823 inode->i_op = &ext4_symlink_inode_operations;
4824 ext4_set_aops(inode);
4825 }
4826 inode_nohighmem(inode);
4827 } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
4828 S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
4829 inode->i_op = &ext4_special_inode_operations;
4830 if (raw_inode->i_block[0])
4831 init_special_inode(inode, inode->i_mode,
4832 old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
4833 else
4834 init_special_inode(inode, inode->i_mode,
4835 new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
4836 } else if (ino == EXT4_BOOT_LOADER_INO) {
4837 make_bad_inode(inode);
4838 } else {
4839 ret = -EFSCORRUPTED;
4840 ext4_error_inode(inode, function, line, 0,
4841 "iget: bogus i_mode (%o)", inode->i_mode);
4842 goto bad_inode;
4843 }
4844 if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb))
4845 ext4_error_inode(inode, function, line, 0,
4846 "casefold flag without casefold feature");
4847 brelse(iloc.bh);
4848
4849 unlock_new_inode(inode);
4850 return inode;
4851
4852bad_inode:
4853 brelse(iloc.bh);
4854 iget_failed(inode);
4855 return ERR_PTR(ret);
4856}
4857
4858static int ext4_inode_blocks_set(handle_t *handle,
4859 struct ext4_inode *raw_inode,
4860 struct ext4_inode_info *ei)
4861{
4862 struct inode *inode = &(ei->vfs_inode);
4863 u64 i_blocks = READ_ONCE(inode->i_blocks);
4864 struct super_block *sb = inode->i_sb;
4865
4866 if (i_blocks <= ~0U) {
4867
4868
4869
4870
4871 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
4872 raw_inode->i_blocks_high = 0;
4873 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4874 return 0;
4875 }
4876
4877
4878
4879
4880
4881
4882 if (!ext4_has_feature_huge_file(sb))
4883 return -EFSCORRUPTED;
4884
4885 if (i_blocks <= 0xffffffffffffULL) {
4886
4887
4888
4889
4890 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
4891 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4892 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4893 } else {
4894 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4895
4896 i_blocks = i_blocks >> (inode->i_blkbits - 9);
4897 raw_inode->i_blocks_lo = cpu_to_le32(i_blocks);
4898 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4899 }
4900 return 0;
4901}
4902
4903static void __ext4_update_other_inode_time(struct super_block *sb,
4904 unsigned long orig_ino,
4905 unsigned long ino,
4906 struct ext4_inode *raw_inode)
4907{
4908 struct inode *inode;
4909
4910 inode = find_inode_by_ino_rcu(sb, ino);
4911 if (!inode)
4912 return;
4913
4914 if (!inode_is_dirtytime_only(inode))
4915 return;
4916
4917 spin_lock(&inode->i_lock);
4918 if (inode_is_dirtytime_only(inode)) {
4919 struct ext4_inode_info *ei = EXT4_I(inode);
4920
4921 inode->i_state &= ~I_DIRTY_TIME;
4922 spin_unlock(&inode->i_lock);
4923
4924 spin_lock(&ei->i_raw_lock);
4925 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
4926 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
4927 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
4928 ext4_inode_csum_set(inode, raw_inode, ei);
4929 spin_unlock(&ei->i_raw_lock);
4930 trace_ext4_other_inode_update_time(inode, orig_ino);
4931 return;
4932 }
4933 spin_unlock(&inode->i_lock);
4934}
4935
4936
4937
4938
4939
4940static void ext4_update_other_inodes_time(struct super_block *sb,
4941 unsigned long orig_ino, char *buf)
4942{
4943 unsigned long ino;
4944 int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4945 int inode_size = EXT4_INODE_SIZE(sb);
4946
4947
4948
4949
4950
4951
4952 ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
4953 rcu_read_lock();
4954 for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
4955 if (ino == orig_ino)
4956 continue;
4957 __ext4_update_other_inode_time(sb, orig_ino, ino,
4958 (struct ext4_inode *)buf);
4959 }
4960 rcu_read_unlock();
4961}
4962
4963
4964
4965
4966
4967
4968
4969
4970static int ext4_do_update_inode(handle_t *handle,
4971 struct inode *inode,
4972 struct ext4_iloc *iloc)
4973{
4974 struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
4975 struct ext4_inode_info *ei = EXT4_I(inode);
4976 struct buffer_head *bh = iloc->bh;
4977 struct super_block *sb = inode->i_sb;
4978 int err = 0, block;
4979 int need_datasync = 0, set_large_file = 0;
4980 uid_t i_uid;
4981 gid_t i_gid;
4982 projid_t i_projid;
4983
4984 spin_lock(&ei->i_raw_lock);
4985
4986
4987
4988
4989
4990 if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
4991 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
4992
4993 err = ext4_inode_blocks_set(handle, raw_inode, ei);
4994
4995 raw_inode->i_mode = cpu_to_le16(inode->i_mode);
4996 i_uid = i_uid_read(inode);
4997 i_gid = i_gid_read(inode);
4998 i_projid = from_kprojid(&init_user_ns, ei->i_projid);
4999 if (!(test_opt(inode->i_sb, NO_UID32))) {
5000 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
5001 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
5002
5003
5004
5005
5006
5007 if (ei->i_dtime && list_empty(&ei->i_orphan)) {
5008 raw_inode->i_uid_high = 0;
5009 raw_inode->i_gid_high = 0;
5010 } else {
5011 raw_inode->i_uid_high =
5012 cpu_to_le16(high_16_bits(i_uid));
5013 raw_inode->i_gid_high =
5014 cpu_to_le16(high_16_bits(i_gid));
5015 }
5016 } else {
5017 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
5018 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
5019 raw_inode->i_uid_high = 0;
5020 raw_inode->i_gid_high = 0;
5021 }
5022 raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
5023
5024 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
5025 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
5026 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
5027 EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
5028
5029 raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
5030 raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
5031 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
5032 raw_inode->i_file_acl_high =
5033 cpu_to_le16(ei->i_file_acl >> 32);
5034 raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
5035 if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode)) {
5036 ext4_isize_set(raw_inode, ei->i_disksize);
5037 need_datasync = 1;
5038 }
5039 if (ei->i_disksize > 0x7fffffffULL) {
5040 if (!ext4_has_feature_large_file(sb) ||
5041 EXT4_SB(sb)->s_es->s_rev_level ==
5042 cpu_to_le32(EXT4_GOOD_OLD_REV))
5043 set_large_file = 1;
5044 }
5045 raw_inode->i_generation = cpu_to_le32(inode->i_generation);
5046 if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
5047 if (old_valid_dev(inode->i_rdev)) {
5048 raw_inode->i_block[0] =
5049 cpu_to_le32(old_encode_dev(inode->i_rdev));
5050 raw_inode->i_block[1] = 0;
5051 } else {
5052 raw_inode->i_block[0] = 0;
5053 raw_inode->i_block[1] =
5054 cpu_to_le32(new_encode_dev(inode->i_rdev));
5055 raw_inode->i_block[2] = 0;
5056 }
5057 } else if (!ext4_has_inline_data(inode)) {
5058 for (block = 0; block < EXT4_N_BLOCKS; block++)
5059 raw_inode->i_block[block] = ei->i_data[block];
5060 }
5061
5062 if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
5063 u64 ivers = ext4_inode_peek_iversion(inode);
5064
5065 raw_inode->i_disk_version = cpu_to_le32(ivers);
5066 if (ei->i_extra_isize) {
5067 if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
5068 raw_inode->i_version_hi =
5069 cpu_to_le32(ivers >> 32);
5070 raw_inode->i_extra_isize =
5071 cpu_to_le16(ei->i_extra_isize);
5072 }
5073 }
5074
5075 if (i_projid != EXT4_DEF_PROJID &&
5076 !ext4_has_feature_project(inode->i_sb))
5077 err = err ?: -EFSCORRUPTED;
5078
5079 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
5080 EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
5081 raw_inode->i_projid = cpu_to_le32(i_projid);
5082
5083 ext4_inode_csum_set(inode, raw_inode, ei);
5084 spin_unlock(&ei->i_raw_lock);
5085 if (err) {
5086 EXT4_ERROR_INODE(inode, "corrupted inode contents");
5087 goto out_brelse;
5088 }
5089
5090 if (inode->i_sb->s_flags & SB_LAZYTIME)
5091 ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
5092 bh->b_data);
5093
5094 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
5095 err = ext4_handle_dirty_metadata(handle, NULL, bh);
5096 if (err)
5097 goto out_error;
5098 ext4_clear_inode_state(inode, EXT4_STATE_NEW);
5099 if (set_large_file) {
5100 BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
5101 err = ext4_journal_get_write_access(handle, sb,
5102 EXT4_SB(sb)->s_sbh,
5103 EXT4_JTR_NONE);
5104 if (err)
5105 goto out_error;
5106 lock_buffer(EXT4_SB(sb)->s_sbh);
5107 ext4_set_feature_large_file(sb);
5108 ext4_superblock_csum_set(sb);
5109 unlock_buffer(EXT4_SB(sb)->s_sbh);
5110 ext4_handle_sync(handle);
5111 err = ext4_handle_dirty_metadata(handle, NULL,
5112 EXT4_SB(sb)->s_sbh);
5113 }
5114 ext4_update_inode_fsync_trans(handle, inode, need_datasync);
5115out_error:
5116 ext4_std_error(inode->i_sb, err);
5117out_brelse:
5118 brelse(bh);
5119 return err;
5120}
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
5157{
5158 int err;
5159
5160 if (WARN_ON_ONCE(current->flags & PF_MEMALLOC) ||
5161 sb_rdonly(inode->i_sb))
5162 return 0;
5163
5164 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5165 return -EIO;
5166
5167 if (EXT4_SB(inode->i_sb)->s_journal) {
5168 if (ext4_journal_current_handle()) {
5169 jbd_debug(1, "called recursively, non-PF_MEMALLOC!\n");
5170 dump_stack();
5171 return -EIO;
5172 }
5173
5174
5175
5176
5177
5178
5179 if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
5180 return 0;
5181
5182 err = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
5183 EXT4_I(inode)->i_sync_tid);
5184 } else {
5185 struct ext4_iloc iloc;
5186
5187 err = __ext4_get_inode_loc_noinmem(inode, &iloc);
5188 if (err)
5189 return err;
5190
5191
5192
5193
5194 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
5195 sync_dirty_buffer(iloc.bh);
5196 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
5197 ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,
5198 "IO error syncing inode");
5199 err = -EIO;
5200 }
5201 brelse(iloc.bh);
5202 }
5203 return err;
5204}
5205
5206
5207
5208
5209
5210
5211static void ext4_wait_for_tail_page_commit(struct inode *inode)
5212{
5213 struct page *page;
5214 unsigned offset;
5215 journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
5216 tid_t commit_tid = 0;
5217 int ret;
5218
5219 offset = inode->i_size & (PAGE_SIZE - 1);
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229 if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))
5230 return;
5231 while (1) {
5232 page = find_lock_page(inode->i_mapping,
5233 inode->i_size >> PAGE_SHIFT);
5234 if (!page)
5235 return;
5236 ret = __ext4_journalled_invalidatepage(page, offset,
5237 PAGE_SIZE - offset);
5238 unlock_page(page);
5239 put_page(page);
5240 if (ret != -EBUSY)
5241 return;
5242 commit_tid = 0;
5243 read_lock(&journal->j_state_lock);
5244 if (journal->j_committing_transaction)
5245 commit_tid = journal->j_committing_transaction->t_tid;
5246 read_unlock(&journal->j_state_lock);
5247 if (commit_tid)
5248 jbd2_log_wait_commit(journal, commit_tid);
5249 }
5250}
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry,
5277 struct iattr *attr)
5278{
5279 struct inode *inode = d_inode(dentry);
5280 int error, rc = 0;
5281 int orphan = 0;
5282 const unsigned int ia_valid = attr->ia_valid;
5283
5284 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5285 return -EIO;
5286
5287 if (unlikely(IS_IMMUTABLE(inode)))
5288 return -EPERM;
5289
5290 if (unlikely(IS_APPEND(inode) &&
5291 (ia_valid & (ATTR_MODE | ATTR_UID |
5292 ATTR_GID | ATTR_TIMES_SET))))
5293 return -EPERM;
5294
5295 error = setattr_prepare(mnt_userns, dentry, attr);
5296 if (error)
5297 return error;
5298
5299 error = fscrypt_prepare_setattr(dentry, attr);
5300 if (error)
5301 return error;
5302
5303 error = fsverity_prepare_setattr(dentry, attr);
5304 if (error)
5305 return error;
5306
5307 if (is_quota_modification(inode, attr)) {
5308 error = dquot_initialize(inode);
5309 if (error)
5310 return error;
5311 }
5312 ext4_fc_start_update(inode);
5313 if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) ||
5314 (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) {
5315 handle_t *handle;
5316
5317
5318
5319 handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
5320 (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
5321 EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
5322 if (IS_ERR(handle)) {
5323 error = PTR_ERR(handle);
5324 goto err_out;
5325 }
5326
5327
5328
5329
5330 down_read(&EXT4_I(inode)->xattr_sem);
5331 error = dquot_transfer(inode, attr);
5332 up_read(&EXT4_I(inode)->xattr_sem);
5333
5334 if (error) {
5335 ext4_journal_stop(handle);
5336 ext4_fc_stop_update(inode);
5337 return error;
5338 }
5339
5340
5341 if (attr->ia_valid & ATTR_UID)
5342 inode->i_uid = attr->ia_uid;
5343 if (attr->ia_valid & ATTR_GID)
5344 inode->i_gid = attr->ia_gid;
5345 error = ext4_mark_inode_dirty(handle, inode);
5346 ext4_journal_stop(handle);
5347 if (unlikely(error)) {
5348 ext4_fc_stop_update(inode);
5349 return error;
5350 }
5351 }
5352
5353 if (attr->ia_valid & ATTR_SIZE) {
5354 handle_t *handle;
5355 loff_t oldsize = inode->i_size;
5356 int shrink = (attr->ia_size < inode->i_size);
5357
5358 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
5359 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5360
5361 if (attr->ia_size > sbi->s_bitmap_maxbytes) {
5362 ext4_fc_stop_update(inode);
5363 return -EFBIG;
5364 }
5365 }
5366 if (!S_ISREG(inode->i_mode)) {
5367 ext4_fc_stop_update(inode);
5368 return -EINVAL;
5369 }
5370
5371 if (IS_I_VERSION(inode) && attr->ia_size != inode->i_size)
5372 inode_inc_iversion(inode);
5373
5374 if (shrink) {
5375 if (ext4_should_order_data(inode)) {
5376 error = ext4_begin_ordered_truncate(inode,
5377 attr->ia_size);
5378 if (error)
5379 goto err_out;
5380 }
5381
5382
5383
5384
5385 inode_dio_wait(inode);
5386 }
5387
5388 filemap_invalidate_lock(inode->i_mapping);
5389
5390 rc = ext4_break_layouts(inode);
5391 if (rc) {
5392 filemap_invalidate_unlock(inode->i_mapping);
5393 goto err_out;
5394 }
5395
5396 if (attr->ia_size != inode->i_size) {
5397 handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
5398 if (IS_ERR(handle)) {
5399 error = PTR_ERR(handle);
5400 goto out_mmap_sem;
5401 }
5402 if (ext4_handle_valid(handle) && shrink) {
5403 error = ext4_orphan_add(handle, inode);
5404 orphan = 1;
5405 }
5406
5407
5408
5409
5410 if (!shrink) {
5411 inode->i_mtime = current_time(inode);
5412 inode->i_ctime = inode->i_mtime;
5413 }
5414
5415 if (shrink)
5416 ext4_fc_track_range(handle, inode,
5417 (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5418 inode->i_sb->s_blocksize_bits,
5419 (oldsize > 0 ? oldsize - 1 : 0) >>
5420 inode->i_sb->s_blocksize_bits);
5421 else
5422 ext4_fc_track_range(
5423 handle, inode,
5424 (oldsize > 0 ? oldsize - 1 : oldsize) >>
5425 inode->i_sb->s_blocksize_bits,
5426 (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5427 inode->i_sb->s_blocksize_bits);
5428
5429 down_write(&EXT4_I(inode)->i_data_sem);
5430 EXT4_I(inode)->i_disksize = attr->ia_size;
5431 rc = ext4_mark_inode_dirty(handle, inode);
5432 if (!error)
5433 error = rc;
5434
5435
5436
5437
5438
5439 if (!error)
5440 i_size_write(inode, attr->ia_size);
5441 up_write(&EXT4_I(inode)->i_data_sem);
5442 ext4_journal_stop(handle);
5443 if (error)
5444 goto out_mmap_sem;
5445 if (!shrink) {
5446 pagecache_isize_extended(inode, oldsize,
5447 inode->i_size);
5448 } else if (ext4_should_journal_data(inode)) {
5449 ext4_wait_for_tail_page_commit(inode);
5450 }
5451 }
5452
5453
5454
5455
5456
5457 truncate_pagecache(inode, inode->i_size);
5458
5459
5460
5461
5462 if (attr->ia_size <= oldsize) {
5463 rc = ext4_truncate(inode);
5464 if (rc)
5465 error = rc;
5466 }
5467out_mmap_sem:
5468 filemap_invalidate_unlock(inode->i_mapping);
5469 }
5470
5471 if (!error) {
5472 setattr_copy(mnt_userns, inode, attr);
5473 mark_inode_dirty(inode);
5474 }
5475
5476
5477
5478
5479
5480 if (orphan && inode->i_nlink)
5481 ext4_orphan_del(NULL, inode);
5482
5483 if (!error && (ia_valid & ATTR_MODE))
5484 rc = posix_acl_chmod(mnt_userns, inode, inode->i_mode);
5485
5486err_out:
5487 if (error)
5488 ext4_std_error(inode->i_sb, error);
5489 if (!error)
5490 error = rc;
5491 ext4_fc_stop_update(inode);
5492 return error;
5493}
5494
5495int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path,
5496 struct kstat *stat, u32 request_mask, unsigned int query_flags)
5497{
5498 struct inode *inode = d_inode(path->dentry);
5499 struct ext4_inode *raw_inode;
5500 struct ext4_inode_info *ei = EXT4_I(inode);
5501 unsigned int flags;
5502
5503 if ((request_mask & STATX_BTIME) &&
5504 EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
5505 stat->result_mask |= STATX_BTIME;
5506 stat->btime.tv_sec = ei->i_crtime.tv_sec;
5507 stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
5508 }
5509
5510 flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
5511 if (flags & EXT4_APPEND_FL)
5512 stat->attributes |= STATX_ATTR_APPEND;
5513 if (flags & EXT4_COMPR_FL)
5514 stat->attributes |= STATX_ATTR_COMPRESSED;
5515 if (flags & EXT4_ENCRYPT_FL)
5516 stat->attributes |= STATX_ATTR_ENCRYPTED;
5517 if (flags & EXT4_IMMUTABLE_FL)
5518 stat->attributes |= STATX_ATTR_IMMUTABLE;
5519 if (flags & EXT4_NODUMP_FL)
5520 stat->attributes |= STATX_ATTR_NODUMP;
5521 if (flags & EXT4_VERITY_FL)
5522 stat->attributes |= STATX_ATTR_VERITY;
5523
5524 stat->attributes_mask |= (STATX_ATTR_APPEND |
5525 STATX_ATTR_COMPRESSED |
5526 STATX_ATTR_ENCRYPTED |
5527 STATX_ATTR_IMMUTABLE |
5528 STATX_ATTR_NODUMP |
5529 STATX_ATTR_VERITY);
5530
5531 generic_fillattr(mnt_userns, inode, stat);
5532 return 0;
5533}
5534
5535int ext4_file_getattr(struct user_namespace *mnt_userns,
5536 const struct path *path, struct kstat *stat,
5537 u32 request_mask, unsigned int query_flags)
5538{
5539 struct inode *inode = d_inode(path->dentry);
5540 u64 delalloc_blocks;
5541
5542 ext4_getattr(mnt_userns, path, stat, request_mask, query_flags);
5543
5544
5545
5546
5547
5548
5549
5550 if (unlikely(ext4_has_inline_data(inode)))
5551 stat->blocks += (stat->size + 511) >> 9;
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563 delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
5564 EXT4_I(inode)->i_reserved_data_blocks);
5565 stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
5566 return 0;
5567}
5568
5569static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
5570 int pextents)
5571{
5572 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
5573 return ext4_ind_trans_blocks(inode, lblocks);
5574 return ext4_ext_index_trans_blocks(inode, pextents);
5575}
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
5589 int pextents)
5590{
5591 ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
5592 int gdpblocks;
5593 int idxblocks;
5594 int ret = 0;
5595
5596
5597
5598
5599
5600 idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
5601
5602 ret = idxblocks;
5603
5604
5605
5606
5607
5608 groups = idxblocks + pextents;
5609 gdpblocks = groups;
5610 if (groups > ngroups)
5611 groups = ngroups;
5612 if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
5613 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
5614
5615
5616 ret += groups + gdpblocks;
5617
5618
5619 ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
5620
5621 return ret;
5622}
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634int ext4_writepage_trans_blocks(struct inode *inode)
5635{
5636 int bpp = ext4_journal_blocks_per_page(inode);
5637 int ret;
5638
5639 ret = ext4_meta_trans_blocks(inode, bpp, bpp);
5640
5641
5642 if (ext4_should_journal_data(inode))
5643 ret += bpp;
5644 return ret;
5645}
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
5657{
5658 return ext4_meta_trans_blocks(inode, nrblocks, 1);
5659}
5660
5661
5662
5663
5664
5665int ext4_mark_iloc_dirty(handle_t *handle,
5666 struct inode *inode, struct ext4_iloc *iloc)
5667{
5668 int err = 0;
5669
5670 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
5671 put_bh(iloc->bh);
5672 return -EIO;
5673 }
5674 ext4_fc_track_inode(handle, inode);
5675
5676 if (IS_I_VERSION(inode))
5677 inode_inc_iversion(inode);
5678
5679
5680 get_bh(iloc->bh);
5681
5682
5683 err = ext4_do_update_inode(handle, inode, iloc);
5684 put_bh(iloc->bh);
5685 return err;
5686}
5687
5688
5689
5690
5691
5692
5693int
5694ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
5695 struct ext4_iloc *iloc)
5696{
5697 int err;
5698
5699 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5700 return -EIO;
5701
5702 err = ext4_get_inode_loc(inode, iloc);
5703 if (!err) {
5704 BUFFER_TRACE(iloc->bh, "get_write_access");
5705 err = ext4_journal_get_write_access(handle, inode->i_sb,
5706 iloc->bh, EXT4_JTR_NONE);
5707 if (err) {
5708 brelse(iloc->bh);
5709 iloc->bh = NULL;
5710 }
5711 }
5712 ext4_std_error(inode->i_sb, err);
5713 return err;
5714}
5715
5716static int __ext4_expand_extra_isize(struct inode *inode,
5717 unsigned int new_extra_isize,
5718 struct ext4_iloc *iloc,
5719 handle_t *handle, int *no_expand)
5720{
5721 struct ext4_inode *raw_inode;
5722 struct ext4_xattr_ibody_header *header;
5723 unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
5724 struct ext4_inode_info *ei = EXT4_I(inode);
5725 int error;
5726
5727
5728 if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
5729 (ei->i_extra_isize & 3)) {
5730 EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
5731 ei->i_extra_isize,
5732 EXT4_INODE_SIZE(inode->i_sb));
5733 return -EFSCORRUPTED;
5734 }
5735 if ((new_extra_isize < ei->i_extra_isize) ||
5736 (new_extra_isize < 4) ||
5737 (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
5738 return -EINVAL;
5739
5740 raw_inode = ext4_raw_inode(iloc);
5741
5742 header = IHDR(inode, raw_inode);
5743
5744
5745 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
5746 header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
5747 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
5748 EXT4_I(inode)->i_extra_isize, 0,
5749 new_extra_isize - EXT4_I(inode)->i_extra_isize);
5750 EXT4_I(inode)->i_extra_isize = new_extra_isize;
5751 return 0;
5752 }
5753
5754
5755 error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
5756 raw_inode, handle);
5757 if (error) {
5758
5759
5760
5761 *no_expand = 1;
5762 }
5763
5764 return error;
5765}
5766
5767
5768
5769
5770
5771static int ext4_try_to_expand_extra_isize(struct inode *inode,
5772 unsigned int new_extra_isize,
5773 struct ext4_iloc iloc,
5774 handle_t *handle)
5775{
5776 int no_expand;
5777 int error;
5778
5779 if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
5780 return -EOVERFLOW;
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791 if (ext4_journal_extend(handle,
5792 EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0)
5793 return -ENOSPC;
5794
5795 if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
5796 return -EBUSY;
5797
5798 error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
5799 handle, &no_expand);
5800 ext4_write_unlock_xattr(inode, &no_expand);
5801
5802 return error;
5803}
5804
5805int ext4_expand_extra_isize(struct inode *inode,
5806 unsigned int new_extra_isize,
5807 struct ext4_iloc *iloc)
5808{
5809 handle_t *handle;
5810 int no_expand;
5811 int error, rc;
5812
5813 if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
5814 brelse(iloc->bh);
5815 return -EOVERFLOW;
5816 }
5817
5818 handle = ext4_journal_start(inode, EXT4_HT_INODE,
5819 EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
5820 if (IS_ERR(handle)) {
5821 error = PTR_ERR(handle);
5822 brelse(iloc->bh);
5823 return error;
5824 }
5825
5826 ext4_write_lock_xattr(inode, &no_expand);
5827
5828 BUFFER_TRACE(iloc->bh, "get_write_access");
5829 error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh,
5830 EXT4_JTR_NONE);
5831 if (error) {
5832 brelse(iloc->bh);
5833 goto out_unlock;
5834 }
5835
5836 error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
5837 handle, &no_expand);
5838
5839 rc = ext4_mark_iloc_dirty(handle, inode, iloc);
5840 if (!error)
5841 error = rc;
5842
5843out_unlock:
5844 ext4_write_unlock_xattr(inode, &no_expand);
5845 ext4_journal_stop(handle);
5846 return error;
5847}
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,
5863 const char *func, unsigned int line)
5864{
5865 struct ext4_iloc iloc;
5866 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5867 int err;
5868
5869 might_sleep();
5870 trace_ext4_mark_inode_dirty(inode, _RET_IP_);
5871 err = ext4_reserve_inode_write(handle, inode, &iloc);
5872 if (err)
5873 goto out;
5874
5875 if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
5876 ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
5877 iloc, handle);
5878
5879 err = ext4_mark_iloc_dirty(handle, inode, &iloc);
5880out:
5881 if (unlikely(err))
5882 ext4_error_inode_err(inode, func, line, 0, err,
5883 "mark_inode_dirty error");
5884 return err;
5885}
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901void ext4_dirty_inode(struct inode *inode, int flags)
5902{
5903 handle_t *handle;
5904
5905 handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
5906 if (IS_ERR(handle))
5907 return;
5908 ext4_mark_inode_dirty(handle, inode);
5909 ext4_journal_stop(handle);
5910}
5911
5912int ext4_change_inode_journal_flag(struct inode *inode, int val)
5913{
5914 journal_t *journal;
5915 handle_t *handle;
5916 int err;
5917 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929 journal = EXT4_JOURNAL(inode);
5930 if (!journal)
5931 return 0;
5932 if (is_journal_aborted(journal))
5933 return -EROFS;
5934
5935
5936 inode_dio_wait(inode);
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946 if (val) {
5947 filemap_invalidate_lock(inode->i_mapping);
5948 err = filemap_write_and_wait(inode->i_mapping);
5949 if (err < 0) {
5950 filemap_invalidate_unlock(inode->i_mapping);
5951 return err;
5952 }
5953 }
5954
5955 percpu_down_write(&sbi->s_writepages_rwsem);
5956 jbd2_journal_lock_updates(journal);
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966 if (val)
5967 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
5968 else {
5969 err = jbd2_journal_flush(journal, 0);
5970 if (err < 0) {
5971 jbd2_journal_unlock_updates(journal);
5972 percpu_up_write(&sbi->s_writepages_rwsem);
5973 return err;
5974 }
5975 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
5976 }
5977 ext4_set_aops(inode);
5978
5979 jbd2_journal_unlock_updates(journal);
5980 percpu_up_write(&sbi->s_writepages_rwsem);
5981
5982 if (val)
5983 filemap_invalidate_unlock(inode->i_mapping);
5984
5985
5986
5987 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
5988 if (IS_ERR(handle))
5989 return PTR_ERR(handle);
5990
5991 ext4_fc_mark_ineligible(inode->i_sb,
5992 EXT4_FC_REASON_JOURNAL_FLAG_CHANGE);
5993 err = ext4_mark_inode_dirty(handle, inode);
5994 ext4_handle_sync(handle);
5995 ext4_journal_stop(handle);
5996 ext4_std_error(inode->i_sb, err);
5997
5998 return err;
5999}
6000
6001static int ext4_bh_unmapped(handle_t *handle, struct inode *inode,
6002 struct buffer_head *bh)
6003{
6004 return !buffer_mapped(bh);
6005}
6006
6007vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf)
6008{
6009 struct vm_area_struct *vma = vmf->vma;
6010 struct page *page = vmf->page;
6011 loff_t size;
6012 unsigned long len;
6013 int err;
6014 vm_fault_t ret;
6015 struct file *file = vma->vm_file;
6016 struct inode *inode = file_inode(file);
6017 struct address_space *mapping = inode->i_mapping;
6018 handle_t *handle;
6019 get_block_t *get_block;
6020 int retries = 0;
6021
6022 if (unlikely(IS_IMMUTABLE(inode)))
6023 return VM_FAULT_SIGBUS;
6024
6025 sb_start_pagefault(inode->i_sb);
6026 file_update_time(vma->vm_file);
6027
6028 filemap_invalidate_lock_shared(mapping);
6029
6030 err = ext4_convert_inline_data(inode);
6031 if (err)
6032 goto out_ret;
6033
6034
6035
6036
6037
6038
6039
6040 if (ext4_should_journal_data(inode))
6041 goto retry_alloc;
6042
6043
6044 if (test_opt(inode->i_sb, DELALLOC) &&
6045 !ext4_nonda_switch(inode->i_sb)) {
6046 do {
6047 err = block_page_mkwrite(vma, vmf,
6048 ext4_da_get_block_prep);
6049 } while (err == -ENOSPC &&
6050 ext4_should_retry_alloc(inode->i_sb, &retries));
6051 goto out_ret;
6052 }
6053
6054 lock_page(page);
6055 size = i_size_read(inode);
6056
6057 if (page->mapping != mapping || page_offset(page) > size) {
6058 unlock_page(page);
6059 ret = VM_FAULT_NOPAGE;
6060 goto out;
6061 }
6062
6063 if (page->index == size >> PAGE_SHIFT)
6064 len = size & ~PAGE_MASK;
6065 else
6066 len = PAGE_SIZE;
6067
6068
6069
6070
6071
6072
6073
6074 if (page_has_buffers(page)) {
6075 if (!ext4_walk_page_buffers(NULL, inode, page_buffers(page),
6076 0, len, NULL,
6077 ext4_bh_unmapped)) {
6078
6079 wait_for_stable_page(page);
6080 ret = VM_FAULT_LOCKED;
6081 goto out;
6082 }
6083 }
6084 unlock_page(page);
6085
6086 if (ext4_should_dioread_nolock(inode))
6087 get_block = ext4_get_block_unwritten;
6088 else
6089 get_block = ext4_get_block;
6090retry_alloc:
6091 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
6092 ext4_writepage_trans_blocks(inode));
6093 if (IS_ERR(handle)) {
6094 ret = VM_FAULT_SIGBUS;
6095 goto out;
6096 }
6097
6098
6099
6100
6101
6102 if (!ext4_should_journal_data(inode)) {
6103 err = block_page_mkwrite(vma, vmf, get_block);
6104 } else {
6105 lock_page(page);
6106 size = i_size_read(inode);
6107
6108 if (page->mapping != mapping || page_offset(page) > size) {
6109 ret = VM_FAULT_NOPAGE;
6110 goto out_error;
6111 }
6112
6113 if (page->index == size >> PAGE_SHIFT)
6114 len = size & ~PAGE_MASK;
6115 else
6116 len = PAGE_SIZE;
6117
6118 err = __block_write_begin(page, 0, len, ext4_get_block);
6119 if (!err) {
6120 ret = VM_FAULT_SIGBUS;
6121 if (ext4_walk_page_buffers(handle, inode,
6122 page_buffers(page), 0, len, NULL,
6123 do_journal_get_write_access))
6124 goto out_error;
6125 if (ext4_walk_page_buffers(handle, inode,
6126 page_buffers(page), 0, len, NULL,
6127 write_end_fn))
6128 goto out_error;
6129 if (ext4_jbd2_inode_add_write(handle, inode,
6130 page_offset(page), len))
6131 goto out_error;
6132 ext4_set_inode_state(inode, EXT4_STATE_JDATA);
6133 } else {
6134 unlock_page(page);
6135 }
6136 }
6137 ext4_journal_stop(handle);
6138 if (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
6139 goto retry_alloc;
6140out_ret:
6141 ret = block_page_mkwrite_return(err);
6142out:
6143 filemap_invalidate_unlock_shared(mapping);
6144 sb_end_pagefault(inode->i_sb);
6145 return ret;
6146out_error:
6147 unlock_page(page);
6148 ext4_journal_stop(handle);
6149 goto out;
6150}
6151