1
2
3
4
5
6
7
8
9
10
11#include <linux/fs.h>
12#include <linux/f2fs_fs.h>
13#include <linux/buffer_head.h>
14#include <linux/mpage.h>
15#include <linux/writeback.h>
16#include <linux/backing-dev.h>
17#include <linux/pagevec.h>
18#include <linux/blkdev.h>
19#include <linux/bio.h>
20#include <linux/prefetch.h>
21#include <linux/uio.h>
22#include <linux/mm.h>
23#include <linux/memcontrol.h>
24#include <linux/cleancache.h>
25#include <linux/sched/signal.h>
26
27#include "f2fs.h"
28#include "node.h"
29#include "segment.h"
30#include "trace.h"
31#include <trace/events/f2fs.h>
32
33static bool __is_cp_guaranteed(struct page *page)
34{
35 struct address_space *mapping = page->mapping;
36 struct inode *inode;
37 struct f2fs_sb_info *sbi;
38
39 if (!mapping)
40 return false;
41
42 inode = mapping->host;
43 sbi = F2FS_I_SB(inode);
44
45 if (inode->i_ino == F2FS_META_INO(sbi) ||
46 inode->i_ino == F2FS_NODE_INO(sbi) ||
47 S_ISDIR(inode->i_mode) ||
48 is_cold_data(page))
49 return true;
50 return false;
51}
52
53static void f2fs_read_end_io(struct bio *bio)
54{
55 struct bio_vec *bvec;
56 int i;
57
58#ifdef CONFIG_F2FS_FAULT_INJECTION
59 if (time_to_inject(F2FS_P_SB(bio_first_page_all(bio)), FAULT_IO)) {
60 f2fs_show_injection_info(FAULT_IO);
61 bio->bi_status = BLK_STS_IOERR;
62 }
63#endif
64
65 if (f2fs_bio_encrypted(bio)) {
66 if (bio->bi_status) {
67 fscrypt_release_ctx(bio->bi_private);
68 } else {
69 fscrypt_decrypt_bio_pages(bio->bi_private, bio);
70 return;
71 }
72 }
73
74 bio_for_each_segment_all(bvec, bio, i) {
75 struct page *page = bvec->bv_page;
76
77 if (!bio->bi_status) {
78 if (!PageUptodate(page))
79 SetPageUptodate(page);
80 } else {
81 ClearPageUptodate(page);
82 SetPageError(page);
83 }
84 unlock_page(page);
85 }
86 bio_put(bio);
87}
88
89static void f2fs_write_end_io(struct bio *bio)
90{
91 struct f2fs_sb_info *sbi = bio->bi_private;
92 struct bio_vec *bvec;
93 int i;
94
95 bio_for_each_segment_all(bvec, bio, i) {
96 struct page *page = bvec->bv_page;
97 enum count_type type = WB_DATA_TYPE(page);
98
99 if (IS_DUMMY_WRITTEN_PAGE(page)) {
100 set_page_private(page, (unsigned long)NULL);
101 ClearPagePrivate(page);
102 unlock_page(page);
103 mempool_free(page, sbi->write_io_dummy);
104
105 if (unlikely(bio->bi_status))
106 f2fs_stop_checkpoint(sbi, true);
107 continue;
108 }
109
110 fscrypt_pullback_bio_page(&page, true);
111
112 if (unlikely(bio->bi_status)) {
113 mapping_set_error(page->mapping, -EIO);
114 if (type == F2FS_WB_CP_DATA)
115 f2fs_stop_checkpoint(sbi, true);
116 }
117
118 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) &&
119 page->index != nid_of_node(page));
120
121 dec_page_count(sbi, type);
122 clear_cold_data(page);
123 end_page_writeback(page);
124 }
125 if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
126 wq_has_sleeper(&sbi->cp_wait))
127 wake_up(&sbi->cp_wait);
128
129 bio_put(bio);
130}
131
132
133
134
135struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
136 block_t blk_addr, struct bio *bio)
137{
138 struct block_device *bdev = sbi->sb->s_bdev;
139 int i;
140
141 for (i = 0; i < sbi->s_ndevs; i++) {
142 if (FDEV(i).start_blk <= blk_addr &&
143 FDEV(i).end_blk >= blk_addr) {
144 blk_addr -= FDEV(i).start_blk;
145 bdev = FDEV(i).bdev;
146 break;
147 }
148 }
149 if (bio) {
150 bio_set_dev(bio, bdev);
151 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
152 }
153 return bdev;
154}
155
156int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
157{
158 int i;
159
160 for (i = 0; i < sbi->s_ndevs; i++)
161 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
162 return i;
163 return 0;
164}
165
166static bool __same_bdev(struct f2fs_sb_info *sbi,
167 block_t blk_addr, struct bio *bio)
168{
169 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL);
170 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno;
171}
172
173
174
175
176static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
177 struct writeback_control *wbc,
178 int npages, bool is_read,
179 enum page_type type, enum temp_type temp)
180{
181 struct bio *bio;
182
183 bio = f2fs_bio_alloc(sbi, npages, true);
184
185 f2fs_target_device(sbi, blk_addr, bio);
186 if (is_read) {
187 bio->bi_end_io = f2fs_read_end_io;
188 bio->bi_private = NULL;
189 } else {
190 bio->bi_end_io = f2fs_write_end_io;
191 bio->bi_private = sbi;
192 bio->bi_write_hint = io_type_to_rw_hint(sbi, type, temp);
193 }
194 if (wbc)
195 wbc_init_bio(wbc, bio);
196
197 return bio;
198}
199
200static inline void __submit_bio(struct f2fs_sb_info *sbi,
201 struct bio *bio, enum page_type type)
202{
203 if (!is_read_io(bio_op(bio))) {
204 unsigned int start;
205
206 if (type != DATA && type != NODE)
207 goto submit_io;
208
209 if (f2fs_sb_has_blkzoned(sbi->sb) && current->plug)
210 blk_finish_plug(current->plug);
211
212 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
213 start %= F2FS_IO_SIZE(sbi);
214
215 if (start == 0)
216 goto submit_io;
217
218
219 for (; start < F2FS_IO_SIZE(sbi); start++) {
220 struct page *page =
221 mempool_alloc(sbi->write_io_dummy,
222 GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
223 f2fs_bug_on(sbi, !page);
224
225 SetPagePrivate(page);
226 set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
227 lock_page(page);
228 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
229 f2fs_bug_on(sbi, 1);
230 }
231
232
233
234
235 if (type == NODE)
236 set_sbi_flag(sbi, SBI_NEED_CP);
237 }
238submit_io:
239 if (is_read_io(bio_op(bio)))
240 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
241 else
242 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
243 submit_bio(bio);
244}
245
246static void __submit_merged_bio(struct f2fs_bio_info *io)
247{
248 struct f2fs_io_info *fio = &io->fio;
249
250 if (!io->bio)
251 return;
252
253 bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
254
255 if (is_read_io(fio->op))
256 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
257 else
258 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
259
260 __submit_bio(io->sbi, io->bio, fio->type);
261 io->bio = NULL;
262}
263
264static bool __has_merged_page(struct f2fs_bio_info *io,
265 struct inode *inode, nid_t ino, pgoff_t idx)
266{
267 struct bio_vec *bvec;
268 struct page *target;
269 int i;
270
271 if (!io->bio)
272 return false;
273
274 if (!inode && !ino)
275 return true;
276
277 bio_for_each_segment_all(bvec, io->bio, i) {
278
279 if (bvec->bv_page->mapping)
280 target = bvec->bv_page;
281 else
282 target = fscrypt_control_page(bvec->bv_page);
283
284 if (idx != target->index)
285 continue;
286
287 if (inode && inode == target->mapping->host)
288 return true;
289 if (ino && ino == ino_of_node(target))
290 return true;
291 }
292
293 return false;
294}
295
296static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
297 nid_t ino, pgoff_t idx, enum page_type type)
298{
299 enum page_type btype = PAGE_TYPE_OF_BIO(type);
300 enum temp_type temp;
301 struct f2fs_bio_info *io;
302 bool ret = false;
303
304 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
305 io = sbi->write_io[btype] + temp;
306
307 down_read(&io->io_rwsem);
308 ret = __has_merged_page(io, inode, ino, idx);
309 up_read(&io->io_rwsem);
310
311
312 if (ret || btype == META)
313 break;
314 }
315 return ret;
316}
317
318static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
319 enum page_type type, enum temp_type temp)
320{
321 enum page_type btype = PAGE_TYPE_OF_BIO(type);
322 struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
323
324 down_write(&io->io_rwsem);
325
326
327 if (type >= META_FLUSH) {
328 io->fio.type = META_FLUSH;
329 io->fio.op = REQ_OP_WRITE;
330 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
331 if (!test_opt(sbi, NOBARRIER))
332 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
333 }
334 __submit_merged_bio(io);
335 up_write(&io->io_rwsem);
336}
337
338static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
339 struct inode *inode, nid_t ino, pgoff_t idx,
340 enum page_type type, bool force)
341{
342 enum temp_type temp;
343
344 if (!force && !has_merged_page(sbi, inode, ino, idx, type))
345 return;
346
347 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
348
349 __f2fs_submit_merged_write(sbi, type, temp);
350
351
352 if (type >= META)
353 break;
354 }
355}
356
357void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
358{
359 __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
360}
361
362void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
363 struct inode *inode, nid_t ino, pgoff_t idx,
364 enum page_type type)
365{
366 __submit_merged_write_cond(sbi, inode, ino, idx, type, false);
367}
368
369void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
370{
371 f2fs_submit_merged_write(sbi, DATA);
372 f2fs_submit_merged_write(sbi, NODE);
373 f2fs_submit_merged_write(sbi, META);
374}
375
376
377
378
379
380int f2fs_submit_page_bio(struct f2fs_io_info *fio)
381{
382 struct bio *bio;
383 struct page *page = fio->encrypted_page ?
384 fio->encrypted_page : fio->page;
385
386 verify_block_addr(fio, fio->new_blkaddr);
387 trace_f2fs_submit_page_bio(page, fio);
388 f2fs_trace_ios(fio, 0);
389
390
391 bio = __bio_alloc(fio->sbi, fio->new_blkaddr, fio->io_wbc,
392 1, is_read_io(fio->op), fio->type, fio->temp);
393
394 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
395 bio_put(bio);
396 return -EFAULT;
397 }
398 bio_set_op_attrs(bio, fio->op, fio->op_flags);
399
400 __submit_bio(fio->sbi, bio, fio->type);
401
402 if (!is_read_io(fio->op))
403 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
404 return 0;
405}
406
407int f2fs_submit_page_write(struct f2fs_io_info *fio)
408{
409 struct f2fs_sb_info *sbi = fio->sbi;
410 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
411 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
412 struct page *bio_page;
413 int err = 0;
414
415 f2fs_bug_on(sbi, is_read_io(fio->op));
416
417 down_write(&io->io_rwsem);
418next:
419 if (fio->in_list) {
420 spin_lock(&io->io_lock);
421 if (list_empty(&io->io_list)) {
422 spin_unlock(&io->io_lock);
423 goto out_fail;
424 }
425 fio = list_first_entry(&io->io_list,
426 struct f2fs_io_info, list);
427 list_del(&fio->list);
428 spin_unlock(&io->io_lock);
429 }
430
431 if (fio->old_blkaddr != NEW_ADDR)
432 verify_block_addr(fio, fio->old_blkaddr);
433 verify_block_addr(fio, fio->new_blkaddr);
434
435 bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
436
437
438 fio->submitted = true;
439
440 inc_page_count(sbi, WB_DATA_TYPE(bio_page));
441
442 if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
443 (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
444 !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
445 __submit_merged_bio(io);
446alloc_new:
447 if (io->bio == NULL) {
448 if ((fio->type == DATA || fio->type == NODE) &&
449 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
450 err = -EAGAIN;
451 dec_page_count(sbi, WB_DATA_TYPE(bio_page));
452 goto out_fail;
453 }
454 io->bio = __bio_alloc(sbi, fio->new_blkaddr, fio->io_wbc,
455 BIO_MAX_PAGES, false,
456 fio->type, fio->temp);
457 io->fio = *fio;
458 }
459
460 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
461 __submit_merged_bio(io);
462 goto alloc_new;
463 }
464
465 if (fio->io_wbc)
466 wbc_account_io(fio->io_wbc, bio_page, PAGE_SIZE);
467
468 io->last_block_in_bio = fio->new_blkaddr;
469 f2fs_trace_ios(fio, 0);
470
471 trace_f2fs_submit_page_write(fio->page, fio);
472
473 if (fio->in_list)
474 goto next;
475out_fail:
476 up_write(&io->io_rwsem);
477 return err;
478}
479
480static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr,
481 unsigned nr_pages)
482{
483 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
484 struct fscrypt_ctx *ctx = NULL;
485 struct bio *bio;
486
487 if (f2fs_encrypted_file(inode)) {
488 ctx = fscrypt_get_ctx(inode, GFP_NOFS);
489 if (IS_ERR(ctx))
490 return ERR_CAST(ctx);
491
492
493 f2fs_wait_on_block_writeback(sbi, blkaddr);
494 }
495
496 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), false);
497 if (!bio) {
498 if (ctx)
499 fscrypt_release_ctx(ctx);
500 return ERR_PTR(-ENOMEM);
501 }
502 f2fs_target_device(sbi, blkaddr, bio);
503 bio->bi_end_io = f2fs_read_end_io;
504 bio->bi_private = ctx;
505 bio_set_op_attrs(bio, REQ_OP_READ, 0);
506
507 return bio;
508}
509
510
511static int f2fs_submit_page_read(struct inode *inode, struct page *page,
512 block_t blkaddr)
513{
514 struct bio *bio = f2fs_grab_read_bio(inode, blkaddr, 1);
515
516 if (IS_ERR(bio))
517 return PTR_ERR(bio);
518
519 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
520 bio_put(bio);
521 return -EFAULT;
522 }
523 __submit_bio(F2FS_I_SB(inode), bio, DATA);
524 return 0;
525}
526
527static void __set_data_blkaddr(struct dnode_of_data *dn)
528{
529 struct f2fs_node *rn = F2FS_NODE(dn->node_page);
530 __le32 *addr_array;
531 int base = 0;
532
533 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode))
534 base = get_extra_isize(dn->inode);
535
536
537 addr_array = blkaddr_in_node(rn);
538 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
539}
540
541
542
543
544
545
546
547void set_data_blkaddr(struct dnode_of_data *dn)
548{
549 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
550 __set_data_blkaddr(dn);
551 if (set_page_dirty(dn->node_page))
552 dn->node_changed = true;
553}
554
555void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
556{
557 dn->data_blkaddr = blkaddr;
558 set_data_blkaddr(dn);
559 f2fs_update_extent_cache(dn);
560}
561
562
563int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
564{
565 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
566 int err;
567
568 if (!count)
569 return 0;
570
571 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
572 return -EPERM;
573 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
574 return err;
575
576 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
577 dn->ofs_in_node, count);
578
579 f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
580
581 for (; count > 0; dn->ofs_in_node++) {
582 block_t blkaddr = datablock_addr(dn->inode,
583 dn->node_page, dn->ofs_in_node);
584 if (blkaddr == NULL_ADDR) {
585 dn->data_blkaddr = NEW_ADDR;
586 __set_data_blkaddr(dn);
587 count--;
588 }
589 }
590
591 if (set_page_dirty(dn->node_page))
592 dn->node_changed = true;
593 return 0;
594}
595
596
597int reserve_new_block(struct dnode_of_data *dn)
598{
599 unsigned int ofs_in_node = dn->ofs_in_node;
600 int ret;
601
602 ret = reserve_new_blocks(dn, 1);
603 dn->ofs_in_node = ofs_in_node;
604 return ret;
605}
606
607int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
608{
609 bool need_put = dn->inode_page ? false : true;
610 int err;
611
612 err = get_dnode_of_data(dn, index, ALLOC_NODE);
613 if (err)
614 return err;
615
616 if (dn->data_blkaddr == NULL_ADDR)
617 err = reserve_new_block(dn);
618 if (err || need_put)
619 f2fs_put_dnode(dn);
620 return err;
621}
622
623int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
624{
625 struct extent_info ei = {0,0,0};
626 struct inode *inode = dn->inode;
627
628 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
629 dn->data_blkaddr = ei.blk + index - ei.fofs;
630 return 0;
631 }
632
633 return f2fs_reserve_block(dn, index);
634}
635
636struct page *get_read_data_page(struct inode *inode, pgoff_t index,
637 int op_flags, bool for_write)
638{
639 struct address_space *mapping = inode->i_mapping;
640 struct dnode_of_data dn;
641 struct page *page;
642 struct extent_info ei = {0,0,0};
643 int err;
644
645 page = f2fs_grab_cache_page(mapping, index, for_write);
646 if (!page)
647 return ERR_PTR(-ENOMEM);
648
649 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
650 dn.data_blkaddr = ei.blk + index - ei.fofs;
651 goto got_it;
652 }
653
654 set_new_dnode(&dn, inode, NULL, NULL, 0);
655 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
656 if (err)
657 goto put_err;
658 f2fs_put_dnode(&dn);
659
660 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
661 err = -ENOENT;
662 goto put_err;
663 }
664got_it:
665 if (PageUptodate(page)) {
666 unlock_page(page);
667 return page;
668 }
669
670
671
672
673
674
675
676 if (dn.data_blkaddr == NEW_ADDR) {
677 zero_user_segment(page, 0, PAGE_SIZE);
678 if (!PageUptodate(page))
679 SetPageUptodate(page);
680 unlock_page(page);
681 return page;
682 }
683
684 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr);
685 if (err)
686 goto put_err;
687 return page;
688
689put_err:
690 f2fs_put_page(page, 1);
691 return ERR_PTR(err);
692}
693
694struct page *find_data_page(struct inode *inode, pgoff_t index)
695{
696 struct address_space *mapping = inode->i_mapping;
697 struct page *page;
698
699 page = find_get_page(mapping, index);
700 if (page && PageUptodate(page))
701 return page;
702 f2fs_put_page(page, 0);
703
704 page = get_read_data_page(inode, index, 0, false);
705 if (IS_ERR(page))
706 return page;
707
708 if (PageUptodate(page))
709 return page;
710
711 wait_on_page_locked(page);
712 if (unlikely(!PageUptodate(page))) {
713 f2fs_put_page(page, 0);
714 return ERR_PTR(-EIO);
715 }
716 return page;
717}
718
719
720
721
722
723
724struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
725 bool for_write)
726{
727 struct address_space *mapping = inode->i_mapping;
728 struct page *page;
729repeat:
730 page = get_read_data_page(inode, index, 0, for_write);
731 if (IS_ERR(page))
732 return page;
733
734
735 lock_page(page);
736 if (unlikely(page->mapping != mapping)) {
737 f2fs_put_page(page, 1);
738 goto repeat;
739 }
740 if (unlikely(!PageUptodate(page))) {
741 f2fs_put_page(page, 1);
742 return ERR_PTR(-EIO);
743 }
744 return page;
745}
746
747
748
749
750
751
752
753
754
755
756struct page *get_new_data_page(struct inode *inode,
757 struct page *ipage, pgoff_t index, bool new_i_size)
758{
759 struct address_space *mapping = inode->i_mapping;
760 struct page *page;
761 struct dnode_of_data dn;
762 int err;
763
764 page = f2fs_grab_cache_page(mapping, index, true);
765 if (!page) {
766
767
768
769
770 f2fs_put_page(ipage, 1);
771 return ERR_PTR(-ENOMEM);
772 }
773
774 set_new_dnode(&dn, inode, ipage, NULL, 0);
775 err = f2fs_reserve_block(&dn, index);
776 if (err) {
777 f2fs_put_page(page, 1);
778 return ERR_PTR(err);
779 }
780 if (!ipage)
781 f2fs_put_dnode(&dn);
782
783 if (PageUptodate(page))
784 goto got_it;
785
786 if (dn.data_blkaddr == NEW_ADDR) {
787 zero_user_segment(page, 0, PAGE_SIZE);
788 if (!PageUptodate(page))
789 SetPageUptodate(page);
790 } else {
791 f2fs_put_page(page, 1);
792
793
794 f2fs_bug_on(F2FS_I_SB(inode), ipage);
795 page = get_lock_data_page(inode, index, true);
796 if (IS_ERR(page))
797 return page;
798 }
799got_it:
800 if (new_i_size && i_size_read(inode) <
801 ((loff_t)(index + 1) << PAGE_SHIFT))
802 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
803 return page;
804}
805
806static int __allocate_data_block(struct dnode_of_data *dn, int seg_type)
807{
808 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
809 struct f2fs_summary sum;
810 struct node_info ni;
811 pgoff_t fofs;
812 blkcnt_t count = 1;
813 int err;
814
815 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
816 return -EPERM;
817
818 dn->data_blkaddr = datablock_addr(dn->inode,
819 dn->node_page, dn->ofs_in_node);
820 if (dn->data_blkaddr == NEW_ADDR)
821 goto alloc;
822
823 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
824 return err;
825
826alloc:
827 get_node_info(sbi, dn->nid, &ni);
828 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
829
830 allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
831 &sum, seg_type, NULL, false);
832 set_data_blkaddr(dn);
833
834
835 fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
836 dn->ofs_in_node;
837 if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
838 f2fs_i_size_write(dn->inode,
839 ((loff_t)(fofs + 1) << PAGE_SHIFT));
840 return 0;
841}
842
843int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
844{
845 struct inode *inode = file_inode(iocb->ki_filp);
846 struct f2fs_map_blocks map;
847 int flag;
848 int err = 0;
849 bool direct_io = iocb->ki_flags & IOCB_DIRECT;
850
851
852 if (direct_io) {
853 err = f2fs_convert_inline_inode(inode);
854 if (err)
855 return err;
856 }
857
858 if (is_inode_flag_set(inode, FI_NO_PREALLOC))
859 return 0;
860
861 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
862 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
863 if (map.m_len > map.m_lblk)
864 map.m_len -= map.m_lblk;
865 else
866 map.m_len = 0;
867
868 map.m_next_pgofs = NULL;
869 map.m_next_extent = NULL;
870 map.m_seg_type = NO_CHECK_TYPE;
871
872 if (direct_io) {
873 map.m_seg_type = rw_hint_to_seg_type(iocb->ki_hint);
874 flag = f2fs_force_buffered_io(inode, WRITE) ?
875 F2FS_GET_BLOCK_PRE_AIO :
876 F2FS_GET_BLOCK_PRE_DIO;
877 goto map_blocks;
878 }
879 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) {
880 err = f2fs_convert_inline_inode(inode);
881 if (err)
882 return err;
883 }
884 if (f2fs_has_inline_data(inode))
885 return err;
886
887 flag = F2FS_GET_BLOCK_PRE_AIO;
888
889map_blocks:
890 err = f2fs_map_blocks(inode, &map, 1, flag);
891 if (map.m_len > 0 && err == -ENOSPC) {
892 if (!direct_io)
893 set_inode_flag(inode, FI_NO_PREALLOC);
894 err = 0;
895 }
896 return err;
897}
898
899static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
900{
901 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
902 if (lock)
903 down_read(&sbi->node_change);
904 else
905 up_read(&sbi->node_change);
906 } else {
907 if (lock)
908 f2fs_lock_op(sbi);
909 else
910 f2fs_unlock_op(sbi);
911 }
912}
913
914
915
916
917
918
919
920
921
922
923int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
924 int create, int flag)
925{
926 unsigned int maxblocks = map->m_len;
927 struct dnode_of_data dn;
928 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
929 int mode = create ? ALLOC_NODE : LOOKUP_NODE;
930 pgoff_t pgofs, end_offset, end;
931 int err = 0, ofs = 1;
932 unsigned int ofs_in_node, last_ofs_in_node;
933 blkcnt_t prealloc;
934 struct extent_info ei = {0,0,0};
935 block_t blkaddr;
936 unsigned int start_pgofs;
937
938 if (!maxblocks)
939 return 0;
940
941 map->m_len = 0;
942 map->m_flags = 0;
943
944
945 pgofs = (pgoff_t)map->m_lblk;
946 end = pgofs + maxblocks;
947
948 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
949 map->m_pblk = ei.blk + pgofs - ei.fofs;
950 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
951 map->m_flags = F2FS_MAP_MAPPED;
952 if (map->m_next_extent)
953 *map->m_next_extent = pgofs + map->m_len;
954 goto out;
955 }
956
957next_dnode:
958 if (create)
959 __do_map_lock(sbi, flag, true);
960
961
962 set_new_dnode(&dn, inode, NULL, NULL, 0);
963 err = get_dnode_of_data(&dn, pgofs, mode);
964 if (err) {
965 if (flag == F2FS_GET_BLOCK_BMAP)
966 map->m_pblk = 0;
967 if (err == -ENOENT) {
968 err = 0;
969 if (map->m_next_pgofs)
970 *map->m_next_pgofs =
971 get_next_page_offset(&dn, pgofs);
972 if (map->m_next_extent)
973 *map->m_next_extent =
974 get_next_page_offset(&dn, pgofs);
975 }
976 goto unlock_out;
977 }
978
979 start_pgofs = pgofs;
980 prealloc = 0;
981 last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
982 end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
983
984next_block:
985 blkaddr = datablock_addr(dn.inode, dn.node_page, dn.ofs_in_node);
986
987 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
988 if (create) {
989 if (unlikely(f2fs_cp_error(sbi))) {
990 err = -EIO;
991 goto sync_out;
992 }
993 if (flag == F2FS_GET_BLOCK_PRE_AIO) {
994 if (blkaddr == NULL_ADDR) {
995 prealloc++;
996 last_ofs_in_node = dn.ofs_in_node;
997 }
998 } else {
999 err = __allocate_data_block(&dn,
1000 map->m_seg_type);
1001 if (!err)
1002 set_inode_flag(inode, FI_APPEND_WRITE);
1003 }
1004 if (err)
1005 goto sync_out;
1006 map->m_flags |= F2FS_MAP_NEW;
1007 blkaddr = dn.data_blkaddr;
1008 } else {
1009 if (flag == F2FS_GET_BLOCK_BMAP) {
1010 map->m_pblk = 0;
1011 goto sync_out;
1012 }
1013 if (flag == F2FS_GET_BLOCK_PRECACHE)
1014 goto sync_out;
1015 if (flag == F2FS_GET_BLOCK_FIEMAP &&
1016 blkaddr == NULL_ADDR) {
1017 if (map->m_next_pgofs)
1018 *map->m_next_pgofs = pgofs + 1;
1019 goto sync_out;
1020 }
1021 if (flag != F2FS_GET_BLOCK_FIEMAP) {
1022
1023 if (map->m_next_pgofs)
1024 *map->m_next_pgofs = pgofs + 1;
1025 goto sync_out;
1026 }
1027 }
1028 }
1029
1030 if (flag == F2FS_GET_BLOCK_PRE_AIO)
1031 goto skip;
1032
1033 if (map->m_len == 0) {
1034
1035 if (blkaddr == NEW_ADDR)
1036 map->m_flags |= F2FS_MAP_UNWRITTEN;
1037 map->m_flags |= F2FS_MAP_MAPPED;
1038
1039 map->m_pblk = blkaddr;
1040 map->m_len = 1;
1041 } else if ((map->m_pblk != NEW_ADDR &&
1042 blkaddr == (map->m_pblk + ofs)) ||
1043 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
1044 flag == F2FS_GET_BLOCK_PRE_DIO) {
1045 ofs++;
1046 map->m_len++;
1047 } else {
1048 goto sync_out;
1049 }
1050
1051skip:
1052 dn.ofs_in_node++;
1053 pgofs++;
1054
1055
1056 if (flag == F2FS_GET_BLOCK_PRE_AIO &&
1057 (pgofs == end || dn.ofs_in_node == end_offset)) {
1058
1059 dn.ofs_in_node = ofs_in_node;
1060 err = reserve_new_blocks(&dn, prealloc);
1061 if (err)
1062 goto sync_out;
1063
1064 map->m_len += dn.ofs_in_node - ofs_in_node;
1065 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
1066 err = -ENOSPC;
1067 goto sync_out;
1068 }
1069 dn.ofs_in_node = end_offset;
1070 }
1071
1072 if (pgofs >= end)
1073 goto sync_out;
1074 else if (dn.ofs_in_node < end_offset)
1075 goto next_block;
1076
1077 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1078 if (map->m_flags & F2FS_MAP_MAPPED) {
1079 unsigned int ofs = start_pgofs - map->m_lblk;
1080
1081 f2fs_update_extent_cache_range(&dn,
1082 start_pgofs, map->m_pblk + ofs,
1083 map->m_len - ofs);
1084 }
1085 }
1086
1087 f2fs_put_dnode(&dn);
1088
1089 if (create) {
1090 __do_map_lock(sbi, flag, false);
1091 f2fs_balance_fs(sbi, dn.node_changed);
1092 }
1093 goto next_dnode;
1094
1095sync_out:
1096 if (flag == F2FS_GET_BLOCK_PRECACHE) {
1097 if (map->m_flags & F2FS_MAP_MAPPED) {
1098 unsigned int ofs = start_pgofs - map->m_lblk;
1099
1100 f2fs_update_extent_cache_range(&dn,
1101 start_pgofs, map->m_pblk + ofs,
1102 map->m_len - ofs);
1103 }
1104 if (map->m_next_extent)
1105 *map->m_next_extent = pgofs + 1;
1106 }
1107 f2fs_put_dnode(&dn);
1108unlock_out:
1109 if (create) {
1110 __do_map_lock(sbi, flag, false);
1111 f2fs_balance_fs(sbi, dn.node_changed);
1112 }
1113out:
1114 trace_f2fs_map_blocks(inode, map, err);
1115 return err;
1116}
1117
1118bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len)
1119{
1120 struct f2fs_map_blocks map;
1121 block_t last_lblk;
1122 int err;
1123
1124 if (pos + len > i_size_read(inode))
1125 return false;
1126
1127 map.m_lblk = F2FS_BYTES_TO_BLK(pos);
1128 map.m_next_pgofs = NULL;
1129 map.m_next_extent = NULL;
1130 map.m_seg_type = NO_CHECK_TYPE;
1131 last_lblk = F2FS_BLK_ALIGN(pos + len);
1132
1133 while (map.m_lblk < last_lblk) {
1134 map.m_len = last_lblk - map.m_lblk;
1135 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT);
1136 if (err || map.m_len == 0)
1137 return false;
1138 map.m_lblk += map.m_len;
1139 }
1140 return true;
1141}
1142
1143static int __get_data_block(struct inode *inode, sector_t iblock,
1144 struct buffer_head *bh, int create, int flag,
1145 pgoff_t *next_pgofs, int seg_type)
1146{
1147 struct f2fs_map_blocks map;
1148 int err;
1149
1150 map.m_lblk = iblock;
1151 map.m_len = bh->b_size >> inode->i_blkbits;
1152 map.m_next_pgofs = next_pgofs;
1153 map.m_next_extent = NULL;
1154 map.m_seg_type = seg_type;
1155
1156 err = f2fs_map_blocks(inode, &map, create, flag);
1157 if (!err) {
1158 map_bh(bh, inode->i_sb, map.m_pblk);
1159 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
1160 bh->b_size = (u64)map.m_len << inode->i_blkbits;
1161 }
1162 return err;
1163}
1164
1165static int get_data_block(struct inode *inode, sector_t iblock,
1166 struct buffer_head *bh_result, int create, int flag,
1167 pgoff_t *next_pgofs)
1168{
1169 return __get_data_block(inode, iblock, bh_result, create,
1170 flag, next_pgofs,
1171 NO_CHECK_TYPE);
1172}
1173
1174static int get_data_block_dio(struct inode *inode, sector_t iblock,
1175 struct buffer_head *bh_result, int create)
1176{
1177 return __get_data_block(inode, iblock, bh_result, create,
1178 F2FS_GET_BLOCK_DEFAULT, NULL,
1179 rw_hint_to_seg_type(
1180 inode->i_write_hint));
1181}
1182
1183static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1184 struct buffer_head *bh_result, int create)
1185{
1186
1187 if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
1188 return -EFBIG;
1189
1190 return __get_data_block(inode, iblock, bh_result, create,
1191 F2FS_GET_BLOCK_BMAP, NULL,
1192 NO_CHECK_TYPE);
1193}
1194
1195static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1196{
1197 return (offset >> inode->i_blkbits);
1198}
1199
1200static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1201{
1202 return (blk << inode->i_blkbits);
1203}
1204
1205static int f2fs_xattr_fiemap(struct inode *inode,
1206 struct fiemap_extent_info *fieinfo)
1207{
1208 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1209 struct page *page;
1210 struct node_info ni;
1211 __u64 phys = 0, len;
1212 __u32 flags;
1213 nid_t xnid = F2FS_I(inode)->i_xattr_nid;
1214 int err = 0;
1215
1216 if (f2fs_has_inline_xattr(inode)) {
1217 int offset;
1218
1219 page = f2fs_grab_cache_page(NODE_MAPPING(sbi),
1220 inode->i_ino, false);
1221 if (!page)
1222 return -ENOMEM;
1223
1224 get_node_info(sbi, inode->i_ino, &ni);
1225
1226 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1227 offset = offsetof(struct f2fs_inode, i_addr) +
1228 sizeof(__le32) * (DEF_ADDRS_PER_INODE -
1229 get_inline_xattr_addrs(inode));
1230
1231 phys += offset;
1232 len = inline_xattr_size(inode);
1233
1234 f2fs_put_page(page, 1);
1235
1236 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED;
1237
1238 if (!xnid)
1239 flags |= FIEMAP_EXTENT_LAST;
1240
1241 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1242 if (err || err == 1)
1243 return err;
1244 }
1245
1246 if (xnid) {
1247 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false);
1248 if (!page)
1249 return -ENOMEM;
1250
1251 get_node_info(sbi, xnid, &ni);
1252
1253 phys = (__u64)blk_to_logical(inode, ni.blk_addr);
1254 len = inode->i_sb->s_blocksize;
1255
1256 f2fs_put_page(page, 1);
1257
1258 flags = FIEMAP_EXTENT_LAST;
1259 }
1260
1261 if (phys)
1262 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags);
1263
1264 return (err < 0 ? err : 0);
1265}
1266
1267int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1268 u64 start, u64 len)
1269{
1270 struct buffer_head map_bh;
1271 sector_t start_blk, last_blk;
1272 pgoff_t next_pgofs;
1273 u64 logical = 0, phys = 0, size = 0;
1274 u32 flags = 0;
1275 int ret = 0;
1276
1277 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) {
1278 ret = f2fs_precache_extents(inode);
1279 if (ret)
1280 return ret;
1281 }
1282
1283 ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC | FIEMAP_FLAG_XATTR);
1284 if (ret)
1285 return ret;
1286
1287 inode_lock(inode);
1288
1289 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) {
1290 ret = f2fs_xattr_fiemap(inode, fieinfo);
1291 goto out;
1292 }
1293
1294 if (f2fs_has_inline_data(inode)) {
1295 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1296 if (ret != -EAGAIN)
1297 goto out;
1298 }
1299
1300 if (logical_to_blk(inode, len) == 0)
1301 len = blk_to_logical(inode, 1);
1302
1303 start_blk = logical_to_blk(inode, start);
1304 last_blk = logical_to_blk(inode, start + len - 1);
1305
1306next:
1307 memset(&map_bh, 0, sizeof(struct buffer_head));
1308 map_bh.b_size = len;
1309
1310 ret = get_data_block(inode, start_blk, &map_bh, 0,
1311 F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
1312 if (ret)
1313 goto out;
1314
1315
1316 if (!buffer_mapped(&map_bh)) {
1317 start_blk = next_pgofs;
1318
1319 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1320 F2FS_I_SB(inode)->max_file_blocks))
1321 goto prep_next;
1322
1323 flags |= FIEMAP_EXTENT_LAST;
1324 }
1325
1326 if (size) {
1327 if (f2fs_encrypted_inode(inode))
1328 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1329
1330 ret = fiemap_fill_next_extent(fieinfo, logical,
1331 phys, size, flags);
1332 }
1333
1334 if (start_blk > last_blk || ret)
1335 goto out;
1336
1337 logical = blk_to_logical(inode, start_blk);
1338 phys = blk_to_logical(inode, map_bh.b_blocknr);
1339 size = map_bh.b_size;
1340 flags = 0;
1341 if (buffer_unwritten(&map_bh))
1342 flags = FIEMAP_EXTENT_UNWRITTEN;
1343
1344 start_blk += logical_to_blk(inode, size);
1345
1346prep_next:
1347 cond_resched();
1348 if (fatal_signal_pending(current))
1349 ret = -EINTR;
1350 else
1351 goto next;
1352out:
1353 if (ret == 1)
1354 ret = 0;
1355
1356 inode_unlock(inode);
1357 return ret;
1358}
1359
1360
1361
1362
1363
1364static int f2fs_mpage_readpages(struct address_space *mapping,
1365 struct list_head *pages, struct page *page,
1366 unsigned nr_pages)
1367{
1368 struct bio *bio = NULL;
1369 sector_t last_block_in_bio = 0;
1370 struct inode *inode = mapping->host;
1371 const unsigned blkbits = inode->i_blkbits;
1372 const unsigned blocksize = 1 << blkbits;
1373 sector_t block_in_file;
1374 sector_t last_block;
1375 sector_t last_block_in_file;
1376 sector_t block_nr;
1377 struct f2fs_map_blocks map;
1378
1379 map.m_pblk = 0;
1380 map.m_lblk = 0;
1381 map.m_len = 0;
1382 map.m_flags = 0;
1383 map.m_next_pgofs = NULL;
1384 map.m_next_extent = NULL;
1385 map.m_seg_type = NO_CHECK_TYPE;
1386
1387 for (; nr_pages; nr_pages--) {
1388 if (pages) {
1389 page = list_last_entry(pages, struct page, lru);
1390
1391 prefetchw(&page->flags);
1392 list_del(&page->lru);
1393 if (add_to_page_cache_lru(page, mapping,
1394 page->index,
1395 readahead_gfp_mask(mapping)))
1396 goto next_page;
1397 }
1398
1399 block_in_file = (sector_t)page->index;
1400 last_block = block_in_file + nr_pages;
1401 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1402 blkbits;
1403 if (last_block > last_block_in_file)
1404 last_block = last_block_in_file;
1405
1406
1407
1408
1409 if ((map.m_flags & F2FS_MAP_MAPPED) &&
1410 block_in_file > map.m_lblk &&
1411 block_in_file < (map.m_lblk + map.m_len))
1412 goto got_it;
1413
1414
1415
1416
1417
1418 map.m_flags = 0;
1419
1420 if (block_in_file < last_block) {
1421 map.m_lblk = block_in_file;
1422 map.m_len = last_block - block_in_file;
1423
1424 if (f2fs_map_blocks(inode, &map, 0,
1425 F2FS_GET_BLOCK_DEFAULT))
1426 goto set_error_page;
1427 }
1428got_it:
1429 if ((map.m_flags & F2FS_MAP_MAPPED)) {
1430 block_nr = map.m_pblk + block_in_file - map.m_lblk;
1431 SetPageMappedToDisk(page);
1432
1433 if (!PageUptodate(page) && !cleancache_get_page(page)) {
1434 SetPageUptodate(page);
1435 goto confused;
1436 }
1437 } else {
1438 zero_user_segment(page, 0, PAGE_SIZE);
1439 if (!PageUptodate(page))
1440 SetPageUptodate(page);
1441 unlock_page(page);
1442 goto next_page;
1443 }
1444
1445
1446
1447
1448
1449 if (bio && (last_block_in_bio != block_nr - 1 ||
1450 !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
1451submit_and_realloc:
1452 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1453 bio = NULL;
1454 }
1455 if (bio == NULL) {
1456 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages);
1457 if (IS_ERR(bio)) {
1458 bio = NULL;
1459 goto set_error_page;
1460 }
1461 }
1462
1463 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1464 goto submit_and_realloc;
1465
1466 last_block_in_bio = block_nr;
1467 goto next_page;
1468set_error_page:
1469 SetPageError(page);
1470 zero_user_segment(page, 0, PAGE_SIZE);
1471 unlock_page(page);
1472 goto next_page;
1473confused:
1474 if (bio) {
1475 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1476 bio = NULL;
1477 }
1478 unlock_page(page);
1479next_page:
1480 if (pages)
1481 put_page(page);
1482 }
1483 BUG_ON(pages && !list_empty(pages));
1484 if (bio)
1485 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1486 return 0;
1487}
1488
1489static int f2fs_read_data_page(struct file *file, struct page *page)
1490{
1491 struct inode *inode = page->mapping->host;
1492 int ret = -EAGAIN;
1493
1494 trace_f2fs_readpage(page, DATA);
1495
1496
1497 if (f2fs_has_inline_data(inode))
1498 ret = f2fs_read_inline_data(inode, page);
1499 if (ret == -EAGAIN)
1500 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
1501 return ret;
1502}
1503
1504static int f2fs_read_data_pages(struct file *file,
1505 struct address_space *mapping,
1506 struct list_head *pages, unsigned nr_pages)
1507{
1508 struct inode *inode = mapping->host;
1509 struct page *page = list_last_entry(pages, struct page, lru);
1510
1511 trace_f2fs_readpages(inode, page, nr_pages);
1512
1513
1514 if (f2fs_has_inline_data(inode))
1515 return 0;
1516
1517 return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
1518}
1519
1520static int encrypt_one_page(struct f2fs_io_info *fio)
1521{
1522 struct inode *inode = fio->page->mapping->host;
1523 gfp_t gfp_flags = GFP_NOFS;
1524
1525 if (!f2fs_encrypted_file(inode))
1526 return 0;
1527
1528
1529 f2fs_wait_on_block_writeback(fio->sbi, fio->old_blkaddr);
1530
1531retry_encrypt:
1532 fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1533 PAGE_SIZE, 0, fio->page->index, gfp_flags);
1534 if (!IS_ERR(fio->encrypted_page))
1535 return 0;
1536
1537
1538 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
1539 f2fs_flush_merged_writes(fio->sbi);
1540 congestion_wait(BLK_RW_ASYNC, HZ/50);
1541 gfp_flags |= __GFP_NOFAIL;
1542 goto retry_encrypt;
1543 }
1544 return PTR_ERR(fio->encrypted_page);
1545}
1546
1547static inline bool check_inplace_update_policy(struct inode *inode,
1548 struct f2fs_io_info *fio)
1549{
1550 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1551 unsigned int policy = SM_I(sbi)->ipu_policy;
1552
1553 if (policy & (0x1 << F2FS_IPU_FORCE))
1554 return true;
1555 if (policy & (0x1 << F2FS_IPU_SSR) && need_SSR(sbi))
1556 return true;
1557 if (policy & (0x1 << F2FS_IPU_UTIL) &&
1558 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1559 return true;
1560 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && need_SSR(sbi) &&
1561 utilization(sbi) > SM_I(sbi)->min_ipu_util)
1562 return true;
1563
1564
1565
1566
1567 if (policy & (0x1 << F2FS_IPU_ASYNC) &&
1568 fio && fio->op == REQ_OP_WRITE &&
1569 !(fio->op_flags & REQ_SYNC) &&
1570 !f2fs_encrypted_inode(inode))
1571 return true;
1572
1573
1574 if (policy & (0x1 << F2FS_IPU_FSYNC) &&
1575 is_inode_flag_set(inode, FI_NEED_IPU))
1576 return true;
1577
1578 return false;
1579}
1580
1581bool should_update_inplace(struct inode *inode, struct f2fs_io_info *fio)
1582{
1583 if (f2fs_is_pinned_file(inode))
1584 return true;
1585
1586
1587 if (file_is_cold(inode))
1588 return true;
1589
1590 return check_inplace_update_policy(inode, fio);
1591}
1592
1593bool should_update_outplace(struct inode *inode, struct f2fs_io_info *fio)
1594{
1595 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1596
1597 if (test_opt(sbi, LFS))
1598 return true;
1599 if (S_ISDIR(inode->i_mode))
1600 return true;
1601 if (f2fs_is_atomic_file(inode))
1602 return true;
1603 if (fio) {
1604 if (is_cold_data(fio->page))
1605 return true;
1606 if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1607 return true;
1608 }
1609 return false;
1610}
1611
1612static inline bool need_inplace_update(struct f2fs_io_info *fio)
1613{
1614 struct inode *inode = fio->page->mapping->host;
1615
1616 if (should_update_outplace(inode, fio))
1617 return false;
1618
1619 return should_update_inplace(inode, fio);
1620}
1621
1622static inline bool valid_ipu_blkaddr(struct f2fs_io_info *fio)
1623{
1624 if (fio->old_blkaddr == NEW_ADDR)
1625 return false;
1626 if (fio->old_blkaddr == NULL_ADDR)
1627 return false;
1628 return true;
1629}
1630
1631int do_write_data_page(struct f2fs_io_info *fio)
1632{
1633 struct page *page = fio->page;
1634 struct inode *inode = page->mapping->host;
1635 struct dnode_of_data dn;
1636 struct extent_info ei = {0,0,0};
1637 bool ipu_force = false;
1638 int err = 0;
1639
1640 set_new_dnode(&dn, inode, NULL, NULL, 0);
1641 if (need_inplace_update(fio) &&
1642 f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1643 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
1644
1645 if (valid_ipu_blkaddr(fio)) {
1646 ipu_force = true;
1647 fio->need_lock = LOCK_DONE;
1648 goto got_it;
1649 }
1650 }
1651
1652
1653 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1654 return -EAGAIN;
1655
1656 err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1657 if (err)
1658 goto out;
1659
1660 fio->old_blkaddr = dn.data_blkaddr;
1661
1662
1663 if (fio->old_blkaddr == NULL_ADDR) {
1664 ClearPageUptodate(page);
1665 goto out_writepage;
1666 }
1667got_it:
1668
1669
1670
1671
1672 if (ipu_force || (valid_ipu_blkaddr(fio) && need_inplace_update(fio))) {
1673 err = encrypt_one_page(fio);
1674 if (err)
1675 goto out_writepage;
1676
1677 set_page_writeback(page);
1678 f2fs_put_dnode(&dn);
1679 if (fio->need_lock == LOCK_REQ)
1680 f2fs_unlock_op(fio->sbi);
1681 err = rewrite_data_page(fio);
1682 trace_f2fs_do_write_data_page(fio->page, IPU);
1683 set_inode_flag(inode, FI_UPDATE_WRITE);
1684 return err;
1685 }
1686
1687 if (fio->need_lock == LOCK_RETRY) {
1688 if (!f2fs_trylock_op(fio->sbi)) {
1689 err = -EAGAIN;
1690 goto out_writepage;
1691 }
1692 fio->need_lock = LOCK_REQ;
1693 }
1694
1695 err = encrypt_one_page(fio);
1696 if (err)
1697 goto out_writepage;
1698
1699 set_page_writeback(page);
1700
1701
1702 write_data_page(&dn, fio);
1703 trace_f2fs_do_write_data_page(page, OPU);
1704 set_inode_flag(inode, FI_APPEND_WRITE);
1705 if (page->index == 0)
1706 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1707out_writepage:
1708 f2fs_put_dnode(&dn);
1709out:
1710 if (fio->need_lock == LOCK_REQ)
1711 f2fs_unlock_op(fio->sbi);
1712 return err;
1713}
1714
1715static int __write_data_page(struct page *page, bool *submitted,
1716 struct writeback_control *wbc,
1717 enum iostat_type io_type)
1718{
1719 struct inode *inode = page->mapping->host;
1720 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1721 loff_t i_size = i_size_read(inode);
1722 const pgoff_t end_index = ((unsigned long long) i_size)
1723 >> PAGE_SHIFT;
1724 loff_t psize = (page->index + 1) << PAGE_SHIFT;
1725 unsigned offset = 0;
1726 bool need_balance_fs = false;
1727 int err = 0;
1728 struct f2fs_io_info fio = {
1729 .sbi = sbi,
1730 .ino = inode->i_ino,
1731 .type = DATA,
1732 .op = REQ_OP_WRITE,
1733 .op_flags = wbc_to_write_flags(wbc),
1734 .old_blkaddr = NULL_ADDR,
1735 .page = page,
1736 .encrypted_page = NULL,
1737 .submitted = false,
1738 .need_lock = LOCK_RETRY,
1739 .io_type = io_type,
1740 .io_wbc = wbc,
1741 };
1742
1743 trace_f2fs_writepage(page, DATA);
1744
1745
1746 if (unlikely(f2fs_cp_error(sbi))) {
1747 mapping_set_error(page->mapping, -EIO);
1748 goto out;
1749 }
1750
1751 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1752 goto redirty_out;
1753
1754 if (page->index < end_index)
1755 goto write;
1756
1757
1758
1759
1760
1761 offset = i_size & (PAGE_SIZE - 1);
1762 if ((page->index >= end_index + 1) || !offset)
1763 goto out;
1764
1765 zero_user_segment(page, offset, PAGE_SIZE);
1766write:
1767 if (f2fs_is_drop_cache(inode))
1768 goto out;
1769
1770 if (f2fs_is_volatile_file(inode) && (!page->index ||
1771 (!wbc->for_reclaim &&
1772 available_free_memory(sbi, BASE_CHECK))))
1773 goto redirty_out;
1774
1775
1776 if (S_ISDIR(inode->i_mode)) {
1777 fio.need_lock = LOCK_DONE;
1778 err = do_write_data_page(&fio);
1779 goto done;
1780 }
1781
1782 if (!wbc->for_reclaim)
1783 need_balance_fs = true;
1784 else if (has_not_enough_free_secs(sbi, 0, 0))
1785 goto redirty_out;
1786 else
1787 set_inode_flag(inode, FI_HOT_DATA);
1788
1789 err = -EAGAIN;
1790 if (f2fs_has_inline_data(inode)) {
1791 err = f2fs_write_inline_data(inode, page);
1792 if (!err)
1793 goto out;
1794 }
1795
1796 if (err == -EAGAIN) {
1797 err = do_write_data_page(&fio);
1798 if (err == -EAGAIN) {
1799 fio.need_lock = LOCK_REQ;
1800 err = do_write_data_page(&fio);
1801 }
1802 }
1803
1804 if (err) {
1805 file_set_keep_isize(inode);
1806 } else {
1807 down_write(&F2FS_I(inode)->i_sem);
1808 if (F2FS_I(inode)->last_disk_size < psize)
1809 F2FS_I(inode)->last_disk_size = psize;
1810 up_write(&F2FS_I(inode)->i_sem);
1811 }
1812
1813done:
1814 if (err && err != -ENOENT)
1815 goto redirty_out;
1816
1817out:
1818 inode_dec_dirty_pages(inode);
1819 if (err)
1820 ClearPageUptodate(page);
1821
1822 if (wbc->for_reclaim) {
1823 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
1824 clear_inode_flag(inode, FI_HOT_DATA);
1825 remove_dirty_inode(inode);
1826 submitted = NULL;
1827 }
1828
1829 unlock_page(page);
1830 if (!S_ISDIR(inode->i_mode))
1831 f2fs_balance_fs(sbi, need_balance_fs);
1832
1833 if (unlikely(f2fs_cp_error(sbi))) {
1834 f2fs_submit_merged_write(sbi, DATA);
1835 submitted = NULL;
1836 }
1837
1838 if (submitted)
1839 *submitted = fio.submitted;
1840
1841 return 0;
1842
1843redirty_out:
1844 redirty_page_for_writepage(wbc, page);
1845 if (!err)
1846 return AOP_WRITEPAGE_ACTIVATE;
1847 unlock_page(page);
1848 return err;
1849}
1850
1851static int f2fs_write_data_page(struct page *page,
1852 struct writeback_control *wbc)
1853{
1854 return __write_data_page(page, NULL, wbc, FS_DATA_IO);
1855}
1856
1857
1858
1859
1860
1861
1862static int f2fs_write_cache_pages(struct address_space *mapping,
1863 struct writeback_control *wbc,
1864 enum iostat_type io_type)
1865{
1866 int ret = 0;
1867 int done = 0;
1868 struct pagevec pvec;
1869 int nr_pages;
1870 pgoff_t uninitialized_var(writeback_index);
1871 pgoff_t index;
1872 pgoff_t end;
1873 pgoff_t done_index;
1874 pgoff_t last_idx = ULONG_MAX;
1875 int cycled;
1876 int range_whole = 0;
1877 int tag;
1878
1879 pagevec_init(&pvec);
1880
1881 if (get_dirty_pages(mapping->host) <=
1882 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
1883 set_inode_flag(mapping->host, FI_HOT_DATA);
1884 else
1885 clear_inode_flag(mapping->host, FI_HOT_DATA);
1886
1887 if (wbc->range_cyclic) {
1888 writeback_index = mapping->writeback_index;
1889 index = writeback_index;
1890 if (index == 0)
1891 cycled = 1;
1892 else
1893 cycled = 0;
1894 end = -1;
1895 } else {
1896 index = wbc->range_start >> PAGE_SHIFT;
1897 end = wbc->range_end >> PAGE_SHIFT;
1898 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1899 range_whole = 1;
1900 cycled = 1;
1901 }
1902 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1903 tag = PAGECACHE_TAG_TOWRITE;
1904 else
1905 tag = PAGECACHE_TAG_DIRTY;
1906retry:
1907 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1908 tag_pages_for_writeback(mapping, index, end);
1909 done_index = index;
1910 while (!done && (index <= end)) {
1911 int i;
1912
1913 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
1914 tag);
1915 if (nr_pages == 0)
1916 break;
1917
1918 for (i = 0; i < nr_pages; i++) {
1919 struct page *page = pvec.pages[i];
1920 bool submitted = false;
1921
1922 done_index = page->index;
1923retry_write:
1924 lock_page(page);
1925
1926 if (unlikely(page->mapping != mapping)) {
1927continue_unlock:
1928 unlock_page(page);
1929 continue;
1930 }
1931
1932 if (!PageDirty(page)) {
1933
1934 goto continue_unlock;
1935 }
1936
1937 if (PageWriteback(page)) {
1938 if (wbc->sync_mode != WB_SYNC_NONE)
1939 f2fs_wait_on_page_writeback(page,
1940 DATA, true);
1941 else
1942 goto continue_unlock;
1943 }
1944
1945 BUG_ON(PageWriteback(page));
1946 if (!clear_page_dirty_for_io(page))
1947 goto continue_unlock;
1948
1949 ret = __write_data_page(page, &submitted, wbc, io_type);
1950 if (unlikely(ret)) {
1951
1952
1953
1954
1955 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1956 unlock_page(page);
1957 ret = 0;
1958 continue;
1959 } else if (ret == -EAGAIN) {
1960 ret = 0;
1961 if (wbc->sync_mode == WB_SYNC_ALL) {
1962 cond_resched();
1963 congestion_wait(BLK_RW_ASYNC,
1964 HZ/50);
1965 goto retry_write;
1966 }
1967 continue;
1968 }
1969 done_index = page->index + 1;
1970 done = 1;
1971 break;
1972 } else if (submitted) {
1973 last_idx = page->index;
1974 }
1975
1976
1977 if ((atomic_read(&F2FS_M_SB(mapping)->wb_sync_req) ||
1978 --wbc->nr_to_write <= 0) &&
1979 wbc->sync_mode == WB_SYNC_NONE) {
1980 done = 1;
1981 break;
1982 }
1983 }
1984 pagevec_release(&pvec);
1985 cond_resched();
1986 }
1987
1988 if (!cycled && !done) {
1989 cycled = 1;
1990 index = 0;
1991 end = writeback_index - 1;
1992 goto retry;
1993 }
1994 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1995 mapping->writeback_index = done_index;
1996
1997 if (last_idx != ULONG_MAX)
1998 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
1999 0, last_idx, DATA);
2000
2001 return ret;
2002}
2003
2004int __f2fs_write_data_pages(struct address_space *mapping,
2005 struct writeback_control *wbc,
2006 enum iostat_type io_type)
2007{
2008 struct inode *inode = mapping->host;
2009 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2010 struct blk_plug plug;
2011 int ret;
2012
2013
2014 if (!mapping->a_ops->writepage)
2015 return 0;
2016
2017
2018 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
2019 return 0;
2020
2021
2022 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
2023 goto skip_write;
2024
2025 if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
2026 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
2027 available_free_memory(sbi, DIRTY_DENTS))
2028 goto skip_write;
2029
2030
2031 if (is_inode_flag_set(inode, FI_DO_DEFRAG))
2032 goto skip_write;
2033
2034 trace_f2fs_writepages(mapping->host, wbc, DATA);
2035
2036
2037 if (wbc->sync_mode == WB_SYNC_ALL)
2038 atomic_inc(&sbi->wb_sync_req);
2039 else if (atomic_read(&sbi->wb_sync_req))
2040 goto skip_write;
2041
2042 blk_start_plug(&plug);
2043 ret = f2fs_write_cache_pages(mapping, wbc, io_type);
2044 blk_finish_plug(&plug);
2045
2046 if (wbc->sync_mode == WB_SYNC_ALL)
2047 atomic_dec(&sbi->wb_sync_req);
2048
2049
2050
2051
2052
2053 remove_dirty_inode(inode);
2054 return ret;
2055
2056skip_write:
2057 wbc->pages_skipped += get_dirty_pages(inode);
2058 trace_f2fs_writepages(mapping->host, wbc, DATA);
2059 return 0;
2060}
2061
2062static int f2fs_write_data_pages(struct address_space *mapping,
2063 struct writeback_control *wbc)
2064{
2065 struct inode *inode = mapping->host;
2066
2067 return __f2fs_write_data_pages(mapping, wbc,
2068 F2FS_I(inode)->cp_task == current ?
2069 FS_CP_DATA_IO : FS_DATA_IO);
2070}
2071
2072static void f2fs_write_failed(struct address_space *mapping, loff_t to)
2073{
2074 struct inode *inode = mapping->host;
2075 loff_t i_size = i_size_read(inode);
2076
2077 if (to > i_size) {
2078 down_write(&F2FS_I(inode)->i_mmap_sem);
2079 truncate_pagecache(inode, i_size);
2080 truncate_blocks(inode, i_size, true);
2081 up_write(&F2FS_I(inode)->i_mmap_sem);
2082 }
2083}
2084
2085static int prepare_write_begin(struct f2fs_sb_info *sbi,
2086 struct page *page, loff_t pos, unsigned len,
2087 block_t *blk_addr, bool *node_changed)
2088{
2089 struct inode *inode = page->mapping->host;
2090 pgoff_t index = page->index;
2091 struct dnode_of_data dn;
2092 struct page *ipage;
2093 bool locked = false;
2094 struct extent_info ei = {0,0,0};
2095 int err = 0;
2096
2097
2098
2099
2100
2101 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
2102 !is_inode_flag_set(inode, FI_NO_PREALLOC))
2103 return 0;
2104
2105 if (f2fs_has_inline_data(inode) ||
2106 (pos & PAGE_MASK) >= i_size_read(inode)) {
2107 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
2108 locked = true;
2109 }
2110restart:
2111
2112 ipage = get_node_page(sbi, inode->i_ino);
2113 if (IS_ERR(ipage)) {
2114 err = PTR_ERR(ipage);
2115 goto unlock_out;
2116 }
2117
2118 set_new_dnode(&dn, inode, ipage, ipage, 0);
2119
2120 if (f2fs_has_inline_data(inode)) {
2121 if (pos + len <= MAX_INLINE_DATA(inode)) {
2122 read_inline_data(page, ipage);
2123 set_inode_flag(inode, FI_DATA_EXIST);
2124 if (inode->i_nlink)
2125 set_inline_node(ipage);
2126 } else {
2127 err = f2fs_convert_inline_page(&dn, page);
2128 if (err)
2129 goto out;
2130 if (dn.data_blkaddr == NULL_ADDR)
2131 err = f2fs_get_block(&dn, index);
2132 }
2133 } else if (locked) {
2134 err = f2fs_get_block(&dn, index);
2135 } else {
2136 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
2137 dn.data_blkaddr = ei.blk + index - ei.fofs;
2138 } else {
2139
2140 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
2141 if (err || dn.data_blkaddr == NULL_ADDR) {
2142 f2fs_put_dnode(&dn);
2143 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
2144 true);
2145 locked = true;
2146 goto restart;
2147 }
2148 }
2149 }
2150
2151
2152 *blk_addr = dn.data_blkaddr;
2153 *node_changed = dn.node_changed;
2154out:
2155 f2fs_put_dnode(&dn);
2156unlock_out:
2157 if (locked)
2158 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
2159 return err;
2160}
2161
2162static int f2fs_write_begin(struct file *file, struct address_space *mapping,
2163 loff_t pos, unsigned len, unsigned flags,
2164 struct page **pagep, void **fsdata)
2165{
2166 struct inode *inode = mapping->host;
2167 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2168 struct page *page = NULL;
2169 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
2170 bool need_balance = false, drop_atomic = false;
2171 block_t blkaddr = NULL_ADDR;
2172 int err = 0;
2173
2174 trace_f2fs_write_begin(inode, pos, len, flags);
2175
2176 if (f2fs_is_atomic_file(inode) &&
2177 !available_free_memory(sbi, INMEM_PAGES)) {
2178 err = -ENOMEM;
2179 drop_atomic = true;
2180 goto fail;
2181 }
2182
2183
2184
2185
2186
2187
2188 if (index != 0) {
2189 err = f2fs_convert_inline_inode(inode);
2190 if (err)
2191 goto fail;
2192 }
2193repeat:
2194
2195
2196
2197
2198 page = f2fs_pagecache_get_page(mapping, index,
2199 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
2200 if (!page) {
2201 err = -ENOMEM;
2202 goto fail;
2203 }
2204
2205 *pagep = page;
2206
2207 err = prepare_write_begin(sbi, page, pos, len,
2208 &blkaddr, &need_balance);
2209 if (err)
2210 goto fail;
2211
2212 if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
2213 unlock_page(page);
2214 f2fs_balance_fs(sbi, true);
2215 lock_page(page);
2216 if (page->mapping != mapping) {
2217
2218 f2fs_put_page(page, 1);
2219 goto repeat;
2220 }
2221 }
2222
2223 f2fs_wait_on_page_writeback(page, DATA, false);
2224
2225
2226 if (f2fs_encrypted_file(inode))
2227 f2fs_wait_on_block_writeback(sbi, blkaddr);
2228
2229 if (len == PAGE_SIZE || PageUptodate(page))
2230 return 0;
2231
2232 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
2233 zero_user_segment(page, len, PAGE_SIZE);
2234 return 0;
2235 }
2236
2237 if (blkaddr == NEW_ADDR) {
2238 zero_user_segment(page, 0, PAGE_SIZE);
2239 SetPageUptodate(page);
2240 } else {
2241 err = f2fs_submit_page_read(inode, page, blkaddr);
2242 if (err)
2243 goto fail;
2244
2245 lock_page(page);
2246 if (unlikely(page->mapping != mapping)) {
2247 f2fs_put_page(page, 1);
2248 goto repeat;
2249 }
2250 if (unlikely(!PageUptodate(page))) {
2251 err = -EIO;
2252 goto fail;
2253 }
2254 }
2255 return 0;
2256
2257fail:
2258 f2fs_put_page(page, 1);
2259 f2fs_write_failed(mapping, pos + len);
2260 if (drop_atomic)
2261 drop_inmem_pages_all(sbi);
2262 return err;
2263}
2264
2265static int f2fs_write_end(struct file *file,
2266 struct address_space *mapping,
2267 loff_t pos, unsigned len, unsigned copied,
2268 struct page *page, void *fsdata)
2269{
2270 struct inode *inode = page->mapping->host;
2271
2272 trace_f2fs_write_end(inode, pos, len, copied);
2273
2274
2275
2276
2277
2278
2279 if (!PageUptodate(page)) {
2280 if (unlikely(copied != len))
2281 copied = 0;
2282 else
2283 SetPageUptodate(page);
2284 }
2285 if (!copied)
2286 goto unlock_out;
2287
2288 set_page_dirty(page);
2289
2290 if (pos + copied > i_size_read(inode))
2291 f2fs_i_size_write(inode, pos + copied);
2292unlock_out:
2293 f2fs_put_page(page, 1);
2294 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
2295 return copied;
2296}
2297
2298static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2299 loff_t offset)
2300{
2301 unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
2302
2303 if (offset & blocksize_mask)
2304 return -EINVAL;
2305
2306 if (iov_iter_alignment(iter) & blocksize_mask)
2307 return -EINVAL;
2308
2309 return 0;
2310}
2311
2312static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2313{
2314 struct address_space *mapping = iocb->ki_filp->f_mapping;
2315 struct inode *inode = mapping->host;
2316 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2317 size_t count = iov_iter_count(iter);
2318 loff_t offset = iocb->ki_pos;
2319 int rw = iov_iter_rw(iter);
2320 int err;
2321 enum rw_hint hint = iocb->ki_hint;
2322 int whint_mode = F2FS_OPTION(sbi).whint_mode;
2323
2324 err = check_direct_IO(inode, iter, offset);
2325 if (err)
2326 return err;
2327
2328 if (f2fs_force_buffered_io(inode, rw))
2329 return 0;
2330
2331 trace_f2fs_direct_IO_enter(inode, offset, count, rw);
2332
2333 if (rw == WRITE && whint_mode == WHINT_MODE_OFF)
2334 iocb->ki_hint = WRITE_LIFE_NOT_SET;
2335
2336 if (!down_read_trylock(&F2FS_I(inode)->dio_rwsem[rw])) {
2337 if (iocb->ki_flags & IOCB_NOWAIT) {
2338 iocb->ki_hint = hint;
2339 err = -EAGAIN;
2340 goto out;
2341 }
2342 down_read(&F2FS_I(inode)->dio_rwsem[rw]);
2343 }
2344
2345 err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
2346 up_read(&F2FS_I(inode)->dio_rwsem[rw]);
2347
2348 if (rw == WRITE) {
2349 if (whint_mode == WHINT_MODE_OFF)
2350 iocb->ki_hint = hint;
2351 if (err > 0) {
2352 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO,
2353 err);
2354 set_inode_flag(inode, FI_UPDATE_WRITE);
2355 } else if (err < 0) {
2356 f2fs_write_failed(mapping, offset + count);
2357 }
2358 }
2359
2360out:
2361 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
2362
2363 return err;
2364}
2365
2366void f2fs_invalidate_page(struct page *page, unsigned int offset,
2367 unsigned int length)
2368{
2369 struct inode *inode = page->mapping->host;
2370 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2371
2372 if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
2373 (offset % PAGE_SIZE || length != PAGE_SIZE))
2374 return;
2375
2376 if (PageDirty(page)) {
2377 if (inode->i_ino == F2FS_META_INO(sbi)) {
2378 dec_page_count(sbi, F2FS_DIRTY_META);
2379 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
2380 dec_page_count(sbi, F2FS_DIRTY_NODES);
2381 } else {
2382 inode_dec_dirty_pages(inode);
2383 remove_dirty_inode(inode);
2384 }
2385 }
2386
2387
2388 if (IS_ATOMIC_WRITTEN_PAGE(page))
2389 return drop_inmem_page(inode, page);
2390
2391 set_page_private(page, 0);
2392 ClearPagePrivate(page);
2393}
2394
2395int f2fs_release_page(struct page *page, gfp_t wait)
2396{
2397
2398 if (PageDirty(page))
2399 return 0;
2400
2401
2402 if (IS_ATOMIC_WRITTEN_PAGE(page))
2403 return 0;
2404
2405 set_page_private(page, 0);
2406 ClearPagePrivate(page);
2407 return 1;
2408}
2409
2410
2411
2412
2413
2414void f2fs_set_page_dirty_nobuffers(struct page *page)
2415{
2416 struct address_space *mapping = page->mapping;
2417 unsigned long flags;
2418
2419 if (unlikely(!mapping))
2420 return;
2421
2422 spin_lock(&mapping->private_lock);
2423 lock_page_memcg(page);
2424 SetPageDirty(page);
2425 spin_unlock(&mapping->private_lock);
2426
2427 xa_lock_irqsave(&mapping->i_pages, flags);
2428 WARN_ON_ONCE(!PageUptodate(page));
2429 account_page_dirtied(page, mapping);
2430 radix_tree_tag_set(&mapping->i_pages,
2431 page_index(page), PAGECACHE_TAG_DIRTY);
2432 xa_unlock_irqrestore(&mapping->i_pages, flags);
2433 unlock_page_memcg(page);
2434
2435 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2436 return;
2437}
2438
2439static int f2fs_set_data_page_dirty(struct page *page)
2440{
2441 struct address_space *mapping = page->mapping;
2442 struct inode *inode = mapping->host;
2443
2444 trace_f2fs_set_page_dirty(page, DATA);
2445
2446 if (!PageUptodate(page))
2447 SetPageUptodate(page);
2448
2449 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
2450 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
2451 register_inmem_page(inode, page);
2452 return 1;
2453 }
2454
2455
2456
2457
2458 return 0;
2459 }
2460
2461 if (!PageDirty(page)) {
2462 f2fs_set_page_dirty_nobuffers(page);
2463 update_dirty_page(inode, page);
2464 return 1;
2465 }
2466 return 0;
2467}
2468
2469static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2470{
2471 struct inode *inode = mapping->host;
2472
2473 if (f2fs_has_inline_data(inode))
2474 return 0;
2475
2476
2477 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2478 filemap_write_and_wait(mapping);
2479
2480 return generic_block_bmap(mapping, block, get_data_block_bmap);
2481}
2482
2483#ifdef CONFIG_MIGRATION
2484#include <linux/migrate.h>
2485
2486int f2fs_migrate_page(struct address_space *mapping,
2487 struct page *newpage, struct page *page, enum migrate_mode mode)
2488{
2489 int rc, extra_count;
2490 struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2491 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2492
2493 BUG_ON(PageWriteback(page));
2494
2495
2496 if (atomic_written) {
2497 if (mode != MIGRATE_SYNC)
2498 return -EBUSY;
2499 if (!mutex_trylock(&fi->inmem_lock))
2500 return -EAGAIN;
2501 }
2502
2503
2504
2505
2506
2507
2508 extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
2509 rc = migrate_page_move_mapping(mapping, newpage,
2510 page, NULL, mode, extra_count);
2511 if (rc != MIGRATEPAGE_SUCCESS) {
2512 if (atomic_written)
2513 mutex_unlock(&fi->inmem_lock);
2514 return rc;
2515 }
2516
2517 if (atomic_written) {
2518 struct inmem_pages *cur;
2519 list_for_each_entry(cur, &fi->inmem_pages, list)
2520 if (cur->page == page) {
2521 cur->page = newpage;
2522 break;
2523 }
2524 mutex_unlock(&fi->inmem_lock);
2525 put_page(page);
2526 get_page(newpage);
2527 }
2528
2529 if (PagePrivate(page))
2530 SetPagePrivate(newpage);
2531 set_page_private(newpage, page_private(page));
2532
2533 if (mode != MIGRATE_SYNC_NO_COPY)
2534 migrate_page_copy(newpage, page);
2535 else
2536 migrate_page_states(newpage, page);
2537
2538 return MIGRATEPAGE_SUCCESS;
2539}
2540#endif
2541
2542const struct address_space_operations f2fs_dblock_aops = {
2543 .readpage = f2fs_read_data_page,
2544 .readpages = f2fs_read_data_pages,
2545 .writepage = f2fs_write_data_page,
2546 .writepages = f2fs_write_data_pages,
2547 .write_begin = f2fs_write_begin,
2548 .write_end = f2fs_write_end,
2549 .set_page_dirty = f2fs_set_data_page_dirty,
2550 .invalidatepage = f2fs_invalidate_page,
2551 .releasepage = f2fs_release_page,
2552 .direct_IO = f2fs_direct_IO,
2553 .bmap = f2fs_bmap,
2554#ifdef CONFIG_MIGRATION
2555 .migratepage = f2fs_migrate_page,
2556#endif
2557};
2558
