1
2
3
4
5
6
7
8
9
10
11
12#include <linux/export.h>
13#include <linux/compiler.h>
14#include <linux/dax.h>
15#include <linux/fs.h>
16#include <linux/sched/signal.h>
17#include <linux/uaccess.h>
18#include <linux/capability.h>
19#include <linux/kernel_stat.h>
20#include <linux/gfp.h>
21#include <linux/mm.h>
22#include <linux/swap.h>
23#include <linux/mman.h>
24#include <linux/pagemap.h>
25#include <linux/file.h>
26#include <linux/uio.h>
27#include <linux/hash.h>
28#include <linux/writeback.h>
29#include <linux/backing-dev.h>
30#include <linux/pagevec.h>
31#include <linux/blkdev.h>
32#include <linux/security.h>
33#include <linux/cpuset.h>
34#include <linux/hugetlb.h>
35#include <linux/memcontrol.h>
36#include <linux/cleancache.h>
37#include <linux/shmem_fs.h>
38#include <linux/rmap.h>
39#include "internal.h"
40
41#define CREATE_TRACE_POINTS
42#include <trace/events/filemap.h>
43
44
45
46
47#include <linux/buffer_head.h>
48
49#include <asm/mman.h>
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114static int page_cache_tree_insert(struct address_space *mapping,
115 struct page *page, void **shadowp)
116{
117 struct radix_tree_node *node;
118 void **slot;
119 int error;
120
121 error = __radix_tree_create(&mapping->page_tree, page->index, 0,
122 &node, &slot);
123 if (error)
124 return error;
125 if (*slot) {
126 void *p;
127
128 p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
129 if (!radix_tree_exceptional_entry(p))
130 return -EEXIST;
131
132 mapping->nrexceptional--;
133 if (shadowp)
134 *shadowp = p;
135 }
136 __radix_tree_replace(&mapping->page_tree, node, slot, page,
137 workingset_lookup_update(mapping));
138 mapping->nrpages++;
139 return 0;
140}
141
142static void page_cache_tree_delete(struct address_space *mapping,
143 struct page *page, void *shadow)
144{
145 int i, nr;
146
147
148 nr = PageHuge(page) ? 1 : hpage_nr_pages(page);
149
150 VM_BUG_ON_PAGE(!PageLocked(page), page);
151 VM_BUG_ON_PAGE(PageTail(page), page);
152 VM_BUG_ON_PAGE(nr != 1 && shadow, page);
153
154 for (i = 0; i < nr; i++) {
155 struct radix_tree_node *node;
156 void **slot;
157
158 __radix_tree_lookup(&mapping->page_tree, page->index + i,
159 &node, &slot);
160
161 VM_BUG_ON_PAGE(!node && nr != 1, page);
162
163 radix_tree_clear_tags(&mapping->page_tree, node, slot);
164 __radix_tree_replace(&mapping->page_tree, node, slot, shadow,
165 workingset_lookup_update(mapping));
166 }
167
168 page->mapping = NULL;
169
170
171 if (shadow) {
172 mapping->nrexceptional += nr;
173
174
175
176
177
178
179 smp_wmb();
180 }
181 mapping->nrpages -= nr;
182}
183
184static void unaccount_page_cache_page(struct address_space *mapping,
185 struct page *page)
186{
187 int nr;
188
189
190
191
192
193
194 if (PageUptodate(page) && PageMappedToDisk(page))
195 cleancache_put_page(page);
196 else
197 cleancache_invalidate_page(mapping, page);
198
199 VM_BUG_ON_PAGE(PageTail(page), page);
200 VM_BUG_ON_PAGE(page_mapped(page), page);
201 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
202 int mapcount;
203
204 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
205 current->comm, page_to_pfn(page));
206 dump_page(page, "still mapped when deleted");
207 dump_stack();
208 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
209
210 mapcount = page_mapcount(page);
211 if (mapping_exiting(mapping) &&
212 page_count(page) >= mapcount + 2) {
213
214
215
216
217
218
219 page_mapcount_reset(page);
220 page_ref_sub(page, mapcount);
221 }
222 }
223
224
225 if (PageHuge(page))
226 return;
227
228 nr = hpage_nr_pages(page);
229
230 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
231 if (PageSwapBacked(page)) {
232 __mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);
233 if (PageTransHuge(page))
234 __dec_node_page_state(page, NR_SHMEM_THPS);
235 } else {
236 VM_BUG_ON_PAGE(PageTransHuge(page), page);
237 }
238
239
240
241
242
243
244
245
246
247
248
249 if (WARN_ON_ONCE(PageDirty(page)))
250 account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
251}
252
253
254
255
256
257
258void __delete_from_page_cache(struct page *page, void *shadow)
259{
260 struct address_space *mapping = page->mapping;
261
262 trace_mm_filemap_delete_from_page_cache(page);
263
264 unaccount_page_cache_page(mapping, page);
265 page_cache_tree_delete(mapping, page, shadow);
266}
267
268static void page_cache_free_page(struct address_space *mapping,
269 struct page *page)
270{
271 void (*freepage)(struct page *);
272
273 freepage = mapping->a_ops->freepage;
274 if (freepage)
275 freepage(page);
276
277 if (PageTransHuge(page) && !PageHuge(page)) {
278 page_ref_sub(page, HPAGE_PMD_NR);
279 VM_BUG_ON_PAGE(page_count(page) <= 0, page);
280 } else {
281 put_page(page);
282 }
283}
284
285
286
287
288
289
290
291
292
293void delete_from_page_cache(struct page *page)
294{
295 struct address_space *mapping = page_mapping(page);
296 unsigned long flags;
297
298 BUG_ON(!PageLocked(page));
299 spin_lock_irqsave(&mapping->tree_lock, flags);
300 __delete_from_page_cache(page, NULL);
301 spin_unlock_irqrestore(&mapping->tree_lock, flags);
302
303 page_cache_free_page(mapping, page);
304}
305EXPORT_SYMBOL(delete_from_page_cache);
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321static void
322page_cache_tree_delete_batch(struct address_space *mapping,
323 struct pagevec *pvec)
324{
325 struct radix_tree_iter iter;
326 void **slot;
327 int total_pages = 0;
328 int i = 0, tail_pages = 0;
329 struct page *page;
330 pgoff_t start;
331
332 start = pvec->pages[0]->index;
333 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
334 if (i >= pagevec_count(pvec) && !tail_pages)
335 break;
336 page = radix_tree_deref_slot_protected(slot,
337 &mapping->tree_lock);
338 if (radix_tree_exceptional_entry(page))
339 continue;
340 if (!tail_pages) {
341
342
343
344
345
346 if (page != pvec->pages[i])
347 continue;
348 WARN_ON_ONCE(!PageLocked(page));
349 if (PageTransHuge(page) && !PageHuge(page))
350 tail_pages = HPAGE_PMD_NR - 1;
351 page->mapping = NULL;
352
353
354
355
356 i++;
357 } else {
358 tail_pages--;
359 }
360 radix_tree_clear_tags(&mapping->page_tree, iter.node, slot);
361 __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL,
362 workingset_lookup_update(mapping));
363 total_pages++;
364 }
365 mapping->nrpages -= total_pages;
366}
367
368void delete_from_page_cache_batch(struct address_space *mapping,
369 struct pagevec *pvec)
370{
371 int i;
372 unsigned long flags;
373
374 if (!pagevec_count(pvec))
375 return;
376
377 spin_lock_irqsave(&mapping->tree_lock, flags);
378 for (i = 0; i < pagevec_count(pvec); i++) {
379 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]);
380
381 unaccount_page_cache_page(mapping, pvec->pages[i]);
382 }
383 page_cache_tree_delete_batch(mapping, pvec);
384 spin_unlock_irqrestore(&mapping->tree_lock, flags);
385
386 for (i = 0; i < pagevec_count(pvec); i++)
387 page_cache_free_page(mapping, pvec->pages[i]);
388}
389
390int filemap_check_errors(struct address_space *mapping)
391{
392 int ret = 0;
393
394 if (test_bit(AS_ENOSPC, &mapping->flags) &&
395 test_and_clear_bit(AS_ENOSPC, &mapping->flags))
396 ret = -ENOSPC;
397 if (test_bit(AS_EIO, &mapping->flags) &&
398 test_and_clear_bit(AS_EIO, &mapping->flags))
399 ret = -EIO;
400 return ret;
401}
402EXPORT_SYMBOL(filemap_check_errors);
403
404static int filemap_check_and_keep_errors(struct address_space *mapping)
405{
406
407 if (test_bit(AS_EIO, &mapping->flags))
408 return -EIO;
409 if (test_bit(AS_ENOSPC, &mapping->flags))
410 return -ENOSPC;
411 return 0;
412}
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
430 loff_t end, int sync_mode)
431{
432 int ret;
433 struct writeback_control wbc = {
434 .sync_mode = sync_mode,
435 .nr_to_write = LONG_MAX,
436 .range_start = start,
437 .range_end = end,
438 };
439
440 if (!mapping_cap_writeback_dirty(mapping))
441 return 0;
442
443 wbc_attach_fdatawrite_inode(&wbc, mapping->host);
444 ret = do_writepages(mapping, &wbc);
445 wbc_detach_inode(&wbc);
446 return ret;
447}
448
449static inline int __filemap_fdatawrite(struct address_space *mapping,
450 int sync_mode)
451{
452 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
453}
454
455int filemap_fdatawrite(struct address_space *mapping)
456{
457 return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
458}
459EXPORT_SYMBOL(filemap_fdatawrite);
460
461int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
462 loff_t end)
463{
464 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
465}
466EXPORT_SYMBOL(filemap_fdatawrite_range);
467
468
469
470
471
472
473
474
475int filemap_flush(struct address_space *mapping)
476{
477 return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
478}
479EXPORT_SYMBOL(filemap_flush);
480
481
482
483
484
485
486
487
488
489
490bool filemap_range_has_page(struct address_space *mapping,
491 loff_t start_byte, loff_t end_byte)
492{
493 pgoff_t index = start_byte >> PAGE_SHIFT;
494 pgoff_t end = end_byte >> PAGE_SHIFT;
495 struct page *page;
496
497 if (end_byte < start_byte)
498 return false;
499
500 if (mapping->nrpages == 0)
501 return false;
502
503 if (!find_get_pages_range(mapping, &index, end, 1, &page))
504 return false;
505 put_page(page);
506 return true;
507}
508EXPORT_SYMBOL(filemap_range_has_page);
509
510static void __filemap_fdatawait_range(struct address_space *mapping,
511 loff_t start_byte, loff_t end_byte)
512{
513 pgoff_t index = start_byte >> PAGE_SHIFT;
514 pgoff_t end = end_byte >> PAGE_SHIFT;
515 struct pagevec pvec;
516 int nr_pages;
517
518 if (end_byte < start_byte)
519 return;
520
521 pagevec_init(&pvec);
522 while (index <= end) {
523 unsigned i;
524
525 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index,
526 end, PAGECACHE_TAG_WRITEBACK);
527 if (!nr_pages)
528 break;
529
530 for (i = 0; i < nr_pages; i++) {
531 struct page *page = pvec.pages[i];
532
533 wait_on_page_writeback(page);
534 ClearPageError(page);
535 }
536 pagevec_release(&pvec);
537 cond_resched();
538 }
539}
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
556 loff_t end_byte)
557{
558 __filemap_fdatawait_range(mapping, start_byte, end_byte);
559 return filemap_check_errors(mapping);
560}
561EXPORT_SYMBOL(filemap_fdatawait_range);
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte)
578{
579 struct address_space *mapping = file->f_mapping;
580
581 __filemap_fdatawait_range(mapping, start_byte, end_byte);
582 return file_check_and_advance_wb_err(file);
583}
584EXPORT_SYMBOL(file_fdatawait_range);
585
586
587
588
589
590
591
592
593
594
595
596
597
598int filemap_fdatawait_keep_errors(struct address_space *mapping)
599{
600 __filemap_fdatawait_range(mapping, 0, LLONG_MAX);
601 return filemap_check_and_keep_errors(mapping);
602}
603EXPORT_SYMBOL(filemap_fdatawait_keep_errors);
604
605static bool mapping_needs_writeback(struct address_space *mapping)
606{
607 return (!dax_mapping(mapping) && mapping->nrpages) ||
608 (dax_mapping(mapping) && mapping->nrexceptional);
609}
610
611int filemap_write_and_wait(struct address_space *mapping)
612{
613 int err = 0;
614
615 if (mapping_needs_writeback(mapping)) {
616 err = filemap_fdatawrite(mapping);
617
618
619
620
621
622
623 if (err != -EIO) {
624 int err2 = filemap_fdatawait(mapping);
625 if (!err)
626 err = err2;
627 } else {
628
629 filemap_check_errors(mapping);
630 }
631 } else {
632 err = filemap_check_errors(mapping);
633 }
634 return err;
635}
636EXPORT_SYMBOL(filemap_write_and_wait);
637
638
639
640
641
642
643
644
645
646
647
648
649int filemap_write_and_wait_range(struct address_space *mapping,
650 loff_t lstart, loff_t lend)
651{
652 int err = 0;
653
654 if (mapping_needs_writeback(mapping)) {
655 err = __filemap_fdatawrite_range(mapping, lstart, lend,
656 WB_SYNC_ALL);
657
658 if (err != -EIO) {
659 int err2 = filemap_fdatawait_range(mapping,
660 lstart, lend);
661 if (!err)
662 err = err2;
663 } else {
664
665 filemap_check_errors(mapping);
666 }
667 } else {
668 err = filemap_check_errors(mapping);
669 }
670 return err;
671}
672EXPORT_SYMBOL(filemap_write_and_wait_range);
673
674void __filemap_set_wb_err(struct address_space *mapping, int err)
675{
676 errseq_t eseq = errseq_set(&mapping->wb_err, err);
677
678 trace_filemap_set_wb_err(mapping, eseq);
679}
680EXPORT_SYMBOL(__filemap_set_wb_err);
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704int file_check_and_advance_wb_err(struct file *file)
705{
706 int err = 0;
707 errseq_t old = READ_ONCE(file->f_wb_err);
708 struct address_space *mapping = file->f_mapping;
709
710
711 if (errseq_check(&mapping->wb_err, old)) {
712
713 spin_lock(&file->f_lock);
714 old = file->f_wb_err;
715 err = errseq_check_and_advance(&mapping->wb_err,
716 &file->f_wb_err);
717 trace_file_check_and_advance_wb_err(file, old);
718 spin_unlock(&file->f_lock);
719 }
720
721
722
723
724
725
726 clear_bit(AS_EIO, &mapping->flags);
727 clear_bit(AS_ENOSPC, &mapping->flags);
728 return err;
729}
730EXPORT_SYMBOL(file_check_and_advance_wb_err);
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend)
747{
748 int err = 0, err2;
749 struct address_space *mapping = file->f_mapping;
750
751 if (mapping_needs_writeback(mapping)) {
752 err = __filemap_fdatawrite_range(mapping, lstart, lend,
753 WB_SYNC_ALL);
754
755 if (err != -EIO)
756 __filemap_fdatawait_range(mapping, lstart, lend);
757 }
758 err2 = file_check_and_advance_wb_err(file);
759 if (!err)
760 err = err2;
761 return err;
762}
763EXPORT_SYMBOL(file_write_and_wait_range);
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
781{
782 int error;
783
784 VM_BUG_ON_PAGE(!PageLocked(old), old);
785 VM_BUG_ON_PAGE(!PageLocked(new), new);
786 VM_BUG_ON_PAGE(new->mapping, new);
787
788 error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
789 if (!error) {
790 struct address_space *mapping = old->mapping;
791 void (*freepage)(struct page *);
792 unsigned long flags;
793
794 pgoff_t offset = old->index;
795 freepage = mapping->a_ops->freepage;
796
797 get_page(new);
798 new->mapping = mapping;
799 new->index = offset;
800
801 spin_lock_irqsave(&mapping->tree_lock, flags);
802 __delete_from_page_cache(old, NULL);
803 error = page_cache_tree_insert(mapping, new, NULL);
804 BUG_ON(error);
805
806
807
808
809 if (!PageHuge(new))
810 __inc_node_page_state(new, NR_FILE_PAGES);
811 if (PageSwapBacked(new))
812 __inc_node_page_state(new, NR_SHMEM);
813 spin_unlock_irqrestore(&mapping->tree_lock, flags);
814 mem_cgroup_migrate(old, new);
815 radix_tree_preload_end();
816 if (freepage)
817 freepage(old);
818 put_page(old);
819 }
820
821 return error;
822}
823EXPORT_SYMBOL_GPL(replace_page_cache_page);
824
825static int __add_to_page_cache_locked(struct page *page,
826 struct address_space *mapping,
827 pgoff_t offset, gfp_t gfp_mask,
828 void **shadowp)
829{
830 int huge = PageHuge(page);
831 struct mem_cgroup *memcg;
832 int error;
833
834 VM_BUG_ON_PAGE(!PageLocked(page), page);
835 VM_BUG_ON_PAGE(PageSwapBacked(page), page);
836
837 if (!huge) {
838 error = mem_cgroup_try_charge(page, current->mm,
839 gfp_mask, &memcg, false);
840 if (error)
841 return error;
842 }
843
844 error = radix_tree_maybe_preload(gfp_mask & ~__GFP_HIGHMEM);
845 if (error) {
846 if (!huge)
847 mem_cgroup_cancel_charge(page, memcg, false);
848 return error;
849 }
850
851 get_page(page);
852 page->mapping = mapping;
853 page->index = offset;
854
855 spin_lock_irq(&mapping->tree_lock);
856 error = page_cache_tree_insert(mapping, page, shadowp);
857 radix_tree_preload_end();
858 if (unlikely(error))
859 goto err_insert;
860
861
862 if (!huge)
863 __inc_node_page_state(page, NR_FILE_PAGES);
864 spin_unlock_irq(&mapping->tree_lock);
865 if (!huge)
866 mem_cgroup_commit_charge(page, memcg, false, false);
867 trace_mm_filemap_add_to_page_cache(page);
868 return 0;
869err_insert:
870 page->mapping = NULL;
871
872 spin_unlock_irq(&mapping->tree_lock);
873 if (!huge)
874 mem_cgroup_cancel_charge(page, memcg, false);
875 put_page(page);
876 return error;
877}
878
879
880
881
882
883
884
885
886
887
888
889int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
890 pgoff_t offset, gfp_t gfp_mask)
891{
892 return __add_to_page_cache_locked(page, mapping, offset,
893 gfp_mask, NULL);
894}
895EXPORT_SYMBOL(add_to_page_cache_locked);
896
897int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
898 pgoff_t offset, gfp_t gfp_mask)
899{
900 void *shadow = NULL;
901 int ret;
902
903 __SetPageLocked(page);
904 ret = __add_to_page_cache_locked(page, mapping, offset,
905 gfp_mask, &shadow);
906 if (unlikely(ret))
907 __ClearPageLocked(page);
908 else {
909
910
911
912
913
914
915
916
917 if (!(gfp_mask & __GFP_WRITE) &&
918 shadow && workingset_refault(shadow)) {
919 SetPageActive(page);
920 workingset_activation(page);
921 } else
922 ClearPageActive(page);
923 lru_cache_add(page);
924 }
925 return ret;
926}
927EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
928
929#ifdef CONFIG_NUMA
930struct page *__page_cache_alloc(gfp_t gfp)
931{
932 int n;
933 struct page *page;
934
935 if (cpuset_do_page_mem_spread()) {
936 unsigned int cpuset_mems_cookie;
937 do {
938 cpuset_mems_cookie = read_mems_allowed_begin();
939 n = cpuset_mem_spread_node();
940 page = __alloc_pages_node(n, gfp, 0);
941 } while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
942
943 return page;
944 }
945 return alloc_pages(gfp, 0);
946}
947EXPORT_SYMBOL(__page_cache_alloc);
948#endif
949
950
951
952
953
954
955
956
957
958
959
960#define PAGE_WAIT_TABLE_BITS 8
961#define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
962static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
963
964static wait_queue_head_t *page_waitqueue(struct page *page)
965{
966 return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)];
967}
968
969void __init pagecache_init(void)
970{
971 int i;
972
973 for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
974 init_waitqueue_head(&page_wait_table[i]);
975
976 page_writeback_init();
977}
978
979
980struct wait_page_key {
981 struct page *page;
982 int bit_nr;
983 int page_match;
984};
985
986struct wait_page_queue {
987 struct page *page;
988 int bit_nr;
989 wait_queue_entry_t wait;
990};
991
992static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg)
993{
994 struct wait_page_key *key = arg;
995 struct wait_page_queue *wait_page
996 = container_of(wait, struct wait_page_queue, wait);
997
998 if (wait_page->page != key->page)
999 return 0;
1000 key->page_match = 1;
1001
1002 if (wait_page->bit_nr != key->bit_nr)
1003 return 0;
1004
1005
1006 if (test_bit(key->bit_nr, &key->page->flags))
1007 return -1;
1008
1009 return autoremove_wake_function(wait, mode, sync, key);
1010}
1011
1012static void wake_up_page_bit(struct page *page, int bit_nr)
1013{
1014 wait_queue_head_t *q = page_waitqueue(page);
1015 struct wait_page_key key;
1016 unsigned long flags;
1017 wait_queue_entry_t bookmark;
1018
1019 key.page = page;
1020 key.bit_nr = bit_nr;
1021 key.page_match = 0;
1022
1023 bookmark.flags = 0;
1024 bookmark.private = NULL;
1025 bookmark.func = NULL;
1026 INIT_LIST_HEAD(&bookmark.entry);
1027
1028 spin_lock_irqsave(&q->lock, flags);
1029 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
1030
1031 while (bookmark.flags & WQ_FLAG_BOOKMARK) {
1032
1033
1034
1035
1036
1037
1038 spin_unlock_irqrestore(&q->lock, flags);
1039 cpu_relax();
1040 spin_lock_irqsave(&q->lock, flags);
1041 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
1042 }
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053 if (!waitqueue_active(q) || !key.page_match) {
1054 ClearPageWaiters(page);
1055
1056
1057
1058
1059
1060
1061
1062 }
1063 spin_unlock_irqrestore(&q->lock, flags);
1064}
1065
1066static void wake_up_page(struct page *page, int bit)
1067{
1068 if (!PageWaiters(page))
1069 return;
1070 wake_up_page_bit(page, bit);
1071}
1072
1073static inline int wait_on_page_bit_common(wait_queue_head_t *q,
1074 struct page *page, int bit_nr, int state, bool lock)
1075{
1076 struct wait_page_queue wait_page;
1077 wait_queue_entry_t *wait = &wait_page.wait;
1078 int ret = 0;
1079
1080 init_wait(wait);
1081 wait->flags = lock ? WQ_FLAG_EXCLUSIVE : 0;
1082 wait->func = wake_page_function;
1083 wait_page.page = page;
1084 wait_page.bit_nr = bit_nr;
1085
1086 for (;;) {
1087 spin_lock_irq(&q->lock);
1088
1089 if (likely(list_empty(&wait->entry))) {
1090 __add_wait_queue_entry_tail(q, wait);
1091 SetPageWaiters(page);
1092 }
1093
1094 set_current_state(state);
1095
1096 spin_unlock_irq(&q->lock);
1097
1098 if (likely(test_bit(bit_nr, &page->flags))) {
1099 io_schedule();
1100 }
1101
1102 if (lock) {
1103 if (!test_and_set_bit_lock(bit_nr, &page->flags))
1104 break;
1105 } else {
1106 if (!test_bit(bit_nr, &page->flags))
1107 break;
1108 }
1109
1110 if (unlikely(signal_pending_state(state, current))) {
1111 ret = -EINTR;
1112 break;
1113 }
1114 }
1115
1116 finish_wait(q, wait);
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126 return ret;
1127}
1128
1129void wait_on_page_bit(struct page *page, int bit_nr)
1130{
1131 wait_queue_head_t *q = page_waitqueue(page);
1132 wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, false);
1133}
1134EXPORT_SYMBOL(wait_on_page_bit);
1135
1136int wait_on_page_bit_killable(struct page *page, int bit_nr)
1137{
1138 wait_queue_head_t *q = page_waitqueue(page);
1139 return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, false);
1140}
1141EXPORT_SYMBOL(wait_on_page_bit_killable);
1142
1143
1144
1145
1146
1147
1148
1149
1150void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter)
1151{
1152 wait_queue_head_t *q = page_waitqueue(page);
1153 unsigned long flags;
1154
1155 spin_lock_irqsave(&q->lock, flags);
1156 __add_wait_queue_entry_tail(q, waiter);
1157 SetPageWaiters(page);
1158 spin_unlock_irqrestore(&q->lock, flags);
1159}
1160EXPORT_SYMBOL_GPL(add_page_wait_queue);
1161
1162#ifndef clear_bit_unlock_is_negative_byte
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem)
1177{
1178 clear_bit_unlock(nr, mem);
1179
1180 return test_bit(PG_waiters, mem);
1181}
1182
1183#endif
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200void unlock_page(struct page *page)
1201{
1202 BUILD_BUG_ON(PG_waiters != 7);
1203 page = compound_head(page);
1204 VM_BUG_ON_PAGE(!PageLocked(page), page);
1205 if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags))
1206 wake_up_page_bit(page, PG_locked);
1207}
1208EXPORT_SYMBOL(unlock_page);
1209
1210
1211
1212
1213
1214void end_page_writeback(struct page *page)
1215{
1216
1217
1218
1219
1220
1221
1222
1223 if (PageReclaim(page)) {
1224 ClearPageReclaim(page);
1225 rotate_reclaimable_page(page);
1226 }
1227
1228 if (!test_clear_page_writeback(page))
1229 BUG();
1230
1231 smp_mb__after_atomic();
1232 wake_up_page(page, PG_writeback);
1233}
1234EXPORT_SYMBOL(end_page_writeback);
1235
1236
1237
1238
1239
1240void page_endio(struct page *page, bool is_write, int err)
1241{
1242 if (!is_write) {
1243 if (!err) {
1244 SetPageUptodate(page);
1245 } else {
1246 ClearPageUptodate(page);
1247 SetPageError(page);
1248 }
1249 unlock_page(page);
1250 } else {
1251 if (err) {
1252 struct address_space *mapping;
1253
1254 SetPageError(page);
1255 mapping = page_mapping(page);
1256 if (mapping)
1257 mapping_set_error(mapping, err);
1258 }
1259 end_page_writeback(page);
1260 }
1261}
1262EXPORT_SYMBOL_GPL(page_endio);
1263
1264
1265
1266
1267
1268void __lock_page(struct page *__page)
1269{
1270 struct page *page = compound_head(__page);
1271 wait_queue_head_t *q = page_waitqueue(page);
1272 wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, true);
1273}
1274EXPORT_SYMBOL(__lock_page);
1275
1276int __lock_page_killable(struct page *__page)
1277{
1278 struct page *page = compound_head(__page);
1279 wait_queue_head_t *q = page_waitqueue(page);
1280 return wait_on_page_bit_common(q, page, PG_locked, TASK_KILLABLE, true);
1281}
1282EXPORT_SYMBOL_GPL(__lock_page_killable);
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
1296 unsigned int flags)
1297{
1298 if (flags & FAULT_FLAG_ALLOW_RETRY) {
1299
1300
1301
1302
1303 if (flags & FAULT_FLAG_RETRY_NOWAIT)
1304 return 0;
1305
1306 up_read(&mm->mmap_sem);
1307 if (flags & FAULT_FLAG_KILLABLE)
1308 wait_on_page_locked_killable(page);
1309 else
1310 wait_on_page_locked(page);
1311 return 0;
1312 } else {
1313 if (flags & FAULT_FLAG_KILLABLE) {
1314 int ret;
1315
1316 ret = __lock_page_killable(page);
1317 if (ret) {
1318 up_read(&mm->mmap_sem);
1319 return 0;
1320 }
1321 } else
1322 __lock_page(page);
1323 return 1;
1324 }
1325}
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348pgoff_t page_cache_next_hole(struct address_space *mapping,
1349 pgoff_t index, unsigned long max_scan)
1350{
1351 unsigned long i;
1352
1353 for (i = 0; i < max_scan; i++) {
1354 struct page *page;
1355
1356 page = radix_tree_lookup(&mapping->page_tree, index);
1357 if (!page || radix_tree_exceptional_entry(page))
1358 break;
1359 index++;
1360 if (index == 0)
1361 break;
1362 }
1363
1364 return index;
1365}
1366EXPORT_SYMBOL(page_cache_next_hole);
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389pgoff_t page_cache_prev_hole(struct address_space *mapping,
1390 pgoff_t index, unsigned long max_scan)
1391{
1392 unsigned long i;
1393
1394 for (i = 0; i < max_scan; i++) {
1395 struct page *page;
1396
1397 page = radix_tree_lookup(&mapping->page_tree, index);
1398 if (!page || radix_tree_exceptional_entry(page))
1399 break;
1400 index--;
1401 if (index == ULONG_MAX)
1402 break;
1403 }
1404
1405 return index;
1406}
1407EXPORT_SYMBOL(page_cache_prev_hole);
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
1423{
1424 void **pagep;
1425 struct page *head, *page;
1426
1427 rcu_read_lock();
1428repeat:
1429 page = NULL;
1430 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
1431 if (pagep) {
1432 page = radix_tree_deref_slot(pagep);
1433 if (unlikely(!page))
1434 goto out;
1435 if (radix_tree_exception(page)) {
1436 if (radix_tree_deref_retry(page))
1437 goto repeat;
1438
1439
1440
1441
1442
1443 goto out;
1444 }
1445
1446 head = compound_head(page);
1447 if (!page_cache_get_speculative(head))
1448 goto repeat;
1449
1450
1451 if (compound_head(page) != head) {
1452 put_page(head);
1453 goto repeat;
1454 }
1455
1456
1457
1458
1459
1460
1461 if (unlikely(page != *pagep)) {
1462 put_page(head);
1463 goto repeat;
1464 }
1465 }
1466out:
1467 rcu_read_unlock();
1468
1469 return page;
1470}
1471EXPORT_SYMBOL(find_get_entry);
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
1490{
1491 struct page *page;
1492
1493repeat:
1494 page = find_get_entry(mapping, offset);
1495 if (page && !radix_tree_exception(page)) {
1496 lock_page(page);
1497
1498 if (unlikely(page_mapping(page) != mapping)) {
1499 unlock_page(page);
1500 put_page(page);
1501 goto repeat;
1502 }
1503 VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
1504 }
1505 return page;
1506}
1507EXPORT_SYMBOL(find_lock_entry);
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
1535 int fgp_flags, gfp_t gfp_mask)
1536{
1537 struct page *page;
1538
1539repeat:
1540 page = find_get_entry(mapping, offset);
1541 if (radix_tree_exceptional_entry(page))
1542 page = NULL;
1543 if (!page)
1544 goto no_page;
1545
1546 if (fgp_flags & FGP_LOCK) {
1547 if (fgp_flags & FGP_NOWAIT) {
1548 if (!trylock_page(page)) {
1549 put_page(page);
1550 return NULL;
1551 }
1552 } else {
1553 lock_page(page);
1554 }
1555
1556
1557 if (unlikely(page->mapping != mapping)) {
1558 unlock_page(page);
1559 put_page(page);
1560 goto repeat;
1561 }
1562 VM_BUG_ON_PAGE(page->index != offset, page);
1563 }
1564
1565 if (page && (fgp_flags & FGP_ACCESSED))
1566 mark_page_accessed(page);
1567
1568no_page:
1569 if (!page && (fgp_flags & FGP_CREAT)) {
1570 int err;
1571 if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
1572 gfp_mask |= __GFP_WRITE;
1573 if (fgp_flags & FGP_NOFS)
1574 gfp_mask &= ~__GFP_FS;
1575
1576 page = __page_cache_alloc(gfp_mask);
1577 if (!page)
1578 return NULL;
1579
1580 if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK)))
1581 fgp_flags |= FGP_LOCK;
1582
1583
1584 if (fgp_flags & FGP_ACCESSED)
1585 __SetPageReferenced(page);
1586
1587 err = add_to_page_cache_lru(page, mapping, offset,
1588 gfp_mask & GFP_RECLAIM_MASK);
1589 if (unlikely(err)) {
1590 put_page(page);
1591 page = NULL;
1592 if (err == -EEXIST)
1593 goto repeat;
1594 }
1595 }
1596
1597 return page;
1598}
1599EXPORT_SYMBOL(pagecache_get_page);
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624unsigned find_get_entries(struct address_space *mapping,
1625 pgoff_t start, unsigned int nr_entries,
1626 struct page **entries, pgoff_t *indices)
1627{
1628 void **slot;
1629 unsigned int ret = 0;
1630 struct radix_tree_iter iter;
1631
1632 if (!nr_entries)
1633 return 0;
1634
1635 rcu_read_lock();
1636 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
1637 struct page *head, *page;
1638repeat:
1639 page = radix_tree_deref_slot(slot);
1640 if (unlikely(!page))
1641 continue;
1642 if (radix_tree_exception(page)) {
1643 if (radix_tree_deref_retry(page)) {
1644 slot = radix_tree_iter_retry(&iter);
1645 continue;
1646 }
1647
1648
1649
1650
1651
1652 goto export;
1653 }
1654
1655 head = compound_head(page);
1656 if (!page_cache_get_speculative(head))
1657 goto repeat;
1658
1659
1660 if (compound_head(page) != head) {
1661 put_page(head);
1662 goto repeat;
1663 }
1664
1665
1666 if (unlikely(page != *slot)) {
1667 put_page(head);
1668 goto repeat;
1669 }
1670export:
1671 indices[ret] = iter.index;
1672 entries[ret] = page;
1673 if (++ret == nr_entries)
1674 break;
1675 }
1676 rcu_read_unlock();
1677 return ret;
1678}
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
1702 pgoff_t end, unsigned int nr_pages,
1703 struct page **pages)
1704{
1705 struct radix_tree_iter iter;
1706 void **slot;
1707 unsigned ret = 0;
1708
1709 if (unlikely(!nr_pages))
1710 return 0;
1711
1712 rcu_read_lock();
1713 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, *start) {
1714 struct page *head, *page;
1715
1716 if (iter.index > end)
1717 break;
1718repeat:
1719 page = radix_tree_deref_slot(slot);
1720 if (unlikely(!page))
1721 continue;
1722
1723 if (radix_tree_exception(page)) {
1724 if (radix_tree_deref_retry(page)) {
1725 slot = radix_tree_iter_retry(&iter);
1726 continue;
1727 }
1728
1729
1730
1731
1732
1733 continue;
1734 }
1735
1736 head = compound_head(page);
1737 if (!page_cache_get_speculative(head))
1738 goto repeat;
1739
1740
1741 if (compound_head(page) != head) {
1742 put_page(head);
1743 goto repeat;
1744 }
1745
1746
1747 if (unlikely(page != *slot)) {
1748 put_page(head);
1749 goto repeat;
1750 }
1751
1752 pages[ret] = page;
1753 if (++ret == nr_pages) {
1754 *start = pages[ret - 1]->index + 1;
1755 goto out;
1756 }
1757 }
1758
1759
1760
1761
1762
1763
1764
1765 if (end == (pgoff_t)-1)
1766 *start = (pgoff_t)-1;
1767 else
1768 *start = end + 1;
1769out:
1770 rcu_read_unlock();
1771
1772 return ret;
1773}
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
1788 unsigned int nr_pages, struct page **pages)
1789{
1790 struct radix_tree_iter iter;
1791 void **slot;
1792 unsigned int ret = 0;
1793
1794 if (unlikely(!nr_pages))
1795 return 0;
1796
1797 rcu_read_lock();
1798 radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
1799 struct page *head, *page;
1800repeat:
1801 page = radix_tree_deref_slot(slot);
1802
1803 if (unlikely(!page))
1804 break;
1805
1806 if (radix_tree_exception(page)) {
1807 if (radix_tree_deref_retry(page)) {
1808 slot = radix_tree_iter_retry(&iter);
1809 continue;
1810 }
1811
1812
1813
1814
1815
1816 break;
1817 }
1818
1819 head = compound_head(page);
1820 if (!page_cache_get_speculative(head))
1821 goto repeat;
1822
1823
1824 if (compound_head(page) != head) {
1825 put_page(head);
1826 goto repeat;
1827 }
1828
1829
1830 if (unlikely(page != *slot)) {
1831 put_page(head);
1832 goto repeat;
1833 }
1834
1835
1836
1837
1838
1839
1840 if (page->mapping == NULL || page_to_pgoff(page) != iter.index) {
1841 put_page(page);
1842 break;
1843 }
1844
1845 pages[ret] = page;
1846 if (++ret == nr_pages)
1847 break;
1848 }
1849 rcu_read_unlock();
1850 return ret;
1851}
1852EXPORT_SYMBOL(find_get_pages_contig);
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
1867 pgoff_t end, int tag, unsigned int nr_pages,
1868 struct page **pages)
1869{
1870 struct radix_tree_iter iter;
1871 void **slot;
1872 unsigned ret = 0;
1873
1874 if (unlikely(!nr_pages))
1875 return 0;
1876
1877 rcu_read_lock();
1878 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1879 &iter, *index, tag) {
1880 struct page *head, *page;
1881
1882 if (iter.index > end)
1883 break;
1884repeat:
1885 page = radix_tree_deref_slot(slot);
1886 if (unlikely(!page))
1887 continue;
1888
1889 if (radix_tree_exception(page)) {
1890 if (radix_tree_deref_retry(page)) {
1891 slot = radix_tree_iter_retry(&iter);
1892 continue;
1893 }
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905 continue;
1906 }
1907
1908 head = compound_head(page);
1909 if (!page_cache_get_speculative(head))
1910 goto repeat;
1911
1912
1913 if (compound_head(page) != head) {
1914 put_page(head);
1915 goto repeat;
1916 }
1917
1918
1919 if (unlikely(page != *slot)) {
1920 put_page(head);
1921 goto repeat;
1922 }
1923
1924 pages[ret] = page;
1925 if (++ret == nr_pages) {
1926 *index = pages[ret - 1]->index + 1;
1927 goto out;
1928 }
1929 }
1930
1931
1932
1933
1934
1935
1936
1937 if (end == (pgoff_t)-1)
1938 *index = (pgoff_t)-1;
1939 else
1940 *index = end + 1;
1941out:
1942 rcu_read_unlock();
1943
1944 return ret;
1945}
1946EXPORT_SYMBOL(find_get_pages_range_tag);
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
1961 int tag, unsigned int nr_entries,
1962 struct page **entries, pgoff_t *indices)
1963{
1964 void **slot;
1965 unsigned int ret = 0;
1966 struct radix_tree_iter iter;
1967
1968 if (!nr_entries)
1969 return 0;
1970
1971 rcu_read_lock();
1972 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1973 &iter, start, tag) {
1974 struct page *head, *page;
1975repeat:
1976 page = radix_tree_deref_slot(slot);
1977 if (unlikely(!page))
1978 continue;
1979 if (radix_tree_exception(page)) {
1980 if (radix_tree_deref_retry(page)) {
1981 slot = radix_tree_iter_retry(&iter);
1982 continue;
1983 }
1984
1985
1986
1987
1988
1989
1990 goto export;
1991 }
1992
1993 head = compound_head(page);
1994 if (!page_cache_get_speculative(head))
1995 goto repeat;
1996
1997
1998 if (compound_head(page) != head) {
1999 put_page(head);
2000 goto repeat;
2001 }
2002
2003
2004 if (unlikely(page != *slot)) {
2005 put_page(head);
2006 goto repeat;
2007 }
2008export:
2009 indices[ret] = iter.index;
2010 entries[ret] = page;
2011 if (++ret == nr_entries)
2012 break;
2013 }
2014 rcu_read_unlock();
2015 return ret;
2016}
2017EXPORT_SYMBOL(find_get_entries_tag);
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034static void shrink_readahead_size_eio(struct file *filp,
2035 struct file_ra_state *ra)
2036{
2037 ra->ra_pages /= 4;
2038}
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052static ssize_t generic_file_buffered_read(struct kiocb *iocb,
2053 struct iov_iter *iter, ssize_t written)
2054{
2055 struct file *filp = iocb->ki_filp;
2056 struct address_space *mapping = filp->f_mapping;
2057 struct inode *inode = mapping->host;
2058 struct file_ra_state *ra = &filp->f_ra;
2059 loff_t *ppos = &iocb->ki_pos;
2060 pgoff_t index;
2061 pgoff_t last_index;
2062 pgoff_t prev_index;
2063 unsigned long offset;
2064 unsigned int prev_offset;
2065 int error = 0;
2066
2067 if (unlikely(*ppos >= inode->i_sb->s_maxbytes))
2068 return 0;
2069 iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
2070
2071 index = *ppos >> PAGE_SHIFT;
2072 prev_index = ra->prev_pos >> PAGE_SHIFT;
2073 prev_offset = ra->prev_pos & (PAGE_SIZE-1);
2074 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT;
2075 offset = *ppos & ~PAGE_MASK;
2076
2077 for (;;) {
2078 struct page *page;
2079 pgoff_t end_index;
2080 loff_t isize;
2081 unsigned long nr, ret;
2082
2083 cond_resched();
2084find_page:
2085 if (fatal_signal_pending(current)) {
2086 error = -EINTR;
2087 goto out;
2088 }
2089
2090 page = find_get_page(mapping, index);
2091 if (!page) {
2092 if (iocb->ki_flags & IOCB_NOWAIT)
2093 goto would_block;
2094 page_cache_sync_readahead(mapping,
2095 ra, filp,
2096 index, last_index - index);
2097 page = find_get_page(mapping, index);
2098 if (unlikely(page == NULL))
2099 goto no_cached_page;
2100 }
2101 if (PageReadahead(page)) {
2102 page_cache_async_readahead(mapping,
2103 ra, filp, page,
2104 index, last_index - index);
2105 }
2106 if (!PageUptodate(page)) {
2107 if (iocb->ki_flags & IOCB_NOWAIT) {
2108 put_page(page);
2109 goto would_block;
2110 }
2111
2112
2113
2114
2115
2116
2117 error = wait_on_page_locked_killable(page);
2118 if (unlikely(error))
2119 goto readpage_error;
2120 if (PageUptodate(page))
2121 goto page_ok;
2122
2123 if (inode->i_blkbits == PAGE_SHIFT ||
2124 !mapping->a_ops->is_partially_uptodate)
2125 goto page_not_up_to_date;
2126
2127 if (unlikely(iter->type & ITER_PIPE))
2128 goto page_not_up_to_date;
2129 if (!trylock_page(page))
2130 goto page_not_up_to_date;
2131
2132 if (!page->mapping)
2133 goto page_not_up_to_date_locked;
2134 if (!mapping->a_ops->is_partially_uptodate(page,
2135 offset, iter->count))
2136 goto page_not_up_to_date_locked;
2137 unlock_page(page);
2138 }
2139page_ok:
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149 isize = i_size_read(inode);
2150 end_index = (isize - 1) >> PAGE_SHIFT;
2151 if (unlikely(!isize || index > end_index)) {
2152 put_page(page);
2153 goto out;
2154 }
2155
2156
2157 nr = PAGE_SIZE;
2158 if (index == end_index) {
2159 nr = ((isize - 1) & ~PAGE_MASK) + 1;
2160 if (nr <= offset) {
2161 put_page(page);
2162 goto out;
2163 }
2164 }
2165 nr = nr - offset;
2166
2167
2168
2169
2170
2171 if (mapping_writably_mapped(mapping))
2172 flush_dcache_page(page);
2173
2174
2175
2176
2177
2178 if (prev_index != index || offset != prev_offset)
2179 mark_page_accessed(page);
2180 prev_index = index;
2181
2182
2183
2184
2185
2186
2187 ret = copy_page_to_iter(page, offset, nr, iter);
2188 offset += ret;
2189 index += offset >> PAGE_SHIFT;
2190 offset &= ~PAGE_MASK;
2191 prev_offset = offset;
2192
2193 put_page(page);
2194 written += ret;
2195 if (!iov_iter_count(iter))
2196 goto out;
2197 if (ret < nr) {
2198 error = -EFAULT;
2199 goto out;
2200 }
2201 continue;
2202
2203page_not_up_to_date:
2204
2205 error = lock_page_killable(page);
2206 if (unlikely(error))
2207 goto readpage_error;
2208
2209page_not_up_to_date_locked:
2210
2211 if (!page->mapping) {
2212 unlock_page(page);
2213 put_page(page);
2214 continue;
2215 }
2216
2217
2218 if (PageUptodate(page)) {
2219 unlock_page(page);
2220 goto page_ok;
2221 }
2222
2223readpage:
2224
2225
2226
2227
2228
2229 ClearPageError(page);
2230
2231 error = mapping->a_ops->readpage(filp, page);
2232
2233 if (unlikely(error)) {
2234 if (error == AOP_TRUNCATED_PAGE) {
2235 put_page(page);
2236 error = 0;
2237 goto find_page;
2238 }
2239 goto readpage_error;
2240 }
2241
2242 if (!PageUptodate(page)) {
2243 error = lock_page_killable(page);
2244 if (unlikely(error))
2245 goto readpage_error;
2246 if (!PageUptodate(page)) {
2247 if (page->mapping == NULL) {
2248
2249
2250
2251 unlock_page(page);
2252 put_page(page);
2253 goto find_page;
2254 }
2255 unlock_page(page);
2256 shrink_readahead_size_eio(filp, ra);
2257 error = -EIO;
2258 goto readpage_error;
2259 }
2260 unlock_page(page);
2261 }
2262
2263 goto page_ok;
2264
2265readpage_error:
2266
2267 put_page(page);
2268 goto out;
2269
2270no_cached_page:
2271
2272
2273
2274
2275 page = page_cache_alloc(mapping);
2276 if (!page) {
2277 error = -ENOMEM;
2278 goto out;
2279 }
2280 error = add_to_page_cache_lru(page, mapping, index,
2281 mapping_gfp_constraint(mapping, GFP_KERNEL));
2282 if (error) {
2283 put_page(page);
2284 if (error == -EEXIST) {
2285 error = 0;
2286 goto find_page;
2287 }
2288 goto out;
2289 }
2290 goto readpage;
2291 }
2292
2293would_block:
2294 error = -EAGAIN;
2295out:
2296 ra->prev_pos = prev_index;
2297 ra->prev_pos <<= PAGE_SHIFT;
2298 ra->prev_pos |= prev_offset;
2299
2300 *ppos = ((loff_t)index << PAGE_SHIFT) + offset;
2301 file_accessed(filp);
2302 return written ? written : error;
2303}
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313ssize_t
2314generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
2315{
2316 size_t count = iov_iter_count(iter);
2317 ssize_t retval = 0;
2318
2319 if (!count)
2320 goto out;
2321
2322 if (iocb->ki_flags & IOCB_DIRECT) {
2323 struct file *file = iocb->ki_filp;
2324 struct address_space *mapping = file->f_mapping;
2325 struct inode *inode = mapping->host;
2326 loff_t size;
2327
2328 size = i_size_read(inode);
2329 if (iocb->ki_flags & IOCB_NOWAIT) {
2330 if (filemap_range_has_page(mapping, iocb->ki_pos,
2331 iocb->ki_pos + count - 1))
2332 return -EAGAIN;
2333 } else {
2334 retval = filemap_write_and_wait_range(mapping,
2335 iocb->ki_pos,
2336 iocb->ki_pos + count - 1);
2337 if (retval < 0)
2338 goto out;
2339 }
2340
2341 file_accessed(file);
2342
2343 retval = mapping->a_ops->direct_IO(iocb, iter);
2344 if (retval >= 0) {
2345 iocb->ki_pos += retval;
2346 count -= retval;
2347 }
2348 iov_iter_revert(iter, count - iov_iter_count(iter));
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359 if (retval < 0 || !count || iocb->ki_pos >= size ||
2360 IS_DAX(inode))
2361 goto out;
2362 }
2363
2364 retval = generic_file_buffered_read(iocb, iter, retval);
2365out:
2366 return retval;
2367}
2368EXPORT_SYMBOL(generic_file_read_iter);
2369
2370#ifdef CONFIG_MMU
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380static int page_cache_read(struct file *file, pgoff_t offset, gfp_t gfp_mask)
2381{
2382 struct address_space *mapping = file->f_mapping;
2383 struct page *page;
2384 int ret;
2385
2386 do {
2387 page = __page_cache_alloc(gfp_mask);
2388 if (!page)
2389 return -ENOMEM;
2390
2391 ret = add_to_page_cache_lru(page, mapping, offset, gfp_mask & GFP_KERNEL);
2392 if (ret == 0)
2393 ret = mapping->a_ops->readpage(file, page);
2394 else if (ret == -EEXIST)
2395 ret = 0;
2396
2397 put_page(page);
2398
2399 } while (ret == AOP_TRUNCATED_PAGE);
2400
2401 return ret;
2402}
2403
2404#define MMAP_LOTSAMISS (100)
2405
2406
2407
2408
2409
2410static void do_sync_mmap_readahead(struct vm_area_struct *vma,
2411 struct file_ra_state *ra,
2412 struct file *file,
2413 pgoff_t offset)
2414{
2415 struct address_space *mapping = file->f_mapping;
2416
2417
2418 if (vma->vm_flags & VM_RAND_READ)
2419 return;
2420 if (!ra->ra_pages)
2421 return;
2422
2423 if (vma->vm_flags & VM_SEQ_READ) {
2424 page_cache_sync_readahead(mapping, ra, file, offset,
2425 ra->ra_pages);
2426 return;
2427 }
2428
2429
2430 if (ra->mmap_miss < MMAP_LOTSAMISS * 10)
2431 ra->mmap_miss++;
2432
2433
2434
2435
2436
2437 if (ra->mmap_miss > MMAP_LOTSAMISS)
2438 return;
2439
2440
2441
2442
2443 ra->start = max_t(long, 0, offset - ra->ra_pages / 2);
2444 ra->size = ra->ra_pages;
2445 ra->async_size = ra->ra_pages / 4;
2446 ra_submit(ra, mapping, file);
2447}
2448
2449
2450
2451
2452
2453static void do_async_mmap_readahead(struct vm_area_struct *vma,
2454 struct file_ra_state *ra,
2455 struct file *file,
2456 struct page *page,
2457 pgoff_t offset)
2458{
2459 struct address_space *mapping = file->f_mapping;
2460
2461
2462 if (vma->vm_flags & VM_RAND_READ)
2463 return;
2464 if (ra->mmap_miss > 0)
2465 ra->mmap_miss--;
2466 if (PageReadahead(page))
2467 page_cache_async_readahead(mapping, ra, file,
2468 page, offset, ra->ra_pages);
2469}
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494int filemap_fault(struct vm_fault *vmf)
2495{
2496 int error;
2497 struct file *file = vmf->vma->vm_file;
2498 struct address_space *mapping = file->f_mapping;
2499 struct file_ra_state *ra = &file->f_ra;
2500 struct inode *inode = mapping->host;
2501 pgoff_t offset = vmf->pgoff;
2502 pgoff_t max_off;
2503 struct page *page;
2504 int ret = 0;
2505
2506 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2507 if (unlikely(offset >= max_off))
2508 return VM_FAULT_SIGBUS;
2509
2510
2511
2512
2513 page = find_get_page(mapping, offset);
2514 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
2515
2516
2517
2518
2519 do_async_mmap_readahead(vmf->vma, ra, file, page, offset);
2520 } else if (!page) {
2521
2522 do_sync_mmap_readahead(vmf->vma, ra, file, offset);
2523 count_vm_event(PGMAJFAULT);
2524 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
2525 ret = VM_FAULT_MAJOR;
2526retry_find:
2527 page = find_get_page(mapping, offset);
2528 if (!page)
2529 goto no_cached_page;
2530 }
2531
2532 if (!lock_page_or_retry(page, vmf->vma->vm_mm, vmf->flags)) {
2533 put_page(page);
2534 return ret | VM_FAULT_RETRY;
2535 }
2536
2537
2538 if (unlikely(page->mapping != mapping)) {
2539 unlock_page(page);
2540 put_page(page);
2541 goto retry_find;
2542 }
2543 VM_BUG_ON_PAGE(page->index != offset, page);
2544
2545
2546
2547
2548
2549 if (unlikely(!PageUptodate(page)))
2550 goto page_not_uptodate;
2551
2552
2553
2554
2555
2556 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2557 if (unlikely(offset >= max_off)) {
2558 unlock_page(page);
2559 put_page(page);
2560 return VM_FAULT_SIGBUS;
2561 }
2562
2563 vmf->page = page;
2564 return ret | VM_FAULT_LOCKED;
2565
2566no_cached_page:
2567
2568
2569
2570
2571 error = page_cache_read(file, offset, vmf->gfp_mask);
2572
2573
2574
2575
2576
2577
2578 if (error >= 0)
2579 goto retry_find;
2580
2581
2582
2583
2584
2585
2586 if (error == -ENOMEM)
2587 return VM_FAULT_OOM;
2588 return VM_FAULT_SIGBUS;
2589
2590page_not_uptodate:
2591
2592
2593
2594
2595
2596
2597 ClearPageError(page);
2598 error = mapping->a_ops->readpage(file, page);
2599 if (!error) {
2600 wait_on_page_locked(page);
2601 if (!PageUptodate(page))
2602 error = -EIO;
2603 }
2604 put_page(page);
2605
2606 if (!error || error == AOP_TRUNCATED_PAGE)
2607 goto retry_find;
2608
2609
2610 shrink_readahead_size_eio(file, ra);
2611 return VM_FAULT_SIGBUS;
2612}
2613EXPORT_SYMBOL(filemap_fault);
2614
2615void filemap_map_pages(struct vm_fault *vmf,
2616 pgoff_t start_pgoff, pgoff_t end_pgoff)
2617{
2618 struct radix_tree_iter iter;
2619 void **slot;
2620 struct file *file = vmf->vma->vm_file;
2621 struct address_space *mapping = file->f_mapping;
2622 pgoff_t last_pgoff = start_pgoff;
2623 unsigned long max_idx;
2624 struct page *head, *page;
2625
2626 rcu_read_lock();
2627 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter,
2628 start_pgoff) {
2629 if (iter.index > end_pgoff)
2630 break;
2631repeat:
2632 page = radix_tree_deref_slot(slot);
2633 if (unlikely(!page))
2634 goto next;
2635 if (radix_tree_exception(page)) {
2636 if (radix_tree_deref_retry(page)) {
2637 slot = radix_tree_iter_retry(&iter);
2638 continue;
2639 }
2640 goto next;
2641 }
2642
2643 head = compound_head(page);
2644 if (!page_cache_get_speculative(head))
2645 goto repeat;
2646
2647
2648 if (compound_head(page) != head) {
2649 put_page(head);
2650 goto repeat;
2651 }
2652
2653
2654 if (unlikely(page != *slot)) {
2655 put_page(head);
2656 goto repeat;
2657 }
2658
2659 if (!PageUptodate(page) ||
2660 PageReadahead(page) ||
2661 PageHWPoison(page))
2662 goto skip;
2663 if (!trylock_page(page))
2664 goto skip;
2665
2666 if (page->mapping != mapping || !PageUptodate(page))
2667 goto unlock;
2668
2669 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
2670 if (page->index >= max_idx)
2671 goto unlock;
2672
2673 if (file->f_ra.mmap_miss > 0)
2674 file->f_ra.mmap_miss--;
2675
2676 vmf->address += (iter.index - last_pgoff) << PAGE_SHIFT;
2677 if (vmf->pte)
2678 vmf->pte += iter.index - last_pgoff;
2679 last_pgoff = iter.index;
2680 if (alloc_set_pte(vmf, NULL, page))
2681 goto unlock;
2682 unlock_page(page);
2683 goto next;
2684unlock:
2685 unlock_page(page);
2686skip:
2687 put_page(page);
2688next:
2689
2690 if (pmd_trans_huge(*vmf->pmd))
2691 break;
2692 if (iter.index == end_pgoff)
2693 break;
2694 }
2695 rcu_read_unlock();
2696}
2697EXPORT_SYMBOL(filemap_map_pages);
2698
2699int filemap_page_mkwrite(struct vm_fault *vmf)
2700{
2701 struct page *page = vmf->page;
2702 struct inode *inode = file_inode(vmf->vma->vm_file);
2703 int ret = VM_FAULT_LOCKED;
2704
2705 sb_start_pagefault(inode->i_sb);
2706 file_update_time(vmf->vma->vm_file);
2707 lock_page(page);
2708 if (page->mapping != inode->i_mapping) {
2709 unlock_page(page);
2710 ret = VM_FAULT_NOPAGE;
2711 goto out;
2712 }
2713
2714
2715
2716
2717
2718 set_page_dirty(page);
2719 wait_for_stable_page(page);
2720out:
2721 sb_end_pagefault(inode->i_sb);
2722 return ret;
2723}
2724EXPORT_SYMBOL(filemap_page_mkwrite);
2725
2726const struct vm_operations_struct generic_file_vm_ops = {
2727 .fault = filemap_fault,
2728 .map_pages = filemap_map_pages,
2729 .page_mkwrite = filemap_page_mkwrite,
2730};
2731
2732
2733
2734int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2735{
2736 struct address_space *mapping = file->f_mapping;
2737
2738 if (!mapping->a_ops->readpage)
2739 return -ENOEXEC;
2740 file_accessed(file);
2741 vma->vm_ops = &generic_file_vm_ops;
2742 return 0;
2743}
2744
2745
2746
2747
2748int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
2749{
2750 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2751 return -EINVAL;
2752 return generic_file_mmap(file, vma);
2753}
2754#else
2755int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2756{
2757 return -ENOSYS;
2758}
2759int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
2760{
2761 return -ENOSYS;
2762}
2763#endif
2764
2765EXPORT_SYMBOL(generic_file_mmap);
2766EXPORT_SYMBOL(generic_file_readonly_mmap);
2767
2768static struct page *wait_on_page_read(struct page *page)
2769{
2770 if (!IS_ERR(page)) {
2771 wait_on_page_locked(page);
2772 if (!PageUptodate(page)) {
2773 put_page(page);
2774 page = ERR_PTR(-EIO);
2775 }
2776 }
2777 return page;
2778}
2779
2780static struct page *do_read_cache_page(struct address_space *mapping,
2781 pgoff_t index,
2782 int (*filler)(void *, struct page *),
2783 void *data,
2784 gfp_t gfp)
2785{
2786 struct page *page;
2787 int err;
2788repeat:
2789 page = find_get_page(mapping, index);
2790 if (!page) {
2791 page = __page_cache_alloc(gfp);
2792 if (!page)
2793 return ERR_PTR(-ENOMEM);
2794 err = add_to_page_cache_lru(page, mapping, index, gfp);
2795 if (unlikely(err)) {
2796 put_page(page);
2797 if (err == -EEXIST)
2798 goto repeat;
2799
2800 return ERR_PTR(err);
2801 }
2802
2803filler:
2804 err = filler(data, page);
2805 if (err < 0) {
2806 put_page(page);
2807 return ERR_PTR(err);
2808 }
2809
2810 page = wait_on_page_read(page);
2811 if (IS_ERR(page))
2812 return page;
2813 goto out;
2814 }
2815 if (PageUptodate(page))
2816 goto out;
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849 wait_on_page_locked(page);
2850 if (PageUptodate(page))
2851 goto out;
2852
2853
2854 lock_page(page);
2855
2856
2857 if (!page->mapping) {
2858 unlock_page(page);
2859 put_page(page);
2860 goto repeat;
2861 }
2862
2863
2864 if (PageUptodate(page)) {
2865 unlock_page(page);
2866 goto out;
2867 }
2868 goto filler;
2869
2870out:
2871 mark_page_accessed(page);
2872 return page;
2873}
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887struct page *read_cache_page(struct address_space *mapping,
2888 pgoff_t index,
2889 int (*filler)(void *, struct page *),
2890 void *data)
2891{
2892 return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
2893}
2894EXPORT_SYMBOL(read_cache_page);
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907struct page *read_cache_page_gfp(struct address_space *mapping,
2908 pgoff_t index,
2909 gfp_t gfp)
2910{
2911 filler_t *filler = (filler_t *)mapping->a_ops->readpage;
2912
2913 return do_read_cache_page(mapping, index, filler, NULL, gfp);
2914}
2915EXPORT_SYMBOL(read_cache_page_gfp);
2916
2917
2918
2919
2920
2921
2922
2923
2924inline ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
2925{
2926 struct file *file = iocb->ki_filp;
2927 struct inode *inode = file->f_mapping->host;
2928 unsigned long limit = rlimit(RLIMIT_FSIZE);
2929 loff_t pos;
2930
2931 if (!iov_iter_count(from))
2932 return 0;
2933
2934
2935 if (iocb->ki_flags & IOCB_APPEND)
2936 iocb->ki_pos = i_size_read(inode);
2937
2938 pos = iocb->ki_pos;
2939
2940 if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT))
2941 return -EINVAL;
2942
2943 if (limit != RLIM_INFINITY) {
2944 if (iocb->ki_pos >= limit) {
2945 send_sig(SIGXFSZ, current, 0);
2946 return -EFBIG;
2947 }
2948 iov_iter_truncate(from, limit - (unsigned long)pos);
2949 }
2950
2951
2952
2953
2954 if (unlikely(pos + iov_iter_count(from) > MAX_NON_LFS &&
2955 !(file->f_flags & O_LARGEFILE))) {
2956 if (pos >= MAX_NON_LFS)
2957 return -EFBIG;
2958 iov_iter_truncate(from, MAX_NON_LFS - (unsigned long)pos);
2959 }
2960
2961
2962
2963
2964
2965
2966
2967
2968 if (unlikely(pos >= inode->i_sb->s_maxbytes))
2969 return -EFBIG;
2970
2971 iov_iter_truncate(from, inode->i_sb->s_maxbytes - pos);
2972 return iov_iter_count(from);
2973}
2974EXPORT_SYMBOL(generic_write_checks);
2975
2976int pagecache_write_begin(struct file *file, struct address_space *mapping,
2977 loff_t pos, unsigned len, unsigned flags,
2978 struct page **pagep, void **fsdata)
2979{
2980 const struct address_space_operations *aops = mapping->a_ops;
2981
2982 return aops->write_begin(file, mapping, pos, len, flags,
2983 pagep, fsdata);
2984}
2985EXPORT_SYMBOL(pagecache_write_begin);
2986
2987int pagecache_write_end(struct file *file, struct address_space *mapping,
2988 loff_t pos, unsigned len, unsigned copied,
2989 struct page *page, void *fsdata)
2990{
2991 const struct address_space_operations *aops = mapping->a_ops;
2992
2993 return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
2994}
2995EXPORT_SYMBOL(pagecache_write_end);
2996
2997ssize_t
2998generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
2999{
3000 struct file *file = iocb->ki_filp;
3001 struct address_space *mapping = file->f_mapping;
3002 struct inode *inode = mapping->host;
3003 loff_t pos = iocb->ki_pos;
3004 ssize_t written;
3005 size_t write_len;
3006 pgoff_t end;
3007
3008 write_len = iov_iter_count(from);
3009 end = (pos + write_len - 1) >> PAGE_SHIFT;
3010
3011 if (iocb->ki_flags & IOCB_NOWAIT) {
3012
3013 if (filemap_range_has_page(inode->i_mapping, pos,
3014 pos + iov_iter_count(from)))
3015 return -EAGAIN;
3016 } else {
3017 written = filemap_write_and_wait_range(mapping, pos,
3018 pos + write_len - 1);
3019 if (written)
3020 goto out;
3021 }
3022
3023
3024
3025
3026
3027
3028
3029 written = invalidate_inode_pages2_range(mapping,
3030 pos >> PAGE_SHIFT, end);
3031
3032
3033
3034
3035 if (written) {
3036 if (written == -EBUSY)
3037 return 0;
3038 goto out;
3039 }
3040
3041 written = mapping->a_ops->direct_IO(iocb, from);
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056 if (mapping->nrpages)
3057 invalidate_inode_pages2_range(mapping,
3058 pos >> PAGE_SHIFT, end);
3059
3060 if (written > 0) {
3061 pos += written;
3062 write_len -= written;
3063 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
3064 i_size_write(inode, pos);
3065 mark_inode_dirty(inode);
3066 }
3067 iocb->ki_pos = pos;
3068 }
3069 iov_iter_revert(from, write_len - iov_iter_count(from));
3070out:
3071 return written;
3072}
3073EXPORT_SYMBOL(generic_file_direct_write);
3074
3075
3076
3077
3078
3079struct page *grab_cache_page_write_begin(struct address_space *mapping,
3080 pgoff_t index, unsigned flags)
3081{
3082 struct page *page;
3083 int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT;
3084
3085 if (flags & AOP_FLAG_NOFS)
3086 fgp_flags |= FGP_NOFS;
3087
3088 page = pagecache_get_page(mapping, index, fgp_flags,
3089 mapping_gfp_mask(mapping));
3090 if (page)
3091 wait_for_stable_page(page);
3092
3093 return page;
3094}
3095EXPORT_SYMBOL(grab_cache_page_write_begin);
3096
3097ssize_t generic_perform_write(struct file *file,
3098 struct iov_iter *i, loff_t pos)
3099{
3100 struct address_space *mapping = file->f_mapping;
3101 const struct address_space_operations *a_ops = mapping->a_ops;
3102 long status = 0;
3103 ssize_t written = 0;
3104 unsigned int flags = 0;
3105
3106 do {
3107 struct page *page;
3108 unsigned long offset;
3109 unsigned long bytes;
3110 size_t copied;
3111 void *fsdata;
3112
3113 offset = (pos & (PAGE_SIZE - 1));
3114 bytes = min_t(unsigned long, PAGE_SIZE - offset,
3115 iov_iter_count(i));
3116
3117again:
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128 if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
3129 status = -EFAULT;
3130 break;
3131 }
3132
3133 if (fatal_signal_pending(current)) {
3134 status = -EINTR;
3135 break;
3136 }
3137
3138 status = a_ops->write_begin(file, mapping, pos, bytes, flags,
3139 &page, &fsdata);
3140 if (unlikely(status < 0))
3141 break;
3142
3143 if (mapping_writably_mapped(mapping))
3144 flush_dcache_page(page);
3145
3146 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
3147 flush_dcache_page(page);
3148
3149 status = a_ops->write_end(file, mapping, pos, bytes, copied,
3150 page, fsdata);
3151 if (unlikely(status < 0))
3152 break;
3153 copied = status;
3154
3155 cond_resched();
3156
3157 iov_iter_advance(i, copied);
3158 if (unlikely(copied == 0)) {
3159
3160
3161
3162
3163
3164
3165
3166
3167 bytes = min_t(unsigned long, PAGE_SIZE - offset,
3168 iov_iter_single_seg_count(i));
3169 goto again;
3170 }
3171 pos += copied;
3172 written += copied;
3173
3174 balance_dirty_pages_ratelimited(mapping);
3175 } while (iov_iter_count(i));
3176
3177 return written ? written : status;
3178}
3179EXPORT_SYMBOL(generic_perform_write);
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3199{
3200 struct file *file = iocb->ki_filp;
3201 struct address_space * mapping = file->f_mapping;
3202 struct inode *inode = mapping->host;
3203 ssize_t written = 0;
3204 ssize_t err;
3205 ssize_t status;
3206
3207
3208 current->backing_dev_info = inode_to_bdi(inode);
3209 err = file_remove_privs(file);
3210 if (err)
3211 goto out;
3212
3213 err = file_update_time(file);
3214 if (err)
3215 goto out;
3216
3217 if (iocb->ki_flags & IOCB_DIRECT) {
3218 loff_t pos, endbyte;
3219
3220 written = generic_file_direct_write(iocb, from);
3221
3222
3223
3224
3225
3226
3227
3228 if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
3229 goto out;
3230
3231 status = generic_perform_write(file, from, pos = iocb->ki_pos);
3232
3233
3234
3235
3236
3237
3238
3239 if (unlikely(status < 0)) {
3240 err = status;
3241 goto out;
3242 }
3243
3244
3245
3246
3247
3248 endbyte = pos + status - 1;
3249 err = filemap_write_and_wait_range(mapping, pos, endbyte);
3250 if (err == 0) {
3251 iocb->ki_pos = endbyte + 1;
3252 written += status;
3253 invalidate_mapping_pages(mapping,
3254 pos >> PAGE_SHIFT,
3255 endbyte >> PAGE_SHIFT);
3256 } else {
3257
3258
3259
3260
3261 }
3262 } else {
3263 written = generic_perform_write(file, from, iocb->ki_pos);
3264 if (likely(written > 0))
3265 iocb->ki_pos += written;
3266 }
3267out:
3268 current->backing_dev_info = NULL;
3269 return written ? written : err;
3270}
3271EXPORT_SYMBOL(__generic_file_write_iter);
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3283{
3284 struct file *file = iocb->ki_filp;
3285 struct inode *inode = file->f_mapping->host;
3286 ssize_t ret;
3287
3288 inode_lock(inode);
3289 ret = generic_write_checks(iocb, from);
3290 if (ret > 0)
3291 ret = __generic_file_write_iter(iocb, from);
3292 inode_unlock(inode);
3293
3294 if (ret > 0)
3295 ret = generic_write_sync(iocb, ret);
3296 return ret;
3297}
3298EXPORT_SYMBOL(generic_file_write_iter);
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317int try_to_release_page(struct page *page, gfp_t gfp_mask)
3318{
3319 struct address_space * const mapping = page->mapping;
3320
3321 BUG_ON(!PageLocked(page));
3322 if (PageWriteback(page))
3323 return 0;
3324
3325 if (mapping && mapping->a_ops->releasepage)
3326 return mapping->a_ops->releasepage(page, gfp_mask);
3327 return try_to_free_buffers(page);
3328}
3329
3330EXPORT_SYMBOL(try_to_release_page);
3331