1
2
3
4
5
6
7
8
9
10
11
12#include <linux/export.h>
13#include <linux/compiler.h>
14#include <linux/dax.h>
15#include <linux/fs.h>
16#include <linux/sched/signal.h>
17#include <linux/uaccess.h>
18#include <linux/capability.h>
19#include <linux/kernel_stat.h>
20#include <linux/gfp.h>
21#include <linux/mm.h>
22#include <linux/swap.h>
23#include <linux/mman.h>
24#include <linux/pagemap.h>
25#include <linux/file.h>
26#include <linux/uio.h>
27#include <linux/hash.h>
28#include <linux/writeback.h>
29#include <linux/backing-dev.h>
30#include <linux/pagevec.h>
31#include <linux/blkdev.h>
32#include <linux/security.h>
33#include <linux/cpuset.h>
34#include <linux/hugetlb.h>
35#include <linux/memcontrol.h>
36#include <linux/cleancache.h>
37#include <linux/shmem_fs.h>
38#include <linux/rmap.h>
39#include "internal.h"
40
41#define CREATE_TRACE_POINTS
42#include <trace/events/filemap.h>
43
44
45
46
47#include <linux/buffer_head.h>
48
49#include <asm/mman.h>
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114static int page_cache_tree_insert(struct address_space *mapping,
115 struct page *page, void **shadowp)
116{
117 struct radix_tree_node *node;
118 void **slot;
119 int error;
120
121 error = __radix_tree_create(&mapping->i_pages, page->index, 0,
122 &node, &slot);
123 if (error)
124 return error;
125 if (*slot) {
126 void *p;
127
128 p = radix_tree_deref_slot_protected(slot,
129 &mapping->i_pages.xa_lock);
130 if (!radix_tree_exceptional_entry(p))
131 return -EEXIST;
132
133 mapping->nrexceptional--;
134 if (shadowp)
135 *shadowp = p;
136 }
137 __radix_tree_replace(&mapping->i_pages, node, slot, page,
138 workingset_lookup_update(mapping));
139 mapping->nrpages++;
140 return 0;
141}
142
143static void page_cache_tree_delete(struct address_space *mapping,
144 struct page *page, void *shadow)
145{
146 int i, nr;
147
148
149 nr = PageHuge(page) ? 1 : hpage_nr_pages(page);
150
151 VM_BUG_ON_PAGE(!PageLocked(page), page);
152 VM_BUG_ON_PAGE(PageTail(page), page);
153 VM_BUG_ON_PAGE(nr != 1 && shadow, page);
154
155 for (i = 0; i < nr; i++) {
156 struct radix_tree_node *node;
157 void **slot;
158
159 __radix_tree_lookup(&mapping->i_pages, page->index + i,
160 &node, &slot);
161
162 VM_BUG_ON_PAGE(!node && nr != 1, page);
163
164 radix_tree_clear_tags(&mapping->i_pages, node, slot);
165 __radix_tree_replace(&mapping->i_pages, node, slot, shadow,
166 workingset_lookup_update(mapping));
167 }
168
169 page->mapping = NULL;
170
171
172 if (shadow) {
173 mapping->nrexceptional += nr;
174
175
176
177
178
179
180 smp_wmb();
181 }
182 mapping->nrpages -= nr;
183}
184
185static void unaccount_page_cache_page(struct address_space *mapping,
186 struct page *page)
187{
188 int nr;
189
190
191
192
193
194
195 if (PageUptodate(page) && PageMappedToDisk(page))
196 cleancache_put_page(page);
197 else
198 cleancache_invalidate_page(mapping, page);
199
200 VM_BUG_ON_PAGE(PageTail(page), page);
201 VM_BUG_ON_PAGE(page_mapped(page), page);
202 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
203 int mapcount;
204
205 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
206 current->comm, page_to_pfn(page));
207 dump_page(page, "still mapped when deleted");
208 dump_stack();
209 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
210
211 mapcount = page_mapcount(page);
212 if (mapping_exiting(mapping) &&
213 page_count(page) >= mapcount + 2) {
214
215
216
217
218
219
220 page_mapcount_reset(page);
221 page_ref_sub(page, mapcount);
222 }
223 }
224
225
226 if (PageHuge(page))
227 return;
228
229 nr = hpage_nr_pages(page);
230
231 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
232 if (PageSwapBacked(page)) {
233 __mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);
234 if (PageTransHuge(page))
235 __dec_node_page_state(page, NR_SHMEM_THPS);
236 } else {
237 VM_BUG_ON_PAGE(PageTransHuge(page), page);
238 }
239
240
241
242
243
244
245
246
247
248
249
250 if (WARN_ON_ONCE(PageDirty(page)))
251 account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
252}
253
254
255
256
257
258
259void __delete_from_page_cache(struct page *page, void *shadow)
260{
261 struct address_space *mapping = page->mapping;
262
263 trace_mm_filemap_delete_from_page_cache(page);
264
265 unaccount_page_cache_page(mapping, page);
266 page_cache_tree_delete(mapping, page, shadow);
267}
268
269static void page_cache_free_page(struct address_space *mapping,
270 struct page *page)
271{
272 void (*freepage)(struct page *);
273
274 freepage = mapping->a_ops->freepage;
275 if (freepage)
276 freepage(page);
277
278 if (PageTransHuge(page) && !PageHuge(page)) {
279 page_ref_sub(page, HPAGE_PMD_NR);
280 VM_BUG_ON_PAGE(page_count(page) <= 0, page);
281 } else {
282 put_page(page);
283 }
284}
285
286
287
288
289
290
291
292
293
294void delete_from_page_cache(struct page *page)
295{
296 struct address_space *mapping = page_mapping(page);
297 unsigned long flags;
298
299 BUG_ON(!PageLocked(page));
300 xa_lock_irqsave(&mapping->i_pages, flags);
301 __delete_from_page_cache(page, NULL);
302 xa_unlock_irqrestore(&mapping->i_pages, flags);
303
304 page_cache_free_page(mapping, page);
305}
306EXPORT_SYMBOL(delete_from_page_cache);
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322static void
323page_cache_tree_delete_batch(struct address_space *mapping,
324 struct pagevec *pvec)
325{
326 struct radix_tree_iter iter;
327 void **slot;
328 int total_pages = 0;
329 int i = 0, tail_pages = 0;
330 struct page *page;
331 pgoff_t start;
332
333 start = pvec->pages[0]->index;
334 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
335 if (i >= pagevec_count(pvec) && !tail_pages)
336 break;
337 page = radix_tree_deref_slot_protected(slot,
338 &mapping->i_pages.xa_lock);
339 if (radix_tree_exceptional_entry(page))
340 continue;
341 if (!tail_pages) {
342
343
344
345
346
347 if (page != pvec->pages[i])
348 continue;
349 WARN_ON_ONCE(!PageLocked(page));
350 if (PageTransHuge(page) && !PageHuge(page))
351 tail_pages = HPAGE_PMD_NR - 1;
352 page->mapping = NULL;
353
354
355
356
357 i++;
358 } else {
359 tail_pages--;
360 }
361 radix_tree_clear_tags(&mapping->i_pages, iter.node, slot);
362 __radix_tree_replace(&mapping->i_pages, iter.node, slot, NULL,
363 workingset_lookup_update(mapping));
364 total_pages++;
365 }
366 mapping->nrpages -= total_pages;
367}
368
369void delete_from_page_cache_batch(struct address_space *mapping,
370 struct pagevec *pvec)
371{
372 int i;
373 unsigned long flags;
374
375 if (!pagevec_count(pvec))
376 return;
377
378 xa_lock_irqsave(&mapping->i_pages, flags);
379 for (i = 0; i < pagevec_count(pvec); i++) {
380 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]);
381
382 unaccount_page_cache_page(mapping, pvec->pages[i]);
383 }
384 page_cache_tree_delete_batch(mapping, pvec);
385 xa_unlock_irqrestore(&mapping->i_pages, flags);
386
387 for (i = 0; i < pagevec_count(pvec); i++)
388 page_cache_free_page(mapping, pvec->pages[i]);
389}
390
391int filemap_check_errors(struct address_space *mapping)
392{
393 int ret = 0;
394
395 if (test_bit(AS_ENOSPC, &mapping->flags) &&
396 test_and_clear_bit(AS_ENOSPC, &mapping->flags))
397 ret = -ENOSPC;
398 if (test_bit(AS_EIO, &mapping->flags) &&
399 test_and_clear_bit(AS_EIO, &mapping->flags))
400 ret = -EIO;
401 return ret;
402}
403EXPORT_SYMBOL(filemap_check_errors);
404
405static int filemap_check_and_keep_errors(struct address_space *mapping)
406{
407
408 if (test_bit(AS_EIO, &mapping->flags))
409 return -EIO;
410 if (test_bit(AS_ENOSPC, &mapping->flags))
411 return -ENOSPC;
412 return 0;
413}
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
431 loff_t end, int sync_mode)
432{
433 int ret;
434 struct writeback_control wbc = {
435 .sync_mode = sync_mode,
436 .nr_to_write = LONG_MAX,
437 .range_start = start,
438 .range_end = end,
439 };
440
441 if (!mapping_cap_writeback_dirty(mapping))
442 return 0;
443
444 wbc_attach_fdatawrite_inode(&wbc, mapping->host);
445 ret = do_writepages(mapping, &wbc);
446 wbc_detach_inode(&wbc);
447 return ret;
448}
449
450static inline int __filemap_fdatawrite(struct address_space *mapping,
451 int sync_mode)
452{
453 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
454}
455
456int filemap_fdatawrite(struct address_space *mapping)
457{
458 return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
459}
460EXPORT_SYMBOL(filemap_fdatawrite);
461
462int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
463 loff_t end)
464{
465 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
466}
467EXPORT_SYMBOL(filemap_fdatawrite_range);
468
469
470
471
472
473
474
475
476int filemap_flush(struct address_space *mapping)
477{
478 return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
479}
480EXPORT_SYMBOL(filemap_flush);
481
482
483
484
485
486
487
488
489
490
491bool filemap_range_has_page(struct address_space *mapping,
492 loff_t start_byte, loff_t end_byte)
493{
494 pgoff_t index = start_byte >> PAGE_SHIFT;
495 pgoff_t end = end_byte >> PAGE_SHIFT;
496 struct page *page;
497
498 if (end_byte < start_byte)
499 return false;
500
501 if (mapping->nrpages == 0)
502 return false;
503
504 if (!find_get_pages_range(mapping, &index, end, 1, &page))
505 return false;
506 put_page(page);
507 return true;
508}
509EXPORT_SYMBOL(filemap_range_has_page);
510
511static void __filemap_fdatawait_range(struct address_space *mapping,
512 loff_t start_byte, loff_t end_byte)
513{
514 pgoff_t index = start_byte >> PAGE_SHIFT;
515 pgoff_t end = end_byte >> PAGE_SHIFT;
516 struct pagevec pvec;
517 int nr_pages;
518
519 if (end_byte < start_byte)
520 return;
521
522 pagevec_init(&pvec);
523 while (index <= end) {
524 unsigned i;
525
526 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index,
527 end, PAGECACHE_TAG_WRITEBACK);
528 if (!nr_pages)
529 break;
530
531 for (i = 0; i < nr_pages; i++) {
532 struct page *page = pvec.pages[i];
533
534 wait_on_page_writeback(page);
535 ClearPageError(page);
536 }
537 pagevec_release(&pvec);
538 cond_resched();
539 }
540}
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
557 loff_t end_byte)
558{
559 __filemap_fdatawait_range(mapping, start_byte, end_byte);
560 return filemap_check_errors(mapping);
561}
562EXPORT_SYMBOL(filemap_fdatawait_range);
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte)
579{
580 struct address_space *mapping = file->f_mapping;
581
582 __filemap_fdatawait_range(mapping, start_byte, end_byte);
583 return file_check_and_advance_wb_err(file);
584}
585EXPORT_SYMBOL(file_fdatawait_range);
586
587
588
589
590
591
592
593
594
595
596
597
598
599int filemap_fdatawait_keep_errors(struct address_space *mapping)
600{
601 __filemap_fdatawait_range(mapping, 0, LLONG_MAX);
602 return filemap_check_and_keep_errors(mapping);
603}
604EXPORT_SYMBOL(filemap_fdatawait_keep_errors);
605
606static bool mapping_needs_writeback(struct address_space *mapping)
607{
608 return (!dax_mapping(mapping) && mapping->nrpages) ||
609 (dax_mapping(mapping) && mapping->nrexceptional);
610}
611
612int filemap_write_and_wait(struct address_space *mapping)
613{
614 int err = 0;
615
616 if (mapping_needs_writeback(mapping)) {
617 err = filemap_fdatawrite(mapping);
618
619
620
621
622
623
624 if (err != -EIO) {
625 int err2 = filemap_fdatawait(mapping);
626 if (!err)
627 err = err2;
628 } else {
629
630 filemap_check_errors(mapping);
631 }
632 } else {
633 err = filemap_check_errors(mapping);
634 }
635 return err;
636}
637EXPORT_SYMBOL(filemap_write_and_wait);
638
639
640
641
642
643
644
645
646
647
648
649
650int filemap_write_and_wait_range(struct address_space *mapping,
651 loff_t lstart, loff_t lend)
652{
653 int err = 0;
654
655 if (mapping_needs_writeback(mapping)) {
656 err = __filemap_fdatawrite_range(mapping, lstart, lend,
657 WB_SYNC_ALL);
658
659 if (err != -EIO) {
660 int err2 = filemap_fdatawait_range(mapping,
661 lstart, lend);
662 if (!err)
663 err = err2;
664 } else {
665
666 filemap_check_errors(mapping);
667 }
668 } else {
669 err = filemap_check_errors(mapping);
670 }
671 return err;
672}
673EXPORT_SYMBOL(filemap_write_and_wait_range);
674
675void __filemap_set_wb_err(struct address_space *mapping, int err)
676{
677 errseq_t eseq = errseq_set(&mapping->wb_err, err);
678
679 trace_filemap_set_wb_err(mapping, eseq);
680}
681EXPORT_SYMBOL(__filemap_set_wb_err);
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705int file_check_and_advance_wb_err(struct file *file)
706{
707 int err = 0;
708 errseq_t old = READ_ONCE(file->f_wb_err);
709 struct address_space *mapping = file->f_mapping;
710
711
712 if (errseq_check(&mapping->wb_err, old)) {
713
714 spin_lock(&file->f_lock);
715 old = file->f_wb_err;
716 err = errseq_check_and_advance(&mapping->wb_err,
717 &file->f_wb_err);
718 trace_file_check_and_advance_wb_err(file, old);
719 spin_unlock(&file->f_lock);
720 }
721
722
723
724
725
726
727 clear_bit(AS_EIO, &mapping->flags);
728 clear_bit(AS_ENOSPC, &mapping->flags);
729 return err;
730}
731EXPORT_SYMBOL(file_check_and_advance_wb_err);
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend)
748{
749 int err = 0, err2;
750 struct address_space *mapping = file->f_mapping;
751
752 if (mapping_needs_writeback(mapping)) {
753 err = __filemap_fdatawrite_range(mapping, lstart, lend,
754 WB_SYNC_ALL);
755
756 if (err != -EIO)
757 __filemap_fdatawait_range(mapping, lstart, lend);
758 }
759 err2 = file_check_and_advance_wb_err(file);
760 if (!err)
761 err = err2;
762 return err;
763}
764EXPORT_SYMBOL(file_write_and_wait_range);
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
782{
783 int error;
784
785 VM_BUG_ON_PAGE(!PageLocked(old), old);
786 VM_BUG_ON_PAGE(!PageLocked(new), new);
787 VM_BUG_ON_PAGE(new->mapping, new);
788
789 error = radix_tree_preload(gfp_mask & GFP_RECLAIM_MASK);
790 if (!error) {
791 struct address_space *mapping = old->mapping;
792 void (*freepage)(struct page *);
793 unsigned long flags;
794
795 pgoff_t offset = old->index;
796 freepage = mapping->a_ops->freepage;
797
798 get_page(new);
799 new->mapping = mapping;
800 new->index = offset;
801
802 xa_lock_irqsave(&mapping->i_pages, flags);
803 __delete_from_page_cache(old, NULL);
804 error = page_cache_tree_insert(mapping, new, NULL);
805 BUG_ON(error);
806
807
808
809
810 if (!PageHuge(new))
811 __inc_node_page_state(new, NR_FILE_PAGES);
812 if (PageSwapBacked(new))
813 __inc_node_page_state(new, NR_SHMEM);
814 xa_unlock_irqrestore(&mapping->i_pages, flags);
815 mem_cgroup_migrate(old, new);
816 radix_tree_preload_end();
817 if (freepage)
818 freepage(old);
819 put_page(old);
820 }
821
822 return error;
823}
824EXPORT_SYMBOL_GPL(replace_page_cache_page);
825
826static int __add_to_page_cache_locked(struct page *page,
827 struct address_space *mapping,
828 pgoff_t offset, gfp_t gfp_mask,
829 void **shadowp)
830{
831 int huge = PageHuge(page);
832 struct mem_cgroup *memcg;
833 int error;
834
835 VM_BUG_ON_PAGE(!PageLocked(page), page);
836 VM_BUG_ON_PAGE(PageSwapBacked(page), page);
837
838 if (!huge) {
839 error = mem_cgroup_try_charge(page, current->mm,
840 gfp_mask, &memcg, false);
841 if (error)
842 return error;
843 }
844
845 error = radix_tree_maybe_preload(gfp_mask & GFP_RECLAIM_MASK);
846 if (error) {
847 if (!huge)
848 mem_cgroup_cancel_charge(page, memcg, false);
849 return error;
850 }
851
852 get_page(page);
853 page->mapping = mapping;
854 page->index = offset;
855
856 xa_lock_irq(&mapping->i_pages);
857 error = page_cache_tree_insert(mapping, page, shadowp);
858 radix_tree_preload_end();
859 if (unlikely(error))
860 goto err_insert;
861
862
863 if (!huge)
864 __inc_node_page_state(page, NR_FILE_PAGES);
865 xa_unlock_irq(&mapping->i_pages);
866 if (!huge)
867 mem_cgroup_commit_charge(page, memcg, false, false);
868 trace_mm_filemap_add_to_page_cache(page);
869 return 0;
870err_insert:
871 page->mapping = NULL;
872
873 xa_unlock_irq(&mapping->i_pages);
874 if (!huge)
875 mem_cgroup_cancel_charge(page, memcg, false);
876 put_page(page);
877 return error;
878}
879
880
881
882
883
884
885
886
887
888
889
890int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
891 pgoff_t offset, gfp_t gfp_mask)
892{
893 return __add_to_page_cache_locked(page, mapping, offset,
894 gfp_mask, NULL);
895}
896EXPORT_SYMBOL(add_to_page_cache_locked);
897
898int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
899 pgoff_t offset, gfp_t gfp_mask)
900{
901 void *shadow = NULL;
902 int ret;
903
904 __SetPageLocked(page);
905 ret = __add_to_page_cache_locked(page, mapping, offset,
906 gfp_mask, &shadow);
907 if (unlikely(ret))
908 __ClearPageLocked(page);
909 else {
910
911
912
913
914
915
916
917
918 if (!(gfp_mask & __GFP_WRITE) &&
919 shadow && workingset_refault(shadow)) {
920 SetPageActive(page);
921 workingset_activation(page);
922 } else
923 ClearPageActive(page);
924 lru_cache_add(page);
925 }
926 return ret;
927}
928EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
929
930#ifdef CONFIG_NUMA
931struct page *__page_cache_alloc(gfp_t gfp)
932{
933 int n;
934 struct page *page;
935
936 if (cpuset_do_page_mem_spread()) {
937 unsigned int cpuset_mems_cookie;
938 do {
939 cpuset_mems_cookie = read_mems_allowed_begin();
940 n = cpuset_mem_spread_node();
941 page = __alloc_pages_node(n, gfp, 0);
942 } while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
943
944 return page;
945 }
946 return alloc_pages(gfp, 0);
947}
948EXPORT_SYMBOL(__page_cache_alloc);
949#endif
950
951
952
953
954
955
956
957
958
959
960
961#define PAGE_WAIT_TABLE_BITS 8
962#define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
963static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
964
965static wait_queue_head_t *page_waitqueue(struct page *page)
966{
967 return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)];
968}
969
970void __init pagecache_init(void)
971{
972 int i;
973
974 for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
975 init_waitqueue_head(&page_wait_table[i]);
976
977 page_writeback_init();
978}
979
980
981struct wait_page_key {
982 struct page *page;
983 int bit_nr;
984 int page_match;
985};
986
987struct wait_page_queue {
988 struct page *page;
989 int bit_nr;
990 wait_queue_entry_t wait;
991};
992
993static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg)
994{
995 struct wait_page_key *key = arg;
996 struct wait_page_queue *wait_page
997 = container_of(wait, struct wait_page_queue, wait);
998
999 if (wait_page->page != key->page)
1000 return 0;
1001 key->page_match = 1;
1002
1003 if (wait_page->bit_nr != key->bit_nr)
1004 return 0;
1005
1006
1007 if (test_bit(key->bit_nr, &key->page->flags))
1008 return -1;
1009
1010 return autoremove_wake_function(wait, mode, sync, key);
1011}
1012
1013static void wake_up_page_bit(struct page *page, int bit_nr)
1014{
1015 wait_queue_head_t *q = page_waitqueue(page);
1016 struct wait_page_key key;
1017 unsigned long flags;
1018 wait_queue_entry_t bookmark;
1019
1020 key.page = page;
1021 key.bit_nr = bit_nr;
1022 key.page_match = 0;
1023
1024 bookmark.flags = 0;
1025 bookmark.private = NULL;
1026 bookmark.func = NULL;
1027 INIT_LIST_HEAD(&bookmark.entry);
1028
1029 spin_lock_irqsave(&q->lock, flags);
1030 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
1031
1032 while (bookmark.flags & WQ_FLAG_BOOKMARK) {
1033
1034
1035
1036
1037
1038
1039 spin_unlock_irqrestore(&q->lock, flags);
1040 cpu_relax();
1041 spin_lock_irqsave(&q->lock, flags);
1042 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
1043 }
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054 if (!waitqueue_active(q) || !key.page_match) {
1055 ClearPageWaiters(page);
1056
1057
1058
1059
1060
1061
1062
1063 }
1064 spin_unlock_irqrestore(&q->lock, flags);
1065}
1066
1067static void wake_up_page(struct page *page, int bit)
1068{
1069 if (!PageWaiters(page))
1070 return;
1071 wake_up_page_bit(page, bit);
1072}
1073
1074static inline int wait_on_page_bit_common(wait_queue_head_t *q,
1075 struct page *page, int bit_nr, int state, bool lock)
1076{
1077 struct wait_page_queue wait_page;
1078 wait_queue_entry_t *wait = &wait_page.wait;
1079 int ret = 0;
1080
1081 init_wait(wait);
1082 wait->flags = lock ? WQ_FLAG_EXCLUSIVE : 0;
1083 wait->func = wake_page_function;
1084 wait_page.page = page;
1085 wait_page.bit_nr = bit_nr;
1086
1087 for (;;) {
1088 spin_lock_irq(&q->lock);
1089
1090 if (likely(list_empty(&wait->entry))) {
1091 __add_wait_queue_entry_tail(q, wait);
1092 SetPageWaiters(page);
1093 }
1094
1095 set_current_state(state);
1096
1097 spin_unlock_irq(&q->lock);
1098
1099 if (likely(test_bit(bit_nr, &page->flags))) {
1100 io_schedule();
1101 }
1102
1103 if (lock) {
1104 if (!test_and_set_bit_lock(bit_nr, &page->flags))
1105 break;
1106 } else {
1107 if (!test_bit(bit_nr, &page->flags))
1108 break;
1109 }
1110
1111 if (unlikely(signal_pending_state(state, current))) {
1112 ret = -EINTR;
1113 break;
1114 }
1115 }
1116
1117 finish_wait(q, wait);
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127 return ret;
1128}
1129
1130void wait_on_page_bit(struct page *page, int bit_nr)
1131{
1132 wait_queue_head_t *q = page_waitqueue(page);
1133 wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, false);
1134}
1135EXPORT_SYMBOL(wait_on_page_bit);
1136
1137int wait_on_page_bit_killable(struct page *page, int bit_nr)
1138{
1139 wait_queue_head_t *q = page_waitqueue(page);
1140 return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, false);
1141}
1142EXPORT_SYMBOL(wait_on_page_bit_killable);
1143
1144
1145
1146
1147
1148
1149
1150
1151void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter)
1152{
1153 wait_queue_head_t *q = page_waitqueue(page);
1154 unsigned long flags;
1155
1156 spin_lock_irqsave(&q->lock, flags);
1157 __add_wait_queue_entry_tail(q, waiter);
1158 SetPageWaiters(page);
1159 spin_unlock_irqrestore(&q->lock, flags);
1160}
1161EXPORT_SYMBOL_GPL(add_page_wait_queue);
1162
1163#ifndef clear_bit_unlock_is_negative_byte
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem)
1178{
1179 clear_bit_unlock(nr, mem);
1180
1181 return test_bit(PG_waiters, mem);
1182}
1183
1184#endif
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201void unlock_page(struct page *page)
1202{
1203 BUILD_BUG_ON(PG_waiters != 7);
1204 page = compound_head(page);
1205 VM_BUG_ON_PAGE(!PageLocked(page), page);
1206 if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags))
1207 wake_up_page_bit(page, PG_locked);
1208}
1209EXPORT_SYMBOL(unlock_page);
1210
1211
1212
1213
1214
1215void end_page_writeback(struct page *page)
1216{
1217
1218
1219
1220
1221
1222
1223
1224 if (PageReclaim(page)) {
1225 ClearPageReclaim(page);
1226 rotate_reclaimable_page(page);
1227 }
1228
1229 if (!test_clear_page_writeback(page))
1230 BUG();
1231
1232 smp_mb__after_atomic();
1233 wake_up_page(page, PG_writeback);
1234}
1235EXPORT_SYMBOL(end_page_writeback);
1236
1237
1238
1239
1240
1241void page_endio(struct page *page, bool is_write, int err)
1242{
1243 if (!is_write) {
1244 if (!err) {
1245 SetPageUptodate(page);
1246 } else {
1247 ClearPageUptodate(page);
1248 SetPageError(page);
1249 }
1250 unlock_page(page);
1251 } else {
1252 if (err) {
1253 struct address_space *mapping;
1254
1255 SetPageError(page);
1256 mapping = page_mapping(page);
1257 if (mapping)
1258 mapping_set_error(mapping, err);
1259 }
1260 end_page_writeback(page);
1261 }
1262}
1263EXPORT_SYMBOL_GPL(page_endio);
1264
1265
1266
1267
1268
1269void __lock_page(struct page *__page)
1270{
1271 struct page *page = compound_head(__page);
1272 wait_queue_head_t *q = page_waitqueue(page);
1273 wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, true);
1274}
1275EXPORT_SYMBOL(__lock_page);
1276
1277int __lock_page_killable(struct page *__page)
1278{
1279 struct page *page = compound_head(__page);
1280 wait_queue_head_t *q = page_waitqueue(page);
1281 return wait_on_page_bit_common(q, page, PG_locked, TASK_KILLABLE, true);
1282}
1283EXPORT_SYMBOL_GPL(__lock_page_killable);
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
1297 unsigned int flags)
1298{
1299 if (flags & FAULT_FLAG_ALLOW_RETRY) {
1300
1301
1302
1303
1304 if (flags & FAULT_FLAG_RETRY_NOWAIT)
1305 return 0;
1306
1307 up_read(&mm->mmap_sem);
1308 if (flags & FAULT_FLAG_KILLABLE)
1309 wait_on_page_locked_killable(page);
1310 else
1311 wait_on_page_locked(page);
1312 return 0;
1313 } else {
1314 if (flags & FAULT_FLAG_KILLABLE) {
1315 int ret;
1316
1317 ret = __lock_page_killable(page);
1318 if (ret) {
1319 up_read(&mm->mmap_sem);
1320 return 0;
1321 }
1322 } else
1323 __lock_page(page);
1324 return 1;
1325 }
1326}
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349pgoff_t page_cache_next_hole(struct address_space *mapping,
1350 pgoff_t index, unsigned long max_scan)
1351{
1352 unsigned long i;
1353
1354 for (i = 0; i < max_scan; i++) {
1355 struct page *page;
1356
1357 page = radix_tree_lookup(&mapping->i_pages, index);
1358 if (!page || radix_tree_exceptional_entry(page))
1359 break;
1360 index++;
1361 if (index == 0)
1362 break;
1363 }
1364
1365 return index;
1366}
1367EXPORT_SYMBOL(page_cache_next_hole);
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390pgoff_t page_cache_prev_hole(struct address_space *mapping,
1391 pgoff_t index, unsigned long max_scan)
1392{
1393 unsigned long i;
1394
1395 for (i = 0; i < max_scan; i++) {
1396 struct page *page;
1397
1398 page = radix_tree_lookup(&mapping->i_pages, index);
1399 if (!page || radix_tree_exceptional_entry(page))
1400 break;
1401 index--;
1402 if (index == ULONG_MAX)
1403 break;
1404 }
1405
1406 return index;
1407}
1408EXPORT_SYMBOL(page_cache_prev_hole);
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
1424{
1425 void **pagep;
1426 struct page *head, *page;
1427
1428 rcu_read_lock();
1429repeat:
1430 page = NULL;
1431 pagep = radix_tree_lookup_slot(&mapping->i_pages, offset);
1432 if (pagep) {
1433 page = radix_tree_deref_slot(pagep);
1434 if (unlikely(!page))
1435 goto out;
1436 if (radix_tree_exception(page)) {
1437 if (radix_tree_deref_retry(page))
1438 goto repeat;
1439
1440
1441
1442
1443
1444 goto out;
1445 }
1446
1447 head = compound_head(page);
1448 if (!page_cache_get_speculative(head))
1449 goto repeat;
1450
1451
1452 if (compound_head(page) != head) {
1453 put_page(head);
1454 goto repeat;
1455 }
1456
1457
1458
1459
1460
1461
1462 if (unlikely(page != *pagep)) {
1463 put_page(head);
1464 goto repeat;
1465 }
1466 }
1467out:
1468 rcu_read_unlock();
1469
1470 return page;
1471}
1472EXPORT_SYMBOL(find_get_entry);
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
1491{
1492 struct page *page;
1493
1494repeat:
1495 page = find_get_entry(mapping, offset);
1496 if (page && !radix_tree_exception(page)) {
1497 lock_page(page);
1498
1499 if (unlikely(page_mapping(page) != mapping)) {
1500 unlock_page(page);
1501 put_page(page);
1502 goto repeat;
1503 }
1504 VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
1505 }
1506 return page;
1507}
1508EXPORT_SYMBOL(find_lock_entry);
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
1536 int fgp_flags, gfp_t gfp_mask)
1537{
1538 struct page *page;
1539
1540repeat:
1541 page = find_get_entry(mapping, offset);
1542 if (radix_tree_exceptional_entry(page))
1543 page = NULL;
1544 if (!page)
1545 goto no_page;
1546
1547 if (fgp_flags & FGP_LOCK) {
1548 if (fgp_flags & FGP_NOWAIT) {
1549 if (!trylock_page(page)) {
1550 put_page(page);
1551 return NULL;
1552 }
1553 } else {
1554 lock_page(page);
1555 }
1556
1557
1558 if (unlikely(page->mapping != mapping)) {
1559 unlock_page(page);
1560 put_page(page);
1561 goto repeat;
1562 }
1563 VM_BUG_ON_PAGE(page->index != offset, page);
1564 }
1565
1566 if (page && (fgp_flags & FGP_ACCESSED))
1567 mark_page_accessed(page);
1568
1569no_page:
1570 if (!page && (fgp_flags & FGP_CREAT)) {
1571 int err;
1572 if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
1573 gfp_mask |= __GFP_WRITE;
1574 if (fgp_flags & FGP_NOFS)
1575 gfp_mask &= ~__GFP_FS;
1576
1577 page = __page_cache_alloc(gfp_mask);
1578 if (!page)
1579 return NULL;
1580
1581 if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK)))
1582 fgp_flags |= FGP_LOCK;
1583
1584
1585 if (fgp_flags & FGP_ACCESSED)
1586 __SetPageReferenced(page);
1587
1588 err = add_to_page_cache_lru(page, mapping, offset, gfp_mask);
1589 if (unlikely(err)) {
1590 put_page(page);
1591 page = NULL;
1592 if (err == -EEXIST)
1593 goto repeat;
1594 }
1595 }
1596
1597 return page;
1598}
1599EXPORT_SYMBOL(pagecache_get_page);
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624unsigned find_get_entries(struct address_space *mapping,
1625 pgoff_t start, unsigned int nr_entries,
1626 struct page **entries, pgoff_t *indices)
1627{
1628 void **slot;
1629 unsigned int ret = 0;
1630 struct radix_tree_iter iter;
1631
1632 if (!nr_entries)
1633 return 0;
1634
1635 rcu_read_lock();
1636 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1637 struct page *head, *page;
1638repeat:
1639 page = radix_tree_deref_slot(slot);
1640 if (unlikely(!page))
1641 continue;
1642 if (radix_tree_exception(page)) {
1643 if (radix_tree_deref_retry(page)) {
1644 slot = radix_tree_iter_retry(&iter);
1645 continue;
1646 }
1647
1648
1649
1650
1651
1652 goto export;
1653 }
1654
1655 head = compound_head(page);
1656 if (!page_cache_get_speculative(head))
1657 goto repeat;
1658
1659
1660 if (compound_head(page) != head) {
1661 put_page(head);
1662 goto repeat;
1663 }
1664
1665
1666 if (unlikely(page != *slot)) {
1667 put_page(head);
1668 goto repeat;
1669 }
1670export:
1671 indices[ret] = iter.index;
1672 entries[ret] = page;
1673 if (++ret == nr_entries)
1674 break;
1675 }
1676 rcu_read_unlock();
1677 return ret;
1678}
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
1702 pgoff_t end, unsigned int nr_pages,
1703 struct page **pages)
1704{
1705 struct radix_tree_iter iter;
1706 void **slot;
1707 unsigned ret = 0;
1708
1709 if (unlikely(!nr_pages))
1710 return 0;
1711
1712 rcu_read_lock();
1713 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, *start) {
1714 struct page *head, *page;
1715
1716 if (iter.index > end)
1717 break;
1718repeat:
1719 page = radix_tree_deref_slot(slot);
1720 if (unlikely(!page))
1721 continue;
1722
1723 if (radix_tree_exception(page)) {
1724 if (radix_tree_deref_retry(page)) {
1725 slot = radix_tree_iter_retry(&iter);
1726 continue;
1727 }
1728
1729
1730
1731
1732
1733 continue;
1734 }
1735
1736 head = compound_head(page);
1737 if (!page_cache_get_speculative(head))
1738 goto repeat;
1739
1740
1741 if (compound_head(page) != head) {
1742 put_page(head);
1743 goto repeat;
1744 }
1745
1746
1747 if (unlikely(page != *slot)) {
1748 put_page(head);
1749 goto repeat;
1750 }
1751
1752 pages[ret] = page;
1753 if (++ret == nr_pages) {
1754 *start = pages[ret - 1]->index + 1;
1755 goto out;
1756 }
1757 }
1758
1759
1760
1761
1762
1763
1764
1765 if (end == (pgoff_t)-1)
1766 *start = (pgoff_t)-1;
1767 else
1768 *start = end + 1;
1769out:
1770 rcu_read_unlock();
1771
1772 return ret;
1773}
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
1788 unsigned int nr_pages, struct page **pages)
1789{
1790 struct radix_tree_iter iter;
1791 void **slot;
1792 unsigned int ret = 0;
1793
1794 if (unlikely(!nr_pages))
1795 return 0;
1796
1797 rcu_read_lock();
1798 radix_tree_for_each_contig(slot, &mapping->i_pages, &iter, index) {
1799 struct page *head, *page;
1800repeat:
1801 page = radix_tree_deref_slot(slot);
1802
1803 if (unlikely(!page))
1804 break;
1805
1806 if (radix_tree_exception(page)) {
1807 if (radix_tree_deref_retry(page)) {
1808 slot = radix_tree_iter_retry(&iter);
1809 continue;
1810 }
1811
1812
1813
1814
1815
1816 break;
1817 }
1818
1819 head = compound_head(page);
1820 if (!page_cache_get_speculative(head))
1821 goto repeat;
1822
1823
1824 if (compound_head(page) != head) {
1825 put_page(head);
1826 goto repeat;
1827 }
1828
1829
1830 if (unlikely(page != *slot)) {
1831 put_page(head);
1832 goto repeat;
1833 }
1834
1835
1836
1837
1838
1839
1840 if (page->mapping == NULL || page_to_pgoff(page) != iter.index) {
1841 put_page(page);
1842 break;
1843 }
1844
1845 pages[ret] = page;
1846 if (++ret == nr_pages)
1847 break;
1848 }
1849 rcu_read_unlock();
1850 return ret;
1851}
1852EXPORT_SYMBOL(find_get_pages_contig);
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
1867 pgoff_t end, int tag, unsigned int nr_pages,
1868 struct page **pages)
1869{
1870 struct radix_tree_iter iter;
1871 void **slot;
1872 unsigned ret = 0;
1873
1874 if (unlikely(!nr_pages))
1875 return 0;
1876
1877 rcu_read_lock();
1878 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, *index, tag) {
1879 struct page *head, *page;
1880
1881 if (iter.index > end)
1882 break;
1883repeat:
1884 page = radix_tree_deref_slot(slot);
1885 if (unlikely(!page))
1886 continue;
1887
1888 if (radix_tree_exception(page)) {
1889 if (radix_tree_deref_retry(page)) {
1890 slot = radix_tree_iter_retry(&iter);
1891 continue;
1892 }
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904 continue;
1905 }
1906
1907 head = compound_head(page);
1908 if (!page_cache_get_speculative(head))
1909 goto repeat;
1910
1911
1912 if (compound_head(page) != head) {
1913 put_page(head);
1914 goto repeat;
1915 }
1916
1917
1918 if (unlikely(page != *slot)) {
1919 put_page(head);
1920 goto repeat;
1921 }
1922
1923 pages[ret] = page;
1924 if (++ret == nr_pages) {
1925 *index = pages[ret - 1]->index + 1;
1926 goto out;
1927 }
1928 }
1929
1930
1931
1932
1933
1934
1935
1936 if (end == (pgoff_t)-1)
1937 *index = (pgoff_t)-1;
1938 else
1939 *index = end + 1;
1940out:
1941 rcu_read_unlock();
1942
1943 return ret;
1944}
1945EXPORT_SYMBOL(find_get_pages_range_tag);
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
1960 int tag, unsigned int nr_entries,
1961 struct page **entries, pgoff_t *indices)
1962{
1963 void **slot;
1964 unsigned int ret = 0;
1965 struct radix_tree_iter iter;
1966
1967 if (!nr_entries)
1968 return 0;
1969
1970 rcu_read_lock();
1971 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, tag) {
1972 struct page *head, *page;
1973repeat:
1974 page = radix_tree_deref_slot(slot);
1975 if (unlikely(!page))
1976 continue;
1977 if (radix_tree_exception(page)) {
1978 if (radix_tree_deref_retry(page)) {
1979 slot = radix_tree_iter_retry(&iter);
1980 continue;
1981 }
1982
1983
1984
1985
1986
1987
1988 goto export;
1989 }
1990
1991 head = compound_head(page);
1992 if (!page_cache_get_speculative(head))
1993 goto repeat;
1994
1995
1996 if (compound_head(page) != head) {
1997 put_page(head);
1998 goto repeat;
1999 }
2000
2001
2002 if (unlikely(page != *slot)) {
2003 put_page(head);
2004 goto repeat;
2005 }
2006export:
2007 indices[ret] = iter.index;
2008 entries[ret] = page;
2009 if (++ret == nr_entries)
2010 break;
2011 }
2012 rcu_read_unlock();
2013 return ret;
2014}
2015EXPORT_SYMBOL(find_get_entries_tag);
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032static void shrink_readahead_size_eio(struct file *filp,
2033 struct file_ra_state *ra)
2034{
2035 ra->ra_pages /= 4;
2036}
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050static ssize_t generic_file_buffered_read(struct kiocb *iocb,
2051 struct iov_iter *iter, ssize_t written)
2052{
2053 struct file *filp = iocb->ki_filp;
2054 struct address_space *mapping = filp->f_mapping;
2055 struct inode *inode = mapping->host;
2056 struct file_ra_state *ra = &filp->f_ra;
2057 loff_t *ppos = &iocb->ki_pos;
2058 pgoff_t index;
2059 pgoff_t last_index;
2060 pgoff_t prev_index;
2061 unsigned long offset;
2062 unsigned int prev_offset;
2063 int error = 0;
2064
2065 if (unlikely(*ppos >= inode->i_sb->s_maxbytes))
2066 return 0;
2067 iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
2068
2069 index = *ppos >> PAGE_SHIFT;
2070 prev_index = ra->prev_pos >> PAGE_SHIFT;
2071 prev_offset = ra->prev_pos & (PAGE_SIZE-1);
2072 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT;
2073 offset = *ppos & ~PAGE_MASK;
2074
2075 for (;;) {
2076 struct page *page;
2077 pgoff_t end_index;
2078 loff_t isize;
2079 unsigned long nr, ret;
2080
2081 cond_resched();
2082find_page:
2083 if (fatal_signal_pending(current)) {
2084 error = -EINTR;
2085 goto out;
2086 }
2087
2088 page = find_get_page(mapping, index);
2089 if (!page) {
2090 if (iocb->ki_flags & IOCB_NOWAIT)
2091 goto would_block;
2092 page_cache_sync_readahead(mapping,
2093 ra, filp,
2094 index, last_index - index);
2095 page = find_get_page(mapping, index);
2096 if (unlikely(page == NULL))
2097 goto no_cached_page;
2098 }
2099 if (PageReadahead(page)) {
2100 page_cache_async_readahead(mapping,
2101 ra, filp, page,
2102 index, last_index - index);
2103 }
2104 if (!PageUptodate(page)) {
2105 if (iocb->ki_flags & IOCB_NOWAIT) {
2106 put_page(page);
2107 goto would_block;
2108 }
2109
2110
2111
2112
2113
2114
2115 error = wait_on_page_locked_killable(page);
2116 if (unlikely(error))
2117 goto readpage_error;
2118 if (PageUptodate(page))
2119 goto page_ok;
2120
2121 if (inode->i_blkbits == PAGE_SHIFT ||
2122 !mapping->a_ops->is_partially_uptodate)
2123 goto page_not_up_to_date;
2124
2125 if (unlikely(iter->type & ITER_PIPE))
2126 goto page_not_up_to_date;
2127 if (!trylock_page(page))
2128 goto page_not_up_to_date;
2129
2130 if (!page->mapping)
2131 goto page_not_up_to_date_locked;
2132 if (!mapping->a_ops->is_partially_uptodate(page,
2133 offset, iter->count))
2134 goto page_not_up_to_date_locked;
2135 unlock_page(page);
2136 }
2137page_ok:
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147 isize = i_size_read(inode);
2148 end_index = (isize - 1) >> PAGE_SHIFT;
2149 if (unlikely(!isize || index > end_index)) {
2150 put_page(page);
2151 goto out;
2152 }
2153
2154
2155 nr = PAGE_SIZE;
2156 if (index == end_index) {
2157 nr = ((isize - 1) & ~PAGE_MASK) + 1;
2158 if (nr <= offset) {
2159 put_page(page);
2160 goto out;
2161 }
2162 }
2163 nr = nr - offset;
2164
2165
2166
2167
2168
2169 if (mapping_writably_mapped(mapping))
2170 flush_dcache_page(page);
2171
2172
2173
2174
2175
2176 if (prev_index != index || offset != prev_offset)
2177 mark_page_accessed(page);
2178 prev_index = index;
2179
2180
2181
2182
2183
2184
2185 ret = copy_page_to_iter(page, offset, nr, iter);
2186 offset += ret;
2187 index += offset >> PAGE_SHIFT;
2188 offset &= ~PAGE_MASK;
2189 prev_offset = offset;
2190
2191 put_page(page);
2192 written += ret;
2193 if (!iov_iter_count(iter))
2194 goto out;
2195 if (ret < nr) {
2196 error = -EFAULT;
2197 goto out;
2198 }
2199 continue;
2200
2201page_not_up_to_date:
2202
2203 error = lock_page_killable(page);
2204 if (unlikely(error))
2205 goto readpage_error;
2206
2207page_not_up_to_date_locked:
2208
2209 if (!page->mapping) {
2210 unlock_page(page);
2211 put_page(page);
2212 continue;
2213 }
2214
2215
2216 if (PageUptodate(page)) {
2217 unlock_page(page);
2218 goto page_ok;
2219 }
2220
2221readpage:
2222
2223
2224
2225
2226
2227 ClearPageError(page);
2228
2229 error = mapping->a_ops->readpage(filp, page);
2230
2231 if (unlikely(error)) {
2232 if (error == AOP_TRUNCATED_PAGE) {
2233 put_page(page);
2234 error = 0;
2235 goto find_page;
2236 }
2237 goto readpage_error;
2238 }
2239
2240 if (!PageUptodate(page)) {
2241 error = lock_page_killable(page);
2242 if (unlikely(error))
2243 goto readpage_error;
2244 if (!PageUptodate(page)) {
2245 if (page->mapping == NULL) {
2246
2247
2248
2249 unlock_page(page);
2250 put_page(page);
2251 goto find_page;
2252 }
2253 unlock_page(page);
2254 shrink_readahead_size_eio(filp, ra);
2255 error = -EIO;
2256 goto readpage_error;
2257 }
2258 unlock_page(page);
2259 }
2260
2261 goto page_ok;
2262
2263readpage_error:
2264
2265 put_page(page);
2266 goto out;
2267
2268no_cached_page:
2269
2270
2271
2272
2273 page = page_cache_alloc(mapping);
2274 if (!page) {
2275 error = -ENOMEM;
2276 goto out;
2277 }
2278 error = add_to_page_cache_lru(page, mapping, index,
2279 mapping_gfp_constraint(mapping, GFP_KERNEL));
2280 if (error) {
2281 put_page(page);
2282 if (error == -EEXIST) {
2283 error = 0;
2284 goto find_page;
2285 }
2286 goto out;
2287 }
2288 goto readpage;
2289 }
2290
2291would_block:
2292 error = -EAGAIN;
2293out:
2294 ra->prev_pos = prev_index;
2295 ra->prev_pos <<= PAGE_SHIFT;
2296 ra->prev_pos |= prev_offset;
2297
2298 *ppos = ((loff_t)index << PAGE_SHIFT) + offset;
2299 file_accessed(filp);
2300 return written ? written : error;
2301}
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311ssize_t
2312generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
2313{
2314 size_t count = iov_iter_count(iter);
2315 ssize_t retval = 0;
2316
2317 if (!count)
2318 goto out;
2319
2320 if (iocb->ki_flags & IOCB_DIRECT) {
2321 struct file *file = iocb->ki_filp;
2322 struct address_space *mapping = file->f_mapping;
2323 struct inode *inode = mapping->host;
2324 loff_t size;
2325
2326 size = i_size_read(inode);
2327 if (iocb->ki_flags & IOCB_NOWAIT) {
2328 if (filemap_range_has_page(mapping, iocb->ki_pos,
2329 iocb->ki_pos + count - 1))
2330 return -EAGAIN;
2331 } else {
2332 retval = filemap_write_and_wait_range(mapping,
2333 iocb->ki_pos,
2334 iocb->ki_pos + count - 1);
2335 if (retval < 0)
2336 goto out;
2337 }
2338
2339 file_accessed(file);
2340
2341 retval = mapping->a_ops->direct_IO(iocb, iter);
2342 if (retval >= 0) {
2343 iocb->ki_pos += retval;
2344 count -= retval;
2345 }
2346 iov_iter_revert(iter, count - iov_iter_count(iter));
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357 if (retval < 0 || !count || iocb->ki_pos >= size ||
2358 IS_DAX(inode))
2359 goto out;
2360 }
2361
2362 retval = generic_file_buffered_read(iocb, iter, retval);
2363out:
2364 return retval;
2365}
2366EXPORT_SYMBOL(generic_file_read_iter);
2367
2368#ifdef CONFIG_MMU
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378static int page_cache_read(struct file *file, pgoff_t offset, gfp_t gfp_mask)
2379{
2380 struct address_space *mapping = file->f_mapping;
2381 struct page *page;
2382 int ret;
2383
2384 do {
2385 page = __page_cache_alloc(gfp_mask);
2386 if (!page)
2387 return -ENOMEM;
2388
2389 ret = add_to_page_cache_lru(page, mapping, offset, gfp_mask);
2390 if (ret == 0)
2391 ret = mapping->a_ops->readpage(file, page);
2392 else if (ret == -EEXIST)
2393 ret = 0;
2394
2395 put_page(page);
2396
2397 } while (ret == AOP_TRUNCATED_PAGE);
2398
2399 return ret;
2400}
2401
2402#define MMAP_LOTSAMISS (100)
2403
2404
2405
2406
2407
2408static void do_sync_mmap_readahead(struct vm_area_struct *vma,
2409 struct file_ra_state *ra,
2410 struct file *file,
2411 pgoff_t offset)
2412{
2413 struct address_space *mapping = file->f_mapping;
2414
2415
2416 if (vma->vm_flags & VM_RAND_READ)
2417 return;
2418 if (!ra->ra_pages)
2419 return;
2420
2421 if (vma->vm_flags & VM_SEQ_READ) {
2422 page_cache_sync_readahead(mapping, ra, file, offset,
2423 ra->ra_pages);
2424 return;
2425 }
2426
2427
2428 if (ra->mmap_miss < MMAP_LOTSAMISS * 10)
2429 ra->mmap_miss++;
2430
2431
2432
2433
2434
2435 if (ra->mmap_miss > MMAP_LOTSAMISS)
2436 return;
2437
2438
2439
2440
2441 ra->start = max_t(long, 0, offset - ra->ra_pages / 2);
2442 ra->size = ra->ra_pages;
2443 ra->async_size = ra->ra_pages / 4;
2444 ra_submit(ra, mapping, file);
2445}
2446
2447
2448
2449
2450
2451static void do_async_mmap_readahead(struct vm_area_struct *vma,
2452 struct file_ra_state *ra,
2453 struct file *file,
2454 struct page *page,
2455 pgoff_t offset)
2456{
2457 struct address_space *mapping = file->f_mapping;
2458
2459
2460 if (vma->vm_flags & VM_RAND_READ)
2461 return;
2462 if (ra->mmap_miss > 0)
2463 ra->mmap_miss--;
2464 if (PageReadahead(page))
2465 page_cache_async_readahead(mapping, ra, file,
2466 page, offset, ra->ra_pages);
2467}
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492vm_fault_t filemap_fault(struct vm_fault *vmf)
2493{
2494 int error;
2495 struct file *file = vmf->vma->vm_file;
2496 struct address_space *mapping = file->f_mapping;
2497 struct file_ra_state *ra = &file->f_ra;
2498 struct inode *inode = mapping->host;
2499 pgoff_t offset = vmf->pgoff;
2500 pgoff_t max_off;
2501 struct page *page;
2502 vm_fault_t ret = 0;
2503
2504 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2505 if (unlikely(offset >= max_off))
2506 return VM_FAULT_SIGBUS;
2507
2508
2509
2510
2511 page = find_get_page(mapping, offset);
2512 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
2513
2514
2515
2516
2517 do_async_mmap_readahead(vmf->vma, ra, file, page, offset);
2518 } else if (!page) {
2519
2520 do_sync_mmap_readahead(vmf->vma, ra, file, offset);
2521 count_vm_event(PGMAJFAULT);
2522 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
2523 ret = VM_FAULT_MAJOR;
2524retry_find:
2525 page = find_get_page(mapping, offset);
2526 if (!page)
2527 goto no_cached_page;
2528 }
2529
2530 if (!lock_page_or_retry(page, vmf->vma->vm_mm, vmf->flags)) {
2531 put_page(page);
2532 return ret | VM_FAULT_RETRY;
2533 }
2534
2535
2536 if (unlikely(page->mapping != mapping)) {
2537 unlock_page(page);
2538 put_page(page);
2539 goto retry_find;
2540 }
2541 VM_BUG_ON_PAGE(page->index != offset, page);
2542
2543
2544
2545
2546
2547 if (unlikely(!PageUptodate(page)))
2548 goto page_not_uptodate;
2549
2550
2551
2552
2553
2554 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2555 if (unlikely(offset >= max_off)) {
2556 unlock_page(page);
2557 put_page(page);
2558 return VM_FAULT_SIGBUS;
2559 }
2560
2561 vmf->page = page;
2562 return ret | VM_FAULT_LOCKED;
2563
2564no_cached_page:
2565
2566
2567
2568
2569 error = page_cache_read(file, offset, vmf->gfp_mask);
2570
2571
2572
2573
2574
2575
2576 if (error >= 0)
2577 goto retry_find;
2578
2579
2580
2581
2582
2583
2584 if (error == -ENOMEM)
2585 return VM_FAULT_OOM;
2586 return VM_FAULT_SIGBUS;
2587
2588page_not_uptodate:
2589
2590
2591
2592
2593
2594
2595 ClearPageError(page);
2596 error = mapping->a_ops->readpage(file, page);
2597 if (!error) {
2598 wait_on_page_locked(page);
2599 if (!PageUptodate(page))
2600 error = -EIO;
2601 }
2602 put_page(page);
2603
2604 if (!error || error == AOP_TRUNCATED_PAGE)
2605 goto retry_find;
2606
2607
2608 shrink_readahead_size_eio(file, ra);
2609 return VM_FAULT_SIGBUS;
2610}
2611EXPORT_SYMBOL(filemap_fault);
2612
2613void filemap_map_pages(struct vm_fault *vmf,
2614 pgoff_t start_pgoff, pgoff_t end_pgoff)
2615{
2616 struct radix_tree_iter iter;
2617 void **slot;
2618 struct file *file = vmf->vma->vm_file;
2619 struct address_space *mapping = file->f_mapping;
2620 pgoff_t last_pgoff = start_pgoff;
2621 unsigned long max_idx;
2622 struct page *head, *page;
2623
2624 rcu_read_lock();
2625 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start_pgoff) {
2626 if (iter.index > end_pgoff)
2627 break;
2628repeat:
2629 page = radix_tree_deref_slot(slot);
2630 if (unlikely(!page))
2631 goto next;
2632 if (radix_tree_exception(page)) {
2633 if (radix_tree_deref_retry(page)) {
2634 slot = radix_tree_iter_retry(&iter);
2635 continue;
2636 }
2637 goto next;
2638 }
2639
2640 head = compound_head(page);
2641 if (!page_cache_get_speculative(head))
2642 goto repeat;
2643
2644
2645 if (compound_head(page) != head) {
2646 put_page(head);
2647 goto repeat;
2648 }
2649
2650
2651 if (unlikely(page != *slot)) {
2652 put_page(head);
2653 goto repeat;
2654 }
2655
2656 if (!PageUptodate(page) ||
2657 PageReadahead(page) ||
2658 PageHWPoison(page))
2659 goto skip;
2660 if (!trylock_page(page))
2661 goto skip;
2662
2663 if (page->mapping != mapping || !PageUptodate(page))
2664 goto unlock;
2665
2666 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
2667 if (page->index >= max_idx)
2668 goto unlock;
2669
2670 if (file->f_ra.mmap_miss > 0)
2671 file->f_ra.mmap_miss--;
2672
2673 vmf->address += (iter.index - last_pgoff) << PAGE_SHIFT;
2674 if (vmf->pte)
2675 vmf->pte += iter.index - last_pgoff;
2676 last_pgoff = iter.index;
2677 if (alloc_set_pte(vmf, NULL, page))
2678 goto unlock;
2679 unlock_page(page);
2680 goto next;
2681unlock:
2682 unlock_page(page);
2683skip:
2684 put_page(page);
2685next:
2686
2687 if (pmd_trans_huge(*vmf->pmd))
2688 break;
2689 if (iter.index == end_pgoff)
2690 break;
2691 }
2692 rcu_read_unlock();
2693}
2694EXPORT_SYMBOL(filemap_map_pages);
2695
2696vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf)
2697{
2698 struct page *page = vmf->page;
2699 struct inode *inode = file_inode(vmf->vma->vm_file);
2700 vm_fault_t ret = VM_FAULT_LOCKED;
2701
2702 sb_start_pagefault(inode->i_sb);
2703 file_update_time(vmf->vma->vm_file);
2704 lock_page(page);
2705 if (page->mapping != inode->i_mapping) {
2706 unlock_page(page);
2707 ret = VM_FAULT_NOPAGE;
2708 goto out;
2709 }
2710
2711
2712
2713
2714
2715 set_page_dirty(page);
2716 wait_for_stable_page(page);
2717out:
2718 sb_end_pagefault(inode->i_sb);
2719 return ret;
2720}
2721
2722const struct vm_operations_struct generic_file_vm_ops = {
2723 .fault = filemap_fault,
2724 .map_pages = filemap_map_pages,
2725 .page_mkwrite = filemap_page_mkwrite,
2726};
2727
2728
2729
2730int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2731{
2732 struct address_space *mapping = file->f_mapping;
2733
2734 if (!mapping->a_ops->readpage)
2735 return -ENOEXEC;
2736 file_accessed(file);
2737 vma->vm_ops = &generic_file_vm_ops;
2738 return 0;
2739}
2740
2741
2742
2743
2744int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
2745{
2746 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2747 return -EINVAL;
2748 return generic_file_mmap(file, vma);
2749}
2750#else
2751int filemap_page_mkwrite(struct vm_fault *vmf)
2752{
2753 return -ENOSYS;
2754}
2755int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2756{
2757 return -ENOSYS;
2758}
2759int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
2760{
2761 return -ENOSYS;
2762}
2763#endif
2764
2765EXPORT_SYMBOL(filemap_page_mkwrite);
2766EXPORT_SYMBOL(generic_file_mmap);
2767EXPORT_SYMBOL(generic_file_readonly_mmap);
2768
2769static struct page *wait_on_page_read(struct page *page)
2770{
2771 if (!IS_ERR(page)) {
2772 wait_on_page_locked(page);
2773 if (!PageUptodate(page)) {
2774 put_page(page);
2775 page = ERR_PTR(-EIO);
2776 }
2777 }
2778 return page;
2779}
2780
2781static struct page *do_read_cache_page(struct address_space *mapping,
2782 pgoff_t index,
2783 int (*filler)(void *, struct page *),
2784 void *data,
2785 gfp_t gfp)
2786{
2787 struct page *page;
2788 int err;
2789repeat:
2790 page = find_get_page(mapping, index);
2791 if (!page) {
2792 page = __page_cache_alloc(gfp);
2793 if (!page)
2794 return ERR_PTR(-ENOMEM);
2795 err = add_to_page_cache_lru(page, mapping, index, gfp);
2796 if (unlikely(err)) {
2797 put_page(page);
2798 if (err == -EEXIST)
2799 goto repeat;
2800
2801 return ERR_PTR(err);
2802 }
2803
2804filler:
2805 err = filler(data, page);
2806 if (err < 0) {
2807 put_page(page);
2808 return ERR_PTR(err);
2809 }
2810
2811 page = wait_on_page_read(page);
2812 if (IS_ERR(page))
2813 return page;
2814 goto out;
2815 }
2816 if (PageUptodate(page))
2817 goto out;
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850 wait_on_page_locked(page);
2851 if (PageUptodate(page))
2852 goto out;
2853
2854
2855 lock_page(page);
2856
2857
2858 if (!page->mapping) {
2859 unlock_page(page);
2860 put_page(page);
2861 goto repeat;
2862 }
2863
2864
2865 if (PageUptodate(page)) {
2866 unlock_page(page);
2867 goto out;
2868 }
2869 goto filler;
2870
2871out:
2872 mark_page_accessed(page);
2873 return page;
2874}
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888struct page *read_cache_page(struct address_space *mapping,
2889 pgoff_t index,
2890 int (*filler)(void *, struct page *),
2891 void *data)
2892{
2893 return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
2894}
2895EXPORT_SYMBOL(read_cache_page);
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908struct page *read_cache_page_gfp(struct address_space *mapping,
2909 pgoff_t index,
2910 gfp_t gfp)
2911{
2912 filler_t *filler = (filler_t *)mapping->a_ops->readpage;
2913
2914 return do_read_cache_page(mapping, index, filler, NULL, gfp);
2915}
2916EXPORT_SYMBOL(read_cache_page_gfp);
2917
2918
2919
2920
2921
2922
2923
2924
2925inline ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
2926{
2927 struct file *file = iocb->ki_filp;
2928 struct inode *inode = file->f_mapping->host;
2929 unsigned long limit = rlimit(RLIMIT_FSIZE);
2930 loff_t pos;
2931
2932 if (!iov_iter_count(from))
2933 return 0;
2934
2935
2936 if (iocb->ki_flags & IOCB_APPEND)
2937 iocb->ki_pos = i_size_read(inode);
2938
2939 pos = iocb->ki_pos;
2940
2941 if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT))
2942 return -EINVAL;
2943
2944 if (limit != RLIM_INFINITY) {
2945 if (iocb->ki_pos >= limit) {
2946 send_sig(SIGXFSZ, current, 0);
2947 return -EFBIG;
2948 }
2949 iov_iter_truncate(from, limit - (unsigned long)pos);
2950 }
2951
2952
2953
2954
2955 if (unlikely(pos + iov_iter_count(from) > MAX_NON_LFS &&
2956 !(file->f_flags & O_LARGEFILE))) {
2957 if (pos >= MAX_NON_LFS)
2958 return -EFBIG;
2959 iov_iter_truncate(from, MAX_NON_LFS - (unsigned long)pos);
2960 }
2961
2962
2963
2964
2965
2966
2967
2968
2969 if (unlikely(pos >= inode->i_sb->s_maxbytes))
2970 return -EFBIG;
2971
2972 iov_iter_truncate(from, inode->i_sb->s_maxbytes - pos);
2973 return iov_iter_count(from);
2974}
2975EXPORT_SYMBOL(generic_write_checks);
2976
2977int pagecache_write_begin(struct file *file, struct address_space *mapping,
2978 loff_t pos, unsigned len, unsigned flags,
2979 struct page **pagep, void **fsdata)
2980{
2981 const struct address_space_operations *aops = mapping->a_ops;
2982
2983 return aops->write_begin(file, mapping, pos, len, flags,
2984 pagep, fsdata);
2985}
2986EXPORT_SYMBOL(pagecache_write_begin);
2987
2988int pagecache_write_end(struct file *file, struct address_space *mapping,
2989 loff_t pos, unsigned len, unsigned copied,
2990 struct page *page, void *fsdata)
2991{
2992 const struct address_space_operations *aops = mapping->a_ops;
2993
2994 return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
2995}
2996EXPORT_SYMBOL(pagecache_write_end);
2997
2998ssize_t
2999generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
3000{
3001 struct file *file = iocb->ki_filp;
3002 struct address_space *mapping = file->f_mapping;
3003 struct inode *inode = mapping->host;
3004 loff_t pos = iocb->ki_pos;
3005 ssize_t written;
3006 size_t write_len;
3007 pgoff_t end;
3008
3009 write_len = iov_iter_count(from);
3010 end = (pos + write_len - 1) >> PAGE_SHIFT;
3011
3012 if (iocb->ki_flags & IOCB_NOWAIT) {
3013
3014 if (filemap_range_has_page(inode->i_mapping, pos,
3015 pos + iov_iter_count(from)))
3016 return -EAGAIN;
3017 } else {
3018 written = filemap_write_and_wait_range(mapping, pos,
3019 pos + write_len - 1);
3020 if (written)
3021 goto out;
3022 }
3023
3024
3025
3026
3027
3028
3029
3030 written = invalidate_inode_pages2_range(mapping,
3031 pos >> PAGE_SHIFT, end);
3032
3033
3034
3035
3036 if (written) {
3037 if (written == -EBUSY)
3038 return 0;
3039 goto out;
3040 }
3041
3042 written = mapping->a_ops->direct_IO(iocb, from);
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057 if (mapping->nrpages)
3058 invalidate_inode_pages2_range(mapping,
3059 pos >> PAGE_SHIFT, end);
3060
3061 if (written > 0) {
3062 pos += written;
3063 write_len -= written;
3064 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
3065 i_size_write(inode, pos);
3066 mark_inode_dirty(inode);
3067 }
3068 iocb->ki_pos = pos;
3069 }
3070 iov_iter_revert(from, write_len - iov_iter_count(from));
3071out:
3072 return written;
3073}
3074EXPORT_SYMBOL(generic_file_direct_write);
3075
3076
3077
3078
3079
3080struct page *grab_cache_page_write_begin(struct address_space *mapping,
3081 pgoff_t index, unsigned flags)
3082{
3083 struct page *page;
3084 int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT;
3085
3086 if (flags & AOP_FLAG_NOFS)
3087 fgp_flags |= FGP_NOFS;
3088
3089 page = pagecache_get_page(mapping, index, fgp_flags,
3090 mapping_gfp_mask(mapping));
3091 if (page)
3092 wait_for_stable_page(page);
3093
3094 return page;
3095}
3096EXPORT_SYMBOL(grab_cache_page_write_begin);
3097
3098ssize_t generic_perform_write(struct file *file,
3099 struct iov_iter *i, loff_t pos)
3100{
3101 struct address_space *mapping = file->f_mapping;
3102 const struct address_space_operations *a_ops = mapping->a_ops;
3103 long status = 0;
3104 ssize_t written = 0;
3105 unsigned int flags = 0;
3106
3107 do {
3108 struct page *page;
3109 unsigned long offset;
3110 unsigned long bytes;
3111 size_t copied;
3112 void *fsdata;
3113
3114 offset = (pos & (PAGE_SIZE - 1));
3115 bytes = min_t(unsigned long, PAGE_SIZE - offset,
3116 iov_iter_count(i));
3117
3118again:
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129 if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
3130 status = -EFAULT;
3131 break;
3132 }
3133
3134 if (fatal_signal_pending(current)) {
3135 status = -EINTR;
3136 break;
3137 }
3138
3139 status = a_ops->write_begin(file, mapping, pos, bytes, flags,
3140 &page, &fsdata);
3141 if (unlikely(status < 0))
3142 break;
3143
3144 if (mapping_writably_mapped(mapping))
3145 flush_dcache_page(page);
3146
3147 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
3148 flush_dcache_page(page);
3149
3150 status = a_ops->write_end(file, mapping, pos, bytes, copied,
3151 page, fsdata);
3152 if (unlikely(status < 0))
3153 break;
3154 copied = status;
3155
3156 cond_resched();
3157
3158 iov_iter_advance(i, copied);
3159 if (unlikely(copied == 0)) {
3160
3161
3162
3163
3164
3165
3166
3167
3168 bytes = min_t(unsigned long, PAGE_SIZE - offset,
3169 iov_iter_single_seg_count(i));
3170 goto again;
3171 }
3172 pos += copied;
3173 written += copied;
3174
3175 balance_dirty_pages_ratelimited(mapping);
3176 } while (iov_iter_count(i));
3177
3178 return written ? written : status;
3179}
3180EXPORT_SYMBOL(generic_perform_write);
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3200{
3201 struct file *file = iocb->ki_filp;
3202 struct address_space * mapping = file->f_mapping;
3203 struct inode *inode = mapping->host;
3204 ssize_t written = 0;
3205 ssize_t err;
3206 ssize_t status;
3207
3208
3209 current->backing_dev_info = inode_to_bdi(inode);
3210 err = file_remove_privs(file);
3211 if (err)
3212 goto out;
3213
3214 err = file_update_time(file);
3215 if (err)
3216 goto out;
3217
3218 if (iocb->ki_flags & IOCB_DIRECT) {
3219 loff_t pos, endbyte;
3220
3221 written = generic_file_direct_write(iocb, from);
3222
3223
3224
3225
3226
3227
3228
3229 if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
3230 goto out;
3231
3232 status = generic_perform_write(file, from, pos = iocb->ki_pos);
3233
3234
3235
3236
3237
3238
3239
3240 if (unlikely(status < 0)) {
3241 err = status;
3242 goto out;
3243 }
3244
3245
3246
3247
3248
3249 endbyte = pos + status - 1;
3250 err = filemap_write_and_wait_range(mapping, pos, endbyte);
3251 if (err == 0) {
3252 iocb->ki_pos = endbyte + 1;
3253 written += status;
3254 invalidate_mapping_pages(mapping,
3255 pos >> PAGE_SHIFT,
3256 endbyte >> PAGE_SHIFT);
3257 } else {
3258
3259
3260
3261
3262 }
3263 } else {
3264 written = generic_perform_write(file, from, iocb->ki_pos);
3265 if (likely(written > 0))
3266 iocb->ki_pos += written;
3267 }
3268out:
3269 current->backing_dev_info = NULL;
3270 return written ? written : err;
3271}
3272EXPORT_SYMBOL(__generic_file_write_iter);
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
3284{
3285 struct file *file = iocb->ki_filp;
3286 struct inode *inode = file->f_mapping->host;
3287 ssize_t ret;
3288
3289 inode_lock(inode);
3290 ret = generic_write_checks(iocb, from);
3291 if (ret > 0)
3292 ret = __generic_file_write_iter(iocb, from);
3293 inode_unlock(inode);
3294
3295 if (ret > 0)
3296 ret = generic_write_sync(iocb, ret);
3297 return ret;
3298}
3299EXPORT_SYMBOL(generic_file_write_iter);
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318int try_to_release_page(struct page *page, gfp_t gfp_mask)
3319{
3320 struct address_space * const mapping = page->mapping;
3321
3322 BUG_ON(!PageLocked(page));
3323 if (PageWriteback(page))
3324 return 0;
3325
3326 if (mapping && mapping->a_ops->releasepage)
3327 return mapping->a_ops->releasepage(page, gfp_mask);
3328 return try_to_free_buffers(page);
3329}
3330
3331EXPORT_SYMBOL(try_to_release_page);
3332