1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/backing-dev.h>
13#include <linux/dax.h>
14#include <linux/gfp.h>
15#include <linux/mm.h>
16#include <linux/swap.h>
17#include <linux/export.h>
18#include <linux/pagemap.h>
19#include <linux/highmem.h>
20#include <linux/pagevec.h>
21#include <linux/task_io_accounting_ops.h>
22#include <linux/buffer_head.h>
23
24#include <linux/shmem_fs.h>
25#include <linux/cleancache.h>
26#include <linux/rmap.h>
27#include "internal.h"
28
29
30
31
32
33
34static inline void __clear_shadow_entry(struct address_space *mapping,
35 pgoff_t index, void *entry)
36{
37 XA_STATE(xas, &mapping->i_pages, index);
38
39 xas_set_update(&xas, workingset_update_node);
40 if (xas_load(&xas) != entry)
41 return;
42 xas_store(&xas, NULL);
43 mapping->nrexceptional--;
44}
45
46static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
47 void *entry)
48{
49 xa_lock_irq(&mapping->i_pages);
50 __clear_shadow_entry(mapping, index, entry);
51 xa_unlock_irq(&mapping->i_pages);
52}
53
54
55
56
57
58
59static void truncate_exceptional_pvec_entries(struct address_space *mapping,
60 struct pagevec *pvec, pgoff_t *indices,
61 pgoff_t end)
62{
63 int i, j;
64 bool dax, lock;
65
66
67 if (shmem_mapping(mapping))
68 return;
69
70 for (j = 0; j < pagevec_count(pvec); j++)
71 if (xa_is_value(pvec->pages[j]))
72 break;
73
74 if (j == pagevec_count(pvec))
75 return;
76
77 dax = dax_mapping(mapping);
78 lock = !dax && indices[j] < end;
79 if (lock)
80 xa_lock_irq(&mapping->i_pages);
81
82 for (i = j; i < pagevec_count(pvec); i++) {
83 struct page *page = pvec->pages[i];
84 pgoff_t index = indices[i];
85
86 if (!xa_is_value(page)) {
87 pvec->pages[j++] = page;
88 continue;
89 }
90
91 if (index >= end)
92 continue;
93
94 if (unlikely(dax)) {
95 dax_delete_mapping_entry(mapping, index);
96 continue;
97 }
98
99 __clear_shadow_entry(mapping, index, page);
100 }
101
102 if (lock)
103 xa_unlock_irq(&mapping->i_pages);
104 pvec->nr = j;
105}
106
107
108
109
110
111static int invalidate_exceptional_entry(struct address_space *mapping,
112 pgoff_t index, void *entry)
113{
114
115 if (shmem_mapping(mapping) || dax_mapping(mapping))
116 return 1;
117 clear_shadow_entry(mapping, index, entry);
118 return 1;
119}
120
121
122
123
124
125static int invalidate_exceptional_entry2(struct address_space *mapping,
126 pgoff_t index, void *entry)
127{
128
129 if (shmem_mapping(mapping))
130 return 1;
131 if (dax_mapping(mapping))
132 return dax_invalidate_mapping_entry_sync(mapping, index);
133 clear_shadow_entry(mapping, index, entry);
134 return 1;
135}
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152void do_invalidatepage(struct page *page, unsigned int offset,
153 unsigned int length)
154{
155 void (*invalidatepage)(struct page *, unsigned int, unsigned int);
156
157 invalidatepage = page->mapping->a_ops->invalidatepage;
158#ifdef CONFIG_BLOCK
159 if (!invalidatepage)
160 invalidatepage = block_invalidatepage;
161#endif
162 if (invalidatepage)
163 (*invalidatepage)(page, offset, length);
164}
165
166
167
168
169
170
171
172
173
174
175
176static void
177truncate_cleanup_page(struct address_space *mapping, struct page *page)
178{
179 if (page_mapped(page)) {
180 unsigned int nr = thp_nr_pages(page);
181 unmap_mapping_pages(mapping, page->index, nr, false);
182 }
183
184 if (page_has_private(page))
185 do_invalidatepage(page, 0, thp_size(page));
186
187
188
189
190
191
192 cancel_dirty_page(page);
193 ClearPageMappedToDisk(page);
194}
195
196
197
198
199
200
201
202
203
204static int
205invalidate_complete_page(struct address_space *mapping, struct page *page)
206{
207 int ret;
208
209 if (page->mapping != mapping)
210 return 0;
211
212 if (page_has_private(page) && !try_to_release_page(page, 0))
213 return 0;
214
215 ret = remove_mapping(mapping, page);
216
217 return ret;
218}
219
220int truncate_inode_page(struct address_space *mapping, struct page *page)
221{
222 VM_BUG_ON_PAGE(PageTail(page), page);
223
224 if (page->mapping != mapping)
225 return -EIO;
226
227 truncate_cleanup_page(mapping, page);
228 delete_from_page_cache(page);
229 return 0;
230}
231
232
233
234
235int generic_error_remove_page(struct address_space *mapping, struct page *page)
236{
237 if (!mapping)
238 return -EINVAL;
239
240
241
242
243 if (!S_ISREG(mapping->host->i_mode))
244 return -EIO;
245 return truncate_inode_page(mapping, page);
246}
247EXPORT_SYMBOL(generic_error_remove_page);
248
249
250
251
252
253
254
255int invalidate_inode_page(struct page *page)
256{
257 struct address_space *mapping = page_mapping(page);
258 if (!mapping)
259 return 0;
260 if (PageDirty(page) || PageWriteback(page))
261 return 0;
262 if (page_mapped(page))
263 return 0;
264 return invalidate_complete_page(mapping, page);
265}
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291void truncate_inode_pages_range(struct address_space *mapping,
292 loff_t lstart, loff_t lend)
293{
294 pgoff_t start;
295 pgoff_t end;
296 unsigned int partial_start;
297 unsigned int partial_end;
298 struct pagevec pvec;
299 pgoff_t indices[PAGEVEC_SIZE];
300 pgoff_t index;
301 int i;
302
303 if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
304 goto out;
305
306
307 partial_start = lstart & (PAGE_SIZE - 1);
308 partial_end = (lend + 1) & (PAGE_SIZE - 1);
309
310
311
312
313
314
315
316 start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
317 if (lend == -1)
318
319
320
321
322
323 end = -1;
324 else
325 end = (lend + 1) >> PAGE_SHIFT;
326
327 pagevec_init(&pvec);
328 index = start;
329 while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
330 min(end - index, (pgoff_t)PAGEVEC_SIZE),
331 indices)) {
332
333
334
335
336
337 struct pagevec locked_pvec;
338
339 pagevec_init(&locked_pvec);
340 for (i = 0; i < pagevec_count(&pvec); i++) {
341 struct page *page = pvec.pages[i];
342
343
344 index = indices[i];
345 if (index >= end)
346 break;
347
348 if (xa_is_value(page))
349 continue;
350
351 if (!trylock_page(page))
352 continue;
353 WARN_ON(page_to_index(page) != index);
354 if (PageWriteback(page)) {
355 unlock_page(page);
356 continue;
357 }
358 if (page->mapping != mapping) {
359 unlock_page(page);
360 continue;
361 }
362 pagevec_add(&locked_pvec, page);
363 }
364 for (i = 0; i < pagevec_count(&locked_pvec); i++)
365 truncate_cleanup_page(mapping, locked_pvec.pages[i]);
366 delete_from_page_cache_batch(mapping, &locked_pvec);
367 for (i = 0; i < pagevec_count(&locked_pvec); i++)
368 unlock_page(locked_pvec.pages[i]);
369 truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);
370 pagevec_release(&pvec);
371 cond_resched();
372 index++;
373 }
374 if (partial_start) {
375 struct page *page = find_lock_page(mapping, start - 1);
376 if (page) {
377 unsigned int top = PAGE_SIZE;
378 if (start > end) {
379
380 top = partial_end;
381 partial_end = 0;
382 }
383 wait_on_page_writeback(page);
384 zero_user_segment(page, partial_start, top);
385 cleancache_invalidate_page(mapping, page);
386 if (page_has_private(page))
387 do_invalidatepage(page, partial_start,
388 top - partial_start);
389 unlock_page(page);
390 put_page(page);
391 }
392 }
393 if (partial_end) {
394 struct page *page = find_lock_page(mapping, end);
395 if (page) {
396 wait_on_page_writeback(page);
397 zero_user_segment(page, 0, partial_end);
398 cleancache_invalidate_page(mapping, page);
399 if (page_has_private(page))
400 do_invalidatepage(page, 0,
401 partial_end);
402 unlock_page(page);
403 put_page(page);
404 }
405 }
406
407
408
409
410 if (start >= end)
411 goto out;
412
413 index = start;
414 for ( ; ; ) {
415 cond_resched();
416 if (!pagevec_lookup_entries(&pvec, mapping, index,
417 min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) {
418
419 if (index == start)
420 break;
421
422 index = start;
423 continue;
424 }
425 if (index == start && indices[0] >= end) {
426
427 pagevec_remove_exceptionals(&pvec);
428 pagevec_release(&pvec);
429 break;
430 }
431
432 for (i = 0; i < pagevec_count(&pvec); i++) {
433 struct page *page = pvec.pages[i];
434
435
436 index = indices[i];
437 if (index >= end) {
438
439 index = start - 1;
440 break;
441 }
442
443 if (xa_is_value(page))
444 continue;
445
446 lock_page(page);
447 WARN_ON(page_to_index(page) != index);
448 wait_on_page_writeback(page);
449 truncate_inode_page(mapping, page);
450 unlock_page(page);
451 }
452 truncate_exceptional_pvec_entries(mapping, &pvec, indices, end);
453 pagevec_release(&pvec);
454 index++;
455 }
456
457out:
458 cleancache_invalidate_inode(mapping);
459}
460EXPORT_SYMBOL(truncate_inode_pages_range);
461
462
463
464
465
466
467
468
469
470
471
472
473
474void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
475{
476 truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
477}
478EXPORT_SYMBOL(truncate_inode_pages);
479
480
481
482
483
484
485
486
487
488
489void truncate_inode_pages_final(struct address_space *mapping)
490{
491 unsigned long nrexceptional;
492 unsigned long nrpages;
493
494
495
496
497
498
499
500
501 mapping_set_exiting(mapping);
502
503
504
505
506
507
508 nrpages = mapping->nrpages;
509 smp_rmb();
510 nrexceptional = mapping->nrexceptional;
511
512 if (nrpages || nrexceptional) {
513
514
515
516
517
518
519 xa_lock_irq(&mapping->i_pages);
520 xa_unlock_irq(&mapping->i_pages);
521 }
522
523
524
525
526
527 truncate_inode_pages(mapping, 0);
528}
529EXPORT_SYMBOL(truncate_inode_pages_final);
530
531static unsigned long __invalidate_mapping_pages(struct address_space *mapping,
532 pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
533{
534 pgoff_t indices[PAGEVEC_SIZE];
535 struct pagevec pvec;
536 pgoff_t index = start;
537 unsigned long ret;
538 unsigned long count = 0;
539 int i;
540
541 pagevec_init(&pvec);
542 while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
543 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
544 indices)) {
545 for (i = 0; i < pagevec_count(&pvec); i++) {
546 struct page *page = pvec.pages[i];
547
548
549 index = indices[i];
550 if (index > end)
551 break;
552
553 if (xa_is_value(page)) {
554 invalidate_exceptional_entry(mapping, index,
555 page);
556 continue;
557 }
558
559 if (!trylock_page(page))
560 continue;
561
562 WARN_ON(page_to_index(page) != index);
563
564
565 if (PageTransTail(page)) {
566 unlock_page(page);
567 continue;
568 } else if (PageTransHuge(page)) {
569 index += HPAGE_PMD_NR - 1;
570 i += HPAGE_PMD_NR - 1;
571
572
573
574
575
576 if (index > end) {
577 unlock_page(page);
578 continue;
579 }
580
581
582 get_page(page);
583
584
585
586
587
588 pagevec_remove_exceptionals(&pvec);
589 pagevec_release(&pvec);
590 }
591
592 ret = invalidate_inode_page(page);
593 unlock_page(page);
594
595
596
597
598 if (!ret) {
599 deactivate_file_page(page);
600
601 if (nr_pagevec)
602 (*nr_pagevec)++;
603 }
604
605 if (PageTransHuge(page))
606 put_page(page);
607 count += ret;
608 }
609 pagevec_remove_exceptionals(&pvec);
610 pagevec_release(&pvec);
611 cond_resched();
612 index++;
613 }
614 return count;
615}
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632unsigned long invalidate_mapping_pages(struct address_space *mapping,
633 pgoff_t start, pgoff_t end)
634{
635 return __invalidate_mapping_pages(mapping, start, end, NULL);
636}
637EXPORT_SYMBOL(invalidate_mapping_pages);
638
639
640
641
642
643
644void invalidate_mapping_pagevec(struct address_space *mapping,
645 pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
646{
647 __invalidate_mapping_pages(mapping, start, end, nr_pagevec);
648}
649
650
651
652
653
654
655
656
657static int
658invalidate_complete_page2(struct address_space *mapping, struct page *page)
659{
660 unsigned long flags;
661
662 if (page->mapping != mapping)
663 return 0;
664
665 if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
666 return 0;
667
668 xa_lock_irqsave(&mapping->i_pages, flags);
669 if (PageDirty(page))
670 goto failed;
671
672 BUG_ON(page_has_private(page));
673 __delete_from_page_cache(page, NULL);
674 xa_unlock_irqrestore(&mapping->i_pages, flags);
675
676 if (mapping->a_ops->freepage)
677 mapping->a_ops->freepage(page);
678
679 put_page(page);
680 return 1;
681failed:
682 xa_unlock_irqrestore(&mapping->i_pages, flags);
683 return 0;
684}
685
686static int do_launder_page(struct address_space *mapping, struct page *page)
687{
688 if (!PageDirty(page))
689 return 0;
690 if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
691 return 0;
692 return mapping->a_ops->launder_page(page);
693}
694
695
696
697
698
699
700
701
702
703
704
705
706int invalidate_inode_pages2_range(struct address_space *mapping,
707 pgoff_t start, pgoff_t end)
708{
709 pgoff_t indices[PAGEVEC_SIZE];
710 struct pagevec pvec;
711 pgoff_t index;
712 int i;
713 int ret = 0;
714 int ret2 = 0;
715 int did_range_unmap = 0;
716
717 if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
718 goto out;
719
720 pagevec_init(&pvec);
721 index = start;
722 while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
723 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
724 indices)) {
725 for (i = 0; i < pagevec_count(&pvec); i++) {
726 struct page *page = pvec.pages[i];
727
728
729 index = indices[i];
730 if (index > end)
731 break;
732
733 if (xa_is_value(page)) {
734 if (!invalidate_exceptional_entry2(mapping,
735 index, page))
736 ret = -EBUSY;
737 continue;
738 }
739
740 lock_page(page);
741 WARN_ON(page_to_index(page) != index);
742 if (page->mapping != mapping) {
743 unlock_page(page);
744 continue;
745 }
746 wait_on_page_writeback(page);
747 if (page_mapped(page)) {
748 if (!did_range_unmap) {
749
750
751
752 unmap_mapping_pages(mapping, index,
753 (1 + end - index), false);
754 did_range_unmap = 1;
755 } else {
756
757
758
759 unmap_mapping_pages(mapping, index,
760 1, false);
761 }
762 }
763 BUG_ON(page_mapped(page));
764 ret2 = do_launder_page(mapping, page);
765 if (ret2 == 0) {
766 if (!invalidate_complete_page2(mapping, page))
767 ret2 = -EBUSY;
768 }
769 if (ret2 < 0)
770 ret = ret2;
771 unlock_page(page);
772 }
773 pagevec_remove_exceptionals(&pvec);
774 pagevec_release(&pvec);
775 cond_resched();
776 index++;
777 }
778
779
780
781
782
783
784
785 if (dax_mapping(mapping)) {
786 unmap_mapping_pages(mapping, start, end - start + 1, false);
787 }
788out:
789 cleancache_invalidate_inode(mapping);
790 return ret;
791}
792EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
793
794
795
796
797
798
799
800
801
802
803int invalidate_inode_pages2(struct address_space *mapping)
804{
805 return invalidate_inode_pages2_range(mapping, 0, -1);
806}
807EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824void truncate_pagecache(struct inode *inode, loff_t newsize)
825{
826 struct address_space *mapping = inode->i_mapping;
827 loff_t holebegin = round_up(newsize, PAGE_SIZE);
828
829
830
831
832
833
834
835
836
837
838 unmap_mapping_range(mapping, holebegin, 0, 1);
839 truncate_inode_pages(mapping, newsize);
840 unmap_mapping_range(mapping, holebegin, 0, 1);
841}
842EXPORT_SYMBOL(truncate_pagecache);
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857void truncate_setsize(struct inode *inode, loff_t newsize)
858{
859 loff_t oldsize = inode->i_size;
860
861 i_size_write(inode, newsize);
862 if (newsize > oldsize)
863 pagecache_isize_extended(inode, oldsize, newsize);
864 truncate_pagecache(inode, newsize);
865}
866EXPORT_SYMBOL(truncate_setsize);
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
888{
889 int bsize = i_blocksize(inode);
890 loff_t rounded_from;
891 struct page *page;
892 pgoff_t index;
893
894 WARN_ON(to > inode->i_size);
895
896 if (from >= to || bsize == PAGE_SIZE)
897 return;
898
899 rounded_from = round_up(from, bsize);
900 if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
901 return;
902
903 index = from >> PAGE_SHIFT;
904 page = find_lock_page(inode->i_mapping, index);
905
906 if (!page)
907 return;
908
909
910
911
912 if (page_mkclean(page))
913 set_page_dirty(page);
914 unlock_page(page);
915 put_page(page);
916}
917EXPORT_SYMBOL(pagecache_isize_extended);
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
933{
934 struct address_space *mapping = inode->i_mapping;
935 loff_t unmap_start = round_up(lstart, PAGE_SIZE);
936 loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
937
938
939
940
941
942
943
944
945
946
947
948
949
950 if ((u64)unmap_end > (u64)unmap_start)
951 unmap_mapping_range(mapping, unmap_start,
952 1 + unmap_end - unmap_start, 0);
953 truncate_inode_pages_range(mapping, lstart, lend);
954}
955EXPORT_SYMBOL(truncate_pagecache_range);
956