1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48#include <linux/mm.h>
49#include <linux/sched/mm.h>
50#include <linux/sched/task.h>
51#include <linux/pagemap.h>
52#include <linux/swap.h>
53#include <linux/swapops.h>
54#include <linux/slab.h>
55#include <linux/init.h>
56#include <linux/ksm.h>
57#include <linux/rmap.h>
58#include <linux/rcupdate.h>
59#include <linux/export.h>
60#include <linux/memcontrol.h>
61#include <linux/mmu_notifier.h>
62#include <linux/migrate.h>
63#include <linux/hugetlb.h>
64#include <linux/backing-dev.h>
65#include <linux/page_idle.h>
66
67#include <asm/tlbflush.h>
68
69#include <trace/events/tlb.h>
70
71#include "internal.h"
72
73static struct kmem_cache *anon_vma_cachep;
74static struct kmem_cache *anon_vma_chain_cachep;
75
76static inline struct anon_vma *anon_vma_alloc(void)
77{
78 struct anon_vma *anon_vma;
79
80 anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
81 if (anon_vma) {
82 atomic_set(&anon_vma->refcount, 1);
83 anon_vma->degree = 1;
84 anon_vma->parent = anon_vma;
85
86
87
88
89 anon_vma->root = anon_vma;
90 }
91
92 return anon_vma;
93}
94
95static inline void anon_vma_free(struct anon_vma *anon_vma)
96{
97 VM_BUG_ON(atomic_read(&anon_vma->refcount));
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116 might_sleep();
117 if (rwsem_is_locked(&anon_vma->root->rwsem)) {
118 anon_vma_lock_write(anon_vma);
119 anon_vma_unlock_write(anon_vma);
120 }
121
122 kmem_cache_free(anon_vma_cachep, anon_vma);
123}
124
125static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp)
126{
127 return kmem_cache_alloc(anon_vma_chain_cachep, gfp);
128}
129
130static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain)
131{
132 kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain);
133}
134
135static void anon_vma_chain_link(struct vm_area_struct *vma,
136 struct anon_vma_chain *avc,
137 struct anon_vma *anon_vma)
138{
139 avc->vma = vma;
140 avc->anon_vma = anon_vma;
141 list_add(&avc->same_vma, &vma->anon_vma_chain);
142 anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
143}
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173int __anon_vma_prepare(struct vm_area_struct *vma)
174{
175 struct mm_struct *mm = vma->vm_mm;
176 struct anon_vma *anon_vma, *allocated;
177 struct anon_vma_chain *avc;
178
179 might_sleep();
180
181 avc = anon_vma_chain_alloc(GFP_KERNEL);
182 if (!avc)
183 goto out_enomem;
184
185 anon_vma = find_mergeable_anon_vma(vma);
186 allocated = NULL;
187 if (!anon_vma) {
188 anon_vma = anon_vma_alloc();
189 if (unlikely(!anon_vma))
190 goto out_enomem_free_avc;
191 allocated = anon_vma;
192 }
193
194 anon_vma_lock_write(anon_vma);
195
196 spin_lock(&mm->page_table_lock);
197 if (likely(!vma->anon_vma)) {
198 vma->anon_vma = anon_vma;
199 anon_vma_chain_link(vma, avc, anon_vma);
200
201 anon_vma->degree++;
202 allocated = NULL;
203 avc = NULL;
204 }
205 spin_unlock(&mm->page_table_lock);
206 anon_vma_unlock_write(anon_vma);
207
208 if (unlikely(allocated))
209 put_anon_vma(allocated);
210 if (unlikely(avc))
211 anon_vma_chain_free(avc);
212
213 return 0;
214
215 out_enomem_free_avc:
216 anon_vma_chain_free(avc);
217 out_enomem:
218 return -ENOMEM;
219}
220
221
222
223
224
225
226
227
228
229static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma)
230{
231 struct anon_vma *new_root = anon_vma->root;
232 if (new_root != root) {
233 if (WARN_ON_ONCE(root))
234 up_write(&root->rwsem);
235 root = new_root;
236 down_write(&root->rwsem);
237 }
238 return root;
239}
240
241static inline void unlock_anon_vma_root(struct anon_vma *root)
242{
243 if (root)
244 up_write(&root->rwsem);
245}
246
247
248
249
250
251
252
253
254
255
256
257
258
259int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
260{
261 struct anon_vma_chain *avc, *pavc;
262 struct anon_vma *root = NULL;
263
264 list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
265 struct anon_vma *anon_vma;
266
267 avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN);
268 if (unlikely(!avc)) {
269 unlock_anon_vma_root(root);
270 root = NULL;
271 avc = anon_vma_chain_alloc(GFP_KERNEL);
272 if (!avc)
273 goto enomem_failure;
274 }
275 anon_vma = pavc->anon_vma;
276 root = lock_anon_vma_root(root, anon_vma);
277 anon_vma_chain_link(dst, avc, anon_vma);
278
279
280
281
282
283
284
285
286
287 if (!dst->anon_vma && anon_vma != src->anon_vma &&
288 anon_vma->degree < 2)
289 dst->anon_vma = anon_vma;
290 }
291 if (dst->anon_vma)
292 dst->anon_vma->degree++;
293 unlock_anon_vma_root(root);
294 return 0;
295
296 enomem_failure:
297
298
299
300
301
302
303 dst->anon_vma = NULL;
304 unlink_anon_vmas(dst);
305 return -ENOMEM;
306}
307
308
309
310
311
312
313int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
314{
315 struct anon_vma_chain *avc;
316 struct anon_vma *anon_vma;
317 int error;
318
319
320 if (!pvma->anon_vma)
321 return 0;
322
323
324 vma->anon_vma = NULL;
325
326
327
328
329
330 error = anon_vma_clone(vma, pvma);
331 if (error)
332 return error;
333
334
335 if (vma->anon_vma)
336 return 0;
337
338
339 anon_vma = anon_vma_alloc();
340 if (!anon_vma)
341 goto out_error;
342 avc = anon_vma_chain_alloc(GFP_KERNEL);
343 if (!avc)
344 goto out_error_free_anon_vma;
345
346
347
348
349
350 anon_vma->root = pvma->anon_vma->root;
351 anon_vma->parent = pvma->anon_vma;
352
353
354
355
356
357 get_anon_vma(anon_vma->root);
358
359 vma->anon_vma = anon_vma;
360 anon_vma_lock_write(anon_vma);
361 anon_vma_chain_link(vma, avc, anon_vma);
362 anon_vma->parent->degree++;
363 anon_vma_unlock_write(anon_vma);
364
365 return 0;
366
367 out_error_free_anon_vma:
368 put_anon_vma(anon_vma);
369 out_error:
370 unlink_anon_vmas(vma);
371 return -ENOMEM;
372}
373
374void unlink_anon_vmas(struct vm_area_struct *vma)
375{
376 struct anon_vma_chain *avc, *next;
377 struct anon_vma *root = NULL;
378
379
380
381
382
383 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
384 struct anon_vma *anon_vma = avc->anon_vma;
385
386 root = lock_anon_vma_root(root, anon_vma);
387 anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
388
389
390
391
392
393 if (RB_EMPTY_ROOT(&anon_vma->rb_root)) {
394 anon_vma->parent->degree--;
395 continue;
396 }
397
398 list_del(&avc->same_vma);
399 anon_vma_chain_free(avc);
400 }
401 if (vma->anon_vma)
402 vma->anon_vma->degree--;
403 unlock_anon_vma_root(root);
404
405
406
407
408
409
410 list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) {
411 struct anon_vma *anon_vma = avc->anon_vma;
412
413 VM_WARN_ON(anon_vma->degree);
414 put_anon_vma(anon_vma);
415
416 list_del(&avc->same_vma);
417 anon_vma_chain_free(avc);
418 }
419}
420
421static void anon_vma_ctor(void *data)
422{
423 struct anon_vma *anon_vma = data;
424
425 init_rwsem(&anon_vma->rwsem);
426 atomic_set(&anon_vma->refcount, 0);
427 anon_vma->rb_root = RB_ROOT;
428}
429
430void __init anon_vma_init(void)
431{
432 anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
433 0, SLAB_TYPESAFE_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT,
434 anon_vma_ctor);
435 anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain,
436 SLAB_PANIC|SLAB_ACCOUNT);
437}
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462struct anon_vma *page_get_anon_vma(struct page *page)
463{
464 struct anon_vma *anon_vma = NULL;
465 unsigned long anon_mapping;
466
467 rcu_read_lock();
468 anon_mapping = (unsigned long)READ_ONCE(page->mapping);
469 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
470 goto out;
471 if (!page_mapped(page))
472 goto out;
473
474 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
475 if (!atomic_inc_not_zero(&anon_vma->refcount)) {
476 anon_vma = NULL;
477 goto out;
478 }
479
480
481
482
483
484
485
486
487 if (!page_mapped(page)) {
488 rcu_read_unlock();
489 put_anon_vma(anon_vma);
490 return NULL;
491 }
492out:
493 rcu_read_unlock();
494
495 return anon_vma;
496}
497
498
499
500
501
502
503
504
505struct anon_vma *page_lock_anon_vma_read(struct page *page)
506{
507 struct anon_vma *anon_vma = NULL;
508 struct anon_vma *root_anon_vma;
509 unsigned long anon_mapping;
510
511 rcu_read_lock();
512 anon_mapping = (unsigned long)READ_ONCE(page->mapping);
513 if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
514 goto out;
515 if (!page_mapped(page))
516 goto out;
517
518 anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
519 root_anon_vma = READ_ONCE(anon_vma->root);
520 if (down_read_trylock(&root_anon_vma->rwsem)) {
521
522
523
524
525
526 if (!page_mapped(page)) {
527 up_read(&root_anon_vma->rwsem);
528 anon_vma = NULL;
529 }
530 goto out;
531 }
532
533
534 if (!atomic_inc_not_zero(&anon_vma->refcount)) {
535 anon_vma = NULL;
536 goto out;
537 }
538
539 if (!page_mapped(page)) {
540 rcu_read_unlock();
541 put_anon_vma(anon_vma);
542 return NULL;
543 }
544
545
546 rcu_read_unlock();
547 anon_vma_lock_read(anon_vma);
548
549 if (atomic_dec_and_test(&anon_vma->refcount)) {
550
551
552
553
554
555 anon_vma_unlock_read(anon_vma);
556 __put_anon_vma(anon_vma);
557 anon_vma = NULL;
558 }
559
560 return anon_vma;
561
562out:
563 rcu_read_unlock();
564 return anon_vma;
565}
566
567void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
568{
569 anon_vma_unlock_read(anon_vma);
570}
571
572#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
573
574
575
576
577
578
579void try_to_unmap_flush(void)
580{
581 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
582 int cpu;
583
584 if (!tlb_ubc->flush_required)
585 return;
586
587 cpu = get_cpu();
588
589 if (cpumask_test_cpu(cpu, &tlb_ubc->cpumask)) {
590 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
591 local_flush_tlb();
592 trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
593 }
594
595 if (cpumask_any_but(&tlb_ubc->cpumask, cpu) < nr_cpu_ids)
596 flush_tlb_others(&tlb_ubc->cpumask, NULL, 0, TLB_FLUSH_ALL);
597 cpumask_clear(&tlb_ubc->cpumask);
598 tlb_ubc->flush_required = false;
599 tlb_ubc->writable = false;
600 put_cpu();
601}
602
603
604void try_to_unmap_flush_dirty(void)
605{
606 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
607
608 if (tlb_ubc->writable)
609 try_to_unmap_flush();
610}
611
612static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)
613{
614 struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc;
615
616 cpumask_or(&tlb_ubc->cpumask, &tlb_ubc->cpumask, mm_cpumask(mm));
617 tlb_ubc->flush_required = true;
618
619
620
621
622
623
624 if (writable)
625 tlb_ubc->writable = true;
626}
627
628
629
630
631
632static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
633{
634 bool should_defer = false;
635
636 if (!(flags & TTU_BATCH_FLUSH))
637 return false;
638
639
640 if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids)
641 should_defer = true;
642 put_cpu();
643
644 return should_defer;
645}
646#else
647static void set_tlb_ubc_flush_pending(struct mm_struct *mm, bool writable)
648{
649}
650
651static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags)
652{
653 return false;
654}
655#endif
656
657
658
659
660
661unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma)
662{
663 unsigned long address;
664 if (PageAnon(page)) {
665 struct anon_vma *page__anon_vma = page_anon_vma(page);
666
667
668
669
670 if (!vma->anon_vma || !page__anon_vma ||
671 vma->anon_vma->root != page__anon_vma->root)
672 return -EFAULT;
673 } else if (page->mapping) {
674 if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping)
675 return -EFAULT;
676 } else
677 return -EFAULT;
678 address = __vma_address(page, vma);
679 if (unlikely(address < vma->vm_start || address >= vma->vm_end))
680 return -EFAULT;
681 return address;
682}
683
684pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address)
685{
686 pgd_t *pgd;
687 p4d_t *p4d;
688 pud_t *pud;
689 pmd_t *pmd = NULL;
690 pmd_t pmde;
691
692 pgd = pgd_offset(mm, address);
693 if (!pgd_present(*pgd))
694 goto out;
695
696 p4d = p4d_offset(pgd, address);
697 if (!p4d_present(*p4d))
698 goto out;
699
700 pud = pud_offset(p4d, address);
701 if (!pud_present(*pud))
702 goto out;
703
704 pmd = pmd_offset(pud, address);
705
706
707
708
709
710 pmde = *pmd;
711 barrier();
712 if (!pmd_present(pmde) || pmd_trans_huge(pmde))
713 pmd = NULL;
714out:
715 return pmd;
716}
717
718struct page_referenced_arg {
719 int mapcount;
720 int referenced;
721 unsigned long vm_flags;
722 struct mem_cgroup *memcg;
723};
724
725
726
727static bool page_referenced_one(struct page *page, struct vm_area_struct *vma,
728 unsigned long address, void *arg)
729{
730 struct page_referenced_arg *pra = arg;
731 struct page_vma_mapped_walk pvmw = {
732 .page = page,
733 .vma = vma,
734 .address = address,
735 };
736 int referenced = 0;
737
738 while (page_vma_mapped_walk(&pvmw)) {
739 address = pvmw.address;
740
741 if (vma->vm_flags & VM_LOCKED) {
742 page_vma_mapped_walk_done(&pvmw);
743 pra->vm_flags |= VM_LOCKED;
744 return false;
745 }
746
747 if (pvmw.pte) {
748 if (ptep_clear_flush_young_notify(vma, address,
749 pvmw.pte)) {
750
751
752
753
754
755
756
757
758 if (likely(!(vma->vm_flags & VM_SEQ_READ)))
759 referenced++;
760 }
761 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
762 if (pmdp_clear_flush_young_notify(vma, address,
763 pvmw.pmd))
764 referenced++;
765 } else {
766
767 WARN_ON_ONCE(1);
768 }
769
770 pra->mapcount--;
771 }
772
773 if (referenced)
774 clear_page_idle(page);
775 if (test_and_clear_page_young(page))
776 referenced++;
777
778 if (referenced) {
779 pra->referenced++;
780 pra->vm_flags |= vma->vm_flags;
781 }
782
783 if (!pra->mapcount)
784 return false;
785
786 return true;
787}
788
789static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg)
790{
791 struct page_referenced_arg *pra = arg;
792 struct mem_cgroup *memcg = pra->memcg;
793
794 if (!mm_match_cgroup(vma->vm_mm, memcg))
795 return true;
796
797 return false;
798}
799
800
801
802
803
804
805
806
807
808
809
810int page_referenced(struct page *page,
811 int is_locked,
812 struct mem_cgroup *memcg,
813 unsigned long *vm_flags)
814{
815 int we_locked = 0;
816 struct page_referenced_arg pra = {
817 .mapcount = total_mapcount(page),
818 .memcg = memcg,
819 };
820 struct rmap_walk_control rwc = {
821 .rmap_one = page_referenced_one,
822 .arg = (void *)&pra,
823 .anon_lock = page_lock_anon_vma_read,
824 };
825
826 *vm_flags = 0;
827 if (!page_mapped(page))
828 return 0;
829
830 if (!page_rmapping(page))
831 return 0;
832
833 if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
834 we_locked = trylock_page(page);
835 if (!we_locked)
836 return 1;
837 }
838
839
840
841
842
843
844 if (memcg) {
845 rwc.invalid_vma = invalid_page_referenced_vma;
846 }
847
848 rmap_walk(page, &rwc);
849 *vm_flags = pra.vm_flags;
850
851 if (we_locked)
852 unlock_page(page);
853
854 return pra.referenced;
855}
856
857static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
858 unsigned long address, void *arg)
859{
860 struct page_vma_mapped_walk pvmw = {
861 .page = page,
862 .vma = vma,
863 .address = address,
864 .flags = PVMW_SYNC,
865 };
866 int *cleaned = arg;
867
868 while (page_vma_mapped_walk(&pvmw)) {
869 int ret = 0;
870 address = pvmw.address;
871 if (pvmw.pte) {
872 pte_t entry;
873 pte_t *pte = pvmw.pte;
874
875 if (!pte_dirty(*pte) && !pte_write(*pte))
876 continue;
877
878 flush_cache_page(vma, address, pte_pfn(*pte));
879 entry = ptep_clear_flush(vma, address, pte);
880 entry = pte_wrprotect(entry);
881 entry = pte_mkclean(entry);
882 set_pte_at(vma->vm_mm, address, pte, entry);
883 ret = 1;
884 } else {
885#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
886 pmd_t *pmd = pvmw.pmd;
887 pmd_t entry;
888
889 if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
890 continue;
891
892 flush_cache_page(vma, address, page_to_pfn(page));
893 entry = pmdp_huge_clear_flush(vma, address, pmd);
894 entry = pmd_wrprotect(entry);
895 entry = pmd_mkclean(entry);
896 set_pmd_at(vma->vm_mm, address, pmd, entry);
897 ret = 1;
898#else
899
900 WARN_ON_ONCE(1);
901#endif
902 }
903
904 if (ret) {
905 mmu_notifier_invalidate_page(vma->vm_mm, address);
906 (*cleaned)++;
907 }
908 }
909
910 return true;
911}
912
913static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg)
914{
915 if (vma->vm_flags & VM_SHARED)
916 return false;
917
918 return true;
919}
920
921int page_mkclean(struct page *page)
922{
923 int cleaned = 0;
924 struct address_space *mapping;
925 struct rmap_walk_control rwc = {
926 .arg = (void *)&cleaned,
927 .rmap_one = page_mkclean_one,
928 .invalid_vma = invalid_mkclean_vma,
929 };
930
931 BUG_ON(!PageLocked(page));
932
933 if (!page_mapped(page))
934 return 0;
935
936 mapping = page_mapping(page);
937 if (!mapping)
938 return 0;
939
940 rmap_walk(page, &rwc);
941
942 return cleaned;
943}
944EXPORT_SYMBOL_GPL(page_mkclean);
945
946
947
948
949
950
951
952
953
954
955
956void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma)
957{
958 struct anon_vma *anon_vma = vma->anon_vma;
959
960 page = compound_head(page);
961
962 VM_BUG_ON_PAGE(!PageLocked(page), page);
963 VM_BUG_ON_VMA(!anon_vma, vma);
964
965 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
966
967
968
969
970
971 WRITE_ONCE(page->mapping, (struct address_space *) anon_vma);
972}
973
974
975
976
977
978
979
980
981static void __page_set_anon_rmap(struct page *page,
982 struct vm_area_struct *vma, unsigned long address, int exclusive)
983{
984 struct anon_vma *anon_vma = vma->anon_vma;
985
986 BUG_ON(!anon_vma);
987
988 if (PageAnon(page))
989 return;
990
991
992
993
994
995
996 if (!exclusive)
997 anon_vma = anon_vma->root;
998
999 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1000 page->mapping = (struct address_space *) anon_vma;
1001 page->index = linear_page_index(vma, address);
1002}
1003
1004
1005
1006
1007
1008
1009
1010static void __page_check_anon_rmap(struct page *page,
1011 struct vm_area_struct *vma, unsigned long address)
1012{
1013#ifdef CONFIG_DEBUG_VM
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026 BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root);
1027 BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address));
1028#endif
1029}
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043void page_add_anon_rmap(struct page *page,
1044 struct vm_area_struct *vma, unsigned long address, bool compound)
1045{
1046 do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0);
1047}
1048
1049
1050
1051
1052
1053
1054void do_page_add_anon_rmap(struct page *page,
1055 struct vm_area_struct *vma, unsigned long address, int flags)
1056{
1057 bool compound = flags & RMAP_COMPOUND;
1058 bool first;
1059
1060 if (compound) {
1061 atomic_t *mapcount;
1062 VM_BUG_ON_PAGE(!PageLocked(page), page);
1063 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
1064 mapcount = compound_mapcount_ptr(page);
1065 first = atomic_inc_and_test(mapcount);
1066 } else {
1067 first = atomic_inc_and_test(&page->_mapcount);
1068 }
1069
1070 if (first) {
1071 int nr = compound ? hpage_nr_pages(page) : 1;
1072
1073
1074
1075
1076
1077
1078 if (compound)
1079 __inc_node_page_state(page, NR_ANON_THPS);
1080 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
1081 }
1082 if (unlikely(PageKsm(page)))
1083 return;
1084
1085 VM_BUG_ON_PAGE(!PageLocked(page), page);
1086
1087
1088 if (first)
1089 __page_set_anon_rmap(page, vma, address,
1090 flags & RMAP_EXCLUSIVE);
1091 else
1092 __page_check_anon_rmap(page, vma, address);
1093}
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106void page_add_new_anon_rmap(struct page *page,
1107 struct vm_area_struct *vma, unsigned long address, bool compound)
1108{
1109 int nr = compound ? hpage_nr_pages(page) : 1;
1110
1111 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
1112 __SetPageSwapBacked(page);
1113 if (compound) {
1114 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
1115
1116 atomic_set(compound_mapcount_ptr(page), 0);
1117 __inc_node_page_state(page, NR_ANON_THPS);
1118 } else {
1119
1120 VM_BUG_ON_PAGE(PageTransCompound(page), page);
1121
1122 atomic_set(&page->_mapcount, 0);
1123 }
1124 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr);
1125 __page_set_anon_rmap(page, vma, address, 1);
1126}
1127
1128
1129
1130
1131
1132
1133
1134void page_add_file_rmap(struct page *page, bool compound)
1135{
1136 int i, nr = 1;
1137
1138 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
1139 lock_page_memcg(page);
1140 if (compound && PageTransHuge(page)) {
1141 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1142 if (atomic_inc_and_test(&page[i]._mapcount))
1143 nr++;
1144 }
1145 if (!atomic_inc_and_test(compound_mapcount_ptr(page)))
1146 goto out;
1147 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
1148 __inc_node_page_state(page, NR_SHMEM_PMDMAPPED);
1149 } else {
1150 if (PageTransCompound(page) && page_mapping(page)) {
1151 VM_WARN_ON_ONCE(!PageLocked(page));
1152
1153 SetPageDoubleMap(compound_head(page));
1154 if (PageMlocked(page))
1155 clear_page_mlock(compound_head(page));
1156 }
1157 if (!atomic_inc_and_test(&page->_mapcount))
1158 goto out;
1159 }
1160 __mod_node_page_state(page_pgdat(page), NR_FILE_MAPPED, nr);
1161 mod_memcg_page_state(page, NR_FILE_MAPPED, nr);
1162out:
1163 unlock_page_memcg(page);
1164}
1165
1166static void page_remove_file_rmap(struct page *page, bool compound)
1167{
1168 int i, nr = 1;
1169
1170 VM_BUG_ON_PAGE(compound && !PageHead(page), page);
1171 lock_page_memcg(page);
1172
1173
1174 if (unlikely(PageHuge(page))) {
1175
1176 atomic_dec(compound_mapcount_ptr(page));
1177 goto out;
1178 }
1179
1180
1181 if (compound && PageTransHuge(page)) {
1182 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1183 if (atomic_add_negative(-1, &page[i]._mapcount))
1184 nr++;
1185 }
1186 if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
1187 goto out;
1188 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
1189 __dec_node_page_state(page, NR_SHMEM_PMDMAPPED);
1190 } else {
1191 if (!atomic_add_negative(-1, &page->_mapcount))
1192 goto out;
1193 }
1194
1195
1196
1197
1198
1199
1200 __mod_node_page_state(page_pgdat(page), NR_FILE_MAPPED, -nr);
1201 mod_memcg_page_state(page, NR_FILE_MAPPED, -nr);
1202
1203 if (unlikely(PageMlocked(page)))
1204 clear_page_mlock(page);
1205out:
1206 unlock_page_memcg(page);
1207}
1208
1209static void page_remove_anon_compound_rmap(struct page *page)
1210{
1211 int i, nr;
1212
1213 if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
1214 return;
1215
1216
1217 if (unlikely(PageHuge(page)))
1218 return;
1219
1220 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1221 return;
1222
1223 __dec_node_page_state(page, NR_ANON_THPS);
1224
1225 if (TestClearPageDoubleMap(page)) {
1226
1227
1228
1229
1230 for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
1231 if (atomic_add_negative(-1, &page[i]._mapcount))
1232 nr++;
1233 }
1234 } else {
1235 nr = HPAGE_PMD_NR;
1236 }
1237
1238 if (unlikely(PageMlocked(page)))
1239 clear_page_mlock(page);
1240
1241 if (nr) {
1242 __mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr);
1243 deferred_split_huge_page(page);
1244 }
1245}
1246
1247
1248
1249
1250
1251
1252
1253
1254void page_remove_rmap(struct page *page, bool compound)
1255{
1256 if (!PageAnon(page))
1257 return page_remove_file_rmap(page, compound);
1258
1259 if (compound)
1260 return page_remove_anon_compound_rmap(page);
1261
1262
1263 if (!atomic_add_negative(-1, &page->_mapcount))
1264 return;
1265
1266
1267
1268
1269
1270
1271 __dec_node_page_state(page, NR_ANON_MAPPED);
1272
1273 if (unlikely(PageMlocked(page)))
1274 clear_page_mlock(page);
1275
1276 if (PageTransCompound(page))
1277 deferred_split_huge_page(compound_head(page));
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288}
1289
1290
1291
1292
1293static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
1294 unsigned long address, void *arg)
1295{
1296 struct mm_struct *mm = vma->vm_mm;
1297 struct page_vma_mapped_walk pvmw = {
1298 .page = page,
1299 .vma = vma,
1300 .address = address,
1301 };
1302 pte_t pteval;
1303 struct page *subpage;
1304 bool ret = true;
1305 enum ttu_flags flags = (enum ttu_flags)arg;
1306
1307
1308 if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED))
1309 return true;
1310
1311 if (flags & TTU_SPLIT_HUGE_PMD) {
1312 split_huge_pmd_address(vma, address,
1313 flags & TTU_MIGRATION, page);
1314 }
1315
1316 while (page_vma_mapped_walk(&pvmw)) {
1317
1318
1319
1320
1321
1322 if (!(flags & TTU_IGNORE_MLOCK)) {
1323 if (vma->vm_flags & VM_LOCKED) {
1324
1325 if (!PageTransCompound(page)) {
1326
1327
1328
1329
1330 mlock_vma_page(page);
1331 }
1332 ret = false;
1333 page_vma_mapped_walk_done(&pvmw);
1334 break;
1335 }
1336 if (flags & TTU_MUNLOCK)
1337 continue;
1338 }
1339
1340
1341 VM_BUG_ON_PAGE(!pvmw.pte, page);
1342
1343 subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
1344 address = pvmw.address;
1345
1346
1347 if (!(flags & TTU_IGNORE_ACCESS)) {
1348 if (ptep_clear_flush_young_notify(vma, address,
1349 pvmw.pte)) {
1350 ret = false;
1351 page_vma_mapped_walk_done(&pvmw);
1352 break;
1353 }
1354 }
1355
1356
1357 flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
1358 if (should_defer_flush(mm, flags)) {
1359
1360
1361
1362
1363
1364
1365
1366
1367 pteval = ptep_get_and_clear(mm, address, pvmw.pte);
1368
1369 set_tlb_ubc_flush_pending(mm, pte_dirty(pteval));
1370 } else {
1371 pteval = ptep_clear_flush(vma, address, pvmw.pte);
1372 }
1373
1374
1375 if (pte_dirty(pteval))
1376 set_page_dirty(page);
1377
1378
1379 update_hiwater_rss(mm);
1380
1381 if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) {
1382 if (PageHuge(page)) {
1383 int nr = 1 << compound_order(page);
1384 hugetlb_count_sub(nr, mm);
1385 } else {
1386 dec_mm_counter(mm, mm_counter(page));
1387 }
1388
1389 pteval = swp_entry_to_pte(make_hwpoison_entry(subpage));
1390 set_pte_at(mm, address, pvmw.pte, pteval);
1391 } else if (pte_unused(pteval)) {
1392
1393
1394
1395
1396
1397 dec_mm_counter(mm, mm_counter(page));
1398 } else if (IS_ENABLED(CONFIG_MIGRATION) &&
1399 (flags & TTU_MIGRATION)) {
1400 swp_entry_t entry;
1401 pte_t swp_pte;
1402
1403
1404
1405
1406
1407 entry = make_migration_entry(subpage,
1408 pte_write(pteval));
1409 swp_pte = swp_entry_to_pte(entry);
1410 if (pte_soft_dirty(pteval))
1411 swp_pte = pte_swp_mksoft_dirty(swp_pte);
1412 set_pte_at(mm, address, pvmw.pte, swp_pte);
1413 } else if (PageAnon(page)) {
1414 swp_entry_t entry = { .val = page_private(subpage) };
1415 pte_t swp_pte;
1416
1417
1418
1419
1420 if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) {
1421 WARN_ON_ONCE(1);
1422 ret = false;
1423 page_vma_mapped_walk_done(&pvmw);
1424 break;
1425 }
1426
1427
1428 if (!PageSwapBacked(page)) {
1429 if (!PageDirty(page)) {
1430 dec_mm_counter(mm, MM_ANONPAGES);
1431 goto discard;
1432 }
1433
1434
1435
1436
1437
1438 set_pte_at(mm, address, pvmw.pte, pteval);
1439 SetPageSwapBacked(page);
1440 ret = false;
1441 page_vma_mapped_walk_done(&pvmw);
1442 break;
1443 }
1444
1445 if (swap_duplicate(entry) < 0) {
1446 set_pte_at(mm, address, pvmw.pte, pteval);
1447 ret = false;
1448 page_vma_mapped_walk_done(&pvmw);
1449 break;
1450 }
1451 if (list_empty(&mm->mmlist)) {
1452 spin_lock(&mmlist_lock);
1453 if (list_empty(&mm->mmlist))
1454 list_add(&mm->mmlist, &init_mm.mmlist);
1455 spin_unlock(&mmlist_lock);
1456 }
1457 dec_mm_counter(mm, MM_ANONPAGES);
1458 inc_mm_counter(mm, MM_SWAPENTS);
1459 swp_pte = swp_entry_to_pte(entry);
1460 if (pte_soft_dirty(pteval))
1461 swp_pte = pte_swp_mksoft_dirty(swp_pte);
1462 set_pte_at(mm, address, pvmw.pte, swp_pte);
1463 } else
1464 dec_mm_counter(mm, mm_counter_file(page));
1465discard:
1466 page_remove_rmap(subpage, PageHuge(page));
1467 put_page(page);
1468 mmu_notifier_invalidate_page(mm, address);
1469 }
1470 return ret;
1471}
1472
1473bool is_vma_temporary_stack(struct vm_area_struct *vma)
1474{
1475 int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
1476
1477 if (!maybe_stack)
1478 return false;
1479
1480 if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) ==
1481 VM_STACK_INCOMPLETE_SETUP)
1482 return true;
1483
1484 return false;
1485}
1486
1487static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg)
1488{
1489 return is_vma_temporary_stack(vma);
1490}
1491
1492static int page_mapcount_is_zero(struct page *page)
1493{
1494 return !total_mapcount(page);
1495}
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507bool try_to_unmap(struct page *page, enum ttu_flags flags)
1508{
1509 struct rmap_walk_control rwc = {
1510 .rmap_one = try_to_unmap_one,
1511 .arg = (void *)flags,
1512 .done = page_mapcount_is_zero,
1513 .anon_lock = page_lock_anon_vma_read,
1514 };
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524 if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page))
1525 rwc.invalid_vma = invalid_migration_vma;
1526
1527 if (flags & TTU_RMAP_LOCKED)
1528 rmap_walk_locked(page, &rwc);
1529 else
1530 rmap_walk(page, &rwc);
1531
1532 return !page_mapcount(page) ? true : false;
1533}
1534
1535static int page_not_mapped(struct page *page)
1536{
1537 return !page_mapped(page);
1538};
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549void try_to_munlock(struct page *page)
1550{
1551 struct rmap_walk_control rwc = {
1552 .rmap_one = try_to_unmap_one,
1553 .arg = (void *)TTU_MUNLOCK,
1554 .done = page_not_mapped,
1555 .anon_lock = page_lock_anon_vma_read,
1556
1557 };
1558
1559 VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page);
1560 VM_BUG_ON_PAGE(PageCompound(page) && PageDoubleMap(page), page);
1561
1562 rmap_walk(page, &rwc);
1563}
1564
1565void __put_anon_vma(struct anon_vma *anon_vma)
1566{
1567 struct anon_vma *root = anon_vma->root;
1568
1569 anon_vma_free(anon_vma);
1570 if (root != anon_vma && atomic_dec_and_test(&root->refcount))
1571 anon_vma_free(root);
1572}
1573
1574static struct anon_vma *rmap_walk_anon_lock(struct page *page,
1575 struct rmap_walk_control *rwc)
1576{
1577 struct anon_vma *anon_vma;
1578
1579 if (rwc->anon_lock)
1580 return rwc->anon_lock(page);
1581
1582
1583
1584
1585
1586
1587
1588 anon_vma = page_anon_vma(page);
1589 if (!anon_vma)
1590 return NULL;
1591
1592 anon_vma_lock_read(anon_vma);
1593 return anon_vma;
1594}
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610static void rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc,
1611 bool locked)
1612{
1613 struct anon_vma *anon_vma;
1614 pgoff_t pgoff_start, pgoff_end;
1615 struct anon_vma_chain *avc;
1616
1617 if (locked) {
1618 anon_vma = page_anon_vma(page);
1619
1620 VM_BUG_ON_PAGE(!anon_vma, page);
1621 } else {
1622 anon_vma = rmap_walk_anon_lock(page, rwc);
1623 }
1624 if (!anon_vma)
1625 return;
1626
1627 pgoff_start = page_to_pgoff(page);
1628 pgoff_end = pgoff_start + hpage_nr_pages(page) - 1;
1629 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
1630 pgoff_start, pgoff_end) {
1631 struct vm_area_struct *vma = avc->vma;
1632 unsigned long address = vma_address(page, vma);
1633
1634 cond_resched();
1635
1636 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
1637 continue;
1638
1639 if (!rwc->rmap_one(page, vma, address, rwc->arg))
1640 break;
1641 if (rwc->done && rwc->done(page))
1642 break;
1643 }
1644
1645 if (!locked)
1646 anon_vma_unlock_read(anon_vma);
1647}
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662static void rmap_walk_file(struct page *page, struct rmap_walk_control *rwc,
1663 bool locked)
1664{
1665 struct address_space *mapping = page_mapping(page);
1666 pgoff_t pgoff_start, pgoff_end;
1667 struct vm_area_struct *vma;
1668
1669
1670
1671
1672
1673
1674
1675 VM_BUG_ON_PAGE(!PageLocked(page), page);
1676
1677 if (!mapping)
1678 return;
1679
1680 pgoff_start = page_to_pgoff(page);
1681 pgoff_end = pgoff_start + hpage_nr_pages(page) - 1;
1682 if (!locked)
1683 i_mmap_lock_read(mapping);
1684 vma_interval_tree_foreach(vma, &mapping->i_mmap,
1685 pgoff_start, pgoff_end) {
1686 unsigned long address = vma_address(page, vma);
1687
1688 cond_resched();
1689
1690 if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg))
1691 continue;
1692
1693 if (!rwc->rmap_one(page, vma, address, rwc->arg))
1694 goto done;
1695 if (rwc->done && rwc->done(page))
1696 goto done;
1697 }
1698
1699done:
1700 if (!locked)
1701 i_mmap_unlock_read(mapping);
1702}
1703
1704void rmap_walk(struct page *page, struct rmap_walk_control *rwc)
1705{
1706 if (unlikely(PageKsm(page)))
1707 rmap_walk_ksm(page, rwc);
1708 else if (PageAnon(page))
1709 rmap_walk_anon(page, rwc, false);
1710 else
1711 rmap_walk_file(page, rwc, false);
1712}
1713
1714
1715void rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc)
1716{
1717
1718 VM_BUG_ON_PAGE(PageKsm(page), page);
1719 if (PageAnon(page))
1720 rmap_walk_anon(page, rwc, true);
1721 else
1722 rmap_walk_file(page, rwc, true);
1723}
1724
1725#ifdef CONFIG_HUGETLB_PAGE
1726
1727
1728
1729
1730
1731static void __hugepage_set_anon_rmap(struct page *page,
1732 struct vm_area_struct *vma, unsigned long address, int exclusive)
1733{
1734 struct anon_vma *anon_vma = vma->anon_vma;
1735
1736 BUG_ON(!anon_vma);
1737
1738 if (PageAnon(page))
1739 return;
1740 if (!exclusive)
1741 anon_vma = anon_vma->root;
1742
1743 anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
1744 page->mapping = (struct address_space *) anon_vma;
1745 page->index = linear_page_index(vma, address);
1746}
1747
1748void hugepage_add_anon_rmap(struct page *page,
1749 struct vm_area_struct *vma, unsigned long address)
1750{
1751 struct anon_vma *anon_vma = vma->anon_vma;
1752 int first;
1753
1754 BUG_ON(!PageLocked(page));
1755 BUG_ON(!anon_vma);
1756
1757 first = atomic_inc_and_test(compound_mapcount_ptr(page));
1758 if (first)
1759 __hugepage_set_anon_rmap(page, vma, address, 0);
1760}
1761
1762void hugepage_add_new_anon_rmap(struct page *page,
1763 struct vm_area_struct *vma, unsigned long address)
1764{
1765 BUG_ON(address < vma->vm_start || address >= vma->vm_end);
1766 atomic_set(compound_mapcount_ptr(page), 0);
1767 __hugepage_set_anon_rmap(page, vma, address, 1);
1768}
1769#endif
1770