1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/migrate.h>
17#include <linux/export.h>
18#include <linux/swap.h>
19#include <linux/swapops.h>
20#include <linux/pagemap.h>
21#include <linux/buffer_head.h>
22#include <linux/mm_inline.h>
23#include <linux/nsproxy.h>
24#include <linux/pagevec.h>
25#include <linux/ksm.h>
26#include <linux/rmap.h>
27#include <linux/topology.h>
28#include <linux/cpu.h>
29#include <linux/cpuset.h>
30#include <linux/writeback.h>
31#include <linux/mempolicy.h>
32#include <linux/vmalloc.h>
33#include <linux/security.h>
34#include <linux/backing-dev.h>
35#include <linux/compaction.h>
36#include <linux/syscalls.h>
37#include <linux/compat.h>
38#include <linux/hugetlb.h>
39#include <linux/hugetlb_cgroup.h>
40#include <linux/gfp.h>
41#include <linux/pfn_t.h>
42#include <linux/memremap.h>
43#include <linux/userfaultfd_k.h>
44#include <linux/balloon_compaction.h>
45#include <linux/mmu_notifier.h>
46#include <linux/page_idle.h>
47#include <linux/page_owner.h>
48#include <linux/sched/mm.h>
49#include <linux/ptrace.h>
50
51#include <asm/tlbflush.h>
52
53#define CREATE_TRACE_POINTS
54#include <trace/events/migrate.h>
55
56#include "internal.h"
57
58
59
60
61
62
63int migrate_prep(void)
64{
65
66
67
68
69
70
71 lru_add_drain_all();
72
73 return 0;
74}
75
76
77int migrate_prep_local(void)
78{
79 lru_add_drain();
80
81 return 0;
82}
83
84int isolate_movable_page(struct page *page, isolate_mode_t mode)
85{
86 struct address_space *mapping;
87
88
89
90
91
92
93
94
95
96
97 if (unlikely(!get_page_unless_zero(page)))
98 goto out;
99
100
101
102
103
104
105 if (unlikely(!__PageMovable(page)))
106 goto out_putpage;
107
108
109
110
111
112
113
114
115
116
117
118 if (unlikely(!trylock_page(page)))
119 goto out_putpage;
120
121 if (!PageMovable(page) || PageIsolated(page))
122 goto out_no_isolated;
123
124 mapping = page_mapping(page);
125 VM_BUG_ON_PAGE(!mapping, page);
126
127 if (!mapping->a_ops->isolate_page(page, mode))
128 goto out_no_isolated;
129
130
131 WARN_ON_ONCE(PageIsolated(page));
132 __SetPageIsolated(page);
133 unlock_page(page);
134
135 return 0;
136
137out_no_isolated:
138 unlock_page(page);
139out_putpage:
140 put_page(page);
141out:
142 return -EBUSY;
143}
144
145
146void putback_movable_page(struct page *page)
147{
148 struct address_space *mapping;
149
150 VM_BUG_ON_PAGE(!PageLocked(page), page);
151 VM_BUG_ON_PAGE(!PageMovable(page), page);
152 VM_BUG_ON_PAGE(!PageIsolated(page), page);
153
154 mapping = page_mapping(page);
155 mapping->a_ops->putback_page(page);
156 __ClearPageIsolated(page);
157}
158
159
160
161
162
163
164
165
166
167void putback_movable_pages(struct list_head *l)
168{
169 struct page *page;
170 struct page *page2;
171
172 list_for_each_entry_safe(page, page2, l, lru) {
173 if (unlikely(PageHuge(page))) {
174 putback_active_hugepage(page);
175 continue;
176 }
177 list_del(&page->lru);
178
179
180
181
182
183 if (unlikely(__PageMovable(page))) {
184 VM_BUG_ON_PAGE(!PageIsolated(page), page);
185 lock_page(page);
186 if (PageMovable(page))
187 putback_movable_page(page);
188 else
189 __ClearPageIsolated(page);
190 unlock_page(page);
191 put_page(page);
192 } else {
193 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
194 page_is_file_cache(page), -hpage_nr_pages(page));
195 putback_lru_page(page);
196 }
197 }
198}
199
200
201
202
203static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
204 unsigned long addr, void *old)
205{
206 struct page_vma_mapped_walk pvmw = {
207 .page = old,
208 .vma = vma,
209 .address = addr,
210 .flags = PVMW_SYNC | PVMW_MIGRATION,
211 };
212 struct page *new;
213 pte_t pte;
214 swp_entry_t entry;
215
216 VM_BUG_ON_PAGE(PageTail(page), page);
217 while (page_vma_mapped_walk(&pvmw)) {
218 if (PageKsm(page))
219 new = page;
220 else
221 new = page - pvmw.page->index +
222 linear_page_index(vma, pvmw.address);
223
224#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
225
226 if (!pvmw.pte) {
227 VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
228 remove_migration_pmd(&pvmw, new);
229 continue;
230 }
231#endif
232
233 get_page(new);
234 pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
235 if (pte_swp_soft_dirty(*pvmw.pte))
236 pte = pte_mksoft_dirty(pte);
237
238
239
240
241 entry = pte_to_swp_entry(*pvmw.pte);
242 if (is_write_migration_entry(entry))
243 pte = maybe_mkwrite(pte, vma);
244
245 if (unlikely(is_zone_device_page(new))) {
246 if (is_device_private_page(new)) {
247 entry = make_device_private_entry(new, pte_write(pte));
248 pte = swp_entry_to_pte(entry);
249 }
250 }
251
252#ifdef CONFIG_HUGETLB_PAGE
253 if (PageHuge(new)) {
254 pte = pte_mkhuge(pte);
255 pte = arch_make_huge_pte(pte, vma, new, 0);
256 set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
257 if (PageAnon(new))
258 hugepage_add_anon_rmap(new, vma, pvmw.address);
259 else
260 page_dup_rmap(new, true);
261 } else
262#endif
263 {
264 set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
265
266 if (PageAnon(new))
267 page_add_anon_rmap(new, vma, pvmw.address, false);
268 else
269 page_add_file_rmap(new, false);
270 }
271 if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
272 mlock_vma_page(new);
273
274 if (PageTransHuge(page) && PageMlocked(page))
275 clear_page_mlock(page);
276
277
278 update_mmu_cache(vma, pvmw.address, pvmw.pte);
279 }
280
281 return true;
282}
283
284
285
286
287
288void remove_migration_ptes(struct page *old, struct page *new, bool locked)
289{
290 struct rmap_walk_control rwc = {
291 .rmap_one = remove_migration_pte,
292 .arg = old,
293 };
294
295 if (locked)
296 rmap_walk_locked(new, &rwc);
297 else
298 rmap_walk(new, &rwc);
299}
300
301
302
303
304
305
306void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
307 spinlock_t *ptl)
308{
309 pte_t pte;
310 swp_entry_t entry;
311 struct page *page;
312
313 spin_lock(ptl);
314 pte = *ptep;
315 if (!is_swap_pte(pte))
316 goto out;
317
318 entry = pte_to_swp_entry(pte);
319 if (!is_migration_entry(entry))
320 goto out;
321
322 page = migration_entry_to_page(entry);
323
324
325
326
327
328
329 if (!get_page_unless_zero(page))
330 goto out;
331 pte_unmap_unlock(ptep, ptl);
332 put_and_wait_on_page_locked(page);
333 return;
334out:
335 pte_unmap_unlock(ptep, ptl);
336}
337
338void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
339 unsigned long address)
340{
341 spinlock_t *ptl = pte_lockptr(mm, pmd);
342 pte_t *ptep = pte_offset_map(pmd, address);
343 __migration_entry_wait(mm, ptep, ptl);
344}
345
346void migration_entry_wait_huge(struct vm_area_struct *vma,
347 struct mm_struct *mm, pte_t *pte)
348{
349 spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
350 __migration_entry_wait(mm, pte, ptl);
351}
352
353#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
354void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
355{
356 spinlock_t *ptl;
357 struct page *page;
358
359 ptl = pmd_lock(mm, pmd);
360 if (!is_pmd_migration_entry(*pmd))
361 goto unlock;
362 page = migration_entry_to_page(pmd_to_swp_entry(*pmd));
363 if (!get_page_unless_zero(page))
364 goto unlock;
365 spin_unlock(ptl);
366 put_and_wait_on_page_locked(page);
367 return;
368unlock:
369 spin_unlock(ptl);
370}
371#endif
372
373static int expected_page_refs(struct address_space *mapping, struct page *page)
374{
375 int expected_count = 1;
376
377
378
379
380
381 expected_count += is_device_private_page(page);
382 if (mapping)
383 expected_count += hpage_nr_pages(page) + page_has_private(page);
384
385 return expected_count;
386}
387
388
389
390
391
392
393
394
395
396int migrate_page_move_mapping(struct address_space *mapping,
397 struct page *newpage, struct page *page, int extra_count)
398{
399 XA_STATE(xas, &mapping->i_pages, page_index(page));
400 struct zone *oldzone, *newzone;
401 int dirty;
402 int expected_count = expected_page_refs(mapping, page) + extra_count;
403
404 if (!mapping) {
405
406 if (page_count(page) != expected_count)
407 return -EAGAIN;
408
409
410 newpage->index = page->index;
411 newpage->mapping = page->mapping;
412 if (PageSwapBacked(page))
413 __SetPageSwapBacked(newpage);
414
415 return MIGRATEPAGE_SUCCESS;
416 }
417
418 oldzone = page_zone(page);
419 newzone = page_zone(newpage);
420
421 xas_lock_irq(&xas);
422 if (page_count(page) != expected_count || xas_load(&xas) != page) {
423 xas_unlock_irq(&xas);
424 return -EAGAIN;
425 }
426
427 if (!page_ref_freeze(page, expected_count)) {
428 xas_unlock_irq(&xas);
429 return -EAGAIN;
430 }
431
432
433
434
435
436 newpage->index = page->index;
437 newpage->mapping = page->mapping;
438 page_ref_add(newpage, hpage_nr_pages(page));
439 if (PageSwapBacked(page)) {
440 __SetPageSwapBacked(newpage);
441 if (PageSwapCache(page)) {
442 SetPageSwapCache(newpage);
443 set_page_private(newpage, page_private(page));
444 }
445 } else {
446 VM_BUG_ON_PAGE(PageSwapCache(page), page);
447 }
448
449
450 dirty = PageDirty(page);
451 if (dirty) {
452 ClearPageDirty(page);
453 SetPageDirty(newpage);
454 }
455
456 xas_store(&xas, newpage);
457 if (PageTransHuge(page)) {
458 int i;
459
460 for (i = 1; i < HPAGE_PMD_NR; i++) {
461 xas_next(&xas);
462 xas_store(&xas, newpage + i);
463 }
464 }
465
466
467
468
469
470
471 page_ref_unfreeze(page, expected_count - hpage_nr_pages(page));
472
473 xas_unlock(&xas);
474
475
476
477
478
479
480
481
482
483
484
485
486 if (newzone != oldzone) {
487 __dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES);
488 __inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES);
489 if (PageSwapBacked(page) && !PageSwapCache(page)) {
490 __dec_node_state(oldzone->zone_pgdat, NR_SHMEM);
491 __inc_node_state(newzone->zone_pgdat, NR_SHMEM);
492 }
493 if (dirty && mapping_cap_account_dirty(mapping)) {
494 __dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY);
495 __dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING);
496 __inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY);
497 __inc_zone_state(newzone, NR_ZONE_WRITE_PENDING);
498 }
499 }
500 local_irq_enable();
501
502 return MIGRATEPAGE_SUCCESS;
503}
504EXPORT_SYMBOL(migrate_page_move_mapping);
505
506
507
508
509
510int migrate_huge_page_move_mapping(struct address_space *mapping,
511 struct page *newpage, struct page *page)
512{
513 XA_STATE(xas, &mapping->i_pages, page_index(page));
514 int expected_count;
515
516 xas_lock_irq(&xas);
517 expected_count = 2 + page_has_private(page);
518 if (page_count(page) != expected_count || xas_load(&xas) != page) {
519 xas_unlock_irq(&xas);
520 return -EAGAIN;
521 }
522
523 if (!page_ref_freeze(page, expected_count)) {
524 xas_unlock_irq(&xas);
525 return -EAGAIN;
526 }
527
528 newpage->index = page->index;
529 newpage->mapping = page->mapping;
530
531 get_page(newpage);
532
533 xas_store(&xas, newpage);
534
535 page_ref_unfreeze(page, expected_count - 1);
536
537 xas_unlock_irq(&xas);
538
539 return MIGRATEPAGE_SUCCESS;
540}
541
542
543
544
545
546
547static void __copy_gigantic_page(struct page *dst, struct page *src,
548 int nr_pages)
549{
550 int i;
551 struct page *dst_base = dst;
552 struct page *src_base = src;
553
554 for (i = 0; i < nr_pages; ) {
555 cond_resched();
556 copy_highpage(dst, src);
557
558 i++;
559 dst = mem_map_next(dst, dst_base, i);
560 src = mem_map_next(src, src_base, i);
561 }
562}
563
564static void copy_huge_page(struct page *dst, struct page *src)
565{
566 int i;
567 int nr_pages;
568
569 if (PageHuge(src)) {
570
571 struct hstate *h = page_hstate(src);
572 nr_pages = pages_per_huge_page(h);
573
574 if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
575 __copy_gigantic_page(dst, src, nr_pages);
576 return;
577 }
578 } else {
579
580 BUG_ON(!PageTransHuge(src));
581 nr_pages = hpage_nr_pages(src);
582 }
583
584 for (i = 0; i < nr_pages; i++) {
585 cond_resched();
586 copy_highpage(dst + i, src + i);
587 }
588}
589
590
591
592
593void migrate_page_states(struct page *newpage, struct page *page)
594{
595 int cpupid;
596
597 if (PageError(page))
598 SetPageError(newpage);
599 if (PageReferenced(page))
600 SetPageReferenced(newpage);
601 if (PageUptodate(page))
602 SetPageUptodate(newpage);
603 if (TestClearPageActive(page)) {
604 VM_BUG_ON_PAGE(PageUnevictable(page), page);
605 SetPageActive(newpage);
606 } else if (TestClearPageUnevictable(page))
607 SetPageUnevictable(newpage);
608 if (PageWorkingset(page))
609 SetPageWorkingset(newpage);
610 if (PageChecked(page))
611 SetPageChecked(newpage);
612 if (PageMappedToDisk(page))
613 SetPageMappedToDisk(newpage);
614
615
616 if (PageDirty(page))
617 SetPageDirty(newpage);
618
619 if (page_is_young(page))
620 set_page_young(newpage);
621 if (page_is_idle(page))
622 set_page_idle(newpage);
623
624
625
626
627
628 cpupid = page_cpupid_xchg_last(page, -1);
629 page_cpupid_xchg_last(newpage, cpupid);
630
631 ksm_migrate_page(newpage, page);
632
633
634
635
636 if (PageSwapCache(page))
637 ClearPageSwapCache(page);
638 ClearPagePrivate(page);
639 set_page_private(page, 0);
640
641
642
643
644
645 if (PageWriteback(newpage))
646 end_page_writeback(newpage);
647
648 copy_page_owner(page, newpage);
649
650 mem_cgroup_migrate(page, newpage);
651}
652EXPORT_SYMBOL(migrate_page_states);
653
654void migrate_page_copy(struct page *newpage, struct page *page)
655{
656 if (PageHuge(page) || PageTransHuge(page))
657 copy_huge_page(newpage, page);
658 else
659 copy_highpage(newpage, page);
660
661 migrate_page_states(newpage, page);
662}
663EXPORT_SYMBOL(migrate_page_copy);
664
665
666
667
668
669
670
671
672
673
674
675int migrate_page(struct address_space *mapping,
676 struct page *newpage, struct page *page,
677 enum migrate_mode mode)
678{
679 int rc;
680
681 BUG_ON(PageWriteback(page));
682
683 rc = migrate_page_move_mapping(mapping, newpage, page, 0);
684
685 if (rc != MIGRATEPAGE_SUCCESS)
686 return rc;
687
688 if (mode != MIGRATE_SYNC_NO_COPY)
689 migrate_page_copy(newpage, page);
690 else
691 migrate_page_states(newpage, page);
692 return MIGRATEPAGE_SUCCESS;
693}
694EXPORT_SYMBOL(migrate_page);
695
696#ifdef CONFIG_BLOCK
697
698static bool buffer_migrate_lock_buffers(struct buffer_head *head,
699 enum migrate_mode mode)
700{
701 struct buffer_head *bh = head;
702
703
704 if (mode != MIGRATE_ASYNC) {
705 do {
706 lock_buffer(bh);
707 bh = bh->b_this_page;
708
709 } while (bh != head);
710
711 return true;
712 }
713
714
715 do {
716 if (!trylock_buffer(bh)) {
717
718
719
720
721 struct buffer_head *failed_bh = bh;
722 bh = head;
723 while (bh != failed_bh) {
724 unlock_buffer(bh);
725 bh = bh->b_this_page;
726 }
727 return false;
728 }
729
730 bh = bh->b_this_page;
731 } while (bh != head);
732 return true;
733}
734
735static int __buffer_migrate_page(struct address_space *mapping,
736 struct page *newpage, struct page *page, enum migrate_mode mode,
737 bool check_refs)
738{
739 struct buffer_head *bh, *head;
740 int rc;
741 int expected_count;
742
743 if (!page_has_buffers(page))
744 return migrate_page(mapping, newpage, page, mode);
745
746
747 expected_count = expected_page_refs(mapping, page);
748 if (page_count(page) != expected_count)
749 return -EAGAIN;
750
751 head = page_buffers(page);
752 if (!buffer_migrate_lock_buffers(head, mode))
753 return -EAGAIN;
754
755 if (check_refs) {
756 bool busy;
757 bool invalidated = false;
758
759recheck_buffers:
760 busy = false;
761 spin_lock(&mapping->private_lock);
762 bh = head;
763 do {
764 if (atomic_read(&bh->b_count)) {
765 busy = true;
766 break;
767 }
768 bh = bh->b_this_page;
769 } while (bh != head);
770 if (busy) {
771 if (invalidated) {
772 rc = -EAGAIN;
773 goto unlock_buffers;
774 }
775 spin_unlock(&mapping->private_lock);
776 invalidate_bh_lrus();
777 invalidated = true;
778 goto recheck_buffers;
779 }
780 }
781
782 rc = migrate_page_move_mapping(mapping, newpage, page, 0);
783 if (rc != MIGRATEPAGE_SUCCESS)
784 goto unlock_buffers;
785
786 ClearPagePrivate(page);
787 set_page_private(newpage, page_private(page));
788 set_page_private(page, 0);
789 put_page(page);
790 get_page(newpage);
791
792 bh = head;
793 do {
794 set_bh_page(bh, newpage, bh_offset(bh));
795 bh = bh->b_this_page;
796
797 } while (bh != head);
798
799 SetPagePrivate(newpage);
800
801 if (mode != MIGRATE_SYNC_NO_COPY)
802 migrate_page_copy(newpage, page);
803 else
804 migrate_page_states(newpage, page);
805
806 rc = MIGRATEPAGE_SUCCESS;
807unlock_buffers:
808 if (check_refs)
809 spin_unlock(&mapping->private_lock);
810 bh = head;
811 do {
812 unlock_buffer(bh);
813 bh = bh->b_this_page;
814
815 } while (bh != head);
816
817 return rc;
818}
819
820
821
822
823
824
825int buffer_migrate_page(struct address_space *mapping,
826 struct page *newpage, struct page *page, enum migrate_mode mode)
827{
828 return __buffer_migrate_page(mapping, newpage, page, mode, false);
829}
830EXPORT_SYMBOL(buffer_migrate_page);
831
832
833
834
835
836
837
838int buffer_migrate_page_norefs(struct address_space *mapping,
839 struct page *newpage, struct page *page, enum migrate_mode mode)
840{
841 return __buffer_migrate_page(mapping, newpage, page, mode, true);
842}
843#endif
844
845
846
847
848static int writeout(struct address_space *mapping, struct page *page)
849{
850 struct writeback_control wbc = {
851 .sync_mode = WB_SYNC_NONE,
852 .nr_to_write = 1,
853 .range_start = 0,
854 .range_end = LLONG_MAX,
855 .for_reclaim = 1
856 };
857 int rc;
858
859 if (!mapping->a_ops->writepage)
860
861 return -EINVAL;
862
863 if (!clear_page_dirty_for_io(page))
864
865 return -EAGAIN;
866
867
868
869
870
871
872
873
874
875 remove_migration_ptes(page, page, false);
876
877 rc = mapping->a_ops->writepage(page, &wbc);
878
879 if (rc != AOP_WRITEPAGE_ACTIVATE)
880
881 lock_page(page);
882
883 return (rc < 0) ? -EIO : -EAGAIN;
884}
885
886
887
888
889static int fallback_migrate_page(struct address_space *mapping,
890 struct page *newpage, struct page *page, enum migrate_mode mode)
891{
892 if (PageDirty(page)) {
893
894 switch (mode) {
895 case MIGRATE_SYNC:
896 case MIGRATE_SYNC_NO_COPY:
897 break;
898 default:
899 return -EBUSY;
900 }
901 return writeout(mapping, page);
902 }
903
904
905
906
907
908 if (page_has_private(page) &&
909 !try_to_release_page(page, GFP_KERNEL))
910 return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
911
912 return migrate_page(mapping, newpage, page, mode);
913}
914
915
916
917
918
919
920
921
922
923
924
925
926static int move_to_new_page(struct page *newpage, struct page *page,
927 enum migrate_mode mode)
928{
929 struct address_space *mapping;
930 int rc = -EAGAIN;
931 bool is_lru = !__PageMovable(page);
932
933 VM_BUG_ON_PAGE(!PageLocked(page), page);
934 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
935
936 mapping = page_mapping(page);
937
938 if (likely(is_lru)) {
939 if (!mapping)
940 rc = migrate_page(mapping, newpage, page, mode);
941 else if (mapping->a_ops->migratepage)
942
943
944
945
946
947
948
949 rc = mapping->a_ops->migratepage(mapping, newpage,
950 page, mode);
951 else
952 rc = fallback_migrate_page(mapping, newpage,
953 page, mode);
954 } else {
955
956
957
958
959 VM_BUG_ON_PAGE(!PageIsolated(page), page);
960 if (!PageMovable(page)) {
961 rc = MIGRATEPAGE_SUCCESS;
962 __ClearPageIsolated(page);
963 goto out;
964 }
965
966 rc = mapping->a_ops->migratepage(mapping, newpage,
967 page, mode);
968 WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
969 !PageIsolated(page));
970 }
971
972
973
974
975
976 if (rc == MIGRATEPAGE_SUCCESS) {
977 if (__PageMovable(page)) {
978 VM_BUG_ON_PAGE(!PageIsolated(page), page);
979
980
981
982
983
984 __ClearPageIsolated(page);
985 }
986
987
988
989
990
991
992 if (!PageMappingFlags(page))
993 page->mapping = NULL;
994
995 if (likely(!is_zone_device_page(newpage)))
996 flush_dcache_page(newpage);
997
998 }
999out:
1000 return rc;
1001}
1002
1003static int __unmap_and_move(struct page *page, struct page *newpage,
1004 int force, enum migrate_mode mode)
1005{
1006 int rc = -EAGAIN;
1007 int page_was_mapped = 0;
1008 struct anon_vma *anon_vma = NULL;
1009 bool is_lru = !__PageMovable(page);
1010
1011 if (!trylock_page(page)) {
1012 if (!force || mode == MIGRATE_ASYNC)
1013 goto out;
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028 if (current->flags & PF_MEMALLOC)
1029 goto out;
1030
1031 lock_page(page);
1032 }
1033
1034 if (PageWriteback(page)) {
1035
1036
1037
1038
1039
1040
1041 switch (mode) {
1042 case MIGRATE_SYNC:
1043 case MIGRATE_SYNC_NO_COPY:
1044 break;
1045 default:
1046 rc = -EBUSY;
1047 goto out_unlock;
1048 }
1049 if (!force)
1050 goto out_unlock;
1051 wait_on_page_writeback(page);
1052 }
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068 if (PageAnon(page) && !PageKsm(page))
1069 anon_vma = page_get_anon_vma(page);
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079 if (unlikely(!trylock_page(newpage)))
1080 goto out_unlock;
1081
1082 if (unlikely(!is_lru)) {
1083 rc = move_to_new_page(newpage, page, mode);
1084 goto out_unlock_both;
1085 }
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099 if (!page->mapping) {
1100 VM_BUG_ON_PAGE(PageAnon(page), page);
1101 if (page_has_private(page)) {
1102 try_to_free_buffers(page);
1103 goto out_unlock_both;
1104 }
1105 } else if (page_mapped(page)) {
1106
1107 VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
1108 page);
1109 try_to_unmap(page,
1110 TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
1111 page_was_mapped = 1;
1112 }
1113
1114 if (!page_mapped(page))
1115 rc = move_to_new_page(newpage, page, mode);
1116
1117 if (page_was_mapped)
1118 remove_migration_ptes(page,
1119 rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
1120
1121out_unlock_both:
1122 unlock_page(newpage);
1123out_unlock:
1124
1125 if (anon_vma)
1126 put_anon_vma(anon_vma);
1127 unlock_page(page);
1128out:
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138 if (rc == MIGRATEPAGE_SUCCESS) {
1139 if (unlikely(!is_lru))
1140 put_page(newpage);
1141 else
1142 putback_lru_page(newpage);
1143 }
1144
1145 return rc;
1146}
1147
1148
1149
1150
1151
1152#if defined(CONFIG_ARM) && \
1153 defined(GCC_VERSION) && GCC_VERSION < 40900 && GCC_VERSION >= 40700
1154#define ICE_noinline noinline
1155#else
1156#define ICE_noinline
1157#endif
1158
1159
1160
1161
1162
1163static ICE_noinline int unmap_and_move(new_page_t get_new_page,
1164 free_page_t put_new_page,
1165 unsigned long private, struct page *page,
1166 int force, enum migrate_mode mode,
1167 enum migrate_reason reason)
1168{
1169 int rc = MIGRATEPAGE_SUCCESS;
1170 struct page *newpage;
1171
1172 if (!thp_migration_supported() && PageTransHuge(page))
1173 return -ENOMEM;
1174
1175 newpage = get_new_page(page, private);
1176 if (!newpage)
1177 return -ENOMEM;
1178
1179 if (page_count(page) == 1) {
1180
1181 ClearPageActive(page);
1182 ClearPageUnevictable(page);
1183 if (unlikely(__PageMovable(page))) {
1184 lock_page(page);
1185 if (!PageMovable(page))
1186 __ClearPageIsolated(page);
1187 unlock_page(page);
1188 }
1189 if (put_new_page)
1190 put_new_page(newpage, private);
1191 else
1192 put_page(newpage);
1193 goto out;
1194 }
1195
1196 rc = __unmap_and_move(page, newpage, force, mode);
1197 if (rc == MIGRATEPAGE_SUCCESS)
1198 set_page_owner_migrate_reason(newpage, reason);
1199
1200out:
1201 if (rc != -EAGAIN) {
1202
1203
1204
1205
1206
1207
1208 list_del(&page->lru);
1209
1210
1211
1212
1213
1214
1215 if (likely(!__PageMovable(page)))
1216 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
1217 page_is_file_cache(page), -hpage_nr_pages(page));
1218 }
1219
1220
1221
1222
1223
1224
1225 if (rc == MIGRATEPAGE_SUCCESS) {
1226 put_page(page);
1227 if (reason == MR_MEMORY_FAILURE) {
1228
1229
1230
1231
1232
1233 if (set_hwpoison_free_buddy_page(page))
1234 num_poisoned_pages_inc();
1235 }
1236 } else {
1237 if (rc != -EAGAIN) {
1238 if (likely(!__PageMovable(page))) {
1239 putback_lru_page(page);
1240 goto put_new;
1241 }
1242
1243 lock_page(page);
1244 if (PageMovable(page))
1245 putback_movable_page(page);
1246 else
1247 __ClearPageIsolated(page);
1248 unlock_page(page);
1249 put_page(page);
1250 }
1251put_new:
1252 if (put_new_page)
1253 put_new_page(newpage, private);
1254 else
1255 put_page(newpage);
1256 }
1257
1258 return rc;
1259}
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279static int unmap_and_move_huge_page(new_page_t get_new_page,
1280 free_page_t put_new_page, unsigned long private,
1281 struct page *hpage, int force,
1282 enum migrate_mode mode, int reason)
1283{
1284 int rc = -EAGAIN;
1285 int page_was_mapped = 0;
1286 struct page *new_hpage;
1287 struct anon_vma *anon_vma = NULL;
1288
1289
1290
1291
1292
1293
1294
1295
1296 if (!hugepage_migration_supported(page_hstate(hpage))) {
1297 putback_active_hugepage(hpage);
1298 return -ENOSYS;
1299 }
1300
1301 new_hpage = get_new_page(hpage, private);
1302 if (!new_hpage)
1303 return -ENOMEM;
1304
1305 if (!trylock_page(hpage)) {
1306 if (!force)
1307 goto out;
1308 switch (mode) {
1309 case MIGRATE_SYNC:
1310 case MIGRATE_SYNC_NO_COPY:
1311 break;
1312 default:
1313 goto out;
1314 }
1315 lock_page(hpage);
1316 }
1317
1318
1319
1320
1321
1322
1323 if (page_private(hpage) && !page_mapping(hpage)) {
1324 rc = -EBUSY;
1325 goto out_unlock;
1326 }
1327
1328 if (PageAnon(hpage))
1329 anon_vma = page_get_anon_vma(hpage);
1330
1331 if (unlikely(!trylock_page(new_hpage)))
1332 goto put_anon;
1333
1334 if (page_mapped(hpage)) {
1335 try_to_unmap(hpage,
1336 TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
1337 page_was_mapped = 1;
1338 }
1339
1340 if (!page_mapped(hpage))
1341 rc = move_to_new_page(new_hpage, hpage, mode);
1342
1343 if (page_was_mapped)
1344 remove_migration_ptes(hpage,
1345 rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
1346
1347 unlock_page(new_hpage);
1348
1349put_anon:
1350 if (anon_vma)
1351 put_anon_vma(anon_vma);
1352
1353 if (rc == MIGRATEPAGE_SUCCESS) {
1354 move_hugetlb_state(hpage, new_hpage, reason);
1355 put_new_page = NULL;
1356 }
1357
1358out_unlock:
1359 unlock_page(hpage);
1360out:
1361 if (rc != -EAGAIN)
1362 putback_active_hugepage(hpage);
1363
1364
1365
1366
1367
1368
1369 if (put_new_page)
1370 put_new_page(new_hpage, private);
1371 else
1372 putback_active_hugepage(new_hpage);
1373
1374 return rc;
1375}
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398int migrate_pages(struct list_head *from, new_page_t get_new_page,
1399 free_page_t put_new_page, unsigned long private,
1400 enum migrate_mode mode, int reason)
1401{
1402 int retry = 1;
1403 int nr_failed = 0;
1404 int nr_succeeded = 0;
1405 int pass = 0;
1406 struct page *page;
1407 struct page *page2;
1408 int swapwrite = current->flags & PF_SWAPWRITE;
1409 int rc;
1410
1411 if (!swapwrite)
1412 current->flags |= PF_SWAPWRITE;
1413
1414 for(pass = 0; pass < 10 && retry; pass++) {
1415 retry = 0;
1416
1417 list_for_each_entry_safe(page, page2, from, lru) {
1418retry:
1419 cond_resched();
1420
1421 if (PageHuge(page))
1422 rc = unmap_and_move_huge_page(get_new_page,
1423 put_new_page, private, page,
1424 pass > 2, mode, reason);
1425 else
1426 rc = unmap_and_move(get_new_page, put_new_page,
1427 private, page, pass > 2, mode,
1428 reason);
1429
1430 switch(rc) {
1431 case -ENOMEM:
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443 if (PageTransHuge(page) && !PageHuge(page)) {
1444 lock_page(page);
1445 rc = split_huge_page_to_list(page, from);
1446 unlock_page(page);
1447 if (!rc) {
1448 list_safe_reset_next(page, page2, lru);
1449 goto retry;
1450 }
1451 }
1452 nr_failed++;
1453 goto out;
1454 case -EAGAIN:
1455 retry++;
1456 break;
1457 case MIGRATEPAGE_SUCCESS:
1458 nr_succeeded++;
1459 break;
1460 default:
1461
1462
1463
1464
1465
1466
1467 nr_failed++;
1468 break;
1469 }
1470 }
1471 }
1472 nr_failed += retry;
1473 rc = nr_failed;
1474out:
1475 if (nr_succeeded)
1476 count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
1477 if (nr_failed)
1478 count_vm_events(PGMIGRATE_FAIL, nr_failed);
1479 trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
1480
1481 if (!swapwrite)
1482 current->flags &= ~PF_SWAPWRITE;
1483
1484 return rc;
1485}
1486
1487#ifdef CONFIG_NUMA
1488
1489static int store_status(int __user *status, int start, int value, int nr)
1490{
1491 while (nr-- > 0) {
1492 if (put_user(value, status + start))
1493 return -EFAULT;
1494 start++;
1495 }
1496
1497 return 0;
1498}
1499
1500static int do_move_pages_to_node(struct mm_struct *mm,
1501 struct list_head *pagelist, int node)
1502{
1503 int err;
1504
1505 if (list_empty(pagelist))
1506 return 0;
1507
1508 err = migrate_pages(pagelist, alloc_new_node_page, NULL, node,
1509 MIGRATE_SYNC, MR_SYSCALL);
1510 if (err)
1511 putback_movable_pages(pagelist);
1512 return err;
1513}
1514
1515
1516
1517
1518
1519
1520
1521
1522static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
1523 int node, struct list_head *pagelist, bool migrate_all)
1524{
1525 struct vm_area_struct *vma;
1526 struct page *page;
1527 unsigned int follflags;
1528 int err;
1529
1530 down_read(&mm->mmap_sem);
1531 err = -EFAULT;
1532 vma = find_vma(mm, addr);
1533 if (!vma || addr < vma->vm_start || !vma_migratable(vma))
1534 goto out;
1535
1536
1537 follflags = FOLL_GET | FOLL_DUMP;
1538 page = follow_page(vma, addr, follflags);
1539
1540 err = PTR_ERR(page);
1541 if (IS_ERR(page))
1542 goto out;
1543
1544 err = -ENOENT;
1545 if (!page)
1546 goto out;
1547
1548 err = 0;
1549 if (page_to_nid(page) == node)
1550 goto out_putpage;
1551
1552 err = -EACCES;
1553 if (page_mapcount(page) > 1 && !migrate_all)
1554 goto out_putpage;
1555
1556 if (PageHuge(page)) {
1557 if (PageHead(page)) {
1558 isolate_huge_page(page, pagelist);
1559 err = 0;
1560 }
1561 } else {
1562 struct page *head;
1563
1564 head = compound_head(page);
1565 err = isolate_lru_page(head);
1566 if (err)
1567 goto out_putpage;
1568
1569 err = 0;
1570 list_add_tail(&head->lru, pagelist);
1571 mod_node_page_state(page_pgdat(head),
1572 NR_ISOLATED_ANON + page_is_file_cache(head),
1573 hpage_nr_pages(head));
1574 }
1575out_putpage:
1576
1577
1578
1579
1580
1581 put_page(page);
1582out:
1583 up_read(&mm->mmap_sem);
1584 return err;
1585}
1586
1587
1588
1589
1590
1591static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
1592 unsigned long nr_pages,
1593 const void __user * __user *pages,
1594 const int __user *nodes,
1595 int __user *status, int flags)
1596{
1597 int current_node = NUMA_NO_NODE;
1598 LIST_HEAD(pagelist);
1599 int start, i;
1600 int err = 0, err1;
1601
1602 migrate_prep();
1603
1604 for (i = start = 0; i < nr_pages; i++) {
1605 const void __user *p;
1606 unsigned long addr;
1607 int node;
1608
1609 err = -EFAULT;
1610 if (get_user(p, pages + i))
1611 goto out_flush;
1612 if (get_user(node, nodes + i))
1613 goto out_flush;
1614 addr = (unsigned long)p;
1615
1616 err = -ENODEV;
1617 if (node < 0 || node >= MAX_NUMNODES)
1618 goto out_flush;
1619 if (!node_state(node, N_MEMORY))
1620 goto out_flush;
1621
1622 err = -EACCES;
1623 if (!node_isset(node, task_nodes))
1624 goto out_flush;
1625
1626 if (current_node == NUMA_NO_NODE) {
1627 current_node = node;
1628 start = i;
1629 } else if (node != current_node) {
1630 err = do_move_pages_to_node(mm, &pagelist, current_node);
1631 if (err)
1632 goto out;
1633 err = store_status(status, start, current_node, i - start);
1634 if (err)
1635 goto out;
1636 start = i;
1637 current_node = node;
1638 }
1639
1640
1641
1642
1643
1644 err = add_page_for_migration(mm, addr, current_node,
1645 &pagelist, flags & MPOL_MF_MOVE_ALL);
1646 if (!err)
1647 continue;
1648
1649 err = store_status(status, i, err, 1);
1650 if (err)
1651 goto out_flush;
1652
1653 err = do_move_pages_to_node(mm, &pagelist, current_node);
1654 if (err)
1655 goto out;
1656 if (i > start) {
1657 err = store_status(status, start, current_node, i - start);
1658 if (err)
1659 goto out;
1660 }
1661 current_node = NUMA_NO_NODE;
1662 }
1663out_flush:
1664 if (list_empty(&pagelist))
1665 return err;
1666
1667
1668 err1 = do_move_pages_to_node(mm, &pagelist, current_node);
1669 if (!err1)
1670 err1 = store_status(status, start, current_node, i - start);
1671 if (!err)
1672 err = err1;
1673out:
1674 return err;
1675}
1676
1677
1678
1679
1680static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
1681 const void __user **pages, int *status)
1682{
1683 unsigned long i;
1684
1685 down_read(&mm->mmap_sem);
1686
1687 for (i = 0; i < nr_pages; i++) {
1688 unsigned long addr = (unsigned long)(*pages);
1689 struct vm_area_struct *vma;
1690 struct page *page;
1691 int err = -EFAULT;
1692
1693 vma = find_vma(mm, addr);
1694 if (!vma || addr < vma->vm_start)
1695 goto set_status;
1696
1697
1698 page = follow_page(vma, addr, FOLL_DUMP);
1699
1700 err = PTR_ERR(page);
1701 if (IS_ERR(page))
1702 goto set_status;
1703
1704 err = page ? page_to_nid(page) : -ENOENT;
1705set_status:
1706 *status = err;
1707
1708 pages++;
1709 status++;
1710 }
1711
1712 up_read(&mm->mmap_sem);
1713}
1714
1715
1716
1717
1718
1719static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
1720 const void __user * __user *pages,
1721 int __user *status)
1722{
1723#define DO_PAGES_STAT_CHUNK_NR 16
1724 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1725 int chunk_status[DO_PAGES_STAT_CHUNK_NR];
1726
1727 while (nr_pages) {
1728 unsigned long chunk_nr;
1729
1730 chunk_nr = nr_pages;
1731 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1732 chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1733
1734 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1735 break;
1736
1737 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1738
1739 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1740 break;
1741
1742 pages += chunk_nr;
1743 status += chunk_nr;
1744 nr_pages -= chunk_nr;
1745 }
1746 return nr_pages ? -EFAULT : 0;
1747}
1748
1749
1750
1751
1752
1753static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
1754 const void __user * __user *pages,
1755 const int __user *nodes,
1756 int __user *status, int flags)
1757{
1758 struct task_struct *task;
1759 struct mm_struct *mm;
1760 int err;
1761 nodemask_t task_nodes;
1762
1763
1764 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1765 return -EINVAL;
1766
1767 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1768 return -EPERM;
1769
1770
1771 rcu_read_lock();
1772 task = pid ? find_task_by_vpid(pid) : current;
1773 if (!task) {
1774 rcu_read_unlock();
1775 return -ESRCH;
1776 }
1777 get_task_struct(task);
1778
1779
1780
1781
1782
1783 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
1784 rcu_read_unlock();
1785 err = -EPERM;
1786 goto out;
1787 }
1788 rcu_read_unlock();
1789
1790 err = security_task_movememory(task);
1791 if (err)
1792 goto out;
1793
1794 task_nodes = cpuset_mems_allowed(task);
1795 mm = get_task_mm(task);
1796 put_task_struct(task);
1797
1798 if (!mm)
1799 return -EINVAL;
1800
1801 if (nodes)
1802 err = do_pages_move(mm, task_nodes, nr_pages, pages,
1803 nodes, status, flags);
1804 else
1805 err = do_pages_stat(mm, nr_pages, pages, status);
1806
1807 mmput(mm);
1808 return err;
1809
1810out:
1811 put_task_struct(task);
1812 return err;
1813}
1814
1815SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1816 const void __user * __user *, pages,
1817 const int __user *, nodes,
1818 int __user *, status, int, flags)
1819{
1820 return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1821}
1822
1823#ifdef CONFIG_COMPAT
1824COMPAT_SYSCALL_DEFINE6(move_pages, pid_t, pid, compat_ulong_t, nr_pages,
1825 compat_uptr_t __user *, pages32,
1826 const int __user *, nodes,
1827 int __user *, status,
1828 int, flags)
1829{
1830 const void __user * __user *pages;
1831 int i;
1832
1833 pages = compat_alloc_user_space(nr_pages * sizeof(void *));
1834 for (i = 0; i < nr_pages; i++) {
1835 compat_uptr_t p;
1836
1837 if (get_user(p, pages32 + i) ||
1838 put_user(compat_ptr(p), pages + i))
1839 return -EFAULT;
1840 }
1841 return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
1842}
1843#endif
1844
1845#ifdef CONFIG_NUMA_BALANCING
1846
1847
1848
1849
1850static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
1851 unsigned long nr_migrate_pages)
1852{
1853 int z;
1854
1855 for (z = pgdat->nr_zones - 1; z >= 0; z--) {
1856 struct zone *zone = pgdat->node_zones + z;
1857
1858 if (!populated_zone(zone))
1859 continue;
1860
1861
1862 if (!zone_watermark_ok(zone, 0,
1863 high_wmark_pages(zone) +
1864 nr_migrate_pages,
1865 0, 0))
1866 continue;
1867 return true;
1868 }
1869 return false;
1870}
1871
1872static struct page *alloc_misplaced_dst_page(struct page *page,
1873 unsigned long data)
1874{
1875 int nid = (int) data;
1876 struct page *newpage;
1877
1878 newpage = __alloc_pages_node(nid,
1879 (GFP_HIGHUSER_MOVABLE |
1880 __GFP_THISNODE | __GFP_NOMEMALLOC |
1881 __GFP_NORETRY | __GFP_NOWARN) &
1882 ~__GFP_RECLAIM, 0);
1883
1884 return newpage;
1885}
1886
1887static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
1888{
1889 int page_lru;
1890
1891 VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
1892
1893
1894 if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page)))
1895 return 0;
1896
1897 if (isolate_lru_page(page))
1898 return 0;
1899
1900
1901
1902
1903
1904
1905
1906
1907 if (PageTransHuge(page) && page_count(page) != 3) {
1908 putback_lru_page(page);
1909 return 0;
1910 }
1911
1912 page_lru = page_is_file_cache(page);
1913 mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
1914 hpage_nr_pages(page));
1915
1916
1917
1918
1919
1920
1921 put_page(page);
1922 return 1;
1923}
1924
1925bool pmd_trans_migrating(pmd_t pmd)
1926{
1927 struct page *page = pmd_page(pmd);
1928 return PageLocked(page);
1929}
1930
1931
1932
1933
1934
1935
1936int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
1937 int node)
1938{
1939 pg_data_t *pgdat = NODE_DATA(node);
1940 int isolated;
1941 int nr_remaining;
1942 LIST_HEAD(migratepages);
1943
1944
1945
1946
1947
1948 if (page_mapcount(page) != 1 && page_is_file_cache(page) &&
1949 (vma->vm_flags & VM_EXEC))
1950 goto out;
1951
1952
1953
1954
1955
1956 if (page_is_file_cache(page) && PageDirty(page))
1957 goto out;
1958
1959 isolated = numamigrate_isolate_page(pgdat, page);
1960 if (!isolated)
1961 goto out;
1962
1963 list_add(&page->lru, &migratepages);
1964 nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
1965 NULL, node, MIGRATE_ASYNC,
1966 MR_NUMA_MISPLACED);
1967 if (nr_remaining) {
1968 if (!list_empty(&migratepages)) {
1969 list_del(&page->lru);
1970 dec_node_page_state(page, NR_ISOLATED_ANON +
1971 page_is_file_cache(page));
1972 putback_lru_page(page);
1973 }
1974 isolated = 0;
1975 } else
1976 count_vm_numa_event(NUMA_PAGE_MIGRATE);
1977 BUG_ON(!list_empty(&migratepages));
1978 return isolated;
1979
1980out:
1981 put_page(page);
1982 return 0;
1983}
1984#endif
1985
1986#if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1987
1988
1989
1990
1991int migrate_misplaced_transhuge_page(struct mm_struct *mm,
1992 struct vm_area_struct *vma,
1993 pmd_t *pmd, pmd_t entry,
1994 unsigned long address,
1995 struct page *page, int node)
1996{
1997 spinlock_t *ptl;
1998 pg_data_t *pgdat = NODE_DATA(node);
1999 int isolated = 0;
2000 struct page *new_page = NULL;
2001 int page_lru = page_is_file_cache(page);
2002 unsigned long start = address & HPAGE_PMD_MASK;
2003
2004 new_page = alloc_pages_node(node,
2005 (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
2006 HPAGE_PMD_ORDER);
2007 if (!new_page)
2008 goto out_fail;
2009 prep_transhuge_page(new_page);
2010
2011 isolated = numamigrate_isolate_page(pgdat, page);
2012 if (!isolated) {
2013 put_page(new_page);
2014 goto out_fail;
2015 }
2016
2017
2018 __SetPageLocked(new_page);
2019 if (PageSwapBacked(page))
2020 __SetPageSwapBacked(new_page);
2021
2022
2023 new_page->mapping = page->mapping;
2024 new_page->index = page->index;
2025
2026 flush_cache_range(vma, start, start + HPAGE_PMD_SIZE);
2027 migrate_page_copy(new_page, page);
2028 WARN_ON(PageLRU(new_page));
2029
2030
2031 ptl = pmd_lock(mm, pmd);
2032 if (unlikely(!pmd_same(*pmd, entry) || !page_ref_freeze(page, 2))) {
2033 spin_unlock(ptl);
2034
2035
2036 if (TestClearPageActive(new_page))
2037 SetPageActive(page);
2038 if (TestClearPageUnevictable(new_page))
2039 SetPageUnevictable(page);
2040
2041 unlock_page(new_page);
2042 put_page(new_page);
2043
2044
2045 get_page(page);
2046 putback_lru_page(page);
2047 mod_node_page_state(page_pgdat(page),
2048 NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
2049
2050 goto out_unlock;
2051 }
2052
2053 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
2054 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064 page_add_anon_rmap(new_page, vma, start, true);
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076 set_pmd_at(mm, start, pmd, entry);
2077 update_mmu_cache_pmd(vma, address, &entry);
2078
2079 page_ref_unfreeze(page, 2);
2080 mlock_migrate_page(new_page, page);
2081 page_remove_rmap(page, true);
2082 set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
2083
2084 spin_unlock(ptl);
2085
2086
2087 get_page(new_page);
2088 putback_lru_page(new_page);
2089
2090 unlock_page(new_page);
2091 unlock_page(page);
2092 put_page(page);
2093 put_page(page);
2094
2095 count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR);
2096 count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
2097
2098 mod_node_page_state(page_pgdat(page),
2099 NR_ISOLATED_ANON + page_lru,
2100 -HPAGE_PMD_NR);
2101 return isolated;
2102
2103out_fail:
2104 count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
2105 ptl = pmd_lock(mm, pmd);
2106 if (pmd_same(*pmd, entry)) {
2107 entry = pmd_modify(entry, vma->vm_page_prot);
2108 set_pmd_at(mm, start, pmd, entry);
2109 update_mmu_cache_pmd(vma, address, &entry);
2110 }
2111 spin_unlock(ptl);
2112
2113out_unlock:
2114 unlock_page(page);
2115 put_page(page);
2116 return 0;
2117}
2118#endif
2119
2120#endif
2121
2122#if defined(CONFIG_MIGRATE_VMA_HELPER)
2123struct migrate_vma {
2124 struct vm_area_struct *vma;
2125 unsigned long *dst;
2126 unsigned long *src;
2127 unsigned long cpages;
2128 unsigned long npages;
2129 unsigned long start;
2130 unsigned long end;
2131};
2132
2133static int migrate_vma_collect_hole(unsigned long start,
2134 unsigned long end,
2135 struct mm_walk *walk)
2136{
2137 struct migrate_vma *migrate = walk->private;
2138 unsigned long addr;
2139
2140 for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
2141 migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
2142 migrate->dst[migrate->npages] = 0;
2143 migrate->npages++;
2144 migrate->cpages++;
2145 }
2146
2147 return 0;
2148}
2149
2150static int migrate_vma_collect_skip(unsigned long start,
2151 unsigned long end,
2152 struct mm_walk *walk)
2153{
2154 struct migrate_vma *migrate = walk->private;
2155 unsigned long addr;
2156
2157 for (addr = start & PAGE_MASK; addr < end; addr += PAGE_SIZE) {
2158 migrate->dst[migrate->npages] = 0;
2159 migrate->src[migrate->npages++] = 0;
2160 }
2161
2162 return 0;
2163}
2164
2165static int migrate_vma_collect_pmd(pmd_t *pmdp,
2166 unsigned long start,
2167 unsigned long end,
2168 struct mm_walk *walk)
2169{
2170 struct migrate_vma *migrate = walk->private;
2171 struct vm_area_struct *vma = walk->vma;
2172 struct mm_struct *mm = vma->vm_mm;
2173 unsigned long addr = start, unmapped = 0;
2174 spinlock_t *ptl;
2175 pte_t *ptep;
2176
2177again:
2178 if (pmd_none(*pmdp))
2179 return migrate_vma_collect_hole(start, end, walk);
2180
2181 if (pmd_trans_huge(*pmdp)) {
2182 struct page *page;
2183
2184 ptl = pmd_lock(mm, pmdp);
2185 if (unlikely(!pmd_trans_huge(*pmdp))) {
2186 spin_unlock(ptl);
2187 goto again;
2188 }
2189
2190 page = pmd_page(*pmdp);
2191 if (is_huge_zero_page(page)) {
2192 spin_unlock(ptl);
2193 split_huge_pmd(vma, pmdp, addr);
2194 if (pmd_trans_unstable(pmdp))
2195 return migrate_vma_collect_skip(start, end,
2196 walk);
2197 } else {
2198 int ret;
2199
2200 get_page(page);
2201 spin_unlock(ptl);
2202 if (unlikely(!trylock_page(page)))
2203 return migrate_vma_collect_skip(start, end,
2204 walk);
2205 ret = split_huge_page(page);
2206 unlock_page(page);
2207 put_page(page);
2208 if (ret)
2209 return migrate_vma_collect_skip(start, end,
2210 walk);
2211 if (pmd_none(*pmdp))
2212 return migrate_vma_collect_hole(start, end,
2213 walk);
2214 }
2215 }
2216
2217 if (unlikely(pmd_bad(*pmdp)))
2218 return migrate_vma_collect_skip(start, end, walk);
2219
2220 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2221 arch_enter_lazy_mmu_mode();
2222
2223 for (; addr < end; addr += PAGE_SIZE, ptep++) {
2224 unsigned long mpfn, pfn;
2225 struct page *page;
2226 swp_entry_t entry;
2227 pte_t pte;
2228
2229 pte = *ptep;
2230 pfn = pte_pfn(pte);
2231
2232 if (pte_none(pte)) {
2233 mpfn = MIGRATE_PFN_MIGRATE;
2234 migrate->cpages++;
2235 pfn = 0;
2236 goto next;
2237 }
2238
2239 if (!pte_present(pte)) {
2240 mpfn = pfn = 0;
2241
2242
2243
2244
2245
2246
2247 entry = pte_to_swp_entry(pte);
2248 if (!is_device_private_entry(entry))
2249 goto next;
2250
2251 page = device_private_entry_to_page(entry);
2252 mpfn = migrate_pfn(page_to_pfn(page))|
2253 MIGRATE_PFN_DEVICE | MIGRATE_PFN_MIGRATE;
2254 if (is_write_device_private_entry(entry))
2255 mpfn |= MIGRATE_PFN_WRITE;
2256 } else {
2257 if (is_zero_pfn(pfn)) {
2258 mpfn = MIGRATE_PFN_MIGRATE;
2259 migrate->cpages++;
2260 pfn = 0;
2261 goto next;
2262 }
2263 page = vm_normal_page(migrate->vma, addr, pte);
2264 mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
2265 mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
2266 }
2267
2268
2269 if (!page || !page->mapping || PageTransCompound(page)) {
2270 mpfn = pfn = 0;
2271 goto next;
2272 }
2273 pfn = page_to_pfn(page);
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284 get_page(page);
2285 migrate->cpages++;
2286
2287
2288
2289
2290
2291
2292 if (trylock_page(page)) {
2293 pte_t swp_pte;
2294
2295 mpfn |= MIGRATE_PFN_LOCKED;
2296 ptep_get_and_clear(mm, addr, ptep);
2297
2298
2299 entry = make_migration_entry(page, mpfn &
2300 MIGRATE_PFN_WRITE);
2301 swp_pte = swp_entry_to_pte(entry);
2302 if (pte_soft_dirty(pte))
2303 swp_pte = pte_swp_mksoft_dirty(swp_pte);
2304 set_pte_at(mm, addr, ptep, swp_pte);
2305
2306
2307
2308
2309
2310
2311 page_remove_rmap(page, false);
2312 put_page(page);
2313
2314 if (pte_present(pte))
2315 unmapped++;
2316 }
2317
2318next:
2319 migrate->dst[migrate->npages] = 0;
2320 migrate->src[migrate->npages++] = mpfn;
2321 }
2322 arch_leave_lazy_mmu_mode();
2323 pte_unmap_unlock(ptep - 1, ptl);
2324
2325
2326 if (unmapped)
2327 flush_tlb_range(walk->vma, start, end);
2328
2329 return 0;
2330}
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340static void migrate_vma_collect(struct migrate_vma *migrate)
2341{
2342 struct mmu_notifier_range range;
2343 struct mm_walk mm_walk = {
2344 .pmd_entry = migrate_vma_collect_pmd,
2345 .pte_hole = migrate_vma_collect_hole,
2346 .vma = migrate->vma,
2347 .mm = migrate->vma->vm_mm,
2348 .private = migrate,
2349 };
2350
2351 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, NULL, mm_walk.mm,
2352 migrate->start,
2353 migrate->end);
2354 mmu_notifier_invalidate_range_start(&range);
2355 walk_page_range(migrate->start, migrate->end, &mm_walk);
2356 mmu_notifier_invalidate_range_end(&range);
2357
2358 migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
2359}
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369static bool migrate_vma_check_page(struct page *page)
2370{
2371
2372
2373
2374
2375
2376 int extra = 1;
2377
2378
2379
2380
2381
2382
2383 if (PageCompound(page))
2384 return false;
2385
2386
2387 if (is_zone_device_page(page)) {
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401 return is_device_private_page(page);
2402 }
2403
2404
2405 if (page_mapping(page))
2406 extra += 1 + page_has_private(page);
2407
2408 if ((page_count(page) - extra) > page_mapcount(page))
2409 return false;
2410
2411 return true;
2412}
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423static void migrate_vma_prepare(struct migrate_vma *migrate)
2424{
2425 const unsigned long npages = migrate->npages;
2426 const unsigned long start = migrate->start;
2427 unsigned long addr, i, restore = 0;
2428 bool allow_drain = true;
2429
2430 lru_add_drain();
2431
2432 for (i = 0; (i < npages) && migrate->cpages; i++) {
2433 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2434 bool remap = true;
2435
2436 if (!page)
2437 continue;
2438
2439 if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
2440
2441
2442
2443
2444
2445
2446
2447
2448 if (!trylock_page(page)) {
2449 migrate->src[i] = 0;
2450 migrate->cpages--;
2451 put_page(page);
2452 continue;
2453 }
2454 remap = false;
2455 migrate->src[i] |= MIGRATE_PFN_LOCKED;
2456 }
2457
2458
2459 if (!is_zone_device_page(page)) {
2460 if (!PageLRU(page) && allow_drain) {
2461
2462 lru_add_drain_all();
2463 allow_drain = false;
2464 }
2465
2466 if (isolate_lru_page(page)) {
2467 if (remap) {
2468 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2469 migrate->cpages--;
2470 restore++;
2471 } else {
2472 migrate->src[i] = 0;
2473 unlock_page(page);
2474 migrate->cpages--;
2475 put_page(page);
2476 }
2477 continue;
2478 }
2479
2480
2481 put_page(page);
2482 }
2483
2484 if (!migrate_vma_check_page(page)) {
2485 if (remap) {
2486 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2487 migrate->cpages--;
2488 restore++;
2489
2490 if (!is_zone_device_page(page)) {
2491 get_page(page);
2492 putback_lru_page(page);
2493 }
2494 } else {
2495 migrate->src[i] = 0;
2496 unlock_page(page);
2497 migrate->cpages--;
2498
2499 if (!is_zone_device_page(page))
2500 putback_lru_page(page);
2501 else
2502 put_page(page);
2503 }
2504 }
2505 }
2506
2507 for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) {
2508 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2509
2510 if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2511 continue;
2512
2513 remove_migration_pte(page, migrate->vma, addr, page);
2514
2515 migrate->src[i] = 0;
2516 unlock_page(page);
2517 put_page(page);
2518 restore--;
2519 }
2520}
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533static void migrate_vma_unmap(struct migrate_vma *migrate)
2534{
2535 int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS;
2536 const unsigned long npages = migrate->npages;
2537 const unsigned long start = migrate->start;
2538 unsigned long addr, i, restore = 0;
2539
2540 for (i = 0; i < npages; i++) {
2541 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2542
2543 if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
2544 continue;
2545
2546 if (page_mapped(page)) {
2547 try_to_unmap(page, flags);
2548 if (page_mapped(page))
2549 goto restore;
2550 }
2551
2552 if (migrate_vma_check_page(page))
2553 continue;
2554
2555restore:
2556 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2557 migrate->cpages--;
2558 restore++;
2559 }
2560
2561 for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) {
2562 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2563
2564 if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
2565 continue;
2566
2567 remove_migration_ptes(page, page, false);
2568
2569 migrate->src[i] = 0;
2570 unlock_page(page);
2571 restore--;
2572
2573 if (is_zone_device_page(page))
2574 put_page(page);
2575 else
2576 putback_lru_page(page);
2577 }
2578}
2579
2580static void migrate_vma_insert_page(struct migrate_vma *migrate,
2581 unsigned long addr,
2582 struct page *page,
2583 unsigned long *src,
2584 unsigned long *dst)
2585{
2586 struct vm_area_struct *vma = migrate->vma;
2587 struct mm_struct *mm = vma->vm_mm;
2588 struct mem_cgroup *memcg;
2589 bool flush = false;
2590 spinlock_t *ptl;
2591 pte_t entry;
2592 pgd_t *pgdp;
2593 p4d_t *p4dp;
2594 pud_t *pudp;
2595 pmd_t *pmdp;
2596 pte_t *ptep;
2597
2598
2599 if (!vma_is_anonymous(vma))
2600 goto abort;
2601
2602 pgdp = pgd_offset(mm, addr);
2603 p4dp = p4d_alloc(mm, pgdp, addr);
2604 if (!p4dp)
2605 goto abort;
2606 pudp = pud_alloc(mm, p4dp, addr);
2607 if (!pudp)
2608 goto abort;
2609 pmdp = pmd_alloc(mm, pudp, addr);
2610 if (!pmdp)
2611 goto abort;
2612
2613 if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
2614 goto abort;
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626 if (pte_alloc(mm, pmdp))
2627 goto abort;
2628
2629
2630 if (unlikely(pmd_trans_unstable(pmdp)))
2631 goto abort;
2632
2633 if (unlikely(anon_vma_prepare(vma)))
2634 goto abort;
2635 if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg, false))
2636 goto abort;
2637
2638
2639
2640
2641
2642
2643 __SetPageUptodate(page);
2644
2645 if (is_zone_device_page(page)) {
2646 if (is_device_private_page(page)) {
2647 swp_entry_t swp_entry;
2648
2649 swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
2650 entry = swp_entry_to_pte(swp_entry);
2651 }
2652 } else {
2653 entry = mk_pte(page, vma->vm_page_prot);
2654 if (vma->vm_flags & VM_WRITE)
2655 entry = pte_mkwrite(pte_mkdirty(entry));
2656 }
2657
2658 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
2659
2660 if (pte_present(*ptep)) {
2661 unsigned long pfn = pte_pfn(*ptep);
2662
2663 if (!is_zero_pfn(pfn)) {
2664 pte_unmap_unlock(ptep, ptl);
2665 mem_cgroup_cancel_charge(page, memcg, false);
2666 goto abort;
2667 }
2668 flush = true;
2669 } else if (!pte_none(*ptep)) {
2670 pte_unmap_unlock(ptep, ptl);
2671 mem_cgroup_cancel_charge(page, memcg, false);
2672 goto abort;
2673 }
2674
2675
2676
2677
2678
2679 if (userfaultfd_missing(vma)) {
2680 pte_unmap_unlock(ptep, ptl);
2681 mem_cgroup_cancel_charge(page, memcg, false);
2682 goto abort;
2683 }
2684
2685 inc_mm_counter(mm, MM_ANONPAGES);
2686 page_add_new_anon_rmap(page, vma, addr, false);
2687 mem_cgroup_commit_charge(page, memcg, false, false);
2688 if (!is_zone_device_page(page))
2689 lru_cache_add_active_or_unevictable(page, vma);
2690 get_page(page);
2691
2692 if (flush) {
2693 flush_cache_page(vma, addr, pte_pfn(*ptep));
2694 ptep_clear_flush_notify(vma, addr, ptep);
2695 set_pte_at_notify(mm, addr, ptep, entry);
2696 update_mmu_cache(vma, addr, ptep);
2697 } else {
2698
2699 set_pte_at(mm, addr, ptep, entry);
2700 update_mmu_cache(vma, addr, ptep);
2701 }
2702
2703 pte_unmap_unlock(ptep, ptl);
2704 *src = MIGRATE_PFN_MIGRATE;
2705 return;
2706
2707abort:
2708 *src &= ~MIGRATE_PFN_MIGRATE;
2709}
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719static void migrate_vma_pages(struct migrate_vma *migrate)
2720{
2721 const unsigned long npages = migrate->npages;
2722 const unsigned long start = migrate->start;
2723 struct mmu_notifier_range range;
2724 unsigned long addr, i;
2725 bool notified = false;
2726
2727 for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
2728 struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2729 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2730 struct address_space *mapping;
2731 int r;
2732
2733 if (!newpage) {
2734 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2735 continue;
2736 }
2737
2738 if (!page) {
2739 if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE)) {
2740 continue;
2741 }
2742 if (!notified) {
2743 notified = true;
2744
2745 mmu_notifier_range_init(&range,
2746 MMU_NOTIFY_CLEAR, 0,
2747 NULL,
2748 migrate->vma->vm_mm,
2749 addr, migrate->end);
2750 mmu_notifier_invalidate_range_start(&range);
2751 }
2752 migrate_vma_insert_page(migrate, addr, newpage,
2753 &migrate->src[i],
2754 &migrate->dst[i]);
2755 continue;
2756 }
2757
2758 mapping = page_mapping(page);
2759
2760 if (is_zone_device_page(newpage)) {
2761 if (is_device_private_page(newpage)) {
2762
2763
2764
2765
2766 if (mapping) {
2767 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2768 continue;
2769 }
2770 } else {
2771
2772
2773
2774
2775 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2776 continue;
2777 }
2778 }
2779
2780 r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
2781 if (r != MIGRATEPAGE_SUCCESS)
2782 migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
2783 }
2784
2785
2786
2787
2788
2789
2790 if (notified)
2791 mmu_notifier_invalidate_range_only_end(&range);
2792}
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805static void migrate_vma_finalize(struct migrate_vma *migrate)
2806{
2807 const unsigned long npages = migrate->npages;
2808 unsigned long i;
2809
2810 for (i = 0; i < npages; i++) {
2811 struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
2812 struct page *page = migrate_pfn_to_page(migrate->src[i]);
2813
2814 if (!page) {
2815 if (newpage) {
2816 unlock_page(newpage);
2817 put_page(newpage);
2818 }
2819 continue;
2820 }
2821
2822 if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
2823 if (newpage) {
2824 unlock_page(newpage);
2825 put_page(newpage);
2826 }
2827 newpage = page;
2828 }
2829
2830 remove_migration_ptes(page, newpage, false);
2831 unlock_page(page);
2832 migrate->cpages--;
2833
2834 if (is_zone_device_page(page))
2835 put_page(page);
2836 else
2837 putback_lru_page(page);
2838
2839 if (newpage != page) {
2840 unlock_page(newpage);
2841 if (is_zone_device_page(newpage))
2842 put_page(newpage);
2843 else
2844 putback_lru_page(newpage);
2845 }
2846 }
2847}
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900int migrate_vma(const struct migrate_vma_ops *ops,
2901 struct vm_area_struct *vma,
2902 unsigned long start,
2903 unsigned long end,
2904 unsigned long *src,
2905 unsigned long *dst,
2906 void *private)
2907{
2908 struct migrate_vma migrate;
2909
2910
2911 start &= PAGE_MASK;
2912 end &= PAGE_MASK;
2913 if (!vma || is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL) ||
2914 vma_is_dax(vma))
2915 return -EINVAL;
2916 if (start < vma->vm_start || start >= vma->vm_end)
2917 return -EINVAL;
2918 if (end <= vma->vm_start || end > vma->vm_end)
2919 return -EINVAL;
2920 if (!ops || !src || !dst || start >= end)
2921 return -EINVAL;
2922
2923 memset(src, 0, sizeof(*src) * ((end - start) >> PAGE_SHIFT));
2924 migrate.src = src;
2925 migrate.dst = dst;
2926 migrate.start = start;
2927 migrate.npages = 0;
2928 migrate.cpages = 0;
2929 migrate.end = end;
2930 migrate.vma = vma;
2931
2932
2933 migrate_vma_collect(&migrate);
2934 if (!migrate.cpages)
2935 return 0;
2936
2937
2938 migrate_vma_prepare(&migrate);
2939 if (!migrate.cpages)
2940 return 0;
2941
2942
2943 migrate_vma_unmap(&migrate);
2944 if (!migrate.cpages)
2945 return 0;
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955 ops->alloc_and_copy(vma, src, dst, start, end, private);
2956
2957
2958 migrate_vma_pages(&migrate);
2959
2960 ops->finalize_and_map(vma, src, dst, start, end, private);
2961
2962
2963 migrate_vma_finalize(&migrate);
2964
2965 return 0;
2966}
2967EXPORT_SYMBOL(migrate_vma);
2968#endif
2969