1
2
3
4
5
6
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/mm.h>
11#include <linux/sched.h>
12#include <linux/sched/coredump.h>
13#include <linux/sched/numa_balancing.h>
14#include <linux/highmem.h>
15#include <linux/hugetlb.h>
16#include <linux/mmu_notifier.h>
17#include <linux/rmap.h>
18#include <linux/swap.h>
19#include <linux/shrinker.h>
20#include <linux/mm_inline.h>
21#include <linux/swapops.h>
22#include <linux/dax.h>
23#include <linux/khugepaged.h>
24#include <linux/freezer.h>
25#include <linux/pfn_t.h>
26#include <linux/mman.h>
27#include <linux/memremap.h>
28#include <linux/pagemap.h>
29#include <linux/debugfs.h>
30#include <linux/migrate.h>
31#include <linux/hashtable.h>
32#include <linux/userfaultfd_k.h>
33#include <linux/page_idle.h>
34#include <linux/shmem_fs.h>
35#include <linux/oom.h>
36
37#include <asm/tlb.h>
38#include <asm/pgalloc.h>
39#include "internal.h"
40
41
42
43
44
45
46
47
48
49unsigned long transparent_hugepage_flags __read_mostly =
50#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
51 (1<<TRANSPARENT_HUGEPAGE_FLAG)|
52#endif
53#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
54 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
55#endif
56 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
57 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
58 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
59
60static struct shrinker deferred_split_shrinker;
61
62static atomic_t huge_zero_refcount;
63struct page *huge_zero_page __read_mostly;
64
65static struct page *get_huge_zero_page(void)
66{
67 struct page *zero_page;
68retry:
69 if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
70 return READ_ONCE(huge_zero_page);
71
72 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
73 HPAGE_PMD_ORDER);
74 if (!zero_page) {
75 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
76 return NULL;
77 }
78 count_vm_event(THP_ZERO_PAGE_ALLOC);
79 preempt_disable();
80 if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
81 preempt_enable();
82 __free_pages(zero_page, compound_order(zero_page));
83 goto retry;
84 }
85
86
87 atomic_set(&huge_zero_refcount, 2);
88 preempt_enable();
89 return READ_ONCE(huge_zero_page);
90}
91
92static void put_huge_zero_page(void)
93{
94
95
96
97
98 BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
99}
100
101struct page *mm_get_huge_zero_page(struct mm_struct *mm)
102{
103 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
104 return READ_ONCE(huge_zero_page);
105
106 if (!get_huge_zero_page())
107 return NULL;
108
109 if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
110 put_huge_zero_page();
111
112 return READ_ONCE(huge_zero_page);
113}
114
115void mm_put_huge_zero_page(struct mm_struct *mm)
116{
117 if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
118 put_huge_zero_page();
119}
120
121static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
122 struct shrink_control *sc)
123{
124
125 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
126}
127
128static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
129 struct shrink_control *sc)
130{
131 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
132 struct page *zero_page = xchg(&huge_zero_page, NULL);
133 BUG_ON(zero_page == NULL);
134 __free_pages(zero_page, compound_order(zero_page));
135 return HPAGE_PMD_NR;
136 }
137
138 return 0;
139}
140
141static struct shrinker huge_zero_page_shrinker = {
142 .count_objects = shrink_huge_zero_page_count,
143 .scan_objects = shrink_huge_zero_page_scan,
144 .seeks = DEFAULT_SEEKS,
145};
146
147#ifdef CONFIG_SYSFS
148static ssize_t enabled_show(struct kobject *kobj,
149 struct kobj_attribute *attr, char *buf)
150{
151 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
152 return sprintf(buf, "[always] madvise never\n");
153 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
154 return sprintf(buf, "always [madvise] never\n");
155 else
156 return sprintf(buf, "always madvise [never]\n");
157}
158
159static ssize_t enabled_store(struct kobject *kobj,
160 struct kobj_attribute *attr,
161 const char *buf, size_t count)
162{
163 ssize_t ret = count;
164
165 if (!memcmp("always", buf,
166 min(sizeof("always")-1, count))) {
167 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
168 set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
169 } else if (!memcmp("madvise", buf,
170 min(sizeof("madvise")-1, count))) {
171 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
172 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
173 } else if (!memcmp("never", buf,
174 min(sizeof("never")-1, count))) {
175 clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags);
176 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags);
177 } else
178 ret = -EINVAL;
179
180 if (ret > 0) {
181 int err = start_stop_khugepaged();
182 if (err)
183 ret = err;
184 }
185 return ret;
186}
187static struct kobj_attribute enabled_attr =
188 __ATTR(enabled, 0644, enabled_show, enabled_store);
189
190ssize_t single_hugepage_flag_show(struct kobject *kobj,
191 struct kobj_attribute *attr, char *buf,
192 enum transparent_hugepage_flag flag)
193{
194 return sprintf(buf, "%d\n",
195 !!test_bit(flag, &transparent_hugepage_flags));
196}
197
198ssize_t single_hugepage_flag_store(struct kobject *kobj,
199 struct kobj_attribute *attr,
200 const char *buf, size_t count,
201 enum transparent_hugepage_flag flag)
202{
203 unsigned long value;
204 int ret;
205
206 ret = kstrtoul(buf, 10, &value);
207 if (ret < 0)
208 return ret;
209 if (value > 1)
210 return -EINVAL;
211
212 if (value)
213 set_bit(flag, &transparent_hugepage_flags);
214 else
215 clear_bit(flag, &transparent_hugepage_flags);
216
217 return count;
218}
219
220static ssize_t defrag_show(struct kobject *kobj,
221 struct kobj_attribute *attr, char *buf)
222{
223 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
224 return sprintf(buf, "[always] defer defer+madvise madvise never\n");
225 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
226 return sprintf(buf, "always [defer] defer+madvise madvise never\n");
227 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
228 return sprintf(buf, "always defer [defer+madvise] madvise never\n");
229 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
230 return sprintf(buf, "always defer defer+madvise [madvise] never\n");
231 return sprintf(buf, "always defer defer+madvise madvise [never]\n");
232}
233
234static ssize_t defrag_store(struct kobject *kobj,
235 struct kobj_attribute *attr,
236 const char *buf, size_t count)
237{
238 if (!memcmp("always", buf,
239 min(sizeof("always")-1, count))) {
240 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
241 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
242 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
243 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
244 } else if (!memcmp("defer+madvise", buf,
245 min(sizeof("defer+madvise")-1, count))) {
246 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
247 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
248 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
249 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
250 } else if (!memcmp("defer", buf,
251 min(sizeof("defer")-1, count))) {
252 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
253 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
254 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
255 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
256 } else if (!memcmp("madvise", buf,
257 min(sizeof("madvise")-1, count))) {
258 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
259 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
260 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
261 set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
262 } else if (!memcmp("never", buf,
263 min(sizeof("never")-1, count))) {
264 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
265 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
266 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
267 clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
268 } else
269 return -EINVAL;
270
271 return count;
272}
273static struct kobj_attribute defrag_attr =
274 __ATTR(defrag, 0644, defrag_show, defrag_store);
275
276static ssize_t use_zero_page_show(struct kobject *kobj,
277 struct kobj_attribute *attr, char *buf)
278{
279 return single_hugepage_flag_show(kobj, attr, buf,
280 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
281}
282static ssize_t use_zero_page_store(struct kobject *kobj,
283 struct kobj_attribute *attr, const char *buf, size_t count)
284{
285 return single_hugepage_flag_store(kobj, attr, buf, count,
286 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
287}
288static struct kobj_attribute use_zero_page_attr =
289 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
290
291static ssize_t hpage_pmd_size_show(struct kobject *kobj,
292 struct kobj_attribute *attr, char *buf)
293{
294 return sprintf(buf, "%lu\n", HPAGE_PMD_SIZE);
295}
296static struct kobj_attribute hpage_pmd_size_attr =
297 __ATTR_RO(hpage_pmd_size);
298
299#ifdef CONFIG_DEBUG_VM
300static ssize_t debug_cow_show(struct kobject *kobj,
301 struct kobj_attribute *attr, char *buf)
302{
303 return single_hugepage_flag_show(kobj, attr, buf,
304 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
305}
306static ssize_t debug_cow_store(struct kobject *kobj,
307 struct kobj_attribute *attr,
308 const char *buf, size_t count)
309{
310 return single_hugepage_flag_store(kobj, attr, buf, count,
311 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
312}
313static struct kobj_attribute debug_cow_attr =
314 __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
315#endif
316
317static struct attribute *hugepage_attr[] = {
318 &enabled_attr.attr,
319 &defrag_attr.attr,
320 &use_zero_page_attr.attr,
321 &hpage_pmd_size_attr.attr,
322#if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
323 &shmem_enabled_attr.attr,
324#endif
325#ifdef CONFIG_DEBUG_VM
326 &debug_cow_attr.attr,
327#endif
328 NULL,
329};
330
331static const struct attribute_group hugepage_attr_group = {
332 .attrs = hugepage_attr,
333};
334
335static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
336{
337 int err;
338
339 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
340 if (unlikely(!*hugepage_kobj)) {
341 pr_err("failed to create transparent hugepage kobject\n");
342 return -ENOMEM;
343 }
344
345 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
346 if (err) {
347 pr_err("failed to register transparent hugepage group\n");
348 goto delete_obj;
349 }
350
351 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
352 if (err) {
353 pr_err("failed to register transparent hugepage group\n");
354 goto remove_hp_group;
355 }
356
357 return 0;
358
359remove_hp_group:
360 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
361delete_obj:
362 kobject_put(*hugepage_kobj);
363 return err;
364}
365
366static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
367{
368 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
369 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
370 kobject_put(hugepage_kobj);
371}
372#else
373static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
374{
375 return 0;
376}
377
378static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
379{
380}
381#endif
382
383static int __init hugepage_init(void)
384{
385 int err;
386 struct kobject *hugepage_kobj;
387
388 if (!has_transparent_hugepage()) {
389 transparent_hugepage_flags = 0;
390 return -EINVAL;
391 }
392
393
394
395
396 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER);
397
398
399
400
401 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
402
403 err = hugepage_init_sysfs(&hugepage_kobj);
404 if (err)
405 goto err_sysfs;
406
407 err = khugepaged_init();
408 if (err)
409 goto err_slab;
410
411 err = register_shrinker(&huge_zero_page_shrinker);
412 if (err)
413 goto err_hzp_shrinker;
414 err = register_shrinker(&deferred_split_shrinker);
415 if (err)
416 goto err_split_shrinker;
417
418
419
420
421
422
423 if (totalram_pages < (512 << (20 - PAGE_SHIFT))) {
424 transparent_hugepage_flags = 0;
425 return 0;
426 }
427
428 err = start_stop_khugepaged();
429 if (err)
430 goto err_khugepaged;
431
432 return 0;
433err_khugepaged:
434 unregister_shrinker(&deferred_split_shrinker);
435err_split_shrinker:
436 unregister_shrinker(&huge_zero_page_shrinker);
437err_hzp_shrinker:
438 khugepaged_destroy();
439err_slab:
440 hugepage_exit_sysfs(hugepage_kobj);
441err_sysfs:
442 return err;
443}
444subsys_initcall(hugepage_init);
445
446static int __init setup_transparent_hugepage(char *str)
447{
448 int ret = 0;
449 if (!str)
450 goto out;
451 if (!strcmp(str, "always")) {
452 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
453 &transparent_hugepage_flags);
454 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
455 &transparent_hugepage_flags);
456 ret = 1;
457 } else if (!strcmp(str, "madvise")) {
458 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
459 &transparent_hugepage_flags);
460 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
461 &transparent_hugepage_flags);
462 ret = 1;
463 } else if (!strcmp(str, "never")) {
464 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
465 &transparent_hugepage_flags);
466 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
467 &transparent_hugepage_flags);
468 ret = 1;
469 }
470out:
471 if (!ret)
472 pr_warn("transparent_hugepage= cannot parse, ignored\n");
473 return ret;
474}
475__setup("transparent_hugepage=", setup_transparent_hugepage);
476
477pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
478{
479 if (likely(vma->vm_flags & VM_WRITE))
480 pmd = pmd_mkwrite(pmd);
481 return pmd;
482}
483
484static inline struct list_head *page_deferred_list(struct page *page)
485{
486
487 return &page[2].deferred_list;
488}
489
490void prep_transhuge_page(struct page *page)
491{
492
493
494
495
496
497 INIT_LIST_HEAD(page_deferred_list(page));
498 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
499}
500
501unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long len,
502 loff_t off, unsigned long flags, unsigned long size)
503{
504 unsigned long addr;
505 loff_t off_end = off + len;
506 loff_t off_align = round_up(off, size);
507 unsigned long len_pad;
508
509 if (off_end <= off_align || (off_end - off_align) < size)
510 return 0;
511
512 len_pad = len + size;
513 if (len_pad < len || (off + len_pad) < off)
514 return 0;
515
516 addr = current->mm->get_unmapped_area(filp, 0, len_pad,
517 off >> PAGE_SHIFT, flags);
518 if (IS_ERR_VALUE(addr))
519 return 0;
520
521 addr += (off - addr) & (size - 1);
522 return addr;
523}
524
525unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
526 unsigned long len, unsigned long pgoff, unsigned long flags)
527{
528 loff_t off = (loff_t)pgoff << PAGE_SHIFT;
529
530 if (addr)
531 goto out;
532 if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD))
533 goto out;
534
535 addr = __thp_get_unmapped_area(filp, len, off, flags, PMD_SIZE);
536 if (addr)
537 return addr;
538
539 out:
540 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
541}
542EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
543
544static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf,
545 struct page *page, gfp_t gfp)
546{
547 struct vm_area_struct *vma = vmf->vma;
548 struct mem_cgroup *memcg;
549 pgtable_t pgtable;
550 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
551 vm_fault_t ret = 0;
552
553 VM_BUG_ON_PAGE(!PageCompound(page), page);
554
555 if (mem_cgroup_try_charge_delay(page, vma->vm_mm, gfp, &memcg, true)) {
556 put_page(page);
557 count_vm_event(THP_FAULT_FALLBACK);
558 return VM_FAULT_FALLBACK;
559 }
560
561 pgtable = pte_alloc_one(vma->vm_mm, haddr);
562 if (unlikely(!pgtable)) {
563 ret = VM_FAULT_OOM;
564 goto release;
565 }
566
567 clear_huge_page(page, vmf->address, HPAGE_PMD_NR);
568
569
570
571
572
573 __SetPageUptodate(page);
574
575 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
576 if (unlikely(!pmd_none(*vmf->pmd))) {
577 goto unlock_release;
578 } else {
579 pmd_t entry;
580
581 ret = check_stable_address_space(vma->vm_mm);
582 if (ret)
583 goto unlock_release;
584
585
586 if (userfaultfd_missing(vma)) {
587 vm_fault_t ret2;
588
589 spin_unlock(vmf->ptl);
590 mem_cgroup_cancel_charge(page, memcg, true);
591 put_page(page);
592 pte_free(vma->vm_mm, pgtable);
593 ret2 = handle_userfault(vmf, VM_UFFD_MISSING);
594 VM_BUG_ON(ret2 & VM_FAULT_FALLBACK);
595 return ret2;
596 }
597
598 entry = mk_huge_pmd(page, vma->vm_page_prot);
599 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
600 page_add_new_anon_rmap(page, vma, haddr, true);
601 mem_cgroup_commit_charge(page, memcg, false, true);
602 lru_cache_add_active_or_unevictable(page, vma);
603 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
604 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
605 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
606 mm_inc_nr_ptes(vma->vm_mm);
607 spin_unlock(vmf->ptl);
608 count_vm_event(THP_FAULT_ALLOC);
609 }
610
611 return 0;
612unlock_release:
613 spin_unlock(vmf->ptl);
614release:
615 if (pgtable)
616 pte_free(vma->vm_mm, pgtable);
617 mem_cgroup_cancel_charge(page, memcg, true);
618 put_page(page);
619 return ret;
620
621}
622
623
624
625
626
627
628
629
630
631
632static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
633{
634 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
635
636 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
637 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
638 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
639 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
640 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
641 return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :
642 __GFP_KSWAPD_RECLAIM);
643 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
644 return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM :
645 0);
646 return GFP_TRANSHUGE_LIGHT;
647}
648
649
650static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
651 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
652 struct page *zero_page)
653{
654 pmd_t entry;
655 if (!pmd_none(*pmd))
656 return false;
657 entry = mk_pmd(zero_page, vma->vm_page_prot);
658 entry = pmd_mkhuge(entry);
659 if (pgtable)
660 pgtable_trans_huge_deposit(mm, pmd, pgtable);
661 set_pmd_at(mm, haddr, pmd, entry);
662 mm_inc_nr_ptes(mm);
663 return true;
664}
665
666vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)
667{
668 struct vm_area_struct *vma = vmf->vma;
669 gfp_t gfp;
670 struct page *page;
671 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
672
673 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
674 return VM_FAULT_FALLBACK;
675 if (unlikely(anon_vma_prepare(vma)))
676 return VM_FAULT_OOM;
677 if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
678 return VM_FAULT_OOM;
679 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
680 !mm_forbids_zeropage(vma->vm_mm) &&
681 transparent_hugepage_use_zero_page()) {
682 pgtable_t pgtable;
683 struct page *zero_page;
684 bool set;
685 vm_fault_t ret;
686 pgtable = pte_alloc_one(vma->vm_mm, haddr);
687 if (unlikely(!pgtable))
688 return VM_FAULT_OOM;
689 zero_page = mm_get_huge_zero_page(vma->vm_mm);
690 if (unlikely(!zero_page)) {
691 pte_free(vma->vm_mm, pgtable);
692 count_vm_event(THP_FAULT_FALLBACK);
693 return VM_FAULT_FALLBACK;
694 }
695 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
696 ret = 0;
697 set = false;
698 if (pmd_none(*vmf->pmd)) {
699 ret = check_stable_address_space(vma->vm_mm);
700 if (ret) {
701 spin_unlock(vmf->ptl);
702 } else if (userfaultfd_missing(vma)) {
703 spin_unlock(vmf->ptl);
704 ret = handle_userfault(vmf, VM_UFFD_MISSING);
705 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
706 } else {
707 set_huge_zero_page(pgtable, vma->vm_mm, vma,
708 haddr, vmf->pmd, zero_page);
709 spin_unlock(vmf->ptl);
710 set = true;
711 }
712 } else
713 spin_unlock(vmf->ptl);
714 if (!set)
715 pte_free(vma->vm_mm, pgtable);
716 return ret;
717 }
718 gfp = alloc_hugepage_direct_gfpmask(vma);
719 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
720 if (unlikely(!page)) {
721 count_vm_event(THP_FAULT_FALLBACK);
722 return VM_FAULT_FALLBACK;
723 }
724 prep_transhuge_page(page);
725 return __do_huge_pmd_anonymous_page(vmf, page, gfp);
726}
727
728static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
729 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write,
730 pgtable_t pgtable)
731{
732 struct mm_struct *mm = vma->vm_mm;
733 pmd_t entry;
734 spinlock_t *ptl;
735
736 ptl = pmd_lock(mm, pmd);
737 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
738 if (pfn_t_devmap(pfn))
739 entry = pmd_mkdevmap(entry);
740 if (write) {
741 entry = pmd_mkyoung(pmd_mkdirty(entry));
742 entry = maybe_pmd_mkwrite(entry, vma);
743 }
744
745 if (pgtable) {
746 pgtable_trans_huge_deposit(mm, pmd, pgtable);
747 mm_inc_nr_ptes(mm);
748 }
749
750 set_pmd_at(mm, addr, pmd, entry);
751 update_mmu_cache_pmd(vma, addr, pmd);
752 spin_unlock(ptl);
753}
754
755vm_fault_t vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
756 pmd_t *pmd, pfn_t pfn, bool write)
757{
758 pgprot_t pgprot = vma->vm_page_prot;
759 pgtable_t pgtable = NULL;
760
761
762
763
764
765 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
766 !pfn_t_devmap(pfn));
767 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
768 (VM_PFNMAP|VM_MIXEDMAP));
769 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
770
771 if (addr < vma->vm_start || addr >= vma->vm_end)
772 return VM_FAULT_SIGBUS;
773
774 if (arch_needs_pgtable_deposit()) {
775 pgtable = pte_alloc_one(vma->vm_mm, addr);
776 if (!pgtable)
777 return VM_FAULT_OOM;
778 }
779
780 track_pfn_insert(vma, &pgprot, pfn);
781
782 insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write, pgtable);
783 return VM_FAULT_NOPAGE;
784}
785EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
786
787#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
788static pud_t maybe_pud_mkwrite(pud_t pud, struct vm_area_struct *vma)
789{
790 if (likely(vma->vm_flags & VM_WRITE))
791 pud = pud_mkwrite(pud);
792 return pud;
793}
794
795static void insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
796 pud_t *pud, pfn_t pfn, pgprot_t prot, bool write)
797{
798 struct mm_struct *mm = vma->vm_mm;
799 pud_t entry;
800 spinlock_t *ptl;
801
802 ptl = pud_lock(mm, pud);
803 entry = pud_mkhuge(pfn_t_pud(pfn, prot));
804 if (pfn_t_devmap(pfn))
805 entry = pud_mkdevmap(entry);
806 if (write) {
807 entry = pud_mkyoung(pud_mkdirty(entry));
808 entry = maybe_pud_mkwrite(entry, vma);
809 }
810 set_pud_at(mm, addr, pud, entry);
811 update_mmu_cache_pud(vma, addr, pud);
812 spin_unlock(ptl);
813}
814
815vm_fault_t vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
816 pud_t *pud, pfn_t pfn, bool write)
817{
818 pgprot_t pgprot = vma->vm_page_prot;
819
820
821
822
823
824 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
825 !pfn_t_devmap(pfn));
826 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
827 (VM_PFNMAP|VM_MIXEDMAP));
828 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
829
830 if (addr < vma->vm_start || addr >= vma->vm_end)
831 return VM_FAULT_SIGBUS;
832
833 track_pfn_insert(vma, &pgprot, pfn);
834
835 insert_pfn_pud(vma, addr, pud, pfn, pgprot, write);
836 return VM_FAULT_NOPAGE;
837}
838EXPORT_SYMBOL_GPL(vmf_insert_pfn_pud);
839#endif
840
841static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
842 pmd_t *pmd, int flags)
843{
844 pmd_t _pmd;
845
846 _pmd = pmd_mkyoung(*pmd);
847 if (flags & FOLL_WRITE)
848 _pmd = pmd_mkdirty(_pmd);
849 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
850 pmd, _pmd, flags & FOLL_WRITE))
851 update_mmu_cache_pmd(vma, addr, pmd);
852}
853
854struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
855 pmd_t *pmd, int flags)
856{
857 unsigned long pfn = pmd_pfn(*pmd);
858 struct mm_struct *mm = vma->vm_mm;
859 struct dev_pagemap *pgmap;
860 struct page *page;
861
862 assert_spin_locked(pmd_lockptr(mm, pmd));
863
864
865
866
867
868 WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set");
869
870 if (flags & FOLL_WRITE && !pmd_write(*pmd))
871 return NULL;
872
873 if (pmd_present(*pmd) && pmd_devmap(*pmd))
874 ;
875 else
876 return NULL;
877
878 if (flags & FOLL_TOUCH)
879 touch_pmd(vma, addr, pmd, flags);
880
881
882
883
884
885 if (!(flags & FOLL_GET))
886 return ERR_PTR(-EEXIST);
887
888 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
889 pgmap = get_dev_pagemap(pfn, NULL);
890 if (!pgmap)
891 return ERR_PTR(-EFAULT);
892 page = pfn_to_page(pfn);
893 get_page(page);
894 put_dev_pagemap(pgmap);
895
896 return page;
897}
898
899int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
900 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
901 struct vm_area_struct *vma)
902{
903 spinlock_t *dst_ptl, *src_ptl;
904 struct page *src_page;
905 pmd_t pmd;
906 pgtable_t pgtable = NULL;
907 int ret = -ENOMEM;
908
909
910 if (!vma_is_anonymous(vma))
911 return 0;
912
913 pgtable = pte_alloc_one(dst_mm, addr);
914 if (unlikely(!pgtable))
915 goto out;
916
917 dst_ptl = pmd_lock(dst_mm, dst_pmd);
918 src_ptl = pmd_lockptr(src_mm, src_pmd);
919 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
920
921 ret = -EAGAIN;
922 pmd = *src_pmd;
923
924#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
925 if (unlikely(is_swap_pmd(pmd))) {
926 swp_entry_t entry = pmd_to_swp_entry(pmd);
927
928 VM_BUG_ON(!is_pmd_migration_entry(pmd));
929 if (is_write_migration_entry(entry)) {
930 make_migration_entry_read(&entry);
931 pmd = swp_entry_to_pmd(entry);
932 if (pmd_swp_soft_dirty(*src_pmd))
933 pmd = pmd_swp_mksoft_dirty(pmd);
934 set_pmd_at(src_mm, addr, src_pmd, pmd);
935 }
936 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
937 mm_inc_nr_ptes(dst_mm);
938 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
939 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
940 ret = 0;
941 goto out_unlock;
942 }
943#endif
944
945 if (unlikely(!pmd_trans_huge(pmd))) {
946 pte_free(dst_mm, pgtable);
947 goto out_unlock;
948 }
949
950
951
952
953
954 if (is_huge_zero_pmd(pmd)) {
955 struct page *zero_page;
956
957
958
959
960
961 zero_page = mm_get_huge_zero_page(dst_mm);
962 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
963 zero_page);
964 ret = 0;
965 goto out_unlock;
966 }
967
968 src_page = pmd_page(pmd);
969 VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
970 get_page(src_page);
971 page_dup_rmap(src_page, true);
972 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
973 mm_inc_nr_ptes(dst_mm);
974 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
975
976 pmdp_set_wrprotect(src_mm, addr, src_pmd);
977 pmd = pmd_mkold(pmd_wrprotect(pmd));
978 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
979
980 ret = 0;
981out_unlock:
982 spin_unlock(src_ptl);
983 spin_unlock(dst_ptl);
984out:
985 return ret;
986}
987
988#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
989static void touch_pud(struct vm_area_struct *vma, unsigned long addr,
990 pud_t *pud, int flags)
991{
992 pud_t _pud;
993
994 _pud = pud_mkyoung(*pud);
995 if (flags & FOLL_WRITE)
996 _pud = pud_mkdirty(_pud);
997 if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK,
998 pud, _pud, flags & FOLL_WRITE))
999 update_mmu_cache_pud(vma, addr, pud);
1000}
1001
1002struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
1003 pud_t *pud, int flags)
1004{
1005 unsigned long pfn = pud_pfn(*pud);
1006 struct mm_struct *mm = vma->vm_mm;
1007 struct dev_pagemap *pgmap;
1008 struct page *page;
1009
1010 assert_spin_locked(pud_lockptr(mm, pud));
1011
1012 if (flags & FOLL_WRITE && !pud_write(*pud))
1013 return NULL;
1014
1015 if (pud_present(*pud) && pud_devmap(*pud))
1016 ;
1017 else
1018 return NULL;
1019
1020 if (flags & FOLL_TOUCH)
1021 touch_pud(vma, addr, pud, flags);
1022
1023
1024
1025
1026
1027 if (!(flags & FOLL_GET))
1028 return ERR_PTR(-EEXIST);
1029
1030 pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
1031 pgmap = get_dev_pagemap(pfn, NULL);
1032 if (!pgmap)
1033 return ERR_PTR(-EFAULT);
1034 page = pfn_to_page(pfn);
1035 get_page(page);
1036 put_dev_pagemap(pgmap);
1037
1038 return page;
1039}
1040
1041int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1042 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
1043 struct vm_area_struct *vma)
1044{
1045 spinlock_t *dst_ptl, *src_ptl;
1046 pud_t pud;
1047 int ret;
1048
1049 dst_ptl = pud_lock(dst_mm, dst_pud);
1050 src_ptl = pud_lockptr(src_mm, src_pud);
1051 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
1052
1053 ret = -EAGAIN;
1054 pud = *src_pud;
1055 if (unlikely(!pud_trans_huge(pud) && !pud_devmap(pud)))
1056 goto out_unlock;
1057
1058
1059
1060
1061
1062
1063 if (is_huge_zero_pud(pud)) {
1064
1065 }
1066
1067 pudp_set_wrprotect(src_mm, addr, src_pud);
1068 pud = pud_mkold(pud_wrprotect(pud));
1069 set_pud_at(dst_mm, addr, dst_pud, pud);
1070
1071 ret = 0;
1072out_unlock:
1073 spin_unlock(src_ptl);
1074 spin_unlock(dst_ptl);
1075 return ret;
1076}
1077
1078void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
1079{
1080 pud_t entry;
1081 unsigned long haddr;
1082 bool write = vmf->flags & FAULT_FLAG_WRITE;
1083
1084 vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud);
1085 if (unlikely(!pud_same(*vmf->pud, orig_pud)))
1086 goto unlock;
1087
1088 entry = pud_mkyoung(orig_pud);
1089 if (write)
1090 entry = pud_mkdirty(entry);
1091 haddr = vmf->address & HPAGE_PUD_MASK;
1092 if (pudp_set_access_flags(vmf->vma, haddr, vmf->pud, entry, write))
1093 update_mmu_cache_pud(vmf->vma, vmf->address, vmf->pud);
1094
1095unlock:
1096 spin_unlock(vmf->ptl);
1097}
1098#endif
1099
1100void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd)
1101{
1102 pmd_t entry;
1103 unsigned long haddr;
1104 bool write = vmf->flags & FAULT_FLAG_WRITE;
1105
1106 vmf->ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
1107 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1108 goto unlock;
1109
1110 entry = pmd_mkyoung(orig_pmd);
1111 if (write)
1112 entry = pmd_mkdirty(entry);
1113 haddr = vmf->address & HPAGE_PMD_MASK;
1114 if (pmdp_set_access_flags(vmf->vma, haddr, vmf->pmd, entry, write))
1115 update_mmu_cache_pmd(vmf->vma, vmf->address, vmf->pmd);
1116
1117unlock:
1118 spin_unlock(vmf->ptl);
1119}
1120
1121static vm_fault_t do_huge_pmd_wp_page_fallback(struct vm_fault *vmf,
1122 pmd_t orig_pmd, struct page *page)
1123{
1124 struct vm_area_struct *vma = vmf->vma;
1125 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1126 struct mem_cgroup *memcg;
1127 pgtable_t pgtable;
1128 pmd_t _pmd;
1129 int i;
1130 vm_fault_t ret = 0;
1131 struct page **pages;
1132 unsigned long mmun_start;
1133 unsigned long mmun_end;
1134
1135 pages = kmalloc_array(HPAGE_PMD_NR, sizeof(struct page *),
1136 GFP_KERNEL);
1137 if (unlikely(!pages)) {
1138 ret |= VM_FAULT_OOM;
1139 goto out;
1140 }
1141
1142 for (i = 0; i < HPAGE_PMD_NR; i++) {
1143 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE, vma,
1144 vmf->address, page_to_nid(page));
1145 if (unlikely(!pages[i] ||
1146 mem_cgroup_try_charge_delay(pages[i], vma->vm_mm,
1147 GFP_KERNEL, &memcg, false))) {
1148 if (pages[i])
1149 put_page(pages[i]);
1150 while (--i >= 0) {
1151 memcg = (void *)page_private(pages[i]);
1152 set_page_private(pages[i], 0);
1153 mem_cgroup_cancel_charge(pages[i], memcg,
1154 false);
1155 put_page(pages[i]);
1156 }
1157 kfree(pages);
1158 ret |= VM_FAULT_OOM;
1159 goto out;
1160 }
1161 set_page_private(pages[i], (unsigned long)memcg);
1162 }
1163
1164 for (i = 0; i < HPAGE_PMD_NR; i++) {
1165 copy_user_highpage(pages[i], page + i,
1166 haddr + PAGE_SIZE * i, vma);
1167 __SetPageUptodate(pages[i]);
1168 cond_resched();
1169 }
1170
1171 mmun_start = haddr;
1172 mmun_end = haddr + HPAGE_PMD_SIZE;
1173 mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
1174
1175 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1176 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1177 goto out_free_pages;
1178 VM_BUG_ON_PAGE(!PageHead(page), page);
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188 pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
1189
1190 pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, vmf->pmd);
1191 pmd_populate(vma->vm_mm, &_pmd, pgtable);
1192
1193 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
1194 pte_t entry;
1195 entry = mk_pte(pages[i], vma->vm_page_prot);
1196 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
1197 memcg = (void *)page_private(pages[i]);
1198 set_page_private(pages[i], 0);
1199 page_add_new_anon_rmap(pages[i], vmf->vma, haddr, false);
1200 mem_cgroup_commit_charge(pages[i], memcg, false, false);
1201 lru_cache_add_active_or_unevictable(pages[i], vma);
1202 vmf->pte = pte_offset_map(&_pmd, haddr);
1203 VM_BUG_ON(!pte_none(*vmf->pte));
1204 set_pte_at(vma->vm_mm, haddr, vmf->pte, entry);
1205 pte_unmap(vmf->pte);
1206 }
1207 kfree(pages);
1208
1209 smp_wmb();
1210 pmd_populate(vma->vm_mm, vmf->pmd, pgtable);
1211 page_remove_rmap(page, true);
1212 spin_unlock(vmf->ptl);
1213
1214
1215
1216
1217
1218 mmu_notifier_invalidate_range_only_end(vma->vm_mm, mmun_start,
1219 mmun_end);
1220
1221 ret |= VM_FAULT_WRITE;
1222 put_page(page);
1223
1224out:
1225 return ret;
1226
1227out_free_pages:
1228 spin_unlock(vmf->ptl);
1229 mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
1230 for (i = 0; i < HPAGE_PMD_NR; i++) {
1231 memcg = (void *)page_private(pages[i]);
1232 set_page_private(pages[i], 0);
1233 mem_cgroup_cancel_charge(pages[i], memcg, false);
1234 put_page(pages[i]);
1235 }
1236 kfree(pages);
1237 goto out;
1238}
1239
1240vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd)
1241{
1242 struct vm_area_struct *vma = vmf->vma;
1243 struct page *page = NULL, *new_page;
1244 struct mem_cgroup *memcg;
1245 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1246 unsigned long mmun_start;
1247 unsigned long mmun_end;
1248 gfp_t huge_gfp;
1249 vm_fault_t ret = 0;
1250
1251 vmf->ptl = pmd_lockptr(vma->vm_mm, vmf->pmd);
1252 VM_BUG_ON_VMA(!vma->anon_vma, vma);
1253 if (is_huge_zero_pmd(orig_pmd))
1254 goto alloc;
1255 spin_lock(vmf->ptl);
1256 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd)))
1257 goto out_unlock;
1258
1259 page = pmd_page(orig_pmd);
1260 VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
1261
1262
1263
1264
1265 if (!trylock_page(page)) {
1266 get_page(page);
1267 spin_unlock(vmf->ptl);
1268 lock_page(page);
1269 spin_lock(vmf->ptl);
1270 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
1271 unlock_page(page);
1272 put_page(page);
1273 goto out_unlock;
1274 }
1275 put_page(page);
1276 }
1277 if (reuse_swap_page(page, NULL)) {
1278 pmd_t entry;
1279 entry = pmd_mkyoung(orig_pmd);
1280 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1281 if (pmdp_set_access_flags(vma, haddr, vmf->pmd, entry, 1))
1282 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1283 ret |= VM_FAULT_WRITE;
1284 unlock_page(page);
1285 goto out_unlock;
1286 }
1287 unlock_page(page);
1288 get_page(page);
1289 spin_unlock(vmf->ptl);
1290alloc:
1291 if (transparent_hugepage_enabled(vma) &&
1292 !transparent_hugepage_debug_cow()) {
1293 huge_gfp = alloc_hugepage_direct_gfpmask(vma);
1294 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1295 } else
1296 new_page = NULL;
1297
1298 if (likely(new_page)) {
1299 prep_transhuge_page(new_page);
1300 } else {
1301 if (!page) {
1302 split_huge_pmd(vma, vmf->pmd, vmf->address);
1303 ret |= VM_FAULT_FALLBACK;
1304 } else {
1305 ret = do_huge_pmd_wp_page_fallback(vmf, orig_pmd, page);
1306 if (ret & VM_FAULT_OOM) {
1307 split_huge_pmd(vma, vmf->pmd, vmf->address);
1308 ret |= VM_FAULT_FALLBACK;
1309 }
1310 put_page(page);
1311 }
1312 count_vm_event(THP_FAULT_FALLBACK);
1313 goto out;
1314 }
1315
1316 if (unlikely(mem_cgroup_try_charge_delay(new_page, vma->vm_mm,
1317 huge_gfp, &memcg, true))) {
1318 put_page(new_page);
1319 split_huge_pmd(vma, vmf->pmd, vmf->address);
1320 if (page)
1321 put_page(page);
1322 ret |= VM_FAULT_FALLBACK;
1323 count_vm_event(THP_FAULT_FALLBACK);
1324 goto out;
1325 }
1326
1327 count_vm_event(THP_FAULT_ALLOC);
1328
1329 if (!page)
1330 clear_huge_page(new_page, vmf->address, HPAGE_PMD_NR);
1331 else
1332 copy_user_huge_page(new_page, page, vmf->address,
1333 vma, HPAGE_PMD_NR);
1334 __SetPageUptodate(new_page);
1335
1336 mmun_start = haddr;
1337 mmun_end = haddr + HPAGE_PMD_SIZE;
1338 mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
1339
1340 spin_lock(vmf->ptl);
1341 if (page)
1342 put_page(page);
1343 if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) {
1344 spin_unlock(vmf->ptl);
1345 mem_cgroup_cancel_charge(new_page, memcg, true);
1346 put_page(new_page);
1347 goto out_mn;
1348 } else {
1349 pmd_t entry;
1350 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
1351 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1352 pmdp_huge_clear_flush_notify(vma, haddr, vmf->pmd);
1353 page_add_new_anon_rmap(new_page, vma, haddr, true);
1354 mem_cgroup_commit_charge(new_page, memcg, false, true);
1355 lru_cache_add_active_or_unevictable(new_page, vma);
1356 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
1357 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1358 if (!page) {
1359 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1360 } else {
1361 VM_BUG_ON_PAGE(!PageHead(page), page);
1362 page_remove_rmap(page, true);
1363 put_page(page);
1364 }
1365 ret |= VM_FAULT_WRITE;
1366 }
1367 spin_unlock(vmf->ptl);
1368out_mn:
1369
1370
1371
1372
1373 mmu_notifier_invalidate_range_only_end(vma->vm_mm, mmun_start,
1374 mmun_end);
1375out:
1376 return ret;
1377out_unlock:
1378 spin_unlock(vmf->ptl);
1379 return ret;
1380}
1381
1382
1383
1384
1385
1386static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)
1387{
1388 return pmd_write(pmd) ||
1389 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));
1390}
1391
1392struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1393 unsigned long addr,
1394 pmd_t *pmd,
1395 unsigned int flags)
1396{
1397 struct mm_struct *mm = vma->vm_mm;
1398 struct page *page = NULL;
1399
1400 assert_spin_locked(pmd_lockptr(mm, pmd));
1401
1402 if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))
1403 goto out;
1404
1405
1406 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1407 return ERR_PTR(-EFAULT);
1408
1409
1410 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
1411 goto out;
1412
1413 page = pmd_page(*pmd);
1414 VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
1415 if (flags & FOLL_TOUCH)
1416 touch_pmd(vma, addr, pmd, flags);
1417 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439 if (PageAnon(page) && compound_mapcount(page) != 1)
1440 goto skip_mlock;
1441 if (PageDoubleMap(page) || !page->mapping)
1442 goto skip_mlock;
1443 if (!trylock_page(page))
1444 goto skip_mlock;
1445 lru_add_drain();
1446 if (page->mapping && !PageDoubleMap(page))
1447 mlock_vma_page(page);
1448 unlock_page(page);
1449 }
1450skip_mlock:
1451 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1452 VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
1453 if (flags & FOLL_GET)
1454 get_page(page);
1455
1456out:
1457 return page;
1458}
1459
1460
1461vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
1462{
1463 struct vm_area_struct *vma = vmf->vma;
1464 struct anon_vma *anon_vma = NULL;
1465 struct page *page;
1466 unsigned long haddr = vmf->address & HPAGE_PMD_MASK;
1467 int page_nid = -1, this_nid = numa_node_id();
1468 int target_nid, last_cpupid = -1;
1469 bool page_locked;
1470 bool migrated = false;
1471 bool was_writable;
1472 int flags = 0;
1473
1474 vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
1475 if (unlikely(!pmd_same(pmd, *vmf->pmd)))
1476 goto out_unlock;
1477
1478
1479
1480
1481
1482
1483 if (unlikely(pmd_trans_migrating(*vmf->pmd))) {
1484 page = pmd_page(*vmf->pmd);
1485 if (!get_page_unless_zero(page))
1486 goto out_unlock;
1487 spin_unlock(vmf->ptl);
1488 wait_on_page_locked(page);
1489 put_page(page);
1490 goto out;
1491 }
1492
1493 page = pmd_page(pmd);
1494 BUG_ON(is_huge_zero_page(page));
1495 page_nid = page_to_nid(page);
1496 last_cpupid = page_cpupid_last(page);
1497 count_vm_numa_event(NUMA_HINT_FAULTS);
1498 if (page_nid == this_nid) {
1499 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
1500 flags |= TNF_FAULT_LOCAL;
1501 }
1502
1503
1504 if (!pmd_savedwrite(pmd))
1505 flags |= TNF_NO_GROUP;
1506
1507
1508
1509
1510
1511 page_locked = trylock_page(page);
1512 target_nid = mpol_misplaced(page, vma, haddr);
1513 if (target_nid == -1) {
1514
1515 if (page_locked)
1516 goto clear_pmdnuma;
1517 }
1518
1519
1520 if (!page_locked) {
1521 page_nid = -1;
1522 if (!get_page_unless_zero(page))
1523 goto out_unlock;
1524 spin_unlock(vmf->ptl);
1525 wait_on_page_locked(page);
1526 put_page(page);
1527 goto out;
1528 }
1529
1530
1531
1532
1533
1534 get_page(page);
1535 spin_unlock(vmf->ptl);
1536 anon_vma = page_lock_anon_vma_read(page);
1537
1538
1539 spin_lock(vmf->ptl);
1540 if (unlikely(!pmd_same(pmd, *vmf->pmd))) {
1541 unlock_page(page);
1542 put_page(page);
1543 page_nid = -1;
1544 goto out_unlock;
1545 }
1546
1547
1548 if (unlikely(!anon_vma)) {
1549 put_page(page);
1550 page_nid = -1;
1551 goto clear_pmdnuma;
1552 }
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565 if (mm_tlb_flush_pending(vma->vm_mm))
1566 flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
1567
1568
1569
1570
1571
1572 spin_unlock(vmf->ptl);
1573
1574 migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma,
1575 vmf->pmd, pmd, vmf->address, page, target_nid);
1576 if (migrated) {
1577 flags |= TNF_MIGRATED;
1578 page_nid = target_nid;
1579 } else
1580 flags |= TNF_MIGRATE_FAIL;
1581
1582 goto out;
1583clear_pmdnuma:
1584 BUG_ON(!PageLocked(page));
1585 was_writable = pmd_savedwrite(pmd);
1586 pmd = pmd_modify(pmd, vma->vm_page_prot);
1587 pmd = pmd_mkyoung(pmd);
1588 if (was_writable)
1589 pmd = pmd_mkwrite(pmd);
1590 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
1591 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1592 unlock_page(page);
1593out_unlock:
1594 spin_unlock(vmf->ptl);
1595
1596out:
1597 if (anon_vma)
1598 page_unlock_anon_vma_read(anon_vma);
1599
1600 if (page_nid != -1)
1601 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR,
1602 flags);
1603
1604 return 0;
1605}
1606
1607
1608
1609
1610
1611bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1612 pmd_t *pmd, unsigned long addr, unsigned long next)
1613{
1614 spinlock_t *ptl;
1615 pmd_t orig_pmd;
1616 struct page *page;
1617 struct mm_struct *mm = tlb->mm;
1618 bool ret = false;
1619
1620 tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);
1621
1622 ptl = pmd_trans_huge_lock(pmd, vma);
1623 if (!ptl)
1624 goto out_unlocked;
1625
1626 orig_pmd = *pmd;
1627 if (is_huge_zero_pmd(orig_pmd))
1628 goto out;
1629
1630 if (unlikely(!pmd_present(orig_pmd))) {
1631 VM_BUG_ON(thp_migration_supported() &&
1632 !is_pmd_migration_entry(orig_pmd));
1633 goto out;
1634 }
1635
1636 page = pmd_page(orig_pmd);
1637
1638
1639
1640
1641 if (page_mapcount(page) != 1)
1642 goto out;
1643
1644 if (!trylock_page(page))
1645 goto out;
1646
1647
1648
1649
1650
1651 if (next - addr != HPAGE_PMD_SIZE) {
1652 get_page(page);
1653 spin_unlock(ptl);
1654 split_huge_page(page);
1655 unlock_page(page);
1656 put_page(page);
1657 goto out_unlocked;
1658 }
1659
1660 if (PageDirty(page))
1661 ClearPageDirty(page);
1662 unlock_page(page);
1663
1664 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1665 pmdp_invalidate(vma, addr, pmd);
1666 orig_pmd = pmd_mkold(orig_pmd);
1667 orig_pmd = pmd_mkclean(orig_pmd);
1668
1669 set_pmd_at(mm, addr, pmd, orig_pmd);
1670 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1671 }
1672
1673 mark_page_lazyfree(page);
1674 ret = true;
1675out:
1676 spin_unlock(ptl);
1677out_unlocked:
1678 return ret;
1679}
1680
1681static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
1682{
1683 pgtable_t pgtable;
1684
1685 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1686 pte_free(mm, pgtable);
1687 mm_dec_nr_ptes(mm);
1688}
1689
1690int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1691 pmd_t *pmd, unsigned long addr)
1692{
1693 pmd_t orig_pmd;
1694 spinlock_t *ptl;
1695
1696 tlb_remove_check_page_size_change(tlb, HPAGE_PMD_SIZE);
1697
1698 ptl = __pmd_trans_huge_lock(pmd, vma);
1699 if (!ptl)
1700 return 0;
1701
1702
1703
1704
1705
1706
1707 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1708 tlb->fullmm);
1709 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1710 if (vma_is_dax(vma)) {
1711 if (arch_needs_pgtable_deposit())
1712 zap_deposited_table(tlb->mm, pmd);
1713 spin_unlock(ptl);
1714 if (is_huge_zero_pmd(orig_pmd))
1715 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE);
1716 } else if (is_huge_zero_pmd(orig_pmd)) {
1717 zap_deposited_table(tlb->mm, pmd);
1718 spin_unlock(ptl);
1719 tlb_remove_page_size(tlb, pmd_page(orig_pmd), HPAGE_PMD_SIZE);
1720 } else {
1721 struct page *page = NULL;
1722 int flush_needed = 1;
1723
1724 if (pmd_present(orig_pmd)) {
1725 page = pmd_page(orig_pmd);
1726 page_remove_rmap(page, true);
1727 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1728 VM_BUG_ON_PAGE(!PageHead(page), page);
1729 } else if (thp_migration_supported()) {
1730 swp_entry_t entry;
1731
1732 VM_BUG_ON(!is_pmd_migration_entry(orig_pmd));
1733 entry = pmd_to_swp_entry(orig_pmd);
1734 page = pfn_to_page(swp_offset(entry));
1735 flush_needed = 0;
1736 } else
1737 WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
1738
1739 if (PageAnon(page)) {
1740 zap_deposited_table(tlb->mm, pmd);
1741 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1742 } else {
1743 if (arch_needs_pgtable_deposit())
1744 zap_deposited_table(tlb->mm, pmd);
1745 add_mm_counter(tlb->mm, mm_counter_file(page), -HPAGE_PMD_NR);
1746 }
1747
1748 spin_unlock(ptl);
1749 if (flush_needed)
1750 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
1751 }
1752 return 1;
1753}
1754
1755#ifndef pmd_move_must_withdraw
1756static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl,
1757 spinlock_t *old_pmd_ptl,
1758 struct vm_area_struct *vma)
1759{
1760
1761
1762
1763
1764
1765
1766 return (new_pmd_ptl != old_pmd_ptl) && vma_is_anonymous(vma);
1767}
1768#endif
1769
1770static pmd_t move_soft_dirty_pmd(pmd_t pmd)
1771{
1772#ifdef CONFIG_MEM_SOFT_DIRTY
1773 if (unlikely(is_pmd_migration_entry(pmd)))
1774 pmd = pmd_swp_mksoft_dirty(pmd);
1775 else if (pmd_present(pmd))
1776 pmd = pmd_mksoft_dirty(pmd);
1777#endif
1778 return pmd;
1779}
1780
1781bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
1782 unsigned long new_addr, unsigned long old_end,
1783 pmd_t *old_pmd, pmd_t *new_pmd)
1784{
1785 spinlock_t *old_ptl, *new_ptl;
1786 pmd_t pmd;
1787 struct mm_struct *mm = vma->vm_mm;
1788 bool force_flush = false;
1789
1790 if ((old_addr & ~HPAGE_PMD_MASK) ||
1791 (new_addr & ~HPAGE_PMD_MASK) ||
1792 old_end - old_addr < HPAGE_PMD_SIZE)
1793 return false;
1794
1795
1796
1797
1798
1799 if (WARN_ON(!pmd_none(*new_pmd))) {
1800 VM_BUG_ON(pmd_trans_huge(*new_pmd));
1801 return false;
1802 }
1803
1804
1805
1806
1807
1808 old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
1809 if (old_ptl) {
1810 new_ptl = pmd_lockptr(mm, new_pmd);
1811 if (new_ptl != old_ptl)
1812 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
1813 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
1814 if (pmd_present(pmd))
1815 force_flush = true;
1816 VM_BUG_ON(!pmd_none(*new_pmd));
1817
1818 if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) {
1819 pgtable_t pgtable;
1820 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1821 pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
1822 }
1823 pmd = move_soft_dirty_pmd(pmd);
1824 set_pmd_at(mm, new_addr, new_pmd, pmd);
1825 if (force_flush)
1826 flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
1827 if (new_ptl != old_ptl)
1828 spin_unlock(new_ptl);
1829 spin_unlock(old_ptl);
1830 return true;
1831 }
1832 return false;
1833}
1834
1835
1836
1837
1838
1839
1840
1841int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
1842 unsigned long addr, pgprot_t newprot, int prot_numa)
1843{
1844 struct mm_struct *mm = vma->vm_mm;
1845 spinlock_t *ptl;
1846 pmd_t entry;
1847 bool preserve_write;
1848 int ret;
1849
1850 ptl = __pmd_trans_huge_lock(pmd, vma);
1851 if (!ptl)
1852 return 0;
1853
1854 preserve_write = prot_numa && pmd_write(*pmd);
1855 ret = 1;
1856
1857#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1858 if (is_swap_pmd(*pmd)) {
1859 swp_entry_t entry = pmd_to_swp_entry(*pmd);
1860
1861 VM_BUG_ON(!is_pmd_migration_entry(*pmd));
1862 if (is_write_migration_entry(entry)) {
1863 pmd_t newpmd;
1864
1865
1866
1867
1868 make_migration_entry_read(&entry);
1869 newpmd = swp_entry_to_pmd(entry);
1870 if (pmd_swp_soft_dirty(*pmd))
1871 newpmd = pmd_swp_mksoft_dirty(newpmd);
1872 set_pmd_at(mm, addr, pmd, newpmd);
1873 }
1874 goto unlock;
1875 }
1876#endif
1877
1878
1879
1880
1881
1882
1883 if (prot_numa && is_huge_zero_pmd(*pmd))
1884 goto unlock;
1885
1886 if (prot_numa && pmd_protnone(*pmd))
1887 goto unlock;
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910 entry = pmdp_invalidate(vma, addr, pmd);
1911
1912 entry = pmd_modify(entry, newprot);
1913 if (preserve_write)
1914 entry = pmd_mk_savedwrite(entry);
1915 ret = HPAGE_PMD_NR;
1916 set_pmd_at(mm, addr, pmd, entry);
1917 BUG_ON(vma_is_anonymous(vma) && !preserve_write && pmd_write(entry));
1918unlock:
1919 spin_unlock(ptl);
1920 return ret;
1921}
1922
1923
1924
1925
1926
1927
1928
1929spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
1930{
1931 spinlock_t *ptl;
1932 ptl = pmd_lock(vma->vm_mm, pmd);
1933 if (likely(is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) ||
1934 pmd_devmap(*pmd)))
1935 return ptl;
1936 spin_unlock(ptl);
1937 return NULL;
1938}
1939
1940
1941
1942
1943
1944
1945
1946spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma)
1947{
1948 spinlock_t *ptl;
1949
1950 ptl = pud_lock(vma->vm_mm, pud);
1951 if (likely(pud_trans_huge(*pud) || pud_devmap(*pud)))
1952 return ptl;
1953 spin_unlock(ptl);
1954 return NULL;
1955}
1956
1957#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1958int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
1959 pud_t *pud, unsigned long addr)
1960{
1961 pud_t orig_pud;
1962 spinlock_t *ptl;
1963
1964 ptl = __pud_trans_huge_lock(pud, vma);
1965 if (!ptl)
1966 return 0;
1967
1968
1969
1970
1971
1972
1973 orig_pud = pudp_huge_get_and_clear_full(tlb->mm, addr, pud,
1974 tlb->fullmm);
1975 tlb_remove_pud_tlb_entry(tlb, pud, addr);
1976 if (vma_is_dax(vma)) {
1977 spin_unlock(ptl);
1978
1979 } else {
1980
1981 BUG();
1982 }
1983 return 1;
1984}
1985
1986static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud,
1987 unsigned long haddr)
1988{
1989 VM_BUG_ON(haddr & ~HPAGE_PUD_MASK);
1990 VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
1991 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma);
1992 VM_BUG_ON(!pud_trans_huge(*pud) && !pud_devmap(*pud));
1993
1994 count_vm_event(THP_SPLIT_PUD);
1995
1996 pudp_huge_clear_flush_notify(vma, haddr, pud);
1997}
1998
1999void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
2000 unsigned long address)
2001{
2002 spinlock_t *ptl;
2003 struct mm_struct *mm = vma->vm_mm;
2004 unsigned long haddr = address & HPAGE_PUD_MASK;
2005
2006 mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PUD_SIZE);
2007 ptl = pud_lock(mm, pud);
2008 if (unlikely(!pud_trans_huge(*pud) && !pud_devmap(*pud)))
2009 goto out;
2010 __split_huge_pud_locked(vma, pud, haddr);
2011
2012out:
2013 spin_unlock(ptl);
2014
2015
2016
2017
2018 mmu_notifier_invalidate_range_only_end(mm, haddr, haddr +
2019 HPAGE_PUD_SIZE);
2020}
2021#endif
2022
2023static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2024 unsigned long haddr, pmd_t *pmd)
2025{
2026 struct mm_struct *mm = vma->vm_mm;
2027 pgtable_t pgtable;
2028 pmd_t _pmd;
2029 int i;
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039 pmdp_huge_clear_flush(vma, haddr, pmd);
2040
2041 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2042 pmd_populate(mm, &_pmd, pgtable);
2043
2044 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2045 pte_t *pte, entry;
2046 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
2047 entry = pte_mkspecial(entry);
2048 pte = pte_offset_map(&_pmd, haddr);
2049 VM_BUG_ON(!pte_none(*pte));
2050 set_pte_at(mm, haddr, pte, entry);
2051 pte_unmap(pte);
2052 }
2053 smp_wmb();
2054 pmd_populate(mm, pmd, pgtable);
2055}
2056
2057static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
2058 unsigned long haddr, bool freeze)
2059{
2060 struct mm_struct *mm = vma->vm_mm;
2061 struct page *page;
2062 pgtable_t pgtable;
2063 pmd_t old_pmd, _pmd;
2064 bool young, write, soft_dirty, pmd_migration = false;
2065 unsigned long addr;
2066 int i;
2067
2068 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
2069 VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2070 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
2071 VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)
2072 && !pmd_devmap(*pmd));
2073
2074 count_vm_event(THP_SPLIT_PMD);
2075
2076 if (!vma_is_anonymous(vma)) {
2077 _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2078
2079
2080
2081
2082 if (arch_needs_pgtable_deposit())
2083 zap_deposited_table(mm, pmd);
2084 if (vma_is_dax(vma))
2085 return;
2086 page = pmd_page(_pmd);
2087 if (!PageDirty(page) && pmd_dirty(_pmd))
2088 set_page_dirty(page);
2089 if (!PageReferenced(page) && pmd_young(_pmd))
2090 SetPageReferenced(page);
2091 page_remove_rmap(page, true);
2092 put_page(page);
2093 add_mm_counter(mm, mm_counter_file(page), -HPAGE_PMD_NR);
2094 return;
2095 } else if (is_huge_zero_pmd(*pmd)) {
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105 return __split_huge_zero_page_pmd(vma, haddr, pmd);
2106 }
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128 old_pmd = pmdp_invalidate(vma, haddr, pmd);
2129
2130#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
2131 pmd_migration = is_pmd_migration_entry(old_pmd);
2132 if (pmd_migration) {
2133 swp_entry_t entry;
2134
2135 entry = pmd_to_swp_entry(old_pmd);
2136 page = pfn_to_page(swp_offset(entry));
2137 } else
2138#endif
2139 page = pmd_page(old_pmd);
2140 VM_BUG_ON_PAGE(!page_count(page), page);
2141 page_ref_add(page, HPAGE_PMD_NR - 1);
2142 if (pmd_dirty(old_pmd))
2143 SetPageDirty(page);
2144 write = pmd_write(old_pmd);
2145 young = pmd_young(old_pmd);
2146 soft_dirty = pmd_soft_dirty(old_pmd);
2147
2148
2149
2150
2151
2152 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2153 pmd_populate(mm, &_pmd, pgtable);
2154
2155 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
2156 pte_t entry, *pte;
2157
2158
2159
2160
2161
2162 if (freeze || pmd_migration) {
2163 swp_entry_t swp_entry;
2164 swp_entry = make_migration_entry(page + i, write);
2165 entry = swp_entry_to_pte(swp_entry);
2166 if (soft_dirty)
2167 entry = pte_swp_mksoft_dirty(entry);
2168 } else {
2169 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
2170 entry = maybe_mkwrite(entry, vma);
2171 if (!write)
2172 entry = pte_wrprotect(entry);
2173 if (!young)
2174 entry = pte_mkold(entry);
2175 if (soft_dirty)
2176 entry = pte_mksoft_dirty(entry);
2177 }
2178 pte = pte_offset_map(&_pmd, addr);
2179 BUG_ON(!pte_none(*pte));
2180 set_pte_at(mm, addr, pte, entry);
2181 atomic_inc(&page[i]._mapcount);
2182 pte_unmap(pte);
2183 }
2184
2185
2186
2187
2188
2189 if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) {
2190 for (i = 0; i < HPAGE_PMD_NR; i++)
2191 atomic_inc(&page[i]._mapcount);
2192 }
2193
2194 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
2195
2196 __dec_node_page_state(page, NR_ANON_THPS);
2197 if (TestClearPageDoubleMap(page)) {
2198
2199 for (i = 0; i < HPAGE_PMD_NR; i++)
2200 atomic_dec(&page[i]._mapcount);
2201 }
2202 }
2203
2204 smp_wmb();
2205 pmd_populate(mm, pmd, pgtable);
2206
2207 if (freeze) {
2208 for (i = 0; i < HPAGE_PMD_NR; i++) {
2209 page_remove_rmap(page + i, false);
2210 put_page(page + i);
2211 }
2212 }
2213}
2214
2215void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2216 unsigned long address, bool freeze, struct page *page)
2217{
2218 spinlock_t *ptl;
2219 struct mm_struct *mm = vma->vm_mm;
2220 unsigned long haddr = address & HPAGE_PMD_MASK;
2221
2222 mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
2223 ptl = pmd_lock(mm, pmd);
2224
2225
2226
2227
2228
2229 VM_BUG_ON(freeze && !page);
2230 if (page && page != pmd_page(*pmd))
2231 goto out;
2232
2233 if (pmd_trans_huge(*pmd)) {
2234 page = pmd_page(*pmd);
2235 if (PageMlocked(page))
2236 clear_page_mlock(page);
2237 } else if (!(pmd_devmap(*pmd) || is_pmd_migration_entry(*pmd)))
2238 goto out;
2239 __split_huge_pmd_locked(vma, pmd, haddr, freeze);
2240out:
2241 spin_unlock(ptl);
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255 mmu_notifier_invalidate_range_only_end(mm, haddr, haddr +
2256 HPAGE_PMD_SIZE);
2257}
2258
2259void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
2260 bool freeze, struct page *page)
2261{
2262 pgd_t *pgd;
2263 p4d_t *p4d;
2264 pud_t *pud;
2265 pmd_t *pmd;
2266
2267 pgd = pgd_offset(vma->vm_mm, address);
2268 if (!pgd_present(*pgd))
2269 return;
2270
2271 p4d = p4d_offset(pgd, address);
2272 if (!p4d_present(*p4d))
2273 return;
2274
2275 pud = pud_offset(p4d, address);
2276 if (!pud_present(*pud))
2277 return;
2278
2279 pmd = pmd_offset(pud, address);
2280
2281 __split_huge_pmd(vma, pmd, address, freeze, page);
2282}
2283
2284void vma_adjust_trans_huge(struct vm_area_struct *vma,
2285 unsigned long start,
2286 unsigned long end,
2287 long adjust_next)
2288{
2289
2290
2291
2292
2293
2294 if (start & ~HPAGE_PMD_MASK &&
2295 (start & HPAGE_PMD_MASK) >= vma->vm_start &&
2296 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2297 split_huge_pmd_address(vma, start, false, NULL);
2298
2299
2300
2301
2302
2303
2304 if (end & ~HPAGE_PMD_MASK &&
2305 (end & HPAGE_PMD_MASK) >= vma->vm_start &&
2306 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
2307 split_huge_pmd_address(vma, end, false, NULL);
2308
2309
2310
2311
2312
2313
2314 if (adjust_next > 0) {
2315 struct vm_area_struct *next = vma->vm_next;
2316 unsigned long nstart = next->vm_start;
2317 nstart += adjust_next << PAGE_SHIFT;
2318 if (nstart & ~HPAGE_PMD_MASK &&
2319 (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
2320 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
2321 split_huge_pmd_address(next, nstart, false, NULL);
2322 }
2323}
2324
2325static void freeze_page(struct page *page)
2326{
2327 enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS |
2328 TTU_RMAP_LOCKED | TTU_SPLIT_HUGE_PMD;
2329 bool unmap_success;
2330
2331 VM_BUG_ON_PAGE(!PageHead(page), page);
2332
2333 if (PageAnon(page))
2334 ttu_flags |= TTU_SPLIT_FREEZE;
2335
2336 unmap_success = try_to_unmap(page, ttu_flags);
2337 VM_BUG_ON_PAGE(!unmap_success, page);
2338}
2339
2340static void unfreeze_page(struct page *page)
2341{
2342 int i;
2343 if (PageTransHuge(page)) {
2344 remove_migration_ptes(page, page, true);
2345 } else {
2346 for (i = 0; i < HPAGE_PMD_NR; i++)
2347 remove_migration_ptes(page + i, page + i, true);
2348 }
2349}
2350
2351static void __split_huge_page_tail(struct page *head, int tail,
2352 struct lruvec *lruvec, struct list_head *list)
2353{
2354 struct page *page_tail = head + tail;
2355
2356 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
2357
2358
2359
2360
2361
2362
2363
2364 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
2365 page_tail->flags |= (head->flags &
2366 ((1L << PG_referenced) |
2367 (1L << PG_swapbacked) |
2368 (1L << PG_swapcache) |
2369 (1L << PG_mlocked) |
2370 (1L << PG_uptodate) |
2371 (1L << PG_active) |
2372 (1L << PG_locked) |
2373 (1L << PG_unevictable) |
2374 (1L << PG_dirty)));
2375
2376
2377 smp_wmb();
2378
2379
2380
2381
2382
2383
2384
2385 clear_compound_head(page_tail);
2386
2387
2388 page_ref_unfreeze(page_tail, 1 + (!PageAnon(head) ||
2389 PageSwapCache(head)));
2390
2391 if (page_is_young(head))
2392 set_page_young(page_tail);
2393 if (page_is_idle(head))
2394 set_page_idle(page_tail);
2395
2396
2397 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
2398 page_tail);
2399 page_tail->mapping = head->mapping;
2400
2401 page_tail->index = head->index + tail;
2402 page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
2403
2404
2405
2406
2407
2408
2409 lru_add_page_tail(head, page_tail, lruvec, list);
2410}
2411
2412static void __split_huge_page(struct page *page, struct list_head *list,
2413 unsigned long flags)
2414{
2415 struct page *head = compound_head(page);
2416 struct zone *zone = page_zone(head);
2417 struct lruvec *lruvec;
2418 pgoff_t end = -1;
2419 int i;
2420
2421 lruvec = mem_cgroup_page_lruvec(head, zone->zone_pgdat);
2422
2423
2424 mem_cgroup_split_huge_fixup(head);
2425
2426 if (!PageAnon(page))
2427 end = DIV_ROUND_UP(i_size_read(head->mapping->host), PAGE_SIZE);
2428
2429 for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
2430 __split_huge_page_tail(head, i, lruvec, list);
2431
2432 if (head[i].index >= end) {
2433 ClearPageDirty(head + i);
2434 __delete_from_page_cache(head + i, NULL);
2435 if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head))
2436 shmem_uncharge(head->mapping->host, 1);
2437 put_page(head + i);
2438 }
2439 }
2440
2441 ClearPageCompound(head);
2442
2443 if (PageAnon(head)) {
2444
2445 if (PageSwapCache(head))
2446 page_ref_add(head, 2);
2447 else
2448 page_ref_inc(head);
2449 } else {
2450
2451 page_ref_add(head, 2);
2452 xa_unlock(&head->mapping->i_pages);
2453 }
2454
2455 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2456
2457 unfreeze_page(head);
2458
2459 for (i = 0; i < HPAGE_PMD_NR; i++) {
2460 struct page *subpage = head + i;
2461 if (subpage == page)
2462 continue;
2463 unlock_page(subpage);
2464
2465
2466
2467
2468
2469
2470
2471
2472 put_page(subpage);
2473 }
2474}
2475
2476int total_mapcount(struct page *page)
2477{
2478 int i, compound, ret;
2479
2480 VM_BUG_ON_PAGE(PageTail(page), page);
2481
2482 if (likely(!PageCompound(page)))
2483 return atomic_read(&page->_mapcount) + 1;
2484
2485 compound = compound_mapcount(page);
2486 if (PageHuge(page))
2487 return compound;
2488 ret = compound;
2489 for (i = 0; i < HPAGE_PMD_NR; i++)
2490 ret += atomic_read(&page[i]._mapcount) + 1;
2491
2492 if (!PageAnon(page))
2493 return ret - compound * HPAGE_PMD_NR;
2494 if (PageDoubleMap(page))
2495 ret -= HPAGE_PMD_NR;
2496 return ret;
2497}
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523int page_trans_huge_mapcount(struct page *page, int *total_mapcount)
2524{
2525 int i, ret, _total_mapcount, mapcount;
2526
2527
2528 VM_BUG_ON_PAGE(PageHuge(page), page);
2529
2530 if (likely(!PageTransCompound(page))) {
2531 mapcount = atomic_read(&page->_mapcount) + 1;
2532 if (total_mapcount)
2533 *total_mapcount = mapcount;
2534 return mapcount;
2535 }
2536
2537 page = compound_head(page);
2538
2539 _total_mapcount = ret = 0;
2540 for (i = 0; i < HPAGE_PMD_NR; i++) {
2541 mapcount = atomic_read(&page[i]._mapcount) + 1;
2542 ret = max(ret, mapcount);
2543 _total_mapcount += mapcount;
2544 }
2545 if (PageDoubleMap(page)) {
2546 ret -= 1;
2547 _total_mapcount -= HPAGE_PMD_NR;
2548 }
2549 mapcount = compound_mapcount(page);
2550 ret += mapcount;
2551 _total_mapcount += mapcount;
2552 if (total_mapcount)
2553 *total_mapcount = _total_mapcount;
2554 return ret;
2555}
2556
2557
2558bool can_split_huge_page(struct page *page, int *pextra_pins)
2559{
2560 int extra_pins;
2561
2562
2563 if (PageAnon(page))
2564 extra_pins = PageSwapCache(page) ? HPAGE_PMD_NR : 0;
2565 else
2566 extra_pins = HPAGE_PMD_NR;
2567 if (pextra_pins)
2568 *pextra_pins = extra_pins;
2569 return total_mapcount(page) == page_count(page) - extra_pins - 1;
2570}
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591int split_huge_page_to_list(struct page *page, struct list_head *list)
2592{
2593 struct page *head = compound_head(page);
2594 struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
2595 struct anon_vma *anon_vma = NULL;
2596 struct address_space *mapping = NULL;
2597 int count, mapcount, extra_pins, ret;
2598 bool mlocked;
2599 unsigned long flags;
2600
2601 VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
2602 VM_BUG_ON_PAGE(!PageLocked(page), page);
2603 VM_BUG_ON_PAGE(!PageCompound(page), page);
2604
2605 if (PageWriteback(page))
2606 return -EBUSY;
2607
2608 if (PageAnon(head)) {
2609
2610
2611
2612
2613
2614
2615
2616
2617 anon_vma = page_get_anon_vma(head);
2618 if (!anon_vma) {
2619 ret = -EBUSY;
2620 goto out;
2621 }
2622 mapping = NULL;
2623 anon_vma_lock_write(anon_vma);
2624 } else {
2625 mapping = head->mapping;
2626
2627
2628 if (!mapping) {
2629 ret = -EBUSY;
2630 goto out;
2631 }
2632
2633 anon_vma = NULL;
2634 i_mmap_lock_read(mapping);
2635 }
2636
2637
2638
2639
2640
2641 if (!can_split_huge_page(head, &extra_pins)) {
2642 ret = -EBUSY;
2643 goto out_unlock;
2644 }
2645
2646 mlocked = PageMlocked(page);
2647 freeze_page(head);
2648 VM_BUG_ON_PAGE(compound_mapcount(head), head);
2649
2650
2651 if (mlocked)
2652 lru_add_drain();
2653
2654
2655 spin_lock_irqsave(zone_lru_lock(page_zone(head)), flags);
2656
2657 if (mapping) {
2658 void **pslot;
2659
2660 xa_lock(&mapping->i_pages);
2661 pslot = radix_tree_lookup_slot(&mapping->i_pages,
2662 page_index(head));
2663
2664
2665
2666
2667 if (radix_tree_deref_slot_protected(pslot,
2668 &mapping->i_pages.xa_lock) != head)
2669 goto fail;
2670 }
2671
2672
2673 spin_lock(&pgdata->split_queue_lock);
2674 count = page_count(head);
2675 mapcount = total_mapcount(head);
2676 if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) {
2677 if (!list_empty(page_deferred_list(head))) {
2678 pgdata->split_queue_len--;
2679 list_del(page_deferred_list(head));
2680 }
2681 if (mapping)
2682 __dec_node_page_state(page, NR_SHMEM_THPS);
2683 spin_unlock(&pgdata->split_queue_lock);
2684 __split_huge_page(page, list, flags);
2685 if (PageSwapCache(head)) {
2686 swp_entry_t entry = { .val = page_private(head) };
2687
2688 ret = split_swap_cluster(entry);
2689 } else
2690 ret = 0;
2691 } else {
2692 if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) {
2693 pr_alert("total_mapcount: %u, page_count(): %u\n",
2694 mapcount, count);
2695 if (PageTail(page))
2696 dump_page(head, NULL);
2697 dump_page(page, "total_mapcount(head) > 0");
2698 BUG();
2699 }
2700 spin_unlock(&pgdata->split_queue_lock);
2701fail: if (mapping)
2702 xa_unlock(&mapping->i_pages);
2703 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2704 unfreeze_page(head);
2705 ret = -EBUSY;
2706 }
2707
2708out_unlock:
2709 if (anon_vma) {
2710 anon_vma_unlock_write(anon_vma);
2711 put_anon_vma(anon_vma);
2712 }
2713 if (mapping)
2714 i_mmap_unlock_read(mapping);
2715out:
2716 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
2717 return ret;
2718}
2719
2720void free_transhuge_page(struct page *page)
2721{
2722 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2723 unsigned long flags;
2724
2725 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2726 if (!list_empty(page_deferred_list(page))) {
2727 pgdata->split_queue_len--;
2728 list_del(page_deferred_list(page));
2729 }
2730 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2731 free_compound_page(page);
2732}
2733
2734void deferred_split_huge_page(struct page *page)
2735{
2736 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2737 unsigned long flags;
2738
2739 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
2740
2741 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2742 if (list_empty(page_deferred_list(page))) {
2743 count_vm_event(THP_DEFERRED_SPLIT_PAGE);
2744 list_add_tail(page_deferred_list(page), &pgdata->split_queue);
2745 pgdata->split_queue_len++;
2746 }
2747 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2748}
2749
2750static unsigned long deferred_split_count(struct shrinker *shrink,
2751 struct shrink_control *sc)
2752{
2753 struct pglist_data *pgdata = NODE_DATA(sc->nid);
2754 return READ_ONCE(pgdata->split_queue_len);
2755}
2756
2757static unsigned long deferred_split_scan(struct shrinker *shrink,
2758 struct shrink_control *sc)
2759{
2760 struct pglist_data *pgdata = NODE_DATA(sc->nid);
2761 unsigned long flags;
2762 LIST_HEAD(list), *pos, *next;
2763 struct page *page;
2764 int split = 0;
2765
2766 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2767
2768 list_for_each_safe(pos, next, &pgdata->split_queue) {
2769 page = list_entry((void *)pos, struct page, mapping);
2770 page = compound_head(page);
2771 if (get_page_unless_zero(page)) {
2772 list_move(page_deferred_list(page), &list);
2773 } else {
2774
2775 list_del_init(page_deferred_list(page));
2776 pgdata->split_queue_len--;
2777 }
2778 if (!--sc->nr_to_scan)
2779 break;
2780 }
2781 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2782
2783 list_for_each_safe(pos, next, &list) {
2784 page = list_entry((void *)pos, struct page, mapping);
2785 if (!trylock_page(page))
2786 goto next;
2787
2788 if (!split_huge_page(page))
2789 split++;
2790 unlock_page(page);
2791next:
2792 put_page(page);
2793 }
2794
2795 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2796 list_splice_tail(&list, &pgdata->split_queue);
2797 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2798
2799
2800
2801
2802
2803 if (!split && list_empty(&pgdata->split_queue))
2804 return SHRINK_STOP;
2805 return split;
2806}
2807
2808static struct shrinker deferred_split_shrinker = {
2809 .count_objects = deferred_split_count,
2810 .scan_objects = deferred_split_scan,
2811 .seeks = DEFAULT_SEEKS,
2812 .flags = SHRINKER_NUMA_AWARE,
2813};
2814
2815#ifdef CONFIG_DEBUG_FS
2816static int split_huge_pages_set(void *data, u64 val)
2817{
2818 struct zone *zone;
2819 struct page *page;
2820 unsigned long pfn, max_zone_pfn;
2821 unsigned long total = 0, split = 0;
2822
2823 if (val != 1)
2824 return -EINVAL;
2825
2826 for_each_populated_zone(zone) {
2827 max_zone_pfn = zone_end_pfn(zone);
2828 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
2829 if (!pfn_valid(pfn))
2830 continue;
2831
2832 page = pfn_to_page(pfn);
2833 if (!get_page_unless_zero(page))
2834 continue;
2835
2836 if (zone != page_zone(page))
2837 goto next;
2838
2839 if (!PageHead(page) || PageHuge(page) || !PageLRU(page))
2840 goto next;
2841
2842 total++;
2843 lock_page(page);
2844 if (!split_huge_page(page))
2845 split++;
2846 unlock_page(page);
2847next:
2848 put_page(page);
2849 }
2850 }
2851
2852 pr_info("%lu of %lu THP split\n", split, total);
2853
2854 return 0;
2855}
2856DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
2857 "%llu\n");
2858
2859static int __init split_huge_pages_debugfs(void)
2860{
2861 void *ret;
2862
2863 ret = debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
2864 &split_huge_pages_fops);
2865 if (!ret)
2866 pr_warn("Failed to create split_huge_pages in debugfs");
2867 return 0;
2868}
2869late_initcall(split_huge_pages_debugfs);
2870#endif
2871
2872#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
2873void set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw,
2874 struct page *page)
2875{
2876 struct vm_area_struct *vma = pvmw->vma;
2877 struct mm_struct *mm = vma->vm_mm;
2878 unsigned long address = pvmw->address;
2879 pmd_t pmdval;
2880 swp_entry_t entry;
2881 pmd_t pmdswp;
2882
2883 if (!(pvmw->pmd && !pvmw->pte))
2884 return;
2885
2886 flush_cache_range(vma, address, address + HPAGE_PMD_SIZE);
2887 pmdval = *pvmw->pmd;
2888 pmdp_invalidate(vma, address, pvmw->pmd);
2889 if (pmd_dirty(pmdval))
2890 set_page_dirty(page);
2891 entry = make_migration_entry(page, pmd_write(pmdval));
2892 pmdswp = swp_entry_to_pmd(entry);
2893 if (pmd_soft_dirty(pmdval))
2894 pmdswp = pmd_swp_mksoft_dirty(pmdswp);
2895 set_pmd_at(mm, address, pvmw->pmd, pmdswp);
2896 page_remove_rmap(page, true);
2897 put_page(page);
2898}
2899
2900void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new)
2901{
2902 struct vm_area_struct *vma = pvmw->vma;
2903 struct mm_struct *mm = vma->vm_mm;
2904 unsigned long address = pvmw->address;
2905 unsigned long mmun_start = address & HPAGE_PMD_MASK;
2906 pmd_t pmde;
2907 swp_entry_t entry;
2908
2909 if (!(pvmw->pmd && !pvmw->pte))
2910 return;
2911
2912 entry = pmd_to_swp_entry(*pvmw->pmd);
2913 get_page(new);
2914 pmde = pmd_mkold(mk_huge_pmd(new, vma->vm_page_prot));
2915 if (pmd_swp_soft_dirty(*pvmw->pmd))
2916 pmde = pmd_mksoft_dirty(pmde);
2917 if (is_write_migration_entry(entry))
2918 pmde = maybe_pmd_mkwrite(pmde, vma);
2919
2920 flush_cache_range(vma, mmun_start, mmun_start + HPAGE_PMD_SIZE);
2921 if (PageAnon(new))
2922 page_add_anon_rmap(new, vma, mmun_start, true);
2923 else
2924 page_add_file_rmap(new, true);
2925 set_pmd_at(mm, mmun_start, pvmw->pmd, pmde);
2926 if ((vma->vm_flags & VM_LOCKED) && !PageDoubleMap(new))
2927 mlock_vma_page(new);
2928 update_mmu_cache_pmd(vma, address, pvmw->pmd);
2929}
2930#endif
2931