1
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/err.h>
5#include <linux/spinlock.h>
6
7#include <linux/mm.h>
8#include <linux/memremap.h>
9#include <linux/pagemap.h>
10#include <linux/rmap.h>
11#include <linux/swap.h>
12#include <linux/swapops.h>
13#include <linux/secretmem.h>
14
15#include <linux/sched/signal.h>
16#include <linux/rwsem.h>
17#include <linux/hugetlb.h>
18#include <linux/migrate.h>
19#include <linux/mm_inline.h>
20#include <linux/sched/mm.h>
21
22#include <asm/mmu_context.h>
23#include <asm/tlbflush.h>
24
25#include "internal.h"
26
27struct follow_page_context {
28 struct dev_pagemap *pgmap;
29 unsigned int page_mask;
30};
31
32static void hpage_pincount_add(struct page *page, int refs)
33{
34 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
35 VM_BUG_ON_PAGE(page != compound_head(page), page);
36
37 atomic_add(refs, compound_pincount_ptr(page));
38}
39
40static void hpage_pincount_sub(struct page *page, int refs)
41{
42 VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
43 VM_BUG_ON_PAGE(page != compound_head(page), page);
44
45 atomic_sub(refs, compound_pincount_ptr(page));
46}
47
48
49static void put_page_refs(struct page *page, int refs)
50{
51#ifdef CONFIG_DEBUG_VM
52 if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))
53 return;
54#endif
55
56
57
58
59
60 if (refs > 1)
61 page_ref_sub(page, refs - 1);
62 put_page(page);
63}
64
65
66
67
68
69static inline struct page *try_get_compound_head(struct page *page, int refs)
70{
71 struct page *head = compound_head(page);
72
73 if (WARN_ON_ONCE(page_ref_count(head) < 0))
74 return NULL;
75 if (unlikely(!page_cache_add_speculative(head, refs)))
76 return NULL;
77
78
79
80
81
82
83
84
85
86
87 if (unlikely(compound_head(page) != head)) {
88 put_page_refs(head, refs);
89 return NULL;
90 }
91
92 return head;
93}
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127struct page *try_grab_compound_head(struct page *page,
128 int refs, unsigned int flags)
129{
130 if (flags & FOLL_GET)
131 return try_get_compound_head(page, refs);
132 else if (flags & FOLL_PIN) {
133
134
135
136
137
138 if (unlikely((flags & FOLL_LONGTERM) &&
139 !is_pinnable_page(page)))
140 return NULL;
141
142
143
144
145
146 page = try_get_compound_head(page, refs);
147 if (!page)
148 return NULL;
149
150
151
152
153
154
155
156
157
158
159
160 if (hpage_pincount_available(page))
161 hpage_pincount_add(page, refs);
162 else
163 page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));
164
165 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,
166 refs);
167
168 return page;
169 }
170
171 WARN_ON_ONCE(1);
172 return NULL;
173}
174
175static void put_compound_head(struct page *page, int refs, unsigned int flags)
176{
177 if (flags & FOLL_PIN) {
178 mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
179 refs);
180
181 if (hpage_pincount_available(page))
182 hpage_pincount_sub(page, refs);
183 else
184 refs *= GUP_PIN_COUNTING_BIAS;
185 }
186
187 put_page_refs(page, refs);
188}
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209bool __must_check try_grab_page(struct page *page, unsigned int flags)
210{
211 if (!(flags & (FOLL_GET | FOLL_PIN)))
212 return true;
213
214 return try_grab_compound_head(page, 1, flags);
215}
216
217
218
219
220
221
222
223
224
225
226void unpin_user_page(struct page *page)
227{
228 put_compound_head(compound_head(page), 1, FOLL_PIN);
229}
230EXPORT_SYMBOL(unpin_user_page);
231
232static inline void compound_range_next(unsigned long i, unsigned long npages,
233 struct page **list, struct page **head,
234 unsigned int *ntails)
235{
236 struct page *next, *page;
237 unsigned int nr = 1;
238
239 if (i >= npages)
240 return;
241
242 next = *list + i;
243 page = compound_head(next);
244 if (PageCompound(page) && compound_order(page) >= 1)
245 nr = min_t(unsigned int,
246 page + compound_nr(page) - next, npages - i);
247
248 *head = page;
249 *ntails = nr;
250}
251
252#define for_each_compound_range(__i, __list, __npages, __head, __ntails) \
253 for (__i = 0, \
254 compound_range_next(__i, __npages, __list, &(__head), &(__ntails)); \
255 __i < __npages; __i += __ntails, \
256 compound_range_next(__i, __npages, __list, &(__head), &(__ntails)))
257
258static inline void compound_next(unsigned long i, unsigned long npages,
259 struct page **list, struct page **head,
260 unsigned int *ntails)
261{
262 struct page *page;
263 unsigned int nr;
264
265 if (i >= npages)
266 return;
267
268 page = compound_head(list[i]);
269 for (nr = i + 1; nr < npages; nr++) {
270 if (compound_head(list[nr]) != page)
271 break;
272 }
273
274 *head = page;
275 *ntails = nr - i;
276}
277
278#define for_each_compound_head(__i, __list, __npages, __head, __ntails) \
279 for (__i = 0, \
280 compound_next(__i, __npages, __list, &(__head), &(__ntails)); \
281 __i < __npages; __i += __ntails, \
282 compound_next(__i, __npages, __list, &(__head), &(__ntails)))
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
307 bool make_dirty)
308{
309 unsigned long index;
310 struct page *head;
311 unsigned int ntails;
312
313 if (!make_dirty) {
314 unpin_user_pages(pages, npages);
315 return;
316 }
317
318 for_each_compound_head(index, pages, npages, head, ntails) {
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339 if (!PageDirty(head))
340 set_page_dirty_lock(head);
341 put_compound_head(head, ntails, FOLL_PIN);
342 }
343}
344EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367void unpin_user_page_range_dirty_lock(struct page *page, unsigned long npages,
368 bool make_dirty)
369{
370 unsigned long index;
371 struct page *head;
372 unsigned int ntails;
373
374 for_each_compound_range(index, &page, npages, head, ntails) {
375 if (make_dirty && !PageDirty(head))
376 set_page_dirty_lock(head);
377 put_compound_head(head, ntails, FOLL_PIN);
378 }
379}
380EXPORT_SYMBOL(unpin_user_page_range_dirty_lock);
381
382
383
384
385
386
387
388
389
390
391void unpin_user_pages(struct page **pages, unsigned long npages)
392{
393 unsigned long index;
394 struct page *head;
395 unsigned int ntails;
396
397
398
399
400
401
402 if (WARN_ON(IS_ERR_VALUE(npages)))
403 return;
404
405 for_each_compound_head(index, pages, npages, head, ntails)
406 put_compound_head(head, ntails, FOLL_PIN);
407}
408EXPORT_SYMBOL(unpin_user_pages);
409
410
411
412
413
414
415static inline void mm_set_has_pinned_flag(unsigned long *mm_flags)
416{
417 if (!test_bit(MMF_HAS_PINNED, mm_flags))
418 set_bit(MMF_HAS_PINNED, mm_flags);
419}
420
421#ifdef CONFIG_MMU
422static struct page *no_page_table(struct vm_area_struct *vma,
423 unsigned int flags)
424{
425
426
427
428
429
430
431
432
433 if ((flags & FOLL_DUMP) &&
434 (vma_is_anonymous(vma) || !vma->vm_ops->fault))
435 return ERR_PTR(-EFAULT);
436 return NULL;
437}
438
439static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
440 pte_t *pte, unsigned int flags)
441{
442
443 if (flags & FOLL_GET)
444 return -EFAULT;
445
446 if (flags & FOLL_TOUCH) {
447 pte_t entry = *pte;
448
449 if (flags & FOLL_WRITE)
450 entry = pte_mkdirty(entry);
451 entry = pte_mkyoung(entry);
452
453 if (!pte_same(*pte, entry)) {
454 set_pte_at(vma->vm_mm, address, pte, entry);
455 update_mmu_cache(vma, address, pte);
456 }
457 }
458
459
460 return -EEXIST;
461}
462
463
464
465
466
467static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
468{
469 return pte_write(pte) ||
470 ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
471}
472
473static struct page *follow_page_pte(struct vm_area_struct *vma,
474 unsigned long address, pmd_t *pmd, unsigned int flags,
475 struct dev_pagemap **pgmap)
476{
477 struct mm_struct *mm = vma->vm_mm;
478 struct page *page;
479 spinlock_t *ptl;
480 pte_t *ptep, pte;
481 int ret;
482
483
484 if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
485 (FOLL_PIN | FOLL_GET)))
486 return ERR_PTR(-EINVAL);
487retry:
488 if (unlikely(pmd_bad(*pmd)))
489 return no_page_table(vma, flags);
490
491 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
492 pte = *ptep;
493 if (!pte_present(pte)) {
494 swp_entry_t entry;
495
496
497
498
499
500 if (likely(!(flags & FOLL_MIGRATION)))
501 goto no_page;
502 if (pte_none(pte))
503 goto no_page;
504 entry = pte_to_swp_entry(pte);
505 if (!is_migration_entry(entry))
506 goto no_page;
507 pte_unmap_unlock(ptep, ptl);
508 migration_entry_wait(mm, pmd, address);
509 goto retry;
510 }
511 if ((flags & FOLL_NUMA) && pte_protnone(pte))
512 goto no_page;
513 if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
514 pte_unmap_unlock(ptep, ptl);
515 return NULL;
516 }
517
518 page = vm_normal_page(vma, address, pte);
519 if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
520
521
522
523
524
525 *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
526 if (*pgmap)
527 page = pte_page(pte);
528 else
529 goto no_page;
530 } else if (unlikely(!page)) {
531 if (flags & FOLL_DUMP) {
532
533 page = ERR_PTR(-EFAULT);
534 goto out;
535 }
536
537 if (is_zero_pfn(pte_pfn(pte))) {
538 page = pte_page(pte);
539 } else {
540 ret = follow_pfn_pte(vma, address, ptep, flags);
541 page = ERR_PTR(ret);
542 goto out;
543 }
544 }
545
546
547 if (unlikely(!try_grab_page(page, flags))) {
548 page = ERR_PTR(-ENOMEM);
549 goto out;
550 }
551
552
553
554
555
556 if (flags & FOLL_PIN) {
557 ret = arch_make_page_accessible(page);
558 if (ret) {
559 unpin_user_page(page);
560 page = ERR_PTR(ret);
561 goto out;
562 }
563 }
564 if (flags & FOLL_TOUCH) {
565 if ((flags & FOLL_WRITE) &&
566 !pte_dirty(pte) && !PageDirty(page))
567 set_page_dirty(page);
568
569
570
571
572
573 mark_page_accessed(page);
574 }
575 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
576
577 if (PageTransCompound(page))
578 goto out;
579
580
581
582
583
584
585
586
587
588
589 if (page->mapping && trylock_page(page)) {
590 lru_add_drain();
591
592
593
594
595
596
597 mlock_vma_page(page);
598 unlock_page(page);
599 }
600 }
601out:
602 pte_unmap_unlock(ptep, ptl);
603 return page;
604no_page:
605 pte_unmap_unlock(ptep, ptl);
606 if (!pte_none(pte))
607 return NULL;
608 return no_page_table(vma, flags);
609}
610
611static struct page *follow_pmd_mask(struct vm_area_struct *vma,
612 unsigned long address, pud_t *pudp,
613 unsigned int flags,
614 struct follow_page_context *ctx)
615{
616 pmd_t *pmd, pmdval;
617 spinlock_t *ptl;
618 struct page *page;
619 struct mm_struct *mm = vma->vm_mm;
620
621 pmd = pmd_offset(pudp, address);
622
623
624
625
626 pmdval = READ_ONCE(*pmd);
627 if (pmd_none(pmdval))
628 return no_page_table(vma, flags);
629 if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) {
630 page = follow_huge_pmd(mm, address, pmd, flags);
631 if (page)
632 return page;
633 return no_page_table(vma, flags);
634 }
635 if (is_hugepd(__hugepd(pmd_val(pmdval)))) {
636 page = follow_huge_pd(vma, address,
637 __hugepd(pmd_val(pmdval)), flags,
638 PMD_SHIFT);
639 if (page)
640 return page;
641 return no_page_table(vma, flags);
642 }
643retry:
644 if (!pmd_present(pmdval)) {
645 if (likely(!(flags & FOLL_MIGRATION)))
646 return no_page_table(vma, flags);
647 VM_BUG_ON(thp_migration_supported() &&
648 !is_pmd_migration_entry(pmdval));
649 if (is_pmd_migration_entry(pmdval))
650 pmd_migration_entry_wait(mm, pmd);
651 pmdval = READ_ONCE(*pmd);
652
653
654
655
656 if (pmd_none(pmdval))
657 return no_page_table(vma, flags);
658 goto retry;
659 }
660 if (pmd_devmap(pmdval)) {
661 ptl = pmd_lock(mm, pmd);
662 page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
663 spin_unlock(ptl);
664 if (page)
665 return page;
666 }
667 if (likely(!pmd_trans_huge(pmdval)))
668 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
669
670 if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
671 return no_page_table(vma, flags);
672
673retry_locked:
674 ptl = pmd_lock(mm, pmd);
675 if (unlikely(pmd_none(*pmd))) {
676 spin_unlock(ptl);
677 return no_page_table(vma, flags);
678 }
679 if (unlikely(!pmd_present(*pmd))) {
680 spin_unlock(ptl);
681 if (likely(!(flags & FOLL_MIGRATION)))
682 return no_page_table(vma, flags);
683 pmd_migration_entry_wait(mm, pmd);
684 goto retry_locked;
685 }
686 if (unlikely(!pmd_trans_huge(*pmd))) {
687 spin_unlock(ptl);
688 return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
689 }
690 if (flags & FOLL_SPLIT_PMD) {
691 int ret;
692 page = pmd_page(*pmd);
693 if (is_huge_zero_page(page)) {
694 spin_unlock(ptl);
695 ret = 0;
696 split_huge_pmd(vma, pmd, address);
697 if (pmd_trans_unstable(pmd))
698 ret = -EBUSY;
699 } else {
700 spin_unlock(ptl);
701 split_huge_pmd(vma, pmd, address);
702 ret = pte_alloc(mm, pmd) ? -ENOMEM : 0;
703 }
704
705 return ret ? ERR_PTR(ret) :
706 follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
707 }
708 page = follow_trans_huge_pmd(vma, address, pmd, flags);
709 spin_unlock(ptl);
710 ctx->page_mask = HPAGE_PMD_NR - 1;
711 return page;
712}
713
714static struct page *follow_pud_mask(struct vm_area_struct *vma,
715 unsigned long address, p4d_t *p4dp,
716 unsigned int flags,
717 struct follow_page_context *ctx)
718{
719 pud_t *pud;
720 spinlock_t *ptl;
721 struct page *page;
722 struct mm_struct *mm = vma->vm_mm;
723
724 pud = pud_offset(p4dp, address);
725 if (pud_none(*pud))
726 return no_page_table(vma, flags);
727 if (pud_huge(*pud) && is_vm_hugetlb_page(vma)) {
728 page = follow_huge_pud(mm, address, pud, flags);
729 if (page)
730 return page;
731 return no_page_table(vma, flags);
732 }
733 if (is_hugepd(__hugepd(pud_val(*pud)))) {
734 page = follow_huge_pd(vma, address,
735 __hugepd(pud_val(*pud)), flags,
736 PUD_SHIFT);
737 if (page)
738 return page;
739 return no_page_table(vma, flags);
740 }
741 if (pud_devmap(*pud)) {
742 ptl = pud_lock(mm, pud);
743 page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
744 spin_unlock(ptl);
745 if (page)
746 return page;
747 }
748 if (unlikely(pud_bad(*pud)))
749 return no_page_table(vma, flags);
750
751 return follow_pmd_mask(vma, address, pud, flags, ctx);
752}
753
754static struct page *follow_p4d_mask(struct vm_area_struct *vma,
755 unsigned long address, pgd_t *pgdp,
756 unsigned int flags,
757 struct follow_page_context *ctx)
758{
759 p4d_t *p4d;
760 struct page *page;
761
762 p4d = p4d_offset(pgdp, address);
763 if (p4d_none(*p4d))
764 return no_page_table(vma, flags);
765 BUILD_BUG_ON(p4d_huge(*p4d));
766 if (unlikely(p4d_bad(*p4d)))
767 return no_page_table(vma, flags);
768
769 if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
770 page = follow_huge_pd(vma, address,
771 __hugepd(p4d_val(*p4d)), flags,
772 P4D_SHIFT);
773 if (page)
774 return page;
775 return no_page_table(vma, flags);
776 }
777 return follow_pud_mask(vma, address, p4d, flags, ctx);
778}
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799static struct page *follow_page_mask(struct vm_area_struct *vma,
800 unsigned long address, unsigned int flags,
801 struct follow_page_context *ctx)
802{
803 pgd_t *pgd;
804 struct page *page;
805 struct mm_struct *mm = vma->vm_mm;
806
807 ctx->page_mask = 0;
808
809
810 page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
811 if (!IS_ERR(page)) {
812 WARN_ON_ONCE(flags & (FOLL_GET | FOLL_PIN));
813 return page;
814 }
815
816 pgd = pgd_offset(mm, address);
817
818 if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
819 return no_page_table(vma, flags);
820
821 if (pgd_huge(*pgd)) {
822 page = follow_huge_pgd(mm, address, pgd, flags);
823 if (page)
824 return page;
825 return no_page_table(vma, flags);
826 }
827 if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
828 page = follow_huge_pd(vma, address,
829 __hugepd(pgd_val(*pgd)), flags,
830 PGDIR_SHIFT);
831 if (page)
832 return page;
833 return no_page_table(vma, flags);
834 }
835
836 return follow_p4d_mask(vma, address, pgd, flags, ctx);
837}
838
839struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
840 unsigned int foll_flags)
841{
842 struct follow_page_context ctx = { NULL };
843 struct page *page;
844
845 if (vma_is_secretmem(vma))
846 return NULL;
847
848 page = follow_page_mask(vma, address, foll_flags, &ctx);
849 if (ctx.pgmap)
850 put_dev_pagemap(ctx.pgmap);
851 return page;
852}
853
854static int get_gate_page(struct mm_struct *mm, unsigned long address,
855 unsigned int gup_flags, struct vm_area_struct **vma,
856 struct page **page)
857{
858 pgd_t *pgd;
859 p4d_t *p4d;
860 pud_t *pud;
861 pmd_t *pmd;
862 pte_t *pte;
863 int ret = -EFAULT;
864
865
866 if (gup_flags & FOLL_WRITE)
867 return -EFAULT;
868 if (address > TASK_SIZE)
869 pgd = pgd_offset_k(address);
870 else
871 pgd = pgd_offset_gate(mm, address);
872 if (pgd_none(*pgd))
873 return -EFAULT;
874 p4d = p4d_offset(pgd, address);
875 if (p4d_none(*p4d))
876 return -EFAULT;
877 pud = pud_offset(p4d, address);
878 if (pud_none(*pud))
879 return -EFAULT;
880 pmd = pmd_offset(pud, address);
881 if (!pmd_present(*pmd))
882 return -EFAULT;
883 VM_BUG_ON(pmd_trans_huge(*pmd));
884 pte = pte_offset_map(pmd, address);
885 if (pte_none(*pte))
886 goto unmap;
887 *vma = get_gate_vma(mm);
888 if (!page)
889 goto out;
890 *page = vm_normal_page(*vma, address, *pte);
891 if (!*page) {
892 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
893 goto unmap;
894 *page = pte_page(*pte);
895 }
896 if (unlikely(!try_grab_page(*page, gup_flags))) {
897 ret = -ENOMEM;
898 goto unmap;
899 }
900out:
901 ret = 0;
902unmap:
903 pte_unmap(pte);
904 return ret;
905}
906
907
908
909
910
911
912static int faultin_page(struct vm_area_struct *vma,
913 unsigned long address, unsigned int *flags, int *locked)
914{
915 unsigned int fault_flags = 0;
916 vm_fault_t ret;
917
918
919 if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
920 return -ENOENT;
921 if (*flags & FOLL_WRITE)
922 fault_flags |= FAULT_FLAG_WRITE;
923 if (*flags & FOLL_REMOTE)
924 fault_flags |= FAULT_FLAG_REMOTE;
925 if (locked)
926 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
927 if (*flags & FOLL_NOWAIT)
928 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
929 if (*flags & FOLL_TRIED) {
930
931
932
933
934 fault_flags |= FAULT_FLAG_TRIED;
935 }
936
937 ret = handle_mm_fault(vma, address, fault_flags, NULL);
938 if (ret & VM_FAULT_ERROR) {
939 int err = vm_fault_to_errno(ret, *flags);
940
941 if (err)
942 return err;
943 BUG();
944 }
945
946 if (ret & VM_FAULT_RETRY) {
947 if (locked && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
948 *locked = 0;
949 return -EBUSY;
950 }
951
952
953
954
955
956
957
958
959
960
961 if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
962 *flags |= FOLL_COW;
963 return 0;
964}
965
966static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
967{
968 vm_flags_t vm_flags = vma->vm_flags;
969 int write = (gup_flags & FOLL_WRITE);
970 int foreign = (gup_flags & FOLL_REMOTE);
971
972 if (vm_flags & (VM_IO | VM_PFNMAP))
973 return -EFAULT;
974
975 if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))
976 return -EFAULT;
977
978 if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma))
979 return -EOPNOTSUPP;
980
981 if (vma_is_secretmem(vma))
982 return -EFAULT;
983
984 if (write) {
985 if (!(vm_flags & VM_WRITE)) {
986 if (!(gup_flags & FOLL_FORCE))
987 return -EFAULT;
988
989
990
991
992
993
994
995
996
997 if (!is_cow_mapping(vm_flags))
998 return -EFAULT;
999 }
1000 } else if (!(vm_flags & VM_READ)) {
1001 if (!(gup_flags & FOLL_FORCE))
1002 return -EFAULT;
1003
1004
1005
1006
1007 if (!(vm_flags & VM_MAYREAD))
1008 return -EFAULT;
1009 }
1010
1011
1012
1013
1014 if (!arch_vma_access_permitted(vma, write, false, foreign))
1015 return -EFAULT;
1016 return 0;
1017}
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079static long __get_user_pages(struct mm_struct *mm,
1080 unsigned long start, unsigned long nr_pages,
1081 unsigned int gup_flags, struct page **pages,
1082 struct vm_area_struct **vmas, int *locked)
1083{
1084 long ret = 0, i = 0;
1085 struct vm_area_struct *vma = NULL;
1086 struct follow_page_context ctx = { NULL };
1087
1088 if (!nr_pages)
1089 return 0;
1090
1091 start = untagged_addr(start);
1092
1093 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
1094
1095
1096
1097
1098
1099
1100 if (!(gup_flags & FOLL_FORCE))
1101 gup_flags |= FOLL_NUMA;
1102
1103 do {
1104 struct page *page;
1105 unsigned int foll_flags = gup_flags;
1106 unsigned int page_increm;
1107
1108
1109 if (!vma || start >= vma->vm_end) {
1110 vma = find_extend_vma(mm, start);
1111 if (!vma && in_gate_area(mm, start)) {
1112 ret = get_gate_page(mm, start & PAGE_MASK,
1113 gup_flags, &vma,
1114 pages ? &pages[i] : NULL);
1115 if (ret)
1116 goto out;
1117 ctx.page_mask = 0;
1118 goto next_page;
1119 }
1120
1121 if (!vma) {
1122 ret = -EFAULT;
1123 goto out;
1124 }
1125 ret = check_vma_flags(vma, gup_flags);
1126 if (ret)
1127 goto out;
1128
1129 if (is_vm_hugetlb_page(vma)) {
1130 i = follow_hugetlb_page(mm, vma, pages, vmas,
1131 &start, &nr_pages, i,
1132 gup_flags, locked);
1133 if (locked && *locked == 0) {
1134
1135
1136
1137
1138
1139 BUG_ON(gup_flags & FOLL_NOWAIT);
1140 goto out;
1141 }
1142 continue;
1143 }
1144 }
1145retry:
1146
1147
1148
1149
1150 if (fatal_signal_pending(current)) {
1151 ret = -EINTR;
1152 goto out;
1153 }
1154 cond_resched();
1155
1156 page = follow_page_mask(vma, start, foll_flags, &ctx);
1157 if (!page) {
1158 ret = faultin_page(vma, start, &foll_flags, locked);
1159 switch (ret) {
1160 case 0:
1161 goto retry;
1162 case -EBUSY:
1163 ret = 0;
1164 fallthrough;
1165 case -EFAULT:
1166 case -ENOMEM:
1167 case -EHWPOISON:
1168 goto out;
1169 case -ENOENT:
1170 goto next_page;
1171 }
1172 BUG();
1173 } else if (PTR_ERR(page) == -EEXIST) {
1174
1175
1176
1177
1178 goto next_page;
1179 } else if (IS_ERR(page)) {
1180 ret = PTR_ERR(page);
1181 goto out;
1182 }
1183 if (pages) {
1184 pages[i] = page;
1185 flush_anon_page(vma, page, start);
1186 flush_dcache_page(page);
1187 ctx.page_mask = 0;
1188 }
1189next_page:
1190 if (vmas) {
1191 vmas[i] = vma;
1192 ctx.page_mask = 0;
1193 }
1194 page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
1195 if (page_increm > nr_pages)
1196 page_increm = nr_pages;
1197 i += page_increm;
1198 start += page_increm * PAGE_SIZE;
1199 nr_pages -= page_increm;
1200 } while (nr_pages);
1201out:
1202 if (ctx.pgmap)
1203 put_dev_pagemap(ctx.pgmap);
1204 return i ? i : ret;
1205}
1206
1207static bool vma_permits_fault(struct vm_area_struct *vma,
1208 unsigned int fault_flags)
1209{
1210 bool write = !!(fault_flags & FAULT_FLAG_WRITE);
1211 bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE);
1212 vm_flags_t vm_flags = write ? VM_WRITE : VM_READ;
1213
1214 if (!(vm_flags & vma->vm_flags))
1215 return false;
1216
1217
1218
1219
1220
1221
1222
1223
1224 if (!arch_vma_access_permitted(vma, write, false, foreign))
1225 return false;
1226
1227 return true;
1228}
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259int fixup_user_fault(struct mm_struct *mm,
1260 unsigned long address, unsigned int fault_flags,
1261 bool *unlocked)
1262{
1263 struct vm_area_struct *vma;
1264 vm_fault_t ret;
1265
1266 address = untagged_addr(address);
1267
1268 if (unlocked)
1269 fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
1270
1271retry:
1272 vma = find_extend_vma(mm, address);
1273 if (!vma || address < vma->vm_start)
1274 return -EFAULT;
1275
1276 if (!vma_permits_fault(vma, fault_flags))
1277 return -EFAULT;
1278
1279 if ((fault_flags & FAULT_FLAG_KILLABLE) &&
1280 fatal_signal_pending(current))
1281 return -EINTR;
1282
1283 ret = handle_mm_fault(vma, address, fault_flags, NULL);
1284 if (ret & VM_FAULT_ERROR) {
1285 int err = vm_fault_to_errno(ret, 0);
1286
1287 if (err)
1288 return err;
1289 BUG();
1290 }
1291
1292 if (ret & VM_FAULT_RETRY) {
1293 mmap_read_lock(mm);
1294 *unlocked = true;
1295 fault_flags |= FAULT_FLAG_TRIED;
1296 goto retry;
1297 }
1298
1299 return 0;
1300}
1301EXPORT_SYMBOL_GPL(fixup_user_fault);
1302
1303
1304
1305
1306
1307static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
1308 unsigned long start,
1309 unsigned long nr_pages,
1310 struct page **pages,
1311 struct vm_area_struct **vmas,
1312 int *locked,
1313 unsigned int flags)
1314{
1315 long ret, pages_done;
1316 bool lock_dropped;
1317
1318 if (locked) {
1319
1320 BUG_ON(vmas);
1321
1322 BUG_ON(*locked != 1);
1323 }
1324
1325 if (flags & FOLL_PIN)
1326 mm_set_has_pinned_flag(&mm->flags);
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337 if (pages && !(flags & FOLL_PIN))
1338 flags |= FOLL_GET;
1339
1340 pages_done = 0;
1341 lock_dropped = false;
1342 for (;;) {
1343 ret = __get_user_pages(mm, start, nr_pages, flags, pages,
1344 vmas, locked);
1345 if (!locked)
1346
1347 return ret;
1348
1349
1350 if (!*locked) {
1351 BUG_ON(ret < 0);
1352 BUG_ON(ret >= nr_pages);
1353 }
1354
1355 if (ret > 0) {
1356 nr_pages -= ret;
1357 pages_done += ret;
1358 if (!nr_pages)
1359 break;
1360 }
1361 if (*locked) {
1362
1363
1364
1365
1366 if (!pages_done)
1367 pages_done = ret;
1368 break;
1369 }
1370
1371
1372
1373
1374 if (likely(pages))
1375 pages += ret;
1376 start += ret << PAGE_SHIFT;
1377 lock_dropped = true;
1378
1379retry:
1380
1381
1382
1383
1384
1385
1386
1387
1388 if (fatal_signal_pending(current)) {
1389 if (!pages_done)
1390 pages_done = -EINTR;
1391 break;
1392 }
1393
1394 ret = mmap_read_lock_killable(mm);
1395 if (ret) {
1396 BUG_ON(ret > 0);
1397 if (!pages_done)
1398 pages_done = ret;
1399 break;
1400 }
1401
1402 *locked = 1;
1403 ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED,
1404 pages, NULL, locked);
1405 if (!*locked) {
1406
1407 BUG_ON(ret != 0);
1408 goto retry;
1409 }
1410 if (ret != 1) {
1411 BUG_ON(ret > 1);
1412 if (!pages_done)
1413 pages_done = ret;
1414 break;
1415 }
1416 nr_pages--;
1417 pages_done++;
1418 if (!nr_pages)
1419 break;
1420 if (likely(pages))
1421 pages++;
1422 start += PAGE_SIZE;
1423 }
1424 if (lock_dropped && *locked) {
1425
1426
1427
1428
1429 mmap_read_unlock(mm);
1430 *locked = 0;
1431 }
1432 return pages_done;
1433}
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455long populate_vma_page_range(struct vm_area_struct *vma,
1456 unsigned long start, unsigned long end, int *locked)
1457{
1458 struct mm_struct *mm = vma->vm_mm;
1459 unsigned long nr_pages = (end - start) / PAGE_SIZE;
1460 int gup_flags;
1461
1462 VM_BUG_ON(!PAGE_ALIGNED(start));
1463 VM_BUG_ON(!PAGE_ALIGNED(end));
1464 VM_BUG_ON_VMA(start < vma->vm_start, vma);
1465 VM_BUG_ON_VMA(end > vma->vm_end, vma);
1466 mmap_assert_locked(mm);
1467
1468 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
1469 if (vma->vm_flags & VM_LOCKONFAULT)
1470 gup_flags &= ~FOLL_POPULATE;
1471
1472
1473
1474
1475
1476 if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
1477 gup_flags |= FOLL_WRITE;
1478
1479
1480
1481
1482
1483 if (vma_is_accessible(vma))
1484 gup_flags |= FOLL_FORCE;
1485
1486
1487
1488
1489
1490 return __get_user_pages(mm, start, nr_pages, gup_flags,
1491 NULL, NULL, locked);
1492}
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start,
1518 unsigned long end, bool write, int *locked)
1519{
1520 struct mm_struct *mm = vma->vm_mm;
1521 unsigned long nr_pages = (end - start) / PAGE_SIZE;
1522 int gup_flags;
1523
1524 VM_BUG_ON(!PAGE_ALIGNED(start));
1525 VM_BUG_ON(!PAGE_ALIGNED(end));
1526 VM_BUG_ON_VMA(start < vma->vm_start, vma);
1527 VM_BUG_ON_VMA(end > vma->vm_end, vma);
1528 mmap_assert_locked(mm);
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK | FOLL_HWPOISON;
1541 if (write)
1542 gup_flags |= FOLL_WRITE;
1543
1544
1545
1546
1547
1548 if (check_vma_flags(vma, gup_flags))
1549 return -EINVAL;
1550
1551 return __get_user_pages(mm, start, nr_pages, gup_flags,
1552 NULL, NULL, locked);
1553}
1554
1555
1556
1557
1558
1559
1560
1561
1562int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
1563{
1564 struct mm_struct *mm = current->mm;
1565 unsigned long end, nstart, nend;
1566 struct vm_area_struct *vma = NULL;
1567 int locked = 0;
1568 long ret = 0;
1569
1570 end = start + len;
1571
1572 for (nstart = start; nstart < end; nstart = nend) {
1573
1574
1575
1576
1577 if (!locked) {
1578 locked = 1;
1579 mmap_read_lock(mm);
1580 vma = find_vma(mm, nstart);
1581 } else if (nstart >= vma->vm_end)
1582 vma = vma->vm_next;
1583 if (!vma || vma->vm_start >= end)
1584 break;
1585
1586
1587
1588
1589 nend = min(end, vma->vm_end);
1590 if (vma->vm_flags & (VM_IO | VM_PFNMAP))
1591 continue;
1592 if (nstart < vma->vm_start)
1593 nstart = vma->vm_start;
1594
1595
1596
1597
1598
1599 ret = populate_vma_page_range(vma, nstart, nend, &locked);
1600 if (ret < 0) {
1601 if (ignore_errors) {
1602 ret = 0;
1603 continue;
1604 }
1605 break;
1606 }
1607 nend = nstart + ret * PAGE_SIZE;
1608 ret = 0;
1609 }
1610 if (locked)
1611 mmap_read_unlock(mm);
1612 return ret;
1613}
1614#else
1615static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
1616 unsigned long nr_pages, struct page **pages,
1617 struct vm_area_struct **vmas, int *locked,
1618 unsigned int foll_flags)
1619{
1620 struct vm_area_struct *vma;
1621 unsigned long vm_flags;
1622 long i;
1623
1624
1625
1626
1627 vm_flags = (foll_flags & FOLL_WRITE) ?
1628 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
1629 vm_flags &= (foll_flags & FOLL_FORCE) ?
1630 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
1631
1632 for (i = 0; i < nr_pages; i++) {
1633 vma = find_vma(mm, start);
1634 if (!vma)
1635 goto finish_or_fault;
1636
1637
1638 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
1639 !(vm_flags & vma->vm_flags))
1640 goto finish_or_fault;
1641
1642 if (pages) {
1643 pages[i] = virt_to_page(start);
1644 if (pages[i])
1645 get_page(pages[i]);
1646 }
1647 if (vmas)
1648 vmas[i] = vma;
1649 start = (start + PAGE_SIZE) & PAGE_MASK;
1650 }
1651
1652 return i;
1653
1654finish_or_fault:
1655 return i ? : -EFAULT;
1656}
1657#endif
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673#ifdef CONFIG_ELF_CORE
1674struct page *get_dump_page(unsigned long addr)
1675{
1676 struct mm_struct *mm = current->mm;
1677 struct page *page;
1678 int locked = 1;
1679 int ret;
1680
1681 if (mmap_read_lock_killable(mm))
1682 return NULL;
1683 ret = __get_user_pages_locked(mm, addr, 1, &page, NULL, &locked,
1684 FOLL_FORCE | FOLL_DUMP | FOLL_GET);
1685 if (locked)
1686 mmap_read_unlock(mm);
1687 return (ret == 1) ? page : NULL;
1688}
1689#endif
1690
1691#ifdef CONFIG_MIGRATION
1692
1693
1694
1695
1696
1697
1698static long check_and_migrate_movable_pages(unsigned long nr_pages,
1699 struct page **pages,
1700 unsigned int gup_flags)
1701{
1702 unsigned long i;
1703 unsigned long isolation_error_count = 0;
1704 bool drain_allow = true;
1705 LIST_HEAD(movable_page_list);
1706 long ret = 0;
1707 struct page *prev_head = NULL;
1708 struct page *head;
1709 struct migration_target_control mtc = {
1710 .nid = NUMA_NO_NODE,
1711 .gfp_mask = GFP_USER | __GFP_NOWARN,
1712 };
1713
1714 for (i = 0; i < nr_pages; i++) {
1715 head = compound_head(pages[i]);
1716 if (head == prev_head)
1717 continue;
1718 prev_head = head;
1719
1720
1721
1722
1723 if (!is_pinnable_page(head)) {
1724 if (PageHuge(head)) {
1725 if (!isolate_huge_page(head, &movable_page_list))
1726 isolation_error_count++;
1727 } else {
1728 if (!PageLRU(head) && drain_allow) {
1729 lru_add_drain_all();
1730 drain_allow = false;
1731 }
1732
1733 if (isolate_lru_page(head)) {
1734 isolation_error_count++;
1735 continue;
1736 }
1737 list_add_tail(&head->lru, &movable_page_list);
1738 mod_node_page_state(page_pgdat(head),
1739 NR_ISOLATED_ANON +
1740 page_is_file_lru(head),
1741 thp_nr_pages(head));
1742 }
1743 }
1744 }
1745
1746
1747
1748
1749
1750 if (list_empty(&movable_page_list) && !isolation_error_count)
1751 return nr_pages;
1752
1753 if (gup_flags & FOLL_PIN) {
1754 unpin_user_pages(pages, nr_pages);
1755 } else {
1756 for (i = 0; i < nr_pages; i++)
1757 put_page(pages[i]);
1758 }
1759 if (!list_empty(&movable_page_list)) {
1760 ret = migrate_pages(&movable_page_list, alloc_migration_target,
1761 NULL, (unsigned long)&mtc, MIGRATE_SYNC,
1762 MR_LONGTERM_PIN, NULL);
1763 if (ret && !list_empty(&movable_page_list))
1764 putback_movable_pages(&movable_page_list);
1765 }
1766
1767 return ret > 0 ? -ENOMEM : ret;
1768}
1769#else
1770static long check_and_migrate_movable_pages(unsigned long nr_pages,
1771 struct page **pages,
1772 unsigned int gup_flags)
1773{
1774 return nr_pages;
1775}
1776#endif
1777
1778
1779
1780
1781
1782static long __gup_longterm_locked(struct mm_struct *mm,
1783 unsigned long start,
1784 unsigned long nr_pages,
1785 struct page **pages,
1786 struct vm_area_struct **vmas,
1787 unsigned int gup_flags)
1788{
1789 unsigned int flags;
1790 long rc;
1791
1792 if (!(gup_flags & FOLL_LONGTERM))
1793 return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1794 NULL, gup_flags);
1795 flags = memalloc_pin_save();
1796 do {
1797 rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1798 NULL, gup_flags);
1799 if (rc <= 0)
1800 break;
1801 rc = check_and_migrate_movable_pages(rc, pages, gup_flags);
1802 } while (!rc);
1803 memalloc_pin_restore(flags);
1804
1805 return rc;
1806}
1807
1808static bool is_valid_gup_flags(unsigned int gup_flags)
1809{
1810
1811
1812
1813
1814 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1815 return false;
1816
1817
1818
1819
1820
1821 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1822 return false;
1823
1824 return true;
1825}
1826
1827#ifdef CONFIG_MMU
1828static long __get_user_pages_remote(struct mm_struct *mm,
1829 unsigned long start, unsigned long nr_pages,
1830 unsigned int gup_flags, struct page **pages,
1831 struct vm_area_struct **vmas, int *locked)
1832{
1833
1834
1835
1836
1837
1838
1839
1840 if (gup_flags & FOLL_LONGTERM) {
1841 if (WARN_ON_ONCE(locked))
1842 return -EINVAL;
1843
1844
1845
1846
1847 return __gup_longterm_locked(mm, start, nr_pages, pages,
1848 vmas, gup_flags | FOLL_TOUCH |
1849 FOLL_REMOTE);
1850 }
1851
1852 return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
1853 locked,
1854 gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1855}
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917long get_user_pages_remote(struct mm_struct *mm,
1918 unsigned long start, unsigned long nr_pages,
1919 unsigned int gup_flags, struct page **pages,
1920 struct vm_area_struct **vmas, int *locked)
1921{
1922 if (!is_valid_gup_flags(gup_flags))
1923 return -EINVAL;
1924
1925 return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
1926 pages, vmas, locked);
1927}
1928EXPORT_SYMBOL(get_user_pages_remote);
1929
1930#else
1931long get_user_pages_remote(struct mm_struct *mm,
1932 unsigned long start, unsigned long nr_pages,
1933 unsigned int gup_flags, struct page **pages,
1934 struct vm_area_struct **vmas, int *locked)
1935{
1936 return 0;
1937}
1938
1939static long __get_user_pages_remote(struct mm_struct *mm,
1940 unsigned long start, unsigned long nr_pages,
1941 unsigned int gup_flags, struct page **pages,
1942 struct vm_area_struct **vmas, int *locked)
1943{
1944 return 0;
1945}
1946#endif
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964long get_user_pages(unsigned long start, unsigned long nr_pages,
1965 unsigned int gup_flags, struct page **pages,
1966 struct vm_area_struct **vmas)
1967{
1968 if (!is_valid_gup_flags(gup_flags))
1969 return -EINVAL;
1970
1971 return __gup_longterm_locked(current->mm, start, nr_pages,
1972 pages, vmas, gup_flags | FOLL_TOUCH);
1973}
1974EXPORT_SYMBOL(get_user_pages);
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
2011 unsigned int gup_flags, struct page **pages,
2012 int *locked)
2013{
2014
2015
2016
2017
2018
2019
2020 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
2021 return -EINVAL;
2022
2023
2024
2025
2026 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
2027 return -EINVAL;
2028
2029 return __get_user_pages_locked(current->mm, start, nr_pages,
2030 pages, NULL, locked,
2031 gup_flags | FOLL_TOUCH);
2032}
2033EXPORT_SYMBOL(get_user_pages_locked);
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2051 struct page **pages, unsigned int gup_flags)
2052{
2053 struct mm_struct *mm = current->mm;
2054 int locked = 1;
2055 long ret;
2056
2057
2058
2059
2060
2061
2062
2063 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
2064 return -EINVAL;
2065
2066 mmap_read_lock(mm);
2067 ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
2068 &locked, gup_flags | FOLL_TOUCH);
2069 if (locked)
2070 mmap_read_unlock(mm);
2071 return ret;
2072}
2073EXPORT_SYMBOL(get_user_pages_unlocked);
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108#ifdef CONFIG_HAVE_FAST_GUP
2109
2110static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
2111 unsigned int flags,
2112 struct page **pages)
2113{
2114 while ((*nr) - nr_start) {
2115 struct page *page = pages[--(*nr)];
2116
2117 ClearPageReferenced(page);
2118 if (flags & FOLL_PIN)
2119 unpin_user_page(page);
2120 else
2121 put_page(page);
2122 }
2123}
2124
2125#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
2126static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
2127 unsigned int flags, struct page **pages, int *nr)
2128{
2129 struct dev_pagemap *pgmap = NULL;
2130 int nr_start = *nr, ret = 0;
2131 pte_t *ptep, *ptem;
2132
2133 ptem = ptep = pte_offset_map(&pmd, addr);
2134 do {
2135 pte_t pte = ptep_get_lockless(ptep);
2136 struct page *head, *page;
2137
2138
2139
2140
2141
2142 if (pte_protnone(pte))
2143 goto pte_unmap;
2144
2145 if (!pte_access_permitted(pte, flags & FOLL_WRITE))
2146 goto pte_unmap;
2147
2148 if (pte_devmap(pte)) {
2149 if (unlikely(flags & FOLL_LONGTERM))
2150 goto pte_unmap;
2151
2152 pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
2153 if (unlikely(!pgmap)) {
2154 undo_dev_pagemap(nr, nr_start, flags, pages);
2155 goto pte_unmap;
2156 }
2157 } else if (pte_special(pte))
2158 goto pte_unmap;
2159
2160 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2161 page = pte_page(pte);
2162
2163 head = try_grab_compound_head(page, 1, flags);
2164 if (!head)
2165 goto pte_unmap;
2166
2167 if (unlikely(page_is_secretmem(page))) {
2168 put_compound_head(head, 1, flags);
2169 goto pte_unmap;
2170 }
2171
2172 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2173 put_compound_head(head, 1, flags);
2174 goto pte_unmap;
2175 }
2176
2177 VM_BUG_ON_PAGE(compound_head(page) != head, page);
2178
2179
2180
2181
2182
2183
2184
2185 if (flags & FOLL_PIN) {
2186 ret = arch_make_page_accessible(page);
2187 if (ret) {
2188 unpin_user_page(page);
2189 goto pte_unmap;
2190 }
2191 }
2192 SetPageReferenced(page);
2193 pages[*nr] = page;
2194 (*nr)++;
2195
2196 } while (ptep++, addr += PAGE_SIZE, addr != end);
2197
2198 ret = 1;
2199
2200pte_unmap:
2201 if (pgmap)
2202 put_dev_pagemap(pgmap);
2203 pte_unmap(ptem);
2204 return ret;
2205}
2206#else
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
2218 unsigned int flags, struct page **pages, int *nr)
2219{
2220 return 0;
2221}
2222#endif
2223
2224#if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
2225static int __gup_device_huge(unsigned long pfn, unsigned long addr,
2226 unsigned long end, unsigned int flags,
2227 struct page **pages, int *nr)
2228{
2229 int nr_start = *nr;
2230 struct dev_pagemap *pgmap = NULL;
2231 int ret = 1;
2232
2233 do {
2234 struct page *page = pfn_to_page(pfn);
2235
2236 pgmap = get_dev_pagemap(pfn, pgmap);
2237 if (unlikely(!pgmap)) {
2238 undo_dev_pagemap(nr, nr_start, flags, pages);
2239 ret = 0;
2240 break;
2241 }
2242 SetPageReferenced(page);
2243 pages[*nr] = page;
2244 if (unlikely(!try_grab_page(page, flags))) {
2245 undo_dev_pagemap(nr, nr_start, flags, pages);
2246 ret = 0;
2247 break;
2248 }
2249 (*nr)++;
2250 pfn++;
2251 } while (addr += PAGE_SIZE, addr != end);
2252
2253 put_dev_pagemap(pgmap);
2254 return ret;
2255}
2256
2257static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2258 unsigned long end, unsigned int flags,
2259 struct page **pages, int *nr)
2260{
2261 unsigned long fault_pfn;
2262 int nr_start = *nr;
2263
2264 fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2265 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2266 return 0;
2267
2268 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2269 undo_dev_pagemap(nr, nr_start, flags, pages);
2270 return 0;
2271 }
2272 return 1;
2273}
2274
2275static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2276 unsigned long end, unsigned int flags,
2277 struct page **pages, int *nr)
2278{
2279 unsigned long fault_pfn;
2280 int nr_start = *nr;
2281
2282 fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2283 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2284 return 0;
2285
2286 if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2287 undo_dev_pagemap(nr, nr_start, flags, pages);
2288 return 0;
2289 }
2290 return 1;
2291}
2292#else
2293static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2294 unsigned long end, unsigned int flags,
2295 struct page **pages, int *nr)
2296{
2297 BUILD_BUG();
2298 return 0;
2299}
2300
2301static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
2302 unsigned long end, unsigned int flags,
2303 struct page **pages, int *nr)
2304{
2305 BUILD_BUG();
2306 return 0;
2307}
2308#endif
2309
2310static int record_subpages(struct page *page, unsigned long addr,
2311 unsigned long end, struct page **pages)
2312{
2313 int nr;
2314
2315 for (nr = 0; addr != end; addr += PAGE_SIZE)
2316 pages[nr++] = page++;
2317
2318 return nr;
2319}
2320
2321#ifdef CONFIG_ARCH_HAS_HUGEPD
2322static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
2323 unsigned long sz)
2324{
2325 unsigned long __boundary = (addr + sz) & ~(sz-1);
2326 return (__boundary - 1 < end - 1) ? __boundary : end;
2327}
2328
2329static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
2330 unsigned long end, unsigned int flags,
2331 struct page **pages, int *nr)
2332{
2333 unsigned long pte_end;
2334 struct page *head, *page;
2335 pte_t pte;
2336 int refs;
2337
2338 pte_end = (addr + sz) & ~(sz-1);
2339 if (pte_end < end)
2340 end = pte_end;
2341
2342 pte = huge_ptep_get(ptep);
2343
2344 if (!pte_access_permitted(pte, flags & FOLL_WRITE))
2345 return 0;
2346
2347
2348 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
2349
2350 head = pte_page(pte);
2351 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2352 refs = record_subpages(page, addr, end, pages + *nr);
2353
2354 head = try_grab_compound_head(head, refs, flags);
2355 if (!head)
2356 return 0;
2357
2358 if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2359 put_compound_head(head, refs, flags);
2360 return 0;
2361 }
2362
2363 *nr += refs;
2364 SetPageReferenced(head);
2365 return 1;
2366}
2367
2368static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2369 unsigned int pdshift, unsigned long end, unsigned int flags,
2370 struct page **pages, int *nr)
2371{
2372 pte_t *ptep;
2373 unsigned long sz = 1UL << hugepd_shift(hugepd);
2374 unsigned long next;
2375
2376 ptep = hugepte_offset(hugepd, addr, pdshift);
2377 do {
2378 next = hugepte_addr_end(addr, end, sz);
2379 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
2380 return 0;
2381 } while (ptep++, addr = next, addr != end);
2382
2383 return 1;
2384}
2385#else
2386static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2387 unsigned int pdshift, unsigned long end, unsigned int flags,
2388 struct page **pages, int *nr)
2389{
2390 return 0;
2391}
2392#endif
2393
2394static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2395 unsigned long end, unsigned int flags,
2396 struct page **pages, int *nr)
2397{
2398 struct page *head, *page;
2399 int refs;
2400
2401 if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
2402 return 0;
2403
2404 if (pmd_devmap(orig)) {
2405 if (unlikely(flags & FOLL_LONGTERM))
2406 return 0;
2407 return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
2408 pages, nr);
2409 }
2410
2411 page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2412 refs = record_subpages(page, addr, end, pages + *nr);
2413
2414 head = try_grab_compound_head(pmd_page(orig), refs, flags);
2415 if (!head)
2416 return 0;
2417
2418 if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
2419 put_compound_head(head, refs, flags);
2420 return 0;
2421 }
2422
2423 *nr += refs;
2424 SetPageReferenced(head);
2425 return 1;
2426}
2427
2428static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2429 unsigned long end, unsigned int flags,
2430 struct page **pages, int *nr)
2431{
2432 struct page *head, *page;
2433 int refs;
2434
2435 if (!pud_access_permitted(orig, flags & FOLL_WRITE))
2436 return 0;
2437
2438 if (pud_devmap(orig)) {
2439 if (unlikely(flags & FOLL_LONGTERM))
2440 return 0;
2441 return __gup_device_huge_pud(orig, pudp, addr, end, flags,
2442 pages, nr);
2443 }
2444
2445 page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2446 refs = record_subpages(page, addr, end, pages + *nr);
2447
2448 head = try_grab_compound_head(pud_page(orig), refs, flags);
2449 if (!head)
2450 return 0;
2451
2452 if (unlikely(pud_val(orig) != pud_val(*pudp))) {
2453 put_compound_head(head, refs, flags);
2454 return 0;
2455 }
2456
2457 *nr += refs;
2458 SetPageReferenced(head);
2459 return 1;
2460}
2461
2462static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
2463 unsigned long end, unsigned int flags,
2464 struct page **pages, int *nr)
2465{
2466 int refs;
2467 struct page *head, *page;
2468
2469 if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
2470 return 0;
2471
2472 BUILD_BUG_ON(pgd_devmap(orig));
2473
2474 page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2475 refs = record_subpages(page, addr, end, pages + *nr);
2476
2477 head = try_grab_compound_head(pgd_page(orig), refs, flags);
2478 if (!head)
2479 return 0;
2480
2481 if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
2482 put_compound_head(head, refs, flags);
2483 return 0;
2484 }
2485
2486 *nr += refs;
2487 SetPageReferenced(head);
2488 return 1;
2489}
2490
2491static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
2492 unsigned int flags, struct page **pages, int *nr)
2493{
2494 unsigned long next;
2495 pmd_t *pmdp;
2496
2497 pmdp = pmd_offset_lockless(pudp, pud, addr);
2498 do {
2499 pmd_t pmd = READ_ONCE(*pmdp);
2500
2501 next = pmd_addr_end(addr, end);
2502 if (!pmd_present(pmd))
2503 return 0;
2504
2505 if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
2506 pmd_devmap(pmd))) {
2507
2508
2509
2510
2511
2512 if (pmd_protnone(pmd))
2513 return 0;
2514
2515 if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
2516 pages, nr))
2517 return 0;
2518
2519 } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
2520
2521
2522
2523
2524 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
2525 PMD_SHIFT, next, flags, pages, nr))
2526 return 0;
2527 } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
2528 return 0;
2529 } while (pmdp++, addr = next, addr != end);
2530
2531 return 1;
2532}
2533
2534static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
2535 unsigned int flags, struct page **pages, int *nr)
2536{
2537 unsigned long next;
2538 pud_t *pudp;
2539
2540 pudp = pud_offset_lockless(p4dp, p4d, addr);
2541 do {
2542 pud_t pud = READ_ONCE(*pudp);
2543
2544 next = pud_addr_end(addr, end);
2545 if (unlikely(!pud_present(pud)))
2546 return 0;
2547 if (unlikely(pud_huge(pud))) {
2548 if (!gup_huge_pud(pud, pudp, addr, next, flags,
2549 pages, nr))
2550 return 0;
2551 } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
2552 if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
2553 PUD_SHIFT, next, flags, pages, nr))
2554 return 0;
2555 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
2556 return 0;
2557 } while (pudp++, addr = next, addr != end);
2558
2559 return 1;
2560}
2561
2562static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
2563 unsigned int flags, struct page **pages, int *nr)
2564{
2565 unsigned long next;
2566 p4d_t *p4dp;
2567
2568 p4dp = p4d_offset_lockless(pgdp, pgd, addr);
2569 do {
2570 p4d_t p4d = READ_ONCE(*p4dp);
2571
2572 next = p4d_addr_end(addr, end);
2573 if (p4d_none(p4d))
2574 return 0;
2575 BUILD_BUG_ON(p4d_huge(p4d));
2576 if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
2577 if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
2578 P4D_SHIFT, next, flags, pages, nr))
2579 return 0;
2580 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
2581 return 0;
2582 } while (p4dp++, addr = next, addr != end);
2583
2584 return 1;
2585}
2586
2587static void gup_pgd_range(unsigned long addr, unsigned long end,
2588 unsigned int flags, struct page **pages, int *nr)
2589{
2590 unsigned long next;
2591 pgd_t *pgdp;
2592
2593 pgdp = pgd_offset(current->mm, addr);
2594 do {
2595 pgd_t pgd = READ_ONCE(*pgdp);
2596
2597 next = pgd_addr_end(addr, end);
2598 if (pgd_none(pgd))
2599 return;
2600 if (unlikely(pgd_huge(pgd))) {
2601 if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
2602 pages, nr))
2603 return;
2604 } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
2605 if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
2606 PGDIR_SHIFT, next, flags, pages, nr))
2607 return;
2608 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
2609 return;
2610 } while (pgdp++, addr = next, addr != end);
2611}
2612#else
2613static inline void gup_pgd_range(unsigned long addr, unsigned long end,
2614 unsigned int flags, struct page **pages, int *nr)
2615{
2616}
2617#endif
2618
2619#ifndef gup_fast_permitted
2620
2621
2622
2623
2624static bool gup_fast_permitted(unsigned long start, unsigned long end)
2625{
2626 return true;
2627}
2628#endif
2629
2630static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
2631 unsigned int gup_flags, struct page **pages)
2632{
2633 int ret;
2634
2635
2636
2637
2638
2639 if (gup_flags & FOLL_LONGTERM) {
2640 mmap_read_lock(current->mm);
2641 ret = __gup_longterm_locked(current->mm,
2642 start, nr_pages,
2643 pages, NULL, gup_flags);
2644 mmap_read_unlock(current->mm);
2645 } else {
2646 ret = get_user_pages_unlocked(start, nr_pages,
2647 pages, gup_flags);
2648 }
2649
2650 return ret;
2651}
2652
2653static unsigned long lockless_pages_from_mm(unsigned long start,
2654 unsigned long end,
2655 unsigned int gup_flags,
2656 struct page **pages)
2657{
2658 unsigned long flags;
2659 int nr_pinned = 0;
2660 unsigned seq;
2661
2662 if (!IS_ENABLED(CONFIG_HAVE_FAST_GUP) ||
2663 !gup_fast_permitted(start, end))
2664 return 0;
2665
2666 if (gup_flags & FOLL_PIN) {
2667 seq = raw_read_seqcount(¤t->mm->write_protect_seq);
2668 if (seq & 1)
2669 return 0;
2670 }
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683 local_irq_save(flags);
2684 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
2685 local_irq_restore(flags);
2686
2687
2688
2689
2690
2691 if (gup_flags & FOLL_PIN) {
2692 if (read_seqcount_retry(¤t->mm->write_protect_seq, seq)) {
2693 unpin_user_pages(pages, nr_pinned);
2694 return 0;
2695 }
2696 }
2697 return nr_pinned;
2698}
2699
2700static int internal_get_user_pages_fast(unsigned long start,
2701 unsigned long nr_pages,
2702 unsigned int gup_flags,
2703 struct page **pages)
2704{
2705 unsigned long len, end;
2706 unsigned long nr_pinned;
2707 int ret;
2708
2709 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
2710 FOLL_FORCE | FOLL_PIN | FOLL_GET |
2711 FOLL_FAST_ONLY)))
2712 return -EINVAL;
2713
2714 if (gup_flags & FOLL_PIN)
2715 mm_set_has_pinned_flag(¤t->mm->flags);
2716
2717 if (!(gup_flags & FOLL_FAST_ONLY))
2718 might_lock_read(¤t->mm->mmap_lock);
2719
2720 start = untagged_addr(start) & PAGE_MASK;
2721 len = nr_pages << PAGE_SHIFT;
2722 if (check_add_overflow(start, len, &end))
2723 return 0;
2724 if (unlikely(!access_ok((void __user *)start, len)))
2725 return -EFAULT;
2726
2727 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
2728 if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
2729 return nr_pinned;
2730
2731
2732 start += nr_pinned << PAGE_SHIFT;
2733 pages += nr_pinned;
2734 ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags,
2735 pages);
2736 if (ret < 0) {
2737
2738
2739
2740
2741 if (nr_pinned)
2742 return nr_pinned;
2743 return ret;
2744 }
2745 return ret + nr_pinned;
2746}
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768int get_user_pages_fast_only(unsigned long start, int nr_pages,
2769 unsigned int gup_flags, struct page **pages)
2770{
2771 int nr_pinned;
2772
2773
2774
2775
2776
2777
2778
2779 gup_flags |= FOLL_GET | FOLL_FAST_ONLY;
2780
2781 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2782 pages);
2783
2784
2785
2786
2787
2788
2789
2790 if (nr_pinned < 0)
2791 nr_pinned = 0;
2792
2793 return nr_pinned;
2794}
2795EXPORT_SYMBOL_GPL(get_user_pages_fast_only);
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813int get_user_pages_fast(unsigned long start, int nr_pages,
2814 unsigned int gup_flags, struct page **pages)
2815{
2816 if (!is_valid_gup_flags(gup_flags))
2817 return -EINVAL;
2818
2819
2820
2821
2822
2823
2824
2825 gup_flags |= FOLL_GET;
2826 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2827}
2828EXPORT_SYMBOL_GPL(get_user_pages_fast);
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846int pin_user_pages_fast(unsigned long start, int nr_pages,
2847 unsigned int gup_flags, struct page **pages)
2848{
2849
2850 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2851 return -EINVAL;
2852
2853 gup_flags |= FOLL_PIN;
2854 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2855}
2856EXPORT_SYMBOL_GPL(pin_user_pages_fast);
2857
2858
2859
2860
2861
2862
2863
2864int pin_user_pages_fast_only(unsigned long start, int nr_pages,
2865 unsigned int gup_flags, struct page **pages)
2866{
2867 int nr_pinned;
2868
2869
2870
2871
2872
2873 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2874 return 0;
2875
2876
2877
2878
2879 gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY);
2880 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2881 pages);
2882
2883
2884
2885
2886
2887 if (nr_pinned < 0)
2888 nr_pinned = 0;
2889
2890 return nr_pinned;
2891}
2892EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917long pin_user_pages_remote(struct mm_struct *mm,
2918 unsigned long start, unsigned long nr_pages,
2919 unsigned int gup_flags, struct page **pages,
2920 struct vm_area_struct **vmas, int *locked)
2921{
2922
2923 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2924 return -EINVAL;
2925
2926 gup_flags |= FOLL_PIN;
2927 return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
2928 pages, vmas, locked);
2929}
2930EXPORT_SYMBOL(pin_user_pages_remote);
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950long pin_user_pages(unsigned long start, unsigned long nr_pages,
2951 unsigned int gup_flags, struct page **pages,
2952 struct vm_area_struct **vmas)
2953{
2954
2955 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2956 return -EINVAL;
2957
2958 gup_flags |= FOLL_PIN;
2959 return __gup_longterm_locked(current->mm, start, nr_pages,
2960 pages, vmas, gup_flags);
2961}
2962EXPORT_SYMBOL(pin_user_pages);
2963
2964
2965
2966
2967
2968
2969long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
2970 struct page **pages, unsigned int gup_flags)
2971{
2972
2973 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2974 return -EINVAL;
2975
2976 gup_flags |= FOLL_PIN;
2977 return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);
2978}
2979EXPORT_SYMBOL(pin_user_pages_unlocked);
2980
2981
2982
2983
2984
2985
2986long pin_user_pages_locked(unsigned long start, unsigned long nr_pages,
2987 unsigned int gup_flags, struct page **pages,
2988 int *locked)
2989{
2990
2991
2992
2993
2994
2995
2996 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
2997 return -EINVAL;
2998
2999
3000 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
3001 return -EINVAL;
3002
3003 gup_flags |= FOLL_PIN;
3004 return __get_user_pages_locked(current->mm, start, nr_pages,
3005 pages, NULL, locked,
3006 gup_flags | FOLL_TOUCH);
3007}
3008EXPORT_SYMBOL(pin_user_pages_locked);
3009