1
2
3
4
5
6
7
8
9
10
11#include <linux/sched.h>
12#include <linux/mm.h>
13#include <linux/vmstat.h>
14#include <linux/highmem.h>
15#include <asm/pgtable.h>
16
17static inline pte_t gup_get_pte(pte_t *ptep)
18{
19#ifndef CONFIG_X2TLB
20 return ACCESS_ONCE(*ptep);
21#else
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55 pte_t pte;
56
57retry:
58 pte.pte_low = ptep->pte_low;
59 smp_rmb();
60 pte.pte_high = ptep->pte_high;
61 smp_rmb();
62 if (unlikely(pte.pte_low != ptep->pte_low))
63 goto retry;
64
65 return pte;
66#endif
67}
68
69
70
71
72
73
74static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
75 unsigned long end, int write, struct page **pages, int *nr)
76{
77 u64 mask, result;
78 pte_t *ptep;
79
80#ifdef CONFIG_X2TLB
81 result = _PAGE_PRESENT | _PAGE_EXT(_PAGE_EXT_KERN_READ | _PAGE_EXT_USER_READ);
82 if (write)
83 result |= _PAGE_EXT(_PAGE_EXT_KERN_WRITE | _PAGE_EXT_USER_WRITE);
84#elif defined(CONFIG_SUPERH64)
85 result = _PAGE_PRESENT | _PAGE_USER | _PAGE_READ;
86 if (write)
87 result |= _PAGE_WRITE;
88#else
89 result = _PAGE_PRESENT | _PAGE_USER;
90 if (write)
91 result |= _PAGE_RW;
92#endif
93
94 mask = result | _PAGE_SPECIAL;
95
96 ptep = pte_offset_map(&pmd, addr);
97 do {
98 pte_t pte = gup_get_pte(ptep);
99 struct page *page;
100
101 if ((pte_val(pte) & mask) != result) {
102 pte_unmap(ptep);
103 return 0;
104 }
105 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
106 page = pte_page(pte);
107 get_page(page);
108 pages[*nr] = page;
109 (*nr)++;
110
111 } while (ptep++, addr += PAGE_SIZE, addr != end);
112 pte_unmap(ptep - 1);
113
114 return 1;
115}
116
117static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
118 int write, struct page **pages, int *nr)
119{
120 unsigned long next;
121 pmd_t *pmdp;
122
123 pmdp = pmd_offset(&pud, addr);
124 do {
125 pmd_t pmd = *pmdp;
126
127 next = pmd_addr_end(addr, end);
128 if (pmd_none(pmd))
129 return 0;
130 if (!gup_pte_range(pmd, addr, next, write, pages, nr))
131 return 0;
132 } while (pmdp++, addr = next, addr != end);
133
134 return 1;
135}
136
137static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
138 int write, struct page **pages, int *nr)
139{
140 unsigned long next;
141 pud_t *pudp;
142
143 pudp = pud_offset(&pgd, addr);
144 do {
145 pud_t pud = *pudp;
146
147 next = pud_addr_end(addr, end);
148 if (pud_none(pud))
149 return 0;
150 if (!gup_pmd_range(pud, addr, next, write, pages, nr))
151 return 0;
152 } while (pudp++, addr = next, addr != end);
153
154 return 1;
155}
156
157
158
159
160
161int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
162 struct page **pages)
163{
164 struct mm_struct *mm = current->mm;
165 unsigned long addr, len, end;
166 unsigned long next;
167 unsigned long flags;
168 pgd_t *pgdp;
169 int nr = 0;
170
171 start &= PAGE_MASK;
172 addr = start;
173 len = (unsigned long) nr_pages << PAGE_SHIFT;
174 end = start + len;
175 if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
176 (void __user *)start, len)))
177 return 0;
178
179
180
181
182
183 local_irq_save(flags);
184 pgdp = pgd_offset(mm, addr);
185 do {
186 pgd_t pgd = *pgdp;
187
188 next = pgd_addr_end(addr, end);
189 if (pgd_none(pgd))
190 break;
191 if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
192 break;
193 } while (pgdp++, addr = next, addr != end);
194 local_irq_restore(flags);
195
196 return nr;
197}
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215int get_user_pages_fast(unsigned long start, int nr_pages, int write,
216 struct page **pages)
217{
218 struct mm_struct *mm = current->mm;
219 unsigned long addr, len, end;
220 unsigned long next;
221 pgd_t *pgdp;
222 int nr = 0;
223
224 start &= PAGE_MASK;
225 addr = start;
226 len = (unsigned long) nr_pages << PAGE_SHIFT;
227
228 end = start + len;
229 if (end < start)
230 goto slow_irqon;
231
232 local_irq_disable();
233 pgdp = pgd_offset(mm, addr);
234 do {
235 pgd_t pgd = *pgdp;
236
237 next = pgd_addr_end(addr, end);
238 if (pgd_none(pgd))
239 goto slow;
240 if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
241 goto slow;
242 } while (pgdp++, addr = next, addr != end);
243 local_irq_enable();
244
245 VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
246 return nr;
247
248 {
249 int ret;
250
251slow:
252 local_irq_enable();
253slow_irqon:
254
255 start += nr << PAGE_SHIFT;
256 pages += nr;
257
258 down_read(&mm->mmap_sem);
259 ret = get_user_pages(current, mm, start,
260 (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
261 up_read(&mm->mmap_sem);
262
263
264 if (nr > 0) {
265 if (ret < 0)
266 ret = nr;
267 else
268 ret += nr;
269 }
270
271 return ret;
272 }
273}
274