1
2
3
4
5
6
7
8
9
10#include <linux/pagemap.h>
11#include <linux/gfp.h>
12#include <linux/mm.h>
13#include <linux/mman.h>
14#include <linux/syscalls.h>
15#include <linux/swap.h>
16#include <linux/swapops.h>
17#include <linux/hugetlb.h>
18
19#include <asm/uaccess.h>
20#include <asm/pgtable.h>
21
22static void mincore_hugetlb_page_range(struct vm_area_struct *vma,
23 unsigned long addr, unsigned long end,
24 unsigned char *vec)
25{
26#ifdef CONFIG_HUGETLB_PAGE
27 struct hstate *h;
28
29 h = hstate_vma(vma);
30 while (1) {
31 unsigned char present;
32 pte_t *ptep;
33
34
35
36
37 ptep = huge_pte_offset(current->mm,
38 addr & huge_page_mask(h));
39 present = ptep && !huge_pte_none(huge_ptep_get(ptep));
40 while (1) {
41 *vec = present;
42 vec++;
43 addr += PAGE_SIZE;
44 if (addr == end)
45 return;
46
47 if (!(addr & ~huge_page_mask(h)))
48 break;
49 }
50 }
51#else
52 BUG();
53#endif
54}
55
56
57
58
59
60
61
62static unsigned char mincore_page(struct address_space *mapping, pgoff_t pgoff)
63{
64 unsigned char present = 0;
65 struct page *page;
66
67
68
69
70
71
72
73 page = find_get_page(mapping, pgoff);
74#ifdef CONFIG_SWAP
75
76 if (radix_tree_exceptional_entry(page)) {
77 swp_entry_t swap = radix_to_swp_entry(page);
78 page = find_get_page(swap_address_space(swap), swap.val);
79 }
80#endif
81 if (page) {
82 present = PageUptodate(page);
83 page_cache_release(page);
84 }
85
86 return present;
87}
88
89static void mincore_unmapped_range(struct vm_area_struct *vma,
90 unsigned long addr, unsigned long end,
91 unsigned char *vec)
92{
93 unsigned long nr = (end - addr) >> PAGE_SHIFT;
94 int i;
95
96 if (vma->vm_file) {
97 pgoff_t pgoff;
98
99 pgoff = linear_page_index(vma, addr);
100 for (i = 0; i < nr; i++, pgoff++)
101 vec[i] = mincore_page(vma->vm_file->f_mapping, pgoff);
102 } else {
103 for (i = 0; i < nr; i++)
104 vec[i] = 0;
105 }
106}
107
108static void mincore_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
109 unsigned long addr, unsigned long end,
110 unsigned char *vec)
111{
112 unsigned long next;
113 spinlock_t *ptl;
114 pte_t *ptep;
115
116 ptep = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
117 do {
118 pte_t pte = *ptep;
119 pgoff_t pgoff;
120
121 next = addr + PAGE_SIZE;
122 if (pte_none(pte))
123 mincore_unmapped_range(vma, addr, next, vec);
124 else if (pte_present(pte))
125 *vec = 1;
126 else if (pte_file(pte)) {
127 pgoff = pte_to_pgoff(pte);
128 *vec = mincore_page(vma->vm_file->f_mapping, pgoff);
129 } else {
130 swp_entry_t entry = pte_to_swp_entry(pte);
131
132 if (is_migration_entry(entry)) {
133
134 *vec = 1;
135 } else {
136#ifdef CONFIG_SWAP
137 pgoff = entry.val;
138 *vec = mincore_page(swap_address_space(entry),
139 pgoff);
140#else
141 WARN_ON(1);
142 *vec = 1;
143#endif
144 }
145 }
146 vec++;
147 } while (ptep++, addr = next, addr != end);
148 pte_unmap_unlock(ptep - 1, ptl);
149}
150
151static void mincore_pmd_range(struct vm_area_struct *vma, pud_t *pud,
152 unsigned long addr, unsigned long end,
153 unsigned char *vec)
154{
155 unsigned long next;
156 pmd_t *pmd;
157
158 pmd = pmd_offset(pud, addr);
159 do {
160 next = pmd_addr_end(addr, end);
161 if (pmd_trans_huge(*pmd)) {
162 if (mincore_huge_pmd(vma, pmd, addr, next, vec)) {
163 vec += (next - addr) >> PAGE_SHIFT;
164 continue;
165 }
166
167 }
168 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
169 mincore_unmapped_range(vma, addr, next, vec);
170 else
171 mincore_pte_range(vma, pmd, addr, next, vec);
172 vec += (next - addr) >> PAGE_SHIFT;
173 } while (pmd++, addr = next, addr != end);
174}
175
176static void mincore_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
177 unsigned long addr, unsigned long end,
178 unsigned char *vec)
179{
180 unsigned long next;
181 pud_t *pud;
182
183 pud = pud_offset(pgd, addr);
184 do {
185 next = pud_addr_end(addr, end);
186 if (pud_none_or_clear_bad(pud))
187 mincore_unmapped_range(vma, addr, next, vec);
188 else
189 mincore_pmd_range(vma, pud, addr, next, vec);
190 vec += (next - addr) >> PAGE_SHIFT;
191 } while (pud++, addr = next, addr != end);
192}
193
194static void mincore_page_range(struct vm_area_struct *vma,
195 unsigned long addr, unsigned long end,
196 unsigned char *vec)
197{
198 unsigned long next;
199 pgd_t *pgd;
200
201 pgd = pgd_offset(vma->vm_mm, addr);
202 do {
203 next = pgd_addr_end(addr, end);
204 if (pgd_none_or_clear_bad(pgd))
205 mincore_unmapped_range(vma, addr, next, vec);
206 else
207 mincore_pud_range(vma, pgd, addr, next, vec);
208 vec += (next - addr) >> PAGE_SHIFT;
209 } while (pgd++, addr = next, addr != end);
210}
211
212
213
214
215
216
217static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *vec)
218{
219 struct vm_area_struct *vma;
220 unsigned long end;
221
222 vma = find_vma(current->mm, addr);
223 if (!vma || addr < vma->vm_start)
224 return -ENOMEM;
225
226 end = min(vma->vm_end, addr + (pages << PAGE_SHIFT));
227
228 if (is_vm_hugetlb_page(vma)) {
229 mincore_hugetlb_page_range(vma, addr, end, vec);
230 return (end - addr) >> PAGE_SHIFT;
231 }
232
233 end = pmd_addr_end(addr, end);
234
235 if (is_vm_hugetlb_page(vma))
236 mincore_hugetlb_page_range(vma, addr, end, vec);
237 else
238 mincore_page_range(vma, addr, end, vec);
239
240 return (end - addr) >> PAGE_SHIFT;
241}
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len,
268 unsigned char __user *, vec)
269{
270 long retval;
271 unsigned long pages;
272 unsigned char *tmp;
273
274
275 if (start & ~PAGE_CACHE_MASK)
276 return -EINVAL;
277
278
279 if (!access_ok(VERIFY_READ, (void __user *) start, len))
280 return -ENOMEM;
281
282
283 pages = len >> PAGE_SHIFT;
284 pages += (len & ~PAGE_MASK) != 0;
285
286 if (!access_ok(VERIFY_WRITE, vec, pages))
287 return -EFAULT;
288
289 tmp = (void *) __get_free_page(GFP_USER);
290 if (!tmp)
291 return -EAGAIN;
292
293 retval = 0;
294 while (pages) {
295
296
297
298
299 down_read(¤t->mm->mmap_sem);
300 retval = do_mincore(start, min(pages, PAGE_SIZE), tmp);
301 up_read(¤t->mm->mmap_sem);
302
303 if (retval <= 0)
304 break;
305 if (copy_to_user(vec, tmp, retval)) {
306 retval = -EFAULT;
307 break;
308 }
309 pages -= retval;
310 vec += retval;
311 start += retval << PAGE_SHIFT;
312 retval = 0;
313 }
314 free_page((unsigned long) tmp);
315 return retval;
316}
317