1#include <linux/mm.h>
2#include <linux/highmem.h>
3#include <linux/sched.h>
4#include <linux/hugetlb.h>
5
6static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
7 struct mm_walk *walk)
8{
9 pte_t *pte;
10 int err = 0;
11
12 pte = pte_offset_map(pmd, addr);
13 for (;;) {
14 err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk);
15 if (err)
16 break;
17 addr += PAGE_SIZE;
18 if (addr == end)
19 break;
20 pte++;
21 }
22
23 pte_unmap(pte);
24 return err;
25}
26
27static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
28 struct mm_walk *walk)
29{
30 pmd_t *pmd;
31 unsigned long next;
32 int err = 0;
33
34 pmd = pmd_offset(pud, addr);
35 do {
36again:
37 next = pmd_addr_end(addr, end);
38 if (pmd_none(*pmd) || !walk->vma) {
39 if (walk->pte_hole)
40 err = walk->pte_hole(addr, next, walk);
41 if (err)
42 break;
43 continue;
44 }
45
46
47
48
49 if (walk->pmd_entry)
50 err = walk->pmd_entry(pmd, addr, next, walk);
51 if (err)
52 break;
53
54
55
56
57
58 if (!walk->pte_entry)
59 continue;
60
61 split_huge_pmd(walk->vma, pmd, addr);
62 if (pmd_trans_unstable(pmd))
63 goto again;
64 err = walk_pte_range(pmd, addr, next, walk);
65 if (err)
66 break;
67 } while (pmd++, addr = next, addr != end);
68
69 return err;
70}
71
72static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end,
73 struct mm_walk *walk)
74{
75 pud_t *pud;
76 unsigned long next;
77 int err = 0;
78
79 pud = pud_offset(pgd, addr);
80 do {
81 next = pud_addr_end(addr, end);
82 if (pud_none_or_clear_bad(pud)) {
83 if (walk->pte_hole)
84 err = walk->pte_hole(addr, next, walk);
85 if (err)
86 break;
87 continue;
88 }
89 if (walk->pmd_entry || walk->pte_entry)
90 err = walk_pmd_range(pud, addr, next, walk);
91 if (err)
92 break;
93 } while (pud++, addr = next, addr != end);
94
95 return err;
96}
97
98static int walk_pgd_range(unsigned long addr, unsigned long end,
99 struct mm_walk *walk)
100{
101 pgd_t *pgd;
102 unsigned long next;
103 int err = 0;
104
105 pgd = pgd_offset(walk->mm, addr);
106 do {
107 next = pgd_addr_end(addr, end);
108 if (pgd_none_or_clear_bad(pgd)) {
109 if (walk->pte_hole)
110 err = walk->pte_hole(addr, next, walk);
111 if (err)
112 break;
113 continue;
114 }
115 if (walk->pmd_entry || walk->pte_entry)
116 err = walk_pud_range(pgd, addr, next, walk);
117 if (err)
118 break;
119 } while (pgd++, addr = next, addr != end);
120
121 return err;
122}
123
124#ifdef CONFIG_HUGETLB_PAGE
125static unsigned long hugetlb_entry_end(struct hstate *h, unsigned long addr,
126 unsigned long end)
127{
128 unsigned long boundary = (addr & huge_page_mask(h)) + huge_page_size(h);
129 return boundary < end ? boundary : end;
130}
131
132static int walk_hugetlb_range(unsigned long addr, unsigned long end,
133 struct mm_walk *walk)
134{
135 struct vm_area_struct *vma = walk->vma;
136 struct hstate *h = hstate_vma(vma);
137 unsigned long next;
138 unsigned long hmask = huge_page_mask(h);
139 pte_t *pte;
140 int err = 0;
141
142 do {
143 next = hugetlb_entry_end(h, addr, end);
144 pte = huge_pte_offset(walk->mm, addr & hmask);
145 if (pte && walk->hugetlb_entry)
146 err = walk->hugetlb_entry(pte, hmask, addr, next, walk);
147 if (err)
148 break;
149 } while (addr = next, addr != end);
150
151 return err;
152}
153
154#else
155static int walk_hugetlb_range(unsigned long addr, unsigned long end,
156 struct mm_walk *walk)
157{
158 return 0;
159}
160
161#endif
162
163
164
165
166
167
168
169static int walk_page_test(unsigned long start, unsigned long end,
170 struct mm_walk *walk)
171{
172 struct vm_area_struct *vma = walk->vma;
173
174 if (walk->test_walk)
175 return walk->test_walk(start, end, walk);
176
177
178
179
180
181
182
183
184
185 if (vma->vm_flags & VM_PFNMAP) {
186 int err = 1;
187 if (walk->pte_hole)
188 err = walk->pte_hole(start, end, walk);
189 return err ? err : 1;
190 }
191 return 0;
192}
193
194static int __walk_page_range(unsigned long start, unsigned long end,
195 struct mm_walk *walk)
196{
197 int err = 0;
198 struct vm_area_struct *vma = walk->vma;
199
200 if (vma && is_vm_hugetlb_page(vma)) {
201 if (walk->hugetlb_entry)
202 err = walk_hugetlb_range(start, end, walk);
203 } else
204 err = walk_pgd_range(start, end, walk);
205
206 return err;
207}
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239int walk_page_range(unsigned long start, unsigned long end,
240 struct mm_walk *walk)
241{
242 int err = 0;
243 unsigned long next;
244 struct vm_area_struct *vma;
245
246 if (start >= end)
247 return -EINVAL;
248
249 if (!walk->mm)
250 return -EINVAL;
251
252 VM_BUG_ON_MM(!rwsem_is_locked(&walk->mm->mmap_sem), walk->mm);
253
254 vma = find_vma(walk->mm, start);
255 do {
256 if (!vma) {
257 walk->vma = NULL;
258 next = end;
259 } else if (start < vma->vm_start) {
260 walk->vma = NULL;
261 next = min(end, vma->vm_start);
262 } else {
263 walk->vma = vma;
264 next = min(end, vma->vm_end);
265 vma = vma->vm_next;
266
267 err = walk_page_test(start, next, walk);
268 if (err > 0) {
269
270
271
272
273
274 err = 0;
275 continue;
276 }
277 if (err < 0)
278 break;
279 }
280 if (walk->vma || walk->pte_hole)
281 err = __walk_page_range(start, next, walk);
282 if (err)
283 break;
284 } while (start = next, start < end);
285 return err;
286}
287
288int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk)
289{
290 int err;
291
292 if (!walk->mm)
293 return -EINVAL;
294
295 VM_BUG_ON(!rwsem_is_locked(&walk->mm->mmap_sem));
296 VM_BUG_ON(!vma);
297 walk->vma = vma;
298 err = walk_page_test(vma->vm_start, vma->vm_end, walk);
299 if (err > 0)
300 return 0;
301 if (err < 0)
302 return err;
303 return __walk_page_range(vma->vm_start, vma->vm_end, walk);
304}
305