1
2#include <linux/pagewalk.h>
3#include <linux/hugetlb.h>
4#include <linux/bitops.h>
5#include <linux/mmu_notifier.h>
6#include <asm/cacheflush.h>
7#include <asm/tlbflush.h>
8
9
10
11
12
13
14
15
16struct wp_walk {
17 struct mmu_notifier_range range;
18 unsigned long tlbflush_start;
19 unsigned long tlbflush_end;
20 unsigned long total;
21};
22
23
24
25
26
27
28
29
30
31
32static int wp_pte(pte_t *pte, unsigned long addr, unsigned long end,
33 struct mm_walk *walk)
34{
35 struct wp_walk *wpwalk = walk->private;
36 pte_t ptent = *pte;
37
38 if (pte_write(ptent)) {
39 pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte);
40
41 ptent = pte_wrprotect(old_pte);
42 ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent);
43 wpwalk->total++;
44 wpwalk->tlbflush_start = min(wpwalk->tlbflush_start, addr);
45 wpwalk->tlbflush_end = max(wpwalk->tlbflush_end,
46 addr + PAGE_SIZE);
47 }
48
49 return 0;
50}
51
52
53
54
55
56
57
58
59
60
61
62
63struct clean_walk {
64 struct wp_walk base;
65 pgoff_t bitmap_pgoff;
66 unsigned long *bitmap;
67 pgoff_t start;
68 pgoff_t end;
69};
70
71#define to_clean_walk(_wpwalk) container_of(_wpwalk, struct clean_walk, base)
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86static int clean_record_pte(pte_t *pte, unsigned long addr,
87 unsigned long end, struct mm_walk *walk)
88{
89 struct wp_walk *wpwalk = walk->private;
90 struct clean_walk *cwalk = to_clean_walk(wpwalk);
91 pte_t ptent = *pte;
92
93 if (pte_dirty(ptent)) {
94 pgoff_t pgoff = ((addr - walk->vma->vm_start) >> PAGE_SHIFT) +
95 walk->vma->vm_pgoff - cwalk->bitmap_pgoff;
96 pte_t old_pte = ptep_modify_prot_start(walk->vma, addr, pte);
97
98 ptent = pte_mkclean(old_pte);
99 ptep_modify_prot_commit(walk->vma, addr, pte, old_pte, ptent);
100
101 wpwalk->total++;
102 wpwalk->tlbflush_start = min(wpwalk->tlbflush_start, addr);
103 wpwalk->tlbflush_end = max(wpwalk->tlbflush_end,
104 addr + PAGE_SIZE);
105
106 __set_bit(pgoff, cwalk->bitmap);
107 cwalk->start = min(cwalk->start, pgoff);
108 cwalk->end = max(cwalk->end, pgoff + 1);
109 }
110
111 return 0;
112}
113
114
115
116
117
118
119
120
121
122
123static int wp_clean_pmd_entry(pmd_t *pmd, unsigned long addr, unsigned long end,
124 struct mm_walk *walk)
125{
126 pmd_t pmdval = pmd_read_atomic(pmd);
127
128 if (!pmd_trans_unstable(&pmdval))
129 return 0;
130
131 if (pmd_none(pmdval)) {
132 walk->action = ACTION_AGAIN;
133 return 0;
134 }
135
136
137 walk->action = ACTION_CONTINUE;
138 if (pmd_trans_huge(pmdval) || pmd_devmap(pmdval))
139 WARN_ON(pmd_write(pmdval) || pmd_dirty(pmdval));
140
141 return 0;
142}
143
144
145
146
147
148
149
150
151
152
153static int wp_clean_pud_entry(pud_t *pud, unsigned long addr, unsigned long end,
154 struct mm_walk *walk)
155{
156 pud_t pudval = READ_ONCE(*pud);
157
158 if (!pud_trans_unstable(&pudval))
159 return 0;
160
161 if (pud_none(pudval)) {
162 walk->action = ACTION_AGAIN;
163 return 0;
164 }
165
166
167 walk->action = ACTION_CONTINUE;
168 if (pud_trans_huge(pudval) || pud_devmap(pudval))
169 WARN_ON(pud_write(pudval) || pud_dirty(pudval));
170
171 return 0;
172}
173
174
175
176
177
178
179
180static int wp_clean_pre_vma(unsigned long start, unsigned long end,
181 struct mm_walk *walk)
182{
183 struct wp_walk *wpwalk = walk->private;
184
185 wpwalk->tlbflush_start = end;
186 wpwalk->tlbflush_end = start;
187
188 mmu_notifier_range_init(&wpwalk->range, MMU_NOTIFY_PROTECTION_PAGE, 0,
189 walk->vma, walk->mm, start, end);
190 mmu_notifier_invalidate_range_start(&wpwalk->range);
191 flush_cache_range(walk->vma, start, end);
192
193
194
195
196
197
198 inc_tlb_flush_pending(walk->mm);
199
200 return 0;
201}
202
203
204
205
206
207
208
209static void wp_clean_post_vma(struct mm_walk *walk)
210{
211 struct wp_walk *wpwalk = walk->private;
212
213 if (mm_tlb_flush_nested(walk->mm))
214 flush_tlb_range(walk->vma, wpwalk->range.start,
215 wpwalk->range.end);
216 else if (wpwalk->tlbflush_end > wpwalk->tlbflush_start)
217 flush_tlb_range(walk->vma, wpwalk->tlbflush_start,
218 wpwalk->tlbflush_end);
219
220 mmu_notifier_invalidate_range_end(&wpwalk->range);
221 dec_tlb_flush_pending(walk->mm);
222}
223
224
225
226
227
228
229static int wp_clean_test_walk(unsigned long start, unsigned long end,
230 struct mm_walk *walk)
231{
232 unsigned long vm_flags = READ_ONCE(walk->vma->vm_flags);
233
234
235 if ((vm_flags & (VM_SHARED | VM_MAYWRITE | VM_HUGETLB)) !=
236 (VM_SHARED | VM_MAYWRITE))
237 return 1;
238
239 return 0;
240}
241
242static const struct mm_walk_ops clean_walk_ops = {
243 .pte_entry = clean_record_pte,
244 .pmd_entry = wp_clean_pmd_entry,
245 .pud_entry = wp_clean_pud_entry,
246 .test_walk = wp_clean_test_walk,
247 .pre_vma = wp_clean_pre_vma,
248 .post_vma = wp_clean_post_vma
249};
250
251static const struct mm_walk_ops wp_walk_ops = {
252 .pte_entry = wp_pte,
253 .pmd_entry = wp_clean_pmd_entry,
254 .pud_entry = wp_clean_pud_entry,
255 .test_walk = wp_clean_test_walk,
256 .pre_vma = wp_clean_pre_vma,
257 .post_vma = wp_clean_post_vma
258};
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274unsigned long wp_shared_mapping_range(struct address_space *mapping,
275 pgoff_t first_index, pgoff_t nr)
276{
277 struct wp_walk wpwalk = { .total = 0 };
278
279 i_mmap_lock_read(mapping);
280 WARN_ON(walk_page_mapping(mapping, first_index, nr, &wp_walk_ops,
281 &wpwalk));
282 i_mmap_unlock_read(mapping);
283
284 return wpwalk.total;
285}
286EXPORT_SYMBOL_GPL(wp_shared_mapping_range);
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323unsigned long clean_record_shared_mapping_range(struct address_space *mapping,
324 pgoff_t first_index, pgoff_t nr,
325 pgoff_t bitmap_pgoff,
326 unsigned long *bitmap,
327 pgoff_t *start,
328 pgoff_t *end)
329{
330 bool none_set = (*start >= *end);
331 struct clean_walk cwalk = {
332 .base = { .total = 0 },
333 .bitmap_pgoff = bitmap_pgoff,
334 .bitmap = bitmap,
335 .start = none_set ? nr : *start,
336 .end = none_set ? 0 : *end,
337 };
338
339 i_mmap_lock_read(mapping);
340 WARN_ON(walk_page_mapping(mapping, first_index, nr, &clean_walk_ops,
341 &cwalk.base));
342 i_mmap_unlock_read(mapping);
343
344 *start = cwalk.start;
345 *end = cwalk.end;
346
347 return cwalk.base.total;
348}
349EXPORT_SYMBOL_GPL(clean_record_shared_mapping_range);
350