1#ifndef _LINUX_MMU_NOTIFIER_H
2#define _LINUX_MMU_NOTIFIER_H
3
4#include <linux/list.h>
5#include <linux/spinlock.h>
6#include <linux/mm_types.h>
7#include <linux/srcu.h>
8
9struct mmu_notifier;
10struct mmu_notifier_ops;
11
12#ifdef CONFIG_MMU_NOTIFIER
13
14
15
16
17
18
19
20struct mmu_notifier_mm {
21
22 struct hlist_head list;
23
24 spinlock_t lock;
25};
26
27struct mmu_notifier_ops {
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51 void (*release)(struct mmu_notifier *mn,
52 struct mm_struct *mm);
53
54
55
56
57
58
59
60
61
62
63 int (*clear_flush_young)(struct mmu_notifier *mn,
64 struct mm_struct *mm,
65 unsigned long start,
66 unsigned long end);
67
68
69
70
71
72
73
74 int (*test_young)(struct mmu_notifier *mn,
75 struct mm_struct *mm,
76 unsigned long address);
77
78
79
80
81
82 void (*change_pte)(struct mmu_notifier *mn,
83 struct mm_struct *mm,
84 unsigned long address,
85 pte_t pte);
86
87
88
89
90
91
92
93
94 void (*invalidate_page)(struct mmu_notifier *mn,
95 struct mm_struct *mm,
96 unsigned long address);
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141 void (*invalidate_range_start)(struct mmu_notifier *mn,
142 struct mm_struct *mm,
143 unsigned long start, unsigned long end);
144 void (*invalidate_range_end)(struct mmu_notifier *mn,
145 struct mm_struct *mm,
146 unsigned long start, unsigned long end);
147};
148
149
150
151
152
153
154
155
156
157
158
159
160struct mmu_notifier {
161 struct hlist_node hlist;
162 const struct mmu_notifier_ops *ops;
163};
164
165static inline int mm_has_notifiers(struct mm_struct *mm)
166{
167 return unlikely(mm->mmu_notifier_mm);
168}
169
170extern int mmu_notifier_register(struct mmu_notifier *mn,
171 struct mm_struct *mm);
172extern int __mmu_notifier_register(struct mmu_notifier *mn,
173 struct mm_struct *mm);
174extern void mmu_notifier_unregister(struct mmu_notifier *mn,
175 struct mm_struct *mm);
176extern void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
177 struct mm_struct *mm);
178extern void __mmu_notifier_mm_destroy(struct mm_struct *mm);
179extern void __mmu_notifier_release(struct mm_struct *mm);
180extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
181 unsigned long start,
182 unsigned long end);
183extern int __mmu_notifier_test_young(struct mm_struct *mm,
184 unsigned long address);
185extern void __mmu_notifier_change_pte(struct mm_struct *mm,
186 unsigned long address, pte_t pte);
187extern void __mmu_notifier_invalidate_page(struct mm_struct *mm,
188 unsigned long address);
189extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
190 unsigned long start, unsigned long end);
191extern void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
192 unsigned long start, unsigned long end);
193
194static inline void mmu_notifier_release(struct mm_struct *mm)
195{
196 if (mm_has_notifiers(mm))
197 __mmu_notifier_release(mm);
198}
199
200static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
201 unsigned long start,
202 unsigned long end)
203{
204 if (mm_has_notifiers(mm))
205 return __mmu_notifier_clear_flush_young(mm, start, end);
206 return 0;
207}
208
209static inline int mmu_notifier_test_young(struct mm_struct *mm,
210 unsigned long address)
211{
212 if (mm_has_notifiers(mm))
213 return __mmu_notifier_test_young(mm, address);
214 return 0;
215}
216
217static inline void mmu_notifier_change_pte(struct mm_struct *mm,
218 unsigned long address, pte_t pte)
219{
220 if (mm_has_notifiers(mm))
221 __mmu_notifier_change_pte(mm, address, pte);
222}
223
224static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
225 unsigned long address)
226{
227 if (mm_has_notifiers(mm))
228 __mmu_notifier_invalidate_page(mm, address);
229}
230
231static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
232 unsigned long start, unsigned long end)
233{
234 if (mm_has_notifiers(mm))
235 __mmu_notifier_invalidate_range_start(mm, start, end);
236}
237
238static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
239 unsigned long start, unsigned long end)
240{
241 if (mm_has_notifiers(mm))
242 __mmu_notifier_invalidate_range_end(mm, start, end);
243}
244
245static inline void mmu_notifier_mm_init(struct mm_struct *mm)
246{
247 mm->mmu_notifier_mm = NULL;
248}
249
250static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
251{
252 if (mm_has_notifiers(mm))
253 __mmu_notifier_mm_destroy(mm);
254}
255
256#define ptep_clear_flush_young_notify(__vma, __address, __ptep) \
257({ \
258 int __young; \
259 struct vm_area_struct *___vma = __vma; \
260 unsigned long ___address = __address; \
261 __young = ptep_clear_flush_young(___vma, ___address, __ptep); \
262 __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \
263 ___address, \
264 ___address + \
265 PAGE_SIZE); \
266 __young; \
267})
268
269#define pmdp_clear_flush_young_notify(__vma, __address, __pmdp) \
270({ \
271 int __young; \
272 struct vm_area_struct *___vma = __vma; \
273 unsigned long ___address = __address; \
274 __young = pmdp_clear_flush_young(___vma, ___address, __pmdp); \
275 __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \
276 ___address, \
277 ___address + \
278 PMD_SIZE); \
279 __young; \
280})
281
282
283
284
285
286
287
288
289
290
291
292#define set_pte_at_notify(__mm, __address, __ptep, __pte) \
293({ \
294 struct mm_struct *___mm = __mm; \
295 unsigned long ___address = __address; \
296 pte_t ___pte = __pte; \
297 \
298 mmu_notifier_change_pte(___mm, ___address, ___pte); \
299 set_pte_at(___mm, ___address, __ptep, ___pte); \
300})
301
302extern void mmu_notifier_call_srcu(struct rcu_head *rcu,
303 void (*func)(struct rcu_head *rcu));
304extern void mmu_notifier_synchronize(void);
305
306#else
307
308static inline void mmu_notifier_release(struct mm_struct *mm)
309{
310}
311
312static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm,
313 unsigned long start,
314 unsigned long end)
315{
316 return 0;
317}
318
319static inline int mmu_notifier_test_young(struct mm_struct *mm,
320 unsigned long address)
321{
322 return 0;
323}
324
325static inline void mmu_notifier_change_pte(struct mm_struct *mm,
326 unsigned long address, pte_t pte)
327{
328}
329
330static inline void mmu_notifier_invalidate_page(struct mm_struct *mm,
331 unsigned long address)
332{
333}
334
335static inline void mmu_notifier_invalidate_range_start(struct mm_struct *mm,
336 unsigned long start, unsigned long end)
337{
338}
339
340static inline void mmu_notifier_invalidate_range_end(struct mm_struct *mm,
341 unsigned long start, unsigned long end)
342{
343}
344
345static inline void mmu_notifier_mm_init(struct mm_struct *mm)
346{
347}
348
349static inline void mmu_notifier_mm_destroy(struct mm_struct *mm)
350{
351}
352
353#define ptep_clear_flush_young_notify ptep_clear_flush_young
354#define pmdp_clear_flush_young_notify pmdp_clear_flush_young
355#define set_pte_at_notify set_pte_at
356
357#endif
358
359#endif
360