1
2
3
4
5
6
7
8
9
10
11
12#include <linux/rculist.h>
13#include <linux/mmu_notifier.h>
14#include <linux/export.h>
15#include <linux/mm.h>
16#include <linux/err.h>
17#include <linux/srcu.h>
18#include <linux/rcupdate.h>
19#include <linux/sched.h>
20#include <linux/slab.h>
21
22
23static struct srcu_struct srcu;
24
25
26
27
28
29
30void mmu_notifier_call_srcu(struct rcu_head *rcu,
31 void (*func)(struct rcu_head *rcu))
32{
33 call_srcu(&srcu, rcu, func);
34}
35EXPORT_SYMBOL_GPL(mmu_notifier_call_srcu);
36
37void mmu_notifier_synchronize(void)
38{
39
40 srcu_barrier(&srcu);
41}
42EXPORT_SYMBOL_GPL(mmu_notifier_synchronize);
43
44
45
46
47
48
49
50
51
52
53
54
55
56void __mmu_notifier_release(struct mm_struct *mm)
57{
58 struct mmu_notifier *mn;
59 int id;
60
61
62
63
64
65 id = srcu_read_lock(&srcu);
66 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)
67
68
69
70
71
72
73 if (mn->ops->release)
74 mn->ops->release(mn, mm);
75
76 spin_lock(&mm->mmu_notifier_mm->lock);
77 while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {
78 mn = hlist_entry(mm->mmu_notifier_mm->list.first,
79 struct mmu_notifier,
80 hlist);
81
82
83
84
85
86
87 hlist_del_init_rcu(&mn->hlist);
88 }
89 spin_unlock(&mm->mmu_notifier_mm->lock);
90 srcu_read_unlock(&srcu, id);
91
92
93
94
95
96
97
98
99
100
101 synchronize_srcu(&srcu);
102}
103
104
105
106
107
108
109int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
110 unsigned long start,
111 unsigned long end)
112{
113 struct mmu_notifier *mn;
114 int young = 0, id;
115
116 id = srcu_read_lock(&srcu);
117 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
118 if (mn->ops->clear_flush_young)
119 young |= mn->ops->clear_flush_young(mn, mm, start, end);
120 }
121 srcu_read_unlock(&srcu, id);
122
123 return young;
124}
125
126int __mmu_notifier_clear_young(struct mm_struct *mm,
127 unsigned long start,
128 unsigned long end)
129{
130 struct mmu_notifier *mn;
131 int young = 0, id;
132
133 id = srcu_read_lock(&srcu);
134 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
135 if (mn->ops->clear_young)
136 young |= mn->ops->clear_young(mn, mm, start, end);
137 }
138 srcu_read_unlock(&srcu, id);
139
140 return young;
141}
142
143int __mmu_notifier_test_young(struct mm_struct *mm,
144 unsigned long address)
145{
146 struct mmu_notifier *mn;
147 int young = 0, id;
148
149 id = srcu_read_lock(&srcu);
150 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
151 if (mn->ops->test_young) {
152 young = mn->ops->test_young(mn, mm, address);
153 if (young)
154 break;
155 }
156 }
157 srcu_read_unlock(&srcu, id);
158
159 return young;
160}
161
162void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
163 pte_t pte)
164{
165 struct mmu_notifier *mn;
166 int id;
167
168 id = srcu_read_lock(&srcu);
169 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
170 if (mn->ops->change_pte)
171 mn->ops->change_pte(mn, mm, address, pte);
172 }
173 srcu_read_unlock(&srcu, id);
174}
175
176void __mmu_notifier_invalidate_page(struct mm_struct *mm,
177 unsigned long address)
178{
179 struct mmu_notifier *mn;
180 int id;
181
182 id = srcu_read_lock(&srcu);
183 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
184 if (mn->ops->invalidate_page)
185 mn->ops->invalidate_page(mn, mm, address);
186 }
187 srcu_read_unlock(&srcu, id);
188}
189
190void __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
191 unsigned long start, unsigned long end)
192{
193 struct mmu_notifier *mn;
194 int id;
195
196 id = srcu_read_lock(&srcu);
197 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
198 if (mn->ops->invalidate_range_start)
199 mn->ops->invalidate_range_start(mn, mm, start, end);
200 }
201 srcu_read_unlock(&srcu, id);
202}
203EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);
204
205void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
206 unsigned long start, unsigned long end)
207{
208 struct mmu_notifier *mn;
209 int id;
210
211 id = srcu_read_lock(&srcu);
212 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
213
214
215
216
217
218
219
220
221 if (mn->ops->invalidate_range)
222 mn->ops->invalidate_range(mn, mm, start, end);
223 if (mn->ops->invalidate_range_end)
224 mn->ops->invalidate_range_end(mn, mm, start, end);
225 }
226 srcu_read_unlock(&srcu, id);
227}
228EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);
229
230void __mmu_notifier_invalidate_range(struct mm_struct *mm,
231 unsigned long start, unsigned long end)
232{
233 struct mmu_notifier *mn;
234 int id;
235
236 id = srcu_read_lock(&srcu);
237 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
238 if (mn->ops->invalidate_range)
239 mn->ops->invalidate_range(mn, mm, start, end);
240 }
241 srcu_read_unlock(&srcu, id);
242}
243EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range);
244
245static int do_mmu_notifier_register(struct mmu_notifier *mn,
246 struct mm_struct *mm,
247 int take_mmap_sem)
248{
249 struct mmu_notifier_mm *mmu_notifier_mm;
250 int ret;
251
252 BUG_ON(atomic_read(&mm->mm_users) <= 0);
253
254
255
256
257
258 BUG_ON(!srcu.per_cpu_ref);
259
260 ret = -ENOMEM;
261 mmu_notifier_mm = kmalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);
262 if (unlikely(!mmu_notifier_mm))
263 goto out;
264
265 if (take_mmap_sem)
266 down_write(&mm->mmap_sem);
267 ret = mm_take_all_locks(mm);
268 if (unlikely(ret))
269 goto out_clean;
270
271 if (!mm_has_notifiers(mm)) {
272 INIT_HLIST_HEAD(&mmu_notifier_mm->list);
273 spin_lock_init(&mmu_notifier_mm->lock);
274
275 mm->mmu_notifier_mm = mmu_notifier_mm;
276 mmu_notifier_mm = NULL;
277 }
278 atomic_inc(&mm->mm_count);
279
280
281
282
283
284
285
286
287
288 spin_lock(&mm->mmu_notifier_mm->lock);
289 hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);
290 spin_unlock(&mm->mmu_notifier_mm->lock);
291
292 mm_drop_all_locks(mm);
293out_clean:
294 if (take_mmap_sem)
295 up_write(&mm->mmap_sem);
296 kfree(mmu_notifier_mm);
297out:
298 BUG_ON(atomic_read(&mm->mm_users) <= 0);
299 return ret;
300}
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)
316{
317 return do_mmu_notifier_register(mn, mm, 1);
318}
319EXPORT_SYMBOL_GPL(mmu_notifier_register);
320
321
322
323
324
325int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)
326{
327 return do_mmu_notifier_register(mn, mm, 0);
328}
329EXPORT_SYMBOL_GPL(__mmu_notifier_register);
330
331
332void __mmu_notifier_mm_destroy(struct mm_struct *mm)
333{
334 BUG_ON(!hlist_empty(&mm->mmu_notifier_mm->list));
335 kfree(mm->mmu_notifier_mm);
336 mm->mmu_notifier_mm = LIST_POISON1;
337}
338
339
340
341
342
343
344
345
346
347
348
349void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)
350{
351 BUG_ON(atomic_read(&mm->mm_count) <= 0);
352
353 if (!hlist_unhashed(&mn->hlist)) {
354
355
356
357
358 int id;
359
360 id = srcu_read_lock(&srcu);
361
362
363
364
365 if (mn->ops->release)
366 mn->ops->release(mn, mm);
367 srcu_read_unlock(&srcu, id);
368
369 spin_lock(&mm->mmu_notifier_mm->lock);
370
371
372
373
374 hlist_del_init_rcu(&mn->hlist);
375 spin_unlock(&mm->mmu_notifier_mm->lock);
376 }
377
378
379
380
381
382 synchronize_srcu(&srcu);
383
384 BUG_ON(atomic_read(&mm->mm_count) <= 0);
385
386 mmdrop(mm);
387}
388EXPORT_SYMBOL_GPL(mmu_notifier_unregister);
389
390
391
392
393void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
394 struct mm_struct *mm)
395{
396 spin_lock(&mm->mmu_notifier_mm->lock);
397
398
399
400
401 hlist_del_init_rcu(&mn->hlist);
402 spin_unlock(&mm->mmu_notifier_mm->lock);
403
404 BUG_ON(atomic_read(&mm->mm_count) <= 0);
405 mmdrop(mm);
406}
407EXPORT_SYMBOL_GPL(mmu_notifier_unregister_no_release);
408
409static int __init mmu_notifier_init(void)
410{
411 return init_srcu_struct(&srcu);
412}
413subsys_initcall(mmu_notifier_init);
414