1
2
3
4
5
6
7
8
9
10
11
12#include <linux/rculist.h>
13#include <linux/mmu_notifier.h>
14#include <linux/export.h>
15#include <linux/mm.h>
16#include <linux/err.h>
17#include <linux/srcu.h>
18#include <linux/rcupdate.h>
19#include <linux/sched.h>
20#include <linux/sched/mm.h>
21#include <linux/slab.h>
22
23
24DEFINE_STATIC_SRCU(srcu);
25
26
27
28
29
30
31void mmu_notifier_call_srcu(struct rcu_head *rcu,
32 void (*func)(struct rcu_head *rcu))
33{
34 call_srcu(&srcu, rcu, func);
35}
36EXPORT_SYMBOL_GPL(mmu_notifier_call_srcu);
37
38void mmu_notifier_synchronize(void)
39{
40
41 srcu_barrier(&srcu);
42}
43EXPORT_SYMBOL_GPL(mmu_notifier_synchronize);
44
45
46
47
48
49
50
51
52
53
54
55
56
57void __mmu_notifier_release(struct mm_struct *mm)
58{
59 struct mmu_notifier *mn;
60 int id;
61
62
63
64
65
66 id = srcu_read_lock(&srcu);
67 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist)
68
69
70
71
72
73
74 if (mn->ops->release)
75 mn->ops->release(mn, mm);
76
77 spin_lock(&mm->mmu_notifier_mm->lock);
78 while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) {
79 mn = hlist_entry(mm->mmu_notifier_mm->list.first,
80 struct mmu_notifier,
81 hlist);
82
83
84
85
86
87
88 hlist_del_init_rcu(&mn->hlist);
89 }
90 spin_unlock(&mm->mmu_notifier_mm->lock);
91 srcu_read_unlock(&srcu, id);
92
93
94
95
96
97
98
99
100
101
102 synchronize_srcu(&srcu);
103}
104
105
106
107
108
109
110int __mmu_notifier_clear_flush_young(struct mm_struct *mm,
111 unsigned long start,
112 unsigned long end)
113{
114 struct mmu_notifier *mn;
115 int young = 0, id;
116
117 id = srcu_read_lock(&srcu);
118 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
119 if (mn->ops->clear_flush_young)
120 young |= mn->ops->clear_flush_young(mn, mm, start, end);
121 }
122 srcu_read_unlock(&srcu, id);
123
124 return young;
125}
126
127int __mmu_notifier_clear_young(struct mm_struct *mm,
128 unsigned long start,
129 unsigned long end)
130{
131 struct mmu_notifier *mn;
132 int young = 0, id;
133
134 id = srcu_read_lock(&srcu);
135 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
136 if (mn->ops->clear_young)
137 young |= mn->ops->clear_young(mn, mm, start, end);
138 }
139 srcu_read_unlock(&srcu, id);
140
141 return young;
142}
143
144int __mmu_notifier_test_young(struct mm_struct *mm,
145 unsigned long address)
146{
147 struct mmu_notifier *mn;
148 int young = 0, id;
149
150 id = srcu_read_lock(&srcu);
151 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
152 if (mn->ops->test_young) {
153 young = mn->ops->test_young(mn, mm, address);
154 if (young)
155 break;
156 }
157 }
158 srcu_read_unlock(&srcu, id);
159
160 return young;
161}
162
163void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address,
164 pte_t pte)
165{
166 struct mmu_notifier *mn;
167 int id;
168
169 id = srcu_read_lock(&srcu);
170 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
171 if (mn->ops->change_pte)
172 mn->ops->change_pte(mn, mm, address, pte);
173 }
174 srcu_read_unlock(&srcu, id);
175}
176
177int __mmu_notifier_invalidate_range_start(struct mm_struct *mm,
178 unsigned long start, unsigned long end,
179 bool blockable)
180{
181 struct mmu_notifier *mn;
182 int ret = 0;
183 int id;
184
185 id = srcu_read_lock(&srcu);
186 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
187 if (mn->ops->invalidate_range_start) {
188 int _ret = mn->ops->invalidate_range_start(mn, mm, start, end, blockable);
189 if (_ret) {
190 pr_info("%pS callback failed with %d in %sblockable context.\n",
191 mn->ops->invalidate_range_start, _ret,
192 !blockable ? "non-" : "");
193 ret = _ret;
194 }
195 }
196 }
197 srcu_read_unlock(&srcu, id);
198
199 return ret;
200}
201EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_start);
202
203void __mmu_notifier_invalidate_range_end(struct mm_struct *mm,
204 unsigned long start,
205 unsigned long end,
206 bool only_end)
207{
208 struct mmu_notifier *mn;
209 int id;
210
211 id = srcu_read_lock(&srcu);
212 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
213
214
215
216
217
218
219
220
221
222
223
224
225
226 if (!only_end && mn->ops->invalidate_range)
227 mn->ops->invalidate_range(mn, mm, start, end);
228 if (mn->ops->invalidate_range_end)
229 mn->ops->invalidate_range_end(mn, mm, start, end);
230 }
231 srcu_read_unlock(&srcu, id);
232}
233EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range_end);
234
235void __mmu_notifier_invalidate_range(struct mm_struct *mm,
236 unsigned long start, unsigned long end)
237{
238 struct mmu_notifier *mn;
239 int id;
240
241 id = srcu_read_lock(&srcu);
242 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
243 if (mn->ops->invalidate_range)
244 mn->ops->invalidate_range(mn, mm, start, end);
245 }
246 srcu_read_unlock(&srcu, id);
247}
248EXPORT_SYMBOL_GPL(__mmu_notifier_invalidate_range);
249
250
251
252
253
254bool mm_has_blockable_invalidate_notifiers(struct mm_struct *mm)
255{
256 struct mmu_notifier *mn;
257 int id;
258 bool ret = false;
259
260 WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem));
261
262 if (!mm_has_notifiers(mm))
263 return ret;
264
265 id = srcu_read_lock(&srcu);
266 hlist_for_each_entry_rcu(mn, &mm->mmu_notifier_mm->list, hlist) {
267 if (!mn->ops->invalidate_range &&
268 !mn->ops->invalidate_range_start &&
269 !mn->ops->invalidate_range_end)
270 continue;
271
272 if (!(mn->ops->flags & MMU_INVALIDATE_DOES_NOT_BLOCK)) {
273 ret = true;
274 break;
275 }
276 }
277 srcu_read_unlock(&srcu, id);
278 return ret;
279}
280
281static int do_mmu_notifier_register(struct mmu_notifier *mn,
282 struct mm_struct *mm,
283 int take_mmap_sem)
284{
285 struct mmu_notifier_mm *mmu_notifier_mm;
286 int ret;
287
288 BUG_ON(atomic_read(&mm->mm_users) <= 0);
289
290 ret = -ENOMEM;
291 mmu_notifier_mm = kmalloc(sizeof(struct mmu_notifier_mm), GFP_KERNEL);
292 if (unlikely(!mmu_notifier_mm))
293 goto out;
294
295 if (take_mmap_sem)
296 down_write(&mm->mmap_sem);
297 ret = mm_take_all_locks(mm);
298 if (unlikely(ret))
299 goto out_clean;
300
301 if (!mm_has_notifiers(mm)) {
302 INIT_HLIST_HEAD(&mmu_notifier_mm->list);
303 spin_lock_init(&mmu_notifier_mm->lock);
304
305 mm->mmu_notifier_mm = mmu_notifier_mm;
306 mmu_notifier_mm = NULL;
307 }
308 mmgrab(mm);
309
310
311
312
313
314
315
316
317
318 spin_lock(&mm->mmu_notifier_mm->lock);
319 hlist_add_head(&mn->hlist, &mm->mmu_notifier_mm->list);
320 spin_unlock(&mm->mmu_notifier_mm->lock);
321
322 mm_drop_all_locks(mm);
323out_clean:
324 if (take_mmap_sem)
325 up_write(&mm->mmap_sem);
326 kfree(mmu_notifier_mm);
327out:
328 BUG_ON(atomic_read(&mm->mm_users) <= 0);
329 return ret;
330}
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345int mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)
346{
347 return do_mmu_notifier_register(mn, mm, 1);
348}
349EXPORT_SYMBOL_GPL(mmu_notifier_register);
350
351
352
353
354
355int __mmu_notifier_register(struct mmu_notifier *mn, struct mm_struct *mm)
356{
357 return do_mmu_notifier_register(mn, mm, 0);
358}
359EXPORT_SYMBOL_GPL(__mmu_notifier_register);
360
361
362void __mmu_notifier_mm_destroy(struct mm_struct *mm)
363{
364 BUG_ON(!hlist_empty(&mm->mmu_notifier_mm->list));
365 kfree(mm->mmu_notifier_mm);
366 mm->mmu_notifier_mm = LIST_POISON1;
367}
368
369
370
371
372
373
374
375
376
377
378
379void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm)
380{
381 BUG_ON(atomic_read(&mm->mm_count) <= 0);
382
383 if (!hlist_unhashed(&mn->hlist)) {
384
385
386
387
388 int id;
389
390 id = srcu_read_lock(&srcu);
391
392
393
394
395 if (mn->ops->release)
396 mn->ops->release(mn, mm);
397 srcu_read_unlock(&srcu, id);
398
399 spin_lock(&mm->mmu_notifier_mm->lock);
400
401
402
403
404 hlist_del_init_rcu(&mn->hlist);
405 spin_unlock(&mm->mmu_notifier_mm->lock);
406 }
407
408
409
410
411
412 synchronize_srcu(&srcu);
413
414 BUG_ON(atomic_read(&mm->mm_count) <= 0);
415
416 mmdrop(mm);
417}
418EXPORT_SYMBOL_GPL(mmu_notifier_unregister);
419
420
421
422
423void mmu_notifier_unregister_no_release(struct mmu_notifier *mn,
424 struct mm_struct *mm)
425{
426 spin_lock(&mm->mmu_notifier_mm->lock);
427
428
429
430
431 hlist_del_init_rcu(&mn->hlist);
432 spin_unlock(&mm->mmu_notifier_mm->lock);
433
434 BUG_ON(atomic_read(&mm->mm_count) <= 0);
435 mmdrop(mm);
436}
437EXPORT_SYMBOL_GPL(mmu_notifier_unregister_no_release);
438