1
2
3
4
5
6
7#include "sched.h"
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141#ifdef CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE
142#define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK \
143 (MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE \
144 | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)
145#else
146#define MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK 0
147#endif
148
149#ifdef CONFIG_RSEQ
150#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK \
151 (MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ \
152 | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ_BITMASK)
153#else
154#define MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ_BITMASK 0
155#endif
156
157#define MEMBARRIER_CMD_BITMASK \
158 (MEMBARRIER_CMD_GLOBAL | MEMBARRIER_CMD_GLOBAL_EXPEDITED \
159 | MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED \
160 | MEMBARRIER_CMD_PRIVATE_EXPEDITED \
161 | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED \
162 | MEMBARRIER_PRIVATE_EXPEDITED_SYNC_CORE_BITMASK)
163
164static void ipi_mb(void *info)
165{
166 smp_mb();
167}
168
169static void ipi_sync_core(void *info)
170{
171
172
173
174
175
176
177
178
179
180
181 smp_mb();
182
183 sync_core_before_usermode();
184}
185
186static void ipi_rseq(void *info)
187{
188
189
190
191
192
193
194
195 smp_mb();
196 rseq_preempt(current);
197}
198
199static void ipi_sync_rq_state(void *info)
200{
201 struct mm_struct *mm = (struct mm_struct *) info;
202
203 if (current->mm != mm)
204 return;
205 this_cpu_write(runqueues.membarrier_state,
206 atomic_read(&mm->membarrier_state));
207
208
209
210
211
212
213 smp_mb();
214}
215
216void membarrier_exec_mmap(struct mm_struct *mm)
217{
218
219
220
221
222
223 smp_mb();
224 atomic_set(&mm->membarrier_state, 0);
225
226
227
228
229 this_cpu_write(runqueues.membarrier_state, 0);
230}
231
232void membarrier_update_current_mm(struct mm_struct *next_mm)
233{
234 struct rq *rq = this_rq();
235 int membarrier_state = 0;
236
237 if (next_mm)
238 membarrier_state = atomic_read(&next_mm->membarrier_state);
239 if (READ_ONCE(rq->membarrier_state) == membarrier_state)
240 return;
241 WRITE_ONCE(rq->membarrier_state, membarrier_state);
242}
243
244static int membarrier_global_expedited(void)
245{
246 int cpu;
247 cpumask_var_t tmpmask;
248
249 if (num_online_cpus() == 1)
250 return 0;
251
252
253
254
255
256 smp_mb();
257
258 if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
259 return -ENOMEM;
260
261 cpus_read_lock();
262 rcu_read_lock();
263 for_each_online_cpu(cpu) {
264 struct task_struct *p;
265
266
267
268
269
270
271
272
273
274 if (cpu == raw_smp_processor_id())
275 continue;
276
277 if (!(READ_ONCE(cpu_rq(cpu)->membarrier_state) &
278 MEMBARRIER_STATE_GLOBAL_EXPEDITED))
279 continue;
280
281
282
283
284
285 p = rcu_dereference(cpu_rq(cpu)->curr);
286 if (!p->mm)
287 continue;
288
289 __cpumask_set_cpu(cpu, tmpmask);
290 }
291 rcu_read_unlock();
292
293 preempt_disable();
294 smp_call_function_many(tmpmask, ipi_mb, NULL, 1);
295 preempt_enable();
296
297 free_cpumask_var(tmpmask);
298 cpus_read_unlock();
299
300
301
302
303
304
305 smp_mb();
306 return 0;
307}
308
309static int membarrier_private_expedited(int flags, int cpu_id)
310{
311 cpumask_var_t tmpmask;
312 struct mm_struct *mm = current->mm;
313 smp_call_func_t ipi_func = ipi_mb;
314
315 if (flags == MEMBARRIER_FLAG_SYNC_CORE) {
316 if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
317 return -EINVAL;
318 if (!(atomic_read(&mm->membarrier_state) &
319 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY))
320 return -EPERM;
321 ipi_func = ipi_sync_core;
322 } else if (flags == MEMBARRIER_FLAG_RSEQ) {
323 if (!IS_ENABLED(CONFIG_RSEQ))
324 return -EINVAL;
325 if (!(atomic_read(&mm->membarrier_state) &
326 MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY))
327 return -EPERM;
328 ipi_func = ipi_rseq;
329 } else {
330 WARN_ON_ONCE(flags);
331 if (!(atomic_read(&mm->membarrier_state) &
332 MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY))
333 return -EPERM;
334 }
335
336 if (flags != MEMBARRIER_FLAG_SYNC_CORE &&
337 (atomic_read(&mm->mm_users) == 1 || num_online_cpus() == 1))
338 return 0;
339
340
341
342
343
344 smp_mb();
345
346 if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
347 return -ENOMEM;
348
349 cpus_read_lock();
350
351 if (cpu_id >= 0) {
352 struct task_struct *p;
353
354 if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id))
355 goto out;
356 rcu_read_lock();
357 p = rcu_dereference(cpu_rq(cpu_id)->curr);
358 if (!p || p->mm != mm) {
359 rcu_read_unlock();
360 goto out;
361 }
362 rcu_read_unlock();
363 } else {
364 int cpu;
365
366 rcu_read_lock();
367 for_each_online_cpu(cpu) {
368 struct task_struct *p;
369
370 p = rcu_dereference(cpu_rq(cpu)->curr);
371 if (p && p->mm == mm)
372 __cpumask_set_cpu(cpu, tmpmask);
373 }
374 rcu_read_unlock();
375 }
376
377 if (cpu_id >= 0) {
378
379
380
381
382 smp_call_function_single(cpu_id, ipi_func, NULL, 1);
383 } else {
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401 if (flags != MEMBARRIER_FLAG_SYNC_CORE) {
402 preempt_disable();
403 smp_call_function_many(tmpmask, ipi_func, NULL, true);
404 preempt_enable();
405 } else {
406 on_each_cpu_mask(tmpmask, ipi_func, NULL, true);
407 }
408 }
409
410out:
411 if (cpu_id < 0)
412 free_cpumask_var(tmpmask);
413 cpus_read_unlock();
414
415
416
417
418
419
420 smp_mb();
421
422 return 0;
423}
424
425static int sync_runqueues_membarrier_state(struct mm_struct *mm)
426{
427 int membarrier_state = atomic_read(&mm->membarrier_state);
428 cpumask_var_t tmpmask;
429 int cpu;
430
431 if (atomic_read(&mm->mm_users) == 1 || num_online_cpus() == 1) {
432 this_cpu_write(runqueues.membarrier_state, membarrier_state);
433
434
435
436
437
438
439
440
441 smp_mb();
442 return 0;
443 }
444
445 if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
446 return -ENOMEM;
447
448
449
450
451
452
453 synchronize_rcu();
454
455
456
457
458
459
460
461
462 cpus_read_lock();
463 rcu_read_lock();
464 for_each_online_cpu(cpu) {
465 struct rq *rq = cpu_rq(cpu);
466 struct task_struct *p;
467
468 p = rcu_dereference(rq->curr);
469 if (p && p->mm == mm)
470 __cpumask_set_cpu(cpu, tmpmask);
471 }
472 rcu_read_unlock();
473
474 on_each_cpu_mask(tmpmask, ipi_sync_rq_state, mm, true);
475
476 free_cpumask_var(tmpmask);
477 cpus_read_unlock();
478
479 return 0;
480}
481
482static int membarrier_register_global_expedited(void)
483{
484 struct task_struct *p = current;
485 struct mm_struct *mm = p->mm;
486 int ret;
487
488 if (atomic_read(&mm->membarrier_state) &
489 MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY)
490 return 0;
491 atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED, &mm->membarrier_state);
492 ret = sync_runqueues_membarrier_state(mm);
493 if (ret)
494 return ret;
495 atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY,
496 &mm->membarrier_state);
497
498 return 0;
499}
500
501static int membarrier_register_private_expedited(int flags)
502{
503 struct task_struct *p = current;
504 struct mm_struct *mm = p->mm;
505 int ready_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY,
506 set_state = MEMBARRIER_STATE_PRIVATE_EXPEDITED,
507 ret;
508
509 if (flags == MEMBARRIER_FLAG_SYNC_CORE) {
510 if (!IS_ENABLED(CONFIG_ARCH_HAS_MEMBARRIER_SYNC_CORE))
511 return -EINVAL;
512 ready_state =
513 MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY;
514 } else if (flags == MEMBARRIER_FLAG_RSEQ) {
515 if (!IS_ENABLED(CONFIG_RSEQ))
516 return -EINVAL;
517 ready_state =
518 MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ_READY;
519 } else {
520 WARN_ON_ONCE(flags);
521 }
522
523
524
525
526
527
528 if ((atomic_read(&mm->membarrier_state) & ready_state) == ready_state)
529 return 0;
530 if (flags & MEMBARRIER_FLAG_SYNC_CORE)
531 set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE;
532 if (flags & MEMBARRIER_FLAG_RSEQ)
533 set_state |= MEMBARRIER_STATE_PRIVATE_EXPEDITED_RSEQ;
534 atomic_or(set_state, &mm->membarrier_state);
535 ret = sync_runqueues_membarrier_state(mm);
536 if (ret)
537 return ret;
538 atomic_or(ready_state, &mm->membarrier_state);
539
540 return 0;
541}
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579SYSCALL_DEFINE3(membarrier, int, cmd, unsigned int, flags, int, cpu_id)
580{
581 switch (cmd) {
582 case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
583 if (unlikely(flags && flags != MEMBARRIER_CMD_FLAG_CPU))
584 return -EINVAL;
585 break;
586 default:
587 if (unlikely(flags))
588 return -EINVAL;
589 }
590
591 if (!(flags & MEMBARRIER_CMD_FLAG_CPU))
592 cpu_id = -1;
593
594 switch (cmd) {
595 case MEMBARRIER_CMD_QUERY:
596 {
597 int cmd_mask = MEMBARRIER_CMD_BITMASK;
598
599 if (tick_nohz_full_enabled())
600 cmd_mask &= ~MEMBARRIER_CMD_GLOBAL;
601 return cmd_mask;
602 }
603 case MEMBARRIER_CMD_GLOBAL:
604
605 if (tick_nohz_full_enabled())
606 return -EINVAL;
607 if (num_online_cpus() > 1)
608 synchronize_rcu();
609 return 0;
610 case MEMBARRIER_CMD_GLOBAL_EXPEDITED:
611 return membarrier_global_expedited();
612 case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED:
613 return membarrier_register_global_expedited();
614 case MEMBARRIER_CMD_PRIVATE_EXPEDITED:
615 return membarrier_private_expedited(0, cpu_id);
616 case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
617 return membarrier_register_private_expedited(0);
618 case MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE:
619 return membarrier_private_expedited(MEMBARRIER_FLAG_SYNC_CORE, cpu_id);
620 case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE:
621 return membarrier_register_private_expedited(MEMBARRIER_FLAG_SYNC_CORE);
622 case MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ:
623 return membarrier_private_expedited(MEMBARRIER_FLAG_RSEQ, cpu_id);
624 case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ:
625 return membarrier_register_private_expedited(MEMBARRIER_FLAG_RSEQ);
626 default:
627 return -EINVAL;
628 }
629}
630