1
2
3
4
5
6
7
8
9
10
11#include <linux/completion.h>
12#include <linux/cpu.h>
13#include <linux/init.h>
14#include <linux/kthread.h>
15#include <linux/export.h>
16#include <linux/percpu.h>
17#include <linux/sched.h>
18#include <linux/stop_machine.h>
19#include <linux/interrupt.h>
20#include <linux/kallsyms.h>
21#include <linux/smpboot.h>
22#include <linux/atomic.h>
23#include <linux/nmi.h>
24#include <linux/sched/wake_q.h>
25
26
27
28
29
30struct cpu_stop_done {
31 atomic_t nr_todo;
32 int ret;
33 struct completion completion;
34};
35
36
37struct cpu_stopper {
38 struct task_struct *thread;
39
40 spinlock_t lock;
41 bool enabled;
42 struct list_head works;
43
44 struct cpu_stop_work stop_work;
45};
46
47static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
48static bool stop_machine_initialized = false;
49
50
51static DEFINE_MUTEX(stop_cpus_mutex);
52static bool stop_cpus_in_progress;
53
54static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
55{
56 memset(done, 0, sizeof(*done));
57 atomic_set(&done->nr_todo, nr_todo);
58 init_completion(&done->completion);
59}
60
61
62static void cpu_stop_signal_done(struct cpu_stop_done *done)
63{
64 if (atomic_dec_and_test(&done->nr_todo))
65 complete(&done->completion);
66}
67
68static void __cpu_stop_queue_work(struct cpu_stopper *stopper,
69 struct cpu_stop_work *work,
70 struct wake_q_head *wakeq)
71{
72 list_add_tail(&work->list, &stopper->works);
73 wake_q_add(wakeq, stopper->thread);
74}
75
76
77static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work)
78{
79 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
80 DEFINE_WAKE_Q(wakeq);
81 unsigned long flags;
82 bool enabled;
83
84 spin_lock_irqsave(&stopper->lock, flags);
85 enabled = stopper->enabled;
86 if (enabled)
87 __cpu_stop_queue_work(stopper, work, &wakeq);
88 else if (work->done)
89 cpu_stop_signal_done(work->done);
90 spin_unlock_irqrestore(&stopper->lock, flags);
91
92 wake_up_q(&wakeq);
93
94 return enabled;
95}
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
122{
123 struct cpu_stop_done done;
124 struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
125
126 cpu_stop_init_done(&done, 1);
127 if (!cpu_stop_queue_work(cpu, &work))
128 return -ENOENT;
129
130
131
132
133 cond_resched();
134 wait_for_completion(&done.completion);
135 return done.ret;
136}
137
138
139enum multi_stop_state {
140
141 MULTI_STOP_NONE,
142
143 MULTI_STOP_PREPARE,
144
145 MULTI_STOP_DISABLE_IRQ,
146
147 MULTI_STOP_RUN,
148
149 MULTI_STOP_EXIT,
150};
151
152struct multi_stop_data {
153 cpu_stop_fn_t fn;
154 void *data;
155
156 unsigned int num_threads;
157 const struct cpumask *active_cpus;
158
159 enum multi_stop_state state;
160 atomic_t thread_ack;
161};
162
163static void set_state(struct multi_stop_data *msdata,
164 enum multi_stop_state newstate)
165{
166
167 atomic_set(&msdata->thread_ack, msdata->num_threads);
168 smp_wmb();
169 msdata->state = newstate;
170}
171
172
173static void ack_state(struct multi_stop_data *msdata)
174{
175 if (atomic_dec_and_test(&msdata->thread_ack))
176 set_state(msdata, msdata->state + 1);
177}
178
179
180static int multi_cpu_stop(void *data)
181{
182 struct multi_stop_data *msdata = data;
183 enum multi_stop_state curstate = MULTI_STOP_NONE;
184 int cpu = smp_processor_id(), err = 0;
185 unsigned long flags;
186 bool is_active;
187
188
189
190
191
192 local_save_flags(flags);
193
194 if (!msdata->active_cpus)
195 is_active = cpu == cpumask_first(cpu_online_mask);
196 else
197 is_active = cpumask_test_cpu(cpu, msdata->active_cpus);
198
199
200 do {
201
202 cpu_relax_yield();
203 if (msdata->state != curstate) {
204 curstate = msdata->state;
205 switch (curstate) {
206 case MULTI_STOP_DISABLE_IRQ:
207 local_irq_disable();
208 hard_irq_disable();
209 break;
210 case MULTI_STOP_RUN:
211 if (is_active)
212 err = msdata->fn(msdata->data);
213 break;
214 default:
215 break;
216 }
217 ack_state(msdata);
218 } else if (curstate > MULTI_STOP_PREPARE) {
219
220
221
222
223
224 touch_nmi_watchdog();
225 }
226 } while (curstate != MULTI_STOP_EXIT);
227
228 local_irq_restore(flags);
229 return err;
230}
231
232static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1,
233 int cpu2, struct cpu_stop_work *work2)
234{
235 struct cpu_stopper *stopper1 = per_cpu_ptr(&cpu_stopper, cpu1);
236 struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2);
237 DEFINE_WAKE_Q(wakeq);
238 int err;
239retry:
240 spin_lock_irq(&stopper1->lock);
241 spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
242
243 err = -ENOENT;
244 if (!stopper1->enabled || !stopper2->enabled)
245 goto unlock;
246
247
248
249
250
251
252
253
254
255
256 err = -EDEADLK;
257 if (unlikely(stop_cpus_in_progress))
258 goto unlock;
259
260 err = 0;
261 __cpu_stop_queue_work(stopper1, work1, &wakeq);
262 __cpu_stop_queue_work(stopper2, work2, &wakeq);
263unlock:
264 spin_unlock(&stopper2->lock);
265 spin_unlock_irq(&stopper1->lock);
266
267 if (unlikely(err == -EDEADLK)) {
268 while (stop_cpus_in_progress)
269 cpu_relax();
270 goto retry;
271 }
272
273 wake_up_q(&wakeq);
274
275 return err;
276}
277
278
279
280
281
282
283
284
285
286
287
288int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg)
289{
290 struct cpu_stop_done done;
291 struct cpu_stop_work work1, work2;
292 struct multi_stop_data msdata;
293
294 msdata = (struct multi_stop_data){
295 .fn = fn,
296 .data = arg,
297 .num_threads = 2,
298 .active_cpus = cpumask_of(cpu1),
299 };
300
301 work1 = work2 = (struct cpu_stop_work){
302 .fn = multi_cpu_stop,
303 .arg = &msdata,
304 .done = &done
305 };
306
307 cpu_stop_init_done(&done, 2);
308 set_state(&msdata, MULTI_STOP_PREPARE);
309
310 if (cpu1 > cpu2)
311 swap(cpu1, cpu2);
312 if (cpu_stop_queue_two_works(cpu1, &work1, cpu2, &work2))
313 return -ENOENT;
314
315 wait_for_completion(&done.completion);
316 return done.ret;
317}
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
338 struct cpu_stop_work *work_buf)
339{
340 *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
341 return cpu_stop_queue_work(cpu, work_buf);
342}
343
344static bool queue_stop_cpus_work(const struct cpumask *cpumask,
345 cpu_stop_fn_t fn, void *arg,
346 struct cpu_stop_done *done)
347{
348 struct cpu_stop_work *work;
349 unsigned int cpu;
350 bool queued = false;
351
352
353
354
355
356
357 preempt_disable();
358 stop_cpus_in_progress = true;
359 for_each_cpu(cpu, cpumask) {
360 work = &per_cpu(cpu_stopper.stop_work, cpu);
361 work->fn = fn;
362 work->arg = arg;
363 work->done = done;
364 if (cpu_stop_queue_work(cpu, work))
365 queued = true;
366 }
367 stop_cpus_in_progress = false;
368 preempt_enable();
369
370 return queued;
371}
372
373static int __stop_cpus(const struct cpumask *cpumask,
374 cpu_stop_fn_t fn, void *arg)
375{
376 struct cpu_stop_done done;
377
378 cpu_stop_init_done(&done, cpumask_weight(cpumask));
379 if (!queue_stop_cpus_work(cpumask, fn, arg, &done))
380 return -ENOENT;
381 wait_for_completion(&done.completion);
382 return done.ret;
383}
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
414{
415 int ret;
416
417
418 mutex_lock(&stop_cpus_mutex);
419 ret = __stop_cpus(cpumask, fn, arg);
420 mutex_unlock(&stop_cpus_mutex);
421 return ret;
422}
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442int try_stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
443{
444 int ret;
445
446
447 if (!mutex_trylock(&stop_cpus_mutex))
448 return -EAGAIN;
449 ret = __stop_cpus(cpumask, fn, arg);
450 mutex_unlock(&stop_cpus_mutex);
451 return ret;
452}
453
454static int cpu_stop_should_run(unsigned int cpu)
455{
456 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
457 unsigned long flags;
458 int run;
459
460 spin_lock_irqsave(&stopper->lock, flags);
461 run = !list_empty(&stopper->works);
462 spin_unlock_irqrestore(&stopper->lock, flags);
463 return run;
464}
465
466static void cpu_stopper_thread(unsigned int cpu)
467{
468 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
469 struct cpu_stop_work *work;
470
471repeat:
472 work = NULL;
473 spin_lock_irq(&stopper->lock);
474 if (!list_empty(&stopper->works)) {
475 work = list_first_entry(&stopper->works,
476 struct cpu_stop_work, list);
477 list_del_init(&work->list);
478 }
479 spin_unlock_irq(&stopper->lock);
480
481 if (work) {
482 cpu_stop_fn_t fn = work->fn;
483 void *arg = work->arg;
484 struct cpu_stop_done *done = work->done;
485 int ret;
486
487
488 preempt_count_inc();
489 ret = fn(arg);
490 if (done) {
491 if (ret)
492 done->ret = ret;
493 cpu_stop_signal_done(done);
494 }
495 preempt_count_dec();
496 WARN_ONCE(preempt_count(),
497 "cpu_stop: %pf(%p) leaked preempt count\n", fn, arg);
498 goto repeat;
499 }
500}
501
502void stop_machine_park(int cpu)
503{
504 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
505
506
507
508
509
510 stopper->enabled = false;
511 kthread_park(stopper->thread);
512}
513
514extern void sched_set_stop_task(int cpu, struct task_struct *stop);
515
516static void cpu_stop_create(unsigned int cpu)
517{
518 sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));
519}
520
521static void cpu_stop_park(unsigned int cpu)
522{
523 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
524
525 WARN_ON(!list_empty(&stopper->works));
526}
527
528void stop_machine_unpark(int cpu)
529{
530 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
531
532 stopper->enabled = true;
533 kthread_unpark(stopper->thread);
534}
535
536static struct smp_hotplug_thread cpu_stop_threads = {
537 .store = &cpu_stopper.thread,
538 .thread_should_run = cpu_stop_should_run,
539 .thread_fn = cpu_stopper_thread,
540 .thread_comm = "migration/%u",
541 .create = cpu_stop_create,
542 .park = cpu_stop_park,
543 .selfparking = true,
544};
545
546static int __init cpu_stop_init(void)
547{
548 unsigned int cpu;
549
550 for_each_possible_cpu(cpu) {
551 struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
552
553 spin_lock_init(&stopper->lock);
554 INIT_LIST_HEAD(&stopper->works);
555 }
556
557 BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
558 stop_machine_unpark(raw_smp_processor_id());
559 stop_machine_initialized = true;
560 return 0;
561}
562early_initcall(cpu_stop_init);
563
564int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
565 const struct cpumask *cpus)
566{
567 struct multi_stop_data msdata = {
568 .fn = fn,
569 .data = data,
570 .num_threads = num_online_cpus(),
571 .active_cpus = cpus,
572 };
573
574 lockdep_assert_cpus_held();
575
576 if (!stop_machine_initialized) {
577
578
579
580
581
582 unsigned long flags;
583 int ret;
584
585 WARN_ON_ONCE(msdata.num_threads != 1);
586
587 local_irq_save(flags);
588 hard_irq_disable();
589 ret = (*fn)(data);
590 local_irq_restore(flags);
591
592 return ret;
593 }
594
595
596 set_state(&msdata, MULTI_STOP_PREPARE);
597 return stop_cpus(cpu_online_mask, multi_cpu_stop, &msdata);
598}
599
600int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
601{
602 int ret;
603
604
605 cpus_read_lock();
606 ret = stop_machine_cpuslocked(fn, data, cpus);
607 cpus_read_unlock();
608 return ret;
609}
610EXPORT_SYMBOL_GPL(stop_machine);
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634int stop_machine_from_inactive_cpu(cpu_stop_fn_t fn, void *data,
635 const struct cpumask *cpus)
636{
637 struct multi_stop_data msdata = { .fn = fn, .data = data,
638 .active_cpus = cpus };
639 struct cpu_stop_done done;
640 int ret;
641
642
643 BUG_ON(cpu_active(raw_smp_processor_id()));
644 msdata.num_threads = num_active_cpus() + 1;
645
646
647 while (!mutex_trylock(&stop_cpus_mutex))
648 cpu_relax();
649
650
651 set_state(&msdata, MULTI_STOP_PREPARE);
652 cpu_stop_init_done(&done, num_active_cpus());
653 queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
654 &done);
655 ret = multi_cpu_stop(&msdata);
656
657
658 while (!completion_done(&done.completion))
659 cpu_relax();
660
661 mutex_unlock(&stop_cpus_mutex);
662 return ret ?: done.ret;
663}
664