1
2
3
4
5
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
13#include <linux/oom.h>
14#include <linux/rcupdate.h>
15#include <linux/export.h>
16#include <linux/bug.h>
17#include <linux/kthread.h>
18#include <linux/stop_machine.h>
19#include <linux/mutex.h>
20#include <linux/gfp.h>
21#include <linux/suspend.h>
22
23#include "smpboot.h"
24
25#ifdef CONFIG_SMP
26
27static DEFINE_MUTEX(cpu_add_remove_lock);
28
29
30
31
32
33void cpu_maps_update_begin(void)
34{
35 mutex_lock(&cpu_add_remove_lock);
36}
37
38void cpu_maps_update_done(void)
39{
40 mutex_unlock(&cpu_add_remove_lock);
41}
42
43static RAW_NOTIFIER_HEAD(cpu_chain);
44
45
46
47
48static int cpu_hotplug_disabled;
49
50#ifdef CONFIG_HOTPLUG_CPU
51
52static struct {
53 struct task_struct *active_writer;
54 struct mutex lock;
55
56
57
58
59 int refcount;
60} cpu_hotplug = {
61 .active_writer = NULL,
62 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
63 .refcount = 0,
64};
65
66void get_online_cpus(void)
67{
68 might_sleep();
69 if (cpu_hotplug.active_writer == current)
70 return;
71 mutex_lock(&cpu_hotplug.lock);
72 cpu_hotplug.refcount++;
73 mutex_unlock(&cpu_hotplug.lock);
74
75}
76EXPORT_SYMBOL_GPL(get_online_cpus);
77
78void put_online_cpus(void)
79{
80 if (cpu_hotplug.active_writer == current)
81 return;
82 mutex_lock(&cpu_hotplug.lock);
83
84 if (WARN_ON(!cpu_hotplug.refcount))
85 cpu_hotplug.refcount++;
86
87 if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
88 wake_up_process(cpu_hotplug.active_writer);
89 mutex_unlock(&cpu_hotplug.lock);
90
91}
92EXPORT_SYMBOL_GPL(put_online_cpus);
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116static void cpu_hotplug_begin(void)
117{
118 cpu_hotplug.active_writer = current;
119
120 for (;;) {
121 mutex_lock(&cpu_hotplug.lock);
122 if (likely(!cpu_hotplug.refcount))
123 break;
124 __set_current_state(TASK_UNINTERRUPTIBLE);
125 mutex_unlock(&cpu_hotplug.lock);
126 schedule();
127 }
128}
129
130static void cpu_hotplug_done(void)
131{
132 cpu_hotplug.active_writer = NULL;
133 mutex_unlock(&cpu_hotplug.lock);
134}
135
136
137
138
139
140
141
142
143void cpu_hotplug_disable(void)
144{
145 cpu_maps_update_begin();
146 cpu_hotplug_disabled = 1;
147 cpu_maps_update_done();
148}
149
150void cpu_hotplug_enable(void)
151{
152 cpu_maps_update_begin();
153 cpu_hotplug_disabled = 0;
154 cpu_maps_update_done();
155}
156
157#else
158static void cpu_hotplug_begin(void) {}
159static void cpu_hotplug_done(void) {}
160#endif
161
162
163int __ref register_cpu_notifier(struct notifier_block *nb)
164{
165 int ret;
166 cpu_maps_update_begin();
167 ret = raw_notifier_chain_register(&cpu_chain, nb);
168 cpu_maps_update_done();
169 return ret;
170}
171
172static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
173 int *nr_calls)
174{
175 int ret;
176
177 ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
178 nr_calls);
179
180 return notifier_to_errno(ret);
181}
182
183static int cpu_notify(unsigned long val, void *v)
184{
185 return __cpu_notify(val, v, -1, NULL);
186}
187
188#ifdef CONFIG_HOTPLUG_CPU
189
190static void cpu_notify_nofail(unsigned long val, void *v)
191{
192 BUG_ON(cpu_notify(val, v));
193}
194EXPORT_SYMBOL(register_cpu_notifier);
195
196void __ref unregister_cpu_notifier(struct notifier_block *nb)
197{
198 cpu_maps_update_begin();
199 raw_notifier_chain_unregister(&cpu_chain, nb);
200 cpu_maps_update_done();
201}
202EXPORT_SYMBOL(unregister_cpu_notifier);
203
204
205
206
207
208
209
210
211
212
213
214
215
216void clear_tasks_mm_cpumask(int cpu)
217{
218 struct task_struct *p;
219
220
221
222
223
224
225
226
227 WARN_ON(cpu_online(cpu));
228 rcu_read_lock();
229 for_each_process(p) {
230 struct task_struct *t;
231
232
233
234
235
236 t = find_lock_task_mm(p);
237 if (!t)
238 continue;
239 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
240 task_unlock(t);
241 }
242 rcu_read_unlock();
243}
244
245static inline void check_for_tasks(int cpu)
246{
247 struct task_struct *p;
248 cputime_t utime, stime;
249
250 write_lock_irq(&tasklist_lock);
251 for_each_process(p) {
252 task_cputime(p, &utime, &stime);
253 if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
254 (utime || stime))
255 printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
256 "(state = %ld, flags = %x)\n",
257 p->comm, task_pid_nr(p), cpu,
258 p->state, p->flags);
259 }
260 write_unlock_irq(&tasklist_lock);
261}
262
263struct take_cpu_down_param {
264 unsigned long mod;
265 void *hcpu;
266};
267
268
269static int __ref take_cpu_down(void *_param)
270{
271 struct take_cpu_down_param *param = _param;
272 int err;
273
274
275 err = __cpu_disable();
276 if (err < 0)
277 return err;
278
279 cpu_notify(CPU_DYING | param->mod, param->hcpu);
280
281 kthread_park(current);
282 return 0;
283}
284
285
286static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
287{
288 int err, nr_calls = 0;
289 void *hcpu = (void *)(long)cpu;
290 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
291 struct take_cpu_down_param tcd_param = {
292 .mod = mod,
293 .hcpu = hcpu,
294 };
295
296 if (num_online_cpus() == 1)
297 return -EBUSY;
298
299 if (!cpu_online(cpu))
300 return -EINVAL;
301
302 cpu_hotplug_begin();
303
304 err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
305 if (err) {
306 nr_calls--;
307 __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
308 printk("%s: attempt to take down CPU %u failed\n",
309 __func__, cpu);
310 goto out_release;
311 }
312 smpboot_park_threads(cpu);
313
314 err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
315 if (err) {
316
317 smpboot_unpark_threads(cpu);
318 cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
319 goto out_release;
320 }
321 BUG_ON(cpu_online(cpu));
322
323
324
325
326
327
328
329
330 while (!idle_cpu(cpu))
331 cpu_relax();
332
333
334 __cpu_die(cpu);
335
336
337 cpu_notify_nofail(CPU_DEAD | mod, hcpu);
338
339 check_for_tasks(cpu);
340
341out_release:
342 cpu_hotplug_done();
343 if (!err)
344 cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
345 return err;
346}
347
348int __ref cpu_down(unsigned int cpu)
349{
350 int err;
351
352 cpu_maps_update_begin();
353
354 if (cpu_hotplug_disabled) {
355 err = -EBUSY;
356 goto out;
357 }
358
359 err = _cpu_down(cpu, 0);
360
361out:
362 cpu_maps_update_done();
363 return err;
364}
365EXPORT_SYMBOL(cpu_down);
366#endif
367
368
369static int _cpu_up(unsigned int cpu, int tasks_frozen)
370{
371 int ret, nr_calls = 0;
372 void *hcpu = (void *)(long)cpu;
373 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
374 struct task_struct *idle;
375
376 cpu_hotplug_begin();
377
378 if (cpu_online(cpu) || !cpu_present(cpu)) {
379 ret = -EINVAL;
380 goto out;
381 }
382
383 idle = idle_thread_get(cpu);
384 if (IS_ERR(idle)) {
385 ret = PTR_ERR(idle);
386 goto out;
387 }
388
389 ret = smpboot_create_threads(cpu);
390 if (ret)
391 goto out;
392
393 ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
394 if (ret) {
395 nr_calls--;
396 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
397 __func__, cpu);
398 goto out_notify;
399 }
400
401
402 ret = __cpu_up(cpu, idle);
403 if (ret != 0)
404 goto out_notify;
405 BUG_ON(!cpu_online(cpu));
406
407
408 smpboot_unpark_threads(cpu);
409
410
411 cpu_notify(CPU_ONLINE | mod, hcpu);
412
413out_notify:
414 if (ret != 0)
415 __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
416out:
417 cpu_hotplug_done();
418
419 return ret;
420}
421
422int cpu_up(unsigned int cpu)
423{
424 int err = 0;
425
426#ifdef CONFIG_MEMORY_HOTPLUG
427 int nid;
428 pg_data_t *pgdat;
429#endif
430
431 if (!cpu_possible(cpu)) {
432 printk(KERN_ERR "can't online cpu %d because it is not "
433 "configured as may-hotadd at boot time\n", cpu);
434#if defined(CONFIG_IA64)
435 printk(KERN_ERR "please check additional_cpus= boot "
436 "parameter\n");
437#endif
438 return -EINVAL;
439 }
440
441#ifdef CONFIG_MEMORY_HOTPLUG
442 nid = cpu_to_node(cpu);
443 if (!node_online(nid)) {
444 err = mem_online_node(nid);
445 if (err)
446 return err;
447 }
448
449 pgdat = NODE_DATA(nid);
450 if (!pgdat) {
451 printk(KERN_ERR
452 "Can't online cpu %d due to NULL pgdat\n", cpu);
453 return -ENOMEM;
454 }
455
456 if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
457 mutex_lock(&zonelists_mutex);
458 build_all_zonelists(NULL, NULL);
459 mutex_unlock(&zonelists_mutex);
460 }
461#endif
462
463 cpu_maps_update_begin();
464
465 if (cpu_hotplug_disabled) {
466 err = -EBUSY;
467 goto out;
468 }
469
470 err = _cpu_up(cpu, 0);
471
472out:
473 cpu_maps_update_done();
474 return err;
475}
476EXPORT_SYMBOL_GPL(cpu_up);
477
478#ifdef CONFIG_PM_SLEEP_SMP
479static cpumask_var_t frozen_cpus;
480
481int disable_nonboot_cpus(void)
482{
483 int cpu, first_cpu, error = 0;
484
485 cpu_maps_update_begin();
486 first_cpu = cpumask_first(cpu_online_mask);
487
488
489
490
491 cpumask_clear(frozen_cpus);
492
493 printk("Disabling non-boot CPUs ...\n");
494 for_each_online_cpu(cpu) {
495 if (cpu == first_cpu)
496 continue;
497 error = _cpu_down(cpu, 1);
498 if (!error)
499 cpumask_set_cpu(cpu, frozen_cpus);
500 else {
501 printk(KERN_ERR "Error taking CPU%d down: %d\n",
502 cpu, error);
503 break;
504 }
505 }
506
507 if (!error) {
508 BUG_ON(num_online_cpus() > 1);
509
510 cpu_hotplug_disabled = 1;
511 } else {
512 printk(KERN_ERR "Non-boot CPUs are not disabled\n");
513 }
514 cpu_maps_update_done();
515 return error;
516}
517
518void __weak arch_enable_nonboot_cpus_begin(void)
519{
520}
521
522void __weak arch_enable_nonboot_cpus_end(void)
523{
524}
525
526void __ref enable_nonboot_cpus(void)
527{
528 int cpu, error;
529
530
531 cpu_maps_update_begin();
532 cpu_hotplug_disabled = 0;
533 if (cpumask_empty(frozen_cpus))
534 goto out;
535
536 printk(KERN_INFO "Enabling non-boot CPUs ...\n");
537
538 arch_enable_nonboot_cpus_begin();
539
540 for_each_cpu(cpu, frozen_cpus) {
541 error = _cpu_up(cpu, 1);
542 if (!error) {
543 printk(KERN_INFO "CPU%d is up\n", cpu);
544 continue;
545 }
546 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
547 }
548
549 arch_enable_nonboot_cpus_end();
550
551 cpumask_clear(frozen_cpus);
552out:
553 cpu_maps_update_done();
554}
555
556static int __init alloc_frozen_cpus(void)
557{
558 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
559 return -ENOMEM;
560 return 0;
561}
562core_initcall(alloc_frozen_cpus);
563
564
565
566
567
568
569
570
571
572
573
574
575static int
576cpu_hotplug_pm_callback(struct notifier_block *nb,
577 unsigned long action, void *ptr)
578{
579 switch (action) {
580
581 case PM_SUSPEND_PREPARE:
582 case PM_HIBERNATION_PREPARE:
583 cpu_hotplug_disable();
584 break;
585
586 case PM_POST_SUSPEND:
587 case PM_POST_HIBERNATION:
588 cpu_hotplug_enable();
589 break;
590
591 default:
592 return NOTIFY_DONE;
593 }
594
595 return NOTIFY_OK;
596}
597
598
599static int __init cpu_hotplug_pm_sync_init(void)
600{
601
602
603
604
605
606 pm_notifier(cpu_hotplug_pm_callback, 0);
607 return 0;
608}
609core_initcall(cpu_hotplug_pm_sync_init);
610
611#endif
612
613
614
615
616
617
618
619
620
621void notify_cpu_starting(unsigned int cpu)
622{
623 unsigned long val = CPU_STARTING;
624
625#ifdef CONFIG_PM_SLEEP_SMP
626 if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
627 val = CPU_STARTING_FROZEN;
628#endif
629 cpu_notify(val, (void *)(long)cpu);
630}
631
632#endif
633
634
635
636
637
638
639
640
641
642
643#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
644#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
645#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
646#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
647
648const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
649
650 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
651 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
652#if BITS_PER_LONG > 32
653 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
654 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
655#endif
656};
657EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
658
659const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
660EXPORT_SYMBOL(cpu_all_bits);
661
662#ifdef CONFIG_INIT_ALL_POSSIBLE
663static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
664 = CPU_BITS_ALL;
665#else
666static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
667#endif
668const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
669EXPORT_SYMBOL(cpu_possible_mask);
670
671static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
672const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
673EXPORT_SYMBOL(cpu_online_mask);
674
675static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
676const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
677EXPORT_SYMBOL(cpu_present_mask);
678
679static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
680const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
681EXPORT_SYMBOL(cpu_active_mask);
682
683void set_cpu_possible(unsigned int cpu, bool possible)
684{
685 if (possible)
686 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
687 else
688 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
689}
690
691void set_cpu_present(unsigned int cpu, bool present)
692{
693 if (present)
694 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
695 else
696 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
697}
698
699void set_cpu_online(unsigned int cpu, bool online)
700{
701 if (online)
702 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
703 else
704 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
705}
706
707void set_cpu_active(unsigned int cpu, bool active)
708{
709 if (active)
710 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
711 else
712 cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
713}
714
715void init_cpu_present(const struct cpumask *src)
716{
717 cpumask_copy(to_cpumask(cpu_present_bits), src);
718}
719
720void init_cpu_possible(const struct cpumask *src)
721{
722 cpumask_copy(to_cpumask(cpu_possible_bits), src);
723}
724
725void init_cpu_online(const struct cpumask *src)
726{
727 cpumask_copy(to_cpumask(cpu_online_bits), src);
728}
729