1
2
3
4
5
6#include <linux/proc_fs.h>
7#include <linux/smp.h>
8#include <linux/init.h>
9#include <linux/notifier.h>
10#include <linux/sched.h>
11#include <linux/unistd.h>
12#include <linux/cpu.h>
13#include <linux/oom.h>
14#include <linux/rcupdate.h>
15#include <linux/export.h>
16#include <linux/bug.h>
17#include <linux/kthread.h>
18#include <linux/stop_machine.h>
19#include <linux/mutex.h>
20#include <linux/gfp.h>
21#include <linux/suspend.h>
22
23#include "smpboot.h"
24
25#ifdef CONFIG_SMP
26
27static DEFINE_MUTEX(cpu_add_remove_lock);
28
29
30
31
32
33void cpu_maps_update_begin(void)
34{
35 mutex_lock(&cpu_add_remove_lock);
36}
37
38void cpu_maps_update_done(void)
39{
40 mutex_unlock(&cpu_add_remove_lock);
41}
42
43static RAW_NOTIFIER_HEAD(cpu_chain);
44
45
46
47
48static int cpu_hotplug_disabled;
49
50#ifdef CONFIG_HOTPLUG_CPU
51
52static struct {
53 struct task_struct *active_writer;
54 struct mutex lock;
55
56
57
58
59 int refcount;
60} cpu_hotplug = {
61 .active_writer = NULL,
62 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
63 .refcount = 0,
64};
65
66void get_online_cpus(void)
67{
68 might_sleep();
69 if (cpu_hotplug.active_writer == current)
70 return;
71 mutex_lock(&cpu_hotplug.lock);
72 cpu_hotplug.refcount++;
73 mutex_unlock(&cpu_hotplug.lock);
74
75}
76EXPORT_SYMBOL_GPL(get_online_cpus);
77
78void put_online_cpus(void)
79{
80 if (cpu_hotplug.active_writer == current)
81 return;
82 mutex_lock(&cpu_hotplug.lock);
83
84 if (WARN_ON(!cpu_hotplug.refcount))
85 cpu_hotplug.refcount++;
86
87 if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
88 wake_up_process(cpu_hotplug.active_writer);
89 mutex_unlock(&cpu_hotplug.lock);
90
91}
92EXPORT_SYMBOL_GPL(put_online_cpus);
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116static void cpu_hotplug_begin(void)
117{
118 cpu_hotplug.active_writer = current;
119
120 for (;;) {
121 mutex_lock(&cpu_hotplug.lock);
122 if (likely(!cpu_hotplug.refcount))
123 break;
124 __set_current_state(TASK_UNINTERRUPTIBLE);
125 mutex_unlock(&cpu_hotplug.lock);
126 schedule();
127 }
128}
129
130static void cpu_hotplug_done(void)
131{
132 cpu_hotplug.active_writer = NULL;
133 mutex_unlock(&cpu_hotplug.lock);
134}
135
136#else
137static void cpu_hotplug_begin(void) {}
138static void cpu_hotplug_done(void) {}
139#endif
140
141
142int __ref register_cpu_notifier(struct notifier_block *nb)
143{
144 int ret;
145 cpu_maps_update_begin();
146 ret = raw_notifier_chain_register(&cpu_chain, nb);
147 cpu_maps_update_done();
148 return ret;
149}
150
151static int __cpu_notify(unsigned long val, void *v, int nr_to_call,
152 int *nr_calls)
153{
154 int ret;
155
156 ret = __raw_notifier_call_chain(&cpu_chain, val, v, nr_to_call,
157 nr_calls);
158
159 return notifier_to_errno(ret);
160}
161
162static int cpu_notify(unsigned long val, void *v)
163{
164 return __cpu_notify(val, v, -1, NULL);
165}
166
167#ifdef CONFIG_HOTPLUG_CPU
168
169static void cpu_notify_nofail(unsigned long val, void *v)
170{
171 BUG_ON(cpu_notify(val, v));
172}
173EXPORT_SYMBOL(register_cpu_notifier);
174
175void __ref unregister_cpu_notifier(struct notifier_block *nb)
176{
177 cpu_maps_update_begin();
178 raw_notifier_chain_unregister(&cpu_chain, nb);
179 cpu_maps_update_done();
180}
181EXPORT_SYMBOL(unregister_cpu_notifier);
182
183
184
185
186
187
188
189
190
191
192
193
194
195void clear_tasks_mm_cpumask(int cpu)
196{
197 struct task_struct *p;
198
199
200
201
202
203
204
205
206 WARN_ON(cpu_online(cpu));
207 rcu_read_lock();
208 for_each_process(p) {
209 struct task_struct *t;
210
211
212
213
214
215 t = find_lock_task_mm(p);
216 if (!t)
217 continue;
218 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
219 task_unlock(t);
220 }
221 rcu_read_unlock();
222}
223
224static inline void check_for_tasks(int cpu)
225{
226 struct task_struct *p;
227 cputime_t utime, stime;
228
229 write_lock_irq(&tasklist_lock);
230 for_each_process(p) {
231 task_cputime(p, &utime, &stime);
232 if (task_cpu(p) == cpu && p->state == TASK_RUNNING &&
233 (utime || stime))
234 printk(KERN_WARNING "Task %s (pid = %d) is on cpu %d "
235 "(state = %ld, flags = %x)\n",
236 p->comm, task_pid_nr(p), cpu,
237 p->state, p->flags);
238 }
239 write_unlock_irq(&tasklist_lock);
240}
241
242struct take_cpu_down_param {
243 unsigned long mod;
244 void *hcpu;
245};
246
247
248static int __ref take_cpu_down(void *_param)
249{
250 struct take_cpu_down_param *param = _param;
251 int err;
252
253
254 err = __cpu_disable();
255 if (err < 0)
256 return err;
257
258 cpu_notify(CPU_DYING | param->mod, param->hcpu);
259
260 kthread_park(current);
261 return 0;
262}
263
264
265static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
266{
267 int err, nr_calls = 0;
268 void *hcpu = (void *)(long)cpu;
269 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
270 struct take_cpu_down_param tcd_param = {
271 .mod = mod,
272 .hcpu = hcpu,
273 };
274
275 if (num_online_cpus() == 1)
276 return -EBUSY;
277
278 if (!cpu_online(cpu))
279 return -EINVAL;
280
281 cpu_hotplug_begin();
282
283 err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls);
284 if (err) {
285 nr_calls--;
286 __cpu_notify(CPU_DOWN_FAILED | mod, hcpu, nr_calls, NULL);
287 printk("%s: attempt to take down CPU %u failed\n",
288 __func__, cpu);
289 goto out_release;
290 }
291 smpboot_park_threads(cpu);
292
293 err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
294 if (err) {
295
296 smpboot_unpark_threads(cpu);
297 cpu_notify_nofail(CPU_DOWN_FAILED | mod, hcpu);
298 goto out_release;
299 }
300 BUG_ON(cpu_online(cpu));
301
302
303
304
305
306
307
308
309 while (!idle_cpu(cpu))
310 cpu_relax();
311
312
313 __cpu_die(cpu);
314
315
316 cpu_notify_nofail(CPU_DEAD | mod, hcpu);
317
318 check_for_tasks(cpu);
319
320out_release:
321 cpu_hotplug_done();
322 if (!err)
323 cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu);
324 return err;
325}
326
327int __ref cpu_down(unsigned int cpu)
328{
329 int err;
330
331 cpu_maps_update_begin();
332
333 if (cpu_hotplug_disabled) {
334 err = -EBUSY;
335 goto out;
336 }
337
338 err = _cpu_down(cpu, 0);
339
340out:
341 cpu_maps_update_done();
342 return err;
343}
344EXPORT_SYMBOL(cpu_down);
345#endif
346
347
348static int __cpuinit _cpu_up(unsigned int cpu, int tasks_frozen)
349{
350 int ret, nr_calls = 0;
351 void *hcpu = (void *)(long)cpu;
352 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
353 struct task_struct *idle;
354
355 cpu_hotplug_begin();
356
357 if (cpu_online(cpu) || !cpu_present(cpu)) {
358 ret = -EINVAL;
359 goto out;
360 }
361
362 idle = idle_thread_get(cpu);
363 if (IS_ERR(idle)) {
364 ret = PTR_ERR(idle);
365 goto out;
366 }
367
368 ret = smpboot_create_threads(cpu);
369 if (ret)
370 goto out;
371
372 ret = __cpu_notify(CPU_UP_PREPARE | mod, hcpu, -1, &nr_calls);
373 if (ret) {
374 nr_calls--;
375 printk(KERN_WARNING "%s: attempt to bring up CPU %u failed\n",
376 __func__, cpu);
377 goto out_notify;
378 }
379
380
381 ret = __cpu_up(cpu, idle);
382 if (ret != 0)
383 goto out_notify;
384 BUG_ON(!cpu_online(cpu));
385
386
387 smpboot_unpark_threads(cpu);
388
389
390 cpu_notify(CPU_ONLINE | mod, hcpu);
391
392out_notify:
393 if (ret != 0)
394 __cpu_notify(CPU_UP_CANCELED | mod, hcpu, nr_calls, NULL);
395out:
396 cpu_hotplug_done();
397
398 return ret;
399}
400
401int __cpuinit cpu_up(unsigned int cpu)
402{
403 int err = 0;
404
405#ifdef CONFIG_MEMORY_HOTPLUG
406 int nid;
407 pg_data_t *pgdat;
408#endif
409
410 if (!cpu_possible(cpu)) {
411 printk(KERN_ERR "can't online cpu %d because it is not "
412 "configured as may-hotadd at boot time\n", cpu);
413#if defined(CONFIG_IA64)
414 printk(KERN_ERR "please check additional_cpus= boot "
415 "parameter\n");
416#endif
417 return -EINVAL;
418 }
419
420#ifdef CONFIG_MEMORY_HOTPLUG
421 nid = cpu_to_node(cpu);
422 if (!node_online(nid)) {
423 err = mem_online_node(nid);
424 if (err)
425 return err;
426 }
427
428 pgdat = NODE_DATA(nid);
429 if (!pgdat) {
430 printk(KERN_ERR
431 "Can't online cpu %d due to NULL pgdat\n", cpu);
432 return -ENOMEM;
433 }
434
435 if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
436 mutex_lock(&zonelists_mutex);
437 build_all_zonelists(NULL, NULL);
438 mutex_unlock(&zonelists_mutex);
439 }
440#endif
441
442 cpu_maps_update_begin();
443
444 if (cpu_hotplug_disabled) {
445 err = -EBUSY;
446 goto out;
447 }
448
449 err = _cpu_up(cpu, 0);
450
451out:
452 cpu_maps_update_done();
453 return err;
454}
455EXPORT_SYMBOL_GPL(cpu_up);
456
457#ifdef CONFIG_PM_SLEEP_SMP
458static cpumask_var_t frozen_cpus;
459
460int disable_nonboot_cpus(void)
461{
462 int cpu, first_cpu, error = 0;
463
464 cpu_maps_update_begin();
465 first_cpu = cpumask_first(cpu_online_mask);
466
467
468
469
470 cpumask_clear(frozen_cpus);
471
472 printk("Disabling non-boot CPUs ...\n");
473 for_each_online_cpu(cpu) {
474 if (cpu == first_cpu)
475 continue;
476 error = _cpu_down(cpu, 1);
477 if (!error)
478 cpumask_set_cpu(cpu, frozen_cpus);
479 else {
480 printk(KERN_ERR "Error taking CPU%d down: %d\n",
481 cpu, error);
482 break;
483 }
484 }
485
486 if (!error) {
487 BUG_ON(num_online_cpus() > 1);
488
489 cpu_hotplug_disabled = 1;
490 } else {
491 printk(KERN_ERR "Non-boot CPUs are not disabled\n");
492 }
493 cpu_maps_update_done();
494 return error;
495}
496
497void __weak arch_enable_nonboot_cpus_begin(void)
498{
499}
500
501void __weak arch_enable_nonboot_cpus_end(void)
502{
503}
504
505void __ref enable_nonboot_cpus(void)
506{
507 int cpu, error;
508
509
510 cpu_maps_update_begin();
511 cpu_hotplug_disabled = 0;
512 if (cpumask_empty(frozen_cpus))
513 goto out;
514
515 printk(KERN_INFO "Enabling non-boot CPUs ...\n");
516
517 arch_enable_nonboot_cpus_begin();
518
519 for_each_cpu(cpu, frozen_cpus) {
520 error = _cpu_up(cpu, 1);
521 if (!error) {
522 printk(KERN_INFO "CPU%d is up\n", cpu);
523 continue;
524 }
525 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
526 }
527
528 arch_enable_nonboot_cpus_end();
529
530 cpumask_clear(frozen_cpus);
531out:
532 cpu_maps_update_done();
533}
534
535static int __init alloc_frozen_cpus(void)
536{
537 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
538 return -ENOMEM;
539 return 0;
540}
541core_initcall(alloc_frozen_cpus);
542
543
544
545
546
547
548
549
550
551
552
553
554void cpu_hotplug_disable_before_freeze(void)
555{
556 cpu_maps_update_begin();
557 cpu_hotplug_disabled = 1;
558 cpu_maps_update_done();
559}
560
561
562
563
564
565
566void cpu_hotplug_enable_after_thaw(void)
567{
568 cpu_maps_update_begin();
569 cpu_hotplug_disabled = 0;
570 cpu_maps_update_done();
571}
572
573
574
575
576
577
578
579
580
581
582
583
584static int
585cpu_hotplug_pm_callback(struct notifier_block *nb,
586 unsigned long action, void *ptr)
587{
588 switch (action) {
589
590 case PM_SUSPEND_PREPARE:
591 case PM_HIBERNATION_PREPARE:
592 cpu_hotplug_disable_before_freeze();
593 break;
594
595 case PM_POST_SUSPEND:
596 case PM_POST_HIBERNATION:
597 cpu_hotplug_enable_after_thaw();
598 break;
599
600 default:
601 return NOTIFY_DONE;
602 }
603
604 return NOTIFY_OK;
605}
606
607
608static int __init cpu_hotplug_pm_sync_init(void)
609{
610
611
612
613
614
615 pm_notifier(cpu_hotplug_pm_callback, 0);
616 return 0;
617}
618core_initcall(cpu_hotplug_pm_sync_init);
619
620#endif
621
622
623
624
625
626
627
628
629
630void __cpuinit notify_cpu_starting(unsigned int cpu)
631{
632 unsigned long val = CPU_STARTING;
633
634#ifdef CONFIG_PM_SLEEP_SMP
635 if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
636 val = CPU_STARTING_FROZEN;
637#endif
638 cpu_notify(val, (void *)(long)cpu);
639}
640
641#endif
642
643
644
645
646
647
648
649
650
651
652#define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
653#define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
654#define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
655#define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
656
657const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
658
659 MASK_DECLARE_8(0), MASK_DECLARE_8(8),
660 MASK_DECLARE_8(16), MASK_DECLARE_8(24),
661#if BITS_PER_LONG > 32
662 MASK_DECLARE_8(32), MASK_DECLARE_8(40),
663 MASK_DECLARE_8(48), MASK_DECLARE_8(56),
664#endif
665};
666EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
667
668const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
669EXPORT_SYMBOL(cpu_all_bits);
670
671#ifdef CONFIG_INIT_ALL_POSSIBLE
672static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
673 = CPU_BITS_ALL;
674#else
675static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
676#endif
677const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
678EXPORT_SYMBOL(cpu_possible_mask);
679
680static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
681const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
682EXPORT_SYMBOL(cpu_online_mask);
683
684static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
685const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
686EXPORT_SYMBOL(cpu_present_mask);
687
688static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
689const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
690EXPORT_SYMBOL(cpu_active_mask);
691
692void set_cpu_possible(unsigned int cpu, bool possible)
693{
694 if (possible)
695 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
696 else
697 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
698}
699
700void set_cpu_present(unsigned int cpu, bool present)
701{
702 if (present)
703 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
704 else
705 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
706}
707
708void set_cpu_online(unsigned int cpu, bool online)
709{
710 if (online)
711 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
712 else
713 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
714}
715
716void set_cpu_active(unsigned int cpu, bool active)
717{
718 if (active)
719 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
720 else
721 cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
722}
723
724void init_cpu_present(const struct cpumask *src)
725{
726 cpumask_copy(to_cpumask(cpu_present_bits), src);
727}
728
729void init_cpu_possible(const struct cpumask *src)
730{
731 cpumask_copy(to_cpumask(cpu_possible_bits), src);
732}
733
734void init_cpu_online(const struct cpumask *src)
735{
736 cpumask_copy(to_cpumask(cpu_online_bits), src);
737}
738