1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42#include <linux/init.h>
43#include <linux/smp.h>
44#include <linux/module.h>
45#include <linux/sched.h>
46#include <linux/percpu.h>
47#include <linux/bootmem.h>
48#include <linux/err.h>
49#include <linux/nmi.h>
50#include <linux/tboot.h>
51#include <linux/stackprotector.h>
52#include <linux/gfp.h>
53
54#include <asm/acpi.h>
55#include <asm/desc.h>
56#include <asm/nmi.h>
57#include <asm/irq.h>
58#include <asm/idle.h>
59#include <asm/trampoline.h>
60#include <asm/cpu.h>
61#include <asm/numa.h>
62#include <asm/pgtable.h>
63#include <asm/tlbflush.h>
64#include <asm/mtrr.h>
65#include <asm/mwait.h>
66#include <asm/apic.h>
67#include <asm/io_apic.h>
68#include <asm/setup.h>
69#include <asm/uv/uv.h>
70#include <linux/mc146818rtc.h>
71
72#include <asm/smpboot_hooks.h>
73#include <asm/i8259.h>
74
75
76DEFINE_PER_CPU(int, cpu_state) = { 0 };
77
78
79
80
81
82#ifdef CONFIG_HOTPLUG_CPU
83
84
85
86
87static DEFINE_PER_CPU(struct task_struct *, idle_thread_array);
88#define get_idle_for_cpu(x) (per_cpu(idle_thread_array, x))
89#define set_idle_for_cpu(x, p) (per_cpu(idle_thread_array, x) = (p))
90
91
92
93
94
95static DEFINE_MUTEX(x86_cpu_hotplug_driver_mutex);
96
97void cpu_hotplug_driver_lock(void)
98{
99 mutex_lock(&x86_cpu_hotplug_driver_mutex);
100}
101
102void cpu_hotplug_driver_unlock(void)
103{
104 mutex_unlock(&x86_cpu_hotplug_driver_mutex);
105}
106
107ssize_t arch_cpu_probe(const char *buf, size_t count) { return -1; }
108ssize_t arch_cpu_release(const char *buf, size_t count) { return -1; }
109#else
110static struct task_struct *idle_thread_array[NR_CPUS] __cpuinitdata ;
111#define get_idle_for_cpu(x) (idle_thread_array[(x)])
112#define set_idle_for_cpu(x, p) (idle_thread_array[(x)] = (p))
113#endif
114
115
116int smp_num_siblings = 1;
117EXPORT_SYMBOL(smp_num_siblings);
118
119
120DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
121
122
123DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
124EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
125
126
127DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
128EXPORT_PER_CPU_SYMBOL(cpu_core_map);
129
130DEFINE_PER_CPU(cpumask_var_t, cpu_llc_shared_map);
131
132
133DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
134EXPORT_PER_CPU_SYMBOL(cpu_info);
135
136atomic_t init_deasserted;
137
138
139
140
141
142static void __cpuinit smp_callin(void)
143{
144 int cpuid, phys_id;
145 unsigned long timeout;
146
147
148
149
150
151
152
153 if (apic->wait_for_init_deassert)
154 apic->wait_for_init_deassert(&init_deasserted);
155
156
157
158
159 phys_id = read_apic_id();
160 cpuid = smp_processor_id();
161 if (cpumask_test_cpu(cpuid, cpu_callin_mask)) {
162 panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__,
163 phys_id, cpuid);
164 }
165 pr_debug("CPU#%d (phys ID: %d) waiting for CALLOUT\n", cpuid, phys_id);
166
167
168
169
170
171
172
173
174
175
176
177
178 timeout = jiffies + 2*HZ;
179 while (time_before(jiffies, timeout)) {
180
181
182
183 if (cpumask_test_cpu(cpuid, cpu_callout_mask))
184 break;
185 cpu_relax();
186 }
187
188 if (!time_before(jiffies, timeout)) {
189 panic("%s: CPU%d started up but did not get a callout!\n",
190 __func__, cpuid);
191 }
192
193
194
195
196
197
198
199
200 pr_debug("CALLIN, before setup_local_APIC().\n");
201 if (apic->smp_callin_clear_local_apic)
202 apic->smp_callin_clear_local_apic();
203 setup_local_APIC();
204 end_local_APIC_setup();
205
206
207
208
209 setup_vector_irq(smp_processor_id());
210
211
212
213
214
215
216 local_irq_enable();
217 calibrate_delay();
218 local_irq_disable();
219 pr_debug("Stack at about %p\n", &cpuid);
220
221
222
223
224 smp_store_cpu_info(cpuid);
225
226
227
228
229
230 set_cpu_sibling_map(raw_smp_processor_id());
231 wmb();
232
233 notify_cpu_starting(cpuid);
234
235
236
237
238 cpumask_set_cpu(cpuid, cpu_callin_mask);
239}
240
241
242
243
244notrace static void __cpuinit start_secondary(void *unused)
245{
246
247
248
249
250
251 cpu_init();
252 preempt_disable();
253 smp_callin();
254
255#ifdef CONFIG_X86_32
256
257 load_cr3(swapper_pg_dir);
258 __flush_tlb_all();
259#endif
260
261
262 barrier();
263
264
265
266 check_tsc_sync_target();
267
268
269
270
271
272
273
274
275
276
277
278
279
280 ipi_call_lock();
281 lock_vector_lock();
282 set_cpu_online(smp_processor_id(), true);
283 unlock_vector_lock();
284 ipi_call_unlock();
285 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
286 x86_platform.nmi_init();
287
288
289 local_irq_enable();
290
291
292 boot_init_stack_canary();
293
294 x86_cpuinit.setup_percpu_clockev();
295
296 wmb();
297 cpu_idle();
298}
299
300
301
302
303
304
305void __cpuinit smp_store_cpu_info(int id)
306{
307 struct cpuinfo_x86 *c = &cpu_data(id);
308
309 *c = boot_cpu_data;
310 c->cpu_index = id;
311 if (id != 0)
312 identify_secondary_cpu(c);
313}
314
315static void __cpuinit link_thread_siblings(int cpu1, int cpu2)
316{
317 cpumask_set_cpu(cpu1, cpu_sibling_mask(cpu2));
318 cpumask_set_cpu(cpu2, cpu_sibling_mask(cpu1));
319 cpumask_set_cpu(cpu1, cpu_core_mask(cpu2));
320 cpumask_set_cpu(cpu2, cpu_core_mask(cpu1));
321 cpumask_set_cpu(cpu1, cpu_llc_shared_mask(cpu2));
322 cpumask_set_cpu(cpu2, cpu_llc_shared_mask(cpu1));
323}
324
325
326void __cpuinit set_cpu_sibling_map(int cpu)
327{
328 int i;
329 struct cpuinfo_x86 *c = &cpu_data(cpu);
330
331 cpumask_set_cpu(cpu, cpu_sibling_setup_mask);
332
333 if (smp_num_siblings > 1) {
334 for_each_cpu(i, cpu_sibling_setup_mask) {
335 struct cpuinfo_x86 *o = &cpu_data(i);
336
337 if (cpu_has(c, X86_FEATURE_TOPOEXT)) {
338 if (c->phys_proc_id == o->phys_proc_id &&
339 per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i) &&
340 c->compute_unit_id == o->compute_unit_id)
341 link_thread_siblings(cpu, i);
342 } else if (c->phys_proc_id == o->phys_proc_id &&
343 c->cpu_core_id == o->cpu_core_id) {
344 link_thread_siblings(cpu, i);
345 }
346 }
347 } else {
348 cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
349 }
350
351 cpumask_set_cpu(cpu, cpu_llc_shared_mask(cpu));
352
353 if (__this_cpu_read(cpu_info.x86_max_cores) == 1) {
354 cpumask_copy(cpu_core_mask(cpu), cpu_sibling_mask(cpu));
355 c->booted_cores = 1;
356 return;
357 }
358
359 for_each_cpu(i, cpu_sibling_setup_mask) {
360 if (per_cpu(cpu_llc_id, cpu) != BAD_APICID &&
361 per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) {
362 cpumask_set_cpu(i, cpu_llc_shared_mask(cpu));
363 cpumask_set_cpu(cpu, cpu_llc_shared_mask(i));
364 }
365 if (c->phys_proc_id == cpu_data(i).phys_proc_id) {
366 cpumask_set_cpu(i, cpu_core_mask(cpu));
367 cpumask_set_cpu(cpu, cpu_core_mask(i));
368
369
370
371 if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) {
372
373
374
375
376 if (cpumask_first(cpu_sibling_mask(i)) == i)
377 c->booted_cores++;
378
379
380
381
382 if (i != cpu)
383 cpu_data(i).booted_cores++;
384 } else if (i != cpu && !c->booted_cores)
385 c->booted_cores = cpu_data(i).booted_cores;
386 }
387 }
388}
389
390
391const struct cpumask *cpu_coregroup_mask(int cpu)
392{
393 struct cpuinfo_x86 *c = &cpu_data(cpu);
394
395
396
397
398 if ((sched_mc_power_savings || sched_smt_power_savings) &&
399 !(cpu_has(c, X86_FEATURE_AMD_DCM)))
400 return cpu_core_mask(cpu);
401 else
402 return cpu_llc_shared_mask(cpu);
403}
404
405static void impress_friends(void)
406{
407 int cpu;
408 unsigned long bogosum = 0;
409
410
411
412 pr_debug("Before bogomips.\n");
413 for_each_possible_cpu(cpu)
414 if (cpumask_test_cpu(cpu, cpu_callout_mask))
415 bogosum += cpu_data(cpu).loops_per_jiffy;
416 printk(KERN_INFO
417 "Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
418 num_online_cpus(),
419 bogosum/(500000/HZ),
420 (bogosum/(5000/HZ))%100);
421
422 pr_debug("Before bogocount - setting activated=1.\n");
423}
424
425void __inquire_remote_apic(int apicid)
426{
427 unsigned i, regs[] = { APIC_ID >> 4, APIC_LVR >> 4, APIC_SPIV >> 4 };
428 char *names[] = { "ID", "VERSION", "SPIV" };
429 int timeout;
430 u32 status;
431
432 printk(KERN_INFO "Inquiring remote APIC 0x%x...\n", apicid);
433
434 for (i = 0; i < ARRAY_SIZE(regs); i++) {
435 printk(KERN_INFO "... APIC 0x%x %s: ", apicid, names[i]);
436
437
438
439
440 status = safe_apic_wait_icr_idle();
441 if (status)
442 printk(KERN_CONT
443 "a previous APIC delivery may have failed\n");
444
445 apic_icr_write(APIC_DM_REMRD | regs[i], apicid);
446
447 timeout = 0;
448 do {
449 udelay(100);
450 status = apic_read(APIC_ICR) & APIC_ICR_RR_MASK;
451 } while (status == APIC_ICR_RR_INPROG && timeout++ < 1000);
452
453 switch (status) {
454 case APIC_ICR_RR_VALID:
455 status = apic_read(APIC_RRR);
456 printk(KERN_CONT "%08x\n", status);
457 break;
458 default:
459 printk(KERN_CONT "failed\n");
460 }
461 }
462}
463
464
465
466
467
468
469int __cpuinit
470wakeup_secondary_cpu_via_nmi(int logical_apicid, unsigned long start_eip)
471{
472 unsigned long send_status, accept_status = 0;
473 int maxlvt;
474
475
476
477
478 apic_icr_write(APIC_DM_NMI | apic->dest_logical, logical_apicid);
479
480 pr_debug("Waiting for send to finish...\n");
481 send_status = safe_apic_wait_icr_idle();
482
483
484
485
486 udelay(200);
487 if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
488 maxlvt = lapic_get_maxlvt();
489 if (maxlvt > 3)
490 apic_write(APIC_ESR, 0);
491 accept_status = (apic_read(APIC_ESR) & 0xEF);
492 }
493 pr_debug("NMI sent.\n");
494
495 if (send_status)
496 printk(KERN_ERR "APIC never delivered???\n");
497 if (accept_status)
498 printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
499
500 return (send_status | accept_status);
501}
502
503static int __cpuinit
504wakeup_secondary_cpu_via_init(int phys_apicid, unsigned long start_eip)
505{
506 unsigned long send_status, accept_status = 0;
507 int maxlvt, num_starts, j;
508
509 maxlvt = lapic_get_maxlvt();
510
511
512
513
514 if (APIC_INTEGRATED(apic_version[phys_apicid])) {
515 if (maxlvt > 3)
516 apic_write(APIC_ESR, 0);
517 apic_read(APIC_ESR);
518 }
519
520 pr_debug("Asserting INIT.\n");
521
522
523
524
525
526
527
528 apic_icr_write(APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT,
529 phys_apicid);
530
531 pr_debug("Waiting for send to finish...\n");
532 send_status = safe_apic_wait_icr_idle();
533
534 mdelay(10);
535
536 pr_debug("Deasserting INIT.\n");
537
538
539
540 apic_icr_write(APIC_INT_LEVELTRIG | APIC_DM_INIT, phys_apicid);
541
542 pr_debug("Waiting for send to finish...\n");
543 send_status = safe_apic_wait_icr_idle();
544
545 mb();
546 atomic_set(&init_deasserted, 1);
547
548
549
550
551
552
553
554 if (APIC_INTEGRATED(apic_version[phys_apicid]))
555 num_starts = 2;
556 else
557 num_starts = 0;
558
559
560
561
562
563 startup_ipi_hook(phys_apicid, (unsigned long) start_secondary,
564 stack_start);
565
566
567
568
569 pr_debug("#startup loops: %d.\n", num_starts);
570
571 for (j = 1; j <= num_starts; j++) {
572 pr_debug("Sending STARTUP #%d.\n", j);
573 if (maxlvt > 3)
574 apic_write(APIC_ESR, 0);
575 apic_read(APIC_ESR);
576 pr_debug("After apic_write.\n");
577
578
579
580
581
582
583
584
585 apic_icr_write(APIC_DM_STARTUP | (start_eip >> 12),
586 phys_apicid);
587
588
589
590
591 udelay(300);
592
593 pr_debug("Startup point 1.\n");
594
595 pr_debug("Waiting for send to finish...\n");
596 send_status = safe_apic_wait_icr_idle();
597
598
599
600
601 udelay(200);
602 if (maxlvt > 3)
603 apic_write(APIC_ESR, 0);
604 accept_status = (apic_read(APIC_ESR) & 0xEF);
605 if (send_status || accept_status)
606 break;
607 }
608 pr_debug("After Startup.\n");
609
610 if (send_status)
611 printk(KERN_ERR "APIC never delivered???\n");
612 if (accept_status)
613 printk(KERN_ERR "APIC delivery error (%lx).\n", accept_status);
614
615 return (send_status | accept_status);
616}
617
618struct create_idle {
619 struct work_struct work;
620 struct task_struct *idle;
621 struct completion done;
622 int cpu;
623};
624
625static void __cpuinit do_fork_idle(struct work_struct *work)
626{
627 struct create_idle *c_idle =
628 container_of(work, struct create_idle, work);
629
630 c_idle->idle = fork_idle(c_idle->cpu);
631 complete(&c_idle->done);
632}
633
634
635static void __cpuinit announce_cpu(int cpu, int apicid)
636{
637 static int current_node = -1;
638 int node = early_cpu_to_node(cpu);
639
640 if (system_state == SYSTEM_BOOTING) {
641 if (node != current_node) {
642 if (current_node > (-1))
643 pr_cont(" Ok.\n");
644 current_node = node;
645 pr_info("Booting Node %3d, Processors ", node);
646 }
647 pr_cont(" #%d%s", cpu, cpu == (nr_cpu_ids - 1) ? " Ok.\n" : "");
648 return;
649 } else
650 pr_info("Booting Node %d Processor %d APIC 0x%x\n",
651 node, cpu, apicid);
652}
653
654
655
656
657
658
659
660static int __cpuinit do_boot_cpu(int apicid, int cpu)
661{
662 unsigned long boot_error = 0;
663 unsigned long start_ip;
664 int timeout;
665 struct create_idle c_idle = {
666 .cpu = cpu,
667 .done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done),
668 };
669
670 INIT_WORK_ONSTACK(&c_idle.work, do_fork_idle);
671
672 alternatives_smp_switch(1);
673
674 c_idle.idle = get_idle_for_cpu(cpu);
675
676
677
678
679
680 if (c_idle.idle) {
681 c_idle.idle->thread.sp = (unsigned long) (((struct pt_regs *)
682 (THREAD_SIZE + task_stack_page(c_idle.idle))) - 1);
683 init_idle(c_idle.idle, cpu);
684 goto do_rest;
685 }
686
687 schedule_work(&c_idle.work);
688 wait_for_completion(&c_idle.done);
689
690 if (IS_ERR(c_idle.idle)) {
691 printk("failed fork for CPU %d\n", cpu);
692 destroy_work_on_stack(&c_idle.work);
693 return PTR_ERR(c_idle.idle);
694 }
695
696 set_idle_for_cpu(cpu, c_idle.idle);
697do_rest:
698 per_cpu(current_task, cpu) = c_idle.idle;
699#ifdef CONFIG_X86_32
700
701 irq_ctx_init(cpu);
702#else
703 clear_tsk_thread_flag(c_idle.idle, TIF_FORK);
704 initial_gs = per_cpu_offset(cpu);
705 per_cpu(kernel_stack, cpu) =
706 (unsigned long)task_stack_page(c_idle.idle) -
707 KERNEL_STACK_OFFSET + THREAD_SIZE;
708#endif
709 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
710 initial_code = (unsigned long)start_secondary;
711 stack_start = c_idle.idle->thread.sp;
712
713
714 start_ip = trampoline_address();
715
716
717 announce_cpu(cpu, apicid);
718
719
720
721
722
723
724 printk(KERN_DEBUG "smpboot cpu %d: start_ip = %lx\n", cpu, start_ip);
725
726 atomic_set(&init_deasserted, 0);
727
728 if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
729
730 pr_debug("Setting warm reset code and vector.\n");
731
732 smpboot_setup_warm_reset_vector(start_ip);
733
734
735
736 if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid])) {
737 apic_write(APIC_ESR, 0);
738 apic_read(APIC_ESR);
739 }
740 }
741
742
743
744
745
746 if (apic->wakeup_secondary_cpu)
747 boot_error = apic->wakeup_secondary_cpu(apicid, start_ip);
748 else
749 boot_error = wakeup_secondary_cpu_via_init(apicid, start_ip);
750
751 if (!boot_error) {
752
753
754
755 pr_debug("Before Callout %d.\n", cpu);
756 cpumask_set_cpu(cpu, cpu_callout_mask);
757 pr_debug("After Callout %d.\n", cpu);
758
759
760
761
762 for (timeout = 0; timeout < 50000; timeout++) {
763 if (cpumask_test_cpu(cpu, cpu_callin_mask))
764 break;
765 udelay(100);
766
767
768
769
770
771
772 schedule();
773 }
774
775 if (cpumask_test_cpu(cpu, cpu_callin_mask))
776 pr_debug("CPU%d: has booted.\n", cpu);
777 else {
778 boot_error = 1;
779 if (*(volatile u32 *)TRAMPOLINE_SYM(trampoline_status)
780 == 0xA5A5A5A5)
781
782 pr_err("CPU%d: Stuck ??\n", cpu);
783 else
784
785 pr_err("CPU%d: Not responding.\n", cpu);
786 if (apic->inquire_remote_apic)
787 apic->inquire_remote_apic(apicid);
788 }
789 }
790
791 if (boot_error) {
792
793 numa_remove_cpu(cpu);
794
795
796 cpumask_clear_cpu(cpu, cpu_callout_mask);
797
798
799 cpumask_clear_cpu(cpu, cpu_initialized_mask);
800
801 set_cpu_present(cpu, false);
802 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
803 }
804
805
806 *(volatile u32 *)TRAMPOLINE_SYM(trampoline_status) = 0;
807
808 if (get_uv_system_type() != UV_NON_UNIQUE_APIC) {
809
810
811
812 smpboot_restore_warm_reset_vector();
813 }
814
815 destroy_work_on_stack(&c_idle.work);
816 return boot_error;
817}
818
819int __cpuinit native_cpu_up(unsigned int cpu)
820{
821 int apicid = apic->cpu_present_to_apicid(cpu);
822 unsigned long flags;
823 int err;
824
825 WARN_ON(irqs_disabled());
826
827 pr_debug("++++++++++++++++++++=_---CPU UP %u\n", cpu);
828
829 if (apicid == BAD_APICID || apicid == boot_cpu_physical_apicid ||
830 !physid_isset(apicid, phys_cpu_present_map)) {
831 printk(KERN_ERR "%s: bad cpu %d\n", __func__, cpu);
832 return -EINVAL;
833 }
834
835
836
837
838 if (cpumask_test_cpu(cpu, cpu_callin_mask)) {
839 pr_debug("do_boot_cpu %d Already started\n", cpu);
840 return -ENOSYS;
841 }
842
843
844
845
846
847 mtrr_save_state();
848
849 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
850
851 err = do_boot_cpu(apicid, cpu);
852 if (err) {
853 pr_debug("do_boot_cpu failed %d\n", err);
854 return -EIO;
855 }
856
857
858
859
860
861 local_irq_save(flags);
862 check_tsc_sync_source(cpu);
863 local_irq_restore(flags);
864
865 while (!cpu_online(cpu)) {
866 cpu_relax();
867 touch_nmi_watchdog();
868 }
869
870 return 0;
871}
872
873
874
875
876void arch_disable_smp_support(void)
877{
878 disable_ioapic_support();
879}
880
881
882
883
884
885
886static __init void disable_smp(void)
887{
888 init_cpu_present(cpumask_of(0));
889 init_cpu_possible(cpumask_of(0));
890 smpboot_clear_io_apic_irqs();
891
892 if (smp_found_config)
893 physid_set_mask_of_physid(boot_cpu_physical_apicid, &phys_cpu_present_map);
894 else
895 physid_set_mask_of_physid(0, &phys_cpu_present_map);
896 cpumask_set_cpu(0, cpu_sibling_mask(0));
897 cpumask_set_cpu(0, cpu_core_mask(0));
898}
899
900
901
902
903static int __init smp_sanity_check(unsigned max_cpus)
904{
905 preempt_disable();
906
907#if !defined(CONFIG_X86_BIGSMP) && defined(CONFIG_X86_32)
908 if (def_to_bigsmp && nr_cpu_ids > 8) {
909 unsigned int cpu;
910 unsigned nr;
911
912 printk(KERN_WARNING
913 "More than 8 CPUs detected - skipping them.\n"
914 "Use CONFIG_X86_BIGSMP.\n");
915
916 nr = 0;
917 for_each_present_cpu(cpu) {
918 if (nr >= 8)
919 set_cpu_present(cpu, false);
920 nr++;
921 }
922
923 nr = 0;
924 for_each_possible_cpu(cpu) {
925 if (nr >= 8)
926 set_cpu_possible(cpu, false);
927 nr++;
928 }
929
930 nr_cpu_ids = 8;
931 }
932#endif
933
934 if (!physid_isset(hard_smp_processor_id(), phys_cpu_present_map)) {
935 printk(KERN_WARNING
936 "weird, boot CPU (#%d) not listed by the BIOS.\n",
937 hard_smp_processor_id());
938
939 physid_set(hard_smp_processor_id(), phys_cpu_present_map);
940 }
941
942
943
944
945
946 if (!smp_found_config && !acpi_lapic) {
947 preempt_enable();
948 printk(KERN_NOTICE "SMP motherboard not detected.\n");
949 disable_smp();
950 if (APIC_init_uniprocessor())
951 printk(KERN_NOTICE "Local APIC not detected."
952 " Using dummy APIC emulation.\n");
953 return -1;
954 }
955
956
957
958
959
960 if (!apic->check_phys_apicid_present(boot_cpu_physical_apicid)) {
961 printk(KERN_NOTICE
962 "weird, boot CPU (#%d) not listed by the BIOS.\n",
963 boot_cpu_physical_apicid);
964 physid_set(hard_smp_processor_id(), phys_cpu_present_map);
965 }
966 preempt_enable();
967
968
969
970
971 if (APIC_INTEGRATED(apic_version[boot_cpu_physical_apicid]) &&
972 !cpu_has_apic) {
973 if (!disable_apic) {
974 pr_err("BIOS bug, local APIC #%d not detected!...\n",
975 boot_cpu_physical_apicid);
976 pr_err("... forcing use of dummy APIC emulation."
977 "(tell your hw vendor)\n");
978 }
979 smpboot_clear_io_apic();
980 disable_ioapic_support();
981 return -1;
982 }
983
984 verify_local_APIC();
985
986
987
988
989 if (!max_cpus) {
990 printk(KERN_INFO "SMP mode deactivated.\n");
991 smpboot_clear_io_apic();
992
993 connect_bsp_APIC();
994 setup_local_APIC();
995 bsp_end_local_APIC_setup();
996 return -1;
997 }
998
999 return 0;
1000}
1001
1002static void __init smp_cpu_index_default(void)
1003{
1004 int i;
1005 struct cpuinfo_x86 *c;
1006
1007 for_each_possible_cpu(i) {
1008 c = &cpu_data(i);
1009
1010 c->cpu_index = nr_cpu_ids;
1011 }
1012}
1013
1014
1015
1016
1017
1018void __init native_smp_prepare_cpus(unsigned int max_cpus)
1019{
1020 unsigned int i;
1021
1022 preempt_disable();
1023 smp_cpu_index_default();
1024
1025
1026
1027
1028 smp_store_cpu_info(0);
1029 cpumask_copy(cpu_callin_mask, cpumask_of(0));
1030 mb();
1031
1032 current_thread_info()->cpu = 0;
1033 for_each_possible_cpu(i) {
1034 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
1035 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
1036 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
1037 }
1038 set_cpu_sibling_map(0);
1039
1040
1041 if (smp_sanity_check(max_cpus) < 0) {
1042 printk(KERN_INFO "SMP disabled\n");
1043 disable_smp();
1044 goto out;
1045 }
1046
1047 default_setup_apic_routing();
1048
1049 preempt_disable();
1050 if (read_apic_id() != boot_cpu_physical_apicid) {
1051 panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
1052 read_apic_id(), boot_cpu_physical_apicid);
1053
1054 }
1055 preempt_enable();
1056
1057 connect_bsp_APIC();
1058
1059
1060
1061
1062 setup_local_APIC();
1063
1064
1065
1066
1067 if (!skip_ioapic_setup && nr_ioapics)
1068 enable_IO_APIC();
1069
1070 bsp_end_local_APIC_setup();
1071
1072 if (apic->setup_portio_remap)
1073 apic->setup_portio_remap();
1074
1075 smpboot_setup_io_apic();
1076
1077
1078
1079
1080 printk(KERN_INFO "CPU%d: ", 0);
1081 print_cpu_info(&cpu_data(0));
1082 x86_init.timers.setup_percpu_clockev();
1083
1084 if (is_uv_system())
1085 uv_system_init();
1086
1087 set_mtrr_aps_delayed_init();
1088out:
1089 preempt_enable();
1090}
1091
1092void arch_disable_nonboot_cpus_begin(void)
1093{
1094
1095
1096
1097
1098 skip_smp_alternatives = true;
1099}
1100
1101void arch_disable_nonboot_cpus_end(void)
1102{
1103 skip_smp_alternatives = false;
1104}
1105
1106void arch_enable_nonboot_cpus_begin(void)
1107{
1108 set_mtrr_aps_delayed_init();
1109}
1110
1111void arch_enable_nonboot_cpus_end(void)
1112{
1113 mtrr_aps_init();
1114}
1115
1116
1117
1118
1119void __init native_smp_prepare_boot_cpu(void)
1120{
1121 int me = smp_processor_id();
1122 switch_to_new_gdt(me);
1123
1124 cpumask_set_cpu(me, cpu_callout_mask);
1125 per_cpu(cpu_state, me) = CPU_ONLINE;
1126}
1127
1128void __init native_smp_cpus_done(unsigned int max_cpus)
1129{
1130 pr_debug("Boot done.\n");
1131
1132 impress_friends();
1133#ifdef CONFIG_X86_IO_APIC
1134 setup_ioapic_dest();
1135#endif
1136 mtrr_aps_init();
1137}
1138
1139static int __initdata setup_possible_cpus = -1;
1140static int __init _setup_possible_cpus(char *str)
1141{
1142 get_option(&str, &setup_possible_cpus);
1143 return 0;
1144}
1145early_param("possible_cpus", _setup_possible_cpus);
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165__init void prefill_possible_map(void)
1166{
1167 int i, possible;
1168
1169
1170 if (!num_processors)
1171 num_processors = 1;
1172
1173 i = setup_max_cpus ?: 1;
1174 if (setup_possible_cpus == -1) {
1175 possible = num_processors;
1176#ifdef CONFIG_HOTPLUG_CPU
1177 if (setup_max_cpus)
1178 possible += disabled_cpus;
1179#else
1180 if (possible > i)
1181 possible = i;
1182#endif
1183 } else
1184 possible = setup_possible_cpus;
1185
1186 total_cpus = max_t(int, possible, num_processors + disabled_cpus);
1187
1188
1189 if (possible > nr_cpu_ids) {
1190 printk(KERN_WARNING
1191 "%d Processors exceeds NR_CPUS limit of %d\n",
1192 possible, nr_cpu_ids);
1193 possible = nr_cpu_ids;
1194 }
1195
1196#ifdef CONFIG_HOTPLUG_CPU
1197 if (!setup_max_cpus)
1198#endif
1199 if (possible > i) {
1200 printk(KERN_WARNING
1201 "%d Processors exceeds max_cpus limit of %u\n",
1202 possible, setup_max_cpus);
1203 possible = i;
1204 }
1205
1206 printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
1207 possible, max_t(int, possible - num_processors, 0));
1208
1209 for (i = 0; i < possible; i++)
1210 set_cpu_possible(i, true);
1211 for (; i < NR_CPUS; i++)
1212 set_cpu_possible(i, false);
1213
1214 nr_cpu_ids = possible;
1215}
1216
1217#ifdef CONFIG_HOTPLUG_CPU
1218
1219static void remove_siblinginfo(int cpu)
1220{
1221 int sibling;
1222 struct cpuinfo_x86 *c = &cpu_data(cpu);
1223
1224 for_each_cpu(sibling, cpu_core_mask(cpu)) {
1225 cpumask_clear_cpu(cpu, cpu_core_mask(sibling));
1226
1227
1228
1229 if (cpumask_weight(cpu_sibling_mask(cpu)) == 1)
1230 cpu_data(sibling).booted_cores--;
1231 }
1232
1233 for_each_cpu(sibling, cpu_sibling_mask(cpu))
1234 cpumask_clear_cpu(cpu, cpu_sibling_mask(sibling));
1235 cpumask_clear(cpu_sibling_mask(cpu));
1236 cpumask_clear(cpu_core_mask(cpu));
1237 c->phys_proc_id = 0;
1238 c->cpu_core_id = 0;
1239 cpumask_clear_cpu(cpu, cpu_sibling_setup_mask);
1240}
1241
1242static void __ref remove_cpu_from_maps(int cpu)
1243{
1244 set_cpu_online(cpu, false);
1245 cpumask_clear_cpu(cpu, cpu_callout_mask);
1246 cpumask_clear_cpu(cpu, cpu_callin_mask);
1247
1248 cpumask_clear_cpu(cpu, cpu_initialized_mask);
1249 numa_remove_cpu(cpu);
1250}
1251
1252void cpu_disable_common(void)
1253{
1254 int cpu = smp_processor_id();
1255
1256 remove_siblinginfo(cpu);
1257
1258
1259 lock_vector_lock();
1260 remove_cpu_from_maps(cpu);
1261 unlock_vector_lock();
1262 fixup_irqs();
1263}
1264
1265int native_cpu_disable(void)
1266{
1267 int cpu = smp_processor_id();
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277 if (cpu == 0)
1278 return -EBUSY;
1279
1280 clear_local_APIC();
1281
1282 cpu_disable_common();
1283 return 0;
1284}
1285
1286void native_cpu_die(unsigned int cpu)
1287{
1288
1289 unsigned int i;
1290
1291 for (i = 0; i < 10; i++) {
1292
1293 if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1294 if (system_state == SYSTEM_RUNNING)
1295 pr_info("CPU %u is now offline\n", cpu);
1296
1297 if (1 == num_online_cpus())
1298 alternatives_smp_switch(0);
1299 return;
1300 }
1301 msleep(100);
1302 }
1303 pr_err("CPU %u didn't die...\n", cpu);
1304}
1305
1306void play_dead_common(void)
1307{
1308 idle_task_exit();
1309 reset_lazy_tlbstate();
1310 c1e_remove_cpu(raw_smp_processor_id());
1311
1312 mb();
1313
1314 __this_cpu_write(cpu_state, CPU_DEAD);
1315
1316
1317
1318
1319 local_irq_disable();
1320}
1321
1322
1323
1324
1325
1326static inline void mwait_play_dead(void)
1327{
1328 unsigned int eax, ebx, ecx, edx;
1329 unsigned int highest_cstate = 0;
1330 unsigned int highest_subcstate = 0;
1331 int i;
1332 void *mwait_ptr;
1333 struct cpuinfo_x86 *c = __this_cpu_ptr(&cpu_info);
1334
1335 if (!(cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)))
1336 return;
1337 if (!cpu_has(__this_cpu_ptr(&cpu_info), X86_FEATURE_CLFLSH))
1338 return;
1339 if (__this_cpu_read(cpu_info.cpuid_level) < CPUID_MWAIT_LEAF)
1340 return;
1341
1342 eax = CPUID_MWAIT_LEAF;
1343 ecx = 0;
1344 native_cpuid(&eax, &ebx, &ecx, &edx);
1345
1346
1347
1348
1349
1350 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED)) {
1351 eax = 0;
1352 } else {
1353 edx >>= MWAIT_SUBSTATE_SIZE;
1354 for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
1355 if (edx & MWAIT_SUBSTATE_MASK) {
1356 highest_cstate = i;
1357 highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
1358 }
1359 }
1360 eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
1361 (highest_subcstate - 1);
1362 }
1363
1364
1365
1366
1367
1368
1369 mwait_ptr = ¤t_thread_info()->flags;
1370
1371 wbinvd();
1372
1373 while (1) {
1374
1375
1376
1377
1378
1379
1380
1381 clflush(mwait_ptr);
1382 __monitor(mwait_ptr, 0, 0);
1383 mb();
1384 __mwait(eax, 0);
1385 }
1386}
1387
1388static inline void hlt_play_dead(void)
1389{
1390 if (__this_cpu_read(cpu_info.x86) >= 4)
1391 wbinvd();
1392
1393 while (1) {
1394 native_halt();
1395 }
1396}
1397
1398void native_play_dead(void)
1399{
1400 play_dead_common();
1401 tboot_shutdown(TB_SHUTDOWN_WFS);
1402
1403 mwait_play_dead();
1404 hlt_play_dead();
1405}
1406
1407#else
1408int native_cpu_disable(void)
1409{
1410 return -ENOSYS;
1411}
1412
1413void native_cpu_die(unsigned int cpu)
1414{
1415
1416 BUG();
1417}
1418
1419void native_play_dead(void)
1420{
1421 BUG();
1422}
1423
1424#endif
1425