1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27#include <linux/acpi.h>
28#include <linux/memblock.h>
29#include <linux/cpu.h>
30#include <linux/delay.h>
31#include <linux/init.h>
32#include <linux/interrupt.h>
33#include <linux/irq.h>
34#include <linux/kernel.h>
35#include <linux/kernel_stat.h>
36#include <linux/mm.h>
37#include <linux/notifier.h>
38#include <linux/smp.h>
39#include <linux/spinlock.h>
40#include <linux/efi.h>
41#include <linux/percpu.h>
42#include <linux/bitops.h>
43
44#include <linux/atomic.h>
45#include <asm/cache.h>
46#include <asm/current.h>
47#include <asm/delay.h>
48#include <asm/efi.h>
49#include <asm/io.h>
50#include <asm/irq.h>
51#include <asm/mca.h>
52#include <asm/page.h>
53#include <asm/processor.h>
54#include <asm/ptrace.h>
55#include <asm/sal.h>
56#include <asm/tlbflush.h>
57#include <asm/unistd.h>
58
59#define SMP_DEBUG 0
60
61#if SMP_DEBUG
62#define Dprintk(x...) printk(x)
63#else
64#define Dprintk(x...)
65#endif
66
67#ifdef CONFIG_HOTPLUG_CPU
68#ifdef CONFIG_PERMIT_BSP_REMOVE
69#define bsp_remove_ok 1
70#else
71#define bsp_remove_ok 0
72#endif
73
74
75
76
77struct sal_to_os_boot sal_boot_rendez_state[NR_CPUS];
78
79
80
81
82
83struct sal_to_os_boot *sal_state_for_booting_cpu = &sal_boot_rendez_state[0];
84
85#define set_brendez_area(x) (sal_state_for_booting_cpu = &sal_boot_rendez_state[(x)]);
86
87#else
88#define set_brendez_area(x)
89#endif
90
91
92
93
94
95#define MASTER (0)
96#define SLAVE (SMP_CACHE_BYTES/8)
97
98#define NUM_ROUNDS 64
99#define NUM_ITERS 5
100
101static DEFINE_SPINLOCK(itc_sync_lock);
102static volatile unsigned long go[SLAVE + 1];
103
104#define DEBUG_ITC_SYNC 0
105
106extern void start_ap (void);
107extern unsigned long ia64_iobase;
108
109struct task_struct *task_for_booting_cpu;
110
111
112
113
114DEFINE_PER_CPU(int, cpu_state);
115
116cpumask_t cpu_core_map[NR_CPUS] __cacheline_aligned;
117EXPORT_SYMBOL(cpu_core_map);
118DEFINE_PER_CPU_SHARED_ALIGNED(cpumask_t, cpu_sibling_map);
119EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
120
121int smp_num_siblings = 1;
122
123
124volatile int ia64_cpu_to_sapicid[NR_CPUS];
125EXPORT_SYMBOL(ia64_cpu_to_sapicid);
126
127static cpumask_t cpu_callin_map;
128
129struct smp_boot_data smp_boot_data __initdata;
130
131unsigned long ap_wakeup_vector = -1;
132
133char __initdata no_int_routing;
134
135unsigned char smp_int_redirect;
136
137#ifdef CONFIG_FORCE_CPEI_RETARGET
138#define CPEI_OVERRIDE_DEFAULT (1)
139#else
140#define CPEI_OVERRIDE_DEFAULT (0)
141#endif
142
143unsigned int force_cpei_retarget = CPEI_OVERRIDE_DEFAULT;
144
145static int __init
146cmdl_force_cpei(char *str)
147{
148 int value=0;
149
150 get_option (&str, &value);
151 force_cpei_retarget = value;
152
153 return 1;
154}
155
156__setup("force_cpei=", cmdl_force_cpei);
157
158static int __init
159nointroute (char *str)
160{
161 no_int_routing = 1;
162 printk ("no_int_routing on\n");
163 return 1;
164}
165
166__setup("nointroute", nointroute);
167
168static void fix_b0_for_bsp(void)
169{
170#ifdef CONFIG_HOTPLUG_CPU
171 int cpuid;
172 static int fix_bsp_b0 = 1;
173
174 cpuid = smp_processor_id();
175
176
177
178
179 if (!(fix_bsp_b0 && cpuid))
180 return;
181
182 sal_boot_rendez_state[0].br[0] = sal_boot_rendez_state[cpuid].br[0];
183 printk ("Fixed BSP b0 value from CPU %d\n", cpuid);
184
185 fix_bsp_b0 = 0;
186#endif
187}
188
189void
190sync_master (void *arg)
191{
192 unsigned long flags, i;
193
194 go[MASTER] = 0;
195
196 local_irq_save(flags);
197 {
198 for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
199 while (!go[MASTER])
200 cpu_relax();
201 go[MASTER] = 0;
202 go[SLAVE] = ia64_get_itc();
203 }
204 }
205 local_irq_restore(flags);
206}
207
208
209
210
211
212
213static inline long
214get_delta (long *rt, long *master)
215{
216 unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
217 unsigned long tcenter, t0, t1, tm;
218 long i;
219
220 for (i = 0; i < NUM_ITERS; ++i) {
221 t0 = ia64_get_itc();
222 go[MASTER] = 1;
223 while (!(tm = go[SLAVE]))
224 cpu_relax();
225 go[SLAVE] = 0;
226 t1 = ia64_get_itc();
227
228 if (t1 - t0 < best_t1 - best_t0)
229 best_t0 = t0, best_t1 = t1, best_tm = tm;
230 }
231
232 *rt = best_t1 - best_t0;
233 *master = best_tm - best_t0;
234
235
236 tcenter = (best_t0/2 + best_t1/2);
237 if (best_t0 % 2 + best_t1 % 2 == 2)
238 ++tcenter;
239 return tcenter - best_tm;
240}
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274void
275ia64_sync_itc (unsigned int master)
276{
277 long i, delta, adj, adjust_latency = 0, done = 0;
278 unsigned long flags, rt, master_time_stamp, bound;
279#if DEBUG_ITC_SYNC
280 struct {
281 long rt;
282 long master;
283 long diff;
284 long lat;
285 } t[NUM_ROUNDS];
286#endif
287
288
289
290
291
292
293
294 BUG_ON((ia64_get_itv() & (1 << 16)) == 0);
295
296 go[MASTER] = 1;
297
298 if (smp_call_function_single(master, sync_master, NULL, 0) < 0) {
299 printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master);
300 return;
301 }
302
303 while (go[MASTER])
304 cpu_relax();
305
306 spin_lock_irqsave(&itc_sync_lock, flags);
307 {
308 for (i = 0; i < NUM_ROUNDS; ++i) {
309 delta = get_delta(&rt, &master_time_stamp);
310 if (delta == 0) {
311 done = 1;
312 bound = rt;
313 }
314
315 if (!done) {
316 if (i > 0) {
317 adjust_latency += -delta;
318 adj = -delta + adjust_latency/4;
319 } else
320 adj = -delta;
321
322 ia64_set_itc(ia64_get_itc() + adj);
323 }
324#if DEBUG_ITC_SYNC
325 t[i].rt = rt;
326 t[i].master = master_time_stamp;
327 t[i].diff = delta;
328 t[i].lat = adjust_latency/4;
329#endif
330 }
331 }
332 spin_unlock_irqrestore(&itc_sync_lock, flags);
333
334#if DEBUG_ITC_SYNC
335 for (i = 0; i < NUM_ROUNDS; ++i)
336 printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
337 t[i].rt, t[i].master, t[i].diff, t[i].lat);
338#endif
339
340 printk(KERN_INFO "CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, "
341 "maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt);
342}
343
344
345
346
347static inline void smp_setup_percpu_timer(void)
348{
349}
350
351static void
352smp_callin (void)
353{
354 int cpuid, phys_id, itc_master;
355 struct cpuinfo_ia64 *last_cpuinfo, *this_cpuinfo;
356 extern void ia64_init_itm(void);
357 extern volatile int time_keeper_id;
358
359 cpuid = smp_processor_id();
360 phys_id = hard_smp_processor_id();
361 itc_master = time_keeper_id;
362
363 if (cpu_online(cpuid)) {
364 printk(KERN_ERR "huh, phys CPU#0x%x, CPU#0x%x already present??\n",
365 phys_id, cpuid);
366 BUG();
367 }
368
369 fix_b0_for_bsp();
370
371
372
373
374 set_numa_node(cpu_to_node_map[cpuid]);
375 set_numa_mem(local_memory_node(cpu_to_node_map[cpuid]));
376
377 spin_lock(&vector_lock);
378
379 __setup_vector_irq(cpuid);
380 notify_cpu_starting(cpuid);
381 set_cpu_online(cpuid, true);
382 per_cpu(cpu_state, cpuid) = CPU_ONLINE;
383 spin_unlock(&vector_lock);
384
385 smp_setup_percpu_timer();
386
387 ia64_mca_cmc_vector_setup();
388
389 local_irq_enable();
390
391 if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) {
392
393
394
395
396
397
398 Dprintk("Going to syncup ITC with ITC Master.\n");
399 ia64_sync_itc(itc_master);
400 }
401
402
403
404
405 ia64_init_itm();
406
407
408
409
410
411 last_cpuinfo = cpu_data(cpuid - 1);
412 this_cpuinfo = local_cpu_data;
413 if (last_cpuinfo->itc_freq != this_cpuinfo->itc_freq ||
414 last_cpuinfo->proc_freq != this_cpuinfo->proc_freq ||
415 last_cpuinfo->features != this_cpuinfo->features ||
416 last_cpuinfo->revision != this_cpuinfo->revision ||
417 last_cpuinfo->family != this_cpuinfo->family ||
418 last_cpuinfo->archrev != this_cpuinfo->archrev ||
419 last_cpuinfo->model != this_cpuinfo->model)
420 calibrate_delay();
421 local_cpu_data->loops_per_jiffy = loops_per_jiffy;
422
423
424
425
426 cpumask_set_cpu(cpuid, &cpu_callin_map);
427 Dprintk("Stack on CPU %d at about %p\n",cpuid, &cpuid);
428}
429
430
431
432
433
434int
435start_secondary (void *unused)
436{
437
438 ia64_set_kr(IA64_KR_IO_BASE, __pa(ia64_iobase));
439#ifndef CONFIG_PRINTK_TIME
440 Dprintk("start_secondary: starting CPU 0x%x\n", hard_smp_processor_id());
441#endif
442 efi_map_pal_code();
443 cpu_init();
444 smp_callin();
445
446 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
447 return 0;
448}
449
450static int
451do_boot_cpu (int sapicid, int cpu, struct task_struct *idle)
452{
453 int timeout;
454
455 task_for_booting_cpu = idle;
456 Dprintk("Sending wakeup vector %lu to AP 0x%x/0x%x.\n", ap_wakeup_vector, cpu, sapicid);
457
458 set_brendez_area(cpu);
459 ia64_send_ipi(cpu, ap_wakeup_vector, IA64_IPI_DM_INT, 0);
460
461
462
463
464 Dprintk("Waiting on callin_map ...");
465 for (timeout = 0; timeout < 100000; timeout++) {
466 if (cpumask_test_cpu(cpu, &cpu_callin_map))
467 break;
468 barrier();
469 udelay(100);
470 }
471 Dprintk("\n");
472
473 if (!cpumask_test_cpu(cpu, &cpu_callin_map)) {
474 printk(KERN_ERR "Processor 0x%x/0x%x is stuck.\n", cpu, sapicid);
475 ia64_cpu_to_sapicid[cpu] = -1;
476 set_cpu_online(cpu, false);
477 return -EINVAL;
478 }
479 return 0;
480}
481
482static int __init
483decay (char *str)
484{
485 int ticks;
486 get_option (&str, &ticks);
487 return 1;
488}
489
490__setup("decay=", decay);
491
492
493
494
495void __init
496smp_build_cpu_map (void)
497{
498 int sapicid, cpu, i;
499 int boot_cpu_id = hard_smp_processor_id();
500
501 for (cpu = 0; cpu < NR_CPUS; cpu++) {
502 ia64_cpu_to_sapicid[cpu] = -1;
503 }
504
505 ia64_cpu_to_sapicid[0] = boot_cpu_id;
506 init_cpu_present(cpumask_of(0));
507 set_cpu_possible(0, true);
508 for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
509 sapicid = smp_boot_data.cpu_phys_id[i];
510 if (sapicid == boot_cpu_id)
511 continue;
512 set_cpu_present(cpu, true);
513 set_cpu_possible(cpu, true);
514 ia64_cpu_to_sapicid[cpu] = sapicid;
515 cpu++;
516 }
517}
518
519
520
521
522void __init
523smp_prepare_cpus (unsigned int max_cpus)
524{
525 int boot_cpu_id = hard_smp_processor_id();
526
527
528
529
530
531 smp_setup_percpu_timer();
532
533 cpumask_set_cpu(0, &cpu_callin_map);
534
535 local_cpu_data->loops_per_jiffy = loops_per_jiffy;
536 ia64_cpu_to_sapicid[0] = boot_cpu_id;
537
538 printk(KERN_INFO "Boot processor id 0x%x/0x%x\n", 0, boot_cpu_id);
539
540 current_thread_info()->cpu = 0;
541
542
543
544
545 if (!max_cpus) {
546 printk(KERN_INFO "SMP mode deactivated.\n");
547 init_cpu_online(cpumask_of(0));
548 init_cpu_present(cpumask_of(0));
549 init_cpu_possible(cpumask_of(0));
550 return;
551 }
552}
553
554void smp_prepare_boot_cpu(void)
555{
556 set_cpu_online(smp_processor_id(), true);
557 cpumask_set_cpu(smp_processor_id(), &cpu_callin_map);
558 set_numa_node(cpu_to_node_map[smp_processor_id()]);
559 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
560}
561
562#ifdef CONFIG_HOTPLUG_CPU
563static inline void
564clear_cpu_sibling_map(int cpu)
565{
566 int i;
567
568 for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu))
569 cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i));
570 for_each_cpu(i, &cpu_core_map[cpu])
571 cpumask_clear_cpu(cpu, &cpu_core_map[i]);
572
573 per_cpu(cpu_sibling_map, cpu) = cpu_core_map[cpu] = CPU_MASK_NONE;
574}
575
576static void
577remove_siblinginfo(int cpu)
578{
579 if (cpu_data(cpu)->threads_per_core == 1 &&
580 cpu_data(cpu)->cores_per_socket == 1) {
581 cpumask_clear_cpu(cpu, &cpu_core_map[cpu]);
582 cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, cpu));
583 return;
584 }
585
586
587 clear_cpu_sibling_map(cpu);
588}
589
590extern void fixup_irqs(void);
591
592int migrate_platform_irqs(unsigned int cpu)
593{
594 int new_cpei_cpu;
595 struct irq_data *data = NULL;
596 const struct cpumask *mask;
597 int retval = 0;
598
599
600
601
602 if (cpe_vector > 0 && is_cpu_cpei_target(cpu)) {
603 printk ("CPU (%d) is CPEI Target\n", cpu);
604 if (can_cpei_retarget()) {
605
606
607
608 new_cpei_cpu = cpumask_any(cpu_online_mask);
609 mask = cpumask_of(new_cpei_cpu);
610 set_cpei_target_cpu(new_cpei_cpu);
611 data = irq_get_irq_data(ia64_cpe_irq);
612
613
614
615
616
617 if (data && data->chip) {
618 data->chip->irq_disable(data);
619 data->chip->irq_set_affinity(data, mask, false);
620 data->chip->irq_enable(data);
621 printk ("Re-targeting CPEI to cpu %d\n", new_cpei_cpu);
622 }
623 }
624 if (!data) {
625 printk ("Unable to retarget CPEI, offline cpu [%d] failed\n", cpu);
626 retval = -EBUSY;
627 }
628 }
629 return retval;
630}
631
632
633int __cpu_disable(void)
634{
635 int cpu = smp_processor_id();
636
637
638
639
640 if (cpu == 0 && !bsp_remove_ok) {
641 printk ("Your platform does not support removal of BSP\n");
642 return (-EBUSY);
643 }
644
645 set_cpu_online(cpu, false);
646
647 if (migrate_platform_irqs(cpu)) {
648 set_cpu_online(cpu, true);
649 return -EBUSY;
650 }
651
652 remove_siblinginfo(cpu);
653 fixup_irqs();
654 local_flush_tlb_all();
655 cpumask_clear_cpu(cpu, &cpu_callin_map);
656 return 0;
657}
658
659void __cpu_die(unsigned int cpu)
660{
661 unsigned int i;
662
663 for (i = 0; i < 100; i++) {
664
665 if (per_cpu(cpu_state, cpu) == CPU_DEAD)
666 {
667 printk ("CPU %d is now offline\n", cpu);
668 return;
669 }
670 msleep(100);
671 }
672 printk(KERN_ERR "CPU %u didn't die...\n", cpu);
673}
674#endif
675
676void
677smp_cpus_done (unsigned int dummy)
678{
679 int cpu;
680 unsigned long bogosum = 0;
681
682
683
684
685
686 for_each_online_cpu(cpu) {
687 bogosum += cpu_data(cpu)->loops_per_jiffy;
688 }
689
690 printk(KERN_INFO "Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
691 (int)num_online_cpus(), bogosum/(500000/HZ), (bogosum/(5000/HZ))%100);
692}
693
694static inline void set_cpu_sibling_map(int cpu)
695{
696 int i;
697
698 for_each_online_cpu(i) {
699 if ((cpu_data(cpu)->socket_id == cpu_data(i)->socket_id)) {
700 cpumask_set_cpu(i, &cpu_core_map[cpu]);
701 cpumask_set_cpu(cpu, &cpu_core_map[i]);
702 if (cpu_data(cpu)->core_id == cpu_data(i)->core_id) {
703 cpumask_set_cpu(i,
704 &per_cpu(cpu_sibling_map, cpu));
705 cpumask_set_cpu(cpu,
706 &per_cpu(cpu_sibling_map, i));
707 }
708 }
709 }
710}
711
712int
713__cpu_up(unsigned int cpu, struct task_struct *tidle)
714{
715 int ret;
716 int sapicid;
717
718 sapicid = ia64_cpu_to_sapicid[cpu];
719 if (sapicid == -1)
720 return -EINVAL;
721
722
723
724
725
726 if (cpumask_test_cpu(cpu, &cpu_callin_map))
727 return -EINVAL;
728
729 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
730
731 ret = do_boot_cpu(sapicid, cpu, tidle);
732 if (ret < 0)
733 return ret;
734
735 if (cpu_data(cpu)->threads_per_core == 1 &&
736 cpu_data(cpu)->cores_per_socket == 1) {
737 cpumask_set_cpu(cpu, &per_cpu(cpu_sibling_map, cpu));
738 cpumask_set_cpu(cpu, &cpu_core_map[cpu]);
739 return 0;
740 }
741
742 set_cpu_sibling_map(cpu);
743
744 return 0;
745}
746
747
748
749
750
751
752
753void __init
754init_smp_config(void)
755{
756 struct fptr {
757 unsigned long fp;
758 unsigned long gp;
759 } *ap_startup;
760 long sal_ret;
761
762
763 ap_startup = (struct fptr *) start_ap;
764 sal_ret = ia64_sal_set_vectors(SAL_VECTOR_OS_BOOT_RENDEZ,
765 ia64_tpa(ap_startup->fp), ia64_tpa(ap_startup->gp), 0, 0, 0, 0);
766 if (sal_ret < 0)
767 printk(KERN_ERR "SMP: Can't set SAL AP Boot Rendezvous: %s\n",
768 ia64_sal_strerror(sal_ret));
769}
770
771
772
773
774
775void identify_siblings(struct cpuinfo_ia64 *c)
776{
777 long status;
778 u16 pltid;
779 pal_logical_to_physical_t info;
780
781 status = ia64_pal_logical_to_phys(-1, &info);
782 if (status != PAL_STATUS_SUCCESS) {
783 if (status != PAL_STATUS_UNIMPLEMENTED) {
784 printk(KERN_ERR
785 "ia64_pal_logical_to_phys failed with %ld\n",
786 status);
787 return;
788 }
789
790 info.overview_ppid = 0;
791 info.overview_cpp = 1;
792 info.overview_tpc = 1;
793 }
794
795 status = ia64_sal_physical_id_info(&pltid);
796 if (status != PAL_STATUS_SUCCESS) {
797 if (status != PAL_STATUS_UNIMPLEMENTED)
798 printk(KERN_ERR
799 "ia64_sal_pltid failed with %ld\n",
800 status);
801 return;
802 }
803
804 c->socket_id = (pltid << 8) | info.overview_ppid;
805
806 if (info.overview_cpp == 1 && info.overview_tpc == 1)
807 return;
808
809 c->cores_per_socket = info.overview_cpp;
810 c->threads_per_core = info.overview_tpc;
811 c->num_log = info.overview_num_log;
812
813 c->core_id = info.log1_cid;
814 c->thread_id = info.log1_tid;
815}
816
817
818
819
820
821
822
823int is_multithreading_enabled(void)
824{
825 int i, j;
826
827 for_each_present_cpu(i) {
828 for_each_present_cpu(j) {
829 if (j == i)
830 continue;
831 if ((cpu_data(j)->socket_id == cpu_data(i)->socket_id)) {
832 if (cpu_data(j)->core_id == cpu_data(i)->core_id)
833 return 1;
834 }
835 }
836 }
837 return 0;
838}
839EXPORT_SYMBOL_GPL(is_multithreading_enabled);
840