1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27#include <linux/acpi.h>
28#include <linux/memblock.h>
29#include <linux/cpu.h>
30#include <linux/delay.h>
31#include <linux/init.h>
32#include <linux/interrupt.h>
33#include <linux/irq.h>
34#include <linux/kernel.h>
35#include <linux/kernel_stat.h>
36#include <linux/mm.h>
37#include <linux/notifier.h>
38#include <linux/smp.h>
39#include <linux/spinlock.h>
40#include <linux/efi.h>
41#include <linux/percpu.h>
42#include <linux/bitops.h>
43
44#include <linux/atomic.h>
45#include <asm/cache.h>
46#include <asm/current.h>
47#include <asm/delay.h>
48#include <asm/io.h>
49#include <asm/irq.h>
50#include <asm/machvec.h>
51#include <asm/mca.h>
52#include <asm/page.h>
53#include <asm/pgalloc.h>
54#include <asm/pgtable.h>
55#include <asm/processor.h>
56#include <asm/ptrace.h>
57#include <asm/sal.h>
58#include <asm/tlbflush.h>
59#include <asm/unistd.h>
60#include <asm/sn/arch.h>
61
62#define SMP_DEBUG 0
63
64#if SMP_DEBUG
65#define Dprintk(x...) printk(x)
66#else
67#define Dprintk(x...)
68#endif
69
70#ifdef CONFIG_HOTPLUG_CPU
71#ifdef CONFIG_PERMIT_BSP_REMOVE
72#define bsp_remove_ok 1
73#else
74#define bsp_remove_ok 0
75#endif
76
77
78
79
80struct sal_to_os_boot sal_boot_rendez_state[NR_CPUS];
81
82
83
84
85
86struct sal_to_os_boot *sal_state_for_booting_cpu = &sal_boot_rendez_state[0];
87
88#define set_brendez_area(x) (sal_state_for_booting_cpu = &sal_boot_rendez_state[(x)]);
89
90#else
91#define set_brendez_area(x)
92#endif
93
94
95
96
97
98#define MASTER (0)
99#define SLAVE (SMP_CACHE_BYTES/8)
100
101#define NUM_ROUNDS 64
102#define NUM_ITERS 5
103
104static DEFINE_SPINLOCK(itc_sync_lock);
105static volatile unsigned long go[SLAVE + 1];
106
107#define DEBUG_ITC_SYNC 0
108
109extern void start_ap (void);
110extern unsigned long ia64_iobase;
111
112struct task_struct *task_for_booting_cpu;
113
114
115
116
117DEFINE_PER_CPU(int, cpu_state);
118
119cpumask_t cpu_core_map[NR_CPUS] __cacheline_aligned;
120EXPORT_SYMBOL(cpu_core_map);
121DEFINE_PER_CPU_SHARED_ALIGNED(cpumask_t, cpu_sibling_map);
122EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
123
124int smp_num_siblings = 1;
125
126
127volatile int ia64_cpu_to_sapicid[NR_CPUS];
128EXPORT_SYMBOL(ia64_cpu_to_sapicid);
129
130static cpumask_t cpu_callin_map;
131
132struct smp_boot_data smp_boot_data __initdata;
133
134unsigned long ap_wakeup_vector = -1;
135
136char __initdata no_int_routing;
137
138unsigned char smp_int_redirect;
139
140#ifdef CONFIG_FORCE_CPEI_RETARGET
141#define CPEI_OVERRIDE_DEFAULT (1)
142#else
143#define CPEI_OVERRIDE_DEFAULT (0)
144#endif
145
146unsigned int force_cpei_retarget = CPEI_OVERRIDE_DEFAULT;
147
148static int __init
149cmdl_force_cpei(char *str)
150{
151 int value=0;
152
153 get_option (&str, &value);
154 force_cpei_retarget = value;
155
156 return 1;
157}
158
159__setup("force_cpei=", cmdl_force_cpei);
160
161static int __init
162nointroute (char *str)
163{
164 no_int_routing = 1;
165 printk ("no_int_routing on\n");
166 return 1;
167}
168
169__setup("nointroute", nointroute);
170
171static void fix_b0_for_bsp(void)
172{
173#ifdef CONFIG_HOTPLUG_CPU
174 int cpuid;
175 static int fix_bsp_b0 = 1;
176
177 cpuid = smp_processor_id();
178
179
180
181
182 if (!(fix_bsp_b0 && cpuid))
183 return;
184
185 sal_boot_rendez_state[0].br[0] = sal_boot_rendez_state[cpuid].br[0];
186 printk ("Fixed BSP b0 value from CPU %d\n", cpuid);
187
188 fix_bsp_b0 = 0;
189#endif
190}
191
192void
193sync_master (void *arg)
194{
195 unsigned long flags, i;
196
197 go[MASTER] = 0;
198
199 local_irq_save(flags);
200 {
201 for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
202 while (!go[MASTER])
203 cpu_relax();
204 go[MASTER] = 0;
205 go[SLAVE] = ia64_get_itc();
206 }
207 }
208 local_irq_restore(flags);
209}
210
211
212
213
214
215
216static inline long
217get_delta (long *rt, long *master)
218{
219 unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
220 unsigned long tcenter, t0, t1, tm;
221 long i;
222
223 for (i = 0; i < NUM_ITERS; ++i) {
224 t0 = ia64_get_itc();
225 go[MASTER] = 1;
226 while (!(tm = go[SLAVE]))
227 cpu_relax();
228 go[SLAVE] = 0;
229 t1 = ia64_get_itc();
230
231 if (t1 - t0 < best_t1 - best_t0)
232 best_t0 = t0, best_t1 = t1, best_tm = tm;
233 }
234
235 *rt = best_t1 - best_t0;
236 *master = best_tm - best_t0;
237
238
239 tcenter = (best_t0/2 + best_t1/2);
240 if (best_t0 % 2 + best_t1 % 2 == 2)
241 ++tcenter;
242 return tcenter - best_tm;
243}
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277void
278ia64_sync_itc (unsigned int master)
279{
280 long i, delta, adj, adjust_latency = 0, done = 0;
281 unsigned long flags, rt, master_time_stamp, bound;
282#if DEBUG_ITC_SYNC
283 struct {
284 long rt;
285 long master;
286 long diff;
287 long lat;
288 } t[NUM_ROUNDS];
289#endif
290
291
292
293
294
295
296
297 BUG_ON((ia64_get_itv() & (1 << 16)) == 0);
298
299 go[MASTER] = 1;
300
301 if (smp_call_function_single(master, sync_master, NULL, 0) < 0) {
302 printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master);
303 return;
304 }
305
306 while (go[MASTER])
307 cpu_relax();
308
309 spin_lock_irqsave(&itc_sync_lock, flags);
310 {
311 for (i = 0; i < NUM_ROUNDS; ++i) {
312 delta = get_delta(&rt, &master_time_stamp);
313 if (delta == 0) {
314 done = 1;
315 bound = rt;
316 }
317
318 if (!done) {
319 if (i > 0) {
320 adjust_latency += -delta;
321 adj = -delta + adjust_latency/4;
322 } else
323 adj = -delta;
324
325 ia64_set_itc(ia64_get_itc() + adj);
326 }
327#if DEBUG_ITC_SYNC
328 t[i].rt = rt;
329 t[i].master = master_time_stamp;
330 t[i].diff = delta;
331 t[i].lat = adjust_latency/4;
332#endif
333 }
334 }
335 spin_unlock_irqrestore(&itc_sync_lock, flags);
336
337#if DEBUG_ITC_SYNC
338 for (i = 0; i < NUM_ROUNDS; ++i)
339 printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
340 t[i].rt, t[i].master, t[i].diff, t[i].lat);
341#endif
342
343 printk(KERN_INFO "CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, "
344 "maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt);
345}
346
347
348
349
350static inline void smp_setup_percpu_timer(void)
351{
352}
353
354static void
355smp_callin (void)
356{
357 int cpuid, phys_id, itc_master;
358 struct cpuinfo_ia64 *last_cpuinfo, *this_cpuinfo;
359 extern void ia64_init_itm(void);
360 extern volatile int time_keeper_id;
361
362#ifdef CONFIG_PERFMON
363 extern void pfm_init_percpu(void);
364#endif
365
366 cpuid = smp_processor_id();
367 phys_id = hard_smp_processor_id();
368 itc_master = time_keeper_id;
369
370 if (cpu_online(cpuid)) {
371 printk(KERN_ERR "huh, phys CPU#0x%x, CPU#0x%x already present??\n",
372 phys_id, cpuid);
373 BUG();
374 }
375
376 fix_b0_for_bsp();
377
378
379
380
381 set_numa_node(cpu_to_node_map[cpuid]);
382 set_numa_mem(local_memory_node(cpu_to_node_map[cpuid]));
383
384 spin_lock(&vector_lock);
385
386 __setup_vector_irq(cpuid);
387 notify_cpu_starting(cpuid);
388 set_cpu_online(cpuid, true);
389 per_cpu(cpu_state, cpuid) = CPU_ONLINE;
390 spin_unlock(&vector_lock);
391
392 smp_setup_percpu_timer();
393
394 ia64_mca_cmc_vector_setup();
395
396#ifdef CONFIG_PERFMON
397 pfm_init_percpu();
398#endif
399
400 local_irq_enable();
401
402 if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) {
403
404
405
406
407
408
409 Dprintk("Going to syncup ITC with ITC Master.\n");
410 ia64_sync_itc(itc_master);
411 }
412
413
414
415
416 ia64_init_itm();
417
418
419
420
421
422 last_cpuinfo = cpu_data(cpuid - 1);
423 this_cpuinfo = local_cpu_data;
424 if (last_cpuinfo->itc_freq != this_cpuinfo->itc_freq ||
425 last_cpuinfo->proc_freq != this_cpuinfo->proc_freq ||
426 last_cpuinfo->features != this_cpuinfo->features ||
427 last_cpuinfo->revision != this_cpuinfo->revision ||
428 last_cpuinfo->family != this_cpuinfo->family ||
429 last_cpuinfo->archrev != this_cpuinfo->archrev ||
430 last_cpuinfo->model != this_cpuinfo->model)
431 calibrate_delay();
432 local_cpu_data->loops_per_jiffy = loops_per_jiffy;
433
434
435
436
437 cpumask_set_cpu(cpuid, &cpu_callin_map);
438 Dprintk("Stack on CPU %d at about %p\n",cpuid, &cpuid);
439}
440
441
442
443
444
445int
446start_secondary (void *unused)
447{
448
449 ia64_set_kr(IA64_KR_IO_BASE, __pa(ia64_iobase));
450#ifndef CONFIG_PRINTK_TIME
451 Dprintk("start_secondary: starting CPU 0x%x\n", hard_smp_processor_id());
452#endif
453 efi_map_pal_code();
454 cpu_init();
455 preempt_disable();
456 smp_callin();
457
458 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
459 return 0;
460}
461
462static int
463do_boot_cpu (int sapicid, int cpu, struct task_struct *idle)
464{
465 int timeout;
466
467 task_for_booting_cpu = idle;
468 Dprintk("Sending wakeup vector %lu to AP 0x%x/0x%x.\n", ap_wakeup_vector, cpu, sapicid);
469
470 set_brendez_area(cpu);
471 platform_send_ipi(cpu, ap_wakeup_vector, IA64_IPI_DM_INT, 0);
472
473
474
475
476 Dprintk("Waiting on callin_map ...");
477 for (timeout = 0; timeout < 100000; timeout++) {
478 if (cpumask_test_cpu(cpu, &cpu_callin_map))
479 break;
480 barrier();
481 udelay(100);
482 }
483 Dprintk("\n");
484
485 if (!cpumask_test_cpu(cpu, &cpu_callin_map)) {
486 printk(KERN_ERR "Processor 0x%x/0x%x is stuck.\n", cpu, sapicid);
487 ia64_cpu_to_sapicid[cpu] = -1;
488 set_cpu_online(cpu, false);
489 return -EINVAL;
490 }
491 return 0;
492}
493
494static int __init
495decay (char *str)
496{
497 int ticks;
498 get_option (&str, &ticks);
499 return 1;
500}
501
502__setup("decay=", decay);
503
504
505
506
507void __init
508smp_build_cpu_map (void)
509{
510 int sapicid, cpu, i;
511 int boot_cpu_id = hard_smp_processor_id();
512
513 for (cpu = 0; cpu < NR_CPUS; cpu++) {
514 ia64_cpu_to_sapicid[cpu] = -1;
515 }
516
517 ia64_cpu_to_sapicid[0] = boot_cpu_id;
518 init_cpu_present(cpumask_of(0));
519 set_cpu_possible(0, true);
520 for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
521 sapicid = smp_boot_data.cpu_phys_id[i];
522 if (sapicid == boot_cpu_id)
523 continue;
524 set_cpu_present(cpu, true);
525 set_cpu_possible(cpu, true);
526 ia64_cpu_to_sapicid[cpu] = sapicid;
527 cpu++;
528 }
529}
530
531
532
533
534void __init
535smp_prepare_cpus (unsigned int max_cpus)
536{
537 int boot_cpu_id = hard_smp_processor_id();
538
539
540
541
542
543 smp_setup_percpu_timer();
544
545 cpumask_set_cpu(0, &cpu_callin_map);
546
547 local_cpu_data->loops_per_jiffy = loops_per_jiffy;
548 ia64_cpu_to_sapicid[0] = boot_cpu_id;
549
550 printk(KERN_INFO "Boot processor id 0x%x/0x%x\n", 0, boot_cpu_id);
551
552 current_thread_info()->cpu = 0;
553
554
555
556
557 if (!max_cpus) {
558 printk(KERN_INFO "SMP mode deactivated.\n");
559 init_cpu_online(cpumask_of(0));
560 init_cpu_present(cpumask_of(0));
561 init_cpu_possible(cpumask_of(0));
562 return;
563 }
564}
565
566void smp_prepare_boot_cpu(void)
567{
568 set_cpu_online(smp_processor_id(), true);
569 cpumask_set_cpu(smp_processor_id(), &cpu_callin_map);
570 set_numa_node(cpu_to_node_map[smp_processor_id()]);
571 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
572}
573
574#ifdef CONFIG_HOTPLUG_CPU
575static inline void
576clear_cpu_sibling_map(int cpu)
577{
578 int i;
579
580 for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu))
581 cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i));
582 for_each_cpu(i, &cpu_core_map[cpu])
583 cpumask_clear_cpu(cpu, &cpu_core_map[i]);
584
585 per_cpu(cpu_sibling_map, cpu) = cpu_core_map[cpu] = CPU_MASK_NONE;
586}
587
588static void
589remove_siblinginfo(int cpu)
590{
591 int last = 0;
592
593 if (cpu_data(cpu)->threads_per_core == 1 &&
594 cpu_data(cpu)->cores_per_socket == 1) {
595 cpumask_clear_cpu(cpu, &cpu_core_map[cpu]);
596 cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, cpu));
597 return;
598 }
599
600 last = (cpumask_weight(&cpu_core_map[cpu]) == 1 ? 1 : 0);
601
602
603 clear_cpu_sibling_map(cpu);
604}
605
606extern void fixup_irqs(void);
607
608int migrate_platform_irqs(unsigned int cpu)
609{
610 int new_cpei_cpu;
611 struct irq_data *data = NULL;
612 const struct cpumask *mask;
613 int retval = 0;
614
615
616
617
618 if (cpe_vector > 0 && is_cpu_cpei_target(cpu)) {
619 printk ("CPU (%d) is CPEI Target\n", cpu);
620 if (can_cpei_retarget()) {
621
622
623
624 new_cpei_cpu = cpumask_any(cpu_online_mask);
625 mask = cpumask_of(new_cpei_cpu);
626 set_cpei_target_cpu(new_cpei_cpu);
627 data = irq_get_irq_data(ia64_cpe_irq);
628
629
630
631
632
633 if (data && data->chip) {
634 data->chip->irq_disable(data);
635 data->chip->irq_set_affinity(data, mask, false);
636 data->chip->irq_enable(data);
637 printk ("Re-targeting CPEI to cpu %d\n", new_cpei_cpu);
638 }
639 }
640 if (!data) {
641 printk ("Unable to retarget CPEI, offline cpu [%d] failed\n", cpu);
642 retval = -EBUSY;
643 }
644 }
645 return retval;
646}
647
648
649int __cpu_disable(void)
650{
651 int cpu = smp_processor_id();
652
653
654
655
656 if (cpu == 0 && !bsp_remove_ok) {
657 printk ("Your platform does not support removal of BSP\n");
658 return (-EBUSY);
659 }
660
661 if (ia64_platform_is("sn2")) {
662 if (!sn_cpu_disable_allowed(cpu))
663 return -EBUSY;
664 }
665
666 set_cpu_online(cpu, false);
667
668 if (migrate_platform_irqs(cpu)) {
669 set_cpu_online(cpu, true);
670 return -EBUSY;
671 }
672
673 remove_siblinginfo(cpu);
674 fixup_irqs();
675 local_flush_tlb_all();
676 cpumask_clear_cpu(cpu, &cpu_callin_map);
677 return 0;
678}
679
680void __cpu_die(unsigned int cpu)
681{
682 unsigned int i;
683
684 for (i = 0; i < 100; i++) {
685
686 if (per_cpu(cpu_state, cpu) == CPU_DEAD)
687 {
688 printk ("CPU %d is now offline\n", cpu);
689 return;
690 }
691 msleep(100);
692 }
693 printk(KERN_ERR "CPU %u didn't die...\n", cpu);
694}
695#endif
696
697void
698smp_cpus_done (unsigned int dummy)
699{
700 int cpu;
701 unsigned long bogosum = 0;
702
703
704
705
706
707 for_each_online_cpu(cpu) {
708 bogosum += cpu_data(cpu)->loops_per_jiffy;
709 }
710
711 printk(KERN_INFO "Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
712 (int)num_online_cpus(), bogosum/(500000/HZ), (bogosum/(5000/HZ))%100);
713}
714
715static inline void set_cpu_sibling_map(int cpu)
716{
717 int i;
718
719 for_each_online_cpu(i) {
720 if ((cpu_data(cpu)->socket_id == cpu_data(i)->socket_id)) {
721 cpumask_set_cpu(i, &cpu_core_map[cpu]);
722 cpumask_set_cpu(cpu, &cpu_core_map[i]);
723 if (cpu_data(cpu)->core_id == cpu_data(i)->core_id) {
724 cpumask_set_cpu(i,
725 &per_cpu(cpu_sibling_map, cpu));
726 cpumask_set_cpu(cpu,
727 &per_cpu(cpu_sibling_map, i));
728 }
729 }
730 }
731}
732
733int
734__cpu_up(unsigned int cpu, struct task_struct *tidle)
735{
736 int ret;
737 int sapicid;
738
739 sapicid = ia64_cpu_to_sapicid[cpu];
740 if (sapicid == -1)
741 return -EINVAL;
742
743
744
745
746
747 if (cpumask_test_cpu(cpu, &cpu_callin_map))
748 return -EINVAL;
749
750 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
751
752 ret = do_boot_cpu(sapicid, cpu, tidle);
753 if (ret < 0)
754 return ret;
755
756 if (cpu_data(cpu)->threads_per_core == 1 &&
757 cpu_data(cpu)->cores_per_socket == 1) {
758 cpumask_set_cpu(cpu, &per_cpu(cpu_sibling_map, cpu));
759 cpumask_set_cpu(cpu, &cpu_core_map[cpu]);
760 return 0;
761 }
762
763 set_cpu_sibling_map(cpu);
764
765 return 0;
766}
767
768
769
770
771
772
773
774void __init
775init_smp_config(void)
776{
777 struct fptr {
778 unsigned long fp;
779 unsigned long gp;
780 } *ap_startup;
781 long sal_ret;
782
783
784 ap_startup = (struct fptr *) start_ap;
785 sal_ret = ia64_sal_set_vectors(SAL_VECTOR_OS_BOOT_RENDEZ,
786 ia64_tpa(ap_startup->fp), ia64_tpa(ap_startup->gp), 0, 0, 0, 0);
787 if (sal_ret < 0)
788 printk(KERN_ERR "SMP: Can't set SAL AP Boot Rendezvous: %s\n",
789 ia64_sal_strerror(sal_ret));
790}
791
792
793
794
795
796void identify_siblings(struct cpuinfo_ia64 *c)
797{
798 long status;
799 u16 pltid;
800 pal_logical_to_physical_t info;
801
802 status = ia64_pal_logical_to_phys(-1, &info);
803 if (status != PAL_STATUS_SUCCESS) {
804 if (status != PAL_STATUS_UNIMPLEMENTED) {
805 printk(KERN_ERR
806 "ia64_pal_logical_to_phys failed with %ld\n",
807 status);
808 return;
809 }
810
811 info.overview_ppid = 0;
812 info.overview_cpp = 1;
813 info.overview_tpc = 1;
814 }
815
816 status = ia64_sal_physical_id_info(&pltid);
817 if (status != PAL_STATUS_SUCCESS) {
818 if (status != PAL_STATUS_UNIMPLEMENTED)
819 printk(KERN_ERR
820 "ia64_sal_pltid failed with %ld\n",
821 status);
822 return;
823 }
824
825 c->socket_id = (pltid << 8) | info.overview_ppid;
826
827 if (info.overview_cpp == 1 && info.overview_tpc == 1)
828 return;
829
830 c->cores_per_socket = info.overview_cpp;
831 c->threads_per_core = info.overview_tpc;
832 c->num_log = info.overview_num_log;
833
834 c->core_id = info.log1_cid;
835 c->thread_id = info.log1_tid;
836}
837
838
839
840
841
842
843
844int is_multithreading_enabled(void)
845{
846 int i, j;
847
848 for_each_present_cpu(i) {
849 for_each_present_cpu(j) {
850 if (j == i)
851 continue;
852 if ((cpu_data(j)->socket_id == cpu_data(i)->socket_id)) {
853 if (cpu_data(j)->core_id == cpu_data(i)->core_id)
854 return 1;
855 }
856 }
857 }
858 return 0;
859}
860EXPORT_SYMBOL_GPL(is_multithreading_enabled);
861