1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include <linux/module.h>
26#include <linux/acpi.h>
27#include <linux/bootmem.h>
28#include <linux/cpu.h>
29#include <linux/delay.h>
30#include <linux/init.h>
31#include <linux/interrupt.h>
32#include <linux/irq.h>
33#include <linux/kernel.h>
34#include <linux/kernel_stat.h>
35#include <linux/mm.h>
36#include <linux/notifier.h>
37#include <linux/smp.h>
38#include <linux/spinlock.h>
39#include <linux/efi.h>
40#include <linux/percpu.h>
41#include <linux/bitops.h>
42
43#include <linux/atomic.h>
44#include <asm/cache.h>
45#include <asm/current.h>
46#include <asm/delay.h>
47#include <asm/io.h>
48#include <asm/irq.h>
49#include <asm/machvec.h>
50#include <asm/mca.h>
51#include <asm/page.h>
52#include <asm/paravirt.h>
53#include <asm/pgalloc.h>
54#include <asm/pgtable.h>
55#include <asm/processor.h>
56#include <asm/ptrace.h>
57#include <asm/sal.h>
58#include <asm/tlbflush.h>
59#include <asm/unistd.h>
60#include <asm/sn/arch.h>
61
62#define SMP_DEBUG 0
63
64#if SMP_DEBUG
65#define Dprintk(x...) printk(x)
66#else
67#define Dprintk(x...)
68#endif
69
70#ifdef CONFIG_HOTPLUG_CPU
71#ifdef CONFIG_PERMIT_BSP_REMOVE
72#define bsp_remove_ok 1
73#else
74#define bsp_remove_ok 0
75#endif
76
77
78
79
80struct sal_to_os_boot sal_boot_rendez_state[NR_CPUS];
81
82
83
84
85
86struct sal_to_os_boot *sal_state_for_booting_cpu = &sal_boot_rendez_state[0];
87
88#define set_brendez_area(x) (sal_state_for_booting_cpu = &sal_boot_rendez_state[(x)]);
89
90#else
91#define set_brendez_area(x)
92#endif
93
94
95
96
97
98#define MASTER (0)
99#define SLAVE (SMP_CACHE_BYTES/8)
100
101#define NUM_ROUNDS 64
102#define NUM_ITERS 5
103
104static DEFINE_SPINLOCK(itc_sync_lock);
105static volatile unsigned long go[SLAVE + 1];
106
107#define DEBUG_ITC_SYNC 0
108
109extern void start_ap (void);
110extern unsigned long ia64_iobase;
111
112struct task_struct *task_for_booting_cpu;
113
114
115
116
117DEFINE_PER_CPU(int, cpu_state);
118
119cpumask_t cpu_core_map[NR_CPUS] __cacheline_aligned;
120EXPORT_SYMBOL(cpu_core_map);
121DEFINE_PER_CPU_SHARED_ALIGNED(cpumask_t, cpu_sibling_map);
122EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
123
124int smp_num_siblings = 1;
125
126
127volatile int ia64_cpu_to_sapicid[NR_CPUS];
128EXPORT_SYMBOL(ia64_cpu_to_sapicid);
129
130static volatile cpumask_t cpu_callin_map;
131
132struct smp_boot_data smp_boot_data __initdata;
133
134unsigned long ap_wakeup_vector = -1;
135
136char __initdata no_int_routing;
137
138unsigned char smp_int_redirect;
139
140#ifdef CONFIG_FORCE_CPEI_RETARGET
141#define CPEI_OVERRIDE_DEFAULT (1)
142#else
143#define CPEI_OVERRIDE_DEFAULT (0)
144#endif
145
146unsigned int force_cpei_retarget = CPEI_OVERRIDE_DEFAULT;
147
148static int __init
149cmdl_force_cpei(char *str)
150{
151 int value=0;
152
153 get_option (&str, &value);
154 force_cpei_retarget = value;
155
156 return 1;
157}
158
159__setup("force_cpei=", cmdl_force_cpei);
160
161static int __init
162nointroute (char *str)
163{
164 no_int_routing = 1;
165 printk ("no_int_routing on\n");
166 return 1;
167}
168
169__setup("nointroute", nointroute);
170
171static void fix_b0_for_bsp(void)
172{
173#ifdef CONFIG_HOTPLUG_CPU
174 int cpuid;
175 static int fix_bsp_b0 = 1;
176
177 cpuid = smp_processor_id();
178
179
180
181
182 if (!(fix_bsp_b0 && cpuid))
183 return;
184
185 sal_boot_rendez_state[0].br[0] = sal_boot_rendez_state[cpuid].br[0];
186 printk ("Fixed BSP b0 value from CPU %d\n", cpuid);
187
188 fix_bsp_b0 = 0;
189#endif
190}
191
192void
193sync_master (void *arg)
194{
195 unsigned long flags, i;
196
197 go[MASTER] = 0;
198
199 local_irq_save(flags);
200 {
201 for (i = 0; i < NUM_ROUNDS*NUM_ITERS; ++i) {
202 while (!go[MASTER])
203 cpu_relax();
204 go[MASTER] = 0;
205 go[SLAVE] = ia64_get_itc();
206 }
207 }
208 local_irq_restore(flags);
209}
210
211
212
213
214
215
216static inline long
217get_delta (long *rt, long *master)
218{
219 unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
220 unsigned long tcenter, t0, t1, tm;
221 long i;
222
223 for (i = 0; i < NUM_ITERS; ++i) {
224 t0 = ia64_get_itc();
225 go[MASTER] = 1;
226 while (!(tm = go[SLAVE]))
227 cpu_relax();
228 go[SLAVE] = 0;
229 t1 = ia64_get_itc();
230
231 if (t1 - t0 < best_t1 - best_t0)
232 best_t0 = t0, best_t1 = t1, best_tm = tm;
233 }
234
235 *rt = best_t1 - best_t0;
236 *master = best_tm - best_t0;
237
238
239 tcenter = (best_t0/2 + best_t1/2);
240 if (best_t0 % 2 + best_t1 % 2 == 2)
241 ++tcenter;
242 return tcenter - best_tm;
243}
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277void
278ia64_sync_itc (unsigned int master)
279{
280 long i, delta, adj, adjust_latency = 0, done = 0;
281 unsigned long flags, rt, master_time_stamp, bound;
282#if DEBUG_ITC_SYNC
283 struct {
284 long rt;
285 long master;
286 long diff;
287 long lat;
288 } t[NUM_ROUNDS];
289#endif
290
291
292
293
294
295
296
297 BUG_ON((ia64_get_itv() & (1 << 16)) == 0);
298
299 go[MASTER] = 1;
300
301 if (smp_call_function_single(master, sync_master, NULL, 0) < 0) {
302 printk(KERN_ERR "sync_itc: failed to get attention of CPU %u!\n", master);
303 return;
304 }
305
306 while (go[MASTER])
307 cpu_relax();
308
309 spin_lock_irqsave(&itc_sync_lock, flags);
310 {
311 for (i = 0; i < NUM_ROUNDS; ++i) {
312 delta = get_delta(&rt, &master_time_stamp);
313 if (delta == 0) {
314 done = 1;
315 bound = rt;
316 }
317
318 if (!done) {
319 if (i > 0) {
320 adjust_latency += -delta;
321 adj = -delta + adjust_latency/4;
322 } else
323 adj = -delta;
324
325 ia64_set_itc(ia64_get_itc() + adj);
326 }
327#if DEBUG_ITC_SYNC
328 t[i].rt = rt;
329 t[i].master = master_time_stamp;
330 t[i].diff = delta;
331 t[i].lat = adjust_latency/4;
332#endif
333 }
334 }
335 spin_unlock_irqrestore(&itc_sync_lock, flags);
336
337#if DEBUG_ITC_SYNC
338 for (i = 0; i < NUM_ROUNDS; ++i)
339 printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
340 t[i].rt, t[i].master, t[i].diff, t[i].lat);
341#endif
342
343 printk(KERN_INFO "CPU %d: synchronized ITC with CPU %u (last diff %ld cycles, "
344 "maxerr %lu cycles)\n", smp_processor_id(), master, delta, rt);
345}
346
347
348
349
350static inline void __devinit
351smp_setup_percpu_timer (void)
352{
353}
354
355static void __cpuinit
356smp_callin (void)
357{
358 int cpuid, phys_id, itc_master;
359 struct cpuinfo_ia64 *last_cpuinfo, *this_cpuinfo;
360 extern void ia64_init_itm(void);
361 extern volatile int time_keeper_id;
362
363#ifdef CONFIG_PERFMON
364 extern void pfm_init_percpu(void);
365#endif
366
367 cpuid = smp_processor_id();
368 phys_id = hard_smp_processor_id();
369 itc_master = time_keeper_id;
370
371 if (cpu_online(cpuid)) {
372 printk(KERN_ERR "huh, phys CPU#0x%x, CPU#0x%x already present??\n",
373 phys_id, cpuid);
374 BUG();
375 }
376
377 fix_b0_for_bsp();
378
379
380
381
382 set_numa_node(cpu_to_node_map[cpuid]);
383 set_numa_mem(local_memory_node(cpu_to_node_map[cpuid]));
384
385 spin_lock(&vector_lock);
386
387 __setup_vector_irq(cpuid);
388 notify_cpu_starting(cpuid);
389 set_cpu_online(cpuid, true);
390 per_cpu(cpu_state, cpuid) = CPU_ONLINE;
391 spin_unlock(&vector_lock);
392
393 smp_setup_percpu_timer();
394
395 ia64_mca_cmc_vector_setup();
396
397#ifdef CONFIG_PERFMON
398 pfm_init_percpu();
399#endif
400
401 local_irq_enable();
402
403 if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) {
404
405
406
407
408
409
410 Dprintk("Going to syncup ITC with ITC Master.\n");
411 ia64_sync_itc(itc_master);
412 }
413
414
415
416
417 ia64_init_itm();
418
419
420
421
422
423 last_cpuinfo = cpu_data(cpuid - 1);
424 this_cpuinfo = local_cpu_data;
425 if (last_cpuinfo->itc_freq != this_cpuinfo->itc_freq ||
426 last_cpuinfo->proc_freq != this_cpuinfo->proc_freq ||
427 last_cpuinfo->features != this_cpuinfo->features ||
428 last_cpuinfo->revision != this_cpuinfo->revision ||
429 last_cpuinfo->family != this_cpuinfo->family ||
430 last_cpuinfo->archrev != this_cpuinfo->archrev ||
431 last_cpuinfo->model != this_cpuinfo->model)
432 calibrate_delay();
433 local_cpu_data->loops_per_jiffy = loops_per_jiffy;
434
435
436
437
438 cpu_set(cpuid, cpu_callin_map);
439 Dprintk("Stack on CPU %d at about %p\n",cpuid, &cpuid);
440}
441
442
443
444
445
446int __cpuinit
447start_secondary (void *unused)
448{
449
450 ia64_set_kr(IA64_KR_IO_BASE, __pa(ia64_iobase));
451#ifndef CONFIG_PRINTK_TIME
452 Dprintk("start_secondary: starting CPU 0x%x\n", hard_smp_processor_id());
453#endif
454 efi_map_pal_code();
455 cpu_init();
456 preempt_disable();
457 smp_callin();
458
459 cpu_idle();
460 return 0;
461}
462
463struct pt_regs * __cpuinit idle_regs(struct pt_regs *regs)
464{
465 return NULL;
466}
467
468static int __cpuinit
469do_boot_cpu (int sapicid, int cpu, struct task_struct *idle)
470{
471 int timeout;
472
473 task_for_booting_cpu = idle;
474 Dprintk("Sending wakeup vector %lu to AP 0x%x/0x%x.\n", ap_wakeup_vector, cpu, sapicid);
475
476 set_brendez_area(cpu);
477 platform_send_ipi(cpu, ap_wakeup_vector, IA64_IPI_DM_INT, 0);
478
479
480
481
482 Dprintk("Waiting on callin_map ...");
483 for (timeout = 0; timeout < 100000; timeout++) {
484 if (cpu_isset(cpu, cpu_callin_map))
485 break;
486 udelay(100);
487 }
488 Dprintk("\n");
489
490 if (!cpu_isset(cpu, cpu_callin_map)) {
491 printk(KERN_ERR "Processor 0x%x/0x%x is stuck.\n", cpu, sapicid);
492 ia64_cpu_to_sapicid[cpu] = -1;
493 set_cpu_online(cpu, false);
494 return -EINVAL;
495 }
496 return 0;
497}
498
499static int __init
500decay (char *str)
501{
502 int ticks;
503 get_option (&str, &ticks);
504 return 1;
505}
506
507__setup("decay=", decay);
508
509
510
511
512void __init
513smp_build_cpu_map (void)
514{
515 int sapicid, cpu, i;
516 int boot_cpu_id = hard_smp_processor_id();
517
518 for (cpu = 0; cpu < NR_CPUS; cpu++) {
519 ia64_cpu_to_sapicid[cpu] = -1;
520 }
521
522 ia64_cpu_to_sapicid[0] = boot_cpu_id;
523 init_cpu_present(cpumask_of(0));
524 set_cpu_possible(0, true);
525 for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
526 sapicid = smp_boot_data.cpu_phys_id[i];
527 if (sapicid == boot_cpu_id)
528 continue;
529 set_cpu_present(cpu, true);
530 set_cpu_possible(cpu, true);
531 ia64_cpu_to_sapicid[cpu] = sapicid;
532 cpu++;
533 }
534}
535
536
537
538
539void __init
540smp_prepare_cpus (unsigned int max_cpus)
541{
542 int boot_cpu_id = hard_smp_processor_id();
543
544
545
546
547
548 smp_setup_percpu_timer();
549
550 cpu_set(0, cpu_callin_map);
551
552 local_cpu_data->loops_per_jiffy = loops_per_jiffy;
553 ia64_cpu_to_sapicid[0] = boot_cpu_id;
554
555 printk(KERN_INFO "Boot processor id 0x%x/0x%x\n", 0, boot_cpu_id);
556
557 current_thread_info()->cpu = 0;
558
559
560
561
562 if (!max_cpus) {
563 printk(KERN_INFO "SMP mode deactivated.\n");
564 init_cpu_online(cpumask_of(0));
565 init_cpu_present(cpumask_of(0));
566 init_cpu_possible(cpumask_of(0));
567 return;
568 }
569}
570
571void __devinit smp_prepare_boot_cpu(void)
572{
573 set_cpu_online(smp_processor_id(), true);
574 cpu_set(smp_processor_id(), cpu_callin_map);
575 set_numa_node(cpu_to_node_map[smp_processor_id()]);
576 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
577 paravirt_post_smp_prepare_boot_cpu();
578}
579
580#ifdef CONFIG_HOTPLUG_CPU
581static inline void
582clear_cpu_sibling_map(int cpu)
583{
584 int i;
585
586 for_each_cpu_mask(i, per_cpu(cpu_sibling_map, cpu))
587 cpu_clear(cpu, per_cpu(cpu_sibling_map, i));
588 for_each_cpu_mask(i, cpu_core_map[cpu])
589 cpu_clear(cpu, cpu_core_map[i]);
590
591 per_cpu(cpu_sibling_map, cpu) = cpu_core_map[cpu] = CPU_MASK_NONE;
592}
593
594static void
595remove_siblinginfo(int cpu)
596{
597 int last = 0;
598
599 if (cpu_data(cpu)->threads_per_core == 1 &&
600 cpu_data(cpu)->cores_per_socket == 1) {
601 cpu_clear(cpu, cpu_core_map[cpu]);
602 cpu_clear(cpu, per_cpu(cpu_sibling_map, cpu));
603 return;
604 }
605
606 last = (cpus_weight(cpu_core_map[cpu]) == 1 ? 1 : 0);
607
608
609 clear_cpu_sibling_map(cpu);
610}
611
612extern void fixup_irqs(void);
613
614int migrate_platform_irqs(unsigned int cpu)
615{
616 int new_cpei_cpu;
617 struct irq_data *data = NULL;
618 const struct cpumask *mask;
619 int retval = 0;
620
621
622
623
624 if (cpe_vector > 0 && is_cpu_cpei_target(cpu)) {
625 printk ("CPU (%d) is CPEI Target\n", cpu);
626 if (can_cpei_retarget()) {
627
628
629
630 new_cpei_cpu = cpumask_any(cpu_online_mask);
631 mask = cpumask_of(new_cpei_cpu);
632 set_cpei_target_cpu(new_cpei_cpu);
633 data = irq_get_irq_data(ia64_cpe_irq);
634
635
636
637
638
639 if (data && data->chip) {
640 data->chip->irq_disable(data);
641 data->chip->irq_set_affinity(data, mask, false);
642 data->chip->irq_enable(data);
643 printk ("Re-targeting CPEI to cpu %d\n", new_cpei_cpu);
644 }
645 }
646 if (!data) {
647 printk ("Unable to retarget CPEI, offline cpu [%d] failed\n", cpu);
648 retval = -EBUSY;
649 }
650 }
651 return retval;
652}
653
654
655int __cpu_disable(void)
656{
657 int cpu = smp_processor_id();
658
659
660
661
662 if (cpu == 0 && !bsp_remove_ok) {
663 printk ("Your platform does not support removal of BSP\n");
664 return (-EBUSY);
665 }
666
667 if (ia64_platform_is("sn2")) {
668 if (!sn_cpu_disable_allowed(cpu))
669 return -EBUSY;
670 }
671
672 set_cpu_online(cpu, false);
673
674 if (migrate_platform_irqs(cpu)) {
675 set_cpu_online(cpu, true);
676 return -EBUSY;
677 }
678
679 remove_siblinginfo(cpu);
680 fixup_irqs();
681 local_flush_tlb_all();
682 cpu_clear(cpu, cpu_callin_map);
683 return 0;
684}
685
686void __cpu_die(unsigned int cpu)
687{
688 unsigned int i;
689
690 for (i = 0; i < 100; i++) {
691
692 if (per_cpu(cpu_state, cpu) == CPU_DEAD)
693 {
694 printk ("CPU %d is now offline\n", cpu);
695 return;
696 }
697 msleep(100);
698 }
699 printk(KERN_ERR "CPU %u didn't die...\n", cpu);
700}
701#endif
702
703void
704smp_cpus_done (unsigned int dummy)
705{
706 int cpu;
707 unsigned long bogosum = 0;
708
709
710
711
712
713 for_each_online_cpu(cpu) {
714 bogosum += cpu_data(cpu)->loops_per_jiffy;
715 }
716
717 printk(KERN_INFO "Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
718 (int)num_online_cpus(), bogosum/(500000/HZ), (bogosum/(5000/HZ))%100);
719}
720
721static inline void __devinit
722set_cpu_sibling_map(int cpu)
723{
724 int i;
725
726 for_each_online_cpu(i) {
727 if ((cpu_data(cpu)->socket_id == cpu_data(i)->socket_id)) {
728 cpu_set(i, cpu_core_map[cpu]);
729 cpu_set(cpu, cpu_core_map[i]);
730 if (cpu_data(cpu)->core_id == cpu_data(i)->core_id) {
731 cpu_set(i, per_cpu(cpu_sibling_map, cpu));
732 cpu_set(cpu, per_cpu(cpu_sibling_map, i));
733 }
734 }
735 }
736}
737
738int __cpuinit
739__cpu_up(unsigned int cpu, struct task_struct *tidle)
740{
741 int ret;
742 int sapicid;
743
744 sapicid = ia64_cpu_to_sapicid[cpu];
745 if (sapicid == -1)
746 return -EINVAL;
747
748
749
750
751
752 if (cpu_isset(cpu, cpu_callin_map))
753 return -EINVAL;
754
755 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
756
757 ret = do_boot_cpu(sapicid, cpu, tidle);
758 if (ret < 0)
759 return ret;
760
761 if (cpu_data(cpu)->threads_per_core == 1 &&
762 cpu_data(cpu)->cores_per_socket == 1) {
763 cpu_set(cpu, per_cpu(cpu_sibling_map, cpu));
764 cpu_set(cpu, cpu_core_map[cpu]);
765 return 0;
766 }
767
768 set_cpu_sibling_map(cpu);
769
770 return 0;
771}
772
773
774
775
776
777
778
779void __init
780init_smp_config(void)
781{
782 struct fptr {
783 unsigned long fp;
784 unsigned long gp;
785 } *ap_startup;
786 long sal_ret;
787
788
789 ap_startup = (struct fptr *) start_ap;
790 sal_ret = ia64_sal_set_vectors(SAL_VECTOR_OS_BOOT_RENDEZ,
791 ia64_tpa(ap_startup->fp), ia64_tpa(ap_startup->gp), 0, 0, 0, 0);
792 if (sal_ret < 0)
793 printk(KERN_ERR "SMP: Can't set SAL AP Boot Rendezvous: %s\n",
794 ia64_sal_strerror(sal_ret));
795}
796
797
798
799
800
801void __devinit
802identify_siblings(struct cpuinfo_ia64 *c)
803{
804 long status;
805 u16 pltid;
806 pal_logical_to_physical_t info;
807
808 status = ia64_pal_logical_to_phys(-1, &info);
809 if (status != PAL_STATUS_SUCCESS) {
810 if (status != PAL_STATUS_UNIMPLEMENTED) {
811 printk(KERN_ERR
812 "ia64_pal_logical_to_phys failed with %ld\n",
813 status);
814 return;
815 }
816
817 info.overview_ppid = 0;
818 info.overview_cpp = 1;
819 info.overview_tpc = 1;
820 }
821
822 status = ia64_sal_physical_id_info(&pltid);
823 if (status != PAL_STATUS_SUCCESS) {
824 if (status != PAL_STATUS_UNIMPLEMENTED)
825 printk(KERN_ERR
826 "ia64_sal_pltid failed with %ld\n",
827 status);
828 return;
829 }
830
831 c->socket_id = (pltid << 8) | info.overview_ppid;
832
833 if (info.overview_cpp == 1 && info.overview_tpc == 1)
834 return;
835
836 c->cores_per_socket = info.overview_cpp;
837 c->threads_per_core = info.overview_tpc;
838 c->num_log = info.overview_num_log;
839
840 c->core_id = info.log1_cid;
841 c->thread_id = info.log1_tid;
842}
843
844
845
846
847
848
849
850int is_multithreading_enabled(void)
851{
852 int i, j;
853
854 for_each_present_cpu(i) {
855 for_each_present_cpu(j) {
856 if (j == i)
857 continue;
858 if ((cpu_data(j)->socket_id == cpu_data(i)->socket_id)) {
859 if (cpu_data(j)->core_id == cpu_data(i)->core_id)
860 return 1;
861 }
862 }
863 }
864 return 0;
865}
866EXPORT_SYMBOL_GPL(is_multithreading_enabled);
867