1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#undef DEBUG
19
20#include <linux/kernel.h>
21#include <linux/export.h>
22#include <linux/sched.h>
23#include <linux/smp.h>
24#include <linux/interrupt.h>
25#include <linux/delay.h>
26#include <linux/init.h>
27#include <linux/spinlock.h>
28#include <linux/cache.h>
29#include <linux/err.h>
30#include <linux/device.h>
31#include <linux/cpu.h>
32#include <linux/notifier.h>
33#include <linux/topology.h>
34
35#include <asm/ptrace.h>
36#include <linux/atomic.h>
37#include <asm/irq.h>
38#include <asm/hw_irq.h>
39#include <asm/kvm_ppc.h>
40#include <asm/page.h>
41#include <asm/pgtable.h>
42#include <asm/prom.h>
43#include <asm/smp.h>
44#include <asm/time.h>
45#include <asm/machdep.h>
46#include <asm/cputhreads.h>
47#include <asm/cputable.h>
48#include <asm/mpic.h>
49#include <asm/vdso_datapage.h>
50#ifdef CONFIG_PPC64
51#include <asm/paca.h>
52#endif
53#include <asm/vdso.h>
54#include <asm/debug.h>
55#include <asm/kexec.h>
56
57#ifdef DEBUG
58#include <asm/udbg.h>
59#define DBG(fmt...) udbg_printf(fmt)
60#else
61#define DBG(fmt...)
62#endif
63
64#ifdef CONFIG_HOTPLUG_CPU
65
66static DEFINE_PER_CPU(int, cpu_state) = { 0 };
67#endif
68
69struct thread_info *secondary_ti;
70
71DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
72DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
73
74EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
75EXPORT_PER_CPU_SYMBOL(cpu_core_map);
76
77
78struct smp_ops_t *smp_ops;
79
80
81volatile unsigned int cpu_callin_map[NR_CPUS];
82
83int smt_enabled_at_boot = 1;
84
85static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL;
86
87
88
89
90
91
92int smp_generic_cpu_bootable(unsigned int nr)
93{
94
95
96
97 if (system_state == SYSTEM_BOOTING && cpu_has_feature(CPU_FTR_SMT)) {
98 if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0)
99 return 0;
100 if (smt_enabled_at_boot
101 && cpu_thread_in_core(nr) >= smt_enabled_at_boot)
102 return 0;
103 }
104
105 return 1;
106}
107
108
109#ifdef CONFIG_PPC64
110int smp_generic_kick_cpu(int nr)
111{
112 BUG_ON(nr < 0 || nr >= NR_CPUS);
113
114
115
116
117
118
119 if (!paca[nr].cpu_start) {
120 paca[nr].cpu_start = 1;
121 smp_mb();
122 return 0;
123 }
124
125#ifdef CONFIG_HOTPLUG_CPU
126
127
128
129
130 generic_set_cpu_up(nr);
131 smp_wmb();
132 smp_send_reschedule(nr);
133#endif
134
135 return 0;
136}
137#endif
138
139static irqreturn_t call_function_action(int irq, void *data)
140{
141 generic_smp_call_function_interrupt();
142 return IRQ_HANDLED;
143}
144
145static irqreturn_t reschedule_action(int irq, void *data)
146{
147 scheduler_ipi();
148 return IRQ_HANDLED;
149}
150
151static irqreturn_t tick_broadcast_ipi_action(int irq, void *data)
152{
153 tick_broadcast_ipi_handler();
154 return IRQ_HANDLED;
155}
156
157static irqreturn_t debug_ipi_action(int irq, void *data)
158{
159 if (crash_ipi_function_ptr) {
160 crash_ipi_function_ptr(get_irq_regs());
161 return IRQ_HANDLED;
162 }
163
164#ifdef CONFIG_DEBUGGER
165 debugger_ipi(get_irq_regs());
166#endif
167
168 return IRQ_HANDLED;
169}
170
171static irq_handler_t smp_ipi_action[] = {
172 [PPC_MSG_CALL_FUNCTION] = call_function_action,
173 [PPC_MSG_RESCHEDULE] = reschedule_action,
174 [PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,
175 [PPC_MSG_DEBUGGER_BREAK] = debug_ipi_action,
176};
177
178const char *smp_ipi_name[] = {
179 [PPC_MSG_CALL_FUNCTION] = "ipi call function",
180 [PPC_MSG_RESCHEDULE] = "ipi reschedule",
181 [PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
182 [PPC_MSG_DEBUGGER_BREAK] = "ipi debugger",
183};
184
185
186int smp_request_message_ipi(int virq, int msg)
187{
188 int err;
189
190 if (msg < 0 || msg > PPC_MSG_DEBUGGER_BREAK) {
191 return -EINVAL;
192 }
193#if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC)
194 if (msg == PPC_MSG_DEBUGGER_BREAK) {
195 return 1;
196 }
197#endif
198 err = request_irq(virq, smp_ipi_action[msg],
199 IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND,
200 smp_ipi_name[msg], NULL);
201 WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n",
202 virq, smp_ipi_name[msg], err);
203
204 return err;
205}
206
207#ifdef CONFIG_PPC_SMP_MUXED_IPI
208struct cpu_messages {
209 long messages;
210 unsigned long data;
211};
212static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message);
213
214void smp_muxed_ipi_set_data(int cpu, unsigned long data)
215{
216 struct cpu_messages *info = &per_cpu(ipi_message, cpu);
217
218 info->data = data;
219}
220
221void smp_muxed_ipi_set_message(int cpu, int msg)
222{
223 struct cpu_messages *info = &per_cpu(ipi_message, cpu);
224 char *message = (char *)&info->messages;
225
226
227
228
229 smp_mb();
230 message[msg] = 1;
231}
232
233void smp_muxed_ipi_message_pass(int cpu, int msg)
234{
235 struct cpu_messages *info = &per_cpu(ipi_message, cpu);
236
237 smp_muxed_ipi_set_message(cpu, msg);
238
239
240
241
242 smp_ops->cause_ipi(cpu, info->data);
243}
244
245#ifdef __BIG_ENDIAN__
246#define IPI_MESSAGE(A) (1uL << ((BITS_PER_LONG - 8) - 8 * (A)))
247#else
248#define IPI_MESSAGE(A) (1uL << (8 * (A)))
249#endif
250
251irqreturn_t smp_ipi_demux(void)
252{
253 struct cpu_messages *info = this_cpu_ptr(&ipi_message);
254 unsigned long all;
255
256 mb();
257
258 do {
259 all = xchg(&info->messages, 0);
260#if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
261
262
263
264
265
266
267
268 if (all & IPI_MESSAGE(PPC_MSG_RM_HOST_ACTION))
269 kvmppc_xics_ipi_action();
270#endif
271 if (all & IPI_MESSAGE(PPC_MSG_CALL_FUNCTION))
272 generic_smp_call_function_interrupt();
273 if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE))
274 scheduler_ipi();
275 if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))
276 tick_broadcast_ipi_handler();
277 if (all & IPI_MESSAGE(PPC_MSG_DEBUGGER_BREAK))
278 debug_ipi_action(0, NULL);
279 } while (info->messages);
280
281 return IRQ_HANDLED;
282}
283#endif
284
285static inline void do_message_pass(int cpu, int msg)
286{
287 if (smp_ops->message_pass)
288 smp_ops->message_pass(cpu, msg);
289#ifdef CONFIG_PPC_SMP_MUXED_IPI
290 else
291 smp_muxed_ipi_message_pass(cpu, msg);
292#endif
293}
294
295void smp_send_reschedule(int cpu)
296{
297 if (likely(smp_ops))
298 do_message_pass(cpu, PPC_MSG_RESCHEDULE);
299}
300EXPORT_SYMBOL_GPL(smp_send_reschedule);
301
302void arch_send_call_function_single_ipi(int cpu)
303{
304 do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
305}
306
307void arch_send_call_function_ipi_mask(const struct cpumask *mask)
308{
309 unsigned int cpu;
310
311 for_each_cpu(cpu, mask)
312 do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
313}
314
315#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
316void tick_broadcast(const struct cpumask *mask)
317{
318 unsigned int cpu;
319
320 for_each_cpu(cpu, mask)
321 do_message_pass(cpu, PPC_MSG_TICK_BROADCAST);
322}
323#endif
324
325#if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC)
326void smp_send_debugger_break(void)
327{
328 int cpu;
329 int me = raw_smp_processor_id();
330
331 if (unlikely(!smp_ops))
332 return;
333
334 for_each_online_cpu(cpu)
335 if (cpu != me)
336 do_message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
337}
338#endif
339
340#ifdef CONFIG_KEXEC
341void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
342{
343 crash_ipi_function_ptr = crash_ipi_callback;
344 if (crash_ipi_callback) {
345 mb();
346 smp_send_debugger_break();
347 }
348}
349#endif
350
351static void stop_this_cpu(void *dummy)
352{
353
354 set_cpu_online(smp_processor_id(), false);
355
356 local_irq_disable();
357 while (1)
358 ;
359}
360
361void smp_send_stop(void)
362{
363 smp_call_function(stop_this_cpu, NULL, 0);
364}
365
366struct thread_info *current_set[NR_CPUS];
367
368static void smp_store_cpu_info(int id)
369{
370 per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR);
371#ifdef CONFIG_PPC_FSL_BOOK3E
372 per_cpu(next_tlbcam_idx, id)
373 = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
374#endif
375}
376
377void __init smp_prepare_cpus(unsigned int max_cpus)
378{
379 unsigned int cpu;
380
381 DBG("smp_prepare_cpus\n");
382
383
384
385
386
387 BUG_ON(boot_cpuid != smp_processor_id());
388
389
390 smp_store_cpu_info(boot_cpuid);
391 cpu_callin_map[boot_cpuid] = 1;
392
393 for_each_possible_cpu(cpu) {
394 zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu),
395 GFP_KERNEL, cpu_to_node(cpu));
396 zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu),
397 GFP_KERNEL, cpu_to_node(cpu));
398
399
400
401 if (cpu_present(cpu)) {
402 set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]);
403 set_cpu_numa_mem(cpu,
404 local_memory_node(numa_cpu_lookup_table[cpu]));
405 }
406 }
407
408 cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
409 cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
410
411 if (smp_ops && smp_ops->probe)
412 smp_ops->probe();
413}
414
415void smp_prepare_boot_cpu(void)
416{
417 BUG_ON(smp_processor_id() != boot_cpuid);
418#ifdef CONFIG_PPC64
419 paca[boot_cpuid].__current = current;
420#endif
421 set_numa_node(numa_cpu_lookup_table[boot_cpuid]);
422 current_set[boot_cpuid] = task_thread_info(current);
423}
424
425#ifdef CONFIG_HOTPLUG_CPU
426
427int generic_cpu_disable(void)
428{
429 unsigned int cpu = smp_processor_id();
430
431 if (cpu == boot_cpuid)
432 return -EBUSY;
433
434 set_cpu_online(cpu, false);
435#ifdef CONFIG_PPC64
436 vdso_data->processorCount--;
437#endif
438 migrate_irqs();
439 return 0;
440}
441
442void generic_cpu_die(unsigned int cpu)
443{
444 int i;
445
446 for (i = 0; i < 100; i++) {
447 smp_rmb();
448 if (is_cpu_dead(cpu))
449 return;
450 msleep(100);
451 }
452 printk(KERN_ERR "CPU%d didn't die...\n", cpu);
453}
454
455void generic_set_cpu_dead(unsigned int cpu)
456{
457 per_cpu(cpu_state, cpu) = CPU_DEAD;
458}
459
460
461
462
463
464
465void generic_set_cpu_up(unsigned int cpu)
466{
467 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
468}
469
470int generic_check_cpu_restart(unsigned int cpu)
471{
472 return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE;
473}
474
475int is_cpu_dead(unsigned int cpu)
476{
477 return per_cpu(cpu_state, cpu) == CPU_DEAD;
478}
479
480static bool secondaries_inhibited(void)
481{
482 return kvm_hv_mode_active();
483}
484
485#else
486
487#define secondaries_inhibited() 0
488
489#endif
490
491static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
492{
493 struct thread_info *ti = task_thread_info(idle);
494
495#ifdef CONFIG_PPC64
496 paca[cpu].__current = idle;
497 paca[cpu].kstack = (unsigned long)ti + THREAD_SIZE - STACK_FRAME_OVERHEAD;
498#endif
499 ti->cpu = cpu;
500 secondary_ti = current_set[cpu] = ti;
501}
502
503int __cpu_up(unsigned int cpu, struct task_struct *tidle)
504{
505 int rc, c;
506
507
508
509
510 if (threads_per_core > 1 && secondaries_inhibited() &&
511 cpu_thread_in_subcore(cpu))
512 return -EBUSY;
513
514 if (smp_ops == NULL ||
515 (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
516 return -EINVAL;
517
518 cpu_idle_thread_init(cpu, tidle);
519
520
521
522
523 cpu_callin_map[cpu] = 0;
524
525
526
527
528
529 smp_mb();
530
531
532 DBG("smp: kicking cpu %d\n", cpu);
533 rc = smp_ops->kick_cpu(cpu);
534 if (rc) {
535 pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc);
536 return rc;
537 }
538
539
540
541
542
543
544 if (system_state < SYSTEM_RUNNING)
545 for (c = 50000; c && !cpu_callin_map[cpu]; c--)
546 udelay(100);
547#ifdef CONFIG_HOTPLUG_CPU
548 else
549
550
551
552
553 for (c = 5000; c && !cpu_callin_map[cpu]; c--)
554 msleep(1);
555#endif
556
557 if (!cpu_callin_map[cpu]) {
558 printk(KERN_ERR "Processor %u is stuck.\n", cpu);
559 return -ENOENT;
560 }
561
562 DBG("Processor %u found.\n", cpu);
563
564 if (smp_ops->give_timebase)
565 smp_ops->give_timebase();
566
567
568 while (!cpu_online(cpu) || !cpu_active(cpu))
569 cpu_relax();
570
571 return 0;
572}
573
574
575
576
577int cpu_to_core_id(int cpu)
578{
579 struct device_node *np;
580 const __be32 *reg;
581 int id = -1;
582
583 np = of_get_cpu_node(cpu, NULL);
584 if (!np)
585 goto out;
586
587 reg = of_get_property(np, "reg", NULL);
588 if (!reg)
589 goto out;
590
591 id = be32_to_cpup(reg);
592out:
593 of_node_put(np);
594 return id;
595}
596
597
598int cpu_core_index_of_thread(int cpu)
599{
600 return cpu >> threads_shift;
601}
602EXPORT_SYMBOL_GPL(cpu_core_index_of_thread);
603
604int cpu_first_thread_of_core(int core)
605{
606 return core << threads_shift;
607}
608EXPORT_SYMBOL_GPL(cpu_first_thread_of_core);
609
610static void traverse_siblings_chip_id(int cpu, bool add, int chipid)
611{
612 const struct cpumask *mask;
613 struct device_node *np;
614 int i, plen;
615 const __be32 *prop;
616
617 mask = add ? cpu_online_mask : cpu_present_mask;
618 for_each_cpu(i, mask) {
619 np = of_get_cpu_node(i, NULL);
620 if (!np)
621 continue;
622 prop = of_get_property(np, "ibm,chip-id", &plen);
623 if (prop && plen == sizeof(int) &&
624 of_read_number(prop, 1) == chipid) {
625 if (add) {
626 cpumask_set_cpu(cpu, cpu_core_mask(i));
627 cpumask_set_cpu(i, cpu_core_mask(cpu));
628 } else {
629 cpumask_clear_cpu(cpu, cpu_core_mask(i));
630 cpumask_clear_cpu(i, cpu_core_mask(cpu));
631 }
632 }
633 of_node_put(np);
634 }
635}
636
637
638
639
640static struct device_node *cpu_to_l2cache(int cpu)
641{
642 struct device_node *np;
643 struct device_node *cache;
644
645 if (!cpu_present(cpu))
646 return NULL;
647
648 np = of_get_cpu_node(cpu, NULL);
649 if (np == NULL)
650 return NULL;
651
652 cache = of_find_next_cache_node(np);
653
654 of_node_put(np);
655
656 return cache;
657}
658
659static void traverse_core_siblings(int cpu, bool add)
660{
661 struct device_node *l2_cache, *np;
662 const struct cpumask *mask;
663 int i, chip, plen;
664 const __be32 *prop;
665
666
667 np = of_get_cpu_node(cpu, NULL);
668 if (np) {
669 chip = -1;
670 prop = of_get_property(np, "ibm,chip-id", &plen);
671 if (prop && plen == sizeof(int))
672 chip = of_read_number(prop, 1);
673 of_node_put(np);
674 if (chip >= 0) {
675 traverse_siblings_chip_id(cpu, add, chip);
676 return;
677 }
678 }
679
680 l2_cache = cpu_to_l2cache(cpu);
681 mask = add ? cpu_online_mask : cpu_present_mask;
682 for_each_cpu(i, mask) {
683 np = cpu_to_l2cache(i);
684 if (!np)
685 continue;
686 if (np == l2_cache) {
687 if (add) {
688 cpumask_set_cpu(cpu, cpu_core_mask(i));
689 cpumask_set_cpu(i, cpu_core_mask(cpu));
690 } else {
691 cpumask_clear_cpu(cpu, cpu_core_mask(i));
692 cpumask_clear_cpu(i, cpu_core_mask(cpu));
693 }
694 }
695 of_node_put(np);
696 }
697 of_node_put(l2_cache);
698}
699
700
701void start_secondary(void *unused)
702{
703 unsigned int cpu = smp_processor_id();
704 int i, base;
705
706 atomic_inc(&init_mm.mm_count);
707 current->active_mm = &init_mm;
708
709 smp_store_cpu_info(cpu);
710 set_dec(tb_ticks_per_jiffy);
711 preempt_disable();
712 cpu_callin_map[cpu] = 1;
713
714 if (smp_ops->setup_cpu)
715 smp_ops->setup_cpu(cpu);
716 if (smp_ops->take_timebase)
717 smp_ops->take_timebase();
718
719 secondary_cpu_time_init();
720
721#ifdef CONFIG_PPC64
722 if (system_state == SYSTEM_RUNNING)
723 vdso_data->processorCount++;
724
725 vdso_getcpu_init();
726#endif
727
728 base = cpu_first_thread_sibling(cpu);
729 for (i = 0; i < threads_per_core; i++) {
730 if (cpu_is_offline(base + i) && (cpu != base + i))
731 continue;
732 cpumask_set_cpu(cpu, cpu_sibling_mask(base + i));
733 cpumask_set_cpu(base + i, cpu_sibling_mask(cpu));
734
735
736
737
738
739 cpumask_set_cpu(cpu, cpu_core_mask(base + i));
740 cpumask_set_cpu(base + i, cpu_core_mask(cpu));
741 }
742 traverse_core_siblings(cpu, true);
743
744 set_numa_node(numa_cpu_lookup_table[cpu]);
745 set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
746
747 smp_wmb();
748 notify_cpu_starting(cpu);
749 set_cpu_online(cpu, true);
750
751 local_irq_enable();
752
753 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
754
755 BUG();
756}
757
758int setup_profiling_timer(unsigned int multiplier)
759{
760 return 0;
761}
762
763#ifdef CONFIG_SCHED_SMT
764
765static int powerpc_smt_flags(void)
766{
767 int flags = SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
768
769 if (cpu_has_feature(CPU_FTR_ASYM_SMT)) {
770 printk_once(KERN_INFO "Enabling Asymmetric SMT scheduling\n");
771 flags |= SD_ASYM_PACKING;
772 }
773 return flags;
774}
775#endif
776
777static struct sched_domain_topology_level powerpc_topology[] = {
778#ifdef CONFIG_SCHED_SMT
779 { cpu_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) },
780#endif
781 { cpu_cpu_mask, SD_INIT_NAME(DIE) },
782 { NULL, },
783};
784
785void __init smp_cpus_done(unsigned int max_cpus)
786{
787 cpumask_var_t old_mask;
788
789
790
791
792
793 alloc_cpumask_var(&old_mask, GFP_NOWAIT);
794 cpumask_copy(old_mask, tsk_cpus_allowed(current));
795 set_cpus_allowed_ptr(current, cpumask_of(boot_cpuid));
796
797 if (smp_ops && smp_ops->setup_cpu)
798 smp_ops->setup_cpu(boot_cpuid);
799
800 set_cpus_allowed_ptr(current, old_mask);
801
802 free_cpumask_var(old_mask);
803
804 if (smp_ops && smp_ops->bringup_done)
805 smp_ops->bringup_done();
806
807 dump_numa_cpu_topology();
808
809 set_sched_topology(powerpc_topology);
810
811}
812
813#ifdef CONFIG_HOTPLUG_CPU
814int __cpu_disable(void)
815{
816 int cpu = smp_processor_id();
817 int base, i;
818 int err;
819
820 if (!smp_ops->cpu_disable)
821 return -ENOSYS;
822
823 err = smp_ops->cpu_disable();
824 if (err)
825 return err;
826
827
828 base = cpu_first_thread_sibling(cpu);
829 for (i = 0; i < threads_per_core; i++) {
830 cpumask_clear_cpu(cpu, cpu_sibling_mask(base + i));
831 cpumask_clear_cpu(base + i, cpu_sibling_mask(cpu));
832 cpumask_clear_cpu(cpu, cpu_core_mask(base + i));
833 cpumask_clear_cpu(base + i, cpu_core_mask(cpu));
834 }
835 traverse_core_siblings(cpu, false);
836
837 return 0;
838}
839
840void __cpu_die(unsigned int cpu)
841{
842 if (smp_ops->cpu_die)
843 smp_ops->cpu_die(cpu);
844}
845
846void cpu_die(void)
847{
848 if (ppc_md.cpu_die)
849 ppc_md.cpu_die();
850
851
852 start_secondary_resume();
853}
854
855#endif
856