1
2
3
4
5
6
7
8
9
10
11
12
13
14#undef DEBUG
15
16#include <linux/kernel.h>
17#include <linux/export.h>
18#include <linux/sched/mm.h>
19#include <linux/sched/task_stack.h>
20#include <linux/sched/topology.h>
21#include <linux/smp.h>
22#include <linux/interrupt.h>
23#include <linux/delay.h>
24#include <linux/init.h>
25#include <linux/spinlock.h>
26#include <linux/cache.h>
27#include <linux/err.h>
28#include <linux/device.h>
29#include <linux/cpu.h>
30#include <linux/notifier.h>
31#include <linux/topology.h>
32#include <linux/profile.h>
33#include <linux/processor.h>
34#include <linux/random.h>
35#include <linux/stackprotector.h>
36#include <linux/pgtable.h>
37#include <linux/clockchips.h>
38
39#include <asm/ptrace.h>
40#include <linux/atomic.h>
41#include <asm/irq.h>
42#include <asm/hw_irq.h>
43#include <asm/kvm_ppc.h>
44#include <asm/dbell.h>
45#include <asm/page.h>
46#include <asm/prom.h>
47#include <asm/smp.h>
48#include <asm/time.h>
49#include <asm/machdep.h>
50#include <asm/cputhreads.h>
51#include <asm/cputable.h>
52#include <asm/mpic.h>
53#include <asm/vdso_datapage.h>
54#ifdef CONFIG_PPC64
55#include <asm/paca.h>
56#endif
57#include <asm/vdso.h>
58#include <asm/debug.h>
59#include <asm/kexec.h>
60#include <asm/asm-prototypes.h>
61#include <asm/cpu_has_feature.h>
62#include <asm/ftrace.h>
63#include <asm/kup.h>
64
65#ifdef DEBUG
66#include <asm/udbg.h>
67#define DBG(fmt...) udbg_printf(fmt)
68#else
69#define DBG(fmt...)
70#endif
71
72#ifdef CONFIG_HOTPLUG_CPU
73
74static DEFINE_PER_CPU(int, cpu_state) = { 0 };
75#endif
76
77struct task_struct *secondary_current;
78bool has_big_cores;
79bool coregroup_enabled;
80bool thread_group_shares_l2;
81
82DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
83DEFINE_PER_CPU(cpumask_var_t, cpu_smallcore_map);
84DEFINE_PER_CPU(cpumask_var_t, cpu_l2_cache_map);
85DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
86static DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map);
87
88EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
89EXPORT_PER_CPU_SYMBOL(cpu_l2_cache_map);
90EXPORT_PER_CPU_SYMBOL(cpu_core_map);
91EXPORT_SYMBOL_GPL(has_big_cores);
92
93enum {
94#ifdef CONFIG_SCHED_SMT
95 smt_idx,
96#endif
97 cache_idx,
98 mc_idx,
99 die_idx,
100};
101
102#define MAX_THREAD_LIST_SIZE 8
103#define THREAD_GROUP_SHARE_L1 1
104#define THREAD_GROUP_SHARE_L2 2
105struct thread_groups {
106 unsigned int property;
107 unsigned int nr_groups;
108 unsigned int threads_per_group;
109 unsigned int thread_list[MAX_THREAD_LIST_SIZE];
110};
111
112
113#define MAX_THREAD_GROUP_PROPERTIES 2
114
115struct thread_groups_list {
116 unsigned int nr_properties;
117 struct thread_groups property_tgs[MAX_THREAD_GROUP_PROPERTIES];
118};
119
120static struct thread_groups_list tgl[NR_CPUS] __initdata;
121
122
123
124
125static DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map);
126
127
128
129
130
131
132static DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map);
133
134
135struct smp_ops_t *smp_ops;
136
137
138volatile unsigned int cpu_callin_map[NR_CPUS];
139
140int smt_enabled_at_boot = 1;
141
142
143
144
145
146
147int smp_generic_cpu_bootable(unsigned int nr)
148{
149
150
151
152 if (system_state < SYSTEM_RUNNING && cpu_has_feature(CPU_FTR_SMT)) {
153 if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0)
154 return 0;
155 if (smt_enabled_at_boot
156 && cpu_thread_in_core(nr) >= smt_enabled_at_boot)
157 return 0;
158 }
159
160 return 1;
161}
162
163
164#ifdef CONFIG_PPC64
165int smp_generic_kick_cpu(int nr)
166{
167 if (nr < 0 || nr >= nr_cpu_ids)
168 return -EINVAL;
169
170
171
172
173
174
175 if (!paca_ptrs[nr]->cpu_start) {
176 paca_ptrs[nr]->cpu_start = 1;
177 smp_mb();
178 return 0;
179 }
180
181#ifdef CONFIG_HOTPLUG_CPU
182
183
184
185
186 generic_set_cpu_up(nr);
187 smp_wmb();
188 smp_send_reschedule(nr);
189#endif
190
191 return 0;
192}
193#endif
194
195static irqreturn_t call_function_action(int irq, void *data)
196{
197 generic_smp_call_function_interrupt();
198 return IRQ_HANDLED;
199}
200
201static irqreturn_t reschedule_action(int irq, void *data)
202{
203 scheduler_ipi();
204 return IRQ_HANDLED;
205}
206
207#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
208static irqreturn_t tick_broadcast_ipi_action(int irq, void *data)
209{
210 timer_broadcast_interrupt();
211 return IRQ_HANDLED;
212}
213#endif
214
215#ifdef CONFIG_NMI_IPI
216static irqreturn_t nmi_ipi_action(int irq, void *data)
217{
218 smp_handle_nmi_ipi(get_irq_regs());
219 return IRQ_HANDLED;
220}
221#endif
222
223static irq_handler_t smp_ipi_action[] = {
224 [PPC_MSG_CALL_FUNCTION] = call_function_action,
225 [PPC_MSG_RESCHEDULE] = reschedule_action,
226#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
227 [PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,
228#endif
229#ifdef CONFIG_NMI_IPI
230 [PPC_MSG_NMI_IPI] = nmi_ipi_action,
231#endif
232};
233
234
235
236
237
238
239const char *smp_ipi_name[] = {
240 [PPC_MSG_CALL_FUNCTION] = "ipi call function",
241 [PPC_MSG_RESCHEDULE] = "ipi reschedule",
242#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
243 [PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
244#endif
245#ifdef CONFIG_NMI_IPI
246 [PPC_MSG_NMI_IPI] = "nmi ipi",
247#endif
248};
249
250
251int smp_request_message_ipi(int virq, int msg)
252{
253 int err;
254
255 if (msg < 0 || msg > PPC_MSG_NMI_IPI)
256 return -EINVAL;
257#ifndef CONFIG_NMI_IPI
258 if (msg == PPC_MSG_NMI_IPI)
259 return 1;
260#endif
261
262 err = request_irq(virq, smp_ipi_action[msg],
263 IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND,
264 smp_ipi_name[msg], NULL);
265 WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n",
266 virq, smp_ipi_name[msg], err);
267
268 return err;
269}
270
271#ifdef CONFIG_PPC_SMP_MUXED_IPI
272struct cpu_messages {
273 long messages;
274};
275static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message);
276
277void smp_muxed_ipi_set_message(int cpu, int msg)
278{
279 struct cpu_messages *info = &per_cpu(ipi_message, cpu);
280 char *message = (char *)&info->messages;
281
282
283
284
285 smp_mb();
286 message[msg] = 1;
287}
288
289void smp_muxed_ipi_message_pass(int cpu, int msg)
290{
291 smp_muxed_ipi_set_message(cpu, msg);
292
293
294
295
296
297 smp_ops->cause_ipi(cpu);
298}
299
300#ifdef __BIG_ENDIAN__
301#define IPI_MESSAGE(A) (1uL << ((BITS_PER_LONG - 8) - 8 * (A)))
302#else
303#define IPI_MESSAGE(A) (1uL << (8 * (A)))
304#endif
305
306irqreturn_t smp_ipi_demux(void)
307{
308 mb();
309
310 return smp_ipi_demux_relaxed();
311}
312
313
314irqreturn_t smp_ipi_demux_relaxed(void)
315{
316 struct cpu_messages *info;
317 unsigned long all;
318
319 info = this_cpu_ptr(&ipi_message);
320 do {
321 all = xchg(&info->messages, 0);
322#if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
323
324
325
326
327
328
329
330 if (all & IPI_MESSAGE(PPC_MSG_RM_HOST_ACTION))
331 kvmppc_xics_ipi_action();
332#endif
333 if (all & IPI_MESSAGE(PPC_MSG_CALL_FUNCTION))
334 generic_smp_call_function_interrupt();
335 if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE))
336 scheduler_ipi();
337#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
338 if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))
339 timer_broadcast_interrupt();
340#endif
341#ifdef CONFIG_NMI_IPI
342 if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI))
343 nmi_ipi_action(0, NULL);
344#endif
345 } while (info->messages);
346
347 return IRQ_HANDLED;
348}
349#endif
350
351static inline void do_message_pass(int cpu, int msg)
352{
353 if (smp_ops->message_pass)
354 smp_ops->message_pass(cpu, msg);
355#ifdef CONFIG_PPC_SMP_MUXED_IPI
356 else
357 smp_muxed_ipi_message_pass(cpu, msg);
358#endif
359}
360
361void smp_send_reschedule(int cpu)
362{
363 if (likely(smp_ops))
364 do_message_pass(cpu, PPC_MSG_RESCHEDULE);
365}
366EXPORT_SYMBOL_GPL(smp_send_reschedule);
367
368void arch_send_call_function_single_ipi(int cpu)
369{
370 do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
371}
372
373void arch_send_call_function_ipi_mask(const struct cpumask *mask)
374{
375 unsigned int cpu;
376
377 for_each_cpu(cpu, mask)
378 do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
379}
380
381#ifdef CONFIG_NMI_IPI
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403static atomic_t __nmi_ipi_lock = ATOMIC_INIT(0);
404static struct cpumask nmi_ipi_pending_mask;
405static bool nmi_ipi_busy = false;
406static void (*nmi_ipi_function)(struct pt_regs *) = NULL;
407
408static void nmi_ipi_lock_start(unsigned long *flags)
409{
410 raw_local_irq_save(*flags);
411 hard_irq_disable();
412 while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {
413 raw_local_irq_restore(*flags);
414 spin_until_cond(atomic_read(&__nmi_ipi_lock) == 0);
415 raw_local_irq_save(*flags);
416 hard_irq_disable();
417 }
418}
419
420static void nmi_ipi_lock(void)
421{
422 while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)
423 spin_until_cond(atomic_read(&__nmi_ipi_lock) == 0);
424}
425
426static void nmi_ipi_unlock(void)
427{
428 smp_mb();
429 WARN_ON(atomic_read(&__nmi_ipi_lock) != 1);
430 atomic_set(&__nmi_ipi_lock, 0);
431}
432
433static void nmi_ipi_unlock_end(unsigned long *flags)
434{
435 nmi_ipi_unlock();
436 raw_local_irq_restore(*flags);
437}
438
439
440
441
442int smp_handle_nmi_ipi(struct pt_regs *regs)
443{
444 void (*fn)(struct pt_regs *) = NULL;
445 unsigned long flags;
446 int me = raw_smp_processor_id();
447 int ret = 0;
448
449
450
451
452
453
454 nmi_ipi_lock_start(&flags);
455 if (cpumask_test_cpu(me, &nmi_ipi_pending_mask)) {
456 cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
457 fn = READ_ONCE(nmi_ipi_function);
458 WARN_ON_ONCE(!fn);
459 ret = 1;
460 }
461 nmi_ipi_unlock_end(&flags);
462
463 if (fn)
464 fn(regs);
465
466 return ret;
467}
468
469static void do_smp_send_nmi_ipi(int cpu, bool safe)
470{
471 if (!safe && smp_ops->cause_nmi_ipi && smp_ops->cause_nmi_ipi(cpu))
472 return;
473
474 if (cpu >= 0) {
475 do_message_pass(cpu, PPC_MSG_NMI_IPI);
476 } else {
477 int c;
478
479 for_each_online_cpu(c) {
480 if (c == raw_smp_processor_id())
481 continue;
482 do_message_pass(c, PPC_MSG_NMI_IPI);
483 }
484 }
485}
486
487
488
489
490
491
492
493static int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *),
494 u64 delay_us, bool safe)
495{
496 unsigned long flags;
497 int me = raw_smp_processor_id();
498 int ret = 1;
499
500 BUG_ON(cpu == me);
501 BUG_ON(cpu < 0 && cpu != NMI_IPI_ALL_OTHERS);
502
503 if (unlikely(!smp_ops))
504 return 0;
505
506 nmi_ipi_lock_start(&flags);
507 while (nmi_ipi_busy) {
508 nmi_ipi_unlock_end(&flags);
509 spin_until_cond(!nmi_ipi_busy);
510 nmi_ipi_lock_start(&flags);
511 }
512 nmi_ipi_busy = true;
513 nmi_ipi_function = fn;
514
515 WARN_ON_ONCE(!cpumask_empty(&nmi_ipi_pending_mask));
516
517 if (cpu < 0) {
518
519 cpumask_copy(&nmi_ipi_pending_mask, cpu_online_mask);
520 cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
521 } else {
522 cpumask_set_cpu(cpu, &nmi_ipi_pending_mask);
523 }
524
525 nmi_ipi_unlock();
526
527
528
529 do_smp_send_nmi_ipi(cpu, safe);
530
531 nmi_ipi_lock();
532
533 while (!cpumask_empty(&nmi_ipi_pending_mask)) {
534 nmi_ipi_unlock();
535 udelay(1);
536 nmi_ipi_lock();
537 if (delay_us) {
538 delay_us--;
539 if (!delay_us)
540 break;
541 }
542 }
543
544 if (!cpumask_empty(&nmi_ipi_pending_mask)) {
545
546 ret = 0;
547 cpumask_clear(&nmi_ipi_pending_mask);
548 }
549
550 nmi_ipi_function = NULL;
551 nmi_ipi_busy = false;
552
553 nmi_ipi_unlock_end(&flags);
554
555 return ret;
556}
557
558int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)
559{
560 return __smp_send_nmi_ipi(cpu, fn, delay_us, false);
561}
562
563int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)
564{
565 return __smp_send_nmi_ipi(cpu, fn, delay_us, true);
566}
567#endif
568
569#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
570void tick_broadcast(const struct cpumask *mask)
571{
572 unsigned int cpu;
573
574 for_each_cpu(cpu, mask)
575 do_message_pass(cpu, PPC_MSG_TICK_BROADCAST);
576}
577#endif
578
579#ifdef CONFIG_DEBUGGER
580static void debugger_ipi_callback(struct pt_regs *regs)
581{
582 debugger_ipi(regs);
583}
584
585void smp_send_debugger_break(void)
586{
587 smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, debugger_ipi_callback, 1000000);
588}
589#endif
590
591#ifdef CONFIG_KEXEC_CORE
592void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
593{
594 int cpu;
595
596 smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_ipi_callback, 1000000);
597 if (kdump_in_progress() && crash_wake_offline) {
598 for_each_present_cpu(cpu) {
599 if (cpu_online(cpu))
600 continue;
601
602
603
604
605
606
607
608
609
610 do_smp_send_nmi_ipi(cpu, false);
611 }
612 }
613}
614#endif
615
616#ifdef CONFIG_NMI_IPI
617static void nmi_stop_this_cpu(struct pt_regs *regs)
618{
619
620
621
622 set_cpu_online(smp_processor_id(), false);
623
624 spin_begin();
625 while (1)
626 spin_cpu_relax();
627}
628
629void smp_send_stop(void)
630{
631 smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, nmi_stop_this_cpu, 1000000);
632}
633
634#else
635
636static void stop_this_cpu(void *dummy)
637{
638 hard_irq_disable();
639
640
641
642
643
644
645
646 set_cpu_online(smp_processor_id(), false);
647
648 spin_begin();
649 while (1)
650 spin_cpu_relax();
651}
652
653void smp_send_stop(void)
654{
655 static bool stopped = false;
656
657
658
659
660
661
662
663 if (stopped)
664 return;
665
666 stopped = true;
667
668 smp_call_function(stop_this_cpu, NULL, 0);
669}
670#endif
671
672struct task_struct *current_set[NR_CPUS];
673
674static void smp_store_cpu_info(int id)
675{
676 per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR);
677#ifdef CONFIG_PPC_FSL_BOOK3E
678 per_cpu(next_tlbcam_idx, id)
679 = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
680#endif
681}
682
683
684
685
686
687
688static void set_cpus_related(int i, int j, struct cpumask *(*get_cpumask)(int))
689{
690 cpumask_set_cpu(i, get_cpumask(j));
691 cpumask_set_cpu(j, get_cpumask(i));
692}
693
694#ifdef CONFIG_HOTPLUG_CPU
695static void set_cpus_unrelated(int i, int j,
696 struct cpumask *(*get_cpumask)(int))
697{
698 cpumask_clear_cpu(i, get_cpumask(j));
699 cpumask_clear_cpu(j, get_cpumask(i));
700}
701#endif
702
703
704
705
706
707static void or_cpumasks_related(int i, int j, struct cpumask *(*srcmask)(int),
708 struct cpumask *(*dstmask)(int))
709{
710 struct cpumask *mask;
711 int k;
712
713 mask = srcmask(j);
714 for_each_cpu(k, srcmask(i))
715 cpumask_or(dstmask(k), dstmask(k), mask);
716
717 if (i == j)
718 return;
719
720 mask = srcmask(i);
721 for_each_cpu(k, srcmask(j))
722 cpumask_or(dstmask(k), dstmask(k), mask);
723}
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784static int parse_thread_groups(struct device_node *dn,
785 struct thread_groups_list *tglp)
786{
787 unsigned int property_idx = 0;
788 u32 *thread_group_array;
789 size_t total_threads;
790 int ret = 0, count;
791 u32 *thread_list;
792 int i = 0;
793
794 count = of_property_count_u32_elems(dn, "ibm,thread-groups");
795 thread_group_array = kcalloc(count, sizeof(u32), GFP_KERNEL);
796 ret = of_property_read_u32_array(dn, "ibm,thread-groups",
797 thread_group_array, count);
798 if (ret)
799 goto out_free;
800
801 while (i < count && property_idx < MAX_THREAD_GROUP_PROPERTIES) {
802 int j;
803 struct thread_groups *tg = &tglp->property_tgs[property_idx++];
804
805 tg->property = thread_group_array[i];
806 tg->nr_groups = thread_group_array[i + 1];
807 tg->threads_per_group = thread_group_array[i + 2];
808 total_threads = tg->nr_groups * tg->threads_per_group;
809
810 thread_list = &thread_group_array[i + 3];
811
812 for (j = 0; j < total_threads; j++)
813 tg->thread_list[j] = thread_list[j];
814 i = i + 3 + total_threads;
815 }
816
817 tglp->nr_properties = property_idx;
818
819out_free:
820 kfree(thread_group_array);
821 return ret;
822}
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838static int get_cpu_thread_group_start(int cpu, struct thread_groups *tg)
839{
840 int hw_cpu_id = get_hard_smp_processor_id(cpu);
841 int i, j;
842
843 for (i = 0; i < tg->nr_groups; i++) {
844 int group_start = i * tg->threads_per_group;
845
846 for (j = 0; j < tg->threads_per_group; j++) {
847 int idx = group_start + j;
848
849 if (tg->thread_list[idx] == hw_cpu_id)
850 return group_start;
851 }
852 }
853
854 return -1;
855}
856
857static struct thread_groups *__init get_thread_groups(int cpu,
858 int group_property,
859 int *err)
860{
861 struct device_node *dn = of_get_cpu_node(cpu, NULL);
862 struct thread_groups_list *cpu_tgl = &tgl[cpu];
863 struct thread_groups *tg = NULL;
864 int i;
865 *err = 0;
866
867 if (!dn) {
868 *err = -ENODATA;
869 return NULL;
870 }
871
872 if (!cpu_tgl->nr_properties) {
873 *err = parse_thread_groups(dn, cpu_tgl);
874 if (*err)
875 goto out;
876 }
877
878 for (i = 0; i < cpu_tgl->nr_properties; i++) {
879 if (cpu_tgl->property_tgs[i].property == group_property) {
880 tg = &cpu_tgl->property_tgs[i];
881 break;
882 }
883 }
884
885 if (!tg)
886 *err = -EINVAL;
887out:
888 of_node_put(dn);
889 return tg;
890}
891
892static int __init init_thread_group_cache_map(int cpu, int cache_property)
893
894{
895 int first_thread = cpu_first_thread_sibling(cpu);
896 int i, cpu_group_start = -1, err = 0;
897 struct thread_groups *tg = NULL;
898 cpumask_var_t *mask = NULL;
899
900 if (cache_property != THREAD_GROUP_SHARE_L1 &&
901 cache_property != THREAD_GROUP_SHARE_L2)
902 return -EINVAL;
903
904 tg = get_thread_groups(cpu, cache_property, &err);
905 if (!tg)
906 return err;
907
908 cpu_group_start = get_cpu_thread_group_start(cpu, tg);
909
910 if (unlikely(cpu_group_start == -1)) {
911 WARN_ON_ONCE(1);
912 return -ENODATA;
913 }
914
915 if (cache_property == THREAD_GROUP_SHARE_L1)
916 mask = &per_cpu(thread_group_l1_cache_map, cpu);
917 else if (cache_property == THREAD_GROUP_SHARE_L2)
918 mask = &per_cpu(thread_group_l2_cache_map, cpu);
919
920 zalloc_cpumask_var_node(mask, GFP_KERNEL, cpu_to_node(cpu));
921
922 for (i = first_thread; i < first_thread + threads_per_core; i++) {
923 int i_group_start = get_cpu_thread_group_start(i, tg);
924
925 if (unlikely(i_group_start == -1)) {
926 WARN_ON_ONCE(1);
927 return -ENODATA;
928 }
929
930 if (i_group_start == cpu_group_start)
931 cpumask_set_cpu(i, *mask);
932 }
933
934 return 0;
935}
936
937static bool shared_caches;
938
939#ifdef CONFIG_SCHED_SMT
940
941static int powerpc_smt_flags(void)
942{
943 int flags = SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
944
945 if (cpu_has_feature(CPU_FTR_ASYM_SMT)) {
946 printk_once(KERN_INFO "Enabling Asymmetric SMT scheduling\n");
947 flags |= SD_ASYM_PACKING;
948 }
949 return flags;
950}
951#endif
952
953
954
955
956
957
958
959static int powerpc_shared_cache_flags(void)
960{
961 return SD_SHARE_PKG_RESOURCES;
962}
963
964
965
966
967
968static const struct cpumask *shared_cache_mask(int cpu)
969{
970 return per_cpu(cpu_l2_cache_map, cpu);
971}
972
973#ifdef CONFIG_SCHED_SMT
974static const struct cpumask *smallcore_smt_mask(int cpu)
975{
976 return cpu_smallcore_mask(cpu);
977}
978#endif
979
980static struct cpumask *cpu_coregroup_mask(int cpu)
981{
982 return per_cpu(cpu_coregroup_map, cpu);
983}
984
985static bool has_coregroup_support(void)
986{
987 return coregroup_enabled;
988}
989
990static const struct cpumask *cpu_mc_mask(int cpu)
991{
992 return cpu_coregroup_mask(cpu);
993}
994
995static struct sched_domain_topology_level powerpc_topology[] = {
996#ifdef CONFIG_SCHED_SMT
997 { cpu_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) },
998#endif
999 { shared_cache_mask, powerpc_shared_cache_flags, SD_INIT_NAME(CACHE) },
1000 { cpu_mc_mask, SD_INIT_NAME(MC) },
1001 { cpu_cpu_mask, SD_INIT_NAME(DIE) },
1002 { NULL, },
1003};
1004
1005static int __init init_big_cores(void)
1006{
1007 int cpu;
1008
1009 for_each_possible_cpu(cpu) {
1010 int err = init_thread_group_cache_map(cpu, THREAD_GROUP_SHARE_L1);
1011
1012 if (err)
1013 return err;
1014
1015 zalloc_cpumask_var_node(&per_cpu(cpu_smallcore_map, cpu),
1016 GFP_KERNEL,
1017 cpu_to_node(cpu));
1018 }
1019
1020 has_big_cores = true;
1021
1022 for_each_possible_cpu(cpu) {
1023 int err = init_thread_group_cache_map(cpu, THREAD_GROUP_SHARE_L2);
1024
1025 if (err)
1026 return err;
1027 }
1028
1029 thread_group_shares_l2 = true;
1030 pr_debug("L2 cache only shared by the threads in the small core\n");
1031 return 0;
1032}
1033
1034void __init smp_prepare_cpus(unsigned int max_cpus)
1035{
1036 unsigned int cpu;
1037
1038 DBG("smp_prepare_cpus\n");
1039
1040
1041
1042
1043
1044 BUG_ON(boot_cpuid != smp_processor_id());
1045
1046
1047 smp_store_cpu_info(boot_cpuid);
1048 cpu_callin_map[boot_cpuid] = 1;
1049
1050 for_each_possible_cpu(cpu) {
1051 zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu),
1052 GFP_KERNEL, cpu_to_node(cpu));
1053 zalloc_cpumask_var_node(&per_cpu(cpu_l2_cache_map, cpu),
1054 GFP_KERNEL, cpu_to_node(cpu));
1055 zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu),
1056 GFP_KERNEL, cpu_to_node(cpu));
1057 if (has_coregroup_support())
1058 zalloc_cpumask_var_node(&per_cpu(cpu_coregroup_map, cpu),
1059 GFP_KERNEL, cpu_to_node(cpu));
1060
1061#ifdef CONFIG_NUMA
1062
1063
1064
1065 if (cpu_present(cpu)) {
1066 set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]);
1067 set_cpu_numa_mem(cpu,
1068 local_memory_node(numa_cpu_lookup_table[cpu]));
1069 }
1070#endif
1071 }
1072
1073
1074 cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
1075 cpumask_set_cpu(boot_cpuid, cpu_l2_cache_mask(boot_cpuid));
1076 cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
1077
1078 if (has_coregroup_support())
1079 cpumask_set_cpu(boot_cpuid, cpu_coregroup_mask(boot_cpuid));
1080
1081 init_big_cores();
1082 if (has_big_cores) {
1083 cpumask_set_cpu(boot_cpuid,
1084 cpu_smallcore_mask(boot_cpuid));
1085 }
1086
1087 if (cpu_to_chip_id(boot_cpuid) != -1) {
1088 int idx = num_possible_cpus() / threads_per_core;
1089
1090
1091
1092
1093
1094
1095
1096 chip_id_lookup_table = kcalloc(idx, sizeof(int), GFP_KERNEL);
1097 if (chip_id_lookup_table)
1098 memset(chip_id_lookup_table, -1, sizeof(int) * idx);
1099 }
1100
1101 if (smp_ops && smp_ops->probe)
1102 smp_ops->probe();
1103}
1104
1105void smp_prepare_boot_cpu(void)
1106{
1107 BUG_ON(smp_processor_id() != boot_cpuid);
1108#ifdef CONFIG_PPC64
1109 paca_ptrs[boot_cpuid]->__current = current;
1110#endif
1111 set_numa_node(numa_cpu_lookup_table[boot_cpuid]);
1112 current_set[boot_cpuid] = current;
1113}
1114
1115#ifdef CONFIG_HOTPLUG_CPU
1116
1117int generic_cpu_disable(void)
1118{
1119 unsigned int cpu = smp_processor_id();
1120
1121 if (cpu == boot_cpuid)
1122 return -EBUSY;
1123
1124 set_cpu_online(cpu, false);
1125#ifdef CONFIG_PPC64
1126 vdso_data->processorCount--;
1127#endif
1128
1129 irq_migrate_all_off_this_cpu();
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139 local_irq_enable();
1140 mdelay(1);
1141 local_irq_disable();
1142
1143 return 0;
1144}
1145
1146void generic_cpu_die(unsigned int cpu)
1147{
1148 int i;
1149
1150 for (i = 0; i < 100; i++) {
1151 smp_rmb();
1152 if (is_cpu_dead(cpu))
1153 return;
1154 msleep(100);
1155 }
1156 printk(KERN_ERR "CPU%d didn't die...\n", cpu);
1157}
1158
1159void generic_set_cpu_dead(unsigned int cpu)
1160{
1161 per_cpu(cpu_state, cpu) = CPU_DEAD;
1162}
1163
1164
1165
1166
1167
1168
1169void generic_set_cpu_up(unsigned int cpu)
1170{
1171 per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
1172}
1173
1174int generic_check_cpu_restart(unsigned int cpu)
1175{
1176 return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE;
1177}
1178
1179int is_cpu_dead(unsigned int cpu)
1180{
1181 return per_cpu(cpu_state, cpu) == CPU_DEAD;
1182}
1183
1184static bool secondaries_inhibited(void)
1185{
1186 return kvm_hv_mode_active();
1187}
1188
1189#else
1190
1191#define secondaries_inhibited() 0
1192
1193#endif
1194
1195static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
1196{
1197#ifdef CONFIG_PPC64
1198 paca_ptrs[cpu]->__current = idle;
1199 paca_ptrs[cpu]->kstack = (unsigned long)task_stack_page(idle) +
1200 THREAD_SIZE - STACK_FRAME_OVERHEAD;
1201#endif
1202 idle->cpu = cpu;
1203 secondary_current = current_set[cpu] = idle;
1204}
1205
1206int __cpu_up(unsigned int cpu, struct task_struct *tidle)
1207{
1208 int rc, c;
1209
1210
1211
1212
1213 if (threads_per_core > 1 && secondaries_inhibited() &&
1214 cpu_thread_in_subcore(cpu))
1215 return -EBUSY;
1216
1217 if (smp_ops == NULL ||
1218 (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
1219 return -EINVAL;
1220
1221 cpu_idle_thread_init(cpu, tidle);
1222
1223
1224
1225
1226
1227 if (smp_ops->prepare_cpu) {
1228 rc = smp_ops->prepare_cpu(cpu);
1229 if (rc)
1230 return rc;
1231 }
1232
1233
1234
1235
1236 cpu_callin_map[cpu] = 0;
1237
1238
1239
1240
1241
1242 smp_mb();
1243
1244
1245 DBG("smp: kicking cpu %d\n", cpu);
1246 rc = smp_ops->kick_cpu(cpu);
1247 if (rc) {
1248 pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc);
1249 return rc;
1250 }
1251
1252
1253
1254
1255
1256
1257 if (system_state < SYSTEM_RUNNING)
1258 for (c = 50000; c && !cpu_callin_map[cpu]; c--)
1259 udelay(100);
1260#ifdef CONFIG_HOTPLUG_CPU
1261 else
1262
1263
1264
1265
1266 for (c = 5000; c && !cpu_callin_map[cpu]; c--)
1267 msleep(1);
1268#endif
1269
1270 if (!cpu_callin_map[cpu]) {
1271 printk(KERN_ERR "Processor %u is stuck.\n", cpu);
1272 return -ENOENT;
1273 }
1274
1275 DBG("Processor %u found.\n", cpu);
1276
1277 if (smp_ops->give_timebase)
1278 smp_ops->give_timebase();
1279
1280
1281 spin_until_cond(cpu_online(cpu));
1282
1283 return 0;
1284}
1285
1286
1287
1288
1289int cpu_to_core_id(int cpu)
1290{
1291 struct device_node *np;
1292 const __be32 *reg;
1293 int id = -1;
1294
1295 np = of_get_cpu_node(cpu, NULL);
1296 if (!np)
1297 goto out;
1298
1299 reg = of_get_property(np, "reg", NULL);
1300 if (!reg)
1301 goto out;
1302
1303 id = be32_to_cpup(reg);
1304out:
1305 of_node_put(np);
1306 return id;
1307}
1308EXPORT_SYMBOL_GPL(cpu_to_core_id);
1309
1310
1311int cpu_core_index_of_thread(int cpu)
1312{
1313 return cpu >> threads_shift;
1314}
1315EXPORT_SYMBOL_GPL(cpu_core_index_of_thread);
1316
1317int cpu_first_thread_of_core(int core)
1318{
1319 return core << threads_shift;
1320}
1321EXPORT_SYMBOL_GPL(cpu_first_thread_of_core);
1322
1323
1324
1325
1326static struct device_node *cpu_to_l2cache(int cpu)
1327{
1328 struct device_node *np;
1329 struct device_node *cache;
1330
1331 if (!cpu_present(cpu))
1332 return NULL;
1333
1334 np = of_get_cpu_node(cpu, NULL);
1335 if (np == NULL)
1336 return NULL;
1337
1338 cache = of_find_next_cache_node(np);
1339
1340 of_node_put(np);
1341
1342 return cache;
1343}
1344
1345static bool update_mask_by_l2(int cpu, cpumask_var_t *mask)
1346{
1347 struct cpumask *(*submask_fn)(int) = cpu_sibling_mask;
1348 struct device_node *l2_cache, *np;
1349 int i;
1350
1351 if (has_big_cores)
1352 submask_fn = cpu_smallcore_mask;
1353
1354
1355
1356
1357
1358 if (thread_group_shares_l2) {
1359 cpumask_set_cpu(cpu, cpu_l2_cache_mask(cpu));
1360
1361 for_each_cpu(i, per_cpu(thread_group_l2_cache_map, cpu)) {
1362 if (cpu_online(i))
1363 set_cpus_related(i, cpu, cpu_l2_cache_mask);
1364 }
1365
1366
1367 if (!cpumask_equal(submask_fn(cpu), cpu_l2_cache_mask(cpu)) &&
1368 !cpumask_subset(submask_fn(cpu), cpu_l2_cache_mask(cpu))) {
1369 pr_warn_once("CPU %d : Inconsistent L1 and L2 cache siblings\n",
1370 cpu);
1371 }
1372
1373 return true;
1374 }
1375
1376 l2_cache = cpu_to_l2cache(cpu);
1377 if (!l2_cache || !*mask) {
1378
1379 for_each_cpu(i, submask_fn(cpu))
1380 set_cpus_related(cpu, i, cpu_l2_cache_mask);
1381
1382 return false;
1383 }
1384
1385 cpumask_and(*mask, cpu_online_mask, cpu_cpu_mask(cpu));
1386
1387
1388 or_cpumasks_related(cpu, cpu, submask_fn, cpu_l2_cache_mask);
1389
1390
1391 cpumask_andnot(*mask, *mask, cpu_l2_cache_mask(cpu));
1392
1393 for_each_cpu(i, *mask) {
1394
1395
1396
1397
1398 np = cpu_to_l2cache(i);
1399
1400
1401 if (np == l2_cache) {
1402 or_cpumasks_related(cpu, i, submask_fn, cpu_l2_cache_mask);
1403 cpumask_andnot(*mask, *mask, submask_fn(i));
1404 } else {
1405 cpumask_andnot(*mask, *mask, cpu_l2_cache_mask(i));
1406 }
1407
1408 of_node_put(np);
1409 }
1410 of_node_put(l2_cache);
1411
1412 return true;
1413}
1414
1415#ifdef CONFIG_HOTPLUG_CPU
1416static void remove_cpu_from_masks(int cpu)
1417{
1418 struct cpumask *(*mask_fn)(int) = cpu_sibling_mask;
1419 int i;
1420
1421 if (shared_caches)
1422 mask_fn = cpu_l2_cache_mask;
1423
1424 for_each_cpu(i, mask_fn(cpu)) {
1425 set_cpus_unrelated(cpu, i, cpu_l2_cache_mask);
1426 set_cpus_unrelated(cpu, i, cpu_sibling_mask);
1427 if (has_big_cores)
1428 set_cpus_unrelated(cpu, i, cpu_smallcore_mask);
1429 }
1430
1431 for_each_cpu(i, cpu_core_mask(cpu))
1432 set_cpus_unrelated(cpu, i, cpu_core_mask);
1433
1434 if (has_coregroup_support()) {
1435 for_each_cpu(i, cpu_coregroup_mask(cpu))
1436 set_cpus_unrelated(cpu, i, cpu_coregroup_mask);
1437 }
1438}
1439#endif
1440
1441static inline void add_cpu_to_smallcore_masks(int cpu)
1442{
1443 int i;
1444
1445 if (!has_big_cores)
1446 return;
1447
1448 cpumask_set_cpu(cpu, cpu_smallcore_mask(cpu));
1449
1450 for_each_cpu(i, per_cpu(thread_group_l1_cache_map, cpu)) {
1451 if (cpu_online(i))
1452 set_cpus_related(i, cpu, cpu_smallcore_mask);
1453 }
1454}
1455
1456static void update_coregroup_mask(int cpu, cpumask_var_t *mask)
1457{
1458 struct cpumask *(*submask_fn)(int) = cpu_sibling_mask;
1459 int coregroup_id = cpu_to_coregroup_id(cpu);
1460 int i;
1461
1462 if (shared_caches)
1463 submask_fn = cpu_l2_cache_mask;
1464
1465 if (!*mask) {
1466
1467 for_each_cpu(i, submask_fn(cpu))
1468 set_cpus_related(cpu, i, cpu_coregroup_mask);
1469
1470 return;
1471 }
1472
1473 cpumask_and(*mask, cpu_online_mask, cpu_cpu_mask(cpu));
1474
1475
1476 or_cpumasks_related(cpu, cpu, submask_fn, cpu_coregroup_mask);
1477
1478
1479 cpumask_andnot(*mask, *mask, cpu_coregroup_mask(cpu));
1480
1481 for_each_cpu(i, *mask) {
1482
1483 if (coregroup_id == cpu_to_coregroup_id(i)) {
1484 or_cpumasks_related(cpu, i, submask_fn, cpu_coregroup_mask);
1485 cpumask_andnot(*mask, *mask, submask_fn(i));
1486 } else {
1487 cpumask_andnot(*mask, *mask, cpu_coregroup_mask(i));
1488 }
1489 }
1490}
1491
1492static void add_cpu_to_masks(int cpu)
1493{
1494 struct cpumask *(*submask_fn)(int) = cpu_sibling_mask;
1495 int first_thread = cpu_first_thread_sibling(cpu);
1496 cpumask_var_t mask;
1497 int chip_id = -1;
1498 bool ret;
1499 int i;
1500
1501
1502
1503
1504
1505 cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
1506
1507 for (i = first_thread; i < first_thread + threads_per_core; i++)
1508 if (cpu_online(i))
1509 set_cpus_related(i, cpu, cpu_sibling_mask);
1510
1511 add_cpu_to_smallcore_masks(cpu);
1512
1513
1514 ret = alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu));
1515 update_mask_by_l2(cpu, &mask);
1516
1517 if (has_coregroup_support())
1518 update_coregroup_mask(cpu, &mask);
1519
1520 if (chip_id_lookup_table && ret)
1521 chip_id = cpu_to_chip_id(cpu);
1522
1523 if (chip_id == -1) {
1524 cpumask_copy(per_cpu(cpu_core_map, cpu), cpu_cpu_mask(cpu));
1525 goto out;
1526 }
1527
1528 if (shared_caches)
1529 submask_fn = cpu_l2_cache_mask;
1530
1531
1532 or_cpumasks_related(cpu, cpu, submask_fn, cpu_core_mask);
1533
1534
1535 cpumask_andnot(mask, cpu_online_mask, cpu_core_mask(cpu));
1536
1537 for_each_cpu(i, mask) {
1538 if (chip_id == cpu_to_chip_id(i)) {
1539 or_cpumasks_related(cpu, i, submask_fn, cpu_core_mask);
1540 cpumask_andnot(mask, mask, submask_fn(i));
1541 } else {
1542 cpumask_andnot(mask, mask, cpu_core_mask(i));
1543 }
1544 }
1545
1546out:
1547 free_cpumask_var(mask);
1548}
1549
1550
1551void start_secondary(void *unused)
1552{
1553 unsigned int cpu = raw_smp_processor_id();
1554
1555
1556 if (IS_ENABLED(CONFIG_PPC32))
1557 setup_kup();
1558
1559 mmgrab(&init_mm);
1560 current->active_mm = &init_mm;
1561
1562 smp_store_cpu_info(cpu);
1563 set_dec(tb_ticks_per_jiffy);
1564 rcu_cpu_starting(cpu);
1565 cpu_callin_map[cpu] = 1;
1566
1567 if (smp_ops->setup_cpu)
1568 smp_ops->setup_cpu(cpu);
1569 if (smp_ops->take_timebase)
1570 smp_ops->take_timebase();
1571
1572 secondary_cpu_time_init();
1573
1574#ifdef CONFIG_PPC64
1575 if (system_state == SYSTEM_RUNNING)
1576 vdso_data->processorCount++;
1577
1578 vdso_getcpu_init();
1579#endif
1580 set_numa_node(numa_cpu_lookup_table[cpu]);
1581 set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
1582
1583
1584 add_cpu_to_masks(cpu);
1585
1586
1587
1588
1589
1590 if (!shared_caches) {
1591 struct cpumask *(*sibling_mask)(int) = cpu_sibling_mask;
1592 struct cpumask *mask = cpu_l2_cache_mask(cpu);
1593
1594 if (has_big_cores)
1595 sibling_mask = cpu_smallcore_mask;
1596
1597 if (cpumask_weight(mask) > cpumask_weight(sibling_mask(cpu)))
1598 shared_caches = true;
1599 }
1600
1601 smp_wmb();
1602 notify_cpu_starting(cpu);
1603 set_cpu_online(cpu, true);
1604
1605 boot_init_stack_canary();
1606
1607 local_irq_enable();
1608
1609
1610 this_cpu_enable_ftrace();
1611
1612 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
1613
1614 BUG();
1615}
1616
1617int setup_profiling_timer(unsigned int multiplier)
1618{
1619 return 0;
1620}
1621
1622static void fixup_topology(void)
1623{
1624 int i;
1625
1626#ifdef CONFIG_SCHED_SMT
1627 if (has_big_cores) {
1628 pr_info("Big cores detected but using small core scheduling\n");
1629 powerpc_topology[smt_idx].mask = smallcore_smt_mask;
1630 }
1631#endif
1632
1633 if (!has_coregroup_support())
1634 powerpc_topology[mc_idx].mask = powerpc_topology[cache_idx].mask;
1635
1636
1637
1638
1639
1640
1641
1642 for (i = 1; i <= die_idx; i++) {
1643 if (powerpc_topology[i].mask != powerpc_topology[i - 1].mask)
1644 continue;
1645
1646 if (powerpc_topology[i].sd_flags && powerpc_topology[i - 1].sd_flags &&
1647 powerpc_topology[i].sd_flags != powerpc_topology[i - 1].sd_flags)
1648 continue;
1649
1650 if (!powerpc_topology[i - 1].sd_flags)
1651 powerpc_topology[i - 1].sd_flags = powerpc_topology[i].sd_flags;
1652
1653 powerpc_topology[i].mask = powerpc_topology[i + 1].mask;
1654 powerpc_topology[i].sd_flags = powerpc_topology[i + 1].sd_flags;
1655#ifdef CONFIG_SCHED_DEBUG
1656 powerpc_topology[i].name = powerpc_topology[i + 1].name;
1657#endif
1658 }
1659}
1660
1661void __init smp_cpus_done(unsigned int max_cpus)
1662{
1663
1664
1665
1666 if (smp_ops && smp_ops->setup_cpu)
1667 smp_ops->setup_cpu(boot_cpuid);
1668
1669 if (smp_ops && smp_ops->bringup_done)
1670 smp_ops->bringup_done();
1671
1672 dump_numa_cpu_topology();
1673
1674 fixup_topology();
1675 set_sched_topology(powerpc_topology);
1676}
1677
1678#ifdef CONFIG_HOTPLUG_CPU
1679int __cpu_disable(void)
1680{
1681 int cpu = smp_processor_id();
1682 int err;
1683
1684 if (!smp_ops->cpu_disable)
1685 return -ENOSYS;
1686
1687 this_cpu_disable_ftrace();
1688
1689 err = smp_ops->cpu_disable();
1690 if (err)
1691 return err;
1692
1693
1694 remove_cpu_from_masks(cpu);
1695
1696 return 0;
1697}
1698
1699void __cpu_die(unsigned int cpu)
1700{
1701 if (smp_ops->cpu_die)
1702 smp_ops->cpu_die(cpu);
1703}
1704
1705void arch_cpu_idle_dead(void)
1706{
1707 sched_preempt_enable_no_resched();
1708
1709
1710
1711
1712
1713 this_cpu_disable_ftrace();
1714
1715 if (smp_ops->cpu_offline_self)
1716 smp_ops->cpu_offline_self();
1717
1718
1719 start_secondary_resume();
1720}
1721
1722#endif
1723