1
2
3
4
5
6
7
8
9#include <linux/export.h>
10#include <linux/string.h>
11#include <linux/sched.h>
12#include <linux/init.h>
13#include <linux/kernel.h>
14#include <linux/reboot.h>
15#include <linux/delay.h>
16#include <linux/initrd.h>
17#include <linux/seq_file.h>
18#include <linux/ioport.h>
19#include <linux/console.h>
20#include <linux/utsname.h>
21#include <linux/tty.h>
22#include <linux/root_dev.h>
23#include <linux/notifier.h>
24#include <linux/cpu.h>
25#include <linux/unistd.h>
26#include <linux/serial.h>
27#include <linux/serial_8250.h>
28#include <linux/memblock.h>
29#include <linux/pci.h>
30#include <linux/lockdep.h>
31#include <linux/memory.h>
32#include <linux/nmi.h>
33
34#include <asm/debugfs.h>
35#include <asm/io.h>
36#include <asm/kdump.h>
37#include <asm/prom.h>
38#include <asm/processor.h>
39#include <asm/pgtable.h>
40#include <asm/smp.h>
41#include <asm/elf.h>
42#include <asm/machdep.h>
43#include <asm/paca.h>
44#include <asm/time.h>
45#include <asm/cputable.h>
46#include <asm/dt_cpu_ftrs.h>
47#include <asm/sections.h>
48#include <asm/btext.h>
49#include <asm/nvram.h>
50#include <asm/setup.h>
51#include <asm/rtas.h>
52#include <asm/iommu.h>
53#include <asm/serial.h>
54#include <asm/cache.h>
55#include <asm/page.h>
56#include <asm/mmu.h>
57#include <asm/firmware.h>
58#include <asm/xmon.h>
59#include <asm/udbg.h>
60#include <asm/kexec.h>
61#include <asm/code-patching.h>
62#include <asm/livepatch.h>
63#include <asm/opal.h>
64#include <asm/cputhreads.h>
65#include <asm/hw_irq.h>
66#include <asm/feature-fixups.h>
67#include <asm/kup.h>
68#include <asm/early_ioremap.h>
69
70#include "setup.h"
71
72int spinning_secondaries;
73u64 ppc64_pft_size;
74
75struct ppc64_caches ppc64_caches = {
76 .l1d = {
77 .block_size = 0x40,
78 .log_block_size = 6,
79 },
80 .l1i = {
81 .block_size = 0x40,
82 .log_block_size = 6
83 },
84};
85EXPORT_SYMBOL_GPL(ppc64_caches);
86
87#if defined(CONFIG_PPC_BOOK3E) && defined(CONFIG_SMP)
88void __init setup_tlb_core_data(void)
89{
90 int cpu;
91
92 BUILD_BUG_ON(offsetof(struct tlb_core_data, lock) != 0);
93
94 for_each_possible_cpu(cpu) {
95 int first = cpu_first_thread_sibling(cpu);
96
97
98
99
100
101
102 if (cpu_first_thread_sibling(boot_cpuid) == first)
103 first = boot_cpuid;
104
105 paca_ptrs[cpu]->tcd_ptr = &paca_ptrs[first]->tcd;
106
107
108
109
110
111
112
113 WARN_ONCE(smt_enabled_at_boot >= 2 &&
114 !mmu_has_feature(MMU_FTR_USE_TLBRSRV) &&
115 book3e_htw_mode != PPC_HTW_E6500,
116 "%s: unsupported MMU configuration\n", __func__);
117 }
118}
119#endif
120
121#ifdef CONFIG_SMP
122
123static char *smt_enabled_cmdline;
124
125
126void __init check_smt_enabled(void)
127{
128 struct device_node *dn;
129 const char *smt_option;
130
131
132 smt_enabled_at_boot = threads_per_core;
133
134
135 if (smt_enabled_cmdline) {
136 if (!strcmp(smt_enabled_cmdline, "on"))
137 smt_enabled_at_boot = threads_per_core;
138 else if (!strcmp(smt_enabled_cmdline, "off"))
139 smt_enabled_at_boot = 0;
140 else {
141 int smt;
142 int rc;
143
144 rc = kstrtoint(smt_enabled_cmdline, 10, &smt);
145 if (!rc)
146 smt_enabled_at_boot =
147 min(threads_per_core, smt);
148 }
149 } else {
150 dn = of_find_node_by_path("/options");
151 if (dn) {
152 smt_option = of_get_property(dn, "ibm,smt-enabled",
153 NULL);
154
155 if (smt_option) {
156 if (!strcmp(smt_option, "on"))
157 smt_enabled_at_boot = threads_per_core;
158 else if (!strcmp(smt_option, "off"))
159 smt_enabled_at_boot = 0;
160 }
161
162 of_node_put(dn);
163 }
164 }
165}
166
167
168static int __init early_smt_enabled(char *p)
169{
170 smt_enabled_cmdline = p;
171 return 0;
172}
173early_param("smt-enabled", early_smt_enabled);
174
175#endif
176
177
178static void __init fixup_boot_paca(void)
179{
180
181 get_paca()->cpu_start = 1;
182
183 get_paca()->data_offset = 0;
184
185 irq_soft_mask_set(IRQS_DISABLED);
186}
187
188static void __init configure_exceptions(void)
189{
190
191
192
193
194 setup_kdump_trampoline();
195
196
197 if (firmware_has_feature(FW_FEATURE_SET_MODE)) {
198
199 pseries_enable_reloc_on_exc();
200
201
202
203
204
205
206
207
208
209#ifdef __LITTLE_ENDIAN__
210 pseries_little_endian_exceptions();
211#endif
212 } else {
213
214 if (firmware_has_feature(FW_FEATURE_OPAL))
215 opal_configure_cores();
216
217
218 }
219}
220
221static void cpu_ready_for_interrupts(void)
222{
223
224
225
226
227
228
229
230 if (cpu_has_feature(CPU_FTR_HVMODE) &&
231 cpu_has_feature(CPU_FTR_ARCH_207S)) {
232 unsigned long lpcr = mfspr(SPRN_LPCR);
233 mtspr(SPRN_LPCR, lpcr | LPCR_AIL_3);
234 }
235
236
237
238
239
240
241
242
243
244 if (cpu_has_feature(CPU_FTR_HVMODE)) {
245 if (cpu_has_feature(CPU_FTR_TM_COMP))
246 mtspr(SPRN_HFSCR, mfspr(SPRN_HFSCR) | HFSCR_TM);
247 else
248 mtspr(SPRN_HFSCR, mfspr(SPRN_HFSCR) & ~HFSCR_TM);
249 }
250
251
252 get_paca()->kernel_msr = MSR_KERNEL;
253}
254
255unsigned long spr_default_dscr = 0;
256
257void __init record_spr_defaults(void)
258{
259 if (early_cpu_has_feature(CPU_FTR_DSCR))
260 spr_default_dscr = mfspr(SPRN_DSCR);
261}
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282void __init early_setup(unsigned long dt_ptr)
283{
284 static __initdata struct paca_struct boot_paca;
285
286
287
288
289 if (!dt_cpu_ftrs_init(__va(dt_ptr)))
290
291 identify_cpu(0, mfspr(SPRN_PVR));
292
293
294 initialise_paca(&boot_paca, 0);
295 setup_paca(&boot_paca);
296 fixup_boot_paca();
297
298
299
300
301 udbg_early_init();
302
303 udbg_printf(" -> %s(), dt_ptr: 0x%lx\n", __func__, dt_ptr);
304
305
306
307
308
309
310 early_init_devtree(__va(dt_ptr));
311
312
313 if (boot_cpuid != 0) {
314
315 memset(&paca_ptrs[0], 0x88, sizeof(paca_ptrs[0]));
316 }
317 setup_paca(paca_ptrs[boot_cpuid]);
318 fixup_boot_paca();
319
320
321
322
323
324 configure_exceptions();
325
326
327
328
329
330 setup_kup();
331
332
333 apply_feature_fixups();
334 setup_feature_keys();
335
336 early_ioremap_setup();
337
338
339 early_init_mmu();
340
341
342
343
344
345
346 record_spr_defaults();
347
348
349
350
351
352
353 cpu_ready_for_interrupts();
354
355
356
357
358
359
360 this_cpu_enable_ftrace();
361
362 udbg_printf(" <- %s()\n", __func__);
363
364#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
365
366
367
368
369
370
371
372
373 btext_map();
374#endif
375}
376
377#ifdef CONFIG_SMP
378void early_setup_secondary(void)
379{
380
381 irq_soft_mask_set(IRQS_DISABLED);
382
383
384 early_init_mmu_secondary();
385
386
387 setup_kup();
388
389
390
391
392
393
394 cpu_ready_for_interrupts();
395}
396
397#endif
398
399void panic_smp_self_stop(void)
400{
401 hard_irq_disable();
402 spin_begin();
403 while (1)
404 spin_cpu_relax();
405}
406
407#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE)
408static bool use_spinloop(void)
409{
410 if (IS_ENABLED(CONFIG_PPC_BOOK3S)) {
411
412
413
414
415
416 if (firmware_has_feature(FW_FEATURE_OPAL))
417 return false;
418 return true;
419 }
420
421
422
423
424
425 return of_property_read_bool(of_chosen, "linux,booted-from-kexec");
426}
427
428void smp_release_cpus(void)
429{
430 unsigned long *ptr;
431 int i;
432
433 if (!use_spinloop())
434 return;
435
436
437
438
439
440
441
442 ptr = (unsigned long *)((unsigned long)&__secondary_hold_spinloop
443 - PHYSICAL_START);
444 *ptr = ppc_function_entry(generic_secondary_smp_init);
445
446
447 for (i = 0; i < 100000; i++) {
448 mb();
449 HMT_low();
450 if (spinning_secondaries == 0)
451 break;
452 udelay(1);
453 }
454 pr_debug("spinning_secondaries = %d\n", spinning_secondaries);
455}
456#endif
457
458
459
460
461
462
463
464
465
466static void init_cache_info(struct ppc_cache_info *info, u32 size, u32 lsize,
467 u32 bsize, u32 sets)
468{
469 info->size = size;
470 info->sets = sets;
471 info->line_size = lsize;
472 info->block_size = bsize;
473 info->log_block_size = __ilog2(bsize);
474 if (bsize)
475 info->blocks_per_page = PAGE_SIZE / bsize;
476 else
477 info->blocks_per_page = 0;
478
479 if (sets == 0)
480 info->assoc = 0xffff;
481 else
482 info->assoc = size / (sets * lsize);
483}
484
485static bool __init parse_cache_info(struct device_node *np,
486 bool icache,
487 struct ppc_cache_info *info)
488{
489 static const char *ipropnames[] __initdata = {
490 "i-cache-size",
491 "i-cache-sets",
492 "i-cache-block-size",
493 "i-cache-line-size",
494 };
495 static const char *dpropnames[] __initdata = {
496 "d-cache-size",
497 "d-cache-sets",
498 "d-cache-block-size",
499 "d-cache-line-size",
500 };
501 const char **propnames = icache ? ipropnames : dpropnames;
502 const __be32 *sizep, *lsizep, *bsizep, *setsp;
503 u32 size, lsize, bsize, sets;
504 bool success = true;
505
506 size = 0;
507 sets = -1u;
508 lsize = bsize = cur_cpu_spec->dcache_bsize;
509 sizep = of_get_property(np, propnames[0], NULL);
510 if (sizep != NULL)
511 size = be32_to_cpu(*sizep);
512 setsp = of_get_property(np, propnames[1], NULL);
513 if (setsp != NULL)
514 sets = be32_to_cpu(*setsp);
515 bsizep = of_get_property(np, propnames[2], NULL);
516 lsizep = of_get_property(np, propnames[3], NULL);
517 if (bsizep == NULL)
518 bsizep = lsizep;
519 if (lsizep != NULL)
520 lsize = be32_to_cpu(*lsizep);
521 if (bsizep != NULL)
522 bsize = be32_to_cpu(*bsizep);
523 if (sizep == NULL || bsizep == NULL || lsizep == NULL)
524 success = false;
525
526
527
528
529
530
531
532 if (sets == 1)
533 sets = 0;
534 else if (sets == 0)
535 sets = 1;
536
537 init_cache_info(info, size, lsize, bsize, sets);
538
539 return success;
540}
541
542void __init initialize_cache_info(void)
543{
544 struct device_node *cpu = NULL, *l2, *l3 = NULL;
545 u32 pvr;
546
547
548
549
550
551
552
553 pvr = PVR_VER(mfspr(SPRN_PVR));
554 if (pvr == PVR_POWER8 || pvr == PVR_POWER8E ||
555 pvr == PVR_POWER8NVL) {
556
557 init_cache_info(&ppc64_caches.l1i, 0x8000, 128, 128, 32);
558 init_cache_info(&ppc64_caches.l1d, 0x10000, 128, 128, 64);
559 init_cache_info(&ppc64_caches.l2, 0x80000, 128, 0, 512);
560 init_cache_info(&ppc64_caches.l3, 0x800000, 128, 0, 8192);
561 } else
562 cpu = of_find_node_by_type(NULL, "cpu");
563
564
565
566
567
568 if (cpu) {
569 if (!parse_cache_info(cpu, false, &ppc64_caches.l1d))
570 pr_warn("Argh, can't find dcache properties !\n");
571
572 if (!parse_cache_info(cpu, true, &ppc64_caches.l1i))
573 pr_warn("Argh, can't find icache properties !\n");
574
575
576
577
578
579 l2 = of_find_next_cache_node(cpu);
580 of_node_put(cpu);
581 if (l2) {
582 parse_cache_info(l2, false, &ppc64_caches.l2);
583 l3 = of_find_next_cache_node(l2);
584 of_node_put(l2);
585 }
586 if (l3) {
587 parse_cache_info(l3, false, &ppc64_caches.l3);
588 of_node_put(l3);
589 }
590 }
591
592
593 dcache_bsize = ppc64_caches.l1d.block_size;
594 icache_bsize = ppc64_caches.l1i.block_size;
595
596 cur_cpu_spec->dcache_bsize = dcache_bsize;
597 cur_cpu_spec->icache_bsize = icache_bsize;
598}
599
600
601
602
603
604
605
606
607
608
609__init u64 ppc64_bolted_size(void)
610{
611#ifdef CONFIG_PPC_BOOK3E
612
613
614 if (early_mmu_has_feature(MMU_FTR_TYPE_FSL_E))
615 return linear_map_top;
616
617 return 1ul << 30;
618#else
619
620 if (early_radix_enabled())
621 return ULONG_MAX;
622
623
624 if (early_mmu_has_feature(MMU_FTR_1T_SEGMENT))
625 return 1UL << SID_SHIFT_1T;
626 return 1UL << SID_SHIFT;
627#endif
628}
629
630static void *__init alloc_stack(unsigned long limit, int cpu)
631{
632 void *ptr;
633
634 BUILD_BUG_ON(STACK_INT_FRAME_SIZE % 16);
635
636 ptr = memblock_alloc_try_nid(THREAD_SIZE, THREAD_ALIGN,
637 MEMBLOCK_LOW_LIMIT, limit,
638 early_cpu_to_node(cpu));
639 if (!ptr)
640 panic("cannot allocate stacks");
641
642 return ptr;
643}
644
645void __init irqstack_early_init(void)
646{
647 u64 limit = ppc64_bolted_size();
648 unsigned int i;
649
650
651
652
653
654
655 for_each_possible_cpu(i) {
656 softirq_ctx[i] = alloc_stack(limit, i);
657 hardirq_ctx[i] = alloc_stack(limit, i);
658 }
659}
660
661#ifdef CONFIG_PPC_BOOK3E
662void __init exc_lvl_early_init(void)
663{
664 unsigned int i;
665
666 for_each_possible_cpu(i) {
667 void *sp;
668
669 sp = alloc_stack(ULONG_MAX, i);
670 critirq_ctx[i] = sp;
671 paca_ptrs[i]->crit_kstack = sp + THREAD_SIZE;
672
673 sp = alloc_stack(ULONG_MAX, i);
674 dbgirq_ctx[i] = sp;
675 paca_ptrs[i]->dbg_kstack = sp + THREAD_SIZE;
676
677 sp = alloc_stack(ULONG_MAX, i);
678 mcheckirq_ctx[i] = sp;
679 paca_ptrs[i]->mc_kstack = sp + THREAD_SIZE;
680 }
681
682 if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))
683 patch_exception(0x040, exc_debug_debug_book3e);
684}
685#endif
686
687
688
689
690
691
692void __init emergency_stack_init(void)
693{
694 u64 limit;
695 unsigned int i;
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711 limit = min(ppc64_bolted_size(), ppc64_rma_size);
712
713 for_each_possible_cpu(i) {
714 paca_ptrs[i]->emergency_sp = alloc_stack(limit, i) + THREAD_SIZE;
715
716#ifdef CONFIG_PPC_BOOK3S_64
717
718 paca_ptrs[i]->nmi_emergency_sp = alloc_stack(limit, i) + THREAD_SIZE;
719
720
721 paca_ptrs[i]->mc_emergency_sp = alloc_stack(limit, i) + THREAD_SIZE;
722#endif
723 }
724}
725
726#ifdef CONFIG_SMP
727#define PCPU_DYN_SIZE ()
728
729static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align)
730{
731 return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS),
732 MEMBLOCK_ALLOC_ACCESSIBLE,
733 early_cpu_to_node(cpu));
734
735}
736
737static void __init pcpu_fc_free(void *ptr, size_t size)
738{
739 memblock_free(__pa(ptr), size);
740}
741
742static int pcpu_cpu_distance(unsigned int from, unsigned int to)
743{
744 if (early_cpu_to_node(from) == early_cpu_to_node(to))
745 return LOCAL_DISTANCE;
746 else
747 return REMOTE_DISTANCE;
748}
749
750unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
751EXPORT_SYMBOL(__per_cpu_offset);
752
753void __init setup_per_cpu_areas(void)
754{
755 const size_t dyn_size = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;
756 size_t atom_size;
757 unsigned long delta;
758 unsigned int cpu;
759 int rc;
760
761
762
763
764
765
766 if (mmu_linear_psize == MMU_PAGE_4K)
767 atom_size = PAGE_SIZE;
768 else
769 atom_size = 1 << 20;
770
771 rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance,
772 pcpu_fc_alloc, pcpu_fc_free);
773 if (rc < 0)
774 panic("cannot initialize percpu area (err=%d)", rc);
775
776 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
777 for_each_possible_cpu(cpu) {
778 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
779 paca_ptrs[cpu]->data_offset = __per_cpu_offset[cpu];
780 }
781}
782#endif
783
784#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
785unsigned long memory_block_size_bytes(void)
786{
787 if (ppc_md.memory_block_size)
788 return ppc_md.memory_block_size();
789
790 return MIN_MEMORY_BLOCK_SIZE;
791}
792#endif
793
794#if defined(CONFIG_PPC_INDIRECT_PIO) || defined(CONFIG_PPC_INDIRECT_MMIO)
795struct ppc_pci_io ppc_pci_io;
796EXPORT_SYMBOL(ppc_pci_io);
797#endif
798
799#ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF
800u64 hw_nmi_get_sample_period(int watchdog_thresh)
801{
802 return ppc_proc_freq * watchdog_thresh;
803}
804#endif
805
806
807
808
809
810
811
812
813
814static int __init disable_hardlockup_detector(void)
815{
816#ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF
817 hardlockup_detector_disable();
818#else
819 if (firmware_has_feature(FW_FEATURE_LPAR))
820 hardlockup_detector_disable();
821#endif
822
823 return 0;
824}
825early_initcall(disable_hardlockup_detector);
826
827#ifdef CONFIG_PPC_BOOK3S_64
828static enum l1d_flush_type enabled_flush_types;
829static void *l1d_flush_fallback_area;
830static bool no_rfi_flush;
831bool rfi_flush;
832
833static int __init handle_no_rfi_flush(char *p)
834{
835 pr_info("rfi-flush: disabled on command line.");
836 no_rfi_flush = true;
837 return 0;
838}
839early_param("no_rfi_flush", handle_no_rfi_flush);
840
841
842
843
844
845static int __init handle_no_pti(char *p)
846{
847 pr_info("rfi-flush: disabling due to 'nopti' on command line.\n");
848 handle_no_rfi_flush(NULL);
849 return 0;
850}
851early_param("nopti", handle_no_pti);
852
853static void do_nothing(void *unused)
854{
855
856
857
858
859}
860
861void rfi_flush_enable(bool enable)
862{
863 if (enable) {
864 do_rfi_flush_fixups(enabled_flush_types);
865 on_each_cpu(do_nothing, NULL, 1);
866 } else
867 do_rfi_flush_fixups(L1D_FLUSH_NONE);
868
869 rfi_flush = enable;
870}
871
872static void __ref init_fallback_flush(void)
873{
874 u64 l1d_size, limit;
875 int cpu;
876
877
878 if (l1d_flush_fallback_area)
879 return;
880
881 l1d_size = ppc64_caches.l1d.size;
882
883
884
885
886
887
888
889
890 if (!l1d_size)
891 l1d_size = (64 * 1024);
892
893 limit = min(ppc64_bolted_size(), ppc64_rma_size);
894
895
896
897
898
899
900 l1d_flush_fallback_area = memblock_alloc_try_nid(l1d_size * 2,
901 l1d_size, MEMBLOCK_LOW_LIMIT,
902 limit, NUMA_NO_NODE);
903 if (!l1d_flush_fallback_area)
904 panic("%s: Failed to allocate %llu bytes align=0x%llx max_addr=%pa\n",
905 __func__, l1d_size * 2, l1d_size, &limit);
906
907
908 for_each_possible_cpu(cpu) {
909 struct paca_struct *paca = paca_ptrs[cpu];
910 paca->rfi_flush_fallback_area = l1d_flush_fallback_area;
911 paca->l1d_flush_size = l1d_size;
912 }
913}
914
915void setup_rfi_flush(enum l1d_flush_type types, bool enable)
916{
917 if (types & L1D_FLUSH_FALLBACK) {
918 pr_info("rfi-flush: fallback displacement flush available\n");
919 init_fallback_flush();
920 }
921
922 if (types & L1D_FLUSH_ORI)
923 pr_info("rfi-flush: ori type flush available\n");
924
925 if (types & L1D_FLUSH_MTTRIG)
926 pr_info("rfi-flush: mttrig type flush available\n");
927
928 enabled_flush_types = types;
929
930 if (!no_rfi_flush && !cpu_mitigations_off())
931 rfi_flush_enable(enable);
932}
933
934#ifdef CONFIG_DEBUG_FS
935static int rfi_flush_set(void *data, u64 val)
936{
937 bool enable;
938
939 if (val == 1)
940 enable = true;
941 else if (val == 0)
942 enable = false;
943 else
944 return -EINVAL;
945
946
947 if (enable != rfi_flush)
948 rfi_flush_enable(enable);
949
950 return 0;
951}
952
953static int rfi_flush_get(void *data, u64 *val)
954{
955 *val = rfi_flush ? 1 : 0;
956 return 0;
957}
958
959DEFINE_SIMPLE_ATTRIBUTE(fops_rfi_flush, rfi_flush_get, rfi_flush_set, "%llu\n");
960
961static __init int rfi_flush_debugfs_init(void)
962{
963 debugfs_create_file("rfi_flush", 0600, powerpc_debugfs_root, NULL, &fops_rfi_flush);
964 return 0;
965}
966device_initcall(rfi_flush_debugfs_init);
967#endif
968#endif
969