1
2
3#define USE_EARLY_PGTABLE_L5
4
5#include <linux/memblock.h>
6#include <linux/linkage.h>
7#include <linux/bitops.h>
8#include <linux/kernel.h>
9#include <linux/export.h>
10#include <linux/percpu.h>
11#include <linux/string.h>
12#include <linux/ctype.h>
13#include <linux/delay.h>
14#include <linux/sched/mm.h>
15#include <linux/sched/clock.h>
16#include <linux/sched/task.h>
17#include <linux/sched/smt.h>
18#include <linux/init.h>
19#include <linux/kprobes.h>
20#include <linux/kgdb.h>
21#include <linux/smp.h>
22#include <linux/io.h>
23#include <linux/syscore_ops.h>
24#include <linux/pgtable.h>
25
26#include <asm/cmdline.h>
27#include <asm/stackprotector.h>
28#include <asm/perf_event.h>
29#include <asm/mmu_context.h>
30#include <asm/doublefault.h>
31#include <asm/archrandom.h>
32#include <asm/hypervisor.h>
33#include <asm/processor.h>
34#include <asm/tlbflush.h>
35#include <asm/debugreg.h>
36#include <asm/sections.h>
37#include <asm/vsyscall.h>
38#include <linux/topology.h>
39#include <linux/cpumask.h>
40#include <linux/atomic.h>
41#include <asm/proto.h>
42#include <asm/setup.h>
43#include <asm/apic.h>
44#include <asm/desc.h>
45#include <asm/fpu/internal.h>
46#include <asm/mtrr.h>
47#include <asm/hwcap2.h>
48#include <linux/numa.h>
49#include <asm/numa.h>
50#include <asm/asm.h>
51#include <asm/bugs.h>
52#include <asm/cpu.h>
53#include <asm/mce.h>
54#include <asm/msr.h>
55#include <asm/memtype.h>
56#include <asm/microcode.h>
57#include <asm/microcode_intel.h>
58#include <asm/intel-family.h>
59#include <asm/cpu_device_id.h>
60#include <asm/uv/uv.h>
61#include <asm/sigframe.h>
62
63#include "cpu.h"
64
65u32 elf_hwcap2 __read_mostly;
66
67
68cpumask_var_t cpu_initialized_mask;
69cpumask_var_t cpu_callout_mask;
70cpumask_var_t cpu_callin_mask;
71
72
73cpumask_var_t cpu_sibling_setup_mask;
74
75
76int smp_num_siblings = 1;
77EXPORT_SYMBOL(smp_num_siblings);
78
79
80DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
81
82u16 get_llc_id(unsigned int cpu)
83{
84 return per_cpu(cpu_llc_id, cpu);
85}
86EXPORT_SYMBOL_GPL(get_llc_id);
87
88
89void __init setup_cpu_local_masks(void)
90{
91 alloc_bootmem_cpumask_var(&cpu_initialized_mask);
92 alloc_bootmem_cpumask_var(&cpu_callin_mask);
93 alloc_bootmem_cpumask_var(&cpu_callout_mask);
94 alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask);
95}
96
97static void default_init(struct cpuinfo_x86 *c)
98{
99#ifdef CONFIG_X86_64
100 cpu_detect_cache_sizes(c);
101#else
102
103
104 if (c->cpuid_level == -1) {
105
106 if (c->x86 == 4)
107 strcpy(c->x86_model_id, "486");
108 else if (c->x86 == 3)
109 strcpy(c->x86_model_id, "386");
110 }
111#endif
112}
113
114static const struct cpu_dev default_cpu = {
115 .c_init = default_init,
116 .c_vendor = "Unknown",
117 .c_x86_vendor = X86_VENDOR_UNKNOWN,
118};
119
120static const struct cpu_dev *this_cpu = &default_cpu;
121
122DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
123#ifdef CONFIG_X86_64
124
125
126
127
128
129
130
131
132 [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
133 [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
134 [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
135 [GDT_ENTRY_DEFAULT_USER32_CS] = GDT_ENTRY_INIT(0xc0fb, 0, 0xfffff),
136 [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f3, 0, 0xfffff),
137 [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xa0fb, 0, 0xfffff),
138#else
139 [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(0xc09a, 0, 0xfffff),
140 [GDT_ENTRY_KERNEL_DS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
141 [GDT_ENTRY_DEFAULT_USER_CS] = GDT_ENTRY_INIT(0xc0fa, 0, 0xfffff),
142 [GDT_ENTRY_DEFAULT_USER_DS] = GDT_ENTRY_INIT(0xc0f2, 0, 0xfffff),
143
144
145
146
147
148
149 [GDT_ENTRY_PNPBIOS_CS32] = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
150
151 [GDT_ENTRY_PNPBIOS_CS16] = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
152
153 [GDT_ENTRY_PNPBIOS_DS] = GDT_ENTRY_INIT(0x0092, 0, 0xffff),
154
155 [GDT_ENTRY_PNPBIOS_TS1] = GDT_ENTRY_INIT(0x0092, 0, 0),
156
157 [GDT_ENTRY_PNPBIOS_TS2] = GDT_ENTRY_INIT(0x0092, 0, 0),
158
159
160
161
162
163 [GDT_ENTRY_APMBIOS_BASE] = GDT_ENTRY_INIT(0x409a, 0, 0xffff),
164
165 [GDT_ENTRY_APMBIOS_BASE+1] = GDT_ENTRY_INIT(0x009a, 0, 0xffff),
166
167 [GDT_ENTRY_APMBIOS_BASE+2] = GDT_ENTRY_INIT(0x4092, 0, 0xffff),
168
169 [GDT_ENTRY_ESPFIX_SS] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
170 [GDT_ENTRY_PERCPU] = GDT_ENTRY_INIT(0xc092, 0, 0xfffff),
171#endif
172} };
173EXPORT_PER_CPU_SYMBOL_GPL(gdt_page);
174
175#ifdef CONFIG_X86_64
176static int __init x86_nopcid_setup(char *s)
177{
178
179 if (s)
180 return -EINVAL;
181
182
183 if (!boot_cpu_has(X86_FEATURE_PCID))
184 return 0;
185
186 setup_clear_cpu_cap(X86_FEATURE_PCID);
187 pr_info("nopcid: PCID feature disabled\n");
188 return 0;
189}
190early_param("nopcid", x86_nopcid_setup);
191#endif
192
193static int __init x86_noinvpcid_setup(char *s)
194{
195
196 if (s)
197 return -EINVAL;
198
199
200 if (!boot_cpu_has(X86_FEATURE_INVPCID))
201 return 0;
202
203 setup_clear_cpu_cap(X86_FEATURE_INVPCID);
204 pr_info("noinvpcid: INVPCID feature disabled\n");
205 return 0;
206}
207early_param("noinvpcid", x86_noinvpcid_setup);
208
209#ifdef CONFIG_X86_32
210static int cachesize_override = -1;
211static int disable_x86_serial_nr = 1;
212
213static int __init cachesize_setup(char *str)
214{
215 get_option(&str, &cachesize_override);
216 return 1;
217}
218__setup("cachesize=", cachesize_setup);
219
220static int __init x86_sep_setup(char *s)
221{
222 setup_clear_cpu_cap(X86_FEATURE_SEP);
223 return 1;
224}
225__setup("nosep", x86_sep_setup);
226
227
228static inline int flag_is_changeable_p(u32 flag)
229{
230 u32 f1, f2;
231
232
233
234
235
236
237
238
239 asm volatile ("pushfl \n\t"
240 "pushfl \n\t"
241 "popl %0 \n\t"
242 "movl %0, %1 \n\t"
243 "xorl %2, %0 \n\t"
244 "pushl %0 \n\t"
245 "popfl \n\t"
246 "pushfl \n\t"
247 "popl %0 \n\t"
248 "popfl \n\t"
249
250 : "=&r" (f1), "=&r" (f2)
251 : "ir" (flag));
252
253 return ((f1^f2) & flag) != 0;
254}
255
256
257int have_cpuid_p(void)
258{
259 return flag_is_changeable_p(X86_EFLAGS_ID);
260}
261
262static void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
263{
264 unsigned long lo, hi;
265
266 if (!cpu_has(c, X86_FEATURE_PN) || !disable_x86_serial_nr)
267 return;
268
269
270
271 rdmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
272 lo |= 0x200000;
273 wrmsr(MSR_IA32_BBL_CR_CTL, lo, hi);
274
275 pr_notice("CPU serial number disabled.\n");
276 clear_cpu_cap(c, X86_FEATURE_PN);
277
278
279 c->cpuid_level = cpuid_eax(0);
280}
281
282static int __init x86_serial_nr_setup(char *s)
283{
284 disable_x86_serial_nr = 0;
285 return 1;
286}
287__setup("serialnumber", x86_serial_nr_setup);
288#else
289static inline int flag_is_changeable_p(u32 flag)
290{
291 return 1;
292}
293static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
294{
295}
296#endif
297
298static __init int setup_disable_smep(char *arg)
299{
300 setup_clear_cpu_cap(X86_FEATURE_SMEP);
301 return 1;
302}
303__setup("nosmep", setup_disable_smep);
304
305static __always_inline void setup_smep(struct cpuinfo_x86 *c)
306{
307 if (cpu_has(c, X86_FEATURE_SMEP))
308 cr4_set_bits(X86_CR4_SMEP);
309}
310
311static __init int setup_disable_smap(char *arg)
312{
313 setup_clear_cpu_cap(X86_FEATURE_SMAP);
314 return 1;
315}
316__setup("nosmap", setup_disable_smap);
317
318static __always_inline void setup_smap(struct cpuinfo_x86 *c)
319{
320 unsigned long eflags = native_save_fl();
321
322
323 BUG_ON(eflags & X86_EFLAGS_AC);
324
325 if (cpu_has(c, X86_FEATURE_SMAP)) {
326#ifdef CONFIG_X86_SMAP
327 cr4_set_bits(X86_CR4_SMAP);
328#else
329 clear_cpu_cap(c, X86_FEATURE_SMAP);
330 cr4_clear_bits(X86_CR4_SMAP);
331#endif
332 }
333}
334
335static __always_inline void setup_umip(struct cpuinfo_x86 *c)
336{
337
338 if (!cpu_feature_enabled(X86_FEATURE_UMIP))
339 goto out;
340
341
342 if (!cpu_has(c, X86_FEATURE_UMIP))
343 goto out;
344
345 cr4_set_bits(X86_CR4_UMIP);
346
347 pr_info_once("x86/cpu: User Mode Instruction Prevention (UMIP) activated\n");
348
349 return;
350
351out:
352
353
354
355
356 cr4_clear_bits(X86_CR4_UMIP);
357}
358
359
360static const unsigned long cr4_pinned_mask =
361 X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_UMIP | X86_CR4_FSGSBASE;
362static DEFINE_STATIC_KEY_FALSE_RO(cr_pinning);
363static unsigned long cr4_pinned_bits __ro_after_init;
364
365void native_write_cr0(unsigned long val)
366{
367 unsigned long bits_missing = 0;
368
369set_register:
370 asm volatile("mov %0,%%cr0": "+r" (val) : : "memory");
371
372 if (static_branch_likely(&cr_pinning)) {
373 if (unlikely((val & X86_CR0_WP) != X86_CR0_WP)) {
374 bits_missing = X86_CR0_WP;
375 val |= bits_missing;
376 goto set_register;
377 }
378
379 WARN_ONCE(bits_missing, "CR0 WP bit went missing!?\n");
380 }
381}
382EXPORT_SYMBOL(native_write_cr0);
383
384void native_write_cr4(unsigned long val)
385{
386 unsigned long bits_changed = 0;
387
388set_register:
389 asm volatile("mov %0,%%cr4": "+r" (val) : : "memory");
390
391 if (static_branch_likely(&cr_pinning)) {
392 if (unlikely((val & cr4_pinned_mask) != cr4_pinned_bits)) {
393 bits_changed = (val & cr4_pinned_mask) ^ cr4_pinned_bits;
394 val = (val & ~cr4_pinned_mask) | cr4_pinned_bits;
395 goto set_register;
396 }
397
398 WARN_ONCE(bits_changed, "pinned CR4 bits changed: 0x%lx!?\n",
399 bits_changed);
400 }
401}
402#if IS_MODULE(CONFIG_LKDTM)
403EXPORT_SYMBOL_GPL(native_write_cr4);
404#endif
405
406void cr4_update_irqsoff(unsigned long set, unsigned long clear)
407{
408 unsigned long newval, cr4 = this_cpu_read(cpu_tlbstate.cr4);
409
410 lockdep_assert_irqs_disabled();
411
412 newval = (cr4 & ~clear) | set;
413 if (newval != cr4) {
414 this_cpu_write(cpu_tlbstate.cr4, newval);
415 __write_cr4(newval);
416 }
417}
418EXPORT_SYMBOL(cr4_update_irqsoff);
419
420
421unsigned long cr4_read_shadow(void)
422{
423 return this_cpu_read(cpu_tlbstate.cr4);
424}
425EXPORT_SYMBOL_GPL(cr4_read_shadow);
426
427void cr4_init(void)
428{
429 unsigned long cr4 = __read_cr4();
430
431 if (boot_cpu_has(X86_FEATURE_PCID))
432 cr4 |= X86_CR4_PCIDE;
433 if (static_branch_likely(&cr_pinning))
434 cr4 = (cr4 & ~cr4_pinned_mask) | cr4_pinned_bits;
435
436 __write_cr4(cr4);
437
438
439 this_cpu_write(cpu_tlbstate.cr4, cr4);
440}
441
442
443
444
445
446
447static void __init setup_cr_pinning(void)
448{
449 cr4_pinned_bits = this_cpu_read(cpu_tlbstate.cr4) & cr4_pinned_mask;
450 static_key_enable(&cr_pinning.key);
451}
452
453static __init int x86_nofsgsbase_setup(char *arg)
454{
455
456 if (strlen(arg))
457 return 0;
458
459
460 if (!boot_cpu_has(X86_FEATURE_FSGSBASE))
461 return 1;
462
463 setup_clear_cpu_cap(X86_FEATURE_FSGSBASE);
464 pr_info("FSGSBASE disabled via kernel command line\n");
465 return 1;
466}
467__setup("nofsgsbase", x86_nofsgsbase_setup);
468
469
470
471
472static bool pku_disabled;
473
474static __always_inline void setup_pku(struct cpuinfo_x86 *c)
475{
476 if (c == &boot_cpu_data) {
477 if (pku_disabled || !cpu_feature_enabled(X86_FEATURE_PKU))
478 return;
479
480
481
482
483 setup_force_cpu_cap(X86_FEATURE_OSPKE);
484
485 } else if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) {
486 return;
487 }
488
489 cr4_set_bits(X86_CR4_PKE);
490
491 pkru_write_default();
492}
493
494#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
495static __init int setup_disable_pku(char *arg)
496{
497
498
499
500
501
502
503
504
505
506
507
508 pr_info("x86: 'nopku' specified, disabling Memory Protection Keys\n");
509 pku_disabled = true;
510 return 1;
511}
512__setup("nopku", setup_disable_pku);
513#endif
514
515
516
517
518
519
520struct cpuid_dependent_feature {
521 u32 feature;
522 u32 level;
523};
524
525static const struct cpuid_dependent_feature
526cpuid_dependent_features[] = {
527 { X86_FEATURE_MWAIT, 0x00000005 },
528 { X86_FEATURE_DCA, 0x00000009 },
529 { X86_FEATURE_XSAVE, 0x0000000d },
530 { 0, 0 }
531};
532
533static void filter_cpuid_features(struct cpuinfo_x86 *c, bool warn)
534{
535 const struct cpuid_dependent_feature *df;
536
537 for (df = cpuid_dependent_features; df->feature; df++) {
538
539 if (!cpu_has(c, df->feature))
540 continue;
541
542
543
544
545
546
547
548 if (!((s32)df->level < 0 ?
549 (u32)df->level > (u32)c->extended_cpuid_level :
550 (s32)df->level > (s32)c->cpuid_level))
551 continue;
552
553 clear_cpu_cap(c, df->feature);
554 if (!warn)
555 continue;
556
557 pr_warn("CPU: CPU feature " X86_CAP_FMT " disabled, no CPUID level 0x%x\n",
558 x86_cap_flag(df->feature), df->level);
559 }
560}
561
562
563
564
565
566
567
568
569
570static const char *table_lookup_model(struct cpuinfo_x86 *c)
571{
572#ifdef CONFIG_X86_32
573 const struct legacy_cpu_model_info *info;
574
575 if (c->x86_model >= 16)
576 return NULL;
577
578 if (!this_cpu)
579 return NULL;
580
581 info = this_cpu->legacy_models;
582
583 while (info->family) {
584 if (info->family == c->x86)
585 return info->model_names[c->x86_model];
586 info++;
587 }
588#endif
589 return NULL;
590}
591
592
593__u32 cpu_caps_cleared[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
594__u32 cpu_caps_set[NCAPINTS + NBUGINTS] __aligned(sizeof(unsigned long));
595
596void load_percpu_segment(int cpu)
597{
598#ifdef CONFIG_X86_32
599 loadsegment(fs, __KERNEL_PERCPU);
600#else
601 __loadsegment_simple(gs, 0);
602 wrmsrl(MSR_GS_BASE, cpu_kernelmode_gs_base(cpu));
603#endif
604}
605
606#ifdef CONFIG_X86_32
607
608DEFINE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
609#endif
610
611
612void load_direct_gdt(int cpu)
613{
614 struct desc_ptr gdt_descr;
615
616 gdt_descr.address = (long)get_cpu_gdt_rw(cpu);
617 gdt_descr.size = GDT_SIZE - 1;
618 load_gdt(&gdt_descr);
619}
620EXPORT_SYMBOL_GPL(load_direct_gdt);
621
622
623void load_fixmap_gdt(int cpu)
624{
625 struct desc_ptr gdt_descr;
626
627 gdt_descr.address = (long)get_cpu_gdt_ro(cpu);
628 gdt_descr.size = GDT_SIZE - 1;
629 load_gdt(&gdt_descr);
630}
631EXPORT_SYMBOL_GPL(load_fixmap_gdt);
632
633
634
635
636
637void switch_to_new_gdt(int cpu)
638{
639
640 load_direct_gdt(cpu);
641
642 load_percpu_segment(cpu);
643}
644
645static const struct cpu_dev *cpu_devs[X86_VENDOR_NUM] = {};
646
647static void get_model_name(struct cpuinfo_x86 *c)
648{
649 unsigned int *v;
650 char *p, *q, *s;
651
652 if (c->extended_cpuid_level < 0x80000004)
653 return;
654
655 v = (unsigned int *)c->x86_model_id;
656 cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]);
657 cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]);
658 cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]);
659 c->x86_model_id[48] = 0;
660
661
662 p = q = s = &c->x86_model_id[0];
663
664 while (*p == ' ')
665 p++;
666
667 while (*p) {
668
669 if (!isspace(*p))
670 s = q;
671
672 *q++ = *p++;
673 }
674
675 *(s + 1) = '\0';
676}
677
678void detect_num_cpu_cores(struct cpuinfo_x86 *c)
679{
680 unsigned int eax, ebx, ecx, edx;
681
682 c->x86_max_cores = 1;
683 if (!IS_ENABLED(CONFIG_SMP) || c->cpuid_level < 4)
684 return;
685
686 cpuid_count(4, 0, &eax, &ebx, &ecx, &edx);
687 if (eax & 0x1f)
688 c->x86_max_cores = (eax >> 26) + 1;
689}
690
691void cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
692{
693 unsigned int n, dummy, ebx, ecx, edx, l2size;
694
695 n = c->extended_cpuid_level;
696
697 if (n >= 0x80000005) {
698 cpuid(0x80000005, &dummy, &ebx, &ecx, &edx);
699 c->x86_cache_size = (ecx>>24) + (edx>>24);
700#ifdef CONFIG_X86_64
701
702 c->x86_tlbsize = 0;
703#endif
704 }
705
706 if (n < 0x80000006)
707 return;
708
709 cpuid(0x80000006, &dummy, &ebx, &ecx, &edx);
710 l2size = ecx >> 16;
711
712#ifdef CONFIG_X86_64
713 c->x86_tlbsize += ((ebx >> 16) & 0xfff) + (ebx & 0xfff);
714#else
715
716 if (this_cpu->legacy_cache_size)
717 l2size = this_cpu->legacy_cache_size(c, l2size);
718
719
720 if (cachesize_override != -1)
721 l2size = cachesize_override;
722
723 if (l2size == 0)
724 return;
725#endif
726
727 c->x86_cache_size = l2size;
728}
729
730u16 __read_mostly tlb_lli_4k[NR_INFO];
731u16 __read_mostly tlb_lli_2m[NR_INFO];
732u16 __read_mostly tlb_lli_4m[NR_INFO];
733u16 __read_mostly tlb_lld_4k[NR_INFO];
734u16 __read_mostly tlb_lld_2m[NR_INFO];
735u16 __read_mostly tlb_lld_4m[NR_INFO];
736u16 __read_mostly tlb_lld_1g[NR_INFO];
737
738static void cpu_detect_tlb(struct cpuinfo_x86 *c)
739{
740 if (this_cpu->c_detect_tlb)
741 this_cpu->c_detect_tlb(c);
742
743 pr_info("Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
744 tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
745 tlb_lli_4m[ENTRIES]);
746
747 pr_info("Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d, 1GB %d\n",
748 tlb_lld_4k[ENTRIES], tlb_lld_2m[ENTRIES],
749 tlb_lld_4m[ENTRIES], tlb_lld_1g[ENTRIES]);
750}
751
752int detect_ht_early(struct cpuinfo_x86 *c)
753{
754#ifdef CONFIG_SMP
755 u32 eax, ebx, ecx, edx;
756
757 if (!cpu_has(c, X86_FEATURE_HT))
758 return -1;
759
760 if (cpu_has(c, X86_FEATURE_CMP_LEGACY))
761 return -1;
762
763 if (cpu_has(c, X86_FEATURE_XTOPOLOGY))
764 return -1;
765
766 cpuid(1, &eax, &ebx, &ecx, &edx);
767
768 smp_num_siblings = (ebx & 0xff0000) >> 16;
769 if (smp_num_siblings == 1)
770 pr_info_once("CPU0: Hyper-Threading is disabled\n");
771#endif
772 return 0;
773}
774
775void detect_ht(struct cpuinfo_x86 *c)
776{
777#ifdef CONFIG_SMP
778 int index_msb, core_bits;
779
780 if (detect_ht_early(c) < 0)
781 return;
782
783 index_msb = get_count_order(smp_num_siblings);
784 c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, index_msb);
785
786 smp_num_siblings = smp_num_siblings / c->x86_max_cores;
787
788 index_msb = get_count_order(smp_num_siblings);
789
790 core_bits = get_count_order(c->x86_max_cores);
791
792 c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, index_msb) &
793 ((1 << core_bits) - 1);
794#endif
795}
796
797static void get_cpu_vendor(struct cpuinfo_x86 *c)
798{
799 char *v = c->x86_vendor_id;
800 int i;
801
802 for (i = 0; i < X86_VENDOR_NUM; i++) {
803 if (!cpu_devs[i])
804 break;
805
806 if (!strcmp(v, cpu_devs[i]->c_ident[0]) ||
807 (cpu_devs[i]->c_ident[1] &&
808 !strcmp(v, cpu_devs[i]->c_ident[1]))) {
809
810 this_cpu = cpu_devs[i];
811 c->x86_vendor = this_cpu->c_x86_vendor;
812 return;
813 }
814 }
815
816 pr_err_once("CPU: vendor_id '%s' unknown, using generic init.\n" \
817 "CPU: Your system may be unstable.\n", v);
818
819 c->x86_vendor = X86_VENDOR_UNKNOWN;
820 this_cpu = &default_cpu;
821}
822
823void cpu_detect(struct cpuinfo_x86 *c)
824{
825
826 cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
827 (unsigned int *)&c->x86_vendor_id[0],
828 (unsigned int *)&c->x86_vendor_id[8],
829 (unsigned int *)&c->x86_vendor_id[4]);
830
831 c->x86 = 4;
832
833 if (c->cpuid_level >= 0x00000001) {
834 u32 junk, tfms, cap0, misc;
835
836 cpuid(0x00000001, &tfms, &misc, &junk, &cap0);
837 c->x86 = x86_family(tfms);
838 c->x86_model = x86_model(tfms);
839 c->x86_stepping = x86_stepping(tfms);
840
841 if (cap0 & (1<<19)) {
842 c->x86_clflush_size = ((misc >> 8) & 0xff) * 8;
843 c->x86_cache_alignment = c->x86_clflush_size;
844 }
845 }
846}
847
848static void apply_forced_caps(struct cpuinfo_x86 *c)
849{
850 int i;
851
852 for (i = 0; i < NCAPINTS + NBUGINTS; i++) {
853 c->x86_capability[i] &= ~cpu_caps_cleared[i];
854 c->x86_capability[i] |= cpu_caps_set[i];
855 }
856}
857
858static void init_speculation_control(struct cpuinfo_x86 *c)
859{
860
861
862
863
864
865
866 if (cpu_has(c, X86_FEATURE_SPEC_CTRL)) {
867 set_cpu_cap(c, X86_FEATURE_IBRS);
868 set_cpu_cap(c, X86_FEATURE_IBPB);
869 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
870 }
871
872 if (cpu_has(c, X86_FEATURE_INTEL_STIBP))
873 set_cpu_cap(c, X86_FEATURE_STIBP);
874
875 if (cpu_has(c, X86_FEATURE_SPEC_CTRL_SSBD) ||
876 cpu_has(c, X86_FEATURE_VIRT_SSBD))
877 set_cpu_cap(c, X86_FEATURE_SSBD);
878
879 if (cpu_has(c, X86_FEATURE_AMD_IBRS)) {
880 set_cpu_cap(c, X86_FEATURE_IBRS);
881 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
882 }
883
884 if (cpu_has(c, X86_FEATURE_AMD_IBPB))
885 set_cpu_cap(c, X86_FEATURE_IBPB);
886
887 if (cpu_has(c, X86_FEATURE_AMD_STIBP)) {
888 set_cpu_cap(c, X86_FEATURE_STIBP);
889 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
890 }
891
892 if (cpu_has(c, X86_FEATURE_AMD_SSBD)) {
893 set_cpu_cap(c, X86_FEATURE_SSBD);
894 set_cpu_cap(c, X86_FEATURE_MSR_SPEC_CTRL);
895 clear_cpu_cap(c, X86_FEATURE_VIRT_SSBD);
896 }
897}
898
899void get_cpu_cap(struct cpuinfo_x86 *c)
900{
901 u32 eax, ebx, ecx, edx;
902
903
904 if (c->cpuid_level >= 0x00000001) {
905 cpuid(0x00000001, &eax, &ebx, &ecx, &edx);
906
907 c->x86_capability[CPUID_1_ECX] = ecx;
908 c->x86_capability[CPUID_1_EDX] = edx;
909 }
910
911
912 if (c->cpuid_level >= 0x00000006)
913 c->x86_capability[CPUID_6_EAX] = cpuid_eax(0x00000006);
914
915
916 if (c->cpuid_level >= 0x00000007) {
917 cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
918 c->x86_capability[CPUID_7_0_EBX] = ebx;
919 c->x86_capability[CPUID_7_ECX] = ecx;
920 c->x86_capability[CPUID_7_EDX] = edx;
921
922
923 if (eax >= 1) {
924 cpuid_count(0x00000007, 1, &eax, &ebx, &ecx, &edx);
925 c->x86_capability[CPUID_7_1_EAX] = eax;
926 }
927 }
928
929
930 if (c->cpuid_level >= 0x0000000d) {
931 cpuid_count(0x0000000d, 1, &eax, &ebx, &ecx, &edx);
932
933 c->x86_capability[CPUID_D_1_EAX] = eax;
934 }
935
936
937 eax = cpuid_eax(0x80000000);
938 c->extended_cpuid_level = eax;
939
940 if ((eax & 0xffff0000) == 0x80000000) {
941 if (eax >= 0x80000001) {
942 cpuid(0x80000001, &eax, &ebx, &ecx, &edx);
943
944 c->x86_capability[CPUID_8000_0001_ECX] = ecx;
945 c->x86_capability[CPUID_8000_0001_EDX] = edx;
946 }
947 }
948
949 if (c->extended_cpuid_level >= 0x80000007) {
950 cpuid(0x80000007, &eax, &ebx, &ecx, &edx);
951
952 c->x86_capability[CPUID_8000_0007_EBX] = ebx;
953 c->x86_power = edx;
954 }
955
956 if (c->extended_cpuid_level >= 0x80000008) {
957 cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
958 c->x86_capability[CPUID_8000_0008_EBX] = ebx;
959 }
960
961 if (c->extended_cpuid_level >= 0x8000000a)
962 c->x86_capability[CPUID_8000_000A_EDX] = cpuid_edx(0x8000000a);
963
964 if (c->extended_cpuid_level >= 0x8000001f)
965 c->x86_capability[CPUID_8000_001F_EAX] = cpuid_eax(0x8000001f);
966
967 init_scattered_cpuid_features(c);
968 init_speculation_control(c);
969
970
971
972
973
974
975 apply_forced_caps(c);
976}
977
978void get_cpu_address_sizes(struct cpuinfo_x86 *c)
979{
980 u32 eax, ebx, ecx, edx;
981
982 if (c->extended_cpuid_level >= 0x80000008) {
983 cpuid(0x80000008, &eax, &ebx, &ecx, &edx);
984
985 c->x86_virt_bits = (eax >> 8) & 0xff;
986 c->x86_phys_bits = eax & 0xff;
987 }
988#ifdef CONFIG_X86_32
989 else if (cpu_has(c, X86_FEATURE_PAE) || cpu_has(c, X86_FEATURE_PSE36))
990 c->x86_phys_bits = 36;
991#endif
992 c->x86_cache_bits = c->x86_phys_bits;
993}
994
995static void identify_cpu_without_cpuid(struct cpuinfo_x86 *c)
996{
997#ifdef CONFIG_X86_32
998 int i;
999
1000
1001
1002
1003
1004 if (flag_is_changeable_p(X86_EFLAGS_AC))
1005 c->x86 = 4;
1006 else
1007 c->x86 = 3;
1008
1009 for (i = 0; i < X86_VENDOR_NUM; i++)
1010 if (cpu_devs[i] && cpu_devs[i]->c_identify) {
1011 c->x86_vendor_id[0] = 0;
1012 cpu_devs[i]->c_identify(c);
1013 if (c->x86_vendor_id[0]) {
1014 get_cpu_vendor(c);
1015 break;
1016 }
1017 }
1018#endif
1019}
1020
1021#define NO_SPECULATION BIT(0)
1022#define NO_MELTDOWN BIT(1)
1023#define NO_SSB BIT(2)
1024#define NO_L1TF BIT(3)
1025#define NO_MDS BIT(4)
1026#define MSBDS_ONLY BIT(5)
1027#define NO_SWAPGS BIT(6)
1028#define NO_ITLB_MULTIHIT BIT(7)
1029#define NO_SPECTRE_V2 BIT(8)
1030
1031#define VULNWL(vendor, family, model, whitelist) \
1032 X86_MATCH_VENDOR_FAM_MODEL(vendor, family, model, whitelist)
1033
1034#define VULNWL_INTEL(model, whitelist) \
1035 VULNWL(INTEL, 6, INTEL_FAM6_##model, whitelist)
1036
1037#define VULNWL_AMD(family, whitelist) \
1038 VULNWL(AMD, family, X86_MODEL_ANY, whitelist)
1039
1040#define VULNWL_HYGON(family, whitelist) \
1041 VULNWL(HYGON, family, X86_MODEL_ANY, whitelist)
1042
1043static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = {
1044 VULNWL(ANY, 4, X86_MODEL_ANY, NO_SPECULATION),
1045 VULNWL(CENTAUR, 5, X86_MODEL_ANY, NO_SPECULATION),
1046 VULNWL(INTEL, 5, X86_MODEL_ANY, NO_SPECULATION),
1047 VULNWL(NSC, 5, X86_MODEL_ANY, NO_SPECULATION),
1048
1049
1050 VULNWL_INTEL(ATOM_SALTWELL, NO_SPECULATION | NO_ITLB_MULTIHIT),
1051 VULNWL_INTEL(ATOM_SALTWELL_TABLET, NO_SPECULATION | NO_ITLB_MULTIHIT),
1052 VULNWL_INTEL(ATOM_SALTWELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT),
1053 VULNWL_INTEL(ATOM_BONNELL, NO_SPECULATION | NO_ITLB_MULTIHIT),
1054 VULNWL_INTEL(ATOM_BONNELL_MID, NO_SPECULATION | NO_ITLB_MULTIHIT),
1055
1056 VULNWL_INTEL(ATOM_SILVERMONT, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1057 VULNWL_INTEL(ATOM_SILVERMONT_D, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1058 VULNWL_INTEL(ATOM_SILVERMONT_MID, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1059 VULNWL_INTEL(ATOM_AIRMONT, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1060 VULNWL_INTEL(XEON_PHI_KNL, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1061 VULNWL_INTEL(XEON_PHI_KNM, NO_SSB | NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1062
1063 VULNWL_INTEL(CORE_YONAH, NO_SSB),
1064
1065 VULNWL_INTEL(ATOM_AIRMONT_MID, NO_L1TF | MSBDS_ONLY | NO_SWAPGS | NO_ITLB_MULTIHIT),
1066 VULNWL_INTEL(ATOM_AIRMONT_NP, NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1067
1068 VULNWL_INTEL(ATOM_GOLDMONT, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1069 VULNWL_INTEL(ATOM_GOLDMONT_D, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1070 VULNWL_INTEL(ATOM_GOLDMONT_PLUS, NO_MDS | NO_L1TF | NO_SWAPGS | NO_ITLB_MULTIHIT),
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080 VULNWL_INTEL(ATOM_TREMONT_D, NO_ITLB_MULTIHIT),
1081
1082
1083 VULNWL_AMD(0x0f, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT),
1084 VULNWL_AMD(0x10, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT),
1085 VULNWL_AMD(0x11, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT),
1086 VULNWL_AMD(0x12, NO_MELTDOWN | NO_SSB | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT),
1087
1088
1089 VULNWL_AMD(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT),
1090 VULNWL_HYGON(X86_FAMILY_ANY, NO_MELTDOWN | NO_L1TF | NO_MDS | NO_SWAPGS | NO_ITLB_MULTIHIT),
1091
1092
1093 VULNWL(CENTAUR, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS),
1094 VULNWL(ZHAOXIN, 7, X86_MODEL_ANY, NO_SPECTRE_V2 | NO_SWAPGS),
1095 {}
1096};
1097
1098#define VULNBL_INTEL_STEPPINGS(model, steppings, issues) \
1099 X86_MATCH_VENDOR_FAM_MODEL_STEPPINGS_FEATURE(INTEL, 6, \
1100 INTEL_FAM6_##model, steppings, \
1101 X86_FEATURE_ANY, issues)
1102
1103#define SRBDS BIT(0)
1104
1105static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = {
1106 VULNBL_INTEL_STEPPINGS(IVYBRIDGE, X86_STEPPING_ANY, SRBDS),
1107 VULNBL_INTEL_STEPPINGS(HASWELL, X86_STEPPING_ANY, SRBDS),
1108 VULNBL_INTEL_STEPPINGS(HASWELL_L, X86_STEPPING_ANY, SRBDS),
1109 VULNBL_INTEL_STEPPINGS(HASWELL_G, X86_STEPPING_ANY, SRBDS),
1110 VULNBL_INTEL_STEPPINGS(BROADWELL_G, X86_STEPPING_ANY, SRBDS),
1111 VULNBL_INTEL_STEPPINGS(BROADWELL, X86_STEPPING_ANY, SRBDS),
1112 VULNBL_INTEL_STEPPINGS(SKYLAKE_L, X86_STEPPING_ANY, SRBDS),
1113 VULNBL_INTEL_STEPPINGS(SKYLAKE, X86_STEPPING_ANY, SRBDS),
1114 VULNBL_INTEL_STEPPINGS(KABYLAKE_L, X86_STEPPINGS(0x0, 0xC), SRBDS),
1115 VULNBL_INTEL_STEPPINGS(KABYLAKE, X86_STEPPINGS(0x0, 0xD), SRBDS),
1116 {}
1117};
1118
1119static bool __init cpu_matches(const struct x86_cpu_id *table, unsigned long which)
1120{
1121 const struct x86_cpu_id *m = x86_match_cpu(table);
1122
1123 return m && !!(m->driver_data & which);
1124}
1125
1126u64 x86_read_arch_cap_msr(void)
1127{
1128 u64 ia32_cap = 0;
1129
1130 if (boot_cpu_has(X86_FEATURE_ARCH_CAPABILITIES))
1131 rdmsrl(MSR_IA32_ARCH_CAPABILITIES, ia32_cap);
1132
1133 return ia32_cap;
1134}
1135
1136static void __init cpu_set_bug_bits(struct cpuinfo_x86 *c)
1137{
1138 u64 ia32_cap = x86_read_arch_cap_msr();
1139
1140
1141 if (!cpu_matches(cpu_vuln_whitelist, NO_ITLB_MULTIHIT) &&
1142 !(ia32_cap & ARCH_CAP_PSCHANGE_MC_NO))
1143 setup_force_cpu_bug(X86_BUG_ITLB_MULTIHIT);
1144
1145 if (cpu_matches(cpu_vuln_whitelist, NO_SPECULATION))
1146 return;
1147
1148 setup_force_cpu_bug(X86_BUG_SPECTRE_V1);
1149
1150 if (!cpu_matches(cpu_vuln_whitelist, NO_SPECTRE_V2))
1151 setup_force_cpu_bug(X86_BUG_SPECTRE_V2);
1152
1153 if (!cpu_matches(cpu_vuln_whitelist, NO_SSB) &&
1154 !(ia32_cap & ARCH_CAP_SSB_NO) &&
1155 !cpu_has(c, X86_FEATURE_AMD_SSB_NO))
1156 setup_force_cpu_bug(X86_BUG_SPEC_STORE_BYPASS);
1157
1158 if (ia32_cap & ARCH_CAP_IBRS_ALL)
1159 setup_force_cpu_cap(X86_FEATURE_IBRS_ENHANCED);
1160
1161 if (!cpu_matches(cpu_vuln_whitelist, NO_MDS) &&
1162 !(ia32_cap & ARCH_CAP_MDS_NO)) {
1163 setup_force_cpu_bug(X86_BUG_MDS);
1164 if (cpu_matches(cpu_vuln_whitelist, MSBDS_ONLY))
1165 setup_force_cpu_bug(X86_BUG_MSBDS_ONLY);
1166 }
1167
1168 if (!cpu_matches(cpu_vuln_whitelist, NO_SWAPGS))
1169 setup_force_cpu_bug(X86_BUG_SWAPGS);
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181 if (!(ia32_cap & ARCH_CAP_TAA_NO) &&
1182 (cpu_has(c, X86_FEATURE_RTM) ||
1183 (ia32_cap & ARCH_CAP_TSX_CTRL_MSR)))
1184 setup_force_cpu_bug(X86_BUG_TAA);
1185
1186
1187
1188
1189
1190 if ((cpu_has(c, X86_FEATURE_RDRAND) ||
1191 cpu_has(c, X86_FEATURE_RDSEED)) &&
1192 cpu_matches(cpu_vuln_blacklist, SRBDS))
1193 setup_force_cpu_bug(X86_BUG_SRBDS);
1194
1195 if (cpu_matches(cpu_vuln_whitelist, NO_MELTDOWN))
1196 return;
1197
1198
1199 if (ia32_cap & ARCH_CAP_RDCL_NO)
1200 return;
1201
1202 setup_force_cpu_bug(X86_BUG_CPU_MELTDOWN);
1203
1204 if (cpu_matches(cpu_vuln_whitelist, NO_L1TF))
1205 return;
1206
1207 setup_force_cpu_bug(X86_BUG_L1TF);
1208}
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219static void detect_nopl(void)
1220{
1221#ifdef CONFIG_X86_32
1222 setup_clear_cpu_cap(X86_FEATURE_NOPL);
1223#else
1224 setup_force_cpu_cap(X86_FEATURE_NOPL);
1225#endif
1226}
1227
1228
1229
1230
1231
1232static void __init cpu_parse_early_param(void)
1233{
1234 char arg[128];
1235 char *argptr = arg;
1236 int arglen, res, bit;
1237
1238#ifdef CONFIG_X86_32
1239 if (cmdline_find_option_bool(boot_command_line, "no387"))
1240#ifdef CONFIG_MATH_EMULATION
1241 setup_clear_cpu_cap(X86_FEATURE_FPU);
1242#else
1243 pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
1244#endif
1245
1246 if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
1247 setup_clear_cpu_cap(X86_FEATURE_FXSR);
1248#endif
1249
1250 if (cmdline_find_option_bool(boot_command_line, "noxsave"))
1251 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
1252
1253 if (cmdline_find_option_bool(boot_command_line, "noxsaveopt"))
1254 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT);
1255
1256 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1257 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1258
1259 arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1260 if (arglen <= 0)
1261 return;
1262
1263 pr_info("Clearing CPUID bits:");
1264 do {
1265 res = get_option(&argptr, &bit);
1266 if (res == 0 || res == 3)
1267 break;
1268
1269
1270 if (res == 1 && arglen >= sizeof(arg))
1271 break;
1272
1273 if (bit >= 0 && bit < NCAPINTS * 32) {
1274 pr_cont(" " X86_CAP_FMT, x86_cap_flag(bit));
1275 setup_clear_cpu_cap(bit);
1276 }
1277 } while (res == 2);
1278 pr_cont("\n");
1279}
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290static void __init early_identify_cpu(struct cpuinfo_x86 *c)
1291{
1292#ifdef CONFIG_X86_64
1293 c->x86_clflush_size = 64;
1294 c->x86_phys_bits = 36;
1295 c->x86_virt_bits = 48;
1296#else
1297 c->x86_clflush_size = 32;
1298 c->x86_phys_bits = 32;
1299 c->x86_virt_bits = 32;
1300#endif
1301 c->x86_cache_alignment = c->x86_clflush_size;
1302
1303 memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1304 c->extended_cpuid_level = 0;
1305
1306 if (!have_cpuid_p())
1307 identify_cpu_without_cpuid(c);
1308
1309
1310 if (have_cpuid_p()) {
1311 cpu_detect(c);
1312 get_cpu_vendor(c);
1313 get_cpu_cap(c);
1314 get_cpu_address_sizes(c);
1315 setup_force_cpu_cap(X86_FEATURE_CPUID);
1316 cpu_parse_early_param();
1317
1318 if (this_cpu->c_early_init)
1319 this_cpu->c_early_init(c);
1320
1321 c->cpu_index = 0;
1322 filter_cpuid_features(c, false);
1323
1324 if (this_cpu->c_bsp_init)
1325 this_cpu->c_bsp_init(c);
1326 } else {
1327 setup_clear_cpu_cap(X86_FEATURE_CPUID);
1328 }
1329
1330 setup_force_cpu_cap(X86_FEATURE_ALWAYS);
1331
1332 cpu_set_bug_bits(c);
1333
1334 sld_setup(c);
1335
1336 fpu__init_system(c);
1337
1338 init_sigframe_size();
1339
1340#ifdef CONFIG_X86_32
1341
1342
1343
1344
1345 setup_clear_cpu_cap(X86_FEATURE_PCID);
1346#endif
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360 if (!pgtable_l5_enabled())
1361 setup_clear_cpu_cap(X86_FEATURE_LA57);
1362
1363 detect_nopl();
1364}
1365
1366void __init early_cpu_init(void)
1367{
1368 const struct cpu_dev *const *cdev;
1369 int count = 0;
1370
1371#ifdef CONFIG_PROCESSOR_SELECT
1372 pr_info("KERNEL supported cpus:\n");
1373#endif
1374
1375 for (cdev = __x86_cpu_dev_start; cdev < __x86_cpu_dev_end; cdev++) {
1376 const struct cpu_dev *cpudev = *cdev;
1377
1378 if (count >= X86_VENDOR_NUM)
1379 break;
1380 cpu_devs[count] = cpudev;
1381 count++;
1382
1383#ifdef CONFIG_PROCESSOR_SELECT
1384 {
1385 unsigned int j;
1386
1387 for (j = 0; j < 2; j++) {
1388 if (!cpudev->c_ident[j])
1389 continue;
1390 pr_info(" %s %s\n", cpudev->c_vendor,
1391 cpudev->c_ident[j]);
1392 }
1393 }
1394#endif
1395 }
1396 early_identify_cpu(&boot_cpu_data);
1397}
1398
1399static void detect_null_seg_behavior(struct cpuinfo_x86 *c)
1400{
1401#ifdef CONFIG_X86_64
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417 unsigned long old_base, tmp;
1418 rdmsrl(MSR_FS_BASE, old_base);
1419 wrmsrl(MSR_FS_BASE, 1);
1420 loadsegment(fs, 0);
1421 rdmsrl(MSR_FS_BASE, tmp);
1422 if (tmp != 0)
1423 set_cpu_bug(c, X86_BUG_NULL_SEG);
1424 wrmsrl(MSR_FS_BASE, old_base);
1425#endif
1426}
1427
1428static void generic_identify(struct cpuinfo_x86 *c)
1429{
1430 c->extended_cpuid_level = 0;
1431
1432 if (!have_cpuid_p())
1433 identify_cpu_without_cpuid(c);
1434
1435
1436 if (!have_cpuid_p())
1437 return;
1438
1439 cpu_detect(c);
1440
1441 get_cpu_vendor(c);
1442
1443 get_cpu_cap(c);
1444
1445 get_cpu_address_sizes(c);
1446
1447 if (c->cpuid_level >= 0x00000001) {
1448 c->initial_apicid = (cpuid_ebx(1) >> 24) & 0xFF;
1449#ifdef CONFIG_X86_32
1450# ifdef CONFIG_SMP
1451 c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1452# else
1453 c->apicid = c->initial_apicid;
1454# endif
1455#endif
1456 c->phys_proc_id = c->initial_apicid;
1457 }
1458
1459 get_model_name(c);
1460
1461 detect_null_seg_behavior(c);
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476#ifdef CONFIG_X86_32
1477 set_cpu_bug(c, X86_BUG_ESPFIX);
1478#endif
1479}
1480
1481
1482
1483
1484
1485static void validate_apic_and_package_id(struct cpuinfo_x86 *c)
1486{
1487#ifdef CONFIG_SMP
1488 unsigned int apicid, cpu = smp_processor_id();
1489
1490 apicid = apic->cpu_present_to_apicid(cpu);
1491
1492 if (apicid != c->apicid) {
1493 pr_err(FW_BUG "CPU%u: APIC id mismatch. Firmware: %x APIC: %x\n",
1494 cpu, apicid, c->initial_apicid);
1495 }
1496 BUG_ON(topology_update_package_map(c->phys_proc_id, cpu));
1497 BUG_ON(topology_update_die_map(c->cpu_die_id, cpu));
1498#else
1499 c->logical_proc_id = 0;
1500#endif
1501}
1502
1503
1504
1505
1506static void identify_cpu(struct cpuinfo_x86 *c)
1507{
1508 int i;
1509
1510 c->loops_per_jiffy = loops_per_jiffy;
1511 c->x86_cache_size = 0;
1512 c->x86_vendor = X86_VENDOR_UNKNOWN;
1513 c->x86_model = c->x86_stepping = 0;
1514 c->x86_vendor_id[0] = '\0';
1515 c->x86_model_id[0] = '\0';
1516 c->x86_max_cores = 1;
1517 c->x86_coreid_bits = 0;
1518 c->cu_id = 0xff;
1519#ifdef CONFIG_X86_64
1520 c->x86_clflush_size = 64;
1521 c->x86_phys_bits = 36;
1522 c->x86_virt_bits = 48;
1523#else
1524 c->cpuid_level = -1;
1525 c->x86_clflush_size = 32;
1526 c->x86_phys_bits = 32;
1527 c->x86_virt_bits = 32;
1528#endif
1529 c->x86_cache_alignment = c->x86_clflush_size;
1530 memset(&c->x86_capability, 0, sizeof(c->x86_capability));
1531#ifdef CONFIG_X86_VMX_FEATURE_NAMES
1532 memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
1533#endif
1534
1535 generic_identify(c);
1536
1537 if (this_cpu->c_identify)
1538 this_cpu->c_identify(c);
1539
1540
1541 apply_forced_caps(c);
1542
1543#ifdef CONFIG_X86_64
1544 c->apicid = apic->phys_pkg_id(c->initial_apicid, 0);
1545#endif
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557 if (this_cpu->c_init)
1558 this_cpu->c_init(c);
1559
1560
1561 squash_the_stupid_serial_number(c);
1562
1563
1564 setup_smep(c);
1565 setup_smap(c);
1566 setup_umip(c);
1567
1568
1569 if (cpu_has(c, X86_FEATURE_FSGSBASE)) {
1570 cr4_set_bits(X86_CR4_FSGSBASE);
1571 elf_hwcap2 |= HWCAP2_FSGSBASE;
1572 }
1573
1574
1575
1576
1577
1578
1579
1580 filter_cpuid_features(c, true);
1581
1582
1583 if (!c->x86_model_id[0]) {
1584 const char *p;
1585 p = table_lookup_model(c);
1586 if (p)
1587 strcpy(c->x86_model_id, p);
1588 else
1589
1590 sprintf(c->x86_model_id, "%02x/%02x",
1591 c->x86, c->x86_model);
1592 }
1593
1594#ifdef CONFIG_X86_64
1595 detect_ht(c);
1596#endif
1597
1598 x86_init_rdrand(c);
1599 setup_pku(c);
1600
1601
1602
1603
1604
1605 apply_forced_caps(c);
1606
1607
1608
1609
1610
1611
1612
1613 if (c != &boot_cpu_data) {
1614
1615 for (i = 0; i < NCAPINTS; i++)
1616 boot_cpu_data.x86_capability[i] &= c->x86_capability[i];
1617
1618
1619 for (i = NCAPINTS; i < NCAPINTS + NBUGINTS; i++)
1620 c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
1621 }
1622
1623
1624 mcheck_cpu_init(c);
1625
1626 select_idle_routine(c);
1627
1628#ifdef CONFIG_NUMA
1629 numa_add_cpu(smp_processor_id());
1630#endif
1631}
1632
1633
1634
1635
1636
1637#ifdef CONFIG_X86_32
1638void enable_sep_cpu(void)
1639{
1640 struct tss_struct *tss;
1641 int cpu;
1642
1643 if (!boot_cpu_has(X86_FEATURE_SEP))
1644 return;
1645
1646 cpu = get_cpu();
1647 tss = &per_cpu(cpu_tss_rw, cpu);
1648
1649
1650
1651
1652
1653
1654 tss->x86_tss.ss1 = __KERNEL_CS;
1655 wrmsr(MSR_IA32_SYSENTER_CS, tss->x86_tss.ss1, 0);
1656 wrmsr(MSR_IA32_SYSENTER_ESP, (unsigned long)(cpu_entry_stack(cpu) + 1), 0);
1657 wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long)entry_SYSENTER_32, 0);
1658
1659 put_cpu();
1660}
1661#endif
1662
1663void __init identify_boot_cpu(void)
1664{
1665 identify_cpu(&boot_cpu_data);
1666#ifdef CONFIG_X86_32
1667 sysenter_setup();
1668 enable_sep_cpu();
1669#endif
1670 cpu_detect_tlb(&boot_cpu_data);
1671 setup_cr_pinning();
1672
1673 tsx_init();
1674}
1675
1676void identify_secondary_cpu(struct cpuinfo_x86 *c)
1677{
1678 BUG_ON(c == &boot_cpu_data);
1679 identify_cpu(c);
1680#ifdef CONFIG_X86_32
1681 enable_sep_cpu();
1682#endif
1683 mtrr_ap_init();
1684 validate_apic_and_package_id(c);
1685 x86_spec_ctrl_setup_ap();
1686 update_srbds_msr();
1687}
1688
1689static __init int setup_noclflush(char *arg)
1690{
1691 setup_clear_cpu_cap(X86_FEATURE_CLFLUSH);
1692 setup_clear_cpu_cap(X86_FEATURE_CLFLUSHOPT);
1693 return 1;
1694}
1695__setup("noclflush", setup_noclflush);
1696
1697void print_cpu_info(struct cpuinfo_x86 *c)
1698{
1699 const char *vendor = NULL;
1700
1701 if (c->x86_vendor < X86_VENDOR_NUM) {
1702 vendor = this_cpu->c_vendor;
1703 } else {
1704 if (c->cpuid_level >= 0)
1705 vendor = c->x86_vendor_id;
1706 }
1707
1708 if (vendor && !strstr(c->x86_model_id, vendor))
1709 pr_cont("%s ", vendor);
1710
1711 if (c->x86_model_id[0])
1712 pr_cont("%s", c->x86_model_id);
1713 else
1714 pr_cont("%d86", c->x86);
1715
1716 pr_cont(" (family: 0x%x, model: 0x%x", c->x86, c->x86_model);
1717
1718 if (c->x86_stepping || c->cpuid_level >= 0)
1719 pr_cont(", stepping: 0x%x)\n", c->x86_stepping);
1720 else
1721 pr_cont(")\n");
1722}
1723
1724
1725
1726
1727
1728static __init int setup_clearcpuid(char *arg)
1729{
1730 return 1;
1731}
1732__setup("clearcpuid=", setup_clearcpuid);
1733
1734#ifdef CONFIG_X86_64
1735DEFINE_PER_CPU_FIRST(struct fixed_percpu_data,
1736 fixed_percpu_data) __aligned(PAGE_SIZE) __visible;
1737EXPORT_PER_CPU_SYMBOL_GPL(fixed_percpu_data);
1738
1739
1740
1741
1742
1743DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
1744 &init_task;
1745EXPORT_PER_CPU_SYMBOL(current_task);
1746
1747DEFINE_PER_CPU(void *, hardirq_stack_ptr);
1748DEFINE_PER_CPU(bool, hardirq_stack_inuse);
1749
1750DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
1751EXPORT_PER_CPU_SYMBOL(__preempt_count);
1752
1753DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
1754
1755
1756void syscall_init(void)
1757{
1758 wrmsr(MSR_STAR, 0, (__USER32_CS << 16) | __KERNEL_CS);
1759 wrmsrl(MSR_LSTAR, (unsigned long)entry_SYSCALL_64);
1760
1761#ifdef CONFIG_IA32_EMULATION
1762 wrmsrl(MSR_CSTAR, (unsigned long)entry_SYSCALL_compat);
1763
1764
1765
1766
1767
1768
1769 wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
1770 wrmsrl_safe(MSR_IA32_SYSENTER_ESP,
1771 (unsigned long)(cpu_entry_stack(smp_processor_id()) + 1));
1772 wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)entry_SYSENTER_compat);
1773#else
1774 wrmsrl(MSR_CSTAR, (unsigned long)ignore_sysret);
1775 wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)GDT_ENTRY_INVALID_SEG);
1776 wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
1777 wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL);
1778#endif
1779
1780
1781
1782
1783
1784 wrmsrl(MSR_SYSCALL_MASK,
1785 X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF|
1786 X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF|
1787 X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF|
1788 X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF|
1789 X86_EFLAGS_AC|X86_EFLAGS_ID);
1790}
1791
1792#else
1793
1794DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
1795EXPORT_PER_CPU_SYMBOL(current_task);
1796DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT;
1797EXPORT_PER_CPU_SYMBOL(__preempt_count);
1798
1799
1800
1801
1802
1803
1804DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
1805 (unsigned long)&init_thread_union + THREAD_SIZE;
1806EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
1807
1808#ifdef CONFIG_STACKPROTECTOR
1809DEFINE_PER_CPU(unsigned long, __stack_chk_guard);
1810EXPORT_PER_CPU_SYMBOL(__stack_chk_guard);
1811#endif
1812
1813#endif
1814
1815
1816
1817
1818static void clear_all_debug_regs(void)
1819{
1820 int i;
1821
1822 for (i = 0; i < 8; i++) {
1823
1824 if ((i == 4) || (i == 5))
1825 continue;
1826
1827 set_debugreg(0, i);
1828 }
1829}
1830
1831#ifdef CONFIG_KGDB
1832
1833
1834
1835
1836static void dbg_restore_debug_regs(void)
1837{
1838 if (unlikely(kgdb_connected && arch_kgdb_ops.correct_hw_break))
1839 arch_kgdb_ops.correct_hw_break();
1840}
1841#else
1842#define dbg_restore_debug_regs()
1843#endif
1844
1845static void wait_for_master_cpu(int cpu)
1846{
1847#ifdef CONFIG_SMP
1848
1849
1850
1851
1852 WARN_ON(cpumask_test_and_set_cpu(cpu, cpu_initialized_mask));
1853 while (!cpumask_test_cpu(cpu, cpu_callout_mask))
1854 cpu_relax();
1855#endif
1856}
1857
1858#ifdef CONFIG_X86_64
1859static inline void setup_getcpu(int cpu)
1860{
1861 unsigned long cpudata = vdso_encode_cpunode(cpu, early_cpu_to_node(cpu));
1862 struct desc_struct d = { };
1863
1864 if (boot_cpu_has(X86_FEATURE_RDTSCP) || boot_cpu_has(X86_FEATURE_RDPID))
1865 wrmsr(MSR_TSC_AUX, cpudata, 0);
1866
1867
1868 d.limit0 = cpudata;
1869 d.limit1 = cpudata >> 16;
1870
1871 d.type = 5;
1872 d.dpl = 3;
1873 d.s = 1;
1874 d.p = 1;
1875 d.d = 1;
1876
1877 write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_CPUNODE, &d, DESCTYPE_S);
1878}
1879
1880static inline void ucode_cpu_init(int cpu)
1881{
1882 if (cpu)
1883 load_ucode_ap();
1884}
1885
1886static inline void tss_setup_ist(struct tss_struct *tss)
1887{
1888
1889 tss->x86_tss.ist[IST_INDEX_DF] = __this_cpu_ist_top_va(DF);
1890 tss->x86_tss.ist[IST_INDEX_NMI] = __this_cpu_ist_top_va(NMI);
1891 tss->x86_tss.ist[IST_INDEX_DB] = __this_cpu_ist_top_va(DB);
1892 tss->x86_tss.ist[IST_INDEX_MCE] = __this_cpu_ist_top_va(MCE);
1893
1894 tss->x86_tss.ist[IST_INDEX_VC] = __this_cpu_ist_top_va(VC);
1895}
1896
1897#else
1898
1899static inline void setup_getcpu(int cpu) { }
1900
1901static inline void ucode_cpu_init(int cpu)
1902{
1903 show_ucode_info_early();
1904}
1905
1906static inline void tss_setup_ist(struct tss_struct *tss) { }
1907
1908#endif
1909
1910static inline void tss_setup_io_bitmap(struct tss_struct *tss)
1911{
1912 tss->x86_tss.io_bitmap_base = IO_BITMAP_OFFSET_INVALID;
1913
1914#ifdef CONFIG_X86_IOPL_IOPERM
1915 tss->io_bitmap.prev_max = 0;
1916 tss->io_bitmap.prev_sequence = 0;
1917 memset(tss->io_bitmap.bitmap, 0xff, sizeof(tss->io_bitmap.bitmap));
1918
1919
1920
1921
1922 tss->io_bitmap.mapall[IO_BITMAP_LONGS] = ~0UL;
1923#endif
1924}
1925
1926
1927
1928
1929
1930void cpu_init_exception_handling(void)
1931{
1932 struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw);
1933 int cpu = raw_smp_processor_id();
1934
1935
1936 setup_getcpu(cpu);
1937
1938
1939 tss_setup_ist(tss);
1940 tss_setup_io_bitmap(tss);
1941 set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss);
1942
1943 load_TR_desc();
1944
1945
1946 load_current_idt();
1947}
1948
1949
1950
1951
1952
1953
1954
1955void cpu_init(void)
1956{
1957 struct task_struct *cur = current;
1958 int cpu = raw_smp_processor_id();
1959
1960 wait_for_master_cpu(cpu);
1961
1962 ucode_cpu_init(cpu);
1963
1964#ifdef CONFIG_NUMA
1965 if (this_cpu_read(numa_node) == 0 &&
1966 early_cpu_to_node(cpu) != NUMA_NO_NODE)
1967 set_numa_node(early_cpu_to_node(cpu));
1968#endif
1969 pr_debug("Initializing CPU#%d\n", cpu);
1970
1971 if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) ||
1972 boot_cpu_has(X86_FEATURE_TSC) || boot_cpu_has(X86_FEATURE_DE))
1973 cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1974
1975
1976
1977
1978
1979 switch_to_new_gdt(cpu);
1980
1981 if (IS_ENABLED(CONFIG_X86_64)) {
1982 loadsegment(fs, 0);
1983 memset(cur->thread.tls_array, 0, GDT_ENTRY_TLS_ENTRIES * 8);
1984 syscall_init();
1985
1986 wrmsrl(MSR_FS_BASE, 0);
1987 wrmsrl(MSR_KERNEL_GS_BASE, 0);
1988 barrier();
1989
1990 x2apic_setup();
1991 }
1992
1993 mmgrab(&init_mm);
1994 cur->active_mm = &init_mm;
1995 BUG_ON(cur->mm);
1996 initialize_tlbstate_and_flush();
1997 enter_lazy_tlb(&init_mm, cur);
1998
1999
2000
2001
2002
2003 load_sp0((unsigned long)(cpu_entry_stack(cpu) + 1));
2004
2005 load_mm_ldt(&init_mm);
2006
2007 clear_all_debug_regs();
2008 dbg_restore_debug_regs();
2009
2010 doublefault_init_cpu_tss();
2011
2012 fpu__init_cpu();
2013
2014 if (is_uv_system())
2015 uv_cpu_init();
2016
2017 load_fixmap_gdt(cpu);
2018}
2019
2020#ifdef CONFIG_SMP
2021void cpu_init_secondary(void)
2022{
2023
2024
2025
2026
2027 cpu_init_exception_handling();
2028 cpu_init();
2029}
2030#endif
2031
2032
2033
2034
2035
2036
2037void microcode_check(void)
2038{
2039 struct cpuinfo_x86 info;
2040
2041 perf_check_microcode();
2042
2043
2044 info.cpuid_level = cpuid_eax(0);
2045
2046
2047
2048
2049
2050
2051 memcpy(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability));
2052
2053 get_cpu_cap(&info);
2054
2055 if (!memcmp(&info.x86_capability, &boot_cpu_data.x86_capability, sizeof(info.x86_capability)))
2056 return;
2057
2058 pr_warn("x86/CPU: CPU features have changed after loading microcode, but might not take effect.\n");
2059 pr_warn("x86/CPU: Please consider either early loading through initrd/built-in or a potential BIOS update.\n");
2060}
2061
2062
2063
2064
2065void arch_smt_update(void)
2066{
2067
2068 cpu_bugs_smt_update();
2069
2070 apic_smt_update();
2071}
2072