1
2#include <linux/export.h>
3#include <linux/bitops.h>
4#include <linux/elf.h>
5#include <linux/mm.h>
6
7#include <linux/io.h>
8#include <linux/sched.h>
9#include <linux/sched/clock.h>
10#include <linux/random.h>
11#include <linux/topology.h>
12#include <asm/processor.h>
13#include <asm/apic.h>
14#include <asm/cacheinfo.h>
15#include <asm/cpu.h>
16#include <asm/spec-ctrl.h>
17#include <asm/smp.h>
18#include <asm/numa.h>
19#include <asm/pci-direct.h>
20#include <asm/delay.h>
21#include <asm/debugreg.h>
22#include <asm/resctrl.h>
23
24#ifdef CONFIG_X86_64
25# include <asm/mmconfig.h>
26#endif
27
28#include "cpu.h"
29
30static const int amd_erratum_383[];
31static const int amd_erratum_400[];
32static const int amd_erratum_1054[];
33static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum);
34
35
36
37
38
39
40static u32 nodes_per_socket = 1;
41
42static inline int rdmsrl_amd_safe(unsigned msr, unsigned long long *p)
43{
44 u32 gprs[8] = { 0 };
45 int err;
46
47 WARN_ONCE((boot_cpu_data.x86 != 0xf),
48 "%s should only be used on K8!\n", __func__);
49
50 gprs[1] = msr;
51 gprs[7] = 0x9c5a203a;
52
53 err = rdmsr_safe_regs(gprs);
54
55 *p = gprs[0] | ((u64)gprs[2] << 32);
56
57 return err;
58}
59
60static inline int wrmsrl_amd_safe(unsigned msr, unsigned long long val)
61{
62 u32 gprs[8] = { 0 };
63
64 WARN_ONCE((boot_cpu_data.x86 != 0xf),
65 "%s should only be used on K8!\n", __func__);
66
67 gprs[0] = (u32)val;
68 gprs[1] = msr;
69 gprs[2] = val >> 32;
70 gprs[7] = 0x9c5a203a;
71
72 return wrmsr_safe_regs(gprs);
73}
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89#ifdef CONFIG_X86_32
90extern __visible void vide(void);
91__asm__(".text\n"
92 ".globl vide\n"
93 ".type vide, @function\n"
94 ".align 4\n"
95 "vide: ret\n");
96#endif
97
98static void init_amd_k5(struct cpuinfo_x86 *c)
99{
100#ifdef CONFIG_X86_32
101
102
103
104
105
106
107#define CBAR (0xfffc)
108#define CBAR_ENB (0x80000000)
109#define CBAR_KEY (0X000000CB)
110 if (c->x86_model == 9 || c->x86_model == 10) {
111 if (inl(CBAR) & CBAR_ENB)
112 outl(0 | CBAR_KEY, CBAR);
113 }
114#endif
115}
116
117static void init_amd_k6(struct cpuinfo_x86 *c)
118{
119#ifdef CONFIG_X86_32
120 u32 l, h;
121 int mbytes = get_num_physpages() >> (20-PAGE_SHIFT);
122
123 if (c->x86_model < 6) {
124
125 if (c->x86_model == 0) {
126 clear_cpu_cap(c, X86_FEATURE_APIC);
127 set_cpu_cap(c, X86_FEATURE_PGE);
128 }
129 return;
130 }
131
132 if (c->x86_model == 6 && c->x86_stepping == 1) {
133 const int K6_BUG_LOOP = 1000000;
134 int n;
135 void (*f_vide)(void);
136 u64 d, d2;
137
138 pr_info("AMD K6 stepping B detected - ");
139
140
141
142
143
144
145 n = K6_BUG_LOOP;
146 f_vide = vide;
147 OPTIMIZER_HIDE_VAR(f_vide);
148 d = rdtsc();
149 while (n--)
150 f_vide();
151 d2 = rdtsc();
152 d = d2-d;
153
154 if (d > 20*K6_BUG_LOOP)
155 pr_cont("system stability may be impaired when more than 32 MB are used.\n");
156 else
157 pr_cont("probably OK (after B9730xxxx).\n");
158 }
159
160
161 if (c->x86_model < 8 ||
162 (c->x86_model == 8 && c->x86_stepping < 8)) {
163
164 if (mbytes > 508)
165 mbytes = 508;
166
167 rdmsr(MSR_K6_WHCR, l, h);
168 if ((l&0x0000FFFF) == 0) {
169 unsigned long flags;
170 l = (1<<0)|((mbytes/4)<<1);
171 local_irq_save(flags);
172 wbinvd();
173 wrmsr(MSR_K6_WHCR, l, h);
174 local_irq_restore(flags);
175 pr_info("Enabling old style K6 write allocation for %d Mb\n",
176 mbytes);
177 }
178 return;
179 }
180
181 if ((c->x86_model == 8 && c->x86_stepping > 7) ||
182 c->x86_model == 9 || c->x86_model == 13) {
183
184
185 if (mbytes > 4092)
186 mbytes = 4092;
187
188 rdmsr(MSR_K6_WHCR, l, h);
189 if ((l&0xFFFF0000) == 0) {
190 unsigned long flags;
191 l = ((mbytes>>2)<<22)|(1<<16);
192 local_irq_save(flags);
193 wbinvd();
194 wrmsr(MSR_K6_WHCR, l, h);
195 local_irq_restore(flags);
196 pr_info("Enabling new style K6 write allocation for %d Mb\n",
197 mbytes);
198 }
199
200 return;
201 }
202
203 if (c->x86_model == 10) {
204
205
206 return;
207 }
208#endif
209}
210
211static void init_amd_k7(struct cpuinfo_x86 *c)
212{
213#ifdef CONFIG_X86_32
214 u32 l, h;
215
216
217
218
219
220
221 if (c->x86_model >= 6 && c->x86_model <= 10) {
222 if (!cpu_has(c, X86_FEATURE_XMM)) {
223 pr_info("Enabling disabled K7/SSE Support.\n");
224 msr_clear_bit(MSR_K7_HWCR, 15);
225 set_cpu_cap(c, X86_FEATURE_XMM);
226 }
227 }
228
229
230
231
232
233
234 if ((c->x86_model == 8 && c->x86_stepping >= 1) || (c->x86_model > 8)) {
235 rdmsr(MSR_K7_CLK_CTL, l, h);
236 if ((l & 0xfff00000) != 0x20000000) {
237 pr_info("CPU: CLK_CTL MSR was %x. Reprogramming to %x\n",
238 l, ((l & 0x000fffff)|0x20000000));
239 wrmsr(MSR_K7_CLK_CTL, (l & 0x000fffff)|0x20000000, h);
240 }
241 }
242
243
244 if (!c->cpu_index)
245 return;
246
247
248
249
250
251
252 if ((c->x86_model == 6) && ((c->x86_stepping == 0) ||
253 (c->x86_stepping == 1)))
254 return;
255
256
257 if ((c->x86_model == 7) && (c->x86_stepping == 0))
258 return;
259
260
261
262
263
264
265
266
267 if (((c->x86_model == 6) && (c->x86_stepping >= 2)) ||
268 ((c->x86_model == 7) && (c->x86_stepping >= 1)) ||
269 (c->x86_model > 7))
270 if (cpu_has(c, X86_FEATURE_MP))
271 return;
272
273
274
275
276
277
278
279 WARN_ONCE(1, "WARNING: This combination of AMD"
280 " processors is not suitable for SMP.\n");
281 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_NOW_UNRELIABLE);
282#endif
283}
284
285#ifdef CONFIG_NUMA
286
287
288
289
290static int nearby_node(int apicid)
291{
292 int i, node;
293
294 for (i = apicid - 1; i >= 0; i--) {
295 node = __apicid_to_node[i];
296 if (node != NUMA_NO_NODE && node_online(node))
297 return node;
298 }
299 for (i = apicid + 1; i < MAX_LOCAL_APIC; i++) {
300 node = __apicid_to_node[i];
301 if (node != NUMA_NO_NODE && node_online(node))
302 return node;
303 }
304 return first_node(node_online_map);
305}
306#endif
307
308
309
310
311
312
313static void legacy_fixup_core_id(struct cpuinfo_x86 *c)
314{
315 u32 cus_per_node;
316
317 if (c->x86 >= 0x17)
318 return;
319
320 cus_per_node = c->x86_max_cores / nodes_per_socket;
321 c->cpu_core_id %= cus_per_node;
322}
323
324
325
326
327
328
329
330static void amd_get_topology(struct cpuinfo_x86 *c)
331{
332 int cpu = smp_processor_id();
333
334
335 if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
336 int err;
337 u32 eax, ebx, ecx, edx;
338
339 cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
340
341 c->cpu_die_id = ecx & 0xff;
342
343 if (c->x86 == 0x15)
344 c->cu_id = ebx & 0xff;
345
346 if (c->x86 >= 0x17) {
347 c->cpu_core_id = ebx & 0xff;
348
349 if (smp_num_siblings > 1)
350 c->x86_max_cores /= smp_num_siblings;
351 }
352
353
354
355
356
357 err = detect_extended_topology(c);
358 if (!err)
359 c->x86_coreid_bits = get_count_order(c->x86_max_cores);
360
361 cacheinfo_amd_init_llc_id(c, cpu);
362
363 } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) {
364 u64 value;
365
366 rdmsrl(MSR_FAM10H_NODE_ID, value);
367 c->cpu_die_id = value & 7;
368
369 per_cpu(cpu_llc_id, cpu) = c->cpu_die_id;
370 } else
371 return;
372
373 if (nodes_per_socket > 1) {
374 set_cpu_cap(c, X86_FEATURE_AMD_DCM);
375 legacy_fixup_core_id(c);
376 }
377}
378
379
380
381
382
383static void amd_detect_cmp(struct cpuinfo_x86 *c)
384{
385 unsigned bits;
386 int cpu = smp_processor_id();
387
388 bits = c->x86_coreid_bits;
389
390 c->cpu_core_id = c->initial_apicid & ((1 << bits)-1);
391
392 c->phys_proc_id = c->initial_apicid >> bits;
393
394 per_cpu(cpu_llc_id, cpu) = c->cpu_die_id = c->phys_proc_id;
395}
396
397static void amd_detect_ppin(struct cpuinfo_x86 *c)
398{
399 unsigned long long val;
400
401 if (!cpu_has(c, X86_FEATURE_AMD_PPIN))
402 return;
403
404
405 if (rdmsrl_safe(MSR_AMD_PPIN_CTL, &val))
406 goto clear_ppin;
407
408
409 if ((val & 3UL) == 1UL)
410 goto clear_ppin;
411
412
413 if (!(val & 2UL)) {
414 wrmsrl_safe(MSR_AMD_PPIN_CTL, val | 2UL);
415 rdmsrl_safe(MSR_AMD_PPIN_CTL, &val);
416 }
417
418
419 if (val & 2UL)
420 return;
421
422clear_ppin:
423 clear_cpu_cap(c, X86_FEATURE_AMD_PPIN);
424}
425
426u32 amd_get_nodes_per_socket(void)
427{
428 return nodes_per_socket;
429}
430EXPORT_SYMBOL_GPL(amd_get_nodes_per_socket);
431
432static void srat_detect_node(struct cpuinfo_x86 *c)
433{
434#ifdef CONFIG_NUMA
435 int cpu = smp_processor_id();
436 int node;
437 unsigned apicid = c->apicid;
438
439 node = numa_cpu_node(cpu);
440 if (node == NUMA_NO_NODE)
441 node = get_llc_id(cpu);
442
443
444
445
446
447
448 if (x86_cpuinit.fixup_cpu_id)
449 x86_cpuinit.fixup_cpu_id(c, node);
450
451 if (!node_online(node)) {
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471 int ht_nodeid = c->initial_apicid;
472
473 if (__apicid_to_node[ht_nodeid] != NUMA_NO_NODE)
474 node = __apicid_to_node[ht_nodeid];
475
476 if (!node_online(node))
477 node = nearby_node(apicid);
478 }
479 numa_set_node(cpu, node);
480#endif
481}
482
483static void early_init_amd_mc(struct cpuinfo_x86 *c)
484{
485#ifdef CONFIG_SMP
486 unsigned bits, ecx;
487
488
489 if (c->extended_cpuid_level < 0x80000008)
490 return;
491
492 ecx = cpuid_ecx(0x80000008);
493
494 c->x86_max_cores = (ecx & 0xff) + 1;
495
496
497 bits = (ecx >> 12) & 0xF;
498
499
500 if (bits == 0) {
501 while ((1 << bits) < c->x86_max_cores)
502 bits++;
503 }
504
505 c->x86_coreid_bits = bits;
506#endif
507}
508
509static void bsp_init_amd(struct cpuinfo_x86 *c)
510{
511 if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) {
512
513 if (c->x86 > 0x10 ||
514 (c->x86 == 0x10 && c->x86_model >= 0x2)) {
515 u64 val;
516
517 rdmsrl(MSR_K7_HWCR, val);
518 if (!(val & BIT(24)))
519 pr_warn(FW_BUG "TSC doesn't count with P0 frequency!\n");
520 }
521 }
522
523 if (c->x86 == 0x15) {
524 unsigned long upperbit;
525 u32 cpuid, assoc;
526
527 cpuid = cpuid_edx(0x80000005);
528 assoc = cpuid >> 16 & 0xff;
529 upperbit = ((cpuid >> 24) << 10) / assoc;
530
531 va_align.mask = (upperbit - 1) & PAGE_MASK;
532 va_align.flags = ALIGN_VA_32 | ALIGN_VA_64;
533
534
535 va_align.bits = get_random_int() & va_align.mask;
536 }
537
538 if (cpu_has(c, X86_FEATURE_MWAITX))
539 use_mwaitx_delay();
540
541 if (boot_cpu_has(X86_FEATURE_TOPOEXT)) {
542 u32 ecx;
543
544 ecx = cpuid_ecx(0x8000001e);
545 __max_die_per_package = nodes_per_socket = ((ecx >> 8) & 7) + 1;
546 } else if (boot_cpu_has(X86_FEATURE_NODEID_MSR)) {
547 u64 value;
548
549 rdmsrl(MSR_FAM10H_NODE_ID, value);
550 __max_die_per_package = nodes_per_socket = ((value >> 3) & 7) + 1;
551 }
552
553 if (!boot_cpu_has(X86_FEATURE_AMD_SSBD) &&
554 !boot_cpu_has(X86_FEATURE_VIRT_SSBD) &&
555 c->x86 >= 0x15 && c->x86 <= 0x17) {
556 unsigned int bit;
557
558 switch (c->x86) {
559 case 0x15: bit = 54; break;
560 case 0x16: bit = 33; break;
561 case 0x17: bit = 10; break;
562 default: return;
563 }
564
565
566
567
568 if (!rdmsrl_safe(MSR_AMD64_LS_CFG, &x86_amd_ls_cfg_base)) {
569 setup_force_cpu_cap(X86_FEATURE_LS_CFG_SSBD);
570 setup_force_cpu_cap(X86_FEATURE_SSBD);
571 x86_amd_ls_cfg_ssbd_mask = 1ULL << bit;
572 }
573 }
574
575 resctrl_cpu_detect(c);
576}
577
578static void early_detect_mem_encrypt(struct cpuinfo_x86 *c)
579{
580 u64 msr;
581
582
583
584
585
586
587
588
589
590
591
592
593
594 if (cpu_has(c, X86_FEATURE_SME) || cpu_has(c, X86_FEATURE_SEV)) {
595
596 rdmsrl(MSR_AMD64_SYSCFG, msr);
597 if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT))
598 goto clear_all;
599
600
601
602
603
604
605 c->x86_phys_bits -= (cpuid_ebx(0x8000001f) >> 6) & 0x3f;
606
607 if (IS_ENABLED(CONFIG_X86_32))
608 goto clear_all;
609
610 rdmsrl(MSR_K7_HWCR, msr);
611 if (!(msr & MSR_K7_HWCR_SMMLOCK))
612 goto clear_sev;
613
614 return;
615
616clear_all:
617 setup_clear_cpu_cap(X86_FEATURE_SME);
618clear_sev:
619 setup_clear_cpu_cap(X86_FEATURE_SEV);
620 setup_clear_cpu_cap(X86_FEATURE_SEV_ES);
621 }
622}
623
624static void early_init_amd(struct cpuinfo_x86 *c)
625{
626 u64 value;
627 u32 dummy;
628
629 early_init_amd_mc(c);
630
631 if (c->x86 >= 0xf)
632 set_cpu_cap(c, X86_FEATURE_K8);
633
634 rdmsr_safe(MSR_AMD64_PATCH_LEVEL, &c->microcode, &dummy);
635
636
637
638
639
640 if (c->x86_power & (1 << 8)) {
641 set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC);
642 set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC);
643 }
644
645
646 if (c->x86_power & BIT(12))
647 set_cpu_cap(c, X86_FEATURE_ACC_POWER);
648
649
650 if (c->x86_power & BIT(14))
651 set_cpu_cap(c, X86_FEATURE_RAPL);
652
653#ifdef CONFIG_X86_64
654 set_cpu_cap(c, X86_FEATURE_SYSCALL32);
655#else
656
657 if (c->x86 == 5)
658 if (c->x86_model == 13 || c->x86_model == 9 ||
659 (c->x86_model == 8 && c->x86_stepping >= 8))
660 set_cpu_cap(c, X86_FEATURE_K6_MTRR);
661#endif
662#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_PCI)
663
664
665
666
667
668
669 if (boot_cpu_has(X86_FEATURE_APIC)) {
670 if (c->x86 > 0x16)
671 set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
672 else if (c->x86 >= 0xf) {
673
674 unsigned int val;
675
676 val = read_pci_config(0, 24, 0, 0x68);
677 if ((val >> 17 & 0x3) == 0x3)
678 set_cpu_cap(c, X86_FEATURE_EXTD_APICID);
679 }
680 }
681#endif
682
683
684
685
686
687
688 set_cpu_cap(c, X86_FEATURE_VMMCALL);
689
690
691 if (c->x86 == 0x16 && c->x86_model <= 0xf)
692 msr_set_bit(MSR_AMD64_LS_CFG, 15);
693
694
695
696
697
698
699
700 if (cpu_has_amd_erratum(c, amd_erratum_400))
701 set_cpu_bug(c, X86_BUG_AMD_E400);
702
703 early_detect_mem_encrypt(c);
704
705
706 if (c->x86 == 0x15 &&
707 (c->x86_model >= 0x10 && c->x86_model <= 0x6f) &&
708 !cpu_has(c, X86_FEATURE_TOPOEXT)) {
709
710 if (msr_set_bit(0xc0011005, 54) > 0) {
711 rdmsrl(0xc0011005, value);
712 if (value & BIT_64(54)) {
713 set_cpu_cap(c, X86_FEATURE_TOPOEXT);
714 pr_info_once(FW_INFO "CPU: Re-enabling disabled Topology Extensions Support.\n");
715 }
716 }
717 }
718
719 if (cpu_has(c, X86_FEATURE_TOPOEXT))
720 smp_num_siblings = ((cpuid_ebx(0x8000001e) >> 8) & 0xff) + 1;
721}
722
723static void init_amd_k8(struct cpuinfo_x86 *c)
724{
725 u32 level;
726 u64 value;
727
728
729 level = cpuid_eax(1);
730 if ((level >= 0x0f48 && level < 0x0f50) || level >= 0x0f58)
731 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
732
733
734
735
736
737
738 if (c->x86_model < 0x14 && cpu_has(c, X86_FEATURE_LAHF_LM)) {
739 clear_cpu_cap(c, X86_FEATURE_LAHF_LM);
740 if (!rdmsrl_amd_safe(0xc001100d, &value)) {
741 value &= ~BIT_64(32);
742 wrmsrl_amd_safe(0xc001100d, value);
743 }
744 }
745
746 if (!c->x86_model_id[0])
747 strcpy(c->x86_model_id, "Hammer");
748
749#ifdef CONFIG_SMP
750
751
752
753
754
755
756
757 msr_set_bit(MSR_K7_HWCR, 6);
758#endif
759 set_cpu_bug(c, X86_BUG_SWAPGS_FENCE);
760}
761
762static void init_amd_gh(struct cpuinfo_x86 *c)
763{
764#ifdef CONFIG_MMCONF_FAM10H
765
766 if (c == &boot_cpu_data)
767 check_enable_amd_mmconf_dmi();
768
769 fam10h_check_enable_mmcfg();
770#endif
771
772
773
774
775
776
777
778
779
780 msr_set_bit(MSR_AMD64_MCx_MASK(4), 10);
781
782
783
784
785
786
787
788
789
790
791 msr_clear_bit(MSR_AMD64_BU_CFG2, 24);
792
793 if (cpu_has_amd_erratum(c, amd_erratum_383))
794 set_cpu_bug(c, X86_BUG_AMD_TLB_MMATCH);
795}
796
797#define MSR_AMD64_DE_CFG 0xC0011029
798
799static void init_amd_ln(struct cpuinfo_x86 *c)
800{
801
802
803
804
805 msr_set_bit(MSR_AMD64_DE_CFG, 31);
806}
807
808static bool rdrand_force;
809
810static int __init rdrand_cmdline(char *str)
811{
812 if (!str)
813 return -EINVAL;
814
815 if (!strcmp(str, "force"))
816 rdrand_force = true;
817 else
818 return -EINVAL;
819
820 return 0;
821}
822early_param("rdrand", rdrand_cmdline);
823
824static void clear_rdrand_cpuid_bit(struct cpuinfo_x86 *c)
825{
826
827
828
829
830
831 if (!IS_ENABLED(CONFIG_PM_SLEEP))
832 return;
833
834
835
836
837
838 if (!(cpuid_ecx(1) & BIT(30)) || rdrand_force)
839 return;
840
841 msr_clear_bit(MSR_AMD64_CPUID_FN_1, 62);
842
843
844
845
846
847 if (cpuid_ecx(1) & BIT(30)) {
848 pr_info_once("BIOS may not properly restore RDRAND after suspend, but hypervisor does not support hiding RDRAND via CPUID.\n");
849 return;
850 }
851
852 clear_cpu_cap(c, X86_FEATURE_RDRAND);
853 pr_info_once("BIOS may not properly restore RDRAND after suspend, hiding RDRAND via CPUID. Use rdrand=force to reenable.\n");
854}
855
856static void init_amd_jg(struct cpuinfo_x86 *c)
857{
858
859
860
861
862
863 clear_rdrand_cpuid_bit(c);
864}
865
866static void init_amd_bd(struct cpuinfo_x86 *c)
867{
868 u64 value;
869
870
871
872
873
874 if ((c->x86_model >= 0x02) && (c->x86_model < 0x20)) {
875 if (!rdmsrl_safe(MSR_F15H_IC_CFG, &value) && !(value & 0x1E)) {
876 value |= 0x1E;
877 wrmsrl_safe(MSR_F15H_IC_CFG, value);
878 }
879 }
880
881
882
883
884
885
886 clear_rdrand_cpuid_bit(c);
887}
888
889static void init_amd_zn(struct cpuinfo_x86 *c)
890{
891 set_cpu_cap(c, X86_FEATURE_ZEN);
892
893#ifdef CONFIG_NUMA
894 node_reclaim_distance = 32;
895#endif
896
897
898
899
900
901 if (!cpu_has(c, X86_FEATURE_HYPERVISOR) && !cpu_has(c, X86_FEATURE_CPB))
902 set_cpu_cap(c, X86_FEATURE_CPB);
903}
904
905static void init_amd(struct cpuinfo_x86 *c)
906{
907 early_init_amd(c);
908
909
910
911
912
913 clear_cpu_cap(c, 0*32+31);
914
915 if (c->x86 >= 0x10)
916 set_cpu_cap(c, X86_FEATURE_REP_GOOD);
917
918
919 c->apicid = hard_smp_processor_id();
920
921
922 if (c->x86 < 6)
923 clear_cpu_cap(c, X86_FEATURE_MCE);
924
925 switch (c->x86) {
926 case 4: init_amd_k5(c); break;
927 case 5: init_amd_k6(c); break;
928 case 6: init_amd_k7(c); break;
929 case 0xf: init_amd_k8(c); break;
930 case 0x10: init_amd_gh(c); break;
931 case 0x12: init_amd_ln(c); break;
932 case 0x15: init_amd_bd(c); break;
933 case 0x16: init_amd_jg(c); break;
934 case 0x17: fallthrough;
935 case 0x19: init_amd_zn(c); break;
936 }
937
938
939
940
941
942 if ((c->x86 >= 6) && (!cpu_has(c, X86_FEATURE_XSAVEERPTR)))
943 set_cpu_bug(c, X86_BUG_FXSAVE_LEAK);
944
945 cpu_detect_cache_sizes(c);
946
947 amd_detect_cmp(c);
948 amd_get_topology(c);
949 srat_detect_node(c);
950 amd_detect_ppin(c);
951
952 init_amd_cacheinfo(c);
953
954 if (cpu_has(c, X86_FEATURE_XMM2)) {
955
956
957
958
959
960
961 msr_set_bit(MSR_F10H_DECFG,
962 MSR_F10H_DECFG_LFENCE_SERIALIZE_BIT);
963
964
965 set_cpu_cap(c, X86_FEATURE_LFENCE_RDTSC);
966 }
967
968
969
970
971
972 if (c->x86 > 0x11)
973 set_cpu_cap(c, X86_FEATURE_ARAT);
974
975
976 if (!cpu_has(c, X86_FEATURE_3DNOWPREFETCH))
977 if (cpu_has(c, X86_FEATURE_3DNOW) || cpu_has(c, X86_FEATURE_LM))
978 set_cpu_cap(c, X86_FEATURE_3DNOWPREFETCH);
979
980
981 if (!cpu_has(c, X86_FEATURE_XENPV))
982 set_cpu_bug(c, X86_BUG_SYSRET_SS_ATTRS);
983
984
985
986
987
988
989 if (cpu_has(c, X86_FEATURE_IRPERF) &&
990 !cpu_has_amd_erratum(c, amd_erratum_1054))
991 msr_set_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT);
992}
993
994#ifdef CONFIG_X86_32
995static unsigned int amd_size_cache(struct cpuinfo_x86 *c, unsigned int size)
996{
997
998 if (c->x86 == 6) {
999
1000 if (c->x86_model == 3 && c->x86_stepping == 0)
1001 size = 64;
1002
1003 if (c->x86_model == 4 &&
1004 (c->x86_stepping == 0 || c->x86_stepping == 1))
1005 size = 256;
1006 }
1007 return size;
1008}
1009#endif
1010
1011static void cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
1012{
1013 u32 ebx, eax, ecx, edx;
1014 u16 mask = 0xfff;
1015
1016 if (c->x86 < 0xf)
1017 return;
1018
1019 if (c->extended_cpuid_level < 0x80000006)
1020 return;
1021
1022 cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
1023
1024 tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
1025 tlb_lli_4k[ENTRIES] = ebx & mask;
1026
1027
1028
1029
1030
1031 if (c->x86 == 0xf) {
1032 cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
1033 mask = 0xff;
1034 }
1035
1036
1037 if (!((eax >> 16) & mask))
1038 tlb_lld_2m[ENTRIES] = (cpuid_eax(0x80000005) >> 16) & 0xff;
1039 else
1040 tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;
1041
1042
1043 tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;
1044
1045
1046 if (!(eax & mask)) {
1047
1048 if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
1049 tlb_lli_2m[ENTRIES] = 1024;
1050 } else {
1051 cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
1052 tlb_lli_2m[ENTRIES] = eax & 0xff;
1053 }
1054 } else
1055 tlb_lli_2m[ENTRIES] = eax & mask;
1056
1057 tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
1058}
1059
1060static const struct cpu_dev amd_cpu_dev = {
1061 .c_vendor = "AMD",
1062 .c_ident = { "AuthenticAMD" },
1063#ifdef CONFIG_X86_32
1064 .legacy_models = {
1065 { .family = 4, .model_names =
1066 {
1067 [3] = "486 DX/2",
1068 [7] = "486 DX/2-WB",
1069 [8] = "486 DX/4",
1070 [9] = "486 DX/4-WB",
1071 [14] = "Am5x86-WT",
1072 [15] = "Am5x86-WB"
1073 }
1074 },
1075 },
1076 .legacy_cache_size = amd_size_cache,
1077#endif
1078 .c_early_init = early_init_amd,
1079 .c_detect_tlb = cpu_detect_tlb_amd,
1080 .c_bsp_init = bsp_init_amd,
1081 .c_init = init_amd,
1082 .c_x86_vendor = X86_VENDOR_AMD,
1083};
1084
1085cpu_dev_register(amd_cpu_dev);
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104#define AMD_LEGACY_ERRATUM(...) { -1, __VA_ARGS__, 0 }
1105#define AMD_OSVW_ERRATUM(osvw_id, ...) { osvw_id, __VA_ARGS__, 0 }
1106#define AMD_MODEL_RANGE(f, m_start, s_start, m_end, s_end) \
1107 ((f << 24) | (m_start << 16) | (s_start << 12) | (m_end << 4) | (s_end))
1108#define AMD_MODEL_RANGE_FAMILY(range) (((range) >> 24) & 0xff)
1109#define AMD_MODEL_RANGE_START(range) (((range) >> 12) & 0xfff)
1110#define AMD_MODEL_RANGE_END(range) ((range) & 0xfff)
1111
1112static const int amd_erratum_400[] =
1113 AMD_OSVW_ERRATUM(1, AMD_MODEL_RANGE(0xf, 0x41, 0x2, 0xff, 0xf),
1114 AMD_MODEL_RANGE(0x10, 0x2, 0x1, 0xff, 0xf));
1115
1116static const int amd_erratum_383[] =
1117 AMD_OSVW_ERRATUM(3, AMD_MODEL_RANGE(0x10, 0, 0, 0xff, 0xf));
1118
1119
1120static const int amd_erratum_1054[] =
1121 AMD_LEGACY_ERRATUM(AMD_MODEL_RANGE(0x17, 0, 0, 0x2f, 0xf));
1122
1123static bool cpu_has_amd_erratum(struct cpuinfo_x86 *cpu, const int *erratum)
1124{
1125 int osvw_id = *erratum++;
1126 u32 range;
1127 u32 ms;
1128
1129 if (osvw_id >= 0 && osvw_id < 65536 &&
1130 cpu_has(cpu, X86_FEATURE_OSVW)) {
1131 u64 osvw_len;
1132
1133 rdmsrl(MSR_AMD64_OSVW_ID_LENGTH, osvw_len);
1134 if (osvw_id < osvw_len) {
1135 u64 osvw_bits;
1136
1137 rdmsrl(MSR_AMD64_OSVW_STATUS + (osvw_id >> 6),
1138 osvw_bits);
1139 return osvw_bits & (1ULL << (osvw_id & 0x3f));
1140 }
1141 }
1142
1143
1144 ms = (cpu->x86_model << 4) | cpu->x86_stepping;
1145 while ((range = *erratum++))
1146 if ((cpu->x86 == AMD_MODEL_RANGE_FAMILY(range)) &&
1147 (ms >= AMD_MODEL_RANGE_START(range)) &&
1148 (ms <= AMD_MODEL_RANGE_END(range)))
1149 return true;
1150
1151 return false;
1152}
1153
1154void set_dr_addr_mask(unsigned long mask, int dr)
1155{
1156 if (!boot_cpu_has(X86_FEATURE_BPEXT))
1157 return;
1158
1159 switch (dr) {
1160 case 0:
1161 wrmsr(MSR_F16H_DR0_ADDR_MASK, mask, 0);
1162 break;
1163 case 1:
1164 case 2:
1165 case 3:
1166 wrmsr(MSR_F16H_DR1_ADDR_MASK - 1 + dr, mask, 0);
1167 break;
1168 default:
1169 break;
1170 }
1171}
1172
1173u32 amd_get_highest_perf(void)
1174{
1175 struct cpuinfo_x86 *c = &boot_cpu_data;
1176
1177 if (c->x86 == 0x17 && ((c->x86_model >= 0x30 && c->x86_model < 0x40) ||
1178 (c->x86_model >= 0x70 && c->x86_model < 0x80)))
1179 return 166;
1180
1181 if (c->x86 == 0x19 && ((c->x86_model >= 0x20 && c->x86_model < 0x30) ||
1182 (c->x86_model >= 0x40 && c->x86_model < 0x70)))
1183 return 166;
1184
1185 return 255;
1186}
1187EXPORT_SYMBOL_GPL(amd_get_highest_perf);
1188