1
2
3
4
5
6
7
8
9
10
11
12#include <linux/kvm_host.h>
13#include <linux/export.h>
14#include <linux/vmalloc.h>
15#include <linux/uaccess.h>
16#include <linux/sched/stat.h>
17
18#include <asm/processor.h>
19#include <asm/user.h>
20#include <asm/fpu/xstate.h>
21#include "cpuid.h"
22#include "lapic.h"
23#include "mmu.h"
24#include "trace.h"
25#include "pmu.h"
26
27
28
29
30
31u32 kvm_cpu_caps[NCAPINTS] __read_mostly;
32EXPORT_SYMBOL_GPL(kvm_cpu_caps);
33
34static u32 xstate_required_size(u64 xstate_bv, bool compacted)
35{
36 int feature_bit = 0;
37 u32 ret = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET;
38
39 xstate_bv &= XFEATURE_MASK_EXTEND;
40 while (xstate_bv) {
41 if (xstate_bv & 0x1) {
42 u32 eax, ebx, ecx, edx, offset;
43 cpuid_count(0xD, feature_bit, &eax, &ebx, &ecx, &edx);
44 offset = compacted ? ret : ebx;
45 ret = max(ret, offset + eax);
46 }
47
48 xstate_bv >>= 1;
49 feature_bit++;
50 }
51
52 return ret;
53}
54
55#define F feature_bit
56
57static int kvm_check_cpuid(struct kvm_vcpu *vcpu)
58{
59 struct kvm_cpuid_entry2 *best;
60
61
62
63
64
65 best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
66 if (best) {
67 int vaddr_bits = (best->eax & 0xff00) >> 8;
68
69 if (vaddr_bits != 48 && vaddr_bits != 57 && vaddr_bits != 0)
70 return -EINVAL;
71 }
72
73 return 0;
74}
75
76void kvm_update_cpuid_runtime(struct kvm_vcpu *vcpu)
77{
78 struct kvm_cpuid_entry2 *best;
79
80 best = kvm_find_cpuid_entry(vcpu, 1, 0);
81 if (best) {
82
83 if (boot_cpu_has(X86_FEATURE_XSAVE))
84 cpuid_entry_change(best, X86_FEATURE_OSXSAVE,
85 kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE));
86
87 cpuid_entry_change(best, X86_FEATURE_APIC,
88 vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE);
89 }
90
91 best = kvm_find_cpuid_entry(vcpu, 7, 0);
92 if (best && boot_cpu_has(X86_FEATURE_PKU) && best->function == 0x7)
93 cpuid_entry_change(best, X86_FEATURE_OSPKE,
94 kvm_read_cr4_bits(vcpu, X86_CR4_PKE));
95
96 best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
97 if (best)
98 best->ebx = xstate_required_size(vcpu->arch.xcr0, false);
99
100 best = kvm_find_cpuid_entry(vcpu, 0xD, 1);
101 if (best && (cpuid_entry_has(best, X86_FEATURE_XSAVES) ||
102 cpuid_entry_has(best, X86_FEATURE_XSAVEC)))
103 best->ebx = xstate_required_size(vcpu->arch.xcr0, true);
104
105 best = kvm_find_cpuid_entry(vcpu, KVM_CPUID_FEATURES, 0);
106 if (kvm_hlt_in_guest(vcpu->kvm) && best &&
107 (best->eax & (1 << KVM_FEATURE_PV_UNHALT)))
108 best->eax &= ~(1 << KVM_FEATURE_PV_UNHALT);
109
110 if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT)) {
111 best = kvm_find_cpuid_entry(vcpu, 0x1, 0);
112 if (best)
113 cpuid_entry_change(best, X86_FEATURE_MWAIT,
114 vcpu->arch.ia32_misc_enable_msr &
115 MSR_IA32_MISC_ENABLE_MWAIT);
116 }
117}
118
119static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)
120{
121 struct kvm_lapic *apic = vcpu->arch.apic;
122 struct kvm_cpuid_entry2 *best;
123
124 kvm_x86_ops.vcpu_after_set_cpuid(vcpu);
125
126 best = kvm_find_cpuid_entry(vcpu, 1, 0);
127 if (best && apic) {
128 if (cpuid_entry_has(best, X86_FEATURE_TSC_DEADLINE_TIMER))
129 apic->lapic_timer.timer_mode_mask = 3 << 17;
130 else
131 apic->lapic_timer.timer_mode_mask = 1 << 17;
132
133 kvm_apic_set_version(vcpu);
134 }
135
136 best = kvm_find_cpuid_entry(vcpu, 0xD, 0);
137 if (!best)
138 vcpu->arch.guest_supported_xcr0 = 0;
139 else
140 vcpu->arch.guest_supported_xcr0 =
141 (best->eax | ((u64)best->edx << 32)) & supported_xcr0;
142
143 vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu);
144 kvm_mmu_reset_context(vcpu);
145
146 kvm_pmu_refresh(vcpu);
147 vcpu->arch.cr4_guest_rsvd_bits =
148 __cr4_reserved_bits(guest_cpuid_has, vcpu);
149 kvm_x86_ops.update_exception_bitmap(vcpu);
150}
151
152static int is_efer_nx(void)
153{
154 return host_efer & EFER_NX;
155}
156
157static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu)
158{
159 int i;
160 struct kvm_cpuid_entry2 *e, *entry;
161
162 entry = NULL;
163 for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
164 e = &vcpu->arch.cpuid_entries[i];
165 if (e->function == 0x80000001) {
166 entry = e;
167 break;
168 }
169 }
170 if (entry && cpuid_entry_has(entry, X86_FEATURE_NX) && !is_efer_nx()) {
171 cpuid_entry_clear(entry, X86_FEATURE_NX);
172 printk(KERN_INFO "kvm: guest NX capability removed\n");
173 }
174}
175
176int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu)
177{
178 struct kvm_cpuid_entry2 *best;
179
180 best = kvm_find_cpuid_entry(vcpu, 0x80000000, 0);
181 if (!best || best->eax < 0x80000008)
182 goto not_found;
183 best = kvm_find_cpuid_entry(vcpu, 0x80000008, 0);
184 if (best)
185 return best->eax & 0xff;
186not_found:
187 return 36;
188}
189EXPORT_SYMBOL_GPL(cpuid_query_maxphyaddr);
190
191
192int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
193 struct kvm_cpuid *cpuid,
194 struct kvm_cpuid_entry __user *entries)
195{
196 int r, i;
197 struct kvm_cpuid_entry *cpuid_entries = NULL;
198
199 r = -E2BIG;
200 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
201 goto out;
202 if (cpuid->nent) {
203 cpuid_entries = vmemdup_user(entries,
204 array_size(sizeof(struct kvm_cpuid_entry),
205 cpuid->nent));
206 if (IS_ERR(cpuid_entries)) {
207 r = PTR_ERR(cpuid_entries);
208 goto out;
209 }
210 }
211 for (i = 0; i < cpuid->nent; i++) {
212 vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function;
213 vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax;
214 vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx;
215 vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx;
216 vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx;
217 vcpu->arch.cpuid_entries[i].index = 0;
218 vcpu->arch.cpuid_entries[i].flags = 0;
219 vcpu->arch.cpuid_entries[i].padding[0] = 0;
220 vcpu->arch.cpuid_entries[i].padding[1] = 0;
221 vcpu->arch.cpuid_entries[i].padding[2] = 0;
222 }
223 vcpu->arch.cpuid_nent = cpuid->nent;
224 r = kvm_check_cpuid(vcpu);
225 if (r) {
226 vcpu->arch.cpuid_nent = 0;
227 kvfree(cpuid_entries);
228 goto out;
229 }
230
231 cpuid_fix_nx_cap(vcpu);
232 kvm_update_cpuid_runtime(vcpu);
233 kvm_vcpu_after_set_cpuid(vcpu);
234
235 kvfree(cpuid_entries);
236out:
237 return r;
238}
239
240int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu,
241 struct kvm_cpuid2 *cpuid,
242 struct kvm_cpuid_entry2 __user *entries)
243{
244 int r;
245
246 r = -E2BIG;
247 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
248 goto out;
249 r = -EFAULT;
250 if (copy_from_user(&vcpu->arch.cpuid_entries, entries,
251 cpuid->nent * sizeof(struct kvm_cpuid_entry2)))
252 goto out;
253 vcpu->arch.cpuid_nent = cpuid->nent;
254 r = kvm_check_cpuid(vcpu);
255 if (r) {
256 vcpu->arch.cpuid_nent = 0;
257 goto out;
258 }
259
260 kvm_update_cpuid_runtime(vcpu);
261 kvm_vcpu_after_set_cpuid(vcpu);
262out:
263 return r;
264}
265
266int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu,
267 struct kvm_cpuid2 *cpuid,
268 struct kvm_cpuid_entry2 __user *entries)
269{
270 int r;
271
272 r = -E2BIG;
273 if (cpuid->nent < vcpu->arch.cpuid_nent)
274 goto out;
275 r = -EFAULT;
276 if (copy_to_user(entries, &vcpu->arch.cpuid_entries,
277 vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2)))
278 goto out;
279 return 0;
280
281out:
282 cpuid->nent = vcpu->arch.cpuid_nent;
283 return r;
284}
285
286static __always_inline void kvm_cpu_cap_mask(enum cpuid_leafs leaf, u32 mask)
287{
288 const struct cpuid_reg cpuid = x86_feature_cpuid(leaf * 32);
289 struct kvm_cpuid_entry2 entry;
290
291 reverse_cpuid_check(leaf);
292 kvm_cpu_caps[leaf] &= mask;
293
294 cpuid_count(cpuid.function, cpuid.index,
295 &entry.eax, &entry.ebx, &entry.ecx, &entry.edx);
296
297 kvm_cpu_caps[leaf] &= *__cpuid_entry_get_reg(&entry, cpuid.reg);
298}
299
300void kvm_set_cpu_caps(void)
301{
302 unsigned int f_nx = is_efer_nx() ? F(NX) : 0;
303#ifdef CONFIG_X86_64
304 unsigned int f_gbpages = F(GBPAGES);
305 unsigned int f_lm = F(LM);
306#else
307 unsigned int f_gbpages = 0;
308 unsigned int f_lm = 0;
309#endif
310
311 BUILD_BUG_ON(sizeof(kvm_cpu_caps) >
312 sizeof(boot_cpu_data.x86_capability));
313
314 memcpy(&kvm_cpu_caps, &boot_cpu_data.x86_capability,
315 sizeof(kvm_cpu_caps));
316
317 kvm_cpu_cap_mask(CPUID_1_ECX,
318
319
320
321
322 F(XMM3) | F(PCLMULQDQ) | 0 |
323 0 |
324 0 | F(SSSE3) | 0 | 0 |
325 F(FMA) | F(CX16) | 0 | F(PDCM) |
326 F(PCID) | 0 | F(XMM4_1) |
327 F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) |
328 0 | F(AES) | F(XSAVE) | 0 | F(AVX) |
329 F(F16C) | F(RDRAND)
330 );
331
332 kvm_cpu_cap_set(X86_FEATURE_X2APIC);
333
334 kvm_cpu_cap_mask(CPUID_1_EDX,
335 F(FPU) | F(VME) | F(DE) | F(PSE) |
336 F(TSC) | F(MSR) | F(PAE) | F(MCE) |
337 F(CX8) | F(APIC) | 0 | F(SEP) |
338 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
339 F(PAT) | F(PSE36) | 0 | F(CLFLUSH) |
340 0 | F(MMX) |
341 F(FXSR) | F(XMM) | F(XMM2) | F(SELFSNOOP) |
342 0
343 );
344
345 kvm_cpu_cap_mask(CPUID_7_0_EBX,
346 F(FSGSBASE) | F(BMI1) | F(HLE) | F(AVX2) | F(SMEP) |
347 F(BMI2) | F(ERMS) | 0 | F(RTM) | 0 | F(RDSEED) |
348 F(ADX) | F(SMAP) | F(AVX512IFMA) | F(AVX512F) | F(AVX512PF) |
349 F(AVX512ER) | F(AVX512CD) | F(CLFLUSHOPT) | F(CLWB) | F(AVX512DQ) |
350 F(SHA_NI) | F(AVX512BW) | F(AVX512VL) | 0
351 );
352
353 kvm_cpu_cap_mask(CPUID_7_ECX,
354 F(AVX512VBMI) | F(LA57) | F(PKU) | 0 | F(RDPID) |
355 F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) |
356 F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) |
357 F(CLDEMOTE) | F(MOVDIRI) | F(MOVDIR64B) | 0
358 );
359
360 if (cpuid_ecx(7) & F(LA57))
361 kvm_cpu_cap_set(X86_FEATURE_LA57);
362
363
364
365
366
367 if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE))
368 kvm_cpu_cap_clear(X86_FEATURE_PKU);
369
370 kvm_cpu_cap_mask(CPUID_7_EDX,
371 F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) |
372 F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) |
373 F(MD_CLEAR) | F(AVX512_VP2INTERSECT) | F(FSRM) |
374 F(SERIALIZE)
375 );
376
377
378 kvm_cpu_cap_set(X86_FEATURE_TSC_ADJUST);
379 kvm_cpu_cap_set(X86_FEATURE_ARCH_CAPABILITIES);
380
381 if (boot_cpu_has(X86_FEATURE_IBPB) && boot_cpu_has(X86_FEATURE_IBRS))
382 kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL);
383 if (boot_cpu_has(X86_FEATURE_STIBP))
384 kvm_cpu_cap_set(X86_FEATURE_INTEL_STIBP);
385 if (boot_cpu_has(X86_FEATURE_AMD_SSBD))
386 kvm_cpu_cap_set(X86_FEATURE_SPEC_CTRL_SSBD);
387
388 kvm_cpu_cap_mask(CPUID_7_1_EAX,
389 F(AVX512_BF16)
390 );
391
392 kvm_cpu_cap_mask(CPUID_D_1_EAX,
393 F(XSAVEOPT) | F(XSAVEC) | F(XGETBV1) | F(XSAVES)
394 );
395
396 kvm_cpu_cap_mask(CPUID_8000_0001_ECX,
397 F(LAHF_LM) | F(CMP_LEGACY) | 0 | 0 |
398 F(CR8_LEGACY) | F(ABM) | F(SSE4A) | F(MISALIGNSSE) |
399 F(3DNOWPREFETCH) | F(OSVW) | 0 | F(XOP) |
400 0 | F(FMA4) | F(TBM) |
401 F(TOPOEXT) | F(PERFCTR_CORE)
402 );
403
404 kvm_cpu_cap_mask(CPUID_8000_0001_EDX,
405 F(FPU) | F(VME) | F(DE) | F(PSE) |
406 F(TSC) | F(MSR) | F(PAE) | F(MCE) |
407 F(CX8) | F(APIC) | 0 | F(SYSCALL) |
408 F(MTRR) | F(PGE) | F(MCA) | F(CMOV) |
409 F(PAT) | F(PSE36) | 0 |
410 f_nx | 0 | F(MMXEXT) | F(MMX) |
411 F(FXSR) | F(FXSR_OPT) | f_gbpages | F(RDTSCP) |
412 0 | f_lm | F(3DNOWEXT) | F(3DNOW)
413 );
414
415 if (!tdp_enabled && IS_ENABLED(CONFIG_X86_64))
416 kvm_cpu_cap_set(X86_FEATURE_GBPAGES);
417
418 kvm_cpu_cap_mask(CPUID_8000_0008_EBX,
419 F(CLZERO) | F(XSAVEERPTR) |
420 F(WBNOINVD) | F(AMD_IBPB) | F(AMD_IBRS) | F(AMD_SSBD) | F(VIRT_SSBD) |
421 F(AMD_SSB_NO) | F(AMD_STIBP) | F(AMD_STIBP_ALWAYS_ON)
422 );
423
424
425
426
427
428
429 if (boot_cpu_has(X86_FEATURE_IBPB))
430 kvm_cpu_cap_set(X86_FEATURE_AMD_IBPB);
431 if (boot_cpu_has(X86_FEATURE_IBRS))
432 kvm_cpu_cap_set(X86_FEATURE_AMD_IBRS);
433 if (boot_cpu_has(X86_FEATURE_STIBP))
434 kvm_cpu_cap_set(X86_FEATURE_AMD_STIBP);
435 if (boot_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD))
436 kvm_cpu_cap_set(X86_FEATURE_AMD_SSBD);
437 if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
438 kvm_cpu_cap_set(X86_FEATURE_AMD_SSB_NO);
439
440
441
442
443 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
444 !boot_cpu_has(X86_FEATURE_AMD_SSBD))
445 kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD);
446
447
448
449
450
451 kvm_cpu_cap_mask(CPUID_8000_000A_EDX, 0);
452
453 kvm_cpu_cap_mask(CPUID_C000_0001_EDX,
454 F(XSTORE) | F(XSTORE_EN) | F(XCRYPT) | F(XCRYPT_EN) |
455 F(ACE2) | F(ACE2_EN) | F(PHE) | F(PHE_EN) |
456 F(PMM) | F(PMM_EN)
457 );
458}
459EXPORT_SYMBOL_GPL(kvm_set_cpu_caps);
460
461struct kvm_cpuid_array {
462 struct kvm_cpuid_entry2 *entries;
463 int maxnent;
464 int nent;
465};
466
467static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array,
468 u32 function, u32 index)
469{
470 struct kvm_cpuid_entry2 *entry;
471
472 if (array->nent >= array->maxnent)
473 return NULL;
474
475 entry = &array->entries[array->nent++];
476
477 entry->function = function;
478 entry->index = index;
479 entry->flags = 0;
480
481 cpuid_count(entry->function, entry->index,
482 &entry->eax, &entry->ebx, &entry->ecx, &entry->edx);
483
484 switch (function) {
485 case 4:
486 case 7:
487 case 0xb:
488 case 0xd:
489 case 0xf:
490 case 0x10:
491 case 0x12:
492 case 0x14:
493 case 0x17:
494 case 0x18:
495 case 0x1f:
496 case 0x8000001d:
497 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
498 break;
499 }
500
501 return entry;
502}
503
504static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func)
505{
506 struct kvm_cpuid_entry2 *entry;
507
508 if (array->nent >= array->maxnent)
509 return -E2BIG;
510
511 entry = &array->entries[array->nent];
512 entry->function = func;
513 entry->index = 0;
514 entry->flags = 0;
515
516 switch (func) {
517 case 0:
518 entry->eax = 7;
519 ++array->nent;
520 break;
521 case 1:
522 entry->ecx = F(MOVBE);
523 ++array->nent;
524 break;
525 case 7:
526 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
527 entry->eax = 0;
528 entry->ecx = F(RDPID);
529 ++array->nent;
530 default:
531 break;
532 }
533
534 return 0;
535}
536
537static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
538{
539 struct kvm_cpuid_entry2 *entry;
540 int r, i, max_idx;
541
542
543 get_cpu();
544
545 r = -E2BIG;
546
547 entry = do_host_cpuid(array, function, 0);
548 if (!entry)
549 goto out;
550
551 switch (function) {
552 case 0:
553
554 entry->eax = min(entry->eax, 0x1fU);
555 break;
556 case 1:
557 cpuid_entry_override(entry, CPUID_1_EDX);
558 cpuid_entry_override(entry, CPUID_1_ECX);
559 break;
560 case 2:
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576 WARN_ON_ONCE((entry->eax & 0xff) > 1);
577 break;
578
579 case 4:
580 case 0x8000001d:
581
582
583
584
585 for (i = 1; entry->eax & 0x1f; ++i) {
586 entry = do_host_cpuid(array, function, i);
587 if (!entry)
588 goto out;
589 }
590 break;
591 case 6:
592 entry->eax = 0x4;
593 entry->ebx = 0;
594 entry->ecx = 0;
595 entry->edx = 0;
596 break;
597
598 case 7:
599 entry->eax = min(entry->eax, 1u);
600 cpuid_entry_override(entry, CPUID_7_0_EBX);
601 cpuid_entry_override(entry, CPUID_7_ECX);
602 cpuid_entry_override(entry, CPUID_7_EDX);
603
604
605 if (entry->eax == 1) {
606 entry = do_host_cpuid(array, function, 1);
607 if (!entry)
608 goto out;
609
610 cpuid_entry_override(entry, CPUID_7_1_EAX);
611 entry->ebx = 0;
612 entry->ecx = 0;
613 entry->edx = 0;
614 }
615 break;
616 case 9:
617 break;
618 case 0xa: {
619 struct x86_pmu_capability cap;
620 union cpuid10_eax eax;
621 union cpuid10_edx edx;
622
623 perf_get_x86_pmu_capability(&cap);
624
625
626
627
628
629 if (!cap.version)
630 memset(&cap, 0, sizeof(cap));
631
632 eax.split.version_id = min(cap.version, 2);
633 eax.split.num_counters = cap.num_counters_gp;
634 eax.split.bit_width = cap.bit_width_gp;
635 eax.split.mask_length = cap.events_mask_len;
636
637 edx.split.num_counters_fixed = min(cap.num_counters_fixed, MAX_FIXED_COUNTERS);
638 edx.split.bit_width_fixed = cap.bit_width_fixed;
639 edx.split.reserved = 0;
640
641 entry->eax = eax.full;
642 entry->ebx = cap.events_mask;
643 entry->ecx = 0;
644 entry->edx = edx.full;
645 break;
646 }
647
648
649
650
651 case 0x1f:
652 case 0xb:
653
654
655
656
657
658 for (i = 1; entry->ecx & 0xff00; ++i) {
659 entry = do_host_cpuid(array, function, i);
660 if (!entry)
661 goto out;
662 }
663 break;
664 case 0xd:
665 entry->eax &= supported_xcr0;
666 entry->ebx = xstate_required_size(supported_xcr0, false);
667 entry->ecx = entry->ebx;
668 entry->edx &= supported_xcr0 >> 32;
669 if (!supported_xcr0)
670 break;
671
672 entry = do_host_cpuid(array, function, 1);
673 if (!entry)
674 goto out;
675
676 cpuid_entry_override(entry, CPUID_D_1_EAX);
677 if (entry->eax & (F(XSAVES)|F(XSAVEC)))
678 entry->ebx = xstate_required_size(supported_xcr0 | supported_xss,
679 true);
680 else {
681 WARN_ON_ONCE(supported_xss != 0);
682 entry->ebx = 0;
683 }
684 entry->ecx &= supported_xss;
685 entry->edx &= supported_xss >> 32;
686
687 for (i = 2; i < 64; ++i) {
688 bool s_state;
689 if (supported_xcr0 & BIT_ULL(i))
690 s_state = false;
691 else if (supported_xss & BIT_ULL(i))
692 s_state = true;
693 else
694 continue;
695
696 entry = do_host_cpuid(array, function, i);
697 if (!entry)
698 goto out;
699
700
701
702
703
704
705
706
707
708 if (WARN_ON_ONCE(!entry->eax || (entry->ecx & 0x1) != s_state)) {
709 --array->nent;
710 continue;
711 }
712 entry->edx = 0;
713 }
714 break;
715
716 case 0x14:
717 if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) {
718 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
719 break;
720 }
721
722 for (i = 1, max_idx = entry->eax; i <= max_idx; ++i) {
723 if (!do_host_cpuid(array, function, i))
724 goto out;
725 }
726 break;
727 case KVM_CPUID_SIGNATURE: {
728 static const char signature[12] = "KVMKVMKVM\0\0";
729 const u32 *sigptr = (const u32 *)signature;
730 entry->eax = KVM_CPUID_FEATURES;
731 entry->ebx = sigptr[0];
732 entry->ecx = sigptr[1];
733 entry->edx = sigptr[2];
734 break;
735 }
736 case KVM_CPUID_FEATURES:
737 entry->eax = (1 << KVM_FEATURE_CLOCKSOURCE) |
738 (1 << KVM_FEATURE_NOP_IO_DELAY) |
739 (1 << KVM_FEATURE_CLOCKSOURCE2) |
740 (1 << KVM_FEATURE_ASYNC_PF) |
741 (1 << KVM_FEATURE_PV_EOI) |
742 (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT) |
743 (1 << KVM_FEATURE_PV_UNHALT) |
744 (1 << KVM_FEATURE_PV_TLB_FLUSH) |
745 (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) |
746 (1 << KVM_FEATURE_PV_SEND_IPI) |
747 (1 << KVM_FEATURE_POLL_CONTROL) |
748 (1 << KVM_FEATURE_PV_SCHED_YIELD) |
749 (1 << KVM_FEATURE_ASYNC_PF_INT);
750
751 if (sched_info_on())
752 entry->eax |= (1 << KVM_FEATURE_STEAL_TIME);
753
754 entry->ebx = 0;
755 entry->ecx = 0;
756 entry->edx = 0;
757 break;
758 case 0x80000000:
759 entry->eax = min(entry->eax, 0x8000001f);
760 break;
761 case 0x80000001:
762 cpuid_entry_override(entry, CPUID_8000_0001_EDX);
763 cpuid_entry_override(entry, CPUID_8000_0001_ECX);
764 break;
765 case 0x80000006:
766
767 break;
768 case 0x80000007:
769
770 entry->edx &= (1 << 8);
771
772 entry->edx &= boot_cpu_data.x86_power;
773 entry->eax = entry->ebx = entry->ecx = 0;
774 break;
775 case 0x80000008: {
776 unsigned g_phys_as = (entry->eax >> 16) & 0xff;
777 unsigned virt_as = max((entry->eax >> 8) & 0xff, 48U);
778 unsigned phys_as = entry->eax & 0xff;
779
780 if (!g_phys_as)
781 g_phys_as = phys_as;
782 entry->eax = g_phys_as | (virt_as << 8);
783 entry->edx = 0;
784 cpuid_entry_override(entry, CPUID_8000_0008_EBX);
785 break;
786 }
787 case 0x8000000A:
788 if (!kvm_cpu_cap_has(X86_FEATURE_SVM)) {
789 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
790 break;
791 }
792 entry->eax = 1;
793 entry->ebx = 8;
794
795 entry->ecx = 0;
796 cpuid_entry_override(entry, CPUID_8000_000A_EDX);
797 break;
798 case 0x80000019:
799 entry->ecx = entry->edx = 0;
800 break;
801 case 0x8000001a:
802 case 0x8000001e:
803 break;
804
805 case 0x8000001F:
806 if (!boot_cpu_has(X86_FEATURE_SEV))
807 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
808 break;
809
810 case 0xC0000000:
811
812 entry->eax = min(entry->eax, 0xC0000004);
813 break;
814 case 0xC0000001:
815 cpuid_entry_override(entry, CPUID_C000_0001_EDX);
816 break;
817 case 3:
818 case 5:
819 case 0xC0000002:
820 case 0xC0000003:
821 case 0xC0000004:
822 default:
823 entry->eax = entry->ebx = entry->ecx = entry->edx = 0;
824 break;
825 }
826
827 r = 0;
828
829out:
830 put_cpu();
831
832 return r;
833}
834
835static int do_cpuid_func(struct kvm_cpuid_array *array, u32 func,
836 unsigned int type)
837{
838 if (type == KVM_GET_EMULATED_CPUID)
839 return __do_cpuid_func_emulated(array, func);
840
841 return __do_cpuid_func(array, func);
842}
843
844#define CENTAUR_CPUID_SIGNATURE 0xC0000000
845
846static int get_cpuid_func(struct kvm_cpuid_array *array, u32 func,
847 unsigned int type)
848{
849 u32 limit;
850 int r;
851
852 if (func == CENTAUR_CPUID_SIGNATURE &&
853 boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR)
854 return 0;
855
856 r = do_cpuid_func(array, func, type);
857 if (r)
858 return r;
859
860 limit = array->entries[array->nent - 1].eax;
861 for (func = func + 1; func <= limit; ++func) {
862 r = do_cpuid_func(array, func, type);
863 if (r)
864 break;
865 }
866
867 return r;
868}
869
870static bool sanity_check_entries(struct kvm_cpuid_entry2 __user *entries,
871 __u32 num_entries, unsigned int ioctl_type)
872{
873 int i;
874 __u32 pad[3];
875
876 if (ioctl_type != KVM_GET_EMULATED_CPUID)
877 return false;
878
879
880
881
882
883
884
885
886
887 for (i = 0; i < num_entries; i++) {
888 if (copy_from_user(pad, entries[i].padding, sizeof(pad)))
889 return true;
890
891 if (pad[0] || pad[1] || pad[2])
892 return true;
893 }
894 return false;
895}
896
897int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid,
898 struct kvm_cpuid_entry2 __user *entries,
899 unsigned int type)
900{
901 static const u32 funcs[] = {
902 0, 0x80000000, CENTAUR_CPUID_SIGNATURE, KVM_CPUID_SIGNATURE,
903 };
904
905 struct kvm_cpuid_array array = {
906 .nent = 0,
907 };
908 int r, i;
909
910 if (cpuid->nent < 1)
911 return -E2BIG;
912 if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
913 cpuid->nent = KVM_MAX_CPUID_ENTRIES;
914
915 if (sanity_check_entries(entries, cpuid->nent, type))
916 return -EINVAL;
917
918 array.entries = vzalloc(array_size(sizeof(struct kvm_cpuid_entry2),
919 cpuid->nent));
920 if (!array.entries)
921 return -ENOMEM;
922
923 array.maxnent = cpuid->nent;
924
925 for (i = 0; i < ARRAY_SIZE(funcs); i++) {
926 r = get_cpuid_func(&array, funcs[i], type);
927 if (r)
928 goto out_free;
929 }
930 cpuid->nent = array.nent;
931
932 if (copy_to_user(entries, array.entries,
933 array.nent * sizeof(struct kvm_cpuid_entry2)))
934 r = -EFAULT;
935
936out_free:
937 vfree(array.entries);
938 return r;
939}
940
941struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu,
942 u32 function, u32 index)
943{
944 struct kvm_cpuid_entry2 *e;
945 int i;
946
947 for (i = 0; i < vcpu->arch.cpuid_nent; ++i) {
948 e = &vcpu->arch.cpuid_entries[i];
949
950 if (e->function == function && (e->index == index ||
951 !(e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX)))
952 return e;
953 }
954 return NULL;
955}
956EXPORT_SYMBOL_GPL(kvm_find_cpuid_entry);
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986static struct kvm_cpuid_entry2 *
987get_out_of_range_cpuid_entry(struct kvm_vcpu *vcpu, u32 *fn_ptr, u32 index)
988{
989 struct kvm_cpuid_entry2 *basic, *class;
990 u32 function = *fn_ptr;
991
992 basic = kvm_find_cpuid_entry(vcpu, 0, 0);
993 if (!basic)
994 return NULL;
995
996 if (is_guest_vendor_amd(basic->ebx, basic->ecx, basic->edx) ||
997 is_guest_vendor_hygon(basic->ebx, basic->ecx, basic->edx))
998 return NULL;
999
1000 if (function >= 0x40000000 && function <= 0x4fffffff)
1001 class = kvm_find_cpuid_entry(vcpu, function & 0xffffff00, 0);
1002 else if (function >= 0xc0000000)
1003 class = kvm_find_cpuid_entry(vcpu, 0xc0000000, 0);
1004 else
1005 class = kvm_find_cpuid_entry(vcpu, function & 0x80000000, 0);
1006
1007 if (class && function <= class->eax)
1008 return NULL;
1009
1010
1011
1012
1013
1014
1015
1016 *fn_ptr = basic->eax;
1017
1018
1019
1020
1021
1022
1023 return kvm_find_cpuid_entry(vcpu, basic->eax, index);
1024}
1025
1026bool kvm_cpuid(struct kvm_vcpu *vcpu, u32 *eax, u32 *ebx,
1027 u32 *ecx, u32 *edx, bool exact_only)
1028{
1029 u32 orig_function = *eax, function = *eax, index = *ecx;
1030 struct kvm_cpuid_entry2 *entry;
1031 bool exact, used_max_basic = false;
1032
1033 entry = kvm_find_cpuid_entry(vcpu, function, index);
1034 exact = !!entry;
1035
1036 if (!entry && !exact_only) {
1037 entry = get_out_of_range_cpuid_entry(vcpu, &function, index);
1038 used_max_basic = !!entry;
1039 }
1040
1041 if (entry) {
1042 *eax = entry->eax;
1043 *ebx = entry->ebx;
1044 *ecx = entry->ecx;
1045 *edx = entry->edx;
1046 if (function == 7 && index == 0) {
1047 u64 data;
1048 if (!__kvm_get_msr(vcpu, MSR_IA32_TSX_CTRL, &data, true) &&
1049 (data & TSX_CTRL_CPUID_CLEAR))
1050 *ebx &= ~(F(RTM) | F(HLE));
1051 }
1052 } else {
1053 *eax = *ebx = *ecx = *edx = 0;
1054
1055
1056
1057
1058
1059
1060
1061 if (function == 0xb || function == 0x1f) {
1062 entry = kvm_find_cpuid_entry(vcpu, function, 1);
1063 if (entry) {
1064 *ecx = index & 0xff;
1065 *edx = entry->edx;
1066 }
1067 }
1068 }
1069 trace_kvm_cpuid(orig_function, index, *eax, *ebx, *ecx, *edx, exact,
1070 used_max_basic);
1071 return exact;
1072}
1073EXPORT_SYMBOL_GPL(kvm_cpuid);
1074
1075int kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
1076{
1077 u32 eax, ebx, ecx, edx;
1078
1079 if (cpuid_fault_enabled(vcpu) && !kvm_require_cpl(vcpu, 0))
1080 return 1;
1081
1082 eax = kvm_rax_read(vcpu);
1083 ecx = kvm_rcx_read(vcpu);
1084 kvm_cpuid(vcpu, &eax, &ebx, &ecx, &edx, false);
1085 kvm_rax_write(vcpu, eax);
1086 kvm_rbx_write(vcpu, ebx);
1087 kvm_rcx_write(vcpu, ecx);
1088 kvm_rdx_write(vcpu, edx);
1089 return kvm_skip_emulated_instruction(vcpu);
1090}
1091EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
1092