1
2
3
4
5
6
7
8
9
10#include <linux/efi.h>
11#include <linux/types.h>
12#include <linux/bitfield.h>
13#include <linux/io.h>
14#include <asm/apic.h>
15#include <asm/desc.h>
16#include <asm/hypervisor.h>
17#include <asm/hyperv-tlfs.h>
18#include <asm/mshyperv.h>
19#include <asm/idtentry.h>
20#include <linux/kexec.h>
21#include <linux/version.h>
22#include <linux/vmalloc.h>
23#include <linux/mm.h>
24#include <linux/hyperv.h>
25#include <linux/slab.h>
26#include <linux/kernel.h>
27#include <linux/cpuhotplug.h>
28#include <linux/syscore_ops.h>
29#include <clocksource/hyperv_timer.h>
30#include <linux/highmem.h>
31#include <linux/swiotlb.h>
32
33int hyperv_init_cpuhp;
34u64 hv_current_partition_id = ~0ull;
35EXPORT_SYMBOL_GPL(hv_current_partition_id);
36
37void *hv_hypercall_pg;
38EXPORT_SYMBOL_GPL(hv_hypercall_pg);
39
40union hv_ghcb * __percpu *hv_ghcb_pg;
41
42
43static void *hv_hypercall_pg_saved;
44
45struct hv_vp_assist_page **hv_vp_assist_page;
46EXPORT_SYMBOL_GPL(hv_vp_assist_page);
47
48static int hyperv_init_ghcb(void)
49{
50 u64 ghcb_gpa;
51 void *ghcb_va;
52 void **ghcb_base;
53
54 if (!hv_isolation_type_snp())
55 return 0;
56
57 if (!hv_ghcb_pg)
58 return -EINVAL;
59
60
61
62
63
64
65 rdmsrl(MSR_AMD64_SEV_ES_GHCB, ghcb_gpa);
66 ghcb_va = memremap(ghcb_gpa, HV_HYP_PAGE_SIZE, MEMREMAP_WB);
67 if (!ghcb_va)
68 return -ENOMEM;
69
70 ghcb_base = (void **)this_cpu_ptr(hv_ghcb_pg);
71 *ghcb_base = ghcb_va;
72
73 return 0;
74}
75
76static int hv_cpu_init(unsigned int cpu)
77{
78 union hv_vp_assist_msr_contents msr = { 0 };
79 struct hv_vp_assist_page **hvp = &hv_vp_assist_page[smp_processor_id()];
80 int ret;
81
82 ret = hv_common_cpu_init(cpu);
83 if (ret)
84 return ret;
85
86 if (!hv_vp_assist_page)
87 return 0;
88
89 if (!*hvp) {
90 if (hv_root_partition) {
91
92
93
94
95 rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
96 *hvp = memremap(msr.pfn <<
97 HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT,
98 PAGE_SIZE, MEMREMAP_WB);
99 } else {
100
101
102
103
104
105
106
107
108 *hvp = __vmalloc(PAGE_SIZE, GFP_KERNEL | __GFP_ZERO);
109 if (*hvp)
110 msr.pfn = vmalloc_to_pfn(*hvp);
111 }
112 WARN_ON(!(*hvp));
113 if (*hvp) {
114 msr.enable = 1;
115 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
116 }
117 }
118
119 return hyperv_init_ghcb();
120}
121
122static void (*hv_reenlightenment_cb)(void);
123
124static void hv_reenlightenment_notify(struct work_struct *dummy)
125{
126 struct hv_tsc_emulation_status emu_status;
127
128 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
129
130
131 if (hv_reenlightenment_cb && emu_status.inprogress)
132 hv_reenlightenment_cb();
133}
134static DECLARE_DELAYED_WORK(hv_reenlightenment_work, hv_reenlightenment_notify);
135
136void hyperv_stop_tsc_emulation(void)
137{
138 u64 freq;
139 struct hv_tsc_emulation_status emu_status;
140
141 rdmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
142 emu_status.inprogress = 0;
143 wrmsrl(HV_X64_MSR_TSC_EMULATION_STATUS, *(u64 *)&emu_status);
144
145 rdmsrl(HV_X64_MSR_TSC_FREQUENCY, freq);
146 tsc_khz = div64_u64(freq, 1000);
147}
148EXPORT_SYMBOL_GPL(hyperv_stop_tsc_emulation);
149
150static inline bool hv_reenlightenment_available(void)
151{
152
153
154
155
156 return ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS &&
157 ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE &&
158 ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT;
159}
160
161DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment)
162{
163 ack_APIC_irq();
164 inc_irq_stat(irq_hv_reenlightenment_count);
165 schedule_delayed_work(&hv_reenlightenment_work, HZ/10);
166}
167
168void set_hv_tscchange_cb(void (*cb)(void))
169{
170 struct hv_reenlightenment_control re_ctrl = {
171 .vector = HYPERV_REENLIGHTENMENT_VECTOR,
172 .enabled = 1,
173 };
174 struct hv_tsc_emulation_control emu_ctrl = {.enabled = 1};
175
176 if (!hv_reenlightenment_available()) {
177 pr_warn("Hyper-V: reenlightenment support is unavailable\n");
178 return;
179 }
180
181 if (!hv_vp_index)
182 return;
183
184 hv_reenlightenment_cb = cb;
185
186
187 wmb();
188
189 re_ctrl.target_vp = hv_vp_index[get_cpu()];
190
191 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
192 wrmsrl(HV_X64_MSR_TSC_EMULATION_CONTROL, *((u64 *)&emu_ctrl));
193
194 put_cpu();
195}
196EXPORT_SYMBOL_GPL(set_hv_tscchange_cb);
197
198void clear_hv_tscchange_cb(void)
199{
200 struct hv_reenlightenment_control re_ctrl;
201
202 if (!hv_reenlightenment_available())
203 return;
204
205 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
206 re_ctrl.enabled = 0;
207 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *(u64 *)&re_ctrl);
208
209 hv_reenlightenment_cb = NULL;
210}
211EXPORT_SYMBOL_GPL(clear_hv_tscchange_cb);
212
213static int hv_cpu_die(unsigned int cpu)
214{
215 struct hv_reenlightenment_control re_ctrl;
216 unsigned int new_cpu;
217 void **ghcb_va;
218
219 if (hv_ghcb_pg) {
220 ghcb_va = (void **)this_cpu_ptr(hv_ghcb_pg);
221 if (*ghcb_va)
222 memunmap(*ghcb_va);
223 *ghcb_va = NULL;
224 }
225
226 hv_common_cpu_die(cpu);
227
228 if (hv_vp_assist_page && hv_vp_assist_page[cpu]) {
229 union hv_vp_assist_msr_contents msr = { 0 };
230 if (hv_root_partition) {
231
232
233
234
235
236
237 memunmap(hv_vp_assist_page[cpu]);
238 hv_vp_assist_page[cpu] = NULL;
239 rdmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
240 msr.enable = 0;
241 }
242 wrmsrl(HV_X64_MSR_VP_ASSIST_PAGE, msr.as_uint64);
243 }
244
245 if (hv_reenlightenment_cb == NULL)
246 return 0;
247
248 rdmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
249 if (re_ctrl.target_vp == hv_vp_index[cpu]) {
250
251
252
253
254
255 new_cpu = cpumask_any_but(cpu_online_mask, cpu);
256
257 if (new_cpu < nr_cpu_ids)
258 re_ctrl.target_vp = hv_vp_index[new_cpu];
259 else
260 re_ctrl.enabled = 0;
261
262 wrmsrl(HV_X64_MSR_REENLIGHTENMENT_CONTROL, *((u64 *)&re_ctrl));
263 }
264
265 return 0;
266}
267
268static int __init hv_pci_init(void)
269{
270 int gen2vm = efi_enabled(EFI_BOOT);
271
272
273
274
275
276
277 if (gen2vm)
278 return 0;
279
280
281 return 1;
282}
283
284static int hv_suspend(void)
285{
286 union hv_x64_msr_hypercall_contents hypercall_msr;
287 int ret;
288
289 if (hv_root_partition)
290 return -EPERM;
291
292
293
294
295
296
297
298
299 hv_hypercall_pg_saved = hv_hypercall_pg;
300 hv_hypercall_pg = NULL;
301
302
303 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
304 hypercall_msr.enable = 0;
305 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
306
307 ret = hv_cpu_die(0);
308 return ret;
309}
310
311static void hv_resume(void)
312{
313 union hv_x64_msr_hypercall_contents hypercall_msr;
314 int ret;
315
316 ret = hv_cpu_init(0);
317 WARN_ON(ret);
318
319
320 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
321 hypercall_msr.enable = 1;
322 hypercall_msr.guest_physical_address =
323 vmalloc_to_pfn(hv_hypercall_pg_saved);
324 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
325
326 hv_hypercall_pg = hv_hypercall_pg_saved;
327 hv_hypercall_pg_saved = NULL;
328
329
330
331
332
333 if (hv_reenlightenment_cb)
334 set_hv_tscchange_cb(hv_reenlightenment_cb);
335}
336
337
338static struct syscore_ops hv_syscore_ops = {
339 .suspend = hv_suspend,
340 .resume = hv_resume,
341};
342
343static void (* __initdata old_setup_percpu_clockev)(void);
344
345static void __init hv_stimer_setup_percpu_clockev(void)
346{
347
348
349
350
351 (void)hv_stimer_alloc(false);
352
353
354
355
356
357
358 if (old_setup_percpu_clockev)
359 old_setup_percpu_clockev();
360}
361
362static void __init hv_get_partition_id(void)
363{
364 struct hv_get_partition_id *output_page;
365 u64 status;
366 unsigned long flags;
367
368 local_irq_save(flags);
369 output_page = *this_cpu_ptr(hyperv_pcpu_output_arg);
370 status = hv_do_hypercall(HVCALL_GET_PARTITION_ID, NULL, output_page);
371 if (!hv_result_success(status)) {
372
373 pr_err("Failed to get partition ID: %lld\n", status);
374 BUG();
375 }
376 hv_current_partition_id = output_page->partition_id;
377 local_irq_restore(flags);
378}
379
380
381
382
383
384
385
386
387
388void __init hyperv_init(void)
389{
390 u64 guest_id;
391 union hv_x64_msr_hypercall_contents hypercall_msr;
392 int cpuhp;
393
394 if (x86_hyper_type != X86_HYPER_MS_HYPERV)
395 return;
396
397 if (hv_common_init())
398 return;
399
400 hv_vp_assist_page = kcalloc(num_possible_cpus(),
401 sizeof(*hv_vp_assist_page), GFP_KERNEL);
402 if (!hv_vp_assist_page) {
403 ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED;
404 goto common_free;
405 }
406
407 if (hv_isolation_type_snp()) {
408 hv_ghcb_pg = alloc_percpu(union hv_ghcb *);
409 if (!hv_ghcb_pg)
410 goto free_vp_assist_page;
411 }
412
413 cpuhp = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/hyperv_init:online",
414 hv_cpu_init, hv_cpu_die);
415 if (cpuhp < 0)
416 goto free_ghcb_page;
417
418
419
420
421
422
423 guest_id = generate_guest_id(0, LINUX_VERSION_CODE, 0);
424 wrmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
425
426
427 hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, guest_id);
428
429 hv_hypercall_pg = __vmalloc_node_range(PAGE_SIZE, 1, VMALLOC_START,
430 VMALLOC_END, GFP_KERNEL, PAGE_KERNEL_ROX,
431 VM_FLUSH_RESET_PERMS, NUMA_NO_NODE,
432 __builtin_return_address(0));
433 if (hv_hypercall_pg == NULL)
434 goto clean_guest_os_id;
435
436 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
437 hypercall_msr.enable = 1;
438
439 if (hv_root_partition) {
440 struct page *pg;
441 void *src, *dst;
442
443
444
445
446
447
448
449
450
451
452
453 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
454
455 pg = vmalloc_to_page(hv_hypercall_pg);
456 dst = kmap(pg);
457 src = memremap(hypercall_msr.guest_physical_address << PAGE_SHIFT, PAGE_SIZE,
458 MEMREMAP_WB);
459 BUG_ON(!(src && dst));
460 memcpy(dst, src, HV_HYP_PAGE_SIZE);
461 memunmap(src);
462 kunmap(pg);
463 } else {
464 hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg);
465 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
466 }
467
468
469
470
471
472
473
474
475 old_setup_percpu_clockev = x86_init.timers.setup_percpu_clockev;
476 x86_init.timers.setup_percpu_clockev = hv_stimer_setup_percpu_clockev;
477
478 hv_apic_init();
479
480 x86_init.pci.arch_init = hv_pci_init;
481
482 register_syscore_ops(&hv_syscore_ops);
483
484 hyperv_init_cpuhp = cpuhp;
485
486 if (cpuid_ebx(HYPERV_CPUID_FEATURES) & HV_ACCESS_PARTITION_ID)
487 hv_get_partition_id();
488
489 BUG_ON(hv_root_partition && hv_current_partition_id == ~0ull);
490
491#ifdef CONFIG_PCI_MSI
492
493
494
495
496 if (hv_root_partition)
497 x86_init.irqs.create_pci_msi_domain = hv_create_pci_msi_domain;
498#endif
499
500
501 hv_query_ext_cap(0);
502
503#ifdef CONFIG_SWIOTLB
504
505
506
507
508
509 if (hv_is_isolation_supported())
510 swiotlb_update_mem_attributes();
511#endif
512
513 return;
514
515clean_guest_os_id:
516 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
517 hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, 0);
518 cpuhp_remove_state(cpuhp);
519free_ghcb_page:
520 free_percpu(hv_ghcb_pg);
521free_vp_assist_page:
522 kfree(hv_vp_assist_page);
523 hv_vp_assist_page = NULL;
524common_free:
525 hv_common_free();
526}
527
528
529
530
531void hyperv_cleanup(void)
532{
533 union hv_x64_msr_hypercall_contents hypercall_msr;
534
535 unregister_syscore_ops(&hv_syscore_ops);
536
537
538 wrmsrl(HV_X64_MSR_GUEST_OS_ID, 0);
539 hv_ghcb_msr_write(HV_X64_MSR_GUEST_OS_ID, 0);
540
541
542
543
544
545
546 hv_hypercall_pg = NULL;
547
548
549 hypercall_msr.as_uint64 = 0;
550 wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
551
552
553 hypercall_msr.as_uint64 = 0;
554 wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64);
555}
556
557void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die)
558{
559 static bool panic_reported;
560 u64 guest_id;
561
562 if (in_die && !panic_on_oops)
563 return;
564
565
566
567
568
569
570 if (panic_reported)
571 return;
572 panic_reported = true;
573
574 rdmsrl(HV_X64_MSR_GUEST_OS_ID, guest_id);
575
576 wrmsrl(HV_X64_MSR_CRASH_P0, err);
577 wrmsrl(HV_X64_MSR_CRASH_P1, guest_id);
578 wrmsrl(HV_X64_MSR_CRASH_P2, regs->ip);
579 wrmsrl(HV_X64_MSR_CRASH_P3, regs->ax);
580 wrmsrl(HV_X64_MSR_CRASH_P4, regs->sp);
581
582
583
584
585 wrmsrl(HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_CRASH_NOTIFY);
586}
587EXPORT_SYMBOL_GPL(hyperv_report_panic);
588
589bool hv_is_hyperv_initialized(void)
590{
591 union hv_x64_msr_hypercall_contents hypercall_msr;
592
593
594
595
596
597 if (x86_hyper_type != X86_HYPER_MS_HYPERV)
598 return false;
599
600
601
602
603
604 hypercall_msr.as_uint64 = 0;
605 rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
606
607 return hypercall_msr.enable;
608}
609EXPORT_SYMBOL_GPL(hv_is_hyperv_initialized);
610