1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/cpu.h>
15#include <linux/kernel.h>
16#include <linux/init.h>
17#include <linux/smp.h>
18#include <linux/preempt.h>
19#include <linux/hardirq.h>
20#include <linux/percpu.h>
21#include <linux/delay.h>
22#include <linux/start_kernel.h>
23#include <linux/sched.h>
24#include <linux/kprobes.h>
25#include <linux/bootmem.h>
26#include <linux/module.h>
27#include <linux/mm.h>
28#include <linux/page-flags.h>
29#include <linux/highmem.h>
30#include <linux/console.h>
31#include <linux/pci.h>
32#include <linux/gfp.h>
33#include <linux/memblock.h>
34#include <linux/edd.h>
35#include <linux/crash_dump.h>
36#include <linux/frame.h>
37
38#ifdef CONFIG_KEXEC_CORE
39#include <linux/kexec.h>
40#endif
41
42#include <xen/xen.h>
43#include <xen/events.h>
44#include <xen/interface/xen.h>
45#include <xen/interface/version.h>
46#include <xen/interface/physdev.h>
47#include <xen/interface/vcpu.h>
48#include <xen/interface/memory.h>
49#include <xen/interface/xen-mca.h>
50#include <xen/features.h>
51#include <xen/page.h>
52#include <xen/hvm.h>
53#include <xen/hvc-console.h>
54#include <xen/acpi.h>
55
56#include <asm/paravirt.h>
57#include <asm/apic.h>
58#include <asm/page.h>
59#include <asm/xen/pci.h>
60#include <asm/xen/hypercall.h>
61#include <asm/xen/hypervisor.h>
62#include <asm/fixmap.h>
63#include <asm/processor.h>
64#include <asm/proto.h>
65#include <asm/msr-index.h>
66#include <asm/traps.h>
67#include <asm/setup.h>
68#include <asm/desc.h>
69#include <asm/pgalloc.h>
70#include <asm/pgtable.h>
71#include <asm/tlbflush.h>
72#include <asm/reboot.h>
73#include <asm/stackprotector.h>
74#include <asm/hypervisor.h>
75#include <asm/mwait.h>
76#include <asm/pci_x86.h>
77#include <asm/pat.h>
78
79#ifdef CONFIG_ACPI
80#include <linux/acpi.h>
81#include <asm/acpi.h>
82#include <acpi/pdc_intel.h>
83#include <acpi/processor.h>
84#include <xen/interface/platform.h>
85#endif
86
87#include "xen-ops.h"
88#include "mmu.h"
89#include "smp.h"
90#include "multicalls.h"
91
92EXPORT_SYMBOL_GPL(hypercall_page);
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
111
112
113
114
115
116
117DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
118
119
120DEFINE_PER_CPU(uint32_t, xen_vcpu_id);
121EXPORT_PER_CPU_SYMBOL(xen_vcpu_id);
122
123enum xen_domain_type xen_domain_type = XEN_NATIVE;
124EXPORT_SYMBOL_GPL(xen_domain_type);
125
126unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
127EXPORT_SYMBOL(machine_to_phys_mapping);
128unsigned long machine_to_phys_nr;
129EXPORT_SYMBOL(machine_to_phys_nr);
130
131struct start_info *xen_start_info;
132EXPORT_SYMBOL_GPL(xen_start_info);
133
134struct shared_info xen_dummy_shared_info;
135
136void *xen_initial_gdt;
137
138RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
139__read_mostly int xen_have_vector_callback;
140EXPORT_SYMBOL_GPL(xen_have_vector_callback);
141
142
143
144
145
146struct shared_info *HYPERVISOR_shared_info = &xen_dummy_shared_info;
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161int xen_have_vcpu_info_placement = 1;
162
163struct tls_descs {
164 struct desc_struct desc[3];
165};
166
167
168
169
170
171
172
173
174static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc);
175
176#define XEN_HVM_CPUID_VCPU_ID_PRESENT (1u << 3)
177
178static int xen_vcpu_setup_restore(int cpu)
179{
180 int rc = 0;
181
182
183 xen_vcpu_info_reset(cpu);
184
185
186
187
188
189 if (xen_pv_domain() ||
190 (xen_hvm_domain() && cpu_online(cpu))) {
191 rc = xen_vcpu_setup(cpu);
192 }
193
194 return rc;
195}
196
197
198
199
200
201
202void xen_vcpu_restore(void)
203{
204 int cpu, rc;
205
206 for_each_possible_cpu(cpu) {
207 bool other_cpu = (cpu != smp_processor_id());
208 bool is_up;
209
210 if (xen_vcpu_nr(cpu) == XEN_VCPU_ID_INVALID)
211 continue;
212
213
214 is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up,
215 xen_vcpu_nr(cpu), NULL) > 0;
216
217 if (other_cpu && is_up &&
218 HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL))
219 BUG();
220
221 if (xen_pv_domain() || xen_feature(XENFEAT_hvm_safe_pvclock))
222 xen_setup_runstate_info(cpu);
223
224 rc = xen_vcpu_setup_restore(cpu);
225 if (rc)
226 pr_emerg_once("vcpu restore failed for cpu=%d err=%d. "
227 "System will hang.\n", cpu, rc);
228
229
230
231
232
233
234
235
236
237 if (other_cpu && is_up && (rc == 0) &&
238 HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL))
239 BUG();
240 }
241}
242
243void xen_vcpu_info_reset(int cpu)
244{
245 if (xen_vcpu_nr(cpu) < MAX_VIRT_CPUS) {
246 per_cpu(xen_vcpu, cpu) =
247 &HYPERVISOR_shared_info->vcpu_info[xen_vcpu_nr(cpu)];
248 } else {
249
250 per_cpu(xen_vcpu, cpu) = NULL;
251 }
252}
253
254int xen_vcpu_setup(int cpu)
255{
256 struct vcpu_register_vcpu_info info;
257 int err;
258 struct vcpu_info *vcpup;
259
260 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info);
261
262
263
264
265
266
267
268
269
270
271
272
273 if (xen_hvm_domain()) {
274 if (per_cpu(xen_vcpu, cpu) == &per_cpu(xen_vcpu_info, cpu))
275 return 0;
276 }
277
278 if (xen_have_vcpu_info_placement) {
279 vcpup = &per_cpu(xen_vcpu_info, cpu);
280 info.mfn = arbitrary_virt_to_mfn(vcpup);
281 info.offset = offset_in_page(vcpup);
282
283
284
285
286
287
288
289
290
291
292
293 err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info,
294 xen_vcpu_nr(cpu), &info);
295
296 if (err) {
297 pr_warn_once("register_vcpu_info failed: cpu=%d err=%d\n",
298 cpu, err);
299 xen_have_vcpu_info_placement = 0;
300 } else {
301
302
303
304
305 per_cpu(xen_vcpu, cpu) = vcpup;
306 }
307 }
308
309 if (!xen_have_vcpu_info_placement)
310 xen_vcpu_info_reset(cpu);
311
312 return ((per_cpu(xen_vcpu, cpu) == NULL) ? -ENODEV : 0);
313}
314
315static void __init xen_banner(void)
316{
317 unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL);
318 struct xen_extraversion extra;
319 HYPERVISOR_xen_version(XENVER_extraversion, &extra);
320
321 printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
322 pv_info.name);
323 printk(KERN_INFO "Xen version: %d.%d%s%s\n",
324 version >> 16, version & 0xffff, extra.extraversion,
325 xen_feature(XENFEAT_mmu_pt_update_preserve_ad) ? " (preserve-AD)" : "");
326}
327
328bool
329xen_running_on_version_or_later(unsigned int major, unsigned int minor)
330{
331 unsigned int version;
332
333 if (!xen_domain())
334 return false;
335
336 version = HYPERVISOR_xen_version(XENVER_version, NULL);
337 if ((((version >> 16) == major) && ((version & 0xffff) >= minor)) ||
338 ((version >> 16) > major))
339 return true;
340 return false;
341}
342
343#define CPUID_THERM_POWER_LEAF 6
344#define APERFMPERF_PRESENT 0
345
346static __read_mostly unsigned int cpuid_leaf1_edx_mask = ~0;
347static __read_mostly unsigned int cpuid_leaf1_ecx_mask = ~0;
348
349static __read_mostly unsigned int cpuid_leaf1_ecx_set_mask;
350static __read_mostly unsigned int cpuid_leaf5_ecx_val;
351static __read_mostly unsigned int cpuid_leaf5_edx_val;
352
353static void xen_cpuid(unsigned int *ax, unsigned int *bx,
354 unsigned int *cx, unsigned int *dx)
355{
356 unsigned maskebx = ~0;
357 unsigned maskecx = ~0;
358 unsigned maskedx = ~0;
359 unsigned setecx = 0;
360
361
362
363
364 switch (*ax) {
365 case 1:
366 maskecx = cpuid_leaf1_ecx_mask;
367 setecx = cpuid_leaf1_ecx_set_mask;
368 maskedx = cpuid_leaf1_edx_mask;
369 break;
370
371 case CPUID_MWAIT_LEAF:
372
373 *ax = 0;
374 *bx = 0;
375 *cx = cpuid_leaf5_ecx_val;
376 *dx = cpuid_leaf5_edx_val;
377 return;
378
379 case CPUID_THERM_POWER_LEAF:
380
381 maskecx = ~(1 << APERFMPERF_PRESENT);
382 break;
383
384 case 0xb:
385
386 maskebx = 0;
387 break;
388 }
389
390 asm(XEN_EMULATE_PREFIX "cpuid"
391 : "=a" (*ax),
392 "=b" (*bx),
393 "=c" (*cx),
394 "=d" (*dx)
395 : "0" (*ax), "2" (*cx));
396
397 *bx &= maskebx;
398 *cx &= maskecx;
399 *cx |= setecx;
400 *dx &= maskedx;
401}
402STACK_FRAME_NON_STANDARD(xen_cpuid);
403
404static bool __init xen_check_mwait(void)
405{
406#ifdef CONFIG_ACPI
407 struct xen_platform_op op = {
408 .cmd = XENPF_set_processor_pminfo,
409 .u.set_pminfo.id = -1,
410 .u.set_pminfo.type = XEN_PM_PDC,
411 };
412 uint32_t buf[3];
413 unsigned int ax, bx, cx, dx;
414 unsigned int mwait_mask;
415
416
417
418
419
420
421
422
423
424 if (!xen_initial_domain())
425 return false;
426
427
428
429
430
431 if (!xen_running_on_version_or_later(4, 2))
432 return false;
433
434 ax = 1;
435 cx = 0;
436
437 native_cpuid(&ax, &bx, &cx, &dx);
438
439 mwait_mask = (1 << (X86_FEATURE_EST % 32)) |
440 (1 << (X86_FEATURE_MWAIT % 32));
441
442 if ((cx & mwait_mask) != mwait_mask)
443 return false;
444
445
446
447
448
449 ax = CPUID_MWAIT_LEAF;
450 bx = 0;
451 cx = 0;
452 dx = 0;
453
454 native_cpuid(&ax, &bx, &cx, &dx);
455
456
457
458
459 buf[0] = ACPI_PDC_REVISION_ID;
460 buf[1] = 1;
461 buf[2] = (ACPI_PDC_C_CAPABILITY_SMP | ACPI_PDC_EST_CAPABILITY_SWSMP);
462
463 set_xen_guest_handle(op.u.set_pminfo.pdc, buf);
464
465 if ((HYPERVISOR_dom0_op(&op) == 0) &&
466 (buf[2] & (ACPI_PDC_C_C1_FFH | ACPI_PDC_C_C2C3_FFH))) {
467 cpuid_leaf5_ecx_val = cx;
468 cpuid_leaf5_edx_val = dx;
469 }
470 return true;
471#else
472 return false;
473#endif
474}
475static void __init xen_init_cpuid_mask(void)
476{
477 unsigned int ax, bx, cx, dx;
478 unsigned int xsave_mask;
479
480 cpuid_leaf1_edx_mask =
481 ~((1 << X86_FEATURE_MTRR) |
482 (1 << X86_FEATURE_ACC));
483
484 if (!xen_initial_domain())
485 cpuid_leaf1_edx_mask &=
486 ~((1 << X86_FEATURE_APIC) |
487 (1 << X86_FEATURE_ACPI));
488
489 cpuid_leaf1_ecx_mask &= ~(1 << (X86_FEATURE_X2APIC % 32));
490
491 ax = 1;
492 cx = 0;
493 xen_cpuid(&ax, &bx, &cx, &dx);
494
495 xsave_mask =
496 (1 << (X86_FEATURE_XSAVE % 32)) |
497 (1 << (X86_FEATURE_OSXSAVE % 32));
498
499
500 if ((cx & xsave_mask) != xsave_mask)
501 cpuid_leaf1_ecx_mask &= ~xsave_mask;
502 if (xen_check_mwait())
503 cpuid_leaf1_ecx_set_mask = (1 << (X86_FEATURE_MWAIT % 32));
504}
505
506static void xen_set_debugreg(int reg, unsigned long val)
507{
508 HYPERVISOR_set_debugreg(reg, val);
509}
510
511static unsigned long xen_get_debugreg(int reg)
512{
513 return HYPERVISOR_get_debugreg(reg);
514}
515
516static void xen_end_context_switch(struct task_struct *next)
517{
518 xen_mc_flush();
519 paravirt_end_context_switch(next);
520}
521
522static unsigned long xen_store_tr(void)
523{
524 return 0;
525}
526
527
528
529
530
531
532
533static void set_aliased_prot(void *v, pgprot_t prot)
534{
535 int level;
536 pte_t *ptep;
537 pte_t pte;
538 unsigned long pfn;
539 struct page *page;
540
541 ptep = lookup_address((unsigned long)v, &level);
542 BUG_ON(ptep == NULL);
543
544 pfn = pte_pfn(*ptep);
545 page = pfn_to_page(pfn);
546
547 pte = pfn_pte(pfn, prot);
548
549 if (HYPERVISOR_update_va_mapping((unsigned long)v, pte, 0))
550 BUG();
551
552 if (!PageHighMem(page)) {
553 void *av = __va(PFN_PHYS(pfn));
554
555 if (av != v)
556 if (HYPERVISOR_update_va_mapping((unsigned long)av, pte, 0))
557 BUG();
558 } else
559 kmap_flush_unused();
560}
561
562static void xen_alloc_ldt(struct desc_struct *ldt, unsigned entries)
563{
564 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
565 int i;
566
567 for(i = 0; i < entries; i += entries_per_page)
568 set_aliased_prot(ldt + i, PAGE_KERNEL_RO);
569}
570
571static void xen_free_ldt(struct desc_struct *ldt, unsigned entries)
572{
573 const unsigned entries_per_page = PAGE_SIZE / LDT_ENTRY_SIZE;
574 int i;
575
576 for(i = 0; i < entries; i += entries_per_page)
577 set_aliased_prot(ldt + i, PAGE_KERNEL);
578}
579
580static void xen_set_ldt(const void *addr, unsigned entries)
581{
582 struct mmuext_op *op;
583 struct multicall_space mcs = xen_mc_entry(sizeof(*op));
584
585 trace_xen_cpu_set_ldt(addr, entries);
586
587 op = mcs.args;
588 op->cmd = MMUEXT_SET_LDT;
589 op->arg1.linear_addr = (unsigned long)addr;
590 op->arg2.nr_ents = entries;
591
592 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF);
593
594 xen_mc_issue(PARAVIRT_LAZY_CPU);
595}
596
597static void xen_load_gdt(const struct desc_ptr *dtr)
598{
599 unsigned long va = dtr->address;
600 unsigned int size = dtr->size + 1;
601 unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
602 unsigned long frames[pages];
603 int f;
604
605
606
607
608
609
610 BUG_ON(size > 65536);
611 BUG_ON(va & ~PAGE_MASK);
612
613 for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
614 int level;
615 pte_t *ptep;
616 unsigned long pfn, mfn;
617 void *virt;
618
619
620
621
622
623
624
625
626 ptep = lookup_address(va, &level);
627 BUG_ON(ptep == NULL);
628
629 pfn = pte_pfn(*ptep);
630 mfn = pfn_to_mfn(pfn);
631 virt = __va(PFN_PHYS(pfn));
632
633 frames[f] = mfn;
634
635 make_lowmem_page_readonly((void *)va);
636 make_lowmem_page_readonly(virt);
637 }
638
639 if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
640 BUG();
641}
642
643
644
645
646static void __init xen_load_gdt_boot(const struct desc_ptr *dtr)
647{
648 unsigned long va = dtr->address;
649 unsigned int size = dtr->size + 1;
650 unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
651 unsigned long frames[pages];
652 int f;
653
654
655
656
657
658
659 BUG_ON(size > 65536);
660 BUG_ON(va & ~PAGE_MASK);
661
662 for (f = 0; va < dtr->address + size; va += PAGE_SIZE, f++) {
663 pte_t pte;
664 unsigned long pfn, mfn;
665
666 pfn = virt_to_pfn(va);
667 mfn = pfn_to_mfn(pfn);
668
669 pte = pfn_pte(pfn, PAGE_KERNEL_RO);
670
671 if (HYPERVISOR_update_va_mapping((unsigned long)va, pte, 0))
672 BUG();
673
674 frames[f] = mfn;
675 }
676
677 if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
678 BUG();
679}
680
681static inline bool desc_equal(const struct desc_struct *d1,
682 const struct desc_struct *d2)
683{
684 return d1->a == d2->a && d1->b == d2->b;
685}
686
687static void load_TLS_descriptor(struct thread_struct *t,
688 unsigned int cpu, unsigned int i)
689{
690 struct desc_struct *shadow = &per_cpu(shadow_tls_desc, cpu).desc[i];
691 struct desc_struct *gdt;
692 xmaddr_t maddr;
693 struct multicall_space mc;
694
695 if (desc_equal(shadow, &t->tls_array[i]))
696 return;
697
698 *shadow = t->tls_array[i];
699
700 gdt = get_cpu_gdt_table(cpu);
701 maddr = arbitrary_virt_to_machine(&gdt[GDT_ENTRY_TLS_MIN+i]);
702 mc = __xen_mc_entry(0);
703
704 MULTI_update_descriptor(mc.mc, maddr.maddr, t->tls_array[i]);
705}
706
707static void xen_load_tls(struct thread_struct *t, unsigned int cpu)
708{
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727 if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_CPU) {
728#ifdef CONFIG_X86_32
729 lazy_load_gs(0);
730#else
731 loadsegment(fs, 0);
732#endif
733 }
734
735 xen_mc_batch();
736
737 load_TLS_descriptor(t, cpu, 0);
738 load_TLS_descriptor(t, cpu, 1);
739 load_TLS_descriptor(t, cpu, 2);
740
741 xen_mc_issue(PARAVIRT_LAZY_CPU);
742}
743
744#ifdef CONFIG_X86_64
745static void xen_load_gs_index(unsigned int idx)
746{
747 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, idx))
748 BUG();
749}
750#endif
751
752static void xen_write_ldt_entry(struct desc_struct *dt, int entrynum,
753 const void *ptr)
754{
755 xmaddr_t mach_lp = arbitrary_virt_to_machine(&dt[entrynum]);
756 u64 entry = *(u64 *)ptr;
757
758 trace_xen_cpu_write_ldt_entry(dt, entrynum, entry);
759
760 preempt_disable();
761
762 xen_mc_flush();
763 if (HYPERVISOR_update_descriptor(mach_lp.maddr, entry))
764 BUG();
765
766 preempt_enable();
767}
768
769static int cvt_gate_to_trap(int vector, const gate_desc *val,
770 struct trap_info *info)
771{
772 unsigned long addr;
773
774 if (val->type != GATE_TRAP && val->type != GATE_INTERRUPT)
775 return 0;
776
777 info->vector = vector;
778
779 addr = gate_offset(*val);
780#ifdef CONFIG_X86_64
781
782
783
784
785
786
787
788 if (addr == (unsigned long)debug)
789 addr = (unsigned long)xen_debug;
790 else if (addr == (unsigned long)int3)
791 addr = (unsigned long)xen_int3;
792 else if (addr == (unsigned long)stack_segment)
793 addr = (unsigned long)xen_stack_segment;
794 else if (addr == (unsigned long)double_fault ||
795 addr == (unsigned long)nmi) {
796
797 return 0;
798#ifdef CONFIG_X86_MCE
799 } else if (addr == (unsigned long)machine_check) {
800
801
802
803
804 ;
805#endif
806 } else {
807
808 if (WARN_ON(val->ist != 0))
809 return 0;
810 }
811#endif
812 info->address = addr;
813
814 info->cs = gate_segment(*val);
815 info->flags = val->dpl;
816
817 if (val->type == GATE_INTERRUPT)
818 info->flags |= 1 << 2;
819
820 return 1;
821}
822
823
824static DEFINE_PER_CPU(struct desc_ptr, idt_desc);
825
826
827
828static void xen_write_idt_entry(gate_desc *dt, int entrynum, const gate_desc *g)
829{
830 unsigned long p = (unsigned long)&dt[entrynum];
831 unsigned long start, end;
832
833 trace_xen_cpu_write_idt_entry(dt, entrynum, g);
834
835 preempt_disable();
836
837 start = __this_cpu_read(idt_desc.address);
838 end = start + __this_cpu_read(idt_desc.size) + 1;
839
840 xen_mc_flush();
841
842 native_write_idt_entry(dt, entrynum, g);
843
844 if (p >= start && (p + 8) <= end) {
845 struct trap_info info[2];
846
847 info[1].address = 0;
848
849 if (cvt_gate_to_trap(entrynum, g, &info[0]))
850 if (HYPERVISOR_set_trap_table(info))
851 BUG();
852 }
853
854 preempt_enable();
855}
856
857static void xen_convert_trap_info(const struct desc_ptr *desc,
858 struct trap_info *traps)
859{
860 unsigned in, out, count;
861
862 count = (desc->size+1) / sizeof(gate_desc);
863 BUG_ON(count > 256);
864
865 for (in = out = 0; in < count; in++) {
866 gate_desc *entry = (gate_desc*)(desc->address) + in;
867
868 if (cvt_gate_to_trap(in, entry, &traps[out]))
869 out++;
870 }
871 traps[out].address = 0;
872}
873
874void xen_copy_trap_info(struct trap_info *traps)
875{
876 const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
877
878 xen_convert_trap_info(desc, traps);
879}
880
881
882
883
884static void xen_load_idt(const struct desc_ptr *desc)
885{
886 static DEFINE_SPINLOCK(lock);
887 static struct trap_info traps[257];
888
889 trace_xen_cpu_load_idt(desc);
890
891 spin_lock(&lock);
892
893 __get_cpu_var(idt_desc) = *desc;
894
895 xen_convert_trap_info(desc, traps);
896
897 xen_mc_flush();
898 if (HYPERVISOR_set_trap_table(traps))
899 BUG();
900
901 spin_unlock(&lock);
902}
903
904
905
906static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
907 const void *desc, int type)
908{
909 trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);
910
911 preempt_disable();
912
913 switch (type) {
914 case DESC_LDT:
915 case DESC_TSS:
916
917 break;
918
919 default: {
920 xmaddr_t maddr = arbitrary_virt_to_machine(&dt[entry]);
921
922 xen_mc_flush();
923 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
924 BUG();
925 }
926
927 }
928
929 preempt_enable();
930}
931
932
933
934
935
936static void __init xen_write_gdt_entry_boot(struct desc_struct *dt, int entry,
937 const void *desc, int type)
938{
939 trace_xen_cpu_write_gdt_entry(dt, entry, desc, type);
940
941 switch (type) {
942 case DESC_LDT:
943 case DESC_TSS:
944
945 break;
946
947 default: {
948 xmaddr_t maddr = virt_to_machine(&dt[entry]);
949
950 if (HYPERVISOR_update_descriptor(maddr.maddr, *(u64 *)desc))
951 dt[entry] = *(struct desc_struct *)desc;
952 }
953
954 }
955}
956
957static void xen_load_sp0(struct tss_struct *tss,
958 struct thread_struct *thread)
959{
960 struct multicall_space mcs;
961
962 mcs = xen_mc_entry(0);
963 MULTI_stack_switch(mcs.mc, __KERNEL_DS, thread->sp0);
964 xen_mc_issue(PARAVIRT_LAZY_CPU);
965}
966
967static void xen_set_iopl_mask(unsigned mask)
968{
969 struct physdev_set_iopl set_iopl;
970
971
972 set_iopl.iopl = (mask == 0) ? 1 : (mask >> 12) & 3;
973 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
974}
975
976static void xen_io_delay(void)
977{
978}
979
980#ifdef CONFIG_X86_LOCAL_APIC
981static unsigned long xen_set_apic_id(unsigned int x)
982{
983 WARN_ON(1);
984 return x;
985}
986static unsigned int xen_get_apic_id(unsigned long x)
987{
988 return ((x)>>24) & 0xFFu;
989}
990static u32 xen_apic_read(u32 reg)
991{
992 struct xen_platform_op op = {
993 .cmd = XENPF_get_cpuinfo,
994 .interface_version = XENPF_INTERFACE_VERSION,
995 .u.pcpu_info.xen_cpuid = 0,
996 };
997 int ret = 0;
998
999
1000
1001 if (!xen_initial_domain() || smp_processor_id())
1002 return 0;
1003
1004 if (reg == APIC_LVR)
1005 return 0x10;
1006
1007 if (reg != APIC_ID)
1008 return 0;
1009
1010 ret = HYPERVISOR_dom0_op(&op);
1011 if (ret)
1012 return 0;
1013
1014 return op.u.pcpu_info.apic_id << 24;
1015}
1016
1017static void xen_apic_write(u32 reg, u32 val)
1018{
1019
1020 WARN_ON(1);
1021}
1022
1023static u64 xen_apic_icr_read(void)
1024{
1025 return 0;
1026}
1027
1028static void xen_apic_icr_write(u32 low, u32 id)
1029{
1030
1031 WARN_ON(1);
1032}
1033
1034static void xen_apic_wait_icr_idle(void)
1035{
1036 return;
1037}
1038
1039static u32 xen_safe_apic_wait_icr_idle(void)
1040{
1041 return 0;
1042}
1043
1044static void set_xen_basic_apic_ops(void)
1045{
1046 apic->read = xen_apic_read;
1047 apic->write = xen_apic_write;
1048 apic->icr_read = xen_apic_icr_read;
1049 apic->icr_write = xen_apic_icr_write;
1050 apic->wait_icr_idle = xen_apic_wait_icr_idle;
1051 apic->safe_wait_icr_idle = xen_safe_apic_wait_icr_idle;
1052 apic->set_apic_id = xen_set_apic_id;
1053 apic->get_apic_id = xen_get_apic_id;
1054
1055#ifdef CONFIG_SMP
1056 apic->send_IPI_allbutself = xen_send_IPI_allbutself;
1057 apic->send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself;
1058 apic->send_IPI_mask = xen_send_IPI_mask;
1059 apic->send_IPI_all = xen_send_IPI_all;
1060 apic->send_IPI_self = xen_send_IPI_self;
1061#endif
1062}
1063
1064#endif
1065
1066static void xen_clts(void)
1067{
1068 struct multicall_space mcs;
1069
1070 mcs = xen_mc_entry(0);
1071
1072 MULTI_fpu_taskswitch(mcs.mc, 0);
1073
1074 xen_mc_issue(PARAVIRT_LAZY_CPU);
1075}
1076
1077static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
1078
1079static unsigned long xen_read_cr0(void)
1080{
1081 unsigned long cr0 = this_cpu_read(xen_cr0_value);
1082
1083 if (unlikely(cr0 == 0)) {
1084 cr0 = native_read_cr0();
1085 this_cpu_write(xen_cr0_value, cr0);
1086 }
1087
1088 return cr0;
1089}
1090
1091static void xen_write_cr0(unsigned long cr0)
1092{
1093 struct multicall_space mcs;
1094
1095 this_cpu_write(xen_cr0_value, cr0);
1096
1097
1098
1099 mcs = xen_mc_entry(0);
1100
1101 MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0);
1102
1103 xen_mc_issue(PARAVIRT_LAZY_CPU);
1104}
1105
1106static void xen_write_cr4(unsigned long cr4)
1107{
1108 cr4 &= ~X86_CR4_PGE;
1109 cr4 &= ~X86_CR4_PSE;
1110
1111 native_write_cr4(cr4);
1112}
1113#ifdef CONFIG_X86_64
1114static inline unsigned long xen_read_cr8(void)
1115{
1116 return 0;
1117}
1118static inline void xen_write_cr8(unsigned long val)
1119{
1120 BUG_ON(val);
1121}
1122#endif
1123static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
1124{
1125 int ret;
1126
1127 ret = 0;
1128
1129 switch (msr) {
1130#ifdef CONFIG_X86_64
1131 unsigned which;
1132 u64 base;
1133
1134 case MSR_FS_BASE: which = SEGBASE_FS; goto set;
1135 case MSR_KERNEL_GS_BASE: which = SEGBASE_GS_USER; goto set;
1136 case MSR_GS_BASE: which = SEGBASE_GS_KERNEL; goto set;
1137
1138 set:
1139 base = ((u64)high << 32) | low;
1140 if (HYPERVISOR_set_segment_base(which, base) != 0)
1141 ret = -EIO;
1142 break;
1143#endif
1144
1145 case MSR_STAR:
1146 case MSR_CSTAR:
1147 case MSR_LSTAR:
1148 case MSR_SYSCALL_MASK:
1149 case MSR_IA32_SYSENTER_CS:
1150 case MSR_IA32_SYSENTER_ESP:
1151 case MSR_IA32_SYSENTER_EIP:
1152
1153
1154
1155 break;
1156
1157 case MSR_IA32_CR_PAT:
1158 if (smp_processor_id() == 0)
1159 xen_set_pat(((u64)high << 32) | low);
1160 break;
1161
1162 default:
1163 ret = native_write_msr_safe(msr, low, high);
1164 }
1165
1166 return ret;
1167}
1168
1169void xen_setup_shared_info(void)
1170{
1171 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
1172 set_fixmap(FIX_PARAVIRT_BOOTMAP,
1173 xen_start_info->shared_info);
1174
1175 HYPERVISOR_shared_info =
1176 (struct shared_info *)fix_to_virt(FIX_PARAVIRT_BOOTMAP);
1177 } else
1178 HYPERVISOR_shared_info =
1179 (struct shared_info *)__va(xen_start_info->shared_info);
1180
1181#ifndef CONFIG_SMP
1182
1183 xen_setup_vcpu_info_placement();
1184#endif
1185
1186 xen_setup_mfn_list_list();
1187}
1188
1189
1190void xen_setup_vcpu_info_placement(void)
1191{
1192 int cpu;
1193
1194 for_each_possible_cpu(cpu) {
1195
1196 per_cpu(xen_vcpu_id, cpu) = cpu;
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206 (void) xen_vcpu_setup(cpu);
1207 }
1208
1209
1210
1211
1212
1213 if (xen_have_vcpu_info_placement) {
1214 pv_irq_ops.save_fl = __PV_IS_CALLEE_SAVE(xen_save_fl_direct);
1215 pv_irq_ops.restore_fl = __PV_IS_CALLEE_SAVE(xen_restore_fl_direct);
1216 pv_irq_ops.irq_disable = __PV_IS_CALLEE_SAVE(xen_irq_disable_direct);
1217 pv_irq_ops.irq_enable = __PV_IS_CALLEE_SAVE(xen_irq_enable_direct);
1218 pv_mmu_ops.read_cr2 = xen_read_cr2_direct;
1219 }
1220}
1221
1222static unsigned xen_patch(u8 type, u16 clobbers, void *insnbuf,
1223 unsigned long addr, unsigned len)
1224{
1225 char *start, *end, *reloc;
1226 unsigned ret;
1227
1228 start = end = reloc = NULL;
1229
1230#define SITE(op, x) \
1231 case PARAVIRT_PATCH(op.x): \
1232 if (xen_have_vcpu_info_placement) { \
1233 start = (char *)xen_##x##_direct; \
1234 end = xen_##x##_direct_end; \
1235 reloc = xen_##x##_direct_reloc; \
1236 } \
1237 goto patch_site
1238
1239 switch (type) {
1240 SITE(pv_irq_ops, irq_enable);
1241 SITE(pv_irq_ops, irq_disable);
1242 SITE(pv_irq_ops, save_fl);
1243 SITE(pv_irq_ops, restore_fl);
1244#undef SITE
1245
1246 patch_site:
1247 if (start == NULL || (end-start) > len)
1248 goto default_patch;
1249
1250 ret = paravirt_patch_insns(insnbuf, len, start, end);
1251
1252
1253
1254
1255
1256 if (reloc > start && reloc < end) {
1257 int reloc_off = reloc - start;
1258 long *relocp = (long *)(insnbuf + reloc_off);
1259 long delta = start - (char *)addr;
1260
1261 *relocp += delta;
1262 }
1263 break;
1264
1265 default_patch:
1266 default:
1267 ret = paravirt_patch_default(type, clobbers, insnbuf,
1268 addr, len);
1269 break;
1270 }
1271
1272 return ret;
1273}
1274
1275static const struct pv_info xen_info __initconst = {
1276 .paravirt_enabled = 1,
1277 .shared_kernel_pmd = 0,
1278
1279#ifdef CONFIG_X86_64
1280 .extra_user_64bit_cs = FLAT_USER_CS64,
1281#endif
1282
1283 .name = "Xen",
1284};
1285
1286static const struct pv_init_ops xen_init_ops __initconst = {
1287 .patch = xen_patch,
1288};
1289
1290static const struct pv_cpu_ops xen_cpu_ops __initconst = {
1291 .cpuid = xen_cpuid,
1292
1293 .set_debugreg = xen_set_debugreg,
1294 .get_debugreg = xen_get_debugreg,
1295
1296 .clts = xen_clts,
1297
1298 .read_cr0 = xen_read_cr0,
1299 .write_cr0 = xen_write_cr0,
1300
1301 .read_cr4 = native_read_cr4,
1302 .read_cr4_safe = native_read_cr4_safe,
1303 .write_cr4 = xen_write_cr4,
1304
1305#ifdef CONFIG_X86_64
1306 .read_cr8 = xen_read_cr8,
1307 .write_cr8 = xen_write_cr8,
1308#endif
1309
1310 .wbinvd = native_wbinvd,
1311
1312 .read_msr = native_read_msr_safe,
1313 .write_msr = xen_write_msr_safe,
1314
1315 .read_pmc = native_read_pmc,
1316
1317 .iret = xen_iret,
1318 .irq_enable_sysexit = xen_sysexit,
1319#ifdef CONFIG_X86_64
1320 .usergs_sysret32 = xen_sysret32,
1321 .usergs_sysret64 = xen_sysret64,
1322#endif
1323
1324 .load_tr_desc = paravirt_nop,
1325 .set_ldt = xen_set_ldt,
1326 .load_gdt = xen_load_gdt,
1327 .load_idt = xen_load_idt,
1328 .load_tls = xen_load_tls,
1329#ifdef CONFIG_X86_64
1330 .load_gs_index = xen_load_gs_index,
1331#endif
1332
1333 .alloc_ldt = xen_alloc_ldt,
1334 .free_ldt = xen_free_ldt,
1335
1336 .store_idt = native_store_idt,
1337 .store_tr = xen_store_tr,
1338
1339 .write_ldt_entry = xen_write_ldt_entry,
1340 .write_gdt_entry = xen_write_gdt_entry,
1341 .write_idt_entry = xen_write_idt_entry,
1342 .load_sp0 = xen_load_sp0,
1343
1344 .set_iopl_mask = xen_set_iopl_mask,
1345 .io_delay = xen_io_delay,
1346
1347
1348 .swapgs = paravirt_nop,
1349
1350 .start_context_switch = paravirt_start_context_switch,
1351 .end_context_switch = xen_end_context_switch,
1352};
1353
1354static const struct pv_apic_ops xen_apic_ops __initconst = {
1355#ifdef CONFIG_X86_LOCAL_APIC
1356 .startup_ipi_hook = paravirt_nop,
1357#endif
1358};
1359
1360static void xen_reboot(int reason)
1361{
1362 struct sched_shutdown r = { .reason = reason };
1363
1364 if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r))
1365 BUG();
1366}
1367
1368static void xen_restart(char *msg)
1369{
1370 xen_reboot(SHUTDOWN_reboot);
1371}
1372
1373static void xen_emergency_restart(void)
1374{
1375 xen_reboot(SHUTDOWN_reboot);
1376}
1377
1378static void xen_machine_halt(void)
1379{
1380 xen_reboot(SHUTDOWN_poweroff);
1381}
1382
1383static void xen_machine_power_off(void)
1384{
1385 if (pm_power_off)
1386 pm_power_off();
1387 xen_reboot(SHUTDOWN_poweroff);
1388}
1389
1390static void xen_crash_shutdown(struct pt_regs *regs)
1391{
1392 xen_reboot(SHUTDOWN_crash);
1393}
1394
1395static int
1396xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr)
1397{
1398 xen_reboot(SHUTDOWN_crash);
1399 return NOTIFY_DONE;
1400}
1401
1402static struct notifier_block xen_panic_block = {
1403 .notifier_call= xen_panic_event,
1404};
1405
1406int xen_panic_handler_init(void)
1407{
1408 atomic_notifier_chain_register(&panic_notifier_list, &xen_panic_block);
1409 return 0;
1410}
1411
1412static const struct machine_ops xen_machine_ops __initconst = {
1413 .restart = xen_restart,
1414 .halt = xen_machine_halt,
1415 .power_off = xen_machine_power_off,
1416 .shutdown = xen_machine_halt,
1417 .crash_shutdown = xen_crash_shutdown,
1418 .emergency_restart = xen_emergency_restart,
1419};
1420
1421static void __init xen_boot_params_init_edd(void)
1422{
1423#if IS_ENABLED(CONFIG_EDD)
1424 struct xen_platform_op op;
1425 struct edd_info *edd_info;
1426 u32 *mbr_signature;
1427 unsigned nr;
1428 int ret;
1429
1430 edd_info = boot_params.eddbuf;
1431 mbr_signature = boot_params.edd_mbr_sig_buffer;
1432
1433 op.cmd = XENPF_firmware_info;
1434
1435 op.u.firmware_info.type = XEN_FW_DISK_INFO;
1436 for (nr = 0; nr < EDDMAXNR; nr++) {
1437 struct edd_info *info = edd_info + nr;
1438
1439 op.u.firmware_info.index = nr;
1440 info->params.length = sizeof(info->params);
1441 set_xen_guest_handle(op.u.firmware_info.u.disk_info.edd_params,
1442 &info->params);
1443 ret = HYPERVISOR_dom0_op(&op);
1444 if (ret)
1445 break;
1446
1447#define C(x) info->x = op.u.firmware_info.u.disk_info.x
1448 C(device);
1449 C(version);
1450 C(interface_support);
1451 C(legacy_max_cylinder);
1452 C(legacy_max_head);
1453 C(legacy_sectors_per_track);
1454#undef C
1455 }
1456 boot_params.eddbuf_entries = nr;
1457
1458 op.u.firmware_info.type = XEN_FW_DISK_MBR_SIGNATURE;
1459 for (nr = 0; nr < EDD_MBR_SIG_MAX; nr++) {
1460 op.u.firmware_info.index = nr;
1461 ret = HYPERVISOR_dom0_op(&op);
1462 if (ret)
1463 break;
1464 mbr_signature[nr] = op.u.firmware_info.u.disk_mbr_signature.mbr_signature;
1465 }
1466 boot_params.edd_mbr_sig_buf_entries = nr;
1467#endif
1468}
1469
1470
1471
1472
1473
1474
1475static void __init xen_setup_stackprotector(void)
1476{
1477 pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry_boot;
1478 pv_cpu_ops.load_gdt = xen_load_gdt_boot;
1479
1480 setup_stack_canary_segment(0);
1481 switch_to_new_gdt(0);
1482
1483 pv_cpu_ops.write_gdt_entry = xen_write_gdt_entry;
1484 pv_cpu_ops.load_gdt = xen_load_gdt;
1485}
1486
1487
1488asmlinkage void __init xen_start_kernel(void)
1489{
1490 struct physdev_set_iopl set_iopl;
1491 int rc;
1492
1493 if (!xen_start_info)
1494 return;
1495
1496 xen_domain_type = XEN_PV_DOMAIN;
1497
1498 xen_setup_machphys_mapping();
1499
1500
1501 pv_info = xen_info;
1502 pv_init_ops = xen_init_ops;
1503 pv_cpu_ops = xen_cpu_ops;
1504 pv_apic_ops = xen_apic_ops;
1505
1506 x86_init.resources.memory_setup = xen_memory_setup;
1507 x86_init.oem.arch_setup = xen_arch_setup;
1508 x86_init.oem.banner = xen_banner;
1509
1510 xen_init_time_ops();
1511
1512
1513
1514
1515
1516 xen_init_mmu_ops();
1517
1518
1519 __supported_pte_mask &= ~_PAGE_GLOBAL;
1520#if 0
1521 if (!xen_initial_domain())
1522#endif
1523 __supported_pte_mask &= ~(_PAGE_PWT | _PAGE_PCD);
1524
1525 __supported_pte_mask |= _PAGE_IOMAP;
1526
1527
1528
1529
1530
1531 __userpte_alloc_gfp &= ~__GFP_HIGHMEM;
1532
1533
1534 x86_configure_nx();
1535
1536 xen_setup_features();
1537
1538
1539 if (!xen_feature(XENFEAT_auto_translated_physmap))
1540 xen_build_dynamic_phys_to_machine();
1541
1542
1543
1544
1545
1546 xen_setup_stackprotector();
1547
1548 xen_init_irq_ops();
1549 xen_init_cpuid_mask();
1550
1551 setup_clear_cpu_cap(X86_FEATURE_SME);
1552
1553#ifdef CONFIG_X86_LOCAL_APIC
1554
1555
1556
1557 set_xen_basic_apic_ops();
1558#endif
1559
1560 if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
1561 pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
1562 pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
1563 }
1564
1565 machine_ops = xen_machine_ops;
1566
1567
1568
1569
1570
1571
1572 xen_initial_gdt = &per_cpu(gdt_page, 0);
1573
1574 xen_smp_init();
1575
1576#ifdef CONFIG_ACPI_NUMA
1577
1578
1579
1580
1581
1582 acpi_numa = -1;
1583#endif
1584#ifdef CONFIG_X86_PAT
1585
1586
1587
1588
1589
1590 pat_enabled = 0;
1591#endif
1592
1593 per_cpu(xen_vcpu_id, 0) = 0;
1594
1595
1596
1597
1598
1599
1600
1601
1602 xen_vcpu_info_reset(0);
1603
1604 local_irq_disable();
1605 early_boot_irqs_disabled = true;
1606
1607 xen_raw_console_write("mapping kernel into physical memory\n");
1608 xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, xen_start_info->nr_pages);
1609
1610
1611 xen_build_mfn_list_list();
1612
1613
1614
1615#ifdef CONFIG_X86_32
1616 pv_info.kernel_rpl = 1;
1617 if (xen_feature(XENFEAT_supervisor_mode_kernel))
1618 pv_info.kernel_rpl = 0;
1619#else
1620 pv_info.kernel_rpl = 0;
1621#endif
1622
1623 xen_reserve_top();
1624
1625
1626
1627
1628
1629 set_iopl.iopl = 1;
1630 rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
1631 if (rc != 0)
1632 xen_raw_printk("physdev_op failed %d\n", rc);
1633
1634#ifdef CONFIG_X86_32
1635
1636 cpu_detect(&new_cpu_data);
1637 new_cpu_data.hard_math = 1;
1638 new_cpu_data.wp_works_ok = 1;
1639 new_cpu_data.x86_capability[0] = cpuid_edx(1);
1640#endif
1641
1642
1643 boot_params.hdr.type_of_loader = (9 << 4) | 0;
1644 boot_params.hdr.ramdisk_image = xen_start_info->mod_start
1645 ? __pa(xen_start_info->mod_start) : 0;
1646 boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
1647 boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
1648
1649 if (!xen_initial_domain()) {
1650 add_preferred_console("xenboot", 0, NULL);
1651 add_preferred_console("tty", 0, NULL);
1652 add_preferred_console("hvc", 0, NULL);
1653 if (pci_xen)
1654 x86_init.pci.arch_init = pci_xen_init;
1655 } else {
1656 const struct dom0_vga_console_info *info =
1657 (void *)((char *)xen_start_info +
1658 xen_start_info->console.dom0.info_off);
1659 struct xen_platform_op op = {
1660 .cmd = XENPF_firmware_info,
1661 .interface_version = XENPF_INTERFACE_VERSION,
1662 .u.firmware_info.type = XEN_FW_KBD_SHIFT_FLAGS,
1663 };
1664
1665 xen_init_vga(info, xen_start_info->console.dom0.info_size);
1666 xen_start_info->console.domU.mfn = 0;
1667 xen_start_info->console.domU.evtchn = 0;
1668
1669 if (HYPERVISOR_dom0_op(&op) == 0)
1670 boot_params.kbd_status = op.u.firmware_info.u.kbd_shift_flags;
1671
1672 xen_init_apic();
1673
1674
1675 pci_request_acs();
1676
1677 xen_acpi_sleep_register();
1678
1679
1680 x86_init.mpparse.find_smp_config = x86_init_noop;
1681 x86_init.mpparse.get_smp_config = x86_init_uint_noop;
1682
1683 xen_boot_params_init_edd();
1684 }
1685#ifdef CONFIG_PCI
1686
1687 pci_probe &= ~PCI_PROBE_BIOS;
1688#endif
1689 xen_raw_console_write("about to get started...\n");
1690
1691
1692 xen_setup_runstate_info(0);
1693
1694
1695#ifdef CONFIG_X86_32
1696 i386_start_kernel();
1697#else
1698 x86_64_start_reservations((char *)__pa_symbol(&boot_params));
1699#endif
1700}
1701
1702void __ref xen_hvm_init_shared_info(void)
1703{
1704 struct xen_add_to_physmap xatp;
1705 static struct shared_info *shared_info_page = 0;
1706
1707 if (!shared_info_page)
1708 shared_info_page = (struct shared_info *)
1709 extend_brk(PAGE_SIZE, PAGE_SIZE);
1710 xatp.domid = DOMID_SELF;
1711 xatp.idx = 0;
1712 xatp.space = XENMAPSPACE_shared_info;
1713 xatp.gpfn = __pa(shared_info_page) >> PAGE_SHIFT;
1714 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
1715 BUG();
1716
1717 HYPERVISOR_shared_info = (struct shared_info *)shared_info_page;
1718}
1719
1720#ifdef CONFIG_XEN_PVHVM
1721static void __init init_hvm_pv_info(void)
1722{
1723 int major, minor;
1724 uint32_t eax, ebx, ecx, edx, pages, msr, base;
1725 u64 pfn;
1726
1727 base = xen_cpuid_base();
1728 cpuid(base + 1, &eax, &ebx, &ecx, &edx);
1729
1730 major = eax >> 16;
1731 minor = eax & 0xffff;
1732 printk(KERN_INFO "Xen version %d.%d.\n", major, minor);
1733
1734 cpuid(base + 2, &pages, &msr, &ecx, &edx);
1735
1736 pfn = __pa(hypercall_page);
1737 wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));
1738
1739 xen_setup_features();
1740
1741 cpuid(base + 4, &eax, &ebx, &ecx, &edx);
1742 if (eax & XEN_HVM_CPUID_VCPU_ID_PRESENT)
1743 this_cpu_write(xen_vcpu_id, ebx);
1744 else
1745 this_cpu_write(xen_vcpu_id, smp_processor_id());
1746
1747 pv_info.name = "Xen HVM";
1748
1749 xen_domain_type = XEN_HVM_DOMAIN;
1750}
1751
1752static int xen_hvm_cpu_notify(struct notifier_block *self, unsigned long action,
1753 void *hcpu)
1754{
1755 int cpu = (long)hcpu;
1756 switch (action) {
1757 case CPU_UP_PREPARE:
1758 if (cpu_acpi_id(cpu) != U32_MAX)
1759 per_cpu(xen_vcpu_id, cpu) = cpu_acpi_id(cpu);
1760 else
1761 per_cpu(xen_vcpu_id, cpu) = cpu;
1762
1763 if (xen_vcpu_setup(cpu))
1764 return NOTIFY_BAD;
1765
1766 if (xen_have_vector_callback) {
1767 if (xen_feature(XENFEAT_hvm_safe_pvclock))
1768 xen_setup_timer(cpu);
1769 }
1770 break;
1771 default:
1772 break;
1773 }
1774 return NOTIFY_OK;
1775}
1776
1777static struct notifier_block xen_hvm_cpu_notifier = {
1778 .notifier_call = xen_hvm_cpu_notify,
1779};
1780
1781#ifdef CONFIG_KEXEC_CORE
1782static void xen_hvm_shutdown(void)
1783{
1784 native_machine_shutdown();
1785 if (kexec_in_progress)
1786 xen_reboot(SHUTDOWN_soft_reset);
1787}
1788
1789static void xen_hvm_crash_shutdown(struct pt_regs *regs)
1790{
1791 native_machine_crash_shutdown(regs);
1792 xen_reboot(SHUTDOWN_soft_reset);
1793}
1794#endif
1795
1796static void __init xen_hvm_guest_init(void)
1797{
1798 init_hvm_pv_info();
1799
1800 xen_hvm_init_shared_info();
1801
1802
1803
1804
1805
1806
1807 xen_vcpu_info_reset(0);
1808
1809 xen_panic_handler_init();
1810
1811 if (xen_feature(XENFEAT_hvm_callback_vector))
1812 xen_have_vector_callback = 1;
1813 xen_hvm_smp_init();
1814 register_cpu_notifier(&xen_hvm_cpu_notifier);
1815 xen_unplug_emulated_devices();
1816 x86_init.irqs.intr_init = xen_init_IRQ;
1817 xen_hvm_init_time_ops();
1818 xen_hvm_init_mmu_ops();
1819#ifdef CONFIG_KEXEC_CORE
1820 if (xen_running_on_version_or_later(3, 2)) {
1821 machine_ops.shutdown = xen_hvm_shutdown;
1822 machine_ops.crash_shutdown = xen_hvm_crash_shutdown;
1823 }
1824#endif
1825}
1826
1827static uint32_t __init xen_hvm_platform(void)
1828{
1829 uint32_t eax, ebx, ecx, edx, base;
1830 int major, minor;
1831
1832 if (xen_pv_domain())
1833 return 0;
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843 base = xen_cpuid_base();
1844
1845 if (base && is_kdump_kernel()) {
1846 cpuid(base + 1, &eax, &ebx, &ecx, &edx);
1847 major = eax >> 16;
1848 minor = eax & 0xffff;
1849 if (major == 3 && minor < 2)
1850 return 0;
1851 }
1852
1853 return base;
1854}
1855
1856bool xen_hvm_need_lapic(void)
1857{
1858 if (xen_pv_domain())
1859 return false;
1860 if (!xen_hvm_domain())
1861 return false;
1862 if (xen_feature(XENFEAT_hvm_pirqs) && xen_have_vector_callback)
1863 return false;
1864 return true;
1865}
1866EXPORT_SYMBOL_GPL(xen_hvm_need_lapic);
1867
1868const struct hypervisor_x86 x86_hyper_xen_hvm __refconst = {
1869 .name = "Xen HVM",
1870 .detect = xen_hvm_platform,
1871 .init_platform = xen_hvm_guest_init,
1872 .x2apic_available = xen_x2apic_para_available,
1873};
1874EXPORT_SYMBOL(x86_hyper_xen_hvm);
1875#endif
1876