1#ifndef _ASM_X86_PARAVIRT_TYPES_H
2#define _ASM_X86_PARAVIRT_TYPES_H
3
4
5#define CLBR_NONE 0
6#define CLBR_EAX (1 << 0)
7#define CLBR_ECX (1 << 1)
8#define CLBR_EDX (1 << 2)
9#define CLBR_EDI (1 << 3)
10
11#ifdef CONFIG_X86_32
12
13#define CLBR_ANY ((1 << 4) - 1)
14
15#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX)
16#define CLBR_RET_REG (CLBR_EAX | CLBR_EDX)
17#define CLBR_SCRATCH (0)
18#else
19#define CLBR_RAX CLBR_EAX
20#define CLBR_RCX CLBR_ECX
21#define CLBR_RDX CLBR_EDX
22#define CLBR_RDI CLBR_EDI
23#define CLBR_RSI (1 << 4)
24#define CLBR_R8 (1 << 5)
25#define CLBR_R9 (1 << 6)
26#define CLBR_R10 (1 << 7)
27#define CLBR_R11 (1 << 8)
28
29#define CLBR_ANY ((1 << 9) - 1)
30
31#define CLBR_ARG_REGS (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
32 CLBR_RCX | CLBR_R8 | CLBR_R9)
33#define CLBR_RET_REG (CLBR_RAX)
34#define CLBR_SCRATCH (CLBR_R10 | CLBR_R11)
35
36#endif
37
38#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
39
40#ifndef __ASSEMBLY__
41
42#include <asm/desc_defs.h>
43#include <asm/kmap_types.h>
44
45struct page;
46struct thread_struct;
47struct desc_ptr;
48struct tss_struct;
49struct mm_struct;
50struct desc_struct;
51struct task_struct;
52struct cpumask;
53
54
55
56
57
58struct paravirt_callee_save {
59 void *func;
60};
61
62
63struct pv_info {
64 unsigned int kernel_rpl;
65 int shared_kernel_pmd;
66 int paravirt_enabled;
67 const char *name;
68};
69
70struct pv_init_ops {
71
72
73
74
75
76
77
78
79 unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
80 unsigned long addr, unsigned len);
81};
82
83
84struct pv_lazy_ops {
85
86 void (*enter)(void);
87 void (*leave)(void);
88};
89
90struct pv_time_ops {
91 unsigned long long (*sched_clock)(void);
92 unsigned long (*get_tsc_khz)(void);
93};
94
95struct pv_cpu_ops {
96
97 unsigned long (*get_debugreg)(int regno);
98 void (*set_debugreg)(int regno, unsigned long value);
99
100 void (*clts)(void);
101
102 unsigned long (*read_cr0)(void);
103 void (*write_cr0)(unsigned long);
104
105 unsigned long (*read_cr4_safe)(void);
106 unsigned long (*read_cr4)(void);
107 void (*write_cr4)(unsigned long);
108
109#ifdef CONFIG_X86_64
110 unsigned long (*read_cr8)(void);
111 void (*write_cr8)(unsigned long);
112#endif
113
114
115 void (*load_tr_desc)(void);
116 void (*load_gdt)(const struct desc_ptr *);
117 void (*load_idt)(const struct desc_ptr *);
118 void (*store_gdt)(struct desc_ptr *);
119 void (*store_idt)(struct desc_ptr *);
120 void (*set_ldt)(const void *desc, unsigned entries);
121 unsigned long (*store_tr)(void);
122 void (*load_tls)(struct thread_struct *t, unsigned int cpu);
123#ifdef CONFIG_X86_64
124 void (*load_gs_index)(unsigned int idx);
125#endif
126 void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
127 const void *desc);
128 void (*write_gdt_entry)(struct desc_struct *,
129 int entrynum, const void *desc, int size);
130 void (*write_idt_entry)(gate_desc *,
131 int entrynum, const gate_desc *gate);
132 void (*alloc_ldt)(struct desc_struct *ldt, unsigned entries);
133 void (*free_ldt)(struct desc_struct *ldt, unsigned entries);
134
135 void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t);
136
137 void (*set_iopl_mask)(unsigned mask);
138
139 void (*wbinvd)(void);
140 void (*io_delay)(void);
141
142
143 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
144 unsigned int *ecx, unsigned int *edx);
145
146
147
148 u64 (*read_msr)(unsigned int msr, int *err);
149 int (*rdmsr_regs)(u32 *regs);
150 int (*write_msr)(unsigned int msr, unsigned low, unsigned high);
151 int (*wrmsr_regs)(u32 *regs);
152
153 u64 (*read_tsc)(void);
154 u64 (*read_pmc)(int counter);
155 unsigned long long (*read_tscp)(unsigned int *aux);
156
157
158
159
160
161
162
163 void (*irq_enable_sysexit)(void);
164
165
166
167
168
169
170
171 void (*usergs_sysret64)(void);
172
173
174
175
176
177
178
179 void (*usergs_sysret32)(void);
180
181
182
183 void (*iret)(void);
184
185 void (*swapgs)(void);
186
187 void (*start_context_switch)(struct task_struct *prev);
188 void (*end_context_switch)(struct task_struct *next);
189};
190
191struct pv_irq_ops {
192
193
194
195
196
197
198
199
200
201 struct paravirt_callee_save save_fl;
202 struct paravirt_callee_save restore_fl;
203 struct paravirt_callee_save irq_disable;
204 struct paravirt_callee_save irq_enable;
205
206 void (*safe_halt)(void);
207 void (*halt)(void);
208
209#ifdef CONFIG_X86_64
210 void (*adjust_exception_frame)(void);
211#endif
212};
213
214struct pv_apic_ops {
215#ifdef CONFIG_X86_LOCAL_APIC
216 void (*startup_ipi_hook)(int phys_apicid,
217 unsigned long start_eip,
218 unsigned long start_esp);
219#endif
220};
221
222struct pv_mmu_ops {
223 unsigned long (*read_cr2)(void);
224 void (*write_cr2)(unsigned long);
225
226 unsigned long (*read_cr3)(void);
227 void (*write_cr3)(unsigned long);
228
229
230
231
232
233 void (*activate_mm)(struct mm_struct *prev,
234 struct mm_struct *next);
235 void (*dup_mmap)(struct mm_struct *oldmm,
236 struct mm_struct *mm);
237 void (*exit_mmap)(struct mm_struct *mm);
238
239
240
241 void (*flush_tlb_user)(void);
242 void (*flush_tlb_kernel)(void);
243 void (*flush_tlb_single)(unsigned long addr);
244 void (*flush_tlb_others)(const struct cpumask *cpus,
245 struct mm_struct *mm,
246 unsigned long va);
247
248
249 int (*pgd_alloc)(struct mm_struct *mm);
250 void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
251
252
253
254
255
256 void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn);
257 void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn);
258 void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn);
259 void (*release_pte)(unsigned long pfn);
260 void (*release_pmd)(unsigned long pfn);
261 void (*release_pud)(unsigned long pfn);
262
263
264 void (*set_pte)(pte_t *ptep, pte_t pteval);
265 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
266 pte_t *ptep, pte_t pteval);
267 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
268 void (*set_pmd_at)(struct mm_struct *mm, unsigned long addr,
269 pmd_t *pmdp, pmd_t pmdval);
270 void (*pte_update)(struct mm_struct *mm, unsigned long addr,
271 pte_t *ptep);
272 void (*pte_update_defer)(struct mm_struct *mm,
273 unsigned long addr, pte_t *ptep);
274 void (*pmd_update)(struct mm_struct *mm, unsigned long addr,
275 pmd_t *pmdp);
276 void (*pmd_update_defer)(struct mm_struct *mm,
277 unsigned long addr, pmd_t *pmdp);
278
279 pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
280 pte_t *ptep);
281 void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
282 pte_t *ptep, pte_t pte);
283
284 struct paravirt_callee_save pte_val;
285 struct paravirt_callee_save make_pte;
286
287 struct paravirt_callee_save pgd_val;
288 struct paravirt_callee_save make_pgd;
289
290#if PAGETABLE_LEVELS >= 3
291#ifdef CONFIG_X86_PAE
292 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
293 void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
294 pte_t *ptep);
295 void (*pmd_clear)(pmd_t *pmdp);
296
297#endif
298
299 void (*set_pud)(pud_t *pudp, pud_t pudval);
300
301 struct paravirt_callee_save pmd_val;
302 struct paravirt_callee_save make_pmd;
303
304#if PAGETABLE_LEVELS == 4
305 struct paravirt_callee_save pud_val;
306 struct paravirt_callee_save make_pud;
307
308 void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
309#endif
310#endif
311
312 struct pv_lazy_ops lazy_mode;
313
314
315
316
317
318 void (*set_fixmap)(unsigned idx,
319 phys_addr_t phys, pgprot_t flags);
320};
321
322struct arch_spinlock;
323struct pv_lock_ops {
324 int (*spin_is_locked)(struct arch_spinlock *lock);
325 int (*spin_is_contended)(struct arch_spinlock *lock);
326 void (*spin_lock)(struct arch_spinlock *lock);
327 void (*spin_lock_flags)(struct arch_spinlock *lock, unsigned long flags);
328 int (*spin_trylock)(struct arch_spinlock *lock);
329 void (*spin_unlock)(struct arch_spinlock *lock);
330};
331
332
333
334
335struct paravirt_patch_template {
336 struct pv_init_ops pv_init_ops;
337 struct pv_time_ops pv_time_ops;
338 struct pv_cpu_ops pv_cpu_ops;
339 struct pv_irq_ops pv_irq_ops;
340 struct pv_apic_ops pv_apic_ops;
341 struct pv_mmu_ops pv_mmu_ops;
342 struct pv_lock_ops pv_lock_ops;
343};
344
345extern struct pv_info pv_info;
346extern struct pv_init_ops pv_init_ops;
347extern struct pv_time_ops pv_time_ops;
348extern struct pv_cpu_ops pv_cpu_ops;
349extern struct pv_irq_ops pv_irq_ops;
350extern struct pv_apic_ops pv_apic_ops;
351extern struct pv_mmu_ops pv_mmu_ops;
352extern struct pv_lock_ops pv_lock_ops;
353
354#define PARAVIRT_PATCH(x) \
355 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
356
357#define paravirt_type(op) \
358 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
359 [paravirt_opptr] "i" (&(op))
360#define paravirt_clobber(clobber) \
361 [paravirt_clobber] "i" (clobber)
362
363
364
365
366
367#define _paravirt_alt(insn_string, type, clobber) \
368 "771:\n\t" insn_string "\n" "772:\n" \
369 ".pushsection .parainstructions,\"a\"\n" \
370 _ASM_ALIGN "\n" \
371 _ASM_PTR " 771b\n" \
372 " .byte " type "\n" \
373 " .byte 772b-771b\n" \
374 " .short " clobber "\n" \
375 ".popsection\n"
376
377
378#define paravirt_alt(insn_string) \
379 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
380
381
382#define DEF_NATIVE(ops, name, code) \
383 extern const char start_##ops##_##name[], end_##ops##_##name[]; \
384 asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
385
386unsigned paravirt_patch_nop(void);
387unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
388unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
389unsigned paravirt_patch_ignore(unsigned len);
390unsigned paravirt_patch_call(void *insnbuf,
391 const void *target, u16 tgt_clobbers,
392 unsigned long addr, u16 site_clobbers,
393 unsigned len);
394unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
395 unsigned long addr, unsigned len);
396unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
397 unsigned long addr, unsigned len);
398
399unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
400 const char *start, const char *end);
401
402unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
403 unsigned long addr, unsigned len);
404
405int paravirt_disable_iospace(void);
406
407
408
409
410
411
412
413#define PARAVIRT_CALL "call *%c[paravirt_opptr];"
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479#ifdef CONFIG_X86_32
480#define PVOP_VCALL_ARGS \
481 unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx
482#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
483
484#define PVOP_CALL_ARG1(x) "a" ((unsigned long)(x))
485#define PVOP_CALL_ARG2(x) "d" ((unsigned long)(x))
486#define PVOP_CALL_ARG3(x) "c" ((unsigned long)(x))
487
488#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
489 "=c" (__ecx)
490#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
491
492#define PVOP_VCALLEE_CLOBBERS "=a" (__eax), "=d" (__edx)
493#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
494
495#define EXTRA_CLOBBERS
496#define VEXTRA_CLOBBERS
497#else
498
499#define PVOP_VCALL_ARGS \
500 unsigned long __edi = __edi, __esi = __esi, \
501 __edx = __edx, __ecx = __ecx, __eax = __eax
502#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
503
504#define PVOP_CALL_ARG1(x) "D" ((unsigned long)(x))
505#define PVOP_CALL_ARG2(x) "S" ((unsigned long)(x))
506#define PVOP_CALL_ARG3(x) "d" ((unsigned long)(x))
507#define PVOP_CALL_ARG4(x) "c" ((unsigned long)(x))
508
509#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
510 "=S" (__esi), "=d" (__edx), \
511 "=c" (__ecx)
512#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
513
514
515#define PVOP_VCALLEE_CLOBBERS "=a" (__eax)
516#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
517
518#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
519#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
520#endif
521
522#ifdef CONFIG_PARAVIRT_DEBUG
523#define PVOP_TEST_NULL(op) BUG_ON(op == NULL)
524#else
525#define PVOP_TEST_NULL(op) ((void)op)
526#endif
527
528#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr, \
529 pre, post, ...) \
530 ({ \
531 rettype __ret; \
532 PVOP_CALL_ARGS; \
533 PVOP_TEST_NULL(op); \
534 \
535 \
536 if (sizeof(rettype) > sizeof(unsigned long)) { \
537 asm volatile(pre \
538 paravirt_alt(PARAVIRT_CALL) \
539 post \
540 : call_clbr \
541 : paravirt_type(op), \
542 paravirt_clobber(clbr), \
543 ##__VA_ARGS__ \
544 : "memory", "cc" extra_clbr); \
545 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
546 } else { \
547 asm volatile(pre \
548 paravirt_alt(PARAVIRT_CALL) \
549 post \
550 : call_clbr \
551 : paravirt_type(op), \
552 paravirt_clobber(clbr), \
553 ##__VA_ARGS__ \
554 : "memory", "cc" extra_clbr); \
555 __ret = (rettype)__eax; \
556 } \
557 __ret; \
558 })
559
560#define __PVOP_CALL(rettype, op, pre, post, ...) \
561 ____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS, \
562 EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)
563
564#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...) \
565 ____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
566 PVOP_CALLEE_CLOBBERS, , \
567 pre, post, ##__VA_ARGS__)
568
569
570#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
571 ({ \
572 PVOP_VCALL_ARGS; \
573 PVOP_TEST_NULL(op); \
574 asm volatile(pre \
575 paravirt_alt(PARAVIRT_CALL) \
576 post \
577 : call_clbr \
578 : paravirt_type(op), \
579 paravirt_clobber(clbr), \
580 ##__VA_ARGS__ \
581 : "memory", "cc" extra_clbr); \
582 })
583
584#define __PVOP_VCALL(op, pre, post, ...) \
585 ____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS, \
586 VEXTRA_CLOBBERS, \
587 pre, post, ##__VA_ARGS__)
588
589#define __PVOP_VCALLEESAVE(op, pre, post, ...) \
590 ____PVOP_VCALL(op.func, CLBR_RET_REG, \
591 PVOP_VCALLEE_CLOBBERS, , \
592 pre, post, ##__VA_ARGS__)
593
594
595
596#define PVOP_CALL0(rettype, op) \
597 __PVOP_CALL(rettype, op, "", "")
598#define PVOP_VCALL0(op) \
599 __PVOP_VCALL(op, "", "")
600
601#define PVOP_CALLEE0(rettype, op) \
602 __PVOP_CALLEESAVE(rettype, op, "", "")
603#define PVOP_VCALLEE0(op) \
604 __PVOP_VCALLEESAVE(op, "", "")
605
606
607#define PVOP_CALL1(rettype, op, arg1) \
608 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
609#define PVOP_VCALL1(op, arg1) \
610 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))
611
612#define PVOP_CALLEE1(rettype, op, arg1) \
613 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
614#define PVOP_VCALLEE1(op, arg1) \
615 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))
616
617
618#define PVOP_CALL2(rettype, op, arg1, arg2) \
619 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
620 PVOP_CALL_ARG2(arg2))
621#define PVOP_VCALL2(op, arg1, arg2) \
622 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
623 PVOP_CALL_ARG2(arg2))
624
625#define PVOP_CALLEE2(rettype, op, arg1, arg2) \
626 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
627 PVOP_CALL_ARG2(arg2))
628#define PVOP_VCALLEE2(op, arg1, arg2) \
629 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1), \
630 PVOP_CALL_ARG2(arg2))
631
632
633#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
634 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
635 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
636#define PVOP_VCALL3(op, arg1, arg2, arg3) \
637 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
638 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
639
640
641#ifdef CONFIG_X86_32
642#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
643 __PVOP_CALL(rettype, op, \
644 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
645 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
646 PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
647#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
648 __PVOP_VCALL(op, \
649 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
650 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
651 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
652#else
653#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
654 __PVOP_CALL(rettype, op, "", "", \
655 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
656 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
657#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
658 __PVOP_VCALL(op, "", "", \
659 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
660 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
661#endif
662
663
664enum paravirt_lazy_mode {
665 PARAVIRT_LAZY_NONE,
666 PARAVIRT_LAZY_MMU,
667 PARAVIRT_LAZY_CPU,
668};
669
670enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
671void paravirt_start_context_switch(struct task_struct *prev);
672void paravirt_end_context_switch(struct task_struct *next);
673
674void paravirt_enter_lazy_mmu(void);
675void paravirt_leave_lazy_mmu(void);
676
677void _paravirt_nop(void);
678u32 _paravirt_ident_32(u32);
679u64 _paravirt_ident_64(u64);
680
681#define paravirt_nop ((void *)_paravirt_nop)
682
683
684struct paravirt_patch_site {
685 u8 *instr;
686 u8 instrtype;
687 u8 len;
688 u16 clobbers;
689};
690
691extern struct paravirt_patch_site __parainstructions[],
692 __parainstructions_end[];
693
694#endif
695
696#endif
697