1#ifndef _ASM_X86_PARAVIRT_TYPES_H
2#define _ASM_X86_PARAVIRT_TYPES_H
3
4
5#define CLBR_NONE 0
6#define CLBR_EAX (1 << 0)
7#define CLBR_ECX (1 << 1)
8#define CLBR_EDX (1 << 2)
9#define CLBR_EDI (1 << 3)
10
11#ifdef CONFIG_X86_32
12
13#define CLBR_ANY ((1 << 4) - 1)
14
15#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX)
16#define CLBR_RET_REG (CLBR_EAX | CLBR_EDX)
17#define CLBR_SCRATCH (0)
18#else
19#define CLBR_RAX CLBR_EAX
20#define CLBR_RCX CLBR_ECX
21#define CLBR_RDX CLBR_EDX
22#define CLBR_RDI CLBR_EDI
23#define CLBR_RSI (1 << 4)
24#define CLBR_R8 (1 << 5)
25#define CLBR_R9 (1 << 6)
26#define CLBR_R10 (1 << 7)
27#define CLBR_R11 (1 << 8)
28
29#define CLBR_ANY ((1 << 9) - 1)
30
31#define CLBR_ARG_REGS (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
32 CLBR_RCX | CLBR_R8 | CLBR_R9)
33#define CLBR_RET_REG (CLBR_RAX)
34#define CLBR_SCRATCH (CLBR_R10 | CLBR_R11)
35
36#endif
37
38#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
39
40#ifndef __ASSEMBLY__
41
42#include <asm/desc_defs.h>
43#include <asm/kmap_types.h>
44
45struct page;
46struct thread_struct;
47struct desc_ptr;
48struct tss_struct;
49struct mm_struct;
50struct desc_struct;
51struct task_struct;
52struct cpumask;
53
54
55
56
57
58struct paravirt_callee_save {
59 void *func;
60};
61
62
63struct pv_info {
64 unsigned int kernel_rpl;
65 int shared_kernel_pmd;
66 int paravirt_enabled;
67 const char *name;
68};
69
70struct pv_init_ops {
71
72
73
74
75
76
77
78
79 unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
80 unsigned long addr, unsigned len);
81};
82
83
84struct pv_lazy_ops {
85
86 void (*enter)(void);
87 void (*leave)(void);
88};
89
90struct pv_time_ops {
91 unsigned long long (*sched_clock)(void);
92 unsigned long (*get_tsc_khz)(void);
93};
94
95struct pv_cpu_ops {
96
97 unsigned long (*get_debugreg)(int regno);
98 void (*set_debugreg)(int regno, unsigned long value);
99
100 void (*clts)(void);
101
102 unsigned long (*read_cr0)(void);
103 void (*write_cr0)(unsigned long);
104
105 unsigned long (*read_cr4_safe)(void);
106 unsigned long (*read_cr4)(void);
107 void (*write_cr4)(unsigned long);
108
109#ifdef CONFIG_X86_64
110 unsigned long (*read_cr8)(void);
111 void (*write_cr8)(unsigned long);
112#endif
113
114
115 void (*load_tr_desc)(void);
116 void (*load_gdt)(const struct desc_ptr *);
117 void (*load_idt)(const struct desc_ptr *);
118 void (*store_gdt)(struct desc_ptr *);
119 void (*store_idt)(struct desc_ptr *);
120 void (*set_ldt)(const void *desc, unsigned entries);
121 unsigned long (*store_tr)(void);
122 void (*load_tls)(struct thread_struct *t, unsigned int cpu);
123#ifdef CONFIG_X86_64
124 void (*load_gs_index)(unsigned int idx);
125#endif
126 void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
127 const void *desc);
128 void (*write_gdt_entry)(struct desc_struct *,
129 int entrynum, const void *desc, int size);
130 void (*write_idt_entry)(gate_desc *,
131 int entrynum, const gate_desc *gate);
132 void (*alloc_ldt)(struct desc_struct *ldt, unsigned entries);
133 void (*free_ldt)(struct desc_struct *ldt, unsigned entries);
134
135 void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t);
136
137 void (*set_iopl_mask)(unsigned mask);
138
139 void (*wbinvd)(void);
140 void (*io_delay)(void);
141
142
143 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
144 unsigned int *ecx, unsigned int *edx);
145
146
147
148 u64 (*read_msr)(unsigned int msr, int *err);
149 int (*rdmsr_regs)(u32 *regs);
150 int (*write_msr)(unsigned int msr, unsigned low, unsigned high);
151 int (*wrmsr_regs)(u32 *regs);
152
153 u64 (*read_tsc)(void);
154 u64 (*read_pmc)(int counter);
155 unsigned long long (*read_tscp)(unsigned int *aux);
156
157
158
159
160
161
162
163 void (*irq_enable_sysexit)(void);
164
165
166
167
168
169
170
171 void (*usergs_sysret64)(void);
172
173
174
175
176
177
178
179 void (*usergs_sysret32)(void);
180
181
182
183 void (*iret)(void);
184
185 void (*swapgs)(void);
186
187 void (*start_context_switch)(struct task_struct *prev);
188 void (*end_context_switch)(struct task_struct *next);
189};
190
191struct pv_irq_ops {
192
193
194
195
196
197
198
199
200
201 struct paravirt_callee_save save_fl;
202 struct paravirt_callee_save restore_fl;
203 struct paravirt_callee_save irq_disable;
204 struct paravirt_callee_save irq_enable;
205
206 void (*safe_halt)(void);
207 void (*halt)(void);
208
209#ifdef CONFIG_X86_64
210 void (*adjust_exception_frame)(void);
211#endif
212};
213
214struct pv_apic_ops {
215#ifdef CONFIG_X86_LOCAL_APIC
216 void (*startup_ipi_hook)(int phys_apicid,
217 unsigned long start_eip,
218 unsigned long start_esp);
219#endif
220};
221
222struct pv_mmu_ops {
223 unsigned long (*read_cr2)(void);
224 void (*write_cr2)(unsigned long);
225
226 unsigned long (*read_cr3)(void);
227 void (*write_cr3)(unsigned long);
228
229
230
231
232
233 void (*activate_mm)(struct mm_struct *prev,
234 struct mm_struct *next);
235 void (*dup_mmap)(struct mm_struct *oldmm,
236 struct mm_struct *mm);
237 void (*exit_mmap)(struct mm_struct *mm);
238
239
240
241 void (*flush_tlb_user)(void);
242 void (*flush_tlb_kernel)(void);
243 void (*flush_tlb_single)(unsigned long addr);
244 void (*flush_tlb_others)(const struct cpumask *cpus,
245 struct mm_struct *mm,
246 unsigned long va);
247
248
249 int (*pgd_alloc)(struct mm_struct *mm);
250 void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
251
252
253
254
255
256 void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn);
257 void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn);
258 void (*alloc_pmd_clone)(unsigned long pfn, unsigned long clonepfn, unsigned long start, unsigned long count);
259 void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn);
260 void (*release_pte)(unsigned long pfn);
261 void (*release_pmd)(unsigned long pfn);
262 void (*release_pud)(unsigned long pfn);
263
264
265 void (*set_pte)(pte_t *ptep, pte_t pteval);
266 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
267 pte_t *ptep, pte_t pteval);
268 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
269 void (*pte_update)(struct mm_struct *mm, unsigned long addr,
270 pte_t *ptep);
271 void (*pte_update_defer)(struct mm_struct *mm,
272 unsigned long addr, pte_t *ptep);
273
274 pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
275 pte_t *ptep);
276 void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
277 pte_t *ptep, pte_t pte);
278
279 struct paravirt_callee_save pte_val;
280 struct paravirt_callee_save make_pte;
281
282 struct paravirt_callee_save pgd_val;
283 struct paravirt_callee_save make_pgd;
284
285#if PAGETABLE_LEVELS >= 3
286#ifdef CONFIG_X86_PAE
287 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
288 void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
289 pte_t *ptep);
290 void (*pmd_clear)(pmd_t *pmdp);
291
292#endif
293
294 void (*set_pud)(pud_t *pudp, pud_t pudval);
295
296 struct paravirt_callee_save pmd_val;
297 struct paravirt_callee_save make_pmd;
298
299#if PAGETABLE_LEVELS == 4
300 struct paravirt_callee_save pud_val;
301 struct paravirt_callee_save make_pud;
302
303 void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
304#endif
305#endif
306
307#ifdef CONFIG_HIGHPTE
308 void *(*kmap_atomic_pte)(struct page *page, enum km_type type);
309#endif
310
311 struct pv_lazy_ops lazy_mode;
312
313
314
315
316
317 void (*set_fixmap)(unsigned idx,
318 phys_addr_t phys, pgprot_t flags);
319};
320
321struct raw_spinlock;
322struct pv_lock_ops {
323 int (*spin_is_locked)(struct raw_spinlock *lock);
324 int (*spin_is_contended)(struct raw_spinlock *lock);
325 void (*spin_lock)(struct raw_spinlock *lock);
326 void (*spin_lock_flags)(struct raw_spinlock *lock, unsigned long flags);
327 int (*spin_trylock)(struct raw_spinlock *lock);
328 void (*spin_unlock)(struct raw_spinlock *lock);
329};
330
331
332
333
334struct paravirt_patch_template {
335 struct pv_init_ops pv_init_ops;
336 struct pv_time_ops pv_time_ops;
337 struct pv_cpu_ops pv_cpu_ops;
338 struct pv_irq_ops pv_irq_ops;
339 struct pv_apic_ops pv_apic_ops;
340 struct pv_mmu_ops pv_mmu_ops;
341 struct pv_lock_ops pv_lock_ops;
342};
343
344extern struct pv_info pv_info;
345extern struct pv_init_ops pv_init_ops;
346extern struct pv_time_ops pv_time_ops;
347extern struct pv_cpu_ops pv_cpu_ops;
348extern struct pv_irq_ops pv_irq_ops;
349extern struct pv_apic_ops pv_apic_ops;
350extern struct pv_mmu_ops pv_mmu_ops;
351extern struct pv_lock_ops pv_lock_ops;
352
353#define PARAVIRT_PATCH(x) \
354 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
355
356#define paravirt_type(op) \
357 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
358 [paravirt_opptr] "i" (&(op))
359#define paravirt_clobber(clobber) \
360 [paravirt_clobber] "i" (clobber)
361
362
363
364
365
366#define _paravirt_alt(insn_string, type, clobber) \
367 "771:\n\t" insn_string "\n" "772:\n" \
368 ".pushsection .parainstructions,\"a\"\n" \
369 _ASM_ALIGN "\n" \
370 _ASM_PTR " 771b\n" \
371 " .byte " type "\n" \
372 " .byte 772b-771b\n" \
373 " .short " clobber "\n" \
374 ".popsection\n"
375
376
377#define paravirt_alt(insn_string) \
378 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
379
380
381#define DEF_NATIVE(ops, name, code) \
382 extern const char start_##ops##_##name[], end_##ops##_##name[]; \
383 asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
384
385unsigned paravirt_patch_nop(void);
386unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
387unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
388unsigned paravirt_patch_ignore(unsigned len);
389unsigned paravirt_patch_call(void *insnbuf,
390 const void *target, u16 tgt_clobbers,
391 unsigned long addr, u16 site_clobbers,
392 unsigned len);
393unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
394 unsigned long addr, unsigned len);
395unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
396 unsigned long addr, unsigned len);
397
398unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
399 const char *start, const char *end);
400
401unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
402 unsigned long addr, unsigned len);
403
404int paravirt_disable_iospace(void);
405
406
407
408
409
410
411
412#define PARAVIRT_CALL "call *%c[paravirt_opptr];"
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478#ifdef CONFIG_X86_32
479#define PVOP_VCALL_ARGS \
480 unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx
481#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
482
483#define PVOP_CALL_ARG1(x) "a" ((unsigned long)(x))
484#define PVOP_CALL_ARG2(x) "d" ((unsigned long)(x))
485#define PVOP_CALL_ARG3(x) "c" ((unsigned long)(x))
486
487#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
488 "=c" (__ecx)
489#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
490
491#define PVOP_VCALLEE_CLOBBERS "=a" (__eax), "=d" (__edx)
492#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
493
494#define EXTRA_CLOBBERS
495#define VEXTRA_CLOBBERS
496#else
497
498#define PVOP_VCALL_ARGS \
499 unsigned long __edi = __edi, __esi = __esi, \
500 __edx = __edx, __ecx = __ecx, __eax = __eax
501#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
502
503#define PVOP_CALL_ARG1(x) "D" ((unsigned long)(x))
504#define PVOP_CALL_ARG2(x) "S" ((unsigned long)(x))
505#define PVOP_CALL_ARG3(x) "d" ((unsigned long)(x))
506#define PVOP_CALL_ARG4(x) "c" ((unsigned long)(x))
507
508#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
509 "=S" (__esi), "=d" (__edx), \
510 "=c" (__ecx)
511#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
512
513
514#define PVOP_VCALLEE_CLOBBERS "=a" (__eax)
515#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
516
517#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
518#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
519#endif
520
521#ifdef CONFIG_PARAVIRT_DEBUG
522#define PVOP_TEST_NULL(op) BUG_ON(op == NULL)
523#else
524#define PVOP_TEST_NULL(op) ((void)op)
525#endif
526
527#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr, \
528 pre, post, ...) \
529 ({ \
530 rettype __ret; \
531 PVOP_CALL_ARGS; \
532 PVOP_TEST_NULL(op); \
533 \
534 \
535 if (sizeof(rettype) > sizeof(unsigned long)) { \
536 asm volatile(pre \
537 paravirt_alt(PARAVIRT_CALL) \
538 post \
539 : call_clbr \
540 : paravirt_type(op), \
541 paravirt_clobber(clbr), \
542 ##__VA_ARGS__ \
543 : "memory", "cc" extra_clbr); \
544 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
545 } else { \
546 asm volatile(pre \
547 paravirt_alt(PARAVIRT_CALL) \
548 post \
549 : call_clbr \
550 : paravirt_type(op), \
551 paravirt_clobber(clbr), \
552 ##__VA_ARGS__ \
553 : "memory", "cc" extra_clbr); \
554 __ret = (rettype)__eax; \
555 } \
556 __ret; \
557 })
558
559#define __PVOP_CALL(rettype, op, pre, post, ...) \
560 ____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS, \
561 EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)
562
563#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...) \
564 ____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
565 PVOP_CALLEE_CLOBBERS, , \
566 pre, post, ##__VA_ARGS__)
567
568
569#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
570 ({ \
571 PVOP_VCALL_ARGS; \
572 PVOP_TEST_NULL(op); \
573 asm volatile(pre \
574 paravirt_alt(PARAVIRT_CALL) \
575 post \
576 : call_clbr \
577 : paravirt_type(op), \
578 paravirt_clobber(clbr), \
579 ##__VA_ARGS__ \
580 : "memory", "cc" extra_clbr); \
581 })
582
583#define __PVOP_VCALL(op, pre, post, ...) \
584 ____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS, \
585 VEXTRA_CLOBBERS, \
586 pre, post, ##__VA_ARGS__)
587
588#define __PVOP_VCALLEESAVE(op, pre, post, ...) \
589 ____PVOP_VCALL(op.func, CLBR_RET_REG, \
590 PVOP_VCALLEE_CLOBBERS, , \
591 pre, post, ##__VA_ARGS__)
592
593
594
595#define PVOP_CALL0(rettype, op) \
596 __PVOP_CALL(rettype, op, "", "")
597#define PVOP_VCALL0(op) \
598 __PVOP_VCALL(op, "", "")
599
600#define PVOP_CALLEE0(rettype, op) \
601 __PVOP_CALLEESAVE(rettype, op, "", "")
602#define PVOP_VCALLEE0(op) \
603 __PVOP_VCALLEESAVE(op, "", "")
604
605
606#define PVOP_CALL1(rettype, op, arg1) \
607 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
608#define PVOP_VCALL1(op, arg1) \
609 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))
610
611#define PVOP_CALLEE1(rettype, op, arg1) \
612 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
613#define PVOP_VCALLEE1(op, arg1) \
614 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))
615
616
617#define PVOP_CALL2(rettype, op, arg1, arg2) \
618 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
619 PVOP_CALL_ARG2(arg2))
620#define PVOP_VCALL2(op, arg1, arg2) \
621 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
622 PVOP_CALL_ARG2(arg2))
623
624#define PVOP_CALLEE2(rettype, op, arg1, arg2) \
625 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
626 PVOP_CALL_ARG2(arg2))
627#define PVOP_VCALLEE2(op, arg1, arg2) \
628 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1), \
629 PVOP_CALL_ARG2(arg2))
630
631
632#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
633 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
634 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
635#define PVOP_VCALL3(op, arg1, arg2, arg3) \
636 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
637 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
638
639
640#ifdef CONFIG_X86_32
641#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
642 __PVOP_CALL(rettype, op, \
643 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
644 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
645 PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
646#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
647 __PVOP_VCALL(op, \
648 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
649 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
650 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
651#else
652#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
653 __PVOP_CALL(rettype, op, "", "", \
654 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
655 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
656#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
657 __PVOP_VCALL(op, "", "", \
658 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
659 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
660#endif
661
662
663enum paravirt_lazy_mode {
664 PARAVIRT_LAZY_NONE,
665 PARAVIRT_LAZY_MMU,
666 PARAVIRT_LAZY_CPU,
667};
668
669enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
670void paravirt_start_context_switch(struct task_struct *prev);
671void paravirt_end_context_switch(struct task_struct *next);
672
673void paravirt_enter_lazy_mmu(void);
674void paravirt_leave_lazy_mmu(void);
675
676void _paravirt_nop(void);
677u32 _paravirt_ident_32(u32);
678u64 _paravirt_ident_64(u64);
679
680#define paravirt_nop ((void *)_paravirt_nop)
681
682
683struct paravirt_patch_site {
684 u8 *instr;
685 u8 instrtype;
686 u8 len;
687 u16 clobbers;
688};
689
690extern struct paravirt_patch_site __parainstructions[],
691 __parainstructions_end[];
692
693#endif
694
695#endif
696