1#ifndef _ASM_X86_PARAVIRT_TYPES_H
2#define _ASM_X86_PARAVIRT_TYPES_H
3
4
5#define CLBR_NONE 0
6#define CLBR_EAX (1 << 0)
7#define CLBR_ECX (1 << 1)
8#define CLBR_EDX (1 << 2)
9#define CLBR_EDI (1 << 3)
10
11#ifdef CONFIG_X86_32
12
13#define CLBR_ANY ((1 << 4) - 1)
14
15#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX)
16#define CLBR_RET_REG (CLBR_EAX | CLBR_EDX)
17#define CLBR_SCRATCH (0)
18#else
19#define CLBR_RAX CLBR_EAX
20#define CLBR_RCX CLBR_ECX
21#define CLBR_RDX CLBR_EDX
22#define CLBR_RDI CLBR_EDI
23#define CLBR_RSI (1 << 4)
24#define CLBR_R8 (1 << 5)
25#define CLBR_R9 (1 << 6)
26#define CLBR_R10 (1 << 7)
27#define CLBR_R11 (1 << 8)
28
29#define CLBR_ANY ((1 << 9) - 1)
30
31#define CLBR_ARG_REGS (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
32 CLBR_RCX | CLBR_R8 | CLBR_R9)
33#define CLBR_RET_REG (CLBR_RAX)
34#define CLBR_SCRATCH (CLBR_R10 | CLBR_R11)
35
36#endif
37
38#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
39
40#ifndef __ASSEMBLY__
41
42#include <asm/desc_defs.h>
43#include <asm/kmap_types.h>
44#include <asm/pgtable_types.h>
45
46struct page;
47struct thread_struct;
48struct desc_ptr;
49struct tss_struct;
50struct mm_struct;
51struct desc_struct;
52struct task_struct;
53struct cpumask;
54
55
56
57
58
59struct paravirt_callee_save {
60 void *func;
61};
62
63
64struct pv_info {
65 unsigned int kernel_rpl;
66 int shared_kernel_pmd;
67
68#ifdef CONFIG_X86_64
69 u16 extra_user_64bit_cs;
70#endif
71
72 int paravirt_enabled;
73 const char *name;
74};
75
76struct pv_init_ops {
77
78
79
80
81
82
83
84
85 unsigned (*patch)(u8 type, u16 clobber, void *insnbuf,
86 unsigned long addr, unsigned len);
87};
88
89
90struct pv_lazy_ops {
91
92 void (*enter)(void);
93 void (*leave)(void);
94 void (*flush)(void);
95};
96
97struct pv_time_ops {
98 unsigned long long (*sched_clock)(void);
99 unsigned long long (*steal_clock)(int cpu);
100 unsigned long (*get_tsc_khz)(void);
101};
102
103struct pv_cpu_ops {
104
105 unsigned long (*get_debugreg)(int regno);
106 void (*set_debugreg)(int regno, unsigned long value);
107
108 void (*clts)(void);
109
110 unsigned long (*read_cr0)(void);
111 void (*write_cr0)(unsigned long);
112
113 unsigned long (*read_cr4_safe)(void);
114 unsigned long (*read_cr4)(void);
115 void (*write_cr4)(unsigned long);
116
117#ifdef CONFIG_X86_64
118 unsigned long (*read_cr8)(void);
119 void (*write_cr8)(unsigned long);
120#endif
121
122
123 void (*load_tr_desc)(void);
124 void (*load_gdt)(const struct desc_ptr *);
125 void (*load_idt)(const struct desc_ptr *);
126
127 void (*store_idt)(struct desc_ptr *);
128 void (*set_ldt)(const void *desc, unsigned entries);
129 unsigned long (*store_tr)(void);
130 void (*load_tls)(struct thread_struct *t, unsigned int cpu);
131#ifdef CONFIG_X86_64
132 void (*load_gs_index)(unsigned int idx);
133#endif
134 void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
135 const void *desc);
136 void (*write_gdt_entry)(struct desc_struct *,
137 int entrynum, const void *desc, int size);
138 void (*write_idt_entry)(gate_desc *,
139 int entrynum, const gate_desc *gate);
140 void (*alloc_ldt)(struct desc_struct *ldt, unsigned entries);
141 void (*free_ldt)(struct desc_struct *ldt, unsigned entries);
142
143 void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t);
144
145 void (*set_iopl_mask)(unsigned mask);
146
147 void (*wbinvd)(void);
148 void (*io_delay)(void);
149
150
151 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
152 unsigned int *ecx, unsigned int *edx);
153
154
155
156 u64 (*read_msr)(unsigned int msr, int *err);
157 int (*write_msr)(unsigned int msr, unsigned low, unsigned high);
158
159 u64 (*read_tsc)(void);
160 u64 (*read_pmc)(int counter);
161 unsigned long long (*read_tscp)(unsigned int *aux);
162
163
164
165
166
167
168
169 void (*irq_enable_sysexit)(void);
170
171
172
173
174
175
176
177 void (*usergs_sysret64)(void);
178
179
180
181
182
183
184
185 void (*usergs_sysret32)(void);
186
187
188
189 void (*iret)(void);
190
191 void (*swapgs)(void);
192
193 void (*start_context_switch)(struct task_struct *prev);
194 void (*end_context_switch)(struct task_struct *next);
195};
196
197struct pv_irq_ops {
198
199
200
201
202
203
204
205
206
207 struct paravirt_callee_save save_fl;
208 struct paravirt_callee_save restore_fl;
209 struct paravirt_callee_save irq_disable;
210 struct paravirt_callee_save irq_enable;
211
212 void (*safe_halt)(void);
213 void (*halt)(void);
214
215#ifdef CONFIG_X86_64
216 void (*adjust_exception_frame)(void);
217#endif
218};
219
220struct pv_apic_ops {
221#ifdef CONFIG_X86_LOCAL_APIC
222 void (*startup_ipi_hook)(int phys_apicid,
223 unsigned long start_eip,
224 unsigned long start_esp);
225#endif
226};
227
228struct pv_mmu_ops {
229 unsigned long (*read_cr2)(void);
230 void (*write_cr2)(unsigned long);
231
232 unsigned long (*read_cr3)(void);
233 void (*write_cr3)(unsigned long);
234
235
236
237
238
239 void (*activate_mm)(struct mm_struct *prev,
240 struct mm_struct *next);
241 void (*dup_mmap)(struct mm_struct *oldmm,
242 struct mm_struct *mm);
243 void (*exit_mmap)(struct mm_struct *mm);
244
245
246
247 void (*flush_tlb_user)(void);
248 void (*flush_tlb_kernel)(void);
249 void (*flush_tlb_single)(unsigned long addr);
250 void (*flush_tlb_others)(const struct cpumask *cpus,
251 struct mm_struct *mm,
252 unsigned long start,
253 unsigned long end);
254
255
256 int (*pgd_alloc)(struct mm_struct *mm);
257 void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
258
259
260
261
262
263 void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn);
264 void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn);
265 void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn);
266 void (*release_pte)(unsigned long pfn);
267 void (*release_pmd)(unsigned long pfn);
268 void (*release_pud)(unsigned long pfn);
269
270
271 void (*set_pte)(pte_t *ptep, pte_t pteval);
272 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
273 pte_t *ptep, pte_t pteval);
274 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
275 void (*set_pmd_at)(struct mm_struct *mm, unsigned long addr,
276 pmd_t *pmdp, pmd_t pmdval);
277 void (*pte_update)(struct mm_struct *mm, unsigned long addr,
278 pte_t *ptep);
279 void (*pte_update_defer)(struct mm_struct *mm,
280 unsigned long addr, pte_t *ptep);
281 void (*pmd_update)(struct mm_struct *mm, unsigned long addr,
282 pmd_t *pmdp);
283 void (*pmd_update_defer)(struct mm_struct *mm,
284 unsigned long addr, pmd_t *pmdp);
285
286 pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr,
287 pte_t *ptep);
288 void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
289 pte_t *ptep, pte_t pte);
290
291 struct paravirt_callee_save pte_val;
292 struct paravirt_callee_save make_pte;
293
294 struct paravirt_callee_save pgd_val;
295 struct paravirt_callee_save make_pgd;
296
297#if PAGETABLE_LEVELS >= 3
298#ifdef CONFIG_X86_PAE
299 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
300 void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
301 pte_t *ptep);
302 void (*pmd_clear)(pmd_t *pmdp);
303
304#endif
305
306 void (*set_pud)(pud_t *pudp, pud_t pudval);
307
308 struct paravirt_callee_save pmd_val;
309 struct paravirt_callee_save make_pmd;
310
311#if PAGETABLE_LEVELS == 4
312 struct paravirt_callee_save pud_val;
313 struct paravirt_callee_save make_pud;
314
315 void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
316#endif
317#endif
318
319 struct pv_lazy_ops lazy_mode;
320
321
322
323
324
325 void (*set_fixmap)(unsigned idx,
326 phys_addr_t phys, pgprot_t flags);
327};
328
329struct arch_spinlock;
330struct pv_lock_ops {
331 int (*spin_is_locked)(struct arch_spinlock *lock);
332 int (*spin_is_contended)(struct arch_spinlock *lock);
333 void (*spin_lock)(struct arch_spinlock *lock);
334 void (*spin_lock_flags)(struct arch_spinlock *lock, unsigned long flags);
335 int (*spin_trylock)(struct arch_spinlock *lock);
336 void (*spin_unlock)(struct arch_spinlock *lock);
337};
338
339
340
341
342struct paravirt_patch_template {
343 struct pv_init_ops pv_init_ops;
344 struct pv_time_ops pv_time_ops;
345 struct pv_cpu_ops pv_cpu_ops;
346 struct pv_irq_ops pv_irq_ops;
347 struct pv_apic_ops pv_apic_ops;
348 struct pv_mmu_ops pv_mmu_ops;
349 struct pv_lock_ops pv_lock_ops;
350};
351
352extern struct pv_info pv_info;
353extern struct pv_init_ops pv_init_ops;
354extern struct pv_time_ops pv_time_ops;
355extern struct pv_cpu_ops pv_cpu_ops;
356extern struct pv_irq_ops pv_irq_ops;
357extern struct pv_apic_ops pv_apic_ops;
358extern struct pv_mmu_ops pv_mmu_ops;
359extern struct pv_lock_ops pv_lock_ops;
360
361#define PARAVIRT_PATCH(x) \
362 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
363
364#define paravirt_type(op) \
365 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
366 [paravirt_opptr] "i" (&(op))
367#define paravirt_clobber(clobber) \
368 [paravirt_clobber] "i" (clobber)
369
370
371
372
373
374#define _paravirt_alt(insn_string, type, clobber) \
375 "771:\n\t" insn_string "\n" "772:\n" \
376 ".pushsection .parainstructions,\"a\"\n" \
377 _ASM_ALIGN "\n" \
378 _ASM_PTR " 771b\n" \
379 " .byte " type "\n" \
380 " .byte 772b-771b\n" \
381 " .short " clobber "\n" \
382 ".popsection\n"
383
384
385#define paravirt_alt(insn_string) \
386 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
387
388
389#define DEF_NATIVE(ops, name, code) \
390 extern const char start_##ops##_##name[], end_##ops##_##name[]; \
391 asm("start_" #ops "_" #name ": " code "; end_" #ops "_" #name ":")
392
393unsigned paravirt_patch_nop(void);
394unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len);
395unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
396unsigned paravirt_patch_ignore(unsigned len);
397unsigned paravirt_patch_call(void *insnbuf,
398 const void *target, u16 tgt_clobbers,
399 unsigned long addr, u16 site_clobbers,
400 unsigned len);
401unsigned paravirt_patch_jmp(void *insnbuf, const void *target,
402 unsigned long addr, unsigned len);
403unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf,
404 unsigned long addr, unsigned len);
405
406unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
407 const char *start, const char *end);
408
409unsigned native_patch(u8 type, u16 clobbers, void *ibuf,
410 unsigned long addr, unsigned len);
411
412int paravirt_disable_iospace(void);
413
414
415
416
417
418
419
420#define PARAVIRT_CALL "call *%c[paravirt_opptr];"
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486#ifdef CONFIG_X86_32
487#define PVOP_VCALL_ARGS \
488 unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx
489#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
490
491#define PVOP_CALL_ARG1(x) "a" ((unsigned long)(x))
492#define PVOP_CALL_ARG2(x) "d" ((unsigned long)(x))
493#define PVOP_CALL_ARG3(x) "c" ((unsigned long)(x))
494
495#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
496 "=c" (__ecx)
497#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
498
499#define PVOP_VCALLEE_CLOBBERS "=a" (__eax), "=d" (__edx)
500#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
501
502#define EXTRA_CLOBBERS
503#define VEXTRA_CLOBBERS
504#else
505
506#define PVOP_VCALL_ARGS \
507 unsigned long __edi = __edi, __esi = __esi, \
508 __edx = __edx, __ecx = __ecx, __eax = __eax
509#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
510
511#define PVOP_CALL_ARG1(x) "D" ((unsigned long)(x))
512#define PVOP_CALL_ARG2(x) "S" ((unsigned long)(x))
513#define PVOP_CALL_ARG3(x) "d" ((unsigned long)(x))
514#define PVOP_CALL_ARG4(x) "c" ((unsigned long)(x))
515
516#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
517 "=S" (__esi), "=d" (__edx), \
518 "=c" (__ecx)
519#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
520
521
522#define PVOP_VCALLEE_CLOBBERS "=a" (__eax)
523#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
524
525#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
526#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
527#endif
528
529#ifdef CONFIG_PARAVIRT_DEBUG
530#define PVOP_TEST_NULL(op) BUG_ON(op == NULL)
531#else
532#define PVOP_TEST_NULL(op) ((void)op)
533#endif
534
535#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr, \
536 pre, post, ...) \
537 ({ \
538 rettype __ret; \
539 PVOP_CALL_ARGS; \
540 PVOP_TEST_NULL(op); \
541 \
542 \
543 if (sizeof(rettype) > sizeof(unsigned long)) { \
544 asm volatile(pre \
545 paravirt_alt(PARAVIRT_CALL) \
546 post \
547 : call_clbr \
548 : paravirt_type(op), \
549 paravirt_clobber(clbr), \
550 ##__VA_ARGS__ \
551 : "memory", "cc" extra_clbr); \
552 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
553 } else { \
554 asm volatile(pre \
555 paravirt_alt(PARAVIRT_CALL) \
556 post \
557 : call_clbr \
558 : paravirt_type(op), \
559 paravirt_clobber(clbr), \
560 ##__VA_ARGS__ \
561 : "memory", "cc" extra_clbr); \
562 __ret = (rettype)__eax; \
563 } \
564 __ret; \
565 })
566
567#define __PVOP_CALL(rettype, op, pre, post, ...) \
568 ____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS, \
569 EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)
570
571#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...) \
572 ____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
573 PVOP_CALLEE_CLOBBERS, , \
574 pre, post, ##__VA_ARGS__)
575
576
577#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
578 ({ \
579 PVOP_VCALL_ARGS; \
580 PVOP_TEST_NULL(op); \
581 asm volatile(pre \
582 paravirt_alt(PARAVIRT_CALL) \
583 post \
584 : call_clbr \
585 : paravirt_type(op), \
586 paravirt_clobber(clbr), \
587 ##__VA_ARGS__ \
588 : "memory", "cc" extra_clbr); \
589 })
590
591#define __PVOP_VCALL(op, pre, post, ...) \
592 ____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS, \
593 VEXTRA_CLOBBERS, \
594 pre, post, ##__VA_ARGS__)
595
596#define __PVOP_VCALLEESAVE(op, pre, post, ...) \
597 ____PVOP_VCALL(op.func, CLBR_RET_REG, \
598 PVOP_VCALLEE_CLOBBERS, , \
599 pre, post, ##__VA_ARGS__)
600
601
602
603#define PVOP_CALL0(rettype, op) \
604 __PVOP_CALL(rettype, op, "", "")
605#define PVOP_VCALL0(op) \
606 __PVOP_VCALL(op, "", "")
607
608#define PVOP_CALLEE0(rettype, op) \
609 __PVOP_CALLEESAVE(rettype, op, "", "")
610#define PVOP_VCALLEE0(op) \
611 __PVOP_VCALLEESAVE(op, "", "")
612
613
614#define PVOP_CALL1(rettype, op, arg1) \
615 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
616#define PVOP_VCALL1(op, arg1) \
617 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))
618
619#define PVOP_CALLEE1(rettype, op, arg1) \
620 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
621#define PVOP_VCALLEE1(op, arg1) \
622 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))
623
624
625#define PVOP_CALL2(rettype, op, arg1, arg2) \
626 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
627 PVOP_CALL_ARG2(arg2))
628#define PVOP_VCALL2(op, arg1, arg2) \
629 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
630 PVOP_CALL_ARG2(arg2))
631
632#define PVOP_CALLEE2(rettype, op, arg1, arg2) \
633 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
634 PVOP_CALL_ARG2(arg2))
635#define PVOP_VCALLEE2(op, arg1, arg2) \
636 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1), \
637 PVOP_CALL_ARG2(arg2))
638
639
640#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
641 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
642 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
643#define PVOP_VCALL3(op, arg1, arg2, arg3) \
644 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
645 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
646
647
648#ifdef CONFIG_X86_32
649#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
650 __PVOP_CALL(rettype, op, \
651 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
652 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
653 PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
654#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
655 __PVOP_VCALL(op, \
656 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
657 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
658 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
659#else
660#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
661 __PVOP_CALL(rettype, op, "", "", \
662 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
663 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
664#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
665 __PVOP_VCALL(op, "", "", \
666 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
667 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
668#endif
669
670
671enum paravirt_lazy_mode {
672 PARAVIRT_LAZY_NONE,
673 PARAVIRT_LAZY_MMU,
674 PARAVIRT_LAZY_CPU,
675};
676
677enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
678void paravirt_start_context_switch(struct task_struct *prev);
679void paravirt_end_context_switch(struct task_struct *next);
680
681void paravirt_enter_lazy_mmu(void);
682void paravirt_leave_lazy_mmu(void);
683void paravirt_flush_lazy_mmu(void);
684
685void _paravirt_nop(void);
686u32 _paravirt_ident_32(u32);
687u64 _paravirt_ident_64(u64);
688
689#define paravirt_nop ((void *)_paravirt_nop)
690
691
692struct paravirt_patch_site {
693 u8 *instr;
694 u8 instrtype;
695 u8 len;
696 u16 clobbers;
697};
698
699extern struct paravirt_patch_site __parainstructions[],
700 __parainstructions_end[];
701
702#endif
703
704#endif
705