1
2#ifndef _ASM_X86_PARAVIRT_TYPES_H
3#define _ASM_X86_PARAVIRT_TYPES_H
4
5
6#define CLBR_NONE 0
7#define CLBR_EAX (1 << 0)
8#define CLBR_ECX (1 << 1)
9#define CLBR_EDX (1 << 2)
10#define CLBR_EDI (1 << 3)
11
12#ifdef CONFIG_X86_32
13
14#define CLBR_ANY ((1 << 4) - 1)
15
16#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX)
17#define CLBR_RET_REG (CLBR_EAX | CLBR_EDX)
18#define CLBR_SCRATCH (0)
19#else
20#define CLBR_RAX CLBR_EAX
21#define CLBR_RCX CLBR_ECX
22#define CLBR_RDX CLBR_EDX
23#define CLBR_RDI CLBR_EDI
24#define CLBR_RSI (1 << 4)
25#define CLBR_R8 (1 << 5)
26#define CLBR_R9 (1 << 6)
27#define CLBR_R10 (1 << 7)
28#define CLBR_R11 (1 << 8)
29
30#define CLBR_ANY ((1 << 9) - 1)
31
32#define CLBR_ARG_REGS (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
33 CLBR_RCX | CLBR_R8 | CLBR_R9)
34#define CLBR_RET_REG (CLBR_RAX)
35#define CLBR_SCRATCH (CLBR_R10 | CLBR_R11)
36
37#endif
38
39#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
40
41#ifndef __ASSEMBLY__
42
43#include <asm/desc_defs.h>
44#include <asm/kmap_types.h>
45#include <asm/pgtable_types.h>
46#include <asm/nospec-branch.h>
47
48struct page;
49struct thread_struct;
50struct desc_ptr;
51struct tss_struct;
52struct mm_struct;
53struct desc_struct;
54struct task_struct;
55struct cpumask;
56struct flush_tlb_info;
57struct mmu_gather;
58struct vm_area_struct;
59
60
61
62
63
64struct paravirt_callee_save {
65 void *func;
66};
67
68
69struct pv_info {
70#ifdef CONFIG_PARAVIRT_XXL
71 unsigned int kernel_rpl;
72 int shared_kernel_pmd;
73
74#ifdef CONFIG_X86_64
75 u16 extra_user_64bit_cs;
76#endif
77#endif
78
79 const char *name;
80};
81
82struct pv_init_ops {
83
84
85
86
87
88
89
90
91 unsigned (*patch)(u8 type, void *insnbuf,
92 unsigned long addr, unsigned len);
93} __no_randomize_layout;
94
95#ifdef CONFIG_PARAVIRT_XXL
96struct pv_lazy_ops {
97
98 void (*enter)(void);
99 void (*leave)(void);
100 void (*flush)(void);
101} __no_randomize_layout;
102#endif
103
104struct pv_time_ops {
105 unsigned long long (*sched_clock)(void);
106 unsigned long long (*steal_clock)(int cpu);
107} __no_randomize_layout;
108
109struct pv_cpu_ops {
110
111 void (*io_delay)(void);
112
113#ifdef CONFIG_PARAVIRT_XXL
114 unsigned long (*get_debugreg)(int regno);
115 void (*set_debugreg)(int regno, unsigned long value);
116
117 unsigned long (*read_cr0)(void);
118 void (*write_cr0)(unsigned long);
119
120 void (*write_cr4)(unsigned long);
121
122#ifdef CONFIG_X86_64
123 unsigned long (*read_cr8)(void);
124 void (*write_cr8)(unsigned long);
125#endif
126
127
128 void (*load_tr_desc)(void);
129 void (*load_gdt)(const struct desc_ptr *);
130 void (*load_idt)(const struct desc_ptr *);
131 void (*set_ldt)(const void *desc, unsigned entries);
132 unsigned long (*store_tr)(void);
133 void (*load_tls)(struct thread_struct *t, unsigned int cpu);
134#ifdef CONFIG_X86_64
135 void (*load_gs_index)(unsigned int idx);
136#endif
137 void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
138 const void *desc);
139 void (*write_gdt_entry)(struct desc_struct *,
140 int entrynum, const void *desc, int size);
141 void (*write_idt_entry)(gate_desc *,
142 int entrynum, const gate_desc *gate);
143 void (*alloc_ldt)(struct desc_struct *ldt, unsigned entries);
144 void (*free_ldt)(struct desc_struct *ldt, unsigned entries);
145
146 void (*load_sp0)(unsigned long sp0);
147
148 void (*set_iopl_mask)(unsigned mask);
149
150 void (*wbinvd)(void);
151
152
153 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
154 unsigned int *ecx, unsigned int *edx);
155
156
157 u64 (*read_msr)(unsigned int msr);
158 void (*write_msr)(unsigned int msr, unsigned low, unsigned high);
159
160
161
162
163
164 u64 (*read_msr_safe)(unsigned int msr, int *err);
165 int (*write_msr_safe)(unsigned int msr, unsigned low, unsigned high);
166
167 u64 (*read_pmc)(int counter);
168
169
170
171
172
173
174
175 void (*usergs_sysret64)(void);
176
177
178
179 void (*iret)(void);
180
181 void (*swapgs)(void);
182
183 void (*start_context_switch)(struct task_struct *prev);
184 void (*end_context_switch)(struct task_struct *next);
185#endif
186} __no_randomize_layout;
187
188struct pv_irq_ops {
189#ifdef CONFIG_PARAVIRT_XXL
190
191
192
193
194
195
196
197
198
199 struct paravirt_callee_save save_fl;
200 struct paravirt_callee_save restore_fl;
201 struct paravirt_callee_save irq_disable;
202 struct paravirt_callee_save irq_enable;
203
204 void (*safe_halt)(void);
205 void (*halt)(void);
206#endif
207} __no_randomize_layout;
208
209struct pv_mmu_ops {
210
211 void (*flush_tlb_user)(void);
212 void (*flush_tlb_kernel)(void);
213 void (*flush_tlb_one_user)(unsigned long addr);
214 void (*flush_tlb_others)(const struct cpumask *cpus,
215 const struct flush_tlb_info *info);
216
217 void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
218
219
220 void (*exit_mmap)(struct mm_struct *mm);
221
222#ifdef CONFIG_PARAVIRT_XXL
223 unsigned long (*read_cr2)(void);
224 void (*write_cr2)(unsigned long);
225
226 unsigned long (*read_cr3)(void);
227 void (*write_cr3)(unsigned long);
228
229
230 void (*activate_mm)(struct mm_struct *prev,
231 struct mm_struct *next);
232 void (*dup_mmap)(struct mm_struct *oldmm,
233 struct mm_struct *mm);
234
235
236 int (*pgd_alloc)(struct mm_struct *mm);
237 void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
238
239
240
241
242
243 void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn);
244 void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn);
245 void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn);
246 void (*alloc_p4d)(struct mm_struct *mm, unsigned long pfn);
247 void (*release_pte)(unsigned long pfn);
248 void (*release_pmd)(unsigned long pfn);
249 void (*release_pud)(unsigned long pfn);
250 void (*release_p4d)(unsigned long pfn);
251
252
253 void (*set_pte)(pte_t *ptep, pte_t pteval);
254 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
255 pte_t *ptep, pte_t pteval);
256 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
257
258 pte_t (*ptep_modify_prot_start)(struct vm_area_struct *vma, unsigned long addr,
259 pte_t *ptep);
260 void (*ptep_modify_prot_commit)(struct vm_area_struct *vma, unsigned long addr,
261 pte_t *ptep, pte_t pte);
262
263 struct paravirt_callee_save pte_val;
264 struct paravirt_callee_save make_pte;
265
266 struct paravirt_callee_save pgd_val;
267 struct paravirt_callee_save make_pgd;
268
269#if CONFIG_PGTABLE_LEVELS >= 3
270#ifdef CONFIG_X86_PAE
271 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
272 void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
273 pte_t *ptep);
274 void (*pmd_clear)(pmd_t *pmdp);
275
276#endif
277
278 void (*set_pud)(pud_t *pudp, pud_t pudval);
279
280 struct paravirt_callee_save pmd_val;
281 struct paravirt_callee_save make_pmd;
282
283#if CONFIG_PGTABLE_LEVELS >= 4
284 struct paravirt_callee_save pud_val;
285 struct paravirt_callee_save make_pud;
286
287 void (*set_p4d)(p4d_t *p4dp, p4d_t p4dval);
288
289#if CONFIG_PGTABLE_LEVELS >= 5
290 struct paravirt_callee_save p4d_val;
291 struct paravirt_callee_save make_p4d;
292
293 void (*set_pgd)(pgd_t *pgdp, pgd_t pgdval);
294#endif
295
296#endif
297
298#endif
299
300 struct pv_lazy_ops lazy_mode;
301
302
303
304
305
306 void (*set_fixmap)(unsigned idx,
307 phys_addr_t phys, pgprot_t flags);
308#endif
309} __no_randomize_layout;
310
311struct arch_spinlock;
312#ifdef CONFIG_SMP
313#include <asm/spinlock_types.h>
314#endif
315
316struct qspinlock;
317
318struct pv_lock_ops {
319 void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val);
320 struct paravirt_callee_save queued_spin_unlock;
321
322 void (*wait)(u8 *ptr, u8 val);
323 void (*kick)(int cpu);
324
325 struct paravirt_callee_save vcpu_is_preempted;
326} __no_randomize_layout;
327
328
329
330
331struct paravirt_patch_template {
332 struct pv_init_ops init;
333 struct pv_time_ops time;
334 struct pv_cpu_ops cpu;
335 struct pv_irq_ops irq;
336 struct pv_mmu_ops mmu;
337 struct pv_lock_ops lock;
338} __no_randomize_layout;
339
340extern struct pv_info pv_info;
341extern struct paravirt_patch_template pv_ops;
342
343#define PARAVIRT_PATCH(x) \
344 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
345
346#define paravirt_type(op) \
347 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
348 [paravirt_opptr] "i" (&(pv_ops.op))
349#define paravirt_clobber(clobber) \
350 [paravirt_clobber] "i" (clobber)
351
352
353
354
355
356#define _paravirt_alt(insn_string, type, clobber) \
357 "771:\n\t" insn_string "\n" "772:\n" \
358 ".pushsection .parainstructions,\"a\"\n" \
359 _ASM_ALIGN "\n" \
360 _ASM_PTR " 771b\n" \
361 " .byte " type "\n" \
362 " .byte 772b-771b\n" \
363 " .short " clobber "\n" \
364 ".popsection\n"
365
366
367#define paravirt_alt(insn_string) \
368 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
369
370
371#define NATIVE_LABEL(a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t"
372
373#define DEF_NATIVE(ops, name, code) \
374 __visible extern const char start_##ops##_##name[], end_##ops##_##name[]; \
375 asm(NATIVE_LABEL("start_", ops, name) code NATIVE_LABEL("end_", ops, name))
376
377unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len);
378unsigned paravirt_patch_default(u8 type, void *insnbuf,
379 unsigned long addr, unsigned len);
380
381unsigned paravirt_patch_insns(void *insnbuf, unsigned len,
382 const char *start, const char *end);
383
384unsigned native_patch(u8 type, void *ibuf, unsigned long addr, unsigned len);
385
386int paravirt_disable_iospace(void);
387
388
389
390
391
392
393
394#define PARAVIRT_CALL \
395 ANNOTATE_RETPOLINE_SAFE \
396 "call *%c[paravirt_opptr];"
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462#ifdef CONFIG_X86_32
463#define PVOP_VCALL_ARGS \
464 unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx;
465
466#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
467
468#define PVOP_CALL_ARG1(x) "a" ((unsigned long)(x))
469#define PVOP_CALL_ARG2(x) "d" ((unsigned long)(x))
470#define PVOP_CALL_ARG3(x) "c" ((unsigned long)(x))
471
472#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
473 "=c" (__ecx)
474#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
475
476#define PVOP_VCALLEE_CLOBBERS "=a" (__eax), "=d" (__edx)
477#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
478
479#define EXTRA_CLOBBERS
480#define VEXTRA_CLOBBERS
481#else
482
483#define PVOP_VCALL_ARGS \
484 unsigned long __edi = __edi, __esi = __esi, \
485 __edx = __edx, __ecx = __ecx, __eax = __eax;
486
487#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
488
489#define PVOP_CALL_ARG1(x) "D" ((unsigned long)(x))
490#define PVOP_CALL_ARG2(x) "S" ((unsigned long)(x))
491#define PVOP_CALL_ARG3(x) "d" ((unsigned long)(x))
492#define PVOP_CALL_ARG4(x) "c" ((unsigned long)(x))
493
494#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
495 "=S" (__esi), "=d" (__edx), \
496 "=c" (__ecx)
497#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
498
499
500#define PVOP_VCALLEE_CLOBBERS "=a" (__eax)
501#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
502
503#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
504#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
505#endif
506
507#ifdef CONFIG_PARAVIRT_DEBUG
508#define PVOP_TEST_NULL(op) BUG_ON(pv_ops.op == NULL)
509#else
510#define PVOP_TEST_NULL(op) ((void)pv_ops.op)
511#endif
512
513#define PVOP_RETMASK(rettype) \
514 ({ unsigned long __mask = ~0UL; \
515 switch (sizeof(rettype)) { \
516 case 1: __mask = 0xffUL; break; \
517 case 2: __mask = 0xffffUL; break; \
518 case 4: __mask = 0xffffffffUL; break; \
519 default: break; \
520 } \
521 __mask; \
522 })
523
524
525#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr, \
526 pre, post, ...) \
527 ({ \
528 rettype __ret; \
529 PVOP_CALL_ARGS; \
530 PVOP_TEST_NULL(op); \
531 \
532 \
533 if (sizeof(rettype) > sizeof(unsigned long)) { \
534 asm volatile(pre \
535 paravirt_alt(PARAVIRT_CALL) \
536 post \
537 : call_clbr, ASM_CALL_CONSTRAINT \
538 : paravirt_type(op), \
539 paravirt_clobber(clbr), \
540 ##__VA_ARGS__ \
541 : "memory", "cc" extra_clbr); \
542 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
543 } else { \
544 asm volatile(pre \
545 paravirt_alt(PARAVIRT_CALL) \
546 post \
547 : call_clbr, ASM_CALL_CONSTRAINT \
548 : paravirt_type(op), \
549 paravirt_clobber(clbr), \
550 ##__VA_ARGS__ \
551 : "memory", "cc" extra_clbr); \
552 __ret = (rettype)(__eax & PVOP_RETMASK(rettype)); \
553 } \
554 __ret; \
555 })
556
557#define __PVOP_CALL(rettype, op, pre, post, ...) \
558 ____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS, \
559 EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)
560
561#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...) \
562 ____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
563 PVOP_CALLEE_CLOBBERS, , \
564 pre, post, ##__VA_ARGS__)
565
566
567#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
568 ({ \
569 PVOP_VCALL_ARGS; \
570 PVOP_TEST_NULL(op); \
571 asm volatile(pre \
572 paravirt_alt(PARAVIRT_CALL) \
573 post \
574 : call_clbr, ASM_CALL_CONSTRAINT \
575 : paravirt_type(op), \
576 paravirt_clobber(clbr), \
577 ##__VA_ARGS__ \
578 : "memory", "cc" extra_clbr); \
579 })
580
581#define __PVOP_VCALL(op, pre, post, ...) \
582 ____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS, \
583 VEXTRA_CLOBBERS, \
584 pre, post, ##__VA_ARGS__)
585
586#define __PVOP_VCALLEESAVE(op, pre, post, ...) \
587 ____PVOP_VCALL(op.func, CLBR_RET_REG, \
588 PVOP_VCALLEE_CLOBBERS, , \
589 pre, post, ##__VA_ARGS__)
590
591
592
593#define PVOP_CALL0(rettype, op) \
594 __PVOP_CALL(rettype, op, "", "")
595#define PVOP_VCALL0(op) \
596 __PVOP_VCALL(op, "", "")
597
598#define PVOP_CALLEE0(rettype, op) \
599 __PVOP_CALLEESAVE(rettype, op, "", "")
600#define PVOP_VCALLEE0(op) \
601 __PVOP_VCALLEESAVE(op, "", "")
602
603
604#define PVOP_CALL1(rettype, op, arg1) \
605 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
606#define PVOP_VCALL1(op, arg1) \
607 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))
608
609#define PVOP_CALLEE1(rettype, op, arg1) \
610 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
611#define PVOP_VCALLEE1(op, arg1) \
612 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))
613
614
615#define PVOP_CALL2(rettype, op, arg1, arg2) \
616 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
617 PVOP_CALL_ARG2(arg2))
618#define PVOP_VCALL2(op, arg1, arg2) \
619 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
620 PVOP_CALL_ARG2(arg2))
621
622#define PVOP_CALLEE2(rettype, op, arg1, arg2) \
623 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
624 PVOP_CALL_ARG2(arg2))
625#define PVOP_VCALLEE2(op, arg1, arg2) \
626 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1), \
627 PVOP_CALL_ARG2(arg2))
628
629
630#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
631 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
632 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
633#define PVOP_VCALL3(op, arg1, arg2, arg3) \
634 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
635 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
636
637
638#ifdef CONFIG_X86_32
639#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
640 __PVOP_CALL(rettype, op, \
641 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
642 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
643 PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
644#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
645 __PVOP_VCALL(op, \
646 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
647 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
648 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
649#else
650#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
651 __PVOP_CALL(rettype, op, "", "", \
652 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
653 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
654#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
655 __PVOP_VCALL(op, "", "", \
656 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
657 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
658#endif
659
660
661enum paravirt_lazy_mode {
662 PARAVIRT_LAZY_NONE,
663 PARAVIRT_LAZY_MMU,
664 PARAVIRT_LAZY_CPU,
665};
666
667enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
668void paravirt_start_context_switch(struct task_struct *prev);
669void paravirt_end_context_switch(struct task_struct *next);
670
671void paravirt_enter_lazy_mmu(void);
672void paravirt_leave_lazy_mmu(void);
673void paravirt_flush_lazy_mmu(void);
674
675void _paravirt_nop(void);
676u64 _paravirt_ident_64(u64);
677
678#define paravirt_nop ((void *)_paravirt_nop)
679
680
681struct paravirt_patch_site {
682 u8 *instr;
683 u8 instrtype;
684 u8 len;
685};
686
687extern struct paravirt_patch_site __parainstructions[],
688 __parainstructions_end[];
689
690#endif
691
692#endif
693