1
2#ifndef _ASM_X86_PARAVIRT_TYPES_H
3#define _ASM_X86_PARAVIRT_TYPES_H
4
5
6#define CLBR_NONE 0
7#define CLBR_EAX (1 << 0)
8#define CLBR_ECX (1 << 1)
9#define CLBR_EDX (1 << 2)
10#define CLBR_EDI (1 << 3)
11
12#ifdef CONFIG_X86_32
13
14#define CLBR_ANY ((1 << 4) - 1)
15
16#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX)
17#define CLBR_RET_REG (CLBR_EAX | CLBR_EDX)
18#define CLBR_SCRATCH (0)
19#else
20#define CLBR_RAX CLBR_EAX
21#define CLBR_RCX CLBR_ECX
22#define CLBR_RDX CLBR_EDX
23#define CLBR_RDI CLBR_EDI
24#define CLBR_RSI (1 << 4)
25#define CLBR_R8 (1 << 5)
26#define CLBR_R9 (1 << 6)
27#define CLBR_R10 (1 << 7)
28#define CLBR_R11 (1 << 8)
29
30#define CLBR_ANY ((1 << 9) - 1)
31
32#define CLBR_ARG_REGS (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
33 CLBR_RCX | CLBR_R8 | CLBR_R9)
34#define CLBR_RET_REG (CLBR_RAX)
35#define CLBR_SCRATCH (CLBR_R10 | CLBR_R11)
36
37#endif
38
39#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
40
41#ifndef __ASSEMBLY__
42
43#include <asm/desc_defs.h>
44#include <asm/kmap_types.h>
45#include <asm/pgtable_types.h>
46#include <asm/nospec-branch.h>
47
48struct page;
49struct thread_struct;
50struct desc_ptr;
51struct tss_struct;
52struct mm_struct;
53struct desc_struct;
54struct task_struct;
55struct cpumask;
56struct flush_tlb_info;
57struct mmu_gather;
58struct vm_area_struct;
59
60
61
62
63
64struct paravirt_callee_save {
65 void *func;
66};
67
68
69struct pv_info {
70#ifdef CONFIG_PARAVIRT_XXL
71 unsigned int kernel_rpl;
72 int shared_kernel_pmd;
73
74#ifdef CONFIG_X86_64
75 u16 extra_user_64bit_cs;
76#endif
77#endif
78
79 const char *name;
80};
81
82struct pv_init_ops {
83
84
85
86
87
88
89
90
91 unsigned (*patch)(u8 type, void *insn_buff,
92 unsigned long addr, unsigned len);
93} __no_randomize_layout;
94
95#ifdef CONFIG_PARAVIRT_XXL
96struct pv_lazy_ops {
97
98 void (*enter)(void);
99 void (*leave)(void);
100 void (*flush)(void);
101} __no_randomize_layout;
102#endif
103
104struct pv_time_ops {
105 unsigned long long (*sched_clock)(void);
106 unsigned long long (*steal_clock)(int cpu);
107} __no_randomize_layout;
108
109struct pv_cpu_ops {
110
111 void (*io_delay)(void);
112
113#ifdef CONFIG_PARAVIRT_XXL
114 unsigned long (*get_debugreg)(int regno);
115 void (*set_debugreg)(int regno, unsigned long value);
116
117 unsigned long (*read_cr0)(void);
118 void (*write_cr0)(unsigned long);
119
120 void (*write_cr4)(unsigned long);
121
122#ifdef CONFIG_X86_64
123 unsigned long (*read_cr8)(void);
124 void (*write_cr8)(unsigned long);
125#endif
126
127
128 void (*load_tr_desc)(void);
129 void (*load_gdt)(const struct desc_ptr *);
130 void (*load_idt)(const struct desc_ptr *);
131 void (*set_ldt)(const void *desc, unsigned entries);
132 unsigned long (*store_tr)(void);
133 void (*load_tls)(struct thread_struct *t, unsigned int cpu);
134#ifdef CONFIG_X86_64
135 void (*load_gs_index)(unsigned int idx);
136#endif
137 void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum,
138 const void *desc);
139 void (*write_gdt_entry)(struct desc_struct *,
140 int entrynum, const void *desc, int size);
141 void (*write_idt_entry)(gate_desc *,
142 int entrynum, const gate_desc *gate);
143 void (*alloc_ldt)(struct desc_struct *ldt, unsigned entries);
144 void (*free_ldt)(struct desc_struct *ldt, unsigned entries);
145
146 void (*load_sp0)(unsigned long sp0);
147
148 void (*set_iopl_mask)(unsigned mask);
149
150 void (*wbinvd)(void);
151
152
153 void (*cpuid)(unsigned int *eax, unsigned int *ebx,
154 unsigned int *ecx, unsigned int *edx);
155
156
157 u64 (*read_msr)(unsigned int msr);
158 void (*write_msr)(unsigned int msr, unsigned low, unsigned high);
159
160
161
162
163
164 u64 (*read_msr_safe)(unsigned int msr, int *err);
165 int (*write_msr_safe)(unsigned int msr, unsigned low, unsigned high);
166
167 u64 (*read_pmc)(int counter);
168
169
170
171
172
173
174
175 void (*usergs_sysret64)(void);
176
177
178
179 void (*iret)(void);
180
181 void (*swapgs)(void);
182
183 void (*start_context_switch)(struct task_struct *prev);
184 void (*end_context_switch)(struct task_struct *next);
185#endif
186} __no_randomize_layout;
187
188struct pv_irq_ops {
189#ifdef CONFIG_PARAVIRT_XXL
190
191
192
193
194
195
196
197
198
199 struct paravirt_callee_save save_fl;
200 struct paravirt_callee_save restore_fl;
201 struct paravirt_callee_save irq_disable;
202 struct paravirt_callee_save irq_enable;
203
204 void (*safe_halt)(void);
205 void (*halt)(void);
206#endif
207} __no_randomize_layout;
208
209struct pv_mmu_ops {
210
211 void (*flush_tlb_user)(void);
212 void (*flush_tlb_kernel)(void);
213 void (*flush_tlb_one_user)(unsigned long addr);
214 void (*flush_tlb_others)(const struct cpumask *cpus,
215 const struct flush_tlb_info *info);
216
217 void (*tlb_remove_table)(struct mmu_gather *tlb, void *table);
218
219
220 void (*exit_mmap)(struct mm_struct *mm);
221
222#ifdef CONFIG_PARAVIRT_XXL
223 struct paravirt_callee_save read_cr2;
224 void (*write_cr2)(unsigned long);
225
226 unsigned long (*read_cr3)(void);
227 void (*write_cr3)(unsigned long);
228
229
230 void (*activate_mm)(struct mm_struct *prev,
231 struct mm_struct *next);
232 void (*dup_mmap)(struct mm_struct *oldmm,
233 struct mm_struct *mm);
234
235
236 int (*pgd_alloc)(struct mm_struct *mm);
237 void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd);
238
239
240
241
242
243 void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn);
244 void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn);
245 void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn);
246 void (*alloc_p4d)(struct mm_struct *mm, unsigned long pfn);
247 void (*release_pte)(unsigned long pfn);
248 void (*release_pmd)(unsigned long pfn);
249 void (*release_pud)(unsigned long pfn);
250 void (*release_p4d)(unsigned long pfn);
251
252
253 void (*set_pte)(pte_t *ptep, pte_t pteval);
254 void (*set_pte_at)(struct mm_struct *mm, unsigned long addr,
255 pte_t *ptep, pte_t pteval);
256 void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval);
257
258 pte_t (*ptep_modify_prot_start)(struct vm_area_struct *vma, unsigned long addr,
259 pte_t *ptep);
260 void (*ptep_modify_prot_commit)(struct vm_area_struct *vma, unsigned long addr,
261 pte_t *ptep, pte_t pte);
262
263 struct paravirt_callee_save pte_val;
264 struct paravirt_callee_save make_pte;
265
266 struct paravirt_callee_save pgd_val;
267 struct paravirt_callee_save make_pgd;
268
269#if CONFIG_PGTABLE_LEVELS >= 3
270#ifdef CONFIG_X86_PAE
271 void (*set_pte_atomic)(pte_t *ptep, pte_t pteval);
272 void (*pte_clear)(struct mm_struct *mm, unsigned long addr,
273 pte_t *ptep);
274 void (*pmd_clear)(pmd_t *pmdp);
275
276#endif
277
278 void (*set_pud)(pud_t *pudp, pud_t pudval);
279
280 struct paravirt_callee_save pmd_val;
281 struct paravirt_callee_save make_pmd;
282
283#if CONFIG_PGTABLE_LEVELS >= 4
284 struct paravirt_callee_save pud_val;
285 struct paravirt_callee_save make_pud;
286
287 void (*set_p4d)(p4d_t *p4dp, p4d_t p4dval);
288
289#if CONFIG_PGTABLE_LEVELS >= 5
290 struct paravirt_callee_save p4d_val;
291 struct paravirt_callee_save make_p4d;
292
293 void (*set_pgd)(pgd_t *pgdp, pgd_t pgdval);
294#endif
295
296#endif
297
298#endif
299
300 struct pv_lazy_ops lazy_mode;
301
302
303
304
305
306 void (*set_fixmap)(unsigned idx,
307 phys_addr_t phys, pgprot_t flags);
308#endif
309} __no_randomize_layout;
310
311struct arch_spinlock;
312#ifdef CONFIG_SMP
313#include <asm/spinlock_types.h>
314#endif
315
316struct qspinlock;
317
318struct pv_lock_ops {
319 void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val);
320 struct paravirt_callee_save queued_spin_unlock;
321
322 void (*wait)(u8 *ptr, u8 val);
323 void (*kick)(int cpu);
324
325 struct paravirt_callee_save vcpu_is_preempted;
326} __no_randomize_layout;
327
328
329
330
331struct paravirt_patch_template {
332 struct pv_init_ops init;
333 struct pv_time_ops time;
334 struct pv_cpu_ops cpu;
335 struct pv_irq_ops irq;
336 struct pv_mmu_ops mmu;
337 struct pv_lock_ops lock;
338} __no_randomize_layout;
339
340extern struct pv_info pv_info;
341extern struct paravirt_patch_template pv_ops;
342
343#define PARAVIRT_PATCH(x) \
344 (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
345
346#define paravirt_type(op) \
347 [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \
348 [paravirt_opptr] "i" (&(pv_ops.op))
349#define paravirt_clobber(clobber) \
350 [paravirt_clobber] "i" (clobber)
351
352
353
354
355
356#define _paravirt_alt(insn_string, type, clobber) \
357 "771:\n\t" insn_string "\n" "772:\n" \
358 ".pushsection .parainstructions,\"a\"\n" \
359 _ASM_ALIGN "\n" \
360 _ASM_PTR " 771b\n" \
361 " .byte " type "\n" \
362 " .byte 772b-771b\n" \
363 " .short " clobber "\n" \
364 ".popsection\n"
365
366
367#define paravirt_alt(insn_string) \
368 _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]")
369
370
371#define NATIVE_LABEL(a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t"
372
373unsigned paravirt_patch_ident_64(void *insn_buff, unsigned len);
374unsigned paravirt_patch_default(u8 type, void *insn_buff, unsigned long addr, unsigned len);
375unsigned paravirt_patch_insns(void *insn_buff, unsigned len, const char *start, const char *end);
376
377unsigned native_patch(u8 type, void *insn_buff, unsigned long addr, unsigned len);
378
379int paravirt_disable_iospace(void);
380
381
382
383
384
385
386
387#define PARAVIRT_CALL \
388 ANNOTATE_RETPOLINE_SAFE \
389 "call *%c[paravirt_opptr];"
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455#ifdef CONFIG_X86_32
456#define PVOP_VCALL_ARGS \
457 unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx;
458
459#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
460
461#define PVOP_CALL_ARG1(x) "a" ((unsigned long)(x))
462#define PVOP_CALL_ARG2(x) "d" ((unsigned long)(x))
463#define PVOP_CALL_ARG3(x) "c" ((unsigned long)(x))
464
465#define PVOP_VCALL_CLOBBERS "=a" (__eax), "=d" (__edx), \
466 "=c" (__ecx)
467#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS
468
469#define PVOP_VCALLEE_CLOBBERS "=a" (__eax), "=d" (__edx)
470#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
471
472#define EXTRA_CLOBBERS
473#define VEXTRA_CLOBBERS
474#else
475
476#define PVOP_VCALL_ARGS \
477 unsigned long __edi = __edi, __esi = __esi, \
478 __edx = __edx, __ecx = __ecx, __eax = __eax;
479
480#define PVOP_CALL_ARGS PVOP_VCALL_ARGS
481
482#define PVOP_CALL_ARG1(x) "D" ((unsigned long)(x))
483#define PVOP_CALL_ARG2(x) "S" ((unsigned long)(x))
484#define PVOP_CALL_ARG3(x) "d" ((unsigned long)(x))
485#define PVOP_CALL_ARG4(x) "c" ((unsigned long)(x))
486
487#define PVOP_VCALL_CLOBBERS "=D" (__edi), \
488 "=S" (__esi), "=d" (__edx), \
489 "=c" (__ecx)
490#define PVOP_CALL_CLOBBERS PVOP_VCALL_CLOBBERS, "=a" (__eax)
491
492
493#define PVOP_VCALLEE_CLOBBERS "=a" (__eax)
494#define PVOP_CALLEE_CLOBBERS PVOP_VCALLEE_CLOBBERS
495
496#define EXTRA_CLOBBERS , "r8", "r9", "r10", "r11"
497#define VEXTRA_CLOBBERS , "rax", "r8", "r9", "r10", "r11"
498#endif
499
500#ifdef CONFIG_PARAVIRT_DEBUG
501#define PVOP_TEST_NULL(op) BUG_ON(pv_ops.op == NULL)
502#else
503#define PVOP_TEST_NULL(op) ((void)pv_ops.op)
504#endif
505
506#define PVOP_RETMASK(rettype) \
507 ({ unsigned long __mask = ~0UL; \
508 switch (sizeof(rettype)) { \
509 case 1: __mask = 0xffUL; break; \
510 case 2: __mask = 0xffffUL; break; \
511 case 4: __mask = 0xffffffffUL; break; \
512 default: break; \
513 } \
514 __mask; \
515 })
516
517
518#define ____PVOP_CALL(rettype, op, clbr, call_clbr, extra_clbr, \
519 pre, post, ...) \
520 ({ \
521 rettype __ret; \
522 PVOP_CALL_ARGS; \
523 PVOP_TEST_NULL(op); \
524 \
525 \
526 if (sizeof(rettype) > sizeof(unsigned long)) { \
527 asm volatile(pre \
528 paravirt_alt(PARAVIRT_CALL) \
529 post \
530 : call_clbr, ASM_CALL_CONSTRAINT \
531 : paravirt_type(op), \
532 paravirt_clobber(clbr), \
533 ##__VA_ARGS__ \
534 : "memory", "cc" extra_clbr); \
535 __ret = (rettype)((((u64)__edx) << 32) | __eax); \
536 } else { \
537 asm volatile(pre \
538 paravirt_alt(PARAVIRT_CALL) \
539 post \
540 : call_clbr, ASM_CALL_CONSTRAINT \
541 : paravirt_type(op), \
542 paravirt_clobber(clbr), \
543 ##__VA_ARGS__ \
544 : "memory", "cc" extra_clbr); \
545 __ret = (rettype)(__eax & PVOP_RETMASK(rettype)); \
546 } \
547 __ret; \
548 })
549
550#define __PVOP_CALL(rettype, op, pre, post, ...) \
551 ____PVOP_CALL(rettype, op, CLBR_ANY, PVOP_CALL_CLOBBERS, \
552 EXTRA_CLOBBERS, pre, post, ##__VA_ARGS__)
553
554#define __PVOP_CALLEESAVE(rettype, op, pre, post, ...) \
555 ____PVOP_CALL(rettype, op.func, CLBR_RET_REG, \
556 PVOP_CALLEE_CLOBBERS, , \
557 pre, post, ##__VA_ARGS__)
558
559
560#define ____PVOP_VCALL(op, clbr, call_clbr, extra_clbr, pre, post, ...) \
561 ({ \
562 PVOP_VCALL_ARGS; \
563 PVOP_TEST_NULL(op); \
564 asm volatile(pre \
565 paravirt_alt(PARAVIRT_CALL) \
566 post \
567 : call_clbr, ASM_CALL_CONSTRAINT \
568 : paravirt_type(op), \
569 paravirt_clobber(clbr), \
570 ##__VA_ARGS__ \
571 : "memory", "cc" extra_clbr); \
572 })
573
574#define __PVOP_VCALL(op, pre, post, ...) \
575 ____PVOP_VCALL(op, CLBR_ANY, PVOP_VCALL_CLOBBERS, \
576 VEXTRA_CLOBBERS, \
577 pre, post, ##__VA_ARGS__)
578
579#define __PVOP_VCALLEESAVE(op, pre, post, ...) \
580 ____PVOP_VCALL(op.func, CLBR_RET_REG, \
581 PVOP_VCALLEE_CLOBBERS, , \
582 pre, post, ##__VA_ARGS__)
583
584
585
586#define PVOP_CALL0(rettype, op) \
587 __PVOP_CALL(rettype, op, "", "")
588#define PVOP_VCALL0(op) \
589 __PVOP_VCALL(op, "", "")
590
591#define PVOP_CALLEE0(rettype, op) \
592 __PVOP_CALLEESAVE(rettype, op, "", "")
593#define PVOP_VCALLEE0(op) \
594 __PVOP_VCALLEESAVE(op, "", "")
595
596
597#define PVOP_CALL1(rettype, op, arg1) \
598 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
599#define PVOP_VCALL1(op, arg1) \
600 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1))
601
602#define PVOP_CALLEE1(rettype, op, arg1) \
603 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1))
604#define PVOP_VCALLEE1(op, arg1) \
605 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1))
606
607
608#define PVOP_CALL2(rettype, op, arg1, arg2) \
609 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
610 PVOP_CALL_ARG2(arg2))
611#define PVOP_VCALL2(op, arg1, arg2) \
612 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
613 PVOP_CALL_ARG2(arg2))
614
615#define PVOP_CALLEE2(rettype, op, arg1, arg2) \
616 __PVOP_CALLEESAVE(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
617 PVOP_CALL_ARG2(arg2))
618#define PVOP_VCALLEE2(op, arg1, arg2) \
619 __PVOP_VCALLEESAVE(op, "", "", PVOP_CALL_ARG1(arg1), \
620 PVOP_CALL_ARG2(arg2))
621
622
623#define PVOP_CALL3(rettype, op, arg1, arg2, arg3) \
624 __PVOP_CALL(rettype, op, "", "", PVOP_CALL_ARG1(arg1), \
625 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
626#define PVOP_VCALL3(op, arg1, arg2, arg3) \
627 __PVOP_VCALL(op, "", "", PVOP_CALL_ARG1(arg1), \
628 PVOP_CALL_ARG2(arg2), PVOP_CALL_ARG3(arg3))
629
630
631#ifdef CONFIG_X86_32
632#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
633 __PVOP_CALL(rettype, op, \
634 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
635 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
636 PVOP_CALL_ARG3(arg3), [_arg4] "mr" ((u32)(arg4)))
637#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
638 __PVOP_VCALL(op, \
639 "push %[_arg4];", "lea 4(%%esp),%%esp;", \
640 "0" ((u32)(arg1)), "1" ((u32)(arg2)), \
641 "2" ((u32)(arg3)), [_arg4] "mr" ((u32)(arg4)))
642#else
643#define PVOP_CALL4(rettype, op, arg1, arg2, arg3, arg4) \
644 __PVOP_CALL(rettype, op, "", "", \
645 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
646 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
647#define PVOP_VCALL4(op, arg1, arg2, arg3, arg4) \
648 __PVOP_VCALL(op, "", "", \
649 PVOP_CALL_ARG1(arg1), PVOP_CALL_ARG2(arg2), \
650 PVOP_CALL_ARG3(arg3), PVOP_CALL_ARG4(arg4))
651#endif
652
653
654enum paravirt_lazy_mode {
655 PARAVIRT_LAZY_NONE,
656 PARAVIRT_LAZY_MMU,
657 PARAVIRT_LAZY_CPU,
658};
659
660enum paravirt_lazy_mode paravirt_get_lazy_mode(void);
661void paravirt_start_context_switch(struct task_struct *prev);
662void paravirt_end_context_switch(struct task_struct *next);
663
664void paravirt_enter_lazy_mmu(void);
665void paravirt_leave_lazy_mmu(void);
666void paravirt_flush_lazy_mmu(void);
667
668void _paravirt_nop(void);
669u64 _paravirt_ident_64(u64);
670
671#define paravirt_nop ((void *)_paravirt_nop)
672
673
674struct paravirt_patch_site {
675 u8 *instr;
676 u8 type;
677 u8 len;
678};
679
680extern struct paravirt_patch_site __parainstructions[],
681 __parainstructions_end[];
682
683#endif
684
685#endif
686