LXR linux/arch/x86/xen/xen-asm.S

   1/* SPDX-License-Identifier: GPL-2.0 */
   2/*
   3 * Asm versions of Xen pv-ops, suitable for direct use.
   4 *
   5 * We only bother with direct forms (ie, vcpu in percpu data) of the
   6 * operations here; the indirect forms are better handled in C.
   7 */
   8
   9#include <asm/errno.h>
  10#include <asm/asm-offsets.h>
  11#include <asm/percpu.h>
  12#include <asm/processor-flags.h>
  13#include <asm/segment.h>
  14#include <asm/thread_info.h>
  15#include <asm/asm.h>
  16#include <asm/frame.h>
  17#include <asm/unwind_hints.h>
  18
  19#include <xen/interface/xen.h>
  20
  21#include <linux/init.h>
  22#include <linux/linkage.h>
  23
  24/*
  25 * Enable events.  This clears the event mask and tests the pending
  26 * event status with one and operation.  If there are pending events,
  27 * then enter the hypervisor to get them handled.
  28 */
  29SYM_FUNC_START(xen_irq_enable_direct)
  30        FRAME_BEGIN
  31        /* Unmask events */
  32        movb $0, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
  33
  34        /*
  35         * Preempt here doesn't matter because that will deal with any
  36         * pending interrupts.  The pending check may end up being run
  37         * on the wrong CPU, but that doesn't hurt.
  38         */
  39
  40        /* Test for pending */
  41        testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_pending
  42        jz 1f
  43
  44        call check_events
  451:
  46        FRAME_END
  47        ret
  48SYM_FUNC_END(xen_irq_enable_direct)
  49
  50
  51/*
  52 * Disabling events is simply a matter of making the event mask
  53 * non-zero.
  54 */
  55SYM_FUNC_START(xen_irq_disable_direct)
  56        movb $1, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
  57        ret
  58SYM_FUNC_END(xen_irq_disable_direct)
  59
  60/*
  61 * (xen_)save_fl is used to get the current interrupt enable status.
  62 * Callers expect the status to be in X86_EFLAGS_IF, and other bits
  63 * may be set in the return value.  We take advantage of this by
  64 * making sure that X86_EFLAGS_IF has the right value (and other bits
  65 * in that byte are 0), but other bits in the return value are
  66 * undefined.  We need to toggle the state of the bit, because Xen and
  67 * x86 use opposite senses (mask vs enable).
  68 */
  69SYM_FUNC_START(xen_save_fl_direct)
  70        testb $0xff, PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_mask
  71        setz %ah
  72        addb %ah, %ah
  73        ret
  74SYM_FUNC_END(xen_save_fl_direct)
  75
  76/*
  77 * Force an event check by making a hypercall, but preserve regs
  78 * before making the call.
  79 */
  80SYM_FUNC_START(check_events)
  81        FRAME_BEGIN
  82        push %rax
  83        push %rcx
  84        push %rdx
  85        push %rsi
  86        push %rdi
  87        push %r8
  88        push %r9
  89        push %r10
  90        push %r11
  91        call xen_force_evtchn_callback
  92        pop %r11
  93        pop %r10
  94        pop %r9
  95        pop %r8
  96        pop %rdi
  97        pop %rsi
  98        pop %rdx
  99        pop %rcx
 100        pop %rax
 101        FRAME_END
 102        ret
 103SYM_FUNC_END(check_events)
 104
 105SYM_FUNC_START(xen_read_cr2)
 106        FRAME_BEGIN
 107        _ASM_MOV PER_CPU_VAR(xen_vcpu), %_ASM_AX
 108        _ASM_MOV XEN_vcpu_info_arch_cr2(%_ASM_AX), %_ASM_AX
 109        FRAME_END
 110        ret
 111SYM_FUNC_END(xen_read_cr2);
 112
 113SYM_FUNC_START(xen_read_cr2_direct)
 114        FRAME_BEGIN
 115        _ASM_MOV PER_CPU_VAR(xen_vcpu_info) + XEN_vcpu_info_arch_cr2, %_ASM_AX
 116        FRAME_END
 117        ret
 118SYM_FUNC_END(xen_read_cr2_direct);
 119
 120.macro xen_pv_trap name
 121SYM_CODE_START(xen_\name)
 122        UNWIND_HINT_EMPTY
 123        pop %rcx
 124        pop %r11
 125        jmp  \name
 126SYM_CODE_END(xen_\name)
 127_ASM_NOKPROBE(xen_\name)
 128.endm
 129
 130xen_pv_trap asm_exc_divide_error
 131xen_pv_trap asm_xenpv_exc_debug
 132xen_pv_trap asm_exc_int3
 133xen_pv_trap asm_xenpv_exc_nmi
 134xen_pv_trap asm_exc_overflow
 135xen_pv_trap asm_exc_bounds
 136xen_pv_trap asm_exc_invalid_op
 137xen_pv_trap asm_exc_device_not_available
 138xen_pv_trap asm_xenpv_exc_double_fault
 139xen_pv_trap asm_exc_coproc_segment_overrun
 140xen_pv_trap asm_exc_invalid_tss
 141xen_pv_trap asm_exc_segment_not_present
 142xen_pv_trap asm_exc_stack_segment
 143xen_pv_trap asm_exc_general_protection
 144xen_pv_trap asm_exc_page_fault
 145xen_pv_trap asm_exc_spurious_interrupt_bug
 146xen_pv_trap asm_exc_coprocessor_error
 147xen_pv_trap asm_exc_alignment_check
 148#ifdef CONFIG_X86_MCE
 149xen_pv_trap asm_xenpv_exc_machine_check
 150#endif /* CONFIG_X86_MCE */
 151xen_pv_trap asm_exc_simd_coprocessor_error
 152#ifdef CONFIG_IA32_EMULATION
 153xen_pv_trap entry_INT80_compat
 154#endif
 155xen_pv_trap asm_exc_xen_unknown_trap
 156xen_pv_trap asm_exc_xen_hypervisor_callback
 157
 158        __INIT
 159SYM_CODE_START(xen_early_idt_handler_array)
 160        i = 0
 161        .rept NUM_EXCEPTION_VECTORS
 162        UNWIND_HINT_EMPTY
 163        pop %rcx
 164        pop %r11
 165        jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE
 166        i = i + 1
 167        .fill xen_early_idt_handler_array + i*XEN_EARLY_IDT_HANDLER_SIZE - ., 1, 0xcc
 168        .endr
 169SYM_CODE_END(xen_early_idt_handler_array)
 170        __FINIT
 171
 172hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
 173/*
 174 * Xen64 iret frame:
 175 *
 176 *      ss
 177 *      rsp
 178 *      rflags
 179 *      cs
 180 *      rip             <-- standard iret frame
 181 *
 182 *      flags
 183 *
 184 *      rcx             }
 185 *      r11             }<-- pushed by hypercall page
 186 * rsp->rax             }
 187 */
 188SYM_CODE_START(xen_iret)
 189        UNWIND_HINT_EMPTY
 190        pushq $0
 191        jmp hypercall_iret
 192SYM_CODE_END(xen_iret)
 193
 194/*
 195 * Xen handles syscall callbacks much like ordinary exceptions, which
 196 * means we have:
 197 * - kernel gs
 198 * - kernel rsp
 199 * - an iret-like stack frame on the stack (including rcx and r11):
 200 *      ss
 201 *      rsp
 202 *      rflags
 203 *      cs
 204 *      rip
 205 *      r11
 206 * rsp->rcx
 207 */
 208
 209/* Normal 64-bit system call target */
 210SYM_CODE_START(xen_syscall_target)
 211        UNWIND_HINT_EMPTY
 212        popq %rcx
 213        popq %r11
 214
 215        /*
 216         * Neither Xen nor the kernel really knows what the old SS and
 217         * CS were.  The kernel expects __USER_DS and __USER_CS, so
 218         * report those values even though Xen will guess its own values.
 219         */
 220        movq $__USER_DS, 4*8(%rsp)
 221        movq $__USER_CS, 1*8(%rsp)
 222
 223        jmp entry_SYSCALL_64_after_hwframe
 224SYM_CODE_END(xen_syscall_target)
 225
 226#ifdef CONFIG_IA32_EMULATION
 227
 228/* 32-bit compat syscall target */
 229SYM_CODE_START(xen_syscall32_target)
 230        UNWIND_HINT_EMPTY
 231        popq %rcx
 232        popq %r11
 233
 234        /*
 235         * Neither Xen nor the kernel really knows what the old SS and
 236         * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
 237         * report those values even though Xen will guess its own values.
 238         */
 239        movq $__USER32_DS, 4*8(%rsp)
 240        movq $__USER32_CS, 1*8(%rsp)
 241
 242        jmp entry_SYSCALL_compat_after_hwframe
 243SYM_CODE_END(xen_syscall32_target)
 244
 245/* 32-bit compat sysenter target */
 246SYM_CODE_START(xen_sysenter_target)
 247        UNWIND_HINT_EMPTY
 248        /*
 249         * NB: Xen is polite and clears TF from EFLAGS for us.  This means
 250         * that we don't need to guard against single step exceptions here.
 251         */
 252        popq %rcx
 253        popq %r11
 254
 255        /*
 256         * Neither Xen nor the kernel really knows what the old SS and
 257         * CS were.  The kernel expects __USER32_DS and __USER32_CS, so
 258         * report those values even though Xen will guess its own values.
 259         */
 260        movq $__USER32_DS, 4*8(%rsp)
 261        movq $__USER32_CS, 1*8(%rsp)
 262
 263        jmp entry_SYSENTER_compat_after_hwframe
 264SYM_CODE_END(xen_sysenter_target)
 265
 266#else /* !CONFIG_IA32_EMULATION */
 267
 268SYM_CODE_START(xen_syscall32_target)
 269SYM_CODE_START(xen_sysenter_target)
 270        UNWIND_HINT_EMPTY
 271        lea 16(%rsp), %rsp      /* strip %rcx, %r11 */
 272        mov $-ENOSYS, %rax
 273        pushq $0
 274        jmp hypercall_iret
 275SYM_CODE_END(xen_sysenter_target)
 276SYM_CODE_END(xen_syscall32_target)
 277
 278#endif  /* CONFIG_IA32_EMULATION */
 279