/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * derived from drivers/kvm/kvm_main.c
 *
 * Copyright (C) 2006 Qumranet, Inc.
 * Copyright (C) 2008 Qumranet, Inc.
 * Copyright IBM Corporation, 2008
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
 *
 * Authors:
 *   Avi Kivity   <avi@qumranet.com>
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Amit Shah    <amit.shah@qumranet.com>
 *   Ben-Ami Yassour <benami@il.ibm.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

#include <linux/kvm_host.h>
#include "irq.h"
#include "mmu.h"
#include "i8254.h"
#include "tss.h"
#include "kvm_cache_regs.h"
#include "x86.h"
#include "cpuid.h"

#include <linux/clocksource.h>
#include <linux/interrupt.h>
#include <linux/kvm.h>
#include <linux/fs.h>
#include <linux/vmalloc.h>
#include <linux/module.h>
#include <linux/mman.h>
#include <linux/highmem.h>
#include <linux/iommu.h>
#include <linux/intel-iommu.h>
#include <linux/cpufreq.h>
#include <linux/user-return-notifier.h>
#include <linux/srcu.h>
#include <linux/slab.h>
#include <linux/perf_event.h>
#include <linux/uaccess.h>
#include <linux/hash.h>
#include <linux/pci.h>
#include <linux/timekeeper_internal.h>
#include <linux/pvclock_gtod.h>
#include <trace/events/kvm.h>

#define CREATE_TRACE_POINTS
#include "trace.h"

#include <asm/debugreg.h>
#include <asm/msr.h>
#include <asm/desc.h>
#include <asm/mtrr.h>
#include <asm/mce.h>
#include <asm/i387.h>
#include <asm/fpu-internal.h> /* Ugh! */
#include <asm/xcr.h>
#include <asm/pvclock.h>
#include <asm/div64.h>

#define MAX_IO_MSRS 256
#define KVM_MAX_MCE_BANKS 32
#define KVM_MCE_CAP_SUPPORTED (MCG_CTL_P | MCG_SER_P)

#define emul_to_vcpu(ctxt) \
        container_of(ctxt, struct kvm_vcpu, arch.emulate_ctxt)

/* EFER defaults:
 * - enable syscall per default because its emulated by KVM
 * - enable LME and LMA per default on 64 bit KVM
 */
#ifdef CONFIG_X86_64
static
u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA));
#else
static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
#endif

#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU

static void update_cr8_intercept(struct kvm_vcpu *vcpu);
static void process_nmi(struct kvm_vcpu *vcpu);

struct kvm_x86_ops *kvm_x86_ops;
EXPORT_SYMBOL_GPL(kvm_x86_ops);

static bool ignore_msrs = 0;
module_param(ignore_msrs, bool, S_IRUGO | S_IWUSR);

unsigned int min_timer_period_us = 500;
module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);

bool kvm_has_tsc_control;
EXPORT_SYMBOL_GPL(kvm_has_tsc_control);
u32  kvm_max_guest_tsc_khz;
EXPORT_SYMBOL_GPL(kvm_max_guest_tsc_khz);

/* tsc tolerance in parts per million - default to 1/2 of the NTP threshold */
static u32 tsc_tolerance_ppm = 250;
module_param(tsc_tolerance_ppm, uint, S_IRUGO | S_IWUSR);

static bool backwards_tsc_observed = false;

#define KVM_NR_SHARED_MSRS 16

struct kvm_shared_msrs_global {
        int nr;
        u32 msrs[KVM_NR_SHARED_MSRS];
};

struct kvm_shared_msrs {
        struct user_return_notifier urn;
        bool registered;
        struct kvm_shared_msr_values {
                u64 host;
                u64 curr;
        } values[KVM_NR_SHARED_MSRS];
};

static struct kvm_shared_msrs_global __read_mostly shared_msrs_global;
static struct kvm_shared_msrs __percpu *shared_msrs;

struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "pf_fixed", VCPU_STAT(pf_fixed) },
        { "pf_guest", VCPU_STAT(pf_guest) },
        { "tlb_flush", VCPU_STAT(tlb_flush) },
        { "invlpg", VCPU_STAT(invlpg) },
        { "exits", VCPU_STAT(exits) },
        { "io_exits", VCPU_STAT(io_exits) },
        { "mmio_exits", VCPU_STAT(mmio_exits) },
        { "signal_exits", VCPU_STAT(signal_exits) },
        { "irq_window", VCPU_STAT(irq_window_exits) },
        { "nmi_window", VCPU_STAT(nmi_window_exits) },
        { "halt_exits", VCPU_STAT(halt_exits) },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "hypercalls", VCPU_STAT(hypercalls) },
        { "request_irq", VCPU_STAT(request_irq_exits) },
        { "irq_exits", VCPU_STAT(irq_exits) },
        { "host_state_reload", VCPU_STAT(host_state_reload) },
        { "efer_reload", VCPU_STAT(efer_reload) },
        { "fpu_reload", VCPU_STAT(fpu_reload) },
        { "insn_emulation", VCPU_STAT(insn_emulation) },
        { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) },
        { "irq_injections", VCPU_STAT(irq_injections) },
        { "nmi_injections", VCPU_STAT(nmi_injections) },
        { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) },
        { "mmu_pte_write", VM_STAT(mmu_pte_write) },
        { "mmu_pte_updated", VM_STAT(mmu_pte_updated) },
        { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) },
        { "mmu_flooded", VM_STAT(mmu_flooded) },
        { "mmu_recycled", VM_STAT(mmu_recycled) },
        { "mmu_cache_miss", VM_STAT(mmu_cache_miss) },
        { "mmu_unsync", VM_STAT(mmu_unsync) },
        { "remote_tlb_flush", VM_STAT(remote_tlb_flush) },
        { "largepages", VM_STAT(lpages) },
        { NULL }
};

u64 __read_mostly host_xcr0;

static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt);

static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu)
{
        int i;
        for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++)
                vcpu->arch.apf.gfns[i] = ~0;
}

static void kvm_on_user_return(struct user_return_notifier *urn)
{
        unsigned slot;
        struct kvm_shared_msrs *locals
                = container_of(urn, struct kvm_shared_msrs, urn);
        struct kvm_shared_msr_values *values;

        for (slot = 0; slot < shared_msrs_global.nr; ++slot) {
                values = &locals->values[slot];
                if (values->host != values->curr) {
                        wrmsrl(shared_msrs_global.msrs[slot], values->host);
                        values->curr = values->host;
                }
        }
        locals->registered = false;
        user_return_notifier_unregister(urn);
}

static void shared_msr_update(unsigned slot, u32 msr)
{
        u64 value;
        unsigned int cpu = smp_processor_id();
        struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);

        /* only read, and nobody should modify it at this time,
         * so don't need lock */
        if (slot >= shared_msrs_global.nr) {
                printk(KERN_ERR "kvm: invalid MSR slot!");
                return;
        }
        rdmsrl_safe(msr, &value);
        smsr->values[slot].host = value;
        smsr->values[slot].curr = value;
}

void kvm_define_shared_msr(unsigned slot, u32 msr)
{
        if (slot >= shared_msrs_global.nr)
                shared_msrs_global.nr = slot + 1;
        shared_msrs_global.msrs[slot] = msr;
        /* we need ensured the shared_msr_global have been updated */
        smp_wmb();
}
EXPORT_SYMBOL_GPL(kvm_define_shared_msr);

static void kvm_shared_msr_cpu_online(void)
{
        unsigned i;

        for (i = 0; i < shared_msrs_global.nr; ++i)
                shared_msr_update(i, shared_msrs_global.msrs[i]);
}

void kvm_set_shared_msr(unsigned slot, u64 value, u64 mask)
{
        unsigned int cpu = smp_processor_id();
        struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);

        if (((value ^ smsr->values[slot].curr) & mask) == 0)
                return;
        smsr->values[slot].curr = value;
        wrmsrl(shared_msrs_global.msrs[slot], value);
        if (!smsr->registered) {
                smsr->urn.on_user_return = kvm_on_user_return;
                user_return_notifier_register(&smsr->urn);
                smsr->registered = true;
        }
}
EXPORT_SYMBOL_GPL(kvm_set_shared_msr);

static void drop_user_return_notifiers(void *ignore)
{
        unsigned int cpu = smp_processor_id();
        struct kvm_shared_msrs *smsr = per_cpu_ptr(shared_msrs, cpu);

        if (smsr->registered)
                kvm_on_user_return(&smsr->urn);
}

u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
{
        return vcpu->arch.apic_base;
}
EXPORT_SYMBOL_GPL(kvm_get_apic_base);

int kvm_set_apic_base(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
        u64 old_state = vcpu->arch.apic_base &
                (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
        u64 new_state = msr_info->data &
                (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE);
        u64 reserved_bits = ((~0ULL) << cpuid_maxphyaddr(vcpu)) |
                0x2ff | (guest_cpuid_has_x2apic(vcpu) ? 0 : X2APIC_ENABLE);

        if (!msr_info->host_initiated &&
            ((msr_info->data & reserved_bits) != 0 ||
             new_state == X2APIC_ENABLE ||
             (new_state == MSR_IA32_APICBASE_ENABLE &&
              old_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE)) ||
             (new_state == (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE) &&
              old_state == 0)))
                return 1;

        kvm_lapic_set_base(vcpu, msr_info->data);
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_apic_base);

asmlinkage __visible void kvm_spurious_fault(void)
{
        /* Fault while not rebooting.  We want the trace. */
        BUG();
}
EXPORT_SYMBOL_GPL(kvm_spurious_fault);

#define EXCPT_BENIGN            0
#define EXCPT_CONTRIBUTORY      1
#define EXCPT_PF                2

static int exception_class(int vector)
{
        switch (vector) {
        case PF_VECTOR:
                return EXCPT_PF;
        case DE_VECTOR:
        case TS_VECTOR:
        case NP_VECTOR:
        case SS_VECTOR:
        case GP_VECTOR:
                return EXCPT_CONTRIBUTORY;
        default:
                break;
        }
        return EXCPT_BENIGN;
}

static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
                unsigned nr, bool has_error, u32 error_code,
                bool reinject)
{
        u32 prev_nr;
        int class1, class2;

        kvm_make_request(KVM_REQ_EVENT, vcpu);

        if (!vcpu->arch.exception.pending) {
        queue:
                vcpu->arch.exception.pending = true;
                vcpu->arch.exception.has_error_code = has_error;
                vcpu->arch.exception.nr = nr;
                vcpu->arch.exception.error_code = error_code;
                vcpu->arch.exception.reinject = reinject;
                return;
        }

        /* to check exception */
        prev_nr = vcpu->arch.exception.nr;
        if (prev_nr == DF_VECTOR) {
                /* triple fault -> shutdown */
                kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
                return;
        }
        class1 = exception_class(prev_nr);
        class2 = exception_class(nr);
        if ((class1 == EXCPT_CONTRIBUTORY && class2 == EXCPT_CONTRIBUTORY)
                || (class1 == EXCPT_PF && class2 != EXCPT_BENIGN)) {
                /* generate double fault per SDM Table 5-5 */
                vcpu->arch.exception.pending = true;
                vcpu->arch.exception.has_error_code = true;
                vcpu->arch.exception.nr = DF_VECTOR;
                vcpu->arch.exception.error_code = 0;
        } else
                /* replace previous exception with a new one in a hope
                   that instruction re-execution will regenerate lost
                   exception */
                goto queue;
}

void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
        kvm_multiple_exception(vcpu, nr, false, 0, false);
}
EXPORT_SYMBOL_GPL(kvm_queue_exception);

void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
{
        kvm_multiple_exception(vcpu, nr, false, 0, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception);

void kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
{
        if (err)
                kvm_inject_gp(vcpu, 0);
        else
                kvm_x86_ops->skip_emulated_instruction(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_complete_insn_gp);

void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
{
        ++vcpu->stat.pf_guest;
        vcpu->arch.cr2 = fault->address;
        kvm_queue_exception_e(vcpu, PF_VECTOR, fault->error_code);
}
EXPORT_SYMBOL_GPL(kvm_inject_page_fault);

void kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
{
        if (mmu_is_nested(vcpu) && !fault->nested_page_fault)
                vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault);
        else
                vcpu->arch.mmu.inject_page_fault(vcpu, fault);
}

void kvm_inject_nmi(struct kvm_vcpu *vcpu)
{
        atomic_inc(&vcpu->arch.nmi_queued);
        kvm_make_request(KVM_REQ_NMI, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_inject_nmi);

void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
        kvm_multiple_exception(vcpu, nr, true, error_code, false);
}
EXPORT_SYMBOL_GPL(kvm_queue_exception_e);

void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
{
        kvm_multiple_exception(vcpu, nr, true, error_code, true);
}
EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);

/*
 * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
 * a #GP and return false.
 */
bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl)
{
        if (kvm_x86_ops->get_cpl(vcpu) <= required_cpl)
                return true;
        kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
        return false;
}
EXPORT_SYMBOL_GPL(kvm_require_cpl);

/*
 * This function will be used to read from the physical memory of the currently
 * running guest. The difference to kvm_read_guest_page is that this function
 * can read from guest physical or from the guest's guest physical memory.
 */
int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
                            gfn_t ngfn, void *data, int offset, int len,
                            u32 access)
{
        gfn_t real_gfn;
        gpa_t ngpa;

        ngpa     = gfn_to_gpa(ngfn);
        real_gfn = mmu->translate_gpa(vcpu, ngpa, access);
        if (real_gfn == UNMAPPED_GVA)
                return -EFAULT;

        real_gfn = gpa_to_gfn(real_gfn);

        return kvm_read_guest_page(vcpu->kvm, real_gfn, data, offset, len);
}
EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu);

int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
                               void *data, int offset, int len, u32 access)
{
        return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn,
                                       data, offset, len, access);
}

/*
 * Load the pae pdptrs.  Return true is they are all valid.
 */
int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3)
{
        gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
        unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
        int i;
        int ret;
        u64 pdpte[ARRAY_SIZE(mmu->pdptrs)];

        ret = kvm_read_guest_page_mmu(vcpu, mmu, pdpt_gfn, pdpte,
                                      offset * sizeof(u64), sizeof(pdpte),
                                      PFERR_USER_MASK|PFERR_WRITE_MASK);
        if (ret < 0) {
                ret = 0;
                goto out;
        }
        for (i = 0; i < ARRAY_SIZE(pdpte); ++i) {
                if (is_present_gpte(pdpte[i]) &&
                    (pdpte[i] & vcpu->arch.mmu.rsvd_bits_mask[0][2])) {
                        ret = 0;
                        goto out;
                }
        }
        ret = 1;

        memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
        __set_bit(VCPU_EXREG_PDPTR,
                  (unsigned long *)&vcpu->arch.regs_avail);
        __set_bit(VCPU_EXREG_PDPTR,
                  (unsigned long *)&vcpu->arch.regs_dirty);
out:

        return ret;
}
EXPORT_SYMBOL_GPL(load_pdptrs);

static bool pdptrs_changed(struct kvm_vcpu *vcpu)
{
        u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)];
        bool changed = true;
        int offset;
        gfn_t gfn;
        int r;

        if (is_long_mode(vcpu) || !is_pae(vcpu))
                return false;

        if (!test_bit(VCPU_EXREG_PDPTR,
                      (unsigned long *)&vcpu->arch.regs_avail))
                return true;

        gfn = (kvm_read_cr3(vcpu) & ~31u) >> PAGE_SHIFT;
        offset = (kvm_read_cr3(vcpu) & ~31u) & (PAGE_SIZE - 1);
        r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte),
                                       PFERR_USER_MASK | PFERR_WRITE_MASK);
        if (r < 0)
                goto out;
        changed = memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0;
out:

        return changed;
}

int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
{
        unsigned long old_cr0 = kvm_read_cr0(vcpu);
        unsigned long update_bits = X86_CR0_PG | X86_CR0_WP |
                                    X86_CR0_CD | X86_CR0_NW;

        cr0 |= X86_CR0_ET;

#ifdef CONFIG_X86_64
        if (cr0 & 0xffffffff00000000UL)
                return 1;
#endif

        cr0 &= ~CR0_RESERVED_BITS;

        if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD))
                return 1;

        if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE))
                return 1;

        if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
#ifdef CONFIG_X86_64
                if ((vcpu->arch.efer & EFER_LME)) {
                        int cs_db, cs_l;

                        if (!is_pae(vcpu))
                                return 1;
                        kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
                        if (cs_l)
                                return 1;
                } else
#endif
                if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
                                                 kvm_read_cr3(vcpu)))
                        return 1;
        }

        if (!(cr0 & X86_CR0_PG) && kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE))
                return 1;

        kvm_x86_ops->set_cr0(vcpu, cr0);

        if ((cr0 ^ old_cr0) & X86_CR0_PG) {
                kvm_clear_async_pf_completion_queue(vcpu);
                kvm_async_pf_hash_reset(vcpu);
        }

        if ((cr0 ^ old_cr0) & update_bits)
                kvm_mmu_reset_context(vcpu);
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_cr0);

void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
{
        (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f));
}
EXPORT_SYMBOL_GPL(kvm_lmsw);

static void kvm_load_guest_xcr0(struct kvm_vcpu *vcpu)
{
        if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE) &&
                        !vcpu->guest_xcr0_loaded) {
                /* kvm_set_xcr() also depends on this */
                xsetbv(XCR_XFEATURE_ENABLED_MASK, vcpu->arch.xcr0);
                vcpu->guest_xcr0_loaded = 1;
        }
}

static void kvm_put_guest_xcr0(struct kvm_vcpu *vcpu)
{
        if (vcpu->guest_xcr0_loaded) {
                if (vcpu->arch.xcr0 != host_xcr0)
                        xsetbv(XCR_XFEATURE_ENABLED_MASK, host_xcr0);
                vcpu->guest_xcr0_loaded = 0;
        }
}

int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
        u64 xcr0 = xcr;
        u64 old_xcr0 = vcpu->arch.xcr0;
        u64 valid_bits;

        /* Only support XCR_XFEATURE_ENABLED_MASK(xcr0) now  */
        if (index != XCR_XFEATURE_ENABLED_MASK)
                return 1;
        if (!(xcr0 & XSTATE_FP))
                return 1;
        if ((xcr0 & XSTATE_YMM) && !(xcr0 & XSTATE_SSE))
                return 1;

        /*
         * Do not allow the guest to set bits that we do not support
         * saving.  However, xcr0 bit 0 is always set, even if the
         * emulated CPU does not support XSAVE (see fx_init).
         */
        valid_bits = vcpu->arch.guest_supported_xcr0 | XSTATE_FP;
        if (xcr0 & ~valid_bits)
                return 1;

        if ((!(xcr0 & XSTATE_BNDREGS)) != (!(xcr0 & XSTATE_BNDCSR)))
                return 1;

        kvm_put_guest_xcr0(vcpu);
        vcpu->arch.xcr0 = xcr0;

        if ((xcr0 ^ old_xcr0) & XSTATE_EXTEND_MASK)
                kvm_update_cpuid(vcpu);
        return 0;
}

int kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr)
{
        if (kvm_x86_ops->get_cpl(vcpu) != 0 ||
            __kvm_set_xcr(vcpu, index, xcr)) {
                kvm_inject_gp(vcpu, 0);
                return 1;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_xcr);

int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
{
        unsigned long old_cr4 = kvm_read_cr4(vcpu);
        unsigned long pdptr_bits = X86_CR4_PGE | X86_CR4_PSE |
                                   X86_CR4_PAE | X86_CR4_SMEP;
        if (cr4 & CR4_RESERVED_BITS)
                return 1;

        if (!guest_cpuid_has_xsave(vcpu) && (cr4 & X86_CR4_OSXSAVE))
                return 1;

        if (!guest_cpuid_has_smep(vcpu) && (cr4 & X86_CR4_SMEP))
                return 1;

        if (!guest_cpuid_has_smap(vcpu) && (cr4 & X86_CR4_SMAP))
                return 1;

        if (!guest_cpuid_has_fsgsbase(vcpu) && (cr4 & X86_CR4_FSGSBASE))
                return 1;

        if (is_long_mode(vcpu)) {
                if (!(cr4 & X86_CR4_PAE))
                        return 1;
        } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE)
                   && ((cr4 ^ old_cr4) & pdptr_bits)
                   && !load_pdptrs(vcpu, vcpu->arch.walk_mmu,
                                   kvm_read_cr3(vcpu)))
                return 1;

        if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) {
                if (!guest_cpuid_has_pcid(vcpu))
                        return 1;

                /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */
                if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu))
                        return 1;
        }

        if (kvm_x86_ops->set_cr4(vcpu, cr4))
                return 1;

        if (((cr4 ^ old_cr4) & pdptr_bits) ||
            (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE)))
                kvm_mmu_reset_context(vcpu);

        if ((cr4 ^ old_cr4) & X86_CR4_SMAP)
                update_permission_bitmask(vcpu, vcpu->arch.walk_mmu, false);

        if ((cr4 ^ old_cr4) & X86_CR4_OSXSAVE)
                kvm_update_cpuid(vcpu);

        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_cr4);

int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
{
        if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) {
                kvm_mmu_sync_roots(vcpu);
                kvm_mmu_flush_tlb(vcpu);
                return 0;
        }

        if (is_long_mode(vcpu)) {
                if (kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE)) {
                        if (cr3 & CR3_PCID_ENABLED_RESERVED_BITS)
                                return 1;
                } else
                        if (cr3 & CR3_L_MODE_RESERVED_BITS)
                                return 1;
        } else {
                if (is_pae(vcpu)) {
                        if (cr3 & CR3_PAE_RESERVED_BITS)
                                return 1;
                        if (is_paging(vcpu) &&
                            !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))
                                return 1;
                }
                /*
                 * We don't check reserved bits in nonpae mode, because
                 * this isn't enforced, and VMware depends on this.
                 */
        }

        vcpu->arch.cr3 = cr3;
        __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
        kvm_mmu_new_cr3(vcpu);
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_cr3);

int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
{
        if (cr8 & CR8_RESERVED_BITS)
                return 1;
        if (irqchip_in_kernel(vcpu->kvm))
                kvm_lapic_set_tpr(vcpu, cr8);
        else
                vcpu->arch.cr8 = cr8;
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_cr8);

unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
{
        if (irqchip_in_kernel(vcpu->kvm))
                return kvm_lapic_get_cr8(vcpu);
        else
                return vcpu->arch.cr8;
}
EXPORT_SYMBOL_GPL(kvm_get_cr8);

static void kvm_update_dr6(struct kvm_vcpu *vcpu)
{
        if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
                kvm_x86_ops->set_dr6(vcpu, vcpu->arch.dr6);
}

static void kvm_update_dr7(struct kvm_vcpu *vcpu)
{
        unsigned long dr7;

        if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
                dr7 = vcpu->arch.guest_debug_dr7;
        else
                dr7 = vcpu->arch.dr7;
        kvm_x86_ops->set_dr7(vcpu, dr7);
        vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
        if (dr7 & DR7_BP_EN_MASK)
                vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
}

static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
        switch (dr) {
        case 0 ... 3:
                vcpu->arch.db[dr] = val;
                if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
                        vcpu->arch.eff_db[dr] = val;
                break;
        case 4:
                if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
                        return 1; /* #UD */
                /* fall through */
        case 6:
                if (val & 0xffffffff00000000ULL)
                        return -1; /* #GP */
                vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1;
                kvm_update_dr6(vcpu);
                break;
        case 5:
                if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
                        return 1; /* #UD */
                /* fall through */
        default: /* 7 */
                if (val & 0xffffffff00000000ULL)
                        return -1; /* #GP */
                vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
                kvm_update_dr7(vcpu);
                break;
        }

        return 0;
}

int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
{
        int res;

        res = __kvm_set_dr(vcpu, dr, val);
        if (res > 0)
                kvm_queue_exception(vcpu, UD_VECTOR);
        else if (res < 0)
                kvm_inject_gp(vcpu, 0);

        return res;
}
EXPORT_SYMBOL_GPL(kvm_set_dr);

static int _kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
{
        switch (dr) {
        case 0 ... 3:
                *val = vcpu->arch.db[dr];
                break;
        case 4:
                if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
                        return 1;
                /* fall through */
        case 6:
                if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
                        *val = vcpu->arch.dr6;
                else
                        *val = kvm_x86_ops->get_dr6(vcpu);
                break;
        case 5:
                if (kvm_read_cr4_bits(vcpu, X86_CR4_DE))
                        return 1;
                /* fall through */
        default: /* 7 */
                *val = vcpu->arch.dr7;
                break;
        }

        return 0;
}

int kvm_get_dr(struct kvm_vcpu *vcpu, int dr, unsigned long *val)
{
        if (_kvm_get_dr(vcpu, dr, val)) {
                kvm_queue_exception(vcpu, UD_VECTOR);
                return 1;
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_dr);

bool kvm_rdpmc(struct kvm_vcpu *vcpu)
{
        u32 ecx = kvm_register_read(vcpu, VCPU_REGS_RCX);
        u64 data;
        int err;

        err = kvm_pmu_read_pmc(vcpu, ecx, &data);
        if (err)
                return err;
        kvm_register_write(vcpu, VCPU_REGS_RAX, (u32)data);
        kvm_register_write(vcpu, VCPU_REGS_RDX, data >> 32);
        return err;
}
EXPORT_SYMBOL_GPL(kvm_rdpmc);

/*
 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
 *
 * This list is modified at module load time to reflect the
 * capabilities of the host cpu. This capabilities test skips MSRs that are
 * kvm-specific. Those are put in the beginning of the list.
 */

#define KVM_SAVE_MSRS_BEGIN     12
static u32 msrs_to_save[] = {
        MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK,
        MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW,
        HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL,
        HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC,
        HV_X64_MSR_APIC_ASSIST_PAGE, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME,
        MSR_KVM_PV_EOI_EN,
        MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
        MSR_STAR,
#ifdef CONFIG_X86_64
        MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
#endif
        MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA,
        MSR_IA32_FEATURE_CONTROL, MSR_IA32_BNDCFGS
};

static unsigned num_msrs_to_save;

static const u32 emulated_msrs[] = {
        MSR_IA32_TSC_ADJUST,
        MSR_IA32_TSCDEADLINE,
        MSR_IA32_MISC_ENABLE,
        MSR_IA32_MCG_STATUS,
        MSR_IA32_MCG_CTL,
};

bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer)
{
        if (efer & efer_reserved_bits)
                return false;

        if (efer & EFER_FFXSR) {
                struct kvm_cpuid_entry2 *feat;

                feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
                if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT)))
                        return false;
        }

        if (efer & EFER_SVME) {
                struct kvm_cpuid_entry2 *feat;

                feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0);
                if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM)))
                        return false;
        }

        return true;
}
EXPORT_SYMBOL_GPL(kvm_valid_efer);

static int set_efer(struct kvm_vcpu *vcpu, u64 efer)
{
        u64 old_efer = vcpu->arch.efer;

        if (!kvm_valid_efer(vcpu, efer))
                return 1;

        if (is_paging(vcpu)
            && (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME))
                return 1;

        efer &= ~EFER_LMA;
        efer |= vcpu->arch.efer & EFER_LMA;

        kvm_x86_ops->set_efer(vcpu, efer);

        /* Update reserved bits */
        if ((efer ^ old_efer) & EFER_NX)
                kvm_mmu_reset_context(vcpu);

        return 0;
}

void kvm_enable_efer_bits(u64 mask)
{
       efer_reserved_bits &= ~mask;
}
EXPORT_SYMBOL_GPL(kvm_enable_efer_bits);


/*
 * Writes msr value into into the appropriate "register".
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
int kvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
{
        return kvm_x86_ops->set_msr(vcpu, msr);
}

/*
 * Adapt set_msr() to msr_io()'s calling convention
 */
static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
{
        struct msr_data msr;

        msr.data = *data;
        msr.index = index;
        msr.host_initiated = true;
        return kvm_set_msr(vcpu, &msr);
}

#ifdef CONFIG_X86_64
struct pvclock_gtod_data {
        seqcount_t      seq;

        struct { /* extract of a clocksource struct */
                int vclock_mode;
                cycle_t cycle_last;
                cycle_t mask;
                u32     mult;
                u32     shift;
        } clock;

        /* open coded 'struct timespec' */
        u64             monotonic_time_snsec;
        time_t          monotonic_time_sec;
};

static struct pvclock_gtod_data pvclock_gtod_data;

static void update_pvclock_gtod(struct timekeeper *tk)
{
        struct pvclock_gtod_data *vdata = &pvclock_gtod_data;

        write_seqcount_begin(&vdata->seq);

        /* copy pvclock gtod data */
        vdata->clock.vclock_mode        = tk->clock->archdata.vclock_mode;
        vdata->clock.cycle_last         = tk->clock->cycle_last;
        vdata->clock.mask               = tk->clock->mask;
        vdata->clock.mult               = tk->mult;
        vdata->clock.shift              = tk->shift;

        vdata->monotonic_time_sec       = tk->xtime_sec
                                        + tk->wall_to_monotonic.tv_sec;
        vdata->monotonic_time_snsec     = tk->xtime_nsec
                                        + (tk->wall_to_monotonic.tv_nsec
                                                << tk->shift);
        while (vdata->monotonic_time_snsec >=
                                        (((u64)NSEC_PER_SEC) << tk->shift)) {
                vdata->monotonic_time_snsec -=
                                        ((u64)NSEC_PER_SEC) << tk->shift;
                vdata->monotonic_time_sec++;
        }

        write_seqcount_end(&vdata->seq);
}
#endif


static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock)
{
        int version;
        int r;
        struct pvclock_wall_clock wc;
        struct timespec boot;

        if (!wall_clock)
                return;

        r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version));
        if (r)
                return;

        if (version & 1)
                ++version;  /* first time write, random junk */

        ++version;

        kvm_write_guest(kvm, wall_clock, &version, sizeof(version));

        /*
         * The guest calculates current wall clock time by adding
         * system time (updated by kvm_guest_time_update below) to the
         * wall clock specified here.  guest system time equals host
         * system time for us, thus we must fill in host boot time here.
         */
        getboottime(&boot);

        if (kvm->arch.kvmclock_offset) {
                struct timespec ts = ns_to_timespec(kvm->arch.kvmclock_offset);
                boot = timespec_sub(boot, ts);
        }
        wc.sec = boot.tv_sec;
        wc.nsec = boot.tv_nsec;
        wc.version = version;

        kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc));

        version++;
        kvm_write_guest(kvm, wall_clock, &version, sizeof(version));
}

static uint32_t div_frac(uint32_t dividend, uint32_t divisor)
{
        uint32_t quotient, remainder;

        /* Don't try to replace with do_div(), this one calculates
         * "(dividend << 32) / divisor" */
        __asm__ ( "divl %4"
                  : "=a" (quotient), "=d" (remainder)
                  : "0" (0), "1" (dividend), "r" (divisor) );
        return quotient;
}

static void kvm_get_time_scale(uint32_t scaled_khz, uint32_t base_khz,
                               s8 *pshift, u32 *pmultiplier)
{
        uint64_t scaled64;
        int32_t  shift = 0;
        uint64_t tps64;
        uint32_t tps32;

        tps64 = base_khz * 1000LL;
        scaled64 = scaled_khz * 1000LL;
        while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) {
                tps64 >>= 1;
                shift--;
        }

        tps32 = (uint32_t)tps64;
        while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) {
                if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000)
                        scaled64 >>= 1;
                else
                        tps32 <<= 1;
                shift++;
        }

        *pshift = shift;
        *pmultiplier = div_frac(scaled64, tps32);

        pr_debug("%s: base_khz %u => %u, shift %d, mul %u\n",
                 __func__, base_khz, scaled_khz, shift, *pmultiplier);
}

static inline u64 get_kernel_ns(void)
{
        struct timespec ts;

        ktime_get_ts(&ts);
        monotonic_to_bootbased(&ts);
        return timespec_to_ns(&ts);
}

#ifdef CONFIG_X86_64
static atomic_t kvm_guest_has_master_clock = ATOMIC_INIT(0);
#endif

static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
unsigned long max_tsc_khz;

static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec)
{
        return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult,
                                   vcpu->arch.virtual_tsc_shift);
}

static u32 adjust_tsc_khz(u32 khz, s32 ppm)
{
        u64 v = (u64)khz * (1000000 + ppm);
        do_div(v, 1000000);
        return v;
}

static void kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 this_tsc_khz)
{
        u32 thresh_lo, thresh_hi;
        int use_scaling = 0;

        /* tsc_khz can be zero if TSC calibration fails */
        if (this_tsc_khz == 0)
                return;

        /* Compute a scale to convert nanoseconds in TSC cycles */
        kvm_get_time_scale(this_tsc_khz, NSEC_PER_SEC / 1000,
                           &vcpu->arch.virtual_tsc_shift,
                           &vcpu->arch.virtual_tsc_mult);
        vcpu->arch.virtual_tsc_khz = this_tsc_khz;

        /*
         * Compute the variation in TSC rate which is acceptable
         * within the range of tolerance and decide if the
         * rate being applied is within that bounds of the hardware
         * rate.  If so, no scaling or compensation need be done.
         */
        thresh_lo = adjust_tsc_khz(tsc_khz, -tsc_tolerance_ppm);
        thresh_hi = adjust_tsc_khz(tsc_khz, tsc_tolerance_ppm);
        if (this_tsc_khz < thresh_lo || this_tsc_khz > thresh_hi) {
                pr_debug("kvm: requested TSC rate %u falls outside tolerance [%u,%u]\n", this_tsc_khz, thresh_lo, thresh_hi);
                use_scaling = 1;
        }
        kvm_x86_ops->set_tsc_khz(vcpu, this_tsc_khz, use_scaling);
}

static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns)
{
        u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec,
                                      vcpu->arch.virtual_tsc_mult,
                                      vcpu->arch.virtual_tsc_shift);
        tsc += vcpu->arch.this_tsc_write;
        return tsc;
}

void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
{
#ifdef CONFIG_X86_64
        bool vcpus_matched;
        bool do_request = false;
        struct kvm_arch *ka = &vcpu->kvm->arch;
        struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

        vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
                         atomic_read(&vcpu->kvm->online_vcpus));

        if (vcpus_matched && gtod->clock.vclock_mode == VCLOCK_TSC)
                if (!ka->use_master_clock)
                        do_request = 1;

        if (!vcpus_matched && ka->use_master_clock)
                        do_request = 1;

        if (do_request)
                kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu);

        trace_kvm_track_tsc(vcpu->vcpu_id, ka->nr_vcpus_matched_tsc,
                            atomic_read(&vcpu->kvm->online_vcpus),
                            ka->use_master_clock, gtod->clock.vclock_mode);
#endif
}

static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
{
        u64 curr_offset = kvm_x86_ops->read_tsc_offset(vcpu);
        vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
}

void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
{
        struct kvm *kvm = vcpu->kvm;
        u64 offset, ns, elapsed;
        unsigned long flags;
        s64 usdiff;
        bool matched;
        u64 data = msr->data;

        raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
        offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
        ns = get_kernel_ns();
        elapsed = ns - kvm->arch.last_tsc_nsec;

        if (vcpu->arch.virtual_tsc_khz) {
                int faulted = 0;

                /* n.b - signed multiplication and division required */
                usdiff = data - kvm->arch.last_tsc_write;
#ifdef CONFIG_X86_64
                usdiff = (usdiff * 1000) / vcpu->arch.virtual_tsc_khz;
#else
                /* do_div() only does unsigned */
                asm("1: idivl %[divisor]\n"
                    "2: xor %%edx, %%edx\n"
                    "   movl $0, %[faulted]\n"
                    "3:\n"
                    ".section .fixup,\"ax\"\n"
                    "4: movl $1, %[faulted]\n"
                    "   jmp  3b\n"
                    ".previous\n"

                _ASM_EXTABLE(1b, 4b)

                : "=A"(usdiff), [faulted] "=r" (faulted)
                : "A"(usdiff * 1000), [divisor] "rm"(vcpu->arch.virtual_tsc_khz));

#endif
                do_div(elapsed, 1000);
                usdiff -= elapsed;
                if (usdiff < 0)
                        usdiff = -usdiff;

                /* idivl overflow => difference is larger than USEC_PER_SEC */
                if (faulted)
                        usdiff = USEC_PER_SEC;
        } else
                usdiff = USEC_PER_SEC; /* disable TSC match window below */

        /*
         * Special case: TSC write with a small delta (1 second) of virtual
         * cycle time against real time is interpreted as an attempt to
         * synchronize the CPU.
         *
         * For a reliable TSC, we can match TSC offsets, and for an unstable
         * TSC, we add elapsed time in this computation.  We could let the
         * compensation code attempt to catch up if we fall behind, but
         * it's better to try to match offsets from the beginning.
         */
        if (usdiff < USEC_PER_SEC &&
            vcpu->arch.virtual_tsc_khz == kvm->arch.last_tsc_khz) {
                if (!check_tsc_unstable()) {
                        offset = kvm->arch.cur_tsc_offset;
                        pr_debug("kvm: matched tsc offset for %llu\n", data);
                } else {
                        u64 delta = nsec_to_cycles(vcpu, elapsed);
                        data += delta;
                        offset = kvm_x86_ops->compute_tsc_offset(vcpu, data);
                        pr_debug("kvm: adjusted tsc offset by %llu\n", delta);
                }
                matched = true;
        } else {
                /*
                 * We split periods of matched TSC writes into generations.
                 * For each generation, we track the original measured
                 * nanosecond time, offset, and write, so if TSCs are in
                 * sync, we can match exact offset, and if not, we can match
                 * exact software computation in compute_guest_tsc()
                 *
                 * These values are tracked in kvm->arch.cur_xxx variables.
                 */
                kvm->arch.cur_tsc_generation++;
                kvm->arch.cur_tsc_nsec = ns;
                kvm->arch.cur_tsc_write = data;
                kvm->arch.cur_tsc_offset = offset;
                matched = false;
                pr_debug("kvm: new tsc generation %u, clock %llu\n",
                         kvm->arch.cur_tsc_generation, data);
        }

        /*
         * We also track th most recent recorded KHZ, write and time to
         * allow the matching interval to be extended at each write.
         */
        kvm->arch.last_tsc_nsec = ns;
        kvm->arch.last_tsc_write = data;
        kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz;

        vcpu->arch.last_guest_tsc = data;

        /* Keep track of which generation this VCPU has synchronized to */
        vcpu->arch.this_tsc_generation = kvm->arch.cur_tsc_generation;
        vcpu->arch.this_tsc_nsec = kvm->arch.cur_tsc_nsec;
        vcpu->arch.this_tsc_write = kvm->arch.cur_tsc_write;

        if (guest_cpuid_has_tsc_adjust(vcpu) && !msr->host_initiated)
                update_ia32_tsc_adjust_msr(vcpu, offset);
        kvm_x86_ops->write_tsc_offset(vcpu, offset);
        raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);

        spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
        if (matched)
                kvm->arch.nr_vcpus_matched_tsc++;
        else
                kvm->arch.nr_vcpus_matched_tsc = 0;

        kvm_track_tsc_matching(vcpu);
        spin_unlock(&kvm->arch.pvclock_gtod_sync_lock);
}

EXPORT_SYMBOL_GPL(kvm_write_tsc);

#ifdef CONFIG_X86_64

static cycle_t read_tsc(void)
{
        cycle_t ret;
        u64 last;

        /*
         * Empirically, a fence (of type that depends on the CPU)
         * before rdtsc is enough to ensure that rdtsc is ordered
         * with respect to loads.  The various CPU manuals are unclear
         * as to whether rdtsc can be reordered with later loads,
         * but no one has ever seen it happen.
         */
        rdtsc_barrier();
        ret = (cycle_t)vget_cycles();

        last = pvclock_gtod_data.clock.cycle_last;

        if (likely(ret >= last))
                return ret;

        /*
         * GCC likes to generate cmov here, but this branch is extremely
         * predictable (it's just a funciton of time and the likely is
         * very likely) and there's a data dependence, so force GCC
         * to generate a branch instead.  I don't barrier() because
         * we don't actually need a barrier, and if this function
         * ever gets inlined it will generate worse code.
         */
        asm volatile ("");
        return last;
}

static inline u64 vgettsc(cycle_t *cycle_now)
{
        long v;
        struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

        *cycle_now = read_tsc();

        v = (*cycle_now - gtod->clock.cycle_last) & gtod->clock.mask;
        return v * gtod->clock.mult;
}

static int do_monotonic(struct timespec *ts, cycle_t *cycle_now)
{
        unsigned long seq;
        u64 ns;
        int mode;
        struct pvclock_gtod_data *gtod = &pvclock_gtod_data;

        ts->tv_nsec = 0;
        do {
                seq = read_seqcount_begin(&gtod->seq);
                mode = gtod->clock.vclock_mode;
                ts->tv_sec = gtod->monotonic_time_sec;
                ns = gtod->monotonic_time_snsec;
                ns += vgettsc(cycle_now);
                ns >>= gtod->clock.shift;
        } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
        timespec_add_ns(ts, ns);

        return mode;
}

/* returns true if host is using tsc clocksource */
static bool kvm_get_time_and_clockread(s64 *kernel_ns, cycle_t *cycle_now)
{
        struct timespec ts;

        /* checked again under seqlock below */
        if (pvclock_gtod_data.clock.vclock_mode != VCLOCK_TSC)
                return false;

        if (do_monotonic(&ts, cycle_now) != VCLOCK_TSC)
                return false;

        monotonic_to_bootbased(&ts);
        *kernel_ns = timespec_to_ns(&ts);

        return true;
}
#endif

/*
 *
 * Assuming a stable TSC across physical CPUS, and a stable TSC
 * across virtual CPUs, the following condition is possible.
 * Each numbered line represents an event visible to both
 * CPUs at the next numbered event.
 *
 * "timespecX" represents host monotonic time. "tscX" represents
 * RDTSC value.
 *
 *              VCPU0 on CPU0           |       VCPU1 on CPU1
 *
 * 1.  read timespec0,tsc0
 * 2.                                   | timespec1 = timespec0 + N
 *                                      | tsc1 = tsc0 + M
 * 3. transition to guest               | transition to guest
 * 4. ret0 = timespec0 + (rdtsc - tsc0) |
 * 5.                                   | ret1 = timespec1 + (rdtsc - tsc1)
 *                                      | ret1 = timespec0 + N + (rdtsc - (tsc0 + M))
 *
 * Since ret0 update is visible to VCPU1 at time 5, to obey monotonicity:
 *
 *      - ret0 < ret1
 *      - timespec0 + (rdtsc - tsc0) < timespec0 + N + (rdtsc - (tsc0 + M))
 *              ...
 *      - 0 < N - M => M < N
 *
 * That is, when timespec0 != timespec1, M < N. Unfortunately that is not
 * always the case (the difference between two distinct xtime instances
 * might be smaller then the difference between corresponding TSC reads,
 * when updating guest vcpus pvclock areas).
 *
 * To avoid that problem, do not allow visibility of distinct
 * system_timestamp/tsc_timestamp values simultaneously: use a master
 * copy of host monotonic time values. Update that master copy
 * in lockstep.
 *
 * Rely on synchronization of host TSCs and guest TSCs for monotonicity.
 *
 */

static void pvclock_update_vm_gtod_copy(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
        struct kvm_arch *ka = &kvm->arch;
        int vclock_mode;
        bool host_tsc_clocksource, vcpus_matched;

        vcpus_matched = (ka->nr_vcpus_matched_tsc + 1 ==
                        atomic_read(&kvm->online_vcpus));

        /*
         * If the host uses TSC clock, then passthrough TSC as stable
         * to the guest.
         */
        host_tsc_clocksource = kvm_get_time_and_clockread(
                                        &ka->master_kernel_ns,
                                        &ka->master_cycle_now);

        ka->use_master_clock = host_tsc_clocksource && vcpus_matched
                                && !backwards_tsc_observed;

        if (ka->use_master_clock)
                atomic_set(&kvm_guest_has_master_clock, 1);

        vclock_mode = pvclock_gtod_data.clock.vclock_mode;
        trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode,
                                        vcpus_matched);
#endif
}

static void kvm_gen_update_masterclock(struct kvm *kvm)
{
#ifdef CONFIG_X86_64
        int i;
        struct kvm_vcpu *vcpu;
        struct kvm_arch *ka = &kvm->arch;

        spin_lock(&ka->pvclock_gtod_sync_lock);
        kvm_make_mclock_inprogress_request(kvm);
        /* no guest entries from this point */
        pvclock_update_vm_gtod_copy(kvm);

        kvm_for_each_vcpu(i, vcpu, kvm)
                set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);

        /* guest entries allowed */
        kvm_for_each_vcpu(i, vcpu, kvm)
                clear_bit(KVM_REQ_MCLOCK_INPROGRESS, &vcpu->requests);

        spin_unlock(&ka->pvclock_gtod_sync_lock);
#endif
}

static int kvm_guest_time_update(struct kvm_vcpu *v)
{
        unsigned long flags, this_tsc_khz;
        struct kvm_vcpu_arch *vcpu = &v->arch;
        struct kvm_arch *ka = &v->kvm->arch;
        s64 kernel_ns;
        u64 tsc_timestamp, host_tsc;
        struct pvclock_vcpu_time_info guest_hv_clock;
        u8 pvclock_flags;
        bool use_master_clock;

        kernel_ns = 0;
        host_tsc = 0;

        /*
         * If the host uses TSC clock, then passthrough TSC as stable
         * to the guest.
         */
        spin_lock(&ka->pvclock_gtod_sync_lock);
        use_master_clock = ka->use_master_clock;
        if (use_master_clock) {
                host_tsc = ka->master_cycle_now;
                kernel_ns = ka->master_kernel_ns;
        }
        spin_unlock(&ka->pvclock_gtod_sync_lock);

        /* Keep irq disabled to prevent changes to the clock */
        local_irq_save(flags);
        this_tsc_khz = __get_cpu_var(cpu_tsc_khz);
        if (unlikely(this_tsc_khz == 0)) {
                local_irq_restore(flags);
                kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
                return 1;
        }
        if (!use_master_clock) {
                host_tsc = native_read_tsc();
                kernel_ns = get_kernel_ns();
        }

        tsc_timestamp = kvm_x86_ops->read_l1_tsc(v, host_tsc);

        /*
         * We may have to catch up the TSC to match elapsed wall clock
         * time for two reasons, even if kvmclock is used.
         *   1) CPU could have been running below the maximum TSC rate
         *   2) Broken TSC compensation resets the base at each VCPU
         *      entry to avoid unknown leaps of TSC even when running
         *      again on the same CPU.  This may cause apparent elapsed
         *      time to disappear, and the guest to stand still or run
         *      very slowly.
         */
        if (vcpu->tsc_catchup) {
                u64 tsc = compute_guest_tsc(v, kernel_ns);
                if (tsc > tsc_timestamp) {
                        adjust_tsc_offset_guest(v, tsc - tsc_timestamp);
                        tsc_timestamp = tsc;
                }
        }

        local_irq_restore(flags);

        if (!vcpu->pv_time_enabled)
                return 0;

        if (unlikely(vcpu->hw_tsc_khz != this_tsc_khz)) {
                kvm_get_time_scale(NSEC_PER_SEC / 1000, this_tsc_khz,
                                   &vcpu->hv_clock.tsc_shift,
                                   &vcpu->hv_clock.tsc_to_system_mul);
                vcpu->hw_tsc_khz = this_tsc_khz;
        }

        /* With all the info we got, fill in the values */
        vcpu->hv_clock.tsc_timestamp = tsc_timestamp;
        vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
        vcpu->last_guest_tsc = tsc_timestamp;

        /*
         * The interface expects us to write an even number signaling that the
         * update is finished. Since the guest won't see the intermediate
         * state, we just increase by 2 at the end.
         */
        vcpu->hv_clock.version += 2;

        if (unlikely(kvm_read_guest_cached(v->kvm, &vcpu->pv_time,
                &guest_hv_clock, sizeof(guest_hv_clock))))
                return 0;

        /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */
        pvclock_flags = (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED);

        if (vcpu->pvclock_set_guest_stopped_request) {
                pvclock_flags |= PVCLOCK_GUEST_STOPPED;
                vcpu->pvclock_set_guest_stopped_request = false;
        }

        /* If the host uses TSC clocksource, then it is stable */
        if (use_master_clock)
                pvclock_flags |= PVCLOCK_TSC_STABLE_BIT;

        vcpu->hv_clock.flags = pvclock_flags;

        kvm_write_guest_cached(v->kvm, &vcpu->pv_time,
                                &vcpu->hv_clock,
                                sizeof(vcpu->hv_clock));
        return 0;
}

/*
 * kvmclock updates which are isolated to a given vcpu, such as
 * vcpu->cpu migration, should not allow system_timestamp from
 * the rest of the vcpus to remain static. Otherwise ntp frequency
 * correction applies to one vcpu's system_timestamp but not
 * the others.
 *
 * So in those cases, request a kvmclock update for all vcpus.
 * We need to rate-limit these requests though, as they can
 * considerably slow guests that have a large number of vcpus.
 * The time for a remote vcpu to update its kvmclock is bound
 * by the delay we use to rate-limit the updates.
 */

#define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

static void kvmclock_update_fn(struct work_struct *work)
{
        int i;
        struct delayed_work *dwork = to_delayed_work(work);
        struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
                                           kvmclock_update_work);
        struct kvm *kvm = container_of(ka, struct kvm, arch);
        struct kvm_vcpu *vcpu;

        kvm_for_each_vcpu(i, vcpu, kvm) {
                set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
                kvm_vcpu_kick(vcpu);
        }
}

static void kvm_gen_kvmclock_update(struct kvm_vcpu *v)
{
        struct kvm *kvm = v->kvm;

        set_bit(KVM_REQ_CLOCK_UPDATE, &v->requests);
        schedule_delayed_work(&kvm->arch.kvmclock_update_work,
                                        KVMCLOCK_UPDATE_DELAY);
}

#define KVMCLOCK_SYNC_PERIOD (300 * HZ)

static void kvmclock_sync_fn(struct work_struct *work)
{
        struct delayed_work *dwork = to_delayed_work(work);
        struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
                                           kvmclock_sync_work);
        struct kvm *kvm = container_of(ka, struct kvm, arch);

        schedule_delayed_work(&kvm->arch.kvmclock_update_work, 0);
        schedule_delayed_work(&kvm->arch.kvmclock_sync_work,
                                        KVMCLOCK_SYNC_PERIOD);
}

static bool msr_mtrr_valid(unsigned msr)
{
        switch (msr) {
        case 0x200 ... 0x200 + 2 * KVM_NR_VAR_MTRR - 1:
        case MSR_MTRRfix64K_00000:
        case MSR_MTRRfix16K_80000:
        case MSR_MTRRfix16K_A0000:
        case MSR_MTRRfix4K_C0000:
        case MSR_MTRRfix4K_C8000:
        case MSR_MTRRfix4K_D0000:
        case MSR_MTRRfix4K_D8000:
        case MSR_MTRRfix4K_E0000:
        case MSR_MTRRfix4K_E8000:
        case MSR_MTRRfix4K_F0000:
        case MSR_MTRRfix4K_F8000:
        case MSR_MTRRdefType:
        case MSR_IA32_CR_PAT:
                return true;
        case 0x2f8:
                return true;
        }
        return false;
}

static bool valid_pat_type(unsigned t)
{
        return t < 8 && (1 << t) & 0xf3; /* 0, 1, 4, 5, 6, 7 */
}

static bool valid_mtrr_type(unsigned t)
{
        return t < 8 && (1 << t) & 0x73; /* 0, 1, 4, 5, 6 */
}

static bool mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
        int i;

        if (!msr_mtrr_valid(msr))
                return false;

        if (msr == MSR_IA32_CR_PAT) {
                for (i = 0; i < 8; i++)
                        if (!valid_pat_type((data >> (i * 8)) & 0xff))
                                return false;
                return true;
        } else if (msr == MSR_MTRRdefType) {
                if (data & ~0xcff)
                        return false;
                return valid_mtrr_type(data & 0xff);
        } else if (msr >= MSR_MTRRfix64K_00000 && msr <= MSR_MTRRfix4K_F8000) {
                for (i = 0; i < 8 ; i++)
                        if (!valid_mtrr_type((data >> (i * 8)) & 0xff))
                                return false;
                return true;
        }

        /* variable MTRRs */
        return valid_mtrr_type(data & 0xff);
}

static int set_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
        u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;

        if (!mtrr_valid(vcpu, msr, data))
                return 1;

        if (msr == MSR_MTRRdefType) {
                vcpu->arch.mtrr_state.def_type = data;
                vcpu->arch.mtrr_state.enabled = (data & 0xc00) >> 10;
        } else if (msr == MSR_MTRRfix64K_00000)
                p[0] = data;
        else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
                p[1 + msr - MSR_MTRRfix16K_80000] = data;
        else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
                p[3 + msr - MSR_MTRRfix4K_C0000] = data;
        else if (msr == MSR_IA32_CR_PAT)
                vcpu->arch.pat = data;
        else {  /* Variable MTRRs */
                int idx, is_mtrr_mask;
                u64 *pt;

                idx = (msr - 0x200) / 2;
                is_mtrr_mask = msr - 0x200 - 2 * idx;
                if (!is_mtrr_mask)
                        pt =
                          (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
                else
                        pt =
                          (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
                *pt = data;
        }

        kvm_mmu_reset_context(vcpu);
        return 0;
}

static int set_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
        u64 mcg_cap = vcpu->arch.mcg_cap;
        unsigned bank_num = mcg_cap & 0xff;

        switch (msr) {
        case MSR_IA32_MCG_STATUS:
                vcpu->arch.mcg_status = data;
                break;
        case MSR_IA32_MCG_CTL:
                if (!(mcg_cap & MCG_CTL_P))
                        return 1;
                if (data != 0 && data != ~(u64)0)
                        return -1;
                vcpu->arch.mcg_ctl = data;
                break;
        default:
                if (msr >= MSR_IA32_MC0_CTL &&
                    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
                        u32 offset = msr - MSR_IA32_MC0_CTL;
                        /* only 0 or all 1s can be written to IA32_MCi_CTL
                         * some Linux kernels though clear bit 10 in bank 4 to
                         * workaround a BIOS/GART TBL issue on AMD K8s, ignore
                         * this to avoid an uncatched #GP in the guest
                         */
                        if ((offset & 0x3) == 0 &&
                            data != 0 && (data | (1 << 10)) != ~(u64)0)
                                return -1;
                        vcpu->arch.mce_banks[offset] = data;
                        break;
                }
                return 1;
        }
        return 0;
}

static int xen_hvm_config(struct kvm_vcpu *vcpu, u64 data)
{
        struct kvm *kvm = vcpu->kvm;
        int lm = is_long_mode(vcpu);
        u8 *blob_addr = lm ? (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_64
                : (u8 *)(long)kvm->arch.xen_hvm_config.blob_addr_32;
        u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
                : kvm->arch.xen_hvm_config.blob_size_32;
        u32 page_num = data & ~PAGE_MASK;
        u64 page_addr = data & PAGE_MASK;
        u8 *page;
        int r;

        r = -E2BIG;
        if (page_num >= blob_size)
                goto out;
        r = -ENOMEM;
        page = memdup_user(blob_addr + (page_num * PAGE_SIZE), PAGE_SIZE);
        if (IS_ERR(page)) {
                r = PTR_ERR(page);
                goto out;
        }
        if (kvm_write_guest(kvm, page_addr, page, PAGE_SIZE))
                goto out_free;
        r = 0;
out_free:
        kfree(page);
out:
        return r;
}

static bool kvm_hv_hypercall_enabled(struct kvm *kvm)
{
        return kvm->arch.hv_hypercall & HV_X64_MSR_HYPERCALL_ENABLE;
}

static bool kvm_hv_msr_partition_wide(u32 msr)
{
        bool r = false;
        switch (msr) {
        case HV_X64_MSR_GUEST_OS_ID:
        case HV_X64_MSR_HYPERCALL:
        case HV_X64_MSR_REFERENCE_TSC:
        case HV_X64_MSR_TIME_REF_COUNT:
                r = true;
                break;
        }

        return r;
}

static int set_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
        struct kvm *kvm = vcpu->kvm;

        switch (msr) {
        case HV_X64_MSR_GUEST_OS_ID:
                kvm->arch.hv_guest_os_id = data;
                /* setting guest os id to zero disables hypercall page */
                if (!kvm->arch.hv_guest_os_id)
                        kvm->arch.hv_hypercall &= ~HV_X64_MSR_HYPERCALL_ENABLE;
                break;
        case HV_X64_MSR_HYPERCALL: {
                u64 gfn;
                unsigned long addr;
                u8 instructions[4];

                /* if guest os id is not set hypercall should remain disabled */
                if (!kvm->arch.hv_guest_os_id)
                        break;
                if (!(data & HV_X64_MSR_HYPERCALL_ENABLE)) {
                        kvm->arch.hv_hypercall = data;
                        break;
                }
                gfn = data >> HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT;
                addr = gfn_to_hva(kvm, gfn);
                if (kvm_is_error_hva(addr))
                        return 1;
                kvm_x86_ops->patch_hypercall(vcpu, instructions);
                ((unsigned char *)instructions)[3] = 0xc3; /* ret */
                if (__copy_to_user((void __user *)addr, instructions, 4))
                        return 1;
                kvm->arch.hv_hypercall = data;
                mark_page_dirty(kvm, gfn);
                break;
        }
        case HV_X64_MSR_REFERENCE_TSC: {
                u64 gfn;
                HV_REFERENCE_TSC_PAGE tsc_ref;
                memset(&tsc_ref, 0, sizeof(tsc_ref));
                kvm->arch.hv_tsc_page = data;
                if (!(data & HV_X64_MSR_TSC_REFERENCE_ENABLE))
                        break;
                gfn = data >> HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT;
                if (kvm_write_guest(kvm, data,
                        &tsc_ref, sizeof(tsc_ref)))
                        return 1;
                mark_page_dirty(kvm, gfn);
                break;
        }
        default:
                vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
                            "data 0x%llx\n", msr, data);
                return 1;
        }
        return 0;
}

static int set_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 data)
{
        switch (msr) {
        case HV_X64_MSR_APIC_ASSIST_PAGE: {
                u64 gfn;
                unsigned long addr;

                if (!(data & HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE)) {
                        vcpu->arch.hv_vapic = data;
                        break;
                }
                gfn = data >> HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT;
                addr = gfn_to_hva(vcpu->kvm, gfn);
                if (kvm_is_error_hva(addr))
                        return 1;
                if (__clear_user((void __user *)addr, PAGE_SIZE))
                        return 1;
                vcpu->arch.hv_vapic = data;
                mark_page_dirty(vcpu->kvm, gfn);
                break;
        }
        case HV_X64_MSR_EOI:
                return kvm_hv_vapic_msr_write(vcpu, APIC_EOI, data);
        case HV_X64_MSR_ICR:
                return kvm_hv_vapic_msr_write(vcpu, APIC_ICR, data);
        case HV_X64_MSR_TPR:
                return kvm_hv_vapic_msr_write(vcpu, APIC_TASKPRI, data);
        default:
                vcpu_unimpl(vcpu, "HYPER-V unimplemented wrmsr: 0x%x "
                            "data 0x%llx\n", msr, data);
                return 1;
        }

        return 0;
}

static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
{
        gpa_t gpa = data & ~0x3f;

        /* Bits 2:5 are reserved, Should be zero */
        if (data & 0x3c)
                return 1;

        vcpu->arch.apf.msr_val = data;

        if (!(data & KVM_ASYNC_PF_ENABLED)) {
                kvm_clear_async_pf_completion_queue(vcpu);
                kvm_async_pf_hash_reset(vcpu);
                return 0;
        }

        if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
                                        sizeof(u32)))
                return 1;

        vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS);
        kvm_async_pf_wakeup_all(vcpu);
        return 0;
}

static void kvmclock_reset(struct kvm_vcpu *vcpu)
{
        vcpu->arch.pv_time_enabled = false;
}

static void accumulate_steal_time(struct kvm_vcpu *vcpu)
{
        u64 delta;

        if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
                return;

        delta = current->sched_info.run_delay - vcpu->arch.st.last_steal;
        vcpu->arch.st.last_steal = current->sched_info.run_delay;
        vcpu->arch.st.accum_steal = delta;
}

static void record_steal_time(struct kvm_vcpu *vcpu)
{
        if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
                return;

        if (unlikely(kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
                &vcpu->arch.st.steal, sizeof(struct kvm_steal_time))))
                return;

        vcpu->arch.st.steal.steal += vcpu->arch.st.accum_steal;
        vcpu->arch.st.steal.version += 2;
        vcpu->arch.st.accum_steal = 0;

        kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.st.stime,
                &vcpu->arch.st.steal, sizeof(struct kvm_steal_time));
}

int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
{
        bool pr = false;
        u32 msr = msr_info->index;
        u64 data = msr_info->data;

        switch (msr) {
        case MSR_AMD64_NB_CFG:
        case MSR_IA32_UCODE_REV:
        case MSR_IA32_UCODE_WRITE:
        case MSR_VM_HSAVE_PA:
        case MSR_AMD64_PATCH_LOADER:
        case MSR_AMD64_BU_CFG2:
                break;

        case MSR_EFER:
                return set_efer(vcpu, data);
        case MSR_K7_HWCR:
                data &= ~(u64)0x40;     /* ignore flush filter disable */
                data &= ~(u64)0x100;    /* ignore ignne emulation enable */
                data &= ~(u64)0x8;      /* ignore TLB cache disable */
                if (data != 0) {
                        vcpu_unimpl(vcpu, "unimplemented HWCR wrmsr: 0x%llx\n",
                                    data);
                        return 1;
                }
                break;
        case MSR_FAM10H_MMIO_CONF_BASE:
                if (data != 0) {
                        vcpu_unimpl(vcpu, "unimplemented MMIO_CONF_BASE wrmsr: "
                                    "0x%llx\n", data);
                        return 1;
                }
                break;
        case MSR_IA32_DEBUGCTLMSR:
                if (!data) {
                        /* We support the non-activated case already */
                        break;
                } else if (data & ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_BTF)) {
                        /* Values other than LBR and BTF are vendor-specific,
                           thus reserved and should throw a #GP */
                        return 1;
                }
                vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTLMSR 0x%llx, nop\n",
                            __func__, data);
                break;
        case 0x200 ... 0x2ff:
                return set_msr_mtrr(vcpu, msr, data);
        case MSR_IA32_APICBASE:
                return kvm_set_apic_base(vcpu, msr_info);
        case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
                return kvm_x2apic_msr_write(vcpu, msr, data);
        case MSR_IA32_TSCDEADLINE:
                kvm_set_lapic_tscdeadline_msr(vcpu, data);
                break;
        case MSR_IA32_TSC_ADJUST:
                if (guest_cpuid_has_tsc_adjust(vcpu)) {
                        if (!msr_info->host_initiated) {
                                u64 adj = data - vcpu->arch.ia32_tsc_adjust_msr;
                                kvm_x86_ops->adjust_tsc_offset(vcpu, adj, true);
                        }
                        vcpu->arch.ia32_tsc_adjust_msr = data;
                }
                break;
        case MSR_IA32_MISC_ENABLE:
                vcpu->arch.ia32_misc_enable_msr = data;
                break;
        case MSR_KVM_WALL_CLOCK_NEW:
        case MSR_KVM_WALL_CLOCK:
                vcpu->kvm->arch.wall_clock = data;
                kvm_write_wall_clock(vcpu->kvm, data);
                break;
        case MSR_KVM_SYSTEM_TIME_NEW:
        case MSR_KVM_SYSTEM_TIME: {
                u64 gpa_offset;
                kvmclock_reset(vcpu);

                vcpu->arch.time = data;
                kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);

                /* we verify if the enable bit is set... */
                if (!(data & 1))
                        break;

                gpa_offset = data & ~(PAGE_MASK | 1);

                if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
                     &vcpu->arch.pv_time, data & ~1ULL,
                     sizeof(struct pvclock_vcpu_time_info)))
                        vcpu->arch.pv_time_enabled = false;
                else
                        vcpu->arch.pv_time_enabled = true;

                break;
        }
        case MSR_KVM_ASYNC_PF_EN:
                if (kvm_pv_enable_async_pf(vcpu, data))
                        return 1;
                break;
        case MSR_KVM_STEAL_TIME:

                if (unlikely(!sched_info_on()))
                        return 1;

                if (data & KVM_STEAL_RESERVED_MASK)
                        return 1;

                if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.st.stime,
                                                data & KVM_STEAL_VALID_BITS,
                                                sizeof(struct kvm_steal_time)))
                        return 1;

                vcpu->arch.st.msr_val = data;

                if (!(data & KVM_MSR_ENABLED))
                        break;

                vcpu->arch.st.last_steal = current->sched_info.run_delay;

                preempt_disable();
                accumulate_steal_time(vcpu);
                preempt_enable();

                kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);

                break;
        case MSR_KVM_PV_EOI_EN:
                if (kvm_lapic_enable_pv_eoi(vcpu, data))
                        return 1;
                break;

        case MSR_IA32_MCG_CTL:
        case MSR_IA32_MCG_STATUS:
        case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
                return set_msr_mce(vcpu, msr, data);

        /* Performance counters are not protected by a CPUID bit,
         * so we should check all of them in the generic path for the sake of
         * cross vendor migration.
         * Writing a zero into the event select MSRs disables them,
         * which we perfectly emulate ;-). Any other value should be at least
         * reported, some guests depend on them.
         */
        case MSR_K7_EVNTSEL0:
        case MSR_K7_EVNTSEL1:
        case MSR_K7_EVNTSEL2:
        case MSR_K7_EVNTSEL3:
                if (data != 0)
                        vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
                                    "0x%x data 0x%llx\n", msr, data);
                break;
        /* at least RHEL 4 unconditionally writes to the perfctr registers,
         * so we ignore writes to make it happy.
         */
        case MSR_K7_PERFCTR0:
        case MSR_K7_PERFCTR1:
        case MSR_K7_PERFCTR2:
        case MSR_K7_PERFCTR3:
                vcpu_unimpl(vcpu, "unimplemented perfctr wrmsr: "
                            "0x%x data 0x%llx\n", msr, data);
                break;
        case MSR_P6_PERFCTR0:
        case MSR_P6_PERFCTR1:
                pr = true;
        case MSR_P6_EVNTSEL0:
        case MSR_P6_EVNTSEL1:
                if (kvm_pmu_msr(vcpu, msr))
                        return kvm_pmu_set_msr(vcpu, msr_info);

                if (pr || data != 0)
                        vcpu_unimpl(vcpu, "disabled perfctr wrmsr: "
                                    "0x%x data 0x%llx\n", msr, data);
                break;
        case MSR_K7_CLK_CTL:
                /*
                 * Ignore all writes to this no longer documented MSR.
                 * Writes are only relevant for old K7 processors,
                 * all pre-dating SVM, but a recommended workaround from
                 * AMD for these chips. It is possible to specify the
                 * affected processor models on the command line, hence
                 * the need to ignore the workaround.
                 */
                break;
        case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
                if (kvm_hv_msr_partition_wide(msr)) {
                        int r;
                        mutex_lock(&vcpu->kvm->lock);
                        r = set_msr_hyperv_pw(vcpu, msr, data);
                        mutex_unlock(&vcpu->kvm->lock);
                        return r;
                } else
                        return set_msr_hyperv(vcpu, msr, data);
                break;
        case MSR_IA32_BBL_CR_CTL3:
                /* Drop writes to this legacy MSR -- see rdmsr
                 * counterpart for further detail.
                 */
                vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data);
                break;
        case MSR_AMD64_OSVW_ID_LENGTH:
                if (!guest_cpuid_has_osvw(vcpu))
                        return 1;
                vcpu->arch.osvw.length = data;
                break;
        case MSR_AMD64_OSVW_STATUS:
                if (!guest_cpuid_has_osvw(vcpu))
                        return 1;
                vcpu->arch.osvw.status = data;
                break;
        default:
                if (msr && (msr == vcpu->kvm->arch.xen_hvm_config.msr))
                        return xen_hvm_config(vcpu, data);
                if (kvm_pmu_msr(vcpu, msr))
                        return kvm_pmu_set_msr(vcpu, msr_info);
                if (!ignore_msrs) {
                        vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n",
                                    msr, data);
                        return 1;
                } else {
                        vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n",
                                    msr, data);
                        break;
                }
        }
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_set_msr_common);


/*
 * Reads an msr value (of 'msr_index') into 'pdata'.
 * Returns 0 on success, non-0 otherwise.
 * Assumes vcpu_load() was already called.
 */
int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
{
        return kvm_x86_ops->get_msr(vcpu, msr_index, pdata);
}

static int get_msr_mtrr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
        u64 *p = (u64 *)&vcpu->arch.mtrr_state.fixed_ranges;

        if (!msr_mtrr_valid(msr))
                return 1;

        if (msr == MSR_MTRRdefType)
                *pdata = vcpu->arch.mtrr_state.def_type +
                         (vcpu->arch.mtrr_state.enabled << 10);
        else if (msr == MSR_MTRRfix64K_00000)
                *pdata = p[0];
        else if (msr == MSR_MTRRfix16K_80000 || msr == MSR_MTRRfix16K_A0000)
                *pdata = p[1 + msr - MSR_MTRRfix16K_80000];
        else if (msr >= MSR_MTRRfix4K_C0000 && msr <= MSR_MTRRfix4K_F8000)
                *pdata = p[3 + msr - MSR_MTRRfix4K_C0000];
        else if (msr == MSR_IA32_CR_PAT)
                *pdata = vcpu->arch.pat;
        else {  /* Variable MTRRs */
                int idx, is_mtrr_mask;
                u64 *pt;

                idx = (msr - 0x200) / 2;
                is_mtrr_mask = msr - 0x200 - 2 * idx;
                if (!is_mtrr_mask)
                        pt =
                          (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].base_lo;
                else
                        pt =
                          (u64 *)&vcpu->arch.mtrr_state.var_ranges[idx].mask_lo;
                *pdata = *pt;
        }

        return 0;
}

static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
        u64 data;
        u64 mcg_cap = vcpu->arch.mcg_cap;
        unsigned bank_num = mcg_cap & 0xff;

        switch (msr) {
        case MSR_IA32_P5_MC_ADDR:
        case MSR_IA32_P5_MC_TYPE:
                data = 0;
                break;
        case MSR_IA32_MCG_CAP:
                data = vcpu->arch.mcg_cap;
                break;
        case MSR_IA32_MCG_CTL:
                if (!(mcg_cap & MCG_CTL_P))
                        return 1;
                data = vcpu->arch.mcg_ctl;
                break;
        case MSR_IA32_MCG_STATUS:
                data = vcpu->arch.mcg_status;
                break;
        default:
                if (msr >= MSR_IA32_MC0_CTL &&
                    msr < MSR_IA32_MC0_CTL + 4 * bank_num) {
                        u32 offset = msr - MSR_IA32_MC0_CTL;
                        data = vcpu->arch.mce_banks[offset];
                        break;
                }
                return 1;
        }
        *pdata = data;
        return 0;
}

static int get_msr_hyperv_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
        u64 data = 0;
        struct kvm *kvm = vcpu->kvm;

        switch (msr) {
        case HV_X64_MSR_GUEST_OS_ID:
                data = kvm->arch.hv_guest_os_id;
                break;
        case HV_X64_MSR_HYPERCALL:
                data = kvm->arch.hv_hypercall;
                break;
        case HV_X64_MSR_TIME_REF_COUNT: {
                data =
                     div_u64(get_kernel_ns() + kvm->arch.kvmclock_offset, 100);
                break;
        }
        case HV_X64_MSR_REFERENCE_TSC:
                data = kvm->arch.hv_tsc_page;
                break;
        default:
                vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
                return 1;
        }

        *pdata = data;
        return 0;
}

static int get_msr_hyperv(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
        u64 data = 0;

        switch (msr) {
        case HV_X64_MSR_VP_INDEX: {
                int r;
                struct kvm_vcpu *v;
                kvm_for_each_vcpu(r, v, vcpu->kvm) {
                        if (v == vcpu) {
                                data = r;
                                break;
                        }
                }
                break;
        }
        case HV_X64_MSR_EOI:
                return kvm_hv_vapic_msr_read(vcpu, APIC_EOI, pdata);
        case HV_X64_MSR_ICR:
                return kvm_hv_vapic_msr_read(vcpu, APIC_ICR, pdata);
        case HV_X64_MSR_TPR:
                return kvm_hv_vapic_msr_read(vcpu, APIC_TASKPRI, pdata);
        case HV_X64_MSR_APIC_ASSIST_PAGE:
                data = vcpu->arch.hv_vapic;
                break;
        default:
                vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr);
                return 1;
        }
        *pdata = data;
        return 0;
}

int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
{
        u64 data;

        switch (msr) {
        case MSR_IA32_PLATFORM_ID:
        case MSR_IA32_EBL_CR_POWERON:
        case MSR_IA32_DEBUGCTLMSR:
        case MSR_IA32_LASTBRANCHFROMIP:
        case MSR_IA32_LASTBRANCHTOIP:
        case MSR_IA32_LASTINTFROMIP:
        case MSR_IA32_LASTINTTOIP:
        case MSR_K8_SYSCFG:
        case MSR_K7_HWCR:
        case MSR_VM_HSAVE_PA:
        case MSR_K7_EVNTSEL0:
        case MSR_K7_PERFCTR0:
        case MSR_K8_INT_PENDING_MSG:
        case MSR_AMD64_NB_CFG:
        case MSR_FAM10H_MMIO_CONF_BASE:
        case MSR_AMD64_BU_CFG2:
                data = 0;
                break;
        case MSR_P6_PERFCTR0:
        case MSR_P6_PERFCTR1:
        case MSR_P6_EVNTSEL0:
        case MSR_P6_EVNTSEL1:
                if (kvm_pmu_msr(vcpu, msr))
                        return kvm_pmu_get_msr(vcpu, msr, pdata);
                data = 0;
                break;
        case MSR_IA32_UCODE_REV:
                data = 0x100000000ULL;
                break;
        case MSR_MTRRcap:
                data = 0x500 | KVM_NR_VAR_MTRR;
                break;
        case 0x200 ... 0x2ff:
                return get_msr_mtrr(vcpu, msr, pdata);
        case 0xcd: /* fsb frequency */
                data = 3;
                break;
                /*
                 * MSR_EBC_FREQUENCY_ID
                 * Conservative value valid for even the basic CPU models.
                 * Models 0,1: 000 in bits 23:21 indicating a bus speed of
                 * 100MHz, model 2 000 in bits 18:16 indicating 100MHz,
                 * and 266MHz for model 3, or 4. Set Core Clock
                 * Frequency to System Bus Frequency Ratio to 1 (bits
                 * 31:24) even though these are only valid for CPU
                 * models > 2, however guests may end up dividing or
                 * multiplying by zero otherwise.
                 */
        case MSR_EBC_FREQUENCY_ID:
                data = 1 << 24;
                break;
        case MSR_IA32_APICBASE:
                data = kvm_get_apic_base(vcpu);
                break;
        case APIC_BASE_MSR ... APIC_BASE_MSR + 0x3ff:
                return kvm_x2apic_msr_read(vcpu, msr, pdata);
                break;
        case MSR_IA32_TSCDEADLINE:
                data = kvm_get_lapic_tscdeadline_msr(vcpu);
                break;
        case MSR_IA32_TSC_ADJUST:
                data = (u64)vcpu->arch.ia32_tsc_adjust_msr;
                break;
        case MSR_IA32_MISC_ENABLE:
                data = vcpu->arch.ia32_misc_enable_msr;
                break;
        case MSR_IA32_PERF_STATUS:
                /* TSC increment by tick */
                data = 1000ULL;
                /* CPU multiplier */
                data |= (((uint64_t)4ULL) << 40);
                break;
        case MSR_EFER:
                data = vcpu->arch.efer;
                break;
        case MSR_KVM_WALL_CLOCK:
        case MSR_KVM_WALL_CLOCK_NEW:
                data = vcpu->kvm->arch.wall_clock;
                break;
        case MSR_KVM_SYSTEM_TIME:
        case MSR_KVM_SYSTEM_TIME_NEW:
                data = vcpu->arch.time;
                break;
        case MSR_KVM_ASYNC_PF_EN:
                data = vcpu->arch.apf.msr_val;
                break;
        case MSR_KVM_STEAL_TIME:
                data = vcpu->arch.st.msr_val;
                break;
        case MSR_KVM_PV_EOI_EN:
                data = vcpu->arch.pv_eoi.msr_val;
                break;
        case MSR_IA32_P5_MC_ADDR:
        case MSR_IA32_P5_MC_TYPE:
        case MSR_IA32_MCG_CAP:
        case MSR_IA32_MCG_CTL:
        case MSR_IA32_MCG_STATUS:
        case MSR_IA32_MC0_CTL ... MSR_IA32_MC0_CTL + 4 * KVM_MAX_MCE_BANKS - 1:
                return get_msr_mce(vcpu, msr, pdata);
        case MSR_K7_CLK_CTL:
                /*
                 * Provide expected ramp-up count for K7. All other
                 * are set to zero, indicating minimum divisors for
                 * every field.
                 *
                 * This prevents guest kernels on AMD host with CPU
                 * type 6, model 8 and higher from exploding due to
                 * the rdmsr failing.
                 */
                data = 0x20000000;
                break;
        case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15:
                if (kvm_hv_msr_partition_wide(msr)) {
                        int r;
                        mutex_lock(&vcpu->kvm->lock);
                        r = get_msr_hyperv_pw(vcpu, msr, pdata);
                        mutex_unlock(&vcpu->kvm->lock);
                        return r;
                } else
                        return get_msr_hyperv(vcpu, msr, pdata);
                break;
        case MSR_IA32_BBL_CR_CTL3:
                /* This legacy MSR exists but isn't fully documented in current
                 * silicon.  It is however accessed by winxp in very narrow
                 * scenarios where it sets bit #19, itself documented as
                 * a "reserved" bit.  Best effort attempt to source coherent
                 * read data here should the balance of the register be
                 * interpreted by the guest:
                 *
                 * L2 cache control register 3: 64GB range, 256KB size,
                 * enabled, latency 0x1, configured
                 */
                data = 0xbe702111;
                break;
        case MSR_AMD64_OSVW_ID_LENGTH:
                if (!guest_cpuid_has_osvw(vcpu))
                        return 1;
                data = vcpu->arch.osvw.length;
                break;
        case MSR_AMD64_OSVW_STATUS:
                if (!guest_cpuid_has_osvw(vcpu))
                        return 1;
                data = vcpu->arch.osvw.status;
                break;
        default:
                if (kvm_pmu_msr(vcpu, msr))
                        return kvm_pmu_get_msr(vcpu, msr, pdata);
                if (!ignore_msrs) {
                        vcpu_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr);
                        return 1;
                } else {
                        vcpu_unimpl(vcpu, "ignored rdmsr: 0x%x\n", msr);
                        data = 0;
                }
                break;
        }
        *pdata = data;
        return 0;
}
EXPORT_SYMBOL_GPL(kvm_get_msr_common);

/*
 * Read or write a bunch of msrs. All parameters are kernel addresses.
 *
 * @return number of msrs set successfully.
 */
static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs,
                    struct kvm_msr_entry *entries,
                    int (*do_msr)(struct kvm_vcpu *vcpu,
                                  unsigned index, u64 *data))
{
        int i, idx;

        idx = srcu_read_lock(&vcpu->kvm->srcu);
        for (i = 0; i < msrs->nmsrs; ++i)
                if (do_msr(vcpu, entries[i].index, &entries[i].data))
                        break;
        srcu_read_unlock(&vcpu->kvm->srcu, idx);

        return i;
}

/*
 * Read or write a bunch of msrs. Parameters are user addresses.
 *
 * @return number of msrs set successfully.
 */
static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs,
                  int (*do_msr)(struct kvm_vcpu *vcpu,
                                unsigned index, u64 *data),
                  int writeback)
{
        struct kvm_msrs msrs;
        struct kvm_msr_entry *entries;
        int r, n;
        unsigned size;

        r = -EFAULT;
        if (copy_from_user(&msrs, user_msrs, sizeof msrs))
                goto out;

        r = -E2BIG;
        if (msrs.nmsrs >= MAX_IO_MSRS)
                goto out;

        size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
        entries = memdup_user(user_msrs->entries, size);
        if (IS_ERR(entries)) {
                r = PTR_ERR(entries);
                goto out;
        }

        r = n = __msr_io(vcpu, &msrs, entries, do_msr);
        if (r < 0)
                goto out_free;

        r = -EFAULT;
        if (writeback && copy_to_user(user_msrs->entries, entries, size))
                goto out_free;

        r = n;

out_free:
        kfree(entries);
out:
        return r;
}

int kvm_dev_ioctl_check_extension(long ext)
{
        int r;

        switch (ext) {
        case KVM_CAP_IRQCHIP:
        case KVM_CAP_HLT:
        case KVM_CAP_MMU_SHADOW_CACHE_CONTROL:
        case KVM_CAP_SET_TSS_ADDR:
        case KVM_CAP_EXT_CPUID:
        case KVM_CAP_EXT_EMUL_CPUID:
        case KVM_CAP_CLOCKSOURCE:
        case KVM_CAP_PIT:
        case KVM_CAP_NOP_IO_DELAY:
        case KVM_CAP_MP_STATE:
        case KVM_CAP_SYNC_MMU:
        case KVM_CAP_USER_NMI:
        case KVM_CAP_REINJECT_CONTROL:
        case KVM_CAP_IRQ_INJECT_STATUS:
        case KVM_CAP_IRQFD:
        case KVM_CAP_IOEVENTFD:
        case KVM_CAP_PIT2:
        case KVM_CAP_PIT_STATE2:
        case KVM_CAP_SET_IDENTITY_MAP_ADDR:
        case KVM_CAP_XEN_HVM:
        case KVM_CAP_ADJUST_CLOCK:
        case KVM_CAP_VCPU_EVENTS:
        case KVM_CAP_HYPERV:
        case KVM_CAP_HYPERV_VAPIC:
        case KVM_CAP_HYPERV_SPIN:
        case KVM_CAP_PCI_SEGMENT:
        case KVM_CAP_DEBUGREGS:
        case KVM_CAP_X86_ROBUST_SINGLESTEP:
        case KVM_CAP_XSAVE:
        case KVM_CAP_ASYNC_PF:
        case KVM_CAP_GET_TSC_KHZ:
        case KVM_CAP_KVMCLOCK_CTRL:
        case KVM_CAP_READONLY_MEM:
        case KVM_CAP_HYPERV_TIME:
        case KVM_CAP_IOAPIC_POLARITY_IGNORED:
#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
        case KVM_CAP_ASSIGN_DEV_IRQ:
        case KVM_CAP_PCI_2_3:
#endif
                r = 1;
                break;
        case KVM_CAP_COALESCED_MMIO:
                r = KVM_COALESCED_MMIO_PAGE_OFFSET;
                break;
        case KVM_CAP_VAPIC:
                r = !kvm_x86_ops->cpu_has_accelerated_tpr();
                break;
        case KVM_CAP_NR_VCPUS:
                r = KVM_SOFT_MAX_VCPUS;
                break;
        case KVM_CAP_MAX_VCPUS:
                r = KVM_MAX_VCPUS;
                break;
        case KVM_CAP_NR_MEMSLOTS:
                r = KVM_USER_MEM_SLOTS;
                break;
        case KVM_CAP_PV_MMU:    /* obsolete */
                r = 0;
                break;
#ifdef CONFIG_KVM_DEVICE_ASSIGNMENT
        case KVM_CAP_IOMMU:
                r = iommu_present(&pci_bus_type);
                break;
#endif
        case KVM_CAP_MCE:
                r = KVM_MAX_MCE_BANKS;
                break;
        case KVM_CAP_XCRS:
                r = cpu_has_xsave;
                break;
        case KVM_CAP_TSC_CONTROL:
                r = kvm_has_tsc_control;
                break;
        case KVM_CAP_TSC_DEADLINE_TIMER:
                r = boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER);
                break;
        default:
                r = 0;
                break;
        }
        return r;

}

long kvm_arch_dev_ioctl(struct file *filp,
                        unsigned int ioctl, unsigned long arg)
{
        void __user *argp = (void __user *)arg;
        long r;

        switch (ioctl) {
        case KVM_GET_MSR_INDEX_LIST: {
                struct kvm_msr_list __user *user_msr_list = argp;
                struct kvm_msr_list msr_list;
                unsigned n;

                r = -EFAULT;
                if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
                        goto out;
                n = msr_list.nmsrs;
                msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs);
                if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
                        goto out;
                r = -E2BIG;
                if (n < msr_list.nmsrs)
                        goto out;
                r = -EFAULT;
                if (copy_to_user(user_msr_list->indices, &msrs_to_save,
                                 num_msrs_to_save * sizeof(u32)))
                        goto out;
                if (copy_to_user(user_msr_list->indices + num_msrs_to_save,
                                 &emulated_msrs,
                                 ARRAY_SIZE(emulated_msrs) * sizeof(u32)))
                        goto out;
                r = 0;
                break;
        }
        case KVM_GET_SUPPORTED_CPUID:
        case KVM_GET_EMULATED_CPUID: {
                struct kvm_cpuid2 __user *cpuid_arg = argp;
                struct kvm_cpuid2 cpuid;

                r = -EFAULT;
                if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
                        goto out;

                r = kvm_dev_ioctl_get_cpuid(&cpuid, cpuid_arg->entries,
                                            ioctl);
                if (r)
                        goto out;

                r = -EFAULT;
                if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid))
                        goto out;
                r = 0;
                break;
        }
        case KVM_X86_GET_MCE_CAP_SUPPORTED: {
                u64 mce_cap;

                mce_cap = KVM_MCE_CAP_SUPPORTED;
                r = -EFAULT;
                if (copy_to_user(argp, &mce_cap, sizeof mce_cap))
                        goto out;
                r = 0;
                break;
        }
        default:
                r = -EINVAL;
        }
out:
        return r;
}

static void wbinvd_ipi(void *garbage)
{
        wbinvd();
}

static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
{
        return kvm_arch_has_noncoherent_dma(vcpu->kvm);
}

void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
        /* Address WBINVD may be executed by guest */
        if (need_emulate_wbinvd(vcpu)) {
                if (kvm_x86_ops->has_wbinvd_exit())
                        cpumask_set_cpu(cpu, vcpu->arch.wbinvd_dirty_mask);
                else if (vcpu->cpu != -1 && vcpu->cpu != cpu)
                        smp_call_function_single(vcpu->cpu,
                                        wbinvd_ipi, NULL, 1);
        }

        kvm_x86_ops->vcpu_load(vcpu, cpu);

        /* Apply any externally detected TSC adjustments (due to suspend) */
        if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
                adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
                vcpu->arch.tsc_offset_adjustment = 0;
                set_bit(KVM_REQ_CLOCK_UPDATE, &vcpu->requests);
        }

        if (unlikely(vcpu->cpu != cpu) || check_tsc_unstable()) {
                s64 tsc_delta = !vcpu->arch.last_host_tsc ? 0 :
                                native_read_tsc() - vcpu->arch.last_host_tsc;
                if (tsc_delta < 0)
                        mark_tsc_unstable("KVM discovered backwards TSC");
                if (check_tsc_unstable()) {
                        u64 offset = kvm_x86_ops->compute_tsc_offset(vcpu,
                                                vcpu->arch.last_guest_tsc);
                        kvm_x86_ops->write_tsc_offset(vcpu, offset);
                        vcpu->arch.tsc_catchup = 1;
                }
                /*
                 * On a host with synchronized TSC, there is no need to update
                 * kvmclock on vcpu->cpu migration
                 */
                if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
                        kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
                if (vcpu->cpu != cpu)
                        kvm_migrate_timers(vcpu);
                vcpu->cpu = cpu;
        }

        accumulate_steal_time(vcpu);
        kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
}

void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
        kvm_x86_ops->vcpu_put(vcpu);
        kvm_put_guest_fpu(vcpu);
        vcpu->arch.last_host_tsc = native_read_tsc();
}

static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
                                    struct kvm_lapic_state *s)
{
        kvm_x86_ops->sync_pir_to_irr(vcpu);
        memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s);

        return 0;
}

static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
                                    struct kvm_lapic_state *s)
{
        kvm_apic_post_state_restore(vcpu, s);
        update_cr8_intercept(vcpu);

        return 0;
}

static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
                                    struct kvm_interrupt *irq)
{
        if (irq->irq >= KVM_NR_INTERRUPTS)
                return -EINVAL;
        if (irqchip_in_kernel(vcpu->kvm))
                return -ENXIO;

        kvm_queue_interrupt(vcpu, irq->irq, false);
        kvm_make_request(KVM_REQ_EVENT, vcpu);

        return 0;
}

static int kvm_vcpu_ioctl_nmi(struct kvm_vcpu *vcpu)
{
        kvm_inject_nmi(vcpu);

        return 0;
}

static int vcpu_ioctl_tpr_access_reporting(struct kvm_vcpu *vcpu,
                                           struct kvm_tpr_access_ctl *tac)
{
        if (tac->flags)
                return -EINVAL;
        vcpu->arch.tpr_access_reporting = !!tac->enabled;
        return 0;
}

static int kvm_vcpu_ioctl_x86_setup_mce(struct kvm_vcpu *vcpu,
                                        u64 mcg_cap)
{
        int r;
        unsigned bank_num = mcg_cap & 0xff, bank;

        r = -EINVAL;
        if (!bank_num || bank_num >= KVM_MAX_MCE_BANKS)
                goto out;
        if (mcg_cap & ~(KVM_MCE_CAP_SUPPORTED | 0xff | 0xff0000))
                goto out;
        r = 0;
        vcpu->arch.mcg_cap = mcg_cap;
        /* Init IA32_MCG_CTL to all 1s */
        if (mcg_cap & MCG_CTL_P)
                vcpu->arch.mcg_ctl = ~(u64)0;
        /* Init IA32_MCi_CTL to all 1s */
        for (bank = 0; bank < bank_num; bank++)
                vcpu->arch.mce_banks[bank*4] = ~(u64)0;
out:
        return r;
}

static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu,
                                      struct kvm_x86_mce *mce)
{
        u64 mcg_cap = vcpu->arch.mcg_cap;
        unsigned bank_num = mcg_cap & 0xff;
        u64 *banks = vcpu->arch.mce_banks;

        if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL))
                return -EINVAL;
        /*
         * if IA32_MCG_CTL is not all 1s, the uncorrected error
         * reporting is disabled
         */
        if ((mce->status & MCI_STATUS_UC) && (mcg_cap & MCG_CTL_P) &&
            vcpu->arch.mcg_ctl != ~(u64)0)
                return 0;
        banks += 4 * mce->bank;
        /*
         * if IA32_MCi_CTL is not all 1s, the uncorrected error
         * reporting is disabled for the bank
         */
        if ((mce->status & MCI_STATUS_UC) && banks[0] != ~(u64)0)
                return 0;
        if (mce->status & MCI_STATUS_UC) {
                if ((vcpu->arch.mcg_status & MCG_STATUS_MCIP) ||
                    !kvm_read_cr4_bits(vcpu, X86_CR4_MCE)) {
                        kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
                        return 0;
                }
                if (banks[1] & MCI_STATUS_VAL)
                        mce->status |= MCI_STATUS_OVER;
                banks[2] = mce->addr;
                banks[3] = mce->misc;
                vcpu->arch.mcg_status = mce->mcg_status;
                banks[1] = mce->status;
                kvm_queue_exception(vcpu, MC_VECTOR);
        } else if (!(banks[1] & MCI_STATUS_VAL)
                   || !(banks[1] & MCI_STATUS_UC)) {
                if (banks[1] & MCI_STATUS_VAL)
                        mce->status |= MCI_STATUS_OVER;
                banks[2] = mce->addr;
                banks[3] = mce->misc;
                banks[1] = mce->status;
        } else
                banks[1] |= MCI_STATUS_OVER;
        return 0;
}

static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu,
                                               struct kvm_vcpu_events *events)
{
        process_nmi(vcpu);
        events->exception.injected =
                vcpu->arch.exception.pending &&
                !kvm_exception_is_soft(vcpu->arch.exception.nr);
        events->exception.nr = vcpu->arch.exception.nr;
        events->exception.has_error_code = vcpu->arch.exception.has_error_code;
        events->exception.pad = 0;
        events->exception.error_code = vcpu->arch.exception.error_code;

        events->interrupt.injected =
                vcpu->arch.interrupt.pending && !vcpu->arch.interrupt.soft;
        events->interrupt.nr = vcpu->arch.interrupt.nr;
        events->interrupt.soft = 0;
        events->interrupt.shadow =
                kvm_x86_ops->get_interrupt_shadow(vcpu,
                        KVM_X86_SHADOW_INT_MOV_SS | KVM_X86_SHADOW_INT_STI);

        events->nmi.injected = vcpu->arch.nmi_injected;
        events->nmi.pending = vcpu->arch.nmi_pending != 0;
        events->nmi.masked = kvm_x86_ops->get_nmi_mask(vcpu);
        events->nmi.pad = 0;

        events->sipi_vector = 0; /* never valid when reporting to user space */

        events->flags = (KVM_VCPUEVENT_VALID_NMI_PENDING
                         | KVM_VCPUEVENT_VALID_SHADOW);
        memset(&events->reserved, 0, sizeof(events->reserved));
}