From aca7f96600b170e470b3056aba0ed8d7df8d330d Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Mon, 17 Sep 2007 14:57:49 -0500 Subject: KVM: x86 emulator: Add vmmcall/vmcall to x86_emulate (v3) Add vmmcall/vmcall to x86_emulate. Future patch will implement functionality for these instructions. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index bd46de6bf891..84af9cc737fa 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1380,6 +1380,12 @@ twobyte_insn: u16 size; unsigned long address; + case 0: /* vmcall */ + if (modrm_mod != 3 || modrm_rm != 1) + goto cannot_emulate; + + /* nop */ + break; case 2: /* lgdt */ rc = read_descriptor(ctxt, ops, src.ptr, &size, &address, op_bytes); @@ -1387,12 +1393,17 @@ twobyte_insn: goto done; realmode_lgdt(ctxt->vcpu, size, address); break; - case 3: /* lidt */ - rc = read_descriptor(ctxt, ops, src.ptr, - &size, &address, op_bytes); - if (rc) - goto done; - realmode_lidt(ctxt->vcpu, size, address); + case 3: /* lidt/vmmcall */ + if (modrm_mod == 3 && modrm_rm == 1) { + /* nop */ + } else { + rc = read_descriptor(ctxt, ops, src.ptr, + &size, &address, + op_bytes); + if (rc) + goto done; + realmode_lidt(ctxt->vcpu, size, address); + } break; case 4: /* smsw */ if (modrm_mod != 3) -- cgit v1.2.3 From 7aa81cc04781b5b99a0647ec04533599d78cd219 Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Mon, 17 Sep 2007 14:57:50 -0500 Subject: KVM: Refactor hypercall infrastructure (v3) This patch refactors the current hypercall infrastructure to better support live migration and SMP. It eliminates the hypercall page by trapping the UD exception that would occur if you used the wrong hypercall instruction for the underlying architecture and replacing it with the right one lazily. A fall-out of this patch is that the unhandled hypercalls no longer trap to userspace. There is very little reason though to use a hypercall to communicate with userspace as PIO or MMIO can be used. There is no code in tree that uses userspace hypercalls. [avi: fix #ud injection on vmx] Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 8 +-- drivers/kvm/kvm_main.c | 156 ++++++++++++++------------------------------- drivers/kvm/svm.c | 19 +++++- drivers/kvm/vmx.c | 29 ++++++++- drivers/kvm/x86_emulate.c | 11 +++- include/linux/kvm_para.h | 159 ++++++++++++++++++++++++++++------------------ 6 files changed, 199 insertions(+), 183 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3b0bc4bda5f2..da9c3aa1c08c 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -46,6 +46,7 @@ #define KVM_MAX_CPUID_ENTRIES 40 #define DE_VECTOR 0 +#define UD_VECTOR 6 #define NM_VECTOR 7 #define DF_VECTOR 8 #define TS_VECTOR 10 @@ -317,9 +318,6 @@ struct kvm_vcpu { unsigned long cr0; unsigned long cr2; unsigned long cr3; - gpa_t para_state_gpa; - struct page *para_state_page; - gpa_t hypercall_gpa; unsigned long cr4; unsigned long cr8; u64 pdptrs[4]; /* pae */ @@ -622,7 +620,9 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); int kvm_mmu_load(struct kvm_vcpu *vcpu); void kvm_mmu_unload(struct kvm_vcpu *vcpu); -int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run); +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu); static inline void kvm_guest_enter(void) { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index c0f372f1d761..1c662f63b7a9 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -1362,51 +1363,61 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_emulate_halt); -int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run) +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) { - unsigned long nr, a0, a1, a2, a3, a4, a5, ret; + unsigned long nr, a0, a1, a2, a3, ret; kvm_x86_ops->cache_regs(vcpu); - ret = -KVM_EINVAL; -#ifdef CONFIG_X86_64 - if (is_long_mode(vcpu)) { - nr = vcpu->regs[VCPU_REGS_RAX]; - a0 = vcpu->regs[VCPU_REGS_RDI]; - a1 = vcpu->regs[VCPU_REGS_RSI]; - a2 = vcpu->regs[VCPU_REGS_RDX]; - a3 = vcpu->regs[VCPU_REGS_RCX]; - a4 = vcpu->regs[VCPU_REGS_R8]; - a5 = vcpu->regs[VCPU_REGS_R9]; - } else -#endif - { - nr = vcpu->regs[VCPU_REGS_RBX] & -1u; - a0 = vcpu->regs[VCPU_REGS_RAX] & -1u; - a1 = vcpu->regs[VCPU_REGS_RCX] & -1u; - a2 = vcpu->regs[VCPU_REGS_RDX] & -1u; - a3 = vcpu->regs[VCPU_REGS_RSI] & -1u; - a4 = vcpu->regs[VCPU_REGS_RDI] & -1u; - a5 = vcpu->regs[VCPU_REGS_RBP] & -1u; + + nr = vcpu->regs[VCPU_REGS_RAX]; + a0 = vcpu->regs[VCPU_REGS_RBX]; + a1 = vcpu->regs[VCPU_REGS_RCX]; + a2 = vcpu->regs[VCPU_REGS_RDX]; + a3 = vcpu->regs[VCPU_REGS_RSI]; + + if (!is_long_mode(vcpu)) { + nr &= 0xFFFFFFFF; + a0 &= 0xFFFFFFFF; + a1 &= 0xFFFFFFFF; + a2 &= 0xFFFFFFFF; + a3 &= 0xFFFFFFFF; } + switch (nr) { default: - run->hypercall.nr = nr; - run->hypercall.args[0] = a0; - run->hypercall.args[1] = a1; - run->hypercall.args[2] = a2; - run->hypercall.args[3] = a3; - run->hypercall.args[4] = a4; - run->hypercall.args[5] = a5; - run->hypercall.ret = ret; - run->hypercall.longmode = is_long_mode(vcpu); - kvm_x86_ops->decache_regs(vcpu); - return 0; + ret = -KVM_ENOSYS; + break; } vcpu->regs[VCPU_REGS_RAX] = ret; kvm_x86_ops->decache_regs(vcpu); - return 1; + return 0; +} +EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu) +{ + char instruction[3]; + int ret = 0; + + mutex_lock(&vcpu->kvm->lock); + + /* + * Blow out the MMU to ensure that no other VCPU has an active mapping + * to ensure that the updated hypercall appears atomically across all + * VCPUs. + */ + kvm_mmu_zap_all(vcpu->kvm); + + kvm_x86_ops->cache_regs(vcpu); + kvm_x86_ops->patch_hypercall(vcpu, instruction); + if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu) + != X86EMUL_CONTINUE) + ret = -EFAULT; + + mutex_unlock(&vcpu->kvm->lock); + + return ret; } -EXPORT_SYMBOL_GPL(kvm_hypercall); static u64 mk_cr_64(u64 curr_cr, u32 new_val) { @@ -1474,75 +1485,6 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, } } -/* - * Register the para guest with the host: - */ -static int vcpu_register_para(struct kvm_vcpu *vcpu, gpa_t para_state_gpa) -{ - struct kvm_vcpu_para_state *para_state; - hpa_t para_state_hpa, hypercall_hpa; - struct page *para_state_page; - unsigned char *hypercall; - gpa_t hypercall_gpa; - - printk(KERN_DEBUG "kvm: guest trying to enter paravirtual mode\n"); - printk(KERN_DEBUG ".... para_state_gpa: %08Lx\n", para_state_gpa); - - /* - * Needs to be page aligned: - */ - if (para_state_gpa != PAGE_ALIGN(para_state_gpa)) - goto err_gp; - - para_state_hpa = gpa_to_hpa(vcpu, para_state_gpa); - printk(KERN_DEBUG ".... para_state_hpa: %08Lx\n", para_state_hpa); - if (is_error_hpa(para_state_hpa)) - goto err_gp; - - mark_page_dirty(vcpu->kvm, para_state_gpa >> PAGE_SHIFT); - para_state_page = pfn_to_page(para_state_hpa >> PAGE_SHIFT); - para_state = kmap(para_state_page); - - printk(KERN_DEBUG ".... guest version: %d\n", para_state->guest_version); - printk(KERN_DEBUG ".... size: %d\n", para_state->size); - - para_state->host_version = KVM_PARA_API_VERSION; - /* - * We cannot support guests that try to register themselves - * with a newer API version than the host supports: - */ - if (para_state->guest_version > KVM_PARA_API_VERSION) { - para_state->ret = -KVM_EINVAL; - goto err_kunmap_skip; - } - - hypercall_gpa = para_state->hypercall_gpa; - hypercall_hpa = gpa_to_hpa(vcpu, hypercall_gpa); - printk(KERN_DEBUG ".... hypercall_hpa: %08Lx\n", hypercall_hpa); - if (is_error_hpa(hypercall_hpa)) { - para_state->ret = -KVM_EINVAL; - goto err_kunmap_skip; - } - - printk(KERN_DEBUG "kvm: para guest successfully registered.\n"); - vcpu->para_state_page = para_state_page; - vcpu->para_state_gpa = para_state_gpa; - vcpu->hypercall_gpa = hypercall_gpa; - - mark_page_dirty(vcpu->kvm, hypercall_gpa >> PAGE_SHIFT); - hypercall = kmap_atomic(pfn_to_page(hypercall_hpa >> PAGE_SHIFT), - KM_USER1) + (hypercall_hpa & ~PAGE_MASK); - kvm_x86_ops->patch_hypercall(vcpu, hypercall); - kunmap_atomic(hypercall, KM_USER1); - - para_state->ret = 0; -err_kunmap_skip: - kunmap(para_state_page); - return 0; -err_gp: - return 1; -} - int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) { u64 data; @@ -1656,12 +1598,6 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) case MSR_IA32_MISC_ENABLE: vcpu->ia32_misc_enable_msr = data; break; - /* - * This is the 'probe whether the host is KVM' logic: - */ - case MSR_KVM_API_MAGIC: - return vcpu_register_para(vcpu, data); - default: pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); return 1; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index ced4ac1955db..794d95416f7b 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -476,7 +476,8 @@ static void init_vmcb(struct vmcb *vmcb) INTERCEPT_DR5_MASK | INTERCEPT_DR7_MASK; - control->intercept_exceptions = 1 << PF_VECTOR; + control->intercept_exceptions = (1 << PF_VECTOR) | + (1 << UD_VECTOR); control->intercept = (1ULL << INTERCEPT_INTR) | @@ -979,6 +980,17 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) return 0; } +static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + int er; + + er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0); + if (er != EMULATE_DONE) + inject_ud(&svm->vcpu); + + return 1; +} + static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); @@ -1045,7 +1057,8 @@ static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { svm->next_rip = svm->vmcb->save.rip + 3; skip_emulated_instruction(&svm->vcpu); - return kvm_hypercall(&svm->vcpu, kvm_run); + kvm_emulate_hypercall(&svm->vcpu); + return 1; } static int invalid_op_interception(struct vcpu_svm *svm, @@ -1241,6 +1254,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, [SVM_EXIT_WRITE_DR3] = emulate_on_interception, [SVM_EXIT_WRITE_DR5] = emulate_on_interception, [SVM_EXIT_WRITE_DR7] = emulate_on_interception, + [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception, [SVM_EXIT_INTR] = nop_on_interception, @@ -1675,7 +1689,6 @@ svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) hypercall[0] = 0x0f; hypercall[1] = 0x01; hypercall[2] = 0xd9; - hypercall[3] = 0xc3; } static void svm_check_processor_compat(void *rtn) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 5b397b6c9f93..47c827d3007c 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -164,6 +164,13 @@ static inline int is_no_device(u32 intr_info) (INTR_TYPE_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK); } +static inline int is_invalid_opcode(u32 intr_info) +{ + return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | + INTR_INFO_VALID_MASK)) == + (INTR_TYPE_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK); +} + static inline int is_external_interrupt(u32 intr_info) { return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK)) @@ -315,7 +322,7 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu) { u32 eb; - eb = 1u << PF_VECTOR; + eb = (1u << PF_VECTOR) | (1u << UD_VECTOR); if (!vcpu->fpu_active) eb |= 1u << NM_VECTOR; if (vcpu->guest_debug.enabled) @@ -560,6 +567,14 @@ static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) INTR_INFO_VALID_MASK); } +static void vmx_inject_ud(struct kvm_vcpu *vcpu) +{ + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + UD_VECTOR | + INTR_TYPE_EXCEPTION | + INTR_INFO_VALID_MASK); +} + /* * Swap MSR entry in host/guest MSR entry array. */ @@ -1771,6 +1786,14 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } + if (is_invalid_opcode(intr_info)) { + er = emulate_instruction(vcpu, kvm_run, 0, 0); + if (er != EMULATE_DONE) + vmx_inject_ud(vcpu); + + return 1; + } + error_code = 0; rip = vmcs_readl(GUEST_RIP); if (intr_info & INTR_INFO_DELIEVER_CODE_MASK) @@ -1873,7 +1896,6 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) hypercall[0] = 0x0f; hypercall[1] = 0x01; hypercall[2] = 0xc1; - hypercall[3] = 0xc3; } static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) @@ -2059,7 +2081,8 @@ static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { skip_emulated_instruction(vcpu); - return kvm_hypercall(vcpu, kvm_run); + kvm_emulate_hypercall(vcpu); + return 1; } /* diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 84af9cc737fa..f12bc2c74040 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1384,7 +1384,11 @@ twobyte_insn: if (modrm_mod != 3 || modrm_rm != 1) goto cannot_emulate; - /* nop */ + rc = kvm_fix_hypercall(ctxt->vcpu); + if (rc) + goto done; + + kvm_emulate_hypercall(ctxt->vcpu); break; case 2: /* lgdt */ rc = read_descriptor(ctxt, ops, src.ptr, @@ -1395,7 +1399,10 @@ twobyte_insn: break; case 3: /* lidt/vmmcall */ if (modrm_mod == 3 && modrm_rm == 1) { - /* nop */ + rc = kvm_fix_hypercall(ctxt->vcpu); + if (rc) + goto done; + kvm_emulate_hypercall(ctxt->vcpu); } else { rc = read_descriptor(ctxt, ops, src.ptr, &size, &address, diff --git a/include/linux/kvm_para.h b/include/linux/kvm_para.h index 3b292565a693..cc5dfb433706 100644 --- a/include/linux/kvm_para.h +++ b/include/linux/kvm_para.h @@ -1,73 +1,110 @@ #ifndef __LINUX_KVM_PARA_H #define __LINUX_KVM_PARA_H -/* - * Guest OS interface for KVM paravirtualization - * - * Note: this interface is totally experimental, and is certain to change - * as we make progress. +/* This CPUID returns the signature 'KVMKVMKVM' in ebx, ecx, and edx. It + * should be used to determine that a VM is running under KVM. */ +#define KVM_CPUID_SIGNATURE 0x40000000 -/* - * Per-VCPU descriptor area shared between guest and host. Writable to - * both guest and host. Registered with the host by the guest when - * a guest acknowledges paravirtual mode. - * - * NOTE: all addresses are guest-physical addresses (gpa), to make it - * easier for the hypervisor to map between the various addresses. - */ -struct kvm_vcpu_para_state { - /* - * API version information for compatibility. If there's any support - * mismatch (too old host trying to execute too new guest) then - * the host will deny entry into paravirtual mode. Any other - * combination (new host + old guest and new host + new guest) - * is supposed to work - new host versions will support all old - * guest API versions. - */ - u32 guest_version; - u32 host_version; - u32 size; - u32 ret; - - /* - * The address of the vm exit instruction (VMCALL or VMMCALL), - * which the host will patch according to the CPU model the - * VM runs on: - */ - u64 hypercall_gpa; - -} __attribute__ ((aligned(PAGE_SIZE))); - -#define KVM_PARA_API_VERSION 1 - -/* - * This is used for an RDMSR's ECX parameter to probe for a KVM host. - * Hopefully no CPU vendor will use up this number. This is placed well - * out of way of the typical space occupied by CPU vendors' MSR indices, - * and we think (or at least hope) it wont be occupied in the future - * either. +/* This CPUID returns a feature bitmap in eax. Before enabling a particular + * paravirtualization, the appropriate feature bit should be checked. */ -#define MSR_KVM_API_MAGIC 0x87655678 +#define KVM_CPUID_FEATURES 0x40000001 -#define KVM_EINVAL 1 +/* Return values for hypercalls */ +#define KVM_ENOSYS 1000 -/* - * Hypercall calling convention: - * - * Each hypercall may have 0-6 parameters. - * - * 64-bit hypercall index is in RAX, goes from 0 to __NR_hypercalls-1 - * - * 64-bit parameters 1-6 are in the standard gcc x86_64 calling convention - * order: RDI, RSI, RDX, RCX, R8, R9. - * - * 32-bit index is EBX, parameters are: EAX, ECX, EDX, ESI, EDI, EBP. - * (the first 3 are according to the gcc regparm calling convention) +#ifdef __KERNEL__ +#include + +/* This instruction is vmcall. On non-VT architectures, it will generate a + * trap that we will then rewrite to the appropriate instruction. + */ +#define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1" + +/* For KVM hypercalls, a three-byte sequence of either the vmrun or the vmmrun + * instruction. The hypervisor may replace it with something else but only the + * instructions are guaranteed to be supported. * - * No registers are clobbered by the hypercall, except that the - * return value is in RAX. + * Up to four arguments may be passed in rbx, rcx, rdx, and rsi respectively. + * The hypercall number should be placed in rax and the return value will be + * placed in rax. No other registers will be clobbered unless explicited + * noted by the particular hypercall. */ -#define __NR_hypercalls 0 + +static inline long kvm_hypercall0(unsigned int nr) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr)); + return ret; +} + +static inline long kvm_hypercall1(unsigned int nr, unsigned long p1) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1)); + return ret; +} + +static inline long kvm_hypercall2(unsigned int nr, unsigned long p1, + unsigned long p2) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2)); + return ret; +} + +static inline long kvm_hypercall3(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2), "d"(p3)); + return ret; +} + +static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, + unsigned long p2, unsigned long p3, + unsigned long p4) +{ + long ret; + asm volatile(KVM_HYPERCALL + : "=a"(ret) + : "a"(nr), "b"(p1), "c"(p2), "d"(p3), "S"(p4)); + return ret; +} + +static inline int kvm_para_available(void) +{ + unsigned int eax, ebx, ecx, edx; + char signature[13]; + + cpuid(KVM_CPUID_SIGNATURE, &eax, &ebx, &ecx, &edx); + memcpy(signature + 0, &ebx, 4); + memcpy(signature + 4, &ecx, 4); + memcpy(signature + 8, &edx, 4); + signature[12] = 0; + + if (strcmp(signature, "KVMKVMKVM") == 0) + return 1; + + return 0; +} + +static inline int kvm_para_has_feature(unsigned int feature) +{ + if (cpuid_eax(KVM_CPUID_FEATURES) & (1UL << feature)) + return 1; + return 0; +} + +#endif #endif -- cgit v1.2.3 From a7ddce3afc8326870b9e5e02fa41e028bffb10a5 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:26:38 +0200 Subject: KVM: x86 emulator: remove unused functions Remove #ifdef functions never used Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 39 --------------------------------------- 1 file changed, 39 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index f12bc2c74040..9ea82f84743e 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1639,42 +1639,3 @@ cannot_emulate: DPRINTF("Cannot emulate %02x\n", b); return -1; } - -#ifdef __XEN__ - -#include -#include - -int -x86_emulate_read_std(unsigned long addr, - unsigned long *val, - unsigned int bytes, struct x86_emulate_ctxt *ctxt) -{ - unsigned int rc; - - *val = 0; - - if ((rc = copy_from_user((void *)val, (void *)addr, bytes)) != 0) { - propagate_page_fault(addr + bytes - rc, 0); /* read fault */ - return X86EMUL_PROPAGATE_FAULT; - } - - return X86EMUL_CONTINUE; -} - -int -x86_emulate_write_std(unsigned long addr, - unsigned long val, - unsigned int bytes, struct x86_emulate_ctxt *ctxt) -{ - unsigned int rc; - - if ((rc = copy_to_user((void *)addr, (void *)&val, bytes)) != 0) { - propagate_page_fault(addr + bytes - rc, PGERR_write_access); - return X86EMUL_PROPAGATE_FAULT; - } - - return X86EMUL_CONTINUE; -} - -#endif -- cgit v1.2.3 From e4e03deda83b1f2fc37ccbfc1eef27e86e8ed4e9 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:52:50 +0200 Subject: KVM: x86 emulator: move all x86_emulate_memop() to a structure Move all x86_emulate_memop() common variables between decode and execute to a structure decode_cache. This will help in later separating decode and emulate. struct decode_cache { u8 twobyte; u8 b; u8 lock_prefix; u8 rep_prefix; u8 op_bytes; u8 ad_bytes; struct operand src; struct operand dst; unsigned long *override_base; unsigned int d; unsigned long regs[NR_VCPU_REGS]; unsigned long eip; /* modrm */ u8 modrm; u8 modrm_mod; u8 modrm_reg; u8 modrm_rm; u8 use_modrm_ea; unsigned long modrm_ea; unsigned long modrm_val; }; Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 919 ++++++++++++++++++++++++---------------------- drivers/kvm/x86_emulate.h | 34 ++ 2 files changed, 518 insertions(+), 435 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 9ea82f84743e..d7026cb8fc70 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -222,13 +222,6 @@ static u16 twobyte_table[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; -/* Type, address-of, and value of an instruction's operand. */ -struct operand { - enum { OP_REG, OP_MEM, OP_IMM } type; - unsigned int bytes; - unsigned long val, orig_val, *ptr; -}; - /* EFLAGS bit definitions. */ #define EFLG_OF (1<<11) #define EFLG_DF (1<<10) @@ -431,24 +424,26 @@ struct operand { /* Access/update address held in a register, based on addressing mode. */ #define address_mask(reg) \ - ((ad_bytes == sizeof(unsigned long)) ? \ - (reg) : ((reg) & ((1UL << (ad_bytes << 3)) - 1))) + ((c->ad_bytes == sizeof(unsigned long)) ? \ + (reg) : ((reg) & ((1UL << (c->ad_bytes << 3)) - 1))) #define register_address(base, reg) \ ((base) + address_mask(reg)) #define register_address_increment(reg, inc) \ do { \ /* signed type ensures sign extension to long */ \ int _inc = (inc); \ - if ( ad_bytes == sizeof(unsigned long) ) \ + if (c->ad_bytes == sizeof(unsigned long)) \ (reg) += _inc; \ else \ - (reg) = ((reg) & ~((1UL << (ad_bytes << 3)) - 1)) | \ - (((reg) + _inc) & ((1UL << (ad_bytes << 3)) - 1)); \ + (reg) = ((reg) & \ + ~((1UL << (c->ad_bytes << 3)) - 1)) | \ + (((reg) + _inc) & \ + ((1UL << (c->ad_bytes << 3)) - 1)); \ } while (0) #define JMP_REL(rel) \ do { \ - register_address_increment(_eip, rel); \ + register_address_increment(c->eip, rel); \ } while (0) /* @@ -524,39 +519,35 @@ static int test_cc(unsigned int condition, unsigned int flags) int x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { - unsigned d; - u8 b, sib, twobyte = 0, rex_prefix = 0; - u8 modrm, modrm_mod = 0, modrm_reg = 0, modrm_rm = 0; - unsigned long *override_base = NULL; - unsigned int op_bytes, ad_bytes, lock_prefix = 0, rep_prefix = 0, i; + struct decode_cache *c = &ctxt->decode; + u8 sib, rex_prefix = 0; + unsigned int i; int rc = 0; - struct operand src, dst; unsigned long cr2 = ctxt->cr2; int mode = ctxt->mode; - unsigned long modrm_ea; - int use_modrm_ea, index_reg = 0, base_reg = 0, scale, rip_relative = 0; + int index_reg = 0, base_reg = 0, scale, rip_relative = 0; int no_wb = 0; u64 msr_data; /* Shadow copy of register state. Committed on successful emulation. */ - unsigned long _regs[NR_VCPU_REGS]; - unsigned long _eip = ctxt->vcpu->rip, _eflags = ctxt->eflags; - unsigned long modrm_val = 0; + unsigned long _eflags = ctxt->eflags; - memcpy(_regs, ctxt->vcpu->regs, sizeof _regs); + memset(c, 0, sizeof(struct decode_cache)); + c->eip = ctxt->vcpu->rip; + memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); switch (mode) { case X86EMUL_MODE_REAL: case X86EMUL_MODE_PROT16: - op_bytes = ad_bytes = 2; + c->op_bytes = c->ad_bytes = 2; break; case X86EMUL_MODE_PROT32: - op_bytes = ad_bytes = 4; + c->op_bytes = c->ad_bytes = 4; break; #ifdef CONFIG_X86_64 case X86EMUL_MODE_PROT64: - op_bytes = 4; - ad_bytes = 8; + c->op_bytes = 4; + c->ad_bytes = 8; break; #endif default: @@ -565,40 +556,42 @@ x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* Legacy prefixes. */ for (i = 0; i < 8; i++) { - switch (b = insn_fetch(u8, 1, _eip)) { + switch (c->b = insn_fetch(u8, 1, c->eip)) { case 0x66: /* operand-size override */ - op_bytes ^= 6; /* switch between 2/4 bytes */ + c->op_bytes ^= 6; /* switch between 2/4 bytes */ break; case 0x67: /* address-size override */ if (mode == X86EMUL_MODE_PROT64) - ad_bytes ^= 12; /* switch between 4/8 bytes */ + /* switch between 4/8 bytes */ + c->ad_bytes ^= 12; else - ad_bytes ^= 6; /* switch between 2/4 bytes */ + /* switch between 2/4 bytes */ + c->ad_bytes ^= 6; break; case 0x2e: /* CS override */ - override_base = &ctxt->cs_base; + c->override_base = &ctxt->cs_base; break; case 0x3e: /* DS override */ - override_base = &ctxt->ds_base; + c->override_base = &ctxt->ds_base; break; case 0x26: /* ES override */ - override_base = &ctxt->es_base; + c->override_base = &ctxt->es_base; break; case 0x64: /* FS override */ - override_base = &ctxt->fs_base; + c->override_base = &ctxt->fs_base; break; case 0x65: /* GS override */ - override_base = &ctxt->gs_base; + c->override_base = &ctxt->gs_base; break; case 0x36: /* SS override */ - override_base = &ctxt->ss_base; + c->override_base = &ctxt->ss_base; break; case 0xf0: /* LOCK */ - lock_prefix = 1; + c->lock_prefix = 1; break; case 0xf2: /* REPNE/REPNZ */ case 0xf3: /* REP/REPE/REPZ */ - rep_prefix = 1; + c->rep_prefix = 1; break; default: goto done_prefixes; @@ -608,177 +601,182 @@ x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) done_prefixes: /* REX prefix. */ - if ((mode == X86EMUL_MODE_PROT64) && ((b & 0xf0) == 0x40)) { - rex_prefix = b; - if (b & 8) - op_bytes = 8; /* REX.W */ - modrm_reg = (b & 4) << 1; /* REX.R */ - index_reg = (b & 2) << 2; /* REX.X */ - modrm_rm = base_reg = (b & 1) << 3; /* REG.B */ - b = insn_fetch(u8, 1, _eip); + if ((mode == X86EMUL_MODE_PROT64) && ((c->b & 0xf0) == 0x40)) { + rex_prefix = c->b; + if (c->b & 8) + c->op_bytes = 8; /* REX.W */ + c->modrm_reg = (c->b & 4) << 1; /* REX.R */ + index_reg = (c->b & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (c->b & 1) << 3; /* REG.B */ + c->b = insn_fetch(u8, 1, c->eip); } /* Opcode byte(s). */ - d = opcode_table[b]; - if (d == 0) { + c->d = opcode_table[c->b]; + if (c->d == 0) { /* Two-byte opcode? */ - if (b == 0x0f) { - twobyte = 1; - b = insn_fetch(u8, 1, _eip); - d = twobyte_table[b]; + if (c->b == 0x0f) { + c->twobyte = 1; + c->b = insn_fetch(u8, 1, c->eip); + c->d = twobyte_table[c->b]; } /* Unrecognised? */ - if (d == 0) + if (c->d == 0) goto cannot_emulate; } /* ModRM and SIB bytes. */ - if (d & ModRM) { - modrm = insn_fetch(u8, 1, _eip); - modrm_mod |= (modrm & 0xc0) >> 6; - modrm_reg |= (modrm & 0x38) >> 3; - modrm_rm |= (modrm & 0x07); - modrm_ea = 0; - use_modrm_ea = 1; - - if (modrm_mod == 3) { - modrm_val = *(unsigned long *) - decode_register(modrm_rm, _regs, d & ByteOp); + if (c->d & ModRM) { + c->modrm = insn_fetch(u8, 1, c->eip); + c->modrm_mod |= (c->modrm & 0xc0) >> 6; + c->modrm_reg |= (c->modrm & 0x38) >> 3; + c->modrm_rm |= (c->modrm & 0x07); + c->modrm_ea = 0; + c->use_modrm_ea = 1; + + if (c->modrm_mod == 3) { + c->modrm_val = *(unsigned long *) + decode_register(c->modrm_rm, c->regs, c->d & ByteOp); goto modrm_done; } - if (ad_bytes == 2) { - unsigned bx = _regs[VCPU_REGS_RBX]; - unsigned bp = _regs[VCPU_REGS_RBP]; - unsigned si = _regs[VCPU_REGS_RSI]; - unsigned di = _regs[VCPU_REGS_RDI]; + if (c->ad_bytes == 2) { + unsigned bx = c->regs[VCPU_REGS_RBX]; + unsigned bp = c->regs[VCPU_REGS_RBP]; + unsigned si = c->regs[VCPU_REGS_RSI]; + unsigned di = c->regs[VCPU_REGS_RDI]; /* 16-bit ModR/M decode. */ - switch (modrm_mod) { + switch (c->modrm_mod) { case 0: - if (modrm_rm == 6) - modrm_ea += insn_fetch(u16, 2, _eip); + if (c->modrm_rm == 6) + c->modrm_ea += + insn_fetch(u16, 2, c->eip); break; case 1: - modrm_ea += insn_fetch(s8, 1, _eip); + c->modrm_ea += insn_fetch(s8, 1, c->eip); break; case 2: - modrm_ea += insn_fetch(u16, 2, _eip); + c->modrm_ea += insn_fetch(u16, 2, c->eip); break; } - switch (modrm_rm) { + switch (c->modrm_rm) { case 0: - modrm_ea += bx + si; + c->modrm_ea += bx + si; break; case 1: - modrm_ea += bx + di; + c->modrm_ea += bx + di; break; case 2: - modrm_ea += bp + si; + c->modrm_ea += bp + si; break; case 3: - modrm_ea += bp + di; + c->modrm_ea += bp + di; break; case 4: - modrm_ea += si; + c->modrm_ea += si; break; case 5: - modrm_ea += di; + c->modrm_ea += di; break; case 6: - if (modrm_mod != 0) - modrm_ea += bp; + if (c->modrm_mod != 0) + c->modrm_ea += bp; break; case 7: - modrm_ea += bx; + c->modrm_ea += bx; break; } - if (modrm_rm == 2 || modrm_rm == 3 || - (modrm_rm == 6 && modrm_mod != 0)) - if (!override_base) - override_base = &ctxt->ss_base; - modrm_ea = (u16)modrm_ea; + if (c->modrm_rm == 2 || c->modrm_rm == 3 || + (c->modrm_rm == 6 && c->modrm_mod != 0)) + if (!c->override_base) + c->override_base = &ctxt->ss_base; + c->modrm_ea = (u16)c->modrm_ea; } else { /* 32/64-bit ModR/M decode. */ - switch (modrm_rm) { + switch (c->modrm_rm) { case 4: case 12: - sib = insn_fetch(u8, 1, _eip); + sib = insn_fetch(u8, 1, c->eip); index_reg |= (sib >> 3) & 7; base_reg |= sib & 7; scale = sib >> 6; switch (base_reg) { case 5: - if (modrm_mod != 0) - modrm_ea += _regs[base_reg]; + if (c->modrm_mod != 0) + c->modrm_ea += + c->regs[base_reg]; else - modrm_ea += insn_fetch(s32, 4, _eip); + c->modrm_ea += + insn_fetch(s32, 4, c->eip); break; default: - modrm_ea += _regs[base_reg]; + c->modrm_ea += c->regs[base_reg]; } switch (index_reg) { case 4: break; default: - modrm_ea += _regs[index_reg] << scale; + c->modrm_ea += + c->regs[index_reg] << scale; } break; case 5: - if (modrm_mod != 0) - modrm_ea += _regs[modrm_rm]; + if (c->modrm_mod != 0) + c->modrm_ea += c->regs[c->modrm_rm]; else if (mode == X86EMUL_MODE_PROT64) rip_relative = 1; break; default: - modrm_ea += _regs[modrm_rm]; + c->modrm_ea += c->regs[c->modrm_rm]; break; } - switch (modrm_mod) { + switch (c->modrm_mod) { case 0: - if (modrm_rm == 5) - modrm_ea += insn_fetch(s32, 4, _eip); + if (c->modrm_rm == 5) + c->modrm_ea += + insn_fetch(s32, 4, c->eip); break; case 1: - modrm_ea += insn_fetch(s8, 1, _eip); + c->modrm_ea += insn_fetch(s8, 1, c->eip); break; case 2: - modrm_ea += insn_fetch(s32, 4, _eip); + c->modrm_ea += insn_fetch(s32, 4, c->eip); break; } } - if (!override_base) - override_base = &ctxt->ds_base; + if (!c->override_base) + c->override_base = &ctxt->ds_base; if (mode == X86EMUL_MODE_PROT64 && - override_base != &ctxt->fs_base && - override_base != &ctxt->gs_base) - override_base = NULL; + c->override_base != &ctxt->fs_base && + c->override_base != &ctxt->gs_base) + c->override_base = NULL; - if (override_base) - modrm_ea += *override_base; + if (c->override_base) + c->modrm_ea += *c->override_base; if (rip_relative) { - modrm_ea += _eip; - switch (d & SrcMask) { + c->modrm_ea += c->eip; + switch (c->d & SrcMask) { case SrcImmByte: - modrm_ea += 1; + c->modrm_ea += 1; break; case SrcImm: - if (d & ByteOp) - modrm_ea += 1; + if (c->d & ByteOp) + c->modrm_ea += 1; else - if (op_bytes == 8) - modrm_ea += 4; + if (c->op_bytes == 8) + c->modrm_ea += 4; else - modrm_ea += op_bytes; + c->modrm_ea += c->op_bytes; } } - if (ad_bytes != 8) - modrm_ea = (u32)modrm_ea; - cr2 = modrm_ea; + if (c->ad_bytes != 8) + c->modrm_ea = (u32)c->modrm_ea; + cr2 = c->modrm_ea; modrm_done: ; } @@ -787,200 +785,210 @@ done_prefixes: * Decode and fetch the source operand: register, memory * or immediate. */ - switch (d & SrcMask) { + switch (c->d & SrcMask) { case SrcNone: break; case SrcReg: - src.type = OP_REG; - if (d & ByteOp) { - src.ptr = decode_register(modrm_reg, _regs, + c->src.type = OP_REG; + if (c->d & ByteOp) { + c->src.ptr = + decode_register(c->modrm_reg, c->regs, (rex_prefix == 0)); - src.val = src.orig_val = *(u8 *) src.ptr; - src.bytes = 1; + c->src.val = c->src.orig_val = *(u8 *)c->src.ptr; + c->src.bytes = 1; } else { - src.ptr = decode_register(modrm_reg, _regs, 0); - switch ((src.bytes = op_bytes)) { + c->src.ptr = + decode_register(c->modrm_reg, c->regs, 0); + switch ((c->src.bytes = c->op_bytes)) { case 2: - src.val = src.orig_val = *(u16 *) src.ptr; + c->src.val = c->src.orig_val = + *(u16 *) c->src.ptr; break; case 4: - src.val = src.orig_val = *(u32 *) src.ptr; + c->src.val = c->src.orig_val = + *(u32 *) c->src.ptr; break; case 8: - src.val = src.orig_val = *(u64 *) src.ptr; + c->src.val = c->src.orig_val = + *(u64 *) c->src.ptr; break; } } break; case SrcMem16: - src.bytes = 2; + c->src.bytes = 2; goto srcmem_common; case SrcMem32: - src.bytes = 4; + c->src.bytes = 4; goto srcmem_common; case SrcMem: - src.bytes = (d & ByteOp) ? 1 : op_bytes; + c->src.bytes = (c->d & ByteOp) ? 1 : + c->op_bytes; /* Don't fetch the address for invlpg: it could be unmapped. */ - if (twobyte && b == 0x01 && modrm_reg == 7) + if (c->twobyte && c->b == 0x01 + && c->modrm_reg == 7) break; srcmem_common: /* * For instructions with a ModR/M byte, switch to register * access if Mod = 3. */ - if ((d & ModRM) && modrm_mod == 3) { - src.type = OP_REG; + if ((c->d & ModRM) && c->modrm_mod == 3) { + c->src.type = OP_REG; break; } - src.type = OP_MEM; - src.ptr = (unsigned long *)cr2; - src.val = 0; - if ((rc = ops->read_emulated((unsigned long)src.ptr, - &src.val, src.bytes, ctxt->vcpu)) != 0) + c->src.type = OP_MEM; + c->src.ptr = (unsigned long *)cr2; + c->src.val = 0; + if ((rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, ctxt->vcpu)) != 0) goto done; - src.orig_val = src.val; + c->src.orig_val = c->src.val; break; case SrcImm: - src.type = OP_IMM; - src.ptr = (unsigned long *)_eip; - src.bytes = (d & ByteOp) ? 1 : op_bytes; - if (src.bytes == 8) - src.bytes = 4; + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + if (c->src.bytes == 8) + c->src.bytes = 4; /* NB. Immediates are sign-extended as necessary. */ - switch (src.bytes) { + switch (c->src.bytes) { case 1: - src.val = insn_fetch(s8, 1, _eip); + c->src.val = insn_fetch(s8, 1, c->eip); break; case 2: - src.val = insn_fetch(s16, 2, _eip); + c->src.val = insn_fetch(s16, 2, c->eip); break; case 4: - src.val = insn_fetch(s32, 4, _eip); + c->src.val = insn_fetch(s32, 4, c->eip); break; } break; case SrcImmByte: - src.type = OP_IMM; - src.ptr = (unsigned long *)_eip; - src.bytes = 1; - src.val = insn_fetch(s8, 1, _eip); + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = 1; + c->src.val = insn_fetch(s8, 1, c->eip); break; } /* Decode and fetch the destination operand: register or memory. */ - switch (d & DstMask) { + switch (c->d & DstMask) { case ImplicitOps: /* Special instructions do their own operand decoding. */ goto special_insn; case DstReg: - dst.type = OP_REG; - if ((d & ByteOp) - && !(twobyte && (b == 0xb6 || b == 0xb7))) { - dst.ptr = decode_register(modrm_reg, _regs, + c->dst.type = OP_REG; + if ((c->d & ByteOp) + && !(c->twobyte && + (c->b == 0xb6 || c->b == 0xb7))) { + c->dst.ptr = + decode_register(c->modrm_reg, c->regs, (rex_prefix == 0)); - dst.val = *(u8 *) dst.ptr; - dst.bytes = 1; + c->dst.val = *(u8 *) c->dst.ptr; + c->dst.bytes = 1; } else { - dst.ptr = decode_register(modrm_reg, _regs, 0); - switch ((dst.bytes = op_bytes)) { + c->dst.ptr = + decode_register(c->modrm_reg, c->regs, 0); + switch ((c->dst.bytes = c->op_bytes)) { case 2: - dst.val = *(u16 *)dst.ptr; + c->dst.val = *(u16 *)c->dst.ptr; break; case 4: - dst.val = *(u32 *)dst.ptr; + c->dst.val = *(u32 *)c->dst.ptr; break; case 8: - dst.val = *(u64 *)dst.ptr; + c->dst.val = *(u64 *)c->dst.ptr; break; } } break; case DstMem: - dst.type = OP_MEM; - dst.ptr = (unsigned long *)cr2; - dst.bytes = (d & ByteOp) ? 1 : op_bytes; - dst.val = 0; - /* - * For instructions with a ModR/M byte, switch to register - * access if Mod = 3. - */ - if ((d & ModRM) && modrm_mod == 3) { - dst.type = OP_REG; + c->dst.type = OP_MEM; + c->dst.ptr = (unsigned long *)cr2; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.val = 0; + if ((c->d & ModRM) && c->modrm_mod == 3) { + c->dst.type = OP_REG; break; } - if (d & BitOp) { - unsigned long mask = ~(dst.bytes * 8 - 1); + if (c->d & BitOp) { + unsigned long mask = ~(c->dst.bytes * 8 - 1); - dst.ptr = (void *)dst.ptr + (src.val & mask) / 8; + c->dst.ptr = (void *)c->dst.ptr + + (c->src.val & mask) / 8; } - if (!(d & Mov) && /* optimisation - avoid slow emulated read */ - ((rc = ops->read_emulated((unsigned long)dst.ptr, - &dst.val, dst.bytes, ctxt->vcpu)) != 0)) + if (!(c->d & Mov) && + /* optimisation - avoid slow emulated read */ + ((rc = ops->read_emulated((unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0)) goto done; break; } - dst.orig_val = dst.val; + c->dst.orig_val = c->dst.val; - if (twobyte) + if (c->twobyte) goto twobyte_insn; - switch (b) { + switch (c->b) { case 0x00 ... 0x05: add: /* add */ - emulate_2op_SrcV("add", src, dst, _eflags); + emulate_2op_SrcV("add", c->src, c->dst, _eflags); break; case 0x08 ... 0x0d: or: /* or */ - emulate_2op_SrcV("or", src, dst, _eflags); + emulate_2op_SrcV("or", c->src, c->dst, _eflags); break; case 0x10 ... 0x15: adc: /* adc */ - emulate_2op_SrcV("adc", src, dst, _eflags); + emulate_2op_SrcV("adc", c->src, c->dst, _eflags); break; case 0x18 ... 0x1d: sbb: /* sbb */ - emulate_2op_SrcV("sbb", src, dst, _eflags); + emulate_2op_SrcV("sbb", c->src, c->dst, _eflags); break; case 0x20 ... 0x23: and: /* and */ - emulate_2op_SrcV("and", src, dst, _eflags); + emulate_2op_SrcV("and", c->src, c->dst, _eflags); break; case 0x24: /* and al imm8 */ - dst.type = OP_REG; - dst.ptr = &_regs[VCPU_REGS_RAX]; - dst.val = *(u8 *)dst.ptr; - dst.bytes = 1; - dst.orig_val = dst.val; + c->dst.type = OP_REG; + c->dst.ptr = &c->regs[VCPU_REGS_RAX]; + c->dst.val = *(u8 *)c->dst.ptr; + c->dst.bytes = 1; + c->dst.orig_val = c->dst.val; goto and; case 0x25: /* and ax imm16, or eax imm32 */ - dst.type = OP_REG; - dst.bytes = op_bytes; - dst.ptr = &_regs[VCPU_REGS_RAX]; - if (op_bytes == 2) - dst.val = *(u16 *)dst.ptr; + c->dst.type = OP_REG; + c->dst.bytes = c->op_bytes; + c->dst.ptr = &c->regs[VCPU_REGS_RAX]; + if (c->op_bytes == 2) + c->dst.val = *(u16 *)c->dst.ptr; else - dst.val = *(u32 *)dst.ptr; - dst.orig_val = dst.val; + c->dst.val = *(u32 *)c->dst.ptr; + c->dst.orig_val = c->dst.val; goto and; case 0x28 ... 0x2d: sub: /* sub */ - emulate_2op_SrcV("sub", src, dst, _eflags); + emulate_2op_SrcV("sub", c->src, c->dst, _eflags); break; case 0x30 ... 0x35: xor: /* xor */ - emulate_2op_SrcV("xor", src, dst, _eflags); + emulate_2op_SrcV("xor", c->src, c->dst, _eflags); break; case 0x38 ... 0x3d: cmp: /* cmp */ - emulate_2op_SrcV("cmp", src, dst, _eflags); + emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); break; case 0x63: /* movsxd */ if (mode != X86EMUL_MODE_PROT64) goto cannot_emulate; - dst.val = (s32) src.val; + c->dst.val = (s32) c->src.val; break; case 0x80 ... 0x83: /* Grp1 */ - switch (modrm_reg) { + switch (c->modrm_reg) { case 0: goto add; case 1: @@ -1001,155 +1009,164 @@ done_prefixes: break; case 0x84 ... 0x85: test: /* test */ - emulate_2op_SrcV("test", src, dst, _eflags); + emulate_2op_SrcV("test", c->src, c->dst, _eflags); break; case 0x86 ... 0x87: /* xchg */ /* Write back the register source. */ - switch (dst.bytes) { + switch (c->dst.bytes) { case 1: - *(u8 *) src.ptr = (u8) dst.val; + *(u8 *) c->src.ptr = (u8) c->dst.val; break; case 2: - *(u16 *) src.ptr = (u16) dst.val; + *(u16 *) c->src.ptr = (u16) c->dst.val; break; case 4: - *src.ptr = (u32) dst.val; + *c->src.ptr = (u32) c->dst.val; break; /* 64b reg: zero-extend */ case 8: - *src.ptr = dst.val; + *c->src.ptr = c->dst.val; break; } /* * Write back the memory destination with implicit LOCK * prefix. */ - dst.val = src.val; - lock_prefix = 1; + c->dst.val = c->src.val; + c->lock_prefix = 1; break; case 0x88 ... 0x8b: /* mov */ goto mov; case 0x8d: /* lea r16/r32, m */ - dst.val = modrm_val; + c->dst.val = c->modrm_val; break; case 0x8f: /* pop (sole member of Grp1a) */ /* 64-bit mode: POP always pops a 64-bit operand. */ if (mode == X86EMUL_MODE_PROT64) - dst.bytes = 8; - if ((rc = ops->read_std(register_address(ctxt->ss_base, - _regs[VCPU_REGS_RSP]), - &dst.val, dst.bytes, ctxt->vcpu)) != 0) + c->dst.bytes = 8; + if ((rc = ops->read_std(register_address( + ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), + &c->dst.val, + c->dst.bytes, + ctxt->vcpu)) != 0) goto done; - register_address_increment(_regs[VCPU_REGS_RSP], dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RSP], + c->dst.bytes); break; case 0xa0 ... 0xa1: /* mov */ - dst.ptr = (unsigned long *)&_regs[VCPU_REGS_RAX]; - dst.val = src.val; - _eip += ad_bytes; /* skip src displacement */ + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + c->dst.val = c->src.val; + /* skip src displacement */ + c->eip += c->ad_bytes; break; case 0xa2 ... 0xa3: /* mov */ - dst.val = (unsigned long)_regs[VCPU_REGS_RAX]; - _eip += ad_bytes; /* skip dst displacement */ + c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; + /* skip c->dst displacement */ + c->eip += c->ad_bytes; break; case 0xc0 ... 0xc1: grp2: /* Grp2 */ - switch (modrm_reg) { + switch (c->modrm_reg) { case 0: /* rol */ - emulate_2op_SrcB("rol", src, dst, _eflags); + emulate_2op_SrcB("rol", c->src, c->dst, _eflags); break; case 1: /* ror */ - emulate_2op_SrcB("ror", src, dst, _eflags); + emulate_2op_SrcB("ror", c->src, c->dst, _eflags); break; case 2: /* rcl */ - emulate_2op_SrcB("rcl", src, dst, _eflags); + emulate_2op_SrcB("rcl", c->src, c->dst, _eflags); break; case 3: /* rcr */ - emulate_2op_SrcB("rcr", src, dst, _eflags); + emulate_2op_SrcB("rcr", c->src, c->dst, _eflags); break; case 4: /* sal/shl */ case 6: /* sal/shl */ - emulate_2op_SrcB("sal", src, dst, _eflags); + emulate_2op_SrcB("sal", c->src, c->dst, _eflags); break; case 5: /* shr */ - emulate_2op_SrcB("shr", src, dst, _eflags); + emulate_2op_SrcB("shr", c->src, c->dst, _eflags); break; case 7: /* sar */ - emulate_2op_SrcB("sar", src, dst, _eflags); + emulate_2op_SrcB("sar", c->src, c->dst, _eflags); break; } break; case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ mov: - dst.val = src.val; + c->dst.val = c->src.val; break; case 0xd0 ... 0xd1: /* Grp2 */ - src.val = 1; + c->src.val = 1; goto grp2; case 0xd2 ... 0xd3: /* Grp2 */ - src.val = _regs[VCPU_REGS_RCX]; + c->src.val = c->regs[VCPU_REGS_RCX]; goto grp2; case 0xf6 ... 0xf7: /* Grp3 */ - switch (modrm_reg) { + switch (c->modrm_reg) { case 0 ... 1: /* test */ /* * Special case in Grp3: test has an immediate * source operand. */ - src.type = OP_IMM; - src.ptr = (unsigned long *)_eip; - src.bytes = (d & ByteOp) ? 1 : op_bytes; - if (src.bytes == 8) - src.bytes = 4; - switch (src.bytes) { + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = (c->d & ByteOp) ? 1 : + c->op_bytes; + if (c->src.bytes == 8) + c->src.bytes = 4; + switch (c->src.bytes) { case 1: - src.val = insn_fetch(s8, 1, _eip); + c->src.val = insn_fetch(s8, 1, c->eip); break; case 2: - src.val = insn_fetch(s16, 2, _eip); + c->src.val = insn_fetch(s16, 2, c->eip); break; case 4: - src.val = insn_fetch(s32, 4, _eip); + c->src.val = insn_fetch(s32, 4, c->eip); break; } goto test; case 2: /* not */ - dst.val = ~dst.val; + c->dst.val = ~c->dst.val; break; case 3: /* neg */ - emulate_1op("neg", dst, _eflags); + emulate_1op("neg", c->dst, _eflags); break; default: goto cannot_emulate; } break; case 0xfe ... 0xff: /* Grp4/Grp5 */ - switch (modrm_reg) { + switch (c->modrm_reg) { case 0: /* inc */ - emulate_1op("inc", dst, _eflags); + emulate_1op("inc", c->dst, _eflags); break; case 1: /* dec */ - emulate_1op("dec", dst, _eflags); + emulate_1op("dec", c->dst, _eflags); break; case 4: /* jmp abs */ - if (b == 0xff) - _eip = dst.val; + if (c->b == 0xff) + c->eip = c->dst.val; else goto cannot_emulate; break; case 6: /* push */ /* 64-bit mode: PUSH always pushes a 64-bit operand. */ if (mode == X86EMUL_MODE_PROT64) { - dst.bytes = 8; - if ((rc = ops->read_std((unsigned long)dst.ptr, - &dst.val, 8, - ctxt->vcpu)) != 0) + c->dst.bytes = 8; + if ((rc = ops->read_std( + (unsigned long)c->dst.ptr, + &c->dst.val, 8, + ctxt->vcpu)) != 0) goto done; } - register_address_increment(_regs[VCPU_REGS_RSP], - -dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->dst.bytes); if ((rc = ops->write_emulated( register_address(ctxt->ss_base, - _regs[VCPU_REGS_RSP]), - &dst.val, dst.bytes, ctxt->vcpu)) != 0) + c->regs[VCPU_REGS_RSP]), + &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0) goto done; no_wb = 1; break; @@ -1161,34 +1178,40 @@ done_prefixes: writeback: if (!no_wb) { - switch (dst.type) { + switch (c->dst.type) { case OP_REG: - /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ - switch (dst.bytes) { + /* The 4-byte case *is* correct: + * in 64-bit mode we zero-extend. + */ + switch (c->dst.bytes) { case 1: - *(u8 *)dst.ptr = (u8)dst.val; + *(u8 *)c->dst.ptr = (u8)c->dst.val; break; case 2: - *(u16 *)dst.ptr = (u16)dst.val; + *(u16 *)c->dst.ptr = (u16)c->dst.val; break; case 4: - *dst.ptr = (u32)dst.val; + *c->dst.ptr = (u32)c->dst.val; break; /* 64b: zero-ext */ case 8: - *dst.ptr = dst.val; + *c->dst.ptr = c->dst.val; break; } break; case OP_MEM: - if (lock_prefix) - rc = ops->cmpxchg_emulated((unsigned long)dst. - ptr, &dst.orig_val, - &dst.val, dst.bytes, - ctxt->vcpu); + if (c->lock_prefix) + rc = ops->cmpxchg_emulated( + (unsigned long)c->dst.ptr, + &c->dst.orig_val, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); else - rc = ops->write_emulated((unsigned long)dst.ptr, - &dst.val, dst.bytes, - ctxt->vcpu); + rc = ops->write_emulated( + (unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); if (rc != 0) goto done; default: @@ -1197,173 +1220,185 @@ writeback: } /* Commit shadow register state. */ - memcpy(ctxt->vcpu->regs, _regs, sizeof _regs); + memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); ctxt->eflags = _eflags; - ctxt->vcpu->rip = _eip; + ctxt->vcpu->rip = c->eip; done: return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; special_insn: - if (twobyte) + if (c->twobyte) goto twobyte_special_insn; - switch(b) { + switch (c->b) { case 0x50 ... 0x57: /* push reg */ - if (op_bytes == 2) - src.val = (u16) _regs[b & 0x7]; + if (c->op_bytes == 2) + c->src.val = (u16) c->regs[c->b & 0x7]; else - src.val = (u32) _regs[b & 0x7]; - dst.type = OP_MEM; - dst.bytes = op_bytes; - dst.val = src.val; - register_address_increment(_regs[VCPU_REGS_RSP], -op_bytes); - dst.ptr = (void *) register_address( - ctxt->ss_base, _regs[VCPU_REGS_RSP]); + c->src.val = (u32) c->regs[c->b & 0x7]; + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->op_bytes); + c->dst.ptr = (void *) register_address( + ctxt->ss_base, c->regs[VCPU_REGS_RSP]); break; case 0x58 ... 0x5f: /* pop reg */ - dst.ptr = (unsigned long *)&_regs[b & 0x7]; + c->dst.ptr = + (unsigned long *)&c->regs[c->b & 0x7]; pop_instruction: if ((rc = ops->read_std(register_address(ctxt->ss_base, - _regs[VCPU_REGS_RSP]), dst.ptr, op_bytes, ctxt->vcpu)) - != 0) + c->regs[VCPU_REGS_RSP]), c->dst.ptr, + c->op_bytes, ctxt->vcpu)) != 0) goto done; - register_address_increment(_regs[VCPU_REGS_RSP], op_bytes); + register_address_increment(c->regs[VCPU_REGS_RSP], + c->op_bytes); no_wb = 1; /* Disable writeback. */ break; case 0x6a: /* push imm8 */ - src.val = 0L; - src.val = insn_fetch(s8, 1, _eip); - push: - dst.type = OP_MEM; - dst.bytes = op_bytes; - dst.val = src.val; - register_address_increment(_regs[VCPU_REGS_RSP], -op_bytes); - dst.ptr = (void *) register_address(ctxt->ss_base, - _regs[VCPU_REGS_RSP]); + c->src.val = 0L; + c->src.val = insn_fetch(s8, 1, c->eip); +push: + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->op_bytes); + c->dst.ptr = (void *) register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]); break; case 0x6c: /* insb */ case 0x6d: /* insw/insd */ if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 1, /* in */ - (d & ByteOp) ? 1 : op_bytes, /* size */ - rep_prefix ? - address_mask(_regs[VCPU_REGS_RCX]) : 1, /* count */ - (_eflags & EFLG_DF), /* down */ + 1, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (_eflags & EFLG_DF), register_address(ctxt->es_base, - _regs[VCPU_REGS_RDI]), /* address */ - rep_prefix, - _regs[VCPU_REGS_RDX] /* port */ - ) == 0) + c->regs[VCPU_REGS_RDI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) return -1; return 0; case 0x6e: /* outsb */ case 0x6f: /* outsw/outsd */ if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 0, /* in */ - (d & ByteOp) ? 1 : op_bytes, /* size */ - rep_prefix ? - address_mask(_regs[VCPU_REGS_RCX]) : 1, /* count */ - (_eflags & EFLG_DF), /* down */ - register_address(override_base ? - *override_base : ctxt->ds_base, - _regs[VCPU_REGS_RSI]), /* address */ - rep_prefix, - _regs[VCPU_REGS_RDX] /* port */ - ) == 0) + 0, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (_eflags & EFLG_DF), + register_address(c->override_base ? + *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) return -1; return 0; case 0x70 ... 0x7f: /* jcc (short) */ { - int rel = insn_fetch(s8, 1, _eip); + int rel = insn_fetch(s8, 1, c->eip); - if (test_cc(b, _eflags)) + if (test_cc(c->b, _eflags)) JMP_REL(rel); break; } case 0x9c: /* pushf */ - src.val = (unsigned long) _eflags; + c->src.val = (unsigned long) _eflags; goto push; case 0x9d: /* popf */ - dst.ptr = (unsigned long *) &_eflags; + c->dst.ptr = (unsigned long *) &_eflags; goto pop_instruction; case 0xc3: /* ret */ - dst.ptr = &_eip; + c->dst.ptr = &c->eip; goto pop_instruction; case 0xf4: /* hlt */ ctxt->vcpu->halt_request = 1; goto done; } - if (rep_prefix) { - if (_regs[VCPU_REGS_RCX] == 0) { - ctxt->vcpu->rip = _eip; + if (c->rep_prefix) { + if (c->regs[VCPU_REGS_RCX] == 0) { + ctxt->vcpu->rip = c->eip; goto done; } - _regs[VCPU_REGS_RCX]--; - _eip = ctxt->vcpu->rip; + c->regs[VCPU_REGS_RCX]--; + c->eip = ctxt->vcpu->rip; } - switch (b) { + switch (c->b) { case 0xa4 ... 0xa5: /* movs */ - dst.type = OP_MEM; - dst.bytes = (d & ByteOp) ? 1 : op_bytes; - dst.ptr = (unsigned long *)register_address(ctxt->es_base, - _regs[VCPU_REGS_RDI]); + c->dst.type = OP_MEM; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); if ((rc = ops->read_emulated(register_address( - override_base ? *override_base : ctxt->ds_base, - _regs[VCPU_REGS_RSI]), &dst.val, dst.bytes, ctxt->vcpu)) != 0) + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0) goto done; - register_address_increment(_regs[VCPU_REGS_RSI], - (_eflags & EFLG_DF) ? -dst.bytes : dst.bytes); - register_address_increment(_regs[VCPU_REGS_RDI], - (_eflags & EFLG_DF) ? -dst.bytes : dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RSI], + (_eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RDI], + (_eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); break; case 0xa6 ... 0xa7: /* cmps */ DPRINTF("Urk! I don't handle CMPS.\n"); goto cannot_emulate; case 0xaa ... 0xab: /* stos */ - dst.type = OP_MEM; - dst.bytes = (d & ByteOp) ? 1 : op_bytes; - dst.ptr = (unsigned long *)cr2; - dst.val = _regs[VCPU_REGS_RAX]; - register_address_increment(_regs[VCPU_REGS_RDI], - (_eflags & EFLG_DF) ? -dst.bytes : dst.bytes); + c->dst.type = OP_MEM; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)cr2; + c->dst.val = c->regs[VCPU_REGS_RAX]; + register_address_increment(c->regs[VCPU_REGS_RDI], + (_eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); break; case 0xac ... 0xad: /* lods */ - dst.type = OP_REG; - dst.bytes = (d & ByteOp) ? 1 : op_bytes; - dst.ptr = (unsigned long *)&_regs[VCPU_REGS_RAX]; - if ((rc = ops->read_emulated(cr2, &dst.val, dst.bytes, + c->dst.type = OP_REG; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + if ((rc = ops->read_emulated(cr2, &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0) goto done; - register_address_increment(_regs[VCPU_REGS_RSI], - (_eflags & EFLG_DF) ? -dst.bytes : dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RSI], + (_eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); break; case 0xae ... 0xaf: /* scas */ DPRINTF("Urk! I don't handle SCAS.\n"); goto cannot_emulate; case 0xe8: /* call (near) */ { long int rel; - switch (op_bytes) { + switch (c->op_bytes) { case 2: - rel = insn_fetch(s16, 2, _eip); + rel = insn_fetch(s16, 2, c->eip); break; case 4: - rel = insn_fetch(s32, 4, _eip); + rel = insn_fetch(s32, 4, c->eip); break; case 8: - rel = insn_fetch(s64, 8, _eip); + rel = insn_fetch(s64, 8, c->eip); break; default: DPRINTF("Call: Invalid op_bytes\n"); goto cannot_emulate; } - src.val = (unsigned long) _eip; + c->src.val = (unsigned long) c->eip; JMP_REL(rel); - op_bytes = ad_bytes; + c->op_bytes = c->ad_bytes; goto push; } case 0xe9: /* jmp rel */ case 0xeb: /* jmp rel short */ - JMP_REL(src.val); + JMP_REL(c->src.val); no_wb = 1; /* Disable writeback. */ break; @@ -1372,16 +1407,16 @@ special_insn: goto writeback; twobyte_insn: - switch (b) { + switch (c->b) { case 0x01: /* lgdt, lidt, lmsw */ /* Disable writeback. */ no_wb = 1; - switch (modrm_reg) { + switch (c->modrm_reg) { u16 size; unsigned long address; case 0: /* vmcall */ - if (modrm_mod != 3 || modrm_rm != 1) + if (c->modrm_mod != 3 || c->modrm_rm != 1) goto cannot_emulate; rc = kvm_fix_hypercall(ctxt->vcpu); @@ -1391,37 +1426,37 @@ twobyte_insn: kvm_emulate_hypercall(ctxt->vcpu); break; case 2: /* lgdt */ - rc = read_descriptor(ctxt, ops, src.ptr, - &size, &address, op_bytes); + rc = read_descriptor(ctxt, ops, c->src.ptr, + &size, &address, c->op_bytes); if (rc) goto done; realmode_lgdt(ctxt->vcpu, size, address); break; case 3: /* lidt/vmmcall */ - if (modrm_mod == 3 && modrm_rm == 1) { + if (c->modrm_mod == 3 && c->modrm_rm == 1) { rc = kvm_fix_hypercall(ctxt->vcpu); if (rc) goto done; kvm_emulate_hypercall(ctxt->vcpu); } else { - rc = read_descriptor(ctxt, ops, src.ptr, + rc = read_descriptor(ctxt, ops, c->src.ptr, &size, &address, - op_bytes); + c->op_bytes); if (rc) goto done; realmode_lidt(ctxt->vcpu, size, address); } break; case 4: /* smsw */ - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - *(u16 *)&_regs[modrm_rm] + *(u16 *)&c->regs[c->modrm_rm] = realmode_get_cr(ctxt->vcpu, 0); break; case 6: /* lmsw */ - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - realmode_lmsw(ctxt->vcpu, (u16)modrm_val, &_eflags); + realmode_lmsw(ctxt->vcpu, (u16)c->modrm_val, &_eflags); break; case 7: /* invlpg*/ emulate_invlpg(ctxt->vcpu, cr2); @@ -1432,24 +1467,26 @@ twobyte_insn: break; case 0x21: /* mov from dr to reg */ no_wb = 1; - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - rc = emulator_get_dr(ctxt, modrm_reg, &_regs[modrm_rm]); + rc = emulator_get_dr(ctxt, c->modrm_reg, + &c->regs[c->modrm_rm]); break; case 0x23: /* mov from reg to dr */ no_wb = 1; - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - rc = emulator_set_dr(ctxt, modrm_reg, _regs[modrm_rm]); + rc = emulator_set_dr(ctxt, c->modrm_reg, + c->regs[c->modrm_rm]); break; case 0x40 ... 0x4f: /* cmov */ - dst.val = dst.orig_val = src.val; + c->dst.val = c->dst.orig_val = c->src.val; no_wb = 1; /* * First, assume we're decoding an even cmov opcode * (lsb == 0). */ - switch ((b & 15) >> 1) { + switch ((c->b & 15) >> 1) { case 0: /* cmovo */ no_wb = (_eflags & EFLG_OF) ? 0 : 1; break; @@ -1477,46 +1514,50 @@ twobyte_insn: break; } /* Odd cmov opcodes (lsb == 1) have inverted sense. */ - no_wb ^= b & 1; + no_wb ^= c->b & 1; break; case 0xa3: bt: /* bt */ - src.val &= (dst.bytes << 3) - 1; /* only subword offset */ - emulate_2op_SrcV_nobyte("bt", src, dst, _eflags); + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("bt", c->src, c->dst, _eflags); break; case 0xab: bts: /* bts */ - src.val &= (dst.bytes << 3) - 1; /* only subword offset */ - emulate_2op_SrcV_nobyte("bts", src, dst, _eflags); + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("bts", c->src, c->dst, _eflags); break; case 0xb0 ... 0xb1: /* cmpxchg */ /* * Save real source value, then compare EAX against * destination. */ - src.orig_val = src.val; - src.val = _regs[VCPU_REGS_RAX]; - emulate_2op_SrcV("cmp", src, dst, _eflags); + c->src.orig_val = c->src.val; + c->src.val = c->regs[VCPU_REGS_RAX]; + emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); if (_eflags & EFLG_ZF) { /* Success: write back to memory. */ - dst.val = src.orig_val; + c->dst.val = c->src.orig_val; } else { /* Failure: write the value we saw to EAX. */ - dst.type = OP_REG; - dst.ptr = (unsigned long *)&_regs[VCPU_REGS_RAX]; + c->dst.type = OP_REG; + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; } break; case 0xb3: btr: /* btr */ - src.val &= (dst.bytes << 3) - 1; /* only subword offset */ - emulate_2op_SrcV_nobyte("btr", src, dst, _eflags); + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("btr", c->src, c->dst, _eflags); break; case 0xb6 ... 0xb7: /* movzx */ - dst.bytes = op_bytes; - dst.val = (d & ByteOp) ? (u8) src.val : (u16) src.val; + c->dst.bytes = c->op_bytes; + c->dst.val = (c->d & ByteOp) ? (u8) c->src.val + : (u16) c->src.val; break; case 0xba: /* Grp8 */ - switch (modrm_reg & 3) { + switch (c->modrm_reg & 3) { case 0: goto bt; case 1: @@ -1529,16 +1570,19 @@ twobyte_insn: break; case 0xbb: btc: /* btc */ - src.val &= (dst.bytes << 3) - 1; /* only subword offset */ - emulate_2op_SrcV_nobyte("btc", src, dst, _eflags); + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("btc", c->src, c->dst, _eflags); break; case 0xbe ... 0xbf: /* movsx */ - dst.bytes = op_bytes; - dst.val = (d & ByteOp) ? (s8) src.val : (s16) src.val; + c->dst.bytes = c->op_bytes; + c->dst.val = (c->d & ByteOp) ? (s8) c->src.val : + (s16) c->src.val; break; case 0xc3: /* movnti */ - dst.bytes = op_bytes; - dst.val = (op_bytes == 4) ? (u32) src.val : (u64) src.val; + c->dst.bytes = c->op_bytes; + c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val : + (u64) c->src.val; break; } goto writeback; @@ -1546,7 +1590,7 @@ twobyte_insn: twobyte_special_insn: /* Disable writeback. */ no_wb = 1; - switch (b) { + switch (c->b) { case 0x06: emulate_clts(ctxt->vcpu); break; @@ -1558,56 +1602,59 @@ twobyte_special_insn: case 0x18: /* Grp16 (prefetch/nop) */ break; case 0x20: /* mov cr, reg */ - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - _regs[modrm_rm] = realmode_get_cr(ctxt->vcpu, modrm_reg); + c->regs[c->modrm_rm] = + realmode_get_cr(ctxt->vcpu, c->modrm_reg); break; case 0x22: /* mov reg, cr */ - if (modrm_mod != 3) + if (c->modrm_mod != 3) goto cannot_emulate; - realmode_set_cr(ctxt->vcpu, modrm_reg, modrm_val, &_eflags); + realmode_set_cr(ctxt->vcpu, + c->modrm_reg, c->modrm_val, &_eflags); break; case 0x30: /* wrmsr */ - msr_data = (u32)_regs[VCPU_REGS_RAX] - | ((u64)_regs[VCPU_REGS_RDX] << 32); - rc = kvm_set_msr(ctxt->vcpu, _regs[VCPU_REGS_RCX], msr_data); + msr_data = (u32)c->regs[VCPU_REGS_RAX] + | ((u64)c->regs[VCPU_REGS_RDX] << 32); + rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); if (rc) { kvm_x86_ops->inject_gp(ctxt->vcpu, 0); - _eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->rip; } rc = X86EMUL_CONTINUE; break; case 0x32: /* rdmsr */ - rc = kvm_get_msr(ctxt->vcpu, _regs[VCPU_REGS_RCX], &msr_data); + rc = kvm_get_msr(ctxt->vcpu, + c->regs[VCPU_REGS_RCX], &msr_data); if (rc) { kvm_x86_ops->inject_gp(ctxt->vcpu, 0); - _eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->rip; } else { - _regs[VCPU_REGS_RAX] = (u32)msr_data; - _regs[VCPU_REGS_RDX] = msr_data >> 32; + c->regs[VCPU_REGS_RAX] = (u32)msr_data; + c->regs[VCPU_REGS_RDX] = msr_data >> 32; } rc = X86EMUL_CONTINUE; break; case 0x80 ... 0x8f: /* jnz rel, etc*/ { long int rel; - switch (op_bytes) { + switch (c->op_bytes) { case 2: - rel = insn_fetch(s16, 2, _eip); + rel = insn_fetch(s16, 2, c->eip); break; case 4: - rel = insn_fetch(s32, 4, _eip); + rel = insn_fetch(s32, 4, c->eip); break; case 8: - rel = insn_fetch(s64, 8, _eip); + rel = insn_fetch(s64, 8, c->eip); break; default: DPRINTF("jnz: Invalid op_bytes\n"); goto cannot_emulate; } - if (test_cc(b, _eflags)) + if (test_cc(c->b, _eflags)) JMP_REL(rel); break; } @@ -1617,14 +1664,16 @@ twobyte_special_insn: if ((rc = ops->read_emulated(cr2, &old, 8, ctxt->vcpu)) != 0) goto done; - if (((u32) (old >> 0) != (u32) _regs[VCPU_REGS_RAX]) || - ((u32) (old >> 32) != (u32) _regs[VCPU_REGS_RDX])) { - _regs[VCPU_REGS_RAX] = (u32) (old >> 0); - _regs[VCPU_REGS_RDX] = (u32) (old >> 32); + if (((u32) (old >> 0) != + (u32) c->regs[VCPU_REGS_RAX]) || + ((u32) (old >> 32) != + (u32) c->regs[VCPU_REGS_RDX])) { + c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); + c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); _eflags &= ~EFLG_ZF; } else { - new = ((u64)_regs[VCPU_REGS_RCX] << 32) - | (u32) _regs[VCPU_REGS_RBX]; + new = ((u64)c->regs[VCPU_REGS_RCX] << 32) + | (u32) c->regs[VCPU_REGS_RBX]; if ((rc = ops->cmpxchg_emulated(cr2, &old, &new, 8, ctxt->vcpu)) != 0) goto done; @@ -1636,6 +1685,6 @@ twobyte_special_insn: goto writeback; cannot_emulate: - DPRINTF("Cannot emulate %02x\n", b); + DPRINTF("Cannot emulate %02x\n", c->b); return -1; } diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index 92c73aa7f9ac..c354200d5834 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -112,6 +112,36 @@ struct x86_emulate_ops { }; +/* Type, address-of, and value of an instruction's operand. */ +struct operand { + enum { OP_REG, OP_MEM, OP_IMM } type; + unsigned int bytes; + unsigned long val, orig_val, *ptr; +}; + +struct decode_cache { + u8 twobyte; + u8 b; + u8 lock_prefix; + u8 rep_prefix; + u8 op_bytes; + u8 ad_bytes; + struct operand src; + struct operand dst; + unsigned long *override_base; + unsigned int d; + unsigned long regs[NR_VCPU_REGS]; + unsigned long eip; + /* modrm */ + u8 modrm; + u8 modrm_mod; + u8 modrm_reg; + u8 modrm_rm; + u8 use_modrm_ea; + unsigned long modrm_ea; + unsigned long modrm_val; +}; + struct x86_emulate_ctxt { /* Register state before/after emulation. */ struct kvm_vcpu *vcpu; @@ -129,6 +159,10 @@ struct x86_emulate_ctxt { unsigned long ss_base; unsigned long gs_base; unsigned long fs_base; + + /* decode cache */ + + struct decode_cache decode; }; /* Execution mode, passed to the emulator. */ -- cgit v1.2.3 From 8b4caf6650808024c37ec4b29cf81b308af998b1 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:27:19 +0200 Subject: KVM: x86 emulator: move all decoding process to function x86_decode_insn() Split the decoding process into a new function x86_decode_insn(). Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 77 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 24 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index d7026cb8fc70..540589c5d427 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -517,20 +517,16 @@ static int test_cc(unsigned int condition, unsigned int flags) } int -x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) +x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; u8 sib, rex_prefix = 0; unsigned int i; int rc = 0; - unsigned long cr2 = ctxt->cr2; int mode = ctxt->mode; int index_reg = 0, base_reg = 0, scale, rip_relative = 0; - int no_wb = 0; - u64 msr_data; /* Shadow copy of register state. Committed on successful emulation. */ - unsigned long _eflags = ctxt->eflags; memset(c, 0, sizeof(struct decode_cache)); c->eip = ctxt->vcpu->rip; @@ -622,8 +618,10 @@ done_prefixes: } /* Unrecognised? */ - if (c->d == 0) - goto cannot_emulate; + if (c->d == 0) { + DPRINTF("Cannot emulate %02x\n", c->b); + return -1; + } } /* ModRM and SIB bytes. */ @@ -776,7 +774,6 @@ done_prefixes: } if (c->ad_bytes != 8) c->modrm_ea = (u32)c->modrm_ea; - cr2 = c->modrm_ea; modrm_done: ; } @@ -838,13 +835,6 @@ done_prefixes: break; } c->src.type = OP_MEM; - c->src.ptr = (unsigned long *)cr2; - c->src.val = 0; - if ((rc = ops->read_emulated((unsigned long)c->src.ptr, - &c->src.val, - c->src.bytes, ctxt->vcpu)) != 0) - goto done; - c->src.orig_val = c->src.val; break; case SrcImm: c->src.type = OP_IMM; @@ -877,7 +867,7 @@ done_prefixes: switch (c->d & DstMask) { case ImplicitOps: /* Special instructions do their own operand decoding. */ - goto special_insn; + return 0; case DstReg: c->dst.type = OP_REG; if ((c->d & ByteOp) @@ -905,14 +895,54 @@ done_prefixes: } break; case DstMem: - c->dst.type = OP_MEM; - c->dst.ptr = (unsigned long *)cr2; - c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.val = 0; if ((c->d & ModRM) && c->modrm_mod == 3) { c->dst.type = OP_REG; break; } + c->dst.type = OP_MEM; + break; + } + +done: + return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; +} + +int +x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) +{ + unsigned long cr2 = ctxt->cr2; + int no_wb = 0; + u64 msr_data; + unsigned long _eflags = ctxt->eflags; + struct decode_cache *c = &ctxt->decode; + int rc; + + rc = x86_decode_insn(ctxt, ops); + if (rc) + return rc; + + if ((c->d & ModRM) && (c->modrm_mod != 3)) + cr2 = c->modrm_ea; + + if (c->src.type == OP_MEM) { + c->src.ptr = (unsigned long *)cr2; + c->src.val = 0; + if ((rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, + ctxt->vcpu)) != 0) + goto done; + c->src.orig_val = c->src.val; + } + + if ((c->d & DstMask) == ImplicitOps) + goto special_insn; + + + if (c->dst.type == OP_MEM) { + c->dst.ptr = (unsigned long *)cr2; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.val = 0; if (c->d & BitOp) { unsigned long mask = ~(c->dst.bytes * 8 - 1); @@ -925,7 +955,6 @@ done_prefixes: &c->dst.val, c->dst.bytes, ctxt->vcpu)) != 0)) goto done; - break; } c->dst.orig_val = c->dst.val; @@ -983,7 +1012,7 @@ done_prefixes: emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); break; case 0x63: /* movsxd */ - if (mode != X86EMUL_MODE_PROT64) + if (ctxt->mode != X86EMUL_MODE_PROT64) goto cannot_emulate; c->dst.val = (s32) c->src.val; break; @@ -1041,7 +1070,7 @@ done_prefixes: break; case 0x8f: /* pop (sole member of Grp1a) */ /* 64-bit mode: POP always pops a 64-bit operand. */ - if (mode == X86EMUL_MODE_PROT64) + if (ctxt->mode == X86EMUL_MODE_PROT64) c->dst.bytes = 8; if ((rc = ops->read_std(register_address( ctxt->ss_base, @@ -1152,7 +1181,7 @@ done_prefixes: break; case 6: /* push */ /* 64-bit mode: PUSH always pushes a 64-bit operand. */ - if (mode == X86EMUL_MODE_PROT64) { + if (ctxt->mode == X86EMUL_MODE_PROT64) { c->dst.bytes = 8; if ((rc = ops->read_std( (unsigned long)c->dst.ptr, -- cgit v1.2.3 From 1be3aa47182e94944e57b176a5c4ee4e74f1ce33 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:27:27 +0200 Subject: KVM: emulate_instruction() calls now x86_decode_insn() and x86_emulate_insn() emulate_instruction() calls now x86_decode_insn() and x86_emulate_insn(). x86_emulate_insn() is x86_emulate_memop() without the decoding part. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 5 ++++- drivers/kvm/x86_emulate.c | 8 ++------ drivers/kvm/x86_emulate.h | 11 ++++------- 3 files changed, 10 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 1c662f63b7a9..800ab5028ed3 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1287,7 +1287,10 @@ int emulate_instruction(struct kvm_vcpu *vcpu, vcpu->mmio_is_write = 0; vcpu->pio.string = 0; - r = x86_emulate_memop(&emulate_ctxt, &emulate_ops); + r = x86_decode_insn(&emulate_ctxt, &emulate_ops); + if (r == 0) + r = x86_emulate_insn(&emulate_ctxt, &emulate_ops); + if (vcpu->pio.string) return EMULATE_DO_MMIO; diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 540589c5d427..c191093982d8 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -908,18 +908,14 @@ done: } int -x86_emulate_memop(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) +x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { unsigned long cr2 = ctxt->cr2; int no_wb = 0; u64 msr_data; unsigned long _eflags = ctxt->eflags; struct decode_cache *c = &ctxt->decode; - int rc; - - rc = x86_decode_insn(ctxt, ops); - if (rc) - return rc; + int rc = 0; if ((c->d & ModRM) && (c->modrm_mod != 3)) cr2 = c->modrm_ea; diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index c354200d5834..28acad416bca 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -178,12 +178,9 @@ struct x86_emulate_ctxt { #define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64 #endif -/* - * x86_emulate_memop: Emulate an instruction that faulted attempting to - * read/write a 'special' memory area. - * Returns -1 on failure, 0 on success. - */ -int x86_emulate_memop(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops); +int x86_decode_insn(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops); +int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops); #endif /* __X86_EMULATE_H__ */ -- cgit v1.2.3 From 3427318fd2244737a466a06a93c5fe579852f871 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 18 Sep 2007 11:27:37 +0200 Subject: KVM: Call x86_decode_insn() only when needed Move emulate_ctxt to kvm_vcpu to keep emulate context when we exit from kvm module. Call x86_decode_insn() only when needed. Modify x86_emulate_insn() to not modify the context if it must be re-entered. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 8 ++++- drivers/kvm/kvm_main.c | 77 +++++++++++++++++++++++++++-------------------- drivers/kvm/svm.c | 9 +++--- drivers/kvm/vmx.c | 9 +++--- drivers/kvm/x86_emulate.c | 24 +++++++++++++-- 5 files changed, 82 insertions(+), 45 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index da9c3aa1c08c..e885b190b798 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -207,6 +207,8 @@ enum { VCPU_SREG_LDTR, }; +#include "x86_emulate.h" + struct kvm_pio_request { unsigned long count; int cur_count; @@ -380,6 +382,10 @@ struct kvm_vcpu { int cpuid_nent; struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES]; + + /* emulate context */ + + struct x86_emulate_ctxt emulate_ctxt; }; struct kvm_mem_alias { @@ -555,7 +561,7 @@ enum emulation_result { }; int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long cr2, u16 error_code); + unsigned long cr2, u16 error_code, int no_decode); void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 800ab5028ed3..710483669f34 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1251,45 +1251,56 @@ struct x86_emulate_ops emulate_ops = { int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, unsigned long cr2, - u16 error_code) + u16 error_code, + int no_decode) { - struct x86_emulate_ctxt emulate_ctxt; - int r; - int cs_db, cs_l; + int r = 0; vcpu->mmio_fault_cr2 = cr2; kvm_x86_ops->cache_regs(vcpu); - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - - emulate_ctxt.vcpu = vcpu; - emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - emulate_ctxt.cr2 = cr2; - emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM) - ? X86EMUL_MODE_REAL : cs_l - ? X86EMUL_MODE_PROT64 : cs_db - ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; - - if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) { - emulate_ctxt.cs_base = 0; - emulate_ctxt.ds_base = 0; - emulate_ctxt.es_base = 0; - emulate_ctxt.ss_base = 0; - } else { - emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS); - emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS); - emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES); - emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS); - } - - emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS); - emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS); - vcpu->mmio_is_write = 0; vcpu->pio.string = 0; - r = x86_decode_insn(&emulate_ctxt, &emulate_ops); + + if (!no_decode) { + int cs_db, cs_l; + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + + vcpu->emulate_ctxt.vcpu = vcpu; + vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); + vcpu->emulate_ctxt.cr2 = cr2; + vcpu->emulate_ctxt.mode = + (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) + ? X86EMUL_MODE_REAL : cs_l + ? X86EMUL_MODE_PROT64 : cs_db + ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; + + if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) { + vcpu->emulate_ctxt.cs_base = 0; + vcpu->emulate_ctxt.ds_base = 0; + vcpu->emulate_ctxt.es_base = 0; + vcpu->emulate_ctxt.ss_base = 0; + } else { + vcpu->emulate_ctxt.cs_base = + get_segment_base(vcpu, VCPU_SREG_CS); + vcpu->emulate_ctxt.ds_base = + get_segment_base(vcpu, VCPU_SREG_DS); + vcpu->emulate_ctxt.es_base = + get_segment_base(vcpu, VCPU_SREG_ES); + vcpu->emulate_ctxt.ss_base = + get_segment_base(vcpu, VCPU_SREG_SS); + } + + vcpu->emulate_ctxt.gs_base = + get_segment_base(vcpu, VCPU_SREG_GS); + vcpu->emulate_ctxt.fs_base = + get_segment_base(vcpu, VCPU_SREG_FS); + + r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + } + if (r == 0) - r = x86_emulate_insn(&emulate_ctxt, &emulate_ops); + r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); if (vcpu->pio.string) return EMULATE_DO_MMIO; @@ -1313,7 +1324,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->set_rflags(vcpu, emulate_ctxt.eflags); + kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags); if (vcpu->mmio_is_write) { vcpu->mmio_needed = 0; @@ -2055,7 +2066,7 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->mmio_read_completed = 1; vcpu->mmio_needed = 0; r = emulate_instruction(vcpu, kvm_run, - vcpu->mmio_fault_cr2, 0); + vcpu->mmio_fault_cr2, 0, 1); if (r == EMULATE_DO_MMIO) { /* * Read-modify-write. Back to userspace. diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 794d95416f7b..f268bd51f337 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -960,7 +960,7 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) return 1; } er = emulate_instruction(&svm->vcpu, kvm_run, fault_address, - error_code); + error_code, 0); mutex_unlock(&kvm->lock); switch (er) { @@ -984,7 +984,7 @@ static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { int er; - er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0); + er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0); if (er != EMULATE_DONE) inject_ud(&svm->vcpu); @@ -1027,7 +1027,8 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) string = (io_info & SVM_IOIO_STR_MASK) != 0; if (string) { - if (emulate_instruction(&svm->vcpu, kvm_run, 0, 0) == EMULATE_DO_MMIO) + if (emulate_instruction(&svm->vcpu, + kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) return 0; return 1; } @@ -1086,7 +1087,7 @@ static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) static int emulate_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - if (emulate_instruction(&svm->vcpu, NULL, 0, 0) != EMULATE_DONE) + if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE) pr_unimpl(&svm->vcpu, "%s: failed\n", __FUNCTION__); return 1; } diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 47c827d3007c..91768d5dbfb9 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1750,7 +1750,7 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, * Cause the #SS fault with 0 error code in VM86 mode. */ if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) - if (emulate_instruction(vcpu, NULL, 0, 0) == EMULATE_DONE) + if (emulate_instruction(vcpu, NULL, 0, 0, 0) == EMULATE_DONE) return 1; return 0; } @@ -1787,7 +1787,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) } if (is_invalid_opcode(intr_info)) { - er = emulate_instruction(vcpu, kvm_run, 0, 0); + er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); if (er != EMULATE_DONE) vmx_inject_ud(vcpu); @@ -1812,7 +1812,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } - er = emulate_instruction(vcpu, kvm_run, cr2, error_code); + er = emulate_instruction(vcpu, kvm_run, cr2, error_code, 0); mutex_unlock(&vcpu->kvm->lock); switch (er) { @@ -1873,7 +1873,8 @@ static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) string = (exit_qualification & 16) != 0; if (string) { - if (emulate_instruction(vcpu, kvm_run, 0, 0) == EMULATE_DO_MMIO) + if (emulate_instruction(vcpu, + kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) return 0; return 1; } diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index c191093982d8..dc9d2a870fbc 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -913,10 +913,19 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) unsigned long cr2 = ctxt->cr2; int no_wb = 0; u64 msr_data; + unsigned long saved_eip = 0; unsigned long _eflags = ctxt->eflags; struct decode_cache *c = &ctxt->decode; int rc = 0; + /* Shadow copy of register state. Committed on successful emulation. + * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't + * modify them. + */ + + memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); + saved_eip = c->eip; + if ((c->d & ModRM) && (c->modrm_mod != 3)) cr2 = c->modrm_ea; @@ -1250,7 +1259,11 @@ writeback: ctxt->vcpu->rip = c->eip; done: - return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; + if (rc == X86EMUL_UNHANDLEABLE) { + c->eip = saved_eip; + return -1; + } + return 0; special_insn: if (c->twobyte) @@ -1305,8 +1318,10 @@ push: register_address(ctxt->es_base, c->regs[VCPU_REGS_RDI]), c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; return -1; + } return 0; case 0x6e: /* outsb */ case 0x6f: /* outsw/outsd */ @@ -1321,8 +1336,10 @@ push: ctxt->ds_base, c->regs[VCPU_REGS_RSI]), c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; return -1; + } return 0; case 0x70 ... 0x7f: /* jcc (short) */ { int rel = insn_fetch(s8, 1, c->eip); @@ -1711,5 +1728,6 @@ twobyte_special_insn: cannot_emulate: DPRINTF("Cannot emulate %02x\n", c->b); + c->eip = saved_eip; return -1; } -- cgit v1.2.3 From 51c6cf662b4b361a09fbd324f4c67875d9bcfbea Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 29 Aug 2007 03:48:05 +0300 Subject: KVM: VMX: Further reduce efer reloads KVM avoids reloading the efer msr when the difference between the guest and host values consist of the long mode bits (which are switched by hardware) and the NX bit (which is emulated by the KVM MMU). This patch also allows KVM to ignore SCE (syscall enable) when the guest is running in 32-bit mode. This is because the syscall instruction is not available in 32-bit mode on Intel processors, so the SCE bit is effectively meaningless. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 61 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 24 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 91768d5dbfb9..8eb49e055ec0 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -57,6 +57,7 @@ struct vcpu_vmx { u16 fs_sel, gs_sel, ldt_sel; int gs_ldt_reload_needed; int fs_reload_needed; + int guest_efer_loaded; }host_state; }; @@ -74,8 +75,6 @@ static DEFINE_PER_CPU(struct vmcs *, current_vmcs); static struct page *vmx_io_bitmap_a; static struct page *vmx_io_bitmap_b; -#define EFER_SAVE_RESTORE_BITS ((u64)EFER_SCE) - static struct vmcs_config { int size; int order; @@ -138,18 +137,6 @@ static void save_msrs(struct kvm_msr_entry *e, int n) rdmsrl(e[i].index, e[i].data); } -static inline u64 msr_efer_save_restore_bits(struct kvm_msr_entry msr) -{ - return (u64)msr.data & EFER_SAVE_RESTORE_BITS; -} - -static inline int msr_efer_need_save_restore(struct vcpu_vmx *vmx) -{ - int efer_offset = vmx->msr_offset_efer; - return msr_efer_save_restore_bits(vmx->host_msrs[efer_offset]) != - msr_efer_save_restore_bits(vmx->guest_msrs[efer_offset]); -} - static inline int is_page_fault(u32 intr_info) { return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | @@ -351,16 +338,42 @@ static void reload_tss(void) static void load_transition_efer(struct vcpu_vmx *vmx) { - u64 trans_efer; int efer_offset = vmx->msr_offset_efer; + u64 host_efer = vmx->host_msrs[efer_offset].data; + u64 guest_efer = vmx->guest_msrs[efer_offset].data; + u64 ignore_bits; + + if (efer_offset < 0) + return; + /* + * NX is emulated; LMA and LME handled by hardware; SCE meaninless + * outside long mode + */ + ignore_bits = EFER_NX | EFER_SCE; +#ifdef CONFIG_X86_64 + ignore_bits |= EFER_LMA | EFER_LME; + /* SCE is meaningful only in long mode on Intel */ + if (guest_efer & EFER_LMA) + ignore_bits &= ~(u64)EFER_SCE; +#endif + if ((guest_efer & ~ignore_bits) == (host_efer & ~ignore_bits)) + return; - trans_efer = vmx->host_msrs[efer_offset].data; - trans_efer &= ~EFER_SAVE_RESTORE_BITS; - trans_efer |= msr_efer_save_restore_bits(vmx->guest_msrs[efer_offset]); - wrmsrl(MSR_EFER, trans_efer); + vmx->host_state.guest_efer_loaded = 1; + guest_efer &= ~ignore_bits; + guest_efer |= host_efer & ignore_bits; + wrmsrl(MSR_EFER, guest_efer); vmx->vcpu.stat.efer_reload++; } +static void reload_host_efer(struct vcpu_vmx *vmx) +{ + if (vmx->host_state.guest_efer_loaded) { + vmx->host_state.guest_efer_loaded = 0; + load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1); + } +} + static void vmx_save_host_state(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -406,8 +419,7 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) } #endif load_msrs(vmx->guest_msrs, vmx->save_nmsrs); - if (msr_efer_need_save_restore(vmx)) - load_transition_efer(vmx); + load_transition_efer(vmx); } static void vmx_load_host_state(struct vcpu_vmx *vmx) @@ -436,8 +448,7 @@ static void vmx_load_host_state(struct vcpu_vmx *vmx) reload_tss(); save_msrs(vmx->guest_msrs, vmx->save_nmsrs); load_msrs(vmx->host_msrs, vmx->save_nmsrs); - if (msr_efer_need_save_restore(vmx)) - load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1); + reload_host_efer(vmx); } /* @@ -727,8 +738,10 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) #ifdef CONFIG_X86_64 case MSR_EFER: ret = kvm_set_msr_common(vcpu, msr_index, data); - if (vmx->host_state.loaded) + if (vmx->host_state.loaded) { + reload_host_efer(vmx); load_transition_efer(vmx); + } break; case MSR_FS_BASE: vmcs_writel(GUEST_FS_BASE, data); -- cgit v1.2.3 From c7addb902054195b995114df154e061c7d604f69 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 16 Sep 2007 18:58:32 +0200 Subject: KVM: Allow not-present guest page faults to bypass kvm There are two classes of page faults trapped by kvm: - host page faults, where the fault is needed to allow kvm to install the shadow pte or update the guest accessed and dirty bits - guest page faults, where the guest has faulted and kvm simply injects the fault back into the guest to handle The second class, guest page faults, is pure overhead. We can eliminate some of it on vmx using the following evil trick: - when we set up a shadow page table entry, if the corresponding guest pte is not present, set up the shadow pte as not present - if the guest pte _is_ present, mark the shadow pte as present but also set one of the reserved bits in the shadow pte - tell the vmx hardware not to trap faults which have the present bit clear With this, normal page-not-present faults go directly to the guest, bypassing kvm entirely. Unfortunately, this trick only works on Intel hardware, as AMD lacks a way to discriminate among page faults based on error code. It is also a little risky since it uses reserved bits which might become unreserved in the future, so a module parameter is provided to disable it. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 ++ drivers/kvm/kvm_main.c | 4 ++- drivers/kvm/mmu.c | 89 ++++++++++++++++++++++++++++++++++++----------- drivers/kvm/paging_tmpl.h | 52 ++++++++++++++++++++------- drivers/kvm/vmx.c | 11 ++++-- 5 files changed, 122 insertions(+), 37 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index e885b190b798..7de948e9e64e 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -150,6 +150,8 @@ struct kvm_mmu { int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); void (*free)(struct kvm_vcpu *vcpu); gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); + void (*prefetch_page)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page); hpa_t root_hpa; int root_level; int shadow_root_level; @@ -536,6 +538,7 @@ void kvm_mmu_module_exit(void); void kvm_mmu_destroy(struct kvm_vcpu *vcpu); int kvm_mmu_create(struct kvm_vcpu *vcpu); int kvm_mmu_setup(struct kvm_vcpu *vcpu); +void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 710483669f34..82cc7ae0fc83 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -3501,7 +3501,9 @@ int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out; - return r; + kvm_mmu_set_nonpresent_ptes(0ull, 0ull); + + return 0; out_free: kmem_cache_destroy(kvm_vcpu_cache); diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index feb5ac986c5d..069ce83f018e 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -156,6 +156,16 @@ static struct kmem_cache *pte_chain_cache; static struct kmem_cache *rmap_desc_cache; static struct kmem_cache *mmu_page_header_cache; +static u64 __read_mostly shadow_trap_nonpresent_pte; +static u64 __read_mostly shadow_notrap_nonpresent_pte; + +void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte) +{ + shadow_trap_nonpresent_pte = trap_pte; + shadow_notrap_nonpresent_pte = notrap_pte; +} +EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes); + static int is_write_protection(struct kvm_vcpu *vcpu) { return vcpu->cr0 & X86_CR0_WP; @@ -176,6 +186,13 @@ static int is_present_pte(unsigned long pte) return pte & PT_PRESENT_MASK; } +static int is_shadow_present_pte(u64 pte) +{ + pte &= ~PT_SHADOW_IO_MARK; + return pte != shadow_trap_nonpresent_pte + && pte != shadow_notrap_nonpresent_pte; +} + static int is_writeble_pte(unsigned long pte) { return pte & PT_WRITABLE_MASK; @@ -450,7 +467,7 @@ static int is_empty_shadow_page(u64 *spt) u64 *end; for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++) - if (*pos != 0) { + if ((*pos & ~PT_SHADOW_IO_MARK) != shadow_trap_nonpresent_pte) { printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__, pos, *pos); return 0; @@ -632,6 +649,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, page->gfn = gfn; page->role = role; hlist_add_head(&page->hash_link, bucket); + vcpu->mmu.prefetch_page(vcpu, page); if (!metaphysical) rmap_write_protect(vcpu, gfn); return page; @@ -648,9 +666,9 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, if (page->role.level == PT_PAGE_TABLE_LEVEL) { for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { - if (pt[i] & PT_PRESENT_MASK) + if (is_shadow_present_pte(pt[i])) rmap_remove(&pt[i]); - pt[i] = 0; + pt[i] = shadow_trap_nonpresent_pte; } kvm_flush_remote_tlbs(kvm); return; @@ -659,8 +677,8 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { ent = pt[i]; - pt[i] = 0; - if (!(ent & PT_PRESENT_MASK)) + pt[i] = shadow_trap_nonpresent_pte; + if (!is_shadow_present_pte(ent)) continue; ent &= PT64_BASE_ADDR_MASK; mmu_page_remove_parent_pte(page_header(ent), &pt[i]); @@ -691,7 +709,7 @@ static void kvm_mmu_zap_page(struct kvm *kvm, } BUG_ON(!parent_pte); kvm_mmu_put_page(page, parent_pte); - set_shadow_pte(parent_pte, 0); + set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte); } kvm_mmu_page_unlink_children(kvm, page); if (!page->root_count) { @@ -798,7 +816,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) if (level == 1) { pte = table[index]; - if (is_present_pte(pte) && is_writeble_pte(pte)) + if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) return 0; mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v); @@ -808,7 +826,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) return 0; } - if (table[index] == 0) { + if (table[index] == shadow_trap_nonpresent_pte) { struct kvm_mmu_page *new_table; gfn_t pseudo_gfn; @@ -829,6 +847,15 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) } } +static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp) +{ + int i; + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) + sp->spt[i] = shadow_trap_nonpresent_pte; +} + static void mmu_free_roots(struct kvm_vcpu *vcpu) { int i; @@ -943,6 +970,7 @@ static int nonpaging_init_context(struct kvm_vcpu *vcpu) context->page_fault = nonpaging_page_fault; context->gva_to_gpa = nonpaging_gva_to_gpa; context->free = nonpaging_free; + context->prefetch_page = nonpaging_prefetch_page; context->root_level = 0; context->shadow_root_level = PT32E_ROOT_LEVEL; context->root_hpa = INVALID_PAGE; @@ -989,6 +1017,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level) context->new_cr3 = paging_new_cr3; context->page_fault = paging64_page_fault; context->gva_to_gpa = paging64_gva_to_gpa; + context->prefetch_page = paging64_prefetch_page; context->free = paging_free; context->root_level = level; context->shadow_root_level = level; @@ -1009,6 +1038,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu) context->page_fault = paging32_page_fault; context->gva_to_gpa = paging32_gva_to_gpa; context->free = paging_free; + context->prefetch_page = paging32_prefetch_page; context->root_level = PT32_ROOT_LEVEL; context->shadow_root_level = PT32E_ROOT_LEVEL; context->root_hpa = INVALID_PAGE; @@ -1081,7 +1111,7 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *child; pte = *spte; - if (is_present_pte(pte)) { + if (is_shadow_present_pte(pte)) { if (page->role.level == PT_PAGE_TABLE_LEVEL) rmap_remove(spte); else { @@ -1089,22 +1119,25 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, mmu_page_remove_parent_pte(child, spte); } } - set_shadow_pte(spte, 0); + set_shadow_pte(spte, shadow_trap_nonpresent_pte); kvm_flush_remote_tlbs(vcpu->kvm); } static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, u64 *spte, - const void *new, int bytes) + const void *new, int bytes, + int offset_in_pte) { if (page->role.level != PT_PAGE_TABLE_LEVEL) return; if (page->role.glevels == PT32_ROOT_LEVEL) - paging32_update_pte(vcpu, page, spte, new, bytes); + paging32_update_pte(vcpu, page, spte, new, bytes, + offset_in_pte); else - paging64_update_pte(vcpu, page, spte, new, bytes); + paging64_update_pte(vcpu, page, spte, new, bytes, + offset_in_pte); } void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, @@ -1126,6 +1159,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, int npte; pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); + kvm_mmu_audit(vcpu, "pre pte write"); if (gfn == vcpu->last_pt_write_gfn) { ++vcpu->last_pt_write_count; if (vcpu->last_pt_write_count >= 3) @@ -1181,10 +1215,12 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, spte = &page->spt[page_offset / sizeof(*spte)]; while (npte--) { mmu_pte_write_zap_pte(vcpu, page, spte); - mmu_pte_write_new_pte(vcpu, page, spte, new, bytes); + mmu_pte_write_new_pte(vcpu, page, spte, new, bytes, + page_offset & (pte_size - 1)); ++spte; } } + kvm_mmu_audit(vcpu, "post pte write"); } int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) @@ -1359,22 +1395,33 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) { u64 ent = pt[i]; - if (!(ent & PT_PRESENT_MASK)) + if (ent == shadow_trap_nonpresent_pte) continue; va = canonicalize(va); - if (level > 1) + if (level > 1) { + if (ent == shadow_notrap_nonpresent_pte) + printk(KERN_ERR "audit: (%s) nontrapping pte" + " in nonleaf level: levels %d gva %lx" + " level %d pte %llx\n", audit_msg, + vcpu->mmu.root_level, va, level, ent); + audit_mappings_page(vcpu, ent, va, level - 1); - else { + } else { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); hpa_t hpa = gpa_to_hpa(vcpu, gpa); - if ((ent & PT_PRESENT_MASK) + if (is_shadow_present_pte(ent) && (ent & PT64_BASE_ADDR_MASK) != hpa) - printk(KERN_ERR "audit error: (%s) levels %d" - " gva %lx gpa %llx hpa %llx ent %llx\n", + printk(KERN_ERR "xx audit error: (%s) levels %d" + " gva %lx gpa %llx hpa %llx ent %llx %d\n", audit_msg, vcpu->mmu.root_level, - va, gpa, hpa, ent); + va, gpa, hpa, ent, is_shadow_present_pte(ent)); + else if (ent == shadow_notrap_nonpresent_pte + && !is_error_hpa(hpa)) + printk(KERN_ERR "audit: (%s) notrap shadow," + " valid guest gva %lx\n", audit_msg, va); + } } } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 6b094b44f8fb..99ac9b15f773 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -31,6 +31,7 @@ #define PT_INDEX(addr, level) PT64_INDEX(addr, level) #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level) + #define PT_LEVEL_BITS PT64_LEVEL_BITS #ifdef CONFIG_X86_64 #define PT_MAX_FULL_LEVELS 4 #else @@ -45,6 +46,7 @@ #define PT_INDEX(addr, level) PT32_INDEX(addr, level) #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level) + #define PT_LEVEL_BITS PT32_LEVEL_BITS #define PT_MAX_FULL_LEVELS 2 #else #error Invalid PTTYPE value @@ -211,12 +213,12 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, { hpa_t paddr; int dirty = gpte & PT_DIRTY_MASK; - u64 spte = *shadow_pte; - int was_rmapped = is_rmap_pte(spte); + u64 spte; + int was_rmapped = is_rmap_pte(*shadow_pte); pgprintk("%s: spte %llx gpte %llx access %llx write_fault %d" " user_fault %d gfn %lx\n", - __FUNCTION__, spte, (u64)gpte, access_bits, + __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, write_fault, user_fault, gfn); if (write_fault && !dirty) { @@ -236,7 +238,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, FNAME(mark_pagetable_dirty)(vcpu->kvm, walker); } - spte |= PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK; + spte = PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK; spte |= gpte & PT64_NX_MASK; if (!dirty) access_bits &= ~PT_WRITABLE_MASK; @@ -248,10 +250,8 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, spte |= PT_USER_MASK; if (is_error_hpa(paddr)) { - spte |= gaddr; - spte |= PT_SHADOW_IO_MARK; - spte &= ~PT_PRESENT_MASK; - set_shadow_pte(shadow_pte, spte); + set_shadow_pte(shadow_pte, + shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); return; } @@ -286,6 +286,7 @@ unshadowed: if (access_bits & PT_WRITABLE_MASK) mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT); + pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); if (!was_rmapped) @@ -304,14 +305,18 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, } static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, - u64 *spte, const void *pte, int bytes) + u64 *spte, const void *pte, int bytes, + int offset_in_pte) { pt_element_t gpte; - if (bytes < sizeof(pt_element_t)) - return; gpte = *(const pt_element_t *)pte; - if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) + if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { + if (!offset_in_pte && !is_present_pte(gpte)) + set_shadow_pte(spte, shadow_notrap_nonpresent_pte); + return; + } + if (bytes < sizeof(pt_element_t)) return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); FNAME(set_pte)(vcpu, gpte, spte, PT_USER_MASK | PT_WRITABLE_MASK, 0, @@ -368,7 +373,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, unsigned hugepage_access = 0; shadow_ent = ((u64 *)__va(shadow_addr)) + index; - if (is_present_pte(*shadow_ent) || is_io_pte(*shadow_ent)) { + if (is_shadow_present_pte(*shadow_ent)) { if (level == PT_PAGE_TABLE_LEVEL) break; shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK; @@ -500,6 +505,26 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) return gpa; } +static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp) +{ + int i; + pt_element_t *gpt; + + if (sp->role.metaphysical || PTTYPE == 32) { + nonpaging_prefetch_page(vcpu, sp); + return; + } + + gpt = kmap_atomic(gfn_to_page(vcpu->kvm, sp->gfn), KM_USER0); + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) + if (is_present_pte(gpt[i])) + sp->spt[i] = shadow_trap_nonpresent_pte; + else + sp->spt[i] = shadow_notrap_nonpresent_pte; + kunmap_atomic(gpt, KM_USER0); +} + #undef pt_element_t #undef guest_walker #undef FNAME @@ -508,4 +533,5 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) #undef SHADOW_PT_INDEX #undef PT_LEVEL_MASK #undef PT_DIR_BASE_ADDR_MASK +#undef PT_LEVEL_BITS #undef PT_MAX_FULL_LEVELS diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 8eb49e055ec0..27a3318fa6c2 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -33,6 +34,9 @@ MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); +static int bypass_guest_pf = 1; +module_param(bypass_guest_pf, bool, 0); + struct vmcs { u32 revision_id; u32 abort; @@ -1535,8 +1539,8 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) } vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); - vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0); - vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0); + vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); + vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ vmcs_writel(HOST_CR0, read_cr0()); /* 22.2.3 */ @@ -2582,6 +2586,9 @@ static int __init vmx_init(void) if (r) goto out1; + if (bypass_guest_pf) + kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull); + return 0; out1: -- cgit v1.2.3 From 12b7d28fc102b772eb70f98491587ec5ee717baf Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 23 Sep 2007 14:10:49 +0200 Subject: KVM: MMU: Make flooding detection work when guest page faults are bypassed When we allow guest page faults to reach the guests directly, we lose the fault tracking which allows us to detect demand paging. So we provide an alternate mechnism by clearing the accessed bit when we set a pte, and checking it later to see if the guest actually used it. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/mmu.c | 21 ++++++++++++++++++++- drivers/kvm/paging_tmpl.h | 9 ++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 7de948e9e64e..08ffc829f07f 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -346,6 +346,7 @@ struct kvm_vcpu { gfn_t last_pt_write_gfn; int last_pt_write_count; + u64 *last_pte_updated; struct kvm_guest_debug guest_debug; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 069ce83f018e..d347e895736e 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -692,6 +692,15 @@ static void kvm_mmu_put_page(struct kvm_mmu_page *page, mmu_page_remove_parent_pte(page, parent_pte); } +static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) +{ + int i; + + for (i = 0; i < KVM_MAX_VCPUS; ++i) + if (kvm->vcpus[i]) + kvm->vcpus[i]->last_pte_updated = NULL; +} + static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *page) { @@ -717,6 +726,7 @@ static void kvm_mmu_zap_page(struct kvm *kvm, kvm_mmu_free_page(kvm, page); } else list_move(&page->link, &kvm->active_mmu_pages); + kvm_mmu_reset_last_pte_updated(kvm); } static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) @@ -1140,6 +1150,13 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, offset_in_pte); } +static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) +{ + u64 *spte = vcpu->last_pte_updated; + + return !!(spte && (*spte & PT_ACCESSED_MASK)); +} + void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes) { @@ -1160,13 +1177,15 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); kvm_mmu_audit(vcpu, "pre pte write"); - if (gfn == vcpu->last_pt_write_gfn) { + if (gfn == vcpu->last_pt_write_gfn + && !last_updated_pte_accessed(vcpu)) { ++vcpu->last_pt_write_count; if (vcpu->last_pt_write_count >= 3) flooded = 1; } else { vcpu->last_pt_write_gfn = gfn; vcpu->last_pt_write_count = 1; + vcpu->last_pte_updated = NULL; } index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &vcpu->kvm->mmu_page_hash[index]; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 99ac9b15f773..be0f85231da9 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -238,7 +238,12 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, FNAME(mark_pagetable_dirty)(vcpu->kvm, walker); } - spte = PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_DIRTY_MASK; + /* + * We don't set the accessed bit, since we sometimes want to see + * whether the guest actually used the pte (in order to detect + * demand paging). + */ + spte = PT_PRESENT_MASK | PT_DIRTY_MASK; spte |= gpte & PT64_NX_MASK; if (!dirty) access_bits &= ~PT_WRITABLE_MASK; @@ -291,6 +296,8 @@ unshadowed: page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); if (!was_rmapped) rmap_add(vcpu, shadow_pte); + if (!ptwrite || !*ptwrite) + vcpu->last_pte_updated = shadow_pte; } static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, -- cgit v1.2.3 From 217648638ccb62dfeea5ac5fe768539cdee61ed0 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 18 Sep 2007 14:05:16 -0500 Subject: KVM: MMU: Ignore reserved bits in cr3 in non-pae mode This patch removes the fault injected when the guest attempts to set reserved bits in cr3. X86 hardware doesn't generate a fault when setting reserved bits. The result of this patch is that vmware-server, running within a kvm guest, boots and runs memtest from an iso. Signed-off-by: Ryan Harper Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 82cc7ae0fc83..2d55bab41634 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -554,14 +554,11 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) inject_gp(vcpu); return; } - } else { - if (cr3 & CR3_NONPAE_RESERVED_BITS) { - printk(KERN_DEBUG - "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); - return; - } } + /* + * We don't check reserved bits in nonpae mode, because + * this isn't enforced, and VMware depends on this. + */ } mutex_lock(&vcpu->kvm->lock); -- cgit v1.2.3 From 8cdbd2c9bf3108c41563eef586f22fa8b5174de0 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Mon, 24 Sep 2007 11:10:54 +0200 Subject: KVM: x86 emulator: split some decoding into functions for readability To improve readability, move push, writeback, and grp 1a/2/3/4/5/9 emulation parts into functions. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 451 +++++++++++++++++++++++++++------------------- 1 file changed, 266 insertions(+), 185 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index dc9d2a870fbc..a15609279283 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -907,6 +907,244 @@ done: return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; } +static inline void emulate_push(struct x86_emulate_ctxt *ctxt) +{ + struct decode_cache *c = &ctxt->decode; + + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], -c->op_bytes); + c->dst.ptr = (void *) register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]); +} + +static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc; + + /* 64-bit mode: POP always pops a 64-bit operand. */ + + if (ctxt->mode == X86EMUL_MODE_PROT64) + c->dst.bytes = 8; + + rc = ops->read_std(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), + &c->dst.val, c->dst.bytes, ctxt->vcpu); + if (rc != 0) + return rc; + + register_address_increment(c->regs[VCPU_REGS_RSP], c->dst.bytes); + + return 0; +} + +static inline void emulate_grp2(struct decode_cache *c, unsigned long *_eflags) +{ + switch (c->modrm_reg) { + case 0: /* rol */ + emulate_2op_SrcB("rol", c->src, c->dst, *_eflags); + break; + case 1: /* ror */ + emulate_2op_SrcB("ror", c->src, c->dst, *_eflags); + break; + case 2: /* rcl */ + emulate_2op_SrcB("rcl", c->src, c->dst, *_eflags); + break; + case 3: /* rcr */ + emulate_2op_SrcB("rcr", c->src, c->dst, *_eflags); + break; + case 4: /* sal/shl */ + case 6: /* sal/shl */ + emulate_2op_SrcB("sal", c->src, c->dst, *_eflags); + break; + case 5: /* shr */ + emulate_2op_SrcB("shr", c->src, c->dst, *_eflags); + break; + case 7: /* sar */ + emulate_2op_SrcB("sar", c->src, c->dst, *_eflags); + break; + } +} + +static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long *_eflags) +{ + struct decode_cache *c = &ctxt->decode; + int rc = 0; + + switch (c->modrm_reg) { + case 0 ... 1: /* test */ + /* + * Special case in Grp3: test has an immediate + * source operand. + */ + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + if (c->src.bytes == 8) + c->src.bytes = 4; + switch (c->src.bytes) { + case 1: + c->src.val = insn_fetch(s8, 1, c->eip); + break; + case 2: + c->src.val = insn_fetch(s16, 2, c->eip); + break; + case 4: + c->src.val = insn_fetch(s32, 4, c->eip); + break; + } + emulate_2op_SrcV("test", c->src, c->dst, *_eflags); + break; + case 2: /* not */ + c->dst.val = ~c->dst.val; + break; + case 3: /* neg */ + emulate_1op("neg", c->dst, *_eflags); + break; + default: + DPRINTF("Cannot emulate %02x\n", c->b); + rc = X86EMUL_UNHANDLEABLE; + break; + } +done: + return rc; +} + +static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long *_eflags, + int *no_wb) +{ + struct decode_cache *c = &ctxt->decode; + int rc; + + switch (c->modrm_reg) { + case 0: /* inc */ + emulate_1op("inc", c->dst, *_eflags); + break; + case 1: /* dec */ + emulate_1op("dec", c->dst, *_eflags); + break; + case 4: /* jmp abs */ + if (c->b == 0xff) + c->eip = c->dst.val; + else { + DPRINTF("Cannot emulate %02x\n", c->b); + return X86EMUL_UNHANDLEABLE; + } + break; + case 6: /* push */ + + /* 64-bit mode: PUSH always pushes a 64-bit operand. */ + + if (ctxt->mode == X86EMUL_MODE_PROT64) { + c->dst.bytes = 8; + rc = ops->read_std((unsigned long)c->dst.ptr, + &c->dst.val, 8, ctxt->vcpu); + if (rc != 0) + return rc; + } + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->dst.bytes); + rc = ops->write_emulated(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), &c->dst.val, + c->dst.bytes, ctxt->vcpu); + if (rc != 0) + return rc; + *no_wb = 1; + break; + default: + DPRINTF("Cannot emulate %02x\n", c->b); + return X86EMUL_UNHANDLEABLE; + } + return 0; +} + +static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long *_eflags, + unsigned long cr2) +{ + struct decode_cache *c = &ctxt->decode; + u64 old, new; + int rc; + + rc = ops->read_emulated(cr2, &old, 8, ctxt->vcpu); + if (rc != 0) + return rc; + + if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) || + ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) { + + c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); + c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); + *_eflags &= ~EFLG_ZF; + + } else { + new = ((u64)c->regs[VCPU_REGS_RCX] << 32) | + (u32) c->regs[VCPU_REGS_RBX]; + + rc = ops->cmpxchg_emulated(cr2, &old, &new, 8, ctxt->vcpu); + if (rc != 0) + return rc; + *_eflags |= EFLG_ZF; + } + return 0; +} + +static inline int writeback(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + int rc; + struct decode_cache *c = &ctxt->decode; + + switch (c->dst.type) { + case OP_REG: + /* The 4-byte case *is* correct: + * in 64-bit mode we zero-extend. + */ + switch (c->dst.bytes) { + case 1: + *(u8 *)c->dst.ptr = (u8)c->dst.val; + break; + case 2: + *(u16 *)c->dst.ptr = (u16)c->dst.val; + break; + case 4: + *c->dst.ptr = (u32)c->dst.val; + break; /* 64b: zero-ext */ + case 8: + *c->dst.ptr = c->dst.val; + break; + } + break; + case OP_MEM: + if (c->lock_prefix) + rc = ops->cmpxchg_emulated( + (unsigned long)c->dst.ptr, + &c->dst.orig_val, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); + else + rc = ops->write_emulated( + (unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); + if (rc != 0) + return rc; + default: + break; + } + return 0; +} + int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { @@ -1042,7 +1280,6 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) } break; case 0x84 ... 0x85: - test: /* test */ emulate_2op_SrcV("test", c->src, c->dst, _eflags); break; case 0x86 ... 0x87: /* xchg */ @@ -1074,18 +1311,9 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) c->dst.val = c->modrm_val; break; case 0x8f: /* pop (sole member of Grp1a) */ - /* 64-bit mode: POP always pops a 64-bit operand. */ - if (ctxt->mode == X86EMUL_MODE_PROT64) - c->dst.bytes = 8; - if ((rc = ops->read_std(register_address( - ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), - &c->dst.val, - c->dst.bytes, - ctxt->vcpu)) != 0) + rc = emulate_grp1a(ctxt, ops); + if (rc != 0) goto done; - register_address_increment(c->regs[VCPU_REGS_RSP], - c->dst.bytes); break; case 0xa0 ... 0xa1: /* mov */ c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; @@ -1099,31 +1327,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) c->eip += c->ad_bytes; break; case 0xc0 ... 0xc1: - grp2: /* Grp2 */ - switch (c->modrm_reg) { - case 0: /* rol */ - emulate_2op_SrcB("rol", c->src, c->dst, _eflags); - break; - case 1: /* ror */ - emulate_2op_SrcB("ror", c->src, c->dst, _eflags); - break; - case 2: /* rcl */ - emulate_2op_SrcB("rcl", c->src, c->dst, _eflags); - break; - case 3: /* rcr */ - emulate_2op_SrcB("rcr", c->src, c->dst, _eflags); - break; - case 4: /* sal/shl */ - case 6: /* sal/shl */ - emulate_2op_SrcB("sal", c->src, c->dst, _eflags); - break; - case 5: /* shr */ - emulate_2op_SrcB("shr", c->src, c->dst, _eflags); - break; - case 7: /* sar */ - emulate_2op_SrcB("sar", c->src, c->dst, _eflags); - break; - } + emulate_grp2(c, &_eflags); break; case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ mov: @@ -1131,126 +1335,29 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) break; case 0xd0 ... 0xd1: /* Grp2 */ c->src.val = 1; - goto grp2; + emulate_grp2(c, &_eflags); + break; case 0xd2 ... 0xd3: /* Grp2 */ c->src.val = c->regs[VCPU_REGS_RCX]; - goto grp2; + emulate_grp2(c, &_eflags); + break; case 0xf6 ... 0xf7: /* Grp3 */ - switch (c->modrm_reg) { - case 0 ... 1: /* test */ - /* - * Special case in Grp3: test has an immediate - * source operand. - */ - c->src.type = OP_IMM; - c->src.ptr = (unsigned long *)c->eip; - c->src.bytes = (c->d & ByteOp) ? 1 : - c->op_bytes; - if (c->src.bytes == 8) - c->src.bytes = 4; - switch (c->src.bytes) { - case 1: - c->src.val = insn_fetch(s8, 1, c->eip); - break; - case 2: - c->src.val = insn_fetch(s16, 2, c->eip); - break; - case 4: - c->src.val = insn_fetch(s32, 4, c->eip); - break; - } - goto test; - case 2: /* not */ - c->dst.val = ~c->dst.val; - break; - case 3: /* neg */ - emulate_1op("neg", c->dst, _eflags); - break; - default: - goto cannot_emulate; - } + rc = emulate_grp3(ctxt, ops, &_eflags); + if (rc != 0) + goto done; break; case 0xfe ... 0xff: /* Grp4/Grp5 */ - switch (c->modrm_reg) { - case 0: /* inc */ - emulate_1op("inc", c->dst, _eflags); - break; - case 1: /* dec */ - emulate_1op("dec", c->dst, _eflags); - break; - case 4: /* jmp abs */ - if (c->b == 0xff) - c->eip = c->dst.val; - else - goto cannot_emulate; - break; - case 6: /* push */ - /* 64-bit mode: PUSH always pushes a 64-bit operand. */ - if (ctxt->mode == X86EMUL_MODE_PROT64) { - c->dst.bytes = 8; - if ((rc = ops->read_std( - (unsigned long)c->dst.ptr, - &c->dst.val, 8, - ctxt->vcpu)) != 0) - goto done; - } - register_address_increment(c->regs[VCPU_REGS_RSP], - -c->dst.bytes); - if ((rc = ops->write_emulated( - register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), - &c->dst.val, - c->dst.bytes, ctxt->vcpu)) != 0) - goto done; - no_wb = 1; - break; - default: - goto cannot_emulate; - } + rc = emulate_grp45(ctxt, ops, &_eflags, &no_wb); + if (rc != 0) + goto done; break; } writeback: if (!no_wb) { - switch (c->dst.type) { - case OP_REG: - /* The 4-byte case *is* correct: - * in 64-bit mode we zero-extend. - */ - switch (c->dst.bytes) { - case 1: - *(u8 *)c->dst.ptr = (u8)c->dst.val; - break; - case 2: - *(u16 *)c->dst.ptr = (u16)c->dst.val; - break; - case 4: - *c->dst.ptr = (u32)c->dst.val; - break; /* 64b: zero-ext */ - case 8: - *c->dst.ptr = c->dst.val; - break; - } - break; - case OP_MEM: - if (c->lock_prefix) - rc = ops->cmpxchg_emulated( - (unsigned long)c->dst.ptr, - &c->dst.orig_val, - &c->dst.val, - c->dst.bytes, - ctxt->vcpu); - else - rc = ops->write_emulated( - (unsigned long)c->dst.ptr, - &c->dst.val, - c->dst.bytes, - ctxt->vcpu); - if (rc != 0) - goto done; - default: - break; - } + rc = writeback(ctxt, ops); + if (rc != 0) + goto done; } /* Commit shadow register state. */ @@ -1283,8 +1390,7 @@ special_insn: ctxt->ss_base, c->regs[VCPU_REGS_RSP]); break; case 0x58 ... 0x5f: /* pop reg */ - c->dst.ptr = - (unsigned long *)&c->regs[c->b & 0x7]; + c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; pop_instruction: if ((rc = ops->read_std(register_address(ctxt->ss_base, c->regs[VCPU_REGS_RSP]), c->dst.ptr, @@ -1298,14 +1404,7 @@ special_insn: case 0x6a: /* push imm8 */ c->src.val = 0L; c->src.val = insn_fetch(s8, 1, c->eip); -push: - c->dst.type = OP_MEM; - c->dst.bytes = c->op_bytes; - c->dst.val = c->src.val; - register_address_increment(c->regs[VCPU_REGS_RSP], - -c->op_bytes); - c->dst.ptr = (void *) register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]); + emulate_push(ctxt); break; case 0x6c: /* insb */ case 0x6d: /* insw/insd */ @@ -1350,7 +1449,8 @@ push: } case 0x9c: /* pushf */ c->src.val = (unsigned long) _eflags; - goto push; + emulate_push(ctxt); + break; case 0x9d: /* popf */ c->dst.ptr = (unsigned long *) &_eflags; goto pop_instruction; @@ -1436,7 +1536,8 @@ push: c->src.val = (unsigned long) c->eip; JMP_REL(rel); c->op_bytes = c->ad_bytes; - goto push; + emulate_push(ctxt); + break; } case 0xe9: /* jmp rel */ case 0xeb: /* jmp rel short */ @@ -1511,8 +1612,7 @@ twobyte_insn: no_wb = 1; if (c->modrm_mod != 3) goto cannot_emulate; - rc = emulator_get_dr(ctxt, c->modrm_reg, - &c->regs[c->modrm_rm]); + rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]); break; case 0x23: /* mov from reg to dr */ no_wb = 1; @@ -1668,8 +1768,7 @@ twobyte_special_insn: break; case 0x32: /* rdmsr */ - rc = kvm_get_msr(ctxt->vcpu, - c->regs[VCPU_REGS_RCX], &msr_data); + rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); if (rc) { kvm_x86_ops->inject_gp(ctxt->vcpu, 0); c->eip = ctxt->vcpu->rip; @@ -1701,28 +1800,10 @@ twobyte_special_insn: break; } case 0xc7: /* Grp9 (cmpxchg8b) */ - { - u64 old, new; - if ((rc = ops->read_emulated(cr2, &old, 8, ctxt->vcpu)) - != 0) - goto done; - if (((u32) (old >> 0) != - (u32) c->regs[VCPU_REGS_RAX]) || - ((u32) (old >> 32) != - (u32) c->regs[VCPU_REGS_RDX])) { - c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); - c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); - _eflags &= ~EFLG_ZF; - } else { - new = ((u64)c->regs[VCPU_REGS_RCX] << 32) - | (u32) c->regs[VCPU_REGS_RBX]; - if ((rc = ops->cmpxchg_emulated(cr2, &old, - &new, 8, ctxt->vcpu)) != 0) - goto done; - _eflags |= EFLG_ZF; - } - break; - } + rc = emulate_grp9(ctxt, ops, &_eflags, cr2); + if (rc != 0) + goto done; + break; } goto writeback; -- cgit v1.2.3 From 05f086f87ef17da70d5f4e9b063e9ea31ab20f28 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Mon, 24 Sep 2007 11:10:55 +0200 Subject: KVM: x86 emulator: remove _eflags and use directly ctxt->eflags. Remove _eflags and use directly ctxt->eflags. Caching eflags is not needed as it is restored to vcpu by kvm_main.c:emulate_instruction() from ctxt->eflags only if emulation doesn't fail. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 121 ++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 62 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index a15609279283..76b813b6365c 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -941,37 +941,37 @@ static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, return 0; } -static inline void emulate_grp2(struct decode_cache *c, unsigned long *_eflags) +static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt) { + struct decode_cache *c = &ctxt->decode; switch (c->modrm_reg) { case 0: /* rol */ - emulate_2op_SrcB("rol", c->src, c->dst, *_eflags); + emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags); break; case 1: /* ror */ - emulate_2op_SrcB("ror", c->src, c->dst, *_eflags); + emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags); break; case 2: /* rcl */ - emulate_2op_SrcB("rcl", c->src, c->dst, *_eflags); + emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags); break; case 3: /* rcr */ - emulate_2op_SrcB("rcr", c->src, c->dst, *_eflags); + emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags); break; case 4: /* sal/shl */ case 6: /* sal/shl */ - emulate_2op_SrcB("sal", c->src, c->dst, *_eflags); + emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags); break; case 5: /* shr */ - emulate_2op_SrcB("shr", c->src, c->dst, *_eflags); + emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags); break; case 7: /* sar */ - emulate_2op_SrcB("sar", c->src, c->dst, *_eflags); + emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags); break; } } static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - unsigned long *_eflags) + struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; int rc = 0; @@ -998,13 +998,13 @@ static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt, c->src.val = insn_fetch(s32, 4, c->eip); break; } - emulate_2op_SrcV("test", c->src, c->dst, *_eflags); + emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags); break; case 2: /* not */ c->dst.val = ~c->dst.val; break; case 3: /* neg */ - emulate_1op("neg", c->dst, *_eflags); + emulate_1op("neg", c->dst, ctxt->eflags); break; default: DPRINTF("Cannot emulate %02x\n", c->b); @@ -1017,7 +1017,6 @@ done: static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops, - unsigned long *_eflags, int *no_wb) { struct decode_cache *c = &ctxt->decode; @@ -1025,10 +1024,10 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, switch (c->modrm_reg) { case 0: /* inc */ - emulate_1op("inc", c->dst, *_eflags); + emulate_1op("inc", c->dst, ctxt->eflags); break; case 1: /* dec */ - emulate_1op("dec", c->dst, *_eflags); + emulate_1op("dec", c->dst, ctxt->eflags); break; case 4: /* jmp abs */ if (c->b == 0xff) @@ -1067,7 +1066,6 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops, - unsigned long *_eflags, unsigned long cr2) { struct decode_cache *c = &ctxt->decode; @@ -1083,7 +1081,7 @@ static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); - *_eflags &= ~EFLG_ZF; + ctxt->eflags &= ~EFLG_ZF; } else { new = ((u64)c->regs[VCPU_REGS_RCX] << 32) | @@ -1092,7 +1090,7 @@ static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, rc = ops->cmpxchg_emulated(cr2, &old, &new, 8, ctxt->vcpu); if (rc != 0) return rc; - *_eflags |= EFLG_ZF; + ctxt->eflags |= EFLG_ZF; } return 0; } @@ -1152,7 +1150,6 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) int no_wb = 0; u64 msr_data; unsigned long saved_eip = 0; - unsigned long _eflags = ctxt->eflags; struct decode_cache *c = &ctxt->decode; int rc = 0; @@ -1207,23 +1204,23 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) switch (c->b) { case 0x00 ... 0x05: add: /* add */ - emulate_2op_SrcV("add", c->src, c->dst, _eflags); + emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags); break; case 0x08 ... 0x0d: or: /* or */ - emulate_2op_SrcV("or", c->src, c->dst, _eflags); + emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags); break; case 0x10 ... 0x15: adc: /* adc */ - emulate_2op_SrcV("adc", c->src, c->dst, _eflags); + emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags); break; case 0x18 ... 0x1d: sbb: /* sbb */ - emulate_2op_SrcV("sbb", c->src, c->dst, _eflags); + emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags); break; case 0x20 ... 0x23: and: /* and */ - emulate_2op_SrcV("and", c->src, c->dst, _eflags); + emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags); break; case 0x24: /* and al imm8 */ c->dst.type = OP_REG; @@ -1244,15 +1241,15 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) goto and; case 0x28 ... 0x2d: sub: /* sub */ - emulate_2op_SrcV("sub", c->src, c->dst, _eflags); + emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags); break; case 0x30 ... 0x35: xor: /* xor */ - emulate_2op_SrcV("xor", c->src, c->dst, _eflags); + emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags); break; case 0x38 ... 0x3d: cmp: /* cmp */ - emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); break; case 0x63: /* movsxd */ if (ctxt->mode != X86EMUL_MODE_PROT64) @@ -1280,7 +1277,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) } break; case 0x84 ... 0x85: - emulate_2op_SrcV("test", c->src, c->dst, _eflags); + emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags); break; case 0x86 ... 0x87: /* xchg */ /* Write back the register source. */ @@ -1327,7 +1324,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) c->eip += c->ad_bytes; break; case 0xc0 ... 0xc1: - emulate_grp2(c, &_eflags); + emulate_grp2(ctxt); break; case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ mov: @@ -1335,19 +1332,19 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) break; case 0xd0 ... 0xd1: /* Grp2 */ c->src.val = 1; - emulate_grp2(c, &_eflags); + emulate_grp2(ctxt); break; case 0xd2 ... 0xd3: /* Grp2 */ c->src.val = c->regs[VCPU_REGS_RCX]; - emulate_grp2(c, &_eflags); + emulate_grp2(ctxt); break; case 0xf6 ... 0xf7: /* Grp3 */ - rc = emulate_grp3(ctxt, ops, &_eflags); + rc = emulate_grp3(ctxt, ops); if (rc != 0) goto done; break; case 0xfe ... 0xff: /* Grp4/Grp5 */ - rc = emulate_grp45(ctxt, ops, &_eflags, &no_wb); + rc = emulate_grp45(ctxt, ops, &no_wb); if (rc != 0) goto done; break; @@ -1362,7 +1359,6 @@ writeback: /* Commit shadow register state. */ memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); - ctxt->eflags = _eflags; ctxt->vcpu->rip = c->eip; done: @@ -1413,7 +1409,7 @@ special_insn: (c->d & ByteOp) ? 1 : c->op_bytes, c->rep_prefix ? address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (_eflags & EFLG_DF), + (ctxt->eflags & EFLG_DF), register_address(ctxt->es_base, c->regs[VCPU_REGS_RDI]), c->rep_prefix, @@ -1429,7 +1425,7 @@ special_insn: (c->d & ByteOp) ? 1 : c->op_bytes, c->rep_prefix ? address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (_eflags & EFLG_DF), + (ctxt->eflags & EFLG_DF), register_address(c->override_base ? *c->override_base : ctxt->ds_base, @@ -1443,16 +1439,16 @@ special_insn: case 0x70 ... 0x7f: /* jcc (short) */ { int rel = insn_fetch(s8, 1, c->eip); - if (test_cc(c->b, _eflags)) + if (test_cc(c->b, ctxt->eflags)) JMP_REL(rel); break; } case 0x9c: /* pushf */ - c->src.val = (unsigned long) _eflags; + c->src.val = (unsigned long) ctxt->eflags; emulate_push(ctxt); break; case 0x9d: /* popf */ - c->dst.ptr = (unsigned long *) &_eflags; + c->dst.ptr = (unsigned long *) &ctxt->eflags; goto pop_instruction; case 0xc3: /* ret */ c->dst.ptr = &c->eip; @@ -1484,10 +1480,10 @@ special_insn: c->dst.bytes, ctxt->vcpu)) != 0) goto done; register_address_increment(c->regs[VCPU_REGS_RSI], - (_eflags & EFLG_DF) ? -c->dst.bytes + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes : c->dst.bytes); register_address_increment(c->regs[VCPU_REGS_RDI], - (_eflags & EFLG_DF) ? -c->dst.bytes + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes : c->dst.bytes); break; case 0xa6 ... 0xa7: /* cmps */ @@ -1499,7 +1495,7 @@ special_insn: c->dst.ptr = (unsigned long *)cr2; c->dst.val = c->regs[VCPU_REGS_RAX]; register_address_increment(c->regs[VCPU_REGS_RDI], - (_eflags & EFLG_DF) ? -c->dst.bytes + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes : c->dst.bytes); break; case 0xac ... 0xad: /* lods */ @@ -1511,7 +1507,7 @@ special_insn: ctxt->vcpu)) != 0) goto done; register_address_increment(c->regs[VCPU_REGS_RSI], - (_eflags & EFLG_DF) ? -c->dst.bytes + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes : c->dst.bytes); break; case 0xae ... 0xaf: /* scas */ @@ -1599,7 +1595,8 @@ twobyte_insn: case 6: /* lmsw */ if (c->modrm_mod != 3) goto cannot_emulate; - realmode_lmsw(ctxt->vcpu, (u16)c->modrm_val, &_eflags); + realmode_lmsw(ctxt->vcpu, (u16)c->modrm_val, + &ctxt->eflags); break; case 7: /* invlpg*/ emulate_invlpg(ctxt->vcpu, cr2); @@ -1630,29 +1627,29 @@ twobyte_insn: */ switch ((c->b & 15) >> 1) { case 0: /* cmovo */ - no_wb = (_eflags & EFLG_OF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_OF) ? 0 : 1; break; case 1: /* cmovb/cmovc/cmovnae */ - no_wb = (_eflags & EFLG_CF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_CF) ? 0 : 1; break; case 2: /* cmovz/cmove */ - no_wb = (_eflags & EFLG_ZF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_ZF) ? 0 : 1; break; case 3: /* cmovbe/cmovna */ - no_wb = (_eflags & (EFLG_CF | EFLG_ZF)) ? 0 : 1; + no_wb = (ctxt->eflags & (EFLG_CF | EFLG_ZF)) ? 0 : 1; break; case 4: /* cmovs */ - no_wb = (_eflags & EFLG_SF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_SF) ? 0 : 1; break; case 5: /* cmovp/cmovpe */ - no_wb = (_eflags & EFLG_PF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_PF) ? 0 : 1; break; case 7: /* cmovle/cmovng */ - no_wb = (_eflags & EFLG_ZF) ? 0 : 1; + no_wb = (ctxt->eflags & EFLG_ZF) ? 0 : 1; /* fall through */ case 6: /* cmovl/cmovnge */ - no_wb &= (!(_eflags & EFLG_SF) != - !(_eflags & EFLG_OF)) ? 0 : 1; + no_wb &= (!(ctxt->eflags & EFLG_SF) != + !(ctxt->eflags & EFLG_OF)) ? 0 : 1; break; } /* Odd cmov opcodes (lsb == 1) have inverted sense. */ @@ -1662,13 +1659,13 @@ twobyte_insn: bt: /* bt */ /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("bt", c->src, c->dst, _eflags); + emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags); break; case 0xab: bts: /* bts */ /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("bts", c->src, c->dst, _eflags); + emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags); break; case 0xb0 ... 0xb1: /* cmpxchg */ /* @@ -1677,8 +1674,8 @@ twobyte_insn: */ c->src.orig_val = c->src.val; c->src.val = c->regs[VCPU_REGS_RAX]; - emulate_2op_SrcV("cmp", c->src, c->dst, _eflags); - if (_eflags & EFLG_ZF) { + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); + if (ctxt->eflags & EFLG_ZF) { /* Success: write back to memory. */ c->dst.val = c->src.orig_val; } else { @@ -1691,7 +1688,7 @@ twobyte_insn: btr: /* btr */ /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("btr", c->src, c->dst, _eflags); + emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags); break; case 0xb6 ... 0xb7: /* movzx */ c->dst.bytes = c->op_bytes; @@ -1714,7 +1711,7 @@ twobyte_insn: btc: /* btc */ /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("btc", c->src, c->dst, _eflags); + emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags); break; case 0xbe ... 0xbf: /* movsx */ c->dst.bytes = c->op_bytes; @@ -1753,7 +1750,7 @@ twobyte_special_insn: if (c->modrm_mod != 3) goto cannot_emulate; realmode_set_cr(ctxt->vcpu, - c->modrm_reg, c->modrm_val, &_eflags); + c->modrm_reg, c->modrm_val, &ctxt->eflags); break; case 0x30: /* wrmsr */ @@ -1795,12 +1792,12 @@ twobyte_special_insn: DPRINTF("jnz: Invalid op_bytes\n"); goto cannot_emulate; } - if (test_cc(c->b, _eflags)) + if (test_cc(c->b, ctxt->eflags)) JMP_REL(rel); break; } case 0xc7: /* Grp9 (cmpxchg8b) */ - rc = emulate_grp9(ctxt, ops, &_eflags, cr2); + rc = emulate_grp9(ctxt, ops, cr2); if (rc != 0) goto done; break; -- cgit v1.2.3 From a01af5ec5163cdc51c00f6dcb22d7766199377cf Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Mon, 24 Sep 2007 11:10:56 +0200 Subject: KVM: x86 emulator: Remove no_wb, use dst.type = OP_NONE instead Remove no_wb, use dst.type = OP_NONE instead, idea stollen from xen-3.1 Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 76 +++++++++++++++-------------------------------- drivers/kvm/x86_emulate.h | 2 +- 2 files changed, 25 insertions(+), 53 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 76b813b6365c..6161e3f66585 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1016,8 +1016,7 @@ done: } static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - int *no_wb) + struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; int rc; @@ -1055,7 +1054,7 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, c->dst.bytes, ctxt->vcpu); if (rc != 0) return rc; - *no_wb = 1; + c->dst.type = OP_NONE; break; default: DPRINTF("Cannot emulate %02x\n", c->b); @@ -1137,6 +1136,10 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt, ctxt->vcpu); if (rc != 0) return rc; + break; + case OP_NONE: + /* no writeback */ + break; default: break; } @@ -1147,7 +1150,6 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { unsigned long cr2 = ctxt->cr2; - int no_wb = 0; u64 msr_data; unsigned long saved_eip = 0; struct decode_cache *c = &ctxt->decode; @@ -1344,18 +1346,16 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) goto done; break; case 0xfe ... 0xff: /* Grp4/Grp5 */ - rc = emulate_grp45(ctxt, ops, &no_wb); + rc = emulate_grp45(ctxt, ops); if (rc != 0) goto done; break; } writeback: - if (!no_wb) { - rc = writeback(ctxt, ops); - if (rc != 0) - goto done; - } + rc = writeback(ctxt, ops); + if (rc != 0) + goto done; /* Commit shadow register state. */ memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); @@ -1395,7 +1395,7 @@ special_insn: register_address_increment(c->regs[VCPU_REGS_RSP], c->op_bytes); - no_wb = 1; /* Disable writeback. */ + c->dst.type = OP_NONE; /* Disable writeback. */ break; case 0x6a: /* push imm8 */ c->src.val = 0L; @@ -1538,7 +1538,7 @@ special_insn: case 0xe9: /* jmp rel */ case 0xeb: /* jmp rel short */ JMP_REL(c->src.val); - no_wb = 1; /* Disable writeback. */ + c->dst.type = OP_NONE; /* Disable writeback. */ break; @@ -1548,8 +1548,6 @@ special_insn: twobyte_insn: switch (c->b) { case 0x01: /* lgdt, lidt, lmsw */ - /* Disable writeback. */ - no_wb = 1; switch (c->modrm_reg) { u16 size; unsigned long address; @@ -1604,56 +1602,30 @@ twobyte_insn: default: goto cannot_emulate; } + /* Disable writeback. */ + c->dst.type = OP_NONE; break; case 0x21: /* mov from dr to reg */ - no_wb = 1; if (c->modrm_mod != 3) goto cannot_emulate; rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]); + if (rc) + goto cannot_emulate; + c->dst.type = OP_NONE; /* no writeback */ break; case 0x23: /* mov from reg to dr */ - no_wb = 1; if (c->modrm_mod != 3) goto cannot_emulate; rc = emulator_set_dr(ctxt, c->modrm_reg, c->regs[c->modrm_rm]); + if (rc) + goto cannot_emulate; + c->dst.type = OP_NONE; /* no writeback */ break; case 0x40 ... 0x4f: /* cmov */ c->dst.val = c->dst.orig_val = c->src.val; - no_wb = 1; - /* - * First, assume we're decoding an even cmov opcode - * (lsb == 0). - */ - switch ((c->b & 15) >> 1) { - case 0: /* cmovo */ - no_wb = (ctxt->eflags & EFLG_OF) ? 0 : 1; - break; - case 1: /* cmovb/cmovc/cmovnae */ - no_wb = (ctxt->eflags & EFLG_CF) ? 0 : 1; - break; - case 2: /* cmovz/cmove */ - no_wb = (ctxt->eflags & EFLG_ZF) ? 0 : 1; - break; - case 3: /* cmovbe/cmovna */ - no_wb = (ctxt->eflags & (EFLG_CF | EFLG_ZF)) ? 0 : 1; - break; - case 4: /* cmovs */ - no_wb = (ctxt->eflags & EFLG_SF) ? 0 : 1; - break; - case 5: /* cmovp/cmovpe */ - no_wb = (ctxt->eflags & EFLG_PF) ? 0 : 1; - break; - case 7: /* cmovle/cmovng */ - no_wb = (ctxt->eflags & EFLG_ZF) ? 0 : 1; - /* fall through */ - case 6: /* cmovl/cmovnge */ - no_wb &= (!(ctxt->eflags & EFLG_SF) != - !(ctxt->eflags & EFLG_OF)) ? 0 : 1; - break; - } - /* Odd cmov opcodes (lsb == 1) have inverted sense. */ - no_wb ^= c->b & 1; + if (!test_cc(c->b, ctxt->eflags)) + c->dst.type = OP_NONE; /* no writeback */ break; case 0xa3: bt: /* bt */ @@ -1727,8 +1699,6 @@ twobyte_insn: goto writeback; twobyte_special_insn: - /* Disable writeback. */ - no_wb = 1; switch (c->b) { case 0x06: emulate_clts(ctxt->vcpu); @@ -1802,6 +1772,8 @@ twobyte_special_insn: goto done; break; } + /* Disable writeback. */ + c->dst.type = OP_NONE; goto writeback; cannot_emulate: diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index 28acad416bca..f03b1287700a 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -114,7 +114,7 @@ struct x86_emulate_ops { /* Type, address-of, and value of an instruction's operand. */ struct operand { - enum { OP_REG, OP_MEM, OP_IMM } type; + enum { OP_REG, OP_MEM, OP_IMM, OP_NONE } type; unsigned int bytes; unsigned long val, orig_val, *ptr; }; -- cgit v1.2.3 From e4f8e03956de4b1a1dc4bbbdefa9c973d01ee91f Mon Sep 17 00:00:00 2001 From: Qing He Date: Mon, 24 Sep 2007 17:22:13 +0800 Subject: KVM: x86_emulator: no writeback for bt Signed-off-by: Qing He Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 6161e3f66585..0afe660aec4b 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1629,6 +1629,7 @@ twobyte_insn: break; case 0xa3: bt: /* bt */ + c->dst.type = OP_NONE; /* only subword offset */ c->src.val &= (c->dst.bytes << 3) - 1; emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags); -- cgit v1.2.3 From a22436b7b8ec9b14a0451d9ac0fdc9d370bd7800 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Mon, 24 Sep 2007 17:00:58 +0200 Subject: KVM: Purify x86_decode_insn() error case management The only valid case is on protected page access, other cases are errors. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 2d55bab41634..888b0924c17b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1251,7 +1251,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, u16 error_code, int no_decode) { - int r = 0; + int r; vcpu->mmio_fault_cr2 = cr2; kvm_x86_ops->cache_regs(vcpu); @@ -1294,10 +1294,14 @@ int emulate_instruction(struct kvm_vcpu *vcpu, get_segment_base(vcpu, VCPU_SREG_FS); r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + if (r) { + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; + return EMULATE_FAIL; + } } - if (r == 0) - r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); + r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); if (vcpu->pio.string) return EMULATE_DO_MMIO; -- cgit v1.2.3 From b4c6abfef400c0f74d9b86a149a6719706cfdbbc Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Tue, 25 Sep 2007 13:36:40 +0200 Subject: KVM: x86 emulator: Any legacy prefix after a REX prefix nullifies its effect This patch modifies the management of REX prefix according behavior I saw in Xen 3.1. In Xen, this modification has been introduced by Jan Beulich. http://lists.xensource.com/archives/html/xen-changelog/2007-01/msg00081.html Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 0afe660aec4b..e294d8409571 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -521,7 +521,6 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; u8 sib, rex_prefix = 0; - unsigned int i; int rc = 0; int mode = ctxt->mode; int index_reg = 0, base_reg = 0, scale, rip_relative = 0; @@ -551,7 +550,7 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) } /* Legacy prefixes. */ - for (i = 0; i < 8; i++) { + for (;;) { switch (c->b = insn_fetch(u8, 1, c->eip)) { case 0x66: /* operand-size override */ c->op_bytes ^= 6; /* switch between 2/4 bytes */ @@ -582,6 +581,11 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) case 0x36: /* SS override */ c->override_base = &ctxt->ss_base; break; + case 0x40 ... 0x4f: /* REX */ + if (mode != X86EMUL_MODE_PROT64) + goto done_prefixes; + rex_prefix = c->b; + continue; case 0xf0: /* LOCK */ c->lock_prefix = 1; break; @@ -592,19 +596,21 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) default: goto done_prefixes; } + + /* Any legacy prefix after a REX prefix nullifies its effect. */ + + rex_prefix = 0; } done_prefixes: /* REX prefix. */ - if ((mode == X86EMUL_MODE_PROT64) && ((c->b & 0xf0) == 0x40)) { - rex_prefix = c->b; - if (c->b & 8) + if (rex_prefix) { + if (rex_prefix & 8) c->op_bytes = 8; /* REX.W */ - c->modrm_reg = (c->b & 4) << 1; /* REX.R */ - index_reg = (c->b & 2) << 2; /* REX.X */ - c->modrm_rm = base_reg = (c->b & 1) << 3; /* REG.B */ - c->b = insn_fetch(u8, 1, c->eip); + c->modrm_reg = (rex_prefix & 4) << 1; /* REX.R */ + index_reg = (rex_prefix & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (rex_prefix & 1) << 3; /* REG.B */ } /* Opcode byte(s). */ -- cgit v1.2.3 From eae5ecb5b9043812968fae7ad9d74bf5e7a50245 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 30 Sep 2007 10:50:12 +0200 Subject: KVM: VMX: Don't clear the vmcs if the vcpu is not loaded on any processor Noted by Eddie Dong. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 27a3318fa6c2..cc2844203c24 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -225,7 +225,9 @@ static void __vcpu_clear(void *arg) static void vcpu_clear(struct vcpu_vmx *vmx) { - if (vmx->vcpu.cpu != raw_smp_processor_id() && vmx->vcpu.cpu != -1) + if (vmx->vcpu.cpu == -1) + return; + if (vmx->vcpu.cpu != raw_smp_processor_id()) smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1); else -- cgit v1.2.3 From f566e09fc2c9f4164e1f0017c8c1c7a18bad7d72 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 30 Sep 2007 11:02:53 +0200 Subject: KVM: VMX: Simplify vcpu_clear() Now that smp_call_function_single() knows how to call a function on the current cpu, there's no need to check explicitly. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index cc2844203c24..439873a25cca 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -227,11 +227,7 @@ static void vcpu_clear(struct vcpu_vmx *vmx) { if (vmx->vcpu.cpu == -1) return; - if (vmx->vcpu.cpu != raw_smp_processor_id()) - smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, - vmx, 0, 1); - else - __vcpu_clear(vmx); + smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1); vmx->launched = 0; } -- cgit v1.2.3 From 290fc38da8187b53b78dd4d5ab27a20b88ef8b61 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Thu, 27 Sep 2007 14:11:22 +0200 Subject: KVM: Remove the usage of page->private field by rmap When kvm uses user-allocated pages in the future for the guest, we won't be able to use page->private for rmap, since page->rmap is reserved for the filesystem. So we move the rmap base pointers to the memory slot. A side effect of this is that we need to store the gfn of each gpte in the shadow pages, since the memory slot is addressed by gfn, instead of hfn like struct page. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 6 ++- drivers/kvm/kvm_main.c | 11 ++++- drivers/kvm/mmu.c | 122 ++++++++++++++++++++++++++-------------------- drivers/kvm/paging_tmpl.h | 3 +- 4 files changed, 86 insertions(+), 56 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 08ffc829f07f..80cfb99fffe0 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -126,6 +126,8 @@ struct kvm_mmu_page { union kvm_mmu_page_role role; u64 *spt; + /* hold the gfn of each spte inside spt */ + gfn_t *gfns; unsigned long slot_bitmap; /* One bit set per slot which has memory * in this shadow page. */ @@ -159,7 +161,7 @@ struct kvm_mmu { u64 *pae_root; }; -#define KVM_NR_MEM_OBJS 20 +#define KVM_NR_MEM_OBJS 40 struct kvm_mmu_memory_cache { int nobjs; @@ -402,6 +404,7 @@ struct kvm_memory_slot { unsigned long npages; unsigned long flags; struct page **phys_mem; + unsigned long *rmap; unsigned long *dirty_bitmap; }; @@ -554,6 +557,7 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); extern hpa_t bad_page_address; +gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); void mark_page_dirty(struct kvm *kvm, gfn_t gfn); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 888b0924c17b..9510e2276ca3 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -309,6 +309,8 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, __free_page(free->phys_mem[i]); vfree(free->phys_mem); } + if (!dont || free->rmap != dont->rmap) + vfree(free->rmap); if (!dont || free->dirty_bitmap != dont->dirty_bitmap) vfree(free->dirty_bitmap); @@ -719,13 +721,18 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, if (!new.phys_mem) goto out_unlock; + new.rmap = vmalloc(npages * sizeof(struct page*)); + + if (!new.rmap) + goto out_unlock; + memset(new.phys_mem, 0, npages * sizeof(struct page *)); + memset(new.rmap, 0, npages * sizeof(*new.rmap)); for (i = 0; i < npages; ++i) { new.phys_mem[i] = alloc_page(GFP_HIGHUSER | __GFP_ZERO); if (!new.phys_mem[i]) goto out_unlock; - set_page_private(new.phys_mem[i],0); } } @@ -909,7 +916,7 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) return r; } -static gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) +gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) { int i; struct kvm_mem_alias *alias; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index d347e895736e..72757db15065 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -276,7 +276,7 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) rmap_desc_cache, 1); if (r) goto out; - r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 4); + r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 8); if (r) goto out; r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache, @@ -326,36 +326,53 @@ static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd) kfree(rd); } +/* + * Take gfn and return the reverse mapping to it. + * Note: gfn must be unaliased before this function get called + */ + +static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot; + + slot = gfn_to_memslot(kvm, gfn); + return &slot->rmap[gfn - slot->base_gfn]; +} + /* * Reverse mapping data structures: * - * If page->private bit zero is zero, then page->private points to the - * shadow page table entry that points to page_address(page). + * If rmapp bit zero is zero, then rmapp point to the shadw page table entry + * that points to page_address(page). * - * If page->private bit zero is one, (then page->private & ~1) points - * to a struct kvm_rmap_desc containing more mappings. + * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc + * containing more mappings. */ -static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte) +static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) { - struct page *page; + struct kvm_mmu_page *page; struct kvm_rmap_desc *desc; + unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; - page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - if (!page_private(page)) { + gfn = unalias_gfn(vcpu->kvm, gfn); + page = page_header(__pa(spte)); + page->gfns[spte - page->spt] = gfn; + rmapp = gfn_to_rmap(vcpu->kvm, gfn); + if (!*rmapp) { rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte); - set_page_private(page,(unsigned long)spte); - } else if (!(page_private(page) & 1)) { + *rmapp = (unsigned long)spte; + } else if (!(*rmapp & 1)) { rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte); desc = mmu_alloc_rmap_desc(vcpu); - desc->shadow_ptes[0] = (u64 *)page_private(page); + desc->shadow_ptes[0] = (u64 *)*rmapp; desc->shadow_ptes[1] = spte; - set_page_private(page,(unsigned long)desc | 1); + *rmapp = (unsigned long)desc | 1; } else { rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte); - desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul); + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); while (desc->shadow_ptes[RMAP_EXT-1] && desc->more) desc = desc->more; if (desc->shadow_ptes[RMAP_EXT-1]) { @@ -368,7 +385,7 @@ static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte) } } -static void rmap_desc_remove_entry(struct page *page, +static void rmap_desc_remove_entry(unsigned long *rmapp, struct kvm_rmap_desc *desc, int i, struct kvm_rmap_desc *prev_desc) @@ -382,44 +399,46 @@ static void rmap_desc_remove_entry(struct page *page, if (j != 0) return; if (!prev_desc && !desc->more) - set_page_private(page,(unsigned long)desc->shadow_ptes[0]); + *rmapp = (unsigned long)desc->shadow_ptes[0]; else if (prev_desc) prev_desc->more = desc->more; else - set_page_private(page,(unsigned long)desc->more | 1); + *rmapp = (unsigned long)desc->more | 1; mmu_free_rmap_desc(desc); } -static void rmap_remove(u64 *spte) +static void rmap_remove(struct kvm *kvm, u64 *spte) { - struct page *page; struct kvm_rmap_desc *desc; struct kvm_rmap_desc *prev_desc; + struct kvm_mmu_page *page; + unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; - page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - if (!page_private(page)) { + page = page_header(__pa(spte)); + rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]); + if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); BUG(); - } else if (!(page_private(page) & 1)) { + } else if (!(*rmapp & 1)) { rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte); - if ((u64 *)page_private(page) != spte) { + if ((u64 *)*rmapp != spte) { printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n", spte, *spte); BUG(); } - set_page_private(page,0); + *rmapp = 0; } else { rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte); - desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul); + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); prev_desc = NULL; while (desc) { for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) if (desc->shadow_ptes[i] == spte) { - rmap_desc_remove_entry(page, + rmap_desc_remove_entry(rmapp, desc, i, prev_desc); return; @@ -433,28 +452,25 @@ static void rmap_remove(u64 *spte) static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) { - struct kvm *kvm = vcpu->kvm; - struct page *page; struct kvm_rmap_desc *desc; + unsigned long *rmapp; u64 *spte; - page = gfn_to_page(kvm, gfn); - BUG_ON(!page); + gfn = unalias_gfn(vcpu->kvm, gfn); + rmapp = gfn_to_rmap(vcpu->kvm, gfn); - while (page_private(page)) { - if (!(page_private(page) & 1)) - spte = (u64 *)page_private(page); + while (*rmapp) { + if (!(*rmapp & 1)) + spte = (u64 *)*rmapp; else { - desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul); + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); spte = desc->shadow_ptes[0]; } BUG_ON(!spte); - BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT - != page_to_pfn(page)); BUG_ON(!(*spte & PT_PRESENT_MASK)); BUG_ON(!(*spte & PT_WRITABLE_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - rmap_remove(spte); + rmap_remove(vcpu->kvm, spte); set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); kvm_flush_remote_tlbs(vcpu->kvm); } @@ -482,6 +498,7 @@ static void kvm_mmu_free_page(struct kvm *kvm, ASSERT(is_empty_shadow_page(page_head->spt)); list_del(&page_head->link); __free_page(virt_to_page(page_head->spt)); + __free_page(virt_to_page(page_head->gfns)); kfree(page_head); ++kvm->n_free_mmu_pages; } @@ -502,6 +519,7 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache, sizeof *page); page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); + page->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); set_page_private(virt_to_page(page->spt), (unsigned long)page); list_add(&page->link, &vcpu->kvm->active_mmu_pages); ASSERT(is_empty_shadow_page(page->spt)); @@ -667,7 +685,7 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, if (page->role.level == PT_PAGE_TABLE_LEVEL) { for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { if (is_shadow_present_pte(pt[i])) - rmap_remove(&pt[i]); + rmap_remove(kvm, &pt[i]); pt[i] = shadow_trap_nonpresent_pte; } kvm_flush_remote_tlbs(kvm); @@ -832,7 +850,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) page_header_update_slot(vcpu->kvm, table, v); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; - rmap_add(vcpu, &table[index]); + rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); return 0; } @@ -1123,7 +1141,7 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, pte = *spte; if (is_shadow_present_pte(pte)) { if (page->role.level == PT_PAGE_TABLE_LEVEL) - rmap_remove(spte); + rmap_remove(vcpu->kvm, spte); else { child = page_header(pte & PT64_BASE_ADDR_MASK); mmu_page_remove_parent_pte(child, spte); @@ -1340,7 +1358,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ if (pt[i] & PT_WRITABLE_MASK) { - rmap_remove(&pt[i]); + rmap_remove(kvm, &pt[i]); pt[i] &= ~PT_WRITABLE_MASK; } } @@ -1470,15 +1488,15 @@ static int count_rmaps(struct kvm_vcpu *vcpu) struct kvm_rmap_desc *d; for (j = 0; j < m->npages; ++j) { - struct page *page = m->phys_mem[j]; + unsigned long *rmapp = &m->rmap[j]; - if (!page->private) + if (!*rmapp) continue; - if (!(page->private & 1)) { + if (!(*rmapp & 1)) { ++nmaps; continue; } - d = (struct kvm_rmap_desc *)(page->private & ~1ul); + d = (struct kvm_rmap_desc *)(*rmapp & ~1ul); while (d) { for (k = 0; k < RMAP_EXT; ++k) if (d->shadow_ptes[k]) @@ -1530,18 +1548,18 @@ static void audit_rmap(struct kvm_vcpu *vcpu) static void audit_write_protection(struct kvm_vcpu *vcpu) { struct kvm_mmu_page *page; + struct kvm_memory_slot *slot; + unsigned long *rmapp; + gfn_t gfn; list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) { - hfn_t hfn; - struct page *pg; - if (page->role.metaphysical) continue; - hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT) - >> PAGE_SHIFT; - pg = pfn_to_page(hfn); - if (pg->private) + slot = gfn_to_memslot(vcpu->kvm, page->gfn); + gfn = unalias_gfn(vcpu->kvm, page->gfn); + rmapp = &slot->rmap[gfn - slot->base_gfn]; + if (*rmapp) printk(KERN_ERR "%s: (%s) shadow page has writable" " mappings: gfn %lx role %x\n", __FUNCTION__, audit_msg, page->gfn, diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index be0f85231da9..fbe595f880af 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -295,7 +295,8 @@ unshadowed: set_shadow_pte(shadow_pte, spte); page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); if (!was_rmapped) - rmap_add(vcpu, shadow_pte); + rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } -- cgit v1.2.3 From 195aefde9cc2cee38dd54ef92a866721fba4413e Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Mon, 1 Oct 2007 22:14:18 +0200 Subject: KVM: Add general accessors to read and write guest memory Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 9 +++ drivers/kvm/kvm_main.c | 160 +++++++++++++++++++++++++++++++++++++++---------- drivers/kvm/vmx.c | 43 ++++++------- 3 files changed, 158 insertions(+), 54 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 80cfb99fffe0..1965438f18cc 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -559,6 +559,15 @@ extern hpa_t bad_page_address; gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); +int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, + int len); +int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); +int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, + int offset, int len); +int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, + unsigned long len); +int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len); +int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); void mark_page_dirty(struct kvm *kvm, gfn_t gfn); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 9510e2276ca3..cac328b8421c 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -400,22 +400,16 @@ static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; int i; - u64 *pdpt; int ret; - struct page *page; u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; mutex_lock(&vcpu->kvm->lock); - page = gfn_to_page(vcpu->kvm, pdpt_gfn); - if (!page) { + ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, + offset * sizeof(u64), sizeof(pdpte)); + if (ret < 0) { ret = 0; goto out; } - - pdpt = kmap_atomic(page, KM_USER0); - memcpy(pdpte, pdpt+offset, sizeof(pdpte)); - kunmap_atomic(pdpt, KM_USER0); - for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) { ret = 0; @@ -962,6 +956,127 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) } EXPORT_SYMBOL_GPL(gfn_to_page); +static int next_segment(unsigned long len, int offset) +{ + if (len > PAGE_SIZE - offset) + return PAGE_SIZE - offset; + else + return len; +} + +int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, + int len) +{ + void *page_virt; + struct page *page; + + page = gfn_to_page(kvm, gfn); + if (!page) + return -EFAULT; + page_virt = kmap_atomic(page, KM_USER0); + + memcpy(data, page_virt + offset, len); + + kunmap_atomic(page_virt, KM_USER0); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_read_guest_page); + +int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + data += seg; + ++gfn; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_read_guest); + +int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, + int offset, int len) +{ + void *page_virt; + struct page *page; + + page = gfn_to_page(kvm, gfn); + if (!page) + return -EFAULT; + page_virt = kmap_atomic(page, KM_USER0); + + memcpy(page_virt + offset, data, len); + + kunmap_atomic(page_virt, KM_USER0); + mark_page_dirty(kvm, gfn); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_write_guest_page); + +int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, + unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + data += seg; + ++gfn; + } + return 0; +} + +int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) +{ + void *page_virt; + struct page *page; + + page = gfn_to_page(kvm, gfn); + if (!page) + return -EFAULT; + page_virt = kmap_atomic(page, KM_USER0); + + memset(page_virt + offset, 0, len); + + kunmap_atomic(page_virt, KM_USER0); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_clear_guest_page); + +int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_clear_guest_page(kvm, gfn, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + ++gfn; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_clear_guest); + /* WARNING: Does not work on aliased pages. */ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) { @@ -988,21 +1103,13 @@ int emulator_read_std(unsigned long addr, gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); unsigned offset = addr & (PAGE_SIZE-1); unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); - unsigned long pfn; - struct page *page; - void *page_virt; + int ret; if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; - pfn = gpa >> PAGE_SHIFT; - page = gfn_to_page(vcpu->kvm, pfn); - if (!page) + ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); + if (ret < 0) return X86EMUL_UNHANDLEABLE; - page_virt = kmap_atomic(page, KM_USER0); - - memcpy(data, page_virt + offset, tocopy); - - kunmap_atomic(page_virt, KM_USER0); bytes -= tocopy; data += tocopy; @@ -1095,19 +1202,12 @@ static int emulator_read_emulated(unsigned long addr, static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, const void *val, int bytes) { - struct page *page; - void *virt; + int ret; - if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT)) - return 0; - page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); - if (!page) + ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); + if (ret < 0) return 0; - mark_page_dirty(vcpu->kvm, gpa >> PAGE_SHIFT); - virt = kmap_atomic(page, KM_USER0); kvm_mmu_pte_write(vcpu, gpa, val, bytes); - memcpy(virt + offset_in_page(gpa), val, bytes); - kunmap_atomic(virt, KM_USER0); return 1; } diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 439873a25cca..894fd45ecc98 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1387,33 +1387,28 @@ static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) static int init_rmode_tss(struct kvm* kvm) { - struct page *p1, *p2, *p3; gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT; - char *page; - - p1 = gfn_to_page(kvm, fn++); - p2 = gfn_to_page(kvm, fn++); - p3 = gfn_to_page(kvm, fn); + u16 data = 0; + int r; - if (!p1 || !p2 || !p3) { - kvm_printf(kvm,"%s: gfn_to_page failed\n", __FUNCTION__); + r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); + if (r < 0) + return 0; + data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; + r = kvm_write_guest_page(kvm, fn++, &data, 0x66, sizeof(u16)); + if (r < 0) + return 0; + r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE); + if (r < 0) + return 0; + r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); + if (r < 0) + return 0; + data = ~0; + r = kvm_write_guest_page(kvm, fn, &data, RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1, + sizeof(u8)); + if (r < 0) return 0; - } - - page = kmap_atomic(p1, KM_USER0); - clear_page(page); - *(u16*)(page + 0x66) = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; - kunmap_atomic(page, KM_USER0); - - page = kmap_atomic(p2, KM_USER0); - clear_page(page); - kunmap_atomic(page, KM_USER0); - - page = kmap_atomic(p3, KM_USER0); - clear_page(page); - *(page + RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1) = ~0; - kunmap_atomic(page, KM_USER0); - return 1; } -- cgit v1.2.3 From 82ce2c96831f049a37118733ced5c8f7c8848102 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 2 Oct 2007 18:52:55 +0200 Subject: KVM: Allow dynamic allocation of the mmu shadow cache size The user is now able to set how many mmu pages will be allocated to the guest. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 7 ++++++- drivers/kvm/kvm_main.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/mmu.c | 40 ++++++++++++++++++++++++++++++++++++++-- include/linux/kvm.h | 3 +++ 4 files changed, 94 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 1965438f18cc..9f10c373b74c 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -40,6 +40,8 @@ #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 +#define KVM_PERMILLE_MMU_PAGES 20 +#define KVM_MIN_ALLOC_MMU_PAGES 64 #define KVM_NUM_MMU_PAGES 1024 #define KVM_MIN_FREE_MMU_PAGES 5 #define KVM_REFILL_PAGES 25 @@ -418,7 +420,9 @@ struct kvm { * Hash table of struct kvm_mmu_page. */ struct list_head active_mmu_pages; - int n_free_mmu_pages; + unsigned int n_free_mmu_pages; + unsigned int n_requested_mmu_pages; + unsigned int n_alloc_mmu_pages; struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; unsigned long rmap_overflow; @@ -547,6 +551,7 @@ void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); void kvm_mmu_zap_all(struct kvm *kvm); +void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index cac328b8421c..2bb1f2f66efa 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -743,6 +743,24 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, if (mem->slot >= kvm->nmemslots) kvm->nmemslots = mem->slot + 1; + if (!kvm->n_requested_mmu_pages) { + unsigned int n_pages; + + if (npages) { + n_pages = npages * KVM_PERMILLE_MMU_PAGES / 1000; + kvm_mmu_change_mmu_pages(kvm, kvm->n_alloc_mmu_pages + + n_pages); + } else { + unsigned int nr_mmu_pages; + + n_pages = old.npages * KVM_PERMILLE_MMU_PAGES / 1000; + nr_mmu_pages = kvm->n_alloc_mmu_pages - n_pages; + nr_mmu_pages = max(nr_mmu_pages, + (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); + kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + } + } + *memslot = new; kvm_mmu_slot_remove_write_access(kvm, mem->slot); @@ -760,6 +778,26 @@ out: return r; } +static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, + u32 kvm_nr_mmu_pages) +{ + if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) + return -EINVAL; + + mutex_lock(&kvm->lock); + + kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); + kvm->n_requested_mmu_pages = kvm_nr_mmu_pages; + + mutex_unlock(&kvm->lock); + return 0; +} + +static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) +{ + return kvm->n_alloc_mmu_pages; +} + /* * Get (and clear) the dirty memory log for a memory slot. */ @@ -3071,6 +3109,14 @@ static long kvm_vm_ioctl(struct file *filp, goto out; break; } + case KVM_SET_NR_MMU_PAGES: + r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); + if (r) + goto out; + break; + case KVM_GET_NR_MMU_PAGES: + r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); + break; case KVM_GET_DIRTY_LOG: { struct kvm_dirty_log log; @@ -3278,6 +3324,7 @@ static long kvm_dev_ioctl(struct file *filp, switch (ext) { case KVM_CAP_IRQCHIP: case KVM_CAP_HLT: + case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: r = 1; break; default: diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 72757db15065..6cda1feb9a95 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -747,6 +747,40 @@ static void kvm_mmu_zap_page(struct kvm *kvm, kvm_mmu_reset_last_pte_updated(kvm); } +/* + * Changing the number of mmu pages allocated to the vm + * Note: if kvm_nr_mmu_pages is too small, you will get dead lock + */ +void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages) +{ + /* + * If we set the number of mmu pages to be smaller be than the + * number of actived pages , we must to free some mmu pages before we + * change the value + */ + + if ((kvm->n_alloc_mmu_pages - kvm->n_free_mmu_pages) > + kvm_nr_mmu_pages) { + int n_used_mmu_pages = kvm->n_alloc_mmu_pages + - kvm->n_free_mmu_pages; + + while (n_used_mmu_pages > kvm_nr_mmu_pages) { + struct kvm_mmu_page *page; + + page = container_of(kvm->active_mmu_pages.prev, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(kvm, page); + n_used_mmu_pages--; + } + kvm->n_free_mmu_pages = 0; + } + else + kvm->n_free_mmu_pages += kvm_nr_mmu_pages + - kvm->n_alloc_mmu_pages; + + kvm->n_alloc_mmu_pages = kvm_nr_mmu_pages; +} + static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) { unsigned index; @@ -1297,8 +1331,10 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) ASSERT(vcpu); - vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES; - + if (vcpu->kvm->n_requested_mmu_pages) + vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_requested_mmu_pages; + else + vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_alloc_mmu_pages; /* * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64. * Therefore we need to allocate shadow page tables in the first diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 057a7f34ee36..d2fd973d81fe 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -347,11 +347,14 @@ struct kvm_signal_mask { */ #define KVM_CAP_IRQCHIP 0 #define KVM_CAP_HLT 1 +#define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2 /* * ioctls for VM fds */ #define KVM_SET_MEMORY_REGION _IOW(KVMIO, 0x40, struct kvm_memory_region) +#define KVM_SET_NR_MMU_PAGES _IO(KVMIO, 0x44) +#define KVM_GET_NR_MMU_PAGES _IO(KVMIO, 0x45) /* * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns * a vcpu fd. -- cgit v1.2.3 From d589444e924bc72e42fa94853f9096589d69374d Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Oct 2007 10:48:30 +1000 Subject: KVM: Add kvm_free_lapic() to pair with kvm_create_lapic() Instead of the asymetry of kvm_free_apic, implement kvm_free_lapic(). And guess what? I found a minor bug: we don't need to hrtimer_cancel() from kvm_main.c, because we do that in kvm_free_apic(). Also: 1) kvm_vcpu_uninit should be the reverse order from kvm_vcpu_init. 2) Don't set apic->regs_page to zero before freeing apic. Signed-off-by: Rusty Russell Signed-off-by: Avi Kivity --- drivers/kvm/irq.h | 2 +- drivers/kvm/kvm_main.c | 4 +--- drivers/kvm/lapic.c | 19 +++++++++---------- 3 files changed, 11 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 11fc014e2b30..508280ea6fcf 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -139,7 +139,7 @@ int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); int kvm_create_lapic(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu); -void kvm_free_apic(struct kvm_lapic *apic); +void kvm_free_lapic(struct kvm_vcpu *vcpu); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 2bb1f2f66efa..b19734606cd9 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -268,10 +268,8 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_init); void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) { + kvm_free_lapic(vcpu); kvm_mmu_destroy(vcpu); - if (vcpu->apic) - hrtimer_cancel(&vcpu->apic->timer.dev); - kvm_free_apic(vcpu->apic); free_page((unsigned long)vcpu->pio_data); free_page((unsigned long)vcpu->run); } diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 238fcad3cece..8e8dab08be58 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -762,19 +762,17 @@ static int apic_mmio_range(struct kvm_io_device *this, gpa_t addr) return ret; } -void kvm_free_apic(struct kvm_lapic *apic) +void kvm_free_lapic(struct kvm_vcpu *vcpu) { - if (!apic) + if (!vcpu->apic) return; - hrtimer_cancel(&apic->timer.dev); + hrtimer_cancel(&vcpu->apic->timer.dev); - if (apic->regs_page) { - __free_page(apic->regs_page); - apic->regs_page = 0; - } + if (vcpu->apic->regs_page) + __free_page(vcpu->apic->regs_page); - kfree(apic); + kfree(vcpu->apic); } /* @@ -962,7 +960,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) if (apic->regs_page == NULL) { printk(KERN_ERR "malloc apic regs error for vcpu %x\n", vcpu->vcpu_id); - goto nomem; + goto nomem_free_apic; } apic->regs = page_address(apic->regs_page); memset(apic->regs, 0, PAGE_SIZE); @@ -980,8 +978,9 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) apic->dev.private = apic; return 0; +nomem_free_apic: + kfree(apic); nomem: - kvm_free_apic(apic); return -ENOMEM; } EXPORT_SYMBOL_GPL(kvm_create_lapic); -- cgit v1.2.3 From 76fafa5e22bd82e92d2734852ba23f17322d675a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Oct 2007 10:50:48 +1000 Subject: KVM: Hoist kvm_create_lapic() into kvm_vcpu_init() Move kvm_create_lapic() into kvm_vcpu_init(), rather than having svm and vmx do it. And make it return the error rather than a fairly random -ENOMEM. This also solves the problem that neither svm.c nor vmx.c actually handles the error path properly. Signed-off-by: Rusty Russell Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 10 +++++++++- drivers/kvm/svm.c | 6 ------ drivers/kvm/vmx.c | 6 ------ 3 files changed, 9 insertions(+), 13 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index b19734606cd9..9ea9277014aa 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -255,14 +255,22 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) if (r < 0) goto fail_free_pio_data; + if (irqchip_in_kernel(kvm)) { + r = kvm_create_lapic(vcpu); + if (r < 0) + goto fail_mmu_destroy; + } + return 0; +fail_mmu_destroy: + kvm_mmu_destroy(vcpu); fail_free_pio_data: free_page((unsigned long)vcpu->pio_data); fail_free_run: free_page((unsigned long)vcpu->run); fail: - return -ENOMEM; + return r; } EXPORT_SYMBOL_GPL(kvm_vcpu_init); diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index f268bd51f337..fb2e591d5397 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -588,12 +588,6 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) if (err) goto free_svm; - if (irqchip_in_kernel(kvm)) { - err = kvm_create_lapic(&svm->vcpu); - if (err < 0) - goto free_svm; - } - page = alloc_page(GFP_KERNEL); if (!page) { err = -ENOMEM; diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 894fd45ecc98..7b742901e783 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2431,12 +2431,6 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) if (err) goto free_vcpu; - if (irqchip_in_kernel(kvm)) { - err = kvm_create_lapic(&vmx->vcpu); - if (err < 0) - goto free_vcpu; - } - vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!vmx->guest_msrs) { err = -ENOMEM; -- cgit v1.2.3 From 7e620d16b8838bc0ad5b27d2dd55796270cd588c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 8 Oct 2007 10:55:29 +1000 Subject: KVM: Remove gratuitous casts from lapic.c Since vcpu->apic is of the correct type, there's not need to cast. Signed-off-by: Rusty Russell Signed-off-by: Avi Kivity --- drivers/kvm/lapic.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 8e8dab08be58..554e73ad33f0 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -172,7 +172,7 @@ static inline int apic_find_highest_irr(struct kvm_lapic *apic) int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; int highest_irr; if (!apic) @@ -783,7 +783,7 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu) void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; if (!apic) return; @@ -792,7 +792,7 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; u64 tpr; if (!apic) @@ -805,7 +805,7 @@ EXPORT_SYMBOL_GPL(kvm_lapic_get_cr8); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; if (!apic) { value |= MSR_IA32_APICBASE_BSP; @@ -882,7 +882,7 @@ EXPORT_SYMBOL_GPL(kvm_lapic_reset); int kvm_lapic_enabled(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = (struct kvm_lapic *)vcpu->apic; + struct kvm_lapic *apic = vcpu->apic; int ret = 0; if (!apic) -- cgit v1.2.3 From d77c26fce93d07802db97498959587eb9347b31d Mon Sep 17 00:00:00 2001 From: Mike Day Date: Mon, 8 Oct 2007 09:02:08 -0400 Subject: KVM: CodingStyle cleanup Signed-off-by: Mike D. Day Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 32 ++++++++++---------- drivers/kvm/kvm_main.c | 58 +++++++++++++++++++----------------- drivers/kvm/lapic.c | 3 +- drivers/kvm/mmu.c | 10 ++++--- drivers/kvm/paging_tmpl.h | 2 +- drivers/kvm/svm.c | 48 ++++++++++++++---------------- drivers/kvm/svm.h | 2 +- drivers/kvm/vmx.c | 60 ++++++++++++++++++++----------------- drivers/kvm/vmx.h | 8 ++--- drivers/kvm/x86_emulate.c | 76 +++++++++++++++++++++++------------------------ 10 files changed, 151 insertions(+), 148 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 9f10c373b74c..ec5b498945ae 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -528,7 +528,7 @@ extern struct kvm_x86_ops *kvm_x86_ops; if (printk_ratelimit()) \ printk(KERN_ERR "kvm: %i: cpu%i " fmt, \ current->tgid, (vcpu)->vcpu_id , ## __VA_ARGS__); \ - } while(0) + } while (0) #define kvm_printf(kvm, fmt ...) printk(KERN_DEBUG fmt) #define vcpu_printf(vcpu, fmt...) kvm_printf(vcpu->kvm, fmt) @@ -598,7 +598,7 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); struct x86_emulate_ctxt; -int kvm_emulate_pio (struct kvm_vcpu *vcpu, struct kvm_run *run, int in, +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, int size, unsigned port); int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, int size, unsigned long count, int down, @@ -607,7 +607,7 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); int kvm_emulate_halt(struct kvm_vcpu *vcpu); int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); int emulate_clts(struct kvm_vcpu *vcpu); -int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest); int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value); @@ -631,7 +631,7 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); void kvm_flush_remote_tlbs(struct kvm *kvm); int emulator_read_std(unsigned long addr, - void *val, + void *val, unsigned int bytes, struct kvm_vcpu *vcpu); int emulator_write_emulated(unsigned long addr, @@ -721,55 +721,55 @@ static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) static inline u16 read_fs(void) { u16 seg; - asm ("mov %%fs, %0" : "=g"(seg)); + asm("mov %%fs, %0" : "=g"(seg)); return seg; } static inline u16 read_gs(void) { u16 seg; - asm ("mov %%gs, %0" : "=g"(seg)); + asm("mov %%gs, %0" : "=g"(seg)); return seg; } static inline u16 read_ldt(void) { u16 ldt; - asm ("sldt %0" : "=g"(ldt)); + asm("sldt %0" : "=g"(ldt)); return ldt; } static inline void load_fs(u16 sel) { - asm ("mov %0, %%fs" : : "rm"(sel)); + asm("mov %0, %%fs" : : "rm"(sel)); } static inline void load_gs(u16 sel) { - asm ("mov %0, %%gs" : : "rm"(sel)); + asm("mov %0, %%gs" : : "rm"(sel)); } #ifndef load_ldt static inline void load_ldt(u16 sel) { - asm ("lldt %0" : : "rm"(sel)); + asm("lldt %0" : : "rm"(sel)); } #endif static inline void get_idt(struct descriptor_table *table) { - asm ("sidt %0" : "=m"(*table)); + asm("sidt %0" : "=m"(*table)); } static inline void get_gdt(struct descriptor_table *table) { - asm ("sgdt %0" : "=m"(*table)); + asm("sgdt %0" : "=m"(*table)); } static inline unsigned long read_tr_base(void) { u16 tr; - asm ("str %0" : "=g"(tr)); + asm("str %0" : "=g"(tr)); return segment_base(tr); } @@ -785,17 +785,17 @@ static inline unsigned long read_msr(unsigned long msr) static inline void fx_save(struct i387_fxsave_struct *image) { - asm ("fxsave (%0)":: "r" (image)); + asm("fxsave (%0)":: "r" (image)); } static inline void fx_restore(struct i387_fxsave_struct *image) { - asm ("fxrstor (%0)":: "r" (image)); + asm("fxrstor (%0)":: "r" (image)); } static inline void fpu_init(void) { - asm ("finit"); + asm("finit"); } static inline u32 get_rdx_init_val(void) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 9ea9277014aa..a1983d2d5b8f 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -104,7 +104,7 @@ static struct dentry *debugfs_dir; #define EFER_RESERVED_BITS 0xfffffffffffff2fe #ifdef CONFIG_X86_64 -// LDT or TSS descriptor in the GDT. 16 bytes. +/* LDT or TSS descriptor in the GDT. 16 bytes. */ struct segment_descriptor_64 { struct segment_descriptor s; u32 base_higher; @@ -121,27 +121,27 @@ unsigned long segment_base(u16 selector) struct descriptor_table gdt; struct segment_descriptor *d; unsigned long table_base; - typedef unsigned long ul; unsigned long v; if (selector == 0) return 0; - asm ("sgdt %0" : "=m"(gdt)); + asm("sgdt %0" : "=m"(gdt)); table_base = gdt.base; if (selector & 4) { /* from ldt */ u16 ldt_selector; - asm ("sldt %0" : "=g"(ldt_selector)); + asm("sldt %0" : "=g"(ldt_selector)); table_base = segment_base(ldt_selector); } d = (struct segment_descriptor *)(table_base + (selector & ~7)); - v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24); + v = d->base_low | ((unsigned long)d->base_mid << 16) | + ((unsigned long)d->base_high << 24); #ifdef CONFIG_X86_64 - if (d->system == 0 - && (d->type == 2 || d->type == 9 || d->type == 11)) - v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32; + if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11)) + v |= ((unsigned long) \ + ((struct segment_descriptor_64 *)d)->base_higher) << 32; #endif return v; } @@ -721,7 +721,7 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, if (!new.phys_mem) goto out_unlock; - new.rmap = vmalloc(npages * sizeof(struct page*)); + new.rmap = vmalloc(npages * sizeof(struct page *)); if (!new.rmap) goto out_unlock; @@ -904,17 +904,17 @@ static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) r = 0; switch (chip->chip_id) { case KVM_IRQCHIP_PIC_MASTER: - memcpy (&chip->chip.pic, + memcpy(&chip->chip.pic, &pic_irqchip(kvm)->pics[0], sizeof(struct kvm_pic_state)); break; case KVM_IRQCHIP_PIC_SLAVE: - memcpy (&chip->chip.pic, + memcpy(&chip->chip.pic, &pic_irqchip(kvm)->pics[1], sizeof(struct kvm_pic_state)); break; case KVM_IRQCHIP_IOAPIC: - memcpy (&chip->chip.ioapic, + memcpy(&chip->chip.ioapic, ioapic_irqchip(kvm), sizeof(struct kvm_ioapic_state)); break; @@ -932,17 +932,17 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) r = 0; switch (chip->chip_id) { case KVM_IRQCHIP_PIC_MASTER: - memcpy (&pic_irqchip(kvm)->pics[0], + memcpy(&pic_irqchip(kvm)->pics[0], &chip->chip.pic, sizeof(struct kvm_pic_state)); break; case KVM_IRQCHIP_PIC_SLAVE: - memcpy (&pic_irqchip(kvm)->pics[1], + memcpy(&pic_irqchip(kvm)->pics[1], &chip->chip.pic, sizeof(struct kvm_pic_state)); break; case KVM_IRQCHIP_IOAPIC: - memcpy (ioapic_irqchip(kvm), + memcpy(ioapic_irqchip(kvm), &chip->chip.ioapic, sizeof(struct kvm_ioapic_state)); break; @@ -1341,7 +1341,7 @@ int emulate_clts(struct kvm_vcpu *vcpu) return X86EMUL_CONTINUE; } -int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest) +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) { struct kvm_vcpu *vcpu = ctxt->vcpu; @@ -1934,7 +1934,7 @@ static void pio_string_write(struct kvm_io_device *pio_dev, mutex_unlock(&vcpu->kvm->lock); } -int kvm_emulate_pio (struct kvm_vcpu *vcpu, struct kvm_run *run, int in, +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, int size, unsigned port) { struct kvm_io_device *pio_dev; @@ -2089,7 +2089,7 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) int r; if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { - printk("vcpu %d received sipi with vector # %x\n", + pr_debug("vcpu %d received sipi with vector # %x\n", vcpu->vcpu_id, vcpu->sipi_vector); kvm_lapic_reset(vcpu); kvm_x86_ops->vcpu_reset(vcpu); @@ -2363,7 +2363,8 @@ static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, sizeof sregs->interrupt_bitmap); pending_vec = kvm_x86_ops->get_irq(vcpu); if (pending_vec >= 0) - set_bit(pending_vec, (unsigned long *)sregs->interrupt_bitmap); + set_bit(pending_vec, + (unsigned long *)sregs->interrupt_bitmap); } else memcpy(sregs->interrupt_bitmap, vcpu->irq_pending, sizeof sregs->interrupt_bitmap); @@ -2436,7 +2437,8 @@ static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, /* Only pending external irq is handled here */ if (pending_vec < max_bits) { kvm_x86_ops->set_irq(vcpu, pending_vec); - printk("Set back pending irq %d\n", pending_vec); + pr_debug("Set back pending irq %d\n", + pending_vec); } } @@ -3155,8 +3157,7 @@ static long kvm_vm_ioctl(struct file *filp, kvm->vpic = NULL; goto out; } - } - else + } else goto out; break; case KVM_IRQ_LINE: { @@ -3448,7 +3449,7 @@ static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, } static int kvm_reboot(struct notifier_block *notifier, unsigned long val, - void *v) + void *v) { if (val == SYS_RESTART) { /* @@ -3655,7 +3656,7 @@ int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, r = misc_register(&kvm_dev); if (r) { - printk (KERN_ERR "kvm: misc device register failed\n"); + printk(KERN_ERR "kvm: misc device register failed\n"); goto out_free; } @@ -3683,6 +3684,7 @@ out: kvm_x86_ops = NULL; return r; } +EXPORT_SYMBOL_GPL(kvm_init_x86); void kvm_exit_x86(void) { @@ -3696,6 +3698,7 @@ void kvm_exit_x86(void) kvm_x86_ops->hardware_unsetup(); kvm_x86_ops = NULL; } +EXPORT_SYMBOL_GPL(kvm_exit_x86); static __init int kvm_init(void) { @@ -3710,7 +3713,9 @@ static __init int kvm_init(void) kvm_init_msr_list(); - if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) { + bad_page = alloc_page(GFP_KERNEL); + + if (bad_page == NULL) { r = -ENOMEM; goto out; } @@ -3736,6 +3741,3 @@ static __exit void kvm_exit(void) module_init(kvm_init) module_exit(kvm_exit) - -EXPORT_SYMBOL_GPL(kvm_init_x86); -EXPORT_SYMBOL_GPL(kvm_exit_x86); diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 554e73ad33f0..e15b42e48862 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -906,8 +906,7 @@ static int __apic_timer_fn(struct kvm_lapic *apic) wait_queue_head_t *q = &apic->vcpu->wq; atomic_inc(&apic->timer.pending); - if (waitqueue_active(q)) - { + if (waitqueue_active(q)) { apic->vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; wake_up_interruptible(q); } diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 6cda1feb9a95..ece0aa4e4c9f 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -90,7 +90,8 @@ static int dbg = 1; #define PT32_DIR_PSE36_SIZE 4 #define PT32_DIR_PSE36_SHIFT 13 -#define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT) +#define PT32_DIR_PSE36_MASK \ + (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT) #define PT_FIRST_AVAIL_BITS_SHIFT 9 @@ -103,7 +104,7 @@ static int dbg = 1; #define PT64_LEVEL_BITS 9 #define PT64_LEVEL_SHIFT(level) \ - ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS ) + (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS) #define PT64_LEVEL_MASK(level) \ (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level)) @@ -115,7 +116,7 @@ static int dbg = 1; #define PT32_LEVEL_BITS 10 #define PT32_LEVEL_SHIFT(level) \ - ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS ) + (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS) #define PT32_LEVEL_MASK(level) \ (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level)) @@ -1489,7 +1490,8 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, printk(KERN_ERR "xx audit error: (%s) levels %d" " gva %lx gpa %llx hpa %llx ent %llx %d\n", audit_msg, vcpu->mmu.root_level, - va, gpa, hpa, ent, is_shadow_present_pte(ent)); + va, gpa, hpa, ent, + is_shadow_present_pte(ent)); else if (ent == shadow_notrap_nonpresent_pte && !is_error_hpa(hpa)) printk(KERN_ERR "audit: (%s) notrap shadow," diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index fbe595f880af..447d2c31f0cb 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -163,7 +163,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->page = pfn_to_page(paddr >> PAGE_SHIFT); walker->table = kmap_atomic(walker->page, KM_USER0); --walker->level; - walker->table_gfn[walker->level - 1 ] = table_gfn; + walker->table_gfn[walker->level - 1] = table_gfn; pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); } diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index fb2e591d5397..7b21576b62bc 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -229,12 +229,11 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) printk(KERN_DEBUG "%s: NOP\n", __FUNCTION__); return; } - if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE) { + if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE) printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n", __FUNCTION__, svm->vmcb->save.rip, svm->next_rip); - } vcpu->rip = svm->vmcb->save.rip = svm->next_rip; svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; @@ -312,7 +311,7 @@ static void svm_hardware_enable(void *garbage) svm_data->next_asid = svm_data->max_asid + 1; svm_features = cpuid_edx(SVM_CPUID_FUNC); - asm volatile ( "sgdt %0" : "=m"(gdt_descr) ); + asm volatile ("sgdt %0" : "=m"(gdt_descr)); gdt = (struct desc_struct *)gdt_descr.address; svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); @@ -544,8 +543,7 @@ static void init_vmcb(struct vmcb *vmcb) init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); save->efer = MSR_EFER_SVME_MASK; - - save->dr6 = 0xffff0ff0; + save->dr6 = 0xffff0ff0; save->dr7 = 0x400; save->rflags = 2; save->rip = 0x0000fff0; @@ -783,7 +781,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) svm->vmcb->save.efer |= KVM_EFER_LMA | KVM_EFER_LME; } - if (is_paging(vcpu) && !(cr0 & X86_CR0_PG) ) { + if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { vcpu->shadow_efer &= ~KVM_EFER_LMA; svm->vmcb->save.efer &= ~(KVM_EFER_LMA | KVM_EFER_LME); } @@ -1010,7 +1008,7 @@ static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - u32 io_info = svm->vmcb->control.exit_info_1; //address size bug? + u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ int size, down, in, string, rep; unsigned port; @@ -1316,7 +1314,7 @@ static void reload_tss(struct kvm_vcpu *vcpu) int cpu = raw_smp_processor_id(); struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu); - svm_data->tss_desc->type = 9; //available 32/64-bit TSS + svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */ load_TR_desc(); } @@ -1434,9 +1432,9 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, * Interrupts blocked. Wait for unblock. */ if (!svm->vcpu.interrupt_window_open && - (svm->vcpu.irq_summary || kvm_run->request_interrupt_window)) { + (svm->vcpu.irq_summary || kvm_run->request_interrupt_window)) control->intercept |= 1ULL << INTERCEPT_VINTR; - } else + else control->intercept &= ~(1ULL << INTERCEPT_VINTR); } @@ -1581,23 +1579,23 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) : : [svm]"a"(svm), [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), - [rbx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_RBP])) + [rbx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RBP])) #ifdef CONFIG_X86_64 - ,[r8 ]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R8])), - [r9 ]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R9 ])), - [r10]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_svm,vcpu.regs[VCPU_REGS_R15])) + , [r8]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R15])) #endif - : "cc", "memory" ); + : "cc", "memory"); if ((svm->vmcb->save.dr7 & 0xff)) load_db_regs(svm->host_db_regs); diff --git a/drivers/kvm/svm.h b/drivers/kvm/svm.h index 3b1b0f35b6cb..5fa277c0187c 100644 --- a/drivers/kvm/svm.h +++ b/drivers/kvm/svm.h @@ -311,7 +311,7 @@ struct __attribute__ ((__packed__)) vmcb { #define SVM_EXIT_ERR -1 -#define SVM_CR0_SELECTIVE_MASK (1 << 3 | 1) // TS and MP +#define SVM_CR0_SELECTIVE_MASK (1 << 3 | 1) /* TS and MP */ #define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda" #define SVM_VMRUN ".byte 0x0f, 0x01, 0xd8" diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 7b742901e783..6955580bb69e 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -62,7 +62,7 @@ struct vcpu_vmx { int gs_ldt_reload_needed; int fs_reload_needed; int guest_efer_loaded; - }host_state; + } host_state; }; @@ -271,7 +271,7 @@ static void vmcs_writel(unsigned long field, unsigned long value) u8 error; asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0" - : "=q"(error) : "a"(value), "d"(field) : "cc" ); + : "=q"(error) : "a"(value), "d"(field) : "cc"); if (unlikely(error)) vmwrite_error(field, value); } @@ -415,10 +415,10 @@ static void vmx_save_host_state(struct kvm_vcpu *vcpu) #endif #ifdef CONFIG_X86_64 - if (is_long_mode(&vmx->vcpu)) { + if (is_long_mode(&vmx->vcpu)) save_msrs(vmx->host_msrs + vmx->msr_offset_kernel_gs_base, 1); - } + #endif load_msrs(vmx->guest_msrs, vmx->save_nmsrs); load_transition_efer(vmx); @@ -845,7 +845,7 @@ static int vmx_get_irq(struct kvm_vcpu *vcpu) if (is_external_interrupt(idtv_info_field)) return idtv_info_field & VECTORING_INFO_VECTOR_MASK; else - printk("pending exception: not handled yet\n"); + printk(KERN_DEBUG "pending exception: not handled yet\n"); } return -1; } @@ -893,7 +893,7 @@ static void hardware_disable(void *garbage) } static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, - u32 msr, u32* result) + u32 msr, u32 *result) { u32 vmx_msr_low, vmx_msr_high; u32 ctl = ctl_min | ctl_opt; @@ -1102,7 +1102,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu) vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); } -static gva_t rmode_tss_base(struct kvm* kvm) +static gva_t rmode_tss_base(struct kvm *kvm) { gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3; return base_gfn << PAGE_SHIFT; @@ -1385,7 +1385,7 @@ static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) vmcs_writel(GUEST_GDTR_BASE, dt->base); } -static int init_rmode_tss(struct kvm* kvm) +static int init_rmode_tss(struct kvm *kvm) { gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT; u16 data = 0; @@ -1494,7 +1494,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_writel(GUEST_RIP, 0); vmcs_writel(GUEST_RSP, 0); - //todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 + /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ vmcs_writel(GUEST_DR7, 0x400); vmcs_writel(GUEST_GDTR_BASE, 0); @@ -1561,7 +1561,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) get_idt(&dt); vmcs_writel(HOST_IDTR_BASE, dt.base); /* 22.2.4 */ - asm ("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return)); + asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return)); vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */ vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); @@ -1613,7 +1613,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); vmx->vcpu.cr0 = 0x60000010; - vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); // enter rmode + vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); /* enter rmode */ vmx_set_cr4(&vmx->vcpu, 0); #ifdef CONFIG_X86_64 vmx_set_efer(&vmx->vcpu, 0); @@ -1644,7 +1644,7 @@ static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq) u16 sp = vmcs_readl(GUEST_RSP); u32 ss_limit = vmcs_read32(GUEST_SS_LIMIT); - if (sp > ss_limit || sp < 6 ) { + if (sp > ss_limit || sp < 6) { vcpu_printf(vcpu, "%s: #SS, rsp 0x%lx ss 0x%lx limit 0x%x\n", __FUNCTION__, vmcs_readl(GUEST_RSP), @@ -1664,15 +1664,18 @@ static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq) ip = vmcs_readl(GUEST_RIP); - if (emulator_write_emulated(ss_base + sp - 2, &flags, 2, vcpu) != X86EMUL_CONTINUE || - emulator_write_emulated(ss_base + sp - 4, &cs, 2, vcpu) != X86EMUL_CONTINUE || - emulator_write_emulated(ss_base + sp - 6, &ip, 2, vcpu) != X86EMUL_CONTINUE) { + if (emulator_write_emulated( + ss_base + sp - 2, &flags, 2, vcpu) != X86EMUL_CONTINUE || + emulator_write_emulated( + ss_base + sp - 4, &cs, 2, vcpu) != X86EMUL_CONTINUE || + emulator_write_emulated( + ss_base + sp - 6, &ip, 2, vcpu) != X86EMUL_CONTINUE) { vcpu_printf(vcpu, "%s: write guest err\n", __FUNCTION__); return; } vmcs_writel(GUEST_RFLAGS, flags & - ~( X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF)); + ~(X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF)); vmcs_write16(GUEST_CS_SELECTOR, ent[1]) ; vmcs_writel(GUEST_CS_BASE, ent[1] << 4); vmcs_writel(GUEST_RIP, ent[0]); @@ -1777,10 +1780,9 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if ((vect_info & VECTORING_INFO_VALID_MASK) && - !is_page_fault(intr_info)) { + !is_page_fault(intr_info)) printk(KERN_ERR "%s: unexpected, vectoring info 0x%x " "intr info 0x%x\n", __FUNCTION__, vect_info, intr_info); - } if (!irqchip_in_kernel(vcpu->kvm) && is_external_interrupt(vect_info)) { int irq = vect_info & VECTORING_INFO_VECTOR_MASK; @@ -1831,7 +1833,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) case EMULATE_DO_MMIO: ++vcpu->stat.mmio_exits; return 0; - case EMULATE_FAIL: + case EMULATE_FAIL: kvm_report_emulation_failure(vcpu, "pagetable"); break; default: @@ -1849,7 +1851,8 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } - if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == (INTR_TYPE_EXCEPTION | 1)) { + if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == + (INTR_TYPE_EXCEPTION | 1)) { kvm_run->exit_reason = KVM_EXIT_DEBUG; return 0; } @@ -2138,8 +2141,8 @@ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) return 0; } - if ( (vectoring_info & VECTORING_INFO_VALID_MASK) && - exit_reason != EXIT_REASON_EXCEPTION_NMI ) + if ((vectoring_info & VECTORING_INFO_VALID_MASK) && + exit_reason != EXIT_REASON_EXCEPTION_NMI) printk(KERN_WARNING "%s: unexpected, valid vectoring info and " "exit reason is 0x%x\n", __FUNCTION__, exit_reason); if (exit_reason < kvm_vmx_max_exit_handlers @@ -2238,7 +2241,7 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) */ vmcs_writel(HOST_CR0, read_cr0()); - asm ( + asm( /* Store host registers */ #ifdef CONFIG_X86_64 "push %%rax; push %%rbx; push %%rdx;" @@ -2342,8 +2345,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])), [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP])), #ifdef CONFIG_X86_64 - [r8 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8 ])), - [r9 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9 ])), + [r8]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9])), [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])), [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])), [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])), @@ -2352,11 +2355,12 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])), #endif [cr2]"i"(offsetof(struct kvm_vcpu, cr2)) - : "cc", "memory" ); + : "cc", "memory"); - vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; + vcpu->interrupt_window_open = + (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; - asm ("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS)); + asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS)); vmx->launched = 1; intr_info = vmcs_read32(VM_EXIT_INTR_INFO); diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index fd4e14666088..270d477a2aa6 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -234,9 +234,9 @@ enum vmcs_field { /* * Exit Qualifications for MOV for Control Register Access */ -#define CONTROL_REG_ACCESS_NUM 0x7 /* 2:0, number of control register */ +#define CONTROL_REG_ACCESS_NUM 0x7 /* 2:0, number of control reg.*/ #define CONTROL_REG_ACCESS_TYPE 0x30 /* 5:4, access type */ -#define CONTROL_REG_ACCESS_REG 0xf00 /* 10:8, general purpose register */ +#define CONTROL_REG_ACCESS_REG 0xf00 /* 10:8, general purpose reg. */ #define LMSW_SOURCE_DATA_SHIFT 16 #define LMSW_SOURCE_DATA (0xFFFF << LMSW_SOURCE_DATA_SHIFT) /* 16:31 lmsw source */ #define REG_EAX (0 << 8) @@ -259,11 +259,11 @@ enum vmcs_field { /* * Exit Qualifications for MOV for Debug Register Access */ -#define DEBUG_REG_ACCESS_NUM 0x7 /* 2:0, number of debug register */ +#define DEBUG_REG_ACCESS_NUM 0x7 /* 2:0, number of debug reg. */ #define DEBUG_REG_ACCESS_TYPE 0x10 /* 4, direction of access */ #define TYPE_MOV_TO_DR (0 << 4) #define TYPE_MOV_FROM_DR (1 << 4) -#define DEBUG_REG_ACCESS_REG 0xf00 /* 11:8, general purpose register */ +#define DEBUG_REG_ACCESS_REG 0xf00 /* 11:8, general purpose reg. */ /* segment AR */ diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index e294d8409571..75fd23bade9c 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -23,7 +23,7 @@ #include #include #include -#define DPRINTF(_f, _a ...) printf( _f , ## _a ) +#define DPRINTF(_f, _a ...) printf(_f , ## _a) #else #include "kvm.h" #define DPRINTF(x...) do {} while (0) @@ -285,21 +285,21 @@ static u16 twobyte_table[256] = { switch ((_dst).bytes) { \ case 2: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","4","2") \ + _PRE_EFLAGS("0", "4", "2") \ _op"w %"_wx"3,%1; " \ - _POST_EFLAGS("0","4","2") \ + _POST_EFLAGS("0", "4", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : _wy ((_src).val), "i" (EFLAGS_MASK) ); \ + : _wy ((_src).val), "i" (EFLAGS_MASK)); \ break; \ case 4: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","4","2") \ + _PRE_EFLAGS("0", "4", "2") \ _op"l %"_lx"3,%1; " \ - _POST_EFLAGS("0","4","2") \ + _POST_EFLAGS("0", "4", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : _ly ((_src).val), "i" (EFLAGS_MASK) ); \ + : _ly ((_src).val), "i" (EFLAGS_MASK)); \ break; \ case 8: \ __emulate_2op_8byte(_op, _src, _dst, \ @@ -311,16 +311,15 @@ static u16 twobyte_table[256] = { #define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \ do { \ unsigned long _tmp; \ - switch ( (_dst).bytes ) \ - { \ + switch ((_dst).bytes) { \ case 1: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","4","2") \ + _PRE_EFLAGS("0", "4", "2") \ _op"b %"_bx"3,%1; " \ - _POST_EFLAGS("0","4","2") \ + _POST_EFLAGS("0", "4", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : _by ((_src).val), "i" (EFLAGS_MASK) ); \ + : _by ((_src).val), "i" (EFLAGS_MASK)); \ break; \ default: \ __emulate_2op_nobyte(_op, _src, _dst, _eflags, \ @@ -349,34 +348,33 @@ static u16 twobyte_table[256] = { do { \ unsigned long _tmp; \ \ - switch ( (_dst).bytes ) \ - { \ + switch ((_dst).bytes) { \ case 1: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","3","2") \ + _PRE_EFLAGS("0", "3", "2") \ _op"b %1; " \ - _POST_EFLAGS("0","3","2") \ + _POST_EFLAGS("0", "3", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : "i" (EFLAGS_MASK) ); \ + : "i" (EFLAGS_MASK)); \ break; \ case 2: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","3","2") \ + _PRE_EFLAGS("0", "3", "2") \ _op"w %1; " \ - _POST_EFLAGS("0","3","2") \ + _POST_EFLAGS("0", "3", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : "i" (EFLAGS_MASK) ); \ + : "i" (EFLAGS_MASK)); \ break; \ case 4: \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","3","2") \ + _PRE_EFLAGS("0", "3", "2") \ _op"l %1; " \ - _POST_EFLAGS("0","3","2") \ + _POST_EFLAGS("0", "3", "2") \ : "=m" (_eflags), "=m" ((_dst).val), \ "=&r" (_tmp) \ - : "i" (EFLAGS_MASK) ); \ + : "i" (EFLAGS_MASK)); \ break; \ case 8: \ __emulate_1op_8byte(_op, _dst, _eflags); \ @@ -389,21 +387,21 @@ static u16 twobyte_table[256] = { #define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy) \ do { \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","4","2") \ + _PRE_EFLAGS("0", "4", "2") \ _op"q %"_qx"3,%1; " \ - _POST_EFLAGS("0","4","2") \ + _POST_EFLAGS("0", "4", "2") \ : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \ - : _qy ((_src).val), "i" (EFLAGS_MASK) ); \ + : _qy ((_src).val), "i" (EFLAGS_MASK)); \ } while (0) #define __emulate_1op_8byte(_op, _dst, _eflags) \ do { \ __asm__ __volatile__ ( \ - _PRE_EFLAGS("0","3","2") \ + _PRE_EFLAGS("0", "3", "2") \ _op"q %1; " \ - _POST_EFLAGS("0","3","2") \ + _POST_EFLAGS("0", "3", "2") \ : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \ - : "i" (EFLAGS_MASK) ); \ + : "i" (EFLAGS_MASK)); \ } while (0) #elif defined(__i386__) @@ -415,8 +413,8 @@ static u16 twobyte_table[256] = { #define insn_fetch(_type, _size, _eip) \ ({ unsigned long _x; \ rc = ops->read_std((unsigned long)(_eip) + ctxt->cs_base, &_x, \ - (_size), ctxt->vcpu); \ - if ( rc != 0 ) \ + (_size), ctxt->vcpu); \ + if (rc != 0) \ goto done; \ (_eip) += (_size); \ (_type)_x; \ @@ -780,7 +778,7 @@ done_prefixes: } if (c->ad_bytes != 8) c->modrm_ea = (u32)c->modrm_ea; - modrm_done: +modrm_done: ; } @@ -828,10 +826,9 @@ done_prefixes: c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; /* Don't fetch the address for invlpg: it could be unmapped. */ - if (c->twobyte && c->b == 0x01 - && c->modrm_reg == 7) + if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7) break; - srcmem_common: + srcmem_common: /* * For instructions with a ModR/M byte, switch to register * access if Mod = 3. @@ -1175,10 +1172,11 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (c->src.type == OP_MEM) { c->src.ptr = (unsigned long *)cr2; c->src.val = 0; - if ((rc = ops->read_emulated((unsigned long)c->src.ptr, - &c->src.val, - c->src.bytes, - ctxt->vcpu)) != 0) + rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, + ctxt->vcpu); + if (rc != 0) goto done; c->src.orig_val = c->src.val; } -- cgit v1.2.3 From 6fc138d2278078990f597cb1f62fde9e5b458f96 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 9 Oct 2007 19:20:39 +0200 Subject: KVM: Support assigning userspace memory to the guest Instead of having the kernel allocate memory to the guest, let userspace allocate it and pass the address to the kernel. This is required for s390 support, but also enables features like memory sharing and using hugetlbfs backed memory. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 81 +++++++++++++++++++++++++++++++++++++++++++------- include/linux/kvm.h | 12 ++++++++ 3 files changed, 83 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ec5b498945ae..3eaed4dafec7 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -408,6 +408,7 @@ struct kvm_memory_slot { struct page **phys_mem; unsigned long *rmap; unsigned long *dirty_bitmap; + int user_alloc; /* user allocated memory */ }; struct kvm { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index a1983d2d5b8f..22b143feb66d 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -300,19 +301,40 @@ static struct kvm *kvm_create_vm(void) return kvm; } +static void kvm_free_userspace_physmem(struct kvm_memory_slot *free) +{ + int i; + + for (i = 0; i < free->npages; ++i) { + if (free->phys_mem[i]) { + if (!PageReserved(free->phys_mem[i])) + SetPageDirty(free->phys_mem[i]); + page_cache_release(free->phys_mem[i]); + } + } +} + +static void kvm_free_kernel_physmem(struct kvm_memory_slot *free) +{ + int i; + + for (i = 0; i < free->npages; ++i) + if (free->phys_mem[i]) + __free_page(free->phys_mem[i]); +} + /* * Free any memory in @free but not in @dont. */ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, struct kvm_memory_slot *dont) { - int i; - if (!dont || free->phys_mem != dont->phys_mem) if (free->phys_mem) { - for (i = 0; i < free->npages; ++i) - if (free->phys_mem[i]) - __free_page(free->phys_mem[i]); + if (free->user_alloc) + kvm_free_userspace_physmem(free); + else + kvm_free_kernel_physmem(free); vfree(free->phys_mem); } if (!dont || free->rmap != dont->rmap) @@ -652,7 +674,9 @@ EXPORT_SYMBOL_GPL(fx_init); * Discontiguous memory is allowed, mostly for framebuffers. */ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct kvm_memory_region *mem) + struct + kvm_userspace_memory_region *mem, + int user_alloc) { int r; gfn_t base_gfn; @@ -728,11 +752,27 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, memset(new.phys_mem, 0, npages * sizeof(struct page *)); memset(new.rmap, 0, npages * sizeof(*new.rmap)); - for (i = 0; i < npages; ++i) { - new.phys_mem[i] = alloc_page(GFP_HIGHUSER - | __GFP_ZERO); - if (!new.phys_mem[i]) + if (user_alloc) { + unsigned long pages_num; + + new.user_alloc = 1; + down_read(¤t->mm->mmap_sem); + + pages_num = get_user_pages(current, current->mm, + mem->userspace_addr, + npages, 1, 1, new.phys_mem, + NULL); + + up_read(¤t->mm->mmap_sem); + if (pages_num != npages) goto out_unlock; + } else { + for (i = 0; i < npages; ++i) { + new.phys_mem[i] = alloc_page(GFP_HIGHUSER + | __GFP_ZERO); + if (!new.phys_mem[i]) + goto out_unlock; + } } } @@ -3108,11 +3148,29 @@ static long kvm_vm_ioctl(struct file *filp, break; case KVM_SET_MEMORY_REGION: { struct kvm_memory_region kvm_mem; + struct kvm_userspace_memory_region kvm_userspace_mem; r = -EFAULT; if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem)) goto out; - r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_mem); + kvm_userspace_mem.slot = kvm_mem.slot; + kvm_userspace_mem.flags = kvm_mem.flags; + kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr; + kvm_userspace_mem.memory_size = kvm_mem.memory_size; + r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0); + if (r) + goto out; + break; + } + case KVM_SET_USER_MEMORY_REGION: { + struct kvm_userspace_memory_region kvm_userspace_mem; + + r = -EFAULT; + if (copy_from_user(&kvm_userspace_mem, argp, + sizeof kvm_userspace_mem)) + goto out; + + r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1); if (r) goto out; break; @@ -3332,6 +3390,7 @@ static long kvm_dev_ioctl(struct file *filp, case KVM_CAP_IRQCHIP: case KVM_CAP_HLT: case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: + case KVM_CAP_USER_MEMORY: r = 1; break; default: diff --git a/include/linux/kvm.h b/include/linux/kvm.h index d2fd973d81fe..971f465e26fa 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -23,6 +23,15 @@ struct kvm_memory_region { __u64 memory_size; /* bytes */ }; +/* for KVM_SET_USER_MEMORY_REGION */ +struct kvm_userspace_memory_region { + __u32 slot; + __u32 flags; + __u64 guest_phys_addr; + __u64 memory_size; /* bytes */ + __u64 userspace_addr; /* start of the userspace allocated memory */ +}; + /* for kvm_memory_region::flags */ #define KVM_MEM_LOG_DIRTY_PAGES 1UL @@ -348,6 +357,7 @@ struct kvm_signal_mask { #define KVM_CAP_IRQCHIP 0 #define KVM_CAP_HLT 1 #define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2 +#define KVM_CAP_USER_MEMORY 3 /* * ioctls for VM fds @@ -355,6 +365,8 @@ struct kvm_signal_mask { #define KVM_SET_MEMORY_REGION _IOW(KVMIO, 0x40, struct kvm_memory_region) #define KVM_SET_NR_MMU_PAGES _IO(KVMIO, 0x44) #define KVM_GET_NR_MMU_PAGES _IO(KVMIO, 0x45) +#define KVM_SET_USER_MEMORY_REGION _IOW(KVMIO, 0x46,\ + struct kvm_userspace_memory_region) /* * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns * a vcpu fd. -- cgit v1.2.3 From 043405e10001fe7aae60c46a57189515f13a6468 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Wed, 10 Oct 2007 17:16:19 +0200 Subject: KVM: Move x86 msr handling to new files x86.[ch] Signed-off-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/Makefile | 2 +- drivers/kvm/kvm.h | 4 ++ drivers/kvm/kvm_main.c | 69 ++------------------------------- drivers/kvm/x86.c | 102 +++++++++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/x86.h | 16 ++++++++ 5 files changed, 126 insertions(+), 67 deletions(-) create mode 100644 drivers/kvm/x86.c create mode 100644 drivers/kvm/x86.h (limited to 'drivers') diff --git a/drivers/kvm/Makefile b/drivers/kvm/Makefile index e5a8f4d3e973..cf18ad46e987 100644 --- a/drivers/kvm/Makefile +++ b/drivers/kvm/Makefile @@ -2,7 +2,7 @@ # Makefile for Kernel-based Virtual Machine module # -kvm-objs := kvm_main.o mmu.o x86_emulate.o i8259.o irq.o lapic.o ioapic.o +kvm-objs := kvm_main.o x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o ioapic.o obj-$(CONFIG_KVM) += kvm.o kvm-intel-objs = vmx.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3eaed4dafec7..9c9c1d7f5b30 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -653,6 +653,10 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); int kvm_fix_hypercall(struct kvm_vcpu *vcpu); +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +__init void kvm_arch_init(void); + static inline void kvm_guest_enter(void) { current->flags |= PF_VCPU; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 22b143feb66d..ec696887b222 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -16,6 +16,7 @@ */ #include "kvm.h" +#include "x86.h" #include "x86_emulate.h" #include "segment_descriptor.h" #include "irq.h" @@ -2507,43 +2508,6 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) } EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); -/* - * List of msr numbers which we expose to userspace through KVM_GET_MSRS - * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. - * - * This list is modified at module load time to reflect the - * capabilities of the host cpu. - */ -static u32 msrs_to_save[] = { - MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, - MSR_K6_STAR, -#ifdef CONFIG_X86_64 - MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, -#endif - MSR_IA32_TIME_STAMP_COUNTER, -}; - -static unsigned num_msrs_to_save; - -static u32 emulated_msrs[] = { - MSR_IA32_MISC_ENABLE, -}; - -static __init void kvm_init_msr_list(void) -{ - u32 dummy[2]; - unsigned i, j; - - for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) { - if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) - continue; - if (j < i) - msrs_to_save[j] = msrs_to_save[i]; - j++; - } - num_msrs_to_save = j; -} - /* * Adapt set_msr() to msr_io()'s calling convention */ @@ -3356,33 +3320,6 @@ static long kvm_dev_ioctl(struct file *filp, goto out; r = kvm_dev_ioctl_create_vm(); break; - case KVM_GET_MSR_INDEX_LIST: { - struct kvm_msr_list __user *user_msr_list = argp; - struct kvm_msr_list msr_list; - unsigned n; - - r = -EFAULT; - if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list)) - goto out; - n = msr_list.nmsrs; - msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs); - if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list)) - goto out; - r = -E2BIG; - if (n < num_msrs_to_save) - goto out; - r = -EFAULT; - if (copy_to_user(user_msr_list->indices, &msrs_to_save, - num_msrs_to_save * sizeof(u32))) - goto out; - if (copy_to_user(user_msr_list->indices - + num_msrs_to_save * sizeof(u32), - &emulated_msrs, - ARRAY_SIZE(emulated_msrs) * sizeof(u32))) - goto out; - r = 0; - break; - } case KVM_CHECK_EXTENSION: { int ext = (long)argp; @@ -3406,7 +3343,7 @@ static long kvm_dev_ioctl(struct file *filp, r = 2 * PAGE_SIZE; break; default: - ; + return kvm_arch_dev_ioctl(filp, ioctl, arg); } out: return r; @@ -3770,7 +3707,7 @@ static __init int kvm_init(void) kvm_init_debug(); - kvm_init_msr_list(); + kvm_arch_init(); bad_page = alloc_page(GFP_KERNEL); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c new file mode 100644 index 000000000000..437902cf178d --- /dev/null +++ b/drivers/kvm/x86.c @@ -0,0 +1,102 @@ +/* + * Kernel-based Virtual Machine driver for Linux + * + * derived from drivers/kvm/kvm_main.c + * + * Copyright (C) 2006 Qumranet, Inc. + * + * Authors: + * Avi Kivity + * Yaniv Kamay + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#include "x86.h" + +#include + +/* + * List of msr numbers which we expose to userspace through KVM_GET_MSRS + * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. + * + * This list is modified at module load time to reflect the + * capabilities of the host cpu. + */ +static u32 msrs_to_save[] = { + MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, + MSR_K6_STAR, +#ifdef CONFIG_X86_64 + MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, +#endif + MSR_IA32_TIME_STAMP_COUNTER, +}; + +static unsigned num_msrs_to_save; + +static u32 emulated_msrs[] = { + MSR_IA32_MISC_ENABLE, +}; + +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + long r; + + switch (ioctl) { + case KVM_GET_MSR_INDEX_LIST: { + struct kvm_msr_list __user *user_msr_list = argp; + struct kvm_msr_list msr_list; + unsigned n; + + r = -EFAULT; + if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list)) + goto out; + n = msr_list.nmsrs; + msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs); + if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list)) + goto out; + r = -E2BIG; + if (n < num_msrs_to_save) + goto out; + r = -EFAULT; + if (copy_to_user(user_msr_list->indices, &msrs_to_save, + num_msrs_to_save * sizeof(u32))) + goto out; + if (copy_to_user(user_msr_list->indices + + num_msrs_to_save * sizeof(u32), + &emulated_msrs, + ARRAY_SIZE(emulated_msrs) * sizeof(u32))) + goto out; + r = 0; + break; + } + default: + r = -EINVAL; + } +out: + return r; +} + +static __init void kvm_init_msr_list(void) +{ + u32 dummy[2]; + unsigned i, j; + + for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) { + if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) + continue; + if (j < i) + msrs_to_save[j] = msrs_to_save[i]; + j++; + } + num_msrs_to_save = j; +} + +__init void kvm_arch_init(void) +{ + kvm_init_msr_list(); +} diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h new file mode 100644 index 000000000000..1e2f71bd805d --- /dev/null +++ b/drivers/kvm/x86.h @@ -0,0 +1,16 @@ +#/* + * Kernel-based Virtual Machine driver for Linux + * + * This header defines architecture specific interfaces, x86 version + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#ifndef KVM_X86_H +#define KVM_X86_H + +#include "kvm.h" + +#endif -- cgit v1.2.3 From f67a46f4aa1212b38696ac6b6a82b4323cea61aa Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Wed, 10 Oct 2007 19:25:50 -0500 Subject: KVM: MMU: Clean up MMU functions to take struct kvm when appropriate Some of the MMU functions take a struct kvm_vcpu even though they affect all VCPUs. This patch cleans up some of them to instead take a struct kvm. This makes things a bit more clear. The main thing that was confusing me was whether certain functions need to be called on all VCPUs. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 18 +++++++++--------- drivers/kvm/paging_tmpl.h | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index ece0aa4e4c9f..a5ca9457e929 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -606,7 +606,7 @@ static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page, BUG(); } -static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu, +static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) { unsigned index; @@ -616,7 +616,7 @@ static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu, pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->mmu_page_hash[index]; + bucket = &kvm->mmu_page_hash[index]; hlist_for_each_entry(page, node, bucket, hash_link) if (page->gfn == gfn && !page->role.metaphysical) { pgprintk("%s: found role %x\n", @@ -782,7 +782,7 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages) kvm->n_alloc_mmu_pages = kvm_nr_mmu_pages; } -static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) +static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) { unsigned index; struct hlist_head *bucket; @@ -793,25 +793,25 @@ static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn) pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); r = 0; index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->mmu_page_hash[index]; + bucket = &kvm->mmu_page_hash[index]; hlist_for_each_entry_safe(page, node, n, bucket, hash_link) if (page->gfn == gfn && !page->role.metaphysical) { pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, page->role.word); - kvm_mmu_zap_page(vcpu->kvm, page); + kvm_mmu_zap_page(kvm, page); r = 1; } return r; } -static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn) +static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) { struct kvm_mmu_page *page; - while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) { + while ((page = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { pgprintk("%s: zap %lx %x\n", __FUNCTION__, gfn, page->role.word); - kvm_mmu_zap_page(vcpu->kvm, page); + kvm_mmu_zap_page(kvm, page); } } @@ -1299,7 +1299,7 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); - return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT); + return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); } void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 447d2c31f0cb..4f6edf85d13f 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -268,11 +268,11 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, spte |= PT_WRITABLE_MASK; if (user_fault) { - mmu_unshadow(vcpu, gfn); + mmu_unshadow(vcpu->kvm, gfn); goto unshadowed; } - shadow = kvm_mmu_lookup_page(vcpu, gfn); + shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn); if (shadow) { pgprintk("%s: found shadow page for %lx, marking ro\n", __FUNCTION__, gfn); -- cgit v1.2.3 From 4a4c99248713e878e1e2880015d01049aec805f3 Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Wed, 10 Oct 2007 20:08:41 -0500 Subject: KVM: MMU: More struct kvm_vcpu -> struct kvm cleanups This time, the biggest change is gpa_to_hpa. The translation of GPA to HPA does not depend on the VCPU state unlike GVA to GPA so there's no need to pass in the kvm_vcpu. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/mmu.c | 26 +++++++++++++------------- drivers/kvm/paging_tmpl.h | 6 +++--- 3 files changed, 17 insertions(+), 17 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 9c9c1d7f5b30..d56962d49aa6 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -554,7 +554,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); void kvm_mmu_zap_all(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); -hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa); +hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index a5ca9457e929..d046ba807763 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -451,14 +451,14 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) } } -static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) +static void rmap_write_protect(struct kvm *kvm, u64 gfn) { struct kvm_rmap_desc *desc; unsigned long *rmapp; u64 *spte; - gfn = unalias_gfn(vcpu->kvm, gfn); - rmapp = gfn_to_rmap(vcpu->kvm, gfn); + gfn = unalias_gfn(kvm, gfn); + rmapp = gfn_to_rmap(kvm, gfn); while (*rmapp) { if (!(*rmapp & 1)) @@ -471,9 +471,9 @@ static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) BUG_ON(!(*spte & PT_PRESENT_MASK)); BUG_ON(!(*spte & PT_WRITABLE_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - rmap_remove(vcpu->kvm, spte); + rmap_remove(kvm, spte); set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); - kvm_flush_remote_tlbs(vcpu->kvm); + kvm_flush_remote_tlbs(kvm); } } @@ -670,7 +670,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, hlist_add_head(&page->hash_link, bucket); vcpu->mmu.prefetch_page(vcpu, page); if (!metaphysical) - rmap_write_protect(vcpu, gfn); + rmap_write_protect(vcpu->kvm, gfn); return page; } @@ -823,19 +823,19 @@ static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa) __set_bit(slot, &page_head->slot_bitmap); } -hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa) +hpa_t safe_gpa_to_hpa(struct kvm *kvm, gpa_t gpa) { - hpa_t hpa = gpa_to_hpa(vcpu, gpa); + hpa_t hpa = gpa_to_hpa(kvm, gpa); return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa; } -hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa) +hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa) { struct page *page; ASSERT((gpa & HPA_ERR_MASK) == 0); - page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); + page = gfn_to_page(kvm, gpa >> PAGE_SHIFT); if (!page) return gpa | HPA_ERR_MASK; return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) @@ -848,7 +848,7 @@ hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva) if (gpa == UNMAPPED_GVA) return UNMAPPED_GVA; - return gpa_to_hpa(vcpu, gpa); + return gpa_to_hpa(vcpu->kvm, gpa); } struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) @@ -857,7 +857,7 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) if (gpa == UNMAPPED_GVA) return NULL; - return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT); + return pfn_to_page(gpa_to_hpa(vcpu->kvm, gpa) >> PAGE_SHIFT); } static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) @@ -1012,7 +1012,7 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); - paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK); + paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK); if (is_error_hpa(paddr)) return 1; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 4f6edf85d13f..8e1e4ca6ea4e 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -103,7 +103,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); slot = gfn_to_memslot(vcpu->kvm, table_gfn); - hpa = safe_gpa_to_hpa(vcpu, root & PT64_BASE_ADDR_MASK); + hpa = safe_gpa_to_hpa(vcpu->kvm, root & PT64_BASE_ADDR_MASK); walker->page = pfn_to_page(hpa >> PAGE_SHIFT); walker->table = kmap_atomic(walker->page, KM_USER0); @@ -159,7 +159,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->inherited_ar &= walker->table[index]; table_gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; kunmap_atomic(walker->table, KM_USER0); - paddr = safe_gpa_to_hpa(vcpu, table_gfn << PAGE_SHIFT); + paddr = safe_gpa_to_hpa(vcpu->kvm, table_gfn << PAGE_SHIFT); walker->page = pfn_to_page(paddr >> PAGE_SHIFT); walker->table = kmap_atomic(walker->page, KM_USER0); --walker->level; @@ -248,7 +248,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (!dirty) access_bits &= ~PT_WRITABLE_MASK; - paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK); + paddr = gpa_to_hpa(vcpu->kvm, gaddr & PT64_BASE_ADDR_MASK); spte |= PT_PRESENT_MASK; if (access_bits & PT_USER_MASK) -- cgit v1.2.3 From e3c5e7ec9efe0ebd47fa812cc86f01c51905edf6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 12:32:30 +0200 Subject: KVM: Move guest pte dirty bit management to the guest pagetable walker This is more consistent with the accessed bit management, and makes the dirty bit available earlier for other purposes. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 5 +++++ drivers/kvm/paging_tmpl.h | 31 ++++++++----------------------- 2 files changed, 13 insertions(+), 23 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index d046ba807763..e6616a6c9cef 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -199,6 +199,11 @@ static int is_writeble_pte(unsigned long pte) return pte & PT_WRITABLE_MASK; } +static int is_dirty_pte(unsigned long pte) +{ + return pte & PT_DIRTY_MASK; +} + static int is_io_pte(unsigned long pte) { return pte & PT_SHADOW_IO_MARK; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 8e1e4ca6ea4e..da36e48fd026 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -144,6 +144,10 @@ static int FNAME(walk_addr)(struct guest_walker *walker, if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; + if (write_fault && !is_dirty_pte(*ptep)) { + mark_page_dirty(vcpu->kvm, table_gfn); + *ptep |= PT_DIRTY_MASK; + } break; } @@ -153,6 +157,10 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->gfn = (*ptep & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); + if (write_fault && !is_dirty_pte(*ptep)) { + mark_page_dirty(vcpu->kvm, table_gfn); + *ptep |= PT_DIRTY_MASK; + } break; } @@ -194,12 +202,6 @@ err: return 0; } -static void FNAME(mark_pagetable_dirty)(struct kvm *kvm, - struct guest_walker *walker) -{ - mark_page_dirty(kvm, walker->table_gfn[walker->level - 1]); -} - static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, u64 *shadow_pte, gpa_t gaddr, @@ -221,23 +223,6 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, write_fault, user_fault, gfn); - if (write_fault && !dirty) { - pt_element_t *guest_ent, *tmp = NULL; - - if (walker->ptep) - guest_ent = walker->ptep; - else { - tmp = kmap_atomic(walker->page, KM_USER0); - guest_ent = &tmp[walker->index]; - } - - *guest_ent |= PT_DIRTY_MASK; - if (!walker->ptep) - kunmap_atomic(tmp, KM_USER0); - dirty = 1; - FNAME(mark_pagetable_dirty)(vcpu->kvm, walker); - } - /* * We don't set the accessed bit, since we sometimes want to see * whether the guest actually used the pte (in order to detect -- cgit v1.2.3 From c22e3514fceb2f514093ce1d19a2f660ac7347ae Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:08:41 +0200 Subject: KVM: MMU: Fix nx access bit for huge pages We must set the bit before the shift, otherwise the wrong bit gets set. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index da36e48fd026..e07cb2e006c2 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -382,9 +382,9 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, metaphysical = 1; hugepage_access = walker->pte; hugepage_access &= PT_USER_MASK | PT_WRITABLE_MASK; + hugepage_access >>= PT_WRITABLE_SHIFT; if (walker->pte & PT64_NX_MASK) hugepage_access |= (1 << 2); - hugepage_access >>= PT_WRITABLE_SHIFT; table_gfn = (walker->pte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; } else { -- cgit v1.2.3 From cc70e7374df1e3a56d718e8ca330619f316511a6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:12:24 +0200 Subject: KVM: MMU: Disable write access on clean large pages By forcing clean huge pages to be read-only, we have separate roles for the shadow of a clean large page and the shadow of a dirty large page. This is necessary because different ptes will be instantiated for the two cases, even for read faults. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index e07cb2e006c2..4538b1533d5b 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -382,6 +382,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, metaphysical = 1; hugepage_access = walker->pte; hugepage_access &= PT_USER_MASK | PT_WRITABLE_MASK; + if (!is_dirty_pte(walker->pte)) + hugepage_access &= ~PT_WRITABLE_MASK; hugepage_access >>= PT_WRITABLE_SHIFT; if (walker->pte & PT64_NX_MASK) hugepage_access |= (1 << 2); -- cgit v1.2.3 From 6bfccdc9ae55bb2e0be2e015d7d1ba061f4071c5 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:13:49 +0200 Subject: KVM: MMU: Instantiate real-mode shadows as user writable shadows This is consistent with real-mode permissions. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index e6616a6c9cef..f52604a7f9e8 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -902,7 +902,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) >> PAGE_SHIFT; new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, v, level - 1, - 1, 0, &table[index]); + 1, 3, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); return -ENOMEM; -- cgit v1.2.3 From 5df34a86f917024b67f9e7c850153390973cdfe3 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:22:59 +0200 Subject: KVM: MMU: Move dirty bit updates to a separate function Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 4538b1533d5b..a0f84a5379a4 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -69,6 +69,17 @@ struct guest_walker { u32 error_code; }; +static void FNAME(update_dirty_bit)(struct kvm_vcpu *vcpu, + int write_fault, + pt_element_t *ptep, + gfn_t table_gfn) +{ + if (write_fault && !is_dirty_pte(*ptep)) { + mark_page_dirty(vcpu->kvm, table_gfn); + *ptep |= PT_DIRTY_MASK; + } +} + /* * Fetch a guest pte for a guest virtual address */ @@ -144,10 +155,8 @@ static int FNAME(walk_addr)(struct guest_walker *walker, if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; - if (write_fault && !is_dirty_pte(*ptep)) { - mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_DIRTY_MASK; - } + FNAME(update_dirty_bit)(vcpu, write_fault, ptep, + table_gfn); break; } @@ -157,10 +166,8 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->gfn = (*ptep & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); - if (write_fault && !is_dirty_pte(*ptep)) { - mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_DIRTY_MASK; - } + FNAME(update_dirty_bit)(vcpu, write_fault, ptep, + table_gfn); break; } -- cgit v1.2.3 From c4fcc2724628c6548748ec80a90b548fc300e81f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 11 Oct 2007 15:30:21 +0200 Subject: KVM: MMU: When updating the dirty bit, inform the mmu about it Since the mmu uses different shadow pages for dirty large pages and clean large pages, this allows the mmu to drop ptes that are now invalid. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index a0f84a5379a4..a9e687b5c1e9 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -74,9 +74,14 @@ static void FNAME(update_dirty_bit)(struct kvm_vcpu *vcpu, pt_element_t *ptep, gfn_t table_gfn) { + gpa_t pte_gpa; + if (write_fault && !is_dirty_pte(*ptep)) { mark_page_dirty(vcpu->kvm, table_gfn); *ptep |= PT_DIRTY_MASK; + pte_gpa = ((gpa_t)table_gfn << PAGE_SHIFT); + pte_gpa += offset_in_page(ptep); + kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)ptep, sizeof(*ptep)); } } -- cgit v1.2.3 From 313a3dc75da20630e549441932a7654223f8d72a Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Thu, 11 Oct 2007 19:16:52 +0200 Subject: KVM: Portability: split kvm_vcpu_ioctl This patch splits kvm_vcpu_ioctl into archtecture independent parts, and x86 specific parts which go to kvm_arch_vcpu_ioctl in x86.c. Common ioctls for all architectures are: KVM_RUN, KVM_GET/SET_(S-)REGS, KVM_TRANSLATE, KVM_INTERRUPT, KVM_DEBUG_GUEST, KVM_SET_SIGNAL_MASK, KVM_GET/SET_FPU Note that some PPC chips don't have an FPU, so we might need an #ifdef around KVM_GET/SET_FPU one day. x86 specific ioctls are: KVM_GET/SET_LAPIC, KVM_SET_CPUID, KVM_GET/SET_MSRS An interresting aspect is vcpu_load/vcpu_put. We now have a common vcpu_load/put which does the preemption stuff, and an architecture specific kvm_arch_vcpu_load/put. In the x86 case, this one calls the vmx/svm function defined in kvm_x86_ops. Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Reviewed-by: Christian Ehrhardt Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 9 ++ drivers/kvm/kvm_main.c | 200 ++------------------------------------------ drivers/kvm/x86.c | 219 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 194 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index d56962d49aa6..1edf8a5e365e 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -537,6 +537,10 @@ extern struct kvm_x86_ops *kvm_x86_ops; int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id); void kvm_vcpu_uninit(struct kvm_vcpu *vcpu); +void vcpu_load(struct kvm_vcpu *vcpu); +void vcpu_put(struct kvm_vcpu *vcpu); + + int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, struct module *module); void kvm_exit_x86(void); @@ -655,6 +659,11 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu); long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); +long kvm_arch_vcpu_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); + __init void kvm_arch_init(void); static inline void kvm_guest_enter(void) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index ec696887b222..5fd2864b7811 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -90,8 +90,6 @@ static struct kvm_stats_debugfs_item { static struct dentry *debugfs_dir; -#define MAX_IO_MSRS 256 - #define CR0_RESERVED_BITS \ (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ @@ -179,21 +177,21 @@ EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); /* * Switches to specified vcpu, until a matching vcpu_put() */ -static void vcpu_load(struct kvm_vcpu *vcpu) +void vcpu_load(struct kvm_vcpu *vcpu) { int cpu; mutex_lock(&vcpu->mutex); cpu = get_cpu(); preempt_notifier_register(&vcpu->preempt_notifier); - kvm_x86_ops->vcpu_load(vcpu, cpu); + kvm_arch_vcpu_load(vcpu, cpu); put_cpu(); } -static void vcpu_put(struct kvm_vcpu *vcpu) +void vcpu_put(struct kvm_vcpu *vcpu) { preempt_disable(); - kvm_x86_ops->vcpu_put(vcpu); + kvm_arch_vcpu_put(vcpu); preempt_notifier_unregister(&vcpu->preempt_notifier); preempt_enable(); mutex_unlock(&vcpu->mutex); @@ -2508,86 +2506,6 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) } EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); -/* - * Adapt set_msr() to msr_io()'s calling convention - */ -static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) -{ - return kvm_set_msr(vcpu, index, *data); -} - -/* - * Read or write a bunch of msrs. All parameters are kernel addresses. - * - * @return number of msrs set successfully. - */ -static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, - struct kvm_msr_entry *entries, - int (*do_msr)(struct kvm_vcpu *vcpu, - unsigned index, u64 *data)) -{ - int i; - - vcpu_load(vcpu); - - for (i = 0; i < msrs->nmsrs; ++i) - if (do_msr(vcpu, entries[i].index, &entries[i].data)) - break; - - vcpu_put(vcpu); - - return i; -} - -/* - * Read or write a bunch of msrs. Parameters are user addresses. - * - * @return number of msrs set successfully. - */ -static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, - int (*do_msr)(struct kvm_vcpu *vcpu, - unsigned index, u64 *data), - int writeback) -{ - struct kvm_msrs msrs; - struct kvm_msr_entry *entries; - int r, n; - unsigned size; - - r = -EFAULT; - if (copy_from_user(&msrs, user_msrs, sizeof msrs)) - goto out; - - r = -E2BIG; - if (msrs.nmsrs >= MAX_IO_MSRS) - goto out; - - r = -ENOMEM; - size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; - entries = vmalloc(size); - if (!entries) - goto out; - - r = -EFAULT; - if (copy_from_user(entries, user_msrs->entries, size)) - goto out_free; - - r = n = __msr_io(vcpu, &msrs, entries, do_msr); - if (r < 0) - goto out_free; - - r = -EFAULT; - if (writeback && copy_to_user(user_msrs->entries, entries, size)) - goto out_free; - - r = n; - -out_free: - vfree(entries); -out: - return r; -} - /* * Translate a guest virtual address to a guest physical address. */ @@ -2761,48 +2679,6 @@ free_vcpu: return r; } -static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) -{ - u64 efer; - int i; - struct kvm_cpuid_entry *e, *entry; - - rdmsrl(MSR_EFER, efer); - entry = NULL; - for (i = 0; i < vcpu->cpuid_nent; ++i) { - e = &vcpu->cpuid_entries[i]; - if (e->function == 0x80000001) { - entry = e; - break; - } - } - if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) { - entry->edx &= ~(1 << 20); - printk(KERN_INFO "kvm: guest NX capability removed\n"); - } -} - -static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, - struct kvm_cpuid *cpuid, - struct kvm_cpuid_entry __user *entries) -{ - int r; - - r = -E2BIG; - if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) - goto out; - r = -EFAULT; - if (copy_from_user(&vcpu->cpuid_entries, entries, - cpuid->nent * sizeof(struct kvm_cpuid_entry))) - goto out; - vcpu->cpuid_nent = cpuid->nent; - cpuid_fix_nx_cap(vcpu); - return 0; - -out: - return r; -} - static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) { if (sigset) { @@ -2875,33 +2751,12 @@ static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) return 0; } -static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, - struct kvm_lapic_state *s) -{ - vcpu_load(vcpu); - memcpy(s->regs, vcpu->apic->regs, sizeof *s); - vcpu_put(vcpu); - - return 0; -} - -static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, - struct kvm_lapic_state *s) -{ - vcpu_load(vcpu); - memcpy(vcpu->apic->regs, s->regs, sizeof *s); - kvm_apic_post_state_restore(vcpu); - vcpu_put(vcpu); - - return 0; -} - static long kvm_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { struct kvm_vcpu *vcpu = filp->private_data; void __user *argp = (void __user *)arg; - int r = -EINVAL; + int r; switch (ioctl) { case KVM_RUN: @@ -2999,24 +2854,6 @@ static long kvm_vcpu_ioctl(struct file *filp, r = 0; break; } - case KVM_GET_MSRS: - r = msr_io(vcpu, argp, kvm_get_msr, 1); - break; - case KVM_SET_MSRS: - r = msr_io(vcpu, argp, do_set_msr, 0); - break; - case KVM_SET_CPUID: { - struct kvm_cpuid __user *cpuid_arg = argp; - struct kvm_cpuid cpuid; - - r = -EFAULT; - if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) - goto out; - r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); - if (r) - goto out; - break; - } case KVM_SET_SIGNAL_MASK: { struct kvm_signal_mask __user *sigmask_arg = argp; struct kvm_signal_mask kvm_sigmask; @@ -3065,33 +2902,8 @@ static long kvm_vcpu_ioctl(struct file *filp, r = 0; break; } - case KVM_GET_LAPIC: { - struct kvm_lapic_state lapic; - - memset(&lapic, 0, sizeof lapic); - r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &lapic, sizeof lapic)) - goto out; - r = 0; - break; - } - case KVM_SET_LAPIC: { - struct kvm_lapic_state lapic; - - r = -EFAULT; - if (copy_from_user(&lapic, argp, sizeof lapic)) - goto out; - r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);; - if (r) - goto out; - r = 0; - break; - } default: - ; + r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); } out: return r; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 437902cf178d..1fe209dd4caf 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -14,10 +14,18 @@ * */ +#include "kvm.h" #include "x86.h" +#include "irq.h" + +#include +#include +#include #include +#define MAX_IO_MSRS 256 + /* * List of msr numbers which we expose to userspace through KVM_GET_MSRS * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. @@ -40,6 +48,86 @@ static u32 emulated_msrs[] = { MSR_IA32_MISC_ENABLE, }; +/* + * Adapt set_msr() to msr_io()'s calling convention + */ +static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) +{ + return kvm_set_msr(vcpu, index, *data); +} + +/* + * Read or write a bunch of msrs. All parameters are kernel addresses. + * + * @return number of msrs set successfully. + */ +static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, + struct kvm_msr_entry *entries, + int (*do_msr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data)) +{ + int i; + + vcpu_load(vcpu); + + for (i = 0; i < msrs->nmsrs; ++i) + if (do_msr(vcpu, entries[i].index, &entries[i].data)) + break; + + vcpu_put(vcpu); + + return i; +} + +/* + * Read or write a bunch of msrs. Parameters are user addresses. + * + * @return number of msrs set successfully. + */ +static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, + int (*do_msr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data), + int writeback) +{ + struct kvm_msrs msrs; + struct kvm_msr_entry *entries; + int r, n; + unsigned size; + + r = -EFAULT; + if (copy_from_user(&msrs, user_msrs, sizeof msrs)) + goto out; + + r = -E2BIG; + if (msrs.nmsrs >= MAX_IO_MSRS) + goto out; + + r = -ENOMEM; + size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; + entries = vmalloc(size); + if (!entries) + goto out; + + r = -EFAULT; + if (copy_from_user(entries, user_msrs->entries, size)) + goto out_free; + + r = n = __msr_io(vcpu, &msrs, entries, do_msr); + if (r < 0) + goto out_free; + + r = -EFAULT; + if (writeback && copy_to_user(user_msrs->entries, entries, size)) + goto out_free; + + r = n; + +out_free: + vfree(entries); +out: + return r; +} + long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -81,6 +169,137 @@ out: return r; } +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + kvm_x86_ops->vcpu_load(vcpu, cpu); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->vcpu_put(vcpu); +} + +static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) +{ + u64 efer; + int i; + struct kvm_cpuid_entry *e, *entry; + + rdmsrl(MSR_EFER, efer); + entry = NULL; + for (i = 0; i < vcpu->cpuid_nent; ++i) { + e = &vcpu->cpuid_entries[i]; + if (e->function == 0x80000001) { + entry = e; + break; + } + } + if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) { + entry->edx &= ~(1 << 20); + printk(KERN_INFO "kvm: guest NX capability removed\n"); + } +} + +static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, + struct kvm_cpuid *cpuid, + struct kvm_cpuid_entry __user *entries) +{ + int r; + + r = -E2BIG; + if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) + goto out; + r = -EFAULT; + if (copy_from_user(&vcpu->cpuid_entries, entries, + cpuid->nent * sizeof(struct kvm_cpuid_entry))) + goto out; + vcpu->cpuid_nent = cpuid->nent; + cpuid_fix_nx_cap(vcpu); + return 0; + +out: + return r; +} + +static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, + struct kvm_lapic_state *s) +{ + vcpu_load(vcpu); + memcpy(s->regs, vcpu->apic->regs, sizeof *s); + vcpu_put(vcpu); + + return 0; +} + +static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, + struct kvm_lapic_state *s) +{ + vcpu_load(vcpu); + memcpy(vcpu->apic->regs, s->regs, sizeof *s); + kvm_apic_post_state_restore(vcpu); + vcpu_put(vcpu); + + return 0; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + int r; + + switch (ioctl) { + case KVM_GET_LAPIC: { + struct kvm_lapic_state lapic; + + memset(&lapic, 0, sizeof lapic); + r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &lapic, sizeof lapic)) + goto out; + r = 0; + break; + } + case KVM_SET_LAPIC: { + struct kvm_lapic_state lapic; + + r = -EFAULT; + if (copy_from_user(&lapic, argp, sizeof lapic)) + goto out; + r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);; + if (r) + goto out; + r = 0; + break; + } + case KVM_SET_CPUID: { + struct kvm_cpuid __user *cpuid_arg = argp; + struct kvm_cpuid cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); + if (r) + goto out; + break; + } + case KVM_GET_MSRS: + r = msr_io(vcpu, argp, kvm_get_msr, 1); + break; + case KVM_SET_MSRS: + r = msr_io(vcpu, argp, do_set_msr, 0); + break; + default: + r = -EINVAL; + } +out: + return r; +} + static __init void kvm_init_msr_list(void) { u32 dummy[2]; -- cgit v1.2.3 From e4d47f404b59adf22f639fedc943db99f090f9ee Mon Sep 17 00:00:00 2001 From: Qing He Date: Mon, 24 Sep 2007 17:39:41 +0800 Subject: KVM: apic round robin cleanup If no apic is enabled in the bitmap of an interrupt delivery with delivery mode of lowest priority, a warning should be reported rather than select a fallback vcpu Signed-off-by: Qing He Signed-off-by: Eddie (Yaozu) Dong Signed-off-by: Avi Kivity --- drivers/kvm/lapic.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index e15b42e48862..8840f9dc0bca 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -395,10 +395,9 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, unsigned long bitmap) { - int vcpu_id; int last; int next; - struct kvm_lapic *apic; + struct kvm_lapic *apic = NULL; last = kvm->round_robin_prev_vcpu; next = last; @@ -415,14 +414,8 @@ struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, } while (next != last); kvm->round_robin_prev_vcpu = next; - if (!apic) { - vcpu_id = ffs(bitmap) - 1; - if (vcpu_id < 0) { - vcpu_id = 0; - printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n"); - } - apic = kvm->vcpus[vcpu_id]->apic; - } + if (!apic) + printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n"); return apic; } -- cgit v1.2.3 From e25e3ed56fce591e09477b0ce043d90f97988f37 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Fri, 12 Oct 2007 11:01:59 +0200 Subject: KVM: Add some \n in ioapic_debug() Add new-line at end of debug strings. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index c7992e667fdb..8503d99b8339 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -40,8 +40,11 @@ #include #include #include "irq.h" -/* #define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ +#if 0 +#define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) +#else #define ioapic_debug(fmt, arg...) +#endif static void ioapic_deliver(struct kvm_ioapic *vioapic, int irq); static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic, @@ -113,7 +116,7 @@ static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val) default: index = (ioapic->ioregsel - 0x10) >> 1; - ioapic_debug("change redir index %x val %x", index, val); + ioapic_debug("change redir index %x val %x\n", index, val); if (index >= IOAPIC_NUM_PINS) return; if (ioapic->ioregsel & 1) { @@ -134,7 +137,7 @@ static void ioapic_inj_irq(struct kvm_ioapic *ioapic, struct kvm_lapic *target, u8 vector, u8 trig_mode, u8 delivery_mode) { - ioapic_debug("irq %d trig %d deliv %d", vector, trig_mode, + ioapic_debug("irq %d trig %d deliv %d\n", vector, trig_mode, delivery_mode); ASSERT((delivery_mode == dest_Fixed) || @@ -151,7 +154,7 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, struct kvm *kvm = ioapic->kvm; struct kvm_vcpu *vcpu; - ioapic_debug("dest %d dest_mode %d", dest, dest_mode); + ioapic_debug("dest %d dest_mode %d\n", dest, dest_mode); if (dest_mode == 0) { /* Physical mode. */ if (dest == 0xFF) { /* Broadcast. */ @@ -179,7 +182,7 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, kvm_apic_match_logical_addr(vcpu->apic, dest)) mask |= 1 << vcpu->vcpu_id; } - ioapic_debug("mask %x", mask); + ioapic_debug("mask %x\n", mask); return mask; } @@ -196,12 +199,12 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) int vcpu_id; ioapic_debug("dest=%x dest_mode=%x delivery_mode=%x " - "vector=%x trig_mode=%x", + "vector=%x trig_mode=%x\n", dest, dest_mode, delivery_mode, vector, trig_mode); deliver_bitmask = ioapic_get_delivery_bitmask(ioapic, dest, dest_mode); if (!deliver_bitmask) { - ioapic_debug("no target on destination"); + ioapic_debug("no target on destination\n"); return; } @@ -214,7 +217,7 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) trig_mode, delivery_mode); else ioapic_debug("null round robin: " - "mask=%x vector=%x delivery_mode=%x", + "mask=%x vector=%x delivery_mode=%x\n", deliver_bitmask, vector, dest_LowestPrio); break; case dest_Fixed: @@ -304,7 +307,7 @@ static void ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len, struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; u32 result; - ioapic_debug("addr %lx", (unsigned long)addr); + ioapic_debug("addr %lx\n", (unsigned long)addr); ASSERT(!(addr & 0xf)); /* check alignment */ addr &= 0xff; @@ -341,8 +344,8 @@ static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; u32 data; - ioapic_debug("ioapic_mmio_write addr=%lx len=%d val=%p\n", - addr, len, val); + ioapic_debug("ioapic_mmio_write addr=%p len=%d val=%p\n", + (void*)addr, len, val); ASSERT(!(addr & 0xf)); /* check alignment */ if (len == 4 || len == 8) data = *(u32 *) val; -- cgit v1.2.3 From ab6ef34b906546d85d92dbc3e0fb4e01cce05f62 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 16 Oct 2007 16:23:22 +0200 Subject: KVM: Move apic timer interrupt backlog processing to common code Beside the obvious goodness of making code more common, this prevents a livelock with the next patch which moves interrupt injection out of the critical section. Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 ++ drivers/kvm/svm.c | 1 - drivers/kvm/vmx.c | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 5fd2864b7811..4c96817929fd 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -2144,6 +2144,8 @@ again: if (unlikely(r)) goto out; + kvm_inject_pending_timer_irqs(vcpu); + preempt_disable(); kvm_x86_ops->prepare_guest_switch(vcpu); diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 7b21576b62bc..ef068d2dddd7 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1355,7 +1355,6 @@ static void svm_intr_assist(struct kvm_vcpu *vcpu) struct vmcb *vmcb = svm->vmcb; int intr_vector = -1; - kvm_inject_pending_timer_irqs(vcpu); if ((vmcb->control.exit_int_info & SVM_EVTINJ_VALID) && ((vmcb->control.exit_int_info & SVM_EVTINJ_TYPE_MASK) == 0)) { intr_vector = vmcb->control.exit_int_info & diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 6955580bb69e..9f77ddbeb025 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2191,7 +2191,6 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) int has_ext_irq, interrupt_window_open; int vector; - kvm_inject_pending_timer_irqs(vcpu); update_tpr_threshold(vcpu); has_ext_irq = kvm_cpu_has_interrupt(vcpu); -- cgit v1.2.3 From 3176bc3e59cc4e30ac10888f08124b31969a7a88 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 16 Oct 2007 17:22:08 +0200 Subject: KVM: Rename KVM_TLB_FLUSH to KVM_REQ_TLB_FLUSH We now have a new namespace, KVM_REQ_*, for bits in vcpu->requests. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/kvm_main.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 1edf8a5e365e..6ae7b6332e32 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -67,7 +67,7 @@ /* * vcpu->requests bit members */ -#define KVM_TLB_FLUSH 0 +#define KVM_REQ_TLB_FLUSH 0 /* * Address types: diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 4c96817929fd..1879b409bed2 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -212,7 +212,7 @@ void kvm_flush_remote_tlbs(struct kvm *kvm) vcpu = kvm->vcpus[i]; if (!vcpu) continue; - if (test_and_set_bit(KVM_TLB_FLUSH, &vcpu->requests)) + if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) continue; cpu = vcpu->cpu; if (cpu != -1 && cpu != raw_smp_processor_id()) @@ -2171,7 +2171,7 @@ again: kvm_guest_enter(); if (vcpu->requests) - if (test_and_clear_bit(KVM_TLB_FLUSH, &vcpu->requests)) + if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) kvm_x86_ops->tlb_flush(vcpu); kvm_x86_ops->run(vcpu, kvm_run); -- cgit v1.2.3 From d77a25074a8f845401f0eb786ebb8996e45d9e22 Mon Sep 17 00:00:00 2001 From: Nitin A Kamble Date: Fri, 12 Oct 2007 17:40:33 -0700 Subject: KVM: x86 emulator: Implement emulation of instruction: inc & dec Instructions: inc r16/r32 (opcode 0x40-0x47) dec r16/r32 (opcode 0x48-0x4f) Signed-off-by: Nitin A Kamble Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 75fd23bade9c..988c6498640f 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -96,8 +96,12 @@ static u8 opcode_table[256] = { ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, 0, 0, 0, 0, - /* 0x40 - 0x4F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0x40 - 0x47 */ + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + /* 0x48 - 0x4F */ + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, /* 0x50 - 0x57 */ ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, @@ -1376,6 +1380,18 @@ special_insn: if (c->twobyte) goto twobyte_special_insn; switch (c->b) { + case 0x40 ... 0x47: /* inc r16/r32 */ + c->dst.bytes = c->op_bytes; + c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; + c->dst.val = *c->dst.ptr; + emulate_1op("inc", c->dst, ctxt->eflags); + break; + case 0x48 ... 0x4f: /* dec r16/r32 */ + c->dst.bytes = c->op_bytes; + c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; + c->dst.val = *c->dst.ptr; + emulate_1op("dec", c->dst, ctxt->eflags); + break; case 0x50 ... 0x57: /* push reg */ if (c->op_bytes == 2) c->src.val = (u16) c->regs[c->b & 0x7]; -- cgit v1.2.3 From 42bf3f0a1f5a25be26e6bb53162cdee82410310d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 17 Oct 2007 12:18:47 +0200 Subject: KVM: MMU: Simplify page table walker Simplify the walker level loop not to carry so much information from one loop to the next. In addition to being complex, this made kmap_atomic() critical sections difficult to manage. As a result of this change, kmap_atomic() sections are limited to actually touching the guest pte, which allows the other functions called from the walker to do sleepy operations. This will happen when we enable swapping. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 124 ++++++++++++++++++---------------------------- 1 file changed, 48 insertions(+), 76 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index a9e687b5c1e9..bab1b7f8d705 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -59,32 +59,12 @@ struct guest_walker { int level; gfn_t table_gfn[PT_MAX_FULL_LEVELS]; - pt_element_t *table; pt_element_t pte; - pt_element_t *ptep; - struct page *page; - int index; pt_element_t inherited_ar; gfn_t gfn; u32 error_code; }; -static void FNAME(update_dirty_bit)(struct kvm_vcpu *vcpu, - int write_fault, - pt_element_t *ptep, - gfn_t table_gfn) -{ - gpa_t pte_gpa; - - if (write_fault && !is_dirty_pte(*ptep)) { - mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_DIRTY_MASK; - pte_gpa = ((gpa_t)table_gfn << PAGE_SHIFT); - pte_gpa += offset_in_page(ptep); - kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)ptep, sizeof(*ptep)); - } -} - /* * Fetch a guest pte for a guest virtual address */ @@ -94,105 +74,99 @@ static int FNAME(walk_addr)(struct guest_walker *walker, { hpa_t hpa; struct kvm_memory_slot *slot; - pt_element_t *ptep; - pt_element_t root; + struct page *page; + pt_element_t *table; + pt_element_t pte; gfn_t table_gfn; + unsigned index; + gpa_t pte_gpa; pgprintk("%s: addr %lx\n", __FUNCTION__, addr); walker->level = vcpu->mmu.root_level; - walker->table = NULL; - walker->page = NULL; - walker->ptep = NULL; - root = vcpu->cr3; + pte = vcpu->cr3; #if PTTYPE == 64 if (!is_long_mode(vcpu)) { - walker->ptep = &vcpu->pdptrs[(addr >> 30) & 3]; - root = *walker->ptep; - walker->pte = root; - if (!(root & PT_PRESENT_MASK)) + pte = vcpu->pdptrs[(addr >> 30) & 3]; + if (!is_present_pte(pte)) goto not_present; --walker->level; } #endif - table_gfn = (root & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; - walker->table_gfn[walker->level - 1] = table_gfn; - pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, - walker->level - 1, table_gfn); - slot = gfn_to_memslot(vcpu->kvm, table_gfn); - hpa = safe_gpa_to_hpa(vcpu->kvm, root & PT64_BASE_ADDR_MASK); - walker->page = pfn_to_page(hpa >> PAGE_SHIFT); - walker->table = kmap_atomic(walker->page, KM_USER0); - ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0); walker->inherited_ar = PT_USER_MASK | PT_WRITABLE_MASK; for (;;) { - int index = PT_INDEX(addr, walker->level); - hpa_t paddr; + index = PT_INDEX(addr, walker->level); - ptep = &walker->table[index]; - walker->index = index; - ASSERT(((unsigned long)walker->table & PAGE_MASK) == - ((unsigned long)ptep & PAGE_MASK)); + table_gfn = (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + walker->table_gfn[walker->level - 1] = table_gfn; + pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, + walker->level - 1, table_gfn); + + slot = gfn_to_memslot(vcpu->kvm, table_gfn); + hpa = safe_gpa_to_hpa(vcpu->kvm, pte & PT64_BASE_ADDR_MASK); + page = pfn_to_page(hpa >> PAGE_SHIFT); - if (!is_present_pte(*ptep)) + table = kmap_atomic(page, KM_USER0); + pte = table[index]; + kunmap_atomic(table, KM_USER0); + + if (!is_present_pte(pte)) goto not_present; - if (write_fault && !is_writeble_pte(*ptep)) + if (write_fault && !is_writeble_pte(pte)) if (user_fault || is_write_protection(vcpu)) goto access_error; - if (user_fault && !(*ptep & PT_USER_MASK)) + if (user_fault && !(pte & PT_USER_MASK)) goto access_error; #if PTTYPE == 64 - if (fetch_fault && is_nx(vcpu) && (*ptep & PT64_NX_MASK)) + if (fetch_fault && is_nx(vcpu) && (pte & PT64_NX_MASK)) goto access_error; #endif - if (!(*ptep & PT_ACCESSED_MASK)) { + if (!(pte & PT_ACCESSED_MASK)) { mark_page_dirty(vcpu->kvm, table_gfn); - *ptep |= PT_ACCESSED_MASK; + pte |= PT_ACCESSED_MASK; + table = kmap_atomic(page, KM_USER0); + table[index] = pte; + kunmap_atomic(table, KM_USER0); } if (walker->level == PT_PAGE_TABLE_LEVEL) { - walker->gfn = (*ptep & PT_BASE_ADDR_MASK) - >> PAGE_SHIFT; - FNAME(update_dirty_bit)(vcpu, write_fault, ptep, - table_gfn); + walker->gfn = (pte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; break; } if (walker->level == PT_DIRECTORY_LEVEL - && (*ptep & PT_PAGE_SIZE_MASK) + && (pte & PT_PAGE_SIZE_MASK) && (PTTYPE == 64 || is_pse(vcpu))) { - walker->gfn = (*ptep & PT_DIR_BASE_ADDR_MASK) + walker->gfn = (pte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); - FNAME(update_dirty_bit)(vcpu, write_fault, ptep, - table_gfn); break; } - walker->inherited_ar &= walker->table[index]; - table_gfn = (*ptep & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; - kunmap_atomic(walker->table, KM_USER0); - paddr = safe_gpa_to_hpa(vcpu->kvm, table_gfn << PAGE_SHIFT); - walker->page = pfn_to_page(paddr >> PAGE_SHIFT); - walker->table = kmap_atomic(walker->page, KM_USER0); + walker->inherited_ar &= pte; --walker->level; - walker->table_gfn[walker->level - 1] = table_gfn; - pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, - walker->level - 1, table_gfn); } - walker->pte = *ptep; - if (walker->page) - walker->ptep = NULL; - if (walker->table) - kunmap_atomic(walker->table, KM_USER0); - pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)*ptep); + + if (write_fault && !is_dirty_pte(pte)) { + mark_page_dirty(vcpu->kvm, table_gfn); + pte |= PT_DIRTY_MASK; + table = kmap_atomic(page, KM_USER0); + table[index] = pte; + kunmap_atomic(table, KM_USER0); + pte_gpa = table_gfn << PAGE_SHIFT; + pte_gpa += index * sizeof(pt_element_t); + kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); + } + + walker->pte = pte; + pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)pte); return 1; not_present: @@ -209,8 +183,6 @@ err: walker->error_code |= PFERR_USER_MASK; if (fetch_fault) walker->error_code |= PFERR_FETCH_MASK; - if (walker->table) - kunmap_atomic(walker->table, KM_USER0); return 0; } -- cgit v1.2.3 From b284be576471b1521dd04c0845beedb790cbb9d7 Mon Sep 17 00:00:00 2001 From: Nitin A Kamble Date: Tue, 16 Oct 2007 18:23:27 -0700 Subject: KVM: x86 emulator: cmc, clc, cli, sti Instruction: cmc, clc, cli, sti opcodes: 0xf5, 0xf8, 0xfa, 0xfb respectively. [avi: fix reference to EFLG_IF which is not defined anywhere] Signed-off-by: Nitin A Kamble Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 988c6498640f..e962de331606 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -161,10 +161,10 @@ static u8 opcode_table[256] = { ImplicitOps, SrcImm|ImplicitOps, 0, SrcImmByte|ImplicitOps, 0, 0, 0, 0, /* 0xF0 - 0xF7 */ 0, 0, 0, 0, - ImplicitOps, 0, + ImplicitOps, ImplicitOps, ByteOp | DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM, /* 0xF8 - 0xFF */ - 0, 0, 0, 0, + ImplicitOps, 0, ImplicitOps, ImplicitOps, 0, 0, ByteOp | DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM }; @@ -1476,6 +1476,23 @@ special_insn: case 0xf4: /* hlt */ ctxt->vcpu->halt_request = 1; goto done; + case 0xf5: /* cmc */ + /* complement carry flag from eflags reg */ + ctxt->eflags ^= EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xf8: /* clc */ + ctxt->eflags &= ~EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfa: /* cli */ + ctxt->eflags &= ~X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfb: /* sti */ + ctxt->eflags |= X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; } if (c->rep_prefix) { if (c->regs[VCPU_REGS_RCX] == 0) { -- cgit v1.2.3 From 98348e9507ace5fda95432ff8ca23f13e7f66176 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 16 Oct 2007 14:42:30 +0200 Subject: KVM: MMU: Add rmap_next(), a helper for walking kvm rmaps Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index f52604a7f9e8..14e54e331f50 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -456,28 +456,53 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) } } -static void rmap_write_protect(struct kvm *kvm, u64 gfn) +static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte) { struct kvm_rmap_desc *desc; + struct kvm_rmap_desc *prev_desc; + u64 *prev_spte; + int i; + + if (!*rmapp) + return NULL; + else if (!(*rmapp & 1)) { + if (!spte) + return (u64 *)*rmapp; + return NULL; + } + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); + prev_desc = NULL; + prev_spte = NULL; + while (desc) { + for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) { + if (prev_spte == spte) + return desc->shadow_ptes[i]; + prev_spte = desc->shadow_ptes[i]; + } + desc = desc->more; + } + return NULL; +} + +static void rmap_write_protect(struct kvm *kvm, u64 gfn) +{ unsigned long *rmapp; u64 *spte; + u64 *prev_spte; gfn = unalias_gfn(kvm, gfn); rmapp = gfn_to_rmap(kvm, gfn); - while (*rmapp) { - if (!(*rmapp & 1)) - spte = (u64 *)*rmapp; - else { - desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); - spte = desc->shadow_ptes[0]; - } + spte = rmap_next(kvm, rmapp, NULL); + while (spte) { BUG_ON(!spte); BUG_ON(!(*spte & PT_PRESENT_MASK)); BUG_ON(!(*spte & PT_WRITABLE_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - rmap_remove(kvm, spte); - set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); + prev_spte = spte; + spte = rmap_next(kvm, rmapp, spte); + rmap_remove(kvm, prev_spte); + set_shadow_pte(prev_spte, *prev_spte & ~PT_WRITABLE_MASK); kvm_flush_remote_tlbs(kvm); } } -- cgit v1.2.3 From 9647c14c98687d0abf5197e74b9d1448ab6ebb95 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 16 Oct 2007 14:43:46 +0200 Subject: KVM: MMU: Keep a reverse mapping of non-writable translations The current kvm mmu only reverse maps writable translation. This is used to write-protect a page in case it becomes a pagetable. But with swapping support, we need a reverse mapping of read-only pages as well: when we evict a page, we need to remove any mapping to it, whether writable or not. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 14e54e331f50..bbf5eb427dc6 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -211,8 +211,8 @@ static int is_io_pte(unsigned long pte) static int is_rmap_pte(u64 pte) { - return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK)) - == (PT_WRITABLE_MASK | PT_PRESENT_MASK); + return pte != shadow_trap_nonpresent_pte + && pte != shadow_notrap_nonpresent_pte; } static void set_shadow_pte(u64 *sptep, u64 spte) @@ -488,7 +488,6 @@ static void rmap_write_protect(struct kvm *kvm, u64 gfn) { unsigned long *rmapp; u64 *spte; - u64 *prev_spte; gfn = unalias_gfn(kvm, gfn); rmapp = gfn_to_rmap(kvm, gfn); @@ -497,13 +496,11 @@ static void rmap_write_protect(struct kvm *kvm, u64 gfn) while (spte) { BUG_ON(!spte); BUG_ON(!(*spte & PT_PRESENT_MASK)); - BUG_ON(!(*spte & PT_WRITABLE_MASK)); rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - prev_spte = spte; - spte = rmap_next(kvm, rmapp, spte); - rmap_remove(kvm, prev_spte); - set_shadow_pte(prev_spte, *prev_spte & ~PT_WRITABLE_MASK); + if (is_writeble_pte(*spte)) + set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); kvm_flush_remote_tlbs(kvm); + spte = rmap_next(kvm, rmapp, spte); } } @@ -908,14 +905,18 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) table = __va(table_addr); if (level == 1) { + int was_rmapped; + pte = table[index]; + was_rmapped = is_rmap_pte(pte); if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) return 0; mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; - rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); + if (!was_rmapped) + rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); return 0; } @@ -1424,10 +1425,8 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) pt = page->spt; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ - if (pt[i] & PT_WRITABLE_MASK) { - rmap_remove(kvm, &pt[i]); + if (pt[i] & PT_WRITABLE_MASK) pt[i] &= ~PT_WRITABLE_MASK; - } } } -- cgit v1.2.3 From cea7bb21280e3a825e64b54740edc5d3e6e4193c Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Wed, 17 Oct 2007 19:17:48 +0200 Subject: KVM: MMU: Make gfn_to_page() always safe In case the page is not present in the guest memory map, return a dummy page the guest can scribble on. This simplifies error checking in its users. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 ++- drivers/kvm/kvm_main.c | 26 ++++++++++++++------------ drivers/kvm/mmu.c | 16 +++++----------- drivers/kvm/paging_tmpl.h | 7 ++----- 4 files changed, 23 insertions(+), 29 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 6ae7b6332e32..0c17c76d030f 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -565,8 +565,9 @@ static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva); struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); -extern hpa_t bad_page_address; +extern struct page *bad_page; +int is_error_page(struct page *page); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 1879b409bed2..47000be25479 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -993,6 +993,12 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) return r; } +int is_error_page(struct page *page) +{ + return page == bad_page; +} +EXPORT_SYMBOL_GPL(is_error_page); + gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) { int i; @@ -1034,7 +1040,7 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) gfn = unalias_gfn(kvm, gfn); slot = __gfn_to_memslot(kvm, gfn); if (!slot) - return NULL; + return bad_page; return slot->phys_mem[gfn - slot->base_gfn]; } EXPORT_SYMBOL_GPL(gfn_to_page); @@ -1054,7 +1060,7 @@ int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, struct page *page; page = gfn_to_page(kvm, gfn); - if (!page) + if (is_error_page(page)) return -EFAULT; page_virt = kmap_atomic(page, KM_USER0); @@ -1092,7 +1098,7 @@ int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, struct page *page; page = gfn_to_page(kvm, gfn); - if (!page) + if (is_error_page(page)) return -EFAULT; page_virt = kmap_atomic(page, KM_USER0); @@ -1130,7 +1136,7 @@ int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) struct page *page; page = gfn_to_page(kvm, gfn); - if (!page) + if (is_error_page(page)) return -EFAULT; page_virt = kmap_atomic(page, KM_USER0); @@ -3068,7 +3074,7 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; page = gfn_to_page(kvm, pgoff); - if (!page) + if (is_error_page(page)) return NOPAGE_SIGBUS; get_page(page); if (type != NULL) @@ -3383,7 +3389,7 @@ static struct sys_device kvm_sysdev = { .cls = &kvm_sysdev_class, }; -hpa_t bad_page_address; +struct page *bad_page; static inline struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) @@ -3512,7 +3518,6 @@ EXPORT_SYMBOL_GPL(kvm_exit_x86); static __init int kvm_init(void) { - static struct page *bad_page; int r; r = kvm_mmu_module_init(); @@ -3523,16 +3528,13 @@ static __init int kvm_init(void) kvm_arch_init(); - bad_page = alloc_page(GFP_KERNEL); + bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (bad_page == NULL) { r = -ENOMEM; goto out; } - bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT; - memset(__va(bad_page_address), 0, PAGE_SIZE); - return 0; out: @@ -3545,7 +3547,7 @@ out4: static __exit void kvm_exit(void) { kvm_exit_debug(); - __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT)); + __free_page(bad_page); kvm_mmu_module_exit(); } diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index bbf5eb427dc6..2ad14fbdcfa0 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -850,23 +850,17 @@ static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa) __set_bit(slot, &page_head->slot_bitmap); } -hpa_t safe_gpa_to_hpa(struct kvm *kvm, gpa_t gpa) -{ - hpa_t hpa = gpa_to_hpa(kvm, gpa); - - return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa; -} - hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa) { struct page *page; + hpa_t hpa; ASSERT((gpa & HPA_ERR_MASK) == 0); page = gfn_to_page(kvm, gpa >> PAGE_SHIFT); - if (!page) - return gpa | HPA_ERR_MASK; - return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) - | (gpa & (PAGE_SIZE-1)); + hpa = ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) | (gpa & (PAGE_SIZE-1)); + if (is_error_page(page)) + return hpa | HPA_ERR_MASK; + return hpa; } hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index bab1b7f8d705..572e5b6d9a7a 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -72,8 +72,6 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - hpa_t hpa; - struct kvm_memory_slot *slot; struct page *page; pt_element_t *table; pt_element_t pte; @@ -105,9 +103,8 @@ static int FNAME(walk_addr)(struct guest_walker *walker, pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); - slot = gfn_to_memslot(vcpu->kvm, table_gfn); - hpa = safe_gpa_to_hpa(vcpu->kvm, pte & PT64_BASE_ADDR_MASK); - page = pfn_to_page(hpa >> PAGE_SHIFT); + page = gfn_to_page(vcpu->kvm, (pte & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT); table = kmap_atomic(page, KM_USER0); pte = table[index]; -- cgit v1.2.3 From 8a7ae055f3533b520401c170ac55e30628b34df5 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Thu, 18 Oct 2007 11:09:33 +0200 Subject: KVM: MMU: Partial swapping of guest memory This allows guest memory to be swapped. Pages which are currently mapped via shadow page tables are pinned into memory, but all other pages can be freely swapped. The patch makes gfn_to_page() elevate the page's reference count, and introduces kvm_release_page() that pairs with it. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 ++ drivers/kvm/kvm_main.c | 83 ++++++++++++++++++++++++++--------------------- drivers/kvm/mmu.c | 14 +++++++- drivers/kvm/paging_tmpl.h | 26 +++++++++++++-- 4 files changed, 84 insertions(+), 41 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 0c17c76d030f..df0711ce9baa 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -409,6 +409,7 @@ struct kvm_memory_slot { unsigned long *rmap; unsigned long *dirty_bitmap; int user_alloc; /* user allocated memory */ + unsigned long userspace_addr; }; struct kvm { @@ -570,6 +571,7 @@ extern struct page *bad_page; int is_error_page(struct page *page); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); +void kvm_release_page(struct page *page); int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, int len); int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 47000be25479..f86a47c2f255 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -300,19 +300,6 @@ static struct kvm *kvm_create_vm(void) return kvm; } -static void kvm_free_userspace_physmem(struct kvm_memory_slot *free) -{ - int i; - - for (i = 0; i < free->npages; ++i) { - if (free->phys_mem[i]) { - if (!PageReserved(free->phys_mem[i])) - SetPageDirty(free->phys_mem[i]); - page_cache_release(free->phys_mem[i]); - } - } -} - static void kvm_free_kernel_physmem(struct kvm_memory_slot *free) { int i; @@ -330,9 +317,7 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, { if (!dont || free->phys_mem != dont->phys_mem) if (free->phys_mem) { - if (free->user_alloc) - kvm_free_userspace_physmem(free); - else + if (!free->user_alloc) kvm_free_kernel_physmem(free); vfree(free->phys_mem); } @@ -361,7 +346,7 @@ static void free_pio_guest_pages(struct kvm_vcpu *vcpu) for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) if (vcpu->pio.guest_pages[i]) { - __free_page(vcpu->pio.guest_pages[i]); + kvm_release_page(vcpu->pio.guest_pages[i]); vcpu->pio.guest_pages[i] = NULL; } } @@ -752,19 +737,8 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, memset(new.phys_mem, 0, npages * sizeof(struct page *)); memset(new.rmap, 0, npages * sizeof(*new.rmap)); if (user_alloc) { - unsigned long pages_num; - new.user_alloc = 1; - down_read(¤t->mm->mmap_sem); - - pages_num = get_user_pages(current, current->mm, - mem->userspace_addr, - npages, 1, 1, new.phys_mem, - NULL); - - up_read(¤t->mm->mmap_sem); - if (pages_num != npages) - goto out_unlock; + new.userspace_addr = mem->userspace_addr; } else { for (i = 0; i < npages; ++i) { new.phys_mem[i] = alloc_page(GFP_HIGHUSER @@ -1039,12 +1013,39 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) gfn = unalias_gfn(kvm, gfn); slot = __gfn_to_memslot(kvm, gfn); - if (!slot) + if (!slot) { + get_page(bad_page); return bad_page; + } + if (slot->user_alloc) { + struct page *page[1]; + int npages; + + down_read(¤t->mm->mmap_sem); + npages = get_user_pages(current, current->mm, + slot->userspace_addr + + (gfn - slot->base_gfn) * PAGE_SIZE, 1, + 1, 1, page, NULL); + up_read(¤t->mm->mmap_sem); + if (npages != 1) { + get_page(bad_page); + return bad_page; + } + return page[0]; + } + get_page(slot->phys_mem[gfn - slot->base_gfn]); return slot->phys_mem[gfn - slot->base_gfn]; } EXPORT_SYMBOL_GPL(gfn_to_page); +void kvm_release_page(struct page *page) +{ + if (!PageReserved(page)) + SetPageDirty(page); + put_page(page); +} +EXPORT_SYMBOL_GPL(kvm_release_page); + static int next_segment(unsigned long len, int offset) { if (len > PAGE_SIZE - offset) @@ -1060,13 +1061,16 @@ int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, struct page *page; page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) + if (is_error_page(page)) { + kvm_release_page(page); return -EFAULT; + } page_virt = kmap_atomic(page, KM_USER0); memcpy(data, page_virt + offset, len); kunmap_atomic(page_virt, KM_USER0); + kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_read_guest_page); @@ -1098,14 +1102,17 @@ int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, struct page *page; page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) + if (is_error_page(page)) { + kvm_release_page(page); return -EFAULT; + } page_virt = kmap_atomic(page, KM_USER0); memcpy(page_virt + offset, data, len); kunmap_atomic(page_virt, KM_USER0); mark_page_dirty(kvm, gfn); + kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_write_guest_page); @@ -1136,13 +1143,16 @@ int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) struct page *page; page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) + if (is_error_page(page)) { + kvm_release_page(page); return -EFAULT; + } page_virt = kmap_atomic(page, KM_USER0); memset(page_virt + offset, 0, len); kunmap_atomic(page_virt, KM_USER0); + kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_clear_guest_page); @@ -2070,8 +2080,6 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, for (i = 0; i < nr_pages; ++i) { mutex_lock(&vcpu->kvm->lock); page = gva_to_page(vcpu, address + i * PAGE_SIZE); - if (page) - get_page(page); vcpu->pio.guest_pages[i] = page; mutex_unlock(&vcpu->kvm->lock); if (!page) { @@ -3074,9 +3082,10 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; page = gfn_to_page(kvm, pgoff); - if (is_error_page(page)) + if (is_error_page(page)) { + kvm_release_page(page); return NOPAGE_SIGBUS; - get_page(page); + } if (type != NULL) *type = VM_FAULT_MINOR; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 2ad14fbdcfa0..5d7af4bde595 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -425,6 +425,8 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) if (!is_rmap_pte(*spte)) return; page = page_header(__pa(spte)); + kvm_release_page(pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> + PAGE_SHIFT)); rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]); if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); @@ -911,6 +913,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) PT_USER_MASK; if (!was_rmapped) rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); + else + kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); return 0; } @@ -925,6 +929,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) 1, 3, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); + kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); return -ENOMEM; } @@ -1039,8 +1044,11 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK); - if (is_error_hpa(paddr)) + if (is_error_hpa(paddr)) { + kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); return 1; + } return nonpaging_map(vcpu, addr & PAGE_MASK, paddr); } @@ -1507,6 +1515,7 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, } else { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); hpa_t hpa = gpa_to_hpa(vcpu, gpa); + struct page *page; if (is_shadow_present_pte(ent) && (ent & PT64_BASE_ADDR_MASK) != hpa) @@ -1519,6 +1528,9 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, && !is_error_hpa(hpa)) printk(KERN_ERR "audit: (%s) notrap shadow," " valid guest gva %lx\n", audit_msg, va); + page = pfn_to_page((gpa & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT); + kvm_release_page(page); } } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 572e5b6d9a7a..0f0266af3f68 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -72,7 +72,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - struct page *page; + struct page *page = NULL; pt_element_t *table; pt_element_t pte; gfn_t table_gfn; @@ -149,6 +149,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->inherited_ar &= pte; --walker->level; + kvm_release_page(page); } if (write_fault && !is_dirty_pte(pte)) { @@ -162,6 +163,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); } + kvm_release_page(page); walker->pte = pte; pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)pte); return 1; @@ -180,6 +182,8 @@ err: walker->error_code |= PFERR_USER_MASK; if (fetch_fault) walker->error_code |= PFERR_FETCH_MASK; + if (page) + kvm_release_page(page); return 0; } @@ -223,6 +227,8 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (is_error_hpa(paddr)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); + kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); return; } @@ -260,9 +266,20 @@ unshadowed: pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); - if (!was_rmapped) + if (!was_rmapped) { rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + if (!is_rmap_pte(*shadow_pte)) { + struct page *page; + + page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT); + kvm_release_page(page); + } + } + else + kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } @@ -486,19 +503,22 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, { int i; pt_element_t *gpt; + struct page *page; if (sp->role.metaphysical || PTTYPE == 32) { nonpaging_prefetch_page(vcpu, sp); return; } - gpt = kmap_atomic(gfn_to_page(vcpu->kvm, sp->gfn), KM_USER0); + page = gfn_to_page(vcpu->kvm, sp->gfn); + gpt = kmap_atomic(page, KM_USER0); for (i = 0; i < PT64_ENT_PER_PAGE; ++i) if (is_present_pte(gpt[i])) sp->spt[i] = shadow_trap_nonpresent_pte; else sp->spt[i] = shadow_notrap_nonpresent_pte; kunmap_atomic(gpt, KM_USER0); + kvm_release_page(page); } #undef pt_element_t -- cgit v1.2.3 From e56a7a28e21aa2d1af659d8f38952411ce9ae40e Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Thu, 18 Oct 2007 14:39:10 +0200 Subject: KVM: Use virtual cpu accounting if available for guest times. ppc and s390 offer the possibility to track process times precisely by looking at cpu timer on every context switch, irq, softirq etc. We can use that infrastructure as well for guest time accounting. We need to account the used time before we change the state. This patch adds a call to account_system_vtime to kvm_guest_enter and kvm_guest exit. If CONFIG_VIRT_CPU_ACCOUNTING is not set, account_system_vtime is defined in hardirq.h as an empty function, which means this patch does not change the behaviour on other platforms. I compile tested this patch on x86 and function tested the patch on s390. Signed-off-by: Christian Borntraeger Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index df0711ce9baa..e8a21e8cb2ad 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -671,11 +672,13 @@ __init void kvm_arch_init(void); static inline void kvm_guest_enter(void) { + account_system_vtime(current); current->flags |= PF_VCPU; } static inline void kvm_guest_exit(void) { + account_system_vtime(current); current->flags &= ~PF_VCPU; } -- cgit v1.2.3 From 8d4e1288ebb753d3140d81cb349f22b0a6829a4a Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Thu, 18 Oct 2007 09:59:34 -0500 Subject: KVM: Allocate userspace memory for older userspace Allocate a userspace buffer for older userspaces. Also eliminate phys_mem buffer. The memset() in kvmctl really kills initial memory usage but swapping works even with old userspaces. A side effect is that maximum guest side is reduced for older userspace on i386. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 -- drivers/kvm/kvm_main.c | 83 ++++++++++++++++++-------------------------------- 2 files changed, 30 insertions(+), 55 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index e8a21e8cb2ad..eb006ed696c1 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -406,10 +406,8 @@ struct kvm_memory_slot { gfn_t base_gfn; unsigned long npages; unsigned long flags; - struct page **phys_mem; unsigned long *rmap; unsigned long *dirty_bitmap; - int user_alloc; /* user allocated memory */ unsigned long userspace_addr; }; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index f86a47c2f255..8f6c21d02656 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -300,36 +301,21 @@ static struct kvm *kvm_create_vm(void) return kvm; } -static void kvm_free_kernel_physmem(struct kvm_memory_slot *free) -{ - int i; - - for (i = 0; i < free->npages; ++i) - if (free->phys_mem[i]) - __free_page(free->phys_mem[i]); -} - /* * Free any memory in @free but not in @dont. */ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, struct kvm_memory_slot *dont) { - if (!dont || free->phys_mem != dont->phys_mem) - if (free->phys_mem) { - if (!free->user_alloc) - kvm_free_kernel_physmem(free); - vfree(free->phys_mem); - } if (!dont || free->rmap != dont->rmap) vfree(free->rmap); if (!dont || free->dirty_bitmap != dont->dirty_bitmap) vfree(free->dirty_bitmap); - free->phys_mem = NULL; free->npages = 0; free->dirty_bitmap = NULL; + free->rmap = NULL; } static void kvm_free_physmem(struct kvm *kvm) @@ -712,10 +698,6 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, goto out_unlock; } - /* Deallocate if slot is being removed */ - if (!npages) - new.phys_mem = NULL; - /* Free page dirty bitmap if unneeded */ if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) new.dirty_bitmap = NULL; @@ -723,29 +705,27 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, r = -ENOMEM; /* Allocate if a slot is being created */ - if (npages && !new.phys_mem) { - new.phys_mem = vmalloc(npages * sizeof(struct page *)); - - if (!new.phys_mem) - goto out_unlock; - + if (npages && !new.rmap) { new.rmap = vmalloc(npages * sizeof(struct page *)); if (!new.rmap) goto out_unlock; - memset(new.phys_mem, 0, npages * sizeof(struct page *)); memset(new.rmap, 0, npages * sizeof(*new.rmap)); - if (user_alloc) { - new.user_alloc = 1; + + if (user_alloc) new.userspace_addr = mem->userspace_addr; - } else { - for (i = 0; i < npages; ++i) { - new.phys_mem[i] = alloc_page(GFP_HIGHUSER - | __GFP_ZERO); - if (!new.phys_mem[i]) - goto out_unlock; - } + else { + down_write(¤t->mm->mmap_sem); + new.userspace_addr = do_mmap(NULL, 0, + npages * PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, + 0); + up_write(¤t->mm->mmap_sem); + + if (IS_ERR((void *)new.userspace_addr)) + goto out_unlock; } } @@ -1010,6 +990,8 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; + struct page *page[1]; + int npages; gfn = unalias_gfn(kvm, gfn); slot = __gfn_to_memslot(kvm, gfn); @@ -1017,24 +999,19 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) get_page(bad_page); return bad_page; } - if (slot->user_alloc) { - struct page *page[1]; - int npages; - - down_read(¤t->mm->mmap_sem); - npages = get_user_pages(current, current->mm, - slot->userspace_addr - + (gfn - slot->base_gfn) * PAGE_SIZE, 1, - 1, 1, page, NULL); - up_read(¤t->mm->mmap_sem); - if (npages != 1) { - get_page(bad_page); - return bad_page; - } - return page[0]; + + down_read(¤t->mm->mmap_sem); + npages = get_user_pages(current, current->mm, + slot->userspace_addr + + (gfn - slot->base_gfn) * PAGE_SIZE, 1, + 1, 1, page, NULL); + up_read(¤t->mm->mmap_sem); + if (npages != 1) { + get_page(bad_page); + return bad_page; } - get_page(slot->phys_mem[gfn - slot->base_gfn]); - return slot->phys_mem[gfn - slot->base_gfn]; + + return page[0]; } EXPORT_SYMBOL_GPL(gfn_to_page); -- cgit v1.2.3 From 34c16eecf78ed4cf01f39ac7211f5b57942ec899 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sat, 20 Oct 2007 15:34:38 +0800 Subject: KVM: Portability: Split kvm_vcpu into arch dependent and independent parts (part 1) First step to split kvm_vcpu. Currently, we just use an macro to define the common fields in kvm_vcpu for all archs, and all archs need to define its own kvm_vcpu struct. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 2 + drivers/kvm/irq.c | 1 + drivers/kvm/kvm.h | 154 ++++++++-------------------------------------- drivers/kvm/kvm_main.c | 4 +- drivers/kvm/lapic.c | 2 + drivers/kvm/mmu.c | 1 + drivers/kvm/svm.c | 2 +- drivers/kvm/vmx.c | 1 + drivers/kvm/x86.h | 117 +++++++++++++++++++++++++++++++++++ drivers/kvm/x86_emulate.c | 1 + 10 files changed, 155 insertions(+), 130 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index 8503d99b8339..e14b7c724e67 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -27,6 +27,8 @@ */ #include "kvm.h" +#include "x86.h" + #include #include #include diff --git a/drivers/kvm/irq.c b/drivers/kvm/irq.c index 7628c7ff628f..59b47c55fc76 100644 --- a/drivers/kvm/irq.c +++ b/drivers/kvm/irq.c @@ -22,6 +22,7 @@ #include #include "kvm.h" +#include "x86.h" #include "irq.h" /* diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index eb006ed696c1..db18d278c1c0 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -308,93 +308,37 @@ struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr); void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev); -struct kvm_vcpu { - struct kvm *kvm; - struct preempt_notifier preempt_notifier; - int vcpu_id; - struct mutex mutex; - int cpu; - u64 host_tsc; - struct kvm_run *run; - int interrupt_window_open; - int guest_mode; - unsigned long requests; - unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */ - DECLARE_BITMAP(irq_pending, KVM_NR_INTERRUPTS); - unsigned long regs[NR_VCPU_REGS]; /* for rsp: vcpu_load_rsp_rip() */ - unsigned long rip; /* needs vcpu_load_rsp_rip() */ - - unsigned long cr0; - unsigned long cr2; - unsigned long cr3; - unsigned long cr4; - unsigned long cr8; - u64 pdptrs[4]; /* pae */ - u64 shadow_efer; - u64 apic_base; - struct kvm_lapic *apic; /* kernel irqchip context */ -#define VCPU_MP_STATE_RUNNABLE 0 -#define VCPU_MP_STATE_UNINITIALIZED 1 -#define VCPU_MP_STATE_INIT_RECEIVED 2 -#define VCPU_MP_STATE_SIPI_RECEIVED 3 -#define VCPU_MP_STATE_HALTED 4 - int mp_state; - int sipi_vector; - u64 ia32_misc_enable_msr; - - struct kvm_mmu mmu; - - struct kvm_mmu_memory_cache mmu_pte_chain_cache; - struct kvm_mmu_memory_cache mmu_rmap_desc_cache; - struct kvm_mmu_memory_cache mmu_page_cache; - struct kvm_mmu_memory_cache mmu_page_header_cache; - - gfn_t last_pt_write_gfn; - int last_pt_write_count; - u64 *last_pte_updated; - - struct kvm_guest_debug guest_debug; - - struct i387_fxsave_struct host_fx_image; - struct i387_fxsave_struct guest_fx_image; - int fpu_active; - int guest_fpu_loaded; - - int mmio_needed; - int mmio_read_completed; - int mmio_is_write; - int mmio_size; - unsigned char mmio_data[8]; +#ifdef CONFIG_HAS_IOMEM +#define KVM_VCPU_MMIO \ + int mmio_needed; \ + int mmio_read_completed; \ + int mmio_is_write; \ + int mmio_size; \ + unsigned char mmio_data[8]; \ gpa_t mmio_phys_addr; - gva_t mmio_fault_cr2; - struct kvm_pio_request pio; - void *pio_data; - wait_queue_head_t wq; - int sigset_active; - sigset_t sigset; +#else +#define KVM_VCPU_MMIO - struct kvm_stat stat; +#endif - struct { - int active; - u8 save_iopl; - struct kvm_save_segment { - u16 selector; - unsigned long base; - u32 limit; - u32 ar; - } tr, es, ds, fs, gs; - } rmode; - int halt_request; /* real mode on Intel only */ - - int cpuid_nent; - struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES]; - - /* emulate context */ - - struct x86_emulate_ctxt emulate_ctxt; -}; +#define KVM_VCPU_COMM \ + struct kvm *kvm; \ + struct preempt_notifier preempt_notifier; \ + int vcpu_id; \ + struct mutex mutex; \ + int cpu; \ + struct kvm_run *run; \ + int guest_mode; \ + unsigned long requests; \ + struct kvm_guest_debug guest_debug; \ + int fpu_active; \ + int guest_fpu_loaded; \ + wait_queue_head_t wq; \ + int sigset_active; \ + sigset_t sigset; \ + struct kvm_stat stat; \ + KVM_VCPU_MMIO struct kvm_mem_alias { gfn_t base_gfn; @@ -680,50 +624,6 @@ static inline void kvm_guest_exit(void) current->flags &= ~PF_VCPU; } -static inline int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) -{ - return vcpu->mmu.page_fault(vcpu, gva, error_code); -} - -static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) -{ - if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) - __kvm_mmu_free_some_pages(vcpu); -} - -static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) -{ - if (likely(vcpu->mmu.root_hpa != INVALID_PAGE)) - return 0; - - return kvm_mmu_load(vcpu); -} - -static inline int is_long_mode(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - return vcpu->shadow_efer & EFER_LME; -#else - return 0; -#endif -} - -static inline int is_pae(struct kvm_vcpu *vcpu) -{ - return vcpu->cr4 & X86_CR4_PAE; -} - -static inline int is_pse(struct kvm_vcpu *vcpu) -{ - return vcpu->cr4 & X86_CR4_PSE; -} - -static inline int is_paging(struct kvm_vcpu *vcpu) -{ - return vcpu->cr0 & X86_CR0_PG; -} - static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) { return slot - kvm->memslots; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 8f6c21d02656..0b23657f434c 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -2244,7 +2244,7 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (r) goto out; } - +#if CONFIG_HAS_IOMEM if (vcpu->mmio_needed) { memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); vcpu->mmio_read_completed = 1; @@ -2259,7 +2259,7 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) goto out; } } - +#endif if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { kvm_x86_ops->cache_regs(vcpu); vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 8840f9dc0bca..64f74bd7093a 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -18,6 +18,8 @@ */ #include "kvm.h" +#include "x86.h" + #include #include #include diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 5d7af4bde595..d9c5950cfae1 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -19,6 +19,7 @@ #include "vmx.h" #include "kvm.h" +#include "x86.h" #include #include diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index ef068d2dddd7..035c8e6898f1 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -13,7 +13,7 @@ * the COPYING file in the top-level directory. * */ - +#include "x86.h" #include "kvm_svm.h" #include "x86_emulate.h" #include "irq.h" diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 9f77ddbeb025..87ff35128825 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -16,6 +16,7 @@ */ #include "kvm.h" +#include "x86.h" #include "x86_emulate.h" #include "irq.h" #include "vmx.h" diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 1e2f71bd805d..01452b552db3 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -13,4 +13,121 @@ #include "kvm.h" +#include +#include + +#include +#include + +struct kvm_vcpu { + KVM_VCPU_COMM; + u64 host_tsc; + int interrupt_window_open; + unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */ + DECLARE_BITMAP(irq_pending, KVM_NR_INTERRUPTS); + unsigned long regs[NR_VCPU_REGS]; /* for rsp: vcpu_load_rsp_rip() */ + unsigned long rip; /* needs vcpu_load_rsp_rip() */ + + unsigned long cr0; + unsigned long cr2; + unsigned long cr3; + unsigned long cr4; + unsigned long cr8; + u64 pdptrs[4]; /* pae */ + u64 shadow_efer; + u64 apic_base; + struct kvm_lapic *apic; /* kernel irqchip context */ +#define VCPU_MP_STATE_RUNNABLE 0 +#define VCPU_MP_STATE_UNINITIALIZED 1 +#define VCPU_MP_STATE_INIT_RECEIVED 2 +#define VCPU_MP_STATE_SIPI_RECEIVED 3 +#define VCPU_MP_STATE_HALTED 4 + int mp_state; + int sipi_vector; + u64 ia32_misc_enable_msr; + + struct kvm_mmu mmu; + + struct kvm_mmu_memory_cache mmu_pte_chain_cache; + struct kvm_mmu_memory_cache mmu_rmap_desc_cache; + struct kvm_mmu_memory_cache mmu_page_cache; + struct kvm_mmu_memory_cache mmu_page_header_cache; + + gfn_t last_pt_write_gfn; + int last_pt_write_count; + u64 *last_pte_updated; + + + struct i387_fxsave_struct host_fx_image; + struct i387_fxsave_struct guest_fx_image; + + gva_t mmio_fault_cr2; + struct kvm_pio_request pio; + void *pio_data; + + struct { + int active; + u8 save_iopl; + struct kvm_save_segment { + u16 selector; + unsigned long base; + u32 limit; + u32 ar; + } tr, es, ds, fs, gs; + } rmode; + int halt_request; /* real mode on Intel only */ + + int cpuid_nent; + struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES]; + + /* emulate context */ + + struct x86_emulate_ctxt emulate_ctxt; +}; + +static inline int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, + u32 error_code) +{ + return vcpu->mmu.page_fault(vcpu, gva, error_code); +} + +static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) +{ + if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + __kvm_mmu_free_some_pages(vcpu); +} + +static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) +{ + if (likely(vcpu->mmu.root_hpa != INVALID_PAGE)) + return 0; + + return kvm_mmu_load(vcpu); +} + +static inline int is_long_mode(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + return vcpu->shadow_efer & EFER_LME; +#else + return 0; +#endif +} + +static inline int is_pae(struct kvm_vcpu *vcpu) +{ + return vcpu->cr4 & X86_CR4_PAE; +} + +static inline int is_pse(struct kvm_vcpu *vcpu) +{ + return vcpu->cr4 & X86_CR4_PSE; +} + +static inline int is_paging(struct kvm_vcpu *vcpu) +{ + return vcpu->cr0 & X86_CR0_PG; +} + + #endif diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index e962de331606..73e3580c88e4 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -26,6 +26,7 @@ #define DPRINTF(_f, _a ...) printf(_f , ## _a) #else #include "kvm.h" +#include "x86.h" #define DPRINTF(x...) do {} while (0) #endif #include "x86_emulate.h" -- cgit v1.2.3 From e00c8cf29b9798eb9918469b0cce1766e0ae40d7 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 21 Oct 2007 11:00:39 +0200 Subject: KVM: Move vmx_vcpu_reset() out of vmx_vcpu_setup() Split guest reset code out of vmx_vcpu_setup(). Besides being cleaner, this moves the realmode tss setup (which can sleep) outside vmx_vcpu_setup() (which is executed with preemption enabled). [izik: remove unused variable] Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/kvm_main.c | 8 ++- drivers/kvm/svm.c | 4 +- drivers/kvm/vmx.c | 178 ++++++++++++++++++++++++------------------------- 4 files changed, 99 insertions(+), 93 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index db18d278c1c0..f7181a407be1 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -412,7 +412,7 @@ struct kvm_x86_ops { /* Create, but do not attach this VCPU */ struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); void (*vcpu_free)(struct kvm_vcpu *vcpu); - void (*vcpu_reset)(struct kvm_vcpu *vcpu); + int (*vcpu_reset)(struct kvm_vcpu *vcpu); void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 0b23657f434c..a6e3165043d1 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -2122,7 +2122,9 @@ static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) pr_debug("vcpu %d received sipi with vector # %x\n", vcpu->vcpu_id, vcpu->sipi_vector); kvm_lapic_reset(vcpu); - kvm_x86_ops->vcpu_reset(vcpu); + r = kvm_x86_ops->vcpu_reset(vcpu); + if (r) + return r; vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; } @@ -2637,7 +2639,9 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); vcpu_load(vcpu); - r = kvm_mmu_setup(vcpu); + r = kvm_x86_ops->vcpu_reset(vcpu); + if (r == 0) + r = kvm_mmu_setup(vcpu); vcpu_put(vcpu); if (r < 0) goto free_vcpu; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 035c8e6898f1..953c111544e1 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -557,7 +557,7 @@ static void init_vmcb(struct vmcb *vmcb) /* rdx = ?? */ } -static void svm_vcpu_reset(struct kvm_vcpu *vcpu) +static int svm_vcpu_reset(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -568,6 +568,8 @@ static void svm_vcpu_reset(struct kvm_vcpu *vcpu) svm->vmcb->save.cs.base = svm->vcpu.sipi_vector << 12; svm->vmcb->save.cs.selector = svm->vcpu.sipi_vector << 8; } + + return 0; } static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 87ff35128825..768ea88beb45 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1433,92 +1433,15 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) unsigned long a; struct descriptor_table dt; int i; - int ret = 0; unsigned long kvm_vmx_return; - u64 msr; u32 exec_control; - if (!init_rmode_tss(vmx->vcpu.kvm)) { - ret = -ENOMEM; - goto out; - } - - vmx->vcpu.rmode.active = 0; - - vmx->vcpu.regs[VCPU_REGS_RDX] = get_rdx_init_val(); - set_cr8(&vmx->vcpu, 0); - msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; - if (vmx->vcpu.vcpu_id == 0) - msr |= MSR_IA32_APICBASE_BSP; - kvm_set_apic_base(&vmx->vcpu, msr); - - fx_init(&vmx->vcpu); - - /* - * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode - * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh. - */ - if (vmx->vcpu.vcpu_id == 0) { - vmcs_write16(GUEST_CS_SELECTOR, 0xf000); - vmcs_writel(GUEST_CS_BASE, 0x000f0000); - } else { - vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.sipi_vector << 8); - vmcs_writel(GUEST_CS_BASE, vmx->vcpu.sipi_vector << 12); - } - vmcs_write32(GUEST_CS_LIMIT, 0xffff); - vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); - - seg_setup(VCPU_SREG_DS); - seg_setup(VCPU_SREG_ES); - seg_setup(VCPU_SREG_FS); - seg_setup(VCPU_SREG_GS); - seg_setup(VCPU_SREG_SS); - - vmcs_write16(GUEST_TR_SELECTOR, 0); - vmcs_writel(GUEST_TR_BASE, 0); - vmcs_write32(GUEST_TR_LIMIT, 0xffff); - vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); - - vmcs_write16(GUEST_LDTR_SELECTOR, 0); - vmcs_writel(GUEST_LDTR_BASE, 0); - vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); - vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); - - vmcs_write32(GUEST_SYSENTER_CS, 0); - vmcs_writel(GUEST_SYSENTER_ESP, 0); - vmcs_writel(GUEST_SYSENTER_EIP, 0); - - vmcs_writel(GUEST_RFLAGS, 0x02); - if (vmx->vcpu.vcpu_id == 0) - vmcs_writel(GUEST_RIP, 0xfff0); - else - vmcs_writel(GUEST_RIP, 0); - vmcs_writel(GUEST_RSP, 0); - - /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ - vmcs_writel(GUEST_DR7, 0x400); - - vmcs_writel(GUEST_GDTR_BASE, 0); - vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); - - vmcs_writel(GUEST_IDTR_BASE, 0); - vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); - - vmcs_write32(GUEST_ACTIVITY_STATE, 0); - vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); - vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); - /* I/O */ vmcs_write64(IO_BITMAP_A, page_to_phys(vmx_io_bitmap_a)); vmcs_write64(IO_BITMAP_B, page_to_phys(vmx_io_bitmap_b)); - guest_write_tsc(0); - vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */ - /* Special registers */ - vmcs_write64(GUEST_IA32_DEBUGCTL, 0); - /* Control */ vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, vmcs_config.pin_based_exec_ctrl); @@ -1593,13 +1516,100 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) ++vmx->nmsrs; } - setup_msrs(vmx); - vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl); /* 22.2.1, 20.8.1 */ vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl); + vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); + vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); + + return 0; +} + +static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u64 msr; + int ret; + + if (!init_rmode_tss(vmx->vcpu.kvm)) { + ret = -ENOMEM; + goto out; + } + + vmx->vcpu.rmode.active = 0; + + vmx->vcpu.regs[VCPU_REGS_RDX] = get_rdx_init_val(); + set_cr8(&vmx->vcpu, 0); + msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; + if (vmx->vcpu.vcpu_id == 0) + msr |= MSR_IA32_APICBASE_BSP; + kvm_set_apic_base(&vmx->vcpu, msr); + + fx_init(&vmx->vcpu); + + /* + * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode + * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh. + */ + if (vmx->vcpu.vcpu_id == 0) { + vmcs_write16(GUEST_CS_SELECTOR, 0xf000); + vmcs_writel(GUEST_CS_BASE, 0x000f0000); + } else { + vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.sipi_vector << 8); + vmcs_writel(GUEST_CS_BASE, vmx->vcpu.sipi_vector << 12); + } + vmcs_write32(GUEST_CS_LIMIT, 0xffff); + vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); + + seg_setup(VCPU_SREG_DS); + seg_setup(VCPU_SREG_ES); + seg_setup(VCPU_SREG_FS); + seg_setup(VCPU_SREG_GS); + seg_setup(VCPU_SREG_SS); + + vmcs_write16(GUEST_TR_SELECTOR, 0); + vmcs_writel(GUEST_TR_BASE, 0); + vmcs_write32(GUEST_TR_LIMIT, 0xffff); + vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); + + vmcs_write16(GUEST_LDTR_SELECTOR, 0); + vmcs_writel(GUEST_LDTR_BASE, 0); + vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); + vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); + + vmcs_write32(GUEST_SYSENTER_CS, 0); + vmcs_writel(GUEST_SYSENTER_ESP, 0); + vmcs_writel(GUEST_SYSENTER_EIP, 0); + + vmcs_writel(GUEST_RFLAGS, 0x02); + if (vmx->vcpu.vcpu_id == 0) + vmcs_writel(GUEST_RIP, 0xfff0); + else + vmcs_writel(GUEST_RIP, 0); + vmcs_writel(GUEST_RSP, 0); + + /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ + vmcs_writel(GUEST_DR7, 0x400); + + vmcs_writel(GUEST_GDTR_BASE, 0); + vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); + + vmcs_writel(GUEST_IDTR_BASE, 0); + vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); + + vmcs_write32(GUEST_ACTIVITY_STATE, 0); + vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); + vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); + + guest_write_tsc(0); + + /* Special registers */ + vmcs_write64(GUEST_IA32_DEBUGCTL, 0); + + setup_msrs(vmx); + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ #ifdef CONFIG_X86_64 @@ -1610,9 +1620,6 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_write32(TPR_THRESHOLD, 0); #endif - vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); - vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); - vmx->vcpu.cr0 = 0x60000010; vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); /* enter rmode */ vmx_set_cr4(&vmx->vcpu, 0); @@ -1628,13 +1635,6 @@ out: return ret; } -static void vmx_vcpu_reset(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - vmx_vcpu_setup(vmx); -} - static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq) { u16 ent[2]; -- cgit v1.2.3 From 60395224d94945544f1f9dce5566981844bf0e77 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 21 Oct 2007 11:03:36 +0200 Subject: KVM: Add a might_sleep() annotation to gfn_to_page() This will help trap accesses to guest memory in atomic context. Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index a6e3165043d1..453e98e251da 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -993,6 +993,8 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) struct page *page[1]; int npages; + might_sleep(); + gfn = unalias_gfn(kvm, gfn); slot = __gfn_to_memslot(kvm, gfn); if (!slot) { -- cgit v1.2.3 From 2fcceae145645ec6a4fab0a806bb674d7484d97b Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Wed, 10 Oct 2007 12:14:25 +0200 Subject: KVM: Export PIC reset for kernel device reset Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- drivers/kvm/i8259.c | 6 ++---- drivers/kvm/irq.h | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/i8259.c b/drivers/kvm/i8259.c index a679157bc599..f0dc2eeadf0b 100644 --- a/drivers/kvm/i8259.c +++ b/drivers/kvm/i8259.c @@ -181,10 +181,8 @@ int kvm_pic_read_irq(struct kvm_pic *s) return intno; } -static void pic_reset(void *opaque) +void kvm_pic_reset(struct kvm_kpic_state *s) { - struct kvm_kpic_state *s = opaque; - s->last_irr = 0; s->irr = 0; s->imr = 0; @@ -209,7 +207,7 @@ static void pic_ioport_write(void *opaque, u32 addr, u32 val) addr &= 1; if (addr == 0) { if (val & 0x10) { - pic_reset(s); /* init */ + kvm_pic_reset(s); /* init */ /* * deassert a pending interrupt */ diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 508280ea6fcf..e08ae1002bd8 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -139,6 +139,7 @@ int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); int kvm_create_lapic(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu); +void kvm_pic_reset(struct kvm_kpic_state *s); void kvm_free_lapic(struct kvm_vcpu *vcpu); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); -- cgit v1.2.3 From 8c392696e749171531d155ea5cefdfc1c970fd18 Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Wed, 10 Oct 2007 12:15:54 +0200 Subject: KVM: Split IOAPIC reset function and export for kernel RESET Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 17 +++++++++++++---- drivers/kvm/irq.h | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index e14b7c724e67..cf1d50b3549b 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -371,18 +371,27 @@ static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, } } +void kvm_ioapic_reset(struct kvm_ioapic *ioapic) +{ + int i; + + for (i = 0; i < IOAPIC_NUM_PINS; i++) + ioapic->redirtbl[i].fields.mask = 1; + ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS; + ioapic->ioregsel = 0; + ioapic->irr = 0; + ioapic->id = 0; +} + int kvm_ioapic_init(struct kvm *kvm) { struct kvm_ioapic *ioapic; - int i; ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL); if (!ioapic) return -ENOMEM; kvm->vioapic = ioapic; - for (i = 0; i < IOAPIC_NUM_PINS; i++) - ioapic->redirtbl[i].fields.mask = 1; - ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS; + kvm_ioapic_reset(ioapic); ioapic->dev.read = ioapic_mmio_read; ioapic->dev.write = ioapic_mmio_write; ioapic->dev.in_range = ioapic_in_range; diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index e08ae1002bd8..5ad3cfd7622d 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -140,6 +140,7 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); int kvm_create_lapic(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu); void kvm_pic_reset(struct kvm_kpic_state *s); +void kvm_ioapic_reset(struct kvm_ioapic *ioapic); void kvm_free_lapic(struct kvm_vcpu *vcpu); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); -- cgit v1.2.3 From 80b14b5b32cb0a98234283daf9b5a6643e1a1ef3 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Thu, 25 Oct 2007 11:54:04 +0200 Subject: KVM: Unmap kernel-allocated memory on slot destruction kvm_vm_ioctl_set_memory_region() is able to remove memory in addition to adding it. Therefore when using kernel swapping support for old userspaces, we need to munmap the memory if the user request to remove it Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index f7181a407be1..12de42c789a2 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -353,6 +353,7 @@ struct kvm_memory_slot { unsigned long *rmap; unsigned long *dirty_bitmap; unsigned long userspace_addr; + int user_alloc; }; struct kvm { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 453e98e251da..ab2c77c9ecef 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -713,6 +713,7 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, memset(new.rmap, 0, npages * sizeof(*new.rmap)); + new.user_alloc = user_alloc; if (user_alloc) new.userspace_addr = mem->userspace_addr; else { @@ -727,6 +728,19 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, if (IS_ERR((void *)new.userspace_addr)) goto out_unlock; } + } else { + if (!old.user_alloc && old.rmap) { + int ret; + + down_write(¤t->mm->mmap_sem); + ret = do_munmap(current->mm, old.userspace_addr, + old.npages * PAGE_SIZE); + up_write(¤t->mm->mmap_sem); + if (ret < 0) + printk(KERN_WARNING + "kvm_vm_ioctl_set_memory_region: " + "failed to munmap memory\n"); + } } /* Allocate page dirty bitmap if needed */ -- cgit v1.2.3 From 210c7c4d7f200f00ec34960a8c96bb990cbd266d Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Wed, 24 Oct 2007 23:52:57 +0200 Subject: KVM: Export memory slot allocation mechanism Remove kvm memory slot allocation mechanism from the ioctl and put it to exported function. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ drivers/kvm/kvm_main.c | 17 +++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 12de42c789a2..f3dda088e34b 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -513,6 +513,9 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); extern struct page *bad_page; int is_error_page(struct page *page); +int kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); void kvm_release_page(struct page *page); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index ab2c77c9ecef..5113cbf75674 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -643,10 +643,9 @@ EXPORT_SYMBOL_GPL(fx_init); * * Discontiguous memory is allowed, mostly for framebuffers. */ -static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct - kvm_userspace_memory_region *mem, - int user_alloc) +int kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc) { int r; gfn_t base_gfn; @@ -789,6 +788,16 @@ out_unlock: kvm_free_physmem_slot(&new, &old); out: return r; + +} +EXPORT_SYMBOL_GPL(kvm_set_memory_region); + +static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, + struct + kvm_userspace_memory_region *mem, + int user_alloc) +{ + return kvm_set_memory_region(kvm, mem, user_alloc); } static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, -- cgit v1.2.3 From e0d62c7f48605119a7f9fa632e77561c89928963 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Wed, 24 Oct 2007 23:57:46 +0200 Subject: KVM: Add kernel-internal memory slots Reserve a few memory slots for kernel internal use. This is good for case you have to register memory region and you want to be sure it was not registered from userspace, and for case you want to register a memory region that won't be seen from userspace. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 6 +++++- drivers/kvm/kvm_main.c | 22 +++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index f3dda088e34b..82c3b13e5d0a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -41,6 +41,8 @@ #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 +/* memory slots that does not exposed to userspace */ +#define KVM_PRIVATE_MEM_SLOTS 4 #define KVM_PERMILLE_MMU_PAGES 20 #define KVM_MIN_ALLOC_MMU_PAGES 64 #define KVM_NUM_MMU_PAGES 1024 @@ -361,7 +363,8 @@ struct kvm { int naliases; struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; int nmemslots; - struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS]; + struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + + KVM_PRIVATE_MEM_SLOTS]; /* * Hash table of struct kvm_mmu_page. */ @@ -529,6 +532,7 @@ int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len); int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); +int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); void mark_page_dirty(struct kvm *kvm, gfn_t gfn); enum emulation_result { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 5113cbf75674..a044856bd02b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -660,7 +660,7 @@ int kvm_set_memory_region(struct kvm *kvm, goto out; if (mem->guest_phys_addr & (PAGE_SIZE - 1)) goto out; - if (mem->slot >= KVM_MEMORY_SLOTS) + if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS) goto out; if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) goto out; @@ -797,6 +797,8 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, kvm_userspace_memory_region *mem, int user_alloc) { + if (mem->slot >= KVM_MEMORY_SLOTS) + return -EINVAL; return kvm_set_memory_region(kvm, mem, user_alloc); } @@ -1010,6 +1012,22 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) return __gfn_to_memslot(kvm, gfn); } +int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) +{ + int i; + + gfn = unalias_gfn(kvm, gfn); + for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { + struct kvm_memory_slot *memslot = &kvm->memslots[i]; + + if (gfn >= memslot->base_gfn + && gfn < memslot->base_gfn + memslot->npages) + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); + struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; @@ -3087,6 +3105,8 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, struct page *page; pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + if (!kvm_is_visible_gfn(kvm, pgoff)) + return NOPAGE_SIGBUS; page = gfn_to_page(kvm, pgoff); if (is_error_page(page)) { kvm_release_page(page); -- cgit v1.2.3 From cbc9402297b9a233981f74587786364cda21c771 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Thu, 25 Oct 2007 00:29:55 +0200 Subject: KVM: Add ioctl to tss address from userspace, Currently kvm has a wart in that it requires three extra pages for use as a tss when emulating real mode on Intel. This patch moves the allocation internally, only requiring userspace to tell us where in the physical address space we can place the tss. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ drivers/kvm/kvm_main.c | 16 ++++++++++++++++ drivers/kvm/svm.c | 7 +++++++ drivers/kvm/vmx.c | 27 +++++++++++++++++++++++++-- include/linux/kvm.h | 2 ++ 5 files changed, 53 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 82c3b13e5d0a..3d07d9b1b815 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -382,6 +382,7 @@ struct kvm { struct kvm_pic *vpic; struct kvm_ioapic *vioapic; int round_robin_prev_vcpu; + unsigned int tss_addr; }; static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) @@ -467,6 +468,8 @@ struct kvm_x86_ops { void (*inject_pending_irq)(struct kvm_vcpu *vcpu); void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, struct kvm_run *run); + + int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); }; extern struct kvm_x86_ops *kvm_x86_ops; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index a044856bd02b..50fd040b9ea9 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -2661,6 +2661,16 @@ static int create_vcpu_fd(struct kvm_vcpu *vcpu) return fd; } +static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) +{ + int ret; + + if (addr > (unsigned int)(-3 * PAGE_SIZE)) + return -1; + ret = kvm_x86_ops->set_tss_addr(kvm, addr); + return ret; +} + /* * Creates some virtual cpus. Good luck creating more than one. */ @@ -2957,6 +2967,11 @@ static long kvm_vm_ioctl(struct file *filp, int r = -EINVAL; switch (ioctl) { + case KVM_SET_TSS_ADDR: + r = kvm_vm_ioctl_set_tss_addr(kvm, arg); + if (r < 0) + goto out; + break; case KVM_CREATE_VCPU: r = kvm_vm_ioctl_create_vcpu(kvm, arg); if (r < 0) @@ -3183,6 +3198,7 @@ static long kvm_dev_ioctl(struct file *filp, case KVM_CAP_HLT: case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: case KVM_CAP_USER_MEMORY: + case KVM_CAP_SET_TSS_ADDR: r = 1; break; default: diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 953c111544e1..beb1e684b68a 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1439,6 +1439,11 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, control->intercept &= ~(1ULL << INTERCEPT_VINTR); } +static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) +{ + return 0; +} + static void save_db_regs(unsigned long *db_regs) { asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0])); @@ -1744,6 +1749,8 @@ static struct kvm_x86_ops svm_x86_ops = { .set_irq = svm_set_irq, .inject_pending_irq = svm_intr_assist, .inject_pending_vectors = do_interrupt_requests, + + .set_tss_addr = svm_set_tss_addr, }; static int __init svm_init(void) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 768ea88beb45..46b29184a4d8 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1105,8 +1105,12 @@ static void enter_pmode(struct kvm_vcpu *vcpu) static gva_t rmode_tss_base(struct kvm *kvm) { - gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3; - return base_gfn << PAGE_SHIFT; + if (!kvm->tss_addr) { + gfn_t base_gfn = kvm->memslots[0].base_gfn + + kvm->memslots[0].npages - 3; + return base_gfn << PAGE_SHIFT; + } + return kvm->tss_addr; } static void fix_rmode_seg(int seg, struct kvm_save_segment *save) @@ -1735,6 +1739,23 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); } +static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) +{ + int ret; + struct kvm_userspace_memory_region tss_mem = { + .slot = 8, + .guest_phys_addr = addr, + .memory_size = PAGE_SIZE * 3, + .flags = 0, + }; + + ret = kvm_set_memory_region(kvm, &tss_mem, 0); + if (ret) + return ret; + kvm->tss_addr = addr; + return 0; +} + static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu) { struct kvm_guest_debug *dbg = &vcpu->guest_debug; @@ -2543,6 +2564,8 @@ static struct kvm_x86_ops vmx_x86_ops = { .set_irq = vmx_inject_irq, .inject_pending_irq = vmx_intr_assist, .inject_pending_vectors = do_interrupt_requests, + + .set_tss_addr = vmx_set_tss_addr, }; static int __init vmx_init(void) diff --git a/include/linux/kvm.h b/include/linux/kvm.h index 971f465e26fa..71d33d62944f 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -358,6 +358,7 @@ struct kvm_signal_mask { #define KVM_CAP_HLT 1 #define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2 #define KVM_CAP_USER_MEMORY 3 +#define KVM_CAP_SET_TSS_ADDR 4 /* * ioctls for VM fds @@ -367,6 +368,7 @@ struct kvm_signal_mask { #define KVM_GET_NR_MMU_PAGES _IO(KVMIO, 0x45) #define KVM_SET_USER_MEMORY_REGION _IOW(KVMIO, 0x46,\ struct kvm_userspace_memory_region) +#define KVM_SET_TSS_ADDR _IO(KVMIO, 0x47) /* * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns * a vcpu fd. -- cgit v1.2.3 From c20363006af64cf397519da5e984b18e6bdffd82 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Thu, 25 Oct 2007 14:18:52 +0200 Subject: KVM: VMX: Let gcc to choose which registers to save (x86_64) This patch lets GCC to determine which registers to save when we switch to/from a VCPU in the case of intel x86_64. * Original code saves following registers: rax, rbx, rcx, rdx, rsi, rdi, rbp, r8, r9, r10, r11, r12, r13, r14, r15 * Patched code: - informs GCC that we modify following registers using the clobber description: rbx, rdi, rsi, r8, r9, r10, r11, r12, r13, r14, r15 - doesn't save rax because it is an output operand (vmx->fail) - cannot put rcx in clobber description because it is an input operand, but as we modify it and we want to keep its value (vcpu), we must save it (pop/push) - rbp is saved (pop/push) because GCC seems to ignore its use in the clobber description. - rdx is saved (pop/push) because it is reserved by GCC (REGPARM) and cannot be put in the clobber description. - line "mov (%%rsp), %3 \n\t" has been removed because %3 is rcx and rcx is restored just after. - line ASM_VMX_VMWRITE_RSP_RDX() is moved out of the ifdef/else/endif Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 46b29184a4d8..56c9bcc82836 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2265,16 +2265,12 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) asm( /* Store host registers */ #ifdef CONFIG_X86_64 - "push %%rax; push %%rbx; push %%rdx;" - "push %%rsi; push %%rdi; push %%rbp;" - "push %%r8; push %%r9; push %%r10; push %%r11;" - "push %%r12; push %%r13; push %%r14; push %%r15;" + "push %%rdx; push %%rbp;" "push %%rcx \n\t" - ASM_VMX_VMWRITE_RSP_RDX "\n\t" #else "pusha; push %%ecx \n\t" - ASM_VMX_VMWRITE_RSP_RDX "\n\t" #endif + ASM_VMX_VMWRITE_RSP_RDX "\n\t" /* Check if vmlaunch of vmresume is needed */ "cmp $0, %1 \n\t" /* Load guest registers. Don't clobber flags. */ @@ -2333,12 +2329,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "mov %%r15, %c[r15](%3) \n\t" "mov %%cr2, %%rax \n\t" "mov %%rax, %c[cr2](%3) \n\t" - "mov (%%rsp), %3 \n\t" - "pop %%rcx; pop %%r15; pop %%r14; pop %%r13; pop %%r12;" - "pop %%r11; pop %%r10; pop %%r9; pop %%r8;" - "pop %%rbp; pop %%rdi; pop %%rsi;" - "pop %%rdx; pop %%rbx; pop %%rax \n\t" + "pop %%rcx; pop %%rbp; pop %%rdx \n\t" #else "xchg %3, (%%esp) \n\t" "mov %%eax, %c[rax](%3) \n\t" @@ -2376,7 +2368,12 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])), #endif [cr2]"i"(offsetof(struct kvm_vcpu, cr2)) - : "cc", "memory"); + : "cc", "memory" +#ifdef CONFIG_X86_64 + , "rbx", "rdi", "rsi" + , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" +#endif + ); vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; -- cgit v1.2.3 From ff593e5abebd899b0b41c11280e2fbeff4103375 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Thu, 25 Oct 2007 14:18:55 +0200 Subject: KVM: VMX: Let gcc to choose which registers to save (i386) This patch lets GCC to determine which registers to save when we switch to/from a VCPU in the case of intel i386. * Original code saves following registers: eax, ebx, ecx, edx, edi, esi, ebp (using popa) * Patched code: - informs GCC that we modify following registers using the clobber description: ebx, edi, rsi - doesn't save eax because it is an output operand (vmx->fail) - cannot put ecx in clobber description because it is an input operand, but as we modify it and we want to keep its value (vcpu), we must save it (pop/push) - ebp is saved (pop/push) because GCC seems to ignore its use the clobber description. - edx is saved (pop/push) because it is reserved by GCC (REGPARM) and cannot be put in the clobber description. - line "mov (%%esp), %3 \n\t" has been removed because %3 is ecx and ecx is restored just after. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 56c9bcc82836..2d7d638d72d0 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2268,7 +2268,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "push %%rdx; push %%rbp;" "push %%rcx \n\t" #else - "pusha; push %%ecx \n\t" + "push %%edx; push %%ebp;" + "push %%ecx \n\t" #endif ASM_VMX_VMWRITE_RSP_RDX "\n\t" /* Check if vmlaunch of vmresume is needed */ @@ -2342,9 +2343,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "mov %%ebp, %c[rbp](%3) \n\t" "mov %%cr2, %%eax \n\t" "mov %%eax, %c[cr2](%3) \n\t" - "mov (%%esp), %3 \n\t" - "pop %%ecx; popa \n\t" + "pop %%ecx; pop %%ebp; pop %%edx \n\t" #endif "setbe %0 \n\t" : "=q" (vmx->fail) @@ -2372,6 +2372,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #ifdef CONFIG_X86_64 , "rbx", "rdi", "rsi" , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" +#else + , "ebx", "edi", "rsi" #endif ); -- cgit v1.2.3 From 54a08c0449683f763dc69e653d17e8c120e16299 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Thu, 25 Oct 2007 14:18:53 +0200 Subject: KVM: SVM: Let gcc to choose which registers to save (x86_64) This patch lets GCC to determine which registers to save when we switch to/from a VCPU in the case of AMD x86_64. * Original code saves following registers: rbx, rcx, rdx, rsi, rdi, rbp, r8, r9, r10, r11, r12, r13, r14, r15 * Patched code: - informs GCC that we modify following registers using the clobber description: rbx, rcx, rdx, rsi, rdi r8, r9, r10, r11, r12, r13, r14, r15 - rbp is saved (pop/push) because GCC seems to ignore its use in the clobber description. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index beb1e684b68a..debd0912a016 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1499,10 +1499,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) asm volatile ( #ifdef CONFIG_X86_64 - "push %%rbx; push %%rcx; push %%rdx;" - "push %%rsi; push %%rdi; push %%rbp;" - "push %%r8; push %%r9; push %%r10; push %%r11;" - "push %%r12; push %%r13; push %%r14; push %%r15;" + "push %%rbp; \n\t" #else "push %%ebx; push %%ecx; push %%edx;" "push %%esi; push %%edi; push %%ebp;" @@ -1567,10 +1564,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "mov %%r14, %c[r14](%[svm]) \n\t" "mov %%r15, %c[r15](%[svm]) \n\t" - "pop %%r15; pop %%r14; pop %%r13; pop %%r12;" - "pop %%r11; pop %%r10; pop %%r9; pop %%r8;" - "pop %%rbp; pop %%rdi; pop %%rsi;" - "pop %%rdx; pop %%rcx; pop %%rbx; \n\t" + "pop %%rbp; \n\t" #else "mov %%ebx, %c[rbx](%[svm]) \n\t" "mov %%ecx, %c[rcx](%[svm]) \n\t" @@ -1601,7 +1595,12 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) [r14]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R14])), [r15]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R15])) #endif - : "cc", "memory"); + : "cc", "memory" +#ifdef CONFIG_X86_64 + , "rbx", "rcx", "rdx", "rsi", "rdi" + , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" +#endif + ); if ((svm->vmcb->save.dr7 & 0xff)) load_db_regs(svm->host_db_regs); -- cgit v1.2.3 From fe7935d49fbe33308c1b5f0e35137989da851010 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Thu, 25 Oct 2007 14:18:54 +0200 Subject: KVM: SVM: Let gcc to choose which registers to save (i386) This patch lets GCC to determine which registers to save when we switch to/from a VCPU in the case of AMD i386 * Original code saves following registers: ebx, ecx, edx, esi, edi, ebp * Patched code: - informs GCC that we modify following registers using the clobber description: ebx, ecx, edx, esi, edi - rbp is saved (pop/push) because GCC seems to ignore its use in the clobber description. Signed-off-by: Laurent Vivier Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index debd0912a016..fe2a5e8b0524 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1501,8 +1501,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #ifdef CONFIG_X86_64 "push %%rbp; \n\t" #else - "push %%ebx; push %%ecx; push %%edx;" - "push %%esi; push %%edi; push %%ebp;" + "push %%ebp; \n\t" #endif #ifdef CONFIG_X86_64 @@ -1573,8 +1572,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) "mov %%edi, %c[rdi](%[svm]) \n\t" "mov %%ebp, %c[rbp](%[svm]) \n\t" - "pop %%ebp; pop %%edi; pop %%esi;" - "pop %%edx; pop %%ecx; pop %%ebx; \n\t" + "pop %%ebp; \n\t" #endif : : [svm]"a"(svm), @@ -1599,6 +1597,8 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #ifdef CONFIG_X86_64 , "rbx", "rcx", "rdx", "rsi", "rdi" , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" +#else + , "ebx", "ecx", "edx" , "esi", "edi" #endif ); -- cgit v1.2.3 From c7e75a3db4ecd952e7a5562cea1b27007bf0c01c Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 28 Oct 2007 16:34:25 +0200 Subject: KVM: x86 emulator: don't depend on cr2 for mov abs emulation The 'mov abs' instruction family (opcodes 0xa0 - 0xa3) still depends on cr2 provided by the page fault handler. This is wrong for several reasons: - if an instruction accessed misaligned data that crosses a page boundary, and if the fault happened on the second page, cr2 will point at the second page, not the data itself. - if we're emulating in real mode, or due to a FlexPriority exit, there is no cr2 generated. So, this change adds decoding for this instruction form and drops reliance on cr2. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 50 ++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 20 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 73e3580c88e4..087a8208f873 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -63,8 +63,9 @@ /* Destination is only written; never read. */ #define Mov (1<<7) #define BitOp (1<<8) +#define MemAbs (1<<9) /* Memory operand is absolute displacement */ -static u8 opcode_table[256] = { +static u16 opcode_table[256] = { /* 0x00 - 0x07 */ ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, @@ -134,8 +135,8 @@ static u8 opcode_table[256] = { /* 0x90 - 0x9F */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps, ImplicitOps, 0, 0, /* 0xA0 - 0xA7 */ - ByteOp | DstReg | SrcMem | Mov, DstReg | SrcMem | Mov, - ByteOp | DstMem | SrcReg | Mov, DstMem | SrcReg | Mov, + ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, + ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs, ByteOp | ImplicitOps | Mov, ImplicitOps | Mov, ByteOp | ImplicitOps, ImplicitOps, /* 0xA8 - 0xAF */ @@ -755,16 +756,6 @@ done_prefixes: break; } } - if (!c->override_base) - c->override_base = &ctxt->ds_base; - if (mode == X86EMUL_MODE_PROT64 && - c->override_base != &ctxt->fs_base && - c->override_base != &ctxt->gs_base) - c->override_base = NULL; - - if (c->override_base) - c->modrm_ea += *c->override_base; - if (rip_relative) { c->modrm_ea += c->eip; switch (c->d & SrcMask) { @@ -781,12 +772,35 @@ done_prefixes: c->modrm_ea += c->op_bytes; } } - if (c->ad_bytes != 8) - c->modrm_ea = (u32)c->modrm_ea; modrm_done: ; + } else if (c->d & MemAbs) { + switch (c->ad_bytes) { + case 2: + c->modrm_ea = insn_fetch(u16, 2, c->eip); + break; + case 4: + c->modrm_ea = insn_fetch(u32, 4, c->eip); + break; + case 8: + c->modrm_ea = insn_fetch(u64, 8, c->eip); + break; + } + } + if (!c->override_base) + c->override_base = &ctxt->ds_base; + if (mode == X86EMUL_MODE_PROT64 && + c->override_base != &ctxt->fs_base && + c->override_base != &ctxt->gs_base) + c->override_base = NULL; + + if (c->override_base) + c->modrm_ea += *c->override_base; + + if (c->ad_bytes != 8) + c->modrm_ea = (u32)c->modrm_ea; /* * Decode and fetch the source operand: register, memory * or immediate. @@ -1171,7 +1185,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); saved_eip = c->eip; - if ((c->d & ModRM) && (c->modrm_mod != 3)) + if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) cr2 = c->modrm_ea; if (c->src.type == OP_MEM) { @@ -1326,13 +1340,9 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) case 0xa0 ... 0xa1: /* mov */ c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; c->dst.val = c->src.val; - /* skip src displacement */ - c->eip += c->ad_bytes; break; case 0xa2 ... 0xa3: /* mov */ c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; - /* skip c->dst displacement */ - c->eip += c->ad_bytes; break; case 0xc0 ... 0xc1: emulate_grp2(ctxt); -- cgit v1.2.3 From 3067714cf59bd4a6dbf788b709485bc62c1ff845 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 28 Oct 2007 18:48:59 +0200 Subject: KVM: Move page fault processing to common code The code that dispatches the page fault and emulates if we failed to map is duplicated across vmx and svm. Merge it to simplify further bugfixing. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 36 ++++++++++++++++++++++++++++++++++++ drivers/kvm/svm.c | 33 +-------------------------------- drivers/kvm/vmx.c | 29 +---------------------------- drivers/kvm/x86.h | 6 +----- 4 files changed, 39 insertions(+), 65 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index d9c5950cfae1..ace3cb86214b 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1347,6 +1347,42 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) } } +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) +{ + int r; + enum emulation_result er; + + mutex_lock(&vcpu->kvm->lock); + r = vcpu->mmu.page_fault(vcpu, cr2, error_code); + if (r < 0) + goto out; + + if (!r) { + r = 1; + goto out; + } + + er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0); + mutex_unlock(&vcpu->kvm->lock); + + switch (er) { + case EMULATE_DONE: + return 1; + case EMULATE_DO_MMIO: + ++vcpu->stat.mmio_exits; + return 0; + case EMULATE_FAIL: + kvm_report_emulation_failure(vcpu, "pagetable"); + return 1; + default: + BUG(); + } +out: + mutex_unlock(&vcpu->kvm->lock); + return r; +} +EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); + static void free_mmu_pages(struct kvm_vcpu *vcpu) { struct kvm_mmu_page *page; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index fe2a5e8b0524..97863f8fd001 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -933,45 +933,14 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) struct kvm *kvm = svm->vcpu.kvm; u64 fault_address; u32 error_code; - enum emulation_result er; - int r; if (!irqchip_in_kernel(kvm) && is_external_interrupt(exit_int_info)) push_irq(&svm->vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK); - mutex_lock(&kvm->lock); - fault_address = svm->vmcb->control.exit_info_2; error_code = svm->vmcb->control.exit_info_1; - r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); - if (r < 0) { - mutex_unlock(&kvm->lock); - return r; - } - if (!r) { - mutex_unlock(&kvm->lock); - return 1; - } - er = emulate_instruction(&svm->vcpu, kvm_run, fault_address, - error_code, 0); - mutex_unlock(&kvm->lock); - - switch (er) { - case EMULATE_DONE: - return 1; - case EMULATE_DO_MMIO: - ++svm->vcpu.stat.mmio_exits; - return 0; - case EMULATE_FAIL: - kvm_report_emulation_failure(&svm->vcpu, "pagetable"); - break; - default: - BUG(); - } - - kvm_run->exit_reason = KVM_EXIT_UNKNOWN; - return 0; + return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); } static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 2d7d638d72d0..7fe834cb0d81 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1796,7 +1796,6 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) unsigned long cr2, rip; u32 vect_info; enum emulation_result er; - int r; vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); intr_info = vmcs_read32(VM_EXIT_INTR_INFO); @@ -1834,33 +1833,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); if (is_page_fault(intr_info)) { cr2 = vmcs_readl(EXIT_QUALIFICATION); - - mutex_lock(&vcpu->kvm->lock); - r = kvm_mmu_page_fault(vcpu, cr2, error_code); - if (r < 0) { - mutex_unlock(&vcpu->kvm->lock); - return r; - } - if (!r) { - mutex_unlock(&vcpu->kvm->lock); - return 1; - } - - er = emulate_instruction(vcpu, kvm_run, cr2, error_code, 0); - mutex_unlock(&vcpu->kvm->lock); - - switch (er) { - case EMULATE_DONE: - return 1; - case EMULATE_DO_MMIO: - ++vcpu->stat.mmio_exits; - return 0; - case EMULATE_FAIL: - kvm_report_emulation_failure(vcpu, "pagetable"); - break; - default: - BUG(); - } + return kvm_mmu_page_fault(vcpu, cr2, error_code); } if (vcpu->rmode.active && diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 01452b552db3..20da8e9751c0 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -85,11 +85,7 @@ struct kvm_vcpu { struct x86_emulate_ctxt emulate_ctxt; }; -static inline int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) -{ - return vcpu->mmu.page_fault(vcpu, gva, error_code); -} +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { -- cgit v1.2.3 From b733bfb524af69612f85c36a511f0109c5e3fe8d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 28 Oct 2007 18:52:05 +0200 Subject: KVM: MMU: Topup the mmu memory preallocation caches before emulating an insn Emulation may cause a shadow pte to be instantiated, which requires memory resources. Make sure the caches are filled to avoid an oops. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index ace3cb86214b..9be54a5e858e 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1362,6 +1362,10 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) goto out; } + r = mmu_topup_memory_caches(vcpu); + if (r) + goto out; + er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0); mutex_unlock(&vcpu->kvm->lock); -- cgit v1.2.3 From 1fe779f8eccd16e527315e1bafd2b3a876ff2489 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Mon, 29 Oct 2007 16:08:35 +0100 Subject: KVM: Portability: Split kvm_vm_ioctl v3 This patch splits kvm_vm_ioctl into archtecture independent parts, and x86 specific parts which go to kvm_arch_vcpu_ioctl in x86.c. The patch is unchanged since last submission. Common ioctls for all architectures are: KVM_CREATE_VCPU, KVM_GET_DIRTY_LOG, KVM_SET_USER_MEMORY_REGION x86 specific ioctls are: KVM_SET_MEMORY_REGION, KVM_GET/SET_NR_MMU_PAGES, KVM_SET_MEMORY_ALIAS, KVM_CREATE_IRQCHIP, KVM_CREATE_IRQ_LINE, KVM_GET/SET_IRQCHIP KVM_SET_TSS_ADDR Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 7 ++ drivers/kvm/kvm_main.c | 255 ++---------------------------------------------- drivers/kvm/x86.c | 258 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 271 insertions(+), 249 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3d07d9b1b815..516f79ffd126 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -620,6 +620,13 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); +int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, + struct + kvm_userspace_memory_region *mem, + int user_alloc); +long kvm_arch_vm_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +void kvm_arch_destroy_vm(struct kvm *kvm); __init void kvm_arch_init(void); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 50fd040b9ea9..c632e3a3b514 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -792,36 +792,16 @@ out: } EXPORT_SYMBOL_GPL(kvm_set_memory_region); -static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct - kvm_userspace_memory_region *mem, - int user_alloc) +int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, + struct + kvm_userspace_memory_region *mem, + int user_alloc) { if (mem->slot >= KVM_MEMORY_SLOTS) return -EINVAL; return kvm_set_memory_region(kvm, mem, user_alloc); } -static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, - u32 kvm_nr_mmu_pages) -{ - if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) - return -EINVAL; - - mutex_lock(&kvm->lock); - - kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); - kvm->n_requested_mmu_pages = kvm_nr_mmu_pages; - - mutex_unlock(&kvm->lock); - return 0; -} - -static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) -{ - return kvm->n_alloc_mmu_pages; -} - /* * Get (and clear) the dirty memory log for a memory slot. */ @@ -867,111 +847,6 @@ out: return r; } -/* - * Set a new alias region. Aliases map a portion of physical memory into - * another portion. This is useful for memory windows, for example the PC - * VGA region. - */ -static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, - struct kvm_memory_alias *alias) -{ - int r, n; - struct kvm_mem_alias *p; - - r = -EINVAL; - /* General sanity checks */ - if (alias->memory_size & (PAGE_SIZE - 1)) - goto out; - if (alias->guest_phys_addr & (PAGE_SIZE - 1)) - goto out; - if (alias->slot >= KVM_ALIAS_SLOTS) - goto out; - if (alias->guest_phys_addr + alias->memory_size - < alias->guest_phys_addr) - goto out; - if (alias->target_phys_addr + alias->memory_size - < alias->target_phys_addr) - goto out; - - mutex_lock(&kvm->lock); - - p = &kvm->aliases[alias->slot]; - p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; - p->npages = alias->memory_size >> PAGE_SHIFT; - p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT; - - for (n = KVM_ALIAS_SLOTS; n > 0; --n) - if (kvm->aliases[n - 1].npages) - break; - kvm->naliases = n; - - kvm_mmu_zap_all(kvm); - - mutex_unlock(&kvm->lock); - - return 0; - -out: - return r; -} - -static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) -{ - int r; - - r = 0; - switch (chip->chip_id) { - case KVM_IRQCHIP_PIC_MASTER: - memcpy(&chip->chip.pic, - &pic_irqchip(kvm)->pics[0], - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_PIC_SLAVE: - memcpy(&chip->chip.pic, - &pic_irqchip(kvm)->pics[1], - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_IOAPIC: - memcpy(&chip->chip.ioapic, - ioapic_irqchip(kvm), - sizeof(struct kvm_ioapic_state)); - break; - default: - r = -EINVAL; - break; - } - return r; -} - -static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) -{ - int r; - - r = 0; - switch (chip->chip_id) { - case KVM_IRQCHIP_PIC_MASTER: - memcpy(&pic_irqchip(kvm)->pics[0], - &chip->chip.pic, - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_PIC_SLAVE: - memcpy(&pic_irqchip(kvm)->pics[1], - &chip->chip.pic, - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_IOAPIC: - memcpy(ioapic_irqchip(kvm), - &chip->chip.ioapic, - sizeof(struct kvm_ioapic_state)); - break; - default: - r = -EINVAL; - break; - } - kvm_pic_update_irq(pic_irqchip(kvm)); - return r; -} - int is_error_page(struct page *page) { return page == bad_page; @@ -2661,16 +2536,6 @@ static int create_vcpu_fd(struct kvm_vcpu *vcpu) return fd; } -static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) -{ - int ret; - - if (addr > (unsigned int)(-3 * PAGE_SIZE)) - return -1; - ret = kvm_x86_ops->set_tss_addr(kvm, addr); - return ret; -} - /* * Creates some virtual cpus. Good luck creating more than one. */ @@ -2964,35 +2829,14 @@ static long kvm_vm_ioctl(struct file *filp, { struct kvm *kvm = filp->private_data; void __user *argp = (void __user *)arg; - int r = -EINVAL; + int r; switch (ioctl) { - case KVM_SET_TSS_ADDR: - r = kvm_vm_ioctl_set_tss_addr(kvm, arg); - if (r < 0) - goto out; - break; case KVM_CREATE_VCPU: r = kvm_vm_ioctl_create_vcpu(kvm, arg); if (r < 0) goto out; break; - case KVM_SET_MEMORY_REGION: { - struct kvm_memory_region kvm_mem; - struct kvm_userspace_memory_region kvm_userspace_mem; - - r = -EFAULT; - if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem)) - goto out; - kvm_userspace_mem.slot = kvm_mem.slot; - kvm_userspace_mem.flags = kvm_mem.flags; - kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr; - kvm_userspace_mem.memory_size = kvm_mem.memory_size; - r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0); - if (r) - goto out; - break; - } case KVM_SET_USER_MEMORY_REGION: { struct kvm_userspace_memory_region kvm_userspace_mem; @@ -3006,14 +2850,6 @@ static long kvm_vm_ioctl(struct file *filp, goto out; break; } - case KVM_SET_NR_MMU_PAGES: - r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); - if (r) - goto out; - break; - case KVM_GET_NR_MMU_PAGES: - r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); - break; case KVM_GET_DIRTY_LOG: { struct kvm_dirty_log log; @@ -3025,87 +2861,8 @@ static long kvm_vm_ioctl(struct file *filp, goto out; break; } - case KVM_SET_MEMORY_ALIAS: { - struct kvm_memory_alias alias; - - r = -EFAULT; - if (copy_from_user(&alias, argp, sizeof alias)) - goto out; - r = kvm_vm_ioctl_set_memory_alias(kvm, &alias); - if (r) - goto out; - break; - } - case KVM_CREATE_IRQCHIP: - r = -ENOMEM; - kvm->vpic = kvm_create_pic(kvm); - if (kvm->vpic) { - r = kvm_ioapic_init(kvm); - if (r) { - kfree(kvm->vpic); - kvm->vpic = NULL; - goto out; - } - } else - goto out; - break; - case KVM_IRQ_LINE: { - struct kvm_irq_level irq_event; - - r = -EFAULT; - if (copy_from_user(&irq_event, argp, sizeof irq_event)) - goto out; - if (irqchip_in_kernel(kvm)) { - mutex_lock(&kvm->lock); - if (irq_event.irq < 16) - kvm_pic_set_irq(pic_irqchip(kvm), - irq_event.irq, - irq_event.level); - kvm_ioapic_set_irq(kvm->vioapic, - irq_event.irq, - irq_event.level); - mutex_unlock(&kvm->lock); - r = 0; - } - break; - } - case KVM_GET_IRQCHIP: { - /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ - struct kvm_irqchip chip; - - r = -EFAULT; - if (copy_from_user(&chip, argp, sizeof chip)) - goto out; - r = -ENXIO; - if (!irqchip_in_kernel(kvm)) - goto out; - r = kvm_vm_ioctl_get_irqchip(kvm, &chip); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &chip, sizeof chip)) - goto out; - r = 0; - break; - } - case KVM_SET_IRQCHIP: { - /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ - struct kvm_irqchip chip; - - r = -EFAULT; - if (copy_from_user(&chip, argp, sizeof chip)) - goto out; - r = -ENXIO; - if (!irqchip_in_kernel(kvm)) - goto out; - r = kvm_vm_ioctl_set_irqchip(kvm, &chip); - if (r) - goto out; - r = 0; - break; - } default: - ; + r = kvm_arch_vm_ioctl(filp, ioctl, arg); } out: return r; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 1fe209dd4caf..b84cb6707f78 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -300,6 +300,264 @@ out: return r; } +static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) +{ + int ret; + + if (addr > (unsigned int)(-3 * PAGE_SIZE)) + return -1; + ret = kvm_x86_ops->set_tss_addr(kvm, addr); + return ret; +} + +static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, + u32 kvm_nr_mmu_pages) +{ + if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) + return -EINVAL; + + mutex_lock(&kvm->lock); + + kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); + kvm->n_requested_mmu_pages = kvm_nr_mmu_pages; + + mutex_unlock(&kvm->lock); + return 0; +} + +static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) +{ + return kvm->n_alloc_mmu_pages; +} + +/* + * Set a new alias region. Aliases map a portion of physical memory into + * another portion. This is useful for memory windows, for example the PC + * VGA region. + */ +static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, + struct kvm_memory_alias *alias) +{ + int r, n; + struct kvm_mem_alias *p; + + r = -EINVAL; + /* General sanity checks */ + if (alias->memory_size & (PAGE_SIZE - 1)) + goto out; + if (alias->guest_phys_addr & (PAGE_SIZE - 1)) + goto out; + if (alias->slot >= KVM_ALIAS_SLOTS) + goto out; + if (alias->guest_phys_addr + alias->memory_size + < alias->guest_phys_addr) + goto out; + if (alias->target_phys_addr + alias->memory_size + < alias->target_phys_addr) + goto out; + + mutex_lock(&kvm->lock); + + p = &kvm->aliases[alias->slot]; + p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; + p->npages = alias->memory_size >> PAGE_SHIFT; + p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT; + + for (n = KVM_ALIAS_SLOTS; n > 0; --n) + if (kvm->aliases[n - 1].npages) + break; + kvm->naliases = n; + + kvm_mmu_zap_all(kvm); + + mutex_unlock(&kvm->lock); + + return 0; + +out: + return r; +} + +static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) +{ + int r; + + r = 0; + switch (chip->chip_id) { + case KVM_IRQCHIP_PIC_MASTER: + memcpy(&chip->chip.pic, + &pic_irqchip(kvm)->pics[0], + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_PIC_SLAVE: + memcpy(&chip->chip.pic, + &pic_irqchip(kvm)->pics[1], + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_IOAPIC: + memcpy(&chip->chip.ioapic, + ioapic_irqchip(kvm), + sizeof(struct kvm_ioapic_state)); + break; + default: + r = -EINVAL; + break; + } + return r; +} + +static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) +{ + int r; + + r = 0; + switch (chip->chip_id) { + case KVM_IRQCHIP_PIC_MASTER: + memcpy(&pic_irqchip(kvm)->pics[0], + &chip->chip.pic, + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_PIC_SLAVE: + memcpy(&pic_irqchip(kvm)->pics[1], + &chip->chip.pic, + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_IOAPIC: + memcpy(ioapic_irqchip(kvm), + &chip->chip.ioapic, + sizeof(struct kvm_ioapic_state)); + break; + default: + r = -EINVAL; + break; + } + kvm_pic_update_irq(pic_irqchip(kvm)); + return r; +} + +long kvm_arch_vm_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm = filp->private_data; + void __user *argp = (void __user *)arg; + int r = -EINVAL; + + switch (ioctl) { + case KVM_SET_TSS_ADDR: + r = kvm_vm_ioctl_set_tss_addr(kvm, arg); + if (r < 0) + goto out; + break; + case KVM_SET_MEMORY_REGION: { + struct kvm_memory_region kvm_mem; + struct kvm_userspace_memory_region kvm_userspace_mem; + + r = -EFAULT; + if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem)) + goto out; + kvm_userspace_mem.slot = kvm_mem.slot; + kvm_userspace_mem.flags = kvm_mem.flags; + kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr; + kvm_userspace_mem.memory_size = kvm_mem.memory_size; + r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0); + if (r) + goto out; + break; + } + case KVM_SET_NR_MMU_PAGES: + r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); + if (r) + goto out; + break; + case KVM_GET_NR_MMU_PAGES: + r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); + break; + case KVM_SET_MEMORY_ALIAS: { + struct kvm_memory_alias alias; + + r = -EFAULT; + if (copy_from_user(&alias, argp, sizeof alias)) + goto out; + r = kvm_vm_ioctl_set_memory_alias(kvm, &alias); + if (r) + goto out; + break; + } + case KVM_CREATE_IRQCHIP: + r = -ENOMEM; + kvm->vpic = kvm_create_pic(kvm); + if (kvm->vpic) { + r = kvm_ioapic_init(kvm); + if (r) { + kfree(kvm->vpic); + kvm->vpic = NULL; + goto out; + } + } else + goto out; + break; + case KVM_IRQ_LINE: { + struct kvm_irq_level irq_event; + + r = -EFAULT; + if (copy_from_user(&irq_event, argp, sizeof irq_event)) + goto out; + if (irqchip_in_kernel(kvm)) { + mutex_lock(&kvm->lock); + if (irq_event.irq < 16) + kvm_pic_set_irq(pic_irqchip(kvm), + irq_event.irq, + irq_event.level); + kvm_ioapic_set_irq(kvm->vioapic, + irq_event.irq, + irq_event.level); + mutex_unlock(&kvm->lock); + r = 0; + } + break; + } + case KVM_GET_IRQCHIP: { + /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ + struct kvm_irqchip chip; + + r = -EFAULT; + if (copy_from_user(&chip, argp, sizeof chip)) + goto out; + r = -ENXIO; + if (!irqchip_in_kernel(kvm)) + goto out; + r = kvm_vm_ioctl_get_irqchip(kvm, &chip); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &chip, sizeof chip)) + goto out; + r = 0; + break; + } + case KVM_SET_IRQCHIP: { + /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ + struct kvm_irqchip chip; + + r = -EFAULT; + if (copy_from_user(&chip, argp, sizeof chip)) + goto out; + r = -ENXIO; + if (!irqchip_in_kernel(kvm)) + goto out; + r = kvm_vm_ioctl_set_irqchip(kvm, &chip); + if (r) + goto out; + r = 0; + break; + } + default: + ; + } +out: + return r; +} + static __init void kvm_init_msr_list(void) { u32 dummy[2]; -- cgit v1.2.3 From 5fb76f9be1a050a25e21a44ab2003c9d36a72a77 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Mon, 29 Oct 2007 16:08:51 +0100 Subject: KVM: Portability: Move memory segmentation to x86.c This patch moves the definition of segment_descriptor_64 for AMD64 and EM64T from kvm_main.c to segment_descriptor.h. It also adds a proper #ifndef...#define...#endif around that header file. The implementation of segment_base is moved from kvm_main.c to x86.c. Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 42 ---------------------------------------- drivers/kvm/segment_descriptor.h | 12 ++++++++++++ drivers/kvm/x86.c | 33 +++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 42 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index c632e3a3b514..cf163bddfbd3 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -18,7 +18,6 @@ #include "kvm.h" #include "x86.h" #include "x86_emulate.h" -#include "segment_descriptor.h" #include "irq.h" #include @@ -104,50 +103,9 @@ static struct dentry *debugfs_dir; #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) #define EFER_RESERVED_BITS 0xfffffffffffff2fe -#ifdef CONFIG_X86_64 -/* LDT or TSS descriptor in the GDT. 16 bytes. */ -struct segment_descriptor_64 { - struct segment_descriptor s; - u32 base_higher; - u32 pad_zero; -}; - -#endif - static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, unsigned long arg); -unsigned long segment_base(u16 selector) -{ - struct descriptor_table gdt; - struct segment_descriptor *d; - unsigned long table_base; - unsigned long v; - - if (selector == 0) - return 0; - - asm("sgdt %0" : "=m"(gdt)); - table_base = gdt.base; - - if (selector & 4) { /* from ldt */ - u16 ldt_selector; - - asm("sldt %0" : "=g"(ldt_selector)); - table_base = segment_base(ldt_selector); - } - d = (struct segment_descriptor *)(table_base + (selector & ~7)); - v = d->base_low | ((unsigned long)d->base_mid << 16) | - ((unsigned long)d->base_high << 24); -#ifdef CONFIG_X86_64 - if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11)) - v |= ((unsigned long) \ - ((struct segment_descriptor_64 *)d)->base_higher) << 32; -#endif - return v; -} -EXPORT_SYMBOL_GPL(segment_base); - static inline int valid_vcpu(int n) { return likely(n >= 0 && n < KVM_MAX_VCPUS); diff --git a/drivers/kvm/segment_descriptor.h b/drivers/kvm/segment_descriptor.h index 71fdf458619a..56fc4c873389 100644 --- a/drivers/kvm/segment_descriptor.h +++ b/drivers/kvm/segment_descriptor.h @@ -1,3 +1,6 @@ +#ifndef __SEGMENT_DESCRIPTOR_H +#define __SEGMENT_DESCRIPTOR_H + struct segment_descriptor { u16 limit_low; u16 base_low; @@ -14,4 +17,13 @@ struct segment_descriptor { u8 base_high; } __attribute__((packed)); +#ifdef CONFIG_X86_64 +/* LDT or TSS descriptor in the GDT. 16 bytes. */ +struct segment_descriptor_64 { + struct segment_descriptor s; + u32 base_higher; + u32 pad_zero; +}; +#endif +#endif diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index b84cb6707f78..5a959220410a 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -16,16 +16,49 @@ #include "kvm.h" #include "x86.h" +#include "segment_descriptor.h" #include "irq.h" #include #include #include +#include #include #define MAX_IO_MSRS 256 +unsigned long segment_base(u16 selector) +{ + struct descriptor_table gdt; + struct segment_descriptor *d; + unsigned long table_base; + unsigned long v; + + if (selector == 0) + return 0; + + asm("sgdt %0" : "=m"(gdt)); + table_base = gdt.base; + + if (selector & 4) { /* from ldt */ + u16 ldt_selector; + + asm("sldt %0" : "=g"(ldt_selector)); + table_base = segment_base(ldt_selector); + } + d = (struct segment_descriptor *)(table_base + (selector & ~7)); + v = d->base_low | ((unsigned long)d->base_mid << 16) | + ((unsigned long)d->base_high << 24); +#ifdef CONFIG_X86_64 + if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11)) + v |= ((unsigned long) \ + ((struct segment_descriptor_64 *)d)->base_higher) << 32; +#endif + return v; +} +EXPORT_SYMBOL_GPL(segment_base); + /* * List of msr numbers which we expose to userspace through KVM_GET_MSRS * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. -- cgit v1.2.3 From 6866b83ed7240bf4a7c50836ee10f61c8534503f Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Mon, 29 Oct 2007 16:09:10 +0100 Subject: KVM: Portability: move get/set_apic_base to x86.c This patch moves the implementation of get_apic_base and set_apic_base from kvm_main.c to x86.c Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 19 ------------------- drivers/kvm/x86.c | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index cf163bddfbd3..e8972a82544b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -556,25 +556,6 @@ unsigned long get_cr8(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(get_cr8); -u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) -{ - if (irqchip_in_kernel(vcpu->kvm)) - return vcpu->apic_base; - else - return vcpu->apic_base; -} -EXPORT_SYMBOL_GPL(kvm_get_apic_base); - -void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) -{ - /* TODO: reserve bits check */ - if (irqchip_in_kernel(vcpu->kvm)) - kvm_lapic_set_base(vcpu, data); - else - vcpu->apic_base = data; -} -EXPORT_SYMBOL_GPL(kvm_set_apic_base); - void fx_init(struct kvm_vcpu *vcpu) { unsigned after_mxcsr_mask; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5a959220410a..c26e3715bee8 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -59,6 +59,25 @@ unsigned long segment_base(u16 selector) } EXPORT_SYMBOL_GPL(segment_base); +u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) +{ + if (irqchip_in_kernel(vcpu->kvm)) + return vcpu->apic_base; + else + return vcpu->apic_base; +} +EXPORT_SYMBOL_GPL(kvm_get_apic_base); + +void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) +{ + /* TODO: reserve bits check */ + if (irqchip_in_kernel(vcpu->kvm)) + kvm_lapic_set_base(vcpu, data); + else + vcpu->apic_base = data; +} +EXPORT_SYMBOL_GPL(kvm_set_apic_base); + /* * List of msr numbers which we expose to userspace through KVM_GET_MSRS * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. -- cgit v1.2.3 From a03490ed29d2771c675d4d9c0ffe22e19a1757f3 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Mon, 29 Oct 2007 16:09:35 +0100 Subject: KVM: Portability: Move control register helper functions to x86.c This patch moves the definitions of CR0_RESERVED_BITS, CR4_RESERVED_BITS, and CR8_RESERVED_BITS along with the following functions from kvm_main.c to x86.c: set_cr0(), set_cr3(), set_cr4(), set_cr8(), get_cr8(), lmsw(), load_pdptrs() The static function wrapper inject_gp is duplicated in kvm_main.c and x86.c for now, the version in kvm_main.c should disappear once the last user of it is gone too. The function load_pdptrs is no longer static, and now defined in x86.h for the time being, until the last user of it is gone from kvm_main.c. Signed-off-by: Carsten Otte Reviewed-by: Christian Borntraeger Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 219 ----------------------------------------------- drivers/kvm/x86.c | 224 +++++++++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/x86.h | 2 +- 3 files changed, 225 insertions(+), 220 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index e8972a82544b..8f7125710d02 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -90,17 +90,6 @@ static struct kvm_stats_debugfs_item { static struct dentry *debugfs_dir; -#define CR0_RESERVED_BITS \ - (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ - | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ - | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) -#define CR4_RESERVED_BITS \ - (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ - | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ - | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \ - | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE)) - -#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) #define EFER_RESERVED_BITS 0xfffffffffffff2fe static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, @@ -348,214 +337,6 @@ static void inject_gp(struct kvm_vcpu *vcpu) kvm_x86_ops->inject_gp(vcpu, 0); } -/* - * Load the pae pdptrs. Return true is they are all valid. - */ -static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; - unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; - int i; - int ret; - u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; - - mutex_lock(&vcpu->kvm->lock); - ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, - offset * sizeof(u64), sizeof(pdpte)); - if (ret < 0) { - ret = 0; - goto out; - } - for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { - if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) { - ret = 0; - goto out; - } - } - ret = 1; - - memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs)); -out: - mutex_unlock(&vcpu->kvm->lock); - - return ret; -} - -void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ - if (cr0 & CR0_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", - cr0, vcpu->cr0); - inject_gp(vcpu); - return; - } - - if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) { - printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n"); - inject_gp(vcpu); - return; - } - - if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) { - printk(KERN_DEBUG "set_cr0: #GP, set PG flag " - "and a clear PE flag\n"); - inject_gp(vcpu); - return; - } - - if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { -#ifdef CONFIG_X86_64 - if ((vcpu->shadow_efer & EFER_LME)) { - int cs_db, cs_l; - - if (!is_pae(vcpu)) { - printk(KERN_DEBUG "set_cr0: #GP, start paging " - "in long mode while PAE is disabled\n"); - inject_gp(vcpu); - return; - } - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - if (cs_l) { - printk(KERN_DEBUG "set_cr0: #GP, start paging " - "in long mode while CS.L == 1\n"); - inject_gp(vcpu); - return; - - } - } else -#endif - if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { - printk(KERN_DEBUG "set_cr0: #GP, pdptrs " - "reserved bits\n"); - inject_gp(vcpu); - return; - } - - } - - kvm_x86_ops->set_cr0(vcpu, cr0); - vcpu->cr0 = cr0; - - mutex_lock(&vcpu->kvm->lock); - kvm_mmu_reset_context(vcpu); - mutex_unlock(&vcpu->kvm->lock); - return; -} -EXPORT_SYMBOL_GPL(set_cr0); - -void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) -{ - set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f)); -} -EXPORT_SYMBOL_GPL(lmsw); - -void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - if (cr4 & CR4_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n"); - inject_gp(vcpu); - return; - } - - if (is_long_mode(vcpu)) { - if (!(cr4 & X86_CR4_PAE)) { - printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while " - "in long mode\n"); - inject_gp(vcpu); - return; - } - } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) - && !load_pdptrs(vcpu, vcpu->cr3)) { - printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); - inject_gp(vcpu); - return; - } - - if (cr4 & X86_CR4_VMXE) { - printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n"); - inject_gp(vcpu); - return; - } - kvm_x86_ops->set_cr4(vcpu, cr4); - vcpu->cr4 = cr4; - mutex_lock(&vcpu->kvm->lock); - kvm_mmu_reset_context(vcpu); - mutex_unlock(&vcpu->kvm->lock); -} -EXPORT_SYMBOL_GPL(set_cr4); - -void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - if (is_long_mode(vcpu)) { - if (cr3 & CR3_L_MODE_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); - return; - } - } else { - if (is_pae(vcpu)) { - if (cr3 & CR3_PAE_RESERVED_BITS) { - printk(KERN_DEBUG - "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); - return; - } - if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) { - printk(KERN_DEBUG "set_cr3: #GP, pdptrs " - "reserved bits\n"); - inject_gp(vcpu); - return; - } - } - /* - * We don't check reserved bits in nonpae mode, because - * this isn't enforced, and VMware depends on this. - */ - } - - mutex_lock(&vcpu->kvm->lock); - /* - * Does the new cr3 value map to physical memory? (Note, we - * catch an invalid cr3 even in real-mode, because it would - * cause trouble later on when we turn on paging anyway.) - * - * A real CPU would silently accept an invalid cr3 and would - * attempt to use it - with largely undefined (and often hard - * to debug) behavior on the guest side. - */ - if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) - inject_gp(vcpu); - else { - vcpu->cr3 = cr3; - vcpu->mmu.new_cr3(vcpu); - } - mutex_unlock(&vcpu->kvm->lock); -} -EXPORT_SYMBOL_GPL(set_cr3); - -void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - if (cr8 & CR8_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8); - inject_gp(vcpu); - return; - } - if (irqchip_in_kernel(vcpu->kvm)) - kvm_lapic_set_tpr(vcpu, cr8); - else - vcpu->cr8 = cr8; -} -EXPORT_SYMBOL_GPL(set_cr8); - -unsigned long get_cr8(struct kvm_vcpu *vcpu) -{ - if (irqchip_in_kernel(vcpu->kvm)) - return kvm_lapic_get_cr8(vcpu); - else - return vcpu->cr8; -} -EXPORT_SYMBOL_GPL(get_cr8); - void fx_init(struct kvm_vcpu *vcpu) { unsigned after_mxcsr_mask; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c26e3715bee8..a728af8a83e5 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -27,6 +27,17 @@ #include #define MAX_IO_MSRS 256 +#define CR0_RESERVED_BITS \ + (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ + | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ + | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) +#define CR4_RESERVED_BITS \ + (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ + | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ + | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \ + | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE)) + +#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) unsigned long segment_base(u16 selector) { @@ -78,6 +89,219 @@ void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) } EXPORT_SYMBOL_GPL(kvm_set_apic_base); +static void inject_gp(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->inject_gp(vcpu, 0); +} + +/* + * Load the pae pdptrs. Return true is they are all valid. + */ +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; + unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; + int i; + int ret; + u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; + + mutex_lock(&vcpu->kvm->lock); + ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, + offset * sizeof(u64), sizeof(pdpte)); + if (ret < 0) { + ret = 0; + goto out; + } + for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { + if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) { + ret = 0; + goto out; + } + } + ret = 1; + + memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs)); +out: + mutex_unlock(&vcpu->kvm->lock); + + return ret; +} + +void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ + if (cr0 & CR0_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", + cr0, vcpu->cr0); + inject_gp(vcpu); + return; + } + + if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) { + printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n"); + inject_gp(vcpu); + return; + } + + if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) { + printk(KERN_DEBUG "set_cr0: #GP, set PG flag " + "and a clear PE flag\n"); + inject_gp(vcpu); + return; + } + + if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { +#ifdef CONFIG_X86_64 + if ((vcpu->shadow_efer & EFER_LME)) { + int cs_db, cs_l; + + if (!is_pae(vcpu)) { + printk(KERN_DEBUG "set_cr0: #GP, start paging " + "in long mode while PAE is disabled\n"); + inject_gp(vcpu); + return; + } + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + if (cs_l) { + printk(KERN_DEBUG "set_cr0: #GP, start paging " + "in long mode while CS.L == 1\n"); + inject_gp(vcpu); + return; + + } + } else +#endif + if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { + printk(KERN_DEBUG "set_cr0: #GP, pdptrs " + "reserved bits\n"); + inject_gp(vcpu); + return; + } + + } + + kvm_x86_ops->set_cr0(vcpu, cr0); + vcpu->cr0 = cr0; + + mutex_lock(&vcpu->kvm->lock); + kvm_mmu_reset_context(vcpu); + mutex_unlock(&vcpu->kvm->lock); + return; +} +EXPORT_SYMBOL_GPL(set_cr0); + +void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) +{ + set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f)); +} +EXPORT_SYMBOL_GPL(lmsw); + +void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + if (cr4 & CR4_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n"); + inject_gp(vcpu); + return; + } + + if (is_long_mode(vcpu)) { + if (!(cr4 & X86_CR4_PAE)) { + printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while " + "in long mode\n"); + inject_gp(vcpu); + return; + } + } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) + && !load_pdptrs(vcpu, vcpu->cr3)) { + printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); + inject_gp(vcpu); + return; + } + + if (cr4 & X86_CR4_VMXE) { + printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n"); + inject_gp(vcpu); + return; + } + kvm_x86_ops->set_cr4(vcpu, cr4); + vcpu->cr4 = cr4; + mutex_lock(&vcpu->kvm->lock); + kvm_mmu_reset_context(vcpu); + mutex_unlock(&vcpu->kvm->lock); +} +EXPORT_SYMBOL_GPL(set_cr4); + +void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + if (is_long_mode(vcpu)) { + if (cr3 & CR3_L_MODE_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); + inject_gp(vcpu); + return; + } + } else { + if (is_pae(vcpu)) { + if (cr3 & CR3_PAE_RESERVED_BITS) { + printk(KERN_DEBUG + "set_cr3: #GP, reserved bits\n"); + inject_gp(vcpu); + return; + } + if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) { + printk(KERN_DEBUG "set_cr3: #GP, pdptrs " + "reserved bits\n"); + inject_gp(vcpu); + return; + } + } + /* + * We don't check reserved bits in nonpae mode, because + * this isn't enforced, and VMware depends on this. + */ + } + + mutex_lock(&vcpu->kvm->lock); + /* + * Does the new cr3 value map to physical memory? (Note, we + * catch an invalid cr3 even in real-mode, because it would + * cause trouble later on when we turn on paging anyway.) + * + * A real CPU would silently accept an invalid cr3 and would + * attempt to use it - with largely undefined (and often hard + * to debug) behavior on the guest side. + */ + if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) + inject_gp(vcpu); + else { + vcpu->cr3 = cr3; + vcpu->mmu.new_cr3(vcpu); + } + mutex_unlock(&vcpu->kvm->lock); +} +EXPORT_SYMBOL_GPL(set_cr3); + +void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) +{ + if (cr8 & CR8_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8); + inject_gp(vcpu); + return; + } + if (irqchip_in_kernel(vcpu->kvm)) + kvm_lapic_set_tpr(vcpu, cr8); + else + vcpu->cr8 = cr8; +} +EXPORT_SYMBOL_GPL(set_cr8); + +unsigned long get_cr8(struct kvm_vcpu *vcpu) +{ + if (irqchip_in_kernel(vcpu->kvm)) + return kvm_lapic_get_cr8(vcpu); + else + return vcpu->cr8; +} +EXPORT_SYMBOL_GPL(get_cr8); + /* * List of msr numbers which we expose to userspace through KVM_GET_MSRS * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 20da8e9751c0..5592456c36ad 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -125,5 +125,5 @@ static inline int is_paging(struct kvm_vcpu *vcpu) return vcpu->cr0 & X86_CR0_PG; } - +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); #endif -- cgit v1.2.3 From f78e0e2ee498e8f847500b565792c7d7634dcf54 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Mon, 29 Oct 2007 09:40:42 +0800 Subject: KVM: VMX: Enable memory mapped TPR shadow (FlexPriority) This patch based on CR8/TPR patch, and enable the TPR shadow (FlexPriority) for 32bit Windows. Since TPR is accessed very frequently by 32bit Windows, especially SMP guest, with FlexPriority enabled, we saw significant performance gain. Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++ drivers/kvm/kvm_main.c | 56 ++++++++++++++++------- drivers/kvm/vmx.c | 117 +++++++++++++++++++++++++++++++++++++++++++------ drivers/kvm/vmx.h | 5 +++ 4 files changed, 152 insertions(+), 30 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 516f79ffd126..22317d6f66ae 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -383,6 +383,7 @@ struct kvm { struct kvm_ioapic *vioapic; int round_robin_prev_vcpu; unsigned int tss_addr; + struct page *apic_access_page; }; static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) @@ -522,6 +523,9 @@ int is_error_page(struct page *page); int kvm_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, int user_alloc); +int __kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); void kvm_release_page(struct page *page); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 8f7125710d02..ac5ed00e9065 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -362,10 +362,12 @@ EXPORT_SYMBOL_GPL(fx_init); * space. * * Discontiguous memory is allowed, mostly for framebuffers. + * + * Must be called holding kvm->lock. */ -int kvm_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - int user_alloc) +int __kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc) { int r; gfn_t base_gfn; @@ -392,8 +394,6 @@ int kvm_set_memory_region(struct kvm *kvm, if (!npages) mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; - mutex_lock(&kvm->lock); - new = old = *memslot; new.base_gfn = base_gfn; @@ -403,7 +403,7 @@ int kvm_set_memory_region(struct kvm *kvm, /* Disallow changing a memory slot's size. */ r = -EINVAL; if (npages && old.npages && npages != old.npages) - goto out_unlock; + goto out_free; /* Check for overlaps */ r = -EEXIST; @@ -414,7 +414,7 @@ int kvm_set_memory_region(struct kvm *kvm, continue; if (!((base_gfn + npages <= s->base_gfn) || (base_gfn >= s->base_gfn + s->npages))) - goto out_unlock; + goto out_free; } /* Free page dirty bitmap if unneeded */ @@ -428,7 +428,7 @@ int kvm_set_memory_region(struct kvm *kvm, new.rmap = vmalloc(npages * sizeof(struct page *)); if (!new.rmap) - goto out_unlock; + goto out_free; memset(new.rmap, 0, npages * sizeof(*new.rmap)); @@ -445,7 +445,7 @@ int kvm_set_memory_region(struct kvm *kvm, up_write(¤t->mm->mmap_sem); if (IS_ERR((void *)new.userspace_addr)) - goto out_unlock; + goto out_free; } } else { if (!old.user_alloc && old.rmap) { @@ -468,7 +468,7 @@ int kvm_set_memory_region(struct kvm *kvm, new.dirty_bitmap = vmalloc(dirty_bytes); if (!new.dirty_bitmap) - goto out_unlock; + goto out_free; memset(new.dirty_bitmap, 0, dirty_bytes); } @@ -498,18 +498,28 @@ int kvm_set_memory_region(struct kvm *kvm, kvm_mmu_slot_remove_write_access(kvm, mem->slot); kvm_flush_remote_tlbs(kvm); - mutex_unlock(&kvm->lock); - kvm_free_physmem_slot(&old, &new); return 0; -out_unlock: - mutex_unlock(&kvm->lock); +out_free: kvm_free_physmem_slot(&new, &old); out: return r; } +EXPORT_SYMBOL_GPL(__kvm_set_memory_region); + +int kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc) +{ + int r; + + mutex_lock(&kvm->lock); + r = __kvm_set_memory_region(kvm, mem, user_alloc); + mutex_unlock(&kvm->lock); + return r; +} EXPORT_SYMBOL_GPL(kvm_set_memory_region); int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, @@ -888,14 +898,21 @@ static int emulator_read_emulated(unsigned long addr, memcpy(val, vcpu->mmio_data, bytes); vcpu->mmio_read_completed = 0; return X86EMUL_CONTINUE; - } else if (emulator_read_std(addr, val, bytes, vcpu) - == X86EMUL_CONTINUE) - return X86EMUL_CONTINUE; + } gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + + if (emulator_read_std(addr, val, bytes, vcpu) + == X86EMUL_CONTINUE) + return X86EMUL_CONTINUE; if (gpa == UNMAPPED_GVA) return X86EMUL_PROPAGATE_FAULT; +mmio: /* * Is this MMIO handled locally? */ @@ -938,9 +955,14 @@ static int emulator_write_emulated_onepage(unsigned long addr, return X86EMUL_PROPAGATE_FAULT; } + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + if (emulator_write_phys(vcpu, gpa, val, bytes)) return X86EMUL_CONTINUE; +mmio: /* * Is this MMIO handled locally? */ diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 7fe834cb0d81..eca422e9506d 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -86,6 +86,7 @@ static struct vmcs_config { u32 revision_id; u32 pin_based_exec_ctrl; u32 cpu_based_exec_ctrl; + u32 cpu_based_2nd_exec_ctrl; u32 vmexit_ctrl; u32 vmentry_ctrl; } vmcs_config; @@ -179,6 +180,29 @@ static inline int vm_need_tpr_shadow(struct kvm *kvm) return ((cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm))); } +static inline int cpu_has_secondary_exec_ctrls(void) +{ + return (vmcs_config.cpu_based_exec_ctrl & + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); +} + +static inline int vm_need_secondary_exec_ctrls(struct kvm *kvm) +{ + return ((cpu_has_secondary_exec_ctrls()) && (irqchip_in_kernel(kvm))); +} + +static inline int cpu_has_vmx_virtualize_apic_accesses(void) +{ + return (vmcs_config.cpu_based_2nd_exec_ctrl & + SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); +} + +static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm) +{ + return ((cpu_has_vmx_virtualize_apic_accesses()) && + (irqchip_in_kernel(kvm))); +} + static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr) { int i; @@ -918,6 +942,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) u32 min, opt; u32 _pin_based_exec_control = 0; u32 _cpu_based_exec_control = 0; + u32 _cpu_based_2nd_exec_control = 0; u32 _vmexit_control = 0; u32 _vmentry_control = 0; @@ -935,11 +960,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MOV_DR_EXITING | CPU_BASED_USE_TSC_OFFSETING; -#ifdef CONFIG_X86_64 - opt = CPU_BASED_TPR_SHADOW; -#else - opt = 0; -#endif + opt = CPU_BASED_TPR_SHADOW | + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS, &_cpu_based_exec_control) < 0) return -EIO; @@ -948,6 +970,18 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING & ~CPU_BASED_CR8_STORE_EXITING; #endif + if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { + min = 0; + opt = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; + if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS2, + &_cpu_based_2nd_exec_control) < 0) + return -EIO; + } +#ifndef CONFIG_X86_64 + if (!(_cpu_based_2nd_exec_control & + SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) + _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; +#endif min = 0; #ifdef CONFIG_X86_64 @@ -985,6 +1019,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; + vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; vmcs_conf->vmexit_ctrl = _vmexit_control; vmcs_conf->vmentry_ctrl = _vmentry_control; @@ -1427,6 +1462,27 @@ static void seg_setup(int seg) vmcs_write32(sf->ar_bytes, 0x93); } +static int alloc_apic_access_page(struct kvm *kvm) +{ + struct kvm_userspace_memory_region kvm_userspace_mem; + int r = 0; + + mutex_lock(&kvm->lock); + if (kvm->apic_access_page) + goto out; + kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT; + kvm_userspace_mem.flags = 0; + kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL; + kvm_userspace_mem.memory_size = PAGE_SIZE; + r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0); + if (r) + goto out; + kvm->apic_access_page = gfn_to_page(kvm, 0xfee00); +out: + mutex_unlock(&kvm->lock); + return r; +} + /* * Sets up the vmcs for emulated real mode. */ @@ -1458,8 +1514,14 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) CPU_BASED_CR8_LOAD_EXITING; #endif } + if (!vm_need_secondary_exec_ctrls(vmx->vcpu.kvm)) + exec_control &= ~CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); + if (vm_need_secondary_exec_ctrls(vmx->vcpu.kvm)) + vmcs_write32(SECONDARY_VM_EXEC_CONTROL, + vmcs_config.cpu_based_2nd_exec_ctrl); + vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ @@ -1528,6 +1590,10 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); + if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + if (alloc_apic_access_page(vmx->vcpu.kvm) != 0) + return -ENOMEM; + return 0; } @@ -1616,13 +1682,17 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ -#ifdef CONFIG_X86_64 - vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); - if (vm_need_tpr_shadow(vmx->vcpu.kvm)) - vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - page_to_phys(vmx->vcpu.apic->regs_page)); - vmcs_write32(TPR_THRESHOLD, 0); -#endif + if (cpu_has_vmx_tpr_shadow()) { + vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); + if (vm_need_tpr_shadow(vmx->vcpu.kvm)) + vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, + page_to_phys(vmx->vcpu.apic->regs_page)); + vmcs_write32(TPR_THRESHOLD, 0); + } + + if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + vmcs_write64(APIC_ACCESS_ADDR, + page_to_phys(vmx->vcpu.kvm->apic_access_page)); vmx->vcpu.cr0 = 0x60000010; vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); /* enter rmode */ @@ -2094,6 +2164,26 @@ static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } +static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + u64 exit_qualification; + enum emulation_result er; + unsigned long offset; + + exit_qualification = vmcs_read64(EXIT_QUALIFICATION); + offset = exit_qualification & 0xffful; + + er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); + + if (er != EMULATE_DONE) { + printk(KERN_ERR + "Fail to handle apic access vmexit! Offset is 0x%lx\n", + offset); + return -ENOTSUPP; + } + return 1; +} + /* * The exit handlers return 1 if the exit was handled fully and guest execution * may resume. Otherwise they set the kvm_run parameter to indicate what needs @@ -2113,7 +2203,8 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window, [EXIT_REASON_HLT] = handle_halt, [EXIT_REASON_VMCALL] = handle_vmcall, - [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold + [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, + [EXIT_REASON_APIC_ACCESS] = handle_apic_access, }; static const int kvm_vmx_max_exit_handlers = diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index 270d477a2aa6..c84bd3733153 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -89,6 +89,8 @@ enum vmcs_field { TSC_OFFSET_HIGH = 0x00002011, VIRTUAL_APIC_PAGE_ADDR = 0x00002012, VIRTUAL_APIC_PAGE_ADDR_HIGH = 0x00002013, + APIC_ACCESS_ADDR = 0x00002014, + APIC_ACCESS_ADDR_HIGH = 0x00002015, VMCS_LINK_POINTER = 0x00002800, VMCS_LINK_POINTER_HIGH = 0x00002801, GUEST_IA32_DEBUGCTL = 0x00002802, @@ -214,6 +216,7 @@ enum vmcs_field { #define EXIT_REASON_MSR_WRITE 32 #define EXIT_REASON_MWAIT_INSTRUCTION 36 #define EXIT_REASON_TPR_BELOW_THRESHOLD 43 +#define EXIT_REASON_APIC_ACCESS 44 /* * Interruption-information format @@ -307,4 +310,6 @@ enum vmcs_field { #define MSR_IA32_FEATURE_CONTROL_LOCKED 0x1 #define MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED 0x4 +#define APIC_ACCESS_PAGE_PRIVATE_MEMSLOT 9 + #endif -- cgit v1.2.3 From aab61cc0d28f6fab0c2c9137d95dea54c7dbcf46 Mon Sep 17 00:00:00 2001 From: Anthony Liguori Date: Mon, 29 Oct 2007 15:15:20 -0500 Subject: KVM: Fix gfn_to_page() acquiring mmap_sem twice KVM's nopage handler calls gfn_to_page() which acquires the mmap_sem when calling out to get_user_pages(). nopage handlers are already invoked with the mmap_sem held though. Introduce a __gfn_to_page() for use by the nopage handler which requires the lock to already be held. This was noticed by tglx. Signed-off-by: Anthony Liguori Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index ac5ed00e9065..f439e45233f2 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -633,7 +633,10 @@ int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) } EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); -struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) +/* + * Requires current->mm->mmap_sem to be held + */ +static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *slot; struct page *page[1]; @@ -648,12 +651,10 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) return bad_page; } - down_read(¤t->mm->mmap_sem); npages = get_user_pages(current, current->mm, slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE, 1, 1, 1, page, NULL); - up_read(¤t->mm->mmap_sem); if (npages != 1) { get_page(bad_page); return bad_page; @@ -661,6 +662,18 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) return page[0]; } + +struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) +{ + struct page *page; + + down_read(¤t->mm->mmap_sem); + page = __gfn_to_page(kvm, gfn); + up_read(¤t->mm->mmap_sem); + + return page; +} + EXPORT_SYMBOL_GPL(gfn_to_page); void kvm_release_page(struct page *page) @@ -2621,7 +2634,8 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; if (!kvm_is_visible_gfn(kvm, pgoff)) return NOPAGE_SIGBUS; - page = gfn_to_page(kvm, pgoff); + /* current->mm->mmap_sem is already held so call lockless version */ + page = __gfn_to_page(kvm, pgoff); if (is_error_page(page)) { kvm_release_page(page); return NOPAGE_SIGBUS; -- cgit v1.2.3 From 15c4a6406f6c40632260861e1db7c539e79dcf1a Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Tue, 30 Oct 2007 18:44:17 +0100 Subject: KVM: Portability: Move kvm_get/set_msr[_common] to x86.c This patch moves the implementation of the functions of kvm_get/set_msr, kvm_get/set_msr_common, and set_efer from kvm_main.c to x86.c. The definition of EFER_RESERVED_BITS is moved too. Signed-off-by: Carsten Otte Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 133 ------------------------------------------------ drivers/kvm/x86.c | 134 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 133 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index f439e45233f2..7acf4cb07793 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -90,8 +90,6 @@ static struct kvm_stats_debugfs_item { static struct dentry *debugfs_dir; -#define EFER_RESERVED_BITS 0xfffffffffffff2fe - static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, unsigned long arg); @@ -1356,137 +1354,6 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, } } -int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) -{ - u64 data; - - switch (msr) { - case 0xc0010010: /* SYSCFG */ - case 0xc0010015: /* HWCR */ - case MSR_IA32_PLATFORM_ID: - case MSR_IA32_P5_MC_ADDR: - case MSR_IA32_P5_MC_TYPE: - case MSR_IA32_MC0_CTL: - case MSR_IA32_MCG_STATUS: - case MSR_IA32_MCG_CAP: - case MSR_IA32_MC0_MISC: - case MSR_IA32_MC0_MISC+4: - case MSR_IA32_MC0_MISC+8: - case MSR_IA32_MC0_MISC+12: - case MSR_IA32_MC0_MISC+16: - case MSR_IA32_UCODE_REV: - case MSR_IA32_PERF_STATUS: - case MSR_IA32_EBL_CR_POWERON: - /* MTRR registers */ - case 0xfe: - case 0x200 ... 0x2ff: - data = 0; - break; - case 0xcd: /* fsb frequency */ - data = 3; - break; - case MSR_IA32_APICBASE: - data = kvm_get_apic_base(vcpu); - break; - case MSR_IA32_MISC_ENABLE: - data = vcpu->ia32_misc_enable_msr; - break; -#ifdef CONFIG_X86_64 - case MSR_EFER: - data = vcpu->shadow_efer; - break; -#endif - default: - pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr); - return 1; - } - *pdata = data; - return 0; -} -EXPORT_SYMBOL_GPL(kvm_get_msr_common); - -/* - * Reads an msr value (of 'msr_index') into 'pdata'. - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) -{ - return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); -} - -#ifdef CONFIG_X86_64 - -static void set_efer(struct kvm_vcpu *vcpu, u64 efer) -{ - if (efer & EFER_RESERVED_BITS) { - printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n", - efer); - inject_gp(vcpu); - return; - } - - if (is_paging(vcpu) - && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) { - printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); - inject_gp(vcpu); - return; - } - - kvm_x86_ops->set_efer(vcpu, efer); - - efer &= ~EFER_LMA; - efer |= vcpu->shadow_efer & EFER_LMA; - - vcpu->shadow_efer = efer; -} - -#endif - -int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) -{ - switch (msr) { -#ifdef CONFIG_X86_64 - case MSR_EFER: - set_efer(vcpu, data); - break; -#endif - case MSR_IA32_MC0_STATUS: - pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n", - __FUNCTION__, data); - break; - case MSR_IA32_MCG_STATUS: - pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n", - __FUNCTION__, data); - break; - case MSR_IA32_UCODE_REV: - case MSR_IA32_UCODE_WRITE: - case 0x200 ... 0x2ff: /* MTRRs */ - break; - case MSR_IA32_APICBASE: - kvm_set_apic_base(vcpu, data); - break; - case MSR_IA32_MISC_ENABLE: - vcpu->ia32_misc_enable_msr = data; - break; - default: - pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); - return 1; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_set_msr_common); - -/* - * Writes msr value into into the appropriate "register". - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) -{ - return kvm_x86_ops->set_msr(vcpu, msr_index, data); -} - void kvm_resched(struct kvm_vcpu *vcpu) { if (!need_resched()) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index a728af8a83e5..786274347512 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -38,6 +38,7 @@ | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE)) #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) +#define EFER_RESERVED_BITS 0xfffffffffffff2fe unsigned long segment_base(u16 selector) { @@ -324,6 +325,44 @@ static u32 emulated_msrs[] = { MSR_IA32_MISC_ENABLE, }; +#ifdef CONFIG_X86_64 + +static void set_efer(struct kvm_vcpu *vcpu, u64 efer) +{ + if (efer & EFER_RESERVED_BITS) { + printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n", + efer); + inject_gp(vcpu); + return; + } + + if (is_paging(vcpu) + && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) { + printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); + inject_gp(vcpu); + return; + } + + kvm_x86_ops->set_efer(vcpu, efer); + + efer &= ~EFER_LMA; + efer |= vcpu->shadow_efer & EFER_LMA; + + vcpu->shadow_efer = efer; +} + +#endif + +/* + * Writes msr value into into the appropriate "register". + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) +{ + return kvm_x86_ops->set_msr(vcpu, msr_index, data); +} + /* * Adapt set_msr() to msr_io()'s calling convention */ @@ -332,6 +371,101 @@ static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) return kvm_set_msr(vcpu, index, *data); } + +int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) +{ + switch (msr) { +#ifdef CONFIG_X86_64 + case MSR_EFER: + set_efer(vcpu, data); + break; +#endif + case MSR_IA32_MC0_STATUS: + pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n", + __FUNCTION__, data); + break; + case MSR_IA32_MCG_STATUS: + pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n", + __FUNCTION__, data); + break; + case MSR_IA32_UCODE_REV: + case MSR_IA32_UCODE_WRITE: + case 0x200 ... 0x2ff: /* MTRRs */ + break; + case MSR_IA32_APICBASE: + kvm_set_apic_base(vcpu, data); + break; + case MSR_IA32_MISC_ENABLE: + vcpu->ia32_misc_enable_msr = data; + break; + default: + pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_set_msr_common); + + +/* + * Reads an msr value (of 'msr_index') into 'pdata'. + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) +{ + return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); +} + +int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) +{ + u64 data; + + switch (msr) { + case 0xc0010010: /* SYSCFG */ + case 0xc0010015: /* HWCR */ + case MSR_IA32_PLATFORM_ID: + case MSR_IA32_P5_MC_ADDR: + case MSR_IA32_P5_MC_TYPE: + case MSR_IA32_MC0_CTL: + case MSR_IA32_MCG_STATUS: + case MSR_IA32_MCG_CAP: + case MSR_IA32_MC0_MISC: + case MSR_IA32_MC0_MISC+4: + case MSR_IA32_MC0_MISC+8: + case MSR_IA32_MC0_MISC+12: + case MSR_IA32_MC0_MISC+16: + case MSR_IA32_UCODE_REV: + case MSR_IA32_PERF_STATUS: + case MSR_IA32_EBL_CR_POWERON: + /* MTRR registers */ + case 0xfe: + case 0x200 ... 0x2ff: + data = 0; + break; + case 0xcd: /* fsb frequency */ + data = 3; + break; + case MSR_IA32_APICBASE: + data = kvm_get_apic_base(vcpu); + break; + case MSR_IA32_MISC_ENABLE: + data = vcpu->ia32_misc_enable_msr; + break; +#ifdef CONFIG_X86_64 + case MSR_EFER: + data = vcpu->shadow_efer; + break; +#endif + default: + pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr); + return 1; + } + *pdata = data; + return 0; +} +EXPORT_SYMBOL_GPL(kvm_get_msr_common); + /* * Read or write a bunch of msrs. All parameters are kernel addresses. * -- cgit v1.2.3 From bbd9b64e37aff5aa715ec5e168425790f5983bf1 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Tue, 30 Oct 2007 18:44:21 +0100 Subject: KVM: Portability: Move x86 emulation and mmio device hook to x86.c This patch moves the following functions to from kvm_main.c to x86.c: emulator_read/write_std, vcpu_find_pervcpu_dev, vcpu_find_mmio_dev, emulator_read/write_emulated, emulator_write_phys, emulator_write_emulated_onepage, emulator_cmpxchg_emulated, get_setment_base, emulate_invlpg, emulate_clts, emulator_get/set_dr, kvm_report_emulation_failure, emulate_instruction The following data type is moved to x86.c: struct x86_emulate_ops emulate_ops Signed-off-by: Carsten Otte Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 357 ------------------------------------------------ drivers/kvm/x86.c | 358 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 358 insertions(+), 357 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7acf4cb07793..1a56d76560de 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -827,369 +827,12 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) } } -int emulator_read_std(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - void *data = val; - - while (bytes) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); - unsigned offset = addr & (PAGE_SIZE-1); - unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); - int ret; - - if (gpa == UNMAPPED_GVA) - return X86EMUL_PROPAGATE_FAULT; - ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); - if (ret < 0) - return X86EMUL_UNHANDLEABLE; - - bytes -= tocopy; - data += tocopy; - addr += tocopy; - } - - return X86EMUL_CONTINUE; -} -EXPORT_SYMBOL_GPL(emulator_read_std); - -static int emulator_write_std(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes); - return X86EMUL_UNHANDLEABLE; -} - -/* - * Only apic need an MMIO device hook, so shortcut now.. - */ -static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - struct kvm_io_device *dev; - - if (vcpu->apic) { - dev = &vcpu->apic->dev; - if (dev->in_range(dev, addr)) - return dev; - } - return NULL; -} - -static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - struct kvm_io_device *dev; - - dev = vcpu_find_pervcpu_dev(vcpu, addr); - if (dev == NULL) - dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr); - return dev; -} - static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, gpa_t addr) { return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); } -static int emulator_read_emulated(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - struct kvm_io_device *mmio_dev; - gpa_t gpa; - - if (vcpu->mmio_read_completed) { - memcpy(val, vcpu->mmio_data, bytes); - vcpu->mmio_read_completed = 0; - return X86EMUL_CONTINUE; - } - - gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); - - /* For APIC access vmexit */ - if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) - goto mmio; - - if (emulator_read_std(addr, val, bytes, vcpu) - == X86EMUL_CONTINUE) - return X86EMUL_CONTINUE; - if (gpa == UNMAPPED_GVA) - return X86EMUL_PROPAGATE_FAULT; - -mmio: - /* - * Is this MMIO handled locally? - */ - mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); - if (mmio_dev) { - kvm_iodevice_read(mmio_dev, gpa, bytes, val); - return X86EMUL_CONTINUE; - } - - vcpu->mmio_needed = 1; - vcpu->mmio_phys_addr = gpa; - vcpu->mmio_size = bytes; - vcpu->mmio_is_write = 0; - - return X86EMUL_UNHANDLEABLE; -} - -static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, - const void *val, int bytes) -{ - int ret; - - ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); - if (ret < 0) - return 0; - kvm_mmu_pte_write(vcpu, gpa, val, bytes); - return 1; -} - -static int emulator_write_emulated_onepage(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - struct kvm_io_device *mmio_dev; - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); - - if (gpa == UNMAPPED_GVA) { - kvm_x86_ops->inject_page_fault(vcpu, addr, 2); - return X86EMUL_PROPAGATE_FAULT; - } - - /* For APIC access vmexit */ - if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) - goto mmio; - - if (emulator_write_phys(vcpu, gpa, val, bytes)) - return X86EMUL_CONTINUE; - -mmio: - /* - * Is this MMIO handled locally? - */ - mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); - if (mmio_dev) { - kvm_iodevice_write(mmio_dev, gpa, bytes, val); - return X86EMUL_CONTINUE; - } - - vcpu->mmio_needed = 1; - vcpu->mmio_phys_addr = gpa; - vcpu->mmio_size = bytes; - vcpu->mmio_is_write = 1; - memcpy(vcpu->mmio_data, val, bytes); - - return X86EMUL_CONTINUE; -} - -int emulator_write_emulated(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - /* Crossing a page boundary? */ - if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { - int rc, now; - - now = -addr & ~PAGE_MASK; - rc = emulator_write_emulated_onepage(addr, val, now, vcpu); - if (rc != X86EMUL_CONTINUE) - return rc; - addr += now; - val += now; - bytes -= now; - } - return emulator_write_emulated_onepage(addr, val, bytes, vcpu); -} -EXPORT_SYMBOL_GPL(emulator_write_emulated); - -static int emulator_cmpxchg_emulated(unsigned long addr, - const void *old, - const void *new, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - static int reported; - - if (!reported) { - reported = 1; - printk(KERN_WARNING "kvm: emulating exchange as write\n"); - } - return emulator_write_emulated(addr, new, bytes, vcpu); -} - -static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) -{ - return kvm_x86_ops->get_segment_base(vcpu, seg); -} - -int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) -{ - return X86EMUL_CONTINUE; -} - -int emulate_clts(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS); - return X86EMUL_CONTINUE; -} - -int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) -{ - struct kvm_vcpu *vcpu = ctxt->vcpu; - - switch (dr) { - case 0 ... 3: - *dest = kvm_x86_ops->get_dr(vcpu, dr); - return X86EMUL_CONTINUE; - default: - pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr); - return X86EMUL_UNHANDLEABLE; - } -} - -int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) -{ - unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U; - int exception; - - kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception); - if (exception) { - /* FIXME: better handling */ - return X86EMUL_UNHANDLEABLE; - } - return X86EMUL_CONTINUE; -} - -void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) -{ - static int reported; - u8 opcodes[4]; - unsigned long rip = vcpu->rip; - unsigned long rip_linear; - - rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); - - if (reported) - return; - - emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); - - printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", - context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); - reported = 1; -} -EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); - -struct x86_emulate_ops emulate_ops = { - .read_std = emulator_read_std, - .write_std = emulator_write_std, - .read_emulated = emulator_read_emulated, - .write_emulated = emulator_write_emulated, - .cmpxchg_emulated = emulator_cmpxchg_emulated, -}; - -int emulate_instruction(struct kvm_vcpu *vcpu, - struct kvm_run *run, - unsigned long cr2, - u16 error_code, - int no_decode) -{ - int r; - - vcpu->mmio_fault_cr2 = cr2; - kvm_x86_ops->cache_regs(vcpu); - - vcpu->mmio_is_write = 0; - vcpu->pio.string = 0; - - if (!no_decode) { - int cs_db, cs_l; - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - - vcpu->emulate_ctxt.vcpu = vcpu; - vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - vcpu->emulate_ctxt.cr2 = cr2; - vcpu->emulate_ctxt.mode = - (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) - ? X86EMUL_MODE_REAL : cs_l - ? X86EMUL_MODE_PROT64 : cs_db - ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; - - if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) { - vcpu->emulate_ctxt.cs_base = 0; - vcpu->emulate_ctxt.ds_base = 0; - vcpu->emulate_ctxt.es_base = 0; - vcpu->emulate_ctxt.ss_base = 0; - } else { - vcpu->emulate_ctxt.cs_base = - get_segment_base(vcpu, VCPU_SREG_CS); - vcpu->emulate_ctxt.ds_base = - get_segment_base(vcpu, VCPU_SREG_DS); - vcpu->emulate_ctxt.es_base = - get_segment_base(vcpu, VCPU_SREG_ES); - vcpu->emulate_ctxt.ss_base = - get_segment_base(vcpu, VCPU_SREG_SS); - } - - vcpu->emulate_ctxt.gs_base = - get_segment_base(vcpu, VCPU_SREG_GS); - vcpu->emulate_ctxt.fs_base = - get_segment_base(vcpu, VCPU_SREG_FS); - - r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); - if (r) { - if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) - return EMULATE_DONE; - return EMULATE_FAIL; - } - } - - r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); - - if (vcpu->pio.string) - return EMULATE_DO_MMIO; - - if ((r || vcpu->mmio_is_write) && run) { - run->exit_reason = KVM_EXIT_MMIO; - run->mmio.phys_addr = vcpu->mmio_phys_addr; - memcpy(run->mmio.data, vcpu->mmio_data, 8); - run->mmio.len = vcpu->mmio_size; - run->mmio.is_write = vcpu->mmio_is_write; - } - - if (r) { - if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) - return EMULATE_DONE; - if (!vcpu->mmio_needed) { - kvm_report_emulation_failure(vcpu, "mmio"); - return EMULATE_FAIL; - } - return EMULATE_DO_MMIO; - } - - kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags); - - if (vcpu->mmio_is_write) { - vcpu->mmio_needed = 0; - return EMULATE_DO_MMIO; - } - - return EMULATE_DONE; -} -EXPORT_SYMBOL_GPL(emulate_instruction); - /* * The vCPU has executed a HLT instruction with in-kernel mode enabled. */ diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 786274347512..fe3733d8ece5 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -983,6 +983,364 @@ static __init void kvm_init_msr_list(void) num_msrs_to_save = j; } +/* + * Only apic need an MMIO device hook, so shortcut now.. + */ +static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + struct kvm_io_device *dev; + + if (vcpu->apic) { + dev = &vcpu->apic->dev; + if (dev->in_range(dev, addr)) + return dev; + } + return NULL; +} + + +static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + struct kvm_io_device *dev; + + dev = vcpu_find_pervcpu_dev(vcpu, addr); + if (dev == NULL) + dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr); + return dev; +} + +int emulator_read_std(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + void *data = val; + + while (bytes) { + gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + unsigned offset = addr & (PAGE_SIZE-1); + unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); + int ret; + + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; + ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); + if (ret < 0) + return X86EMUL_UNHANDLEABLE; + + bytes -= tocopy; + data += tocopy; + addr += tocopy; + } + + return X86EMUL_CONTINUE; +} +EXPORT_SYMBOL_GPL(emulator_read_std); + +static int emulator_write_std(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes); + return X86EMUL_UNHANDLEABLE; +} + +static int emulator_read_emulated(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + struct kvm_io_device *mmio_dev; + gpa_t gpa; + + if (vcpu->mmio_read_completed) { + memcpy(val, vcpu->mmio_data, bytes); + vcpu->mmio_read_completed = 0; + return X86EMUL_CONTINUE; + } + + gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + + if (emulator_read_std(addr, val, bytes, vcpu) + == X86EMUL_CONTINUE) + return X86EMUL_CONTINUE; + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; + +mmio: + /* + * Is this MMIO handled locally? + */ + mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); + if (mmio_dev) { + kvm_iodevice_read(mmio_dev, gpa, bytes, val); + return X86EMUL_CONTINUE; + } + + vcpu->mmio_needed = 1; + vcpu->mmio_phys_addr = gpa; + vcpu->mmio_size = bytes; + vcpu->mmio_is_write = 0; + + return X86EMUL_UNHANDLEABLE; +} + +static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, + const void *val, int bytes) +{ + int ret; + + ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); + if (ret < 0) + return 0; + kvm_mmu_pte_write(vcpu, gpa, val, bytes); + return 1; +} + +static int emulator_write_emulated_onepage(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + struct kvm_io_device *mmio_dev; + gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + + if (gpa == UNMAPPED_GVA) { + kvm_x86_ops->inject_page_fault(vcpu, addr, 2); + return X86EMUL_PROPAGATE_FAULT; + } + + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + + if (emulator_write_phys(vcpu, gpa, val, bytes)) + return X86EMUL_CONTINUE; + +mmio: + /* + * Is this MMIO handled locally? + */ + mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); + if (mmio_dev) { + kvm_iodevice_write(mmio_dev, gpa, bytes, val); + return X86EMUL_CONTINUE; + } + + vcpu->mmio_needed = 1; + vcpu->mmio_phys_addr = gpa; + vcpu->mmio_size = bytes; + vcpu->mmio_is_write = 1; + memcpy(vcpu->mmio_data, val, bytes); + + return X86EMUL_CONTINUE; +} + +int emulator_write_emulated(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + /* Crossing a page boundary? */ + if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { + int rc, now; + + now = -addr & ~PAGE_MASK; + rc = emulator_write_emulated_onepage(addr, val, now, vcpu); + if (rc != X86EMUL_CONTINUE) + return rc; + addr += now; + val += now; + bytes -= now; + } + return emulator_write_emulated_onepage(addr, val, bytes, vcpu); +} +EXPORT_SYMBOL_GPL(emulator_write_emulated); + +static int emulator_cmpxchg_emulated(unsigned long addr, + const void *old, + const void *new, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + static int reported; + + if (!reported) { + reported = 1; + printk(KERN_WARNING "kvm: emulating exchange as write\n"); + } + return emulator_write_emulated(addr, new, bytes, vcpu); +} + +static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) +{ + return kvm_x86_ops->get_segment_base(vcpu, seg); +} + +int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) +{ + return X86EMUL_CONTINUE; +} + +int emulate_clts(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS); + return X86EMUL_CONTINUE; +} + +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) +{ + struct kvm_vcpu *vcpu = ctxt->vcpu; + + switch (dr) { + case 0 ... 3: + *dest = kvm_x86_ops->get_dr(vcpu, dr); + return X86EMUL_CONTINUE; + default: + pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr); + return X86EMUL_UNHANDLEABLE; + } +} + +int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) +{ + unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U; + int exception; + + kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception); + if (exception) { + /* FIXME: better handling */ + return X86EMUL_UNHANDLEABLE; + } + return X86EMUL_CONTINUE; +} + +void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) +{ + static int reported; + u8 opcodes[4]; + unsigned long rip = vcpu->rip; + unsigned long rip_linear; + + rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); + + if (reported) + return; + + emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); + + printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", + context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); + reported = 1; +} +EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); + +struct x86_emulate_ops emulate_ops = { + .read_std = emulator_read_std, + .write_std = emulator_write_std, + .read_emulated = emulator_read_emulated, + .write_emulated = emulator_write_emulated, + .cmpxchg_emulated = emulator_cmpxchg_emulated, +}; + +int emulate_instruction(struct kvm_vcpu *vcpu, + struct kvm_run *run, + unsigned long cr2, + u16 error_code, + int no_decode) +{ + int r; + + vcpu->mmio_fault_cr2 = cr2; + kvm_x86_ops->cache_regs(vcpu); + + vcpu->mmio_is_write = 0; + vcpu->pio.string = 0; + + if (!no_decode) { + int cs_db, cs_l; + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + + vcpu->emulate_ctxt.vcpu = vcpu; + vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); + vcpu->emulate_ctxt.cr2 = cr2; + vcpu->emulate_ctxt.mode = + (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) + ? X86EMUL_MODE_REAL : cs_l + ? X86EMUL_MODE_PROT64 : cs_db + ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; + + if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) { + vcpu->emulate_ctxt.cs_base = 0; + vcpu->emulate_ctxt.ds_base = 0; + vcpu->emulate_ctxt.es_base = 0; + vcpu->emulate_ctxt.ss_base = 0; + } else { + vcpu->emulate_ctxt.cs_base = + get_segment_base(vcpu, VCPU_SREG_CS); + vcpu->emulate_ctxt.ds_base = + get_segment_base(vcpu, VCPU_SREG_DS); + vcpu->emulate_ctxt.es_base = + get_segment_base(vcpu, VCPU_SREG_ES); + vcpu->emulate_ctxt.ss_base = + get_segment_base(vcpu, VCPU_SREG_SS); + } + + vcpu->emulate_ctxt.gs_base = + get_segment_base(vcpu, VCPU_SREG_GS); + vcpu->emulate_ctxt.fs_base = + get_segment_base(vcpu, VCPU_SREG_FS); + + r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + if (r) { + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; + return EMULATE_FAIL; + } + } + + r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); + + if (vcpu->pio.string) + return EMULATE_DO_MMIO; + + if ((r || vcpu->mmio_is_write) && run) { + run->exit_reason = KVM_EXIT_MMIO; + run->mmio.phys_addr = vcpu->mmio_phys_addr; + memcpy(run->mmio.data, vcpu->mmio_data, 8); + run->mmio.len = vcpu->mmio_size; + run->mmio.is_write = vcpu->mmio_is_write; + } + + if (r) { + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; + if (!vcpu->mmio_needed) { + kvm_report_emulation_failure(vcpu, "mmio"); + return EMULATE_FAIL; + } + return EMULATE_DO_MMIO; + } + + kvm_x86_ops->decache_regs(vcpu); + kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags); + + if (vcpu->mmio_is_write) { + vcpu->mmio_needed = 0; + return EMULATE_DO_MMIO; + } + + return EMULATE_DONE; +} +EXPORT_SYMBOL_GPL(emulate_instruction); + __init void kvm_arch_init(void) { kvm_init_msr_list(); -- cgit v1.2.3 From de7d789acd7f373268194bb48dc0690c975ab8e6 Mon Sep 17 00:00:00 2001 From: Carsten Otte Date: Tue, 30 Oct 2007 18:44:25 +0100 Subject: KVM: Portability: Move pio emulation functions to x86.c This patch moves implementation of the following functions from kvm_main.c to x86.c: free_pio_guest_pages, vcpu_find_pio_dev, pio_copy_data, complete_pio, kernel_pio, pio_string_write, kvm_emulate_pio, kvm_emulate_pio_string The function inject_gp, which was duplicated by yesterday's patch series, is removed from kvm_main.c now because it is not needed anymore. Signed-off-by: Carsten Otte Acked-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 248 ------------------------------------------------- drivers/kvm/x86.c | 243 ++++++++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/x86.h | 1 + 3 files changed, 244 insertions(+), 248 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 1a56d76560de..2025cdfb4593 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -271,17 +271,6 @@ static void kvm_free_physmem(struct kvm *kvm) kvm_free_physmem_slot(&kvm->memslots[i], NULL); } -static void free_pio_guest_pages(struct kvm_vcpu *vcpu) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) - if (vcpu->pio.guest_pages[i]) { - kvm_release_page(vcpu->pio.guest_pages[i]); - vcpu->pio.guest_pages[i] = NULL; - } -} - static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) { vcpu_load(vcpu); @@ -330,11 +319,6 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) return 0; } -static void inject_gp(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->inject_gp(vcpu, 0); -} - void fx_init(struct kvm_vcpu *vcpu) { unsigned after_mxcsr_mask; @@ -827,12 +811,6 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) } } -static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); -} - /* * The vCPU has executed a HLT instruction with in-kernel mode enabled. */ @@ -1042,232 +1020,6 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); -static int pio_copy_data(struct kvm_vcpu *vcpu) -{ - void *p = vcpu->pio_data; - void *q; - unsigned bytes; - int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1; - - q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE, - PAGE_KERNEL); - if (!q) { - free_pio_guest_pages(vcpu); - return -ENOMEM; - } - q += vcpu->pio.guest_page_offset; - bytes = vcpu->pio.size * vcpu->pio.cur_count; - if (vcpu->pio.in) - memcpy(q, p, bytes); - else - memcpy(p, q, bytes); - q -= vcpu->pio.guest_page_offset; - vunmap(q); - free_pio_guest_pages(vcpu); - return 0; -} - -static int complete_pio(struct kvm_vcpu *vcpu) -{ - struct kvm_pio_request *io = &vcpu->pio; - long delta; - int r; - - kvm_x86_ops->cache_regs(vcpu); - - if (!io->string) { - if (io->in) - memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data, - io->size); - } else { - if (io->in) { - r = pio_copy_data(vcpu); - if (r) { - kvm_x86_ops->cache_regs(vcpu); - return r; - } - } - - delta = 1; - if (io->rep) { - delta *= io->cur_count; - /* - * The size of the register should really depend on - * current address size. - */ - vcpu->regs[VCPU_REGS_RCX] -= delta; - } - if (io->down) - delta = -delta; - delta *= io->size; - if (io->in) - vcpu->regs[VCPU_REGS_RDI] += delta; - else - vcpu->regs[VCPU_REGS_RSI] += delta; - } - - kvm_x86_ops->decache_regs(vcpu); - - io->count -= io->cur_count; - io->cur_count = 0; - - return 0; -} - -static void kernel_pio(struct kvm_io_device *pio_dev, - struct kvm_vcpu *vcpu, - void *pd) -{ - /* TODO: String I/O for in kernel device */ - - mutex_lock(&vcpu->kvm->lock); - if (vcpu->pio.in) - kvm_iodevice_read(pio_dev, vcpu->pio.port, - vcpu->pio.size, - pd); - else - kvm_iodevice_write(pio_dev, vcpu->pio.port, - vcpu->pio.size, - pd); - mutex_unlock(&vcpu->kvm->lock); -} - -static void pio_string_write(struct kvm_io_device *pio_dev, - struct kvm_vcpu *vcpu) -{ - struct kvm_pio_request *io = &vcpu->pio; - void *pd = vcpu->pio_data; - int i; - - mutex_lock(&vcpu->kvm->lock); - for (i = 0; i < io->cur_count; i++) { - kvm_iodevice_write(pio_dev, io->port, - io->size, - pd); - pd += io->size; - } - mutex_unlock(&vcpu->kvm->lock); -} - -int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned port) -{ - struct kvm_io_device *pio_dev; - - vcpu->run->exit_reason = KVM_EXIT_IO; - vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->pio.size = size; - vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1; - vcpu->run->io.port = vcpu->pio.port = port; - vcpu->pio.in = in; - vcpu->pio.string = 0; - vcpu->pio.down = 0; - vcpu->pio.guest_page_offset = 0; - vcpu->pio.rep = 0; - - kvm_x86_ops->cache_regs(vcpu); - memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4); - kvm_x86_ops->decache_regs(vcpu); - - kvm_x86_ops->skip_emulated_instruction(vcpu); - - pio_dev = vcpu_find_pio_dev(vcpu, port); - if (pio_dev) { - kernel_pio(pio_dev, vcpu, vcpu->pio_data); - complete_pio(vcpu); - return 1; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_emulate_pio); - -int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned long count, int down, - gva_t address, int rep, unsigned port) -{ - unsigned now, in_page; - int i, ret = 0; - int nr_pages = 1; - struct page *page; - struct kvm_io_device *pio_dev; - - vcpu->run->exit_reason = KVM_EXIT_IO; - vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->pio.size = size; - vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count; - vcpu->run->io.port = vcpu->pio.port = port; - vcpu->pio.in = in; - vcpu->pio.string = 1; - vcpu->pio.down = down; - vcpu->pio.guest_page_offset = offset_in_page(address); - vcpu->pio.rep = rep; - - if (!count) { - kvm_x86_ops->skip_emulated_instruction(vcpu); - return 1; - } - - if (!down) - in_page = PAGE_SIZE - offset_in_page(address); - else - in_page = offset_in_page(address) + size; - now = min(count, (unsigned long)in_page / size); - if (!now) { - /* - * String I/O straddles page boundary. Pin two guest pages - * so that we satisfy atomicity constraints. Do just one - * transaction to avoid complexity. - */ - nr_pages = 2; - now = 1; - } - if (down) { - /* - * String I/O in reverse. Yuck. Kill the guest, fix later. - */ - pr_unimpl(vcpu, "guest string pio down\n"); - inject_gp(vcpu); - return 1; - } - vcpu->run->io.count = now; - vcpu->pio.cur_count = now; - - if (vcpu->pio.cur_count == vcpu->pio.count) - kvm_x86_ops->skip_emulated_instruction(vcpu); - - for (i = 0; i < nr_pages; ++i) { - mutex_lock(&vcpu->kvm->lock); - page = gva_to_page(vcpu, address + i * PAGE_SIZE); - vcpu->pio.guest_pages[i] = page; - mutex_unlock(&vcpu->kvm->lock); - if (!page) { - inject_gp(vcpu); - free_pio_guest_pages(vcpu); - return 1; - } - } - - pio_dev = vcpu_find_pio_dev(vcpu, port); - if (!vcpu->pio.in) { - /* string PIO write */ - ret = pio_copy_data(vcpu); - if (ret >= 0 && pio_dev) { - pio_string_write(pio_dev, vcpu); - complete_pio(vcpu); - if (vcpu->pio.count == 0) - ret = 1; - } - } else if (pio_dev) - pr_unimpl(vcpu, "no string pio read support yet, " - "port %x size %d count %ld\n", - port, size, count); - - return ret; -} -EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); - /* * Check if userspace requested an interrupt window, and that the * interrupt window is open. diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index fe3733d8ece5..f75e7d7d9ead 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1341,6 +1341,249 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } EXPORT_SYMBOL_GPL(emulate_instruction); +static void free_pio_guest_pages(struct kvm_vcpu *vcpu) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) + if (vcpu->pio.guest_pages[i]) { + kvm_release_page(vcpu->pio.guest_pages[i]); + vcpu->pio.guest_pages[i] = NULL; + } +} + +static int pio_copy_data(struct kvm_vcpu *vcpu) +{ + void *p = vcpu->pio_data; + void *q; + unsigned bytes; + int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1; + + q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE, + PAGE_KERNEL); + if (!q) { + free_pio_guest_pages(vcpu); + return -ENOMEM; + } + q += vcpu->pio.guest_page_offset; + bytes = vcpu->pio.size * vcpu->pio.cur_count; + if (vcpu->pio.in) + memcpy(q, p, bytes); + else + memcpy(p, q, bytes); + q -= vcpu->pio.guest_page_offset; + vunmap(q); + free_pio_guest_pages(vcpu); + return 0; +} + +int complete_pio(struct kvm_vcpu *vcpu) +{ + struct kvm_pio_request *io = &vcpu->pio; + long delta; + int r; + + kvm_x86_ops->cache_regs(vcpu); + + if (!io->string) { + if (io->in) + memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data, + io->size); + } else { + if (io->in) { + r = pio_copy_data(vcpu); + if (r) { + kvm_x86_ops->cache_regs(vcpu); + return r; + } + } + + delta = 1; + if (io->rep) { + delta *= io->cur_count; + /* + * The size of the register should really depend on + * current address size. + */ + vcpu->regs[VCPU_REGS_RCX] -= delta; + } + if (io->down) + delta = -delta; + delta *= io->size; + if (io->in) + vcpu->regs[VCPU_REGS_RDI] += delta; + else + vcpu->regs[VCPU_REGS_RSI] += delta; + } + + kvm_x86_ops->decache_regs(vcpu); + + io->count -= io->cur_count; + io->cur_count = 0; + + return 0; +} + +static void kernel_pio(struct kvm_io_device *pio_dev, + struct kvm_vcpu *vcpu, + void *pd) +{ + /* TODO: String I/O for in kernel device */ + + mutex_lock(&vcpu->kvm->lock); + if (vcpu->pio.in) + kvm_iodevice_read(pio_dev, vcpu->pio.port, + vcpu->pio.size, + pd); + else + kvm_iodevice_write(pio_dev, vcpu->pio.port, + vcpu->pio.size, + pd); + mutex_unlock(&vcpu->kvm->lock); +} + +static void pio_string_write(struct kvm_io_device *pio_dev, + struct kvm_vcpu *vcpu) +{ + struct kvm_pio_request *io = &vcpu->pio; + void *pd = vcpu->pio_data; + int i; + + mutex_lock(&vcpu->kvm->lock); + for (i = 0; i < io->cur_count; i++) { + kvm_iodevice_write(pio_dev, io->port, + io->size, + pd); + pd += io->size; + } + mutex_unlock(&vcpu->kvm->lock); +} + +static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); +} + +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned port) +{ + struct kvm_io_device *pio_dev; + + vcpu->run->exit_reason = KVM_EXIT_IO; + vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; + vcpu->run->io.size = vcpu->pio.size = size; + vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; + vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1; + vcpu->run->io.port = vcpu->pio.port = port; + vcpu->pio.in = in; + vcpu->pio.string = 0; + vcpu->pio.down = 0; + vcpu->pio.guest_page_offset = 0; + vcpu->pio.rep = 0; + + kvm_x86_ops->cache_regs(vcpu); + memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4); + kvm_x86_ops->decache_regs(vcpu); + + kvm_x86_ops->skip_emulated_instruction(vcpu); + + pio_dev = vcpu_find_pio_dev(vcpu, port); + if (pio_dev) { + kernel_pio(pio_dev, vcpu, vcpu->pio_data); + complete_pio(vcpu); + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_emulate_pio); + +int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned long count, int down, + gva_t address, int rep, unsigned port) +{ + unsigned now, in_page; + int i, ret = 0; + int nr_pages = 1; + struct page *page; + struct kvm_io_device *pio_dev; + + vcpu->run->exit_reason = KVM_EXIT_IO; + vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; + vcpu->run->io.size = vcpu->pio.size = size; + vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; + vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count; + vcpu->run->io.port = vcpu->pio.port = port; + vcpu->pio.in = in; + vcpu->pio.string = 1; + vcpu->pio.down = down; + vcpu->pio.guest_page_offset = offset_in_page(address); + vcpu->pio.rep = rep; + + if (!count) { + kvm_x86_ops->skip_emulated_instruction(vcpu); + return 1; + } + + if (!down) + in_page = PAGE_SIZE - offset_in_page(address); + else + in_page = offset_in_page(address) + size; + now = min(count, (unsigned long)in_page / size); + if (!now) { + /* + * String I/O straddles page boundary. Pin two guest pages + * so that we satisfy atomicity constraints. Do just one + * transaction to avoid complexity. + */ + nr_pages = 2; + now = 1; + } + if (down) { + /* + * String I/O in reverse. Yuck. Kill the guest, fix later. + */ + pr_unimpl(vcpu, "guest string pio down\n"); + inject_gp(vcpu); + return 1; + } + vcpu->run->io.count = now; + vcpu->pio.cur_count = now; + + if (vcpu->pio.cur_count == vcpu->pio.count) + kvm_x86_ops->skip_emulated_instruction(vcpu); + + for (i = 0; i < nr_pages; ++i) { + mutex_lock(&vcpu->kvm->lock); + page = gva_to_page(vcpu, address + i * PAGE_SIZE); + vcpu->pio.guest_pages[i] = page; + mutex_unlock(&vcpu->kvm->lock); + if (!page) { + inject_gp(vcpu); + free_pio_guest_pages(vcpu); + return 1; + } + } + + pio_dev = vcpu_find_pio_dev(vcpu, port); + if (!vcpu->pio.in) { + /* string PIO write */ + ret = pio_copy_data(vcpu); + if (ret >= 0 && pio_dev) { + pio_string_write(pio_dev, vcpu); + complete_pio(vcpu); + if (vcpu->pio.count == 0) + ret = 1; + } + } else if (pio_dev) + pr_unimpl(vcpu, "no string pio read support yet, " + "port %x size %d count %ld\n", + port, size, count); + + return ret; +} +EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); + __init void kvm_arch_init(void) { kvm_init_msr_list(); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 5592456c36ad..663b822b4ddb 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -126,4 +126,5 @@ static inline int is_paging(struct kvm_vcpu *vcpu) } int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); +int complete_pio(struct kvm_vcpu *vcpu); #endif -- cgit v1.2.3 From 3c118e24af821d68dca0ba81e9499820c840c133 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 31 Oct 2007 10:27:04 +0200 Subject: KVM: x86 emulator: Extract the common code of SrcReg and DstReg Share the common parts of SrcReg and DstReg decoding. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 80 ++++++++++++++++++----------------------------- 1 file changed, 31 insertions(+), 49 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 087a8208f873..58ceb6616364 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -520,6 +520,34 @@ static int test_cc(unsigned int condition, unsigned int flags) return (!!rc ^ (condition & 1)); } +static void decode_register_operand(struct operand *op, + struct decode_cache *c, + int highbyte_regs, + int inhibit_bytereg) +{ + op->type = OP_REG; + if ((c->d & ByteOp) && !inhibit_bytereg) { + op->ptr = decode_register(c->modrm_reg, c->regs, highbyte_regs); + op->val = *(u8 *)op->ptr; + op->bytes = 1; + } else { + op->ptr = decode_register(c->modrm_reg, c->regs, 0); + op->bytes = c->op_bytes; + switch (op->bytes) { + case 2: + op->val = *(u16 *)op->ptr; + break; + case 4: + op->val = *(u32 *)op->ptr; + break; + case 8: + op->val = *(u64 *) op->ptr; + break; + } + } + op->orig_val = op->val; +} + int x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { @@ -809,31 +837,7 @@ modrm_done: case SrcNone: break; case SrcReg: - c->src.type = OP_REG; - if (c->d & ByteOp) { - c->src.ptr = - decode_register(c->modrm_reg, c->regs, - (rex_prefix == 0)); - c->src.val = c->src.orig_val = *(u8 *)c->src.ptr; - c->src.bytes = 1; - } else { - c->src.ptr = - decode_register(c->modrm_reg, c->regs, 0); - switch ((c->src.bytes = c->op_bytes)) { - case 2: - c->src.val = c->src.orig_val = - *(u16 *) c->src.ptr; - break; - case 4: - c->src.val = c->src.orig_val = - *(u32 *) c->src.ptr; - break; - case 8: - c->src.val = c->src.orig_val = - *(u64 *) c->src.ptr; - break; - } - } + decode_register_operand(&c->src, c, rex_prefix == 0, 0); break; case SrcMem16: c->src.bytes = 2; @@ -891,30 +895,8 @@ modrm_done: /* Special instructions do their own operand decoding. */ return 0; case DstReg: - c->dst.type = OP_REG; - if ((c->d & ByteOp) - && !(c->twobyte && - (c->b == 0xb6 || c->b == 0xb7))) { - c->dst.ptr = - decode_register(c->modrm_reg, c->regs, - (rex_prefix == 0)); - c->dst.val = *(u8 *) c->dst.ptr; - c->dst.bytes = 1; - } else { - c->dst.ptr = - decode_register(c->modrm_reg, c->regs, 0); - switch ((c->dst.bytes = c->op_bytes)) { - case 2: - c->dst.val = *(u16 *)c->dst.ptr; - break; - case 4: - c->dst.val = *(u32 *)c->dst.ptr; - break; - case 8: - c->dst.val = *(u64 *)c->dst.ptr; - break; - } - } + decode_register_operand(&c->dst, c, rex_prefix == 0, + c->twobyte && (c->b == 0xb6 || c->b == 0xb7)); break; case DstMem: if ((c->d & ModRM) && c->modrm_mod == 3) { -- cgit v1.2.3 From 33615aa956521923eab0552994b5961cd3034042 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 31 Oct 2007 11:15:56 +0200 Subject: KVM: x86 emulator: centralize decoding of one-byte register access insns Instructions like 'inc reg' that have the register operand encoded in the opcode are currently specially decoded. Extend decode_register_operand() to handle that case, indicated by having DstReg or SrcReg without ModRM. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 103 +++++++++++++++++++++------------------------- drivers/kvm/x86_emulate.h | 1 + 2 files changed, 47 insertions(+), 57 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 58ceb6616364..884e4a2e47e6 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -99,17 +99,13 @@ static u16 opcode_table[256] = { ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, 0, 0, 0, 0, /* 0x40 - 0x47 */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, /* 0x48 - 0x4F */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, /* 0x50 - 0x57 */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, /* 0x58 - 0x5F */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, /* 0x60 - 0x67 */ 0, 0, 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ , 0, 0, 0, 0, @@ -525,13 +521,17 @@ static void decode_register_operand(struct operand *op, int highbyte_regs, int inhibit_bytereg) { + unsigned reg = c->modrm_reg; + + if (!(c->d & ModRM)) + reg = (c->b & 7) | ((c->rex_prefix & 1) << 3); op->type = OP_REG; if ((c->d & ByteOp) && !inhibit_bytereg) { - op->ptr = decode_register(c->modrm_reg, c->regs, highbyte_regs); + op->ptr = decode_register(reg, c->regs, highbyte_regs); op->val = *(u8 *)op->ptr; op->bytes = 1; } else { - op->ptr = decode_register(c->modrm_reg, c->regs, 0); + op->ptr = decode_register(reg, c->regs, 0); op->bytes = c->op_bytes; switch (op->bytes) { case 2: @@ -552,7 +552,7 @@ int x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; - u8 sib, rex_prefix = 0; + u8 sib; int rc = 0; int mode = ctxt->mode; int index_reg = 0, base_reg = 0, scale, rip_relative = 0; @@ -616,7 +616,7 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) case 0x40 ... 0x4f: /* REX */ if (mode != X86EMUL_MODE_PROT64) goto done_prefixes; - rex_prefix = c->b; + c->rex_prefix = c->b; continue; case 0xf0: /* LOCK */ c->lock_prefix = 1; @@ -631,18 +631,18 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* Any legacy prefix after a REX prefix nullifies its effect. */ - rex_prefix = 0; + c->rex_prefix = 0; } done_prefixes: /* REX prefix. */ - if (rex_prefix) { - if (rex_prefix & 8) + if (c->rex_prefix) { + if (c->rex_prefix & 8) c->op_bytes = 8; /* REX.W */ - c->modrm_reg = (rex_prefix & 4) << 1; /* REX.R */ - index_reg = (rex_prefix & 2) << 2; /* REX.X */ - c->modrm_rm = base_reg = (rex_prefix & 1) << 3; /* REG.B */ + c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */ + index_reg = (c->rex_prefix & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */ } /* Opcode byte(s). */ @@ -837,7 +837,7 @@ modrm_done: case SrcNone: break; case SrcReg: - decode_register_operand(&c->src, c, rex_prefix == 0, 0); + decode_register_operand(&c->src, c, c->rex_prefix == 0, 0); break; case SrcMem16: c->src.bytes = 2; @@ -895,7 +895,7 @@ modrm_done: /* Special instructions do their own operand decoding. */ return 0; case DstReg: - decode_register_operand(&c->dst, c, rex_prefix == 0, + decode_register_operand(&c->dst, c, c->rex_prefix == 0, c->twobyte && (c->b == 0xb6 || c->b == 0xb7)); break; case DstMem: @@ -1258,6 +1258,32 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) cmp: /* cmp */ emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); break; + case 0x40 ... 0x47: /* inc r16/r32 */ + emulate_1op("inc", c->dst, ctxt->eflags); + break; + case 0x48 ... 0x4f: /* dec r16/r32 */ + emulate_1op("dec", c->dst, ctxt->eflags); + break; + case 0x50 ... 0x57: /* push reg */ + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->op_bytes); + c->dst.ptr = (void *) register_address( + ctxt->ss_base, c->regs[VCPU_REGS_RSP]); + break; + case 0x58 ... 0x5f: /* pop reg */ + pop_instruction: + if ((rc = ops->read_std(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), c->dst.ptr, + c->op_bytes, ctxt->vcpu)) != 0) + goto done; + + register_address_increment(c->regs[VCPU_REGS_RSP], + c->op_bytes); + c->dst.type = OP_NONE; /* Disable writeback. */ + break; case 0x63: /* movsxd */ if (ctxt->mode != X86EMUL_MODE_PROT64) goto cannot_emulate; @@ -1373,43 +1399,6 @@ special_insn: if (c->twobyte) goto twobyte_special_insn; switch (c->b) { - case 0x40 ... 0x47: /* inc r16/r32 */ - c->dst.bytes = c->op_bytes; - c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; - c->dst.val = *c->dst.ptr; - emulate_1op("inc", c->dst, ctxt->eflags); - break; - case 0x48 ... 0x4f: /* dec r16/r32 */ - c->dst.bytes = c->op_bytes; - c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; - c->dst.val = *c->dst.ptr; - emulate_1op("dec", c->dst, ctxt->eflags); - break; - case 0x50 ... 0x57: /* push reg */ - if (c->op_bytes == 2) - c->src.val = (u16) c->regs[c->b & 0x7]; - else - c->src.val = (u32) c->regs[c->b & 0x7]; - c->dst.type = OP_MEM; - c->dst.bytes = c->op_bytes; - c->dst.val = c->src.val; - register_address_increment(c->regs[VCPU_REGS_RSP], - -c->op_bytes); - c->dst.ptr = (void *) register_address( - ctxt->ss_base, c->regs[VCPU_REGS_RSP]); - break; - case 0x58 ... 0x5f: /* pop reg */ - c->dst.ptr = (unsigned long *)&c->regs[c->b & 0x7]; - pop_instruction: - if ((rc = ops->read_std(register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), c->dst.ptr, - c->op_bytes, ctxt->vcpu)) != 0) - goto done; - - register_address_increment(c->regs[VCPU_REGS_RSP], - c->op_bytes); - c->dst.type = OP_NONE; /* Disable writeback. */ - break; case 0x6a: /* push imm8 */ c->src.val = 0L; c->src.val = insn_fetch(s8, 1, c->eip); diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index f03b1287700a..e34868b42c33 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -126,6 +126,7 @@ struct decode_cache { u8 rep_prefix; u8 op_bytes; u8 ad_bytes; + u8 rex_prefix; struct operand src; struct operand dst; unsigned long *override_base; -- cgit v1.2.3 From 9f1ef3f8f5f8ade2561d969181b152c225b82a3e Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 31 Oct 2007 11:21:06 +0200 Subject: KVM: Simplify decode_register_operand() calling convention Now that rex_prefix is part of the decode cache, there is no need to pass it along. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 884e4a2e47e6..b352a6c4d4a2 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -518,10 +518,10 @@ static int test_cc(unsigned int condition, unsigned int flags) static void decode_register_operand(struct operand *op, struct decode_cache *c, - int highbyte_regs, int inhibit_bytereg) { unsigned reg = c->modrm_reg; + int highbyte_regs = c->rex_prefix == 0; if (!(c->d & ModRM)) reg = (c->b & 7) | ((c->rex_prefix & 1) << 3); @@ -837,7 +837,7 @@ modrm_done: case SrcNone: break; case SrcReg: - decode_register_operand(&c->src, c, c->rex_prefix == 0, 0); + decode_register_operand(&c->src, c, 0); break; case SrcMem16: c->src.bytes = 2; @@ -895,7 +895,7 @@ modrm_done: /* Special instructions do their own operand decoding. */ return 0; case DstReg: - decode_register_operand(&c->dst, c, c->rex_prefix == 0, + decode_register_operand(&c->dst, c, c->twobyte && (c->b == 0xb6 || c->b == 0xb7)); break; case DstMem: -- cgit v1.2.3 From 3b6fff198c2ae3ca87ee8c9103d52ac9e8f40dd5 Mon Sep 17 00:00:00 2001 From: Uri Lublin Date: Tue, 30 Oct 2007 10:42:09 +0200 Subject: KVM: Make mark_page_dirty() work for aliased pages too. Recommended by Izik Eidus. Signed-off-by: Uri Lublin Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 2025cdfb4593..cac66d505f57 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -796,11 +796,11 @@ int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) } EXPORT_SYMBOL_GPL(kvm_clear_guest); -/* WARNING: Does not work on aliased pages. */ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *memslot; + gfn = unalias_gfn(kvm, gfn); memslot = __gfn_to_memslot(kvm, gfn); if (memslot && memslot->dirty_bitmap) { unsigned long rel_gfn = gfn - memslot->base_gfn; -- cgit v1.2.3 From 1c73ef66502311d9aff7fed7d7f970288329c6cb Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 1 Nov 2007 06:31:28 +0200 Subject: KVM: x86 emulator: Hoist modrm and abs decoding into separate functions Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 337 ++++++++++++++++++++++++---------------------- 1 file changed, 177 insertions(+), 160 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index b352a6c4d4a2..64888a6edc27 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -548,14 +548,182 @@ static void decode_register_operand(struct operand *op, op->orig_val = op->val; } +static int decode_modrm(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + u8 sib; + int index_reg = 0, base_reg = 0, scale, rip_relative = 0; + int rc = 0; + + if (c->rex_prefix) { + c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */ + index_reg = (c->rex_prefix & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */ + } + + c->modrm = insn_fetch(u8, 1, c->eip); + c->modrm_mod |= (c->modrm & 0xc0) >> 6; + c->modrm_reg |= (c->modrm & 0x38) >> 3; + c->modrm_rm |= (c->modrm & 0x07); + c->modrm_ea = 0; + c->use_modrm_ea = 1; + + if (c->modrm_mod == 3) { + c->modrm_val = *(unsigned long *) + decode_register(c->modrm_rm, c->regs, c->d & ByteOp); + return rc; + } + + if (c->ad_bytes == 2) { + unsigned bx = c->regs[VCPU_REGS_RBX]; + unsigned bp = c->regs[VCPU_REGS_RBP]; + unsigned si = c->regs[VCPU_REGS_RSI]; + unsigned di = c->regs[VCPU_REGS_RDI]; + + /* 16-bit ModR/M decode. */ + switch (c->modrm_mod) { + case 0: + if (c->modrm_rm == 6) + c->modrm_ea += insn_fetch(u16, 2, c->eip); + break; + case 1: + c->modrm_ea += insn_fetch(s8, 1, c->eip); + break; + case 2: + c->modrm_ea += insn_fetch(u16, 2, c->eip); + break; + } + switch (c->modrm_rm) { + case 0: + c->modrm_ea += bx + si; + break; + case 1: + c->modrm_ea += bx + di; + break; + case 2: + c->modrm_ea += bp + si; + break; + case 3: + c->modrm_ea += bp + di; + break; + case 4: + c->modrm_ea += si; + break; + case 5: + c->modrm_ea += di; + break; + case 6: + if (c->modrm_mod != 0) + c->modrm_ea += bp; + break; + case 7: + c->modrm_ea += bx; + break; + } + if (c->modrm_rm == 2 || c->modrm_rm == 3 || + (c->modrm_rm == 6 && c->modrm_mod != 0)) + if (!c->override_base) + c->override_base = &ctxt->ss_base; + c->modrm_ea = (u16)c->modrm_ea; + } else { + /* 32/64-bit ModR/M decode. */ + switch (c->modrm_rm) { + case 4: + case 12: + sib = insn_fetch(u8, 1, c->eip); + index_reg |= (sib >> 3) & 7; + base_reg |= sib & 7; + scale = sib >> 6; + + switch (base_reg) { + case 5: + if (c->modrm_mod != 0) + c->modrm_ea += c->regs[base_reg]; + else + c->modrm_ea += + insn_fetch(s32, 4, c->eip); + break; + default: + c->modrm_ea += c->regs[base_reg]; + } + switch (index_reg) { + case 4: + break; + default: + c->modrm_ea += c->regs[index_reg] << scale; + } + break; + case 5: + if (c->modrm_mod != 0) + c->modrm_ea += c->regs[c->modrm_rm]; + else if (ctxt->mode == X86EMUL_MODE_PROT64) + rip_relative = 1; + break; + default: + c->modrm_ea += c->regs[c->modrm_rm]; + break; + } + switch (c->modrm_mod) { + case 0: + if (c->modrm_rm == 5) + c->modrm_ea += insn_fetch(s32, 4, c->eip); + break; + case 1: + c->modrm_ea += insn_fetch(s8, 1, c->eip); + break; + case 2: + c->modrm_ea += insn_fetch(s32, 4, c->eip); + break; + } + } + if (rip_relative) { + c->modrm_ea += c->eip; + switch (c->d & SrcMask) { + case SrcImmByte: + c->modrm_ea += 1; + break; + case SrcImm: + if (c->d & ByteOp) + c->modrm_ea += 1; + else + if (c->op_bytes == 8) + c->modrm_ea += 4; + else + c->modrm_ea += c->op_bytes; + } + } +done: + return rc; +} + +static int decode_abs(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc = 0; + + switch (c->ad_bytes) { + case 2: + c->modrm_ea = insn_fetch(u16, 2, c->eip); + break; + case 4: + c->modrm_ea = insn_fetch(u32, 4, c->eip); + break; + case 8: + c->modrm_ea = insn_fetch(u64, 8, c->eip); + break; + } +done: + return rc; +} + int x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { struct decode_cache *c = &ctxt->decode; - u8 sib; int rc = 0; int mode = ctxt->mode; - int index_reg = 0, base_reg = 0, scale, rip_relative = 0; /* Shadow copy of register state. Committed on successful emulation. */ @@ -637,13 +805,9 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) done_prefixes: /* REX prefix. */ - if (c->rex_prefix) { + if (c->rex_prefix) if (c->rex_prefix & 8) c->op_bytes = 8; /* REX.W */ - c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */ - index_reg = (c->rex_prefix & 2) << 2; /* REX.X */ - c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */ - } /* Opcode byte(s). */ c->d = opcode_table[c->b]; @@ -663,159 +827,12 @@ done_prefixes: } /* ModRM and SIB bytes. */ - if (c->d & ModRM) { - c->modrm = insn_fetch(u8, 1, c->eip); - c->modrm_mod |= (c->modrm & 0xc0) >> 6; - c->modrm_reg |= (c->modrm & 0x38) >> 3; - c->modrm_rm |= (c->modrm & 0x07); - c->modrm_ea = 0; - c->use_modrm_ea = 1; - - if (c->modrm_mod == 3) { - c->modrm_val = *(unsigned long *) - decode_register(c->modrm_rm, c->regs, c->d & ByteOp); - goto modrm_done; - } - - if (c->ad_bytes == 2) { - unsigned bx = c->regs[VCPU_REGS_RBX]; - unsigned bp = c->regs[VCPU_REGS_RBP]; - unsigned si = c->regs[VCPU_REGS_RSI]; - unsigned di = c->regs[VCPU_REGS_RDI]; - - /* 16-bit ModR/M decode. */ - switch (c->modrm_mod) { - case 0: - if (c->modrm_rm == 6) - c->modrm_ea += - insn_fetch(u16, 2, c->eip); - break; - case 1: - c->modrm_ea += insn_fetch(s8, 1, c->eip); - break; - case 2: - c->modrm_ea += insn_fetch(u16, 2, c->eip); - break; - } - switch (c->modrm_rm) { - case 0: - c->modrm_ea += bx + si; - break; - case 1: - c->modrm_ea += bx + di; - break; - case 2: - c->modrm_ea += bp + si; - break; - case 3: - c->modrm_ea += bp + di; - break; - case 4: - c->modrm_ea += si; - break; - case 5: - c->modrm_ea += di; - break; - case 6: - if (c->modrm_mod != 0) - c->modrm_ea += bp; - break; - case 7: - c->modrm_ea += bx; - break; - } - if (c->modrm_rm == 2 || c->modrm_rm == 3 || - (c->modrm_rm == 6 && c->modrm_mod != 0)) - if (!c->override_base) - c->override_base = &ctxt->ss_base; - c->modrm_ea = (u16)c->modrm_ea; - } else { - /* 32/64-bit ModR/M decode. */ - switch (c->modrm_rm) { - case 4: - case 12: - sib = insn_fetch(u8, 1, c->eip); - index_reg |= (sib >> 3) & 7; - base_reg |= sib & 7; - scale = sib >> 6; - - switch (base_reg) { - case 5: - if (c->modrm_mod != 0) - c->modrm_ea += - c->regs[base_reg]; - else - c->modrm_ea += - insn_fetch(s32, 4, c->eip); - break; - default: - c->modrm_ea += c->regs[base_reg]; - } - switch (index_reg) { - case 4: - break; - default: - c->modrm_ea += - c->regs[index_reg] << scale; - - } - break; - case 5: - if (c->modrm_mod != 0) - c->modrm_ea += c->regs[c->modrm_rm]; - else if (mode == X86EMUL_MODE_PROT64) - rip_relative = 1; - break; - default: - c->modrm_ea += c->regs[c->modrm_rm]; - break; - } - switch (c->modrm_mod) { - case 0: - if (c->modrm_rm == 5) - c->modrm_ea += - insn_fetch(s32, 4, c->eip); - break; - case 1: - c->modrm_ea += insn_fetch(s8, 1, c->eip); - break; - case 2: - c->modrm_ea += insn_fetch(s32, 4, c->eip); - break; - } - } - if (rip_relative) { - c->modrm_ea += c->eip; - switch (c->d & SrcMask) { - case SrcImmByte: - c->modrm_ea += 1; - break; - case SrcImm: - if (c->d & ByteOp) - c->modrm_ea += 1; - else - if (c->op_bytes == 8) - c->modrm_ea += 4; - else - c->modrm_ea += c->op_bytes; - } - } -modrm_done: - ; - } else if (c->d & MemAbs) { - switch (c->ad_bytes) { - case 2: - c->modrm_ea = insn_fetch(u16, 2, c->eip); - break; - case 4: - c->modrm_ea = insn_fetch(u32, 4, c->eip); - break; - case 8: - c->modrm_ea = insn_fetch(u64, 8, c->eip); - break; - } - - } + if (c->d & ModRM) + rc = decode_modrm(ctxt, ops); + else if (c->d & MemAbs) + rc = decode_abs(ctxt, ops); + if (rc) + goto done; if (!c->override_base) c->override_base = &ctxt->ds_base; -- cgit v1.2.3 From 417bc3041f5e66df1ce7f03d8fc481c3b12f250a Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Wed, 31 Oct 2007 17:24:23 -0500 Subject: KVM: Portability: Make exported debugfs data architecture-specific Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 7 +++++++ drivers/kvm/kvm_main.c | 25 ------------------------- drivers/kvm/x86.c | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 25 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 22317d6f66ae..d030a82966f5 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -761,4 +761,11 @@ static inline u32 get_rdx_init_val(void) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) +struct kvm_stats_debugfs_item { + const char *name; + int offset; + struct dentry *dentry; +}; +extern struct kvm_stats_debugfs_item debugfs_entries[]; + #endif diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index cac66d505f57..58a5f399ad85 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -63,31 +63,6 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_cache); static __read_mostly struct preempt_ops kvm_preempt_ops; -#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x) - -static struct kvm_stats_debugfs_item { - const char *name; - int offset; - struct dentry *dentry; -} debugfs_entries[] = { - { "pf_fixed", STAT_OFFSET(pf_fixed) }, - { "pf_guest", STAT_OFFSET(pf_guest) }, - { "tlb_flush", STAT_OFFSET(tlb_flush) }, - { "invlpg", STAT_OFFSET(invlpg) }, - { "exits", STAT_OFFSET(exits) }, - { "io_exits", STAT_OFFSET(io_exits) }, - { "mmio_exits", STAT_OFFSET(mmio_exits) }, - { "signal_exits", STAT_OFFSET(signal_exits) }, - { "irq_window", STAT_OFFSET(irq_window_exits) }, - { "halt_exits", STAT_OFFSET(halt_exits) }, - { "halt_wakeup", STAT_OFFSET(halt_wakeup) }, - { "request_irq", STAT_OFFSET(request_irq_exits) }, - { "irq_exits", STAT_OFFSET(irq_exits) }, - { "light_exits", STAT_OFFSET(light_exits) }, - { "efer_reload", STAT_OFFSET(efer_reload) }, - { NULL } -}; - static struct dentry *debugfs_dir; static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index f75e7d7d9ead..c1f10e58f4d2 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -40,6 +40,28 @@ #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) #define EFER_RESERVED_BITS 0xfffffffffffff2fe +#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x) + +struct kvm_stats_debugfs_item debugfs_entries[] = { + { "pf_fixed", STAT_OFFSET(pf_fixed) }, + { "pf_guest", STAT_OFFSET(pf_guest) }, + { "tlb_flush", STAT_OFFSET(tlb_flush) }, + { "invlpg", STAT_OFFSET(invlpg) }, + { "exits", STAT_OFFSET(exits) }, + { "io_exits", STAT_OFFSET(io_exits) }, + { "mmio_exits", STAT_OFFSET(mmio_exits) }, + { "signal_exits", STAT_OFFSET(signal_exits) }, + { "irq_window", STAT_OFFSET(irq_window_exits) }, + { "halt_exits", STAT_OFFSET(halt_exits) }, + { "halt_wakeup", STAT_OFFSET(halt_wakeup) }, + { "request_irq", STAT_OFFSET(request_irq_exits) }, + { "irq_exits", STAT_OFFSET(irq_exits) }, + { "light_exits", STAT_OFFSET(light_exits) }, + { "efer_reload", STAT_OFFSET(efer_reload) }, + { NULL } +}; + + unsigned long segment_base(u16 selector) { struct descriptor_table gdt; -- cgit v1.2.3 From 8776e5194f7bb847906e3561c4dba12ed66ebfb6 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Wed, 31 Oct 2007 17:24:24 -0500 Subject: KVM: Portability: Move x86 instruction emulation code to x86.c Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 177 +------------------------------------------------ drivers/kvm/x86.c | 175 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 176 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index d030a82966f5..ef2a6a8328ea 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -591,6 +591,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); void fx_init(struct kvm_vcpu *vcpu); +void kvm_vcpu_block(struct kvm_vcpu *vcpu); void kvm_resched(struct kvm_vcpu *vcpu); void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 58a5f399ad85..57573ebf02ba 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -789,7 +789,7 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) /* * The vCPU has executed a HLT instruction with in-kernel mode enabled. */ -static void kvm_vcpu_block(struct kvm_vcpu *vcpu) +void kvm_vcpu_block(struct kvm_vcpu *vcpu) { DECLARE_WAITQUEUE(wait, current); @@ -812,144 +812,6 @@ static void kvm_vcpu_block(struct kvm_vcpu *vcpu) remove_wait_queue(&vcpu->wq, &wait); } -int kvm_emulate_halt(struct kvm_vcpu *vcpu) -{ - ++vcpu->stat.halt_exits; - if (irqchip_in_kernel(vcpu->kvm)) { - vcpu->mp_state = VCPU_MP_STATE_HALTED; - kvm_vcpu_block(vcpu); - if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE) - return -EINTR; - return 1; - } else { - vcpu->run->exit_reason = KVM_EXIT_HLT; - return 0; - } -} -EXPORT_SYMBOL_GPL(kvm_emulate_halt); - -int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) -{ - unsigned long nr, a0, a1, a2, a3, ret; - - kvm_x86_ops->cache_regs(vcpu); - - nr = vcpu->regs[VCPU_REGS_RAX]; - a0 = vcpu->regs[VCPU_REGS_RBX]; - a1 = vcpu->regs[VCPU_REGS_RCX]; - a2 = vcpu->regs[VCPU_REGS_RDX]; - a3 = vcpu->regs[VCPU_REGS_RSI]; - - if (!is_long_mode(vcpu)) { - nr &= 0xFFFFFFFF; - a0 &= 0xFFFFFFFF; - a1 &= 0xFFFFFFFF; - a2 &= 0xFFFFFFFF; - a3 &= 0xFFFFFFFF; - } - - switch (nr) { - default: - ret = -KVM_ENOSYS; - break; - } - vcpu->regs[VCPU_REGS_RAX] = ret; - kvm_x86_ops->decache_regs(vcpu); - return 0; -} -EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); - -int kvm_fix_hypercall(struct kvm_vcpu *vcpu) -{ - char instruction[3]; - int ret = 0; - - mutex_lock(&vcpu->kvm->lock); - - /* - * Blow out the MMU to ensure that no other VCPU has an active mapping - * to ensure that the updated hypercall appears atomically across all - * VCPUs. - */ - kvm_mmu_zap_all(vcpu->kvm); - - kvm_x86_ops->cache_regs(vcpu); - kvm_x86_ops->patch_hypercall(vcpu, instruction); - if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu) - != X86EMUL_CONTINUE) - ret = -EFAULT; - - mutex_unlock(&vcpu->kvm->lock); - - return ret; -} - -static u64 mk_cr_64(u64 curr_cr, u32 new_val) -{ - return (curr_cr & ~((1ULL << 32) - 1)) | new_val; -} - -void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) -{ - struct descriptor_table dt = { limit, base }; - - kvm_x86_ops->set_gdt(vcpu, &dt); -} - -void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) -{ - struct descriptor_table dt = { limit, base }; - - kvm_x86_ops->set_idt(vcpu, &dt); -} - -void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, - unsigned long *rflags) -{ - lmsw(vcpu, msw); - *rflags = kvm_x86_ops->get_rflags(vcpu); -} - -unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) -{ - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - switch (cr) { - case 0: - return vcpu->cr0; - case 2: - return vcpu->cr2; - case 3: - return vcpu->cr3; - case 4: - return vcpu->cr4; - default: - vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); - return 0; - } -} - -void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, - unsigned long *rflags) -{ - switch (cr) { - case 0: - set_cr0(vcpu, mk_cr_64(vcpu->cr0, val)); - *rflags = kvm_x86_ops->get_rflags(vcpu); - break; - case 2: - vcpu->cr2 = val; - break; - case 3: - set_cr3(vcpu, val); - break; - case 4: - set_cr4(vcpu, mk_cr_64(vcpu->cr4, val)); - break; - default: - vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); - } -} - void kvm_resched(struct kvm_vcpu *vcpu) { if (!need_resched()) @@ -958,43 +820,6 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) -{ - int i; - u32 function; - struct kvm_cpuid_entry *e, *best; - - kvm_x86_ops->cache_regs(vcpu); - function = vcpu->regs[VCPU_REGS_RAX]; - vcpu->regs[VCPU_REGS_RAX] = 0; - vcpu->regs[VCPU_REGS_RBX] = 0; - vcpu->regs[VCPU_REGS_RCX] = 0; - vcpu->regs[VCPU_REGS_RDX] = 0; - best = NULL; - for (i = 0; i < vcpu->cpuid_nent; ++i) { - e = &vcpu->cpuid_entries[i]; - if (e->function == function) { - best = e; - break; - } - /* - * Both basic or both extended? - */ - if (((e->function ^ function) & 0x80000000) == 0) - if (!best || e->function > best->function) - best = e; - } - if (best) { - vcpu->regs[VCPU_REGS_RAX] = best->eax; - vcpu->regs[VCPU_REGS_RBX] = best->ebx; - vcpu->regs[VCPU_REGS_RCX] = best->ecx; - vcpu->regs[VCPU_REGS_RDX] = best->edx; - } - kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->skip_emulated_instruction(vcpu); -} -EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); - /* * Check if userspace requested an interrupt window, and that the * interrupt window is open. diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c1f10e58f4d2..2cf7ebab50f4 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1610,3 +1610,178 @@ __init void kvm_arch_init(void) { kvm_init_msr_list(); } + +int kvm_emulate_halt(struct kvm_vcpu *vcpu) +{ + ++vcpu->stat.halt_exits; + if (irqchip_in_kernel(vcpu->kvm)) { + vcpu->mp_state = VCPU_MP_STATE_HALTED; + kvm_vcpu_block(vcpu); + if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE) + return -EINTR; + return 1; + } else { + vcpu->run->exit_reason = KVM_EXIT_HLT; + return 0; + } +} +EXPORT_SYMBOL_GPL(kvm_emulate_halt); + +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) +{ + unsigned long nr, a0, a1, a2, a3, ret; + + kvm_x86_ops->cache_regs(vcpu); + + nr = vcpu->regs[VCPU_REGS_RAX]; + a0 = vcpu->regs[VCPU_REGS_RBX]; + a1 = vcpu->regs[VCPU_REGS_RCX]; + a2 = vcpu->regs[VCPU_REGS_RDX]; + a3 = vcpu->regs[VCPU_REGS_RSI]; + + if (!is_long_mode(vcpu)) { + nr &= 0xFFFFFFFF; + a0 &= 0xFFFFFFFF; + a1 &= 0xFFFFFFFF; + a2 &= 0xFFFFFFFF; + a3 &= 0xFFFFFFFF; + } + + switch (nr) { + default: + ret = -KVM_ENOSYS; + break; + } + vcpu->regs[VCPU_REGS_RAX] = ret; + kvm_x86_ops->decache_regs(vcpu); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu) +{ + char instruction[3]; + int ret = 0; + + mutex_lock(&vcpu->kvm->lock); + + /* + * Blow out the MMU to ensure that no other VCPU has an active mapping + * to ensure that the updated hypercall appears atomically across all + * VCPUs. + */ + kvm_mmu_zap_all(vcpu->kvm); + + kvm_x86_ops->cache_regs(vcpu); + kvm_x86_ops->patch_hypercall(vcpu, instruction); + if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu) + != X86EMUL_CONTINUE) + ret = -EFAULT; + + mutex_unlock(&vcpu->kvm->lock); + + return ret; +} + +static u64 mk_cr_64(u64 curr_cr, u32 new_val) +{ + return (curr_cr & ~((1ULL << 32) - 1)) | new_val; +} + +void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) +{ + struct descriptor_table dt = { limit, base }; + + kvm_x86_ops->set_gdt(vcpu, &dt); +} + +void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) +{ + struct descriptor_table dt = { limit, base }; + + kvm_x86_ops->set_idt(vcpu, &dt); +} + +void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, + unsigned long *rflags) +{ + lmsw(vcpu, msw); + *rflags = kvm_x86_ops->get_rflags(vcpu); +} + +unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) +{ + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + switch (cr) { + case 0: + return vcpu->cr0; + case 2: + return vcpu->cr2; + case 3: + return vcpu->cr3; + case 4: + return vcpu->cr4; + default: + vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); + return 0; + } +} + +void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, + unsigned long *rflags) +{ + switch (cr) { + case 0: + set_cr0(vcpu, mk_cr_64(vcpu->cr0, val)); + *rflags = kvm_x86_ops->get_rflags(vcpu); + break; + case 2: + vcpu->cr2 = val; + break; + case 3: + set_cr3(vcpu, val); + break; + case 4: + set_cr4(vcpu, mk_cr_64(vcpu->cr4, val)); + break; + default: + vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); + } +} + +void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) +{ + int i; + u32 function; + struct kvm_cpuid_entry *e, *best; + + kvm_x86_ops->cache_regs(vcpu); + function = vcpu->regs[VCPU_REGS_RAX]; + vcpu->regs[VCPU_REGS_RAX] = 0; + vcpu->regs[VCPU_REGS_RBX] = 0; + vcpu->regs[VCPU_REGS_RCX] = 0; + vcpu->regs[VCPU_REGS_RDX] = 0; + best = NULL; + for (i = 0; i < vcpu->cpuid_nent; ++i) { + e = &vcpu->cpuid_entries[i]; + if (e->function == function) { + best = e; + break; + } + /* + * Both basic or both extended? + */ + if (((e->function ^ function) & 0x80000000) == 0) + if (!best || e->function > best->function) + best = e; + } + if (best) { + vcpu->regs[VCPU_REGS_RAX] = best->eax; + vcpu->regs[VCPU_REGS_RBX] = best->ebx; + vcpu->regs[VCPU_REGS_RCX] = best->ecx; + vcpu->regs[VCPU_REGS_RDX] = best->edx; + } + kvm_x86_ops->decache_regs(vcpu); + kvm_x86_ops->skip_emulated_instruction(vcpu); +} +EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); -- cgit v1.2.3 From d075206073286dca84768137af0a0bf3d11f0663 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Wed, 31 Oct 2007 17:24:25 -0500 Subject: KVM: Portability: Move x86 FPU handling to x86.c Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 ++ drivers/kvm/kvm_main.c | 107 +------------------------------------------------ drivers/kvm/x86.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 105 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ef2a6a8328ea..469ca42c2a19 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -633,6 +633,9 @@ long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); void kvm_arch_destroy_vm(struct kvm *kvm); +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); + __init void kvm_arch_init(void); static inline void kvm_guest_enter(void) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 57573ebf02ba..7230f48ba08e 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -73,28 +73,6 @@ static inline int valid_vcpu(int n) return likely(n >= 0 && n < KVM_MAX_VCPUS); } -void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) -{ - if (!vcpu->fpu_active || vcpu->guest_fpu_loaded) - return; - - vcpu->guest_fpu_loaded = 1; - fx_save(&vcpu->host_fx_image); - fx_restore(&vcpu->guest_fx_image); -} -EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); - -void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) -{ - if (!vcpu->guest_fpu_loaded) - return; - - vcpu->guest_fpu_loaded = 0; - fx_save(&vcpu->guest_fx_image); - fx_restore(&vcpu->host_fx_image); -} -EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); - /* * Switches to specified vcpu, until a matching vcpu_put() */ @@ -294,26 +272,6 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) return 0; } -void fx_init(struct kvm_vcpu *vcpu) -{ - unsigned after_mxcsr_mask; - - /* Initialize guest FPU by resetting ours and saving into guest's */ - preempt_disable(); - fx_save(&vcpu->host_fx_image); - fpu_init(); - fx_save(&vcpu->guest_fx_image); - fx_restore(&vcpu->host_fx_image); - preempt_enable(); - - vcpu->cr0 |= X86_CR0_ET; - after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space); - vcpu->guest_fx_image.mxcsr = 0x1f80; - memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask, - 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask); -} -EXPORT_SYMBOL_GPL(fx_init); - /* * Allocate some memory and give it an address in the guest physical address * space. @@ -1422,67 +1380,6 @@ static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) return 0; } -/* - * fxsave fpu state. Taken from x86_64/processor.h. To be killed when - * we have asm/x86/processor.h - */ -struct fxsave { - u16 cwd; - u16 swd; - u16 twd; - u16 fop; - u64 rip; - u64 rdp; - u32 mxcsr; - u32 mxcsr_mask; - u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */ -#ifdef CONFIG_X86_64 - u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */ -#else - u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */ -#endif -}; - -static int kvm_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) -{ - struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; - - vcpu_load(vcpu); - - memcpy(fpu->fpr, fxsave->st_space, 128); - fpu->fcw = fxsave->cwd; - fpu->fsw = fxsave->swd; - fpu->ftwx = fxsave->twd; - fpu->last_opcode = fxsave->fop; - fpu->last_ip = fxsave->rip; - fpu->last_dp = fxsave->rdp; - memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space); - - vcpu_put(vcpu); - - return 0; -} - -static int kvm_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) -{ - struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; - - vcpu_load(vcpu); - - memcpy(fxsave->st_space, fpu->fpr, 128); - fxsave->cwd = fpu->fcw; - fxsave->swd = fpu->fsw; - fxsave->twd = fpu->ftwx; - fxsave->fop = fpu->last_opcode; - fxsave->rip = fpu->last_ip; - fxsave->rdp = fpu->last_dp; - memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space); - - vcpu_put(vcpu); - - return 0; -} - static long kvm_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1613,7 +1510,7 @@ static long kvm_vcpu_ioctl(struct file *filp, struct kvm_fpu fpu; memset(&fpu, 0, sizeof fpu); - r = kvm_vcpu_ioctl_get_fpu(vcpu, &fpu); + r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu); if (r) goto out; r = -EFAULT; @@ -1628,7 +1525,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&fpu, argp, sizeof fpu)) goto out; - r = kvm_vcpu_ioctl_set_fpu(vcpu, &fpu); + r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu); if (r) goto out; r = 0; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 2cf7ebab50f4..ef1661f10b48 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1785,3 +1785,106 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) kvm_x86_ops->skip_emulated_instruction(vcpu); } EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); + +/* + * fxsave fpu state. Taken from x86_64/processor.h. To be killed when + * we have asm/x86/processor.h + */ +struct fxsave { + u16 cwd; + u16 swd; + u16 twd; + u16 fop; + u64 rip; + u64 rdp; + u32 mxcsr; + u32 mxcsr_mask; + u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */ +#ifdef CONFIG_X86_64 + u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */ +#else + u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */ +#endif +}; + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; + + vcpu_load(vcpu); + + memcpy(fpu->fpr, fxsave->st_space, 128); + fpu->fcw = fxsave->cwd; + fpu->fsw = fxsave->swd; + fpu->ftwx = fxsave->twd; + fpu->last_opcode = fxsave->fop; + fpu->last_ip = fxsave->rip; + fpu->last_dp = fxsave->rdp; + memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; + + vcpu_load(vcpu); + + memcpy(fxsave->st_space, fpu->fpr, 128); + fxsave->cwd = fpu->fcw; + fxsave->swd = fpu->fsw; + fxsave->twd = fpu->ftwx; + fxsave->fop = fpu->last_opcode; + fxsave->rip = fpu->last_ip; + fxsave->rdp = fpu->last_dp; + memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space); + + vcpu_put(vcpu); + + return 0; +} + +void fx_init(struct kvm_vcpu *vcpu) +{ + unsigned after_mxcsr_mask; + + /* Initialize guest FPU by resetting ours and saving into guest's */ + preempt_disable(); + fx_save(&vcpu->host_fx_image); + fpu_init(); + fx_save(&vcpu->guest_fx_image); + fx_restore(&vcpu->host_fx_image); + preempt_enable(); + + vcpu->cr0 |= X86_CR0_ET; + after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space); + vcpu->guest_fx_image.mxcsr = 0x1f80; + memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask, + 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask); +} +EXPORT_SYMBOL_GPL(fx_init); + +void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) +{ + if (!vcpu->fpu_active || vcpu->guest_fpu_loaded) + return; + + vcpu->guest_fpu_loaded = 1; + fx_save(&vcpu->host_fx_image); + fx_restore(&vcpu->guest_fx_image); +} +EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); + +void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) +{ + if (!vcpu->guest_fpu_loaded) + return; + + vcpu->guest_fpu_loaded = 0; + fx_save(&vcpu->guest_fx_image); + fx_restore(&vcpu->host_fx_image); +} +EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); -- cgit v1.2.3 From b6c7a5dccf9471f4891df722dbd0700ce56eb2e2 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Thu, 1 Nov 2007 14:16:10 -0500 Subject: KVM: Portability: Move x86 vcpu ioctl handlers to x86.c Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 10 ++ drivers/kvm/kvm_main.c | 442 +------------------------------------------------ drivers/kvm/x86.c | 427 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 443 insertions(+), 436 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 469ca42c2a19..3f5ffc37480d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -636,6 +636,16 @@ void kvm_arch_destroy_vm(struct kvm *kvm); int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs); +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs); +int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg); +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); + __init void kvm_arch_init(void); static inline void kvm_guest_enter(void) diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7230f48ba08e..8665531d9287 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -778,422 +778,6 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -/* - * Check if userspace requested an interrupt window, and that the - * interrupt window is open. - * - * No need to exit to userspace if we already have an interrupt queued. - */ -static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - return (!vcpu->irq_summary && - kvm_run->request_interrupt_window && - vcpu->interrupt_window_open && - (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF)); -} - -static void post_kvm_run_save(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0; - kvm_run->cr8 = get_cr8(vcpu); - kvm_run->apic_base = kvm_get_apic_base(vcpu); - if (irqchip_in_kernel(vcpu->kvm)) - kvm_run->ready_for_interrupt_injection = 1; - else - kvm_run->ready_for_interrupt_injection = - (vcpu->interrupt_window_open && - vcpu->irq_summary == 0); -} - -static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - int r; - - if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { - pr_debug("vcpu %d received sipi with vector # %x\n", - vcpu->vcpu_id, vcpu->sipi_vector); - kvm_lapic_reset(vcpu); - r = kvm_x86_ops->vcpu_reset(vcpu); - if (r) - return r; - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; - } - -preempted: - if (vcpu->guest_debug.enabled) - kvm_x86_ops->guest_debug_pre(vcpu); - -again: - r = kvm_mmu_reload(vcpu); - if (unlikely(r)) - goto out; - - kvm_inject_pending_timer_irqs(vcpu); - - preempt_disable(); - - kvm_x86_ops->prepare_guest_switch(vcpu); - kvm_load_guest_fpu(vcpu); - - local_irq_disable(); - - if (signal_pending(current)) { - local_irq_enable(); - preempt_enable(); - r = -EINTR; - kvm_run->exit_reason = KVM_EXIT_INTR; - ++vcpu->stat.signal_exits; - goto out; - } - - if (irqchip_in_kernel(vcpu->kvm)) - kvm_x86_ops->inject_pending_irq(vcpu); - else if (!vcpu->mmio_read_completed) - kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); - - vcpu->guest_mode = 1; - kvm_guest_enter(); - - if (vcpu->requests) - if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) - kvm_x86_ops->tlb_flush(vcpu); - - kvm_x86_ops->run(vcpu, kvm_run); - - vcpu->guest_mode = 0; - local_irq_enable(); - - ++vcpu->stat.exits; - - /* - * We must have an instruction between local_irq_enable() and - * kvm_guest_exit(), so the timer interrupt isn't delayed by - * the interrupt shadow. The stat.exits increment will do nicely. - * But we need to prevent reordering, hence this barrier(): - */ - barrier(); - - kvm_guest_exit(); - - preempt_enable(); - - /* - * Profile KVM exit RIPs: - */ - if (unlikely(prof_on == KVM_PROFILING)) { - kvm_x86_ops->cache_regs(vcpu); - profile_hit(KVM_PROFILING, (void *)vcpu->rip); - } - - r = kvm_x86_ops->handle_exit(kvm_run, vcpu); - - if (r > 0) { - if (dm_request_for_irq_injection(vcpu, kvm_run)) { - r = -EINTR; - kvm_run->exit_reason = KVM_EXIT_INTR; - ++vcpu->stat.request_irq_exits; - goto out; - } - if (!need_resched()) { - ++vcpu->stat.light_exits; - goto again; - } - } - -out: - if (r > 0) { - kvm_resched(vcpu); - goto preempted; - } - - post_kvm_run_save(vcpu, kvm_run); - - return r; -} - - -static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - int r; - sigset_t sigsaved; - - vcpu_load(vcpu); - - if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) { - kvm_vcpu_block(vcpu); - vcpu_put(vcpu); - return -EAGAIN; - } - - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); - - /* re-sync apic's tpr */ - if (!irqchip_in_kernel(vcpu->kvm)) - set_cr8(vcpu, kvm_run->cr8); - - if (vcpu->pio.cur_count) { - r = complete_pio(vcpu); - if (r) - goto out; - } -#if CONFIG_HAS_IOMEM - if (vcpu->mmio_needed) { - memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); - vcpu->mmio_read_completed = 1; - vcpu->mmio_needed = 0; - r = emulate_instruction(vcpu, kvm_run, - vcpu->mmio_fault_cr2, 0, 1); - if (r == EMULATE_DO_MMIO) { - /* - * Read-modify-write. Back to userspace. - */ - r = 0; - goto out; - } - } -#endif - if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { - kvm_x86_ops->cache_regs(vcpu); - vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; - kvm_x86_ops->decache_regs(vcpu); - } - - r = __vcpu_run(vcpu, kvm_run); - -out: - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &sigsaved, NULL); - - vcpu_put(vcpu); - return r; -} - -static int kvm_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, - struct kvm_regs *regs) -{ - vcpu_load(vcpu); - - kvm_x86_ops->cache_regs(vcpu); - - regs->rax = vcpu->regs[VCPU_REGS_RAX]; - regs->rbx = vcpu->regs[VCPU_REGS_RBX]; - regs->rcx = vcpu->regs[VCPU_REGS_RCX]; - regs->rdx = vcpu->regs[VCPU_REGS_RDX]; - regs->rsi = vcpu->regs[VCPU_REGS_RSI]; - regs->rdi = vcpu->regs[VCPU_REGS_RDI]; - regs->rsp = vcpu->regs[VCPU_REGS_RSP]; - regs->rbp = vcpu->regs[VCPU_REGS_RBP]; -#ifdef CONFIG_X86_64 - regs->r8 = vcpu->regs[VCPU_REGS_R8]; - regs->r9 = vcpu->regs[VCPU_REGS_R9]; - regs->r10 = vcpu->regs[VCPU_REGS_R10]; - regs->r11 = vcpu->regs[VCPU_REGS_R11]; - regs->r12 = vcpu->regs[VCPU_REGS_R12]; - regs->r13 = vcpu->regs[VCPU_REGS_R13]; - regs->r14 = vcpu->regs[VCPU_REGS_R14]; - regs->r15 = vcpu->regs[VCPU_REGS_R15]; -#endif - - regs->rip = vcpu->rip; - regs->rflags = kvm_x86_ops->get_rflags(vcpu); - - /* - * Don't leak debug flags in case they were set for guest debugging - */ - if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep) - regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); - - vcpu_put(vcpu); - - return 0; -} - -static int kvm_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, - struct kvm_regs *regs) -{ - vcpu_load(vcpu); - - vcpu->regs[VCPU_REGS_RAX] = regs->rax; - vcpu->regs[VCPU_REGS_RBX] = regs->rbx; - vcpu->regs[VCPU_REGS_RCX] = regs->rcx; - vcpu->regs[VCPU_REGS_RDX] = regs->rdx; - vcpu->regs[VCPU_REGS_RSI] = regs->rsi; - vcpu->regs[VCPU_REGS_RDI] = regs->rdi; - vcpu->regs[VCPU_REGS_RSP] = regs->rsp; - vcpu->regs[VCPU_REGS_RBP] = regs->rbp; -#ifdef CONFIG_X86_64 - vcpu->regs[VCPU_REGS_R8] = regs->r8; - vcpu->regs[VCPU_REGS_R9] = regs->r9; - vcpu->regs[VCPU_REGS_R10] = regs->r10; - vcpu->regs[VCPU_REGS_R11] = regs->r11; - vcpu->regs[VCPU_REGS_R12] = regs->r12; - vcpu->regs[VCPU_REGS_R13] = regs->r13; - vcpu->regs[VCPU_REGS_R14] = regs->r14; - vcpu->regs[VCPU_REGS_R15] = regs->r15; -#endif - - vcpu->rip = regs->rip; - kvm_x86_ops->set_rflags(vcpu, regs->rflags); - - kvm_x86_ops->decache_regs(vcpu); - - vcpu_put(vcpu); - - return 0; -} - -static void get_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - return kvm_x86_ops->get_segment(vcpu, var, seg); -} - -static int kvm_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - struct descriptor_table dt; - int pending_vec; - - vcpu_load(vcpu); - - get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - get_segment(vcpu, &sregs->es, VCPU_SREG_ES); - get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_x86_ops->get_idt(vcpu, &dt); - sregs->idt.limit = dt.limit; - sregs->idt.base = dt.base; - kvm_x86_ops->get_gdt(vcpu, &dt); - sregs->gdt.limit = dt.limit; - sregs->gdt.base = dt.base; - - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - sregs->cr0 = vcpu->cr0; - sregs->cr2 = vcpu->cr2; - sregs->cr3 = vcpu->cr3; - sregs->cr4 = vcpu->cr4; - sregs->cr8 = get_cr8(vcpu); - sregs->efer = vcpu->shadow_efer; - sregs->apic_base = kvm_get_apic_base(vcpu); - - if (irqchip_in_kernel(vcpu->kvm)) { - memset(sregs->interrupt_bitmap, 0, - sizeof sregs->interrupt_bitmap); - pending_vec = kvm_x86_ops->get_irq(vcpu); - if (pending_vec >= 0) - set_bit(pending_vec, - (unsigned long *)sregs->interrupt_bitmap); - } else - memcpy(sregs->interrupt_bitmap, vcpu->irq_pending, - sizeof sregs->interrupt_bitmap); - - vcpu_put(vcpu); - - return 0; -} - -static void set_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - return kvm_x86_ops->set_segment(vcpu, var, seg); -} - -static int kvm_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - int mmu_reset_needed = 0; - int i, pending_vec, max_bits; - struct descriptor_table dt; - - vcpu_load(vcpu); - - dt.limit = sregs->idt.limit; - dt.base = sregs->idt.base; - kvm_x86_ops->set_idt(vcpu, &dt); - dt.limit = sregs->gdt.limit; - dt.base = sregs->gdt.base; - kvm_x86_ops->set_gdt(vcpu, &dt); - - vcpu->cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->cr3 != sregs->cr3; - vcpu->cr3 = sregs->cr3; - - set_cr8(vcpu, sregs->cr8); - - mmu_reset_needed |= vcpu->shadow_efer != sregs->efer; -#ifdef CONFIG_X86_64 - kvm_x86_ops->set_efer(vcpu, sregs->efer); -#endif - kvm_set_apic_base(vcpu, sregs->apic_base); - - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - - mmu_reset_needed |= vcpu->cr0 != sregs->cr0; - vcpu->cr0 = sregs->cr0; - kvm_x86_ops->set_cr0(vcpu, sregs->cr0); - - mmu_reset_needed |= vcpu->cr4 != sregs->cr4; - kvm_x86_ops->set_cr4(vcpu, sregs->cr4); - if (!is_long_mode(vcpu) && is_pae(vcpu)) - load_pdptrs(vcpu, vcpu->cr3); - - if (mmu_reset_needed) - kvm_mmu_reset_context(vcpu); - - if (!irqchip_in_kernel(vcpu->kvm)) { - memcpy(vcpu->irq_pending, sregs->interrupt_bitmap, - sizeof vcpu->irq_pending); - vcpu->irq_summary = 0; - for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i) - if (vcpu->irq_pending[i]) - __set_bit(i, &vcpu->irq_summary); - } else { - max_bits = (sizeof sregs->interrupt_bitmap) << 3; - pending_vec = find_first_bit( - (const unsigned long *)sregs->interrupt_bitmap, - max_bits); - /* Only pending external irq is handled here */ - if (pending_vec < max_bits) { - kvm_x86_ops->set_irq(vcpu, pending_vec); - pr_debug("Set back pending irq %d\n", - pending_vec); - } - } - - set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - set_segment(vcpu, &sregs->es, VCPU_SREG_ES); - set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - vcpu_put(vcpu); - - return 0; -} - -void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) -{ - struct kvm_segment cs; - - get_segment(vcpu, &cs, VCPU_SREG_CS); - *db = cs.db; - *l = cs.l; -} -EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); - /* * Translate a guest virtual address to a guest physical address. */ @@ -1233,20 +817,6 @@ static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, return 0; } -static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, - struct kvm_debug_guest *dbg) -{ - int r; - - vcpu_load(vcpu); - - r = kvm_x86_ops->set_guest_debug(vcpu, dbg); - - vcpu_put(vcpu); - - return r; -} - static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma, unsigned long address, int *type) @@ -1392,13 +962,13 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EINVAL; if (arg) goto out; - r = kvm_vcpu_ioctl_run(vcpu, vcpu->run); + r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); break; case KVM_GET_REGS: { struct kvm_regs kvm_regs; memset(&kvm_regs, 0, sizeof kvm_regs); - r = kvm_vcpu_ioctl_get_regs(vcpu, &kvm_regs); + r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs); if (r) goto out; r = -EFAULT; @@ -1413,7 +983,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs)) goto out; - r = kvm_vcpu_ioctl_set_regs(vcpu, &kvm_regs); + r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs); if (r) goto out; r = 0; @@ -1423,7 +993,7 @@ static long kvm_vcpu_ioctl(struct file *filp, struct kvm_sregs kvm_sregs; memset(&kvm_sregs, 0, sizeof kvm_sregs); - r = kvm_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs); + r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs); if (r) goto out; r = -EFAULT; @@ -1438,7 +1008,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs)) goto out; - r = kvm_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs); + r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs); if (r) goto out; r = 0; @@ -1477,7 +1047,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&dbg, argp, sizeof dbg)) goto out; - r = kvm_vcpu_ioctl_debug_guest(vcpu, &dbg); + r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg); if (r) goto out; r = 0; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index ef1661f10b48..394da6605364 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1786,6 +1786,433 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); +/* + * Check if userspace requested an interrupt window, and that the + * interrupt window is open. + * + * No need to exit to userspace if we already have an interrupt queued. + */ +static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + return (!vcpu->irq_summary && + kvm_run->request_interrupt_window && + vcpu->interrupt_window_open && + (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF)); +} + +static void post_kvm_run_save(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0; + kvm_run->cr8 = get_cr8(vcpu); + kvm_run->apic_base = kvm_get_apic_base(vcpu); + if (irqchip_in_kernel(vcpu->kvm)) + kvm_run->ready_for_interrupt_injection = 1; + else + kvm_run->ready_for_interrupt_injection = + (vcpu->interrupt_window_open && + vcpu->irq_summary == 0); +} + +static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + int r; + + if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { + pr_debug("vcpu %d received sipi with vector # %x\n", + vcpu->vcpu_id, vcpu->sipi_vector); + kvm_lapic_reset(vcpu); + r = kvm_x86_ops->vcpu_reset(vcpu); + if (r) + return r; + vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + } + +preempted: + if (vcpu->guest_debug.enabled) + kvm_x86_ops->guest_debug_pre(vcpu); + +again: + r = kvm_mmu_reload(vcpu); + if (unlikely(r)) + goto out; + + kvm_inject_pending_timer_irqs(vcpu); + + preempt_disable(); + + kvm_x86_ops->prepare_guest_switch(vcpu); + kvm_load_guest_fpu(vcpu); + + local_irq_disable(); + + if (signal_pending(current)) { + local_irq_enable(); + preempt_enable(); + r = -EINTR; + kvm_run->exit_reason = KVM_EXIT_INTR; + ++vcpu->stat.signal_exits; + goto out; + } + + if (irqchip_in_kernel(vcpu->kvm)) + kvm_x86_ops->inject_pending_irq(vcpu); + else if (!vcpu->mmio_read_completed) + kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); + + vcpu->guest_mode = 1; + kvm_guest_enter(); + + if (vcpu->requests) + if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) + kvm_x86_ops->tlb_flush(vcpu); + + kvm_x86_ops->run(vcpu, kvm_run); + + vcpu->guest_mode = 0; + local_irq_enable(); + + ++vcpu->stat.exits; + + /* + * We must have an instruction between local_irq_enable() and + * kvm_guest_exit(), so the timer interrupt isn't delayed by + * the interrupt shadow. The stat.exits increment will do nicely. + * But we need to prevent reordering, hence this barrier(): + */ + barrier(); + + kvm_guest_exit(); + + preempt_enable(); + + /* + * Profile KVM exit RIPs: + */ + if (unlikely(prof_on == KVM_PROFILING)) { + kvm_x86_ops->cache_regs(vcpu); + profile_hit(KVM_PROFILING, (void *)vcpu->rip); + } + + r = kvm_x86_ops->handle_exit(kvm_run, vcpu); + + if (r > 0) { + if (dm_request_for_irq_injection(vcpu, kvm_run)) { + r = -EINTR; + kvm_run->exit_reason = KVM_EXIT_INTR; + ++vcpu->stat.request_irq_exits; + goto out; + } + if (!need_resched()) { + ++vcpu->stat.light_exits; + goto again; + } + } + +out: + if (r > 0) { + kvm_resched(vcpu); + goto preempted; + } + + post_kvm_run_save(vcpu, kvm_run); + + return r; +} + +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + int r; + sigset_t sigsaved; + + vcpu_load(vcpu); + + if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) { + kvm_vcpu_block(vcpu); + vcpu_put(vcpu); + return -EAGAIN; + } + + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); + + /* re-sync apic's tpr */ + if (!irqchip_in_kernel(vcpu->kvm)) + set_cr8(vcpu, kvm_run->cr8); + + if (vcpu->pio.cur_count) { + r = complete_pio(vcpu); + if (r) + goto out; + } +#if CONFIG_HAS_IOMEM + if (vcpu->mmio_needed) { + memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); + vcpu->mmio_read_completed = 1; + vcpu->mmio_needed = 0; + r = emulate_instruction(vcpu, kvm_run, + vcpu->mmio_fault_cr2, 0, 1); + if (r == EMULATE_DO_MMIO) { + /* + * Read-modify-write. Back to userspace. + */ + r = 0; + goto out; + } + } +#endif + if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { + kvm_x86_ops->cache_regs(vcpu); + vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; + kvm_x86_ops->decache_regs(vcpu); + } + + r = __vcpu_run(vcpu, kvm_run); + +out: + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); + + vcpu_put(vcpu); + return r; +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu_load(vcpu); + + kvm_x86_ops->cache_regs(vcpu); + + regs->rax = vcpu->regs[VCPU_REGS_RAX]; + regs->rbx = vcpu->regs[VCPU_REGS_RBX]; + regs->rcx = vcpu->regs[VCPU_REGS_RCX]; + regs->rdx = vcpu->regs[VCPU_REGS_RDX]; + regs->rsi = vcpu->regs[VCPU_REGS_RSI]; + regs->rdi = vcpu->regs[VCPU_REGS_RDI]; + regs->rsp = vcpu->regs[VCPU_REGS_RSP]; + regs->rbp = vcpu->regs[VCPU_REGS_RBP]; +#ifdef CONFIG_X86_64 + regs->r8 = vcpu->regs[VCPU_REGS_R8]; + regs->r9 = vcpu->regs[VCPU_REGS_R9]; + regs->r10 = vcpu->regs[VCPU_REGS_R10]; + regs->r11 = vcpu->regs[VCPU_REGS_R11]; + regs->r12 = vcpu->regs[VCPU_REGS_R12]; + regs->r13 = vcpu->regs[VCPU_REGS_R13]; + regs->r14 = vcpu->regs[VCPU_REGS_R14]; + regs->r15 = vcpu->regs[VCPU_REGS_R15]; +#endif + + regs->rip = vcpu->rip; + regs->rflags = kvm_x86_ops->get_rflags(vcpu); + + /* + * Don't leak debug flags in case they were set for guest debugging + */ + if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep) + regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu_load(vcpu); + + vcpu->regs[VCPU_REGS_RAX] = regs->rax; + vcpu->regs[VCPU_REGS_RBX] = regs->rbx; + vcpu->regs[VCPU_REGS_RCX] = regs->rcx; + vcpu->regs[VCPU_REGS_RDX] = regs->rdx; + vcpu->regs[VCPU_REGS_RSI] = regs->rsi; + vcpu->regs[VCPU_REGS_RDI] = regs->rdi; + vcpu->regs[VCPU_REGS_RSP] = regs->rsp; + vcpu->regs[VCPU_REGS_RBP] = regs->rbp; +#ifdef CONFIG_X86_64 + vcpu->regs[VCPU_REGS_R8] = regs->r8; + vcpu->regs[VCPU_REGS_R9] = regs->r9; + vcpu->regs[VCPU_REGS_R10] = regs->r10; + vcpu->regs[VCPU_REGS_R11] = regs->r11; + vcpu->regs[VCPU_REGS_R12] = regs->r12; + vcpu->regs[VCPU_REGS_R13] = regs->r13; + vcpu->regs[VCPU_REGS_R14] = regs->r14; + vcpu->regs[VCPU_REGS_R15] = regs->r15; +#endif + + vcpu->rip = regs->rip; + kvm_x86_ops->set_rflags(vcpu, regs->rflags); + + kvm_x86_ops->decache_regs(vcpu); + + vcpu_put(vcpu); + + return 0; +} + +static void get_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + return kvm_x86_ops->get_segment(vcpu, var, seg); +} + +void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) +{ + struct kvm_segment cs; + + get_segment(vcpu, &cs, VCPU_SREG_CS); + *db = cs.db; + *l = cs.l; +} +EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + struct descriptor_table dt; + int pending_vec; + + vcpu_load(vcpu); + + get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + get_segment(vcpu, &sregs->es, VCPU_SREG_ES); + get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_x86_ops->get_idt(vcpu, &dt); + sregs->idt.limit = dt.limit; + sregs->idt.base = dt.base; + kvm_x86_ops->get_gdt(vcpu, &dt); + sregs->gdt.limit = dt.limit; + sregs->gdt.base = dt.base; + + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + sregs->cr0 = vcpu->cr0; + sregs->cr2 = vcpu->cr2; + sregs->cr3 = vcpu->cr3; + sregs->cr4 = vcpu->cr4; + sregs->cr8 = get_cr8(vcpu); + sregs->efer = vcpu->shadow_efer; + sregs->apic_base = kvm_get_apic_base(vcpu); + + if (irqchip_in_kernel(vcpu->kvm)) { + memset(sregs->interrupt_bitmap, 0, + sizeof sregs->interrupt_bitmap); + pending_vec = kvm_x86_ops->get_irq(vcpu); + if (pending_vec >= 0) + set_bit(pending_vec, + (unsigned long *)sregs->interrupt_bitmap); + } else + memcpy(sregs->interrupt_bitmap, vcpu->irq_pending, + sizeof sregs->interrupt_bitmap); + + vcpu_put(vcpu); + + return 0; +} + +static void set_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + return kvm_x86_ops->set_segment(vcpu, var, seg); +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + int mmu_reset_needed = 0; + int i, pending_vec, max_bits; + struct descriptor_table dt; + + vcpu_load(vcpu); + + dt.limit = sregs->idt.limit; + dt.base = sregs->idt.base; + kvm_x86_ops->set_idt(vcpu, &dt); + dt.limit = sregs->gdt.limit; + dt.base = sregs->gdt.base; + kvm_x86_ops->set_gdt(vcpu, &dt); + + vcpu->cr2 = sregs->cr2; + mmu_reset_needed |= vcpu->cr3 != sregs->cr3; + vcpu->cr3 = sregs->cr3; + + set_cr8(vcpu, sregs->cr8); + + mmu_reset_needed |= vcpu->shadow_efer != sregs->efer; +#ifdef CONFIG_X86_64 + kvm_x86_ops->set_efer(vcpu, sregs->efer); +#endif + kvm_set_apic_base(vcpu, sregs->apic_base); + + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + + mmu_reset_needed |= vcpu->cr0 != sregs->cr0; + vcpu->cr0 = sregs->cr0; + kvm_x86_ops->set_cr0(vcpu, sregs->cr0); + + mmu_reset_needed |= vcpu->cr4 != sregs->cr4; + kvm_x86_ops->set_cr4(vcpu, sregs->cr4); + if (!is_long_mode(vcpu) && is_pae(vcpu)) + load_pdptrs(vcpu, vcpu->cr3); + + if (mmu_reset_needed) + kvm_mmu_reset_context(vcpu); + + if (!irqchip_in_kernel(vcpu->kvm)) { + memcpy(vcpu->irq_pending, sregs->interrupt_bitmap, + sizeof vcpu->irq_pending); + vcpu->irq_summary = 0; + for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i) + if (vcpu->irq_pending[i]) + __set_bit(i, &vcpu->irq_summary); + } else { + max_bits = (sizeof sregs->interrupt_bitmap) << 3; + pending_vec = find_first_bit( + (const unsigned long *)sregs->interrupt_bitmap, + max_bits); + /* Only pending external irq is handled here */ + if (pending_vec < max_bits) { + kvm_x86_ops->set_irq(vcpu, pending_vec); + pr_debug("Set back pending irq %d\n", + pending_vec); + } + } + + set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + set_segment(vcpu, &sregs->es, VCPU_SREG_ES); + set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg) +{ + int r; + + vcpu_load(vcpu); + + r = kvm_x86_ops->set_guest_debug(vcpu, dbg); + + vcpu_put(vcpu); + + return r; +} + /* * fxsave fpu state. Taken from x86_64/processor.h. To be killed when * we have asm/x86/processor.h -- cgit v1.2.3 From 12264760e46077a65c1240ac0b27dfa34b402158 Mon Sep 17 00:00:00 2001 From: Dor Laor Date: Wed, 7 Nov 2007 16:20:06 +0200 Subject: KVM: Add make_page_dirty() to kvm_clear_guest_page() Every write access to guest pages should be tracked. Signed-off-by: Dor Laor Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 8665531d9287..8d6e55f95fbb 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -706,6 +706,7 @@ int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) kunmap_atomic(page_virt, KM_USER0); kvm_release_page(page); + mark_page_dirty(kvm, gfn); return 0; } EXPORT_SYMBOL_GPL(kvm_clear_guest_page); -- cgit v1.2.3 From 9c5623e3e42e94927d02a6693875badf15692970 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 8 Nov 2007 18:19:20 +0200 Subject: KVM: VMX: Use vmx to inject real-mode interrupts Instead of injecting real-mode interrupts by writing the interrupt frame into guest memory, abuse vmx by injecting a software interrupt. We need to pretend the software interrupt instruction had a length > 0, so we have to adjust rip backward. This lets us not to mess with writing guest memory, which is complex and also sleeps. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 53 ++++------------------------------------------------- drivers/kvm/vmx.h | 1 + 2 files changed, 5 insertions(+), 49 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index eca422e9506d..d2c25e25d3aa 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1709,58 +1709,13 @@ out: return ret; } -static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq) -{ - u16 ent[2]; - u16 cs; - u16 ip; - unsigned long flags; - unsigned long ss_base = vmcs_readl(GUEST_SS_BASE); - u16 sp = vmcs_readl(GUEST_RSP); - u32 ss_limit = vmcs_read32(GUEST_SS_LIMIT); - - if (sp > ss_limit || sp < 6) { - vcpu_printf(vcpu, "%s: #SS, rsp 0x%lx ss 0x%lx limit 0x%x\n", - __FUNCTION__, - vmcs_readl(GUEST_RSP), - vmcs_readl(GUEST_SS_BASE), - vmcs_read32(GUEST_SS_LIMIT)); - return; - } - - if (emulator_read_std(irq * sizeof(ent), &ent, sizeof(ent), vcpu) != - X86EMUL_CONTINUE) { - vcpu_printf(vcpu, "%s: read guest err\n", __FUNCTION__); - return; - } - - flags = vmcs_readl(GUEST_RFLAGS); - cs = vmcs_readl(GUEST_CS_BASE) >> 4; - ip = vmcs_readl(GUEST_RIP); - - - if (emulator_write_emulated( - ss_base + sp - 2, &flags, 2, vcpu) != X86EMUL_CONTINUE || - emulator_write_emulated( - ss_base + sp - 4, &cs, 2, vcpu) != X86EMUL_CONTINUE || - emulator_write_emulated( - ss_base + sp - 6, &ip, 2, vcpu) != X86EMUL_CONTINUE) { - vcpu_printf(vcpu, "%s: write guest err\n", __FUNCTION__); - return; - } - - vmcs_writel(GUEST_RFLAGS, flags & - ~(X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF)); - vmcs_write16(GUEST_CS_SELECTOR, ent[1]) ; - vmcs_writel(GUEST_CS_BASE, ent[1] << 4); - vmcs_writel(GUEST_RIP, ent[0]); - vmcs_writel(GUEST_RSP, (vmcs_readl(GUEST_RSP) & ~0xffff) | (sp - 6)); -} - static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) { if (vcpu->rmode.active) { - inject_rmode_irq(vcpu, irq); + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK); + vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); + vmcs_writel(GUEST_RIP, vmcs_readl(GUEST_RIP) - 1); return; } vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index c84bd3733153..d757b36c2fbe 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -233,6 +233,7 @@ enum vmcs_field { #define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ #define INTR_TYPE_EXCEPTION (3 << 8) /* processor exception */ +#define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ /* * Exit Qualifications for MOV for Control Register Access -- cgit v1.2.3 From 1155f76a8166ae6fc88e7d73eb6817eb9012d476 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 22 Nov 2007 11:30:47 +0200 Subject: KVM: VMX: Read & store IDT_VECTORING_INFO_FIELD We'll want to write to it in order to fix real-mode irq injection problems, but it is a read-only field. Storing it in a variable solves that issue. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index d2c25e25d3aa..f045f4005eaa 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -48,6 +48,7 @@ struct vcpu_vmx { struct kvm_vcpu vcpu; int launched; u8 fail; + u32 idt_vectoring_info; struct kvm_msr_entry *guest_msrs; struct kvm_msr_entry *host_msrs; int nmsrs; @@ -863,9 +864,10 @@ static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) static int vmx_get_irq(struct kvm_vcpu *vcpu) { + struct vcpu_vmx *vmx = to_vmx(vcpu); u32 idtv_info_field; - idtv_info_field = vmcs_read32(IDT_VECTORING_INFO_FIELD); + idtv_info_field = vmx->idt_vectoring_info; if (idtv_info_field & INTR_INFO_VALID_MASK) { if (is_external_interrupt(idtv_info_field)) return idtv_info_field & VECTORING_INFO_VECTOR_MASK; @@ -1817,12 +1819,13 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { + struct vcpu_vmx *vmx = to_vmx(vcpu); u32 intr_info, error_code; unsigned long cr2, rip; u32 vect_info; enum emulation_result er; - vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + vect_info = vmx->idt_vectoring_info; intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if ((vect_info & VECTORING_INFO_VALID_MASK) && @@ -2171,9 +2174,9 @@ static const int kvm_vmx_max_exit_handlers = */ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) { - u32 vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); u32 exit_reason = vmcs_read32(VM_EXIT_REASON); struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 vectoring_info = vmx->idt_vectoring_info; if (unlikely(vmx->fail)) { kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; @@ -2228,6 +2231,7 @@ static void enable_irq_window(struct kvm_vcpu *vcpu) static void vmx_intr_assist(struct kvm_vcpu *vcpu) { + struct vcpu_vmx *vmx = to_vmx(vcpu); u32 idtv_info_field, intr_info_field; int has_ext_irq, interrupt_window_open; int vector; @@ -2236,7 +2240,7 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) has_ext_irq = kvm_cpu_has_interrupt(vcpu); intr_info_field = vmcs_read32(VM_ENTRY_INTR_INFO_FIELD); - idtv_info_field = vmcs_read32(IDT_VECTORING_INFO_FIELD); + idtv_info_field = vmx->idt_vectoring_info; if (intr_info_field & INTR_INFO_VALID_MASK) { if (idtv_info_field & INTR_INFO_VALID_MASK) { /* TODO: fault when IDT_Vectoring */ @@ -2396,6 +2400,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #endif ); + vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; @@ -2413,7 +2419,8 @@ static void vmx_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, u32 err_code) { - u32 vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 vect_info = vmx->idt_vectoring_info; ++vcpu->stat.pf_guest; -- cgit v1.2.3 From 9c8cba3761d4741cfd53ecc40604fac01f52128a Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 22 Nov 2007 11:42:59 +0200 Subject: KVM: Fix faults during injection of real-mode interrupts If vmx fails to inject a real-mode interrupt while fetching the interrupt redirection table, it fails to record this in the vectoring information field. So we detect this condition and do it ourselves. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index f045f4005eaa..26719aa9701e 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -65,7 +65,13 @@ struct vcpu_vmx { int fs_reload_needed; int guest_efer_loaded; } host_state; - + struct { + struct { + bool pending; + u8 vector; + unsigned rip; + } irq; + } rmode; }; static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) @@ -1713,11 +1719,16 @@ out: static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) { + struct vcpu_vmx *vmx = to_vmx(vcpu); + if (vcpu->rmode.active) { + vmx->rmode.irq.pending = true; + vmx->rmode.irq.vector = irq; + vmx->rmode.irq.rip = vmcs_readl(GUEST_RIP); vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK); vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); - vmcs_writel(GUEST_RIP, vmcs_readl(GUEST_RIP) - 1); + vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip - 1); return; } vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, @@ -2251,6 +2262,17 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) return; } if (unlikely(idtv_info_field & INTR_INFO_VALID_MASK)) { + if ((idtv_info_field & VECTORING_INFO_TYPE_MASK) + == INTR_TYPE_EXT_INTR + && vcpu->rmode.active) { + u8 vect = idtv_info_field & VECTORING_INFO_VECTOR_MASK; + + vmx_inject_irq(vcpu, vect); + if (unlikely(has_ext_irq)) + enable_irq_window(vcpu); + return; + } + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, idtv_info_field); vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); @@ -2275,6 +2297,29 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) enable_irq_window(vcpu); } +/* + * Failure to inject an interrupt should give us the information + * in IDT_VECTORING_INFO_FIELD. However, if the failure occurs + * when fetching the interrupt redirection bitmap in the real-mode + * tss, this doesn't happen. So we do it ourselves. + */ +static void fixup_rmode_irq(struct vcpu_vmx *vmx) +{ + vmx->rmode.irq.pending = 0; + if (vmcs_readl(GUEST_RIP) + 1 != vmx->rmode.irq.rip) + return; + vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip); + if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) { + vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK; + vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR; + return; + } + vmx->idt_vectoring_info = + VECTORING_INFO_VALID_MASK + | INTR_TYPE_EXT_INTR + | vmx->rmode.irq.vector; +} + static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -2401,6 +2446,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) ); vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + if (vmx->rmode.irq.pending) + fixup_rmode_irq(vmx); vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; -- cgit v1.2.3 From 8a70cc3d0f4877f862ac9cace2e61e4e5116b502 Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Sun, 11 Nov 2007 12:27:20 +0200 Subject: KVM: VMX: Comment VMX primary/secondary exec ctl definitions Add comments for secondary/primary Processor-Based VM-execution controls. Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- drivers/kvm/vmx.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index d757b36c2fbe..6d32bc69285e 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -25,6 +25,9 @@ * */ +/* + * Definitions of Primary Processor-Based VM-Execution Controls. + */ #define CPU_BASED_VIRTUAL_INTR_PENDING 0x00000004 #define CPU_BASED_USE_TSC_OFFSETING 0x00000008 #define CPU_BASED_HLT_EXITING 0x00000080 @@ -42,6 +45,11 @@ #define CPU_BASED_MONITOR_EXITING 0x20000000 #define CPU_BASED_PAUSE_EXITING 0x40000000 #define CPU_BASED_ACTIVATE_SECONDARY_CONTROLS 0x80000000 +/* + * Definitions of Secondary Processor-Based VM-Execution Controls. + */ +#define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 + #define PIN_BASED_EXT_INTR_MASK 0x00000001 #define PIN_BASED_NMI_EXITING 0x00000008 @@ -54,8 +62,6 @@ #define VM_ENTRY_SMM 0x00000400 #define VM_ENTRY_DEACT_DUAL_MONITOR 0x00000800 -#define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 - /* VMCS Encodings */ enum vmcs_field { GUEST_ES_SELECTOR = 0x00000800, -- cgit v1.2.3 From e5edaa01c4cea5f60c617fac989c6458df0ecc4e Mon Sep 17 00:00:00 2001 From: Eddie Dong Date: Sun, 11 Nov 2007 12:28:35 +0200 Subject: KVM: VMX: wbinvd exiting Add wbinvd VM Exit support to prepare for pass-through device cache emulation and also enhance real time responsiveness. Signed-off-by: Yaozu (Eddie) Dong Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 11 ++++++++++- drivers/kvm/vmx.h | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 26719aa9701e..c3bde7532714 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -980,7 +980,8 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) #endif if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { min = 0; - opt = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; + opt = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | + SECONDARY_EXEC_WBINVD_EXITING; if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS2, &_cpu_based_2nd_exec_control) < 0) return -EIO; @@ -2133,6 +2134,13 @@ static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return 1; } +static int handle_wbinvd(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + skip_emulated_instruction(vcpu); + /* TODO: Add support for VT-d/pass-through device */ + return 1; +} + static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { u64 exit_qualification; @@ -2174,6 +2182,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, [EXIT_REASON_VMCALL] = handle_vmcall, [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, [EXIT_REASON_APIC_ACCESS] = handle_apic_access, + [EXIT_REASON_WBINVD] = handle_wbinvd, }; static const int kvm_vmx_max_exit_handlers = diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h index 6d32bc69285e..d52ae8d7303d 100644 --- a/drivers/kvm/vmx.h +++ b/drivers/kvm/vmx.h @@ -49,6 +49,7 @@ * Definitions of Secondary Processor-Based VM-Execution Controls. */ #define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 +#define SECONDARY_EXEC_WBINVD_EXITING 0x00000040 #define PIN_BASED_EXT_INTR_MASK 0x00000001 @@ -223,6 +224,7 @@ enum vmcs_field { #define EXIT_REASON_MWAIT_INSTRUCTION 36 #define EXIT_REASON_TPR_BELOW_THRESHOLD 43 #define EXIT_REASON_APIC_ACCESS 44 +#define EXIT_REASON_WBINVD 54 /* * Interruption-information format -- cgit v1.2.3 From 906e608b05b87b650e509491e41c89575358df5e Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sun, 11 Nov 2007 14:48:17 +0200 Subject: KVM: x86 emulator: remove 8 bytes operands emulator for call near instruction it is removed beacuse it isnt supported on a real host Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 64888a6edc27..e6979475bee7 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1558,9 +1558,6 @@ special_insn: case 4: rel = insn_fetch(s32, 4, c->eip); break; - case 8: - rel = insn_fetch(s64, 8, c->eip); - break; default: DPRINTF("Call: Invalid op_bytes\n"); goto cannot_emulate; -- cgit v1.2.3 From 1a6f4d7fbd11e539630cd5637311a4e55fae60ef Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 11 Nov 2007 18:37:32 +0200 Subject: KVM: Simplify CPU_TASKS_FROZEN cpu notifier handling Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 8d6e55f95fbb..d395c987894e 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1335,21 +1335,19 @@ static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, { int cpu = (long)v; + val &= ~CPU_TASKS_FROZEN; switch (val) { case CPU_DYING: - case CPU_DYING_FROZEN: printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", cpu); hardware_disable(NULL); break; case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", cpu); smp_call_function_single(cpu, hardware_disable, NULL, 0, 1); break; case CPU_ONLINE: - case CPU_ONLINE_FROZEN: printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", cpu); smp_call_function_single(cpu, hardware_enable, NULL, 0, 1); -- cgit v1.2.3 From f9d46eb0e4a5b5e0926ca61c19f8c8bbb9496b28 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sun, 11 Nov 2007 22:02:22 +0200 Subject: KVM: add kvm_is_error_hva() Check for the "error hva", an address outside the user address space that signals a bad gfn. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 11 +++++++++++ 2 files changed, 12 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3f5ffc37480d..64983240adca 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -520,6 +520,7 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); extern struct page *bad_page; int is_error_page(struct page *page); +int kvm_is_error_hva(unsigned long addr); int kvm_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, int user_alloc); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index d395c987894e..dfb65e2f87ee 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -498,6 +498,17 @@ int is_error_page(struct page *page) } EXPORT_SYMBOL_GPL(is_error_page); +static inline unsigned long bad_hva(void) +{ + return PAGE_OFFSET; +} + +int kvm_is_error_hva(unsigned long addr) +{ + return addr == bad_hva(); +} +EXPORT_SYMBOL_GPL(kvm_is_error_hva); + gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) { int i; -- cgit v1.2.3 From 539cb6608ca804e7805d8e88c83377d991a552b1 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sun, 11 Nov 2007 22:05:04 +0200 Subject: KVM: introduce gfn_to_hva() Convert a guest frame number to the corresponding host virtual address. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index dfb65e2f87ee..04544aecf22f 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -559,28 +559,37 @@ int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) } EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); +static unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot; + + gfn = unalias_gfn(kvm, gfn); + slot = __gfn_to_memslot(kvm, gfn); + if (!slot) + return bad_hva(); + return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE); +} + /* * Requires current->mm->mmap_sem to be held */ static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn) { - struct kvm_memory_slot *slot; struct page *page[1]; + unsigned long addr; int npages; might_sleep(); - gfn = unalias_gfn(kvm, gfn); - slot = __gfn_to_memslot(kvm, gfn); - if (!slot) { + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) { get_page(bad_page); return bad_page; } - npages = get_user_pages(current, current->mm, - slot->userspace_addr - + (gfn - slot->base_gfn) * PAGE_SIZE, 1, - 1, 1, page, NULL); + npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page, + NULL); + if (npages != 1) { get_page(bad_page); return bad_page; -- cgit v1.2.3 From e0506bcba5992650b1190de9125f5963a30f32e2 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Sun, 11 Nov 2007 22:10:22 +0200 Subject: KVM: Change kvm_{read,write}_guest() to use copy_{from,to}_user() This changes kvm_write_guest_page/kvm_read_guest_page to use copy_to_user/read_from_user, as a result we get better speed and better dirty bit tracking. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 04544aecf22f..d68901bd0693 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -630,20 +630,15 @@ static int next_segment(unsigned long len, int offset) int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, int len) { - void *page_virt; - struct page *page; + int r; + unsigned long addr; - page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) { - kvm_release_page(page); + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) + return -EFAULT; + r = copy_from_user(data, (void __user *)addr + offset, len); + if (r) return -EFAULT; - } - page_virt = kmap_atomic(page, KM_USER0); - - memcpy(data, page_virt + offset, len); - - kunmap_atomic(page_virt, KM_USER0); - kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_read_guest_page); @@ -671,21 +666,16 @@ EXPORT_SYMBOL_GPL(kvm_read_guest); int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, int offset, int len) { - void *page_virt; - struct page *page; + int r; + unsigned long addr; - page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) { - kvm_release_page(page); + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) + return -EFAULT; + r = copy_to_user((void __user *)addr + offset, data, len); + if (r) return -EFAULT; - } - page_virt = kmap_atomic(page, KM_USER0); - - memcpy(page_virt + offset, data, len); - - kunmap_atomic(page_virt, KM_USER0); mark_page_dirty(kvm, gfn); - kvm_release_page(page); return 0; } EXPORT_SYMBOL_GPL(kvm_write_guest_page); -- cgit v1.2.3 From d825ed0a97b8e82597dd652bfe51fed3fa9b3c9a Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:08:51 +0800 Subject: KVM: Portability: Move some includes to x86.c Move some includes to x86.c from kvm_main.c, since the related functions have been moved to x86.c Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 -- drivers/kvm/x86.c | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index d68901bd0693..6aaba79f8c5a 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -17,7 +17,6 @@ #include "kvm.h" #include "x86.h" -#include "x86_emulate.h" #include "irq.h" #include @@ -44,7 +43,6 @@ #include #include -#include #include #include #include diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 394da6605364..4282a0fb2962 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -16,6 +16,7 @@ #include "kvm.h" #include "x86.h" +#include "x86_emulate.h" #include "segment_descriptor.h" #include "irq.h" @@ -25,6 +26,7 @@ #include #include +#include #define MAX_IO_MSRS 256 #define CR0_RESERVED_BITS \ -- cgit v1.2.3 From 97896d04a14669b146c17d779b81ec7a339deeb3 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:09:30 +0800 Subject: KVM: Portability: Move kvm_x86_ops to x86.c Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 1 - drivers/kvm/x86.c | 2 ++ drivers/kvm/x86.h | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 6aaba79f8c5a..47a76c3a4c81 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -55,7 +55,6 @@ static LIST_HEAD(vm_list); static cpumask_t cpus_hardware_enabled; -struct kvm_x86_ops *kvm_x86_ops; struct kmem_cache *kvm_vcpu_cache; EXPORT_SYMBOL_GPL(kvm_vcpu_cache); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 4282a0fb2962..2edc53ec8b3b 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -44,6 +44,8 @@ #define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x) +struct kvm_x86_ops *kvm_x86_ops; + struct kvm_stats_debugfs_item debugfs_entries[] = { { "pf_fixed", STAT_OFFSET(pf_fixed) }, { "pf_guest", STAT_OFFSET(pf_guest) }, diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 663b822b4ddb..ec32c26a5118 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -85,6 +85,8 @@ struct kvm_vcpu { struct x86_emulate_ctxt emulate_ctxt; }; +extern struct kvm_x86_ops *kvm_x86_ops; + int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From e9b11c17552afe684e9e5d0444309a3ddf410116 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:38:21 +0800 Subject: KVM: Portability: Add vcpu and hardware management arch hooks Add the following hooks: void decache_vcpus_on_cpu(int cpu); int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu); int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); void kvm_arch_hardware_enable(void *garbage); void kvm_arch_hardware_disable(void *garbage); int kvm_arch_hardware_setup(void); void kvm_arch_hardware_unsetup(void); void kvm_arch_check_processor_compat(void *rtn); Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 19 ++++++ drivers/kvm/kvm_main.c | 113 ++++++----------------------------- drivers/kvm/x86.c | 157 +++++++++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/x86.h | 3 + 4 files changed, 197 insertions(+), 95 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 64983240adca..bca07c6d0460 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -492,6 +492,8 @@ void kvm_vcpu_uninit(struct kvm_vcpu *vcpu); void vcpu_load(struct kvm_vcpu *vcpu); void vcpu_put(struct kvm_vcpu *vcpu); +void decache_vcpus_on_cpu(int cpu); + int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, struct module *module); @@ -649,6 +651,23 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); __init void kvm_arch_init(void); +int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); + +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); +struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); +void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu); + +int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); +void kvm_arch_hardware_enable(void *garbage); +void kvm_arch_hardware_disable(void *garbage); +int kvm_arch_hardware_setup(void); +void kvm_arch_hardware_unsetup(void); +void kvm_arch_check_processor_compat(void *rtn); + + static inline void kvm_guest_enter(void) { account_system_vtime(current); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 47a76c3a4c81..7fdfed52dbe7 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -50,8 +50,8 @@ MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); -static DEFINE_SPINLOCK(kvm_lock); -static LIST_HEAD(vm_list); +DEFINE_SPINLOCK(kvm_lock); +LIST_HEAD(vm_list); static cpumask_t cpus_hardware_enabled; @@ -124,13 +124,8 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) mutex_init(&vcpu->mutex); vcpu->cpu = -1; - vcpu->mmu.root_hpa = INVALID_PAGE; vcpu->kvm = kvm; vcpu->vcpu_id = id; - if (!irqchip_in_kernel(kvm) || id == 0) - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; - else - vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED; init_waitqueue_head(&vcpu->wq); page = alloc_page(GFP_KERNEL | __GFP_ZERO); @@ -140,29 +135,11 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) } vcpu->run = page_address(page); - page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) { - r = -ENOMEM; - goto fail_free_run; - } - vcpu->pio_data = page_address(page); - - r = kvm_mmu_create(vcpu); + r = kvm_arch_vcpu_init(vcpu); if (r < 0) - goto fail_free_pio_data; - - if (irqchip_in_kernel(kvm)) { - r = kvm_create_lapic(vcpu); - if (r < 0) - goto fail_mmu_destroy; - } - + goto fail_free_run; return 0; -fail_mmu_destroy: - kvm_mmu_destroy(vcpu); -fail_free_pio_data: - free_page((unsigned long)vcpu->pio_data); fail_free_run: free_page((unsigned long)vcpu->run); fail: @@ -172,9 +149,7 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_init); void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) { - kvm_free_lapic(vcpu); - kvm_mmu_destroy(vcpu); - free_page((unsigned long)vcpu->pio_data); + kvm_arch_vcpu_uninit(vcpu); free_page((unsigned long)vcpu->run); } EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); @@ -240,7 +215,7 @@ static void kvm_free_vcpus(struct kvm *kvm) kvm_unload_vcpu_mmu(kvm->vcpus[i]); for (i = 0; i < KVM_MAX_VCPUS; ++i) { if (kvm->vcpus[i]) { - kvm_x86_ops->vcpu_free(kvm->vcpus[i]); + kvm_arch_vcpu_free(kvm->vcpus[i]); kvm->vcpus[i] = NULL; } } @@ -900,28 +875,17 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) if (!valid_vcpu(n)) return -EINVAL; - vcpu = kvm_x86_ops->vcpu_create(kvm, n); + vcpu = kvm_arch_vcpu_create(kvm, n); if (IS_ERR(vcpu)) return PTR_ERR(vcpu); preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); - /* We do fxsave: this must be aligned. */ - BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); - - vcpu_load(vcpu); - r = kvm_x86_ops->vcpu_reset(vcpu); - if (r == 0) - r = kvm_mmu_setup(vcpu); - vcpu_put(vcpu); - if (r < 0) - goto free_vcpu; - mutex_lock(&kvm->lock); if (kvm->vcpus[n]) { r = -EEXIST; mutex_unlock(&kvm->lock); - goto mmu_unload; + goto vcpu_destroy; } kvm->vcpus[n] = vcpu; mutex_unlock(&kvm->lock); @@ -936,14 +900,8 @@ unlink: mutex_lock(&kvm->lock); kvm->vcpus[n] = NULL; mutex_unlock(&kvm->lock); - -mmu_unload: - vcpu_load(vcpu); - kvm_mmu_unload(vcpu); - vcpu_put(vcpu); - -free_vcpu: - kvm_x86_ops->vcpu_free(vcpu); +vcpu_destroy: + kvm_arch_vcpu_destory(vcpu); return r; } @@ -1281,41 +1239,6 @@ static struct miscdevice kvm_dev = { &kvm_chardev_ops, }; -/* - * Make sure that a cpu that is being hot-unplugged does not have any vcpus - * cached on it. - */ -static void decache_vcpus_on_cpu(int cpu) -{ - struct kvm *vm; - struct kvm_vcpu *vcpu; - int i; - - spin_lock(&kvm_lock); - list_for_each_entry(vm, &vm_list, vm_list) - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - vcpu = vm->vcpus[i]; - if (!vcpu) - continue; - /* - * If the vcpu is locked, then it is running on some - * other cpu and therefore it is not cached on the - * cpu in question. - * - * If it's not locked, check the last cpu it executed - * on. - */ - if (mutex_trylock(&vcpu->mutex)) { - if (vcpu->cpu == cpu) { - kvm_x86_ops->vcpu_decache(vcpu); - vcpu->cpu = -1; - } - mutex_unlock(&vcpu->mutex); - } - } - spin_unlock(&kvm_lock); -} - static void hardware_enable(void *junk) { int cpu = raw_smp_processor_id(); @@ -1323,7 +1246,7 @@ static void hardware_enable(void *junk) if (cpu_isset(cpu, cpus_hardware_enabled)) return; cpu_set(cpu, cpus_hardware_enabled); - kvm_x86_ops->hardware_enable(NULL); + kvm_arch_hardware_enable(NULL); } static void hardware_disable(void *junk) @@ -1334,7 +1257,7 @@ static void hardware_disable(void *junk) return; cpu_clear(cpu, cpus_hardware_enabled); decache_vcpus_on_cpu(cpu); - kvm_x86_ops->hardware_disable(NULL); + kvm_arch_hardware_disable(NULL); } static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, @@ -1500,7 +1423,7 @@ static void kvm_sched_in(struct preempt_notifier *pn, int cpu) { struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); - kvm_x86_ops->vcpu_load(vcpu, cpu); + kvm_arch_vcpu_load(vcpu, cpu); } static void kvm_sched_out(struct preempt_notifier *pn, @@ -1508,7 +1431,7 @@ static void kvm_sched_out(struct preempt_notifier *pn, { struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); - kvm_x86_ops->vcpu_put(vcpu); + kvm_arch_vcpu_put(vcpu); } int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, @@ -1533,13 +1456,13 @@ int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, kvm_x86_ops = ops; - r = kvm_x86_ops->hardware_setup(); + r = kvm_arch_hardware_setup(); if (r < 0) goto out; for_each_online_cpu(cpu) { smp_call_function_single(cpu, - kvm_x86_ops->check_processor_compatibility, + kvm_arch_check_processor_compat, &r, 0, 1); if (r < 0) goto out_free_0; @@ -1594,7 +1517,7 @@ out_free_2: out_free_1: on_each_cpu(hardware_disable, NULL, 0, 1); out_free_0: - kvm_x86_ops->hardware_unsetup(); + kvm_arch_hardware_unsetup(); out: kvm_x86_ops = NULL; return r; @@ -1610,7 +1533,7 @@ void kvm_exit_x86(void) unregister_reboot_notifier(&kvm_reboot_notifier); unregister_cpu_notifier(&kvm_cpu_notifier); on_each_cpu(hardware_disable, NULL, 0, 1); - kvm_x86_ops->hardware_unsetup(); + kvm_arch_hardware_unsetup(); kvm_x86_ops = NULL; } EXPORT_SYMBOL_GPL(kvm_exit_x86); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 2edc53ec8b3b..4902b35060f5 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -564,6 +564,41 @@ out: return r; } +/* + * Make sure that a cpu that is being hot-unplugged does not have any vcpus + * cached on it. + */ +void decache_vcpus_on_cpu(int cpu) +{ + struct kvm *vm; + struct kvm_vcpu *vcpu; + int i; + + spin_lock(&kvm_lock); + list_for_each_entry(vm, &vm_list, vm_list) + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + vcpu = vm->vcpus[i]; + if (!vcpu) + continue; + /* + * If the vcpu is locked, then it is running on some + * other cpu and therefore it is not cached on the + * cpu in question. + * + * If it's not locked, check the last cpu it executed + * on. + */ + if (mutex_trylock(&vcpu->mutex)) { + if (vcpu->cpu == cpu) { + kvm_x86_ops->vcpu_decache(vcpu); + vcpu->cpu = -1; + } + mutex_unlock(&vcpu->mutex); + } + } + spin_unlock(&kvm_lock); +} + long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -2319,3 +2354,125 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) fx_restore(&vcpu->host_fx_image); } EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); + +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->vcpu_free(vcpu); +} + +struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, + unsigned int id) +{ + int r; + struct kvm_vcpu *vcpu = kvm_x86_ops->vcpu_create(kvm, id); + + if (IS_ERR(vcpu)) { + r = -ENOMEM; + goto fail; + } + + /* We do fxsave: this must be aligned. */ + BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); + + vcpu_load(vcpu); + r = kvm_arch_vcpu_reset(vcpu); + if (r == 0) + r = kvm_mmu_setup(vcpu); + vcpu_put(vcpu); + if (r < 0) + goto free_vcpu; + + return vcpu; +free_vcpu: + kvm_x86_ops->vcpu_free(vcpu); +fail: + return ERR_PTR(r); +} + +void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu) +{ + vcpu_load(vcpu); + kvm_mmu_unload(vcpu); + vcpu_put(vcpu); + + kvm_x86_ops->vcpu_free(vcpu); +} + +int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) +{ + return kvm_x86_ops->vcpu_reset(vcpu); +} + +void kvm_arch_hardware_enable(void *garbage) +{ + kvm_x86_ops->hardware_enable(garbage); +} + +void kvm_arch_hardware_disable(void *garbage) +{ + kvm_x86_ops->hardware_disable(garbage); +} + +int kvm_arch_hardware_setup(void) +{ + return kvm_x86_ops->hardware_setup(); +} + +void kvm_arch_hardware_unsetup(void) +{ + kvm_x86_ops->hardware_unsetup(); +} + +void kvm_arch_check_processor_compat(void *rtn) +{ + kvm_x86_ops->check_processor_compatibility(rtn); +} + +int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) +{ + struct page *page; + struct kvm *kvm; + int r; + + BUG_ON(vcpu->kvm == NULL); + kvm = vcpu->kvm; + + vcpu->mmu.root_hpa = INVALID_PAGE; + if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0) + vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + else + vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) { + r = -ENOMEM; + goto fail; + } + vcpu->pio_data = page_address(page); + + r = kvm_mmu_create(vcpu); + if (r < 0) + goto fail_free_pio_data; + + if (irqchip_in_kernel(kvm)) { + r = kvm_create_lapic(vcpu); + if (r < 0) + goto fail_mmu_destroy; + } + + return 0; + +fail_mmu_destroy: + kvm_mmu_destroy(vcpu); +fail_free_pio_data: + free_page((unsigned long)vcpu->pio_data); +fail: + return r; +} + +void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) +{ + kvm_free_lapic(vcpu); + kvm_mmu_destroy(vcpu); + free_page((unsigned long)vcpu->pio_data); +} diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index ec32c26a5118..4df064100226 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -19,6 +19,9 @@ #include #include +extern spinlock_t kvm_lock; +extern struct list_head vm_list; + struct kvm_vcpu { KVM_VCPU_COMM; u64 host_tsc; -- cgit v1.2.3 From cb498ea2ce1d3f3c0bc0a2522241dca10263e437 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:39:31 +0800 Subject: KVM: Portability: Combine kvm_init and kvm_init_x86 Will be called once arch module registers itself. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++-- drivers/kvm/kvm_main.c | 61 ++++++++++++++++++-------------------------------- drivers/kvm/svm.c | 4 ++-- drivers/kvm/vmx.c | 4 ++-- 4 files changed, 28 insertions(+), 45 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index bca07c6d0460..5e7be1504664 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -495,9 +495,9 @@ void vcpu_put(struct kvm_vcpu *vcpu); void decache_vcpus_on_cpu(int cpu); -int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, +int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, struct module *module); -void kvm_exit_x86(void); +void kvm_exit(void); int kvm_mmu_module_init(void); void kvm_mmu_module_exit(void); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7fdfed52dbe7..2b3736e7483c 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1434,12 +1434,27 @@ static void kvm_sched_out(struct preempt_notifier *pn, kvm_arch_vcpu_put(vcpu); } -int kvm_init_x86(struct kvm_x86_ops *ops, unsigned int vcpu_size, +int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, struct module *module) { int r; int cpu; + r = kvm_mmu_module_init(); + if (r) + goto out4; + + kvm_init_debug(); + + kvm_arch_init(); + + bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + + if (bad_page == NULL) { + r = -ENOMEM; + goto out; + } + if (kvm_x86_ops) { printk(KERN_ERR "kvm: already loaded the other module\n"); return -EEXIST; @@ -1520,11 +1535,14 @@ out_free_0: kvm_arch_hardware_unsetup(); out: kvm_x86_ops = NULL; + kvm_exit_debug(); + kvm_mmu_module_exit(); +out4: return r; } -EXPORT_SYMBOL_GPL(kvm_init_x86); +EXPORT_SYMBOL_GPL(kvm_init); -void kvm_exit_x86(void) +void kvm_exit(void) { misc_deregister(&kvm_dev); kmem_cache_destroy(kvm_vcpu_cache); @@ -1535,43 +1553,8 @@ void kvm_exit_x86(void) on_each_cpu(hardware_disable, NULL, 0, 1); kvm_arch_hardware_unsetup(); kvm_x86_ops = NULL; -} -EXPORT_SYMBOL_GPL(kvm_exit_x86); - -static __init int kvm_init(void) -{ - int r; - - r = kvm_mmu_module_init(); - if (r) - goto out4; - - kvm_init_debug(); - - kvm_arch_init(); - - bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); - - if (bad_page == NULL) { - r = -ENOMEM; - goto out; - } - - return 0; - -out: - kvm_exit_debug(); - kvm_mmu_module_exit(); -out4: - return r; -} - -static __exit void kvm_exit(void) -{ kvm_exit_debug(); __free_page(bad_page); kvm_mmu_module_exit(); } - -module_init(kvm_init) -module_exit(kvm_exit) +EXPORT_SYMBOL_GPL(kvm_exit); diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 97863f8fd001..0ccbef18476a 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1723,13 +1723,13 @@ static struct kvm_x86_ops svm_x86_ops = { static int __init svm_init(void) { - return kvm_init_x86(&svm_x86_ops, sizeof(struct vcpu_svm), + return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm), THIS_MODULE); } static void __exit svm_exit(void) { - kvm_exit_x86(); + kvm_exit(); } module_init(svm_init) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index c3bde7532714..529162345c12 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2677,7 +2677,7 @@ static int __init vmx_init(void) memset(iova, 0xff, PAGE_SIZE); kunmap(vmx_io_bitmap_b); - r = kvm_init_x86(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE); + r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE); if (r) goto out1; @@ -2698,7 +2698,7 @@ static void __exit vmx_exit(void) __free_page(vmx_io_bitmap_b); __free_page(vmx_io_bitmap_a); - kvm_exit_x86(); + kvm_exit(); } module_init(vmx_init) -- cgit v1.2.3 From f8c16bbaa9e14b309ffcf29cac0ea377a35b0dd2 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 14 Nov 2007 20:40:21 +0800 Subject: KVM: Portability: Move x86 specific code from kvm_init() to kvm_arch() Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 5 +++-- drivers/kvm/kvm_main.c | 26 ++++++-------------------- drivers/kvm/x86.c | 27 ++++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 23 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 5e7be1504664..96d9c7da14c2 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -495,7 +495,7 @@ void vcpu_put(struct kvm_vcpu *vcpu); void decache_vcpus_on_cpu(int cpu); -int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, +int kvm_init(void *opaque, unsigned int vcpu_size, struct module *module); void kvm_exit(void); @@ -649,7 +649,8 @@ int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg); int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); -__init void kvm_arch_init(void); +int kvm_arch_init(void *opaque); +void kvm_arch_exit(void); int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 2b3736e7483c..e0f20d020281 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1434,7 +1434,7 @@ static void kvm_sched_out(struct preempt_notifier *pn, kvm_arch_vcpu_put(vcpu); } -int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, +int kvm_init(void *opaque, unsigned int vcpu_size, struct module *module) { int r; @@ -1446,7 +1446,9 @@ int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, kvm_init_debug(); - kvm_arch_init(); + r = kvm_arch_init(opaque); + if (r) + goto out4; bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); @@ -1455,22 +1457,6 @@ int kvm_init(struct kvm_x86_ops *ops, unsigned int vcpu_size, goto out; } - if (kvm_x86_ops) { - printk(KERN_ERR "kvm: already loaded the other module\n"); - return -EEXIST; - } - - if (!ops->cpu_has_kvm_support()) { - printk(KERN_ERR "kvm: no hardware support\n"); - return -EOPNOTSUPP; - } - if (ops->disabled_by_bios()) { - printk(KERN_ERR "kvm: disabled by bios\n"); - return -EOPNOTSUPP; - } - - kvm_x86_ops = ops; - r = kvm_arch_hardware_setup(); if (r < 0) goto out; @@ -1534,7 +1520,7 @@ out_free_1: out_free_0: kvm_arch_hardware_unsetup(); out: - kvm_x86_ops = NULL; + kvm_arch_exit(); kvm_exit_debug(); kvm_mmu_module_exit(); out4: @@ -1552,7 +1538,7 @@ void kvm_exit(void) unregister_cpu_notifier(&kvm_cpu_notifier); on_each_cpu(hardware_disable, NULL, 0, 1); kvm_arch_hardware_unsetup(); - kvm_x86_ops = NULL; + kvm_arch_exit(); kvm_exit_debug(); __free_page(bad_page); kvm_mmu_module_exit(); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 4902b35060f5..bbfa810bf8bd 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1645,11 +1645,36 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, } EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); -__init void kvm_arch_init(void) +int kvm_arch_init(void *opaque) { + struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; + kvm_init_msr_list(); + + if (kvm_x86_ops) { + printk(KERN_ERR "kvm: already loaded the other module\n"); + return -EEXIST; + } + + if (!ops->cpu_has_kvm_support()) { + printk(KERN_ERR "kvm: no hardware support\n"); + return -EOPNOTSUPP; + } + if (ops->disabled_by_bios()) { + printk(KERN_ERR "kvm: disabled by bios\n"); + return -EOPNOTSUPP; + } + + kvm_x86_ops = ops; + + return 0; } +void kvm_arch_exit(void) +{ + kvm_x86_ops = NULL; + } + int kvm_emulate_halt(struct kvm_vcpu *vcpu) { ++vcpu->stat.halt_exits; -- cgit v1.2.3 From a7e6c88a78b7c3d67491bf90b2b95e66437dbdb6 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Thu, 15 Nov 2007 14:52:28 +0800 Subject: KVM: x86 emulator: modify 'lods', and 'stos' not to depend on CR2 The current 'lods' and 'stos' is depending on incoming CR2 rather than decode memory address from registers. Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index e6979475bee7..8e2162fc6f70 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1528,7 +1528,9 @@ special_insn: case 0xaa ... 0xab: /* stos */ c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.ptr = (unsigned long *)cr2; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); c->dst.val = c->regs[VCPU_REGS_RAX]; register_address_increment(c->regs[VCPU_REGS_RDI], (ctxt->eflags & EFLG_DF) ? -c->dst.bytes @@ -1538,9 +1540,13 @@ special_insn: c->dst.type = OP_REG; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; - if ((rc = ops->read_emulated(cr2, &c->dst.val, - c->dst.bytes, - ctxt->vcpu)) != 0) + if ((rc = ops->read_emulated(register_address( + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + &c->dst.val, + c->dst.bytes, + ctxt->vcpu)) != 0) goto done; register_address_increment(c->regs[VCPU_REGS_RSI], (ctxt->eflags & EFLG_DF) ? -c->dst.bytes -- cgit v1.2.3 From 018d00d2fef27fc65f2917bec8f72b93615c18e1 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Thu, 15 Nov 2007 23:07:47 +0800 Subject: KVM: Portability: move KVM_CHECK_EXTENSION Make KVM_CHECK_EXTENSION code into a function, all archs can define its capability independently. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ drivers/kvm/kvm_main.c | 18 ++---------------- drivers/kvm/x86.c | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 16 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 96d9c7da14c2..a7be0733bc48 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -628,6 +628,9 @@ long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); + +int kvm_dev_ioctl_check_extension(long ext); + int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index e0f20d020281..c782f27265de 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1198,23 +1198,9 @@ static long kvm_dev_ioctl(struct file *filp, goto out; r = kvm_dev_ioctl_create_vm(); break; - case KVM_CHECK_EXTENSION: { - int ext = (long)argp; - - switch (ext) { - case KVM_CAP_IRQCHIP: - case KVM_CAP_HLT: - case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: - case KVM_CAP_USER_MEMORY: - case KVM_CAP_SET_TSS_ADDR: - r = 1; - break; - default: - r = 0; - break; - } + case KVM_CHECK_EXTENSION: + r = kvm_dev_ioctl_check_extension((long)argp); break; - } case KVM_GET_VCPU_MMAP_SIZE: r = -EINVAL; if (arg) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index bbfa810bf8bd..609792637fb4 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -599,6 +599,26 @@ void decache_vcpus_on_cpu(int cpu) spin_unlock(&kvm_lock); } +int kvm_dev_ioctl_check_extension(long ext) +{ + int r; + + switch (ext) { + case KVM_CAP_IRQCHIP: + case KVM_CAP_HLT: + case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: + case KVM_CAP_USER_MEMORY: + case KVM_CAP_SET_TSS_ADDR: + r = 1; + break; + default: + r = 0; + break; + } + return r; + +} + long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { -- cgit v1.2.3 From e08aa78ae5747c6e1dc525e8a40f23c2ea61d757 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 15 Nov 2007 18:06:18 +0200 Subject: KVM: VMX: Consolidate register usage in vmx_vcpu_run() We pass vcpu, vmx->fail, and vmx->launched to assembly code, but all three are fields within vmx. Consolidate by only passing in vmx and offsets for the rest. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 146 +++++++++++++++++++++++++++--------------------------- 1 file changed, 73 insertions(+), 73 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 529162345c12..84c77fed9d17 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2350,36 +2350,36 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #endif ASM_VMX_VMWRITE_RSP_RDX "\n\t" /* Check if vmlaunch of vmresume is needed */ - "cmp $0, %1 \n\t" + "cmpl $0, %c[launched](%0) \n\t" /* Load guest registers. Don't clobber flags. */ #ifdef CONFIG_X86_64 - "mov %c[cr2](%3), %%rax \n\t" + "mov %c[cr2](%0), %%rax \n\t" "mov %%rax, %%cr2 \n\t" - "mov %c[rax](%3), %%rax \n\t" - "mov %c[rbx](%3), %%rbx \n\t" - "mov %c[rdx](%3), %%rdx \n\t" - "mov %c[rsi](%3), %%rsi \n\t" - "mov %c[rdi](%3), %%rdi \n\t" - "mov %c[rbp](%3), %%rbp \n\t" - "mov %c[r8](%3), %%r8 \n\t" - "mov %c[r9](%3), %%r9 \n\t" - "mov %c[r10](%3), %%r10 \n\t" - "mov %c[r11](%3), %%r11 \n\t" - "mov %c[r12](%3), %%r12 \n\t" - "mov %c[r13](%3), %%r13 \n\t" - "mov %c[r14](%3), %%r14 \n\t" - "mov %c[r15](%3), %%r15 \n\t" - "mov %c[rcx](%3), %%rcx \n\t" /* kills %3 (rcx) */ + "mov %c[rax](%0), %%rax \n\t" + "mov %c[rbx](%0), %%rbx \n\t" + "mov %c[rdx](%0), %%rdx \n\t" + "mov %c[rsi](%0), %%rsi \n\t" + "mov %c[rdi](%0), %%rdi \n\t" + "mov %c[rbp](%0), %%rbp \n\t" + "mov %c[r8](%0), %%r8 \n\t" + "mov %c[r9](%0), %%r9 \n\t" + "mov %c[r10](%0), %%r10 \n\t" + "mov %c[r11](%0), %%r11 \n\t" + "mov %c[r12](%0), %%r12 \n\t" + "mov %c[r13](%0), %%r13 \n\t" + "mov %c[r14](%0), %%r14 \n\t" + "mov %c[r15](%0), %%r15 \n\t" + "mov %c[rcx](%0), %%rcx \n\t" /* kills %0 (rcx) */ #else - "mov %c[cr2](%3), %%eax \n\t" + "mov %c[cr2](%0), %%eax \n\t" "mov %%eax, %%cr2 \n\t" - "mov %c[rax](%3), %%eax \n\t" - "mov %c[rbx](%3), %%ebx \n\t" - "mov %c[rdx](%3), %%edx \n\t" - "mov %c[rsi](%3), %%esi \n\t" - "mov %c[rdi](%3), %%edi \n\t" - "mov %c[rbp](%3), %%ebp \n\t" - "mov %c[rcx](%3), %%ecx \n\t" /* kills %3 (ecx) */ + "mov %c[rax](%0), %%eax \n\t" + "mov %c[rbx](%0), %%ebx \n\t" + "mov %c[rdx](%0), %%edx \n\t" + "mov %c[rsi](%0), %%esi \n\t" + "mov %c[rdi](%0), %%edi \n\t" + "mov %c[rbp](%0), %%ebp \n\t" + "mov %c[rcx](%0), %%ecx \n\t" /* kills %0 (ecx) */ #endif /* Enter guest mode */ "jne .Llaunched \n\t" @@ -2389,62 +2389,62 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) ".Lkvm_vmx_return: " /* Save guest registers, load host registers, keep flags */ #ifdef CONFIG_X86_64 - "xchg %3, (%%rsp) \n\t" - "mov %%rax, %c[rax](%3) \n\t" - "mov %%rbx, %c[rbx](%3) \n\t" - "pushq (%%rsp); popq %c[rcx](%3) \n\t" - "mov %%rdx, %c[rdx](%3) \n\t" - "mov %%rsi, %c[rsi](%3) \n\t" - "mov %%rdi, %c[rdi](%3) \n\t" - "mov %%rbp, %c[rbp](%3) \n\t" - "mov %%r8, %c[r8](%3) \n\t" - "mov %%r9, %c[r9](%3) \n\t" - "mov %%r10, %c[r10](%3) \n\t" - "mov %%r11, %c[r11](%3) \n\t" - "mov %%r12, %c[r12](%3) \n\t" - "mov %%r13, %c[r13](%3) \n\t" - "mov %%r14, %c[r14](%3) \n\t" - "mov %%r15, %c[r15](%3) \n\t" + "xchg %0, (%%rsp) \n\t" + "mov %%rax, %c[rax](%0) \n\t" + "mov %%rbx, %c[rbx](%0) \n\t" + "pushq (%%rsp); popq %c[rcx](%0) \n\t" + "mov %%rdx, %c[rdx](%0) \n\t" + "mov %%rsi, %c[rsi](%0) \n\t" + "mov %%rdi, %c[rdi](%0) \n\t" + "mov %%rbp, %c[rbp](%0) \n\t" + "mov %%r8, %c[r8](%0) \n\t" + "mov %%r9, %c[r9](%0) \n\t" + "mov %%r10, %c[r10](%0) \n\t" + "mov %%r11, %c[r11](%0) \n\t" + "mov %%r12, %c[r12](%0) \n\t" + "mov %%r13, %c[r13](%0) \n\t" + "mov %%r14, %c[r14](%0) \n\t" + "mov %%r15, %c[r15](%0) \n\t" "mov %%cr2, %%rax \n\t" - "mov %%rax, %c[cr2](%3) \n\t" + "mov %%rax, %c[cr2](%0) \n\t" - "pop %%rcx; pop %%rbp; pop %%rdx \n\t" + "pop %%rbp; pop %%rbp; pop %%rdx \n\t" #else - "xchg %3, (%%esp) \n\t" - "mov %%eax, %c[rax](%3) \n\t" - "mov %%ebx, %c[rbx](%3) \n\t" - "pushl (%%esp); popl %c[rcx](%3) \n\t" - "mov %%edx, %c[rdx](%3) \n\t" - "mov %%esi, %c[rsi](%3) \n\t" - "mov %%edi, %c[rdi](%3) \n\t" - "mov %%ebp, %c[rbp](%3) \n\t" + "xchg %0, (%%esp) \n\t" + "mov %%eax, %c[rax](%0) \n\t" + "mov %%ebx, %c[rbx](%0) \n\t" + "pushl (%%esp); popl %c[rcx](%0) \n\t" + "mov %%edx, %c[rdx](%0) \n\t" + "mov %%esi, %c[rsi](%0) \n\t" + "mov %%edi, %c[rdi](%0) \n\t" + "mov %%ebp, %c[rbp](%0) \n\t" "mov %%cr2, %%eax \n\t" - "mov %%eax, %c[cr2](%3) \n\t" + "mov %%eax, %c[cr2](%0) \n\t" - "pop %%ecx; pop %%ebp; pop %%edx \n\t" + "pop %%ebp; pop %%ebp; pop %%edx \n\t" #endif - "setbe %0 \n\t" - : "=q" (vmx->fail) - : "r"(vmx->launched), "d"((unsigned long)HOST_RSP), - "c"(vcpu), - [rax]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RAX])), - [rbx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP])), + "setbe %c[fail](%0) \n\t" + : : "c"(vmx), "d"((unsigned long)HOST_RSP), + [launched]"i"(offsetof(struct vcpu_vmx, launched)), + [fail]"i"(offsetof(struct vcpu_vmx, fail)), + [rax]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RAX])), + [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RBP])), #ifdef CONFIG_X86_64 - [r8]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])), + [r8]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R15])), #endif - [cr2]"i"(offsetof(struct kvm_vcpu, cr2)) + [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.cr2)) : "cc", "memory" #ifdef CONFIG_X86_64 , "rbx", "rdi", "rsi" -- cgit v1.2.3 From 8b0067913d9d0439d4cf3c1f7314ba71fc7b2958 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 16 Nov 2007 13:05:55 +0800 Subject: KVM: Portability: Make kvm_vcpu_ioctl_translate arch dependent Move kvm_vcpu_ioctl_translate to arch, since mmu would be put under arch. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 +++ drivers/kvm/kvm_main.c | 24 +----------------------- drivers/kvm/x86.c | 22 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 23 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index a7be0733bc48..e34e246eafec 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -642,6 +642,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm); int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr); + int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index c782f27265de..3946025f3d37 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -761,28 +761,6 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -/* - * Translate a guest virtual address to a guest physical address. - */ -static int kvm_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, - struct kvm_translation *tr) -{ - unsigned long vaddr = tr->linear_address; - gpa_t gpa; - - vcpu_load(vcpu); - mutex_lock(&vcpu->kvm->lock); - gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr); - tr->physical_address = gpa; - tr->valid = gpa != UNMAPPED_GVA; - tr->writeable = 1; - tr->usermode = 0; - mutex_unlock(&vcpu->kvm->lock); - vcpu_put(vcpu); - - return 0; -} - static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) { @@ -986,7 +964,7 @@ static long kvm_vcpu_ioctl(struct file *filp, r = -EFAULT; if (copy_from_user(&tr, argp, sizeof tr)) goto out; - r = kvm_vcpu_ioctl_translate(vcpu, &tr); + r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); if (r) goto out; r = -EFAULT; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 609792637fb4..f1746afab9e5 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2318,6 +2318,28 @@ struct fxsave { #endif }; +/* + * Translate a guest virtual address to a guest physical address. + */ +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + unsigned long vaddr = tr->linear_address; + gpa_t gpa; + + vcpu_load(vcpu); + mutex_lock(&vcpu->kvm->lock); + gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr); + tr->physical_address = gpa; + tr->valid = gpa != UNMAPPED_GVA; + tr->writeable = 1; + tr->usermode = 0; + mutex_unlock(&vcpu->kvm->lock); + vcpu_put(vcpu); + + return 0; +} + int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; -- cgit v1.2.3 From 56919c5c97aa07b150311c5c9802af07872ee2b0 Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Mon, 12 Nov 2007 20:06:51 -0800 Subject: KVM: Remove ptr comparisons to 0 Fix sparse warnings "Using plain integer as NULL pointer" Signed-off-by: Joe Perches Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/kvm_main.c | 3 ++- drivers/kvm/svm.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index e34e246eafec..c4ad66bb5e08 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -398,7 +398,7 @@ static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) static inline int irqchip_in_kernel(struct kvm *kvm) { - return pic_irqchip(kvm) != 0; + return pic_irqchip(kvm) != NULL; } struct descriptor_table { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 3946025f3d37..7335435f528e 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1449,7 +1449,8 @@ int kvm_init(void *opaque, unsigned int vcpu_size, /* A kmem cache lets us meet the alignment requirements of fx_save. */ kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, - __alignof__(struct kvm_vcpu), 0, 0); + __alignof__(struct kvm_vcpu), + 0, NULL); if (!kvm_vcpu_cache) { r = -ENOMEM; goto out_free_4; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 0ccbef18476a..a1a7f39ff3f1 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1271,7 +1271,7 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) exit_code); if (exit_code >= ARRAY_SIZE(svm_exit_handlers) - || svm_exit_handlers[exit_code] == 0) { + || !svm_exit_handlers[exit_code]) { kvm_run->exit_reason = KVM_EXIT_UNKNOWN; kvm_run->hw.hardware_exit_reason = exit_code; return 0; -- cgit v1.2.3 From a16b043cc96db4a01abb337bef4a51cebcfcbb1b Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 16 Nov 2007 14:38:21 +0800 Subject: KVM: Remove __init attributes for kvm_init_debug and kvm_init_msr_list Since their callers are not declared with __init. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 2 +- drivers/kvm/x86.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7335435f528e..e24489b409cf 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1332,7 +1332,7 @@ static u64 stat_get(void *_offset) DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n"); -static __init void kvm_init_debug(void) +static void kvm_init_debug(void) { struct kvm_stats_debugfs_item *p; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index f1746afab9e5..abb7beeb8f54 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1049,7 +1049,7 @@ out: return r; } -static __init void kvm_init_msr_list(void) +static void kvm_init_msr_list(void) { u32 dummy[2]; unsigned i, j; -- cgit v1.2.3 From d19a9cd275b0fdc793d1bb8b644b7aad0517e4bc Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 18 Nov 2007 18:43:45 +0800 Subject: KVM: Portability: Add two hooks to handle kvm_create and destroy vm Add two arch hooks to handle kvm_create_vm and kvm destroy_vm. Now, just put io_bus init and destory in common. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++++ drivers/kvm/kvm_main.c | 42 ++++++------------------------------------ drivers/kvm/x86.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 36 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index c4ad66bb5e08..59e001c3ff09 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -674,6 +674,10 @@ int kvm_arch_hardware_setup(void); void kvm_arch_hardware_unsetup(void); void kvm_arch_check_processor_compat(void *rtn); +void kvm_free_physmem(struct kvm *kvm); + +struct kvm *kvm_arch_create_vm(void); +void kvm_arch_destroy_vm(struct kvm *kvm); static inline void kvm_guest_enter(void) { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index e24489b409cf..bde3cf741892 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -156,18 +156,18 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); static struct kvm *kvm_create_vm(void) { - struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); + struct kvm *kvm = kvm_arch_create_vm(); - if (!kvm) - return ERR_PTR(-ENOMEM); + if (IS_ERR(kvm)) + goto out; kvm_io_bus_init(&kvm->pio_bus); mutex_init(&kvm->lock); - INIT_LIST_HEAD(&kvm->active_mmu_pages); kvm_io_bus_init(&kvm->mmio_bus); spin_lock(&kvm_lock); list_add(&kvm->vm_list, &vm_list); spin_unlock(&kvm_lock); +out: return kvm; } @@ -188,7 +188,7 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free, free->rmap = NULL; } -static void kvm_free_physmem(struct kvm *kvm) +void kvm_free_physmem(struct kvm *kvm) { int i; @@ -196,32 +196,6 @@ static void kvm_free_physmem(struct kvm *kvm) kvm_free_physmem_slot(&kvm->memslots[i], NULL); } -static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) -{ - vcpu_load(vcpu); - kvm_mmu_unload(vcpu); - vcpu_put(vcpu); -} - -static void kvm_free_vcpus(struct kvm *kvm) -{ - unsigned int i; - - /* - * Unpin any mmu pages first. - */ - for (i = 0; i < KVM_MAX_VCPUS; ++i) - if (kvm->vcpus[i]) - kvm_unload_vcpu_mmu(kvm->vcpus[i]); - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - if (kvm->vcpus[i]) { - kvm_arch_vcpu_free(kvm->vcpus[i]); - kvm->vcpus[i] = NULL; - } - } - -} - static void kvm_destroy_vm(struct kvm *kvm) { spin_lock(&kvm_lock); @@ -229,11 +203,7 @@ static void kvm_destroy_vm(struct kvm *kvm) spin_unlock(&kvm_lock); kvm_io_bus_destroy(&kvm->pio_bus); kvm_io_bus_destroy(&kvm->mmio_bus); - kfree(kvm->vpic); - kfree(kvm->vioapic); - kvm_free_vcpus(kvm); - kvm_free_physmem(kvm); - kfree(kvm); + kvm_arch_destroy_vm(kvm); } static int kvm_vm_release(struct inode *inode, struct file *filp) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index abb7beeb8f54..b7c72ac36735 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2543,3 +2543,50 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) kvm_mmu_destroy(vcpu); free_page((unsigned long)vcpu->pio_data); } + +struct kvm *kvm_arch_create_vm(void) +{ + struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); + + if (!kvm) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&kvm->active_mmu_pages); + + return kvm; +} + +static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) +{ + vcpu_load(vcpu); + kvm_mmu_unload(vcpu); + vcpu_put(vcpu); +} + +static void kvm_free_vcpus(struct kvm *kvm) +{ + unsigned int i; + + /* + * Unpin any mmu pages first. + */ + for (i = 0; i < KVM_MAX_VCPUS; ++i) + if (kvm->vcpus[i]) + kvm_unload_vcpu_mmu(kvm->vcpus[i]); + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + if (kvm->vcpus[i]) { + kvm_arch_vcpu_free(kvm->vcpus[i]); + kvm->vcpus[i] = NULL; + } + } + +} + +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + kfree(kvm->vpic); + kfree(kvm->vioapic); + kvm_free_vcpus(kvm); + kvm_free_physmem(kvm); + kfree(kvm); +} -- cgit v1.2.3 From e1beb1d37c3187aa12c6463bd15ba594e9986761 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 13:50:24 +0200 Subject: KVM: Replace 'light_exits' stat with 'host_state_reload' This is a little more accurate (since it counts actual reloads, not potential reloads), and reverses the sense of the statistic to measure a bad event like most of the other stats (e.g. we want to minimize all counters). Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/svm.c | 1 + drivers/kvm/vmx.c | 1 + drivers/kvm/x86.c | 6 ++---- 4 files changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 59e001c3ff09..04efe8833e09 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -246,7 +246,7 @@ struct kvm_stat { u32 halt_wakeup; u32 request_irq_exits; u32 irq_exits; - u32 light_exits; + u32 host_state_reload; u32 efer_reload; }; diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index a1a7f39ff3f1..0d32304944fc 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -654,6 +654,7 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); int i; + ++vcpu->stat.host_state_reload; for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 84c77fed9d17..7130f315afdf 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -463,6 +463,7 @@ static void vmx_load_host_state(struct vcpu_vmx *vmx) if (!vmx->host_state.loaded) return; + ++vmx->vcpu.stat.host_state_reload; vmx->host_state.loaded = 0; if (vmx->host_state.fs_reload_needed) load_fs(vmx->host_state.fs_sel); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index b7c72ac36735..923dfd4e0afd 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -60,7 +60,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "halt_wakeup", STAT_OFFSET(halt_wakeup) }, { "request_irq", STAT_OFFSET(request_irq_exits) }, { "irq_exits", STAT_OFFSET(irq_exits) }, - { "light_exits", STAT_OFFSET(light_exits) }, + { "host_state_reload", STAT_OFFSET(host_state_reload) }, { "efer_reload", STAT_OFFSET(efer_reload) }, { NULL } }; @@ -1988,10 +1988,8 @@ again: ++vcpu->stat.request_irq_exits; goto out; } - if (!need_resched()) { - ++vcpu->stat.light_exits; + if (!need_resched()) goto again; - } } out: -- cgit v1.2.3 From f096ed8588b53bf802b84862c6c4d00d25d1ed72 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 13:54:33 +0200 Subject: KVM: Add fpu_reload counter Measure the number of times we switch the fpu state. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/x86.c | 2 ++ 2 files changed, 3 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 04efe8833e09..a85c5903591f 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -248,6 +248,7 @@ struct kvm_stat { u32 irq_exits; u32 host_state_reload; u32 efer_reload; + u32 fpu_reload; }; struct kvm_io_device { diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 923dfd4e0afd..c1211e125b43 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -62,6 +62,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "irq_exits", STAT_OFFSET(irq_exits) }, { "host_state_reload", STAT_OFFSET(host_state_reload) }, { "efer_reload", STAT_OFFSET(efer_reload) }, + { "fpu_reload", STAT_OFFSET(fpu_reload) }, { NULL } }; @@ -2417,6 +2418,7 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) vcpu->guest_fpu_loaded = 0; fx_save(&vcpu->guest_fx_image); fx_restore(&vcpu->host_fx_image); + ++vcpu->stat.fpu_reload; } EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); -- cgit v1.2.3 From f2b5756bb3fbdca912c4890e444c18650389d8ae Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 15:17:51 +0200 Subject: KVM: Add instruction emulation statistics --- drivers/kvm/kvm.h | 2 ++ drivers/kvm/x86.c | 4 ++++ 2 files changed, 6 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index a85c5903591f..5a8a9af3593a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -249,6 +249,8 @@ struct kvm_stat { u32 host_state_reload; u32 efer_reload; u32 fpu_reload; + u32 insn_emulation; + u32 insn_emulation_fail; }; struct kvm_io_device { diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c1211e125b43..a46b95b3651c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -63,6 +63,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "host_state_reload", STAT_OFFSET(host_state_reload) }, { "efer_reload", STAT_OFFSET(efer_reload) }, { "fpu_reload", STAT_OFFSET(fpu_reload) }, + { "insn_emulation", STAT_OFFSET(insn_emulation) }, + { "insn_emulation_fail", STAT_OFFSET(insn_emulation_fail) }, { NULL } }; @@ -1381,7 +1383,9 @@ int emulate_instruction(struct kvm_vcpu *vcpu, get_segment_base(vcpu, VCPU_SREG_FS); r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + ++vcpu->stat.insn_emulation; if (r) { + ++vcpu->stat.insn_emulation_fail; if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) return EMULATE_DONE; return EMULATE_FAIL; -- cgit v1.2.3 From ba1389b7a04de07e6231693b7ebb34f5b5d1a3e6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 16:24:12 +0200 Subject: KVM: Extend stats support for VM stats This is in addition to the current virtual cpu statistics. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 14 ++++++++++++-- drivers/kvm/kvm_main.c | 26 +++++++++++++++++++++++--- drivers/kvm/x86.c | 39 ++++++++++++++++++++------------------- 3 files changed, 55 insertions(+), 24 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 5a8a9af3593a..d3171f9c9c01 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -231,7 +231,7 @@ struct kvm_pio_request { int rep; }; -struct kvm_stat { +struct kvm_vcpu_stat { u32 pf_fixed; u32 pf_guest; u32 tlb_flush; @@ -342,7 +342,7 @@ void kvm_io_bus_register_dev(struct kvm_io_bus *bus, wait_queue_head_t wq; \ int sigset_active; \ sigset_t sigset; \ - struct kvm_stat stat; \ + struct kvm_vcpu_stat stat; \ KVM_VCPU_MMIO struct kvm_mem_alias { @@ -361,6 +361,9 @@ struct kvm_memory_slot { int user_alloc; }; +struct kvm_vm_stat { +}; + struct kvm { struct mutex lock; /* protects everything except vcpus */ int naliases; @@ -387,6 +390,7 @@ struct kvm { int round_robin_prev_vcpu; unsigned int tss_addr; struct page *apic_access_page; + struct kvm_vm_stat stat; }; static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) @@ -809,9 +813,15 @@ static inline u32 get_rdx_init_val(void) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) +enum kvm_stat_kind { + KVM_STAT_VM, + KVM_STAT_VCPU, +}; + struct kvm_stats_debugfs_item { const char *name; int offset; + enum kvm_stat_kind kind; struct dentry *dentry; }; extern struct kvm_stats_debugfs_item debugfs_entries[]; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index bde3cf741892..b0b6ff2a8cbe 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1281,7 +1281,22 @@ static struct notifier_block kvm_cpu_notifier = { .priority = 20, /* must be > scheduler priority */ }; -static u64 stat_get(void *_offset) +static u64 vm_stat_get(void *_offset) +{ + unsigned offset = (long)_offset; + u64 total = 0; + struct kvm *kvm; + + spin_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) + total += *(u32 *)((void *)kvm + offset); + spin_unlock(&kvm_lock); + return total; +} + +DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); + +static u64 vcpu_stat_get(void *_offset) { unsigned offset = (long)_offset; u64 total = 0; @@ -1300,7 +1315,12 @@ static u64 stat_get(void *_offset) return total; } -DEFINE_SIMPLE_ATTRIBUTE(stat_fops, stat_get, NULL, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); + +static struct file_operations *stat_fops[] = { + [KVM_STAT_VCPU] = &vcpu_stat_fops, + [KVM_STAT_VM] = &vm_stat_fops, +}; static void kvm_init_debug(void) { @@ -1310,7 +1330,7 @@ static void kvm_init_debug(void) for (p = debugfs_entries; p->name; ++p) p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir, (void *)(long)p->offset, - &stat_fops); + stat_fops[p->kind]); } static void kvm_exit_debug(void) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index a46b95b3651c..016abc3357e9 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -42,29 +42,30 @@ #define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) #define EFER_RESERVED_BITS 0xfffffffffffff2fe -#define STAT_OFFSET(x) offsetof(struct kvm_vcpu, stat.x) +#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM +#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU struct kvm_x86_ops *kvm_x86_ops; struct kvm_stats_debugfs_item debugfs_entries[] = { - { "pf_fixed", STAT_OFFSET(pf_fixed) }, - { "pf_guest", STAT_OFFSET(pf_guest) }, - { "tlb_flush", STAT_OFFSET(tlb_flush) }, - { "invlpg", STAT_OFFSET(invlpg) }, - { "exits", STAT_OFFSET(exits) }, - { "io_exits", STAT_OFFSET(io_exits) }, - { "mmio_exits", STAT_OFFSET(mmio_exits) }, - { "signal_exits", STAT_OFFSET(signal_exits) }, - { "irq_window", STAT_OFFSET(irq_window_exits) }, - { "halt_exits", STAT_OFFSET(halt_exits) }, - { "halt_wakeup", STAT_OFFSET(halt_wakeup) }, - { "request_irq", STAT_OFFSET(request_irq_exits) }, - { "irq_exits", STAT_OFFSET(irq_exits) }, - { "host_state_reload", STAT_OFFSET(host_state_reload) }, - { "efer_reload", STAT_OFFSET(efer_reload) }, - { "fpu_reload", STAT_OFFSET(fpu_reload) }, - { "insn_emulation", STAT_OFFSET(insn_emulation) }, - { "insn_emulation_fail", STAT_OFFSET(insn_emulation_fail) }, + { "pf_fixed", VCPU_STAT(pf_fixed) }, + { "pf_guest", VCPU_STAT(pf_guest) }, + { "tlb_flush", VCPU_STAT(tlb_flush) }, + { "invlpg", VCPU_STAT(invlpg) }, + { "exits", VCPU_STAT(exits) }, + { "io_exits", VCPU_STAT(io_exits) }, + { "mmio_exits", VCPU_STAT(mmio_exits) }, + { "signal_exits", VCPU_STAT(signal_exits) }, + { "irq_window", VCPU_STAT(irq_window_exits) }, + { "halt_exits", VCPU_STAT(halt_exits) }, + { "halt_wakeup", VCPU_STAT(halt_wakeup) }, + { "request_irq", VCPU_STAT(request_irq_exits) }, + { "irq_exits", VCPU_STAT(irq_exits) }, + { "host_state_reload", VCPU_STAT(host_state_reload) }, + { "efer_reload", VCPU_STAT(efer_reload) }, + { "fpu_reload", VCPU_STAT(fpu_reload) }, + { "insn_emulation", VCPU_STAT(insn_emulation) }, + { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, { NULL } }; -- cgit v1.2.3 From 4cee576493b6abc95cc7447a65f1b9d2b40b8f20 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 18 Nov 2007 16:37:07 +0200 Subject: KVM: MMU: Add some mmu statistics Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 6 ++++++ drivers/kvm/mmu.c | 9 ++++++++- drivers/kvm/x86.c | 6 ++++++ 3 files changed, 20 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index d3171f9c9c01..bdcc44e6b6d7 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -362,6 +362,12 @@ struct kvm_memory_slot { }; struct kvm_vm_stat { + u32 mmu_shadow_zapped; + u32 mmu_pte_write; + u32 mmu_pte_updated; + u32 mmu_pde_zapped; + u32 mmu_flooded; + u32 mmu_recycled; }; struct kvm { diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 9be54a5e858e..87d8e70fe502 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -755,6 +755,7 @@ static void kvm_mmu_zap_page(struct kvm *kvm, { u64 *parent_pte; + ++kvm->stat.mmu_shadow_zapped; while (page->multimapped || page->parent_pte) { if (!page->multimapped) parent_pte = page->parent_pte; @@ -1226,9 +1227,12 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, const void *new, int bytes, int offset_in_pte) { - if (page->role.level != PT_PAGE_TABLE_LEVEL) + if (page->role.level != PT_PAGE_TABLE_LEVEL) { + ++vcpu->kvm->stat.mmu_pde_zapped; return; + } + ++vcpu->kvm->stat.mmu_pte_updated; if (page->role.glevels == PT32_ROOT_LEVEL) paging32_update_pte(vcpu, page, spte, new, bytes, offset_in_pte); @@ -1263,6 +1267,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, int npte; pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); + ++vcpu->kvm->stat.mmu_pte_write; kvm_mmu_audit(vcpu, "pre pte write"); if (gfn == vcpu->last_pt_write_gfn && !last_updated_pte_accessed(vcpu)) { @@ -1296,6 +1301,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, pgprintk("misaligned: gpa %llx bytes %d role %x\n", gpa, bytes, page->role.word); kvm_mmu_zap_page(vcpu->kvm, page); + ++vcpu->kvm->stat.mmu_flooded; continue; } page_offset = offset; @@ -1344,6 +1350,7 @@ void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) page = container_of(vcpu->kvm->active_mmu_pages.prev, struct kvm_mmu_page, link); kvm_mmu_zap_page(vcpu->kvm, page); + ++vcpu->kvm->stat.mmu_recycled; } } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 016abc3357e9..fdc7632d7620 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -66,6 +66,12 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "fpu_reload", VCPU_STAT(fpu_reload) }, { "insn_emulation", VCPU_STAT(insn_emulation) }, { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, + { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, + { "mmu_pte_write", VM_STAT(mmu_pte_write) }, + { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, + { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, + { "mmu_flooded", VM_STAT(mmu_flooded) }, + { "mmu_recycled", VM_STAT(mmu_recycled) }, { NULL } }; -- cgit v1.2.3 From 9327fd1195b77e67b86db8c1c1c4c4f7db009b00 Mon Sep 17 00:00:00 2001 From: Amit Shah Date: Thu, 15 Nov 2007 18:38:46 +0200 Subject: KVM: Make unloading of FPU state when putting vcpu arch-independent Instead of having each architecture do it individually, we do this in the arch-independent code (just x86 as of now). [avi: add svm to the mix, which was added to mainline during the 2.6.24-rc process] Signed-off-by: Amit Shah Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 1 - drivers/kvm/vmx.c | 1 - drivers/kvm/x86.c | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 0d32304944fc..fb3721d88dbf 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -659,7 +659,6 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); rdtscll(vcpu->host_tsc); - kvm_put_guest_fpu(vcpu); } static void svm_vcpu_decache(struct kvm_vcpu *vcpu) diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 7130f315afdf..0c082faaa6db 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -541,7 +541,6 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) static void vmx_vcpu_put(struct kvm_vcpu *vcpu) { vmx_load_host_state(to_vmx(vcpu)); - kvm_put_guest_fpu(vcpu); } static void vmx_fpu_activate(struct kvm_vcpu *vcpu) diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index fdc7632d7620..9618fcba887e 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -678,6 +678,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) { kvm_x86_ops->vcpu_put(vcpu); + kvm_put_guest_fpu(vcpu); } static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 5bb064dcdeb7ab341e2f8a3e2fc34faa63b1662c Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 18 Nov 2007 20:29:43 +0800 Subject: KVM: Portability: Move kvm_vcpu_ioctl_get_dirty_log to arch-specific file Meanwhile keep the interface in common, and leave as more logic in common as possible. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 5 +++++ drivers/kvm/kvm_main.c | 19 ++++--------------- drivers/kvm/x86.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index bdcc44e6b6d7..c1aa84f7ca0a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -644,6 +644,11 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); int kvm_dev_ioctl_check_extension(long ext); +int kvm_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log, int *is_dirty); +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log); + int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index b0b6ff2a8cbe..5f11e6b09458 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -389,19 +389,14 @@ int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, return kvm_set_memory_region(kvm, mem, user_alloc); } -/* - * Get (and clear) the dirty memory log for a memory slot. - */ -static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log) +int kvm_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log, int *is_dirty) { struct kvm_memory_slot *memslot; int r, i; int n; unsigned long any = 0; - mutex_lock(&kvm->lock); - r = -EINVAL; if (log->slot >= KVM_MEMORY_SLOTS) goto out; @@ -420,17 +415,11 @@ static int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) goto out; - /* If nothing is dirty, don't bother messing with page tables. */ - if (any) { - kvm_mmu_slot_remove_write_access(kvm, log->slot); - kvm_flush_remote_tlbs(kvm); - memset(memslot->dirty_bitmap, 0, n); - } + if (any) + *is_dirty = 1; r = 0; - out: - mutex_unlock(&kvm->lock); return r; } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 9618fcba887e..935e2769b787 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -937,6 +937,37 @@ static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) return r; } +/* + * Get (and clear) the dirty memory log for a memory slot. + */ +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log) +{ + int r; + int n; + struct kvm_memory_slot *memslot; + int is_dirty = 0; + + mutex_lock(&kvm->lock); + + r = kvm_get_dirty_log(kvm, log, &is_dirty); + if (r) + goto out; + + /* If nothing is dirty, don't bother messing with page tables. */ + if (is_dirty) { + kvm_mmu_slot_remove_write_access(kvm, log->slot); + kvm_flush_remote_tlbs(kvm); + memslot = &kvm->memslots[log->slot]; + n = ALIGN(memslot->npages, BITS_PER_LONG) / 8; + memset(memslot->dirty_bitmap, 0, n); + } + r = 0; +out: + mutex_unlock(&kvm->lock); + return r; +} + long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { -- cgit v1.2.3 From 56c6d28a9afdca0d48dd618276e055f19c0306bb Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 18 Nov 2007 20:43:21 +0800 Subject: KVM: Portability: MMU initialization and teardown split Move out kvm_mmu init and exit functionality from kvm_main.c Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 8 -------- drivers/kvm/x86.c | 24 +++++++++++++++++++----- 2 files changed, 19 insertions(+), 13 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 5f11e6b09458..595ed323fc53 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1383,10 +1383,6 @@ int kvm_init(void *opaque, unsigned int vcpu_size, int r; int cpu; - r = kvm_mmu_module_init(); - if (r) - goto out4; - kvm_init_debug(); r = kvm_arch_init(opaque); @@ -1446,8 +1442,6 @@ int kvm_init(void *opaque, unsigned int vcpu_size, kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out; - kvm_mmu_set_nonpresent_ptes(0ull, 0ull); - return 0; out_free: @@ -1466,7 +1460,6 @@ out_free_0: out: kvm_arch_exit(); kvm_exit_debug(); - kvm_mmu_module_exit(); out4: return r; } @@ -1485,6 +1478,5 @@ void kvm_exit(void) kvm_arch_exit(); kvm_exit_debug(); __free_page(bad_page); - kvm_mmu_module_exit(); } EXPORT_SYMBOL_GPL(kvm_exit); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 935e2769b787..2257a0aab32f 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1711,33 +1711,47 @@ EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); int kvm_arch_init(void *opaque) { + int r; struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; + r = kvm_mmu_module_init(); + if (r) + goto out_fail; + kvm_init_msr_list(); if (kvm_x86_ops) { printk(KERN_ERR "kvm: already loaded the other module\n"); - return -EEXIST; + r = -EEXIST; + goto out; } if (!ops->cpu_has_kvm_support()) { printk(KERN_ERR "kvm: no hardware support\n"); - return -EOPNOTSUPP; + r = -EOPNOTSUPP; + goto out; } if (ops->disabled_by_bios()) { printk(KERN_ERR "kvm: disabled by bios\n"); - return -EOPNOTSUPP; + r = -EOPNOTSUPP; + goto out; } kvm_x86_ops = ops; - + kvm_mmu_set_nonpresent_ptes(0ull, 0ull); return 0; + +out: + kvm_mmu_module_exit(); +out_fail: + return r; } void kvm_arch_exit(void) { kvm_x86_ops = NULL; - } + kvm_mmu_module_exit(); +} int kvm_emulate_halt(struct kvm_vcpu *vcpu) { -- cgit v1.2.3 From cd6e8f87efb74d91100d84894021dd12fea1a8a5 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 14:33:37 +0800 Subject: KVM: Portability: Move some macro definitions from kvm.h to x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 33 --------------------------------- drivers/kvm/x86.h | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 33 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index c1aa84f7ca0a..aceecf4e4f2d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -20,24 +20,6 @@ #include #include -#define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) -#define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) -#define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) - -#define KVM_GUEST_CR0_MASK \ - (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE \ - | X86_CR0_NW | X86_CR0_CD) -#define KVM_VM_CR0_ALWAYS_ON \ - (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE | X86_CR0_TS \ - | X86_CR0_MP) -#define KVM_GUEST_CR4_MASK \ - (X86_CR4_VME | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_VMXE) -#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) -#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) - -#define INVALID_PAGE (~(hpa_t)0) -#define UNMAPPED_GVA (~(gpa_t)0) - #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 @@ -50,21 +32,6 @@ #define KVM_REFILL_PAGES 25 #define KVM_MAX_CPUID_ENTRIES 40 -#define DE_VECTOR 0 -#define UD_VECTOR 6 -#define NM_VECTOR 7 -#define DF_VECTOR 8 -#define TS_VECTOR 10 -#define NP_VECTOR 11 -#define SS_VECTOR 12 -#define GP_VECTOR 13 -#define PF_VECTOR 14 - -#define SELECTOR_TI_MASK (1 << 2) -#define SELECTOR_RPL_MASK 0x03 - -#define IOPL_SHIFT 12 - #define KVM_PIO_PAGE_OFFSET 1 /* diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 4df064100226..ec1d66940c80 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -19,6 +19,39 @@ #include #include +#define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) +#define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) +#define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) + +#define KVM_GUEST_CR0_MASK \ + (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE \ + | X86_CR0_NW | X86_CR0_CD) +#define KVM_VM_CR0_ALWAYS_ON \ + (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE | X86_CR0_TS \ + | X86_CR0_MP) +#define KVM_GUEST_CR4_MASK \ + (X86_CR4_VME | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_VMXE) +#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) +#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) + +#define INVALID_PAGE (~(hpa_t)0) +#define UNMAPPED_GVA (~(gpa_t)0) + +#define DE_VECTOR 0 +#define UD_VECTOR 6 +#define NM_VECTOR 7 +#define DF_VECTOR 8 +#define TS_VECTOR 10 +#define NP_VECTOR 11 +#define SS_VECTOR 12 +#define GP_VECTOR 13 +#define PF_VECTOR 14 + +#define SELECTOR_TI_MASK (1 << 2) +#define SELECTOR_RPL_MASK 0x03 + +#define IOPL_SHIFT 12 + extern spinlock_t kvm_lock; extern struct list_head vm_list; -- cgit v1.2.3 From ea4a5ff80c24a25034a0c0468f6d1f47d1782076 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 14:40:47 +0800 Subject: KVM: Portability: Move struct kvm_x86_ops definition to x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 69 ------------------------------------------------------- drivers/kvm/x86.h | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 69 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index aceecf4e4f2d..e4e1ff7e6f9e 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -386,75 +386,6 @@ struct descriptor_table { unsigned long base; } __attribute__((packed)); -struct kvm_x86_ops { - int (*cpu_has_kvm_support)(void); /* __init */ - int (*disabled_by_bios)(void); /* __init */ - void (*hardware_enable)(void *dummy); /* __init */ - void (*hardware_disable)(void *dummy); - void (*check_processor_compatibility)(void *rtn); - int (*hardware_setup)(void); /* __init */ - void (*hardware_unsetup)(void); /* __exit */ - - /* Create, but do not attach this VCPU */ - struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); - void (*vcpu_free)(struct kvm_vcpu *vcpu); - int (*vcpu_reset)(struct kvm_vcpu *vcpu); - - void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); - void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); - void (*vcpu_put)(struct kvm_vcpu *vcpu); - void (*vcpu_decache)(struct kvm_vcpu *vcpu); - - int (*set_guest_debug)(struct kvm_vcpu *vcpu, - struct kvm_debug_guest *dbg); - void (*guest_debug_pre)(struct kvm_vcpu *vcpu); - int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); - int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); - u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); - void (*get_segment)(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg); - void (*set_segment)(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg); - void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); - void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); - void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); - void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); - void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); - void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); - void (*get_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*set_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*get_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*set_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - unsigned long (*get_dr)(struct kvm_vcpu *vcpu, int dr); - void (*set_dr)(struct kvm_vcpu *vcpu, int dr, unsigned long value, - int *exception); - void (*cache_regs)(struct kvm_vcpu *vcpu); - void (*decache_regs)(struct kvm_vcpu *vcpu); - unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); - void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); - - void (*tlb_flush)(struct kvm_vcpu *vcpu); - void (*inject_page_fault)(struct kvm_vcpu *vcpu, - unsigned long addr, u32 err_code); - - void (*inject_gp)(struct kvm_vcpu *vcpu, unsigned err_code); - - void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run); - int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); - void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); - void (*patch_hypercall)(struct kvm_vcpu *vcpu, - unsigned char *hypercall_addr); - int (*get_irq)(struct kvm_vcpu *vcpu); - void (*set_irq)(struct kvm_vcpu *vcpu, int vec); - void (*inject_pending_irq)(struct kvm_vcpu *vcpu); - void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, - struct kvm_run *run); - - int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); -}; - -extern struct kvm_x86_ops *kvm_x86_ops; - /* The guest did something we don't support. */ #define pr_unimpl(vcpu, fmt, ...) \ do { \ diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index ec1d66940c80..77b4092c0542 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -121,6 +121,73 @@ struct kvm_vcpu { struct x86_emulate_ctxt emulate_ctxt; }; +struct kvm_x86_ops { + int (*cpu_has_kvm_support)(void); /* __init */ + int (*disabled_by_bios)(void); /* __init */ + void (*hardware_enable)(void *dummy); /* __init */ + void (*hardware_disable)(void *dummy); + void (*check_processor_compatibility)(void *rtn); + int (*hardware_setup)(void); /* __init */ + void (*hardware_unsetup)(void); /* __exit */ + + /* Create, but do not attach this VCPU */ + struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); + void (*vcpu_free)(struct kvm_vcpu *vcpu); + int (*vcpu_reset)(struct kvm_vcpu *vcpu); + + void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); + void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); + void (*vcpu_put)(struct kvm_vcpu *vcpu); + void (*vcpu_decache)(struct kvm_vcpu *vcpu); + + int (*set_guest_debug)(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg); + void (*guest_debug_pre)(struct kvm_vcpu *vcpu); + int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); + int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); + u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); + void (*get_segment)(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg); + void (*set_segment)(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg); + void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); + void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); + void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); + void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); + void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); + void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); + void (*get_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*set_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*get_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*set_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + unsigned long (*get_dr)(struct kvm_vcpu *vcpu, int dr); + void (*set_dr)(struct kvm_vcpu *vcpu, int dr, unsigned long value, + int *exception); + void (*cache_regs)(struct kvm_vcpu *vcpu); + void (*decache_regs)(struct kvm_vcpu *vcpu); + unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); + void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); + + void (*tlb_flush)(struct kvm_vcpu *vcpu); + void (*inject_page_fault)(struct kvm_vcpu *vcpu, + unsigned long addr, u32 err_code); + + void (*inject_gp)(struct kvm_vcpu *vcpu, unsigned err_code); + + void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run); + int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); + void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); + void (*patch_hypercall)(struct kvm_vcpu *vcpu, + unsigned char *hypercall_addr); + int (*get_irq)(struct kvm_vcpu *vcpu); + void (*set_irq)(struct kvm_vcpu *vcpu, int vec); + void (*inject_pending_irq)(struct kvm_vcpu *vcpu); + void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, + struct kvm_run *run); + + int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); +}; + extern struct kvm_x86_ops *kvm_x86_ops; int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); -- cgit v1.2.3 From 2b3ccfa0c5c7738a08f473b5d00b78f87935de72 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 14:56:05 +0800 Subject: KVM: Portability: Move vcpu regs enumeration definition to x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 35 ----------------------------------- drivers/kvm/x86.h | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 35 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index e4e1ff7e6f9e..1c4de50ac06a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -150,41 +150,6 @@ struct kvm_guest_debug { int singlestep; }; -enum { - VCPU_REGS_RAX = 0, - VCPU_REGS_RCX = 1, - VCPU_REGS_RDX = 2, - VCPU_REGS_RBX = 3, - VCPU_REGS_RSP = 4, - VCPU_REGS_RBP = 5, - VCPU_REGS_RSI = 6, - VCPU_REGS_RDI = 7, -#ifdef CONFIG_X86_64 - VCPU_REGS_R8 = 8, - VCPU_REGS_R9 = 9, - VCPU_REGS_R10 = 10, - VCPU_REGS_R11 = 11, - VCPU_REGS_R12 = 12, - VCPU_REGS_R13 = 13, - VCPU_REGS_R14 = 14, - VCPU_REGS_R15 = 15, -#endif - NR_VCPU_REGS -}; - -enum { - VCPU_SREG_CS, - VCPU_SREG_DS, - VCPU_SREG_ES, - VCPU_SREG_FS, - VCPU_SREG_GS, - VCPU_SREG_SS, - VCPU_SREG_TR, - VCPU_SREG_LDTR, -}; - -#include "x86_emulate.h" - struct kvm_pio_request { unsigned long count; int cur_count; diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 77b4092c0542..77a4a4ab9391 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -55,6 +55,41 @@ extern spinlock_t kvm_lock; extern struct list_head vm_list; +enum { + VCPU_REGS_RAX = 0, + VCPU_REGS_RCX = 1, + VCPU_REGS_RDX = 2, + VCPU_REGS_RBX = 3, + VCPU_REGS_RSP = 4, + VCPU_REGS_RBP = 5, + VCPU_REGS_RSI = 6, + VCPU_REGS_RDI = 7, +#ifdef CONFIG_X86_64 + VCPU_REGS_R8 = 8, + VCPU_REGS_R9 = 9, + VCPU_REGS_R10 = 10, + VCPU_REGS_R11 = 11, + VCPU_REGS_R12 = 12, + VCPU_REGS_R13 = 13, + VCPU_REGS_R14 = 14, + VCPU_REGS_R15 = 15, +#endif + NR_VCPU_REGS +}; + +enum { + VCPU_SREG_CS, + VCPU_SREG_DS, + VCPU_SREG_ES, + VCPU_SREG_FS, + VCPU_SREG_GS, + VCPU_SREG_SS, + VCPU_SREG_TR, + VCPU_SREG_LDTR, +}; + +#include "x86_emulate.h" + struct kvm_vcpu { KVM_VCPU_COMM; u64 host_tsc; -- cgit v1.2.3 From ec6d273deb56b1607e2acaad3df4bca42f135cd7 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 15:08:31 +0800 Subject: KVM: Move some static inline functions out from kvm.h into x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 109 ----------------------------------------------------- drivers/kvm/x86.h | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 109 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 1c4de50ac06a..41f6ee2134de 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -577,115 +577,6 @@ static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) return slot - kvm->memslots; } -static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) -{ - struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); - - return (struct kvm_mmu_page *)page_private(page); -} - -static inline u16 read_fs(void) -{ - u16 seg; - asm("mov %%fs, %0" : "=g"(seg)); - return seg; -} - -static inline u16 read_gs(void) -{ - u16 seg; - asm("mov %%gs, %0" : "=g"(seg)); - return seg; -} - -static inline u16 read_ldt(void) -{ - u16 ldt; - asm("sldt %0" : "=g"(ldt)); - return ldt; -} - -static inline void load_fs(u16 sel) -{ - asm("mov %0, %%fs" : : "rm"(sel)); -} - -static inline void load_gs(u16 sel) -{ - asm("mov %0, %%gs" : : "rm"(sel)); -} - -#ifndef load_ldt -static inline void load_ldt(u16 sel) -{ - asm("lldt %0" : : "rm"(sel)); -} -#endif - -static inline void get_idt(struct descriptor_table *table) -{ - asm("sidt %0" : "=m"(*table)); -} - -static inline void get_gdt(struct descriptor_table *table) -{ - asm("sgdt %0" : "=m"(*table)); -} - -static inline unsigned long read_tr_base(void) -{ - u16 tr; - asm("str %0" : "=g"(tr)); - return segment_base(tr); -} - -#ifdef CONFIG_X86_64 -static inline unsigned long read_msr(unsigned long msr) -{ - u64 value; - - rdmsrl(msr, value); - return value; -} -#endif - -static inline void fx_save(struct i387_fxsave_struct *image) -{ - asm("fxsave (%0)":: "r" (image)); -} - -static inline void fx_restore(struct i387_fxsave_struct *image) -{ - asm("fxrstor (%0)":: "r" (image)); -} - -static inline void fpu_init(void) -{ - asm("finit"); -} - -static inline u32 get_rdx_init_val(void) -{ - return 0x600; /* P6 family */ -} - -#define ASM_VMX_VMCLEAR_RAX ".byte 0x66, 0x0f, 0xc7, 0x30" -#define ASM_VMX_VMLAUNCH ".byte 0x0f, 0x01, 0xc2" -#define ASM_VMX_VMRESUME ".byte 0x0f, 0x01, 0xc3" -#define ASM_VMX_VMPTRLD_RAX ".byte 0x0f, 0xc7, 0x30" -#define ASM_VMX_VMREAD_RDX_RAX ".byte 0x0f, 0x78, 0xd0" -#define ASM_VMX_VMWRITE_RAX_RDX ".byte 0x0f, 0x79, 0xd0" -#define ASM_VMX_VMWRITE_RSP_RDX ".byte 0x0f, 0x79, 0xd4" -#define ASM_VMX_VMXOFF ".byte 0x0f, 0x01, 0xc4" -#define ASM_VMX_VMXON_RAX ".byte 0xf3, 0x0f, 0xc7, 0x30" - -#define MSR_IA32_TIME_STAMP_COUNTER 0x010 - -#define TSS_IOPB_BASE_OFFSET 0x66 -#define TSS_BASE_SIZE 0x68 -#define TSS_IOPB_SIZE (65536 / 8) -#define TSS_REDIRECTION_SIZE (256 / 8) -#define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) enum kvm_stat_kind { KVM_STAT_VM, diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 77a4a4ab9391..f1c43cafb0ca 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -267,4 +267,114 @@ static inline int is_paging(struct kvm_vcpu *vcpu) int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); int complete_pio(struct kvm_vcpu *vcpu); + +static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) +{ + struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); + + return (struct kvm_mmu_page *)page_private(page); +} + +static inline u16 read_fs(void) +{ + u16 seg; + asm("mov %%fs, %0" : "=g"(seg)); + return seg; +} + +static inline u16 read_gs(void) +{ + u16 seg; + asm("mov %%gs, %0" : "=g"(seg)); + return seg; +} + +static inline u16 read_ldt(void) +{ + u16 ldt; + asm("sldt %0" : "=g"(ldt)); + return ldt; +} + +static inline void load_fs(u16 sel) +{ + asm("mov %0, %%fs" : : "rm"(sel)); +} + +static inline void load_gs(u16 sel) +{ + asm("mov %0, %%gs" : : "rm"(sel)); +} + +#ifndef load_ldt +static inline void load_ldt(u16 sel) +{ + asm("lldt %0" : : "rm"(sel)); +} +#endif + +static inline void get_idt(struct descriptor_table *table) +{ + asm("sidt %0" : "=m"(*table)); +} + +static inline void get_gdt(struct descriptor_table *table) +{ + asm("sgdt %0" : "=m"(*table)); +} + +static inline unsigned long read_tr_base(void) +{ + u16 tr; + asm("str %0" : "=g"(tr)); + return segment_base(tr); +} + +#ifdef CONFIG_X86_64 +static inline unsigned long read_msr(unsigned long msr) +{ + u64 value; + + rdmsrl(msr, value); + return value; +} +#endif + +static inline void fx_save(struct i387_fxsave_struct *image) +{ + asm("fxsave (%0)":: "r" (image)); +} + +static inline void fx_restore(struct i387_fxsave_struct *image) +{ + asm("fxrstor (%0)":: "r" (image)); +} + +static inline void fpu_init(void) +{ + asm("finit"); +} + +static inline u32 get_rdx_init_val(void) +{ + return 0x600; /* P6 family */ +} + +#define ASM_VMX_VMCLEAR_RAX ".byte 0x66, 0x0f, 0xc7, 0x30" +#define ASM_VMX_VMLAUNCH ".byte 0x0f, 0x01, 0xc2" +#define ASM_VMX_VMRESUME ".byte 0x0f, 0x01, 0xc3" +#define ASM_VMX_VMPTRLD_RAX ".byte 0x0f, 0xc7, 0x30" +#define ASM_VMX_VMREAD_RDX_RAX ".byte 0x0f, 0x78, 0xd0" +#define ASM_VMX_VMWRITE_RAX_RDX ".byte 0x0f, 0x79, 0xd0" +#define ASM_VMX_VMWRITE_RSP_RDX ".byte 0x0f, 0x79, 0xd4" +#define ASM_VMX_VMXOFF ".byte 0x0f, 0x01, 0xc4" +#define ASM_VMX_VMXON_RAX ".byte 0xf3, 0x0f, 0xc7, 0x30" + +#define MSR_IA32_TIME_STAMP_COUNTER 0x010 + +#define TSS_IOPB_BASE_OFFSET 0x66 +#define TSS_BASE_SIZE 0x68 +#define TSS_IOPB_SIZE (65536 / 8) +#define TSS_REDIRECTION_SIZE (256 / 8) +#define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) #endif -- cgit v1.2.3 From 54f1585a8d0131bc3993902d4441f4049daed0d5 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Mon, 19 Nov 2007 15:24:28 +0800 Subject: KVM: Portability: Move some function declarations to x86.h Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 84 ------------------------------------------------------- drivers/kvm/x86.h | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 84 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 41f6ee2134de..19014564ddbb 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -375,19 +375,6 @@ int kvm_init(void *opaque, unsigned int vcpu_size, struct module *module); void kvm_exit(void); -int kvm_mmu_module_init(void); -void kvm_mmu_module_exit(void); - -void kvm_mmu_destroy(struct kvm_vcpu *vcpu); -int kvm_mmu_create(struct kvm_vcpu *vcpu); -int kvm_mmu_setup(struct kvm_vcpu *vcpu); -void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); - -int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); -void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); -void kvm_mmu_zap_all(struct kvm *kvm); -void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); - hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) @@ -421,83 +408,12 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); void mark_page_dirty(struct kvm *kvm, gfn_t gfn); -enum emulation_result { - EMULATE_DONE, /* no further processing */ - EMULATE_DO_MMIO, /* kvm_run filled with mmio request */ - EMULATE_FAIL, /* can't emulate this instruction */ -}; - -int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long cr2, u16 error_code, int no_decode); -void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); -void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); -void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); -void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, - unsigned long *rflags); - -unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr); -void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value, - unsigned long *rflags); -int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data); -int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); - -struct x86_emulate_ctxt; - -int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned port); -int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned long count, int down, - gva_t address, int rep, unsigned port); -void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); -int kvm_emulate_halt(struct kvm_vcpu *vcpu); -int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); -int emulate_clts(struct kvm_vcpu *vcpu); -int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, - unsigned long *dest); -int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, - unsigned long value); - -void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr0); -unsigned long get_cr8(struct kvm_vcpu *vcpu); -void lmsw(struct kvm_vcpu *vcpu, unsigned long msw); -void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); - -int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); -int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); - -void fx_init(struct kvm_vcpu *vcpu); - void kvm_vcpu_block(struct kvm_vcpu *vcpu); void kvm_resched(struct kvm_vcpu *vcpu); void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); void kvm_flush_remote_tlbs(struct kvm *kvm); -int emulator_read_std(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); -int emulator_write_emulated(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); - -unsigned long segment_base(u16 selector); - -void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, - const u8 *new, int bytes); -int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); -void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); -int kvm_mmu_load(struct kvm_vcpu *vcpu); -void kvm_mmu_unload(struct kvm_vcpu *vcpu); - -int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); - -int kvm_fix_hypercall(struct kvm_vcpu *vcpu); - long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); long kvm_arch_vcpu_ioctl(struct file *filp, diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index f1c43cafb0ca..90b791bf6287 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -225,6 +225,90 @@ struct kvm_x86_ops { extern struct kvm_x86_ops *kvm_x86_ops; +int kvm_mmu_module_init(void); +void kvm_mmu_module_exit(void); + +void kvm_mmu_destroy(struct kvm_vcpu *vcpu); +int kvm_mmu_create(struct kvm_vcpu *vcpu); +int kvm_mmu_setup(struct kvm_vcpu *vcpu); +void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); + +int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); +void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); +void kvm_mmu_zap_all(struct kvm *kvm); +void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); + +enum emulation_result { + EMULATE_DONE, /* no further processing */ + EMULATE_DO_MMIO, /* kvm_run filled with mmio request */ + EMULATE_FAIL, /* can't emulate this instruction */ +}; + +int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, + unsigned long cr2, u16 error_code, int no_decode); +void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); +void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); +void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); +void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, + unsigned long *rflags); + +unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr); +void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value, + unsigned long *rflags); +int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data); +int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); + +struct x86_emulate_ctxt; + +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned port); +int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned long count, int down, + gva_t address, int rep, unsigned port); +void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); +int kvm_emulate_halt(struct kvm_vcpu *vcpu); +int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); +int emulate_clts(struct kvm_vcpu *vcpu); +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, + unsigned long *dest); +int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, + unsigned long value); + +void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr0); +unsigned long get_cr8(struct kvm_vcpu *vcpu); +void lmsw(struct kvm_vcpu *vcpu, unsigned long msw); +void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); + +int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); +int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); + +void fx_init(struct kvm_vcpu *vcpu); + +int emulator_read_std(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); +int emulator_write_emulated(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); + +unsigned long segment_base(u16 selector); + +void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, + const u8 *new, int bytes); +int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); +void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); +int kvm_mmu_load(struct kvm_vcpu *vcpu); +void kvm_mmu_unload(struct kvm_vcpu *vcpu); + +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu); + int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 15b00f32d53f31fe194b17ca0384a37756b55be2 Mon Sep 17 00:00:00 2001 From: Jan Kiszka Date: Mon, 19 Nov 2007 10:21:45 +0100 Subject: KVM: VMX: Force seg.base == (seg.sel << 4) in real mode Ensure that segment.base == segment.selector << 4 when entering the real mode on Intel so that the CPU will not bark at us. This fixes some old protected mode demo from http://www.x86.org/articles/pmbasics/tspec_a1_doc.htm. Signed-off-by: Jan Kiszka Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 0c082faaa6db..b4c0bdce7b34 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1165,7 +1165,8 @@ static void fix_rmode_seg(int seg, struct kvm_save_segment *save) save->base = vmcs_readl(sf->base); save->limit = vmcs_read32(sf->limit); save->ar = vmcs_read32(sf->ar_bytes); - vmcs_write16(sf->selector, vmcs_readl(sf->base) >> 4); + vmcs_write16(sf->selector, save->base >> 4); + vmcs_write32(sf->base, save->base & 0xfffff); vmcs_write32(sf->limit, 0xffff); vmcs_write32(sf->ar_bytes, 0xf3); } -- cgit v1.2.3 From ec8d4eaefc06636eb8fc83610c6605168796b4f7 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Mon, 19 Nov 2007 11:28:19 +0200 Subject: KVM: MMU: Change guest pte access to kvm_{read,write}_guest() Things are simpler and more regular this way. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 0f0266af3f68..be6640178f6b 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -72,7 +72,6 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - struct page *page = NULL; pt_element_t *table; pt_element_t pte; gfn_t table_gfn; @@ -99,16 +98,13 @@ static int FNAME(walk_addr)(struct guest_walker *walker, index = PT_INDEX(addr, walker->level); table_gfn = (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + pte_gpa = table_gfn << PAGE_SHIFT; + pte_gpa += index * sizeof(pt_element_t); walker->table_gfn[walker->level - 1] = table_gfn; pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); - page = gfn_to_page(vcpu->kvm, (pte & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); - - table = kmap_atomic(page, KM_USER0); - pte = table[index]; - kunmap_atomic(table, KM_USER0); + kvm_read_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); if (!is_present_pte(pte)) goto not_present; @@ -128,9 +124,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, if (!(pte & PT_ACCESSED_MASK)) { mark_page_dirty(vcpu->kvm, table_gfn); pte |= PT_ACCESSED_MASK; - table = kmap_atomic(page, KM_USER0); - table[index] = pte; - kunmap_atomic(table, KM_USER0); + kvm_write_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); } if (walker->level == PT_PAGE_TABLE_LEVEL) { @@ -149,21 +143,15 @@ static int FNAME(walk_addr)(struct guest_walker *walker, walker->inherited_ar &= pte; --walker->level; - kvm_release_page(page); } if (write_fault && !is_dirty_pte(pte)) { mark_page_dirty(vcpu->kvm, table_gfn); pte |= PT_DIRTY_MASK; - table = kmap_atomic(page, KM_USER0); - table[index] = pte; - kunmap_atomic(table, KM_USER0); - pte_gpa = table_gfn << PAGE_SHIFT; - pte_gpa += index * sizeof(pt_element_t); + kvm_write_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); } - kvm_release_page(page); walker->pte = pte; pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)pte); return 1; @@ -182,8 +170,6 @@ err: walker->error_code |= PFERR_USER_MASK; if (fetch_fault) walker->error_code |= PFERR_FETCH_MASK; - if (page) - kvm_release_page(page); return 0; } -- cgit v1.2.3 From 3e021bf5053bc45448e43cf1d7d020ddf1409714 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Mon, 19 Nov 2007 11:16:57 +0200 Subject: KVM: Simplify kvm_clear_guest_page() Use kvm_write_guest_page() with empty_zero_page, instead of doing kmap and memset. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 595ed323fc53..e83a178dc80f 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -46,6 +46,7 @@ #include #include #include +#include MODULE_AUTHOR("Qumranet"); MODULE_LICENSE("GPL"); @@ -633,22 +634,7 @@ int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) { - void *page_virt; - struct page *page; - - page = gfn_to_page(kvm, gfn); - if (is_error_page(page)) { - kvm_release_page(page); - return -EFAULT; - } - page_virt = kmap_atomic(page, KM_USER0); - - memset(page_virt + offset, 0, len); - - kunmap_atomic(page_virt, KM_USER0); - kvm_release_page(page); - mark_page_dirty(kvm, gfn); - return 0; + return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len); } EXPORT_SYMBOL_GPL(kvm_clear_guest_page); -- cgit v1.2.3 From 971535ff65da2e967a1105a3b2fa3693fc83ca8f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 19 Nov 2007 18:44:15 +0200 Subject: KVM: MMU: Remove unused variable Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index be6640178f6b..77a2b22492bf 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -72,7 +72,6 @@ static int FNAME(walk_addr)(struct guest_walker *walker, struct kvm_vcpu *vcpu, gva_t addr, int write_fault, int user_fault, int fetch_fault) { - pt_element_t *table; pt_element_t pte; gfn_t table_gfn; unsigned index; -- cgit v1.2.3 From 00fc9f5ae5988fb180f5a288451a741b35cc806e Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 19 Nov 2007 14:04:45 -0600 Subject: KVM: Remove unused "rmap_overflow" variable Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 19014564ddbb..ba78a45811cd 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -318,7 +318,6 @@ struct kvm { unsigned int n_alloc_mmu_pages; struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; - unsigned long rmap_overflow; struct list_head vm_list; struct file *filp; struct kvm_io_bus mmio_bus; -- cgit v1.2.3 From d40ccc6246ed19ca529043998942a6b450c0e498 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 19 Nov 2007 14:04:43 -0600 Subject: KVM: Correct consistent typo: "destory" -> "destroy" Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 2 +- drivers/kvm/kvm_main.c | 2 +- drivers/kvm/x86.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ba78a45811cd..ccba9584355c 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -461,7 +461,7 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); -void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); void kvm_arch_hardware_enable(void *garbage); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index e83a178dc80f..4e1bd9488470 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -824,7 +824,7 @@ unlink: kvm->vcpus[n] = NULL; mutex_unlock(&kvm->lock); vcpu_destroy: - kvm_arch_vcpu_destory(vcpu); + kvm_arch_vcpu_destroy(vcpu); return r; } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 2257a0aab32f..5a1b72fbaeaa 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2513,7 +2513,7 @@ fail: return ERR_PTR(r); } -void kvm_arch_vcpu_destory(struct kvm_vcpu *vcpu) +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) { vcpu_load(vcpu); kvm_mmu_unload(vcpu); -- cgit v1.2.3 From 7faa8f6fcc119ce680a90a5f011841b466b1e161 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 19 Nov 2007 14:04:44 -0600 Subject: KVM: Move misplaced comment Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ccba9584355c..52e80183e050 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -135,15 +135,15 @@ struct kvm_mmu { #define KVM_NR_MEM_OBJS 40 +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ struct kvm_mmu_memory_cache { int nobjs; void *objects[KVM_NR_MEM_OBJS]; }; -/* - * We don't want allocation failures within the mmu code, so we preallocate - * enough memory for a single page fault in a cache. - */ struct kvm_guest_debug { int enabled; unsigned long bp[4]; -- cgit v1.2.3 From 2065b3727ecdb64450597d70f7e13af00b85dbd8 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 20 Nov 2007 11:30:04 +0200 Subject: KVM: MMU: Fix potential memory leak with smp real-mode When we map a page, we check whether some other vcpu mapped it for us and if so, bail out. But we should decrease the refcount on the page as we do so. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 87d8e70fe502..8add4d5c6840 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -907,8 +907,10 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) pte = table[index]; was_rmapped = is_rmap_pte(pte); - if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) + if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) { + kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); return 0; + } mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | -- cgit v1.2.3 From b4231d61807cac8d9d257eb6979c1685fa9a171f Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 20 Nov 2007 11:49:33 +0200 Subject: KVM: MMU: Selectively set PageDirty when releasing guest memory Improve dirty bit setting for pages that kvm release, until now every page that we released we marked dirty, from now only pages that have potential to get dirty we mark dirty. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 ++- drivers/kvm/kvm_main.c | 12 +++++++++--- drivers/kvm/mmu.c | 23 +++++++++++++++-------- drivers/kvm/paging_tmpl.h | 12 ++++++------ drivers/kvm/x86.c | 2 +- 5 files changed, 33 insertions(+), 19 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 52e80183e050..c2acd74389fa 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -393,7 +393,8 @@ int __kvm_set_memory_region(struct kvm *kvm, int user_alloc); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); -void kvm_release_page(struct page *page); +void kvm_release_page_clean(struct page *page); +void kvm_release_page_dirty(struct page *page); int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, int len); int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 4e1bd9488470..729573b844e5 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -543,13 +543,19 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) EXPORT_SYMBOL_GPL(gfn_to_page); -void kvm_release_page(struct page *page) +void kvm_release_page_clean(struct page *page) +{ + put_page(page); +} +EXPORT_SYMBOL_GPL(kvm_release_page_clean); + +void kvm_release_page_dirty(struct page *page) { if (!PageReserved(page)) SetPageDirty(page); put_page(page); } -EXPORT_SYMBOL_GPL(kvm_release_page); +EXPORT_SYMBOL_GPL(kvm_release_page_dirty); static int next_segment(unsigned long len, int offset) { @@ -1055,7 +1061,7 @@ static struct page *kvm_vm_nopage(struct vm_area_struct *vma, /* current->mm->mmap_sem is already held so call lockless version */ page = __gfn_to_page(kvm, pgoff); if (is_error_page(page)) { - kvm_release_page(page); + kvm_release_page_clean(page); return NOPAGE_SIGBUS; } if (type != NULL) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 8add4d5c6840..4624f3789b9a 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -420,14 +420,18 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) struct kvm_rmap_desc *desc; struct kvm_rmap_desc *prev_desc; struct kvm_mmu_page *page; + struct page *release_page; unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; page = page_header(__pa(spte)); - kvm_release_page(pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> - PAGE_SHIFT)); + release_page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + if (is_writeble_pte(*spte)) + kvm_release_page_dirty(release_page); + else + kvm_release_page_clean(release_page); rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]); if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); @@ -893,7 +897,9 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) { int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->mmu.root_hpa; + struct page *page; + page = pfn_to_page(p >> PAGE_SHIFT); for (; ; level--) { u32 index = PT64_INDEX(v, level); u64 *table; @@ -908,7 +914,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) pte = table[index]; was_rmapped = is_rmap_pte(pte); if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) { - kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); + kvm_release_page_clean(page); return 0; } mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); @@ -918,7 +924,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) if (!was_rmapped) rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); else - kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); + kvm_release_page_clean(page); + return 0; } @@ -933,7 +940,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) 1, 3, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); - kvm_release_page(pfn_to_page(p >> PAGE_SHIFT)); + kvm_release_page_clean(page); return -ENOMEM; } @@ -1049,8 +1056,8 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK); if (is_error_hpa(paddr)) { - kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); return 1; } @@ -1580,7 +1587,7 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, " valid guest gva %lx\n", audit_msg, va); page = pfn_to_page((gpa & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - kvm_release_page(page); + kvm_release_page_clean(page); } } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 77a2b22492bf..bf15d127a48f 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -212,8 +212,8 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (is_error_hpa(paddr)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); - kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); return; } @@ -259,12 +259,12 @@ unshadowed: page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - kvm_release_page(page); + kvm_release_page_clean(page); } } else - kvm_release_page(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) + >> PAGE_SHIFT)); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } @@ -503,7 +503,7 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, else sp->spt[i] = shadow_notrap_nonpresent_pte; kunmap_atomic(gpt, KM_USER0); - kvm_release_page(page); + kvm_release_page_clean(page); } #undef pt_element_t diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5a1b72fbaeaa..6212984a2e6c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1472,7 +1472,7 @@ static void free_pio_guest_pages(struct kvm_vcpu *vcpu) for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) if (vcpu->pio.guest_pages[i]) { - kvm_release_page(vcpu->pio.guest_pages[i]); + kvm_release_page_dirty(vcpu->pio.guest_pages[i]); vcpu->pio.guest_pages[i] = NULL; } } -- cgit v1.2.3 From 93a0039c8d93074d5f92dfb69f6a7d453905d002 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 12:49:31 +0200 Subject: KVM: x86 emulator: retire ->write_std() Theoretically used to acccess memory known to be ordinary RAM, it was never implemented. It is questionable whether it is possible to implement it correctly. Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 10 ---------- drivers/kvm/x86_emulate.h | 11 ----------- 2 files changed, 21 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 6212984a2e6c..5a54e328088d 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1162,15 +1162,6 @@ int emulator_read_std(unsigned long addr, } EXPORT_SYMBOL_GPL(emulator_read_std); -static int emulator_write_std(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - pr_unimpl(vcpu, "emulator_write_std: addr %lx n %d\n", addr, bytes); - return X86EMUL_UNHANDLEABLE; -} - static int emulator_read_emulated(unsigned long addr, void *val, unsigned int bytes, @@ -1367,7 +1358,6 @@ EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); struct x86_emulate_ops emulate_ops = { .read_std = emulator_read_std, - .write_std = emulator_write_std, .read_emulated = emulator_read_emulated, .write_emulated = emulator_write_emulated, .cmpxchg_emulated = emulator_cmpxchg_emulated, diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index e34868b42c33..a62bf14bbf89 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -62,17 +62,6 @@ struct x86_emulate_ops { int (*read_std)(unsigned long addr, void *val, unsigned int bytes, struct kvm_vcpu *vcpu); - /* - * write_std: Write bytes of standard (non-emulated/special) memory. - * Used for stack operations, and others. - * @addr: [IN ] Linear address to which to write. - * @val: [IN ] Value to write to memory (low-order bytes used as - * required). - * @bytes: [IN ] Number of bytes to write to memory. - */ - int (*write_std)(unsigned long addr, const void *val, - unsigned int bytes, struct kvm_vcpu *vcpu); - /* * read_emulated: Read bytes from emulated/special memory area. * @addr: [IN ] Linear address from which to read. -- cgit v1.2.3 From 6226686954c4cce3d63ffe1777e60360fcbf0b83 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 13:15:52 +0200 Subject: KVM: x86 emulator: prefetch up to 15 bytes of the instruction executed Instead of fetching one byte at a time, prefetch 15 bytes (or until the next page boundary) to avoid guest page table walks. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 38 ++++++++++++++++++++++++++++++++++++-- drivers/kvm/x86_emulate.h | 7 +++++++ 2 files changed, 43 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 8e2162fc6f70..6e7f774d1751 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -414,8 +414,7 @@ static u16 twobyte_table[256] = { /* Fetch next part of the instruction being emulated. */ #define insn_fetch(_type, _size, _eip) \ ({ unsigned long _x; \ - rc = ops->read_std((unsigned long)(_eip) + ctxt->cs_base, &_x, \ - (_size), ctxt->vcpu); \ + rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \ if (rc != 0) \ goto done; \ (_eip) += (_size); \ @@ -446,6 +445,41 @@ static u16 twobyte_table[256] = { register_address_increment(c->eip, rel); \ } while (0) +static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long linear, u8 *dest) +{ + struct fetch_cache *fc = &ctxt->decode.fetch; + int rc; + int size; + + if (linear < fc->start || linear >= fc->end) { + size = min(15UL, PAGE_SIZE - offset_in_page(linear)); + rc = ops->read_std(linear, fc->data, size, ctxt->vcpu); + if (rc) + return rc; + fc->start = linear; + fc->end = linear + size; + } + *dest = fc->data[linear - fc->start]; + return 0; +} + +static int do_insn_fetch(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long eip, void *dest, unsigned size) +{ + int rc = 0; + + eip += ctxt->cs_base; + while (size--) { + rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++); + if (rc) + return rc; + } + return 0; +} + /* * Given the 'reg' portion of a ModRM byte, and a register block, return a * pointer into the block that addresses the relevant register. diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index a62bf14bbf89..4603b2bf3488 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -108,6 +108,12 @@ struct operand { unsigned long val, orig_val, *ptr; }; +struct fetch_cache { + u8 data[15]; + unsigned long start; + unsigned long end; +}; + struct decode_cache { u8 twobyte; u8 b; @@ -130,6 +136,7 @@ struct decode_cache { u8 use_modrm_ea; unsigned long modrm_ea; unsigned long modrm_val; + struct fetch_cache fetch; }; struct x86_emulate_ctxt { -- cgit v1.2.3 From 3ad82a7e874c5d6c4045090cc01d7794dd9eb21c Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Tue, 20 Nov 2007 13:11:38 +0800 Subject: KVM: Recalculate mmu pages needed for every memory region change Instead of incrementally changing the mmu cache size for every memory slot operation, recalculate it from scratch. This is simpler and safer. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 21 ++++----------------- drivers/kvm/mmu.c | 19 +++++++++++++++++++ drivers/kvm/x86.h | 1 + 3 files changed, 24 insertions(+), 17 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 729573b844e5..93ecafbfb1b6 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -333,26 +333,13 @@ int __kvm_set_memory_region(struct kvm *kvm, if (mem->slot >= kvm->nmemslots) kvm->nmemslots = mem->slot + 1; + *memslot = new; + if (!kvm->n_requested_mmu_pages) { - unsigned int n_pages; - - if (npages) { - n_pages = npages * KVM_PERMILLE_MMU_PAGES / 1000; - kvm_mmu_change_mmu_pages(kvm, kvm->n_alloc_mmu_pages + - n_pages); - } else { - unsigned int nr_mmu_pages; - - n_pages = old.npages * KVM_PERMILLE_MMU_PAGES / 1000; - nr_mmu_pages = kvm->n_alloc_mmu_pages - n_pages; - nr_mmu_pages = max(nr_mmu_pages, - (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); - kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); - } + unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); + kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); } - *memslot = new; - kvm_mmu_slot_remove_write_access(kvm, mem->slot); kvm_flush_remote_tlbs(kvm); diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 4624f3789b9a..101cd5377a89 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1535,6 +1535,25 @@ nomem: return -ENOMEM; } +/* + * Caculate mmu pages needed for kvm. + */ +unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm) +{ + int i; + unsigned int nr_mmu_pages; + unsigned int nr_pages = 0; + + for (i = 0; i < kvm->nmemslots; i++) + nr_pages += kvm->memslots[i].npages; + + nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; + nr_mmu_pages = max(nr_mmu_pages, + (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); + + return nr_mmu_pages; +} + #ifdef AUDIT static const char *audit_msg; diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 90b791bf6287..71f2477d03fd 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -236,6 +236,7 @@ void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); void kvm_mmu_zap_all(struct kvm *kvm); +unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); enum emulation_result { -- cgit v1.2.3 From 0de10343b3ca7aa34dd606145748f73ed19f627e Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Tue, 20 Nov 2007 16:25:04 +0800 Subject: KVM: Portability: Split kvm_set_memory_region() to have an arch callout Moving !user_alloc case to kvm_arch to avoid unnecessary code logic in non-x86 platform. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++++ drivers/kvm/kvm_main.c | 38 +++++-------------------------------- drivers/kvm/x86.c | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 33 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index c2acd74389fa..49094a221f6a 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -391,6 +391,10 @@ int kvm_set_memory_region(struct kvm *kvm, int __kvm_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region *mem, int user_alloc); +int kvm_arch_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot old, + int user_alloc); gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); void kvm_release_page_clean(struct page *page); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 93ecafbfb1b6..9dd6ad3c6c7b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -291,33 +291,7 @@ int __kvm_set_memory_region(struct kvm *kvm, memset(new.rmap, 0, npages * sizeof(*new.rmap)); new.user_alloc = user_alloc; - if (user_alloc) - new.userspace_addr = mem->userspace_addr; - else { - down_write(¤t->mm->mmap_sem); - new.userspace_addr = do_mmap(NULL, 0, - npages * PAGE_SIZE, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - 0); - up_write(¤t->mm->mmap_sem); - - if (IS_ERR((void *)new.userspace_addr)) - goto out_free; - } - } else { - if (!old.user_alloc && old.rmap) { - int ret; - - down_write(¤t->mm->mmap_sem); - ret = do_munmap(current->mm, old.userspace_addr, - old.npages * PAGE_SIZE); - up_write(¤t->mm->mmap_sem); - if (ret < 0) - printk(KERN_WARNING - "kvm_vm_ioctl_set_memory_region: " - "failed to munmap memory\n"); - } + new.userspace_addr = mem->userspace_addr; } /* Allocate page dirty bitmap if needed */ @@ -335,14 +309,12 @@ int __kvm_set_memory_region(struct kvm *kvm, *memslot = new; - if (!kvm->n_requested_mmu_pages) { - unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); - kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc); + if (r) { + *memslot = old; + goto out_free; } - kvm_mmu_slot_remove_write_access(kvm, mem->slot); - kvm_flush_remote_tlbs(kvm); - kvm_free_physmem_slot(&old, &new); return 0; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5a54e328088d..6abb2ed1a908 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -2637,3 +2638,53 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_free_physmem(kvm); kfree(kvm); } + +int kvm_arch_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot old, + int user_alloc) +{ + int npages = mem->memory_size >> PAGE_SHIFT; + struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot]; + + /*To keep backward compatibility with older userspace, + *x86 needs to hanlde !user_alloc case. + */ + if (!user_alloc) { + if (npages && !old.rmap) { + down_write(¤t->mm->mmap_sem); + memslot->userspace_addr = do_mmap(NULL, 0, + npages * PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, + 0); + up_write(¤t->mm->mmap_sem); + + if (IS_ERR((void *)memslot->userspace_addr)) + return PTR_ERR((void *)memslot->userspace_addr); + } else { + if (!old.user_alloc && old.rmap) { + int ret; + + down_write(¤t->mm->mmap_sem); + ret = do_munmap(current->mm, old.userspace_addr, + old.npages * PAGE_SIZE); + up_write(¤t->mm->mmap_sem); + if (ret < 0) + printk(KERN_WARNING + "kvm_vm_ioctl_set_memory_region: " + "failed to munmap memory\n"); + } + } + } + + if (!kvm->n_requested_mmu_pages) { + unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); + kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + } + + kvm_mmu_slot_remove_write_access(kvm, mem->slot); + kvm_flush_remote_tlbs(kvm); + + return 0; +} -- cgit v1.2.3 From 26e5215fdc6cf7b5a8b1269134095abbb7338b3c Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 15:30:24 +0200 Subject: KVM: Split vcpu creation to avoid vcpu_load() before preemption setup Split kvm_arch_vcpu_create() into kvm_arch_vcpu_create() and kvm_arch_vcpu_setup(), enabling preemption notification between the two. This mean that we can now do vcpu_load() within kvm_arch_vcpu_setup(). Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 4 ++++ drivers/kvm/x86.c | 16 +++++++--------- 3 files changed, 12 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 49094a221f6a..b65f5dee4b1b 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -466,6 +466,7 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); +int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 9dd6ad3c6c7b..7939b5c9a4b4 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -769,6 +769,10 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); + r = kvm_arch_vcpu_setup(vcpu); + if (r) + goto vcpu_destroy; + mutex_lock(&kvm->lock); if (kvm->vcpus[n]) { r = -EEXIST; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 6abb2ed1a908..b482b6a8a828 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2478,13 +2478,12 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id) { - int r; - struct kvm_vcpu *vcpu = kvm_x86_ops->vcpu_create(kvm, id); + return kvm_x86_ops->vcpu_create(kvm, id); +} - if (IS_ERR(vcpu)) { - r = -ENOMEM; - goto fail; - } +int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) +{ + int r; /* We do fxsave: this must be aligned. */ BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); @@ -2497,11 +2496,10 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, if (r < 0) goto free_vcpu; - return vcpu; + return 0; free_vcpu: kvm_x86_ops->vcpu_free(vcpu); -fail: - return ERR_PTR(r); + return r; } void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From e5a4c8cad952d6e060991841f3632578ab3ae133 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 21:39:54 +0200 Subject: KVM: MMU: Implement guest page fault bypass for nonpae I spent an hour worrying why I see so many guest page faults on FC6 i386. Turns out bypass wasn't implemented for nonpae. Implement it so it doesn't happen again. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index bf15d127a48f..92b93134a2a6 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -486,19 +486,22 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { - int i; + int i, offset = 0; pt_element_t *gpt; struct page *page; - if (sp->role.metaphysical || PTTYPE == 32) { + if (sp->role.metaphysical + || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) { nonpaging_prefetch_page(vcpu, sp); return; } + if (PTTYPE == 32) + offset = sp->role.quadrant << PT64_LEVEL_BITS; page = gfn_to_page(vcpu->kvm, sp->gfn); gpt = kmap_atomic(page, KM_USER0); for (i = 0; i < PT64_ENT_PER_PAGE; ++i) - if (is_present_pte(gpt[i])) + if (is_present_pte(gpt[offset + i])) sp->spt[i] = shadow_trap_nonpresent_pte; else sp->spt[i] = shadow_notrap_nonpresent_pte; -- cgit v1.2.3 From 0f74a24c59b814c1c8085251cbea48d339f0c7c6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 20 Nov 2007 23:01:14 +0200 Subject: KVM: Add statistic for remote tlb flushes Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 3 +++ drivers/kvm/x86.c | 1 + 3 files changed, 5 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index b65f5dee4b1b..048849d97b35 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -300,6 +300,7 @@ struct kvm_vm_stat { u32 mmu_pde_zapped; u32 mmu_flooded; u32 mmu_recycled; + u32 remote_tlb_flush; }; struct kvm { diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 7939b5c9a4b4..aec6b67cfebb 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -115,6 +115,9 @@ void kvm_flush_remote_tlbs(struct kvm *kvm) if (cpu != -1 && cpu != raw_smp_processor_id()) cpu_set(cpu, cpus); } + if (cpus_empty(cpus)) + return; + ++kvm->stat.remote_tlb_flush; smp_call_function_mask(cpus, ack_flush, NULL, 1); } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index b482b6a8a828..ac09f381f47f 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -73,6 +73,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, { "mmu_flooded", VM_STAT(mmu_flooded) }, { "mmu_recycled", VM_STAT(mmu_recycled) }, + { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, { NULL } }; -- cgit v1.2.3 From 79539cec0c3c38d35a1e3e5310d2c562ae6e82b8 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 02:06:21 +0200 Subject: KVM: MMU: Avoid unnecessary remote tlb flushes when guest updates a pte If all we're doing is increasing permissions on a pte (typical for demand paging), then there's not need to flush remote tlbs. Worst case they'll get a spurious page fault. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 101cd5377a89..281dd5f9310c 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -134,6 +134,8 @@ static int dbg = 1; #define PT32_DIR_BASE_ADDR_MASK \ (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1)) +#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \ + | PT64_NX_MASK) #define PFERR_PRESENT_MASK (1U << 0) #define PFERR_WRITE_MASK (1U << 1) @@ -1227,7 +1229,6 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, } } set_shadow_pte(spte, shadow_trap_nonpresent_pte); - kvm_flush_remote_tlbs(vcpu->kvm); } static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, @@ -1250,6 +1251,27 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, offset_in_pte); } +static bool need_remote_flush(u64 old, u64 new) +{ + if (!is_shadow_present_pte(old)) + return false; + if (!is_shadow_present_pte(new)) + return true; + if ((old ^ new) & PT64_BASE_ADDR_MASK) + return true; + old ^= PT64_NX_MASK; + new ^= PT64_NX_MASK; + return (old & ~new & PT64_PERM_MASK) != 0; +} + +static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new) +{ + if (need_remote_flush(old, new)) + kvm_flush_remote_tlbs(vcpu->kvm); + else + kvm_mmu_flush_tlb(vcpu); +} + static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) { u64 *spte = vcpu->last_pte_updated; @@ -1265,6 +1287,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, struct hlist_node *node, *n; struct hlist_head *bucket; unsigned index; + u64 entry; u64 *spte; unsigned offset = offset_in_page(gpa); unsigned pte_size; @@ -1335,9 +1358,11 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } spte = &page->spt[page_offset / sizeof(*spte)]; while (npte--) { + entry = *spte; mmu_pte_write_zap_pte(vcpu, page, spte); mmu_pte_write_new_pte(vcpu, page, spte, new, bytes, page_offset & (pte_size - 1)); + mmu_pte_write_flush_tlb(vcpu, entry, *spte); ++spte; } } -- cgit v1.2.3 From d835dfecd00fd770288dcd9a46c0e0966d526fdf Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 02:57:59 +0200 Subject: KVM: Don't bother the mmu if cr3 load doesn't change cr3 If the guest requests just a tlb flush, don't take the vm lock and drop the mmu context pointlessly. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 2 +- drivers/kvm/x86.c | 25 +++++++++++++++++++++++++ drivers/kvm/x86.h | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 281dd5f9310c..346aa65a08d5 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1086,7 +1086,7 @@ static int nonpaging_init_context(struct kvm_vcpu *vcpu) return 0; } -static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) +void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) { ++vcpu->stat.tlb_flush; kvm_x86_ops->tlb_flush(vcpu); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index ac09f381f47f..15e1203faef0 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -166,6 +166,26 @@ out: return ret; } +static bool pdptrs_changed(struct kvm_vcpu *vcpu) +{ + u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; + bool changed = true; + int r; + + if (is_long_mode(vcpu) || !is_pae(vcpu)) + return false; + + mutex_lock(&vcpu->kvm->lock); + r = kvm_read_guest(vcpu->kvm, vcpu->cr3 & ~31u, pdpte, sizeof(pdpte)); + if (r < 0) + goto out; + changed = memcmp(pdpte, vcpu->pdptrs, sizeof(pdpte)) != 0; +out: + mutex_unlock(&vcpu->kvm->lock); + + return changed; +} + void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { if (cr0 & CR0_RESERVED_BITS) { @@ -271,6 +291,11 @@ EXPORT_SYMBOL_GPL(set_cr4); void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { + if (cr3 == vcpu->cr3 && !pdptrs_changed(vcpu)) { + kvm_mmu_flush_tlb(vcpu); + return; + } + if (is_long_mode(vcpu)) { if (cr3 & CR3_L_MODE_RESERVED_BITS) { printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 71f2477d03fd..b1528c9f566f 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -299,6 +299,7 @@ int emulator_write_emulated(unsigned long addr, unsigned long segment_base(u16 selector); +void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu); void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes); int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); -- cgit v1.2.3 From b238f7bc2dd5e0e8ecbd28863d6d03e490c652ce Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Tue, 20 Nov 2007 12:02:12 +0200 Subject: KVM: MMU: Code cleanup Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 92b93134a2a6..6e013015f0a8 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -187,6 +187,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, int dirty = gpte & PT_DIRTY_MASK; u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); + struct page *page; pgprintk("%s: spte %llx gpte %llx access %llx write_fault %d" " user_fault %d gfn %lx\n", @@ -205,6 +206,12 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, paddr = gpa_to_hpa(vcpu->kvm, gaddr & PT64_BASE_ADDR_MASK); + /* + * the reason paddr get mask even that it isnt pte is beacuse the + * HPA_ERR_MASK bit might be used to signal error + */ + page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + spte |= PT_PRESENT_MASK; if (access_bits & PT_USER_MASK) spte |= PT_USER_MASK; @@ -212,8 +219,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (is_error_hpa(paddr)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); - kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(page); return; } @@ -254,17 +260,11 @@ unshadowed: if (!was_rmapped) { rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - if (!is_rmap_pte(*shadow_pte)) { - struct page *page; - - page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); + if (!is_rmap_pte(*shadow_pte)) kvm_release_page_clean(page); - } } else - kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + kvm_release_page_clean(page); if (!ptwrite || !*ptwrite) vcpu->last_pte_updated = shadow_pte; } -- cgit v1.2.3 From 5fb07ddb183eb4728f240e09c3305d271ce61d48 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 12:35:07 +0200 Subject: KVM: MMU: Introduce and use gpte_to_gfn() Instead of repretitively open-coding this. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 6e013015f0a8..6f79ae87d0cf 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -52,6 +52,9 @@ #error Invalid PTTYPE value #endif +#define gpte_to_gfn FNAME(gpte_to_gfn) +#define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde) + /* * The guest_walker structure emulates the behavior of the hardware page * table walker. @@ -65,6 +68,16 @@ struct guest_walker { u32 error_code; }; +static gfn_t gpte_to_gfn(pt_element_t gpte) +{ + return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; +} + +static gfn_t gpte_to_gfn_pde(pt_element_t gpte) +{ + return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; +} + /* * Fetch a guest pte for a guest virtual address */ @@ -96,7 +109,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, for (;;) { index = PT_INDEX(addr, walker->level); - table_gfn = (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT; + table_gfn = gpte_to_gfn(pte); pte_gpa = table_gfn << PAGE_SHIFT; pte_gpa += index * sizeof(pt_element_t); walker->table_gfn[walker->level - 1] = table_gfn; @@ -127,15 +140,14 @@ static int FNAME(walk_addr)(struct guest_walker *walker, } if (walker->level == PT_PAGE_TABLE_LEVEL) { - walker->gfn = (pte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; + walker->gfn = gpte_to_gfn(pte); break; } if (walker->level == PT_DIRECTORY_LEVEL && (pte & PT_PAGE_SIZE_MASK) && (PTTYPE == 64 || is_pse(vcpu))) { - walker->gfn = (pte & PT_DIR_BASE_ADDR_MASK) - >> PAGE_SHIFT; + walker->gfn = gpte_to_gfn_pde(pte); walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); break; } @@ -296,8 +308,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); FNAME(set_pte)(vcpu, gpte, spte, PT_USER_MASK | PT_WRITABLE_MASK, 0, - 0, NULL, NULL, - (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT); + 0, NULL, NULL, gpte_to_gfn(gpte)); } static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, @@ -370,8 +381,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, hugepage_access >>= PT_WRITABLE_SHIFT; if (walker->pte & PT64_NX_MASK) hugepage_access |= (1 << 2); - table_gfn = (walker->pte & PT_BASE_ADDR_MASK) - >> PAGE_SHIFT; + table_gfn = gpte_to_gfn(walker->pte); } else { metaphysical = 0; table_gfn = walker->table_gfn[level - 2]; @@ -519,3 +529,5 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, #undef PT_DIR_BASE_ADDR_MASK #undef PT_LEVEL_BITS #undef PT_MAX_FULL_LEVELS +#undef gpte_to_gfn +#undef gpte_to_gfn_pde -- cgit v1.2.3 From da928521b7a095e85f6ac92a6c106bca7e17625f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 13:54:47 +0200 Subject: KVM: MMU: Move pse36 handling to the guest walker Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 7 +++++++ drivers/kvm/paging_tmpl.h | 5 ++--- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 346aa65a08d5..a9fed594ac58 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -218,6 +218,13 @@ static int is_rmap_pte(u64 pte) && pte != shadow_notrap_nonpresent_pte; } +static gfn_t pse36_gfn_delta(u32 gpte) +{ + int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT; + + return (gpte & PT32_DIR_PSE36_MASK) << shift; +} + static void set_shadow_pte(u64 *sptep, u64 spte) { #ifdef CONFIG_X86_64 diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 6f79ae87d0cf..dceb4b9137a4 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -149,6 +149,8 @@ static int FNAME(walk_addr)(struct guest_walker *walker, && (PTTYPE == 64 || is_pse(vcpu))) { walker->gfn = gpte_to_gfn_pde(pte); walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); + if (PTTYPE == 32 && is_cpuid_PSE36()) + walker->gfn += pse36_gfn_delta(pte); break; } @@ -320,9 +322,6 @@ static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, access_bits &= gpde; gaddr = (gpa_t)gfn << PAGE_SHIFT; - if (PTTYPE == 32 && is_cpuid_PSE36()) - gaddr |= (gpde & PT32_DIR_PSE36_MASK) << - (32 - PT32_DIR_PSE36_SHIFT); FNAME(set_pte_common)(vcpu, shadow_pte, gaddr, gpde, access_bits, user_fault, write_fault, ptwrite, walker, gfn); -- cgit v1.2.3 From 4e542370c7570bdb582ab8c751ab899ff9e4214a Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:08:40 +0200 Subject: KVM: MMU: Remove extra gaddr parameter from set_pte_common() Similar information is available in the gfn parameter, so use that. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 1 + drivers/kvm/paging_tmpl.h | 29 +++++++++-------------------- 2 files changed, 10 insertions(+), 20 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index a9fed594ac58..c3362ba33b0a 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -29,6 +29,7 @@ #include #include +#include #undef MMU_DEBUG diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index dceb4b9137a4..cc373ed1f049 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -188,7 +188,6 @@ err: static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, u64 *shadow_pte, - gpa_t gaddr, pt_element_t gpte, u64 access_bits, int user_fault, @@ -197,7 +196,6 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, struct guest_walker *walker, gfn_t gfn) { - hpa_t paddr; int dirty = gpte & PT_DIRTY_MASK; u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); @@ -218,26 +216,20 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, if (!dirty) access_bits &= ~PT_WRITABLE_MASK; - paddr = gpa_to_hpa(vcpu->kvm, gaddr & PT64_BASE_ADDR_MASK); - - /* - * the reason paddr get mask even that it isnt pte is beacuse the - * HPA_ERR_MASK bit might be used to signal error - */ - page = pfn_to_page((paddr & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + page = gfn_to_page(vcpu->kvm, gfn); spte |= PT_PRESENT_MASK; if (access_bits & PT_USER_MASK) spte |= PT_USER_MASK; - if (is_error_hpa(paddr)) { + if (is_error_page(page)) { set_shadow_pte(shadow_pte, shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); kvm_release_page_clean(page); return; } - spte |= paddr; + spte |= page_to_phys(page); if ((access_bits & PT_WRITABLE_MASK) || (write_fault && !is_write_protection(vcpu) && !user_fault)) { @@ -266,14 +258,14 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, unshadowed: if (access_bits & PT_WRITABLE_MASK) - mark_page_dirty(vcpu->kvm, gaddr >> PAGE_SHIFT); + mark_page_dirty(vcpu->kvm, gfn); pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); - page_header_update_slot(vcpu->kvm, shadow_pte, gaddr); + page_header_update_slot(vcpu->kvm, shadow_pte, + (gpa_t)gfn << PAGE_SHIFT); if (!was_rmapped) { - rmap_add(vcpu, shadow_pte, (gaddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); + rmap_add(vcpu, shadow_pte, gfn); if (!is_rmap_pte(*shadow_pte)) kvm_release_page_clean(page); } @@ -289,7 +281,7 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, struct guest_walker *walker, gfn_t gfn) { access_bits &= gpte; - FNAME(set_pte_common)(vcpu, shadow_pte, gpte & PT_BASE_ADDR_MASK, + FNAME(set_pte_common)(vcpu, shadow_pte, gpte, access_bits, user_fault, write_fault, ptwrite, walker, gfn); } @@ -318,11 +310,8 @@ static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, int user_fault, int write_fault, int *ptwrite, struct guest_walker *walker, gfn_t gfn) { - gpa_t gaddr; - access_bits &= gpde; - gaddr = (gpa_t)gfn << PAGE_SHIFT; - FNAME(set_pte_common)(vcpu, shadow_pte, gaddr, + FNAME(set_pte_common)(vcpu, shadow_pte, gpde, access_bits, user_fault, write_fault, ptwrite, walker, gfn); } -- cgit v1.2.3 From 050e64992f90abee4e1e77b57fe7abf1ba22d330 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:11:49 +0200 Subject: KVM: MMU: Remove set_pde() It is now identical to set_pte(). Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index cc373ed1f049..062f4f574649 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -305,17 +305,6 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, 0, NULL, NULL, gpte_to_gfn(gpte)); } -static void FNAME(set_pde)(struct kvm_vcpu *vcpu, pt_element_t gpde, - u64 *shadow_pte, u64 access_bits, - int user_fault, int write_fault, int *ptwrite, - struct guest_walker *walker, gfn_t gfn) -{ - access_bits &= gpde; - FNAME(set_pte_common)(vcpu, shadow_pte, - gpde, access_bits, user_fault, write_fault, - ptwrite, walker, gfn); -} - /* * Fetch a shadow pte for a specific level in the paging hierarchy. */ @@ -384,16 +373,10 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, prev_shadow_ent = shadow_ent; } - if (walker->level == PT_DIRECTORY_LEVEL) { - FNAME(set_pde)(vcpu, walker->pte, shadow_ent, - walker->inherited_ar, user_fault, write_fault, - ptwrite, walker, walker->gfn); - } else { - ASSERT(walker->level == PT_PAGE_TABLE_LEVEL); - FNAME(set_pte)(vcpu, walker->pte, shadow_ent, - walker->inherited_ar, user_fault, write_fault, - ptwrite, walker, walker->gfn); - } + FNAME(set_pte)(vcpu, walker->pte, shadow_ent, + walker->inherited_ar, user_fault, write_fault, + ptwrite, walker, walker->gfn); + return shadow_ent; } -- cgit v1.2.3 From 230c9a8f2354dc51f55bf81147f408d8fc2f3dd1 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:16:30 +0200 Subject: KVM: MMU: Merge set_pte() and set_pte_common() Since set_pte() is now the only caller of set_pte_common(), merge the two functions. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 062f4f574649..54a6ee802dc2 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -186,15 +186,11 @@ err: return 0; } -static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, - u64 *shadow_pte, - pt_element_t gpte, - u64 access_bits, - int user_fault, - int write_fault, - int *ptwrite, - struct guest_walker *walker, - gfn_t gfn) +static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, + u64 *shadow_pte, u64 access_bits, + int user_fault, int write_fault, + int *ptwrite, struct guest_walker *walker, + gfn_t gfn) { int dirty = gpte & PT_DIRTY_MASK; u64 spte; @@ -206,6 +202,7 @@ static void FNAME(set_pte_common)(struct kvm_vcpu *vcpu, __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, write_fault, user_fault, gfn); + access_bits &= gpte; /* * We don't set the accessed bit, since we sometimes want to see * whether the guest actually used the pte (in order to detect @@ -275,17 +272,6 @@ unshadowed: vcpu->last_pte_updated = shadow_pte; } -static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, - u64 *shadow_pte, u64 access_bits, - int user_fault, int write_fault, int *ptwrite, - struct guest_walker *walker, gfn_t gfn) -{ - access_bits &= gpte; - FNAME(set_pte_common)(vcpu, shadow_pte, - gpte, access_bits, user_fault, write_fault, - ptwrite, walker, gfn); -} - static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, u64 *spte, const void *pte, int bytes, int offset_in_pte) -- cgit v1.2.3 From 38c335f1f57bd9edc3cd894e8f5f3fc133320ad3 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:20:22 +0200 Subject: KVM: MMU: Adjust page_header_update_slot() to accept a gfn instead of a gpa Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 7 ++++--- drivers/kvm/paging_tmpl.h | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index c3362ba33b0a..1dcffc49eff8 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -860,9 +860,9 @@ static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) } } -static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa) +static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) { - int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT)); + int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn)); struct kvm_mmu_page *page_head = page_header(__pa(pte)); __set_bit(slot, &page_head->slot_bitmap); @@ -928,7 +928,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) return 0; } mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); - page_header_update_slot(vcpu->kvm, table, v); + page_header_update_slot(vcpu->kvm, table, + v >> PAGE_SHIFT); table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK; if (!was_rmapped) diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 54a6ee802dc2..a3da98bfd6a6 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -259,8 +259,7 @@ unshadowed: pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); set_shadow_pte(shadow_pte, spte); - page_header_update_slot(vcpu->kvm, shadow_pte, - (gpa_t)gfn << PAGE_SHIFT); + page_header_update_slot(vcpu->kvm, shadow_pte, gfn); if (!was_rmapped) { rmap_add(vcpu, shadow_pte, gfn); if (!is_rmap_pte(*shadow_pte)) -- cgit v1.2.3 From 1755fbcc6666906cada43c1c839ca1fc6616d4a6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:44:45 +0200 Subject: KVM: MMU: Introduce gfn_to_gpa() Converting a frame number to an address is tricky since the data type changes size. Introduce a function to do it. This fixes an actual bug when accessing guest ptes. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++++ drivers/kvm/paging_tmpl.h | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 048849d97b35..eda82cded88e 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -499,6 +499,10 @@ static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) return slot - kvm->memslots; } +static inline gpa_t gfn_to_gpa(gfn_t gfn) +{ + return (gpa_t)gfn << PAGE_SHIFT; +} enum kvm_stat_kind { KVM_STAT_VM, diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index a3da98bfd6a6..b24bc7c86078 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -110,7 +110,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, index = PT_INDEX(addr, walker->level); table_gfn = gpte_to_gfn(pte); - pte_gpa = table_gfn << PAGE_SHIFT; + pte_gpa = gfn_to_gpa(table_gfn); pte_gpa += index * sizeof(pt_element_t); walker->table_gfn[walker->level - 1] = table_gfn; pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, @@ -442,7 +442,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) r = FNAME(walk_addr)(&walker, vcpu, vaddr, 0, 0, 0); if (r) { - gpa = (gpa_t)walker.gfn << PAGE_SHIFT; + gpa = gfn_to_gpa(walker.gfn); gpa |= vaddr & ~PAGE_MASK; } -- cgit v1.2.3 From 3f3e7124f66df2d36751b0c97eabe38a13752fff Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:54:16 +0200 Subject: KVM: MMU: Simplify nonpaging_map() Instead of passing an hpa, pass a regular struct page. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 1dcffc49eff8..1965185bbe23 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -903,13 +903,11 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } -static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) +static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, struct page *page) { int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->mmu.root_hpa; - struct page *page; - page = pfn_to_page(p >> PAGE_SHIFT); for (; ; level--) { u32 index = PT64_INDEX(v, level); u64 *table; @@ -930,8 +928,9 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p) mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); page_header_update_slot(vcpu->kvm, table, v >> PAGE_SHIFT); - table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK | - PT_USER_MASK; + table[index] = page_to_phys(page) + | PT_PRESENT_MASK | PT_WRITABLE_MASK + | PT_USER_MASK; if (!was_rmapped) rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); else @@ -1050,10 +1049,9 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) } static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) + u32 error_code) { - gpa_t addr = gva; - hpa_t paddr; + struct page *page; int r; r = mmu_topup_memory_caches(vcpu); @@ -1063,16 +1061,14 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, ASSERT(vcpu); ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); + page = gfn_to_page(vcpu->kvm, gva >> PAGE_SHIFT); - paddr = gpa_to_hpa(vcpu->kvm, addr & PT64_BASE_ADDR_MASK); - - if (is_error_hpa(paddr)) { - kvm_release_page_clean(pfn_to_page((paddr & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT)); + if (is_error_page(page)) { + kvm_release_page_clean(page); return 1; } - return nonpaging_map(vcpu, addr & PAGE_MASK, paddr); + return nonpaging_map(vcpu, gva & PAGE_MASK, page); } static void nonpaging_free(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 0d81f2966a61953c4cc7397342f5f907b3a6e257 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 14:57:44 +0200 Subject: KVM: MMU: Remove gva_to_hpa() No longer used. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 - drivers/kvm/mmu.c | 9 --------- 2 files changed, 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index eda82cded88e..31315bcf09c3 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -379,7 +379,6 @@ hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } -hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva); struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); extern struct page *bad_page; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 1965185bbe23..6aa0319ede4d 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -881,15 +881,6 @@ hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa) return hpa; } -hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva) -{ - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); - - if (gpa == UNMAPPED_GVA) - return UNMAPPED_GVA; - return gpa_to_hpa(vcpu->kvm, gpa); -} - struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); -- cgit v1.2.3 From 1d28f5f4a4984be4cd7200ed512c94517c13e392 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 15:01:44 +0200 Subject: KVM: Remove gpa_to_hpa() Converting last uses along the way. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 - drivers/kvm/mmu.c | 21 +++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 31315bcf09c3..1fd8158ced89 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -375,7 +375,6 @@ int kvm_init(void *opaque, unsigned int vcpu_size, struct module *module); void kvm_exit(void); -hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa); #define HPA_MSB ((sizeof(hpa_t) * 8) - 1) #define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 6aa0319ede4d..9b75b102b8d1 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -868,26 +868,13 @@ static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) __set_bit(slot, &page_head->slot_bitmap); } -hpa_t gpa_to_hpa(struct kvm *kvm, gpa_t gpa) -{ - struct page *page; - hpa_t hpa; - - ASSERT((gpa & HPA_ERR_MASK) == 0); - page = gfn_to_page(kvm, gpa >> PAGE_SHIFT); - hpa = ((hpa_t)page_to_pfn(page) << PAGE_SHIFT) | (gpa & (PAGE_SIZE-1)); - if (is_error_page(page)) - return hpa | HPA_ERR_MASK; - return hpa; -} - struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); if (gpa == UNMAPPED_GVA) return NULL; - return pfn_to_page(gpa_to_hpa(vcpu->kvm, gpa) >> PAGE_SHIFT); + return gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); } static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) @@ -1611,8 +1598,8 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, audit_mappings_page(vcpu, ent, va, level - 1); } else { gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); - hpa_t hpa = gpa_to_hpa(vcpu, gpa); - struct page *page; + struct page *page = gpa_to_page(vcpu, gpa); + hpa_t hpa = page_to_phys(page); if (is_shadow_present_pte(ent) && (ent & PT64_BASE_ADDR_MASK) != hpa) @@ -1625,8 +1612,6 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, && !is_error_hpa(hpa)) printk(KERN_ERR "audit: (%s) notrap shadow," " valid guest gva %lx\n", audit_msg, va); - page = pfn_to_page((gpa & PT64_BASE_ADDR_MASK) - >> PAGE_SHIFT); kvm_release_page_clean(page); } -- cgit v1.2.3 From 4db3531487bdde4027b701da7c1b8d094ae2cc91 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 15:28:32 +0200 Subject: KVM: MMU: Rename variables of type 'struct kvm_mmu_page *' These are traditionally named 'page', but even more traditionally, that name is reserved for variables that point to a 'struct page'. Rename them to 'sp' (for "shadow page"). Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 300 ++++++++++++++++++++++++++---------------------------- 1 file changed, 146 insertions(+), 154 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 9b75b102b8d1..86896da3e837 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -367,7 +367,7 @@ static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn) */ static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct kvm_rmap_desc *desc; unsigned long *rmapp; int i; @@ -375,8 +375,8 @@ static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) if (!is_rmap_pte(*spte)) return; gfn = unalias_gfn(vcpu->kvm, gfn); - page = page_header(__pa(spte)); - page->gfns[spte - page->spt] = gfn; + sp = page_header(__pa(spte)); + sp->gfns[spte - sp->spt] = gfn; rmapp = gfn_to_rmap(vcpu->kvm, gfn); if (!*rmapp) { rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte); @@ -429,20 +429,20 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) { struct kvm_rmap_desc *desc; struct kvm_rmap_desc *prev_desc; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct page *release_page; unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; - page = page_header(__pa(spte)); + sp = page_header(__pa(spte)); release_page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); if (is_writeble_pte(*spte)) kvm_release_page_dirty(release_page); else kvm_release_page_clean(release_page); - rmapp = gfn_to_rmap(kvm, page->gfns[spte - page->spt]); + rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt]); if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); BUG(); @@ -537,14 +537,13 @@ static int is_empty_shadow_page(u64 *spt) } #endif -static void kvm_mmu_free_page(struct kvm *kvm, - struct kvm_mmu_page *page_head) +static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp) { - ASSERT(is_empty_shadow_page(page_head->spt)); - list_del(&page_head->link); - __free_page(virt_to_page(page_head->spt)); - __free_page(virt_to_page(page_head->gfns)); - kfree(page_head); + ASSERT(is_empty_shadow_page(sp->spt)); + list_del(&sp->link); + __free_page(virt_to_page(sp->spt)); + __free_page(virt_to_page(sp->gfns)); + kfree(sp); ++kvm->n_free_mmu_pages; } @@ -556,27 +555,26 @@ static unsigned kvm_page_table_hashfn(gfn_t gfn) static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, u64 *parent_pte) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; if (!vcpu->kvm->n_free_mmu_pages) return NULL; - page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache, - sizeof *page); - page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); - page->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); - set_page_private(virt_to_page(page->spt), (unsigned long)page); - list_add(&page->link, &vcpu->kvm->active_mmu_pages); - ASSERT(is_empty_shadow_page(page->spt)); - page->slot_bitmap = 0; - page->multimapped = 0; - page->parent_pte = parent_pte; + sp = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache, sizeof *sp); + sp->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); + sp->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); + set_page_private(virt_to_page(sp->spt), (unsigned long)sp); + list_add(&sp->link, &vcpu->kvm->active_mmu_pages); + ASSERT(is_empty_shadow_page(sp->spt)); + sp->slot_bitmap = 0; + sp->multimapped = 0; + sp->parent_pte = parent_pte; --vcpu->kvm->n_free_mmu_pages; - return page; + return sp; } static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page, u64 *parent_pte) + struct kvm_mmu_page *sp, u64 *parent_pte) { struct kvm_pte_chain *pte_chain; struct hlist_node *node; @@ -584,20 +582,20 @@ static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, if (!parent_pte) return; - if (!page->multimapped) { - u64 *old = page->parent_pte; + if (!sp->multimapped) { + u64 *old = sp->parent_pte; if (!old) { - page->parent_pte = parent_pte; + sp->parent_pte = parent_pte; return; } - page->multimapped = 1; + sp->multimapped = 1; pte_chain = mmu_alloc_pte_chain(vcpu); - INIT_HLIST_HEAD(&page->parent_ptes); - hlist_add_head(&pte_chain->link, &page->parent_ptes); + INIT_HLIST_HEAD(&sp->parent_ptes); + hlist_add_head(&pte_chain->link, &sp->parent_ptes); pte_chain->parent_ptes[0] = old; } - hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) { + hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) { if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1]) continue; for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) @@ -608,23 +606,23 @@ static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, } pte_chain = mmu_alloc_pte_chain(vcpu); BUG_ON(!pte_chain); - hlist_add_head(&pte_chain->link, &page->parent_ptes); + hlist_add_head(&pte_chain->link, &sp->parent_ptes); pte_chain->parent_ptes[0] = parent_pte; } -static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page, +static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp, u64 *parent_pte) { struct kvm_pte_chain *pte_chain; struct hlist_node *node; int i; - if (!page->multimapped) { - BUG_ON(page->parent_pte != parent_pte); - page->parent_pte = NULL; + if (!sp->multimapped) { + BUG_ON(sp->parent_pte != parent_pte); + sp->parent_pte = NULL; return; } - hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) + hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) { if (!pte_chain->parent_ptes[i]) break; @@ -640,9 +638,9 @@ static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page, if (i == 0) { hlist_del(&pte_chain->link); mmu_free_pte_chain(pte_chain); - if (hlist_empty(&page->parent_ptes)) { - page->multimapped = 0; - page->parent_pte = NULL; + if (hlist_empty(&sp->parent_ptes)) { + sp->multimapped = 0; + sp->parent_pte = NULL; } } return; @@ -650,22 +648,21 @@ static void mmu_page_remove_parent_pte(struct kvm_mmu_page *page, BUG(); } -static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, - gfn_t gfn) +static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) { unsigned index; struct hlist_head *bucket; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct hlist_node *node; pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &kvm->mmu_page_hash[index]; - hlist_for_each_entry(page, node, bucket, hash_link) - if (page->gfn == gfn && !page->role.metaphysical) { + hlist_for_each_entry(sp, node, bucket, hash_link) + if (sp->gfn == gfn && !sp->role.metaphysical) { pgprintk("%s: found role %x\n", - __FUNCTION__, page->role.word); - return page; + __FUNCTION__, sp->role.word); + return sp; } return NULL; } @@ -682,7 +679,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, unsigned index; unsigned quadrant; struct hlist_head *bucket; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct hlist_node *node; role.word = 0; @@ -699,35 +696,35 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, gfn, role.word); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &vcpu->kvm->mmu_page_hash[index]; - hlist_for_each_entry(page, node, bucket, hash_link) - if (page->gfn == gfn && page->role.word == role.word) { - mmu_page_add_parent_pte(vcpu, page, parent_pte); + hlist_for_each_entry(sp, node, bucket, hash_link) + if (sp->gfn == gfn && sp->role.word == role.word) { + mmu_page_add_parent_pte(vcpu, sp, parent_pte); pgprintk("%s: found\n", __FUNCTION__); - return page; + return sp; } - page = kvm_mmu_alloc_page(vcpu, parent_pte); - if (!page) - return page; + sp = kvm_mmu_alloc_page(vcpu, parent_pte); + if (!sp) + return sp; pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word); - page->gfn = gfn; - page->role = role; - hlist_add_head(&page->hash_link, bucket); - vcpu->mmu.prefetch_page(vcpu, page); + sp->gfn = gfn; + sp->role = role; + hlist_add_head(&sp->hash_link, bucket); + vcpu->mmu.prefetch_page(vcpu, sp); if (!metaphysical) rmap_write_protect(vcpu->kvm, gfn); - return page; + return sp; } static void kvm_mmu_page_unlink_children(struct kvm *kvm, - struct kvm_mmu_page *page) + struct kvm_mmu_page *sp) { unsigned i; u64 *pt; u64 ent; - pt = page->spt; + pt = sp->spt; - if (page->role.level == PT_PAGE_TABLE_LEVEL) { + if (sp->role.level == PT_PAGE_TABLE_LEVEL) { for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { if (is_shadow_present_pte(pt[i])) rmap_remove(kvm, &pt[i]); @@ -749,10 +746,9 @@ static void kvm_mmu_page_unlink_children(struct kvm *kvm, kvm_flush_remote_tlbs(kvm); } -static void kvm_mmu_put_page(struct kvm_mmu_page *page, - u64 *parent_pte) +static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte) { - mmu_page_remove_parent_pte(page, parent_pte); + mmu_page_remove_parent_pte(sp, parent_pte); } static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) @@ -764,32 +760,31 @@ static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) kvm->vcpus[i]->last_pte_updated = NULL; } -static void kvm_mmu_zap_page(struct kvm *kvm, - struct kvm_mmu_page *page) +static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) { u64 *parent_pte; ++kvm->stat.mmu_shadow_zapped; - while (page->multimapped || page->parent_pte) { - if (!page->multimapped) - parent_pte = page->parent_pte; + while (sp->multimapped || sp->parent_pte) { + if (!sp->multimapped) + parent_pte = sp->parent_pte; else { struct kvm_pte_chain *chain; - chain = container_of(page->parent_ptes.first, + chain = container_of(sp->parent_ptes.first, struct kvm_pte_chain, link); parent_pte = chain->parent_ptes[0]; } BUG_ON(!parent_pte); - kvm_mmu_put_page(page, parent_pte); + kvm_mmu_put_page(sp, parent_pte); set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte); } - kvm_mmu_page_unlink_children(kvm, page); - if (!page->root_count) { - hlist_del(&page->hash_link); - kvm_mmu_free_page(kvm, page); + kvm_mmu_page_unlink_children(kvm, sp); + if (!sp->root_count) { + hlist_del(&sp->hash_link); + kvm_mmu_free_page(kvm, sp); } else - list_move(&page->link, &kvm->active_mmu_pages); + list_move(&sp->link, &kvm->active_mmu_pages); kvm_mmu_reset_last_pte_updated(kvm); } @@ -831,7 +826,7 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) { unsigned index; struct hlist_head *bucket; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct hlist_node *node, *n; int r; @@ -839,11 +834,11 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) r = 0; index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &kvm->mmu_page_hash[index]; - hlist_for_each_entry_safe(page, node, n, bucket, hash_link) - if (page->gfn == gfn && !page->role.metaphysical) { + hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) + if (sp->gfn == gfn && !sp->role.metaphysical) { pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, - page->role.word); - kvm_mmu_zap_page(kvm, page); + sp->role.word); + kvm_mmu_zap_page(kvm, sp); r = 1; } return r; @@ -851,21 +846,20 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; - while ((page = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { - pgprintk("%s: zap %lx %x\n", - __FUNCTION__, gfn, page->role.word); - kvm_mmu_zap_page(kvm, page); + while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { + pgprintk("%s: zap %lx %x\n", __FUNCTION__, gfn, sp->role.word); + kvm_mmu_zap_page(kvm, sp); } } static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) { int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn)); - struct kvm_mmu_page *page_head = page_header(__pa(pte)); + struct kvm_mmu_page *sp = page_header(__pa(pte)); - __set_bit(slot, &page_head->slot_bitmap); + __set_bit(slot, &sp->slot_bitmap); } struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) @@ -951,7 +945,7 @@ static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, static void mmu_free_roots(struct kvm_vcpu *vcpu) { int i; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; if (!VALID_PAGE(vcpu->mmu.root_hpa)) return; @@ -959,8 +953,8 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) { hpa_t root = vcpu->mmu.root_hpa; - page = page_header(root); - --page->root_count; + sp = page_header(root); + --sp->root_count; vcpu->mmu.root_hpa = INVALID_PAGE; return; } @@ -970,8 +964,8 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) if (root) { root &= PT64_BASE_ADDR_MASK; - page = page_header(root); - --page->root_count; + sp = page_header(root); + --sp->root_count; } vcpu->mmu.pae_root[i] = INVALID_PAGE; } @@ -982,7 +976,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) { int i; gfn_t root_gfn; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; root_gfn = vcpu->cr3 >> PAGE_SHIFT; @@ -991,10 +985,10 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) hpa_t root = vcpu->mmu.root_hpa; ASSERT(!VALID_PAGE(root)); - page = kvm_mmu_get_page(vcpu, root_gfn, 0, - PT64_ROOT_LEVEL, 0, 0, NULL); - root = __pa(page->spt); - ++page->root_count; + sp = kvm_mmu_get_page(vcpu, root_gfn, 0, + PT64_ROOT_LEVEL, 0, 0, NULL); + root = __pa(sp->spt); + ++sp->root_count; vcpu->mmu.root_hpa = root; return; } @@ -1011,11 +1005,11 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT; } else if (vcpu->mmu.root_level == 0) root_gfn = 0; - page = kvm_mmu_get_page(vcpu, root_gfn, i << 30, - PT32_ROOT_LEVEL, !is_paging(vcpu), - 0, NULL); - root = __pa(page->spt); - ++page->root_count; + sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, + PT32_ROOT_LEVEL, !is_paging(vcpu), + 0, NULL); + root = __pa(sp->spt); + ++sp->root_count; vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; } vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root); @@ -1196,7 +1190,7 @@ void kvm_mmu_unload(struct kvm_vcpu *vcpu) } static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page, + struct kvm_mmu_page *sp, u64 *spte) { u64 pte; @@ -1204,7 +1198,7 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, pte = *spte; if (is_shadow_present_pte(pte)) { - if (page->role.level == PT_PAGE_TABLE_LEVEL) + if (sp->role.level == PT_PAGE_TABLE_LEVEL) rmap_remove(vcpu->kvm, spte); else { child = page_header(pte & PT64_BASE_ADDR_MASK); @@ -1215,23 +1209,21 @@ static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, } static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page, + struct kvm_mmu_page *sp, u64 *spte, const void *new, int bytes, int offset_in_pte) { - if (page->role.level != PT_PAGE_TABLE_LEVEL) { + if (sp->role.level != PT_PAGE_TABLE_LEVEL) { ++vcpu->kvm->stat.mmu_pde_zapped; return; } ++vcpu->kvm->stat.mmu_pte_updated; - if (page->role.glevels == PT32_ROOT_LEVEL) - paging32_update_pte(vcpu, page, spte, new, bytes, - offset_in_pte); + if (sp->role.glevels == PT32_ROOT_LEVEL) + paging32_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte); else - paging64_update_pte(vcpu, page, spte, new, bytes, - offset_in_pte); + paging64_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte); } static bool need_remote_flush(u64 old, u64 new) @@ -1266,7 +1258,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new, int bytes) { gfn_t gfn = gpa >> PAGE_SHIFT; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct hlist_node *node, *n; struct hlist_head *bucket; unsigned index; @@ -1296,10 +1288,10 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &vcpu->kvm->mmu_page_hash[index]; - hlist_for_each_entry_safe(page, node, n, bucket, hash_link) { - if (page->gfn != gfn || page->role.metaphysical) + hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) { + if (sp->gfn != gfn || sp->role.metaphysical) continue; - pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; + pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); misaligned |= bytes < 4; if (misaligned || flooded) { @@ -1314,15 +1306,15 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, * page. */ pgprintk("misaligned: gpa %llx bytes %d role %x\n", - gpa, bytes, page->role.word); - kvm_mmu_zap_page(vcpu->kvm, page); + gpa, bytes, sp->role.word); + kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_flooded; continue; } page_offset = offset; - level = page->role.level; + level = sp->role.level; npte = 1; - if (page->role.glevels == PT32_ROOT_LEVEL) { + if (sp->role.glevels == PT32_ROOT_LEVEL) { page_offset <<= 1; /* 32->64 */ /* * A 32-bit pde maps 4MB while the shadow pdes map @@ -1336,14 +1328,14 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, } quadrant = page_offset >> PAGE_SHIFT; page_offset &= ~PAGE_MASK; - if (quadrant != page->role.quadrant) + if (quadrant != sp->role.quadrant) continue; } - spte = &page->spt[page_offset / sizeof(*spte)]; + spte = &sp->spt[page_offset / sizeof(*spte)]; while (npte--) { entry = *spte; - mmu_pte_write_zap_pte(vcpu, page, spte); - mmu_pte_write_new_pte(vcpu, page, spte, new, bytes, + mmu_pte_write_zap_pte(vcpu, sp, spte); + mmu_pte_write_new_pte(vcpu, sp, spte, new, bytes, page_offset & (pte_size - 1)); mmu_pte_write_flush_tlb(vcpu, entry, *spte); ++spte; @@ -1362,11 +1354,11 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; - page = container_of(vcpu->kvm->active_mmu_pages.prev, - struct kvm_mmu_page, link); - kvm_mmu_zap_page(vcpu->kvm, page); + sp = container_of(vcpu->kvm->active_mmu_pages.prev, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_recycled; } } @@ -1413,12 +1405,12 @@ EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); static void free_mmu_pages(struct kvm_vcpu *vcpu) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; while (!list_empty(&vcpu->kvm->active_mmu_pages)) { - page = container_of(vcpu->kvm->active_mmu_pages.next, - struct kvm_mmu_page, link); - kvm_mmu_zap_page(vcpu->kvm, page); + sp = container_of(vcpu->kvm->active_mmu_pages.next, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(vcpu->kvm, sp); } free_page((unsigned long)vcpu->mmu.pae_root); } @@ -1480,16 +1472,16 @@ void kvm_mmu_destroy(struct kvm_vcpu *vcpu) void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; - list_for_each_entry(page, &kvm->active_mmu_pages, link) { + list_for_each_entry(sp, &kvm->active_mmu_pages, link) { int i; u64 *pt; - if (!test_bit(slot, &page->slot_bitmap)) + if (!test_bit(slot, &sp->slot_bitmap)) continue; - pt = page->spt; + pt = sp->spt; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) /* avoid RMW */ if (pt[i] & PT_WRITABLE_MASK) @@ -1499,10 +1491,10 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) void kvm_mmu_zap_all(struct kvm *kvm) { - struct kvm_mmu_page *page, *node; + struct kvm_mmu_page *sp, *node; - list_for_each_entry_safe(page, node, &kvm->active_mmu_pages, link) - kvm_mmu_zap_page(kvm, page); + list_for_each_entry_safe(sp, node, &kvm->active_mmu_pages, link) + kvm_mmu_zap_page(kvm, sp); kvm_flush_remote_tlbs(kvm); } @@ -1668,13 +1660,13 @@ static int count_rmaps(struct kvm_vcpu *vcpu) static int count_writable_mappings(struct kvm_vcpu *vcpu) { int nmaps = 0; - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; int i; - list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) { - u64 *pt = page->spt; + list_for_each_entry(sp, &vcpu->kvm->active_mmu_pages, link) { + u64 *pt = sp->spt; - if (page->role.level != PT_PAGE_TABLE_LEVEL) + if (sp->role.level != PT_PAGE_TABLE_LEVEL) continue; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { @@ -1702,23 +1694,23 @@ static void audit_rmap(struct kvm_vcpu *vcpu) static void audit_write_protection(struct kvm_vcpu *vcpu) { - struct kvm_mmu_page *page; + struct kvm_mmu_page *sp; struct kvm_memory_slot *slot; unsigned long *rmapp; gfn_t gfn; - list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) { - if (page->role.metaphysical) + list_for_each_entry(sp, &vcpu->kvm->active_mmu_pages, link) { + if (sp->role.metaphysical) continue; - slot = gfn_to_memslot(vcpu->kvm, page->gfn); - gfn = unalias_gfn(vcpu->kvm, page->gfn); + slot = gfn_to_memslot(vcpu->kvm, sp->gfn); + gfn = unalias_gfn(vcpu->kvm, sp->gfn); rmapp = &slot->rmap[gfn - slot->base_gfn]; if (*rmapp) printk(KERN_ERR "%s: (%s) shadow page has writable" " mappings: gfn %lx role %x\n", - __FUNCTION__, audit_msg, page->gfn, - page->role.word); + __FUNCTION__, audit_msg, sp->gfn, + sp->role.word); } } -- cgit v1.2.3 From 76c35c6e99cb46b936b88cc795c9c886e7fe7bd4 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 15:32:41 +0200 Subject: KVM: MMU: Rename 'release_page' Rename the awkwardly named variable. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 86896da3e837..0cb6580956ba 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -430,18 +430,18 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) struct kvm_rmap_desc *desc; struct kvm_rmap_desc *prev_desc; struct kvm_mmu_page *sp; - struct page *release_page; + struct page *page; unsigned long *rmapp; int i; if (!is_rmap_pte(*spte)) return; sp = page_header(__pa(spte)); - release_page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); if (is_writeble_pte(*spte)) - kvm_release_page_dirty(release_page); + kvm_release_page_dirty(page); else - kvm_release_page_clean(release_page); + kvm_release_page_clean(page); rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt]); if (!*rmapp) { printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); -- cgit v1.2.3 From 6d4e4c4fca5be806b888d606894d914847e82d78 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Wed, 21 Nov 2007 16:41:05 +0200 Subject: KVM: Disallow fork() and similar games when using a VM We don't want the meaning of guest userspace changing under our feet. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/kvm_main.c | 9 +++++++++ kernel/fork.c | 1 + 3 files changed, 11 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 1fd8158ced89..be18620bd656 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -305,6 +305,7 @@ struct kvm_vm_stat { struct kvm { struct mutex lock; /* protects everything except vcpus */ + struct mm_struct *mm; /* userspace tied to this vm */ int naliases; struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; int nmemslots; diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index aec6b67cfebb..0efd759e585f 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -165,6 +165,8 @@ static struct kvm *kvm_create_vm(void) if (IS_ERR(kvm)) goto out; + kvm->mm = current->mm; + atomic_inc(&kvm->mm->mm_count); kvm_io_bus_init(&kvm->pio_bus); mutex_init(&kvm->lock); kvm_io_bus_init(&kvm->mmio_bus); @@ -202,12 +204,15 @@ void kvm_free_physmem(struct kvm *kvm) static void kvm_destroy_vm(struct kvm *kvm) { + struct mm_struct *mm = kvm->mm; + spin_lock(&kvm_lock); list_del(&kvm->vm_list); spin_unlock(&kvm_lock); kvm_io_bus_destroy(&kvm->pio_bus); kvm_io_bus_destroy(&kvm->mmio_bus); kvm_arch_destroy_vm(kvm); + mmdrop(mm); } static int kvm_vm_release(struct inode *inode, struct file *filp) @@ -818,6 +823,8 @@ static long kvm_vcpu_ioctl(struct file *filp, void __user *argp = (void __user *)arg; int r; + if (vcpu->kvm->mm != current->mm) + return -EIO; switch (ioctl) { case KVM_RUN: r = -EINVAL; @@ -976,6 +983,8 @@ static long kvm_vm_ioctl(struct file *filp, void __user *argp = (void __user *)arg; int r; + if (kvm->mm != current->mm) + return -EIO; switch (ioctl) { case KVM_CREATE_VCPU: r = kvm_vm_ioctl_create_vcpu(kvm, arg); diff --git a/kernel/fork.c b/kernel/fork.c index 314f5101d2b0..05e0b6f4365b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -393,6 +393,7 @@ void fastcall __mmdrop(struct mm_struct *mm) destroy_context(mm); free_mm(mm); } +EXPORT_SYMBOL_GPL(__mmdrop); /* * Decrement the use count and release all resources for an mm. -- cgit v1.2.3 From 0771671749b59a507b6da4efb931c44d9691e248 Mon Sep 17 00:00:00 2001 From: Dan Kenigsberg Date: Wed, 21 Nov 2007 17:10:04 +0200 Subject: KVM: Enhance guest cpuid management The current cpuid management suffers from several problems, which inhibit passing through the host feature set to the guest: - No way to tell which features the host supports While some features can be supported with no changes to kvm, others need explicit support. That means kvm needs to vet the feature set before it is passed to the guest. - No support for indexed or stateful cpuid entries Some cpuid entries depend on ecx as well as on eax, or on internal state in the processor (running cpuid multiple times with the same input returns different output). The current cpuid machinery only supports keying on eax. - No support for save/restore/migrate The internal state above needs to be exposed to userspace so it can be saved or migrated. This patch adds extended cpuid support by means of three new ioctls: - KVM_GET_SUPPORTED_CPUID: get all cpuid entries the host (and kvm) supports - KVM_SET_CPUID2: sets the vcpu's cpuid table - KVM_GET_CPUID2: gets the vcpu's cpuid table, including hidden state [avi: fix original KVM_SET_CPUID not removing nx on non-nx hosts as it did before] Signed-off-by: Dan Kenigsberg Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 331 ++++++++++++++++++++++++++++++++++++++++++++++++-- drivers/kvm/x86.h | 2 +- include/asm-x86/kvm.h | 21 ++++ include/linux/kvm.h | 4 + 4 files changed, 348 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 15e1203faef0..7237cb25f77d 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -646,6 +646,7 @@ int kvm_dev_ioctl_check_extension(long ext) case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: case KVM_CAP_USER_MEMORY: case KVM_CAP_SET_TSS_ADDR: + case KVM_CAP_EXT_CPUID: r = 1; break; default: @@ -708,13 +709,19 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_put_guest_fpu(vcpu); } -static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) +static int is_efer_nx(void) { u64 efer; - int i; - struct kvm_cpuid_entry *e, *entry; rdmsrl(MSR_EFER, efer); + return efer & EFER_NX; +} + +static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) +{ + int i; + struct kvm_cpuid_entry2 *e, *entry; + entry = NULL; for (i = 0; i < vcpu->cpuid_nent; ++i) { e = &vcpu->cpuid_entries[i]; @@ -723,15 +730,56 @@ static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) break; } } - if (entry && (entry->edx & (1 << 20)) && !(efer & EFER_NX)) { + if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) { entry->edx &= ~(1 << 20); printk(KERN_INFO "kvm: guest NX capability removed\n"); } } +/* when an old userspace process fills a new kernel module */ static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid *cpuid, struct kvm_cpuid_entry __user *entries) +{ + int r, i; + struct kvm_cpuid_entry *cpuid_entries; + + r = -E2BIG; + if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) + goto out; + r = -ENOMEM; + cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent); + if (!cpuid_entries) + goto out; + r = -EFAULT; + if (copy_from_user(cpuid_entries, entries, + cpuid->nent * sizeof(struct kvm_cpuid_entry))) + goto out_free; + for (i = 0; i < cpuid->nent; i++) { + vcpu->cpuid_entries[i].function = cpuid_entries[i].function; + vcpu->cpuid_entries[i].eax = cpuid_entries[i].eax; + vcpu->cpuid_entries[i].ebx = cpuid_entries[i].ebx; + vcpu->cpuid_entries[i].ecx = cpuid_entries[i].ecx; + vcpu->cpuid_entries[i].edx = cpuid_entries[i].edx; + vcpu->cpuid_entries[i].index = 0; + vcpu->cpuid_entries[i].flags = 0; + vcpu->cpuid_entries[i].padding[0] = 0; + vcpu->cpuid_entries[i].padding[1] = 0; + vcpu->cpuid_entries[i].padding[2] = 0; + } + vcpu->cpuid_nent = cpuid->nent; + cpuid_fix_nx_cap(vcpu); + r = 0; + +out_free: + vfree(cpuid_entries); +out: + return r; +} + +static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) { int r; @@ -740,16 +788,198 @@ static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, goto out; r = -EFAULT; if (copy_from_user(&vcpu->cpuid_entries, entries, - cpuid->nent * sizeof(struct kvm_cpuid_entry))) + cpuid->nent * sizeof(struct kvm_cpuid_entry2))) goto out; vcpu->cpuid_nent = cpuid->nent; - cpuid_fix_nx_cap(vcpu); return 0; out: return r; } +static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) +{ + int r; + + r = -E2BIG; + if (cpuid->nent < vcpu->cpuid_nent) + goto out; + r = -EFAULT; + if (copy_to_user(entries, &vcpu->cpuid_entries, + vcpu->cpuid_nent * sizeof(struct kvm_cpuid_entry2))) + goto out; + return 0; + +out: + cpuid->nent = vcpu->cpuid_nent; + return r; +} + +static inline u32 bit(int bitno) +{ + return 1 << (bitno & 31); +} + +static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, + u32 index) +{ + entry->function = function; + entry->index = index; + cpuid_count(entry->function, entry->index, + &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); + entry->flags = 0; +} + +static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, + u32 index, int *nent, int maxnent) +{ + const u32 kvm_supported_word0_x86_features = bit(X86_FEATURE_FPU) | + bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) | + bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) | + bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) | + bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) | + bit(X86_FEATURE_SEP) | bit(X86_FEATURE_PGE) | + bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) | + bit(X86_FEATURE_CLFLSH) | bit(X86_FEATURE_MMX) | + bit(X86_FEATURE_FXSR) | bit(X86_FEATURE_XMM) | + bit(X86_FEATURE_XMM2) | bit(X86_FEATURE_SELFSNOOP); + const u32 kvm_supported_word1_x86_features = bit(X86_FEATURE_FPU) | + bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) | + bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) | + bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) | + bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) | + bit(X86_FEATURE_PGE) | + bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) | + bit(X86_FEATURE_MMX) | bit(X86_FEATURE_FXSR) | + bit(X86_FEATURE_SYSCALL) | + (bit(X86_FEATURE_NX) && is_efer_nx()) | +#ifdef CONFIG_X86_64 + bit(X86_FEATURE_LM) | +#endif + bit(X86_FEATURE_MMXEXT) | + bit(X86_FEATURE_3DNOWEXT) | + bit(X86_FEATURE_3DNOW); + const u32 kvm_supported_word3_x86_features = + bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16); + const u32 kvm_supported_word6_x86_features = + bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY); + + /* all func 2 cpuid_count() should be called on the same cpu */ + get_cpu(); + do_cpuid_1_ent(entry, function, index); + ++*nent; + + switch (function) { + case 0: + entry->eax = min(entry->eax, (u32)0xb); + break; + case 1: + entry->edx &= kvm_supported_word0_x86_features; + entry->ecx &= kvm_supported_word3_x86_features; + break; + /* function 2 entries are STATEFUL. That is, repeated cpuid commands + * may return different values. This forces us to get_cpu() before + * issuing the first command, and also to emulate this annoying behavior + * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */ + case 2: { + int t, times = entry->eax & 0xff; + + entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; + for (t = 1; t < times && *nent < maxnent; ++t) { + do_cpuid_1_ent(&entry[t], function, 0); + entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; + ++*nent; + } + break; + } + /* function 4 and 0xb have additional index. */ + case 4: { + int index, cache_type; + + entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + /* read more entries until cache_type is zero */ + for (index = 1; *nent < maxnent; ++index) { + cache_type = entry[index - 1].eax & 0x1f; + if (!cache_type) + break; + do_cpuid_1_ent(&entry[index], function, index); + entry[index].flags |= + KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + ++*nent; + } + break; + } + case 0xb: { + int index, level_type; + + entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + /* read more entries until level_type is zero */ + for (index = 1; *nent < maxnent; ++index) { + level_type = entry[index - 1].ecx & 0xff; + if (!level_type) + break; + do_cpuid_1_ent(&entry[index], function, index); + entry[index].flags |= + KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + ++*nent; + } + break; + } + case 0x80000000: + entry->eax = min(entry->eax, 0x8000001a); + break; + case 0x80000001: + entry->edx &= kvm_supported_word1_x86_features; + entry->ecx &= kvm_supported_word6_x86_features; + break; + } + put_cpu(); +} + +static int kvm_vm_ioctl_get_supported_cpuid(struct kvm *kvm, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) +{ + struct kvm_cpuid_entry2 *cpuid_entries; + int limit, nent = 0, r = -E2BIG; + u32 func; + + if (cpuid->nent < 1) + goto out; + r = -ENOMEM; + cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent); + if (!cpuid_entries) + goto out; + + do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent); + limit = cpuid_entries[0].eax; + for (func = 1; func <= limit && nent < cpuid->nent; ++func) + do_cpuid_ent(&cpuid_entries[nent], func, 0, + &nent, cpuid->nent); + r = -E2BIG; + if (nent >= cpuid->nent) + goto out_free; + + do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent); + limit = cpuid_entries[nent - 1].eax; + for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func) + do_cpuid_ent(&cpuid_entries[nent], func, 0, + &nent, cpuid->nent); + r = -EFAULT; + if (copy_to_user(entries, cpuid_entries, + nent * sizeof(struct kvm_cpuid_entry2))) + goto out_free; + cpuid->nent = nent; + r = 0; + +out_free: + vfree(cpuid_entries); +out: + return r; +} + static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) { @@ -816,6 +1046,36 @@ long kvm_arch_vcpu_ioctl(struct file *filp, goto out; break; } + case KVM_SET_CPUID2: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + break; + } + case KVM_GET_CPUID2: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) + goto out; + r = 0; + break; + } case KVM_GET_MSRS: r = msr_io(vcpu, argp, kvm_get_msr, 1); break; @@ -1111,6 +1371,24 @@ long kvm_arch_vm_ioctl(struct file *filp, r = 0; break; } + case KVM_GET_SUPPORTED_CPUID: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vm_ioctl_get_supported_cpuid(kvm, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + + r = -EFAULT; + if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) + goto out; + r = 0; + break; + } default: ; } @@ -1908,14 +2186,47 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, } } +static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i) +{ + struct kvm_cpuid_entry2 *e = &vcpu->cpuid_entries[i]; + int j, nent = vcpu->cpuid_nent; + + e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT; + /* when no next entry is found, the current entry[i] is reselected */ + for (j = i + 1; j == i; j = (j + 1) % nent) { + struct kvm_cpuid_entry2 *ej = &vcpu->cpuid_entries[j]; + if (ej->function == e->function) { + ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; + return j; + } + } + return 0; /* silence gcc, even though control never reaches here */ +} + +/* find an entry with matching function, matching index (if needed), and that + * should be read next (if it's stateful) */ +static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e, + u32 function, u32 index) +{ + if (e->function != function) + return 0; + if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index) + return 0; + if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) && + !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT)) + return 0; + return 1; +} + void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) { int i; - u32 function; - struct kvm_cpuid_entry *e, *best; + u32 function, index; + struct kvm_cpuid_entry2 *e, *best; kvm_x86_ops->cache_regs(vcpu); function = vcpu->regs[VCPU_REGS_RAX]; + index = vcpu->regs[VCPU_REGS_RCX]; vcpu->regs[VCPU_REGS_RAX] = 0; vcpu->regs[VCPU_REGS_RBX] = 0; vcpu->regs[VCPU_REGS_RCX] = 0; @@ -1923,7 +2234,9 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) best = NULL; for (i = 0; i < vcpu->cpuid_nent; ++i) { e = &vcpu->cpuid_entries[i]; - if (e->function == function) { + if (is_matching_cpuid_entry(e, function, index)) { + if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) + move_to_next_stateful_cpuid_entry(vcpu, i); best = e; break; } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index b1528c9f566f..78ab1e108d8b 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -149,7 +149,7 @@ struct kvm_vcpu { int halt_request; /* real mode on Intel only */ int cpuid_nent; - struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES]; + struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; /* emulate context */ diff --git a/include/asm-x86/kvm.h b/include/asm-x86/kvm.h index 4837d759bec0..17afa81d6d66 100644 --- a/include/asm-x86/kvm.h +++ b/include/asm-x86/kvm.h @@ -151,5 +151,26 @@ struct kvm_cpuid { struct kvm_cpuid_entry entries[0]; }; +struct kvm_cpuid_entry2 { + __u32 function; + __u32 index; + __u32 flags; + __u32 eax; + __u32 ebx; + __u32 ecx; + __u32 edx; + __u32 padding[3]; +}; + +#define KVM_CPUID_FLAG_SIGNIFCANT_INDEX 1 +#define KVM_CPUID_FLAG_STATEFUL_FUNC 2 +#define KVM_CPUID_FLAG_STATE_READ_NEXT 4 + +/* for KVM_SET_CPUID2 */ +struct kvm_cpuid2 { + __u32 nent; + __u32 padding; + struct kvm_cpuid_entry2 entries[0]; +}; #endif diff --git a/include/linux/kvm.h b/include/linux/kvm.h index fd4f900fcce3..b751552f2e30 100644 --- a/include/linux/kvm.h +++ b/include/linux/kvm.h @@ -224,6 +224,7 @@ struct kvm_signal_mask { #define KVM_CAP_MMU_SHADOW_CACHE_CONTROL 2 #define KVM_CAP_USER_MEMORY 3 #define KVM_CAP_SET_TSS_ADDR 4 +#define KVM_CAP_EXT_CPUID 5 /* * ioctls for VM fds @@ -241,6 +242,7 @@ struct kvm_signal_mask { #define KVM_CREATE_VCPU _IO(KVMIO, 0x41) #define KVM_GET_DIRTY_LOG _IOW(KVMIO, 0x42, struct kvm_dirty_log) #define KVM_SET_MEMORY_ALIAS _IOW(KVMIO, 0x43, struct kvm_memory_alias) +#define KVM_GET_SUPPORTED_CPUID _IOWR(KVMIO, 0x48, struct kvm_cpuid2) /* Device model IOC */ #define KVM_CREATE_IRQCHIP _IO(KVMIO, 0x60) #define KVM_IRQ_LINE _IOW(KVMIO, 0x61, struct kvm_irq_level) @@ -266,5 +268,7 @@ struct kvm_signal_mask { #define KVM_SET_FPU _IOW(KVMIO, 0x8d, struct kvm_fpu) #define KVM_GET_LAPIC _IOR(KVMIO, 0x8e, struct kvm_lapic_state) #define KVM_SET_LAPIC _IOW(KVMIO, 0x8f, struct kvm_lapic_state) +#define KVM_SET_CPUID2 _IOW(KVMIO, 0x90, struct kvm_cpuid2) +#define KVM_GET_CPUID2 _IOWR(KVMIO, 0x91, struct kvm_cpuid2) #endif -- cgit v1.2.3 From 83ff3b9d4a0ade8f8eb9757fd4b36c501ba161e4 Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Wed, 21 Nov 2007 14:33:25 +0800 Subject: KVM: VMX: Remove the secondary execute control dependency on irqchip The state of SECONDARY_VM_EXEC_CONTROL shouldn't depend on in-kernel IRQ chip, this patch fix this. Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index b4c0bdce7b34..fc5e7c8381ce 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -193,11 +193,6 @@ static inline int cpu_has_secondary_exec_ctrls(void) CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); } -static inline int vm_need_secondary_exec_ctrls(struct kvm *kvm) -{ - return ((cpu_has_secondary_exec_ctrls()) && (irqchip_in_kernel(kvm))); -} - static inline int cpu_has_vmx_virtualize_apic_accesses(void) { return (vmcs_config.cpu_based_2nd_exec_ctrl & @@ -1524,13 +1519,15 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) CPU_BASED_CR8_LOAD_EXITING; #endif } - if (!vm_need_secondary_exec_ctrls(vmx->vcpu.kvm)) - exec_control &= ~CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); - if (vm_need_secondary_exec_ctrls(vmx->vcpu.kvm)) - vmcs_write32(SECONDARY_VM_EXEC_CONTROL, - vmcs_config.cpu_based_2nd_exec_ctrl); + if (cpu_has_secondary_exec_ctrls()) { + exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; + if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + exec_control &= + ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; + vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); + } vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); -- cgit v1.2.3 From e9f85cde99fa011db256f6683ba81d155e0d3f57 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Thu, 22 Nov 2007 11:20:33 +0800 Subject: KVM: Portability: Move unalias_gfn to arch dependent file Non-x86 archs don't need this mechanism. Move it to arch, and keep its interface in common. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 14 -------------- drivers/kvm/x86.c | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 0efd759e585f..721e660023f6 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -408,20 +408,6 @@ int kvm_is_error_hva(unsigned long addr) } EXPORT_SYMBOL_GPL(kvm_is_error_hva); -gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) -{ - int i; - struct kvm_mem_alias *alias; - - for (i = 0; i < kvm->naliases; ++i) { - alias = &kvm->aliases[i]; - if (gfn >= alias->base_gfn - && gfn < alias->base_gfn + alias->npages) - return alias->target_gfn + gfn - alias->base_gfn; - } - return gfn; -} - static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { int i; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 7237cb25f77d..2be59364a49d 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1119,6 +1119,20 @@ static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) return kvm->n_alloc_mmu_pages; } +gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) +{ + int i; + struct kvm_mem_alias *alias; + + for (i = 0; i < kvm->naliases; ++i) { + alias = &kvm->aliases[i]; + if (gfn >= alias->base_gfn + && gfn < alias->base_gfn + alias->npages) + return alias->target_gfn + gfn - alias->base_gfn; + } + return gfn; +} + /* * Set a new alias region. Aliases map a portion of physical memory into * another portion. This is useful for memory windows, for example the PC -- cgit v1.2.3 From 90e0a28f6b7241c7793f2ebd540c349580170446 Mon Sep 17 00:00:00 2001 From: Guillaume Thouvenin Date: Thu, 22 Nov 2007 11:32:09 +0100 Subject: KVM: x86 emulator: Make a distinction between repeat prefixes F3 and F2 cmps and scas instructions accept repeat prefixes F3 and F2. So in order to emulate those prefixed instructions we need to be able to know if prefixes are REP/REPE/REPZ or REPNE/REPNZ. Currently kvm doesn't make this distinction. This patch introduces this distinction. Signed-off-by: Guillaume Thouvenin Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 4 +++- drivers/kvm/x86_emulate.h | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 6e7f774d1751..9f8d59a49313 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -824,8 +824,10 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) c->lock_prefix = 1; break; case 0xf2: /* REPNE/REPNZ */ + c->rep_prefix = REPNE_PREFIX; + break; case 0xf3: /* REP/REPE/REPZ */ - c->rep_prefix = 1; + c->rep_prefix = REPE_PREFIX; break; default: goto done_prefixes; diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index 4603b2bf3488..644086e354a7 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -162,6 +162,10 @@ struct x86_emulate_ctxt { struct decode_cache decode; }; +/* Repeat String Operation Prefix */ +#define REPE_PREFIX 1 +#define REPNE_PREFIX 2 + /* Execution mode, passed to the emulator. */ #define X86EMUL_MODE_REAL 0 /* Real mode. */ #define X86EMUL_MODE_PROT16 2 /* 16-bit protected mode. */ -- cgit v1.2.3 From f21b8bf4cc4091b23669987124fd13f758abf6d6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 22 Nov 2007 14:16:12 +0200 Subject: KVM: x86 emulator: address size and operand size overrides are sticky Current implementation is to toggle, which is incorrect. Patch ported from corresponding Xen code. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 9f8d59a49313..3be506ac01a3 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -758,6 +758,7 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) struct decode_cache *c = &ctxt->decode; int rc = 0; int mode = ctxt->mode; + int def_op_bytes, def_ad_bytes; /* Shadow copy of register state. Committed on successful emulation. */ @@ -768,34 +769,38 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) switch (mode) { case X86EMUL_MODE_REAL: case X86EMUL_MODE_PROT16: - c->op_bytes = c->ad_bytes = 2; + def_op_bytes = def_ad_bytes = 2; break; case X86EMUL_MODE_PROT32: - c->op_bytes = c->ad_bytes = 4; + def_op_bytes = def_ad_bytes = 4; break; #ifdef CONFIG_X86_64 case X86EMUL_MODE_PROT64: - c->op_bytes = 4; - c->ad_bytes = 8; + def_op_bytes = 4; + def_ad_bytes = 8; break; #endif default: return -1; } + c->op_bytes = def_op_bytes; + c->ad_bytes = def_ad_bytes; + /* Legacy prefixes. */ for (;;) { switch (c->b = insn_fetch(u8, 1, c->eip)) { case 0x66: /* operand-size override */ - c->op_bytes ^= 6; /* switch between 2/4 bytes */ + /* switch between 2/4 bytes */ + c->op_bytes = def_op_bytes ^ 6; break; case 0x67: /* address-size override */ if (mode == X86EMUL_MODE_PROT64) /* switch between 4/8 bytes */ - c->ad_bytes ^= 12; + c->ad_bytes = def_ad_bytes ^ 12; else /* switch between 2/4 bytes */ - c->ad_bytes ^= 6; + c->ad_bytes = def_ad_bytes ^ 6; break; case 0x2e: /* CS override */ c->override_base = &ctxt->cs_base; -- cgit v1.2.3 From eb9774f0d675922d780f23cbb747f474b0573dea Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 17:45:31 +0200 Subject: KVM: Remove misleading check for mmio during event injection mmio was already handled in kvm_arch_vcpu_ioctl_run(), so no need to check again. Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 2be59364a49d..5e5815604cbd 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2344,7 +2344,7 @@ again: if (irqchip_in_kernel(vcpu->kvm)) kvm_x86_ops->inject_pending_irq(vcpu); - else if (!vcpu->mmio_read_completed) + else kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); vcpu->guest_mode = 1; -- cgit v1.2.3 From 448353caeab1d9180f12194a80790859ddbeebd5 Mon Sep 17 00:00:00 2001 From: Izik Eidus Date: Mon, 26 Nov 2007 14:08:14 +0200 Subject: KVM: MMU: mark pages that were inserted to the shadow pages table as accessed Mark guest pages as accessed when removed from the shadow page tables for better lru processing. Signed-off-by: Izik Eidus Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 0cb6580956ba..9b9d1b6eb2f8 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -438,6 +439,7 @@ static void rmap_remove(struct kvm *kvm, u64 *spte) return; sp = page_header(__pa(spte)); page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + mark_page_accessed(page); if (is_writeble_pte(*spte)) kvm_release_page_dirty(page); else -- cgit v1.2.3 From e8d8d7fe8877c594c08f40cc7c013626cfe3e9cc Mon Sep 17 00:00:00 2001 From: Sheng Yang Date: Fri, 16 Nov 2007 16:29:15 +0800 Subject: KVM: x86 emulator: Rename 'cr2' to 'memop' Previous patches have removed the dependency on cr2; we can now stop passing it to the emulator and rename uses to 'memop'. Signed-off-by: Sheng Yang Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 1 - drivers/kvm/x86_emulate.c | 18 +++++++++--------- drivers/kvm/x86_emulate.h | 1 - 3 files changed, 9 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5e5815604cbd..12f1d6fc92bb 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1702,7 +1702,6 @@ int emulate_instruction(struct kvm_vcpu *vcpu, vcpu->emulate_ctxt.vcpu = vcpu; vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - vcpu->emulate_ctxt.cr2 = cr2; vcpu->emulate_ctxt.mode = (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_REAL : cs_l diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 3be506ac01a3..22fdf0ac6615 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1127,13 +1127,13 @@ static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops, - unsigned long cr2) + unsigned long memop) { struct decode_cache *c = &ctxt->decode; u64 old, new; int rc; - rc = ops->read_emulated(cr2, &old, 8, ctxt->vcpu); + rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu); if (rc != 0) return rc; @@ -1148,7 +1148,7 @@ static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, new = ((u64)c->regs[VCPU_REGS_RCX] << 32) | (u32) c->regs[VCPU_REGS_RBX]; - rc = ops->cmpxchg_emulated(cr2, &old, &new, 8, ctxt->vcpu); + rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu); if (rc != 0) return rc; ctxt->eflags |= EFLG_ZF; @@ -1211,7 +1211,7 @@ static inline int writeback(struct x86_emulate_ctxt *ctxt, int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) { - unsigned long cr2 = ctxt->cr2; + unsigned long memop = 0; u64 msr_data; unsigned long saved_eip = 0; struct decode_cache *c = &ctxt->decode; @@ -1226,10 +1226,10 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) saved_eip = c->eip; if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) - cr2 = c->modrm_ea; + memop = c->modrm_ea; if (c->src.type == OP_MEM) { - c->src.ptr = (unsigned long *)cr2; + c->src.ptr = (unsigned long *)memop; c->src.val = 0; rc = ops->read_emulated((unsigned long)c->src.ptr, &c->src.val, @@ -1245,7 +1245,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (c->dst.type == OP_MEM) { - c->dst.ptr = (unsigned long *)cr2; + c->dst.ptr = (unsigned long *)memop; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; c->dst.val = 0; if (c->d & BitOp) { @@ -1677,7 +1677,7 @@ twobyte_insn: &ctxt->eflags); break; case 7: /* invlpg*/ - emulate_invlpg(ctxt->vcpu, cr2); + emulate_invlpg(ctxt->vcpu, memop); break; default: goto cannot_emulate; @@ -1848,7 +1848,7 @@ twobyte_special_insn: break; } case 0xc7: /* Grp9 (cmpxchg8b) */ - rc = emulate_grp9(ctxt, ops, cr2); + rc = emulate_grp9(ctxt, ops, memop); if (rc != 0) goto done; break; diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h index 644086e354a7..7db91b9bdcd4 100644 --- a/drivers/kvm/x86_emulate.h +++ b/drivers/kvm/x86_emulate.h @@ -145,7 +145,6 @@ struct x86_emulate_ctxt { /* Linear faulting address (if emulating a page-faulting instruction). */ unsigned long eflags; - unsigned long cr2; /* Emulated execution mode, represented by an X86EMUL_MODE value. */ int mode; -- cgit v1.2.3 From d7e5117a2568f7407e98ca85155511ecfe4f0631 Mon Sep 17 00:00:00 2001 From: Guillaume Thouvenin Date: Mon, 26 Nov 2007 13:49:09 +0100 Subject: KVM: x86 emulator: cmps instruction Add emulation for the cmps instruction. This lets OpenBSD boot on kvm. Signed-off-by: Guillaume Thouvenin Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 58 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 22fdf0ac6615..84e536603f5a 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1535,10 +1535,31 @@ special_insn: break; } if (c->rep_prefix) { + /* All REP prefixes have the same first termination condition */ if (c->regs[VCPU_REGS_RCX] == 0) { ctxt->vcpu->rip = c->eip; goto done; } + /* The second termination condition only applies for REPE + * and REPNE. Test if the repeat string operation prefix is + * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the + * corresponding termination condition according to: + * - if REPE/REPZ and ZF = 0 then done + * - if REPNE/REPNZ and ZF = 1 then done + */ + if ((c->b == 0xa6) || (c->b == 0xa7) || + (c->b == 0xae) || (c->b == 0xaf)) { + if ((c->rep_prefix == REPE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == 0)) { + ctxt->vcpu->rip = c->eip; + goto done; + } + if ((c->rep_prefix == REPNE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { + ctxt->vcpu->rip = c->eip; + goto done; + } + } c->regs[VCPU_REGS_RCX]--; c->eip = ctxt->vcpu->rip; } @@ -1564,8 +1585,41 @@ special_insn: : c->dst.bytes); break; case 0xa6 ... 0xa7: /* cmps */ - DPRINTF("Urk! I don't handle CMPS.\n"); - goto cannot_emulate; + c->src.type = OP_NONE; /* Disable writeback. */ + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->src.ptr = (unsigned long *)register_address( + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]); + if ((rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, + ctxt->vcpu)) != 0) + goto done; + + c->dst.type = OP_NONE; /* Disable writeback. */ + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); + if ((rc = ops->read_emulated((unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu)) != 0) + goto done; + + DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr); + + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); + + register_address_increment(c->regs[VCPU_REGS_RSI], + (ctxt->eflags & EFLG_DF) ? -c->src.bytes + : c->src.bytes); + register_address_increment(c->regs[VCPU_REGS_RDI], + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + + break; case 0xaa ... 0xab: /* stos */ c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; -- cgit v1.2.3 From b9fa9d6bc6ac617170c4cf21ca764eb187618c0d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 27 Nov 2007 19:05:37 +0200 Subject: KVM: x86 emulator: Move rep processing before instruction execution Currently rep processing is handled somewhere in the middle of instruction processing. Move it to a sensible place. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 70 ++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 34 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 84e536603f5a..53377f0b202e 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -64,6 +64,7 @@ #define Mov (1<<7) #define BitOp (1<<8) #define MemAbs (1<<9) /* Memory operand is absolute displacement */ +#define String (1<<10) /* String instruction (rep capable) */ static u16 opcode_table[256] = { /* 0x00 - 0x07 */ @@ -133,12 +134,12 @@ static u16 opcode_table[256] = { /* 0xA0 - 0xA7 */ ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs, - ByteOp | ImplicitOps | Mov, ImplicitOps | Mov, - ByteOp | ImplicitOps, ImplicitOps, + ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | String, ImplicitOps | String, /* 0xA8 - 0xAF */ - 0, 0, ByteOp | ImplicitOps | Mov, ImplicitOps | Mov, - ByteOp | ImplicitOps | Mov, ImplicitOps | Mov, - ByteOp | ImplicitOps, ImplicitOps, + 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | String, ImplicitOps | String, /* 0xB0 - 0xBF */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0xC0 - 0xC7 */ @@ -1228,6 +1229,36 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) memop = c->modrm_ea; + if (c->rep_prefix && (c->d & String)) { + /* All REP prefixes have the same first termination condition */ + if (c->regs[VCPU_REGS_RCX] == 0) { + ctxt->vcpu->rip = c->eip; + goto done; + } + /* The second termination condition only applies for REPE + * and REPNE. Test if the repeat string operation prefix is + * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the + * corresponding termination condition according to: + * - if REPE/REPZ and ZF = 0 then done + * - if REPNE/REPNZ and ZF = 1 then done + */ + if ((c->b == 0xa6) || (c->b == 0xa7) || + (c->b == 0xae) || (c->b == 0xaf)) { + if ((c->rep_prefix == REPE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == 0)) { + ctxt->vcpu->rip = c->eip; + goto done; + } + if ((c->rep_prefix == REPNE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { + ctxt->vcpu->rip = c->eip; + goto done; + } + } + c->regs[VCPU_REGS_RCX]--; + c->eip = ctxt->vcpu->rip; + } + if (c->src.type == OP_MEM) { c->src.ptr = (unsigned long *)memop; c->src.val = 0; @@ -1534,35 +1565,6 @@ special_insn: c->dst.type = OP_NONE; /* Disable writeback. */ break; } - if (c->rep_prefix) { - /* All REP prefixes have the same first termination condition */ - if (c->regs[VCPU_REGS_RCX] == 0) { - ctxt->vcpu->rip = c->eip; - goto done; - } - /* The second termination condition only applies for REPE - * and REPNE. Test if the repeat string operation prefix is - * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the - * corresponding termination condition according to: - * - if REPE/REPZ and ZF = 0 then done - * - if REPNE/REPNZ and ZF = 1 then done - */ - if ((c->b == 0xa6) || (c->b == 0xa7) || - (c->b == 0xae) || (c->b == 0xaf)) { - if ((c->rep_prefix == REPE_PREFIX) && - ((ctxt->eflags & EFLG_ZF) == 0)) { - ctxt->vcpu->rip = c->eip; - goto done; - } - if ((c->rep_prefix == REPNE_PREFIX) && - ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { - ctxt->vcpu->rip = c->eip; - goto done; - } - } - c->regs[VCPU_REGS_RCX]--; - c->eip = ctxt->vcpu->rip; - } switch (c->b) { case 0xa4 ... 0xa5: /* movs */ c->dst.type = OP_MEM; -- cgit v1.2.3 From 111de5d60c589bca02b49dce076ca588618e0d1c Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 27 Nov 2007 19:14:21 +0200 Subject: KVM: x86 emulator: unify two switches The rep prefix cleanup left two switch () statements next to each other. Unify them. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 50 ++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 53377f0b202e..499d1ad42bc4 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1541,31 +1541,6 @@ special_insn: case 0x9d: /* popf */ c->dst.ptr = (unsigned long *) &ctxt->eflags; goto pop_instruction; - case 0xc3: /* ret */ - c->dst.ptr = &c->eip; - goto pop_instruction; - case 0xf4: /* hlt */ - ctxt->vcpu->halt_request = 1; - goto done; - case 0xf5: /* cmc */ - /* complement carry flag from eflags reg */ - ctxt->eflags ^= EFLG_CF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xf8: /* clc */ - ctxt->eflags &= ~EFLG_CF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xfa: /* cli */ - ctxt->eflags &= ~X86_EFLAGS_IF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xfb: /* sti */ - ctxt->eflags |= X86_EFLAGS_IF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - } - switch (c->b) { case 0xa4 ... 0xa5: /* movs */ c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; @@ -1652,6 +1627,9 @@ special_insn: case 0xae ... 0xaf: /* scas */ DPRINTF("Urk! I don't handle SCAS.\n"); goto cannot_emulate; + case 0xc3: /* ret */ + c->dst.ptr = &c->eip; + goto pop_instruction; case 0xe8: /* call (near) */ { long int rel; switch (c->op_bytes) { @@ -1676,8 +1654,26 @@ special_insn: JMP_REL(c->src.val); c->dst.type = OP_NONE; /* Disable writeback. */ break; - - + case 0xf4: /* hlt */ + ctxt->vcpu->halt_request = 1; + goto done; + case 0xf5: /* cmc */ + /* complement carry flag from eflags reg */ + ctxt->eflags ^= EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xf8: /* clc */ + ctxt->eflags &= ~EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfa: /* cli */ + ctxt->eflags &= ~X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfb: /* sti */ + ctxt->eflags |= X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; } goto writeback; -- cgit v1.2.3 From 018a98db74d4cc8967cfc570197b14bcbdddb579 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Tue, 27 Nov 2007 19:30:56 +0200 Subject: KVM: x86 emulator: unify four switch statements into two Unify the special instruction switch with the regular instruction switch, and the two byte special instruction switch with the regular two byte instruction switch. That makes it much easier to find an instruction or the place an instruction needs to be added in. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 342 +++++++++++++++++++++++----------------------- 1 file changed, 168 insertions(+), 174 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 499d1ad42bc4..3e3eba70d5ac 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1294,6 +1294,8 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) } c->dst.orig_val = c->dst.val; +special_insn: + if (c->twobyte) goto twobyte_insn; @@ -1378,6 +1380,52 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) goto cannot_emulate; c->dst.val = (s32) c->src.val; break; + case 0x6a: /* push imm8 */ + c->src.val = 0L; + c->src.val = insn_fetch(s8, 1, c->eip); + emulate_push(ctxt); + break; + case 0x6c: /* insb */ + case 0x6d: /* insw/insd */ + if (kvm_emulate_pio_string(ctxt->vcpu, NULL, + 1, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (ctxt->eflags & EFLG_DF), + register_address(ctxt->es_base, + c->regs[VCPU_REGS_RDI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; + return -1; + } + return 0; + case 0x6e: /* outsb */ + case 0x6f: /* outsw/outsd */ + if (kvm_emulate_pio_string(ctxt->vcpu, NULL, + 0, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (ctxt->eflags & EFLG_DF), + register_address(c->override_base ? + *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; + return -1; + } + return 0; + case 0x70 ... 0x7f: /* jcc (short) */ { + int rel = insn_fetch(s8, 1, c->eip); + + if (test_cc(c->b, ctxt->eflags)) + JMP_REL(rel); + break; + } case 0x80 ... 0x83: /* Grp1 */ switch (c->modrm_reg) { case 0: @@ -1434,106 +1482,6 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (rc != 0) goto done; break; - case 0xa0 ... 0xa1: /* mov */ - c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; - c->dst.val = c->src.val; - break; - case 0xa2 ... 0xa3: /* mov */ - c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; - break; - case 0xc0 ... 0xc1: - emulate_grp2(ctxt); - break; - case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ - mov: - c->dst.val = c->src.val; - break; - case 0xd0 ... 0xd1: /* Grp2 */ - c->src.val = 1; - emulate_grp2(ctxt); - break; - case 0xd2 ... 0xd3: /* Grp2 */ - c->src.val = c->regs[VCPU_REGS_RCX]; - emulate_grp2(ctxt); - break; - case 0xf6 ... 0xf7: /* Grp3 */ - rc = emulate_grp3(ctxt, ops); - if (rc != 0) - goto done; - break; - case 0xfe ... 0xff: /* Grp4/Grp5 */ - rc = emulate_grp45(ctxt, ops); - if (rc != 0) - goto done; - break; - } - -writeback: - rc = writeback(ctxt, ops); - if (rc != 0) - goto done; - - /* Commit shadow register state. */ - memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); - ctxt->vcpu->rip = c->eip; - -done: - if (rc == X86EMUL_UNHANDLEABLE) { - c->eip = saved_eip; - return -1; - } - return 0; - -special_insn: - if (c->twobyte) - goto twobyte_special_insn; - switch (c->b) { - case 0x6a: /* push imm8 */ - c->src.val = 0L; - c->src.val = insn_fetch(s8, 1, c->eip); - emulate_push(ctxt); - break; - case 0x6c: /* insb */ - case 0x6d: /* insw/insd */ - if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 1, - (c->d & ByteOp) ? 1 : c->op_bytes, - c->rep_prefix ? - address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (ctxt->eflags & EFLG_DF), - register_address(ctxt->es_base, - c->regs[VCPU_REGS_RDI]), - c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) { - c->eip = saved_eip; - return -1; - } - return 0; - case 0x6e: /* outsb */ - case 0x6f: /* outsw/outsd */ - if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 0, - (c->d & ByteOp) ? 1 : c->op_bytes, - c->rep_prefix ? - address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (ctxt->eflags & EFLG_DF), - register_address(c->override_base ? - *c->override_base : - ctxt->ds_base, - c->regs[VCPU_REGS_RSI]), - c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) { - c->eip = saved_eip; - return -1; - } - return 0; - case 0x70 ... 0x7f: /* jcc (short) */ { - int rel = insn_fetch(s8, 1, c->eip); - - if (test_cc(c->b, ctxt->eflags)) - JMP_REL(rel); - break; - } case 0x9c: /* pushf */ c->src.val = (unsigned long) ctxt->eflags; emulate_push(ctxt); @@ -1541,6 +1489,13 @@ special_insn: case 0x9d: /* popf */ c->dst.ptr = (unsigned long *) &ctxt->eflags; goto pop_instruction; + case 0xa0 ... 0xa1: /* mov */ + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + c->dst.val = c->src.val; + break; + case 0xa2 ... 0xa3: /* mov */ + c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; + break; case 0xa4 ... 0xa5: /* movs */ c->dst.type = OP_MEM; c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; @@ -1627,9 +1582,24 @@ special_insn: case 0xae ... 0xaf: /* scas */ DPRINTF("Urk! I don't handle SCAS.\n"); goto cannot_emulate; + case 0xc0 ... 0xc1: + emulate_grp2(ctxt); + break; case 0xc3: /* ret */ c->dst.ptr = &c->eip; goto pop_instruction; + case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ + mov: + c->dst.val = c->src.val; + break; + case 0xd0 ... 0xd1: /* Grp2 */ + c->src.val = 1; + emulate_grp2(ctxt); + break; + case 0xd2 ... 0xd3: /* Grp2 */ + c->src.val = c->regs[VCPU_REGS_RCX]; + emulate_grp2(ctxt); + break; case 0xe8: /* call (near) */ { long int rel; switch (c->op_bytes) { @@ -1662,6 +1632,11 @@ special_insn: ctxt->eflags ^= EFLG_CF; c->dst.type = OP_NONE; /* Disable writeback. */ break; + case 0xf6 ... 0xf7: /* Grp3 */ + rc = emulate_grp3(ctxt, ops); + if (rc != 0) + goto done; + break; case 0xf8: /* clc */ ctxt->eflags &= ~EFLG_CF; c->dst.type = OP_NONE; /* Disable writeback. */ @@ -1674,8 +1649,28 @@ special_insn: ctxt->eflags |= X86_EFLAGS_IF; c->dst.type = OP_NONE; /* Disable writeback. */ break; + case 0xfe ... 0xff: /* Grp4/Grp5 */ + rc = emulate_grp45(ctxt, ops); + if (rc != 0) + goto done; + break; } - goto writeback; + +writeback: + rc = writeback(ctxt, ops); + if (rc != 0) + goto done; + + /* Commit shadow register state. */ + memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); + ctxt->vcpu->rip = c->eip; + +done: + if (rc == X86EMUL_UNHANDLEABLE) { + c->eip = saved_eip; + return -1; + } + return 0; twobyte_insn: switch (c->b) { @@ -1737,6 +1732,23 @@ twobyte_insn: /* Disable writeback. */ c->dst.type = OP_NONE; break; + case 0x06: + emulate_clts(ctxt->vcpu); + c->dst.type = OP_NONE; + break; + case 0x08: /* invd */ + case 0x09: /* wbinvd */ + case 0x0d: /* GrpP (prefetch) */ + case 0x18: /* Grp16 (prefetch/nop) */ + c->dst.type = OP_NONE; + break; + case 0x20: /* mov cr, reg */ + if (c->modrm_mod != 3) + goto cannot_emulate; + c->regs[c->modrm_rm] = + realmode_get_cr(ctxt->vcpu, c->modrm_reg); + c->dst.type = OP_NONE; /* no writeback */ + break; case 0x21: /* mov from dr to reg */ if (c->modrm_mod != 3) goto cannot_emulate; @@ -1745,6 +1757,13 @@ twobyte_insn: goto cannot_emulate; c->dst.type = OP_NONE; /* no writeback */ break; + case 0x22: /* mov reg, cr */ + if (c->modrm_mod != 3) + goto cannot_emulate; + realmode_set_cr(ctxt->vcpu, + c->modrm_reg, c->modrm_val, &ctxt->eflags); + c->dst.type = OP_NONE; + break; case 0x23: /* mov from reg to dr */ if (c->modrm_mod != 3) goto cannot_emulate; @@ -1754,11 +1773,58 @@ twobyte_insn: goto cannot_emulate; c->dst.type = OP_NONE; /* no writeback */ break; + case 0x30: + /* wrmsr */ + msr_data = (u32)c->regs[VCPU_REGS_RAX] + | ((u64)c->regs[VCPU_REGS_RDX] << 32); + rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); + if (rc) { + kvm_x86_ops->inject_gp(ctxt->vcpu, 0); + c->eip = ctxt->vcpu->rip; + } + rc = X86EMUL_CONTINUE; + c->dst.type = OP_NONE; + break; + case 0x32: + /* rdmsr */ + rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); + if (rc) { + kvm_x86_ops->inject_gp(ctxt->vcpu, 0); + c->eip = ctxt->vcpu->rip; + } else { + c->regs[VCPU_REGS_RAX] = (u32)msr_data; + c->regs[VCPU_REGS_RDX] = msr_data >> 32; + } + rc = X86EMUL_CONTINUE; + c->dst.type = OP_NONE; + break; case 0x40 ... 0x4f: /* cmov */ c->dst.val = c->dst.orig_val = c->src.val; if (!test_cc(c->b, ctxt->eflags)) c->dst.type = OP_NONE; /* no writeback */ break; + case 0x80 ... 0x8f: /* jnz rel, etc*/ { + long int rel; + + switch (c->op_bytes) { + case 2: + rel = insn_fetch(s16, 2, c->eip); + break; + case 4: + rel = insn_fetch(s32, 4, c->eip); + break; + case 8: + rel = insn_fetch(s64, 8, c->eip); + break; + default: + DPRINTF("jnz: Invalid op_bytes\n"); + goto cannot_emulate; + } + if (test_cc(c->b, ctxt->eflags)) + JMP_REL(rel); + c->dst.type = OP_NONE; + break; + } case 0xa3: bt: /* bt */ c->dst.type = OP_NONE; @@ -1828,85 +1894,13 @@ twobyte_insn: c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val : (u64) c->src.val; break; - } - goto writeback; - -twobyte_special_insn: - switch (c->b) { - case 0x06: - emulate_clts(ctxt->vcpu); - break; - case 0x08: /* invd */ - break; - case 0x09: /* wbinvd */ - break; - case 0x0d: /* GrpP (prefetch) */ - case 0x18: /* Grp16 (prefetch/nop) */ - break; - case 0x20: /* mov cr, reg */ - if (c->modrm_mod != 3) - goto cannot_emulate; - c->regs[c->modrm_rm] = - realmode_get_cr(ctxt->vcpu, c->modrm_reg); - break; - case 0x22: /* mov reg, cr */ - if (c->modrm_mod != 3) - goto cannot_emulate; - realmode_set_cr(ctxt->vcpu, - c->modrm_reg, c->modrm_val, &ctxt->eflags); - break; - case 0x30: - /* wrmsr */ - msr_data = (u32)c->regs[VCPU_REGS_RAX] - | ((u64)c->regs[VCPU_REGS_RDX] << 32); - rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); - if (rc) { - kvm_x86_ops->inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->rip; - } - rc = X86EMUL_CONTINUE; - break; - case 0x32: - /* rdmsr */ - rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); - if (rc) { - kvm_x86_ops->inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->rip; - } else { - c->regs[VCPU_REGS_RAX] = (u32)msr_data; - c->regs[VCPU_REGS_RDX] = msr_data >> 32; - } - rc = X86EMUL_CONTINUE; - break; - case 0x80 ... 0x8f: /* jnz rel, etc*/ { - long int rel; - - switch (c->op_bytes) { - case 2: - rel = insn_fetch(s16, 2, c->eip); - break; - case 4: - rel = insn_fetch(s32, 4, c->eip); - break; - case 8: - rel = insn_fetch(s64, 8, c->eip); - break; - default: - DPRINTF("jnz: Invalid op_bytes\n"); - goto cannot_emulate; - } - if (test_cc(c->b, ctxt->eflags)) - JMP_REL(rel); - break; - } case 0xc7: /* Grp9 (cmpxchg8b) */ rc = emulate_grp9(ctxt, ops, memop); if (rc != 0) goto done; + c->dst.type = OP_NONE; break; } - /* Disable writeback. */ - c->dst.type = OP_NONE; goto writeback; cannot_emulate: -- cgit v1.2.3 From f77bc6a420eba845605ff1d53cadf55f94c5e8b7 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Wed, 21 Nov 2007 04:36:41 +0800 Subject: KVM: Portability: Move KVM_INTERRUPT vcpu ioctl to x86.c Other archs doesn't need it. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 29 ----------------------------- drivers/kvm/x86.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 29 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 721e660023f6..c74fb44dd87b 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -665,23 +665,6 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, - struct kvm_interrupt *irq) -{ - if (irq->irq < 0 || irq->irq >= 256) - return -EINVAL; - if (irqchip_in_kernel(vcpu->kvm)) - return -ENXIO; - vcpu_load(vcpu); - - set_bit(irq->irq, vcpu->irq_pending); - set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary); - - vcpu_put(vcpu); - - return 0; -} - static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma, unsigned long address, int *type) @@ -883,18 +866,6 @@ static long kvm_vcpu_ioctl(struct file *filp, r = 0; break; } - case KVM_INTERRUPT: { - struct kvm_interrupt irq; - - r = -EFAULT; - if (copy_from_user(&irq, argp, sizeof irq)) - goto out; - r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); - if (r) - goto out; - r = 0; - break; - } case KVM_DEBUG_GUEST: { struct kvm_debug_guest dbg; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 12f1d6fc92bb..c9e4b67bfb1b 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1001,6 +1001,23 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, return 0; } +static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_interrupt *irq) +{ + if (irq->irq < 0 || irq->irq >= 256) + return -EINVAL; + if (irqchip_in_kernel(vcpu->kvm)) + return -ENXIO; + vcpu_load(vcpu); + + set_bit(irq->irq, vcpu->irq_pending); + set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary); + + vcpu_put(vcpu); + + return 0; +} + long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1034,6 +1051,18 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = 0; break; } + case KVM_INTERRUPT: { + struct kvm_interrupt irq; + + r = -EFAULT; + if (copy_from_user(&irq, argp, sizeof irq)) + goto out; + r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); + if (r) + goto out; + r = 0; + break; + } case KVM_SET_CPUID: { struct kvm_cpuid __user *cpuid_arg = argp; struct kvm_cpuid cpuid; -- cgit v1.2.3 From d23087847184a7417fc69bdfaa8a32834b447bef Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Thu, 29 Nov 2007 15:35:39 +0800 Subject: KVM: Correct kvm_init() error paths not freeing bad_pge. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index c74fb44dd87b..058366f480dc 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -1311,7 +1311,7 @@ int kvm_init(void *opaque, unsigned int vcpu_size, r = kvm_arch_init(opaque); if (r) - goto out4; + goto out_fail; bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); @@ -1322,29 +1322,29 @@ int kvm_init(void *opaque, unsigned int vcpu_size, r = kvm_arch_hardware_setup(); if (r < 0) - goto out; + goto out_free_0; for_each_online_cpu(cpu) { smp_call_function_single(cpu, kvm_arch_check_processor_compat, &r, 0, 1); if (r < 0) - goto out_free_0; + goto out_free_1; } on_each_cpu(hardware_enable, NULL, 0, 1); r = register_cpu_notifier(&kvm_cpu_notifier); if (r) - goto out_free_1; + goto out_free_2; register_reboot_notifier(&kvm_reboot_notifier); r = sysdev_class_register(&kvm_sysdev_class); if (r) - goto out_free_2; + goto out_free_3; r = sysdev_register(&kvm_sysdev); if (r) - goto out_free_3; + goto out_free_4; /* A kmem cache lets us meet the alignment requirements of fx_save. */ kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, @@ -1352,7 +1352,7 @@ int kvm_init(void *opaque, unsigned int vcpu_size, 0, NULL); if (!kvm_vcpu_cache) { r = -ENOMEM; - goto out_free_4; + goto out_free_5; } kvm_chardev_ops.owner = module; @@ -1370,21 +1370,23 @@ int kvm_init(void *opaque, unsigned int vcpu_size, out_free: kmem_cache_destroy(kvm_vcpu_cache); -out_free_4: +out_free_5: sysdev_unregister(&kvm_sysdev); -out_free_3: +out_free_4: sysdev_class_unregister(&kvm_sysdev_class); -out_free_2: +out_free_3: unregister_reboot_notifier(&kvm_reboot_notifier); unregister_cpu_notifier(&kvm_cpu_notifier); -out_free_1: +out_free_2: on_each_cpu(hardware_disable, NULL, 0, 1); -out_free_0: +out_free_1: kvm_arch_hardware_unsetup(); +out_free_0: + __free_page(bad_page); out: kvm_arch_exit(); kvm_exit_debug(); -out4: +out_fail: return r; } EXPORT_SYMBOL_GPL(kvm_init); -- cgit v1.2.3 From fb56dbb31c4738a3918db81fd24da732ce3b4ae6 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 2 Dec 2007 10:50:06 +0200 Subject: KVM: Export include/linux/kvm.h only if $ARCH actually supports KVM Currently, make headers_check barfs due to , which includes, not existing. Rather than add a zillion s, export kvm.h only if the arch actually supports it. Signed-off-by: Avi Kivity --- arch/x86/Kconfig | 1 + drivers/kvm/Kconfig | 7 +++++-- include/linux/Kbuild | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index fb3eea3e38ee..d289cfcf92c4 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -107,6 +107,7 @@ config ARCH_SUPPORTS_OPROFILE bool default y +select HAVE_KVM config ZONE_DMA32 bool diff --git a/drivers/kvm/Kconfig b/drivers/kvm/Kconfig index 656920636cb2..c83e1c9b5129 100644 --- a/drivers/kvm/Kconfig +++ b/drivers/kvm/Kconfig @@ -1,9 +1,12 @@ # # KVM configuration # +config HAVE_KVM + bool + menuconfig VIRTUALIZATION bool "Virtualization" - depends on X86 + depends on HAVE_KVM || X86 default y ---help--- Say Y here to get to see options for using your Linux host to run other @@ -16,7 +19,7 @@ if VIRTUALIZATION config KVM tristate "Kernel-based Virtual Machine (KVM) support" - depends on X86 && EXPERIMENTAL + depends on HAVE_KVM && EXPERIMENTAL select PREEMPT_NOTIFIERS select ANON_INODES ---help--- diff --git a/include/linux/Kbuild b/include/linux/Kbuild index c263a69b5680..85b2482cc736 100644 --- a/include/linux/Kbuild +++ b/include/linux/Kbuild @@ -255,7 +255,7 @@ unifdef-y += kd.h unifdef-y += kernelcapi.h unifdef-y += kernel.h unifdef-y += keyboard.h -unifdef-y += kvm.h +unifdef-$(CONFIG_HAVE_KVM) += kvm.h unifdef-y += llc.h unifdef-y += loop.h unifdef-y += lp.h -- cgit v1.2.3 From 2b5203ee6828e9cabab1e1fb42e53d72ab9351a6 Mon Sep 17 00:00:00 2001 From: Carlo Marcelo Arenas Belon Date: Sat, 1 Dec 2007 06:17:11 -0600 Subject: KVM: SVM: Remove KVM specific defines for MSR_EFER This patch removes the KVM specific defines for MSR_EFER that were being used in the svm support file and migrates all references to use instead the ones from the kernel headers that are used everywhere else and that have the same values. Signed-off-by: Carlo Marcelo Arenas Belon Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index fb3721d88dbf..c75c6b65b651 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -42,9 +42,6 @@ MODULE_LICENSE("GPL"); #define SEG_TYPE_LDT 2 #define SEG_TYPE_BUSY_TSS16 3 -#define KVM_EFER_LMA (1 << 10) -#define KVM_EFER_LME (1 << 8) - #define SVM_FEATURE_NPT (1 << 0) #define SVM_FEATURE_LBRV (1 << 1) #define SVM_DEATURE_SVML (1 << 2) @@ -184,8 +181,8 @@ static inline void flush_guest_tlb(struct kvm_vcpu *vcpu) static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) { - if (!(efer & KVM_EFER_LMA)) - efer &= ~KVM_EFER_LME; + if (!(efer & EFER_LMA)) + efer &= ~EFER_LME; to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK; vcpu->shadow_efer = efer; @@ -777,15 +774,15 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) struct vcpu_svm *svm = to_svm(vcpu); #ifdef CONFIG_X86_64 - if (vcpu->shadow_efer & KVM_EFER_LME) { + if (vcpu->shadow_efer & EFER_LME) { if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { - vcpu->shadow_efer |= KVM_EFER_LMA; - svm->vmcb->save.efer |= KVM_EFER_LMA | KVM_EFER_LME; + vcpu->shadow_efer |= EFER_LMA; + svm->vmcb->save.efer |= EFER_LMA | EFER_LME; } if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { - vcpu->shadow_efer &= ~KVM_EFER_LMA; - svm->vmcb->save.efer &= ~(KVM_EFER_LMA | KVM_EFER_LME); + vcpu->shadow_efer &= ~EFER_LMA; + svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); } } #endif -- cgit v1.2.3 From 8be5453f95933340a42c6e7fc7b66f3bb786fddd Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 2 Dec 2007 22:35:57 +0800 Subject: KVM: Replace kvm_lapic with kvm_vcpu in ioapic/lapic interface This patch replaces lapic structure with kvm_vcpu in ioapic.c, making ioapic independent of the local apic, as required by ia64. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 18 ++++++++---------- drivers/kvm/irq.h | 5 +++-- drivers/kvm/lapic.c | 23 ++++++++++++++++++----- 3 files changed, 29 insertions(+), 17 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index cf1d50b3549b..3629867a76bc 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -136,7 +136,7 @@ static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val) } static void ioapic_inj_irq(struct kvm_ioapic *ioapic, - struct kvm_lapic *target, + struct kvm_vcpu *vcpu, u8 vector, u8 trig_mode, u8 delivery_mode) { ioapic_debug("irq %d trig %d deliv %d\n", vector, trig_mode, @@ -145,7 +145,7 @@ static void ioapic_inj_irq(struct kvm_ioapic *ioapic, ASSERT((delivery_mode == dest_Fixed) || (delivery_mode == dest_LowestPrio)); - kvm_apic_set_irq(target, vector, trig_mode); + kvm_apic_set_irq(vcpu, vector, trig_mode); } static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, @@ -196,7 +196,6 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) u8 vector = ioapic->redirtbl[irq].fields.vector; u8 trig_mode = ioapic->redirtbl[irq].fields.trig_mode; u32 deliver_bitmask; - struct kvm_lapic *target; struct kvm_vcpu *vcpu; int vcpu_id; @@ -212,13 +211,13 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) switch (delivery_mode) { case dest_LowestPrio: - target = - kvm_apic_round_robin(ioapic->kvm, vector, deliver_bitmask); - if (target != NULL) - ioapic_inj_irq(ioapic, target, vector, + vcpu = kvm_get_lowest_prio_vcpu(ioapic->kvm, vector, + deliver_bitmask); + if (vcpu != NULL) + ioapic_inj_irq(ioapic, vcpu, vector, trig_mode, delivery_mode); else - ioapic_debug("null round robin: " + ioapic_debug("null lowest prio vcpu: " "mask=%x vector=%x delivery_mode=%x\n", deliver_bitmask, vector, dest_LowestPrio); break; @@ -229,8 +228,7 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) deliver_bitmask &= ~(1 << vcpu_id); vcpu = ioapic->kvm->vcpus[vcpu_id]; if (vcpu) { - target = vcpu->apic; - ioapic_inj_irq(ioapic, target, vector, + ioapic_inj_irq(ioapic, vcpu, vector, trig_mode, delivery_mode); } } diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 5ad3cfd7622d..78a34dde1cb9 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -145,14 +145,15 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); -struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, + +struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, unsigned long bitmap); u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest); void kvm_ioapic_update_eoi(struct kvm *kvm, int vector); int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda); -int kvm_apic_set_irq(struct kvm_lapic *apic, u8 vec, u8 trig); +int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig); void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu); int kvm_ioapic_init(struct kvm *kvm); void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level); diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 64f74bd7093a..5efa6c0276c6 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -185,8 +185,10 @@ int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); -int kvm_apic_set_irq(struct kvm_lapic *apic, u8 vec, u8 trig) +int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig) { + struct kvm_lapic *apic = vcpu->apic; + if (!apic_test_and_set_irr(vec, apic)) { /* a new pending irq is set in IRR */ if (trig) @@ -394,7 +396,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, return result; } -struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, +static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, unsigned long bitmap) { int last; @@ -422,6 +424,17 @@ struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, return apic; } +struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, + unsigned long bitmap) +{ + struct kvm_lapic *apic; + + apic = kvm_apic_round_robin(kvm, vector, bitmap); + if (apic) + return apic->vcpu; + return NULL; +} + static void apic_set_eoi(struct kvm_lapic *apic) { int vector = apic_find_highest_isr(apic); @@ -453,7 +466,7 @@ static void apic_send_ipi(struct kvm_lapic *apic) unsigned int delivery_mode = icr_low & APIC_MODE_MASK; unsigned int vector = icr_low & APIC_VECTOR_MASK; - struct kvm_lapic *target; + struct kvm_vcpu *target; struct kvm_vcpu *vcpu; unsigned long lpr_map = 0; int i; @@ -480,9 +493,9 @@ static void apic_send_ipi(struct kvm_lapic *apic) } if (delivery_mode == APIC_DM_LOWEST) { - target = kvm_apic_round_robin(vcpu->kvm, vector, lpr_map); + target = kvm_get_lowest_prio_vcpu(vcpu->kvm, vector, lpr_map); if (target != NULL) - __apic_accept_irq(target, delivery_mode, + __apic_accept_irq(target->apic, delivery_mode, vector, level, trig_mode); } } -- cgit v1.2.3 From 0c7ac28d3dd27d718493aae4bbc7de89a2d9d386 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 2 Dec 2007 22:49:09 +0800 Subject: KVM: Replace dest_Lowest_Prio and dest_Fixed with self-defined macros Change dest_Loest_Prio -> IOAPIC_LOWEST_PRIORITY dest_Fixed -> IOAPIC_FIXED the original names are x86 specific, while the ioapic code will be reused for ia64. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 13 +++++-------- drivers/kvm/irq.h | 8 ++++++++ 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index 3629867a76bc..0feae6f7fb2d 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -36,11 +36,8 @@ #include #include #include -#include #include #include -#include -#include #include "irq.h" #if 0 #define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) @@ -142,8 +139,8 @@ static void ioapic_inj_irq(struct kvm_ioapic *ioapic, ioapic_debug("irq %d trig %d deliv %d\n", vector, trig_mode, delivery_mode); - ASSERT((delivery_mode == dest_Fixed) || - (delivery_mode == dest_LowestPrio)); + ASSERT((delivery_mode == IOAPIC_FIXED) || + (delivery_mode == IOAPIC_LOWEST_PRIORITY)); kvm_apic_set_irq(vcpu, vector, trig_mode); } @@ -210,7 +207,7 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) } switch (delivery_mode) { - case dest_LowestPrio: + case IOAPIC_LOWEST_PRIORITY: vcpu = kvm_get_lowest_prio_vcpu(ioapic->kvm, vector, deliver_bitmask); if (vcpu != NULL) @@ -219,9 +216,9 @@ static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) else ioapic_debug("null lowest prio vcpu: " "mask=%x vector=%x delivery_mode=%x\n", - deliver_bitmask, vector, dest_LowestPrio); + deliver_bitmask, vector, IOAPIC_LOWEST_PRIORITY); break; - case dest_Fixed: + case IOAPIC_FIXED: for (vcpu_id = 0; deliver_bitmask != 0; vcpu_id++) { if (!(deliver_bitmask & (1 << vcpu_id))) continue; diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 78a34dde1cb9..75f5f18d801c 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -79,6 +79,14 @@ void kvm_pic_update_irq(struct kvm_pic *s); #define IOAPIC_REG_VERSION 0x01 #define IOAPIC_REG_ARB_ID 0x02 /* x86 IOAPIC only */ +/*ioapic delivery mode*/ +#define IOAPIC_FIXED 0x0 +#define IOAPIC_LOWEST_PRIORITY 0x1 +#define IOAPIC_PMI 0x2 +#define IOAPIC_NMI 0x4 +#define IOAPIC_INIT 0x5 +#define IOAPIC_EXTINT 0x7 + struct kvm_ioapic { u64 base_address; u32 ioregsel; -- cgit v1.2.3 From b1fd3d30ba46360fc94d40ee8dc1f7a4338b19be Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Sun, 2 Dec 2007 22:53:07 +0800 Subject: KVM: Extend ioapic code to support iosapic iosapic supports an additional mmio EOI register compared to ioapic. Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index 0feae6f7fb2d..e7debfafca50 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -360,6 +360,11 @@ static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, case IOAPIC_REG_WINDOW: ioapic_write_indirect(ioapic, data); break; +#ifdef CONFIG_IA64 + case IOAPIC_REG_EOI: + kvm_ioapic_update_eoi(ioapic, data); + break; +#endif default: break; -- cgit v1.2.3 From d77a39d982431ef9efc7e7a1690cee2ce252b95e Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 3 Dec 2007 15:30:23 -0600 Subject: KVM: Portability: Move address types to their own header file Signed-off-by: Hollis Blanchard Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 21 ++------------------- drivers/kvm/types.h | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 drivers/kvm/types.h (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index be18620bd656..3b0ba5ab7104 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -20,6 +20,8 @@ #include #include +#include "types.h" + #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 @@ -39,25 +41,6 @@ */ #define KVM_REQ_TLB_FLUSH 0 -/* - * Address types: - * - * gva - guest virtual address - * gpa - guest physical address - * gfn - guest frame number - * hva - host virtual address - * hpa - host physical address - * hfn - host frame number - */ - -typedef unsigned long gva_t; -typedef u64 gpa_t; -typedef unsigned long gfn_t; - -typedef unsigned long hva_t; -typedef u64 hpa_t; -typedef unsigned long hfn_t; - #define NR_PTE_CHAIN_ENTRIES 5 struct kvm_pte_chain { diff --git a/drivers/kvm/types.h b/drivers/kvm/types.h new file mode 100644 index 000000000000..6ad76231b8ff --- /dev/null +++ b/drivers/kvm/types.h @@ -0,0 +1,41 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +#ifndef __KVM_TYPES_H__ +#define __KVM_TYPES_H__ + +#include + +/* + * Address types: + * + * gva - guest virtual address + * gpa - guest physical address + * gfn - guest frame number + * hva - host virtual address + * hpa - host physical address + * hfn - host frame number + */ + +typedef unsigned long gva_t; +typedef u64 gpa_t; +typedef unsigned long gfn_t; + +typedef unsigned long hva_t; +typedef u64 hpa_t; +typedef unsigned long hfn_t; + +#endif /* __KVM_TYPES_H__ */ -- cgit v1.2.3 From e2174021cfa535dbcaef02dc6f2897019c30731d Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 3 Dec 2007 15:30:24 -0600 Subject: KVM: Portability: Move IO device definitions to its own header file Signed-off-by: Hollis Blanchard Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/iodev.h | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/irq.h | 1 + drivers/kvm/kvm.h | 42 --------------------------------- drivers/kvm/kvm_main.c | 1 + 4 files changed, 65 insertions(+), 42 deletions(-) create mode 100644 drivers/kvm/iodev.h (limited to 'drivers') diff --git a/drivers/kvm/iodev.h b/drivers/kvm/iodev.h new file mode 100644 index 000000000000..eb9e8a71843a --- /dev/null +++ b/drivers/kvm/iodev.h @@ -0,0 +1,63 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __KVM_IODEV_H__ +#define __KVM_IODEV_H__ + +#include "types.h" + +struct kvm_io_device { + void (*read)(struct kvm_io_device *this, + gpa_t addr, + int len, + void *val); + void (*write)(struct kvm_io_device *this, + gpa_t addr, + int len, + const void *val); + int (*in_range)(struct kvm_io_device *this, gpa_t addr); + void (*destructor)(struct kvm_io_device *this); + + void *private; +}; + +static inline void kvm_iodevice_read(struct kvm_io_device *dev, + gpa_t addr, + int len, + void *val) +{ + dev->read(dev, addr, len, val); +} + +static inline void kvm_iodevice_write(struct kvm_io_device *dev, + gpa_t addr, + int len, + const void *val) +{ + dev->write(dev, addr, len, val); +} + +static inline int kvm_iodevice_inrange(struct kvm_io_device *dev, gpa_t addr) +{ + return dev->in_range(dev, addr); +} + +static inline void kvm_iodevice_destructor(struct kvm_io_device *dev) +{ + if (dev->destructor) + dev->destructor(dev); +} + +#endif /* __KVM_IODEV_H__ */ diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 75f5f18d801c..803b9c7db57b 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -23,6 +23,7 @@ #define __IRQ_H #include "kvm.h" +#include "iodev.h" typedef void irq_request_func(void *opaque, int level); diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 3b0ba5ab7104..a1b7d1c00bde 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -168,48 +168,6 @@ struct kvm_vcpu_stat { u32 insn_emulation_fail; }; -struct kvm_io_device { - void (*read)(struct kvm_io_device *this, - gpa_t addr, - int len, - void *val); - void (*write)(struct kvm_io_device *this, - gpa_t addr, - int len, - const void *val); - int (*in_range)(struct kvm_io_device *this, gpa_t addr); - void (*destructor)(struct kvm_io_device *this); - - void *private; -}; - -static inline void kvm_iodevice_read(struct kvm_io_device *dev, - gpa_t addr, - int len, - void *val) -{ - dev->read(dev, addr, len, val); -} - -static inline void kvm_iodevice_write(struct kvm_io_device *dev, - gpa_t addr, - int len, - const void *val) -{ - dev->write(dev, addr, len, val); -} - -static inline int kvm_iodevice_inrange(struct kvm_io_device *dev, gpa_t addr) -{ - return dev->in_range(dev, addr); -} - -static inline void kvm_iodevice_destructor(struct kvm_io_device *dev) -{ - if (dev->destructor) - dev->destructor(dev); -} - /* * It would be nice to use something smarter than a linear search, TBD... * Thankfully we dont expect many devices to register (famous last words :), diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 058366f480dc..14b376ead3da 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -18,6 +18,7 @@ #include "kvm.h" #include "x86.h" #include "irq.h" +#include "iodev.h" #include #include -- cgit v1.2.3 From e01a1b570f66ad318239517adbcc2cbe368d0a46 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 3 Dec 2007 15:30:25 -0600 Subject: KVM: Portability: Stop including x86-specific headers in kvm_main.c Signed-off-by: Hollis Blanchard Signed-off-by: Avi Kivity --- drivers/kvm/i8259.c | 1 + drivers/kvm/irq.h | 7 ++++++- drivers/kvm/kvm.h | 9 ++++----- drivers/kvm/kvm_main.c | 3 --- drivers/kvm/x86.h | 8 ++++++++ 5 files changed, 19 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/i8259.c b/drivers/kvm/i8259.c index f0dc2eeadf0b..b3cad632f3d5 100644 --- a/drivers/kvm/i8259.c +++ b/drivers/kvm/i8259.c @@ -27,6 +27,7 @@ */ #include #include "irq.h" +#include "kvm.h" /* * set irq level. If an edge is detected, then the IRR is set to 1 diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 803b9c7db57b..730a87c173e5 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -22,9 +22,14 @@ #ifndef __IRQ_H #define __IRQ_H -#include "kvm.h" +#include +#include +#include #include "iodev.h" +struct kvm; +struct kvm_vcpu; + typedef void irq_request_func(void *opaque, int level); struct kvm_kpic_state { diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index a1b7d1c00bde..0d3555bf5333 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -288,11 +288,6 @@ static inline int irqchip_in_kernel(struct kvm *kvm) return pic_irqchip(kvm) != NULL; } -struct descriptor_table { - u16 limit; - unsigned long base; -} __attribute__((packed)); - /* The guest did something we don't support. */ #define pr_unimpl(vcpu, fmt, ...) \ do { \ @@ -457,4 +452,8 @@ struct kvm_stats_debugfs_item { }; extern struct kvm_stats_debugfs_item debugfs_entries[]; +#if defined(CONFIG_X86) +#include "x86.h" +#endif + #endif diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 14b376ead3da..0f11046b94fa 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -16,8 +16,6 @@ */ #include "kvm.h" -#include "x86.h" -#include "irq.h" #include "iodev.h" #include @@ -46,7 +44,6 @@ #include #include #include -#include #include MODULE_AUTHOR("Qumranet"); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 78ab1e108d8b..4b7acc76bb34 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -12,6 +12,7 @@ #define KVM_X86_H #include "kvm.h" +#include "irq.h" #include #include @@ -19,6 +20,8 @@ #include #include +#include + #define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) #define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) #define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) @@ -156,6 +159,11 @@ struct kvm_vcpu { struct x86_emulate_ctxt emulate_ctxt; }; +struct descriptor_table { + u16 limit; + unsigned long base; +} __attribute__((packed)); + struct kvm_x86_ops { int (*cpu_has_kvm_support)(void); /* __init */ int (*disabled_by_bios)(void); /* __init */ -- cgit v1.2.3 From 53e0aa7b65ec0d66e34e4d356a641c725cc2e519 Mon Sep 17 00:00:00 2001 From: Hollis Blanchard Date: Mon, 3 Dec 2007 16:15:26 -0600 Subject: KVM: Portability: Create kvm_arch_vcpu_runnable() function This abstracts the detail of x86 hlt and INIT modes into a function. Signed-off-by: Hollis Blanchard Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 3 +-- drivers/kvm/x86.h | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 0f11046b94fa..70664f020849 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -643,8 +643,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) */ while (!kvm_cpu_has_interrupt(vcpu) && !signal_pending(current) - && vcpu->mp_state != VCPU_MP_STATE_RUNNABLE - && vcpu->mp_state != VCPU_MP_STATE_SIPI_RECEIVED) { + && !kvm_arch_vcpu_runnable(vcpu)) { set_current_state(TASK_INTERRUPTIBLE); vcpu_put(vcpu); schedule(); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 4b7acc76bb34..eed796402e3b 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -471,4 +471,11 @@ static inline u32 get_rdx_init_val(void) #define TSS_IOPB_SIZE (65536 / 8) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) + +static inline int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return vcpu->mp_state == VCPU_MP_STATE_RUNNABLE + || vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED; +} + #endif -- cgit v1.2.3 From e4a533a416936753882060bf743222576713fde8 Mon Sep 17 00:00:00 2001 From: "npiggin@suse.de" Date: Wed, 5 Dec 2007 18:15:52 +1100 Subject: KVM: Convert KVM from ->nopage() to ->fault() Signed-off-by: Nick Piggin Cc: kvm-devel@lists.sourceforge.net Cc: avi@qumranet.com Cc: linux-kernel@vger.kernel.org Signed-off-by: Avi Kivity --- drivers/kvm/kvm_main.c | 42 +++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 70664f020849..1a03b67eb921 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -662,30 +662,24 @@ void kvm_resched(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_resched); -static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma, - unsigned long address, - int *type) +static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct kvm_vcpu *vcpu = vma->vm_file->private_data; - unsigned long pgoff; struct page *page; - pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; - if (pgoff == 0) + if (vmf->pgoff == 0) page = virt_to_page(vcpu->run); - else if (pgoff == KVM_PIO_PAGE_OFFSET) + else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) page = virt_to_page(vcpu->pio_data); else - return NOPAGE_SIGBUS; + return VM_FAULT_SIGBUS; get_page(page); - if (type != NULL) - *type = VM_FAULT_MINOR; - - return page; + vmf->page = page; + return 0; } static struct vm_operations_struct kvm_vcpu_vm_ops = { - .nopage = kvm_vcpu_nopage, + .fault = kvm_vcpu_fault, }; static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) @@ -976,31 +970,25 @@ out: return r; } -static struct page *kvm_vm_nopage(struct vm_area_struct *vma, - unsigned long address, - int *type) +static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct kvm *kvm = vma->vm_file->private_data; - unsigned long pgoff; struct page *page; - pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; - if (!kvm_is_visible_gfn(kvm, pgoff)) - return NOPAGE_SIGBUS; + if (!kvm_is_visible_gfn(kvm, vmf->pgoff)) + return VM_FAULT_SIGBUS; /* current->mm->mmap_sem is already held so call lockless version */ - page = __gfn_to_page(kvm, pgoff); + page = __gfn_to_page(kvm, vmf->pgoff); if (is_error_page(page)) { kvm_release_page_clean(page); - return NOPAGE_SIGBUS; + return VM_FAULT_SIGBUS; } - if (type != NULL) - *type = VM_FAULT_MINOR; - - return page; + vmf->page = page; + return 0; } static struct vm_operations_struct kvm_vm_vm_ops = { - .nopage = kvm_vm_nopage, + .fault = kvm_vm_fault, }; static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma) -- cgit v1.2.3 From 4bf8ed8dd2781a5e7603a83f8ee1d4f5aa04ebc4 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Tue, 4 Dec 2007 13:42:16 -0500 Subject: KVM: MMU: Remove unused prev_shadow_ent variable from fetch() Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index b24bc7c86078..3c40a598ceb8 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -300,7 +300,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, hpa_t shadow_addr; int level; u64 *shadow_ent; - u64 *prev_shadow_ent = NULL; if (!is_present_pte(walker->pte)) return NULL; @@ -326,7 +325,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (level == PT_PAGE_TABLE_LEVEL) break; shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK; - prev_shadow_ent = shadow_ent; continue; } @@ -355,7 +353,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; *shadow_ent = shadow_pte; - prev_shadow_ent = shadow_ent; } FNAME(set_pte)(vcpu, walker->pte, shadow_ent, -- cgit v1.2.3 From 298101da2f507c13eaf179ee4507a7c0fe3e7b06 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 13:41:11 +0200 Subject: KVM: Generalize exception injection mechanism Instead of each subarch doing its own thing, add an API for queuing an injection, and manage failed exception injection centerally (i.e., if an inject failed due to a shadow page fault, we need to requeue it). Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 21 +++++++++++++++++++++ drivers/kvm/vmx.c | 20 ++++++++++++++++++++ drivers/kvm/x86.c | 33 ++++++++++++++++++++++++++++++++- drivers/kvm/x86.h | 13 +++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index c75c6b65b651..87072c647f28 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -188,6 +188,25 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) vcpu->shadow_efer = efer; } +static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm->vmcb->control.event_inj = nr + | SVM_EVTINJ_VALID + | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) + | SVM_EVTINJ_TYPE_EXEPT; + svm->vmcb->control.event_inj_err = error_code; +} + +static bool svm_exception_injected(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID); +} + static void svm_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) { struct vcpu_svm *svm = to_svm(vcpu); @@ -1712,6 +1731,8 @@ static struct kvm_x86_ops svm_x86_ops = { .patch_hypercall = svm_patch_hypercall, .get_irq = svm_get_irq, .set_irq = svm_set_irq, + .queue_exception = svm_queue_exception, + .exception_injected = svm_exception_injected, .inject_pending_irq = svm_intr_assist, .inject_pending_vectors = do_interrupt_requests, diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index fc5e7c8381ce..f382956f176a 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -595,6 +595,24 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) vcpu->interrupt_window_open = 1; } +static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code) +{ + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + nr | INTR_TYPE_EXCEPTION + | (has_error_code ? INTR_INFO_DELIEVER_CODE_MASK : 0) + | INTR_INFO_VALID_MASK); + if (has_error_code) + vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); +} + +static bool vmx_exception_injected(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + return !(vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); +} + static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) { printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n", @@ -2641,6 +2659,8 @@ static struct kvm_x86_ops vmx_x86_ops = { .patch_hypercall = vmx_patch_hypercall, .get_irq = vmx_get_irq, .set_irq = vmx_inject_irq, + .queue_exception = vmx_queue_exception, + .exception_injected = vmx_exception_injected, .inject_pending_irq = vmx_intr_assist, .inject_pending_vectors = do_interrupt_requests, diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c9e4b67bfb1b..11440d12a2d3 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -133,6 +133,32 @@ static void inject_gp(struct kvm_vcpu *vcpu) kvm_x86_ops->inject_gp(vcpu, 0); } +void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) +{ + WARN_ON(vcpu->exception.pending); + vcpu->exception.pending = true; + vcpu->exception.has_error_code = false; + vcpu->exception.nr = nr; +} +EXPORT_SYMBOL_GPL(kvm_queue_exception); + +void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) +{ + WARN_ON(vcpu->exception.pending); + vcpu->exception.pending = true; + vcpu->exception.has_error_code = true; + vcpu->exception.nr = nr; + vcpu->exception.error_code = error_code; +} +EXPORT_SYMBOL_GPL(kvm_queue_exception_e); + +static void __queue_exception(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->queue_exception(vcpu, vcpu->exception.nr, + vcpu->exception.has_error_code, + vcpu->exception.error_code); +} + /* * Load the pae pdptrs. Return true is they are all valid. */ @@ -2370,7 +2396,9 @@ again: goto out; } - if (irqchip_in_kernel(vcpu->kvm)) + if (vcpu->exception.pending) + __queue_exception(vcpu); + else if (irqchip_in_kernel(vcpu->kvm)) kvm_x86_ops->inject_pending_irq(vcpu); else kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); @@ -2409,6 +2437,9 @@ again: profile_hit(KVM_PROFILING, (void *)vcpu->rip); } + if (vcpu->exception.pending && kvm_x86_ops->exception_injected(vcpu)) + vcpu->exception.pending = false; + r = kvm_x86_ops->handle_exit(kvm_run, vcpu); if (r > 0) { diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index eed796402e3b..1e71668694ea 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -139,6 +139,13 @@ struct kvm_vcpu { struct kvm_pio_request pio; void *pio_data; + struct kvm_queued_exception { + bool pending; + bool has_error_code; + u8 nr; + u32 error_code; + } exception; + struct { int active; u8 save_iopl; @@ -224,6 +231,9 @@ struct kvm_x86_ops { unsigned char *hypercall_addr); int (*get_irq)(struct kvm_vcpu *vcpu); void (*set_irq)(struct kvm_vcpu *vcpu, int vec); + void (*queue_exception)(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code); + bool (*exception_injected)(struct kvm_vcpu *vcpu); void (*inject_pending_irq)(struct kvm_vcpu *vcpu); void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, struct kvm_run *run); @@ -294,6 +304,9 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); +void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); +void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); + void fx_init(struct kvm_vcpu *vcpu); int emulator_read_std(unsigned long addr, -- cgit v1.2.3 From c3c91fee5195ba5176a6da5ddc2a2822243eb79f Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 14:04:58 +0200 Subject: KVM: Replace page fault injection by the generalized exception queue Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 2 +- drivers/kvm/svm.c | 35 ----------------------------------- drivers/kvm/vmx.c | 32 -------------------------------- drivers/kvm/x86.c | 17 ++++++++++++++++- drivers/kvm/x86.h | 4 ++-- 5 files changed, 19 insertions(+), 71 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 9b9d1b6eb2f8..62a74151f847 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -1081,7 +1081,7 @@ static void inject_page_fault(struct kvm_vcpu *vcpu, u64 addr, u32 err_code) { - kvm_x86_ops->inject_page_fault(vcpu, addr, err_code); + kvm_inject_page_fault(vcpu, addr, err_code); } static void paging_free(struct kvm_vcpu *vcpu) diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 87072c647f28..aa8e90b404a0 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -225,12 +225,6 @@ static void inject_ud(struct kvm_vcpu *vcpu) UD_VECTOR; } -static int is_page_fault(uint32_t info) -{ - info &= SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; - return info == (PF_VECTOR | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT); -} - static int is_external_interrupt(u32 info) { info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; @@ -1624,34 +1618,6 @@ static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) } } -static void svm_inject_page_fault(struct kvm_vcpu *vcpu, - unsigned long addr, - uint32_t err_code) -{ - struct vcpu_svm *svm = to_svm(vcpu); - uint32_t exit_int_info = svm->vmcb->control.exit_int_info; - - ++vcpu->stat.pf_guest; - - if (is_page_fault(exit_int_info)) { - - svm->vmcb->control.event_inj_err = 0; - svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | - SVM_EVTINJ_VALID_ERR | - SVM_EVTINJ_TYPE_EXEPT | - DF_VECTOR; - return; - } - vcpu->cr2 = addr; - svm->vmcb->save.cr2 = addr; - svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | - SVM_EVTINJ_VALID_ERR | - SVM_EVTINJ_TYPE_EXEPT | - PF_VECTOR; - svm->vmcb->control.event_inj_err = err_code; -} - - static int is_disabled(void) { u64 vm_cr; @@ -1721,7 +1687,6 @@ static struct kvm_x86_ops svm_x86_ops = { .set_rflags = svm_set_rflags, .tlb_flush = svm_flush_tlb, - .inject_page_fault = svm_inject_page_fault, .inject_gp = svm_inject_gp, diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index f382956f176a..be0b12e709e5 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2487,37 +2487,6 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) asm("int $2"); } -static void vmx_inject_page_fault(struct kvm_vcpu *vcpu, - unsigned long addr, - u32 err_code) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u32 vect_info = vmx->idt_vectoring_info; - - ++vcpu->stat.pf_guest; - - if (is_page_fault(vect_info)) { - printk(KERN_DEBUG "inject_page_fault: " - "double fault 0x%lx @ 0x%lx\n", - addr, vmcs_readl(GUEST_RIP)); - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 0); - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - DF_VECTOR | - INTR_TYPE_EXCEPTION | - INTR_INFO_DELIEVER_CODE_MASK | - INTR_INFO_VALID_MASK); - return; - } - vcpu->cr2 = addr; - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, err_code); - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - PF_VECTOR | - INTR_TYPE_EXCEPTION | - INTR_INFO_DELIEVER_CODE_MASK | - INTR_INFO_VALID_MASK); - -} - static void vmx_free_vmcs(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -2649,7 +2618,6 @@ static struct kvm_x86_ops vmx_x86_ops = { .set_rflags = vmx_set_rflags, .tlb_flush = vmx_flush_tlb, - .inject_page_fault = vmx_inject_page_fault, .inject_gp = vmx_inject_gp, diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 11440d12a2d3..dc007a32a883 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -142,6 +142,21 @@ void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) } EXPORT_SYMBOL_GPL(kvm_queue_exception); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, + u32 error_code) +{ + ++vcpu->stat.pf_guest; + if (vcpu->exception.pending && vcpu->exception.nr == PF_VECTOR) { + printk(KERN_DEBUG "kvm: inject_page_fault:" + " double fault 0x%lx\n", addr); + vcpu->exception.nr = DF_VECTOR; + vcpu->exception.error_code = 0; + return; + } + vcpu->cr2 = addr; + kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); +} + void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) { WARN_ON(vcpu->exception.pending); @@ -1601,7 +1616,7 @@ static int emulator_write_emulated_onepage(unsigned long addr, gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); if (gpa == UNMAPPED_GVA) { - kvm_x86_ops->inject_page_fault(vcpu, addr, 2); + kvm_inject_page_fault(vcpu, addr, 2); return X86EMUL_PROPAGATE_FAULT; } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 1e71668694ea..d3ac4e2b3a41 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -219,8 +219,6 @@ struct kvm_x86_ops { void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); void (*tlb_flush)(struct kvm_vcpu *vcpu); - void (*inject_page_fault)(struct kvm_vcpu *vcpu, - unsigned long addr, u32 err_code); void (*inject_gp)(struct kvm_vcpu *vcpu, unsigned err_code); @@ -306,6 +304,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2, + u32 error_code); void fx_init(struct kvm_vcpu *vcpu); -- cgit v1.2.3 From c1a5d4f990ce034bcb19aebbb910c07019e60f6b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 14:12:03 +0200 Subject: KVM: Replace #GP injection by the generalized exception queue Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 17 ++--------------- drivers/kvm/vmx.c | 18 ++---------------- drivers/kvm/x86.c | 43 +++++++++++++++++++------------------------ drivers/kvm/x86.h | 7 +++++-- drivers/kvm/x86_emulate.c | 4 ++-- 5 files changed, 30 insertions(+), 59 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index aa8e90b404a0..f9769338c621 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -207,17 +207,6 @@ static bool svm_exception_injected(struct kvm_vcpu *vcpu) return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID); } -static void svm_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | - SVM_EVTINJ_VALID_ERR | - SVM_EVTINJ_TYPE_EXEPT | - GP_VECTOR; - svm->vmcb->control.event_inj_err = error_code; -} - static void inject_ud(struct kvm_vcpu *vcpu) { to_svm(vcpu)->vmcb->control.event_inj = SVM_EVTINJ_VALID | @@ -1115,7 +1104,7 @@ static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) u64 data; if (svm_get_msr(&svm->vcpu, ecx, &data)) - svm_inject_gp(&svm->vcpu, 0); + kvm_inject_gp(&svm->vcpu, 0); else { svm->vmcb->save.rax = data & 0xffffffff; svm->vcpu.regs[VCPU_REGS_RDX] = data >> 32; @@ -1176,7 +1165,7 @@ static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ((u64)(svm->vcpu.regs[VCPU_REGS_RDX] & -1u) << 32); svm->next_rip = svm->vmcb->save.rip + 2; if (svm_set_msr(&svm->vcpu, ecx, data)) - svm_inject_gp(&svm->vcpu, 0); + kvm_inject_gp(&svm->vcpu, 0); else skip_emulated_instruction(&svm->vcpu); return 1; @@ -1688,8 +1677,6 @@ static struct kvm_x86_ops svm_x86_ops = { .tlb_flush = svm_flush_tlb, - .inject_gp = svm_inject_gp, - .run = svm_vcpu_run, .handle_exit = handle_exit, .skip_emulated_instruction = skip_emulated_instruction, diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index be0b12e709e5..3b3c5f7d2e7c 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -613,18 +613,6 @@ static bool vmx_exception_injected(struct kvm_vcpu *vcpu) return !(vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); } -static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code) -{ - printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n", - vmcs_readl(GUEST_RIP)); - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - GP_VECTOR | - INTR_TYPE_EXCEPTION | - INTR_INFO_DELIEVER_CODE_MASK | - INTR_INFO_VALID_MASK); -} - static void vmx_inject_ud(struct kvm_vcpu *vcpu) { vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, @@ -2083,7 +2071,7 @@ static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) u64 data; if (vmx_get_msr(vcpu, ecx, &data)) { - vmx_inject_gp(vcpu, 0); + kvm_inject_gp(vcpu, 0); return 1; } @@ -2101,7 +2089,7 @@ static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32); if (vmx_set_msr(vcpu, ecx, data) != 0) { - vmx_inject_gp(vcpu, 0); + kvm_inject_gp(vcpu, 0); return 1; } @@ -2619,8 +2607,6 @@ static struct kvm_x86_ops vmx_x86_ops = { .tlb_flush = vmx_flush_tlb, - .inject_gp = vmx_inject_gp, - .run = vmx_vcpu_run, .handle_exit = kvm_handle_exit, .skip_emulated_instruction = skip_emulated_instruction, diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index dc007a32a883..6deb052b5f93 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -128,11 +128,6 @@ void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) } EXPORT_SYMBOL_GPL(kvm_set_apic_base); -static void inject_gp(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->inject_gp(vcpu, 0); -} - void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) { WARN_ON(vcpu->exception.pending); @@ -232,20 +227,20 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (cr0 & CR0_RESERVED_BITS) { printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", cr0, vcpu->cr0); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) { printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) { printk(KERN_DEBUG "set_cr0: #GP, set PG flag " "and a clear PE flag\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -257,14 +252,14 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (!is_pae(vcpu)) { printk(KERN_DEBUG "set_cr0: #GP, start paging " "in long mode while PAE is disabled\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); if (cs_l) { printk(KERN_DEBUG "set_cr0: #GP, start paging " "in long mode while CS.L == 1\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -273,7 +268,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { printk(KERN_DEBUG "set_cr0: #GP, pdptrs " "reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -299,7 +294,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { if (cr4 & CR4_RESERVED_BITS) { printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -307,19 +302,19 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) if (!(cr4 & X86_CR4_PAE)) { printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while " "in long mode\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) && !load_pdptrs(vcpu, vcpu->cr3)) { printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if (cr4 & X86_CR4_VMXE) { printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } kvm_x86_ops->set_cr4(vcpu, cr4); @@ -340,7 +335,7 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (is_long_mode(vcpu)) { if (cr3 & CR3_L_MODE_RESERVED_BITS) { printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } } else { @@ -348,13 +343,13 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (cr3 & CR3_PAE_RESERVED_BITS) { printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) { printk(KERN_DEBUG "set_cr3: #GP, pdptrs " "reserved bits\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } } @@ -375,7 +370,7 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) * to debug) behavior on the guest side. */ if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); else { vcpu->cr3 = cr3; vcpu->mmu.new_cr3(vcpu); @@ -388,7 +383,7 @@ void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { if (cr8 & CR8_RESERVED_BITS) { printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if (irqchip_in_kernel(vcpu->kvm)) @@ -436,14 +431,14 @@ static void set_efer(struct kvm_vcpu *vcpu, u64 efer) if (efer & EFER_RESERVED_BITS) { printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n", efer); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } if (is_paging(vcpu) && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) { printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return; } @@ -2047,7 +2042,7 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, * String I/O in reverse. Yuck. Kill the guest, fix later. */ pr_unimpl(vcpu, "guest string pio down\n"); - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); return 1; } vcpu->run->io.count = now; @@ -2062,7 +2057,7 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, vcpu->pio.guest_pages[i] = page; mutex_unlock(&vcpu->kvm->lock); if (!page) { - inject_gp(vcpu); + kvm_inject_gp(vcpu, 0); free_pio_guest_pages(vcpu); return 1; } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index d3ac4e2b3a41..78396d627be0 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -220,8 +220,6 @@ struct kvm_x86_ops { void (*tlb_flush)(struct kvm_vcpu *vcpu); - void (*inject_gp)(struct kvm_vcpu *vcpu, unsigned err_code); - void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run); int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); @@ -467,6 +465,11 @@ static inline u32 get_rdx_init_val(void) return 0x600; /* P6 family */ } +static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) +{ + kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); +} + #define ASM_VMX_VMCLEAR_RAX ".byte 0x66, 0x0f, 0xc7, 0x30" #define ASM_VMX_VMLAUNCH ".byte 0x0f, 0x01, 0xc2" #define ASM_VMX_VMRESUME ".byte 0x0f, 0x01, 0xc3" diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 3e3eba70d5ac..2e259a847697 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -1779,7 +1779,7 @@ twobyte_insn: | ((u64)c->regs[VCPU_REGS_RDX] << 32); rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); if (rc) { - kvm_x86_ops->inject_gp(ctxt->vcpu, 0); + kvm_inject_gp(ctxt->vcpu, 0); c->eip = ctxt->vcpu->rip; } rc = X86EMUL_CONTINUE; @@ -1789,7 +1789,7 @@ twobyte_insn: /* rdmsr */ rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); if (rc) { - kvm_x86_ops->inject_gp(ctxt->vcpu, 0); + kvm_inject_gp(ctxt->vcpu, 0); c->eip = ctxt->vcpu->rip; } else { c->regs[VCPU_REGS_RAX] = (u32)msr_data; -- cgit v1.2.3 From 7ee5d940f5064a7a4f0e53a8ffe755bc26a8b0f1 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 25 Nov 2007 15:22:50 +0200 Subject: KVM: Use generalized exception queue for injecting #UD Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 12 ++---------- drivers/kvm/vmx.c | 11 +---------- 2 files changed, 3 insertions(+), 20 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index f9769338c621..06beed7d4a08 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -207,13 +207,6 @@ static bool svm_exception_injected(struct kvm_vcpu *vcpu) return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID); } -static void inject_ud(struct kvm_vcpu *vcpu) -{ - to_svm(vcpu)->vmcb->control.event_inj = SVM_EVTINJ_VALID | - SVM_EVTINJ_TYPE_EXEPT | - UD_VECTOR; -} - static int is_external_interrupt(u32 info) { info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; @@ -948,8 +941,7 @@ static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0); if (er != EMULATE_DONE) - inject_ud(&svm->vcpu); - + kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; } @@ -1027,7 +1019,7 @@ static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) static int invalid_op_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - inject_ud(&svm->vcpu); + kvm_queue_exception(&svm->vcpu, UD_VECTOR); return 1; } diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 3b3c5f7d2e7c..3b44573c326e 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -613,14 +613,6 @@ static bool vmx_exception_injected(struct kvm_vcpu *vcpu) return !(vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); } -static void vmx_inject_ud(struct kvm_vcpu *vcpu) -{ - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - UD_VECTOR | - INTR_TYPE_EXCEPTION | - INTR_INFO_VALID_MASK); -} - /* * Swap MSR entry in host/guest MSR entry array. */ @@ -1866,8 +1858,7 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (is_invalid_opcode(intr_info)) { er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); if (er != EMULATE_DONE) - vmx_inject_ud(vcpu); - + kvm_queue_exception(vcpu, UD_VECTOR); return 1; } -- cgit v1.2.3 From e934c9c1c8742872a53efb84966d9c1d7b8c8e24 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 6 Dec 2007 16:15:02 +0200 Subject: KVM: x86 emulator: fix eflags preparation for emulation We prepare eflags for the emulated instruction, then clobber it with an 'andl'. Fix by popping eflags as the last thing in the sequence. Patch taken from Xen (16143:959b4b92b6bf) Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 2e259a847697..f423b0e327f4 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -256,21 +256,21 @@ static u16 twobyte_table[256] = { #define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF) /* Before executing instruction: restore necessary bits in EFLAGS. */ -#define _PRE_EFLAGS(_sav, _msk, _tmp) \ - /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); */ \ - "push %"_sav"; " \ - "movl %"_msk",%"_LO32 _tmp"; " \ - "andl %"_LO32 _tmp",("_STK"); " \ - "pushf; " \ - "notl %"_LO32 _tmp"; " \ - "andl %"_LO32 _tmp",("_STK"); " \ - "pop %"_tmp"; " \ - "orl %"_LO32 _tmp",("_STK"); " \ - "popf; " \ - /* _sav &= ~msk; */ \ - "movl %"_msk",%"_LO32 _tmp"; " \ - "notl %"_LO32 _tmp"; " \ - "andl %"_LO32 _tmp",%"_sav"; " +#define _PRE_EFLAGS(_sav, _msk, _tmp) \ + /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \ + "movl %"_sav",%"_LO32 _tmp"; " \ + "push %"_tmp"; " \ + "push %"_tmp"; " \ + "movl %"_msk",%"_LO32 _tmp"; " \ + "andl %"_LO32 _tmp",("_STK"); " \ + "pushf; " \ + "notl %"_LO32 _tmp"; " \ + "andl %"_LO32 _tmp",("_STK"); " \ + "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \ + "pop %"_tmp"; " \ + "orl %"_LO32 _tmp",("_STK"); " \ + "popf; " \ + "pop %"_sav"; " /* After executing instruction: write-back necessary bits in EFLAGS. */ #define _POST_EFLAGS(_sav, _msk, _tmp) \ -- cgit v1.2.3 From e5314067f6a77688a3d36548e7618430ce4a6236 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 6 Dec 2007 16:32:45 +0200 Subject: KVM: VMX: Avoid exit when setting cr8 if the local apic is in the kernel With apic in userspace, we must exit to userspace after a cr8 write in order to update the tpr. But if the apic is in the kernel, the exit is unnecessary. Noticed by Joerg Roedel. Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 3b44573c326e..83084348581a 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1973,6 +1973,8 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu_load_rsp_rip(vcpu); set_cr8(vcpu, vcpu->regs[reg]); skip_emulated_instruction(vcpu); + if (irqchip_in_kernel(vcpu->kvm)) + return 1; kvm_run->exit_reason = KVM_EXIT_SET_TPR; return 0; }; -- cgit v1.2.3 From 152ff9be2ed11e76ed3014b94eacdd6bc9275518 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 6 Dec 2007 15:46:52 +0100 Subject: KVM: SVM: Emulate read/write access to cr8 This patch adds code to emulate the access to the cr8 register to the x86 instruction emulator in kvm. This is needed on svm, where there is no hardware decode for control register access. Signed-off-by: Joerg Roedel Signed-off-by: Markus Rechberger Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 6deb052b5f93..9db4e3242b62 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -2236,6 +2236,8 @@ unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) return vcpu->cr3; case 4: return vcpu->cr4; + case 8: + return get_cr8(vcpu); default: vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); return 0; @@ -2259,6 +2261,9 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, case 4: set_cr4(vcpu, mk_cr_64(vcpu->cr4, val)); break; + case 8: + set_cr8(vcpu, val & 0xfUL); + break; default: vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); } -- cgit v1.2.3 From 6e3d5dfbad4d8c29bdf8ed160f91f955f0efc12b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 6 Dec 2007 18:14:14 +0200 Subject: KVM: x86 emulator: Fix stack instructions on 64-bit mode Stack instructions are always 64-bit on 64-bit mode; many of the emulated stack instructions did not take that into account. Fix by adding a 'Stack' bitflag and setting the operand size appropriately during the decode stage (except for 'push r/m', which is in a group with a few other instructions, so it gets its own treatment). This fixes random crashes on Vista x64. Signed-off-by: Avi Kivity --- drivers/kvm/x86_emulate.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index f423b0e327f4..0a6ab06fde01 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -65,6 +65,7 @@ #define BitOp (1<<8) #define MemAbs (1<<9) /* Memory operand is absolute displacement */ #define String (1<<10) /* String instruction (rep capable) */ +#define Stack (1<<11) /* Stack instruction (push/pop) */ static u16 opcode_table[256] = { /* 0x00 - 0x07 */ @@ -104,14 +105,16 @@ static u16 opcode_table[256] = { /* 0x48 - 0x4F */ DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, /* 0x50 - 0x57 */ - SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, SrcReg, + SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, + SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, /* 0x58 - 0x5F */ - DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, + DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, + DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, /* 0x60 - 0x67 */ 0, 0, 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ , 0, 0, 0, 0, /* 0x68 - 0x6F */ - 0, 0, ImplicitOps|Mov, 0, + 0, 0, ImplicitOps | Mov | Stack, 0, SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */ SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */ /* 0x70 - 0x77 */ @@ -128,9 +131,10 @@ static u16 opcode_table[256] = { /* 0x88 - 0x8F */ ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov, ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - 0, ModRM | DstReg, 0, DstMem | SrcNone | ModRM | Mov, + 0, ModRM | DstReg, 0, DstMem | SrcNone | ModRM | Mov | Stack, /* 0x90 - 0x9F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps, ImplicitOps, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, ImplicitOps | Stack, ImplicitOps | Stack, 0, 0, /* 0xA0 - 0xA7 */ ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs, @@ -144,7 +148,7 @@ static u16 opcode_table[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0xC0 - 0xC7 */ ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM, - 0, ImplicitOps, 0, 0, + 0, ImplicitOps | Stack, 0, 0, ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov, /* 0xC8 - 0xCF */ 0, 0, 0, 0, 0, 0, 0, 0, @@ -157,7 +161,8 @@ static u16 opcode_table[256] = { /* 0xE0 - 0xE7 */ 0, 0, 0, 0, 0, 0, 0, 0, /* 0xE8 - 0xEF */ - ImplicitOps, SrcImm|ImplicitOps, 0, SrcImmByte|ImplicitOps, 0, 0, 0, 0, + ImplicitOps | Stack, SrcImm|ImplicitOps, 0, SrcImmByte|ImplicitOps, + 0, 0, 0, 0, /* 0xF0 - 0xF7 */ 0, 0, 0, 0, ImplicitOps, ImplicitOps, @@ -868,6 +873,9 @@ done_prefixes: } } + if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack)) + c->op_bytes = 8; + /* ModRM and SIB bytes. */ if (c->d & ModRM) rc = decode_modrm(ctxt, ops); @@ -988,11 +996,6 @@ static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, struct decode_cache *c = &ctxt->decode; int rc; - /* 64-bit mode: POP always pops a 64-bit operand. */ - - if (ctxt->mode == X86EMUL_MODE_PROT64) - c->dst.bytes = 8; - rc = ops->read_std(register_address(ctxt->ss_base, c->regs[VCPU_REGS_RSP]), &c->dst.val, c->dst.bytes, ctxt->vcpu); -- cgit v1.2.3 From 80a8119ca3f021037b8513d39dbb0ffd1af86b20 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Thu, 6 Dec 2007 19:50:00 +0200 Subject: KVM: SVM: Trap access to the cr8 register Later we may be able to use the virtual tpr feature, but for now, just trap it. Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 8 ++++++-- drivers/kvm/svm.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 06beed7d4a08..143d271d4431 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -449,11 +449,13 @@ static void init_vmcb(struct vmcb *vmcb) control->intercept_cr_read = INTERCEPT_CR0_MASK | INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK; + INTERCEPT_CR4_MASK | + INTERCEPT_CR8_MASK; control->intercept_cr_write = INTERCEPT_CR0_MASK | INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK; + INTERCEPT_CR4_MASK | + INTERCEPT_CR8_MASK; control->intercept_dr_read = INTERCEPT_DR0_MASK | INTERCEPT_DR1_MASK | @@ -1195,10 +1197,12 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, [SVM_EXIT_READ_CR0] = emulate_on_interception, [SVM_EXIT_READ_CR3] = emulate_on_interception, [SVM_EXIT_READ_CR4] = emulate_on_interception, + [SVM_EXIT_READ_CR8] = emulate_on_interception, /* for now: */ [SVM_EXIT_WRITE_CR0] = emulate_on_interception, [SVM_EXIT_WRITE_CR3] = emulate_on_interception, [SVM_EXIT_WRITE_CR4] = emulate_on_interception, + [SVM_EXIT_WRITE_CR8] = emulate_on_interception, [SVM_EXIT_READ_DR0] = emulate_on_interception, [SVM_EXIT_READ_DR1] = emulate_on_interception, [SVM_EXIT_READ_DR2] = emulate_on_interception, diff --git a/drivers/kvm/svm.h b/drivers/kvm/svm.h index 5fa277c0187c..5fd50491b555 100644 --- a/drivers/kvm/svm.h +++ b/drivers/kvm/svm.h @@ -204,6 +204,7 @@ struct __attribute__ ((__packed__)) vmcb { #define INTERCEPT_CR0_MASK 1 #define INTERCEPT_CR3_MASK (1 << 3) #define INTERCEPT_CR4_MASK (1 << 4) +#define INTERCEPT_CR8_MASK (1 << 8) #define INTERCEPT_DR0_MASK 1 #define INTERCEPT_DR1_MASK (1 << 1) -- cgit v1.2.3 From b3e4e63fd9ee8cd5d9047b89e0d463d5c48ee5b5 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Fri, 7 Dec 2007 07:56:58 -0500 Subject: KVM: MMU: Use cmpxchg for pte updates on walk_addr() In preparation for multi-threaded guest pte walking, use cmpxchg() when updating guest pte's. This guarantees that the assignment of the dirty bit can't be lost if two CPU's are faulting the same address simultaneously. [avi: fix kunmap_atomic() parameters] Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 3c40a598ceb8..8086f827120f 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -34,7 +34,9 @@ #define PT_LEVEL_BITS PT64_LEVEL_BITS #ifdef CONFIG_X86_64 #define PT_MAX_FULL_LEVELS 4 + #define CMPXCHG cmpxchg #else + #define CMPXCHG cmpxchg64 #define PT_MAX_FULL_LEVELS 2 #endif #elif PTTYPE == 32 @@ -48,6 +50,7 @@ #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level) #define PT_LEVEL_BITS PT32_LEVEL_BITS #define PT_MAX_FULL_LEVELS 2 + #define CMPXCHG cmpxchg #else #error Invalid PTTYPE value #endif @@ -78,6 +81,26 @@ static gfn_t gpte_to_gfn_pde(pt_element_t gpte) return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; } +static bool FNAME(cmpxchg_gpte)(struct kvm *kvm, + gfn_t table_gfn, unsigned index, + pt_element_t orig_pte, pt_element_t new_pte) +{ + pt_element_t ret; + pt_element_t *table; + struct page *page; + + page = gfn_to_page(kvm, table_gfn); + table = kmap_atomic(page, KM_USER0); + + ret = CMPXCHG(&table[index], orig_pte, new_pte); + + kunmap_atomic(table, KM_USER0); + + kvm_release_page_dirty(page); + + return (ret != orig_pte); +} + /* * Fetch a guest pte for a guest virtual address */ @@ -91,6 +114,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, gpa_t pte_gpa; pgprintk("%s: addr %lx\n", __FUNCTION__, addr); +walk: walker->level = vcpu->mmu.root_level; pte = vcpu->cr3; #if PTTYPE == 64 @@ -135,8 +159,10 @@ static int FNAME(walk_addr)(struct guest_walker *walker, if (!(pte & PT_ACCESSED_MASK)) { mark_page_dirty(vcpu->kvm, table_gfn); + if (FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, + index, pte, pte|PT_ACCESSED_MASK)) + goto walk; pte |= PT_ACCESSED_MASK; - kvm_write_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); } if (walker->level == PT_PAGE_TABLE_LEVEL) { @@ -159,9 +185,14 @@ static int FNAME(walk_addr)(struct guest_walker *walker, } if (write_fault && !is_dirty_pte(pte)) { + bool ret; + mark_page_dirty(vcpu->kvm, table_gfn); + ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte, + pte|PT_DIRTY_MASK); + if (ret) + goto walk; pte |= PT_DIRTY_MASK; - kvm_write_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); } @@ -484,3 +515,4 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, #undef PT_MAX_FULL_LEVELS #undef gpte_to_gfn #undef gpte_to_gfn_pde +#undef CMPXCHG -- cgit v1.2.3 From fe135d2ceb3d7dc08151b3cbad96565d02cc8676 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 16:15:46 +0200 Subject: KVM: MMU: Simplify calculation of pte access The nx bit is awkwardly placed in the 63rd bit position; furthermore it has a reversed meaning compared to the other bits, which means we can't use a bitwise and to calculate compounded access masks. So, we simplify things by creating a new 3-bit exec/write/user access word, and doing all calculations in that. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 14 ++++++++---- drivers/kvm/paging_tmpl.h | 58 +++++++++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 29 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 62a74151f847..f8a2137c64a2 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -85,7 +85,8 @@ static int dbg = 1; #define PT_PAGE_SIZE_MASK (1ULL << 7) #define PT_PAT_MASK (1ULL << 7) #define PT_GLOBAL_MASK (1ULL << 8) -#define PT64_NX_MASK (1ULL << 63) +#define PT64_NX_SHIFT 63 +#define PT64_NX_MASK (1ULL << PT64_NX_SHIFT) #define PT_PAT_SHIFT 7 #define PT_DIR_PAT_SHIFT 12 @@ -153,6 +154,11 @@ static int dbg = 1; #define RMAP_EXT 4 +#define ACC_EXEC_MASK 1 +#define ACC_WRITE_MASK PT_WRITABLE_MASK +#define ACC_USER_MASK PT_USER_MASK +#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK) + struct kvm_rmap_desc { u64 *shadow_ptes[RMAP_EXT]; struct kvm_rmap_desc *more; @@ -921,7 +927,7 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, struct page *page) >> PAGE_SHIFT; new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, v, level - 1, - 1, 3, &table[index]); + 1, ACC_ALL, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); kvm_release_page_clean(page); @@ -988,7 +994,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) ASSERT(!VALID_PAGE(root)); sp = kvm_mmu_get_page(vcpu, root_gfn, 0, - PT64_ROOT_LEVEL, 0, 0, NULL); + PT64_ROOT_LEVEL, 0, ACC_ALL, NULL); root = __pa(sp->spt); ++sp->root_count; vcpu->mmu.root_hpa = root; @@ -1009,7 +1015,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) root_gfn = 0; sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL, !is_paging(vcpu), - 0, NULL); + ACC_ALL, NULL); root = __pa(sp->spt); ++sp->root_count; vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 8086f827120f..7688cbf413c8 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -66,7 +66,8 @@ struct guest_walker { int level; gfn_t table_gfn[PT_MAX_FULL_LEVELS]; pt_element_t pte; - pt_element_t inherited_ar; + unsigned pt_access; + unsigned pte_access; gfn_t gfn; u32 error_code; }; @@ -110,7 +111,7 @@ static int FNAME(walk_addr)(struct guest_walker *walker, { pt_element_t pte; gfn_t table_gfn; - unsigned index; + unsigned index, pt_access, pte_access; gpa_t pte_gpa; pgprintk("%s: addr %lx\n", __FUNCTION__, addr); @@ -128,7 +129,7 @@ walk: ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0); - walker->inherited_ar = PT_USER_MASK | PT_WRITABLE_MASK; + pt_access = ACC_ALL; for (;;) { index = PT_INDEX(addr, walker->level); @@ -165,6 +166,14 @@ walk: pte |= PT_ACCESSED_MASK; } + pte_access = pte & (PT_WRITABLE_MASK | PT_USER_MASK); + pte_access |= ACC_EXEC_MASK; +#if PTTYPE == 64 + if (is_nx(vcpu)) + pte_access &= ~(pte >> PT64_NX_SHIFT); +#endif + pte_access &= pt_access; + if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = gpte_to_gfn(pte); break; @@ -180,7 +189,7 @@ walk: break; } - walker->inherited_ar &= pte; + pt_access = pte_access; --walker->level; } @@ -197,7 +206,10 @@ walk: } walker->pte = pte; - pgprintk("%s: pte %llx\n", __FUNCTION__, (u64)pte); + walker->pt_access = pt_access; + walker->pte_access = pte_access; + pgprintk("%s: pte %llx pte_access %x pt_access %x\n", + __FUNCTION__, (u64)pte, pt_access, pte_access); return 1; not_present: @@ -218,7 +230,8 @@ err: } static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, - u64 *shadow_pte, u64 access_bits, + u64 *shadow_pte, unsigned pt_access, + unsigned pte_access, int user_fault, int write_fault, int *ptwrite, struct guest_walker *walker, gfn_t gfn) @@ -228,12 +241,11 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, int was_rmapped = is_rmap_pte(*shadow_pte); struct page *page; - pgprintk("%s: spte %llx gpte %llx access %llx write_fault %d" + pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" " user_fault %d gfn %lx\n", - __FUNCTION__, *shadow_pte, (u64)gpte, access_bits, + __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, write_fault, user_fault, gfn); - access_bits &= gpte; /* * We don't set the accessed bit, since we sometimes want to see * whether the guest actually used the pte (in order to detect @@ -242,12 +254,12 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, spte = PT_PRESENT_MASK | PT_DIRTY_MASK; spte |= gpte & PT64_NX_MASK; if (!dirty) - access_bits &= ~PT_WRITABLE_MASK; + pte_access &= ~ACC_WRITE_MASK; page = gfn_to_page(vcpu->kvm, gfn); spte |= PT_PRESENT_MASK; - if (access_bits & PT_USER_MASK) + if (pte_access & ACC_USER_MASK) spte |= PT_USER_MASK; if (is_error_page(page)) { @@ -259,7 +271,7 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, spte |= page_to_phys(page); - if ((access_bits & PT_WRITABLE_MASK) + if ((pte_access & ACC_WRITE_MASK) || (write_fault && !is_write_protection(vcpu) && !user_fault)) { struct kvm_mmu_page *shadow; @@ -273,7 +285,7 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, if (shadow) { pgprintk("%s: found shadow page for %lx, marking ro\n", __FUNCTION__, gfn); - access_bits &= ~PT_WRITABLE_MASK; + pte_access &= ~ACC_WRITE_MASK; if (is_writeble_pte(spte)) { spte &= ~PT_WRITABLE_MASK; kvm_x86_ops->tlb_flush(vcpu); @@ -285,7 +297,7 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, unshadowed: - if (access_bits & PT_WRITABLE_MASK) + if (pte_access & ACC_WRITE_MASK) mark_page_dirty(vcpu->kvm, gfn); pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); @@ -317,8 +329,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, if (bytes < sizeof(pt_element_t)) return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); - FNAME(set_pte)(vcpu, gpte, spte, PT_USER_MASK | PT_WRITABLE_MASK, 0, - 0, NULL, NULL, gpte_to_gfn(gpte)); + FNAME(set_pte)(vcpu, gpte, spte, ACC_ALL, ACC_ALL, + 0, 0, NULL, NULL, gpte_to_gfn(gpte)); } /* @@ -331,6 +343,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, hpa_t shadow_addr; int level; u64 *shadow_ent; + unsigned access = walker->pt_access; if (!is_present_pte(walker->pte)) return NULL; @@ -349,7 +362,6 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, u64 shadow_pte; int metaphysical; gfn_t table_gfn; - unsigned hugepage_access = 0; shadow_ent = ((u64 *)__va(shadow_addr)) + index; if (is_shadow_present_pte(*shadow_ent)) { @@ -365,20 +377,15 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (level - 1 == PT_PAGE_TABLE_LEVEL && walker->level == PT_DIRECTORY_LEVEL) { metaphysical = 1; - hugepage_access = walker->pte; - hugepage_access &= PT_USER_MASK | PT_WRITABLE_MASK; if (!is_dirty_pte(walker->pte)) - hugepage_access &= ~PT_WRITABLE_MASK; - hugepage_access >>= PT_WRITABLE_SHIFT; - if (walker->pte & PT64_NX_MASK) - hugepage_access |= (1 << 2); + access &= ~ACC_WRITE_MASK; table_gfn = gpte_to_gfn(walker->pte); } else { metaphysical = 0; table_gfn = walker->table_gfn[level - 2]; } shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, - metaphysical, hugepage_access, + metaphysical, access, shadow_ent); shadow_addr = __pa(shadow_page->spt); shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK @@ -387,7 +394,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, } FNAME(set_pte)(vcpu, walker->pte, shadow_ent, - walker->inherited_ar, user_fault, write_fault, + access, walker->pte_access & access, + user_fault, write_fault, ptwrite, walker, walker->gfn); return shadow_ent; -- cgit v1.2.3 From 8d87a03aea43535a92efbb180e0ceac94d4bb5db Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 16:37:36 +0200 Subject: KVM: MMU: Set nx bit correctly on shadow ptes While the page table walker correctly generates a guest page fault if a guest tries to execute a non-executable page, the shadow code does not mark it non-executable. This means that if a guest accesses an nx page first with a read access, then subsequent code fetch accesses will succeed. Fix by setting the nx bit on shadow ptes. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 7688cbf413c8..59ba752a6880 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -255,6 +255,8 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, spte |= gpte & PT64_NX_MASK; if (!dirty) pte_access &= ~ACC_WRITE_MASK; + if (!(pte_access & ACC_EXEC_MASK)) + spte |= PT64_NX_MASK; page = gfn_to_page(vcpu->kvm, gfn); -- cgit v1.2.3 From bedbe4ee86195dcd899577828714cc1413beb571 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 16:52:56 +0200 Subject: KVM: MMU: Move pte access calculation into a helper function Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 59ba752a6880..1fc4f9b321ee 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -102,6 +102,18 @@ static bool FNAME(cmpxchg_gpte)(struct kvm *kvm, return (ret != orig_pte); } +static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte) +{ + unsigned access; + + access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK; +#if PTTYPE == 64 + if (is_nx(vcpu)) + access &= ~(gpte >> PT64_NX_SHIFT); +#endif + return access; +} + /* * Fetch a guest pte for a guest virtual address */ @@ -166,13 +178,7 @@ walk: pte |= PT_ACCESSED_MASK; } - pte_access = pte & (PT_WRITABLE_MASK | PT_USER_MASK); - pte_access |= ACC_EXEC_MASK; -#if PTTYPE == 64 - if (is_nx(vcpu)) - pte_access &= ~(pte >> PT64_NX_SHIFT); -#endif - pte_access &= pt_access; + pte_access = pt_access & FNAME(gpte_access)(vcpu, pte); if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = gpte_to_gfn(pte); -- cgit v1.2.3 From 41074d07c78b086b18fc2af590caef05dbcca568 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:00:02 +0200 Subject: KVM: MMU: Fix inherited permissions for emulated guest pte updates When we emulate a guest pte write, we fail to apply the correct inherited permissions from the parent ptes. Now that we store inherited permissions in the shadow page, we can use that to update the pte permissions correctly. Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 4 ++-- drivers/kvm/mmu.c | 4 ++-- drivers/kvm/paging_tmpl.h | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 0d3555bf5333..ceefb4427dbd 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -55,7 +55,7 @@ struct kvm_pte_chain { * bits 4:7 - page table level for this shadow (1-4) * bits 8:9 - page table quadrant for 2-level guests * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) - * bits 17:19 - "access" - the user, writable, and nx bits of a huge page pde + * bits 17:19 - common access permissions for all ptes in this shadow page */ union kvm_mmu_page_role { unsigned word; @@ -65,7 +65,7 @@ union kvm_mmu_page_role { unsigned quadrant : 2; unsigned pad_for_nice_hex_output : 6; unsigned metaphysical : 1; - unsigned hugepage_access : 3; + unsigned access : 3; }; }; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index f8a2137c64a2..cace1e41b683 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -680,7 +680,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, gva_t gaddr, unsigned level, int metaphysical, - unsigned hugepage_access, + unsigned access, u64 *parent_pte) { union kvm_mmu_page_role role; @@ -694,7 +694,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, role.glevels = vcpu->mmu.root_level; role.level = level; role.metaphysical = metaphysical; - role.hugepage_access = hugepage_access; + role.access = access; if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) { quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 1fc4f9b321ee..211fef83be5d 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -327,6 +327,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, int offset_in_pte) { pt_element_t gpte; + unsigned pte_access; gpte = *(const pt_element_t *)pte; if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { @@ -337,7 +338,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, if (bytes < sizeof(pt_element_t)) return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); - FNAME(set_pte)(vcpu, gpte, spte, ACC_ALL, ACC_ALL, + pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); + FNAME(set_pte)(vcpu, gpte, spte, page->role.access, pte_access, 0, 0, NULL, NULL, gpte_to_gfn(gpte)); } -- cgit v1.2.3 From b4ab019ce71efe30790b5aaa396549a16bb20608 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:27:52 +0200 Subject: KVM: MMU: No need to pick up nx bit from guest pte We already set it according to cumulative access permissions. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 211fef83be5d..179ce9e8145e 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -258,7 +258,6 @@ static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, * demand paging). */ spte = PT_PRESENT_MASK | PT_DIRTY_MASK; - spte |= gpte & PT64_NX_MASK; if (!dirty) pte_access &= ~ACC_WRITE_MASK; if (!(pte_access & ACC_EXEC_MASK)) -- cgit v1.2.3 From e3f955042219a595b1c3208e12c20860f23f72fb Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:32:30 +0200 Subject: KVM: MMU: Pass pte dirty flag to set_pte() instead of calculating it on-site This allows us to remove its dependency on pt_element_t. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 179ce9e8145e..ca8d51574118 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -235,14 +235,12 @@ err: return 0; } -static void FNAME(set_pte)(struct kvm_vcpu *vcpu, pt_element_t gpte, - u64 *shadow_pte, unsigned pt_access, - unsigned pte_access, - int user_fault, int write_fault, +static void FNAME(set_pte)(struct kvm_vcpu *vcpu, u64 *shadow_pte, + unsigned pt_access, unsigned pte_access, + int user_fault, int write_fault, int dirty, int *ptwrite, struct guest_walker *walker, gfn_t gfn) { - int dirty = gpte & PT_DIRTY_MASK; u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); struct page *page; @@ -338,8 +336,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); - FNAME(set_pte)(vcpu, gpte, spte, page->role.access, pte_access, - 0, 0, NULL, NULL, gpte_to_gfn(gpte)); + FNAME(set_pte)(vcpu, spte, page->role.access, pte_access, 0, 0, + gpte & PT_DIRTY_MASK, NULL, NULL, gpte_to_gfn(gpte)); } /* @@ -402,9 +400,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, *shadow_ent = shadow_pte; } - FNAME(set_pte)(vcpu, walker->pte, shadow_ent, - access, walker->pte_access & access, - user_fault, write_fault, + FNAME(set_pte)(vcpu, shadow_ent, access, walker->pte_access & access, + user_fault, write_fault, walker->pte & PT_DIRTY_MASK, ptwrite, walker, walker->gfn); return shadow_ent; -- cgit v1.2.3 From 2fbf4cf13f777e1f61ee692fe67d16bddd747700 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:33:46 +0200 Subject: KVM: MMU: Remove walker argument to set_pte() Unused. Signed-off-by: Avi Kivity --- drivers/kvm/paging_tmpl.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index ca8d51574118..2e5a80af22c9 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -238,8 +238,7 @@ err: static void FNAME(set_pte)(struct kvm_vcpu *vcpu, u64 *shadow_pte, unsigned pt_access, unsigned pte_access, int user_fault, int write_fault, int dirty, - int *ptwrite, struct guest_walker *walker, - gfn_t gfn) + int *ptwrite, gfn_t gfn) { u64 spte; int was_rmapped = is_rmap_pte(*shadow_pte); @@ -337,7 +336,7 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); FNAME(set_pte)(vcpu, spte, page->role.access, pte_access, 0, 0, - gpte & PT_DIRTY_MASK, NULL, NULL, gpte_to_gfn(gpte)); + gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); } /* @@ -402,7 +401,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, FNAME(set_pte)(vcpu, shadow_ent, access, walker->pte_access & access, user_fault, write_fault, walker->pte & PT_DIRTY_MASK, - ptwrite, walker, walker->gfn); + ptwrite, walker->gfn); return shadow_ent; } -- cgit v1.2.3 From 1c4f1fd6d5692614e8dc75ee53f7be590f1e878b Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 17:40:31 +0200 Subject: KVM: MMU: Move set_pte() into guest paging mode independent code As set_pte() no longer references either a gpte or the guest walker, we can move it out of paging mode dependent code (which compiles twice and is generally nasty). Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 83 ++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/paging_tmpl.h | 93 +++-------------------------------------------- 2 files changed, 88 insertions(+), 88 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index cace1e41b683..a91e05b42345 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -879,6 +879,89 @@ struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) return gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); } +static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, + unsigned pt_access, unsigned pte_access, + int user_fault, int write_fault, int dirty, + int *ptwrite, gfn_t gfn) +{ + u64 spte; + int was_rmapped = is_rmap_pte(*shadow_pte); + struct page *page; + + pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" + " user_fault %d gfn %lx\n", + __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, + write_fault, user_fault, gfn); + + /* + * We don't set the accessed bit, since we sometimes want to see + * whether the guest actually used the pte (in order to detect + * demand paging). + */ + spte = PT_PRESENT_MASK | PT_DIRTY_MASK; + if (!dirty) + pte_access &= ~ACC_WRITE_MASK; + if (!(pte_access & ACC_EXEC_MASK)) + spte |= PT64_NX_MASK; + + page = gfn_to_page(vcpu->kvm, gfn); + + spte |= PT_PRESENT_MASK; + if (pte_access & ACC_USER_MASK) + spte |= PT_USER_MASK; + + if (is_error_page(page)) { + set_shadow_pte(shadow_pte, + shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); + kvm_release_page_clean(page); + return; + } + + spte |= page_to_phys(page); + + if ((pte_access & ACC_WRITE_MASK) + || (write_fault && !is_write_protection(vcpu) && !user_fault)) { + struct kvm_mmu_page *shadow; + + spte |= PT_WRITABLE_MASK; + if (user_fault) { + mmu_unshadow(vcpu->kvm, gfn); + goto unshadowed; + } + + shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn); + if (shadow) { + pgprintk("%s: found shadow page for %lx, marking ro\n", + __FUNCTION__, gfn); + pte_access &= ~ACC_WRITE_MASK; + if (is_writeble_pte(spte)) { + spte &= ~PT_WRITABLE_MASK; + kvm_x86_ops->tlb_flush(vcpu); + } + if (write_fault) + *ptwrite = 1; + } + } + +unshadowed: + + if (pte_access & ACC_WRITE_MASK) + mark_page_dirty(vcpu->kvm, gfn); + + pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); + set_shadow_pte(shadow_pte, spte); + page_header_update_slot(vcpu->kvm, shadow_pte, gfn); + if (!was_rmapped) { + rmap_add(vcpu, shadow_pte, gfn); + if (!is_rmap_pte(*shadow_pte)) + kvm_release_page_clean(page); + } + else + kvm_release_page_clean(page); + if (!ptwrite || !*ptwrite) + vcpu->last_pte_updated = shadow_pte; +} + static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 2e5a80af22c9..3ab3fb635e16 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -235,89 +235,6 @@ err: return 0; } -static void FNAME(set_pte)(struct kvm_vcpu *vcpu, u64 *shadow_pte, - unsigned pt_access, unsigned pte_access, - int user_fault, int write_fault, int dirty, - int *ptwrite, gfn_t gfn) -{ - u64 spte; - int was_rmapped = is_rmap_pte(*shadow_pte); - struct page *page; - - pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" - " user_fault %d gfn %lx\n", - __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, - write_fault, user_fault, gfn); - - /* - * We don't set the accessed bit, since we sometimes want to see - * whether the guest actually used the pte (in order to detect - * demand paging). - */ - spte = PT_PRESENT_MASK | PT_DIRTY_MASK; - if (!dirty) - pte_access &= ~ACC_WRITE_MASK; - if (!(pte_access & ACC_EXEC_MASK)) - spte |= PT64_NX_MASK; - - page = gfn_to_page(vcpu->kvm, gfn); - - spte |= PT_PRESENT_MASK; - if (pte_access & ACC_USER_MASK) - spte |= PT_USER_MASK; - - if (is_error_page(page)) { - set_shadow_pte(shadow_pte, - shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); - kvm_release_page_clean(page); - return; - } - - spte |= page_to_phys(page); - - if ((pte_access & ACC_WRITE_MASK) - || (write_fault && !is_write_protection(vcpu) && !user_fault)) { - struct kvm_mmu_page *shadow; - - spte |= PT_WRITABLE_MASK; - if (user_fault) { - mmu_unshadow(vcpu->kvm, gfn); - goto unshadowed; - } - - shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn); - if (shadow) { - pgprintk("%s: found shadow page for %lx, marking ro\n", - __FUNCTION__, gfn); - pte_access &= ~ACC_WRITE_MASK; - if (is_writeble_pte(spte)) { - spte &= ~PT_WRITABLE_MASK; - kvm_x86_ops->tlb_flush(vcpu); - } - if (write_fault) - *ptwrite = 1; - } - } - -unshadowed: - - if (pte_access & ACC_WRITE_MASK) - mark_page_dirty(vcpu->kvm, gfn); - - pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); - set_shadow_pte(shadow_pte, spte); - page_header_update_slot(vcpu->kvm, shadow_pte, gfn); - if (!was_rmapped) { - rmap_add(vcpu, shadow_pte, gfn); - if (!is_rmap_pte(*shadow_pte)) - kvm_release_page_clean(page); - } - else - kvm_release_page_clean(page); - if (!ptwrite || !*ptwrite) - vcpu->last_pte_updated = shadow_pte; -} - static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, u64 *spte, const void *pte, int bytes, int offset_in_pte) @@ -335,8 +252,8 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, return; pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); - FNAME(set_pte)(vcpu, spte, page->role.access, pte_access, 0, 0, - gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); + mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0, + gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); } /* @@ -399,9 +316,9 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, *shadow_ent = shadow_pte; } - FNAME(set_pte)(vcpu, shadow_ent, access, walker->pte_access & access, - user_fault, write_fault, walker->pte & PT_DIRTY_MASK, - ptwrite, walker->gfn); + mmu_set_spte(vcpu, shadow_ent, access, walker->pte_access & access, + user_fault, write_fault, walker->pte & PT_DIRTY_MASK, + ptwrite, walker->gfn); return shadow_ent; } -- cgit v1.2.3 From bc750ba860d978fcaac1e0db28774b1f38ae8193 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 18:39:41 +0200 Subject: KVM: MMU: Adjust mmu_set_spte() debug code for gpte removal Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index a91e05b42345..b4dd72645853 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -888,9 +888,9 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, int was_rmapped = is_rmap_pte(*shadow_pte); struct page *page; - pgprintk("%s: spte %llx gpte %llx access %x write_fault %d" + pgprintk("%s: spte %llx access %x write_fault %d" " user_fault %d gfn %lx\n", - __FUNCTION__, *shadow_pte, (u64)gpte, pt_access, + __FUNCTION__, *shadow_pte, pt_access, write_fault, user_fault, gfn); /* -- cgit v1.2.3 From e833240f3c1b0b415efb14eaa102718769d5f063 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 9 Dec 2007 18:43:00 +0200 Subject: KVM: MMU: Use mmu_set_spte() for real-mode shadows In addition to removing some duplicated code, this also handles the unlikely case of real-mode code updating a guest page table. This can happen when one vcpu (in real mode) touches a second vcpu's (in protected mode) page tables, or if a vcpu switches to real mode, touches page tables, and switches back. Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index b4dd72645853..ba71e8d66761 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -966,40 +966,23 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) { } -static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, struct page *page) +static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) { int level = PT32E_ROOT_LEVEL; hpa_t table_addr = vcpu->mmu.root_hpa; + int pt_write = 0; for (; ; level--) { u32 index = PT64_INDEX(v, level); u64 *table; - u64 pte; ASSERT(VALID_PAGE(table_addr)); table = __va(table_addr); if (level == 1) { - int was_rmapped; - - pte = table[index]; - was_rmapped = is_rmap_pte(pte); - if (is_shadow_present_pte(pte) && is_writeble_pte(pte)) { - kvm_release_page_clean(page); - return 0; - } - mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT); - page_header_update_slot(vcpu->kvm, table, - v >> PAGE_SHIFT); - table[index] = page_to_phys(page) - | PT_PRESENT_MASK | PT_WRITABLE_MASK - | PT_USER_MASK; - if (!was_rmapped) - rmap_add(vcpu, &table[index], v >> PAGE_SHIFT); - else - kvm_release_page_clean(page); - - return 0; + mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL, + 0, write, 1, &pt_write, gfn); + return pt_write || is_io_pte(table[index]); } if (table[index] == shadow_trap_nonpresent_pte) { @@ -1013,7 +996,6 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, struct page *page) 1, ACC_ALL, &table[index]); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); - kvm_release_page_clean(page); return -ENOMEM; } @@ -1114,9 +1096,10 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code) { - struct page *page; + gfn_t gfn; int r; + pgprintk("%s: gva %lx error %x\n", __FUNCTION__, gva, error_code); r = mmu_topup_memory_caches(vcpu); if (r) return r; @@ -1124,14 +1107,10 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, ASSERT(vcpu); ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); - page = gfn_to_page(vcpu->kvm, gva >> PAGE_SHIFT); - - if (is_error_page(page)) { - kvm_release_page_clean(page); - return 1; - } + gfn = gva >> PAGE_SHIFT; - return nonpaging_map(vcpu, gva & PAGE_MASK, page); + return nonpaging_map(vcpu, gva & PAGE_MASK, + error_code & PFERR_WRITE_MASK, gfn); } static void nonpaging_free(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From 1d075434149c38d457c30d1f11d9c39210b0bb79 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 6 Dec 2007 21:02:25 +0100 Subject: KVM: SVM: Exit to userspace if write to cr8 and not using in-kernel apic With this patch KVM on SVM will exit to userspace if the guest writes to CR8 and the in-kernel APIC is disabled. Signed-off-by: Joerg Roedel Signed-off-by: Markus Rechberger Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 143d271d4431..442ca818c5a9 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1048,6 +1048,15 @@ static int emulate_on_interception(struct vcpu_svm *svm, return 1; } +static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + emulate_instruction(&svm->vcpu, NULL, 0, 0, 0); + if (irqchip_in_kernel(svm->vcpu.kvm)) + return 1; + kvm_run->exit_reason = KVM_EXIT_SET_TPR; + return 0; +} + static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) { struct vcpu_svm *svm = to_svm(vcpu); @@ -1202,7 +1211,7 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, [SVM_EXIT_WRITE_CR0] = emulate_on_interception, [SVM_EXIT_WRITE_CR3] = emulate_on_interception, [SVM_EXIT_WRITE_CR4] = emulate_on_interception, - [SVM_EXIT_WRITE_CR8] = emulate_on_interception, + [SVM_EXIT_WRITE_CR8] = cr8_write_interception, [SVM_EXIT_READ_DR0] = emulate_on_interception, [SVM_EXIT_READ_DR1] = emulate_on_interception, [SVM_EXIT_READ_DR2] = emulate_on_interception, -- cgit v1.2.3 From 7819026eefee53eaaac3fdce1a2f157c7ea943fe Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Tue, 11 Dec 2007 19:12:27 -0500 Subject: KVM: MMU: Fix SMP shadow instantiation race There is a race where VCPU0 is shadowing a pagetable entry while VCPU1 is updating it, which results in a stale shadow copy. Fix that by comparing the contents of the cached guest pte with the current guest pte after write-protecting the guest pagetable. Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- drivers/kvm/mmu.c | 12 ++++++++---- drivers/kvm/paging_tmpl.h | 29 +++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index ba71e8d66761..92ac0d1106b4 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -681,7 +681,8 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, unsigned level, int metaphysical, unsigned access, - u64 *parent_pte) + u64 *parent_pte, + bool *new_page) { union kvm_mmu_page_role role; unsigned index; @@ -720,6 +721,8 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, vcpu->mmu.prefetch_page(vcpu, sp); if (!metaphysical) rmap_write_protect(vcpu->kvm, gfn); + if (new_page) + *new_page = 1; return sp; } @@ -993,7 +996,8 @@ static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) >> PAGE_SHIFT; new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, v, level - 1, - 1, ACC_ALL, &table[index]); + 1, ACC_ALL, &table[index], + NULL); if (!new_table) { pgprintk("nonpaging_map: ENOMEM\n"); return -ENOMEM; @@ -1059,7 +1063,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) ASSERT(!VALID_PAGE(root)); sp = kvm_mmu_get_page(vcpu, root_gfn, 0, - PT64_ROOT_LEVEL, 0, ACC_ALL, NULL); + PT64_ROOT_LEVEL, 0, ACC_ALL, NULL, NULL); root = __pa(sp->spt); ++sp->root_count; vcpu->mmu.root_hpa = root; @@ -1080,7 +1084,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) root_gfn = 0; sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL, !is_paging(vcpu), - ACC_ALL, NULL); + ACC_ALL, NULL, NULL); root = __pa(sp->spt); ++sp->root_count; vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index 3ab3fb635e16..fb19596c9589 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -65,7 +65,8 @@ struct guest_walker { int level; gfn_t table_gfn[PT_MAX_FULL_LEVELS]; - pt_element_t pte; + pt_element_t ptes[PT_MAX_FULL_LEVELS]; + gpa_t pte_gpa[PT_MAX_FULL_LEVELS]; unsigned pt_access; unsigned pte_access; gfn_t gfn; @@ -150,6 +151,7 @@ walk: pte_gpa = gfn_to_gpa(table_gfn); pte_gpa += index * sizeof(pt_element_t); walker->table_gfn[walker->level - 1] = table_gfn; + walker->pte_gpa[walker->level - 1] = pte_gpa; pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, walker->level - 1, table_gfn); @@ -180,6 +182,8 @@ walk: pte_access = pt_access & FNAME(gpte_access)(vcpu, pte); + walker->ptes[walker->level - 1] = pte; + if (walker->level == PT_PAGE_TABLE_LEVEL) { walker->gfn = gpte_to_gfn(pte); break; @@ -209,9 +213,9 @@ walk: goto walk; pte |= PT_DIRTY_MASK; kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); + walker->ptes[walker->level - 1] = pte; } - walker->pte = pte; walker->pt_access = pt_access; walker->pte_access = pte_access; pgprintk("%s: pte %llx pte_access %x pt_access %x\n", @@ -268,7 +272,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, u64 *shadow_ent; unsigned access = walker->pt_access; - if (!is_present_pte(walker->pte)) + if (!is_present_pte(walker->ptes[walker->level - 1])) return NULL; shadow_addr = vcpu->mmu.root_hpa; @@ -285,6 +289,7 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, u64 shadow_pte; int metaphysical; gfn_t table_gfn; + bool new_page = 0; shadow_ent = ((u64 *)__va(shadow_addr)) + index; if (is_shadow_present_pte(*shadow_ent)) { @@ -300,16 +305,23 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (level - 1 == PT_PAGE_TABLE_LEVEL && walker->level == PT_DIRECTORY_LEVEL) { metaphysical = 1; - if (!is_dirty_pte(walker->pte)) + if (!is_dirty_pte(walker->ptes[level - 1])) access &= ~ACC_WRITE_MASK; - table_gfn = gpte_to_gfn(walker->pte); + table_gfn = gpte_to_gfn(walker->ptes[level - 1]); } else { metaphysical = 0; table_gfn = walker->table_gfn[level - 2]; } shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, metaphysical, access, - shadow_ent); + shadow_ent, &new_page); + if (new_page && !metaphysical) { + pt_element_t curr_pte; + kvm_read_guest(vcpu->kvm, walker->pte_gpa[level - 2], + &curr_pte, sizeof(curr_pte)); + if (curr_pte != walker->ptes[level - 2]) + return NULL; + } shadow_addr = __pa(shadow_page->spt); shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; @@ -317,7 +329,8 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, } mmu_set_spte(vcpu, shadow_ent, access, walker->pte_access & access, - user_fault, write_fault, walker->pte & PT_DIRTY_MASK, + user_fault, write_fault, + walker->ptes[walker->level-1] & PT_DIRTY_MASK, ptwrite, walker->gfn); return shadow_ent; @@ -382,7 +395,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, /* * mmio: emulate if accessible, otherwise its a guest fault. */ - if (is_io_pte(*shadow_pte)) + if (shadow_pte && is_io_pte(*shadow_pte)) return 1; ++vcpu->stat.pf_fixed; -- cgit v1.2.3 From 722f6ecbcf3e53809236b36e6dfd9042924a2d51 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 12 Dec 2007 12:37:24 +0100 Subject: KVM: LAPIC: minor debugging compile fix This patch fixes a compile error of the LAPIC code with APIC debugging enabled. Signed-off-by: Joerg Roedel Signed-off-by: Markus Rechberger Signed-off-by: Avi Kivity --- drivers/kvm/lapic.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 5efa6c0276c6..466c37f02e85 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -829,7 +829,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) /* with FSB delivery interrupt, we can restart APIC functionality */ apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " - "0x%lx.\n", apic->apic_base, apic->base_address); + "0x%lx.\n", apic->vcpu->apic_base, apic->base_address); } -- cgit v1.2.3 From 62b9abaaf8423a3a38b938831b3b8431a6607ad8 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 11 Dec 2007 15:36:57 +0100 Subject: KVM: SVM: support writing 0 to K8 performance counter control registers This lets SVM ignore writes of the value 0 to the performance counter control registers. Thus enabling them will still fail in the guest, but a write of 0 which keeps them disabled is accepted. This is required to boot Windows Vista 64bit. [avi: avoid fall-thru in switch statement] Signed-off-by: Joerg Roedel Signed-off-by: Markus Rechberger Signed-off-by: Avi Kivity --- drivers/kvm/svm.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 442ca818c5a9..ef21804a5c5c 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -1155,7 +1155,20 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) case MSR_IA32_SYSENTER_ESP: svm->vmcb->save.sysenter_esp = data; break; + case MSR_K7_EVNTSEL0: + case MSR_K7_EVNTSEL1: + case MSR_K7_EVNTSEL2: + case MSR_K7_EVNTSEL3: + /* + * only support writing 0 to the performance counters for now + * to make Windows happy. Should be replaced by a real + * performance counter emulation later. + */ + if (data != 0) + goto unhandled; + break; default: + unhandled: return kvm_set_msr_common(vcpu, ecx, data); } return 0; -- cgit v1.2.3 From 2bacc55c7c3c61e356aef06b9a319b4cee90b519 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Wed, 12 Dec 2007 10:46:12 -0500 Subject: KVM: MMU: emulated cmpxchg8b should be atomic on i386 Emulate cmpxchg8b atomically on i386. This is required to avoid a guest pte walker from seeing a splitted write. [avi: make it compile] Signed-off-by: Marcelo Tosatti Signed-off-by: Avi Kivity --- drivers/kvm/x86.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'drivers') diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 9db4e3242b62..3b79684a3c0c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -1674,6 +1675,31 @@ static int emulator_cmpxchg_emulated(unsigned long addr, reported = 1; printk(KERN_WARNING "kvm: emulating exchange as write\n"); } +#ifndef CONFIG_X86_64 + /* guests cmpxchg8b have to be emulated atomically */ + if (bytes == 8) { + gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + struct page *page; + char *addr; + u64 val; + + if (gpa == UNMAPPED_GVA || + (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto emul_write; + + if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK)) + goto emul_write; + + val = *(u64 *)new; + page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); + addr = kmap_atomic(page, KM_USER0); + set_64bit((u64 *)(addr + offset_in_page(gpa)), val); + kunmap_atomic(addr, KM_USER0); + kvm_release_page_dirty(page); + } +emul_write: +#endif + return emulator_write_emulated(addr, new, bytes, vcpu); } -- cgit v1.2.3 From 682c59a3f3f211ed555b17144f2d82eb8286a1db Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Tue, 11 Dec 2007 20:36:00 +0800 Subject: KVM: Portability: Move kvm{pic,ioapic} accesors to x86 specific code Signed-off-by: Zhang Xiantao Signed-off-by: Avi Kivity --- drivers/kvm/irq.h | 18 ++++++++++++++++-- drivers/kvm/kvm.h | 18 +++--------------- drivers/kvm/x86.h | 1 - 3 files changed, 19 insertions(+), 18 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index 730a87c173e5..e88d93957d24 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -26,6 +26,7 @@ #include #include #include "iodev.h" +#include "x86.h" struct kvm; struct kvm_vcpu; @@ -63,8 +64,6 @@ struct kvm_pic { struct kvm_pic *kvm_create_pic(struct kvm *kvm); void kvm_pic_set_irq(void *opaque, int irq, int level); int kvm_pic_read_irq(struct kvm_pic *s); -int kvm_cpu_get_interrupt(struct kvm_vcpu *v); -int kvm_cpu_has_interrupt(struct kvm_vcpu *v); void kvm_pic_update_irq(struct kvm_pic *s); #define IOAPIC_NUM_PINS KVM_IOAPIC_NUM_PINS @@ -147,6 +146,21 @@ do { \ #define ASSERT(x) do { } while (0) #endif +static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) +{ + return kvm->vpic; +} + +static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) +{ + return kvm->vioapic; +} + +static inline int irqchip_in_kernel(struct kvm *kvm) +{ + return pic_irqchip(kvm) != NULL; +} + void kvm_vcpu_kick(struct kvm_vcpu *vcpu); int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ceefb4427dbd..668a8300365d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -273,21 +273,6 @@ struct kvm { struct kvm_vm_stat stat; }; -static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) -{ - return kvm->vpic; -} - -static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) -{ - return kvm->vioapic; -} - -static inline int irqchip_in_kernel(struct kvm *kvm) -{ - return pic_irqchip(kvm) != NULL; -} - /* The guest did something we don't support. */ #define pr_unimpl(vcpu, fmt, ...) \ do { \ @@ -417,6 +402,9 @@ void kvm_free_physmem(struct kvm *kvm); struct kvm *kvm_arch_create_vm(void); void kvm_arch_destroy_vm(struct kvm *kvm); +int kvm_cpu_get_interrupt(struct kvm_vcpu *v); +int kvm_cpu_has_interrupt(struct kvm_vcpu *v); + static inline void kvm_guest_enter(void) { account_system_vtime(current); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 78396d627be0..0fc7020aa1a5 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -12,7 +12,6 @@ #define KVM_X86_H #include "kvm.h" -#include "irq.h" #include #include -- cgit v1.2.3 From ad312c7c79f781c822e37effe41307503a2bb85b Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Thu, 13 Dec 2007 23:50:52 +0800 Subject: KVM: Portability: Introduce kvm_vcpu_arch Move all the architecture-specific fields in kvm_vcpu into a new struct kvm_vcpu_arch. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 10 +- drivers/kvm/kvm_main.c | 2 +- drivers/kvm/lapic.c | 91 ++++---- drivers/kvm/mmu.c | 142 ++++++------- drivers/kvm/paging_tmpl.h | 16 +- drivers/kvm/svm.c | 122 +++++------ drivers/kvm/vmx.c | 213 +++++++++---------- drivers/kvm/x86.c | 522 +++++++++++++++++++++++----------------------- drivers/kvm/x86.h | 25 ++- drivers/kvm/x86_emulate.c | 24 +-- 10 files changed, 586 insertions(+), 581 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index e7debfafca50..04910368c251 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -158,7 +158,7 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, if (dest_mode == 0) { /* Physical mode. */ if (dest == 0xFF) { /* Broadcast. */ for (i = 0; i < KVM_MAX_VCPUS; ++i) - if (kvm->vcpus[i] && kvm->vcpus[i]->apic) + if (kvm->vcpus[i] && kvm->vcpus[i]->arch.apic) mask |= 1 << i; return mask; } @@ -166,8 +166,8 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, vcpu = kvm->vcpus[i]; if (!vcpu) continue; - if (kvm_apic_match_physical_addr(vcpu->apic, dest)) { - if (vcpu->apic) + if (kvm_apic_match_physical_addr(vcpu->arch.apic, dest)) { + if (vcpu->arch.apic) mask = 1 << i; break; } @@ -177,8 +177,8 @@ static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, vcpu = kvm->vcpus[i]; if (!vcpu) continue; - if (vcpu->apic && - kvm_apic_match_logical_addr(vcpu->apic, dest)) + if (vcpu->arch.apic && + kvm_apic_match_logical_addr(vcpu->arch.apic, dest)) mask |= 1 << vcpu->vcpu_id; } ioapic_debug("mask %x\n", mask); diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index 1a03b67eb921..ae2a1bf640bc 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -670,7 +670,7 @@ static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) if (vmf->pgoff == 0) page = virt_to_page(vcpu->run); else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) - page = virt_to_page(vcpu->pio_data); + page = virt_to_page(vcpu->arch.pio_data); else return VM_FAULT_SIGBUS; get_page(page); diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 466c37f02e85..5c9f46784c26 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -58,6 +58,7 @@ #define VEC_POS(v) ((v) & (32 - 1)) #define REG_POS(v) (((v) >> 5) << 4) + static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off) { return *((u32 *) (apic->regs + reg_off)); @@ -90,7 +91,7 @@ static inline void apic_clear_vector(int vec, void *bitmap) static inline int apic_hw_enabled(struct kvm_lapic *apic) { - return (apic)->vcpu->apic_base & MSR_IA32_APICBASE_ENABLE; + return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE; } static inline int apic_sw_enabled(struct kvm_lapic *apic) @@ -174,7 +175,7 @@ static inline int apic_find_highest_irr(struct kvm_lapic *apic) int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; int highest_irr; if (!apic) @@ -187,7 +188,7 @@ EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (!apic_test_and_set_irr(vec, apic)) { /* a new pending irq is set in IRR */ @@ -272,7 +273,7 @@ static int apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, int short_hand, int dest, int dest_mode) { int result = 0; - struct kvm_lapic *target = vcpu->apic; + struct kvm_lapic *target = vcpu->arch.apic; apic_debug("target %p, source %p, dest 0x%x, " "dest_mode 0x%x, short_hand 0x%x", @@ -339,10 +340,10 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, } else apic_clear_vector(vector, apic->regs + APIC_TMR); - if (vcpu->mp_state == VCPU_MP_STATE_RUNNABLE) + if (vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE) kvm_vcpu_kick(vcpu); - else if (vcpu->mp_state == VCPU_MP_STATE_HALTED) { - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + else if (vcpu->arch.mp_state == VCPU_MP_STATE_HALTED) { + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; if (waitqueue_active(&vcpu->wq)) wake_up_interruptible(&vcpu->wq); } @@ -363,11 +364,11 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, case APIC_DM_INIT: if (level) { - if (vcpu->mp_state == VCPU_MP_STATE_RUNNABLE) + if (vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE) printk(KERN_DEBUG "INIT on a runnable vcpu %d\n", vcpu->vcpu_id); - vcpu->mp_state = VCPU_MP_STATE_INIT_RECEIVED; + vcpu->arch.mp_state = VCPU_MP_STATE_INIT_RECEIVED; kvm_vcpu_kick(vcpu); } else { printk(KERN_DEBUG @@ -380,9 +381,9 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, case APIC_DM_STARTUP: printk(KERN_DEBUG "SIPI to vcpu %d vector 0x%02x\n", vcpu->vcpu_id, vector); - if (vcpu->mp_state == VCPU_MP_STATE_INIT_RECEIVED) { - vcpu->sipi_vector = vector; - vcpu->mp_state = VCPU_MP_STATE_SIPI_RECEIVED; + if (vcpu->arch.mp_state == VCPU_MP_STATE_INIT_RECEIVED) { + vcpu->arch.sipi_vector = vector; + vcpu->arch.mp_state = VCPU_MP_STATE_SIPI_RECEIVED; if (waitqueue_active(&vcpu->wq)) wake_up_interruptible(&vcpu->wq); } @@ -411,7 +412,7 @@ static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, next = 0; if (kvm->vcpus[next] == NULL || !test_bit(next, &bitmap)) continue; - apic = kvm->vcpus[next]->apic; + apic = kvm->vcpus[next]->arch.apic; if (apic && apic_enabled(apic)) break; apic = NULL; @@ -482,12 +483,12 @@ static void apic_send_ipi(struct kvm_lapic *apic) if (!vcpu) continue; - if (vcpu->apic && + if (vcpu->arch.apic && apic_match_dest(vcpu, apic, short_hand, dest, dest_mode)) { if (delivery_mode == APIC_DM_LOWEST) set_bit(vcpu->vcpu_id, &lpr_map); else - __apic_accept_irq(vcpu->apic, delivery_mode, + __apic_accept_irq(vcpu->arch.apic, delivery_mode, vector, level, trig_mode); } } @@ -495,7 +496,7 @@ static void apic_send_ipi(struct kvm_lapic *apic) if (delivery_mode == APIC_DM_LOWEST) { target = kvm_get_lowest_prio_vcpu(vcpu->kvm, vector, lpr_map); if (target != NULL) - __apic_accept_irq(target->apic, delivery_mode, + __apic_accept_irq(target->arch.apic, delivery_mode, vector, level, trig_mode); } } @@ -772,15 +773,15 @@ static int apic_mmio_range(struct kvm_io_device *this, gpa_t addr) void kvm_free_lapic(struct kvm_vcpu *vcpu) { - if (!vcpu->apic) + if (!vcpu->arch.apic) return; - hrtimer_cancel(&vcpu->apic->timer.dev); + hrtimer_cancel(&vcpu->arch.apic->timer.dev); - if (vcpu->apic->regs_page) - __free_page(vcpu->apic->regs_page); + if (vcpu->arch.apic->regs_page) + __free_page(vcpu->arch.apic->regs_page); - kfree(vcpu->apic); + kfree(vcpu->arch.apic); } /* @@ -791,7 +792,7 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu) void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (!apic) return; @@ -800,7 +801,7 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; u64 tpr; if (!apic) @@ -813,29 +814,29 @@ EXPORT_SYMBOL_GPL(kvm_lapic_get_cr8); void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (!apic) { value |= MSR_IA32_APICBASE_BSP; - vcpu->apic_base = value; + vcpu->arch.apic_base = value; return; } if (apic->vcpu->vcpu_id) value &= ~MSR_IA32_APICBASE_BSP; - vcpu->apic_base = value; - apic->base_address = apic->vcpu->apic_base & + vcpu->arch.apic_base = value; + apic->base_address = apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_BASE; /* with FSB delivery interrupt, we can restart APIC functionality */ apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " - "0x%lx.\n", apic->vcpu->apic_base, apic->base_address); + "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); } u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu) { - return vcpu->apic_base; + return vcpu->arch.apic_base; } EXPORT_SYMBOL_GPL(kvm_lapic_get_base); @@ -847,7 +848,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu) apic_debug("%s\n", __FUNCTION__); ASSERT(vcpu); - apic = vcpu->apic; + apic = vcpu->arch.apic; ASSERT(apic != NULL); /* Stop the timer in case it's a reset to an active apic */ @@ -878,19 +879,19 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu) update_divide_count(apic); atomic_set(&apic->timer.pending, 0); if (vcpu->vcpu_id == 0) - vcpu->apic_base |= MSR_IA32_APICBASE_BSP; + vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP; apic_update_ppr(apic); apic_debug(KERN_INFO "%s: vcpu=%p, id=%d, base_msr=" "0x%016" PRIx64 ", base_address=0x%0lx.\n", __FUNCTION__, vcpu, kvm_apic_id(apic), - vcpu->apic_base, apic->base_address); + vcpu->arch.apic_base, apic->base_address); } EXPORT_SYMBOL_GPL(kvm_lapic_reset); int kvm_lapic_enabled(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; int ret = 0; if (!apic) @@ -915,7 +916,7 @@ static int __apic_timer_fn(struct kvm_lapic *apic) atomic_inc(&apic->timer.pending); if (waitqueue_active(q)) { - apic->vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + apic->vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; wake_up_interruptible(q); } if (apic_lvtt_period(apic)) { @@ -961,7 +962,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) if (!apic) goto nomem; - vcpu->apic = apic; + vcpu->arch.apic = apic; apic->regs_page = alloc_page(GFP_KERNEL); if (apic->regs_page == NULL) { @@ -976,7 +977,7 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu) hrtimer_init(&apic->timer.dev, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); apic->timer.dev.function = apic_timer_fn; apic->base_address = APIC_DEFAULT_PHYS_BASE; - vcpu->apic_base = APIC_DEFAULT_PHYS_BASE; + vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE; kvm_lapic_reset(vcpu); apic->dev.read = apic_mmio_read; @@ -994,7 +995,7 @@ EXPORT_SYMBOL_GPL(kvm_create_lapic); int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; int highest_irr; if (!apic || !apic_enabled(apic)) @@ -1010,11 +1011,11 @@ int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) { - u32 lvt0 = apic_get_reg(vcpu->apic, APIC_LVT0); + u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0); int r = 0; if (vcpu->vcpu_id == 0) { - if (!apic_hw_enabled(vcpu->apic)) + if (!apic_hw_enabled(vcpu->arch.apic)) r = 1; if ((lvt0 & APIC_LVT_MASKED) == 0 && GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) @@ -1025,7 +1026,7 @@ int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (apic && apic_lvt_enabled(apic, APIC_LVTT) && atomic_read(&apic->timer.pending) > 0) { @@ -1036,7 +1037,7 @@ void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (apic && apic_lvt_vector(apic, APIC_LVTT) == vec) apic->timer.last_update = ktime_add_ns( @@ -1047,7 +1048,7 @@ void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec) int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) { int vector = kvm_apic_has_interrupt(vcpu); - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; if (vector == -1) return -1; @@ -1060,9 +1061,9 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; - apic->base_address = vcpu->apic_base & + apic->base_address = vcpu->arch.apic_base & MSR_IA32_APICBASE_BASE; apic_set_reg(apic, APIC_LVR, APIC_VERSION); apic_update_ppr(apic); @@ -1073,7 +1074,7 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) { - struct kvm_lapic *apic = vcpu->apic; + struct kvm_lapic *apic = vcpu->arch.apic; struct hrtimer *timer; if (!apic) diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 92ac0d1106b4..da1dedb497b8 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -180,7 +180,7 @@ EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes); static int is_write_protection(struct kvm_vcpu *vcpu) { - return vcpu->cr0 & X86_CR0_WP; + return vcpu->arch.cr0 & X86_CR0_WP; } static int is_cpuid_PSE36(void) @@ -190,7 +190,7 @@ static int is_cpuid_PSE36(void) static int is_nx(struct kvm_vcpu *vcpu) { - return vcpu->shadow_efer & EFER_NX; + return vcpu->arch.shadow_efer & EFER_NX; } static int is_present_pte(unsigned long pte) @@ -292,18 +292,18 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) int r; kvm_mmu_free_some_pages(vcpu); - r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache, + r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache, pte_chain_cache, 4); if (r) goto out; - r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache, + r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, rmap_desc_cache, 1); if (r) goto out; - r = mmu_topup_memory_cache_page(&vcpu->mmu_page_cache, 8); + r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8); if (r) goto out; - r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache, + r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache, mmu_page_header_cache, 4); out: return r; @@ -311,10 +311,10 @@ out: static void mmu_free_memory_caches(struct kvm_vcpu *vcpu) { - mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache); - mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache); - mmu_free_memory_cache_page(&vcpu->mmu_page_cache); - mmu_free_memory_cache(&vcpu->mmu_page_header_cache); + mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache); + mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache); + mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache); + mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache); } static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, @@ -330,7 +330,7 @@ static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu) { - return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache, + return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache, sizeof(struct kvm_pte_chain)); } @@ -341,7 +341,7 @@ static void mmu_free_pte_chain(struct kvm_pte_chain *pc) static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu) { - return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache, + return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache, sizeof(struct kvm_rmap_desc)); } @@ -568,9 +568,9 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, if (!vcpu->kvm->n_free_mmu_pages) return NULL; - sp = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache, sizeof *sp); - sp->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); - sp->gfns = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE); + sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp); + sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); + sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); set_page_private(virt_to_page(sp->spt), (unsigned long)sp); list_add(&sp->link, &vcpu->kvm->active_mmu_pages); ASSERT(is_empty_shadow_page(sp->spt)); @@ -692,11 +692,11 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, struct hlist_node *node; role.word = 0; - role.glevels = vcpu->mmu.root_level; + role.glevels = vcpu->arch.mmu.root_level; role.level = level; role.metaphysical = metaphysical; role.access = access; - if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) { + if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) { quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; role.quadrant = quadrant; @@ -718,7 +718,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, sp->gfn = gfn; sp->role = role; hlist_add_head(&sp->hash_link, bucket); - vcpu->mmu.prefetch_page(vcpu, sp); + vcpu->arch.mmu.prefetch_page(vcpu, sp); if (!metaphysical) rmap_write_protect(vcpu->kvm, gfn); if (new_page) @@ -768,7 +768,7 @@ static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) for (i = 0; i < KVM_MAX_VCPUS; ++i) if (kvm->vcpus[i]) - kvm->vcpus[i]->last_pte_updated = NULL; + kvm->vcpus[i]->arch.last_pte_updated = NULL; } static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) @@ -875,7 +875,7 @@ static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); if (gpa == UNMAPPED_GVA) return NULL; @@ -962,7 +962,7 @@ unshadowed: else kvm_release_page_clean(page); if (!ptwrite || !*ptwrite) - vcpu->last_pte_updated = shadow_pte; + vcpu->arch.last_pte_updated = shadow_pte; } static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) @@ -972,7 +972,7 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) { int level = PT32E_ROOT_LEVEL; - hpa_t table_addr = vcpu->mmu.root_hpa; + hpa_t table_addr = vcpu->arch.mmu.root_hpa; int pt_write = 0; for (; ; level--) { @@ -1024,29 +1024,29 @@ static void mmu_free_roots(struct kvm_vcpu *vcpu) int i; struct kvm_mmu_page *sp; - if (!VALID_PAGE(vcpu->mmu.root_hpa)) + if (!VALID_PAGE(vcpu->arch.mmu.root_hpa)) return; #ifdef CONFIG_X86_64 - if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) { - hpa_t root = vcpu->mmu.root_hpa; + if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { + hpa_t root = vcpu->arch.mmu.root_hpa; sp = page_header(root); --sp->root_count; - vcpu->mmu.root_hpa = INVALID_PAGE; + vcpu->arch.mmu.root_hpa = INVALID_PAGE; return; } #endif for (i = 0; i < 4; ++i) { - hpa_t root = vcpu->mmu.pae_root[i]; + hpa_t root = vcpu->arch.mmu.pae_root[i]; if (root) { root &= PT64_BASE_ADDR_MASK; sp = page_header(root); --sp->root_count; } - vcpu->mmu.pae_root[i] = INVALID_PAGE; + vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; } - vcpu->mmu.root_hpa = INVALID_PAGE; + vcpu->arch.mmu.root_hpa = INVALID_PAGE; } static void mmu_alloc_roots(struct kvm_vcpu *vcpu) @@ -1055,41 +1055,41 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) gfn_t root_gfn; struct kvm_mmu_page *sp; - root_gfn = vcpu->cr3 >> PAGE_SHIFT; + root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT; #ifdef CONFIG_X86_64 - if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) { - hpa_t root = vcpu->mmu.root_hpa; + if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { + hpa_t root = vcpu->arch.mmu.root_hpa; ASSERT(!VALID_PAGE(root)); sp = kvm_mmu_get_page(vcpu, root_gfn, 0, PT64_ROOT_LEVEL, 0, ACC_ALL, NULL, NULL); root = __pa(sp->spt); ++sp->root_count; - vcpu->mmu.root_hpa = root; + vcpu->arch.mmu.root_hpa = root; return; } #endif for (i = 0; i < 4; ++i) { - hpa_t root = vcpu->mmu.pae_root[i]; + hpa_t root = vcpu->arch.mmu.pae_root[i]; ASSERT(!VALID_PAGE(root)); - if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) { - if (!is_present_pte(vcpu->pdptrs[i])) { - vcpu->mmu.pae_root[i] = 0; + if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) { + if (!is_present_pte(vcpu->arch.pdptrs[i])) { + vcpu->arch.mmu.pae_root[i] = 0; continue; } - root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT; - } else if (vcpu->mmu.root_level == 0) + root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT; + } else if (vcpu->arch.mmu.root_level == 0) root_gfn = 0; sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL, !is_paging(vcpu), ACC_ALL, NULL, NULL); root = __pa(sp->spt); ++sp->root_count; - vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK; + vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK; } - vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root); + vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root); } static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) @@ -1109,7 +1109,7 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, return r; ASSERT(vcpu); - ASSERT(VALID_PAGE(vcpu->mmu.root_hpa)); + ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa)); gfn = gva >> PAGE_SHIFT; @@ -1124,7 +1124,7 @@ static void nonpaging_free(struct kvm_vcpu *vcpu) static int nonpaging_init_context(struct kvm_vcpu *vcpu) { - struct kvm_mmu *context = &vcpu->mmu; + struct kvm_mmu *context = &vcpu->arch.mmu; context->new_cr3 = nonpaging_new_cr3; context->page_fault = nonpaging_page_fault; @@ -1171,7 +1171,7 @@ static void paging_free(struct kvm_vcpu *vcpu) static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level) { - struct kvm_mmu *context = &vcpu->mmu; + struct kvm_mmu *context = &vcpu->arch.mmu; ASSERT(is_pae(vcpu)); context->new_cr3 = paging_new_cr3; @@ -1192,7 +1192,7 @@ static int paging64_init_context(struct kvm_vcpu *vcpu) static int paging32_init_context(struct kvm_vcpu *vcpu) { - struct kvm_mmu *context = &vcpu->mmu; + struct kvm_mmu *context = &vcpu->arch.mmu; context->new_cr3 = paging_new_cr3; context->page_fault = paging32_page_fault; @@ -1213,7 +1213,7 @@ static int paging32E_init_context(struct kvm_vcpu *vcpu) static int init_kvm_mmu(struct kvm_vcpu *vcpu) { ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa)); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); if (!is_paging(vcpu)) return nonpaging_init_context(vcpu); @@ -1228,9 +1228,9 @@ static int init_kvm_mmu(struct kvm_vcpu *vcpu) static void destroy_kvm_mmu(struct kvm_vcpu *vcpu) { ASSERT(vcpu); - if (VALID_PAGE(vcpu->mmu.root_hpa)) { - vcpu->mmu.free(vcpu); - vcpu->mmu.root_hpa = INVALID_PAGE; + if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) { + vcpu->arch.mmu.free(vcpu); + vcpu->arch.mmu.root_hpa = INVALID_PAGE; } } @@ -1250,7 +1250,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) if (r) goto out; mmu_alloc_roots(vcpu); - kvm_x86_ops->set_cr3(vcpu, vcpu->mmu.root_hpa); + kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa); kvm_mmu_flush_tlb(vcpu); out: mutex_unlock(&vcpu->kvm->lock); @@ -1323,7 +1323,7 @@ static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new) static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) { - u64 *spte = vcpu->last_pte_updated; + u64 *spte = vcpu->arch.last_pte_updated; return !!(spte && (*spte & PT_ACCESSED_MASK)); } @@ -1350,15 +1350,15 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); ++vcpu->kvm->stat.mmu_pte_write; kvm_mmu_audit(vcpu, "pre pte write"); - if (gfn == vcpu->last_pt_write_gfn + if (gfn == vcpu->arch.last_pt_write_gfn && !last_updated_pte_accessed(vcpu)) { - ++vcpu->last_pt_write_count; - if (vcpu->last_pt_write_count >= 3) + ++vcpu->arch.last_pt_write_count; + if (vcpu->arch.last_pt_write_count >= 3) flooded = 1; } else { - vcpu->last_pt_write_gfn = gfn; - vcpu->last_pt_write_count = 1; - vcpu->last_pte_updated = NULL; + vcpu->arch.last_pt_write_gfn = gfn; + vcpu->arch.last_pt_write_count = 1; + vcpu->arch.last_pte_updated = NULL; } index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; bucket = &vcpu->kvm->mmu_page_hash[index]; @@ -1420,7 +1420,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); } @@ -1443,7 +1443,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) enum emulation_result er; mutex_lock(&vcpu->kvm->lock); - r = vcpu->mmu.page_fault(vcpu, cr2, error_code); + r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code); if (r < 0) goto out; @@ -1486,7 +1486,7 @@ static void free_mmu_pages(struct kvm_vcpu *vcpu) struct kvm_mmu_page, link); kvm_mmu_zap_page(vcpu->kvm, sp); } - free_page((unsigned long)vcpu->mmu.pae_root); + free_page((unsigned long)vcpu->arch.mmu.pae_root); } static int alloc_mmu_pages(struct kvm_vcpu *vcpu) @@ -1508,9 +1508,9 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) page = alloc_page(GFP_KERNEL | __GFP_DMA32); if (!page) goto error_1; - vcpu->mmu.pae_root = page_address(page); + vcpu->arch.mmu.pae_root = page_address(page); for (i = 0; i < 4; ++i) - vcpu->mmu.pae_root[i] = INVALID_PAGE; + vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; return 0; @@ -1522,7 +1522,7 @@ error_1: int kvm_mmu_create(struct kvm_vcpu *vcpu) { ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa)); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); return alloc_mmu_pages(vcpu); } @@ -1530,7 +1530,7 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu) int kvm_mmu_setup(struct kvm_vcpu *vcpu) { ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa)); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); return init_kvm_mmu(vcpu); } @@ -1659,11 +1659,11 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, printk(KERN_ERR "audit: (%s) nontrapping pte" " in nonleaf level: levels %d gva %lx" " level %d pte %llx\n", audit_msg, - vcpu->mmu.root_level, va, level, ent); + vcpu->arch.mmu.root_level, va, level, ent); audit_mappings_page(vcpu, ent, va, level - 1); } else { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va); struct page *page = gpa_to_page(vcpu, gpa); hpa_t hpa = page_to_phys(page); @@ -1671,7 +1671,7 @@ static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, && (ent & PT64_BASE_ADDR_MASK) != hpa) printk(KERN_ERR "xx audit error: (%s) levels %d" " gva %lx gpa %llx hpa %llx ent %llx %d\n", - audit_msg, vcpu->mmu.root_level, + audit_msg, vcpu->arch.mmu.root_level, va, gpa, hpa, ent, is_shadow_present_pte(ent)); else if (ent == shadow_notrap_nonpresent_pte @@ -1688,13 +1688,13 @@ static void audit_mappings(struct kvm_vcpu *vcpu) { unsigned i; - if (vcpu->mmu.root_level == 4) - audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4); + if (vcpu->arch.mmu.root_level == 4) + audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4); else for (i = 0; i < 4; ++i) - if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK) + if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK) audit_mappings_page(vcpu, - vcpu->mmu.pae_root[i], + vcpu->arch.mmu.pae_root[i], i << 30, 2); } diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h index fb19596c9589..56b88f7e83ef 100644 --- a/drivers/kvm/paging_tmpl.h +++ b/drivers/kvm/paging_tmpl.h @@ -129,11 +129,11 @@ static int FNAME(walk_addr)(struct guest_walker *walker, pgprintk("%s: addr %lx\n", __FUNCTION__, addr); walk: - walker->level = vcpu->mmu.root_level; - pte = vcpu->cr3; + walker->level = vcpu->arch.mmu.root_level; + pte = vcpu->arch.cr3; #if PTTYPE == 64 if (!is_long_mode(vcpu)) { - pte = vcpu->pdptrs[(addr >> 30) & 3]; + pte = vcpu->arch.pdptrs[(addr >> 30) & 3]; if (!is_present_pte(pte)) goto not_present; --walker->level; @@ -275,10 +275,10 @@ static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, if (!is_present_pte(walker->ptes[walker->level - 1])) return NULL; - shadow_addr = vcpu->mmu.root_hpa; - level = vcpu->mmu.shadow_root_level; + shadow_addr = vcpu->arch.mmu.root_hpa; + level = vcpu->arch.mmu.shadow_root_level; if (level == PT32E_ROOT_LEVEL) { - shadow_addr = vcpu->mmu.pae_root[(addr >> 30) & 3]; + shadow_addr = vcpu->arch.mmu.pae_root[(addr >> 30) & 3]; shadow_addr &= PT64_BASE_ADDR_MASK; --level; } @@ -380,7 +380,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, if (!r) { pgprintk("%s: guest page fault\n", __FUNCTION__); inject_page_fault(vcpu, addr, walker.error_code); - vcpu->last_pt_write_count = 0; /* reset fork detector */ + vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ return 0; } @@ -390,7 +390,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, shadow_pte, *shadow_pte, write_pt); if (!write_pt) - vcpu->last_pt_write_count = 0; /* reset fork detector */ + vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ /* * mmio: emulate if accessible, otherwise its a guest fault. diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index ef21804a5c5c..7888638c02e8 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -99,20 +99,20 @@ static inline u32 svm_has(u32 feat) static inline u8 pop_irq(struct kvm_vcpu *vcpu) { - int word_index = __ffs(vcpu->irq_summary); - int bit_index = __ffs(vcpu->irq_pending[word_index]); + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); int irq = word_index * BITS_PER_LONG + bit_index; - clear_bit(bit_index, &vcpu->irq_pending[word_index]); - if (!vcpu->irq_pending[word_index]) - clear_bit(word_index, &vcpu->irq_summary); + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); return irq; } static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq) { - set_bit(irq, vcpu->irq_pending); - set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary); + set_bit(irq, vcpu->arch.irq_pending); + set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary); } static inline void clgi(void) @@ -185,7 +185,7 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) efer &= ~EFER_LME; to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK; - vcpu->shadow_efer = efer; + vcpu->arch.shadow_efer = efer; } static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, @@ -227,10 +227,10 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) svm->vmcb->save.rip, svm->next_rip); - vcpu->rip = svm->vmcb->save.rip = svm->next_rip; + vcpu->arch.rip = svm->vmcb->save.rip = svm->next_rip; svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; - vcpu->interrupt_window_open = 1; + vcpu->arch.interrupt_window_open = 1; } static int has_svm(void) @@ -559,8 +559,8 @@ static int svm_vcpu_reset(struct kvm_vcpu *vcpu) if (vcpu->vcpu_id != 0) { svm->vmcb->save.rip = 0; - svm->vmcb->save.cs.base = svm->vcpu.sipi_vector << 12; - svm->vmcb->save.cs.selector = svm->vcpu.sipi_vector << 8; + svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12; + svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8; } return 0; @@ -597,9 +597,9 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) fx_init(&svm->vcpu); svm->vcpu.fpu_active = 1; - svm->vcpu.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; + svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; if (svm->vcpu.vcpu_id == 0) - svm->vcpu.apic_base |= MSR_IA32_APICBASE_BSP; + svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP; return &svm->vcpu; @@ -633,7 +633,7 @@ static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) * increasing TSC. */ rdtscll(tsc_this); - delta = vcpu->host_tsc - tsc_this; + delta = vcpu->arch.host_tsc - tsc_this; svm->vmcb->control.tsc_offset += delta; vcpu->cpu = cpu; kvm_migrate_apic_timer(vcpu); @@ -652,7 +652,7 @@ static void svm_vcpu_put(struct kvm_vcpu *vcpu) for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); - rdtscll(vcpu->host_tsc); + rdtscll(vcpu->arch.host_tsc); } static void svm_vcpu_decache(struct kvm_vcpu *vcpu) @@ -663,17 +663,17 @@ static void svm_cache_regs(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - vcpu->regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; - vcpu->regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; - vcpu->rip = svm->vmcb->save.rip; + vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; + vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; + vcpu->arch.rip = svm->vmcb->save.rip; } static void svm_decache_regs(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - svm->vmcb->save.rax = vcpu->regs[VCPU_REGS_RAX]; - svm->vmcb->save.rsp = vcpu->regs[VCPU_REGS_RSP]; - svm->vmcb->save.rip = vcpu->rip; + svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; + svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; + svm->vmcb->save.rip = vcpu->arch.rip; } static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) @@ -771,24 +771,24 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) struct vcpu_svm *svm = to_svm(vcpu); #ifdef CONFIG_X86_64 - if (vcpu->shadow_efer & EFER_LME) { + if (vcpu->arch.shadow_efer & EFER_LME) { if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { - vcpu->shadow_efer |= EFER_LMA; + vcpu->arch.shadow_efer |= EFER_LMA; svm->vmcb->save.efer |= EFER_LMA | EFER_LME; } if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { - vcpu->shadow_efer &= ~EFER_LMA; + vcpu->arch.shadow_efer &= ~EFER_LMA; svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); } } #endif - if ((vcpu->cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) { + if ((vcpu->arch.cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) { svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); vcpu->fpu_active = 1; } - vcpu->cr0 = cr0; + vcpu->arch.cr0 = cr0; cr0 |= X86_CR0_PG | X86_CR0_WP; cr0 &= ~(X86_CR0_CD | X86_CR0_NW); svm->vmcb->save.cr0 = cr0; @@ -796,7 +796,7 @@ static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { - vcpu->cr4 = cr4; + vcpu->arch.cr4 = cr4; to_svm(vcpu)->vmcb->save.cr4 = cr4 | X86_CR4_PAE; } @@ -901,7 +901,7 @@ static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value, svm->db_regs[dr] = value; return; case 4 ... 5: - if (vcpu->cr4 & X86_CR4_DE) { + if (vcpu->arch.cr4 & X86_CR4_DE) { *exception = UD_VECTOR; return; } @@ -950,7 +950,7 @@ static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); - if (!(svm->vcpu.cr0 & X86_CR0_TS)) + if (!(svm->vcpu.arch.cr0 & X86_CR0_TS)) svm->vmcb->save.cr0 &= ~X86_CR0_TS; svm->vcpu.fpu_active = 1; @@ -1103,14 +1103,14 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - u32 ecx = svm->vcpu.regs[VCPU_REGS_RCX]; + u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; u64 data; if (svm_get_msr(&svm->vcpu, ecx, &data)) kvm_inject_gp(&svm->vcpu, 0); else { svm->vmcb->save.rax = data & 0xffffffff; - svm->vcpu.regs[VCPU_REGS_RDX] = data >> 32; + svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32; svm->next_rip = svm->vmcb->save.rip + 2; skip_emulated_instruction(&svm->vcpu); } @@ -1176,9 +1176,9 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) { - u32 ecx = svm->vcpu.regs[VCPU_REGS_RCX]; + u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; u64 data = (svm->vmcb->save.rax & -1u) - | ((u64)(svm->vcpu.regs[VCPU_REGS_RDX] & -1u) << 32); + | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32); svm->next_rip = svm->vmcb->save.rip + 2; if (svm_set_msr(&svm->vcpu, ecx, data)) kvm_inject_gp(&svm->vcpu, 0); @@ -1205,7 +1205,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm, * possible */ if (kvm_run->request_interrupt_window && - !svm->vcpu.irq_summary) { + !svm->vcpu.arch.irq_summary) { ++svm->vcpu.stat.irq_window_exits; kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; return 0; @@ -1382,20 +1382,20 @@ static void kvm_reput_irq(struct vcpu_svm *svm) push_irq(&svm->vcpu, control->int_vector); } - svm->vcpu.interrupt_window_open = + svm->vcpu.arch.interrupt_window_open = !(control->int_state & SVM_INTERRUPT_SHADOW_MASK); } static void svm_do_inject_vector(struct vcpu_svm *svm) { struct kvm_vcpu *vcpu = &svm->vcpu; - int word_index = __ffs(vcpu->irq_summary); - int bit_index = __ffs(vcpu->irq_pending[word_index]); + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); int irq = word_index * BITS_PER_LONG + bit_index; - clear_bit(bit_index, &vcpu->irq_pending[word_index]); - if (!vcpu->irq_pending[word_index]) - clear_bit(word_index, &vcpu->irq_summary); + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); svm_inject_irq(svm, irq); } @@ -1405,11 +1405,11 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, struct vcpu_svm *svm = to_svm(vcpu); struct vmcb_control_area *control = &svm->vmcb->control; - svm->vcpu.interrupt_window_open = + svm->vcpu.arch.interrupt_window_open = (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) && (svm->vmcb->save.rflags & X86_EFLAGS_IF)); - if (svm->vcpu.interrupt_window_open && svm->vcpu.irq_summary) + if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary) /* * If interrupts enabled, and not blocked by sti or mov ss. Good. */ @@ -1418,8 +1418,8 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, /* * Interrupts blocked. Wait for unblock. */ - if (!svm->vcpu.interrupt_window_open && - (svm->vcpu.irq_summary || kvm_run->request_interrupt_window)) + if (!svm->vcpu.arch.interrupt_window_open && + (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window)) control->intercept |= 1ULL << INTERCEPT_VINTR; else control->intercept &= ~(1ULL << INTERCEPT_VINTR); @@ -1471,7 +1471,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) svm->host_cr2 = kvm_read_cr2(); svm->host_dr6 = read_dr6(); svm->host_dr7 = read_dr7(); - svm->vmcb->save.cr2 = vcpu->cr2; + svm->vmcb->save.cr2 = vcpu->arch.cr2; if (svm->vmcb->save.dr7 & 0xff) { write_dr7(0); @@ -1563,21 +1563,21 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) : : [svm]"a"(svm), [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), - [rbx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_RBP])) + [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP])) #ifdef CONFIG_X86_64 - , [r8]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_svm, vcpu.regs[VCPU_REGS_R15])) + , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15])) #endif : "cc", "memory" #ifdef CONFIG_X86_64 @@ -1591,7 +1591,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if ((svm->vmcb->save.dr7 & 0xff)) load_db_regs(svm->host_db_regs); - vcpu->cr2 = svm->vmcb->save.cr2; + vcpu->arch.cr2 = svm->vmcb->save.cr2; write_dr6(svm->host_dr6); write_dr7(svm->host_dr7); diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 83084348581a..cf78ebb2f36e 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -247,7 +247,7 @@ static void __vcpu_clear(void *arg) vmcs_clear(vmx->vmcs); if (per_cpu(current_vmcs, cpu) == vmx->vmcs) per_cpu(current_vmcs, cpu) = NULL; - rdtscll(vmx->vcpu.host_tsc); + rdtscll(vmx->vcpu.arch.host_tsc); } static void vcpu_clear(struct vcpu_vmx *vmx) @@ -343,7 +343,7 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu) eb |= 1u << NM_VECTOR; if (vcpu->guest_debug.enabled) eb |= 1u << 1; - if (vcpu->rmode.active) + if (vcpu->arch.rmode.active) eb = ~0; vmcs_write32(EXCEPTION_BITMAP, eb); } @@ -528,7 +528,7 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) * Make sure the time stamp counter is monotonous. */ rdtscll(tsc_this); - delta = vcpu->host_tsc - tsc_this; + delta = vcpu->arch.host_tsc - tsc_this; vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta); } } @@ -544,7 +544,7 @@ static void vmx_fpu_activate(struct kvm_vcpu *vcpu) return; vcpu->fpu_active = 1; vmcs_clear_bits(GUEST_CR0, X86_CR0_TS); - if (vcpu->cr0 & X86_CR0_TS) + if (vcpu->arch.cr0 & X86_CR0_TS) vmcs_set_bits(GUEST_CR0, X86_CR0_TS); update_exception_bitmap(vcpu); } @@ -570,7 +570,7 @@ static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) { - if (vcpu->rmode.active) + if (vcpu->arch.rmode.active) rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; vmcs_writel(GUEST_RFLAGS, rflags); } @@ -592,7 +592,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) if (interruptibility & 3) vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, interruptibility & ~3); - vcpu->interrupt_window_open = 1; + vcpu->arch.interrupt_window_open = 1; } static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, @@ -661,7 +661,7 @@ static void setup_msrs(struct vcpu_vmx *vmx) * if efer.sce is enabled. */ index = __find_msr_index(vmx, MSR_K6_STAR); - if ((index >= 0) && (vmx->vcpu.shadow_efer & EFER_SCE)) + if ((index >= 0) && (vmx->vcpu.arch.shadow_efer & EFER_SCE)) move_msr_up(vmx, index, save_nmsrs++); } #endif @@ -805,12 +805,12 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) /* * Sync the rsp and rip registers into the vcpu structure. This allows - * registers to be accessed by indexing vcpu->regs. + * registers to be accessed by indexing vcpu->arch.regs. */ static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu) { - vcpu->regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); - vcpu->rip = vmcs_readl(GUEST_RIP); + vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); + vcpu->arch.rip = vmcs_readl(GUEST_RIP); } /* @@ -819,8 +819,8 @@ static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu) */ static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu) { - vmcs_writel(GUEST_RSP, vcpu->regs[VCPU_REGS_RSP]); - vmcs_writel(GUEST_RIP, vcpu->rip); + vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); + vmcs_writel(GUEST_RIP, vcpu->arch.rip); } static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) @@ -1111,15 +1111,15 @@ static void enter_pmode(struct kvm_vcpu *vcpu) { unsigned long flags; - vcpu->rmode.active = 0; + vcpu->arch.rmode.active = 0; - vmcs_writel(GUEST_TR_BASE, vcpu->rmode.tr.base); - vmcs_write32(GUEST_TR_LIMIT, vcpu->rmode.tr.limit); - vmcs_write32(GUEST_TR_AR_BYTES, vcpu->rmode.tr.ar); + vmcs_writel(GUEST_TR_BASE, vcpu->arch.rmode.tr.base); + vmcs_write32(GUEST_TR_LIMIT, vcpu->arch.rmode.tr.limit); + vmcs_write32(GUEST_TR_AR_BYTES, vcpu->arch.rmode.tr.ar); flags = vmcs_readl(GUEST_RFLAGS); flags &= ~(X86_EFLAGS_IOPL | X86_EFLAGS_VM); - flags |= (vcpu->rmode.save_iopl << IOPL_SHIFT); + flags |= (vcpu->arch.rmode.save_iopl << IOPL_SHIFT); vmcs_writel(GUEST_RFLAGS, flags); vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | @@ -1127,10 +1127,10 @@ static void enter_pmode(struct kvm_vcpu *vcpu) update_exception_bitmap(vcpu); - fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->rmode.es); - fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->rmode.ds); - fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->rmode.gs); - fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->rmode.fs); + fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->arch.rmode.es); + fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->arch.rmode.ds); + fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->arch.rmode.gs); + fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->arch.rmode.fs); vmcs_write16(GUEST_SS_SELECTOR, 0); vmcs_write32(GUEST_SS_AR_BYTES, 0x93); @@ -1168,19 +1168,20 @@ static void enter_rmode(struct kvm_vcpu *vcpu) { unsigned long flags; - vcpu->rmode.active = 1; + vcpu->arch.rmode.active = 1; - vcpu->rmode.tr.base = vmcs_readl(GUEST_TR_BASE); + vcpu->arch.rmode.tr.base = vmcs_readl(GUEST_TR_BASE); vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm)); - vcpu->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT); + vcpu->arch.rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT); vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); - vcpu->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES); + vcpu->arch.rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES); vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); flags = vmcs_readl(GUEST_RFLAGS); - vcpu->rmode.save_iopl = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; + vcpu->arch.rmode.save_iopl + = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; @@ -1198,10 +1199,10 @@ static void enter_rmode(struct kvm_vcpu *vcpu) vmcs_writel(GUEST_CS_BASE, 0xf0000); vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4); - fix_rmode_seg(VCPU_SREG_ES, &vcpu->rmode.es); - fix_rmode_seg(VCPU_SREG_DS, &vcpu->rmode.ds); - fix_rmode_seg(VCPU_SREG_GS, &vcpu->rmode.gs); - fix_rmode_seg(VCPU_SREG_FS, &vcpu->rmode.fs); + fix_rmode_seg(VCPU_SREG_ES, &vcpu->arch.rmode.es); + fix_rmode_seg(VCPU_SREG_DS, &vcpu->arch.rmode.ds); + fix_rmode_seg(VCPU_SREG_GS, &vcpu->arch.rmode.gs); + fix_rmode_seg(VCPU_SREG_FS, &vcpu->arch.rmode.fs); kvm_mmu_reset_context(vcpu); init_rmode_tss(vcpu->kvm); @@ -1222,7 +1223,7 @@ static void enter_lmode(struct kvm_vcpu *vcpu) | AR_TYPE_BUSY_64_TSS); } - vcpu->shadow_efer |= EFER_LMA; + vcpu->arch.shadow_efer |= EFER_LMA; find_msr_entry(to_vmx(vcpu), MSR_EFER)->data |= EFER_LMA | EFER_LME; vmcs_write32(VM_ENTRY_CONTROLS, @@ -1232,7 +1233,7 @@ static void enter_lmode(struct kvm_vcpu *vcpu) static void exit_lmode(struct kvm_vcpu *vcpu) { - vcpu->shadow_efer &= ~EFER_LMA; + vcpu->arch.shadow_efer &= ~EFER_LMA; vmcs_write32(VM_ENTRY_CONTROLS, vmcs_read32(VM_ENTRY_CONTROLS) @@ -1243,22 +1244,22 @@ static void exit_lmode(struct kvm_vcpu *vcpu) static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) { - vcpu->cr4 &= KVM_GUEST_CR4_MASK; - vcpu->cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK; + vcpu->arch.cr4 &= KVM_GUEST_CR4_MASK; + vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK; } static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { vmx_fpu_deactivate(vcpu); - if (vcpu->rmode.active && (cr0 & X86_CR0_PE)) + if (vcpu->arch.rmode.active && (cr0 & X86_CR0_PE)) enter_pmode(vcpu); - if (!vcpu->rmode.active && !(cr0 & X86_CR0_PE)) + if (!vcpu->arch.rmode.active && !(cr0 & X86_CR0_PE)) enter_rmode(vcpu); #ifdef CONFIG_X86_64 - if (vcpu->shadow_efer & EFER_LME) { + if (vcpu->arch.shadow_efer & EFER_LME) { if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) enter_lmode(vcpu); if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) @@ -1269,7 +1270,7 @@ static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) vmcs_writel(CR0_READ_SHADOW, cr0); vmcs_writel(GUEST_CR0, (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON); - vcpu->cr0 = cr0; + vcpu->arch.cr0 = cr0; if (!(cr0 & X86_CR0_TS) || !(cr0 & X86_CR0_PE)) vmx_fpu_activate(vcpu); @@ -1278,16 +1279,16 @@ static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { vmcs_writel(GUEST_CR3, cr3); - if (vcpu->cr0 & X86_CR0_PE) + if (vcpu->arch.cr0 & X86_CR0_PE) vmx_fpu_deactivate(vcpu); } static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) { vmcs_writel(CR4_READ_SHADOW, cr4); - vmcs_writel(GUEST_CR4, cr4 | (vcpu->rmode.active ? + vmcs_writel(GUEST_CR4, cr4 | (vcpu->arch.rmode.active ? KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON)); - vcpu->cr4 = cr4; + vcpu->arch.cr4 = cr4; } #ifdef CONFIG_X86_64 @@ -1297,7 +1298,7 @@ static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) struct vcpu_vmx *vmx = to_vmx(vcpu); struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); - vcpu->shadow_efer = efer; + vcpu->arch.shadow_efer = efer; if (efer & EFER_LMA) { vmcs_write32(VM_ENTRY_CONTROLS, vmcs_read32(VM_ENTRY_CONTROLS) | @@ -1374,17 +1375,17 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; u32 ar; - if (vcpu->rmode.active && seg == VCPU_SREG_TR) { - vcpu->rmode.tr.selector = var->selector; - vcpu->rmode.tr.base = var->base; - vcpu->rmode.tr.limit = var->limit; - vcpu->rmode.tr.ar = vmx_segment_access_rights(var); + if (vcpu->arch.rmode.active && seg == VCPU_SREG_TR) { + vcpu->arch.rmode.tr.selector = var->selector; + vcpu->arch.rmode.tr.base = var->base; + vcpu->arch.rmode.tr.limit = var->limit; + vcpu->arch.rmode.tr.ar = vmx_segment_access_rights(var); return; } vmcs_writel(sf->base, var->base); vmcs_write32(sf->limit, var->limit); vmcs_write16(sf->selector, var->selector); - if (vcpu->rmode.active && var->s) { + if (vcpu->arch.rmode.active && var->s) { /* * Hack real-mode segments into vm86 compatibility. */ @@ -1613,9 +1614,9 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) goto out; } - vmx->vcpu.rmode.active = 0; + vmx->vcpu.arch.rmode.active = 0; - vmx->vcpu.regs[VCPU_REGS_RDX] = get_rdx_init_val(); + vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val(); set_cr8(&vmx->vcpu, 0); msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; if (vmx->vcpu.vcpu_id == 0) @@ -1632,8 +1633,8 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_write16(GUEST_CS_SELECTOR, 0xf000); vmcs_writel(GUEST_CS_BASE, 0x000f0000); } else { - vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.sipi_vector << 8); - vmcs_writel(GUEST_CS_BASE, vmx->vcpu.sipi_vector << 12); + vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8); + vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12); } vmcs_write32(GUEST_CS_LIMIT, 0xffff); vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); @@ -1691,7 +1692,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); if (vm_need_tpr_shadow(vmx->vcpu.kvm)) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - page_to_phys(vmx->vcpu.apic->regs_page)); + page_to_phys(vmx->vcpu.arch.apic->regs_page)); vmcs_write32(TPR_THRESHOLD, 0); } @@ -1699,8 +1700,8 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(vmx->vcpu.kvm->apic_access_page)); - vmx->vcpu.cr0 = 0x60000010; - vmx_set_cr0(&vmx->vcpu, vmx->vcpu.cr0); /* enter rmode */ + vmx->vcpu.arch.cr0 = 0x60000010; + vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */ vmx_set_cr4(&vmx->vcpu, 0); #ifdef CONFIG_X86_64 vmx_set_efer(&vmx->vcpu, 0); @@ -1718,7 +1719,7 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (vcpu->rmode.active) { + if (vcpu->arch.rmode.active) { vmx->rmode.irq.pending = true; vmx->rmode.irq.vector = irq; vmx->rmode.irq.rip = vmcs_readl(GUEST_RIP); @@ -1734,13 +1735,13 @@ static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) static void kvm_do_inject_irq(struct kvm_vcpu *vcpu) { - int word_index = __ffs(vcpu->irq_summary); - int bit_index = __ffs(vcpu->irq_pending[word_index]); + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); int irq = word_index * BITS_PER_LONG + bit_index; - clear_bit(bit_index, &vcpu->irq_pending[word_index]); - if (!vcpu->irq_pending[word_index]) - clear_bit(word_index, &vcpu->irq_summary); + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); vmx_inject_irq(vcpu, irq); } @@ -1750,12 +1751,12 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, { u32 cpu_based_vm_exec_control; - vcpu->interrupt_window_open = + vcpu->arch.interrupt_window_open = ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0); - if (vcpu->interrupt_window_open && - vcpu->irq_summary && + if (vcpu->arch.interrupt_window_open && + vcpu->arch.irq_summary && !(vmcs_read32(VM_ENTRY_INTR_INFO_FIELD) & INTR_INFO_VALID_MASK)) /* * If interrupts enabled, and not blocked by sti or mov ss. Good. @@ -1763,8 +1764,8 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, kvm_do_inject_irq(vcpu); cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); - if (!vcpu->interrupt_window_open && - (vcpu->irq_summary || kvm_run->request_interrupt_window)) + if (!vcpu->arch.interrupt_window_open && + (vcpu->arch.irq_summary || kvm_run->request_interrupt_window)) /* * Interrupts blocked. Wait for unblock. */ @@ -1812,7 +1813,7 @@ static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu) static int handle_rmode_exception(struct kvm_vcpu *vcpu, int vec, u32 err_code) { - if (!vcpu->rmode.active) + if (!vcpu->arch.rmode.active) return 0; /* @@ -1843,8 +1844,8 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (!irqchip_in_kernel(vcpu->kvm) && is_external_interrupt(vect_info)) { int irq = vect_info & VECTORING_INFO_VECTOR_MASK; - set_bit(irq, vcpu->irq_pending); - set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary); + set_bit(irq, vcpu->arch.irq_pending); + set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary); } if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) /* nmi */ @@ -1871,11 +1872,11 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) return kvm_mmu_page_fault(vcpu, cr2, error_code); } - if (vcpu->rmode.active && + if (vcpu->arch.rmode.active && handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK, error_code)) { - if (vcpu->halt_request) { - vcpu->halt_request = 0; + if (vcpu->arch.halt_request) { + vcpu->arch.halt_request = 0; return kvm_emulate_halt(vcpu); } return 1; @@ -1956,22 +1957,22 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) switch (cr) { case 0: vcpu_load_rsp_rip(vcpu); - set_cr0(vcpu, vcpu->regs[reg]); + set_cr0(vcpu, vcpu->arch.regs[reg]); skip_emulated_instruction(vcpu); return 1; case 3: vcpu_load_rsp_rip(vcpu); - set_cr3(vcpu, vcpu->regs[reg]); + set_cr3(vcpu, vcpu->arch.regs[reg]); skip_emulated_instruction(vcpu); return 1; case 4: vcpu_load_rsp_rip(vcpu); - set_cr4(vcpu, vcpu->regs[reg]); + set_cr4(vcpu, vcpu->arch.regs[reg]); skip_emulated_instruction(vcpu); return 1; case 8: vcpu_load_rsp_rip(vcpu); - set_cr8(vcpu, vcpu->regs[reg]); + set_cr8(vcpu, vcpu->arch.regs[reg]); skip_emulated_instruction(vcpu); if (irqchip_in_kernel(vcpu->kvm)) return 1; @@ -1982,8 +1983,8 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) case 2: /* clts */ vcpu_load_rsp_rip(vcpu); vmx_fpu_deactivate(vcpu); - vcpu->cr0 &= ~X86_CR0_TS; - vmcs_writel(CR0_READ_SHADOW, vcpu->cr0); + vcpu->arch.cr0 &= ~X86_CR0_TS; + vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0); vmx_fpu_activate(vcpu); skip_emulated_instruction(vcpu); return 1; @@ -1991,13 +1992,13 @@ static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) switch (cr) { case 3: vcpu_load_rsp_rip(vcpu); - vcpu->regs[reg] = vcpu->cr3; + vcpu->arch.regs[reg] = vcpu->arch.cr3; vcpu_put_rsp_rip(vcpu); skip_emulated_instruction(vcpu); return 1; case 8: vcpu_load_rsp_rip(vcpu); - vcpu->regs[reg] = get_cr8(vcpu); + vcpu->arch.regs[reg] = get_cr8(vcpu); vcpu_put_rsp_rip(vcpu); skip_emulated_instruction(vcpu); return 1; @@ -2043,7 +2044,7 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) default: val = 0; } - vcpu->regs[reg] = val; + vcpu->arch.regs[reg] = val; } else { /* mov to dr */ } @@ -2060,7 +2061,7 @@ static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { - u32 ecx = vcpu->regs[VCPU_REGS_RCX]; + u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; u64 data; if (vmx_get_msr(vcpu, ecx, &data)) { @@ -2069,17 +2070,17 @@ static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) } /* FIXME: handling of bits 32:63 of rax, rdx */ - vcpu->regs[VCPU_REGS_RAX] = data & -1u; - vcpu->regs[VCPU_REGS_RDX] = (data >> 32) & -1u; + vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u; + vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u; skip_emulated_instruction(vcpu); return 1; } static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { - u32 ecx = vcpu->regs[VCPU_REGS_RCX]; - u64 data = (vcpu->regs[VCPU_REGS_RAX] & -1u) - | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32); + u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; + u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u) + | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32); if (vmx_set_msr(vcpu, ecx, data) != 0) { kvm_inject_gp(vcpu, 0); @@ -2110,7 +2111,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu, * possible */ if (kvm_run->request_interrupt_window && - !vcpu->irq_summary) { + !vcpu->arch.irq_summary) { kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; ++vcpu->stat.irq_window_exits; return 0; @@ -2270,7 +2271,7 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) if (unlikely(idtv_info_field & INTR_INFO_VALID_MASK)) { if ((idtv_info_field & VECTORING_INFO_TYPE_MASK) == INTR_TYPE_EXT_INTR - && vcpu->rmode.active) { + && vcpu->arch.rmode.active) { u8 vect = idtv_info_field & VECTORING_INFO_VECTOR_MASK; vmx_inject_irq(vcpu, vect); @@ -2424,24 +2425,24 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) : : "c"(vmx), "d"((unsigned long)HOST_RSP), [launched]"i"(offsetof(struct vcpu_vmx, launched)), [fail]"i"(offsetof(struct vcpu_vmx, fail)), - [rax]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RAX])), - [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_RBP])), + [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])), + [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])), #ifdef CONFIG_X86_64 - [r8]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_vmx, vcpu.regs[VCPU_REGS_R15])), + [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])), #endif - [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.cr2)) + [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)) : "cc", "memory" #ifdef CONFIG_X86_64 , "rbx", "rdi", "rsi" @@ -2455,7 +2456,7 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (vmx->rmode.irq.pending) fixup_rmode_irq(vmx); - vcpu->interrupt_window_open = + vcpu->arch.interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS)); diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 3b79684a3c0c..5a2f33a84e4f 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -113,9 +113,9 @@ EXPORT_SYMBOL_GPL(segment_base); u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) { if (irqchip_in_kernel(vcpu->kvm)) - return vcpu->apic_base; + return vcpu->arch.apic_base; else - return vcpu->apic_base; + return vcpu->arch.apic_base; } EXPORT_SYMBOL_GPL(kvm_get_apic_base); @@ -125,16 +125,16 @@ void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) if (irqchip_in_kernel(vcpu->kvm)) kvm_lapic_set_base(vcpu, data); else - vcpu->apic_base = data; + vcpu->arch.apic_base = data; } EXPORT_SYMBOL_GPL(kvm_set_apic_base); void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) { - WARN_ON(vcpu->exception.pending); - vcpu->exception.pending = true; - vcpu->exception.has_error_code = false; - vcpu->exception.nr = nr; + WARN_ON(vcpu->arch.exception.pending); + vcpu->arch.exception.pending = true; + vcpu->arch.exception.has_error_code = false; + vcpu->arch.exception.nr = nr; } EXPORT_SYMBOL_GPL(kvm_queue_exception); @@ -142,32 +142,32 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, u32 error_code) { ++vcpu->stat.pf_guest; - if (vcpu->exception.pending && vcpu->exception.nr == PF_VECTOR) { + if (vcpu->arch.exception.pending && vcpu->arch.exception.nr == PF_VECTOR) { printk(KERN_DEBUG "kvm: inject_page_fault:" " double fault 0x%lx\n", addr); - vcpu->exception.nr = DF_VECTOR; - vcpu->exception.error_code = 0; + vcpu->arch.exception.nr = DF_VECTOR; + vcpu->arch.exception.error_code = 0; return; } - vcpu->cr2 = addr; + vcpu->arch.cr2 = addr; kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); } void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) { - WARN_ON(vcpu->exception.pending); - vcpu->exception.pending = true; - vcpu->exception.has_error_code = true; - vcpu->exception.nr = nr; - vcpu->exception.error_code = error_code; + WARN_ON(vcpu->arch.exception.pending); + vcpu->arch.exception.pending = true; + vcpu->arch.exception.has_error_code = true; + vcpu->arch.exception.nr = nr; + vcpu->arch.exception.error_code = error_code; } EXPORT_SYMBOL_GPL(kvm_queue_exception_e); static void __queue_exception(struct kvm_vcpu *vcpu) { - kvm_x86_ops->queue_exception(vcpu, vcpu->exception.nr, - vcpu->exception.has_error_code, - vcpu->exception.error_code); + kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr, + vcpu->arch.exception.has_error_code, + vcpu->arch.exception.error_code); } /* @@ -179,7 +179,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; int i; int ret; - u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; + u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; mutex_lock(&vcpu->kvm->lock); ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, @@ -196,7 +196,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) } ret = 1; - memcpy(vcpu->pdptrs, pdpte, sizeof(vcpu->pdptrs)); + memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); out: mutex_unlock(&vcpu->kvm->lock); @@ -205,7 +205,7 @@ out: static bool pdptrs_changed(struct kvm_vcpu *vcpu) { - u64 pdpte[ARRAY_SIZE(vcpu->pdptrs)]; + u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; bool changed = true; int r; @@ -213,10 +213,10 @@ static bool pdptrs_changed(struct kvm_vcpu *vcpu) return false; mutex_lock(&vcpu->kvm->lock); - r = kvm_read_guest(vcpu->kvm, vcpu->cr3 & ~31u, pdpte, sizeof(pdpte)); + r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte)); if (r < 0) goto out; - changed = memcmp(pdpte, vcpu->pdptrs, sizeof(pdpte)) != 0; + changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0; out: mutex_unlock(&vcpu->kvm->lock); @@ -227,7 +227,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) { if (cr0 & CR0_RESERVED_BITS) { printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", - cr0, vcpu->cr0); + cr0, vcpu->arch.cr0); kvm_inject_gp(vcpu, 0); return; } @@ -247,7 +247,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { #ifdef CONFIG_X86_64 - if ((vcpu->shadow_efer & EFER_LME)) { + if ((vcpu->arch.shadow_efer & EFER_LME)) { int cs_db, cs_l; if (!is_pae(vcpu)) { @@ -266,7 +266,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } } else #endif - if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) { + if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) { printk(KERN_DEBUG "set_cr0: #GP, pdptrs " "reserved bits\n"); kvm_inject_gp(vcpu, 0); @@ -276,7 +276,7 @@ void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) } kvm_x86_ops->set_cr0(vcpu, cr0); - vcpu->cr0 = cr0; + vcpu->arch.cr0 = cr0; mutex_lock(&vcpu->kvm->lock); kvm_mmu_reset_context(vcpu); @@ -287,7 +287,7 @@ EXPORT_SYMBOL_GPL(set_cr0); void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) { - set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f)); + set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f)); } EXPORT_SYMBOL_GPL(lmsw); @@ -307,7 +307,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return; } } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) - && !load_pdptrs(vcpu, vcpu->cr3)) { + && !load_pdptrs(vcpu, vcpu->arch.cr3)) { printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); kvm_inject_gp(vcpu, 0); return; @@ -319,7 +319,7 @@ void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return; } kvm_x86_ops->set_cr4(vcpu, cr4); - vcpu->cr4 = cr4; + vcpu->arch.cr4 = cr4; mutex_lock(&vcpu->kvm->lock); kvm_mmu_reset_context(vcpu); mutex_unlock(&vcpu->kvm->lock); @@ -328,7 +328,7 @@ EXPORT_SYMBOL_GPL(set_cr4); void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { - if (cr3 == vcpu->cr3 && !pdptrs_changed(vcpu)) { + if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) { kvm_mmu_flush_tlb(vcpu); return; } @@ -373,8 +373,8 @@ void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) kvm_inject_gp(vcpu, 0); else { - vcpu->cr3 = cr3; - vcpu->mmu.new_cr3(vcpu); + vcpu->arch.cr3 = cr3; + vcpu->arch.mmu.new_cr3(vcpu); } mutex_unlock(&vcpu->kvm->lock); } @@ -390,7 +390,7 @@ void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) if (irqchip_in_kernel(vcpu->kvm)) kvm_lapic_set_tpr(vcpu, cr8); else - vcpu->cr8 = cr8; + vcpu->arch.cr8 = cr8; } EXPORT_SYMBOL_GPL(set_cr8); @@ -399,7 +399,7 @@ unsigned long get_cr8(struct kvm_vcpu *vcpu) if (irqchip_in_kernel(vcpu->kvm)) return kvm_lapic_get_cr8(vcpu); else - return vcpu->cr8; + return vcpu->arch.cr8; } EXPORT_SYMBOL_GPL(get_cr8); @@ -437,7 +437,7 @@ static void set_efer(struct kvm_vcpu *vcpu, u64 efer) } if (is_paging(vcpu) - && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) { + && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) { printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); kvm_inject_gp(vcpu, 0); return; @@ -446,9 +446,9 @@ static void set_efer(struct kvm_vcpu *vcpu, u64 efer) kvm_x86_ops->set_efer(vcpu, efer); efer &= ~EFER_LMA; - efer |= vcpu->shadow_efer & EFER_LMA; + efer |= vcpu->arch.shadow_efer & EFER_LMA; - vcpu->shadow_efer = efer; + vcpu->arch.shadow_efer = efer; } #endif @@ -496,7 +496,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) kvm_set_apic_base(vcpu, data); break; case MSR_IA32_MISC_ENABLE: - vcpu->ia32_misc_enable_msr = data; + vcpu->arch.ia32_misc_enable_msr = data; break; default: pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); @@ -550,11 +550,11 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) data = kvm_get_apic_base(vcpu); break; case MSR_IA32_MISC_ENABLE: - data = vcpu->ia32_misc_enable_msr; + data = vcpu->arch.ia32_misc_enable_msr; break; #ifdef CONFIG_X86_64 case MSR_EFER: - data = vcpu->shadow_efer; + data = vcpu->arch.shadow_efer; break; #endif default: @@ -760,8 +760,8 @@ static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) struct kvm_cpuid_entry2 *e, *entry; entry = NULL; - for (i = 0; i < vcpu->cpuid_nent; ++i) { - e = &vcpu->cpuid_entries[i]; + for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { + e = &vcpu->arch.cpuid_entries[i]; if (e->function == 0x80000001) { entry = e; break; @@ -793,18 +793,18 @@ static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, cpuid->nent * sizeof(struct kvm_cpuid_entry))) goto out_free; for (i = 0; i < cpuid->nent; i++) { - vcpu->cpuid_entries[i].function = cpuid_entries[i].function; - vcpu->cpuid_entries[i].eax = cpuid_entries[i].eax; - vcpu->cpuid_entries[i].ebx = cpuid_entries[i].ebx; - vcpu->cpuid_entries[i].ecx = cpuid_entries[i].ecx; - vcpu->cpuid_entries[i].edx = cpuid_entries[i].edx; - vcpu->cpuid_entries[i].index = 0; - vcpu->cpuid_entries[i].flags = 0; - vcpu->cpuid_entries[i].padding[0] = 0; - vcpu->cpuid_entries[i].padding[1] = 0; - vcpu->cpuid_entries[i].padding[2] = 0; - } - vcpu->cpuid_nent = cpuid->nent; + vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function; + vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax; + vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx; + vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx; + vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx; + vcpu->arch.cpuid_entries[i].index = 0; + vcpu->arch.cpuid_entries[i].flags = 0; + vcpu->arch.cpuid_entries[i].padding[0] = 0; + vcpu->arch.cpuid_entries[i].padding[1] = 0; + vcpu->arch.cpuid_entries[i].padding[2] = 0; + } + vcpu->arch.cpuid_nent = cpuid->nent; cpuid_fix_nx_cap(vcpu); r = 0; @@ -824,10 +824,10 @@ static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) goto out; r = -EFAULT; - if (copy_from_user(&vcpu->cpuid_entries, entries, + if (copy_from_user(&vcpu->arch.cpuid_entries, entries, cpuid->nent * sizeof(struct kvm_cpuid_entry2))) goto out; - vcpu->cpuid_nent = cpuid->nent; + vcpu->arch.cpuid_nent = cpuid->nent; return 0; out: @@ -841,16 +841,16 @@ static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, int r; r = -E2BIG; - if (cpuid->nent < vcpu->cpuid_nent) + if (cpuid->nent < vcpu->arch.cpuid_nent) goto out; r = -EFAULT; - if (copy_to_user(entries, &vcpu->cpuid_entries, - vcpu->cpuid_nent * sizeof(struct kvm_cpuid_entry2))) + if (copy_to_user(entries, &vcpu->arch.cpuid_entries, + vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2))) goto out; return 0; out: - cpuid->nent = vcpu->cpuid_nent; + cpuid->nent = vcpu->arch.cpuid_nent; return r; } @@ -1021,7 +1021,7 @@ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) { vcpu_load(vcpu); - memcpy(s->regs, vcpu->apic->regs, sizeof *s); + memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s); vcpu_put(vcpu); return 0; @@ -1031,7 +1031,7 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) { vcpu_load(vcpu); - memcpy(vcpu->apic->regs, s->regs, sizeof *s); + memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); kvm_apic_post_state_restore(vcpu); vcpu_put(vcpu); @@ -1047,8 +1047,8 @@ static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, return -ENXIO; vcpu_load(vcpu); - set_bit(irq->irq, vcpu->irq_pending); - set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary); + set_bit(irq->irq, vcpu->arch.irq_pending); + set_bit(irq->irq / BITS_PER_LONG, &vcpu->arch.irq_summary); vcpu_put(vcpu); @@ -1499,8 +1499,8 @@ static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, { struct kvm_io_device *dev; - if (vcpu->apic) { - dev = &vcpu->apic->dev; + if (vcpu->arch.apic) { + dev = &vcpu->arch.apic->dev; if (dev->in_range(dev, addr)) return dev; } @@ -1527,7 +1527,7 @@ int emulator_read_std(unsigned long addr, void *data = val; while (bytes) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); unsigned offset = addr & (PAGE_SIZE-1); unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); int ret; @@ -1561,7 +1561,7 @@ static int emulator_read_emulated(unsigned long addr, return X86EMUL_CONTINUE; } - gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); /* For APIC access vmexit */ if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) @@ -1609,7 +1609,7 @@ static int emulator_write_emulated_onepage(unsigned long addr, struct kvm_vcpu *vcpu) { struct kvm_io_device *mmio_dev; - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); if (gpa == UNMAPPED_GVA) { kvm_inject_page_fault(vcpu, addr, 2); @@ -1678,7 +1678,7 @@ static int emulator_cmpxchg_emulated(unsigned long addr, #ifndef CONFIG_X86_64 /* guests cmpxchg8b have to be emulated atomically */ if (bytes == 8) { - gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr); + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); struct page *page; char *addr; u64 val; @@ -1715,7 +1715,7 @@ int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) int emulate_clts(struct kvm_vcpu *vcpu) { - kvm_x86_ops->set_cr0(vcpu, vcpu->cr0 & ~X86_CR0_TS); + kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS); return X86EMUL_CONTINUE; } @@ -1750,7 +1750,7 @@ void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) { static int reported; u8 opcodes[4]; - unsigned long rip = vcpu->rip; + unsigned long rip = vcpu->arch.rip; unsigned long rip_linear; rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); @@ -1781,46 +1781,46 @@ int emulate_instruction(struct kvm_vcpu *vcpu, { int r; - vcpu->mmio_fault_cr2 = cr2; + vcpu->arch.mmio_fault_cr2 = cr2; kvm_x86_ops->cache_regs(vcpu); vcpu->mmio_is_write = 0; - vcpu->pio.string = 0; + vcpu->arch.pio.string = 0; if (!no_decode) { int cs_db, cs_l; kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - vcpu->emulate_ctxt.vcpu = vcpu; - vcpu->emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - vcpu->emulate_ctxt.mode = - (vcpu->emulate_ctxt.eflags & X86_EFLAGS_VM) + vcpu->arch.emulate_ctxt.vcpu = vcpu; + vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); + vcpu->arch.emulate_ctxt.mode = + (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM) ? X86EMUL_MODE_REAL : cs_l ? X86EMUL_MODE_PROT64 : cs_db ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; - if (vcpu->emulate_ctxt.mode == X86EMUL_MODE_PROT64) { - vcpu->emulate_ctxt.cs_base = 0; - vcpu->emulate_ctxt.ds_base = 0; - vcpu->emulate_ctxt.es_base = 0; - vcpu->emulate_ctxt.ss_base = 0; + if (vcpu->arch.emulate_ctxt.mode == X86EMUL_MODE_PROT64) { + vcpu->arch.emulate_ctxt.cs_base = 0; + vcpu->arch.emulate_ctxt.ds_base = 0; + vcpu->arch.emulate_ctxt.es_base = 0; + vcpu->arch.emulate_ctxt.ss_base = 0; } else { - vcpu->emulate_ctxt.cs_base = + vcpu->arch.emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS); - vcpu->emulate_ctxt.ds_base = + vcpu->arch.emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS); - vcpu->emulate_ctxt.es_base = + vcpu->arch.emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES); - vcpu->emulate_ctxt.ss_base = + vcpu->arch.emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS); } - vcpu->emulate_ctxt.gs_base = + vcpu->arch.emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS); - vcpu->emulate_ctxt.fs_base = + vcpu->arch.emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS); - r = x86_decode_insn(&vcpu->emulate_ctxt, &emulate_ops); + r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); ++vcpu->stat.insn_emulation; if (r) { ++vcpu->stat.insn_emulation_fail; @@ -1830,9 +1830,9 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } } - r = x86_emulate_insn(&vcpu->emulate_ctxt, &emulate_ops); + r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); - if (vcpu->pio.string) + if (vcpu->arch.pio.string) return EMULATE_DO_MMIO; if ((r || vcpu->mmio_is_write) && run) { @@ -1854,7 +1854,7 @@ int emulate_instruction(struct kvm_vcpu *vcpu, } kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->set_rflags(vcpu, vcpu->emulate_ctxt.eflags); + kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags); if (vcpu->mmio_is_write) { vcpu->mmio_needed = 0; @@ -1869,33 +1869,33 @@ static void free_pio_guest_pages(struct kvm_vcpu *vcpu) { int i; - for (i = 0; i < ARRAY_SIZE(vcpu->pio.guest_pages); ++i) - if (vcpu->pio.guest_pages[i]) { - kvm_release_page_dirty(vcpu->pio.guest_pages[i]); - vcpu->pio.guest_pages[i] = NULL; + for (i = 0; i < ARRAY_SIZE(vcpu->arch.pio.guest_pages); ++i) + if (vcpu->arch.pio.guest_pages[i]) { + kvm_release_page_dirty(vcpu->arch.pio.guest_pages[i]); + vcpu->arch.pio.guest_pages[i] = NULL; } } static int pio_copy_data(struct kvm_vcpu *vcpu) { - void *p = vcpu->pio_data; + void *p = vcpu->arch.pio_data; void *q; unsigned bytes; - int nr_pages = vcpu->pio.guest_pages[1] ? 2 : 1; + int nr_pages = vcpu->arch.pio.guest_pages[1] ? 2 : 1; - q = vmap(vcpu->pio.guest_pages, nr_pages, VM_READ|VM_WRITE, + q = vmap(vcpu->arch.pio.guest_pages, nr_pages, VM_READ|VM_WRITE, PAGE_KERNEL); if (!q) { free_pio_guest_pages(vcpu); return -ENOMEM; } - q += vcpu->pio.guest_page_offset; - bytes = vcpu->pio.size * vcpu->pio.cur_count; - if (vcpu->pio.in) + q += vcpu->arch.pio.guest_page_offset; + bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count; + if (vcpu->arch.pio.in) memcpy(q, p, bytes); else memcpy(p, q, bytes); - q -= vcpu->pio.guest_page_offset; + q -= vcpu->arch.pio.guest_page_offset; vunmap(q); free_pio_guest_pages(vcpu); return 0; @@ -1903,7 +1903,7 @@ static int pio_copy_data(struct kvm_vcpu *vcpu) int complete_pio(struct kvm_vcpu *vcpu) { - struct kvm_pio_request *io = &vcpu->pio; + struct kvm_pio_request *io = &vcpu->arch.pio; long delta; int r; @@ -1911,7 +1911,7 @@ int complete_pio(struct kvm_vcpu *vcpu) if (!io->string) { if (io->in) - memcpy(&vcpu->regs[VCPU_REGS_RAX], vcpu->pio_data, + memcpy(&vcpu->arch.regs[VCPU_REGS_RAX], vcpu->arch.pio_data, io->size); } else { if (io->in) { @@ -1929,15 +1929,15 @@ int complete_pio(struct kvm_vcpu *vcpu) * The size of the register should really depend on * current address size. */ - vcpu->regs[VCPU_REGS_RCX] -= delta; + vcpu->arch.regs[VCPU_REGS_RCX] -= delta; } if (io->down) delta = -delta; delta *= io->size; if (io->in) - vcpu->regs[VCPU_REGS_RDI] += delta; + vcpu->arch.regs[VCPU_REGS_RDI] += delta; else - vcpu->regs[VCPU_REGS_RSI] += delta; + vcpu->arch.regs[VCPU_REGS_RSI] += delta; } kvm_x86_ops->decache_regs(vcpu); @@ -1955,13 +1955,13 @@ static void kernel_pio(struct kvm_io_device *pio_dev, /* TODO: String I/O for in kernel device */ mutex_lock(&vcpu->kvm->lock); - if (vcpu->pio.in) - kvm_iodevice_read(pio_dev, vcpu->pio.port, - vcpu->pio.size, + if (vcpu->arch.pio.in) + kvm_iodevice_read(pio_dev, vcpu->arch.pio.port, + vcpu->arch.pio.size, pd); else - kvm_iodevice_write(pio_dev, vcpu->pio.port, - vcpu->pio.size, + kvm_iodevice_write(pio_dev, vcpu->arch.pio.port, + vcpu->arch.pio.size, pd); mutex_unlock(&vcpu->kvm->lock); } @@ -1969,8 +1969,8 @@ static void kernel_pio(struct kvm_io_device *pio_dev, static void pio_string_write(struct kvm_io_device *pio_dev, struct kvm_vcpu *vcpu) { - struct kvm_pio_request *io = &vcpu->pio; - void *pd = vcpu->pio_data; + struct kvm_pio_request *io = &vcpu->arch.pio; + void *pd = vcpu->arch.pio_data; int i; mutex_lock(&vcpu->kvm->lock); @@ -1996,25 +1996,25 @@ int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, vcpu->run->exit_reason = KVM_EXIT_IO; vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->pio.size = size; + vcpu->run->io.size = vcpu->arch.pio.size = size; vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = 1; - vcpu->run->io.port = vcpu->pio.port = port; - vcpu->pio.in = in; - vcpu->pio.string = 0; - vcpu->pio.down = 0; - vcpu->pio.guest_page_offset = 0; - vcpu->pio.rep = 0; + vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1; + vcpu->run->io.port = vcpu->arch.pio.port = port; + vcpu->arch.pio.in = in; + vcpu->arch.pio.string = 0; + vcpu->arch.pio.down = 0; + vcpu->arch.pio.guest_page_offset = 0; + vcpu->arch.pio.rep = 0; kvm_x86_ops->cache_regs(vcpu); - memcpy(vcpu->pio_data, &vcpu->regs[VCPU_REGS_RAX], 4); + memcpy(vcpu->arch.pio_data, &vcpu->arch.regs[VCPU_REGS_RAX], 4); kvm_x86_ops->decache_regs(vcpu); kvm_x86_ops->skip_emulated_instruction(vcpu); pio_dev = vcpu_find_pio_dev(vcpu, port); if (pio_dev) { - kernel_pio(pio_dev, vcpu, vcpu->pio_data); + kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data); complete_pio(vcpu); return 1; } @@ -2034,15 +2034,15 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, vcpu->run->exit_reason = KVM_EXIT_IO; vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->pio.size = size; + vcpu->run->io.size = vcpu->arch.pio.size = size; vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->pio.count = vcpu->pio.cur_count = count; - vcpu->run->io.port = vcpu->pio.port = port; - vcpu->pio.in = in; - vcpu->pio.string = 1; - vcpu->pio.down = down; - vcpu->pio.guest_page_offset = offset_in_page(address); - vcpu->pio.rep = rep; + vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count; + vcpu->run->io.port = vcpu->arch.pio.port = port; + vcpu->arch.pio.in = in; + vcpu->arch.pio.string = 1; + vcpu->arch.pio.down = down; + vcpu->arch.pio.guest_page_offset = offset_in_page(address); + vcpu->arch.pio.rep = rep; if (!count) { kvm_x86_ops->skip_emulated_instruction(vcpu); @@ -2072,15 +2072,15 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, return 1; } vcpu->run->io.count = now; - vcpu->pio.cur_count = now; + vcpu->arch.pio.cur_count = now; - if (vcpu->pio.cur_count == vcpu->pio.count) + if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count) kvm_x86_ops->skip_emulated_instruction(vcpu); for (i = 0; i < nr_pages; ++i) { mutex_lock(&vcpu->kvm->lock); page = gva_to_page(vcpu, address + i * PAGE_SIZE); - vcpu->pio.guest_pages[i] = page; + vcpu->arch.pio.guest_pages[i] = page; mutex_unlock(&vcpu->kvm->lock); if (!page) { kvm_inject_gp(vcpu, 0); @@ -2090,13 +2090,13 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, } pio_dev = vcpu_find_pio_dev(vcpu, port); - if (!vcpu->pio.in) { + if (!vcpu->arch.pio.in) { /* string PIO write */ ret = pio_copy_data(vcpu); if (ret >= 0 && pio_dev) { pio_string_write(pio_dev, vcpu); complete_pio(vcpu); - if (vcpu->pio.count == 0) + if (vcpu->arch.pio.count == 0) ret = 1; } } else if (pio_dev) @@ -2156,9 +2156,9 @@ int kvm_emulate_halt(struct kvm_vcpu *vcpu) { ++vcpu->stat.halt_exits; if (irqchip_in_kernel(vcpu->kvm)) { - vcpu->mp_state = VCPU_MP_STATE_HALTED; + vcpu->arch.mp_state = VCPU_MP_STATE_HALTED; kvm_vcpu_block(vcpu); - if (vcpu->mp_state != VCPU_MP_STATE_RUNNABLE) + if (vcpu->arch.mp_state != VCPU_MP_STATE_RUNNABLE) return -EINTR; return 1; } else { @@ -2174,11 +2174,11 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) kvm_x86_ops->cache_regs(vcpu); - nr = vcpu->regs[VCPU_REGS_RAX]; - a0 = vcpu->regs[VCPU_REGS_RBX]; - a1 = vcpu->regs[VCPU_REGS_RCX]; - a2 = vcpu->regs[VCPU_REGS_RDX]; - a3 = vcpu->regs[VCPU_REGS_RSI]; + nr = vcpu->arch.regs[VCPU_REGS_RAX]; + a0 = vcpu->arch.regs[VCPU_REGS_RBX]; + a1 = vcpu->arch.regs[VCPU_REGS_RCX]; + a2 = vcpu->arch.regs[VCPU_REGS_RDX]; + a3 = vcpu->arch.regs[VCPU_REGS_RSI]; if (!is_long_mode(vcpu)) { nr &= 0xFFFFFFFF; @@ -2193,7 +2193,7 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) ret = -KVM_ENOSYS; break; } - vcpu->regs[VCPU_REGS_RAX] = ret; + vcpu->arch.regs[VCPU_REGS_RAX] = ret; kvm_x86_ops->decache_regs(vcpu); return 0; } @@ -2215,7 +2215,7 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu) kvm_x86_ops->cache_regs(vcpu); kvm_x86_ops->patch_hypercall(vcpu, instruction); - if (emulator_write_emulated(vcpu->rip, instruction, 3, vcpu) + if (emulator_write_emulated(vcpu->arch.rip, instruction, 3, vcpu) != X86EMUL_CONTINUE) ret = -EFAULT; @@ -2255,13 +2255,13 @@ unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) kvm_x86_ops->decache_cr4_guest_bits(vcpu); switch (cr) { case 0: - return vcpu->cr0; + return vcpu->arch.cr0; case 2: - return vcpu->cr2; + return vcpu->arch.cr2; case 3: - return vcpu->cr3; + return vcpu->arch.cr3; case 4: - return vcpu->cr4; + return vcpu->arch.cr4; case 8: return get_cr8(vcpu); default: @@ -2275,17 +2275,17 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, { switch (cr) { case 0: - set_cr0(vcpu, mk_cr_64(vcpu->cr0, val)); + set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val)); *rflags = kvm_x86_ops->get_rflags(vcpu); break; case 2: - vcpu->cr2 = val; + vcpu->arch.cr2 = val; break; case 3: set_cr3(vcpu, val); break; case 4: - set_cr4(vcpu, mk_cr_64(vcpu->cr4, val)); + set_cr4(vcpu, mk_cr_64(vcpu->arch.cr4, val)); break; case 8: set_cr8(vcpu, val & 0xfUL); @@ -2297,13 +2297,13 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i) { - struct kvm_cpuid_entry2 *e = &vcpu->cpuid_entries[i]; - int j, nent = vcpu->cpuid_nent; + struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i]; + int j, nent = vcpu->arch.cpuid_nent; e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT; /* when no next entry is found, the current entry[i] is reselected */ for (j = i + 1; j == i; j = (j + 1) % nent) { - struct kvm_cpuid_entry2 *ej = &vcpu->cpuid_entries[j]; + struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j]; if (ej->function == e->function) { ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; return j; @@ -2334,15 +2334,15 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) struct kvm_cpuid_entry2 *e, *best; kvm_x86_ops->cache_regs(vcpu); - function = vcpu->regs[VCPU_REGS_RAX]; - index = vcpu->regs[VCPU_REGS_RCX]; - vcpu->regs[VCPU_REGS_RAX] = 0; - vcpu->regs[VCPU_REGS_RBX] = 0; - vcpu->regs[VCPU_REGS_RCX] = 0; - vcpu->regs[VCPU_REGS_RDX] = 0; + function = vcpu->arch.regs[VCPU_REGS_RAX]; + index = vcpu->arch.regs[VCPU_REGS_RCX]; + vcpu->arch.regs[VCPU_REGS_RAX] = 0; + vcpu->arch.regs[VCPU_REGS_RBX] = 0; + vcpu->arch.regs[VCPU_REGS_RCX] = 0; + vcpu->arch.regs[VCPU_REGS_RDX] = 0; best = NULL; - for (i = 0; i < vcpu->cpuid_nent; ++i) { - e = &vcpu->cpuid_entries[i]; + for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { + e = &vcpu->arch.cpuid_entries[i]; if (is_matching_cpuid_entry(e, function, index)) { if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) move_to_next_stateful_cpuid_entry(vcpu, i); @@ -2357,10 +2357,10 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) best = e; } if (best) { - vcpu->regs[VCPU_REGS_RAX] = best->eax; - vcpu->regs[VCPU_REGS_RBX] = best->ebx; - vcpu->regs[VCPU_REGS_RCX] = best->ecx; - vcpu->regs[VCPU_REGS_RDX] = best->edx; + vcpu->arch.regs[VCPU_REGS_RAX] = best->eax; + vcpu->arch.regs[VCPU_REGS_RBX] = best->ebx; + vcpu->arch.regs[VCPU_REGS_RCX] = best->ecx; + vcpu->arch.regs[VCPU_REGS_RDX] = best->edx; } kvm_x86_ops->decache_regs(vcpu); kvm_x86_ops->skip_emulated_instruction(vcpu); @@ -2376,9 +2376,9 @@ EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { - return (!vcpu->irq_summary && + return (!vcpu->arch.irq_summary && kvm_run->request_interrupt_window && - vcpu->interrupt_window_open && + vcpu->arch.interrupt_window_open && (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF)); } @@ -2392,22 +2392,22 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu, kvm_run->ready_for_interrupt_injection = 1; else kvm_run->ready_for_interrupt_injection = - (vcpu->interrupt_window_open && - vcpu->irq_summary == 0); + (vcpu->arch.interrupt_window_open && + vcpu->arch.irq_summary == 0); } static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) { int r; - if (unlikely(vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { + if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { pr_debug("vcpu %d received sipi with vector # %x\n", - vcpu->vcpu_id, vcpu->sipi_vector); + vcpu->vcpu_id, vcpu->arch.sipi_vector); kvm_lapic_reset(vcpu); r = kvm_x86_ops->vcpu_reset(vcpu); if (r) return r; - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; } preempted: @@ -2437,7 +2437,7 @@ again: goto out; } - if (vcpu->exception.pending) + if (vcpu->arch.exception.pending) __queue_exception(vcpu); else if (irqchip_in_kernel(vcpu->kvm)) kvm_x86_ops->inject_pending_irq(vcpu); @@ -2475,11 +2475,11 @@ again: */ if (unlikely(prof_on == KVM_PROFILING)) { kvm_x86_ops->cache_regs(vcpu); - profile_hit(KVM_PROFILING, (void *)vcpu->rip); + profile_hit(KVM_PROFILING, (void *)vcpu->arch.rip); } - if (vcpu->exception.pending && kvm_x86_ops->exception_injected(vcpu)) - vcpu->exception.pending = false; + if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu)) + vcpu->arch.exception.pending = false; r = kvm_x86_ops->handle_exit(kvm_run, vcpu); @@ -2512,7 +2512,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu_load(vcpu); - if (unlikely(vcpu->mp_state == VCPU_MP_STATE_UNINITIALIZED)) { + if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_UNINITIALIZED)) { kvm_vcpu_block(vcpu); vcpu_put(vcpu); return -EAGAIN; @@ -2525,7 +2525,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) if (!irqchip_in_kernel(vcpu->kvm)) set_cr8(vcpu, kvm_run->cr8); - if (vcpu->pio.cur_count) { + if (vcpu->arch.pio.cur_count) { r = complete_pio(vcpu); if (r) goto out; @@ -2536,7 +2536,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) vcpu->mmio_read_completed = 1; vcpu->mmio_needed = 0; r = emulate_instruction(vcpu, kvm_run, - vcpu->mmio_fault_cr2, 0, 1); + vcpu->arch.mmio_fault_cr2, 0, 1); if (r == EMULATE_DO_MMIO) { /* * Read-modify-write. Back to userspace. @@ -2548,7 +2548,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) #endif if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { kvm_x86_ops->cache_regs(vcpu); - vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; + vcpu->arch.regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; kvm_x86_ops->decache_regs(vcpu); } @@ -2568,26 +2568,26 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) kvm_x86_ops->cache_regs(vcpu); - regs->rax = vcpu->regs[VCPU_REGS_RAX]; - regs->rbx = vcpu->regs[VCPU_REGS_RBX]; - regs->rcx = vcpu->regs[VCPU_REGS_RCX]; - regs->rdx = vcpu->regs[VCPU_REGS_RDX]; - regs->rsi = vcpu->regs[VCPU_REGS_RSI]; - regs->rdi = vcpu->regs[VCPU_REGS_RDI]; - regs->rsp = vcpu->regs[VCPU_REGS_RSP]; - regs->rbp = vcpu->regs[VCPU_REGS_RBP]; + regs->rax = vcpu->arch.regs[VCPU_REGS_RAX]; + regs->rbx = vcpu->arch.regs[VCPU_REGS_RBX]; + regs->rcx = vcpu->arch.regs[VCPU_REGS_RCX]; + regs->rdx = vcpu->arch.regs[VCPU_REGS_RDX]; + regs->rsi = vcpu->arch.regs[VCPU_REGS_RSI]; + regs->rdi = vcpu->arch.regs[VCPU_REGS_RDI]; + regs->rsp = vcpu->arch.regs[VCPU_REGS_RSP]; + regs->rbp = vcpu->arch.regs[VCPU_REGS_RBP]; #ifdef CONFIG_X86_64 - regs->r8 = vcpu->regs[VCPU_REGS_R8]; - regs->r9 = vcpu->regs[VCPU_REGS_R9]; - regs->r10 = vcpu->regs[VCPU_REGS_R10]; - regs->r11 = vcpu->regs[VCPU_REGS_R11]; - regs->r12 = vcpu->regs[VCPU_REGS_R12]; - regs->r13 = vcpu->regs[VCPU_REGS_R13]; - regs->r14 = vcpu->regs[VCPU_REGS_R14]; - regs->r15 = vcpu->regs[VCPU_REGS_R15]; + regs->r8 = vcpu->arch.regs[VCPU_REGS_R8]; + regs->r9 = vcpu->arch.regs[VCPU_REGS_R9]; + regs->r10 = vcpu->arch.regs[VCPU_REGS_R10]; + regs->r11 = vcpu->arch.regs[VCPU_REGS_R11]; + regs->r12 = vcpu->arch.regs[VCPU_REGS_R12]; + regs->r13 = vcpu->arch.regs[VCPU_REGS_R13]; + regs->r14 = vcpu->arch.regs[VCPU_REGS_R14]; + regs->r15 = vcpu->arch.regs[VCPU_REGS_R15]; #endif - regs->rip = vcpu->rip; + regs->rip = vcpu->arch.rip; regs->rflags = kvm_x86_ops->get_rflags(vcpu); /* @@ -2605,26 +2605,26 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) { vcpu_load(vcpu); - vcpu->regs[VCPU_REGS_RAX] = regs->rax; - vcpu->regs[VCPU_REGS_RBX] = regs->rbx; - vcpu->regs[VCPU_REGS_RCX] = regs->rcx; - vcpu->regs[VCPU_REGS_RDX] = regs->rdx; - vcpu->regs[VCPU_REGS_RSI] = regs->rsi; - vcpu->regs[VCPU_REGS_RDI] = regs->rdi; - vcpu->regs[VCPU_REGS_RSP] = regs->rsp; - vcpu->regs[VCPU_REGS_RBP] = regs->rbp; + vcpu->arch.regs[VCPU_REGS_RAX] = regs->rax; + vcpu->arch.regs[VCPU_REGS_RBX] = regs->rbx; + vcpu->arch.regs[VCPU_REGS_RCX] = regs->rcx; + vcpu->arch.regs[VCPU_REGS_RDX] = regs->rdx; + vcpu->arch.regs[VCPU_REGS_RSI] = regs->rsi; + vcpu->arch.regs[VCPU_REGS_RDI] = regs->rdi; + vcpu->arch.regs[VCPU_REGS_RSP] = regs->rsp; + vcpu->arch.regs[VCPU_REGS_RBP] = regs->rbp; #ifdef CONFIG_X86_64 - vcpu->regs[VCPU_REGS_R8] = regs->r8; - vcpu->regs[VCPU_REGS_R9] = regs->r9; - vcpu->regs[VCPU_REGS_R10] = regs->r10; - vcpu->regs[VCPU_REGS_R11] = regs->r11; - vcpu->regs[VCPU_REGS_R12] = regs->r12; - vcpu->regs[VCPU_REGS_R13] = regs->r13; - vcpu->regs[VCPU_REGS_R14] = regs->r14; - vcpu->regs[VCPU_REGS_R15] = regs->r15; + vcpu->arch.regs[VCPU_REGS_R8] = regs->r8; + vcpu->arch.regs[VCPU_REGS_R9] = regs->r9; + vcpu->arch.regs[VCPU_REGS_R10] = regs->r10; + vcpu->arch.regs[VCPU_REGS_R11] = regs->r11; + vcpu->arch.regs[VCPU_REGS_R12] = regs->r12; + vcpu->arch.regs[VCPU_REGS_R13] = regs->r13; + vcpu->arch.regs[VCPU_REGS_R14] = regs->r14; + vcpu->arch.regs[VCPU_REGS_R15] = regs->r15; #endif - vcpu->rip = regs->rip; + vcpu->arch.rip = regs->rip; kvm_x86_ops->set_rflags(vcpu, regs->rflags); kvm_x86_ops->decache_regs(vcpu); @@ -2676,12 +2676,12 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, sregs->gdt.base = dt.base; kvm_x86_ops->decache_cr4_guest_bits(vcpu); - sregs->cr0 = vcpu->cr0; - sregs->cr2 = vcpu->cr2; - sregs->cr3 = vcpu->cr3; - sregs->cr4 = vcpu->cr4; + sregs->cr0 = vcpu->arch.cr0; + sregs->cr2 = vcpu->arch.cr2; + sregs->cr3 = vcpu->arch.cr3; + sregs->cr4 = vcpu->arch.cr4; sregs->cr8 = get_cr8(vcpu); - sregs->efer = vcpu->shadow_efer; + sregs->efer = vcpu->arch.shadow_efer; sregs->apic_base = kvm_get_apic_base(vcpu); if (irqchip_in_kernel(vcpu->kvm)) { @@ -2692,7 +2692,7 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, set_bit(pending_vec, (unsigned long *)sregs->interrupt_bitmap); } else - memcpy(sregs->interrupt_bitmap, vcpu->irq_pending, + memcpy(sregs->interrupt_bitmap, vcpu->arch.irq_pending, sizeof sregs->interrupt_bitmap); vcpu_put(vcpu); @@ -2722,13 +2722,13 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, dt.base = sregs->gdt.base; kvm_x86_ops->set_gdt(vcpu, &dt); - vcpu->cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->cr3 != sregs->cr3; - vcpu->cr3 = sregs->cr3; + vcpu->arch.cr2 = sregs->cr2; + mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3; + vcpu->arch.cr3 = sregs->cr3; set_cr8(vcpu, sregs->cr8); - mmu_reset_needed |= vcpu->shadow_efer != sregs->efer; + mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer; #ifdef CONFIG_X86_64 kvm_x86_ops->set_efer(vcpu, sregs->efer); #endif @@ -2736,25 +2736,25 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, kvm_x86_ops->decache_cr4_guest_bits(vcpu); - mmu_reset_needed |= vcpu->cr0 != sregs->cr0; - vcpu->cr0 = sregs->cr0; + mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0; + vcpu->arch.cr0 = sregs->cr0; kvm_x86_ops->set_cr0(vcpu, sregs->cr0); - mmu_reset_needed |= vcpu->cr4 != sregs->cr4; + mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4; kvm_x86_ops->set_cr4(vcpu, sregs->cr4); if (!is_long_mode(vcpu) && is_pae(vcpu)) - load_pdptrs(vcpu, vcpu->cr3); + load_pdptrs(vcpu, vcpu->arch.cr3); if (mmu_reset_needed) kvm_mmu_reset_context(vcpu); if (!irqchip_in_kernel(vcpu->kvm)) { - memcpy(vcpu->irq_pending, sregs->interrupt_bitmap, - sizeof vcpu->irq_pending); - vcpu->irq_summary = 0; - for (i = 0; i < ARRAY_SIZE(vcpu->irq_pending); ++i) - if (vcpu->irq_pending[i]) - __set_bit(i, &vcpu->irq_summary); + memcpy(vcpu->arch.irq_pending, sregs->interrupt_bitmap, + sizeof vcpu->arch.irq_pending); + vcpu->arch.irq_summary = 0; + for (i = 0; i < ARRAY_SIZE(vcpu->arch.irq_pending); ++i) + if (vcpu->arch.irq_pending[i]) + __set_bit(i, &vcpu->arch.irq_summary); } else { max_bits = (sizeof sregs->interrupt_bitmap) << 3; pending_vec = find_first_bit( @@ -2829,7 +2829,7 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, vcpu_load(vcpu); mutex_lock(&vcpu->kvm->lock); - gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr); + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr); tr->physical_address = gpa; tr->valid = gpa != UNMAPPED_GVA; tr->writeable = 1; @@ -2842,7 +2842,7 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { - struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; + struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image; vcpu_load(vcpu); @@ -2862,7 +2862,7 @@ int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) { - struct fxsave *fxsave = (struct fxsave *)&vcpu->guest_fx_image; + struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image; vcpu_load(vcpu); @@ -2886,16 +2886,16 @@ void fx_init(struct kvm_vcpu *vcpu) /* Initialize guest FPU by resetting ours and saving into guest's */ preempt_disable(); - fx_save(&vcpu->host_fx_image); + fx_save(&vcpu->arch.host_fx_image); fpu_init(); - fx_save(&vcpu->guest_fx_image); - fx_restore(&vcpu->host_fx_image); + fx_save(&vcpu->arch.guest_fx_image); + fx_restore(&vcpu->arch.host_fx_image); preempt_enable(); - vcpu->cr0 |= X86_CR0_ET; + vcpu->arch.cr0 |= X86_CR0_ET; after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space); - vcpu->guest_fx_image.mxcsr = 0x1f80; - memset((void *)&vcpu->guest_fx_image + after_mxcsr_mask, + vcpu->arch.guest_fx_image.mxcsr = 0x1f80; + memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask, 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask); } EXPORT_SYMBOL_GPL(fx_init); @@ -2906,8 +2906,8 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) return; vcpu->guest_fpu_loaded = 1; - fx_save(&vcpu->host_fx_image); - fx_restore(&vcpu->guest_fx_image); + fx_save(&vcpu->arch.host_fx_image); + fx_restore(&vcpu->arch.guest_fx_image); } EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); @@ -2917,8 +2917,8 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) return; vcpu->guest_fpu_loaded = 0; - fx_save(&vcpu->guest_fx_image); - fx_restore(&vcpu->host_fx_image); + fx_save(&vcpu->arch.guest_fx_image); + fx_restore(&vcpu->arch.host_fx_image); ++vcpu->stat.fpu_reload; } EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); @@ -2939,7 +2939,7 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) int r; /* We do fxsave: this must be aligned. */ - BUG_ON((unsigned long)&vcpu->host_fx_image & 0xF); + BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF); vcpu_load(vcpu); r = kvm_arch_vcpu_reset(vcpu); @@ -3003,18 +3003,18 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) BUG_ON(vcpu->kvm == NULL); kvm = vcpu->kvm; - vcpu->mmu.root_hpa = INVALID_PAGE; + vcpu->arch.mmu.root_hpa = INVALID_PAGE; if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0) - vcpu->mp_state = VCPU_MP_STATE_RUNNABLE; + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; else - vcpu->mp_state = VCPU_MP_STATE_UNINITIALIZED; + vcpu->arch.mp_state = VCPU_MP_STATE_UNINITIALIZED; page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!page) { r = -ENOMEM; goto fail; } - vcpu->pio_data = page_address(page); + vcpu->arch.pio_data = page_address(page); r = kvm_mmu_create(vcpu); if (r < 0) @@ -3031,7 +3031,7 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) fail_mmu_destroy: kvm_mmu_destroy(vcpu); fail_free_pio_data: - free_page((unsigned long)vcpu->pio_data); + free_page((unsigned long)vcpu->arch.pio_data); fail: return r; } @@ -3040,7 +3040,7 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) { kvm_free_lapic(vcpu); kvm_mmu_destroy(vcpu); - free_page((unsigned long)vcpu->pio_data); + free_page((unsigned long)vcpu->arch.pio_data); } struct kvm *kvm_arch_create_vm(void) diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 0fc7020aa1a5..0e01ac75268c 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -92,8 +92,7 @@ enum { #include "x86_emulate.h" -struct kvm_vcpu { - KVM_VCPU_COMM; +struct kvm_vcpu_arch { u64 host_tsc; int interrupt_window_open; unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */ @@ -130,7 +129,6 @@ struct kvm_vcpu { int last_pt_write_count; u64 *last_pte_updated; - struct i387_fxsave_struct host_fx_image; struct i387_fxsave_struct guest_fx_image; @@ -159,12 +157,17 @@ struct kvm_vcpu { int cpuid_nent; struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; - /* emulate context */ struct x86_emulate_ctxt emulate_ctxt; }; +struct kvm_vcpu { + KVM_VCPU_COMM; + + struct kvm_vcpu_arch arch; +}; + struct descriptor_table { u16 limit; unsigned long base; @@ -339,7 +342,7 @@ static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) { - if (likely(vcpu->mmu.root_hpa != INVALID_PAGE)) + if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) return 0; return kvm_mmu_load(vcpu); @@ -348,7 +351,7 @@ static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) static inline int is_long_mode(struct kvm_vcpu *vcpu) { #ifdef CONFIG_X86_64 - return vcpu->shadow_efer & EFER_LME; + return vcpu->arch.shadow_efer & EFER_LME; #else return 0; #endif @@ -356,17 +359,17 @@ static inline int is_long_mode(struct kvm_vcpu *vcpu) static inline int is_pae(struct kvm_vcpu *vcpu) { - return vcpu->cr4 & X86_CR4_PAE; + return vcpu->arch.cr4 & X86_CR4_PAE; } static inline int is_pse(struct kvm_vcpu *vcpu) { - return vcpu->cr4 & X86_CR4_PSE; + return vcpu->arch.cr4 & X86_CR4_PSE; } static inline int is_paging(struct kvm_vcpu *vcpu) { - return vcpu->cr0 & X86_CR0_PG; + return vcpu->arch.cr0 & X86_CR0_PG; } int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); @@ -489,8 +492,8 @@ static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) static inline int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) { - return vcpu->mp_state == VCPU_MP_STATE_RUNNABLE - || vcpu->mp_state == VCPU_MP_STATE_SIPI_RECEIVED; + return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE + || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; } #endif diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c index 0a6ab06fde01..50b133f68743 100644 --- a/drivers/kvm/x86_emulate.c +++ b/drivers/kvm/x86_emulate.c @@ -769,8 +769,8 @@ x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) /* Shadow copy of register state. Committed on successful emulation. */ memset(c, 0, sizeof(struct decode_cache)); - c->eip = ctxt->vcpu->rip; - memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); + c->eip = ctxt->vcpu->arch.rip; + memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); switch (mode) { case X86EMUL_MODE_REAL: @@ -1226,7 +1226,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) * modify them. */ - memcpy(c->regs, ctxt->vcpu->regs, sizeof c->regs); + memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); saved_eip = c->eip; if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) @@ -1235,7 +1235,7 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) if (c->rep_prefix && (c->d & String)) { /* All REP prefixes have the same first termination condition */ if (c->regs[VCPU_REGS_RCX] == 0) { - ctxt->vcpu->rip = c->eip; + ctxt->vcpu->arch.rip = c->eip; goto done; } /* The second termination condition only applies for REPE @@ -1249,17 +1249,17 @@ x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) (c->b == 0xae) || (c->b == 0xaf)) { if ((c->rep_prefix == REPE_PREFIX) && ((ctxt->eflags & EFLG_ZF) == 0)) { - ctxt->vcpu->rip = c->eip; + ctxt->vcpu->arch.rip = c->eip; goto done; } if ((c->rep_prefix == REPNE_PREFIX) && ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { - ctxt->vcpu->rip = c->eip; + ctxt->vcpu->arch.rip = c->eip; goto done; } } c->regs[VCPU_REGS_RCX]--; - c->eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->arch.rip; } if (c->src.type == OP_MEM) { @@ -1628,7 +1628,7 @@ special_insn: c->dst.type = OP_NONE; /* Disable writeback. */ break; case 0xf4: /* hlt */ - ctxt->vcpu->halt_request = 1; + ctxt->vcpu->arch.halt_request = 1; goto done; case 0xf5: /* cmc */ /* complement carry flag from eflags reg */ @@ -1665,8 +1665,8 @@ writeback: goto done; /* Commit shadow register state. */ - memcpy(ctxt->vcpu->regs, c->regs, sizeof c->regs); - ctxt->vcpu->rip = c->eip; + memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs); + ctxt->vcpu->arch.rip = c->eip; done: if (rc == X86EMUL_UNHANDLEABLE) { @@ -1783,7 +1783,7 @@ twobyte_insn: rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); if (rc) { kvm_inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->arch.rip; } rc = X86EMUL_CONTINUE; c->dst.type = OP_NONE; @@ -1793,7 +1793,7 @@ twobyte_insn: rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); if (rc) { kvm_inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->rip; + c->eip = ctxt->vcpu->arch.rip; } else { c->regs[VCPU_REGS_RAX] = (u32)msr_data; c->regs[VCPU_REGS_RDX] = msr_data >> 32; -- cgit v1.2.3 From 1d737c8a68736db36e0aa502ace9da240704c5ae Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:35:10 +0800 Subject: KVM: Portability: Split mmu-related static inline functions to mmu.h Since these functions need to know the details of kvm or kvm_vcpu structure, it can't be put in x86.h. Create mmu.h to hold them. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 1 + drivers/kvm/mmu.c | 1 + drivers/kvm/mmu.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ drivers/kvm/svm.c | 1 + drivers/kvm/vmx.c | 1 + drivers/kvm/x86.c | 7 +++++++ drivers/kvm/x86.h | 44 -------------------------------------------- 7 files changed, 55 insertions(+), 44 deletions(-) create mode 100644 drivers/kvm/mmu.h (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 668a8300365d..c040bb3ac819 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -396,6 +396,7 @@ void kvm_arch_hardware_disable(void *garbage); int kvm_arch_hardware_setup(void); void kvm_arch_hardware_unsetup(void); void kvm_arch_check_processor_compat(void *rtn); +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); void kvm_free_physmem(struct kvm *kvm); diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index da1dedb497b8..1dc0e8c02c70 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -20,6 +20,7 @@ #include "vmx.h" #include "kvm.h" #include "x86.h" +#include "mmu.h" #include #include diff --git a/drivers/kvm/mmu.h b/drivers/kvm/mmu.h new file mode 100644 index 000000000000..9ebfd1cafe62 --- /dev/null +++ b/drivers/kvm/mmu.h @@ -0,0 +1,44 @@ +#ifndef __KVM_X86_MMU_H +#define __KVM_X86_MMU_H + +#include "kvm.h" + +static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) +{ + if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + __kvm_mmu_free_some_pages(vcpu); +} + +static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) +{ + if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) + return 0; + + return kvm_mmu_load(vcpu); +} + +static inline int is_long_mode(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + return vcpu->arch.shadow_efer & EFER_LME; +#else + return 0; +#endif +} + +static inline int is_pae(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr4 & X86_CR4_PAE; +} + +static inline int is_pse(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr4 & X86_CR4_PSE; +} + +static inline int is_paging(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr0 & X86_CR0_PG; +} + +#endif diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c index 7888638c02e8..e606f6d18669 100644 --- a/drivers/kvm/svm.c +++ b/drivers/kvm/svm.c @@ -17,6 +17,7 @@ #include "kvm_svm.h" #include "x86_emulate.h" #include "irq.h" +#include "mmu.h" #include #include diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index cf78ebb2f36e..d0f431d4fe4d 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -21,6 +21,7 @@ #include "irq.h" #include "vmx.h" #include "segment_descriptor.h" +#include "mmu.h" #include #include diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 5a2f33a84e4f..60f9722a06ba 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -19,6 +19,7 @@ #include "x86_emulate.h" #include "segment_descriptor.h" #include "irq.h" +#include "mmu.h" #include #include @@ -3139,3 +3140,9 @@ int kvm_arch_set_memory_region(struct kvm *kvm, return 0; } + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE + || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; +} diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 0e01ac75268c..0da4be9b0bad 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -334,44 +334,6 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu); int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); -static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) -{ - if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) - __kvm_mmu_free_some_pages(vcpu); -} - -static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) -{ - if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) - return 0; - - return kvm_mmu_load(vcpu); -} - -static inline int is_long_mode(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - return vcpu->arch.shadow_efer & EFER_LME; -#else - return 0; -#endif -} - -static inline int is_pae(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr4 & X86_CR4_PAE; -} - -static inline int is_pse(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr4 & X86_CR4_PSE; -} - -static inline int is_paging(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr0 & X86_CR0_PG; -} - int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); int complete_pio(struct kvm_vcpu *vcpu); @@ -490,10 +452,4 @@ static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) #define TSS_REDIRECTION_SIZE (256 / 8) #define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) -static inline int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE - || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; -} - #endif -- cgit v1.2.3 From d657a98e3c20537d8b4d44aef51cf4311d96f2b0 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:41:22 +0800 Subject: KVM: Portability: Move kvm_vcpu definition back to kvm.h This patches moves kvm_vcpu definition to kvm.h, and finally kvm.h includes x86.h. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/irq.h | 2 +- drivers/kvm/kvm.h | 113 ++++------------------------------------------------ drivers/kvm/types.h | 13 ++++++ drivers/kvm/x86.h | 102 +++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 117 insertions(+), 113 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index e88d93957d24..a688c51d0a39 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -26,7 +26,7 @@ #include #include #include "iodev.h" -#include "x86.h" +#include "kvm.h" struct kvm; struct kvm_vcpu; diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index c040bb3ac819..53717be80bbe 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -22,17 +22,13 @@ #include "types.h" +#include "x86.h" + #define KVM_MAX_VCPUS 4 #define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 /* memory slots that does not exposed to userspace */ #define KVM_PRIVATE_MEM_SLOTS 4 -#define KVM_PERMILLE_MMU_PAGES 20 -#define KVM_MIN_ALLOC_MMU_PAGES 64 -#define KVM_NUM_MMU_PAGES 1024 -#define KVM_MIN_FREE_MMU_PAGES 5 -#define KVM_REFILL_PAGES 25 -#define KVM_MAX_CPUID_ENTRIES 40 #define KVM_PIO_PAGE_OFFSET 1 @@ -41,111 +37,16 @@ */ #define KVM_REQ_TLB_FLUSH 0 -#define NR_PTE_CHAIN_ENTRIES 5 - -struct kvm_pte_chain { - u64 *parent_ptes[NR_PTE_CHAIN_ENTRIES]; - struct hlist_node link; -}; - -/* - * kvm_mmu_page_role, below, is defined as: - * - * bits 0:3 - total guest paging levels (2-4, or zero for real mode) - * bits 4:7 - page table level for this shadow (1-4) - * bits 8:9 - page table quadrant for 2-level guests - * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) - * bits 17:19 - common access permissions for all ptes in this shadow page - */ -union kvm_mmu_page_role { - unsigned word; - struct { - unsigned glevels : 4; - unsigned level : 4; - unsigned quadrant : 2; - unsigned pad_for_nice_hex_output : 6; - unsigned metaphysical : 1; - unsigned access : 3; - }; -}; - -struct kvm_mmu_page { - struct list_head link; - struct hlist_node hash_link; - - /* - * The following two entries are used to key the shadow page in the - * hash table. - */ - gfn_t gfn; - union kvm_mmu_page_role role; - - u64 *spt; - /* hold the gfn of each spte inside spt */ - gfn_t *gfns; - unsigned long slot_bitmap; /* One bit set per slot which has memory - * in this shadow page. - */ - int multimapped; /* More than one parent_pte? */ - int root_count; /* Currently serving as active root */ - union { - u64 *parent_pte; /* !multimapped */ - struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */ - }; -}; struct kvm_vcpu; extern struct kmem_cache *kvm_vcpu_cache; -/* - * x86 supports 3 paging modes (4-level 64-bit, 3-level 64-bit, and 2-level - * 32-bit). The kvm_mmu structure abstracts the details of the current mmu - * mode. - */ -struct kvm_mmu { - void (*new_cr3)(struct kvm_vcpu *vcpu); - int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); - void (*free)(struct kvm_vcpu *vcpu); - gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); - void (*prefetch_page)(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page); - hpa_t root_hpa; - int root_level; - int shadow_root_level; - - u64 *pae_root; -}; - -#define KVM_NR_MEM_OBJS 40 - -/* - * We don't want allocation failures within the mmu code, so we preallocate - * enough memory for a single page fault in a cache. - */ -struct kvm_mmu_memory_cache { - int nobjs; - void *objects[KVM_NR_MEM_OBJS]; -}; - struct kvm_guest_debug { int enabled; unsigned long bp[4]; int singlestep; }; -struct kvm_pio_request { - unsigned long count; - int cur_count; - struct page *guest_pages[2]; - unsigned guest_page_offset; - int in; - int port; - int size; - int string; - int down; - int rep; -}; - struct kvm_vcpu_stat { u32 pf_fixed; u32 pf_guest; @@ -218,6 +119,12 @@ void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_vcpu_stat stat; \ KVM_VCPU_MMIO +struct kvm_vcpu { + KVM_VCPU_COMM; + + struct kvm_vcpu_arch arch; +}; + struct kvm_mem_alias { gfn_t base_gfn; unsigned long npages; @@ -441,8 +348,4 @@ struct kvm_stats_debugfs_item { }; extern struct kvm_stats_debugfs_item debugfs_entries[]; -#if defined(CONFIG_X86) -#include "x86.h" -#endif - #endif diff --git a/drivers/kvm/types.h b/drivers/kvm/types.h index 6ad76231b8ff..1c4e46decb22 100644 --- a/drivers/kvm/types.h +++ b/drivers/kvm/types.h @@ -38,4 +38,17 @@ typedef unsigned long hva_t; typedef u64 hpa_t; typedef unsigned long hfn_t; +struct kvm_pio_request { + unsigned long count; + int cur_count; + struct page *guest_pages[2]; + unsigned guest_page_offset; + int in; + int port; + int size; + int string; + int down; + int rep; +}; + #endif /* __KVM_TYPES_H__ */ diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 0da4be9b0bad..a0dd4473b8f8 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -11,8 +11,6 @@ #ifndef KVM_X86_H #define KVM_X86_H -#include "kvm.h" - #include #include @@ -21,6 +19,8 @@ #include +#include "types.h" + #define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) #define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) #define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) @@ -54,9 +54,19 @@ #define IOPL_SHIFT 12 +#define KVM_PERMILLE_MMU_PAGES 20 +#define KVM_MIN_ALLOC_MMU_PAGES 64 +#define KVM_NUM_MMU_PAGES 1024 +#define KVM_MIN_FREE_MMU_PAGES 5 +#define KVM_REFILL_PAGES 25 +#define KVM_MAX_CPUID_ENTRIES 40 + extern spinlock_t kvm_lock; extern struct list_head vm_list; +struct kvm_vcpu; +struct kvm; + enum { VCPU_REGS_RAX = 0, VCPU_REGS_RCX = 1, @@ -92,6 +102,89 @@ enum { #include "x86_emulate.h" +#define KVM_NR_MEM_OBJS 40 + +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ +struct kvm_mmu_memory_cache { + int nobjs; + void *objects[KVM_NR_MEM_OBJS]; +}; + +#define NR_PTE_CHAIN_ENTRIES 5 + +struct kvm_pte_chain { + u64 *parent_ptes[NR_PTE_CHAIN_ENTRIES]; + struct hlist_node link; +}; + +/* + * kvm_mmu_page_role, below, is defined as: + * + * bits 0:3 - total guest paging levels (2-4, or zero for real mode) + * bits 4:7 - page table level for this shadow (1-4) + * bits 8:9 - page table quadrant for 2-level guests + * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) + * bits 17:19 - common access permissions for all ptes in this shadow page + */ +union kvm_mmu_page_role { + unsigned word; + struct { + unsigned glevels : 4; + unsigned level : 4; + unsigned quadrant : 2; + unsigned pad_for_nice_hex_output : 6; + unsigned metaphysical : 1; + unsigned access : 3; + }; +}; + +struct kvm_mmu_page { + struct list_head link; + struct hlist_node hash_link; + + /* + * The following two entries are used to key the shadow page in the + * hash table. + */ + gfn_t gfn; + union kvm_mmu_page_role role; + + u64 *spt; + /* hold the gfn of each spte inside spt */ + gfn_t *gfns; + unsigned long slot_bitmap; /* One bit set per slot which has memory + * in this shadow page. + */ + int multimapped; /* More than one parent_pte? */ + int root_count; /* Currently serving as active root */ + union { + u64 *parent_pte; /* !multimapped */ + struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */ + }; +}; + +/* + * x86 supports 3 paging modes (4-level 64-bit, 3-level 64-bit, and 2-level + * 32-bit). The kvm_mmu structure abstracts the details of the current mmu + * mode. + */ +struct kvm_mmu { + void (*new_cr3)(struct kvm_vcpu *vcpu); + int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); + void (*free)(struct kvm_vcpu *vcpu); + gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); + void (*prefetch_page)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page); + hpa_t root_hpa; + int root_level; + int shadow_root_level; + + u64 *pae_root; +}; + struct kvm_vcpu_arch { u64 host_tsc; int interrupt_window_open; @@ -162,11 +255,6 @@ struct kvm_vcpu_arch { struct x86_emulate_ctxt emulate_ctxt; }; -struct kvm_vcpu { - KVM_VCPU_COMM; - - struct kvm_vcpu_arch arch; -}; struct descriptor_table { u16 limit; -- cgit v1.2.3 From d17fbbf7384d3b639f3c3299b5248ec4c4404556 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:45:31 +0800 Subject: KVM: Portability: Expand the KVM_VCPU_COMM in kvm_vcpu structure. This patches removes KVM_COMM macro, original it is hold kvm_vcpu common fields. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 53 ++++++++++++++++++++++------------------------------- 1 file changed, 22 insertions(+), 31 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 53717be80bbe..039faa766c57 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -87,41 +87,32 @@ struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr); void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev); +struct kvm_vcpu { + struct kvm *kvm; + struct preempt_notifier preempt_notifier; + int vcpu_id; + struct mutex mutex; + int cpu; + struct kvm_run *run; + int guest_mode; + unsigned long requests; + struct kvm_guest_debug guest_debug; + int fpu_active; + int guest_fpu_loaded; + wait_queue_head_t wq; + int sigset_active; + sigset_t sigset; + struct kvm_vcpu_stat stat; + #ifdef CONFIG_HAS_IOMEM -#define KVM_VCPU_MMIO \ - int mmio_needed; \ - int mmio_read_completed; \ - int mmio_is_write; \ - int mmio_size; \ - unsigned char mmio_data[8]; \ + int mmio_needed; + int mmio_read_completed; + int mmio_is_write; + int mmio_size; + unsigned char mmio_data[8]; gpa_t mmio_phys_addr; - -#else -#define KVM_VCPU_MMIO - #endif -#define KVM_VCPU_COMM \ - struct kvm *kvm; \ - struct preempt_notifier preempt_notifier; \ - int vcpu_id; \ - struct mutex mutex; \ - int cpu; \ - struct kvm_run *run; \ - int guest_mode; \ - unsigned long requests; \ - struct kvm_guest_debug guest_debug; \ - int fpu_active; \ - int guest_fpu_loaded; \ - wait_queue_head_t wq; \ - int sigset_active; \ - sigset_t sigset; \ - struct kvm_vcpu_stat stat; \ - KVM_VCPU_MMIO - -struct kvm_vcpu { - KVM_VCPU_COMM; - struct kvm_vcpu_arch arch; }; -- cgit v1.2.3 From 77b4c255af34e73ea1efd1c3384bbe91361c81e6 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:49:26 +0800 Subject: KVM: Portability: Move kvm_vcpu_stat to x86.h This patches moves kvm_vcpu_stat to x86.h, so every arch can define its own kvm_vcpu_stat structure. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 22 ---------------------- drivers/kvm/x86.h | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 22 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 039faa766c57..9ea7149e741d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -47,28 +47,6 @@ struct kvm_guest_debug { int singlestep; }; -struct kvm_vcpu_stat { - u32 pf_fixed; - u32 pf_guest; - u32 tlb_flush; - u32 invlpg; - - u32 exits; - u32 io_exits; - u32 mmio_exits; - u32 signal_exits; - u32 irq_window_exits; - u32 halt_exits; - u32 halt_wakeup; - u32 request_irq_exits; - u32 irq_exits; - u32 host_state_reload; - u32 efer_reload; - u32 fpu_reload; - u32 insn_emulation; - u32 insn_emulation_fail; -}; - /* * It would be nice to use something smarter than a linear search, TBD... * Thankfully we dont expect many devices to register (famous last words :), diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index a0dd4473b8f8..4ca848536365 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -255,6 +255,27 @@ struct kvm_vcpu_arch { struct x86_emulate_ctxt emulate_ctxt; }; +struct kvm_vcpu_stat { + u32 pf_fixed; + u32 pf_guest; + u32 tlb_flush; + u32 invlpg; + + u32 exits; + u32 io_exits; + u32 mmio_exits; + u32 signal_exits; + u32 irq_window_exits; + u32 halt_exits; + u32 halt_wakeup; + u32 request_irq_exits; + u32 irq_exits; + u32 host_state_reload; + u32 efer_reload; + u32 fpu_reload; + u32 insn_emulation; + u32 insn_emulation_fail; +}; struct descriptor_table { u16 limit; -- cgit v1.2.3 From d69fb81f0554fb980e4b1d3db4e44351c2c4a4a2 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 09:54:20 +0800 Subject: KVM: Portability: Move memslot aliases to new struct kvm_arch This patches create kvm_arch to hold arch-specific kvm fileds and moves fields naliases and aliases to kvm_arch. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 10 +--------- drivers/kvm/x86.c | 10 +++++----- drivers/kvm/x86.h | 13 +++++++++++++ 3 files changed, 19 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 9ea7149e741d..bf5b85c1f094 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -25,7 +25,6 @@ #include "x86.h" #define KVM_MAX_VCPUS 4 -#define KVM_ALIAS_SLOTS 4 #define KVM_MEMORY_SLOTS 8 /* memory slots that does not exposed to userspace */ #define KVM_PRIVATE_MEM_SLOTS 4 @@ -94,12 +93,6 @@ struct kvm_vcpu { struct kvm_vcpu_arch arch; }; -struct kvm_mem_alias { - gfn_t base_gfn; - unsigned long npages; - gfn_t target_gfn; -}; - struct kvm_memory_slot { gfn_t base_gfn; unsigned long npages; @@ -123,8 +116,6 @@ struct kvm_vm_stat { struct kvm { struct mutex lock; /* protects everything except vcpus */ struct mm_struct *mm; /* userspace tied to this vm */ - int naliases; - struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; int nmemslots; struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS]; @@ -147,6 +138,7 @@ struct kvm { unsigned int tss_addr; struct page *apic_access_page; struct kvm_vm_stat stat; + struct kvm_arch arch; }; /* The guest did something we don't support. */ diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 60f9722a06ba..7e1bd526bd5c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1191,8 +1191,8 @@ gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) int i; struct kvm_mem_alias *alias; - for (i = 0; i < kvm->naliases; ++i) { - alias = &kvm->aliases[i]; + for (i = 0; i < kvm->arch.naliases; ++i) { + alias = &kvm->arch.aliases[i]; if (gfn >= alias->base_gfn && gfn < alias->base_gfn + alias->npages) return alias->target_gfn + gfn - alias->base_gfn; @@ -1228,15 +1228,15 @@ static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, mutex_lock(&kvm->lock); - p = &kvm->aliases[alias->slot]; + p = &kvm->arch.aliases[alias->slot]; p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; p->npages = alias->memory_size >> PAGE_SHIFT; p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT; for (n = KVM_ALIAS_SLOTS; n > 0; --n) - if (kvm->aliases[n - 1].npages) + if (kvm->arch.aliases[n - 1].npages) break; - kvm->naliases = n; + kvm->arch.naliases = n; kvm_mmu_zap_all(kvm); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 4ca848536365..be84f2b89095 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -54,6 +54,8 @@ #define IOPL_SHIFT 12 +#define KVM_ALIAS_SLOTS 4 + #define KVM_PERMILLE_MMU_PAGES 20 #define KVM_MIN_ALLOC_MMU_PAGES 64 #define KVM_NUM_MMU_PAGES 1024 @@ -255,6 +257,17 @@ struct kvm_vcpu_arch { struct x86_emulate_ctxt emulate_ctxt; }; +struct kvm_mem_alias { + gfn_t base_gfn; + unsigned long npages; + gfn_t target_gfn; +}; + +struct kvm_arch{ + int naliases; + struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; +}; + struct kvm_vcpu_stat { u32 pf_fixed; u32 pf_guest; -- cgit v1.2.3 From f05e70ac03a6614af12194a014b338ec5594cb5c Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 10:01:48 +0800 Subject: KVM: Portability: Move mmu-related fields to kvm_arch This patches moves mmu-related fields to kvm_arch. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 8 -------- drivers/kvm/mmu.c | 58 ++++++++++++++++++++++++++++--------------------------- drivers/kvm/mmu.h | 2 +- drivers/kvm/x86.c | 8 ++++---- drivers/kvm/x86.h | 9 +++++++++ 5 files changed, 44 insertions(+), 41 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index bf5b85c1f094..65de5e4225f7 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -119,14 +119,6 @@ struct kvm { int nmemslots; struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS]; - /* - * Hash table of struct kvm_mmu_page. - */ - struct list_head active_mmu_pages; - unsigned int n_free_mmu_pages; - unsigned int n_requested_mmu_pages; - unsigned int n_alloc_mmu_pages; - struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; struct list_head vm_list; struct file *filp; diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c index 1dc0e8c02c70..c26d83f86a3a 100644 --- a/drivers/kvm/mmu.c +++ b/drivers/kvm/mmu.c @@ -553,7 +553,7 @@ static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp) __free_page(virt_to_page(sp->spt)); __free_page(virt_to_page(sp->gfns)); kfree(sp); - ++kvm->n_free_mmu_pages; + ++kvm->arch.n_free_mmu_pages; } static unsigned kvm_page_table_hashfn(gfn_t gfn) @@ -566,19 +566,19 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, { struct kvm_mmu_page *sp; - if (!vcpu->kvm->n_free_mmu_pages) + if (!vcpu->kvm->arch.n_free_mmu_pages) return NULL; sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp); sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); set_page_private(virt_to_page(sp->spt), (unsigned long)sp); - list_add(&sp->link, &vcpu->kvm->active_mmu_pages); + list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); ASSERT(is_empty_shadow_page(sp->spt)); sp->slot_bitmap = 0; sp->multimapped = 0; sp->parent_pte = parent_pte; - --vcpu->kvm->n_free_mmu_pages; + --vcpu->kvm->arch.n_free_mmu_pages; return sp; } @@ -666,7 +666,7 @@ static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &kvm->mmu_page_hash[index]; + bucket = &kvm->arch.mmu_page_hash[index]; hlist_for_each_entry(sp, node, bucket, hash_link) if (sp->gfn == gfn && !sp->role.metaphysical) { pgprintk("%s: found role %x\n", @@ -705,7 +705,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__, gfn, role.word); index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->mmu_page_hash[index]; + bucket = &vcpu->kvm->arch.mmu_page_hash[index]; hlist_for_each_entry(sp, node, bucket, hash_link) if (sp->gfn == gfn && sp->role.word == role.word) { mmu_page_add_parent_pte(vcpu, sp, parent_pte); @@ -796,7 +796,7 @@ static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) hlist_del(&sp->hash_link); kvm_mmu_free_page(kvm, sp); } else - list_move(&sp->link, &kvm->active_mmu_pages); + list_move(&sp->link, &kvm->arch.active_mmu_pages); kvm_mmu_reset_last_pte_updated(kvm); } @@ -812,26 +812,26 @@ void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages) * change the value */ - if ((kvm->n_alloc_mmu_pages - kvm->n_free_mmu_pages) > + if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) > kvm_nr_mmu_pages) { - int n_used_mmu_pages = kvm->n_alloc_mmu_pages - - kvm->n_free_mmu_pages; + int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages + - kvm->arch.n_free_mmu_pages; while (n_used_mmu_pages > kvm_nr_mmu_pages) { struct kvm_mmu_page *page; - page = container_of(kvm->active_mmu_pages.prev, + page = container_of(kvm->arch.active_mmu_pages.prev, struct kvm_mmu_page, link); kvm_mmu_zap_page(kvm, page); n_used_mmu_pages--; } - kvm->n_free_mmu_pages = 0; + kvm->arch.n_free_mmu_pages = 0; } else - kvm->n_free_mmu_pages += kvm_nr_mmu_pages - - kvm->n_alloc_mmu_pages; + kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages + - kvm->arch.n_alloc_mmu_pages; - kvm->n_alloc_mmu_pages = kvm_nr_mmu_pages; + kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages; } static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) @@ -845,7 +845,7 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); r = 0; index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &kvm->mmu_page_hash[index]; + bucket = &kvm->arch.mmu_page_hash[index]; hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) if (sp->gfn == gfn && !sp->role.metaphysical) { pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, @@ -1362,7 +1362,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, vcpu->arch.last_pte_updated = NULL; } index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->mmu_page_hash[index]; + bucket = &vcpu->kvm->arch.mmu_page_hash[index]; hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) { if (sp->gfn != gfn || sp->role.metaphysical) continue; @@ -1428,10 +1428,10 @@ int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { - while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) { + while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) { struct kvm_mmu_page *sp; - sp = container_of(vcpu->kvm->active_mmu_pages.prev, + sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev, struct kvm_mmu_page, link); kvm_mmu_zap_page(vcpu->kvm, sp); ++vcpu->kvm->stat.mmu_recycled; @@ -1482,8 +1482,8 @@ static void free_mmu_pages(struct kvm_vcpu *vcpu) { struct kvm_mmu_page *sp; - while (!list_empty(&vcpu->kvm->active_mmu_pages)) { - sp = container_of(vcpu->kvm->active_mmu_pages.next, + while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) { + sp = container_of(vcpu->kvm->arch.active_mmu_pages.next, struct kvm_mmu_page, link); kvm_mmu_zap_page(vcpu->kvm, sp); } @@ -1497,10 +1497,12 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu) ASSERT(vcpu); - if (vcpu->kvm->n_requested_mmu_pages) - vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_requested_mmu_pages; + if (vcpu->kvm->arch.n_requested_mmu_pages) + vcpu->kvm->arch.n_free_mmu_pages = + vcpu->kvm->arch.n_requested_mmu_pages; else - vcpu->kvm->n_free_mmu_pages = vcpu->kvm->n_alloc_mmu_pages; + vcpu->kvm->arch.n_free_mmu_pages = + vcpu->kvm->arch.n_alloc_mmu_pages; /* * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64. * Therefore we need to allocate shadow page tables in the first @@ -1549,7 +1551,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) { struct kvm_mmu_page *sp; - list_for_each_entry(sp, &kvm->active_mmu_pages, link) { + list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) { int i; u64 *pt; @@ -1568,7 +1570,7 @@ void kvm_mmu_zap_all(struct kvm *kvm) { struct kvm_mmu_page *sp, *node; - list_for_each_entry_safe(sp, node, &kvm->active_mmu_pages, link) + list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) kvm_mmu_zap_page(kvm, sp); kvm_flush_remote_tlbs(kvm); @@ -1738,7 +1740,7 @@ static int count_writable_mappings(struct kvm_vcpu *vcpu) struct kvm_mmu_page *sp; int i; - list_for_each_entry(sp, &vcpu->kvm->active_mmu_pages, link) { + list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { u64 *pt = sp->spt; if (sp->role.level != PT_PAGE_TABLE_LEVEL) @@ -1774,7 +1776,7 @@ static void audit_write_protection(struct kvm_vcpu *vcpu) unsigned long *rmapp; gfn_t gfn; - list_for_each_entry(sp, &vcpu->kvm->active_mmu_pages, link) { + list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { if (sp->role.metaphysical) continue; diff --git a/drivers/kvm/mmu.h b/drivers/kvm/mmu.h index 9ebfd1cafe62..cbfc272262df 100644 --- a/drivers/kvm/mmu.h +++ b/drivers/kvm/mmu.h @@ -5,7 +5,7 @@ static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) { - if (unlikely(vcpu->kvm->n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + if (unlikely(vcpu->kvm->arch.n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) __kvm_mmu_free_some_pages(vcpu); } diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index 7e1bd526bd5c..c0e95fb9f46c 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1175,7 +1175,7 @@ static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, mutex_lock(&kvm->lock); kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); - kvm->n_requested_mmu_pages = kvm_nr_mmu_pages; + kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; mutex_unlock(&kvm->lock); return 0; @@ -1183,7 +1183,7 @@ static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) { - return kvm->n_alloc_mmu_pages; + return kvm->arch.n_alloc_mmu_pages; } gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) @@ -3051,7 +3051,7 @@ struct kvm *kvm_arch_create_vm(void) if (!kvm) return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&kvm->active_mmu_pages); + INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); return kvm; } @@ -3130,7 +3130,7 @@ int kvm_arch_set_memory_region(struct kvm *kvm, } } - if (!kvm->n_requested_mmu_pages) { + if (!kvm->arch.n_requested_mmu_pages) { unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index be84f2b89095..5cdc3666e212 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -266,6 +266,15 @@ struct kvm_mem_alias { struct kvm_arch{ int naliases; struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; + + unsigned int n_free_mmu_pages; + unsigned int n_requested_mmu_pages; + unsigned int n_alloc_mmu_pages; + struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; + /* + * Hash table of struct kvm_mmu_page. + */ + struct list_head active_mmu_pages; }; struct kvm_vcpu_stat { -- cgit v1.2.3 From d7deeeb02cf73fd98cb71a6a0a5dffab9ef79556 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 10:17:34 +0800 Subject: KVM: Portability: move vpic and vioapic to kvm_arch This patches moves two fields vpid and vioapic to kvm_arch Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/ioapic.c | 4 ++-- drivers/kvm/irq.h | 4 ++-- drivers/kvm/kvm.h | 2 -- drivers/kvm/x86.c | 14 +++++++------- drivers/kvm/x86.h | 2 ++ 5 files changed, 13 insertions(+), 13 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c index 04910368c251..f8236774c1b4 100644 --- a/drivers/kvm/ioapic.c +++ b/drivers/kvm/ioapic.c @@ -271,7 +271,7 @@ static int get_eoi_gsi(struct kvm_ioapic *ioapic, int vector) void kvm_ioapic_update_eoi(struct kvm *kvm, int vector) { - struct kvm_ioapic *ioapic = kvm->vioapic; + struct kvm_ioapic *ioapic = kvm->arch.vioapic; union ioapic_redir_entry *ent; int gsi; @@ -390,7 +390,7 @@ int kvm_ioapic_init(struct kvm *kvm) ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL); if (!ioapic) return -ENOMEM; - kvm->vioapic = ioapic; + kvm->arch.vioapic = ioapic; kvm_ioapic_reset(ioapic); ioapic->dev.read = ioapic_mmio_read; ioapic->dev.write = ioapic_mmio_write; diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h index a688c51d0a39..6e023dc3f848 100644 --- a/drivers/kvm/irq.h +++ b/drivers/kvm/irq.h @@ -148,12 +148,12 @@ do { \ static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) { - return kvm->vpic; + return kvm->arch.vpic; } static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) { - return kvm->vioapic; + return kvm->arch.vioapic; } static inline int irqchip_in_kernel(struct kvm *kvm) diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index 65de5e4225f7..d9ce916e98ab 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -124,8 +124,6 @@ struct kvm { struct file *filp; struct kvm_io_bus mmio_bus; struct kvm_io_bus pio_bus; - struct kvm_pic *vpic; - struct kvm_ioapic *vioapic; int round_robin_prev_vcpu; unsigned int tss_addr; struct page *apic_access_page; diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c index c0e95fb9f46c..b37c0093d728 100644 --- a/drivers/kvm/x86.c +++ b/drivers/kvm/x86.c @@ -1386,12 +1386,12 @@ long kvm_arch_vm_ioctl(struct file *filp, } case KVM_CREATE_IRQCHIP: r = -ENOMEM; - kvm->vpic = kvm_create_pic(kvm); - if (kvm->vpic) { + kvm->arch.vpic = kvm_create_pic(kvm); + if (kvm->arch.vpic) { r = kvm_ioapic_init(kvm); if (r) { - kfree(kvm->vpic); - kvm->vpic = NULL; + kfree(kvm->arch.vpic); + kvm->arch.vpic = NULL; goto out; } } else @@ -1409,7 +1409,7 @@ long kvm_arch_vm_ioctl(struct file *filp, kvm_pic_set_irq(pic_irqchip(kvm), irq_event.irq, irq_event.level); - kvm_ioapic_set_irq(kvm->vioapic, + kvm_ioapic_set_irq(kvm->arch.vioapic, irq_event.irq, irq_event.level); mutex_unlock(&kvm->lock); @@ -3084,8 +3084,8 @@ static void kvm_free_vcpus(struct kvm *kvm) void kvm_arch_destroy_vm(struct kvm *kvm) { - kfree(kvm->vpic); - kfree(kvm->vioapic); + kfree(kvm->arch.vpic); + kfree(kvm->arch.vioapic); kvm_free_vcpus(kvm); kvm_free_physmem(kvm); kfree(kvm); diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 5cdc3666e212..ef23a30bf048 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -275,6 +275,8 @@ struct kvm_arch{ * Hash table of struct kvm_mmu_page. */ struct list_head active_mmu_pages; + struct kvm_pic *vpic; + struct kvm_ioapic *vioapic; }; struct kvm_vcpu_stat { -- cgit v1.2.3 From bfc6d222bdb1123d12c1193bcd7c755e6617b405 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 10:20:16 +0800 Subject: KVM: Portability: Move round_robin_prev_vcpu and tss_addr to kvm_arch This patches moves two fields round_robin_prev_vcpu and tss to kvm_arch. Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 3 --- drivers/kvm/lapic.c | 4 ++-- drivers/kvm/vmx.c | 12 ++++++------ drivers/kvm/x86.h | 4 ++++ 4 files changed, 12 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index d9ce916e98ab..ab9657a6f0ce 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -124,9 +124,6 @@ struct kvm { struct file *filp; struct kvm_io_bus mmio_bus; struct kvm_io_bus pio_bus; - int round_robin_prev_vcpu; - unsigned int tss_addr; - struct page *apic_access_page; struct kvm_vm_stat stat; struct kvm_arch arch; }; diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c index 5c9f46784c26..8c74bf184a07 100644 --- a/drivers/kvm/lapic.c +++ b/drivers/kvm/lapic.c @@ -404,7 +404,7 @@ static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, int next; struct kvm_lapic *apic = NULL; - last = kvm->round_robin_prev_vcpu; + last = kvm->arch.round_robin_prev_vcpu; next = last; do { @@ -417,7 +417,7 @@ static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, break; apic = NULL; } while (next != last); - kvm->round_robin_prev_vcpu = next; + kvm->arch.round_robin_prev_vcpu = next; if (!apic) printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n"); diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index d0f431d4fe4d..7e61a560aab5 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -1143,12 +1143,12 @@ static void enter_pmode(struct kvm_vcpu *vcpu) static gva_t rmode_tss_base(struct kvm *kvm) { - if (!kvm->tss_addr) { + if (!kvm->arch.tss_addr) { gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3; return base_gfn << PAGE_SHIFT; } - return kvm->tss_addr; + return kvm->arch.tss_addr; } static void fix_rmode_seg(int seg, struct kvm_save_segment *save) @@ -1473,7 +1473,7 @@ static int alloc_apic_access_page(struct kvm *kvm) int r = 0; mutex_lock(&kvm->lock); - if (kvm->apic_access_page) + if (kvm->arch.apic_access_page) goto out; kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT; kvm_userspace_mem.flags = 0; @@ -1482,7 +1482,7 @@ static int alloc_apic_access_page(struct kvm *kvm) r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0); if (r) goto out; - kvm->apic_access_page = gfn_to_page(kvm, 0xfee00); + kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00); out: mutex_unlock(&kvm->lock); return r; @@ -1699,7 +1699,7 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) vmcs_write64(APIC_ACCESS_ADDR, - page_to_phys(vmx->vcpu.kvm->apic_access_page)); + page_to_phys(vmx->vcpu.kvm->arch.apic_access_page)); vmx->vcpu.arch.cr0 = 0x60000010; vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */ @@ -1789,7 +1789,7 @@ static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) ret = kvm_set_memory_region(kvm, &tss_mem, 0); if (ret) return ret; - kvm->tss_addr = addr; + kvm->arch.tss_addr = addr; return 0; } diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index ef23a30bf048..945583934853 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -277,6 +277,10 @@ struct kvm_arch{ struct list_head active_mmu_pages; struct kvm_pic *vpic; struct kvm_ioapic *vioapic; + + int round_robin_prev_vcpu; + unsigned int tss_addr; + struct page *apic_access_page; }; struct kvm_vcpu_stat { -- cgit v1.2.3 From 0711456c0d78335d24e4f92aa40d32c78fd44aa8 Mon Sep 17 00:00:00 2001 From: Zhang Xiantao Date: Fri, 14 Dec 2007 10:23:23 +0800 Subject: KVM: Portability: Move kvm_vm_stat to x86.h This patch moves kvm_vm_stat to x86.h, and every arch can define its own kvm_vm_stat in $arch.h Signed-off-by: Zhang Xiantao Acked-by: Carsten Otte Signed-off-by: Avi Kivity --- drivers/kvm/kvm.h | 10 ---------- drivers/kvm/x86.h | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h index ab9657a6f0ce..bf6a3b330a3d 100644 --- a/drivers/kvm/kvm.h +++ b/drivers/kvm/kvm.h @@ -103,16 +103,6 @@ struct kvm_memory_slot { int user_alloc; }; -struct kvm_vm_stat { - u32 mmu_shadow_zapped; - u32 mmu_pte_write; - u32 mmu_pte_updated; - u32 mmu_pde_zapped; - u32 mmu_flooded; - u32 mmu_recycled; - u32 remote_tlb_flush; -}; - struct kvm { struct mutex lock; /* protects everything except vcpus */ struct mm_struct *mm; /* userspace tied to this vm */ diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h index 945583934853..dfb8091971a9 100644 --- a/drivers/kvm/x86.h +++ b/drivers/kvm/x86.h @@ -283,6 +283,16 @@ struct kvm_arch{ struct page *apic_access_page; }; +struct kvm_vm_stat { + u32 mmu_shadow_zapped; + u32 mmu_pte_write; + u32 mmu_pte_updated; + u32 mmu_pde_zapped; + u32 mmu_flooded; + u32 mmu_recycled; + u32 remote_tlb_flush; +}; + struct kvm_vcpu_stat { u32 pf_fixed; u32 pf_guest; -- cgit v1.2.3 From 9584bf2c93f56656dba0de8f6c75b54ca7995143 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 13 Dec 2007 10:21:10 -0600 Subject: KVM: VMX: Add printk_ratelimit in vmx_intr_assist Add printk_ratelimit check in front of printk. This prevents spamming of the message during 32-bit ubuntu 6.06server install. Previously, it would hang during the partition formatting stage. Signed-off-by: Ryan Harper Signed-off-by: Avi Kivity --- drivers/kvm/vmx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c index 7e61a560aab5..11ca2340d38f 100644 --- a/drivers/kvm/vmx.c +++ b/drivers/kvm/vmx.c @@ -2263,7 +2263,8 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) if (intr_info_field & INTR_INFO_VALID_MASK) { if (idtv_info_field & INTR_INFO_VALID_MASK) { /* TODO: fault when IDT_Vectoring */ - printk(KERN_ERR "Fault when IDT_Vectoring\n"); + if (printk_ratelimit()) + printk(KERN_ERR "Fault when IDT_Vectoring\n"); } if (has_ext_irq) enable_irq_window(vcpu); -- cgit v1.2.3 From edf884172e9828c6234b254208af04655855038d Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 16 Dec 2007 11:02:48 +0200 Subject: KVM: Move arch dependent files to new directory arch/x86/kvm/ This paves the way for multiple architecture support. Note that while ioapic.c could potentially be shared with ia64, it is also moved. Signed-off-by: Avi Kivity --- arch/x86/Kconfig | 2 + arch/x86/Makefile | 2 + arch/x86/kvm/Kconfig | 57 + arch/x86/kvm/Makefile | 15 + arch/x86/kvm/i8259.c | 450 ++++++ arch/x86/kvm/ioapic.c | 400 +++++ arch/x86/kvm/irq.c | 98 ++ arch/x86/kvm/irq.h | 195 +++ arch/x86/kvm/kvm_svm.h | 45 + arch/x86/kvm/lapic.c | 1085 +++++++++++++ arch/x86/kvm/mmu.c | 1805 +++++++++++++++++++++ arch/x86/kvm/mmu.h | 44 + arch/x86/kvm/paging_tmpl.h | 461 ++++++ arch/x86/kvm/segment_descriptor.h | 29 + arch/x86/kvm/svm.c | 1725 ++++++++++++++++++++ arch/x86/kvm/svm.h | 325 ++++ arch/x86/kvm/vmx.c | 2671 +++++++++++++++++++++++++++++++ arch/x86/kvm/vmx.h | 324 ++++ arch/x86/kvm/x86.c | 3146 ++++++++++++++++++++++++++++++++++++ arch/x86/kvm/x86_emulate.c | 1912 ++++++++++++++++++++++ drivers/Kconfig | 2 - drivers/Makefile | 1 - drivers/kvm/Kconfig | 57 - drivers/kvm/Makefile | 10 - drivers/kvm/i8259.c | 449 ------ drivers/kvm/ioapic.c | 402 ----- drivers/kvm/iodev.h | 2 +- drivers/kvm/irq.c | 99 -- drivers/kvm/irq.h | 196 --- drivers/kvm/kvm.h | 289 ---- drivers/kvm/kvm_main.c | 2 +- drivers/kvm/kvm_svm.h | 45 - drivers/kvm/lapic.c | 1087 ------------- drivers/kvm/mmu.c | 1806 --------------------- drivers/kvm/mmu.h | 44 - drivers/kvm/paging_tmpl.h | 461 ------ drivers/kvm/segment_descriptor.h | 29 - drivers/kvm/svm.c | 1725 -------------------- drivers/kvm/svm.h | 325 ---- drivers/kvm/types.h | 54 - drivers/kvm/vmx.c | 2673 ------------------------------- drivers/kvm/vmx.h | 324 ---- drivers/kvm/x86.c | 3148 ------------------------------------- drivers/kvm/x86.h | 602 ------- drivers/kvm/x86_emulate.c | 1913 ---------------------- drivers/kvm/x86_emulate.h | 186 --- include/asm-x86/kvm_host.h | 601 +++++++ include/asm-x86/kvm_x86_emulate.h | 186 +++ include/linux/kvm_host.h | 289 ++++ include/linux/kvm_types.h | 54 + 50 files changed, 15923 insertions(+), 15929 deletions(-) create mode 100644 arch/x86/kvm/Kconfig create mode 100644 arch/x86/kvm/Makefile create mode 100644 arch/x86/kvm/i8259.c create mode 100644 arch/x86/kvm/ioapic.c create mode 100644 arch/x86/kvm/irq.c create mode 100644 arch/x86/kvm/irq.h create mode 100644 arch/x86/kvm/kvm_svm.h create mode 100644 arch/x86/kvm/lapic.c create mode 100644 arch/x86/kvm/mmu.c create mode 100644 arch/x86/kvm/mmu.h create mode 100644 arch/x86/kvm/paging_tmpl.h create mode 100644 arch/x86/kvm/segment_descriptor.h create mode 100644 arch/x86/kvm/svm.c create mode 100644 arch/x86/kvm/svm.h create mode 100644 arch/x86/kvm/vmx.c create mode 100644 arch/x86/kvm/vmx.h create mode 100644 arch/x86/kvm/x86.c create mode 100644 arch/x86/kvm/x86_emulate.c delete mode 100644 drivers/kvm/Kconfig delete mode 100644 drivers/kvm/Makefile delete mode 100644 drivers/kvm/i8259.c delete mode 100644 drivers/kvm/ioapic.c delete mode 100644 drivers/kvm/irq.c delete mode 100644 drivers/kvm/irq.h delete mode 100644 drivers/kvm/kvm.h delete mode 100644 drivers/kvm/kvm_svm.h delete mode 100644 drivers/kvm/lapic.c delete mode 100644 drivers/kvm/mmu.c delete mode 100644 drivers/kvm/mmu.h delete mode 100644 drivers/kvm/paging_tmpl.h delete mode 100644 drivers/kvm/segment_descriptor.h delete mode 100644 drivers/kvm/svm.c delete mode 100644 drivers/kvm/svm.h delete mode 100644 drivers/kvm/types.h delete mode 100644 drivers/kvm/vmx.c delete mode 100644 drivers/kvm/vmx.h delete mode 100644 drivers/kvm/x86.c delete mode 100644 drivers/kvm/x86.h delete mode 100644 drivers/kvm/x86_emulate.c delete mode 100644 drivers/kvm/x86_emulate.h create mode 100644 include/asm-x86/kvm_host.h create mode 100644 include/asm-x86/kvm_x86_emulate.h create mode 100644 include/linux/kvm_host.h create mode 100644 include/linux/kvm_types.h (limited to 'drivers') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index d289cfcf92c4..65b449134cf7 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1599,4 +1599,6 @@ source "security/Kconfig" source "crypto/Kconfig" +source "arch/x86/kvm/Kconfig" + source "lib/Kconfig" diff --git a/arch/x86/Makefile b/arch/x86/Makefile index b08f18261df6..da8f4129780b 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -7,6 +7,8 @@ else KBUILD_DEFCONFIG := $(ARCH)_defconfig endif +core-$(CONFIG_KVM) += arch/x86/kvm/ + # BITS is used as extension for files which are available in a 32 bit # and a 64 bit version to simplify shared Makefiles. # e.g.: obj-y += foo_$(BITS).o diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig new file mode 100644 index 000000000000..c83e1c9b5129 --- /dev/null +++ b/arch/x86/kvm/Kconfig @@ -0,0 +1,57 @@ +# +# KVM configuration +# +config HAVE_KVM + bool + +menuconfig VIRTUALIZATION + bool "Virtualization" + depends on HAVE_KVM || X86 + default y + ---help--- + Say Y here to get to see options for using your Linux host to run other + operating systems inside virtual machines (guests). + This option alone does not add any kernel code. + + If you say N, all options in this submenu will be skipped and disabled. + +if VIRTUALIZATION + +config KVM + tristate "Kernel-based Virtual Machine (KVM) support" + depends on HAVE_KVM && EXPERIMENTAL + select PREEMPT_NOTIFIERS + select ANON_INODES + ---help--- + Support hosting fully virtualized guest machines using hardware + virtualization extensions. You will need a fairly recent + processor equipped with virtualization extensions. You will also + need to select one or more of the processor modules below. + + This module provides access to the hardware capabilities through + a character device node named /dev/kvm. + + To compile this as a module, choose M here: the module + will be called kvm. + + If unsure, say N. + +config KVM_INTEL + tristate "KVM for Intel processors support" + depends on KVM + ---help--- + Provides support for KVM on Intel processors equipped with the VT + extensions. + +config KVM_AMD + tristate "KVM for AMD processors support" + depends on KVM + ---help--- + Provides support for KVM on AMD processors equipped with the AMD-V + (SVM) extensions. + +# OK, it's a little counter-intuitive to do this, but it puts it neatly under +# the virtualization menu. +source drivers/lguest/Kconfig + +endif # VIRTUALIZATION diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile new file mode 100644 index 000000000000..880ffe403b35 --- /dev/null +++ b/arch/x86/kvm/Makefile @@ -0,0 +1,15 @@ +# +# Makefile for Kernel-based Virtual Machine module +# + +common-objs = $(addprefix ../../../drivers/kvm/, kvm_main.o) + +EXTRA_CFLAGS += -I drivers/kvm + +kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \ + ioapic.o +obj-$(CONFIG_KVM) += kvm.o +kvm-intel-objs = vmx.o +obj-$(CONFIG_KVM_INTEL) += kvm-intel.o +kvm-amd-objs = svm.o +obj-$(CONFIG_KVM_AMD) += kvm-amd.o diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c new file mode 100644 index 000000000000..ab29cf2def47 --- /dev/null +++ b/arch/x86/kvm/i8259.c @@ -0,0 +1,450 @@ +/* + * 8259 interrupt controller emulation + * + * Copyright (c) 2003-2004 Fabrice Bellard + * Copyright (c) 2007 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * Authors: + * Yaozu (Eddie) Dong + * Port from Qemu. + */ +#include +#include "irq.h" + +#include + +/* + * set irq level. If an edge is detected, then the IRR is set to 1 + */ +static inline void pic_set_irq1(struct kvm_kpic_state *s, int irq, int level) +{ + int mask; + mask = 1 << irq; + if (s->elcr & mask) /* level triggered */ + if (level) { + s->irr |= mask; + s->last_irr |= mask; + } else { + s->irr &= ~mask; + s->last_irr &= ~mask; + } + else /* edge triggered */ + if (level) { + if ((s->last_irr & mask) == 0) + s->irr |= mask; + s->last_irr |= mask; + } else + s->last_irr &= ~mask; +} + +/* + * return the highest priority found in mask (highest = smallest + * number). Return 8 if no irq + */ +static inline int get_priority(struct kvm_kpic_state *s, int mask) +{ + int priority; + if (mask == 0) + return 8; + priority = 0; + while ((mask & (1 << ((priority + s->priority_add) & 7))) == 0) + priority++; + return priority; +} + +/* + * return the pic wanted interrupt. return -1 if none + */ +static int pic_get_irq(struct kvm_kpic_state *s) +{ + int mask, cur_priority, priority; + + mask = s->irr & ~s->imr; + priority = get_priority(s, mask); + if (priority == 8) + return -1; + /* + * compute current priority. If special fully nested mode on the + * master, the IRQ coming from the slave is not taken into account + * for the priority computation. + */ + mask = s->isr; + if (s->special_fully_nested_mode && s == &s->pics_state->pics[0]) + mask &= ~(1 << 2); + cur_priority = get_priority(s, mask); + if (priority < cur_priority) + /* + * higher priority found: an irq should be generated + */ + return (priority + s->priority_add) & 7; + else + return -1; +} + +/* + * raise irq to CPU if necessary. must be called every time the active + * irq may change + */ +static void pic_update_irq(struct kvm_pic *s) +{ + int irq2, irq; + + irq2 = pic_get_irq(&s->pics[1]); + if (irq2 >= 0) { + /* + * if irq request by slave pic, signal master PIC + */ + pic_set_irq1(&s->pics[0], 2, 1); + pic_set_irq1(&s->pics[0], 2, 0); + } + irq = pic_get_irq(&s->pics[0]); + if (irq >= 0) + s->irq_request(s->irq_request_opaque, 1); + else + s->irq_request(s->irq_request_opaque, 0); +} + +void kvm_pic_update_irq(struct kvm_pic *s) +{ + pic_update_irq(s); +} + +void kvm_pic_set_irq(void *opaque, int irq, int level) +{ + struct kvm_pic *s = opaque; + + pic_set_irq1(&s->pics[irq >> 3], irq & 7, level); + pic_update_irq(s); +} + +/* + * acknowledge interrupt 'irq' + */ +static inline void pic_intack(struct kvm_kpic_state *s, int irq) +{ + if (s->auto_eoi) { + if (s->rotate_on_auto_eoi) + s->priority_add = (irq + 1) & 7; + } else + s->isr |= (1 << irq); + /* + * We don't clear a level sensitive interrupt here + */ + if (!(s->elcr & (1 << irq))) + s->irr &= ~(1 << irq); +} + +int kvm_pic_read_irq(struct kvm_pic *s) +{ + int irq, irq2, intno; + + irq = pic_get_irq(&s->pics[0]); + if (irq >= 0) { + pic_intack(&s->pics[0], irq); + if (irq == 2) { + irq2 = pic_get_irq(&s->pics[1]); + if (irq2 >= 0) + pic_intack(&s->pics[1], irq2); + else + /* + * spurious IRQ on slave controller + */ + irq2 = 7; + intno = s->pics[1].irq_base + irq2; + irq = irq2 + 8; + } else + intno = s->pics[0].irq_base + irq; + } else { + /* + * spurious IRQ on host controller + */ + irq = 7; + intno = s->pics[0].irq_base + irq; + } + pic_update_irq(s); + + return intno; +} + +void kvm_pic_reset(struct kvm_kpic_state *s) +{ + s->last_irr = 0; + s->irr = 0; + s->imr = 0; + s->isr = 0; + s->priority_add = 0; + s->irq_base = 0; + s->read_reg_select = 0; + s->poll = 0; + s->special_mask = 0; + s->init_state = 0; + s->auto_eoi = 0; + s->rotate_on_auto_eoi = 0; + s->special_fully_nested_mode = 0; + s->init4 = 0; +} + +static void pic_ioport_write(void *opaque, u32 addr, u32 val) +{ + struct kvm_kpic_state *s = opaque; + int priority, cmd, irq; + + addr &= 1; + if (addr == 0) { + if (val & 0x10) { + kvm_pic_reset(s); /* init */ + /* + * deassert a pending interrupt + */ + s->pics_state->irq_request(s->pics_state-> + irq_request_opaque, 0); + s->init_state = 1; + s->init4 = val & 1; + if (val & 0x02) + printk(KERN_ERR "single mode not supported"); + if (val & 0x08) + printk(KERN_ERR + "level sensitive irq not supported"); + } else if (val & 0x08) { + if (val & 0x04) + s->poll = 1; + if (val & 0x02) + s->read_reg_select = val & 1; + if (val & 0x40) + s->special_mask = (val >> 5) & 1; + } else { + cmd = val >> 5; + switch (cmd) { + case 0: + case 4: + s->rotate_on_auto_eoi = cmd >> 2; + break; + case 1: /* end of interrupt */ + case 5: + priority = get_priority(s, s->isr); + if (priority != 8) { + irq = (priority + s->priority_add) & 7; + s->isr &= ~(1 << irq); + if (cmd == 5) + s->priority_add = (irq + 1) & 7; + pic_update_irq(s->pics_state); + } + break; + case 3: + irq = val & 7; + s->isr &= ~(1 << irq); + pic_update_irq(s->pics_state); + break; + case 6: + s->priority_add = (val + 1) & 7; + pic_update_irq(s->pics_state); + break; + case 7: + irq = val & 7; + s->isr &= ~(1 << irq); + s->priority_add = (irq + 1) & 7; + pic_update_irq(s->pics_state); + break; + default: + break; /* no operation */ + } + } + } else + switch (s->init_state) { + case 0: /* normal mode */ + s->imr = val; + pic_update_irq(s->pics_state); + break; + case 1: + s->irq_base = val & 0xf8; + s->init_state = 2; + break; + case 2: + if (s->init4) + s->init_state = 3; + else + s->init_state = 0; + break; + case 3: + s->special_fully_nested_mode = (val >> 4) & 1; + s->auto_eoi = (val >> 1) & 1; + s->init_state = 0; + break; + } +} + +static u32 pic_poll_read(struct kvm_kpic_state *s, u32 addr1) +{ + int ret; + + ret = pic_get_irq(s); + if (ret >= 0) { + if (addr1 >> 7) { + s->pics_state->pics[0].isr &= ~(1 << 2); + s->pics_state->pics[0].irr &= ~(1 << 2); + } + s->irr &= ~(1 << ret); + s->isr &= ~(1 << ret); + if (addr1 >> 7 || ret != 2) + pic_update_irq(s->pics_state); + } else { + ret = 0x07; + pic_update_irq(s->pics_state); + } + + return ret; +} + +static u32 pic_ioport_read(void *opaque, u32 addr1) +{ + struct kvm_kpic_state *s = opaque; + unsigned int addr; + int ret; + + addr = addr1; + addr &= 1; + if (s->poll) { + ret = pic_poll_read(s, addr1); + s->poll = 0; + } else + if (addr == 0) + if (s->read_reg_select) + ret = s->isr; + else + ret = s->irr; + else + ret = s->imr; + return ret; +} + +static void elcr_ioport_write(void *opaque, u32 addr, u32 val) +{ + struct kvm_kpic_state *s = opaque; + s->elcr = val & s->elcr_mask; +} + +static u32 elcr_ioport_read(void *opaque, u32 addr1) +{ + struct kvm_kpic_state *s = opaque; + return s->elcr; +} + +static int picdev_in_range(struct kvm_io_device *this, gpa_t addr) +{ + switch (addr) { + case 0x20: + case 0x21: + case 0xa0: + case 0xa1: + case 0x4d0: + case 0x4d1: + return 1; + default: + return 0; + } +} + +static void picdev_write(struct kvm_io_device *this, + gpa_t addr, int len, const void *val) +{ + struct kvm_pic *s = this->private; + unsigned char data = *(unsigned char *)val; + + if (len != 1) { + if (printk_ratelimit()) + printk(KERN_ERR "PIC: non byte write\n"); + return; + } + switch (addr) { + case 0x20: + case 0x21: + case 0xa0: + case 0xa1: + pic_ioport_write(&s->pics[addr >> 7], addr, data); + break; + case 0x4d0: + case 0x4d1: + elcr_ioport_write(&s->pics[addr & 1], addr, data); + break; + } +} + +static void picdev_read(struct kvm_io_device *this, + gpa_t addr, int len, void *val) +{ + struct kvm_pic *s = this->private; + unsigned char data = 0; + + if (len != 1) { + if (printk_ratelimit()) + printk(KERN_ERR "PIC: non byte read\n"); + return; + } + switch (addr) { + case 0x20: + case 0x21: + case 0xa0: + case 0xa1: + data = pic_ioport_read(&s->pics[addr >> 7], addr); + break; + case 0x4d0: + case 0x4d1: + data = elcr_ioport_read(&s->pics[addr & 1], addr); + break; + } + *(unsigned char *)val = data; +} + +/* + * callback when PIC0 irq status changed + */ +static void pic_irq_request(void *opaque, int level) +{ + struct kvm *kvm = opaque; + struct kvm_vcpu *vcpu = kvm->vcpus[0]; + + pic_irqchip(kvm)->output = level; + if (vcpu) + kvm_vcpu_kick(vcpu); +} + +struct kvm_pic *kvm_create_pic(struct kvm *kvm) +{ + struct kvm_pic *s; + s = kzalloc(sizeof(struct kvm_pic), GFP_KERNEL); + if (!s) + return NULL; + s->pics[0].elcr_mask = 0xf8; + s->pics[1].elcr_mask = 0xde; + s->irq_request = pic_irq_request; + s->irq_request_opaque = kvm; + s->pics[0].pics_state = s; + s->pics[1].pics_state = s; + + /* + * Initialize PIO device + */ + s->dev.read = picdev_read; + s->dev.write = picdev_write; + s->dev.in_range = picdev_in_range; + s->dev.private = s; + kvm_io_bus_register_dev(&kvm->pio_bus, &s->dev); + return s; +} diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c new file mode 100644 index 000000000000..72f12f75495d --- /dev/null +++ b/arch/x86/kvm/ioapic.c @@ -0,0 +1,400 @@ +/* + * Copyright (C) 2001 MandrakeSoft S.A. + * + * MandrakeSoft S.A. + * 43, rue d'Aboukir + * 75002 Paris - France + * http://www.linux-mandrake.com/ + * http://www.mandrakesoft.com/ + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Yunhong Jiang + * Yaozu (Eddie) Dong + * Based on Xen 3.1 code. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "irq.h" +#if 0 +#define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) +#else +#define ioapic_debug(fmt, arg...) +#endif +static void ioapic_deliver(struct kvm_ioapic *vioapic, int irq); + +static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic, + unsigned long addr, + unsigned long length) +{ + unsigned long result = 0; + + switch (ioapic->ioregsel) { + case IOAPIC_REG_VERSION: + result = ((((IOAPIC_NUM_PINS - 1) & 0xff) << 16) + | (IOAPIC_VERSION_ID & 0xff)); + break; + + case IOAPIC_REG_APIC_ID: + case IOAPIC_REG_ARB_ID: + result = ((ioapic->id & 0xf) << 24); + break; + + default: + { + u32 redir_index = (ioapic->ioregsel - 0x10) >> 1; + u64 redir_content; + + ASSERT(redir_index < IOAPIC_NUM_PINS); + + redir_content = ioapic->redirtbl[redir_index].bits; + result = (ioapic->ioregsel & 0x1) ? + (redir_content >> 32) & 0xffffffff : + redir_content & 0xffffffff; + break; + } + } + + return result; +} + +static void ioapic_service(struct kvm_ioapic *ioapic, unsigned int idx) +{ + union ioapic_redir_entry *pent; + + pent = &ioapic->redirtbl[idx]; + + if (!pent->fields.mask) { + ioapic_deliver(ioapic, idx); + if (pent->fields.trig_mode == IOAPIC_LEVEL_TRIG) + pent->fields.remote_irr = 1; + } + if (!pent->fields.trig_mode) + ioapic->irr &= ~(1 << idx); +} + +static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val) +{ + unsigned index; + + switch (ioapic->ioregsel) { + case IOAPIC_REG_VERSION: + /* Writes are ignored. */ + break; + + case IOAPIC_REG_APIC_ID: + ioapic->id = (val >> 24) & 0xf; + break; + + case IOAPIC_REG_ARB_ID: + break; + + default: + index = (ioapic->ioregsel - 0x10) >> 1; + + ioapic_debug("change redir index %x val %x\n", index, val); + if (index >= IOAPIC_NUM_PINS) + return; + if (ioapic->ioregsel & 1) { + ioapic->redirtbl[index].bits &= 0xffffffff; + ioapic->redirtbl[index].bits |= (u64) val << 32; + } else { + ioapic->redirtbl[index].bits &= ~0xffffffffULL; + ioapic->redirtbl[index].bits |= (u32) val; + ioapic->redirtbl[index].fields.remote_irr = 0; + } + if (ioapic->irr & (1 << index)) + ioapic_service(ioapic, index); + break; + } +} + +static void ioapic_inj_irq(struct kvm_ioapic *ioapic, + struct kvm_vcpu *vcpu, + u8 vector, u8 trig_mode, u8 delivery_mode) +{ + ioapic_debug("irq %d trig %d deliv %d\n", vector, trig_mode, + delivery_mode); + + ASSERT((delivery_mode == IOAPIC_FIXED) || + (delivery_mode == IOAPIC_LOWEST_PRIORITY)); + + kvm_apic_set_irq(vcpu, vector, trig_mode); +} + +static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, + u8 dest_mode) +{ + u32 mask = 0; + int i; + struct kvm *kvm = ioapic->kvm; + struct kvm_vcpu *vcpu; + + ioapic_debug("dest %d dest_mode %d\n", dest, dest_mode); + + if (dest_mode == 0) { /* Physical mode. */ + if (dest == 0xFF) { /* Broadcast. */ + for (i = 0; i < KVM_MAX_VCPUS; ++i) + if (kvm->vcpus[i] && kvm->vcpus[i]->arch.apic) + mask |= 1 << i; + return mask; + } + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + vcpu = kvm->vcpus[i]; + if (!vcpu) + continue; + if (kvm_apic_match_physical_addr(vcpu->arch.apic, dest)) { + if (vcpu->arch.apic) + mask = 1 << i; + break; + } + } + } else if (dest != 0) /* Logical mode, MDA non-zero. */ + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + vcpu = kvm->vcpus[i]; + if (!vcpu) + continue; + if (vcpu->arch.apic && + kvm_apic_match_logical_addr(vcpu->arch.apic, dest)) + mask |= 1 << vcpu->vcpu_id; + } + ioapic_debug("mask %x\n", mask); + return mask; +} + +static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) +{ + u8 dest = ioapic->redirtbl[irq].fields.dest_id; + u8 dest_mode = ioapic->redirtbl[irq].fields.dest_mode; + u8 delivery_mode = ioapic->redirtbl[irq].fields.delivery_mode; + u8 vector = ioapic->redirtbl[irq].fields.vector; + u8 trig_mode = ioapic->redirtbl[irq].fields.trig_mode; + u32 deliver_bitmask; + struct kvm_vcpu *vcpu; + int vcpu_id; + + ioapic_debug("dest=%x dest_mode=%x delivery_mode=%x " + "vector=%x trig_mode=%x\n", + dest, dest_mode, delivery_mode, vector, trig_mode); + + deliver_bitmask = ioapic_get_delivery_bitmask(ioapic, dest, dest_mode); + if (!deliver_bitmask) { + ioapic_debug("no target on destination\n"); + return; + } + + switch (delivery_mode) { + case IOAPIC_LOWEST_PRIORITY: + vcpu = kvm_get_lowest_prio_vcpu(ioapic->kvm, vector, + deliver_bitmask); + if (vcpu != NULL) + ioapic_inj_irq(ioapic, vcpu, vector, + trig_mode, delivery_mode); + else + ioapic_debug("null lowest prio vcpu: " + "mask=%x vector=%x delivery_mode=%x\n", + deliver_bitmask, vector, IOAPIC_LOWEST_PRIORITY); + break; + case IOAPIC_FIXED: + for (vcpu_id = 0; deliver_bitmask != 0; vcpu_id++) { + if (!(deliver_bitmask & (1 << vcpu_id))) + continue; + deliver_bitmask &= ~(1 << vcpu_id); + vcpu = ioapic->kvm->vcpus[vcpu_id]; + if (vcpu) { + ioapic_inj_irq(ioapic, vcpu, vector, + trig_mode, delivery_mode); + } + } + break; + + /* TODO: NMI */ + default: + printk(KERN_WARNING "Unsupported delivery mode %d\n", + delivery_mode); + break; + } +} + +void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level) +{ + u32 old_irr = ioapic->irr; + u32 mask = 1 << irq; + union ioapic_redir_entry entry; + + if (irq >= 0 && irq < IOAPIC_NUM_PINS) { + entry = ioapic->redirtbl[irq]; + level ^= entry.fields.polarity; + if (!level) + ioapic->irr &= ~mask; + else { + ioapic->irr |= mask; + if ((!entry.fields.trig_mode && old_irr != ioapic->irr) + || !entry.fields.remote_irr) + ioapic_service(ioapic, irq); + } + } +} + +static int get_eoi_gsi(struct kvm_ioapic *ioapic, int vector) +{ + int i; + + for (i = 0; i < IOAPIC_NUM_PINS; i++) + if (ioapic->redirtbl[i].fields.vector == vector) + return i; + return -1; +} + +void kvm_ioapic_update_eoi(struct kvm *kvm, int vector) +{ + struct kvm_ioapic *ioapic = kvm->arch.vioapic; + union ioapic_redir_entry *ent; + int gsi; + + gsi = get_eoi_gsi(ioapic, vector); + if (gsi == -1) { + printk(KERN_WARNING "Can't find redir item for %d EOI\n", + vector); + return; + } + + ent = &ioapic->redirtbl[gsi]; + ASSERT(ent->fields.trig_mode == IOAPIC_LEVEL_TRIG); + + ent->fields.remote_irr = 0; + if (!ent->fields.mask && (ioapic->irr & (1 << gsi))) + ioapic_deliver(ioapic, gsi); +} + +static int ioapic_in_range(struct kvm_io_device *this, gpa_t addr) +{ + struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; + + return ((addr >= ioapic->base_address && + (addr < ioapic->base_address + IOAPIC_MEM_LENGTH))); +} + +static void ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len, + void *val) +{ + struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; + u32 result; + + ioapic_debug("addr %lx\n", (unsigned long)addr); + ASSERT(!(addr & 0xf)); /* check alignment */ + + addr &= 0xff; + switch (addr) { + case IOAPIC_REG_SELECT: + result = ioapic->ioregsel; + break; + + case IOAPIC_REG_WINDOW: + result = ioapic_read_indirect(ioapic, addr, len); + break; + + default: + result = 0; + break; + } + switch (len) { + case 8: + *(u64 *) val = result; + break; + case 1: + case 2: + case 4: + memcpy(val, (char *)&result, len); + break; + default: + printk(KERN_WARNING "ioapic: wrong length %d\n", len); + } +} + +static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, + const void *val) +{ + struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; + u32 data; + + ioapic_debug("ioapic_mmio_write addr=%p len=%d val=%p\n", + (void*)addr, len, val); + ASSERT(!(addr & 0xf)); /* check alignment */ + if (len == 4 || len == 8) + data = *(u32 *) val; + else { + printk(KERN_WARNING "ioapic: Unsupported size %d\n", len); + return; + } + + addr &= 0xff; + switch (addr) { + case IOAPIC_REG_SELECT: + ioapic->ioregsel = data; + break; + + case IOAPIC_REG_WINDOW: + ioapic_write_indirect(ioapic, data); + break; +#ifdef CONFIG_IA64 + case IOAPIC_REG_EOI: + kvm_ioapic_update_eoi(ioapic, data); + break; +#endif + + default: + break; + } +} + +void kvm_ioapic_reset(struct kvm_ioapic *ioapic) +{ + int i; + + for (i = 0; i < IOAPIC_NUM_PINS; i++) + ioapic->redirtbl[i].fields.mask = 1; + ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS; + ioapic->ioregsel = 0; + ioapic->irr = 0; + ioapic->id = 0; +} + +int kvm_ioapic_init(struct kvm *kvm) +{ + struct kvm_ioapic *ioapic; + + ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL); + if (!ioapic) + return -ENOMEM; + kvm->arch.vioapic = ioapic; + kvm_ioapic_reset(ioapic); + ioapic->dev.read = ioapic_mmio_read; + ioapic->dev.write = ioapic_mmio_write; + ioapic->dev.in_range = ioapic_in_range; + ioapic->dev.private = ioapic; + ioapic->kvm = kvm; + kvm_io_bus_register_dev(&kvm->mmio_bus, &ioapic->dev); + return 0; +} diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c new file mode 100644 index 000000000000..07a09aad4fd6 --- /dev/null +++ b/arch/x86/kvm/irq.c @@ -0,0 +1,98 @@ +/* + * irq.c: API for in kernel interrupt controller + * Copyright (c) 2007, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + * Authors: + * Yaozu (Eddie) Dong + * + */ + +#include +#include + +#include "irq.h" + +/* + * check if there is pending interrupt without + * intack. + */ +int kvm_cpu_has_interrupt(struct kvm_vcpu *v) +{ + struct kvm_pic *s; + + if (kvm_apic_has_interrupt(v) == -1) { /* LAPIC */ + if (kvm_apic_accept_pic_intr(v)) { + s = pic_irqchip(v->kvm); /* PIC */ + return s->output; + } else + return 0; + } + return 1; +} +EXPORT_SYMBOL_GPL(kvm_cpu_has_interrupt); + +/* + * Read pending interrupt vector and intack. + */ +int kvm_cpu_get_interrupt(struct kvm_vcpu *v) +{ + struct kvm_pic *s; + int vector; + + vector = kvm_get_apic_interrupt(v); /* APIC */ + if (vector == -1) { + if (kvm_apic_accept_pic_intr(v)) { + s = pic_irqchip(v->kvm); + s->output = 0; /* PIC */ + vector = kvm_pic_read_irq(s); + } + } + return vector; +} +EXPORT_SYMBOL_GPL(kvm_cpu_get_interrupt); + +static void vcpu_kick_intr(void *info) +{ +#ifdef DEBUG + struct kvm_vcpu *vcpu = (struct kvm_vcpu *)info; + printk(KERN_DEBUG "vcpu_kick_intr %p \n", vcpu); +#endif +} + +void kvm_vcpu_kick(struct kvm_vcpu *vcpu) +{ + int ipi_pcpu = vcpu->cpu; + + if (waitqueue_active(&vcpu->wq)) { + wake_up_interruptible(&vcpu->wq); + ++vcpu->stat.halt_wakeup; + } + if (vcpu->guest_mode) + smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0, 0); +} + +void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu) +{ + kvm_inject_apic_timer_irqs(vcpu); + /* TODO: PIT, RTC etc. */ +} +EXPORT_SYMBOL_GPL(kvm_inject_pending_timer_irqs); + +void kvm_timer_intr_post(struct kvm_vcpu *vcpu, int vec) +{ + kvm_apic_timer_intr_post(vcpu, vec); + /* TODO: PIT, RTC etc. */ +} +EXPORT_SYMBOL_GPL(kvm_timer_intr_post); diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h new file mode 100644 index 000000000000..6316638eec9f --- /dev/null +++ b/arch/x86/kvm/irq.h @@ -0,0 +1,195 @@ +/* + * irq.h: in kernel interrupt controller related definitions + * Copyright (c) 2007, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + * Authors: + * Yaozu (Eddie) Dong + * + */ + +#ifndef __IRQ_H +#define __IRQ_H + +#include +#include +#include +#include "iodev.h" + +struct kvm; +struct kvm_vcpu; + +typedef void irq_request_func(void *opaque, int level); + +struct kvm_kpic_state { + u8 last_irr; /* edge detection */ + u8 irr; /* interrupt request register */ + u8 imr; /* interrupt mask register */ + u8 isr; /* interrupt service register */ + u8 priority_add; /* highest irq priority */ + u8 irq_base; + u8 read_reg_select; + u8 poll; + u8 special_mask; + u8 init_state; + u8 auto_eoi; + u8 rotate_on_auto_eoi; + u8 special_fully_nested_mode; + u8 init4; /* true if 4 byte init */ + u8 elcr; /* PIIX edge/trigger selection */ + u8 elcr_mask; + struct kvm_pic *pics_state; +}; + +struct kvm_pic { + struct kvm_kpic_state pics[2]; /* 0 is master pic, 1 is slave pic */ + irq_request_func *irq_request; + void *irq_request_opaque; + int output; /* intr from master PIC */ + struct kvm_io_device dev; +}; + +struct kvm_pic *kvm_create_pic(struct kvm *kvm); +void kvm_pic_set_irq(void *opaque, int irq, int level); +int kvm_pic_read_irq(struct kvm_pic *s); +void kvm_pic_update_irq(struct kvm_pic *s); + +#define IOAPIC_NUM_PINS KVM_IOAPIC_NUM_PINS +#define IOAPIC_VERSION_ID 0x11 /* IOAPIC version */ +#define IOAPIC_EDGE_TRIG 0 +#define IOAPIC_LEVEL_TRIG 1 + +#define IOAPIC_DEFAULT_BASE_ADDRESS 0xfec00000 +#define IOAPIC_MEM_LENGTH 0x100 + +/* Direct registers. */ +#define IOAPIC_REG_SELECT 0x00 +#define IOAPIC_REG_WINDOW 0x10 +#define IOAPIC_REG_EOI 0x40 /* IA64 IOSAPIC only */ + +/* Indirect registers. */ +#define IOAPIC_REG_APIC_ID 0x00 /* x86 IOAPIC only */ +#define IOAPIC_REG_VERSION 0x01 +#define IOAPIC_REG_ARB_ID 0x02 /* x86 IOAPIC only */ + +/*ioapic delivery mode*/ +#define IOAPIC_FIXED 0x0 +#define IOAPIC_LOWEST_PRIORITY 0x1 +#define IOAPIC_PMI 0x2 +#define IOAPIC_NMI 0x4 +#define IOAPIC_INIT 0x5 +#define IOAPIC_EXTINT 0x7 + +struct kvm_ioapic { + u64 base_address; + u32 ioregsel; + u32 id; + u32 irr; + u32 pad; + union ioapic_redir_entry { + u64 bits; + struct { + u8 vector; + u8 delivery_mode:3; + u8 dest_mode:1; + u8 delivery_status:1; + u8 polarity:1; + u8 remote_irr:1; + u8 trig_mode:1; + u8 mask:1; + u8 reserve:7; + u8 reserved[4]; + u8 dest_id; + } fields; + } redirtbl[IOAPIC_NUM_PINS]; + struct kvm_io_device dev; + struct kvm *kvm; +}; + +struct kvm_lapic { + unsigned long base_address; + struct kvm_io_device dev; + struct { + atomic_t pending; + s64 period; /* unit: ns */ + u32 divide_count; + ktime_t last_update; + struct hrtimer dev; + } timer; + struct kvm_vcpu *vcpu; + struct page *regs_page; + void *regs; +}; + +#ifdef DEBUG +#define ASSERT(x) \ +do { \ + if (!(x)) { \ + printk(KERN_EMERG "assertion failed %s: %d: %s\n", \ + __FILE__, __LINE__, #x); \ + BUG(); \ + } \ +} while (0) +#else +#define ASSERT(x) do { } while (0) +#endif + +static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) +{ + return kvm->arch.vpic; +} + +static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) +{ + return kvm->arch.vioapic; +} + +static inline int irqchip_in_kernel(struct kvm *kvm) +{ + return pic_irqchip(kvm) != NULL; +} + +void kvm_vcpu_kick(struct kvm_vcpu *vcpu); +int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); +int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); +int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); +int kvm_create_lapic(struct kvm_vcpu *vcpu); +void kvm_lapic_reset(struct kvm_vcpu *vcpu); +void kvm_pic_reset(struct kvm_kpic_state *s); +void kvm_ioapic_reset(struct kvm_ioapic *ioapic); +void kvm_free_lapic(struct kvm_vcpu *vcpu); +u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); +void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); +void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); + +struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, + unsigned long bitmap); +u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); +void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); +int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest); +void kvm_ioapic_update_eoi(struct kvm *kvm, int vector); +int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda); +int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig); +void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu); +int kvm_ioapic_init(struct kvm *kvm); +void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level); +int kvm_lapic_enabled(struct kvm_vcpu *vcpu); +int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); +void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec); +void kvm_timer_intr_post(struct kvm_vcpu *vcpu, int vec); +void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); +void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); +void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu); + +#endif diff --git a/arch/x86/kvm/kvm_svm.h b/arch/x86/kvm/kvm_svm.h new file mode 100644 index 000000000000..ecdfe97e4635 --- /dev/null +++ b/arch/x86/kvm/kvm_svm.h @@ -0,0 +1,45 @@ +#ifndef __KVM_SVM_H +#define __KVM_SVM_H + +#include +#include +#include +#include +#include + +#include "svm.h" + +static const u32 host_save_user_msrs[] = { +#ifdef CONFIG_X86_64 + MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE, + MSR_FS_BASE, +#endif + MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, +}; + +#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs) +#define NUM_DB_REGS 4 + +struct kvm_vcpu; + +struct vcpu_svm { + struct kvm_vcpu vcpu; + struct vmcb *vmcb; + unsigned long vmcb_pa; + struct svm_cpu_data *svm_data; + uint64_t asid_generation; + + unsigned long db_regs[NUM_DB_REGS]; + + u64 next_rip; + + u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; + u64 host_gs_base; + unsigned long host_cr2; + unsigned long host_db_regs[NUM_DB_REGS]; + unsigned long host_dr6; + unsigned long host_dr7; +}; + +#endif + diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c new file mode 100644 index 000000000000..4076331b01ee --- /dev/null +++ b/arch/x86/kvm/lapic.c @@ -0,0 +1,1085 @@ + +/* + * Local APIC virtualization + * + * Copyright (C) 2006 Qumranet, Inc. + * Copyright (C) 2007 Novell + * Copyright (C) 2007 Intel + * + * Authors: + * Dor Laor + * Gregory Haskins + * Yaozu (Eddie) Dong + * + * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "irq.h" + +#define PRId64 "d" +#define PRIx64 "llx" +#define PRIu64 "u" +#define PRIo64 "o" + +#define APIC_BUS_CYCLE_NS 1 + +/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ +#define apic_debug(fmt, arg...) + +#define APIC_LVT_NUM 6 +/* 14 is the version for Xeon and Pentium 8.4.8*/ +#define APIC_VERSION (0x14UL | ((APIC_LVT_NUM - 1) << 16)) +#define LAPIC_MMIO_LENGTH (1 << 12) +/* followed define is not in apicdef.h */ +#define APIC_SHORT_MASK 0xc0000 +#define APIC_DEST_NOSHORT 0x0 +#define APIC_DEST_MASK 0x800 +#define MAX_APIC_VECTOR 256 + +#define VEC_POS(v) ((v) & (32 - 1)) +#define REG_POS(v) (((v) >> 5) << 4) + +static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off) +{ + return *((u32 *) (apic->regs + reg_off)); +} + +static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) +{ + *((u32 *) (apic->regs + reg_off)) = val; +} + +static inline int apic_test_and_set_vector(int vec, void *bitmap) +{ + return test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); +} + +static inline int apic_test_and_clear_vector(int vec, void *bitmap) +{ + return test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); +} + +static inline void apic_set_vector(int vec, void *bitmap) +{ + set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); +} + +static inline void apic_clear_vector(int vec, void *bitmap) +{ + clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); +} + +static inline int apic_hw_enabled(struct kvm_lapic *apic) +{ + return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE; +} + +static inline int apic_sw_enabled(struct kvm_lapic *apic) +{ + return apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED; +} + +static inline int apic_enabled(struct kvm_lapic *apic) +{ + return apic_sw_enabled(apic) && apic_hw_enabled(apic); +} + +#define LVT_MASK \ + (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) + +#define LINT_MASK \ + (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ + APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) + +static inline int kvm_apic_id(struct kvm_lapic *apic) +{ + return (apic_get_reg(apic, APIC_ID) >> 24) & 0xff; +} + +static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) +{ + return !(apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); +} + +static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) +{ + return apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; +} + +static inline int apic_lvtt_period(struct kvm_lapic *apic) +{ + return apic_get_reg(apic, APIC_LVTT) & APIC_LVT_TIMER_PERIODIC; +} + +static unsigned int apic_lvt_mask[APIC_LVT_NUM] = { + LVT_MASK | APIC_LVT_TIMER_PERIODIC, /* LVTT */ + LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ + LVT_MASK | APIC_MODE_MASK, /* LVTPC */ + LINT_MASK, LINT_MASK, /* LVT0-1 */ + LVT_MASK /* LVTERR */ +}; + +static int find_highest_vector(void *bitmap) +{ + u32 *word = bitmap; + int word_offset = MAX_APIC_VECTOR >> 5; + + while ((word_offset != 0) && (word[(--word_offset) << 2] == 0)) + continue; + + if (likely(!word_offset && !word[0])) + return -1; + else + return fls(word[word_offset << 2]) - 1 + (word_offset << 5); +} + +static inline int apic_test_and_set_irr(int vec, struct kvm_lapic *apic) +{ + return apic_test_and_set_vector(vec, apic->regs + APIC_IRR); +} + +static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) +{ + apic_clear_vector(vec, apic->regs + APIC_IRR); +} + +static inline int apic_find_highest_irr(struct kvm_lapic *apic) +{ + int result; + + result = find_highest_vector(apic->regs + APIC_IRR); + ASSERT(result == -1 || result >= 16); + + return result; +} + +int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + int highest_irr; + + if (!apic) + return 0; + highest_irr = apic_find_highest_irr(apic); + + return highest_irr; +} +EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); + +int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + if (!apic_test_and_set_irr(vec, apic)) { + /* a new pending irq is set in IRR */ + if (trig) + apic_set_vector(vec, apic->regs + APIC_TMR); + else + apic_clear_vector(vec, apic->regs + APIC_TMR); + kvm_vcpu_kick(apic->vcpu); + return 1; + } + return 0; +} + +static inline int apic_find_highest_isr(struct kvm_lapic *apic) +{ + int result; + + result = find_highest_vector(apic->regs + APIC_ISR); + ASSERT(result == -1 || result >= 16); + + return result; +} + +static void apic_update_ppr(struct kvm_lapic *apic) +{ + u32 tpr, isrv, ppr; + int isr; + + tpr = apic_get_reg(apic, APIC_TASKPRI); + isr = apic_find_highest_isr(apic); + isrv = (isr != -1) ? isr : 0; + + if ((tpr & 0xf0) >= (isrv & 0xf0)) + ppr = tpr & 0xff; + else + ppr = isrv & 0xf0; + + apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x", + apic, ppr, isr, isrv); + + apic_set_reg(apic, APIC_PROCPRI, ppr); +} + +static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) +{ + apic_set_reg(apic, APIC_TASKPRI, tpr); + apic_update_ppr(apic); +} + +int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest) +{ + return kvm_apic_id(apic) == dest; +} + +int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda) +{ + int result = 0; + u8 logical_id; + + logical_id = GET_APIC_LOGICAL_ID(apic_get_reg(apic, APIC_LDR)); + + switch (apic_get_reg(apic, APIC_DFR)) { + case APIC_DFR_FLAT: + if (logical_id & mda) + result = 1; + break; + case APIC_DFR_CLUSTER: + if (((logical_id >> 4) == (mda >> 0x4)) + && (logical_id & mda & 0xf)) + result = 1; + break; + default: + printk(KERN_WARNING "Bad DFR vcpu %d: %08x\n", + apic->vcpu->vcpu_id, apic_get_reg(apic, APIC_DFR)); + break; + } + + return result; +} + +static int apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, + int short_hand, int dest, int dest_mode) +{ + int result = 0; + struct kvm_lapic *target = vcpu->arch.apic; + + apic_debug("target %p, source %p, dest 0x%x, " + "dest_mode 0x%x, short_hand 0x%x", + target, source, dest, dest_mode, short_hand); + + ASSERT(!target); + switch (short_hand) { + case APIC_DEST_NOSHORT: + if (dest_mode == 0) { + /* Physical mode. */ + if ((dest == 0xFF) || (dest == kvm_apic_id(target))) + result = 1; + } else + /* Logical mode. */ + result = kvm_apic_match_logical_addr(target, dest); + break; + case APIC_DEST_SELF: + if (target == source) + result = 1; + break; + case APIC_DEST_ALLINC: + result = 1; + break; + case APIC_DEST_ALLBUT: + if (target != source) + result = 1; + break; + default: + printk(KERN_WARNING "Bad dest shorthand value %x\n", + short_hand); + break; + } + + return result; +} + +/* + * Add a pending IRQ into lapic. + * Return 1 if successfully added and 0 if discarded. + */ +static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, + int vector, int level, int trig_mode) +{ + int orig_irr, result = 0; + struct kvm_vcpu *vcpu = apic->vcpu; + + switch (delivery_mode) { + case APIC_DM_FIXED: + case APIC_DM_LOWEST: + /* FIXME add logic for vcpu on reset */ + if (unlikely(!apic_enabled(apic))) + break; + + orig_irr = apic_test_and_set_irr(vector, apic); + if (orig_irr && trig_mode) { + apic_debug("level trig mode repeatedly for vector %d", + vector); + break; + } + + if (trig_mode) { + apic_debug("level trig mode for vector %d", vector); + apic_set_vector(vector, apic->regs + APIC_TMR); + } else + apic_clear_vector(vector, apic->regs + APIC_TMR); + + if (vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE) + kvm_vcpu_kick(vcpu); + else if (vcpu->arch.mp_state == VCPU_MP_STATE_HALTED) { + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; + if (waitqueue_active(&vcpu->wq)) + wake_up_interruptible(&vcpu->wq); + } + + result = (orig_irr == 0); + break; + + case APIC_DM_REMRD: + printk(KERN_DEBUG "Ignoring delivery mode 3\n"); + break; + + case APIC_DM_SMI: + printk(KERN_DEBUG "Ignoring guest SMI\n"); + break; + case APIC_DM_NMI: + printk(KERN_DEBUG "Ignoring guest NMI\n"); + break; + + case APIC_DM_INIT: + if (level) { + if (vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE) + printk(KERN_DEBUG + "INIT on a runnable vcpu %d\n", + vcpu->vcpu_id); + vcpu->arch.mp_state = VCPU_MP_STATE_INIT_RECEIVED; + kvm_vcpu_kick(vcpu); + } else { + printk(KERN_DEBUG + "Ignoring de-assert INIT to vcpu %d\n", + vcpu->vcpu_id); + } + + break; + + case APIC_DM_STARTUP: + printk(KERN_DEBUG "SIPI to vcpu %d vector 0x%02x\n", + vcpu->vcpu_id, vector); + if (vcpu->arch.mp_state == VCPU_MP_STATE_INIT_RECEIVED) { + vcpu->arch.sipi_vector = vector; + vcpu->arch.mp_state = VCPU_MP_STATE_SIPI_RECEIVED; + if (waitqueue_active(&vcpu->wq)) + wake_up_interruptible(&vcpu->wq); + } + break; + + default: + printk(KERN_ERR "TODO: unsupported delivery mode %x\n", + delivery_mode); + break; + } + return result; +} + +static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, + unsigned long bitmap) +{ + int last; + int next; + struct kvm_lapic *apic = NULL; + + last = kvm->arch.round_robin_prev_vcpu; + next = last; + + do { + if (++next == KVM_MAX_VCPUS) + next = 0; + if (kvm->vcpus[next] == NULL || !test_bit(next, &bitmap)) + continue; + apic = kvm->vcpus[next]->arch.apic; + if (apic && apic_enabled(apic)) + break; + apic = NULL; + } while (next != last); + kvm->arch.round_robin_prev_vcpu = next; + + if (!apic) + printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n"); + + return apic; +} + +struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, + unsigned long bitmap) +{ + struct kvm_lapic *apic; + + apic = kvm_apic_round_robin(kvm, vector, bitmap); + if (apic) + return apic->vcpu; + return NULL; +} + +static void apic_set_eoi(struct kvm_lapic *apic) +{ + int vector = apic_find_highest_isr(apic); + + /* + * Not every write EOI will has corresponding ISR, + * one example is when Kernel check timer on setup_IO_APIC + */ + if (vector == -1) + return; + + apic_clear_vector(vector, apic->regs + APIC_ISR); + apic_update_ppr(apic); + + if (apic_test_and_clear_vector(vector, apic->regs + APIC_TMR)) + kvm_ioapic_update_eoi(apic->vcpu->kvm, vector); +} + +static void apic_send_ipi(struct kvm_lapic *apic) +{ + u32 icr_low = apic_get_reg(apic, APIC_ICR); + u32 icr_high = apic_get_reg(apic, APIC_ICR2); + + unsigned int dest = GET_APIC_DEST_FIELD(icr_high); + unsigned int short_hand = icr_low & APIC_SHORT_MASK; + unsigned int trig_mode = icr_low & APIC_INT_LEVELTRIG; + unsigned int level = icr_low & APIC_INT_ASSERT; + unsigned int dest_mode = icr_low & APIC_DEST_MASK; + unsigned int delivery_mode = icr_low & APIC_MODE_MASK; + unsigned int vector = icr_low & APIC_VECTOR_MASK; + + struct kvm_vcpu *target; + struct kvm_vcpu *vcpu; + unsigned long lpr_map = 0; + int i; + + apic_debug("icr_high 0x%x, icr_low 0x%x, " + "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " + "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n", + icr_high, icr_low, short_hand, dest, + trig_mode, level, dest_mode, delivery_mode, vector); + + for (i = 0; i < KVM_MAX_VCPUS; i++) { + vcpu = apic->vcpu->kvm->vcpus[i]; + if (!vcpu) + continue; + + if (vcpu->arch.apic && + apic_match_dest(vcpu, apic, short_hand, dest, dest_mode)) { + if (delivery_mode == APIC_DM_LOWEST) + set_bit(vcpu->vcpu_id, &lpr_map); + else + __apic_accept_irq(vcpu->arch.apic, delivery_mode, + vector, level, trig_mode); + } + } + + if (delivery_mode == APIC_DM_LOWEST) { + target = kvm_get_lowest_prio_vcpu(vcpu->kvm, vector, lpr_map); + if (target != NULL) + __apic_accept_irq(target->arch.apic, delivery_mode, + vector, level, trig_mode); + } +} + +static u32 apic_get_tmcct(struct kvm_lapic *apic) +{ + u64 counter_passed; + ktime_t passed, now; + u32 tmcct; + + ASSERT(apic != NULL); + + now = apic->timer.dev.base->get_time(); + tmcct = apic_get_reg(apic, APIC_TMICT); + + /* if initial count is 0, current count should also be 0 */ + if (tmcct == 0) + return 0; + + if (unlikely(ktime_to_ns(now) <= + ktime_to_ns(apic->timer.last_update))) { + /* Wrap around */ + passed = ktime_add(( { + (ktime_t) { + .tv64 = KTIME_MAX - + (apic->timer.last_update).tv64}; } + ), now); + apic_debug("time elapsed\n"); + } else + passed = ktime_sub(now, apic->timer.last_update); + + counter_passed = div64_64(ktime_to_ns(passed), + (APIC_BUS_CYCLE_NS * apic->timer.divide_count)); + + if (counter_passed > tmcct) { + if (unlikely(!apic_lvtt_period(apic))) { + /* one-shot timers stick at 0 until reset */ + tmcct = 0; + } else { + /* + * periodic timers reset to APIC_TMICT when they + * hit 0. The while loop simulates this happening N + * times. (counter_passed %= tmcct) would also work, + * but might be slower or not work on 32-bit?? + */ + while (counter_passed > tmcct) + counter_passed -= tmcct; + tmcct -= counter_passed; + } + } else { + tmcct -= counter_passed; + } + + return tmcct; +} + +static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) +{ + u32 val = 0; + + if (offset >= LAPIC_MMIO_LENGTH) + return 0; + + switch (offset) { + case APIC_ARBPRI: + printk(KERN_WARNING "Access APIC ARBPRI register " + "which is for P6\n"); + break; + + case APIC_TMCCT: /* Timer CCR */ + val = apic_get_tmcct(apic); + break; + + default: + apic_update_ppr(apic); + val = apic_get_reg(apic, offset); + break; + } + + return val; +} + +static void apic_mmio_read(struct kvm_io_device *this, + gpa_t address, int len, void *data) +{ + struct kvm_lapic *apic = (struct kvm_lapic *)this->private; + unsigned int offset = address - apic->base_address; + unsigned char alignment = offset & 0xf; + u32 result; + + if ((alignment + len) > 4) { + printk(KERN_ERR "KVM_APIC_READ: alignment error %lx %d", + (unsigned long)address, len); + return; + } + result = __apic_read(apic, offset & ~0xf); + + switch (len) { + case 1: + case 2: + case 4: + memcpy(data, (char *)&result + alignment, len); + break; + default: + printk(KERN_ERR "Local APIC read with len = %x, " + "should be 1,2, or 4 instead\n", len); + break; + } +} + +static void update_divide_count(struct kvm_lapic *apic) +{ + u32 tmp1, tmp2, tdcr; + + tdcr = apic_get_reg(apic, APIC_TDCR); + tmp1 = tdcr & 0xf; + tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; + apic->timer.divide_count = 0x1 << (tmp2 & 0x7); + + apic_debug("timer divide count is 0x%x\n", + apic->timer.divide_count); +} + +static void start_apic_timer(struct kvm_lapic *apic) +{ + ktime_t now = apic->timer.dev.base->get_time(); + + apic->timer.last_update = now; + + apic->timer.period = apic_get_reg(apic, APIC_TMICT) * + APIC_BUS_CYCLE_NS * apic->timer.divide_count; + atomic_set(&apic->timer.pending, 0); + hrtimer_start(&apic->timer.dev, + ktime_add_ns(now, apic->timer.period), + HRTIMER_MODE_ABS); + + apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" + PRIx64 ", " + "timer initial count 0x%x, period %lldns, " + "expire @ 0x%016" PRIx64 ".\n", __FUNCTION__, + APIC_BUS_CYCLE_NS, ktime_to_ns(now), + apic_get_reg(apic, APIC_TMICT), + apic->timer.period, + ktime_to_ns(ktime_add_ns(now, + apic->timer.period))); +} + +static void apic_mmio_write(struct kvm_io_device *this, + gpa_t address, int len, const void *data) +{ + struct kvm_lapic *apic = (struct kvm_lapic *)this->private; + unsigned int offset = address - apic->base_address; + unsigned char alignment = offset & 0xf; + u32 val; + + /* + * APIC register must be aligned on 128-bits boundary. + * 32/64/128 bits registers must be accessed thru 32 bits. + * Refer SDM 8.4.1 + */ + if (len != 4 || alignment) { + if (printk_ratelimit()) + printk(KERN_ERR "apic write: bad size=%d %lx\n", + len, (long)address); + return; + } + + val = *(u32 *) data; + + /* too common printing */ + if (offset != APIC_EOI) + apic_debug("%s: offset 0x%x with length 0x%x, and value is " + "0x%x\n", __FUNCTION__, offset, len, val); + + offset &= 0xff0; + + switch (offset) { + case APIC_ID: /* Local APIC ID */ + apic_set_reg(apic, APIC_ID, val); + break; + + case APIC_TASKPRI: + apic_set_tpr(apic, val & 0xff); + break; + + case APIC_EOI: + apic_set_eoi(apic); + break; + + case APIC_LDR: + apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK); + break; + + case APIC_DFR: + apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); + break; + + case APIC_SPIV: + apic_set_reg(apic, APIC_SPIV, val & 0x3ff); + if (!(val & APIC_SPIV_APIC_ENABLED)) { + int i; + u32 lvt_val; + + for (i = 0; i < APIC_LVT_NUM; i++) { + lvt_val = apic_get_reg(apic, + APIC_LVTT + 0x10 * i); + apic_set_reg(apic, APIC_LVTT + 0x10 * i, + lvt_val | APIC_LVT_MASKED); + } + atomic_set(&apic->timer.pending, 0); + + } + break; + + case APIC_ICR: + /* No delay here, so we always clear the pending bit */ + apic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); + apic_send_ipi(apic); + break; + + case APIC_ICR2: + apic_set_reg(apic, APIC_ICR2, val & 0xff000000); + break; + + case APIC_LVTT: + case APIC_LVTTHMR: + case APIC_LVTPC: + case APIC_LVT0: + case APIC_LVT1: + case APIC_LVTERR: + /* TODO: Check vector */ + if (!apic_sw_enabled(apic)) + val |= APIC_LVT_MASKED; + + val &= apic_lvt_mask[(offset - APIC_LVTT) >> 4]; + apic_set_reg(apic, offset, val); + + break; + + case APIC_TMICT: + hrtimer_cancel(&apic->timer.dev); + apic_set_reg(apic, APIC_TMICT, val); + start_apic_timer(apic); + return; + + case APIC_TDCR: + if (val & 4) + printk(KERN_ERR "KVM_WRITE:TDCR %x\n", val); + apic_set_reg(apic, APIC_TDCR, val); + update_divide_count(apic); + break; + + default: + apic_debug("Local APIC Write to read-only register %x\n", + offset); + break; + } + +} + +static int apic_mmio_range(struct kvm_io_device *this, gpa_t addr) +{ + struct kvm_lapic *apic = (struct kvm_lapic *)this->private; + int ret = 0; + + + if (apic_hw_enabled(apic) && + (addr >= apic->base_address) && + (addr < (apic->base_address + LAPIC_MMIO_LENGTH))) + ret = 1; + + return ret; +} + +void kvm_free_lapic(struct kvm_vcpu *vcpu) +{ + if (!vcpu->arch.apic) + return; + + hrtimer_cancel(&vcpu->arch.apic->timer.dev); + + if (vcpu->arch.apic->regs_page) + __free_page(vcpu->arch.apic->regs_page); + + kfree(vcpu->arch.apic); +} + +/* + *---------------------------------------------------------------------- + * LAPIC interface + *---------------------------------------------------------------------- + */ + +void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + if (!apic) + return; + apic_set_tpr(apic, ((cr8 & 0x0f) << 4)); +} + +u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + u64 tpr; + + if (!apic) + return 0; + tpr = (u64) apic_get_reg(apic, APIC_TASKPRI); + + return (tpr & 0xf0) >> 4; +} +EXPORT_SYMBOL_GPL(kvm_lapic_get_cr8); + +void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + if (!apic) { + value |= MSR_IA32_APICBASE_BSP; + vcpu->arch.apic_base = value; + return; + } + if (apic->vcpu->vcpu_id) + value &= ~MSR_IA32_APICBASE_BSP; + + vcpu->arch.apic_base = value; + apic->base_address = apic->vcpu->arch.apic_base & + MSR_IA32_APICBASE_BASE; + + /* with FSB delivery interrupt, we can restart APIC functionality */ + apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " + "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); + +} + +u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.apic_base; +} +EXPORT_SYMBOL_GPL(kvm_lapic_get_base); + +void kvm_lapic_reset(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic; + int i; + + apic_debug("%s\n", __FUNCTION__); + + ASSERT(vcpu); + apic = vcpu->arch.apic; + ASSERT(apic != NULL); + + /* Stop the timer in case it's a reset to an active apic */ + hrtimer_cancel(&apic->timer.dev); + + apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24); + apic_set_reg(apic, APIC_LVR, APIC_VERSION); + + for (i = 0; i < APIC_LVT_NUM; i++) + apic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); + apic_set_reg(apic, APIC_LVT0, + SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); + + apic_set_reg(apic, APIC_DFR, 0xffffffffU); + apic_set_reg(apic, APIC_SPIV, 0xff); + apic_set_reg(apic, APIC_TASKPRI, 0); + apic_set_reg(apic, APIC_LDR, 0); + apic_set_reg(apic, APIC_ESR, 0); + apic_set_reg(apic, APIC_ICR, 0); + apic_set_reg(apic, APIC_ICR2, 0); + apic_set_reg(apic, APIC_TDCR, 0); + apic_set_reg(apic, APIC_TMICT, 0); + for (i = 0; i < 8; i++) { + apic_set_reg(apic, APIC_IRR + 0x10 * i, 0); + apic_set_reg(apic, APIC_ISR + 0x10 * i, 0); + apic_set_reg(apic, APIC_TMR + 0x10 * i, 0); + } + update_divide_count(apic); + atomic_set(&apic->timer.pending, 0); + if (vcpu->vcpu_id == 0) + vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP; + apic_update_ppr(apic); + + apic_debug(KERN_INFO "%s: vcpu=%p, id=%d, base_msr=" + "0x%016" PRIx64 ", base_address=0x%0lx.\n", __FUNCTION__, + vcpu, kvm_apic_id(apic), + vcpu->arch.apic_base, apic->base_address); +} +EXPORT_SYMBOL_GPL(kvm_lapic_reset); + +int kvm_lapic_enabled(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + int ret = 0; + + if (!apic) + return 0; + ret = apic_enabled(apic); + + return ret; +} +EXPORT_SYMBOL_GPL(kvm_lapic_enabled); + +/* + *---------------------------------------------------------------------- + * timer interface + *---------------------------------------------------------------------- + */ + +/* TODO: make sure __apic_timer_fn runs in current pCPU */ +static int __apic_timer_fn(struct kvm_lapic *apic) +{ + int result = 0; + wait_queue_head_t *q = &apic->vcpu->wq; + + atomic_inc(&apic->timer.pending); + if (waitqueue_active(q)) { + apic->vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; + wake_up_interruptible(q); + } + if (apic_lvtt_period(apic)) { + result = 1; + apic->timer.dev.expires = ktime_add_ns( + apic->timer.dev.expires, + apic->timer.period); + } + return result; +} + +static int __inject_apic_timer_irq(struct kvm_lapic *apic) +{ + int vector; + + vector = apic_lvt_vector(apic, APIC_LVTT); + return __apic_accept_irq(apic, APIC_DM_FIXED, vector, 1, 0); +} + +static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) +{ + struct kvm_lapic *apic; + int restart_timer = 0; + + apic = container_of(data, struct kvm_lapic, timer.dev); + + restart_timer = __apic_timer_fn(apic); + + if (restart_timer) + return HRTIMER_RESTART; + else + return HRTIMER_NORESTART; +} + +int kvm_create_lapic(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic; + + ASSERT(vcpu != NULL); + apic_debug("apic_init %d\n", vcpu->vcpu_id); + + apic = kzalloc(sizeof(*apic), GFP_KERNEL); + if (!apic) + goto nomem; + + vcpu->arch.apic = apic; + + apic->regs_page = alloc_page(GFP_KERNEL); + if (apic->regs_page == NULL) { + printk(KERN_ERR "malloc apic regs error for vcpu %x\n", + vcpu->vcpu_id); + goto nomem_free_apic; + } + apic->regs = page_address(apic->regs_page); + memset(apic->regs, 0, PAGE_SIZE); + apic->vcpu = vcpu; + + hrtimer_init(&apic->timer.dev, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); + apic->timer.dev.function = apic_timer_fn; + apic->base_address = APIC_DEFAULT_PHYS_BASE; + vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE; + + kvm_lapic_reset(vcpu); + apic->dev.read = apic_mmio_read; + apic->dev.write = apic_mmio_write; + apic->dev.in_range = apic_mmio_range; + apic->dev.private = apic; + + return 0; +nomem_free_apic: + kfree(apic); +nomem: + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(kvm_create_lapic); + +int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + int highest_irr; + + if (!apic || !apic_enabled(apic)) + return -1; + + apic_update_ppr(apic); + highest_irr = apic_find_highest_irr(apic); + if ((highest_irr == -1) || + ((highest_irr & 0xF0) <= apic_get_reg(apic, APIC_PROCPRI))) + return -1; + return highest_irr; +} + +int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) +{ + u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0); + int r = 0; + + if (vcpu->vcpu_id == 0) { + if (!apic_hw_enabled(vcpu->arch.apic)) + r = 1; + if ((lvt0 & APIC_LVT_MASKED) == 0 && + GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) + r = 1; + } + return r; +} + +void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + if (apic && apic_lvt_enabled(apic, APIC_LVTT) && + atomic_read(&apic->timer.pending) > 0) { + if (__inject_apic_timer_irq(apic)) + atomic_dec(&apic->timer.pending); + } +} + +void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + if (apic && apic_lvt_vector(apic, APIC_LVTT) == vec) + apic->timer.last_update = ktime_add_ns( + apic->timer.last_update, + apic->timer.period); +} + +int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) +{ + int vector = kvm_apic_has_interrupt(vcpu); + struct kvm_lapic *apic = vcpu->arch.apic; + + if (vector == -1) + return -1; + + apic_set_vector(vector, apic->regs + APIC_ISR); + apic_update_ppr(apic); + apic_clear_irr(vector, apic); + return vector; +} + +void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + + apic->base_address = vcpu->arch.apic_base & + MSR_IA32_APICBASE_BASE; + apic_set_reg(apic, APIC_LVR, APIC_VERSION); + apic_update_ppr(apic); + hrtimer_cancel(&apic->timer.dev); + update_divide_count(apic); + start_apic_timer(apic); +} + +void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) +{ + struct kvm_lapic *apic = vcpu->arch.apic; + struct hrtimer *timer; + + if (!apic) + return; + + timer = &apic->timer.dev; + if (hrtimer_cancel(timer)) + hrtimer_start(timer, timer->expires, HRTIMER_MODE_ABS); +} +EXPORT_SYMBOL_GPL(kvm_migrate_apic_timer); diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c new file mode 100644 index 000000000000..401eb7ce3207 --- /dev/null +++ b/arch/x86/kvm/mmu.c @@ -0,0 +1,1805 @@ +/* + * Kernel-based Virtual Machine driver for Linux + * + * This module enables machines with Intel VT-x extensions to run virtual + * machines without emulation or binary translation. + * + * MMU support + * + * Copyright (C) 2006 Qumranet, Inc. + * + * Authors: + * Yaniv Kamay + * Avi Kivity + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#include "vmx.h" +#include "mmu.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#undef MMU_DEBUG + +#undef AUDIT + +#ifdef AUDIT +static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg); +#else +static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {} +#endif + +#ifdef MMU_DEBUG + +#define pgprintk(x...) do { if (dbg) printk(x); } while (0) +#define rmap_printk(x...) do { if (dbg) printk(x); } while (0) + +#else + +#define pgprintk(x...) do { } while (0) +#define rmap_printk(x...) do { } while (0) + +#endif + +#if defined(MMU_DEBUG) || defined(AUDIT) +static int dbg = 1; +#endif + +#ifndef MMU_DEBUG +#define ASSERT(x) do { } while (0) +#else +#define ASSERT(x) \ + if (!(x)) { \ + printk(KERN_WARNING "assertion failed %s:%d: %s\n", \ + __FILE__, __LINE__, #x); \ + } +#endif + +#define PT64_PT_BITS 9 +#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS) +#define PT32_PT_BITS 10 +#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS) + +#define PT_WRITABLE_SHIFT 1 + +#define PT_PRESENT_MASK (1ULL << 0) +#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT) +#define PT_USER_MASK (1ULL << 2) +#define PT_PWT_MASK (1ULL << 3) +#define PT_PCD_MASK (1ULL << 4) +#define PT_ACCESSED_MASK (1ULL << 5) +#define PT_DIRTY_MASK (1ULL << 6) +#define PT_PAGE_SIZE_MASK (1ULL << 7) +#define PT_PAT_MASK (1ULL << 7) +#define PT_GLOBAL_MASK (1ULL << 8) +#define PT64_NX_SHIFT 63 +#define PT64_NX_MASK (1ULL << PT64_NX_SHIFT) + +#define PT_PAT_SHIFT 7 +#define PT_DIR_PAT_SHIFT 12 +#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT) + +#define PT32_DIR_PSE36_SIZE 4 +#define PT32_DIR_PSE36_SHIFT 13 +#define PT32_DIR_PSE36_MASK \ + (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT) + + +#define PT_FIRST_AVAIL_BITS_SHIFT 9 +#define PT64_SECOND_AVAIL_BITS_SHIFT 52 + +#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT) + +#define VALID_PAGE(x) ((x) != INVALID_PAGE) + +#define PT64_LEVEL_BITS 9 + +#define PT64_LEVEL_SHIFT(level) \ + (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS) + +#define PT64_LEVEL_MASK(level) \ + (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level)) + +#define PT64_INDEX(address, level)\ + (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1)) + + +#define PT32_LEVEL_BITS 10 + +#define PT32_LEVEL_SHIFT(level) \ + (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS) + +#define PT32_LEVEL_MASK(level) \ + (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level)) + +#define PT32_INDEX(address, level)\ + (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1)) + + +#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1)) +#define PT64_DIR_BASE_ADDR_MASK \ + (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1)) + +#define PT32_BASE_ADDR_MASK PAGE_MASK +#define PT32_DIR_BASE_ADDR_MASK \ + (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1)) + +#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \ + | PT64_NX_MASK) + +#define PFERR_PRESENT_MASK (1U << 0) +#define PFERR_WRITE_MASK (1U << 1) +#define PFERR_USER_MASK (1U << 2) +#define PFERR_FETCH_MASK (1U << 4) + +#define PT64_ROOT_LEVEL 4 +#define PT32_ROOT_LEVEL 2 +#define PT32E_ROOT_LEVEL 3 + +#define PT_DIRECTORY_LEVEL 2 +#define PT_PAGE_TABLE_LEVEL 1 + +#define RMAP_EXT 4 + +#define ACC_EXEC_MASK 1 +#define ACC_WRITE_MASK PT_WRITABLE_MASK +#define ACC_USER_MASK PT_USER_MASK +#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK) + +struct kvm_rmap_desc { + u64 *shadow_ptes[RMAP_EXT]; + struct kvm_rmap_desc *more; +}; + +static struct kmem_cache *pte_chain_cache; +static struct kmem_cache *rmap_desc_cache; +static struct kmem_cache *mmu_page_header_cache; + +static u64 __read_mostly shadow_trap_nonpresent_pte; +static u64 __read_mostly shadow_notrap_nonpresent_pte; + +void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte) +{ + shadow_trap_nonpresent_pte = trap_pte; + shadow_notrap_nonpresent_pte = notrap_pte; +} +EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes); + +static int is_write_protection(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr0 & X86_CR0_WP; +} + +static int is_cpuid_PSE36(void) +{ + return 1; +} + +static int is_nx(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.shadow_efer & EFER_NX; +} + +static int is_present_pte(unsigned long pte) +{ + return pte & PT_PRESENT_MASK; +} + +static int is_shadow_present_pte(u64 pte) +{ + pte &= ~PT_SHADOW_IO_MARK; + return pte != shadow_trap_nonpresent_pte + && pte != shadow_notrap_nonpresent_pte; +} + +static int is_writeble_pte(unsigned long pte) +{ + return pte & PT_WRITABLE_MASK; +} + +static int is_dirty_pte(unsigned long pte) +{ + return pte & PT_DIRTY_MASK; +} + +static int is_io_pte(unsigned long pte) +{ + return pte & PT_SHADOW_IO_MARK; +} + +static int is_rmap_pte(u64 pte) +{ + return pte != shadow_trap_nonpresent_pte + && pte != shadow_notrap_nonpresent_pte; +} + +static gfn_t pse36_gfn_delta(u32 gpte) +{ + int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT; + + return (gpte & PT32_DIR_PSE36_MASK) << shift; +} + +static void set_shadow_pte(u64 *sptep, u64 spte) +{ +#ifdef CONFIG_X86_64 + set_64bit((unsigned long *)sptep, spte); +#else + set_64bit((unsigned long long *)sptep, spte); +#endif +} + +static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, + struct kmem_cache *base_cache, int min) +{ + void *obj; + + if (cache->nobjs >= min) + return 0; + while (cache->nobjs < ARRAY_SIZE(cache->objects)) { + obj = kmem_cache_zalloc(base_cache, GFP_KERNEL); + if (!obj) + return -ENOMEM; + cache->objects[cache->nobjs++] = obj; + } + return 0; +} + +static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) +{ + while (mc->nobjs) + kfree(mc->objects[--mc->nobjs]); +} + +static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache, + int min) +{ + struct page *page; + + if (cache->nobjs >= min) + return 0; + while (cache->nobjs < ARRAY_SIZE(cache->objects)) { + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + set_page_private(page, 0); + cache->objects[cache->nobjs++] = page_address(page); + } + return 0; +} + +static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc) +{ + while (mc->nobjs) + free_page((unsigned long)mc->objects[--mc->nobjs]); +} + +static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) +{ + int r; + + kvm_mmu_free_some_pages(vcpu); + r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache, + pte_chain_cache, 4); + if (r) + goto out; + r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, + rmap_desc_cache, 1); + if (r) + goto out; + r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8); + if (r) + goto out; + r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache, + mmu_page_header_cache, 4); +out: + return r; +} + +static void mmu_free_memory_caches(struct kvm_vcpu *vcpu) +{ + mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache); + mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache); + mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache); + mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache); +} + +static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, + size_t size) +{ + void *p; + + BUG_ON(!mc->nobjs); + p = mc->objects[--mc->nobjs]; + memset(p, 0, size); + return p; +} + +static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu) +{ + return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache, + sizeof(struct kvm_pte_chain)); +} + +static void mmu_free_pte_chain(struct kvm_pte_chain *pc) +{ + kfree(pc); +} + +static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu) +{ + return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache, + sizeof(struct kvm_rmap_desc)); +} + +static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd) +{ + kfree(rd); +} + +/* + * Take gfn and return the reverse mapping to it. + * Note: gfn must be unaliased before this function get called + */ + +static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot; + + slot = gfn_to_memslot(kvm, gfn); + return &slot->rmap[gfn - slot->base_gfn]; +} + +/* + * Reverse mapping data structures: + * + * If rmapp bit zero is zero, then rmapp point to the shadw page table entry + * that points to page_address(page). + * + * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc + * containing more mappings. + */ +static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) +{ + struct kvm_mmu_page *sp; + struct kvm_rmap_desc *desc; + unsigned long *rmapp; + int i; + + if (!is_rmap_pte(*spte)) + return; + gfn = unalias_gfn(vcpu->kvm, gfn); + sp = page_header(__pa(spte)); + sp->gfns[spte - sp->spt] = gfn; + rmapp = gfn_to_rmap(vcpu->kvm, gfn); + if (!*rmapp) { + rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte); + *rmapp = (unsigned long)spte; + } else if (!(*rmapp & 1)) { + rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte); + desc = mmu_alloc_rmap_desc(vcpu); + desc->shadow_ptes[0] = (u64 *)*rmapp; + desc->shadow_ptes[1] = spte; + *rmapp = (unsigned long)desc | 1; + } else { + rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte); + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); + while (desc->shadow_ptes[RMAP_EXT-1] && desc->more) + desc = desc->more; + if (desc->shadow_ptes[RMAP_EXT-1]) { + desc->more = mmu_alloc_rmap_desc(vcpu); + desc = desc->more; + } + for (i = 0; desc->shadow_ptes[i]; ++i) + ; + desc->shadow_ptes[i] = spte; + } +} + +static void rmap_desc_remove_entry(unsigned long *rmapp, + struct kvm_rmap_desc *desc, + int i, + struct kvm_rmap_desc *prev_desc) +{ + int j; + + for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j) + ; + desc->shadow_ptes[i] = desc->shadow_ptes[j]; + desc->shadow_ptes[j] = NULL; + if (j != 0) + return; + if (!prev_desc && !desc->more) + *rmapp = (unsigned long)desc->shadow_ptes[0]; + else + if (prev_desc) + prev_desc->more = desc->more; + else + *rmapp = (unsigned long)desc->more | 1; + mmu_free_rmap_desc(desc); +} + +static void rmap_remove(struct kvm *kvm, u64 *spte) +{ + struct kvm_rmap_desc *desc; + struct kvm_rmap_desc *prev_desc; + struct kvm_mmu_page *sp; + struct page *page; + unsigned long *rmapp; + int i; + + if (!is_rmap_pte(*spte)) + return; + sp = page_header(__pa(spte)); + page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); + mark_page_accessed(page); + if (is_writeble_pte(*spte)) + kvm_release_page_dirty(page); + else + kvm_release_page_clean(page); + rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt]); + if (!*rmapp) { + printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); + BUG(); + } else if (!(*rmapp & 1)) { + rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte); + if ((u64 *)*rmapp != spte) { + printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n", + spte, *spte); + BUG(); + } + *rmapp = 0; + } else { + rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte); + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); + prev_desc = NULL; + while (desc) { + for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) + if (desc->shadow_ptes[i] == spte) { + rmap_desc_remove_entry(rmapp, + desc, i, + prev_desc); + return; + } + prev_desc = desc; + desc = desc->more; + } + BUG(); + } +} + +static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte) +{ + struct kvm_rmap_desc *desc; + struct kvm_rmap_desc *prev_desc; + u64 *prev_spte; + int i; + + if (!*rmapp) + return NULL; + else if (!(*rmapp & 1)) { + if (!spte) + return (u64 *)*rmapp; + return NULL; + } + desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); + prev_desc = NULL; + prev_spte = NULL; + while (desc) { + for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) { + if (prev_spte == spte) + return desc->shadow_ptes[i]; + prev_spte = desc->shadow_ptes[i]; + } + desc = desc->more; + } + return NULL; +} + +static void rmap_write_protect(struct kvm *kvm, u64 gfn) +{ + unsigned long *rmapp; + u64 *spte; + + gfn = unalias_gfn(kvm, gfn); + rmapp = gfn_to_rmap(kvm, gfn); + + spte = rmap_next(kvm, rmapp, NULL); + while (spte) { + BUG_ON(!spte); + BUG_ON(!(*spte & PT_PRESENT_MASK)); + rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); + if (is_writeble_pte(*spte)) + set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); + kvm_flush_remote_tlbs(kvm); + spte = rmap_next(kvm, rmapp, spte); + } +} + +#ifdef MMU_DEBUG +static int is_empty_shadow_page(u64 *spt) +{ + u64 *pos; + u64 *end; + + for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++) + if ((*pos & ~PT_SHADOW_IO_MARK) != shadow_trap_nonpresent_pte) { + printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__, + pos, *pos); + return 0; + } + return 1; +} +#endif + +static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp) +{ + ASSERT(is_empty_shadow_page(sp->spt)); + list_del(&sp->link); + __free_page(virt_to_page(sp->spt)); + __free_page(virt_to_page(sp->gfns)); + kfree(sp); + ++kvm->arch.n_free_mmu_pages; +} + +static unsigned kvm_page_table_hashfn(gfn_t gfn) +{ + return gfn; +} + +static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, + u64 *parent_pte) +{ + struct kvm_mmu_page *sp; + + if (!vcpu->kvm->arch.n_free_mmu_pages) + return NULL; + + sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp); + sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); + sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); + set_page_private(virt_to_page(sp->spt), (unsigned long)sp); + list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); + ASSERT(is_empty_shadow_page(sp->spt)); + sp->slot_bitmap = 0; + sp->multimapped = 0; + sp->parent_pte = parent_pte; + --vcpu->kvm->arch.n_free_mmu_pages; + return sp; +} + +static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp, u64 *parent_pte) +{ + struct kvm_pte_chain *pte_chain; + struct hlist_node *node; + int i; + + if (!parent_pte) + return; + if (!sp->multimapped) { + u64 *old = sp->parent_pte; + + if (!old) { + sp->parent_pte = parent_pte; + return; + } + sp->multimapped = 1; + pte_chain = mmu_alloc_pte_chain(vcpu); + INIT_HLIST_HEAD(&sp->parent_ptes); + hlist_add_head(&pte_chain->link, &sp->parent_ptes); + pte_chain->parent_ptes[0] = old; + } + hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) { + if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1]) + continue; + for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) + if (!pte_chain->parent_ptes[i]) { + pte_chain->parent_ptes[i] = parent_pte; + return; + } + } + pte_chain = mmu_alloc_pte_chain(vcpu); + BUG_ON(!pte_chain); + hlist_add_head(&pte_chain->link, &sp->parent_ptes); + pte_chain->parent_ptes[0] = parent_pte; +} + +static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp, + u64 *parent_pte) +{ + struct kvm_pte_chain *pte_chain; + struct hlist_node *node; + int i; + + if (!sp->multimapped) { + BUG_ON(sp->parent_pte != parent_pte); + sp->parent_pte = NULL; + return; + } + hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) + for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) { + if (!pte_chain->parent_ptes[i]) + break; + if (pte_chain->parent_ptes[i] != parent_pte) + continue; + while (i + 1 < NR_PTE_CHAIN_ENTRIES + && pte_chain->parent_ptes[i + 1]) { + pte_chain->parent_ptes[i] + = pte_chain->parent_ptes[i + 1]; + ++i; + } + pte_chain->parent_ptes[i] = NULL; + if (i == 0) { + hlist_del(&pte_chain->link); + mmu_free_pte_chain(pte_chain); + if (hlist_empty(&sp->parent_ptes)) { + sp->multimapped = 0; + sp->parent_pte = NULL; + } + } + return; + } + BUG(); +} + +static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) +{ + unsigned index; + struct hlist_head *bucket; + struct kvm_mmu_page *sp; + struct hlist_node *node; + + pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); + index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; + bucket = &kvm->arch.mmu_page_hash[index]; + hlist_for_each_entry(sp, node, bucket, hash_link) + if (sp->gfn == gfn && !sp->role.metaphysical) { + pgprintk("%s: found role %x\n", + __FUNCTION__, sp->role.word); + return sp; + } + return NULL; +} + +static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, + gfn_t gfn, + gva_t gaddr, + unsigned level, + int metaphysical, + unsigned access, + u64 *parent_pte, + bool *new_page) +{ + union kvm_mmu_page_role role; + unsigned index; + unsigned quadrant; + struct hlist_head *bucket; + struct kvm_mmu_page *sp; + struct hlist_node *node; + + role.word = 0; + role.glevels = vcpu->arch.mmu.root_level; + role.level = level; + role.metaphysical = metaphysical; + role.access = access; + if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) { + quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); + quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; + role.quadrant = quadrant; + } + pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__, + gfn, role.word); + index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; + bucket = &vcpu->kvm->arch.mmu_page_hash[index]; + hlist_for_each_entry(sp, node, bucket, hash_link) + if (sp->gfn == gfn && sp->role.word == role.word) { + mmu_page_add_parent_pte(vcpu, sp, parent_pte); + pgprintk("%s: found\n", __FUNCTION__); + return sp; + } + sp = kvm_mmu_alloc_page(vcpu, parent_pte); + if (!sp) + return sp; + pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word); + sp->gfn = gfn; + sp->role = role; + hlist_add_head(&sp->hash_link, bucket); + vcpu->arch.mmu.prefetch_page(vcpu, sp); + if (!metaphysical) + rmap_write_protect(vcpu->kvm, gfn); + if (new_page) + *new_page = 1; + return sp; +} + +static void kvm_mmu_page_unlink_children(struct kvm *kvm, + struct kvm_mmu_page *sp) +{ + unsigned i; + u64 *pt; + u64 ent; + + pt = sp->spt; + + if (sp->role.level == PT_PAGE_TABLE_LEVEL) { + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { + if (is_shadow_present_pte(pt[i])) + rmap_remove(kvm, &pt[i]); + pt[i] = shadow_trap_nonpresent_pte; + } + kvm_flush_remote_tlbs(kvm); + return; + } + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { + ent = pt[i]; + + pt[i] = shadow_trap_nonpresent_pte; + if (!is_shadow_present_pte(ent)) + continue; + ent &= PT64_BASE_ADDR_MASK; + mmu_page_remove_parent_pte(page_header(ent), &pt[i]); + } + kvm_flush_remote_tlbs(kvm); +} + +static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte) +{ + mmu_page_remove_parent_pte(sp, parent_pte); +} + +static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) +{ + int i; + + for (i = 0; i < KVM_MAX_VCPUS; ++i) + if (kvm->vcpus[i]) + kvm->vcpus[i]->arch.last_pte_updated = NULL; +} + +static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) +{ + u64 *parent_pte; + + ++kvm->stat.mmu_shadow_zapped; + while (sp->multimapped || sp->parent_pte) { + if (!sp->multimapped) + parent_pte = sp->parent_pte; + else { + struct kvm_pte_chain *chain; + + chain = container_of(sp->parent_ptes.first, + struct kvm_pte_chain, link); + parent_pte = chain->parent_ptes[0]; + } + BUG_ON(!parent_pte); + kvm_mmu_put_page(sp, parent_pte); + set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte); + } + kvm_mmu_page_unlink_children(kvm, sp); + if (!sp->root_count) { + hlist_del(&sp->hash_link); + kvm_mmu_free_page(kvm, sp); + } else + list_move(&sp->link, &kvm->arch.active_mmu_pages); + kvm_mmu_reset_last_pte_updated(kvm); +} + +/* + * Changing the number of mmu pages allocated to the vm + * Note: if kvm_nr_mmu_pages is too small, you will get dead lock + */ +void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages) +{ + /* + * If we set the number of mmu pages to be smaller be than the + * number of actived pages , we must to free some mmu pages before we + * change the value + */ + + if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) > + kvm_nr_mmu_pages) { + int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages + - kvm->arch.n_free_mmu_pages; + + while (n_used_mmu_pages > kvm_nr_mmu_pages) { + struct kvm_mmu_page *page; + + page = container_of(kvm->arch.active_mmu_pages.prev, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(kvm, page); + n_used_mmu_pages--; + } + kvm->arch.n_free_mmu_pages = 0; + } + else + kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages + - kvm->arch.n_alloc_mmu_pages; + + kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages; +} + +static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) +{ + unsigned index; + struct hlist_head *bucket; + struct kvm_mmu_page *sp; + struct hlist_node *node, *n; + int r; + + pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); + r = 0; + index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; + bucket = &kvm->arch.mmu_page_hash[index]; + hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) + if (sp->gfn == gfn && !sp->role.metaphysical) { + pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, + sp->role.word); + kvm_mmu_zap_page(kvm, sp); + r = 1; + } + return r; +} + +static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_mmu_page *sp; + + while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { + pgprintk("%s: zap %lx %x\n", __FUNCTION__, gfn, sp->role.word); + kvm_mmu_zap_page(kvm, sp); + } +} + +static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) +{ + int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn)); + struct kvm_mmu_page *sp = page_header(__pa(pte)); + + __set_bit(slot, &sp->slot_bitmap); +} + +struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) +{ + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); + + if (gpa == UNMAPPED_GVA) + return NULL; + return gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); +} + +static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, + unsigned pt_access, unsigned pte_access, + int user_fault, int write_fault, int dirty, + int *ptwrite, gfn_t gfn) +{ + u64 spte; + int was_rmapped = is_rmap_pte(*shadow_pte); + struct page *page; + + pgprintk("%s: spte %llx access %x write_fault %d" + " user_fault %d gfn %lx\n", + __FUNCTION__, *shadow_pte, pt_access, + write_fault, user_fault, gfn); + + /* + * We don't set the accessed bit, since we sometimes want to see + * whether the guest actually used the pte (in order to detect + * demand paging). + */ + spte = PT_PRESENT_MASK | PT_DIRTY_MASK; + if (!dirty) + pte_access &= ~ACC_WRITE_MASK; + if (!(pte_access & ACC_EXEC_MASK)) + spte |= PT64_NX_MASK; + + page = gfn_to_page(vcpu->kvm, gfn); + + spte |= PT_PRESENT_MASK; + if (pte_access & ACC_USER_MASK) + spte |= PT_USER_MASK; + + if (is_error_page(page)) { + set_shadow_pte(shadow_pte, + shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); + kvm_release_page_clean(page); + return; + } + + spte |= page_to_phys(page); + + if ((pte_access & ACC_WRITE_MASK) + || (write_fault && !is_write_protection(vcpu) && !user_fault)) { + struct kvm_mmu_page *shadow; + + spte |= PT_WRITABLE_MASK; + if (user_fault) { + mmu_unshadow(vcpu->kvm, gfn); + goto unshadowed; + } + + shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn); + if (shadow) { + pgprintk("%s: found shadow page for %lx, marking ro\n", + __FUNCTION__, gfn); + pte_access &= ~ACC_WRITE_MASK; + if (is_writeble_pte(spte)) { + spte &= ~PT_WRITABLE_MASK; + kvm_x86_ops->tlb_flush(vcpu); + } + if (write_fault) + *ptwrite = 1; + } + } + +unshadowed: + + if (pte_access & ACC_WRITE_MASK) + mark_page_dirty(vcpu->kvm, gfn); + + pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); + set_shadow_pte(shadow_pte, spte); + page_header_update_slot(vcpu->kvm, shadow_pte, gfn); + if (!was_rmapped) { + rmap_add(vcpu, shadow_pte, gfn); + if (!is_rmap_pte(*shadow_pte)) + kvm_release_page_clean(page); + } + else + kvm_release_page_clean(page); + if (!ptwrite || !*ptwrite) + vcpu->arch.last_pte_updated = shadow_pte; +} + +static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) +{ +} + +static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) +{ + int level = PT32E_ROOT_LEVEL; + hpa_t table_addr = vcpu->arch.mmu.root_hpa; + int pt_write = 0; + + for (; ; level--) { + u32 index = PT64_INDEX(v, level); + u64 *table; + + ASSERT(VALID_PAGE(table_addr)); + table = __va(table_addr); + + if (level == 1) { + mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL, + 0, write, 1, &pt_write, gfn); + return pt_write || is_io_pte(table[index]); + } + + if (table[index] == shadow_trap_nonpresent_pte) { + struct kvm_mmu_page *new_table; + gfn_t pseudo_gfn; + + pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK) + >> PAGE_SHIFT; + new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, + v, level - 1, + 1, ACC_ALL, &table[index], + NULL); + if (!new_table) { + pgprintk("nonpaging_map: ENOMEM\n"); + return -ENOMEM; + } + + table[index] = __pa(new_table->spt) | PT_PRESENT_MASK + | PT_WRITABLE_MASK | PT_USER_MASK; + } + table_addr = table[index] & PT64_BASE_ADDR_MASK; + } +} + +static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp) +{ + int i; + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) + sp->spt[i] = shadow_trap_nonpresent_pte; +} + +static void mmu_free_roots(struct kvm_vcpu *vcpu) +{ + int i; + struct kvm_mmu_page *sp; + + if (!VALID_PAGE(vcpu->arch.mmu.root_hpa)) + return; +#ifdef CONFIG_X86_64 + if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { + hpa_t root = vcpu->arch.mmu.root_hpa; + + sp = page_header(root); + --sp->root_count; + vcpu->arch.mmu.root_hpa = INVALID_PAGE; + return; + } +#endif + for (i = 0; i < 4; ++i) { + hpa_t root = vcpu->arch.mmu.pae_root[i]; + + if (root) { + root &= PT64_BASE_ADDR_MASK; + sp = page_header(root); + --sp->root_count; + } + vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; + } + vcpu->arch.mmu.root_hpa = INVALID_PAGE; +} + +static void mmu_alloc_roots(struct kvm_vcpu *vcpu) +{ + int i; + gfn_t root_gfn; + struct kvm_mmu_page *sp; + + root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT; + +#ifdef CONFIG_X86_64 + if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { + hpa_t root = vcpu->arch.mmu.root_hpa; + + ASSERT(!VALID_PAGE(root)); + sp = kvm_mmu_get_page(vcpu, root_gfn, 0, + PT64_ROOT_LEVEL, 0, ACC_ALL, NULL, NULL); + root = __pa(sp->spt); + ++sp->root_count; + vcpu->arch.mmu.root_hpa = root; + return; + } +#endif + for (i = 0; i < 4; ++i) { + hpa_t root = vcpu->arch.mmu.pae_root[i]; + + ASSERT(!VALID_PAGE(root)); + if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) { + if (!is_present_pte(vcpu->arch.pdptrs[i])) { + vcpu->arch.mmu.pae_root[i] = 0; + continue; + } + root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT; + } else if (vcpu->arch.mmu.root_level == 0) + root_gfn = 0; + sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, + PT32_ROOT_LEVEL, !is_paging(vcpu), + ACC_ALL, NULL, NULL); + root = __pa(sp->spt); + ++sp->root_count; + vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK; + } + vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root); +} + +static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) +{ + return vaddr; +} + +static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, + u32 error_code) +{ + gfn_t gfn; + int r; + + pgprintk("%s: gva %lx error %x\n", __FUNCTION__, gva, error_code); + r = mmu_topup_memory_caches(vcpu); + if (r) + return r; + + ASSERT(vcpu); + ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa)); + + gfn = gva >> PAGE_SHIFT; + + return nonpaging_map(vcpu, gva & PAGE_MASK, + error_code & PFERR_WRITE_MASK, gfn); +} + +static void nonpaging_free(struct kvm_vcpu *vcpu) +{ + mmu_free_roots(vcpu); +} + +static int nonpaging_init_context(struct kvm_vcpu *vcpu) +{ + struct kvm_mmu *context = &vcpu->arch.mmu; + + context->new_cr3 = nonpaging_new_cr3; + context->page_fault = nonpaging_page_fault; + context->gva_to_gpa = nonpaging_gva_to_gpa; + context->free = nonpaging_free; + context->prefetch_page = nonpaging_prefetch_page; + context->root_level = 0; + context->shadow_root_level = PT32E_ROOT_LEVEL; + context->root_hpa = INVALID_PAGE; + return 0; +} + +void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) +{ + ++vcpu->stat.tlb_flush; + kvm_x86_ops->tlb_flush(vcpu); +} + +static void paging_new_cr3(struct kvm_vcpu *vcpu) +{ + pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3); + mmu_free_roots(vcpu); +} + +static void inject_page_fault(struct kvm_vcpu *vcpu, + u64 addr, + u32 err_code) +{ + kvm_inject_page_fault(vcpu, addr, err_code); +} + +static void paging_free(struct kvm_vcpu *vcpu) +{ + nonpaging_free(vcpu); +} + +#define PTTYPE 64 +#include "paging_tmpl.h" +#undef PTTYPE + +#define PTTYPE 32 +#include "paging_tmpl.h" +#undef PTTYPE + +static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level) +{ + struct kvm_mmu *context = &vcpu->arch.mmu; + + ASSERT(is_pae(vcpu)); + context->new_cr3 = paging_new_cr3; + context->page_fault = paging64_page_fault; + context->gva_to_gpa = paging64_gva_to_gpa; + context->prefetch_page = paging64_prefetch_page; + context->free = paging_free; + context->root_level = level; + context->shadow_root_level = level; + context->root_hpa = INVALID_PAGE; + return 0; +} + +static int paging64_init_context(struct kvm_vcpu *vcpu) +{ + return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL); +} + +static int paging32_init_context(struct kvm_vcpu *vcpu) +{ + struct kvm_mmu *context = &vcpu->arch.mmu; + + context->new_cr3 = paging_new_cr3; + context->page_fault = paging32_page_fault; + context->gva_to_gpa = paging32_gva_to_gpa; + context->free = paging_free; + context->prefetch_page = paging32_prefetch_page; + context->root_level = PT32_ROOT_LEVEL; + context->shadow_root_level = PT32E_ROOT_LEVEL; + context->root_hpa = INVALID_PAGE; + return 0; +} + +static int paging32E_init_context(struct kvm_vcpu *vcpu) +{ + return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL); +} + +static int init_kvm_mmu(struct kvm_vcpu *vcpu) +{ + ASSERT(vcpu); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); + + if (!is_paging(vcpu)) + return nonpaging_init_context(vcpu); + else if (is_long_mode(vcpu)) + return paging64_init_context(vcpu); + else if (is_pae(vcpu)) + return paging32E_init_context(vcpu); + else + return paging32_init_context(vcpu); +} + +static void destroy_kvm_mmu(struct kvm_vcpu *vcpu) +{ + ASSERT(vcpu); + if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) { + vcpu->arch.mmu.free(vcpu); + vcpu->arch.mmu.root_hpa = INVALID_PAGE; + } +} + +int kvm_mmu_reset_context(struct kvm_vcpu *vcpu) +{ + destroy_kvm_mmu(vcpu); + return init_kvm_mmu(vcpu); +} +EXPORT_SYMBOL_GPL(kvm_mmu_reset_context); + +int kvm_mmu_load(struct kvm_vcpu *vcpu) +{ + int r; + + mutex_lock(&vcpu->kvm->lock); + r = mmu_topup_memory_caches(vcpu); + if (r) + goto out; + mmu_alloc_roots(vcpu); + kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa); + kvm_mmu_flush_tlb(vcpu); +out: + mutex_unlock(&vcpu->kvm->lock); + return r; +} +EXPORT_SYMBOL_GPL(kvm_mmu_load); + +void kvm_mmu_unload(struct kvm_vcpu *vcpu) +{ + mmu_free_roots(vcpu); +} + +static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp, + u64 *spte) +{ + u64 pte; + struct kvm_mmu_page *child; + + pte = *spte; + if (is_shadow_present_pte(pte)) { + if (sp->role.level == PT_PAGE_TABLE_LEVEL) + rmap_remove(vcpu->kvm, spte); + else { + child = page_header(pte & PT64_BASE_ADDR_MASK); + mmu_page_remove_parent_pte(child, spte); + } + } + set_shadow_pte(spte, shadow_trap_nonpresent_pte); +} + +static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp, + u64 *spte, + const void *new, int bytes, + int offset_in_pte) +{ + if (sp->role.level != PT_PAGE_TABLE_LEVEL) { + ++vcpu->kvm->stat.mmu_pde_zapped; + return; + } + + ++vcpu->kvm->stat.mmu_pte_updated; + if (sp->role.glevels == PT32_ROOT_LEVEL) + paging32_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte); + else + paging64_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte); +} + +static bool need_remote_flush(u64 old, u64 new) +{ + if (!is_shadow_present_pte(old)) + return false; + if (!is_shadow_present_pte(new)) + return true; + if ((old ^ new) & PT64_BASE_ADDR_MASK) + return true; + old ^= PT64_NX_MASK; + new ^= PT64_NX_MASK; + return (old & ~new & PT64_PERM_MASK) != 0; +} + +static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new) +{ + if (need_remote_flush(old, new)) + kvm_flush_remote_tlbs(vcpu->kvm); + else + kvm_mmu_flush_tlb(vcpu); +} + +static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) +{ + u64 *spte = vcpu->arch.last_pte_updated; + + return !!(spte && (*spte & PT_ACCESSED_MASK)); +} + +void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, + const u8 *new, int bytes) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + struct kvm_mmu_page *sp; + struct hlist_node *node, *n; + struct hlist_head *bucket; + unsigned index; + u64 entry; + u64 *spte; + unsigned offset = offset_in_page(gpa); + unsigned pte_size; + unsigned page_offset; + unsigned misaligned; + unsigned quadrant; + int level; + int flooded = 0; + int npte; + + pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); + ++vcpu->kvm->stat.mmu_pte_write; + kvm_mmu_audit(vcpu, "pre pte write"); + if (gfn == vcpu->arch.last_pt_write_gfn + && !last_updated_pte_accessed(vcpu)) { + ++vcpu->arch.last_pt_write_count; + if (vcpu->arch.last_pt_write_count >= 3) + flooded = 1; + } else { + vcpu->arch.last_pt_write_gfn = gfn; + vcpu->arch.last_pt_write_count = 1; + vcpu->arch.last_pte_updated = NULL; + } + index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; + bucket = &vcpu->kvm->arch.mmu_page_hash[index]; + hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) { + if (sp->gfn != gfn || sp->role.metaphysical) + continue; + pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; + misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); + misaligned |= bytes < 4; + if (misaligned || flooded) { + /* + * Misaligned accesses are too much trouble to fix + * up; also, they usually indicate a page is not used + * as a page table. + * + * If we're seeing too many writes to a page, + * it may no longer be a page table, or we may be + * forking, in which case it is better to unmap the + * page. + */ + pgprintk("misaligned: gpa %llx bytes %d role %x\n", + gpa, bytes, sp->role.word); + kvm_mmu_zap_page(vcpu->kvm, sp); + ++vcpu->kvm->stat.mmu_flooded; + continue; + } + page_offset = offset; + level = sp->role.level; + npte = 1; + if (sp->role.glevels == PT32_ROOT_LEVEL) { + page_offset <<= 1; /* 32->64 */ + /* + * A 32-bit pde maps 4MB while the shadow pdes map + * only 2MB. So we need to double the offset again + * and zap two pdes instead of one. + */ + if (level == PT32_ROOT_LEVEL) { + page_offset &= ~7; /* kill rounding error */ + page_offset <<= 1; + npte = 2; + } + quadrant = page_offset >> PAGE_SHIFT; + page_offset &= ~PAGE_MASK; + if (quadrant != sp->role.quadrant) + continue; + } + spte = &sp->spt[page_offset / sizeof(*spte)]; + while (npte--) { + entry = *spte; + mmu_pte_write_zap_pte(vcpu, sp, spte); + mmu_pte_write_new_pte(vcpu, sp, spte, new, bytes, + page_offset & (pte_size - 1)); + mmu_pte_write_flush_tlb(vcpu, entry, *spte); + ++spte; + } + } + kvm_mmu_audit(vcpu, "post pte write"); +} + +int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) +{ + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); + + return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); +} + +void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) +{ + while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) { + struct kvm_mmu_page *sp; + + sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(vcpu->kvm, sp); + ++vcpu->kvm->stat.mmu_recycled; + } +} + +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) +{ + int r; + enum emulation_result er; + + mutex_lock(&vcpu->kvm->lock); + r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code); + if (r < 0) + goto out; + + if (!r) { + r = 1; + goto out; + } + + r = mmu_topup_memory_caches(vcpu); + if (r) + goto out; + + er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0); + mutex_unlock(&vcpu->kvm->lock); + + switch (er) { + case EMULATE_DONE: + return 1; + case EMULATE_DO_MMIO: + ++vcpu->stat.mmio_exits; + return 0; + case EMULATE_FAIL: + kvm_report_emulation_failure(vcpu, "pagetable"); + return 1; + default: + BUG(); + } +out: + mutex_unlock(&vcpu->kvm->lock); + return r; +} +EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); + +static void free_mmu_pages(struct kvm_vcpu *vcpu) +{ + struct kvm_mmu_page *sp; + + while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) { + sp = container_of(vcpu->kvm->arch.active_mmu_pages.next, + struct kvm_mmu_page, link); + kvm_mmu_zap_page(vcpu->kvm, sp); + } + free_page((unsigned long)vcpu->arch.mmu.pae_root); +} + +static int alloc_mmu_pages(struct kvm_vcpu *vcpu) +{ + struct page *page; + int i; + + ASSERT(vcpu); + + if (vcpu->kvm->arch.n_requested_mmu_pages) + vcpu->kvm->arch.n_free_mmu_pages = + vcpu->kvm->arch.n_requested_mmu_pages; + else + vcpu->kvm->arch.n_free_mmu_pages = + vcpu->kvm->arch.n_alloc_mmu_pages; + /* + * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64. + * Therefore we need to allocate shadow page tables in the first + * 4GB of memory, which happens to fit the DMA32 zone. + */ + page = alloc_page(GFP_KERNEL | __GFP_DMA32); + if (!page) + goto error_1; + vcpu->arch.mmu.pae_root = page_address(page); + for (i = 0; i < 4; ++i) + vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; + + return 0; + +error_1: + free_mmu_pages(vcpu); + return -ENOMEM; +} + +int kvm_mmu_create(struct kvm_vcpu *vcpu) +{ + ASSERT(vcpu); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); + + return alloc_mmu_pages(vcpu); +} + +int kvm_mmu_setup(struct kvm_vcpu *vcpu) +{ + ASSERT(vcpu); + ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); + + return init_kvm_mmu(vcpu); +} + +void kvm_mmu_destroy(struct kvm_vcpu *vcpu) +{ + ASSERT(vcpu); + + destroy_kvm_mmu(vcpu); + free_mmu_pages(vcpu); + mmu_free_memory_caches(vcpu); +} + +void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) +{ + struct kvm_mmu_page *sp; + + list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) { + int i; + u64 *pt; + + if (!test_bit(slot, &sp->slot_bitmap)) + continue; + + pt = sp->spt; + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) + /* avoid RMW */ + if (pt[i] & PT_WRITABLE_MASK) + pt[i] &= ~PT_WRITABLE_MASK; + } +} + +void kvm_mmu_zap_all(struct kvm *kvm) +{ + struct kvm_mmu_page *sp, *node; + + list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) + kvm_mmu_zap_page(kvm, sp); + + kvm_flush_remote_tlbs(kvm); +} + +void kvm_mmu_module_exit(void) +{ + if (pte_chain_cache) + kmem_cache_destroy(pte_chain_cache); + if (rmap_desc_cache) + kmem_cache_destroy(rmap_desc_cache); + if (mmu_page_header_cache) + kmem_cache_destroy(mmu_page_header_cache); +} + +int kvm_mmu_module_init(void) +{ + pte_chain_cache = kmem_cache_create("kvm_pte_chain", + sizeof(struct kvm_pte_chain), + 0, 0, NULL); + if (!pte_chain_cache) + goto nomem; + rmap_desc_cache = kmem_cache_create("kvm_rmap_desc", + sizeof(struct kvm_rmap_desc), + 0, 0, NULL); + if (!rmap_desc_cache) + goto nomem; + + mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header", + sizeof(struct kvm_mmu_page), + 0, 0, NULL); + if (!mmu_page_header_cache) + goto nomem; + + return 0; + +nomem: + kvm_mmu_module_exit(); + return -ENOMEM; +} + +/* + * Caculate mmu pages needed for kvm. + */ +unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm) +{ + int i; + unsigned int nr_mmu_pages; + unsigned int nr_pages = 0; + + for (i = 0; i < kvm->nmemslots; i++) + nr_pages += kvm->memslots[i].npages; + + nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; + nr_mmu_pages = max(nr_mmu_pages, + (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); + + return nr_mmu_pages; +} + +#ifdef AUDIT + +static const char *audit_msg; + +static gva_t canonicalize(gva_t gva) +{ +#ifdef CONFIG_X86_64 + gva = (long long)(gva << 16) >> 16; +#endif + return gva; +} + +static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, + gva_t va, int level) +{ + u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK); + int i; + gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1)); + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) { + u64 ent = pt[i]; + + if (ent == shadow_trap_nonpresent_pte) + continue; + + va = canonicalize(va); + if (level > 1) { + if (ent == shadow_notrap_nonpresent_pte) + printk(KERN_ERR "audit: (%s) nontrapping pte" + " in nonleaf level: levels %d gva %lx" + " level %d pte %llx\n", audit_msg, + vcpu->arch.mmu.root_level, va, level, ent); + + audit_mappings_page(vcpu, ent, va, level - 1); + } else { + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va); + struct page *page = gpa_to_page(vcpu, gpa); + hpa_t hpa = page_to_phys(page); + + if (is_shadow_present_pte(ent) + && (ent & PT64_BASE_ADDR_MASK) != hpa) + printk(KERN_ERR "xx audit error: (%s) levels %d" + " gva %lx gpa %llx hpa %llx ent %llx %d\n", + audit_msg, vcpu->arch.mmu.root_level, + va, gpa, hpa, ent, + is_shadow_present_pte(ent)); + else if (ent == shadow_notrap_nonpresent_pte + && !is_error_hpa(hpa)) + printk(KERN_ERR "audit: (%s) notrap shadow," + " valid guest gva %lx\n", audit_msg, va); + kvm_release_page_clean(page); + + } + } +} + +static void audit_mappings(struct kvm_vcpu *vcpu) +{ + unsigned i; + + if (vcpu->arch.mmu.root_level == 4) + audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4); + else + for (i = 0; i < 4; ++i) + if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK) + audit_mappings_page(vcpu, + vcpu->arch.mmu.pae_root[i], + i << 30, + 2); +} + +static int count_rmaps(struct kvm_vcpu *vcpu) +{ + int nmaps = 0; + int i, j, k; + + for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { + struct kvm_memory_slot *m = &vcpu->kvm->memslots[i]; + struct kvm_rmap_desc *d; + + for (j = 0; j < m->npages; ++j) { + unsigned long *rmapp = &m->rmap[j]; + + if (!*rmapp) + continue; + if (!(*rmapp & 1)) { + ++nmaps; + continue; + } + d = (struct kvm_rmap_desc *)(*rmapp & ~1ul); + while (d) { + for (k = 0; k < RMAP_EXT; ++k) + if (d->shadow_ptes[k]) + ++nmaps; + else + break; + d = d->more; + } + } + } + return nmaps; +} + +static int count_writable_mappings(struct kvm_vcpu *vcpu) +{ + int nmaps = 0; + struct kvm_mmu_page *sp; + int i; + + list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { + u64 *pt = sp->spt; + + if (sp->role.level != PT_PAGE_TABLE_LEVEL) + continue; + + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { + u64 ent = pt[i]; + + if (!(ent & PT_PRESENT_MASK)) + continue; + if (!(ent & PT_WRITABLE_MASK)) + continue; + ++nmaps; + } + } + return nmaps; +} + +static void audit_rmap(struct kvm_vcpu *vcpu) +{ + int n_rmap = count_rmaps(vcpu); + int n_actual = count_writable_mappings(vcpu); + + if (n_rmap != n_actual) + printk(KERN_ERR "%s: (%s) rmap %d actual %d\n", + __FUNCTION__, audit_msg, n_rmap, n_actual); +} + +static void audit_write_protection(struct kvm_vcpu *vcpu) +{ + struct kvm_mmu_page *sp; + struct kvm_memory_slot *slot; + unsigned long *rmapp; + gfn_t gfn; + + list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { + if (sp->role.metaphysical) + continue; + + slot = gfn_to_memslot(vcpu->kvm, sp->gfn); + gfn = unalias_gfn(vcpu->kvm, sp->gfn); + rmapp = &slot->rmap[gfn - slot->base_gfn]; + if (*rmapp) + printk(KERN_ERR "%s: (%s) shadow page has writable" + " mappings: gfn %lx role %x\n", + __FUNCTION__, audit_msg, sp->gfn, + sp->role.word); + } +} + +static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) +{ + int olddbg = dbg; + + dbg = 0; + audit_msg = msg; + audit_rmap(vcpu); + audit_write_protection(vcpu); + audit_mappings(vcpu); + dbg = olddbg; +} + +#endif diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h new file mode 100644 index 000000000000..1fce19ec7a23 --- /dev/null +++ b/arch/x86/kvm/mmu.h @@ -0,0 +1,44 @@ +#ifndef __KVM_X86_MMU_H +#define __KVM_X86_MMU_H + +#include + +static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) +{ + if (unlikely(vcpu->kvm->arch.n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) + __kvm_mmu_free_some_pages(vcpu); +} + +static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) +{ + if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) + return 0; + + return kvm_mmu_load(vcpu); +} + +static inline int is_long_mode(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + return vcpu->arch.shadow_efer & EFER_LME; +#else + return 0; +#endif +} + +static inline int is_pae(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr4 & X86_CR4_PAE; +} + +static inline int is_pse(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr4 & X86_CR4_PSE; +} + +static inline int is_paging(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.cr0 & X86_CR0_PG; +} + +#endif diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h new file mode 100644 index 000000000000..56b88f7e83ef --- /dev/null +++ b/arch/x86/kvm/paging_tmpl.h @@ -0,0 +1,461 @@ +/* + * Kernel-based Virtual Machine driver for Linux + * + * This module enables machines with Intel VT-x extensions to run virtual + * machines without emulation or binary translation. + * + * MMU support + * + * Copyright (C) 2006 Qumranet, Inc. + * + * Authors: + * Yaniv Kamay + * Avi Kivity + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +/* + * We need the mmu code to access both 32-bit and 64-bit guest ptes, + * so the code in this file is compiled twice, once per pte size. + */ + +#if PTTYPE == 64 + #define pt_element_t u64 + #define guest_walker guest_walker64 + #define FNAME(name) paging##64_##name + #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK + #define PT_DIR_BASE_ADDR_MASK PT64_DIR_BASE_ADDR_MASK + #define PT_INDEX(addr, level) PT64_INDEX(addr, level) + #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) + #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level) + #define PT_LEVEL_BITS PT64_LEVEL_BITS + #ifdef CONFIG_X86_64 + #define PT_MAX_FULL_LEVELS 4 + #define CMPXCHG cmpxchg + #else + #define CMPXCHG cmpxchg64 + #define PT_MAX_FULL_LEVELS 2 + #endif +#elif PTTYPE == 32 + #define pt_element_t u32 + #define guest_walker guest_walker32 + #define FNAME(name) paging##32_##name + #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK + #define PT_DIR_BASE_ADDR_MASK PT32_DIR_BASE_ADDR_MASK + #define PT_INDEX(addr, level) PT32_INDEX(addr, level) + #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) + #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level) + #define PT_LEVEL_BITS PT32_LEVEL_BITS + #define PT_MAX_FULL_LEVELS 2 + #define CMPXCHG cmpxchg +#else + #error Invalid PTTYPE value +#endif + +#define gpte_to_gfn FNAME(gpte_to_gfn) +#define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde) + +/* + * The guest_walker structure emulates the behavior of the hardware page + * table walker. + */ +struct guest_walker { + int level; + gfn_t table_gfn[PT_MAX_FULL_LEVELS]; + pt_element_t ptes[PT_MAX_FULL_LEVELS]; + gpa_t pte_gpa[PT_MAX_FULL_LEVELS]; + unsigned pt_access; + unsigned pte_access; + gfn_t gfn; + u32 error_code; +}; + +static gfn_t gpte_to_gfn(pt_element_t gpte) +{ + return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; +} + +static gfn_t gpte_to_gfn_pde(pt_element_t gpte) +{ + return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; +} + +static bool FNAME(cmpxchg_gpte)(struct kvm *kvm, + gfn_t table_gfn, unsigned index, + pt_element_t orig_pte, pt_element_t new_pte) +{ + pt_element_t ret; + pt_element_t *table; + struct page *page; + + page = gfn_to_page(kvm, table_gfn); + table = kmap_atomic(page, KM_USER0); + + ret = CMPXCHG(&table[index], orig_pte, new_pte); + + kunmap_atomic(table, KM_USER0); + + kvm_release_page_dirty(page); + + return (ret != orig_pte); +} + +static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte) +{ + unsigned access; + + access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK; +#if PTTYPE == 64 + if (is_nx(vcpu)) + access &= ~(gpte >> PT64_NX_SHIFT); +#endif + return access; +} + +/* + * Fetch a guest pte for a guest virtual address + */ +static int FNAME(walk_addr)(struct guest_walker *walker, + struct kvm_vcpu *vcpu, gva_t addr, + int write_fault, int user_fault, int fetch_fault) +{ + pt_element_t pte; + gfn_t table_gfn; + unsigned index, pt_access, pte_access; + gpa_t pte_gpa; + + pgprintk("%s: addr %lx\n", __FUNCTION__, addr); +walk: + walker->level = vcpu->arch.mmu.root_level; + pte = vcpu->arch.cr3; +#if PTTYPE == 64 + if (!is_long_mode(vcpu)) { + pte = vcpu->arch.pdptrs[(addr >> 30) & 3]; + if (!is_present_pte(pte)) + goto not_present; + --walker->level; + } +#endif + ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || + (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0); + + pt_access = ACC_ALL; + + for (;;) { + index = PT_INDEX(addr, walker->level); + + table_gfn = gpte_to_gfn(pte); + pte_gpa = gfn_to_gpa(table_gfn); + pte_gpa += index * sizeof(pt_element_t); + walker->table_gfn[walker->level - 1] = table_gfn; + walker->pte_gpa[walker->level - 1] = pte_gpa; + pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, + walker->level - 1, table_gfn); + + kvm_read_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); + + if (!is_present_pte(pte)) + goto not_present; + + if (write_fault && !is_writeble_pte(pte)) + if (user_fault || is_write_protection(vcpu)) + goto access_error; + + if (user_fault && !(pte & PT_USER_MASK)) + goto access_error; + +#if PTTYPE == 64 + if (fetch_fault && is_nx(vcpu) && (pte & PT64_NX_MASK)) + goto access_error; +#endif + + if (!(pte & PT_ACCESSED_MASK)) { + mark_page_dirty(vcpu->kvm, table_gfn); + if (FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, + index, pte, pte|PT_ACCESSED_MASK)) + goto walk; + pte |= PT_ACCESSED_MASK; + } + + pte_access = pt_access & FNAME(gpte_access)(vcpu, pte); + + walker->ptes[walker->level - 1] = pte; + + if (walker->level == PT_PAGE_TABLE_LEVEL) { + walker->gfn = gpte_to_gfn(pte); + break; + } + + if (walker->level == PT_DIRECTORY_LEVEL + && (pte & PT_PAGE_SIZE_MASK) + && (PTTYPE == 64 || is_pse(vcpu))) { + walker->gfn = gpte_to_gfn_pde(pte); + walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); + if (PTTYPE == 32 && is_cpuid_PSE36()) + walker->gfn += pse36_gfn_delta(pte); + break; + } + + pt_access = pte_access; + --walker->level; + } + + if (write_fault && !is_dirty_pte(pte)) { + bool ret; + + mark_page_dirty(vcpu->kvm, table_gfn); + ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte, + pte|PT_DIRTY_MASK); + if (ret) + goto walk; + pte |= PT_DIRTY_MASK; + kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); + walker->ptes[walker->level - 1] = pte; + } + + walker->pt_access = pt_access; + walker->pte_access = pte_access; + pgprintk("%s: pte %llx pte_access %x pt_access %x\n", + __FUNCTION__, (u64)pte, pt_access, pte_access); + return 1; + +not_present: + walker->error_code = 0; + goto err; + +access_error: + walker->error_code = PFERR_PRESENT_MASK; + +err: + if (write_fault) + walker->error_code |= PFERR_WRITE_MASK; + if (user_fault) + walker->error_code |= PFERR_USER_MASK; + if (fetch_fault) + walker->error_code |= PFERR_FETCH_MASK; + return 0; +} + +static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, + u64 *spte, const void *pte, int bytes, + int offset_in_pte) +{ + pt_element_t gpte; + unsigned pte_access; + + gpte = *(const pt_element_t *)pte; + if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { + if (!offset_in_pte && !is_present_pte(gpte)) + set_shadow_pte(spte, shadow_notrap_nonpresent_pte); + return; + } + if (bytes < sizeof(pt_element_t)) + return; + pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); + pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); + mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0, + gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); +} + +/* + * Fetch a shadow pte for a specific level in the paging hierarchy. + */ +static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, + struct guest_walker *walker, + int user_fault, int write_fault, int *ptwrite) +{ + hpa_t shadow_addr; + int level; + u64 *shadow_ent; + unsigned access = walker->pt_access; + + if (!is_present_pte(walker->ptes[walker->level - 1])) + return NULL; + + shadow_addr = vcpu->arch.mmu.root_hpa; + level = vcpu->arch.mmu.shadow_root_level; + if (level == PT32E_ROOT_LEVEL) { + shadow_addr = vcpu->arch.mmu.pae_root[(addr >> 30) & 3]; + shadow_addr &= PT64_BASE_ADDR_MASK; + --level; + } + + for (; ; level--) { + u32 index = SHADOW_PT_INDEX(addr, level); + struct kvm_mmu_page *shadow_page; + u64 shadow_pte; + int metaphysical; + gfn_t table_gfn; + bool new_page = 0; + + shadow_ent = ((u64 *)__va(shadow_addr)) + index; + if (is_shadow_present_pte(*shadow_ent)) { + if (level == PT_PAGE_TABLE_LEVEL) + break; + shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK; + continue; + } + + if (level == PT_PAGE_TABLE_LEVEL) + break; + + if (level - 1 == PT_PAGE_TABLE_LEVEL + && walker->level == PT_DIRECTORY_LEVEL) { + metaphysical = 1; + if (!is_dirty_pte(walker->ptes[level - 1])) + access &= ~ACC_WRITE_MASK; + table_gfn = gpte_to_gfn(walker->ptes[level - 1]); + } else { + metaphysical = 0; + table_gfn = walker->table_gfn[level - 2]; + } + shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, + metaphysical, access, + shadow_ent, &new_page); + if (new_page && !metaphysical) { + pt_element_t curr_pte; + kvm_read_guest(vcpu->kvm, walker->pte_gpa[level - 2], + &curr_pte, sizeof(curr_pte)); + if (curr_pte != walker->ptes[level - 2]) + return NULL; + } + shadow_addr = __pa(shadow_page->spt); + shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK + | PT_WRITABLE_MASK | PT_USER_MASK; + *shadow_ent = shadow_pte; + } + + mmu_set_spte(vcpu, shadow_ent, access, walker->pte_access & access, + user_fault, write_fault, + walker->ptes[walker->level-1] & PT_DIRTY_MASK, + ptwrite, walker->gfn); + + return shadow_ent; +} + +/* + * Page fault handler. There are several causes for a page fault: + * - there is no shadow pte for the guest pte + * - write access through a shadow pte marked read only so that we can set + * the dirty bit + * - write access to a shadow pte marked read only so we can update the page + * dirty bitmap, when userspace requests it + * - mmio access; in this case we will never install a present shadow pte + * - normal guest page fault due to the guest pte marked not present, not + * writable, or not executable + * + * Returns: 1 if we need to emulate the instruction, 0 otherwise, or + * a negative value on error. + */ +static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, + u32 error_code) +{ + int write_fault = error_code & PFERR_WRITE_MASK; + int user_fault = error_code & PFERR_USER_MASK; + int fetch_fault = error_code & PFERR_FETCH_MASK; + struct guest_walker walker; + u64 *shadow_pte; + int write_pt = 0; + int r; + + pgprintk("%s: addr %lx err %x\n", __FUNCTION__, addr, error_code); + kvm_mmu_audit(vcpu, "pre page fault"); + + r = mmu_topup_memory_caches(vcpu); + if (r) + return r; + + /* + * Look up the shadow pte for the faulting address. + */ + r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault, + fetch_fault); + + /* + * The page is not mapped by the guest. Let the guest handle it. + */ + if (!r) { + pgprintk("%s: guest page fault\n", __FUNCTION__); + inject_page_fault(vcpu, addr, walker.error_code); + vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ + return 0; + } + + shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault, + &write_pt); + pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__, + shadow_pte, *shadow_pte, write_pt); + + if (!write_pt) + vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ + + /* + * mmio: emulate if accessible, otherwise its a guest fault. + */ + if (shadow_pte && is_io_pte(*shadow_pte)) + return 1; + + ++vcpu->stat.pf_fixed; + kvm_mmu_audit(vcpu, "post page fault (fixed)"); + + return write_pt; +} + +static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) +{ + struct guest_walker walker; + gpa_t gpa = UNMAPPED_GVA; + int r; + + r = FNAME(walk_addr)(&walker, vcpu, vaddr, 0, 0, 0); + + if (r) { + gpa = gfn_to_gpa(walker.gfn); + gpa |= vaddr & ~PAGE_MASK; + } + + return gpa; +} + +static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *sp) +{ + int i, offset = 0; + pt_element_t *gpt; + struct page *page; + + if (sp->role.metaphysical + || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) { + nonpaging_prefetch_page(vcpu, sp); + return; + } + + if (PTTYPE == 32) + offset = sp->role.quadrant << PT64_LEVEL_BITS; + page = gfn_to_page(vcpu->kvm, sp->gfn); + gpt = kmap_atomic(page, KM_USER0); + for (i = 0; i < PT64_ENT_PER_PAGE; ++i) + if (is_present_pte(gpt[offset + i])) + sp->spt[i] = shadow_trap_nonpresent_pte; + else + sp->spt[i] = shadow_notrap_nonpresent_pte; + kunmap_atomic(gpt, KM_USER0); + kvm_release_page_clean(page); +} + +#undef pt_element_t +#undef guest_walker +#undef FNAME +#undef PT_BASE_ADDR_MASK +#undef PT_INDEX +#undef SHADOW_PT_INDEX +#undef PT_LEVEL_MASK +#undef PT_DIR_BASE_ADDR_MASK +#undef PT_LEVEL_BITS +#undef PT_MAX_FULL_LEVELS +#undef gpte_to_gfn +#undef gpte_to_gfn_pde +#undef CMPXCHG diff --git a/arch/x86/kvm/segment_descriptor.h b/arch/x86/kvm/segment_descriptor.h new file mode 100644 index 000000000000..56fc4c873389 --- /dev/null +++ b/arch/x86/kvm/segment_descriptor.h @@ -0,0 +1,29 @@ +#ifndef __SEGMENT_DESCRIPTOR_H +#define __SEGMENT_DESCRIPTOR_H + +struct segment_descriptor { + u16 limit_low; + u16 base_low; + u8 base_mid; + u8 type : 4; + u8 system : 1; + u8 dpl : 2; + u8 present : 1; + u8 limit_high : 4; + u8 avl : 1; + u8 long_mode : 1; + u8 default_op : 1; + u8 granularity : 1; + u8 base_high; +} __attribute__((packed)); + +#ifdef CONFIG_X86_64 +/* LDT or TSS descriptor in the GDT. 16 bytes. */ +struct segment_descriptor_64 { + struct segment_descriptor s; + u32 base_higher; + u32 pad_zero; +}; + +#endif +#endif diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c new file mode 100644 index 000000000000..3d4b71a94440 --- /dev/null +++ b/arch/x86/kvm/svm.c @@ -0,0 +1,1725 @@ +/* + * Kernel-based Virtual Machine driver for Linux + * + * AMD SVM support + * + * Copyright (C) 2006 Qumranet, Inc. + * + * Authors: + * Yaniv Kamay + * Avi Kivity + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ +#include + +#include "kvm_svm.h" +#include "irq.h" +#include "mmu.h" + +#include +#include +#include +#include +#include + +#include + +MODULE_AUTHOR("Qumranet"); +MODULE_LICENSE("GPL"); + +#define IOPM_ALLOC_ORDER 2 +#define MSRPM_ALLOC_ORDER 1 + +#define DB_VECTOR 1 +#define UD_VECTOR 6 +#define GP_VECTOR 13 + +#define DR7_GD_MASK (1 << 13) +#define DR6_BD_MASK (1 << 13) + +#define SEG_TYPE_LDT 2 +#define SEG_TYPE_BUSY_TSS16 3 + +#define SVM_FEATURE_NPT (1 << 0) +#define SVM_FEATURE_LBRV (1 << 1) +#define SVM_DEATURE_SVML (1 << 2) + +static void kvm_reput_irq(struct vcpu_svm *svm); + +static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) +{ + return container_of(vcpu, struct vcpu_svm, vcpu); +} + +unsigned long iopm_base; +unsigned long msrpm_base; + +struct kvm_ldttss_desc { + u16 limit0; + u16 base0; + unsigned base1 : 8, type : 5, dpl : 2, p : 1; + unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8; + u32 base3; + u32 zero1; +} __attribute__((packed)); + +struct svm_cpu_data { + int cpu; + + u64 asid_generation; + u32 max_asid; + u32 next_asid; + struct kvm_ldttss_desc *tss_desc; + + struct page *save_area; +}; + +static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); +static uint32_t svm_features; + +struct svm_init_data { + int cpu; + int r; +}; + +static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; + +#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) +#define MSRS_RANGE_SIZE 2048 +#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) + +#define MAX_INST_SIZE 15 + +static inline u32 svm_has(u32 feat) +{ + return svm_features & feat; +} + +static inline u8 pop_irq(struct kvm_vcpu *vcpu) +{ + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); + int irq = word_index * BITS_PER_LONG + bit_index; + + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); + return irq; +} + +static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq) +{ + set_bit(irq, vcpu->arch.irq_pending); + set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary); +} + +static inline void clgi(void) +{ + asm volatile (SVM_CLGI); +} + +static inline void stgi(void) +{ + asm volatile (SVM_STGI); +} + +static inline void invlpga(unsigned long addr, u32 asid) +{ + asm volatile (SVM_INVLPGA :: "a"(addr), "c"(asid)); +} + +static inline unsigned long kvm_read_cr2(void) +{ + unsigned long cr2; + + asm volatile ("mov %%cr2, %0" : "=r" (cr2)); + return cr2; +} + +static inline void kvm_write_cr2(unsigned long val) +{ + asm volatile ("mov %0, %%cr2" :: "r" (val)); +} + +static inline unsigned long read_dr6(void) +{ + unsigned long dr6; + + asm volatile ("mov %%dr6, %0" : "=r" (dr6)); + return dr6; +} + +static inline void write_dr6(unsigned long val) +{ + asm volatile ("mov %0, %%dr6" :: "r" (val)); +} + +static inline unsigned long read_dr7(void) +{ + unsigned long dr7; + + asm volatile ("mov %%dr7, %0" : "=r" (dr7)); + return dr7; +} + +static inline void write_dr7(unsigned long val) +{ + asm volatile ("mov %0, %%dr7" :: "r" (val)); +} + +static inline void force_new_asid(struct kvm_vcpu *vcpu) +{ + to_svm(vcpu)->asid_generation--; +} + +static inline void flush_guest_tlb(struct kvm_vcpu *vcpu) +{ + force_new_asid(vcpu); +} + +static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) +{ + if (!(efer & EFER_LMA)) + efer &= ~EFER_LME; + + to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK; + vcpu->arch.shadow_efer = efer; +} + +static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm->vmcb->control.event_inj = nr + | SVM_EVTINJ_VALID + | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) + | SVM_EVTINJ_TYPE_EXEPT; + svm->vmcb->control.event_inj_err = error_code; +} + +static bool svm_exception_injected(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID); +} + +static int is_external_interrupt(u32 info) +{ + info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; + return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR); +} + +static void skip_emulated_instruction(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (!svm->next_rip) { + printk(KERN_DEBUG "%s: NOP\n", __FUNCTION__); + return; + } + if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE) + printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n", + __FUNCTION__, + svm->vmcb->save.rip, + svm->next_rip); + + vcpu->arch.rip = svm->vmcb->save.rip = svm->next_rip; + svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; + + vcpu->arch.interrupt_window_open = 1; +} + +static int has_svm(void) +{ + uint32_t eax, ebx, ecx, edx; + + if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) { + printk(KERN_INFO "has_svm: not amd\n"); + return 0; + } + + cpuid(0x80000000, &eax, &ebx, &ecx, &edx); + if (eax < SVM_CPUID_FUNC) { + printk(KERN_INFO "has_svm: can't execute cpuid_8000000a\n"); + return 0; + } + + cpuid(0x80000001, &eax, &ebx, &ecx, &edx); + if (!(ecx & (1 << SVM_CPUID_FEATURE_SHIFT))) { + printk(KERN_DEBUG "has_svm: svm not available\n"); + return 0; + } + return 1; +} + +static void svm_hardware_disable(void *garbage) +{ + struct svm_cpu_data *svm_data + = per_cpu(svm_data, raw_smp_processor_id()); + + if (svm_data) { + uint64_t efer; + + wrmsrl(MSR_VM_HSAVE_PA, 0); + rdmsrl(MSR_EFER, efer); + wrmsrl(MSR_EFER, efer & ~MSR_EFER_SVME_MASK); + per_cpu(svm_data, raw_smp_processor_id()) = NULL; + __free_page(svm_data->save_area); + kfree(svm_data); + } +} + +static void svm_hardware_enable(void *garbage) +{ + + struct svm_cpu_data *svm_data; + uint64_t efer; +#ifdef CONFIG_X86_64 + struct desc_ptr gdt_descr; +#else + struct desc_ptr gdt_descr; +#endif + struct desc_struct *gdt; + int me = raw_smp_processor_id(); + + if (!has_svm()) { + printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me); + return; + } + svm_data = per_cpu(svm_data, me); + + if (!svm_data) { + printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n", + me); + return; + } + + svm_data->asid_generation = 1; + svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; + svm_data->next_asid = svm_data->max_asid + 1; + svm_features = cpuid_edx(SVM_CPUID_FUNC); + + asm volatile ("sgdt %0" : "=m"(gdt_descr)); + gdt = (struct desc_struct *)gdt_descr.address; + svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); + + rdmsrl(MSR_EFER, efer); + wrmsrl(MSR_EFER, efer | MSR_EFER_SVME_MASK); + + wrmsrl(MSR_VM_HSAVE_PA, + page_to_pfn(svm_data->save_area) << PAGE_SHIFT); +} + +static int svm_cpu_init(int cpu) +{ + struct svm_cpu_data *svm_data; + int r; + + svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); + if (!svm_data) + return -ENOMEM; + svm_data->cpu = cpu; + svm_data->save_area = alloc_page(GFP_KERNEL); + r = -ENOMEM; + if (!svm_data->save_area) + goto err_1; + + per_cpu(svm_data, cpu) = svm_data; + + return 0; + +err_1: + kfree(svm_data); + return r; + +} + +static void set_msr_interception(u32 *msrpm, unsigned msr, + int read, int write) +{ + int i; + + for (i = 0; i < NUM_MSR_MAPS; i++) { + if (msr >= msrpm_ranges[i] && + msr < msrpm_ranges[i] + MSRS_IN_RANGE) { + u32 msr_offset = (i * MSRS_IN_RANGE + msr - + msrpm_ranges[i]) * 2; + + u32 *base = msrpm + (msr_offset / 32); + u32 msr_shift = msr_offset % 32; + u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1); + *base = (*base & ~(0x3 << msr_shift)) | + (mask << msr_shift); + return; + } + } + BUG(); +} + +static __init int svm_hardware_setup(void) +{ + int cpu; + struct page *iopm_pages; + struct page *msrpm_pages; + void *iopm_va, *msrpm_va; + int r; + + iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER); + + if (!iopm_pages) + return -ENOMEM; + + iopm_va = page_address(iopm_pages); + memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER)); + clear_bit(0x80, iopm_va); /* allow direct access to PC debug port */ + iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; + + + msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); + + r = -ENOMEM; + if (!msrpm_pages) + goto err_1; + + msrpm_va = page_address(msrpm_pages); + memset(msrpm_va, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER)); + msrpm_base = page_to_pfn(msrpm_pages) << PAGE_SHIFT; + +#ifdef CONFIG_X86_64 + set_msr_interception(msrpm_va, MSR_GS_BASE, 1, 1); + set_msr_interception(msrpm_va, MSR_FS_BASE, 1, 1); + set_msr_interception(msrpm_va, MSR_KERNEL_GS_BASE, 1, 1); + set_msr_interception(msrpm_va, MSR_LSTAR, 1, 1); + set_msr_interception(msrpm_va, MSR_CSTAR, 1, 1); + set_msr_interception(msrpm_va, MSR_SYSCALL_MASK, 1, 1); +#endif + set_msr_interception(msrpm_va, MSR_K6_STAR, 1, 1); + set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_CS, 1, 1); + set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_ESP, 1, 1); + set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_EIP, 1, 1); + + for_each_online_cpu(cpu) { + r = svm_cpu_init(cpu); + if (r) + goto err_2; + } + return 0; + +err_2: + __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER); + msrpm_base = 0; +err_1: + __free_pages(iopm_pages, IOPM_ALLOC_ORDER); + iopm_base = 0; + return r; +} + +static __exit void svm_hardware_unsetup(void) +{ + __free_pages(pfn_to_page(msrpm_base >> PAGE_SHIFT), MSRPM_ALLOC_ORDER); + __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER); + iopm_base = msrpm_base = 0; +} + +static void init_seg(struct vmcb_seg *seg) +{ + seg->selector = 0; + seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | + SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ + seg->limit = 0xffff; + seg->base = 0; +} + +static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) +{ + seg->selector = 0; + seg->attrib = SVM_SELECTOR_P_MASK | type; + seg->limit = 0xffff; + seg->base = 0; +} + +static void init_vmcb(struct vmcb *vmcb) +{ + struct vmcb_control_area *control = &vmcb->control; + struct vmcb_save_area *save = &vmcb->save; + + control->intercept_cr_read = INTERCEPT_CR0_MASK | + INTERCEPT_CR3_MASK | + INTERCEPT_CR4_MASK | + INTERCEPT_CR8_MASK; + + control->intercept_cr_write = INTERCEPT_CR0_MASK | + INTERCEPT_CR3_MASK | + INTERCEPT_CR4_MASK | + INTERCEPT_CR8_MASK; + + control->intercept_dr_read = INTERCEPT_DR0_MASK | + INTERCEPT_DR1_MASK | + INTERCEPT_DR2_MASK | + INTERCEPT_DR3_MASK; + + control->intercept_dr_write = INTERCEPT_DR0_MASK | + INTERCEPT_DR1_MASK | + INTERCEPT_DR2_MASK | + INTERCEPT_DR3_MASK | + INTERCEPT_DR5_MASK | + INTERCEPT_DR7_MASK; + + control->intercept_exceptions = (1 << PF_VECTOR) | + (1 << UD_VECTOR); + + + control->intercept = (1ULL << INTERCEPT_INTR) | + (1ULL << INTERCEPT_NMI) | + (1ULL << INTERCEPT_SMI) | + /* + * selective cr0 intercept bug? + * 0: 0f 22 d8 mov %eax,%cr3 + * 3: 0f 20 c0 mov %cr0,%eax + * 6: 0d 00 00 00 80 or $0x80000000,%eax + * b: 0f 22 c0 mov %eax,%cr0 + * set cr3 ->interception + * get cr0 ->interception + * set cr0 -> no interception + */ + /* (1ULL << INTERCEPT_SELECTIVE_CR0) | */ + (1ULL << INTERCEPT_CPUID) | + (1ULL << INTERCEPT_INVD) | + (1ULL << INTERCEPT_HLT) | + (1ULL << INTERCEPT_INVLPGA) | + (1ULL << INTERCEPT_IOIO_PROT) | + (1ULL << INTERCEPT_MSR_PROT) | + (1ULL << INTERCEPT_TASK_SWITCH) | + (1ULL << INTERCEPT_SHUTDOWN) | + (1ULL << INTERCEPT_VMRUN) | + (1ULL << INTERCEPT_VMMCALL) | + (1ULL << INTERCEPT_VMLOAD) | + (1ULL << INTERCEPT_VMSAVE) | + (1ULL << INTERCEPT_STGI) | + (1ULL << INTERCEPT_CLGI) | + (1ULL << INTERCEPT_SKINIT) | + (1ULL << INTERCEPT_WBINVD) | + (1ULL << INTERCEPT_MONITOR) | + (1ULL << INTERCEPT_MWAIT); + + control->iopm_base_pa = iopm_base; + control->msrpm_base_pa = msrpm_base; + control->tsc_offset = 0; + control->int_ctl = V_INTR_MASKING_MASK; + + init_seg(&save->es); + init_seg(&save->ss); + init_seg(&save->ds); + init_seg(&save->fs); + init_seg(&save->gs); + + save->cs.selector = 0xf000; + /* Executable/Readable Code Segment */ + save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | + SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; + save->cs.limit = 0xffff; + /* + * cs.base should really be 0xffff0000, but vmx can't handle that, so + * be consistent with it. + * + * Replace when we have real mode working for vmx. + */ + save->cs.base = 0xf0000; + + save->gdtr.limit = 0xffff; + save->idtr.limit = 0xffff; + + init_sys_seg(&save->ldtr, SEG_TYPE_LDT); + init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); + + save->efer = MSR_EFER_SVME_MASK; + save->dr6 = 0xffff0ff0; + save->dr7 = 0x400; + save->rflags = 2; + save->rip = 0x0000fff0; + + /* + * cr0 val on cpu init should be 0x60000010, we enable cpu + * cache by default. the orderly way is to enable cache in bios. + */ + save->cr0 = 0x00000010 | X86_CR0_PG | X86_CR0_WP; + save->cr4 = X86_CR4_PAE; + /* rdx = ?? */ +} + +static int svm_vcpu_reset(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + init_vmcb(svm->vmcb); + + if (vcpu->vcpu_id != 0) { + svm->vmcb->save.rip = 0; + svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12; + svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8; + } + + return 0; +} + +static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) +{ + struct vcpu_svm *svm; + struct page *page; + int err; + + svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); + if (!svm) { + err = -ENOMEM; + goto out; + } + + err = kvm_vcpu_init(&svm->vcpu, kvm, id); + if (err) + goto free_svm; + + page = alloc_page(GFP_KERNEL); + if (!page) { + err = -ENOMEM; + goto uninit; + } + + svm->vmcb = page_address(page); + clear_page(svm->vmcb); + svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT; + svm->asid_generation = 0; + memset(svm->db_regs, 0, sizeof(svm->db_regs)); + init_vmcb(svm->vmcb); + + fx_init(&svm->vcpu); + svm->vcpu.fpu_active = 1; + svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; + if (svm->vcpu.vcpu_id == 0) + svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP; + + return &svm->vcpu; + +uninit: + kvm_vcpu_uninit(&svm->vcpu); +free_svm: + kmem_cache_free(kvm_vcpu_cache, svm); +out: + return ERR_PTR(err); +} + +static void svm_free_vcpu(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT)); + kvm_vcpu_uninit(vcpu); + kmem_cache_free(kvm_vcpu_cache, svm); +} + +static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + int i; + + if (unlikely(cpu != vcpu->cpu)) { + u64 tsc_this, delta; + + /* + * Make sure that the guest sees a monotonically + * increasing TSC. + */ + rdtscll(tsc_this); + delta = vcpu->arch.host_tsc - tsc_this; + svm->vmcb->control.tsc_offset += delta; + vcpu->cpu = cpu; + kvm_migrate_apic_timer(vcpu); + } + + for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) + rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); +} + +static void svm_vcpu_put(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + int i; + + ++vcpu->stat.host_state_reload; + for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) + wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); + + rdtscll(vcpu->arch.host_tsc); +} + +static void svm_vcpu_decache(struct kvm_vcpu *vcpu) +{ +} + +static void svm_cache_regs(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; + vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; + vcpu->arch.rip = svm->vmcb->save.rip; +} + +static void svm_decache_regs(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; + svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; + svm->vmcb->save.rip = vcpu->arch.rip; +} + +static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) +{ + return to_svm(vcpu)->vmcb->save.rflags; +} + +static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) +{ + to_svm(vcpu)->vmcb->save.rflags = rflags; +} + +static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) +{ + struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; + + switch (seg) { + case VCPU_SREG_CS: return &save->cs; + case VCPU_SREG_DS: return &save->ds; + case VCPU_SREG_ES: return &save->es; + case VCPU_SREG_FS: return &save->fs; + case VCPU_SREG_GS: return &save->gs; + case VCPU_SREG_SS: return &save->ss; + case VCPU_SREG_TR: return &save->tr; + case VCPU_SREG_LDTR: return &save->ldtr; + } + BUG(); + return NULL; +} + +static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) +{ + struct vmcb_seg *s = svm_seg(vcpu, seg); + + return s->base; +} + +static void svm_get_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + struct vmcb_seg *s = svm_seg(vcpu, seg); + + var->base = s->base; + var->limit = s->limit; + var->selector = s->selector; + var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; + var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; + var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; + var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; + var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; + var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; + var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; + var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1; + var->unusable = !var->present; +} + +static void svm_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + dt->limit = svm->vmcb->save.idtr.limit; + dt->base = svm->vmcb->save.idtr.base; +} + +static void svm_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm->vmcb->save.idtr.limit = dt->limit; + svm->vmcb->save.idtr.base = dt->base ; +} + +static void svm_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + dt->limit = svm->vmcb->save.gdtr.limit; + dt->base = svm->vmcb->save.gdtr.base; +} + +static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm->vmcb->save.gdtr.limit = dt->limit; + svm->vmcb->save.gdtr.base = dt->base ; +} + +static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) +{ +} + +static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ + struct vcpu_svm *svm = to_svm(vcpu); + +#ifdef CONFIG_X86_64 + if (vcpu->arch.shadow_efer & EFER_LME) { + if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { + vcpu->arch.shadow_efer |= EFER_LMA; + svm->vmcb->save.efer |= EFER_LMA | EFER_LME; + } + + if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { + vcpu->arch.shadow_efer &= ~EFER_LMA; + svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); + } + } +#endif + if ((vcpu->arch.cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) { + svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); + vcpu->fpu_active = 1; + } + + vcpu->arch.cr0 = cr0; + cr0 |= X86_CR0_PG | X86_CR0_WP; + cr0 &= ~(X86_CR0_CD | X86_CR0_NW); + svm->vmcb->save.cr0 = cr0; +} + +static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + vcpu->arch.cr4 = cr4; + to_svm(vcpu)->vmcb->save.cr4 = cr4 | X86_CR4_PAE; +} + +static void svm_set_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_seg *s = svm_seg(vcpu, seg); + + s->base = var->base; + s->limit = var->limit; + s->selector = var->selector; + if (var->unusable) + s->attrib = 0; + else { + s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); + s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; + s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; + s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT; + s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; + s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; + s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; + s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; + } + if (seg == VCPU_SREG_CS) + svm->vmcb->save.cpl + = (svm->vmcb->save.cs.attrib + >> SVM_SELECTOR_DPL_SHIFT) & 3; + +} + +/* FIXME: + + svm(vcpu)->vmcb->control.int_ctl &= ~V_TPR_MASK; + svm(vcpu)->vmcb->control.int_ctl |= (sregs->cr8 & V_TPR_MASK); + +*/ + +static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) +{ + return -EOPNOTSUPP; +} + +static int svm_get_irq(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + u32 exit_int_info = svm->vmcb->control.exit_int_info; + + if (is_external_interrupt(exit_int_info)) + return exit_int_info & SVM_EVTINJ_VEC_MASK; + return -1; +} + +static void load_host_msrs(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); +#endif +} + +static void save_host_msrs(struct kvm_vcpu *vcpu) +{ +#ifdef CONFIG_X86_64 + rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); +#endif +} + +static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data) +{ + if (svm_data->next_asid > svm_data->max_asid) { + ++svm_data->asid_generation; + svm_data->next_asid = 1; + svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; + } + + svm->vcpu.cpu = svm_data->cpu; + svm->asid_generation = svm_data->asid_generation; + svm->vmcb->control.asid = svm_data->next_asid++; +} + +static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr) +{ + return to_svm(vcpu)->db_regs[dr]; +} + +static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value, + int *exception) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + *exception = 0; + + if (svm->vmcb->save.dr7 & DR7_GD_MASK) { + svm->vmcb->save.dr7 &= ~DR7_GD_MASK; + svm->vmcb->save.dr6 |= DR6_BD_MASK; + *exception = DB_VECTOR; + return; + } + + switch (dr) { + case 0 ... 3: + svm->db_regs[dr] = value; + return; + case 4 ... 5: + if (vcpu->arch.cr4 & X86_CR4_DE) { + *exception = UD_VECTOR; + return; + } + case 7: { + if (value & ~((1ULL << 32) - 1)) { + *exception = GP_VECTOR; + return; + } + svm->vmcb->save.dr7 = value; + return; + } + default: + printk(KERN_DEBUG "%s: unexpected dr %u\n", + __FUNCTION__, dr); + *exception = UD_VECTOR; + return; + } +} + +static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + u32 exit_int_info = svm->vmcb->control.exit_int_info; + struct kvm *kvm = svm->vcpu.kvm; + u64 fault_address; + u32 error_code; + + if (!irqchip_in_kernel(kvm) && + is_external_interrupt(exit_int_info)) + push_irq(&svm->vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK); + + fault_address = svm->vmcb->control.exit_info_2; + error_code = svm->vmcb->control.exit_info_1; + return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); +} + +static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + int er; + + er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0); + if (er != EMULATE_DONE) + kvm_queue_exception(&svm->vcpu, UD_VECTOR); + return 1; +} + +static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); + if (!(svm->vcpu.arch.cr0 & X86_CR0_TS)) + svm->vmcb->save.cr0 &= ~X86_CR0_TS; + svm->vcpu.fpu_active = 1; + + return 1; +} + +static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + /* + * VMCB is undefined after a SHUTDOWN intercept + * so reinitialize it. + */ + clear_page(svm->vmcb); + init_vmcb(svm->vmcb); + + kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; + return 0; +} + +static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ + int size, down, in, string, rep; + unsigned port; + + ++svm->vcpu.stat.io_exits; + + svm->next_rip = svm->vmcb->control.exit_info_2; + + string = (io_info & SVM_IOIO_STR_MASK) != 0; + + if (string) { + if (emulate_instruction(&svm->vcpu, + kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) + return 0; + return 1; + } + + in = (io_info & SVM_IOIO_TYPE_MASK) != 0; + port = io_info >> 16; + size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; + rep = (io_info & SVM_IOIO_REP_MASK) != 0; + down = (svm->vmcb->save.rflags & X86_EFLAGS_DF) != 0; + + return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port); +} + +static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + return 1; +} + +static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + svm->next_rip = svm->vmcb->save.rip + 1; + skip_emulated_instruction(&svm->vcpu); + return kvm_emulate_halt(&svm->vcpu); +} + +static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + svm->next_rip = svm->vmcb->save.rip + 3; + skip_emulated_instruction(&svm->vcpu); + kvm_emulate_hypercall(&svm->vcpu); + return 1; +} + +static int invalid_op_interception(struct vcpu_svm *svm, + struct kvm_run *kvm_run) +{ + kvm_queue_exception(&svm->vcpu, UD_VECTOR); + return 1; +} + +static int task_switch_interception(struct vcpu_svm *svm, + struct kvm_run *kvm_run) +{ + pr_unimpl(&svm->vcpu, "%s: task switch is unsupported\n", __FUNCTION__); + kvm_run->exit_reason = KVM_EXIT_UNKNOWN; + return 0; +} + +static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + svm->next_rip = svm->vmcb->save.rip + 2; + kvm_emulate_cpuid(&svm->vcpu); + return 1; +} + +static int emulate_on_interception(struct vcpu_svm *svm, + struct kvm_run *kvm_run) +{ + if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE) + pr_unimpl(&svm->vcpu, "%s: failed\n", __FUNCTION__); + return 1; +} + +static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + emulate_instruction(&svm->vcpu, NULL, 0, 0, 0); + if (irqchip_in_kernel(svm->vcpu.kvm)) + return 1; + kvm_run->exit_reason = KVM_EXIT_SET_TPR; + return 0; +} + +static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + switch (ecx) { + case MSR_IA32_TIME_STAMP_COUNTER: { + u64 tsc; + + rdtscll(tsc); + *data = svm->vmcb->control.tsc_offset + tsc; + break; + } + case MSR_K6_STAR: + *data = svm->vmcb->save.star; + break; +#ifdef CONFIG_X86_64 + case MSR_LSTAR: + *data = svm->vmcb->save.lstar; + break; + case MSR_CSTAR: + *data = svm->vmcb->save.cstar; + break; + case MSR_KERNEL_GS_BASE: + *data = svm->vmcb->save.kernel_gs_base; + break; + case MSR_SYSCALL_MASK: + *data = svm->vmcb->save.sfmask; + break; +#endif + case MSR_IA32_SYSENTER_CS: + *data = svm->vmcb->save.sysenter_cs; + break; + case MSR_IA32_SYSENTER_EIP: + *data = svm->vmcb->save.sysenter_eip; + break; + case MSR_IA32_SYSENTER_ESP: + *data = svm->vmcb->save.sysenter_esp; + break; + default: + return kvm_get_msr_common(vcpu, ecx, data); + } + return 0; +} + +static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; + u64 data; + + if (svm_get_msr(&svm->vcpu, ecx, &data)) + kvm_inject_gp(&svm->vcpu, 0); + else { + svm->vmcb->save.rax = data & 0xffffffff; + svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32; + svm->next_rip = svm->vmcb->save.rip + 2; + skip_emulated_instruction(&svm->vcpu); + } + return 1; +} + +static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + switch (ecx) { + case MSR_IA32_TIME_STAMP_COUNTER: { + u64 tsc; + + rdtscll(tsc); + svm->vmcb->control.tsc_offset = data - tsc; + break; + } + case MSR_K6_STAR: + svm->vmcb->save.star = data; + break; +#ifdef CONFIG_X86_64 + case MSR_LSTAR: + svm->vmcb->save.lstar = data; + break; + case MSR_CSTAR: + svm->vmcb->save.cstar = data; + break; + case MSR_KERNEL_GS_BASE: + svm->vmcb->save.kernel_gs_base = data; + break; + case MSR_SYSCALL_MASK: + svm->vmcb->save.sfmask = data; + break; +#endif + case MSR_IA32_SYSENTER_CS: + svm->vmcb->save.sysenter_cs = data; + break; + case MSR_IA32_SYSENTER_EIP: + svm->vmcb->save.sysenter_eip = data; + break; + case MSR_IA32_SYSENTER_ESP: + svm->vmcb->save.sysenter_esp = data; + break; + case MSR_K7_EVNTSEL0: + case MSR_K7_EVNTSEL1: + case MSR_K7_EVNTSEL2: + case MSR_K7_EVNTSEL3: + /* + * only support writing 0 to the performance counters for now + * to make Windows happy. Should be replaced by a real + * performance counter emulation later. + */ + if (data != 0) + goto unhandled; + break; + default: + unhandled: + return kvm_set_msr_common(vcpu, ecx, data); + } + return 0; +} + +static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; + u64 data = (svm->vmcb->save.rax & -1u) + | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32); + svm->next_rip = svm->vmcb->save.rip + 2; + if (svm_set_msr(&svm->vcpu, ecx, data)) + kvm_inject_gp(&svm->vcpu, 0); + else + skip_emulated_instruction(&svm->vcpu); + return 1; +} + +static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) +{ + if (svm->vmcb->control.exit_info_1) + return wrmsr_interception(svm, kvm_run); + else + return rdmsr_interception(svm, kvm_run); +} + +static int interrupt_window_interception(struct vcpu_svm *svm, + struct kvm_run *kvm_run) +{ + svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR); + svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; + /* + * If the user space waits to inject interrupts, exit as soon as + * possible + */ + if (kvm_run->request_interrupt_window && + !svm->vcpu.arch.irq_summary) { + ++svm->vcpu.stat.irq_window_exits; + kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; + return 0; + } + + return 1; +} + +static int (*svm_exit_handlers[])(struct vcpu_svm *svm, + struct kvm_run *kvm_run) = { + [SVM_EXIT_READ_CR0] = emulate_on_interception, + [SVM_EXIT_READ_CR3] = emulate_on_interception, + [SVM_EXIT_READ_CR4] = emulate_on_interception, + [SVM_EXIT_READ_CR8] = emulate_on_interception, + /* for now: */ + [SVM_EXIT_WRITE_CR0] = emulate_on_interception, + [SVM_EXIT_WRITE_CR3] = emulate_on_interception, + [SVM_EXIT_WRITE_CR4] = emulate_on_interception, + [SVM_EXIT_WRITE_CR8] = cr8_write_interception, + [SVM_EXIT_READ_DR0] = emulate_on_interception, + [SVM_EXIT_READ_DR1] = emulate_on_interception, + [SVM_EXIT_READ_DR2] = emulate_on_interception, + [SVM_EXIT_READ_DR3] = emulate_on_interception, + [SVM_EXIT_WRITE_DR0] = emulate_on_interception, + [SVM_EXIT_WRITE_DR1] = emulate_on_interception, + [SVM_EXIT_WRITE_DR2] = emulate_on_interception, + [SVM_EXIT_WRITE_DR3] = emulate_on_interception, + [SVM_EXIT_WRITE_DR5] = emulate_on_interception, + [SVM_EXIT_WRITE_DR7] = emulate_on_interception, + [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, + [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, + [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception, + [SVM_EXIT_INTR] = nop_on_interception, + [SVM_EXIT_NMI] = nop_on_interception, + [SVM_EXIT_SMI] = nop_on_interception, + [SVM_EXIT_INIT] = nop_on_interception, + [SVM_EXIT_VINTR] = interrupt_window_interception, + /* [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, */ + [SVM_EXIT_CPUID] = cpuid_interception, + [SVM_EXIT_INVD] = emulate_on_interception, + [SVM_EXIT_HLT] = halt_interception, + [SVM_EXIT_INVLPG] = emulate_on_interception, + [SVM_EXIT_INVLPGA] = invalid_op_interception, + [SVM_EXIT_IOIO] = io_interception, + [SVM_EXIT_MSR] = msr_interception, + [SVM_EXIT_TASK_SWITCH] = task_switch_interception, + [SVM_EXIT_SHUTDOWN] = shutdown_interception, + [SVM_EXIT_VMRUN] = invalid_op_interception, + [SVM_EXIT_VMMCALL] = vmmcall_interception, + [SVM_EXIT_VMLOAD] = invalid_op_interception, + [SVM_EXIT_VMSAVE] = invalid_op_interception, + [SVM_EXIT_STGI] = invalid_op_interception, + [SVM_EXIT_CLGI] = invalid_op_interception, + [SVM_EXIT_SKINIT] = invalid_op_interception, + [SVM_EXIT_WBINVD] = emulate_on_interception, + [SVM_EXIT_MONITOR] = invalid_op_interception, + [SVM_EXIT_MWAIT] = invalid_op_interception, +}; + + +static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + u32 exit_code = svm->vmcb->control.exit_code; + + kvm_reput_irq(svm); + + if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { + kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; + kvm_run->fail_entry.hardware_entry_failure_reason + = svm->vmcb->control.exit_code; + return 0; + } + + if (is_external_interrupt(svm->vmcb->control.exit_int_info) && + exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR) + printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x " + "exit_code 0x%x\n", + __FUNCTION__, svm->vmcb->control.exit_int_info, + exit_code); + + if (exit_code >= ARRAY_SIZE(svm_exit_handlers) + || !svm_exit_handlers[exit_code]) { + kvm_run->exit_reason = KVM_EXIT_UNKNOWN; + kvm_run->hw.hardware_exit_reason = exit_code; + return 0; + } + + return svm_exit_handlers[exit_code](svm, kvm_run); +} + +static void reload_tss(struct kvm_vcpu *vcpu) +{ + int cpu = raw_smp_processor_id(); + + struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu); + svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */ + load_TR_desc(); +} + +static void pre_svm_run(struct vcpu_svm *svm) +{ + int cpu = raw_smp_processor_id(); + + struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu); + + svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; + if (svm->vcpu.cpu != cpu || + svm->asid_generation != svm_data->asid_generation) + new_asid(svm, svm_data); +} + + +static inline void svm_inject_irq(struct vcpu_svm *svm, int irq) +{ + struct vmcb_control_area *control; + + control = &svm->vmcb->control; + control->int_vector = irq; + control->int_ctl &= ~V_INTR_PRIO_MASK; + control->int_ctl |= V_IRQ_MASK | + ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); +} + +static void svm_set_irq(struct kvm_vcpu *vcpu, int irq) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm_inject_irq(svm, irq); +} + +static void svm_intr_assist(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb = svm->vmcb; + int intr_vector = -1; + + if ((vmcb->control.exit_int_info & SVM_EVTINJ_VALID) && + ((vmcb->control.exit_int_info & SVM_EVTINJ_TYPE_MASK) == 0)) { + intr_vector = vmcb->control.exit_int_info & + SVM_EVTINJ_VEC_MASK; + vmcb->control.exit_int_info = 0; + svm_inject_irq(svm, intr_vector); + return; + } + + if (vmcb->control.int_ctl & V_IRQ_MASK) + return; + + if (!kvm_cpu_has_interrupt(vcpu)) + return; + + if (!(vmcb->save.rflags & X86_EFLAGS_IF) || + (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || + (vmcb->control.event_inj & SVM_EVTINJ_VALID)) { + /* unable to deliver irq, set pending irq */ + vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR); + svm_inject_irq(svm, 0x0); + return; + } + /* Okay, we can deliver the interrupt: grab it and update PIC state. */ + intr_vector = kvm_cpu_get_interrupt(vcpu); + svm_inject_irq(svm, intr_vector); + kvm_timer_intr_post(vcpu, intr_vector); +} + +static void kvm_reput_irq(struct vcpu_svm *svm) +{ + struct vmcb_control_area *control = &svm->vmcb->control; + + if ((control->int_ctl & V_IRQ_MASK) + && !irqchip_in_kernel(svm->vcpu.kvm)) { + control->int_ctl &= ~V_IRQ_MASK; + push_irq(&svm->vcpu, control->int_vector); + } + + svm->vcpu.arch.interrupt_window_open = + !(control->int_state & SVM_INTERRUPT_SHADOW_MASK); +} + +static void svm_do_inject_vector(struct vcpu_svm *svm) +{ + struct kvm_vcpu *vcpu = &svm->vcpu; + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); + int irq = word_index * BITS_PER_LONG + bit_index; + + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); + svm_inject_irq(svm, irq); +} + +static void do_interrupt_requests(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_control_area *control = &svm->vmcb->control; + + svm->vcpu.arch.interrupt_window_open = + (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) && + (svm->vmcb->save.rflags & X86_EFLAGS_IF)); + + if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary) + /* + * If interrupts enabled, and not blocked by sti or mov ss. Good. + */ + svm_do_inject_vector(svm); + + /* + * Interrupts blocked. Wait for unblock. + */ + if (!svm->vcpu.arch.interrupt_window_open && + (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window)) + control->intercept |= 1ULL << INTERCEPT_VINTR; + else + control->intercept &= ~(1ULL << INTERCEPT_VINTR); +} + +static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) +{ + return 0; +} + +static void save_db_regs(unsigned long *db_regs) +{ + asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0])); + asm volatile ("mov %%dr1, %0" : "=r"(db_regs[1])); + asm volatile ("mov %%dr2, %0" : "=r"(db_regs[2])); + asm volatile ("mov %%dr3, %0" : "=r"(db_regs[3])); +} + +static void load_db_regs(unsigned long *db_regs) +{ + asm volatile ("mov %0, %%dr0" : : "r"(db_regs[0])); + asm volatile ("mov %0, %%dr1" : : "r"(db_regs[1])); + asm volatile ("mov %0, %%dr2" : : "r"(db_regs[2])); + asm volatile ("mov %0, %%dr3" : : "r"(db_regs[3])); +} + +static void svm_flush_tlb(struct kvm_vcpu *vcpu) +{ + force_new_asid(vcpu); +} + +static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) +{ +} + +static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + struct vcpu_svm *svm = to_svm(vcpu); + u16 fs_selector; + u16 gs_selector; + u16 ldt_selector; + + pre_svm_run(svm); + + save_host_msrs(vcpu); + fs_selector = read_fs(); + gs_selector = read_gs(); + ldt_selector = read_ldt(); + svm->host_cr2 = kvm_read_cr2(); + svm->host_dr6 = read_dr6(); + svm->host_dr7 = read_dr7(); + svm->vmcb->save.cr2 = vcpu->arch.cr2; + + if (svm->vmcb->save.dr7 & 0xff) { + write_dr7(0); + save_db_regs(svm->host_db_regs); + load_db_regs(svm->db_regs); + } + + clgi(); + + local_irq_enable(); + + asm volatile ( +#ifdef CONFIG_X86_64 + "push %%rbp; \n\t" +#else + "push %%ebp; \n\t" +#endif + +#ifdef CONFIG_X86_64 + "mov %c[rbx](%[svm]), %%rbx \n\t" + "mov %c[rcx](%[svm]), %%rcx \n\t" + "mov %c[rdx](%[svm]), %%rdx \n\t" + "mov %c[rsi](%[svm]), %%rsi \n\t" + "mov %c[rdi](%[svm]), %%rdi \n\t" + "mov %c[rbp](%[svm]), %%rbp \n\t" + "mov %c[r8](%[svm]), %%r8 \n\t" + "mov %c[r9](%[svm]), %%r9 \n\t" + "mov %c[r10](%[svm]), %%r10 \n\t" + "mov %c[r11](%[svm]), %%r11 \n\t" + "mov %c[r12](%[svm]), %%r12 \n\t" + "mov %c[r13](%[svm]), %%r13 \n\t" + "mov %c[r14](%[svm]), %%r14 \n\t" + "mov %c[r15](%[svm]), %%r15 \n\t" +#else + "mov %c[rbx](%[svm]), %%ebx \n\t" + "mov %c[rcx](%[svm]), %%ecx \n\t" + "mov %c[rdx](%[svm]), %%edx \n\t" + "mov %c[rsi](%[svm]), %%esi \n\t" + "mov %c[rdi](%[svm]), %%edi \n\t" + "mov %c[rbp](%[svm]), %%ebp \n\t" +#endif + +#ifdef CONFIG_X86_64 + /* Enter guest mode */ + "push %%rax \n\t" + "mov %c[vmcb](%[svm]), %%rax \n\t" + SVM_VMLOAD "\n\t" + SVM_VMRUN "\n\t" + SVM_VMSAVE "\n\t" + "pop %%rax \n\t" +#else + /* Enter guest mode */ + "push %%eax \n\t" + "mov %c[vmcb](%[svm]), %%eax \n\t" + SVM_VMLOAD "\n\t" + SVM_VMRUN "\n\t" + SVM_VMSAVE "\n\t" + "pop %%eax \n\t" +#endif + + /* Save guest registers, load host registers */ +#ifdef CONFIG_X86_64 + "mov %%rbx, %c[rbx](%[svm]) \n\t" + "mov %%rcx, %c[rcx](%[svm]) \n\t" + "mov %%rdx, %c[rdx](%[svm]) \n\t" + "mov %%rsi, %c[rsi](%[svm]) \n\t" + "mov %%rdi, %c[rdi](%[svm]) \n\t" + "mov %%rbp, %c[rbp](%[svm]) \n\t" + "mov %%r8, %c[r8](%[svm]) \n\t" + "mov %%r9, %c[r9](%[svm]) \n\t" + "mov %%r10, %c[r10](%[svm]) \n\t" + "mov %%r11, %c[r11](%[svm]) \n\t" + "mov %%r12, %c[r12](%[svm]) \n\t" + "mov %%r13, %c[r13](%[svm]) \n\t" + "mov %%r14, %c[r14](%[svm]) \n\t" + "mov %%r15, %c[r15](%[svm]) \n\t" + + "pop %%rbp; \n\t" +#else + "mov %%ebx, %c[rbx](%[svm]) \n\t" + "mov %%ecx, %c[rcx](%[svm]) \n\t" + "mov %%edx, %c[rdx](%[svm]) \n\t" + "mov %%esi, %c[rsi](%[svm]) \n\t" + "mov %%edi, %c[rdi](%[svm]) \n\t" + "mov %%ebp, %c[rbp](%[svm]) \n\t" + + "pop %%ebp; \n\t" +#endif + : + : [svm]"a"(svm), + [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), + [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP])) +#ifdef CONFIG_X86_64 + , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15])) +#endif + : "cc", "memory" +#ifdef CONFIG_X86_64 + , "rbx", "rcx", "rdx", "rsi", "rdi" + , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" +#else + , "ebx", "ecx", "edx" , "esi", "edi" +#endif + ); + + if ((svm->vmcb->save.dr7 & 0xff)) + load_db_regs(svm->host_db_regs); + + vcpu->arch.cr2 = svm->vmcb->save.cr2; + + write_dr6(svm->host_dr6); + write_dr7(svm->host_dr7); + kvm_write_cr2(svm->host_cr2); + + load_fs(fs_selector); + load_gs(gs_selector); + load_ldt(ldt_selector); + load_host_msrs(vcpu); + + reload_tss(vcpu); + + local_irq_disable(); + + stgi(); + + svm->next_rip = 0; +} + +static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm->vmcb->save.cr3 = root; + force_new_asid(vcpu); + + if (vcpu->fpu_active) { + svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR); + svm->vmcb->save.cr0 |= X86_CR0_TS; + vcpu->fpu_active = 0; + } +} + +static int is_disabled(void) +{ + u64 vm_cr; + + rdmsrl(MSR_VM_CR, vm_cr); + if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) + return 1; + + return 0; +} + +static void +svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) +{ + /* + * Patch in the VMMCALL instruction: + */ + hypercall[0] = 0x0f; + hypercall[1] = 0x01; + hypercall[2] = 0xd9; +} + +static void svm_check_processor_compat(void *rtn) +{ + *(int *)rtn = 0; +} + +static struct kvm_x86_ops svm_x86_ops = { + .cpu_has_kvm_support = has_svm, + .disabled_by_bios = is_disabled, + .hardware_setup = svm_hardware_setup, + .hardware_unsetup = svm_hardware_unsetup, + .check_processor_compatibility = svm_check_processor_compat, + .hardware_enable = svm_hardware_enable, + .hardware_disable = svm_hardware_disable, + + .vcpu_create = svm_create_vcpu, + .vcpu_free = svm_free_vcpu, + .vcpu_reset = svm_vcpu_reset, + + .prepare_guest_switch = svm_prepare_guest_switch, + .vcpu_load = svm_vcpu_load, + .vcpu_put = svm_vcpu_put, + .vcpu_decache = svm_vcpu_decache, + + .set_guest_debug = svm_guest_debug, + .get_msr = svm_get_msr, + .set_msr = svm_set_msr, + .get_segment_base = svm_get_segment_base, + .get_segment = svm_get_segment, + .set_segment = svm_set_segment, + .get_cs_db_l_bits = kvm_get_cs_db_l_bits, + .decache_cr4_guest_bits = svm_decache_cr4_guest_bits, + .set_cr0 = svm_set_cr0, + .set_cr3 = svm_set_cr3, + .set_cr4 = svm_set_cr4, + .set_efer = svm_set_efer, + .get_idt = svm_get_idt, + .set_idt = svm_set_idt, + .get_gdt = svm_get_gdt, + .set_gdt = svm_set_gdt, + .get_dr = svm_get_dr, + .set_dr = svm_set_dr, + .cache_regs = svm_cache_regs, + .decache_regs = svm_decache_regs, + .get_rflags = svm_get_rflags, + .set_rflags = svm_set_rflags, + + .tlb_flush = svm_flush_tlb, + + .run = svm_vcpu_run, + .handle_exit = handle_exit, + .skip_emulated_instruction = skip_emulated_instruction, + .patch_hypercall = svm_patch_hypercall, + .get_irq = svm_get_irq, + .set_irq = svm_set_irq, + .queue_exception = svm_queue_exception, + .exception_injected = svm_exception_injected, + .inject_pending_irq = svm_intr_assist, + .inject_pending_vectors = do_interrupt_requests, + + .set_tss_addr = svm_set_tss_addr, +}; + +static int __init svm_init(void) +{ + return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm), + THIS_MODULE); +} + +static void __exit svm_exit(void) +{ + kvm_exit(); +} + +module_init(svm_init) +module_exit(svm_exit) diff --git a/arch/x86/kvm/svm.h b/arch/x86/kvm/svm.h new file mode 100644 index 000000000000..5fd50491b555 --- /dev/null +++ b/arch/x86/kvm/svm.h @@ -0,0 +1,325 @@ +#ifndef __SVM_H +#define __SVM_H + +enum { + INTERCEPT_INTR, + INTERCEPT_NMI, + INTERCEPT_SMI, + INTERCEPT_INIT, + INTERCEPT_VINTR, + INTERCEPT_SELECTIVE_CR0, + INTERCEPT_STORE_IDTR, + INTERCEPT_STORE_GDTR, + INTERCEPT_STORE_LDTR, + INTERCEPT_STORE_TR, + INTERCEPT_LOAD_IDTR, + INTERCEPT_LOAD_GDTR, + INTERCEPT_LOAD_LDTR, + INTERCEPT_LOAD_TR, + INTERCEPT_RDTSC, + INTERCEPT_RDPMC, + INTERCEPT_PUSHF, + INTERCEPT_POPF, + INTERCEPT_CPUID, + INTERCEPT_RSM, + INTERCEPT_IRET, + INTERCEPT_INTn, + INTERCEPT_INVD, + INTERCEPT_PAUSE, + INTERCEPT_HLT, + INTERCEPT_INVLPG, + INTERCEPT_INVLPGA, + INTERCEPT_IOIO_PROT, + INTERCEPT_MSR_PROT, + INTERCEPT_TASK_SWITCH, + INTERCEPT_FERR_FREEZE, + INTERCEPT_SHUTDOWN, + INTERCEPT_VMRUN, + INTERCEPT_VMMCALL, + INTERCEPT_VMLOAD, + INTERCEPT_VMSAVE, + INTERCEPT_STGI, + INTERCEPT_CLGI, + INTERCEPT_SKINIT, + INTERCEPT_RDTSCP, + INTERCEPT_ICEBP, + INTERCEPT_WBINVD, + INTERCEPT_MONITOR, + INTERCEPT_MWAIT, + INTERCEPT_MWAIT_COND, +}; + + +struct __attribute__ ((__packed__)) vmcb_control_area { + u16 intercept_cr_read; + u16 intercept_cr_write; + u16 intercept_dr_read; + u16 intercept_dr_write; + u32 intercept_exceptions; + u64 intercept; + u8 reserved_1[44]; + u64 iopm_base_pa; + u64 msrpm_base_pa; + u64 tsc_offset; + u32 asid; + u8 tlb_ctl; + u8 reserved_2[3]; + u32 int_ctl; + u32 int_vector; + u32 int_state; + u8 reserved_3[4]; + u32 exit_code; + u32 exit_code_hi; + u64 exit_info_1; + u64 exit_info_2; + u32 exit_int_info; + u32 exit_int_info_err; + u64 nested_ctl; + u8 reserved_4[16]; + u32 event_inj; + u32 event_inj_err; + u64 nested_cr3; + u64 lbr_ctl; + u8 reserved_5[832]; +}; + + +#define TLB_CONTROL_DO_NOTHING 0 +#define TLB_CONTROL_FLUSH_ALL_ASID 1 + +#define V_TPR_MASK 0x0f + +#define V_IRQ_SHIFT 8 +#define V_IRQ_MASK (1 << V_IRQ_SHIFT) + +#define V_INTR_PRIO_SHIFT 16 +#define V_INTR_PRIO_MASK (0x0f << V_INTR_PRIO_SHIFT) + +#define V_IGN_TPR_SHIFT 20 +#define V_IGN_TPR_MASK (1 << V_IGN_TPR_SHIFT) + +#define V_INTR_MASKING_SHIFT 24 +#define V_INTR_MASKING_MASK (1 << V_INTR_MASKING_SHIFT) + +#define SVM_INTERRUPT_SHADOW_MASK 1 + +#define SVM_IOIO_STR_SHIFT 2 +#define SVM_IOIO_REP_SHIFT 3 +#define SVM_IOIO_SIZE_SHIFT 4 +#define SVM_IOIO_ASIZE_SHIFT 7 + +#define SVM_IOIO_TYPE_MASK 1 +#define SVM_IOIO_STR_MASK (1 << SVM_IOIO_STR_SHIFT) +#define SVM_IOIO_REP_MASK (1 << SVM_IOIO_REP_SHIFT) +#define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) +#define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) + +struct __attribute__ ((__packed__)) vmcb_seg { + u16 selector; + u16 attrib; + u32 limit; + u64 base; +}; + +struct __attribute__ ((__packed__)) vmcb_save_area { + struct vmcb_seg es; + struct vmcb_seg cs; + struct vmcb_seg ss; + struct vmcb_seg ds; + struct vmcb_seg fs; + struct vmcb_seg gs; + struct vmcb_seg gdtr; + struct vmcb_seg ldtr; + struct vmcb_seg idtr; + struct vmcb_seg tr; + u8 reserved_1[43]; + u8 cpl; + u8 reserved_2[4]; + u64 efer; + u8 reserved_3[112]; + u64 cr4; + u64 cr3; + u64 cr0; + u64 dr7; + u64 dr6; + u64 rflags; + u64 rip; + u8 reserved_4[88]; + u64 rsp; + u8 reserved_5[24]; + u64 rax; + u64 star; + u64 lstar; + u64 cstar; + u64 sfmask; + u64 kernel_gs_base; + u64 sysenter_cs; + u64 sysenter_esp; + u64 sysenter_eip; + u64 cr2; + u8 reserved_6[32]; + u64 g_pat; + u64 dbgctl; + u64 br_from; + u64 br_to; + u64 last_excp_from; + u64 last_excp_to; +}; + +struct __attribute__ ((__packed__)) vmcb { + struct vmcb_control_area control; + struct vmcb_save_area save; +}; + +#define SVM_CPUID_FEATURE_SHIFT 2 +#define SVM_CPUID_FUNC 0x8000000a + +#define MSR_EFER_SVME_MASK (1ULL << 12) +#define MSR_VM_CR 0xc0010114 +#define MSR_VM_HSAVE_PA 0xc0010117ULL + +#define SVM_VM_CR_SVM_DISABLE 4 + +#define SVM_SELECTOR_S_SHIFT 4 +#define SVM_SELECTOR_DPL_SHIFT 5 +#define SVM_SELECTOR_P_SHIFT 7 +#define SVM_SELECTOR_AVL_SHIFT 8 +#define SVM_SELECTOR_L_SHIFT 9 +#define SVM_SELECTOR_DB_SHIFT 10 +#define SVM_SELECTOR_G_SHIFT 11 + +#define SVM_SELECTOR_TYPE_MASK (0xf) +#define SVM_SELECTOR_S_MASK (1 << SVM_SELECTOR_S_SHIFT) +#define SVM_SELECTOR_DPL_MASK (3 << SVM_SELECTOR_DPL_SHIFT) +#define SVM_SELECTOR_P_MASK (1 << SVM_SELECTOR_P_SHIFT) +#define SVM_SELECTOR_AVL_MASK (1 << SVM_SELECTOR_AVL_SHIFT) +#define SVM_SELECTOR_L_MASK (1 << SVM_SELECTOR_L_SHIFT) +#define SVM_SELECTOR_DB_MASK (1 << SVM_SELECTOR_DB_SHIFT) +#define SVM_SELECTOR_G_MASK (1 << SVM_SELECTOR_G_SHIFT) + +#define SVM_SELECTOR_WRITE_MASK (1 << 1) +#define SVM_SELECTOR_READ_MASK SVM_SELECTOR_WRITE_MASK +#define SVM_SELECTOR_CODE_MASK (1 << 3) + +#define INTERCEPT_CR0_MASK 1 +#define INTERCEPT_CR3_MASK (1 << 3) +#define INTERCEPT_CR4_MASK (1 << 4) +#define INTERCEPT_CR8_MASK (1 << 8) + +#define INTERCEPT_DR0_MASK 1 +#define INTERCEPT_DR1_MASK (1 << 1) +#define INTERCEPT_DR2_MASK (1 << 2) +#define INTERCEPT_DR3_MASK (1 << 3) +#define INTERCEPT_DR4_MASK (1 << 4) +#define INTERCEPT_DR5_MASK (1 << 5) +#define INTERCEPT_DR6_MASK (1 << 6) +#define INTERCEPT_DR7_MASK (1 << 7) + +#define SVM_EVTINJ_VEC_MASK 0xff + +#define SVM_EVTINJ_TYPE_SHIFT 8 +#define SVM_EVTINJ_TYPE_MASK (7 << SVM_EVTINJ_TYPE_SHIFT) + +#define SVM_EVTINJ_TYPE_INTR (0 << SVM_EVTINJ_TYPE_SHIFT) +#define SVM_EVTINJ_TYPE_NMI (2 << SVM_EVTINJ_TYPE_SHIFT) +#define SVM_EVTINJ_TYPE_EXEPT (3 << SVM_EVTINJ_TYPE_SHIFT) +#define SVM_EVTINJ_TYPE_SOFT (4 << SVM_EVTINJ_TYPE_SHIFT) + +#define SVM_EVTINJ_VALID (1 << 31) +#define SVM_EVTINJ_VALID_ERR (1 << 11) + +#define SVM_EXITINTINFO_VEC_MASK SVM_EVTINJ_VEC_MASK + +#define SVM_EXITINTINFO_TYPE_INTR SVM_EVTINJ_TYPE_INTR +#define SVM_EXITINTINFO_TYPE_NMI SVM_EVTINJ_TYPE_NMI +#define SVM_EXITINTINFO_TYPE_EXEPT SVM_EVTINJ_TYPE_EXEPT +#define SVM_EXITINTINFO_TYPE_SOFT SVM_EVTINJ_TYPE_SOFT + +#define SVM_EXITINTINFO_VALID SVM_EVTINJ_VALID +#define SVM_EXITINTINFO_VALID_ERR SVM_EVTINJ_VALID_ERR + +#define SVM_EXIT_READ_CR0 0x000 +#define SVM_EXIT_READ_CR3 0x003 +#define SVM_EXIT_READ_CR4 0x004 +#define SVM_EXIT_READ_CR8 0x008 +#define SVM_EXIT_WRITE_CR0 0x010 +#define SVM_EXIT_WRITE_CR3 0x013 +#define SVM_EXIT_WRITE_CR4 0x014 +#define SVM_EXIT_WRITE_CR8 0x018 +#define SVM_EXIT_READ_DR0 0x020 +#define SVM_EXIT_READ_DR1 0x021 +#define SVM_EXIT_READ_DR2 0x022 +#define SVM_EXIT_READ_DR3 0x023 +#define SVM_EXIT_READ_DR4 0x024 +#define SVM_EXIT_READ_DR5 0x025 +#define SVM_EXIT_READ_DR6 0x026 +#define SVM_EXIT_READ_DR7 0x027 +#define SVM_EXIT_WRITE_DR0 0x030 +#define SVM_EXIT_WRITE_DR1 0x031 +#define SVM_EXIT_WRITE_DR2 0x032 +#define SVM_EXIT_WRITE_DR3 0x033 +#define SVM_EXIT_WRITE_DR4 0x034 +#define SVM_EXIT_WRITE_DR5 0x035 +#define SVM_EXIT_WRITE_DR6 0x036 +#define SVM_EXIT_WRITE_DR7 0x037 +#define SVM_EXIT_EXCP_BASE 0x040 +#define SVM_EXIT_INTR 0x060 +#define SVM_EXIT_NMI 0x061 +#define SVM_EXIT_SMI 0x062 +#define SVM_EXIT_INIT 0x063 +#define SVM_EXIT_VINTR 0x064 +#define SVM_EXIT_CR0_SEL_WRITE 0x065 +#define SVM_EXIT_IDTR_READ 0x066 +#define SVM_EXIT_GDTR_READ 0x067 +#define SVM_EXIT_LDTR_READ 0x068 +#define SVM_EXIT_TR_READ 0x069 +#define SVM_EXIT_IDTR_WRITE 0x06a +#define SVM_EXIT_GDTR_WRITE 0x06b +#define SVM_EXIT_LDTR_WRITE 0x06c +#define SVM_EXIT_TR_WRITE 0x06d +#define SVM_EXIT_RDTSC 0x06e +#define SVM_EXIT_RDPMC 0x06f +#define SVM_EXIT_PUSHF 0x070 +#define SVM_EXIT_POPF 0x071 +#define SVM_EXIT_CPUID 0x072 +#define SVM_EXIT_RSM 0x073 +#define SVM_EXIT_IRET 0x074 +#define SVM_EXIT_SWINT 0x075 +#define SVM_EXIT_INVD 0x076 +#define SVM_EXIT_PAUSE 0x077 +#define SVM_EXIT_HLT 0x078 +#define SVM_EXIT_INVLPG 0x079 +#define SVM_EXIT_INVLPGA 0x07a +#define SVM_EXIT_IOIO 0x07b +#define SVM_EXIT_MSR 0x07c +#define SVM_EXIT_TASK_SWITCH 0x07d +#define SVM_EXIT_FERR_FREEZE 0x07e +#define SVM_EXIT_SHUTDOWN 0x07f +#define SVM_EXIT_VMRUN 0x080 +#define SVM_EXIT_VMMCALL 0x081 +#define SVM_EXIT_VMLOAD 0x082 +#define SVM_EXIT_VMSAVE 0x083 +#define SVM_EXIT_STGI 0x084 +#define SVM_EXIT_CLGI 0x085 +#define SVM_EXIT_SKINIT 0x086 +#define SVM_EXIT_RDTSCP 0x087 +#define SVM_EXIT_ICEBP 0x088 +#define SVM_EXIT_WBINVD 0x089 +#define SVM_EXIT_MONITOR 0x08a +#define SVM_EXIT_MWAIT 0x08b +#define SVM_EXIT_MWAIT_COND 0x08c +#define SVM_EXIT_NPF 0x400 + +#define SVM_EXIT_ERR -1 + +#define SVM_CR0_SELECTIVE_MASK (1 << 3 | 1) /* TS and MP */ + +#define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda" +#define SVM_VMRUN ".byte 0x0f, 0x01, 0xd8" +#define SVM_VMSAVE ".byte 0x0f, 0x01, 0xdb" +#define SVM_CLGI ".byte 0x0f, 0x01, 0xdd" +#define SVM_STGI ".byte 0x0f, 0x01, 0xdc" +#define SVM_INVLPGA ".byte 0x0f, 0x01, 0xdf" + +#endif + diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c new file mode 100644 index 000000000000..fc494aff5d8b --- /dev/null +++ b/arch/x86/kvm/vmx.c @@ -0,0 +1,2671 @@ +/* + * Kernel-based Virtual Machine driver for Linux + * + * This module enables machines with Intel VT-x extensions to run virtual + * machines without emulation or binary translation. + * + * Copyright (C) 2006 Qumranet, Inc. + * + * Authors: + * Avi Kivity + * Yaniv Kamay + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#include "irq.h" +#include "vmx.h" +#include "segment_descriptor.h" +#include "mmu.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +MODULE_AUTHOR("Qumranet"); +MODULE_LICENSE("GPL"); + +static int bypass_guest_pf = 1; +module_param(bypass_guest_pf, bool, 0); + +struct vmcs { + u32 revision_id; + u32 abort; + char data[0]; +}; + +struct vcpu_vmx { + struct kvm_vcpu vcpu; + int launched; + u8 fail; + u32 idt_vectoring_info; + struct kvm_msr_entry *guest_msrs; + struct kvm_msr_entry *host_msrs; + int nmsrs; + int save_nmsrs; + int msr_offset_efer; +#ifdef CONFIG_X86_64 + int msr_offset_kernel_gs_base; +#endif + struct vmcs *vmcs; + struct { + int loaded; + u16 fs_sel, gs_sel, ldt_sel; + int gs_ldt_reload_needed; + int fs_reload_needed; + int guest_efer_loaded; + } host_state; + struct { + struct { + bool pending; + u8 vector; + unsigned rip; + } irq; + } rmode; +}; + +static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) +{ + return container_of(vcpu, struct vcpu_vmx, vcpu); +} + +static int init_rmode_tss(struct kvm *kvm); + +static DEFINE_PER_CPU(struct vmcs *, vmxarea); +static DEFINE_PER_CPU(struct vmcs *, current_vmcs); + +static struct page *vmx_io_bitmap_a; +static struct page *vmx_io_bitmap_b; + +static struct vmcs_config { + int size; + int order; + u32 revision_id; + u32 pin_based_exec_ctrl; + u32 cpu_based_exec_ctrl; + u32 cpu_based_2nd_exec_ctrl; + u32 vmexit_ctrl; + u32 vmentry_ctrl; +} vmcs_config; + +#define VMX_SEGMENT_FIELD(seg) \ + [VCPU_SREG_##seg] = { \ + .selector = GUEST_##seg##_SELECTOR, \ + .base = GUEST_##seg##_BASE, \ + .limit = GUEST_##seg##_LIMIT, \ + .ar_bytes = GUEST_##seg##_AR_BYTES, \ + } + +static struct kvm_vmx_segment_field { + unsigned selector; + unsigned base; + unsigned limit; + unsigned ar_bytes; +} kvm_vmx_segment_fields[] = { + VMX_SEGMENT_FIELD(CS), + VMX_SEGMENT_FIELD(DS), + VMX_SEGMENT_FIELD(ES), + VMX_SEGMENT_FIELD(FS), + VMX_SEGMENT_FIELD(GS), + VMX_SEGMENT_FIELD(SS), + VMX_SEGMENT_FIELD(TR), + VMX_SEGMENT_FIELD(LDTR), +}; + +/* + * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it + * away by decrementing the array size. + */ +static const u32 vmx_msr_index[] = { +#ifdef CONFIG_X86_64 + MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, MSR_KERNEL_GS_BASE, +#endif + MSR_EFER, MSR_K6_STAR, +}; +#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index) + +static void load_msrs(struct kvm_msr_entry *e, int n) +{ + int i; + + for (i = 0; i < n; ++i) + wrmsrl(e[i].index, e[i].data); +} + +static void save_msrs(struct kvm_msr_entry *e, int n) +{ + int i; + + for (i = 0; i < n; ++i) + rdmsrl(e[i].index, e[i].data); +} + +static inline int is_page_fault(u32 intr_info) +{ + return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | + INTR_INFO_VALID_MASK)) == + (INTR_TYPE_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK); +} + +static inline int is_no_device(u32 intr_info) +{ + return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | + INTR_INFO_VALID_MASK)) == + (INTR_TYPE_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK); +} + +static inline int is_invalid_opcode(u32 intr_info) +{ + return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | + INTR_INFO_VALID_MASK)) == + (INTR_TYPE_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK); +} + +static inline int is_external_interrupt(u32 intr_info) +{ + return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK)) + == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK); +} + +static inline int cpu_has_vmx_tpr_shadow(void) +{ + return (vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW); +} + +static inline int vm_need_tpr_shadow(struct kvm *kvm) +{ + return ((cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm))); +} + +static inline int cpu_has_secondary_exec_ctrls(void) +{ + return (vmcs_config.cpu_based_exec_ctrl & + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); +} + +static inline int cpu_has_vmx_virtualize_apic_accesses(void) +{ + return (vmcs_config.cpu_based_2nd_exec_ctrl & + SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); +} + +static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm) +{ + return ((cpu_has_vmx_virtualize_apic_accesses()) && + (irqchip_in_kernel(kvm))); +} + +static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr) +{ + int i; + + for (i = 0; i < vmx->nmsrs; ++i) + if (vmx->guest_msrs[i].index == msr) + return i; + return -1; +} + +static struct kvm_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr) +{ + int i; + + i = __find_msr_index(vmx, msr); + if (i >= 0) + return &vmx->guest_msrs[i]; + return NULL; +} + +static void vmcs_clear(struct vmcs *vmcs) +{ + u64 phys_addr = __pa(vmcs); + u8 error; + + asm volatile (ASM_VMX_VMCLEAR_RAX "; setna %0" + : "=g"(error) : "a"(&phys_addr), "m"(phys_addr) + : "cc", "memory"); + if (error) + printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n", + vmcs, phys_addr); +} + +static void __vcpu_clear(void *arg) +{ + struct vcpu_vmx *vmx = arg; + int cpu = raw_smp_processor_id(); + + if (vmx->vcpu.cpu == cpu) + vmcs_clear(vmx->vmcs); + if (per_cpu(current_vmcs, cpu) == vmx->vmcs) + per_cpu(current_vmcs, cpu) = NULL; + rdtscll(vmx->vcpu.arch.host_tsc); +} + +static void vcpu_clear(struct vcpu_vmx *vmx) +{ + if (vmx->vcpu.cpu == -1) + return; + smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1); + vmx->launched = 0; +} + +static unsigned long vmcs_readl(unsigned long field) +{ + unsigned long value; + + asm volatile (ASM_VMX_VMREAD_RDX_RAX + : "=a"(value) : "d"(field) : "cc"); + return value; +} + +static u16 vmcs_read16(unsigned long field) +{ + return vmcs_readl(field); +} + +static u32 vmcs_read32(unsigned long field) +{ + return vmcs_readl(field); +} + +static u64 vmcs_read64(unsigned long field) +{ +#ifdef CONFIG_X86_64 + return vmcs_readl(field); +#else + return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32); +#endif +} + +static noinline void vmwrite_error(unsigned long field, unsigned long value) +{ + printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n", + field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); + dump_stack(); +} + +static void vmcs_writel(unsigned long field, unsigned long value) +{ + u8 error; + + asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0" + : "=q"(error) : "a"(value), "d"(field) : "cc"); + if (unlikely(error)) + vmwrite_error(field, value); +} + +static void vmcs_write16(unsigned long field, u16 value) +{ + vmcs_writel(field, value); +} + +static void vmcs_write32(unsigned long field, u32 value) +{ + vmcs_writel(field, value); +} + +static void vmcs_write64(unsigned long field, u64 value) +{ +#ifdef CONFIG_X86_64 + vmcs_writel(field, value); +#else + vmcs_writel(field, value); + asm volatile (""); + vmcs_writel(field+1, value >> 32); +#endif +} + +static void vmcs_clear_bits(unsigned long field, u32 mask) +{ + vmcs_writel(field, vmcs_readl(field) & ~mask); +} + +static void vmcs_set_bits(unsigned long field, u32 mask) +{ + vmcs_writel(field, vmcs_readl(field) | mask); +} + +static void update_exception_bitmap(struct kvm_vcpu *vcpu) +{ + u32 eb; + + eb = (1u << PF_VECTOR) | (1u << UD_VECTOR); + if (!vcpu->fpu_active) + eb |= 1u << NM_VECTOR; + if (vcpu->guest_debug.enabled) + eb |= 1u << 1; + if (vcpu->arch.rmode.active) + eb = ~0; + vmcs_write32(EXCEPTION_BITMAP, eb); +} + +static void reload_tss(void) +{ +#ifndef CONFIG_X86_64 + + /* + * VT restores TR but not its size. Useless. + */ + struct descriptor_table gdt; + struct segment_descriptor *descs; + + get_gdt(&gdt); + descs = (void *)gdt.base; + descs[GDT_ENTRY_TSS].type = 9; /* available TSS */ + load_TR_desc(); +#endif +} + +static void load_transition_efer(struct vcpu_vmx *vmx) +{ + int efer_offset = vmx->msr_offset_efer; + u64 host_efer = vmx->host_msrs[efer_offset].data; + u64 guest_efer = vmx->guest_msrs[efer_offset].data; + u64 ignore_bits; + + if (efer_offset < 0) + return; + /* + * NX is emulated; LMA and LME handled by hardware; SCE meaninless + * outside long mode + */ + ignore_bits = EFER_NX | EFER_SCE; +#ifdef CONFIG_X86_64 + ignore_bits |= EFER_LMA | EFER_LME; + /* SCE is meaningful only in long mode on Intel */ + if (guest_efer & EFER_LMA) + ignore_bits &= ~(u64)EFER_SCE; +#endif + if ((guest_efer & ~ignore_bits) == (host_efer & ~ignore_bits)) + return; + + vmx->host_state.guest_efer_loaded = 1; + guest_efer &= ~ignore_bits; + guest_efer |= host_efer & ignore_bits; + wrmsrl(MSR_EFER, guest_efer); + vmx->vcpu.stat.efer_reload++; +} + +static void reload_host_efer(struct vcpu_vmx *vmx) +{ + if (vmx->host_state.guest_efer_loaded) { + vmx->host_state.guest_efer_loaded = 0; + load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1); + } +} + +static void vmx_save_host_state(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + if (vmx->host_state.loaded) + return; + + vmx->host_state.loaded = 1; + /* + * Set host fs and gs selectors. Unfortunately, 22.2.3 does not + * allow segment selectors with cpl > 0 or ti == 1. + */ + vmx->host_state.ldt_sel = read_ldt(); + vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel; + vmx->host_state.fs_sel = read_fs(); + if (!(vmx->host_state.fs_sel & 7)) { + vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel); + vmx->host_state.fs_reload_needed = 0; + } else { + vmcs_write16(HOST_FS_SELECTOR, 0); + vmx->host_state.fs_reload_needed = 1; + } + vmx->host_state.gs_sel = read_gs(); + if (!(vmx->host_state.gs_sel & 7)) + vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel); + else { + vmcs_write16(HOST_GS_SELECTOR, 0); + vmx->host_state.gs_ldt_reload_needed = 1; + } + +#ifdef CONFIG_X86_64 + vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE)); + vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE)); +#else + vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel)); + vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel)); +#endif + +#ifdef CONFIG_X86_64 + if (is_long_mode(&vmx->vcpu)) + save_msrs(vmx->host_msrs + + vmx->msr_offset_kernel_gs_base, 1); + +#endif + load_msrs(vmx->guest_msrs, vmx->save_nmsrs); + load_transition_efer(vmx); +} + +static void vmx_load_host_state(struct vcpu_vmx *vmx) +{ + unsigned long flags; + + if (!vmx->host_state.loaded) + return; + + ++vmx->vcpu.stat.host_state_reload; + vmx->host_state.loaded = 0; + if (vmx->host_state.fs_reload_needed) + load_fs(vmx->host_state.fs_sel); + if (vmx->host_state.gs_ldt_reload_needed) { + load_ldt(vmx->host_state.ldt_sel); + /* + * If we have to reload gs, we must take care to + * preserve our gs base. + */ + local_irq_save(flags); + load_gs(vmx->host_state.gs_sel); +#ifdef CONFIG_X86_64 + wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE)); +#endif + local_irq_restore(flags); + } + reload_tss(); + save_msrs(vmx->guest_msrs, vmx->save_nmsrs); + load_msrs(vmx->host_msrs, vmx->save_nmsrs); + reload_host_efer(vmx); +} + +/* + * Switches to specified vcpu, until a matching vcpu_put(), but assumes + * vcpu mutex is already taken. + */ +static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u64 phys_addr = __pa(vmx->vmcs); + u64 tsc_this, delta; + + if (vcpu->cpu != cpu) { + vcpu_clear(vmx); + kvm_migrate_apic_timer(vcpu); + } + + if (per_cpu(current_vmcs, cpu) != vmx->vmcs) { + u8 error; + + per_cpu(current_vmcs, cpu) = vmx->vmcs; + asm volatile (ASM_VMX_VMPTRLD_RAX "; setna %0" + : "=g"(error) : "a"(&phys_addr), "m"(phys_addr) + : "cc"); + if (error) + printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n", + vmx->vmcs, phys_addr); + } + + if (vcpu->cpu != cpu) { + struct descriptor_table dt; + unsigned long sysenter_esp; + + vcpu->cpu = cpu; + /* + * Linux uses per-cpu TSS and GDT, so set these when switching + * processors. + */ + vmcs_writel(HOST_TR_BASE, read_tr_base()); /* 22.2.4 */ + get_gdt(&dt); + vmcs_writel(HOST_GDTR_BASE, dt.base); /* 22.2.4 */ + + rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp); + vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */ + + /* + * Make sure the time stamp counter is monotonous. + */ + rdtscll(tsc_this); + delta = vcpu->arch.host_tsc - tsc_this; + vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta); + } +} + +static void vmx_vcpu_put(struct kvm_vcpu *vcpu) +{ + vmx_load_host_state(to_vmx(vcpu)); +} + +static void vmx_fpu_activate(struct kvm_vcpu *vcpu) +{ + if (vcpu->fpu_active) + return; + vcpu->fpu_active = 1; + vmcs_clear_bits(GUEST_CR0, X86_CR0_TS); + if (vcpu->arch.cr0 & X86_CR0_TS) + vmcs_set_bits(GUEST_CR0, X86_CR0_TS); + update_exception_bitmap(vcpu); +} + +static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu) +{ + if (!vcpu->fpu_active) + return; + vcpu->fpu_active = 0; + vmcs_set_bits(GUEST_CR0, X86_CR0_TS); + update_exception_bitmap(vcpu); +} + +static void vmx_vcpu_decache(struct kvm_vcpu *vcpu) +{ + vcpu_clear(to_vmx(vcpu)); +} + +static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) +{ + return vmcs_readl(GUEST_RFLAGS); +} + +static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) +{ + if (vcpu->arch.rmode.active) + rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; + vmcs_writel(GUEST_RFLAGS, rflags); +} + +static void skip_emulated_instruction(struct kvm_vcpu *vcpu) +{ + unsigned long rip; + u32 interruptibility; + + rip = vmcs_readl(GUEST_RIP); + rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN); + vmcs_writel(GUEST_RIP, rip); + + /* + * We emulated an instruction, so temporary interrupt blocking + * should be removed, if set. + */ + interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); + if (interruptibility & 3) + vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, + interruptibility & ~3); + vcpu->arch.interrupt_window_open = 1; +} + +static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code) +{ + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + nr | INTR_TYPE_EXCEPTION + | (has_error_code ? INTR_INFO_DELIEVER_CODE_MASK : 0) + | INTR_INFO_VALID_MASK); + if (has_error_code) + vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); +} + +static bool vmx_exception_injected(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + return !(vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); +} + +/* + * Swap MSR entry in host/guest MSR entry array. + */ +#ifdef CONFIG_X86_64 +static void move_msr_up(struct vcpu_vmx *vmx, int from, int to) +{ + struct kvm_msr_entry tmp; + + tmp = vmx->guest_msrs[to]; + vmx->guest_msrs[to] = vmx->guest_msrs[from]; + vmx->guest_msrs[from] = tmp; + tmp = vmx->host_msrs[to]; + vmx->host_msrs[to] = vmx->host_msrs[from]; + vmx->host_msrs[from] = tmp; +} +#endif + +/* + * Set up the vmcs to automatically save and restore system + * msrs. Don't touch the 64-bit msrs if the guest is in legacy + * mode, as fiddling with msrs is very expensive. + */ +static void setup_msrs(struct vcpu_vmx *vmx) +{ + int save_nmsrs; + + save_nmsrs = 0; +#ifdef CONFIG_X86_64 + if (is_long_mode(&vmx->vcpu)) { + int index; + + index = __find_msr_index(vmx, MSR_SYSCALL_MASK); + if (index >= 0) + move_msr_up(vmx, index, save_nmsrs++); + index = __find_msr_index(vmx, MSR_LSTAR); + if (index >= 0) + move_msr_up(vmx, index, save_nmsrs++); + index = __find_msr_index(vmx, MSR_CSTAR); + if (index >= 0) + move_msr_up(vmx, index, save_nmsrs++); + index = __find_msr_index(vmx, MSR_KERNEL_GS_BASE); + if (index >= 0) + move_msr_up(vmx, index, save_nmsrs++); + /* + * MSR_K6_STAR is only needed on long mode guests, and only + * if efer.sce is enabled. + */ + index = __find_msr_index(vmx, MSR_K6_STAR); + if ((index >= 0) && (vmx->vcpu.arch.shadow_efer & EFER_SCE)) + move_msr_up(vmx, index, save_nmsrs++); + } +#endif + vmx->save_nmsrs = save_nmsrs; + +#ifdef CONFIG_X86_64 + vmx->msr_offset_kernel_gs_base = + __find_msr_index(vmx, MSR_KERNEL_GS_BASE); +#endif + vmx->msr_offset_efer = __find_msr_index(vmx, MSR_EFER); +} + +/* + * reads and returns guest's timestamp counter "register" + * guest_tsc = host_tsc + tsc_offset -- 21.3 + */ +static u64 guest_read_tsc(void) +{ + u64 host_tsc, tsc_offset; + + rdtscll(host_tsc); + tsc_offset = vmcs_read64(TSC_OFFSET); + return host_tsc + tsc_offset; +} + +/* + * writes 'guest_tsc' into guest's timestamp counter "register" + * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc + */ +static void guest_write_tsc(u64 guest_tsc) +{ + u64 host_tsc; + + rdtscll(host_tsc); + vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc); +} + +/* + * Reads an msr value (of 'msr_index') into 'pdata'. + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) +{ + u64 data; + struct kvm_msr_entry *msr; + + if (!pdata) { + printk(KERN_ERR "BUG: get_msr called with NULL pdata\n"); + return -EINVAL; + } + + switch (msr_index) { +#ifdef CONFIG_X86_64 + case MSR_FS_BASE: + data = vmcs_readl(GUEST_FS_BASE); + break; + case MSR_GS_BASE: + data = vmcs_readl(GUEST_GS_BASE); + break; + case MSR_EFER: + return kvm_get_msr_common(vcpu, msr_index, pdata); +#endif + case MSR_IA32_TIME_STAMP_COUNTER: + data = guest_read_tsc(); + break; + case MSR_IA32_SYSENTER_CS: + data = vmcs_read32(GUEST_SYSENTER_CS); + break; + case MSR_IA32_SYSENTER_EIP: + data = vmcs_readl(GUEST_SYSENTER_EIP); + break; + case MSR_IA32_SYSENTER_ESP: + data = vmcs_readl(GUEST_SYSENTER_ESP); + break; + default: + msr = find_msr_entry(to_vmx(vcpu), msr_index); + if (msr) { + data = msr->data; + break; + } + return kvm_get_msr_common(vcpu, msr_index, pdata); + } + + *pdata = data; + return 0; +} + +/* + * Writes msr value into into the appropriate "register". + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + struct kvm_msr_entry *msr; + int ret = 0; + + switch (msr_index) { +#ifdef CONFIG_X86_64 + case MSR_EFER: + ret = kvm_set_msr_common(vcpu, msr_index, data); + if (vmx->host_state.loaded) { + reload_host_efer(vmx); + load_transition_efer(vmx); + } + break; + case MSR_FS_BASE: + vmcs_writel(GUEST_FS_BASE, data); + break; + case MSR_GS_BASE: + vmcs_writel(GUEST_GS_BASE, data); + break; +#endif + case MSR_IA32_SYSENTER_CS: + vmcs_write32(GUEST_SYSENTER_CS, data); + break; + case MSR_IA32_SYSENTER_EIP: + vmcs_writel(GUEST_SYSENTER_EIP, data); + break; + case MSR_IA32_SYSENTER_ESP: + vmcs_writel(GUEST_SYSENTER_ESP, data); + break; + case MSR_IA32_TIME_STAMP_COUNTER: + guest_write_tsc(data); + break; + default: + msr = find_msr_entry(vmx, msr_index); + if (msr) { + msr->data = data; + if (vmx->host_state.loaded) + load_msrs(vmx->guest_msrs, vmx->save_nmsrs); + break; + } + ret = kvm_set_msr_common(vcpu, msr_index, data); + } + + return ret; +} + +/* + * Sync the rsp and rip registers into the vcpu structure. This allows + * registers to be accessed by indexing vcpu->arch.regs. + */ +static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu) +{ + vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); + vcpu->arch.rip = vmcs_readl(GUEST_RIP); +} + +/* + * Syncs rsp and rip back into the vmcs. Should be called after possible + * modification. + */ +static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu) +{ + vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); + vmcs_writel(GUEST_RIP, vcpu->arch.rip); +} + +static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) +{ + unsigned long dr7 = 0x400; + int old_singlestep; + + old_singlestep = vcpu->guest_debug.singlestep; + + vcpu->guest_debug.enabled = dbg->enabled; + if (vcpu->guest_debug.enabled) { + int i; + + dr7 |= 0x200; /* exact */ + for (i = 0; i < 4; ++i) { + if (!dbg->breakpoints[i].enabled) + continue; + vcpu->guest_debug.bp[i] = dbg->breakpoints[i].address; + dr7 |= 2 << (i*2); /* global enable */ + dr7 |= 0 << (i*4+16); /* execution breakpoint */ + } + + vcpu->guest_debug.singlestep = dbg->singlestep; + } else + vcpu->guest_debug.singlestep = 0; + + if (old_singlestep && !vcpu->guest_debug.singlestep) { + unsigned long flags; + + flags = vmcs_readl(GUEST_RFLAGS); + flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); + vmcs_writel(GUEST_RFLAGS, flags); + } + + update_exception_bitmap(vcpu); + vmcs_writel(GUEST_DR7, dr7); + + return 0; +} + +static int vmx_get_irq(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 idtv_info_field; + + idtv_info_field = vmx->idt_vectoring_info; + if (idtv_info_field & INTR_INFO_VALID_MASK) { + if (is_external_interrupt(idtv_info_field)) + return idtv_info_field & VECTORING_INFO_VECTOR_MASK; + else + printk(KERN_DEBUG "pending exception: not handled yet\n"); + } + return -1; +} + +static __init int cpu_has_kvm_support(void) +{ + unsigned long ecx = cpuid_ecx(1); + return test_bit(5, &ecx); /* CPUID.1:ECX.VMX[bit 5] -> VT */ +} + +static __init int vmx_disabled_by_bios(void) +{ + u64 msr; + + rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); + return (msr & (MSR_IA32_FEATURE_CONTROL_LOCKED | + MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED)) + == MSR_IA32_FEATURE_CONTROL_LOCKED; + /* locked but not enabled */ +} + +static void hardware_enable(void *garbage) +{ + int cpu = raw_smp_processor_id(); + u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); + u64 old; + + rdmsrl(MSR_IA32_FEATURE_CONTROL, old); + if ((old & (MSR_IA32_FEATURE_CONTROL_LOCKED | + MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED)) + != (MSR_IA32_FEATURE_CONTROL_LOCKED | + MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED)) + /* enable and lock */ + wrmsrl(MSR_IA32_FEATURE_CONTROL, old | + MSR_IA32_FEATURE_CONTROL_LOCKED | + MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED); + write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */ + asm volatile (ASM_VMX_VMXON_RAX : : "a"(&phys_addr), "m"(phys_addr) + : "memory", "cc"); +} + +static void hardware_disable(void *garbage) +{ + asm volatile (ASM_VMX_VMXOFF : : : "cc"); +} + +static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, + u32 msr, u32 *result) +{ + u32 vmx_msr_low, vmx_msr_high; + u32 ctl = ctl_min | ctl_opt; + + rdmsr(msr, vmx_msr_low, vmx_msr_high); + + ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */ + ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */ + + /* Ensure minimum (required) set of control bits are supported. */ + if (ctl_min & ~ctl) + return -EIO; + + *result = ctl; + return 0; +} + +static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) +{ + u32 vmx_msr_low, vmx_msr_high; + u32 min, opt; + u32 _pin_based_exec_control = 0; + u32 _cpu_based_exec_control = 0; + u32 _cpu_based_2nd_exec_control = 0; + u32 _vmexit_control = 0; + u32 _vmentry_control = 0; + + min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING; + opt = 0; + if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS, + &_pin_based_exec_control) < 0) + return -EIO; + + min = CPU_BASED_HLT_EXITING | +#ifdef CONFIG_X86_64 + CPU_BASED_CR8_LOAD_EXITING | + CPU_BASED_CR8_STORE_EXITING | +#endif + CPU_BASED_USE_IO_BITMAPS | + CPU_BASED_MOV_DR_EXITING | + CPU_BASED_USE_TSC_OFFSETING; + opt = CPU_BASED_TPR_SHADOW | + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; + if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS, + &_cpu_based_exec_control) < 0) + return -EIO; +#ifdef CONFIG_X86_64 + if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) + _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING & + ~CPU_BASED_CR8_STORE_EXITING; +#endif + if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { + min = 0; + opt = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | + SECONDARY_EXEC_WBINVD_EXITING; + if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS2, + &_cpu_based_2nd_exec_control) < 0) + return -EIO; + } +#ifndef CONFIG_X86_64 + if (!(_cpu_based_2nd_exec_control & + SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) + _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; +#endif + + min = 0; +#ifdef CONFIG_X86_64 + min |= VM_EXIT_HOST_ADDR_SPACE_SIZE; +#endif + opt = 0; + if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS, + &_vmexit_control) < 0) + return -EIO; + + min = opt = 0; + if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS, + &_vmentry_control) < 0) + return -EIO; + + rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high); + + /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ + if ((vmx_msr_high & 0x1fff) > PAGE_SIZE) + return -EIO; + +#ifdef CONFIG_X86_64 + /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */ + if (vmx_msr_high & (1u<<16)) + return -EIO; +#endif + + /* Require Write-Back (WB) memory type for VMCS accesses. */ + if (((vmx_msr_high >> 18) & 15) != 6) + return -EIO; + + vmcs_conf->size = vmx_msr_high & 0x1fff; + vmcs_conf->order = get_order(vmcs_config.size); + vmcs_conf->revision_id = vmx_msr_low; + + vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; + vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; + vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; + vmcs_conf->vmexit_ctrl = _vmexit_control; + vmcs_conf->vmentry_ctrl = _vmentry_control; + + return 0; +} + +static struct vmcs *alloc_vmcs_cpu(int cpu) +{ + int node = cpu_to_node(cpu); + struct page *pages; + struct vmcs *vmcs; + + pages = alloc_pages_node(node, GFP_KERNEL, vmcs_config.order); + if (!pages) + return NULL; + vmcs = page_address(pages); + memset(vmcs, 0, vmcs_config.size); + vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */ + return vmcs; +} + +static struct vmcs *alloc_vmcs(void) +{ + return alloc_vmcs_cpu(raw_smp_processor_id()); +} + +static void free_vmcs(struct vmcs *vmcs) +{ + free_pages((unsigned long)vmcs, vmcs_config.order); +} + +static void free_kvm_area(void) +{ + int cpu; + + for_each_online_cpu(cpu) + free_vmcs(per_cpu(vmxarea, cpu)); +} + +static __init int alloc_kvm_area(void) +{ + int cpu; + + for_each_online_cpu(cpu) { + struct vmcs *vmcs; + + vmcs = alloc_vmcs_cpu(cpu); + if (!vmcs) { + free_kvm_area(); + return -ENOMEM; + } + + per_cpu(vmxarea, cpu) = vmcs; + } + return 0; +} + +static __init int hardware_setup(void) +{ + if (setup_vmcs_config(&vmcs_config) < 0) + return -EIO; + return alloc_kvm_area(); +} + +static __exit void hardware_unsetup(void) +{ + free_kvm_area(); +} + +static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save) +{ + struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; + + if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) { + vmcs_write16(sf->selector, save->selector); + vmcs_writel(sf->base, save->base); + vmcs_write32(sf->limit, save->limit); + vmcs_write32(sf->ar_bytes, save->ar); + } else { + u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK) + << AR_DPL_SHIFT; + vmcs_write32(sf->ar_bytes, 0x93 | dpl); + } +} + +static void enter_pmode(struct kvm_vcpu *vcpu) +{ + unsigned long flags; + + vcpu->arch.rmode.active = 0; + + vmcs_writel(GUEST_TR_BASE, vcpu->arch.rmode.tr.base); + vmcs_write32(GUEST_TR_LIMIT, vcpu->arch.rmode.tr.limit); + vmcs_write32(GUEST_TR_AR_BYTES, vcpu->arch.rmode.tr.ar); + + flags = vmcs_readl(GUEST_RFLAGS); + flags &= ~(X86_EFLAGS_IOPL | X86_EFLAGS_VM); + flags |= (vcpu->arch.rmode.save_iopl << IOPL_SHIFT); + vmcs_writel(GUEST_RFLAGS, flags); + + vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | + (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); + + update_exception_bitmap(vcpu); + + fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->arch.rmode.es); + fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->arch.rmode.ds); + fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->arch.rmode.gs); + fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->arch.rmode.fs); + + vmcs_write16(GUEST_SS_SELECTOR, 0); + vmcs_write32(GUEST_SS_AR_BYTES, 0x93); + + vmcs_write16(GUEST_CS_SELECTOR, + vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK); + vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); +} + +static gva_t rmode_tss_base(struct kvm *kvm) +{ + if (!kvm->arch.tss_addr) { + gfn_t base_gfn = kvm->memslots[0].base_gfn + + kvm->memslots[0].npages - 3; + return base_gfn << PAGE_SHIFT; + } + return kvm->arch.tss_addr; +} + +static void fix_rmode_seg(int seg, struct kvm_save_segment *save) +{ + struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; + + save->selector = vmcs_read16(sf->selector); + save->base = vmcs_readl(sf->base); + save->limit = vmcs_read32(sf->limit); + save->ar = vmcs_read32(sf->ar_bytes); + vmcs_write16(sf->selector, save->base >> 4); + vmcs_write32(sf->base, save->base & 0xfffff); + vmcs_write32(sf->limit, 0xffff); + vmcs_write32(sf->ar_bytes, 0xf3); +} + +static void enter_rmode(struct kvm_vcpu *vcpu) +{ + unsigned long flags; + + vcpu->arch.rmode.active = 1; + + vcpu->arch.rmode.tr.base = vmcs_readl(GUEST_TR_BASE); + vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm)); + + vcpu->arch.rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT); + vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); + + vcpu->arch.rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES); + vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); + + flags = vmcs_readl(GUEST_RFLAGS); + vcpu->arch.rmode.save_iopl + = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; + + flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; + + vmcs_writel(GUEST_RFLAGS, flags); + vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); + update_exception_bitmap(vcpu); + + vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4); + vmcs_write32(GUEST_SS_LIMIT, 0xffff); + vmcs_write32(GUEST_SS_AR_BYTES, 0xf3); + + vmcs_write32(GUEST_CS_AR_BYTES, 0xf3); + vmcs_write32(GUEST_CS_LIMIT, 0xffff); + if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000) + vmcs_writel(GUEST_CS_BASE, 0xf0000); + vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4); + + fix_rmode_seg(VCPU_SREG_ES, &vcpu->arch.rmode.es); + fix_rmode_seg(VCPU_SREG_DS, &vcpu->arch.rmode.ds); + fix_rmode_seg(VCPU_SREG_GS, &vcpu->arch.rmode.gs); + fix_rmode_seg(VCPU_SREG_FS, &vcpu->arch.rmode.fs); + + kvm_mmu_reset_context(vcpu); + init_rmode_tss(vcpu->kvm); +} + +#ifdef CONFIG_X86_64 + +static void enter_lmode(struct kvm_vcpu *vcpu) +{ + u32 guest_tr_ar; + + guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); + if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) { + printk(KERN_DEBUG "%s: tss fixup for long mode. \n", + __FUNCTION__); + vmcs_write32(GUEST_TR_AR_BYTES, + (guest_tr_ar & ~AR_TYPE_MASK) + | AR_TYPE_BUSY_64_TSS); + } + + vcpu->arch.shadow_efer |= EFER_LMA; + + find_msr_entry(to_vmx(vcpu), MSR_EFER)->data |= EFER_LMA | EFER_LME; + vmcs_write32(VM_ENTRY_CONTROLS, + vmcs_read32(VM_ENTRY_CONTROLS) + | VM_ENTRY_IA32E_MODE); +} + +static void exit_lmode(struct kvm_vcpu *vcpu) +{ + vcpu->arch.shadow_efer &= ~EFER_LMA; + + vmcs_write32(VM_ENTRY_CONTROLS, + vmcs_read32(VM_ENTRY_CONTROLS) + & ~VM_ENTRY_IA32E_MODE); +} + +#endif + +static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) +{ + vcpu->arch.cr4 &= KVM_GUEST_CR4_MASK; + vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK; +} + +static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ + vmx_fpu_deactivate(vcpu); + + if (vcpu->arch.rmode.active && (cr0 & X86_CR0_PE)) + enter_pmode(vcpu); + + if (!vcpu->arch.rmode.active && !(cr0 & X86_CR0_PE)) + enter_rmode(vcpu); + +#ifdef CONFIG_X86_64 + if (vcpu->arch.shadow_efer & EFER_LME) { + if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) + enter_lmode(vcpu); + if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) + exit_lmode(vcpu); + } +#endif + + vmcs_writel(CR0_READ_SHADOW, cr0); + vmcs_writel(GUEST_CR0, + (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON); + vcpu->arch.cr0 = cr0; + + if (!(cr0 & X86_CR0_TS) || !(cr0 & X86_CR0_PE)) + vmx_fpu_activate(vcpu); +} + +static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + vmcs_writel(GUEST_CR3, cr3); + if (vcpu->arch.cr0 & X86_CR0_PE) + vmx_fpu_deactivate(vcpu); +} + +static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + vmcs_writel(CR4_READ_SHADOW, cr4); + vmcs_writel(GUEST_CR4, cr4 | (vcpu->arch.rmode.active ? + KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON)); + vcpu->arch.cr4 = cr4; +} + +#ifdef CONFIG_X86_64 + +static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); + + vcpu->arch.shadow_efer = efer; + if (efer & EFER_LMA) { + vmcs_write32(VM_ENTRY_CONTROLS, + vmcs_read32(VM_ENTRY_CONTROLS) | + VM_ENTRY_IA32E_MODE); + msr->data = efer; + + } else { + vmcs_write32(VM_ENTRY_CONTROLS, + vmcs_read32(VM_ENTRY_CONTROLS) & + ~VM_ENTRY_IA32E_MODE); + + msr->data = efer & ~EFER_LME; + } + setup_msrs(vmx); +} + +#endif + +static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) +{ + struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; + + return vmcs_readl(sf->base); +} + +static void vmx_get_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; + u32 ar; + + var->base = vmcs_readl(sf->base); + var->limit = vmcs_read32(sf->limit); + var->selector = vmcs_read16(sf->selector); + ar = vmcs_read32(sf->ar_bytes); + if (ar & AR_UNUSABLE_MASK) + ar = 0; + var->type = ar & 15; + var->s = (ar >> 4) & 1; + var->dpl = (ar >> 5) & 3; + var->present = (ar >> 7) & 1; + var->avl = (ar >> 12) & 1; + var->l = (ar >> 13) & 1; + var->db = (ar >> 14) & 1; + var->g = (ar >> 15) & 1; + var->unusable = (ar >> 16) & 1; +} + +static u32 vmx_segment_access_rights(struct kvm_segment *var) +{ + u32 ar; + + if (var->unusable) + ar = 1 << 16; + else { + ar = var->type & 15; + ar |= (var->s & 1) << 4; + ar |= (var->dpl & 3) << 5; + ar |= (var->present & 1) << 7; + ar |= (var->avl & 1) << 12; + ar |= (var->l & 1) << 13; + ar |= (var->db & 1) << 14; + ar |= (var->g & 1) << 15; + } + if (ar == 0) /* a 0 value means unusable */ + ar = AR_UNUSABLE_MASK; + + return ar; +} + +static void vmx_set_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; + u32 ar; + + if (vcpu->arch.rmode.active && seg == VCPU_SREG_TR) { + vcpu->arch.rmode.tr.selector = var->selector; + vcpu->arch.rmode.tr.base = var->base; + vcpu->arch.rmode.tr.limit = var->limit; + vcpu->arch.rmode.tr.ar = vmx_segment_access_rights(var); + return; + } + vmcs_writel(sf->base, var->base); + vmcs_write32(sf->limit, var->limit); + vmcs_write16(sf->selector, var->selector); + if (vcpu->arch.rmode.active && var->s) { + /* + * Hack real-mode segments into vm86 compatibility. + */ + if (var->base == 0xffff0000 && var->selector == 0xf000) + vmcs_writel(sf->base, 0xf0000); + ar = 0xf3; + } else + ar = vmx_segment_access_rights(var); + vmcs_write32(sf->ar_bytes, ar); +} + +static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) +{ + u32 ar = vmcs_read32(GUEST_CS_AR_BYTES); + + *db = (ar >> 14) & 1; + *l = (ar >> 13) & 1; +} + +static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) +{ + dt->limit = vmcs_read32(GUEST_IDTR_LIMIT); + dt->base = vmcs_readl(GUEST_IDTR_BASE); +} + +static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) +{ + vmcs_write32(GUEST_IDTR_LIMIT, dt->limit); + vmcs_writel(GUEST_IDTR_BASE, dt->base); +} + +static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) +{ + dt->limit = vmcs_read32(GUEST_GDTR_LIMIT); + dt->base = vmcs_readl(GUEST_GDTR_BASE); +} + +static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) +{ + vmcs_write32(GUEST_GDTR_LIMIT, dt->limit); + vmcs_writel(GUEST_GDTR_BASE, dt->base); +} + +static int init_rmode_tss(struct kvm *kvm) +{ + gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT; + u16 data = 0; + int r; + + r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); + if (r < 0) + return 0; + data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; + r = kvm_write_guest_page(kvm, fn++, &data, 0x66, sizeof(u16)); + if (r < 0) + return 0; + r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE); + if (r < 0) + return 0; + r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); + if (r < 0) + return 0; + data = ~0; + r = kvm_write_guest_page(kvm, fn, &data, RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1, + sizeof(u8)); + if (r < 0) + return 0; + return 1; +} + +static void seg_setup(int seg) +{ + struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; + + vmcs_write16(sf->selector, 0); + vmcs_writel(sf->base, 0); + vmcs_write32(sf->limit, 0xffff); + vmcs_write32(sf->ar_bytes, 0x93); +} + +static int alloc_apic_access_page(struct kvm *kvm) +{ + struct kvm_userspace_memory_region kvm_userspace_mem; + int r = 0; + + mutex_lock(&kvm->lock); + if (kvm->arch.apic_access_page) + goto out; + kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT; + kvm_userspace_mem.flags = 0; + kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL; + kvm_userspace_mem.memory_size = PAGE_SIZE; + r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0); + if (r) + goto out; + kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00); +out: + mutex_unlock(&kvm->lock); + return r; +} + +/* + * Sets up the vmcs for emulated real mode. + */ +static int vmx_vcpu_setup(struct vcpu_vmx *vmx) +{ + u32 host_sysenter_cs; + u32 junk; + unsigned long a; + struct descriptor_table dt; + int i; + unsigned long kvm_vmx_return; + u32 exec_control; + + /* I/O */ + vmcs_write64(IO_BITMAP_A, page_to_phys(vmx_io_bitmap_a)); + vmcs_write64(IO_BITMAP_B, page_to_phys(vmx_io_bitmap_b)); + + vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */ + + /* Control */ + vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, + vmcs_config.pin_based_exec_ctrl); + + exec_control = vmcs_config.cpu_based_exec_ctrl; + if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) { + exec_control &= ~CPU_BASED_TPR_SHADOW; +#ifdef CONFIG_X86_64 + exec_control |= CPU_BASED_CR8_STORE_EXITING | + CPU_BASED_CR8_LOAD_EXITING; +#endif + } + vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); + + if (cpu_has_secondary_exec_ctrls()) { + exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; + if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + exec_control &= + ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; + vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); + } + + vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); + vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); + vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ + + vmcs_writel(HOST_CR0, read_cr0()); /* 22.2.3 */ + vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */ + vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */ + + vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ + vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ + vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ + vmcs_write16(HOST_FS_SELECTOR, read_fs()); /* 22.2.4 */ + vmcs_write16(HOST_GS_SELECTOR, read_gs()); /* 22.2.4 */ + vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ +#ifdef CONFIG_X86_64 + rdmsrl(MSR_FS_BASE, a); + vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */ + rdmsrl(MSR_GS_BASE, a); + vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */ +#else + vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ + vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ +#endif + + vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ + + get_idt(&dt); + vmcs_writel(HOST_IDTR_BASE, dt.base); /* 22.2.4 */ + + asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return)); + vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */ + vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); + vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); + vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); + + rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk); + vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs); + rdmsrl(MSR_IA32_SYSENTER_ESP, a); + vmcs_writel(HOST_IA32_SYSENTER_ESP, a); /* 22.2.3 */ + rdmsrl(MSR_IA32_SYSENTER_EIP, a); + vmcs_writel(HOST_IA32_SYSENTER_EIP, a); /* 22.2.3 */ + + for (i = 0; i < NR_VMX_MSR; ++i) { + u32 index = vmx_msr_index[i]; + u32 data_low, data_high; + u64 data; + int j = vmx->nmsrs; + + if (rdmsr_safe(index, &data_low, &data_high) < 0) + continue; + if (wrmsr_safe(index, data_low, data_high) < 0) + continue; + data = data_low | ((u64)data_high << 32); + vmx->host_msrs[j].index = index; + vmx->host_msrs[j].reserved = 0; + vmx->host_msrs[j].data = data; + vmx->guest_msrs[j] = vmx->host_msrs[j]; + ++vmx->nmsrs; + } + + vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl); + + /* 22.2.1, 20.8.1 */ + vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl); + + vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); + vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); + + if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + if (alloc_apic_access_page(vmx->vcpu.kvm) != 0) + return -ENOMEM; + + return 0; +} + +static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u64 msr; + int ret; + + if (!init_rmode_tss(vmx->vcpu.kvm)) { + ret = -ENOMEM; + goto out; + } + + vmx->vcpu.arch.rmode.active = 0; + + vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val(); + set_cr8(&vmx->vcpu, 0); + msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; + if (vmx->vcpu.vcpu_id == 0) + msr |= MSR_IA32_APICBASE_BSP; + kvm_set_apic_base(&vmx->vcpu, msr); + + fx_init(&vmx->vcpu); + + /* + * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode + * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh. + */ + if (vmx->vcpu.vcpu_id == 0) { + vmcs_write16(GUEST_CS_SELECTOR, 0xf000); + vmcs_writel(GUEST_CS_BASE, 0x000f0000); + } else { + vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8); + vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12); + } + vmcs_write32(GUEST_CS_LIMIT, 0xffff); + vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); + + seg_setup(VCPU_SREG_DS); + seg_setup(VCPU_SREG_ES); + seg_setup(VCPU_SREG_FS); + seg_setup(VCPU_SREG_GS); + seg_setup(VCPU_SREG_SS); + + vmcs_write16(GUEST_TR_SELECTOR, 0); + vmcs_writel(GUEST_TR_BASE, 0); + vmcs_write32(GUEST_TR_LIMIT, 0xffff); + vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); + + vmcs_write16(GUEST_LDTR_SELECTOR, 0); + vmcs_writel(GUEST_LDTR_BASE, 0); + vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); + vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); + + vmcs_write32(GUEST_SYSENTER_CS, 0); + vmcs_writel(GUEST_SYSENTER_ESP, 0); + vmcs_writel(GUEST_SYSENTER_EIP, 0); + + vmcs_writel(GUEST_RFLAGS, 0x02); + if (vmx->vcpu.vcpu_id == 0) + vmcs_writel(GUEST_RIP, 0xfff0); + else + vmcs_writel(GUEST_RIP, 0); + vmcs_writel(GUEST_RSP, 0); + + /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ + vmcs_writel(GUEST_DR7, 0x400); + + vmcs_writel(GUEST_GDTR_BASE, 0); + vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); + + vmcs_writel(GUEST_IDTR_BASE, 0); + vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); + + vmcs_write32(GUEST_ACTIVITY_STATE, 0); + vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); + vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); + + guest_write_tsc(0); + + /* Special registers */ + vmcs_write64(GUEST_IA32_DEBUGCTL, 0); + + setup_msrs(vmx); + + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ + + if (cpu_has_vmx_tpr_shadow()) { + vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); + if (vm_need_tpr_shadow(vmx->vcpu.kvm)) + vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, + page_to_phys(vmx->vcpu.arch.apic->regs_page)); + vmcs_write32(TPR_THRESHOLD, 0); + } + + if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) + vmcs_write64(APIC_ACCESS_ADDR, + page_to_phys(vmx->vcpu.kvm->arch.apic_access_page)); + + vmx->vcpu.arch.cr0 = 0x60000010; + vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */ + vmx_set_cr4(&vmx->vcpu, 0); +#ifdef CONFIG_X86_64 + vmx_set_efer(&vmx->vcpu, 0); +#endif + vmx_fpu_activate(&vmx->vcpu); + update_exception_bitmap(&vmx->vcpu); + + return 0; + +out: + return ret; +} + +static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + if (vcpu->arch.rmode.active) { + vmx->rmode.irq.pending = true; + vmx->rmode.irq.vector = irq; + vmx->rmode.irq.rip = vmcs_readl(GUEST_RIP); + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK); + vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); + vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip - 1); + return; + } + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, + irq | INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK); +} + +static void kvm_do_inject_irq(struct kvm_vcpu *vcpu) +{ + int word_index = __ffs(vcpu->arch.irq_summary); + int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); + int irq = word_index * BITS_PER_LONG + bit_index; + + clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); + if (!vcpu->arch.irq_pending[word_index]) + clear_bit(word_index, &vcpu->arch.irq_summary); + vmx_inject_irq(vcpu, irq); +} + + +static void do_interrupt_requests(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + u32 cpu_based_vm_exec_control; + + vcpu->arch.interrupt_window_open = + ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && + (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0); + + if (vcpu->arch.interrupt_window_open && + vcpu->arch.irq_summary && + !(vmcs_read32(VM_ENTRY_INTR_INFO_FIELD) & INTR_INFO_VALID_MASK)) + /* + * If interrupts enabled, and not blocked by sti or mov ss. Good. + */ + kvm_do_inject_irq(vcpu); + + cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); + if (!vcpu->arch.interrupt_window_open && + (vcpu->arch.irq_summary || kvm_run->request_interrupt_window)) + /* + * Interrupts blocked. Wait for unblock. + */ + cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING; + else + cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING; + vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); +} + +static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) +{ + int ret; + struct kvm_userspace_memory_region tss_mem = { + .slot = 8, + .guest_phys_addr = addr, + .memory_size = PAGE_SIZE * 3, + .flags = 0, + }; + + ret = kvm_set_memory_region(kvm, &tss_mem, 0); + if (ret) + return ret; + kvm->arch.tss_addr = addr; + return 0; +} + +static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu) +{ + struct kvm_guest_debug *dbg = &vcpu->guest_debug; + + set_debugreg(dbg->bp[0], 0); + set_debugreg(dbg->bp[1], 1); + set_debugreg(dbg->bp[2], 2); + set_debugreg(dbg->bp[3], 3); + + if (dbg->singlestep) { + unsigned long flags; + + flags = vmcs_readl(GUEST_RFLAGS); + flags |= X86_EFLAGS_TF | X86_EFLAGS_RF; + vmcs_writel(GUEST_RFLAGS, flags); + } +} + +static int handle_rmode_exception(struct kvm_vcpu *vcpu, + int vec, u32 err_code) +{ + if (!vcpu->arch.rmode.active) + return 0; + + /* + * Instruction with address size override prefix opcode 0x67 + * Cause the #SS fault with 0 error code in VM86 mode. + */ + if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) + if (emulate_instruction(vcpu, NULL, 0, 0, 0) == EMULATE_DONE) + return 1; + return 0; +} + +static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 intr_info, error_code; + unsigned long cr2, rip; + u32 vect_info; + enum emulation_result er; + + vect_info = vmx->idt_vectoring_info; + intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + + if ((vect_info & VECTORING_INFO_VALID_MASK) && + !is_page_fault(intr_info)) + printk(KERN_ERR "%s: unexpected, vectoring info 0x%x " + "intr info 0x%x\n", __FUNCTION__, vect_info, intr_info); + + if (!irqchip_in_kernel(vcpu->kvm) && is_external_interrupt(vect_info)) { + int irq = vect_info & VECTORING_INFO_VECTOR_MASK; + set_bit(irq, vcpu->arch.irq_pending); + set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary); + } + + if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) /* nmi */ + return 1; /* already handled by vmx_vcpu_run() */ + + if (is_no_device(intr_info)) { + vmx_fpu_activate(vcpu); + return 1; + } + + if (is_invalid_opcode(intr_info)) { + er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); + if (er != EMULATE_DONE) + kvm_queue_exception(vcpu, UD_VECTOR); + return 1; + } + + error_code = 0; + rip = vmcs_readl(GUEST_RIP); + if (intr_info & INTR_INFO_DELIEVER_CODE_MASK) + error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); + if (is_page_fault(intr_info)) { + cr2 = vmcs_readl(EXIT_QUALIFICATION); + return kvm_mmu_page_fault(vcpu, cr2, error_code); + } + + if (vcpu->arch.rmode.active && + handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK, + error_code)) { + if (vcpu->arch.halt_request) { + vcpu->arch.halt_request = 0; + return kvm_emulate_halt(vcpu); + } + return 1; + } + + if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == + (INTR_TYPE_EXCEPTION | 1)) { + kvm_run->exit_reason = KVM_EXIT_DEBUG; + return 0; + } + kvm_run->exit_reason = KVM_EXIT_EXCEPTION; + kvm_run->ex.exception = intr_info & INTR_INFO_VECTOR_MASK; + kvm_run->ex.error_code = error_code; + return 0; +} + +static int handle_external_interrupt(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + ++vcpu->stat.irq_exits; + return 1; +} + +static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; + return 0; +} + +static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + unsigned long exit_qualification; + int size, down, in, string, rep; + unsigned port; + + ++vcpu->stat.io_exits; + exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + string = (exit_qualification & 16) != 0; + + if (string) { + if (emulate_instruction(vcpu, + kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) + return 0; + return 1; + } + + size = (exit_qualification & 7) + 1; + in = (exit_qualification & 8) != 0; + down = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_DF) != 0; + rep = (exit_qualification & 32) != 0; + port = exit_qualification >> 16; + + return kvm_emulate_pio(vcpu, kvm_run, in, size, port); +} + +static void +vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) +{ + /* + * Patch in the VMCALL instruction: + */ + hypercall[0] = 0x0f; + hypercall[1] = 0x01; + hypercall[2] = 0xc1; +} + +static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + unsigned long exit_qualification; + int cr; + int reg; + + exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + cr = exit_qualification & 15; + reg = (exit_qualification >> 8) & 15; + switch ((exit_qualification >> 4) & 3) { + case 0: /* mov to cr */ + switch (cr) { + case 0: + vcpu_load_rsp_rip(vcpu); + set_cr0(vcpu, vcpu->arch.regs[reg]); + skip_emulated_instruction(vcpu); + return 1; + case 3: + vcpu_load_rsp_rip(vcpu); + set_cr3(vcpu, vcpu->arch.regs[reg]); + skip_emulated_instruction(vcpu); + return 1; + case 4: + vcpu_load_rsp_rip(vcpu); + set_cr4(vcpu, vcpu->arch.regs[reg]); + skip_emulated_instruction(vcpu); + return 1; + case 8: + vcpu_load_rsp_rip(vcpu); + set_cr8(vcpu, vcpu->arch.regs[reg]); + skip_emulated_instruction(vcpu); + if (irqchip_in_kernel(vcpu->kvm)) + return 1; + kvm_run->exit_reason = KVM_EXIT_SET_TPR; + return 0; + }; + break; + case 2: /* clts */ + vcpu_load_rsp_rip(vcpu); + vmx_fpu_deactivate(vcpu); + vcpu->arch.cr0 &= ~X86_CR0_TS; + vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0); + vmx_fpu_activate(vcpu); + skip_emulated_instruction(vcpu); + return 1; + case 1: /*mov from cr*/ + switch (cr) { + case 3: + vcpu_load_rsp_rip(vcpu); + vcpu->arch.regs[reg] = vcpu->arch.cr3; + vcpu_put_rsp_rip(vcpu); + skip_emulated_instruction(vcpu); + return 1; + case 8: + vcpu_load_rsp_rip(vcpu); + vcpu->arch.regs[reg] = get_cr8(vcpu); + vcpu_put_rsp_rip(vcpu); + skip_emulated_instruction(vcpu); + return 1; + } + break; + case 3: /* lmsw */ + lmsw(vcpu, (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f); + + skip_emulated_instruction(vcpu); + return 1; + default: + break; + } + kvm_run->exit_reason = 0; + pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n", + (int)(exit_qualification >> 4) & 3, cr); + return 0; +} + +static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + unsigned long exit_qualification; + unsigned long val; + int dr, reg; + + /* + * FIXME: this code assumes the host is debugging the guest. + * need to deal with guest debugging itself too. + */ + exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + dr = exit_qualification & 7; + reg = (exit_qualification >> 8) & 15; + vcpu_load_rsp_rip(vcpu); + if (exit_qualification & 16) { + /* mov from dr */ + switch (dr) { + case 6: + val = 0xffff0ff0; + break; + case 7: + val = 0x400; + break; + default: + val = 0; + } + vcpu->arch.regs[reg] = val; + } else { + /* mov to dr */ + } + vcpu_put_rsp_rip(vcpu); + skip_emulated_instruction(vcpu); + return 1; +} + +static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + kvm_emulate_cpuid(vcpu); + return 1; +} + +static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; + u64 data; + + if (vmx_get_msr(vcpu, ecx, &data)) { + kvm_inject_gp(vcpu, 0); + return 1; + } + + /* FIXME: handling of bits 32:63 of rax, rdx */ + vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u; + vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u; + skip_emulated_instruction(vcpu); + return 1; +} + +static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; + u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u) + | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32); + + if (vmx_set_msr(vcpu, ecx, data) != 0) { + kvm_inject_gp(vcpu, 0); + return 1; + } + + skip_emulated_instruction(vcpu); + return 1; +} + +static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + return 1; +} + +static int handle_interrupt_window(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + u32 cpu_based_vm_exec_control; + + /* clear pending irq */ + cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); + cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING; + vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); + /* + * If the user space waits to inject interrupts, exit as soon as + * possible + */ + if (kvm_run->request_interrupt_window && + !vcpu->arch.irq_summary) { + kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; + ++vcpu->stat.irq_window_exits; + return 0; + } + return 1; +} + +static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + skip_emulated_instruction(vcpu); + return kvm_emulate_halt(vcpu); +} + +static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + skip_emulated_instruction(vcpu); + kvm_emulate_hypercall(vcpu); + return 1; +} + +static int handle_wbinvd(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + skip_emulated_instruction(vcpu); + /* TODO: Add support for VT-d/pass-through device */ + return 1; +} + +static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + u64 exit_qualification; + enum emulation_result er; + unsigned long offset; + + exit_qualification = vmcs_read64(EXIT_QUALIFICATION); + offset = exit_qualification & 0xffful; + + er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); + + if (er != EMULATE_DONE) { + printk(KERN_ERR + "Fail to handle apic access vmexit! Offset is 0x%lx\n", + offset); + return -ENOTSUPP; + } + return 1; +} + +/* + * The exit handlers return 1 if the exit was handled fully and guest execution + * may resume. Otherwise they set the kvm_run parameter to indicate what needs + * to be done to userspace and return 0. + */ +static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) = { + [EXIT_REASON_EXCEPTION_NMI] = handle_exception, + [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, + [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, + [EXIT_REASON_IO_INSTRUCTION] = handle_io, + [EXIT_REASON_CR_ACCESS] = handle_cr, + [EXIT_REASON_DR_ACCESS] = handle_dr, + [EXIT_REASON_CPUID] = handle_cpuid, + [EXIT_REASON_MSR_READ] = handle_rdmsr, + [EXIT_REASON_MSR_WRITE] = handle_wrmsr, + [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window, + [EXIT_REASON_HLT] = handle_halt, + [EXIT_REASON_VMCALL] = handle_vmcall, + [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, + [EXIT_REASON_APIC_ACCESS] = handle_apic_access, + [EXIT_REASON_WBINVD] = handle_wbinvd, +}; + +static const int kvm_vmx_max_exit_handlers = + ARRAY_SIZE(kvm_vmx_exit_handlers); + +/* + * The guest has exited. See if we can fix it or if we need userspace + * assistance. + */ +static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) +{ + u32 exit_reason = vmcs_read32(VM_EXIT_REASON); + struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 vectoring_info = vmx->idt_vectoring_info; + + if (unlikely(vmx->fail)) { + kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; + kvm_run->fail_entry.hardware_entry_failure_reason + = vmcs_read32(VM_INSTRUCTION_ERROR); + return 0; + } + + if ((vectoring_info & VECTORING_INFO_VALID_MASK) && + exit_reason != EXIT_REASON_EXCEPTION_NMI) + printk(KERN_WARNING "%s: unexpected, valid vectoring info and " + "exit reason is 0x%x\n", __FUNCTION__, exit_reason); + if (exit_reason < kvm_vmx_max_exit_handlers + && kvm_vmx_exit_handlers[exit_reason]) + return kvm_vmx_exit_handlers[exit_reason](vcpu, kvm_run); + else { + kvm_run->exit_reason = KVM_EXIT_UNKNOWN; + kvm_run->hw.hardware_exit_reason = exit_reason; + } + return 0; +} + +static void vmx_flush_tlb(struct kvm_vcpu *vcpu) +{ +} + +static void update_tpr_threshold(struct kvm_vcpu *vcpu) +{ + int max_irr, tpr; + + if (!vm_need_tpr_shadow(vcpu->kvm)) + return; + + if (!kvm_lapic_enabled(vcpu) || + ((max_irr = kvm_lapic_find_highest_irr(vcpu)) == -1)) { + vmcs_write32(TPR_THRESHOLD, 0); + return; + } + + tpr = (kvm_lapic_get_cr8(vcpu) & 0x0f) << 4; + vmcs_write32(TPR_THRESHOLD, (max_irr > tpr) ? tpr >> 4 : max_irr >> 4); +} + +static void enable_irq_window(struct kvm_vcpu *vcpu) +{ + u32 cpu_based_vm_exec_control; + + cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); + cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING; + vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); +} + +static void vmx_intr_assist(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 idtv_info_field, intr_info_field; + int has_ext_irq, interrupt_window_open; + int vector; + + update_tpr_threshold(vcpu); + + has_ext_irq = kvm_cpu_has_interrupt(vcpu); + intr_info_field = vmcs_read32(VM_ENTRY_INTR_INFO_FIELD); + idtv_info_field = vmx->idt_vectoring_info; + if (intr_info_field & INTR_INFO_VALID_MASK) { + if (idtv_info_field & INTR_INFO_VALID_MASK) { + /* TODO: fault when IDT_Vectoring */ + if (printk_ratelimit()) + printk(KERN_ERR "Fault when IDT_Vectoring\n"); + } + if (has_ext_irq) + enable_irq_window(vcpu); + return; + } + if (unlikely(idtv_info_field & INTR_INFO_VALID_MASK)) { + if ((idtv_info_field & VECTORING_INFO_TYPE_MASK) + == INTR_TYPE_EXT_INTR + && vcpu->arch.rmode.active) { + u8 vect = idtv_info_field & VECTORING_INFO_VECTOR_MASK; + + vmx_inject_irq(vcpu, vect); + if (unlikely(has_ext_irq)) + enable_irq_window(vcpu); + return; + } + + vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, idtv_info_field); + vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, + vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); + + if (unlikely(idtv_info_field & INTR_INFO_DELIEVER_CODE_MASK)) + vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, + vmcs_read32(IDT_VECTORING_ERROR_CODE)); + if (unlikely(has_ext_irq)) + enable_irq_window(vcpu); + return; + } + if (!has_ext_irq) + return; + interrupt_window_open = + ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && + (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0); + if (interrupt_window_open) { + vector = kvm_cpu_get_interrupt(vcpu); + vmx_inject_irq(vcpu, vector); + kvm_timer_intr_post(vcpu, vector); + } else + enable_irq_window(vcpu); +} + +/* + * Failure to inject an interrupt should give us the information + * in IDT_VECTORING_INFO_FIELD. However, if the failure occurs + * when fetching the interrupt redirection bitmap in the real-mode + * tss, this doesn't happen. So we do it ourselves. + */ +static void fixup_rmode_irq(struct vcpu_vmx *vmx) +{ + vmx->rmode.irq.pending = 0; + if (vmcs_readl(GUEST_RIP) + 1 != vmx->rmode.irq.rip) + return; + vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip); + if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) { + vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK; + vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR; + return; + } + vmx->idt_vectoring_info = + VECTORING_INFO_VALID_MASK + | INTR_TYPE_EXT_INTR + | vmx->rmode.irq.vector; +} + +static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 intr_info; + + /* + * Loading guest fpu may have cleared host cr0.ts + */ + vmcs_writel(HOST_CR0, read_cr0()); + + asm( + /* Store host registers */ +#ifdef CONFIG_X86_64 + "push %%rdx; push %%rbp;" + "push %%rcx \n\t" +#else + "push %%edx; push %%ebp;" + "push %%ecx \n\t" +#endif + ASM_VMX_VMWRITE_RSP_RDX "\n\t" + /* Check if vmlaunch of vmresume is needed */ + "cmpl $0, %c[launched](%0) \n\t" + /* Load guest registers. Don't clobber flags. */ +#ifdef CONFIG_X86_64 + "mov %c[cr2](%0), %%rax \n\t" + "mov %%rax, %%cr2 \n\t" + "mov %c[rax](%0), %%rax \n\t" + "mov %c[rbx](%0), %%rbx \n\t" + "mov %c[rdx](%0), %%rdx \n\t" + "mov %c[rsi](%0), %%rsi \n\t" + "mov %c[rdi](%0), %%rdi \n\t" + "mov %c[rbp](%0), %%rbp \n\t" + "mov %c[r8](%0), %%r8 \n\t" + "mov %c[r9](%0), %%r9 \n\t" + "mov %c[r10](%0), %%r10 \n\t" + "mov %c[r11](%0), %%r11 \n\t" + "mov %c[r12](%0), %%r12 \n\t" + "mov %c[r13](%0), %%r13 \n\t" + "mov %c[r14](%0), %%r14 \n\t" + "mov %c[r15](%0), %%r15 \n\t" + "mov %c[rcx](%0), %%rcx \n\t" /* kills %0 (rcx) */ +#else + "mov %c[cr2](%0), %%eax \n\t" + "mov %%eax, %%cr2 \n\t" + "mov %c[rax](%0), %%eax \n\t" + "mov %c[rbx](%0), %%ebx \n\t" + "mov %c[rdx](%0), %%edx \n\t" + "mov %c[rsi](%0), %%esi \n\t" + "mov %c[rdi](%0), %%edi \n\t" + "mov %c[rbp](%0), %%ebp \n\t" + "mov %c[rcx](%0), %%ecx \n\t" /* kills %0 (ecx) */ +#endif + /* Enter guest mode */ + "jne .Llaunched \n\t" + ASM_VMX_VMLAUNCH "\n\t" + "jmp .Lkvm_vmx_return \n\t" + ".Llaunched: " ASM_VMX_VMRESUME "\n\t" + ".Lkvm_vmx_return: " + /* Save guest registers, load host registers, keep flags */ +#ifdef CONFIG_X86_64 + "xchg %0, (%%rsp) \n\t" + "mov %%rax, %c[rax](%0) \n\t" + "mov %%rbx, %c[rbx](%0) \n\t" + "pushq (%%rsp); popq %c[rcx](%0) \n\t" + "mov %%rdx, %c[rdx](%0) \n\t" + "mov %%rsi, %c[rsi](%0) \n\t" + "mov %%rdi, %c[rdi](%0) \n\t" + "mov %%rbp, %c[rbp](%0) \n\t" + "mov %%r8, %c[r8](%0) \n\t" + "mov %%r9, %c[r9](%0) \n\t" + "mov %%r10, %c[r10](%0) \n\t" + "mov %%r11, %c[r11](%0) \n\t" + "mov %%r12, %c[r12](%0) \n\t" + "mov %%r13, %c[r13](%0) \n\t" + "mov %%r14, %c[r14](%0) \n\t" + "mov %%r15, %c[r15](%0) \n\t" + "mov %%cr2, %%rax \n\t" + "mov %%rax, %c[cr2](%0) \n\t" + + "pop %%rbp; pop %%rbp; pop %%rdx \n\t" +#else + "xchg %0, (%%esp) \n\t" + "mov %%eax, %c[rax](%0) \n\t" + "mov %%ebx, %c[rbx](%0) \n\t" + "pushl (%%esp); popl %c[rcx](%0) \n\t" + "mov %%edx, %c[rdx](%0) \n\t" + "mov %%esi, %c[rsi](%0) \n\t" + "mov %%edi, %c[rdi](%0) \n\t" + "mov %%ebp, %c[rbp](%0) \n\t" + "mov %%cr2, %%eax \n\t" + "mov %%eax, %c[cr2](%0) \n\t" + + "pop %%ebp; pop %%ebp; pop %%edx \n\t" +#endif + "setbe %c[fail](%0) \n\t" + : : "c"(vmx), "d"((unsigned long)HOST_RSP), + [launched]"i"(offsetof(struct vcpu_vmx, launched)), + [fail]"i"(offsetof(struct vcpu_vmx, fail)), + [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])), + [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])), + [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])), + [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])), + [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])), + [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])), + [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])), +#ifdef CONFIG_X86_64 + [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])), + [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])), + [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])), + [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])), + [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])), + [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])), + [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])), + [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])), +#endif + [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)) + : "cc", "memory" +#ifdef CONFIG_X86_64 + , "rbx", "rdi", "rsi" + , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" +#else + , "ebx", "edi", "rsi" +#endif + ); + + vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + if (vmx->rmode.irq.pending) + fixup_rmode_irq(vmx); + + vcpu->arch.interrupt_window_open = + (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; + + asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS)); + vmx->launched = 1; + + intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + + /* We need to handle NMIs before interrupts are enabled */ + if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) /* nmi */ + asm("int $2"); +} + +static void vmx_free_vmcs(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + if (vmx->vmcs) { + on_each_cpu(__vcpu_clear, vmx, 0, 1); + free_vmcs(vmx->vmcs); + vmx->vmcs = NULL; + } +} + +static void vmx_free_vcpu(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + vmx_free_vmcs(vcpu); + kfree(vmx->host_msrs); + kfree(vmx->guest_msrs); + kvm_vcpu_uninit(vcpu); + kmem_cache_free(kvm_vcpu_cache, vmx); +} + +static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) +{ + int err; + struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); + int cpu; + + if (!vmx) + return ERR_PTR(-ENOMEM); + + err = kvm_vcpu_init(&vmx->vcpu, kvm, id); + if (err) + goto free_vcpu; + + vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!vmx->guest_msrs) { + err = -ENOMEM; + goto uninit_vcpu; + } + + vmx->host_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!vmx->host_msrs) + goto free_guest_msrs; + + vmx->vmcs = alloc_vmcs(); + if (!vmx->vmcs) + goto free_msrs; + + vmcs_clear(vmx->vmcs); + + cpu = get_cpu(); + vmx_vcpu_load(&vmx->vcpu, cpu); + err = vmx_vcpu_setup(vmx); + vmx_vcpu_put(&vmx->vcpu); + put_cpu(); + if (err) + goto free_vmcs; + + return &vmx->vcpu; + +free_vmcs: + free_vmcs(vmx->vmcs); +free_msrs: + kfree(vmx->host_msrs); +free_guest_msrs: + kfree(vmx->guest_msrs); +uninit_vcpu: + kvm_vcpu_uninit(&vmx->vcpu); +free_vcpu: + kmem_cache_free(kvm_vcpu_cache, vmx); + return ERR_PTR(err); +} + +static void __init vmx_check_processor_compat(void *rtn) +{ + struct vmcs_config vmcs_conf; + + *(int *)rtn = 0; + if (setup_vmcs_config(&vmcs_conf) < 0) + *(int *)rtn = -EIO; + if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) { + printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n", + smp_processor_id()); + *(int *)rtn = -EIO; + } +} + +static struct kvm_x86_ops vmx_x86_ops = { + .cpu_has_kvm_support = cpu_has_kvm_support, + .disabled_by_bios = vmx_disabled_by_bios, + .hardware_setup = hardware_setup, + .hardware_unsetup = hardware_unsetup, + .check_processor_compatibility = vmx_check_processor_compat, + .hardware_enable = hardware_enable, + .hardware_disable = hardware_disable, + + .vcpu_create = vmx_create_vcpu, + .vcpu_free = vmx_free_vcpu, + .vcpu_reset = vmx_vcpu_reset, + + .prepare_guest_switch = vmx_save_host_state, + .vcpu_load = vmx_vcpu_load, + .vcpu_put = vmx_vcpu_put, + .vcpu_decache = vmx_vcpu_decache, + + .set_guest_debug = set_guest_debug, + .guest_debug_pre = kvm_guest_debug_pre, + .get_msr = vmx_get_msr, + .set_msr = vmx_set_msr, + .get_segment_base = vmx_get_segment_base, + .get_segment = vmx_get_segment, + .set_segment = vmx_set_segment, + .get_cs_db_l_bits = vmx_get_cs_db_l_bits, + .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits, + .set_cr0 = vmx_set_cr0, + .set_cr3 = vmx_set_cr3, + .set_cr4 = vmx_set_cr4, +#ifdef CONFIG_X86_64 + .set_efer = vmx_set_efer, +#endif + .get_idt = vmx_get_idt, + .set_idt = vmx_set_idt, + .get_gdt = vmx_get_gdt, + .set_gdt = vmx_set_gdt, + .cache_regs = vcpu_load_rsp_rip, + .decache_regs = vcpu_put_rsp_rip, + .get_rflags = vmx_get_rflags, + .set_rflags = vmx_set_rflags, + + .tlb_flush = vmx_flush_tlb, + + .run = vmx_vcpu_run, + .handle_exit = kvm_handle_exit, + .skip_emulated_instruction = skip_emulated_instruction, + .patch_hypercall = vmx_patch_hypercall, + .get_irq = vmx_get_irq, + .set_irq = vmx_inject_irq, + .queue_exception = vmx_queue_exception, + .exception_injected = vmx_exception_injected, + .inject_pending_irq = vmx_intr_assist, + .inject_pending_vectors = do_interrupt_requests, + + .set_tss_addr = vmx_set_tss_addr, +}; + +static int __init vmx_init(void) +{ + void *iova; + int r; + + vmx_io_bitmap_a = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); + if (!vmx_io_bitmap_a) + return -ENOMEM; + + vmx_io_bitmap_b = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); + if (!vmx_io_bitmap_b) { + r = -ENOMEM; + goto out; + } + + /* + * Allow direct access to the PC debug port (it is often used for I/O + * delays, but the vmexits simply slow things down). + */ + iova = kmap(vmx_io_bitmap_a); + memset(iova, 0xff, PAGE_SIZE); + clear_bit(0x80, iova); + kunmap(vmx_io_bitmap_a); + + iova = kmap(vmx_io_bitmap_b); + memset(iova, 0xff, PAGE_SIZE); + kunmap(vmx_io_bitmap_b); + + r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE); + if (r) + goto out1; + + if (bypass_guest_pf) + kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull); + + return 0; + +out1: + __free_page(vmx_io_bitmap_b); +out: + __free_page(vmx_io_bitmap_a); + return r; +} + +static void __exit vmx_exit(void) +{ + __free_page(vmx_io_bitmap_b); + __free_page(vmx_io_bitmap_a); + + kvm_exit(); +} + +module_init(vmx_init) +module_exit(vmx_exit) diff --git a/arch/x86/kvm/vmx.h b/arch/x86/kvm/vmx.h new file mode 100644 index 000000000000..d52ae8d7303d --- /dev/null +++ b/arch/x86/kvm/vmx.h @@ -0,0 +1,324 @@ +#ifndef VMX_H +#define VMX_H + +/* + * vmx.h: VMX Architecture related definitions + * Copyright (c) 2004, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + * + * A few random additions are: + * Copyright (C) 2006 Qumranet + * Avi Kivity + * Yaniv Kamay + * + */ + +/* + * Definitions of Primary Processor-Based VM-Execution Controls. + */ +#define CPU_BASED_VIRTUAL_INTR_PENDING 0x00000004 +#define CPU_BASED_USE_TSC_OFFSETING 0x00000008 +#define CPU_BASED_HLT_EXITING 0x00000080 +#define CPU_BASED_INVLPG_EXITING 0x00000200 +#define CPU_BASED_MWAIT_EXITING 0x00000400 +#define CPU_BASED_RDPMC_EXITING 0x00000800 +#define CPU_BASED_RDTSC_EXITING 0x00001000 +#define CPU_BASED_CR8_LOAD_EXITING 0x00080000 +#define CPU_BASED_CR8_STORE_EXITING 0x00100000 +#define CPU_BASED_TPR_SHADOW 0x00200000 +#define CPU_BASED_MOV_DR_EXITING 0x00800000 +#define CPU_BASED_UNCOND_IO_EXITING 0x01000000 +#define CPU_BASED_USE_IO_BITMAPS 0x02000000 +#define CPU_BASED_USE_MSR_BITMAPS 0x10000000 +#define CPU_BASED_MONITOR_EXITING 0x20000000 +#define CPU_BASED_PAUSE_EXITING 0x40000000 +#define CPU_BASED_ACTIVATE_SECONDARY_CONTROLS 0x80000000 +/* + * Definitions of Secondary Processor-Based VM-Execution Controls. + */ +#define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 +#define SECONDARY_EXEC_WBINVD_EXITING 0x00000040 + + +#define PIN_BASED_EXT_INTR_MASK 0x00000001 +#define PIN_BASED_NMI_EXITING 0x00000008 +#define PIN_BASED_VIRTUAL_NMIS 0x00000020 + +#define VM_EXIT_HOST_ADDR_SPACE_SIZE 0x00000200 +#define VM_EXIT_ACK_INTR_ON_EXIT 0x00008000 + +#define VM_ENTRY_IA32E_MODE 0x00000200 +#define VM_ENTRY_SMM 0x00000400 +#define VM_ENTRY_DEACT_DUAL_MONITOR 0x00000800 + +/* VMCS Encodings */ +enum vmcs_field { + GUEST_ES_SELECTOR = 0x00000800, + GUEST_CS_SELECTOR = 0x00000802, + GUEST_SS_SELECTOR = 0x00000804, + GUEST_DS_SELECTOR = 0x00000806, + GUEST_FS_SELECTOR = 0x00000808, + GUEST_GS_SELECTOR = 0x0000080a, + GUEST_LDTR_SELECTOR = 0x0000080c, + GUEST_TR_SELECTOR = 0x0000080e, + HOST_ES_SELECTOR = 0x00000c00, + HOST_CS_SELECTOR = 0x00000c02, + HOST_SS_SELECTOR = 0x00000c04, + HOST_DS_SELECTOR = 0x00000c06, + HOST_FS_SELECTOR = 0x00000c08, + HOST_GS_SELECTOR = 0x00000c0a, + HOST_TR_SELECTOR = 0x00000c0c, + IO_BITMAP_A = 0x00002000, + IO_BITMAP_A_HIGH = 0x00002001, + IO_BITMAP_B = 0x00002002, + IO_BITMAP_B_HIGH = 0x00002003, + MSR_BITMAP = 0x00002004, + MSR_BITMAP_HIGH = 0x00002005, + VM_EXIT_MSR_STORE_ADDR = 0x00002006, + VM_EXIT_MSR_STORE_ADDR_HIGH = 0x00002007, + VM_EXIT_MSR_LOAD_ADDR = 0x00002008, + VM_EXIT_MSR_LOAD_ADDR_HIGH = 0x00002009, + VM_ENTRY_MSR_LOAD_ADDR = 0x0000200a, + VM_ENTRY_MSR_LOAD_ADDR_HIGH = 0x0000200b, + TSC_OFFSET = 0x00002010, + TSC_OFFSET_HIGH = 0x00002011, + VIRTUAL_APIC_PAGE_ADDR = 0x00002012, + VIRTUAL_APIC_PAGE_ADDR_HIGH = 0x00002013, + APIC_ACCESS_ADDR = 0x00002014, + APIC_ACCESS_ADDR_HIGH = 0x00002015, + VMCS_LINK_POINTER = 0x00002800, + VMCS_LINK_POINTER_HIGH = 0x00002801, + GUEST_IA32_DEBUGCTL = 0x00002802, + GUEST_IA32_DEBUGCTL_HIGH = 0x00002803, + PIN_BASED_VM_EXEC_CONTROL = 0x00004000, + CPU_BASED_VM_EXEC_CONTROL = 0x00004002, + EXCEPTION_BITMAP = 0x00004004, + PAGE_FAULT_ERROR_CODE_MASK = 0x00004006, + PAGE_FAULT_ERROR_CODE_MATCH = 0x00004008, + CR3_TARGET_COUNT = 0x0000400a, + VM_EXIT_CONTROLS = 0x0000400c, + VM_EXIT_MSR_STORE_COUNT = 0x0000400e, + VM_EXIT_MSR_LOAD_COUNT = 0x00004010, + VM_ENTRY_CONTROLS = 0x00004012, + VM_ENTRY_MSR_LOAD_COUNT = 0x00004014, + VM_ENTRY_INTR_INFO_FIELD = 0x00004016, + VM_ENTRY_EXCEPTION_ERROR_CODE = 0x00004018, + VM_ENTRY_INSTRUCTION_LEN = 0x0000401a, + TPR_THRESHOLD = 0x0000401c, + SECONDARY_VM_EXEC_CONTROL = 0x0000401e, + VM_INSTRUCTION_ERROR = 0x00004400, + VM_EXIT_REASON = 0x00004402, + VM_EXIT_INTR_INFO = 0x00004404, + VM_EXIT_INTR_ERROR_CODE = 0x00004406, + IDT_VECTORING_INFO_FIELD = 0x00004408, + IDT_VECTORING_ERROR_CODE = 0x0000440a, + VM_EXIT_INSTRUCTION_LEN = 0x0000440c, + VMX_INSTRUCTION_INFO = 0x0000440e, + GUEST_ES_LIMIT = 0x00004800, + GUEST_CS_LIMIT = 0x00004802, + GUEST_SS_LIMIT = 0x00004804, + GUEST_DS_LIMIT = 0x00004806, + GUEST_FS_LIMIT = 0x00004808, + GUEST_GS_LIMIT = 0x0000480a, + GUEST_LDTR_LIMIT = 0x0000480c, + GUEST_TR_LIMIT = 0x0000480e, + GUEST_GDTR_LIMIT = 0x00004810, + GUEST_IDTR_LIMIT = 0x00004812, + GUEST_ES_AR_BYTES = 0x00004814, + GUEST_CS_AR_BYTES = 0x00004816, + GUEST_SS_AR_BYTES = 0x00004818, + GUEST_DS_AR_BYTES = 0x0000481a, + GUEST_FS_AR_BYTES = 0x0000481c, + GUEST_GS_AR_BYTES = 0x0000481e, + GUEST_LDTR_AR_BYTES = 0x00004820, + GUEST_TR_AR_BYTES = 0x00004822, + GUEST_INTERRUPTIBILITY_INFO = 0x00004824, + GUEST_ACTIVITY_STATE = 0X00004826, + GUEST_SYSENTER_CS = 0x0000482A, + HOST_IA32_SYSENTER_CS = 0x00004c00, + CR0_GUEST_HOST_MASK = 0x00006000, + CR4_GUEST_HOST_MASK = 0x00006002, + CR0_READ_SHADOW = 0x00006004, + CR4_READ_SHADOW = 0x00006006, + CR3_TARGET_VALUE0 = 0x00006008, + CR3_TARGET_VALUE1 = 0x0000600a, + CR3_TARGET_VALUE2 = 0x0000600c, + CR3_TARGET_VALUE3 = 0x0000600e, + EXIT_QUALIFICATION = 0x00006400, + GUEST_LINEAR_ADDRESS = 0x0000640a, + GUEST_CR0 = 0x00006800, + GUEST_CR3 = 0x00006802, + GUEST_CR4 = 0x00006804, + GUEST_ES_BASE = 0x00006806, + GUEST_CS_BASE = 0x00006808, + GUEST_SS_BASE = 0x0000680a, + GUEST_DS_BASE = 0x0000680c, + GUEST_FS_BASE = 0x0000680e, + GUEST_GS_BASE = 0x00006810, + GUEST_LDTR_BASE = 0x00006812, + GUEST_TR_BASE = 0x00006814, + GUEST_GDTR_BASE = 0x00006816, + GUEST_IDTR_BASE = 0x00006818, + GUEST_DR7 = 0x0000681a, + GUEST_RSP = 0x0000681c, + GUEST_RIP = 0x0000681e, + GUEST_RFLAGS = 0x00006820, + GUEST_PENDING_DBG_EXCEPTIONS = 0x00006822, + GUEST_SYSENTER_ESP = 0x00006824, + GUEST_SYSENTER_EIP = 0x00006826, + HOST_CR0 = 0x00006c00, + HOST_CR3 = 0x00006c02, + HOST_CR4 = 0x00006c04, + HOST_FS_BASE = 0x00006c06, + HOST_GS_BASE = 0x00006c08, + HOST_TR_BASE = 0x00006c0a, + HOST_GDTR_BASE = 0x00006c0c, + HOST_IDTR_BASE = 0x00006c0e, + HOST_IA32_SYSENTER_ESP = 0x00006c10, + HOST_IA32_SYSENTER_EIP = 0x00006c12, + HOST_RSP = 0x00006c14, + HOST_RIP = 0x00006c16, +}; + +#define VMX_EXIT_REASONS_FAILED_VMENTRY 0x80000000 + +#define EXIT_REASON_EXCEPTION_NMI 0 +#define EXIT_REASON_EXTERNAL_INTERRUPT 1 +#define EXIT_REASON_TRIPLE_FAULT 2 + +#define EXIT_REASON_PENDING_INTERRUPT 7 + +#define EXIT_REASON_TASK_SWITCH 9 +#define EXIT_REASON_CPUID 10 +#define EXIT_REASON_HLT 12 +#define EXIT_REASON_INVLPG 14 +#define EXIT_REASON_RDPMC 15 +#define EXIT_REASON_RDTSC 16 +#define EXIT_REASON_VMCALL 18 +#define EXIT_REASON_VMCLEAR 19 +#define EXIT_REASON_VMLAUNCH 20 +#define EXIT_REASON_VMPTRLD 21 +#define EXIT_REASON_VMPTRST 22 +#define EXIT_REASON_VMREAD 23 +#define EXIT_REASON_VMRESUME 24 +#define EXIT_REASON_VMWRITE 25 +#define EXIT_REASON_VMOFF 26 +#define EXIT_REASON_VMON 27 +#define EXIT_REASON_CR_ACCESS 28 +#define EXIT_REASON_DR_ACCESS 29 +#define EXIT_REASON_IO_INSTRUCTION 30 +#define EXIT_REASON_MSR_READ 31 +#define EXIT_REASON_MSR_WRITE 32 +#define EXIT_REASON_MWAIT_INSTRUCTION 36 +#define EXIT_REASON_TPR_BELOW_THRESHOLD 43 +#define EXIT_REASON_APIC_ACCESS 44 +#define EXIT_REASON_WBINVD 54 + +/* + * Interruption-information format + */ +#define INTR_INFO_VECTOR_MASK 0xff /* 7:0 */ +#define INTR_INFO_INTR_TYPE_MASK 0x700 /* 10:8 */ +#define INTR_INFO_DELIEVER_CODE_MASK 0x800 /* 11 */ +#define INTR_INFO_VALID_MASK 0x80000000 /* 31 */ + +#define VECTORING_INFO_VECTOR_MASK INTR_INFO_VECTOR_MASK +#define VECTORING_INFO_TYPE_MASK INTR_INFO_INTR_TYPE_MASK +#define VECTORING_INFO_DELIEVER_CODE_MASK INTR_INFO_DELIEVER_CODE_MASK +#define VECTORING_INFO_VALID_MASK INTR_INFO_VALID_MASK + +#define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ +#define INTR_TYPE_EXCEPTION (3 << 8) /* processor exception */ +#define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ + +/* + * Exit Qualifications for MOV for Control Register Access + */ +#define CONTROL_REG_ACCESS_NUM 0x7 /* 2:0, number of control reg.*/ +#define CONTROL_REG_ACCESS_TYPE 0x30 /* 5:4, access type */ +#define CONTROL_REG_ACCESS_REG 0xf00 /* 10:8, general purpose reg. */ +#define LMSW_SOURCE_DATA_SHIFT 16 +#define LMSW_SOURCE_DATA (0xFFFF << LMSW_SOURCE_DATA_SHIFT) /* 16:31 lmsw source */ +#define REG_EAX (0 << 8) +#define REG_ECX (1 << 8) +#define REG_EDX (2 << 8) +#define REG_EBX (3 << 8) +#define REG_ESP (4 << 8) +#define REG_EBP (5 << 8) +#define REG_ESI (6 << 8) +#define REG_EDI (7 << 8) +#define REG_R8 (8 << 8) +#define REG_R9 (9 << 8) +#define REG_R10 (10 << 8) +#define REG_R11 (11 << 8) +#define REG_R12 (12 << 8) +#define REG_R13 (13 << 8) +#define REG_R14 (14 << 8) +#define REG_R15 (15 << 8) + +/* + * Exit Qualifications for MOV for Debug Register Access + */ +#define DEBUG_REG_ACCESS_NUM 0x7 /* 2:0, number of debug reg. */ +#define DEBUG_REG_ACCESS_TYPE 0x10 /* 4, direction of access */ +#define TYPE_MOV_TO_DR (0 << 4) +#define TYPE_MOV_FROM_DR (1 << 4) +#define DEBUG_REG_ACCESS_REG 0xf00 /* 11:8, general purpose reg. */ + + +/* segment AR */ +#define SEGMENT_AR_L_MASK (1 << 13) + +#define AR_TYPE_ACCESSES_MASK 1 +#define AR_TYPE_READABLE_MASK (1 << 1) +#define AR_TYPE_WRITEABLE_MASK (1 << 2) +#define AR_TYPE_CODE_MASK (1 << 3) +#define AR_TYPE_MASK 0x0f +#define AR_TYPE_BUSY_64_TSS 11 +#define AR_TYPE_BUSY_32_TSS 11 +#define AR_TYPE_BUSY_16_TSS 3 +#define AR_TYPE_LDT 2 + +#define AR_UNUSABLE_MASK (1 << 16) +#define AR_S_MASK (1 << 4) +#define AR_P_MASK (1 << 7) +#define AR_L_MASK (1 << 13) +#define AR_DB_MASK (1 << 14) +#define AR_G_MASK (1 << 15) +#define AR_DPL_SHIFT 5 +#define AR_DPL(ar) (((ar) >> AR_DPL_SHIFT) & 3) + +#define AR_RESERVD_MASK 0xfffe0f00 + +#define MSR_IA32_VMX_BASIC 0x480 +#define MSR_IA32_VMX_PINBASED_CTLS 0x481 +#define MSR_IA32_VMX_PROCBASED_CTLS 0x482 +#define MSR_IA32_VMX_EXIT_CTLS 0x483 +#define MSR_IA32_VMX_ENTRY_CTLS 0x484 +#define MSR_IA32_VMX_MISC 0x485 +#define MSR_IA32_VMX_CR0_FIXED0 0x486 +#define MSR_IA32_VMX_CR0_FIXED1 0x487 +#define MSR_IA32_VMX_CR4_FIXED0 0x488 +#define MSR_IA32_VMX_CR4_FIXED1 0x489 +#define MSR_IA32_VMX_VMCS_ENUM 0x48a +#define MSR_IA32_VMX_PROCBASED_CTLS2 0x48b + +#define MSR_IA32_FEATURE_CONTROL 0x3a +#define MSR_IA32_FEATURE_CONTROL_LOCKED 0x1 +#define MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED 0x4 + +#define APIC_ACCESS_PAGE_PRIVATE_MEMSLOT 9 + +#endif diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c new file mode 100644 index 000000000000..5902c5cbc1bb --- /dev/null +++ b/arch/x86/kvm/x86.c @@ -0,0 +1,3146 @@ +/* + * Kernel-based Virtual Machine driver for Linux + * + * derived from drivers/kvm/kvm_main.c + * + * Copyright (C) 2006 Qumranet, Inc. + * + * Authors: + * Avi Kivity + * Yaniv Kamay + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#include +#include "segment_descriptor.h" +#include "irq.h" +#include "mmu.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#define MAX_IO_MSRS 256 +#define CR0_RESERVED_BITS \ + (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ + | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ + | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) +#define CR4_RESERVED_BITS \ + (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ + | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ + | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \ + | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE)) + +#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) +#define EFER_RESERVED_BITS 0xfffffffffffff2fe + +#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM +#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU + +struct kvm_x86_ops *kvm_x86_ops; + +struct kvm_stats_debugfs_item debugfs_entries[] = { + { "pf_fixed", VCPU_STAT(pf_fixed) }, + { "pf_guest", VCPU_STAT(pf_guest) }, + { "tlb_flush", VCPU_STAT(tlb_flush) }, + { "invlpg", VCPU_STAT(invlpg) }, + { "exits", VCPU_STAT(exits) }, + { "io_exits", VCPU_STAT(io_exits) }, + { "mmio_exits", VCPU_STAT(mmio_exits) }, + { "signal_exits", VCPU_STAT(signal_exits) }, + { "irq_window", VCPU_STAT(irq_window_exits) }, + { "halt_exits", VCPU_STAT(halt_exits) }, + { "halt_wakeup", VCPU_STAT(halt_wakeup) }, + { "request_irq", VCPU_STAT(request_irq_exits) }, + { "irq_exits", VCPU_STAT(irq_exits) }, + { "host_state_reload", VCPU_STAT(host_state_reload) }, + { "efer_reload", VCPU_STAT(efer_reload) }, + { "fpu_reload", VCPU_STAT(fpu_reload) }, + { "insn_emulation", VCPU_STAT(insn_emulation) }, + { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, + { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, + { "mmu_pte_write", VM_STAT(mmu_pte_write) }, + { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, + { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, + { "mmu_flooded", VM_STAT(mmu_flooded) }, + { "mmu_recycled", VM_STAT(mmu_recycled) }, + { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, + { NULL } +}; + + +unsigned long segment_base(u16 selector) +{ + struct descriptor_table gdt; + struct segment_descriptor *d; + unsigned long table_base; + unsigned long v; + + if (selector == 0) + return 0; + + asm("sgdt %0" : "=m"(gdt)); + table_base = gdt.base; + + if (selector & 4) { /* from ldt */ + u16 ldt_selector; + + asm("sldt %0" : "=g"(ldt_selector)); + table_base = segment_base(ldt_selector); + } + d = (struct segment_descriptor *)(table_base + (selector & ~7)); + v = d->base_low | ((unsigned long)d->base_mid << 16) | + ((unsigned long)d->base_high << 24); +#ifdef CONFIG_X86_64 + if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11)) + v |= ((unsigned long) \ + ((struct segment_descriptor_64 *)d)->base_higher) << 32; +#endif + return v; +} +EXPORT_SYMBOL_GPL(segment_base); + +u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) +{ + if (irqchip_in_kernel(vcpu->kvm)) + return vcpu->arch.apic_base; + else + return vcpu->arch.apic_base; +} +EXPORT_SYMBOL_GPL(kvm_get_apic_base); + +void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) +{ + /* TODO: reserve bits check */ + if (irqchip_in_kernel(vcpu->kvm)) + kvm_lapic_set_base(vcpu, data); + else + vcpu->arch.apic_base = data; +} +EXPORT_SYMBOL_GPL(kvm_set_apic_base); + +void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) +{ + WARN_ON(vcpu->arch.exception.pending); + vcpu->arch.exception.pending = true; + vcpu->arch.exception.has_error_code = false; + vcpu->arch.exception.nr = nr; +} +EXPORT_SYMBOL_GPL(kvm_queue_exception); + +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, + u32 error_code) +{ + ++vcpu->stat.pf_guest; + if (vcpu->arch.exception.pending && vcpu->arch.exception.nr == PF_VECTOR) { + printk(KERN_DEBUG "kvm: inject_page_fault:" + " double fault 0x%lx\n", addr); + vcpu->arch.exception.nr = DF_VECTOR; + vcpu->arch.exception.error_code = 0; + return; + } + vcpu->arch.cr2 = addr; + kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); +} + +void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) +{ + WARN_ON(vcpu->arch.exception.pending); + vcpu->arch.exception.pending = true; + vcpu->arch.exception.has_error_code = true; + vcpu->arch.exception.nr = nr; + vcpu->arch.exception.error_code = error_code; +} +EXPORT_SYMBOL_GPL(kvm_queue_exception_e); + +static void __queue_exception(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr, + vcpu->arch.exception.has_error_code, + vcpu->arch.exception.error_code); +} + +/* + * Load the pae pdptrs. Return true is they are all valid. + */ +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; + unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; + int i; + int ret; + u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; + + mutex_lock(&vcpu->kvm->lock); + ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, + offset * sizeof(u64), sizeof(pdpte)); + if (ret < 0) { + ret = 0; + goto out; + } + for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { + if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) { + ret = 0; + goto out; + } + } + ret = 1; + + memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); +out: + mutex_unlock(&vcpu->kvm->lock); + + return ret; +} + +static bool pdptrs_changed(struct kvm_vcpu *vcpu) +{ + u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; + bool changed = true; + int r; + + if (is_long_mode(vcpu) || !is_pae(vcpu)) + return false; + + mutex_lock(&vcpu->kvm->lock); + r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte)); + if (r < 0) + goto out; + changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0; +out: + mutex_unlock(&vcpu->kvm->lock); + + return changed; +} + +void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ + if (cr0 & CR0_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", + cr0, vcpu->arch.cr0); + kvm_inject_gp(vcpu, 0); + return; + } + + if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) { + printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n"); + kvm_inject_gp(vcpu, 0); + return; + } + + if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) { + printk(KERN_DEBUG "set_cr0: #GP, set PG flag " + "and a clear PE flag\n"); + kvm_inject_gp(vcpu, 0); + return; + } + + if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { +#ifdef CONFIG_X86_64 + if ((vcpu->arch.shadow_efer & EFER_LME)) { + int cs_db, cs_l; + + if (!is_pae(vcpu)) { + printk(KERN_DEBUG "set_cr0: #GP, start paging " + "in long mode while PAE is disabled\n"); + kvm_inject_gp(vcpu, 0); + return; + } + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + if (cs_l) { + printk(KERN_DEBUG "set_cr0: #GP, start paging " + "in long mode while CS.L == 1\n"); + kvm_inject_gp(vcpu, 0); + return; + + } + } else +#endif + if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) { + printk(KERN_DEBUG "set_cr0: #GP, pdptrs " + "reserved bits\n"); + kvm_inject_gp(vcpu, 0); + return; + } + + } + + kvm_x86_ops->set_cr0(vcpu, cr0); + vcpu->arch.cr0 = cr0; + + mutex_lock(&vcpu->kvm->lock); + kvm_mmu_reset_context(vcpu); + mutex_unlock(&vcpu->kvm->lock); + return; +} +EXPORT_SYMBOL_GPL(set_cr0); + +void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) +{ + set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f)); +} +EXPORT_SYMBOL_GPL(lmsw); + +void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + if (cr4 & CR4_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n"); + kvm_inject_gp(vcpu, 0); + return; + } + + if (is_long_mode(vcpu)) { + if (!(cr4 & X86_CR4_PAE)) { + printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while " + "in long mode\n"); + kvm_inject_gp(vcpu, 0); + return; + } + } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) + && !load_pdptrs(vcpu, vcpu->arch.cr3)) { + printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); + kvm_inject_gp(vcpu, 0); + return; + } + + if (cr4 & X86_CR4_VMXE) { + printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n"); + kvm_inject_gp(vcpu, 0); + return; + } + kvm_x86_ops->set_cr4(vcpu, cr4); + vcpu->arch.cr4 = cr4; + mutex_lock(&vcpu->kvm->lock); + kvm_mmu_reset_context(vcpu); + mutex_unlock(&vcpu->kvm->lock); +} +EXPORT_SYMBOL_GPL(set_cr4); + +void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) { + kvm_mmu_flush_tlb(vcpu); + return; + } + + if (is_long_mode(vcpu)) { + if (cr3 & CR3_L_MODE_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); + kvm_inject_gp(vcpu, 0); + return; + } + } else { + if (is_pae(vcpu)) { + if (cr3 & CR3_PAE_RESERVED_BITS) { + printk(KERN_DEBUG + "set_cr3: #GP, reserved bits\n"); + kvm_inject_gp(vcpu, 0); + return; + } + if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) { + printk(KERN_DEBUG "set_cr3: #GP, pdptrs " + "reserved bits\n"); + kvm_inject_gp(vcpu, 0); + return; + } + } + /* + * We don't check reserved bits in nonpae mode, because + * this isn't enforced, and VMware depends on this. + */ + } + + mutex_lock(&vcpu->kvm->lock); + /* + * Does the new cr3 value map to physical memory? (Note, we + * catch an invalid cr3 even in real-mode, because it would + * cause trouble later on when we turn on paging anyway.) + * + * A real CPU would silently accept an invalid cr3 and would + * attempt to use it - with largely undefined (and often hard + * to debug) behavior on the guest side. + */ + if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) + kvm_inject_gp(vcpu, 0); + else { + vcpu->arch.cr3 = cr3; + vcpu->arch.mmu.new_cr3(vcpu); + } + mutex_unlock(&vcpu->kvm->lock); +} +EXPORT_SYMBOL_GPL(set_cr3); + +void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) +{ + if (cr8 & CR8_RESERVED_BITS) { + printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8); + kvm_inject_gp(vcpu, 0); + return; + } + if (irqchip_in_kernel(vcpu->kvm)) + kvm_lapic_set_tpr(vcpu, cr8); + else + vcpu->arch.cr8 = cr8; +} +EXPORT_SYMBOL_GPL(set_cr8); + +unsigned long get_cr8(struct kvm_vcpu *vcpu) +{ + if (irqchip_in_kernel(vcpu->kvm)) + return kvm_lapic_get_cr8(vcpu); + else + return vcpu->arch.cr8; +} +EXPORT_SYMBOL_GPL(get_cr8); + +/* + * List of msr numbers which we expose to userspace through KVM_GET_MSRS + * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. + * + * This list is modified at module load time to reflect the + * capabilities of the host cpu. + */ +static u32 msrs_to_save[] = { + MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, + MSR_K6_STAR, +#ifdef CONFIG_X86_64 + MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, +#endif + MSR_IA32_TIME_STAMP_COUNTER, +}; + +static unsigned num_msrs_to_save; + +static u32 emulated_msrs[] = { + MSR_IA32_MISC_ENABLE, +}; + +#ifdef CONFIG_X86_64 + +static void set_efer(struct kvm_vcpu *vcpu, u64 efer) +{ + if (efer & EFER_RESERVED_BITS) { + printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n", + efer); + kvm_inject_gp(vcpu, 0); + return; + } + + if (is_paging(vcpu) + && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) { + printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); + kvm_inject_gp(vcpu, 0); + return; + } + + kvm_x86_ops->set_efer(vcpu, efer); + + efer &= ~EFER_LMA; + efer |= vcpu->arch.shadow_efer & EFER_LMA; + + vcpu->arch.shadow_efer = efer; +} + +#endif + +/* + * Writes msr value into into the appropriate "register". + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) +{ + return kvm_x86_ops->set_msr(vcpu, msr_index, data); +} + +/* + * Adapt set_msr() to msr_io()'s calling convention + */ +static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) +{ + return kvm_set_msr(vcpu, index, *data); +} + + +int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) +{ + switch (msr) { +#ifdef CONFIG_X86_64 + case MSR_EFER: + set_efer(vcpu, data); + break; +#endif + case MSR_IA32_MC0_STATUS: + pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n", + __FUNCTION__, data); + break; + case MSR_IA32_MCG_STATUS: + pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n", + __FUNCTION__, data); + break; + case MSR_IA32_UCODE_REV: + case MSR_IA32_UCODE_WRITE: + case 0x200 ... 0x2ff: /* MTRRs */ + break; + case MSR_IA32_APICBASE: + kvm_set_apic_base(vcpu, data); + break; + case MSR_IA32_MISC_ENABLE: + vcpu->arch.ia32_misc_enable_msr = data; + break; + default: + pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_set_msr_common); + + +/* + * Reads an msr value (of 'msr_index') into 'pdata'. + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) +{ + return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); +} + +int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) +{ + u64 data; + + switch (msr) { + case 0xc0010010: /* SYSCFG */ + case 0xc0010015: /* HWCR */ + case MSR_IA32_PLATFORM_ID: + case MSR_IA32_P5_MC_ADDR: + case MSR_IA32_P5_MC_TYPE: + case MSR_IA32_MC0_CTL: + case MSR_IA32_MCG_STATUS: + case MSR_IA32_MCG_CAP: + case MSR_IA32_MC0_MISC: + case MSR_IA32_MC0_MISC+4: + case MSR_IA32_MC0_MISC+8: + case MSR_IA32_MC0_MISC+12: + case MSR_IA32_MC0_MISC+16: + case MSR_IA32_UCODE_REV: + case MSR_IA32_PERF_STATUS: + case MSR_IA32_EBL_CR_POWERON: + /* MTRR registers */ + case 0xfe: + case 0x200 ... 0x2ff: + data = 0; + break; + case 0xcd: /* fsb frequency */ + data = 3; + break; + case MSR_IA32_APICBASE: + data = kvm_get_apic_base(vcpu); + break; + case MSR_IA32_MISC_ENABLE: + data = vcpu->arch.ia32_misc_enable_msr; + break; +#ifdef CONFIG_X86_64 + case MSR_EFER: + data = vcpu->arch.shadow_efer; + break; +#endif + default: + pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr); + return 1; + } + *pdata = data; + return 0; +} +EXPORT_SYMBOL_GPL(kvm_get_msr_common); + +/* + * Read or write a bunch of msrs. All parameters are kernel addresses. + * + * @return number of msrs set successfully. + */ +static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, + struct kvm_msr_entry *entries, + int (*do_msr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data)) +{ + int i; + + vcpu_load(vcpu); + + for (i = 0; i < msrs->nmsrs; ++i) + if (do_msr(vcpu, entries[i].index, &entries[i].data)) + break; + + vcpu_put(vcpu); + + return i; +} + +/* + * Read or write a bunch of msrs. Parameters are user addresses. + * + * @return number of msrs set successfully. + */ +static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, + int (*do_msr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data), + int writeback) +{ + struct kvm_msrs msrs; + struct kvm_msr_entry *entries; + int r, n; + unsigned size; + + r = -EFAULT; + if (copy_from_user(&msrs, user_msrs, sizeof msrs)) + goto out; + + r = -E2BIG; + if (msrs.nmsrs >= MAX_IO_MSRS) + goto out; + + r = -ENOMEM; + size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; + entries = vmalloc(size); + if (!entries) + goto out; + + r = -EFAULT; + if (copy_from_user(entries, user_msrs->entries, size)) + goto out_free; + + r = n = __msr_io(vcpu, &msrs, entries, do_msr); + if (r < 0) + goto out_free; + + r = -EFAULT; + if (writeback && copy_to_user(user_msrs->entries, entries, size)) + goto out_free; + + r = n; + +out_free: + vfree(entries); +out: + return r; +} + +/* + * Make sure that a cpu that is being hot-unplugged does not have any vcpus + * cached on it. + */ +void decache_vcpus_on_cpu(int cpu) +{ + struct kvm *vm; + struct kvm_vcpu *vcpu; + int i; + + spin_lock(&kvm_lock); + list_for_each_entry(vm, &vm_list, vm_list) + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + vcpu = vm->vcpus[i]; + if (!vcpu) + continue; + /* + * If the vcpu is locked, then it is running on some + * other cpu and therefore it is not cached on the + * cpu in question. + * + * If it's not locked, check the last cpu it executed + * on. + */ + if (mutex_trylock(&vcpu->mutex)) { + if (vcpu->cpu == cpu) { + kvm_x86_ops->vcpu_decache(vcpu); + vcpu->cpu = -1; + } + mutex_unlock(&vcpu->mutex); + } + } + spin_unlock(&kvm_lock); +} + +int kvm_dev_ioctl_check_extension(long ext) +{ + int r; + + switch (ext) { + case KVM_CAP_IRQCHIP: + case KVM_CAP_HLT: + case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: + case KVM_CAP_USER_MEMORY: + case KVM_CAP_SET_TSS_ADDR: + case KVM_CAP_EXT_CPUID: + r = 1; + break; + default: + r = 0; + break; + } + return r; + +} + +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + long r; + + switch (ioctl) { + case KVM_GET_MSR_INDEX_LIST: { + struct kvm_msr_list __user *user_msr_list = argp; + struct kvm_msr_list msr_list; + unsigned n; + + r = -EFAULT; + if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list)) + goto out; + n = msr_list.nmsrs; + msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs); + if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list)) + goto out; + r = -E2BIG; + if (n < num_msrs_to_save) + goto out; + r = -EFAULT; + if (copy_to_user(user_msr_list->indices, &msrs_to_save, + num_msrs_to_save * sizeof(u32))) + goto out; + if (copy_to_user(user_msr_list->indices + + num_msrs_to_save * sizeof(u32), + &emulated_msrs, + ARRAY_SIZE(emulated_msrs) * sizeof(u32))) + goto out; + r = 0; + break; + } + default: + r = -EINVAL; + } +out: + return r; +} + +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + kvm_x86_ops->vcpu_load(vcpu, cpu); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->vcpu_put(vcpu); + kvm_put_guest_fpu(vcpu); +} + +static int is_efer_nx(void) +{ + u64 efer; + + rdmsrl(MSR_EFER, efer); + return efer & EFER_NX; +} + +static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) +{ + int i; + struct kvm_cpuid_entry2 *e, *entry; + + entry = NULL; + for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { + e = &vcpu->arch.cpuid_entries[i]; + if (e->function == 0x80000001) { + entry = e; + break; + } + } + if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) { + entry->edx &= ~(1 << 20); + printk(KERN_INFO "kvm: guest NX capability removed\n"); + } +} + +/* when an old userspace process fills a new kernel module */ +static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, + struct kvm_cpuid *cpuid, + struct kvm_cpuid_entry __user *entries) +{ + int r, i; + struct kvm_cpuid_entry *cpuid_entries; + + r = -E2BIG; + if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) + goto out; + r = -ENOMEM; + cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent); + if (!cpuid_entries) + goto out; + r = -EFAULT; + if (copy_from_user(cpuid_entries, entries, + cpuid->nent * sizeof(struct kvm_cpuid_entry))) + goto out_free; + for (i = 0; i < cpuid->nent; i++) { + vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function; + vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax; + vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx; + vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx; + vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx; + vcpu->arch.cpuid_entries[i].index = 0; + vcpu->arch.cpuid_entries[i].flags = 0; + vcpu->arch.cpuid_entries[i].padding[0] = 0; + vcpu->arch.cpuid_entries[i].padding[1] = 0; + vcpu->arch.cpuid_entries[i].padding[2] = 0; + } + vcpu->arch.cpuid_nent = cpuid->nent; + cpuid_fix_nx_cap(vcpu); + r = 0; + +out_free: + vfree(cpuid_entries); +out: + return r; +} + +static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) +{ + int r; + + r = -E2BIG; + if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) + goto out; + r = -EFAULT; + if (copy_from_user(&vcpu->arch.cpuid_entries, entries, + cpuid->nent * sizeof(struct kvm_cpuid_entry2))) + goto out; + vcpu->arch.cpuid_nent = cpuid->nent; + return 0; + +out: + return r; +} + +static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) +{ + int r; + + r = -E2BIG; + if (cpuid->nent < vcpu->arch.cpuid_nent) + goto out; + r = -EFAULT; + if (copy_to_user(entries, &vcpu->arch.cpuid_entries, + vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2))) + goto out; + return 0; + +out: + cpuid->nent = vcpu->arch.cpuid_nent; + return r; +} + +static inline u32 bit(int bitno) +{ + return 1 << (bitno & 31); +} + +static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, + u32 index) +{ + entry->function = function; + entry->index = index; + cpuid_count(entry->function, entry->index, + &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); + entry->flags = 0; +} + +static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, + u32 index, int *nent, int maxnent) +{ + const u32 kvm_supported_word0_x86_features = bit(X86_FEATURE_FPU) | + bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) | + bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) | + bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) | + bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) | + bit(X86_FEATURE_SEP) | bit(X86_FEATURE_PGE) | + bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) | + bit(X86_FEATURE_CLFLSH) | bit(X86_FEATURE_MMX) | + bit(X86_FEATURE_FXSR) | bit(X86_FEATURE_XMM) | + bit(X86_FEATURE_XMM2) | bit(X86_FEATURE_SELFSNOOP); + const u32 kvm_supported_word1_x86_features = bit(X86_FEATURE_FPU) | + bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) | + bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) | + bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) | + bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) | + bit(X86_FEATURE_PGE) | + bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) | + bit(X86_FEATURE_MMX) | bit(X86_FEATURE_FXSR) | + bit(X86_FEATURE_SYSCALL) | + (bit(X86_FEATURE_NX) && is_efer_nx()) | +#ifdef CONFIG_X86_64 + bit(X86_FEATURE_LM) | +#endif + bit(X86_FEATURE_MMXEXT) | + bit(X86_FEATURE_3DNOWEXT) | + bit(X86_FEATURE_3DNOW); + const u32 kvm_supported_word3_x86_features = + bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16); + const u32 kvm_supported_word6_x86_features = + bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY); + + /* all func 2 cpuid_count() should be called on the same cpu */ + get_cpu(); + do_cpuid_1_ent(entry, function, index); + ++*nent; + + switch (function) { + case 0: + entry->eax = min(entry->eax, (u32)0xb); + break; + case 1: + entry->edx &= kvm_supported_word0_x86_features; + entry->ecx &= kvm_supported_word3_x86_features; + break; + /* function 2 entries are STATEFUL. That is, repeated cpuid commands + * may return different values. This forces us to get_cpu() before + * issuing the first command, and also to emulate this annoying behavior + * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */ + case 2: { + int t, times = entry->eax & 0xff; + + entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; + for (t = 1; t < times && *nent < maxnent; ++t) { + do_cpuid_1_ent(&entry[t], function, 0); + entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; + ++*nent; + } + break; + } + /* function 4 and 0xb have additional index. */ + case 4: { + int index, cache_type; + + entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + /* read more entries until cache_type is zero */ + for (index = 1; *nent < maxnent; ++index) { + cache_type = entry[index - 1].eax & 0x1f; + if (!cache_type) + break; + do_cpuid_1_ent(&entry[index], function, index); + entry[index].flags |= + KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + ++*nent; + } + break; + } + case 0xb: { + int index, level_type; + + entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + /* read more entries until level_type is zero */ + for (index = 1; *nent < maxnent; ++index) { + level_type = entry[index - 1].ecx & 0xff; + if (!level_type) + break; + do_cpuid_1_ent(&entry[index], function, index); + entry[index].flags |= + KVM_CPUID_FLAG_SIGNIFCANT_INDEX; + ++*nent; + } + break; + } + case 0x80000000: + entry->eax = min(entry->eax, 0x8000001a); + break; + case 0x80000001: + entry->edx &= kvm_supported_word1_x86_features; + entry->ecx &= kvm_supported_word6_x86_features; + break; + } + put_cpu(); +} + +static int kvm_vm_ioctl_get_supported_cpuid(struct kvm *kvm, + struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 __user *entries) +{ + struct kvm_cpuid_entry2 *cpuid_entries; + int limit, nent = 0, r = -E2BIG; + u32 func; + + if (cpuid->nent < 1) + goto out; + r = -ENOMEM; + cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent); + if (!cpuid_entries) + goto out; + + do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent); + limit = cpuid_entries[0].eax; + for (func = 1; func <= limit && nent < cpuid->nent; ++func) + do_cpuid_ent(&cpuid_entries[nent], func, 0, + &nent, cpuid->nent); + r = -E2BIG; + if (nent >= cpuid->nent) + goto out_free; + + do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent); + limit = cpuid_entries[nent - 1].eax; + for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func) + do_cpuid_ent(&cpuid_entries[nent], func, 0, + &nent, cpuid->nent); + r = -EFAULT; + if (copy_to_user(entries, cpuid_entries, + nent * sizeof(struct kvm_cpuid_entry2))) + goto out_free; + cpuid->nent = nent; + r = 0; + +out_free: + vfree(cpuid_entries); +out: + return r; +} + +static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, + struct kvm_lapic_state *s) +{ + vcpu_load(vcpu); + memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s); + vcpu_put(vcpu); + + return 0; +} + +static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, + struct kvm_lapic_state *s) +{ + vcpu_load(vcpu); + memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); + kvm_apic_post_state_restore(vcpu); + vcpu_put(vcpu); + + return 0; +} + +static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_interrupt *irq) +{ + if (irq->irq < 0 || irq->irq >= 256) + return -EINVAL; + if (irqchip_in_kernel(vcpu->kvm)) + return -ENXIO; + vcpu_load(vcpu); + + set_bit(irq->irq, vcpu->arch.irq_pending); + set_bit(irq->irq / BITS_PER_LONG, &vcpu->arch.irq_summary); + + vcpu_put(vcpu); + + return 0; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + int r; + + switch (ioctl) { + case KVM_GET_LAPIC: { + struct kvm_lapic_state lapic; + + memset(&lapic, 0, sizeof lapic); + r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &lapic, sizeof lapic)) + goto out; + r = 0; + break; + } + case KVM_SET_LAPIC: { + struct kvm_lapic_state lapic; + + r = -EFAULT; + if (copy_from_user(&lapic, argp, sizeof lapic)) + goto out; + r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);; + if (r) + goto out; + r = 0; + break; + } + case KVM_INTERRUPT: { + struct kvm_interrupt irq; + + r = -EFAULT; + if (copy_from_user(&irq, argp, sizeof irq)) + goto out; + r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); + if (r) + goto out; + r = 0; + break; + } + case KVM_SET_CPUID: { + struct kvm_cpuid __user *cpuid_arg = argp; + struct kvm_cpuid cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); + if (r) + goto out; + break; + } + case KVM_SET_CPUID2: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + break; + } + case KVM_GET_CPUID2: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) + goto out; + r = 0; + break; + } + case KVM_GET_MSRS: + r = msr_io(vcpu, argp, kvm_get_msr, 1); + break; + case KVM_SET_MSRS: + r = msr_io(vcpu, argp, do_set_msr, 0); + break; + default: + r = -EINVAL; + } +out: + return r; +} + +static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) +{ + int ret; + + if (addr > (unsigned int)(-3 * PAGE_SIZE)) + return -1; + ret = kvm_x86_ops->set_tss_addr(kvm, addr); + return ret; +} + +static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, + u32 kvm_nr_mmu_pages) +{ + if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) + return -EINVAL; + + mutex_lock(&kvm->lock); + + kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); + kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; + + mutex_unlock(&kvm->lock); + return 0; +} + +static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) +{ + return kvm->arch.n_alloc_mmu_pages; +} + +gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) +{ + int i; + struct kvm_mem_alias *alias; + + for (i = 0; i < kvm->arch.naliases; ++i) { + alias = &kvm->arch.aliases[i]; + if (gfn >= alias->base_gfn + && gfn < alias->base_gfn + alias->npages) + return alias->target_gfn + gfn - alias->base_gfn; + } + return gfn; +} + +/* + * Set a new alias region. Aliases map a portion of physical memory into + * another portion. This is useful for memory windows, for example the PC + * VGA region. + */ +static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, + struct kvm_memory_alias *alias) +{ + int r, n; + struct kvm_mem_alias *p; + + r = -EINVAL; + /* General sanity checks */ + if (alias->memory_size & (PAGE_SIZE - 1)) + goto out; + if (alias->guest_phys_addr & (PAGE_SIZE - 1)) + goto out; + if (alias->slot >= KVM_ALIAS_SLOTS) + goto out; + if (alias->guest_phys_addr + alias->memory_size + < alias->guest_phys_addr) + goto out; + if (alias->target_phys_addr + alias->memory_size + < alias->target_phys_addr) + goto out; + + mutex_lock(&kvm->lock); + + p = &kvm->arch.aliases[alias->slot]; + p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; + p->npages = alias->memory_size >> PAGE_SHIFT; + p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT; + + for (n = KVM_ALIAS_SLOTS; n > 0; --n) + if (kvm->arch.aliases[n - 1].npages) + break; + kvm->arch.naliases = n; + + kvm_mmu_zap_all(kvm); + + mutex_unlock(&kvm->lock); + + return 0; + +out: + return r; +} + +static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) +{ + int r; + + r = 0; + switch (chip->chip_id) { + case KVM_IRQCHIP_PIC_MASTER: + memcpy(&chip->chip.pic, + &pic_irqchip(kvm)->pics[0], + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_PIC_SLAVE: + memcpy(&chip->chip.pic, + &pic_irqchip(kvm)->pics[1], + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_IOAPIC: + memcpy(&chip->chip.ioapic, + ioapic_irqchip(kvm), + sizeof(struct kvm_ioapic_state)); + break; + default: + r = -EINVAL; + break; + } + return r; +} + +static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) +{ + int r; + + r = 0; + switch (chip->chip_id) { + case KVM_IRQCHIP_PIC_MASTER: + memcpy(&pic_irqchip(kvm)->pics[0], + &chip->chip.pic, + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_PIC_SLAVE: + memcpy(&pic_irqchip(kvm)->pics[1], + &chip->chip.pic, + sizeof(struct kvm_pic_state)); + break; + case KVM_IRQCHIP_IOAPIC: + memcpy(ioapic_irqchip(kvm), + &chip->chip.ioapic, + sizeof(struct kvm_ioapic_state)); + break; + default: + r = -EINVAL; + break; + } + kvm_pic_update_irq(pic_irqchip(kvm)); + return r; +} + +/* + * Get (and clear) the dirty memory log for a memory slot. + */ +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log) +{ + int r; + int n; + struct kvm_memory_slot *memslot; + int is_dirty = 0; + + mutex_lock(&kvm->lock); + + r = kvm_get_dirty_log(kvm, log, &is_dirty); + if (r) + goto out; + + /* If nothing is dirty, don't bother messing with page tables. */ + if (is_dirty) { + kvm_mmu_slot_remove_write_access(kvm, log->slot); + kvm_flush_remote_tlbs(kvm); + memslot = &kvm->memslots[log->slot]; + n = ALIGN(memslot->npages, BITS_PER_LONG) / 8; + memset(memslot->dirty_bitmap, 0, n); + } + r = 0; +out: + mutex_unlock(&kvm->lock); + return r; +} + +long kvm_arch_vm_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm = filp->private_data; + void __user *argp = (void __user *)arg; + int r = -EINVAL; + + switch (ioctl) { + case KVM_SET_TSS_ADDR: + r = kvm_vm_ioctl_set_tss_addr(kvm, arg); + if (r < 0) + goto out; + break; + case KVM_SET_MEMORY_REGION: { + struct kvm_memory_region kvm_mem; + struct kvm_userspace_memory_region kvm_userspace_mem; + + r = -EFAULT; + if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem)) + goto out; + kvm_userspace_mem.slot = kvm_mem.slot; + kvm_userspace_mem.flags = kvm_mem.flags; + kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr; + kvm_userspace_mem.memory_size = kvm_mem.memory_size; + r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0); + if (r) + goto out; + break; + } + case KVM_SET_NR_MMU_PAGES: + r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); + if (r) + goto out; + break; + case KVM_GET_NR_MMU_PAGES: + r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); + break; + case KVM_SET_MEMORY_ALIAS: { + struct kvm_memory_alias alias; + + r = -EFAULT; + if (copy_from_user(&alias, argp, sizeof alias)) + goto out; + r = kvm_vm_ioctl_set_memory_alias(kvm, &alias); + if (r) + goto out; + break; + } + case KVM_CREATE_IRQCHIP: + r = -ENOMEM; + kvm->arch.vpic = kvm_create_pic(kvm); + if (kvm->arch.vpic) { + r = kvm_ioapic_init(kvm); + if (r) { + kfree(kvm->arch.vpic); + kvm->arch.vpic = NULL; + goto out; + } + } else + goto out; + break; + case KVM_IRQ_LINE: { + struct kvm_irq_level irq_event; + + r = -EFAULT; + if (copy_from_user(&irq_event, argp, sizeof irq_event)) + goto out; + if (irqchip_in_kernel(kvm)) { + mutex_lock(&kvm->lock); + if (irq_event.irq < 16) + kvm_pic_set_irq(pic_irqchip(kvm), + irq_event.irq, + irq_event.level); + kvm_ioapic_set_irq(kvm->arch.vioapic, + irq_event.irq, + irq_event.level); + mutex_unlock(&kvm->lock); + r = 0; + } + break; + } + case KVM_GET_IRQCHIP: { + /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ + struct kvm_irqchip chip; + + r = -EFAULT; + if (copy_from_user(&chip, argp, sizeof chip)) + goto out; + r = -ENXIO; + if (!irqchip_in_kernel(kvm)) + goto out; + r = kvm_vm_ioctl_get_irqchip(kvm, &chip); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &chip, sizeof chip)) + goto out; + r = 0; + break; + } + case KVM_SET_IRQCHIP: { + /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ + struct kvm_irqchip chip; + + r = -EFAULT; + if (copy_from_user(&chip, argp, sizeof chip)) + goto out; + r = -ENXIO; + if (!irqchip_in_kernel(kvm)) + goto out; + r = kvm_vm_ioctl_set_irqchip(kvm, &chip); + if (r) + goto out; + r = 0; + break; + } + case KVM_GET_SUPPORTED_CPUID: { + struct kvm_cpuid2 __user *cpuid_arg = argp; + struct kvm_cpuid2 cpuid; + + r = -EFAULT; + if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) + goto out; + r = kvm_vm_ioctl_get_supported_cpuid(kvm, &cpuid, + cpuid_arg->entries); + if (r) + goto out; + + r = -EFAULT; + if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) + goto out; + r = 0; + break; + } + default: + ; + } +out: + return r; +} + +static void kvm_init_msr_list(void) +{ + u32 dummy[2]; + unsigned i, j; + + for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) { + if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) + continue; + if (j < i) + msrs_to_save[j] = msrs_to_save[i]; + j++; + } + num_msrs_to_save = j; +} + +/* + * Only apic need an MMIO device hook, so shortcut now.. + */ +static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + struct kvm_io_device *dev; + + if (vcpu->arch.apic) { + dev = &vcpu->arch.apic->dev; + if (dev->in_range(dev, addr)) + return dev; + } + return NULL; +} + + +static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + struct kvm_io_device *dev; + + dev = vcpu_find_pervcpu_dev(vcpu, addr); + if (dev == NULL) + dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr); + return dev; +} + +int emulator_read_std(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + void *data = val; + + while (bytes) { + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + unsigned offset = addr & (PAGE_SIZE-1); + unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); + int ret; + + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; + ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); + if (ret < 0) + return X86EMUL_UNHANDLEABLE; + + bytes -= tocopy; + data += tocopy; + addr += tocopy; + } + + return X86EMUL_CONTINUE; +} +EXPORT_SYMBOL_GPL(emulator_read_std); + +static int emulator_read_emulated(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + struct kvm_io_device *mmio_dev; + gpa_t gpa; + + if (vcpu->mmio_read_completed) { + memcpy(val, vcpu->mmio_data, bytes); + vcpu->mmio_read_completed = 0; + return X86EMUL_CONTINUE; + } + + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + + if (emulator_read_std(addr, val, bytes, vcpu) + == X86EMUL_CONTINUE) + return X86EMUL_CONTINUE; + if (gpa == UNMAPPED_GVA) + return X86EMUL_PROPAGATE_FAULT; + +mmio: + /* + * Is this MMIO handled locally? + */ + mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); + if (mmio_dev) { + kvm_iodevice_read(mmio_dev, gpa, bytes, val); + return X86EMUL_CONTINUE; + } + + vcpu->mmio_needed = 1; + vcpu->mmio_phys_addr = gpa; + vcpu->mmio_size = bytes; + vcpu->mmio_is_write = 0; + + return X86EMUL_UNHANDLEABLE; +} + +static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, + const void *val, int bytes) +{ + int ret; + + ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); + if (ret < 0) + return 0; + kvm_mmu_pte_write(vcpu, gpa, val, bytes); + return 1; +} + +static int emulator_write_emulated_onepage(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + struct kvm_io_device *mmio_dev; + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + + if (gpa == UNMAPPED_GVA) { + kvm_inject_page_fault(vcpu, addr, 2); + return X86EMUL_PROPAGATE_FAULT; + } + + /* For APIC access vmexit */ + if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto mmio; + + if (emulator_write_phys(vcpu, gpa, val, bytes)) + return X86EMUL_CONTINUE; + +mmio: + /* + * Is this MMIO handled locally? + */ + mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); + if (mmio_dev) { + kvm_iodevice_write(mmio_dev, gpa, bytes, val); + return X86EMUL_CONTINUE; + } + + vcpu->mmio_needed = 1; + vcpu->mmio_phys_addr = gpa; + vcpu->mmio_size = bytes; + vcpu->mmio_is_write = 1; + memcpy(vcpu->mmio_data, val, bytes); + + return X86EMUL_CONTINUE; +} + +int emulator_write_emulated(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + /* Crossing a page boundary? */ + if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { + int rc, now; + + now = -addr & ~PAGE_MASK; + rc = emulator_write_emulated_onepage(addr, val, now, vcpu); + if (rc != X86EMUL_CONTINUE) + return rc; + addr += now; + val += now; + bytes -= now; + } + return emulator_write_emulated_onepage(addr, val, bytes, vcpu); +} +EXPORT_SYMBOL_GPL(emulator_write_emulated); + +static int emulator_cmpxchg_emulated(unsigned long addr, + const void *old, + const void *new, + unsigned int bytes, + struct kvm_vcpu *vcpu) +{ + static int reported; + + if (!reported) { + reported = 1; + printk(KERN_WARNING "kvm: emulating exchange as write\n"); + } +#ifndef CONFIG_X86_64 + /* guests cmpxchg8b have to be emulated atomically */ + if (bytes == 8) { + gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); + struct page *page; + char *addr; + u64 val; + + if (gpa == UNMAPPED_GVA || + (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) + goto emul_write; + + if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK)) + goto emul_write; + + val = *(u64 *)new; + page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); + addr = kmap_atomic(page, KM_USER0); + set_64bit((u64 *)(addr + offset_in_page(gpa)), val); + kunmap_atomic(addr, KM_USER0); + kvm_release_page_dirty(page); + } +emul_write: +#endif + + return emulator_write_emulated(addr, new, bytes, vcpu); +} + +static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) +{ + return kvm_x86_ops->get_segment_base(vcpu, seg); +} + +int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) +{ + return X86EMUL_CONTINUE; +} + +int emulate_clts(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS); + return X86EMUL_CONTINUE; +} + +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) +{ + struct kvm_vcpu *vcpu = ctxt->vcpu; + + switch (dr) { + case 0 ... 3: + *dest = kvm_x86_ops->get_dr(vcpu, dr); + return X86EMUL_CONTINUE; + default: + pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr); + return X86EMUL_UNHANDLEABLE; + } +} + +int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) +{ + unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U; + int exception; + + kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception); + if (exception) { + /* FIXME: better handling */ + return X86EMUL_UNHANDLEABLE; + } + return X86EMUL_CONTINUE; +} + +void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) +{ + static int reported; + u8 opcodes[4]; + unsigned long rip = vcpu->arch.rip; + unsigned long rip_linear; + + rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); + + if (reported) + return; + + emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); + + printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", + context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); + reported = 1; +} +EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); + +struct x86_emulate_ops emulate_ops = { + .read_std = emulator_read_std, + .read_emulated = emulator_read_emulated, + .write_emulated = emulator_write_emulated, + .cmpxchg_emulated = emulator_cmpxchg_emulated, +}; + +int emulate_instruction(struct kvm_vcpu *vcpu, + struct kvm_run *run, + unsigned long cr2, + u16 error_code, + int no_decode) +{ + int r; + + vcpu->arch.mmio_fault_cr2 = cr2; + kvm_x86_ops->cache_regs(vcpu); + + vcpu->mmio_is_write = 0; + vcpu->arch.pio.string = 0; + + if (!no_decode) { + int cs_db, cs_l; + kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); + + vcpu->arch.emulate_ctxt.vcpu = vcpu; + vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); + vcpu->arch.emulate_ctxt.mode = + (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM) + ? X86EMUL_MODE_REAL : cs_l + ? X86EMUL_MODE_PROT64 : cs_db + ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; + + if (vcpu->arch.emulate_ctxt.mode == X86EMUL_MODE_PROT64) { + vcpu->arch.emulate_ctxt.cs_base = 0; + vcpu->arch.emulate_ctxt.ds_base = 0; + vcpu->arch.emulate_ctxt.es_base = 0; + vcpu->arch.emulate_ctxt.ss_base = 0; + } else { + vcpu->arch.emulate_ctxt.cs_base = + get_segment_base(vcpu, VCPU_SREG_CS); + vcpu->arch.emulate_ctxt.ds_base = + get_segment_base(vcpu, VCPU_SREG_DS); + vcpu->arch.emulate_ctxt.es_base = + get_segment_base(vcpu, VCPU_SREG_ES); + vcpu->arch.emulate_ctxt.ss_base = + get_segment_base(vcpu, VCPU_SREG_SS); + } + + vcpu->arch.emulate_ctxt.gs_base = + get_segment_base(vcpu, VCPU_SREG_GS); + vcpu->arch.emulate_ctxt.fs_base = + get_segment_base(vcpu, VCPU_SREG_FS); + + r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); + ++vcpu->stat.insn_emulation; + if (r) { + ++vcpu->stat.insn_emulation_fail; + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; + return EMULATE_FAIL; + } + } + + r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); + + if (vcpu->arch.pio.string) + return EMULATE_DO_MMIO; + + if ((r || vcpu->mmio_is_write) && run) { + run->exit_reason = KVM_EXIT_MMIO; + run->mmio.phys_addr = vcpu->mmio_phys_addr; + memcpy(run->mmio.data, vcpu->mmio_data, 8); + run->mmio.len = vcpu->mmio_size; + run->mmio.is_write = vcpu->mmio_is_write; + } + + if (r) { + if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) + return EMULATE_DONE; + if (!vcpu->mmio_needed) { + kvm_report_emulation_failure(vcpu, "mmio"); + return EMULATE_FAIL; + } + return EMULATE_DO_MMIO; + } + + kvm_x86_ops->decache_regs(vcpu); + kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags); + + if (vcpu->mmio_is_write) { + vcpu->mmio_needed = 0; + return EMULATE_DO_MMIO; + } + + return EMULATE_DONE; +} +EXPORT_SYMBOL_GPL(emulate_instruction); + +static void free_pio_guest_pages(struct kvm_vcpu *vcpu) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(vcpu->arch.pio.guest_pages); ++i) + if (vcpu->arch.pio.guest_pages[i]) { + kvm_release_page_dirty(vcpu->arch.pio.guest_pages[i]); + vcpu->arch.pio.guest_pages[i] = NULL; + } +} + +static int pio_copy_data(struct kvm_vcpu *vcpu) +{ + void *p = vcpu->arch.pio_data; + void *q; + unsigned bytes; + int nr_pages = vcpu->arch.pio.guest_pages[1] ? 2 : 1; + + q = vmap(vcpu->arch.pio.guest_pages, nr_pages, VM_READ|VM_WRITE, + PAGE_KERNEL); + if (!q) { + free_pio_guest_pages(vcpu); + return -ENOMEM; + } + q += vcpu->arch.pio.guest_page_offset; + bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count; + if (vcpu->arch.pio.in) + memcpy(q, p, bytes); + else + memcpy(p, q, bytes); + q -= vcpu->arch.pio.guest_page_offset; + vunmap(q); + free_pio_guest_pages(vcpu); + return 0; +} + +int complete_pio(struct kvm_vcpu *vcpu) +{ + struct kvm_pio_request *io = &vcpu->arch.pio; + long delta; + int r; + + kvm_x86_ops->cache_regs(vcpu); + + if (!io->string) { + if (io->in) + memcpy(&vcpu->arch.regs[VCPU_REGS_RAX], vcpu->arch.pio_data, + io->size); + } else { + if (io->in) { + r = pio_copy_data(vcpu); + if (r) { + kvm_x86_ops->cache_regs(vcpu); + return r; + } + } + + delta = 1; + if (io->rep) { + delta *= io->cur_count; + /* + * The size of the register should really depend on + * current address size. + */ + vcpu->arch.regs[VCPU_REGS_RCX] -= delta; + } + if (io->down) + delta = -delta; + delta *= io->size; + if (io->in) + vcpu->arch.regs[VCPU_REGS_RDI] += delta; + else + vcpu->arch.regs[VCPU_REGS_RSI] += delta; + } + + kvm_x86_ops->decache_regs(vcpu); + + io->count -= io->cur_count; + io->cur_count = 0; + + return 0; +} + +static void kernel_pio(struct kvm_io_device *pio_dev, + struct kvm_vcpu *vcpu, + void *pd) +{ + /* TODO: String I/O for in kernel device */ + + mutex_lock(&vcpu->kvm->lock); + if (vcpu->arch.pio.in) + kvm_iodevice_read(pio_dev, vcpu->arch.pio.port, + vcpu->arch.pio.size, + pd); + else + kvm_iodevice_write(pio_dev, vcpu->arch.pio.port, + vcpu->arch.pio.size, + pd); + mutex_unlock(&vcpu->kvm->lock); +} + +static void pio_string_write(struct kvm_io_device *pio_dev, + struct kvm_vcpu *vcpu) +{ + struct kvm_pio_request *io = &vcpu->arch.pio; + void *pd = vcpu->arch.pio_data; + int i; + + mutex_lock(&vcpu->kvm->lock); + for (i = 0; i < io->cur_count; i++) { + kvm_iodevice_write(pio_dev, io->port, + io->size, + pd); + pd += io->size; + } + mutex_unlock(&vcpu->kvm->lock); +} + +static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, + gpa_t addr) +{ + return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); +} + +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned port) +{ + struct kvm_io_device *pio_dev; + + vcpu->run->exit_reason = KVM_EXIT_IO; + vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; + vcpu->run->io.size = vcpu->arch.pio.size = size; + vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; + vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1; + vcpu->run->io.port = vcpu->arch.pio.port = port; + vcpu->arch.pio.in = in; + vcpu->arch.pio.string = 0; + vcpu->arch.pio.down = 0; + vcpu->arch.pio.guest_page_offset = 0; + vcpu->arch.pio.rep = 0; + + kvm_x86_ops->cache_regs(vcpu); + memcpy(vcpu->arch.pio_data, &vcpu->arch.regs[VCPU_REGS_RAX], 4); + kvm_x86_ops->decache_regs(vcpu); + + kvm_x86_ops->skip_emulated_instruction(vcpu); + + pio_dev = vcpu_find_pio_dev(vcpu, port); + if (pio_dev) { + kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data); + complete_pio(vcpu); + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_emulate_pio); + +int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned long count, int down, + gva_t address, int rep, unsigned port) +{ + unsigned now, in_page; + int i, ret = 0; + int nr_pages = 1; + struct page *page; + struct kvm_io_device *pio_dev; + + vcpu->run->exit_reason = KVM_EXIT_IO; + vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; + vcpu->run->io.size = vcpu->arch.pio.size = size; + vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; + vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count; + vcpu->run->io.port = vcpu->arch.pio.port = port; + vcpu->arch.pio.in = in; + vcpu->arch.pio.string = 1; + vcpu->arch.pio.down = down; + vcpu->arch.pio.guest_page_offset = offset_in_page(address); + vcpu->arch.pio.rep = rep; + + if (!count) { + kvm_x86_ops->skip_emulated_instruction(vcpu); + return 1; + } + + if (!down) + in_page = PAGE_SIZE - offset_in_page(address); + else + in_page = offset_in_page(address) + size; + now = min(count, (unsigned long)in_page / size); + if (!now) { + /* + * String I/O straddles page boundary. Pin two guest pages + * so that we satisfy atomicity constraints. Do just one + * transaction to avoid complexity. + */ + nr_pages = 2; + now = 1; + } + if (down) { + /* + * String I/O in reverse. Yuck. Kill the guest, fix later. + */ + pr_unimpl(vcpu, "guest string pio down\n"); + kvm_inject_gp(vcpu, 0); + return 1; + } + vcpu->run->io.count = now; + vcpu->arch.pio.cur_count = now; + + if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count) + kvm_x86_ops->skip_emulated_instruction(vcpu); + + for (i = 0; i < nr_pages; ++i) { + mutex_lock(&vcpu->kvm->lock); + page = gva_to_page(vcpu, address + i * PAGE_SIZE); + vcpu->arch.pio.guest_pages[i] = page; + mutex_unlock(&vcpu->kvm->lock); + if (!page) { + kvm_inject_gp(vcpu, 0); + free_pio_guest_pages(vcpu); + return 1; + } + } + + pio_dev = vcpu_find_pio_dev(vcpu, port); + if (!vcpu->arch.pio.in) { + /* string PIO write */ + ret = pio_copy_data(vcpu); + if (ret >= 0 && pio_dev) { + pio_string_write(pio_dev, vcpu); + complete_pio(vcpu); + if (vcpu->arch.pio.count == 0) + ret = 1; + } + } else if (pio_dev) + pr_unimpl(vcpu, "no string pio read support yet, " + "port %x size %d count %ld\n", + port, size, count); + + return ret; +} +EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); + +int kvm_arch_init(void *opaque) +{ + int r; + struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; + + r = kvm_mmu_module_init(); + if (r) + goto out_fail; + + kvm_init_msr_list(); + + if (kvm_x86_ops) { + printk(KERN_ERR "kvm: already loaded the other module\n"); + r = -EEXIST; + goto out; + } + + if (!ops->cpu_has_kvm_support()) { + printk(KERN_ERR "kvm: no hardware support\n"); + r = -EOPNOTSUPP; + goto out; + } + if (ops->disabled_by_bios()) { + printk(KERN_ERR "kvm: disabled by bios\n"); + r = -EOPNOTSUPP; + goto out; + } + + kvm_x86_ops = ops; + kvm_mmu_set_nonpresent_ptes(0ull, 0ull); + return 0; + +out: + kvm_mmu_module_exit(); +out_fail: + return r; +} + +void kvm_arch_exit(void) +{ + kvm_x86_ops = NULL; + kvm_mmu_module_exit(); +} + +int kvm_emulate_halt(struct kvm_vcpu *vcpu) +{ + ++vcpu->stat.halt_exits; + if (irqchip_in_kernel(vcpu->kvm)) { + vcpu->arch.mp_state = VCPU_MP_STATE_HALTED; + kvm_vcpu_block(vcpu); + if (vcpu->arch.mp_state != VCPU_MP_STATE_RUNNABLE) + return -EINTR; + return 1; + } else { + vcpu->run->exit_reason = KVM_EXIT_HLT; + return 0; + } +} +EXPORT_SYMBOL_GPL(kvm_emulate_halt); + +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) +{ + unsigned long nr, a0, a1, a2, a3, ret; + + kvm_x86_ops->cache_regs(vcpu); + + nr = vcpu->arch.regs[VCPU_REGS_RAX]; + a0 = vcpu->arch.regs[VCPU_REGS_RBX]; + a1 = vcpu->arch.regs[VCPU_REGS_RCX]; + a2 = vcpu->arch.regs[VCPU_REGS_RDX]; + a3 = vcpu->arch.regs[VCPU_REGS_RSI]; + + if (!is_long_mode(vcpu)) { + nr &= 0xFFFFFFFF; + a0 &= 0xFFFFFFFF; + a1 &= 0xFFFFFFFF; + a2 &= 0xFFFFFFFF; + a3 &= 0xFFFFFFFF; + } + + switch (nr) { + default: + ret = -KVM_ENOSYS; + break; + } + vcpu->arch.regs[VCPU_REGS_RAX] = ret; + kvm_x86_ops->decache_regs(vcpu); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu) +{ + char instruction[3]; + int ret = 0; + + mutex_lock(&vcpu->kvm->lock); + + /* + * Blow out the MMU to ensure that no other VCPU has an active mapping + * to ensure that the updated hypercall appears atomically across all + * VCPUs. + */ + kvm_mmu_zap_all(vcpu->kvm); + + kvm_x86_ops->cache_regs(vcpu); + kvm_x86_ops->patch_hypercall(vcpu, instruction); + if (emulator_write_emulated(vcpu->arch.rip, instruction, 3, vcpu) + != X86EMUL_CONTINUE) + ret = -EFAULT; + + mutex_unlock(&vcpu->kvm->lock); + + return ret; +} + +static u64 mk_cr_64(u64 curr_cr, u32 new_val) +{ + return (curr_cr & ~((1ULL << 32) - 1)) | new_val; +} + +void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) +{ + struct descriptor_table dt = { limit, base }; + + kvm_x86_ops->set_gdt(vcpu, &dt); +} + +void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) +{ + struct descriptor_table dt = { limit, base }; + + kvm_x86_ops->set_idt(vcpu, &dt); +} + +void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, + unsigned long *rflags) +{ + lmsw(vcpu, msw); + *rflags = kvm_x86_ops->get_rflags(vcpu); +} + +unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) +{ + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + switch (cr) { + case 0: + return vcpu->arch.cr0; + case 2: + return vcpu->arch.cr2; + case 3: + return vcpu->arch.cr3; + case 4: + return vcpu->arch.cr4; + case 8: + return get_cr8(vcpu); + default: + vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); + return 0; + } +} + +void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, + unsigned long *rflags) +{ + switch (cr) { + case 0: + set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val)); + *rflags = kvm_x86_ops->get_rflags(vcpu); + break; + case 2: + vcpu->arch.cr2 = val; + break; + case 3: + set_cr3(vcpu, val); + break; + case 4: + set_cr4(vcpu, mk_cr_64(vcpu->arch.cr4, val)); + break; + case 8: + set_cr8(vcpu, val & 0xfUL); + break; + default: + vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); + } +} + +static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i) +{ + struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i]; + int j, nent = vcpu->arch.cpuid_nent; + + e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT; + /* when no next entry is found, the current entry[i] is reselected */ + for (j = i + 1; j == i; j = (j + 1) % nent) { + struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j]; + if (ej->function == e->function) { + ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; + return j; + } + } + return 0; /* silence gcc, even though control never reaches here */ +} + +/* find an entry with matching function, matching index (if needed), and that + * should be read next (if it's stateful) */ +static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e, + u32 function, u32 index) +{ + if (e->function != function) + return 0; + if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index) + return 0; + if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) && + !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT)) + return 0; + return 1; +} + +void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) +{ + int i; + u32 function, index; + struct kvm_cpuid_entry2 *e, *best; + + kvm_x86_ops->cache_regs(vcpu); + function = vcpu->arch.regs[VCPU_REGS_RAX]; + index = vcpu->arch.regs[VCPU_REGS_RCX]; + vcpu->arch.regs[VCPU_REGS_RAX] = 0; + vcpu->arch.regs[VCPU_REGS_RBX] = 0; + vcpu->arch.regs[VCPU_REGS_RCX] = 0; + vcpu->arch.regs[VCPU_REGS_RDX] = 0; + best = NULL; + for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { + e = &vcpu->arch.cpuid_entries[i]; + if (is_matching_cpuid_entry(e, function, index)) { + if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) + move_to_next_stateful_cpuid_entry(vcpu, i); + best = e; + break; + } + /* + * Both basic or both extended? + */ + if (((e->function ^ function) & 0x80000000) == 0) + if (!best || e->function > best->function) + best = e; + } + if (best) { + vcpu->arch.regs[VCPU_REGS_RAX] = best->eax; + vcpu->arch.regs[VCPU_REGS_RBX] = best->ebx; + vcpu->arch.regs[VCPU_REGS_RCX] = best->ecx; + vcpu->arch.regs[VCPU_REGS_RDX] = best->edx; + } + kvm_x86_ops->decache_regs(vcpu); + kvm_x86_ops->skip_emulated_instruction(vcpu); +} +EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); + +/* + * Check if userspace requested an interrupt window, and that the + * interrupt window is open. + * + * No need to exit to userspace if we already have an interrupt queued. + */ +static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + return (!vcpu->arch.irq_summary && + kvm_run->request_interrupt_window && + vcpu->arch.interrupt_window_open && + (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF)); +} + +static void post_kvm_run_save(struct kvm_vcpu *vcpu, + struct kvm_run *kvm_run) +{ + kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0; + kvm_run->cr8 = get_cr8(vcpu); + kvm_run->apic_base = kvm_get_apic_base(vcpu); + if (irqchip_in_kernel(vcpu->kvm)) + kvm_run->ready_for_interrupt_injection = 1; + else + kvm_run->ready_for_interrupt_injection = + (vcpu->arch.interrupt_window_open && + vcpu->arch.irq_summary == 0); +} + +static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + int r; + + if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { + pr_debug("vcpu %d received sipi with vector # %x\n", + vcpu->vcpu_id, vcpu->arch.sipi_vector); + kvm_lapic_reset(vcpu); + r = kvm_x86_ops->vcpu_reset(vcpu); + if (r) + return r; + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; + } + +preempted: + if (vcpu->guest_debug.enabled) + kvm_x86_ops->guest_debug_pre(vcpu); + +again: + r = kvm_mmu_reload(vcpu); + if (unlikely(r)) + goto out; + + kvm_inject_pending_timer_irqs(vcpu); + + preempt_disable(); + + kvm_x86_ops->prepare_guest_switch(vcpu); + kvm_load_guest_fpu(vcpu); + + local_irq_disable(); + + if (signal_pending(current)) { + local_irq_enable(); + preempt_enable(); + r = -EINTR; + kvm_run->exit_reason = KVM_EXIT_INTR; + ++vcpu->stat.signal_exits; + goto out; + } + + if (vcpu->arch.exception.pending) + __queue_exception(vcpu); + else if (irqchip_in_kernel(vcpu->kvm)) + kvm_x86_ops->inject_pending_irq(vcpu); + else + kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); + + vcpu->guest_mode = 1; + kvm_guest_enter(); + + if (vcpu->requests) + if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) + kvm_x86_ops->tlb_flush(vcpu); + + kvm_x86_ops->run(vcpu, kvm_run); + + vcpu->guest_mode = 0; + local_irq_enable(); + + ++vcpu->stat.exits; + + /* + * We must have an instruction between local_irq_enable() and + * kvm_guest_exit(), so the timer interrupt isn't delayed by + * the interrupt shadow. The stat.exits increment will do nicely. + * But we need to prevent reordering, hence this barrier(): + */ + barrier(); + + kvm_guest_exit(); + + preempt_enable(); + + /* + * Profile KVM exit RIPs: + */ + if (unlikely(prof_on == KVM_PROFILING)) { + kvm_x86_ops->cache_regs(vcpu); + profile_hit(KVM_PROFILING, (void *)vcpu->arch.rip); + } + + if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu)) + vcpu->arch.exception.pending = false; + + r = kvm_x86_ops->handle_exit(kvm_run, vcpu); + + if (r > 0) { + if (dm_request_for_irq_injection(vcpu, kvm_run)) { + r = -EINTR; + kvm_run->exit_reason = KVM_EXIT_INTR; + ++vcpu->stat.request_irq_exits; + goto out; + } + if (!need_resched()) + goto again; + } + +out: + if (r > 0) { + kvm_resched(vcpu); + goto preempted; + } + + post_kvm_run_save(vcpu, kvm_run); + + return r; +} + +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +{ + int r; + sigset_t sigsaved; + + vcpu_load(vcpu); + + if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_UNINITIALIZED)) { + kvm_vcpu_block(vcpu); + vcpu_put(vcpu); + return -EAGAIN; + } + + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); + + /* re-sync apic's tpr */ + if (!irqchip_in_kernel(vcpu->kvm)) + set_cr8(vcpu, kvm_run->cr8); + + if (vcpu->arch.pio.cur_count) { + r = complete_pio(vcpu); + if (r) + goto out; + } +#if CONFIG_HAS_IOMEM + if (vcpu->mmio_needed) { + memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); + vcpu->mmio_read_completed = 1; + vcpu->mmio_needed = 0; + r = emulate_instruction(vcpu, kvm_run, + vcpu->arch.mmio_fault_cr2, 0, 1); + if (r == EMULATE_DO_MMIO) { + /* + * Read-modify-write. Back to userspace. + */ + r = 0; + goto out; + } + } +#endif + if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { + kvm_x86_ops->cache_regs(vcpu); + vcpu->arch.regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; + kvm_x86_ops->decache_regs(vcpu); + } + + r = __vcpu_run(vcpu, kvm_run); + +out: + if (vcpu->sigset_active) + sigprocmask(SIG_SETMASK, &sigsaved, NULL); + + vcpu_put(vcpu); + return r; +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu_load(vcpu); + + kvm_x86_ops->cache_regs(vcpu); + + regs->rax = vcpu->arch.regs[VCPU_REGS_RAX]; + regs->rbx = vcpu->arch.regs[VCPU_REGS_RBX]; + regs->rcx = vcpu->arch.regs[VCPU_REGS_RCX]; + regs->rdx = vcpu->arch.regs[VCPU_REGS_RDX]; + regs->rsi = vcpu->arch.regs[VCPU_REGS_RSI]; + regs->rdi = vcpu->arch.regs[VCPU_REGS_RDI]; + regs->rsp = vcpu->arch.regs[VCPU_REGS_RSP]; + regs->rbp = vcpu->arch.regs[VCPU_REGS_RBP]; +#ifdef CONFIG_X86_64 + regs->r8 = vcpu->arch.regs[VCPU_REGS_R8]; + regs->r9 = vcpu->arch.regs[VCPU_REGS_R9]; + regs->r10 = vcpu->arch.regs[VCPU_REGS_R10]; + regs->r11 = vcpu->arch.regs[VCPU_REGS_R11]; + regs->r12 = vcpu->arch.regs[VCPU_REGS_R12]; + regs->r13 = vcpu->arch.regs[VCPU_REGS_R13]; + regs->r14 = vcpu->arch.regs[VCPU_REGS_R14]; + regs->r15 = vcpu->arch.regs[VCPU_REGS_R15]; +#endif + + regs->rip = vcpu->arch.rip; + regs->rflags = kvm_x86_ops->get_rflags(vcpu); + + /* + * Don't leak debug flags in case they were set for guest debugging + */ + if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep) + regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu_load(vcpu); + + vcpu->arch.regs[VCPU_REGS_RAX] = regs->rax; + vcpu->arch.regs[VCPU_REGS_RBX] = regs->rbx; + vcpu->arch.regs[VCPU_REGS_RCX] = regs->rcx; + vcpu->arch.regs[VCPU_REGS_RDX] = regs->rdx; + vcpu->arch.regs[VCPU_REGS_RSI] = regs->rsi; + vcpu->arch.regs[VCPU_REGS_RDI] = regs->rdi; + vcpu->arch.regs[VCPU_REGS_RSP] = regs->rsp; + vcpu->arch.regs[VCPU_REGS_RBP] = regs->rbp; +#ifdef CONFIG_X86_64 + vcpu->arch.regs[VCPU_REGS_R8] = regs->r8; + vcpu->arch.regs[VCPU_REGS_R9] = regs->r9; + vcpu->arch.regs[VCPU_REGS_R10] = regs->r10; + vcpu->arch.regs[VCPU_REGS_R11] = regs->r11; + vcpu->arch.regs[VCPU_REGS_R12] = regs->r12; + vcpu->arch.regs[VCPU_REGS_R13] = regs->r13; + vcpu->arch.regs[VCPU_REGS_R14] = regs->r14; + vcpu->arch.regs[VCPU_REGS_R15] = regs->r15; +#endif + + vcpu->arch.rip = regs->rip; + kvm_x86_ops->set_rflags(vcpu, regs->rflags); + + kvm_x86_ops->decache_regs(vcpu); + + vcpu_put(vcpu); + + return 0; +} + +static void get_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + return kvm_x86_ops->get_segment(vcpu, var, seg); +} + +void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) +{ + struct kvm_segment cs; + + get_segment(vcpu, &cs, VCPU_SREG_CS); + *db = cs.db; + *l = cs.l; +} +EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + struct descriptor_table dt; + int pending_vec; + + vcpu_load(vcpu); + + get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + get_segment(vcpu, &sregs->es, VCPU_SREG_ES); + get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_x86_ops->get_idt(vcpu, &dt); + sregs->idt.limit = dt.limit; + sregs->idt.base = dt.base; + kvm_x86_ops->get_gdt(vcpu, &dt); + sregs->gdt.limit = dt.limit; + sregs->gdt.base = dt.base; + + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + sregs->cr0 = vcpu->arch.cr0; + sregs->cr2 = vcpu->arch.cr2; + sregs->cr3 = vcpu->arch.cr3; + sregs->cr4 = vcpu->arch.cr4; + sregs->cr8 = get_cr8(vcpu); + sregs->efer = vcpu->arch.shadow_efer; + sregs->apic_base = kvm_get_apic_base(vcpu); + + if (irqchip_in_kernel(vcpu->kvm)) { + memset(sregs->interrupt_bitmap, 0, + sizeof sregs->interrupt_bitmap); + pending_vec = kvm_x86_ops->get_irq(vcpu); + if (pending_vec >= 0) + set_bit(pending_vec, + (unsigned long *)sregs->interrupt_bitmap); + } else + memcpy(sregs->interrupt_bitmap, vcpu->arch.irq_pending, + sizeof sregs->interrupt_bitmap); + + vcpu_put(vcpu); + + return 0; +} + +static void set_segment(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg) +{ + return kvm_x86_ops->set_segment(vcpu, var, seg); +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + int mmu_reset_needed = 0; + int i, pending_vec, max_bits; + struct descriptor_table dt; + + vcpu_load(vcpu); + + dt.limit = sregs->idt.limit; + dt.base = sregs->idt.base; + kvm_x86_ops->set_idt(vcpu, &dt); + dt.limit = sregs->gdt.limit; + dt.base = sregs->gdt.base; + kvm_x86_ops->set_gdt(vcpu, &dt); + + vcpu->arch.cr2 = sregs->cr2; + mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3; + vcpu->arch.cr3 = sregs->cr3; + + set_cr8(vcpu, sregs->cr8); + + mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer; +#ifdef CONFIG_X86_64 + kvm_x86_ops->set_efer(vcpu, sregs->efer); +#endif + kvm_set_apic_base(vcpu, sregs->apic_base); + + kvm_x86_ops->decache_cr4_guest_bits(vcpu); + + mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0; + vcpu->arch.cr0 = sregs->cr0; + kvm_x86_ops->set_cr0(vcpu, sregs->cr0); + + mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4; + kvm_x86_ops->set_cr4(vcpu, sregs->cr4); + if (!is_long_mode(vcpu) && is_pae(vcpu)) + load_pdptrs(vcpu, vcpu->arch.cr3); + + if (mmu_reset_needed) + kvm_mmu_reset_context(vcpu); + + if (!irqchip_in_kernel(vcpu->kvm)) { + memcpy(vcpu->arch.irq_pending, sregs->interrupt_bitmap, + sizeof vcpu->arch.irq_pending); + vcpu->arch.irq_summary = 0; + for (i = 0; i < ARRAY_SIZE(vcpu->arch.irq_pending); ++i) + if (vcpu->arch.irq_pending[i]) + __set_bit(i, &vcpu->arch.irq_summary); + } else { + max_bits = (sizeof sregs->interrupt_bitmap) << 3; + pending_vec = find_first_bit( + (const unsigned long *)sregs->interrupt_bitmap, + max_bits); + /* Only pending external irq is handled here */ + if (pending_vec < max_bits) { + kvm_x86_ops->set_irq(vcpu, pending_vec); + pr_debug("Set back pending irq %d\n", + pending_vec); + } + } + + set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + set_segment(vcpu, &sregs->es, VCPU_SREG_ES); + set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg) +{ + int r; + + vcpu_load(vcpu); + + r = kvm_x86_ops->set_guest_debug(vcpu, dbg); + + vcpu_put(vcpu); + + return r; +} + +/* + * fxsave fpu state. Taken from x86_64/processor.h. To be killed when + * we have asm/x86/processor.h + */ +struct fxsave { + u16 cwd; + u16 swd; + u16 twd; + u16 fop; + u64 rip; + u64 rdp; + u32 mxcsr; + u32 mxcsr_mask; + u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */ +#ifdef CONFIG_X86_64 + u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */ +#else + u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */ +#endif +}; + +/* + * Translate a guest virtual address to a guest physical address. + */ +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + unsigned long vaddr = tr->linear_address; + gpa_t gpa; + + vcpu_load(vcpu); + mutex_lock(&vcpu->kvm->lock); + gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr); + tr->physical_address = gpa; + tr->valid = gpa != UNMAPPED_GVA; + tr->writeable = 1; + tr->usermode = 0; + mutex_unlock(&vcpu->kvm->lock); + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image; + + vcpu_load(vcpu); + + memcpy(fpu->fpr, fxsave->st_space, 128); + fpu->fcw = fxsave->cwd; + fpu->fsw = fxsave->swd; + fpu->ftwx = fxsave->twd; + fpu->last_opcode = fxsave->fop; + fpu->last_ip = fxsave->rip; + fpu->last_dp = fxsave->rdp; + memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space); + + vcpu_put(vcpu); + + return 0; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image; + + vcpu_load(vcpu); + + memcpy(fxsave->st_space, fpu->fpr, 128); + fxsave->cwd = fpu->fcw; + fxsave->swd = fpu->fsw; + fxsave->twd = fpu->ftwx; + fxsave->fop = fpu->last_opcode; + fxsave->rip = fpu->last_ip; + fxsave->rdp = fpu->last_dp; + memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space); + + vcpu_put(vcpu); + + return 0; +} + +void fx_init(struct kvm_vcpu *vcpu) +{ + unsigned after_mxcsr_mask; + + /* Initialize guest FPU by resetting ours and saving into guest's */ + preempt_disable(); + fx_save(&vcpu->arch.host_fx_image); + fpu_init(); + fx_save(&vcpu->arch.guest_fx_image); + fx_restore(&vcpu->arch.host_fx_image); + preempt_enable(); + + vcpu->arch.cr0 |= X86_CR0_ET; + after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space); + vcpu->arch.guest_fx_image.mxcsr = 0x1f80; + memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask, + 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask); +} +EXPORT_SYMBOL_GPL(fx_init); + +void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) +{ + if (!vcpu->fpu_active || vcpu->guest_fpu_loaded) + return; + + vcpu->guest_fpu_loaded = 1; + fx_save(&vcpu->arch.host_fx_image); + fx_restore(&vcpu->arch.guest_fx_image); +} +EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); + +void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) +{ + if (!vcpu->guest_fpu_loaded) + return; + + vcpu->guest_fpu_loaded = 0; + fx_save(&vcpu->arch.guest_fx_image); + fx_restore(&vcpu->arch.host_fx_image); + ++vcpu->stat.fpu_reload; +} +EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); + +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) +{ + kvm_x86_ops->vcpu_free(vcpu); +} + +struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, + unsigned int id) +{ + return kvm_x86_ops->vcpu_create(kvm, id); +} + +int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) +{ + int r; + + /* We do fxsave: this must be aligned. */ + BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF); + + vcpu_load(vcpu); + r = kvm_arch_vcpu_reset(vcpu); + if (r == 0) + r = kvm_mmu_setup(vcpu); + vcpu_put(vcpu); + if (r < 0) + goto free_vcpu; + + return 0; +free_vcpu: + kvm_x86_ops->vcpu_free(vcpu); + return r; +} + +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + vcpu_load(vcpu); + kvm_mmu_unload(vcpu); + vcpu_put(vcpu); + + kvm_x86_ops->vcpu_free(vcpu); +} + +int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) +{ + return kvm_x86_ops->vcpu_reset(vcpu); +} + +void kvm_arch_hardware_enable(void *garbage) +{ + kvm_x86_ops->hardware_enable(garbage); +} + +void kvm_arch_hardware_disable(void *garbage) +{ + kvm_x86_ops->hardware_disable(garbage); +} + +int kvm_arch_hardware_setup(void) +{ + return kvm_x86_ops->hardware_setup(); +} + +void kvm_arch_hardware_unsetup(void) +{ + kvm_x86_ops->hardware_unsetup(); +} + +void kvm_arch_check_processor_compat(void *rtn) +{ + kvm_x86_ops->check_processor_compatibility(rtn); +} + +int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) +{ + struct page *page; + struct kvm *kvm; + int r; + + BUG_ON(vcpu->kvm == NULL); + kvm = vcpu->kvm; + + vcpu->arch.mmu.root_hpa = INVALID_PAGE; + if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0) + vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; + else + vcpu->arch.mp_state = VCPU_MP_STATE_UNINITIALIZED; + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) { + r = -ENOMEM; + goto fail; + } + vcpu->arch.pio_data = page_address(page); + + r = kvm_mmu_create(vcpu); + if (r < 0) + goto fail_free_pio_data; + + if (irqchip_in_kernel(kvm)) { + r = kvm_create_lapic(vcpu); + if (r < 0) + goto fail_mmu_destroy; + } + + return 0; + +fail_mmu_destroy: + kvm_mmu_destroy(vcpu); +fail_free_pio_data: + free_page((unsigned long)vcpu->arch.pio_data); +fail: + return r; +} + +void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) +{ + kvm_free_lapic(vcpu); + kvm_mmu_destroy(vcpu); + free_page((unsigned long)vcpu->arch.pio_data); +} + +struct kvm *kvm_arch_create_vm(void) +{ + struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); + + if (!kvm) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); + + return kvm; +} + +static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) +{ + vcpu_load(vcpu); + kvm_mmu_unload(vcpu); + vcpu_put(vcpu); +} + +static void kvm_free_vcpus(struct kvm *kvm) +{ + unsigned int i; + + /* + * Unpin any mmu pages first. + */ + for (i = 0; i < KVM_MAX_VCPUS; ++i) + if (kvm->vcpus[i]) + kvm_unload_vcpu_mmu(kvm->vcpus[i]); + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + if (kvm->vcpus[i]) { + kvm_arch_vcpu_free(kvm->vcpus[i]); + kvm->vcpus[i] = NULL; + } + } + +} + +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + kfree(kvm->arch.vpic); + kfree(kvm->arch.vioapic); + kvm_free_vcpus(kvm); + kvm_free_physmem(kvm); + kfree(kvm); +} + +int kvm_arch_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot old, + int user_alloc) +{ + int npages = mem->memory_size >> PAGE_SHIFT; + struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot]; + + /*To keep backward compatibility with older userspace, + *x86 needs to hanlde !user_alloc case. + */ + if (!user_alloc) { + if (npages && !old.rmap) { + down_write(¤t->mm->mmap_sem); + memslot->userspace_addr = do_mmap(NULL, 0, + npages * PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, + 0); + up_write(¤t->mm->mmap_sem); + + if (IS_ERR((void *)memslot->userspace_addr)) + return PTR_ERR((void *)memslot->userspace_addr); + } else { + if (!old.user_alloc && old.rmap) { + int ret; + + down_write(¤t->mm->mmap_sem); + ret = do_munmap(current->mm, old.userspace_addr, + old.npages * PAGE_SIZE); + up_write(¤t->mm->mmap_sem); + if (ret < 0) + printk(KERN_WARNING + "kvm_vm_ioctl_set_memory_region: " + "failed to munmap memory\n"); + } + } + } + + if (!kvm->arch.n_requested_mmu_pages) { + unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); + kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); + } + + kvm_mmu_slot_remove_write_access(kvm, mem->slot); + kvm_flush_remote_tlbs(kvm); + + return 0; +} + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE + || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; +} diff --git a/arch/x86/kvm/x86_emulate.c b/arch/x86/kvm/x86_emulate.c new file mode 100644 index 000000000000..79586003397a --- /dev/null +++ b/arch/x86/kvm/x86_emulate.c @@ -0,0 +1,1912 @@ +/****************************************************************************** + * x86_emulate.c + * + * Generic x86 (32-bit and 64-bit) instruction decoder and emulator. + * + * Copyright (c) 2005 Keir Fraser + * + * Linux coding style, mod r/m decoder, segment base fixes, real-mode + * privileged instructions: + * + * Copyright (C) 2006 Qumranet + * + * Avi Kivity + * Yaniv Kamay + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4 + */ + +#ifndef __KERNEL__ +#include +#include +#include +#define DPRINTF(_f, _a ...) printf(_f , ## _a) +#else +#include +#define DPRINTF(x...) do {} while (0) +#endif +#include +#include + +/* + * Opcode effective-address decode tables. + * Note that we only emulate instructions that have at least one memory + * operand (excluding implicit stack references). We assume that stack + * references and instruction fetches will never occur in special memory + * areas that require emulation. So, for example, 'mov ,' need + * not be handled. + */ + +/* Operand sizes: 8-bit operands or specified/overridden size. */ +#define ByteOp (1<<0) /* 8-bit operands. */ +/* Destination operand type. */ +#define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */ +#define DstReg (2<<1) /* Register operand. */ +#define DstMem (3<<1) /* Memory operand. */ +#define DstMask (3<<1) +/* Source operand type. */ +#define SrcNone (0<<3) /* No source operand. */ +#define SrcImplicit (0<<3) /* Source operand is implicit in the opcode. */ +#define SrcReg (1<<3) /* Register operand. */ +#define SrcMem (2<<3) /* Memory operand. */ +#define SrcMem16 (3<<3) /* Memory operand (16-bit). */ +#define SrcMem32 (4<<3) /* Memory operand (32-bit). */ +#define SrcImm (5<<3) /* Immediate operand. */ +#define SrcImmByte (6<<3) /* 8-bit sign-extended immediate operand. */ +#define SrcMask (7<<3) +/* Generic ModRM decode. */ +#define ModRM (1<<6) +/* Destination is only written; never read. */ +#define Mov (1<<7) +#define BitOp (1<<8) +#define MemAbs (1<<9) /* Memory operand is absolute displacement */ +#define String (1<<10) /* String instruction (rep capable) */ +#define Stack (1<<11) /* Stack instruction (push/pop) */ + +static u16 opcode_table[256] = { + /* 0x00 - 0x07 */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, + 0, 0, 0, 0, + /* 0x08 - 0x0F */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, + 0, 0, 0, 0, + /* 0x10 - 0x17 */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, + 0, 0, 0, 0, + /* 0x18 - 0x1F */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, + 0, 0, 0, 0, + /* 0x20 - 0x27 */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, + SrcImmByte, SrcImm, 0, 0, + /* 0x28 - 0x2F */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, + 0, 0, 0, 0, + /* 0x30 - 0x37 */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, + 0, 0, 0, 0, + /* 0x38 - 0x3F */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, + 0, 0, 0, 0, + /* 0x40 - 0x47 */ + DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, + /* 0x48 - 0x4F */ + DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, + /* 0x50 - 0x57 */ + SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, + SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, + /* 0x58 - 0x5F */ + DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, + DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, + /* 0x60 - 0x67 */ + 0, 0, 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ , + 0, 0, 0, 0, + /* 0x68 - 0x6F */ + 0, 0, ImplicitOps | Mov | Stack, 0, + SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */ + SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */ + /* 0x70 - 0x77 */ + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + /* 0x78 - 0x7F */ + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + /* 0x80 - 0x87 */ + ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM, + ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM, + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, + /* 0x88 - 0x8F */ + ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov, + ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + 0, ModRM | DstReg, 0, DstMem | SrcNone | ModRM | Mov | Stack, + /* 0x90 - 0x9F */ + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, ImplicitOps | Stack, ImplicitOps | Stack, 0, 0, + /* 0xA0 - 0xA7 */ + ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, + ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs, + ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | String, ImplicitOps | String, + /* 0xA8 - 0xAF */ + 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, + ByteOp | ImplicitOps | String, ImplicitOps | String, + /* 0xB0 - 0xBF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0xC0 - 0xC7 */ + ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM, + 0, ImplicitOps | Stack, 0, 0, + ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov, + /* 0xC8 - 0xCF */ + 0, 0, 0, 0, 0, 0, 0, 0, + /* 0xD0 - 0xD7 */ + ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, + ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, + 0, 0, 0, 0, + /* 0xD8 - 0xDF */ + 0, 0, 0, 0, 0, 0, 0, 0, + /* 0xE0 - 0xE7 */ + 0, 0, 0, 0, 0, 0, 0, 0, + /* 0xE8 - 0xEF */ + ImplicitOps | Stack, SrcImm|ImplicitOps, 0, SrcImmByte|ImplicitOps, + 0, 0, 0, 0, + /* 0xF0 - 0xF7 */ + 0, 0, 0, 0, + ImplicitOps, ImplicitOps, + ByteOp | DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM, + /* 0xF8 - 0xFF */ + ImplicitOps, 0, ImplicitOps, ImplicitOps, + 0, 0, ByteOp | DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM +}; + +static u16 twobyte_table[256] = { + /* 0x00 - 0x0F */ + 0, SrcMem | ModRM | DstReg, 0, 0, 0, 0, ImplicitOps, 0, + ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0, + /* 0x10 - 0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0, + /* 0x20 - 0x2F */ + ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + /* 0x30 - 0x3F */ + ImplicitOps, 0, ImplicitOps, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0x40 - 0x47 */ + DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + /* 0x48 - 0x4F */ + DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, + /* 0x50 - 0x5F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0x60 - 0x6F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0x70 - 0x7F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0x80 - 0x8F */ + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, + /* 0x90 - 0x9F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0xA0 - 0xA7 */ + 0, 0, 0, DstMem | SrcReg | ModRM | BitOp, 0, 0, 0, 0, + /* 0xA8 - 0xAF */ + 0, 0, 0, DstMem | SrcReg | ModRM | BitOp, 0, 0, 0, 0, + /* 0xB0 - 0xB7 */ + ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, 0, + DstMem | SrcReg | ModRM | BitOp, + 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov, + DstReg | SrcMem16 | ModRM | Mov, + /* 0xB8 - 0xBF */ + 0, 0, DstMem | SrcImmByte | ModRM, DstMem | SrcReg | ModRM | BitOp, + 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov, + DstReg | SrcMem16 | ModRM | Mov, + /* 0xC0 - 0xCF */ + 0, 0, 0, DstMem | SrcReg | ModRM | Mov, 0, 0, 0, ImplicitOps | ModRM, + 0, 0, 0, 0, 0, 0, 0, 0, + /* 0xD0 - 0xDF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0xE0 - 0xEF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 0xF0 - 0xFF */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +/* EFLAGS bit definitions. */ +#define EFLG_OF (1<<11) +#define EFLG_DF (1<<10) +#define EFLG_SF (1<<7) +#define EFLG_ZF (1<<6) +#define EFLG_AF (1<<4) +#define EFLG_PF (1<<2) +#define EFLG_CF (1<<0) + +/* + * Instruction emulation: + * Most instructions are emulated directly via a fragment of inline assembly + * code. This allows us to save/restore EFLAGS and thus very easily pick up + * any modified flags. + */ + +#if defined(CONFIG_X86_64) +#define _LO32 "k" /* force 32-bit operand */ +#define _STK "%%rsp" /* stack pointer */ +#elif defined(__i386__) +#define _LO32 "" /* force 32-bit operand */ +#define _STK "%%esp" /* stack pointer */ +#endif + +/* + * These EFLAGS bits are restored from saved value during emulation, and + * any changes are written back to the saved value after emulation. + */ +#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF) + +/* Before executing instruction: restore necessary bits in EFLAGS. */ +#define _PRE_EFLAGS(_sav, _msk, _tmp) \ + /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \ + "movl %"_sav",%"_LO32 _tmp"; " \ + "push %"_tmp"; " \ + "push %"_tmp"; " \ + "movl %"_msk",%"_LO32 _tmp"; " \ + "andl %"_LO32 _tmp",("_STK"); " \ + "pushf; " \ + "notl %"_LO32 _tmp"; " \ + "andl %"_LO32 _tmp",("_STK"); " \ + "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \ + "pop %"_tmp"; " \ + "orl %"_LO32 _tmp",("_STK"); " \ + "popf; " \ + "pop %"_sav"; " + +/* After executing instruction: write-back necessary bits in EFLAGS. */ +#define _POST_EFLAGS(_sav, _msk, _tmp) \ + /* _sav |= EFLAGS & _msk; */ \ + "pushf; " \ + "pop %"_tmp"; " \ + "andl %"_msk",%"_LO32 _tmp"; " \ + "orl %"_LO32 _tmp",%"_sav"; " + +/* Raw emulation: instruction has two explicit operands. */ +#define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \ + do { \ + unsigned long _tmp; \ + \ + switch ((_dst).bytes) { \ + case 2: \ + __asm__ __volatile__ ( \ + _PRE_EFLAGS("0", "4", "2") \ + _op"w %"_wx"3,%1; " \ + _POST_EFLAGS("0", "4", "2") \ + : "=m" (_eflags), "=m" ((_dst).val), \ + "=&r" (_tmp) \ + : _wy ((_src).val), "i" (EFLAGS_MASK)); \ + break; \ + case 4: \ + __asm__ __volatile__ ( \ + _PRE_EFLAGS("0", "4", "2") \ + _op"l %"_lx"3,%1; " \ + _POST_EFLAGS("0", "4", "2") \ + : "=m" (_eflags), "=m" ((_dst).val), \ + "=&r" (_tmp) \ + : _ly ((_src).val), "i" (EFLAGS_MASK)); \ + break; \ + case 8: \ + __emulate_2op_8byte(_op, _src, _dst, \ + _eflags, _qx, _qy); \ + break; \ + } \ + } while (0) + +#define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \ + do { \ + unsigned long _tmp; \ + switch ((_dst).bytes) { \ + case 1: \ + __asm__ __volatile__ ( \ + _PRE_EFLAGS("0", "4", "2") \ + _op"b %"_bx"3,%1; " \ + _POST_EFLAGS("0", "4", "2") \ + : "=m" (_eflags), "=m" ((_dst).val), \ + "=&r" (_tmp) \ + : _by ((_src).val), "i" (EFLAGS_MASK)); \ + break; \ + default: \ + __emulate_2op_nobyte(_op, _src, _dst, _eflags, \ + _wx, _wy, _lx, _ly, _qx, _qy); \ + break; \ + } \ + } while (0) + +/* Source operand is byte-sized and may be restricted to just %cl. */ +#define emulate_2op_SrcB(_op, _src, _dst, _eflags) \ + __emulate_2op(_op, _src, _dst, _eflags, \ + "b", "c", "b", "c", "b", "c", "b", "c") + +/* Source operand is byte, word, long or quad sized. */ +#define emulate_2op_SrcV(_op, _src, _dst, _eflags) \ + __emulate_2op(_op, _src, _dst, _eflags, \ + "b", "q", "w", "r", _LO32, "r", "", "r") + +/* Source operand is word, long or quad sized. */ +#define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \ + __emulate_2op_nobyte(_op, _src, _dst, _eflags, \ + "w", "r", _LO32, "r", "", "r") + +/* Instruction has only one explicit operand (no source operand). */ +#define emulate_1op(_op, _dst, _eflags) \ + do { \ + unsigned long _tmp; \ + \ + switch ((_dst).bytes) { \ + case 1: \ + __asm__ __volatile__ ( \ + _PRE_EFLAGS("0", "3", "2") \ + _op"b %1; " \ + _POST_EFLAGS("0", "3", "2") \ + : "=m" (_eflags), "=m" ((_dst).val), \ + "=&r" (_tmp) \ + : "i" (EFLAGS_MASK)); \ + break; \ + case 2: \ + __asm__ __volatile__ ( \ + _PRE_EFLAGS("0", "3", "2") \ + _op"w %1; " \ + _POST_EFLAGS("0", "3", "2") \ + : "=m" (_eflags), "=m" ((_dst).val), \ + "=&r" (_tmp) \ + : "i" (EFLAGS_MASK)); \ + break; \ + case 4: \ + __asm__ __volatile__ ( \ + _PRE_EFLAGS("0", "3", "2") \ + _op"l %1; " \ + _POST_EFLAGS("0", "3", "2") \ + : "=m" (_eflags), "=m" ((_dst).val), \ + "=&r" (_tmp) \ + : "i" (EFLAGS_MASK)); \ + break; \ + case 8: \ + __emulate_1op_8byte(_op, _dst, _eflags); \ + break; \ + } \ + } while (0) + +/* Emulate an instruction with quadword operands (x86/64 only). */ +#if defined(CONFIG_X86_64) +#define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy) \ + do { \ + __asm__ __volatile__ ( \ + _PRE_EFLAGS("0", "4", "2") \ + _op"q %"_qx"3,%1; " \ + _POST_EFLAGS("0", "4", "2") \ + : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \ + : _qy ((_src).val), "i" (EFLAGS_MASK)); \ + } while (0) + +#define __emulate_1op_8byte(_op, _dst, _eflags) \ + do { \ + __asm__ __volatile__ ( \ + _PRE_EFLAGS("0", "3", "2") \ + _op"q %1; " \ + _POST_EFLAGS("0", "3", "2") \ + : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \ + : "i" (EFLAGS_MASK)); \ + } while (0) + +#elif defined(__i386__) +#define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy) +#define __emulate_1op_8byte(_op, _dst, _eflags) +#endif /* __i386__ */ + +/* Fetch next part of the instruction being emulated. */ +#define insn_fetch(_type, _size, _eip) \ +({ unsigned long _x; \ + rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \ + if (rc != 0) \ + goto done; \ + (_eip) += (_size); \ + (_type)_x; \ +}) + +/* Access/update address held in a register, based on addressing mode. */ +#define address_mask(reg) \ + ((c->ad_bytes == sizeof(unsigned long)) ? \ + (reg) : ((reg) & ((1UL << (c->ad_bytes << 3)) - 1))) +#define register_address(base, reg) \ + ((base) + address_mask(reg)) +#define register_address_increment(reg, inc) \ + do { \ + /* signed type ensures sign extension to long */ \ + int _inc = (inc); \ + if (c->ad_bytes == sizeof(unsigned long)) \ + (reg) += _inc; \ + else \ + (reg) = ((reg) & \ + ~((1UL << (c->ad_bytes << 3)) - 1)) | \ + (((reg) + _inc) & \ + ((1UL << (c->ad_bytes << 3)) - 1)); \ + } while (0) + +#define JMP_REL(rel) \ + do { \ + register_address_increment(c->eip, rel); \ + } while (0) + +static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long linear, u8 *dest) +{ + struct fetch_cache *fc = &ctxt->decode.fetch; + int rc; + int size; + + if (linear < fc->start || linear >= fc->end) { + size = min(15UL, PAGE_SIZE - offset_in_page(linear)); + rc = ops->read_std(linear, fc->data, size, ctxt->vcpu); + if (rc) + return rc; + fc->start = linear; + fc->end = linear + size; + } + *dest = fc->data[linear - fc->start]; + return 0; +} + +static int do_insn_fetch(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long eip, void *dest, unsigned size) +{ + int rc = 0; + + eip += ctxt->cs_base; + while (size--) { + rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++); + if (rc) + return rc; + } + return 0; +} + +/* + * Given the 'reg' portion of a ModRM byte, and a register block, return a + * pointer into the block that addresses the relevant register. + * @highbyte_regs specifies whether to decode AH,CH,DH,BH. + */ +static void *decode_register(u8 modrm_reg, unsigned long *regs, + int highbyte_regs) +{ + void *p; + + p = ®s[modrm_reg]; + if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8) + p = (unsigned char *)®s[modrm_reg & 3] + 1; + return p; +} + +static int read_descriptor(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + void *ptr, + u16 *size, unsigned long *address, int op_bytes) +{ + int rc; + + if (op_bytes == 2) + op_bytes = 3; + *address = 0; + rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2, + ctxt->vcpu); + if (rc) + return rc; + rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes, + ctxt->vcpu); + return rc; +} + +static int test_cc(unsigned int condition, unsigned int flags) +{ + int rc = 0; + + switch ((condition & 15) >> 1) { + case 0: /* o */ + rc |= (flags & EFLG_OF); + break; + case 1: /* b/c/nae */ + rc |= (flags & EFLG_CF); + break; + case 2: /* z/e */ + rc |= (flags & EFLG_ZF); + break; + case 3: /* be/na */ + rc |= (flags & (EFLG_CF|EFLG_ZF)); + break; + case 4: /* s */ + rc |= (flags & EFLG_SF); + break; + case 5: /* p/pe */ + rc |= (flags & EFLG_PF); + break; + case 7: /* le/ng */ + rc |= (flags & EFLG_ZF); + /* fall through */ + case 6: /* l/nge */ + rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF)); + break; + } + + /* Odd condition identifiers (lsb == 1) have inverted sense. */ + return (!!rc ^ (condition & 1)); +} + +static void decode_register_operand(struct operand *op, + struct decode_cache *c, + int inhibit_bytereg) +{ + unsigned reg = c->modrm_reg; + int highbyte_regs = c->rex_prefix == 0; + + if (!(c->d & ModRM)) + reg = (c->b & 7) | ((c->rex_prefix & 1) << 3); + op->type = OP_REG; + if ((c->d & ByteOp) && !inhibit_bytereg) { + op->ptr = decode_register(reg, c->regs, highbyte_regs); + op->val = *(u8 *)op->ptr; + op->bytes = 1; + } else { + op->ptr = decode_register(reg, c->regs, 0); + op->bytes = c->op_bytes; + switch (op->bytes) { + case 2: + op->val = *(u16 *)op->ptr; + break; + case 4: + op->val = *(u32 *)op->ptr; + break; + case 8: + op->val = *(u64 *) op->ptr; + break; + } + } + op->orig_val = op->val; +} + +static int decode_modrm(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + u8 sib; + int index_reg = 0, base_reg = 0, scale, rip_relative = 0; + int rc = 0; + + if (c->rex_prefix) { + c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */ + index_reg = (c->rex_prefix & 2) << 2; /* REX.X */ + c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */ + } + + c->modrm = insn_fetch(u8, 1, c->eip); + c->modrm_mod |= (c->modrm & 0xc0) >> 6; + c->modrm_reg |= (c->modrm & 0x38) >> 3; + c->modrm_rm |= (c->modrm & 0x07); + c->modrm_ea = 0; + c->use_modrm_ea = 1; + + if (c->modrm_mod == 3) { + c->modrm_val = *(unsigned long *) + decode_register(c->modrm_rm, c->regs, c->d & ByteOp); + return rc; + } + + if (c->ad_bytes == 2) { + unsigned bx = c->regs[VCPU_REGS_RBX]; + unsigned bp = c->regs[VCPU_REGS_RBP]; + unsigned si = c->regs[VCPU_REGS_RSI]; + unsigned di = c->regs[VCPU_REGS_RDI]; + + /* 16-bit ModR/M decode. */ + switch (c->modrm_mod) { + case 0: + if (c->modrm_rm == 6) + c->modrm_ea += insn_fetch(u16, 2, c->eip); + break; + case 1: + c->modrm_ea += insn_fetch(s8, 1, c->eip); + break; + case 2: + c->modrm_ea += insn_fetch(u16, 2, c->eip); + break; + } + switch (c->modrm_rm) { + case 0: + c->modrm_ea += bx + si; + break; + case 1: + c->modrm_ea += bx + di; + break; + case 2: + c->modrm_ea += bp + si; + break; + case 3: + c->modrm_ea += bp + di; + break; + case 4: + c->modrm_ea += si; + break; + case 5: + c->modrm_ea += di; + break; + case 6: + if (c->modrm_mod != 0) + c->modrm_ea += bp; + break; + case 7: + c->modrm_ea += bx; + break; + } + if (c->modrm_rm == 2 || c->modrm_rm == 3 || + (c->modrm_rm == 6 && c->modrm_mod != 0)) + if (!c->override_base) + c->override_base = &ctxt->ss_base; + c->modrm_ea = (u16)c->modrm_ea; + } else { + /* 32/64-bit ModR/M decode. */ + switch (c->modrm_rm) { + case 4: + case 12: + sib = insn_fetch(u8, 1, c->eip); + index_reg |= (sib >> 3) & 7; + base_reg |= sib & 7; + scale = sib >> 6; + + switch (base_reg) { + case 5: + if (c->modrm_mod != 0) + c->modrm_ea += c->regs[base_reg]; + else + c->modrm_ea += + insn_fetch(s32, 4, c->eip); + break; + default: + c->modrm_ea += c->regs[base_reg]; + } + switch (index_reg) { + case 4: + break; + default: + c->modrm_ea += c->regs[index_reg] << scale; + } + break; + case 5: + if (c->modrm_mod != 0) + c->modrm_ea += c->regs[c->modrm_rm]; + else if (ctxt->mode == X86EMUL_MODE_PROT64) + rip_relative = 1; + break; + default: + c->modrm_ea += c->regs[c->modrm_rm]; + break; + } + switch (c->modrm_mod) { + case 0: + if (c->modrm_rm == 5) + c->modrm_ea += insn_fetch(s32, 4, c->eip); + break; + case 1: + c->modrm_ea += insn_fetch(s8, 1, c->eip); + break; + case 2: + c->modrm_ea += insn_fetch(s32, 4, c->eip); + break; + } + } + if (rip_relative) { + c->modrm_ea += c->eip; + switch (c->d & SrcMask) { + case SrcImmByte: + c->modrm_ea += 1; + break; + case SrcImm: + if (c->d & ByteOp) + c->modrm_ea += 1; + else + if (c->op_bytes == 8) + c->modrm_ea += 4; + else + c->modrm_ea += c->op_bytes; + } + } +done: + return rc; +} + +static int decode_abs(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc = 0; + + switch (c->ad_bytes) { + case 2: + c->modrm_ea = insn_fetch(u16, 2, c->eip); + break; + case 4: + c->modrm_ea = insn_fetch(u32, 4, c->eip); + break; + case 8: + c->modrm_ea = insn_fetch(u64, 8, c->eip); + break; + } +done: + return rc; +} + +int +x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc = 0; + int mode = ctxt->mode; + int def_op_bytes, def_ad_bytes; + + /* Shadow copy of register state. Committed on successful emulation. */ + + memset(c, 0, sizeof(struct decode_cache)); + c->eip = ctxt->vcpu->arch.rip; + memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); + + switch (mode) { + case X86EMUL_MODE_REAL: + case X86EMUL_MODE_PROT16: + def_op_bytes = def_ad_bytes = 2; + break; + case X86EMUL_MODE_PROT32: + def_op_bytes = def_ad_bytes = 4; + break; +#ifdef CONFIG_X86_64 + case X86EMUL_MODE_PROT64: + def_op_bytes = 4; + def_ad_bytes = 8; + break; +#endif + default: + return -1; + } + + c->op_bytes = def_op_bytes; + c->ad_bytes = def_ad_bytes; + + /* Legacy prefixes. */ + for (;;) { + switch (c->b = insn_fetch(u8, 1, c->eip)) { + case 0x66: /* operand-size override */ + /* switch between 2/4 bytes */ + c->op_bytes = def_op_bytes ^ 6; + break; + case 0x67: /* address-size override */ + if (mode == X86EMUL_MODE_PROT64) + /* switch between 4/8 bytes */ + c->ad_bytes = def_ad_bytes ^ 12; + else + /* switch between 2/4 bytes */ + c->ad_bytes = def_ad_bytes ^ 6; + break; + case 0x2e: /* CS override */ + c->override_base = &ctxt->cs_base; + break; + case 0x3e: /* DS override */ + c->override_base = &ctxt->ds_base; + break; + case 0x26: /* ES override */ + c->override_base = &ctxt->es_base; + break; + case 0x64: /* FS override */ + c->override_base = &ctxt->fs_base; + break; + case 0x65: /* GS override */ + c->override_base = &ctxt->gs_base; + break; + case 0x36: /* SS override */ + c->override_base = &ctxt->ss_base; + break; + case 0x40 ... 0x4f: /* REX */ + if (mode != X86EMUL_MODE_PROT64) + goto done_prefixes; + c->rex_prefix = c->b; + continue; + case 0xf0: /* LOCK */ + c->lock_prefix = 1; + break; + case 0xf2: /* REPNE/REPNZ */ + c->rep_prefix = REPNE_PREFIX; + break; + case 0xf3: /* REP/REPE/REPZ */ + c->rep_prefix = REPE_PREFIX; + break; + default: + goto done_prefixes; + } + + /* Any legacy prefix after a REX prefix nullifies its effect. */ + + c->rex_prefix = 0; + } + +done_prefixes: + + /* REX prefix. */ + if (c->rex_prefix) + if (c->rex_prefix & 8) + c->op_bytes = 8; /* REX.W */ + + /* Opcode byte(s). */ + c->d = opcode_table[c->b]; + if (c->d == 0) { + /* Two-byte opcode? */ + if (c->b == 0x0f) { + c->twobyte = 1; + c->b = insn_fetch(u8, 1, c->eip); + c->d = twobyte_table[c->b]; + } + + /* Unrecognised? */ + if (c->d == 0) { + DPRINTF("Cannot emulate %02x\n", c->b); + return -1; + } + } + + if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack)) + c->op_bytes = 8; + + /* ModRM and SIB bytes. */ + if (c->d & ModRM) + rc = decode_modrm(ctxt, ops); + else if (c->d & MemAbs) + rc = decode_abs(ctxt, ops); + if (rc) + goto done; + + if (!c->override_base) + c->override_base = &ctxt->ds_base; + if (mode == X86EMUL_MODE_PROT64 && + c->override_base != &ctxt->fs_base && + c->override_base != &ctxt->gs_base) + c->override_base = NULL; + + if (c->override_base) + c->modrm_ea += *c->override_base; + + if (c->ad_bytes != 8) + c->modrm_ea = (u32)c->modrm_ea; + /* + * Decode and fetch the source operand: register, memory + * or immediate. + */ + switch (c->d & SrcMask) { + case SrcNone: + break; + case SrcReg: + decode_register_operand(&c->src, c, 0); + break; + case SrcMem16: + c->src.bytes = 2; + goto srcmem_common; + case SrcMem32: + c->src.bytes = 4; + goto srcmem_common; + case SrcMem: + c->src.bytes = (c->d & ByteOp) ? 1 : + c->op_bytes; + /* Don't fetch the address for invlpg: it could be unmapped. */ + if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7) + break; + srcmem_common: + /* + * For instructions with a ModR/M byte, switch to register + * access if Mod = 3. + */ + if ((c->d & ModRM) && c->modrm_mod == 3) { + c->src.type = OP_REG; + break; + } + c->src.type = OP_MEM; + break; + case SrcImm: + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + if (c->src.bytes == 8) + c->src.bytes = 4; + /* NB. Immediates are sign-extended as necessary. */ + switch (c->src.bytes) { + case 1: + c->src.val = insn_fetch(s8, 1, c->eip); + break; + case 2: + c->src.val = insn_fetch(s16, 2, c->eip); + break; + case 4: + c->src.val = insn_fetch(s32, 4, c->eip); + break; + } + break; + case SrcImmByte: + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = 1; + c->src.val = insn_fetch(s8, 1, c->eip); + break; + } + + /* Decode and fetch the destination operand: register or memory. */ + switch (c->d & DstMask) { + case ImplicitOps: + /* Special instructions do their own operand decoding. */ + return 0; + case DstReg: + decode_register_operand(&c->dst, c, + c->twobyte && (c->b == 0xb6 || c->b == 0xb7)); + break; + case DstMem: + if ((c->d & ModRM) && c->modrm_mod == 3) { + c->dst.type = OP_REG; + break; + } + c->dst.type = OP_MEM; + break; + } + +done: + return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; +} + +static inline void emulate_push(struct x86_emulate_ctxt *ctxt) +{ + struct decode_cache *c = &ctxt->decode; + + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], -c->op_bytes); + c->dst.ptr = (void *) register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]); +} + +static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc; + + rc = ops->read_std(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), + &c->dst.val, c->dst.bytes, ctxt->vcpu); + if (rc != 0) + return rc; + + register_address_increment(c->regs[VCPU_REGS_RSP], c->dst.bytes); + + return 0; +} + +static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt) +{ + struct decode_cache *c = &ctxt->decode; + switch (c->modrm_reg) { + case 0: /* rol */ + emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags); + break; + case 1: /* ror */ + emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags); + break; + case 2: /* rcl */ + emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags); + break; + case 3: /* rcr */ + emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags); + break; + case 4: /* sal/shl */ + case 6: /* sal/shl */ + emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags); + break; + case 5: /* shr */ + emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags); + break; + case 7: /* sar */ + emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags); + break; + } +} + +static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc = 0; + + switch (c->modrm_reg) { + case 0 ... 1: /* test */ + /* + * Special case in Grp3: test has an immediate + * source operand. + */ + c->src.type = OP_IMM; + c->src.ptr = (unsigned long *)c->eip; + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + if (c->src.bytes == 8) + c->src.bytes = 4; + switch (c->src.bytes) { + case 1: + c->src.val = insn_fetch(s8, 1, c->eip); + break; + case 2: + c->src.val = insn_fetch(s16, 2, c->eip); + break; + case 4: + c->src.val = insn_fetch(s32, 4, c->eip); + break; + } + emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags); + break; + case 2: /* not */ + c->dst.val = ~c->dst.val; + break; + case 3: /* neg */ + emulate_1op("neg", c->dst, ctxt->eflags); + break; + default: + DPRINTF("Cannot emulate %02x\n", c->b); + rc = X86EMUL_UNHANDLEABLE; + break; + } +done: + return rc; +} + +static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + struct decode_cache *c = &ctxt->decode; + int rc; + + switch (c->modrm_reg) { + case 0: /* inc */ + emulate_1op("inc", c->dst, ctxt->eflags); + break; + case 1: /* dec */ + emulate_1op("dec", c->dst, ctxt->eflags); + break; + case 4: /* jmp abs */ + if (c->b == 0xff) + c->eip = c->dst.val; + else { + DPRINTF("Cannot emulate %02x\n", c->b); + return X86EMUL_UNHANDLEABLE; + } + break; + case 6: /* push */ + + /* 64-bit mode: PUSH always pushes a 64-bit operand. */ + + if (ctxt->mode == X86EMUL_MODE_PROT64) { + c->dst.bytes = 8; + rc = ops->read_std((unsigned long)c->dst.ptr, + &c->dst.val, 8, ctxt->vcpu); + if (rc != 0) + return rc; + } + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->dst.bytes); + rc = ops->write_emulated(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), &c->dst.val, + c->dst.bytes, ctxt->vcpu); + if (rc != 0) + return rc; + c->dst.type = OP_NONE; + break; + default: + DPRINTF("Cannot emulate %02x\n", c->b); + return X86EMUL_UNHANDLEABLE; + } + return 0; +} + +static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops, + unsigned long memop) +{ + struct decode_cache *c = &ctxt->decode; + u64 old, new; + int rc; + + rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu); + if (rc != 0) + return rc; + + if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) || + ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) { + + c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); + c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); + ctxt->eflags &= ~EFLG_ZF; + + } else { + new = ((u64)c->regs[VCPU_REGS_RCX] << 32) | + (u32) c->regs[VCPU_REGS_RBX]; + + rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu); + if (rc != 0) + return rc; + ctxt->eflags |= EFLG_ZF; + } + return 0; +} + +static inline int writeback(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops) +{ + int rc; + struct decode_cache *c = &ctxt->decode; + + switch (c->dst.type) { + case OP_REG: + /* The 4-byte case *is* correct: + * in 64-bit mode we zero-extend. + */ + switch (c->dst.bytes) { + case 1: + *(u8 *)c->dst.ptr = (u8)c->dst.val; + break; + case 2: + *(u16 *)c->dst.ptr = (u16)c->dst.val; + break; + case 4: + *c->dst.ptr = (u32)c->dst.val; + break; /* 64b: zero-ext */ + case 8: + *c->dst.ptr = c->dst.val; + break; + } + break; + case OP_MEM: + if (c->lock_prefix) + rc = ops->cmpxchg_emulated( + (unsigned long)c->dst.ptr, + &c->dst.orig_val, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); + else + rc = ops->write_emulated( + (unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu); + if (rc != 0) + return rc; + break; + case OP_NONE: + /* no writeback */ + break; + default: + break; + } + return 0; +} + +int +x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) +{ + unsigned long memop = 0; + u64 msr_data; + unsigned long saved_eip = 0; + struct decode_cache *c = &ctxt->decode; + int rc = 0; + + /* Shadow copy of register state. Committed on successful emulation. + * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't + * modify them. + */ + + memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); + saved_eip = c->eip; + + if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) + memop = c->modrm_ea; + + if (c->rep_prefix && (c->d & String)) { + /* All REP prefixes have the same first termination condition */ + if (c->regs[VCPU_REGS_RCX] == 0) { + ctxt->vcpu->arch.rip = c->eip; + goto done; + } + /* The second termination condition only applies for REPE + * and REPNE. Test if the repeat string operation prefix is + * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the + * corresponding termination condition according to: + * - if REPE/REPZ and ZF = 0 then done + * - if REPNE/REPNZ and ZF = 1 then done + */ + if ((c->b == 0xa6) || (c->b == 0xa7) || + (c->b == 0xae) || (c->b == 0xaf)) { + if ((c->rep_prefix == REPE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == 0)) { + ctxt->vcpu->arch.rip = c->eip; + goto done; + } + if ((c->rep_prefix == REPNE_PREFIX) && + ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { + ctxt->vcpu->arch.rip = c->eip; + goto done; + } + } + c->regs[VCPU_REGS_RCX]--; + c->eip = ctxt->vcpu->arch.rip; + } + + if (c->src.type == OP_MEM) { + c->src.ptr = (unsigned long *)memop; + c->src.val = 0; + rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, + ctxt->vcpu); + if (rc != 0) + goto done; + c->src.orig_val = c->src.val; + } + + if ((c->d & DstMask) == ImplicitOps) + goto special_insn; + + + if (c->dst.type == OP_MEM) { + c->dst.ptr = (unsigned long *)memop; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.val = 0; + if (c->d & BitOp) { + unsigned long mask = ~(c->dst.bytes * 8 - 1); + + c->dst.ptr = (void *)c->dst.ptr + + (c->src.val & mask) / 8; + } + if (!(c->d & Mov) && + /* optimisation - avoid slow emulated read */ + ((rc = ops->read_emulated((unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0)) + goto done; + } + c->dst.orig_val = c->dst.val; + +special_insn: + + if (c->twobyte) + goto twobyte_insn; + + switch (c->b) { + case 0x00 ... 0x05: + add: /* add */ + emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags); + break; + case 0x08 ... 0x0d: + or: /* or */ + emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags); + break; + case 0x10 ... 0x15: + adc: /* adc */ + emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags); + break; + case 0x18 ... 0x1d: + sbb: /* sbb */ + emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags); + break; + case 0x20 ... 0x23: + and: /* and */ + emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags); + break; + case 0x24: /* and al imm8 */ + c->dst.type = OP_REG; + c->dst.ptr = &c->regs[VCPU_REGS_RAX]; + c->dst.val = *(u8 *)c->dst.ptr; + c->dst.bytes = 1; + c->dst.orig_val = c->dst.val; + goto and; + case 0x25: /* and ax imm16, or eax imm32 */ + c->dst.type = OP_REG; + c->dst.bytes = c->op_bytes; + c->dst.ptr = &c->regs[VCPU_REGS_RAX]; + if (c->op_bytes == 2) + c->dst.val = *(u16 *)c->dst.ptr; + else + c->dst.val = *(u32 *)c->dst.ptr; + c->dst.orig_val = c->dst.val; + goto and; + case 0x28 ... 0x2d: + sub: /* sub */ + emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags); + break; + case 0x30 ... 0x35: + xor: /* xor */ + emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags); + break; + case 0x38 ... 0x3d: + cmp: /* cmp */ + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); + break; + case 0x40 ... 0x47: /* inc r16/r32 */ + emulate_1op("inc", c->dst, ctxt->eflags); + break; + case 0x48 ... 0x4f: /* dec r16/r32 */ + emulate_1op("dec", c->dst, ctxt->eflags); + break; + case 0x50 ... 0x57: /* push reg */ + c->dst.type = OP_MEM; + c->dst.bytes = c->op_bytes; + c->dst.val = c->src.val; + register_address_increment(c->regs[VCPU_REGS_RSP], + -c->op_bytes); + c->dst.ptr = (void *) register_address( + ctxt->ss_base, c->regs[VCPU_REGS_RSP]); + break; + case 0x58 ... 0x5f: /* pop reg */ + pop_instruction: + if ((rc = ops->read_std(register_address(ctxt->ss_base, + c->regs[VCPU_REGS_RSP]), c->dst.ptr, + c->op_bytes, ctxt->vcpu)) != 0) + goto done; + + register_address_increment(c->regs[VCPU_REGS_RSP], + c->op_bytes); + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0x63: /* movsxd */ + if (ctxt->mode != X86EMUL_MODE_PROT64) + goto cannot_emulate; + c->dst.val = (s32) c->src.val; + break; + case 0x6a: /* push imm8 */ + c->src.val = 0L; + c->src.val = insn_fetch(s8, 1, c->eip); + emulate_push(ctxt); + break; + case 0x6c: /* insb */ + case 0x6d: /* insw/insd */ + if (kvm_emulate_pio_string(ctxt->vcpu, NULL, + 1, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (ctxt->eflags & EFLG_DF), + register_address(ctxt->es_base, + c->regs[VCPU_REGS_RDI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; + return -1; + } + return 0; + case 0x6e: /* outsb */ + case 0x6f: /* outsw/outsd */ + if (kvm_emulate_pio_string(ctxt->vcpu, NULL, + 0, + (c->d & ByteOp) ? 1 : c->op_bytes, + c->rep_prefix ? + address_mask(c->regs[VCPU_REGS_RCX]) : 1, + (ctxt->eflags & EFLG_DF), + register_address(c->override_base ? + *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + c->rep_prefix, + c->regs[VCPU_REGS_RDX]) == 0) { + c->eip = saved_eip; + return -1; + } + return 0; + case 0x70 ... 0x7f: /* jcc (short) */ { + int rel = insn_fetch(s8, 1, c->eip); + + if (test_cc(c->b, ctxt->eflags)) + JMP_REL(rel); + break; + } + case 0x80 ... 0x83: /* Grp1 */ + switch (c->modrm_reg) { + case 0: + goto add; + case 1: + goto or; + case 2: + goto adc; + case 3: + goto sbb; + case 4: + goto and; + case 5: + goto sub; + case 6: + goto xor; + case 7: + goto cmp; + } + break; + case 0x84 ... 0x85: + emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags); + break; + case 0x86 ... 0x87: /* xchg */ + /* Write back the register source. */ + switch (c->dst.bytes) { + case 1: + *(u8 *) c->src.ptr = (u8) c->dst.val; + break; + case 2: + *(u16 *) c->src.ptr = (u16) c->dst.val; + break; + case 4: + *c->src.ptr = (u32) c->dst.val; + break; /* 64b reg: zero-extend */ + case 8: + *c->src.ptr = c->dst.val; + break; + } + /* + * Write back the memory destination with implicit LOCK + * prefix. + */ + c->dst.val = c->src.val; + c->lock_prefix = 1; + break; + case 0x88 ... 0x8b: /* mov */ + goto mov; + case 0x8d: /* lea r16/r32, m */ + c->dst.val = c->modrm_val; + break; + case 0x8f: /* pop (sole member of Grp1a) */ + rc = emulate_grp1a(ctxt, ops); + if (rc != 0) + goto done; + break; + case 0x9c: /* pushf */ + c->src.val = (unsigned long) ctxt->eflags; + emulate_push(ctxt); + break; + case 0x9d: /* popf */ + c->dst.ptr = (unsigned long *) &ctxt->eflags; + goto pop_instruction; + case 0xa0 ... 0xa1: /* mov */ + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + c->dst.val = c->src.val; + break; + case 0xa2 ... 0xa3: /* mov */ + c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; + break; + case 0xa4 ... 0xa5: /* movs */ + c->dst.type = OP_MEM; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); + if ((rc = ops->read_emulated(register_address( + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + &c->dst.val, + c->dst.bytes, ctxt->vcpu)) != 0) + goto done; + register_address_increment(c->regs[VCPU_REGS_RSI], + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + register_address_increment(c->regs[VCPU_REGS_RDI], + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + break; + case 0xa6 ... 0xa7: /* cmps */ + c->src.type = OP_NONE; /* Disable writeback. */ + c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->src.ptr = (unsigned long *)register_address( + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]); + if ((rc = ops->read_emulated((unsigned long)c->src.ptr, + &c->src.val, + c->src.bytes, + ctxt->vcpu)) != 0) + goto done; + + c->dst.type = OP_NONE; /* Disable writeback. */ + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); + if ((rc = ops->read_emulated((unsigned long)c->dst.ptr, + &c->dst.val, + c->dst.bytes, + ctxt->vcpu)) != 0) + goto done; + + DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr); + + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); + + register_address_increment(c->regs[VCPU_REGS_RSI], + (ctxt->eflags & EFLG_DF) ? -c->src.bytes + : c->src.bytes); + register_address_increment(c->regs[VCPU_REGS_RDI], + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + + break; + case 0xaa ... 0xab: /* stos */ + c->dst.type = OP_MEM; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)register_address( + ctxt->es_base, + c->regs[VCPU_REGS_RDI]); + c->dst.val = c->regs[VCPU_REGS_RAX]; + register_address_increment(c->regs[VCPU_REGS_RDI], + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + break; + case 0xac ... 0xad: /* lods */ + c->dst.type = OP_REG; + c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + if ((rc = ops->read_emulated(register_address( + c->override_base ? *c->override_base : + ctxt->ds_base, + c->regs[VCPU_REGS_RSI]), + &c->dst.val, + c->dst.bytes, + ctxt->vcpu)) != 0) + goto done; + register_address_increment(c->regs[VCPU_REGS_RSI], + (ctxt->eflags & EFLG_DF) ? -c->dst.bytes + : c->dst.bytes); + break; + case 0xae ... 0xaf: /* scas */ + DPRINTF("Urk! I don't handle SCAS.\n"); + goto cannot_emulate; + case 0xc0 ... 0xc1: + emulate_grp2(ctxt); + break; + case 0xc3: /* ret */ + c->dst.ptr = &c->eip; + goto pop_instruction; + case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ + mov: + c->dst.val = c->src.val; + break; + case 0xd0 ... 0xd1: /* Grp2 */ + c->src.val = 1; + emulate_grp2(ctxt); + break; + case 0xd2 ... 0xd3: /* Grp2 */ + c->src.val = c->regs[VCPU_REGS_RCX]; + emulate_grp2(ctxt); + break; + case 0xe8: /* call (near) */ { + long int rel; + switch (c->op_bytes) { + case 2: + rel = insn_fetch(s16, 2, c->eip); + break; + case 4: + rel = insn_fetch(s32, 4, c->eip); + break; + default: + DPRINTF("Call: Invalid op_bytes\n"); + goto cannot_emulate; + } + c->src.val = (unsigned long) c->eip; + JMP_REL(rel); + c->op_bytes = c->ad_bytes; + emulate_push(ctxt); + break; + } + case 0xe9: /* jmp rel */ + case 0xeb: /* jmp rel short */ + JMP_REL(c->src.val); + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xf4: /* hlt */ + ctxt->vcpu->arch.halt_request = 1; + goto done; + case 0xf5: /* cmc */ + /* complement carry flag from eflags reg */ + ctxt->eflags ^= EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xf6 ... 0xf7: /* Grp3 */ + rc = emulate_grp3(ctxt, ops); + if (rc != 0) + goto done; + break; + case 0xf8: /* clc */ + ctxt->eflags &= ~EFLG_CF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfa: /* cli */ + ctxt->eflags &= ~X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfb: /* sti */ + ctxt->eflags |= X86_EFLAGS_IF; + c->dst.type = OP_NONE; /* Disable writeback. */ + break; + case 0xfe ... 0xff: /* Grp4/Grp5 */ + rc = emulate_grp45(ctxt, ops); + if (rc != 0) + goto done; + break; + } + +writeback: + rc = writeback(ctxt, ops); + if (rc != 0) + goto done; + + /* Commit shadow register state. */ + memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs); + ctxt->vcpu->arch.rip = c->eip; + +done: + if (rc == X86EMUL_UNHANDLEABLE) { + c->eip = saved_eip; + return -1; + } + return 0; + +twobyte_insn: + switch (c->b) { + case 0x01: /* lgdt, lidt, lmsw */ + switch (c->modrm_reg) { + u16 size; + unsigned long address; + + case 0: /* vmcall */ + if (c->modrm_mod != 3 || c->modrm_rm != 1) + goto cannot_emulate; + + rc = kvm_fix_hypercall(ctxt->vcpu); + if (rc) + goto done; + + kvm_emulate_hypercall(ctxt->vcpu); + break; + case 2: /* lgdt */ + rc = read_descriptor(ctxt, ops, c->src.ptr, + &size, &address, c->op_bytes); + if (rc) + goto done; + realmode_lgdt(ctxt->vcpu, size, address); + break; + case 3: /* lidt/vmmcall */ + if (c->modrm_mod == 3 && c->modrm_rm == 1) { + rc = kvm_fix_hypercall(ctxt->vcpu); + if (rc) + goto done; + kvm_emulate_hypercall(ctxt->vcpu); + } else { + rc = read_descriptor(ctxt, ops, c->src.ptr, + &size, &address, + c->op_bytes); + if (rc) + goto done; + realmode_lidt(ctxt->vcpu, size, address); + } + break; + case 4: /* smsw */ + if (c->modrm_mod != 3) + goto cannot_emulate; + *(u16 *)&c->regs[c->modrm_rm] + = realmode_get_cr(ctxt->vcpu, 0); + break; + case 6: /* lmsw */ + if (c->modrm_mod != 3) + goto cannot_emulate; + realmode_lmsw(ctxt->vcpu, (u16)c->modrm_val, + &ctxt->eflags); + break; + case 7: /* invlpg*/ + emulate_invlpg(ctxt->vcpu, memop); + break; + default: + goto cannot_emulate; + } + /* Disable writeback. */ + c->dst.type = OP_NONE; + break; + case 0x06: + emulate_clts(ctxt->vcpu); + c->dst.type = OP_NONE; + break; + case 0x08: /* invd */ + case 0x09: /* wbinvd */ + case 0x0d: /* GrpP (prefetch) */ + case 0x18: /* Grp16 (prefetch/nop) */ + c->dst.type = OP_NONE; + break; + case 0x20: /* mov cr, reg */ + if (c->modrm_mod != 3) + goto cannot_emulate; + c->regs[c->modrm_rm] = + realmode_get_cr(ctxt->vcpu, c->modrm_reg); + c->dst.type = OP_NONE; /* no writeback */ + break; + case 0x21: /* mov from dr to reg */ + if (c->modrm_mod != 3) + goto cannot_emulate; + rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]); + if (rc) + goto cannot_emulate; + c->dst.type = OP_NONE; /* no writeback */ + break; + case 0x22: /* mov reg, cr */ + if (c->modrm_mod != 3) + goto cannot_emulate; + realmode_set_cr(ctxt->vcpu, + c->modrm_reg, c->modrm_val, &ctxt->eflags); + c->dst.type = OP_NONE; + break; + case 0x23: /* mov from reg to dr */ + if (c->modrm_mod != 3) + goto cannot_emulate; + rc = emulator_set_dr(ctxt, c->modrm_reg, + c->regs[c->modrm_rm]); + if (rc) + goto cannot_emulate; + c->dst.type = OP_NONE; /* no writeback */ + break; + case 0x30: + /* wrmsr */ + msr_data = (u32)c->regs[VCPU_REGS_RAX] + | ((u64)c->regs[VCPU_REGS_RDX] << 32); + rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); + if (rc) { + kvm_inject_gp(ctxt->vcpu, 0); + c->eip = ctxt->vcpu->arch.rip; + } + rc = X86EMUL_CONTINUE; + c->dst.type = OP_NONE; + break; + case 0x32: + /* rdmsr */ + rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); + if (rc) { + kvm_inject_gp(ctxt->vcpu, 0); + c->eip = ctxt->vcpu->arch.rip; + } else { + c->regs[VCPU_REGS_RAX] = (u32)msr_data; + c->regs[VCPU_REGS_RDX] = msr_data >> 32; + } + rc = X86EMUL_CONTINUE; + c->dst.type = OP_NONE; + break; + case 0x40 ... 0x4f: /* cmov */ + c->dst.val = c->dst.orig_val = c->src.val; + if (!test_cc(c->b, ctxt->eflags)) + c->dst.type = OP_NONE; /* no writeback */ + break; + case 0x80 ... 0x8f: /* jnz rel, etc*/ { + long int rel; + + switch (c->op_bytes) { + case 2: + rel = insn_fetch(s16, 2, c->eip); + break; + case 4: + rel = insn_fetch(s32, 4, c->eip); + break; + case 8: + rel = insn_fetch(s64, 8, c->eip); + break; + default: + DPRINTF("jnz: Invalid op_bytes\n"); + goto cannot_emulate; + } + if (test_cc(c->b, ctxt->eflags)) + JMP_REL(rel); + c->dst.type = OP_NONE; + break; + } + case 0xa3: + bt: /* bt */ + c->dst.type = OP_NONE; + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags); + break; + case 0xab: + bts: /* bts */ + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags); + break; + case 0xb0 ... 0xb1: /* cmpxchg */ + /* + * Save real source value, then compare EAX against + * destination. + */ + c->src.orig_val = c->src.val; + c->src.val = c->regs[VCPU_REGS_RAX]; + emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); + if (ctxt->eflags & EFLG_ZF) { + /* Success: write back to memory. */ + c->dst.val = c->src.orig_val; + } else { + /* Failure: write the value we saw to EAX. */ + c->dst.type = OP_REG; + c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; + } + break; + case 0xb3: + btr: /* btr */ + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags); + break; + case 0xb6 ... 0xb7: /* movzx */ + c->dst.bytes = c->op_bytes; + c->dst.val = (c->d & ByteOp) ? (u8) c->src.val + : (u16) c->src.val; + break; + case 0xba: /* Grp8 */ + switch (c->modrm_reg & 3) { + case 0: + goto bt; + case 1: + goto bts; + case 2: + goto btr; + case 3: + goto btc; + } + break; + case 0xbb: + btc: /* btc */ + /* only subword offset */ + c->src.val &= (c->dst.bytes << 3) - 1; + emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags); + break; + case 0xbe ... 0xbf: /* movsx */ + c->dst.bytes = c->op_bytes; + c->dst.val = (c->d & ByteOp) ? (s8) c->src.val : + (s16) c->src.val; + break; + case 0xc3: /* movnti */ + c->dst.bytes = c->op_bytes; + c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val : + (u64) c->src.val; + break; + case 0xc7: /* Grp9 (cmpxchg8b) */ + rc = emulate_grp9(ctxt, ops, memop); + if (rc != 0) + goto done; + c->dst.type = OP_NONE; + break; + } + goto writeback; + +cannot_emulate: + DPRINTF("Cannot emulate %02x\n", c->b); + c->eip = saved_eip; + return -1; +} diff --git a/drivers/Kconfig b/drivers/Kconfig index f4076d9e9902..08d4ae201597 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -90,8 +90,6 @@ source "drivers/dca/Kconfig" source "drivers/auxdisplay/Kconfig" -source "drivers/kvm/Kconfig" - source "drivers/uio/Kconfig" source "drivers/virtio/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index d92d4d82d001..9e1f808e43cf 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -47,7 +47,6 @@ obj-$(CONFIG_SPI) += spi/ obj-$(CONFIG_PCCARD) += pcmcia/ obj-$(CONFIG_DIO) += dio/ obj-$(CONFIG_SBUS) += sbus/ -obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_ZORRO) += zorro/ obj-$(CONFIG_MAC) += macintosh/ obj-$(CONFIG_ATA_OVER_ETH) += block/aoe/ diff --git a/drivers/kvm/Kconfig b/drivers/kvm/Kconfig deleted file mode 100644 index c83e1c9b5129..000000000000 --- a/drivers/kvm/Kconfig +++ /dev/null @@ -1,57 +0,0 @@ -# -# KVM configuration -# -config HAVE_KVM - bool - -menuconfig VIRTUALIZATION - bool "Virtualization" - depends on HAVE_KVM || X86 - default y - ---help--- - Say Y here to get to see options for using your Linux host to run other - operating systems inside virtual machines (guests). - This option alone does not add any kernel code. - - If you say N, all options in this submenu will be skipped and disabled. - -if VIRTUALIZATION - -config KVM - tristate "Kernel-based Virtual Machine (KVM) support" - depends on HAVE_KVM && EXPERIMENTAL - select PREEMPT_NOTIFIERS - select ANON_INODES - ---help--- - Support hosting fully virtualized guest machines using hardware - virtualization extensions. You will need a fairly recent - processor equipped with virtualization extensions. You will also - need to select one or more of the processor modules below. - - This module provides access to the hardware capabilities through - a character device node named /dev/kvm. - - To compile this as a module, choose M here: the module - will be called kvm. - - If unsure, say N. - -config KVM_INTEL - tristate "KVM for Intel processors support" - depends on KVM - ---help--- - Provides support for KVM on Intel processors equipped with the VT - extensions. - -config KVM_AMD - tristate "KVM for AMD processors support" - depends on KVM - ---help--- - Provides support for KVM on AMD processors equipped with the AMD-V - (SVM) extensions. - -# OK, it's a little counter-intuitive to do this, but it puts it neatly under -# the virtualization menu. -source drivers/lguest/Kconfig - -endif # VIRTUALIZATION diff --git a/drivers/kvm/Makefile b/drivers/kvm/Makefile deleted file mode 100644 index cf18ad46e987..000000000000 --- a/drivers/kvm/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# -# Makefile for Kernel-based Virtual Machine module -# - -kvm-objs := kvm_main.o x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o ioapic.o -obj-$(CONFIG_KVM) += kvm.o -kvm-intel-objs = vmx.o -obj-$(CONFIG_KVM_INTEL) += kvm-intel.o -kvm-amd-objs = svm.o -obj-$(CONFIG_KVM_AMD) += kvm-amd.o diff --git a/drivers/kvm/i8259.c b/drivers/kvm/i8259.c deleted file mode 100644 index b3cad632f3d5..000000000000 --- a/drivers/kvm/i8259.c +++ /dev/null @@ -1,449 +0,0 @@ -/* - * 8259 interrupt controller emulation - * - * Copyright (c) 2003-2004 Fabrice Bellard - * Copyright (c) 2007 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * Authors: - * Yaozu (Eddie) Dong - * Port from Qemu. - */ -#include -#include "irq.h" -#include "kvm.h" - -/* - * set irq level. If an edge is detected, then the IRR is set to 1 - */ -static inline void pic_set_irq1(struct kvm_kpic_state *s, int irq, int level) -{ - int mask; - mask = 1 << irq; - if (s->elcr & mask) /* level triggered */ - if (level) { - s->irr |= mask; - s->last_irr |= mask; - } else { - s->irr &= ~mask; - s->last_irr &= ~mask; - } - else /* edge triggered */ - if (level) { - if ((s->last_irr & mask) == 0) - s->irr |= mask; - s->last_irr |= mask; - } else - s->last_irr &= ~mask; -} - -/* - * return the highest priority found in mask (highest = smallest - * number). Return 8 if no irq - */ -static inline int get_priority(struct kvm_kpic_state *s, int mask) -{ - int priority; - if (mask == 0) - return 8; - priority = 0; - while ((mask & (1 << ((priority + s->priority_add) & 7))) == 0) - priority++; - return priority; -} - -/* - * return the pic wanted interrupt. return -1 if none - */ -static int pic_get_irq(struct kvm_kpic_state *s) -{ - int mask, cur_priority, priority; - - mask = s->irr & ~s->imr; - priority = get_priority(s, mask); - if (priority == 8) - return -1; - /* - * compute current priority. If special fully nested mode on the - * master, the IRQ coming from the slave is not taken into account - * for the priority computation. - */ - mask = s->isr; - if (s->special_fully_nested_mode && s == &s->pics_state->pics[0]) - mask &= ~(1 << 2); - cur_priority = get_priority(s, mask); - if (priority < cur_priority) - /* - * higher priority found: an irq should be generated - */ - return (priority + s->priority_add) & 7; - else - return -1; -} - -/* - * raise irq to CPU if necessary. must be called every time the active - * irq may change - */ -static void pic_update_irq(struct kvm_pic *s) -{ - int irq2, irq; - - irq2 = pic_get_irq(&s->pics[1]); - if (irq2 >= 0) { - /* - * if irq request by slave pic, signal master PIC - */ - pic_set_irq1(&s->pics[0], 2, 1); - pic_set_irq1(&s->pics[0], 2, 0); - } - irq = pic_get_irq(&s->pics[0]); - if (irq >= 0) - s->irq_request(s->irq_request_opaque, 1); - else - s->irq_request(s->irq_request_opaque, 0); -} - -void kvm_pic_update_irq(struct kvm_pic *s) -{ - pic_update_irq(s); -} - -void kvm_pic_set_irq(void *opaque, int irq, int level) -{ - struct kvm_pic *s = opaque; - - pic_set_irq1(&s->pics[irq >> 3], irq & 7, level); - pic_update_irq(s); -} - -/* - * acknowledge interrupt 'irq' - */ -static inline void pic_intack(struct kvm_kpic_state *s, int irq) -{ - if (s->auto_eoi) { - if (s->rotate_on_auto_eoi) - s->priority_add = (irq + 1) & 7; - } else - s->isr |= (1 << irq); - /* - * We don't clear a level sensitive interrupt here - */ - if (!(s->elcr & (1 << irq))) - s->irr &= ~(1 << irq); -} - -int kvm_pic_read_irq(struct kvm_pic *s) -{ - int irq, irq2, intno; - - irq = pic_get_irq(&s->pics[0]); - if (irq >= 0) { - pic_intack(&s->pics[0], irq); - if (irq == 2) { - irq2 = pic_get_irq(&s->pics[1]); - if (irq2 >= 0) - pic_intack(&s->pics[1], irq2); - else - /* - * spurious IRQ on slave controller - */ - irq2 = 7; - intno = s->pics[1].irq_base + irq2; - irq = irq2 + 8; - } else - intno = s->pics[0].irq_base + irq; - } else { - /* - * spurious IRQ on host controller - */ - irq = 7; - intno = s->pics[0].irq_base + irq; - } - pic_update_irq(s); - - return intno; -} - -void kvm_pic_reset(struct kvm_kpic_state *s) -{ - s->last_irr = 0; - s->irr = 0; - s->imr = 0; - s->isr = 0; - s->priority_add = 0; - s->irq_base = 0; - s->read_reg_select = 0; - s->poll = 0; - s->special_mask = 0; - s->init_state = 0; - s->auto_eoi = 0; - s->rotate_on_auto_eoi = 0; - s->special_fully_nested_mode = 0; - s->init4 = 0; -} - -static void pic_ioport_write(void *opaque, u32 addr, u32 val) -{ - struct kvm_kpic_state *s = opaque; - int priority, cmd, irq; - - addr &= 1; - if (addr == 0) { - if (val & 0x10) { - kvm_pic_reset(s); /* init */ - /* - * deassert a pending interrupt - */ - s->pics_state->irq_request(s->pics_state-> - irq_request_opaque, 0); - s->init_state = 1; - s->init4 = val & 1; - if (val & 0x02) - printk(KERN_ERR "single mode not supported"); - if (val & 0x08) - printk(KERN_ERR - "level sensitive irq not supported"); - } else if (val & 0x08) { - if (val & 0x04) - s->poll = 1; - if (val & 0x02) - s->read_reg_select = val & 1; - if (val & 0x40) - s->special_mask = (val >> 5) & 1; - } else { - cmd = val >> 5; - switch (cmd) { - case 0: - case 4: - s->rotate_on_auto_eoi = cmd >> 2; - break; - case 1: /* end of interrupt */ - case 5: - priority = get_priority(s, s->isr); - if (priority != 8) { - irq = (priority + s->priority_add) & 7; - s->isr &= ~(1 << irq); - if (cmd == 5) - s->priority_add = (irq + 1) & 7; - pic_update_irq(s->pics_state); - } - break; - case 3: - irq = val & 7; - s->isr &= ~(1 << irq); - pic_update_irq(s->pics_state); - break; - case 6: - s->priority_add = (val + 1) & 7; - pic_update_irq(s->pics_state); - break; - case 7: - irq = val & 7; - s->isr &= ~(1 << irq); - s->priority_add = (irq + 1) & 7; - pic_update_irq(s->pics_state); - break; - default: - break; /* no operation */ - } - } - } else - switch (s->init_state) { - case 0: /* normal mode */ - s->imr = val; - pic_update_irq(s->pics_state); - break; - case 1: - s->irq_base = val & 0xf8; - s->init_state = 2; - break; - case 2: - if (s->init4) - s->init_state = 3; - else - s->init_state = 0; - break; - case 3: - s->special_fully_nested_mode = (val >> 4) & 1; - s->auto_eoi = (val >> 1) & 1; - s->init_state = 0; - break; - } -} - -static u32 pic_poll_read(struct kvm_kpic_state *s, u32 addr1) -{ - int ret; - - ret = pic_get_irq(s); - if (ret >= 0) { - if (addr1 >> 7) { - s->pics_state->pics[0].isr &= ~(1 << 2); - s->pics_state->pics[0].irr &= ~(1 << 2); - } - s->irr &= ~(1 << ret); - s->isr &= ~(1 << ret); - if (addr1 >> 7 || ret != 2) - pic_update_irq(s->pics_state); - } else { - ret = 0x07; - pic_update_irq(s->pics_state); - } - - return ret; -} - -static u32 pic_ioport_read(void *opaque, u32 addr1) -{ - struct kvm_kpic_state *s = opaque; - unsigned int addr; - int ret; - - addr = addr1; - addr &= 1; - if (s->poll) { - ret = pic_poll_read(s, addr1); - s->poll = 0; - } else - if (addr == 0) - if (s->read_reg_select) - ret = s->isr; - else - ret = s->irr; - else - ret = s->imr; - return ret; -} - -static void elcr_ioport_write(void *opaque, u32 addr, u32 val) -{ - struct kvm_kpic_state *s = opaque; - s->elcr = val & s->elcr_mask; -} - -static u32 elcr_ioport_read(void *opaque, u32 addr1) -{ - struct kvm_kpic_state *s = opaque; - return s->elcr; -} - -static int picdev_in_range(struct kvm_io_device *this, gpa_t addr) -{ - switch (addr) { - case 0x20: - case 0x21: - case 0xa0: - case 0xa1: - case 0x4d0: - case 0x4d1: - return 1; - default: - return 0; - } -} - -static void picdev_write(struct kvm_io_device *this, - gpa_t addr, int len, const void *val) -{ - struct kvm_pic *s = this->private; - unsigned char data = *(unsigned char *)val; - - if (len != 1) { - if (printk_ratelimit()) - printk(KERN_ERR "PIC: non byte write\n"); - return; - } - switch (addr) { - case 0x20: - case 0x21: - case 0xa0: - case 0xa1: - pic_ioport_write(&s->pics[addr >> 7], addr, data); - break; - case 0x4d0: - case 0x4d1: - elcr_ioport_write(&s->pics[addr & 1], addr, data); - break; - } -} - -static void picdev_read(struct kvm_io_device *this, - gpa_t addr, int len, void *val) -{ - struct kvm_pic *s = this->private; - unsigned char data = 0; - - if (len != 1) { - if (printk_ratelimit()) - printk(KERN_ERR "PIC: non byte read\n"); - return; - } - switch (addr) { - case 0x20: - case 0x21: - case 0xa0: - case 0xa1: - data = pic_ioport_read(&s->pics[addr >> 7], addr); - break; - case 0x4d0: - case 0x4d1: - data = elcr_ioport_read(&s->pics[addr & 1], addr); - break; - } - *(unsigned char *)val = data; -} - -/* - * callback when PIC0 irq status changed - */ -static void pic_irq_request(void *opaque, int level) -{ - struct kvm *kvm = opaque; - struct kvm_vcpu *vcpu = kvm->vcpus[0]; - - pic_irqchip(kvm)->output = level; - if (vcpu) - kvm_vcpu_kick(vcpu); -} - -struct kvm_pic *kvm_create_pic(struct kvm *kvm) -{ - struct kvm_pic *s; - s = kzalloc(sizeof(struct kvm_pic), GFP_KERNEL); - if (!s) - return NULL; - s->pics[0].elcr_mask = 0xf8; - s->pics[1].elcr_mask = 0xde; - s->irq_request = pic_irq_request; - s->irq_request_opaque = kvm; - s->pics[0].pics_state = s; - s->pics[1].pics_state = s; - - /* - * Initialize PIO device - */ - s->dev.read = picdev_read; - s->dev.write = picdev_write; - s->dev.in_range = picdev_in_range; - s->dev.private = s; - kvm_io_bus_register_dev(&kvm->pio_bus, &s->dev); - return s; -} diff --git a/drivers/kvm/ioapic.c b/drivers/kvm/ioapic.c deleted file mode 100644 index f8236774c1b4..000000000000 --- a/drivers/kvm/ioapic.c +++ /dev/null @@ -1,402 +0,0 @@ -/* - * Copyright (C) 2001 MandrakeSoft S.A. - * - * MandrakeSoft S.A. - * 43, rue d'Aboukir - * 75002 Paris - France - * http://www.linux-mandrake.com/ - * http://www.mandrakesoft.com/ - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * - * Yunhong Jiang - * Yaozu (Eddie) Dong - * Based on Xen 3.1 code. - */ - -#include "kvm.h" -#include "x86.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "irq.h" -#if 0 -#define ioapic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) -#else -#define ioapic_debug(fmt, arg...) -#endif -static void ioapic_deliver(struct kvm_ioapic *vioapic, int irq); - -static unsigned long ioapic_read_indirect(struct kvm_ioapic *ioapic, - unsigned long addr, - unsigned long length) -{ - unsigned long result = 0; - - switch (ioapic->ioregsel) { - case IOAPIC_REG_VERSION: - result = ((((IOAPIC_NUM_PINS - 1) & 0xff) << 16) - | (IOAPIC_VERSION_ID & 0xff)); - break; - - case IOAPIC_REG_APIC_ID: - case IOAPIC_REG_ARB_ID: - result = ((ioapic->id & 0xf) << 24); - break; - - default: - { - u32 redir_index = (ioapic->ioregsel - 0x10) >> 1; - u64 redir_content; - - ASSERT(redir_index < IOAPIC_NUM_PINS); - - redir_content = ioapic->redirtbl[redir_index].bits; - result = (ioapic->ioregsel & 0x1) ? - (redir_content >> 32) & 0xffffffff : - redir_content & 0xffffffff; - break; - } - } - - return result; -} - -static void ioapic_service(struct kvm_ioapic *ioapic, unsigned int idx) -{ - union ioapic_redir_entry *pent; - - pent = &ioapic->redirtbl[idx]; - - if (!pent->fields.mask) { - ioapic_deliver(ioapic, idx); - if (pent->fields.trig_mode == IOAPIC_LEVEL_TRIG) - pent->fields.remote_irr = 1; - } - if (!pent->fields.trig_mode) - ioapic->irr &= ~(1 << idx); -} - -static void ioapic_write_indirect(struct kvm_ioapic *ioapic, u32 val) -{ - unsigned index; - - switch (ioapic->ioregsel) { - case IOAPIC_REG_VERSION: - /* Writes are ignored. */ - break; - - case IOAPIC_REG_APIC_ID: - ioapic->id = (val >> 24) & 0xf; - break; - - case IOAPIC_REG_ARB_ID: - break; - - default: - index = (ioapic->ioregsel - 0x10) >> 1; - - ioapic_debug("change redir index %x val %x\n", index, val); - if (index >= IOAPIC_NUM_PINS) - return; - if (ioapic->ioregsel & 1) { - ioapic->redirtbl[index].bits &= 0xffffffff; - ioapic->redirtbl[index].bits |= (u64) val << 32; - } else { - ioapic->redirtbl[index].bits &= ~0xffffffffULL; - ioapic->redirtbl[index].bits |= (u32) val; - ioapic->redirtbl[index].fields.remote_irr = 0; - } - if (ioapic->irr & (1 << index)) - ioapic_service(ioapic, index); - break; - } -} - -static void ioapic_inj_irq(struct kvm_ioapic *ioapic, - struct kvm_vcpu *vcpu, - u8 vector, u8 trig_mode, u8 delivery_mode) -{ - ioapic_debug("irq %d trig %d deliv %d\n", vector, trig_mode, - delivery_mode); - - ASSERT((delivery_mode == IOAPIC_FIXED) || - (delivery_mode == IOAPIC_LOWEST_PRIORITY)); - - kvm_apic_set_irq(vcpu, vector, trig_mode); -} - -static u32 ioapic_get_delivery_bitmask(struct kvm_ioapic *ioapic, u8 dest, - u8 dest_mode) -{ - u32 mask = 0; - int i; - struct kvm *kvm = ioapic->kvm; - struct kvm_vcpu *vcpu; - - ioapic_debug("dest %d dest_mode %d\n", dest, dest_mode); - - if (dest_mode == 0) { /* Physical mode. */ - if (dest == 0xFF) { /* Broadcast. */ - for (i = 0; i < KVM_MAX_VCPUS; ++i) - if (kvm->vcpus[i] && kvm->vcpus[i]->arch.apic) - mask |= 1 << i; - return mask; - } - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - vcpu = kvm->vcpus[i]; - if (!vcpu) - continue; - if (kvm_apic_match_physical_addr(vcpu->arch.apic, dest)) { - if (vcpu->arch.apic) - mask = 1 << i; - break; - } - } - } else if (dest != 0) /* Logical mode, MDA non-zero. */ - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - vcpu = kvm->vcpus[i]; - if (!vcpu) - continue; - if (vcpu->arch.apic && - kvm_apic_match_logical_addr(vcpu->arch.apic, dest)) - mask |= 1 << vcpu->vcpu_id; - } - ioapic_debug("mask %x\n", mask); - return mask; -} - -static void ioapic_deliver(struct kvm_ioapic *ioapic, int irq) -{ - u8 dest = ioapic->redirtbl[irq].fields.dest_id; - u8 dest_mode = ioapic->redirtbl[irq].fields.dest_mode; - u8 delivery_mode = ioapic->redirtbl[irq].fields.delivery_mode; - u8 vector = ioapic->redirtbl[irq].fields.vector; - u8 trig_mode = ioapic->redirtbl[irq].fields.trig_mode; - u32 deliver_bitmask; - struct kvm_vcpu *vcpu; - int vcpu_id; - - ioapic_debug("dest=%x dest_mode=%x delivery_mode=%x " - "vector=%x trig_mode=%x\n", - dest, dest_mode, delivery_mode, vector, trig_mode); - - deliver_bitmask = ioapic_get_delivery_bitmask(ioapic, dest, dest_mode); - if (!deliver_bitmask) { - ioapic_debug("no target on destination\n"); - return; - } - - switch (delivery_mode) { - case IOAPIC_LOWEST_PRIORITY: - vcpu = kvm_get_lowest_prio_vcpu(ioapic->kvm, vector, - deliver_bitmask); - if (vcpu != NULL) - ioapic_inj_irq(ioapic, vcpu, vector, - trig_mode, delivery_mode); - else - ioapic_debug("null lowest prio vcpu: " - "mask=%x vector=%x delivery_mode=%x\n", - deliver_bitmask, vector, IOAPIC_LOWEST_PRIORITY); - break; - case IOAPIC_FIXED: - for (vcpu_id = 0; deliver_bitmask != 0; vcpu_id++) { - if (!(deliver_bitmask & (1 << vcpu_id))) - continue; - deliver_bitmask &= ~(1 << vcpu_id); - vcpu = ioapic->kvm->vcpus[vcpu_id]; - if (vcpu) { - ioapic_inj_irq(ioapic, vcpu, vector, - trig_mode, delivery_mode); - } - } - break; - - /* TODO: NMI */ - default: - printk(KERN_WARNING "Unsupported delivery mode %d\n", - delivery_mode); - break; - } -} - -void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level) -{ - u32 old_irr = ioapic->irr; - u32 mask = 1 << irq; - union ioapic_redir_entry entry; - - if (irq >= 0 && irq < IOAPIC_NUM_PINS) { - entry = ioapic->redirtbl[irq]; - level ^= entry.fields.polarity; - if (!level) - ioapic->irr &= ~mask; - else { - ioapic->irr |= mask; - if ((!entry.fields.trig_mode && old_irr != ioapic->irr) - || !entry.fields.remote_irr) - ioapic_service(ioapic, irq); - } - } -} - -static int get_eoi_gsi(struct kvm_ioapic *ioapic, int vector) -{ - int i; - - for (i = 0; i < IOAPIC_NUM_PINS; i++) - if (ioapic->redirtbl[i].fields.vector == vector) - return i; - return -1; -} - -void kvm_ioapic_update_eoi(struct kvm *kvm, int vector) -{ - struct kvm_ioapic *ioapic = kvm->arch.vioapic; - union ioapic_redir_entry *ent; - int gsi; - - gsi = get_eoi_gsi(ioapic, vector); - if (gsi == -1) { - printk(KERN_WARNING "Can't find redir item for %d EOI\n", - vector); - return; - } - - ent = &ioapic->redirtbl[gsi]; - ASSERT(ent->fields.trig_mode == IOAPIC_LEVEL_TRIG); - - ent->fields.remote_irr = 0; - if (!ent->fields.mask && (ioapic->irr & (1 << gsi))) - ioapic_deliver(ioapic, gsi); -} - -static int ioapic_in_range(struct kvm_io_device *this, gpa_t addr) -{ - struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; - - return ((addr >= ioapic->base_address && - (addr < ioapic->base_address + IOAPIC_MEM_LENGTH))); -} - -static void ioapic_mmio_read(struct kvm_io_device *this, gpa_t addr, int len, - void *val) -{ - struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; - u32 result; - - ioapic_debug("addr %lx\n", (unsigned long)addr); - ASSERT(!(addr & 0xf)); /* check alignment */ - - addr &= 0xff; - switch (addr) { - case IOAPIC_REG_SELECT: - result = ioapic->ioregsel; - break; - - case IOAPIC_REG_WINDOW: - result = ioapic_read_indirect(ioapic, addr, len); - break; - - default: - result = 0; - break; - } - switch (len) { - case 8: - *(u64 *) val = result; - break; - case 1: - case 2: - case 4: - memcpy(val, (char *)&result, len); - break; - default: - printk(KERN_WARNING "ioapic: wrong length %d\n", len); - } -} - -static void ioapic_mmio_write(struct kvm_io_device *this, gpa_t addr, int len, - const void *val) -{ - struct kvm_ioapic *ioapic = (struct kvm_ioapic *)this->private; - u32 data; - - ioapic_debug("ioapic_mmio_write addr=%p len=%d val=%p\n", - (void*)addr, len, val); - ASSERT(!(addr & 0xf)); /* check alignment */ - if (len == 4 || len == 8) - data = *(u32 *) val; - else { - printk(KERN_WARNING "ioapic: Unsupported size %d\n", len); - return; - } - - addr &= 0xff; - switch (addr) { - case IOAPIC_REG_SELECT: - ioapic->ioregsel = data; - break; - - case IOAPIC_REG_WINDOW: - ioapic_write_indirect(ioapic, data); - break; -#ifdef CONFIG_IA64 - case IOAPIC_REG_EOI: - kvm_ioapic_update_eoi(ioapic, data); - break; -#endif - - default: - break; - } -} - -void kvm_ioapic_reset(struct kvm_ioapic *ioapic) -{ - int i; - - for (i = 0; i < IOAPIC_NUM_PINS; i++) - ioapic->redirtbl[i].fields.mask = 1; - ioapic->base_address = IOAPIC_DEFAULT_BASE_ADDRESS; - ioapic->ioregsel = 0; - ioapic->irr = 0; - ioapic->id = 0; -} - -int kvm_ioapic_init(struct kvm *kvm) -{ - struct kvm_ioapic *ioapic; - - ioapic = kzalloc(sizeof(struct kvm_ioapic), GFP_KERNEL); - if (!ioapic) - return -ENOMEM; - kvm->arch.vioapic = ioapic; - kvm_ioapic_reset(ioapic); - ioapic->dev.read = ioapic_mmio_read; - ioapic->dev.write = ioapic_mmio_write; - ioapic->dev.in_range = ioapic_in_range; - ioapic->dev.private = ioapic; - ioapic->kvm = kvm; - kvm_io_bus_register_dev(&kvm->mmio_bus, &ioapic->dev); - return 0; -} diff --git a/drivers/kvm/iodev.h b/drivers/kvm/iodev.h index eb9e8a71843a..c14e642027b2 100644 --- a/drivers/kvm/iodev.h +++ b/drivers/kvm/iodev.h @@ -16,7 +16,7 @@ #ifndef __KVM_IODEV_H__ #define __KVM_IODEV_H__ -#include "types.h" +#include struct kvm_io_device { void (*read)(struct kvm_io_device *this, diff --git a/drivers/kvm/irq.c b/drivers/kvm/irq.c deleted file mode 100644 index 59b47c55fc76..000000000000 --- a/drivers/kvm/irq.c +++ /dev/null @@ -1,99 +0,0 @@ -/* - * irq.c: API for in kernel interrupt controller - * Copyright (c) 2007, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 Temple - * Place - Suite 330, Boston, MA 02111-1307 USA. - * Authors: - * Yaozu (Eddie) Dong - * - */ - -#include - -#include "kvm.h" -#include "x86.h" -#include "irq.h" - -/* - * check if there is pending interrupt without - * intack. - */ -int kvm_cpu_has_interrupt(struct kvm_vcpu *v) -{ - struct kvm_pic *s; - - if (kvm_apic_has_interrupt(v) == -1) { /* LAPIC */ - if (kvm_apic_accept_pic_intr(v)) { - s = pic_irqchip(v->kvm); /* PIC */ - return s->output; - } else - return 0; - } - return 1; -} -EXPORT_SYMBOL_GPL(kvm_cpu_has_interrupt); - -/* - * Read pending interrupt vector and intack. - */ -int kvm_cpu_get_interrupt(struct kvm_vcpu *v) -{ - struct kvm_pic *s; - int vector; - - vector = kvm_get_apic_interrupt(v); /* APIC */ - if (vector == -1) { - if (kvm_apic_accept_pic_intr(v)) { - s = pic_irqchip(v->kvm); - s->output = 0; /* PIC */ - vector = kvm_pic_read_irq(s); - } - } - return vector; -} -EXPORT_SYMBOL_GPL(kvm_cpu_get_interrupt); - -static void vcpu_kick_intr(void *info) -{ -#ifdef DEBUG - struct kvm_vcpu *vcpu = (struct kvm_vcpu *)info; - printk(KERN_DEBUG "vcpu_kick_intr %p \n", vcpu); -#endif -} - -void kvm_vcpu_kick(struct kvm_vcpu *vcpu) -{ - int ipi_pcpu = vcpu->cpu; - - if (waitqueue_active(&vcpu->wq)) { - wake_up_interruptible(&vcpu->wq); - ++vcpu->stat.halt_wakeup; - } - if (vcpu->guest_mode) - smp_call_function_single(ipi_pcpu, vcpu_kick_intr, vcpu, 0, 0); -} - -void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu) -{ - kvm_inject_apic_timer_irqs(vcpu); - /* TODO: PIT, RTC etc. */ -} -EXPORT_SYMBOL_GPL(kvm_inject_pending_timer_irqs); - -void kvm_timer_intr_post(struct kvm_vcpu *vcpu, int vec) -{ - kvm_apic_timer_intr_post(vcpu, vec); - /* TODO: PIT, RTC etc. */ -} -EXPORT_SYMBOL_GPL(kvm_timer_intr_post); diff --git a/drivers/kvm/irq.h b/drivers/kvm/irq.h deleted file mode 100644 index 6e023dc3f848..000000000000 --- a/drivers/kvm/irq.h +++ /dev/null @@ -1,196 +0,0 @@ -/* - * irq.h: in kernel interrupt controller related definitions - * Copyright (c) 2007, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 Temple - * Place - Suite 330, Boston, MA 02111-1307 USA. - * Authors: - * Yaozu (Eddie) Dong - * - */ - -#ifndef __IRQ_H -#define __IRQ_H - -#include -#include -#include -#include "iodev.h" -#include "kvm.h" - -struct kvm; -struct kvm_vcpu; - -typedef void irq_request_func(void *opaque, int level); - -struct kvm_kpic_state { - u8 last_irr; /* edge detection */ - u8 irr; /* interrupt request register */ - u8 imr; /* interrupt mask register */ - u8 isr; /* interrupt service register */ - u8 priority_add; /* highest irq priority */ - u8 irq_base; - u8 read_reg_select; - u8 poll; - u8 special_mask; - u8 init_state; - u8 auto_eoi; - u8 rotate_on_auto_eoi; - u8 special_fully_nested_mode; - u8 init4; /* true if 4 byte init */ - u8 elcr; /* PIIX edge/trigger selection */ - u8 elcr_mask; - struct kvm_pic *pics_state; -}; - -struct kvm_pic { - struct kvm_kpic_state pics[2]; /* 0 is master pic, 1 is slave pic */ - irq_request_func *irq_request; - void *irq_request_opaque; - int output; /* intr from master PIC */ - struct kvm_io_device dev; -}; - -struct kvm_pic *kvm_create_pic(struct kvm *kvm); -void kvm_pic_set_irq(void *opaque, int irq, int level); -int kvm_pic_read_irq(struct kvm_pic *s); -void kvm_pic_update_irq(struct kvm_pic *s); - -#define IOAPIC_NUM_PINS KVM_IOAPIC_NUM_PINS -#define IOAPIC_VERSION_ID 0x11 /* IOAPIC version */ -#define IOAPIC_EDGE_TRIG 0 -#define IOAPIC_LEVEL_TRIG 1 - -#define IOAPIC_DEFAULT_BASE_ADDRESS 0xfec00000 -#define IOAPIC_MEM_LENGTH 0x100 - -/* Direct registers. */ -#define IOAPIC_REG_SELECT 0x00 -#define IOAPIC_REG_WINDOW 0x10 -#define IOAPIC_REG_EOI 0x40 /* IA64 IOSAPIC only */ - -/* Indirect registers. */ -#define IOAPIC_REG_APIC_ID 0x00 /* x86 IOAPIC only */ -#define IOAPIC_REG_VERSION 0x01 -#define IOAPIC_REG_ARB_ID 0x02 /* x86 IOAPIC only */ - -/*ioapic delivery mode*/ -#define IOAPIC_FIXED 0x0 -#define IOAPIC_LOWEST_PRIORITY 0x1 -#define IOAPIC_PMI 0x2 -#define IOAPIC_NMI 0x4 -#define IOAPIC_INIT 0x5 -#define IOAPIC_EXTINT 0x7 - -struct kvm_ioapic { - u64 base_address; - u32 ioregsel; - u32 id; - u32 irr; - u32 pad; - union ioapic_redir_entry { - u64 bits; - struct { - u8 vector; - u8 delivery_mode:3; - u8 dest_mode:1; - u8 delivery_status:1; - u8 polarity:1; - u8 remote_irr:1; - u8 trig_mode:1; - u8 mask:1; - u8 reserve:7; - u8 reserved[4]; - u8 dest_id; - } fields; - } redirtbl[IOAPIC_NUM_PINS]; - struct kvm_io_device dev; - struct kvm *kvm; -}; - -struct kvm_lapic { - unsigned long base_address; - struct kvm_io_device dev; - struct { - atomic_t pending; - s64 period; /* unit: ns */ - u32 divide_count; - ktime_t last_update; - struct hrtimer dev; - } timer; - struct kvm_vcpu *vcpu; - struct page *regs_page; - void *regs; -}; - -#ifdef DEBUG -#define ASSERT(x) \ -do { \ - if (!(x)) { \ - printk(KERN_EMERG "assertion failed %s: %d: %s\n", \ - __FILE__, __LINE__, #x); \ - BUG(); \ - } \ -} while (0) -#else -#define ASSERT(x) do { } while (0) -#endif - -static inline struct kvm_pic *pic_irqchip(struct kvm *kvm) -{ - return kvm->arch.vpic; -} - -static inline struct kvm_ioapic *ioapic_irqchip(struct kvm *kvm) -{ - return kvm->arch.vioapic; -} - -static inline int irqchip_in_kernel(struct kvm *kvm) -{ - return pic_irqchip(kvm) != NULL; -} - -void kvm_vcpu_kick(struct kvm_vcpu *vcpu); -int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); -int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); -int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); -int kvm_create_lapic(struct kvm_vcpu *vcpu); -void kvm_lapic_reset(struct kvm_vcpu *vcpu); -void kvm_pic_reset(struct kvm_kpic_state *s); -void kvm_ioapic_reset(struct kvm_ioapic *ioapic); -void kvm_free_lapic(struct kvm_vcpu *vcpu); -u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); -void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); -void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value); - -struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, - unsigned long bitmap); -u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); -void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); -int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest); -void kvm_ioapic_update_eoi(struct kvm *kvm, int vector); -int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda); -int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig); -void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu); -int kvm_ioapic_init(struct kvm *kvm); -void kvm_ioapic_set_irq(struct kvm_ioapic *ioapic, int irq, int level); -int kvm_lapic_enabled(struct kvm_vcpu *vcpu); -int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); -void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec); -void kvm_timer_intr_post(struct kvm_vcpu *vcpu, int vec); -void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); -void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); -void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu); - -#endif diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h deleted file mode 100644 index bf6a3b330a3d..000000000000 --- a/drivers/kvm/kvm.h +++ /dev/null @@ -1,289 +0,0 @@ -#ifndef __KVM_H -#define __KVM_H - -/* - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "types.h" - -#include "x86.h" - -#define KVM_MAX_VCPUS 4 -#define KVM_MEMORY_SLOTS 8 -/* memory slots that does not exposed to userspace */ -#define KVM_PRIVATE_MEM_SLOTS 4 - -#define KVM_PIO_PAGE_OFFSET 1 - -/* - * vcpu->requests bit members - */ -#define KVM_REQ_TLB_FLUSH 0 - - -struct kvm_vcpu; -extern struct kmem_cache *kvm_vcpu_cache; - -struct kvm_guest_debug { - int enabled; - unsigned long bp[4]; - int singlestep; -}; - -/* - * It would be nice to use something smarter than a linear search, TBD... - * Thankfully we dont expect many devices to register (famous last words :), - * so until then it will suffice. At least its abstracted so we can change - * in one place. - */ -struct kvm_io_bus { - int dev_count; -#define NR_IOBUS_DEVS 6 - struct kvm_io_device *devs[NR_IOBUS_DEVS]; -}; - -void kvm_io_bus_init(struct kvm_io_bus *bus); -void kvm_io_bus_destroy(struct kvm_io_bus *bus); -struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr); -void kvm_io_bus_register_dev(struct kvm_io_bus *bus, - struct kvm_io_device *dev); - -struct kvm_vcpu { - struct kvm *kvm; - struct preempt_notifier preempt_notifier; - int vcpu_id; - struct mutex mutex; - int cpu; - struct kvm_run *run; - int guest_mode; - unsigned long requests; - struct kvm_guest_debug guest_debug; - int fpu_active; - int guest_fpu_loaded; - wait_queue_head_t wq; - int sigset_active; - sigset_t sigset; - struct kvm_vcpu_stat stat; - -#ifdef CONFIG_HAS_IOMEM - int mmio_needed; - int mmio_read_completed; - int mmio_is_write; - int mmio_size; - unsigned char mmio_data[8]; - gpa_t mmio_phys_addr; -#endif - - struct kvm_vcpu_arch arch; -}; - -struct kvm_memory_slot { - gfn_t base_gfn; - unsigned long npages; - unsigned long flags; - unsigned long *rmap; - unsigned long *dirty_bitmap; - unsigned long userspace_addr; - int user_alloc; -}; - -struct kvm { - struct mutex lock; /* protects everything except vcpus */ - struct mm_struct *mm; /* userspace tied to this vm */ - int nmemslots; - struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + - KVM_PRIVATE_MEM_SLOTS]; - struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; - struct list_head vm_list; - struct file *filp; - struct kvm_io_bus mmio_bus; - struct kvm_io_bus pio_bus; - struct kvm_vm_stat stat; - struct kvm_arch arch; -}; - -/* The guest did something we don't support. */ -#define pr_unimpl(vcpu, fmt, ...) \ - do { \ - if (printk_ratelimit()) \ - printk(KERN_ERR "kvm: %i: cpu%i " fmt, \ - current->tgid, (vcpu)->vcpu_id , ## __VA_ARGS__); \ - } while (0) - -#define kvm_printf(kvm, fmt ...) printk(KERN_DEBUG fmt) -#define vcpu_printf(vcpu, fmt...) kvm_printf(vcpu->kvm, fmt) - -int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id); -void kvm_vcpu_uninit(struct kvm_vcpu *vcpu); - -void vcpu_load(struct kvm_vcpu *vcpu); -void vcpu_put(struct kvm_vcpu *vcpu); - -void decache_vcpus_on_cpu(int cpu); - - -int kvm_init(void *opaque, unsigned int vcpu_size, - struct module *module); -void kvm_exit(void); - -#define HPA_MSB ((sizeof(hpa_t) * 8) - 1) -#define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) -static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } -struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); - -extern struct page *bad_page; - -int is_error_page(struct page *page); -int kvm_is_error_hva(unsigned long addr); -int kvm_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - int user_alloc); -int __kvm_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - int user_alloc); -int kvm_arch_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - struct kvm_memory_slot old, - int user_alloc); -gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); -struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); -void kvm_release_page_clean(struct page *page); -void kvm_release_page_dirty(struct page *page); -int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, - int len); -int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); -int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, - int offset, int len); -int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, - unsigned long len); -int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len); -int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); -struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); -int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); -void mark_page_dirty(struct kvm *kvm, gfn_t gfn); - -void kvm_vcpu_block(struct kvm_vcpu *vcpu); -void kvm_resched(struct kvm_vcpu *vcpu); -void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); -void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); -void kvm_flush_remote_tlbs(struct kvm *kvm); - -long kvm_arch_dev_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg); -long kvm_arch_vcpu_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg); -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); -void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); - -int kvm_dev_ioctl_check_extension(long ext); - -int kvm_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log, int *is_dirty); -int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log); - -int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct - kvm_userspace_memory_region *mem, - int user_alloc); -long kvm_arch_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg); -void kvm_arch_destroy_vm(struct kvm *kvm); - -int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); -int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); - -int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, - struct kvm_translation *tr); - -int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); -int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); -int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs); -int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs); -int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, - struct kvm_debug_guest *dbg); -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); - -int kvm_arch_init(void *opaque); -void kvm_arch_exit(void); - -int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); -void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); - -void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); -void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); -struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); -int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu); -void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); - -int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); -void kvm_arch_hardware_enable(void *garbage); -void kvm_arch_hardware_disable(void *garbage); -int kvm_arch_hardware_setup(void); -void kvm_arch_hardware_unsetup(void); -void kvm_arch_check_processor_compat(void *rtn); -int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); - -void kvm_free_physmem(struct kvm *kvm); - -struct kvm *kvm_arch_create_vm(void); -void kvm_arch_destroy_vm(struct kvm *kvm); - -int kvm_cpu_get_interrupt(struct kvm_vcpu *v); -int kvm_cpu_has_interrupt(struct kvm_vcpu *v); - -static inline void kvm_guest_enter(void) -{ - account_system_vtime(current); - current->flags |= PF_VCPU; -} - -static inline void kvm_guest_exit(void) -{ - account_system_vtime(current); - current->flags &= ~PF_VCPU; -} - -static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) -{ - return slot - kvm->memslots; -} - -static inline gpa_t gfn_to_gpa(gfn_t gfn) -{ - return (gpa_t)gfn << PAGE_SHIFT; -} - -enum kvm_stat_kind { - KVM_STAT_VM, - KVM_STAT_VCPU, -}; - -struct kvm_stats_debugfs_item { - const char *name; - int offset; - enum kvm_stat_kind kind; - struct dentry *dentry; -}; -extern struct kvm_stats_debugfs_item debugfs_entries[]; - -#endif diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c index ae2a1bf640bc..4026d7d64296 100644 --- a/drivers/kvm/kvm_main.c +++ b/drivers/kvm/kvm_main.c @@ -15,9 +15,9 @@ * */ -#include "kvm.h" #include "iodev.h" +#include #include #include #include diff --git a/drivers/kvm/kvm_svm.h b/drivers/kvm/kvm_svm.h deleted file mode 100644 index a0e415daef5b..000000000000 --- a/drivers/kvm/kvm_svm.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef __KVM_SVM_H -#define __KVM_SVM_H - -#include -#include -#include -#include - -#include "svm.h" -#include "kvm.h" - -static const u32 host_save_user_msrs[] = { -#ifdef CONFIG_X86_64 - MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE, - MSR_FS_BASE, -#endif - MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, -}; - -#define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs) -#define NUM_DB_REGS 4 - -struct kvm_vcpu; - -struct vcpu_svm { - struct kvm_vcpu vcpu; - struct vmcb *vmcb; - unsigned long vmcb_pa; - struct svm_cpu_data *svm_data; - uint64_t asid_generation; - - unsigned long db_regs[NUM_DB_REGS]; - - u64 next_rip; - - u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; - u64 host_gs_base; - unsigned long host_cr2; - unsigned long host_db_regs[NUM_DB_REGS]; - unsigned long host_dr6; - unsigned long host_dr7; -}; - -#endif - diff --git a/drivers/kvm/lapic.c b/drivers/kvm/lapic.c deleted file mode 100644 index 8c74bf184a07..000000000000 --- a/drivers/kvm/lapic.c +++ /dev/null @@ -1,1087 +0,0 @@ - -/* - * Local APIC virtualization - * - * Copyright (C) 2006 Qumranet, Inc. - * Copyright (C) 2007 Novell - * Copyright (C) 2007 Intel - * - * Authors: - * Dor Laor - * Gregory Haskins - * Yaozu (Eddie) Dong - * - * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation. - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - */ - -#include "kvm.h" -#include "x86.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "irq.h" - -#define PRId64 "d" -#define PRIx64 "llx" -#define PRIu64 "u" -#define PRIo64 "o" - -#define APIC_BUS_CYCLE_NS 1 - -/* #define apic_debug(fmt,arg...) printk(KERN_WARNING fmt,##arg) */ -#define apic_debug(fmt, arg...) - -#define APIC_LVT_NUM 6 -/* 14 is the version for Xeon and Pentium 8.4.8*/ -#define APIC_VERSION (0x14UL | ((APIC_LVT_NUM - 1) << 16)) -#define LAPIC_MMIO_LENGTH (1 << 12) -/* followed define is not in apicdef.h */ -#define APIC_SHORT_MASK 0xc0000 -#define APIC_DEST_NOSHORT 0x0 -#define APIC_DEST_MASK 0x800 -#define MAX_APIC_VECTOR 256 - -#define VEC_POS(v) ((v) & (32 - 1)) -#define REG_POS(v) (((v) >> 5) << 4) - -static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off) -{ - return *((u32 *) (apic->regs + reg_off)); -} - -static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) -{ - *((u32 *) (apic->regs + reg_off)) = val; -} - -static inline int apic_test_and_set_vector(int vec, void *bitmap) -{ - return test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); -} - -static inline int apic_test_and_clear_vector(int vec, void *bitmap) -{ - return test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); -} - -static inline void apic_set_vector(int vec, void *bitmap) -{ - set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); -} - -static inline void apic_clear_vector(int vec, void *bitmap) -{ - clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); -} - -static inline int apic_hw_enabled(struct kvm_lapic *apic) -{ - return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE; -} - -static inline int apic_sw_enabled(struct kvm_lapic *apic) -{ - return apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED; -} - -static inline int apic_enabled(struct kvm_lapic *apic) -{ - return apic_sw_enabled(apic) && apic_hw_enabled(apic); -} - -#define LVT_MASK \ - (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK) - -#define LINT_MASK \ - (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ - APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) - -static inline int kvm_apic_id(struct kvm_lapic *apic) -{ - return (apic_get_reg(apic, APIC_ID) >> 24) & 0xff; -} - -static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) -{ - return !(apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); -} - -static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) -{ - return apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; -} - -static inline int apic_lvtt_period(struct kvm_lapic *apic) -{ - return apic_get_reg(apic, APIC_LVTT) & APIC_LVT_TIMER_PERIODIC; -} - -static unsigned int apic_lvt_mask[APIC_LVT_NUM] = { - LVT_MASK | APIC_LVT_TIMER_PERIODIC, /* LVTT */ - LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ - LVT_MASK | APIC_MODE_MASK, /* LVTPC */ - LINT_MASK, LINT_MASK, /* LVT0-1 */ - LVT_MASK /* LVTERR */ -}; - -static int find_highest_vector(void *bitmap) -{ - u32 *word = bitmap; - int word_offset = MAX_APIC_VECTOR >> 5; - - while ((word_offset != 0) && (word[(--word_offset) << 2] == 0)) - continue; - - if (likely(!word_offset && !word[0])) - return -1; - else - return fls(word[word_offset << 2]) - 1 + (word_offset << 5); -} - -static inline int apic_test_and_set_irr(int vec, struct kvm_lapic *apic) -{ - return apic_test_and_set_vector(vec, apic->regs + APIC_IRR); -} - -static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) -{ - apic_clear_vector(vec, apic->regs + APIC_IRR); -} - -static inline int apic_find_highest_irr(struct kvm_lapic *apic) -{ - int result; - - result = find_highest_vector(apic->regs + APIC_IRR); - ASSERT(result == -1 || result >= 16); - - return result; -} - -int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - int highest_irr; - - if (!apic) - return 0; - highest_irr = apic_find_highest_irr(apic); - - return highest_irr; -} -EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr); - -int kvm_apic_set_irq(struct kvm_vcpu *vcpu, u8 vec, u8 trig) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - - if (!apic_test_and_set_irr(vec, apic)) { - /* a new pending irq is set in IRR */ - if (trig) - apic_set_vector(vec, apic->regs + APIC_TMR); - else - apic_clear_vector(vec, apic->regs + APIC_TMR); - kvm_vcpu_kick(apic->vcpu); - return 1; - } - return 0; -} - -static inline int apic_find_highest_isr(struct kvm_lapic *apic) -{ - int result; - - result = find_highest_vector(apic->regs + APIC_ISR); - ASSERT(result == -1 || result >= 16); - - return result; -} - -static void apic_update_ppr(struct kvm_lapic *apic) -{ - u32 tpr, isrv, ppr; - int isr; - - tpr = apic_get_reg(apic, APIC_TASKPRI); - isr = apic_find_highest_isr(apic); - isrv = (isr != -1) ? isr : 0; - - if ((tpr & 0xf0) >= (isrv & 0xf0)) - ppr = tpr & 0xff; - else - ppr = isrv & 0xf0; - - apic_debug("vlapic %p, ppr 0x%x, isr 0x%x, isrv 0x%x", - apic, ppr, isr, isrv); - - apic_set_reg(apic, APIC_PROCPRI, ppr); -} - -static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr) -{ - apic_set_reg(apic, APIC_TASKPRI, tpr); - apic_update_ppr(apic); -} - -int kvm_apic_match_physical_addr(struct kvm_lapic *apic, u16 dest) -{ - return kvm_apic_id(apic) == dest; -} - -int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda) -{ - int result = 0; - u8 logical_id; - - logical_id = GET_APIC_LOGICAL_ID(apic_get_reg(apic, APIC_LDR)); - - switch (apic_get_reg(apic, APIC_DFR)) { - case APIC_DFR_FLAT: - if (logical_id & mda) - result = 1; - break; - case APIC_DFR_CLUSTER: - if (((logical_id >> 4) == (mda >> 0x4)) - && (logical_id & mda & 0xf)) - result = 1; - break; - default: - printk(KERN_WARNING "Bad DFR vcpu %d: %08x\n", - apic->vcpu->vcpu_id, apic_get_reg(apic, APIC_DFR)); - break; - } - - return result; -} - -static int apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, - int short_hand, int dest, int dest_mode) -{ - int result = 0; - struct kvm_lapic *target = vcpu->arch.apic; - - apic_debug("target %p, source %p, dest 0x%x, " - "dest_mode 0x%x, short_hand 0x%x", - target, source, dest, dest_mode, short_hand); - - ASSERT(!target); - switch (short_hand) { - case APIC_DEST_NOSHORT: - if (dest_mode == 0) { - /* Physical mode. */ - if ((dest == 0xFF) || (dest == kvm_apic_id(target))) - result = 1; - } else - /* Logical mode. */ - result = kvm_apic_match_logical_addr(target, dest); - break; - case APIC_DEST_SELF: - if (target == source) - result = 1; - break; - case APIC_DEST_ALLINC: - result = 1; - break; - case APIC_DEST_ALLBUT: - if (target != source) - result = 1; - break; - default: - printk(KERN_WARNING "Bad dest shorthand value %x\n", - short_hand); - break; - } - - return result; -} - -/* - * Add a pending IRQ into lapic. - * Return 1 if successfully added and 0 if discarded. - */ -static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, - int vector, int level, int trig_mode) -{ - int orig_irr, result = 0; - struct kvm_vcpu *vcpu = apic->vcpu; - - switch (delivery_mode) { - case APIC_DM_FIXED: - case APIC_DM_LOWEST: - /* FIXME add logic for vcpu on reset */ - if (unlikely(!apic_enabled(apic))) - break; - - orig_irr = apic_test_and_set_irr(vector, apic); - if (orig_irr && trig_mode) { - apic_debug("level trig mode repeatedly for vector %d", - vector); - break; - } - - if (trig_mode) { - apic_debug("level trig mode for vector %d", vector); - apic_set_vector(vector, apic->regs + APIC_TMR); - } else - apic_clear_vector(vector, apic->regs + APIC_TMR); - - if (vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE) - kvm_vcpu_kick(vcpu); - else if (vcpu->arch.mp_state == VCPU_MP_STATE_HALTED) { - vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; - if (waitqueue_active(&vcpu->wq)) - wake_up_interruptible(&vcpu->wq); - } - - result = (orig_irr == 0); - break; - - case APIC_DM_REMRD: - printk(KERN_DEBUG "Ignoring delivery mode 3\n"); - break; - - case APIC_DM_SMI: - printk(KERN_DEBUG "Ignoring guest SMI\n"); - break; - case APIC_DM_NMI: - printk(KERN_DEBUG "Ignoring guest NMI\n"); - break; - - case APIC_DM_INIT: - if (level) { - if (vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE) - printk(KERN_DEBUG - "INIT on a runnable vcpu %d\n", - vcpu->vcpu_id); - vcpu->arch.mp_state = VCPU_MP_STATE_INIT_RECEIVED; - kvm_vcpu_kick(vcpu); - } else { - printk(KERN_DEBUG - "Ignoring de-assert INIT to vcpu %d\n", - vcpu->vcpu_id); - } - - break; - - case APIC_DM_STARTUP: - printk(KERN_DEBUG "SIPI to vcpu %d vector 0x%02x\n", - vcpu->vcpu_id, vector); - if (vcpu->arch.mp_state == VCPU_MP_STATE_INIT_RECEIVED) { - vcpu->arch.sipi_vector = vector; - vcpu->arch.mp_state = VCPU_MP_STATE_SIPI_RECEIVED; - if (waitqueue_active(&vcpu->wq)) - wake_up_interruptible(&vcpu->wq); - } - break; - - default: - printk(KERN_ERR "TODO: unsupported delivery mode %x\n", - delivery_mode); - break; - } - return result; -} - -static struct kvm_lapic *kvm_apic_round_robin(struct kvm *kvm, u8 vector, - unsigned long bitmap) -{ - int last; - int next; - struct kvm_lapic *apic = NULL; - - last = kvm->arch.round_robin_prev_vcpu; - next = last; - - do { - if (++next == KVM_MAX_VCPUS) - next = 0; - if (kvm->vcpus[next] == NULL || !test_bit(next, &bitmap)) - continue; - apic = kvm->vcpus[next]->arch.apic; - if (apic && apic_enabled(apic)) - break; - apic = NULL; - } while (next != last); - kvm->arch.round_robin_prev_vcpu = next; - - if (!apic) - printk(KERN_DEBUG "vcpu not ready for apic_round_robin\n"); - - return apic; -} - -struct kvm_vcpu *kvm_get_lowest_prio_vcpu(struct kvm *kvm, u8 vector, - unsigned long bitmap) -{ - struct kvm_lapic *apic; - - apic = kvm_apic_round_robin(kvm, vector, bitmap); - if (apic) - return apic->vcpu; - return NULL; -} - -static void apic_set_eoi(struct kvm_lapic *apic) -{ - int vector = apic_find_highest_isr(apic); - - /* - * Not every write EOI will has corresponding ISR, - * one example is when Kernel check timer on setup_IO_APIC - */ - if (vector == -1) - return; - - apic_clear_vector(vector, apic->regs + APIC_ISR); - apic_update_ppr(apic); - - if (apic_test_and_clear_vector(vector, apic->regs + APIC_TMR)) - kvm_ioapic_update_eoi(apic->vcpu->kvm, vector); -} - -static void apic_send_ipi(struct kvm_lapic *apic) -{ - u32 icr_low = apic_get_reg(apic, APIC_ICR); - u32 icr_high = apic_get_reg(apic, APIC_ICR2); - - unsigned int dest = GET_APIC_DEST_FIELD(icr_high); - unsigned int short_hand = icr_low & APIC_SHORT_MASK; - unsigned int trig_mode = icr_low & APIC_INT_LEVELTRIG; - unsigned int level = icr_low & APIC_INT_ASSERT; - unsigned int dest_mode = icr_low & APIC_DEST_MASK; - unsigned int delivery_mode = icr_low & APIC_MODE_MASK; - unsigned int vector = icr_low & APIC_VECTOR_MASK; - - struct kvm_vcpu *target; - struct kvm_vcpu *vcpu; - unsigned long lpr_map = 0; - int i; - - apic_debug("icr_high 0x%x, icr_low 0x%x, " - "short_hand 0x%x, dest 0x%x, trig_mode 0x%x, level 0x%x, " - "dest_mode 0x%x, delivery_mode 0x%x, vector 0x%x\n", - icr_high, icr_low, short_hand, dest, - trig_mode, level, dest_mode, delivery_mode, vector); - - for (i = 0; i < KVM_MAX_VCPUS; i++) { - vcpu = apic->vcpu->kvm->vcpus[i]; - if (!vcpu) - continue; - - if (vcpu->arch.apic && - apic_match_dest(vcpu, apic, short_hand, dest, dest_mode)) { - if (delivery_mode == APIC_DM_LOWEST) - set_bit(vcpu->vcpu_id, &lpr_map); - else - __apic_accept_irq(vcpu->arch.apic, delivery_mode, - vector, level, trig_mode); - } - } - - if (delivery_mode == APIC_DM_LOWEST) { - target = kvm_get_lowest_prio_vcpu(vcpu->kvm, vector, lpr_map); - if (target != NULL) - __apic_accept_irq(target->arch.apic, delivery_mode, - vector, level, trig_mode); - } -} - -static u32 apic_get_tmcct(struct kvm_lapic *apic) -{ - u64 counter_passed; - ktime_t passed, now; - u32 tmcct; - - ASSERT(apic != NULL); - - now = apic->timer.dev.base->get_time(); - tmcct = apic_get_reg(apic, APIC_TMICT); - - /* if initial count is 0, current count should also be 0 */ - if (tmcct == 0) - return 0; - - if (unlikely(ktime_to_ns(now) <= - ktime_to_ns(apic->timer.last_update))) { - /* Wrap around */ - passed = ktime_add(( { - (ktime_t) { - .tv64 = KTIME_MAX - - (apic->timer.last_update).tv64}; } - ), now); - apic_debug("time elapsed\n"); - } else - passed = ktime_sub(now, apic->timer.last_update); - - counter_passed = div64_64(ktime_to_ns(passed), - (APIC_BUS_CYCLE_NS * apic->timer.divide_count)); - - if (counter_passed > tmcct) { - if (unlikely(!apic_lvtt_period(apic))) { - /* one-shot timers stick at 0 until reset */ - tmcct = 0; - } else { - /* - * periodic timers reset to APIC_TMICT when they - * hit 0. The while loop simulates this happening N - * times. (counter_passed %= tmcct) would also work, - * but might be slower or not work on 32-bit?? - */ - while (counter_passed > tmcct) - counter_passed -= tmcct; - tmcct -= counter_passed; - } - } else { - tmcct -= counter_passed; - } - - return tmcct; -} - -static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset) -{ - u32 val = 0; - - if (offset >= LAPIC_MMIO_LENGTH) - return 0; - - switch (offset) { - case APIC_ARBPRI: - printk(KERN_WARNING "Access APIC ARBPRI register " - "which is for P6\n"); - break; - - case APIC_TMCCT: /* Timer CCR */ - val = apic_get_tmcct(apic); - break; - - default: - apic_update_ppr(apic); - val = apic_get_reg(apic, offset); - break; - } - - return val; -} - -static void apic_mmio_read(struct kvm_io_device *this, - gpa_t address, int len, void *data) -{ - struct kvm_lapic *apic = (struct kvm_lapic *)this->private; - unsigned int offset = address - apic->base_address; - unsigned char alignment = offset & 0xf; - u32 result; - - if ((alignment + len) > 4) { - printk(KERN_ERR "KVM_APIC_READ: alignment error %lx %d", - (unsigned long)address, len); - return; - } - result = __apic_read(apic, offset & ~0xf); - - switch (len) { - case 1: - case 2: - case 4: - memcpy(data, (char *)&result + alignment, len); - break; - default: - printk(KERN_ERR "Local APIC read with len = %x, " - "should be 1,2, or 4 instead\n", len); - break; - } -} - -static void update_divide_count(struct kvm_lapic *apic) -{ - u32 tmp1, tmp2, tdcr; - - tdcr = apic_get_reg(apic, APIC_TDCR); - tmp1 = tdcr & 0xf; - tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; - apic->timer.divide_count = 0x1 << (tmp2 & 0x7); - - apic_debug("timer divide count is 0x%x\n", - apic->timer.divide_count); -} - -static void start_apic_timer(struct kvm_lapic *apic) -{ - ktime_t now = apic->timer.dev.base->get_time(); - - apic->timer.last_update = now; - - apic->timer.period = apic_get_reg(apic, APIC_TMICT) * - APIC_BUS_CYCLE_NS * apic->timer.divide_count; - atomic_set(&apic->timer.pending, 0); - hrtimer_start(&apic->timer.dev, - ktime_add_ns(now, apic->timer.period), - HRTIMER_MODE_ABS); - - apic_debug("%s: bus cycle is %" PRId64 "ns, now 0x%016" - PRIx64 ", " - "timer initial count 0x%x, period %lldns, " - "expire @ 0x%016" PRIx64 ".\n", __FUNCTION__, - APIC_BUS_CYCLE_NS, ktime_to_ns(now), - apic_get_reg(apic, APIC_TMICT), - apic->timer.period, - ktime_to_ns(ktime_add_ns(now, - apic->timer.period))); -} - -static void apic_mmio_write(struct kvm_io_device *this, - gpa_t address, int len, const void *data) -{ - struct kvm_lapic *apic = (struct kvm_lapic *)this->private; - unsigned int offset = address - apic->base_address; - unsigned char alignment = offset & 0xf; - u32 val; - - /* - * APIC register must be aligned on 128-bits boundary. - * 32/64/128 bits registers must be accessed thru 32 bits. - * Refer SDM 8.4.1 - */ - if (len != 4 || alignment) { - if (printk_ratelimit()) - printk(KERN_ERR "apic write: bad size=%d %lx\n", - len, (long)address); - return; - } - - val = *(u32 *) data; - - /* too common printing */ - if (offset != APIC_EOI) - apic_debug("%s: offset 0x%x with length 0x%x, and value is " - "0x%x\n", __FUNCTION__, offset, len, val); - - offset &= 0xff0; - - switch (offset) { - case APIC_ID: /* Local APIC ID */ - apic_set_reg(apic, APIC_ID, val); - break; - - case APIC_TASKPRI: - apic_set_tpr(apic, val & 0xff); - break; - - case APIC_EOI: - apic_set_eoi(apic); - break; - - case APIC_LDR: - apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK); - break; - - case APIC_DFR: - apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); - break; - - case APIC_SPIV: - apic_set_reg(apic, APIC_SPIV, val & 0x3ff); - if (!(val & APIC_SPIV_APIC_ENABLED)) { - int i; - u32 lvt_val; - - for (i = 0; i < APIC_LVT_NUM; i++) { - lvt_val = apic_get_reg(apic, - APIC_LVTT + 0x10 * i); - apic_set_reg(apic, APIC_LVTT + 0x10 * i, - lvt_val | APIC_LVT_MASKED); - } - atomic_set(&apic->timer.pending, 0); - - } - break; - - case APIC_ICR: - /* No delay here, so we always clear the pending bit */ - apic_set_reg(apic, APIC_ICR, val & ~(1 << 12)); - apic_send_ipi(apic); - break; - - case APIC_ICR2: - apic_set_reg(apic, APIC_ICR2, val & 0xff000000); - break; - - case APIC_LVTT: - case APIC_LVTTHMR: - case APIC_LVTPC: - case APIC_LVT0: - case APIC_LVT1: - case APIC_LVTERR: - /* TODO: Check vector */ - if (!apic_sw_enabled(apic)) - val |= APIC_LVT_MASKED; - - val &= apic_lvt_mask[(offset - APIC_LVTT) >> 4]; - apic_set_reg(apic, offset, val); - - break; - - case APIC_TMICT: - hrtimer_cancel(&apic->timer.dev); - apic_set_reg(apic, APIC_TMICT, val); - start_apic_timer(apic); - return; - - case APIC_TDCR: - if (val & 4) - printk(KERN_ERR "KVM_WRITE:TDCR %x\n", val); - apic_set_reg(apic, APIC_TDCR, val); - update_divide_count(apic); - break; - - default: - apic_debug("Local APIC Write to read-only register %x\n", - offset); - break; - } - -} - -static int apic_mmio_range(struct kvm_io_device *this, gpa_t addr) -{ - struct kvm_lapic *apic = (struct kvm_lapic *)this->private; - int ret = 0; - - - if (apic_hw_enabled(apic) && - (addr >= apic->base_address) && - (addr < (apic->base_address + LAPIC_MMIO_LENGTH))) - ret = 1; - - return ret; -} - -void kvm_free_lapic(struct kvm_vcpu *vcpu) -{ - if (!vcpu->arch.apic) - return; - - hrtimer_cancel(&vcpu->arch.apic->timer.dev); - - if (vcpu->arch.apic->regs_page) - __free_page(vcpu->arch.apic->regs_page); - - kfree(vcpu->arch.apic); -} - -/* - *---------------------------------------------------------------------- - * LAPIC interface - *---------------------------------------------------------------------- - */ - -void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - - if (!apic) - return; - apic_set_tpr(apic, ((cr8 & 0x0f) << 4)); -} - -u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - u64 tpr; - - if (!apic) - return 0; - tpr = (u64) apic_get_reg(apic, APIC_TASKPRI); - - return (tpr & 0xf0) >> 4; -} -EXPORT_SYMBOL_GPL(kvm_lapic_get_cr8); - -void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - - if (!apic) { - value |= MSR_IA32_APICBASE_BSP; - vcpu->arch.apic_base = value; - return; - } - if (apic->vcpu->vcpu_id) - value &= ~MSR_IA32_APICBASE_BSP; - - vcpu->arch.apic_base = value; - apic->base_address = apic->vcpu->arch.apic_base & - MSR_IA32_APICBASE_BASE; - - /* with FSB delivery interrupt, we can restart APIC functionality */ - apic_debug("apic base msr is 0x%016" PRIx64 ", and base address is " - "0x%lx.\n", apic->vcpu->arch.apic_base, apic->base_address); - -} - -u64 kvm_lapic_get_base(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.apic_base; -} -EXPORT_SYMBOL_GPL(kvm_lapic_get_base); - -void kvm_lapic_reset(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic; - int i; - - apic_debug("%s\n", __FUNCTION__); - - ASSERT(vcpu); - apic = vcpu->arch.apic; - ASSERT(apic != NULL); - - /* Stop the timer in case it's a reset to an active apic */ - hrtimer_cancel(&apic->timer.dev); - - apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24); - apic_set_reg(apic, APIC_LVR, APIC_VERSION); - - for (i = 0; i < APIC_LVT_NUM; i++) - apic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED); - apic_set_reg(apic, APIC_LVT0, - SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); - - apic_set_reg(apic, APIC_DFR, 0xffffffffU); - apic_set_reg(apic, APIC_SPIV, 0xff); - apic_set_reg(apic, APIC_TASKPRI, 0); - apic_set_reg(apic, APIC_LDR, 0); - apic_set_reg(apic, APIC_ESR, 0); - apic_set_reg(apic, APIC_ICR, 0); - apic_set_reg(apic, APIC_ICR2, 0); - apic_set_reg(apic, APIC_TDCR, 0); - apic_set_reg(apic, APIC_TMICT, 0); - for (i = 0; i < 8; i++) { - apic_set_reg(apic, APIC_IRR + 0x10 * i, 0); - apic_set_reg(apic, APIC_ISR + 0x10 * i, 0); - apic_set_reg(apic, APIC_TMR + 0x10 * i, 0); - } - update_divide_count(apic); - atomic_set(&apic->timer.pending, 0); - if (vcpu->vcpu_id == 0) - vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP; - apic_update_ppr(apic); - - apic_debug(KERN_INFO "%s: vcpu=%p, id=%d, base_msr=" - "0x%016" PRIx64 ", base_address=0x%0lx.\n", __FUNCTION__, - vcpu, kvm_apic_id(apic), - vcpu->arch.apic_base, apic->base_address); -} -EXPORT_SYMBOL_GPL(kvm_lapic_reset); - -int kvm_lapic_enabled(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - int ret = 0; - - if (!apic) - return 0; - ret = apic_enabled(apic); - - return ret; -} -EXPORT_SYMBOL_GPL(kvm_lapic_enabled); - -/* - *---------------------------------------------------------------------- - * timer interface - *---------------------------------------------------------------------- - */ - -/* TODO: make sure __apic_timer_fn runs in current pCPU */ -static int __apic_timer_fn(struct kvm_lapic *apic) -{ - int result = 0; - wait_queue_head_t *q = &apic->vcpu->wq; - - atomic_inc(&apic->timer.pending); - if (waitqueue_active(q)) { - apic->vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; - wake_up_interruptible(q); - } - if (apic_lvtt_period(apic)) { - result = 1; - apic->timer.dev.expires = ktime_add_ns( - apic->timer.dev.expires, - apic->timer.period); - } - return result; -} - -static int __inject_apic_timer_irq(struct kvm_lapic *apic) -{ - int vector; - - vector = apic_lvt_vector(apic, APIC_LVTT); - return __apic_accept_irq(apic, APIC_DM_FIXED, vector, 1, 0); -} - -static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) -{ - struct kvm_lapic *apic; - int restart_timer = 0; - - apic = container_of(data, struct kvm_lapic, timer.dev); - - restart_timer = __apic_timer_fn(apic); - - if (restart_timer) - return HRTIMER_RESTART; - else - return HRTIMER_NORESTART; -} - -int kvm_create_lapic(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic; - - ASSERT(vcpu != NULL); - apic_debug("apic_init %d\n", vcpu->vcpu_id); - - apic = kzalloc(sizeof(*apic), GFP_KERNEL); - if (!apic) - goto nomem; - - vcpu->arch.apic = apic; - - apic->regs_page = alloc_page(GFP_KERNEL); - if (apic->regs_page == NULL) { - printk(KERN_ERR "malloc apic regs error for vcpu %x\n", - vcpu->vcpu_id); - goto nomem_free_apic; - } - apic->regs = page_address(apic->regs_page); - memset(apic->regs, 0, PAGE_SIZE); - apic->vcpu = vcpu; - - hrtimer_init(&apic->timer.dev, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); - apic->timer.dev.function = apic_timer_fn; - apic->base_address = APIC_DEFAULT_PHYS_BASE; - vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE; - - kvm_lapic_reset(vcpu); - apic->dev.read = apic_mmio_read; - apic->dev.write = apic_mmio_write; - apic->dev.in_range = apic_mmio_range; - apic->dev.private = apic; - - return 0; -nomem_free_apic: - kfree(apic); -nomem: - return -ENOMEM; -} -EXPORT_SYMBOL_GPL(kvm_create_lapic); - -int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - int highest_irr; - - if (!apic || !apic_enabled(apic)) - return -1; - - apic_update_ppr(apic); - highest_irr = apic_find_highest_irr(apic); - if ((highest_irr == -1) || - ((highest_irr & 0xF0) <= apic_get_reg(apic, APIC_PROCPRI))) - return -1; - return highest_irr; -} - -int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) -{ - u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0); - int r = 0; - - if (vcpu->vcpu_id == 0) { - if (!apic_hw_enabled(vcpu->arch.apic)) - r = 1; - if ((lvt0 & APIC_LVT_MASKED) == 0 && - GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) - r = 1; - } - return r; -} - -void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - - if (apic && apic_lvt_enabled(apic, APIC_LVTT) && - atomic_read(&apic->timer.pending) > 0) { - if (__inject_apic_timer_irq(apic)) - atomic_dec(&apic->timer.pending); - } -} - -void kvm_apic_timer_intr_post(struct kvm_vcpu *vcpu, int vec) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - - if (apic && apic_lvt_vector(apic, APIC_LVTT) == vec) - apic->timer.last_update = ktime_add_ns( - apic->timer.last_update, - apic->timer.period); -} - -int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu) -{ - int vector = kvm_apic_has_interrupt(vcpu); - struct kvm_lapic *apic = vcpu->arch.apic; - - if (vector == -1) - return -1; - - apic_set_vector(vector, apic->regs + APIC_ISR); - apic_update_ppr(apic); - apic_clear_irr(vector, apic); - return vector; -} - -void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - - apic->base_address = vcpu->arch.apic_base & - MSR_IA32_APICBASE_BASE; - apic_set_reg(apic, APIC_LVR, APIC_VERSION); - apic_update_ppr(apic); - hrtimer_cancel(&apic->timer.dev); - update_divide_count(apic); - start_apic_timer(apic); -} - -void kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) -{ - struct kvm_lapic *apic = vcpu->arch.apic; - struct hrtimer *timer; - - if (!apic) - return; - - timer = &apic->timer.dev; - if (hrtimer_cancel(timer)) - hrtimer_start(timer, timer->expires, HRTIMER_MODE_ABS); -} -EXPORT_SYMBOL_GPL(kvm_migrate_apic_timer); diff --git a/drivers/kvm/mmu.c b/drivers/kvm/mmu.c deleted file mode 100644 index c26d83f86a3a..000000000000 --- a/drivers/kvm/mmu.c +++ /dev/null @@ -1,1806 +0,0 @@ -/* - * Kernel-based Virtual Machine driver for Linux - * - * This module enables machines with Intel VT-x extensions to run virtual - * machines without emulation or binary translation. - * - * MMU support - * - * Copyright (C) 2006 Qumranet, Inc. - * - * Authors: - * Yaniv Kamay - * Avi Kivity - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - * - */ - -#include "vmx.h" -#include "kvm.h" -#include "x86.h" -#include "mmu.h" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#undef MMU_DEBUG - -#undef AUDIT - -#ifdef AUDIT -static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg); -#else -static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {} -#endif - -#ifdef MMU_DEBUG - -#define pgprintk(x...) do { if (dbg) printk(x); } while (0) -#define rmap_printk(x...) do { if (dbg) printk(x); } while (0) - -#else - -#define pgprintk(x...) do { } while (0) -#define rmap_printk(x...) do { } while (0) - -#endif - -#if defined(MMU_DEBUG) || defined(AUDIT) -static int dbg = 1; -#endif - -#ifndef MMU_DEBUG -#define ASSERT(x) do { } while (0) -#else -#define ASSERT(x) \ - if (!(x)) { \ - printk(KERN_WARNING "assertion failed %s:%d: %s\n", \ - __FILE__, __LINE__, #x); \ - } -#endif - -#define PT64_PT_BITS 9 -#define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS) -#define PT32_PT_BITS 10 -#define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS) - -#define PT_WRITABLE_SHIFT 1 - -#define PT_PRESENT_MASK (1ULL << 0) -#define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT) -#define PT_USER_MASK (1ULL << 2) -#define PT_PWT_MASK (1ULL << 3) -#define PT_PCD_MASK (1ULL << 4) -#define PT_ACCESSED_MASK (1ULL << 5) -#define PT_DIRTY_MASK (1ULL << 6) -#define PT_PAGE_SIZE_MASK (1ULL << 7) -#define PT_PAT_MASK (1ULL << 7) -#define PT_GLOBAL_MASK (1ULL << 8) -#define PT64_NX_SHIFT 63 -#define PT64_NX_MASK (1ULL << PT64_NX_SHIFT) - -#define PT_PAT_SHIFT 7 -#define PT_DIR_PAT_SHIFT 12 -#define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT) - -#define PT32_DIR_PSE36_SIZE 4 -#define PT32_DIR_PSE36_SHIFT 13 -#define PT32_DIR_PSE36_MASK \ - (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT) - - -#define PT_FIRST_AVAIL_BITS_SHIFT 9 -#define PT64_SECOND_AVAIL_BITS_SHIFT 52 - -#define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT) - -#define VALID_PAGE(x) ((x) != INVALID_PAGE) - -#define PT64_LEVEL_BITS 9 - -#define PT64_LEVEL_SHIFT(level) \ - (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS) - -#define PT64_LEVEL_MASK(level) \ - (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level)) - -#define PT64_INDEX(address, level)\ - (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1)) - - -#define PT32_LEVEL_BITS 10 - -#define PT32_LEVEL_SHIFT(level) \ - (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS) - -#define PT32_LEVEL_MASK(level) \ - (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level)) - -#define PT32_INDEX(address, level)\ - (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1)) - - -#define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1)) -#define PT64_DIR_BASE_ADDR_MASK \ - (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1)) - -#define PT32_BASE_ADDR_MASK PAGE_MASK -#define PT32_DIR_BASE_ADDR_MASK \ - (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1)) - -#define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \ - | PT64_NX_MASK) - -#define PFERR_PRESENT_MASK (1U << 0) -#define PFERR_WRITE_MASK (1U << 1) -#define PFERR_USER_MASK (1U << 2) -#define PFERR_FETCH_MASK (1U << 4) - -#define PT64_ROOT_LEVEL 4 -#define PT32_ROOT_LEVEL 2 -#define PT32E_ROOT_LEVEL 3 - -#define PT_DIRECTORY_LEVEL 2 -#define PT_PAGE_TABLE_LEVEL 1 - -#define RMAP_EXT 4 - -#define ACC_EXEC_MASK 1 -#define ACC_WRITE_MASK PT_WRITABLE_MASK -#define ACC_USER_MASK PT_USER_MASK -#define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK) - -struct kvm_rmap_desc { - u64 *shadow_ptes[RMAP_EXT]; - struct kvm_rmap_desc *more; -}; - -static struct kmem_cache *pte_chain_cache; -static struct kmem_cache *rmap_desc_cache; -static struct kmem_cache *mmu_page_header_cache; - -static u64 __read_mostly shadow_trap_nonpresent_pte; -static u64 __read_mostly shadow_notrap_nonpresent_pte; - -void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte) -{ - shadow_trap_nonpresent_pte = trap_pte; - shadow_notrap_nonpresent_pte = notrap_pte; -} -EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes); - -static int is_write_protection(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr0 & X86_CR0_WP; -} - -static int is_cpuid_PSE36(void) -{ - return 1; -} - -static int is_nx(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.shadow_efer & EFER_NX; -} - -static int is_present_pte(unsigned long pte) -{ - return pte & PT_PRESENT_MASK; -} - -static int is_shadow_present_pte(u64 pte) -{ - pte &= ~PT_SHADOW_IO_MARK; - return pte != shadow_trap_nonpresent_pte - && pte != shadow_notrap_nonpresent_pte; -} - -static int is_writeble_pte(unsigned long pte) -{ - return pte & PT_WRITABLE_MASK; -} - -static int is_dirty_pte(unsigned long pte) -{ - return pte & PT_DIRTY_MASK; -} - -static int is_io_pte(unsigned long pte) -{ - return pte & PT_SHADOW_IO_MARK; -} - -static int is_rmap_pte(u64 pte) -{ - return pte != shadow_trap_nonpresent_pte - && pte != shadow_notrap_nonpresent_pte; -} - -static gfn_t pse36_gfn_delta(u32 gpte) -{ - int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT; - - return (gpte & PT32_DIR_PSE36_MASK) << shift; -} - -static void set_shadow_pte(u64 *sptep, u64 spte) -{ -#ifdef CONFIG_X86_64 - set_64bit((unsigned long *)sptep, spte); -#else - set_64bit((unsigned long long *)sptep, spte); -#endif -} - -static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache, - struct kmem_cache *base_cache, int min) -{ - void *obj; - - if (cache->nobjs >= min) - return 0; - while (cache->nobjs < ARRAY_SIZE(cache->objects)) { - obj = kmem_cache_zalloc(base_cache, GFP_KERNEL); - if (!obj) - return -ENOMEM; - cache->objects[cache->nobjs++] = obj; - } - return 0; -} - -static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc) -{ - while (mc->nobjs) - kfree(mc->objects[--mc->nobjs]); -} - -static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache, - int min) -{ - struct page *page; - - if (cache->nobjs >= min) - return 0; - while (cache->nobjs < ARRAY_SIZE(cache->objects)) { - page = alloc_page(GFP_KERNEL); - if (!page) - return -ENOMEM; - set_page_private(page, 0); - cache->objects[cache->nobjs++] = page_address(page); - } - return 0; -} - -static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc) -{ - while (mc->nobjs) - free_page((unsigned long)mc->objects[--mc->nobjs]); -} - -static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu) -{ - int r; - - kvm_mmu_free_some_pages(vcpu); - r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache, - pte_chain_cache, 4); - if (r) - goto out; - r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache, - rmap_desc_cache, 1); - if (r) - goto out; - r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8); - if (r) - goto out; - r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache, - mmu_page_header_cache, 4); -out: - return r; -} - -static void mmu_free_memory_caches(struct kvm_vcpu *vcpu) -{ - mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache); - mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache); - mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache); - mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache); -} - -static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, - size_t size) -{ - void *p; - - BUG_ON(!mc->nobjs); - p = mc->objects[--mc->nobjs]; - memset(p, 0, size); - return p; -} - -static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu) -{ - return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache, - sizeof(struct kvm_pte_chain)); -} - -static void mmu_free_pte_chain(struct kvm_pte_chain *pc) -{ - kfree(pc); -} - -static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu) -{ - return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache, - sizeof(struct kvm_rmap_desc)); -} - -static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd) -{ - kfree(rd); -} - -/* - * Take gfn and return the reverse mapping to it. - * Note: gfn must be unaliased before this function get called - */ - -static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn) -{ - struct kvm_memory_slot *slot; - - slot = gfn_to_memslot(kvm, gfn); - return &slot->rmap[gfn - slot->base_gfn]; -} - -/* - * Reverse mapping data structures: - * - * If rmapp bit zero is zero, then rmapp point to the shadw page table entry - * that points to page_address(page). - * - * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc - * containing more mappings. - */ -static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) -{ - struct kvm_mmu_page *sp; - struct kvm_rmap_desc *desc; - unsigned long *rmapp; - int i; - - if (!is_rmap_pte(*spte)) - return; - gfn = unalias_gfn(vcpu->kvm, gfn); - sp = page_header(__pa(spte)); - sp->gfns[spte - sp->spt] = gfn; - rmapp = gfn_to_rmap(vcpu->kvm, gfn); - if (!*rmapp) { - rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte); - *rmapp = (unsigned long)spte; - } else if (!(*rmapp & 1)) { - rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte); - desc = mmu_alloc_rmap_desc(vcpu); - desc->shadow_ptes[0] = (u64 *)*rmapp; - desc->shadow_ptes[1] = spte; - *rmapp = (unsigned long)desc | 1; - } else { - rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte); - desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); - while (desc->shadow_ptes[RMAP_EXT-1] && desc->more) - desc = desc->more; - if (desc->shadow_ptes[RMAP_EXT-1]) { - desc->more = mmu_alloc_rmap_desc(vcpu); - desc = desc->more; - } - for (i = 0; desc->shadow_ptes[i]; ++i) - ; - desc->shadow_ptes[i] = spte; - } -} - -static void rmap_desc_remove_entry(unsigned long *rmapp, - struct kvm_rmap_desc *desc, - int i, - struct kvm_rmap_desc *prev_desc) -{ - int j; - - for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j) - ; - desc->shadow_ptes[i] = desc->shadow_ptes[j]; - desc->shadow_ptes[j] = NULL; - if (j != 0) - return; - if (!prev_desc && !desc->more) - *rmapp = (unsigned long)desc->shadow_ptes[0]; - else - if (prev_desc) - prev_desc->more = desc->more; - else - *rmapp = (unsigned long)desc->more | 1; - mmu_free_rmap_desc(desc); -} - -static void rmap_remove(struct kvm *kvm, u64 *spte) -{ - struct kvm_rmap_desc *desc; - struct kvm_rmap_desc *prev_desc; - struct kvm_mmu_page *sp; - struct page *page; - unsigned long *rmapp; - int i; - - if (!is_rmap_pte(*spte)) - return; - sp = page_header(__pa(spte)); - page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT); - mark_page_accessed(page); - if (is_writeble_pte(*spte)) - kvm_release_page_dirty(page); - else - kvm_release_page_clean(page); - rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt]); - if (!*rmapp) { - printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte); - BUG(); - } else if (!(*rmapp & 1)) { - rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte); - if ((u64 *)*rmapp != spte) { - printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n", - spte, *spte); - BUG(); - } - *rmapp = 0; - } else { - rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte); - desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); - prev_desc = NULL; - while (desc) { - for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) - if (desc->shadow_ptes[i] == spte) { - rmap_desc_remove_entry(rmapp, - desc, i, - prev_desc); - return; - } - prev_desc = desc; - desc = desc->more; - } - BUG(); - } -} - -static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte) -{ - struct kvm_rmap_desc *desc; - struct kvm_rmap_desc *prev_desc; - u64 *prev_spte; - int i; - - if (!*rmapp) - return NULL; - else if (!(*rmapp & 1)) { - if (!spte) - return (u64 *)*rmapp; - return NULL; - } - desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul); - prev_desc = NULL; - prev_spte = NULL; - while (desc) { - for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) { - if (prev_spte == spte) - return desc->shadow_ptes[i]; - prev_spte = desc->shadow_ptes[i]; - } - desc = desc->more; - } - return NULL; -} - -static void rmap_write_protect(struct kvm *kvm, u64 gfn) -{ - unsigned long *rmapp; - u64 *spte; - - gfn = unalias_gfn(kvm, gfn); - rmapp = gfn_to_rmap(kvm, gfn); - - spte = rmap_next(kvm, rmapp, NULL); - while (spte) { - BUG_ON(!spte); - BUG_ON(!(*spte & PT_PRESENT_MASK)); - rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte); - if (is_writeble_pte(*spte)) - set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK); - kvm_flush_remote_tlbs(kvm); - spte = rmap_next(kvm, rmapp, spte); - } -} - -#ifdef MMU_DEBUG -static int is_empty_shadow_page(u64 *spt) -{ - u64 *pos; - u64 *end; - - for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++) - if ((*pos & ~PT_SHADOW_IO_MARK) != shadow_trap_nonpresent_pte) { - printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__, - pos, *pos); - return 0; - } - return 1; -} -#endif - -static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp) -{ - ASSERT(is_empty_shadow_page(sp->spt)); - list_del(&sp->link); - __free_page(virt_to_page(sp->spt)); - __free_page(virt_to_page(sp->gfns)); - kfree(sp); - ++kvm->arch.n_free_mmu_pages; -} - -static unsigned kvm_page_table_hashfn(gfn_t gfn) -{ - return gfn; -} - -static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, - u64 *parent_pte) -{ - struct kvm_mmu_page *sp; - - if (!vcpu->kvm->arch.n_free_mmu_pages) - return NULL; - - sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp); - sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); - sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE); - set_page_private(virt_to_page(sp->spt), (unsigned long)sp); - list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); - ASSERT(is_empty_shadow_page(sp->spt)); - sp->slot_bitmap = 0; - sp->multimapped = 0; - sp->parent_pte = parent_pte; - --vcpu->kvm->arch.n_free_mmu_pages; - return sp; -} - -static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp, u64 *parent_pte) -{ - struct kvm_pte_chain *pte_chain; - struct hlist_node *node; - int i; - - if (!parent_pte) - return; - if (!sp->multimapped) { - u64 *old = sp->parent_pte; - - if (!old) { - sp->parent_pte = parent_pte; - return; - } - sp->multimapped = 1; - pte_chain = mmu_alloc_pte_chain(vcpu); - INIT_HLIST_HEAD(&sp->parent_ptes); - hlist_add_head(&pte_chain->link, &sp->parent_ptes); - pte_chain->parent_ptes[0] = old; - } - hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) { - if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1]) - continue; - for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) - if (!pte_chain->parent_ptes[i]) { - pte_chain->parent_ptes[i] = parent_pte; - return; - } - } - pte_chain = mmu_alloc_pte_chain(vcpu); - BUG_ON(!pte_chain); - hlist_add_head(&pte_chain->link, &sp->parent_ptes); - pte_chain->parent_ptes[0] = parent_pte; -} - -static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp, - u64 *parent_pte) -{ - struct kvm_pte_chain *pte_chain; - struct hlist_node *node; - int i; - - if (!sp->multimapped) { - BUG_ON(sp->parent_pte != parent_pte); - sp->parent_pte = NULL; - return; - } - hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) - for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) { - if (!pte_chain->parent_ptes[i]) - break; - if (pte_chain->parent_ptes[i] != parent_pte) - continue; - while (i + 1 < NR_PTE_CHAIN_ENTRIES - && pte_chain->parent_ptes[i + 1]) { - pte_chain->parent_ptes[i] - = pte_chain->parent_ptes[i + 1]; - ++i; - } - pte_chain->parent_ptes[i] = NULL; - if (i == 0) { - hlist_del(&pte_chain->link); - mmu_free_pte_chain(pte_chain); - if (hlist_empty(&sp->parent_ptes)) { - sp->multimapped = 0; - sp->parent_pte = NULL; - } - } - return; - } - BUG(); -} - -static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) -{ - unsigned index; - struct hlist_head *bucket; - struct kvm_mmu_page *sp; - struct hlist_node *node; - - pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); - index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &kvm->arch.mmu_page_hash[index]; - hlist_for_each_entry(sp, node, bucket, hash_link) - if (sp->gfn == gfn && !sp->role.metaphysical) { - pgprintk("%s: found role %x\n", - __FUNCTION__, sp->role.word); - return sp; - } - return NULL; -} - -static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, - gfn_t gfn, - gva_t gaddr, - unsigned level, - int metaphysical, - unsigned access, - u64 *parent_pte, - bool *new_page) -{ - union kvm_mmu_page_role role; - unsigned index; - unsigned quadrant; - struct hlist_head *bucket; - struct kvm_mmu_page *sp; - struct hlist_node *node; - - role.word = 0; - role.glevels = vcpu->arch.mmu.root_level; - role.level = level; - role.metaphysical = metaphysical; - role.access = access; - if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) { - quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); - quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; - role.quadrant = quadrant; - } - pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__, - gfn, role.word); - index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->arch.mmu_page_hash[index]; - hlist_for_each_entry(sp, node, bucket, hash_link) - if (sp->gfn == gfn && sp->role.word == role.word) { - mmu_page_add_parent_pte(vcpu, sp, parent_pte); - pgprintk("%s: found\n", __FUNCTION__); - return sp; - } - sp = kvm_mmu_alloc_page(vcpu, parent_pte); - if (!sp) - return sp; - pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word); - sp->gfn = gfn; - sp->role = role; - hlist_add_head(&sp->hash_link, bucket); - vcpu->arch.mmu.prefetch_page(vcpu, sp); - if (!metaphysical) - rmap_write_protect(vcpu->kvm, gfn); - if (new_page) - *new_page = 1; - return sp; -} - -static void kvm_mmu_page_unlink_children(struct kvm *kvm, - struct kvm_mmu_page *sp) -{ - unsigned i; - u64 *pt; - u64 ent; - - pt = sp->spt; - - if (sp->role.level == PT_PAGE_TABLE_LEVEL) { - for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { - if (is_shadow_present_pte(pt[i])) - rmap_remove(kvm, &pt[i]); - pt[i] = shadow_trap_nonpresent_pte; - } - kvm_flush_remote_tlbs(kvm); - return; - } - - for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { - ent = pt[i]; - - pt[i] = shadow_trap_nonpresent_pte; - if (!is_shadow_present_pte(ent)) - continue; - ent &= PT64_BASE_ADDR_MASK; - mmu_page_remove_parent_pte(page_header(ent), &pt[i]); - } - kvm_flush_remote_tlbs(kvm); -} - -static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte) -{ - mmu_page_remove_parent_pte(sp, parent_pte); -} - -static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm) -{ - int i; - - for (i = 0; i < KVM_MAX_VCPUS; ++i) - if (kvm->vcpus[i]) - kvm->vcpus[i]->arch.last_pte_updated = NULL; -} - -static void kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) -{ - u64 *parent_pte; - - ++kvm->stat.mmu_shadow_zapped; - while (sp->multimapped || sp->parent_pte) { - if (!sp->multimapped) - parent_pte = sp->parent_pte; - else { - struct kvm_pte_chain *chain; - - chain = container_of(sp->parent_ptes.first, - struct kvm_pte_chain, link); - parent_pte = chain->parent_ptes[0]; - } - BUG_ON(!parent_pte); - kvm_mmu_put_page(sp, parent_pte); - set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte); - } - kvm_mmu_page_unlink_children(kvm, sp); - if (!sp->root_count) { - hlist_del(&sp->hash_link); - kvm_mmu_free_page(kvm, sp); - } else - list_move(&sp->link, &kvm->arch.active_mmu_pages); - kvm_mmu_reset_last_pte_updated(kvm); -} - -/* - * Changing the number of mmu pages allocated to the vm - * Note: if kvm_nr_mmu_pages is too small, you will get dead lock - */ -void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages) -{ - /* - * If we set the number of mmu pages to be smaller be than the - * number of actived pages , we must to free some mmu pages before we - * change the value - */ - - if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) > - kvm_nr_mmu_pages) { - int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages - - kvm->arch.n_free_mmu_pages; - - while (n_used_mmu_pages > kvm_nr_mmu_pages) { - struct kvm_mmu_page *page; - - page = container_of(kvm->arch.active_mmu_pages.prev, - struct kvm_mmu_page, link); - kvm_mmu_zap_page(kvm, page); - n_used_mmu_pages--; - } - kvm->arch.n_free_mmu_pages = 0; - } - else - kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages - - kvm->arch.n_alloc_mmu_pages; - - kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages; -} - -static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) -{ - unsigned index; - struct hlist_head *bucket; - struct kvm_mmu_page *sp; - struct hlist_node *node, *n; - int r; - - pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn); - r = 0; - index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &kvm->arch.mmu_page_hash[index]; - hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) - if (sp->gfn == gfn && !sp->role.metaphysical) { - pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn, - sp->role.word); - kvm_mmu_zap_page(kvm, sp); - r = 1; - } - return r; -} - -static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) -{ - struct kvm_mmu_page *sp; - - while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { - pgprintk("%s: zap %lx %x\n", __FUNCTION__, gfn, sp->role.word); - kvm_mmu_zap_page(kvm, sp); - } -} - -static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn) -{ - int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn)); - struct kvm_mmu_page *sp = page_header(__pa(pte)); - - __set_bit(slot, &sp->slot_bitmap); -} - -struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva) -{ - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); - - if (gpa == UNMAPPED_GVA) - return NULL; - return gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); -} - -static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, - unsigned pt_access, unsigned pte_access, - int user_fault, int write_fault, int dirty, - int *ptwrite, gfn_t gfn) -{ - u64 spte; - int was_rmapped = is_rmap_pte(*shadow_pte); - struct page *page; - - pgprintk("%s: spte %llx access %x write_fault %d" - " user_fault %d gfn %lx\n", - __FUNCTION__, *shadow_pte, pt_access, - write_fault, user_fault, gfn); - - /* - * We don't set the accessed bit, since we sometimes want to see - * whether the guest actually used the pte (in order to detect - * demand paging). - */ - spte = PT_PRESENT_MASK | PT_DIRTY_MASK; - if (!dirty) - pte_access &= ~ACC_WRITE_MASK; - if (!(pte_access & ACC_EXEC_MASK)) - spte |= PT64_NX_MASK; - - page = gfn_to_page(vcpu->kvm, gfn); - - spte |= PT_PRESENT_MASK; - if (pte_access & ACC_USER_MASK) - spte |= PT_USER_MASK; - - if (is_error_page(page)) { - set_shadow_pte(shadow_pte, - shadow_trap_nonpresent_pte | PT_SHADOW_IO_MARK); - kvm_release_page_clean(page); - return; - } - - spte |= page_to_phys(page); - - if ((pte_access & ACC_WRITE_MASK) - || (write_fault && !is_write_protection(vcpu) && !user_fault)) { - struct kvm_mmu_page *shadow; - - spte |= PT_WRITABLE_MASK; - if (user_fault) { - mmu_unshadow(vcpu->kvm, gfn); - goto unshadowed; - } - - shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn); - if (shadow) { - pgprintk("%s: found shadow page for %lx, marking ro\n", - __FUNCTION__, gfn); - pte_access &= ~ACC_WRITE_MASK; - if (is_writeble_pte(spte)) { - spte &= ~PT_WRITABLE_MASK; - kvm_x86_ops->tlb_flush(vcpu); - } - if (write_fault) - *ptwrite = 1; - } - } - -unshadowed: - - if (pte_access & ACC_WRITE_MASK) - mark_page_dirty(vcpu->kvm, gfn); - - pgprintk("%s: setting spte %llx\n", __FUNCTION__, spte); - set_shadow_pte(shadow_pte, spte); - page_header_update_slot(vcpu->kvm, shadow_pte, gfn); - if (!was_rmapped) { - rmap_add(vcpu, shadow_pte, gfn); - if (!is_rmap_pte(*shadow_pte)) - kvm_release_page_clean(page); - } - else - kvm_release_page_clean(page); - if (!ptwrite || !*ptwrite) - vcpu->arch.last_pte_updated = shadow_pte; -} - -static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) -{ -} - -static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) -{ - int level = PT32E_ROOT_LEVEL; - hpa_t table_addr = vcpu->arch.mmu.root_hpa; - int pt_write = 0; - - for (; ; level--) { - u32 index = PT64_INDEX(v, level); - u64 *table; - - ASSERT(VALID_PAGE(table_addr)); - table = __va(table_addr); - - if (level == 1) { - mmu_set_spte(vcpu, &table[index], ACC_ALL, ACC_ALL, - 0, write, 1, &pt_write, gfn); - return pt_write || is_io_pte(table[index]); - } - - if (table[index] == shadow_trap_nonpresent_pte) { - struct kvm_mmu_page *new_table; - gfn_t pseudo_gfn; - - pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK) - >> PAGE_SHIFT; - new_table = kvm_mmu_get_page(vcpu, pseudo_gfn, - v, level - 1, - 1, ACC_ALL, &table[index], - NULL); - if (!new_table) { - pgprintk("nonpaging_map: ENOMEM\n"); - return -ENOMEM; - } - - table[index] = __pa(new_table->spt) | PT_PRESENT_MASK - | PT_WRITABLE_MASK | PT_USER_MASK; - } - table_addr = table[index] & PT64_BASE_ADDR_MASK; - } -} - -static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp) -{ - int i; - - for (i = 0; i < PT64_ENT_PER_PAGE; ++i) - sp->spt[i] = shadow_trap_nonpresent_pte; -} - -static void mmu_free_roots(struct kvm_vcpu *vcpu) -{ - int i; - struct kvm_mmu_page *sp; - - if (!VALID_PAGE(vcpu->arch.mmu.root_hpa)) - return; -#ifdef CONFIG_X86_64 - if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { - hpa_t root = vcpu->arch.mmu.root_hpa; - - sp = page_header(root); - --sp->root_count; - vcpu->arch.mmu.root_hpa = INVALID_PAGE; - return; - } -#endif - for (i = 0; i < 4; ++i) { - hpa_t root = vcpu->arch.mmu.pae_root[i]; - - if (root) { - root &= PT64_BASE_ADDR_MASK; - sp = page_header(root); - --sp->root_count; - } - vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; - } - vcpu->arch.mmu.root_hpa = INVALID_PAGE; -} - -static void mmu_alloc_roots(struct kvm_vcpu *vcpu) -{ - int i; - gfn_t root_gfn; - struct kvm_mmu_page *sp; - - root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT; - -#ifdef CONFIG_X86_64 - if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) { - hpa_t root = vcpu->arch.mmu.root_hpa; - - ASSERT(!VALID_PAGE(root)); - sp = kvm_mmu_get_page(vcpu, root_gfn, 0, - PT64_ROOT_LEVEL, 0, ACC_ALL, NULL, NULL); - root = __pa(sp->spt); - ++sp->root_count; - vcpu->arch.mmu.root_hpa = root; - return; - } -#endif - for (i = 0; i < 4; ++i) { - hpa_t root = vcpu->arch.mmu.pae_root[i]; - - ASSERT(!VALID_PAGE(root)); - if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) { - if (!is_present_pte(vcpu->arch.pdptrs[i])) { - vcpu->arch.mmu.pae_root[i] = 0; - continue; - } - root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT; - } else if (vcpu->arch.mmu.root_level == 0) - root_gfn = 0; - sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, - PT32_ROOT_LEVEL, !is_paging(vcpu), - ACC_ALL, NULL, NULL); - root = __pa(sp->spt); - ++sp->root_count; - vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK; - } - vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root); -} - -static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr) -{ - return vaddr; -} - -static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva, - u32 error_code) -{ - gfn_t gfn; - int r; - - pgprintk("%s: gva %lx error %x\n", __FUNCTION__, gva, error_code); - r = mmu_topup_memory_caches(vcpu); - if (r) - return r; - - ASSERT(vcpu); - ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa)); - - gfn = gva >> PAGE_SHIFT; - - return nonpaging_map(vcpu, gva & PAGE_MASK, - error_code & PFERR_WRITE_MASK, gfn); -} - -static void nonpaging_free(struct kvm_vcpu *vcpu) -{ - mmu_free_roots(vcpu); -} - -static int nonpaging_init_context(struct kvm_vcpu *vcpu) -{ - struct kvm_mmu *context = &vcpu->arch.mmu; - - context->new_cr3 = nonpaging_new_cr3; - context->page_fault = nonpaging_page_fault; - context->gva_to_gpa = nonpaging_gva_to_gpa; - context->free = nonpaging_free; - context->prefetch_page = nonpaging_prefetch_page; - context->root_level = 0; - context->shadow_root_level = PT32E_ROOT_LEVEL; - context->root_hpa = INVALID_PAGE; - return 0; -} - -void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu) -{ - ++vcpu->stat.tlb_flush; - kvm_x86_ops->tlb_flush(vcpu); -} - -static void paging_new_cr3(struct kvm_vcpu *vcpu) -{ - pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3); - mmu_free_roots(vcpu); -} - -static void inject_page_fault(struct kvm_vcpu *vcpu, - u64 addr, - u32 err_code) -{ - kvm_inject_page_fault(vcpu, addr, err_code); -} - -static void paging_free(struct kvm_vcpu *vcpu) -{ - nonpaging_free(vcpu); -} - -#define PTTYPE 64 -#include "paging_tmpl.h" -#undef PTTYPE - -#define PTTYPE 32 -#include "paging_tmpl.h" -#undef PTTYPE - -static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level) -{ - struct kvm_mmu *context = &vcpu->arch.mmu; - - ASSERT(is_pae(vcpu)); - context->new_cr3 = paging_new_cr3; - context->page_fault = paging64_page_fault; - context->gva_to_gpa = paging64_gva_to_gpa; - context->prefetch_page = paging64_prefetch_page; - context->free = paging_free; - context->root_level = level; - context->shadow_root_level = level; - context->root_hpa = INVALID_PAGE; - return 0; -} - -static int paging64_init_context(struct kvm_vcpu *vcpu) -{ - return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL); -} - -static int paging32_init_context(struct kvm_vcpu *vcpu) -{ - struct kvm_mmu *context = &vcpu->arch.mmu; - - context->new_cr3 = paging_new_cr3; - context->page_fault = paging32_page_fault; - context->gva_to_gpa = paging32_gva_to_gpa; - context->free = paging_free; - context->prefetch_page = paging32_prefetch_page; - context->root_level = PT32_ROOT_LEVEL; - context->shadow_root_level = PT32E_ROOT_LEVEL; - context->root_hpa = INVALID_PAGE; - return 0; -} - -static int paging32E_init_context(struct kvm_vcpu *vcpu) -{ - return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL); -} - -static int init_kvm_mmu(struct kvm_vcpu *vcpu) -{ - ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); - - if (!is_paging(vcpu)) - return nonpaging_init_context(vcpu); - else if (is_long_mode(vcpu)) - return paging64_init_context(vcpu); - else if (is_pae(vcpu)) - return paging32E_init_context(vcpu); - else - return paging32_init_context(vcpu); -} - -static void destroy_kvm_mmu(struct kvm_vcpu *vcpu) -{ - ASSERT(vcpu); - if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) { - vcpu->arch.mmu.free(vcpu); - vcpu->arch.mmu.root_hpa = INVALID_PAGE; - } -} - -int kvm_mmu_reset_context(struct kvm_vcpu *vcpu) -{ - destroy_kvm_mmu(vcpu); - return init_kvm_mmu(vcpu); -} -EXPORT_SYMBOL_GPL(kvm_mmu_reset_context); - -int kvm_mmu_load(struct kvm_vcpu *vcpu) -{ - int r; - - mutex_lock(&vcpu->kvm->lock); - r = mmu_topup_memory_caches(vcpu); - if (r) - goto out; - mmu_alloc_roots(vcpu); - kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa); - kvm_mmu_flush_tlb(vcpu); -out: - mutex_unlock(&vcpu->kvm->lock); - return r; -} -EXPORT_SYMBOL_GPL(kvm_mmu_load); - -void kvm_mmu_unload(struct kvm_vcpu *vcpu) -{ - mmu_free_roots(vcpu); -} - -static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp, - u64 *spte) -{ - u64 pte; - struct kvm_mmu_page *child; - - pte = *spte; - if (is_shadow_present_pte(pte)) { - if (sp->role.level == PT_PAGE_TABLE_LEVEL) - rmap_remove(vcpu->kvm, spte); - else { - child = page_header(pte & PT64_BASE_ADDR_MASK); - mmu_page_remove_parent_pte(child, spte); - } - } - set_shadow_pte(spte, shadow_trap_nonpresent_pte); -} - -static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp, - u64 *spte, - const void *new, int bytes, - int offset_in_pte) -{ - if (sp->role.level != PT_PAGE_TABLE_LEVEL) { - ++vcpu->kvm->stat.mmu_pde_zapped; - return; - } - - ++vcpu->kvm->stat.mmu_pte_updated; - if (sp->role.glevels == PT32_ROOT_LEVEL) - paging32_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte); - else - paging64_update_pte(vcpu, sp, spte, new, bytes, offset_in_pte); -} - -static bool need_remote_flush(u64 old, u64 new) -{ - if (!is_shadow_present_pte(old)) - return false; - if (!is_shadow_present_pte(new)) - return true; - if ((old ^ new) & PT64_BASE_ADDR_MASK) - return true; - old ^= PT64_NX_MASK; - new ^= PT64_NX_MASK; - return (old & ~new & PT64_PERM_MASK) != 0; -} - -static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new) -{ - if (need_remote_flush(old, new)) - kvm_flush_remote_tlbs(vcpu->kvm); - else - kvm_mmu_flush_tlb(vcpu); -} - -static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu) -{ - u64 *spte = vcpu->arch.last_pte_updated; - - return !!(spte && (*spte & PT_ACCESSED_MASK)); -} - -void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, - const u8 *new, int bytes) -{ - gfn_t gfn = gpa >> PAGE_SHIFT; - struct kvm_mmu_page *sp; - struct hlist_node *node, *n; - struct hlist_head *bucket; - unsigned index; - u64 entry; - u64 *spte; - unsigned offset = offset_in_page(gpa); - unsigned pte_size; - unsigned page_offset; - unsigned misaligned; - unsigned quadrant; - int level; - int flooded = 0; - int npte; - - pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes); - ++vcpu->kvm->stat.mmu_pte_write; - kvm_mmu_audit(vcpu, "pre pte write"); - if (gfn == vcpu->arch.last_pt_write_gfn - && !last_updated_pte_accessed(vcpu)) { - ++vcpu->arch.last_pt_write_count; - if (vcpu->arch.last_pt_write_count >= 3) - flooded = 1; - } else { - vcpu->arch.last_pt_write_gfn = gfn; - vcpu->arch.last_pt_write_count = 1; - vcpu->arch.last_pte_updated = NULL; - } - index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES; - bucket = &vcpu->kvm->arch.mmu_page_hash[index]; - hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) { - if (sp->gfn != gfn || sp->role.metaphysical) - continue; - pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; - misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); - misaligned |= bytes < 4; - if (misaligned || flooded) { - /* - * Misaligned accesses are too much trouble to fix - * up; also, they usually indicate a page is not used - * as a page table. - * - * If we're seeing too many writes to a page, - * it may no longer be a page table, or we may be - * forking, in which case it is better to unmap the - * page. - */ - pgprintk("misaligned: gpa %llx bytes %d role %x\n", - gpa, bytes, sp->role.word); - kvm_mmu_zap_page(vcpu->kvm, sp); - ++vcpu->kvm->stat.mmu_flooded; - continue; - } - page_offset = offset; - level = sp->role.level; - npte = 1; - if (sp->role.glevels == PT32_ROOT_LEVEL) { - page_offset <<= 1; /* 32->64 */ - /* - * A 32-bit pde maps 4MB while the shadow pdes map - * only 2MB. So we need to double the offset again - * and zap two pdes instead of one. - */ - if (level == PT32_ROOT_LEVEL) { - page_offset &= ~7; /* kill rounding error */ - page_offset <<= 1; - npte = 2; - } - quadrant = page_offset >> PAGE_SHIFT; - page_offset &= ~PAGE_MASK; - if (quadrant != sp->role.quadrant) - continue; - } - spte = &sp->spt[page_offset / sizeof(*spte)]; - while (npte--) { - entry = *spte; - mmu_pte_write_zap_pte(vcpu, sp, spte); - mmu_pte_write_new_pte(vcpu, sp, spte, new, bytes, - page_offset & (pte_size - 1)); - mmu_pte_write_flush_tlb(vcpu, entry, *spte); - ++spte; - } - } - kvm_mmu_audit(vcpu, "post pte write"); -} - -int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) -{ - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva); - - return kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); -} - -void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) -{ - while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) { - struct kvm_mmu_page *sp; - - sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev, - struct kvm_mmu_page, link); - kvm_mmu_zap_page(vcpu->kvm, sp); - ++vcpu->kvm->stat.mmu_recycled; - } -} - -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code) -{ - int r; - enum emulation_result er; - - mutex_lock(&vcpu->kvm->lock); - r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code); - if (r < 0) - goto out; - - if (!r) { - r = 1; - goto out; - } - - r = mmu_topup_memory_caches(vcpu); - if (r) - goto out; - - er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0); - mutex_unlock(&vcpu->kvm->lock); - - switch (er) { - case EMULATE_DONE: - return 1; - case EMULATE_DO_MMIO: - ++vcpu->stat.mmio_exits; - return 0; - case EMULATE_FAIL: - kvm_report_emulation_failure(vcpu, "pagetable"); - return 1; - default: - BUG(); - } -out: - mutex_unlock(&vcpu->kvm->lock); - return r; -} -EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); - -static void free_mmu_pages(struct kvm_vcpu *vcpu) -{ - struct kvm_mmu_page *sp; - - while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) { - sp = container_of(vcpu->kvm->arch.active_mmu_pages.next, - struct kvm_mmu_page, link); - kvm_mmu_zap_page(vcpu->kvm, sp); - } - free_page((unsigned long)vcpu->arch.mmu.pae_root); -} - -static int alloc_mmu_pages(struct kvm_vcpu *vcpu) -{ - struct page *page; - int i; - - ASSERT(vcpu); - - if (vcpu->kvm->arch.n_requested_mmu_pages) - vcpu->kvm->arch.n_free_mmu_pages = - vcpu->kvm->arch.n_requested_mmu_pages; - else - vcpu->kvm->arch.n_free_mmu_pages = - vcpu->kvm->arch.n_alloc_mmu_pages; - /* - * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64. - * Therefore we need to allocate shadow page tables in the first - * 4GB of memory, which happens to fit the DMA32 zone. - */ - page = alloc_page(GFP_KERNEL | __GFP_DMA32); - if (!page) - goto error_1; - vcpu->arch.mmu.pae_root = page_address(page); - for (i = 0; i < 4; ++i) - vcpu->arch.mmu.pae_root[i] = INVALID_PAGE; - - return 0; - -error_1: - free_mmu_pages(vcpu); - return -ENOMEM; -} - -int kvm_mmu_create(struct kvm_vcpu *vcpu) -{ - ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); - - return alloc_mmu_pages(vcpu); -} - -int kvm_mmu_setup(struct kvm_vcpu *vcpu) -{ - ASSERT(vcpu); - ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); - - return init_kvm_mmu(vcpu); -} - -void kvm_mmu_destroy(struct kvm_vcpu *vcpu) -{ - ASSERT(vcpu); - - destroy_kvm_mmu(vcpu); - free_mmu_pages(vcpu); - mmu_free_memory_caches(vcpu); -} - -void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot) -{ - struct kvm_mmu_page *sp; - - list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) { - int i; - u64 *pt; - - if (!test_bit(slot, &sp->slot_bitmap)) - continue; - - pt = sp->spt; - for (i = 0; i < PT64_ENT_PER_PAGE; ++i) - /* avoid RMW */ - if (pt[i] & PT_WRITABLE_MASK) - pt[i] &= ~PT_WRITABLE_MASK; - } -} - -void kvm_mmu_zap_all(struct kvm *kvm) -{ - struct kvm_mmu_page *sp, *node; - - list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) - kvm_mmu_zap_page(kvm, sp); - - kvm_flush_remote_tlbs(kvm); -} - -void kvm_mmu_module_exit(void) -{ - if (pte_chain_cache) - kmem_cache_destroy(pte_chain_cache); - if (rmap_desc_cache) - kmem_cache_destroy(rmap_desc_cache); - if (mmu_page_header_cache) - kmem_cache_destroy(mmu_page_header_cache); -} - -int kvm_mmu_module_init(void) -{ - pte_chain_cache = kmem_cache_create("kvm_pte_chain", - sizeof(struct kvm_pte_chain), - 0, 0, NULL); - if (!pte_chain_cache) - goto nomem; - rmap_desc_cache = kmem_cache_create("kvm_rmap_desc", - sizeof(struct kvm_rmap_desc), - 0, 0, NULL); - if (!rmap_desc_cache) - goto nomem; - - mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header", - sizeof(struct kvm_mmu_page), - 0, 0, NULL); - if (!mmu_page_header_cache) - goto nomem; - - return 0; - -nomem: - kvm_mmu_module_exit(); - return -ENOMEM; -} - -/* - * Caculate mmu pages needed for kvm. - */ -unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm) -{ - int i; - unsigned int nr_mmu_pages; - unsigned int nr_pages = 0; - - for (i = 0; i < kvm->nmemslots; i++) - nr_pages += kvm->memslots[i].npages; - - nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000; - nr_mmu_pages = max(nr_mmu_pages, - (unsigned int) KVM_MIN_ALLOC_MMU_PAGES); - - return nr_mmu_pages; -} - -#ifdef AUDIT - -static const char *audit_msg; - -static gva_t canonicalize(gva_t gva) -{ -#ifdef CONFIG_X86_64 - gva = (long long)(gva << 16) >> 16; -#endif - return gva; -} - -static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte, - gva_t va, int level) -{ - u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK); - int i; - gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1)); - - for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) { - u64 ent = pt[i]; - - if (ent == shadow_trap_nonpresent_pte) - continue; - - va = canonicalize(va); - if (level > 1) { - if (ent == shadow_notrap_nonpresent_pte) - printk(KERN_ERR "audit: (%s) nontrapping pte" - " in nonleaf level: levels %d gva %lx" - " level %d pte %llx\n", audit_msg, - vcpu->arch.mmu.root_level, va, level, ent); - - audit_mappings_page(vcpu, ent, va, level - 1); - } else { - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va); - struct page *page = gpa_to_page(vcpu, gpa); - hpa_t hpa = page_to_phys(page); - - if (is_shadow_present_pte(ent) - && (ent & PT64_BASE_ADDR_MASK) != hpa) - printk(KERN_ERR "xx audit error: (%s) levels %d" - " gva %lx gpa %llx hpa %llx ent %llx %d\n", - audit_msg, vcpu->arch.mmu.root_level, - va, gpa, hpa, ent, - is_shadow_present_pte(ent)); - else if (ent == shadow_notrap_nonpresent_pte - && !is_error_hpa(hpa)) - printk(KERN_ERR "audit: (%s) notrap shadow," - " valid guest gva %lx\n", audit_msg, va); - kvm_release_page_clean(page); - - } - } -} - -static void audit_mappings(struct kvm_vcpu *vcpu) -{ - unsigned i; - - if (vcpu->arch.mmu.root_level == 4) - audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4); - else - for (i = 0; i < 4; ++i) - if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK) - audit_mappings_page(vcpu, - vcpu->arch.mmu.pae_root[i], - i << 30, - 2); -} - -static int count_rmaps(struct kvm_vcpu *vcpu) -{ - int nmaps = 0; - int i, j, k; - - for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { - struct kvm_memory_slot *m = &vcpu->kvm->memslots[i]; - struct kvm_rmap_desc *d; - - for (j = 0; j < m->npages; ++j) { - unsigned long *rmapp = &m->rmap[j]; - - if (!*rmapp) - continue; - if (!(*rmapp & 1)) { - ++nmaps; - continue; - } - d = (struct kvm_rmap_desc *)(*rmapp & ~1ul); - while (d) { - for (k = 0; k < RMAP_EXT; ++k) - if (d->shadow_ptes[k]) - ++nmaps; - else - break; - d = d->more; - } - } - } - return nmaps; -} - -static int count_writable_mappings(struct kvm_vcpu *vcpu) -{ - int nmaps = 0; - struct kvm_mmu_page *sp; - int i; - - list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { - u64 *pt = sp->spt; - - if (sp->role.level != PT_PAGE_TABLE_LEVEL) - continue; - - for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { - u64 ent = pt[i]; - - if (!(ent & PT_PRESENT_MASK)) - continue; - if (!(ent & PT_WRITABLE_MASK)) - continue; - ++nmaps; - } - } - return nmaps; -} - -static void audit_rmap(struct kvm_vcpu *vcpu) -{ - int n_rmap = count_rmaps(vcpu); - int n_actual = count_writable_mappings(vcpu); - - if (n_rmap != n_actual) - printk(KERN_ERR "%s: (%s) rmap %d actual %d\n", - __FUNCTION__, audit_msg, n_rmap, n_actual); -} - -static void audit_write_protection(struct kvm_vcpu *vcpu) -{ - struct kvm_mmu_page *sp; - struct kvm_memory_slot *slot; - unsigned long *rmapp; - gfn_t gfn; - - list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { - if (sp->role.metaphysical) - continue; - - slot = gfn_to_memslot(vcpu->kvm, sp->gfn); - gfn = unalias_gfn(vcpu->kvm, sp->gfn); - rmapp = &slot->rmap[gfn - slot->base_gfn]; - if (*rmapp) - printk(KERN_ERR "%s: (%s) shadow page has writable" - " mappings: gfn %lx role %x\n", - __FUNCTION__, audit_msg, sp->gfn, - sp->role.word); - } -} - -static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) -{ - int olddbg = dbg; - - dbg = 0; - audit_msg = msg; - audit_rmap(vcpu); - audit_write_protection(vcpu); - audit_mappings(vcpu); - dbg = olddbg; -} - -#endif diff --git a/drivers/kvm/mmu.h b/drivers/kvm/mmu.h deleted file mode 100644 index cbfc272262df..000000000000 --- a/drivers/kvm/mmu.h +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef __KVM_X86_MMU_H -#define __KVM_X86_MMU_H - -#include "kvm.h" - -static inline void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu) -{ - if (unlikely(vcpu->kvm->arch.n_free_mmu_pages < KVM_MIN_FREE_MMU_PAGES)) - __kvm_mmu_free_some_pages(vcpu); -} - -static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) -{ - if (likely(vcpu->arch.mmu.root_hpa != INVALID_PAGE)) - return 0; - - return kvm_mmu_load(vcpu); -} - -static inline int is_long_mode(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - return vcpu->arch.shadow_efer & EFER_LME; -#else - return 0; -#endif -} - -static inline int is_pae(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr4 & X86_CR4_PAE; -} - -static inline int is_pse(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr4 & X86_CR4_PSE; -} - -static inline int is_paging(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.cr0 & X86_CR0_PG; -} - -#endif diff --git a/drivers/kvm/paging_tmpl.h b/drivers/kvm/paging_tmpl.h deleted file mode 100644 index 56b88f7e83ef..000000000000 --- a/drivers/kvm/paging_tmpl.h +++ /dev/null @@ -1,461 +0,0 @@ -/* - * Kernel-based Virtual Machine driver for Linux - * - * This module enables machines with Intel VT-x extensions to run virtual - * machines without emulation or binary translation. - * - * MMU support - * - * Copyright (C) 2006 Qumranet, Inc. - * - * Authors: - * Yaniv Kamay - * Avi Kivity - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - * - */ - -/* - * We need the mmu code to access both 32-bit and 64-bit guest ptes, - * so the code in this file is compiled twice, once per pte size. - */ - -#if PTTYPE == 64 - #define pt_element_t u64 - #define guest_walker guest_walker64 - #define FNAME(name) paging##64_##name - #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK - #define PT_DIR_BASE_ADDR_MASK PT64_DIR_BASE_ADDR_MASK - #define PT_INDEX(addr, level) PT64_INDEX(addr, level) - #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) - #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level) - #define PT_LEVEL_BITS PT64_LEVEL_BITS - #ifdef CONFIG_X86_64 - #define PT_MAX_FULL_LEVELS 4 - #define CMPXCHG cmpxchg - #else - #define CMPXCHG cmpxchg64 - #define PT_MAX_FULL_LEVELS 2 - #endif -#elif PTTYPE == 32 - #define pt_element_t u32 - #define guest_walker guest_walker32 - #define FNAME(name) paging##32_##name - #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK - #define PT_DIR_BASE_ADDR_MASK PT32_DIR_BASE_ADDR_MASK - #define PT_INDEX(addr, level) PT32_INDEX(addr, level) - #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level) - #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level) - #define PT_LEVEL_BITS PT32_LEVEL_BITS - #define PT_MAX_FULL_LEVELS 2 - #define CMPXCHG cmpxchg -#else - #error Invalid PTTYPE value -#endif - -#define gpte_to_gfn FNAME(gpte_to_gfn) -#define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde) - -/* - * The guest_walker structure emulates the behavior of the hardware page - * table walker. - */ -struct guest_walker { - int level; - gfn_t table_gfn[PT_MAX_FULL_LEVELS]; - pt_element_t ptes[PT_MAX_FULL_LEVELS]; - gpa_t pte_gpa[PT_MAX_FULL_LEVELS]; - unsigned pt_access; - unsigned pte_access; - gfn_t gfn; - u32 error_code; -}; - -static gfn_t gpte_to_gfn(pt_element_t gpte) -{ - return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; -} - -static gfn_t gpte_to_gfn_pde(pt_element_t gpte) -{ - return (gpte & PT_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; -} - -static bool FNAME(cmpxchg_gpte)(struct kvm *kvm, - gfn_t table_gfn, unsigned index, - pt_element_t orig_pte, pt_element_t new_pte) -{ - pt_element_t ret; - pt_element_t *table; - struct page *page; - - page = gfn_to_page(kvm, table_gfn); - table = kmap_atomic(page, KM_USER0); - - ret = CMPXCHG(&table[index], orig_pte, new_pte); - - kunmap_atomic(table, KM_USER0); - - kvm_release_page_dirty(page); - - return (ret != orig_pte); -} - -static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte) -{ - unsigned access; - - access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK; -#if PTTYPE == 64 - if (is_nx(vcpu)) - access &= ~(gpte >> PT64_NX_SHIFT); -#endif - return access; -} - -/* - * Fetch a guest pte for a guest virtual address - */ -static int FNAME(walk_addr)(struct guest_walker *walker, - struct kvm_vcpu *vcpu, gva_t addr, - int write_fault, int user_fault, int fetch_fault) -{ - pt_element_t pte; - gfn_t table_gfn; - unsigned index, pt_access, pte_access; - gpa_t pte_gpa; - - pgprintk("%s: addr %lx\n", __FUNCTION__, addr); -walk: - walker->level = vcpu->arch.mmu.root_level; - pte = vcpu->arch.cr3; -#if PTTYPE == 64 - if (!is_long_mode(vcpu)) { - pte = vcpu->arch.pdptrs[(addr >> 30) & 3]; - if (!is_present_pte(pte)) - goto not_present; - --walker->level; - } -#endif - ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || - (vcpu->cr3 & CR3_NONPAE_RESERVED_BITS) == 0); - - pt_access = ACC_ALL; - - for (;;) { - index = PT_INDEX(addr, walker->level); - - table_gfn = gpte_to_gfn(pte); - pte_gpa = gfn_to_gpa(table_gfn); - pte_gpa += index * sizeof(pt_element_t); - walker->table_gfn[walker->level - 1] = table_gfn; - walker->pte_gpa[walker->level - 1] = pte_gpa; - pgprintk("%s: table_gfn[%d] %lx\n", __FUNCTION__, - walker->level - 1, table_gfn); - - kvm_read_guest(vcpu->kvm, pte_gpa, &pte, sizeof(pte)); - - if (!is_present_pte(pte)) - goto not_present; - - if (write_fault && !is_writeble_pte(pte)) - if (user_fault || is_write_protection(vcpu)) - goto access_error; - - if (user_fault && !(pte & PT_USER_MASK)) - goto access_error; - -#if PTTYPE == 64 - if (fetch_fault && is_nx(vcpu) && (pte & PT64_NX_MASK)) - goto access_error; -#endif - - if (!(pte & PT_ACCESSED_MASK)) { - mark_page_dirty(vcpu->kvm, table_gfn); - if (FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, - index, pte, pte|PT_ACCESSED_MASK)) - goto walk; - pte |= PT_ACCESSED_MASK; - } - - pte_access = pt_access & FNAME(gpte_access)(vcpu, pte); - - walker->ptes[walker->level - 1] = pte; - - if (walker->level == PT_PAGE_TABLE_LEVEL) { - walker->gfn = gpte_to_gfn(pte); - break; - } - - if (walker->level == PT_DIRECTORY_LEVEL - && (pte & PT_PAGE_SIZE_MASK) - && (PTTYPE == 64 || is_pse(vcpu))) { - walker->gfn = gpte_to_gfn_pde(pte); - walker->gfn += PT_INDEX(addr, PT_PAGE_TABLE_LEVEL); - if (PTTYPE == 32 && is_cpuid_PSE36()) - walker->gfn += pse36_gfn_delta(pte); - break; - } - - pt_access = pte_access; - --walker->level; - } - - if (write_fault && !is_dirty_pte(pte)) { - bool ret; - - mark_page_dirty(vcpu->kvm, table_gfn); - ret = FNAME(cmpxchg_gpte)(vcpu->kvm, table_gfn, index, pte, - pte|PT_DIRTY_MASK); - if (ret) - goto walk; - pte |= PT_DIRTY_MASK; - kvm_mmu_pte_write(vcpu, pte_gpa, (u8 *)&pte, sizeof(pte)); - walker->ptes[walker->level - 1] = pte; - } - - walker->pt_access = pt_access; - walker->pte_access = pte_access; - pgprintk("%s: pte %llx pte_access %x pt_access %x\n", - __FUNCTION__, (u64)pte, pt_access, pte_access); - return 1; - -not_present: - walker->error_code = 0; - goto err; - -access_error: - walker->error_code = PFERR_PRESENT_MASK; - -err: - if (write_fault) - walker->error_code |= PFERR_WRITE_MASK; - if (user_fault) - walker->error_code |= PFERR_USER_MASK; - if (fetch_fault) - walker->error_code |= PFERR_FETCH_MASK; - return 0; -} - -static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, - u64 *spte, const void *pte, int bytes, - int offset_in_pte) -{ - pt_element_t gpte; - unsigned pte_access; - - gpte = *(const pt_element_t *)pte; - if (~gpte & (PT_PRESENT_MASK | PT_ACCESSED_MASK)) { - if (!offset_in_pte && !is_present_pte(gpte)) - set_shadow_pte(spte, shadow_notrap_nonpresent_pte); - return; - } - if (bytes < sizeof(pt_element_t)) - return; - pgprintk("%s: gpte %llx spte %p\n", __FUNCTION__, (u64)gpte, spte); - pte_access = page->role.access & FNAME(gpte_access)(vcpu, gpte); - mmu_set_spte(vcpu, spte, page->role.access, pte_access, 0, 0, - gpte & PT_DIRTY_MASK, NULL, gpte_to_gfn(gpte)); -} - -/* - * Fetch a shadow pte for a specific level in the paging hierarchy. - */ -static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, - struct guest_walker *walker, - int user_fault, int write_fault, int *ptwrite) -{ - hpa_t shadow_addr; - int level; - u64 *shadow_ent; - unsigned access = walker->pt_access; - - if (!is_present_pte(walker->ptes[walker->level - 1])) - return NULL; - - shadow_addr = vcpu->arch.mmu.root_hpa; - level = vcpu->arch.mmu.shadow_root_level; - if (level == PT32E_ROOT_LEVEL) { - shadow_addr = vcpu->arch.mmu.pae_root[(addr >> 30) & 3]; - shadow_addr &= PT64_BASE_ADDR_MASK; - --level; - } - - for (; ; level--) { - u32 index = SHADOW_PT_INDEX(addr, level); - struct kvm_mmu_page *shadow_page; - u64 shadow_pte; - int metaphysical; - gfn_t table_gfn; - bool new_page = 0; - - shadow_ent = ((u64 *)__va(shadow_addr)) + index; - if (is_shadow_present_pte(*shadow_ent)) { - if (level == PT_PAGE_TABLE_LEVEL) - break; - shadow_addr = *shadow_ent & PT64_BASE_ADDR_MASK; - continue; - } - - if (level == PT_PAGE_TABLE_LEVEL) - break; - - if (level - 1 == PT_PAGE_TABLE_LEVEL - && walker->level == PT_DIRECTORY_LEVEL) { - metaphysical = 1; - if (!is_dirty_pte(walker->ptes[level - 1])) - access &= ~ACC_WRITE_MASK; - table_gfn = gpte_to_gfn(walker->ptes[level - 1]); - } else { - metaphysical = 0; - table_gfn = walker->table_gfn[level - 2]; - } - shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, - metaphysical, access, - shadow_ent, &new_page); - if (new_page && !metaphysical) { - pt_element_t curr_pte; - kvm_read_guest(vcpu->kvm, walker->pte_gpa[level - 2], - &curr_pte, sizeof(curr_pte)); - if (curr_pte != walker->ptes[level - 2]) - return NULL; - } - shadow_addr = __pa(shadow_page->spt); - shadow_pte = shadow_addr | PT_PRESENT_MASK | PT_ACCESSED_MASK - | PT_WRITABLE_MASK | PT_USER_MASK; - *shadow_ent = shadow_pte; - } - - mmu_set_spte(vcpu, shadow_ent, access, walker->pte_access & access, - user_fault, write_fault, - walker->ptes[walker->level-1] & PT_DIRTY_MASK, - ptwrite, walker->gfn); - - return shadow_ent; -} - -/* - * Page fault handler. There are several causes for a page fault: - * - there is no shadow pte for the guest pte - * - write access through a shadow pte marked read only so that we can set - * the dirty bit - * - write access to a shadow pte marked read only so we can update the page - * dirty bitmap, when userspace requests it - * - mmio access; in this case we will never install a present shadow pte - * - normal guest page fault due to the guest pte marked not present, not - * writable, or not executable - * - * Returns: 1 if we need to emulate the instruction, 0 otherwise, or - * a negative value on error. - */ -static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr, - u32 error_code) -{ - int write_fault = error_code & PFERR_WRITE_MASK; - int user_fault = error_code & PFERR_USER_MASK; - int fetch_fault = error_code & PFERR_FETCH_MASK; - struct guest_walker walker; - u64 *shadow_pte; - int write_pt = 0; - int r; - - pgprintk("%s: addr %lx err %x\n", __FUNCTION__, addr, error_code); - kvm_mmu_audit(vcpu, "pre page fault"); - - r = mmu_topup_memory_caches(vcpu); - if (r) - return r; - - /* - * Look up the shadow pte for the faulting address. - */ - r = FNAME(walk_addr)(&walker, vcpu, addr, write_fault, user_fault, - fetch_fault); - - /* - * The page is not mapped by the guest. Let the guest handle it. - */ - if (!r) { - pgprintk("%s: guest page fault\n", __FUNCTION__); - inject_page_fault(vcpu, addr, walker.error_code); - vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ - return 0; - } - - shadow_pte = FNAME(fetch)(vcpu, addr, &walker, user_fault, write_fault, - &write_pt); - pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __FUNCTION__, - shadow_pte, *shadow_pte, write_pt); - - if (!write_pt) - vcpu->arch.last_pt_write_count = 0; /* reset fork detector */ - - /* - * mmio: emulate if accessible, otherwise its a guest fault. - */ - if (shadow_pte && is_io_pte(*shadow_pte)) - return 1; - - ++vcpu->stat.pf_fixed; - kvm_mmu_audit(vcpu, "post page fault (fixed)"); - - return write_pt; -} - -static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t vaddr) -{ - struct guest_walker walker; - gpa_t gpa = UNMAPPED_GVA; - int r; - - r = FNAME(walk_addr)(&walker, vcpu, vaddr, 0, 0, 0); - - if (r) { - gpa = gfn_to_gpa(walker.gfn); - gpa |= vaddr & ~PAGE_MASK; - } - - return gpa; -} - -static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *sp) -{ - int i, offset = 0; - pt_element_t *gpt; - struct page *page; - - if (sp->role.metaphysical - || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) { - nonpaging_prefetch_page(vcpu, sp); - return; - } - - if (PTTYPE == 32) - offset = sp->role.quadrant << PT64_LEVEL_BITS; - page = gfn_to_page(vcpu->kvm, sp->gfn); - gpt = kmap_atomic(page, KM_USER0); - for (i = 0; i < PT64_ENT_PER_PAGE; ++i) - if (is_present_pte(gpt[offset + i])) - sp->spt[i] = shadow_trap_nonpresent_pte; - else - sp->spt[i] = shadow_notrap_nonpresent_pte; - kunmap_atomic(gpt, KM_USER0); - kvm_release_page_clean(page); -} - -#undef pt_element_t -#undef guest_walker -#undef FNAME -#undef PT_BASE_ADDR_MASK -#undef PT_INDEX -#undef SHADOW_PT_INDEX -#undef PT_LEVEL_MASK -#undef PT_DIR_BASE_ADDR_MASK -#undef PT_LEVEL_BITS -#undef PT_MAX_FULL_LEVELS -#undef gpte_to_gfn -#undef gpte_to_gfn_pde -#undef CMPXCHG diff --git a/drivers/kvm/segment_descriptor.h b/drivers/kvm/segment_descriptor.h deleted file mode 100644 index 56fc4c873389..000000000000 --- a/drivers/kvm/segment_descriptor.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef __SEGMENT_DESCRIPTOR_H -#define __SEGMENT_DESCRIPTOR_H - -struct segment_descriptor { - u16 limit_low; - u16 base_low; - u8 base_mid; - u8 type : 4; - u8 system : 1; - u8 dpl : 2; - u8 present : 1; - u8 limit_high : 4; - u8 avl : 1; - u8 long_mode : 1; - u8 default_op : 1; - u8 granularity : 1; - u8 base_high; -} __attribute__((packed)); - -#ifdef CONFIG_X86_64 -/* LDT or TSS descriptor in the GDT. 16 bytes. */ -struct segment_descriptor_64 { - struct segment_descriptor s; - u32 base_higher; - u32 pad_zero; -}; - -#endif -#endif diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c deleted file mode 100644 index e606f6d18669..000000000000 --- a/drivers/kvm/svm.c +++ /dev/null @@ -1,1725 +0,0 @@ -/* - * Kernel-based Virtual Machine driver for Linux - * - * AMD SVM support - * - * Copyright (C) 2006 Qumranet, Inc. - * - * Authors: - * Yaniv Kamay - * Avi Kivity - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - * - */ -#include "x86.h" -#include "kvm_svm.h" -#include "x86_emulate.h" -#include "irq.h" -#include "mmu.h" - -#include -#include -#include -#include -#include - -#include - -MODULE_AUTHOR("Qumranet"); -MODULE_LICENSE("GPL"); - -#define IOPM_ALLOC_ORDER 2 -#define MSRPM_ALLOC_ORDER 1 - -#define DB_VECTOR 1 -#define UD_VECTOR 6 -#define GP_VECTOR 13 - -#define DR7_GD_MASK (1 << 13) -#define DR6_BD_MASK (1 << 13) - -#define SEG_TYPE_LDT 2 -#define SEG_TYPE_BUSY_TSS16 3 - -#define SVM_FEATURE_NPT (1 << 0) -#define SVM_FEATURE_LBRV (1 << 1) -#define SVM_DEATURE_SVML (1 << 2) - -static void kvm_reput_irq(struct vcpu_svm *svm); - -static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) -{ - return container_of(vcpu, struct vcpu_svm, vcpu); -} - -unsigned long iopm_base; -unsigned long msrpm_base; - -struct kvm_ldttss_desc { - u16 limit0; - u16 base0; - unsigned base1 : 8, type : 5, dpl : 2, p : 1; - unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8; - u32 base3; - u32 zero1; -} __attribute__((packed)); - -struct svm_cpu_data { - int cpu; - - u64 asid_generation; - u32 max_asid; - u32 next_asid; - struct kvm_ldttss_desc *tss_desc; - - struct page *save_area; -}; - -static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); -static uint32_t svm_features; - -struct svm_init_data { - int cpu; - int r; -}; - -static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; - -#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) -#define MSRS_RANGE_SIZE 2048 -#define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2) - -#define MAX_INST_SIZE 15 - -static inline u32 svm_has(u32 feat) -{ - return svm_features & feat; -} - -static inline u8 pop_irq(struct kvm_vcpu *vcpu) -{ - int word_index = __ffs(vcpu->arch.irq_summary); - int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); - int irq = word_index * BITS_PER_LONG + bit_index; - - clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); - if (!vcpu->arch.irq_pending[word_index]) - clear_bit(word_index, &vcpu->arch.irq_summary); - return irq; -} - -static inline void push_irq(struct kvm_vcpu *vcpu, u8 irq) -{ - set_bit(irq, vcpu->arch.irq_pending); - set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary); -} - -static inline void clgi(void) -{ - asm volatile (SVM_CLGI); -} - -static inline void stgi(void) -{ - asm volatile (SVM_STGI); -} - -static inline void invlpga(unsigned long addr, u32 asid) -{ - asm volatile (SVM_INVLPGA :: "a"(addr), "c"(asid)); -} - -static inline unsigned long kvm_read_cr2(void) -{ - unsigned long cr2; - - asm volatile ("mov %%cr2, %0" : "=r" (cr2)); - return cr2; -} - -static inline void kvm_write_cr2(unsigned long val) -{ - asm volatile ("mov %0, %%cr2" :: "r" (val)); -} - -static inline unsigned long read_dr6(void) -{ - unsigned long dr6; - - asm volatile ("mov %%dr6, %0" : "=r" (dr6)); - return dr6; -} - -static inline void write_dr6(unsigned long val) -{ - asm volatile ("mov %0, %%dr6" :: "r" (val)); -} - -static inline unsigned long read_dr7(void) -{ - unsigned long dr7; - - asm volatile ("mov %%dr7, %0" : "=r" (dr7)); - return dr7; -} - -static inline void write_dr7(unsigned long val) -{ - asm volatile ("mov %0, %%dr7" :: "r" (val)); -} - -static inline void force_new_asid(struct kvm_vcpu *vcpu) -{ - to_svm(vcpu)->asid_generation--; -} - -static inline void flush_guest_tlb(struct kvm_vcpu *vcpu) -{ - force_new_asid(vcpu); -} - -static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) -{ - if (!(efer & EFER_LMA)) - efer &= ~EFER_LME; - - to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK; - vcpu->arch.shadow_efer = efer; -} - -static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, - bool has_error_code, u32 error_code) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->vmcb->control.event_inj = nr - | SVM_EVTINJ_VALID - | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) - | SVM_EVTINJ_TYPE_EXEPT; - svm->vmcb->control.event_inj_err = error_code; -} - -static bool svm_exception_injected(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - return !(svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID); -} - -static int is_external_interrupt(u32 info) -{ - info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID; - return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR); -} - -static void skip_emulated_instruction(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - if (!svm->next_rip) { - printk(KERN_DEBUG "%s: NOP\n", __FUNCTION__); - return; - } - if (svm->next_rip - svm->vmcb->save.rip > MAX_INST_SIZE) - printk(KERN_ERR "%s: ip 0x%llx next 0x%llx\n", - __FUNCTION__, - svm->vmcb->save.rip, - svm->next_rip); - - vcpu->arch.rip = svm->vmcb->save.rip = svm->next_rip; - svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; - - vcpu->arch.interrupt_window_open = 1; -} - -static int has_svm(void) -{ - uint32_t eax, ebx, ecx, edx; - - if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) { - printk(KERN_INFO "has_svm: not amd\n"); - return 0; - } - - cpuid(0x80000000, &eax, &ebx, &ecx, &edx); - if (eax < SVM_CPUID_FUNC) { - printk(KERN_INFO "has_svm: can't execute cpuid_8000000a\n"); - return 0; - } - - cpuid(0x80000001, &eax, &ebx, &ecx, &edx); - if (!(ecx & (1 << SVM_CPUID_FEATURE_SHIFT))) { - printk(KERN_DEBUG "has_svm: svm not available\n"); - return 0; - } - return 1; -} - -static void svm_hardware_disable(void *garbage) -{ - struct svm_cpu_data *svm_data - = per_cpu(svm_data, raw_smp_processor_id()); - - if (svm_data) { - uint64_t efer; - - wrmsrl(MSR_VM_HSAVE_PA, 0); - rdmsrl(MSR_EFER, efer); - wrmsrl(MSR_EFER, efer & ~MSR_EFER_SVME_MASK); - per_cpu(svm_data, raw_smp_processor_id()) = NULL; - __free_page(svm_data->save_area); - kfree(svm_data); - } -} - -static void svm_hardware_enable(void *garbage) -{ - - struct svm_cpu_data *svm_data; - uint64_t efer; -#ifdef CONFIG_X86_64 - struct desc_ptr gdt_descr; -#else - struct desc_ptr gdt_descr; -#endif - struct desc_struct *gdt; - int me = raw_smp_processor_id(); - - if (!has_svm()) { - printk(KERN_ERR "svm_cpu_init: err EOPNOTSUPP on %d\n", me); - return; - } - svm_data = per_cpu(svm_data, me); - - if (!svm_data) { - printk(KERN_ERR "svm_cpu_init: svm_data is NULL on %d\n", - me); - return; - } - - svm_data->asid_generation = 1; - svm_data->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; - svm_data->next_asid = svm_data->max_asid + 1; - svm_features = cpuid_edx(SVM_CPUID_FUNC); - - asm volatile ("sgdt %0" : "=m"(gdt_descr)); - gdt = (struct desc_struct *)gdt_descr.address; - svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); - - rdmsrl(MSR_EFER, efer); - wrmsrl(MSR_EFER, efer | MSR_EFER_SVME_MASK); - - wrmsrl(MSR_VM_HSAVE_PA, - page_to_pfn(svm_data->save_area) << PAGE_SHIFT); -} - -static int svm_cpu_init(int cpu) -{ - struct svm_cpu_data *svm_data; - int r; - - svm_data = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); - if (!svm_data) - return -ENOMEM; - svm_data->cpu = cpu; - svm_data->save_area = alloc_page(GFP_KERNEL); - r = -ENOMEM; - if (!svm_data->save_area) - goto err_1; - - per_cpu(svm_data, cpu) = svm_data; - - return 0; - -err_1: - kfree(svm_data); - return r; - -} - -static void set_msr_interception(u32 *msrpm, unsigned msr, - int read, int write) -{ - int i; - - for (i = 0; i < NUM_MSR_MAPS; i++) { - if (msr >= msrpm_ranges[i] && - msr < msrpm_ranges[i] + MSRS_IN_RANGE) { - u32 msr_offset = (i * MSRS_IN_RANGE + msr - - msrpm_ranges[i]) * 2; - - u32 *base = msrpm + (msr_offset / 32); - u32 msr_shift = msr_offset % 32; - u32 mask = ((write) ? 0 : 2) | ((read) ? 0 : 1); - *base = (*base & ~(0x3 << msr_shift)) | - (mask << msr_shift); - return; - } - } - BUG(); -} - -static __init int svm_hardware_setup(void) -{ - int cpu; - struct page *iopm_pages; - struct page *msrpm_pages; - void *iopm_va, *msrpm_va; - int r; - - iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER); - - if (!iopm_pages) - return -ENOMEM; - - iopm_va = page_address(iopm_pages); - memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER)); - clear_bit(0x80, iopm_va); /* allow direct access to PC debug port */ - iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT; - - - msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); - - r = -ENOMEM; - if (!msrpm_pages) - goto err_1; - - msrpm_va = page_address(msrpm_pages); - memset(msrpm_va, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER)); - msrpm_base = page_to_pfn(msrpm_pages) << PAGE_SHIFT; - -#ifdef CONFIG_X86_64 - set_msr_interception(msrpm_va, MSR_GS_BASE, 1, 1); - set_msr_interception(msrpm_va, MSR_FS_BASE, 1, 1); - set_msr_interception(msrpm_va, MSR_KERNEL_GS_BASE, 1, 1); - set_msr_interception(msrpm_va, MSR_LSTAR, 1, 1); - set_msr_interception(msrpm_va, MSR_CSTAR, 1, 1); - set_msr_interception(msrpm_va, MSR_SYSCALL_MASK, 1, 1); -#endif - set_msr_interception(msrpm_va, MSR_K6_STAR, 1, 1); - set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_CS, 1, 1); - set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_ESP, 1, 1); - set_msr_interception(msrpm_va, MSR_IA32_SYSENTER_EIP, 1, 1); - - for_each_online_cpu(cpu) { - r = svm_cpu_init(cpu); - if (r) - goto err_2; - } - return 0; - -err_2: - __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER); - msrpm_base = 0; -err_1: - __free_pages(iopm_pages, IOPM_ALLOC_ORDER); - iopm_base = 0; - return r; -} - -static __exit void svm_hardware_unsetup(void) -{ - __free_pages(pfn_to_page(msrpm_base >> PAGE_SHIFT), MSRPM_ALLOC_ORDER); - __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER); - iopm_base = msrpm_base = 0; -} - -static void init_seg(struct vmcb_seg *seg) -{ - seg->selector = 0; - seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | - SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */ - seg->limit = 0xffff; - seg->base = 0; -} - -static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) -{ - seg->selector = 0; - seg->attrib = SVM_SELECTOR_P_MASK | type; - seg->limit = 0xffff; - seg->base = 0; -} - -static void init_vmcb(struct vmcb *vmcb) -{ - struct vmcb_control_area *control = &vmcb->control; - struct vmcb_save_area *save = &vmcb->save; - - control->intercept_cr_read = INTERCEPT_CR0_MASK | - INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK | - INTERCEPT_CR8_MASK; - - control->intercept_cr_write = INTERCEPT_CR0_MASK | - INTERCEPT_CR3_MASK | - INTERCEPT_CR4_MASK | - INTERCEPT_CR8_MASK; - - control->intercept_dr_read = INTERCEPT_DR0_MASK | - INTERCEPT_DR1_MASK | - INTERCEPT_DR2_MASK | - INTERCEPT_DR3_MASK; - - control->intercept_dr_write = INTERCEPT_DR0_MASK | - INTERCEPT_DR1_MASK | - INTERCEPT_DR2_MASK | - INTERCEPT_DR3_MASK | - INTERCEPT_DR5_MASK | - INTERCEPT_DR7_MASK; - - control->intercept_exceptions = (1 << PF_VECTOR) | - (1 << UD_VECTOR); - - - control->intercept = (1ULL << INTERCEPT_INTR) | - (1ULL << INTERCEPT_NMI) | - (1ULL << INTERCEPT_SMI) | - /* - * selective cr0 intercept bug? - * 0: 0f 22 d8 mov %eax,%cr3 - * 3: 0f 20 c0 mov %cr0,%eax - * 6: 0d 00 00 00 80 or $0x80000000,%eax - * b: 0f 22 c0 mov %eax,%cr0 - * set cr3 ->interception - * get cr0 ->interception - * set cr0 -> no interception - */ - /* (1ULL << INTERCEPT_SELECTIVE_CR0) | */ - (1ULL << INTERCEPT_CPUID) | - (1ULL << INTERCEPT_INVD) | - (1ULL << INTERCEPT_HLT) | - (1ULL << INTERCEPT_INVLPGA) | - (1ULL << INTERCEPT_IOIO_PROT) | - (1ULL << INTERCEPT_MSR_PROT) | - (1ULL << INTERCEPT_TASK_SWITCH) | - (1ULL << INTERCEPT_SHUTDOWN) | - (1ULL << INTERCEPT_VMRUN) | - (1ULL << INTERCEPT_VMMCALL) | - (1ULL << INTERCEPT_VMLOAD) | - (1ULL << INTERCEPT_VMSAVE) | - (1ULL << INTERCEPT_STGI) | - (1ULL << INTERCEPT_CLGI) | - (1ULL << INTERCEPT_SKINIT) | - (1ULL << INTERCEPT_WBINVD) | - (1ULL << INTERCEPT_MONITOR) | - (1ULL << INTERCEPT_MWAIT); - - control->iopm_base_pa = iopm_base; - control->msrpm_base_pa = msrpm_base; - control->tsc_offset = 0; - control->int_ctl = V_INTR_MASKING_MASK; - - init_seg(&save->es); - init_seg(&save->ss); - init_seg(&save->ds); - init_seg(&save->fs); - init_seg(&save->gs); - - save->cs.selector = 0xf000; - /* Executable/Readable Code Segment */ - save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | - SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK; - save->cs.limit = 0xffff; - /* - * cs.base should really be 0xffff0000, but vmx can't handle that, so - * be consistent with it. - * - * Replace when we have real mode working for vmx. - */ - save->cs.base = 0xf0000; - - save->gdtr.limit = 0xffff; - save->idtr.limit = 0xffff; - - init_sys_seg(&save->ldtr, SEG_TYPE_LDT); - init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); - - save->efer = MSR_EFER_SVME_MASK; - save->dr6 = 0xffff0ff0; - save->dr7 = 0x400; - save->rflags = 2; - save->rip = 0x0000fff0; - - /* - * cr0 val on cpu init should be 0x60000010, we enable cpu - * cache by default. the orderly way is to enable cache in bios. - */ - save->cr0 = 0x00000010 | X86_CR0_PG | X86_CR0_WP; - save->cr4 = X86_CR4_PAE; - /* rdx = ?? */ -} - -static int svm_vcpu_reset(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - init_vmcb(svm->vmcb); - - if (vcpu->vcpu_id != 0) { - svm->vmcb->save.rip = 0; - svm->vmcb->save.cs.base = svm->vcpu.arch.sipi_vector << 12; - svm->vmcb->save.cs.selector = svm->vcpu.arch.sipi_vector << 8; - } - - return 0; -} - -static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) -{ - struct vcpu_svm *svm; - struct page *page; - int err; - - svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); - if (!svm) { - err = -ENOMEM; - goto out; - } - - err = kvm_vcpu_init(&svm->vcpu, kvm, id); - if (err) - goto free_svm; - - page = alloc_page(GFP_KERNEL); - if (!page) { - err = -ENOMEM; - goto uninit; - } - - svm->vmcb = page_address(page); - clear_page(svm->vmcb); - svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT; - svm->asid_generation = 0; - memset(svm->db_regs, 0, sizeof(svm->db_regs)); - init_vmcb(svm->vmcb); - - fx_init(&svm->vcpu); - svm->vcpu.fpu_active = 1; - svm->vcpu.arch.apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; - if (svm->vcpu.vcpu_id == 0) - svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP; - - return &svm->vcpu; - -uninit: - kvm_vcpu_uninit(&svm->vcpu); -free_svm: - kmem_cache_free(kvm_vcpu_cache, svm); -out: - return ERR_PTR(err); -} - -static void svm_free_vcpu(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT)); - kvm_vcpu_uninit(vcpu); - kmem_cache_free(kvm_vcpu_cache, svm); -} - -static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - int i; - - if (unlikely(cpu != vcpu->cpu)) { - u64 tsc_this, delta; - - /* - * Make sure that the guest sees a monotonically - * increasing TSC. - */ - rdtscll(tsc_this); - delta = vcpu->arch.host_tsc - tsc_this; - svm->vmcb->control.tsc_offset += delta; - vcpu->cpu = cpu; - kvm_migrate_apic_timer(vcpu); - } - - for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) - rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); -} - -static void svm_vcpu_put(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - int i; - - ++vcpu->stat.host_state_reload; - for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++) - wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]); - - rdtscll(vcpu->arch.host_tsc); -} - -static void svm_vcpu_decache(struct kvm_vcpu *vcpu) -{ -} - -static void svm_cache_regs(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; - vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; - vcpu->arch.rip = svm->vmcb->save.rip; -} - -static void svm_decache_regs(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; - svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; - svm->vmcb->save.rip = vcpu->arch.rip; -} - -static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu) -{ - return to_svm(vcpu)->vmcb->save.rflags; -} - -static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) -{ - to_svm(vcpu)->vmcb->save.rflags = rflags; -} - -static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) -{ - struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; - - switch (seg) { - case VCPU_SREG_CS: return &save->cs; - case VCPU_SREG_DS: return &save->ds; - case VCPU_SREG_ES: return &save->es; - case VCPU_SREG_FS: return &save->fs; - case VCPU_SREG_GS: return &save->gs; - case VCPU_SREG_SS: return &save->ss; - case VCPU_SREG_TR: return &save->tr; - case VCPU_SREG_LDTR: return &save->ldtr; - } - BUG(); - return NULL; -} - -static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg) -{ - struct vmcb_seg *s = svm_seg(vcpu, seg); - - return s->base; -} - -static void svm_get_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - struct vmcb_seg *s = svm_seg(vcpu, seg); - - var->base = s->base; - var->limit = s->limit; - var->selector = s->selector; - var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; - var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; - var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; - var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; - var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; - var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; - var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; - var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1; - var->unusable = !var->present; -} - -static void svm_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - dt->limit = svm->vmcb->save.idtr.limit; - dt->base = svm->vmcb->save.idtr.base; -} - -static void svm_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->vmcb->save.idtr.limit = dt->limit; - svm->vmcb->save.idtr.base = dt->base ; -} - -static void svm_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - dt->limit = svm->vmcb->save.gdtr.limit; - dt->base = svm->vmcb->save.gdtr.base; -} - -static void svm_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->vmcb->save.gdtr.limit = dt->limit; - svm->vmcb->save.gdtr.base = dt->base ; -} - -static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) -{ -} - -static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ - struct vcpu_svm *svm = to_svm(vcpu); - -#ifdef CONFIG_X86_64 - if (vcpu->arch.shadow_efer & EFER_LME) { - if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { - vcpu->arch.shadow_efer |= EFER_LMA; - svm->vmcb->save.efer |= EFER_LMA | EFER_LME; - } - - if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) { - vcpu->arch.shadow_efer &= ~EFER_LMA; - svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); - } - } -#endif - if ((vcpu->arch.cr0 & X86_CR0_TS) && !(cr0 & X86_CR0_TS)) { - svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); - vcpu->fpu_active = 1; - } - - vcpu->arch.cr0 = cr0; - cr0 |= X86_CR0_PG | X86_CR0_WP; - cr0 &= ~(X86_CR0_CD | X86_CR0_NW); - svm->vmcb->save.cr0 = cr0; -} - -static void svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - vcpu->arch.cr4 = cr4; - to_svm(vcpu)->vmcb->save.cr4 = cr4 | X86_CR4_PAE; -} - -static void svm_set_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - struct vcpu_svm *svm = to_svm(vcpu); - struct vmcb_seg *s = svm_seg(vcpu, seg); - - s->base = var->base; - s->limit = var->limit; - s->selector = var->selector; - if (var->unusable) - s->attrib = 0; - else { - s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); - s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; - s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; - s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT; - s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; - s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; - s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; - s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; - } - if (seg == VCPU_SREG_CS) - svm->vmcb->save.cpl - = (svm->vmcb->save.cs.attrib - >> SVM_SELECTOR_DPL_SHIFT) & 3; - -} - -/* FIXME: - - svm(vcpu)->vmcb->control.int_ctl &= ~V_TPR_MASK; - svm(vcpu)->vmcb->control.int_ctl |= (sregs->cr8 & V_TPR_MASK); - -*/ - -static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) -{ - return -EOPNOTSUPP; -} - -static int svm_get_irq(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - u32 exit_int_info = svm->vmcb->control.exit_int_info; - - if (is_external_interrupt(exit_int_info)) - return exit_int_info & SVM_EVTINJ_VEC_MASK; - return -1; -} - -static void load_host_msrs(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - wrmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); -#endif -} - -static void save_host_msrs(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host_gs_base); -#endif -} - -static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data) -{ - if (svm_data->next_asid > svm_data->max_asid) { - ++svm_data->asid_generation; - svm_data->next_asid = 1; - svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; - } - - svm->vcpu.cpu = svm_data->cpu; - svm->asid_generation = svm_data->asid_generation; - svm->vmcb->control.asid = svm_data->next_asid++; -} - -static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr) -{ - return to_svm(vcpu)->db_regs[dr]; -} - -static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value, - int *exception) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - *exception = 0; - - if (svm->vmcb->save.dr7 & DR7_GD_MASK) { - svm->vmcb->save.dr7 &= ~DR7_GD_MASK; - svm->vmcb->save.dr6 |= DR6_BD_MASK; - *exception = DB_VECTOR; - return; - } - - switch (dr) { - case 0 ... 3: - svm->db_regs[dr] = value; - return; - case 4 ... 5: - if (vcpu->arch.cr4 & X86_CR4_DE) { - *exception = UD_VECTOR; - return; - } - case 7: { - if (value & ~((1ULL << 32) - 1)) { - *exception = GP_VECTOR; - return; - } - svm->vmcb->save.dr7 = value; - return; - } - default: - printk(KERN_DEBUG "%s: unexpected dr %u\n", - __FUNCTION__, dr); - *exception = UD_VECTOR; - return; - } -} - -static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - u32 exit_int_info = svm->vmcb->control.exit_int_info; - struct kvm *kvm = svm->vcpu.kvm; - u64 fault_address; - u32 error_code; - - if (!irqchip_in_kernel(kvm) && - is_external_interrupt(exit_int_info)) - push_irq(&svm->vcpu, exit_int_info & SVM_EVTINJ_VEC_MASK); - - fault_address = svm->vmcb->control.exit_info_2; - error_code = svm->vmcb->control.exit_info_1; - return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); -} - -static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - int er; - - er = emulate_instruction(&svm->vcpu, kvm_run, 0, 0, 0); - if (er != EMULATE_DONE) - kvm_queue_exception(&svm->vcpu, UD_VECTOR); - return 1; -} - -static int nm_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - svm->vmcb->control.intercept_exceptions &= ~(1 << NM_VECTOR); - if (!(svm->vcpu.arch.cr0 & X86_CR0_TS)) - svm->vmcb->save.cr0 &= ~X86_CR0_TS; - svm->vcpu.fpu_active = 1; - - return 1; -} - -static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - /* - * VMCB is undefined after a SHUTDOWN intercept - * so reinitialize it. - */ - clear_page(svm->vmcb); - init_vmcb(svm->vmcb); - - kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; - return 0; -} - -static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ - int size, down, in, string, rep; - unsigned port; - - ++svm->vcpu.stat.io_exits; - - svm->next_rip = svm->vmcb->control.exit_info_2; - - string = (io_info & SVM_IOIO_STR_MASK) != 0; - - if (string) { - if (emulate_instruction(&svm->vcpu, - kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) - return 0; - return 1; - } - - in = (io_info & SVM_IOIO_TYPE_MASK) != 0; - port = io_info >> 16; - size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; - rep = (io_info & SVM_IOIO_REP_MASK) != 0; - down = (svm->vmcb->save.rflags & X86_EFLAGS_DF) != 0; - - return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port); -} - -static int nop_on_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - return 1; -} - -static int halt_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - svm->next_rip = svm->vmcb->save.rip + 1; - skip_emulated_instruction(&svm->vcpu); - return kvm_emulate_halt(&svm->vcpu); -} - -static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - svm->next_rip = svm->vmcb->save.rip + 3; - skip_emulated_instruction(&svm->vcpu); - kvm_emulate_hypercall(&svm->vcpu); - return 1; -} - -static int invalid_op_interception(struct vcpu_svm *svm, - struct kvm_run *kvm_run) -{ - kvm_queue_exception(&svm->vcpu, UD_VECTOR); - return 1; -} - -static int task_switch_interception(struct vcpu_svm *svm, - struct kvm_run *kvm_run) -{ - pr_unimpl(&svm->vcpu, "%s: task switch is unsupported\n", __FUNCTION__); - kvm_run->exit_reason = KVM_EXIT_UNKNOWN; - return 0; -} - -static int cpuid_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - svm->next_rip = svm->vmcb->save.rip + 2; - kvm_emulate_cpuid(&svm->vcpu); - return 1; -} - -static int emulate_on_interception(struct vcpu_svm *svm, - struct kvm_run *kvm_run) -{ - if (emulate_instruction(&svm->vcpu, NULL, 0, 0, 0) != EMULATE_DONE) - pr_unimpl(&svm->vcpu, "%s: failed\n", __FUNCTION__); - return 1; -} - -static int cr8_write_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - emulate_instruction(&svm->vcpu, NULL, 0, 0, 0); - if (irqchip_in_kernel(svm->vcpu.kvm)) - return 1; - kvm_run->exit_reason = KVM_EXIT_SET_TPR; - return 0; -} - -static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - switch (ecx) { - case MSR_IA32_TIME_STAMP_COUNTER: { - u64 tsc; - - rdtscll(tsc); - *data = svm->vmcb->control.tsc_offset + tsc; - break; - } - case MSR_K6_STAR: - *data = svm->vmcb->save.star; - break; -#ifdef CONFIG_X86_64 - case MSR_LSTAR: - *data = svm->vmcb->save.lstar; - break; - case MSR_CSTAR: - *data = svm->vmcb->save.cstar; - break; - case MSR_KERNEL_GS_BASE: - *data = svm->vmcb->save.kernel_gs_base; - break; - case MSR_SYSCALL_MASK: - *data = svm->vmcb->save.sfmask; - break; -#endif - case MSR_IA32_SYSENTER_CS: - *data = svm->vmcb->save.sysenter_cs; - break; - case MSR_IA32_SYSENTER_EIP: - *data = svm->vmcb->save.sysenter_eip; - break; - case MSR_IA32_SYSENTER_ESP: - *data = svm->vmcb->save.sysenter_esp; - break; - default: - return kvm_get_msr_common(vcpu, ecx, data); - } - return 0; -} - -static int rdmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; - u64 data; - - if (svm_get_msr(&svm->vcpu, ecx, &data)) - kvm_inject_gp(&svm->vcpu, 0); - else { - svm->vmcb->save.rax = data & 0xffffffff; - svm->vcpu.arch.regs[VCPU_REGS_RDX] = data >> 32; - svm->next_rip = svm->vmcb->save.rip + 2; - skip_emulated_instruction(&svm->vcpu); - } - return 1; -} - -static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - switch (ecx) { - case MSR_IA32_TIME_STAMP_COUNTER: { - u64 tsc; - - rdtscll(tsc); - svm->vmcb->control.tsc_offset = data - tsc; - break; - } - case MSR_K6_STAR: - svm->vmcb->save.star = data; - break; -#ifdef CONFIG_X86_64 - case MSR_LSTAR: - svm->vmcb->save.lstar = data; - break; - case MSR_CSTAR: - svm->vmcb->save.cstar = data; - break; - case MSR_KERNEL_GS_BASE: - svm->vmcb->save.kernel_gs_base = data; - break; - case MSR_SYSCALL_MASK: - svm->vmcb->save.sfmask = data; - break; -#endif - case MSR_IA32_SYSENTER_CS: - svm->vmcb->save.sysenter_cs = data; - break; - case MSR_IA32_SYSENTER_EIP: - svm->vmcb->save.sysenter_eip = data; - break; - case MSR_IA32_SYSENTER_ESP: - svm->vmcb->save.sysenter_esp = data; - break; - case MSR_K7_EVNTSEL0: - case MSR_K7_EVNTSEL1: - case MSR_K7_EVNTSEL2: - case MSR_K7_EVNTSEL3: - /* - * only support writing 0 to the performance counters for now - * to make Windows happy. Should be replaced by a real - * performance counter emulation later. - */ - if (data != 0) - goto unhandled; - break; - default: - unhandled: - return kvm_set_msr_common(vcpu, ecx, data); - } - return 0; -} - -static int wrmsr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - u32 ecx = svm->vcpu.arch.regs[VCPU_REGS_RCX]; - u64 data = (svm->vmcb->save.rax & -1u) - | ((u64)(svm->vcpu.arch.regs[VCPU_REGS_RDX] & -1u) << 32); - svm->next_rip = svm->vmcb->save.rip + 2; - if (svm_set_msr(&svm->vcpu, ecx, data)) - kvm_inject_gp(&svm->vcpu, 0); - else - skip_emulated_instruction(&svm->vcpu); - return 1; -} - -static int msr_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) -{ - if (svm->vmcb->control.exit_info_1) - return wrmsr_interception(svm, kvm_run); - else - return rdmsr_interception(svm, kvm_run); -} - -static int interrupt_window_interception(struct vcpu_svm *svm, - struct kvm_run *kvm_run) -{ - svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR); - svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; - /* - * If the user space waits to inject interrupts, exit as soon as - * possible - */ - if (kvm_run->request_interrupt_window && - !svm->vcpu.arch.irq_summary) { - ++svm->vcpu.stat.irq_window_exits; - kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; - return 0; - } - - return 1; -} - -static int (*svm_exit_handlers[])(struct vcpu_svm *svm, - struct kvm_run *kvm_run) = { - [SVM_EXIT_READ_CR0] = emulate_on_interception, - [SVM_EXIT_READ_CR3] = emulate_on_interception, - [SVM_EXIT_READ_CR4] = emulate_on_interception, - [SVM_EXIT_READ_CR8] = emulate_on_interception, - /* for now: */ - [SVM_EXIT_WRITE_CR0] = emulate_on_interception, - [SVM_EXIT_WRITE_CR3] = emulate_on_interception, - [SVM_EXIT_WRITE_CR4] = emulate_on_interception, - [SVM_EXIT_WRITE_CR8] = cr8_write_interception, - [SVM_EXIT_READ_DR0] = emulate_on_interception, - [SVM_EXIT_READ_DR1] = emulate_on_interception, - [SVM_EXIT_READ_DR2] = emulate_on_interception, - [SVM_EXIT_READ_DR3] = emulate_on_interception, - [SVM_EXIT_WRITE_DR0] = emulate_on_interception, - [SVM_EXIT_WRITE_DR1] = emulate_on_interception, - [SVM_EXIT_WRITE_DR2] = emulate_on_interception, - [SVM_EXIT_WRITE_DR3] = emulate_on_interception, - [SVM_EXIT_WRITE_DR5] = emulate_on_interception, - [SVM_EXIT_WRITE_DR7] = emulate_on_interception, - [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, - [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, - [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception, - [SVM_EXIT_INTR] = nop_on_interception, - [SVM_EXIT_NMI] = nop_on_interception, - [SVM_EXIT_SMI] = nop_on_interception, - [SVM_EXIT_INIT] = nop_on_interception, - [SVM_EXIT_VINTR] = interrupt_window_interception, - /* [SVM_EXIT_CR0_SEL_WRITE] = emulate_on_interception, */ - [SVM_EXIT_CPUID] = cpuid_interception, - [SVM_EXIT_INVD] = emulate_on_interception, - [SVM_EXIT_HLT] = halt_interception, - [SVM_EXIT_INVLPG] = emulate_on_interception, - [SVM_EXIT_INVLPGA] = invalid_op_interception, - [SVM_EXIT_IOIO] = io_interception, - [SVM_EXIT_MSR] = msr_interception, - [SVM_EXIT_TASK_SWITCH] = task_switch_interception, - [SVM_EXIT_SHUTDOWN] = shutdown_interception, - [SVM_EXIT_VMRUN] = invalid_op_interception, - [SVM_EXIT_VMMCALL] = vmmcall_interception, - [SVM_EXIT_VMLOAD] = invalid_op_interception, - [SVM_EXIT_VMSAVE] = invalid_op_interception, - [SVM_EXIT_STGI] = invalid_op_interception, - [SVM_EXIT_CLGI] = invalid_op_interception, - [SVM_EXIT_SKINIT] = invalid_op_interception, - [SVM_EXIT_WBINVD] = emulate_on_interception, - [SVM_EXIT_MONITOR] = invalid_op_interception, - [SVM_EXIT_MWAIT] = invalid_op_interception, -}; - - -static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - u32 exit_code = svm->vmcb->control.exit_code; - - kvm_reput_irq(svm); - - if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { - kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; - kvm_run->fail_entry.hardware_entry_failure_reason - = svm->vmcb->control.exit_code; - return 0; - } - - if (is_external_interrupt(svm->vmcb->control.exit_int_info) && - exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR) - printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x " - "exit_code 0x%x\n", - __FUNCTION__, svm->vmcb->control.exit_int_info, - exit_code); - - if (exit_code >= ARRAY_SIZE(svm_exit_handlers) - || !svm_exit_handlers[exit_code]) { - kvm_run->exit_reason = KVM_EXIT_UNKNOWN; - kvm_run->hw.hardware_exit_reason = exit_code; - return 0; - } - - return svm_exit_handlers[exit_code](svm, kvm_run); -} - -static void reload_tss(struct kvm_vcpu *vcpu) -{ - int cpu = raw_smp_processor_id(); - - struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu); - svm_data->tss_desc->type = 9; /* available 32/64-bit TSS */ - load_TR_desc(); -} - -static void pre_svm_run(struct vcpu_svm *svm) -{ - int cpu = raw_smp_processor_id(); - - struct svm_cpu_data *svm_data = per_cpu(svm_data, cpu); - - svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; - if (svm->vcpu.cpu != cpu || - svm->asid_generation != svm_data->asid_generation) - new_asid(svm, svm_data); -} - - -static inline void svm_inject_irq(struct vcpu_svm *svm, int irq) -{ - struct vmcb_control_area *control; - - control = &svm->vmcb->control; - control->int_vector = irq; - control->int_ctl &= ~V_INTR_PRIO_MASK; - control->int_ctl |= V_IRQ_MASK | - ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); -} - -static void svm_set_irq(struct kvm_vcpu *vcpu, int irq) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm_inject_irq(svm, irq); -} - -static void svm_intr_assist(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - struct vmcb *vmcb = svm->vmcb; - int intr_vector = -1; - - if ((vmcb->control.exit_int_info & SVM_EVTINJ_VALID) && - ((vmcb->control.exit_int_info & SVM_EVTINJ_TYPE_MASK) == 0)) { - intr_vector = vmcb->control.exit_int_info & - SVM_EVTINJ_VEC_MASK; - vmcb->control.exit_int_info = 0; - svm_inject_irq(svm, intr_vector); - return; - } - - if (vmcb->control.int_ctl & V_IRQ_MASK) - return; - - if (!kvm_cpu_has_interrupt(vcpu)) - return; - - if (!(vmcb->save.rflags & X86_EFLAGS_IF) || - (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || - (vmcb->control.event_inj & SVM_EVTINJ_VALID)) { - /* unable to deliver irq, set pending irq */ - vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR); - svm_inject_irq(svm, 0x0); - return; - } - /* Okay, we can deliver the interrupt: grab it and update PIC state. */ - intr_vector = kvm_cpu_get_interrupt(vcpu); - svm_inject_irq(svm, intr_vector); - kvm_timer_intr_post(vcpu, intr_vector); -} - -static void kvm_reput_irq(struct vcpu_svm *svm) -{ - struct vmcb_control_area *control = &svm->vmcb->control; - - if ((control->int_ctl & V_IRQ_MASK) - && !irqchip_in_kernel(svm->vcpu.kvm)) { - control->int_ctl &= ~V_IRQ_MASK; - push_irq(&svm->vcpu, control->int_vector); - } - - svm->vcpu.arch.interrupt_window_open = - !(control->int_state & SVM_INTERRUPT_SHADOW_MASK); -} - -static void svm_do_inject_vector(struct vcpu_svm *svm) -{ - struct kvm_vcpu *vcpu = &svm->vcpu; - int word_index = __ffs(vcpu->arch.irq_summary); - int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); - int irq = word_index * BITS_PER_LONG + bit_index; - - clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); - if (!vcpu->arch.irq_pending[word_index]) - clear_bit(word_index, &vcpu->arch.irq_summary); - svm_inject_irq(svm, irq); -} - -static void do_interrupt_requests(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - struct vcpu_svm *svm = to_svm(vcpu); - struct vmcb_control_area *control = &svm->vmcb->control; - - svm->vcpu.arch.interrupt_window_open = - (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) && - (svm->vmcb->save.rflags & X86_EFLAGS_IF)); - - if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary) - /* - * If interrupts enabled, and not blocked by sti or mov ss. Good. - */ - svm_do_inject_vector(svm); - - /* - * Interrupts blocked. Wait for unblock. - */ - if (!svm->vcpu.arch.interrupt_window_open && - (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window)) - control->intercept |= 1ULL << INTERCEPT_VINTR; - else - control->intercept &= ~(1ULL << INTERCEPT_VINTR); -} - -static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) -{ - return 0; -} - -static void save_db_regs(unsigned long *db_regs) -{ - asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0])); - asm volatile ("mov %%dr1, %0" : "=r"(db_regs[1])); - asm volatile ("mov %%dr2, %0" : "=r"(db_regs[2])); - asm volatile ("mov %%dr3, %0" : "=r"(db_regs[3])); -} - -static void load_db_regs(unsigned long *db_regs) -{ - asm volatile ("mov %0, %%dr0" : : "r"(db_regs[0])); - asm volatile ("mov %0, %%dr1" : : "r"(db_regs[1])); - asm volatile ("mov %0, %%dr2" : : "r"(db_regs[2])); - asm volatile ("mov %0, %%dr3" : : "r"(db_regs[3])); -} - -static void svm_flush_tlb(struct kvm_vcpu *vcpu) -{ - force_new_asid(vcpu); -} - -static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) -{ -} - -static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - struct vcpu_svm *svm = to_svm(vcpu); - u16 fs_selector; - u16 gs_selector; - u16 ldt_selector; - - pre_svm_run(svm); - - save_host_msrs(vcpu); - fs_selector = read_fs(); - gs_selector = read_gs(); - ldt_selector = read_ldt(); - svm->host_cr2 = kvm_read_cr2(); - svm->host_dr6 = read_dr6(); - svm->host_dr7 = read_dr7(); - svm->vmcb->save.cr2 = vcpu->arch.cr2; - - if (svm->vmcb->save.dr7 & 0xff) { - write_dr7(0); - save_db_regs(svm->host_db_regs); - load_db_regs(svm->db_regs); - } - - clgi(); - - local_irq_enable(); - - asm volatile ( -#ifdef CONFIG_X86_64 - "push %%rbp; \n\t" -#else - "push %%ebp; \n\t" -#endif - -#ifdef CONFIG_X86_64 - "mov %c[rbx](%[svm]), %%rbx \n\t" - "mov %c[rcx](%[svm]), %%rcx \n\t" - "mov %c[rdx](%[svm]), %%rdx \n\t" - "mov %c[rsi](%[svm]), %%rsi \n\t" - "mov %c[rdi](%[svm]), %%rdi \n\t" - "mov %c[rbp](%[svm]), %%rbp \n\t" - "mov %c[r8](%[svm]), %%r8 \n\t" - "mov %c[r9](%[svm]), %%r9 \n\t" - "mov %c[r10](%[svm]), %%r10 \n\t" - "mov %c[r11](%[svm]), %%r11 \n\t" - "mov %c[r12](%[svm]), %%r12 \n\t" - "mov %c[r13](%[svm]), %%r13 \n\t" - "mov %c[r14](%[svm]), %%r14 \n\t" - "mov %c[r15](%[svm]), %%r15 \n\t" -#else - "mov %c[rbx](%[svm]), %%ebx \n\t" - "mov %c[rcx](%[svm]), %%ecx \n\t" - "mov %c[rdx](%[svm]), %%edx \n\t" - "mov %c[rsi](%[svm]), %%esi \n\t" - "mov %c[rdi](%[svm]), %%edi \n\t" - "mov %c[rbp](%[svm]), %%ebp \n\t" -#endif - -#ifdef CONFIG_X86_64 - /* Enter guest mode */ - "push %%rax \n\t" - "mov %c[vmcb](%[svm]), %%rax \n\t" - SVM_VMLOAD "\n\t" - SVM_VMRUN "\n\t" - SVM_VMSAVE "\n\t" - "pop %%rax \n\t" -#else - /* Enter guest mode */ - "push %%eax \n\t" - "mov %c[vmcb](%[svm]), %%eax \n\t" - SVM_VMLOAD "\n\t" - SVM_VMRUN "\n\t" - SVM_VMSAVE "\n\t" - "pop %%eax \n\t" -#endif - - /* Save guest registers, load host registers */ -#ifdef CONFIG_X86_64 - "mov %%rbx, %c[rbx](%[svm]) \n\t" - "mov %%rcx, %c[rcx](%[svm]) \n\t" - "mov %%rdx, %c[rdx](%[svm]) \n\t" - "mov %%rsi, %c[rsi](%[svm]) \n\t" - "mov %%rdi, %c[rdi](%[svm]) \n\t" - "mov %%rbp, %c[rbp](%[svm]) \n\t" - "mov %%r8, %c[r8](%[svm]) \n\t" - "mov %%r9, %c[r9](%[svm]) \n\t" - "mov %%r10, %c[r10](%[svm]) \n\t" - "mov %%r11, %c[r11](%[svm]) \n\t" - "mov %%r12, %c[r12](%[svm]) \n\t" - "mov %%r13, %c[r13](%[svm]) \n\t" - "mov %%r14, %c[r14](%[svm]) \n\t" - "mov %%r15, %c[r15](%[svm]) \n\t" - - "pop %%rbp; \n\t" -#else - "mov %%ebx, %c[rbx](%[svm]) \n\t" - "mov %%ecx, %c[rcx](%[svm]) \n\t" - "mov %%edx, %c[rdx](%[svm]) \n\t" - "mov %%esi, %c[rsi](%[svm]) \n\t" - "mov %%edi, %c[rdi](%[svm]) \n\t" - "mov %%ebp, %c[rbp](%[svm]) \n\t" - - "pop %%ebp; \n\t" -#endif - : - : [svm]"a"(svm), - [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), - [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP])) -#ifdef CONFIG_X86_64 - , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15])) -#endif - : "cc", "memory" -#ifdef CONFIG_X86_64 - , "rbx", "rcx", "rdx", "rsi", "rdi" - , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" -#else - , "ebx", "ecx", "edx" , "esi", "edi" -#endif - ); - - if ((svm->vmcb->save.dr7 & 0xff)) - load_db_regs(svm->host_db_regs); - - vcpu->arch.cr2 = svm->vmcb->save.cr2; - - write_dr6(svm->host_dr6); - write_dr7(svm->host_dr7); - kvm_write_cr2(svm->host_cr2); - - load_fs(fs_selector); - load_gs(gs_selector); - load_ldt(ldt_selector); - load_host_msrs(vcpu); - - reload_tss(vcpu); - - local_irq_disable(); - - stgi(); - - svm->next_rip = 0; -} - -static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->vmcb->save.cr3 = root; - force_new_asid(vcpu); - - if (vcpu->fpu_active) { - svm->vmcb->control.intercept_exceptions |= (1 << NM_VECTOR); - svm->vmcb->save.cr0 |= X86_CR0_TS; - vcpu->fpu_active = 0; - } -} - -static int is_disabled(void) -{ - u64 vm_cr; - - rdmsrl(MSR_VM_CR, vm_cr); - if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE)) - return 1; - - return 0; -} - -static void -svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) -{ - /* - * Patch in the VMMCALL instruction: - */ - hypercall[0] = 0x0f; - hypercall[1] = 0x01; - hypercall[2] = 0xd9; -} - -static void svm_check_processor_compat(void *rtn) -{ - *(int *)rtn = 0; -} - -static struct kvm_x86_ops svm_x86_ops = { - .cpu_has_kvm_support = has_svm, - .disabled_by_bios = is_disabled, - .hardware_setup = svm_hardware_setup, - .hardware_unsetup = svm_hardware_unsetup, - .check_processor_compatibility = svm_check_processor_compat, - .hardware_enable = svm_hardware_enable, - .hardware_disable = svm_hardware_disable, - - .vcpu_create = svm_create_vcpu, - .vcpu_free = svm_free_vcpu, - .vcpu_reset = svm_vcpu_reset, - - .prepare_guest_switch = svm_prepare_guest_switch, - .vcpu_load = svm_vcpu_load, - .vcpu_put = svm_vcpu_put, - .vcpu_decache = svm_vcpu_decache, - - .set_guest_debug = svm_guest_debug, - .get_msr = svm_get_msr, - .set_msr = svm_set_msr, - .get_segment_base = svm_get_segment_base, - .get_segment = svm_get_segment, - .set_segment = svm_set_segment, - .get_cs_db_l_bits = kvm_get_cs_db_l_bits, - .decache_cr4_guest_bits = svm_decache_cr4_guest_bits, - .set_cr0 = svm_set_cr0, - .set_cr3 = svm_set_cr3, - .set_cr4 = svm_set_cr4, - .set_efer = svm_set_efer, - .get_idt = svm_get_idt, - .set_idt = svm_set_idt, - .get_gdt = svm_get_gdt, - .set_gdt = svm_set_gdt, - .get_dr = svm_get_dr, - .set_dr = svm_set_dr, - .cache_regs = svm_cache_regs, - .decache_regs = svm_decache_regs, - .get_rflags = svm_get_rflags, - .set_rflags = svm_set_rflags, - - .tlb_flush = svm_flush_tlb, - - .run = svm_vcpu_run, - .handle_exit = handle_exit, - .skip_emulated_instruction = skip_emulated_instruction, - .patch_hypercall = svm_patch_hypercall, - .get_irq = svm_get_irq, - .set_irq = svm_set_irq, - .queue_exception = svm_queue_exception, - .exception_injected = svm_exception_injected, - .inject_pending_irq = svm_intr_assist, - .inject_pending_vectors = do_interrupt_requests, - - .set_tss_addr = svm_set_tss_addr, -}; - -static int __init svm_init(void) -{ - return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm), - THIS_MODULE); -} - -static void __exit svm_exit(void) -{ - kvm_exit(); -} - -module_init(svm_init) -module_exit(svm_exit) diff --git a/drivers/kvm/svm.h b/drivers/kvm/svm.h deleted file mode 100644 index 5fd50491b555..000000000000 --- a/drivers/kvm/svm.h +++ /dev/null @@ -1,325 +0,0 @@ -#ifndef __SVM_H -#define __SVM_H - -enum { - INTERCEPT_INTR, - INTERCEPT_NMI, - INTERCEPT_SMI, - INTERCEPT_INIT, - INTERCEPT_VINTR, - INTERCEPT_SELECTIVE_CR0, - INTERCEPT_STORE_IDTR, - INTERCEPT_STORE_GDTR, - INTERCEPT_STORE_LDTR, - INTERCEPT_STORE_TR, - INTERCEPT_LOAD_IDTR, - INTERCEPT_LOAD_GDTR, - INTERCEPT_LOAD_LDTR, - INTERCEPT_LOAD_TR, - INTERCEPT_RDTSC, - INTERCEPT_RDPMC, - INTERCEPT_PUSHF, - INTERCEPT_POPF, - INTERCEPT_CPUID, - INTERCEPT_RSM, - INTERCEPT_IRET, - INTERCEPT_INTn, - INTERCEPT_INVD, - INTERCEPT_PAUSE, - INTERCEPT_HLT, - INTERCEPT_INVLPG, - INTERCEPT_INVLPGA, - INTERCEPT_IOIO_PROT, - INTERCEPT_MSR_PROT, - INTERCEPT_TASK_SWITCH, - INTERCEPT_FERR_FREEZE, - INTERCEPT_SHUTDOWN, - INTERCEPT_VMRUN, - INTERCEPT_VMMCALL, - INTERCEPT_VMLOAD, - INTERCEPT_VMSAVE, - INTERCEPT_STGI, - INTERCEPT_CLGI, - INTERCEPT_SKINIT, - INTERCEPT_RDTSCP, - INTERCEPT_ICEBP, - INTERCEPT_WBINVD, - INTERCEPT_MONITOR, - INTERCEPT_MWAIT, - INTERCEPT_MWAIT_COND, -}; - - -struct __attribute__ ((__packed__)) vmcb_control_area { - u16 intercept_cr_read; - u16 intercept_cr_write; - u16 intercept_dr_read; - u16 intercept_dr_write; - u32 intercept_exceptions; - u64 intercept; - u8 reserved_1[44]; - u64 iopm_base_pa; - u64 msrpm_base_pa; - u64 tsc_offset; - u32 asid; - u8 tlb_ctl; - u8 reserved_2[3]; - u32 int_ctl; - u32 int_vector; - u32 int_state; - u8 reserved_3[4]; - u32 exit_code; - u32 exit_code_hi; - u64 exit_info_1; - u64 exit_info_2; - u32 exit_int_info; - u32 exit_int_info_err; - u64 nested_ctl; - u8 reserved_4[16]; - u32 event_inj; - u32 event_inj_err; - u64 nested_cr3; - u64 lbr_ctl; - u8 reserved_5[832]; -}; - - -#define TLB_CONTROL_DO_NOTHING 0 -#define TLB_CONTROL_FLUSH_ALL_ASID 1 - -#define V_TPR_MASK 0x0f - -#define V_IRQ_SHIFT 8 -#define V_IRQ_MASK (1 << V_IRQ_SHIFT) - -#define V_INTR_PRIO_SHIFT 16 -#define V_INTR_PRIO_MASK (0x0f << V_INTR_PRIO_SHIFT) - -#define V_IGN_TPR_SHIFT 20 -#define V_IGN_TPR_MASK (1 << V_IGN_TPR_SHIFT) - -#define V_INTR_MASKING_SHIFT 24 -#define V_INTR_MASKING_MASK (1 << V_INTR_MASKING_SHIFT) - -#define SVM_INTERRUPT_SHADOW_MASK 1 - -#define SVM_IOIO_STR_SHIFT 2 -#define SVM_IOIO_REP_SHIFT 3 -#define SVM_IOIO_SIZE_SHIFT 4 -#define SVM_IOIO_ASIZE_SHIFT 7 - -#define SVM_IOIO_TYPE_MASK 1 -#define SVM_IOIO_STR_MASK (1 << SVM_IOIO_STR_SHIFT) -#define SVM_IOIO_REP_MASK (1 << SVM_IOIO_REP_SHIFT) -#define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) -#define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) - -struct __attribute__ ((__packed__)) vmcb_seg { - u16 selector; - u16 attrib; - u32 limit; - u64 base; -}; - -struct __attribute__ ((__packed__)) vmcb_save_area { - struct vmcb_seg es; - struct vmcb_seg cs; - struct vmcb_seg ss; - struct vmcb_seg ds; - struct vmcb_seg fs; - struct vmcb_seg gs; - struct vmcb_seg gdtr; - struct vmcb_seg ldtr; - struct vmcb_seg idtr; - struct vmcb_seg tr; - u8 reserved_1[43]; - u8 cpl; - u8 reserved_2[4]; - u64 efer; - u8 reserved_3[112]; - u64 cr4; - u64 cr3; - u64 cr0; - u64 dr7; - u64 dr6; - u64 rflags; - u64 rip; - u8 reserved_4[88]; - u64 rsp; - u8 reserved_5[24]; - u64 rax; - u64 star; - u64 lstar; - u64 cstar; - u64 sfmask; - u64 kernel_gs_base; - u64 sysenter_cs; - u64 sysenter_esp; - u64 sysenter_eip; - u64 cr2; - u8 reserved_6[32]; - u64 g_pat; - u64 dbgctl; - u64 br_from; - u64 br_to; - u64 last_excp_from; - u64 last_excp_to; -}; - -struct __attribute__ ((__packed__)) vmcb { - struct vmcb_control_area control; - struct vmcb_save_area save; -}; - -#define SVM_CPUID_FEATURE_SHIFT 2 -#define SVM_CPUID_FUNC 0x8000000a - -#define MSR_EFER_SVME_MASK (1ULL << 12) -#define MSR_VM_CR 0xc0010114 -#define MSR_VM_HSAVE_PA 0xc0010117ULL - -#define SVM_VM_CR_SVM_DISABLE 4 - -#define SVM_SELECTOR_S_SHIFT 4 -#define SVM_SELECTOR_DPL_SHIFT 5 -#define SVM_SELECTOR_P_SHIFT 7 -#define SVM_SELECTOR_AVL_SHIFT 8 -#define SVM_SELECTOR_L_SHIFT 9 -#define SVM_SELECTOR_DB_SHIFT 10 -#define SVM_SELECTOR_G_SHIFT 11 - -#define SVM_SELECTOR_TYPE_MASK (0xf) -#define SVM_SELECTOR_S_MASK (1 << SVM_SELECTOR_S_SHIFT) -#define SVM_SELECTOR_DPL_MASK (3 << SVM_SELECTOR_DPL_SHIFT) -#define SVM_SELECTOR_P_MASK (1 << SVM_SELECTOR_P_SHIFT) -#define SVM_SELECTOR_AVL_MASK (1 << SVM_SELECTOR_AVL_SHIFT) -#define SVM_SELECTOR_L_MASK (1 << SVM_SELECTOR_L_SHIFT) -#define SVM_SELECTOR_DB_MASK (1 << SVM_SELECTOR_DB_SHIFT) -#define SVM_SELECTOR_G_MASK (1 << SVM_SELECTOR_G_SHIFT) - -#define SVM_SELECTOR_WRITE_MASK (1 << 1) -#define SVM_SELECTOR_READ_MASK SVM_SELECTOR_WRITE_MASK -#define SVM_SELECTOR_CODE_MASK (1 << 3) - -#define INTERCEPT_CR0_MASK 1 -#define INTERCEPT_CR3_MASK (1 << 3) -#define INTERCEPT_CR4_MASK (1 << 4) -#define INTERCEPT_CR8_MASK (1 << 8) - -#define INTERCEPT_DR0_MASK 1 -#define INTERCEPT_DR1_MASK (1 << 1) -#define INTERCEPT_DR2_MASK (1 << 2) -#define INTERCEPT_DR3_MASK (1 << 3) -#define INTERCEPT_DR4_MASK (1 << 4) -#define INTERCEPT_DR5_MASK (1 << 5) -#define INTERCEPT_DR6_MASK (1 << 6) -#define INTERCEPT_DR7_MASK (1 << 7) - -#define SVM_EVTINJ_VEC_MASK 0xff - -#define SVM_EVTINJ_TYPE_SHIFT 8 -#define SVM_EVTINJ_TYPE_MASK (7 << SVM_EVTINJ_TYPE_SHIFT) - -#define SVM_EVTINJ_TYPE_INTR (0 << SVM_EVTINJ_TYPE_SHIFT) -#define SVM_EVTINJ_TYPE_NMI (2 << SVM_EVTINJ_TYPE_SHIFT) -#define SVM_EVTINJ_TYPE_EXEPT (3 << SVM_EVTINJ_TYPE_SHIFT) -#define SVM_EVTINJ_TYPE_SOFT (4 << SVM_EVTINJ_TYPE_SHIFT) - -#define SVM_EVTINJ_VALID (1 << 31) -#define SVM_EVTINJ_VALID_ERR (1 << 11) - -#define SVM_EXITINTINFO_VEC_MASK SVM_EVTINJ_VEC_MASK - -#define SVM_EXITINTINFO_TYPE_INTR SVM_EVTINJ_TYPE_INTR -#define SVM_EXITINTINFO_TYPE_NMI SVM_EVTINJ_TYPE_NMI -#define SVM_EXITINTINFO_TYPE_EXEPT SVM_EVTINJ_TYPE_EXEPT -#define SVM_EXITINTINFO_TYPE_SOFT SVM_EVTINJ_TYPE_SOFT - -#define SVM_EXITINTINFO_VALID SVM_EVTINJ_VALID -#define SVM_EXITINTINFO_VALID_ERR SVM_EVTINJ_VALID_ERR - -#define SVM_EXIT_READ_CR0 0x000 -#define SVM_EXIT_READ_CR3 0x003 -#define SVM_EXIT_READ_CR4 0x004 -#define SVM_EXIT_READ_CR8 0x008 -#define SVM_EXIT_WRITE_CR0 0x010 -#define SVM_EXIT_WRITE_CR3 0x013 -#define SVM_EXIT_WRITE_CR4 0x014 -#define SVM_EXIT_WRITE_CR8 0x018 -#define SVM_EXIT_READ_DR0 0x020 -#define SVM_EXIT_READ_DR1 0x021 -#define SVM_EXIT_READ_DR2 0x022 -#define SVM_EXIT_READ_DR3 0x023 -#define SVM_EXIT_READ_DR4 0x024 -#define SVM_EXIT_READ_DR5 0x025 -#define SVM_EXIT_READ_DR6 0x026 -#define SVM_EXIT_READ_DR7 0x027 -#define SVM_EXIT_WRITE_DR0 0x030 -#define SVM_EXIT_WRITE_DR1 0x031 -#define SVM_EXIT_WRITE_DR2 0x032 -#define SVM_EXIT_WRITE_DR3 0x033 -#define SVM_EXIT_WRITE_DR4 0x034 -#define SVM_EXIT_WRITE_DR5 0x035 -#define SVM_EXIT_WRITE_DR6 0x036 -#define SVM_EXIT_WRITE_DR7 0x037 -#define SVM_EXIT_EXCP_BASE 0x040 -#define SVM_EXIT_INTR 0x060 -#define SVM_EXIT_NMI 0x061 -#define SVM_EXIT_SMI 0x062 -#define SVM_EXIT_INIT 0x063 -#define SVM_EXIT_VINTR 0x064 -#define SVM_EXIT_CR0_SEL_WRITE 0x065 -#define SVM_EXIT_IDTR_READ 0x066 -#define SVM_EXIT_GDTR_READ 0x067 -#define SVM_EXIT_LDTR_READ 0x068 -#define SVM_EXIT_TR_READ 0x069 -#define SVM_EXIT_IDTR_WRITE 0x06a -#define SVM_EXIT_GDTR_WRITE 0x06b -#define SVM_EXIT_LDTR_WRITE 0x06c -#define SVM_EXIT_TR_WRITE 0x06d -#define SVM_EXIT_RDTSC 0x06e -#define SVM_EXIT_RDPMC 0x06f -#define SVM_EXIT_PUSHF 0x070 -#define SVM_EXIT_POPF 0x071 -#define SVM_EXIT_CPUID 0x072 -#define SVM_EXIT_RSM 0x073 -#define SVM_EXIT_IRET 0x074 -#define SVM_EXIT_SWINT 0x075 -#define SVM_EXIT_INVD 0x076 -#define SVM_EXIT_PAUSE 0x077 -#define SVM_EXIT_HLT 0x078 -#define SVM_EXIT_INVLPG 0x079 -#define SVM_EXIT_INVLPGA 0x07a -#define SVM_EXIT_IOIO 0x07b -#define SVM_EXIT_MSR 0x07c -#define SVM_EXIT_TASK_SWITCH 0x07d -#define SVM_EXIT_FERR_FREEZE 0x07e -#define SVM_EXIT_SHUTDOWN 0x07f -#define SVM_EXIT_VMRUN 0x080 -#define SVM_EXIT_VMMCALL 0x081 -#define SVM_EXIT_VMLOAD 0x082 -#define SVM_EXIT_VMSAVE 0x083 -#define SVM_EXIT_STGI 0x084 -#define SVM_EXIT_CLGI 0x085 -#define SVM_EXIT_SKINIT 0x086 -#define SVM_EXIT_RDTSCP 0x087 -#define SVM_EXIT_ICEBP 0x088 -#define SVM_EXIT_WBINVD 0x089 -#define SVM_EXIT_MONITOR 0x08a -#define SVM_EXIT_MWAIT 0x08b -#define SVM_EXIT_MWAIT_COND 0x08c -#define SVM_EXIT_NPF 0x400 - -#define SVM_EXIT_ERR -1 - -#define SVM_CR0_SELECTIVE_MASK (1 << 3 | 1) /* TS and MP */ - -#define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda" -#define SVM_VMRUN ".byte 0x0f, 0x01, 0xd8" -#define SVM_VMSAVE ".byte 0x0f, 0x01, 0xdb" -#define SVM_CLGI ".byte 0x0f, 0x01, 0xdd" -#define SVM_STGI ".byte 0x0f, 0x01, 0xdc" -#define SVM_INVLPGA ".byte 0x0f, 0x01, 0xdf" - -#endif - diff --git a/drivers/kvm/types.h b/drivers/kvm/types.h deleted file mode 100644 index 1c4e46decb22..000000000000 --- a/drivers/kvm/types.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ - -#ifndef __KVM_TYPES_H__ -#define __KVM_TYPES_H__ - -#include - -/* - * Address types: - * - * gva - guest virtual address - * gpa - guest physical address - * gfn - guest frame number - * hva - host virtual address - * hpa - host physical address - * hfn - host frame number - */ - -typedef unsigned long gva_t; -typedef u64 gpa_t; -typedef unsigned long gfn_t; - -typedef unsigned long hva_t; -typedef u64 hpa_t; -typedef unsigned long hfn_t; - -struct kvm_pio_request { - unsigned long count; - int cur_count; - struct page *guest_pages[2]; - unsigned guest_page_offset; - int in; - int port; - int size; - int string; - int down; - int rep; -}; - -#endif /* __KVM_TYPES_H__ */ diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c deleted file mode 100644 index 11ca2340d38f..000000000000 --- a/drivers/kvm/vmx.c +++ /dev/null @@ -1,2673 +0,0 @@ -/* - * Kernel-based Virtual Machine driver for Linux - * - * This module enables machines with Intel VT-x extensions to run virtual - * machines without emulation or binary translation. - * - * Copyright (C) 2006 Qumranet, Inc. - * - * Authors: - * Avi Kivity - * Yaniv Kamay - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - * - */ - -#include "kvm.h" -#include "x86.h" -#include "x86_emulate.h" -#include "irq.h" -#include "vmx.h" -#include "segment_descriptor.h" -#include "mmu.h" - -#include -#include -#include -#include -#include -#include - -#include -#include - -MODULE_AUTHOR("Qumranet"); -MODULE_LICENSE("GPL"); - -static int bypass_guest_pf = 1; -module_param(bypass_guest_pf, bool, 0); - -struct vmcs { - u32 revision_id; - u32 abort; - char data[0]; -}; - -struct vcpu_vmx { - struct kvm_vcpu vcpu; - int launched; - u8 fail; - u32 idt_vectoring_info; - struct kvm_msr_entry *guest_msrs; - struct kvm_msr_entry *host_msrs; - int nmsrs; - int save_nmsrs; - int msr_offset_efer; -#ifdef CONFIG_X86_64 - int msr_offset_kernel_gs_base; -#endif - struct vmcs *vmcs; - struct { - int loaded; - u16 fs_sel, gs_sel, ldt_sel; - int gs_ldt_reload_needed; - int fs_reload_needed; - int guest_efer_loaded; - } host_state; - struct { - struct { - bool pending; - u8 vector; - unsigned rip; - } irq; - } rmode; -}; - -static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu) -{ - return container_of(vcpu, struct vcpu_vmx, vcpu); -} - -static int init_rmode_tss(struct kvm *kvm); - -static DEFINE_PER_CPU(struct vmcs *, vmxarea); -static DEFINE_PER_CPU(struct vmcs *, current_vmcs); - -static struct page *vmx_io_bitmap_a; -static struct page *vmx_io_bitmap_b; - -static struct vmcs_config { - int size; - int order; - u32 revision_id; - u32 pin_based_exec_ctrl; - u32 cpu_based_exec_ctrl; - u32 cpu_based_2nd_exec_ctrl; - u32 vmexit_ctrl; - u32 vmentry_ctrl; -} vmcs_config; - -#define VMX_SEGMENT_FIELD(seg) \ - [VCPU_SREG_##seg] = { \ - .selector = GUEST_##seg##_SELECTOR, \ - .base = GUEST_##seg##_BASE, \ - .limit = GUEST_##seg##_LIMIT, \ - .ar_bytes = GUEST_##seg##_AR_BYTES, \ - } - -static struct kvm_vmx_segment_field { - unsigned selector; - unsigned base; - unsigned limit; - unsigned ar_bytes; -} kvm_vmx_segment_fields[] = { - VMX_SEGMENT_FIELD(CS), - VMX_SEGMENT_FIELD(DS), - VMX_SEGMENT_FIELD(ES), - VMX_SEGMENT_FIELD(FS), - VMX_SEGMENT_FIELD(GS), - VMX_SEGMENT_FIELD(SS), - VMX_SEGMENT_FIELD(TR), - VMX_SEGMENT_FIELD(LDTR), -}; - -/* - * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it - * away by decrementing the array size. - */ -static const u32 vmx_msr_index[] = { -#ifdef CONFIG_X86_64 - MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, MSR_KERNEL_GS_BASE, -#endif - MSR_EFER, MSR_K6_STAR, -}; -#define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index) - -static void load_msrs(struct kvm_msr_entry *e, int n) -{ - int i; - - for (i = 0; i < n; ++i) - wrmsrl(e[i].index, e[i].data); -} - -static void save_msrs(struct kvm_msr_entry *e, int n) -{ - int i; - - for (i = 0; i < n; ++i) - rdmsrl(e[i].index, e[i].data); -} - -static inline int is_page_fault(u32 intr_info) -{ - return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | - INTR_INFO_VALID_MASK)) == - (INTR_TYPE_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK); -} - -static inline int is_no_device(u32 intr_info) -{ - return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | - INTR_INFO_VALID_MASK)) == - (INTR_TYPE_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK); -} - -static inline int is_invalid_opcode(u32 intr_info) -{ - return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | - INTR_INFO_VALID_MASK)) == - (INTR_TYPE_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK); -} - -static inline int is_external_interrupt(u32 intr_info) -{ - return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK)) - == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK); -} - -static inline int cpu_has_vmx_tpr_shadow(void) -{ - return (vmcs_config.cpu_based_exec_ctrl & CPU_BASED_TPR_SHADOW); -} - -static inline int vm_need_tpr_shadow(struct kvm *kvm) -{ - return ((cpu_has_vmx_tpr_shadow()) && (irqchip_in_kernel(kvm))); -} - -static inline int cpu_has_secondary_exec_ctrls(void) -{ - return (vmcs_config.cpu_based_exec_ctrl & - CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); -} - -static inline int cpu_has_vmx_virtualize_apic_accesses(void) -{ - return (vmcs_config.cpu_based_2nd_exec_ctrl & - SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); -} - -static inline int vm_need_virtualize_apic_accesses(struct kvm *kvm) -{ - return ((cpu_has_vmx_virtualize_apic_accesses()) && - (irqchip_in_kernel(kvm))); -} - -static int __find_msr_index(struct vcpu_vmx *vmx, u32 msr) -{ - int i; - - for (i = 0; i < vmx->nmsrs; ++i) - if (vmx->guest_msrs[i].index == msr) - return i; - return -1; -} - -static struct kvm_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr) -{ - int i; - - i = __find_msr_index(vmx, msr); - if (i >= 0) - return &vmx->guest_msrs[i]; - return NULL; -} - -static void vmcs_clear(struct vmcs *vmcs) -{ - u64 phys_addr = __pa(vmcs); - u8 error; - - asm volatile (ASM_VMX_VMCLEAR_RAX "; setna %0" - : "=g"(error) : "a"(&phys_addr), "m"(phys_addr) - : "cc", "memory"); - if (error) - printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n", - vmcs, phys_addr); -} - -static void __vcpu_clear(void *arg) -{ - struct vcpu_vmx *vmx = arg; - int cpu = raw_smp_processor_id(); - - if (vmx->vcpu.cpu == cpu) - vmcs_clear(vmx->vmcs); - if (per_cpu(current_vmcs, cpu) == vmx->vmcs) - per_cpu(current_vmcs, cpu) = NULL; - rdtscll(vmx->vcpu.arch.host_tsc); -} - -static void vcpu_clear(struct vcpu_vmx *vmx) -{ - if (vmx->vcpu.cpu == -1) - return; - smp_call_function_single(vmx->vcpu.cpu, __vcpu_clear, vmx, 0, 1); - vmx->launched = 0; -} - -static unsigned long vmcs_readl(unsigned long field) -{ - unsigned long value; - - asm volatile (ASM_VMX_VMREAD_RDX_RAX - : "=a"(value) : "d"(field) : "cc"); - return value; -} - -static u16 vmcs_read16(unsigned long field) -{ - return vmcs_readl(field); -} - -static u32 vmcs_read32(unsigned long field) -{ - return vmcs_readl(field); -} - -static u64 vmcs_read64(unsigned long field) -{ -#ifdef CONFIG_X86_64 - return vmcs_readl(field); -#else - return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32); -#endif -} - -static noinline void vmwrite_error(unsigned long field, unsigned long value) -{ - printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n", - field, value, vmcs_read32(VM_INSTRUCTION_ERROR)); - dump_stack(); -} - -static void vmcs_writel(unsigned long field, unsigned long value) -{ - u8 error; - - asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0" - : "=q"(error) : "a"(value), "d"(field) : "cc"); - if (unlikely(error)) - vmwrite_error(field, value); -} - -static void vmcs_write16(unsigned long field, u16 value) -{ - vmcs_writel(field, value); -} - -static void vmcs_write32(unsigned long field, u32 value) -{ - vmcs_writel(field, value); -} - -static void vmcs_write64(unsigned long field, u64 value) -{ -#ifdef CONFIG_X86_64 - vmcs_writel(field, value); -#else - vmcs_writel(field, value); - asm volatile (""); - vmcs_writel(field+1, value >> 32); -#endif -} - -static void vmcs_clear_bits(unsigned long field, u32 mask) -{ - vmcs_writel(field, vmcs_readl(field) & ~mask); -} - -static void vmcs_set_bits(unsigned long field, u32 mask) -{ - vmcs_writel(field, vmcs_readl(field) | mask); -} - -static void update_exception_bitmap(struct kvm_vcpu *vcpu) -{ - u32 eb; - - eb = (1u << PF_VECTOR) | (1u << UD_VECTOR); - if (!vcpu->fpu_active) - eb |= 1u << NM_VECTOR; - if (vcpu->guest_debug.enabled) - eb |= 1u << 1; - if (vcpu->arch.rmode.active) - eb = ~0; - vmcs_write32(EXCEPTION_BITMAP, eb); -} - -static void reload_tss(void) -{ -#ifndef CONFIG_X86_64 - - /* - * VT restores TR but not its size. Useless. - */ - struct descriptor_table gdt; - struct segment_descriptor *descs; - - get_gdt(&gdt); - descs = (void *)gdt.base; - descs[GDT_ENTRY_TSS].type = 9; /* available TSS */ - load_TR_desc(); -#endif -} - -static void load_transition_efer(struct vcpu_vmx *vmx) -{ - int efer_offset = vmx->msr_offset_efer; - u64 host_efer = vmx->host_msrs[efer_offset].data; - u64 guest_efer = vmx->guest_msrs[efer_offset].data; - u64 ignore_bits; - - if (efer_offset < 0) - return; - /* - * NX is emulated; LMA and LME handled by hardware; SCE meaninless - * outside long mode - */ - ignore_bits = EFER_NX | EFER_SCE; -#ifdef CONFIG_X86_64 - ignore_bits |= EFER_LMA | EFER_LME; - /* SCE is meaningful only in long mode on Intel */ - if (guest_efer & EFER_LMA) - ignore_bits &= ~(u64)EFER_SCE; -#endif - if ((guest_efer & ~ignore_bits) == (host_efer & ~ignore_bits)) - return; - - vmx->host_state.guest_efer_loaded = 1; - guest_efer &= ~ignore_bits; - guest_efer |= host_efer & ignore_bits; - wrmsrl(MSR_EFER, guest_efer); - vmx->vcpu.stat.efer_reload++; -} - -static void reload_host_efer(struct vcpu_vmx *vmx) -{ - if (vmx->host_state.guest_efer_loaded) { - vmx->host_state.guest_efer_loaded = 0; - load_msrs(vmx->host_msrs + vmx->msr_offset_efer, 1); - } -} - -static void vmx_save_host_state(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - if (vmx->host_state.loaded) - return; - - vmx->host_state.loaded = 1; - /* - * Set host fs and gs selectors. Unfortunately, 22.2.3 does not - * allow segment selectors with cpl > 0 or ti == 1. - */ - vmx->host_state.ldt_sel = read_ldt(); - vmx->host_state.gs_ldt_reload_needed = vmx->host_state.ldt_sel; - vmx->host_state.fs_sel = read_fs(); - if (!(vmx->host_state.fs_sel & 7)) { - vmcs_write16(HOST_FS_SELECTOR, vmx->host_state.fs_sel); - vmx->host_state.fs_reload_needed = 0; - } else { - vmcs_write16(HOST_FS_SELECTOR, 0); - vmx->host_state.fs_reload_needed = 1; - } - vmx->host_state.gs_sel = read_gs(); - if (!(vmx->host_state.gs_sel & 7)) - vmcs_write16(HOST_GS_SELECTOR, vmx->host_state.gs_sel); - else { - vmcs_write16(HOST_GS_SELECTOR, 0); - vmx->host_state.gs_ldt_reload_needed = 1; - } - -#ifdef CONFIG_X86_64 - vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE)); - vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE)); -#else - vmcs_writel(HOST_FS_BASE, segment_base(vmx->host_state.fs_sel)); - vmcs_writel(HOST_GS_BASE, segment_base(vmx->host_state.gs_sel)); -#endif - -#ifdef CONFIG_X86_64 - if (is_long_mode(&vmx->vcpu)) - save_msrs(vmx->host_msrs + - vmx->msr_offset_kernel_gs_base, 1); - -#endif - load_msrs(vmx->guest_msrs, vmx->save_nmsrs); - load_transition_efer(vmx); -} - -static void vmx_load_host_state(struct vcpu_vmx *vmx) -{ - unsigned long flags; - - if (!vmx->host_state.loaded) - return; - - ++vmx->vcpu.stat.host_state_reload; - vmx->host_state.loaded = 0; - if (vmx->host_state.fs_reload_needed) - load_fs(vmx->host_state.fs_sel); - if (vmx->host_state.gs_ldt_reload_needed) { - load_ldt(vmx->host_state.ldt_sel); - /* - * If we have to reload gs, we must take care to - * preserve our gs base. - */ - local_irq_save(flags); - load_gs(vmx->host_state.gs_sel); -#ifdef CONFIG_X86_64 - wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE)); -#endif - local_irq_restore(flags); - } - reload_tss(); - save_msrs(vmx->guest_msrs, vmx->save_nmsrs); - load_msrs(vmx->host_msrs, vmx->save_nmsrs); - reload_host_efer(vmx); -} - -/* - * Switches to specified vcpu, until a matching vcpu_put(), but assumes - * vcpu mutex is already taken. - */ -static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u64 phys_addr = __pa(vmx->vmcs); - u64 tsc_this, delta; - - if (vcpu->cpu != cpu) { - vcpu_clear(vmx); - kvm_migrate_apic_timer(vcpu); - } - - if (per_cpu(current_vmcs, cpu) != vmx->vmcs) { - u8 error; - - per_cpu(current_vmcs, cpu) = vmx->vmcs; - asm volatile (ASM_VMX_VMPTRLD_RAX "; setna %0" - : "=g"(error) : "a"(&phys_addr), "m"(phys_addr) - : "cc"); - if (error) - printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n", - vmx->vmcs, phys_addr); - } - - if (vcpu->cpu != cpu) { - struct descriptor_table dt; - unsigned long sysenter_esp; - - vcpu->cpu = cpu; - /* - * Linux uses per-cpu TSS and GDT, so set these when switching - * processors. - */ - vmcs_writel(HOST_TR_BASE, read_tr_base()); /* 22.2.4 */ - get_gdt(&dt); - vmcs_writel(HOST_GDTR_BASE, dt.base); /* 22.2.4 */ - - rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp); - vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */ - - /* - * Make sure the time stamp counter is monotonous. - */ - rdtscll(tsc_this); - delta = vcpu->arch.host_tsc - tsc_this; - vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta); - } -} - -static void vmx_vcpu_put(struct kvm_vcpu *vcpu) -{ - vmx_load_host_state(to_vmx(vcpu)); -} - -static void vmx_fpu_activate(struct kvm_vcpu *vcpu) -{ - if (vcpu->fpu_active) - return; - vcpu->fpu_active = 1; - vmcs_clear_bits(GUEST_CR0, X86_CR0_TS); - if (vcpu->arch.cr0 & X86_CR0_TS) - vmcs_set_bits(GUEST_CR0, X86_CR0_TS); - update_exception_bitmap(vcpu); -} - -static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu) -{ - if (!vcpu->fpu_active) - return; - vcpu->fpu_active = 0; - vmcs_set_bits(GUEST_CR0, X86_CR0_TS); - update_exception_bitmap(vcpu); -} - -static void vmx_vcpu_decache(struct kvm_vcpu *vcpu) -{ - vcpu_clear(to_vmx(vcpu)); -} - -static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu) -{ - return vmcs_readl(GUEST_RFLAGS); -} - -static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) -{ - if (vcpu->arch.rmode.active) - rflags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; - vmcs_writel(GUEST_RFLAGS, rflags); -} - -static void skip_emulated_instruction(struct kvm_vcpu *vcpu) -{ - unsigned long rip; - u32 interruptibility; - - rip = vmcs_readl(GUEST_RIP); - rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN); - vmcs_writel(GUEST_RIP, rip); - - /* - * We emulated an instruction, so temporary interrupt blocking - * should be removed, if set. - */ - interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); - if (interruptibility & 3) - vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, - interruptibility & ~3); - vcpu->arch.interrupt_window_open = 1; -} - -static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, - bool has_error_code, u32 error_code) -{ - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - nr | INTR_TYPE_EXCEPTION - | (has_error_code ? INTR_INFO_DELIEVER_CODE_MASK : 0) - | INTR_INFO_VALID_MASK); - if (has_error_code) - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); -} - -static bool vmx_exception_injected(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - return !(vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK); -} - -/* - * Swap MSR entry in host/guest MSR entry array. - */ -#ifdef CONFIG_X86_64 -static void move_msr_up(struct vcpu_vmx *vmx, int from, int to) -{ - struct kvm_msr_entry tmp; - - tmp = vmx->guest_msrs[to]; - vmx->guest_msrs[to] = vmx->guest_msrs[from]; - vmx->guest_msrs[from] = tmp; - tmp = vmx->host_msrs[to]; - vmx->host_msrs[to] = vmx->host_msrs[from]; - vmx->host_msrs[from] = tmp; -} -#endif - -/* - * Set up the vmcs to automatically save and restore system - * msrs. Don't touch the 64-bit msrs if the guest is in legacy - * mode, as fiddling with msrs is very expensive. - */ -static void setup_msrs(struct vcpu_vmx *vmx) -{ - int save_nmsrs; - - save_nmsrs = 0; -#ifdef CONFIG_X86_64 - if (is_long_mode(&vmx->vcpu)) { - int index; - - index = __find_msr_index(vmx, MSR_SYSCALL_MASK); - if (index >= 0) - move_msr_up(vmx, index, save_nmsrs++); - index = __find_msr_index(vmx, MSR_LSTAR); - if (index >= 0) - move_msr_up(vmx, index, save_nmsrs++); - index = __find_msr_index(vmx, MSR_CSTAR); - if (index >= 0) - move_msr_up(vmx, index, save_nmsrs++); - index = __find_msr_index(vmx, MSR_KERNEL_GS_BASE); - if (index >= 0) - move_msr_up(vmx, index, save_nmsrs++); - /* - * MSR_K6_STAR is only needed on long mode guests, and only - * if efer.sce is enabled. - */ - index = __find_msr_index(vmx, MSR_K6_STAR); - if ((index >= 0) && (vmx->vcpu.arch.shadow_efer & EFER_SCE)) - move_msr_up(vmx, index, save_nmsrs++); - } -#endif - vmx->save_nmsrs = save_nmsrs; - -#ifdef CONFIG_X86_64 - vmx->msr_offset_kernel_gs_base = - __find_msr_index(vmx, MSR_KERNEL_GS_BASE); -#endif - vmx->msr_offset_efer = __find_msr_index(vmx, MSR_EFER); -} - -/* - * reads and returns guest's timestamp counter "register" - * guest_tsc = host_tsc + tsc_offset -- 21.3 - */ -static u64 guest_read_tsc(void) -{ - u64 host_tsc, tsc_offset; - - rdtscll(host_tsc); - tsc_offset = vmcs_read64(TSC_OFFSET); - return host_tsc + tsc_offset; -} - -/* - * writes 'guest_tsc' into guest's timestamp counter "register" - * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc - */ -static void guest_write_tsc(u64 guest_tsc) -{ - u64 host_tsc; - - rdtscll(host_tsc); - vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc); -} - -/* - * Reads an msr value (of 'msr_index') into 'pdata'. - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) -{ - u64 data; - struct kvm_msr_entry *msr; - - if (!pdata) { - printk(KERN_ERR "BUG: get_msr called with NULL pdata\n"); - return -EINVAL; - } - - switch (msr_index) { -#ifdef CONFIG_X86_64 - case MSR_FS_BASE: - data = vmcs_readl(GUEST_FS_BASE); - break; - case MSR_GS_BASE: - data = vmcs_readl(GUEST_GS_BASE); - break; - case MSR_EFER: - return kvm_get_msr_common(vcpu, msr_index, pdata); -#endif - case MSR_IA32_TIME_STAMP_COUNTER: - data = guest_read_tsc(); - break; - case MSR_IA32_SYSENTER_CS: - data = vmcs_read32(GUEST_SYSENTER_CS); - break; - case MSR_IA32_SYSENTER_EIP: - data = vmcs_readl(GUEST_SYSENTER_EIP); - break; - case MSR_IA32_SYSENTER_ESP: - data = vmcs_readl(GUEST_SYSENTER_ESP); - break; - default: - msr = find_msr_entry(to_vmx(vcpu), msr_index); - if (msr) { - data = msr->data; - break; - } - return kvm_get_msr_common(vcpu, msr_index, pdata); - } - - *pdata = data; - return 0; -} - -/* - * Writes msr value into into the appropriate "register". - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - struct kvm_msr_entry *msr; - int ret = 0; - - switch (msr_index) { -#ifdef CONFIG_X86_64 - case MSR_EFER: - ret = kvm_set_msr_common(vcpu, msr_index, data); - if (vmx->host_state.loaded) { - reload_host_efer(vmx); - load_transition_efer(vmx); - } - break; - case MSR_FS_BASE: - vmcs_writel(GUEST_FS_BASE, data); - break; - case MSR_GS_BASE: - vmcs_writel(GUEST_GS_BASE, data); - break; -#endif - case MSR_IA32_SYSENTER_CS: - vmcs_write32(GUEST_SYSENTER_CS, data); - break; - case MSR_IA32_SYSENTER_EIP: - vmcs_writel(GUEST_SYSENTER_EIP, data); - break; - case MSR_IA32_SYSENTER_ESP: - vmcs_writel(GUEST_SYSENTER_ESP, data); - break; - case MSR_IA32_TIME_STAMP_COUNTER: - guest_write_tsc(data); - break; - default: - msr = find_msr_entry(vmx, msr_index); - if (msr) { - msr->data = data; - if (vmx->host_state.loaded) - load_msrs(vmx->guest_msrs, vmx->save_nmsrs); - break; - } - ret = kvm_set_msr_common(vcpu, msr_index, data); - } - - return ret; -} - -/* - * Sync the rsp and rip registers into the vcpu structure. This allows - * registers to be accessed by indexing vcpu->arch.regs. - */ -static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu) -{ - vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP); - vcpu->arch.rip = vmcs_readl(GUEST_RIP); -} - -/* - * Syncs rsp and rip back into the vmcs. Should be called after possible - * modification. - */ -static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu) -{ - vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]); - vmcs_writel(GUEST_RIP, vcpu->arch.rip); -} - -static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) -{ - unsigned long dr7 = 0x400; - int old_singlestep; - - old_singlestep = vcpu->guest_debug.singlestep; - - vcpu->guest_debug.enabled = dbg->enabled; - if (vcpu->guest_debug.enabled) { - int i; - - dr7 |= 0x200; /* exact */ - for (i = 0; i < 4; ++i) { - if (!dbg->breakpoints[i].enabled) - continue; - vcpu->guest_debug.bp[i] = dbg->breakpoints[i].address; - dr7 |= 2 << (i*2); /* global enable */ - dr7 |= 0 << (i*4+16); /* execution breakpoint */ - } - - vcpu->guest_debug.singlestep = dbg->singlestep; - } else - vcpu->guest_debug.singlestep = 0; - - if (old_singlestep && !vcpu->guest_debug.singlestep) { - unsigned long flags; - - flags = vmcs_readl(GUEST_RFLAGS); - flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); - vmcs_writel(GUEST_RFLAGS, flags); - } - - update_exception_bitmap(vcpu); - vmcs_writel(GUEST_DR7, dr7); - - return 0; -} - -static int vmx_get_irq(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u32 idtv_info_field; - - idtv_info_field = vmx->idt_vectoring_info; - if (idtv_info_field & INTR_INFO_VALID_MASK) { - if (is_external_interrupt(idtv_info_field)) - return idtv_info_field & VECTORING_INFO_VECTOR_MASK; - else - printk(KERN_DEBUG "pending exception: not handled yet\n"); - } - return -1; -} - -static __init int cpu_has_kvm_support(void) -{ - unsigned long ecx = cpuid_ecx(1); - return test_bit(5, &ecx); /* CPUID.1:ECX.VMX[bit 5] -> VT */ -} - -static __init int vmx_disabled_by_bios(void) -{ - u64 msr; - - rdmsrl(MSR_IA32_FEATURE_CONTROL, msr); - return (msr & (MSR_IA32_FEATURE_CONTROL_LOCKED | - MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED)) - == MSR_IA32_FEATURE_CONTROL_LOCKED; - /* locked but not enabled */ -} - -static void hardware_enable(void *garbage) -{ - int cpu = raw_smp_processor_id(); - u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); - u64 old; - - rdmsrl(MSR_IA32_FEATURE_CONTROL, old); - if ((old & (MSR_IA32_FEATURE_CONTROL_LOCKED | - MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED)) - != (MSR_IA32_FEATURE_CONTROL_LOCKED | - MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED)) - /* enable and lock */ - wrmsrl(MSR_IA32_FEATURE_CONTROL, old | - MSR_IA32_FEATURE_CONTROL_LOCKED | - MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED); - write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */ - asm volatile (ASM_VMX_VMXON_RAX : : "a"(&phys_addr), "m"(phys_addr) - : "memory", "cc"); -} - -static void hardware_disable(void *garbage) -{ - asm volatile (ASM_VMX_VMXOFF : : : "cc"); -} - -static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt, - u32 msr, u32 *result) -{ - u32 vmx_msr_low, vmx_msr_high; - u32 ctl = ctl_min | ctl_opt; - - rdmsr(msr, vmx_msr_low, vmx_msr_high); - - ctl &= vmx_msr_high; /* bit == 0 in high word ==> must be zero */ - ctl |= vmx_msr_low; /* bit == 1 in low word ==> must be one */ - - /* Ensure minimum (required) set of control bits are supported. */ - if (ctl_min & ~ctl) - return -EIO; - - *result = ctl; - return 0; -} - -static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf) -{ - u32 vmx_msr_low, vmx_msr_high; - u32 min, opt; - u32 _pin_based_exec_control = 0; - u32 _cpu_based_exec_control = 0; - u32 _cpu_based_2nd_exec_control = 0; - u32 _vmexit_control = 0; - u32 _vmentry_control = 0; - - min = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING; - opt = 0; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PINBASED_CTLS, - &_pin_based_exec_control) < 0) - return -EIO; - - min = CPU_BASED_HLT_EXITING | -#ifdef CONFIG_X86_64 - CPU_BASED_CR8_LOAD_EXITING | - CPU_BASED_CR8_STORE_EXITING | -#endif - CPU_BASED_USE_IO_BITMAPS | - CPU_BASED_MOV_DR_EXITING | - CPU_BASED_USE_TSC_OFFSETING; - opt = CPU_BASED_TPR_SHADOW | - CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS, - &_cpu_based_exec_control) < 0) - return -EIO; -#ifdef CONFIG_X86_64 - if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) - _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING & - ~CPU_BASED_CR8_STORE_EXITING; -#endif - if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) { - min = 0; - opt = SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | - SECONDARY_EXEC_WBINVD_EXITING; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_PROCBASED_CTLS2, - &_cpu_based_2nd_exec_control) < 0) - return -EIO; - } -#ifndef CONFIG_X86_64 - if (!(_cpu_based_2nd_exec_control & - SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) - _cpu_based_exec_control &= ~CPU_BASED_TPR_SHADOW; -#endif - - min = 0; -#ifdef CONFIG_X86_64 - min |= VM_EXIT_HOST_ADDR_SPACE_SIZE; -#endif - opt = 0; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_EXIT_CTLS, - &_vmexit_control) < 0) - return -EIO; - - min = opt = 0; - if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS, - &_vmentry_control) < 0) - return -EIO; - - rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high); - - /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ - if ((vmx_msr_high & 0x1fff) > PAGE_SIZE) - return -EIO; - -#ifdef CONFIG_X86_64 - /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */ - if (vmx_msr_high & (1u<<16)) - return -EIO; -#endif - - /* Require Write-Back (WB) memory type for VMCS accesses. */ - if (((vmx_msr_high >> 18) & 15) != 6) - return -EIO; - - vmcs_conf->size = vmx_msr_high & 0x1fff; - vmcs_conf->order = get_order(vmcs_config.size); - vmcs_conf->revision_id = vmx_msr_low; - - vmcs_conf->pin_based_exec_ctrl = _pin_based_exec_control; - vmcs_conf->cpu_based_exec_ctrl = _cpu_based_exec_control; - vmcs_conf->cpu_based_2nd_exec_ctrl = _cpu_based_2nd_exec_control; - vmcs_conf->vmexit_ctrl = _vmexit_control; - vmcs_conf->vmentry_ctrl = _vmentry_control; - - return 0; -} - -static struct vmcs *alloc_vmcs_cpu(int cpu) -{ - int node = cpu_to_node(cpu); - struct page *pages; - struct vmcs *vmcs; - - pages = alloc_pages_node(node, GFP_KERNEL, vmcs_config.order); - if (!pages) - return NULL; - vmcs = page_address(pages); - memset(vmcs, 0, vmcs_config.size); - vmcs->revision_id = vmcs_config.revision_id; /* vmcs revision id */ - return vmcs; -} - -static struct vmcs *alloc_vmcs(void) -{ - return alloc_vmcs_cpu(raw_smp_processor_id()); -} - -static void free_vmcs(struct vmcs *vmcs) -{ - free_pages((unsigned long)vmcs, vmcs_config.order); -} - -static void free_kvm_area(void) -{ - int cpu; - - for_each_online_cpu(cpu) - free_vmcs(per_cpu(vmxarea, cpu)); -} - -static __init int alloc_kvm_area(void) -{ - int cpu; - - for_each_online_cpu(cpu) { - struct vmcs *vmcs; - - vmcs = alloc_vmcs_cpu(cpu); - if (!vmcs) { - free_kvm_area(); - return -ENOMEM; - } - - per_cpu(vmxarea, cpu) = vmcs; - } - return 0; -} - -static __init int hardware_setup(void) -{ - if (setup_vmcs_config(&vmcs_config) < 0) - return -EIO; - return alloc_kvm_area(); -} - -static __exit void hardware_unsetup(void) -{ - free_kvm_area(); -} - -static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save) -{ - struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; - - if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) { - vmcs_write16(sf->selector, save->selector); - vmcs_writel(sf->base, save->base); - vmcs_write32(sf->limit, save->limit); - vmcs_write32(sf->ar_bytes, save->ar); - } else { - u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK) - << AR_DPL_SHIFT; - vmcs_write32(sf->ar_bytes, 0x93 | dpl); - } -} - -static void enter_pmode(struct kvm_vcpu *vcpu) -{ - unsigned long flags; - - vcpu->arch.rmode.active = 0; - - vmcs_writel(GUEST_TR_BASE, vcpu->arch.rmode.tr.base); - vmcs_write32(GUEST_TR_LIMIT, vcpu->arch.rmode.tr.limit); - vmcs_write32(GUEST_TR_AR_BYTES, vcpu->arch.rmode.tr.ar); - - flags = vmcs_readl(GUEST_RFLAGS); - flags &= ~(X86_EFLAGS_IOPL | X86_EFLAGS_VM); - flags |= (vcpu->arch.rmode.save_iopl << IOPL_SHIFT); - vmcs_writel(GUEST_RFLAGS, flags); - - vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~X86_CR4_VME) | - (vmcs_readl(CR4_READ_SHADOW) & X86_CR4_VME)); - - update_exception_bitmap(vcpu); - - fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->arch.rmode.es); - fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->arch.rmode.ds); - fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->arch.rmode.gs); - fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->arch.rmode.fs); - - vmcs_write16(GUEST_SS_SELECTOR, 0); - vmcs_write32(GUEST_SS_AR_BYTES, 0x93); - - vmcs_write16(GUEST_CS_SELECTOR, - vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK); - vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); -} - -static gva_t rmode_tss_base(struct kvm *kvm) -{ - if (!kvm->arch.tss_addr) { - gfn_t base_gfn = kvm->memslots[0].base_gfn + - kvm->memslots[0].npages - 3; - return base_gfn << PAGE_SHIFT; - } - return kvm->arch.tss_addr; -} - -static void fix_rmode_seg(int seg, struct kvm_save_segment *save) -{ - struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; - - save->selector = vmcs_read16(sf->selector); - save->base = vmcs_readl(sf->base); - save->limit = vmcs_read32(sf->limit); - save->ar = vmcs_read32(sf->ar_bytes); - vmcs_write16(sf->selector, save->base >> 4); - vmcs_write32(sf->base, save->base & 0xfffff); - vmcs_write32(sf->limit, 0xffff); - vmcs_write32(sf->ar_bytes, 0xf3); -} - -static void enter_rmode(struct kvm_vcpu *vcpu) -{ - unsigned long flags; - - vcpu->arch.rmode.active = 1; - - vcpu->arch.rmode.tr.base = vmcs_readl(GUEST_TR_BASE); - vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm)); - - vcpu->arch.rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT); - vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); - - vcpu->arch.rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES); - vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); - - flags = vmcs_readl(GUEST_RFLAGS); - vcpu->arch.rmode.save_iopl - = (flags & X86_EFLAGS_IOPL) >> IOPL_SHIFT; - - flags |= X86_EFLAGS_IOPL | X86_EFLAGS_VM; - - vmcs_writel(GUEST_RFLAGS, flags); - vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | X86_CR4_VME); - update_exception_bitmap(vcpu); - - vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4); - vmcs_write32(GUEST_SS_LIMIT, 0xffff); - vmcs_write32(GUEST_SS_AR_BYTES, 0xf3); - - vmcs_write32(GUEST_CS_AR_BYTES, 0xf3); - vmcs_write32(GUEST_CS_LIMIT, 0xffff); - if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000) - vmcs_writel(GUEST_CS_BASE, 0xf0000); - vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4); - - fix_rmode_seg(VCPU_SREG_ES, &vcpu->arch.rmode.es); - fix_rmode_seg(VCPU_SREG_DS, &vcpu->arch.rmode.ds); - fix_rmode_seg(VCPU_SREG_GS, &vcpu->arch.rmode.gs); - fix_rmode_seg(VCPU_SREG_FS, &vcpu->arch.rmode.fs); - - kvm_mmu_reset_context(vcpu); - init_rmode_tss(vcpu->kvm); -} - -#ifdef CONFIG_X86_64 - -static void enter_lmode(struct kvm_vcpu *vcpu) -{ - u32 guest_tr_ar; - - guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES); - if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) { - printk(KERN_DEBUG "%s: tss fixup for long mode. \n", - __FUNCTION__); - vmcs_write32(GUEST_TR_AR_BYTES, - (guest_tr_ar & ~AR_TYPE_MASK) - | AR_TYPE_BUSY_64_TSS); - } - - vcpu->arch.shadow_efer |= EFER_LMA; - - find_msr_entry(to_vmx(vcpu), MSR_EFER)->data |= EFER_LMA | EFER_LME; - vmcs_write32(VM_ENTRY_CONTROLS, - vmcs_read32(VM_ENTRY_CONTROLS) - | VM_ENTRY_IA32E_MODE); -} - -static void exit_lmode(struct kvm_vcpu *vcpu) -{ - vcpu->arch.shadow_efer &= ~EFER_LMA; - - vmcs_write32(VM_ENTRY_CONTROLS, - vmcs_read32(VM_ENTRY_CONTROLS) - & ~VM_ENTRY_IA32E_MODE); -} - -#endif - -static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) -{ - vcpu->arch.cr4 &= KVM_GUEST_CR4_MASK; - vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK; -} - -static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ - vmx_fpu_deactivate(vcpu); - - if (vcpu->arch.rmode.active && (cr0 & X86_CR0_PE)) - enter_pmode(vcpu); - - if (!vcpu->arch.rmode.active && !(cr0 & X86_CR0_PE)) - enter_rmode(vcpu); - -#ifdef CONFIG_X86_64 - if (vcpu->arch.shadow_efer & EFER_LME) { - if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) - enter_lmode(vcpu); - if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) - exit_lmode(vcpu); - } -#endif - - vmcs_writel(CR0_READ_SHADOW, cr0); - vmcs_writel(GUEST_CR0, - (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON); - vcpu->arch.cr0 = cr0; - - if (!(cr0 & X86_CR0_TS) || !(cr0 & X86_CR0_PE)) - vmx_fpu_activate(vcpu); -} - -static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - vmcs_writel(GUEST_CR3, cr3); - if (vcpu->arch.cr0 & X86_CR0_PE) - vmx_fpu_deactivate(vcpu); -} - -static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - vmcs_writel(CR4_READ_SHADOW, cr4); - vmcs_writel(GUEST_CR4, cr4 | (vcpu->arch.rmode.active ? - KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON)); - vcpu->arch.cr4 = cr4; -} - -#ifdef CONFIG_X86_64 - -static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); - - vcpu->arch.shadow_efer = efer; - if (efer & EFER_LMA) { - vmcs_write32(VM_ENTRY_CONTROLS, - vmcs_read32(VM_ENTRY_CONTROLS) | - VM_ENTRY_IA32E_MODE); - msr->data = efer; - - } else { - vmcs_write32(VM_ENTRY_CONTROLS, - vmcs_read32(VM_ENTRY_CONTROLS) & - ~VM_ENTRY_IA32E_MODE); - - msr->data = efer & ~EFER_LME; - } - setup_msrs(vmx); -} - -#endif - -static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) -{ - struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; - - return vmcs_readl(sf->base); -} - -static void vmx_get_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; - u32 ar; - - var->base = vmcs_readl(sf->base); - var->limit = vmcs_read32(sf->limit); - var->selector = vmcs_read16(sf->selector); - ar = vmcs_read32(sf->ar_bytes); - if (ar & AR_UNUSABLE_MASK) - ar = 0; - var->type = ar & 15; - var->s = (ar >> 4) & 1; - var->dpl = (ar >> 5) & 3; - var->present = (ar >> 7) & 1; - var->avl = (ar >> 12) & 1; - var->l = (ar >> 13) & 1; - var->db = (ar >> 14) & 1; - var->g = (ar >> 15) & 1; - var->unusable = (ar >> 16) & 1; -} - -static u32 vmx_segment_access_rights(struct kvm_segment *var) -{ - u32 ar; - - if (var->unusable) - ar = 1 << 16; - else { - ar = var->type & 15; - ar |= (var->s & 1) << 4; - ar |= (var->dpl & 3) << 5; - ar |= (var->present & 1) << 7; - ar |= (var->avl & 1) << 12; - ar |= (var->l & 1) << 13; - ar |= (var->db & 1) << 14; - ar |= (var->g & 1) << 15; - } - if (ar == 0) /* a 0 value means unusable */ - ar = AR_UNUSABLE_MASK; - - return ar; -} - -static void vmx_set_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; - u32 ar; - - if (vcpu->arch.rmode.active && seg == VCPU_SREG_TR) { - vcpu->arch.rmode.tr.selector = var->selector; - vcpu->arch.rmode.tr.base = var->base; - vcpu->arch.rmode.tr.limit = var->limit; - vcpu->arch.rmode.tr.ar = vmx_segment_access_rights(var); - return; - } - vmcs_writel(sf->base, var->base); - vmcs_write32(sf->limit, var->limit); - vmcs_write16(sf->selector, var->selector); - if (vcpu->arch.rmode.active && var->s) { - /* - * Hack real-mode segments into vm86 compatibility. - */ - if (var->base == 0xffff0000 && var->selector == 0xf000) - vmcs_writel(sf->base, 0xf0000); - ar = 0xf3; - } else - ar = vmx_segment_access_rights(var); - vmcs_write32(sf->ar_bytes, ar); -} - -static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) -{ - u32 ar = vmcs_read32(GUEST_CS_AR_BYTES); - - *db = (ar >> 14) & 1; - *l = (ar >> 13) & 1; -} - -static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) -{ - dt->limit = vmcs_read32(GUEST_IDTR_LIMIT); - dt->base = vmcs_readl(GUEST_IDTR_BASE); -} - -static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) -{ - vmcs_write32(GUEST_IDTR_LIMIT, dt->limit); - vmcs_writel(GUEST_IDTR_BASE, dt->base); -} - -static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) -{ - dt->limit = vmcs_read32(GUEST_GDTR_LIMIT); - dt->base = vmcs_readl(GUEST_GDTR_BASE); -} - -static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt) -{ - vmcs_write32(GUEST_GDTR_LIMIT, dt->limit); - vmcs_writel(GUEST_GDTR_BASE, dt->base); -} - -static int init_rmode_tss(struct kvm *kvm) -{ - gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT; - u16 data = 0; - int r; - - r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); - if (r < 0) - return 0; - data = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE; - r = kvm_write_guest_page(kvm, fn++, &data, 0x66, sizeof(u16)); - if (r < 0) - return 0; - r = kvm_clear_guest_page(kvm, fn++, 0, PAGE_SIZE); - if (r < 0) - return 0; - r = kvm_clear_guest_page(kvm, fn, 0, PAGE_SIZE); - if (r < 0) - return 0; - data = ~0; - r = kvm_write_guest_page(kvm, fn, &data, RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1, - sizeof(u8)); - if (r < 0) - return 0; - return 1; -} - -static void seg_setup(int seg) -{ - struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; - - vmcs_write16(sf->selector, 0); - vmcs_writel(sf->base, 0); - vmcs_write32(sf->limit, 0xffff); - vmcs_write32(sf->ar_bytes, 0x93); -} - -static int alloc_apic_access_page(struct kvm *kvm) -{ - struct kvm_userspace_memory_region kvm_userspace_mem; - int r = 0; - - mutex_lock(&kvm->lock); - if (kvm->arch.apic_access_page) - goto out; - kvm_userspace_mem.slot = APIC_ACCESS_PAGE_PRIVATE_MEMSLOT; - kvm_userspace_mem.flags = 0; - kvm_userspace_mem.guest_phys_addr = 0xfee00000ULL; - kvm_userspace_mem.memory_size = PAGE_SIZE; - r = __kvm_set_memory_region(kvm, &kvm_userspace_mem, 0); - if (r) - goto out; - kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00); -out: - mutex_unlock(&kvm->lock); - return r; -} - -/* - * Sets up the vmcs for emulated real mode. - */ -static int vmx_vcpu_setup(struct vcpu_vmx *vmx) -{ - u32 host_sysenter_cs; - u32 junk; - unsigned long a; - struct descriptor_table dt; - int i; - unsigned long kvm_vmx_return; - u32 exec_control; - - /* I/O */ - vmcs_write64(IO_BITMAP_A, page_to_phys(vmx_io_bitmap_a)); - vmcs_write64(IO_BITMAP_B, page_to_phys(vmx_io_bitmap_b)); - - vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */ - - /* Control */ - vmcs_write32(PIN_BASED_VM_EXEC_CONTROL, - vmcs_config.pin_based_exec_ctrl); - - exec_control = vmcs_config.cpu_based_exec_ctrl; - if (!vm_need_tpr_shadow(vmx->vcpu.kvm)) { - exec_control &= ~CPU_BASED_TPR_SHADOW; -#ifdef CONFIG_X86_64 - exec_control |= CPU_BASED_CR8_STORE_EXITING | - CPU_BASED_CR8_LOAD_EXITING; -#endif - } - vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, exec_control); - - if (cpu_has_secondary_exec_ctrls()) { - exec_control = vmcs_config.cpu_based_2nd_exec_ctrl; - if (!vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) - exec_control &= - ~SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; - vmcs_write32(SECONDARY_VM_EXEC_CONTROL, exec_control); - } - - vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, !!bypass_guest_pf); - vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, !!bypass_guest_pf); - vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */ - - vmcs_writel(HOST_CR0, read_cr0()); /* 22.2.3 */ - vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */ - vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */ - - vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */ - vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ - vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */ - vmcs_write16(HOST_FS_SELECTOR, read_fs()); /* 22.2.4 */ - vmcs_write16(HOST_GS_SELECTOR, read_gs()); /* 22.2.4 */ - vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */ -#ifdef CONFIG_X86_64 - rdmsrl(MSR_FS_BASE, a); - vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */ - rdmsrl(MSR_GS_BASE, a); - vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */ -#else - vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */ - vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */ -#endif - - vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */ - - get_idt(&dt); - vmcs_writel(HOST_IDTR_BASE, dt.base); /* 22.2.4 */ - - asm("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return)); - vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */ - vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0); - vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0); - vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0); - - rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk); - vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs); - rdmsrl(MSR_IA32_SYSENTER_ESP, a); - vmcs_writel(HOST_IA32_SYSENTER_ESP, a); /* 22.2.3 */ - rdmsrl(MSR_IA32_SYSENTER_EIP, a); - vmcs_writel(HOST_IA32_SYSENTER_EIP, a); /* 22.2.3 */ - - for (i = 0; i < NR_VMX_MSR; ++i) { - u32 index = vmx_msr_index[i]; - u32 data_low, data_high; - u64 data; - int j = vmx->nmsrs; - - if (rdmsr_safe(index, &data_low, &data_high) < 0) - continue; - if (wrmsr_safe(index, data_low, data_high) < 0) - continue; - data = data_low | ((u64)data_high << 32); - vmx->host_msrs[j].index = index; - vmx->host_msrs[j].reserved = 0; - vmx->host_msrs[j].data = data; - vmx->guest_msrs[j] = vmx->host_msrs[j]; - ++vmx->nmsrs; - } - - vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl); - - /* 22.2.1, 20.8.1 */ - vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl); - - vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); - vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); - - if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) - if (alloc_apic_access_page(vmx->vcpu.kvm) != 0) - return -ENOMEM; - - return 0; -} - -static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u64 msr; - int ret; - - if (!init_rmode_tss(vmx->vcpu.kvm)) { - ret = -ENOMEM; - goto out; - } - - vmx->vcpu.arch.rmode.active = 0; - - vmx->vcpu.arch.regs[VCPU_REGS_RDX] = get_rdx_init_val(); - set_cr8(&vmx->vcpu, 0); - msr = 0xfee00000 | MSR_IA32_APICBASE_ENABLE; - if (vmx->vcpu.vcpu_id == 0) - msr |= MSR_IA32_APICBASE_BSP; - kvm_set_apic_base(&vmx->vcpu, msr); - - fx_init(&vmx->vcpu); - - /* - * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode - * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh. - */ - if (vmx->vcpu.vcpu_id == 0) { - vmcs_write16(GUEST_CS_SELECTOR, 0xf000); - vmcs_writel(GUEST_CS_BASE, 0x000f0000); - } else { - vmcs_write16(GUEST_CS_SELECTOR, vmx->vcpu.arch.sipi_vector << 8); - vmcs_writel(GUEST_CS_BASE, vmx->vcpu.arch.sipi_vector << 12); - } - vmcs_write32(GUEST_CS_LIMIT, 0xffff); - vmcs_write32(GUEST_CS_AR_BYTES, 0x9b); - - seg_setup(VCPU_SREG_DS); - seg_setup(VCPU_SREG_ES); - seg_setup(VCPU_SREG_FS); - seg_setup(VCPU_SREG_GS); - seg_setup(VCPU_SREG_SS); - - vmcs_write16(GUEST_TR_SELECTOR, 0); - vmcs_writel(GUEST_TR_BASE, 0); - vmcs_write32(GUEST_TR_LIMIT, 0xffff); - vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); - - vmcs_write16(GUEST_LDTR_SELECTOR, 0); - vmcs_writel(GUEST_LDTR_BASE, 0); - vmcs_write32(GUEST_LDTR_LIMIT, 0xffff); - vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082); - - vmcs_write32(GUEST_SYSENTER_CS, 0); - vmcs_writel(GUEST_SYSENTER_ESP, 0); - vmcs_writel(GUEST_SYSENTER_EIP, 0); - - vmcs_writel(GUEST_RFLAGS, 0x02); - if (vmx->vcpu.vcpu_id == 0) - vmcs_writel(GUEST_RIP, 0xfff0); - else - vmcs_writel(GUEST_RIP, 0); - vmcs_writel(GUEST_RSP, 0); - - /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ - vmcs_writel(GUEST_DR7, 0x400); - - vmcs_writel(GUEST_GDTR_BASE, 0); - vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); - - vmcs_writel(GUEST_IDTR_BASE, 0); - vmcs_write32(GUEST_IDTR_LIMIT, 0xffff); - - vmcs_write32(GUEST_ACTIVITY_STATE, 0); - vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); - vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); - - guest_write_tsc(0); - - /* Special registers */ - vmcs_write64(GUEST_IA32_DEBUGCTL, 0); - - setup_msrs(vmx); - - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */ - - if (cpu_has_vmx_tpr_shadow()) { - vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); - if (vm_need_tpr_shadow(vmx->vcpu.kvm)) - vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - page_to_phys(vmx->vcpu.arch.apic->regs_page)); - vmcs_write32(TPR_THRESHOLD, 0); - } - - if (vm_need_virtualize_apic_accesses(vmx->vcpu.kvm)) - vmcs_write64(APIC_ACCESS_ADDR, - page_to_phys(vmx->vcpu.kvm->arch.apic_access_page)); - - vmx->vcpu.arch.cr0 = 0x60000010; - vmx_set_cr0(&vmx->vcpu, vmx->vcpu.arch.cr0); /* enter rmode */ - vmx_set_cr4(&vmx->vcpu, 0); -#ifdef CONFIG_X86_64 - vmx_set_efer(&vmx->vcpu, 0); -#endif - vmx_fpu_activate(&vmx->vcpu); - update_exception_bitmap(&vmx->vcpu); - - return 0; - -out: - return ret; -} - -static void vmx_inject_irq(struct kvm_vcpu *vcpu, int irq) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - if (vcpu->arch.rmode.active) { - vmx->rmode.irq.pending = true; - vmx->rmode.irq.vector = irq; - vmx->rmode.irq.rip = vmcs_readl(GUEST_RIP); - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - irq | INTR_TYPE_SOFT_INTR | INTR_INFO_VALID_MASK); - vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); - vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip - 1); - return; - } - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, - irq | INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK); -} - -static void kvm_do_inject_irq(struct kvm_vcpu *vcpu) -{ - int word_index = __ffs(vcpu->arch.irq_summary); - int bit_index = __ffs(vcpu->arch.irq_pending[word_index]); - int irq = word_index * BITS_PER_LONG + bit_index; - - clear_bit(bit_index, &vcpu->arch.irq_pending[word_index]); - if (!vcpu->arch.irq_pending[word_index]) - clear_bit(word_index, &vcpu->arch.irq_summary); - vmx_inject_irq(vcpu, irq); -} - - -static void do_interrupt_requests(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - u32 cpu_based_vm_exec_control; - - vcpu->arch.interrupt_window_open = - ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && - (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0); - - if (vcpu->arch.interrupt_window_open && - vcpu->arch.irq_summary && - !(vmcs_read32(VM_ENTRY_INTR_INFO_FIELD) & INTR_INFO_VALID_MASK)) - /* - * If interrupts enabled, and not blocked by sti or mov ss. Good. - */ - kvm_do_inject_irq(vcpu); - - cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); - if (!vcpu->arch.interrupt_window_open && - (vcpu->arch.irq_summary || kvm_run->request_interrupt_window)) - /* - * Interrupts blocked. Wait for unblock. - */ - cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING; - else - cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING; - vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); -} - -static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) -{ - int ret; - struct kvm_userspace_memory_region tss_mem = { - .slot = 8, - .guest_phys_addr = addr, - .memory_size = PAGE_SIZE * 3, - .flags = 0, - }; - - ret = kvm_set_memory_region(kvm, &tss_mem, 0); - if (ret) - return ret; - kvm->arch.tss_addr = addr; - return 0; -} - -static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu) -{ - struct kvm_guest_debug *dbg = &vcpu->guest_debug; - - set_debugreg(dbg->bp[0], 0); - set_debugreg(dbg->bp[1], 1); - set_debugreg(dbg->bp[2], 2); - set_debugreg(dbg->bp[3], 3); - - if (dbg->singlestep) { - unsigned long flags; - - flags = vmcs_readl(GUEST_RFLAGS); - flags |= X86_EFLAGS_TF | X86_EFLAGS_RF; - vmcs_writel(GUEST_RFLAGS, flags); - } -} - -static int handle_rmode_exception(struct kvm_vcpu *vcpu, - int vec, u32 err_code) -{ - if (!vcpu->arch.rmode.active) - return 0; - - /* - * Instruction with address size override prefix opcode 0x67 - * Cause the #SS fault with 0 error code in VM86 mode. - */ - if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0) - if (emulate_instruction(vcpu, NULL, 0, 0, 0) == EMULATE_DONE) - return 1; - return 0; -} - -static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u32 intr_info, error_code; - unsigned long cr2, rip; - u32 vect_info; - enum emulation_result er; - - vect_info = vmx->idt_vectoring_info; - intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - - if ((vect_info & VECTORING_INFO_VALID_MASK) && - !is_page_fault(intr_info)) - printk(KERN_ERR "%s: unexpected, vectoring info 0x%x " - "intr info 0x%x\n", __FUNCTION__, vect_info, intr_info); - - if (!irqchip_in_kernel(vcpu->kvm) && is_external_interrupt(vect_info)) { - int irq = vect_info & VECTORING_INFO_VECTOR_MASK; - set_bit(irq, vcpu->arch.irq_pending); - set_bit(irq / BITS_PER_LONG, &vcpu->arch.irq_summary); - } - - if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) /* nmi */ - return 1; /* already handled by vmx_vcpu_run() */ - - if (is_no_device(intr_info)) { - vmx_fpu_activate(vcpu); - return 1; - } - - if (is_invalid_opcode(intr_info)) { - er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); - if (er != EMULATE_DONE) - kvm_queue_exception(vcpu, UD_VECTOR); - return 1; - } - - error_code = 0; - rip = vmcs_readl(GUEST_RIP); - if (intr_info & INTR_INFO_DELIEVER_CODE_MASK) - error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); - if (is_page_fault(intr_info)) { - cr2 = vmcs_readl(EXIT_QUALIFICATION); - return kvm_mmu_page_fault(vcpu, cr2, error_code); - } - - if (vcpu->arch.rmode.active && - handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK, - error_code)) { - if (vcpu->arch.halt_request) { - vcpu->arch.halt_request = 0; - return kvm_emulate_halt(vcpu); - } - return 1; - } - - if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == - (INTR_TYPE_EXCEPTION | 1)) { - kvm_run->exit_reason = KVM_EXIT_DEBUG; - return 0; - } - kvm_run->exit_reason = KVM_EXIT_EXCEPTION; - kvm_run->ex.exception = intr_info & INTR_INFO_VECTOR_MASK; - kvm_run->ex.error_code = error_code; - return 0; -} - -static int handle_external_interrupt(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - ++vcpu->stat.irq_exits; - return 1; -} - -static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; - return 0; -} - -static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - unsigned long exit_qualification; - int size, down, in, string, rep; - unsigned port; - - ++vcpu->stat.io_exits; - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - string = (exit_qualification & 16) != 0; - - if (string) { - if (emulate_instruction(vcpu, - kvm_run, 0, 0, 0) == EMULATE_DO_MMIO) - return 0; - return 1; - } - - size = (exit_qualification & 7) + 1; - in = (exit_qualification & 8) != 0; - down = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_DF) != 0; - rep = (exit_qualification & 32) != 0; - port = exit_qualification >> 16; - - return kvm_emulate_pio(vcpu, kvm_run, in, size, port); -} - -static void -vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall) -{ - /* - * Patch in the VMCALL instruction: - */ - hypercall[0] = 0x0f; - hypercall[1] = 0x01; - hypercall[2] = 0xc1; -} - -static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - unsigned long exit_qualification; - int cr; - int reg; - - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - cr = exit_qualification & 15; - reg = (exit_qualification >> 8) & 15; - switch ((exit_qualification >> 4) & 3) { - case 0: /* mov to cr */ - switch (cr) { - case 0: - vcpu_load_rsp_rip(vcpu); - set_cr0(vcpu, vcpu->arch.regs[reg]); - skip_emulated_instruction(vcpu); - return 1; - case 3: - vcpu_load_rsp_rip(vcpu); - set_cr3(vcpu, vcpu->arch.regs[reg]); - skip_emulated_instruction(vcpu); - return 1; - case 4: - vcpu_load_rsp_rip(vcpu); - set_cr4(vcpu, vcpu->arch.regs[reg]); - skip_emulated_instruction(vcpu); - return 1; - case 8: - vcpu_load_rsp_rip(vcpu); - set_cr8(vcpu, vcpu->arch.regs[reg]); - skip_emulated_instruction(vcpu); - if (irqchip_in_kernel(vcpu->kvm)) - return 1; - kvm_run->exit_reason = KVM_EXIT_SET_TPR; - return 0; - }; - break; - case 2: /* clts */ - vcpu_load_rsp_rip(vcpu); - vmx_fpu_deactivate(vcpu); - vcpu->arch.cr0 &= ~X86_CR0_TS; - vmcs_writel(CR0_READ_SHADOW, vcpu->arch.cr0); - vmx_fpu_activate(vcpu); - skip_emulated_instruction(vcpu); - return 1; - case 1: /*mov from cr*/ - switch (cr) { - case 3: - vcpu_load_rsp_rip(vcpu); - vcpu->arch.regs[reg] = vcpu->arch.cr3; - vcpu_put_rsp_rip(vcpu); - skip_emulated_instruction(vcpu); - return 1; - case 8: - vcpu_load_rsp_rip(vcpu); - vcpu->arch.regs[reg] = get_cr8(vcpu); - vcpu_put_rsp_rip(vcpu); - skip_emulated_instruction(vcpu); - return 1; - } - break; - case 3: /* lmsw */ - lmsw(vcpu, (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f); - - skip_emulated_instruction(vcpu); - return 1; - default: - break; - } - kvm_run->exit_reason = 0; - pr_unimpl(vcpu, "unhandled control register: op %d cr %d\n", - (int)(exit_qualification >> 4) & 3, cr); - return 0; -} - -static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - unsigned long exit_qualification; - unsigned long val; - int dr, reg; - - /* - * FIXME: this code assumes the host is debugging the guest. - * need to deal with guest debugging itself too. - */ - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - dr = exit_qualification & 7; - reg = (exit_qualification >> 8) & 15; - vcpu_load_rsp_rip(vcpu); - if (exit_qualification & 16) { - /* mov from dr */ - switch (dr) { - case 6: - val = 0xffff0ff0; - break; - case 7: - val = 0x400; - break; - default: - val = 0; - } - vcpu->arch.regs[reg] = val; - } else { - /* mov to dr */ - } - vcpu_put_rsp_rip(vcpu); - skip_emulated_instruction(vcpu); - return 1; -} - -static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - kvm_emulate_cpuid(vcpu); - return 1; -} - -static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; - u64 data; - - if (vmx_get_msr(vcpu, ecx, &data)) { - kvm_inject_gp(vcpu, 0); - return 1; - } - - /* FIXME: handling of bits 32:63 of rax, rdx */ - vcpu->arch.regs[VCPU_REGS_RAX] = data & -1u; - vcpu->arch.regs[VCPU_REGS_RDX] = (data >> 32) & -1u; - skip_emulated_instruction(vcpu); - return 1; -} - -static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - u32 ecx = vcpu->arch.regs[VCPU_REGS_RCX]; - u64 data = (vcpu->arch.regs[VCPU_REGS_RAX] & -1u) - | ((u64)(vcpu->arch.regs[VCPU_REGS_RDX] & -1u) << 32); - - if (vmx_set_msr(vcpu, ecx, data) != 0) { - kvm_inject_gp(vcpu, 0); - return 1; - } - - skip_emulated_instruction(vcpu); - return 1; -} - -static int handle_tpr_below_threshold(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - return 1; -} - -static int handle_interrupt_window(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - u32 cpu_based_vm_exec_control; - - /* clear pending irq */ - cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); - cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING; - vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); - /* - * If the user space waits to inject interrupts, exit as soon as - * possible - */ - if (kvm_run->request_interrupt_window && - !vcpu->arch.irq_summary) { - kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; - ++vcpu->stat.irq_window_exits; - return 0; - } - return 1; -} - -static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - skip_emulated_instruction(vcpu); - return kvm_emulate_halt(vcpu); -} - -static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - skip_emulated_instruction(vcpu); - kvm_emulate_hypercall(vcpu); - return 1; -} - -static int handle_wbinvd(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - skip_emulated_instruction(vcpu); - /* TODO: Add support for VT-d/pass-through device */ - return 1; -} - -static int handle_apic_access(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - u64 exit_qualification; - enum emulation_result er; - unsigned long offset; - - exit_qualification = vmcs_read64(EXIT_QUALIFICATION); - offset = exit_qualification & 0xffful; - - er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); - - if (er != EMULATE_DONE) { - printk(KERN_ERR - "Fail to handle apic access vmexit! Offset is 0x%lx\n", - offset); - return -ENOTSUPP; - } - return 1; -} - -/* - * The exit handlers return 1 if the exit was handled fully and guest execution - * may resume. Otherwise they set the kvm_run parameter to indicate what needs - * to be done to userspace and return 0. - */ -static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) = { - [EXIT_REASON_EXCEPTION_NMI] = handle_exception, - [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, - [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, - [EXIT_REASON_IO_INSTRUCTION] = handle_io, - [EXIT_REASON_CR_ACCESS] = handle_cr, - [EXIT_REASON_DR_ACCESS] = handle_dr, - [EXIT_REASON_CPUID] = handle_cpuid, - [EXIT_REASON_MSR_READ] = handle_rdmsr, - [EXIT_REASON_MSR_WRITE] = handle_wrmsr, - [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window, - [EXIT_REASON_HLT] = handle_halt, - [EXIT_REASON_VMCALL] = handle_vmcall, - [EXIT_REASON_TPR_BELOW_THRESHOLD] = handle_tpr_below_threshold, - [EXIT_REASON_APIC_ACCESS] = handle_apic_access, - [EXIT_REASON_WBINVD] = handle_wbinvd, -}; - -static const int kvm_vmx_max_exit_handlers = - ARRAY_SIZE(kvm_vmx_exit_handlers); - -/* - * The guest has exited. See if we can fix it or if we need userspace - * assistance. - */ -static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) -{ - u32 exit_reason = vmcs_read32(VM_EXIT_REASON); - struct vcpu_vmx *vmx = to_vmx(vcpu); - u32 vectoring_info = vmx->idt_vectoring_info; - - if (unlikely(vmx->fail)) { - kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; - kvm_run->fail_entry.hardware_entry_failure_reason - = vmcs_read32(VM_INSTRUCTION_ERROR); - return 0; - } - - if ((vectoring_info & VECTORING_INFO_VALID_MASK) && - exit_reason != EXIT_REASON_EXCEPTION_NMI) - printk(KERN_WARNING "%s: unexpected, valid vectoring info and " - "exit reason is 0x%x\n", __FUNCTION__, exit_reason); - if (exit_reason < kvm_vmx_max_exit_handlers - && kvm_vmx_exit_handlers[exit_reason]) - return kvm_vmx_exit_handlers[exit_reason](vcpu, kvm_run); - else { - kvm_run->exit_reason = KVM_EXIT_UNKNOWN; - kvm_run->hw.hardware_exit_reason = exit_reason; - } - return 0; -} - -static void vmx_flush_tlb(struct kvm_vcpu *vcpu) -{ -} - -static void update_tpr_threshold(struct kvm_vcpu *vcpu) -{ - int max_irr, tpr; - - if (!vm_need_tpr_shadow(vcpu->kvm)) - return; - - if (!kvm_lapic_enabled(vcpu) || - ((max_irr = kvm_lapic_find_highest_irr(vcpu)) == -1)) { - vmcs_write32(TPR_THRESHOLD, 0); - return; - } - - tpr = (kvm_lapic_get_cr8(vcpu) & 0x0f) << 4; - vmcs_write32(TPR_THRESHOLD, (max_irr > tpr) ? tpr >> 4 : max_irr >> 4); -} - -static void enable_irq_window(struct kvm_vcpu *vcpu) -{ - u32 cpu_based_vm_exec_control; - - cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL); - cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING; - vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control); -} - -static void vmx_intr_assist(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u32 idtv_info_field, intr_info_field; - int has_ext_irq, interrupt_window_open; - int vector; - - update_tpr_threshold(vcpu); - - has_ext_irq = kvm_cpu_has_interrupt(vcpu); - intr_info_field = vmcs_read32(VM_ENTRY_INTR_INFO_FIELD); - idtv_info_field = vmx->idt_vectoring_info; - if (intr_info_field & INTR_INFO_VALID_MASK) { - if (idtv_info_field & INTR_INFO_VALID_MASK) { - /* TODO: fault when IDT_Vectoring */ - if (printk_ratelimit()) - printk(KERN_ERR "Fault when IDT_Vectoring\n"); - } - if (has_ext_irq) - enable_irq_window(vcpu); - return; - } - if (unlikely(idtv_info_field & INTR_INFO_VALID_MASK)) { - if ((idtv_info_field & VECTORING_INFO_TYPE_MASK) - == INTR_TYPE_EXT_INTR - && vcpu->arch.rmode.active) { - u8 vect = idtv_info_field & VECTORING_INFO_VECTOR_MASK; - - vmx_inject_irq(vcpu, vect); - if (unlikely(has_ext_irq)) - enable_irq_window(vcpu); - return; - } - - vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, idtv_info_field); - vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, - vmcs_read32(VM_EXIT_INSTRUCTION_LEN)); - - if (unlikely(idtv_info_field & INTR_INFO_DELIEVER_CODE_MASK)) - vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, - vmcs_read32(IDT_VECTORING_ERROR_CODE)); - if (unlikely(has_ext_irq)) - enable_irq_window(vcpu); - return; - } - if (!has_ext_irq) - return; - interrupt_window_open = - ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && - (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0); - if (interrupt_window_open) { - vector = kvm_cpu_get_interrupt(vcpu); - vmx_inject_irq(vcpu, vector); - kvm_timer_intr_post(vcpu, vector); - } else - enable_irq_window(vcpu); -} - -/* - * Failure to inject an interrupt should give us the information - * in IDT_VECTORING_INFO_FIELD. However, if the failure occurs - * when fetching the interrupt redirection bitmap in the real-mode - * tss, this doesn't happen. So we do it ourselves. - */ -static void fixup_rmode_irq(struct vcpu_vmx *vmx) -{ - vmx->rmode.irq.pending = 0; - if (vmcs_readl(GUEST_RIP) + 1 != vmx->rmode.irq.rip) - return; - vmcs_writel(GUEST_RIP, vmx->rmode.irq.rip); - if (vmx->idt_vectoring_info & VECTORING_INFO_VALID_MASK) { - vmx->idt_vectoring_info &= ~VECTORING_INFO_TYPE_MASK; - vmx->idt_vectoring_info |= INTR_TYPE_EXT_INTR; - return; - } - vmx->idt_vectoring_info = - VECTORING_INFO_VALID_MASK - | INTR_TYPE_EXT_INTR - | vmx->rmode.irq.vector; -} - -static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - u32 intr_info; - - /* - * Loading guest fpu may have cleared host cr0.ts - */ - vmcs_writel(HOST_CR0, read_cr0()); - - asm( - /* Store host registers */ -#ifdef CONFIG_X86_64 - "push %%rdx; push %%rbp;" - "push %%rcx \n\t" -#else - "push %%edx; push %%ebp;" - "push %%ecx \n\t" -#endif - ASM_VMX_VMWRITE_RSP_RDX "\n\t" - /* Check if vmlaunch of vmresume is needed */ - "cmpl $0, %c[launched](%0) \n\t" - /* Load guest registers. Don't clobber flags. */ -#ifdef CONFIG_X86_64 - "mov %c[cr2](%0), %%rax \n\t" - "mov %%rax, %%cr2 \n\t" - "mov %c[rax](%0), %%rax \n\t" - "mov %c[rbx](%0), %%rbx \n\t" - "mov %c[rdx](%0), %%rdx \n\t" - "mov %c[rsi](%0), %%rsi \n\t" - "mov %c[rdi](%0), %%rdi \n\t" - "mov %c[rbp](%0), %%rbp \n\t" - "mov %c[r8](%0), %%r8 \n\t" - "mov %c[r9](%0), %%r9 \n\t" - "mov %c[r10](%0), %%r10 \n\t" - "mov %c[r11](%0), %%r11 \n\t" - "mov %c[r12](%0), %%r12 \n\t" - "mov %c[r13](%0), %%r13 \n\t" - "mov %c[r14](%0), %%r14 \n\t" - "mov %c[r15](%0), %%r15 \n\t" - "mov %c[rcx](%0), %%rcx \n\t" /* kills %0 (rcx) */ -#else - "mov %c[cr2](%0), %%eax \n\t" - "mov %%eax, %%cr2 \n\t" - "mov %c[rax](%0), %%eax \n\t" - "mov %c[rbx](%0), %%ebx \n\t" - "mov %c[rdx](%0), %%edx \n\t" - "mov %c[rsi](%0), %%esi \n\t" - "mov %c[rdi](%0), %%edi \n\t" - "mov %c[rbp](%0), %%ebp \n\t" - "mov %c[rcx](%0), %%ecx \n\t" /* kills %0 (ecx) */ -#endif - /* Enter guest mode */ - "jne .Llaunched \n\t" - ASM_VMX_VMLAUNCH "\n\t" - "jmp .Lkvm_vmx_return \n\t" - ".Llaunched: " ASM_VMX_VMRESUME "\n\t" - ".Lkvm_vmx_return: " - /* Save guest registers, load host registers, keep flags */ -#ifdef CONFIG_X86_64 - "xchg %0, (%%rsp) \n\t" - "mov %%rax, %c[rax](%0) \n\t" - "mov %%rbx, %c[rbx](%0) \n\t" - "pushq (%%rsp); popq %c[rcx](%0) \n\t" - "mov %%rdx, %c[rdx](%0) \n\t" - "mov %%rsi, %c[rsi](%0) \n\t" - "mov %%rdi, %c[rdi](%0) \n\t" - "mov %%rbp, %c[rbp](%0) \n\t" - "mov %%r8, %c[r8](%0) \n\t" - "mov %%r9, %c[r9](%0) \n\t" - "mov %%r10, %c[r10](%0) \n\t" - "mov %%r11, %c[r11](%0) \n\t" - "mov %%r12, %c[r12](%0) \n\t" - "mov %%r13, %c[r13](%0) \n\t" - "mov %%r14, %c[r14](%0) \n\t" - "mov %%r15, %c[r15](%0) \n\t" - "mov %%cr2, %%rax \n\t" - "mov %%rax, %c[cr2](%0) \n\t" - - "pop %%rbp; pop %%rbp; pop %%rdx \n\t" -#else - "xchg %0, (%%esp) \n\t" - "mov %%eax, %c[rax](%0) \n\t" - "mov %%ebx, %c[rbx](%0) \n\t" - "pushl (%%esp); popl %c[rcx](%0) \n\t" - "mov %%edx, %c[rdx](%0) \n\t" - "mov %%esi, %c[rsi](%0) \n\t" - "mov %%edi, %c[rdi](%0) \n\t" - "mov %%ebp, %c[rbp](%0) \n\t" - "mov %%cr2, %%eax \n\t" - "mov %%eax, %c[cr2](%0) \n\t" - - "pop %%ebp; pop %%ebp; pop %%edx \n\t" -#endif - "setbe %c[fail](%0) \n\t" - : : "c"(vmx), "d"((unsigned long)HOST_RSP), - [launched]"i"(offsetof(struct vcpu_vmx, launched)), - [fail]"i"(offsetof(struct vcpu_vmx, fail)), - [rax]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RAX])), - [rbx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBX])), - [rcx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RCX])), - [rdx]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDX])), - [rsi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RSI])), - [rdi]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RDI])), - [rbp]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_RBP])), -#ifdef CONFIG_X86_64 - [r8]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R8])), - [r9]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R9])), - [r10]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R10])), - [r11]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R11])), - [r12]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R12])), - [r13]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R13])), - [r14]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R14])), - [r15]"i"(offsetof(struct vcpu_vmx, vcpu.arch.regs[VCPU_REGS_R15])), -#endif - [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)) - : "cc", "memory" -#ifdef CONFIG_X86_64 - , "rbx", "rdi", "rsi" - , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" -#else - , "ebx", "edi", "rsi" -#endif - ); - - vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); - if (vmx->rmode.irq.pending) - fixup_rmode_irq(vmx); - - vcpu->arch.interrupt_window_open = - (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0; - - asm("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS)); - vmx->launched = 1; - - intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - - /* We need to handle NMIs before interrupts are enabled */ - if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) /* nmi */ - asm("int $2"); -} - -static void vmx_free_vmcs(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - if (vmx->vmcs) { - on_each_cpu(__vcpu_clear, vmx, 0, 1); - free_vmcs(vmx->vmcs); - vmx->vmcs = NULL; - } -} - -static void vmx_free_vcpu(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - vmx_free_vmcs(vcpu); - kfree(vmx->host_msrs); - kfree(vmx->guest_msrs); - kvm_vcpu_uninit(vcpu); - kmem_cache_free(kvm_vcpu_cache, vmx); -} - -static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) -{ - int err; - struct vcpu_vmx *vmx = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); - int cpu; - - if (!vmx) - return ERR_PTR(-ENOMEM); - - err = kvm_vcpu_init(&vmx->vcpu, kvm, id); - if (err) - goto free_vcpu; - - vmx->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!vmx->guest_msrs) { - err = -ENOMEM; - goto uninit_vcpu; - } - - vmx->host_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!vmx->host_msrs) - goto free_guest_msrs; - - vmx->vmcs = alloc_vmcs(); - if (!vmx->vmcs) - goto free_msrs; - - vmcs_clear(vmx->vmcs); - - cpu = get_cpu(); - vmx_vcpu_load(&vmx->vcpu, cpu); - err = vmx_vcpu_setup(vmx); - vmx_vcpu_put(&vmx->vcpu); - put_cpu(); - if (err) - goto free_vmcs; - - return &vmx->vcpu; - -free_vmcs: - free_vmcs(vmx->vmcs); -free_msrs: - kfree(vmx->host_msrs); -free_guest_msrs: - kfree(vmx->guest_msrs); -uninit_vcpu: - kvm_vcpu_uninit(&vmx->vcpu); -free_vcpu: - kmem_cache_free(kvm_vcpu_cache, vmx); - return ERR_PTR(err); -} - -static void __init vmx_check_processor_compat(void *rtn) -{ - struct vmcs_config vmcs_conf; - - *(int *)rtn = 0; - if (setup_vmcs_config(&vmcs_conf) < 0) - *(int *)rtn = -EIO; - if (memcmp(&vmcs_config, &vmcs_conf, sizeof(struct vmcs_config)) != 0) { - printk(KERN_ERR "kvm: CPU %d feature inconsistency!\n", - smp_processor_id()); - *(int *)rtn = -EIO; - } -} - -static struct kvm_x86_ops vmx_x86_ops = { - .cpu_has_kvm_support = cpu_has_kvm_support, - .disabled_by_bios = vmx_disabled_by_bios, - .hardware_setup = hardware_setup, - .hardware_unsetup = hardware_unsetup, - .check_processor_compatibility = vmx_check_processor_compat, - .hardware_enable = hardware_enable, - .hardware_disable = hardware_disable, - - .vcpu_create = vmx_create_vcpu, - .vcpu_free = vmx_free_vcpu, - .vcpu_reset = vmx_vcpu_reset, - - .prepare_guest_switch = vmx_save_host_state, - .vcpu_load = vmx_vcpu_load, - .vcpu_put = vmx_vcpu_put, - .vcpu_decache = vmx_vcpu_decache, - - .set_guest_debug = set_guest_debug, - .guest_debug_pre = kvm_guest_debug_pre, - .get_msr = vmx_get_msr, - .set_msr = vmx_set_msr, - .get_segment_base = vmx_get_segment_base, - .get_segment = vmx_get_segment, - .set_segment = vmx_set_segment, - .get_cs_db_l_bits = vmx_get_cs_db_l_bits, - .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits, - .set_cr0 = vmx_set_cr0, - .set_cr3 = vmx_set_cr3, - .set_cr4 = vmx_set_cr4, -#ifdef CONFIG_X86_64 - .set_efer = vmx_set_efer, -#endif - .get_idt = vmx_get_idt, - .set_idt = vmx_set_idt, - .get_gdt = vmx_get_gdt, - .set_gdt = vmx_set_gdt, - .cache_regs = vcpu_load_rsp_rip, - .decache_regs = vcpu_put_rsp_rip, - .get_rflags = vmx_get_rflags, - .set_rflags = vmx_set_rflags, - - .tlb_flush = vmx_flush_tlb, - - .run = vmx_vcpu_run, - .handle_exit = kvm_handle_exit, - .skip_emulated_instruction = skip_emulated_instruction, - .patch_hypercall = vmx_patch_hypercall, - .get_irq = vmx_get_irq, - .set_irq = vmx_inject_irq, - .queue_exception = vmx_queue_exception, - .exception_injected = vmx_exception_injected, - .inject_pending_irq = vmx_intr_assist, - .inject_pending_vectors = do_interrupt_requests, - - .set_tss_addr = vmx_set_tss_addr, -}; - -static int __init vmx_init(void) -{ - void *iova; - int r; - - vmx_io_bitmap_a = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); - if (!vmx_io_bitmap_a) - return -ENOMEM; - - vmx_io_bitmap_b = alloc_page(GFP_KERNEL | __GFP_HIGHMEM); - if (!vmx_io_bitmap_b) { - r = -ENOMEM; - goto out; - } - - /* - * Allow direct access to the PC debug port (it is often used for I/O - * delays, but the vmexits simply slow things down). - */ - iova = kmap(vmx_io_bitmap_a); - memset(iova, 0xff, PAGE_SIZE); - clear_bit(0x80, iova); - kunmap(vmx_io_bitmap_a); - - iova = kmap(vmx_io_bitmap_b); - memset(iova, 0xff, PAGE_SIZE); - kunmap(vmx_io_bitmap_b); - - r = kvm_init(&vmx_x86_ops, sizeof(struct vcpu_vmx), THIS_MODULE); - if (r) - goto out1; - - if (bypass_guest_pf) - kvm_mmu_set_nonpresent_ptes(~0xffeull, 0ull); - - return 0; - -out1: - __free_page(vmx_io_bitmap_b); -out: - __free_page(vmx_io_bitmap_a); - return r; -} - -static void __exit vmx_exit(void) -{ - __free_page(vmx_io_bitmap_b); - __free_page(vmx_io_bitmap_a); - - kvm_exit(); -} - -module_init(vmx_init) -module_exit(vmx_exit) diff --git a/drivers/kvm/vmx.h b/drivers/kvm/vmx.h deleted file mode 100644 index d52ae8d7303d..000000000000 --- a/drivers/kvm/vmx.h +++ /dev/null @@ -1,324 +0,0 @@ -#ifndef VMX_H -#define VMX_H - -/* - * vmx.h: VMX Architecture related definitions - * Copyright (c) 2004, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., 59 Temple - * Place - Suite 330, Boston, MA 02111-1307 USA. - * - * A few random additions are: - * Copyright (C) 2006 Qumranet - * Avi Kivity - * Yaniv Kamay - * - */ - -/* - * Definitions of Primary Processor-Based VM-Execution Controls. - */ -#define CPU_BASED_VIRTUAL_INTR_PENDING 0x00000004 -#define CPU_BASED_USE_TSC_OFFSETING 0x00000008 -#define CPU_BASED_HLT_EXITING 0x00000080 -#define CPU_BASED_INVLPG_EXITING 0x00000200 -#define CPU_BASED_MWAIT_EXITING 0x00000400 -#define CPU_BASED_RDPMC_EXITING 0x00000800 -#define CPU_BASED_RDTSC_EXITING 0x00001000 -#define CPU_BASED_CR8_LOAD_EXITING 0x00080000 -#define CPU_BASED_CR8_STORE_EXITING 0x00100000 -#define CPU_BASED_TPR_SHADOW 0x00200000 -#define CPU_BASED_MOV_DR_EXITING 0x00800000 -#define CPU_BASED_UNCOND_IO_EXITING 0x01000000 -#define CPU_BASED_USE_IO_BITMAPS 0x02000000 -#define CPU_BASED_USE_MSR_BITMAPS 0x10000000 -#define CPU_BASED_MONITOR_EXITING 0x20000000 -#define CPU_BASED_PAUSE_EXITING 0x40000000 -#define CPU_BASED_ACTIVATE_SECONDARY_CONTROLS 0x80000000 -/* - * Definitions of Secondary Processor-Based VM-Execution Controls. - */ -#define SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES 0x00000001 -#define SECONDARY_EXEC_WBINVD_EXITING 0x00000040 - - -#define PIN_BASED_EXT_INTR_MASK 0x00000001 -#define PIN_BASED_NMI_EXITING 0x00000008 -#define PIN_BASED_VIRTUAL_NMIS 0x00000020 - -#define VM_EXIT_HOST_ADDR_SPACE_SIZE 0x00000200 -#define VM_EXIT_ACK_INTR_ON_EXIT 0x00008000 - -#define VM_ENTRY_IA32E_MODE 0x00000200 -#define VM_ENTRY_SMM 0x00000400 -#define VM_ENTRY_DEACT_DUAL_MONITOR 0x00000800 - -/* VMCS Encodings */ -enum vmcs_field { - GUEST_ES_SELECTOR = 0x00000800, - GUEST_CS_SELECTOR = 0x00000802, - GUEST_SS_SELECTOR = 0x00000804, - GUEST_DS_SELECTOR = 0x00000806, - GUEST_FS_SELECTOR = 0x00000808, - GUEST_GS_SELECTOR = 0x0000080a, - GUEST_LDTR_SELECTOR = 0x0000080c, - GUEST_TR_SELECTOR = 0x0000080e, - HOST_ES_SELECTOR = 0x00000c00, - HOST_CS_SELECTOR = 0x00000c02, - HOST_SS_SELECTOR = 0x00000c04, - HOST_DS_SELECTOR = 0x00000c06, - HOST_FS_SELECTOR = 0x00000c08, - HOST_GS_SELECTOR = 0x00000c0a, - HOST_TR_SELECTOR = 0x00000c0c, - IO_BITMAP_A = 0x00002000, - IO_BITMAP_A_HIGH = 0x00002001, - IO_BITMAP_B = 0x00002002, - IO_BITMAP_B_HIGH = 0x00002003, - MSR_BITMAP = 0x00002004, - MSR_BITMAP_HIGH = 0x00002005, - VM_EXIT_MSR_STORE_ADDR = 0x00002006, - VM_EXIT_MSR_STORE_ADDR_HIGH = 0x00002007, - VM_EXIT_MSR_LOAD_ADDR = 0x00002008, - VM_EXIT_MSR_LOAD_ADDR_HIGH = 0x00002009, - VM_ENTRY_MSR_LOAD_ADDR = 0x0000200a, - VM_ENTRY_MSR_LOAD_ADDR_HIGH = 0x0000200b, - TSC_OFFSET = 0x00002010, - TSC_OFFSET_HIGH = 0x00002011, - VIRTUAL_APIC_PAGE_ADDR = 0x00002012, - VIRTUAL_APIC_PAGE_ADDR_HIGH = 0x00002013, - APIC_ACCESS_ADDR = 0x00002014, - APIC_ACCESS_ADDR_HIGH = 0x00002015, - VMCS_LINK_POINTER = 0x00002800, - VMCS_LINK_POINTER_HIGH = 0x00002801, - GUEST_IA32_DEBUGCTL = 0x00002802, - GUEST_IA32_DEBUGCTL_HIGH = 0x00002803, - PIN_BASED_VM_EXEC_CONTROL = 0x00004000, - CPU_BASED_VM_EXEC_CONTROL = 0x00004002, - EXCEPTION_BITMAP = 0x00004004, - PAGE_FAULT_ERROR_CODE_MASK = 0x00004006, - PAGE_FAULT_ERROR_CODE_MATCH = 0x00004008, - CR3_TARGET_COUNT = 0x0000400a, - VM_EXIT_CONTROLS = 0x0000400c, - VM_EXIT_MSR_STORE_COUNT = 0x0000400e, - VM_EXIT_MSR_LOAD_COUNT = 0x00004010, - VM_ENTRY_CONTROLS = 0x00004012, - VM_ENTRY_MSR_LOAD_COUNT = 0x00004014, - VM_ENTRY_INTR_INFO_FIELD = 0x00004016, - VM_ENTRY_EXCEPTION_ERROR_CODE = 0x00004018, - VM_ENTRY_INSTRUCTION_LEN = 0x0000401a, - TPR_THRESHOLD = 0x0000401c, - SECONDARY_VM_EXEC_CONTROL = 0x0000401e, - VM_INSTRUCTION_ERROR = 0x00004400, - VM_EXIT_REASON = 0x00004402, - VM_EXIT_INTR_INFO = 0x00004404, - VM_EXIT_INTR_ERROR_CODE = 0x00004406, - IDT_VECTORING_INFO_FIELD = 0x00004408, - IDT_VECTORING_ERROR_CODE = 0x0000440a, - VM_EXIT_INSTRUCTION_LEN = 0x0000440c, - VMX_INSTRUCTION_INFO = 0x0000440e, - GUEST_ES_LIMIT = 0x00004800, - GUEST_CS_LIMIT = 0x00004802, - GUEST_SS_LIMIT = 0x00004804, - GUEST_DS_LIMIT = 0x00004806, - GUEST_FS_LIMIT = 0x00004808, - GUEST_GS_LIMIT = 0x0000480a, - GUEST_LDTR_LIMIT = 0x0000480c, - GUEST_TR_LIMIT = 0x0000480e, - GUEST_GDTR_LIMIT = 0x00004810, - GUEST_IDTR_LIMIT = 0x00004812, - GUEST_ES_AR_BYTES = 0x00004814, - GUEST_CS_AR_BYTES = 0x00004816, - GUEST_SS_AR_BYTES = 0x00004818, - GUEST_DS_AR_BYTES = 0x0000481a, - GUEST_FS_AR_BYTES = 0x0000481c, - GUEST_GS_AR_BYTES = 0x0000481e, - GUEST_LDTR_AR_BYTES = 0x00004820, - GUEST_TR_AR_BYTES = 0x00004822, - GUEST_INTERRUPTIBILITY_INFO = 0x00004824, - GUEST_ACTIVITY_STATE = 0X00004826, - GUEST_SYSENTER_CS = 0x0000482A, - HOST_IA32_SYSENTER_CS = 0x00004c00, - CR0_GUEST_HOST_MASK = 0x00006000, - CR4_GUEST_HOST_MASK = 0x00006002, - CR0_READ_SHADOW = 0x00006004, - CR4_READ_SHADOW = 0x00006006, - CR3_TARGET_VALUE0 = 0x00006008, - CR3_TARGET_VALUE1 = 0x0000600a, - CR3_TARGET_VALUE2 = 0x0000600c, - CR3_TARGET_VALUE3 = 0x0000600e, - EXIT_QUALIFICATION = 0x00006400, - GUEST_LINEAR_ADDRESS = 0x0000640a, - GUEST_CR0 = 0x00006800, - GUEST_CR3 = 0x00006802, - GUEST_CR4 = 0x00006804, - GUEST_ES_BASE = 0x00006806, - GUEST_CS_BASE = 0x00006808, - GUEST_SS_BASE = 0x0000680a, - GUEST_DS_BASE = 0x0000680c, - GUEST_FS_BASE = 0x0000680e, - GUEST_GS_BASE = 0x00006810, - GUEST_LDTR_BASE = 0x00006812, - GUEST_TR_BASE = 0x00006814, - GUEST_GDTR_BASE = 0x00006816, - GUEST_IDTR_BASE = 0x00006818, - GUEST_DR7 = 0x0000681a, - GUEST_RSP = 0x0000681c, - GUEST_RIP = 0x0000681e, - GUEST_RFLAGS = 0x00006820, - GUEST_PENDING_DBG_EXCEPTIONS = 0x00006822, - GUEST_SYSENTER_ESP = 0x00006824, - GUEST_SYSENTER_EIP = 0x00006826, - HOST_CR0 = 0x00006c00, - HOST_CR3 = 0x00006c02, - HOST_CR4 = 0x00006c04, - HOST_FS_BASE = 0x00006c06, - HOST_GS_BASE = 0x00006c08, - HOST_TR_BASE = 0x00006c0a, - HOST_GDTR_BASE = 0x00006c0c, - HOST_IDTR_BASE = 0x00006c0e, - HOST_IA32_SYSENTER_ESP = 0x00006c10, - HOST_IA32_SYSENTER_EIP = 0x00006c12, - HOST_RSP = 0x00006c14, - HOST_RIP = 0x00006c16, -}; - -#define VMX_EXIT_REASONS_FAILED_VMENTRY 0x80000000 - -#define EXIT_REASON_EXCEPTION_NMI 0 -#define EXIT_REASON_EXTERNAL_INTERRUPT 1 -#define EXIT_REASON_TRIPLE_FAULT 2 - -#define EXIT_REASON_PENDING_INTERRUPT 7 - -#define EXIT_REASON_TASK_SWITCH 9 -#define EXIT_REASON_CPUID 10 -#define EXIT_REASON_HLT 12 -#define EXIT_REASON_INVLPG 14 -#define EXIT_REASON_RDPMC 15 -#define EXIT_REASON_RDTSC 16 -#define EXIT_REASON_VMCALL 18 -#define EXIT_REASON_VMCLEAR 19 -#define EXIT_REASON_VMLAUNCH 20 -#define EXIT_REASON_VMPTRLD 21 -#define EXIT_REASON_VMPTRST 22 -#define EXIT_REASON_VMREAD 23 -#define EXIT_REASON_VMRESUME 24 -#define EXIT_REASON_VMWRITE 25 -#define EXIT_REASON_VMOFF 26 -#define EXIT_REASON_VMON 27 -#define EXIT_REASON_CR_ACCESS 28 -#define EXIT_REASON_DR_ACCESS 29 -#define EXIT_REASON_IO_INSTRUCTION 30 -#define EXIT_REASON_MSR_READ 31 -#define EXIT_REASON_MSR_WRITE 32 -#define EXIT_REASON_MWAIT_INSTRUCTION 36 -#define EXIT_REASON_TPR_BELOW_THRESHOLD 43 -#define EXIT_REASON_APIC_ACCESS 44 -#define EXIT_REASON_WBINVD 54 - -/* - * Interruption-information format - */ -#define INTR_INFO_VECTOR_MASK 0xff /* 7:0 */ -#define INTR_INFO_INTR_TYPE_MASK 0x700 /* 10:8 */ -#define INTR_INFO_DELIEVER_CODE_MASK 0x800 /* 11 */ -#define INTR_INFO_VALID_MASK 0x80000000 /* 31 */ - -#define VECTORING_INFO_VECTOR_MASK INTR_INFO_VECTOR_MASK -#define VECTORING_INFO_TYPE_MASK INTR_INFO_INTR_TYPE_MASK -#define VECTORING_INFO_DELIEVER_CODE_MASK INTR_INFO_DELIEVER_CODE_MASK -#define VECTORING_INFO_VALID_MASK INTR_INFO_VALID_MASK - -#define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ -#define INTR_TYPE_EXCEPTION (3 << 8) /* processor exception */ -#define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ - -/* - * Exit Qualifications for MOV for Control Register Access - */ -#define CONTROL_REG_ACCESS_NUM 0x7 /* 2:0, number of control reg.*/ -#define CONTROL_REG_ACCESS_TYPE 0x30 /* 5:4, access type */ -#define CONTROL_REG_ACCESS_REG 0xf00 /* 10:8, general purpose reg. */ -#define LMSW_SOURCE_DATA_SHIFT 16 -#define LMSW_SOURCE_DATA (0xFFFF << LMSW_SOURCE_DATA_SHIFT) /* 16:31 lmsw source */ -#define REG_EAX (0 << 8) -#define REG_ECX (1 << 8) -#define REG_EDX (2 << 8) -#define REG_EBX (3 << 8) -#define REG_ESP (4 << 8) -#define REG_EBP (5 << 8) -#define REG_ESI (6 << 8) -#define REG_EDI (7 << 8) -#define REG_R8 (8 << 8) -#define REG_R9 (9 << 8) -#define REG_R10 (10 << 8) -#define REG_R11 (11 << 8) -#define REG_R12 (12 << 8) -#define REG_R13 (13 << 8) -#define REG_R14 (14 << 8) -#define REG_R15 (15 << 8) - -/* - * Exit Qualifications for MOV for Debug Register Access - */ -#define DEBUG_REG_ACCESS_NUM 0x7 /* 2:0, number of debug reg. */ -#define DEBUG_REG_ACCESS_TYPE 0x10 /* 4, direction of access */ -#define TYPE_MOV_TO_DR (0 << 4) -#define TYPE_MOV_FROM_DR (1 << 4) -#define DEBUG_REG_ACCESS_REG 0xf00 /* 11:8, general purpose reg. */ - - -/* segment AR */ -#define SEGMENT_AR_L_MASK (1 << 13) - -#define AR_TYPE_ACCESSES_MASK 1 -#define AR_TYPE_READABLE_MASK (1 << 1) -#define AR_TYPE_WRITEABLE_MASK (1 << 2) -#define AR_TYPE_CODE_MASK (1 << 3) -#define AR_TYPE_MASK 0x0f -#define AR_TYPE_BUSY_64_TSS 11 -#define AR_TYPE_BUSY_32_TSS 11 -#define AR_TYPE_BUSY_16_TSS 3 -#define AR_TYPE_LDT 2 - -#define AR_UNUSABLE_MASK (1 << 16) -#define AR_S_MASK (1 << 4) -#define AR_P_MASK (1 << 7) -#define AR_L_MASK (1 << 13) -#define AR_DB_MASK (1 << 14) -#define AR_G_MASK (1 << 15) -#define AR_DPL_SHIFT 5 -#define AR_DPL(ar) (((ar) >> AR_DPL_SHIFT) & 3) - -#define AR_RESERVD_MASK 0xfffe0f00 - -#define MSR_IA32_VMX_BASIC 0x480 -#define MSR_IA32_VMX_PINBASED_CTLS 0x481 -#define MSR_IA32_VMX_PROCBASED_CTLS 0x482 -#define MSR_IA32_VMX_EXIT_CTLS 0x483 -#define MSR_IA32_VMX_ENTRY_CTLS 0x484 -#define MSR_IA32_VMX_MISC 0x485 -#define MSR_IA32_VMX_CR0_FIXED0 0x486 -#define MSR_IA32_VMX_CR0_FIXED1 0x487 -#define MSR_IA32_VMX_CR4_FIXED0 0x488 -#define MSR_IA32_VMX_CR4_FIXED1 0x489 -#define MSR_IA32_VMX_VMCS_ENUM 0x48a -#define MSR_IA32_VMX_PROCBASED_CTLS2 0x48b - -#define MSR_IA32_FEATURE_CONTROL 0x3a -#define MSR_IA32_FEATURE_CONTROL_LOCKED 0x1 -#define MSR_IA32_FEATURE_CONTROL_VMXON_ENABLED 0x4 - -#define APIC_ACCESS_PAGE_PRIVATE_MEMSLOT 9 - -#endif diff --git a/drivers/kvm/x86.c b/drivers/kvm/x86.c deleted file mode 100644 index b37c0093d728..000000000000 --- a/drivers/kvm/x86.c +++ /dev/null @@ -1,3148 +0,0 @@ -/* - * Kernel-based Virtual Machine driver for Linux - * - * derived from drivers/kvm/kvm_main.c - * - * Copyright (C) 2006 Qumranet, Inc. - * - * Authors: - * Avi Kivity - * Yaniv Kamay - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - * - */ - -#include "kvm.h" -#include "x86.h" -#include "x86_emulate.h" -#include "segment_descriptor.h" -#include "irq.h" -#include "mmu.h" - -#include -#include -#include -#include -#include -#include - -#include -#include - -#define MAX_IO_MSRS 256 -#define CR0_RESERVED_BITS \ - (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ - | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ - | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) -#define CR4_RESERVED_BITS \ - (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ - | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ - | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR \ - | X86_CR4_OSXMMEXCPT | X86_CR4_VMXE)) - -#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) -#define EFER_RESERVED_BITS 0xfffffffffffff2fe - -#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM -#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU - -struct kvm_x86_ops *kvm_x86_ops; - -struct kvm_stats_debugfs_item debugfs_entries[] = { - { "pf_fixed", VCPU_STAT(pf_fixed) }, - { "pf_guest", VCPU_STAT(pf_guest) }, - { "tlb_flush", VCPU_STAT(tlb_flush) }, - { "invlpg", VCPU_STAT(invlpg) }, - { "exits", VCPU_STAT(exits) }, - { "io_exits", VCPU_STAT(io_exits) }, - { "mmio_exits", VCPU_STAT(mmio_exits) }, - { "signal_exits", VCPU_STAT(signal_exits) }, - { "irq_window", VCPU_STAT(irq_window_exits) }, - { "halt_exits", VCPU_STAT(halt_exits) }, - { "halt_wakeup", VCPU_STAT(halt_wakeup) }, - { "request_irq", VCPU_STAT(request_irq_exits) }, - { "irq_exits", VCPU_STAT(irq_exits) }, - { "host_state_reload", VCPU_STAT(host_state_reload) }, - { "efer_reload", VCPU_STAT(efer_reload) }, - { "fpu_reload", VCPU_STAT(fpu_reload) }, - { "insn_emulation", VCPU_STAT(insn_emulation) }, - { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, - { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, - { "mmu_pte_write", VM_STAT(mmu_pte_write) }, - { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, - { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, - { "mmu_flooded", VM_STAT(mmu_flooded) }, - { "mmu_recycled", VM_STAT(mmu_recycled) }, - { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, - { NULL } -}; - - -unsigned long segment_base(u16 selector) -{ - struct descriptor_table gdt; - struct segment_descriptor *d; - unsigned long table_base; - unsigned long v; - - if (selector == 0) - return 0; - - asm("sgdt %0" : "=m"(gdt)); - table_base = gdt.base; - - if (selector & 4) { /* from ldt */ - u16 ldt_selector; - - asm("sldt %0" : "=g"(ldt_selector)); - table_base = segment_base(ldt_selector); - } - d = (struct segment_descriptor *)(table_base + (selector & ~7)); - v = d->base_low | ((unsigned long)d->base_mid << 16) | - ((unsigned long)d->base_high << 24); -#ifdef CONFIG_X86_64 - if (d->system == 0 && (d->type == 2 || d->type == 9 || d->type == 11)) - v |= ((unsigned long) \ - ((struct segment_descriptor_64 *)d)->base_higher) << 32; -#endif - return v; -} -EXPORT_SYMBOL_GPL(segment_base); - -u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) -{ - if (irqchip_in_kernel(vcpu->kvm)) - return vcpu->arch.apic_base; - else - return vcpu->arch.apic_base; -} -EXPORT_SYMBOL_GPL(kvm_get_apic_base); - -void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) -{ - /* TODO: reserve bits check */ - if (irqchip_in_kernel(vcpu->kvm)) - kvm_lapic_set_base(vcpu, data); - else - vcpu->arch.apic_base = data; -} -EXPORT_SYMBOL_GPL(kvm_set_apic_base); - -void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr) -{ - WARN_ON(vcpu->arch.exception.pending); - vcpu->arch.exception.pending = true; - vcpu->arch.exception.has_error_code = false; - vcpu->arch.exception.nr = nr; -} -EXPORT_SYMBOL_GPL(kvm_queue_exception); - -void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, - u32 error_code) -{ - ++vcpu->stat.pf_guest; - if (vcpu->arch.exception.pending && vcpu->arch.exception.nr == PF_VECTOR) { - printk(KERN_DEBUG "kvm: inject_page_fault:" - " double fault 0x%lx\n", addr); - vcpu->arch.exception.nr = DF_VECTOR; - vcpu->arch.exception.error_code = 0; - return; - } - vcpu->arch.cr2 = addr; - kvm_queue_exception_e(vcpu, PF_VECTOR, error_code); -} - -void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code) -{ - WARN_ON(vcpu->arch.exception.pending); - vcpu->arch.exception.pending = true; - vcpu->arch.exception.has_error_code = true; - vcpu->arch.exception.nr = nr; - vcpu->arch.exception.error_code = error_code; -} -EXPORT_SYMBOL_GPL(kvm_queue_exception_e); - -static void __queue_exception(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->queue_exception(vcpu, vcpu->arch.exception.nr, - vcpu->arch.exception.has_error_code, - vcpu->arch.exception.error_code); -} - -/* - * Load the pae pdptrs. Return true is they are all valid. - */ -int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; - unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2; - int i; - int ret; - u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; - - mutex_lock(&vcpu->kvm->lock); - ret = kvm_read_guest_page(vcpu->kvm, pdpt_gfn, pdpte, - offset * sizeof(u64), sizeof(pdpte)); - if (ret < 0) { - ret = 0; - goto out; - } - for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { - if ((pdpte[i] & 1) && (pdpte[i] & 0xfffffff0000001e6ull)) { - ret = 0; - goto out; - } - } - ret = 1; - - memcpy(vcpu->arch.pdptrs, pdpte, sizeof(vcpu->arch.pdptrs)); -out: - mutex_unlock(&vcpu->kvm->lock); - - return ret; -} - -static bool pdptrs_changed(struct kvm_vcpu *vcpu) -{ - u64 pdpte[ARRAY_SIZE(vcpu->arch.pdptrs)]; - bool changed = true; - int r; - - if (is_long_mode(vcpu) || !is_pae(vcpu)) - return false; - - mutex_lock(&vcpu->kvm->lock); - r = kvm_read_guest(vcpu->kvm, vcpu->arch.cr3 & ~31u, pdpte, sizeof(pdpte)); - if (r < 0) - goto out; - changed = memcmp(pdpte, vcpu->arch.pdptrs, sizeof(pdpte)) != 0; -out: - mutex_unlock(&vcpu->kvm->lock); - - return changed; -} - -void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ - if (cr0 & CR0_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n", - cr0, vcpu->arch.cr0); - kvm_inject_gp(vcpu, 0); - return; - } - - if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) { - printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n"); - kvm_inject_gp(vcpu, 0); - return; - } - - if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) { - printk(KERN_DEBUG "set_cr0: #GP, set PG flag " - "and a clear PE flag\n"); - kvm_inject_gp(vcpu, 0); - return; - } - - if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) { -#ifdef CONFIG_X86_64 - if ((vcpu->arch.shadow_efer & EFER_LME)) { - int cs_db, cs_l; - - if (!is_pae(vcpu)) { - printk(KERN_DEBUG "set_cr0: #GP, start paging " - "in long mode while PAE is disabled\n"); - kvm_inject_gp(vcpu, 0); - return; - } - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - if (cs_l) { - printk(KERN_DEBUG "set_cr0: #GP, start paging " - "in long mode while CS.L == 1\n"); - kvm_inject_gp(vcpu, 0); - return; - - } - } else -#endif - if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->arch.cr3)) { - printk(KERN_DEBUG "set_cr0: #GP, pdptrs " - "reserved bits\n"); - kvm_inject_gp(vcpu, 0); - return; - } - - } - - kvm_x86_ops->set_cr0(vcpu, cr0); - vcpu->arch.cr0 = cr0; - - mutex_lock(&vcpu->kvm->lock); - kvm_mmu_reset_context(vcpu); - mutex_unlock(&vcpu->kvm->lock); - return; -} -EXPORT_SYMBOL_GPL(set_cr0); - -void lmsw(struct kvm_vcpu *vcpu, unsigned long msw) -{ - set_cr0(vcpu, (vcpu->arch.cr0 & ~0x0ful) | (msw & 0x0f)); -} -EXPORT_SYMBOL_GPL(lmsw); - -void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - if (cr4 & CR4_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n"); - kvm_inject_gp(vcpu, 0); - return; - } - - if (is_long_mode(vcpu)) { - if (!(cr4 & X86_CR4_PAE)) { - printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while " - "in long mode\n"); - kvm_inject_gp(vcpu, 0); - return; - } - } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & X86_CR4_PAE) - && !load_pdptrs(vcpu, vcpu->arch.cr3)) { - printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n"); - kvm_inject_gp(vcpu, 0); - return; - } - - if (cr4 & X86_CR4_VMXE) { - printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n"); - kvm_inject_gp(vcpu, 0); - return; - } - kvm_x86_ops->set_cr4(vcpu, cr4); - vcpu->arch.cr4 = cr4; - mutex_lock(&vcpu->kvm->lock); - kvm_mmu_reset_context(vcpu); - mutex_unlock(&vcpu->kvm->lock); -} -EXPORT_SYMBOL_GPL(set_cr4); - -void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - if (cr3 == vcpu->arch.cr3 && !pdptrs_changed(vcpu)) { - kvm_mmu_flush_tlb(vcpu); - return; - } - - if (is_long_mode(vcpu)) { - if (cr3 & CR3_L_MODE_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n"); - kvm_inject_gp(vcpu, 0); - return; - } - } else { - if (is_pae(vcpu)) { - if (cr3 & CR3_PAE_RESERVED_BITS) { - printk(KERN_DEBUG - "set_cr3: #GP, reserved bits\n"); - kvm_inject_gp(vcpu, 0); - return; - } - if (is_paging(vcpu) && !load_pdptrs(vcpu, cr3)) { - printk(KERN_DEBUG "set_cr3: #GP, pdptrs " - "reserved bits\n"); - kvm_inject_gp(vcpu, 0); - return; - } - } - /* - * We don't check reserved bits in nonpae mode, because - * this isn't enforced, and VMware depends on this. - */ - } - - mutex_lock(&vcpu->kvm->lock); - /* - * Does the new cr3 value map to physical memory? (Note, we - * catch an invalid cr3 even in real-mode, because it would - * cause trouble later on when we turn on paging anyway.) - * - * A real CPU would silently accept an invalid cr3 and would - * attempt to use it - with largely undefined (and often hard - * to debug) behavior on the guest side. - */ - if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT))) - kvm_inject_gp(vcpu, 0); - else { - vcpu->arch.cr3 = cr3; - vcpu->arch.mmu.new_cr3(vcpu); - } - mutex_unlock(&vcpu->kvm->lock); -} -EXPORT_SYMBOL_GPL(set_cr3); - -void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - if (cr8 & CR8_RESERVED_BITS) { - printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8); - kvm_inject_gp(vcpu, 0); - return; - } - if (irqchip_in_kernel(vcpu->kvm)) - kvm_lapic_set_tpr(vcpu, cr8); - else - vcpu->arch.cr8 = cr8; -} -EXPORT_SYMBOL_GPL(set_cr8); - -unsigned long get_cr8(struct kvm_vcpu *vcpu) -{ - if (irqchip_in_kernel(vcpu->kvm)) - return kvm_lapic_get_cr8(vcpu); - else - return vcpu->arch.cr8; -} -EXPORT_SYMBOL_GPL(get_cr8); - -/* - * List of msr numbers which we expose to userspace through KVM_GET_MSRS - * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. - * - * This list is modified at module load time to reflect the - * capabilities of the host cpu. - */ -static u32 msrs_to_save[] = { - MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, - MSR_K6_STAR, -#ifdef CONFIG_X86_64 - MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, -#endif - MSR_IA32_TIME_STAMP_COUNTER, -}; - -static unsigned num_msrs_to_save; - -static u32 emulated_msrs[] = { - MSR_IA32_MISC_ENABLE, -}; - -#ifdef CONFIG_X86_64 - -static void set_efer(struct kvm_vcpu *vcpu, u64 efer) -{ - if (efer & EFER_RESERVED_BITS) { - printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n", - efer); - kvm_inject_gp(vcpu, 0); - return; - } - - if (is_paging(vcpu) - && (vcpu->arch.shadow_efer & EFER_LME) != (efer & EFER_LME)) { - printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n"); - kvm_inject_gp(vcpu, 0); - return; - } - - kvm_x86_ops->set_efer(vcpu, efer); - - efer &= ~EFER_LMA; - efer |= vcpu->arch.shadow_efer & EFER_LMA; - - vcpu->arch.shadow_efer = efer; -} - -#endif - -/* - * Writes msr value into into the appropriate "register". - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) -{ - return kvm_x86_ops->set_msr(vcpu, msr_index, data); -} - -/* - * Adapt set_msr() to msr_io()'s calling convention - */ -static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) -{ - return kvm_set_msr(vcpu, index, *data); -} - - -int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) -{ - switch (msr) { -#ifdef CONFIG_X86_64 - case MSR_EFER: - set_efer(vcpu, data); - break; -#endif - case MSR_IA32_MC0_STATUS: - pr_unimpl(vcpu, "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n", - __FUNCTION__, data); - break; - case MSR_IA32_MCG_STATUS: - pr_unimpl(vcpu, "%s: MSR_IA32_MCG_STATUS 0x%llx, nop\n", - __FUNCTION__, data); - break; - case MSR_IA32_UCODE_REV: - case MSR_IA32_UCODE_WRITE: - case 0x200 ... 0x2ff: /* MTRRs */ - break; - case MSR_IA32_APICBASE: - kvm_set_apic_base(vcpu, data); - break; - case MSR_IA32_MISC_ENABLE: - vcpu->arch.ia32_misc_enable_msr = data; - break; - default: - pr_unimpl(vcpu, "unhandled wrmsr: 0x%x\n", msr); - return 1; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_set_msr_common); - - -/* - * Reads an msr value (of 'msr_index') into 'pdata'. - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) -{ - return kvm_x86_ops->get_msr(vcpu, msr_index, pdata); -} - -int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) -{ - u64 data; - - switch (msr) { - case 0xc0010010: /* SYSCFG */ - case 0xc0010015: /* HWCR */ - case MSR_IA32_PLATFORM_ID: - case MSR_IA32_P5_MC_ADDR: - case MSR_IA32_P5_MC_TYPE: - case MSR_IA32_MC0_CTL: - case MSR_IA32_MCG_STATUS: - case MSR_IA32_MCG_CAP: - case MSR_IA32_MC0_MISC: - case MSR_IA32_MC0_MISC+4: - case MSR_IA32_MC0_MISC+8: - case MSR_IA32_MC0_MISC+12: - case MSR_IA32_MC0_MISC+16: - case MSR_IA32_UCODE_REV: - case MSR_IA32_PERF_STATUS: - case MSR_IA32_EBL_CR_POWERON: - /* MTRR registers */ - case 0xfe: - case 0x200 ... 0x2ff: - data = 0; - break; - case 0xcd: /* fsb frequency */ - data = 3; - break; - case MSR_IA32_APICBASE: - data = kvm_get_apic_base(vcpu); - break; - case MSR_IA32_MISC_ENABLE: - data = vcpu->arch.ia32_misc_enable_msr; - break; -#ifdef CONFIG_X86_64 - case MSR_EFER: - data = vcpu->arch.shadow_efer; - break; -#endif - default: - pr_unimpl(vcpu, "unhandled rdmsr: 0x%x\n", msr); - return 1; - } - *pdata = data; - return 0; -} -EXPORT_SYMBOL_GPL(kvm_get_msr_common); - -/* - * Read or write a bunch of msrs. All parameters are kernel addresses. - * - * @return number of msrs set successfully. - */ -static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, - struct kvm_msr_entry *entries, - int (*do_msr)(struct kvm_vcpu *vcpu, - unsigned index, u64 *data)) -{ - int i; - - vcpu_load(vcpu); - - for (i = 0; i < msrs->nmsrs; ++i) - if (do_msr(vcpu, entries[i].index, &entries[i].data)) - break; - - vcpu_put(vcpu); - - return i; -} - -/* - * Read or write a bunch of msrs. Parameters are user addresses. - * - * @return number of msrs set successfully. - */ -static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, - int (*do_msr)(struct kvm_vcpu *vcpu, - unsigned index, u64 *data), - int writeback) -{ - struct kvm_msrs msrs; - struct kvm_msr_entry *entries; - int r, n; - unsigned size; - - r = -EFAULT; - if (copy_from_user(&msrs, user_msrs, sizeof msrs)) - goto out; - - r = -E2BIG; - if (msrs.nmsrs >= MAX_IO_MSRS) - goto out; - - r = -ENOMEM; - size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; - entries = vmalloc(size); - if (!entries) - goto out; - - r = -EFAULT; - if (copy_from_user(entries, user_msrs->entries, size)) - goto out_free; - - r = n = __msr_io(vcpu, &msrs, entries, do_msr); - if (r < 0) - goto out_free; - - r = -EFAULT; - if (writeback && copy_to_user(user_msrs->entries, entries, size)) - goto out_free; - - r = n; - -out_free: - vfree(entries); -out: - return r; -} - -/* - * Make sure that a cpu that is being hot-unplugged does not have any vcpus - * cached on it. - */ -void decache_vcpus_on_cpu(int cpu) -{ - struct kvm *vm; - struct kvm_vcpu *vcpu; - int i; - - spin_lock(&kvm_lock); - list_for_each_entry(vm, &vm_list, vm_list) - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - vcpu = vm->vcpus[i]; - if (!vcpu) - continue; - /* - * If the vcpu is locked, then it is running on some - * other cpu and therefore it is not cached on the - * cpu in question. - * - * If it's not locked, check the last cpu it executed - * on. - */ - if (mutex_trylock(&vcpu->mutex)) { - if (vcpu->cpu == cpu) { - kvm_x86_ops->vcpu_decache(vcpu); - vcpu->cpu = -1; - } - mutex_unlock(&vcpu->mutex); - } - } - spin_unlock(&kvm_lock); -} - -int kvm_dev_ioctl_check_extension(long ext) -{ - int r; - - switch (ext) { - case KVM_CAP_IRQCHIP: - case KVM_CAP_HLT: - case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: - case KVM_CAP_USER_MEMORY: - case KVM_CAP_SET_TSS_ADDR: - case KVM_CAP_EXT_CPUID: - r = 1; - break; - default: - r = 0; - break; - } - return r; - -} - -long kvm_arch_dev_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) -{ - void __user *argp = (void __user *)arg; - long r; - - switch (ioctl) { - case KVM_GET_MSR_INDEX_LIST: { - struct kvm_msr_list __user *user_msr_list = argp; - struct kvm_msr_list msr_list; - unsigned n; - - r = -EFAULT; - if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list)) - goto out; - n = msr_list.nmsrs; - msr_list.nmsrs = num_msrs_to_save + ARRAY_SIZE(emulated_msrs); - if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list)) - goto out; - r = -E2BIG; - if (n < num_msrs_to_save) - goto out; - r = -EFAULT; - if (copy_to_user(user_msr_list->indices, &msrs_to_save, - num_msrs_to_save * sizeof(u32))) - goto out; - if (copy_to_user(user_msr_list->indices - + num_msrs_to_save * sizeof(u32), - &emulated_msrs, - ARRAY_SIZE(emulated_msrs) * sizeof(u32))) - goto out; - r = 0; - break; - } - default: - r = -EINVAL; - } -out: - return r; -} - -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) -{ - kvm_x86_ops->vcpu_load(vcpu, cpu); -} - -void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->vcpu_put(vcpu); - kvm_put_guest_fpu(vcpu); -} - -static int is_efer_nx(void) -{ - u64 efer; - - rdmsrl(MSR_EFER, efer); - return efer & EFER_NX; -} - -static void cpuid_fix_nx_cap(struct kvm_vcpu *vcpu) -{ - int i; - struct kvm_cpuid_entry2 *e, *entry; - - entry = NULL; - for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { - e = &vcpu->arch.cpuid_entries[i]; - if (e->function == 0x80000001) { - entry = e; - break; - } - } - if (entry && (entry->edx & (1 << 20)) && !is_efer_nx()) { - entry->edx &= ~(1 << 20); - printk(KERN_INFO "kvm: guest NX capability removed\n"); - } -} - -/* when an old userspace process fills a new kernel module */ -static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, - struct kvm_cpuid *cpuid, - struct kvm_cpuid_entry __user *entries) -{ - int r, i; - struct kvm_cpuid_entry *cpuid_entries; - - r = -E2BIG; - if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) - goto out; - r = -ENOMEM; - cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry) * cpuid->nent); - if (!cpuid_entries) - goto out; - r = -EFAULT; - if (copy_from_user(cpuid_entries, entries, - cpuid->nent * sizeof(struct kvm_cpuid_entry))) - goto out_free; - for (i = 0; i < cpuid->nent; i++) { - vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function; - vcpu->arch.cpuid_entries[i].eax = cpuid_entries[i].eax; - vcpu->arch.cpuid_entries[i].ebx = cpuid_entries[i].ebx; - vcpu->arch.cpuid_entries[i].ecx = cpuid_entries[i].ecx; - vcpu->arch.cpuid_entries[i].edx = cpuid_entries[i].edx; - vcpu->arch.cpuid_entries[i].index = 0; - vcpu->arch.cpuid_entries[i].flags = 0; - vcpu->arch.cpuid_entries[i].padding[0] = 0; - vcpu->arch.cpuid_entries[i].padding[1] = 0; - vcpu->arch.cpuid_entries[i].padding[2] = 0; - } - vcpu->arch.cpuid_nent = cpuid->nent; - cpuid_fix_nx_cap(vcpu); - r = 0; - -out_free: - vfree(cpuid_entries); -out: - return r; -} - -static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, - struct kvm_cpuid2 *cpuid, - struct kvm_cpuid_entry2 __user *entries) -{ - int r; - - r = -E2BIG; - if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) - goto out; - r = -EFAULT; - if (copy_from_user(&vcpu->arch.cpuid_entries, entries, - cpuid->nent * sizeof(struct kvm_cpuid_entry2))) - goto out; - vcpu->arch.cpuid_nent = cpuid->nent; - return 0; - -out: - return r; -} - -static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, - struct kvm_cpuid2 *cpuid, - struct kvm_cpuid_entry2 __user *entries) -{ - int r; - - r = -E2BIG; - if (cpuid->nent < vcpu->arch.cpuid_nent) - goto out; - r = -EFAULT; - if (copy_to_user(entries, &vcpu->arch.cpuid_entries, - vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2))) - goto out; - return 0; - -out: - cpuid->nent = vcpu->arch.cpuid_nent; - return r; -} - -static inline u32 bit(int bitno) -{ - return 1 << (bitno & 31); -} - -static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, - u32 index) -{ - entry->function = function; - entry->index = index; - cpuid_count(entry->function, entry->index, - &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); - entry->flags = 0; -} - -static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, - u32 index, int *nent, int maxnent) -{ - const u32 kvm_supported_word0_x86_features = bit(X86_FEATURE_FPU) | - bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) | - bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) | - bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) | - bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) | - bit(X86_FEATURE_SEP) | bit(X86_FEATURE_PGE) | - bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) | - bit(X86_FEATURE_CLFLSH) | bit(X86_FEATURE_MMX) | - bit(X86_FEATURE_FXSR) | bit(X86_FEATURE_XMM) | - bit(X86_FEATURE_XMM2) | bit(X86_FEATURE_SELFSNOOP); - const u32 kvm_supported_word1_x86_features = bit(X86_FEATURE_FPU) | - bit(X86_FEATURE_VME) | bit(X86_FEATURE_DE) | - bit(X86_FEATURE_PSE) | bit(X86_FEATURE_TSC) | - bit(X86_FEATURE_MSR) | bit(X86_FEATURE_PAE) | - bit(X86_FEATURE_CX8) | bit(X86_FEATURE_APIC) | - bit(X86_FEATURE_PGE) | - bit(X86_FEATURE_CMOV) | bit(X86_FEATURE_PSE36) | - bit(X86_FEATURE_MMX) | bit(X86_FEATURE_FXSR) | - bit(X86_FEATURE_SYSCALL) | - (bit(X86_FEATURE_NX) && is_efer_nx()) | -#ifdef CONFIG_X86_64 - bit(X86_FEATURE_LM) | -#endif - bit(X86_FEATURE_MMXEXT) | - bit(X86_FEATURE_3DNOWEXT) | - bit(X86_FEATURE_3DNOW); - const u32 kvm_supported_word3_x86_features = - bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16); - const u32 kvm_supported_word6_x86_features = - bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY); - - /* all func 2 cpuid_count() should be called on the same cpu */ - get_cpu(); - do_cpuid_1_ent(entry, function, index); - ++*nent; - - switch (function) { - case 0: - entry->eax = min(entry->eax, (u32)0xb); - break; - case 1: - entry->edx &= kvm_supported_word0_x86_features; - entry->ecx &= kvm_supported_word3_x86_features; - break; - /* function 2 entries are STATEFUL. That is, repeated cpuid commands - * may return different values. This forces us to get_cpu() before - * issuing the first command, and also to emulate this annoying behavior - * in kvm_emulate_cpuid() using KVM_CPUID_FLAG_STATE_READ_NEXT */ - case 2: { - int t, times = entry->eax & 0xff; - - entry->flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; - for (t = 1; t < times && *nent < maxnent; ++t) { - do_cpuid_1_ent(&entry[t], function, 0); - entry[t].flags |= KVM_CPUID_FLAG_STATEFUL_FUNC; - ++*nent; - } - break; - } - /* function 4 and 0xb have additional index. */ - case 4: { - int index, cache_type; - - entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; - /* read more entries until cache_type is zero */ - for (index = 1; *nent < maxnent; ++index) { - cache_type = entry[index - 1].eax & 0x1f; - if (!cache_type) - break; - do_cpuid_1_ent(&entry[index], function, index); - entry[index].flags |= - KVM_CPUID_FLAG_SIGNIFCANT_INDEX; - ++*nent; - } - break; - } - case 0xb: { - int index, level_type; - - entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; - /* read more entries until level_type is zero */ - for (index = 1; *nent < maxnent; ++index) { - level_type = entry[index - 1].ecx & 0xff; - if (!level_type) - break; - do_cpuid_1_ent(&entry[index], function, index); - entry[index].flags |= - KVM_CPUID_FLAG_SIGNIFCANT_INDEX; - ++*nent; - } - break; - } - case 0x80000000: - entry->eax = min(entry->eax, 0x8000001a); - break; - case 0x80000001: - entry->edx &= kvm_supported_word1_x86_features; - entry->ecx &= kvm_supported_word6_x86_features; - break; - } - put_cpu(); -} - -static int kvm_vm_ioctl_get_supported_cpuid(struct kvm *kvm, - struct kvm_cpuid2 *cpuid, - struct kvm_cpuid_entry2 __user *entries) -{ - struct kvm_cpuid_entry2 *cpuid_entries; - int limit, nent = 0, r = -E2BIG; - u32 func; - - if (cpuid->nent < 1) - goto out; - r = -ENOMEM; - cpuid_entries = vmalloc(sizeof(struct kvm_cpuid_entry2) * cpuid->nent); - if (!cpuid_entries) - goto out; - - do_cpuid_ent(&cpuid_entries[0], 0, 0, &nent, cpuid->nent); - limit = cpuid_entries[0].eax; - for (func = 1; func <= limit && nent < cpuid->nent; ++func) - do_cpuid_ent(&cpuid_entries[nent], func, 0, - &nent, cpuid->nent); - r = -E2BIG; - if (nent >= cpuid->nent) - goto out_free; - - do_cpuid_ent(&cpuid_entries[nent], 0x80000000, 0, &nent, cpuid->nent); - limit = cpuid_entries[nent - 1].eax; - for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func) - do_cpuid_ent(&cpuid_entries[nent], func, 0, - &nent, cpuid->nent); - r = -EFAULT; - if (copy_to_user(entries, cpuid_entries, - nent * sizeof(struct kvm_cpuid_entry2))) - goto out_free; - cpuid->nent = nent; - r = 0; - -out_free: - vfree(cpuid_entries); -out: - return r; -} - -static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu, - struct kvm_lapic_state *s) -{ - vcpu_load(vcpu); - memcpy(s->regs, vcpu->arch.apic->regs, sizeof *s); - vcpu_put(vcpu); - - return 0; -} - -static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, - struct kvm_lapic_state *s) -{ - vcpu_load(vcpu); - memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); - kvm_apic_post_state_restore(vcpu); - vcpu_put(vcpu); - - return 0; -} - -static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, - struct kvm_interrupt *irq) -{ - if (irq->irq < 0 || irq->irq >= 256) - return -EINVAL; - if (irqchip_in_kernel(vcpu->kvm)) - return -ENXIO; - vcpu_load(vcpu); - - set_bit(irq->irq, vcpu->arch.irq_pending); - set_bit(irq->irq / BITS_PER_LONG, &vcpu->arch.irq_summary); - - vcpu_put(vcpu); - - return 0; -} - -long kvm_arch_vcpu_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) -{ - struct kvm_vcpu *vcpu = filp->private_data; - void __user *argp = (void __user *)arg; - int r; - - switch (ioctl) { - case KVM_GET_LAPIC: { - struct kvm_lapic_state lapic; - - memset(&lapic, 0, sizeof lapic); - r = kvm_vcpu_ioctl_get_lapic(vcpu, &lapic); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &lapic, sizeof lapic)) - goto out; - r = 0; - break; - } - case KVM_SET_LAPIC: { - struct kvm_lapic_state lapic; - - r = -EFAULT; - if (copy_from_user(&lapic, argp, sizeof lapic)) - goto out; - r = kvm_vcpu_ioctl_set_lapic(vcpu, &lapic);; - if (r) - goto out; - r = 0; - break; - } - case KVM_INTERRUPT: { - struct kvm_interrupt irq; - - r = -EFAULT; - if (copy_from_user(&irq, argp, sizeof irq)) - goto out; - r = kvm_vcpu_ioctl_interrupt(vcpu, &irq); - if (r) - goto out; - r = 0; - break; - } - case KVM_SET_CPUID: { - struct kvm_cpuid __user *cpuid_arg = argp; - struct kvm_cpuid cpuid; - - r = -EFAULT; - if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) - goto out; - r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries); - if (r) - goto out; - break; - } - case KVM_SET_CPUID2: { - struct kvm_cpuid2 __user *cpuid_arg = argp; - struct kvm_cpuid2 cpuid; - - r = -EFAULT; - if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) - goto out; - r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, - cpuid_arg->entries); - if (r) - goto out; - break; - } - case KVM_GET_CPUID2: { - struct kvm_cpuid2 __user *cpuid_arg = argp; - struct kvm_cpuid2 cpuid; - - r = -EFAULT; - if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) - goto out; - r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, - cpuid_arg->entries); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) - goto out; - r = 0; - break; - } - case KVM_GET_MSRS: - r = msr_io(vcpu, argp, kvm_get_msr, 1); - break; - case KVM_SET_MSRS: - r = msr_io(vcpu, argp, do_set_msr, 0); - break; - default: - r = -EINVAL; - } -out: - return r; -} - -static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr) -{ - int ret; - - if (addr > (unsigned int)(-3 * PAGE_SIZE)) - return -1; - ret = kvm_x86_ops->set_tss_addr(kvm, addr); - return ret; -} - -static int kvm_vm_ioctl_set_nr_mmu_pages(struct kvm *kvm, - u32 kvm_nr_mmu_pages) -{ - if (kvm_nr_mmu_pages < KVM_MIN_ALLOC_MMU_PAGES) - return -EINVAL; - - mutex_lock(&kvm->lock); - - kvm_mmu_change_mmu_pages(kvm, kvm_nr_mmu_pages); - kvm->arch.n_requested_mmu_pages = kvm_nr_mmu_pages; - - mutex_unlock(&kvm->lock); - return 0; -} - -static int kvm_vm_ioctl_get_nr_mmu_pages(struct kvm *kvm) -{ - return kvm->arch.n_alloc_mmu_pages; -} - -gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn) -{ - int i; - struct kvm_mem_alias *alias; - - for (i = 0; i < kvm->arch.naliases; ++i) { - alias = &kvm->arch.aliases[i]; - if (gfn >= alias->base_gfn - && gfn < alias->base_gfn + alias->npages) - return alias->target_gfn + gfn - alias->base_gfn; - } - return gfn; -} - -/* - * Set a new alias region. Aliases map a portion of physical memory into - * another portion. This is useful for memory windows, for example the PC - * VGA region. - */ -static int kvm_vm_ioctl_set_memory_alias(struct kvm *kvm, - struct kvm_memory_alias *alias) -{ - int r, n; - struct kvm_mem_alias *p; - - r = -EINVAL; - /* General sanity checks */ - if (alias->memory_size & (PAGE_SIZE - 1)) - goto out; - if (alias->guest_phys_addr & (PAGE_SIZE - 1)) - goto out; - if (alias->slot >= KVM_ALIAS_SLOTS) - goto out; - if (alias->guest_phys_addr + alias->memory_size - < alias->guest_phys_addr) - goto out; - if (alias->target_phys_addr + alias->memory_size - < alias->target_phys_addr) - goto out; - - mutex_lock(&kvm->lock); - - p = &kvm->arch.aliases[alias->slot]; - p->base_gfn = alias->guest_phys_addr >> PAGE_SHIFT; - p->npages = alias->memory_size >> PAGE_SHIFT; - p->target_gfn = alias->target_phys_addr >> PAGE_SHIFT; - - for (n = KVM_ALIAS_SLOTS; n > 0; --n) - if (kvm->arch.aliases[n - 1].npages) - break; - kvm->arch.naliases = n; - - kvm_mmu_zap_all(kvm); - - mutex_unlock(&kvm->lock); - - return 0; - -out: - return r; -} - -static int kvm_vm_ioctl_get_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) -{ - int r; - - r = 0; - switch (chip->chip_id) { - case KVM_IRQCHIP_PIC_MASTER: - memcpy(&chip->chip.pic, - &pic_irqchip(kvm)->pics[0], - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_PIC_SLAVE: - memcpy(&chip->chip.pic, - &pic_irqchip(kvm)->pics[1], - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_IOAPIC: - memcpy(&chip->chip.ioapic, - ioapic_irqchip(kvm), - sizeof(struct kvm_ioapic_state)); - break; - default: - r = -EINVAL; - break; - } - return r; -} - -static int kvm_vm_ioctl_set_irqchip(struct kvm *kvm, struct kvm_irqchip *chip) -{ - int r; - - r = 0; - switch (chip->chip_id) { - case KVM_IRQCHIP_PIC_MASTER: - memcpy(&pic_irqchip(kvm)->pics[0], - &chip->chip.pic, - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_PIC_SLAVE: - memcpy(&pic_irqchip(kvm)->pics[1], - &chip->chip.pic, - sizeof(struct kvm_pic_state)); - break; - case KVM_IRQCHIP_IOAPIC: - memcpy(ioapic_irqchip(kvm), - &chip->chip.ioapic, - sizeof(struct kvm_ioapic_state)); - break; - default: - r = -EINVAL; - break; - } - kvm_pic_update_irq(pic_irqchip(kvm)); - return r; -} - -/* - * Get (and clear) the dirty memory log for a memory slot. - */ -int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log) -{ - int r; - int n; - struct kvm_memory_slot *memslot; - int is_dirty = 0; - - mutex_lock(&kvm->lock); - - r = kvm_get_dirty_log(kvm, log, &is_dirty); - if (r) - goto out; - - /* If nothing is dirty, don't bother messing with page tables. */ - if (is_dirty) { - kvm_mmu_slot_remove_write_access(kvm, log->slot); - kvm_flush_remote_tlbs(kvm); - memslot = &kvm->memslots[log->slot]; - n = ALIGN(memslot->npages, BITS_PER_LONG) / 8; - memset(memslot->dirty_bitmap, 0, n); - } - r = 0; -out: - mutex_unlock(&kvm->lock); - return r; -} - -long kvm_arch_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) -{ - struct kvm *kvm = filp->private_data; - void __user *argp = (void __user *)arg; - int r = -EINVAL; - - switch (ioctl) { - case KVM_SET_TSS_ADDR: - r = kvm_vm_ioctl_set_tss_addr(kvm, arg); - if (r < 0) - goto out; - break; - case KVM_SET_MEMORY_REGION: { - struct kvm_memory_region kvm_mem; - struct kvm_userspace_memory_region kvm_userspace_mem; - - r = -EFAULT; - if (copy_from_user(&kvm_mem, argp, sizeof kvm_mem)) - goto out; - kvm_userspace_mem.slot = kvm_mem.slot; - kvm_userspace_mem.flags = kvm_mem.flags; - kvm_userspace_mem.guest_phys_addr = kvm_mem.guest_phys_addr; - kvm_userspace_mem.memory_size = kvm_mem.memory_size; - r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 0); - if (r) - goto out; - break; - } - case KVM_SET_NR_MMU_PAGES: - r = kvm_vm_ioctl_set_nr_mmu_pages(kvm, arg); - if (r) - goto out; - break; - case KVM_GET_NR_MMU_PAGES: - r = kvm_vm_ioctl_get_nr_mmu_pages(kvm); - break; - case KVM_SET_MEMORY_ALIAS: { - struct kvm_memory_alias alias; - - r = -EFAULT; - if (copy_from_user(&alias, argp, sizeof alias)) - goto out; - r = kvm_vm_ioctl_set_memory_alias(kvm, &alias); - if (r) - goto out; - break; - } - case KVM_CREATE_IRQCHIP: - r = -ENOMEM; - kvm->arch.vpic = kvm_create_pic(kvm); - if (kvm->arch.vpic) { - r = kvm_ioapic_init(kvm); - if (r) { - kfree(kvm->arch.vpic); - kvm->arch.vpic = NULL; - goto out; - } - } else - goto out; - break; - case KVM_IRQ_LINE: { - struct kvm_irq_level irq_event; - - r = -EFAULT; - if (copy_from_user(&irq_event, argp, sizeof irq_event)) - goto out; - if (irqchip_in_kernel(kvm)) { - mutex_lock(&kvm->lock); - if (irq_event.irq < 16) - kvm_pic_set_irq(pic_irqchip(kvm), - irq_event.irq, - irq_event.level); - kvm_ioapic_set_irq(kvm->arch.vioapic, - irq_event.irq, - irq_event.level); - mutex_unlock(&kvm->lock); - r = 0; - } - break; - } - case KVM_GET_IRQCHIP: { - /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ - struct kvm_irqchip chip; - - r = -EFAULT; - if (copy_from_user(&chip, argp, sizeof chip)) - goto out; - r = -ENXIO; - if (!irqchip_in_kernel(kvm)) - goto out; - r = kvm_vm_ioctl_get_irqchip(kvm, &chip); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &chip, sizeof chip)) - goto out; - r = 0; - break; - } - case KVM_SET_IRQCHIP: { - /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ - struct kvm_irqchip chip; - - r = -EFAULT; - if (copy_from_user(&chip, argp, sizeof chip)) - goto out; - r = -ENXIO; - if (!irqchip_in_kernel(kvm)) - goto out; - r = kvm_vm_ioctl_set_irqchip(kvm, &chip); - if (r) - goto out; - r = 0; - break; - } - case KVM_GET_SUPPORTED_CPUID: { - struct kvm_cpuid2 __user *cpuid_arg = argp; - struct kvm_cpuid2 cpuid; - - r = -EFAULT; - if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) - goto out; - r = kvm_vm_ioctl_get_supported_cpuid(kvm, &cpuid, - cpuid_arg->entries); - if (r) - goto out; - - r = -EFAULT; - if (copy_to_user(cpuid_arg, &cpuid, sizeof cpuid)) - goto out; - r = 0; - break; - } - default: - ; - } -out: - return r; -} - -static void kvm_init_msr_list(void) -{ - u32 dummy[2]; - unsigned i, j; - - for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) { - if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0) - continue; - if (j < i) - msrs_to_save[j] = msrs_to_save[i]; - j++; - } - num_msrs_to_save = j; -} - -/* - * Only apic need an MMIO device hook, so shortcut now.. - */ -static struct kvm_io_device *vcpu_find_pervcpu_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - struct kvm_io_device *dev; - - if (vcpu->arch.apic) { - dev = &vcpu->arch.apic->dev; - if (dev->in_range(dev, addr)) - return dev; - } - return NULL; -} - - -static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - struct kvm_io_device *dev; - - dev = vcpu_find_pervcpu_dev(vcpu, addr); - if (dev == NULL) - dev = kvm_io_bus_find_dev(&vcpu->kvm->mmio_bus, addr); - return dev; -} - -int emulator_read_std(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - void *data = val; - - while (bytes) { - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); - unsigned offset = addr & (PAGE_SIZE-1); - unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); - int ret; - - if (gpa == UNMAPPED_GVA) - return X86EMUL_PROPAGATE_FAULT; - ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); - if (ret < 0) - return X86EMUL_UNHANDLEABLE; - - bytes -= tocopy; - data += tocopy; - addr += tocopy; - } - - return X86EMUL_CONTINUE; -} -EXPORT_SYMBOL_GPL(emulator_read_std); - -static int emulator_read_emulated(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - struct kvm_io_device *mmio_dev; - gpa_t gpa; - - if (vcpu->mmio_read_completed) { - memcpy(val, vcpu->mmio_data, bytes); - vcpu->mmio_read_completed = 0; - return X86EMUL_CONTINUE; - } - - gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); - - /* For APIC access vmexit */ - if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) - goto mmio; - - if (emulator_read_std(addr, val, bytes, vcpu) - == X86EMUL_CONTINUE) - return X86EMUL_CONTINUE; - if (gpa == UNMAPPED_GVA) - return X86EMUL_PROPAGATE_FAULT; - -mmio: - /* - * Is this MMIO handled locally? - */ - mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); - if (mmio_dev) { - kvm_iodevice_read(mmio_dev, gpa, bytes, val); - return X86EMUL_CONTINUE; - } - - vcpu->mmio_needed = 1; - vcpu->mmio_phys_addr = gpa; - vcpu->mmio_size = bytes; - vcpu->mmio_is_write = 0; - - return X86EMUL_UNHANDLEABLE; -} - -static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, - const void *val, int bytes) -{ - int ret; - - ret = kvm_write_guest(vcpu->kvm, gpa, val, bytes); - if (ret < 0) - return 0; - kvm_mmu_pte_write(vcpu, gpa, val, bytes); - return 1; -} - -static int emulator_write_emulated_onepage(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - struct kvm_io_device *mmio_dev; - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); - - if (gpa == UNMAPPED_GVA) { - kvm_inject_page_fault(vcpu, addr, 2); - return X86EMUL_PROPAGATE_FAULT; - } - - /* For APIC access vmexit */ - if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) - goto mmio; - - if (emulator_write_phys(vcpu, gpa, val, bytes)) - return X86EMUL_CONTINUE; - -mmio: - /* - * Is this MMIO handled locally? - */ - mmio_dev = vcpu_find_mmio_dev(vcpu, gpa); - if (mmio_dev) { - kvm_iodevice_write(mmio_dev, gpa, bytes, val); - return X86EMUL_CONTINUE; - } - - vcpu->mmio_needed = 1; - vcpu->mmio_phys_addr = gpa; - vcpu->mmio_size = bytes; - vcpu->mmio_is_write = 1; - memcpy(vcpu->mmio_data, val, bytes); - - return X86EMUL_CONTINUE; -} - -int emulator_write_emulated(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - /* Crossing a page boundary? */ - if (((addr + bytes - 1) ^ addr) & PAGE_MASK) { - int rc, now; - - now = -addr & ~PAGE_MASK; - rc = emulator_write_emulated_onepage(addr, val, now, vcpu); - if (rc != X86EMUL_CONTINUE) - return rc; - addr += now; - val += now; - bytes -= now; - } - return emulator_write_emulated_onepage(addr, val, bytes, vcpu); -} -EXPORT_SYMBOL_GPL(emulator_write_emulated); - -static int emulator_cmpxchg_emulated(unsigned long addr, - const void *old, - const void *new, - unsigned int bytes, - struct kvm_vcpu *vcpu) -{ - static int reported; - - if (!reported) { - reported = 1; - printk(KERN_WARNING "kvm: emulating exchange as write\n"); - } -#ifndef CONFIG_X86_64 - /* guests cmpxchg8b have to be emulated atomically */ - if (bytes == 8) { - gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); - struct page *page; - char *addr; - u64 val; - - if (gpa == UNMAPPED_GVA || - (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) - goto emul_write; - - if (((gpa + bytes - 1) & PAGE_MASK) != (gpa & PAGE_MASK)) - goto emul_write; - - val = *(u64 *)new; - page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); - addr = kmap_atomic(page, KM_USER0); - set_64bit((u64 *)(addr + offset_in_page(gpa)), val); - kunmap_atomic(addr, KM_USER0); - kvm_release_page_dirty(page); - } -emul_write: -#endif - - return emulator_write_emulated(addr, new, bytes, vcpu); -} - -static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg) -{ - return kvm_x86_ops->get_segment_base(vcpu, seg); -} - -int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address) -{ - return X86EMUL_CONTINUE; -} - -int emulate_clts(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->set_cr0(vcpu, vcpu->arch.cr0 & ~X86_CR0_TS); - return X86EMUL_CONTINUE; -} - -int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long *dest) -{ - struct kvm_vcpu *vcpu = ctxt->vcpu; - - switch (dr) { - case 0 ... 3: - *dest = kvm_x86_ops->get_dr(vcpu, dr); - return X86EMUL_CONTINUE; - default: - pr_unimpl(vcpu, "%s: unexpected dr %u\n", __FUNCTION__, dr); - return X86EMUL_UNHANDLEABLE; - } -} - -int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value) -{ - unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U; - int exception; - - kvm_x86_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception); - if (exception) { - /* FIXME: better handling */ - return X86EMUL_UNHANDLEABLE; - } - return X86EMUL_CONTINUE; -} - -void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) -{ - static int reported; - u8 opcodes[4]; - unsigned long rip = vcpu->arch.rip; - unsigned long rip_linear; - - rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); - - if (reported) - return; - - emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); - - printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", - context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); - reported = 1; -} -EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); - -struct x86_emulate_ops emulate_ops = { - .read_std = emulator_read_std, - .read_emulated = emulator_read_emulated, - .write_emulated = emulator_write_emulated, - .cmpxchg_emulated = emulator_cmpxchg_emulated, -}; - -int emulate_instruction(struct kvm_vcpu *vcpu, - struct kvm_run *run, - unsigned long cr2, - u16 error_code, - int no_decode) -{ - int r; - - vcpu->arch.mmio_fault_cr2 = cr2; - kvm_x86_ops->cache_regs(vcpu); - - vcpu->mmio_is_write = 0; - vcpu->arch.pio.string = 0; - - if (!no_decode) { - int cs_db, cs_l; - kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); - - vcpu->arch.emulate_ctxt.vcpu = vcpu; - vcpu->arch.emulate_ctxt.eflags = kvm_x86_ops->get_rflags(vcpu); - vcpu->arch.emulate_ctxt.mode = - (vcpu->arch.emulate_ctxt.eflags & X86_EFLAGS_VM) - ? X86EMUL_MODE_REAL : cs_l - ? X86EMUL_MODE_PROT64 : cs_db - ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; - - if (vcpu->arch.emulate_ctxt.mode == X86EMUL_MODE_PROT64) { - vcpu->arch.emulate_ctxt.cs_base = 0; - vcpu->arch.emulate_ctxt.ds_base = 0; - vcpu->arch.emulate_ctxt.es_base = 0; - vcpu->arch.emulate_ctxt.ss_base = 0; - } else { - vcpu->arch.emulate_ctxt.cs_base = - get_segment_base(vcpu, VCPU_SREG_CS); - vcpu->arch.emulate_ctxt.ds_base = - get_segment_base(vcpu, VCPU_SREG_DS); - vcpu->arch.emulate_ctxt.es_base = - get_segment_base(vcpu, VCPU_SREG_ES); - vcpu->arch.emulate_ctxt.ss_base = - get_segment_base(vcpu, VCPU_SREG_SS); - } - - vcpu->arch.emulate_ctxt.gs_base = - get_segment_base(vcpu, VCPU_SREG_GS); - vcpu->arch.emulate_ctxt.fs_base = - get_segment_base(vcpu, VCPU_SREG_FS); - - r = x86_decode_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); - ++vcpu->stat.insn_emulation; - if (r) { - ++vcpu->stat.insn_emulation_fail; - if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) - return EMULATE_DONE; - return EMULATE_FAIL; - } - } - - r = x86_emulate_insn(&vcpu->arch.emulate_ctxt, &emulate_ops); - - if (vcpu->arch.pio.string) - return EMULATE_DO_MMIO; - - if ((r || vcpu->mmio_is_write) && run) { - run->exit_reason = KVM_EXIT_MMIO; - run->mmio.phys_addr = vcpu->mmio_phys_addr; - memcpy(run->mmio.data, vcpu->mmio_data, 8); - run->mmio.len = vcpu->mmio_size; - run->mmio.is_write = vcpu->mmio_is_write; - } - - if (r) { - if (kvm_mmu_unprotect_page_virt(vcpu, cr2)) - return EMULATE_DONE; - if (!vcpu->mmio_needed) { - kvm_report_emulation_failure(vcpu, "mmio"); - return EMULATE_FAIL; - } - return EMULATE_DO_MMIO; - } - - kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->set_rflags(vcpu, vcpu->arch.emulate_ctxt.eflags); - - if (vcpu->mmio_is_write) { - vcpu->mmio_needed = 0; - return EMULATE_DO_MMIO; - } - - return EMULATE_DONE; -} -EXPORT_SYMBOL_GPL(emulate_instruction); - -static void free_pio_guest_pages(struct kvm_vcpu *vcpu) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(vcpu->arch.pio.guest_pages); ++i) - if (vcpu->arch.pio.guest_pages[i]) { - kvm_release_page_dirty(vcpu->arch.pio.guest_pages[i]); - vcpu->arch.pio.guest_pages[i] = NULL; - } -} - -static int pio_copy_data(struct kvm_vcpu *vcpu) -{ - void *p = vcpu->arch.pio_data; - void *q; - unsigned bytes; - int nr_pages = vcpu->arch.pio.guest_pages[1] ? 2 : 1; - - q = vmap(vcpu->arch.pio.guest_pages, nr_pages, VM_READ|VM_WRITE, - PAGE_KERNEL); - if (!q) { - free_pio_guest_pages(vcpu); - return -ENOMEM; - } - q += vcpu->arch.pio.guest_page_offset; - bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count; - if (vcpu->arch.pio.in) - memcpy(q, p, bytes); - else - memcpy(p, q, bytes); - q -= vcpu->arch.pio.guest_page_offset; - vunmap(q); - free_pio_guest_pages(vcpu); - return 0; -} - -int complete_pio(struct kvm_vcpu *vcpu) -{ - struct kvm_pio_request *io = &vcpu->arch.pio; - long delta; - int r; - - kvm_x86_ops->cache_regs(vcpu); - - if (!io->string) { - if (io->in) - memcpy(&vcpu->arch.regs[VCPU_REGS_RAX], vcpu->arch.pio_data, - io->size); - } else { - if (io->in) { - r = pio_copy_data(vcpu); - if (r) { - kvm_x86_ops->cache_regs(vcpu); - return r; - } - } - - delta = 1; - if (io->rep) { - delta *= io->cur_count; - /* - * The size of the register should really depend on - * current address size. - */ - vcpu->arch.regs[VCPU_REGS_RCX] -= delta; - } - if (io->down) - delta = -delta; - delta *= io->size; - if (io->in) - vcpu->arch.regs[VCPU_REGS_RDI] += delta; - else - vcpu->arch.regs[VCPU_REGS_RSI] += delta; - } - - kvm_x86_ops->decache_regs(vcpu); - - io->count -= io->cur_count; - io->cur_count = 0; - - return 0; -} - -static void kernel_pio(struct kvm_io_device *pio_dev, - struct kvm_vcpu *vcpu, - void *pd) -{ - /* TODO: String I/O for in kernel device */ - - mutex_lock(&vcpu->kvm->lock); - if (vcpu->arch.pio.in) - kvm_iodevice_read(pio_dev, vcpu->arch.pio.port, - vcpu->arch.pio.size, - pd); - else - kvm_iodevice_write(pio_dev, vcpu->arch.pio.port, - vcpu->arch.pio.size, - pd); - mutex_unlock(&vcpu->kvm->lock); -} - -static void pio_string_write(struct kvm_io_device *pio_dev, - struct kvm_vcpu *vcpu) -{ - struct kvm_pio_request *io = &vcpu->arch.pio; - void *pd = vcpu->arch.pio_data; - int i; - - mutex_lock(&vcpu->kvm->lock); - for (i = 0; i < io->cur_count; i++) { - kvm_iodevice_write(pio_dev, io->port, - io->size, - pd); - pd += io->size; - } - mutex_unlock(&vcpu->kvm->lock); -} - -static struct kvm_io_device *vcpu_find_pio_dev(struct kvm_vcpu *vcpu, - gpa_t addr) -{ - return kvm_io_bus_find_dev(&vcpu->kvm->pio_bus, addr); -} - -int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned port) -{ - struct kvm_io_device *pio_dev; - - vcpu->run->exit_reason = KVM_EXIT_IO; - vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->arch.pio.size = size; - vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = 1; - vcpu->run->io.port = vcpu->arch.pio.port = port; - vcpu->arch.pio.in = in; - vcpu->arch.pio.string = 0; - vcpu->arch.pio.down = 0; - vcpu->arch.pio.guest_page_offset = 0; - vcpu->arch.pio.rep = 0; - - kvm_x86_ops->cache_regs(vcpu); - memcpy(vcpu->arch.pio_data, &vcpu->arch.regs[VCPU_REGS_RAX], 4); - kvm_x86_ops->decache_regs(vcpu); - - kvm_x86_ops->skip_emulated_instruction(vcpu); - - pio_dev = vcpu_find_pio_dev(vcpu, port); - if (pio_dev) { - kernel_pio(pio_dev, vcpu, vcpu->arch.pio_data); - complete_pio(vcpu); - return 1; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_emulate_pio); - -int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned long count, int down, - gva_t address, int rep, unsigned port) -{ - unsigned now, in_page; - int i, ret = 0; - int nr_pages = 1; - struct page *page; - struct kvm_io_device *pio_dev; - - vcpu->run->exit_reason = KVM_EXIT_IO; - vcpu->run->io.direction = in ? KVM_EXIT_IO_IN : KVM_EXIT_IO_OUT; - vcpu->run->io.size = vcpu->arch.pio.size = size; - vcpu->run->io.data_offset = KVM_PIO_PAGE_OFFSET * PAGE_SIZE; - vcpu->run->io.count = vcpu->arch.pio.count = vcpu->arch.pio.cur_count = count; - vcpu->run->io.port = vcpu->arch.pio.port = port; - vcpu->arch.pio.in = in; - vcpu->arch.pio.string = 1; - vcpu->arch.pio.down = down; - vcpu->arch.pio.guest_page_offset = offset_in_page(address); - vcpu->arch.pio.rep = rep; - - if (!count) { - kvm_x86_ops->skip_emulated_instruction(vcpu); - return 1; - } - - if (!down) - in_page = PAGE_SIZE - offset_in_page(address); - else - in_page = offset_in_page(address) + size; - now = min(count, (unsigned long)in_page / size); - if (!now) { - /* - * String I/O straddles page boundary. Pin two guest pages - * so that we satisfy atomicity constraints. Do just one - * transaction to avoid complexity. - */ - nr_pages = 2; - now = 1; - } - if (down) { - /* - * String I/O in reverse. Yuck. Kill the guest, fix later. - */ - pr_unimpl(vcpu, "guest string pio down\n"); - kvm_inject_gp(vcpu, 0); - return 1; - } - vcpu->run->io.count = now; - vcpu->arch.pio.cur_count = now; - - if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count) - kvm_x86_ops->skip_emulated_instruction(vcpu); - - for (i = 0; i < nr_pages; ++i) { - mutex_lock(&vcpu->kvm->lock); - page = gva_to_page(vcpu, address + i * PAGE_SIZE); - vcpu->arch.pio.guest_pages[i] = page; - mutex_unlock(&vcpu->kvm->lock); - if (!page) { - kvm_inject_gp(vcpu, 0); - free_pio_guest_pages(vcpu); - return 1; - } - } - - pio_dev = vcpu_find_pio_dev(vcpu, port); - if (!vcpu->arch.pio.in) { - /* string PIO write */ - ret = pio_copy_data(vcpu); - if (ret >= 0 && pio_dev) { - pio_string_write(pio_dev, vcpu); - complete_pio(vcpu); - if (vcpu->arch.pio.count == 0) - ret = 1; - } - } else if (pio_dev) - pr_unimpl(vcpu, "no string pio read support yet, " - "port %x size %d count %ld\n", - port, size, count); - - return ret; -} -EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); - -int kvm_arch_init(void *opaque) -{ - int r; - struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; - - r = kvm_mmu_module_init(); - if (r) - goto out_fail; - - kvm_init_msr_list(); - - if (kvm_x86_ops) { - printk(KERN_ERR "kvm: already loaded the other module\n"); - r = -EEXIST; - goto out; - } - - if (!ops->cpu_has_kvm_support()) { - printk(KERN_ERR "kvm: no hardware support\n"); - r = -EOPNOTSUPP; - goto out; - } - if (ops->disabled_by_bios()) { - printk(KERN_ERR "kvm: disabled by bios\n"); - r = -EOPNOTSUPP; - goto out; - } - - kvm_x86_ops = ops; - kvm_mmu_set_nonpresent_ptes(0ull, 0ull); - return 0; - -out: - kvm_mmu_module_exit(); -out_fail: - return r; -} - -void kvm_arch_exit(void) -{ - kvm_x86_ops = NULL; - kvm_mmu_module_exit(); -} - -int kvm_emulate_halt(struct kvm_vcpu *vcpu) -{ - ++vcpu->stat.halt_exits; - if (irqchip_in_kernel(vcpu->kvm)) { - vcpu->arch.mp_state = VCPU_MP_STATE_HALTED; - kvm_vcpu_block(vcpu); - if (vcpu->arch.mp_state != VCPU_MP_STATE_RUNNABLE) - return -EINTR; - return 1; - } else { - vcpu->run->exit_reason = KVM_EXIT_HLT; - return 0; - } -} -EXPORT_SYMBOL_GPL(kvm_emulate_halt); - -int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) -{ - unsigned long nr, a0, a1, a2, a3, ret; - - kvm_x86_ops->cache_regs(vcpu); - - nr = vcpu->arch.regs[VCPU_REGS_RAX]; - a0 = vcpu->arch.regs[VCPU_REGS_RBX]; - a1 = vcpu->arch.regs[VCPU_REGS_RCX]; - a2 = vcpu->arch.regs[VCPU_REGS_RDX]; - a3 = vcpu->arch.regs[VCPU_REGS_RSI]; - - if (!is_long_mode(vcpu)) { - nr &= 0xFFFFFFFF; - a0 &= 0xFFFFFFFF; - a1 &= 0xFFFFFFFF; - a2 &= 0xFFFFFFFF; - a3 &= 0xFFFFFFFF; - } - - switch (nr) { - default: - ret = -KVM_ENOSYS; - break; - } - vcpu->arch.regs[VCPU_REGS_RAX] = ret; - kvm_x86_ops->decache_regs(vcpu); - return 0; -} -EXPORT_SYMBOL_GPL(kvm_emulate_hypercall); - -int kvm_fix_hypercall(struct kvm_vcpu *vcpu) -{ - char instruction[3]; - int ret = 0; - - mutex_lock(&vcpu->kvm->lock); - - /* - * Blow out the MMU to ensure that no other VCPU has an active mapping - * to ensure that the updated hypercall appears atomically across all - * VCPUs. - */ - kvm_mmu_zap_all(vcpu->kvm); - - kvm_x86_ops->cache_regs(vcpu); - kvm_x86_ops->patch_hypercall(vcpu, instruction); - if (emulator_write_emulated(vcpu->arch.rip, instruction, 3, vcpu) - != X86EMUL_CONTINUE) - ret = -EFAULT; - - mutex_unlock(&vcpu->kvm->lock); - - return ret; -} - -static u64 mk_cr_64(u64 curr_cr, u32 new_val) -{ - return (curr_cr & ~((1ULL << 32) - 1)) | new_val; -} - -void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) -{ - struct descriptor_table dt = { limit, base }; - - kvm_x86_ops->set_gdt(vcpu, &dt); -} - -void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base) -{ - struct descriptor_table dt = { limit, base }; - - kvm_x86_ops->set_idt(vcpu, &dt); -} - -void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, - unsigned long *rflags) -{ - lmsw(vcpu, msw); - *rflags = kvm_x86_ops->get_rflags(vcpu); -} - -unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr) -{ - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - switch (cr) { - case 0: - return vcpu->arch.cr0; - case 2: - return vcpu->arch.cr2; - case 3: - return vcpu->arch.cr3; - case 4: - return vcpu->arch.cr4; - case 8: - return get_cr8(vcpu); - default: - vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); - return 0; - } -} - -void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val, - unsigned long *rflags) -{ - switch (cr) { - case 0: - set_cr0(vcpu, mk_cr_64(vcpu->arch.cr0, val)); - *rflags = kvm_x86_ops->get_rflags(vcpu); - break; - case 2: - vcpu->arch.cr2 = val; - break; - case 3: - set_cr3(vcpu, val); - break; - case 4: - set_cr4(vcpu, mk_cr_64(vcpu->arch.cr4, val)); - break; - case 8: - set_cr8(vcpu, val & 0xfUL); - break; - default: - vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr); - } -} - -static int move_to_next_stateful_cpuid_entry(struct kvm_vcpu *vcpu, int i) -{ - struct kvm_cpuid_entry2 *e = &vcpu->arch.cpuid_entries[i]; - int j, nent = vcpu->arch.cpuid_nent; - - e->flags &= ~KVM_CPUID_FLAG_STATE_READ_NEXT; - /* when no next entry is found, the current entry[i] is reselected */ - for (j = i + 1; j == i; j = (j + 1) % nent) { - struct kvm_cpuid_entry2 *ej = &vcpu->arch.cpuid_entries[j]; - if (ej->function == e->function) { - ej->flags |= KVM_CPUID_FLAG_STATE_READ_NEXT; - return j; - } - } - return 0; /* silence gcc, even though control never reaches here */ -} - -/* find an entry with matching function, matching index (if needed), and that - * should be read next (if it's stateful) */ -static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e, - u32 function, u32 index) -{ - if (e->function != function) - return 0; - if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index) - return 0; - if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) && - !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT)) - return 0; - return 1; -} - -void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) -{ - int i; - u32 function, index; - struct kvm_cpuid_entry2 *e, *best; - - kvm_x86_ops->cache_regs(vcpu); - function = vcpu->arch.regs[VCPU_REGS_RAX]; - index = vcpu->arch.regs[VCPU_REGS_RCX]; - vcpu->arch.regs[VCPU_REGS_RAX] = 0; - vcpu->arch.regs[VCPU_REGS_RBX] = 0; - vcpu->arch.regs[VCPU_REGS_RCX] = 0; - vcpu->arch.regs[VCPU_REGS_RDX] = 0; - best = NULL; - for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { - e = &vcpu->arch.cpuid_entries[i]; - if (is_matching_cpuid_entry(e, function, index)) { - if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) - move_to_next_stateful_cpuid_entry(vcpu, i); - best = e; - break; - } - /* - * Both basic or both extended? - */ - if (((e->function ^ function) & 0x80000000) == 0) - if (!best || e->function > best->function) - best = e; - } - if (best) { - vcpu->arch.regs[VCPU_REGS_RAX] = best->eax; - vcpu->arch.regs[VCPU_REGS_RBX] = best->ebx; - vcpu->arch.regs[VCPU_REGS_RCX] = best->ecx; - vcpu->arch.regs[VCPU_REGS_RDX] = best->edx; - } - kvm_x86_ops->decache_regs(vcpu); - kvm_x86_ops->skip_emulated_instruction(vcpu); -} -EXPORT_SYMBOL_GPL(kvm_emulate_cpuid); - -/* - * Check if userspace requested an interrupt window, and that the - * interrupt window is open. - * - * No need to exit to userspace if we already have an interrupt queued. - */ -static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - return (!vcpu->arch.irq_summary && - kvm_run->request_interrupt_window && - vcpu->arch.interrupt_window_open && - (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF)); -} - -static void post_kvm_run_save(struct kvm_vcpu *vcpu, - struct kvm_run *kvm_run) -{ - kvm_run->if_flag = (kvm_x86_ops->get_rflags(vcpu) & X86_EFLAGS_IF) != 0; - kvm_run->cr8 = get_cr8(vcpu); - kvm_run->apic_base = kvm_get_apic_base(vcpu); - if (irqchip_in_kernel(vcpu->kvm)) - kvm_run->ready_for_interrupt_injection = 1; - else - kvm_run->ready_for_interrupt_injection = - (vcpu->arch.interrupt_window_open && - vcpu->arch.irq_summary == 0); -} - -static int __vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - int r; - - if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED)) { - pr_debug("vcpu %d received sipi with vector # %x\n", - vcpu->vcpu_id, vcpu->arch.sipi_vector); - kvm_lapic_reset(vcpu); - r = kvm_x86_ops->vcpu_reset(vcpu); - if (r) - return r; - vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; - } - -preempted: - if (vcpu->guest_debug.enabled) - kvm_x86_ops->guest_debug_pre(vcpu); - -again: - r = kvm_mmu_reload(vcpu); - if (unlikely(r)) - goto out; - - kvm_inject_pending_timer_irqs(vcpu); - - preempt_disable(); - - kvm_x86_ops->prepare_guest_switch(vcpu); - kvm_load_guest_fpu(vcpu); - - local_irq_disable(); - - if (signal_pending(current)) { - local_irq_enable(); - preempt_enable(); - r = -EINTR; - kvm_run->exit_reason = KVM_EXIT_INTR; - ++vcpu->stat.signal_exits; - goto out; - } - - if (vcpu->arch.exception.pending) - __queue_exception(vcpu); - else if (irqchip_in_kernel(vcpu->kvm)) - kvm_x86_ops->inject_pending_irq(vcpu); - else - kvm_x86_ops->inject_pending_vectors(vcpu, kvm_run); - - vcpu->guest_mode = 1; - kvm_guest_enter(); - - if (vcpu->requests) - if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) - kvm_x86_ops->tlb_flush(vcpu); - - kvm_x86_ops->run(vcpu, kvm_run); - - vcpu->guest_mode = 0; - local_irq_enable(); - - ++vcpu->stat.exits; - - /* - * We must have an instruction between local_irq_enable() and - * kvm_guest_exit(), so the timer interrupt isn't delayed by - * the interrupt shadow. The stat.exits increment will do nicely. - * But we need to prevent reordering, hence this barrier(): - */ - barrier(); - - kvm_guest_exit(); - - preempt_enable(); - - /* - * Profile KVM exit RIPs: - */ - if (unlikely(prof_on == KVM_PROFILING)) { - kvm_x86_ops->cache_regs(vcpu); - profile_hit(KVM_PROFILING, (void *)vcpu->arch.rip); - } - - if (vcpu->arch.exception.pending && kvm_x86_ops->exception_injected(vcpu)) - vcpu->arch.exception.pending = false; - - r = kvm_x86_ops->handle_exit(kvm_run, vcpu); - - if (r > 0) { - if (dm_request_for_irq_injection(vcpu, kvm_run)) { - r = -EINTR; - kvm_run->exit_reason = KVM_EXIT_INTR; - ++vcpu->stat.request_irq_exits; - goto out; - } - if (!need_resched()) - goto again; - } - -out: - if (r > 0) { - kvm_resched(vcpu); - goto preempted; - } - - post_kvm_run_save(vcpu, kvm_run); - - return r; -} - -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) -{ - int r; - sigset_t sigsaved; - - vcpu_load(vcpu); - - if (unlikely(vcpu->arch.mp_state == VCPU_MP_STATE_UNINITIALIZED)) { - kvm_vcpu_block(vcpu); - vcpu_put(vcpu); - return -EAGAIN; - } - - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved); - - /* re-sync apic's tpr */ - if (!irqchip_in_kernel(vcpu->kvm)) - set_cr8(vcpu, kvm_run->cr8); - - if (vcpu->arch.pio.cur_count) { - r = complete_pio(vcpu); - if (r) - goto out; - } -#if CONFIG_HAS_IOMEM - if (vcpu->mmio_needed) { - memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8); - vcpu->mmio_read_completed = 1; - vcpu->mmio_needed = 0; - r = emulate_instruction(vcpu, kvm_run, - vcpu->arch.mmio_fault_cr2, 0, 1); - if (r == EMULATE_DO_MMIO) { - /* - * Read-modify-write. Back to userspace. - */ - r = 0; - goto out; - } - } -#endif - if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) { - kvm_x86_ops->cache_regs(vcpu); - vcpu->arch.regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret; - kvm_x86_ops->decache_regs(vcpu); - } - - r = __vcpu_run(vcpu, kvm_run); - -out: - if (vcpu->sigset_active) - sigprocmask(SIG_SETMASK, &sigsaved, NULL); - - vcpu_put(vcpu); - return r; -} - -int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - vcpu_load(vcpu); - - kvm_x86_ops->cache_regs(vcpu); - - regs->rax = vcpu->arch.regs[VCPU_REGS_RAX]; - regs->rbx = vcpu->arch.regs[VCPU_REGS_RBX]; - regs->rcx = vcpu->arch.regs[VCPU_REGS_RCX]; - regs->rdx = vcpu->arch.regs[VCPU_REGS_RDX]; - regs->rsi = vcpu->arch.regs[VCPU_REGS_RSI]; - regs->rdi = vcpu->arch.regs[VCPU_REGS_RDI]; - regs->rsp = vcpu->arch.regs[VCPU_REGS_RSP]; - regs->rbp = vcpu->arch.regs[VCPU_REGS_RBP]; -#ifdef CONFIG_X86_64 - regs->r8 = vcpu->arch.regs[VCPU_REGS_R8]; - regs->r9 = vcpu->arch.regs[VCPU_REGS_R9]; - regs->r10 = vcpu->arch.regs[VCPU_REGS_R10]; - regs->r11 = vcpu->arch.regs[VCPU_REGS_R11]; - regs->r12 = vcpu->arch.regs[VCPU_REGS_R12]; - regs->r13 = vcpu->arch.regs[VCPU_REGS_R13]; - regs->r14 = vcpu->arch.regs[VCPU_REGS_R14]; - regs->r15 = vcpu->arch.regs[VCPU_REGS_R15]; -#endif - - regs->rip = vcpu->arch.rip; - regs->rflags = kvm_x86_ops->get_rflags(vcpu); - - /* - * Don't leak debug flags in case they were set for guest debugging - */ - if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep) - regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); - - vcpu_put(vcpu); - - return 0; -} - -int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - vcpu_load(vcpu); - - vcpu->arch.regs[VCPU_REGS_RAX] = regs->rax; - vcpu->arch.regs[VCPU_REGS_RBX] = regs->rbx; - vcpu->arch.regs[VCPU_REGS_RCX] = regs->rcx; - vcpu->arch.regs[VCPU_REGS_RDX] = regs->rdx; - vcpu->arch.regs[VCPU_REGS_RSI] = regs->rsi; - vcpu->arch.regs[VCPU_REGS_RDI] = regs->rdi; - vcpu->arch.regs[VCPU_REGS_RSP] = regs->rsp; - vcpu->arch.regs[VCPU_REGS_RBP] = regs->rbp; -#ifdef CONFIG_X86_64 - vcpu->arch.regs[VCPU_REGS_R8] = regs->r8; - vcpu->arch.regs[VCPU_REGS_R9] = regs->r9; - vcpu->arch.regs[VCPU_REGS_R10] = regs->r10; - vcpu->arch.regs[VCPU_REGS_R11] = regs->r11; - vcpu->arch.regs[VCPU_REGS_R12] = regs->r12; - vcpu->arch.regs[VCPU_REGS_R13] = regs->r13; - vcpu->arch.regs[VCPU_REGS_R14] = regs->r14; - vcpu->arch.regs[VCPU_REGS_R15] = regs->r15; -#endif - - vcpu->arch.rip = regs->rip; - kvm_x86_ops->set_rflags(vcpu, regs->rflags); - - kvm_x86_ops->decache_regs(vcpu); - - vcpu_put(vcpu); - - return 0; -} - -static void get_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - return kvm_x86_ops->get_segment(vcpu, var, seg); -} - -void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) -{ - struct kvm_segment cs; - - get_segment(vcpu, &cs, VCPU_SREG_CS); - *db = cs.db; - *l = cs.l; -} -EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); - -int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - struct descriptor_table dt; - int pending_vec; - - vcpu_load(vcpu); - - get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - get_segment(vcpu, &sregs->es, VCPU_SREG_ES); - get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_x86_ops->get_idt(vcpu, &dt); - sregs->idt.limit = dt.limit; - sregs->idt.base = dt.base; - kvm_x86_ops->get_gdt(vcpu, &dt); - sregs->gdt.limit = dt.limit; - sregs->gdt.base = dt.base; - - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - sregs->cr0 = vcpu->arch.cr0; - sregs->cr2 = vcpu->arch.cr2; - sregs->cr3 = vcpu->arch.cr3; - sregs->cr4 = vcpu->arch.cr4; - sregs->cr8 = get_cr8(vcpu); - sregs->efer = vcpu->arch.shadow_efer; - sregs->apic_base = kvm_get_apic_base(vcpu); - - if (irqchip_in_kernel(vcpu->kvm)) { - memset(sregs->interrupt_bitmap, 0, - sizeof sregs->interrupt_bitmap); - pending_vec = kvm_x86_ops->get_irq(vcpu); - if (pending_vec >= 0) - set_bit(pending_vec, - (unsigned long *)sregs->interrupt_bitmap); - } else - memcpy(sregs->interrupt_bitmap, vcpu->arch.irq_pending, - sizeof sregs->interrupt_bitmap); - - vcpu_put(vcpu); - - return 0; -} - -static void set_segment(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg) -{ - return kvm_x86_ops->set_segment(vcpu, var, seg); -} - -int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - int mmu_reset_needed = 0; - int i, pending_vec, max_bits; - struct descriptor_table dt; - - vcpu_load(vcpu); - - dt.limit = sregs->idt.limit; - dt.base = sregs->idt.base; - kvm_x86_ops->set_idt(vcpu, &dt); - dt.limit = sregs->gdt.limit; - dt.base = sregs->gdt.base; - kvm_x86_ops->set_gdt(vcpu, &dt); - - vcpu->arch.cr2 = sregs->cr2; - mmu_reset_needed |= vcpu->arch.cr3 != sregs->cr3; - vcpu->arch.cr3 = sregs->cr3; - - set_cr8(vcpu, sregs->cr8); - - mmu_reset_needed |= vcpu->arch.shadow_efer != sregs->efer; -#ifdef CONFIG_X86_64 - kvm_x86_ops->set_efer(vcpu, sregs->efer); -#endif - kvm_set_apic_base(vcpu, sregs->apic_base); - - kvm_x86_ops->decache_cr4_guest_bits(vcpu); - - mmu_reset_needed |= vcpu->arch.cr0 != sregs->cr0; - vcpu->arch.cr0 = sregs->cr0; - kvm_x86_ops->set_cr0(vcpu, sregs->cr0); - - mmu_reset_needed |= vcpu->arch.cr4 != sregs->cr4; - kvm_x86_ops->set_cr4(vcpu, sregs->cr4); - if (!is_long_mode(vcpu) && is_pae(vcpu)) - load_pdptrs(vcpu, vcpu->arch.cr3); - - if (mmu_reset_needed) - kvm_mmu_reset_context(vcpu); - - if (!irqchip_in_kernel(vcpu->kvm)) { - memcpy(vcpu->arch.irq_pending, sregs->interrupt_bitmap, - sizeof vcpu->arch.irq_pending); - vcpu->arch.irq_summary = 0; - for (i = 0; i < ARRAY_SIZE(vcpu->arch.irq_pending); ++i) - if (vcpu->arch.irq_pending[i]) - __set_bit(i, &vcpu->arch.irq_summary); - } else { - max_bits = (sizeof sregs->interrupt_bitmap) << 3; - pending_vec = find_first_bit( - (const unsigned long *)sregs->interrupt_bitmap, - max_bits); - /* Only pending external irq is handled here */ - if (pending_vec < max_bits) { - kvm_x86_ops->set_irq(vcpu, pending_vec); - pr_debug("Set back pending irq %d\n", - pending_vec); - } - } - - set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - set_segment(vcpu, &sregs->es, VCPU_SREG_ES); - set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - vcpu_put(vcpu); - - return 0; -} - -int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, - struct kvm_debug_guest *dbg) -{ - int r; - - vcpu_load(vcpu); - - r = kvm_x86_ops->set_guest_debug(vcpu, dbg); - - vcpu_put(vcpu); - - return r; -} - -/* - * fxsave fpu state. Taken from x86_64/processor.h. To be killed when - * we have asm/x86/processor.h - */ -struct fxsave { - u16 cwd; - u16 swd; - u16 twd; - u16 fop; - u64 rip; - u64 rdp; - u32 mxcsr; - u32 mxcsr_mask; - u32 st_space[32]; /* 8*16 bytes for each FP-reg = 128 bytes */ -#ifdef CONFIG_X86_64 - u32 xmm_space[64]; /* 16*16 bytes for each XMM-reg = 256 bytes */ -#else - u32 xmm_space[32]; /* 8*16 bytes for each XMM-reg = 128 bytes */ -#endif -}; - -/* - * Translate a guest virtual address to a guest physical address. - */ -int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, - struct kvm_translation *tr) -{ - unsigned long vaddr = tr->linear_address; - gpa_t gpa; - - vcpu_load(vcpu); - mutex_lock(&vcpu->kvm->lock); - gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, vaddr); - tr->physical_address = gpa; - tr->valid = gpa != UNMAPPED_GVA; - tr->writeable = 1; - tr->usermode = 0; - mutex_unlock(&vcpu->kvm->lock); - vcpu_put(vcpu); - - return 0; -} - -int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) -{ - struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image; - - vcpu_load(vcpu); - - memcpy(fpu->fpr, fxsave->st_space, 128); - fpu->fcw = fxsave->cwd; - fpu->fsw = fxsave->swd; - fpu->ftwx = fxsave->twd; - fpu->last_opcode = fxsave->fop; - fpu->last_ip = fxsave->rip; - fpu->last_dp = fxsave->rdp; - memcpy(fpu->xmm, fxsave->xmm_space, sizeof fxsave->xmm_space); - - vcpu_put(vcpu); - - return 0; -} - -int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) -{ - struct fxsave *fxsave = (struct fxsave *)&vcpu->arch.guest_fx_image; - - vcpu_load(vcpu); - - memcpy(fxsave->st_space, fpu->fpr, 128); - fxsave->cwd = fpu->fcw; - fxsave->swd = fpu->fsw; - fxsave->twd = fpu->ftwx; - fxsave->fop = fpu->last_opcode; - fxsave->rip = fpu->last_ip; - fxsave->rdp = fpu->last_dp; - memcpy(fxsave->xmm_space, fpu->xmm, sizeof fxsave->xmm_space); - - vcpu_put(vcpu); - - return 0; -} - -void fx_init(struct kvm_vcpu *vcpu) -{ - unsigned after_mxcsr_mask; - - /* Initialize guest FPU by resetting ours and saving into guest's */ - preempt_disable(); - fx_save(&vcpu->arch.host_fx_image); - fpu_init(); - fx_save(&vcpu->arch.guest_fx_image); - fx_restore(&vcpu->arch.host_fx_image); - preempt_enable(); - - vcpu->arch.cr0 |= X86_CR0_ET; - after_mxcsr_mask = offsetof(struct i387_fxsave_struct, st_space); - vcpu->arch.guest_fx_image.mxcsr = 0x1f80; - memset((void *)&vcpu->arch.guest_fx_image + after_mxcsr_mask, - 0, sizeof(struct i387_fxsave_struct) - after_mxcsr_mask); -} -EXPORT_SYMBOL_GPL(fx_init); - -void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) -{ - if (!vcpu->fpu_active || vcpu->guest_fpu_loaded) - return; - - vcpu->guest_fpu_loaded = 1; - fx_save(&vcpu->arch.host_fx_image); - fx_restore(&vcpu->arch.guest_fx_image); -} -EXPORT_SYMBOL_GPL(kvm_load_guest_fpu); - -void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) -{ - if (!vcpu->guest_fpu_loaded) - return; - - vcpu->guest_fpu_loaded = 0; - fx_save(&vcpu->arch.guest_fx_image); - fx_restore(&vcpu->arch.host_fx_image); - ++vcpu->stat.fpu_reload; -} -EXPORT_SYMBOL_GPL(kvm_put_guest_fpu); - -void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) -{ - kvm_x86_ops->vcpu_free(vcpu); -} - -struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, - unsigned int id) -{ - return kvm_x86_ops->vcpu_create(kvm, id); -} - -int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) -{ - int r; - - /* We do fxsave: this must be aligned. */ - BUG_ON((unsigned long)&vcpu->arch.host_fx_image & 0xF); - - vcpu_load(vcpu); - r = kvm_arch_vcpu_reset(vcpu); - if (r == 0) - r = kvm_mmu_setup(vcpu); - vcpu_put(vcpu); - if (r < 0) - goto free_vcpu; - - return 0; -free_vcpu: - kvm_x86_ops->vcpu_free(vcpu); - return r; -} - -void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) -{ - vcpu_load(vcpu); - kvm_mmu_unload(vcpu); - vcpu_put(vcpu); - - kvm_x86_ops->vcpu_free(vcpu); -} - -int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) -{ - return kvm_x86_ops->vcpu_reset(vcpu); -} - -void kvm_arch_hardware_enable(void *garbage) -{ - kvm_x86_ops->hardware_enable(garbage); -} - -void kvm_arch_hardware_disable(void *garbage) -{ - kvm_x86_ops->hardware_disable(garbage); -} - -int kvm_arch_hardware_setup(void) -{ - return kvm_x86_ops->hardware_setup(); -} - -void kvm_arch_hardware_unsetup(void) -{ - kvm_x86_ops->hardware_unsetup(); -} - -void kvm_arch_check_processor_compat(void *rtn) -{ - kvm_x86_ops->check_processor_compatibility(rtn); -} - -int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) -{ - struct page *page; - struct kvm *kvm; - int r; - - BUG_ON(vcpu->kvm == NULL); - kvm = vcpu->kvm; - - vcpu->arch.mmu.root_hpa = INVALID_PAGE; - if (!irqchip_in_kernel(kvm) || vcpu->vcpu_id == 0) - vcpu->arch.mp_state = VCPU_MP_STATE_RUNNABLE; - else - vcpu->arch.mp_state = VCPU_MP_STATE_UNINITIALIZED; - - page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) { - r = -ENOMEM; - goto fail; - } - vcpu->arch.pio_data = page_address(page); - - r = kvm_mmu_create(vcpu); - if (r < 0) - goto fail_free_pio_data; - - if (irqchip_in_kernel(kvm)) { - r = kvm_create_lapic(vcpu); - if (r < 0) - goto fail_mmu_destroy; - } - - return 0; - -fail_mmu_destroy: - kvm_mmu_destroy(vcpu); -fail_free_pio_data: - free_page((unsigned long)vcpu->arch.pio_data); -fail: - return r; -} - -void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) -{ - kvm_free_lapic(vcpu); - kvm_mmu_destroy(vcpu); - free_page((unsigned long)vcpu->arch.pio_data); -} - -struct kvm *kvm_arch_create_vm(void) -{ - struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL); - - if (!kvm) - return ERR_PTR(-ENOMEM); - - INIT_LIST_HEAD(&kvm->arch.active_mmu_pages); - - return kvm; -} - -static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) -{ - vcpu_load(vcpu); - kvm_mmu_unload(vcpu); - vcpu_put(vcpu); -} - -static void kvm_free_vcpus(struct kvm *kvm) -{ - unsigned int i; - - /* - * Unpin any mmu pages first. - */ - for (i = 0; i < KVM_MAX_VCPUS; ++i) - if (kvm->vcpus[i]) - kvm_unload_vcpu_mmu(kvm->vcpus[i]); - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - if (kvm->vcpus[i]) { - kvm_arch_vcpu_free(kvm->vcpus[i]); - kvm->vcpus[i] = NULL; - } - } - -} - -void kvm_arch_destroy_vm(struct kvm *kvm) -{ - kfree(kvm->arch.vpic); - kfree(kvm->arch.vioapic); - kvm_free_vcpus(kvm); - kvm_free_physmem(kvm); - kfree(kvm); -} - -int kvm_arch_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - struct kvm_memory_slot old, - int user_alloc) -{ - int npages = mem->memory_size >> PAGE_SHIFT; - struct kvm_memory_slot *memslot = &kvm->memslots[mem->slot]; - - /*To keep backward compatibility with older userspace, - *x86 needs to hanlde !user_alloc case. - */ - if (!user_alloc) { - if (npages && !old.rmap) { - down_write(¤t->mm->mmap_sem); - memslot->userspace_addr = do_mmap(NULL, 0, - npages * PAGE_SIZE, - PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_ANONYMOUS, - 0); - up_write(¤t->mm->mmap_sem); - - if (IS_ERR((void *)memslot->userspace_addr)) - return PTR_ERR((void *)memslot->userspace_addr); - } else { - if (!old.user_alloc && old.rmap) { - int ret; - - down_write(¤t->mm->mmap_sem); - ret = do_munmap(current->mm, old.userspace_addr, - old.npages * PAGE_SIZE); - up_write(¤t->mm->mmap_sem); - if (ret < 0) - printk(KERN_WARNING - "kvm_vm_ioctl_set_memory_region: " - "failed to munmap memory\n"); - } - } - } - - if (!kvm->arch.n_requested_mmu_pages) { - unsigned int nr_mmu_pages = kvm_mmu_calculate_mmu_pages(kvm); - kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); - } - - kvm_mmu_slot_remove_write_access(kvm, mem->slot); - kvm_flush_remote_tlbs(kvm); - - return 0; -} - -int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.mp_state == VCPU_MP_STATE_RUNNABLE - || vcpu->arch.mp_state == VCPU_MP_STATE_SIPI_RECEIVED; -} diff --git a/drivers/kvm/x86.h b/drivers/kvm/x86.h deleted file mode 100644 index dfb8091971a9..000000000000 --- a/drivers/kvm/x86.h +++ /dev/null @@ -1,602 +0,0 @@ -#/* - * Kernel-based Virtual Machine driver for Linux - * - * This header defines architecture specific interfaces, x86 version - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - * - */ - -#ifndef KVM_X86_H -#define KVM_X86_H - -#include -#include - -#include -#include - -#include - -#include "types.h" - -#define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) -#define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) -#define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) - -#define KVM_GUEST_CR0_MASK \ - (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE \ - | X86_CR0_NW | X86_CR0_CD) -#define KVM_VM_CR0_ALWAYS_ON \ - (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE | X86_CR0_TS \ - | X86_CR0_MP) -#define KVM_GUEST_CR4_MASK \ - (X86_CR4_VME | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_VMXE) -#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) -#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) - -#define INVALID_PAGE (~(hpa_t)0) -#define UNMAPPED_GVA (~(gpa_t)0) - -#define DE_VECTOR 0 -#define UD_VECTOR 6 -#define NM_VECTOR 7 -#define DF_VECTOR 8 -#define TS_VECTOR 10 -#define NP_VECTOR 11 -#define SS_VECTOR 12 -#define GP_VECTOR 13 -#define PF_VECTOR 14 - -#define SELECTOR_TI_MASK (1 << 2) -#define SELECTOR_RPL_MASK 0x03 - -#define IOPL_SHIFT 12 - -#define KVM_ALIAS_SLOTS 4 - -#define KVM_PERMILLE_MMU_PAGES 20 -#define KVM_MIN_ALLOC_MMU_PAGES 64 -#define KVM_NUM_MMU_PAGES 1024 -#define KVM_MIN_FREE_MMU_PAGES 5 -#define KVM_REFILL_PAGES 25 -#define KVM_MAX_CPUID_ENTRIES 40 - -extern spinlock_t kvm_lock; -extern struct list_head vm_list; - -struct kvm_vcpu; -struct kvm; - -enum { - VCPU_REGS_RAX = 0, - VCPU_REGS_RCX = 1, - VCPU_REGS_RDX = 2, - VCPU_REGS_RBX = 3, - VCPU_REGS_RSP = 4, - VCPU_REGS_RBP = 5, - VCPU_REGS_RSI = 6, - VCPU_REGS_RDI = 7, -#ifdef CONFIG_X86_64 - VCPU_REGS_R8 = 8, - VCPU_REGS_R9 = 9, - VCPU_REGS_R10 = 10, - VCPU_REGS_R11 = 11, - VCPU_REGS_R12 = 12, - VCPU_REGS_R13 = 13, - VCPU_REGS_R14 = 14, - VCPU_REGS_R15 = 15, -#endif - NR_VCPU_REGS -}; - -enum { - VCPU_SREG_CS, - VCPU_SREG_DS, - VCPU_SREG_ES, - VCPU_SREG_FS, - VCPU_SREG_GS, - VCPU_SREG_SS, - VCPU_SREG_TR, - VCPU_SREG_LDTR, -}; - -#include "x86_emulate.h" - -#define KVM_NR_MEM_OBJS 40 - -/* - * We don't want allocation failures within the mmu code, so we preallocate - * enough memory for a single page fault in a cache. - */ -struct kvm_mmu_memory_cache { - int nobjs; - void *objects[KVM_NR_MEM_OBJS]; -}; - -#define NR_PTE_CHAIN_ENTRIES 5 - -struct kvm_pte_chain { - u64 *parent_ptes[NR_PTE_CHAIN_ENTRIES]; - struct hlist_node link; -}; - -/* - * kvm_mmu_page_role, below, is defined as: - * - * bits 0:3 - total guest paging levels (2-4, or zero for real mode) - * bits 4:7 - page table level for this shadow (1-4) - * bits 8:9 - page table quadrant for 2-level guests - * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) - * bits 17:19 - common access permissions for all ptes in this shadow page - */ -union kvm_mmu_page_role { - unsigned word; - struct { - unsigned glevels : 4; - unsigned level : 4; - unsigned quadrant : 2; - unsigned pad_for_nice_hex_output : 6; - unsigned metaphysical : 1; - unsigned access : 3; - }; -}; - -struct kvm_mmu_page { - struct list_head link; - struct hlist_node hash_link; - - /* - * The following two entries are used to key the shadow page in the - * hash table. - */ - gfn_t gfn; - union kvm_mmu_page_role role; - - u64 *spt; - /* hold the gfn of each spte inside spt */ - gfn_t *gfns; - unsigned long slot_bitmap; /* One bit set per slot which has memory - * in this shadow page. - */ - int multimapped; /* More than one parent_pte? */ - int root_count; /* Currently serving as active root */ - union { - u64 *parent_pte; /* !multimapped */ - struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */ - }; -}; - -/* - * x86 supports 3 paging modes (4-level 64-bit, 3-level 64-bit, and 2-level - * 32-bit). The kvm_mmu structure abstracts the details of the current mmu - * mode. - */ -struct kvm_mmu { - void (*new_cr3)(struct kvm_vcpu *vcpu); - int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); - void (*free)(struct kvm_vcpu *vcpu); - gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); - void (*prefetch_page)(struct kvm_vcpu *vcpu, - struct kvm_mmu_page *page); - hpa_t root_hpa; - int root_level; - int shadow_root_level; - - u64 *pae_root; -}; - -struct kvm_vcpu_arch { - u64 host_tsc; - int interrupt_window_open; - unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */ - DECLARE_BITMAP(irq_pending, KVM_NR_INTERRUPTS); - unsigned long regs[NR_VCPU_REGS]; /* for rsp: vcpu_load_rsp_rip() */ - unsigned long rip; /* needs vcpu_load_rsp_rip() */ - - unsigned long cr0; - unsigned long cr2; - unsigned long cr3; - unsigned long cr4; - unsigned long cr8; - u64 pdptrs[4]; /* pae */ - u64 shadow_efer; - u64 apic_base; - struct kvm_lapic *apic; /* kernel irqchip context */ -#define VCPU_MP_STATE_RUNNABLE 0 -#define VCPU_MP_STATE_UNINITIALIZED 1 -#define VCPU_MP_STATE_INIT_RECEIVED 2 -#define VCPU_MP_STATE_SIPI_RECEIVED 3 -#define VCPU_MP_STATE_HALTED 4 - int mp_state; - int sipi_vector; - u64 ia32_misc_enable_msr; - - struct kvm_mmu mmu; - - struct kvm_mmu_memory_cache mmu_pte_chain_cache; - struct kvm_mmu_memory_cache mmu_rmap_desc_cache; - struct kvm_mmu_memory_cache mmu_page_cache; - struct kvm_mmu_memory_cache mmu_page_header_cache; - - gfn_t last_pt_write_gfn; - int last_pt_write_count; - u64 *last_pte_updated; - - struct i387_fxsave_struct host_fx_image; - struct i387_fxsave_struct guest_fx_image; - - gva_t mmio_fault_cr2; - struct kvm_pio_request pio; - void *pio_data; - - struct kvm_queued_exception { - bool pending; - bool has_error_code; - u8 nr; - u32 error_code; - } exception; - - struct { - int active; - u8 save_iopl; - struct kvm_save_segment { - u16 selector; - unsigned long base; - u32 limit; - u32 ar; - } tr, es, ds, fs, gs; - } rmode; - int halt_request; /* real mode on Intel only */ - - int cpuid_nent; - struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; - /* emulate context */ - - struct x86_emulate_ctxt emulate_ctxt; -}; - -struct kvm_mem_alias { - gfn_t base_gfn; - unsigned long npages; - gfn_t target_gfn; -}; - -struct kvm_arch{ - int naliases; - struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; - - unsigned int n_free_mmu_pages; - unsigned int n_requested_mmu_pages; - unsigned int n_alloc_mmu_pages; - struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; - /* - * Hash table of struct kvm_mmu_page. - */ - struct list_head active_mmu_pages; - struct kvm_pic *vpic; - struct kvm_ioapic *vioapic; - - int round_robin_prev_vcpu; - unsigned int tss_addr; - struct page *apic_access_page; -}; - -struct kvm_vm_stat { - u32 mmu_shadow_zapped; - u32 mmu_pte_write; - u32 mmu_pte_updated; - u32 mmu_pde_zapped; - u32 mmu_flooded; - u32 mmu_recycled; - u32 remote_tlb_flush; -}; - -struct kvm_vcpu_stat { - u32 pf_fixed; - u32 pf_guest; - u32 tlb_flush; - u32 invlpg; - - u32 exits; - u32 io_exits; - u32 mmio_exits; - u32 signal_exits; - u32 irq_window_exits; - u32 halt_exits; - u32 halt_wakeup; - u32 request_irq_exits; - u32 irq_exits; - u32 host_state_reload; - u32 efer_reload; - u32 fpu_reload; - u32 insn_emulation; - u32 insn_emulation_fail; -}; - -struct descriptor_table { - u16 limit; - unsigned long base; -} __attribute__((packed)); - -struct kvm_x86_ops { - int (*cpu_has_kvm_support)(void); /* __init */ - int (*disabled_by_bios)(void); /* __init */ - void (*hardware_enable)(void *dummy); /* __init */ - void (*hardware_disable)(void *dummy); - void (*check_processor_compatibility)(void *rtn); - int (*hardware_setup)(void); /* __init */ - void (*hardware_unsetup)(void); /* __exit */ - - /* Create, but do not attach this VCPU */ - struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); - void (*vcpu_free)(struct kvm_vcpu *vcpu); - int (*vcpu_reset)(struct kvm_vcpu *vcpu); - - void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); - void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); - void (*vcpu_put)(struct kvm_vcpu *vcpu); - void (*vcpu_decache)(struct kvm_vcpu *vcpu); - - int (*set_guest_debug)(struct kvm_vcpu *vcpu, - struct kvm_debug_guest *dbg); - void (*guest_debug_pre)(struct kvm_vcpu *vcpu); - int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); - int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); - u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); - void (*get_segment)(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg); - void (*set_segment)(struct kvm_vcpu *vcpu, - struct kvm_segment *var, int seg); - void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); - void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); - void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); - void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); - void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); - void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); - void (*get_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*set_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*get_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - void (*set_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); - unsigned long (*get_dr)(struct kvm_vcpu *vcpu, int dr); - void (*set_dr)(struct kvm_vcpu *vcpu, int dr, unsigned long value, - int *exception); - void (*cache_regs)(struct kvm_vcpu *vcpu); - void (*decache_regs)(struct kvm_vcpu *vcpu); - unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); - void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); - - void (*tlb_flush)(struct kvm_vcpu *vcpu); - - void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run); - int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); - void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); - void (*patch_hypercall)(struct kvm_vcpu *vcpu, - unsigned char *hypercall_addr); - int (*get_irq)(struct kvm_vcpu *vcpu); - void (*set_irq)(struct kvm_vcpu *vcpu, int vec); - void (*queue_exception)(struct kvm_vcpu *vcpu, unsigned nr, - bool has_error_code, u32 error_code); - bool (*exception_injected)(struct kvm_vcpu *vcpu); - void (*inject_pending_irq)(struct kvm_vcpu *vcpu); - void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, - struct kvm_run *run); - - int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); -}; - -extern struct kvm_x86_ops *kvm_x86_ops; - -int kvm_mmu_module_init(void); -void kvm_mmu_module_exit(void); - -void kvm_mmu_destroy(struct kvm_vcpu *vcpu); -int kvm_mmu_create(struct kvm_vcpu *vcpu); -int kvm_mmu_setup(struct kvm_vcpu *vcpu); -void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); - -int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); -void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); -void kvm_mmu_zap_all(struct kvm *kvm); -unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); -void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); - -enum emulation_result { - EMULATE_DONE, /* no further processing */ - EMULATE_DO_MMIO, /* kvm_run filled with mmio request */ - EMULATE_FAIL, /* can't emulate this instruction */ -}; - -int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, - unsigned long cr2, u16 error_code, int no_decode); -void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); -void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); -void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); -void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, - unsigned long *rflags); - -unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr); -void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value, - unsigned long *rflags); -int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data); -int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); - -struct x86_emulate_ctxt; - -int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned port); -int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, - int size, unsigned long count, int down, - gva_t address, int rep, unsigned port); -void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); -int kvm_emulate_halt(struct kvm_vcpu *vcpu); -int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); -int emulate_clts(struct kvm_vcpu *vcpu); -int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, - unsigned long *dest); -int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, - unsigned long value); - -void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr0); -void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr0); -unsigned long get_cr8(struct kvm_vcpu *vcpu); -void lmsw(struct kvm_vcpu *vcpu, unsigned long msw); -void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); - -int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); -int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); - -void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); -void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); -void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2, - u32 error_code); - -void fx_init(struct kvm_vcpu *vcpu); - -int emulator_read_std(unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); -int emulator_write_emulated(unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); - -unsigned long segment_base(u16 selector); - -void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu); -void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, - const u8 *new, int bytes); -int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); -void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); -int kvm_mmu_load(struct kvm_vcpu *vcpu); -void kvm_mmu_unload(struct kvm_vcpu *vcpu); - -int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); - -int kvm_fix_hypercall(struct kvm_vcpu *vcpu); - -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); - -int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); -int complete_pio(struct kvm_vcpu *vcpu); - -static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) -{ - struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); - - return (struct kvm_mmu_page *)page_private(page); -} - -static inline u16 read_fs(void) -{ - u16 seg; - asm("mov %%fs, %0" : "=g"(seg)); - return seg; -} - -static inline u16 read_gs(void) -{ - u16 seg; - asm("mov %%gs, %0" : "=g"(seg)); - return seg; -} - -static inline u16 read_ldt(void) -{ - u16 ldt; - asm("sldt %0" : "=g"(ldt)); - return ldt; -} - -static inline void load_fs(u16 sel) -{ - asm("mov %0, %%fs" : : "rm"(sel)); -} - -static inline void load_gs(u16 sel) -{ - asm("mov %0, %%gs" : : "rm"(sel)); -} - -#ifndef load_ldt -static inline void load_ldt(u16 sel) -{ - asm("lldt %0" : : "rm"(sel)); -} -#endif - -static inline void get_idt(struct descriptor_table *table) -{ - asm("sidt %0" : "=m"(*table)); -} - -static inline void get_gdt(struct descriptor_table *table) -{ - asm("sgdt %0" : "=m"(*table)); -} - -static inline unsigned long read_tr_base(void) -{ - u16 tr; - asm("str %0" : "=g"(tr)); - return segment_base(tr); -} - -#ifdef CONFIG_X86_64 -static inline unsigned long read_msr(unsigned long msr) -{ - u64 value; - - rdmsrl(msr, value); - return value; -} -#endif - -static inline void fx_save(struct i387_fxsave_struct *image) -{ - asm("fxsave (%0)":: "r" (image)); -} - -static inline void fx_restore(struct i387_fxsave_struct *image) -{ - asm("fxrstor (%0)":: "r" (image)); -} - -static inline void fpu_init(void) -{ - asm("finit"); -} - -static inline u32 get_rdx_init_val(void) -{ - return 0x600; /* P6 family */ -} - -static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) -{ - kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); -} - -#define ASM_VMX_VMCLEAR_RAX ".byte 0x66, 0x0f, 0xc7, 0x30" -#define ASM_VMX_VMLAUNCH ".byte 0x0f, 0x01, 0xc2" -#define ASM_VMX_VMRESUME ".byte 0x0f, 0x01, 0xc3" -#define ASM_VMX_VMPTRLD_RAX ".byte 0x0f, 0xc7, 0x30" -#define ASM_VMX_VMREAD_RDX_RAX ".byte 0x0f, 0x78, 0xd0" -#define ASM_VMX_VMWRITE_RAX_RDX ".byte 0x0f, 0x79, 0xd0" -#define ASM_VMX_VMWRITE_RSP_RDX ".byte 0x0f, 0x79, 0xd4" -#define ASM_VMX_VMXOFF ".byte 0x0f, 0x01, 0xc4" -#define ASM_VMX_VMXON_RAX ".byte 0xf3, 0x0f, 0xc7, 0x30" - -#define MSR_IA32_TIME_STAMP_COUNTER 0x010 - -#define TSS_IOPB_BASE_OFFSET 0x66 -#define TSS_BASE_SIZE 0x68 -#define TSS_IOPB_SIZE (65536 / 8) -#define TSS_REDIRECTION_SIZE (256 / 8) -#define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) - -#endif diff --git a/drivers/kvm/x86_emulate.c b/drivers/kvm/x86_emulate.c deleted file mode 100644 index 50b133f68743..000000000000 --- a/drivers/kvm/x86_emulate.c +++ /dev/null @@ -1,1913 +0,0 @@ -/****************************************************************************** - * x86_emulate.c - * - * Generic x86 (32-bit and 64-bit) instruction decoder and emulator. - * - * Copyright (c) 2005 Keir Fraser - * - * Linux coding style, mod r/m decoder, segment base fixes, real-mode - * privileged instructions: - * - * Copyright (C) 2006 Qumranet - * - * Avi Kivity - * Yaniv Kamay - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - * - * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4 - */ - -#ifndef __KERNEL__ -#include -#include -#include -#define DPRINTF(_f, _a ...) printf(_f , ## _a) -#else -#include "kvm.h" -#include "x86.h" -#define DPRINTF(x...) do {} while (0) -#endif -#include "x86_emulate.h" -#include - -/* - * Opcode effective-address decode tables. - * Note that we only emulate instructions that have at least one memory - * operand (excluding implicit stack references). We assume that stack - * references and instruction fetches will never occur in special memory - * areas that require emulation. So, for example, 'mov ,' need - * not be handled. - */ - -/* Operand sizes: 8-bit operands or specified/overridden size. */ -#define ByteOp (1<<0) /* 8-bit operands. */ -/* Destination operand type. */ -#define ImplicitOps (1<<1) /* Implicit in opcode. No generic decode. */ -#define DstReg (2<<1) /* Register operand. */ -#define DstMem (3<<1) /* Memory operand. */ -#define DstMask (3<<1) -/* Source operand type. */ -#define SrcNone (0<<3) /* No source operand. */ -#define SrcImplicit (0<<3) /* Source operand is implicit in the opcode. */ -#define SrcReg (1<<3) /* Register operand. */ -#define SrcMem (2<<3) /* Memory operand. */ -#define SrcMem16 (3<<3) /* Memory operand (16-bit). */ -#define SrcMem32 (4<<3) /* Memory operand (32-bit). */ -#define SrcImm (5<<3) /* Immediate operand. */ -#define SrcImmByte (6<<3) /* 8-bit sign-extended immediate operand. */ -#define SrcMask (7<<3) -/* Generic ModRM decode. */ -#define ModRM (1<<6) -/* Destination is only written; never read. */ -#define Mov (1<<7) -#define BitOp (1<<8) -#define MemAbs (1<<9) /* Memory operand is absolute displacement */ -#define String (1<<10) /* String instruction (rep capable) */ -#define Stack (1<<11) /* Stack instruction (push/pop) */ - -static u16 opcode_table[256] = { - /* 0x00 - 0x07 */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, - 0, 0, 0, 0, - /* 0x08 - 0x0F */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, - 0, 0, 0, 0, - /* 0x10 - 0x17 */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, - 0, 0, 0, 0, - /* 0x18 - 0x1F */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, - 0, 0, 0, 0, - /* 0x20 - 0x27 */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, - SrcImmByte, SrcImm, 0, 0, - /* 0x28 - 0x2F */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, - 0, 0, 0, 0, - /* 0x30 - 0x37 */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, - 0, 0, 0, 0, - /* 0x38 - 0x3F */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstReg | SrcMem | ModRM, DstReg | SrcMem | ModRM, - 0, 0, 0, 0, - /* 0x40 - 0x47 */ - DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, - /* 0x48 - 0x4F */ - DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, DstReg, - /* 0x50 - 0x57 */ - SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, - SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, SrcReg | Stack, - /* 0x58 - 0x5F */ - DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, - DstReg | Stack, DstReg | Stack, DstReg | Stack, DstReg | Stack, - /* 0x60 - 0x67 */ - 0, 0, 0, DstReg | SrcMem32 | ModRM | Mov /* movsxd (x86/64) */ , - 0, 0, 0, 0, - /* 0x68 - 0x6F */ - 0, 0, ImplicitOps | Mov | Stack, 0, - SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* insb, insw/insd */ - SrcNone | ByteOp | ImplicitOps, SrcNone | ImplicitOps, /* outsb, outsw/outsd */ - /* 0x70 - 0x77 */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - /* 0x78 - 0x7F */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - /* 0x80 - 0x87 */ - ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImm | ModRM, - ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM, - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, - /* 0x88 - 0x8F */ - ByteOp | DstMem | SrcReg | ModRM | Mov, DstMem | SrcReg | ModRM | Mov, - ByteOp | DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - 0, ModRM | DstReg, 0, DstMem | SrcNone | ModRM | Mov | Stack, - /* 0x90 - 0x9F */ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, ImplicitOps | Stack, ImplicitOps | Stack, 0, 0, - /* 0xA0 - 0xA7 */ - ByteOp | DstReg | SrcMem | Mov | MemAbs, DstReg | SrcMem | Mov | MemAbs, - ByteOp | DstMem | SrcReg | Mov | MemAbs, DstMem | SrcReg | Mov | MemAbs, - ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, - ByteOp | ImplicitOps | String, ImplicitOps | String, - /* 0xA8 - 0xAF */ - 0, 0, ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, - ByteOp | ImplicitOps | Mov | String, ImplicitOps | Mov | String, - ByteOp | ImplicitOps | String, ImplicitOps | String, - /* 0xB0 - 0xBF */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 0xC0 - 0xC7 */ - ByteOp | DstMem | SrcImm | ModRM, DstMem | SrcImmByte | ModRM, - 0, ImplicitOps | Stack, 0, 0, - ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov, - /* 0xC8 - 0xCF */ - 0, 0, 0, 0, 0, 0, 0, 0, - /* 0xD0 - 0xD7 */ - ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, - ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, - 0, 0, 0, 0, - /* 0xD8 - 0xDF */ - 0, 0, 0, 0, 0, 0, 0, 0, - /* 0xE0 - 0xE7 */ - 0, 0, 0, 0, 0, 0, 0, 0, - /* 0xE8 - 0xEF */ - ImplicitOps | Stack, SrcImm|ImplicitOps, 0, SrcImmByte|ImplicitOps, - 0, 0, 0, 0, - /* 0xF0 - 0xF7 */ - 0, 0, 0, 0, - ImplicitOps, ImplicitOps, - ByteOp | DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM, - /* 0xF8 - 0xFF */ - ImplicitOps, 0, ImplicitOps, ImplicitOps, - 0, 0, ByteOp | DstMem | SrcNone | ModRM, DstMem | SrcNone | ModRM -}; - -static u16 twobyte_table[256] = { - /* 0x00 - 0x0F */ - 0, SrcMem | ModRM | DstReg, 0, 0, 0, 0, ImplicitOps, 0, - ImplicitOps, ImplicitOps, 0, 0, 0, ImplicitOps | ModRM, 0, 0, - /* 0x10 - 0x1F */ - 0, 0, 0, 0, 0, 0, 0, 0, ImplicitOps | ModRM, 0, 0, 0, 0, 0, 0, 0, - /* 0x20 - 0x2F */ - ModRM | ImplicitOps, ModRM, ModRM | ImplicitOps, ModRM, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - /* 0x30 - 0x3F */ - ImplicitOps, 0, ImplicitOps, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 0x40 - 0x47 */ - DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - /* 0x48 - 0x4F */ - DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - DstReg | SrcMem | ModRM | Mov, DstReg | SrcMem | ModRM | Mov, - /* 0x50 - 0x5F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 0x60 - 0x6F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 0x70 - 0x7F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 0x80 - 0x8F */ - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - ImplicitOps, ImplicitOps, ImplicitOps, ImplicitOps, - /* 0x90 - 0x9F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 0xA0 - 0xA7 */ - 0, 0, 0, DstMem | SrcReg | ModRM | BitOp, 0, 0, 0, 0, - /* 0xA8 - 0xAF */ - 0, 0, 0, DstMem | SrcReg | ModRM | BitOp, 0, 0, 0, 0, - /* 0xB0 - 0xB7 */ - ByteOp | DstMem | SrcReg | ModRM, DstMem | SrcReg | ModRM, 0, - DstMem | SrcReg | ModRM | BitOp, - 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov, - DstReg | SrcMem16 | ModRM | Mov, - /* 0xB8 - 0xBF */ - 0, 0, DstMem | SrcImmByte | ModRM, DstMem | SrcReg | ModRM | BitOp, - 0, 0, ByteOp | DstReg | SrcMem | ModRM | Mov, - DstReg | SrcMem16 | ModRM | Mov, - /* 0xC0 - 0xCF */ - 0, 0, 0, DstMem | SrcReg | ModRM | Mov, 0, 0, 0, ImplicitOps | ModRM, - 0, 0, 0, 0, 0, 0, 0, 0, - /* 0xD0 - 0xDF */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 0xE0 - 0xEF */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - /* 0xF0 - 0xFF */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 -}; - -/* EFLAGS bit definitions. */ -#define EFLG_OF (1<<11) -#define EFLG_DF (1<<10) -#define EFLG_SF (1<<7) -#define EFLG_ZF (1<<6) -#define EFLG_AF (1<<4) -#define EFLG_PF (1<<2) -#define EFLG_CF (1<<0) - -/* - * Instruction emulation: - * Most instructions are emulated directly via a fragment of inline assembly - * code. This allows us to save/restore EFLAGS and thus very easily pick up - * any modified flags. - */ - -#if defined(CONFIG_X86_64) -#define _LO32 "k" /* force 32-bit operand */ -#define _STK "%%rsp" /* stack pointer */ -#elif defined(__i386__) -#define _LO32 "" /* force 32-bit operand */ -#define _STK "%%esp" /* stack pointer */ -#endif - -/* - * These EFLAGS bits are restored from saved value during emulation, and - * any changes are written back to the saved value after emulation. - */ -#define EFLAGS_MASK (EFLG_OF|EFLG_SF|EFLG_ZF|EFLG_AF|EFLG_PF|EFLG_CF) - -/* Before executing instruction: restore necessary bits in EFLAGS. */ -#define _PRE_EFLAGS(_sav, _msk, _tmp) \ - /* EFLAGS = (_sav & _msk) | (EFLAGS & ~_msk); _sav &= ~_msk; */ \ - "movl %"_sav",%"_LO32 _tmp"; " \ - "push %"_tmp"; " \ - "push %"_tmp"; " \ - "movl %"_msk",%"_LO32 _tmp"; " \ - "andl %"_LO32 _tmp",("_STK"); " \ - "pushf; " \ - "notl %"_LO32 _tmp"; " \ - "andl %"_LO32 _tmp",("_STK"); " \ - "andl %"_LO32 _tmp","__stringify(BITS_PER_LONG/4)"("_STK"); " \ - "pop %"_tmp"; " \ - "orl %"_LO32 _tmp",("_STK"); " \ - "popf; " \ - "pop %"_sav"; " - -/* After executing instruction: write-back necessary bits in EFLAGS. */ -#define _POST_EFLAGS(_sav, _msk, _tmp) \ - /* _sav |= EFLAGS & _msk; */ \ - "pushf; " \ - "pop %"_tmp"; " \ - "andl %"_msk",%"_LO32 _tmp"; " \ - "orl %"_LO32 _tmp",%"_sav"; " - -/* Raw emulation: instruction has two explicit operands. */ -#define __emulate_2op_nobyte(_op,_src,_dst,_eflags,_wx,_wy,_lx,_ly,_qx,_qy) \ - do { \ - unsigned long _tmp; \ - \ - switch ((_dst).bytes) { \ - case 2: \ - __asm__ __volatile__ ( \ - _PRE_EFLAGS("0", "4", "2") \ - _op"w %"_wx"3,%1; " \ - _POST_EFLAGS("0", "4", "2") \ - : "=m" (_eflags), "=m" ((_dst).val), \ - "=&r" (_tmp) \ - : _wy ((_src).val), "i" (EFLAGS_MASK)); \ - break; \ - case 4: \ - __asm__ __volatile__ ( \ - _PRE_EFLAGS("0", "4", "2") \ - _op"l %"_lx"3,%1; " \ - _POST_EFLAGS("0", "4", "2") \ - : "=m" (_eflags), "=m" ((_dst).val), \ - "=&r" (_tmp) \ - : _ly ((_src).val), "i" (EFLAGS_MASK)); \ - break; \ - case 8: \ - __emulate_2op_8byte(_op, _src, _dst, \ - _eflags, _qx, _qy); \ - break; \ - } \ - } while (0) - -#define __emulate_2op(_op,_src,_dst,_eflags,_bx,_by,_wx,_wy,_lx,_ly,_qx,_qy) \ - do { \ - unsigned long _tmp; \ - switch ((_dst).bytes) { \ - case 1: \ - __asm__ __volatile__ ( \ - _PRE_EFLAGS("0", "4", "2") \ - _op"b %"_bx"3,%1; " \ - _POST_EFLAGS("0", "4", "2") \ - : "=m" (_eflags), "=m" ((_dst).val), \ - "=&r" (_tmp) \ - : _by ((_src).val), "i" (EFLAGS_MASK)); \ - break; \ - default: \ - __emulate_2op_nobyte(_op, _src, _dst, _eflags, \ - _wx, _wy, _lx, _ly, _qx, _qy); \ - break; \ - } \ - } while (0) - -/* Source operand is byte-sized and may be restricted to just %cl. */ -#define emulate_2op_SrcB(_op, _src, _dst, _eflags) \ - __emulate_2op(_op, _src, _dst, _eflags, \ - "b", "c", "b", "c", "b", "c", "b", "c") - -/* Source operand is byte, word, long or quad sized. */ -#define emulate_2op_SrcV(_op, _src, _dst, _eflags) \ - __emulate_2op(_op, _src, _dst, _eflags, \ - "b", "q", "w", "r", _LO32, "r", "", "r") - -/* Source operand is word, long or quad sized. */ -#define emulate_2op_SrcV_nobyte(_op, _src, _dst, _eflags) \ - __emulate_2op_nobyte(_op, _src, _dst, _eflags, \ - "w", "r", _LO32, "r", "", "r") - -/* Instruction has only one explicit operand (no source operand). */ -#define emulate_1op(_op, _dst, _eflags) \ - do { \ - unsigned long _tmp; \ - \ - switch ((_dst).bytes) { \ - case 1: \ - __asm__ __volatile__ ( \ - _PRE_EFLAGS("0", "3", "2") \ - _op"b %1; " \ - _POST_EFLAGS("0", "3", "2") \ - : "=m" (_eflags), "=m" ((_dst).val), \ - "=&r" (_tmp) \ - : "i" (EFLAGS_MASK)); \ - break; \ - case 2: \ - __asm__ __volatile__ ( \ - _PRE_EFLAGS("0", "3", "2") \ - _op"w %1; " \ - _POST_EFLAGS("0", "3", "2") \ - : "=m" (_eflags), "=m" ((_dst).val), \ - "=&r" (_tmp) \ - : "i" (EFLAGS_MASK)); \ - break; \ - case 4: \ - __asm__ __volatile__ ( \ - _PRE_EFLAGS("0", "3", "2") \ - _op"l %1; " \ - _POST_EFLAGS("0", "3", "2") \ - : "=m" (_eflags), "=m" ((_dst).val), \ - "=&r" (_tmp) \ - : "i" (EFLAGS_MASK)); \ - break; \ - case 8: \ - __emulate_1op_8byte(_op, _dst, _eflags); \ - break; \ - } \ - } while (0) - -/* Emulate an instruction with quadword operands (x86/64 only). */ -#if defined(CONFIG_X86_64) -#define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy) \ - do { \ - __asm__ __volatile__ ( \ - _PRE_EFLAGS("0", "4", "2") \ - _op"q %"_qx"3,%1; " \ - _POST_EFLAGS("0", "4", "2") \ - : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \ - : _qy ((_src).val), "i" (EFLAGS_MASK)); \ - } while (0) - -#define __emulate_1op_8byte(_op, _dst, _eflags) \ - do { \ - __asm__ __volatile__ ( \ - _PRE_EFLAGS("0", "3", "2") \ - _op"q %1; " \ - _POST_EFLAGS("0", "3", "2") \ - : "=m" (_eflags), "=m" ((_dst).val), "=&r" (_tmp) \ - : "i" (EFLAGS_MASK)); \ - } while (0) - -#elif defined(__i386__) -#define __emulate_2op_8byte(_op, _src, _dst, _eflags, _qx, _qy) -#define __emulate_1op_8byte(_op, _dst, _eflags) -#endif /* __i386__ */ - -/* Fetch next part of the instruction being emulated. */ -#define insn_fetch(_type, _size, _eip) \ -({ unsigned long _x; \ - rc = do_insn_fetch(ctxt, ops, (_eip), &_x, (_size)); \ - if (rc != 0) \ - goto done; \ - (_eip) += (_size); \ - (_type)_x; \ -}) - -/* Access/update address held in a register, based on addressing mode. */ -#define address_mask(reg) \ - ((c->ad_bytes == sizeof(unsigned long)) ? \ - (reg) : ((reg) & ((1UL << (c->ad_bytes << 3)) - 1))) -#define register_address(base, reg) \ - ((base) + address_mask(reg)) -#define register_address_increment(reg, inc) \ - do { \ - /* signed type ensures sign extension to long */ \ - int _inc = (inc); \ - if (c->ad_bytes == sizeof(unsigned long)) \ - (reg) += _inc; \ - else \ - (reg) = ((reg) & \ - ~((1UL << (c->ad_bytes << 3)) - 1)) | \ - (((reg) + _inc) & \ - ((1UL << (c->ad_bytes << 3)) - 1)); \ - } while (0) - -#define JMP_REL(rel) \ - do { \ - register_address_increment(c->eip, rel); \ - } while (0) - -static int do_fetch_insn_byte(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - unsigned long linear, u8 *dest) -{ - struct fetch_cache *fc = &ctxt->decode.fetch; - int rc; - int size; - - if (linear < fc->start || linear >= fc->end) { - size = min(15UL, PAGE_SIZE - offset_in_page(linear)); - rc = ops->read_std(linear, fc->data, size, ctxt->vcpu); - if (rc) - return rc; - fc->start = linear; - fc->end = linear + size; - } - *dest = fc->data[linear - fc->start]; - return 0; -} - -static int do_insn_fetch(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - unsigned long eip, void *dest, unsigned size) -{ - int rc = 0; - - eip += ctxt->cs_base; - while (size--) { - rc = do_fetch_insn_byte(ctxt, ops, eip++, dest++); - if (rc) - return rc; - } - return 0; -} - -/* - * Given the 'reg' portion of a ModRM byte, and a register block, return a - * pointer into the block that addresses the relevant register. - * @highbyte_regs specifies whether to decode AH,CH,DH,BH. - */ -static void *decode_register(u8 modrm_reg, unsigned long *regs, - int highbyte_regs) -{ - void *p; - - p = ®s[modrm_reg]; - if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8) - p = (unsigned char *)®s[modrm_reg & 3] + 1; - return p; -} - -static int read_descriptor(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - void *ptr, - u16 *size, unsigned long *address, int op_bytes) -{ - int rc; - - if (op_bytes == 2) - op_bytes = 3; - *address = 0; - rc = ops->read_std((unsigned long)ptr, (unsigned long *)size, 2, - ctxt->vcpu); - if (rc) - return rc; - rc = ops->read_std((unsigned long)ptr + 2, address, op_bytes, - ctxt->vcpu); - return rc; -} - -static int test_cc(unsigned int condition, unsigned int flags) -{ - int rc = 0; - - switch ((condition & 15) >> 1) { - case 0: /* o */ - rc |= (flags & EFLG_OF); - break; - case 1: /* b/c/nae */ - rc |= (flags & EFLG_CF); - break; - case 2: /* z/e */ - rc |= (flags & EFLG_ZF); - break; - case 3: /* be/na */ - rc |= (flags & (EFLG_CF|EFLG_ZF)); - break; - case 4: /* s */ - rc |= (flags & EFLG_SF); - break; - case 5: /* p/pe */ - rc |= (flags & EFLG_PF); - break; - case 7: /* le/ng */ - rc |= (flags & EFLG_ZF); - /* fall through */ - case 6: /* l/nge */ - rc |= (!(flags & EFLG_SF) != !(flags & EFLG_OF)); - break; - } - - /* Odd condition identifiers (lsb == 1) have inverted sense. */ - return (!!rc ^ (condition & 1)); -} - -static void decode_register_operand(struct operand *op, - struct decode_cache *c, - int inhibit_bytereg) -{ - unsigned reg = c->modrm_reg; - int highbyte_regs = c->rex_prefix == 0; - - if (!(c->d & ModRM)) - reg = (c->b & 7) | ((c->rex_prefix & 1) << 3); - op->type = OP_REG; - if ((c->d & ByteOp) && !inhibit_bytereg) { - op->ptr = decode_register(reg, c->regs, highbyte_regs); - op->val = *(u8 *)op->ptr; - op->bytes = 1; - } else { - op->ptr = decode_register(reg, c->regs, 0); - op->bytes = c->op_bytes; - switch (op->bytes) { - case 2: - op->val = *(u16 *)op->ptr; - break; - case 4: - op->val = *(u32 *)op->ptr; - break; - case 8: - op->val = *(u64 *) op->ptr; - break; - } - } - op->orig_val = op->val; -} - -static int decode_modrm(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) -{ - struct decode_cache *c = &ctxt->decode; - u8 sib; - int index_reg = 0, base_reg = 0, scale, rip_relative = 0; - int rc = 0; - - if (c->rex_prefix) { - c->modrm_reg = (c->rex_prefix & 4) << 1; /* REX.R */ - index_reg = (c->rex_prefix & 2) << 2; /* REX.X */ - c->modrm_rm = base_reg = (c->rex_prefix & 1) << 3; /* REG.B */ - } - - c->modrm = insn_fetch(u8, 1, c->eip); - c->modrm_mod |= (c->modrm & 0xc0) >> 6; - c->modrm_reg |= (c->modrm & 0x38) >> 3; - c->modrm_rm |= (c->modrm & 0x07); - c->modrm_ea = 0; - c->use_modrm_ea = 1; - - if (c->modrm_mod == 3) { - c->modrm_val = *(unsigned long *) - decode_register(c->modrm_rm, c->regs, c->d & ByteOp); - return rc; - } - - if (c->ad_bytes == 2) { - unsigned bx = c->regs[VCPU_REGS_RBX]; - unsigned bp = c->regs[VCPU_REGS_RBP]; - unsigned si = c->regs[VCPU_REGS_RSI]; - unsigned di = c->regs[VCPU_REGS_RDI]; - - /* 16-bit ModR/M decode. */ - switch (c->modrm_mod) { - case 0: - if (c->modrm_rm == 6) - c->modrm_ea += insn_fetch(u16, 2, c->eip); - break; - case 1: - c->modrm_ea += insn_fetch(s8, 1, c->eip); - break; - case 2: - c->modrm_ea += insn_fetch(u16, 2, c->eip); - break; - } - switch (c->modrm_rm) { - case 0: - c->modrm_ea += bx + si; - break; - case 1: - c->modrm_ea += bx + di; - break; - case 2: - c->modrm_ea += bp + si; - break; - case 3: - c->modrm_ea += bp + di; - break; - case 4: - c->modrm_ea += si; - break; - case 5: - c->modrm_ea += di; - break; - case 6: - if (c->modrm_mod != 0) - c->modrm_ea += bp; - break; - case 7: - c->modrm_ea += bx; - break; - } - if (c->modrm_rm == 2 || c->modrm_rm == 3 || - (c->modrm_rm == 6 && c->modrm_mod != 0)) - if (!c->override_base) - c->override_base = &ctxt->ss_base; - c->modrm_ea = (u16)c->modrm_ea; - } else { - /* 32/64-bit ModR/M decode. */ - switch (c->modrm_rm) { - case 4: - case 12: - sib = insn_fetch(u8, 1, c->eip); - index_reg |= (sib >> 3) & 7; - base_reg |= sib & 7; - scale = sib >> 6; - - switch (base_reg) { - case 5: - if (c->modrm_mod != 0) - c->modrm_ea += c->regs[base_reg]; - else - c->modrm_ea += - insn_fetch(s32, 4, c->eip); - break; - default: - c->modrm_ea += c->regs[base_reg]; - } - switch (index_reg) { - case 4: - break; - default: - c->modrm_ea += c->regs[index_reg] << scale; - } - break; - case 5: - if (c->modrm_mod != 0) - c->modrm_ea += c->regs[c->modrm_rm]; - else if (ctxt->mode == X86EMUL_MODE_PROT64) - rip_relative = 1; - break; - default: - c->modrm_ea += c->regs[c->modrm_rm]; - break; - } - switch (c->modrm_mod) { - case 0: - if (c->modrm_rm == 5) - c->modrm_ea += insn_fetch(s32, 4, c->eip); - break; - case 1: - c->modrm_ea += insn_fetch(s8, 1, c->eip); - break; - case 2: - c->modrm_ea += insn_fetch(s32, 4, c->eip); - break; - } - } - if (rip_relative) { - c->modrm_ea += c->eip; - switch (c->d & SrcMask) { - case SrcImmByte: - c->modrm_ea += 1; - break; - case SrcImm: - if (c->d & ByteOp) - c->modrm_ea += 1; - else - if (c->op_bytes == 8) - c->modrm_ea += 4; - else - c->modrm_ea += c->op_bytes; - } - } -done: - return rc; -} - -static int decode_abs(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) -{ - struct decode_cache *c = &ctxt->decode; - int rc = 0; - - switch (c->ad_bytes) { - case 2: - c->modrm_ea = insn_fetch(u16, 2, c->eip); - break; - case 4: - c->modrm_ea = insn_fetch(u32, 4, c->eip); - break; - case 8: - c->modrm_ea = insn_fetch(u64, 8, c->eip); - break; - } -done: - return rc; -} - -int -x86_decode_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) -{ - struct decode_cache *c = &ctxt->decode; - int rc = 0; - int mode = ctxt->mode; - int def_op_bytes, def_ad_bytes; - - /* Shadow copy of register state. Committed on successful emulation. */ - - memset(c, 0, sizeof(struct decode_cache)); - c->eip = ctxt->vcpu->arch.rip; - memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); - - switch (mode) { - case X86EMUL_MODE_REAL: - case X86EMUL_MODE_PROT16: - def_op_bytes = def_ad_bytes = 2; - break; - case X86EMUL_MODE_PROT32: - def_op_bytes = def_ad_bytes = 4; - break; -#ifdef CONFIG_X86_64 - case X86EMUL_MODE_PROT64: - def_op_bytes = 4; - def_ad_bytes = 8; - break; -#endif - default: - return -1; - } - - c->op_bytes = def_op_bytes; - c->ad_bytes = def_ad_bytes; - - /* Legacy prefixes. */ - for (;;) { - switch (c->b = insn_fetch(u8, 1, c->eip)) { - case 0x66: /* operand-size override */ - /* switch between 2/4 bytes */ - c->op_bytes = def_op_bytes ^ 6; - break; - case 0x67: /* address-size override */ - if (mode == X86EMUL_MODE_PROT64) - /* switch between 4/8 bytes */ - c->ad_bytes = def_ad_bytes ^ 12; - else - /* switch between 2/4 bytes */ - c->ad_bytes = def_ad_bytes ^ 6; - break; - case 0x2e: /* CS override */ - c->override_base = &ctxt->cs_base; - break; - case 0x3e: /* DS override */ - c->override_base = &ctxt->ds_base; - break; - case 0x26: /* ES override */ - c->override_base = &ctxt->es_base; - break; - case 0x64: /* FS override */ - c->override_base = &ctxt->fs_base; - break; - case 0x65: /* GS override */ - c->override_base = &ctxt->gs_base; - break; - case 0x36: /* SS override */ - c->override_base = &ctxt->ss_base; - break; - case 0x40 ... 0x4f: /* REX */ - if (mode != X86EMUL_MODE_PROT64) - goto done_prefixes; - c->rex_prefix = c->b; - continue; - case 0xf0: /* LOCK */ - c->lock_prefix = 1; - break; - case 0xf2: /* REPNE/REPNZ */ - c->rep_prefix = REPNE_PREFIX; - break; - case 0xf3: /* REP/REPE/REPZ */ - c->rep_prefix = REPE_PREFIX; - break; - default: - goto done_prefixes; - } - - /* Any legacy prefix after a REX prefix nullifies its effect. */ - - c->rex_prefix = 0; - } - -done_prefixes: - - /* REX prefix. */ - if (c->rex_prefix) - if (c->rex_prefix & 8) - c->op_bytes = 8; /* REX.W */ - - /* Opcode byte(s). */ - c->d = opcode_table[c->b]; - if (c->d == 0) { - /* Two-byte opcode? */ - if (c->b == 0x0f) { - c->twobyte = 1; - c->b = insn_fetch(u8, 1, c->eip); - c->d = twobyte_table[c->b]; - } - - /* Unrecognised? */ - if (c->d == 0) { - DPRINTF("Cannot emulate %02x\n", c->b); - return -1; - } - } - - if (mode == X86EMUL_MODE_PROT64 && (c->d & Stack)) - c->op_bytes = 8; - - /* ModRM and SIB bytes. */ - if (c->d & ModRM) - rc = decode_modrm(ctxt, ops); - else if (c->d & MemAbs) - rc = decode_abs(ctxt, ops); - if (rc) - goto done; - - if (!c->override_base) - c->override_base = &ctxt->ds_base; - if (mode == X86EMUL_MODE_PROT64 && - c->override_base != &ctxt->fs_base && - c->override_base != &ctxt->gs_base) - c->override_base = NULL; - - if (c->override_base) - c->modrm_ea += *c->override_base; - - if (c->ad_bytes != 8) - c->modrm_ea = (u32)c->modrm_ea; - /* - * Decode and fetch the source operand: register, memory - * or immediate. - */ - switch (c->d & SrcMask) { - case SrcNone: - break; - case SrcReg: - decode_register_operand(&c->src, c, 0); - break; - case SrcMem16: - c->src.bytes = 2; - goto srcmem_common; - case SrcMem32: - c->src.bytes = 4; - goto srcmem_common; - case SrcMem: - c->src.bytes = (c->d & ByteOp) ? 1 : - c->op_bytes; - /* Don't fetch the address for invlpg: it could be unmapped. */ - if (c->twobyte && c->b == 0x01 && c->modrm_reg == 7) - break; - srcmem_common: - /* - * For instructions with a ModR/M byte, switch to register - * access if Mod = 3. - */ - if ((c->d & ModRM) && c->modrm_mod == 3) { - c->src.type = OP_REG; - break; - } - c->src.type = OP_MEM; - break; - case SrcImm: - c->src.type = OP_IMM; - c->src.ptr = (unsigned long *)c->eip; - c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - if (c->src.bytes == 8) - c->src.bytes = 4; - /* NB. Immediates are sign-extended as necessary. */ - switch (c->src.bytes) { - case 1: - c->src.val = insn_fetch(s8, 1, c->eip); - break; - case 2: - c->src.val = insn_fetch(s16, 2, c->eip); - break; - case 4: - c->src.val = insn_fetch(s32, 4, c->eip); - break; - } - break; - case SrcImmByte: - c->src.type = OP_IMM; - c->src.ptr = (unsigned long *)c->eip; - c->src.bytes = 1; - c->src.val = insn_fetch(s8, 1, c->eip); - break; - } - - /* Decode and fetch the destination operand: register or memory. */ - switch (c->d & DstMask) { - case ImplicitOps: - /* Special instructions do their own operand decoding. */ - return 0; - case DstReg: - decode_register_operand(&c->dst, c, - c->twobyte && (c->b == 0xb6 || c->b == 0xb7)); - break; - case DstMem: - if ((c->d & ModRM) && c->modrm_mod == 3) { - c->dst.type = OP_REG; - break; - } - c->dst.type = OP_MEM; - break; - } - -done: - return (rc == X86EMUL_UNHANDLEABLE) ? -1 : 0; -} - -static inline void emulate_push(struct x86_emulate_ctxt *ctxt) -{ - struct decode_cache *c = &ctxt->decode; - - c->dst.type = OP_MEM; - c->dst.bytes = c->op_bytes; - c->dst.val = c->src.val; - register_address_increment(c->regs[VCPU_REGS_RSP], -c->op_bytes); - c->dst.ptr = (void *) register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]); -} - -static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) -{ - struct decode_cache *c = &ctxt->decode; - int rc; - - rc = ops->read_std(register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), - &c->dst.val, c->dst.bytes, ctxt->vcpu); - if (rc != 0) - return rc; - - register_address_increment(c->regs[VCPU_REGS_RSP], c->dst.bytes); - - return 0; -} - -static inline void emulate_grp2(struct x86_emulate_ctxt *ctxt) -{ - struct decode_cache *c = &ctxt->decode; - switch (c->modrm_reg) { - case 0: /* rol */ - emulate_2op_SrcB("rol", c->src, c->dst, ctxt->eflags); - break; - case 1: /* ror */ - emulate_2op_SrcB("ror", c->src, c->dst, ctxt->eflags); - break; - case 2: /* rcl */ - emulate_2op_SrcB("rcl", c->src, c->dst, ctxt->eflags); - break; - case 3: /* rcr */ - emulate_2op_SrcB("rcr", c->src, c->dst, ctxt->eflags); - break; - case 4: /* sal/shl */ - case 6: /* sal/shl */ - emulate_2op_SrcB("sal", c->src, c->dst, ctxt->eflags); - break; - case 5: /* shr */ - emulate_2op_SrcB("shr", c->src, c->dst, ctxt->eflags); - break; - case 7: /* sar */ - emulate_2op_SrcB("sar", c->src, c->dst, ctxt->eflags); - break; - } -} - -static inline int emulate_grp3(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) -{ - struct decode_cache *c = &ctxt->decode; - int rc = 0; - - switch (c->modrm_reg) { - case 0 ... 1: /* test */ - /* - * Special case in Grp3: test has an immediate - * source operand. - */ - c->src.type = OP_IMM; - c->src.ptr = (unsigned long *)c->eip; - c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - if (c->src.bytes == 8) - c->src.bytes = 4; - switch (c->src.bytes) { - case 1: - c->src.val = insn_fetch(s8, 1, c->eip); - break; - case 2: - c->src.val = insn_fetch(s16, 2, c->eip); - break; - case 4: - c->src.val = insn_fetch(s32, 4, c->eip); - break; - } - emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags); - break; - case 2: /* not */ - c->dst.val = ~c->dst.val; - break; - case 3: /* neg */ - emulate_1op("neg", c->dst, ctxt->eflags); - break; - default: - DPRINTF("Cannot emulate %02x\n", c->b); - rc = X86EMUL_UNHANDLEABLE; - break; - } -done: - return rc; -} - -static inline int emulate_grp45(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) -{ - struct decode_cache *c = &ctxt->decode; - int rc; - - switch (c->modrm_reg) { - case 0: /* inc */ - emulate_1op("inc", c->dst, ctxt->eflags); - break; - case 1: /* dec */ - emulate_1op("dec", c->dst, ctxt->eflags); - break; - case 4: /* jmp abs */ - if (c->b == 0xff) - c->eip = c->dst.val; - else { - DPRINTF("Cannot emulate %02x\n", c->b); - return X86EMUL_UNHANDLEABLE; - } - break; - case 6: /* push */ - - /* 64-bit mode: PUSH always pushes a 64-bit operand. */ - - if (ctxt->mode == X86EMUL_MODE_PROT64) { - c->dst.bytes = 8; - rc = ops->read_std((unsigned long)c->dst.ptr, - &c->dst.val, 8, ctxt->vcpu); - if (rc != 0) - return rc; - } - register_address_increment(c->regs[VCPU_REGS_RSP], - -c->dst.bytes); - rc = ops->write_emulated(register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), &c->dst.val, - c->dst.bytes, ctxt->vcpu); - if (rc != 0) - return rc; - c->dst.type = OP_NONE; - break; - default: - DPRINTF("Cannot emulate %02x\n", c->b); - return X86EMUL_UNHANDLEABLE; - } - return 0; -} - -static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops, - unsigned long memop) -{ - struct decode_cache *c = &ctxt->decode; - u64 old, new; - int rc; - - rc = ops->read_emulated(memop, &old, 8, ctxt->vcpu); - if (rc != 0) - return rc; - - if (((u32) (old >> 0) != (u32) c->regs[VCPU_REGS_RAX]) || - ((u32) (old >> 32) != (u32) c->regs[VCPU_REGS_RDX])) { - - c->regs[VCPU_REGS_RAX] = (u32) (old >> 0); - c->regs[VCPU_REGS_RDX] = (u32) (old >> 32); - ctxt->eflags &= ~EFLG_ZF; - - } else { - new = ((u64)c->regs[VCPU_REGS_RCX] << 32) | - (u32) c->regs[VCPU_REGS_RBX]; - - rc = ops->cmpxchg_emulated(memop, &old, &new, 8, ctxt->vcpu); - if (rc != 0) - return rc; - ctxt->eflags |= EFLG_ZF; - } - return 0; -} - -static inline int writeback(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops) -{ - int rc; - struct decode_cache *c = &ctxt->decode; - - switch (c->dst.type) { - case OP_REG: - /* The 4-byte case *is* correct: - * in 64-bit mode we zero-extend. - */ - switch (c->dst.bytes) { - case 1: - *(u8 *)c->dst.ptr = (u8)c->dst.val; - break; - case 2: - *(u16 *)c->dst.ptr = (u16)c->dst.val; - break; - case 4: - *c->dst.ptr = (u32)c->dst.val; - break; /* 64b: zero-ext */ - case 8: - *c->dst.ptr = c->dst.val; - break; - } - break; - case OP_MEM: - if (c->lock_prefix) - rc = ops->cmpxchg_emulated( - (unsigned long)c->dst.ptr, - &c->dst.orig_val, - &c->dst.val, - c->dst.bytes, - ctxt->vcpu); - else - rc = ops->write_emulated( - (unsigned long)c->dst.ptr, - &c->dst.val, - c->dst.bytes, - ctxt->vcpu); - if (rc != 0) - return rc; - break; - case OP_NONE: - /* no writeback */ - break; - default: - break; - } - return 0; -} - -int -x86_emulate_insn(struct x86_emulate_ctxt *ctxt, struct x86_emulate_ops *ops) -{ - unsigned long memop = 0; - u64 msr_data; - unsigned long saved_eip = 0; - struct decode_cache *c = &ctxt->decode; - int rc = 0; - - /* Shadow copy of register state. Committed on successful emulation. - * NOTE: we can copy them from vcpu as x86_decode_insn() doesn't - * modify them. - */ - - memcpy(c->regs, ctxt->vcpu->arch.regs, sizeof c->regs); - saved_eip = c->eip; - - if (((c->d & ModRM) && (c->modrm_mod != 3)) || (c->d & MemAbs)) - memop = c->modrm_ea; - - if (c->rep_prefix && (c->d & String)) { - /* All REP prefixes have the same first termination condition */ - if (c->regs[VCPU_REGS_RCX] == 0) { - ctxt->vcpu->arch.rip = c->eip; - goto done; - } - /* The second termination condition only applies for REPE - * and REPNE. Test if the repeat string operation prefix is - * REPE/REPZ or REPNE/REPNZ and if it's the case it tests the - * corresponding termination condition according to: - * - if REPE/REPZ and ZF = 0 then done - * - if REPNE/REPNZ and ZF = 1 then done - */ - if ((c->b == 0xa6) || (c->b == 0xa7) || - (c->b == 0xae) || (c->b == 0xaf)) { - if ((c->rep_prefix == REPE_PREFIX) && - ((ctxt->eflags & EFLG_ZF) == 0)) { - ctxt->vcpu->arch.rip = c->eip; - goto done; - } - if ((c->rep_prefix == REPNE_PREFIX) && - ((ctxt->eflags & EFLG_ZF) == EFLG_ZF)) { - ctxt->vcpu->arch.rip = c->eip; - goto done; - } - } - c->regs[VCPU_REGS_RCX]--; - c->eip = ctxt->vcpu->arch.rip; - } - - if (c->src.type == OP_MEM) { - c->src.ptr = (unsigned long *)memop; - c->src.val = 0; - rc = ops->read_emulated((unsigned long)c->src.ptr, - &c->src.val, - c->src.bytes, - ctxt->vcpu); - if (rc != 0) - goto done; - c->src.orig_val = c->src.val; - } - - if ((c->d & DstMask) == ImplicitOps) - goto special_insn; - - - if (c->dst.type == OP_MEM) { - c->dst.ptr = (unsigned long *)memop; - c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.val = 0; - if (c->d & BitOp) { - unsigned long mask = ~(c->dst.bytes * 8 - 1); - - c->dst.ptr = (void *)c->dst.ptr + - (c->src.val & mask) / 8; - } - if (!(c->d & Mov) && - /* optimisation - avoid slow emulated read */ - ((rc = ops->read_emulated((unsigned long)c->dst.ptr, - &c->dst.val, - c->dst.bytes, ctxt->vcpu)) != 0)) - goto done; - } - c->dst.orig_val = c->dst.val; - -special_insn: - - if (c->twobyte) - goto twobyte_insn; - - switch (c->b) { - case 0x00 ... 0x05: - add: /* add */ - emulate_2op_SrcV("add", c->src, c->dst, ctxt->eflags); - break; - case 0x08 ... 0x0d: - or: /* or */ - emulate_2op_SrcV("or", c->src, c->dst, ctxt->eflags); - break; - case 0x10 ... 0x15: - adc: /* adc */ - emulate_2op_SrcV("adc", c->src, c->dst, ctxt->eflags); - break; - case 0x18 ... 0x1d: - sbb: /* sbb */ - emulate_2op_SrcV("sbb", c->src, c->dst, ctxt->eflags); - break; - case 0x20 ... 0x23: - and: /* and */ - emulate_2op_SrcV("and", c->src, c->dst, ctxt->eflags); - break; - case 0x24: /* and al imm8 */ - c->dst.type = OP_REG; - c->dst.ptr = &c->regs[VCPU_REGS_RAX]; - c->dst.val = *(u8 *)c->dst.ptr; - c->dst.bytes = 1; - c->dst.orig_val = c->dst.val; - goto and; - case 0x25: /* and ax imm16, or eax imm32 */ - c->dst.type = OP_REG; - c->dst.bytes = c->op_bytes; - c->dst.ptr = &c->regs[VCPU_REGS_RAX]; - if (c->op_bytes == 2) - c->dst.val = *(u16 *)c->dst.ptr; - else - c->dst.val = *(u32 *)c->dst.ptr; - c->dst.orig_val = c->dst.val; - goto and; - case 0x28 ... 0x2d: - sub: /* sub */ - emulate_2op_SrcV("sub", c->src, c->dst, ctxt->eflags); - break; - case 0x30 ... 0x35: - xor: /* xor */ - emulate_2op_SrcV("xor", c->src, c->dst, ctxt->eflags); - break; - case 0x38 ... 0x3d: - cmp: /* cmp */ - emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); - break; - case 0x40 ... 0x47: /* inc r16/r32 */ - emulate_1op("inc", c->dst, ctxt->eflags); - break; - case 0x48 ... 0x4f: /* dec r16/r32 */ - emulate_1op("dec", c->dst, ctxt->eflags); - break; - case 0x50 ... 0x57: /* push reg */ - c->dst.type = OP_MEM; - c->dst.bytes = c->op_bytes; - c->dst.val = c->src.val; - register_address_increment(c->regs[VCPU_REGS_RSP], - -c->op_bytes); - c->dst.ptr = (void *) register_address( - ctxt->ss_base, c->regs[VCPU_REGS_RSP]); - break; - case 0x58 ... 0x5f: /* pop reg */ - pop_instruction: - if ((rc = ops->read_std(register_address(ctxt->ss_base, - c->regs[VCPU_REGS_RSP]), c->dst.ptr, - c->op_bytes, ctxt->vcpu)) != 0) - goto done; - - register_address_increment(c->regs[VCPU_REGS_RSP], - c->op_bytes); - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0x63: /* movsxd */ - if (ctxt->mode != X86EMUL_MODE_PROT64) - goto cannot_emulate; - c->dst.val = (s32) c->src.val; - break; - case 0x6a: /* push imm8 */ - c->src.val = 0L; - c->src.val = insn_fetch(s8, 1, c->eip); - emulate_push(ctxt); - break; - case 0x6c: /* insb */ - case 0x6d: /* insw/insd */ - if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 1, - (c->d & ByteOp) ? 1 : c->op_bytes, - c->rep_prefix ? - address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (ctxt->eflags & EFLG_DF), - register_address(ctxt->es_base, - c->regs[VCPU_REGS_RDI]), - c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) { - c->eip = saved_eip; - return -1; - } - return 0; - case 0x6e: /* outsb */ - case 0x6f: /* outsw/outsd */ - if (kvm_emulate_pio_string(ctxt->vcpu, NULL, - 0, - (c->d & ByteOp) ? 1 : c->op_bytes, - c->rep_prefix ? - address_mask(c->regs[VCPU_REGS_RCX]) : 1, - (ctxt->eflags & EFLG_DF), - register_address(c->override_base ? - *c->override_base : - ctxt->ds_base, - c->regs[VCPU_REGS_RSI]), - c->rep_prefix, - c->regs[VCPU_REGS_RDX]) == 0) { - c->eip = saved_eip; - return -1; - } - return 0; - case 0x70 ... 0x7f: /* jcc (short) */ { - int rel = insn_fetch(s8, 1, c->eip); - - if (test_cc(c->b, ctxt->eflags)) - JMP_REL(rel); - break; - } - case 0x80 ... 0x83: /* Grp1 */ - switch (c->modrm_reg) { - case 0: - goto add; - case 1: - goto or; - case 2: - goto adc; - case 3: - goto sbb; - case 4: - goto and; - case 5: - goto sub; - case 6: - goto xor; - case 7: - goto cmp; - } - break; - case 0x84 ... 0x85: - emulate_2op_SrcV("test", c->src, c->dst, ctxt->eflags); - break; - case 0x86 ... 0x87: /* xchg */ - /* Write back the register source. */ - switch (c->dst.bytes) { - case 1: - *(u8 *) c->src.ptr = (u8) c->dst.val; - break; - case 2: - *(u16 *) c->src.ptr = (u16) c->dst.val; - break; - case 4: - *c->src.ptr = (u32) c->dst.val; - break; /* 64b reg: zero-extend */ - case 8: - *c->src.ptr = c->dst.val; - break; - } - /* - * Write back the memory destination with implicit LOCK - * prefix. - */ - c->dst.val = c->src.val; - c->lock_prefix = 1; - break; - case 0x88 ... 0x8b: /* mov */ - goto mov; - case 0x8d: /* lea r16/r32, m */ - c->dst.val = c->modrm_val; - break; - case 0x8f: /* pop (sole member of Grp1a) */ - rc = emulate_grp1a(ctxt, ops); - if (rc != 0) - goto done; - break; - case 0x9c: /* pushf */ - c->src.val = (unsigned long) ctxt->eflags; - emulate_push(ctxt); - break; - case 0x9d: /* popf */ - c->dst.ptr = (unsigned long *) &ctxt->eflags; - goto pop_instruction; - case 0xa0 ... 0xa1: /* mov */ - c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; - c->dst.val = c->src.val; - break; - case 0xa2 ... 0xa3: /* mov */ - c->dst.val = (unsigned long)c->regs[VCPU_REGS_RAX]; - break; - case 0xa4 ... 0xa5: /* movs */ - c->dst.type = OP_MEM; - c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.ptr = (unsigned long *)register_address( - ctxt->es_base, - c->regs[VCPU_REGS_RDI]); - if ((rc = ops->read_emulated(register_address( - c->override_base ? *c->override_base : - ctxt->ds_base, - c->regs[VCPU_REGS_RSI]), - &c->dst.val, - c->dst.bytes, ctxt->vcpu)) != 0) - goto done; - register_address_increment(c->regs[VCPU_REGS_RSI], - (ctxt->eflags & EFLG_DF) ? -c->dst.bytes - : c->dst.bytes); - register_address_increment(c->regs[VCPU_REGS_RDI], - (ctxt->eflags & EFLG_DF) ? -c->dst.bytes - : c->dst.bytes); - break; - case 0xa6 ... 0xa7: /* cmps */ - c->src.type = OP_NONE; /* Disable writeback. */ - c->src.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->src.ptr = (unsigned long *)register_address( - c->override_base ? *c->override_base : - ctxt->ds_base, - c->regs[VCPU_REGS_RSI]); - if ((rc = ops->read_emulated((unsigned long)c->src.ptr, - &c->src.val, - c->src.bytes, - ctxt->vcpu)) != 0) - goto done; - - c->dst.type = OP_NONE; /* Disable writeback. */ - c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.ptr = (unsigned long *)register_address( - ctxt->es_base, - c->regs[VCPU_REGS_RDI]); - if ((rc = ops->read_emulated((unsigned long)c->dst.ptr, - &c->dst.val, - c->dst.bytes, - ctxt->vcpu)) != 0) - goto done; - - DPRINTF("cmps: mem1=0x%p mem2=0x%p\n", c->src.ptr, c->dst.ptr); - - emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); - - register_address_increment(c->regs[VCPU_REGS_RSI], - (ctxt->eflags & EFLG_DF) ? -c->src.bytes - : c->src.bytes); - register_address_increment(c->regs[VCPU_REGS_RDI], - (ctxt->eflags & EFLG_DF) ? -c->dst.bytes - : c->dst.bytes); - - break; - case 0xaa ... 0xab: /* stos */ - c->dst.type = OP_MEM; - c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.ptr = (unsigned long *)register_address( - ctxt->es_base, - c->regs[VCPU_REGS_RDI]); - c->dst.val = c->regs[VCPU_REGS_RAX]; - register_address_increment(c->regs[VCPU_REGS_RDI], - (ctxt->eflags & EFLG_DF) ? -c->dst.bytes - : c->dst.bytes); - break; - case 0xac ... 0xad: /* lods */ - c->dst.type = OP_REG; - c->dst.bytes = (c->d & ByteOp) ? 1 : c->op_bytes; - c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; - if ((rc = ops->read_emulated(register_address( - c->override_base ? *c->override_base : - ctxt->ds_base, - c->regs[VCPU_REGS_RSI]), - &c->dst.val, - c->dst.bytes, - ctxt->vcpu)) != 0) - goto done; - register_address_increment(c->regs[VCPU_REGS_RSI], - (ctxt->eflags & EFLG_DF) ? -c->dst.bytes - : c->dst.bytes); - break; - case 0xae ... 0xaf: /* scas */ - DPRINTF("Urk! I don't handle SCAS.\n"); - goto cannot_emulate; - case 0xc0 ... 0xc1: - emulate_grp2(ctxt); - break; - case 0xc3: /* ret */ - c->dst.ptr = &c->eip; - goto pop_instruction; - case 0xc6 ... 0xc7: /* mov (sole member of Grp11) */ - mov: - c->dst.val = c->src.val; - break; - case 0xd0 ... 0xd1: /* Grp2 */ - c->src.val = 1; - emulate_grp2(ctxt); - break; - case 0xd2 ... 0xd3: /* Grp2 */ - c->src.val = c->regs[VCPU_REGS_RCX]; - emulate_grp2(ctxt); - break; - case 0xe8: /* call (near) */ { - long int rel; - switch (c->op_bytes) { - case 2: - rel = insn_fetch(s16, 2, c->eip); - break; - case 4: - rel = insn_fetch(s32, 4, c->eip); - break; - default: - DPRINTF("Call: Invalid op_bytes\n"); - goto cannot_emulate; - } - c->src.val = (unsigned long) c->eip; - JMP_REL(rel); - c->op_bytes = c->ad_bytes; - emulate_push(ctxt); - break; - } - case 0xe9: /* jmp rel */ - case 0xeb: /* jmp rel short */ - JMP_REL(c->src.val); - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xf4: /* hlt */ - ctxt->vcpu->arch.halt_request = 1; - goto done; - case 0xf5: /* cmc */ - /* complement carry flag from eflags reg */ - ctxt->eflags ^= EFLG_CF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xf6 ... 0xf7: /* Grp3 */ - rc = emulate_grp3(ctxt, ops); - if (rc != 0) - goto done; - break; - case 0xf8: /* clc */ - ctxt->eflags &= ~EFLG_CF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xfa: /* cli */ - ctxt->eflags &= ~X86_EFLAGS_IF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xfb: /* sti */ - ctxt->eflags |= X86_EFLAGS_IF; - c->dst.type = OP_NONE; /* Disable writeback. */ - break; - case 0xfe ... 0xff: /* Grp4/Grp5 */ - rc = emulate_grp45(ctxt, ops); - if (rc != 0) - goto done; - break; - } - -writeback: - rc = writeback(ctxt, ops); - if (rc != 0) - goto done; - - /* Commit shadow register state. */ - memcpy(ctxt->vcpu->arch.regs, c->regs, sizeof c->regs); - ctxt->vcpu->arch.rip = c->eip; - -done: - if (rc == X86EMUL_UNHANDLEABLE) { - c->eip = saved_eip; - return -1; - } - return 0; - -twobyte_insn: - switch (c->b) { - case 0x01: /* lgdt, lidt, lmsw */ - switch (c->modrm_reg) { - u16 size; - unsigned long address; - - case 0: /* vmcall */ - if (c->modrm_mod != 3 || c->modrm_rm != 1) - goto cannot_emulate; - - rc = kvm_fix_hypercall(ctxt->vcpu); - if (rc) - goto done; - - kvm_emulate_hypercall(ctxt->vcpu); - break; - case 2: /* lgdt */ - rc = read_descriptor(ctxt, ops, c->src.ptr, - &size, &address, c->op_bytes); - if (rc) - goto done; - realmode_lgdt(ctxt->vcpu, size, address); - break; - case 3: /* lidt/vmmcall */ - if (c->modrm_mod == 3 && c->modrm_rm == 1) { - rc = kvm_fix_hypercall(ctxt->vcpu); - if (rc) - goto done; - kvm_emulate_hypercall(ctxt->vcpu); - } else { - rc = read_descriptor(ctxt, ops, c->src.ptr, - &size, &address, - c->op_bytes); - if (rc) - goto done; - realmode_lidt(ctxt->vcpu, size, address); - } - break; - case 4: /* smsw */ - if (c->modrm_mod != 3) - goto cannot_emulate; - *(u16 *)&c->regs[c->modrm_rm] - = realmode_get_cr(ctxt->vcpu, 0); - break; - case 6: /* lmsw */ - if (c->modrm_mod != 3) - goto cannot_emulate; - realmode_lmsw(ctxt->vcpu, (u16)c->modrm_val, - &ctxt->eflags); - break; - case 7: /* invlpg*/ - emulate_invlpg(ctxt->vcpu, memop); - break; - default: - goto cannot_emulate; - } - /* Disable writeback. */ - c->dst.type = OP_NONE; - break; - case 0x06: - emulate_clts(ctxt->vcpu); - c->dst.type = OP_NONE; - break; - case 0x08: /* invd */ - case 0x09: /* wbinvd */ - case 0x0d: /* GrpP (prefetch) */ - case 0x18: /* Grp16 (prefetch/nop) */ - c->dst.type = OP_NONE; - break; - case 0x20: /* mov cr, reg */ - if (c->modrm_mod != 3) - goto cannot_emulate; - c->regs[c->modrm_rm] = - realmode_get_cr(ctxt->vcpu, c->modrm_reg); - c->dst.type = OP_NONE; /* no writeback */ - break; - case 0x21: /* mov from dr to reg */ - if (c->modrm_mod != 3) - goto cannot_emulate; - rc = emulator_get_dr(ctxt, c->modrm_reg, &c->regs[c->modrm_rm]); - if (rc) - goto cannot_emulate; - c->dst.type = OP_NONE; /* no writeback */ - break; - case 0x22: /* mov reg, cr */ - if (c->modrm_mod != 3) - goto cannot_emulate; - realmode_set_cr(ctxt->vcpu, - c->modrm_reg, c->modrm_val, &ctxt->eflags); - c->dst.type = OP_NONE; - break; - case 0x23: /* mov from reg to dr */ - if (c->modrm_mod != 3) - goto cannot_emulate; - rc = emulator_set_dr(ctxt, c->modrm_reg, - c->regs[c->modrm_rm]); - if (rc) - goto cannot_emulate; - c->dst.type = OP_NONE; /* no writeback */ - break; - case 0x30: - /* wrmsr */ - msr_data = (u32)c->regs[VCPU_REGS_RAX] - | ((u64)c->regs[VCPU_REGS_RDX] << 32); - rc = kvm_set_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], msr_data); - if (rc) { - kvm_inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->arch.rip; - } - rc = X86EMUL_CONTINUE; - c->dst.type = OP_NONE; - break; - case 0x32: - /* rdmsr */ - rc = kvm_get_msr(ctxt->vcpu, c->regs[VCPU_REGS_RCX], &msr_data); - if (rc) { - kvm_inject_gp(ctxt->vcpu, 0); - c->eip = ctxt->vcpu->arch.rip; - } else { - c->regs[VCPU_REGS_RAX] = (u32)msr_data; - c->regs[VCPU_REGS_RDX] = msr_data >> 32; - } - rc = X86EMUL_CONTINUE; - c->dst.type = OP_NONE; - break; - case 0x40 ... 0x4f: /* cmov */ - c->dst.val = c->dst.orig_val = c->src.val; - if (!test_cc(c->b, ctxt->eflags)) - c->dst.type = OP_NONE; /* no writeback */ - break; - case 0x80 ... 0x8f: /* jnz rel, etc*/ { - long int rel; - - switch (c->op_bytes) { - case 2: - rel = insn_fetch(s16, 2, c->eip); - break; - case 4: - rel = insn_fetch(s32, 4, c->eip); - break; - case 8: - rel = insn_fetch(s64, 8, c->eip); - break; - default: - DPRINTF("jnz: Invalid op_bytes\n"); - goto cannot_emulate; - } - if (test_cc(c->b, ctxt->eflags)) - JMP_REL(rel); - c->dst.type = OP_NONE; - break; - } - case 0xa3: - bt: /* bt */ - c->dst.type = OP_NONE; - /* only subword offset */ - c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("bt", c->src, c->dst, ctxt->eflags); - break; - case 0xab: - bts: /* bts */ - /* only subword offset */ - c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("bts", c->src, c->dst, ctxt->eflags); - break; - case 0xb0 ... 0xb1: /* cmpxchg */ - /* - * Save real source value, then compare EAX against - * destination. - */ - c->src.orig_val = c->src.val; - c->src.val = c->regs[VCPU_REGS_RAX]; - emulate_2op_SrcV("cmp", c->src, c->dst, ctxt->eflags); - if (ctxt->eflags & EFLG_ZF) { - /* Success: write back to memory. */ - c->dst.val = c->src.orig_val; - } else { - /* Failure: write the value we saw to EAX. */ - c->dst.type = OP_REG; - c->dst.ptr = (unsigned long *)&c->regs[VCPU_REGS_RAX]; - } - break; - case 0xb3: - btr: /* btr */ - /* only subword offset */ - c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("btr", c->src, c->dst, ctxt->eflags); - break; - case 0xb6 ... 0xb7: /* movzx */ - c->dst.bytes = c->op_bytes; - c->dst.val = (c->d & ByteOp) ? (u8) c->src.val - : (u16) c->src.val; - break; - case 0xba: /* Grp8 */ - switch (c->modrm_reg & 3) { - case 0: - goto bt; - case 1: - goto bts; - case 2: - goto btr; - case 3: - goto btc; - } - break; - case 0xbb: - btc: /* btc */ - /* only subword offset */ - c->src.val &= (c->dst.bytes << 3) - 1; - emulate_2op_SrcV_nobyte("btc", c->src, c->dst, ctxt->eflags); - break; - case 0xbe ... 0xbf: /* movsx */ - c->dst.bytes = c->op_bytes; - c->dst.val = (c->d & ByteOp) ? (s8) c->src.val : - (s16) c->src.val; - break; - case 0xc3: /* movnti */ - c->dst.bytes = c->op_bytes; - c->dst.val = (c->op_bytes == 4) ? (u32) c->src.val : - (u64) c->src.val; - break; - case 0xc7: /* Grp9 (cmpxchg8b) */ - rc = emulate_grp9(ctxt, ops, memop); - if (rc != 0) - goto done; - c->dst.type = OP_NONE; - break; - } - goto writeback; - -cannot_emulate: - DPRINTF("Cannot emulate %02x\n", c->b); - c->eip = saved_eip; - return -1; -} diff --git a/drivers/kvm/x86_emulate.h b/drivers/kvm/x86_emulate.h deleted file mode 100644 index 7db91b9bdcd4..000000000000 --- a/drivers/kvm/x86_emulate.h +++ /dev/null @@ -1,186 +0,0 @@ -/****************************************************************************** - * x86_emulate.h - * - * Generic x86 (32-bit and 64-bit) instruction decoder and emulator. - * - * Copyright (c) 2005 Keir Fraser - * - * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4 - */ - -#ifndef __X86_EMULATE_H__ -#define __X86_EMULATE_H__ - -struct x86_emulate_ctxt; - -/* - * x86_emulate_ops: - * - * These operations represent the instruction emulator's interface to memory. - * There are two categories of operation: those that act on ordinary memory - * regions (*_std), and those that act on memory regions known to require - * special treatment or emulation (*_emulated). - * - * The emulator assumes that an instruction accesses only one 'emulated memory' - * location, that this location is the given linear faulting address (cr2), and - * that this is one of the instruction's data operands. Instruction fetches and - * stack operations are assumed never to access emulated memory. The emulator - * automatically deduces which operand of a string-move operation is accessing - * emulated memory, and assumes that the other operand accesses normal memory. - * - * NOTES: - * 1. The emulator isn't very smart about emulated vs. standard memory. - * 'Emulated memory' access addresses should be checked for sanity. - * 'Normal memory' accesses may fault, and the caller must arrange to - * detect and handle reentrancy into the emulator via recursive faults. - * Accesses may be unaligned and may cross page boundaries. - * 2. If the access fails (cannot emulate, or a standard access faults) then - * it is up to the memop to propagate the fault to the guest VM via - * some out-of-band mechanism, unknown to the emulator. The memop signals - * failure by returning X86EMUL_PROPAGATE_FAULT to the emulator, which will - * then immediately bail. - * 3. Valid access sizes are 1, 2, 4 and 8 bytes. On x86/32 systems only - * cmpxchg8b_emulated need support 8-byte accesses. - * 4. The emulator cannot handle 64-bit mode emulation on an x86/32 system. - */ -/* Access completed successfully: continue emulation as normal. */ -#define X86EMUL_CONTINUE 0 -/* Access is unhandleable: bail from emulation and return error to caller. */ -#define X86EMUL_UNHANDLEABLE 1 -/* Terminate emulation but return success to the caller. */ -#define X86EMUL_PROPAGATE_FAULT 2 /* propagate a generated fault to guest */ -#define X86EMUL_RETRY_INSTR 2 /* retry the instruction for some reason */ -#define X86EMUL_CMPXCHG_FAILED 2 /* cmpxchg did not see expected value */ -struct x86_emulate_ops { - /* - * read_std: Read bytes of standard (non-emulated/special) memory. - * Used for instruction fetch, stack operations, and others. - * @addr: [IN ] Linear address from which to read. - * @val: [OUT] Value read from memory, zero-extended to 'u_long'. - * @bytes: [IN ] Number of bytes to read from memory. - */ - int (*read_std)(unsigned long addr, void *val, - unsigned int bytes, struct kvm_vcpu *vcpu); - - /* - * read_emulated: Read bytes from emulated/special memory area. - * @addr: [IN ] Linear address from which to read. - * @val: [OUT] Value read from memory, zero-extended to 'u_long'. - * @bytes: [IN ] Number of bytes to read from memory. - */ - int (*read_emulated) (unsigned long addr, - void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); - - /* - * write_emulated: Read bytes from emulated/special memory area. - * @addr: [IN ] Linear address to which to write. - * @val: [IN ] Value to write to memory (low-order bytes used as - * required). - * @bytes: [IN ] Number of bytes to write to memory. - */ - int (*write_emulated) (unsigned long addr, - const void *val, - unsigned int bytes, - struct kvm_vcpu *vcpu); - - /* - * cmpxchg_emulated: Emulate an atomic (LOCKed) CMPXCHG operation on an - * emulated/special memory area. - * @addr: [IN ] Linear address to access. - * @old: [IN ] Value expected to be current at @addr. - * @new: [IN ] Value to write to @addr. - * @bytes: [IN ] Number of bytes to access using CMPXCHG. - */ - int (*cmpxchg_emulated) (unsigned long addr, - const void *old, - const void *new, - unsigned int bytes, - struct kvm_vcpu *vcpu); - -}; - -/* Type, address-of, and value of an instruction's operand. */ -struct operand { - enum { OP_REG, OP_MEM, OP_IMM, OP_NONE } type; - unsigned int bytes; - unsigned long val, orig_val, *ptr; -}; - -struct fetch_cache { - u8 data[15]; - unsigned long start; - unsigned long end; -}; - -struct decode_cache { - u8 twobyte; - u8 b; - u8 lock_prefix; - u8 rep_prefix; - u8 op_bytes; - u8 ad_bytes; - u8 rex_prefix; - struct operand src; - struct operand dst; - unsigned long *override_base; - unsigned int d; - unsigned long regs[NR_VCPU_REGS]; - unsigned long eip; - /* modrm */ - u8 modrm; - u8 modrm_mod; - u8 modrm_reg; - u8 modrm_rm; - u8 use_modrm_ea; - unsigned long modrm_ea; - unsigned long modrm_val; - struct fetch_cache fetch; -}; - -struct x86_emulate_ctxt { - /* Register state before/after emulation. */ - struct kvm_vcpu *vcpu; - - /* Linear faulting address (if emulating a page-faulting instruction). */ - unsigned long eflags; - - /* Emulated execution mode, represented by an X86EMUL_MODE value. */ - int mode; - - unsigned long cs_base; - unsigned long ds_base; - unsigned long es_base; - unsigned long ss_base; - unsigned long gs_base; - unsigned long fs_base; - - /* decode cache */ - - struct decode_cache decode; -}; - -/* Repeat String Operation Prefix */ -#define REPE_PREFIX 1 -#define REPNE_PREFIX 2 - -/* Execution mode, passed to the emulator. */ -#define X86EMUL_MODE_REAL 0 /* Real mode. */ -#define X86EMUL_MODE_PROT16 2 /* 16-bit protected mode. */ -#define X86EMUL_MODE_PROT32 4 /* 32-bit protected mode. */ -#define X86EMUL_MODE_PROT64 8 /* 64-bit (long) mode. */ - -/* Host execution mode. */ -#if defined(__i386__) -#define X86EMUL_MODE_HOST X86EMUL_MODE_PROT32 -#elif defined(CONFIG_X86_64) -#define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64 -#endif - -int x86_decode_insn(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops); -int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, - struct x86_emulate_ops *ops); - -#endif /* __X86_EMULATE_H__ */ diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h new file mode 100644 index 000000000000..28940e1a9713 --- /dev/null +++ b/include/asm-x86/kvm_host.h @@ -0,0 +1,601 @@ +#/* + * Kernel-based Virtual Machine driver for Linux + * + * This header defines architecture specific interfaces, x86 version + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#ifndef ASM_KVM_HOST_H +#define ASM_KVM_HOST_H + +#include +#include + +#include +#include +#include + +#include + +#define CR3_PAE_RESERVED_BITS ((X86_CR3_PWT | X86_CR3_PCD) - 1) +#define CR3_NONPAE_RESERVED_BITS ((PAGE_SIZE-1) & ~(X86_CR3_PWT | X86_CR3_PCD)) +#define CR3_L_MODE_RESERVED_BITS (CR3_NONPAE_RESERVED_BITS|0xFFFFFF0000000000ULL) + +#define KVM_GUEST_CR0_MASK \ + (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE \ + | X86_CR0_NW | X86_CR0_CD) +#define KVM_VM_CR0_ALWAYS_ON \ + (X86_CR0_PG | X86_CR0_PE | X86_CR0_WP | X86_CR0_NE | X86_CR0_TS \ + | X86_CR0_MP) +#define KVM_GUEST_CR4_MASK \ + (X86_CR4_VME | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_PGE | X86_CR4_VMXE) +#define KVM_PMODE_VM_CR4_ALWAYS_ON (X86_CR4_PAE | X86_CR4_VMXE) +#define KVM_RMODE_VM_CR4_ALWAYS_ON (X86_CR4_VME | X86_CR4_PAE | X86_CR4_VMXE) + +#define INVALID_PAGE (~(hpa_t)0) +#define UNMAPPED_GVA (~(gpa_t)0) + +#define DE_VECTOR 0 +#define UD_VECTOR 6 +#define NM_VECTOR 7 +#define DF_VECTOR 8 +#define TS_VECTOR 10 +#define NP_VECTOR 11 +#define SS_VECTOR 12 +#define GP_VECTOR 13 +#define PF_VECTOR 14 + +#define SELECTOR_TI_MASK (1 << 2) +#define SELECTOR_RPL_MASK 0x03 + +#define IOPL_SHIFT 12 + +#define KVM_ALIAS_SLOTS 4 + +#define KVM_PERMILLE_MMU_PAGES 20 +#define KVM_MIN_ALLOC_MMU_PAGES 64 +#define KVM_NUM_MMU_PAGES 1024 +#define KVM_MIN_FREE_MMU_PAGES 5 +#define KVM_REFILL_PAGES 25 +#define KVM_MAX_CPUID_ENTRIES 40 + +extern spinlock_t kvm_lock; +extern struct list_head vm_list; + +struct kvm_vcpu; +struct kvm; + +enum { + VCPU_REGS_RAX = 0, + VCPU_REGS_RCX = 1, + VCPU_REGS_RDX = 2, + VCPU_REGS_RBX = 3, + VCPU_REGS_RSP = 4, + VCPU_REGS_RBP = 5, + VCPU_REGS_RSI = 6, + VCPU_REGS_RDI = 7, +#ifdef CONFIG_X86_64 + VCPU_REGS_R8 = 8, + VCPU_REGS_R9 = 9, + VCPU_REGS_R10 = 10, + VCPU_REGS_R11 = 11, + VCPU_REGS_R12 = 12, + VCPU_REGS_R13 = 13, + VCPU_REGS_R14 = 14, + VCPU_REGS_R15 = 15, +#endif + NR_VCPU_REGS +}; + +enum { + VCPU_SREG_CS, + VCPU_SREG_DS, + VCPU_SREG_ES, + VCPU_SREG_FS, + VCPU_SREG_GS, + VCPU_SREG_SS, + VCPU_SREG_TR, + VCPU_SREG_LDTR, +}; + +#include + +#define KVM_NR_MEM_OBJS 40 + +/* + * We don't want allocation failures within the mmu code, so we preallocate + * enough memory for a single page fault in a cache. + */ +struct kvm_mmu_memory_cache { + int nobjs; + void *objects[KVM_NR_MEM_OBJS]; +}; + +#define NR_PTE_CHAIN_ENTRIES 5 + +struct kvm_pte_chain { + u64 *parent_ptes[NR_PTE_CHAIN_ENTRIES]; + struct hlist_node link; +}; + +/* + * kvm_mmu_page_role, below, is defined as: + * + * bits 0:3 - total guest paging levels (2-4, or zero for real mode) + * bits 4:7 - page table level for this shadow (1-4) + * bits 8:9 - page table quadrant for 2-level guests + * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) + * bits 17:19 - common access permissions for all ptes in this shadow page + */ +union kvm_mmu_page_role { + unsigned word; + struct { + unsigned glevels : 4; + unsigned level : 4; + unsigned quadrant : 2; + unsigned pad_for_nice_hex_output : 6; + unsigned metaphysical : 1; + unsigned access : 3; + }; +}; + +struct kvm_mmu_page { + struct list_head link; + struct hlist_node hash_link; + + /* + * The following two entries are used to key the shadow page in the + * hash table. + */ + gfn_t gfn; + union kvm_mmu_page_role role; + + u64 *spt; + /* hold the gfn of each spte inside spt */ + gfn_t *gfns; + unsigned long slot_bitmap; /* One bit set per slot which has memory + * in this shadow page. + */ + int multimapped; /* More than one parent_pte? */ + int root_count; /* Currently serving as active root */ + union { + u64 *parent_pte; /* !multimapped */ + struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */ + }; +}; + +/* + * x86 supports 3 paging modes (4-level 64-bit, 3-level 64-bit, and 2-level + * 32-bit). The kvm_mmu structure abstracts the details of the current mmu + * mode. + */ +struct kvm_mmu { + void (*new_cr3)(struct kvm_vcpu *vcpu); + int (*page_fault)(struct kvm_vcpu *vcpu, gva_t gva, u32 err); + void (*free)(struct kvm_vcpu *vcpu); + gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, gva_t gva); + void (*prefetch_page)(struct kvm_vcpu *vcpu, + struct kvm_mmu_page *page); + hpa_t root_hpa; + int root_level; + int shadow_root_level; + + u64 *pae_root; +}; + +struct kvm_vcpu_arch { + u64 host_tsc; + int interrupt_window_open; + unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */ + DECLARE_BITMAP(irq_pending, KVM_NR_INTERRUPTS); + unsigned long regs[NR_VCPU_REGS]; /* for rsp: vcpu_load_rsp_rip() */ + unsigned long rip; /* needs vcpu_load_rsp_rip() */ + + unsigned long cr0; + unsigned long cr2; + unsigned long cr3; + unsigned long cr4; + unsigned long cr8; + u64 pdptrs[4]; /* pae */ + u64 shadow_efer; + u64 apic_base; + struct kvm_lapic *apic; /* kernel irqchip context */ +#define VCPU_MP_STATE_RUNNABLE 0 +#define VCPU_MP_STATE_UNINITIALIZED 1 +#define VCPU_MP_STATE_INIT_RECEIVED 2 +#define VCPU_MP_STATE_SIPI_RECEIVED 3 +#define VCPU_MP_STATE_HALTED 4 + int mp_state; + int sipi_vector; + u64 ia32_misc_enable_msr; + + struct kvm_mmu mmu; + + struct kvm_mmu_memory_cache mmu_pte_chain_cache; + struct kvm_mmu_memory_cache mmu_rmap_desc_cache; + struct kvm_mmu_memory_cache mmu_page_cache; + struct kvm_mmu_memory_cache mmu_page_header_cache; + + gfn_t last_pt_write_gfn; + int last_pt_write_count; + u64 *last_pte_updated; + + struct i387_fxsave_struct host_fx_image; + struct i387_fxsave_struct guest_fx_image; + + gva_t mmio_fault_cr2; + struct kvm_pio_request pio; + void *pio_data; + + struct kvm_queued_exception { + bool pending; + bool has_error_code; + u8 nr; + u32 error_code; + } exception; + + struct { + int active; + u8 save_iopl; + struct kvm_save_segment { + u16 selector; + unsigned long base; + u32 limit; + u32 ar; + } tr, es, ds, fs, gs; + } rmode; + int halt_request; /* real mode on Intel only */ + + int cpuid_nent; + struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; + /* emulate context */ + + struct x86_emulate_ctxt emulate_ctxt; +}; + +struct kvm_mem_alias { + gfn_t base_gfn; + unsigned long npages; + gfn_t target_gfn; +}; + +struct kvm_arch{ + int naliases; + struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS]; + + unsigned int n_free_mmu_pages; + unsigned int n_requested_mmu_pages; + unsigned int n_alloc_mmu_pages; + struct hlist_head mmu_page_hash[KVM_NUM_MMU_PAGES]; + /* + * Hash table of struct kvm_mmu_page. + */ + struct list_head active_mmu_pages; + struct kvm_pic *vpic; + struct kvm_ioapic *vioapic; + + int round_robin_prev_vcpu; + unsigned int tss_addr; + struct page *apic_access_page; +}; + +struct kvm_vm_stat { + u32 mmu_shadow_zapped; + u32 mmu_pte_write; + u32 mmu_pte_updated; + u32 mmu_pde_zapped; + u32 mmu_flooded; + u32 mmu_recycled; + u32 remote_tlb_flush; +}; + +struct kvm_vcpu_stat { + u32 pf_fixed; + u32 pf_guest; + u32 tlb_flush; + u32 invlpg; + + u32 exits; + u32 io_exits; + u32 mmio_exits; + u32 signal_exits; + u32 irq_window_exits; + u32 halt_exits; + u32 halt_wakeup; + u32 request_irq_exits; + u32 irq_exits; + u32 host_state_reload; + u32 efer_reload; + u32 fpu_reload; + u32 insn_emulation; + u32 insn_emulation_fail; +}; + +struct descriptor_table { + u16 limit; + unsigned long base; +} __attribute__((packed)); + +struct kvm_x86_ops { + int (*cpu_has_kvm_support)(void); /* __init */ + int (*disabled_by_bios)(void); /* __init */ + void (*hardware_enable)(void *dummy); /* __init */ + void (*hardware_disable)(void *dummy); + void (*check_processor_compatibility)(void *rtn); + int (*hardware_setup)(void); /* __init */ + void (*hardware_unsetup)(void); /* __exit */ + + /* Create, but do not attach this VCPU */ + struct kvm_vcpu *(*vcpu_create)(struct kvm *kvm, unsigned id); + void (*vcpu_free)(struct kvm_vcpu *vcpu); + int (*vcpu_reset)(struct kvm_vcpu *vcpu); + + void (*prepare_guest_switch)(struct kvm_vcpu *vcpu); + void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); + void (*vcpu_put)(struct kvm_vcpu *vcpu); + void (*vcpu_decache)(struct kvm_vcpu *vcpu); + + int (*set_guest_debug)(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg); + void (*guest_debug_pre)(struct kvm_vcpu *vcpu); + int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); + int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); + u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); + void (*get_segment)(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg); + void (*set_segment)(struct kvm_vcpu *vcpu, + struct kvm_segment *var, int seg); + void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); + void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); + void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); + void (*set_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); + void (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); + void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); + void (*get_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*set_idt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*get_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + void (*set_gdt)(struct kvm_vcpu *vcpu, struct descriptor_table *dt); + unsigned long (*get_dr)(struct kvm_vcpu *vcpu, int dr); + void (*set_dr)(struct kvm_vcpu *vcpu, int dr, unsigned long value, + int *exception); + void (*cache_regs)(struct kvm_vcpu *vcpu); + void (*decache_regs)(struct kvm_vcpu *vcpu); + unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); + void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); + + void (*tlb_flush)(struct kvm_vcpu *vcpu); + + void (*run)(struct kvm_vcpu *vcpu, struct kvm_run *run); + int (*handle_exit)(struct kvm_run *run, struct kvm_vcpu *vcpu); + void (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); + void (*patch_hypercall)(struct kvm_vcpu *vcpu, + unsigned char *hypercall_addr); + int (*get_irq)(struct kvm_vcpu *vcpu); + void (*set_irq)(struct kvm_vcpu *vcpu, int vec); + void (*queue_exception)(struct kvm_vcpu *vcpu, unsigned nr, + bool has_error_code, u32 error_code); + bool (*exception_injected)(struct kvm_vcpu *vcpu); + void (*inject_pending_irq)(struct kvm_vcpu *vcpu); + void (*inject_pending_vectors)(struct kvm_vcpu *vcpu, + struct kvm_run *run); + + int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); +}; + +extern struct kvm_x86_ops *kvm_x86_ops; + +int kvm_mmu_module_init(void); +void kvm_mmu_module_exit(void); + +void kvm_mmu_destroy(struct kvm_vcpu *vcpu); +int kvm_mmu_create(struct kvm_vcpu *vcpu); +int kvm_mmu_setup(struct kvm_vcpu *vcpu); +void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte); + +int kvm_mmu_reset_context(struct kvm_vcpu *vcpu); +void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot); +void kvm_mmu_zap_all(struct kvm *kvm); +unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm); +void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages); + +enum emulation_result { + EMULATE_DONE, /* no further processing */ + EMULATE_DO_MMIO, /* kvm_run filled with mmio request */ + EMULATE_FAIL, /* can't emulate this instruction */ +}; + +int emulate_instruction(struct kvm_vcpu *vcpu, struct kvm_run *run, + unsigned long cr2, u16 error_code, int no_decode); +void kvm_report_emulation_failure(struct kvm_vcpu *cvpu, const char *context); +void realmode_lgdt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); +void realmode_lidt(struct kvm_vcpu *vcpu, u16 size, unsigned long address); +void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw, + unsigned long *rflags); + +unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr); +void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value, + unsigned long *rflags); +int kvm_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *data); +int kvm_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); + +struct x86_emulate_ctxt; + +int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned port); +int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, + int size, unsigned long count, int down, + gva_t address, int rep, unsigned port); +void kvm_emulate_cpuid(struct kvm_vcpu *vcpu); +int kvm_emulate_halt(struct kvm_vcpu *vcpu); +int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address); +int emulate_clts(struct kvm_vcpu *vcpu); +int emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr, + unsigned long *dest); +int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, + unsigned long value); + +void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr0); +void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr0); +unsigned long get_cr8(struct kvm_vcpu *vcpu); +void lmsw(struct kvm_vcpu *vcpu, unsigned long msw); +void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l); + +int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); +int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data); + +void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr); +void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); +void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2, + u32 error_code); + +void fx_init(struct kvm_vcpu *vcpu); + +int emulator_read_std(unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); +int emulator_write_emulated(unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); + +unsigned long segment_base(u16 selector); + +void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu); +void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, + const u8 *new, int bytes); +int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); +void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); +int kvm_mmu_load(struct kvm_vcpu *vcpu); +void kvm_mmu_unload(struct kvm_vcpu *vcpu); + +int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); + +int kvm_fix_hypercall(struct kvm_vcpu *vcpu); + +int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code); + +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); +int complete_pio(struct kvm_vcpu *vcpu); + +static inline struct kvm_mmu_page *page_header(hpa_t shadow_page) +{ + struct page *page = pfn_to_page(shadow_page >> PAGE_SHIFT); + + return (struct kvm_mmu_page *)page_private(page); +} + +static inline u16 read_fs(void) +{ + u16 seg; + asm("mov %%fs, %0" : "=g"(seg)); + return seg; +} + +static inline u16 read_gs(void) +{ + u16 seg; + asm("mov %%gs, %0" : "=g"(seg)); + return seg; +} + +static inline u16 read_ldt(void) +{ + u16 ldt; + asm("sldt %0" : "=g"(ldt)); + return ldt; +} + +static inline void load_fs(u16 sel) +{ + asm("mov %0, %%fs" : : "rm"(sel)); +} + +static inline void load_gs(u16 sel) +{ + asm("mov %0, %%gs" : : "rm"(sel)); +} + +#ifndef load_ldt +static inline void load_ldt(u16 sel) +{ + asm("lldt %0" : : "rm"(sel)); +} +#endif + +static inline void get_idt(struct descriptor_table *table) +{ + asm("sidt %0" : "=m"(*table)); +} + +static inline void get_gdt(struct descriptor_table *table) +{ + asm("sgdt %0" : "=m"(*table)); +} + +static inline unsigned long read_tr_base(void) +{ + u16 tr; + asm("str %0" : "=g"(tr)); + return segment_base(tr); +} + +#ifdef CONFIG_X86_64 +static inline unsigned long read_msr(unsigned long msr) +{ + u64 value; + + rdmsrl(msr, value); + return value; +} +#endif + +static inline void fx_save(struct i387_fxsave_struct *image) +{ + asm("fxsave (%0)":: "r" (image)); +} + +static inline void fx_restore(struct i387_fxsave_struct *image) +{ + asm("fxrstor (%0)":: "r" (image)); +} + +static inline void fpu_init(void) +{ + asm("finit"); +} + +static inline u32 get_rdx_init_val(void) +{ + return 0x600; /* P6 family */ +} + +static inline void kvm_inject_gp(struct kvm_vcpu *vcpu, u32 error_code) +{ + kvm_queue_exception_e(vcpu, GP_VECTOR, error_code); +} + +#define ASM_VMX_VMCLEAR_RAX ".byte 0x66, 0x0f, 0xc7, 0x30" +#define ASM_VMX_VMLAUNCH ".byte 0x0f, 0x01, 0xc2" +#define ASM_VMX_VMRESUME ".byte 0x0f, 0x01, 0xc3" +#define ASM_VMX_VMPTRLD_RAX ".byte 0x0f, 0xc7, 0x30" +#define ASM_VMX_VMREAD_RDX_RAX ".byte 0x0f, 0x78, 0xd0" +#define ASM_VMX_VMWRITE_RAX_RDX ".byte 0x0f, 0x79, 0xd0" +#define ASM_VMX_VMWRITE_RSP_RDX ".byte 0x0f, 0x79, 0xd4" +#define ASM_VMX_VMXOFF ".byte 0x0f, 0x01, 0xc4" +#define ASM_VMX_VMXON_RAX ".byte 0xf3, 0x0f, 0xc7, 0x30" + +#define MSR_IA32_TIME_STAMP_COUNTER 0x010 + +#define TSS_IOPB_BASE_OFFSET 0x66 +#define TSS_BASE_SIZE 0x68 +#define TSS_IOPB_SIZE (65536 / 8) +#define TSS_REDIRECTION_SIZE (256 / 8) +#define RMODE_TSS_SIZE (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) + +#endif diff --git a/include/asm-x86/kvm_x86_emulate.h b/include/asm-x86/kvm_x86_emulate.h new file mode 100644 index 000000000000..7db91b9bdcd4 --- /dev/null +++ b/include/asm-x86/kvm_x86_emulate.h @@ -0,0 +1,186 @@ +/****************************************************************************** + * x86_emulate.h + * + * Generic x86 (32-bit and 64-bit) instruction decoder and emulator. + * + * Copyright (c) 2005 Keir Fraser + * + * From: xen-unstable 10676:af9809f51f81a3c43f276f00c81a52ef558afda4 + */ + +#ifndef __X86_EMULATE_H__ +#define __X86_EMULATE_H__ + +struct x86_emulate_ctxt; + +/* + * x86_emulate_ops: + * + * These operations represent the instruction emulator's interface to memory. + * There are two categories of operation: those that act on ordinary memory + * regions (*_std), and those that act on memory regions known to require + * special treatment or emulation (*_emulated). + * + * The emulator assumes that an instruction accesses only one 'emulated memory' + * location, that this location is the given linear faulting address (cr2), and + * that this is one of the instruction's data operands. Instruction fetches and + * stack operations are assumed never to access emulated memory. The emulator + * automatically deduces which operand of a string-move operation is accessing + * emulated memory, and assumes that the other operand accesses normal memory. + * + * NOTES: + * 1. The emulator isn't very smart about emulated vs. standard memory. + * 'Emulated memory' access addresses should be checked for sanity. + * 'Normal memory' accesses may fault, and the caller must arrange to + * detect and handle reentrancy into the emulator via recursive faults. + * Accesses may be unaligned and may cross page boundaries. + * 2. If the access fails (cannot emulate, or a standard access faults) then + * it is up to the memop to propagate the fault to the guest VM via + * some out-of-band mechanism, unknown to the emulator. The memop signals + * failure by returning X86EMUL_PROPAGATE_FAULT to the emulator, which will + * then immediately bail. + * 3. Valid access sizes are 1, 2, 4 and 8 bytes. On x86/32 systems only + * cmpxchg8b_emulated need support 8-byte accesses. + * 4. The emulator cannot handle 64-bit mode emulation on an x86/32 system. + */ +/* Access completed successfully: continue emulation as normal. */ +#define X86EMUL_CONTINUE 0 +/* Access is unhandleable: bail from emulation and return error to caller. */ +#define X86EMUL_UNHANDLEABLE 1 +/* Terminate emulation but return success to the caller. */ +#define X86EMUL_PROPAGATE_FAULT 2 /* propagate a generated fault to guest */ +#define X86EMUL_RETRY_INSTR 2 /* retry the instruction for some reason */ +#define X86EMUL_CMPXCHG_FAILED 2 /* cmpxchg did not see expected value */ +struct x86_emulate_ops { + /* + * read_std: Read bytes of standard (non-emulated/special) memory. + * Used for instruction fetch, stack operations, and others. + * @addr: [IN ] Linear address from which to read. + * @val: [OUT] Value read from memory, zero-extended to 'u_long'. + * @bytes: [IN ] Number of bytes to read from memory. + */ + int (*read_std)(unsigned long addr, void *val, + unsigned int bytes, struct kvm_vcpu *vcpu); + + /* + * read_emulated: Read bytes from emulated/special memory area. + * @addr: [IN ] Linear address from which to read. + * @val: [OUT] Value read from memory, zero-extended to 'u_long'. + * @bytes: [IN ] Number of bytes to read from memory. + */ + int (*read_emulated) (unsigned long addr, + void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); + + /* + * write_emulated: Read bytes from emulated/special memory area. + * @addr: [IN ] Linear address to which to write. + * @val: [IN ] Value to write to memory (low-order bytes used as + * required). + * @bytes: [IN ] Number of bytes to write to memory. + */ + int (*write_emulated) (unsigned long addr, + const void *val, + unsigned int bytes, + struct kvm_vcpu *vcpu); + + /* + * cmpxchg_emulated: Emulate an atomic (LOCKed) CMPXCHG operation on an + * emulated/special memory area. + * @addr: [IN ] Linear address to access. + * @old: [IN ] Value expected to be current at @addr. + * @new: [IN ] Value to write to @addr. + * @bytes: [IN ] Number of bytes to access using CMPXCHG. + */ + int (*cmpxchg_emulated) (unsigned long addr, + const void *old, + const void *new, + unsigned int bytes, + struct kvm_vcpu *vcpu); + +}; + +/* Type, address-of, and value of an instruction's operand. */ +struct operand { + enum { OP_REG, OP_MEM, OP_IMM, OP_NONE } type; + unsigned int bytes; + unsigned long val, orig_val, *ptr; +}; + +struct fetch_cache { + u8 data[15]; + unsigned long start; + unsigned long end; +}; + +struct decode_cache { + u8 twobyte; + u8 b; + u8 lock_prefix; + u8 rep_prefix; + u8 op_bytes; + u8 ad_bytes; + u8 rex_prefix; + struct operand src; + struct operand dst; + unsigned long *override_base; + unsigned int d; + unsigned long regs[NR_VCPU_REGS]; + unsigned long eip; + /* modrm */ + u8 modrm; + u8 modrm_mod; + u8 modrm_reg; + u8 modrm_rm; + u8 use_modrm_ea; + unsigned long modrm_ea; + unsigned long modrm_val; + struct fetch_cache fetch; +}; + +struct x86_emulate_ctxt { + /* Register state before/after emulation. */ + struct kvm_vcpu *vcpu; + + /* Linear faulting address (if emulating a page-faulting instruction). */ + unsigned long eflags; + + /* Emulated execution mode, represented by an X86EMUL_MODE value. */ + int mode; + + unsigned long cs_base; + unsigned long ds_base; + unsigned long es_base; + unsigned long ss_base; + unsigned long gs_base; + unsigned long fs_base; + + /* decode cache */ + + struct decode_cache decode; +}; + +/* Repeat String Operation Prefix */ +#define REPE_PREFIX 1 +#define REPNE_PREFIX 2 + +/* Execution mode, passed to the emulator. */ +#define X86EMUL_MODE_REAL 0 /* Real mode. */ +#define X86EMUL_MODE_PROT16 2 /* 16-bit protected mode. */ +#define X86EMUL_MODE_PROT32 4 /* 32-bit protected mode. */ +#define X86EMUL_MODE_PROT64 8 /* 64-bit (long) mode. */ + +/* Host execution mode. */ +#if defined(__i386__) +#define X86EMUL_MODE_HOST X86EMUL_MODE_PROT32 +#elif defined(CONFIG_X86_64) +#define X86EMUL_MODE_HOST X86EMUL_MODE_PROT64 +#endif + +int x86_decode_insn(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops); +int x86_emulate_insn(struct x86_emulate_ctxt *ctxt, + struct x86_emulate_ops *ops); + +#endif /* __X86_EMULATE_H__ */ diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h new file mode 100644 index 000000000000..a85d5b6943de --- /dev/null +++ b/include/linux/kvm_host.h @@ -0,0 +1,289 @@ +#ifndef __KVM_HOST_H +#define __KVM_HOST_H + +/* + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + +#define KVM_MAX_VCPUS 4 +#define KVM_MEMORY_SLOTS 8 +/* memory slots that does not exposed to userspace */ +#define KVM_PRIVATE_MEM_SLOTS 4 + +#define KVM_PIO_PAGE_OFFSET 1 + +/* + * vcpu->requests bit members + */ +#define KVM_REQ_TLB_FLUSH 0 + + +struct kvm_vcpu; +extern struct kmem_cache *kvm_vcpu_cache; + +struct kvm_guest_debug { + int enabled; + unsigned long bp[4]; + int singlestep; +}; + +/* + * It would be nice to use something smarter than a linear search, TBD... + * Thankfully we dont expect many devices to register (famous last words :), + * so until then it will suffice. At least its abstracted so we can change + * in one place. + */ +struct kvm_io_bus { + int dev_count; +#define NR_IOBUS_DEVS 6 + struct kvm_io_device *devs[NR_IOBUS_DEVS]; +}; + +void kvm_io_bus_init(struct kvm_io_bus *bus); +void kvm_io_bus_destroy(struct kvm_io_bus *bus); +struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr); +void kvm_io_bus_register_dev(struct kvm_io_bus *bus, + struct kvm_io_device *dev); + +struct kvm_vcpu { + struct kvm *kvm; + struct preempt_notifier preempt_notifier; + int vcpu_id; + struct mutex mutex; + int cpu; + struct kvm_run *run; + int guest_mode; + unsigned long requests; + struct kvm_guest_debug guest_debug; + int fpu_active; + int guest_fpu_loaded; + wait_queue_head_t wq; + int sigset_active; + sigset_t sigset; + struct kvm_vcpu_stat stat; + +#ifdef CONFIG_HAS_IOMEM + int mmio_needed; + int mmio_read_completed; + int mmio_is_write; + int mmio_size; + unsigned char mmio_data[8]; + gpa_t mmio_phys_addr; +#endif + + struct kvm_vcpu_arch arch; +}; + +struct kvm_memory_slot { + gfn_t base_gfn; + unsigned long npages; + unsigned long flags; + unsigned long *rmap; + unsigned long *dirty_bitmap; + unsigned long userspace_addr; + int user_alloc; +}; + +struct kvm { + struct mutex lock; /* protects everything except vcpus */ + struct mm_struct *mm; /* userspace tied to this vm */ + int nmemslots; + struct kvm_memory_slot memslots[KVM_MEMORY_SLOTS + + KVM_PRIVATE_MEM_SLOTS]; + struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; + struct list_head vm_list; + struct file *filp; + struct kvm_io_bus mmio_bus; + struct kvm_io_bus pio_bus; + struct kvm_vm_stat stat; + struct kvm_arch arch; +}; + +/* The guest did something we don't support. */ +#define pr_unimpl(vcpu, fmt, ...) \ + do { \ + if (printk_ratelimit()) \ + printk(KERN_ERR "kvm: %i: cpu%i " fmt, \ + current->tgid, (vcpu)->vcpu_id , ## __VA_ARGS__); \ + } while (0) + +#define kvm_printf(kvm, fmt ...) printk(KERN_DEBUG fmt) +#define vcpu_printf(vcpu, fmt...) kvm_printf(vcpu->kvm, fmt) + +int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id); +void kvm_vcpu_uninit(struct kvm_vcpu *vcpu); + +void vcpu_load(struct kvm_vcpu *vcpu); +void vcpu_put(struct kvm_vcpu *vcpu); + +void decache_vcpus_on_cpu(int cpu); + + +int kvm_init(void *opaque, unsigned int vcpu_size, + struct module *module); +void kvm_exit(void); + +#define HPA_MSB ((sizeof(hpa_t) * 8) - 1) +#define HPA_ERR_MASK ((hpa_t)1 << HPA_MSB) +static inline int is_error_hpa(hpa_t hpa) { return hpa >> HPA_MSB; } +struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva); + +extern struct page *bad_page; + +int is_error_page(struct page *page); +int kvm_is_error_hva(unsigned long addr); +int kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc); +int __kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc); +int kvm_arch_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + struct kvm_memory_slot old, + int user_alloc); +gfn_t unalias_gfn(struct kvm *kvm, gfn_t gfn); +struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn); +void kvm_release_page_clean(struct page *page); +void kvm_release_page_dirty(struct page *page); +int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, + int len); +int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); +int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, + int offset, int len); +int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, + unsigned long len); +int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len); +int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len); +struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn); +int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn); +void mark_page_dirty(struct kvm *kvm, gfn_t gfn); + +void kvm_vcpu_block(struct kvm_vcpu *vcpu); +void kvm_resched(struct kvm_vcpu *vcpu); +void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); +void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); +void kvm_flush_remote_tlbs(struct kvm *kvm); + +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +long kvm_arch_vcpu_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); + +int kvm_dev_ioctl_check_extension(long ext); + +int kvm_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log, int *is_dirty); +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log); + +int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, + struct + kvm_userspace_memory_region *mem, + int user_alloc); +long kvm_arch_vm_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg); +void kvm_arch_destroy_vm(struct kvm *kvm); + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu); + +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr); + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs); +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs); +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs); +int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, + struct kvm_debug_guest *dbg); +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); + +int kvm_arch_init(void *opaque); +void kvm_arch_exit(void); + +int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu); + +void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu); +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu); +struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id); +int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu); +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); + +int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu); +void kvm_arch_hardware_enable(void *garbage); +void kvm_arch_hardware_disable(void *garbage); +int kvm_arch_hardware_setup(void); +void kvm_arch_hardware_unsetup(void); +void kvm_arch_check_processor_compat(void *rtn); +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu); + +void kvm_free_physmem(struct kvm *kvm); + +struct kvm *kvm_arch_create_vm(void); +void kvm_arch_destroy_vm(struct kvm *kvm); + +int kvm_cpu_get_interrupt(struct kvm_vcpu *v); +int kvm_cpu_has_interrupt(struct kvm_vcpu *v); + +static inline void kvm_guest_enter(void) +{ + account_system_vtime(current); + current->flags |= PF_VCPU; +} + +static inline void kvm_guest_exit(void) +{ + account_system_vtime(current); + current->flags &= ~PF_VCPU; +} + +static inline int memslot_id(struct kvm *kvm, struct kvm_memory_slot *slot) +{ + return slot - kvm->memslots; +} + +static inline gpa_t gfn_to_gpa(gfn_t gfn) +{ + return (gpa_t)gfn << PAGE_SHIFT; +} + +enum kvm_stat_kind { + KVM_STAT_VM, + KVM_STAT_VCPU, +}; + +struct kvm_stats_debugfs_item { + const char *name; + int offset; + enum kvm_stat_kind kind; + struct dentry *dentry; +}; +extern struct kvm_stats_debugfs_item debugfs_entries[]; + +#endif diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h new file mode 100644 index 000000000000..1c4e46decb22 --- /dev/null +++ b/include/linux/kvm_types.h @@ -0,0 +1,54 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ + +#ifndef __KVM_TYPES_H__ +#define __KVM_TYPES_H__ + +#include + +/* + * Address types: + * + * gva - guest virtual address + * gpa - guest physical address + * gfn - guest frame number + * hva - host virtual address + * hpa - host physical address + * hfn - host frame number + */ + +typedef unsigned long gva_t; +typedef u64 gpa_t; +typedef unsigned long gfn_t; + +typedef unsigned long hva_t; +typedef u64 hpa_t; +typedef unsigned long hfn_t; + +struct kvm_pio_request { + unsigned long count; + int cur_count; + struct page *guest_pages[2]; + unsigned guest_page_offset; + int in; + int port; + int size; + int string; + int down; + int rep; +}; + +#endif /* __KVM_TYPES_H__ */ -- cgit v1.2.3 From 0fce5623ba248d3af0d7fb719d5ac367cc9d5192 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sun, 16 Dec 2007 11:13:16 +0200 Subject: KVM: Move drivers/kvm/* to virt/kvm/ Signed-off-by: Avi Kivity --- arch/x86/kvm/Makefile | 4 +- drivers/kvm/iodev.h | 63 --- drivers/kvm/kvm_main.c | 1393 ------------------------------------------------ virt/kvm/iodev.h | 63 +++ virt/kvm/kvm_main.c | 1393 ++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 1458 insertions(+), 1458 deletions(-) delete mode 100644 drivers/kvm/iodev.h delete mode 100644 drivers/kvm/kvm_main.c create mode 100644 virt/kvm/iodev.h create mode 100644 virt/kvm/kvm_main.c (limited to 'drivers') diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 880ffe403b35..af41534af496 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -2,9 +2,9 @@ # Makefile for Kernel-based Virtual Machine module # -common-objs = $(addprefix ../../../drivers/kvm/, kvm_main.o) +common-objs = $(addprefix ../../../virt/kvm/, kvm_main.o) -EXTRA_CFLAGS += -I drivers/kvm +EXTRA_CFLAGS += -I virt/kvm kvm-objs := $(common-objs) x86.o mmu.o x86_emulate.o i8259.o irq.o lapic.o \ ioapic.o diff --git a/drivers/kvm/iodev.h b/drivers/kvm/iodev.h deleted file mode 100644 index c14e642027b2..000000000000 --- a/drivers/kvm/iodev.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#ifndef __KVM_IODEV_H__ -#define __KVM_IODEV_H__ - -#include - -struct kvm_io_device { - void (*read)(struct kvm_io_device *this, - gpa_t addr, - int len, - void *val); - void (*write)(struct kvm_io_device *this, - gpa_t addr, - int len, - const void *val); - int (*in_range)(struct kvm_io_device *this, gpa_t addr); - void (*destructor)(struct kvm_io_device *this); - - void *private; -}; - -static inline void kvm_iodevice_read(struct kvm_io_device *dev, - gpa_t addr, - int len, - void *val) -{ - dev->read(dev, addr, len, val); -} - -static inline void kvm_iodevice_write(struct kvm_io_device *dev, - gpa_t addr, - int len, - const void *val) -{ - dev->write(dev, addr, len, val); -} - -static inline int kvm_iodevice_inrange(struct kvm_io_device *dev, gpa_t addr) -{ - return dev->in_range(dev, addr); -} - -static inline void kvm_iodevice_destructor(struct kvm_io_device *dev) -{ - if (dev->destructor) - dev->destructor(dev); -} - -#endif /* __KVM_IODEV_H__ */ diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c deleted file mode 100644 index 4026d7d64296..000000000000 --- a/drivers/kvm/kvm_main.c +++ /dev/null @@ -1,1393 +0,0 @@ -/* - * Kernel-based Virtual Machine driver for Linux - * - * This module enables machines with Intel VT-x extensions to run virtual - * machines without emulation or binary translation. - * - * Copyright (C) 2006 Qumranet, Inc. - * - * Authors: - * Avi Kivity - * Yaniv Kamay - * - * This work is licensed under the terms of the GNU GPL, version 2. See - * the COPYING file in the top-level directory. - * - */ - -#include "iodev.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -MODULE_AUTHOR("Qumranet"); -MODULE_LICENSE("GPL"); - -DEFINE_SPINLOCK(kvm_lock); -LIST_HEAD(vm_list); - -static cpumask_t cpus_hardware_enabled; - -struct kmem_cache *kvm_vcpu_cache; -EXPORT_SYMBOL_GPL(kvm_vcpu_cache); - -static __read_mostly struct preempt_ops kvm_preempt_ops; - -static struct dentry *debugfs_dir; - -static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, - unsigned long arg); - -static inline int valid_vcpu(int n) -{ - return likely(n >= 0 && n < KVM_MAX_VCPUS); -} - -/* - * Switches to specified vcpu, until a matching vcpu_put() - */ -void vcpu_load(struct kvm_vcpu *vcpu) -{ - int cpu; - - mutex_lock(&vcpu->mutex); - cpu = get_cpu(); - preempt_notifier_register(&vcpu->preempt_notifier); - kvm_arch_vcpu_load(vcpu, cpu); - put_cpu(); -} - -void vcpu_put(struct kvm_vcpu *vcpu) -{ - preempt_disable(); - kvm_arch_vcpu_put(vcpu); - preempt_notifier_unregister(&vcpu->preempt_notifier); - preempt_enable(); - mutex_unlock(&vcpu->mutex); -} - -static void ack_flush(void *_completed) -{ -} - -void kvm_flush_remote_tlbs(struct kvm *kvm) -{ - int i, cpu; - cpumask_t cpus; - struct kvm_vcpu *vcpu; - - cpus_clear(cpus); - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - vcpu = kvm->vcpus[i]; - if (!vcpu) - continue; - if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) - continue; - cpu = vcpu->cpu; - if (cpu != -1 && cpu != raw_smp_processor_id()) - cpu_set(cpu, cpus); - } - if (cpus_empty(cpus)) - return; - ++kvm->stat.remote_tlb_flush; - smp_call_function_mask(cpus, ack_flush, NULL, 1); -} - -int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) -{ - struct page *page; - int r; - - mutex_init(&vcpu->mutex); - vcpu->cpu = -1; - vcpu->kvm = kvm; - vcpu->vcpu_id = id; - init_waitqueue_head(&vcpu->wq); - - page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!page) { - r = -ENOMEM; - goto fail; - } - vcpu->run = page_address(page); - - r = kvm_arch_vcpu_init(vcpu); - if (r < 0) - goto fail_free_run; - return 0; - -fail_free_run: - free_page((unsigned long)vcpu->run); -fail: - return r; -} -EXPORT_SYMBOL_GPL(kvm_vcpu_init); - -void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) -{ - kvm_arch_vcpu_uninit(vcpu); - free_page((unsigned long)vcpu->run); -} -EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); - -static struct kvm *kvm_create_vm(void) -{ - struct kvm *kvm = kvm_arch_create_vm(); - - if (IS_ERR(kvm)) - goto out; - - kvm->mm = current->mm; - atomic_inc(&kvm->mm->mm_count); - kvm_io_bus_init(&kvm->pio_bus); - mutex_init(&kvm->lock); - kvm_io_bus_init(&kvm->mmio_bus); - spin_lock(&kvm_lock); - list_add(&kvm->vm_list, &vm_list); - spin_unlock(&kvm_lock); -out: - return kvm; -} - -/* - * Free any memory in @free but not in @dont. - */ -static void kvm_free_physmem_slot(struct kvm_memory_slot *free, - struct kvm_memory_slot *dont) -{ - if (!dont || free->rmap != dont->rmap) - vfree(free->rmap); - - if (!dont || free->dirty_bitmap != dont->dirty_bitmap) - vfree(free->dirty_bitmap); - - free->npages = 0; - free->dirty_bitmap = NULL; - free->rmap = NULL; -} - -void kvm_free_physmem(struct kvm *kvm) -{ - int i; - - for (i = 0; i < kvm->nmemslots; ++i) - kvm_free_physmem_slot(&kvm->memslots[i], NULL); -} - -static void kvm_destroy_vm(struct kvm *kvm) -{ - struct mm_struct *mm = kvm->mm; - - spin_lock(&kvm_lock); - list_del(&kvm->vm_list); - spin_unlock(&kvm_lock); - kvm_io_bus_destroy(&kvm->pio_bus); - kvm_io_bus_destroy(&kvm->mmio_bus); - kvm_arch_destroy_vm(kvm); - mmdrop(mm); -} - -static int kvm_vm_release(struct inode *inode, struct file *filp) -{ - struct kvm *kvm = filp->private_data; - - kvm_destroy_vm(kvm); - return 0; -} - -/* - * Allocate some memory and give it an address in the guest physical address - * space. - * - * Discontiguous memory is allowed, mostly for framebuffers. - * - * Must be called holding kvm->lock. - */ -int __kvm_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - int user_alloc) -{ - int r; - gfn_t base_gfn; - unsigned long npages; - unsigned long i; - struct kvm_memory_slot *memslot; - struct kvm_memory_slot old, new; - - r = -EINVAL; - /* General sanity checks */ - if (mem->memory_size & (PAGE_SIZE - 1)) - goto out; - if (mem->guest_phys_addr & (PAGE_SIZE - 1)) - goto out; - if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS) - goto out; - if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) - goto out; - - memslot = &kvm->memslots[mem->slot]; - base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; - npages = mem->memory_size >> PAGE_SHIFT; - - if (!npages) - mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; - - new = old = *memslot; - - new.base_gfn = base_gfn; - new.npages = npages; - new.flags = mem->flags; - - /* Disallow changing a memory slot's size. */ - r = -EINVAL; - if (npages && old.npages && npages != old.npages) - goto out_free; - - /* Check for overlaps */ - r = -EEXIST; - for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { - struct kvm_memory_slot *s = &kvm->memslots[i]; - - if (s == memslot) - continue; - if (!((base_gfn + npages <= s->base_gfn) || - (base_gfn >= s->base_gfn + s->npages))) - goto out_free; - } - - /* Free page dirty bitmap if unneeded */ - if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) - new.dirty_bitmap = NULL; - - r = -ENOMEM; - - /* Allocate if a slot is being created */ - if (npages && !new.rmap) { - new.rmap = vmalloc(npages * sizeof(struct page *)); - - if (!new.rmap) - goto out_free; - - memset(new.rmap, 0, npages * sizeof(*new.rmap)); - - new.user_alloc = user_alloc; - new.userspace_addr = mem->userspace_addr; - } - - /* Allocate page dirty bitmap if needed */ - if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { - unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8; - - new.dirty_bitmap = vmalloc(dirty_bytes); - if (!new.dirty_bitmap) - goto out_free; - memset(new.dirty_bitmap, 0, dirty_bytes); - } - - if (mem->slot >= kvm->nmemslots) - kvm->nmemslots = mem->slot + 1; - - *memslot = new; - - r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc); - if (r) { - *memslot = old; - goto out_free; - } - - kvm_free_physmem_slot(&old, &new); - return 0; - -out_free: - kvm_free_physmem_slot(&new, &old); -out: - return r; - -} -EXPORT_SYMBOL_GPL(__kvm_set_memory_region); - -int kvm_set_memory_region(struct kvm *kvm, - struct kvm_userspace_memory_region *mem, - int user_alloc) -{ - int r; - - mutex_lock(&kvm->lock); - r = __kvm_set_memory_region(kvm, mem, user_alloc); - mutex_unlock(&kvm->lock); - return r; -} -EXPORT_SYMBOL_GPL(kvm_set_memory_region); - -int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, - struct - kvm_userspace_memory_region *mem, - int user_alloc) -{ - if (mem->slot >= KVM_MEMORY_SLOTS) - return -EINVAL; - return kvm_set_memory_region(kvm, mem, user_alloc); -} - -int kvm_get_dirty_log(struct kvm *kvm, - struct kvm_dirty_log *log, int *is_dirty) -{ - struct kvm_memory_slot *memslot; - int r, i; - int n; - unsigned long any = 0; - - r = -EINVAL; - if (log->slot >= KVM_MEMORY_SLOTS) - goto out; - - memslot = &kvm->memslots[log->slot]; - r = -ENOENT; - if (!memslot->dirty_bitmap) - goto out; - - n = ALIGN(memslot->npages, BITS_PER_LONG) / 8; - - for (i = 0; !any && i < n/sizeof(long); ++i) - any = memslot->dirty_bitmap[i]; - - r = -EFAULT; - if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) - goto out; - - if (any) - *is_dirty = 1; - - r = 0; -out: - return r; -} - -int is_error_page(struct page *page) -{ - return page == bad_page; -} -EXPORT_SYMBOL_GPL(is_error_page); - -static inline unsigned long bad_hva(void) -{ - return PAGE_OFFSET; -} - -int kvm_is_error_hva(unsigned long addr) -{ - return addr == bad_hva(); -} -EXPORT_SYMBOL_GPL(kvm_is_error_hva); - -static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn) -{ - int i; - - for (i = 0; i < kvm->nmemslots; ++i) { - struct kvm_memory_slot *memslot = &kvm->memslots[i]; - - if (gfn >= memslot->base_gfn - && gfn < memslot->base_gfn + memslot->npages) - return memslot; - } - return NULL; -} - -struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) -{ - gfn = unalias_gfn(kvm, gfn); - return __gfn_to_memslot(kvm, gfn); -} - -int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) -{ - int i; - - gfn = unalias_gfn(kvm, gfn); - for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { - struct kvm_memory_slot *memslot = &kvm->memslots[i]; - - if (gfn >= memslot->base_gfn - && gfn < memslot->base_gfn + memslot->npages) - return 1; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); - -static unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) -{ - struct kvm_memory_slot *slot; - - gfn = unalias_gfn(kvm, gfn); - slot = __gfn_to_memslot(kvm, gfn); - if (!slot) - return bad_hva(); - return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE); -} - -/* - * Requires current->mm->mmap_sem to be held - */ -static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn) -{ - struct page *page[1]; - unsigned long addr; - int npages; - - might_sleep(); - - addr = gfn_to_hva(kvm, gfn); - if (kvm_is_error_hva(addr)) { - get_page(bad_page); - return bad_page; - } - - npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page, - NULL); - - if (npages != 1) { - get_page(bad_page); - return bad_page; - } - - return page[0]; -} - -struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) -{ - struct page *page; - - down_read(¤t->mm->mmap_sem); - page = __gfn_to_page(kvm, gfn); - up_read(¤t->mm->mmap_sem); - - return page; -} - -EXPORT_SYMBOL_GPL(gfn_to_page); - -void kvm_release_page_clean(struct page *page) -{ - put_page(page); -} -EXPORT_SYMBOL_GPL(kvm_release_page_clean); - -void kvm_release_page_dirty(struct page *page) -{ - if (!PageReserved(page)) - SetPageDirty(page); - put_page(page); -} -EXPORT_SYMBOL_GPL(kvm_release_page_dirty); - -static int next_segment(unsigned long len, int offset) -{ - if (len > PAGE_SIZE - offset) - return PAGE_SIZE - offset; - else - return len; -} - -int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, - int len) -{ - int r; - unsigned long addr; - - addr = gfn_to_hva(kvm, gfn); - if (kvm_is_error_hva(addr)) - return -EFAULT; - r = copy_from_user(data, (void __user *)addr + offset, len); - if (r) - return -EFAULT; - return 0; -} -EXPORT_SYMBOL_GPL(kvm_read_guest_page); - -int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) -{ - gfn_t gfn = gpa >> PAGE_SHIFT; - int seg; - int offset = offset_in_page(gpa); - int ret; - - while ((seg = next_segment(len, offset)) != 0) { - ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); - if (ret < 0) - return ret; - offset = 0; - len -= seg; - data += seg; - ++gfn; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_read_guest); - -int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, - int offset, int len) -{ - int r; - unsigned long addr; - - addr = gfn_to_hva(kvm, gfn); - if (kvm_is_error_hva(addr)) - return -EFAULT; - r = copy_to_user((void __user *)addr + offset, data, len); - if (r) - return -EFAULT; - mark_page_dirty(kvm, gfn); - return 0; -} -EXPORT_SYMBOL_GPL(kvm_write_guest_page); - -int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, - unsigned long len) -{ - gfn_t gfn = gpa >> PAGE_SHIFT; - int seg; - int offset = offset_in_page(gpa); - int ret; - - while ((seg = next_segment(len, offset)) != 0) { - ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); - if (ret < 0) - return ret; - offset = 0; - len -= seg; - data += seg; - ++gfn; - } - return 0; -} - -int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) -{ - return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len); -} -EXPORT_SYMBOL_GPL(kvm_clear_guest_page); - -int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) -{ - gfn_t gfn = gpa >> PAGE_SHIFT; - int seg; - int offset = offset_in_page(gpa); - int ret; - - while ((seg = next_segment(len, offset)) != 0) { - ret = kvm_clear_guest_page(kvm, gfn, offset, seg); - if (ret < 0) - return ret; - offset = 0; - len -= seg; - ++gfn; - } - return 0; -} -EXPORT_SYMBOL_GPL(kvm_clear_guest); - -void mark_page_dirty(struct kvm *kvm, gfn_t gfn) -{ - struct kvm_memory_slot *memslot; - - gfn = unalias_gfn(kvm, gfn); - memslot = __gfn_to_memslot(kvm, gfn); - if (memslot && memslot->dirty_bitmap) { - unsigned long rel_gfn = gfn - memslot->base_gfn; - - /* avoid RMW */ - if (!test_bit(rel_gfn, memslot->dirty_bitmap)) - set_bit(rel_gfn, memslot->dirty_bitmap); - } -} - -/* - * The vCPU has executed a HLT instruction with in-kernel mode enabled. - */ -void kvm_vcpu_block(struct kvm_vcpu *vcpu) -{ - DECLARE_WAITQUEUE(wait, current); - - add_wait_queue(&vcpu->wq, &wait); - - /* - * We will block until either an interrupt or a signal wakes us up - */ - while (!kvm_cpu_has_interrupt(vcpu) - && !signal_pending(current) - && !kvm_arch_vcpu_runnable(vcpu)) { - set_current_state(TASK_INTERRUPTIBLE); - vcpu_put(vcpu); - schedule(); - vcpu_load(vcpu); - } - - __set_current_state(TASK_RUNNING); - remove_wait_queue(&vcpu->wq, &wait); -} - -void kvm_resched(struct kvm_vcpu *vcpu) -{ - if (!need_resched()) - return; - cond_resched(); -} -EXPORT_SYMBOL_GPL(kvm_resched); - -static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) -{ - struct kvm_vcpu *vcpu = vma->vm_file->private_data; - struct page *page; - - if (vmf->pgoff == 0) - page = virt_to_page(vcpu->run); - else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) - page = virt_to_page(vcpu->arch.pio_data); - else - return VM_FAULT_SIGBUS; - get_page(page); - vmf->page = page; - return 0; -} - -static struct vm_operations_struct kvm_vcpu_vm_ops = { - .fault = kvm_vcpu_fault, -}; - -static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) -{ - vma->vm_ops = &kvm_vcpu_vm_ops; - return 0; -} - -static int kvm_vcpu_release(struct inode *inode, struct file *filp) -{ - struct kvm_vcpu *vcpu = filp->private_data; - - fput(vcpu->kvm->filp); - return 0; -} - -static struct file_operations kvm_vcpu_fops = { - .release = kvm_vcpu_release, - .unlocked_ioctl = kvm_vcpu_ioctl, - .compat_ioctl = kvm_vcpu_ioctl, - .mmap = kvm_vcpu_mmap, -}; - -/* - * Allocates an inode for the vcpu. - */ -static int create_vcpu_fd(struct kvm_vcpu *vcpu) -{ - int fd, r; - struct inode *inode; - struct file *file; - - r = anon_inode_getfd(&fd, &inode, &file, - "kvm-vcpu", &kvm_vcpu_fops, vcpu); - if (r) - return r; - atomic_inc(&vcpu->kvm->filp->f_count); - return fd; -} - -/* - * Creates some virtual cpus. Good luck creating more than one. - */ -static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) -{ - int r; - struct kvm_vcpu *vcpu; - - if (!valid_vcpu(n)) - return -EINVAL; - - vcpu = kvm_arch_vcpu_create(kvm, n); - if (IS_ERR(vcpu)) - return PTR_ERR(vcpu); - - preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); - - r = kvm_arch_vcpu_setup(vcpu); - if (r) - goto vcpu_destroy; - - mutex_lock(&kvm->lock); - if (kvm->vcpus[n]) { - r = -EEXIST; - mutex_unlock(&kvm->lock); - goto vcpu_destroy; - } - kvm->vcpus[n] = vcpu; - mutex_unlock(&kvm->lock); - - /* Now it's all set up, let userspace reach it */ - r = create_vcpu_fd(vcpu); - if (r < 0) - goto unlink; - return r; - -unlink: - mutex_lock(&kvm->lock); - kvm->vcpus[n] = NULL; - mutex_unlock(&kvm->lock); -vcpu_destroy: - kvm_arch_vcpu_destroy(vcpu); - return r; -} - -static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) -{ - if (sigset) { - sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); - vcpu->sigset_active = 1; - vcpu->sigset = *sigset; - } else - vcpu->sigset_active = 0; - return 0; -} - -static long kvm_vcpu_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) -{ - struct kvm_vcpu *vcpu = filp->private_data; - void __user *argp = (void __user *)arg; - int r; - - if (vcpu->kvm->mm != current->mm) - return -EIO; - switch (ioctl) { - case KVM_RUN: - r = -EINVAL; - if (arg) - goto out; - r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); - break; - case KVM_GET_REGS: { - struct kvm_regs kvm_regs; - - memset(&kvm_regs, 0, sizeof kvm_regs); - r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs)) - goto out; - r = 0; - break; - } - case KVM_SET_REGS: { - struct kvm_regs kvm_regs; - - r = -EFAULT; - if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs)) - goto out; - r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs); - if (r) - goto out; - r = 0; - break; - } - case KVM_GET_SREGS: { - struct kvm_sregs kvm_sregs; - - memset(&kvm_sregs, 0, sizeof kvm_sregs); - r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs)) - goto out; - r = 0; - break; - } - case KVM_SET_SREGS: { - struct kvm_sregs kvm_sregs; - - r = -EFAULT; - if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs)) - goto out; - r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs); - if (r) - goto out; - r = 0; - break; - } - case KVM_TRANSLATE: { - struct kvm_translation tr; - - r = -EFAULT; - if (copy_from_user(&tr, argp, sizeof tr)) - goto out; - r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &tr, sizeof tr)) - goto out; - r = 0; - break; - } - case KVM_DEBUG_GUEST: { - struct kvm_debug_guest dbg; - - r = -EFAULT; - if (copy_from_user(&dbg, argp, sizeof dbg)) - goto out; - r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg); - if (r) - goto out; - r = 0; - break; - } - case KVM_SET_SIGNAL_MASK: { - struct kvm_signal_mask __user *sigmask_arg = argp; - struct kvm_signal_mask kvm_sigmask; - sigset_t sigset, *p; - - p = NULL; - if (argp) { - r = -EFAULT; - if (copy_from_user(&kvm_sigmask, argp, - sizeof kvm_sigmask)) - goto out; - r = -EINVAL; - if (kvm_sigmask.len != sizeof sigset) - goto out; - r = -EFAULT; - if (copy_from_user(&sigset, sigmask_arg->sigset, - sizeof sigset)) - goto out; - p = &sigset; - } - r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); - break; - } - case KVM_GET_FPU: { - struct kvm_fpu fpu; - - memset(&fpu, 0, sizeof fpu); - r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu); - if (r) - goto out; - r = -EFAULT; - if (copy_to_user(argp, &fpu, sizeof fpu)) - goto out; - r = 0; - break; - } - case KVM_SET_FPU: { - struct kvm_fpu fpu; - - r = -EFAULT; - if (copy_from_user(&fpu, argp, sizeof fpu)) - goto out; - r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu); - if (r) - goto out; - r = 0; - break; - } - default: - r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); - } -out: - return r; -} - -static long kvm_vm_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) -{ - struct kvm *kvm = filp->private_data; - void __user *argp = (void __user *)arg; - int r; - - if (kvm->mm != current->mm) - return -EIO; - switch (ioctl) { - case KVM_CREATE_VCPU: - r = kvm_vm_ioctl_create_vcpu(kvm, arg); - if (r < 0) - goto out; - break; - case KVM_SET_USER_MEMORY_REGION: { - struct kvm_userspace_memory_region kvm_userspace_mem; - - r = -EFAULT; - if (copy_from_user(&kvm_userspace_mem, argp, - sizeof kvm_userspace_mem)) - goto out; - - r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1); - if (r) - goto out; - break; - } - case KVM_GET_DIRTY_LOG: { - struct kvm_dirty_log log; - - r = -EFAULT; - if (copy_from_user(&log, argp, sizeof log)) - goto out; - r = kvm_vm_ioctl_get_dirty_log(kvm, &log); - if (r) - goto out; - break; - } - default: - r = kvm_arch_vm_ioctl(filp, ioctl, arg); - } -out: - return r; -} - -static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) -{ - struct kvm *kvm = vma->vm_file->private_data; - struct page *page; - - if (!kvm_is_visible_gfn(kvm, vmf->pgoff)) - return VM_FAULT_SIGBUS; - /* current->mm->mmap_sem is already held so call lockless version */ - page = __gfn_to_page(kvm, vmf->pgoff); - if (is_error_page(page)) { - kvm_release_page_clean(page); - return VM_FAULT_SIGBUS; - } - vmf->page = page; - return 0; -} - -static struct vm_operations_struct kvm_vm_vm_ops = { - .fault = kvm_vm_fault, -}; - -static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma) -{ - vma->vm_ops = &kvm_vm_vm_ops; - return 0; -} - -static struct file_operations kvm_vm_fops = { - .release = kvm_vm_release, - .unlocked_ioctl = kvm_vm_ioctl, - .compat_ioctl = kvm_vm_ioctl, - .mmap = kvm_vm_mmap, -}; - -static int kvm_dev_ioctl_create_vm(void) -{ - int fd, r; - struct inode *inode; - struct file *file; - struct kvm *kvm; - - kvm = kvm_create_vm(); - if (IS_ERR(kvm)) - return PTR_ERR(kvm); - r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm); - if (r) { - kvm_destroy_vm(kvm); - return r; - } - - kvm->filp = file; - - return fd; -} - -static long kvm_dev_ioctl(struct file *filp, - unsigned int ioctl, unsigned long arg) -{ - void __user *argp = (void __user *)arg; - long r = -EINVAL; - - switch (ioctl) { - case KVM_GET_API_VERSION: - r = -EINVAL; - if (arg) - goto out; - r = KVM_API_VERSION; - break; - case KVM_CREATE_VM: - r = -EINVAL; - if (arg) - goto out; - r = kvm_dev_ioctl_create_vm(); - break; - case KVM_CHECK_EXTENSION: - r = kvm_dev_ioctl_check_extension((long)argp); - break; - case KVM_GET_VCPU_MMAP_SIZE: - r = -EINVAL; - if (arg) - goto out; - r = 2 * PAGE_SIZE; - break; - default: - return kvm_arch_dev_ioctl(filp, ioctl, arg); - } -out: - return r; -} - -static struct file_operations kvm_chardev_ops = { - .unlocked_ioctl = kvm_dev_ioctl, - .compat_ioctl = kvm_dev_ioctl, -}; - -static struct miscdevice kvm_dev = { - KVM_MINOR, - "kvm", - &kvm_chardev_ops, -}; - -static void hardware_enable(void *junk) -{ - int cpu = raw_smp_processor_id(); - - if (cpu_isset(cpu, cpus_hardware_enabled)) - return; - cpu_set(cpu, cpus_hardware_enabled); - kvm_arch_hardware_enable(NULL); -} - -static void hardware_disable(void *junk) -{ - int cpu = raw_smp_processor_id(); - - if (!cpu_isset(cpu, cpus_hardware_enabled)) - return; - cpu_clear(cpu, cpus_hardware_enabled); - decache_vcpus_on_cpu(cpu); - kvm_arch_hardware_disable(NULL); -} - -static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, - void *v) -{ - int cpu = (long)v; - - val &= ~CPU_TASKS_FROZEN; - switch (val) { - case CPU_DYING: - printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", - cpu); - hardware_disable(NULL); - break; - case CPU_UP_CANCELED: - printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", - cpu); - smp_call_function_single(cpu, hardware_disable, NULL, 0, 1); - break; - case CPU_ONLINE: - printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", - cpu); - smp_call_function_single(cpu, hardware_enable, NULL, 0, 1); - break; - } - return NOTIFY_OK; -} - -static int kvm_reboot(struct notifier_block *notifier, unsigned long val, - void *v) -{ - if (val == SYS_RESTART) { - /* - * Some (well, at least mine) BIOSes hang on reboot if - * in vmx root mode. - */ - printk(KERN_INFO "kvm: exiting hardware virtualization\n"); - on_each_cpu(hardware_disable, NULL, 0, 1); - } - return NOTIFY_OK; -} - -static struct notifier_block kvm_reboot_notifier = { - .notifier_call = kvm_reboot, - .priority = 0, -}; - -void kvm_io_bus_init(struct kvm_io_bus *bus) -{ - memset(bus, 0, sizeof(*bus)); -} - -void kvm_io_bus_destroy(struct kvm_io_bus *bus) -{ - int i; - - for (i = 0; i < bus->dev_count; i++) { - struct kvm_io_device *pos = bus->devs[i]; - - kvm_iodevice_destructor(pos); - } -} - -struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr) -{ - int i; - - for (i = 0; i < bus->dev_count; i++) { - struct kvm_io_device *pos = bus->devs[i]; - - if (pos->in_range(pos, addr)) - return pos; - } - - return NULL; -} - -void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev) -{ - BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1)); - - bus->devs[bus->dev_count++] = dev; -} - -static struct notifier_block kvm_cpu_notifier = { - .notifier_call = kvm_cpu_hotplug, - .priority = 20, /* must be > scheduler priority */ -}; - -static u64 vm_stat_get(void *_offset) -{ - unsigned offset = (long)_offset; - u64 total = 0; - struct kvm *kvm; - - spin_lock(&kvm_lock); - list_for_each_entry(kvm, &vm_list, vm_list) - total += *(u32 *)((void *)kvm + offset); - spin_unlock(&kvm_lock); - return total; -} - -DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); - -static u64 vcpu_stat_get(void *_offset) -{ - unsigned offset = (long)_offset; - u64 total = 0; - struct kvm *kvm; - struct kvm_vcpu *vcpu; - int i; - - spin_lock(&kvm_lock); - list_for_each_entry(kvm, &vm_list, vm_list) - for (i = 0; i < KVM_MAX_VCPUS; ++i) { - vcpu = kvm->vcpus[i]; - if (vcpu) - total += *(u32 *)((void *)vcpu + offset); - } - spin_unlock(&kvm_lock); - return total; -} - -DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); - -static struct file_operations *stat_fops[] = { - [KVM_STAT_VCPU] = &vcpu_stat_fops, - [KVM_STAT_VM] = &vm_stat_fops, -}; - -static void kvm_init_debug(void) -{ - struct kvm_stats_debugfs_item *p; - - debugfs_dir = debugfs_create_dir("kvm", NULL); - for (p = debugfs_entries; p->name; ++p) - p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir, - (void *)(long)p->offset, - stat_fops[p->kind]); -} - -static void kvm_exit_debug(void) -{ - struct kvm_stats_debugfs_item *p; - - for (p = debugfs_entries; p->name; ++p) - debugfs_remove(p->dentry); - debugfs_remove(debugfs_dir); -} - -static int kvm_suspend(struct sys_device *dev, pm_message_t state) -{ - hardware_disable(NULL); - return 0; -} - -static int kvm_resume(struct sys_device *dev) -{ - hardware_enable(NULL); - return 0; -} - -static struct sysdev_class kvm_sysdev_class = { - .name = "kvm", - .suspend = kvm_suspend, - .resume = kvm_resume, -}; - -static struct sys_device kvm_sysdev = { - .id = 0, - .cls = &kvm_sysdev_class, -}; - -struct page *bad_page; - -static inline -struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) -{ - return container_of(pn, struct kvm_vcpu, preempt_notifier); -} - -static void kvm_sched_in(struct preempt_notifier *pn, int cpu) -{ - struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); - - kvm_arch_vcpu_load(vcpu, cpu); -} - -static void kvm_sched_out(struct preempt_notifier *pn, - struct task_struct *next) -{ - struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); - - kvm_arch_vcpu_put(vcpu); -} - -int kvm_init(void *opaque, unsigned int vcpu_size, - struct module *module) -{ - int r; - int cpu; - - kvm_init_debug(); - - r = kvm_arch_init(opaque); - if (r) - goto out_fail; - - bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); - - if (bad_page == NULL) { - r = -ENOMEM; - goto out; - } - - r = kvm_arch_hardware_setup(); - if (r < 0) - goto out_free_0; - - for_each_online_cpu(cpu) { - smp_call_function_single(cpu, - kvm_arch_check_processor_compat, - &r, 0, 1); - if (r < 0) - goto out_free_1; - } - - on_each_cpu(hardware_enable, NULL, 0, 1); - r = register_cpu_notifier(&kvm_cpu_notifier); - if (r) - goto out_free_2; - register_reboot_notifier(&kvm_reboot_notifier); - - r = sysdev_class_register(&kvm_sysdev_class); - if (r) - goto out_free_3; - - r = sysdev_register(&kvm_sysdev); - if (r) - goto out_free_4; - - /* A kmem cache lets us meet the alignment requirements of fx_save. */ - kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, - __alignof__(struct kvm_vcpu), - 0, NULL); - if (!kvm_vcpu_cache) { - r = -ENOMEM; - goto out_free_5; - } - - kvm_chardev_ops.owner = module; - - r = misc_register(&kvm_dev); - if (r) { - printk(KERN_ERR "kvm: misc device register failed\n"); - goto out_free; - } - - kvm_preempt_ops.sched_in = kvm_sched_in; - kvm_preempt_ops.sched_out = kvm_sched_out; - - return 0; - -out_free: - kmem_cache_destroy(kvm_vcpu_cache); -out_free_5: - sysdev_unregister(&kvm_sysdev); -out_free_4: - sysdev_class_unregister(&kvm_sysdev_class); -out_free_3: - unregister_reboot_notifier(&kvm_reboot_notifier); - unregister_cpu_notifier(&kvm_cpu_notifier); -out_free_2: - on_each_cpu(hardware_disable, NULL, 0, 1); -out_free_1: - kvm_arch_hardware_unsetup(); -out_free_0: - __free_page(bad_page); -out: - kvm_arch_exit(); - kvm_exit_debug(); -out_fail: - return r; -} -EXPORT_SYMBOL_GPL(kvm_init); - -void kvm_exit(void) -{ - misc_deregister(&kvm_dev); - kmem_cache_destroy(kvm_vcpu_cache); - sysdev_unregister(&kvm_sysdev); - sysdev_class_unregister(&kvm_sysdev_class); - unregister_reboot_notifier(&kvm_reboot_notifier); - unregister_cpu_notifier(&kvm_cpu_notifier); - on_each_cpu(hardware_disable, NULL, 0, 1); - kvm_arch_hardware_unsetup(); - kvm_arch_exit(); - kvm_exit_debug(); - __free_page(bad_page); -} -EXPORT_SYMBOL_GPL(kvm_exit); diff --git a/virt/kvm/iodev.h b/virt/kvm/iodev.h new file mode 100644 index 000000000000..c14e642027b2 --- /dev/null +++ b/virt/kvm/iodev.h @@ -0,0 +1,63 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +#ifndef __KVM_IODEV_H__ +#define __KVM_IODEV_H__ + +#include + +struct kvm_io_device { + void (*read)(struct kvm_io_device *this, + gpa_t addr, + int len, + void *val); + void (*write)(struct kvm_io_device *this, + gpa_t addr, + int len, + const void *val); + int (*in_range)(struct kvm_io_device *this, gpa_t addr); + void (*destructor)(struct kvm_io_device *this); + + void *private; +}; + +static inline void kvm_iodevice_read(struct kvm_io_device *dev, + gpa_t addr, + int len, + void *val) +{ + dev->read(dev, addr, len, val); +} + +static inline void kvm_iodevice_write(struct kvm_io_device *dev, + gpa_t addr, + int len, + const void *val) +{ + dev->write(dev, addr, len, val); +} + +static inline int kvm_iodevice_inrange(struct kvm_io_device *dev, gpa_t addr) +{ + return dev->in_range(dev, addr); +} + +static inline void kvm_iodevice_destructor(struct kvm_io_device *dev) +{ + if (dev->destructor) + dev->destructor(dev); +} + +#endif /* __KVM_IODEV_H__ */ diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c new file mode 100644 index 000000000000..4026d7d64296 --- /dev/null +++ b/virt/kvm/kvm_main.c @@ -0,0 +1,1393 @@ +/* + * Kernel-based Virtual Machine driver for Linux + * + * This module enables machines with Intel VT-x extensions to run virtual + * machines without emulation or binary translation. + * + * Copyright (C) 2006 Qumranet, Inc. + * + * Authors: + * Avi Kivity + * Yaniv Kamay + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + * + */ + +#include "iodev.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +MODULE_AUTHOR("Qumranet"); +MODULE_LICENSE("GPL"); + +DEFINE_SPINLOCK(kvm_lock); +LIST_HEAD(vm_list); + +static cpumask_t cpus_hardware_enabled; + +struct kmem_cache *kvm_vcpu_cache; +EXPORT_SYMBOL_GPL(kvm_vcpu_cache); + +static __read_mostly struct preempt_ops kvm_preempt_ops; + +static struct dentry *debugfs_dir; + +static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, + unsigned long arg); + +static inline int valid_vcpu(int n) +{ + return likely(n >= 0 && n < KVM_MAX_VCPUS); +} + +/* + * Switches to specified vcpu, until a matching vcpu_put() + */ +void vcpu_load(struct kvm_vcpu *vcpu) +{ + int cpu; + + mutex_lock(&vcpu->mutex); + cpu = get_cpu(); + preempt_notifier_register(&vcpu->preempt_notifier); + kvm_arch_vcpu_load(vcpu, cpu); + put_cpu(); +} + +void vcpu_put(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + kvm_arch_vcpu_put(vcpu); + preempt_notifier_unregister(&vcpu->preempt_notifier); + preempt_enable(); + mutex_unlock(&vcpu->mutex); +} + +static void ack_flush(void *_completed) +{ +} + +void kvm_flush_remote_tlbs(struct kvm *kvm) +{ + int i, cpu; + cpumask_t cpus; + struct kvm_vcpu *vcpu; + + cpus_clear(cpus); + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + vcpu = kvm->vcpus[i]; + if (!vcpu) + continue; + if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) + continue; + cpu = vcpu->cpu; + if (cpu != -1 && cpu != raw_smp_processor_id()) + cpu_set(cpu, cpus); + } + if (cpus_empty(cpus)) + return; + ++kvm->stat.remote_tlb_flush; + smp_call_function_mask(cpus, ack_flush, NULL, 1); +} + +int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) +{ + struct page *page; + int r; + + mutex_init(&vcpu->mutex); + vcpu->cpu = -1; + vcpu->kvm = kvm; + vcpu->vcpu_id = id; + init_waitqueue_head(&vcpu->wq); + + page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!page) { + r = -ENOMEM; + goto fail; + } + vcpu->run = page_address(page); + + r = kvm_arch_vcpu_init(vcpu); + if (r < 0) + goto fail_free_run; + return 0; + +fail_free_run: + free_page((unsigned long)vcpu->run); +fail: + return r; +} +EXPORT_SYMBOL_GPL(kvm_vcpu_init); + +void kvm_vcpu_uninit(struct kvm_vcpu *vcpu) +{ + kvm_arch_vcpu_uninit(vcpu); + free_page((unsigned long)vcpu->run); +} +EXPORT_SYMBOL_GPL(kvm_vcpu_uninit); + +static struct kvm *kvm_create_vm(void) +{ + struct kvm *kvm = kvm_arch_create_vm(); + + if (IS_ERR(kvm)) + goto out; + + kvm->mm = current->mm; + atomic_inc(&kvm->mm->mm_count); + kvm_io_bus_init(&kvm->pio_bus); + mutex_init(&kvm->lock); + kvm_io_bus_init(&kvm->mmio_bus); + spin_lock(&kvm_lock); + list_add(&kvm->vm_list, &vm_list); + spin_unlock(&kvm_lock); +out: + return kvm; +} + +/* + * Free any memory in @free but not in @dont. + */ +static void kvm_free_physmem_slot(struct kvm_memory_slot *free, + struct kvm_memory_slot *dont) +{ + if (!dont || free->rmap != dont->rmap) + vfree(free->rmap); + + if (!dont || free->dirty_bitmap != dont->dirty_bitmap) + vfree(free->dirty_bitmap); + + free->npages = 0; + free->dirty_bitmap = NULL; + free->rmap = NULL; +} + +void kvm_free_physmem(struct kvm *kvm) +{ + int i; + + for (i = 0; i < kvm->nmemslots; ++i) + kvm_free_physmem_slot(&kvm->memslots[i], NULL); +} + +static void kvm_destroy_vm(struct kvm *kvm) +{ + struct mm_struct *mm = kvm->mm; + + spin_lock(&kvm_lock); + list_del(&kvm->vm_list); + spin_unlock(&kvm_lock); + kvm_io_bus_destroy(&kvm->pio_bus); + kvm_io_bus_destroy(&kvm->mmio_bus); + kvm_arch_destroy_vm(kvm); + mmdrop(mm); +} + +static int kvm_vm_release(struct inode *inode, struct file *filp) +{ + struct kvm *kvm = filp->private_data; + + kvm_destroy_vm(kvm); + return 0; +} + +/* + * Allocate some memory and give it an address in the guest physical address + * space. + * + * Discontiguous memory is allowed, mostly for framebuffers. + * + * Must be called holding kvm->lock. + */ +int __kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc) +{ + int r; + gfn_t base_gfn; + unsigned long npages; + unsigned long i; + struct kvm_memory_slot *memslot; + struct kvm_memory_slot old, new; + + r = -EINVAL; + /* General sanity checks */ + if (mem->memory_size & (PAGE_SIZE - 1)) + goto out; + if (mem->guest_phys_addr & (PAGE_SIZE - 1)) + goto out; + if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS) + goto out; + if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr) + goto out; + + memslot = &kvm->memslots[mem->slot]; + base_gfn = mem->guest_phys_addr >> PAGE_SHIFT; + npages = mem->memory_size >> PAGE_SHIFT; + + if (!npages) + mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES; + + new = old = *memslot; + + new.base_gfn = base_gfn; + new.npages = npages; + new.flags = mem->flags; + + /* Disallow changing a memory slot's size. */ + r = -EINVAL; + if (npages && old.npages && npages != old.npages) + goto out_free; + + /* Check for overlaps */ + r = -EEXIST; + for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { + struct kvm_memory_slot *s = &kvm->memslots[i]; + + if (s == memslot) + continue; + if (!((base_gfn + npages <= s->base_gfn) || + (base_gfn >= s->base_gfn + s->npages))) + goto out_free; + } + + /* Free page dirty bitmap if unneeded */ + if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES)) + new.dirty_bitmap = NULL; + + r = -ENOMEM; + + /* Allocate if a slot is being created */ + if (npages && !new.rmap) { + new.rmap = vmalloc(npages * sizeof(struct page *)); + + if (!new.rmap) + goto out_free; + + memset(new.rmap, 0, npages * sizeof(*new.rmap)); + + new.user_alloc = user_alloc; + new.userspace_addr = mem->userspace_addr; + } + + /* Allocate page dirty bitmap if needed */ + if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) { + unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8; + + new.dirty_bitmap = vmalloc(dirty_bytes); + if (!new.dirty_bitmap) + goto out_free; + memset(new.dirty_bitmap, 0, dirty_bytes); + } + + if (mem->slot >= kvm->nmemslots) + kvm->nmemslots = mem->slot + 1; + + *memslot = new; + + r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc); + if (r) { + *memslot = old; + goto out_free; + } + + kvm_free_physmem_slot(&old, &new); + return 0; + +out_free: + kvm_free_physmem_slot(&new, &old); +out: + return r; + +} +EXPORT_SYMBOL_GPL(__kvm_set_memory_region); + +int kvm_set_memory_region(struct kvm *kvm, + struct kvm_userspace_memory_region *mem, + int user_alloc) +{ + int r; + + mutex_lock(&kvm->lock); + r = __kvm_set_memory_region(kvm, mem, user_alloc); + mutex_unlock(&kvm->lock); + return r; +} +EXPORT_SYMBOL_GPL(kvm_set_memory_region); + +int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, + struct + kvm_userspace_memory_region *mem, + int user_alloc) +{ + if (mem->slot >= KVM_MEMORY_SLOTS) + return -EINVAL; + return kvm_set_memory_region(kvm, mem, user_alloc); +} + +int kvm_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log, int *is_dirty) +{ + struct kvm_memory_slot *memslot; + int r, i; + int n; + unsigned long any = 0; + + r = -EINVAL; + if (log->slot >= KVM_MEMORY_SLOTS) + goto out; + + memslot = &kvm->memslots[log->slot]; + r = -ENOENT; + if (!memslot->dirty_bitmap) + goto out; + + n = ALIGN(memslot->npages, BITS_PER_LONG) / 8; + + for (i = 0; !any && i < n/sizeof(long); ++i) + any = memslot->dirty_bitmap[i]; + + r = -EFAULT; + if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n)) + goto out; + + if (any) + *is_dirty = 1; + + r = 0; +out: + return r; +} + +int is_error_page(struct page *page) +{ + return page == bad_page; +} +EXPORT_SYMBOL_GPL(is_error_page); + +static inline unsigned long bad_hva(void) +{ + return PAGE_OFFSET; +} + +int kvm_is_error_hva(unsigned long addr) +{ + return addr == bad_hva(); +} +EXPORT_SYMBOL_GPL(kvm_is_error_hva); + +static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn) +{ + int i; + + for (i = 0; i < kvm->nmemslots; ++i) { + struct kvm_memory_slot *memslot = &kvm->memslots[i]; + + if (gfn >= memslot->base_gfn + && gfn < memslot->base_gfn + memslot->npages) + return memslot; + } + return NULL; +} + +struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) +{ + gfn = unalias_gfn(kvm, gfn); + return __gfn_to_memslot(kvm, gfn); +} + +int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) +{ + int i; + + gfn = unalias_gfn(kvm, gfn); + for (i = 0; i < KVM_MEMORY_SLOTS; ++i) { + struct kvm_memory_slot *memslot = &kvm->memslots[i]; + + if (gfn >= memslot->base_gfn + && gfn < memslot->base_gfn + memslot->npages) + return 1; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); + +static unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot; + + gfn = unalias_gfn(kvm, gfn); + slot = __gfn_to_memslot(kvm, gfn); + if (!slot) + return bad_hva(); + return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE); +} + +/* + * Requires current->mm->mmap_sem to be held + */ +static struct page *__gfn_to_page(struct kvm *kvm, gfn_t gfn) +{ + struct page *page[1]; + unsigned long addr; + int npages; + + might_sleep(); + + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) { + get_page(bad_page); + return bad_page; + } + + npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page, + NULL); + + if (npages != 1) { + get_page(bad_page); + return bad_page; + } + + return page[0]; +} + +struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) +{ + struct page *page; + + down_read(¤t->mm->mmap_sem); + page = __gfn_to_page(kvm, gfn); + up_read(¤t->mm->mmap_sem); + + return page; +} + +EXPORT_SYMBOL_GPL(gfn_to_page); + +void kvm_release_page_clean(struct page *page) +{ + put_page(page); +} +EXPORT_SYMBOL_GPL(kvm_release_page_clean); + +void kvm_release_page_dirty(struct page *page) +{ + if (!PageReserved(page)) + SetPageDirty(page); + put_page(page); +} +EXPORT_SYMBOL_GPL(kvm_release_page_dirty); + +static int next_segment(unsigned long len, int offset) +{ + if (len > PAGE_SIZE - offset) + return PAGE_SIZE - offset; + else + return len; +} + +int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, + int len) +{ + int r; + unsigned long addr; + + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) + return -EFAULT; + r = copy_from_user(data, (void __user *)addr + offset, len); + if (r) + return -EFAULT; + return 0; +} +EXPORT_SYMBOL_GPL(kvm_read_guest_page); + +int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_read_guest_page(kvm, gfn, data, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + data += seg; + ++gfn; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_read_guest); + +int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, + int offset, int len) +{ + int r; + unsigned long addr; + + addr = gfn_to_hva(kvm, gfn); + if (kvm_is_error_hva(addr)) + return -EFAULT; + r = copy_to_user((void __user *)addr + offset, data, len); + if (r) + return -EFAULT; + mark_page_dirty(kvm, gfn); + return 0; +} +EXPORT_SYMBOL_GPL(kvm_write_guest_page); + +int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, + unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_write_guest_page(kvm, gfn, data, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + data += seg; + ++gfn; + } + return 0; +} + +int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) +{ + return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len); +} +EXPORT_SYMBOL_GPL(kvm_clear_guest_page); + +int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int seg; + int offset = offset_in_page(gpa); + int ret; + + while ((seg = next_segment(len, offset)) != 0) { + ret = kvm_clear_guest_page(kvm, gfn, offset, seg); + if (ret < 0) + return ret; + offset = 0; + len -= seg; + ++gfn; + } + return 0; +} +EXPORT_SYMBOL_GPL(kvm_clear_guest); + +void mark_page_dirty(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *memslot; + + gfn = unalias_gfn(kvm, gfn); + memslot = __gfn_to_memslot(kvm, gfn); + if (memslot && memslot->dirty_bitmap) { + unsigned long rel_gfn = gfn - memslot->base_gfn; + + /* avoid RMW */ + if (!test_bit(rel_gfn, memslot->dirty_bitmap)) + set_bit(rel_gfn, memslot->dirty_bitmap); + } +} + +/* + * The vCPU has executed a HLT instruction with in-kernel mode enabled. + */ +void kvm_vcpu_block(struct kvm_vcpu *vcpu) +{ + DECLARE_WAITQUEUE(wait, current); + + add_wait_queue(&vcpu->wq, &wait); + + /* + * We will block until either an interrupt or a signal wakes us up + */ + while (!kvm_cpu_has_interrupt(vcpu) + && !signal_pending(current) + && !kvm_arch_vcpu_runnable(vcpu)) { + set_current_state(TASK_INTERRUPTIBLE); + vcpu_put(vcpu); + schedule(); + vcpu_load(vcpu); + } + + __set_current_state(TASK_RUNNING); + remove_wait_queue(&vcpu->wq, &wait); +} + +void kvm_resched(struct kvm_vcpu *vcpu) +{ + if (!need_resched()) + return; + cond_resched(); +} +EXPORT_SYMBOL_GPL(kvm_resched); + +static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct kvm_vcpu *vcpu = vma->vm_file->private_data; + struct page *page; + + if (vmf->pgoff == 0) + page = virt_to_page(vcpu->run); + else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) + page = virt_to_page(vcpu->arch.pio_data); + else + return VM_FAULT_SIGBUS; + get_page(page); + vmf->page = page; + return 0; +} + +static struct vm_operations_struct kvm_vcpu_vm_ops = { + .fault = kvm_vcpu_fault, +}; + +static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma) +{ + vma->vm_ops = &kvm_vcpu_vm_ops; + return 0; +} + +static int kvm_vcpu_release(struct inode *inode, struct file *filp) +{ + struct kvm_vcpu *vcpu = filp->private_data; + + fput(vcpu->kvm->filp); + return 0; +} + +static struct file_operations kvm_vcpu_fops = { + .release = kvm_vcpu_release, + .unlocked_ioctl = kvm_vcpu_ioctl, + .compat_ioctl = kvm_vcpu_ioctl, + .mmap = kvm_vcpu_mmap, +}; + +/* + * Allocates an inode for the vcpu. + */ +static int create_vcpu_fd(struct kvm_vcpu *vcpu) +{ + int fd, r; + struct inode *inode; + struct file *file; + + r = anon_inode_getfd(&fd, &inode, &file, + "kvm-vcpu", &kvm_vcpu_fops, vcpu); + if (r) + return r; + atomic_inc(&vcpu->kvm->filp->f_count); + return fd; +} + +/* + * Creates some virtual cpus. Good luck creating more than one. + */ +static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n) +{ + int r; + struct kvm_vcpu *vcpu; + + if (!valid_vcpu(n)) + return -EINVAL; + + vcpu = kvm_arch_vcpu_create(kvm, n); + if (IS_ERR(vcpu)) + return PTR_ERR(vcpu); + + preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops); + + r = kvm_arch_vcpu_setup(vcpu); + if (r) + goto vcpu_destroy; + + mutex_lock(&kvm->lock); + if (kvm->vcpus[n]) { + r = -EEXIST; + mutex_unlock(&kvm->lock); + goto vcpu_destroy; + } + kvm->vcpus[n] = vcpu; + mutex_unlock(&kvm->lock); + + /* Now it's all set up, let userspace reach it */ + r = create_vcpu_fd(vcpu); + if (r < 0) + goto unlink; + return r; + +unlink: + mutex_lock(&kvm->lock); + kvm->vcpus[n] = NULL; + mutex_unlock(&kvm->lock); +vcpu_destroy: + kvm_arch_vcpu_destroy(vcpu); + return r; +} + +static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) +{ + if (sigset) { + sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP)); + vcpu->sigset_active = 1; + vcpu->sigset = *sigset; + } else + vcpu->sigset_active = 0; + return 0; +} + +static long kvm_vcpu_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + int r; + + if (vcpu->kvm->mm != current->mm) + return -EIO; + switch (ioctl) { + case KVM_RUN: + r = -EINVAL; + if (arg) + goto out; + r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); + break; + case KVM_GET_REGS: { + struct kvm_regs kvm_regs; + + memset(&kvm_regs, 0, sizeof kvm_regs); + r = kvm_arch_vcpu_ioctl_get_regs(vcpu, &kvm_regs); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &kvm_regs, sizeof kvm_regs)) + goto out; + r = 0; + break; + } + case KVM_SET_REGS: { + struct kvm_regs kvm_regs; + + r = -EFAULT; + if (copy_from_user(&kvm_regs, argp, sizeof kvm_regs)) + goto out; + r = kvm_arch_vcpu_ioctl_set_regs(vcpu, &kvm_regs); + if (r) + goto out; + r = 0; + break; + } + case KVM_GET_SREGS: { + struct kvm_sregs kvm_sregs; + + memset(&kvm_sregs, 0, sizeof kvm_sregs); + r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs)) + goto out; + r = 0; + break; + } + case KVM_SET_SREGS: { + struct kvm_sregs kvm_sregs; + + r = -EFAULT; + if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs)) + goto out; + r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs); + if (r) + goto out; + r = 0; + break; + } + case KVM_TRANSLATE: { + struct kvm_translation tr; + + r = -EFAULT; + if (copy_from_user(&tr, argp, sizeof tr)) + goto out; + r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &tr, sizeof tr)) + goto out; + r = 0; + break; + } + case KVM_DEBUG_GUEST: { + struct kvm_debug_guest dbg; + + r = -EFAULT; + if (copy_from_user(&dbg, argp, sizeof dbg)) + goto out; + r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg); + if (r) + goto out; + r = 0; + break; + } + case KVM_SET_SIGNAL_MASK: { + struct kvm_signal_mask __user *sigmask_arg = argp; + struct kvm_signal_mask kvm_sigmask; + sigset_t sigset, *p; + + p = NULL; + if (argp) { + r = -EFAULT; + if (copy_from_user(&kvm_sigmask, argp, + sizeof kvm_sigmask)) + goto out; + r = -EINVAL; + if (kvm_sigmask.len != sizeof sigset) + goto out; + r = -EFAULT; + if (copy_from_user(&sigset, sigmask_arg->sigset, + sizeof sigset)) + goto out; + p = &sigset; + } + r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset); + break; + } + case KVM_GET_FPU: { + struct kvm_fpu fpu; + + memset(&fpu, 0, sizeof fpu); + r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu); + if (r) + goto out; + r = -EFAULT; + if (copy_to_user(argp, &fpu, sizeof fpu)) + goto out; + r = 0; + break; + } + case KVM_SET_FPU: { + struct kvm_fpu fpu; + + r = -EFAULT; + if (copy_from_user(&fpu, argp, sizeof fpu)) + goto out; + r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu); + if (r) + goto out; + r = 0; + break; + } + default: + r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); + } +out: + return r; +} + +static long kvm_vm_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + struct kvm *kvm = filp->private_data; + void __user *argp = (void __user *)arg; + int r; + + if (kvm->mm != current->mm) + return -EIO; + switch (ioctl) { + case KVM_CREATE_VCPU: + r = kvm_vm_ioctl_create_vcpu(kvm, arg); + if (r < 0) + goto out; + break; + case KVM_SET_USER_MEMORY_REGION: { + struct kvm_userspace_memory_region kvm_userspace_mem; + + r = -EFAULT; + if (copy_from_user(&kvm_userspace_mem, argp, + sizeof kvm_userspace_mem)) + goto out; + + r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1); + if (r) + goto out; + break; + } + case KVM_GET_DIRTY_LOG: { + struct kvm_dirty_log log; + + r = -EFAULT; + if (copy_from_user(&log, argp, sizeof log)) + goto out; + r = kvm_vm_ioctl_get_dirty_log(kvm, &log); + if (r) + goto out; + break; + } + default: + r = kvm_arch_vm_ioctl(filp, ioctl, arg); + } +out: + return r; +} + +static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) +{ + struct kvm *kvm = vma->vm_file->private_data; + struct page *page; + + if (!kvm_is_visible_gfn(kvm, vmf->pgoff)) + return VM_FAULT_SIGBUS; + /* current->mm->mmap_sem is already held so call lockless version */ + page = __gfn_to_page(kvm, vmf->pgoff); + if (is_error_page(page)) { + kvm_release_page_clean(page); + return VM_FAULT_SIGBUS; + } + vmf->page = page; + return 0; +} + +static struct vm_operations_struct kvm_vm_vm_ops = { + .fault = kvm_vm_fault, +}; + +static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma) +{ + vma->vm_ops = &kvm_vm_vm_ops; + return 0; +} + +static struct file_operations kvm_vm_fops = { + .release = kvm_vm_release, + .unlocked_ioctl = kvm_vm_ioctl, + .compat_ioctl = kvm_vm_ioctl, + .mmap = kvm_vm_mmap, +}; + +static int kvm_dev_ioctl_create_vm(void) +{ + int fd, r; + struct inode *inode; + struct file *file; + struct kvm *kvm; + + kvm = kvm_create_vm(); + if (IS_ERR(kvm)) + return PTR_ERR(kvm); + r = anon_inode_getfd(&fd, &inode, &file, "kvm-vm", &kvm_vm_fops, kvm); + if (r) { + kvm_destroy_vm(kvm); + return r; + } + + kvm->filp = file; + + return fd; +} + +static long kvm_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + long r = -EINVAL; + + switch (ioctl) { + case KVM_GET_API_VERSION: + r = -EINVAL; + if (arg) + goto out; + r = KVM_API_VERSION; + break; + case KVM_CREATE_VM: + r = -EINVAL; + if (arg) + goto out; + r = kvm_dev_ioctl_create_vm(); + break; + case KVM_CHECK_EXTENSION: + r = kvm_dev_ioctl_check_extension((long)argp); + break; + case KVM_GET_VCPU_MMAP_SIZE: + r = -EINVAL; + if (arg) + goto out; + r = 2 * PAGE_SIZE; + break; + default: + return kvm_arch_dev_ioctl(filp, ioctl, arg); + } +out: + return r; +} + +static struct file_operations kvm_chardev_ops = { + .unlocked_ioctl = kvm_dev_ioctl, + .compat_ioctl = kvm_dev_ioctl, +}; + +static struct miscdevice kvm_dev = { + KVM_MINOR, + "kvm", + &kvm_chardev_ops, +}; + +static void hardware_enable(void *junk) +{ + int cpu = raw_smp_processor_id(); + + if (cpu_isset(cpu, cpus_hardware_enabled)) + return; + cpu_set(cpu, cpus_hardware_enabled); + kvm_arch_hardware_enable(NULL); +} + +static void hardware_disable(void *junk) +{ + int cpu = raw_smp_processor_id(); + + if (!cpu_isset(cpu, cpus_hardware_enabled)) + return; + cpu_clear(cpu, cpus_hardware_enabled); + decache_vcpus_on_cpu(cpu); + kvm_arch_hardware_disable(NULL); +} + +static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val, + void *v) +{ + int cpu = (long)v; + + val &= ~CPU_TASKS_FROZEN; + switch (val) { + case CPU_DYING: + printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", + cpu); + hardware_disable(NULL); + break; + case CPU_UP_CANCELED: + printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n", + cpu); + smp_call_function_single(cpu, hardware_disable, NULL, 0, 1); + break; + case CPU_ONLINE: + printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n", + cpu); + smp_call_function_single(cpu, hardware_enable, NULL, 0, 1); + break; + } + return NOTIFY_OK; +} + +static int kvm_reboot(struct notifier_block *notifier, unsigned long val, + void *v) +{ + if (val == SYS_RESTART) { + /* + * Some (well, at least mine) BIOSes hang on reboot if + * in vmx root mode. + */ + printk(KERN_INFO "kvm: exiting hardware virtualization\n"); + on_each_cpu(hardware_disable, NULL, 0, 1); + } + return NOTIFY_OK; +} + +static struct notifier_block kvm_reboot_notifier = { + .notifier_call = kvm_reboot, + .priority = 0, +}; + +void kvm_io_bus_init(struct kvm_io_bus *bus) +{ + memset(bus, 0, sizeof(*bus)); +} + +void kvm_io_bus_destroy(struct kvm_io_bus *bus) +{ + int i; + + for (i = 0; i < bus->dev_count; i++) { + struct kvm_io_device *pos = bus->devs[i]; + + kvm_iodevice_destructor(pos); + } +} + +struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus, gpa_t addr) +{ + int i; + + for (i = 0; i < bus->dev_count; i++) { + struct kvm_io_device *pos = bus->devs[i]; + + if (pos->in_range(pos, addr)) + return pos; + } + + return NULL; +} + +void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev) +{ + BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1)); + + bus->devs[bus->dev_count++] = dev; +} + +static struct notifier_block kvm_cpu_notifier = { + .notifier_call = kvm_cpu_hotplug, + .priority = 20, /* must be > scheduler priority */ +}; + +static u64 vm_stat_get(void *_offset) +{ + unsigned offset = (long)_offset; + u64 total = 0; + struct kvm *kvm; + + spin_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) + total += *(u32 *)((void *)kvm + offset); + spin_unlock(&kvm_lock); + return total; +} + +DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n"); + +static u64 vcpu_stat_get(void *_offset) +{ + unsigned offset = (long)_offset; + u64 total = 0; + struct kvm *kvm; + struct kvm_vcpu *vcpu; + int i; + + spin_lock(&kvm_lock); + list_for_each_entry(kvm, &vm_list, vm_list) + for (i = 0; i < KVM_MAX_VCPUS; ++i) { + vcpu = kvm->vcpus[i]; + if (vcpu) + total += *(u32 *)((void *)vcpu + offset); + } + spin_unlock(&kvm_lock); + return total; +} + +DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n"); + +static struct file_operations *stat_fops[] = { + [KVM_STAT_VCPU] = &vcpu_stat_fops, + [KVM_STAT_VM] = &vm_stat_fops, +}; + +static void kvm_init_debug(void) +{ + struct kvm_stats_debugfs_item *p; + + debugfs_dir = debugfs_create_dir("kvm", NULL); + for (p = debugfs_entries; p->name; ++p) + p->dentry = debugfs_create_file(p->name, 0444, debugfs_dir, + (void *)(long)p->offset, + stat_fops[p->kind]); +} + +static void kvm_exit_debug(void) +{ + struct kvm_stats_debugfs_item *p; + + for (p = debugfs_entries; p->name; ++p) + debugfs_remove(p->dentry); + debugfs_remove(debugfs_dir); +} + +static int kvm_suspend(struct sys_device *dev, pm_message_t state) +{ + hardware_disable(NULL); + return 0; +} + +static int kvm_resume(struct sys_device *dev) +{ + hardware_enable(NULL); + return 0; +} + +static struct sysdev_class kvm_sysdev_class = { + .name = "kvm", + .suspend = kvm_suspend, + .resume = kvm_resume, +}; + +static struct sys_device kvm_sysdev = { + .id = 0, + .cls = &kvm_sysdev_class, +}; + +struct page *bad_page; + +static inline +struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn) +{ + return container_of(pn, struct kvm_vcpu, preempt_notifier); +} + +static void kvm_sched_in(struct preempt_notifier *pn, int cpu) +{ + struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); + + kvm_arch_vcpu_load(vcpu, cpu); +} + +static void kvm_sched_out(struct preempt_notifier *pn, + struct task_struct *next) +{ + struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); + + kvm_arch_vcpu_put(vcpu); +} + +int kvm_init(void *opaque, unsigned int vcpu_size, + struct module *module) +{ + int r; + int cpu; + + kvm_init_debug(); + + r = kvm_arch_init(opaque); + if (r) + goto out_fail; + + bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + + if (bad_page == NULL) { + r = -ENOMEM; + goto out; + } + + r = kvm_arch_hardware_setup(); + if (r < 0) + goto out_free_0; + + for_each_online_cpu(cpu) { + smp_call_function_single(cpu, + kvm_arch_check_processor_compat, + &r, 0, 1); + if (r < 0) + goto out_free_1; + } + + on_each_cpu(hardware_enable, NULL, 0, 1); + r = register_cpu_notifier(&kvm_cpu_notifier); + if (r) + goto out_free_2; + register_reboot_notifier(&kvm_reboot_notifier); + + r = sysdev_class_register(&kvm_sysdev_class); + if (r) + goto out_free_3; + + r = sysdev_register(&kvm_sysdev); + if (r) + goto out_free_4; + + /* A kmem cache lets us meet the alignment requirements of fx_save. */ + kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, + __alignof__(struct kvm_vcpu), + 0, NULL); + if (!kvm_vcpu_cache) { + r = -ENOMEM; + goto out_free_5; + } + + kvm_chardev_ops.owner = module; + + r = misc_register(&kvm_dev); + if (r) { + printk(KERN_ERR "kvm: misc device register failed\n"); + goto out_free; + } + + kvm_preempt_ops.sched_in = kvm_sched_in; + kvm_preempt_ops.sched_out = kvm_sched_out; + + return 0; + +out_free: + kmem_cache_destroy(kvm_vcpu_cache); +out_free_5: + sysdev_unregister(&kvm_sysdev); +out_free_4: + sysdev_class_unregister(&kvm_sysdev_class); +out_free_3: + unregister_reboot_notifier(&kvm_reboot_notifier); + unregister_cpu_notifier(&kvm_cpu_notifier); +out_free_2: + on_each_cpu(hardware_disable, NULL, 0, 1); +out_free_1: + kvm_arch_hardware_unsetup(); +out_free_0: + __free_page(bad_page); +out: + kvm_arch_exit(); + kvm_exit_debug(); +out_fail: + return r; +} +EXPORT_SYMBOL_GPL(kvm_init); + +void kvm_exit(void) +{ + misc_deregister(&kvm_dev); + kmem_cache_destroy(kvm_vcpu_cache); + sysdev_unregister(&kvm_sysdev); + sysdev_class_unregister(&kvm_sysdev_class); + unregister_reboot_notifier(&kvm_reboot_notifier); + unregister_cpu_notifier(&kvm_cpu_notifier); + on_each_cpu(hardware_disable, NULL, 0, 1); + kvm_arch_hardware_unsetup(); + kvm_arch_exit(); + kvm_exit_debug(); + __free_page(bad_page); +} +EXPORT_SYMBOL_GPL(kvm_exit); -- cgit v1.2.3