/* * Copyright (C) 2012 - Virtual Open Systems and Columbia University * Author: Christoffer Dall * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License, version 2, as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include "interrupts_head.S" .text .pushsection .hyp.text, "ax" /******************************************************************** * Flush per-VMID TLBs * * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); * * We rely on the hardware to broadcast the TLB invalidation to all CPUs * inside the inner-shareable domain (which is the case for all v7 * implementations). If we come across a non-IS SMP implementation, we'll * have to use an IPI based mechanism. Until then, we stick to the simple * hardware assisted version. * * As v7 does not support flushing per IPA, just nuke the whole TLB * instead, ignoring the ipa value. */ ENTRY(__kvm_tlb_flush_vmid_ipa) push {r2, r3} dsb ishst add r0, r0, #KVM_VTTBR ldrd r2, r3, [r0] mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR isb mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored) dsb ish isb mov r2, #0 mov r3, #0 mcrr p15, 6, r2, r3, c2 @ Back to VMID #0 isb @ Not necessary if followed by eret pop {r2, r3} bx lr ENDPROC(__kvm_tlb_flush_vmid_ipa) /** * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs * * Reuses __kvm_tlb_flush_vmid_ipa() for ARMv7, without passing address * parameter */ ENTRY(__kvm_tlb_flush_vmid) b __kvm_tlb_flush_vmid_ipa ENDPROC(__kvm_tlb_flush_vmid) /******************************************************************** * Flush TLBs and instruction caches of all CPUs inside the inner-shareable * domain, for all VMIDs * * void __kvm_flush_vm_context(void); */ ENTRY(__kvm_flush_vm_context) mov r0, #0 @ rn parameter for c15 flushes is SBZ /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */ mcr p15, 4, r0, c8, c3, 4 /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */ mcr p15, 0, r0, c7, c1, 0 dsb ish isb @ Not necessary if followed by eret bx lr ENDPROC(__kvm_flush_vm_context) /******************************************************************** * Hypervisor world-switch code * * * int __kvm_vcpu_run(struct kvm_vcpu *vcpu) */ ENTRY(__kvm_vcpu_run) @ Save the vcpu pointer mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR save_host_regs restore_vgic_state restore_timer_state @ Store hardware CP15 state and load guest state read_cp15_state store_to_vcpu = 0 write_cp15_state read_from_vcpu = 1 @ If the host kernel has not been configured with VFPv3 support, @ then it is safer if we deny guests from using it as well. #ifdef CONFIG_VFPv3 @ Set FPEXC_EN so the guest doesn't trap floating point instructions VFPFMRX r2, FPEXC @ VMRS push {r2} orr r2, r2, #FPEXC_EN VFPFMXR FPEXC, r2 @ VMSR #endif @ Configure Hyp-role configure_hyp_role vmentry @ Trap coprocessor CRx accesses set_hstr vmentry set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)) set_hdcr vmentry @ Write configured ID register into MIDR alias ldr r1, [vcpu, #VCPU_MIDR] mcr p15, 4, r1, c0, c0, 0 @ Write guest view of MPIDR into VMPIDR ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)] mcr p15, 4, r1, c0, c0, 5 @ Set up guest memory translation ldr r1, [vcpu, #VCPU_KVM] add r1, r1, #KVM_VTTBR ldrd r2, r3, [r1] mcrr p15, 6, rr_lo_hi(r2, r3), c2 @ Write VTTBR @ We're all done, just restore the GPRs and go to the guest restore_guest_regs clrex @ Clear exclusive monitor eret __kvm_vcpu_return: /* * return convention: * guest r0, r1, r2 saved on the stack * r0: vcpu pointer * r1: exception code */ save_guest_regs @ Set VMID == 0 mov r2, #0 mov r3, #0 mcrr p15, 6, r2, r3, c2 @ Write VTTBR @ Don't trap coprocessor accesses for host kernel set_hstr vmexit set_hdcr vmexit set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)), after_vfp_restore #ifdef CONFIG_VFPv3 @ Switch VFP/NEON hardware state to the host's add r7, vcpu, #(VCPU_GUEST_CTXT + CPU_CTXT_VFP) store_vfp_state r7 add r7, vcpu, #VCPU_HOST_CTXT ldr r7, [r7] add r7, r7, #CPU_CTXT_VFP restore_vfp_state r7 after_vfp_restore: @ Restore FPEXC_EN which we clobbered on entry pop {r2} VFPFMXR FPEXC, r2 #else after_vfp_restore: #endif @ Reset Hyp-role configure_hyp_role vmexit @ Let host read hardware MIDR mrc p15, 0, r2, c0, c0, 0 mcr p15, 4, r2, c0, c0, 0 @ Back to hardware MPIDR mrc p15, 0, r2, c0, c0, 5 mcr p15, 4, r2, c0, c0, 5 @ Store guest CP15 state and restore host state read_cp15_state store_to_vcpu = 1 write_cp15_state read_from_vcpu = 0 save_timer_state save_vgic_state restore_host_regs clrex @ Clear exclusive monitor #ifndef CONFIG_CPU_ENDIAN_BE8 mov r0, r1 @ Return the return code mov r1, #0 @ Clear upper bits in return value #else @ r1 already has return code mov r0, #0 @ Clear upper bits in return value #endif /* CONFIG_CPU_ENDIAN_BE8 */ bx lr @ return to IOCTL /******************************************************************** * Call function in Hyp mode * * * u64 kvm_call_hyp(void *hypfn, ...); * * This is not really a variadic function in the classic C-way and care must * be taken when calling this to ensure parameters are passed in registers * only, since the stack will change between the caller and the callee. * * Call the function with the first argument containing a pointer to the * function you wish to call in Hyp mode, and subsequent arguments will be * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the * function pointer can be passed). The function being called must be mapped * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are * passed in r0 and r1. * * A function pointer with a value of 0xffffffff has a special meaning, * and is used to implement __hyp_get_vectors in the same way as in * arch/arm/kernel/hyp_stub.S. * * The calling convention follows the standard AAPCS: * r0 - r3: caller save * r12: caller save * rest: callee save */ ENTRY(kvm_call_hyp) hvc #0 bx lr /******************************************************************** * Hypervisor exception vector and handlers * * * The KVM/ARM Hypervisor ABI is defined as follows: * * Entry to Hyp mode from the host kernel will happen _only_ when an HVC * instruction is issued since all traps are disabled when running the host * kernel as per the Hyp-mode initialization at boot time. * * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc * below) when the HVC instruction is called from SVC mode (i.e. a guest or the * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC * instructions are called from within Hyp-mode. * * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode): * Switching to Hyp mode is done through a simple HVC #0 instruction. The * exception vector code will check that the HVC comes from VMID==0 and if * so will push the necessary state (SPSR, lr_usr) on the Hyp stack. * - r0 contains a pointer to a HYP function * - r1, r2, and r3 contain arguments to the above function. * - The HYP function will be called with its arguments in r0, r1 and r2. * On HYP function return, we return directly to SVC. * * Note that the above is used to execute code in Hyp-mode from a host-kernel * point of view, and is a different concept from performing a world-switch and * executing guest code SVC mode (with a VMID != 0). */ /* Handle undef, svc, pabt, or dabt by crashing with a user notice */ .macro bad_exception exception_code, panic_str push {r0-r2} mrrc p15, 6, r0, r1, c2 @ Read VTTBR lsr r1, r1, #16 ands r1, r1, #0xff beq 99f load_vcpu @ Load VCPU pointer .if \exception_code == ARM_EXCEPTION_DATA_ABORT mrc p15, 4, r2, c5, c2, 0 @ HSR mrc p15, 4, r1, c6, c0, 0 @ HDFAR str r2, [vcpu, #VCPU_HSR] str r1, [vcpu, #VCPU_HxFAR] .endif .if \exception_code == ARM_EXCEPTION_PREF_ABORT mrc p15, 4, r2, c5, c2, 0 @ HSR mrc p15, 4, r1, c6, c0, 2 @ HIFAR str r2, [vcpu, #VCPU_HSR] str r1, [vcpu, #VCPU_HxFAR] .endif mov r1, #\exception_code b __kvm_vcpu_return @ We were in the host already. Let's craft a panic-ing return to SVC. 99: mrs r2, cpsr bic r2, r2, #MODE_MASK orr r2, r2, #SVC_MODE THUMB( orr r2, r2, #PSR_T_BIT ) msr spsr_cxsf, r2 mrs r1, ELR_hyp ldr r2, =panic msr ELR_hyp, r2 ldr r0, =\panic_str clrex @ Clear exclusive monitor eret .endm .align 5 __kvm_hyp_vector: .globl __kvm_hyp_vector @ Hyp-mode exception vector W(b) hyp_reset W(b) hyp_undef W(b) hyp_svc W(b) hyp_pabt W(b) hyp_dabt W(b) hyp_hvc W(b) hyp_irq W(b) hyp_fiq .align hyp_reset: b hyp_reset .align hyp_undef: bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str .align hyp_svc: bad_exception ARM_EXCEPTION_HVC, svc_die_str .align hyp_pabt: bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str .align hyp_dabt: bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str .align hyp_hvc: /* * Getting here is either becuase of a trap from a guest or from calling * HVC from the host kernel, which means "switch to Hyp mode". */ push {r0, r1, r2} @ Check syndrome register mrc p15, 4, r1, c5, c2, 0 @ HSR lsr r0, r1, #HSR_EC_SHIFT cmp r0, #HSR_EC_HVC bne guest_trap @ Not HVC instr. /* * Let's check if the HVC came from VMID 0 and allow simple * switch to Hyp mode */ mrrc p15, 6, r0, r2, c2 lsr r2, r2, #16 and r2, r2, #0xff cmp r2, #0 bne guest_trap @ Guest called HVC /* * Getting here means host called HVC, we shift parameters and branch * to Hyp function. */ pop {r0, r1, r2} /* Check for __hyp_get_vectors */ cmp r0, #-1 mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR beq 1f push {lr} mrs lr, SPSR push {lr} mov lr, r0 mov r0, r1 mov r1, r2 mov r2, r3 THUMB( orr lr, #1) blx lr @ Call the HYP function pop {lr} msr SPSR_csxf, lr pop {lr} 1: eret guest_trap: load_vcpu @ Load VCPU pointer to r0 str r1, [vcpu, #VCPU_HSR] @ Check if we need the fault information lsr r1, r1, #HSR_EC_SHIFT #ifdef CONFIG_VFPv3 cmp r1, #HSR_EC_CP_0_13 beq switch_to_guest_vfp #endif cmp r1, #HSR_EC_IABT mrceq p15, 4, r2, c6, c0, 2 @ HIFAR beq 2f cmp r1, #HSR_EC_DABT bne 1f mrc p15, 4, r2, c6, c0, 0 @ HDFAR 2: str r2, [vcpu, #VCPU_HxFAR] /* * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode: * * Abort on the stage 2 translation for a memory access from a * Non-secure PL1 or PL0 mode: * * For any Access flag fault or Translation fault, and also for any * Permission fault on the stage 2 translation of a memory access * made as part of a translation table walk for a stage 1 translation, * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR * is UNKNOWN. */ /* Check for permission fault, and S1PTW */ mrc p15, 4, r1, c5, c2, 0 @ HSR and r0, r1, #HSR_FSC_TYPE cmp r0, #FSC_PERM tsteq r1, #(1 << 7) @ S1PTW mrcne p15, 4, r2, c6, c0, 4 @ HPFAR bne 3f /* Preserve PAR */ mrrc p15, 0, r0, r1, c7 @ PAR push {r0, r1} /* Resolve IPA using the xFAR */ mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR isb mrrc p15, 0, r0, r1, c7 @ PAR tst r0, #1 bne 4f @ Failed translation ubfx r2, r0, #12, #20 lsl r2, r2, #4 orr r2, r2, r1, lsl #24 /* Restore PAR */ pop {r0, r1} mcrr p15, 0, r0, r1, c7 @ PAR 3: load_vcpu @ Load VCPU pointer to r0 str r2, [r0, #VCPU_HPFAR] 1: mov r1, #ARM_EXCEPTION_HVC b __kvm_vcpu_return 4: pop {r0, r1} @ Failed translation, return to guest mcrr p15, 0, r0, r1, c7 @ PAR clrex pop {r0, r1, r2} eret /* * If VFPv3 support is not available, then we will not switch the VFP * registers; however cp10 and cp11 accesses will still trap and fallback * to the regular coprocessor emulation code, which currently will * inject an undefined exception to the guest. */ #ifdef CONFIG_VFPv3 switch_to_guest_vfp: push {r3-r7} @ NEON/VFP used. Turn on VFP access. set_hcptr vmtrap, (HCPTR_TCP(10) | HCPTR_TCP(11)) @ Switch VFP/NEON hardware state to the guest's add r7, r0, #VCPU_HOST_CTXT ldr r7, [r7] add r7, r7, #CPU_CTXT_VFP store_vfp_state r7 add r7, r0, #(VCPU_GUEST_CTXT + CPU_CTXT_VFP) restore_vfp_state r7 pop {r3-r7} pop {r0-r2} clrex eret #endif .align hyp_irq: push {r0, r1, r2} mov r1, #ARM_EXCEPTION_IRQ load_vcpu @ Load VCPU pointer to r0 b __kvm_vcpu_return .align hyp_fiq: b hyp_fiq .ltorg .popsection .pushsection ".rodata" und_die_str: .ascii "unexpected undefined exception in Hyp mode at: %#08x\n" pabt_die_str: .ascii "unexpected prefetch abort in Hyp mode at: %#08x\n" dabt_die_str: .ascii "unexpected data abort in Hyp mode at: %#08x\n" svc_die_str: .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x\n" .popsection