diff options
Diffstat (limited to 'virt/kvm/arm/vgic')
| -rw-r--r-- | virt/kvm/arm/vgic/vgic-init.c | 4 | ||||
| -rw-r--r-- | virt/kvm/arm/vgic/vgic-irqfd.c | 6 | ||||
| -rw-r--r-- | virt/kvm/arm/vgic/vgic-kvm-device.c | 133 | ||||
| -rw-r--r-- | virt/kvm/arm/vgic/vgic-mmio-v3.c | 8 | ||||
| -rw-r--r-- | virt/kvm/arm/vgic/vgic-mmio.c | 2 | ||||
| -rw-r--r-- | virt/kvm/arm/vgic/vgic-mmio.h | 4 | ||||
| -rw-r--r-- | virt/kvm/arm/vgic/vgic-v2.c | 71 | ||||
| -rw-r--r-- | virt/kvm/arm/vgic/vgic.c | 8 | ||||
| -rw-r--r-- | virt/kvm/arm/vgic/vgic.h | 54 |
9 files changed, 148 insertions, 142 deletions
diff --git a/virt/kvm/arm/vgic/vgic-init.c b/virt/kvm/arm/vgic/vgic-init.c index 83777c1cbae0..8cebfbc19e90 100644 --- a/virt/kvm/arm/vgic/vgic-init.c +++ b/virt/kvm/arm/vgic/vgic-init.c @@ -405,6 +405,10 @@ int kvm_vgic_hyp_init(void) break; case GIC_V3: ret = vgic_v3_probe(gic_kvm_info); + if (!ret) { + static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif); + kvm_info("GIC system register CPU interface enabled\n"); + } break; default: ret = -ENODEV; diff --git a/virt/kvm/arm/vgic/vgic-irqfd.c b/virt/kvm/arm/vgic/vgic-irqfd.c index b31a51a14efb..d918dcf26a5a 100644 --- a/virt/kvm/arm/vgic/vgic-irqfd.c +++ b/virt/kvm/arm/vgic/vgic-irqfd.c @@ -46,15 +46,9 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e, * @ue: user api routing entry handle * return 0 on success, -EINVAL on errors. */ -#ifdef KVM_CAP_X2APIC_API int kvm_set_routing_entry(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) -#else -/* Remove this version and the ifdefery once merged into 4.8 */ -int kvm_set_routing_entry(struct kvm_kernel_irq_routing_entry *e, - const struct kvm_irq_routing_entry *ue) -#endif { int r = -EINVAL; diff --git a/virt/kvm/arm/vgic/vgic-kvm-device.c b/virt/kvm/arm/vgic/vgic-kvm-device.c index 1813f93b5cde..ce1f4ed9daf4 100644 --- a/virt/kvm/arm/vgic/vgic-kvm-device.c +++ b/virt/kvm/arm/vgic/vgic-kvm-device.c @@ -71,7 +71,6 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write) addr_ptr = &vgic->vgic_cpu_base; alignment = SZ_4K; break; -#ifdef CONFIG_KVM_ARM_VGIC_V3 case KVM_VGIC_V3_ADDR_TYPE_DIST: type_needed = KVM_DEV_TYPE_ARM_VGIC_V3; addr_ptr = &vgic->vgic_dist_base; @@ -82,7 +81,6 @@ int kvm_vgic_addr(struct kvm *kvm, unsigned long type, u64 *addr, bool write) addr_ptr = &vgic->vgic_redist_base; alignment = SZ_64K; break; -#endif default: r = -ENODEV; goto out; @@ -219,52 +217,65 @@ int kvm_register_vgic_device(unsigned long type) ret = kvm_register_device_ops(&kvm_arm_vgic_v2_ops, KVM_DEV_TYPE_ARM_VGIC_V2); break; -#ifdef CONFIG_KVM_ARM_VGIC_V3 case KVM_DEV_TYPE_ARM_VGIC_V3: ret = kvm_register_device_ops(&kvm_arm_vgic_v3_ops, KVM_DEV_TYPE_ARM_VGIC_V3); + +#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS if (ret) break; ret = kvm_vgic_register_its_device(); - break; #endif + break; } return ret; } -/** vgic_attr_regs_access: allows user space to read/write VGIC registers - * - * @dev: kvm device handle - * @attr: kvm device attribute - * @reg: address the value is read or written - * @is_write: write flag - * - */ -static int vgic_attr_regs_access(struct kvm_device *dev, - struct kvm_device_attr *attr, - u32 *reg, bool is_write) -{ +struct vgic_reg_attr { + struct kvm_vcpu *vcpu; gpa_t addr; - int cpuid, ret, c; - struct kvm_vcpu *vcpu, *tmp_vcpu; - int vcpu_lock_idx = -1; +}; + +static int parse_vgic_v2_attr(struct kvm_device *dev, + struct kvm_device_attr *attr, + struct vgic_reg_attr *reg_attr) +{ + int cpuid; cpuid = (attr->attr & KVM_DEV_ARM_VGIC_CPUID_MASK) >> KVM_DEV_ARM_VGIC_CPUID_SHIFT; - vcpu = kvm_get_vcpu(dev->kvm, cpuid); - addr = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK; - mutex_lock(&dev->kvm->lock); + if (cpuid >= atomic_read(&dev->kvm->online_vcpus)) + return -EINVAL; - ret = vgic_init(dev->kvm); - if (ret) - goto out; + reg_attr->vcpu = kvm_get_vcpu(dev->kvm, cpuid); + reg_attr->addr = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK; - if (cpuid >= atomic_read(&dev->kvm->online_vcpus)) { - ret = -EINVAL; - goto out; + return 0; +} + +/* unlocks vcpus from @vcpu_lock_idx and smaller */ +static void unlock_vcpus(struct kvm *kvm, int vcpu_lock_idx) +{ + struct kvm_vcpu *tmp_vcpu; + + for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) { + tmp_vcpu = kvm_get_vcpu(kvm, vcpu_lock_idx); + mutex_unlock(&tmp_vcpu->mutex); } +} + +static void unlock_all_vcpus(struct kvm *kvm) +{ + unlock_vcpus(kvm, atomic_read(&kvm->online_vcpus) - 1); +} + +/* Returns true if all vcpus were locked, false otherwise */ +static bool lock_all_vcpus(struct kvm *kvm) +{ + struct kvm_vcpu *tmp_vcpu; + int c; /* * Any time a vcpu is run, vcpu_load is called which tries to grab the @@ -272,11 +283,49 @@ static int vgic_attr_regs_access(struct kvm_device *dev, * that no other VCPUs are run and fiddle with the vgic state while we * access it. */ - ret = -EBUSY; - kvm_for_each_vcpu(c, tmp_vcpu, dev->kvm) { - if (!mutex_trylock(&tmp_vcpu->mutex)) - goto out; - vcpu_lock_idx = c; + kvm_for_each_vcpu(c, tmp_vcpu, kvm) { + if (!mutex_trylock(&tmp_vcpu->mutex)) { + unlock_vcpus(kvm, c - 1); + return false; + } + } + + return true; +} + +/** + * vgic_attr_regs_access_v2 - allows user space to access VGIC v2 state + * + * @dev: kvm device handle + * @attr: kvm device attribute + * @reg: address the value is read or written + * @is_write: true if userspace is writing a register + */ +static int vgic_attr_regs_access_v2(struct kvm_device *dev, + struct kvm_device_attr *attr, + u32 *reg, bool is_write) +{ + struct vgic_reg_attr reg_attr; + gpa_t addr; + struct kvm_vcpu *vcpu; + int ret; + + ret = parse_vgic_v2_attr(dev, attr, ®_attr); + if (ret) + return ret; + + vcpu = reg_attr.vcpu; + addr = reg_attr.addr; + + mutex_lock(&dev->kvm->lock); + + ret = vgic_init(dev->kvm); + if (ret) + goto out; + + if (!lock_all_vcpus(dev->kvm)) { + ret = -EBUSY; + goto out; } switch (attr->group) { @@ -291,18 +340,12 @@ static int vgic_attr_regs_access(struct kvm_device *dev, break; } + unlock_all_vcpus(dev->kvm); out: - for (; vcpu_lock_idx >= 0; vcpu_lock_idx--) { - tmp_vcpu = kvm_get_vcpu(dev->kvm, vcpu_lock_idx); - mutex_unlock(&tmp_vcpu->mutex); - } - mutex_unlock(&dev->kvm->lock); return ret; } -/* V2 ops */ - static int vgic_v2_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { @@ -321,7 +364,7 @@ static int vgic_v2_set_attr(struct kvm_device *dev, if (get_user(reg, uaddr)) return -EFAULT; - return vgic_attr_regs_access(dev, attr, ®, true); + return vgic_attr_regs_access_v2(dev, attr, ®, true); } } @@ -343,7 +386,7 @@ static int vgic_v2_get_attr(struct kvm_device *dev, u32 __user *uaddr = (u32 __user *)(long)attr->addr; u32 reg = 0; - ret = vgic_attr_regs_access(dev, attr, ®, false); + ret = vgic_attr_regs_access_v2(dev, attr, ®, false); if (ret) return ret; return put_user(reg, uaddr); @@ -387,10 +430,6 @@ struct kvm_device_ops kvm_arm_vgic_v2_ops = { .has_attr = vgic_v2_has_attr, }; -/* V3 ops */ - -#ifdef CONFIG_KVM_ARM_VGIC_V3 - static int vgic_v3_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { @@ -433,5 +472,3 @@ struct kvm_device_ops kvm_arm_vgic_v3_ops = { .get_attr = vgic_v3_get_attr, .has_attr = vgic_v3_has_attr, }; - -#endif /* CONFIG_KVM_ARM_VGIC_V3 */ diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/virt/kvm/arm/vgic/vgic-mmio-v3.c index 90d81811fdda..0d3c76a4208b 100644 --- a/virt/kvm/arm/vgic/vgic-mmio-v3.c +++ b/virt/kvm/arm/vgic/vgic-mmio-v3.c @@ -23,7 +23,7 @@ #include "vgic-mmio.h" /* extract @num bytes at @offset bytes offset in data */ -unsigned long extract_bytes(unsigned long data, unsigned int offset, +unsigned long extract_bytes(u64 data, unsigned int offset, unsigned int num) { return (data >> (offset * 8)) & GENMASK_ULL(num * 8 - 1, 0); @@ -42,6 +42,7 @@ u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len, return reg | ((u64)val << lower); } +#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS bool vgic_has_its(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; @@ -51,6 +52,7 @@ bool vgic_has_its(struct kvm *kvm) return dist->has_its; } +#endif static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len) @@ -179,7 +181,7 @@ static unsigned long vgic_mmio_read_v3r_typer(struct kvm_vcpu *vcpu, int target_vcpu_id = vcpu->vcpu_id; u64 value; - value = (mpidr & GENMASK(23, 0)) << 32; + value = (u64)(mpidr & GENMASK(23, 0)) << 32; value |= ((target_vcpu_id & 0xffff) << 8); if (target_vcpu_id == atomic_read(&vcpu->kvm->online_vcpus) - 1) value |= GICR_TYPER_LAST; @@ -609,7 +611,7 @@ void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg) bool broadcast; sgi = (reg & ICC_SGI1R_SGI_ID_MASK) >> ICC_SGI1R_SGI_ID_SHIFT; - broadcast = reg & BIT(ICC_SGI1R_IRQ_ROUTING_MODE_BIT); + broadcast = reg & BIT_ULL(ICC_SGI1R_IRQ_ROUTING_MODE_BIT); target_cpus = (reg & ICC_SGI1R_TARGET_LIST_MASK) >> ICC_SGI1R_TARGET_LIST_SHIFT; mpidr = SGI_AFFINITY_LEVEL(reg, 3); mpidr |= SGI_AFFINITY_LEVEL(reg, 2); diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/virt/kvm/arm/vgic/vgic-mmio.c index 3bad3c5ed431..e18b30ddcdce 100644 --- a/virt/kvm/arm/vgic/vgic-mmio.c +++ b/virt/kvm/arm/vgic/vgic-mmio.c @@ -550,11 +550,9 @@ int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address, case VGIC_V2: len = vgic_v2_init_dist_iodev(io_device); break; -#ifdef CONFIG_KVM_ARM_VGIC_V3 case VGIC_V3: len = vgic_v3_init_dist_iodev(io_device); break; -#endif default: BUG_ON(1); } diff --git a/virt/kvm/arm/vgic/vgic-mmio.h b/virt/kvm/arm/vgic/vgic-mmio.h index 0b3ecf9d100e..4c34d39d44a0 100644 --- a/virt/kvm/arm/vgic/vgic-mmio.h +++ b/virt/kvm/arm/vgic/vgic-mmio.h @@ -96,7 +96,7 @@ unsigned long vgic_data_mmio_bus_to_host(const void *val, unsigned int len); void vgic_data_host_to_mmio_bus(void *buf, unsigned int len, unsigned long data); -unsigned long extract_bytes(unsigned long data, unsigned int offset, +unsigned long extract_bytes(u64 data, unsigned int offset, unsigned int num); u64 update_64bit_reg(u64 reg, unsigned int offset, unsigned int len, @@ -162,12 +162,10 @@ unsigned int vgic_v2_init_dist_iodev(struct vgic_io_device *dev); unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev); -#ifdef CONFIG_KVM_ARM_VGIC_V3 u64 vgic_sanitise_outer_cacheability(u64 reg); u64 vgic_sanitise_inner_cacheability(u64 reg); u64 vgic_sanitise_shareability(u64 reg); u64 vgic_sanitise_field(u64 reg, u64 field_mask, int field_shift, u64 (*sanitise_fn)(u64)); -#endif #endif diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/virt/kvm/arm/vgic/vgic-v2.c index 0bf6709d1006..0a063af40565 100644 --- a/virt/kvm/arm/vgic/vgic-v2.c +++ b/virt/kvm/arm/vgic/vgic-v2.c @@ -278,12 +278,14 @@ int vgic_v2_map_resources(struct kvm *kvm) goto out; } - ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base, - kvm_vgic_global_state.vcpu_base, - KVM_VGIC_V2_CPU_SIZE, true); - if (ret) { - kvm_err("Unable to remap VGIC CPU to VCPU\n"); - goto out; + if (!static_branch_unlikely(&vgic_v2_cpuif_trap)) { + ret = kvm_phys_addr_ioremap(kvm, dist->vgic_cpu_base, + kvm_vgic_global_state.vcpu_base, + KVM_VGIC_V2_CPU_SIZE, true); + if (ret) { + kvm_err("Unable to remap VGIC CPU to VCPU\n"); + goto out; + } } dist->ready = true; @@ -294,6 +296,8 @@ out: return ret; } +DEFINE_STATIC_KEY_FALSE(vgic_v2_cpuif_trap); + /** * vgic_v2_probe - probe for a GICv2 compatible interrupt controller in DT * @node: pointer to the DT node @@ -310,45 +314,51 @@ int vgic_v2_probe(const struct gic_kvm_info *info) return -ENXIO; } - if (!PAGE_ALIGNED(info->vcpu.start)) { - kvm_err("GICV physical address 0x%llx not page aligned\n", - (unsigned long long)info->vcpu.start); - return -ENXIO; - } + if (!PAGE_ALIGNED(info->vcpu.start) || + !PAGE_ALIGNED(resource_size(&info->vcpu))) { + kvm_info("GICV region size/alignment is unsafe, using trapping (reduced performance)\n"); + kvm_vgic_global_state.vcpu_base_va = ioremap(info->vcpu.start, + resource_size(&info->vcpu)); + if (!kvm_vgic_global_state.vcpu_base_va) { + kvm_err("Cannot ioremap GICV\n"); + return -ENOMEM; + } - if (!PAGE_ALIGNED(resource_size(&info->vcpu))) { - kvm_err("GICV size 0x%llx not a multiple of page size 0x%lx\n", - (unsigned long long)resource_size(&info->vcpu), - PAGE_SIZE); - return -ENXIO; + ret = create_hyp_io_mappings(kvm_vgic_global_state.vcpu_base_va, + kvm_vgic_global_state.vcpu_base_va + resource_size(&info->vcpu), + info->vcpu.start); + if (ret) { + kvm_err("Cannot map GICV into hyp\n"); + goto out; + } + + static_branch_enable(&vgic_v2_cpuif_trap); } kvm_vgic_global_state.vctrl_base = ioremap(info->vctrl.start, resource_size(&info->vctrl)); if (!kvm_vgic_global_state.vctrl_base) { kvm_err("Cannot ioremap GICH\n"); - return -ENOMEM; + ret = -ENOMEM; + goto out; } vtr = readl_relaxed(kvm_vgic_global_state.vctrl_base + GICH_VTR); kvm_vgic_global_state.nr_lr = (vtr & 0x3f) + 1; - ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2); - if (ret) { - kvm_err("Cannot register GICv2 KVM device\n"); - iounmap(kvm_vgic_global_state.vctrl_base); - return ret; - } - ret = create_hyp_io_mappings(kvm_vgic_global_state.vctrl_base, kvm_vgic_global_state.vctrl_base + resource_size(&info->vctrl), info->vctrl.start); if (ret) { kvm_err("Cannot map VCTRL into hyp\n"); - kvm_unregister_device_ops(KVM_DEV_TYPE_ARM_VGIC_V2); - iounmap(kvm_vgic_global_state.vctrl_base); - return ret; + goto out; + } + + ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V2); + if (ret) { + kvm_err("Cannot register GICv2 KVM device\n"); + goto out; } kvm_vgic_global_state.can_emulate_gicv2 = true; @@ -359,4 +369,11 @@ int vgic_v2_probe(const struct gic_kvm_info *info) kvm_info("vgic-v2@%llx\n", info->vctrl.start); return 0; +out: + if (kvm_vgic_global_state.vctrl_base) + iounmap(kvm_vgic_global_state.vctrl_base); + if (kvm_vgic_global_state.vcpu_base_va) + iounmap(kvm_vgic_global_state.vcpu_base_va); + + return ret; } diff --git a/virt/kvm/arm/vgic/vgic.c b/virt/kvm/arm/vgic/vgic.c index e83b7fe4baae..2893d5ba523a 100644 --- a/virt/kvm/arm/vgic/vgic.c +++ b/virt/kvm/arm/vgic/vgic.c @@ -29,7 +29,7 @@ #define DEBUG_SPINLOCK_BUG_ON(p) #endif -struct vgic_global __section(.hyp.text) kvm_vgic_global_state; +struct vgic_global __section(.hyp.text) kvm_vgic_global_state = {.gicv3_cpuif = STATIC_KEY_FALSE_INIT,}; /* * Locking order is always: @@ -645,6 +645,9 @@ next: /* Sync back the hardware VGIC state into our emulation after a guest's run. */ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) { + if (unlikely(!vgic_initialized(vcpu->kvm))) + return; + vgic_process_maintenance_interrupt(vcpu); vgic_fold_lr_state(vcpu); vgic_prune_ap_list(vcpu); @@ -653,6 +656,9 @@ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) /* Flush our emulation state into the GIC hardware before entering the guest. */ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu) { + if (unlikely(!vgic_initialized(vcpu->kvm))) + return; + spin_lock(&vcpu->arch.vgic_cpu.ap_list_lock); vgic_flush_lr_state(vcpu); spin_unlock(&vcpu->arch.vgic_cpu.ap_list_lock); diff --git a/virt/kvm/arm/vgic/vgic.h b/virt/kvm/arm/vgic/vgic.h index 6c4625c46368..9d9e014765a2 100644 --- a/virt/kvm/arm/vgic/vgic.h +++ b/virt/kvm/arm/vgic/vgic.h @@ -72,7 +72,6 @@ static inline void vgic_get_irq_kref(struct vgic_irq *irq) kref_get(&irq->refcount); } -#ifdef CONFIG_KVM_ARM_VGIC_V3 void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu); void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu); void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr); @@ -84,63 +83,14 @@ void vgic_v3_enable(struct kvm_vcpu *vcpu); int vgic_v3_probe(const struct gic_kvm_info *info); int vgic_v3_map_resources(struct kvm *kvm); int vgic_register_redist_iodevs(struct kvm *kvm, gpa_t dist_base_address); + +#ifdef CONFIG_KVM_ARM_VGIC_V3_ITS int vgic_register_its_iodevs(struct kvm *kvm); bool vgic_has_its(struct kvm *kvm); int kvm_vgic_register_its_device(void); void vgic_enable_lpis(struct kvm_vcpu *vcpu); int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi); #else -static inline void vgic_v3_process_maintenance(struct kvm_vcpu *vcpu) -{ -} - -static inline void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) -{ -} - -static inline void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, - struct vgic_irq *irq, int lr) -{ -} - -static inline void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr) -{ -} - -static inline void vgic_v3_set_underflow(struct kvm_vcpu *vcpu) -{ -} - -static inline -void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) -{ -} - -static inline -void vgic_v3_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr) -{ -} - -static inline void vgic_v3_enable(struct kvm_vcpu *vcpu) -{ -} - -static inline int vgic_v3_probe(const struct gic_kvm_info *info) -{ - return -ENODEV; -} - -static inline int vgic_v3_map_resources(struct kvm *kvm) -{ - return -ENODEV; -} - -static inline int vgic_register_redist_iodevs(struct kvm *kvm, - gpa_t dist_base_address) -{ - return -ENODEV; -} - static inline int vgic_register_its_iodevs(struct kvm *kvm) { return -ENODEV; |