diff options
| author | Anup Patel <apatel@ventanamicro.com> | 2022-10-02 07:48:37 +0300 |
|---|---|---|
| committer | Anup Patel <anup@brainfault.org> | 2022-10-02 07:48:37 +0300 |
| commit | 5ff112484f2e63c5cac9f865181ca7ce467d0f89 (patch) | |
| tree | 894c4781a93fc6f3cd82037f324d894bed5b4a34 /arch/riscv/kvm/tlb.c | |
| parent | 122979aa26cd4a314aae889a0496eb829d50bc9e (diff) | |
| download | linux-5ff112484f2e63c5cac9f865181ca7ce467d0f89.tar.xz | |
RISC-V: KVM: Use Svinval for local TLB maintenance when available
We should prefer HINVAL.GVMA and HINVAL.VVMA instruction for local TLB
maintenance when underlying host supports Svinval extension.
Signed-off-by: Anup Patel <apatel@ventanamicro.com>
Reviewed-by: Andrew Jones <ajones@ventanamicro.com>
Signed-off-by: Anup Patel <anup@brainfault.org>
Diffstat (limited to 'arch/riscv/kvm/tlb.c')
| -rw-r--r-- | arch/riscv/kvm/tlb.c | 60 |
1 files changed, 48 insertions, 12 deletions
diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index 1ce3394b3acf..309d79b3e5cd 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -12,8 +12,12 @@ #include <linux/kvm_host.h> #include <asm/cacheflush.h> #include <asm/csr.h> +#include <asm/hwcap.h> #include <asm/insn-def.h> +#define has_svinval() \ + static_branch_unlikely(&riscv_isa_ext_keys[RISCV_ISA_EXT_KEY_SVINVAL]) + void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, gpa_t gpa, gpa_t gpsz, unsigned long order) @@ -25,9 +29,17 @@ void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, return; } - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) - asm volatile (HFENCE_GVMA(%0, %1) - : : "r" (pos >> 2), "r" (vmid) : "memory"); + if (has_svinval()) { + asm volatile (SFENCE_W_INVAL() ::: "memory"); + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + asm volatile (HINVAL_GVMA(%0, %1) + : : "r" (pos >> 2), "r" (vmid) : "memory"); + asm volatile (SFENCE_INVAL_IR() ::: "memory"); + } else { + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + asm volatile (HFENCE_GVMA(%0, %1) + : : "r" (pos >> 2), "r" (vmid) : "memory"); + } } void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) @@ -45,9 +57,17 @@ void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, return; } - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) - asm volatile(HFENCE_GVMA(%0, zero) - : : "r" (pos >> 2) : "memory"); + if (has_svinval()) { + asm volatile (SFENCE_W_INVAL() ::: "memory"); + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + asm volatile(HINVAL_GVMA(%0, zero) + : : "r" (pos >> 2) : "memory"); + asm volatile (SFENCE_INVAL_IR() ::: "memory"); + } else { + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + asm volatile(HFENCE_GVMA(%0, zero) + : : "r" (pos >> 2) : "memory"); + } } void kvm_riscv_local_hfence_gvma_all(void) @@ -70,9 +90,17 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) - asm volatile(HFENCE_VVMA(%0, %1) - : : "r" (pos), "r" (asid) : "memory"); + if (has_svinval()) { + asm volatile (SFENCE_W_INVAL() ::: "memory"); + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + asm volatile(HINVAL_VVMA(%0, %1) + : : "r" (pos), "r" (asid) : "memory"); + asm volatile (SFENCE_INVAL_IR() ::: "memory"); + } else { + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + asm volatile(HFENCE_VVMA(%0, %1) + : : "r" (pos), "r" (asid) : "memory"); + } csr_write(CSR_HGATP, hgatp); } @@ -102,9 +130,17 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) - asm volatile(HFENCE_VVMA(%0, zero) - : : "r" (pos) : "memory"); + if (has_svinval()) { + asm volatile (SFENCE_W_INVAL() ::: "memory"); + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + asm volatile(HINVAL_VVMA(%0, zero) + : : "r" (pos) : "memory"); + asm volatile (SFENCE_INVAL_IR() ::: "memory"); + } else { + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + asm volatile(HFENCE_VVMA(%0, zero) + : : "r" (pos) : "memory"); + } csr_write(CSR_HGATP, hgatp); } |