diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index fd445e9fa3f9..2ae34632cdcb 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -24,7 +25,12 @@ void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, gpa_t gpa, gpa_t gpsz, unsigned long order) { - gpa_t pos; + gpa_t end, pos, step = BIT(order); + + if (check_add_overflow(gpa, gpsz, &end)) { + kvm_riscv_local_hfence_gvma_vmid_all(vmid); + return; + } if (PTRS_PER_PTE < (gpsz >> order)) { kvm_riscv_local_hfence_gvma_vmid_all(vmid); @@ -33,14 +39,20 @@ void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile (HINVAL_GVMA(%0, %1) : : "r" (pos >> 2), "r" (vmid) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile (HFENCE_GVMA(%0, %1) : : "r" (pos >> 2), "r" (vmid) : "memory"); + if (end - pos <= step) + break; + } } } @@ -52,7 +64,12 @@ void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, unsigned long order) { - gpa_t pos; + gpa_t end, pos, step = BIT(order); + + if (check_add_overflow(gpa, gpsz, &end)) { + kvm_riscv_local_hfence_gvma_all(); + return; + } if (PTRS_PER_PTE < (gpsz >> order)) { kvm_riscv_local_hfence_gvma_all(); @@ -61,14 +78,20 @@ void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile(HINVAL_GVMA(%0, zero) : : "r" (pos >> 2) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) + for (pos = gpa; pos < end; pos += step) { asm volatile(HFENCE_GVMA(%0, zero) : : "r" (pos >> 2) : "memory"); + if (end - pos <= step) + break; + } } } @@ -83,7 +106,13 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, unsigned long gvsz, unsigned long order) { - unsigned long pos, hgatp; + unsigned long end, pos, step = BIT(order); + unsigned long hgatp; + + if (check_add_overflow(gva, gvsz, &end)) { + kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); + return; + } if (PTRS_PER_PTE < (gvsz >> order)) { kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); @@ -94,14 +123,20 @@ void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HINVAL_VVMA(%0, %1) : : "r" (pos), "r" (asid) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HFENCE_VVMA(%0, %1) : : "r" (pos), "r" (asid) : "memory"); + if (end - pos <= step) + break; + } } csr_write(CSR_HGATP, hgatp); @@ -123,7 +158,13 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, unsigned long gva, unsigned long gvsz, unsigned long order) { - unsigned long pos, hgatp; + unsigned long end, pos, step = BIT(order); + unsigned long hgatp; + + if (check_add_overflow(gva, gvsz, &end)) { + kvm_riscv_local_hfence_vvma_all(vmid); + return; + } if (PTRS_PER_PTE < (gvsz >> order)) { kvm_riscv_local_hfence_vvma_all(vmid); @@ -134,14 +175,20 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, if (has_svinval()) { asm volatile (SFENCE_W_INVAL() ::: "memory"); - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HINVAL_VVMA(%0, zero) : : "r" (pos) : "memory"); + if (end - pos <= step) + break; + } asm volatile (SFENCE_INVAL_IR() ::: "memory"); } else { - for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) + for (pos = gva; pos < end; pos += step) { asm volatile(HFENCE_VVMA(%0, zero) : : "r" (pos) : "memory"); + if (end - pos <= step) + break; + } } csr_write(CSR_HGATP, hgatp);