diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 8d06b62e7188..e9b2b0c40ec6 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -157,5 +157,6 @@ extern unsigned long kvm_nvhe_sym(__icache_flags); extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits); extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl); extern unsigned long kvm_nvhe_sym(hyp_nr_cpus); +extern unsigned int kvm_nvhe_sym(hyp_gicv3_nr_lr); #endif /* __ARM64_KVM_HYP_H__ */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index d4efda7b75a3..3732ee9eb0d4 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -2439,6 +2439,8 @@ static int __init init_subsystems(void) switch (err) { case 0: vgic_present = true; + if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + kvm_nvhe_sym(hyp_gicv3_nr_lr) = kvm_vgic_global_state.nr_lr; break; case -ENODEV: case -ENXIO: diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 3cbfae0e3dda..4f2b871199cb 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -56,6 +56,7 @@ int host_stage2_idmap_locked(phys_addr_t addr, u64 size, enum kvm_pgtable_prot p int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id); int kvm_host_prepare_stage2(void *pgt_pool_base); int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd); +void kvm_guest_destroy_stage2(struct pkvm_hyp_vm *vm); void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt); int hyp_pin_shared_mem(void *from, void *to); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 06db299c37a8..1d01c6e547f5 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -24,6 +24,9 @@ DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); +/* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */ +unsigned int hyp_gicv3_nr_lr; + void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) @@ -128,10 +131,18 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; + /* __hyp_running_vcpu must be NULL in a guest context. */ + hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; + hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; - hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE); + /* + * HCR_EL2.VSE is host-owned (a pending virtual SError to inject), not a + * trap-control bit, so it must flow to the hyp vCPU alongside TWI/TWE + * for the vSError to be delivered. sync_hyp_vcpu() reflects it back. + */ + hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWI | HCR_TWE | HCR_VSE); hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & - (HCR_TWI | HCR_TWE); + (HCR_TWI | HCR_TWE | HCR_VSE); hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; @@ -139,6 +150,12 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3; + /* Bound used_lrs by the number of implemented list registers. */ + hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs = + min_t(unsigned int, + hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs, + hyp_gicv3_nr_lr); + hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; } diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 25f04629014e..fee55e49c132 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -217,7 +217,6 @@ static void *guest_s2_zalloc_page(void *mc) memset(addr, 0, PAGE_SIZE); p = hyp_virt_to_page(addr); p->refcount = 1; - p->order = 0; return addr; } @@ -306,23 +305,27 @@ int kvm_guest_prepare_stage2(struct pkvm_hyp_vm *vm, void *pgd) return 0; } +void kvm_guest_destroy_stage2(struct pkvm_hyp_vm *vm) +{ + guest_lock_component(vm); + kvm_pgtable_stage2_destroy(&vm->pgt); + vm->kvm.arch.mmu.pgd_phys = 0ULL; + guest_unlock_component(vm); +} + void reclaim_pgtable_pages(struct pkvm_hyp_vm *vm, struct kvm_hyp_memcache *mc) { struct hyp_page *page; void *addr; /* Dump all pgtable pages in the hyp_pool */ - guest_lock_component(vm); - kvm_pgtable_stage2_destroy(&vm->pgt); - vm->kvm.arch.mmu.pgd_phys = 0ULL; - guest_unlock_component(vm); + kvm_guest_destroy_stage2(vm); /* Drain the hyp_pool into the memcache */ addr = hyp_alloc_pages(&vm->pool, 0); while (addr) { page = hyp_virt_to_page(addr); page->refcount = 0; - page->order = 0; push_hyp_memcache(mc, addr, hyp_virt_to_phys); WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(addr), 1)); addr = hyp_alloc_pages(&vm->pool, 0); @@ -851,6 +854,16 @@ static int __hyp_check_page_state_range(phys_addr_t phys, u64 size, enum pkvm_pa return 0; } +static int __hyp_check_page_count_range(phys_addr_t phys, u64 size) +{ + for_each_hyp_page(page, phys, size) { + if (page->refcount) + return -EBUSY; + } + + return 0; +} + static bool guest_pte_is_poisoned(kvm_pte_t pte) { if (kvm_pte_valid(pte)) @@ -1049,7 +1062,6 @@ int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn) int __pkvm_host_unshare_hyp(u64 pfn) { u64 phys = hyp_pfn_to_phys(pfn); - u64 virt = (u64)__hyp_va(phys); u64 size = PAGE_SIZE; int ret; @@ -1062,10 +1074,9 @@ int __pkvm_host_unshare_hyp(u64 pfn) ret = __hyp_check_page_state_range(phys, size, PKVM_PAGE_SHARED_BORROWED); if (ret) goto unlock; - if (hyp_page_count((void *)virt)) { - ret = -EBUSY; + ret = __hyp_check_page_count_range(phys, size); + if (ret) goto unlock; - } __hyp_set_page_state_range(phys, size, PKVM_NOPAGE); WARN_ON(__host_set_page_state_range(phys, size, PKVM_PAGE_OWNED)); @@ -1128,6 +1139,10 @@ int __pkvm_hyp_donate_host(u64 pfn, u64 nr_pages) if (ret) goto unlock; + ret = __hyp_check_page_count_range(phys, size); + if (ret) + goto unlock; + __hyp_set_page_state_range(phys, size, PKVM_NOPAGE); WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, virt, size) != size); WARN_ON(host_stage2_set_owner_locked(phys, size, PKVM_ID_HOST)); diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c index a1eb27a1a747..57f86aa0f82f 100644 --- a/arch/arm64/kvm/hyp/nvhe/page_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -94,13 +94,22 @@ static void __hyp_attach_page(struct hyp_pool *pool, struct hyp_page *p) { phys_addr_t phys = hyp_page_to_phys(p); - u8 order = p->order; struct hyp_page *buddy; + bool coalesce = true; + u8 order = p->order; - memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order); + /* + * 'external' pages are never coalesced and their ->order field + * untrusted as they bypass hyp_pool_init(). Enforce order-0. + */ + if (phys < pool->range_start || phys >= pool->range_end) { + order = 0; + coalesce = false; + } - /* Skip coalescing for 'external' pages being freed into the pool. */ - if (phys < pool->range_start || phys >= pool->range_end) + memset(hyp_page_to_virt(p), 0, PAGE_SIZE << order); + + if (!coalesce) goto insert; /* @@ -237,8 +246,10 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, /* Init the vmemmap portion */ p = hyp_phys_to_page(phys); - for (i = 0; i < nr_pages; i++) + for (i = 0; i < nr_pages; i++) { hyp_set_page_refcounted(&p[i]); + p[i].order = 0; + } /* Attach the unused pages to the buddy tree */ for (i = reserved_pages; i < nr_pages; i++) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index eb1c10120f9f..3b2c4fbc34d8 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -853,10 +853,12 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, /* Must be called last since this publishes the VM. */ ret = insert_vm_table_entry(handle, hyp_vm); if (ret) - goto err_remove_mappings; + goto err_destroy_stage2; return 0; +err_destroy_stage2: + kvm_guest_destroy_stage2(hyp_vm); err_remove_mappings: unmap_donated_memory(hyp_vm, vm_size); unmap_donated_memory(pgd, pgd_size); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 4da9281312eb..8811ad60cf72 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -501,6 +501,10 @@ static int share_pfn_hyp(u64 pfn) rb_link_node(&this->node, parent, node); rb_insert_color(&this->node, &hyp_shared_pfns); ret = kvm_call_hyp_nvhe(__pkvm_host_share_hyp, pfn); + if (ret) { + rb_erase(&this->node, &hyp_shared_pfns); + kfree(this); + } unlock: mutex_unlock(&hyp_shared_pfns_lock); @@ -520,13 +524,17 @@ static int unshare_pfn_hyp(u64 pfn) goto unlock; } - this->count--; - if (this->count) + if (this->count > 1) { + this->count--; + goto unlock; + } + + ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn); + if (ret) goto unlock; rb_erase(&this->node, &hyp_shared_pfns); kfree(this); - ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_hyp, pfn); unlock: mutex_unlock(&hyp_shared_pfns_lock); @@ -536,8 +544,8 @@ static int unshare_pfn_hyp(u64 pfn) int kvm_share_hyp(void *from, void *to) { phys_addr_t start, end, cur; + int ret = 0; u64 pfn; - int ret; if (is_kernel_in_hyp_mode()) return 0; @@ -559,10 +567,24 @@ int kvm_share_hyp(void *from, void *to) pfn = __phys_to_pfn(cur); ret = share_pfn_hyp(pfn); if (ret) - return ret; + break; } - return 0; + if (!ret) + return 0; + + /* + * Roll back the pages shared by this call. A failed unshare leaks + * the page (it stays shared with the hypervisor and is no longer + * reusable for pKVM) but breaks no isolation guarantee, so warn and + * continue. Not expected in practice. + */ + for (end = cur, cur = start; cur < end; cur += PAGE_SIZE) { + pfn = __phys_to_pfn(cur); + WARN_ON(unshare_pfn_hyp(pfn)); + } + + return ret; } void kvm_unshare_hyp(void *from, void *to) @@ -577,6 +599,11 @@ void kvm_unshare_hyp(void *from, void *to) end = PAGE_ALIGN(__pa(to)); for (cur = start; cur < end; cur += PAGE_SIZE) { pfn = __phys_to_pfn(cur); + /* + * A failed unshare leaks the page: it stays shared with the + * hypervisor and is no longer reusable for pKVM. No isolation + * guarantee is broken, and this is not expected in practice. + */ WARN_ON(unshare_pfn_hyp(pfn)); } }