github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 12:44:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM guest_memfd and x86 CoCo changes for 7.3

Browse Source 
- Forcefully invalidate SNP VMSA pages if their backing guest_memfd page is
    zapped/invalidated, e.g. due to a PUNCH_HOLE in response to a Page-State
    Change request.
 
  - Rework the so called "prepare" and "invalidate" guest_memfd hooks to prepare
    for in-place private<=>shared conversion, and clean up a few warts along the
    way.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEKTobbabEP7vbhhN9OlYIJqCjN/0FAmp7vdQACgkQOlYIJqCj
 N/00vA/+NmxB+Zsr7+TZo4NFiFcs/FK7rBb6psvlLuJN1oNLPw7dzdmPclSyv7li
 RDWRF6SmuujxCO2W4LMc1Fz5IjN8/iYPHsEHnYCCxjDgFdEl7x1MOJ0xuXQOWgQw
 icpWdYgWh9EoaKepn0EFVeF7WZFK7KhxmpJNz5HZEx3p110HfrSHbjl0LikySDst
 19Jck5ZzTCVrbF58e5k6phzvvR6mRNeLS7BKvOM+u5U70+7P9AcTbihd8qz7vy0k
 LlJUBZbp7NywjgIcqMTZGmBvsWoOAcvykBX9hNgjDdobVhh3Ft+hsmOxNt7yOoAl
 llJznwRQvZItWGfGO8o9l+/pJI7bvGzQ6isXqdg3TbxWH7/d6HOdWoGiC8ya2x+k
 UBDvlRgAH9KZuuE7E2m/ggM0hoLbWyNDV2qVo2WnmqbGG4SQRSA5hyq/pIov0hRY
 kr0anBd/Pw8ZoO02RtlrSQkKODbdR7l3/lQDzHzSWKidJ8Eh+iqA30/oIQzQMCOw
 3s/SIB19Jab4UEgS3aqvFNbsj+zXN2tT6NLiiSoodSp+sSP5OMJ46U4Sb/txn9EP
 L035UIyLRAGFsIybvutSckT/Guv9aVeeK0F2CNRvzKrPUhaj3Vi2a7RvQtZZWeVd
 EJLzM07Rx8msGZYq8Eg+9QkrDe5IgVjJvsZ5RrKeS7VJKivpFRU=
 =yUfX
 -----END PGP SIGNATURE-----

Merge tag 'kvm-x86-coco-7.3' of https://github.com/kvm-x86/linux into HEAD

KVM guest_memfd and x86 CoCo changes for 7.3

 - Forcefully invalidate SNP VMSA pages if their backing guest_memfd page is
   zapped/invalidated, e.g. due to a PUNCH_HOLE in response to a Page-State
   Change request.

 - Rework the so called "prepare" and "invalidate" guest_memfd hooks to prepare
   for in-place private<=>shared conversion, and clean up a few warts along the
   way.
This commit is contained in:
Paolo Bonzini 2026-08-18 13:25:39 +02:00
commit 159ed1ae35
12 changed files with 269 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
arch/x86/include/asm/kvm-x86-ops.h
View File

@ -134,6 +134,7 @@ KVM_X86_OP_OPTIONAL(mem_enc_unregister_region)
KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from)
KVM_X86_OP_OPTIONAL(vm_move_enc_context_from)
KVM_X86_OP_OPTIONAL(guest_memory_reclaimed)
KVM_X86_OP_OPTIONAL(reload_vmsa)
KVM_X86_OP(get_feature_msr)
KVM_X86_OP(check_emulate_instruction)
KVM_X86_OP(apic_init_signal_blocked)
@ -145,9 +146,16 @@ KVM_X86_OP(vcpu_deliver_sipi_vector)
KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons);
KVM_X86_OP_OPTIONAL(get_untagged_addr)
KVM_X86_OP_OPTIONAL(alloc_apic_backing_page)
KVM_X86_OP_OPTIONAL_RET0(gmem_prepare)
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
KVM_X86_OP_OPTIONAL_RET0(gmem_make_private)
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
KVM_X86_OP_OPTIONAL(gmem_make_shared)
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
KVM_X86_OP_OPTIONAL(gmem_invalidate_range)
#endif
KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level)
KVM_X86_OP_OPTIONAL(gmem_invalidate)
#endif
#undef KVM_X86_OP

15
arch/x86/include/asm/kvm_host.h
View File

@ -122,6 +122,8 @@
KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_HV_TLB_FLUSH \
KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_VMSA_PAGE_RELOAD \
KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP)
#define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \
KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT)
@ -1878,6 +1880,7 @@ struct kvm_x86_ops {
int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd);
void (*guest_memory_reclaimed)(struct kvm *kvm);
void (*reload_vmsa)(struct kvm_vcpu *vcpu);
int (*get_feature_msr)(u32 msr, u64 *data);
@ -1900,8 +1903,16 @@ struct kvm_x86_ops {
gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags);
void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu);
int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end);
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
kvm_pfn_t nr_pages);
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages);
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range);
#endif
int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
};

3
arch/x86/kvm/Kconfig
View File

@ -160,7 +160,8 @@ config KVM_AMD_SEV
depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m)
select ARCH_HAS_CC_PLATFORM
select KVM_GENERIC_MEMORY_ATTRIBUTES
select HAVE_KVM_ARCH_GMEM_PREPARE
select HAVE_KVM_ARCH_GMEM_CONVERT
select HAVE_KVM_ARCH_GMEM_RECLAIM
select HAVE_KVM_ARCH_GMEM_INVALIDATE
select HAVE_KVM_ARCH_GMEM_POPULATE
help

74
arch/x86/kvm/mmu/mmu.c
View File

@ -6947,20 +6947,11 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm)
kvm_mmu_commit_zap_page(kvm, &invalid_list);
}
/*
* Fast invalidate all shadow pages and use lock-break technique
* to zap obsolete pages.
*
* It's required when memslot is being deleted or VM is being
* destroyed, in these cases, we should ensure that KVM MMU does
* not use any resource of the being-deleted slot or all slots
* after calling the function.
*/
static void kvm_mmu_zap_all_fast(struct kvm *kvm)
static void __kvm_mmu_zap_all_fast_front_half(struct kvm *kvm)
{
lockdep_assert_held(&kvm->slots_lock);
lockdep_assert_held_write(&kvm->mmu_lock);
write_lock(&kvm->mmu_lock);
trace_kvm_mmu_zap_all_fast(kvm);
/*
@ -6997,8 +6988,12 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm)
kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS);
kvm_zap_obsolete_pages(kvm);
}
write_unlock(&kvm->mmu_lock);
static void __kvm_mmu_zap_all_fast_back_half(struct kvm *kvm)
{
lockdep_assert_held(&kvm->slots_lock);
lockdep_assert_not_held(&kvm->mmu_lock);
/*
* Zap the invalidated TDP MMU roots, all SPTEs must be dropped before
@ -7012,6 +7007,24 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm)
kvm_tdp_mmu_zap_invalidated_roots(kvm, true);
}
/*
* Fast invalidate all shadow pages and use lock-break technique
* to zap obsolete pages.
*
* It's required when memslot is being deleted or VM is being
* destroyed, in these cases, we should ensure that KVM MMU does
* not use any resource of the being-deleted slot or all slots
* after calling the function.
*/
static void kvm_mmu_zap_all_fast(struct kvm *kvm)
{
write_lock(&kvm->mmu_lock);
__kvm_mmu_zap_all_fast_front_half(kvm);
write_unlock(&kvm->mmu_lock);
__kvm_mmu_zap_all_fast_back_half(kvm);
}
int kvm_mmu_init_vm(struct kvm *kvm)
{
int r, i;
@ -7586,8 +7599,8 @@ static void kvm_mmu_zap_memslot_pages_and_flush(struct kvm *kvm,
kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
}
static void kvm_mmu_zap_memslot(struct kvm *kvm,
struct kvm_memory_slot *slot)
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
struct kvm_memory_slot *slot)
{
struct kvm_gfn_range range = {
.slot = slot,
@ -7596,27 +7609,28 @@ static void kvm_mmu_zap_memslot(struct kvm *kvm,
.may_block = true,
.attr_filter = KVM_FILTER_PRIVATE | KVM_FILTER_SHARED,
};
bool zap_all = kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&
kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL);
bool flush;
write_lock(&kvm->mmu_lock);
flush = kvm_unmap_gfn_range(kvm, &range);
kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush);
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
if (slot->gmem.file)
kvm_arch_gmem_invalidate_range(kvm, &range);
#endif
if (zap_all) {
__kvm_mmu_zap_all_fast_front_half(kvm);
} else {
flush = kvm_unmap_gfn_range(kvm, &range);
kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush);
}
write_unlock(&kvm->mmu_lock);
}
static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm)
{
return kvm->arch.vm_type == KVM_X86_DEFAULT_VM &&
kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL);
}
void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
struct kvm_memory_slot *slot)
{
if (kvm_memslot_flush_zap_all(kvm))
kvm_mmu_zap_all_fast(kvm);
else
kvm_mmu_zap_memslot(kvm, slot);
if (zap_all)
__kvm_mmu_zap_all_fast_back_half(kvm);
}
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)

197
arch/x86/kvm/svm/sev.c
View File

@ -3998,30 +3998,25 @@ static int snp_begin_psc(struct vcpu_svm *svm)
return snp_do_psc(svm);
}
/*
* Invoked as part of svm_vcpu_reset() processing of an init event.
*/
static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
{
struct vcpu_svm *svm = to_svm(vcpu);
struct kvm_memory_slot *slot;
struct kvm *kvm = vcpu->kvm;
gfn_t gfn = gpa_to_gfn(gpa);
unsigned long mmu_seq;
struct page *page;
kvm_pfn_t pfn;
gfn_t gfn;
guard(mutex)(&svm->sev_es.snp_vmsa_mutex);
lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex);
if (!svm->sev_es.snp_ap_waiting_for_reset)
return;
svm->sev_es.snp_ap_waiting_for_reset = false;
/* Mark the vCPU as offline and not runnable */
vcpu->arch.pv.pv_unhalted = false;
kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED);
/* Clear use of the VMSA */
/*
* Clear use of the VMSA. Ensure snp_guest_vmsa_gpa is written exactly
* once, as it is read locklessly when responding to gfn invalidations.
* Pairs with the READ_ONCE() in sev_gmem_invalidate_range().
*/
svm->vmcb->control.vmsa_pa = INVALID_PAGE;
WRITE_ONCE(svm->sev_es.snp_guest_vmsa_gpa, INVALID_PAGE);
/*
* When replacing the VMSA during SEV-SNP AP creation,
@ -4029,23 +4024,6 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
*/
vmcb_mark_all_dirty(svm->vmcb);
if (!VALID_PAGE(svm->sev_es.snp_vmsa_gpa))
return;
gfn = gpa_to_gfn(svm->sev_es.snp_vmsa_gpa);
svm->sev_es.snp_vmsa_gpa = INVALID_PAGE;
slot = gfn_to_memslot(vcpu->kvm, gfn);
if (!slot)
return;
/*
* The new VMSA will be private memory guest memory, so retrieve the
* PFN from the gmem backend.
*/
if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL))
return;
/*
* From this point forward, the VMSA will always be a guest-mapped page
* rather than the initial one allocated by KVM in svm->sev_es.vmsa. In
@ -4057,20 +4035,83 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
*/
svm->sev_es.snp_has_guest_vmsa = true;
/* Use the new VMSA */
svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
if (!VALID_PAGE(gpa))
return;
/* Mark the vCPU as runnable */
kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
slot = gfn_to_memslot(vcpu->kvm, gfn);
if (!slot)
return;
mmu_seq = kvm->mmu_invalidate_seq;
smp_rmb();
/*
* gmem pages aren't currently migratable, but if this ever changes
* then care should be taken to ensure svm->sev_es.vmsa is pinned
* through some other means.
* The new VMSA will be private memory guest memory, so retrieve the
* PFN from the gmem backend.
*/
if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL))
return;
read_lock(&kvm->mmu_lock);
/*
* Save the guest-provided GPA. If retry is needed, then KVM will try
* again with the same GPA. If the VMSA is usable, then KVM needs to
* track the GPA so that the VMSA can be reloaded if the backing page
* for the GPA is invalidated.
*/
svm->sev_es.snp_guest_vmsa_gpa = gpa;
if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn))
kvm_make_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu);
else
svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
read_unlock(&kvm->mmu_lock);
kvm_release_page_clean(page);
}
/*
* Invoked as part of svm_vcpu_reset() processing of an init event.
*/
static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu)
{
struct vcpu_svm *svm = to_svm(vcpu);
gpa_t gpa;
guard(mutex)(&svm->sev_es.snp_vmsa_mutex);
if (!svm->sev_es.snp_ap_waiting_for_reset)
return;
svm->sev_es.snp_ap_waiting_for_reset = false;
/* Mark the vCPU as offline and not runnable */
vcpu->arch.pv.pv_unhalted = false;
kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED);
gpa = svm->sev_es.snp_pending_vmsa_gpa;
svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE;
__sev_snp_reload_vmsa(vcpu, gpa);
/*
* Mark the vCPU as runnable for CREATE requests, indicated by a valid
* VMSA GPA, even if installing the VMSA failed, so that KVM_RUN will
* fail instead of blocking indefinitely and hanging the vCPU, e.g. if
* the backing guest_memfd page is unavailable.
*/
if (VALID_PAGE(gpa))
kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE);
}
void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu)
{
struct vcpu_sev_es_state *sev_es = &to_svm(vcpu)->sev_es;
guard(mutex)(&sev_es->snp_vmsa_mutex);
__sev_snp_reload_vmsa(vcpu, sev_es->snp_guest_vmsa_gpa);
}
static int sev_snp_ap_creation(struct vcpu_svm *svm)
{
struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm);
@ -4124,10 +4165,10 @@ static int sev_snp_ap_creation(struct vcpu_svm *svm)
return -EINVAL;
}
target_svm->sev_es.snp_vmsa_gpa = svm->vmcb->control.exit_info_2;
target_svm->sev_es.snp_pending_vmsa_gpa = svm->vmcb->control.exit_info_2;
break;
case SVM_VMGEXIT_AP_DESTROY:
target_svm->sev_es.snp_vmsa_gpa = INVALID_PAGE;
target_svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE;
break;
default:
vcpu_unimpl(vcpu, "vmgexit: invalid AP creation request [%#x] from guest\n",
@ -4810,6 +4851,8 @@ int sev_vcpu_create(struct kvm_vcpu *vcpu)
return -ENOMEM;
svm->sev_es.vmsa = page_address(vmsa_page);
svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE;
svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE;
vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm);
@ -5064,15 +5107,7 @@ static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end)
return true;
}
static u8 max_level_for_order(int order)
{
if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M))
return PG_LEVEL_2M;
return PG_LEVEL_4K;
}
static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order)
static bool is_large_rmp_possible(kvm_pfn_t pfn, kvm_pfn_t nr_pages)
{
kvm_pfn_t pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD);
@ -5081,14 +5116,14 @@ static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order)
* PFN is currently shared, then the entire 2M-aligned range can be
* set to private via a single 2M RMP entry.
*/
if (max_level_for_order(order) > PG_LEVEL_4K &&
if (nr_pages >= KVM_PAGES_PER_HPAGE(PG_LEVEL_2M) &&
is_pfn_range_shared(pfn_aligned, pfn_aligned + PTRS_PER_PMD))
return true;
return false;
}
int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages)
{
struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
kvm_pfn_t pfn_aligned;
@ -5099,6 +5134,9 @@ int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
if (!sev_snp_guest(kvm))
return 0;
if (WARN_ON_ONCE(nr_pages != 1))
return -EIO;
rc = snp_lookup_rmpentry(pfn, &assigned, &level);
if (rc) {
pr_err_ratelimited("SEV: Failed to look up RMP entry: GFN %llx PFN %llx error %d\n",
@ -5107,12 +5145,12 @@ int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
}
if (assigned) {
pr_debug("%s: already assigned: gfn %llx pfn %llx max_order %d level %d\n",
__func__, gfn, pfn, max_order, level);
pr_debug("%s: already assigned: gfn %llx pfn %llx nr_pages %llx level %d\n",
__func__, gfn, pfn, nr_pages, level);
return 0;
}
if (is_large_rmp_possible(kvm, pfn, max_order)) {
if (is_large_rmp_possible(pfn, nr_pages)) {
level = PG_LEVEL_2M;
pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD);
gfn_aligned = ALIGN_DOWN(gfn, PTRS_PER_PMD);
@ -5129,22 +5167,22 @@ int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
return -EINVAL;
}
pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx max_order %d level %d\n",
__func__, gfn, pfn, pfn_aligned, max_order, level);
pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx nr_pages %llx level %d\n",
__func__, gfn, pfn, pfn_aligned, nr_pages, level);
return 0;
}
void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages)
{
kvm_pfn_t pfn;
kvm_pfn_t end = pfn + nr_pages;
if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP))
return;
pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, start, end);
pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, pfn, end);
for (pfn = start; pfn < end;) {
while (pfn < end) {
bool use_2m_update = false;
int rc, rmp_level;
bool assigned;
@ -5198,6 +5236,41 @@ void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
}
}
void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range)
{
struct kvm_vcpu *vcpu;
unsigned long i;
lockdep_assert_held_write(&kvm->mmu_lock);
/*
* An unstable result for "is SNP" is a-ok here, thanks to mmu_lock.
* The vCPU's VMSA GPA is invalidated before the vCPU is made visible
* to other tasks, and can only become valid while holding mmu_lock,
* after the VM is fully committed to being an SNP VM.
*/
if (!____sev_snp_guest(kvm))
return;
kvm_for_each_vcpu(i, vcpu, kvm) {
/*
* Read snp_guest_vmsa_gpa without taking the vCPU's VMSA mutex
* (or its generic mutex) as mmu_lock is held, i.e. this task
* can't sleep. The VMSA is invalidated outside of mmu_lock,
* but can only become valid inside of mmu_lock, i.e. the below
* can get false positives, but not false negatives. A false
* positive is benign, as a spurious request simply forces the
* vCPU to re-establish its VMSA.
*/
gpa_t gpa = READ_ONCE(to_svm(vcpu)->sev_es.snp_guest_vmsa_gpa);
if (VALID_PAGE(gpa) &&
gpa_to_gfn(gpa) >= range->start &&
gpa_to_gfn(gpa) < range->end)
kvm_make_request_and_kick(KVM_REQ_VMSA_PAGE_RELOAD, vcpu);
}
}
int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
{
int level, rc;

10
arch/x86/kvm/svm/svm.c
View File

@ -5463,9 +5463,15 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
.mem_enc_register_region = sev_mem_enc_register_region,
.mem_enc_unregister_region = sev_mem_enc_unregister_region,
.guest_memory_reclaimed = sev_guest_memory_reclaimed,
.reload_vmsa = sev_snp_reload_vmsa,
.vm_copy_enc_context_from = sev_vm_copy_enc_context_from,
.vm_move_enc_context_from = sev_vm_move_enc_context_from,
.gmem_make_private = sev_gmem_make_private,
.gmem_make_shared = sev_gmem_make_shared,
.gmem_invalidate_range = sev_gmem_invalidate_range,
.gmem_max_mapping_level = sev_gmem_max_mapping_level,
#endif
.check_emulate_instruction = svm_check_emulate_instruction,
@ -5477,10 +5483,6 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
.vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector,
.vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons,
.alloc_apic_backing_page = svm_alloc_apic_backing_page,
.gmem_prepare = sev_gmem_prepare,
.gmem_invalidate = sev_gmem_invalidate,
.gmem_max_mapping_level = sev_gmem_max_mapping_level,
};
/*

19
arch/x86/kvm/svm/svm.h
View File

@ -272,7 +272,8 @@ struct vcpu_sev_es_state {
u64 ghcb_registered_gpa;
struct mutex snp_vmsa_mutex; /* Used to handle concurrent updates of VMSA. */
gpa_t snp_vmsa_gpa;
gpa_t snp_pending_vmsa_gpa;
gpa_t snp_guest_vmsa_gpa;
bool snp_ap_waiting_for_reset;
bool snp_has_guest_vmsa;
};
@ -996,6 +997,7 @@ static inline struct page *snp_safe_alloc_page(void)
{
return snp_safe_alloc_page_node(numa_node_id(), GFP_KERNEL_ACCOUNT);
}
void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu);
int sev_vcpu_create(struct kvm_vcpu *vcpu);
void sev_free_vcpu(struct kvm_vcpu *vcpu);
@ -1008,8 +1010,9 @@ int sev_cpu_init(struct svm_cpu_data *sd);
int sev_dev_get_attr(u32 group, u64 attr, u64 *val);
extern unsigned int max_sev_asid;
void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code);
int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order);
void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end);
int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages);
void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages);
void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range);
int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private);
struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu);
void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa);
@ -1035,16 +1038,6 @@ static inline int sev_cpu_init(struct svm_cpu_data *sd) { return 0; }
static inline int sev_dev_get_attr(u32 group, u64 attr, u64 *val) { return -ENXIO; }
#define max_sev_asid 0
static inline void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) {}
static inline int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order)
{
return 0;
}
static inline void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) {}
static inline int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private)
{
return 0;
}
static inline struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu)
{
return NULL;

25
arch/x86/kvm/x86.c
View File

@ -3939,7 +3939,10 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
break;
fallthrough;
case KVM_CAP_DISABLE_QUIRKS:
kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks;
mutex_lock(&kvm->lock);
WRITE_ONCE(kvm->arch.disabled_quirks,
kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks));
mutex_unlock(&kvm->lock);
r = 0;
break;
case KVM_CAP_SPLIT_IRQCHIP: {
@ -8167,6 +8170,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
goto out;
}
}
if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu))
kvm_x86_call(reload_vmsa)(vcpu);
}
if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win ||
@ -10590,17 +10595,25 @@ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
return !kvm_arch_has_private_mem(kvm);
}
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE
int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order)
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
kvm_pfn_t nr_pages)
{
return kvm_x86_call(gmem_prepare)(kvm, pfn, gfn, max_order);
return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages);
}
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages)
{
kvm_x86_call(gmem_make_shared)(pfn, nr_pages);
}
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end)
void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range)
{
kvm_x86_call(gmem_invalidate)(start, end);
kvm_x86_call(gmem_invalidate_range)(kvm, range);
}
#endif
#endif

2
arch/x86/kvm/x86.h
View File

@ -304,7 +304,7 @@ static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa)
static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
{
return !(kvm->arch.disabled_quirks & quirk);
return !(READ_ONCE(kvm->arch.disabled_quirks) & quirk);
}
static __always_inline void kvm_request_l1tf_flush_l1d(void)

11
include/linux/kvm_host.h
View File

@ -2573,8 +2573,9 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm,
}
#endif /* CONFIG_KVM_GUEST_MEMFD */
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE
int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order);
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
kvm_pfn_t nr_pages);
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE
@ -2607,8 +2608,12 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src,
kvm_gmem_populate_cb post_populate, void *opaque);
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages);
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end);
void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range);
#endif
#ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY

6
virt/kvm/Kconfig
View File

@ -107,7 +107,11 @@ config KVM_GUEST_MEMFD
select XARRAY_MULTI
bool
config HAVE_KVM_ARCH_GMEM_PREPARE
config HAVE_KVM_ARCH_GMEM_CONVERT
bool
depends on KVM_GUEST_MEMFD
config HAVE_KVM_ARCH_GMEM_RECLAIM
bool
depends on KVM_GUEST_MEMFD

72
virt/kvm/guest_memfd.c
View File

@ -61,52 +61,14 @@ static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn)
return gfn - slot->base_gfn + slot->gmem.pgoff;
}
static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
pgoff_t index, struct folio *folio)
static bool kvm_gmem_is_private_mem(struct inode *inode, pgoff_t index)
{
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE
kvm_pfn_t pfn = folio_file_pfn(folio, index);
gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff;
int rc = kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio));
if (rc) {
pr_warn_ratelimited("gmem: Failed to prepare folio for index %lx GFN %llx PFN %llx error %d.\n",
index, gfn, pfn, rc);
return rc;
}
#endif
return 0;
return !(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED);
}
/*
* Process @folio, which contains @gfn, so that the guest can use it.
* The folio must be locked and the gfn must be contained in @slot.
* On successful return the guest sees a zero page so as to avoid
* leaking host data and the up-to-date flag is set.
*/
static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,
gfn_t gfn, struct folio *folio)
static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index)
{
pgoff_t index;
/*
* Preparing huge folios should always be safe, since it should
* be possible to split them later if needed.
*
* Right now the folio order is always going to be zero, but the
* code is ready for huge folios. The only assumption is that
* the base pgoff of memslots is naturally aligned with the
* requested page order, ensuring that huge folios can also use
* huge page table entries for GPA->HPA mapping.
*
* The order will be passed when creating the guest_memfd, and
* checked when creating memslots.
*/
WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, folio_nr_pages(folio)));
index = kvm_gmem_get_index(slot, gfn);
index = ALIGN_DOWN(index, folio_nr_pages(folio));
return __kvm_gmem_prepare_folio(kvm, slot, index, folio);
return !kvm_gmem_is_private_mem(inode, index);
}
/*
@ -186,6 +148,10 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start,
}
flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range);
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
kvm_arch_gmem_invalidate_range(kvm, &gfn_range);
#endif
}
if (flush)
@ -398,7 +364,7 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf)
if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode))
return VM_FAULT_SIGBUS;
if (!(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED))
if (!kvm_gmem_is_shared_mem(inode, vmf->pgoff))
return VM_FAULT_SIGBUS;
folio = kvm_gmem_get_folio(inode, vmf->pgoff);
@ -524,14 +490,10 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol
return MF_DELAYED;
}
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
static void kvm_gmem_free_folio(struct folio *folio)
{
struct page *page = folio_page(folio, 0);
kvm_pfn_t pfn = page_to_pfn(page);
int order = folio_order(folio);
kvm_arch_gmem_invalidate(pfn, pfn + (1ul << order));
kvm_arch_gmem_reclaim(folio_file_pfn(folio, 0), folio_nr_pages(folio));
}
#endif
@ -539,7 +501,7 @@ static const struct address_space_operations kvm_gmem_aops = {
.dirty_folio = noop_dirty_folio,
.migrate_folio = kvm_gmem_migrate_folio,
.error_remove_folio = kvm_gmem_error_folio,
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
.free_folio = kvm_gmem_free_folio,
#endif
};
@ -794,7 +756,9 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
{
pgoff_t index = kvm_gmem_get_index(slot, gfn);
struct folio *folio;
int r = 0;
int r = 0, __order;
max_order = max_order ?: &__order;
CLASS(gmem_get_file, file)(slot);
if (!file)
@ -809,7 +773,11 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
folio_mark_uptodate(folio);
}
r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio);
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
if (kvm_gmem_is_private_mem(file_inode(file), index))
r = kvm_arch_gmem_make_private(kvm, gfn, *pfn,
(kvm_pfn_t)1 << *max_order);
#endif
folio_unlock(folio);