diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index e75344f4e0cd..410a3eed3a3d 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3159,6 +3159,32 @@ Kernel parameters kvm.enable_vmware_backdoor=[KVM] Support VMware backdoor PV interface. Default is false (don't support). + kvm.gmem_in_place_conversion= + [KVM] Controls whether KVM enables in-place conversion + support for guest_memfd and tracks the private/shared + state of memory per guest_memfd instead of per VM. + + If enabled, KVM enables the KVM_SET_MEMORY_ATTRIBUTES2 + ioctl on guest_memfd file descriptors and disables the + legacy VM-scoped KVM_SET_MEMORY_ATTRIBUTES ioctl for + private memory state tracking. Only the + KVM_MEMORY_ATTRIBUTE_PRIVATE attribute moves to + per-guest_memfd tracking; other attributes remain + per-VM. + + This parameter toggles KVM's in-place conversion + capability support. Whether a VMM uses separate backends + or out-of-place memory management is determined by + userspace VMM design. + + Note, this parameter is only available when + CONFIG_KVM_VM_MEMORY_ATTRIBUTES=y. When + CONFIG_KVM_VM_MEMORY_ATTRIBUTES is not set, in-place + conversion is enabled (and cannot be disabled) if KVM + is built with private memory support. + + Default is N (off). + kvm.nx_huge_pages= [KVM] Controls the software workaround for the X86_BUG_ITLB_MULTIHIT bug. diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index ce01afa2a9a4..4f27ccbc5ba1 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -117,7 +117,7 @@ description: x86 includes both i386 and x86_64. Type: - system, vm, or vcpu. + system, vm, vcpu or guest_memfd. Parameters: what parameters are accepted by the ioctl. @@ -6397,11 +6397,24 @@ mapping for userspace_addr is not required to be valid/populated at the time of KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated on-demand. -When mapping a gfn into the guest, KVM selects shared vs. private, i.e consumes -userspace_addr vs. guest_memfd, based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE -state. At VM creation time, all memory is shared, i.e. the PRIVATE attribute -is '0' for all gfns. Userspace can control whether memory is shared/private by -toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via KVM_SET_MEMORY_ATTRIBUTES as needed. +When mapping a gfn into the guest, guest faults are always serviced from +guest_memfd regardless of whether memory is shared or private. KVM determines +shared vs. private based on the state in guest_memfd, which is the sole +authority on private vs. shared memory. See :ref:`KVM_CREATE_GUEST_MEMFD` to +find out more about the creation-time shared/private status. Userspace can +control whether memory is shared/private by toggling +KVM_MEMORY_ATTRIBUTE_PRIVATE via :ref:`KVM_SET_MEMORY_ATTRIBUTES2` as needed. + +userspace_addr is expected to be the mmap()-ed address corresponding to the +right offset within the guest_memfd. Any mismatch between userspace_addr and +guest_memfd is not validated and is a user error. userspace_addr is only used +for host-side guest accesses such as kvm_read_guest(). + +If in-place conversion is disabled, KVM selects shared vs. private based on the +gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE state. At VM creation time, all memory is +shared, i.e. the PRIVATE attribute is '0' for all gfns. Userspace can control +whether memory is shared/private by toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via +KVM_SET_MEMORY_ATTRIBUTES as needed. S390: ^^^^^ @@ -6409,6 +6422,8 @@ S390: Returns -EINVAL if the VM has the KVM_VM_S390_UCONTROL flag set. Returns -EINVAL if called on a protected VM. +.. _KVM_SET_MEMORY_ATTRIBUTES: + 4.141 KVM_SET_MEMORY_ATTRIBUTES ------------------------------- @@ -6445,6 +6460,10 @@ the state of a gfn/page as needed. The "flags" field is reserved for future extensions and must be '0'. +See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES2`. + +.. _KVM_CREATE_GUEST_MEMFD: + 4.142 KVM_CREATE_GUEST_MEMFD ---------------------------- @@ -6499,10 +6518,10 @@ specified via KVM_CREATE_GUEST_MEMFD. Currently defined flags: page tables. Private memory cannot. ============================ ================================================ -When the KVM MMU performs a PFN lookup to service a guest fault and the backing -guest_memfd has the GUEST_MEMFD_FLAG_MMAP set, then the fault will always be -consumed from guest_memfd, regardless of whether it is a shared or a private -fault. +When the KVM MMU performs a PFN lookup to service a guest fault, the fault will +always be consumed from guest_memfd, regardless of whether it is a shared or a +private fault (unless in-place conversion is disabled and the backing +guest_memfd does not have the GUEST_MEMFD_FLAG_MMAP flag set). See KVM_SET_USER_MEMORY_REGION2 for additional details. @@ -6692,6 +6711,83 @@ significant bit): Userspace should use the defined constants from ```` rather than hardcoding bit positions. +.. _KVM_SET_MEMORY_ATTRIBUTES2: + +4.146 KVM_SET_MEMORY_ATTRIBUTES2 +--------------------------------- + +:Capability: KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES +:Architectures: all +:Type: guest_memfd ioctl +:Parameters: struct kvm_memory_attributes2 (in/out) +:Returns: 0 on success, <0 on error + +Errors: + + ========== =============================================================== + EINVAL The specified `offset` or `size` was invalid (e.g. not + page aligned, causes an overflow, or size is zero). + EFAULT The parameter address was invalid. + ENOMEM Ran out of memory trying to track private/shared state + EAGAIN Pages in the range have outstanding references, see + ``error_offset`` below. + ========== =============================================================== + +KVM_SET_MEMORY_ATTRIBUTES2 is an extension to +KVM_SET_MEMORY_ATTRIBUTES that supports returning (writing) values to +userspace. The original (pre-extension) fields are shared with +KVM_SET_MEMORY_ATTRIBUTES identically. + +Attribute values are shared with KVM_SET_MEMORY_ATTRIBUTES. + +:: + + struct kvm_memory_attributes2 { + union { + __u64 address; + __u64 offset; + }; + __u64 size; + __u64 attributes; + __u64 flags; + __u64 error_offset; + __u64 reserved[11]; + }; + + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) + +The capability KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES enumerates the attributes +that can be set via KVM_SET_MEMORY_ATTRIBUTES2. + +Set attributes for a range of offsets within a guest_memfd to +KVM_MEMORY_ATTRIBUTE_PRIVATE to limit the specified guest_memfd backed +memory range for guest use. Even if the guest_memfd was created with +GUEST_MEMFD_FLAG_MMAP, after a successful call to set +KVM_MEMORY_ATTRIBUTE_PRIVATE, the requested range will not be mappable +into host userspace and will only be mappable by the guest. + +To allow the range to be mappable into host userspace again, call +KVM_SET_MEMORY_ATTRIBUTES2 on the guest_memfd again with +KVM_MEMORY_ATTRIBUTE_PRIVATE unset. + +KVM does not directly manipulate the memory contents of pages during +attribute updates. However, the process of setting these attributes, +which includes operations such as unmapping pages from the host or +stage-2 page tables, may result in side effects on memory contents +that vary across different trusted firmware implementations. + +If this ioctl returns -EAGAIN, the offset of the page with unexpected +refcounts will be returned in ``error_offset``. This can occur if +there are transient refcounts on the pages, taken by other parts of +the kernel. + +Userspace is expected to figure out how to remove all known refcounts +on the shared pages, such as refcounts taken by get_user_pages(), and +try the ioctl again. A possible source of these long term refcounts is +if the guest_memfd memory was pinned in IOMMU page tables. + +See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES`. + .. _kvm_run: 5. The kvm_run structure diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index bd04a908a8db..5977fbe33b98 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -503,7 +503,8 @@ secrets. It is required that the GPA ranges initialized by this command have had the KVM_MEMORY_ATTRIBUTE_PRIVATE attribute set in advance. See the documentation -for KVM_SET_MEMORY_ATTRIBUTES for more details on this aspect. +for KVM_SET_MEMORY_ATTRIBUTES/KVM_SET_MEMORY_ATTRIBUTES2 for more details on +this aspect. Upon success, this command is not guaranteed to have processed the entire range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of @@ -511,9 +512,14 @@ range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of remaining range that has yet to be processed. The caller should continue calling this command until those fields indicate the entire range has been processed, e.g. ``len`` is 0, ``gfn_start`` is equal to the last GFN in the -range plus 1, and ``uaddr`` is the last byte of the userspace-provided source -buffer address plus 1. In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, -``uaddr`` will be ignored completely. +range plus 1, and ``uaddr`` (if specified) is the last byte of the +userspace-provided source buffer address plus 1. + +In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, ``uaddr`` will be +ignored completely. For all other page types, ``uaddr`` is optional if in-place +conversion is enabled (i.e. when the data had been written directly to +guest_memfd while the page was in the shared state) and is required if in-place +conversion is disabled. Parameters (in): struct kvm_sev_snp_launch_update diff --git a/Documentation/virt/kvm/x86/intel-tdx.rst b/Documentation/virt/kvm/x86/intel-tdx.rst index 6a222e9d0954..d8d9409120e6 100644 --- a/Documentation/virt/kvm/x86/intel-tdx.rst +++ b/Documentation/virt/kvm/x86/intel-tdx.rst @@ -158,6 +158,10 @@ KVM_TDX_INIT_MEM_REGION Initialize @nr_pages TDX guest private memory starting from @gpa with userspace provided data from @source_addr. @source_addr must be PAGE_SIZE-aligned. +If guest_memfd in-place conversion is enabled, pass 0 for @source_addr +to represent "no source page". A source page is required if in-place +conversion is not enabled or not supported. + Note, before calling this sub command, memory attribute of the range [gpa, gpa + nr_pages] needs to be private. Userspace can use KVM_SET_MEMORY_ATTRIBUTES to set the attribute. diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 605a9b25f62b..745cb86f9e04 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1773,7 +1773,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ smp_rmb(); - ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, &s2vi.page, NULL); + ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, NULL); if (ret) { /* If result is non-NULL this is a synthetic fault. */ if (!result) @@ -1827,7 +1827,6 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, } out_unlock: - kvm_release_faultin_page(kvm, s2vi.page, !!ret, prot & KVM_PGTABLE_PROT_W); kvm_fault_unlock(kvm); kfree(mapping); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index f0f473978d98..f1c414f1c3fa 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -1559,7 +1559,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) bool write_fault, writable; unsigned long mmu_seq; struct vncr_tlb *vt; - struct page *page; + struct page *page = NULL; u64 va, pfn, gfn; int ret; @@ -1615,7 +1615,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) return -EFAULT; } } else { - ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, NULL); if (ret) { kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE, write_fault, false, false); diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index e213c9ae3e30..67b43c167045 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -150,7 +150,7 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT KVM_X86_OP_OPTIONAL_RET0(gmem_make_private) #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM) KVM_X86_OP_OPTIONAL(gmem_make_shared) #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a902b055c7f2..ab5b90c0d77c 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1738,7 +1738,7 @@ struct kvm_x86_ops { int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM) void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE @@ -1857,9 +1857,14 @@ enum kvm_intr_type { ((vcpu) && (vcpu)->arch.handling_intr_from_guest && \ (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI))) -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#if defined(CONFIG_KVM_SW_PROTECTED_VM) || \ + defined(CONFIG_KVM_INTEL_TDX) || \ + defined(CONFIG_KVM_AMD_SEV) #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) #endif +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() (!!kvm_x86_ops.gmem_make_private) +#endif #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state) diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 42e79c3c87ff..796d33f629d3 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -76,11 +76,22 @@ config KVM_WERROR If in doubt, say "N". +config KVM_VM_MEMORY_ATTRIBUTES + bool "Enable per-VM PRIVATE vs. SHARED attributes (for CoCo VMs)" + depends on KVM_SW_PROTECTED_VM || KVM_INTEL_TDX || KVM_AMD_SEV + help + Enable support for tracking PRIVATE vs. SHARED memory using per-VM + memory attributes. Using per-VM attributes is deprecated in favor of + tracking PRIVATE state in guest_memfd. Select this if you need to run + CoCo VMs using a VMM that doesn't support guest_memfd memory + attributes. + + If unsure, say N. + config KVM_SW_PROTECTED_VM bool "Enable support for KVM software-protected VMs" depends on EXPERT depends on KVM_X86 && X86_64 - select KVM_GENERIC_MEMORY_ATTRIBUTES help Enable support for KVM software-protected VMs. Currently, software- protected VMs are purely a development and testing vehicle for @@ -131,7 +142,6 @@ config KVM_INTEL_TDX bool "Intel Trust Domain Extensions (TDX) support" default y depends on INTEL_TDX_HOST - select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_POPULATE help Provides support for launching Intel Trust Domain Extensions (TDX) @@ -155,7 +165,6 @@ config KVM_AMD_SEV depends on KVM_AMD && X86_64 depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) select ARCH_HAS_CC_PLATFORM - select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_CONVERT select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 9def61fade3b..5a83f09b4d68 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3427,7 +3427,7 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault, is_private = fault->is_private; } else { max_level = PG_LEVEL_NUM; - is_private = kvm_mem_is_private(kvm, gfn); + is_private = kvm_is_private_gfn(kvm, gfn); } max_level = min(max_level, max_huge_page_level); @@ -3640,13 +3640,13 @@ static bool page_fault_can_be_fast(struct kvm *kvm, struct kvm_page_fault *fault * guest spinning on a #PF indefinitely, so don't attempt the fast path * in this case. * - * Note that the kvm_mem_is_private() check might race with an + * Note that the kvm_is_private_gfn() check might race with an * attribute update, but this will either result in the guest spinning * on RET_PF_SPURIOUS until the update completes, or an actual spurious * case might go down the slow path. Either case will resolve itself. */ if (kvm->arch.has_private_mem && - fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) + fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) return false; /* @@ -4634,7 +4634,7 @@ static int kvm_mmu_faultin_pfn_gmem(struct kvm_vcpu *vcpu, } r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn, - &fault->refcounted_page, &max_order); + &max_order); if (r) { kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return r; @@ -4714,7 +4714,7 @@ static int kvm_mmu_faultin_pfn(struct kvm_vcpu *vcpu, * Now that we have a snapshot of mmu_invalidate_seq we can check for a * private vs. shared mismatch. */ - if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) { + if (fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) { kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return -EFAULT; } @@ -5122,7 +5122,7 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, direct_bits = 0; if (kvm_arch_has_private_mem(vcpu->kvm) && - kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa))) + kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(range->gpa))) error_code |= PFERR_PRIVATE_ACCESS; else direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm)); @@ -6595,7 +6595,7 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && !(error_code & PFERR_RSVD_MASK) && vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM && - kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa))) + kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(cr2_or_gpa))) error_code |= PFERR_PRIVATE_ACCESS; r = RET_PF_INVALID; @@ -8108,7 +8108,7 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread); } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, int level) { @@ -8127,8 +8127,8 @@ static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; } -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { struct kvm_memory_slot *slot = range->slot; int level; @@ -8197,18 +8197,18 @@ static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot, const unsigned long end = start + KVM_PAGES_PER_HPAGE(level); if (level == PG_LEVEL_2M) - return kvm_range_has_memory_attributes(kvm, start, end, ~0, attrs); + return kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attrs); for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) { if (hugepage_test_mixed(slot, gfn, level - 1) || - attrs != kvm_get_memory_attributes(kvm, gfn)) + attrs != kvm_get_vm_memory_attributes(kvm, gfn)) return false; } return true; } -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { unsigned long attrs = range->arg.attributes; struct kvm_memory_slot *slot = range->slot; @@ -8301,7 +8301,7 @@ void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm, * be manually checked as the attributes may already be mixed. */ for (gfn = start; gfn < end; gfn += nr_pages) { - unsigned long attrs = kvm_get_memory_attributes(kvm, gfn); + unsigned long attrs = kvm_get_vm_memory_attributes(kvm, gfn); if (hugepage_has_attrs(kvm, slot, gfn, level, attrs)) hugepage_clear_mixed(slot, gfn, level); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 3309960af486..5c421eaa5c07 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2416,7 +2416,7 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, */ if (ret && !snp_page_reclaim(kvm, pfn) && sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID && - sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) { + sev_populate_args->fw_error == SEV_RET_INVALID_PARAM && src_page) { void *src_vaddr = kmap_local_page(src_page); void *dst_vaddr = kmap_local_pfn(pfn); @@ -2449,8 +2449,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) return -EFAULT; - pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d\n", __func__, - params.gfn_start, params.len, params.type, params.flags); + pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d src %llx\n", __func__, + params.gfn_start, params.len, params.type, params.flags, params.uaddr); if (!params.len || !PAGE_ALIGNED(params.len) || params.flags || (params.type != KVM_SEV_SNP_PAGE_TYPE_NORMAL && @@ -2462,7 +2462,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO) src = NULL; - else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr)) + else if ((!gmem_in_place_conversion && !params.uaddr) || + !PAGE_ALIGNED(params.uaddr)) return -EINVAL; else src = u64_to_user_ptr(params.uaddr); @@ -2509,7 +2510,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) params.gfn_start += count; params.len -= count * PAGE_SIZE; - if (params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) + if (src && params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) params.uaddr += count * PAGE_SIZE; if (copy_to_user(u64_to_user_ptr(argp->data), ¶ms, sizeof(params))) @@ -4047,7 +4048,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) struct kvm *kvm = vcpu->kvm; gfn_t gfn = gpa_to_gfn(gpa); unsigned long mmu_seq; - struct page *page; kvm_pfn_t pfn; lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); @@ -4091,7 +4091,7 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) * The new VMSA will be private memory guest memory, so retrieve the * PFN from the gmem backend. */ - if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL)) + if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, NULL)) return; read_lock(&kvm->mmu_lock); @@ -4107,8 +4107,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) else svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); read_unlock(&kvm->mmu_lock); - - kvm_release_page_clean(page); } /* @@ -5034,7 +5032,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) struct kvm_memory_slot *slot; struct kvm *kvm = vcpu->kvm; int order, rmp_level, ret; - struct page *page; + unsigned long mmu_seq; bool assigned; kvm_pfn_t pfn; gfn_t gfn; @@ -5048,7 +5046,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) * userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults * for shared pages should not end up here. */ - if (!kvm_mem_is_private(kvm, gfn)) { + if (!kvm_is_private_gfn(kvm, gfn)) { pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n", gpa); return; @@ -5061,7 +5059,10 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) return; } - ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &page, &order); + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + + ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &order); if (ret) { pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n", gpa); @@ -5070,9 +5071,13 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); if (ret || !assigned) { - pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", - gpa, pfn, ret); - goto out_no_trace; + guard(read_lock)(&kvm->mmu_lock); + + if (!mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) + pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", + gpa, pfn, ret); + + return; } /* @@ -5100,26 +5105,31 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) if (rmp_level == PG_LEVEL_4K) goto out; - ret = snp_rmptable_psmash(pfn); - if (ret) { - /* - * Look it up again. If it's 4K now then the PSMASH may have - * raced with another process and the issue has already resolved - * itself. - */ - if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && - assigned && rmp_level == PG_LEVEL_4K) + scoped_guard(read_lock, &kvm->mmu_lock) { + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) goto out; - pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", - gpa, pfn, ret); + ret = snp_rmptable_psmash(pfn); + if (ret) { + /* + * Look it up again. If it's 4K now then the PSMASH may + * have raced with another process and the issue has + * already resolved itself. If it's not assigned, then + * this must have raced with another process that made + * this page shared. + */ + if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && + ((assigned && rmp_level == PG_LEVEL_4K) || !assigned)) + goto out; + + pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", + gpa, pfn, ret); + } } kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD); out: trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret); -out_no_trace: - kvm_release_page_unused(page); } static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 2cf454ab08ab..e7950b98b4bb 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3184,7 +3184,7 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm)) return -EIO; - kvm_tdx->page_add_src = src_page; + kvm_tdx->page_add_src = src_page ?: pfn_to_page(pfn); ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn); kvm_tdx->page_add_src = NULL; @@ -3230,7 +3230,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c if (copy_from_user(®ion, u64_to_user_ptr(cmd->data), sizeof(region))) return -EFAULT; - if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr || + if (!PAGE_ALIGNED(region.source_addr) || + (!gmem_in_place_conversion && !region.source_addr) || !PAGE_ALIGNED(region.gpa) || !region.nr_pages || region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || !vt_is_tdx_private_gpa(kvm, region.gpa) || @@ -3261,7 +3262,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c break; } - region.source_addr += PAGE_SIZE; + if (region.source_addr) + region.source_addr += PAGE_SIZE; region.gpa += PAGE_SIZE; region.nr_pages--; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index aad065d035fb..40ebfc41ce72 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10144,7 +10144,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm, } } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES kvm_mmu_init_memslot_memory_attributes(kvm, slot); #endif @@ -10641,14 +10641,15 @@ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) } #ifdef CONFIG_KVM_GUEST_MEMFD -/* - * KVM doesn't yet support initializing guest_memfd memory as shared for VMs - * with private memory (the private vs. shared tracking needs to be moved into - * guest_memfd). - */ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) { - return !kvm_arch_has_private_mem(kvm); + /* + * INIT_SHARED is supported if in-place conversion is enabled, or if + * the VM doesn't support private memory. If the VM has private memory + * and in-place conversion is disabled, then guest_memfd can _only_ be + * used for private memory. + */ + return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm); } #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT @@ -10657,6 +10658,11 @@ int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, { return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); } + +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +{ + kvm_x86_call(gmem_make_shared)(pfn, nr_pages); +} #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 864c1f769083..4d4e76a50d77 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -726,27 +726,6 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) } #endif -#ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static inline bool kvm_arch_has_private_mem(struct kvm *kvm) -{ - return false; -} -#endif - -#ifdef CONFIG_KVM_GUEST_MEMFD -bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); - -static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) -{ - u64 flags = GUEST_MEMFD_FLAG_MMAP; - - if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) - flags |= GUEST_MEMFD_FLAG_INIT_SHARED; - - return flags; -} -#endif - #ifndef kvm_arch_has_readonly_mem static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm) { @@ -887,7 +866,7 @@ struct kvm { #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER struct notifier_block pm_notifier; #endif -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* Protected by slots_lock (for writes) and RCU (for reads) */ struct xarray mem_attr_array; #endif @@ -2591,48 +2570,80 @@ static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot) return slot->flags & KVM_MEMSLOT_GMEM_ONLY; } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +static inline unsigned long kvm_get_vm_memory_attributes(struct kvm *kvm, gfn_t gfn) { return xa_to_value(xa_load(&kvm->mem_attr_array, gfn)); } -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs); -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs); +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +static inline bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { - return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; + return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; +} +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ + +#ifdef kvm_arch_has_private_mem +extern bool gmem_in_place_conversion; + +typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn); +DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t); + +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return static_call(__kvm_is_private_gfn)(kvm, gfn); } #else -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +#define gmem_in_place_conversion false + +static inline bool kvm_arch_has_private_mem(struct kvm *kvm) { return false; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ + +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return false; +} +#endif /* kvm_arch_has_private_mem */ #ifdef CONFIG_KVM_GUEST_MEMFD +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn); +bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); + +static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) +{ + u64 flags = GUEST_MEMFD_FLAG_MMAP; + + if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) + flags |= GUEST_MEMFD_FLAG_INIT_SHARED; + + return flags; +} + int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order); + gfn_t gfn, kvm_pfn_t *pfn, int *max_order); #else static inline int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, - kvm_pfn_t *pfn, struct page **page, - int *max_order) + kvm_pfn_t *pfn, int *max_order) { KVM_BUG_ON(1, kvm); return -EIO; } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#ifndef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() false #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h index f37582730ffc..07013f88e655 100644 --- a/include/trace/events/kvm.h +++ b/include/trace/events/kvm.h @@ -364,13 +364,13 @@ TRACE_EVENT(kvm_dirty_ring_exit, TP_printk("vcpu %d", __entry->vcpu_id) ); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* * @start: Starting address of guest memory range * @end: End address of guest memory range * @attr: The value of the attribute being set. */ -TRACE_EVENT(kvm_vm_set_mem_attributes, +TRACE_EVENT(kvm_set_vm_mem_attributes, TP_PROTO(gfn_t start, gfn_t end, unsigned long attr), TP_ARGS(start, end, attr), @@ -389,7 +389,7 @@ TRACE_EVENT(kvm_vm_set_mem_attributes, TP_printk("%#016llx -- %#016llx [0x%lx]", __entry->start, __entry->end, __entry->attr) ); -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ TRACE_EVENT(kvm_unmap_hva_range, TP_PROTO(unsigned long start, unsigned long end), diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index ac2d77d14963..8dff2fc1972e 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -999,6 +999,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_HPAGE_2G 249 #define KVM_CAP_PPC_COMPAT_CAPS 250 #define KVM_CAP_ARM_PMU_V3_STRICT 251 +#define KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES 252 struct kvm_irq_routing_irqchip { __u32 irqchip; @@ -1653,6 +1654,21 @@ struct kvm_memory_attributes { __u64 flags; }; +/* Available with KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES */ +#define KVM_SET_MEMORY_ATTRIBUTES2 _IOWR(KVMIO, 0xd2, struct kvm_memory_attributes2) + +struct kvm_memory_attributes2 { + union { + __u64 address; + __u64 offset; + }; + __u64 size; + __u64 attributes; + __u64 flags; + __u64 error_offset; + __u64 reserved[11]; +}; + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) #define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 1f4bc2a1322d..42b50f7badad 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -156,6 +156,7 @@ TEST_GEN_PROGS_x86 += x86/max_vcpuid_cap_test TEST_GEN_PROGS_x86 += x86/triple_fault_event_test TEST_GEN_PROGS_x86 += x86/recalc_apic_map_test TEST_GEN_PROGS_x86 += x86/aperfmperf_test +TEST_GEN_PROGS_x86 += x86/guest_memfd_conversions_test TEST_GEN_PROGS_x86 += access_tracking_perf_test TEST_GEN_PROGS_x86 += coalesced_io_test TEST_GEN_PROGS_x86 += dirty_log_perf_test diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 5a7a455b5387..777fa3dbf88d 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -392,6 +392,16 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ }) +#define __gmem_ioctl(gmem_fd, cmd, arg) \ + kvm_do_ioctl(gmem_fd, cmd, arg) + +#define gmem_ioctl(gmem_fd, cmd, arg) \ +({ \ + int ret = __gmem_ioctl(gmem_fd, cmd, arg); \ + \ + TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ +}) + /* * Looks up and returns the value corresponding to the capability * (KVM_CAP_*) given by cap. @@ -418,8 +428,19 @@ static inline void vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes); + +/* + * KVM_SET_MEMORY_ATTRIBUTES{,2} overwrites _all_ attributes. These + * flows need significant enhancements to support multiple attributes. + */ +#define TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes) \ + TEST_ASSERT(!(attributes) || (attributes) == KVM_MEMORY_ATTRIBUTE_PRIVATE, \ + "Update me to support multiple attributes!") + static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, - u64 size, u64 attributes) + size_t size, u64 attributes) { struct kvm_memory_attributes attr = { .attributes = attributes, @@ -428,27 +449,109 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, .flags = 0, }; - /* - * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows - * need significant enhancements to support multiple attributes. - */ - TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, - "Update me to support multiple attributes!"); + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); } +static inline int __gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes, + u64 *error_offset) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + .error_offset = 0, + }; + int r; + + r = __gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); + + /* Copy error_offset regardless of r so caller can check. */ + if (error_offset) + *error_offset = attr.error_offset; + + return r; +} + +static inline int __gmem_set_private(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE, + error_offset); +} + +static inline int __gmem_set_shared(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, 0, + error_offset); +} + +static inline void gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + }; + + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); + + __TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) > 0, + "No valid attributes for guest_memfd ioctl!"); + + gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); +} + +static inline void gmem_set_private(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); +} + +static inline void gmem_set_shared(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, 0); +} + +static inline void vm_mem_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, + size_t size, u64 attrs) +{ + if (kvm_has_gmem_attributes) { + gpa_t end = gpa + size; + off_t fd_offset; + gpa_t addr; + size_t len; + int fd; + + for (addr = gpa; addr < end; addr += len) { + fd = kvm_gpa_to_guest_memfd(vm, addr, &fd_offset, &len); + len = min(end - addr, len); + + gmem_set_memory_attributes(fd, fd_offset, len, attrs); + } + } else { + vm_set_memory_attributes(vm, gpa, size, attrs); + } +} static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa, - u64 size) + size_t size) { - vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); + vm_mem_set_memory_attributes(vm, gpa, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); } static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa, - u64 size) + size_t size) { - vm_set_memory_attributes(vm, gpa, size, 0); + vm_mem_set_memory_attributes(vm, gpa, size, 0); } void vm_guest_mem_fallocate(struct kvm_vm *vm, gpa_t gpa, u64 size, @@ -690,17 +793,17 @@ int __vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva); void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags); void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd_fd, u64 guest_memfd_offset); + int gmem_fd, u64 gmem_offset, u64 gmem_flags); #ifndef vm_arch_has_protected_memory static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) @@ -732,6 +835,11 @@ void *addr_gva2hva(struct kvm_vm *vm, gva_t gva); gpa_t addr_hva2gpa(struct kvm_vm *vm, void *hva); void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa); +static inline int kvm_slot_to_fd(struct kvm_vm *vm, u32 slot) +{ + return memslot2region(vm, slot)->fd; +} + #ifndef vcpu_arch_put_guest #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) #endif @@ -1123,6 +1231,9 @@ static inline int pin_self_to_any_cpu(void) return pin_task_to_any_cpu(pthread_self()); } +void pin_pages(void *vaddr, uint64_t size); +void unpin_pages(void); + void kvm_print_vcpu_pinning_help(void); void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[], int nr_vcpus); diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index 76bb11bb79f3..b9c4af6a420d 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -83,21 +83,23 @@ do { \ __builtin_unreachable(); \ } while (0) -extern sigjmp_buf expect_sigbus_jmpbuf; -void expect_sigbus_handler(int signum); +extern __thread sigjmp_buf expect_sigbus_jmpbuf; +extern __thread volatile sig_atomic_t expecting_sigbus; +void catchall_signal_handler(int signum); -#define TEST_EXPECT_SIGBUS(action) \ -do { \ - struct sigaction sa_old, sa_new = { \ - .sa_handler = expect_sigbus_handler, \ - }; \ - \ - sigaction(SIGBUS, &sa_new, &sa_old); \ - if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ - action; \ - TEST_FAIL("'%s' should have triggered SIGBUS", #action); \ - } \ - sigaction(SIGBUS, &sa_old, NULL); \ +#define TEST_EXPECT_SIGBUS(action) \ +do { \ + struct sigaction __sa = {}; \ + \ + TEST_ASSERT_EQ(sigaction(SIGBUS, NULL, &__sa), 0); \ + TEST_ASSERT_EQ(__sa.sa_handler, &catchall_signal_handler); \ + \ + expecting_sigbus = true; \ + if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ + action; \ + TEST_FAIL("'%s' should have triggered SIGBUS", #action);\ + } \ + expecting_sigbus = false; \ } while (0) size_t parse_size(const char *size); @@ -116,6 +118,8 @@ struct kvm_random_state { extern u32 kvm_random_seed; extern struct kvm_random_state kvm_rng; +extern bool kvm_has_gmem_attributes; + struct kvm_random_state new_kvm_random_state(u32 seed); u32 kvm_random_u32(struct kvm_random_state *state); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9ddc047d5c27..0c67d964a9aa 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -19,6 +19,8 @@ #include #include +#include "../../../../../mm/gup_test.h" + #define KVM_UTIL_MIN_PFN 2 u32 kvm_random_seed; @@ -32,6 +34,8 @@ static void kvm_seed_rng(u32 seed) kvm_rng = new_kvm_random_state(kvm_random_seed); } +bool kvm_has_gmem_attributes; + static size_t vcpu_mmap_sz(void); int __open_path_or_exit(const char *path, int flags, const char *enoent_help) @@ -490,6 +494,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, u64 nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus, nr_extra_pages); struct userspace_mem_region *slot0; + u64 gmem_flags = 0; struct kvm_vm *vm; int i, flags; @@ -505,10 +510,20 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, * for CoCo VMs that require GUEST_MEMFD backed private memory. */ flags = 0; - if (is_guest_memfd_required(shape)) + if (is_guest_memfd_required(shape)) { flags |= KVM_MEM_GUEST_MEMFD; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags); + /* + * Allow mmap() and initialize the gmem instance as SHARED if + * in-place conversions are supported, so that the memory can + * be populated/initialized like any other memory region. + */ + if (kvm_has_gmem_attributes) + gmem_flags |= GUEST_MEMFD_FLAG_INIT_SHARED | + GUEST_MEMFD_FLAG_MMAP; + } + + vm_mem_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags, -1, 0, gmem_flags); for (i = 0; i < NR_MEM_REGIONS; i++) vm->memslots[i] = 0; @@ -527,6 +542,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, kvm_seed_rng(kvm_random_seed); sync_global_to_guest(vm, kvm_rng); + sync_global_to_guest(vm, kvm_has_gmem_attributes); kvm_arch_vm_post_create(vm, nr_runnable_vcpus); @@ -642,6 +658,29 @@ int __pin_task_to_cpu(pthread_t task, int cpu) return pthread_setaffinity_np(task, sizeof(cpuset), &cpuset); } +static int gup_test_fd = -1; + +void pin_pages(void *vaddr, uint64_t size) +{ + const struct pin_longterm_test args = { + .addr = (uint64_t)vaddr, + .size = size, + .flags = PIN_LONGTERM_TEST_FLAG_USE_WRITE, + }; + + gup_test_fd = __open_path_or_exit("/sys/kernel/debug/gup_test", O_RDWR, + "Is CONFIG_GUP_TEST enabled?"); + + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_START, &args), 0); +} + +void unpin_pages(void) +{ + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_STOP), 0); + + kvm_free_fd(gup_test_fd); +} + static u32 parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask) { u32 pcpu = atoi_non_negative("CPU number", cpu_str); @@ -970,7 +1009,7 @@ void vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset) + u32 gmem_fd, u64 gmem_offset) { struct kvm_userspace_memory_region2 region = { .slot = slot, @@ -978,8 +1017,8 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, .guest_phys_addr = gpa, .memory_size = size, .userspace_addr = (uintptr_t)hva, - .guest_memfd = guest_memfd, - .guest_memfd_offset = guest_memfd_offset, + .guest_memfd = gmem_fd, + .guest_memfd_offset = gmem_offset, }; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -989,10 +1028,10 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset) + u32 gmem_fd, u64 gmem_offset) { int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva, - guest_memfd, guest_memfd_offset); + gmem_fd, gmem_offset); TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)", errno, strerror(errno)); @@ -1002,13 +1041,16 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, /* FIXME: This thing needs to be ripped apart and rewritten. */ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd, u64 guest_memfd_offset) + int gmem_fd, u64 gmem_offset, u64 gmem_flags) { int ret; struct userspace_mem_region *region; - size_t backing_src_pagesz = get_backing_src_pagesz(src_type); size_t mem_size = npages * vm->page_size; - size_t alignment = 1; + size_t backing_src_pagesz; + off_t mmap_offset; + bool is_gmem_mmap; + size_t alignment; + int mmap_flags; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -1060,58 +1102,40 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, /* Allocate and initialize new mem region structure. */ region = calloc(1, sizeof(*region)); TEST_ASSERT(region != NULL, "Insufficient Memory"); - region->mmap_size = mem_size; - /* - * When using THP mmap is not guaranteed to returned a hugepage aligned - * address so we have to pad the mmap. Padding is not needed for HugeTLB - * because mmap will always return an address aligned to the HugeTLB - * page size. - */ - if (src_type == VM_MEM_SRC_ANONYMOUS_THP) - alignment = max(backing_src_pagesz, alignment); + is_gmem_mmap = (flags & KVM_MEM_GUEST_MEMFD) && + (gmem_flags & GUEST_MEMFD_FLAG_MMAP); + + if (is_gmem_mmap) { + backing_src_pagesz = getpagesize(); + alignment = 1; + mmap_flags = MAP_SHARED; + mmap_offset = gmem_offset; + } else { + backing_src_pagesz = get_backing_src_pagesz(src_type); + /* + * When using THP mmap is not guaranteed to returned a hugepage aligned + * address so we have to pad the mmap. Padding is not needed for HugeTLB + * because mmap will always return an address aligned to the HugeTLB + * page size. + */ + alignment = src_type == VM_MEM_SRC_ANONYMOUS_THP ? backing_src_pagesz : 1; + mmap_flags = vm_mem_backing_src_alias(src_type)->flag; + mmap_offset = 0; + } TEST_ASSERT_EQ(gpa, align_up(gpa, backing_src_pagesz)); + region->mmap_size = mem_size; /* Add enough memory to align up if necessary */ if (alignment > 1) region->mmap_size += alignment; - region->fd = -1; - if (backing_src_is_shared(src_type)) - region->fd = kvm_memfd_alloc(region->mmap_size, - src_type == VM_MEM_SRC_SHARED_HUGETLB); - - region->mmap_start = kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd); - - TEST_ASSERT(!is_backing_src_hugetlb(src_type) || - region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz), - "mmap_start %p is not aligned to HugeTLB page size 0x%lx", - region->mmap_start, backing_src_pagesz); - - /* Align host address */ - region->host_mem = align_ptr_up(region->mmap_start, alignment); - - /* As needed perform madvise */ - if ((src_type == VM_MEM_SRC_ANONYMOUS || - src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { - ret = madvise(region->host_mem, mem_size, - src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); - TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", - region->host_mem, mem_size, - vm_mem_backing_src_alias(src_type)->name); - } - - region->backing_src_type = src_type; - if (flags & KVM_MEM_GUEST_MEMFD) { - if (guest_memfd < 0) { - u32 guest_memfd_flags = 0; - TEST_ASSERT(!guest_memfd_offset, + if (gmem_fd < 0) { + TEST_ASSERT(!gmem_offset, "Offset must be zero when creating new guest_memfd"); - guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags); + gmem_fd = vm_create_guest_memfd(vm, mem_size, gmem_flags); } else { /* * Install a unique fd for each memslot so that the fd @@ -1119,15 +1143,51 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, * needing to track if the fd is owned by the framework * or by the caller. */ - guest_memfd = kvm_dup(guest_memfd); + gmem_fd = kvm_dup(gmem_fd); } - region->region.guest_memfd = guest_memfd; - region->region.guest_memfd_offset = guest_memfd_offset; + region->region.guest_memfd = gmem_fd; + region->region.guest_memfd_offset = gmem_offset; } else { region->region.guest_memfd = -1; } + region->fd = -1; + if (is_gmem_mmap) { + region->fd = kvm_dup(gmem_fd); + } else if (backing_src_is_shared(src_type)) { + region->fd = kvm_memfd_alloc(region->mmap_size, + src_type == VM_MEM_SRC_SHARED_HUGETLB); + } + + region->mmap_start = __kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, + mmap_flags, region->fd, mmap_offset); + + /* Align host address */ + region->host_mem = align_ptr_up(region->mmap_start, alignment); + + if (!is_gmem_mmap) { + TEST_ASSERT(!is_backing_src_hugetlb(src_type) || + region->mmap_start == + align_ptr_up(region->mmap_start, backing_src_pagesz), + "mmap_start %p is not aligned to HugeTLB page size 0x%lx", + region->mmap_start, backing_src_pagesz); + + /* As needed perform madvise */ + if ((src_type == VM_MEM_SRC_ANONYMOUS || + src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { + int advice = src_type == VM_MEM_SRC_ANONYMOUS ? + MADV_NOHUGEPAGE : MADV_HUGEPAGE; + + ret = madvise(region->host_mem, mem_size, advice); + TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", + region->host_mem, mem_size, + vm_mem_backing_src_alias(src_type)->name); + } + } + + region->backing_src_type = src_type; + region->unused_phy_pages = sparsebit_alloc(); if (vm_arch_has_protected_memory(vm)) region->protected_phy_pages = sparsebit_alloc(); @@ -1152,10 +1212,10 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, /* If shared memory, create an alias. */ if (region->fd >= 0) { - region->mmap_alias = kvm_mmap(region->mmap_size, - PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd); + region->mmap_alias = __kvm_mmap(region->mmap_size, + PROT_READ | PROT_WRITE, + mmap_flags, region->fd, + mmap_offset); /* Align host alias address */ region->host_alias = align_ptr_up(region->mmap_alias, alignment); @@ -1166,7 +1226,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags) { - vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0); + vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0, 0); } /* @@ -1299,27 +1359,20 @@ void vm_guest_mem_fallocate(struct kvm_vm *vm, u64 base, u64 size, bool punch_hole) { const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0); - struct userspace_mem_region *region; u64 end = base + size; - gpa_t gpa, len; off_t fd_offset; - int ret; + int fd, ret; + size_t len; + gpa_t gpa; for (gpa = base; gpa < end; gpa += len) { - u64 offset; + fd = kvm_gpa_to_guest_memfd(vm, gpa, &fd_offset, &len); + len = min(end - gpa, len); - region = userspace_mem_region_find(vm, gpa, gpa); - TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, - "Private memory region not found for GPA 0x%lx", gpa); - - offset = gpa - region->region.guest_phys_addr; - fd_offset = region->region.guest_memfd_offset + offset; - len = min_t(u64, end - gpa, region->region.memory_size - offset); - - ret = fallocate(region->region.guest_memfd, mode, fd_offset, len); + ret = fallocate(fd, mode, fd_offset, len); TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx", punch_hole ? "punch hole" : "allocate", gpa, len, - region->region.guest_memfd, mode, fd_offset); + fd, mode, fd_offset); } } @@ -1656,6 +1709,22 @@ void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa) return (void *) ((uintptr_t) region->host_alias + offset); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes) +{ + struct userspace_mem_region *region; + gpa_t gpa_offset; + + region = userspace_mem_region_find(vm, gpa, gpa); + TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, + "guest_memfd memory region not found for GPA 0x%lx", gpa); + + gpa_offset = gpa - region->region.guest_phys_addr; + *fd_offset = region->region.guest_memfd_offset + gpa_offset; + *nr_bytes = region->region.memory_size - gpa_offset; + return region->region.guest_memfd; +} + /* Create an interrupt controller chip for the specified VM. */ void vm_create_irqchip(struct kvm_vm *vm) { @@ -2274,13 +2343,20 @@ __weak void kvm_selftest_arch_init(void) { } -static void report_unexpected_signal(int signum) +__thread sigjmp_buf expect_sigbus_jmpbuf; +__thread volatile sig_atomic_t expecting_sigbus; + +void catchall_signal_handler(int signum) { + switch (signum) { + case SIGBUS: { + if (expecting_sigbus) + siglongjmp(expect_sigbus_jmpbuf, 1); + + TEST_FAIL("Unexpected SIGBUS (%d)\n", signum); + } #define KVM_CASE_SIGNUM(sig) \ case sig: TEST_FAIL("Unexpected " #sig " (%d)\n", signum) - - switch (signum) { - KVM_CASE_SIGNUM(SIGBUS); KVM_CASE_SIGNUM(SIGSEGV); KVM_CASE_SIGNUM(SIGILL); KVM_CASE_SIGNUM(SIGFPE); @@ -2292,12 +2368,13 @@ static void report_unexpected_signal(int signum) void __attribute((constructor)) kvm_selftest_init(void) { struct sigaction sig_sa = { - .sa_handler = report_unexpected_signal, + .sa_handler = catchall_signal_handler, }; /* Tell stdout not to buffer its content. */ setbuf(stdout, NULL); + expecting_sigbus = false; sigaction(SIGBUS, &sig_sa, NULL); sigaction(SIGSEGV, &sig_sa, NULL); sigaction(SIGILL, &sig_sa, NULL); @@ -2306,6 +2383,8 @@ void __attribute((constructor)) kvm_selftest_init(void) srandom(time(0)); kvm_seed_rng(random()); + kvm_has_gmem_attributes = kvm_has_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES); + kvm_selftest_arch_init(); } diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index f9263a715704..f5f7189f96a8 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -19,13 +19,6 @@ #include "test_util.h" #include "kvm_syscalls.h" -sigjmp_buf expect_sigbus_jmpbuf; - -void __attribute__((used)) expect_sigbus_handler(int signum) -{ - siglongjmp(expect_sigbus_jmpbuf, 1); -} - /* * Random number generator that is usable from guest code. This is the * Park-Miller LCG using standard constants. diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c new file mode 100644 index 000000000000..e1c84df0f059 --- /dev/null +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, Google LLC. + */ +#include +#include +#include +#include + +#include +#include +#include + +#include "kvm_util.h" +#include "kvm_syscalls.h" +#include "kselftest_harness.h" +#include "test_util.h" +#include "ucall_common.h" + +FIXTURE(gmem_conversions) { + struct kvm_vcpu *vcpu; + int gmem_fd; + /* HVA of the first byte of the memory mmap()-ed from gmem_fd. */ + char *mem; +}; + +typedef FIXTURE_DATA(gmem_conversions) test_data_t; + +FIXTURE_SETUP(gmem_conversions) { } + +static size_t page_size; + +static void guest_do_rmw(void); +#define GUEST_MEMFD_SHARING_TEST_GVA 0x90000000ULL + +/* + * Defer setup until the individual test is invoked so that tests can specify + * the number of pages and flags for the guest_memfd instance. + */ +static void gmem_conversions_do_setup(test_data_t *t, int nr_pages, + int gmem_flags) +{ + const struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, + }; + /* + * Use high GPA above APIC_DEFAULT_PHYS_BASE to avoid clashing with + * APIC_DEFAULT_PHYS_BASE. + */ + const gpa_t gpa = SZ_4G; + const u32 slot = 1; + struct kvm_vm *vm; + + vm = __vm_create_shape_with_one_vcpu(shape, &t->vcpu, nr_pages, guest_do_rmw); + + vm_mem_add(vm, VM_MEM_SRC_SHMEM, gpa, slot, nr_pages, + KVM_MEM_GUEST_MEMFD, -1, 0, gmem_flags); + + t->gmem_fd = kvm_slot_to_fd(vm, slot); + t->mem = addr_gpa2hva(vm, gpa); + virt_map(vm, GUEST_MEMFD_SHARING_TEST_GVA, gpa, nr_pages); +} + +static void gmem_conversions_do_teardown(test_data_t *t) +{ + /* Use NULL to avoid second free in FIXTURE_TEARDOWN (multipage tests). */ + if (!t->vcpu) + return; + + /* No need to close gmem_fd, it's owned by the VM structure. */ + kvm_vm_free(t->vcpu->vm); + t->vcpu = NULL; +} + +FIXTURE_TEARDOWN(gmem_conversions) +{ + gmem_conversions_do_teardown(self); +} + +/* + * In these test definition macros, __nr_pages and nr_pages is used to set up + * the total number of pages in the guest_memfd under test. This will be + * available in the test definitions as nr_pages. + */ + +#define __GMEM_CONVERSION_TEST(test, __nr_pages, flags) \ +static void __gmem_conversions_##test(test_data_t *t, int nr_pages); \ + \ +TEST_F(gmem_conversions, test) \ +{ \ + const int nr = (__nr_pages); \ + \ + gmem_conversions_do_setup(self, nr, flags); \ + __gmem_conversions_##test(self, nr); \ +} \ +static void __gmem_conversions_##test(test_data_t *t, int nr_pages) \ + +#define GMEM_CONVERSION_TEST(test, __nr_pages, flags) \ + __GMEM_CONVERSION_TEST(test, __nr_pages, (flags) | GUEST_MEMFD_FLAG_MMAP) + +#define __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, __nr_pages) \ + GMEM_CONVERSION_TEST(test, __nr_pages, 0) + +#define GMEM_CONVERSION_TEST_INIT_PRIVATE(test) \ + __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, 1) + +#define __GMEM_CONVERSION_TEST_INIT_SHARED(test, __nr_pages) \ + GMEM_CONVERSION_TEST(test, __nr_pages, GUEST_MEMFD_FLAG_INIT_SHARED) + +#define GMEM_CONVERSION_TEST_INIT_SHARED(test) \ + __GMEM_CONVERSION_TEST_INIT_SHARED(test, 1) + +/* + * Repeats test over nr_pages in a guest_memfd of size nr_pages, providing each + * test iteration with test_page, the index of the page under test in + * guest_memfd. test_page takes values 0..(nr_pages - 1) inclusive. + */ +#define GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(test, __nr_pages) \ +static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \ + const int test_page); \ + \ +TEST_F(gmem_conversions, test) \ +{ \ + const u64 flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; \ + const int nr = (__nr_pages); \ + int i; \ + \ + for (i = 0; i < nr; ++i) { \ + gmem_conversions_do_setup(self, nr, flags); \ + __gmem_conversions_multipage_##test(self, nr, i); \ + gmem_conversions_do_teardown(self); \ + } \ +} \ +static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \ + const int test_page) + +struct guest_check_data { + void *mem; + char expected_val; + char write_val; +}; +static struct guest_check_data guest_data; + +static void guest_do_rmw(void) +{ + for (;;) { + char *mem = READ_ONCE(guest_data.mem); + + GUEST_ASSERT_EQ(READ_ONCE(*mem), READ_ONCE(guest_data.expected_val)); + WRITE_ONCE(*mem, READ_ONCE(guest_data.write_val)); + + GUEST_SYNC(0); + } +} + +static void run_guest_do_rmw(struct kvm_vcpu *vcpu, u64 pgoff, + char expected_val, char write_val) +{ + struct ucall uc; + int r; + + guest_data.mem = (void *)GUEST_MEMFD_SHARING_TEST_GVA + pgoff * page_size; + guest_data.expected_val = expected_val; + guest_data.write_val = write_val; + sync_global_to_guest(vcpu->vm, guest_data); + + do { + r = __vcpu_run(vcpu); + } while (r == -1 && errno == EINTR); + + TEST_ASSERT_EQ(r, 0); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_SYNC: + break; + default: + TEST_FAIL("Unexpected ucall %lu", uc.cmd); + } +} + +static void host_do_rmw(char *mem, u64 pgoff, char expected_val, + char write_val) +{ + TEST_ASSERT_EQ(READ_ONCE(mem[pgoff * page_size]), expected_val); + WRITE_ONCE(mem[pgoff * page_size], write_val); +} + +static void test_private(test_data_t *t, u64 pgoff, char starting_val, + char write_val) +{ + TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[pgoff * page_size], write_val)); + run_guest_do_rmw(t->vcpu, pgoff, starting_val, write_val); + TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[pgoff * page_size])); +} + +static void test_convert_to_private(test_data_t *t, u64 pgoff, + char starting_val, char write_val) +{ + gmem_set_private(t->gmem_fd, pgoff * page_size, page_size); + test_private(t, pgoff, starting_val, write_val); +} + +static void test_shared(test_data_t *t, u64 pgoff, char starting_val, + char host_write_val, char write_val) +{ + host_do_rmw(t->mem, pgoff, starting_val, host_write_val); + run_guest_do_rmw(t->vcpu, pgoff, host_write_val, write_val); + TEST_ASSERT_EQ(READ_ONCE(t->mem[pgoff * page_size]), write_val); +} + +static void test_convert_to_shared(test_data_t *t, u64 pgoff, + char starting_val, char host_write_val, + char write_val) +{ + gmem_set_shared(t->gmem_fd, pgoff * page_size, page_size); + test_shared(t, pgoff, starting_val, host_write_val, write_val); +} + +GMEM_CONVERSION_TEST_INIT_PRIVATE(init_private) +{ + test_private(t, 0, 0, 'A'); + test_convert_to_shared(t, 0, 'A', 'B', 'C'); + test_convert_to_private(t, 0, 'C', 'E'); +} + +GMEM_CONVERSION_TEST_INIT_SHARED(init_shared) +{ + test_shared(t, 0, 0, 'A', 'B'); + test_convert_to_private(t, 0, 'B', 'C'); + test_convert_to_shared(t, 0, 'C', 'D', 'E'); +} + +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(indexing, 4) +{ + int i; + + /* Get a char that varies with both i and n. */ +#define combine(x, n) (((x) << 4) + (n)) +#define i_(n) (combine(i, n)) +#define t_(n) (combine(test_page, n)) + + /* + * Start with the highest index, to catch any errors when, perhaps, the + * first page is returned even for the last index. + */ + for (i = nr_pages - 1; i >= 0; --i) + test_shared(t, i, 0, i_(0), i_(2)); + + test_convert_to_private(t, test_page, t_(2), t_(3)); + + for (i = 0; i < nr_pages; ++i) { + if (i == test_page) + test_private(t, test_page, t_(3), t_(4)); + else + test_shared(t, i, i_(2), i_(3), i_(4)); + } + + test_convert_to_shared(t, test_page, t_(4), t_(5), t_(6)); + + for (i = 0; i < nr_pages; ++i) { + char expected = i == test_page ? t_(6) : i_(4); + + test_shared(t, i, expected, i_(7), i_(8)); + } + +#undef t_ +#undef i_ +#undef combine +} + +/* + * Test that even if there are no folios yet, conversion requests are recorded + * in guest_memfd. + */ +GMEM_CONVERSION_TEST_INIT_SHARED(before_allocation_shared) +{ + test_convert_to_private(t, 0, 0, 'A'); +} + +GMEM_CONVERSION_TEST_INIT_PRIVATE(before_allocation_private) +{ + test_convert_to_shared(t, 0, 0, 'A', 'B'); +} + +/* + * Test that when some of the folios in the conversion range are allocated, + * conversion requests are handled correctly in guest_memfd. Vary the ranges + * allocated before conversion, using test_page, to cover various layouts of + * contiguous allocated and unallocated regions. + */ +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(unallocated_folios, 8) +{ + const int second_page_to_fault = 4; + int i; + + /* + * Fault 2 of the pages to test filemap range operations except when + * test_page == second_page_to_fault. + */ + host_do_rmw(t->mem, test_page, 0, 'A'); + if (test_page != second_page_to_fault) + host_do_rmw(t->mem, second_page_to_fault, 0, 'A'); + + gmem_set_private(t->gmem_fd, 0, nr_pages * page_size); + for (i = 0; i < nr_pages; ++i) { + char expected = (i == test_page || i == second_page_to_fault) ? 'A' : 0; + + test_private(t, i, expected, 'B'); + } + + for (i = 0; i < nr_pages; ++i) + test_convert_to_shared(t, i, 'B', 'C', 'D'); +} + +/* Truncation should not affect shared/private status. */ +GMEM_CONVERSION_TEST_INIT_SHARED(truncate) +{ + host_do_rmw(t->mem, 0, 0, 'A'); + kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size); + host_do_rmw(t->mem, 0, 0, 'A'); + + test_convert_to_private(t, 0, 'A', 'B'); + + kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size); + test_private(t, 0, 0, 'A'); +} + +/* Test that shared/private memory protections work and are seen from any process. */ +GMEM_CONVERSION_TEST_INIT_SHARED(forked_accesses) +{ + enum test_state { + STATE_INIT, + STATE_CHECK_SHARED, + STATE_DONE_CHECKING_SHARED, + STATE_CHECK_PRIVATE, + STATE_DONE_CHECKING_PRIVATE, + }; + + struct sync_state { + pthread_mutex_t mutex; + pthread_cond_t cond; + enum test_state step; + } *sync; + + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; + pid_t child_pid, parent_pid; + int status; + + sync = kvm_mmap(sizeof(*sync), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); + + pthread_mutexattr_init(&mattr); + pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); + pthread_mutex_init(&sync->mutex, &mattr); + pthread_mutexattr_destroy(&mattr); + + pthread_condattr_init(&cattr); + pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); + pthread_cond_init(&sync->cond, &cattr); + pthread_condattr_destroy(&cattr); + + sync->step = STATE_INIT; + +#define TEST_STATE_AWAIT(__state) \ + do { \ + pthread_mutex_lock(&sync->mutex); \ + while (sync->step != (__state)) { \ + struct timespec ts, stop; \ + int ret; \ + \ + clock_gettime(CLOCK_REALTIME, &ts); \ + stop = timespec_add_ns(ts, 100 * 1000000UL); \ + \ + ret = pthread_cond_timedwait(&sync->cond, &sync->mutex, &stop); \ + if (ret == ETIMEDOUT) { \ + bool alive = (child_pid == 0) ? \ + (getppid() == parent_pid) : \ + (waitpid(child_pid, NULL, WNOHANG) == 0); \ + TEST_ASSERT(alive, "Other process exited prematurely"); \ + } else { \ + TEST_ASSERT(!ret, "pthread_cond_timedwait failed"); \ + } \ + } \ + pthread_mutex_unlock(&sync->mutex); \ + } while (0) + +#define TEST_STATE_SET(__state) \ + do { \ + pthread_mutex_lock(&sync->mutex); \ + sync->step = (__state); \ + pthread_cond_broadcast(&sync->cond); \ + pthread_mutex_unlock(&sync->mutex); \ + } while (0) + + parent_pid = getpid(); + child_pid = fork(); + TEST_ASSERT(child_pid != -1, "fork failed"); + + if (child_pid == 0) { + const char inconsequential = 0xdd; + + TEST_STATE_AWAIT(STATE_CHECK_SHARED); + + /* + * This maps the pages into the child process as well, and tests + * that the conversion process will unmap the guest_memfd memory + * from all processes. + */ + host_do_rmw(t->mem, 0, 0xB, 0xC); + + TEST_STATE_SET(STATE_DONE_CHECKING_SHARED); + TEST_STATE_AWAIT(STATE_CHECK_PRIVATE); + + TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[0])); + TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[0], inconsequential)); + + TEST_STATE_SET(STATE_DONE_CHECKING_PRIVATE); + exit(0); + } + + test_shared(t, 0, 0, 0xA, 0xB); + + TEST_STATE_SET(STATE_CHECK_SHARED); + TEST_STATE_AWAIT(STATE_DONE_CHECKING_SHARED); + + test_convert_to_private(t, 0, 0xC, 0xD); + + TEST_STATE_SET(STATE_CHECK_PRIVATE); + TEST_STATE_AWAIT(STATE_DONE_CHECKING_PRIVATE); + + TEST_ASSERT_EQ(waitpid(child_pid, &status, 0), child_pid); + TEST_ASSERT(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "Child exited with unexpected status"); + + pthread_mutex_destroy(&sync->mutex); + pthread_cond_destroy(&sync->cond); + kvm_munmap(sync, sizeof(*sync)); + +#undef TEST_STATE_SET +#undef TEST_STATE_AWAIT +} + +static void test_convert_to_private_fails(test_data_t *t, u64 pgoff, + size_t nr_pages, + u64 expected_error_offset) +{ + /* +1 to make it anything but expected_error_offset. */ + u64 error_offset = expected_error_offset + 1; + u64 offset = pgoff * page_size; + int ret; + + do { + ret = __gmem_set_private(t->gmem_fd, offset, + nr_pages * page_size, &error_offset); + } while (ret == -1 && errno == EINTR); + TEST_ASSERT(ret == -1 && errno == EAGAIN, + "Wanted EAGAIN on page %lu, got %d (ret = %d)", pgoff, + errno, ret); + TEST_ASSERT_EQ(error_offset, expected_error_offset); +} + +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(elevated_refcount, 4) +{ + int i; + + pin_pages(t->mem + test_page * page_size, page_size); + + for (i = 0; i < nr_pages; i++) + test_shared(t, i, 0, 'A', 'B'); + + /* + * Converting in bulk should fail as long any page in the range has + * unexpected refcounts. + */ + test_convert_to_private_fails(t, 0, nr_pages, test_page * page_size); + + for (i = 0; i < nr_pages; i++) { + /* + * Converting page-wise should also fail as long any page in the + * range has unexpected refcounts. + */ + if (i == test_page) + test_convert_to_private_fails(t, i, 1, test_page * page_size); + else + test_convert_to_private(t, i, 'B', 'C'); + } + + unpin_pages(); + + gmem_set_private(t->gmem_fd, 0, nr_pages * page_size); + + for (i = 0; i < nr_pages; i++) { + char expected = i == test_page ? 'B' : 'C'; + + test_private(t, i, expected, 'D'); + } +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) & + KVM_MEMORY_ATTRIBUTE_PRIVATE); + + page_size = getpagesize(); + + return test_harness_run(argc, argv); +} diff --git a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c index 3aa0673f79f3..33b610383693 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c @@ -202,15 +202,18 @@ static void guest_test_explicit_conversion(u64 base_gpa, bool do_fallocate) guest_sync_shared(gpa, size, p3, p4); memcmp_g(gpa, p4, size); - /* Reset the shared memory back to the initial pattern. */ - memset((void *)gpa, init_p, size); - /* * Free (via PUNCH_HOLE) *all* private memory so that the next * iteration starts from a clean slate, e.g. with respect to * whether or not there are pages/folios in guest_mem. */ guest_map_shared(base_gpa, PER_CPU_DATA_SIZE, true); + + /* + * Hole-punching above zeroed private memory. Reset shared + * memory in preparation for the next GUEST_STAGE. + */ + memset((void *)base_gpa, init_p, PER_CPU_DATA_SIZE); } } @@ -303,9 +306,12 @@ static void handle_exit_hypercall(struct kvm_vcpu *vcpu) if (do_fallocate) vm_guest_mem_fallocate(vm, gpa, size, map_shared); - if (set_attributes) - vm_set_memory_attributes(vm, gpa, size, - map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE); + if (set_attributes) { + u64 attrs = map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE; + + vm_mem_set_memory_attributes(vm, gpa, size, attrs); + } + run->hypercall.ret = 0; } @@ -349,8 +355,21 @@ static void *__test_mem_conversions(void *__vcpu) size_t nr_bytes = min_t(size_t, vm->page_size, size - i); u8 *hva = addr_gpa2hva(vm, gpa + i); - /* In all cases, the host should observe the shared data. */ - memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + if (kvm_has_gmem_attributes && + uc.args[0] == SYNC_PRIVATE) { + TEST_EXPECT_SIGBUS(READ_ONCE(*hva)); + } else { + /* + * If not testing in-place conversion + * (dual backing), the host should + * always observe shared data, since the + * shared memory is a separate page. + * + * For SYNC_SHARED, test that the host + * can see shared memory. + */ + memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + } /* For shared, write the new pattern to guest memory. */ if (uc.args[0] == SYNC_SHARED) @@ -366,20 +385,29 @@ static void *__test_mem_conversions(void *__vcpu) } } +/* Align each vCPU's chunk of memory naturally to the size of the backing store. */ +static size_t compute_per_cpu_size(enum vm_mem_backing_src_type src_type) +{ + size_t alignment; + + if (kvm_has_gmem_attributes) + alignment = getpagesize(); + else + alignment = get_backing_src_pagesz(src_type); + + return align_up(PER_CPU_DATA_SIZE, max_t(size_t, SZ_2M, alignment)); +} + static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_vcpus, u32 nr_memslots) { - /* - * Allocate enough memory so that each vCPU's chunk of memory can be - * naturally aligned with respect to the size of the backing store. - */ - const size_t alignment = max_t(size_t, SZ_2M, get_backing_src_pagesz(src_type)); - const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment); + const size_t per_cpu_size = compute_per_cpu_size(src_type); const size_t memfd_size = per_cpu_size * nr_vcpus; const size_t slot_size = memfd_size / nr_memslots; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; pthread_t threads[KVM_MAX_VCPUS]; struct kvm_vm *vm; + u64 gmem_flags; int memfd, i; const struct vm_shape shape = { @@ -394,12 +422,16 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_v vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE)); - memfd = vm_create_guest_memfd(vm, memfd_size, 0); + gmem_flags = 0; + if (kvm_has_gmem_attributes) + gmem_flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; + + memfd = vm_create_guest_memfd(vm, memfd_size, gmem_flags); for (i = 0; i < nr_memslots; i++) vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i, BASE_DATA_SLOT + i, slot_size / vm->page_size, - KVM_MEM_GUEST_MEMFD, memfd, slot_size * i); + KVM_MEM_GUEST_MEMFD, memfd, slot_size * i, gmem_flags); for (i = 0; i < nr_vcpus; i++) { gpa_t gpa = BASE_DATA_GPA + i * per_cpu_size; @@ -459,6 +491,8 @@ int main(int argc, char *argv[]) while ((opt = getopt(argc, argv, "hm:s:n:")) != -1) { switch (opt) { case 's': + TEST_ASSERT(!kvm_has_gmem_attributes, + "src_type is only configurable when testing without in-place conversion"); src_type = parse_backing_src_type(optarg); break; case 'n': diff --git a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c index e6d16e9d6120..f7228e0f1ac6 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c @@ -62,8 +62,9 @@ static void test_private_access_memslot_deleted(void) virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES); - /* Request to access page privately */ - vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); + /* Request to access page privately. */ + if (!kvm_has_gmem_attributes) + vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); kvm_pthread_create(&vm_thread, NULL, (pthread_fn_t)run_vcpu_get_exit_reason, (void *)vcpu); @@ -73,10 +74,26 @@ static void test_private_access_memslot_deleted(void) kvm_pthread_join(vm_thread, &thread_return); exit_reason = (u32)(u64)thread_return; - TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); - TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); - TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); - TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + /* + * If attributes are tracked per-gmem, deleting the memslot that points + * at the gmem instance effectively makes the memory shared, and so the + * read should trigger emulated MMIO. + * + * If attributes are tracked per-VM, deleting the memslot shouldn't + * affect the private attribute, and so KVM should generate a memory + * fault exit (emulated MMIO on private GPAs is disallowed). + */ + if (kvm_has_gmem_attributes) { + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MMIO); + TEST_ASSERT_EQ(vcpu->run->mmio.phys_addr, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->mmio.len, sizeof(u64)); + TEST_ASSERT_EQ(vcpu->run->mmio.is_write, false); + } else { + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); + TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); + TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + } kvm_vm_free(vm); } @@ -87,6 +104,13 @@ static void test_private_access_memslot_not_private(void) struct kvm_vcpu *vcpu; u32 exit_reason; + /* + * Accessing non-private memory as private with a software-protected VM + * isn't possible when doing in-place conversions. + */ + if (kvm_has_gmem_attributes) + return; + vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu, guest_repeatedly_read); diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 24d7dfceb828..c5bfaa1f7749 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -82,9 +82,6 @@ config KVM_ELIDE_TLB_FLUSH_IF_YOUNG config KVM_MMU_LOCKLESS_AGING bool -config KVM_GENERIC_MEMORY_ATTRIBUTES - bool - config KVM_GUEST_MEMFD select XARRAY_MULTI bool diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 625e62e1a031..93f3d29d5751 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -4,9 +4,11 @@ #include #include #include +#include #include #include #include +#include #include "kvm_mm.h" #include "guest_memfd.h" @@ -34,6 +36,13 @@ struct gmem_inode { struct list_head gmem_file_list; u64 flags; + /* + * Every index in this inode, whether memory is populated or + * not, is tracked in attributes. The entire range of indices, + * corresponding to the size of this inode, is represented in + * this maple tree. + */ + struct maple_tree attributes; }; static __always_inline struct gmem_inode *GMEM_I(struct inode *inode) @@ -61,9 +70,28 @@ static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn) return gfn - slot->base_gfn + slot->gmem.pgoff; } +static u64 kvm_gmem_get_default_attributes(struct inode *inode) +{ + bool init_shared = GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED; + + return init_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE; +} + +static u64 kvm_gmem_get_attributes(struct inode *inode, void *entry) +{ + if (WARN_ON_ONCE(!entry)) + return kvm_gmem_get_default_attributes(inode); + + return xa_to_value(entry); +} + static bool kvm_gmem_is_private_mem(struct inode *inode, pgoff_t index) { - return !(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED); + struct maple_tree *mt = &GMEM_I(inode)->attributes; + void *entry = mtree_load(mt, index); + + return kvm_gmem_get_attributes(inode, entry) & + KVM_MEMORY_ATTRIBUTE_PRIVATE; } static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) @@ -71,6 +99,24 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) return !kvm_gmem_is_private_mem(inode, index); } +static bool kvm_gmem_range_has_attributes(struct inode *inode, + pgoff_t index, size_t nr_pages, + u64 attributes) +{ + struct maple_tree *mt = &GMEM_I(inode)->attributes; + pgoff_t end = index + nr_pages - 1; + void *entry; + + lockdep_assert(mt_lock_is_held(mt)); + + mt_for_each(mt, entry, index, end) { + if (kvm_gmem_get_attributes(inode, entry) != attributes) + return false; + } + + return true; +} + /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -112,8 +158,11 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) return folio; } -static enum kvm_gfn_range_filter kvm_gmem_get_invalidate_filter(struct inode *inode) +static enum kvm_gfn_range_filter kvm_gmem_get_all_gfns_filter(struct inode *inode) { + if (gmem_in_place_conversion) + return KVM_FILTER_SHARED | KVM_FILTER_PRIVATE; + if (GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED) return KVM_FILTER_SHARED; @@ -162,15 +211,13 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, } static void kvm_gmem_invalidate_start(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, + enum kvm_gfn_range_filter filter) { - enum kvm_gfn_range_filter attr_filter; struct gmem_file *f; - attr_filter = kvm_gmem_get_invalidate_filter(inode); - kvm_gmem_for_each_file(f, inode) - __kvm_gmem_invalidate_start(f, start, end, attr_filter); + __kvm_gmem_invalidate_start(f, start, end, filter); } static void __kvm_gmem_invalidate_end(struct gmem_file *f, pgoff_t start, @@ -196,6 +243,7 @@ static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t start, static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { + enum kvm_gfn_range_filter filter = kvm_gmem_get_all_gfns_filter(inode); pgoff_t start = offset >> PAGE_SHIFT; pgoff_t end = (offset + len) >> PAGE_SHIFT; @@ -205,7 +253,7 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) */ filemap_invalidate_lock(inode->i_mapping); - kvm_gmem_invalidate_start(inode, start, end); + kvm_gmem_invalidate_start(inode, start, end, filter); truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1); @@ -300,17 +348,56 @@ static int kvm_gmem_release(struct inode *inode, struct file *file) * dereferencing the slot for existing bindings needs to be protected * against memslot updates, specifically so that unbind doesn't race * and free the memslot (kvm_gmem_get_file() will return NULL). - * - * Since .release is called only when the reference count is zero, - * after which file_ref_get() and get_file_active() fail, - * kvm_gmem_get_pfn() cannot be using the file concurrently. - * file_ref_put() provides a full barrier, and get_file_active() the - * matching acquire barrier. */ mutex_lock(&kvm->slots_lock); filemap_invalidate_lock(inode->i_mapping); + /* + * Note! synchronize_srcu() is _not_ needed after nullifying memslot + * bindings as slot->gmem.file cannot be set back to a non-null value + * without the memslot first being deleted. I.e. this relies on the + * synchronize_srcu_expedited() in kvm_swap_active_memslots() to ensure + * kvm_gmem_get_pfn() (which runs with kvm->srcu held for read) can't + * grab a reference to slot->gmem.file even if the struct file object + * is reallocated (for use as a different file). + * + * file_ref_put() provides a full barrier, and __get_file_rcu() the + * matching acquire barrier, to ensure that kvm_gmem_get_file() (via + * __get_file_rcu()) sees refcount==0 or fails the "file reloaded" + * check (file != NULL due to nullifying the file pointer here). + * + * Unlike most other users of get_file_rcu(), where callers don't care + * if they race with a write, only that they have a reference to _a_ + * live file, kvm_gmem_get_pfn() needs to get the exact file that is + * associated with the memslot. Without the aforementioned SRCU + * synchronization, the following could happen: + * + * CPU0 CPU1 + * kvm_gmem_get_pfn() + * slot->gmem.file == A + * kvm_gmem_release()) + * slot->gmem.file = NULL + * + * kvm_set_memory_region() + * slot deleted + * + * kvm_set_memory_region() + * slot created + * slot->gmem.file = B (alloc the same object) + * + * get_file_active() + * file = B + * file_reloaded = B + * + * + * + * Obviously KVM would be broken in many places if the synchronization + * were omitted, but it's important to note that get_file_active() does + * NOT guarantee a reference to the "original" file was obtained, only + * that it grabbed a reference for the returned file, i.e. didn't grab + * a reference for file A, but then returned a pointer to file B. + */ xa_for_each(&f->bindings, index, slot) WRITE_ONCE(slot->gmem.file, NULL); @@ -320,7 +407,7 @@ static int kvm_gmem_release(struct inode *inode, struct file *file) * memory, as its lifetime is associated with the inode, not the file. */ __kvm_gmem_invalidate_start(f, 0, -1ul, - kvm_gmem_get_invalidate_filter(inode)); + kvm_gmem_get_all_gfns_filter(inode)); __kvm_gmem_invalidate_end(f, 0, -1ul); list_del(&f->entry); @@ -364,10 +451,13 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) return VM_FAULT_SIGBUS; - if (!kvm_gmem_is_shared_mem(inode, vmf->pgoff)) - return VM_FAULT_SIGBUS; + filemap_invalidate_lock_shared(inode->i_mapping); + if (kvm_gmem_is_shared_mem(inode, vmf->pgoff)) + folio = kvm_gmem_get_folio(inode, vmf->pgoff); + else + folio = ERR_PTR(-EACCES); + filemap_invalidate_unlock_shared(inode->i_mapping); - folio = kvm_gmem_get_folio(inode, vmf->pgoff); if (IS_ERR(folio)) { if (PTR_ERR(folio) == -EAGAIN) return VM_FAULT_RETRY; @@ -448,11 +538,280 @@ static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) return 0; } +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); + struct inode *inode; + + if (!slot) + return 0; + + CLASS(gmem_get_file, file)(slot); + if (!file) + return 0; + + inode = file_inode(file); + + /* + * Rely on the maple tree's internal RCU lock to ensure a stable result. + * This result can become stale as soon as the lock is dropped, so the + * caller _must_ protect consumption of private vs. shared either by + * holding guest_memfd's invalidate lock for the entire duration, or by + * checking mmu_invalidate_retry_gfn() under mmu_lock to serialize + * against concurrent attribute updates. + */ + return kvm_gmem_is_private_mem(inode, kvm_gmem_get_index(slot, gfn)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_is_private_gfn); + +/* + * Preallocate memory for attributes to be stored on a maple tree, pointed to + * by mas. Adjacent ranges with attributes identical to the new attributes + * will be merged. Also sets mas's bounds up for storing attributes. + * + * This maintains the invariant that ranges with the same attributes will + * always be merged. + */ +static int kvm_gmem_mas_preallocate(struct ma_state *mas, u64 attributes, + pgoff_t start, size_t nr_pages) +{ + pgoff_t end = start + nr_pages; + pgoff_t last = end - 1; + void *entry; + + /* Try extending range. entry is NULL on overflow/wrap-around. */ + mas_set(mas, end); + entry = mas_find(mas, end); + if (entry && xa_to_value(entry) == attributes) + last = mas->last; + + if (start > 0) { + mas_set(mas, start - 1); + entry = mas_find(mas, start - 1); + if (entry && xa_to_value(entry) == attributes) + start = mas->index; + } + + mas_set_range(mas, start, last); + return mas_preallocate(mas, xa_mk_value(attributes), GFP_KERNEL); +} + +static bool __folio_has_outstanding_references(struct folio *folio, + enum lru_cache_drained *drained) +{ + if (folio_maybe_dma_pinned(folio) || folio_mapped(folio)) + return true; + + /* 1 reference held by filemap_get_folios() in the folio batch. */ + lru_cache_drain_for_folio(folio, 1, drained); + + /* + * Outstanding references are anything other than those from the page + * cache, plus 1 temporary reference held by filemap_get_folios() in the + * folio batch. + */ + return folio_ref_count(folio) != folio_nr_pages(folio) + 1; +} + +static bool kvm_gmem_has_outstanding_references(struct inode *inode, + pgoff_t start, size_t nr_pages, + pgoff_t *err_index) +{ + enum lru_cache_drained drained = LRU_CACHE_NOT_DRAINED; + struct address_space *mapping = inode->i_mapping; + pgoff_t last = start + nr_pages - 1; + struct folio_batch fbatch; + pgoff_t next; + int i; + + folio_batch_init(&fbatch); + + next = start; + while (filemap_get_folios(mapping, &next, last, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + + if (__folio_has_outstanding_references(folio, &drained)) { + *err_index = max(start, folio->index); + folio_batch_release(&fbatch); + return true; + } + } + + folio_batch_release(&fbatch); + cond_resched(); + } + + return false; +} + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) +{ + struct folio_batch fbatch; + pgoff_t next = start; + int i; + + folio_batch_init(&fbatch); + while (filemap_get_folios(inode->i_mapping, &next, end - 1, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + pgoff_t start_index, end_index; + kvm_pfn_t start_pfn; + kvm_pfn_t nr_pages; + + start_index = max(start, folio->index); + end_index = min(end, folio_next_index(folio)); + /* + * end_index is either in folio or points to + * the first page of the next folio. Hence, + * all pages in range [start_index, end_index) + * are contiguous. + */ + start_pfn = folio_file_pfn(folio, start_index); + nr_pages = end_index - start_index; + + kvm_arch_gmem_make_shared(start_pfn, nr_pages); + } + + folio_batch_release(&fbatch); + cond_resched(); + } +} +#else +static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) {} +#endif + +static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, + size_t nr_pages, uint64_t attrs, + pgoff_t *err_index) +{ + bool to_private = attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE; + struct address_space *mapping = inode->i_mapping; + struct gmem_inode *gi = GMEM_I(inode); + enum kvm_gfn_range_filter filter; + pgoff_t end = start + nr_pages; + struct maple_tree *mt; + struct ma_state mas; + int r = 0; + + mt = &gi->attributes; + + filemap_invalidate_lock(mapping); + + if (kvm_gmem_range_has_attributes(inode, start, nr_pages, attrs)) + goto out; + + mas_init(&mas, mt, start); + r = kvm_gmem_mas_preallocate(&mas, attrs, start, nr_pages); + if (r) { + *err_index = start; + goto out; + } + + if (to_private) { + /* + * Forcefully unmap the pages from all userspace page tables, + * and then verify there are no outstanding references, e.g. + * acquired via GUP or similar. Tell userspace to try again if + * there are outstanding references and hope that whatever has + * pinned the page will put its reference "soon". + */ + unmap_mapping_pages(mapping, start, nr_pages, false); + + if (kvm_gmem_has_outstanding_references(inode, start, nr_pages, + err_index)) { + mas_destroy(&mas); + r = -EAGAIN; + goto out; + } + } + + /* + * From this point on guest_memfd has performed necessary + * checks and can proceed to do guest-breaking changes. + */ + + filter = to_private ? KVM_FILTER_SHARED : KVM_FILTER_PRIVATE; + kvm_gmem_invalidate_start(inode, start, end, filter); + + if (!to_private && kvm_arch_has_gmem_convert()) + kvm_gmem_make_shared(inode, start, end); + + mas_store_prealloc(&mas, xa_mk_value(attrs)); + + kvm_gmem_invalidate_end(inode, start, end); +out: + filemap_invalidate_unlock(mapping); + return r; +} + +static long kvm_gmem_set_attributes(struct file *file, void __user *argp) +{ + struct gmem_file *f = file->private_data; + struct inode *inode = file_inode(file); + struct kvm_memory_attributes2 attrs; + pgoff_t err_index; + size_t nr_pages; + pgoff_t index; + int i, r; + + if (copy_from_user(&attrs, argp, sizeof(attrs))) + return -EFAULT; + + if (attrs.flags) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(attrs.reserved); i++) { + if (attrs.reserved[i]) + return -EINVAL; + } + if (!kvm_arch_has_private_mem(f->kvm)) + return -EINVAL; + if (attrs.attributes & ~KVM_MEMORY_ATTRIBUTE_PRIVATE) + return -EINVAL; + if (attrs.size == 0 || attrs.offset + attrs.size < attrs.offset) + return -EINVAL; + if (!PAGE_ALIGNED(attrs.offset) || !PAGE_ALIGNED(attrs.size)) + return -EINVAL; + + if (attrs.offset >= i_size_read(inode) || + attrs.offset + attrs.size > i_size_read(inode)) + return -EINVAL; + + nr_pages = attrs.size >> PAGE_SHIFT; + index = attrs.offset >> PAGE_SHIFT; + r = __kvm_gmem_set_attributes(inode, index, nr_pages, attrs.attributes, + &err_index); + if (r) { + attrs.error_offset = ((uint64_t)err_index) << PAGE_SHIFT; + + if (copy_to_user(argp, &attrs, sizeof(attrs))) + return -EFAULT; + } + + return r; +} + +static long kvm_gmem_ioctl(struct file *file, unsigned int ioctl, + unsigned long arg) +{ + switch (ioctl) { + case KVM_SET_MEMORY_ATTRIBUTES2: + if (!gmem_in_place_conversion) + return -ENOTTY; + + return kvm_gmem_set_attributes(file, (void __user *)arg); + default: + return -ENOTTY; + } +} + static struct file_operations kvm_gmem_fops = { .mmap = kvm_gmem_mmap, .open = generic_file_open, .release = kvm_gmem_release, .fallocate = kvm_gmem_fallocate, + .unlocked_ioctl = kvm_gmem_ioctl, }; static int kvm_gmem_migrate_folio(struct address_space *mapping, @@ -465,14 +824,27 @@ static int kvm_gmem_migrate_folio(struct address_space *mapping, static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *folio) { + struct inode *inode = mapping->host; + enum kvm_gfn_range_filter filter; pgoff_t start, end; - filemap_invalidate_lock_shared(mapping); + /* + * memory_failure() invokes ->error_remove_folio() while holding the + * poisoned folio lock. Do not block on mapping->invalidate_lock, as + * a writer that already holds the invalidate lock can be waiting for + * the same folio, e.g. truncation from MADV_REMOVE/FALLOC_FL_PUNCH_HOLE. + * + * If the invalidate lock is contended, fail the recovery instead of + * risking a deadlock in the memory-failure path. + */ + if (!filemap_invalidate_trylock_shared(mapping)) + return -EBUSY; start = folio->index; end = start + folio_nr_pages(folio); - kvm_gmem_invalidate_start(mapping->host, start, end); + filter = kvm_gmem_get_all_gfns_filter(inode); + kvm_gmem_invalidate_start(inode, start, end, filter); /* * Do not truncate the range, what action is taken in response to the @@ -520,6 +892,51 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm) return true; } +static int kvm_gmem_init_inode(struct inode *inode, loff_t size, u64 flags) +{ + struct gmem_inode *gi = GMEM_I(inode); + MA_STATE(mas, &gi->attributes, 0, (size >> PAGE_SHIFT) - 1); + u64 attrs; + int r; + + inode->i_op = &kvm_gmem_iops; + inode->i_mapping->a_ops = &kvm_gmem_aops; + inode->i_mode |= S_IFREG; + inode->i_size = size; + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + + /* + * guest_memfd memory is neither migratable nor swappable: set + * inaccessible to gate off both. + */ + mapping_set_inaccessible(inode->i_mapping); + WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); + + gi->flags = flags; + + mt_set_external_lock(&gi->attributes, + &inode->i_mapping->invalidate_lock); + + /* + * Store default attributes for the entire gmem instance. Ensuring every + * index is represented in the maple tree at all times simplifies the + * conversion and merging logic. + */ + attrs = kvm_gmem_get_default_attributes(inode); + + /* + * Acquire the invalidation lock purely to make lockdep happy. The + * maple tree library expects all stores to be protected via the lock, + * and the library can't know when the tree is reachable only by the + * caller, as is the case here. + */ + filemap_invalidate_lock(inode->i_mapping); + r = mas_store_gfp(&mas, xa_mk_value(attrs), GFP_KERNEL); + filemap_invalidate_unlock(inode->i_mapping); + + return r; +} + static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { static const char *name = "[kvm-gmem]"; @@ -550,16 +967,9 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) goto err_fops; } - inode->i_op = &kvm_gmem_iops; - inode->i_mapping->a_ops = &kvm_gmem_aops; - inode->i_mode |= S_IFREG; - inode->i_size = size; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); - mapping_set_inaccessible(inode->i_mapping); - /* Unmovable mappings are supposed to be marked unevictable as well. */ - WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); - - GMEM_I(inode)->flags = flags; + err = kvm_gmem_init_inode(inode, size, flags); + if (err) + goto err_inode; file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, &kvm_gmem_fops); if (IS_ERR(file)) { @@ -603,19 +1013,21 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) return __kvm_gmem_create(kvm, size, flags); } -int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset) +int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset) { uoff_t size = slot->npages << PAGE_SHIFT; - unsigned long start, end; struct gmem_file *f; struct inode *inode; struct file *file; - int r = -EINVAL; + BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset)); BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff)); + if (WARN_ON_ONCE(slot->flags & KVM_MEMSLOT_GMEM_ONLY)) + return -EINVAL; + file = fget(fd); if (!file) return -EBADF; @@ -632,84 +1044,93 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, if (!PAGE_ALIGNED(offset) || offset + size > i_size_read(inode)) goto err; - filemap_invalidate_lock(inode->i_mapping); - - start = offset >> PAGE_SHIFT; - end = start + slot->npages; - - if (!xa_empty(&f->bindings) && - xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) { - r = -EEXIST; - filemap_invalidate_unlock(inode->i_mapping); - goto err; - } - /* * memslots of flag KVM_MEM_GUEST_MEMFD are immutable to change, so * kvm_gmem_bind() must occur on a new memslot. Because the memslot * is not visible yet, kvm_gmem_get_pfn() is guaranteed to see the file. */ - WRITE_ONCE(slot->gmem.file, file); - slot->gmem.pgoff = start; - if (kvm_gmem_supports_mmap(inode)) + slot->gmem.file = file; + slot->gmem.pgoff = offset >> PAGE_SHIFT; + if (gmem_in_place_conversion || kvm_gmem_supports_mmap(inode)) slot->flags |= KVM_MEMSLOT_GMEM_ONLY; - xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL); - filemap_invalidate_unlock(inode->i_mapping); - /* - * Drop the reference to the file, even on success. The file pins KVM, - * not the other way 'round. Active bindings are invalidated if the - * file is closed before memslots are destroyed. + * Gift the caller a reference to the file. The reference will be + * dropped after bindings are established, or if installing the new + * memslot ultimately fails. */ - r = 0; + return 0; + err: fput(file); - return r; + return -EINVAL; } -static void __kvm_gmem_unbind(struct kvm_memory_slot *slot, struct gmem_file *f) +int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot) { - unsigned long start = slot->gmem.pgoff; - unsigned long end = start + slot->npages; + struct gmem_file *f = slot->gmem.file->private_data; + struct inode *inode = file_inode(slot->gmem.file); + unsigned long start, end; + int r; - xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + if (WARN_ON_ONCE(slot->gmem.file->f_op != &kvm_gmem_fops)) + return -EIO; - /* - * synchronize_srcu(&kvm->srcu) ensured that kvm_gmem_get_pfn() - * cannot see this memslot. - */ - WRITE_ONCE(slot->gmem.file, NULL); + filemap_invalidate_lock(inode->i_mapping); + + start = slot->gmem.pgoff; + end = start + slot->npages; + + if (!xa_empty(&f->bindings) && + xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) { + filemap_invalidate_unlock(inode->i_mapping); + return -EEXIST; + } + + r = xa_err(xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL)); + if (r) + xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + + filemap_invalidate_unlock(inode->i_mapping); + + return r; } void kvm_gmem_unbind(struct kvm_memory_slot *slot) { + struct file *file = slot->gmem.file; + unsigned long start = slot->gmem.pgoff; + unsigned long end = start + slot->npages; + struct gmem_file *f; + /* * Nothing to do if the underlying file was _already_ closed, as * kvm_gmem_release() invalidates and nullifies all bindings. */ - if (!slot->gmem.file) + if (!file) return; - CLASS(gmem_get_file, file)(slot); - /* * However, if the file is _being_ closed, then the bindings need to be * removed as kvm_gmem_release() might not run until after the memslot - * is freed. Note, modifying the bindings is safe even though the file - * is dying as kvm_gmem_release() nullifies slot->gmem.file under - * slots_lock, and only puts its reference to KVM after destroying all - * bindings. I.e. reaching this point means kvm_gmem_release() hasn't - * yet destroyed the bindings or freed the gmem_file, and can't do so - * until the caller drops slots_lock. + * is freed. Modifying the bindings is safe even if the file is dying + * as kvm_gmem_release() nullifies slot->gmem.file under slots_lock, + * and only puts its reference to KVM after destroying all bindings. + * I.e. reaching this point means kvm_gmem_release() hasn't destroyed + * the bindings or freed the gmem_file and can't do so until the caller + * drops slots_lock, so there's no need to verify the file is live. */ - if (!file) { - __kvm_gmem_unbind(slot, slot->gmem.file->private_data); - return; - } + f = file->private_data; filemap_invalidate_lock(file->f_mapping); - __kvm_gmem_unbind(slot, file->private_data); + xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + + /* + * Note, the caller is responsible for ensuring the slot is unreachable + * before unbinding, e.g. by synchronizing SRCU after deleting the slot, + * to guarantee kvm_gmem_get_pfn() can't see the slot+file. + */ + WRITE_ONCE(slot->gmem.file, NULL); filemap_invalidate_unlock(file->f_mapping); } @@ -743,6 +1164,11 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, return ERR_PTR(-EHWPOISON); } + if (!folio_test_uptodate(folio)) { + clear_highpage(folio_page(folio, 0)); + folio_mark_uptodate(folio); + } + *pfn = folio_file_pfn(folio, index); if (max_order) *max_order = 0; @@ -751,8 +1177,7 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, } int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order) + gfn_t gfn, kvm_pfn_t *pfn, int *max_order) { pgoff_t index = kvm_gmem_get_index(slot, gfn); struct folio *folio; @@ -764,28 +1189,24 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, if (!file) return -EFAULT; - folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order); - if (IS_ERR(folio)) - return PTR_ERR(folio); + filemap_invalidate_lock_shared(file_inode(file)->i_mapping); - if (!folio_test_uptodate(folio)) { - clear_highpage(folio_page(folio, 0)); - folio_mark_uptodate(folio); + folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order); + if (IS_ERR(folio)) { + r = PTR_ERR(folio); + goto out; } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - if (kvm_gmem_is_private_mem(file_inode(file), index)) + if (kvm_arch_has_gmem_convert() && + kvm_gmem_is_private_mem(file_inode(file), index)) r = kvm_arch_gmem_make_private(kvm, gfn, *pfn, (kvm_pfn_t)1 << *max_order); -#endif folio_unlock(folio); + folio_put(folio); - if (!r) - *page = folio_file_page(folio, index); - else - folio_put(folio); - +out: + filemap_invalidate_unlock_shared(file_inode(file)->i_mapping); return r; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn); @@ -811,16 +1232,12 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, folio_unlock(folio); - if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1, - KVM_MEMORY_ATTRIBUTE_PRIVATE, - KVM_MEMORY_ATTRIBUTE_PRIVATE)) { + if (!kvm_is_private_gfn(kvm, gfn)) { ret = -EINVAL; goto out_put_folio; } ret = post_populate(kvm, gfn, pfn, src_page, opaque); - if (!ret) - folio_mark_uptodate(folio); out_put_folio: folio_put(folio); @@ -917,6 +1334,15 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) mpol_shared_policy_init(&gi->policy, NULL); + /* + * Memory attributes are protected by the filemap invalidation lock, but + * the lock structure isn't available at this time. Immediately mark + * maple tree as using external locking so that accessing the tree + * before it's fully initialized results in NULL pointer dereferences + * and not more subtle bugs. + */ + mt_init_flags(&gi->attributes, MT_FLAGS_LOCK_EXTERN | MT_FLAGS_USE_RCU); + gi->flags = 0; INIT_LIST_HEAD(&gi->gmem_file_list); return &gi->vfs_inode; @@ -924,7 +1350,26 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) static void kvm_gmem_destroy_inode(struct inode *inode) { - mpol_free_shared_policy(&GMEM_I(inode)->policy); + struct gmem_inode *gi = GMEM_I(inode); + + mpol_free_shared_policy(&gi->policy); + + /* + * Note! Checking for an empty tree is functionally necessary + * to avoid explosions if the tree hasn't been fully + * initialized, i.e. if the inode is being destroyed before + * guest_memfd can set the external lock, lockdep would find + * that the tree's internal ma_lock was not held. + */ + if (!mtree_empty(&gi->attributes)) { + /* + * Acquire the invalidation lock purely to make lockdep happy, + * the inode is unreachable at this point. + */ + filemap_invalidate_lock(inode->i_mapping); + __mt_destroy(&gi->attributes); + filemap_invalidate_unlock(inode->i_mapping); + } } static void kvm_gmem_free_inode(struct inode *inode) diff --git a/virt/kvm/guest_memfd.h b/virt/kvm/guest_memfd.h index 0f9c6f840838..01bd359d27e3 100644 --- a/virt/kvm/guest_memfd.h +++ b/virt/kvm/guest_memfd.h @@ -8,8 +8,9 @@ int kvm_gmem_init(struct module *module); void kvm_gmem_exit(void); int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args); -int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset); +int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset); +int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot); void kvm_gmem_unbind(struct kvm_memory_slot *slot); #else static inline int kvm_gmem_init(struct module *module) @@ -17,9 +18,17 @@ static inline int kvm_gmem_init(struct module *module) return 0; } static inline void kvm_gmem_exit(void) {}; -static inline int kvm_gmem_bind(struct kvm *kvm, - struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset) + +static inline int kvm_gmem_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset) +{ + WARN_ON_ONCE(1); + return -EIO; +} + +static inline int kvm_gmem_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *slot) { WARN_ON_ONCE(1); return -EIO; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 4e212b7c83d3..0d8cc3cc0641 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -104,6 +104,14 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink); static bool __ro_after_init allow_unsafe_mappings; module_param(allow_unsafe_mappings, bool, 0444); +#ifdef kvm_arch_has_private_mem +bool __ro_after_init gmem_in_place_conversion = !IS_ENABLED(CONFIG_KVM_VM_MEMORY_ATTRIBUTES); +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +module_param(gmem_in_place_conversion, bool, 0444); +#endif +EXPORT_SYMBOL_FOR_KVM_INTERNAL(gmem_in_place_conversion); +#endif + /* * Ordering of locks: * @@ -1128,7 +1136,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) spin_lock_init(&kvm->mn_invalidate_lock); rcuwait_init(&kvm->mn_memslots_update_rcuwait); xa_init(&kvm->vcpu_array); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES xa_init_flags(&kvm->mem_attr_array, XA_FLAGS_ACCOUNT); #endif @@ -1313,7 +1321,7 @@ static void kvm_destroy_vm(struct kvm *kvm) cleanup_srcu_struct(&kvm->irq_srcu); srcu_barrier(&kvm->srcu); cleanup_srcu_struct(&kvm->srcu); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES xa_destroy(&kvm->mem_attr_array); #endif kvm_arch_free_vm(kvm); @@ -1762,10 +1770,10 @@ static void kvm_commit_memory_region(struct kvm *kvm, kvm_destroy_dirty_bitmap(old); /* - * Unbind the guest_memfd instance as needed; the @new slot has - * already created its own binding. TODO: Drop the WARN when - * dirty logging guest_memfd memslots is supported. Until then, - * flags-only changes on guest_memfd slots should be impossible. + * TODO: Drop the WARN and do the unbind() call only for MOVE + * when dirty logging guest_memfd memslots is supported. Until + * then, flags-only changes on guest_memfd slots should also be + * impossible; unbind the old memslot for defense-in-depth. */ if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD)) kvm_gmem_unbind(old); @@ -1951,20 +1959,16 @@ static int kvm_set_memslot(struct kvm *kvm, } r = kvm_prepare_memory_region(kvm, old, new, change); - if (r) { - /* - * For DELETE/MOVE, revert the above INVALID change. No - * modifications required since the original slot was preserved - * in the inactive slots. Changing the active memslots also - * release slots_arch_lock. - */ - if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { - kvm_activate_memslot(kvm, invalid_slot, old); - kfree(invalid_slot); - } else { - mutex_unlock(&kvm->slots_arch_lock); + if (r) + goto err; + + if (change == KVM_MR_CREATE && (new->flags & KVM_MEM_GUEST_MEMFD)) { + r = kvm_gmem_commit_memory_region(kvm, new); + if (r) { + kvm_arch_free_memslot(kvm, new); + kvm_destroy_dirty_bitmap(new); + goto err; } - return r; } /* @@ -1997,6 +2001,20 @@ static int kvm_set_memslot(struct kvm *kvm, kvm_commit_memory_region(kvm, old, new, change); return 0; + +err: + /* + * For DELETE/MOVE, revert the above INVALID change. No modifications + * required since the original slot was preserved in the inactive slots. + * Changing the active memslots also release slots_arch_lock. + */ + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { + kvm_activate_memslot(kvm, invalid_slot, old); + kfree(invalid_slot); + } else { + mutex_unlock(&kvm->slots_arch_lock); + } + return r; } static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id, @@ -2125,21 +2143,30 @@ static int kvm_set_memory_region(struct kvm *kvm, new->npages = npages; new->flags = mem->flags; new->userspace_addr = mem->userspace_addr; - if (mem->flags & KVM_MEM_GUEST_MEMFD) { - r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset); + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) { + r = kvm_gmem_prepare_memory_region(kvm, new, mem->guest_memfd, + mem->guest_memfd_offset); if (r) goto out; } r = kvm_set_memslot(kvm, old, new, change); + + /* + * Drop the reference to the gmem file, even on success. The file pins + * KVM, not the other way 'round. Active bindings are invalidated if + * the file is closed before memslots are destroyed. + */ +#ifdef CONFIG_KVM_GUEST_MEMFD + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) + fput(new->gmem.file); +#endif + if (r) - goto out_unbind; + goto out; return 0; -out_unbind: - if (mem->flags & KVM_MEM_GUEST_MEMFD) - kvm_gmem_unbind(new); out: kfree(new); return r; @@ -2439,32 +2466,41 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, } #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static u64 kvm_supported_mem_attributes(struct kvm *kvm) +#ifdef kvm_arch_has_private_mem +static u64 kvm_supports_private_mem(struct kvm *kvm) { - if (!kvm || kvm_arch_has_private_mem(kvm)) - return KVM_MEMORY_ATTRIBUTE_PRIVATE; + return !kvm || kvm_arch_has_private_mem(kvm); +} +#else +#define kvm_supports_private_mem(kvm) false +#endif - return 0; +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm) +{ + if (gmem_in_place_conversion || !kvm_supports_private_mem(kvm)) + return 0; + + return KVM_MEMORY_ATTRIBUTE_PRIVATE; } /* * Returns true if _all_ gfns in the range [@start, @end) have attributes * such that the bits in @mask match @attrs. */ -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs) +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs) { XA_STATE(xas, &kvm->mem_attr_array, start); unsigned long index; void *entry; - mask &= kvm_supported_mem_attributes(kvm); + mask &= kvm_supported_vm_mem_attributes(kvm); if (attrs & ~mask) return false; if (end == start + 1) - return (kvm_get_memory_attributes(kvm, start) & mask) == attrs; + return (kvm_get_vm_memory_attributes(kvm, start) & mask) == attrs; guard(rcu)(); @@ -2557,8 +2593,8 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm, KVM_MMU_UNLOCK(kvm); } -static bool kvm_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +static bool kvm_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { /* * Unconditionally add the range to the invalidation set, regardless of @@ -2573,18 +2609,18 @@ static bool kvm_pre_set_memory_attributes(struct kvm *kvm, */ kvm_mmu_invalidate_range_add(kvm, range->start, range->end); - return kvm_arch_pre_set_memory_attributes(kvm, range); + return kvm_arch_pre_set_vm_memory_attributes(kvm, range); } /* Set @attributes for the gfn range [@start, @end). */ -static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, +static int kvm_set_vm_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, unsigned long attributes) { struct kvm_mmu_notifier_range pre_set_range = { .start = start, .end = end, .arg.attributes = attributes, - .handler = kvm_pre_set_memory_attributes, + .handler = kvm_pre_set_vm_memory_attributes, .on_lock = kvm_mmu_invalidate_start, .flush_on_ret = true, .may_block = true, @@ -2593,7 +2629,7 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, .start = start, .end = end, .arg.attributes = attributes, - .handler = kvm_arch_post_set_memory_attributes, + .handler = kvm_arch_post_set_vm_memory_attributes, .on_lock = kvm_mmu_invalidate_end, .may_block = true, }; @@ -2603,12 +2639,12 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, entry = attributes ? xa_mk_value(attributes) : NULL; - trace_kvm_vm_set_mem_attributes(start, end, attributes); + trace_kvm_set_vm_mem_attributes(start, end, attributes); mutex_lock(&kvm->slots_lock); /* Nothing to do if the entire range has the desired attributes. */ - if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes)) + if (kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attributes)) goto out_unlock; /* @@ -2648,7 +2684,7 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, /* flags is currently not used. */ if (attrs->flags) return -EINVAL; - if (attrs->attributes & ~kvm_supported_mem_attributes(kvm)) + if (attrs->attributes & ~kvm_supported_vm_mem_attributes(kvm)) return -EINVAL; if (attrs->size == 0 || attrs->address + attrs->size < attrs->address) return -EINVAL; @@ -2665,9 +2701,26 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, */ BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long)); - return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes); + return kvm_set_vm_mem_attributes(kvm, start, end, attrs->attributes); } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ + +#ifdef kvm_arch_has_private_mem +DEFINE_STATIC_CALL_RET0(__kvm_is_private_gfn, kvm_is_private_gfn_t); +EXPORT_STATIC_CALL_GPL(__kvm_is_private_gfn); + +static void kvm_init_memory_attributes(void) +{ + if (gmem_in_place_conversion) + static_call_update(__kvm_is_private_gfn, kvm_gmem_is_private_gfn); +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES + else + static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn); +#endif +} +#else +static void kvm_init_memory_attributes(void) { } +#endif struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { @@ -4973,15 +5026,20 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_SYSTEM_EVENT_DATA: case KVM_CAP_DEVICE_CTRL: return 1; -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_CAP_MEMORY_ATTRIBUTES: - return kvm_supported_mem_attributes(kvm); + return kvm_supported_vm_mem_attributes(kvm); #endif #ifdef CONFIG_KVM_GUEST_MEMFD case KVM_CAP_GUEST_MEMFD: return 1; case KVM_CAP_GUEST_MEMFD_FLAGS: return kvm_gmem_get_supported_flags(kvm); + case KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES: + if (!gmem_in_place_conversion || !kvm_supports_private_mem(kvm)) + return 0; + + return KVM_MEMORY_ATTRIBUTE_PRIVATE; #endif default: break; @@ -5377,7 +5435,7 @@ static long kvm_vm_ioctl(struct file *filp, break; } #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_SET_MEMORY_ATTRIBUTES: { struct kvm_memory_attributes attrs; @@ -5388,7 +5446,7 @@ static long kvm_vm_ioctl(struct file *filp, r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs); break; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ case KVM_CREATE_DEVICE: { struct kvm_create_device cd; @@ -6578,6 +6636,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out; + kvm_init_memory_attributes(); kvm_init_debug(); r = kvm_vfio_ops_init();