github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM guest_memfd and CoCo changes for 7.4

Browse Source 
- Take the invalidate lock when unbinding a dying file to fix a race with
    kvm_gmem_invalidate_end() that results in KVM leaving the MMU invalidation
    count elevated.  Not taking the lock in the first place was pure misguided
    paranoid.
 
  - When binding a memslot, purge the bindings if storing the new memslot fails,
    as Xarray's "store range" API can subtly fail partway through.
 
  - Establish memslot<=>guest_memfd bindings *after* the memslot is ready, as
    the memslot becomes reachable the instant the binding is installed.
 
  - Don't grab a page refcount when reloading an SNP VMSA and stop returning a
    "struct page" from guest_memfd's PFN lookup altogether, to ensure that KVM
    doesn't hold any unnecessary refcounts (the lifetime of the page is ensured
    by MMU notifiers and similar hooks).  Preventing unnecessary refcounts
    greatly simplifies supporting in-place SHARED=>PRIVATE conversions.
 
  - Add support for in-place PRIVATE<=>SHARED conversion, i.e. allow a single
    guest_memfd instance to be used for both PRIVATE and SHARED variations of
    CoCo guests' memory (though not at the same time).  This eliminates the need
    to double allocate guest memory for TDX and SNP, and is basically a hard
    requirement for pKVM, which relies on being able to preserve memory contents
    on conversions.
 
  - Hack-a-fix a deadlock bug in ->error_remove_folio() by returning -EBUSY if
    the filemap invalidate lock is contended.  Punt on a proper fix for the
    moment, as a proper fix likely involves reworking truncate_error_folio().
 
  - Add a pile of selftests for in-place conversions.
 
  - Misc guest_memfd cleanups
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEKTobbabEP7vbhhN9OlYIJqCjN/0FAmrRipMACgkQOlYIJqCj
 N/0gbg/7BgQ72N9KIke30rHHgKcmpyxYBZe1r/yKhWyYPcpGjX+7UxeoGegv2CRS
 pbnr0fTAKKlCXRFRVcHrwg2bXY3tLgWMK3i/jGFmh3i6buii3iYRFT0kl63jt1S6
 Ai72VKhAp/EmzzAI9RaWnU2BRAVlD2uurrBIy/N28vgWzfgIFWyOEGSXOyyrn3vG
 BZBbSDPBqYJ2vYuj6UbHfLRbFmtyNWVM/XL+Lm3i6HKZ2uvfwWZSa7w5kqJZ+2hV
 2+P5Ipu0KZkIumxHkndzZuFNtorwIHxbDGwvbyyKrgsREfRnywCGFxTJA8+bxQtm
 IzFx/2UK71Bg4eKN+SGkt242SqjI+zaZWF1NahGsgRO3spRikWNS11+jEjtEiYsw
 H0N+q1rWr4CM6T063aw0TqqNDrpOrxdFBjY75T2jmhNSmgkwnizhi3NO5bIwfgfi
 tioLDXhg6XAPLPOk6xLgaaJnyVFS0aqB1lHVhftDsgtvHpCcpOxXU2g89jf3HZ0q
 8smglIn4mDhZYtQoswT4sBad0ZCTNEUncyCNhnWNyBkG3GwJEMGRgE2HMxtiKIAm
 6eUSrpMpvVBArSJTyb73GWBMioxaPcuTdWAAmUAgs9RfITGgskxSYv7ZJM2kEh3g
 0EGpQe2yK1Ul3hKzI1Ty+rZfCqHNcx6P65pDhKG6tvlxxewfinc=
 =Jrvd
 -----END PGP SIGNATURE-----

Merge tag 'kvm-x86-coco-7.4' of https://github.com/kvm-x86/linux into HEAD

KVM guest_memfd and CoCo changes for 7.4

 - Take the invalidate lock when unbinding a dying file to fix a race with
   kvm_gmem_invalidate_end() that results in KVM leaving the MMU invalidation
   count elevated.  Not taking the lock in the first place was pure misguided
   paranoid.

 - When binding a memslot, purge the bindings if storing the new memslot fails,
   as Xarray's "store range" API can subtly fail partway through.

 - Establish memslot<=>guest_memfd bindings *after* the memslot is ready, as
   the memslot becomes reachable the instant the binding is installed.

 - Don't grab a page refcount when reloading an SNP VMSA and stop returning a
   "struct page" from guest_memfd's PFN lookup altogether, to ensure that KVM
   doesn't hold any unnecessary refcounts (the lifetime of the page is ensured
   by MMU notifiers and similar hooks).  Preventing unnecessary refcounts
   greatly simplifies supporting in-place SHARED=>PRIVATE conversions.

 - Add support for in-place PRIVATE<=>SHARED conversion, i.e. allow a single
   guest_memfd instance to be used for both PRIVATE and SHARED variations of
   CoCo guests' memory (though not at the same time).  This eliminates the need
   to double allocate guest memory for TDX and SNP, and is basically a hard
   requirement for pKVM, which relies on being able to preserve memory contents
   on conversions.

 - Hack-a-fix a deadlock bug in ->error_remove_folio() by returning -EBUSY if
   the filemap invalidate lock is contended.  Punt on a proper fix for the
   moment, as a proper fix likely involves reworking truncate_error_folio().

 - Add a pile of selftests for in-place conversions.

 - Misc guest_memfd cleanups

Conflicts in arch/arm64/kvm/mmu.c: convert to use s2vi.
This commit is contained in:
Paolo Bonzini 2026-10-16 09:45:16 +02:00
commit 29c925376d
28 changed files with 1872 additions and 414 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Documentation/admin-guide/kernel-parameters.txt
View File

@ -3159,6 +3159,32 @@ Kernel parameters
kvm.enable_vmware_backdoor=[KVM] Support VMware backdoor PV interface.
Default is false (don't support).
kvm.gmem_in_place_conversion=
[KVM] Controls whether KVM enables in-place conversion
support for guest_memfd and tracks the private/shared
state of memory per guest_memfd instead of per VM.
If enabled, KVM enables the KVM_SET_MEMORY_ATTRIBUTES2
ioctl on guest_memfd file descriptors and disables the
legacy VM-scoped KVM_SET_MEMORY_ATTRIBUTES ioctl for
private memory state tracking. Only the
KVM_MEMORY_ATTRIBUTE_PRIVATE attribute moves to
per-guest_memfd tracking; other attributes remain
per-VM.
This parameter toggles KVM's in-place conversion
capability support. Whether a VMM uses separate backends
or out-of-place memory management is determined by
userspace VMM design.
Note, this parameter is only available when
CONFIG_KVM_VM_MEMORY_ATTRIBUTES=y. When
CONFIG_KVM_VM_MEMORY_ATTRIBUTES is not set, in-place
conversion is enabled (and cannot be disabled) if KVM
is built with private memory support.
Default is N (off).
kvm.nx_huge_pages=
[KVM] Controls the software workaround for the
X86_BUG_ITLB_MULTIHIT bug.

116
Documentation/virt/kvm/api.rst
View File

@ -117,7 +117,7 @@ description:
x86 includes both i386 and x86_64.
Type:
system, vm, or vcpu.
system, vm, vcpu or guest_memfd.
Parameters:
what parameters are accepted by the ioctl.
@ -6397,11 +6397,24 @@ mapping for userspace_addr is not required to be valid/populated at the time of
KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated
on-demand.
When mapping a gfn into the guest, KVM selects shared vs. private, i.e consumes
userspace_addr vs. guest_memfd, based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE
state. At VM creation time, all memory is shared, i.e. the PRIVATE attribute
is '0' for all gfns. Userspace can control whether memory is shared/private by
toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via KVM_SET_MEMORY_ATTRIBUTES as needed.
When mapping a gfn into the guest, guest faults are always serviced from
guest_memfd regardless of whether memory is shared or private. KVM determines
shared vs. private based on the state in guest_memfd, which is the sole
authority on private vs. shared memory. See :ref:`KVM_CREATE_GUEST_MEMFD` to
find out more about the creation-time shared/private status. Userspace can
control whether memory is shared/private by toggling
KVM_MEMORY_ATTRIBUTE_PRIVATE via :ref:`KVM_SET_MEMORY_ATTRIBUTES2` as needed.
userspace_addr is expected to be the mmap()-ed address corresponding to the
right offset within the guest_memfd. Any mismatch between userspace_addr and
guest_memfd is not validated and is a user error. userspace_addr is only used
for host-side guest accesses such as kvm_read_guest().
If in-place conversion is disabled, KVM selects shared vs. private based on the
gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE state. At VM creation time, all memory is
shared, i.e. the PRIVATE attribute is '0' for all gfns. Userspace can control
whether memory is shared/private by toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via
KVM_SET_MEMORY_ATTRIBUTES as needed.
S390:
^^^^^
@ -6409,6 +6422,8 @@ S390:
Returns -EINVAL if the VM has the KVM_VM_S390_UCONTROL flag set.
Returns -EINVAL if called on a protected VM.
.. _KVM_SET_MEMORY_ATTRIBUTES:
4.141 KVM_SET_MEMORY_ATTRIBUTES
-------------------------------
@ -6445,6 +6460,10 @@ the state of a gfn/page as needed.
The "flags" field is reserved for future extensions and must be '0'.
See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES2`.
.. _KVM_CREATE_GUEST_MEMFD:
4.142 KVM_CREATE_GUEST_MEMFD
----------------------------
@ -6499,10 +6518,10 @@ specified via KVM_CREATE_GUEST_MEMFD. Currently defined flags:
page tables. Private memory cannot.
============================ ================================================
When the KVM MMU performs a PFN lookup to service a guest fault and the backing
guest_memfd has the GUEST_MEMFD_FLAG_MMAP set, then the fault will always be
consumed from guest_memfd, regardless of whether it is a shared or a private
fault.
When the KVM MMU performs a PFN lookup to service a guest fault, the fault will
always be consumed from guest_memfd, regardless of whether it is a shared or a
private fault (unless in-place conversion is disabled and the backing
guest_memfd does not have the GUEST_MEMFD_FLAG_MMAP flag set).
See KVM_SET_USER_MEMORY_REGION2 for additional details.
@ -6692,6 +6711,83 @@ significant bit):
Userspace should use the defined constants from ``<linux/kvm.h>`` rather
than hardcoding bit positions.
.. _KVM_SET_MEMORY_ATTRIBUTES2:
4.146 KVM_SET_MEMORY_ATTRIBUTES2
---------------------------------
:Capability: KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES
:Architectures: all
:Type: guest_memfd ioctl
:Parameters: struct kvm_memory_attributes2 (in/out)
:Returns: 0 on success, <0 on error
Errors:
========== ===============================================================
EINVAL The specified `offset` or `size` was invalid (e.g. not
page aligned, causes an overflow, or size is zero).
EFAULT The parameter address was invalid.
ENOMEM Ran out of memory trying to track private/shared state
EAGAIN Pages in the range have outstanding references, see
``error_offset`` below.
========== ===============================================================
KVM_SET_MEMORY_ATTRIBUTES2 is an extension to
KVM_SET_MEMORY_ATTRIBUTES that supports returning (writing) values to
userspace. The original (pre-extension) fields are shared with
KVM_SET_MEMORY_ATTRIBUTES identically.
Attribute values are shared with KVM_SET_MEMORY_ATTRIBUTES.
::
struct kvm_memory_attributes2 {
union {
__u64 address;
__u64 offset;
};
__u64 size;
__u64 attributes;
__u64 flags;
__u64 error_offset;
__u64 reserved[11];
};
#define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3)
The capability KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES enumerates the attributes
that can be set via KVM_SET_MEMORY_ATTRIBUTES2.
Set attributes for a range of offsets within a guest_memfd to
KVM_MEMORY_ATTRIBUTE_PRIVATE to limit the specified guest_memfd backed
memory range for guest use. Even if the guest_memfd was created with
GUEST_MEMFD_FLAG_MMAP, after a successful call to set
KVM_MEMORY_ATTRIBUTE_PRIVATE, the requested range will not be mappable
into host userspace and will only be mappable by the guest.
To allow the range to be mappable into host userspace again, call
KVM_SET_MEMORY_ATTRIBUTES2 on the guest_memfd again with
KVM_MEMORY_ATTRIBUTE_PRIVATE unset.
KVM does not directly manipulate the memory contents of pages during
attribute updates. However, the process of setting these attributes,
which includes operations such as unmapping pages from the host or
stage-2 page tables, may result in side effects on memory contents
that vary across different trusted firmware implementations.
If this ioctl returns -EAGAIN, the offset of the page with unexpected
refcounts will be returned in ``error_offset``. This can occur if
there are transient refcounts on the pages, taken by other parts of
the kernel.
Userspace is expected to figure out how to remove all known refcounts
on the shared pages, such as refcounts taken by get_user_pages(), and
try the ioctl again. A possible source of these long term refcounts is
if the guest_memfd memory was pinned in IOMMU page tables.
See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES`.
.. _kvm_run:
5. The kvm_run structure

14
Documentation/virt/kvm/x86/amd-memory-encryption.rst
View File

@ -503,7 +503,8 @@ secrets.
It is required that the GPA ranges initialized by this command have had the
KVM_MEMORY_ATTRIBUTE_PRIVATE attribute set in advance. See the documentation
for KVM_SET_MEMORY_ATTRIBUTES for more details on this aspect.
for KVM_SET_MEMORY_ATTRIBUTES/KVM_SET_MEMORY_ATTRIBUTES2 for more details on
this aspect.
Upon success, this command is not guaranteed to have processed the entire
range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of
@ -511,9 +512,14 @@ range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of
remaining range that has yet to be processed. The caller should continue
calling this command until those fields indicate the entire range has been
processed, e.g. ``len`` is 0, ``gfn_start`` is equal to the last GFN in the
range plus 1, and ``uaddr`` is the last byte of the userspace-provided source
buffer address plus 1. In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO,
``uaddr`` will be ignored completely.
range plus 1, and ``uaddr`` (if specified) is the last byte of the
userspace-provided source buffer address plus 1.
In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, ``uaddr`` will be
ignored completely. For all other page types, ``uaddr`` is optional if in-place
conversion is enabled (i.e. when the data had been written directly to
guest_memfd while the page was in the shared state) and is required if in-place
conversion is disabled.
Parameters (in): struct kvm_sev_snp_launch_update

4
Documentation/virt/kvm/x86/intel-tdx.rst
View File

@ -158,6 +158,10 @@ KVM_TDX_INIT_MEM_REGION
Initialize @nr_pages TDX guest private memory starting from @gpa with userspace
provided data from @source_addr. @source_addr must be PAGE_SIZE-aligned.
If guest_memfd in-place conversion is enabled, pass 0 for @source_addr
to represent "no source page". A source page is required if in-place
conversion is not enabled or not supported.
Note, before calling this sub command, memory attribute of the range
[gpa, gpa + nr_pages] needs to be private. Userspace can use
KVM_SET_MEMORY_ATTRIBUTES to set the attribute.

3
arch/arm64/kvm/mmu.c
View File

@ -1773,7 +1773,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
/* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */
smp_rmb();
ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, &s2vi.page, NULL);
ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, NULL);
if (ret) {
/* If result is non-NULL this is a synthetic fault. */
if (!result)
@ -1827,7 +1827,6 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd,
}
out_unlock:
kvm_release_faultin_page(kvm, s2vi.page, !!ret, prot & KVM_PGTABLE_PROT_W);
kvm_fault_unlock(kvm);
kfree(mapping);

4
arch/arm64/kvm/nested.c
View File

@ -1559,7 +1559,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
bool write_fault, writable;
unsigned long mmu_seq;
struct vncr_tlb *vt;
struct page *page;
struct page *page = NULL;
u64 va, pfn, gfn;
int ret;
@ -1615,7 +1615,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem)
return -EFAULT;
}
} else {
ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL);
ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, NULL);
if (ret) {
kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE,
write_fault, false, false);

2
arch/x86/include/asm/kvm-x86-ops.h
View File

@ -150,7 +150,7 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page)
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
KVM_X86_OP_OPTIONAL_RET0(gmem_make_private)
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM)
KVM_X86_OP_OPTIONAL(gmem_make_shared)
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE

9
arch/x86/include/asm/kvm_host.h
View File

@ -1738,7 +1738,7 @@ struct kvm_x86_ops {
int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
kvm_pfn_t nr_pages);
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM
#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM)
void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages);
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE
@ -1857,9 +1857,14 @@ enum kvm_intr_type {
((vcpu) && (vcpu)->arch.handling_intr_from_guest && \
(!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI)))
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#if defined(CONFIG_KVM_SW_PROTECTED_VM) || \
defined(CONFIG_KVM_INTEL_TDX) || \
defined(CONFIG_KVM_AMD_SEV)
#define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem)
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
#define kvm_arch_has_gmem_convert() (!!kvm_x86_ops.gmem_make_private)
#endif
#define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state)

15
arch/x86/kvm/Kconfig
View File

@ -76,11 +76,22 @@ config KVM_WERROR
If in doubt, say "N".
config KVM_VM_MEMORY_ATTRIBUTES
bool "Enable per-VM PRIVATE vs. SHARED attributes (for CoCo VMs)"
depends on KVM_SW_PROTECTED_VM || KVM_INTEL_TDX || KVM_AMD_SEV
help
Enable support for tracking PRIVATE vs. SHARED memory using per-VM
memory attributes. Using per-VM attributes is deprecated in favor of
tracking PRIVATE state in guest_memfd. Select this if you need to run
CoCo VMs using a VMM that doesn't support guest_memfd memory
attributes.
If unsure, say N.
config KVM_SW_PROTECTED_VM
bool "Enable support for KVM software-protected VMs"
depends on EXPERT
depends on KVM_X86 && X86_64
select KVM_GENERIC_MEMORY_ATTRIBUTES
help
Enable support for KVM software-protected VMs. Currently, software-
protected VMs are purely a development and testing vehicle for
@ -131,7 +142,6 @@ config KVM_INTEL_TDX
bool "Intel Trust Domain Extensions (TDX) support"
default y
depends on INTEL_TDX_HOST
select KVM_GENERIC_MEMORY_ATTRIBUTES
select HAVE_KVM_ARCH_GMEM_POPULATE
help
Provides support for launching Intel Trust Domain Extensions (TDX)
@ -155,7 +165,6 @@ config KVM_AMD_SEV
depends on KVM_AMD && X86_64
depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m)
select ARCH_HAS_CC_PLATFORM
select KVM_GENERIC_MEMORY_ATTRIBUTES
select HAVE_KVM_ARCH_GMEM_CONVERT
select HAVE_KVM_ARCH_GMEM_RECLAIM
select HAVE_KVM_ARCH_GMEM_INVALIDATE

30
arch/x86/kvm/mmu/mmu.c
View File

@ -3427,7 +3427,7 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault,
is_private = fault->is_private;
} else {
max_level = PG_LEVEL_NUM;
is_private = kvm_mem_is_private(kvm, gfn);
is_private = kvm_is_private_gfn(kvm, gfn);
}
max_level = min(max_level, max_huge_page_level);
@ -3640,13 +3640,13 @@ static bool page_fault_can_be_fast(struct kvm *kvm, struct kvm_page_fault *fault
* guest spinning on a #PF indefinitely, so don't attempt the fast path
* in this case.
*
* Note that the kvm_mem_is_private() check might race with an
* Note that the kvm_is_private_gfn() check might race with an
* attribute update, but this will either result in the guest spinning
* on RET_PF_SPURIOUS until the update completes, or an actual spurious
* case might go down the slow path. Either case will resolve itself.
*/
if (kvm->arch.has_private_mem &&
fault->is_private != kvm_mem_is_private(kvm, fault->gfn))
fault->is_private != kvm_is_private_gfn(kvm, fault->gfn))
return false;
/*
@ -4634,7 +4634,7 @@ static int kvm_mmu_faultin_pfn_gmem(struct kvm_vcpu *vcpu,
}
r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn,
&fault->refcounted_page, &max_order);
&max_order);
if (r) {
kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
return r;
@ -4714,7 +4714,7 @@ static int kvm_mmu_faultin_pfn(struct kvm_vcpu *vcpu,
* Now that we have a snapshot of mmu_invalidate_seq we can check for a
* private vs. shared mismatch.
*/
if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) {
if (fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) {
kvm_mmu_prepare_memory_fault_exit(vcpu, fault);
return -EFAULT;
}
@ -5122,7 +5122,7 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
direct_bits = 0;
if (kvm_arch_has_private_mem(vcpu->kvm) &&
kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa)))
kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(range->gpa)))
error_code |= PFERR_PRIVATE_ACCESS;
else
direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
@ -6595,7 +6595,7 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err
if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) &&
!(error_code & PFERR_RSVD_MASK) &&
vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM &&
kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)))
kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)))
error_code |= PFERR_PRIVATE_ACCESS;
r = RET_PF_INVALID;
@ -8108,7 +8108,7 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread);
}
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
int level)
{
@ -8127,8 +8127,8 @@ static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn,
lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG;
}
bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range)
bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range)
{
struct kvm_memory_slot *slot = range->slot;
int level;
@ -8197,18 +8197,18 @@ static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot,
const unsigned long end = start + KVM_PAGES_PER_HPAGE(level);
if (level == PG_LEVEL_2M)
return kvm_range_has_memory_attributes(kvm, start, end, ~0, attrs);
return kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attrs);
for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) {
if (hugepage_test_mixed(slot, gfn, level - 1) ||
attrs != kvm_get_memory_attributes(kvm, gfn))
attrs != kvm_get_vm_memory_attributes(kvm, gfn))
return false;
}
return true;
}
bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range)
bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range)
{
unsigned long attrs = range->arg.attributes;
struct kvm_memory_slot *slot = range->slot;
@ -8301,7 +8301,7 @@ void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
* be manually checked as the attributes may already be mixed.
*/
for (gfn = start; gfn < end; gfn += nr_pages) {
unsigned long attrs = kvm_get_memory_attributes(kvm, gfn);
unsigned long attrs = kvm_get_vm_memory_attributes(kvm, gfn);
if (hugepage_has_attrs(kvm, slot, gfn, level, attrs))
hugepage_clear_mixed(slot, gfn, level);

66
arch/x86/kvm/svm/sev.c
View File

@ -2416,7 +2416,7 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
*/
if (ret && !snp_page_reclaim(kvm, pfn) &&
sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID &&
sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) {
sev_populate_args->fw_error == SEV_RET_INVALID_PARAM && src_page) {
void *src_vaddr = kmap_local_page(src_page);
void *dst_vaddr = kmap_local_pfn(pfn);
@ -2449,8 +2449,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
if (copy_from_user(&params, u64_to_user_ptr(argp->data), sizeof(params)))
return -EFAULT;
pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d\n", __func__,
params.gfn_start, params.len, params.type, params.flags);
pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d src %llx\n", __func__,
params.gfn_start, params.len, params.type, params.flags, params.uaddr);
if (!params.len || !PAGE_ALIGNED(params.len) || params.flags ||
(params.type != KVM_SEV_SNP_PAGE_TYPE_NORMAL &&
@ -2462,7 +2462,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO)
src = NULL;
else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr))
else if ((!gmem_in_place_conversion && !params.uaddr) ||
!PAGE_ALIGNED(params.uaddr))
return -EINVAL;
else
src = u64_to_user_ptr(params.uaddr);
@ -2509,7 +2510,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp)
params.gfn_start += count;
params.len -= count * PAGE_SIZE;
if (params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO)
if (src && params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO)
params.uaddr += count * PAGE_SIZE;
if (copy_to_user(u64_to_user_ptr(argp->data), &params, sizeof(params)))
@ -4047,7 +4048,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
struct kvm *kvm = vcpu->kvm;
gfn_t gfn = gpa_to_gfn(gpa);
unsigned long mmu_seq;
struct page *page;
kvm_pfn_t pfn;
lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex);
@ -4091,7 +4091,7 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
* The new VMSA will be private memory guest memory, so retrieve the
* PFN from the gmem backend.
*/
if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL))
if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, NULL))
return;
read_lock(&kvm->mmu_lock);
@ -4107,8 +4107,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa)
else
svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn);
read_unlock(&kvm->mmu_lock);
kvm_release_page_clean(page);
}
/*
@ -5034,7 +5032,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
struct kvm_memory_slot *slot;
struct kvm *kvm = vcpu->kvm;
int order, rmp_level, ret;
struct page *page;
unsigned long mmu_seq;
bool assigned;
kvm_pfn_t pfn;
gfn_t gfn;
@ -5048,7 +5046,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
* userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults
* for shared pages should not end up here.
*/
if (!kvm_mem_is_private(kvm, gfn)) {
if (!kvm_is_private_gfn(kvm, gfn)) {
pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n",
gpa);
return;
@ -5061,7 +5059,10 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
return;
}
ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &page, &order);
mmu_seq = kvm->mmu_invalidate_seq;
smp_rmb();
ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &order);
if (ret) {
pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n",
gpa);
@ -5070,9 +5071,13 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level);
if (ret || !assigned) {
pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n",
gpa, pfn, ret);
goto out_no_trace;
guard(read_lock)(&kvm->mmu_lock);
if (!mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn))
pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n",
gpa, pfn, ret);
return;
}
/*
@ -5100,26 +5105,31 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code)
if (rmp_level == PG_LEVEL_4K)
goto out;
ret = snp_rmptable_psmash(pfn);
if (ret) {
/*
* Look it up again. If it's 4K now then the PSMASH may have
* raced with another process and the issue has already resolved
* itself.
*/
if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) &&
assigned && rmp_level == PG_LEVEL_4K)
scoped_guard(read_lock, &kvm->mmu_lock) {
if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn))
goto out;
pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n",
gpa, pfn, ret);
ret = snp_rmptable_psmash(pfn);
if (ret) {
/*
* Look it up again. If it's 4K now then the PSMASH may
* have raced with another process and the issue has
* already resolved itself. If it's not assigned, then
* this must have raced with another process that made
* this page shared.
*/
if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) &&
((assigned && rmp_level == PG_LEVEL_4K) || !assigned))
goto out;
pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n",
gpa, pfn, ret);
}
}
kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD);
out:
trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret);
out_no_trace:
kvm_release_page_unused(page);
}
static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end)

8
arch/x86/kvm/vmx/tdx.c
View File

@ -3184,7 +3184,7 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm))
return -EIO;
kvm_tdx->page_add_src = src_page;
kvm_tdx->page_add_src = src_page ?: pfn_to_page(pfn);
ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn);
kvm_tdx->page_add_src = NULL;
@ -3230,7 +3230,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
if (copy_from_user(&region, u64_to_user_ptr(cmd->data), sizeof(region)))
return -EFAULT;
if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr ||
if (!PAGE_ALIGNED(region.source_addr) ||
(!gmem_in_place_conversion && !region.source_addr) ||
!PAGE_ALIGNED(region.gpa) || !region.nr_pages ||
region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa ||
!vt_is_tdx_private_gpa(kvm, region.gpa) ||
@ -3261,7 +3262,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c
break;
}
region.source_addr += PAGE_SIZE;
if (region.source_addr)
region.source_addr += PAGE_SIZE;
region.gpa += PAGE_SIZE;
region.nr_pages--;

20
arch/x86/kvm/x86.c
View File

@ -10144,7 +10144,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm,
}
}
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
kvm_mmu_init_memslot_memory_attributes(kvm, slot);
#endif
@ -10641,14 +10641,15 @@ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
}
#ifdef CONFIG_KVM_GUEST_MEMFD
/*
* KVM doesn't yet support initializing guest_memfd memory as shared for VMs
* with private memory (the private vs. shared tracking needs to be moved into
* guest_memfd).
*/
bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
{
return !kvm_arch_has_private_mem(kvm);
/*
* INIT_SHARED is supported if in-place conversion is enabled, or if
* the VM doesn't support private memory. If the VM has private memory
* and in-place conversion is disabled, then guest_memfd can _only_ be
* used for private memory.
*/
return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm);
}
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
@ -10657,6 +10658,11 @@ int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
{
return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages);
}
void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages)
{
kvm_x86_call(gmem_make_shared)(pfn, nr_pages);
}
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM

89
include/linux/kvm_host.h
View File

@ -726,27 +726,6 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu)
}
#endif
#ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
static inline bool kvm_arch_has_private_mem(struct kvm *kvm)
{
return false;
}
#endif
#ifdef CONFIG_KVM_GUEST_MEMFD
bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm);
static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm)
{
u64 flags = GUEST_MEMFD_FLAG_MMAP;
if (!kvm || kvm_arch_supports_gmem_init_shared(kvm))
flags |= GUEST_MEMFD_FLAG_INIT_SHARED;
return flags;
}
#endif
#ifndef kvm_arch_has_readonly_mem
static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm)
{
@ -887,7 +866,7 @@ struct kvm {
#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER
struct notifier_block pm_notifier;
#endif
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
/* Protected by slots_lock (for writes) and RCU (for reads) */
struct xarray mem_attr_array;
#endif
@ -2591,48 +2570,80 @@ static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot)
return slot->flags & KVM_MEMSLOT_GMEM_ONLY;
}
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn)
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
static inline unsigned long kvm_get_vm_memory_attributes(struct kvm *kvm, gfn_t gfn)
{
return xa_to_value(xa_load(&kvm->mem_attr_array, gfn));
}
bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
unsigned long mask, unsigned long attrs);
bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range);
bool kvm_arch_post_set_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range);
bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
unsigned long mask, unsigned long attrs);
bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range);
bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range);
static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
static inline bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn)
{
return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE;
return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE;
}
#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */
#ifdef kvm_arch_has_private_mem
extern bool gmem_in_place_conversion;
typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn);
DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t);
static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn)
{
return static_call(__kvm_is_private_gfn)(kvm, gfn);
}
#else
static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn)
#define gmem_in_place_conversion false
static inline bool kvm_arch_has_private_mem(struct kvm *kvm)
{
return false;
}
#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn)
{
return false;
}
#endif /* kvm_arch_has_private_mem */
#ifdef CONFIG_KVM_GUEST_MEMFD
bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn);
bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm);
static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm)
{
u64 flags = GUEST_MEMFD_FLAG_MMAP;
if (!kvm || kvm_arch_supports_gmem_init_shared(kvm))
flags |= GUEST_MEMFD_FLAG_INIT_SHARED;
return flags;
}
int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
gfn_t gfn, kvm_pfn_t *pfn, struct page **page,
int *max_order);
gfn_t gfn, kvm_pfn_t *pfn, int *max_order);
#else
static inline int kvm_gmem_get_pfn(struct kvm *kvm,
struct kvm_memory_slot *slot, gfn_t gfn,
kvm_pfn_t *pfn, struct page **page,
int *max_order)
kvm_pfn_t *pfn, int *max_order)
{
KVM_BUG_ON(1, kvm);
return -EIO;
}
#endif /* CONFIG_KVM_GUEST_MEMFD */
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
kvm_pfn_t nr_pages);
void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages);
#ifndef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
#define kvm_arch_has_gmem_convert() false
#endif
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE

6
include/trace/events/kvm.h
View File

@ -364,13 +364,13 @@ TRACE_EVENT(kvm_dirty_ring_exit,
TP_printk("vcpu %d", __entry->vcpu_id)
);
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
/*
* @start: Starting address of guest memory range
* @end: End address of guest memory range
* @attr: The value of the attribute being set.
*/
TRACE_EVENT(kvm_vm_set_mem_attributes,
TRACE_EVENT(kvm_set_vm_mem_attributes,
TP_PROTO(gfn_t start, gfn_t end, unsigned long attr),
TP_ARGS(start, end, attr),
@ -389,7 +389,7 @@ TRACE_EVENT(kvm_vm_set_mem_attributes,
TP_printk("%#016llx -- %#016llx [0x%lx]",
__entry->start, __entry->end, __entry->attr)
);
#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */
TRACE_EVENT(kvm_unmap_hva_range,
TP_PROTO(unsigned long start, unsigned long end),

16
include/uapi/linux/kvm.h
View File

@ -999,6 +999,7 @@ struct kvm_enable_cap {
#define KVM_CAP_S390_HPAGE_2G 249
#define KVM_CAP_PPC_COMPAT_CAPS 250
#define KVM_CAP_ARM_PMU_V3_STRICT 251
#define KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES 252
struct kvm_irq_routing_irqchip {
__u32 irqchip;
@ -1653,6 +1654,21 @@ struct kvm_memory_attributes {
__u64 flags;
};
/* Available with KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES */
#define KVM_SET_MEMORY_ATTRIBUTES2 _IOWR(KVMIO, 0xd2, struct kvm_memory_attributes2)
struct kvm_memory_attributes2 {
union {
__u64 address;
__u64 offset;
};
__u64 size;
__u64 attributes;
__u64 flags;
__u64 error_offset;
__u64 reserved[11];
};
#define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3)
#define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd)

1
tools/testing/selftests/kvm/Makefile.kvm
View File

@ -156,6 +156,7 @@ TEST_GEN_PROGS_x86 += x86/max_vcpuid_cap_test
TEST_GEN_PROGS_x86 += x86/triple_fault_event_test
TEST_GEN_PROGS_x86 += x86/recalc_apic_map_test
TEST_GEN_PROGS_x86 += x86/aperfmperf_test
TEST_GEN_PROGS_x86 += x86/guest_memfd_conversions_test
TEST_GEN_PROGS_x86 += access_tracking_perf_test
TEST_GEN_PROGS_x86 += coalesced_io_test
TEST_GEN_PROGS_x86 += dirty_log_perf_test

139
tools/testing/selftests/kvm/include/kvm_util.h
View File

@ -392,6 +392,16 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { }
__TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \
})
#define __gmem_ioctl(gmem_fd, cmd, arg) \
kvm_do_ioctl(gmem_fd, cmd, arg)
#define gmem_ioctl(gmem_fd, cmd, arg) \
({ \
int ret = __gmem_ioctl(gmem_fd, cmd, arg); \
\
TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \
})
/*
* Looks up and returns the value corresponding to the capability
* (KVM_CAP_*) given by cap.
@ -418,8 +428,19 @@ static inline void vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0)
vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap);
}
int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset,
size_t *nr_bytes);
/*
* KVM_SET_MEMORY_ATTRIBUTES{,2} overwrites _all_ attributes. These
* flows need significant enhancements to support multiple attributes.
*/
#define TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes) \
TEST_ASSERT(!(attributes) || (attributes) == KVM_MEMORY_ATTRIBUTE_PRIVATE, \
"Update me to support multiple attributes!")
static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa,
u64 size, u64 attributes)
size_t size, u64 attributes)
{
struct kvm_memory_attributes attr = {
.attributes = attributes,
@ -428,27 +449,109 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa,
.flags = 0,
};
/*
* KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows
* need significant enhancements to support multiple attributes.
*/
TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE,
"Update me to support multiple attributes!");
TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes);
vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr);
}
static inline int __gmem_set_memory_attributes(int fd, u64 offset,
size_t size, u64 attributes,
u64 *error_offset)
{
struct kvm_memory_attributes2 attr = {
.attributes = attributes,
.offset = offset,
.size = size,
.flags = 0,
.error_offset = 0,
};
int r;
r = __gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr);
/* Copy error_offset regardless of r so caller can check. */
if (error_offset)
*error_offset = attr.error_offset;
return r;
}
static inline int __gmem_set_private(int fd, u64 offset, size_t size,
u64 *error_offset)
{
return __gmem_set_memory_attributes(fd, offset, size,
KVM_MEMORY_ATTRIBUTE_PRIVATE,
error_offset);
}
static inline int __gmem_set_shared(int fd, u64 offset, size_t size,
u64 *error_offset)
{
return __gmem_set_memory_attributes(fd, offset, size, 0,
error_offset);
}
static inline void gmem_set_memory_attributes(int fd, u64 offset,
size_t size, u64 attributes)
{
struct kvm_memory_attributes2 attr = {
.attributes = attributes,
.offset = offset,
.size = size,
.flags = 0,
};
TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes);
__TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) > 0,
"No valid attributes for guest_memfd ioctl!");
gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr);
}
static inline void gmem_set_private(int fd, u64 offset, size_t size)
{
gmem_set_memory_attributes(fd, offset, size,
KVM_MEMORY_ATTRIBUTE_PRIVATE);
}
static inline void gmem_set_shared(int fd, u64 offset, size_t size)
{
gmem_set_memory_attributes(fd, offset, size, 0);
}
static inline void vm_mem_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa,
size_t size, u64 attrs)
{
if (kvm_has_gmem_attributes) {
gpa_t end = gpa + size;
off_t fd_offset;
gpa_t addr;
size_t len;
int fd;
for (addr = gpa; addr < end; addr += len) {
fd = kvm_gpa_to_guest_memfd(vm, addr, &fd_offset, &len);
len = min(end - addr, len);
gmem_set_memory_attributes(fd, fd_offset, len, attrs);
}
} else {
vm_set_memory_attributes(vm, gpa, size, attrs);
}
}
static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa,
u64 size)
size_t size)
{
vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE);
vm_mem_set_memory_attributes(vm, gpa, size,
KVM_MEMORY_ATTRIBUTE_PRIVATE);
}
static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa,
u64 size)
size_t size)
{
vm_set_memory_attributes(vm, gpa, size, 0);
vm_mem_set_memory_attributes(vm, gpa, size, 0);
}
void vm_guest_mem_fallocate(struct kvm_vm *vm, gpa_t gpa, u64 size,
@ -690,17 +793,17 @@ int __vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags,
gpa_t gpa, u64 size, void *hva);
void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
gpa_t gpa, u64 size, void *hva,
u32 guest_memfd, u64 guest_memfd_offset);
u32 gmem_fd, u64 gmem_offset);
int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
gpa_t gpa, u64 size, void *hva,
u32 guest_memfd, u64 guest_memfd_offset);
u32 gmem_fd, u64 gmem_offset);
void vm_userspace_mem_region_add(struct kvm_vm *vm,
enum vm_mem_backing_src_type src_type,
gpa_t gpa, u32 slot, u64 npages, u32 flags);
void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
gpa_t gpa, u32 slot, u64 npages, u32 flags,
int guest_memfd_fd, u64 guest_memfd_offset);
int gmem_fd, u64 gmem_offset, u64 gmem_flags);
#ifndef vm_arch_has_protected_memory
static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm)
@ -732,6 +835,11 @@ void *addr_gva2hva(struct kvm_vm *vm, gva_t gva);
gpa_t addr_hva2gpa(struct kvm_vm *vm, void *hva);
void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa);
static inline int kvm_slot_to_fd(struct kvm_vm *vm, u32 slot)
{
return memslot2region(vm, slot)->fd;
}
#ifndef vcpu_arch_put_guest
#define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0)
#endif
@ -1123,6 +1231,9 @@ static inline int pin_self_to_any_cpu(void)
return pin_task_to_any_cpu(pthread_self());
}
void pin_pages(void *vaddr, uint64_t size);
void unpin_pages(void);
void kvm_print_vcpu_pinning_help(void);
void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[],
int nr_vcpus);

32
tools/testing/selftests/kvm/include/test_util.h
View File

@ -83,21 +83,23 @@ do { \
__builtin_unreachable(); \
} while (0)
extern sigjmp_buf expect_sigbus_jmpbuf;
void expect_sigbus_handler(int signum);
extern __thread sigjmp_buf expect_sigbus_jmpbuf;
extern __thread volatile sig_atomic_t expecting_sigbus;
void catchall_signal_handler(int signum);
#define TEST_EXPECT_SIGBUS(action) \
do { \
struct sigaction sa_old, sa_new = { \
.sa_handler = expect_sigbus_handler, \
}; \
\
sigaction(SIGBUS, &sa_new, &sa_old); \
if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \
action; \
TEST_FAIL("'%s' should have triggered SIGBUS", #action); \
} \
sigaction(SIGBUS, &sa_old, NULL); \
#define TEST_EXPECT_SIGBUS(action) \
do { \
struct sigaction __sa = {}; \
\
TEST_ASSERT_EQ(sigaction(SIGBUS, NULL, &__sa), 0); \
TEST_ASSERT_EQ(__sa.sa_handler, &catchall_signal_handler); \
\
expecting_sigbus = true; \
if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \
action; \
TEST_FAIL("'%s' should have triggered SIGBUS", #action);\
} \
expecting_sigbus = false; \
} while (0)
size_t parse_size(const char *size);
@ -116,6 +118,8 @@ struct kvm_random_state {
extern u32 kvm_random_seed;
extern struct kvm_random_state kvm_rng;
extern bool kvm_has_gmem_attributes;
struct kvm_random_state new_kvm_random_state(u32 seed);
u32 kvm_random_u32(struct kvm_random_state *state);

237
tools/testing/selftests/kvm/lib/kvm_util.c
View File

@ -19,6 +19,8 @@
#include <unistd.h>
#include <linux/kernel.h>
#include "../../../../../mm/gup_test.h"
#define KVM_UTIL_MIN_PFN 2
u32 kvm_random_seed;
@ -32,6 +34,8 @@ static void kvm_seed_rng(u32 seed)
kvm_rng = new_kvm_random_state(kvm_random_seed);
}
bool kvm_has_gmem_attributes;
static size_t vcpu_mmap_sz(void);
int __open_path_or_exit(const char *path, int flags, const char *enoent_help)
@ -490,6 +494,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus,
u64 nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus,
nr_extra_pages);
struct userspace_mem_region *slot0;
u64 gmem_flags = 0;
struct kvm_vm *vm;
int i, flags;
@ -505,10 +510,20 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus,
* for CoCo VMs that require GUEST_MEMFD backed private memory.
*/
flags = 0;
if (is_guest_memfd_required(shape))
if (is_guest_memfd_required(shape)) {
flags |= KVM_MEM_GUEST_MEMFD;
vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags);
/*
* Allow mmap() and initialize the gmem instance as SHARED if
* in-place conversions are supported, so that the memory can
* be populated/initialized like any other memory region.
*/
if (kvm_has_gmem_attributes)
gmem_flags |= GUEST_MEMFD_FLAG_INIT_SHARED |
GUEST_MEMFD_FLAG_MMAP;
}
vm_mem_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags, -1, 0, gmem_flags);
for (i = 0; i < NR_MEM_REGIONS; i++)
vm->memslots[i] = 0;
@ -527,6 +542,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus,
kvm_seed_rng(kvm_random_seed);
sync_global_to_guest(vm, kvm_rng);
sync_global_to_guest(vm, kvm_has_gmem_attributes);
kvm_arch_vm_post_create(vm, nr_runnable_vcpus);
@ -642,6 +658,29 @@ int __pin_task_to_cpu(pthread_t task, int cpu)
return pthread_setaffinity_np(task, sizeof(cpuset), &cpuset);
}
static int gup_test_fd = -1;
void pin_pages(void *vaddr, uint64_t size)
{
const struct pin_longterm_test args = {
.addr = (uint64_t)vaddr,
.size = size,
.flags = PIN_LONGTERM_TEST_FLAG_USE_WRITE,
};
gup_test_fd = __open_path_or_exit("/sys/kernel/debug/gup_test", O_RDWR,
"Is CONFIG_GUP_TEST enabled?");
TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_START, &args), 0);
}
void unpin_pages(void)
{
TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_STOP), 0);
kvm_free_fd(gup_test_fd);
}
static u32 parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask)
{
u32 pcpu = atoi_non_negative("CPU number", cpu_str);
@ -970,7 +1009,7 @@ void vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags,
int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
gpa_t gpa, u64 size, void *hva,
u32 guest_memfd, u64 guest_memfd_offset)
u32 gmem_fd, u64 gmem_offset)
{
struct kvm_userspace_memory_region2 region = {
.slot = slot,
@ -978,8 +1017,8 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
.guest_phys_addr = gpa,
.memory_size = size,
.userspace_addr = (uintptr_t)hva,
.guest_memfd = guest_memfd,
.guest_memfd_offset = guest_memfd_offset,
.guest_memfd = gmem_fd,
.guest_memfd_offset = gmem_offset,
};
TEST_REQUIRE_SET_USER_MEMORY_REGION2();
@ -989,10 +1028,10 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
gpa_t gpa, u64 size, void *hva,
u32 guest_memfd, u64 guest_memfd_offset)
u32 gmem_fd, u64 gmem_offset)
{
int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva,
guest_memfd, guest_memfd_offset);
gmem_fd, gmem_offset);
TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)",
errno, strerror(errno));
@ -1002,13 +1041,16 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags,
/* FIXME: This thing needs to be ripped apart and rewritten. */
void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
gpa_t gpa, u32 slot, u64 npages, u32 flags,
int guest_memfd, u64 guest_memfd_offset)
int gmem_fd, u64 gmem_offset, u64 gmem_flags)
{
int ret;
struct userspace_mem_region *region;
size_t backing_src_pagesz = get_backing_src_pagesz(src_type);
size_t mem_size = npages * vm->page_size;
size_t alignment = 1;
size_t backing_src_pagesz;
off_t mmap_offset;
bool is_gmem_mmap;
size_t alignment;
int mmap_flags;
TEST_REQUIRE_SET_USER_MEMORY_REGION2();
@ -1060,58 +1102,40 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
/* Allocate and initialize new mem region structure. */
region = calloc(1, sizeof(*region));
TEST_ASSERT(region != NULL, "Insufficient Memory");
region->mmap_size = mem_size;
/*
* When using THP mmap is not guaranteed to returned a hugepage aligned
* address so we have to pad the mmap. Padding is not needed for HugeTLB
* because mmap will always return an address aligned to the HugeTLB
* page size.
*/
if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
alignment = max(backing_src_pagesz, alignment);
is_gmem_mmap = (flags & KVM_MEM_GUEST_MEMFD) &&
(gmem_flags & GUEST_MEMFD_FLAG_MMAP);
if (is_gmem_mmap) {
backing_src_pagesz = getpagesize();
alignment = 1;
mmap_flags = MAP_SHARED;
mmap_offset = gmem_offset;
} else {
backing_src_pagesz = get_backing_src_pagesz(src_type);
/*
* When using THP mmap is not guaranteed to returned a hugepage aligned
* address so we have to pad the mmap. Padding is not needed for HugeTLB
* because mmap will always return an address aligned to the HugeTLB
* page size.
*/
alignment = src_type == VM_MEM_SRC_ANONYMOUS_THP ? backing_src_pagesz : 1;
mmap_flags = vm_mem_backing_src_alias(src_type)->flag;
mmap_offset = 0;
}
TEST_ASSERT_EQ(gpa, align_up(gpa, backing_src_pagesz));
region->mmap_size = mem_size;
/* Add enough memory to align up if necessary */
if (alignment > 1)
region->mmap_size += alignment;
region->fd = -1;
if (backing_src_is_shared(src_type))
region->fd = kvm_memfd_alloc(region->mmap_size,
src_type == VM_MEM_SRC_SHARED_HUGETLB);
region->mmap_start = kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE,
vm_mem_backing_src_alias(src_type)->flag,
region->fd);
TEST_ASSERT(!is_backing_src_hugetlb(src_type) ||
region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz),
"mmap_start %p is not aligned to HugeTLB page size 0x%lx",
region->mmap_start, backing_src_pagesz);
/* Align host address */
region->host_mem = align_ptr_up(region->mmap_start, alignment);
/* As needed perform madvise */
if ((src_type == VM_MEM_SRC_ANONYMOUS ||
src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {
ret = madvise(region->host_mem, mem_size,
src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",
region->host_mem, mem_size,
vm_mem_backing_src_alias(src_type)->name);
}
region->backing_src_type = src_type;
if (flags & KVM_MEM_GUEST_MEMFD) {
if (guest_memfd < 0) {
u32 guest_memfd_flags = 0;
TEST_ASSERT(!guest_memfd_offset,
if (gmem_fd < 0) {
TEST_ASSERT(!gmem_offset,
"Offset must be zero when creating new guest_memfd");
guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags);
gmem_fd = vm_create_guest_memfd(vm, mem_size, gmem_flags);
} else {
/*
* Install a unique fd for each memslot so that the fd
@ -1119,15 +1143,51 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
* needing to track if the fd is owned by the framework
* or by the caller.
*/
guest_memfd = kvm_dup(guest_memfd);
gmem_fd = kvm_dup(gmem_fd);
}
region->region.guest_memfd = guest_memfd;
region->region.guest_memfd_offset = guest_memfd_offset;
region->region.guest_memfd = gmem_fd;
region->region.guest_memfd_offset = gmem_offset;
} else {
region->region.guest_memfd = -1;
}
region->fd = -1;
if (is_gmem_mmap) {
region->fd = kvm_dup(gmem_fd);
} else if (backing_src_is_shared(src_type)) {
region->fd = kvm_memfd_alloc(region->mmap_size,
src_type == VM_MEM_SRC_SHARED_HUGETLB);
}
region->mmap_start = __kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE,
mmap_flags, region->fd, mmap_offset);
/* Align host address */
region->host_mem = align_ptr_up(region->mmap_start, alignment);
if (!is_gmem_mmap) {
TEST_ASSERT(!is_backing_src_hugetlb(src_type) ||
region->mmap_start ==
align_ptr_up(region->mmap_start, backing_src_pagesz),
"mmap_start %p is not aligned to HugeTLB page size 0x%lx",
region->mmap_start, backing_src_pagesz);
/* As needed perform madvise */
if ((src_type == VM_MEM_SRC_ANONYMOUS ||
src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {
int advice = src_type == VM_MEM_SRC_ANONYMOUS ?
MADV_NOHUGEPAGE : MADV_HUGEPAGE;
ret = madvise(region->host_mem, mem_size, advice);
TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",
region->host_mem, mem_size,
vm_mem_backing_src_alias(src_type)->name);
}
}
region->backing_src_type = src_type;
region->unused_phy_pages = sparsebit_alloc();
if (vm_arch_has_protected_memory(vm))
region->protected_phy_pages = sparsebit_alloc();
@ -1152,10 +1212,10 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type,
/* If shared memory, create an alias. */
if (region->fd >= 0) {
region->mmap_alias = kvm_mmap(region->mmap_size,
PROT_READ | PROT_WRITE,
vm_mem_backing_src_alias(src_type)->flag,
region->fd);
region->mmap_alias = __kvm_mmap(region->mmap_size,
PROT_READ | PROT_WRITE,
mmap_flags, region->fd,
mmap_offset);
/* Align host alias address */
region->host_alias = align_ptr_up(region->mmap_alias, alignment);
@ -1166,7 +1226,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm,
enum vm_mem_backing_src_type src_type,
gpa_t gpa, u32 slot, u64 npages, u32 flags)
{
vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0);
vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0, 0);
}
/*
@ -1299,27 +1359,20 @@ void vm_guest_mem_fallocate(struct kvm_vm *vm, u64 base, u64 size,
bool punch_hole)
{
const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0);
struct userspace_mem_region *region;
u64 end = base + size;
gpa_t gpa, len;
off_t fd_offset;
int ret;
int fd, ret;
size_t len;
gpa_t gpa;
for (gpa = base; gpa < end; gpa += len) {
u64 offset;
fd = kvm_gpa_to_guest_memfd(vm, gpa, &fd_offset, &len);
len = min(end - gpa, len);
region = userspace_mem_region_find(vm, gpa, gpa);
TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD,
"Private memory region not found for GPA 0x%lx", gpa);
offset = gpa - region->region.guest_phys_addr;
fd_offset = region->region.guest_memfd_offset + offset;
len = min_t(u64, end - gpa, region->region.memory_size - offset);
ret = fallocate(region->region.guest_memfd, mode, fd_offset, len);
ret = fallocate(fd, mode, fd_offset, len);
TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx",
punch_hole ? "punch hole" : "allocate", gpa, len,
region->region.guest_memfd, mode, fd_offset);
fd, mode, fd_offset);
}
}
@ -1656,6 +1709,22 @@ void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa)
return (void *) ((uintptr_t) region->host_alias + offset);
}
int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset,
size_t *nr_bytes)
{
struct userspace_mem_region *region;
gpa_t gpa_offset;
region = userspace_mem_region_find(vm, gpa, gpa);
TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD,
"guest_memfd memory region not found for GPA 0x%lx", gpa);
gpa_offset = gpa - region->region.guest_phys_addr;
*fd_offset = region->region.guest_memfd_offset + gpa_offset;
*nr_bytes = region->region.memory_size - gpa_offset;
return region->region.guest_memfd;
}
/* Create an interrupt controller chip for the specified VM. */
void vm_create_irqchip(struct kvm_vm *vm)
{
@ -2274,13 +2343,20 @@ __weak void kvm_selftest_arch_init(void)
{
}
static void report_unexpected_signal(int signum)
__thread sigjmp_buf expect_sigbus_jmpbuf;
__thread volatile sig_atomic_t expecting_sigbus;
void catchall_signal_handler(int signum)
{
switch (signum) {
case SIGBUS: {
if (expecting_sigbus)
siglongjmp(expect_sigbus_jmpbuf, 1);
TEST_FAIL("Unexpected SIGBUS (%d)\n", signum);
}
#define KVM_CASE_SIGNUM(sig) \
case sig: TEST_FAIL("Unexpected " #sig " (%d)\n", signum)
switch (signum) {
KVM_CASE_SIGNUM(SIGBUS);
KVM_CASE_SIGNUM(SIGSEGV);
KVM_CASE_SIGNUM(SIGILL);
KVM_CASE_SIGNUM(SIGFPE);
@ -2292,12 +2368,13 @@ static void report_unexpected_signal(int signum)
void __attribute((constructor)) kvm_selftest_init(void)
{
struct sigaction sig_sa = {
.sa_handler = report_unexpected_signal,
.sa_handler = catchall_signal_handler,
};
/* Tell stdout not to buffer its content. */
setbuf(stdout, NULL);
expecting_sigbus = false;
sigaction(SIGBUS, &sig_sa, NULL);
sigaction(SIGSEGV, &sig_sa, NULL);
sigaction(SIGILL, &sig_sa, NULL);
@ -2306,6 +2383,8 @@ void __attribute((constructor)) kvm_selftest_init(void)
srandom(time(0));
kvm_seed_rng(random());
kvm_has_gmem_attributes = kvm_has_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES);
kvm_selftest_arch_init();
}

7
tools/testing/selftests/kvm/lib/test_util.c
View File

@ -19,13 +19,6 @@
#include "test_util.h"
#include "kvm_syscalls.h"
sigjmp_buf expect_sigbus_jmpbuf;
void __attribute__((used)) expect_sigbus_handler(int signum)
{
siglongjmp(expect_sigbus_jmpbuf, 1);
}
/*
* Random number generator that is usable from guest code. This is the
* Park-Miller LCG using standard constants.

512
tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c Normal file
View File

@ -0,0 +1,512 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (c) 2024, Google LLC.
*/
#include <pthread.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>
#include <linux/align.h>
#include <linux/kvm.h>
#include <linux/sizes.h>
#include "kvm_util.h"
#include "kvm_syscalls.h"
#include "kselftest_harness.h"
#include "test_util.h"
#include "ucall_common.h"
FIXTURE(gmem_conversions) {
struct kvm_vcpu *vcpu;
int gmem_fd;
/* HVA of the first byte of the memory mmap()-ed from gmem_fd. */
char *mem;
};
typedef FIXTURE_DATA(gmem_conversions) test_data_t;
FIXTURE_SETUP(gmem_conversions) { }
static size_t page_size;
static void guest_do_rmw(void);
#define GUEST_MEMFD_SHARING_TEST_GVA 0x90000000ULL
/*
* Defer setup until the individual test is invoked so that tests can specify
* the number of pages and flags for the guest_memfd instance.
*/
static void gmem_conversions_do_setup(test_data_t *t, int nr_pages,
int gmem_flags)
{
const struct vm_shape shape = {
.mode = VM_MODE_DEFAULT,
.type = KVM_X86_SW_PROTECTED_VM,
};
/*
* Use high GPA above APIC_DEFAULT_PHYS_BASE to avoid clashing with
* APIC_DEFAULT_PHYS_BASE.
*/
const gpa_t gpa = SZ_4G;
const u32 slot = 1;
struct kvm_vm *vm;
vm = __vm_create_shape_with_one_vcpu(shape, &t->vcpu, nr_pages, guest_do_rmw);
vm_mem_add(vm, VM_MEM_SRC_SHMEM, gpa, slot, nr_pages,
KVM_MEM_GUEST_MEMFD, -1, 0, gmem_flags);
t->gmem_fd = kvm_slot_to_fd(vm, slot);
t->mem = addr_gpa2hva(vm, gpa);
virt_map(vm, GUEST_MEMFD_SHARING_TEST_GVA, gpa, nr_pages);
}
static void gmem_conversions_do_teardown(test_data_t *t)
{
/* Use NULL to avoid second free in FIXTURE_TEARDOWN (multipage tests). */
if (!t->vcpu)
return;
/* No need to close gmem_fd, it's owned by the VM structure. */
kvm_vm_free(t->vcpu->vm);
t->vcpu = NULL;
}
FIXTURE_TEARDOWN(gmem_conversions)
{
gmem_conversions_do_teardown(self);
}
/*
* In these test definition macros, __nr_pages and nr_pages is used to set up
* the total number of pages in the guest_memfd under test. This will be
* available in the test definitions as nr_pages.
*/
#define __GMEM_CONVERSION_TEST(test, __nr_pages, flags) \
static void __gmem_conversions_##test(test_data_t *t, int nr_pages); \
\
TEST_F(gmem_conversions, test) \
{ \
const int nr = (__nr_pages); \
\
gmem_conversions_do_setup(self, nr, flags); \
__gmem_conversions_##test(self, nr); \
} \
static void __gmem_conversions_##test(test_data_t *t, int nr_pages) \
#define GMEM_CONVERSION_TEST(test, __nr_pages, flags) \
__GMEM_CONVERSION_TEST(test, __nr_pages, (flags) | GUEST_MEMFD_FLAG_MMAP)
#define __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, __nr_pages) \
GMEM_CONVERSION_TEST(test, __nr_pages, 0)
#define GMEM_CONVERSION_TEST_INIT_PRIVATE(test) \
__GMEM_CONVERSION_TEST_INIT_PRIVATE(test, 1)
#define __GMEM_CONVERSION_TEST_INIT_SHARED(test, __nr_pages) \
GMEM_CONVERSION_TEST(test, __nr_pages, GUEST_MEMFD_FLAG_INIT_SHARED)
#define GMEM_CONVERSION_TEST_INIT_SHARED(test) \
__GMEM_CONVERSION_TEST_INIT_SHARED(test, 1)
/*
* Repeats test over nr_pages in a guest_memfd of size nr_pages, providing each
* test iteration with test_page, the index of the page under test in
* guest_memfd. test_page takes values 0..(nr_pages - 1) inclusive.
*/
#define GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(test, __nr_pages) \
static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \
const int test_page); \
\
TEST_F(gmem_conversions, test) \
{ \
const u64 flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; \
const int nr = (__nr_pages); \
int i; \
\
for (i = 0; i < nr; ++i) { \
gmem_conversions_do_setup(self, nr, flags); \
__gmem_conversions_multipage_##test(self, nr, i); \
gmem_conversions_do_teardown(self); \
} \
} \
static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \
const int test_page)
struct guest_check_data {
void *mem;
char expected_val;
char write_val;
};
static struct guest_check_data guest_data;
static void guest_do_rmw(void)
{
for (;;) {
char *mem = READ_ONCE(guest_data.mem);
GUEST_ASSERT_EQ(READ_ONCE(*mem), READ_ONCE(guest_data.expected_val));
WRITE_ONCE(*mem, READ_ONCE(guest_data.write_val));
GUEST_SYNC(0);
}
}
static void run_guest_do_rmw(struct kvm_vcpu *vcpu, u64 pgoff,
char expected_val, char write_val)
{
struct ucall uc;
int r;
guest_data.mem = (void *)GUEST_MEMFD_SHARING_TEST_GVA + pgoff * page_size;
guest_data.expected_val = expected_val;
guest_data.write_val = write_val;
sync_global_to_guest(vcpu->vm, guest_data);
do {
r = __vcpu_run(vcpu);
} while (r == -1 && errno == EINTR);
TEST_ASSERT_EQ(r, 0);
switch (get_ucall(vcpu, &uc)) {
case UCALL_ABORT:
REPORT_GUEST_ASSERT(uc);
case UCALL_SYNC:
break;
default:
TEST_FAIL("Unexpected ucall %lu", uc.cmd);
}
}
static void host_do_rmw(char *mem, u64 pgoff, char expected_val,
char write_val)
{
TEST_ASSERT_EQ(READ_ONCE(mem[pgoff * page_size]), expected_val);
WRITE_ONCE(mem[pgoff * page_size], write_val);
}
static void test_private(test_data_t *t, u64 pgoff, char starting_val,
char write_val)
{
TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[pgoff * page_size], write_val));
run_guest_do_rmw(t->vcpu, pgoff, starting_val, write_val);
TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[pgoff * page_size]));
}
static void test_convert_to_private(test_data_t *t, u64 pgoff,
char starting_val, char write_val)
{
gmem_set_private(t->gmem_fd, pgoff * page_size, page_size);
test_private(t, pgoff, starting_val, write_val);
}
static void test_shared(test_data_t *t, u64 pgoff, char starting_val,
char host_write_val, char write_val)
{
host_do_rmw(t->mem, pgoff, starting_val, host_write_val);
run_guest_do_rmw(t->vcpu, pgoff, host_write_val, write_val);
TEST_ASSERT_EQ(READ_ONCE(t->mem[pgoff * page_size]), write_val);
}
static void test_convert_to_shared(test_data_t *t, u64 pgoff,
char starting_val, char host_write_val,
char write_val)
{
gmem_set_shared(t->gmem_fd, pgoff * page_size, page_size);
test_shared(t, pgoff, starting_val, host_write_val, write_val);
}
GMEM_CONVERSION_TEST_INIT_PRIVATE(init_private)
{
test_private(t, 0, 0, 'A');
test_convert_to_shared(t, 0, 'A', 'B', 'C');
test_convert_to_private(t, 0, 'C', 'E');
}
GMEM_CONVERSION_TEST_INIT_SHARED(init_shared)
{
test_shared(t, 0, 0, 'A', 'B');
test_convert_to_private(t, 0, 'B', 'C');
test_convert_to_shared(t, 0, 'C', 'D', 'E');
}
GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(indexing, 4)
{
int i;
/* Get a char that varies with both i and n. */
#define combine(x, n) (((x) << 4) + (n))
#define i_(n) (combine(i, n))
#define t_(n) (combine(test_page, n))
/*
* Start with the highest index, to catch any errors when, perhaps, the
* first page is returned even for the last index.
*/
for (i = nr_pages - 1; i >= 0; --i)
test_shared(t, i, 0, i_(0), i_(2));
test_convert_to_private(t, test_page, t_(2), t_(3));
for (i = 0; i < nr_pages; ++i) {
if (i == test_page)
test_private(t, test_page, t_(3), t_(4));
else
test_shared(t, i, i_(2), i_(3), i_(4));
}
test_convert_to_shared(t, test_page, t_(4), t_(5), t_(6));
for (i = 0; i < nr_pages; ++i) {
char expected = i == test_page ? t_(6) : i_(4);
test_shared(t, i, expected, i_(7), i_(8));
}
#undef t_
#undef i_
#undef combine
}
/*
* Test that even if there are no folios yet, conversion requests are recorded
* in guest_memfd.
*/
GMEM_CONVERSION_TEST_INIT_SHARED(before_allocation_shared)
{
test_convert_to_private(t, 0, 0, 'A');
}
GMEM_CONVERSION_TEST_INIT_PRIVATE(before_allocation_private)
{
test_convert_to_shared(t, 0, 0, 'A', 'B');
}
/*
* Test that when some of the folios in the conversion range are allocated,
* conversion requests are handled correctly in guest_memfd. Vary the ranges
* allocated before conversion, using test_page, to cover various layouts of
* contiguous allocated and unallocated regions.
*/
GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(unallocated_folios, 8)
{
const int second_page_to_fault = 4;
int i;
/*
* Fault 2 of the pages to test filemap range operations except when
* test_page == second_page_to_fault.
*/
host_do_rmw(t->mem, test_page, 0, 'A');
if (test_page != second_page_to_fault)
host_do_rmw(t->mem, second_page_to_fault, 0, 'A');
gmem_set_private(t->gmem_fd, 0, nr_pages * page_size);
for (i = 0; i < nr_pages; ++i) {
char expected = (i == test_page || i == second_page_to_fault) ? 'A' : 0;
test_private(t, i, expected, 'B');
}
for (i = 0; i < nr_pages; ++i)
test_convert_to_shared(t, i, 'B', 'C', 'D');
}
/* Truncation should not affect shared/private status. */
GMEM_CONVERSION_TEST_INIT_SHARED(truncate)
{
host_do_rmw(t->mem, 0, 0, 'A');
kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size);
host_do_rmw(t->mem, 0, 0, 'A');
test_convert_to_private(t, 0, 'A', 'B');
kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size);
test_private(t, 0, 0, 'A');
}
/* Test that shared/private memory protections work and are seen from any process. */
GMEM_CONVERSION_TEST_INIT_SHARED(forked_accesses)
{
enum test_state {
STATE_INIT,
STATE_CHECK_SHARED,
STATE_DONE_CHECKING_SHARED,
STATE_CHECK_PRIVATE,
STATE_DONE_CHECKING_PRIVATE,
};
struct sync_state {
pthread_mutex_t mutex;
pthread_cond_t cond;
enum test_state step;
} *sync;
pthread_mutexattr_t mattr;
pthread_condattr_t cattr;
pid_t child_pid, parent_pid;
int status;
sync = kvm_mmap(sizeof(*sync), PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1);
pthread_mutexattr_init(&mattr);
pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED);
pthread_mutex_init(&sync->mutex, &mattr);
pthread_mutexattr_destroy(&mattr);
pthread_condattr_init(&cattr);
pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED);
pthread_cond_init(&sync->cond, &cattr);
pthread_condattr_destroy(&cattr);
sync->step = STATE_INIT;
#define TEST_STATE_AWAIT(__state) \
do { \
pthread_mutex_lock(&sync->mutex); \
while (sync->step != (__state)) { \
struct timespec ts, stop; \
int ret; \
\
clock_gettime(CLOCK_REALTIME, &ts); \
stop = timespec_add_ns(ts, 100 * 1000000UL); \
\
ret = pthread_cond_timedwait(&sync->cond, &sync->mutex, &stop); \
if (ret == ETIMEDOUT) { \
bool alive = (child_pid == 0) ? \
(getppid() == parent_pid) : \
(waitpid(child_pid, NULL, WNOHANG) == 0); \
TEST_ASSERT(alive, "Other process exited prematurely"); \
} else { \
TEST_ASSERT(!ret, "pthread_cond_timedwait failed"); \
} \
} \
pthread_mutex_unlock(&sync->mutex); \
} while (0)
#define TEST_STATE_SET(__state) \
do { \
pthread_mutex_lock(&sync->mutex); \
sync->step = (__state); \
pthread_cond_broadcast(&sync->cond); \
pthread_mutex_unlock(&sync->mutex); \
} while (0)
parent_pid = getpid();
child_pid = fork();
TEST_ASSERT(child_pid != -1, "fork failed");
if (child_pid == 0) {
const char inconsequential = 0xdd;
TEST_STATE_AWAIT(STATE_CHECK_SHARED);
/*
* This maps the pages into the child process as well, and tests
* that the conversion process will unmap the guest_memfd memory
* from all processes.
*/
host_do_rmw(t->mem, 0, 0xB, 0xC);
TEST_STATE_SET(STATE_DONE_CHECKING_SHARED);
TEST_STATE_AWAIT(STATE_CHECK_PRIVATE);
TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[0]));
TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[0], inconsequential));
TEST_STATE_SET(STATE_DONE_CHECKING_PRIVATE);
exit(0);
}
test_shared(t, 0, 0, 0xA, 0xB);
TEST_STATE_SET(STATE_CHECK_SHARED);
TEST_STATE_AWAIT(STATE_DONE_CHECKING_SHARED);
test_convert_to_private(t, 0, 0xC, 0xD);
TEST_STATE_SET(STATE_CHECK_PRIVATE);
TEST_STATE_AWAIT(STATE_DONE_CHECKING_PRIVATE);
TEST_ASSERT_EQ(waitpid(child_pid, &status, 0), child_pid);
TEST_ASSERT(WIFEXITED(status) && WEXITSTATUS(status) == 0,
"Child exited with unexpected status");
pthread_mutex_destroy(&sync->mutex);
pthread_cond_destroy(&sync->cond);
kvm_munmap(sync, sizeof(*sync));
#undef TEST_STATE_SET
#undef TEST_STATE_AWAIT
}
static void test_convert_to_private_fails(test_data_t *t, u64 pgoff,
size_t nr_pages,
u64 expected_error_offset)
{
/* +1 to make it anything but expected_error_offset. */
u64 error_offset = expected_error_offset + 1;
u64 offset = pgoff * page_size;
int ret;
do {
ret = __gmem_set_private(t->gmem_fd, offset,
nr_pages * page_size, &error_offset);
} while (ret == -1 && errno == EINTR);
TEST_ASSERT(ret == -1 && errno == EAGAIN,
"Wanted EAGAIN on page %lu, got %d (ret = %d)", pgoff,
errno, ret);
TEST_ASSERT_EQ(error_offset, expected_error_offset);
}
GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(elevated_refcount, 4)
{
int i;
pin_pages(t->mem + test_page * page_size, page_size);
for (i = 0; i < nr_pages; i++)
test_shared(t, i, 0, 'A', 'B');
/*
* Converting in bulk should fail as long any page in the range has
* unexpected refcounts.
*/
test_convert_to_private_fails(t, 0, nr_pages, test_page * page_size);
for (i = 0; i < nr_pages; i++) {
/*
* Converting page-wise should also fail as long any page in the
* range has unexpected refcounts.
*/
if (i == test_page)
test_convert_to_private_fails(t, i, 1, test_page * page_size);
else
test_convert_to_private(t, i, 'B', 'C');
}
unpin_pages();
gmem_set_private(t->gmem_fd, 0, nr_pages * page_size);
for (i = 0; i < nr_pages; i++) {
char expected = i == test_page ? 'B' : 'C';
test_private(t, i, expected, 'D');
}
}
int main(int argc, char *argv[])
{
TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM));
TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) &
KVM_MEMORY_ATTRIBUTE_PRIVATE);
page_size = getpagesize();
return test_harness_run(argc, argv);
}

66
tools/testing/selftests/kvm/x86/private_mem_conversions_test.c
View File

@ -202,15 +202,18 @@ static void guest_test_explicit_conversion(u64 base_gpa, bool do_fallocate)
guest_sync_shared(gpa, size, p3, p4);
memcmp_g(gpa, p4, size);
/* Reset the shared memory back to the initial pattern. */
memset((void *)gpa, init_p, size);
/*
* Free (via PUNCH_HOLE) *all* private memory so that the next
* iteration starts from a clean slate, e.g. with respect to
* whether or not there are pages/folios in guest_mem.
*/
guest_map_shared(base_gpa, PER_CPU_DATA_SIZE, true);
/*
* Hole-punching above zeroed private memory. Reset shared
* memory in preparation for the next GUEST_STAGE.
*/
memset((void *)base_gpa, init_p, PER_CPU_DATA_SIZE);
}
}
@ -303,9 +306,12 @@ static void handle_exit_hypercall(struct kvm_vcpu *vcpu)
if (do_fallocate)
vm_guest_mem_fallocate(vm, gpa, size, map_shared);
if (set_attributes)
vm_set_memory_attributes(vm, gpa, size,
map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE);
if (set_attributes) {
u64 attrs = map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE;
vm_mem_set_memory_attributes(vm, gpa, size, attrs);
}
run->hypercall.ret = 0;
}
@ -349,8 +355,21 @@ static void *__test_mem_conversions(void *__vcpu)
size_t nr_bytes = min_t(size_t, vm->page_size, size - i);
u8 *hva = addr_gpa2hva(vm, gpa + i);
/* In all cases, the host should observe the shared data. */
memcmp_h(hva, gpa + i, uc.args[3], nr_bytes);
if (kvm_has_gmem_attributes &&
uc.args[0] == SYNC_PRIVATE) {
TEST_EXPECT_SIGBUS(READ_ONCE(*hva));
} else {
/*
* If not testing in-place conversion
* (dual backing), the host should
* always observe shared data, since the
* shared memory is a separate page.
*
* For SYNC_SHARED, test that the host
* can see shared memory.
*/
memcmp_h(hva, gpa + i, uc.args[3], nr_bytes);
}
/* For shared, write the new pattern to guest memory. */
if (uc.args[0] == SYNC_SHARED)
@ -366,20 +385,29 @@ static void *__test_mem_conversions(void *__vcpu)
}
}
/* Align each vCPU's chunk of memory naturally to the size of the backing store. */
static size_t compute_per_cpu_size(enum vm_mem_backing_src_type src_type)
{
size_t alignment;
if (kvm_has_gmem_attributes)
alignment = getpagesize();
else
alignment = get_backing_src_pagesz(src_type);
return align_up(PER_CPU_DATA_SIZE, max_t(size_t, SZ_2M, alignment));
}
static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_vcpus,
u32 nr_memslots)
{
/*
* Allocate enough memory so that each vCPU's chunk of memory can be
* naturally aligned with respect to the size of the backing store.
*/
const size_t alignment = max_t(size_t, SZ_2M, get_backing_src_pagesz(src_type));
const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment);
const size_t per_cpu_size = compute_per_cpu_size(src_type);
const size_t memfd_size = per_cpu_size * nr_vcpus;
const size_t slot_size = memfd_size / nr_memslots;
struct kvm_vcpu *vcpus[KVM_MAX_VCPUS];
pthread_t threads[KVM_MAX_VCPUS];
struct kvm_vm *vm;
u64 gmem_flags;
int memfd, i;
const struct vm_shape shape = {
@ -394,12 +422,16 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_v
vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE));
memfd = vm_create_guest_memfd(vm, memfd_size, 0);
gmem_flags = 0;
if (kvm_has_gmem_attributes)
gmem_flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED;
memfd = vm_create_guest_memfd(vm, memfd_size, gmem_flags);
for (i = 0; i < nr_memslots; i++)
vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i,
BASE_DATA_SLOT + i, slot_size / vm->page_size,
KVM_MEM_GUEST_MEMFD, memfd, slot_size * i);
KVM_MEM_GUEST_MEMFD, memfd, slot_size * i, gmem_flags);
for (i = 0; i < nr_vcpus; i++) {
gpa_t gpa = BASE_DATA_GPA + i * per_cpu_size;
@ -459,6 +491,8 @@ int main(int argc, char *argv[])
while ((opt = getopt(argc, argv, "hm:s:n:")) != -1) {
switch (opt) {
case 's':
TEST_ASSERT(!kvm_has_gmem_attributes,
"src_type is only configurable when testing without in-place conversion");
src_type = parse_backing_src_type(optarg);
break;
case 'n':

36
tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c
View File

@ -62,8 +62,9 @@ static void test_private_access_memslot_deleted(void)
virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES);
/* Request to access page privately */
vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE);
/* Request to access page privately. */
if (!kvm_has_gmem_attributes)
vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE);
kvm_pthread_create(&vm_thread, NULL,
(pthread_fn_t)run_vcpu_get_exit_reason, (void *)vcpu);
@ -73,10 +74,26 @@ static void test_private_access_memslot_deleted(void)
kvm_pthread_join(vm_thread, &thread_return);
exit_reason = (u32)(u64)thread_return;
TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT);
TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE);
TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA);
TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE);
/*
* If attributes are tracked per-gmem, deleting the memslot that points
* at the gmem instance effectively makes the memory shared, and so the
* read should trigger emulated MMIO.
*
* If attributes are tracked per-VM, deleting the memslot shouldn't
* affect the private attribute, and so KVM should generate a memory
* fault exit (emulated MMIO on private GPAs is disallowed).
*/
if (kvm_has_gmem_attributes) {
TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MMIO);
TEST_ASSERT_EQ(vcpu->run->mmio.phys_addr, EXITS_TEST_GPA);
TEST_ASSERT_EQ(vcpu->run->mmio.len, sizeof(u64));
TEST_ASSERT_EQ(vcpu->run->mmio.is_write, false);
} else {
TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT);
TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE);
TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA);
TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE);
}
kvm_vm_free(vm);
}
@ -87,6 +104,13 @@ static void test_private_access_memslot_not_private(void)
struct kvm_vcpu *vcpu;
u32 exit_reason;
/*
* Accessing non-private memory as private with a software-protected VM
* isn't possible when doing in-place conversions.
*/
if (kvm_has_gmem_attributes)
return;
vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu,
guest_repeatedly_read);

3
virt/kvm/Kconfig
View File

@ -82,9 +82,6 @@ config KVM_ELIDE_TLB_FLUSH_IF_YOUNG
config KVM_MMU_LOCKLESS_AGING
bool
config KVM_GENERIC_MEMORY_ATTRIBUTES
bool
config KVM_GUEST_MEMFD
select XARRAY_MULTI
bool

649
virt/kvm/guest_memfd.c
View File

@ -4,9 +4,11 @@
#include <linux/falloc.h>
#include <linux/fs.h>
#include <linux/kvm_host.h>
#include <linux/maple_tree.h>
#include <linux/mempolicy.h>
#include <linux/pseudo_fs.h>
#include <linux/pagemap.h>
#include <linux/swap.h>
#include "kvm_mm.h"
#include "guest_memfd.h"
@ -34,6 +36,13 @@ struct gmem_inode {
struct list_head gmem_file_list;
u64 flags;
/*
* Every index in this inode, whether memory is populated or
* not, is tracked in attributes. The entire range of indices,
* corresponding to the size of this inode, is represented in
* this maple tree.
*/
struct maple_tree attributes;
};
static __always_inline struct gmem_inode *GMEM_I(struct inode *inode)
@ -61,9 +70,28 @@ static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn)
return gfn - slot->base_gfn + slot->gmem.pgoff;
}
static u64 kvm_gmem_get_default_attributes(struct inode *inode)
{
bool init_shared = GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED;
return init_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE;
}
static u64 kvm_gmem_get_attributes(struct inode *inode, void *entry)
{
if (WARN_ON_ONCE(!entry))
return kvm_gmem_get_default_attributes(inode);
return xa_to_value(entry);
}
static bool kvm_gmem_is_private_mem(struct inode *inode, pgoff_t index)
{
return !(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED);
struct maple_tree *mt = &GMEM_I(inode)->attributes;
void *entry = mtree_load(mt, index);
return kvm_gmem_get_attributes(inode, entry) &
KVM_MEMORY_ATTRIBUTE_PRIVATE;
}
static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index)
@ -71,6 +99,24 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index)
return !kvm_gmem_is_private_mem(inode, index);
}
static bool kvm_gmem_range_has_attributes(struct inode *inode,
pgoff_t index, size_t nr_pages,
u64 attributes)
{
struct maple_tree *mt = &GMEM_I(inode)->attributes;
pgoff_t end = index + nr_pages - 1;
void *entry;
lockdep_assert(mt_lock_is_held(mt));
mt_for_each(mt, entry, index, end) {
if (kvm_gmem_get_attributes(inode, entry) != attributes)
return false;
}
return true;
}
/*
* Returns a locked folio on success. The caller is responsible for
* setting the up-to-date flag before the memory is mapped into the guest.
@ -112,8 +158,11 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index)
return folio;
}
static enum kvm_gfn_range_filter kvm_gmem_get_invalidate_filter(struct inode *inode)
static enum kvm_gfn_range_filter kvm_gmem_get_all_gfns_filter(struct inode *inode)
{
if (gmem_in_place_conversion)
return KVM_FILTER_SHARED | KVM_FILTER_PRIVATE;
if (GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED)
return KVM_FILTER_SHARED;
@ -162,15 +211,13 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start,
}
static void kvm_gmem_invalidate_start(struct inode *inode, pgoff_t start,
pgoff_t end)
pgoff_t end,
enum kvm_gfn_range_filter filter)
{
enum kvm_gfn_range_filter attr_filter;
struct gmem_file *f;
attr_filter = kvm_gmem_get_invalidate_filter(inode);
kvm_gmem_for_each_file(f, inode)
__kvm_gmem_invalidate_start(f, start, end, attr_filter);
__kvm_gmem_invalidate_start(f, start, end, filter);
}
static void __kvm_gmem_invalidate_end(struct gmem_file *f, pgoff_t start,
@ -196,6 +243,7 @@ static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t start,
static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)
{
enum kvm_gfn_range_filter filter = kvm_gmem_get_all_gfns_filter(inode);
pgoff_t start = offset >> PAGE_SHIFT;
pgoff_t end = (offset + len) >> PAGE_SHIFT;
@ -205,7 +253,7 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)
*/
filemap_invalidate_lock(inode->i_mapping);
kvm_gmem_invalidate_start(inode, start, end);
kvm_gmem_invalidate_start(inode, start, end, filter);
truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1);
@ -300,17 +348,56 @@ static int kvm_gmem_release(struct inode *inode, struct file *file)
* dereferencing the slot for existing bindings needs to be protected
* against memslot updates, specifically so that unbind doesn't race
* and free the memslot (kvm_gmem_get_file() will return NULL).
*
* Since .release is called only when the reference count is zero,
* after which file_ref_get() and get_file_active() fail,
* kvm_gmem_get_pfn() cannot be using the file concurrently.
* file_ref_put() provides a full barrier, and get_file_active() the
* matching acquire barrier.
*/
mutex_lock(&kvm->slots_lock);
filemap_invalidate_lock(inode->i_mapping);
/*
* Note! synchronize_srcu() is _not_ needed after nullifying memslot
* bindings as slot->gmem.file cannot be set back to a non-null value
* without the memslot first being deleted. I.e. this relies on the
* synchronize_srcu_expedited() in kvm_swap_active_memslots() to ensure
* kvm_gmem_get_pfn() (which runs with kvm->srcu held for read) can't
* grab a reference to slot->gmem.file even if the struct file object
* is reallocated (for use as a different file).
*
* file_ref_put() provides a full barrier, and __get_file_rcu() the
* matching acquire barrier, to ensure that kvm_gmem_get_file() (via
* __get_file_rcu()) sees refcount==0 or fails the "file reloaded"
* check (file != NULL due to nullifying the file pointer here).
*
* Unlike most other users of get_file_rcu(), where callers don't care
* if they race with a write, only that they have a reference to _a_
* live file, kvm_gmem_get_pfn() needs to get the exact file that is
* associated with the memslot. Without the aforementioned SRCU
* synchronization, the following could happen:
*
* CPU0 CPU1
* kvm_gmem_get_pfn()
* slot->gmem.file == A
* kvm_gmem_release())
* slot->gmem.file = NULL
*
* kvm_set_memory_region()
* slot deleted
*
* kvm_set_memory_region()
* slot created
* slot->gmem.file = B (alloc the same object)
*
* get_file_active()
* file = B
* file_reloaded = B
*
* <KVM does weird things with an old memslot + new file>
*
* Obviously KVM would be broken in many places if the synchronization
* were omitted, but it's important to note that get_file_active() does
* NOT guarantee a reference to the "original" file was obtained, only
* that it grabbed a reference for the returned file, i.e. didn't grab
* a reference for file A, but then returned a pointer to file B.
*/
xa_for_each(&f->bindings, index, slot)
WRITE_ONCE(slot->gmem.file, NULL);
@ -320,7 +407,7 @@ static int kvm_gmem_release(struct inode *inode, struct file *file)
* memory, as its lifetime is associated with the inode, not the file.
*/
__kvm_gmem_invalidate_start(f, 0, -1ul,
kvm_gmem_get_invalidate_filter(inode));
kvm_gmem_get_all_gfns_filter(inode));
__kvm_gmem_invalidate_end(f, 0, -1ul);
list_del(&f->entry);
@ -364,10 +451,13 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf)
if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode))
return VM_FAULT_SIGBUS;
if (!kvm_gmem_is_shared_mem(inode, vmf->pgoff))
return VM_FAULT_SIGBUS;
filemap_invalidate_lock_shared(inode->i_mapping);
if (kvm_gmem_is_shared_mem(inode, vmf->pgoff))
folio = kvm_gmem_get_folio(inode, vmf->pgoff);
else
folio = ERR_PTR(-EACCES);
filemap_invalidate_unlock_shared(inode->i_mapping);
folio = kvm_gmem_get_folio(inode, vmf->pgoff);
if (IS_ERR(folio)) {
if (PTR_ERR(folio) == -EAGAIN)
return VM_FAULT_RETRY;
@ -448,11 +538,280 @@ static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma)
return 0;
}
bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn)
{
struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
struct inode *inode;
if (!slot)
return 0;
CLASS(gmem_get_file, file)(slot);
if (!file)
return 0;
inode = file_inode(file);
/*
* Rely on the maple tree's internal RCU lock to ensure a stable result.
* This result can become stale as soon as the lock is dropped, so the
* caller _must_ protect consumption of private vs. shared either by
* holding guest_memfd's invalidate lock for the entire duration, or by
* checking mmu_invalidate_retry_gfn() under mmu_lock to serialize
* against concurrent attribute updates.
*/
return kvm_gmem_is_private_mem(inode, kvm_gmem_get_index(slot, gfn));
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_is_private_gfn);
/*
* Preallocate memory for attributes to be stored on a maple tree, pointed to
* by mas. Adjacent ranges with attributes identical to the new attributes
* will be merged. Also sets mas's bounds up for storing attributes.
*
* This maintains the invariant that ranges with the same attributes will
* always be merged.
*/
static int kvm_gmem_mas_preallocate(struct ma_state *mas, u64 attributes,
pgoff_t start, size_t nr_pages)
{
pgoff_t end = start + nr_pages;
pgoff_t last = end - 1;
void *entry;
/* Try extending range. entry is NULL on overflow/wrap-around. */
mas_set(mas, end);
entry = mas_find(mas, end);
if (entry && xa_to_value(entry) == attributes)
last = mas->last;
if (start > 0) {
mas_set(mas, start - 1);
entry = mas_find(mas, start - 1);
if (entry && xa_to_value(entry) == attributes)
start = mas->index;
}
mas_set_range(mas, start, last);
return mas_preallocate(mas, xa_mk_value(attributes), GFP_KERNEL);
}
static bool __folio_has_outstanding_references(struct folio *folio,
enum lru_cache_drained *drained)
{
if (folio_maybe_dma_pinned(folio) || folio_mapped(folio))
return true;
/* 1 reference held by filemap_get_folios() in the folio batch. */
lru_cache_drain_for_folio(folio, 1, drained);
/*
* Outstanding references are anything other than those from the page
* cache, plus 1 temporary reference held by filemap_get_folios() in the
* folio batch.
*/
return folio_ref_count(folio) != folio_nr_pages(folio) + 1;
}
static bool kvm_gmem_has_outstanding_references(struct inode *inode,
pgoff_t start, size_t nr_pages,
pgoff_t *err_index)
{
enum lru_cache_drained drained = LRU_CACHE_NOT_DRAINED;
struct address_space *mapping = inode->i_mapping;
pgoff_t last = start + nr_pages - 1;
struct folio_batch fbatch;
pgoff_t next;
int i;
folio_batch_init(&fbatch);
next = start;
while (filemap_get_folios(mapping, &next, last, &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); ++i) {
struct folio *folio = fbatch.folios[i];
if (__folio_has_outstanding_references(folio, &drained)) {
*err_index = max(start, folio->index);
folio_batch_release(&fbatch);
return true;
}
}
folio_batch_release(&fbatch);
cond_resched();
}
return false;
}
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end)
{
struct folio_batch fbatch;
pgoff_t next = start;
int i;
folio_batch_init(&fbatch);
while (filemap_get_folios(inode->i_mapping, &next, end - 1, &fbatch)) {
for (i = 0; i < folio_batch_count(&fbatch); ++i) {
struct folio *folio = fbatch.folios[i];
pgoff_t start_index, end_index;
kvm_pfn_t start_pfn;
kvm_pfn_t nr_pages;
start_index = max(start, folio->index);
end_index = min(end, folio_next_index(folio));
/*
* end_index is either in folio or points to
* the first page of the next folio. Hence,
* all pages in range [start_index, end_index)
* are contiguous.
*/
start_pfn = folio_file_pfn(folio, start_index);
nr_pages = end_index - start_index;
kvm_arch_gmem_make_shared(start_pfn, nr_pages);
}
folio_batch_release(&fbatch);
cond_resched();
}
}
#else
static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) {}
#endif
static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start,
size_t nr_pages, uint64_t attrs,
pgoff_t *err_index)
{
bool to_private = attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE;
struct address_space *mapping = inode->i_mapping;
struct gmem_inode *gi = GMEM_I(inode);
enum kvm_gfn_range_filter filter;
pgoff_t end = start + nr_pages;
struct maple_tree *mt;
struct ma_state mas;
int r = 0;
mt = &gi->attributes;
filemap_invalidate_lock(mapping);
if (kvm_gmem_range_has_attributes(inode, start, nr_pages, attrs))
goto out;
mas_init(&mas, mt, start);
r = kvm_gmem_mas_preallocate(&mas, attrs, start, nr_pages);
if (r) {
*err_index = start;
goto out;
}
if (to_private) {
/*
* Forcefully unmap the pages from all userspace page tables,
* and then verify there are no outstanding references, e.g.
* acquired via GUP or similar. Tell userspace to try again if
* there are outstanding references and hope that whatever has
* pinned the page will put its reference "soon".
*/
unmap_mapping_pages(mapping, start, nr_pages, false);
if (kvm_gmem_has_outstanding_references(inode, start, nr_pages,
err_index)) {
mas_destroy(&mas);
r = -EAGAIN;
goto out;
}
}
/*
* From this point on guest_memfd has performed necessary
* checks and can proceed to do guest-breaking changes.
*/
filter = to_private ? KVM_FILTER_SHARED : KVM_FILTER_PRIVATE;
kvm_gmem_invalidate_start(inode, start, end, filter);
if (!to_private && kvm_arch_has_gmem_convert())
kvm_gmem_make_shared(inode, start, end);
mas_store_prealloc(&mas, xa_mk_value(attrs));
kvm_gmem_invalidate_end(inode, start, end);
out:
filemap_invalidate_unlock(mapping);
return r;
}
static long kvm_gmem_set_attributes(struct file *file, void __user *argp)
{
struct gmem_file *f = file->private_data;
struct inode *inode = file_inode(file);
struct kvm_memory_attributes2 attrs;
pgoff_t err_index;
size_t nr_pages;
pgoff_t index;
int i, r;
if (copy_from_user(&attrs, argp, sizeof(attrs)))
return -EFAULT;
if (attrs.flags)
return -EINVAL;
for (i = 0; i < ARRAY_SIZE(attrs.reserved); i++) {
if (attrs.reserved[i])
return -EINVAL;
}
if (!kvm_arch_has_private_mem(f->kvm))
return -EINVAL;
if (attrs.attributes & ~KVM_MEMORY_ATTRIBUTE_PRIVATE)
return -EINVAL;
if (attrs.size == 0 || attrs.offset + attrs.size < attrs.offset)
return -EINVAL;
if (!PAGE_ALIGNED(attrs.offset) || !PAGE_ALIGNED(attrs.size))
return -EINVAL;
if (attrs.offset >= i_size_read(inode) ||
attrs.offset + attrs.size > i_size_read(inode))
return -EINVAL;
nr_pages = attrs.size >> PAGE_SHIFT;
index = attrs.offset >> PAGE_SHIFT;
r = __kvm_gmem_set_attributes(inode, index, nr_pages, attrs.attributes,
&err_index);
if (r) {
attrs.error_offset = ((uint64_t)err_index) << PAGE_SHIFT;
if (copy_to_user(argp, &attrs, sizeof(attrs)))
return -EFAULT;
}
return r;
}
static long kvm_gmem_ioctl(struct file *file, unsigned int ioctl,
unsigned long arg)
{
switch (ioctl) {
case KVM_SET_MEMORY_ATTRIBUTES2:
if (!gmem_in_place_conversion)
return -ENOTTY;
return kvm_gmem_set_attributes(file, (void __user *)arg);
default:
return -ENOTTY;
}
}
static struct file_operations kvm_gmem_fops = {
.mmap = kvm_gmem_mmap,
.open = generic_file_open,
.release = kvm_gmem_release,
.fallocate = kvm_gmem_fallocate,
.unlocked_ioctl = kvm_gmem_ioctl,
};
static int kvm_gmem_migrate_folio(struct address_space *mapping,
@ -465,14 +824,27 @@ static int kvm_gmem_migrate_folio(struct address_space *mapping,
static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *folio)
{
struct inode *inode = mapping->host;
enum kvm_gfn_range_filter filter;
pgoff_t start, end;
filemap_invalidate_lock_shared(mapping);
/*
* memory_failure() invokes ->error_remove_folio() while holding the
* poisoned folio lock. Do not block on mapping->invalidate_lock, as
* a writer that already holds the invalidate lock can be waiting for
* the same folio, e.g. truncation from MADV_REMOVE/FALLOC_FL_PUNCH_HOLE.
*
* If the invalidate lock is contended, fail the recovery instead of
* risking a deadlock in the memory-failure path.
*/
if (!filemap_invalidate_trylock_shared(mapping))
return -EBUSY;
start = folio->index;
end = start + folio_nr_pages(folio);
kvm_gmem_invalidate_start(mapping->host, start, end);
filter = kvm_gmem_get_all_gfns_filter(inode);
kvm_gmem_invalidate_start(inode, start, end, filter);
/*
* Do not truncate the range, what action is taken in response to the
@ -520,6 +892,51 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm)
return true;
}
static int kvm_gmem_init_inode(struct inode *inode, loff_t size, u64 flags)
{
struct gmem_inode *gi = GMEM_I(inode);
MA_STATE(mas, &gi->attributes, 0, (size >> PAGE_SHIFT) - 1);
u64 attrs;
int r;
inode->i_op = &kvm_gmem_iops;
inode->i_mapping->a_ops = &kvm_gmem_aops;
inode->i_mode |= S_IFREG;
inode->i_size = size;
mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);
/*
* guest_memfd memory is neither migratable nor swappable: set
* inaccessible to gate off both.
*/
mapping_set_inaccessible(inode->i_mapping);
WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));
gi->flags = flags;
mt_set_external_lock(&gi->attributes,
&inode->i_mapping->invalidate_lock);
/*
* Store default attributes for the entire gmem instance. Ensuring every
* index is represented in the maple tree at all times simplifies the
* conversion and merging logic.
*/
attrs = kvm_gmem_get_default_attributes(inode);
/*
* Acquire the invalidation lock purely to make lockdep happy. The
* maple tree library expects all stores to be protected via the lock,
* and the library can't know when the tree is reachable only by the
* caller, as is the case here.
*/
filemap_invalidate_lock(inode->i_mapping);
r = mas_store_gfp(&mas, xa_mk_value(attrs), GFP_KERNEL);
filemap_invalidate_unlock(inode->i_mapping);
return r;
}
static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
{
static const char *name = "[kvm-gmem]";
@ -550,16 +967,9 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)
goto err_fops;
}
inode->i_op = &kvm_gmem_iops;
inode->i_mapping->a_ops = &kvm_gmem_aops;
inode->i_mode |= S_IFREG;
inode->i_size = size;
mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);
mapping_set_inaccessible(inode->i_mapping);
/* Unmovable mappings are supposed to be marked unevictable as well. */
WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));
GMEM_I(inode)->flags = flags;
err = kvm_gmem_init_inode(inode, size, flags);
if (err)
goto err_inode;
file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, &kvm_gmem_fops);
if (IS_ERR(file)) {
@ -603,19 +1013,21 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)
return __kvm_gmem_create(kvm, size, flags);
}
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
unsigned int fd, uoff_t offset)
int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot,
unsigned int fd, uoff_t offset)
{
uoff_t size = slot->npages << PAGE_SHIFT;
unsigned long start, end;
struct gmem_file *f;
struct inode *inode;
struct file *file;
int r = -EINVAL;
BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset));
BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff));
if (WARN_ON_ONCE(slot->flags & KVM_MEMSLOT_GMEM_ONLY))
return -EINVAL;
file = fget(fd);
if (!file)
return -EBADF;
@ -632,84 +1044,93 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
if (!PAGE_ALIGNED(offset) || offset + size > i_size_read(inode))
goto err;
filemap_invalidate_lock(inode->i_mapping);
start = offset >> PAGE_SHIFT;
end = start + slot->npages;
if (!xa_empty(&f->bindings) &&
xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) {
r = -EEXIST;
filemap_invalidate_unlock(inode->i_mapping);
goto err;
}
/*
* memslots of flag KVM_MEM_GUEST_MEMFD are immutable to change, so
* kvm_gmem_bind() must occur on a new memslot. Because the memslot
* is not visible yet, kvm_gmem_get_pfn() is guaranteed to see the file.
*/
WRITE_ONCE(slot->gmem.file, file);
slot->gmem.pgoff = start;
if (kvm_gmem_supports_mmap(inode))
slot->gmem.file = file;
slot->gmem.pgoff = offset >> PAGE_SHIFT;
if (gmem_in_place_conversion || kvm_gmem_supports_mmap(inode))
slot->flags |= KVM_MEMSLOT_GMEM_ONLY;
xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL);
filemap_invalidate_unlock(inode->i_mapping);
/*
* Drop the reference to the file, even on success. The file pins KVM,
* not the other way 'round. Active bindings are invalidated if the
* file is closed before memslots are destroyed.
* Gift the caller a reference to the file. The reference will be
* dropped after bindings are established, or if installing the new
* memslot ultimately fails.
*/
r = 0;
return 0;
err:
fput(file);
return r;
return -EINVAL;
}
static void __kvm_gmem_unbind(struct kvm_memory_slot *slot, struct gmem_file *f)
int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot)
{
unsigned long start = slot->gmem.pgoff;
unsigned long end = start + slot->npages;
struct gmem_file *f = slot->gmem.file->private_data;
struct inode *inode = file_inode(slot->gmem.file);
unsigned long start, end;
int r;
xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL);
if (WARN_ON_ONCE(slot->gmem.file->f_op != &kvm_gmem_fops))
return -EIO;
/*
* synchronize_srcu(&kvm->srcu) ensured that kvm_gmem_get_pfn()
* cannot see this memslot.
*/
WRITE_ONCE(slot->gmem.file, NULL);
filemap_invalidate_lock(inode->i_mapping);
start = slot->gmem.pgoff;
end = start + slot->npages;
if (!xa_empty(&f->bindings) &&
xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) {
filemap_invalidate_unlock(inode->i_mapping);
return -EEXIST;
}
r = xa_err(xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL));
if (r)
xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL);
filemap_invalidate_unlock(inode->i_mapping);
return r;
}
void kvm_gmem_unbind(struct kvm_memory_slot *slot)
{
struct file *file = slot->gmem.file;
unsigned long start = slot->gmem.pgoff;
unsigned long end = start + slot->npages;
struct gmem_file *f;
/*
* Nothing to do if the underlying file was _already_ closed, as
* kvm_gmem_release() invalidates and nullifies all bindings.
*/
if (!slot->gmem.file)
if (!file)
return;
CLASS(gmem_get_file, file)(slot);
/*
* However, if the file is _being_ closed, then the bindings need to be
* removed as kvm_gmem_release() might not run until after the memslot
* is freed. Note, modifying the bindings is safe even though the file
* is dying as kvm_gmem_release() nullifies slot->gmem.file under
* slots_lock, and only puts its reference to KVM after destroying all
* bindings. I.e. reaching this point means kvm_gmem_release() hasn't
* yet destroyed the bindings or freed the gmem_file, and can't do so
* until the caller drops slots_lock.
* is freed. Modifying the bindings is safe even if the file is dying
* as kvm_gmem_release() nullifies slot->gmem.file under slots_lock,
* and only puts its reference to KVM after destroying all bindings.
* I.e. reaching this point means kvm_gmem_release() hasn't destroyed
* the bindings or freed the gmem_file and can't do so until the caller
* drops slots_lock, so there's no need to verify the file is live.
*/
if (!file) {
__kvm_gmem_unbind(slot, slot->gmem.file->private_data);
return;
}
f = file->private_data;
filemap_invalidate_lock(file->f_mapping);
__kvm_gmem_unbind(slot, file->private_data);
xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL);
/*
* Note, the caller is responsible for ensuring the slot is unreachable
* before unbinding, e.g. by synchronizing SRCU after deleting the slot,
* to guarantee kvm_gmem_get_pfn() can't see the slot+file.
*/
WRITE_ONCE(slot->gmem.file, NULL);
filemap_invalidate_unlock(file->f_mapping);
}
@ -743,6 +1164,11 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file,
return ERR_PTR(-EHWPOISON);
}
if (!folio_test_uptodate(folio)) {
clear_highpage(folio_page(folio, 0));
folio_mark_uptodate(folio);
}
*pfn = folio_file_pfn(folio, index);
if (max_order)
*max_order = 0;
@ -751,8 +1177,7 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file,
}
int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
gfn_t gfn, kvm_pfn_t *pfn, struct page **page,
int *max_order)
gfn_t gfn, kvm_pfn_t *pfn, int *max_order)
{
pgoff_t index = kvm_gmem_get_index(slot, gfn);
struct folio *folio;
@ -764,28 +1189,24 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,
if (!file)
return -EFAULT;
folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order);
if (IS_ERR(folio))
return PTR_ERR(folio);
filemap_invalidate_lock_shared(file_inode(file)->i_mapping);
if (!folio_test_uptodate(folio)) {
clear_highpage(folio_page(folio, 0));
folio_mark_uptodate(folio);
folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order);
if (IS_ERR(folio)) {
r = PTR_ERR(folio);
goto out;
}
#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT
if (kvm_gmem_is_private_mem(file_inode(file), index))
if (kvm_arch_has_gmem_convert() &&
kvm_gmem_is_private_mem(file_inode(file), index))
r = kvm_arch_gmem_make_private(kvm, gfn, *pfn,
(kvm_pfn_t)1 << *max_order);
#endif
folio_unlock(folio);
folio_put(folio);
if (!r)
*page = folio_file_page(folio, index);
else
folio_put(folio);
out:
filemap_invalidate_unlock_shared(file_inode(file)->i_mapping);
return r;
}
EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn);
@ -811,16 +1232,12 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot,
folio_unlock(folio);
if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1,
KVM_MEMORY_ATTRIBUTE_PRIVATE,
KVM_MEMORY_ATTRIBUTE_PRIVATE)) {
if (!kvm_is_private_gfn(kvm, gfn)) {
ret = -EINVAL;
goto out_put_folio;
}
ret = post_populate(kvm, gfn, pfn, src_page, opaque);
if (!ret)
folio_mark_uptodate(folio);
out_put_folio:
folio_put(folio);
@ -917,6 +1334,15 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb)
mpol_shared_policy_init(&gi->policy, NULL);
/*
* Memory attributes are protected by the filemap invalidation lock, but
* the lock structure isn't available at this time. Immediately mark
* maple tree as using external locking so that accessing the tree
* before it's fully initialized results in NULL pointer dereferences
* and not more subtle bugs.
*/
mt_init_flags(&gi->attributes, MT_FLAGS_LOCK_EXTERN | MT_FLAGS_USE_RCU);
gi->flags = 0;
INIT_LIST_HEAD(&gi->gmem_file_list);
return &gi->vfs_inode;
@ -924,7 +1350,26 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb)
static void kvm_gmem_destroy_inode(struct inode *inode)
{
mpol_free_shared_policy(&GMEM_I(inode)->policy);
struct gmem_inode *gi = GMEM_I(inode);
mpol_free_shared_policy(&gi->policy);
/*
* Note! Checking for an empty tree is functionally necessary
* to avoid explosions if the tree hasn't been fully
* initialized, i.e. if the inode is being destroyed before
* guest_memfd can set the external lock, lockdep would find
* that the tree's internal ma_lock was not held.
*/
if (!mtree_empty(&gi->attributes)) {
/*
* Acquire the invalidation lock purely to make lockdep happy,
* the inode is unreachable at this point.
*/
filemap_invalidate_lock(inode->i_mapping);
__mt_destroy(&gi->attributes);
filemap_invalidate_unlock(inode->i_mapping);
}
}
static void kvm_gmem_free_inode(struct inode *inode)

19
virt/kvm/guest_memfd.h
View File

@ -8,8 +8,9 @@
int kvm_gmem_init(struct module *module);
void kvm_gmem_exit(void);
int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args);
int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,
unsigned int fd, uoff_t offset);
int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot,
unsigned int fd, uoff_t offset);
int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot);
void kvm_gmem_unbind(struct kvm_memory_slot *slot);
#else
static inline int kvm_gmem_init(struct module *module)
@ -17,9 +18,17 @@ static inline int kvm_gmem_init(struct module *module)
return 0;
}
static inline void kvm_gmem_exit(void) {};
static inline int kvm_gmem_bind(struct kvm *kvm,
struct kvm_memory_slot *slot,
unsigned int fd, uoff_t offset)
static inline int kvm_gmem_prepare_memory_region(struct kvm *kvm,
struct kvm_memory_slot *slot,
unsigned int fd, uoff_t offset)
{
WARN_ON_ONCE(1);
return -EIO;
}
static inline int kvm_gmem_commit_memory_region(struct kvm *kvm,
struct kvm_memory_slot *slot)
{
WARN_ON_ONCE(1);
return -EIO;

157
virt/kvm/kvm_main.c
View File

@ -104,6 +104,14 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink);
static bool __ro_after_init allow_unsafe_mappings;
module_param(allow_unsafe_mappings, bool, 0444);
#ifdef kvm_arch_has_private_mem
bool __ro_after_init gmem_in_place_conversion = !IS_ENABLED(CONFIG_KVM_VM_MEMORY_ATTRIBUTES);
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
module_param(gmem_in_place_conversion, bool, 0444);
#endif
EXPORT_SYMBOL_FOR_KVM_INTERNAL(gmem_in_place_conversion);
#endif
/*
* Ordering of locks:
*
@ -1128,7 +1136,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname)
spin_lock_init(&kvm->mn_invalidate_lock);
rcuwait_init(&kvm->mn_memslots_update_rcuwait);
xa_init(&kvm->vcpu_array);
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
xa_init_flags(&kvm->mem_attr_array, XA_FLAGS_ACCOUNT);
#endif
@ -1313,7 +1321,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
cleanup_srcu_struct(&kvm->irq_srcu);
srcu_barrier(&kvm->srcu);
cleanup_srcu_struct(&kvm->srcu);
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
xa_destroy(&kvm->mem_attr_array);
#endif
kvm_arch_free_vm(kvm);
@ -1762,10 +1770,10 @@ static void kvm_commit_memory_region(struct kvm *kvm,
kvm_destroy_dirty_bitmap(old);
/*
* Unbind the guest_memfd instance as needed; the @new slot has
* already created its own binding. TODO: Drop the WARN when
* dirty logging guest_memfd memslots is supported. Until then,
* flags-only changes on guest_memfd slots should be impossible.
* TODO: Drop the WARN and do the unbind() call only for MOVE
* when dirty logging guest_memfd memslots is supported. Until
* then, flags-only changes on guest_memfd slots should also be
* impossible; unbind the old memslot for defense-in-depth.
*/
if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD))
kvm_gmem_unbind(old);
@ -1951,20 +1959,16 @@ static int kvm_set_memslot(struct kvm *kvm,
}
r = kvm_prepare_memory_region(kvm, old, new, change);
if (r) {
/*
* For DELETE/MOVE, revert the above INVALID change. No
* modifications required since the original slot was preserved
* in the inactive slots. Changing the active memslots also
* release slots_arch_lock.
*/
if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
kvm_activate_memslot(kvm, invalid_slot, old);
kfree(invalid_slot);
} else {
mutex_unlock(&kvm->slots_arch_lock);
if (r)
goto err;
if (change == KVM_MR_CREATE && (new->flags & KVM_MEM_GUEST_MEMFD)) {
r = kvm_gmem_commit_memory_region(kvm, new);
if (r) {
kvm_arch_free_memslot(kvm, new);
kvm_destroy_dirty_bitmap(new);
goto err;
}
return r;
}
/*
@ -1997,6 +2001,20 @@ static int kvm_set_memslot(struct kvm *kvm,
kvm_commit_memory_region(kvm, old, new, change);
return 0;
err:
/*
* For DELETE/MOVE, revert the above INVALID change. No modifications
* required since the original slot was preserved in the inactive slots.
* Changing the active memslots also release slots_arch_lock.
*/
if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
kvm_activate_memslot(kvm, invalid_slot, old);
kfree(invalid_slot);
} else {
mutex_unlock(&kvm->slots_arch_lock);
}
return r;
}
static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id,
@ -2125,21 +2143,30 @@ static int kvm_set_memory_region(struct kvm *kvm,
new->npages = npages;
new->flags = mem->flags;
new->userspace_addr = mem->userspace_addr;
if (mem->flags & KVM_MEM_GUEST_MEMFD) {
r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset);
if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) {
r = kvm_gmem_prepare_memory_region(kvm, new, mem->guest_memfd,
mem->guest_memfd_offset);
if (r)
goto out;
}
r = kvm_set_memslot(kvm, old, new, change);
/*
* Drop the reference to the gmem file, even on success. The file pins
* KVM, not the other way 'round. Active bindings are invalidated if
* the file is closed before memslots are destroyed.
*/
#ifdef CONFIG_KVM_GUEST_MEMFD
if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD))
fput(new->gmem.file);
#endif
if (r)
goto out_unbind;
goto out;
return 0;
out_unbind:
if (mem->flags & KVM_MEM_GUEST_MEMFD)
kvm_gmem_unbind(new);
out:
kfree(new);
return r;
@ -2439,32 +2466,41 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm,
}
#endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
static u64 kvm_supported_mem_attributes(struct kvm *kvm)
#ifdef kvm_arch_has_private_mem
static u64 kvm_supports_private_mem(struct kvm *kvm)
{
if (!kvm || kvm_arch_has_private_mem(kvm))
return KVM_MEMORY_ATTRIBUTE_PRIVATE;
return !kvm || kvm_arch_has_private_mem(kvm);
}
#else
#define kvm_supports_private_mem(kvm) false
#endif
return 0;
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm)
{
if (gmem_in_place_conversion || !kvm_supports_private_mem(kvm))
return 0;
return KVM_MEMORY_ATTRIBUTE_PRIVATE;
}
/*
* Returns true if _all_ gfns in the range [@start, @end) have attributes
* such that the bits in @mask match @attrs.
*/
bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
unsigned long mask, unsigned long attrs)
bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
unsigned long mask, unsigned long attrs)
{
XA_STATE(xas, &kvm->mem_attr_array, start);
unsigned long index;
void *entry;
mask &= kvm_supported_mem_attributes(kvm);
mask &= kvm_supported_vm_mem_attributes(kvm);
if (attrs & ~mask)
return false;
if (end == start + 1)
return (kvm_get_memory_attributes(kvm, start) & mask) == attrs;
return (kvm_get_vm_memory_attributes(kvm, start) & mask) == attrs;
guard(rcu)();
@ -2557,8 +2593,8 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
KVM_MMU_UNLOCK(kvm);
}
static bool kvm_pre_set_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range)
static bool kvm_pre_set_vm_memory_attributes(struct kvm *kvm,
struct kvm_gfn_range *range)
{
/*
* Unconditionally add the range to the invalidation set, regardless of
@ -2573,18 +2609,18 @@ static bool kvm_pre_set_memory_attributes(struct kvm *kvm,
*/
kvm_mmu_invalidate_range_add(kvm, range->start, range->end);
return kvm_arch_pre_set_memory_attributes(kvm, range);
return kvm_arch_pre_set_vm_memory_attributes(kvm, range);
}
/* Set @attributes for the gfn range [@start, @end). */
static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
static int kvm_set_vm_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
unsigned long attributes)
{
struct kvm_mmu_notifier_range pre_set_range = {
.start = start,
.end = end,
.arg.attributes = attributes,
.handler = kvm_pre_set_memory_attributes,
.handler = kvm_pre_set_vm_memory_attributes,
.on_lock = kvm_mmu_invalidate_start,
.flush_on_ret = true,
.may_block = true,
@ -2593,7 +2629,7 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
.start = start,
.end = end,
.arg.attributes = attributes,
.handler = kvm_arch_post_set_memory_attributes,
.handler = kvm_arch_post_set_vm_memory_attributes,
.on_lock = kvm_mmu_invalidate_end,
.may_block = true,
};
@ -2603,12 +2639,12 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
entry = attributes ? xa_mk_value(attributes) : NULL;
trace_kvm_vm_set_mem_attributes(start, end, attributes);
trace_kvm_set_vm_mem_attributes(start, end, attributes);
mutex_lock(&kvm->slots_lock);
/* Nothing to do if the entire range has the desired attributes. */
if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes))
if (kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attributes))
goto out_unlock;
/*
@ -2648,7 +2684,7 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm,
/* flags is currently not used. */
if (attrs->flags)
return -EINVAL;
if (attrs->attributes & ~kvm_supported_mem_attributes(kvm))
if (attrs->attributes & ~kvm_supported_vm_mem_attributes(kvm))
return -EINVAL;
if (attrs->size == 0 || attrs->address + attrs->size < attrs->address)
return -EINVAL;
@ -2665,9 +2701,26 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm,
*/
BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long));
return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes);
return kvm_set_vm_mem_attributes(kvm, start, end, attrs->attributes);
}
#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */
#ifdef kvm_arch_has_private_mem
DEFINE_STATIC_CALL_RET0(__kvm_is_private_gfn, kvm_is_private_gfn_t);
EXPORT_STATIC_CALL_GPL(__kvm_is_private_gfn);
static void kvm_init_memory_attributes(void)
{
if (gmem_in_place_conversion)
static_call_update(__kvm_is_private_gfn, kvm_gmem_is_private_gfn);
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
else
static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn);
#endif
}
#else
static void kvm_init_memory_attributes(void) { }
#endif
struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
{
@ -4973,15 +5026,20 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
case KVM_CAP_SYSTEM_EVENT_DATA:
case KVM_CAP_DEVICE_CTRL:
return 1;
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
case KVM_CAP_MEMORY_ATTRIBUTES:
return kvm_supported_mem_attributes(kvm);
return kvm_supported_vm_mem_attributes(kvm);
#endif
#ifdef CONFIG_KVM_GUEST_MEMFD
case KVM_CAP_GUEST_MEMFD:
return 1;
case KVM_CAP_GUEST_MEMFD_FLAGS:
return kvm_gmem_get_supported_flags(kvm);
case KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES:
if (!gmem_in_place_conversion || !kvm_supports_private_mem(kvm))
return 0;
return KVM_MEMORY_ATTRIBUTE_PRIVATE;
#endif
default:
break;
@ -5377,7 +5435,7 @@ static long kvm_vm_ioctl(struct file *filp,
break;
}
#endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES
#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES
case KVM_SET_MEMORY_ATTRIBUTES: {
struct kvm_memory_attributes attrs;
@ -5388,7 +5446,7 @@ static long kvm_vm_ioctl(struct file *filp,
r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs);
break;
}
#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */
#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */
case KVM_CREATE_DEVICE: {
struct kvm_create_device cd;
@ -6578,6 +6636,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module)
kvm_preempt_ops.sched_in = kvm_sched_in;
kvm_preempt_ops.sched_out = kvm_sched_out;
kvm_init_memory_attributes();
kvm_init_debug();
r = kvm_vfio_ops_init();