diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ba01c39d6bb5..03636d799837 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3146,6 +3146,32 @@ Kernel parameters kvm.enable_vmware_backdoor=[KVM] Support VMware backdoor PV interface. Default is false (don't support). + kvm.gmem_in_place_conversion= + [KVM] Controls whether KVM enables in-place conversion + support for guest_memfd and tracks the private/shared + state of memory per guest_memfd instead of per VM. + + If enabled, KVM enables the KVM_SET_MEMORY_ATTRIBUTES2 + ioctl on guest_memfd file descriptors and disables the + legacy VM-scoped KVM_SET_MEMORY_ATTRIBUTES ioctl for + private memory state tracking. Only the + KVM_MEMORY_ATTRIBUTE_PRIVATE attribute moves to + per-guest_memfd tracking; other attributes remain + per-VM. + + This parameter toggles KVM's in-place conversion + capability support. Whether a VMM uses separate backends + or out-of-place memory management is determined by + userspace VMM design. + + Note, this parameter is only available when + CONFIG_KVM_PRIVATE_MEMATTR_VM=y. When + CONFIG_KVM_PRIVATE_MEMATTR_VM is not set, in-place + conversion is enabled (and cannot be disabled) if KVM + is built with private memory support. + + Default is N (off). + kvm.nx_huge_pages= [KVM] Controls the software workaround for the X86_BUG_ITLB_MULTIHIT bug. diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e0430cc750c9..e95c9cda085d 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -117,7 +117,7 @@ description: x86 includes both i386 and x86_64. Type: - system, vm, or vcpu. + system, vm, vcpu or guest_memfd. Parameters: what parameters are accepted by the ioctl. @@ -918,7 +918,8 @@ The irq_type field has the following values: - KVM_ARM_IRQ_TYPE_SPI: in-kernel GICv2/GICv3: SPI, irq_id between 32 and 1019 (incl.) (the vcpu_index field is ignored) - in-kernel GICv5: SPI, irq_id between 0 and 65535 (incl.) + in-kernel GICv5: SPI, irq_id between 0 and the configured + number of SPIs minus one (1023 maximum) - KVM_ARM_IRQ_TYPE_PPI: in-kernel GICv2/GICv3: PPI, irq_id between 16 and 31 (incl.) in-kernel GICv5: PPI, irq_id between 0 and 127 (incl.) @@ -1322,6 +1323,9 @@ guests, across different userspace implementations. Nevertheless, userspace can still emulate all Arm exceptions by manipulating individual registers using the KVM_SET_ONE_REG API. +For a protected VM, setting ext_dabt_pending returns -EPERM; see +Documentation/virt/kvm/arm/pkvm.rst. Injecting an SError is unaffected. + See KVM_GET_VCPU_EVENTS for the data structure. Calling this ioctl on a vCPU that hasn't been initialized will return @@ -1545,7 +1549,9 @@ Possible values are: KVM_MP_STATE_RUNNABLE the vcpu is currently running [x86,arm64,riscv,loongarch] KVM_MP_STATE_UNINITIALIZED the vcpu is an application processor (AP) - which has not yet received an INIT signal [x86] + which has not yet received an INIT signal + [x86], or a protected VM's vcpu that the + guest holds powered off [arm64] KVM_MP_STATE_INIT_RECEIVED the vcpu has received an INIT signal, and is now ready for a SIPI [x86] KVM_MP_STATE_HALTED the vcpu has executed a HLT instruction and @@ -1624,6 +1630,12 @@ For arm64/riscv: The only states that are valid are KVM_MP_STATE_STOPPED and KVM_MP_STATE_RUNNABLE which reflect if the vcpu should be paused or not. +On arm64, once a protected VM's vcpu has run, it reads as +KVM_MP_STATE_UNINITIALIZED while the guest holds it powered off, with +CPU_OFF or by not yet bringing it online with CPU_ON, and +KVM_MP_STATE_RUNNABLE and KVM_MP_STATE_SUSPENDED return -EPERM on it: only +the guest can power it on. See Documentation/virt/kvm/arm/pkvm.rst. + On LoongArch, only the KVM_MP_STATE_RUNNABLE state is used to reflect whether the vcpu is runnable. @@ -2324,7 +2336,8 @@ Errors: ENOENT no such register EINVAL invalid register ID, or no such register or used with VMs in protected virtualization mode on s390 - EPERM (arm64) register access not allowed before vcpu finalization + EPERM (arm64) register access not allowed before vcpu + finalization, or after a protected VM's vcpu has run EBUSY (riscv) changing register value not allowed after the vcpu has run at least once ====== ============================================================ @@ -2949,7 +2962,8 @@ Errors include: ENOENT no such register EINVAL invalid register ID, or no such register or used with VMs in protected virtualization mode on s390 - EPERM (arm64) register access not allowed before vcpu finalization + EPERM (arm64) register access not allowed before vcpu + finalization, or after a protected VM's vcpu has run ======== ============================================================ (These error codes are indicative only: do not rely on a specific error @@ -3484,6 +3498,7 @@ Errors: ====== ================================================================= EINVAL the target is unknown, or the combination of features is invalid. ENOENT a features bit specified is unknown. + EPERM the vcpu belongs to a protected VM and has already run. ====== ================================================================= This tells KVM what type of CPU to present to the guest, and what @@ -3512,6 +3527,10 @@ after the vcpu has been run. This will reset the vcpu to its initial state. All calls to this function after the initial call must use the same target and same set of feature flags, otherwise EINVAL will be returned. +For a protected VM this ioctl returns EPERM once the vcpu has run, and the +features it accepts are restricted: KVM_ARM_VCPU_PSCI_0_2 is required. See +Documentation/virt/kvm/arm/pkvm.rst. + Possible features: - KVM_ARM_VCPU_POWER_OFF: Starts the CPU in a power-off state. @@ -3600,20 +3619,14 @@ Possible features: :Parameters: struct kvm_vcpu_init (out) :Returns: 0 on success; -1 on error -Errors: - - ====== ========================================== - ENODEV no preferred target available for the host - ====== ========================================== - This queries KVM for preferred CPU target type which can be emulated by KVM on underlying host. -The ioctl returns struct kvm_vcpu_init instance containing information -about preferred CPU target type and recommended features for it. The -kvm_vcpu_init->features bitmap returned will have feature bits set if -the preferred target recommends setting these features, but this is -not mandatory. +The ioctl returns a struct kvm_vcpu_init instance containing the +preferred CPU target type. The kvm_vcpu_init->features bitmap is +returned empty: userspace selects the vCPU features itself, and their +availability is reported by the capabilities listed under +KVM_ARM_VCPU_INIT. The information returned by this ioctl can be used to prepare an instance of struct kvm_vcpu_init for KVM_ARM_VCPU_INIT ioctl which will result in @@ -3676,7 +3689,7 @@ type KVM_X86_REG_TYPE_MSR, but are NOT enumerated via KVM_GET_REG_LIST. :Capability: KVM_CAP_ARM_SET_DEVICE_ADDR :Architectures: arm64 :Type: vm ioctl -:Parameters: struct kvm_arm_device_address (in) +:Parameters: struct kvm_arm_device_addr (in) :Returns: 0 on success, -1 on error Errors: @@ -3790,6 +3803,9 @@ can be determined by querying the KVM_CAP_GUEST_DEBUG_HW_BPS and KVM_CAP_GUEST_DEBUG_HW_WPS capabilities which return a positive number indicating the number of supported registers. +On arm64, this ioctl returns -EPERM for a protected VM: debugging a +protected guest is not supported. See Documentation/virt/kvm/arm/pkvm.rst. + For ppc, the KVM_CAP_PPC_GUEST_DEBUG_SSTEP capability indicates whether the single-step debug event (KVM_GUESTDBG_SINGLESTEP) is supported. @@ -6381,11 +6397,24 @@ mapping for userspace_addr is not required to be valid/populated at the time of KVM_SET_USER_MEMORY_REGION2, e.g. shared memory can be lazily mapped/allocated on-demand. -When mapping a gfn into the guest, KVM selects shared vs. private, i.e consumes -userspace_addr vs. guest_memfd, based on the gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE -state. At VM creation time, all memory is shared, i.e. the PRIVATE attribute -is '0' for all gfns. Userspace can control whether memory is shared/private by -toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via KVM_SET_MEMORY_ATTRIBUTES as needed. +When mapping a gfn into the guest, guest faults are always serviced from +guest_memfd regardless of whether memory is shared or private. KVM determines +shared vs. private based on the state in guest_memfd, which is the sole +authority on private vs. shared memory. See :ref:`KVM_CREATE_GUEST_MEMFD` to +find out more about the creation-time shared/private status. Userspace can +control whether memory is shared/private by toggling +KVM_MEMORY_ATTRIBUTE_PRIVATE via :ref:`KVM_SET_MEMORY_ATTRIBUTES2` as needed. + +userspace_addr is expected to be the mmap()-ed address corresponding to the +right offset within the guest_memfd. Any mismatch between userspace_addr and +guest_memfd is not validated and is a user error. userspace_addr is only used +for host-side guest accesses such as kvm_read_guest(). + +If in-place conversion is disabled, KVM selects shared vs. private based on the +gfn's KVM_MEMORY_ATTRIBUTE_PRIVATE state. At VM creation time, all memory is +shared, i.e. the PRIVATE attribute is '0' for all gfns. Userspace can control +whether memory is shared/private by toggling KVM_MEMORY_ATTRIBUTE_PRIVATE via +KVM_SET_MEMORY_ATTRIBUTES as needed. S390: ^^^^^ @@ -6393,6 +6422,8 @@ S390: Returns -EINVAL if the VM has the KVM_VM_S390_UCONTROL flag set. Returns -EINVAL if called on a protected VM. +.. _KVM_SET_MEMORY_ATTRIBUTES: + 4.141 KVM_SET_MEMORY_ATTRIBUTES ------------------------------- @@ -6429,6 +6460,10 @@ the state of a gfn/page as needed. The "flags" field is reserved for future extensions and must be '0'. +See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES2`. + +.. _KVM_CREATE_GUEST_MEMFD: + 4.142 KVM_CREATE_GUEST_MEMFD ---------------------------- @@ -6483,10 +6518,10 @@ specified via KVM_CREATE_GUEST_MEMFD. Currently defined flags: page tables. Private memory cannot. ============================ ================================================ -When the KVM MMU performs a PFN lookup to service a guest fault and the backing -guest_memfd has the GUEST_MEMFD_FLAG_MMAP set, then the fault will always be -consumed from guest_memfd, regardless of whether it is a shared or a private -fault. +When the KVM MMU performs a PFN lookup to service a guest fault, the fault will +always be consumed from guest_memfd, regardless of whether it is a shared or a +private fault (unless in-place conversion is disabled and the backing +guest_memfd does not have the GUEST_MEMFD_FLAG_MMAP flag set). See KVM_SET_USER_MEMORY_REGION2 for additional details. @@ -6494,7 +6529,7 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details. --------------------------- :Capability: KVM_CAP_PRE_FAULT_MEMORY -:Architectures: none +:Architectures: x86, s390, arm64 :Type: vcpu ioctl :Parameters: struct kvm_pre_fault_memory (in/out) :Returns: 0 if at least one page is processed, < 0 on error @@ -6502,12 +6537,15 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details. Errors: ========== =============================================================== + EAGAIN A race occurred before progress was made, but a retry may succeed. EINVAL The specified `gpa` and `size` were invalid (e.g. not page aligned, causes an overflow, or size is zero), or the VM is UCONTROL (s390). ENOENT The specified `gpa` is outside defined memslots. + ENOEXEC The vCPU has not been initialised (arm64). EINTR An unmasked signal is pending and no page was processed. EFAULT The parameter address was invalid. + EHWPOISON A poisoned host page was encountered. EOPNOTSUPP Mapping memory for a GPA is unsupported by the hypervisor, and/or for the current vCPU state/mode. EIO unexpected error conditions (also causes a WARN) @@ -6527,7 +6565,17 @@ Errors: KVM_PRE_FAULT_MEMORY populates KVM's stage-2 page tables used to map memory for the current vCPU state. KVM maps memory as if the vCPU generated a stage-2 read page fault, e.g. faults in memory as needed, but doesn't break -CoW. On x86, KVM does not mark any newly created stage-2 PTE as Accessed. +CoW. On arm64, KVM marks newly created stage-2 PTEs as Accessed, as it +does for any stage-2 fault, but leaves the Accessed state of existing PTEs +unchanged. On x86, KVM does not mark any newly created stage-2 PTE as +Accessed, and for s390 it is not applicable. + +On arm64, a GPA is interpreted as an IPA, and never interpreted as the IPA +of a nested guest. Pre-faulting only populates canonical stage-2 page +tables. + +The feature is not supported on arm64 if the protected KVM (pKVM) feature +is enabled. In the case of confidential VM types where there is an initial set up of private guest memory before the guest is 'finalized'/measured, this ioctl @@ -6542,7 +6590,7 @@ When the ioctl returns, the input values are updated to point to the remaining range. If `size` > 0 on return, the caller can just issue the ioctl again with the same `struct kvm_map_memory` argument. -Shadow page tables cannot support this ioctl because they +On x86, shadow page tables cannot support this ioctl because they are indexed by virtual address or nested guest physical address. Calling this ioctl when the guest is using shadow page tables (for example because it is running a nested guest with nested page tables) @@ -6663,6 +6711,83 @@ significant bit): Userspace should use the defined constants from ```` rather than hardcoding bit positions. +.. _KVM_SET_MEMORY_ATTRIBUTES2: + +4.146 KVM_SET_MEMORY_ATTRIBUTES2 +--------------------------------- + +:Capability: KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES +:Architectures: all +:Type: guest_memfd ioctl +:Parameters: struct kvm_memory_attributes2 (in/out) +:Returns: 0 on success, <0 on error + +Errors: + + ========== =============================================================== + EINVAL The specified `offset` or `size` was invalid (e.g. not + page aligned, causes an overflow, or size is zero). + EFAULT The parameter address was invalid. + ENOMEM Ran out of memory trying to track private/shared state + EAGAIN Pages in the range have outstanding references, see + ``error_offset`` below. + ========== =============================================================== + +KVM_SET_MEMORY_ATTRIBUTES2 is an extension to +KVM_SET_MEMORY_ATTRIBUTES that supports returning (writing) values to +userspace. The original (pre-extension) fields are shared with +KVM_SET_MEMORY_ATTRIBUTES identically. + +Attribute values are shared with KVM_SET_MEMORY_ATTRIBUTES. + +:: + + struct kvm_memory_attributes2 { + union { + __u64 address; + __u64 offset; + }; + __u64 size; + __u64 attributes; + __u64 flags; + __u64 error_offset; + __u64 reserved[11]; + }; + + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) + +The capability KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES enumerates the attributes +that can be set via KVM_SET_MEMORY_ATTRIBUTES2. + +Set attributes for a range of offsets within a guest_memfd to +KVM_MEMORY_ATTRIBUTE_PRIVATE to limit the specified guest_memfd backed +memory range for guest use. Even if the guest_memfd was created with +GUEST_MEMFD_FLAG_MMAP, after a successful call to set +KVM_MEMORY_ATTRIBUTE_PRIVATE, the requested range will not be mappable +into host userspace and will only be mappable by the guest. + +To allow the range to be mappable into host userspace again, call +KVM_SET_MEMORY_ATTRIBUTES2 on the guest_memfd again with +KVM_MEMORY_ATTRIBUTE_PRIVATE unset. + +KVM does not directly manipulate the memory contents of pages during +attribute updates. However, the process of setting these attributes, +which includes operations such as unmapping pages from the host or +stage-2 page tables, may result in side effects on memory contents +that vary across different trusted firmware implementations. + +If this ioctl returns -EAGAIN, the offset of the page with unexpected +refcounts will be returned in ``error_offset``. This can occur if +there are transient refcounts on the pages, taken by other parts of +the kernel. + +Userspace is expected to figure out how to remove all known refcounts +on the shared pages, such as refcounts taken by get_user_pages(), and +try the ioctl again. A possible source of these long term refcounts is +if the guest_memfd memory was pinned in IOMMU page tables. + +See also: :ref:`KVM_SET_MEMORY_ATTRIBUTES`. + .. _kvm_run: 5. The kvm_run structure @@ -9635,6 +9760,31 @@ On older versions of Linux, CPU[EAX=1]:ECX[24] (TSC_DEADLINE) is not reported by is present and the kernel has enabled in-kernel emulation of the local APIC. On newer versions, ``KVM_GET_SUPPORTED_CPUID`` does report the bit as available. +Long mode segment limits +~~~~~~~~~~~~~~~~~~~~~~~~ + +CPU[EAX=0x80000008]:EBX[20] (EFER_LMSLE_MBZ) is a "defeature" bit: it is set +when the CPU does *not* support long mode segment limits, and so requires +EFER.LMSLE to be zero. KVM reports the bit via ``KVM_GET_SUPPORTED_CPUID`` if +and only if KVM refuses to set EFER.LMSLE, i.e. if the CPU doesn't support long +mode segment limits, or if nested SVM is unsupported. KVM therefore reports +the bit on all Intel hosts, as KVM allows EFER.LMSLE only when nested SVM is +enabled. + +KVM never reports the bit via ``KVM_GET_EMULATED_CPUID``, but userspace may set +it via ``KVM_SET_CPUID2`` even on a host where KVM doesn't report it. KVM +honors the guest's enumeration and rejects EFER.LMSLE=1 accordingly. That lets +userspace defeature a vCPU on a host that *does* support long mode segment +limits, so that the vCPU can later be migrated to a host that doesn't, e.g. so +that a vCPU created on AMD Rome can be migrated to Milan and later, which +dropped support for long mode segment limits. The opposite direction needs no +emulation, as a host that lacks long mode segment limits already enumerates the +defeature. + +Note, ``KVM_SET_MSRS`` is exempt from the check, as host-initiated MSR writes +skip guest CPUID checks so that userspace can set MSRs before it sets guest +CPUID. ``KVM_SET_SREGS`` and nested VMRUN are not exempt. + CPU topology ~~~~~~~~~~~~ diff --git a/Documentation/virt/kvm/arm/fw-pseudo-registers.rst b/Documentation/virt/kvm/arm/fw-pseudo-registers.rst index d78b53b05dfc..c07471e258ea 100644 --- a/Documentation/virt/kvm/arm/fw-pseudo-registers.rst +++ b/Documentation/virt/kvm/arm/fw-pseudo-registers.rst @@ -33,6 +33,8 @@ The following registers are defined: highest PSCI version implemented by KVM and compatible with v0.2) - Allows any PSCI version implemented by KVM and compatible with v0.2 to be set with SET_ONE_REG + - Returns -EINVAL for a version below v1.1 on a protected VM (see + Documentation/virt/kvm/arm/pkvm.rst) - Affects the whole VM (even if the register view is per-vcpu) * KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: diff --git a/Documentation/virt/kvm/arm/pkvm.rst b/Documentation/virt/kvm/arm/pkvm.rst index 514992a79a83..b75bb8ad6f26 100644 --- a/Documentation/virt/kvm/arm/pkvm.rst +++ b/Documentation/virt/kvm/arm/pkvm.rst @@ -15,7 +15,9 @@ Overview Booting a host kernel with '``kvm-arm.mode=protected``' enables "Protected KVM" (pKVM). During boot, pKVM installs a stage-2 identity map page-table for the host and uses it to isolate the hypervisor -running at EL2 from the rest of the host running at EL1/0. +running at EL2 from the rest of the host running at EL1/0. pKVM supports +GICv3 only: GICv2 is not supported in protected mode, and neither are +GICv5-based guests. pKVM permits creation of protected virtual machines (pVMs) by passing the ``KVM_VM_TYPE_ARM_PROTECTED`` machine type identifier to the @@ -31,7 +33,7 @@ See hypercalls.rst for more details. Isolation mechanisms ==================== -pKVM relies on a number of mechanisms to isolate PVMs from the host: +pKVM relies on a number of mechanisms to isolate pVMs from the host: CPU memory isolation -------------------- @@ -50,8 +52,9 @@ stage-2 identity map as part of the donation procedure. This gives rise to some user-visible differences when compared to non-protected VMs, largely due to the lack of MMU notifiers: -* Memslots cannot be moved or deleted once the pVM has started running. -* Read-only memslots and dirty logging are not supported. +* Memslots cannot be moved or deleted once the pVM has started running + (``-EPERM``). +* Read-only memslots and dirty logging are not supported (``-EPERM``). * With the exception of swap, file-backed pages cannot be mapped into a pVM. * Donated pages are accounted against ``RLIMIT_MLOCK`` and so the VMM @@ -67,12 +70,19 @@ largely due to the lack of MMU notifiers: then it will either return ``-EFAULT`` or forcefully reclaim the memory pages. Reclaimed memory is zeroed by the hypervisor and a subsequent attempt to access it in the pVM will return ``-EFAULT`` - from the ``VCPU_RUN`` ioctl(). + from the ``KVM_RUN`` ioctl(). CPU state isolation ------------------- -Status: **Unimplemented.** +Status: CPU register state of protected vCPUs is managed entirely at EL2. + +pKVM performs the complete context switch for protected vCPUs within the +hypervisor. The hypervisor sets a protected vCPU's initial state, not the +host, and only what each exit needs is synchronised back. + +The user-visible consequences are described under `API behaviour for +protected VMs`_. DMA isolation using an IOMMU ---------------------------- @@ -89,13 +99,153 @@ The FF-A proxy ensures that the host cannot share pVM or hypervisor memory with Trustzone as part of a "confused deputy" attack. The PSCI proxy ensures that CPUs always have the stage-2 identity map -installed when they are executing in the host. +installed when they are executing in the host. This proxy is distinct +from the PSCI handling provided to protected guests, which is described +under `API behaviour for protected VMs`_. Protected VM firmware (pvmfw) ----------------------------- Status: **Unimplemented.** +API behaviour for protected VMs +=============================== + +Protected vCPU state is owned by EL2 (see `CPU state isolation`_). The VMM +configures a vCPU before its first ``KVM_RUN``; afterwards the state is +private to the guest and the ioctls that access it return ``-EPERM``. The +vCPU ioctls' errors follow one rule: ``-EPERM`` means the host asked for +state that the guest owns, ``-EINVAL`` means the request is not valid for a +protected VM, and a feature pKVM doesn't support for protected VMs is +rejected the way KVM rejects one that isn't implemented (``-ENXIO`` from +``KVM_ARM_VCPU_PVTIME_CTRL``, for example). The ioctls themselves are +described in Documentation/virt/kvm/api.rst. + +Boot +---- + +A protected VM boots from a single primary vCPU. Before the first +``KVM_RUN``, the VMM prepares the boot state: + +* Set ``KVM_MP_STATE_RUNNABLE`` on the primary vCPU and + ``KVM_MP_STATE_STOPPED`` on every other vCPU. EL2 allows only one + RUNNABLE primary per protected VM. A second RUNNABLE vCPU fails at its + first ``KVM_RUN``. +* Set the primary vCPU's boot state with ``KVM_SET_ONE_REG``: the kernel + entry address in ``PC`` and the DTB pointer in ``x0``. + +``PC`` and ``x0`` are the only registers EL2 takes from the host. Other +pre-run writes are accepted, but the guest starts from the state the +hypervisor resets it to. + +Secondary vCPUs are started by the guest itself through PSCI ``CPU_ON`` +(see `Power state`_), which supplies their entry point and context ID. +The VMM cannot choose where they boot. + +vCPU state +---------- + +* ``KVM_GET_ONE_REG`` and ``KVM_SET_ONE_REG`` return ``-EPERM`` once the + vCPU has run. Before that, they access the host-side copy from which + EL2 builds the guest's boot state (see `Boot`_). +* ``KVM_ARM_VCPU_INIT`` accepts only the vCPU features that a protected + guest supports and returns ``-EINVAL`` otherwise. + ``KVM_ARM_VCPU_PSCI_0_2`` is required, as EL2 implements PSCI 1.1 for + the guest (see `Power state`_). ``KVM_ARM_VCPU_EL1_32BIT`` is not + supported: protected guests run in AArch64 only and see no AArch32 + support in ``ID_AA64PFR0_EL1``. Once the vCPU has run, + ``KVM_ARM_VCPU_INIT`` returns ``-EPERM``, as re-initialising it would + reset the host-side copy alone. +* ``KVM_SET_VCPU_EVENTS`` returns ``-EPERM`` for external-abort injection + (``ext_dabt_pending``), and ``KVM_CAP_ARM_INJECT_EXT_DABT`` is not + advertised to a protected VM. SError injection is unaffected. +* ``KVM_SET_GUEST_DEBUG`` returns ``-EPERM``, and ``KVM_CAP_SET_GUEST_DEBUG`` + is not advertised to a protected VM (see `Debug`_). + +Power state +----------- + +EL2 implements PSCI 1.1 for a protected guest. The calls that move a +vCPU's power state, ``CPU_ON`` and ``CPU_OFF``, are handled at EL2, and +the host cannot change the outcome: for ``CPU_ON`` the host only +schedules the target, which EL2 has already reset to the entry point +the guest chose, and for ``CPU_OFF`` it only stops scheduling it. +``AFFINITY_INFO`` is handled by the host, which returns ``OFF`` once +it has stopped scheduling every vCPU the query covers, as for a +non-protected VM. A vCPU becomes a valid ``CPU_ON`` target at its first +``KVM_RUN``, and before that EL2 returns ``INVALID_PARAMETERS``. The +platform calls, ``CPU_SUSPEND``, ``SYSTEM_OFF``, ``SYSTEM_RESET`` and +``SYSTEM_RESET2``, are forwarded to the host and behave as for a +non-protected VM, with the ``SYSTEM_*`` calls exiting to the VMM as +``KVM_EXIT_SYSTEM_EVENT``; the ``SYSTEM_RESET2`` reset type and cookie +are in the guest's registers, which ``KVM_GET_ONE_REG`` rejects once +the vCPU has run. Any other function returns ``NOT_SUPPORTED``, and +``PSCI_FEATURES`` reports the same set. Because the host handles those +forwarded calls, ``KVM_SET_ONE_REG`` on ``KVM_REG_ARM_PSCI_VERSION`` +returns ``-EINVAL`` for a version below 1.1. + +Once a vCPU has run, its power state follows the guest's PSCI calls, not +the VMM's. A vCPU that the guest has powered off with ``CPU_OFF``, or has +not yet brought online with ``CPU_ON``, reads as +``KVM_MP_STATE_UNINITIALIZED``, and only an in-guest ``CPU_ON`` can power +it on: ``KVM_SET_MP_STATE`` returns ``-EPERM`` for ``KVM_MP_STATE_RUNNABLE`` +and ``KVM_MP_STATE_SUSPENDED`` on it, accepts ``KVM_MP_STATE_STOPPED`` and +leaves it as it is, and never takes ``KVM_MP_STATE_UNINITIALIZED`` from +the VMM. ``KVM_MP_STATE_STOPPED`` still stops a vCPU EL2 has powered on, +and ``KVM_MP_STATE_RUNNABLE`` resumes it, so the VMM can pause it. + +Other interface differences +--------------------------- + +* ``KVM_CHECK_EXTENSION`` reports only the arm64 capabilities that pKVM + supports for protected guests, and ``KVM_ENABLE_CAP`` accepts only + those. Query them on the VM file descriptor: the system file descriptor + has no VM to filter against. The capabilities the generic KVM code + answers itself (``KVM_CAP_SYNC_MMU`` or ``KVM_CAP_DEVICE_CTRL``, for + example) are reported as for any VM, and don't mean the feature works + for a protected one. The filter does not cover every interface either: + device-fd configuration (for example the VGIC after + ``KVM_CREATE_DEVICE``) and vCPU attributes are unfiltered, so a feature + the filter excludes is rejected by the attribute itself. + ``KVM_ARM_VCPU_PVTIME_CTRL`` returns ``-ENXIO``, for example, as it does + where steal time isn't implemented. +* The vGIC of a protected VM remains host-managed: device creation, + configuration and interrupt injection all work as they do for a + non-protected VM. +* ``KVM_ARM_SET_COUNTER_OFFSET`` and ``KVM_ARM_GET_REG_WRITABLE_MASKS`` + return ``-EINVAL``: their capabilities are not advertised to a protected + VM, whose counter offset and ID registers are set by EL2. A protected + guest sees the physical timebase. +* A protected guest's first access to each page of memory exits to the + host, since the hypervisor cannot tell memory from a device before the + page is mapped. For a store, the host sees the value of the register + the syndrome names, clamped to the access width, and nothing else from + the register file. For a load, the register the syndrome names takes + the value the host returns, as for any emulated access. +* A protected guest that uses a feature it was not given, or executes an + ``SMC``, takes an undefined instruction exception from the hypervisor; + the host is not involved. +* Apart from PSCI, the hypervisor handles a protected guest's SMCCC + calls itself and does not involve the host. A function it does not + implement returns ``NOT_SUPPORTED``, and the host's SMCCC filter does + not apply: the VM device attribute ioctls (``KVM_ARM_VM_SMCCC_CTRL``) + return ``-EINVAL`` for a protected VM. +* The hypervisor can decode a trapped guest access only from the CPU's + instruction syndrome, which is provided only for a load or store of a + single general-purpose register. An access without one (for example a + load/store pair or a SIMD/FP access) cannot be decoded. For a + non-protected VM it can exit to the VMM as ``KVM_EXIT_ARM_NISV``. For a + protected VM it cannot be emulated by the VMM, so the guest takes a + synchronous external abort instead. + +Debug +----- + +Hardware-assisted debugging is not available to protected guests. +``MDSCR_EL1``, ``OSLAR_EL1``, ``OSLSR_EL1``, ``OSDLR_EL1``, and breakpoint +and watchpoint 0 are RAZ/WI; any other debug register access takes an +undefined instruction exception. + Resources ========= diff --git a/Documentation/virt/kvm/devices/arm-vgic-v5.rst b/Documentation/virt/kvm/devices/arm-vgic-v5.rst index 70b9162755c7..8a691c8938b6 100644 --- a/Documentation/virt/kvm/devices/arm-vgic-v5.rst +++ b/Documentation/virt/kvm/devices/arm-vgic-v5.rst @@ -12,12 +12,39 @@ Only one VGIC instance may be instantiated through this API. The created VGIC will act as the VM interrupt controller, requiring emulated user-space devices to inject interrupts to the VGIC instead of directly to CPUs. -Creating a guest GICv5 device requires a GICv5 host. The current VGICv5 device -only supports PPI interrupts. These can either be injected from emulated -in-kernel devices (such as the Arch Timer, or PMU), or via the KVM_IRQ_LINE -ioctl. +Creating a guest GICv5 device requires a GICv5 host. The VGICv5 device supports +PPI, SPI, and LPI interrupts. The PPI and SPI interrupts can either be injected +from emulated in-kernel devices (such as the Arch Timer, or PMU), or via the +KVM_IRQ_LINE ioctl. LPIs are not externally injected, but are handled in +hardware via the LPI IST. Their pending state is driven directly by the guest. Groups: + KVM_DEV_ARM_VGIC_GRP_ADDR + Attributes: + + KVM_VGIC_V5_ADDR_TYPE_IRS (rw, 64-bit) + Base address in the guest physical address space of the GICv5 IRS + (Interrupt Routing Service) register mappings. Only valid for + KVM_DEV_TYPE_ARM_VGIC_V5. This address needs to be 64K aligned and the + region covers 128 KByte - the IRS has a CONFIG_FRAME and a SETLPI_FRAME, + each of which is 64 KBytes in size. + + Setting the address of the IRS in GPA space is mandatory before VGIC + resources are mapped, as the IRS is responsible for handling SPIs and + LPIs. Failure to set the IRS address before the first vCPU run results in + an error. + + KVM_DEV_ARM_VGIC_GRP_NR_IRQS + Attributes: + + A value describing the number of SPIs for this GIC instance. This is + GICv5-specific: unlike GICv2/v3, the value does not include SGIs or PPIs. + The value ranges from 32 to KVM's VGICv5 maximum of 1024 SPIs, in + increments of 32. If userspace does not set this attribute, KVM uses 32 + SPIs by default. + + kvm_device_attr.addr points to a __u32 value. + KVM_DEV_ARM_VGIC_GRP_CTRL Attributes: @@ -48,3 +75,239 @@ Groups: -EFAULT Invalid guest ram access -EBUSY One or more VCPUS are running ======= ======================================================== + + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS + Attributes: + + The attr field of kvm_device_attr encodes two values:: + + bits: | 63 .... 32 | 31 .... 16 | 15 .... 0 | + values: | mpidr | RES | instr | + + The mpidr field encodes the CPU ID based on the affinity information in the + architecture defined MPIDR, and the field is encoded as follows:: + + | 63 .... 56 | 55 .... 48 | 47 .... 40 | 39 .... 32 | + | Aff3 | Aff2 | Aff1 | Aff0 | + + The instr field encodes the system register to access based on the fields + defined in the A64 instruction set encoding for system register access + (RES means the bits are reserved for future use and should be zero):: + + | 15 ... 14 | 13 ... 11 | 10 ... 7 | 6 ... 3 | 2 ... 0 | + | Op 0 | Op1 | CRn | CRm | Op2 | + + All system regs accessed through this API are (rw, 64-bit) and + kvm_device_attr.addr points to a __u64 value. + + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS accesses the CPU interface registers for the + CPU specified by the mpidr field. + + The available registers are: + + ======================= =================================================== + ICC_ICSR_EL1 + ICC_PPI_ENABLER0_EL1 + ICC_PPI_ENABLER1_EL1 + ICC_PPI_SACTIVER0_EL1 ICC_PPI_CACTIVER0_EL1 is not supported. Writes to + ICC_PPI_SACTIVER0_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SACTIVER1_EL1 ICC_PPI_CACTIVER1_EL1 is not supported. Writes to + ICC_PPI_SACTIVER1_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SPENDR0_EL1 ICC_PPI_CPENDR0_EL1 is not supported. Writes to + ICC_PPI_SPENDR0_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_SPENDR1_EL1 ICC_PPI_CPENDR1_EL1 is not supported. Writes to + ICC_PPI_SPENDR1_EL1 are treated as RAW writes of + the underlying state. + ICC_PPI_PRIORITYR0_EL1 + ICC_PPI_PRIORITYR1_EL1 + ICC_PPI_PRIORITYR2_EL1 + ICC_PPI_PRIORITYR3_EL1 + ICC_PPI_PRIORITYR4_EL1 + ICC_PPI_PRIORITYR5_EL1 + ICC_PPI_PRIORITYR6_EL1 + ICC_PPI_PRIORITYR7_EL1 + ICC_PPI_PRIORITYR8_EL1 + ICC_PPI_PRIORITYR9_EL1 + ICC_PPI_PRIORITYR10_EL1 + ICC_PPI_PRIORITYR11_EL1 + ICC_PPI_PRIORITYR12_EL1 + ICC_PPI_PRIORITYR13_EL1 + ICC_PPI_PRIORITYR14_EL1 + ICC_PPI_PRIORITYR15_EL1 + ICC_APR_EL1 + ICC_CR0_EL1 + ICC_PCR_EL1 + ======================= =================================================== + + Errors: + + ======= ============================================================= + -ENXIO Getting or setting this register is not supported + -EBUSY VCPU is running, or write attempted after a VCPU has run + -EINVAL Invalid mpidr or register value supplied + ======= ============================================================= + + + KVM_DEV_ARM_VGIC_GRP_IRS_REGS + Attributes: + The attr field of kvm_device_attr encodes the offset of the IRS register, + relative to the IRS CONFIG_FRAME base address. This is the address that + was provided via KVM_VGIC_V5_ADDR_TYPE_IRS when creating VGICv5 in the + first place. + + kvm_device_attr.addr points to a __u64 value whatever the width + of the addressed register (32/64 bits). 64 bit registers can only + be accessed with full length. + + Writes to read-only registers are ignored by the kernel except for + IRS_IDR0 - IRS_IDR2 and IRS_IDR5 - IRS_IDR7, which are sanity checked to + ensure that they match a sane config. + + IRS_IDR3 and IRS_IDR4 are RAZ/WI, as nested virtualization is not + supported. + + For registers without dedicated userspace accessors, getting or setting a + register uses the same emulated MMIO handlers as guest reads/writes. + Dedicated userspace accessors may instead save or restore migration state + without triggering guest-visible side effects. For example, restoring + IRS_IST_BASER only restores the emulated register state; any host LPI IST + allocation based on the restored IRS_IST_CFGR and IRS_IST_BASER state + happens when KVM_DEV_ARM_VGIC_GRP_IST is restored. + + Once an LPI IST has been allocated, IRS_IST_CFGR is immutable. Userspace + may write back its current value, but setting a different value is + rejected with -EINVAL. + + Once an LPI IST has been allocated, IRS_IST_BASER is immutable. + Userspace may write back its current value, but setting a different value + is rejected with -EINVAL. + + Errors: + + ======= ================================================================= + -ENXIO Offset does not correspond to any supported register + -EFAULT Invalid user pointer for attr->addr + -EINVAL Offset is not 32-bit aligned for 32-bit MMIO registers, not + 64-bit aligned for 64-bit registers, or the supplied register + value is not compatible with the configured VGICv5 IRS state + -EBUSY VGIC is not initialized, one or more VCPUs are running, or a + write is attempted after a VCPU has run + ======= ================================================================= + + KVM_DEV_ARM_VGIC_GRP_IST + Attributes: + This interface is used to either save the state of the IRS's Interrupt + State Tables (ISTs), or to restore them. A get operation saves IST state, + and a set operation restores IST state. kvm_device_attr.attr is reserved + and must be zero. + + The VGIC must be initialized before using this interface. Restore must be + performed before the VM has run. For restore, userspace must have already + restored the IRS state needed to describe any guest LPI IST. + + Saving first asks the IRS to save and quiesce the VM so that interrupt + state has been written back to the ISTs. KVM checks that the VM remains + quiesced while copying out the SPI and LPI IST state. + + kvm_device_attr.addr points to a struct kvm_vgic_v5_ist:: + + struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; + }; + + The SPI and LPI IST buffers contain one little-endian 32-bit IST entry per + interrupt, in interrupt number order. Only the architected 32-bit ISTE + state is exposed to userspace; host IST layout and metadata are private to + KVM. + + The SPI IST buffer is required and its size must be: + + nr_spis * sizeof(__u32) + + where nr_spis is the value returned by KVM_DEV_ARM_VGIC_GRP_NR_IRQS for + the VGICv5 device. For VGICv5 this value is the number of SPIs, not the + total number of interrupts. + + The LPI IST buffer is required if the guest's IRS_IST_CFGR and + IRS_IST_BASER state describes a valid LPI IST, and its size must be: + + BIT(lpi_id_bits) * sizeof(__u32) + + where lpi_id_bits is the LPI ID width configured in IRS_IST_CFGR. If no + LPI IST is configured, lpi_ist_addr and lpi_ist_size must both be zero. + + Errors: + + =========== ============================================================ + -EBUSY One or more VCPUs are running, the VGIC is not initialized, + restore was requested after the VM has run, an LPI IST + already exists, or the save operation completed but the VM + did not remain quiesced + -EINVAL attr->addr is NULL, the userspace IST descriptor is missing + a required buffer, supplies a non-zero LPI buffer when no + LPI IST is configured, has a buffer size that does not match + the configured VGICv5 IST state, or an internal VM table + operation rejected the VM state + -EFAULT Invalid user pointer for attr->addr, spi_ist_addr, or + lpi_ist_addr + -ENXIO Required per-VM VGICv5/IST backing state is missing or + inconsistent + -ENOMEM Restoring IST state failed while allocating the host LPI IST + or tracking pending interrupts + -ETIMEDOUT An IRS save/VM operation timed out + =========== ============================================================ + +IRS Save Sequence: +------------------ + +The following operations are required when saving the virtual GICv5 IRS: + +a) Save the ISTs by issuing KVM_GET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_IST. +b) Save the IRS MMIO register state by issuing KVM_GET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IRS_REGS. + +These two steps may be performed in either order. Saving the ISTs writes all +IST state to userspace-provided buffers and does not update guest memory. + +IRS Restore Sequence: +--------------------- + +The following ordering must be followed when restoring the virtual GICv5 and +IRS: + +a) Create vCPUs. +b) Provide the IRS base address by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_ADDR +c) Restore the number of SPIs by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_NR_IRQS. +d) Initialise the GIC - this sets up the default state and creates the SPI + IST - by issuing KVM_SET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_CTRL with + KVM_DEV_ARM_VGIC_CTRL_INIT +e) Restore the IRS MMIO register state by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IRS_REGS. The IRS registers may be restored in any + order, but any IRS_IDR* state and the IRS_IST_CFGR/IRS_IST_BASER state + describing an LPI IST must be restored before restoring the ISTs. KVM uses + the restored IRS_IST_CFGR and IRS_IST_BASER state to allocate the LPI IST + during the following step. +f) Restore the ISTs by issuing KVM_SET_DEVICE_ATTR on + KVM_DEV_ARM_VGIC_GRP_IST. + +The number of SPIs must be restored before VGIC initialization because +initialization allocates the SPI state and fixes the SPI range exposed by the +IRS ID registers. + +The IRS's MMIO registers must be restored prior to restoring the ISTs as these +are used to convey the number of LPIs the guest has configured to KVM. + +The various ``*_STATUSR`` registers are observational state in the current KVM +implementation. Userspace may save them for validation or debugging purposes, +but they are not required as restore input and do not need to be replayed during +restore. + +Then vCPUs can be started. diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst index deb5c51bc00c..7e23584fd74a 100644 --- a/Documentation/virt/kvm/devices/vcpu.rst +++ b/Documentation/virt/kvm/devices/vcpu.rst @@ -135,7 +135,7 @@ after selecting a hardware PMU. Request that the VCPU uses the specified hardware PMU when creating guest events for the purpose of PMU emulation. The PMU identifier can be read from the "type" file for the desired PMU instance under /sys/devices (or, equivalent, -/sys/bus/even_source). This attribute is particularly useful on heterogeneous +/sys/bus/event_source). This attribute is particularly useful on heterogeneous systems where there are at least two CPU PMUs on the system. The PMU that is set for one VCPU will be used by all the other VCPUs. It isn't possible to set a PMU if a PMU event filter is already present. @@ -231,7 +231,8 @@ Returns: Specifies the base address of the stolen time structure for this VCPU. The base address must be 64 byte aligned and exist within a valid guest memory region. See Documentation/virt/kvm/arm/pvtime.rst for more information -including the layout of the stolen time structure. +including the layout of the stolen time structure. Stolen time is not +supported for protected VMs (see Documentation/virt/kvm/arm/pkvm.rst). 4. GROUP: KVM_VCPU_TSC_CTRL =========================== diff --git a/Documentation/virt/kvm/devices/vm.rst b/Documentation/virt/kvm/devices/vm.rst index a4d39fa1b083..78e63eb9994e 100644 --- a/Documentation/virt/kvm/devices/vm.rst +++ b/Documentation/virt/kvm/devices/vm.rst @@ -280,6 +280,24 @@ Allows user space to disable dea key wrapping, clearing the wrapping key. :Parameters: none :Returns: 0 +4.5. ATTRIBUTE: KVM_S390_VM_CRYPTO_ENABLE_APIE (w/o) +------------------------------------------------------- + +Allows user space to enable AP interpretive execution. + +:Parameters: none +:Returns: 0 for success + -EOPNOTSUPP if AP instructions not available for guest + +4.6. ATTRIBUTE: KVM_S390_VM_CRYPTO_DISABLE_APIE (w/o) +------------------------------------------------------- + +Allows user space to disable AP interpretive execution. + +:Parameters: none +:Returns: 0 for success + -EOPNOTSUPP if AP instructions not available for guest + 5. GROUP: KVM_S390_VM_MIGRATION =============================== diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index bd04a908a8db..eeb0812cb10a 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -170,7 +170,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_update { + struct kvm_sev_launch_update_data { __u64 uaddr; /* userspace address to be encrypted (must be 16-byte aligned) */ __u32 len; /* length of the data to be encrypted (must be 16-byte aligned) */ }; @@ -316,7 +316,7 @@ used by the guest owner with the KVM_SEV_LAUNCH_MEASURE. If len is zero on entry, the measurement blob length is written to len and uaddr is unused. -Parameters (in): struct kvm_sev_attestation +Parameters (in): struct kvm_sev_attestation_report Returns: 0 on success, -negative on error @@ -377,7 +377,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_send_update_data { + struct kvm_sev_send_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; @@ -447,7 +447,7 @@ Returns: 0 on success, -negative on error :: - struct kvm_sev_launch_receive_update_data { + struct kvm_sev_receive_update_data { __u64 hdr_uaddr; /* userspace address containing the packet header */ __u32 hdr_len; @@ -503,7 +503,8 @@ secrets. It is required that the GPA ranges initialized by this command have had the KVM_MEMORY_ATTRIBUTE_PRIVATE attribute set in advance. See the documentation -for KVM_SET_MEMORY_ATTRIBUTES for more details on this aspect. +for KVM_SET_MEMORY_ATTRIBUTES/KVM_SET_MEMORY_ATTRIBUTES2 for more details on +this aspect. Upon success, this command is not guaranteed to have processed the entire range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of @@ -511,9 +512,14 @@ range requested. Instead, the ``gfn_start``, ``uaddr``, and ``len`` fields of remaining range that has yet to be processed. The caller should continue calling this command until those fields indicate the entire range has been processed, e.g. ``len`` is 0, ``gfn_start`` is equal to the last GFN in the -range plus 1, and ``uaddr`` is the last byte of the userspace-provided source -buffer address plus 1. In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, -``uaddr`` will be ignored completely. +range plus 1, and ``uaddr`` (if specified) is the last byte of the +userspace-provided source buffer address plus 1. + +In the case where ``type`` is KVM_SEV_SNP_PAGE_TYPE_ZERO, ``uaddr`` will be +ignored completely. For all other page types, ``uaddr`` is optional if in-place +conversion is enabled (i.e. when the data had been written directly to +guest_memfd while the page was in the shared state) and is required if in-place +conversion is disabled. Parameters (in): struct kvm_sev_snp_launch_update diff --git a/Documentation/virt/kvm/x86/hypercalls.rst b/Documentation/virt/kvm/x86/hypercalls.rst index 521ecf9a8a36..e0b6c11f19ce 100644 --- a/Documentation/virt/kvm/x86/hypercalls.rst +++ b/Documentation/virt/kvm/x86/hypercalls.rst @@ -109,7 +109,7 @@ The template for each hypercall is: Usage: a0: guest physical address where host copies -"struct kvm_clock_offset" structure. +"struct kvm_clock_pairing" structure. a1: clock_type, ATM only KVM_CLOCK_PAIRING_WALLCLOCK (0) is supported (corresponding to the host's CLOCK_REALTIME clock). diff --git a/Documentation/virt/kvm/x86/intel-tdx.rst b/Documentation/virt/kvm/x86/intel-tdx.rst index 6a222e9d0954..d8d9409120e6 100644 --- a/Documentation/virt/kvm/x86/intel-tdx.rst +++ b/Documentation/virt/kvm/x86/intel-tdx.rst @@ -158,6 +158,10 @@ KVM_TDX_INIT_MEM_REGION Initialize @nr_pages TDX guest private memory starting from @gpa with userspace provided data from @source_addr. @source_addr must be PAGE_SIZE-aligned. +If guest_memfd in-place conversion is enabled, pass 0 for @source_addr +to represent "no source page". A source page is required if in-place +conversion is not enabled or not supported. + Note, before calling this sub command, memory attribute of the range [gpa, gpa + nr_pages] needs to be private. Userspace can use KVM_SET_MEMORY_ATTRIBUTES to set the attribute. diff --git a/MAINTAINERS b/MAINTAINERS index 8e940a241dcd..2790a256b406 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14422,6 +14422,8 @@ F: arch/powerpc/include/asm/kvm* F: arch/powerpc/include/uapi/asm/kvm* F: arch/powerpc/kernel/kvm* F: arch/powerpc/kvm/ +F: tools/testing/selftests/kvm/*/powerpc/ +F: tools/testing/selftests/kvm/powerpc/ KERNEL VIRTUAL MACHINE FOR RISC-V (KVM/riscv) M: Anup Patel @@ -14446,6 +14448,7 @@ L: kvm@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git F: Documentation/virt/kvm/s390* +F: arch/s390/include/arm64/* F: arch/s390/include/asm/gmap_helpers.h F: arch/s390/include/asm/kvm* F: arch/s390/include/uapi/asm/kvm* @@ -14458,6 +14461,22 @@ F: tools/testing/selftests/drivers/s390x/uvdevice/ F: tools/testing/selftests/kvm/*/s390/ F: tools/testing/selftests/kvm/s390/ +KERNEL VIRTUAL MACHINE for arm-on-s390 (KVM/s390) +R: Steffen Eiden +R: Andreas Grapentin +R: Marc Zyngier +L: kvm@vger.kernel.org +L: linux-s390@vger.kernel.org +S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git +F: arch/s390/include/arm64/ +F: arch/s390/include/asm/aef.h +F: arch/s390/include/asm/kvm_host_arm64* +F: arch/s390/include/asm/sae* +F: arch/s390/kernel/aef.c +F: arch/s390/kvm/arm64/ +F: arch/s390/tools/*arm64* + KERNEL VIRTUAL MACHINE FOR X86 (KVM/x86) M: Sean Christopherson M: Paolo Bonzini diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 04e1a0d12f8f..ed28f94be428 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -45,6 +45,9 @@ KBUILD_CFLAGS += $(CC_FLAGS_NO_FPU) \ KBUILD_CFLAGS += $(call cc-disable-warning, psabi) KBUILD_AFLAGS += $(compat_vdso) +# Enable all code shared to s390 +KBUILD_CPPFLAGS += -DARM64_S390_COMMON=1 + ifeq ($(call rustc-min-version, 108500),y) KBUILD_RUSTFLAGS += --target=aarch64-unknown-none-softfloat else diff --git a/arch/arm64/include/asm/brk-imm.h b/arch/arm64/include/asm/brk-imm.h index beb42c62b6ac..dd2d153dc0d8 100644 --- a/arch/arm64/include/asm/brk-imm.h +++ b/arch/arm64/include/asm/brk-imm.h @@ -2,6 +2,7 @@ /* * Copyright (C) 2012 ARM Ltd. */ +/* Whole file is shared with s390 */ #ifndef __ASM_BRK_IMM_H #define __ASM_BRK_IMM_H diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index f816f5d77f1a..a584f88dbaf9 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -3,11 +3,12 @@ * Copyright (C) 2013 - ARM Ltd * Author: Marc Zyngier */ +/* Whole file is shared with s390 */ #ifndef __ASM_ESR_H #define __ASM_ESR_H -#include +#include #include #define ESR_ELx_EC_UNKNOWN UL(0x00) @@ -435,7 +436,36 @@ #define ESR_ELx_IT_GCSPOPX 7 #ifndef __ASSEMBLER__ -#include +#include + +static __always_inline bool esr_trap_is_iabt(unsigned long esr) +{ + return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW; +} + +/* Always check for S1PTW *before* using this. */ +static __always_inline bool esr_dabt_is_write(unsigned long esr) +{ + return esr & ESR_ELx_WNR; +} + +static __always_inline bool esr_dabt_is_cm(unsigned long esr) +{ + return esr & ESR_ELx_CM; +} + +static __always_inline bool esr_abt_is_sea(unsigned long esr) +{ + switch (esr & ESR_ELx_FSC) { + case ESR_ELx_FSC_EXTABT: + case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3): + case ESR_ELx_FSC_SECC: + case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3): + return true; + default: + return false; + } +} static inline unsigned long esr_brk_comment(unsigned long esr) { diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 4bfbd827c5aa..678f0daed445 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -3,14 +3,16 @@ * Copyright (C) 2012,2013 - ARM Ltd * Author: Marc Zyngier */ +/* Whole file is shared with s390 */ #ifndef __ARM64_KVM_ARM_H__ #define __ARM64_KVM_ARM_H__ +#include +#include +#include #include -#include #include -#include /* * Because I'm terribly lazy and that repainting the whole of the KVM diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index e5b92ac09e69..7d799cd05dcd 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -62,6 +62,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs, __KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs, __KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_alloc_selftest, MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM), @@ -87,14 +88,20 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___tracing_write_event, __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs, __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs, + __KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident, + __KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident, + __KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend, __KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr, __KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_topup, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaim, + __KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaimable, + __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp, + __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp, MARKER(__KVM_HOST_SMCCC_FUNC_PKVM_ONLY), /* Hypercalls that are available only when pKVM has finalised. */ - __KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp, - __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp, __KVM_HOST_SMCCC_FUNC___pkvm_host_donate_guest, __KVM_HOST_SMCCC_FUNC___pkvm_host_share_guest, __KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_guest, diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index a3c1928bdf74..85289a4b8d52 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -44,6 +44,7 @@ enum exception_type { bool kvm_condition_valid32(const struct kvm_vcpu *vcpu); void kvm_skip_instr32(struct kvm_vcpu *vcpu); +#if ARM64_S390_COMMON == 1 void kvm_inject_undefined(struct kvm_vcpu *vcpu); void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr); int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr); @@ -75,6 +76,8 @@ static inline int kvm_inject_serror(struct kvm_vcpu *vcpu) void kvm_vcpu_wfi(struct kvm_vcpu *vcpu); +#endif /* ARM64_S390_COMMON == 1 */ + void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu); int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2); int kvm_inject_nested_irq(struct kvm_vcpu *vcpu); @@ -133,12 +136,37 @@ static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr) static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) { - return (unsigned long *)&vcpu_gp_regs(vcpu)->pc; + return (unsigned long *)&vcpu->arch.ctxt.regs.pc; } static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) { - return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate; + return (unsigned long *)&vcpu->arch.ctxt.regs.pstate; +} + +static __always_inline unsigned long *vcpu_sp_el0(struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.ctxt.regs.sp; +} + +static __always_inline u64 *vcpu_sp_el1(struct kvm_vcpu *vcpu) +{ + return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); +} + +static __always_inline __u128 *vcpu_vreg(struct kvm_vcpu *vcpu, int n) +{ + return &vcpu->arch.ctxt.fp_regs.vregs[n]; +} + +static __always_inline __u32 *vcpu_fpsr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.ctxt.fp_regs.fpsr; +} + +static __always_inline __u32 *vcpu_fpcr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.ctxt.fp_regs.fpcr; } static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) @@ -159,6 +187,7 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) *vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT; } +#if ARM64_S390_COMMON == 1 /* * vcpu_get_reg and vcpu_set_reg should always be passed a register number * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on @@ -167,16 +196,18 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu) static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu, u8 reg_num) { - return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num]; + return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)[reg_num]; } static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num, unsigned long val) { if (reg_num != 31) - vcpu_gp_regs(vcpu)->regs[reg_num] = val; + vcpu_gp_regs(vcpu)[reg_num] = val; } +#endif /* ARM64_S390_COMMON == 1 */ + static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt) { switch (ctxt->regs.pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) { @@ -379,6 +410,12 @@ static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu) return vcpu->arch.fault.disr_el1; } +#if ARM64_S390_COMMON == 1 +static __always_inline bool esr_abt_is_s1ptw(unsigned long esr) +{ + return esr & ESR_ELx_S1PTW; +} + static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu) { return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK; @@ -411,18 +448,13 @@ static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu) static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW); + return esr_abt_is_s1ptw(kvm_vcpu_get_esr(vcpu)); } /* Always check for S1PTW *before* using this. */ static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR; -} - -static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu) -{ - return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM); + return esr_dabt_is_write(kvm_vcpu_get_esr(vcpu)); } static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu) @@ -443,12 +475,7 @@ static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW; -} - -static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu) -{ - return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu); + return esr_trap_is_iabt(kvm_vcpu_get_esr(vcpu)); } static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) @@ -456,6 +483,13 @@ static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC; } +#endif /* ARM64_S390_COMMON == 1 */ + +static __always_inline bool esr_abt_is_exec_fault(unsigned long esr) +{ + return esr_trap_is_iabt(esr) && !esr_abt_is_s1ptw(esr); +} + static inline bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) { @@ -468,26 +502,9 @@ bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu) return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu)); } -static inline -u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu) -{ - unsigned long esr = kvm_vcpu_get_esr(vcpu); - - BUG_ON(!esr_fsc_is_permission_fault(esr)); - return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL)); -} - static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu) { - switch (kvm_vcpu_trap_get_fault(vcpu)) { - case ESR_ELx_FSC_EXTABT: - case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3): - case ESR_ELx_FSC_SECC: - case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3): - return true; - default: - return false; - } + return esr_abt_is_sea(kvm_vcpu_get_esr(vcpu)); } static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) @@ -496,9 +513,10 @@ static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu) return ESR_ELx_SYS64_ISS_RT(esr); } -static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) +#if ARM64_S390_COMMON == 1 +static inline bool esr_abt_is_write_fault(unsigned long esr) { - if (kvm_vcpu_abt_iss1tw(vcpu)) { + if (esr_abt_is_s1ptw(esr)) { /* * Only a permission fault on a S1PTW should be * considered as a write. Otherwise, page tables baked @@ -511,15 +529,22 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) * first), then a permission fault to allow the flags * to be set. */ - return kvm_vcpu_trap_is_permission_fault(vcpu); + return esr_fsc_is_permission_fault(esr); } - if (kvm_vcpu_trap_is_iabt(vcpu)) + if (esr_trap_is_iabt(esr)) return false; - return kvm_vcpu_dabt_iswrite(vcpu); + return esr_dabt_is_write(esr); } +static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu) +{ + return esr_abt_is_write_fault(kvm_vcpu_get_esr(vcpu)); +} + +#endif /* ARM64_S390_COMMON == 1 */ + static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) { return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; @@ -561,6 +586,7 @@ static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) return vcpu_read_sys_reg(vcpu, r) & bit; } +#if ARM64_S390_COMMON == 1 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu, unsigned long data, unsigned int len) @@ -636,6 +662,8 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu) vcpu_set_flag((v), e); \ } while (0) +#endif /* ARM64_S390_COMMON == 1 */ + /* * Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE * format if E2H isn't set. @@ -737,6 +765,12 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) } } +static inline void kvm_reset_fpsimd(struct kvm_vcpu *vcpu) +{ + memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); +} + +#if ARM64_S390_COMMON == 1 /* Reset a vcpu's core registers. */ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) { @@ -750,15 +784,19 @@ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) pstate = VCPU_RESET_PSTATE_EL1; /* Reset core registers */ - memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); - memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); + memset(vcpu_gp_regs(vcpu), 0, sizeof(vcpu_gp_regs(vcpu))); + *vcpu_pc(vcpu) = 0; + *vcpu_sp_el0(vcpu) = 0; + kvm_reset_fpsimd(vcpu); vcpu->arch.ctxt.spsr_abt = 0; vcpu->arch.ctxt.spsr_und = 0; vcpu->arch.ctxt.spsr_irq = 0; vcpu->arch.ctxt.spsr_fiq = 0; - vcpu_gp_regs(vcpu)->pstate = pstate; + *vcpu_cpsr(vcpu) = pstate; } +#endif /* ARM64_S390_COMMON == 1 */ + /* PSCI reset handling for a vcpu. */ static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu, struct vcpu_reset_state *reset_state) diff --git a/arch/arm64/include/asm/kvm_hcall.h b/arch/arm64/include/asm/kvm_hcall.h new file mode 100644 index 000000000000..5f77a8aa2963 --- /dev/null +++ b/arch/arm64/include/asm/kvm_hcall.h @@ -0,0 +1,258 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * The host<->hyp hypercall interface. + * + * Copyright (C) 2026 Google LLC + * Author: Fuad Tabba + */ + +#ifndef __ARM64_KVM_HCALL_H__ +#define __ARM64_KVM_HCALL_H__ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +typedef u16 pkvm_handle_t; + +enum pkvm_topup_id { + PKVM_TOPUP_HYP_ALLOC, + NR_PKVM_TOPUP_HYP_IDS, + PKVM_TOPUP_HYP_ALLOC_SELFTEST, +}; + +struct kvm; +struct kvm_s2_mmu; +struct kvm_vcpu; +struct vgic_v3_cpu_if; +struct vgic_v5_cpu_if; + +/* + * A host VA carried by a hypercall argument. At EL2 such a pointer must not + * be dereferenced until it is translated with kern_hyp_va_host(); sparse + * flags any use that skips the translation. The tag describes the EL2 view + * only: the host dereferences its own VAs freely. + */ +#if defined(__KVM_NVHE_HYPERVISOR__) && defined(__CHECKER__) +#define __kern __attribute__((noderef, address_space(__kern))) +#else +#define __kern +#endif + +/* + * Hypercall signatures are declared as (type, name) argument pairs. + * __KVM_HCALL_MAP() applies a macro to each pair, in the mold of __MAP() + * in . The ladder is indexed by list entries, two per + * argument; __KVM_HCALL_MAP_N() takes that count explicitly. + */ +#define __KVM_HCALL_MAP2(m, t, a, ...) m(t, a) +#define __KVM_HCALL_MAP4(m, t, a, ...) m(t, a), __KVM_HCALL_MAP2(m, __VA_ARGS__) +#define __KVM_HCALL_MAP6(m, t, a, ...) m(t, a), __KVM_HCALL_MAP4(m, __VA_ARGS__) +#define __KVM_HCALL_MAP8(m, t, a, ...) m(t, a), __KVM_HCALL_MAP6(m, __VA_ARGS__) +#define __KVM_HCALL_MAP10(m, t, a, ...) m(t, a), __KVM_HCALL_MAP8(m, __VA_ARGS__) +#define __KVM_HCALL_MAP12(m, t, a, ...) m(t, a), __KVM_HCALL_MAP10(m, __VA_ARGS__) +#define __KVM_HCALL_MAP_N(n, m, ...) CONCATENATE(__KVM_HCALL_MAP, n)(m, __VA_ARGS__) +#define __KVM_HCALL_MAP(m, ...) __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), m, __VA_ARGS__) + +#define __KVM_HCALL_DECL(t, a) t a +#define __KVM_HCALL_LONG(t, a) unsigned long a +#define __KVM_HCALL_CAST(t, a) (__force t) a +#define __KVM_HCALL_ARGS(t, a) a + +#ifndef __KVM_NVHE_HYPERVISOR__ +#define __kvm_call_hyp_nvhe(f, ...) \ + ({ \ + struct arm_smccc_res res; \ + \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \ + ##__VA_ARGS__, &res); \ + if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \ + res.a1 = -EOPNOTSUPP; \ + \ + res.a1; \ + }) + +/* + * Generate a typed stub for each declared hypercall. kvm_call_hyp_nvhe() + * resolves to the stub, so a call with the wrong argument count or types + * fails to compile instead of being silently truncated to an SMCCC function + * number and a pile of registers. The stub inlines to the same SMCCC call + * the untyped macro used to make. + */ +#define DECLARE_KVM_HOST_HCALL(ret, name, ...) \ + static __always_inline \ + ret nvhe_hvc_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) \ + { \ + return (ret)__kvm_call_hyp_nvhe(name, \ + __KVM_HCALL_MAP(__KVM_HCALL_ARGS, __VA_ARGS__));\ + } + +#define DECLARE_KVM_HOST_HCALL0(ret, name) \ + static __always_inline ret nvhe_hvc_##name(void) \ + { \ + return (ret)__kvm_call_hyp_nvhe(name); \ + } + +#define kvm_call_hyp_nvhe(f, ...) nvhe_hvc_##f(__VA_ARGS__) + +/* + * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, + * where the eret to EL1 acts as a context synchronization event. + */ +#define kvm_call_hyp(f, ...) \ + do { \ + if (has_vhe()) { \ + f(__VA_ARGS__); \ + isb(); \ + } else { \ + kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ + } \ + } while (0) + +#define kvm_call_hyp_ret(f, ...) \ + ({ \ + typeof(f(__VA_ARGS__)) ret; \ + \ + if (has_vhe()) { \ + ret = f(__VA_ARGS__); \ + } else { \ + ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ + } \ + \ + ret; \ + }) +#else /* __KVM_NVHE_HYPERVISOR__ */ +#define kvm_call_hyp(f, ...) f(__VA_ARGS__) +#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) +#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) + +/* + * At EL2 each declaration emits the canonical signature of the hypercall, + * which DEFINE_KVM_HOST_HCALL() in hyp-main.c checks the handler + * definition against. + */ +#define DECLARE_KVM_HOST_HCALL(ret, name, ...) \ + typedef ret kvm_host_hcall_sig_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)); +#define DECLARE_KVM_HOST_HCALL0(ret, name) \ + typedef ret kvm_host_hcall_sig_##name(void); +#endif /* __KVM_NVHE_HYPERVISOR__ */ + +/* Hypercalls that are unavailable once pKVM has finalised. */ +DECLARE_KVM_HOST_HCALL(int, __pkvm_init, + phys_addr_t, phys, unsigned long, size, + unsigned long *, per_cpu_base, u32, hyp_va_bits) +DECLARE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping, + phys_addr_t, phys, size_t, size, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, + enum arm64_hyp_spectre_vector, slot) +DECLARE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) +DECLARE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) +DECLARE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest) + +DECLARE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) + +/* Hypercalls that are always available and common to [nh]VHE/pKVM. */ +DECLARE_KVM_HOST_HCALL(void, __kvm_adjust_pc, + struct kvm_vcpu __kern *, vcpu) +DECLARE_KVM_HOST_HCALL(int, __kvm_vcpu_run, + struct kvm_vcpu __kern *, vcpu) +DECLARE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, + struct kvm_s2_mmu __kern *, mmu) +DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages) +DECLARE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, + struct kvm_s2_mmu __kern *, mmu) +DECLARE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, + u64, cntvoff) +DECLARE_KVM_HOST_HCALL(int, __tracing_load, + void __kern *, desc_hva, size_t, desc_size) +DECLARE_KVM_HOST_HCALL0(void, __tracing_unload) +DECLARE_KVM_HOST_HCALL(int, __tracing_enable, + bool, enable) +DECLARE_KVM_HOST_HCALL(int, __tracing_swap_reader, + unsigned int, cpu) +DECLARE_KVM_HOST_HCALL(void, __tracing_update_clock, + u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc) +DECLARE_KVM_HOST_HCALL(int, __tracing_reset, + unsigned int, cpu) +DECLARE_KVM_HOST_HCALL(int, __tracing_enable_event, + unsigned short, id, bool, enable) +DECLARE_KVM_HOST_HCALL(void, __tracing_write_event, + u64, id) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_resident, + struct vgic_v5_cpu_if __kern *, cpu_if); +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident, + struct vgic_v5_cpu_if __kern *, cpu_if); +DECLARE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, + struct vgic_v3_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, + struct vgic_v3_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, + struct vgic_v5_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, + struct vgic_v5_cpu_if __kern *, cpu_if) +DECLARE_KVM_HOST_HCALL(void, __vgic_v5_vdpend, + u32, intid, bool, pending, u16, vm); +DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, + enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, + enum pkvm_topup_id, id, unsigned long, target) +DECLARE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, + enum pkvm_topup_id, id) + +/* Hypercalls that are available only when pKVM has finalised. */ +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, + u64, pfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp, + u64, pfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, + u64, pfn, u64, gfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_guest, + u64, pfn, u64, gfn, u64, nr_pages, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest, + u64, gfn, u64, prot) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold) +DECLARE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, + u64, gfn) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) +DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm, + struct kvm __kern *, host_kvm, void __kern *, pgd_hva) +DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, + pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu) +DECLARE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) +DECLARE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, + phys_addr_t, phys) +DECLARE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page, + pkvm_handle_t, handle, u64, gfn) +DECLARE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, + pkvm_handle_t, handle) +DECLARE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, + pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2) +DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put) +DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) +DECLARE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, + pkvm_handle_t, handle) + +#endif /* __ARM64_KVM_HCALL_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 406c57c99e84..6dd8462145c6 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -27,9 +28,13 @@ #include #include #include +#include #include #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM +#define __KVM_HAVE_VCPU_RUN_PID_CHANGE #define KVM_HALT_POLL_NS_DEFAULT 500000 @@ -39,6 +44,7 @@ #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS +#if ARM64_S390_COMMON == 1 #define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) @@ -55,6 +61,51 @@ #define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9) #define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10) #define KVM_REQ_VGIC_PROCESS_UPDATE KVM_ARCH_REQ(11) +#define KVM_REQ_RELOAD_GICv5 KVM_ARCH_REQ(12) + +/* Bit indices for kvm_arch::flags */ +enum { + /* + * If we encounter a data abort without valid instruction syndrome + * information, report this to user space. User space can (and + * should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is + * supported. + */ + KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, + /* Memory Tagging Extension enabled for the guest */ + KVM_ARCH_FLAG_MTE_ENABLED, + /* At least one vCPU has ran in the VM */ + KVM_ARCH_FLAG_HAS_RAN_ONCE, + /* The vCPU feature set for the VM is configured */ + KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, + /* PSCI SYSTEM_SUSPEND enabled for the guest */ + KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, + /* VM counter offset */ + KVM_ARCH_FLAG_VM_COUNTER_OFFSET, + /* Timer PPIs made immutable */ + KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE, + /* Initial ID reg values loaded */ + KVM_ARCH_FLAG_ID_REGS_INITIALIZED, + /* Fine-Grained UNDEF initialised */ + KVM_ARCH_FLAG_FGU_INITIALIZED, + /* SVE exposed to guest */ + KVM_ARCH_FLAG_GUEST_HAS_SVE, + /* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */ + KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS, + /* Unhandled SEAs are taken to userspace */ + KVM_ARCH_FLAG_EXIT_SEA, + /* No further ID register changes possible */ + KVM_ARCH_FLAG_ID_REGS_FINAL, +}; + +struct vcpu_reset_state { + unsigned long pc; + unsigned long r0; + bool be; + bool reset; +}; + +#endif /* ARM64_S390_COMMON == 1 */ #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -93,9 +144,22 @@ struct kvm_hyp_memcache { struct pkvm_mapping *mapping; /* only used from EL1 */ #define HYP_MEMCACHE_ACCOUNT_STAGE2 BIT(1) +#define HYP_MEMCACHE_ACCOUNT_KMEMCG BIT(2) unsigned long flags; }; +static inline void init_hyp_memcache(struct kvm_hyp_memcache *mc) +{ + memset(mc, 0, sizeof(*mc)); + mc->mapping = ZERO_SIZE_PTR; /* Prevent allocation, solely useful for stage2 memcache */ +} + +static inline void init_hyp_stage2_memcache(struct kvm_hyp_memcache *mc) +{ + memset(mc, 0, sizeof(*mc)); + mc->flags = HYP_MEMCACHE_ACCOUNT_STAGE2 | HYP_MEMCACHE_ACCOUNT_KMEMCG; +} + static inline void push_hyp_memcache(struct kvm_hyp_memcache *mc, phys_addr_t *p, phys_addr_t (*to_pa)(void *virt)) @@ -152,6 +216,36 @@ struct kvm_vmid { atomic64_t id; }; +/* + * Record of a guest stage-2 mapping, storing canonical and nested IPA + * ranges. Both ranges have the same size. The lower bits of nested.start + * store the index of the nested mmu this mapping belongs to. + */ +struct kvm_guest_s2_mapping { + struct interval_tree_node canonical; + struct interval_tree_node nested; +}; + +struct kvm_vcpu_ops { + void (*vcpu_load)(struct kvm_vcpu *vcpu); + void (*vcpu_put)(struct kvm_vcpu *vcpu); +}; + +struct kvm_gfn_range; +struct kvm_s2_fault_desc; + +struct kvm_vm_s2_ops { + bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + bool (*vm_test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range); + int (*vm_flush_remote_tlbs)(struct kvm *kvm); + int (*vm_flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn, + u64 nr_pages); + void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu, + phys_addr_t start, u64 size, + bool may_block); + int (*vm_mem_abort)(const struct kvm_s2_fault_desc *s2fd); +}; + struct kvm_s2_mmu { struct kvm_vmid vmid; @@ -213,22 +307,24 @@ struct kvm_s2_mmu { u64 tlb_vttbr; u64 tlb_vtcr; + /* Guest s2 mapping records indexed in this MMU's IPA space. */ + struct rb_root_cached guest_s2_mappings; + /* * true when this represents a nested context where virtual * HCR_EL2.VM == 1 */ bool nested_stage2_enabled; -#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS - struct dentry *shadow_pt_debugfs_dentry; -#endif - /* * true when this MMU needs to be unmapped before being used for a new * purpose. */ bool pending_unmap; + /* Index in the S2 MMU array, only valid for a shadow S2 */ + u16 s2_mmu_idx; + /* * 0: Nobody is currently using this, check vttbr for validity * >0: Somebody is actively using this. @@ -253,13 +349,9 @@ struct kvm_smccc_features { unsigned long vendor_hyp_bmap_2; /* Function numbers 64-127 */ }; -typedef u16 pkvm_handle_t; - struct kvm_protected_vm { pkvm_handle_t handle; - struct kvm_hyp_memcache teardown_mc; struct kvm_hyp_memcache stage2_teardown_mc; - bool is_protected; bool is_created; /* @@ -308,8 +400,24 @@ enum fgt_group_id { __NR_FGT_GROUP_IDS__ }; +enum kvm_arm_vm_flavor { + VM_NVHE, + VM_VHE, + /* VMs running on a hyp that doesn't trust */ + MARKER(__VM_DISTRUSTING_HYP), + VM_PKVM, /* Normal guests on pKVM */ + MARKER(__VM_PROTECTED), + VM_PROTECTED_PKVM, /* Protected VM */ + VM_FLAVOR_MAX +}; + struct kvm_arch { struct kvm_s2_mmu mmu; + const struct kvm_vm_s2_ops *vm_s2_ops; + + enum kvm_arm_vm_flavor vm_flavor; + /* Mandated version of PSCI */ + u32 psci_version; /* * Fine-Grained UNDEF, mimicking the FGT layout defined by the @@ -328,47 +436,21 @@ struct kvm_arch { size_t nested_mmus_size; int nested_mmus_next; + /* + * Serializes guest s2 tracking trees access when the mmu_lock + * is only held for read. + */ + spinlock_t guest_s2_tracking_lock; + /* Interrupt controller */ struct vgic_dist vgic; /* Timers */ struct arch_timer_vm_data timer_data; - /* Mandated version of PSCI */ - u32 psci_version; - /* Protects VM-scoped configuration data */ struct mutex config_lock; - /* - * If we encounter a data abort without valid instruction syndrome - * information, report this to user space. User space can (and - * should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is - * supported. - */ -#define KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER 0 - /* Memory Tagging Extension enabled for the guest */ -#define KVM_ARCH_FLAG_MTE_ENABLED 1 - /* At least one vCPU has ran in the VM */ -#define KVM_ARCH_FLAG_HAS_RAN_ONCE 2 - /* The vCPU feature set for the VM is configured */ -#define KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED 3 - /* PSCI SYSTEM_SUSPEND enabled for the guest */ -#define KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED 4 - /* VM counter offset */ -#define KVM_ARCH_FLAG_VM_COUNTER_OFFSET 5 - /* Timer PPIs made immutable */ -#define KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE 6 - /* Initial ID reg values loaded */ -#define KVM_ARCH_FLAG_ID_REGS_INITIALIZED 7 - /* Fine-Grained UNDEF initialised */ -#define KVM_ARCH_FLAG_FGU_INITIALIZED 8 - /* SVE exposed to guest */ -#define KVM_ARCH_FLAG_GUEST_HAS_SVE 9 - /* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */ -#define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS 10 - /* Unhandled SEAs are taken to userspace */ -#define KVM_ARCH_FLAG_EXIT_SEA 11 unsigned long flags; /* VM-wide vCPU feature set */ @@ -837,17 +919,11 @@ extern s64 kvm_nvhe_sym(hyp_physvirt_offset); extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS]; #define hyp_cpu_logical_map CHOOSE_NVHE_SYM(hyp_cpu_logical_map) -struct vcpu_reset_state { - unsigned long pc; - unsigned long r0; - bool be; - bool reset; -}; - struct vncr_tlb; struct kvm_vcpu_arch { struct kvm_cpu_context ctxt; + const struct kvm_vcpu_ops *vcpu_ops; /* * Guest floating point state @@ -928,8 +1004,8 @@ struct kvm_vcpu_arch { /* Cache some mmu pages needed inside spinlock regions */ struct kvm_mmu_memory_cache mmu_page_cache; - /* Pages to top-up the pKVM/EL2 guest pool */ - struct kvm_hyp_memcache pkvm_memcache; + /* Pages to top-up the pKVM/EL2 guest stage-2 pool */ + struct kvm_hyp_memcache stage2_mc; /* Virtual SError ESR to restore when HCR_EL2.VSE is set */ u64 vsesr_el2; @@ -953,6 +1029,7 @@ struct kvm_vcpu_arch { pid_t pid; }; +#if ARM64_S390_COMMON == 1 /* * Each 'flag' is composed of a comma-separated triplet: * @@ -1035,11 +1112,28 @@ struct kvm_vcpu_arch { set; \ }) +#define __vcpu_copy_flag(vt, vs, flagset, f, m) \ + do { \ + typeof(vs->arch.flagset) tmp, val; \ + \ + __build_check_flag(vs, flagset, f, m); \ + \ + val = READ_ONCE(vs->arch.flagset); \ + val &= (m); \ + __vcpu_flags_preempt_disable(); \ + tmp = READ_ONCE(vt->arch.flagset); \ + tmp &= ~(m); \ + tmp |= val; \ + WRITE_ONCE(vt->arch.flagset, tmp); \ + __vcpu_flags_preempt_enable(); \ + } while (0) + #define vcpu_get_flag(v, ...) __vcpu_get_flag((v), __VA_ARGS__) #define vcpu_set_flag(v, ...) __vcpu_set_flag((v), __VA_ARGS__) #define vcpu_clear_flag(v, ...) __vcpu_clear_flag((v), __VA_ARGS__) #define vcpu_test_and_clear_flag(v, ...) \ __vcpu_test_and_clear_flag((v), __VA_ARGS__) +#define vcpu_copy_flag(vt, vs, ...) __vcpu_copy_flag((vt), (vs), __VA_ARGS__) /* KVM_ARM_VCPU_INIT completed */ #define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0)) @@ -1048,6 +1142,11 @@ struct kvm_vcpu_arch { /* pKVM VCPU setup completed */ #define VCPU_PKVM_FINALIZED __vcpu_single_flag(cflags, BIT(2)) +int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); +bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); + +#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED) + /* Exception pending */ #define PENDING_EXCEPTION __vcpu_single_flag(iflags, BIT(0)) /* @@ -1057,6 +1156,8 @@ struct kvm_vcpu_arch { #define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1)) /* Target EL/MODE (not a single flag, but let's abuse the macro) */ #define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1)) +/* Cover both PENDING_EXCEPTION and EXCEPT_MASK for global operations */ +#define PC_UPDATE_REQ __vcpu_single_flag(iflags, GENMASK(3, 0)) /* Host-set: the hyp flushes the non-protected vCPU state in on entry */ #define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(4)) @@ -1106,6 +1207,8 @@ struct kvm_vcpu_arch { /* KVM is currently emulating an L2 to L1 exception */ #define IN_NESTED_EXCEPTION __vcpu_single_flag(sflags, BIT(9)) +#endif /* ARM64_S390_COMMON == 1 */ + #define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl) #define vcpu_sve_zcr_elx(vcpu) \ @@ -1151,6 +1254,9 @@ struct kvm_vcpu_arch { #define vcpu_has_ptrauth(vcpu) false #endif +#define kvm_id_regs_final(kvm) \ + test_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &(kvm)->arch.flags) + #define vcpu_on_unsupported_cpu(vcpu) \ vcpu_get_flag(vcpu, ON_UNSUPPORTED_CPU) @@ -1160,7 +1266,7 @@ struct kvm_vcpu_arch { #define vcpu_clear_on_unsupported_cpu(vcpu) \ vcpu_clear_flag(vcpu, ON_UNSUPPORTED_CPU) -#define vcpu_gp_regs(v) (&(v)->arch.ctxt.regs) +#define vcpu_gp_regs(v) ((v)->arch.ctxt.regs.regs) /* * Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the @@ -1241,14 +1347,6 @@ struct kvm_vcpu_stat { u64 exits; }; -unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); -int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); -int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); - -unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); - int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events); @@ -1260,51 +1358,6 @@ void kvm_arm_resume_guest(struct kvm *kvm); #define vcpu_has_run_once(vcpu) (!!READ_ONCE((vcpu)->pid)) -#ifndef __KVM_NVHE_HYPERVISOR__ -#define kvm_call_hyp_nvhe(f, ...) \ - ({ \ - struct arm_smccc_res res; \ - \ - arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \ - ##__VA_ARGS__, &res); \ - if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \ - res.a1 = -EOPNOTSUPP; \ - \ - res.a1; \ - }) - -/* - * The isb() below is there to guarantee the same behaviour on VHE as on !VHE, - * where the eret to EL1 acts as a context synchronization event. - */ -#define kvm_call_hyp(f, ...) \ - do { \ - if (has_vhe()) { \ - f(__VA_ARGS__); \ - isb(); \ - } else { \ - kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ - } \ - } while(0) - -#define kvm_call_hyp_ret(f, ...) \ - ({ \ - typeof(f(__VA_ARGS__)) ret; \ - \ - if (has_vhe()) { \ - ret = f(__VA_ARGS__); \ - } else { \ - ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \ - } \ - \ - ret; \ - }) -#else /* __KVM_NVHE_HYPERVISOR__ */ -#define kvm_call_hyp(f, ...) f(__VA_ARGS__) -#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__) -#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__) -#endif /* __KVM_NVHE_HYPERVISOR__ */ - int handle_exit(struct kvm_vcpu *vcpu, int exception_index); void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index); @@ -1327,6 +1380,15 @@ int __init populate_nv_trap_config(void); void kvm_calculate_traps(struct kvm_vcpu *vcpu); +#if ARM64_S390_COMMON == 1 +unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); +int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); +int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); +int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); + +unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu); +int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); + /* MMIO helpers */ void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data); unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); @@ -1334,6 +1396,8 @@ unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len); int kvm_handle_mmio_return(struct kvm_vcpu *vcpu); int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa); +#endif /* ARM64_S390_COMMON == 1 */ + /* * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event, * arrived in guest context. For arm64, any event that arrives while a vCPU is @@ -1348,7 +1412,7 @@ long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu); gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu); void kvm_update_stolen_time(struct kvm_vcpu *vcpu); -bool kvm_arm_pvtime_supported(void); +bool kvm_arm_pvtime_supported(struct kvm *kvm); int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu, @@ -1506,14 +1570,30 @@ struct kvm *kvm_arch_alloc_vm(void); #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE -#define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.is_protected) +#ifdef __KVM_NVHE_HYPERVISOR__ +#define kvm_vm_is_protected(kvm) \ + (is_protected_kvm_enabled() && ((kvm)->arch.vm_flavor == VM_PROTECTED_PKVM)) +/* + * Accessing vcpu->kvm from nVHE hyp stub is tricky, as we need to convert the + * pointer to the hyp VA. With pKVM, the nVHE code runs with the hyp_vcpu, + * which is populated correctly and is gated on is_protected_kvm_enabled(). + */ +#define vcpu_is_protected(vcpu) \ + ({ \ + struct kvm *__kvm = READ_ONCE((vcpu)->kvm); \ + \ + (__kvm && kvm_vm_is_protected(__kvm)); \ + }) + +#else + +#define kvm_vm_is_protected(kvm) ((kvm)->arch.vm_flavor >= __VM_PROTECTED) #define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm) -int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature); -bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); +#define kvm_vm_hyp_is_distrusting(kvm) ((kvm)->arch.vm_flavor >= __VM_DISTRUSTING_HYP) -#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED) +#endif /* __KVM_NVHE_HYPERVISOR__ */ #define kvm_has_mte(kvm) \ (system_supports_mte() && \ @@ -1531,11 +1611,15 @@ static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature) return test_bit(feature, ka->vcpu_features); } +#if ARM64_S390_COMMON == 1 + #define kvm_vcpu_has_feature(k, f) __vcpu_has_feature(&(k)->arch, (f)) #define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f)) #define kvm_vcpu_initialized(v) vcpu_get_flag(v, VCPU_INITIALIZED) +#endif /* ARM64_S390_COMMON == 1 */ + int kvm_trng_call(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM extern phys_addr_t hyp_mem_base; @@ -1547,6 +1631,7 @@ static inline void kvm_hyp_reserve(void) { } void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu); bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu); +bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu); static inline u64 *__vm_id_reg(struct kvm_arch *ka, u32 reg) { diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index 4974492744cc..b0e42be3d9ee 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -10,6 +10,7 @@ #include #include #include +#include #include DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt); @@ -88,6 +89,8 @@ void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if); int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu); /* GICv5 */ +void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if); +void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_vmcr_apr(struct vgic_v5_cpu_if *cpu_if); /* No hypercalls for the following */ @@ -95,6 +98,7 @@ void __vgic_v5_save_ppi_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_ppi_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_save_state(struct vgic_v5_cpu_if *cpu_if); void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if); +void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm); #ifdef __KVM_NVHE_HYPERVISOR__ void __timer_enable_traps(struct kvm_vcpu *vcpu); @@ -150,6 +154,9 @@ extern u64 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val); extern unsigned long kvm_nvhe_sym(__icache_flags); extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits); +extern enum mitigation_state kvm_nvhe_sym(spectre_v2_state); +extern enum mitigation_state kvm_nvhe_sym(spectre_v4_state); +extern enum mitigation_state kvm_nvhe_sym(spectre_bhb_state); extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl); extern unsigned long kvm_nvhe_sym(hyp_nr_cpus); extern unsigned int kvm_nvhe_sym(hyp_gicv3_nr_lr); diff --git a/arch/arm64/include/asm/kvm_hypevents.h b/arch/arm64/include/asm/kvm_hypevents.h index 5f6e6789d121..3396deb1a1f8 100644 --- a/arch/arm64/include/asm/kvm_hypevents.h +++ b/arch/arm64/include/asm/kvm_hypevents.h @@ -58,4 +58,18 @@ HYP_EVENT(selftest, ), RE_PRINTK("id=%llu", __entry->id) ); + +/* + * trace_hyp_printk() has too many specificities to be declared with HYP_EVENT(). + * However, we can use a REMOTE_EVENT macro to automatically do the declaration + * for the kernel side. + */ +REMOTE_EVENT_CUSTOM_PRINTK(hyp_printk, + 0, /* id will be overwritten during hyp event init */ + RE_STRUCT( + re_field(u16, fmt_id) + re_field(u64, args[]) + ), + __hyp_trace_printk(evt, seq) +); #endif diff --git a/arch/arm64/include/asm/kvm_hyptrace.h b/arch/arm64/include/asm/kvm_hyptrace.h index de133b735f72..16e8d60abba9 100644 --- a/arch/arm64/include/asm/kvm_hyptrace.h +++ b/arch/arm64/include/asm/kvm_hyptrace.h @@ -23,4 +23,13 @@ extern struct remote_event __hyp_events_end[]; extern struct hyp_event_id __hyp_event_ids_start[]; extern struct hyp_event_id __hyp_event_ids_end[]; +#define HYP_STRING_FMT_MAX_SIZE 127 + +struct hyp_string_fmt { + const char fmt[HYP_STRING_FMT_MAX_SIZE]; + u8 nr_args; +}; + +extern const struct hyp_string_fmt __hyp_string_fmts_start[]; +extern const struct hyp_string_fmt __hyp_string_fmts_end[]; #endif diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 6eae7e7e2a68..32392abf2e21 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -7,6 +7,8 @@ #ifndef __ARM64_KVM_MMU_H__ #define __ARM64_KVM_MMU_H__ +#include + #include #include #include @@ -126,6 +128,9 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) * replace the instructions with `nop`s. */ #ifndef __KVM_VHE_HYPERVISOR__ + if (is_protected_kvm_enabled() && !is_ttbr1_addr(v)) + return v; + asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n" /* mask with va_mask */ "ror %0, %0, #1\n" /* rotate to the first tag bit */ "add %0, %0, #0\n" /* insert the low 12 bits of the tag */ @@ -140,6 +145,14 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) #define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v)))) +/* + * Translate a __kern-tagged host VA, dropping the tag: the only sanctioned + * unwrap. Translation only, no ownership or bounds validation; the result + * carries the pointee type stripped of the tag and of any cv-qualifiers. + */ +#define kern_hyp_va_host(v) \ + ((TYPEOF_UNQUAL(*(v)) *)__kern_hyp_va((unsigned long)(__force void *)(v))) + extern u32 __hyp_va_bits; /* @@ -392,12 +405,10 @@ static inline bool kvm_supports_cacheable_pfnmap(void) #ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS void kvm_s2_ptdump_create_debugfs(struct kvm *kvm); -void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu); -void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu); +void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx); #else static inline void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) {} -static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) {} -static inline void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) {} +static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx) {} #endif /* CONFIG_PTDUMP_STAGE2_DEBUGFS */ #endif /* __ASSEMBLER__ */ diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 586026e85903..0f899addafa8 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -78,6 +78,11 @@ extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid, const union tlbi_info *info, void (*)(struct kvm_s2_mmu *, const union tlbi_info *)); +extern void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, + gpa_t nested_ipa, size_t map_size, + struct kvm_guest_s2_mapping *mapping); +extern void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, + gpa_t nipa, size_t size); extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu); extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu); @@ -166,6 +171,8 @@ extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu, struct kvm_s2_trans *trans); extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2); extern void kvm_nested_s2_wp(struct kvm *kvm); +extern void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, + size_t unmap_size, bool may_block); extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block); extern void kvm_nested_s2_flush(struct kvm *kvm); diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index 41a8687938eb..1c9c57cd6ae4 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -7,10 +7,18 @@ #ifndef __ARM64_KVM_PGTABLE_H__ #define __ARM64_KVM_PGTABLE_H__ +#include #include -#include +#include +#include +#include #include +#include +#include + +struct kvm_s2_mmu; + #define KVM_PGTABLE_FIRST_LEVEL -1 #define KVM_PGTABLE_LAST_LEVEL 3 @@ -532,7 +540,7 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size); * kvm_get_vtcr() - Helper to construct VTCR_EL2 * @mmfr0: Sanitized value of SYS_ID_AA64MMFR0_EL1 register. * @mmfr1: Sanitized value of SYS_ID_AA64MMFR1_EL1 register. - * @phys_shfit: Value to set in VTCR_EL2.T0SZ. + * @phys_shift: IPA size, in bits. * * The VTCR value is common across all the physical CPUs on the system. * We use system wide sanitised values to fill in different fields, @@ -859,6 +867,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, * @addr: Input address for the start of the walk. * @ptep: Pointer to storage for the retrieved PTE. * @level: Pointer to storage for the level of the retrieved PTE. + * @flags: Flags to control the page-table walk + * (see struct kvm_pgtable_visit_ctx). * * The offset of @addr within a page is ignored. * @@ -869,7 +879,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, * Return: 0 on success, negative error code on failure. */ int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, - kvm_pte_t *ptep, s8 *level); + kvm_pte_t *ptep, s8 *level, + enum kvm_pgtable_walk_flags flags); /** * kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index cad60569f061..e7562770e7cb 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -7,9 +7,9 @@ #define __ARM64_KVM_PKVM_H__ #include +#include #include #include -#include #include /* Maximum number of VMs that can co-exist under pKVM. */ @@ -17,7 +17,8 @@ #define HYP_MEMBLOCK_REGIONS 128 -int pkvm_init_host_vm(struct kvm *kvm, unsigned long type); +void pkvm_selftests(void); +int pkvm_init_host_vm(struct kvm *kvm); int pkvm_create_hyp_vm(struct kvm *kvm); bool pkvm_hyp_vm_is_created(struct kvm *kvm); void pkvm_destroy_hyp_vm(struct kvm *kvm); @@ -42,17 +43,50 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) case KVM_CAP_ARM_VM_IPA_SIZE: case KVM_CAP_ARM_PTRAUTH_ADDRESS: case KVM_CAP_ARM_PTRAUTH_GENERIC: + case KVM_CAP_ONE_REG: + case KVM_CAP_MP_STATE: + case KVM_CAP_VCPU_EVENTS: + case KVM_CAP_VCPU_ATTRIBUTES: + case KVM_CAP_IMMEDIATE_EXIT: + case KVM_CAP_IOEVENTFD: + case KVM_CAP_IRQFD_RESAMPLE: + case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2: + case KVM_CAP_ARM_INJECT_SERROR_ESR: return true; case KVM_CAP_ARM_MTE: - return false; case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: + case KVM_CAP_PRE_FAULT_MEMORY: return false; default: - return !kvm || !kvm_vm_is_protected(kvm); + return !kvm || (kvm->arch.vm_flavor == VM_PKVM); } } +/* + * The vCPU features a protected VM may use: checked by the host at + * KVM_ARM_VCPU_INIT, applied by EL2 when the hyp VM is created. + */ +static inline void kvm_pkvm_vcpu_allowed_features(struct kvm *kvm, + unsigned long *allowed) +{ + bitmap_zero(allowed, KVM_VCPU_MAX_FEATURES); + + set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3)) + set_bit(KVM_ARM_VCPU_PMU_V3, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS)) + set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) + set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed); + + if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) + set_bit(KVM_ARM_VCPU_SVE, allowed); +} + /* * Check whether the KVM VM IOCTL is allowed in pKVM. * @@ -204,6 +238,116 @@ struct pkvm_mapping { u64 __subtree_last; /* Internal member for interval tree */ }; +enum pkvm_hyp_req_type { + PKVM_HYP_NO_REQ = 0, + PKVM_HYP_REQ_HYP_ALLOC, + PKVM_HYP_REQ_HYP_ALLOC_SELFTEST, + __PKVM_HYP_REQ_TYPE_MAX, +}; + +#define PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX \ + (sizeof(struct arm_smccc_res) - offsetof(struct arm_smccc_res, a2) - 1) + +struct pkvm_hyp_req { + u8 type; + union { + struct { + u32 nr_pages; + } mem; + struct { + /* Helper for SMCCC encoding/decoding */ + u8 args[PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX]; + } args; + }; +}; + +static inline size_t pkvm_hyp_req_arg_size(u8 type) +{ + struct pkvm_hyp_req *req; + + switch (type) { + case PKVM_HYP_NO_REQ: + return 0; + case PKVM_HYP_REQ_HYP_ALLOC: + case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST: + return sizeof(req->mem); + default: + WARN_ON(1); + } + + return 0; +} + +/* Encode the pending pkvm_hyp_req type into the SMCCC args */ +static inline void +pkvm_hyp_req_to_smccc(struct kvm_cpu_context *host_ctxt, struct pkvm_hyp_req *req) +{ + u8 *dst, type = req->type; + size_t size; + + if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) { + host_ctxt->regs.regs[2] = 0; + return; + } + + size = pkvm_hyp_req_arg_size(type); + if (WARN_ON(size > PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX)) + return; + + dst = (u8 *)&host_ctxt->regs.regs[2]; + *dst = type; + + memcpy(dst + 1, &req->args, size); +} + +/* Return true if a pkvm_hyp_req has been decoded from the SMCCC args */ +static inline bool smccc_to_pkvm_hyp_req(struct pkvm_hyp_req *req, struct arm_smccc_res *res) +{ + u8 *src = (u8 *)res + offsetof(struct arm_smccc_res, a2); + u8 type = *src; + + if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) + return false; + + req->type = type; + memcpy(&req->args, src + 1, pkvm_hyp_req_arg_size(type)); + + return true; +} + +int __pkvm_handle_smccc_req(struct arm_smccc_res *res); + +/** + * pkvm_call_hyp_req() - Issue an HVC that can return hypervisor requests + * @f: Hypervisor function symbol to call. + * @...: Arguments to pass to the hypercall. + * + * Re-issue an HVC and process any pending hypervisor request until completion + * or error. + * + * Only use this helper for HVCs whose hypervisor handlers format their return + * registers with pkvm_hyp_req_to_smccc(). + * + * Return: Result of the hypercall or a negative error if the hyp request + * handling failed. + */ +#define pkvm_call_hyp_req(f, ...) \ +({ \ + struct arm_smccc_res __res; \ + int __ret; \ + do { \ + __ret = -1; \ + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), ##__VA_ARGS__, &__res); \ + if (WARN_ON(__res.a0 != SMCCC_RET_SUCCESS)) \ + break; \ + __ret = __res.a1; \ + if (!__ret) \ + break; \ + __ret = __pkvm_handle_smccc_req(&__res); \ + } while (!__ret); \ + __ret; \ +}) + int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, struct kvm_pgtable_mm_ops *mm_ops); void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt, diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index bd291bd6cfac..ad0ad30b36cd 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -26,6 +26,8 @@ #define GIC_PRIO_PSR_I_SET GICV3_PRIO_PSR_I_SET +#if ARM64_S390_COMMON == 1 + /* Additional SPSR bits not exposed in the UABI */ #define PSR_MODE_THREAD_BIT (1 << 0) #define PSR_IL_BIT (1 << 20) @@ -66,6 +68,8 @@ #define PSR_AA32_IT_MASK 0x0600fc00 /* If-Then execution state mask */ #define PSR_AA32_GE_MASK 0x000f0000 +#endif /* ARM64_S390_COMMON == 1 */ + /* AArch32 CPSR bits, as seen in AArch32 */ #define COMPAT_PSR_DIT_BIT 0x00200000 @@ -96,26 +100,48 @@ #define COMPAT_USER_SZ 296 /* Architecturally defined mapping between AArch32 and AArch64 registers */ +enum { + __compat_fp = 11, + __compat_sp = 13, + __compat_lr = 14, + __compat_sp_hyp = 15, + __compat_lr_irq = 16, + __compat_sp_irq = 17, + __compat_lr_svc = 18, + __compat_sp_svc = 19, + __compat_lr_abt = 20, + __compat_sp_abt = 21, + __compat_lr_und = 22, + __compat_sp_und = 23, + __compat_r8_fiq = 24, + __compat_r9_fiq = 25, + __compat_r10_fiq = 26, + __compat_r11_fiq = 27, + __compat_r12_fiq = 28, + __compat_sp_fiq = 29, + __compat_lr_fiq = 30, +}; + #define compat_usr(x) regs[(x)] -#define compat_fp regs[11] -#define compat_sp regs[13] -#define compat_lr regs[14] -#define compat_sp_hyp regs[15] -#define compat_lr_irq regs[16] -#define compat_sp_irq regs[17] -#define compat_lr_svc regs[18] -#define compat_sp_svc regs[19] -#define compat_lr_abt regs[20] -#define compat_sp_abt regs[21] -#define compat_lr_und regs[22] -#define compat_sp_und regs[23] -#define compat_r8_fiq regs[24] -#define compat_r9_fiq regs[25] -#define compat_r10_fiq regs[26] -#define compat_r11_fiq regs[27] -#define compat_r12_fiq regs[28] -#define compat_sp_fiq regs[29] -#define compat_lr_fiq regs[30] +#define compat_fp regs[__compat_fp] +#define compat_sp regs[__compat_sp] +#define compat_lr regs[__compat_lr] +#define compat_sp_hyp regs[__compat_sp_hyp] +#define compat_lr_irq regs[__compat_lr_irq] +#define compat_sp_irq regs[__compat_sp_irq] +#define compat_lr_svc regs[__compat_lr_svc] +#define compat_sp_svc regs[__compat_sp_svc] +#define compat_lr_abt regs[__compat_lr_abt] +#define compat_sp_abt regs[__compat_sp_abt] +#define compat_lr_und regs[__compat_lr_und] +#define compat_sp_und regs[__compat_sp_und] +#define compat_r8_fiq regs[__compat_r8_fiq] +#define compat_r9_fiq regs[__compat_r9_fiq] +#define compat_r10_fiq regs[__compat_r10_fiq] +#define compat_r11_fiq regs[__compat_r11_fiq] +#define compat_r12_fiq regs[__compat_r12_fiq] +#define compat_sp_fiq regs[__compat_sp_fiq] +#define compat_lr_fiq regs[__compat_lr_fiq] static inline unsigned long compat_psr_to_pstate(const unsigned long psr) { diff --git a/arch/arm64/include/asm/stacktrace/nvhe.h b/arch/arm64/include/asm/stacktrace/nvhe.h index 171f9edef49f..a631a577cbe5 100644 --- a/arch/arm64/include/asm/stacktrace/nvhe.h +++ b/arch/arm64/include/asm/stacktrace/nvhe.h @@ -37,6 +37,14 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state, state->pc = pc; } +DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); +DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info); + +#ifdef CONFIG_PKVM_STACKTRACE +DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], + pkvm_stacktrace); +#endif + #ifndef __KVM_NVHE_HYPERVISOR__ /* * Conventional (non-protected) nVHE HYP stack unwinder @@ -45,8 +53,6 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state, * (by the host in EL1). */ -DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); -DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info); DECLARE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base); void kvm_nvhe_dump_backtrace(unsigned long hyp_offset); diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index cec36606f6a4..e3b60ee03023 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -16,6 +16,8 @@ #include +#if ARM64_S390_COMMON == 1 + /* * ARMv8 ARM reserves the following encoding for system registers: * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview", @@ -50,6 +52,8 @@ #define sys_reg_CRm(id) (((id) >> CRm_shift) & CRm_mask) #define sys_reg_Op2(id) (((id) >> Op2_shift) & Op2_mask) +#endif /* ARM64_S390_COMMON == 1 */ + #ifndef CONFIG_BROKEN_GAS_INST #ifdef __ASSEMBLER__ @@ -114,6 +118,8 @@ #define GSB_SYS_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 0, 31) #define GSB_ACK_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 1, 31) +#if ARM64_S390_COMMON == 1 + /* Data cache zero operations */ #define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2) #define SYS_DC_IGSW sys_insn(1, 0, 7, 6, 4) @@ -286,7 +292,6 @@ #define SYS_APGAKEYLO_EL1 sys_reg(3, 0, 2, 3, 0) #define SYS_APGAKEYHI_EL1 sys_reg(3, 0, 2, 3, 1) -#define SYS_SPSR_EL1 sys_reg(3, 0, 4, 0, 0) #define SYS_ELR_EL1 sys_reg(3, 0, 4, 0, 1) #define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0) @@ -509,7 +514,6 @@ #define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0) #define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6) -#define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) #define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0) #define SYS_SPSR_irq sys_reg(3, 4, 4, 3, 0) @@ -595,7 +599,6 @@ #define SYS_BRBCR_EL12 sys_reg(2, 5, 9, 0, 0) #define SYS_TTBR0_EL12 sys_reg(3, 5, 2, 0, 0) #define SYS_TTBR1_EL12 sys_reg(3, 5, 2, 0, 1) -#define SYS_SPSR_EL12 sys_reg(3, 5, 4, 0, 0) #define SYS_ELR_EL12 sys_reg(3, 5, 4, 0, 1) #define SYS_AFSR0_EL12 sys_reg(3, 5, 5, 1, 0) #define SYS_AFSR1_EL12 sys_reg(3, 5, 5, 1, 1) @@ -826,6 +829,8 @@ #define SCTLR_ELx_A (BIT(1)) #define SCTLR_ELx_M (BIT(0)) +#endif /* ARM64_S390_COMMON == 1 */ + #define INIT_SCTLR_EL2_MMU_ON \ (SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_SA | SCTLR_ELx_I | \ SCTLR_ELx_IESB | SCTLR_ELx_WXN | SCTLR_ELx_ITFSB | \ @@ -847,6 +852,7 @@ SCTLR_EL1_nTLSMD | SCTLR_EL1_EIS | SCTLR_EL1_TSCXT | \ SCTLR_EL1_EOS) +#if ARM64_S390_COMMON == 1 /* MAIR_ELx memory attributes (used by Linux) */ #define MAIR_ATTR_DEVICE_nGnRnE UL(0x00) #define MAIR_ATTR_DEVICE_nGnRE UL(0x04) @@ -1018,7 +1024,7 @@ #define GCS_CAP(x) ((((unsigned long)x) & GCS_CAP_ADDR_MASK) | \ GCS_CAP_VALID_TOKEN) /* - * Definitions for GICv5 instructions + * Definitions for GICv5 instructions for the Current Domain */ #define GICV5_OP_GIC_CDAFF sys_insn(1, 0, 12, 1, 3) #define GICV5_OP_GIC_CDDI sys_insn(1, 0, 12, 2, 0) @@ -1083,6 +1089,19 @@ #define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29) #define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0) +/* + * Definitions for GICv5 instructions for the Virtual Domain + */ +#define GICV5_OP_GIC_VDPEND sys_insn(1, 4, 12, 1, 4) + +/* Shift and mask definitions for GIC VDPEND */ +#define GICV5_GIC_VDPEND_PENDING_MASK BIT_ULL(63) +#define GICV5_GIC_VDPEND_VM_MASK GENMASK_ULL(47, 32) +#define GICV5_GIC_VDPEND_TYPE_MASK GENMASK_ULL(31, 29) +#define GICV5_GIC_VDPEND_ID_MASK GENMASK_ULL(23, 0) + +#endif /* ARM64_S390_COMMON == 1 */ + #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) #define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn) @@ -1232,6 +1251,8 @@ par; \ }) +#if ARM64_S390_COMMON == 1 + #define SYS_FIELD_VALUE(reg, field, val) reg##_##field##_##val #define SYS_FIELD_GET(reg, field, val) \ @@ -1244,6 +1265,8 @@ FIELD_PREP(reg##_##field##_MASK, \ SYS_FIELD_VALUE(reg, field, val)) -#endif +#endif /* ARM64_S390_COMMON == 1 */ + +#endif /* __ASSEMBLER__ */ #endif /* __ASM_SYSREG_H */ diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 019e5e3d892e..65ee09bd1463 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -97,6 +97,11 @@ struct kvm_regs { #define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) #define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K) +/* Supported VGICv5 address types */ +#define KVM_VGIC_V5_ADDR_TYPE_IRS 6 + +#define KVM_VGIC_V5_IRS_SIZE (2 * SZ_64K) + #define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ #define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ #define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */ @@ -418,6 +423,16 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 +#define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10 +#define KVM_DEV_ARM_VGIC_GRP_IST 11 + +struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; +}; + #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT) diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index facda8137d9b..19d602e45523 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -142,6 +142,7 @@ KVM_NVHE_ALIAS(__hyp_rodata_end); #ifdef CONFIG_NVHE_EL2_TRACING KVM_NVHE_ALIAS(__hyp_event_ids_start); KVM_NVHE_ALIAS(__hyp_event_ids_end); +KVM_NVHE_ALIAS(__hyp_string_fmts_start); #endif /* pKVM static key */ diff --git a/arch/arm64/kernel/pi/Makefile b/arch/arm64/kernel/pi/Makefile index 4a0cbedfdcd6..a273ab5af805 100644 --- a/arch/arm64/kernel/pi/Makefile +++ b/arch/arm64/kernel/pi/Makefile @@ -34,6 +34,7 @@ $(obj)/lib-%.pi.o: OBJCOPYFLAGS += --prefix-alloc-sections=.init $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) obj-y := idreg-override.pi.o \ map_kernel.pi.o map_range.pi.o \ diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index af1d72020976..1869b44ae190 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -15,7 +15,7 @@ #ifdef CONFIG_NVHE_EL2_TRACING #define HYPERVISOR_EVENT_IDS \ - . = ALIGN(PAGE_SIZE); \ + . = ALIGN(SZ_8); \ __hyp_event_ids_start = .; \ *(HYP_SECTION_NAME(.event_ids)) \ __hyp_event_ids_end = .; @@ -214,6 +214,14 @@ SECTIONS HYPERVISOR_RODATA_SECTIONS +#ifdef CONFIG_NVHE_EL2_TRACING + .rodata.hyp_string_fmts : { + __hyp_string_fmts_start = .; + *(_hyp_string_fmts) + __hyp_string_fmts_end = .; + } +#endif + .got : { *(.got) } /* * Make sure that the .got.plt is either completely empty or it diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 449154f9a485..48d531e302d3 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -22,21 +22,18 @@ menuconfig KVM select KVM_COMMON select KVM_GENERIC_HARDWARE_ENABLING select HAVE_KVM_CPU_RELAX_INTERCEPT - select KVM_MMIO select KVM_GENERIC_DIRTYLOG_READ_PROTECT select VIRT_XFER_TO_GUEST_WORK select KVM_VFIO select HAVE_KVM_DIRTY_RING_ACQ_REL select NEED_KVM_DIRTY_RING_WITH_BITMAP - select HAVE_KVM_MSI select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_IRQ_BYPASS - select HAVE_KVM_READONLY_MEM - select HAVE_KVM_VCPU_RUN_PID_CHANGE select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS select KVM_GUEST_MEMFD + select KVM_GENERIC_PRE_FAULT_MEMORY help Support hosting virtualized guest machines. diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 59612d2f277c..7aaeeb84e788 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -24,7 +24,8 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \ vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \ - vgic/vgic-v5.o + vgic/vgic-v5.o vgic/vgic-v5-tables.o vgic/vgic-irs-v5.o \ + vgic-sys-reg-v5.o kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 6ac3321f4c57..b2c5ba499aa5 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -528,17 +528,11 @@ static void timer_save_state(struct arch_timer_context *ctx) goto out; switch (index) { - u64 cval; - case TIMER_VTIMER: case TIMER_HVTIMER: timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL)); - cval = read_sysreg_el0(SYS_CNTV_CVAL); - - if (has_broken_cntvoff()) - cval -= timer_get_offset(ctx); - - timer_set_cval(ctx, cval); + if (!has_broken_cntvoff() || !timer_get_offset(ctx)) + timer_set_cval(ctx, read_sysreg_el0(SYS_CNTV_CVAL)); /* Disable the timer */ write_sysreg_el0(0, SYS_CNTV_CTL); @@ -564,11 +558,12 @@ static void timer_save_state(struct arch_timer_context *ctx) case TIMER_PTIMER: case TIMER_HPTIMER: timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL)); - cval = read_sysreg_el0(SYS_CNTP_CVAL); - - cval -= timer_get_offset(ctx); - - timer_set_cval(ctx, cval); + /* + * With an offset, memory already holds the guest's CVAL (the + * trap handler or __deactivate_traps() wrote it). + */ + if (!timer_get_offset(ctx)) + timer_set_cval(ctx, read_sysreg_el0(SYS_CNTP_CVAL)); /* Disable the timer */ write_sysreg_el0(0, SYS_CNTP_CTL); @@ -647,7 +642,8 @@ static void timer_restore_state(struct arch_timer_context *ctx) offset = timer_get_offset(ctx); if (has_broken_cntvoff()) { set_cntvoff(0); - cval += offset; + if (offset) + cval = timer_apply_offset(cval, offset, kvm_phys_timer_read()); } else { set_cntvoff(offset); } @@ -660,7 +656,8 @@ static void timer_restore_state(struct arch_timer_context *ctx) cval = timer_get_cval(ctx); offset = timer_get_offset(ctx); set_cntpoff(offset); - cval += offset; + if (offset) + cval = timer_apply_offset(cval, offset, kvm_phys_timer_read()); write_sysreg_el0(cval, SYS_CNTP_CVAL); isb(); write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL); @@ -1110,8 +1107,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu) timer_context_init(vcpu, i); /* Synchronize offsets across timers of a VM if not already provided */ - if (!vcpu_is_protected(vcpu) && - !test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { + if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) { timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read()); timer_set_offset(vcpu_ptimer(vcpu), 0); } @@ -1133,6 +1129,15 @@ void kvm_timer_init_vm(struct kvm *kvm) */ for (int i = 0; i < NR_KVM_TIMERS; i++) kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]); + + /* + * Protected VMs don't allow the userspace to set counter offsets, + * either set via counter register writes or the dedicated ioctls. + * Pretend the offset has already been set and rely on the default + * offset being 0. + */ + if (kvm_vm_is_protected(kvm)) + set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags); } void kvm_timer_cpu_up(void) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index f2478b561459..d30378a50c53 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -93,6 +93,7 @@ static const struct kvm_ioctl_cap_map vm_ioctl_caps[] = { { KVM_ARM_PREFERRED_TARGET, KVM_CAP_ARM_BASIC }, }; +static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu); /* * Set *ext to the capability. * Return 0 if found, or -EINVAL if no IOCTL matches. @@ -214,6 +215,26 @@ static int kvm_arm_default_max_vcpus(void) return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; } +static int kvm_init_vm_flavor(struct kvm *kvm, unsigned long type) +{ + bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; + + if (is_protected_kvm_enabled()) { + if (protected) + kvm->arch.vm_flavor = VM_PROTECTED_PKVM; + else + kvm->arch.vm_flavor = VM_PKVM; + } else if (protected) { + return -EINVAL; + } else if (has_vhe()) { + kvm->arch.vm_flavor = VM_VHE; + } else { + kvm->arch.vm_flavor = VM_NVHE; + } + + return 0; +} + /** * kvm_arch_init_vm - initializes a VM data structure * @kvm: pointer to the KVM struct @@ -236,6 +257,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) mutex_unlock(&kvm->lock); #endif + ret = kvm_init_vm_flavor(kvm, type); + if (ret) + return ret; + ret = kvm_share_hyp(kvm, kvm + 1); if (ret) return ret; @@ -259,12 +284,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) * If any failures occur after this is successful, make sure to * call __pkvm_unreserve_vm to unreserve the VM in hyp. */ - ret = pkvm_init_host_vm(kvm, type); + ret = pkvm_init_host_vm(kvm); if (ret) goto err_uninit_mmu; - } else if (type & KVM_VM_TYPE_ARM_PROTECTED) { - ret = -EINVAL; - goto err_uninit_mmu; } kvm_vgic_early_init(kvm); @@ -410,6 +432,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_COUNTER_OFFSET: case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS: case KVM_CAP_ARM_SEA_TO_USER: + case KVM_CAP_PRE_FAULT_MEMORY: r = 1; break; case KVM_CAP_SET_GUEST_DEBUG2: @@ -451,7 +474,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = system_supports_mte(); break; case KVM_CAP_STEAL_TIME: - r = kvm_arm_pvtime_supported(); + r = kvm_arm_pvtime_supported(kvm); break; case KVM_CAP_ARM_EL1_32BIT: r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1); @@ -552,6 +575,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) mutex_unlock(&vcpu->mutex); #endif + kvm_init_vcpu_ops(vcpu); + /* Force users to call KVM_ARM_VCPU_INIT */ vcpu_clear_flag(vcpu, VCPU_INITIALIZED); @@ -595,7 +620,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) if (!is_protected_kvm_enabled()) kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); else - free_hyp_memcache(&vcpu->arch.pkvm_memcache); + free_hyp_memcache(&vcpu->arch.stage2_mc); kvm_timer_vcpu_terminate(vcpu); kvm_pmu_vcpu_destroy(vcpu); kvm_vgic_vcpu_destroy(vcpu); @@ -614,33 +639,34 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu) { - if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) { - /* - * Either we're running an L2 guest, and the API/APK bits come - * from L1's HCR_EL2, or API/APK are both set. - */ - if (unlikely(is_nested_ctxt(vcpu))) { - u64 val; + if (!vcpu_has_ptrauth(vcpu)) + return; - val = __vcpu_sys_reg(vcpu, HCR_EL2); - val &= (HCR_API | HCR_APK); - vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); - vcpu->arch.hcr_el2 |= val; - } else { - vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); - } + /* + * Either we're running an L2 guest, and the API/APK bits come + * from L1's HCR_EL2, or API/APK are both set. + */ + if (unlikely(is_nested_ctxt(vcpu))) { + u64 val; - /* - * Save the host keys if there is any chance for the guest - * to use pauth, as the entry code will reload the guest - * keys in that case. - */ - if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { - struct kvm_cpu_context *ctxt; + val = __vcpu_sys_reg(vcpu, HCR_EL2); + val &= (HCR_API | HCR_APK); + vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); + vcpu->arch.hcr_el2 |= val; + } else { + vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK); + } - ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); - ptrauth_save_keys(ctxt); - } + /* + * Save the host keys if there is any chance for the guest + * to use pauth, as the entry code will reload the guest + * keys in that case. + */ + if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) { + struct kvm_cpu_context *ctxt; + + ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt); + ptrauth_save_keys(ctxt); } } @@ -666,14 +692,11 @@ static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu) return single_task_running(); } -void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +static void vcpu_prepare_mmu(struct kvm_vcpu *vcpu) { struct kvm_s2_mmu *mmu; int *last_ran; - if (is_protected_kvm_enabled()) - goto nommu; - if (vcpu_has_nv(vcpu)) kvm_vcpu_load_hw_mmu(vcpu); @@ -703,25 +726,15 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) kvm_call_hyp(__kvm_flush_cpu_context, mmu); *last_ran = vcpu->vcpu_idx; } +} -nommu: - vcpu->cpu = cpu; - - /* - * The timer must be loaded before the vgic to correctly set up physical - * interrupt deactivation in nested state (e.g. timer interrupt). - */ - kvm_timer_vcpu_load(vcpu); - kvm_vgic_load(vcpu); - kvm_vcpu_load_debug(vcpu); - kvm_vcpu_load_fgt(vcpu); - if (has_vhe()) - kvm_vcpu_load_vhe(vcpu); - kvm_arch_vcpu_load_fp(vcpu); - kvm_vcpu_pmu_restore_guest(vcpu); - if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) - kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +static void vcpu_put_mmu(struct kvm_vcpu *vcpu) +{ + kvm_arm_vmid_clear_active(); +} +static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu) +{ if (kvm_vcpu_should_clear_twe(vcpu)) vcpu->arch.hcr_el2 &= ~HCR_TWE; else @@ -731,16 +744,73 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.hcr_el2 &= ~HCR_TWI; else vcpu->arch.hcr_el2 |= HCR_TWI; +} +static void vcpu_load_pvtime(struct kvm_vcpu *vcpu) +{ + if (kvm_arm_is_pvtime_enabled(&vcpu->arch)) + kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu); +} + +static void vhe_vcpu_load(struct kvm_vcpu *vcpu) +{ + vcpu_prepare_mmu(vcpu); + /* + * For VHE, the timer must be loaded before the vgic to correctly + * set up physical interrupt deactivation in nested state (e.g. timer + * interrupt). + */ + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + kvm_vcpu_load_vhe(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); vcpu_set_pauth_traps(vcpu); +} - if (is_protected_kvm_enabled()) { - kvm_call_hyp_nvhe(__pkvm_vcpu_load, - vcpu->kvm->arch.pkvm.handle, - vcpu->vcpu_idx, vcpu->arch.hcr_el2); - kvm_call_hyp(__vgic_v3_restore_vmcr_aprs, - &vcpu->arch.vgic_cpu.vgic_v3); - } +static void nvhe_vcpu_load(struct kvm_vcpu *vcpu) +{ + vcpu_prepare_mmu(vcpu); + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); + vcpu_set_pauth_traps(vcpu); +} + +static void pkvm_vcpu_load(struct kvm_vcpu *vcpu) +{ + kvm_timer_vcpu_load(vcpu); + kvm_vgic_load(vcpu); + kvm_vcpu_load_debug(vcpu); + kvm_vcpu_load_fgt(vcpu); + kvm_arch_vcpu_load_fp(vcpu); + kvm_vcpu_pmu_restore_guest(vcpu); + + vcpu_load_pvtime(vcpu); + vcpu_set_wfx_traps(vcpu); + + kvm_call_hyp_nvhe(__pkvm_vcpu_load, + vcpu->kvm->arch.pkvm.handle, + vcpu->vcpu_idx, vcpu->arch.hcr_el2); + kvm_call_hyp_nvhe(__vgic_v3_restore_vmcr_aprs, + &vcpu->arch.vgic_cpu.vgic_v3); +} + +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + vcpu->cpu = cpu; + vcpu->arch.vcpu_ops->vcpu_load(vcpu); if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus)) vcpu_set_on_unsupported_cpu(vcpu); @@ -748,49 +818,76 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vcpu->arch.pid = pid_nr(vcpu->pid); } -void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +static void vhe_vcpu_put(struct kvm_vcpu *vcpu) { - if (is_protected_kvm_enabled()) { - kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); - kvm_call_hyp_nvhe(__pkvm_vcpu_put); - - /* __pkvm_vcpu_put implies a sync of the state */ - if (!kvm_vm_is_protected(vcpu->kvm)) - vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); - } - kvm_vcpu_put_debug(vcpu); kvm_arch_vcpu_put_fp(vcpu); - if (has_vhe()) - kvm_vcpu_put_vhe(vcpu); + kvm_vcpu_put_vhe(vcpu); kvm_timer_vcpu_put(vcpu); kvm_vgic_put(vcpu); kvm_vcpu_pmu_restore_host(vcpu); + if (vcpu_has_nv(vcpu)) kvm_vcpu_put_hw_mmu(vcpu); - kvm_arm_vmid_clear_active(); + vcpu_put_mmu(vcpu); +} + +static void nvhe_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_vcpu_put_debug(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + kvm_vcpu_pmu_restore_host(vcpu); + vcpu_put_mmu(vcpu); +} + +static void pkvm_vcpu_put(struct kvm_vcpu *vcpu) +{ + kvm_call_hyp_nvhe(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); + kvm_call_hyp_nvhe(__pkvm_vcpu_put); + + /* __pkvm_vcpu_put implies a sync of the state */ + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); + + kvm_vcpu_put_debug(vcpu); + kvm_arch_vcpu_put_fp(vcpu); + kvm_timer_vcpu_put(vcpu); + kvm_vgic_put(vcpu); + kvm_vcpu_pmu_restore_host(vcpu); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + vcpu->arch.vcpu_ops->vcpu_put(vcpu); vcpu_clear_on_unsupported_cpu(vcpu); vcpu->cpu = -1; } -static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) +static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu, u32 mp_state) { - WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + WRITE_ONCE(vcpu->arch.mp_state.mp_state, mp_state); kvm_make_request(KVM_REQ_SLEEP, vcpu); kvm_vcpu_kick(vcpu); } +/* The guest's own CPU_OFF: EL2 has already powered the vCPU off. */ void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) { + u32 mp_state = vcpu_is_protected(vcpu) ? KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_STOPPED; + spin_lock(&vcpu->arch.mp_state_lock); - __kvm_arm_vcpu_power_off(vcpu); + __kvm_arm_vcpu_power_off(vcpu, mp_state); spin_unlock(&vcpu->arch.mp_state_lock); } bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu) { - return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED; + u32 mp_state = READ_ONCE(vcpu->arch.mp_state.mp_state); + + return mp_state == KVM_MP_STATE_STOPPED || mp_state == KVM_MP_STATE_UNINITIALIZED; } static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu) @@ -805,6 +902,12 @@ static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu) return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED; } +/* Only the guest's CPU_ON may start a vCPU EL2 holds powered off. */ +bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu) +{ + return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_UNINITIALIZED; +} + int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -822,12 +925,24 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, switch (mp_state->mp_state) { case KVM_MP_STATE_RUNNABLE: + if (kvm_pkvm_vcpu_is_powered_off(vcpu)) { + ret = -EPERM; + break; + } + WRITE_ONCE(vcpu->arch.mp_state, *mp_state); break; case KVM_MP_STATE_STOPPED: - __kvm_arm_vcpu_power_off(vcpu); + /* Keep EL2's off record: STOPPED over it would let RUNNABLE through. */ + if (!kvm_pkvm_vcpu_is_powered_off(vcpu)) + __kvm_arm_vcpu_power_off(vcpu, KVM_MP_STATE_STOPPED); break; case KVM_MP_STATE_SUSPENDED: + if (kvm_pkvm_vcpu_is_powered_off(vcpu)) { + ret = -EPERM; + break; + } + kvm_arm_vcpu_suspend(vcpu); break; default: @@ -953,7 +1068,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; } - ret = kvm_finalize_sys_regs(vcpu); + ret = kvm_vcpu_finalize_sys_regs(vcpu); if (ret) return ret; @@ -983,13 +1098,13 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; } - ret = vgic_v5_finalize_ppi_state(kvm); + ret = vgic_v5_finalize_ppi_state(vcpu); if (ret) return ret; if (is_protected_kvm_enabled()) { /* Start with the vcpu in a dirty state */ - if (!kvm_vm_is_protected(vcpu->kvm)) + if (vcpu->kvm->arch.vm_flavor == VM_PKVM) vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); ret = pkvm_create_hyp_vm(kvm); if (ret) @@ -1163,6 +1278,14 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) preempt_enable(); } + if (kvm_check_request(KVM_REQ_RELOAD_GICv5, vcpu)) { + /* The IRS enable bit was changed */ + preempt_disable(); + vgic_v5_put(vcpu); + vgic_v5_load(vcpu); + preempt_enable(); + } + if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu)) kvm_vcpu_reload_pmu(vcpu); @@ -1267,7 +1390,7 @@ static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu) * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code * @vcpu: The VCPU pointer * - * This function is called through the VCPU_RUN ioctl called from user space. It + * This function is called through the KVM_RUN ioctl called from user space. It * will execute VM code in a loop until the time slice for the process is used * or some emulation is needed from user space in which case the function will * return with return value 0 and with the kvm_run structure filled in with the @@ -1581,10 +1704,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, return -EINVAL; } -static unsigned long system_supported_vcpu_features(void) +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) { unsigned long features = KVM_VCPU_VALID_FEATURES; + if (vcpu->kvm->arch.vm_flavor == VM_PROTECTED_PKVM) + kvm_pkvm_vcpu_allowed_features(vcpu->kvm, &features); + if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); @@ -1607,6 +1733,7 @@ static unsigned long system_supported_vcpu_features(void) return features; } +#if ARM64_S390_COMMON == 1 static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, const struct kvm_vcpu_init *init) { @@ -1621,7 +1748,12 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, return -ENOENT; } - if (features & ~system_supported_vcpu_features()) + if (features & ~system_supported_vcpu_features(vcpu)) + return -EINVAL; + + /* EL2 implements PSCI 1.1; the host must not dispatch as 0.1. */ + if (vcpu_is_protected(vcpu) && + !test_bit(KVM_ARM_VCPU_PSCI_0_2, &features)) return -EINVAL; /* @@ -1660,6 +1792,8 @@ static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu, KVM_VCPU_MAX_FEATURES); } +#endif /* ARM64_S390_COMMON == 1 */ + static int kvm_setup_vcpu(struct kvm_vcpu *vcpu) { struct kvm *kvm = vcpu->kvm; @@ -1779,7 +1913,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, spin_lock(&vcpu->arch.mp_state_lock); if (power_off) - __kvm_arm_vcpu_power_off(vcpu); + __kvm_arm_vcpu_power_off(vcpu, KVM_MP_STATE_STOPPED); else WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); @@ -1856,6 +1990,24 @@ static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, return __kvm_arm_vcpu_set_events(vcpu, events); } +/* + * Once a protected vCPU has run, the host copy holds the boot state the VMM + * wrote plus what the exit handlers copy out, and EL2 has read mp_state, + * which it does only at hyp vCPU creation. + */ +static long pkvm_filter_vcpu_ioctl(struct kvm_vcpu *vcpu, unsigned int ioctl) +{ + switch (ioctl) { + case KVM_ARM_VCPU_INIT: + case KVM_SET_ONE_REG: + case KVM_GET_ONE_REG: + if (vcpu_is_protected(vcpu) && vcpu_has_run_once(vcpu)) + return -EPERM; + } + + return 0; +} + long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1864,6 +2016,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp, struct kvm_device_attr attr; long r; + r = pkvm_filter_vcpu_ioctl(vcpu, ioctl); + if (r) + return r; + switch (ioctl) { case KVM_ARM_VCPU_INIT: { struct kvm_vcpu_init init; @@ -2109,6 +2265,37 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) } } +static const struct kvm_vcpu_ops vhe_vcpu_ops = { + .vcpu_load = vhe_vcpu_load, + .vcpu_put = vhe_vcpu_put, +}; + +static const struct kvm_vcpu_ops nvhe_vcpu_ops = { + .vcpu_load = nvhe_vcpu_load, + .vcpu_put = nvhe_vcpu_put, +}; + +static const struct kvm_vcpu_ops pkvm_vcpu_ops = { + .vcpu_load = pkvm_vcpu_load, + .vcpu_put = pkvm_vcpu_put, +}; + +#define KVM_VCPU_OPS(flavor, ops) \ + [(flavor)] = &(ops) + +static const struct kvm_vcpu_ops *arm64_vcpu_ops[] = { + KVM_VCPU_OPS(VM_NVHE, nvhe_vcpu_ops), + KVM_VCPU_OPS(VM_VHE, vhe_vcpu_ops), + KVM_VCPU_OPS(VM_PKVM, pkvm_vcpu_ops), + KVM_VCPU_OPS(VM_PROTECTED_PKVM, pkvm_vcpu_ops), +}; + +static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu) +{ + BUILD_BUG_ON(ARRAY_SIZE(arm64_vcpu_ops) != VM_FLAVOR_MAX); + vcpu->arch.vcpu_ops = arm64_vcpu_ops[vcpu->kvm->arch.vm_flavor]; +} + static unsigned long nvhe_percpu_size(void) { return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) - @@ -2626,6 +2813,10 @@ static void kvm_hyp_init_symbols(void) kvm_nvhe_sym(__icache_flags) = __icache_flags; kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits; + kvm_nvhe_sym(spectre_v2_state) = arm64_get_spectre_v2_state(); + kvm_nvhe_sym(spectre_v4_state) = arm64_get_spectre_v4_state(); + kvm_nvhe_sym(spectre_bhb_state) = arm64_get_spectre_bhb_state(); + /* Propagate the FGT state to the nVHE side */ kvm_nvhe_sym(hfgrtr_masks) = hfgrtr_masks; kvm_nvhe_sym(hfgwtr_masks) = hfgwtr_masks; @@ -2918,6 +3109,8 @@ static int __init init_hyp_mode(void) kvm_err("Failed to init hyp memory protection\n"); goto out_err; } + + pkvm_selftests(); } return 0; diff --git a/arch/arm64/include/asm/kvm_define_hypevents.h b/arch/arm64/kvm/define_hypevents.h similarity index 100% rename from arch/arm64/include/asm/kvm_define_hypevents.h rename to arch/arm64/kvm/define_hypevents.h diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index b01d6622b872..1079136d5cd7 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -62,6 +62,7 @@ const struct kvm_stats_header kvm_vcpu_stats_header = { sizeof(kvm_vcpu_stats_desc), }; +#if ARM64_S390_COMMON == 1 static bool core_reg_offset_is_vreg(u64 off) { return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) && @@ -134,19 +135,19 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) KVM_REG_ARM_CORE_REG(regs.regs[30]): off -= KVM_REG_ARM_CORE_REG(regs.regs[0]); off /= 2; - return &vcpu->arch.ctxt.regs.regs[off]; + return &vcpu_gp_regs(vcpu)[off]; case KVM_REG_ARM_CORE_REG(regs.sp): - return &vcpu->arch.ctxt.regs.sp; + return vcpu_sp_el0(vcpu); case KVM_REG_ARM_CORE_REG(regs.pc): - return &vcpu->arch.ctxt.regs.pc; + return vcpu_pc(vcpu); case KVM_REG_ARM_CORE_REG(regs.pstate): - return &vcpu->arch.ctxt.regs.pstate; + return vcpu_cpsr(vcpu); case KVM_REG_ARM_CORE_REG(sp_el1): - return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1); + return vcpu_sp_el1(vcpu); case KVM_REG_ARM_CORE_REG(elr_el1): return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1); @@ -170,13 +171,13 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]); off /= 4; - return &vcpu->arch.ctxt.fp_regs.vregs[off]; + return vcpu_vreg(vcpu, off); case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): - return &vcpu->arch.ctxt.fp_regs.fpsr; + return vcpu_fpsr(vcpu); case KVM_REG_ARM_CORE_REG(fp_regs.fpcr): - return &vcpu->arch.ctxt.fp_regs.fpcr; + return vcpu_fpcr(vcpu); default: return NULL; @@ -306,6 +307,8 @@ static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return err; } +#endif /* ARM64_S390_COMMON == 1 */ + #define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64) #define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64) #define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq))) @@ -543,6 +546,7 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) return -EINVAL; } +#if ARM64_S390_COMMON == 1 static int copy_core_reg_indices(const struct kvm_vcpu *vcpu, u64 __user *uindices) { @@ -591,6 +595,8 @@ static unsigned long num_core_regs(const struct kvm_vcpu *vcpu) return copy_core_reg_indices(vcpu, NULL); } +#endif /* ARM64_S390_COMMON == 1 */ + static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu) { const unsigned int slices = vcpu_sve_slices(vcpu); @@ -786,6 +792,13 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu, u64 esr = events->exception.serror_esr; int ret = 0; + /* + * EL2 injects an external abort only to complete a forwarded abort. + * SError injection is forwarded. + */ + if (vcpu_is_protected(vcpu) && ext_dabt_pending) + return -EPERM; + /* * Immediately commit the pending SEA to the vCPU's architectural * state which is necessary since we do not return a pending SEA @@ -883,6 +896,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, { trace_kvm_set_guest_debug(vcpu, dbg->control); + /* A protected guest's debug state is not exposed to the host. */ + if (vcpu_is_protected(vcpu)) + return -EPERM; + if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) return -EINVAL; diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index 308100ed25de..9709905f21b5 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -46,7 +46,7 @@ alternative_else_nop_endif // Ensure that __guest_enter() always provides a context // synchronization event so that callers don't need ISBs for anything - // that would usually be synchonized by the ERET. + // that would usually be synchronized by the ERET. isb mov x0, #ARM_EXCEPTION_IRQ ret diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 754e2dc1df54..7521bddf73e4 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -70,6 +70,7 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, enum exception_type type) { unsigned long sctlr, vbar, old, new, mode; + struct kvm *kvm; u64 exc_offset; mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); @@ -109,8 +110,10 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, new |= (old & PSR_C_BIT); new |= (old & PSR_V_BIT); - if (kvm_has_mte(kern_hyp_va(vcpu->kvm))) + kvm = vcpu_get_kvm(vcpu); + if (kvm && kvm_has_mte(kvm)) new |= PSR_TCO_BIT; + vcpu_put_kvm(vcpu, kvm); new |= (old & PSR_DIT_BIT); @@ -276,12 +279,12 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) switch(mode) { case PSR_AA32_MODE_ABT: __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr)); - vcpu_gp_regs(vcpu)->compat_lr_abt = return_address; + vcpu_gp_regs(vcpu)[__compat_lr_abt] = return_address; break; case PSR_AA32_MODE_UND: __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr)); - vcpu_gp_regs(vcpu)->compat_lr_und = return_address; + vcpu_gp_regs(vcpu)[__compat_lr_und] = return_address; break; } @@ -350,12 +353,19 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu) */ void __kvm_adjust_pc(struct kvm_vcpu *vcpu) { - if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) { - kvm_inject_exception(vcpu); - vcpu_clear_flag(vcpu, PENDING_EXCEPTION); - vcpu_clear_flag(vcpu, EXCEPT_MASK); - } else if (vcpu_get_flag(vcpu, INCREMENT_PC)) { - kvm_skip_instr(vcpu); - vcpu_clear_flag(vcpu, INCREMENT_PC); + struct kvm_vcpu *target = kvm_adjust_pc_get(vcpu); + + if (!target) + return; + + if (vcpu_get_flag(target, PENDING_EXCEPTION)) { + kvm_inject_exception(target); + vcpu_clear_flag(target, PENDING_EXCEPTION); + vcpu_clear_flag(target, EXCEPT_MASK); + } else if (vcpu_get_flag(target, INCREMENT_PC)) { + kvm_skip_instr(target); + vcpu_clear_flag(target, INCREMENT_PC); } + + kvm_adjust_pc_put(vcpu, target); } diff --git a/arch/arm64/kvm/hyp/hyp-constants.c b/arch/arm64/kvm/hyp/hyp-constants.c index b257a3b4bfc5..b2caae21f271 100644 --- a/arch/arm64/kvm/hyp/hyp-constants.c +++ b/arch/arm64/kvm/hyp/hyp-constants.c @@ -7,7 +7,5 @@ int main(void) { DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page)); - DEFINE(PKVM_HYP_VM_SIZE, sizeof(struct pkvm_hyp_vm)); - DEFINE(PKVM_HYP_VCPU_SIZE, sizeof(struct pkvm_hyp_vcpu)); return 0; } diff --git a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h index 4fdfeabefeb4..d2b9a98bf703 100644 --- a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h +++ b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h @@ -13,6 +13,34 @@ #include #include +#ifdef __KVM_NVHE_HYPERVISOR__ +#include + +/* Under pKVM a host vCPU's ->kvm is host-writable. */ +static inline struct kvm *vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + if (is_protected_kvm_enabled()) + return pkvm_vcpu_get_kvm(vcpu); + + return kern_hyp_va(vcpu->kvm); +} + +static inline void vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ + if (is_protected_kvm_enabled()) + pkvm_vcpu_put_kvm(vcpu, kvm); +} +#else +static inline struct kvm *vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + return vcpu->kvm; +} + +static inline void vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ +} +#endif + static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) { if (vcpu_mode_is_32bit(vcpu)) { @@ -33,11 +61,11 @@ static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) static inline void __kvm_skip_instr(struct kvm_vcpu *vcpu) { *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); - vcpu_gp_regs(vcpu)->pstate = read_sysreg_el2(SYS_SPSR); + *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); kvm_skip_instr(vcpu); - write_sysreg_el2(vcpu_gp_regs(vcpu)->pstate, SYS_SPSR); + write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); } @@ -50,4 +78,24 @@ static inline void kvm_skip_host_instr(void) write_sysreg_el2(read_sysreg_el2(SYS_ELR) + 4, SYS_ELR); } +/* + * Under pKVM, the vCPU __kvm_adjust_pc() adjusts for @vcpu (NULL leaves the + * request on @vcpu for its next entry), and the copy of the consumed + * PC_UPDATE_REQ back to @vcpu. + */ +#ifdef __KVM_NVHE_HYPERVISOR__ +struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu); +void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, struct kvm_vcpu *target); +#else +static inline struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu) +{ + return vcpu; +} + +static inline void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, + struct kvm_vcpu *target) +{ +} +#endif + #endif diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 1ce7130e2549..a2a054fec2e2 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -470,7 +470,7 @@ static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code) u64 spsr; *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR); - arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2); + arm64_mops_reset_regs(&vcpu->arch.ctxt.regs, vcpu->arch.fault.esr_el2); write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR); /* @@ -902,7 +902,7 @@ static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu) /* * Check for the conditions of Cortex-A510's #2077057. When these occur * SPSR_EL2 can't be trusted, but isn't needed either as it is - * unchanged from the value in vcpu_gp_regs(vcpu)->pstate. + * unchanged from the value in vcpu_cpsr(vcpu). * Are we single-stepping the guest, and took a PAC exception from the * active-not-pending state? */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/alloc.h b/arch/arm64/kvm/hyp/include/nvhe/alloc.h new file mode 100644 index 000000000000..bca0168e0046 --- /dev/null +++ b/arch/arm64/kvm/hyp/include/nvhe/alloc.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __KVM_NVHE_ALLOC__ +#define __KVM_NVHE_ALLOC__ +#include + +#include + +void *hyp_alloc(size_t size); +int hyp_alloc_errno(void); +u32 hyp_alloc_topup_needed(void); +void hyp_free(void *addr); + +int hyp_alloc_init(size_t size); +int hyp_alloc_topup(struct kvm_hyp_memcache *host_mc); +unsigned long hyp_alloc_reclaimable(void); +void hyp_alloc_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target); + +#ifdef CONFIG_NVHE_EL2_DEBUG +int hyp_allocator_selftest(void); +u32 hyp_alloc_selftest_topup_needed(void); +int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc); +void hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target); +#else +static inline int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc) { return 0; }; +static inline void +hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target) { }; +#endif +#endif diff --git a/arch/arm64/kvm/hyp/include/nvhe/define_events.h b/arch/arm64/kvm/hyp/include/nvhe/define_events.h index 776d4c6cb702..370e8c2d39fe 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/define_events.h +++ b/arch/arm64/kvm/hyp/include/nvhe/define_events.h @@ -10,5 +10,3 @@ #define HYP_EVENT_MULTI_READ #include #undef HYP_EVENT_MULTI_READ - -#undef HYP_EVENT diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 31cae95ddb71..72b7927a1fa5 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -25,11 +25,14 @@ int hyp_back_vmemmap(phys_addr_t back); int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot); int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot); int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot); +void pkvm_remove_mappings(void *from, void *to); int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr); int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr); int pkvm_check_host_ownership(void); int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr); +int pkvm_map_private_va_range(void *haddr, phys_addr_t phys, size_t size); +phys_addr_t pkvm_private_va_range_pa(void *va); #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h index 5c050f21066a..4373e9b63e87 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/pkvm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/pkvm.h @@ -7,6 +7,7 @@ #ifndef __ARM64_KVM_NVHE_PKVM_H__ #define __ARM64_KVM_NVHE_PKVM_H__ +#include #include #include @@ -35,6 +36,15 @@ struct pkvm_hyp_vcpu { * per-cpu pointer tracking us. Otherwise, NULL if not loaded. */ struct pkvm_hyp_vcpu **loaded_hyp_vcpu; + + /* The previous exit's ARM_EXCEPTION_* code. */ + u32 exit_code; + + /* + * PSCI_0_2_AFFINITY_LEVEL_{OFF, ON_PENDING, ON}. A non-protected + * vCPU is always ON. + */ + int power_state; }; /* @@ -50,8 +60,15 @@ struct pkvm_hyp_vm { struct kvm_pgtable pgt; struct kvm_pgtable_mm_ops mm_ops; struct hyp_pool pool; + unsigned short refcount; hyp_spinlock_t lock; + /* + * The vCPU initialised RUNNABLE: claimed under vm_table_lock, + * released only if its own init fails. + */ + struct pkvm_hyp_vcpu *primary_vcpu; + /* Array of the hyp vCPU structures for this VM. */ struct pkvm_hyp_vcpu *vcpus[]; }; @@ -74,14 +91,24 @@ static inline bool pkvm_hyp_vm_is_protected(struct pkvm_hyp_vm *hyp_vm) return kvm_vm_is_protected(&hyp_vm->kvm); } +static inline void pkvm_hyp_vm_ref_inc(struct pkvm_hyp_vm *hyp_vm) +{ + BUG_ON(hyp_vm->refcount == USHRT_MAX); + hyp_vm->refcount++; +} + +static inline void pkvm_hyp_vm_ref_dec(struct pkvm_hyp_vm *hyp_vm) +{ + BUG_ON(!hyp_vm->refcount); + hyp_vm->refcount--; +} + void pkvm_hyp_vm_table_init(void *tbl); int __pkvm_reserve_vm(void); void __pkvm_unreserve_vm(pkvm_handle_t handle); -int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, - unsigned long pgd_hva); -int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - unsigned long vcpu_hva); +int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva); +int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu); int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn); int __pkvm_start_teardown_vm(pkvm_handle_t handle); @@ -92,6 +119,8 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, unsigned int vcpu_idx); void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu); struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void); +struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu); +void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm); struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle); struct pkvm_hyp_vm *get_np_pkvm_hyp_vm(pkvm_handle_t handle); @@ -101,6 +130,9 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code); bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code); bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code); void kvm_init_pvm_id_regs(struct kvm_vcpu *vcpu); +void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu); int kvm_check_pvm_sysreg_table(void); +int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu); +struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *vm, unsigned long mpidr); #endif /* __ARM64_KVM_NVHE_PKVM_H__ */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h index b6978f03e86a..4116c73263c2 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/spinlock.h +++ b/arch/arm64/kvm/hyp/include/nvhe/spinlock.h @@ -17,6 +17,8 @@ #include #include +#include + typedef union hyp_spinlock { u32 __val; struct { @@ -118,4 +120,6 @@ static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) { } #endif +DEFINE_LOCK_GUARD_1(hyp_spinlock, hyp_spinlock_t, hyp_spin_lock(_T->lock), + hyp_spin_unlock(_T->lock)) #endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */ diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h index 8813ff250f8e..6e3225d8d7d7 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/trace.h +++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h @@ -4,6 +4,7 @@ #include +#include #include static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt) @@ -46,7 +47,71 @@ static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt) void *tracing_reserve_entry(unsigned long length); void tracing_commit_entry(void); -int __tracing_load(unsigned long desc_va, size_t desc_size); +/* + * The trace_hyp_printk boilerplate is too fiddly to be declared with + * HYP_EVENT(): + * + * The string format is stored into a kernel-accessible ELF section. The + * hypervisor only writes the format ID. + * + * The function has a variadic prototype. We have no easy way to know each + * argument width so they must all cast to u64. + */ +#define REMOTE_EVENT_CUSTOM_PRINTK(...) + +#define __TO_U64_0() +#define __TO_U64_1(x) , (u64)(x) +#define __TO_U64_2(x, ...) , (u64)(x) __TO_U64_1(__VA_ARGS__) +#define __TO_U64_3(x, ...) , (u64)(x) __TO_U64_2(__VA_ARGS__) +#define __TO_U64_4(x, ...) , (u64)(x) __TO_U64_3(__VA_ARGS__) +#define __TO_U64_5(x, ...) , (u64)(x) __TO_U64_4(__VA_ARGS__) +#define __TO_U64_6(x, ...) , (u64)(x) __TO_U64_5(__VA_ARGS__) +#define __TO_U64_7(x, ...) , (u64)(x) __TO_U64_6(__VA_ARGS__) +#define __TO_U64_8(x, ...) , (u64)(x) __TO_U64_7(__VA_ARGS__) + +#define __TO_U64_X(N, ...) CONCATENATE(__TO_U64_, N)(__VA_ARGS__) +#define __TO_U64(...) __TO_U64_X(COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__) + +REMOTE_EVENT_FORMAT(hyp_printk, HE_STRUCT(he_field(u16, fmt_id) he_field(u64, args[]))); +extern struct hyp_event_id hyp_event_id_hyp_printk; + +static __always_inline void __trace_hyp_printk(const struct hyp_string_fmt *fmt, int nr_args, ...) +{ + struct remote_event_format_hyp_printk *entry; + va_list va; + int i; + + if (!atomic_read(&hyp_event_id_hyp_printk.enabled)) + return; + + entry = tracing_reserve_entry(struct_size(entry, args, nr_args)); + if (!entry) + return; + + entry->hdr.id = hyp_event_id_hyp_printk.id; + entry->fmt_id = fmt - __hyp_string_fmts_start; + + va_start(va, nr_args); + for (i = 0; i < nr_args; i++) + entry->args[i] = va_arg(va, u64); + va_end(va); + + tracing_commit_entry(); +} + +#define trace_hyp_printk(__fmt, __args...) \ +do { \ + static const struct hyp_string_fmt __used __section("_hyp_string_fmts") \ + __hyp_str_fmt = { \ + .fmt = __fmt, \ + .nr_args = COUNT_ARGS(__args), \ + }; \ + BUILD_BUG_ON(sizeof(__fmt) > HYP_STRING_FMT_MAX_SIZE); \ + /* __TO_U64 prepends a comma if there are arguments */ \ + __trace_hyp_printk(&__hyp_str_fmt, COUNT_ARGS(__args) __TO_U64(__args)); \ +} while (0) + +int __tracing_load(void __kern *desc_va, size_t desc_size); void __tracing_unload(void); int __tracing_enable(bool enable); int __tracing_swap_reader(unsigned int cpu); @@ -58,8 +123,10 @@ static inline void *tracing_reserve_entry(unsigned long length) { return NULL; } static inline void tracing_commit_entry(void) { } #define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \ static inline void trace_##__name(__proto) {} +#define REMOTE_EVENT_CUSTOM_PRINTK(...) +#define trace_hyp_printk(fmt, args...) do { } while (0) -static inline int __tracing_load(unsigned long desc_va, size_t desc_size) { return -ENODEV; } +static inline int __tracing_load(void __kern *desc_va, size_t desc_size) { return -ENODEV; } static inline void __tracing_unload(void) { } static inline int __tracing_enable(bool enable) { return -ENODEV; } static inline int __tracing_swap_reader(unsigned int cpu) { return -ENODEV; } diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile index 2ed489bf544e..56a7e2c05f73 100644 --- a/arch/arm64/kvm/hyp/nvhe/Makefile +++ b/arch/arm64/kvm/hyp/nvhe/Makefile @@ -23,7 +23,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs)) CFLAGS_switch.nvhe.o += -Wno-override-init hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \ - hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \ + hyp-main.o hyp-smp.o psci-relay.o alloc.o early_alloc.o page_alloc.o \ cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \ ../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o @@ -49,6 +49,7 @@ targets += $(hyp-obj) kvm_nvhe.tmp.o kvm_nvhe.rel.o hyp.lds hyp-reloc.S hyp-relo # avoids file name clashes for files shared with VHE. $(obj)/%.nvhe.o: $(src)/%.c FORCE $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) $(obj)/%.nvhe.o: $(src)/%.S FORCE $(call if_changed_rule,as_o_S) diff --git a/arch/arm64/kvm/hyp/nvhe/alloc.c b/arch/arm64/kvm/hyp/nvhe/alloc.c new file mode 100644 index 000000000000..e1e6dd21cda1 --- /dev/null +++ b/arch/arm64/kvm/hyp/nvhe/alloc.c @@ -0,0 +1,1212 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2026 Google LLC + * Author: Vincent Donnefort + * + * This heap allocator manages a reserved VA space range, dynamically mapping + * and unmapping physical pages on-demand to minimise the pKVM hypervisor + * footprint. As memory is reclaimed and relinquished to the host, unmapped + * holes are introduced within the VA space. To prevent orphans mapped regions, + * neighboring unused chunks cannot be merged if they are separated by an + * unmapped region. + * + */ + +#include +#include +#include +#include + +#include +#include + +/* + * A minimum of 16 bytes allows us to comply with the alignment requirement for + * FPSIMD registers that are part of struct pkvm_hyp_vcpu. + * + * It also must be a power of two. + */ +#define MIN_ALLOC_SIZE 16UL + +/* + * struct chunk_hdr - Chunk header + * @next: offset from this chunk header to the next one. + * @prev: offset from this chunk header to the previous one. + * @__unmapped: Internal field containing the offset to the unmapped page + * boundary, multiplexed with the allocation state flag. + * @hash: Hash computed over the chunk header. + * @data: Chunk payload. + */ +struct chunk_hdr { + u32 next; + u32 prev; +#define USED_BIT_MASK 1U + u32 __unmapped; + u32 hash; + char data[]; +} __aligned(MIN_ALLOC_SIZE); + +/* + * struct hyp_allocator - Heap allocator + * @start: Start in the allocator's reserved virtual address range. + * @end: End in the allocator's reserved virtual address range. + * @last_used: Pointer to the end of the last used chunk. This is + * necessary for the last chunk in the list as the + * allocated size of a chunk is derived from the next one. + * @first_unmapped: Pointer to the first unmapped page in the + * allocator's range. This is only necessary and + * updated when no chunk is in the list. + * @head: Head of the chunk list. + * @tail: Tail of the chunk list. + * @mc: Memcache containing pre-allocated pages for mapping. + * @lock: Spinlock protecting the allocator state. + * @errno: Per-CPU error code for allocation failures. + * @topup_needed: Per-CPU page counter needed to top-up the memcache. + */ +struct hyp_allocator { + void *start; + void *end; + void *last_used; + void *first_unmapped; + struct chunk_hdr *head; + struct chunk_hdr *tail; + struct kvm_hyp_memcache mc; + hyp_spinlock_t lock; + int __percpu *errno; + u32 __percpu *topup_needed; +}; + +static u32 chunk_hash_compute(const struct chunk_hdr *chunk) +{ + u32 hash = 0; + + BUILD_BUG_ON(sizeof(*chunk) != 16); + + hash ^= hash_64(*(const u64 *)chunk, 32); + hash ^= hash_32(chunk->__unmapped, 32); + return hash; +} + +static void chunk_set_hash(struct chunk_hdr *chunk) +{ + if (chunk) + chunk->hash = chunk_hash_compute(chunk); +} + +static void chunk_check_hash(const struct chunk_hdr *chunk) +{ + if (chunk) + WARN_ON(chunk->hash != chunk_hash_compute(chunk)); +} + +static bool chunk_is_used(const struct chunk_hdr *chunk) +{ + return !!(chunk->__unmapped & USED_BIT_MASK); +} + +static void chunk_set_used(struct chunk_hdr *chunk) +{ + chunk->__unmapped |= USED_BIT_MASK; +} + +static void chunk_set_unused(struct chunk_hdr *chunk) +{ + chunk->__unmapped &= ~USED_BIT_MASK; +} + +static void *chunk_unmapped(const struct chunk_hdr *chunk) +{ + u32 offset = chunk->__unmapped & ~USED_BIT_MASK; + + if (!offset) + return NULL; + + return (void *)chunk + offset; +} + +static void __chunk_set_unmapped(struct chunk_hdr *chunk, u32 unmapped) +{ + chunk->__unmapped = unmapped | (chunk_is_used(chunk) ? USED_BIT_MASK : 0); +} + +static void chunk_set_unmapped(struct chunk_hdr *chunk, void *unmapped) +{ + WARN_ON(!PAGE_ALIGNED(unmapped)); + + if (unmapped) { + WARN_ON((void *)chunk > unmapped); + __chunk_set_unmapped(chunk, unmapped - (void *)chunk); + } else { + __chunk_set_unmapped(chunk, 0); + } +} + +static void *chunk_data(const struct chunk_hdr *chunk) +{ + return (void *)&chunk->data; +} + +static struct chunk_hdr *__chunk_next(const struct chunk_hdr *chunk) +{ + if (!chunk->next) + return NULL; + + return (struct chunk_hdr *)((void *)chunk + chunk->next); +} + +static struct chunk_hdr *__chunk_prev(const struct chunk_hdr *chunk) +{ + if (!chunk->prev) + return NULL; + + return (struct chunk_hdr *)((void *)chunk - chunk->prev); +} + +static void chunk_set_next(struct chunk_hdr *chunk, struct chunk_hdr *next) +{ + if (!chunk) + return; + + if (next) { + WARN_ON(chunk > next); + chunk->next = (void *)next - (void *)chunk; + } else { + chunk->next = 0; + } +} + +static void chunk_set_prev(struct chunk_hdr *chunk, struct chunk_hdr *prev) +{ + if (!chunk) + return; + + if (prev) { + WARN_ON(chunk < prev); + chunk->prev = (void *)chunk - (void *)prev; + } else { + chunk->prev = 0; + } +} + +static struct chunk_hdr *chunk_get_next(const struct chunk_hdr *chunk) +{ + struct chunk_hdr *next = __chunk_next(chunk); + + chunk_check_hash(next); + return next; +} + +static struct chunk_hdr *chunk_get_prev(const struct chunk_hdr *chunk) +{ + struct chunk_hdr *prev = __chunk_prev(chunk); + + chunk_check_hash(prev); + return prev; +} + +static struct chunk_hdr *chunk_get(struct chunk_hdr *chunk) +{ + chunk_check_hash(chunk); + return chunk; +} + +#define chunk_hdr_size() \ + offsetof(struct chunk_hdr, data) + +#define chunk_min_size() \ + (chunk_hdr_size() + MIN_ALLOC_SIZE) + +static size_t chunk_data_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + void *end; + + if (next) + end = (void *)next; + else + end = allocator->end; + + return end - chunk_data(chunk); +} + +static size_t chunk_mapped_data_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + void *unmapped = chunk_unmapped(chunk); + + if (!unmapped) + return chunk_data_size(chunk, allocator); + + return unmapped - chunk_data(chunk); +} + +static size_t chunk_used_size(const struct chunk_hdr *chunk, struct hyp_allocator *allocator) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + + if (!chunk_is_used(chunk)) + return 0; + + if (next) + return chunk_mapped_data_size(chunk, allocator); + + return allocator->last_used - chunk_data(chunk); +} + +static void chunk_list_insert(struct chunk_hdr *chunk, struct chunk_hdr *prev) +{ + struct chunk_hdr *next = NULL; + + WARN_ON(!chunk); + + if (prev) { + next = chunk_get_next(prev); + chunk_set_next(prev, chunk); + chunk_set_hash(prev); + } + + if (next) { + chunk_set_prev(next, chunk); + chunk_set_hash(next); + } + + chunk_set_next(chunk, next); + chunk_set_prev(chunk, prev); +} + +static void chunk_list_del(struct chunk_hdr *chunk) +{ + struct chunk_hdr *prev, *next; + + WARN_ON(!chunk); + + prev = chunk_get_prev(chunk); + next = chunk_get_next(chunk); + + if (prev) { + chunk_set_next(prev, next); + chunk_set_hash(prev); + } + + if (next) { + chunk_set_prev(next, prev); + chunk_set_hash(next); + } +} + +/* + * Return a fixup start address for chunk creation. It makes sure the chunk + * header doesn't cross any page boundary and that it leaves enough space at the + * start of page. This is intended to prevent orphan mapped regions during chunk + * memory reclaim + */ +static void *chunk_start(void *start) +{ + void *page = PTR_ALIGN(start, PAGE_SIZE); + + if (page - start < chunk_hdr_size()) + return page; + + page = PTR_ALIGN_DOWN(start, PAGE_SIZE); + if (start - page < chunk_min_size()) + return page + chunk_min_size(); + + return start; +} + +static void hyp_allocator_set_errno(struct hyp_allocator *allocator, int errno) +{ + *this_cpu_ptr(allocator->errno) = errno; +} + +static int hyp_allocator_errno(struct hyp_allocator *allocator) +{ + int *errno = this_cpu_ptr(allocator->errno); + int ret = *errno; + + *errno = 0; + + return ret; +} + +static int hyp_allocator_map(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + struct chunk_hdr *next, + void *addr, void *end) +{ + void *unmapped = chunk ? chunk_unmapped(chunk) : allocator->first_unmapped; + void *unmapped_end = next ?: allocator->end; + + /* + * hyp_allocator_can_create_chunk() already validates addr/end + * belong to the chunk. + */ + WARN_ON(end <= addr); + + /* The chunk does not span an unmapped region */ + if (!unmapped) + return 0; + + while (unmapped < end) { + void *page = pop_hyp_memcache(&allocator->mc, hyp_phys_to_virt); + int ret; + + if (!page) { + end = PTR_ALIGN(end, PAGE_SIZE); + *this_cpu_ptr(allocator->topup_needed) = + (unsigned long)(end - unmapped) >> PAGE_SHIFT; + return -ENOMEM; + } + + ret = pkvm_map_private_va_range(unmapped, hyp_virt_to_phys(page), PAGE_SIZE); + if (ret) { + push_hyp_memcache(&allocator->mc, page, hyp_virt_to_phys); + return ret; + } + + unmapped += PAGE_SIZE; + + /* + * Reset the unmap field if we've reached the next chunk or the + * allocator boundary. Chunks are always installed to cover a + * mapped region followed by an unmapped one. Therefore, we + * expect next to be PAGE_ALIGNED. + */ + WARN_ON(!PAGE_ALIGNED(unmapped_end)); + if (unmapped == unmapped_end) + unmapped = NULL; + + if (chunk) { + chunk_set_unmapped(chunk, unmapped); + chunk_set_hash(chunk); + } else { + allocator->first_unmapped = unmapped; + } + + if (!unmapped) + break; + } + + return 0; +} + +static void hyp_allocator_unmap(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + void *addr, void *end) +{ + void *unmap = addr; + + /* + * hyp_allocator_chunk_reclaimable() already computes valid addr/end, no + * need to check them again + */ + WARN_ON(end <= addr); + + while (unmap < end) { + phys_addr_t pa = pkvm_private_va_range_pa((void *)unmap); + void *page = hyp_phys_to_virt(pa); + + push_hyp_memcache(&allocator->mc, page, hyp_virt_to_phys); + unmap += PAGE_SIZE; + } + + pkvm_remove_mappings(addr, end); + + if (chunk) { + chunk_set_unmapped(chunk, addr); + chunk_set_hash(chunk); + } else { + allocator->first_unmapped = addr; + } +} + +static bool hyp_allocator_can_create_chunk(struct hyp_allocator *allocator, + const struct chunk_hdr *prev, + const struct chunk_hdr *next, + void *addr, void *end) +{ + void *page, *unmapped; + + if (addr < allocator->start || end > allocator->end) + return false; + + /* First chunk created must be installed at allocator->start */ + if (!prev) + return addr == allocator->start; + + /* Must not overwrite the next chunk */ + if (next && end > (void *)next) + return false; + + /* Must not overwrite the previous chunk */ + if (addr < (chunk_data(prev) + chunk_used_size(prev, allocator))) + return false; + + /* Header must not cross page boundaries */ + page = PTR_ALIGN(addr, PAGE_SIZE); + if (page != addr && (page - addr) < chunk_hdr_size()) + return false; + + /* Must leave a minimum distance from a page-start to maximise reclaim */ + page = PTR_ALIGN_DOWN(addr, PAGE_SIZE); + if (page != addr && (addr - page) < chunk_min_size()) + return false; + + unmapped = chunk_unmapped(prev); + if (!unmapped) + return true; + + /* Must never create an orphan mapped region */ + if (addr > unmapped) + return false; + + return true; +} + +/* + * Tries to create a new chunk in the allocator whose header starts at @addr and + * whose data finishes at @end. + */ +static struct chunk_hdr *hyp_allocator_create_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *prev, void *addr, + void *end, bool used) +{ + struct chunk_hdr *next, *chunk = addr; + void *unmapped; + int ret; + + if (end > allocator->end) + return ERR_PTR(-E2BIG); + + next = prev ? chunk_get_next(prev) : NULL; + if (!hyp_allocator_can_create_chunk(allocator, prev, next, addr, end)) + return ERR_PTR(-EINVAL); + + ret = hyp_allocator_map(allocator, prev, next, addr, end); + if (ret) + return ERR_PTR(ret); + + memset(chunk, 0, sizeof(*chunk)); + if (used) + chunk_set_used(chunk); + else + chunk_set_unused(chunk); + + /* First chunk, first allocation */ + if (!prev) { + chunk_set_unmapped(chunk, allocator->first_unmapped); + chunk_list_insert(chunk, NULL); + chunk_set_hash(chunk); + + allocator->last_used = end; + allocator->head = allocator->tail = chunk; + return chunk; + } + + /* Last chunk in the list */ + if (!next) { + allocator->last_used = end; + allocator->tail = chunk; + } + + /* Inherit prev's unmapped region */ + unmapped = chunk_unmapped(prev); + chunk_set_unmapped(chunk, unmapped); + chunk_list_insert(chunk, prev); + chunk_set_hash(chunk); + + chunk_set_unmapped(prev, NULL); + chunk_set_hash(prev); + + return chunk; +} + +static bool hyp_allocator_can_destroy_chunk(const struct chunk_hdr *prev, + const struct chunk_hdr *next, + const struct chunk_hdr *chunk) +{ + if (chunk_is_used(chunk)) + return false; + + /* Can't merge down unless we are the last one in the list */ + if (next && chunk_is_used(prev)) + return false; + + /* Must never create an orphan mapped region */ + if (chunk_unmapped(prev)) + return false; + + return true; +} + +static int hyp_allocator_destroy_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *prev, + struct chunk_hdr *chunk) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + + if (!hyp_allocator_can_destroy_chunk(prev, next, chunk)) + return -EINVAL; + + /* Last chunk in the list */ + if (!next) { + allocator->last_used = chunk; + allocator->tail = prev; + } + + chunk_set_unmapped(prev, chunk_unmapped(chunk)); + chunk_set_hash(prev); + chunk_list_del(chunk); + + return 0; +} + +/* + * Return the best unused chunk for recycling, that is the smallest chunk + * fitting the allocation which needs to use the least unmapped region. + */ +static struct chunk_hdr *hyp_allocator_find_efficient_chunk(struct hyp_allocator *allocator, + size_t size) +{ + struct chunk_hdr *chunk, *best_chunk = NULL; + size_t best_data_size = SIZE_MAX; + size_t best_missing = SIZE_MAX; + + chunk = allocator->head; + while (chunk) { + size_t missing, mapped, data_size; + + if (chunk_is_used(chunk)) + goto next; + + data_size = chunk_data_size(chunk, allocator); + if (data_size < size) + goto next; + + mapped = chunk_mapped_data_size(chunk, allocator); + missing = (size > mapped) ? DIV_ROUND_UP(size - mapped, PAGE_SIZE) : 0; + if (missing > best_missing) + goto next; + + if (missing == best_missing && data_size >= best_data_size) + goto next; + + best_missing = missing; + best_data_size = data_size; + best_chunk = chunk; + +next: + chunk = chunk_get_next(chunk); + } + + return best_chunk; +} + +static struct chunk_hdr *hyp_allocator_reuse_chunk(struct hyp_allocator *allocator, + struct chunk_hdr *chunk, size_t size) +{ + struct chunk_hdr *next = chunk_get_next(chunk); + void *start, *end, *split, *split_end; + int ret; + + start = chunk_data(chunk); + end = start + size; + + /* Last chunk in the list, no need to split */ + if (!next) { + split = split_end = NULL; + allocator->last_used = chunk_data(chunk) + size; + } else { + split = chunk_start(end); + split_end = split + chunk_min_size(); + + if (!hyp_allocator_can_create_chunk(allocator, chunk, next, split, split_end)) + split = split_end = NULL; + } + + /* Batch the mapping of the reused chunk and the split */ + ret = hyp_allocator_map(allocator, chunk, next, chunk_data(chunk), split ? split_end : end); + if (ret) + return ERR_PTR(ret); + + if (split) + WARN_ON(IS_ERR_OR_NULL( + hyp_allocator_create_chunk(allocator, chunk, split, split_end, false))); + + chunk_set_used(chunk); + chunk_set_hash(chunk); + + return chunk; +} + +static struct chunk_hdr *hyp_allocator_alloc_chunk(struct hyp_allocator *allocator, size_t size) +{ + struct chunk_hdr *chunk; + void *start, *end; + + /* First allocation */ + if (!allocator->head) { + start = allocator->start; + end = start + chunk_hdr_size() + size; + return hyp_allocator_create_chunk(allocator, NULL, start, end, true); + } + + chunk = hyp_allocator_find_efficient_chunk(allocator, size); + + /* Nothing found, create a new chunk at the end in the list */ + if (!chunk) { + start = chunk_start(chunk_data(allocator->tail) + + chunk_used_size(allocator->tail, allocator)); + end = start + chunk_hdr_size() + size; + return hyp_allocator_create_chunk(allocator, allocator->tail, start, end, true); + } + + return hyp_allocator_reuse_chunk(allocator, chunk, size); +} + +static void *hyp_allocator_alloc(struct hyp_allocator *allocator, size_t size) +{ + struct chunk_hdr *chunk; + + size = max(size, MIN_ALLOC_SIZE); + + /* Ensure we do not overflow ALIGN(MIN_ALLOC_SIZE) */ + if (size > U32_MAX) { + hyp_allocator_set_errno(allocator, -E2BIG); + return NULL; + } + + size = ALIGN(size, MIN_ALLOC_SIZE); + if (size > (allocator->end - allocator->start - chunk_hdr_size())) { + hyp_allocator_set_errno(allocator, -E2BIG); + return NULL; + } + + scoped_guard(hyp_spinlock, &allocator->lock) { + chunk = hyp_allocator_alloc_chunk(allocator, size); + if (IS_ERR_OR_NULL(chunk)) { + hyp_allocator_set_errno(allocator, + IS_ERR(chunk) ? PTR_ERR(chunk) : -EINVAL); + return NULL; + } + } + + memset(chunk_data(chunk), 0, size); + return chunk_data(chunk); +} + +static void hyp_allocator_free(struct hyp_allocator *allocator, void *data) +{ + struct chunk_hdr *chunk, *next, *prev; + + if (!data) + return; + + WARN_ON(!IS_ALIGNED((unsigned long)data, MIN_ALLOC_SIZE)); + WARN_ON(data >= allocator->end || data < allocator->start + chunk_hdr_size()); + + guard(hyp_spinlock)(&allocator->lock); + + chunk = chunk_get(container_of(data, struct chunk_hdr, data)); + WARN_ON(!chunk_is_used(chunk)); + chunk_set_unused(chunk); + chunk_set_hash(chunk); + + next = chunk_get_next(chunk); + if (next) + hyp_allocator_destroy_chunk(allocator, chunk, next); + + prev = chunk_get_prev(chunk); + if (prev) + hyp_allocator_destroy_chunk(allocator, prev, chunk); +} + +static unsigned long hyp_allocator_chunk_reclaimable(struct hyp_allocator *allocator, + const struct chunk_hdr *chunk, + u64 *__addr, u64 *__end) +{ + struct chunk_hdr *next; + void *addr, *end; + + /* Last chunk in the allocator */ + if (chunk == allocator->head && chunk == allocator->tail && !chunk_is_used(chunk)) { + addr = (void *)chunk; + end = chunk_unmapped(chunk); + if (!end) + end = allocator->end; + goto end; + } + + next = chunk_get_next(chunk); + + /* Last chunk in the list we can reclaim, even if used */ + if (!next) { + addr = chunk_data(chunk) + chunk_used_size(chunk, allocator); + addr = PTR_ALIGN(addr, PAGE_SIZE); + end = chunk_unmapped(chunk); + if (!end) + end = allocator->end; + goto end; + } + + if (chunk_is_used(chunk)) + return 0; + + addr = PTR_ALIGN(chunk_data(chunk), PAGE_SIZE); + end = chunk_unmapped(chunk); + if (!end) + end = PTR_ALIGN_DOWN(next, PAGE_SIZE); + +end: + if (addr >= end) + return 0; + + if (__end) + *__end = (u64)end; + if (__addr) + *__addr = (u64)addr; + + return (end - addr) >> PAGE_SHIFT; +} + +static void hyp_allocator_reclaim_chunk(struct hyp_allocator *allocator, struct chunk_hdr *chunk, + void *addr, void *end) +{ + struct chunk_hdr *next; + + WARN_ON(end <= addr); + + /* We are about to destroy the last chunk in the allocator */ + if (addr == allocator->start) { + allocator->tail = allocator->head = chunk = NULL; + goto unmap; + } + + next = chunk_get_next(chunk); + + /* + * Split the reclaimed chunk at the next page boundary, + * this ensures no orphan mapped region is created. Splitting at the page boundary is always + * possible because chunks always leave a minimum distance to the page start. + * + * +--------------+ + * |______________| + * |______________|<- Next chunk + * |_ _ _ __ _ _ _| + * | |<- Page-aligned split + * +--------------+ + * +--------------+ + * | | + * | |<- Page reclaimed + * | | + * | | + * +--------------+ + * +--------------+ + * | | + * |______________| + * |______________|<- Chunk to split + * | | + * +--------------+ + */ + if (next && !chunk_unmapped(chunk) && next != end) + WARN_ON(IS_ERR_OR_NULL(hyp_allocator_create_chunk(allocator, chunk, end, next, + false))); +unmap: + hyp_allocator_unmap(allocator, chunk, addr, end); +} + +/* + * Return the best reclaimable chunk which is the highest chunk in the list + * with the biggest reclaimable region. + */ +static struct chunk_hdr *hyp_allocator_find_reclaimable_chunk(struct hyp_allocator *allocator, + u64 *addr, u64 *end) +{ + struct chunk_hdr *chunk, *best_chunk = NULL; + unsigned long best_reclaimable = 0; + + chunk = allocator->head; + while (chunk) { + u64 __addr, __end; + unsigned long reclaimable = hyp_allocator_chunk_reclaimable(allocator, chunk, + &__addr, &__end); + + /* Favour the top biggest chunks */ + if (reclaimable && reclaimable >= best_reclaimable) { + best_reclaimable = reclaimable; + best_chunk = chunk; + *addr = __addr; + *end = __end; + } + + chunk = chunk_get_next(chunk); + } + + return best_chunk; +} + +static void hyp_allocator_drain_memcache(struct hyp_allocator *allocator, + struct kvm_hyp_memcache *host_mc, + unsigned long target) +{ + struct kvm_hyp_memcache *mc = &allocator->mc; + + while (target && mc->nr_pages) { + void *page = pop_hyp_memcache(mc, hyp_phys_to_virt); + + memset(page, 0, PAGE_SIZE); + kvm_flush_dcache_to_poc(page, PAGE_SIZE); + push_hyp_memcache(host_mc, page, hyp_virt_to_phys); + WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(page), 1)); + + target--; + } +} + +static void hyp_allocator_reclaim(struct hyp_allocator *allocator, struct kvm_hyp_memcache *host_mc, + unsigned long target) +{ + unsigned long reclaimed = 0; + + if (!target) + return; + + guard(hyp_spinlock)(&allocator->lock); + + do { + struct chunk_hdr *chunk; + unsigned long r; + u64 addr, end; + + chunk = hyp_allocator_find_reclaimable_chunk(allocator, &addr, &end); + if (!chunk) + break; + + r = min((end - addr) >> PAGE_SHIFT, target - reclaimed); + addr = end - (r << PAGE_SHIFT); + hyp_allocator_reclaim_chunk(allocator, chunk, (void *)addr, (void *)end); + + reclaimed += r; + } while (reclaimed < target); + + hyp_allocator_drain_memcache(allocator, host_mc, target); +} + +static unsigned long hyp_allocator_reclaimable(struct hyp_allocator *allocator) +{ + unsigned long reclaimable; + struct chunk_hdr *chunk; + + guard(hyp_spinlock)(&allocator->lock); + + /* + * While allocator->mc typically contains transient pages, + * over-allocation or an aborted operation could leave pages behind. + * Account for allocator->mc so stalled topup pages can be reclaimed. + */ + reclaimable = allocator->mc.nr_pages; + + chunk = allocator->head; + while (chunk) { + reclaimable += hyp_allocator_chunk_reclaimable(allocator, chunk, NULL, NULL); + chunk = chunk_get_next(chunk); + } + + return reclaimable; +} + +static int hyp_allocator_topup(struct hyp_allocator *allocator, + struct kvm_hyp_memcache *host_mc) +{ + struct kvm_hyp_memcache *alloc_mc = &allocator->mc; + + guard(hyp_spinlock)(&allocator->lock); + return refill_memcache(alloc_mc, host_mc->nr_pages + alloc_mc->nr_pages, host_mc); +} + +static u32 hyp_allocator_topup_needed(struct hyp_allocator *allocator) +{ + u32 *topup_needed = this_cpu_ptr(allocator->topup_needed); + u32 ret = *topup_needed; + + *topup_needed = 0; + + return ret; +} + +static int hyp_allocator_init(struct hyp_allocator *allocator, size_t size) +{ + unsigned long start; + int ret; + + size = PAGE_ALIGN(size); + + /* constrained by chunk_hdr u32 types */ + if (size > U32_MAX || !size) + return -EINVAL; + + ret = pkvm_alloc_private_va_range(size, &start); + if (ret) + return ret; + + allocator->first_unmapped = allocator->start = (void *)start; + allocator->end = allocator->start + size; + hyp_spin_lock_init(&allocator->lock); + + return 0; +} + +static DEFINE_PER_CPU(int, __hyp_allocator_errno); +static DEFINE_PER_CPU(u32, __hyp_allocator_topup_needed); + +static struct hyp_allocator hyp_allocator = { + .errno = &__hyp_allocator_errno, + .topup_needed = &__hyp_allocator_topup_needed, +}; + +/** + * hyp_alloc() - Allocate memory from the heap allocator + * + * @size: Allocation size in bytes. + * + * Callers must ensure they do not hold pkvm_pgd_lock to avoid deadlocks, + * as the allocator may modify the hyp stage-1 page-table. + * + * Return: A pointer to the allocated memory on success, else NULL. + */ +void *hyp_alloc(size_t size) +{ + return hyp_allocator_alloc(&hyp_allocator, size); +} + +/** + * hyp_free() - Free memory allocated with hyp_alloc() + * + * @data: Address returned by the original hyp_alloc(). + * + * The use of any other address than one returned by hyp_alloc() will cause a + * hypervisor panic. + */ +void hyp_free(void *data) +{ + hyp_allocator_free(&hyp_allocator, data); +} + +/** + * hyp_alloc_errno() - Read the errno on allocation error + * + * Get the return code from an allocation failure. + * + * Return: -ENOMEM if the allocator needs a refill from the host, -E2BIG if + * there is no VA space left else 0. + */ +int hyp_alloc_errno(void) +{ + return hyp_allocator_errno(&hyp_allocator); +} + +static int selftest_init(void); + +int hyp_alloc_init(size_t size) +{ + int ret; + + ret = hyp_allocator_init(&hyp_allocator, size); + if (ret) + return ret; + + return selftest_init(); +} + +void hyp_alloc_reclaim(struct kvm_hyp_memcache *mc, unsigned long target) +{ + hyp_allocator_reclaim(&hyp_allocator, mc, target); +} + +unsigned long hyp_alloc_reclaimable(void) +{ + return hyp_allocator_reclaimable(&hyp_allocator); +} + +int hyp_alloc_topup(struct kvm_hyp_memcache *host_mc) +{ + return hyp_allocator_topup(&hyp_allocator, host_mc); +} + +u32 hyp_alloc_topup_needed(void) +{ + return hyp_allocator_topup_needed(&hyp_allocator); +} + +#ifdef CONFIG_NVHE_EL2_DEBUG +#define SELFTEST_MAX_PAGES 6 +#define SELFTEST_MAX_SIZE (PAGE_SIZE * SELFTEST_MAX_PAGES) + +static DEFINE_PER_CPU(int, __selftest_errno); +static DEFINE_PER_CPU(u32, __selftest_topup_needed); + +static struct hyp_allocator selftest_allocator = { + .errno = &__selftest_errno, + .topup_needed = &__selftest_topup_needed, + .lock = __HYP_SPIN_LOCK_UNLOCKED, +}; + +int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc) +{ + return hyp_allocator_topup(&selftest_allocator, host_mc); +} + +void hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target) +{ + hyp_allocator_reclaim(&selftest_allocator, host_mc, target); +} + +u32 hyp_alloc_selftest_topup_needed(void) +{ + return hyp_allocator_topup_needed(&selftest_allocator); +} + +static int selftest_init(void) +{ + return hyp_allocator_init(&selftest_allocator, SELFTEST_MAX_SIZE); +} + +static void *selftest_alloc(size_t size) +{ + return hyp_allocator_alloc(&selftest_allocator, size); +} + +static void selftest_free(void *addr) +{ + hyp_allocator_free(&selftest_allocator, addr); +} + +static int selftest_errno(void) +{ + return hyp_allocator_errno(&selftest_allocator); +} + +int hyp_allocator_selftest(void) +{ + struct hyp_allocator *allocator = &selftest_allocator; + static DEFINE_HYP_SPINLOCK(selftest_lock); + struct kvm_hyp_memcache host_mc = { }; + void *addr1, *addr2, *addr3, *addr4; + int ret; + + guard(hyp_spinlock)(&selftest_lock); + + if (allocator->mc.nr_pages < SELFTEST_MAX_PAGES) { + *this_cpu_ptr(allocator->topup_needed) = SELFTEST_MAX_PAGES - + allocator->mc.nr_pages; + return -ENOMEM; + } + + selftest_alloc(SELFTEST_MAX_SIZE); + if (selftest_errno() != -E2BIG) + return -EINVAL; + + selftest_alloc(SIZE_MAX); + if (selftest_errno() != -E2BIG) + return -EINVAL; + + /* Test first chunk */ + addr1 = selftest_alloc(0); + if (!addr1 || addr1 != (void *)allocator->start + chunk_hdr_size()) + return -EINVAL; + + /* Test second contiguous chunk with unaligned size */ + addr2 = selftest_alloc(MIN_ALLOC_SIZE + 1); + if (!addr2) + return -EINVAL; + addr3 = selftest_alloc(0); + if (!addr3 || + addr3 != addr2 + (2 * MIN_ALLOC_SIZE) + chunk_hdr_size()) + return -EINVAL; + + selftest_free(addr3); + + /* Test chunk recycling */ + selftest_free(addr1); + if (addr1 != selftest_alloc(0)) + return -EINVAL; + + /* Test chunk forward merging */ + addr3 = selftest_alloc(0); + selftest_free(addr2); + selftest_free(addr1); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 2)) + return -EINVAL; + + selftest_free(addr1); + + /* Test chunk splitting */ + if (addr1 != selftest_alloc(0)) + return -EINVAL; + if (addr2 != selftest_alloc(0)) + return -EINVAL; + + /* Test chunk backward merging */ + selftest_free(addr1); + selftest_free(addr2); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 2)) + return -EINVAL; + + selftest_free(addr1); + + /* Test chunk 3-way merging */ + addr1 = selftest_alloc(0); + addr2 = selftest_alloc(0); + addr4 = selftest_alloc(0); + selftest_free(addr1); + selftest_free(addr3); + selftest_free(addr2); + if (addr1 != selftest_alloc(MIN_ALLOC_SIZE * 3)) + return -EINVAL; + + selftest_free(addr4); + selftest_free(addr1); + + /* Test reclaiming */ + if (addr1 != selftest_alloc(0)) + return -EINVAL; + if (addr2 != selftest_alloc(PAGE_SIZE * 2)) + return -EINVAL; + addr3 = selftest_alloc(0); + addr4 = selftest_alloc(PAGE_SIZE); + + /* Test reclaiming the last chunk of the list */ + selftest_free(addr4); + hyp_allocator_reclaim(allocator, &host_mc, SELFTEST_MAX_PAGES); + if (host_mc.nr_pages != SELFTEST_MAX_PAGES - 3) + return -EINVAL; + + /* Test punching a hole in the middle of a free chunk ... */ + selftest_free(addr2); + hyp_allocator_reclaim(allocator, &host_mc, SELFTEST_MAX_PAGES); + if (host_mc.nr_pages != SELFTEST_MAX_PAGES - 2) + return -EINVAL; + + if (selftest_alloc(PAGE_SIZE)) + return -EINVAL; + if (selftest_errno() != -ENOMEM) + return -EINVAL; + + /* ... and to refill this hole */ + ret = hyp_allocator_topup(allocator, &host_mc); + if (ret) + return ret; + /* Chunk at addr2 was made smaller by the reclaim */ + if (addr2 != selftest_alloc(PAGE_SIZE)) + return -EINVAL; + + /* Test reclaiming the entire allocator from the host */ + selftest_free(addr3); + selftest_free(addr2); + selftest_free(addr1); + if (addr1 != selftest_alloc(SELFTEST_MAX_PAGES * PAGE_SIZE - chunk_hdr_size())) + return -EINVAL; + selftest_free(addr1); + + return 0; +} +#else +static int selftest_init(void) { return 0; } +#endif diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/events.c index add9383aadb5..eba14365e98b 100644 --- a/arch/arm64/kvm/hyp/nvhe/events.c +++ b/arch/arm64/kvm/hyp/nvhe/events.c @@ -7,8 +7,16 @@ #include #include +#include + #include +/* + * The hyp_printk event is not declared with HYP_EVENT in kvm_hypevents.h, + * so we manually add the boilerplate here. + */ +HYP_EVENT(hyp_printk, 0, 0, 0, 0); + int __tracing_enable_event(unsigned short id, bool enable) { struct hyp_event_id *event_id = &__hyp_event_ids_start[id]; diff --git a/arch/arm64/kvm/hyp/nvhe/ffa.c b/arch/arm64/kvm/hyp/nvhe/ffa.c index a327c2bbb6b6..5ad74ab265b5 100644 --- a/arch/arm64/kvm/hyp/nvhe/ffa.c +++ b/arch/arm64/kvm/hyp/nvhe/ffa.c @@ -42,6 +42,9 @@ */ #define HOST_FFA_ID 0 +#define FFA_NOTIF_RECEIVER_ENDP_MASK GENMASK(15, 0) +#define FFA_NOTIF_SENDER_ENDP_MASK GENMASK(31, 16) + /* * A buffer to hold the maximum descriptor size we can see from the host, * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP @@ -71,6 +74,21 @@ static u32 hyp_ffa_version; static bool has_version_negotiated; static hyp_spinlock_t version_lock; +static bool ffa_check_unused_args_sbz(struct kvm_cpu_context *ctxt, int first_reg) +{ + DECLARE_REG(u32, func_id, ctxt, 0); + int reg, end_reg = 7; + + if (hyp_ffa_version >= FFA_VERSION_1_2 && ARM_SMCCC_IS_64(func_id)) + end_reg = 17; + for (reg = first_reg; reg <= end_reg; reg++) { + if (cpu_reg(ctxt, reg)) + return true; + } + + return false; +} + static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno) { *res = (struct arm_smccc_1_2_regs) { @@ -239,6 +257,11 @@ static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res, int ret = 0; void *rx_virt, *tx_virt; + if (ffa_check_unused_args_sbz(ctxt, 4)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) { ret = FFA_RET_INVALID_PARAMETERS; goto out; @@ -315,6 +338,11 @@ static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, id, ctxt, 1); int ret = 0; + if (ffa_check_unused_args_sbz(ctxt, 2)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (id != HOST_FFA_ID) { ret = FFA_RET_INVALID_PARAMETERS; goto out; @@ -421,6 +449,11 @@ static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, int ret = FFA_RET_INVALID_PARAMETERS; u32 nr_ranges; + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) goto out; @@ -483,6 +516,11 @@ static void __do_ffa_mem_xfer(const u64 func_id, int ret = 0; size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version); + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ret = FFA_RET_INVALID_PARAMETERS; + goto out; + } + if (addr_mbz || npages_mbz || fraglen > len || fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { ret = FFA_RET_INVALID_PARAMETERS; @@ -586,6 +624,11 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, int ret = 0; u64 handle; + if (ffa_check_unused_args_sbz(ctxt, 4)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + handle = PACK_HANDLE(handle_lo, handle_hi); hyp_spin_lock(&host_buffers.lock); @@ -691,14 +734,6 @@ static bool ffa_call_supported(u64 func_id) case FFA_RXTX_MAP: case FFA_MEM_DONATE: case FFA_MEM_RETRIEVE_REQ: - /* Optional notification interfaces added in FF-A 1.1 */ - case FFA_NOTIFICATION_BITMAP_CREATE: - case FFA_NOTIFICATION_BITMAP_DESTROY: - case FFA_NOTIFICATION_BIND: - case FFA_NOTIFICATION_UNBIND: - case FFA_NOTIFICATION_SET: - case FFA_NOTIFICATION_GET: - case FFA_NOTIFICATION_INFO_GET: /* Optional interfaces added in FF-A 1.2 */ case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */ case FFA_MSG_SEND_DIRECT_RESP2: /* Optional per 7.5.1 */ @@ -785,6 +820,11 @@ static void do_ffa_version(struct arm_smccc_1_2_regs *res, { DECLARE_REG(u32, ffa_req_version, ctxt, 1); + if (ffa_check_unused_args_sbz(ctxt, 2)) { + res->a0 = FFA_RET_NOT_SUPPORTED; + return; + } + if (FFA_MAJOR_VERSION(ffa_req_version) != 1) { res->a0 = FFA_RET_NOT_SUPPORTED; return; @@ -834,6 +874,11 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, DECLARE_REG(u32, flags, ctxt, 5); u32 count, partition_sz, copy_sz; + if (ffa_check_unused_args_sbz(ctxt, 6)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + hyp_spin_lock(&host_buffers.lock); if (!host_buffers.rx) { ffa_to_smccc_res(res, FFA_RET_BUSY); @@ -878,6 +923,140 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, hyp_spin_unlock(&host_buffers.lock); } +static void do_ffa_notif_bitmap(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, func_id, ctxt, 0); + DECLARE_REG(u32, vmid, ctxt, 1); + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, func_id == FFA_NOTIFICATION_BITMAP_CREATE ? 3 : 2)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (vmid != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_bind(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags > 1) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_unbind(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, reserved, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, 5) || reserved) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_set(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (FIELD_GET(FFA_NOTIF_SENDER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (ffa_check_unused_args_sbz(ctxt, 5)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags & GENMASK(15, 2)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_get(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + DECLARE_REG(u32, endp_id, ctxt, 1); + DECLARE_REG(u32, flags, ctxt, 2); + struct arm_smccc_1_2_regs *args; + + if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (ffa_check_unused_args_sbz(ctxt, 3)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + if (flags & GENMASK(31, 4)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + +static void do_ffa_notif_info_get(struct arm_smccc_1_2_regs *res, + struct kvm_cpu_context *ctxt) +{ + struct arm_smccc_1_2_regs *args; + + if (ffa_check_unused_args_sbz(ctxt, 1)) { + ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS); + return; + } + + args = (void *)&ctxt->regs.regs[0]; + hyp_smccc_1_2_smc(args, res); +} + bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) { struct arm_smccc_1_2_regs res = {0}; @@ -906,6 +1085,11 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) switch (func_id) { case FFA_FEATURES: + if (ffa_check_unused_args_sbz(host_ctxt, 3)) { + ffa_to_smccc_res(&res, FFA_RET_INVALID_PARAMETERS); + goto out_handled; + } + if (!do_ffa_features(&res, host_ctxt)) return false; goto out_handled; @@ -936,6 +1120,26 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) case FFA_PARTITION_INFO_GET: do_ffa_part_get(&res, host_ctxt); goto out_handled; + case FFA_NOTIFICATION_BITMAP_CREATE: + case FFA_NOTIFICATION_BITMAP_DESTROY: + do_ffa_notif_bitmap(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_BIND: + do_ffa_notif_bind(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_UNBIND: + do_ffa_notif_unbind(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_SET: + do_ffa_notif_set(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_GET: + do_ffa_notif_get(&res, host_ctxt); + goto out_handled; + case FFA_NOTIFICATION_INFO_GET: + case FFA_FN64_NOTIFICATION_INFO_GET: + do_ffa_notif_info_get(&res, host_ctxt); + goto out_handled; } if (ffa_call_supported(func_id)) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index ac64a036b0a9..137a8144d09e 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -4,10 +4,13 @@ * Author: Andrew Scull */ +#include + #include #include #include +#include #include #include @@ -17,6 +20,7 @@ #include #include +#include #include #include #include @@ -26,11 +30,449 @@ DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params); +/* + * Define a hypercall handler: handle_ unmarshals the arguments from + * the host context and hands them, correctly typed, to the body that + * follows the macro. The parameter list is type-checked against the + * signature declared in , so the handler cannot drift + * from what the typed caller stubs marshal in. Modelled on the syscall + * wrappers. + */ +/* Truncate the fixed list of argument registers to the declared signature. */ +#define KVM_HOST_HCALL_REGS(...) \ + __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), __KVM_HCALL_ARGS \ + ,, cpu_reg(host_ctxt, 1),, cpu_reg(host_ctxt, 2) \ + ,, cpu_reg(host_ctxt, 3),, cpu_reg(host_ctxt, 4) \ + ,, cpu_reg(host_ctxt, 5),, cpu_reg(host_ctxt, 6)) + +#define set_cpu_reg_ulong(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_u64(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_int(ctxt, r, v) { cpu_reg(ctxt, r) = v; } +#define set_cpu_reg_void(ctxt, r, v) { v; } +#define set_cpu_reg(ctxt, r, t, v) set_cpu_reg_##t(ctxt, r, v) + +#define DEFINE_KVM_HOST_HCALL(ret, name, ...) \ + static kvm_host_hcall_sig_##name __do_##name; \ + static __always_inline \ + ret __se_##name(__KVM_HCALL_MAP(__KVM_HCALL_LONG, __VA_ARGS__)) \ + { \ + return __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_CAST, __VA_ARGS__)); \ + } \ + static void handle_##name(struct kvm_cpu_context *host_ctxt) \ + { \ + set_cpu_reg(host_ctxt, 1, ret, __se_##name(KVM_HOST_HCALL_REGS(__VA_ARGS__))); \ + } \ + static ret __do_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) + +#define DEFINE_KVM_HOST_HCALL0(ret, name) \ + static kvm_host_hcall_sig_##name __do_##name; \ + static void handle_##name(struct kvm_cpu_context *host_ctxt) \ + { \ + set_cpu_reg(host_ctxt, 1, ret, __do_##name()); \ + } \ + static ret __do_##name(void) + +/* + * Encode a hypervisor request in the host SMCCC return registers for known + * error numbers. + * + * Must be paired with pkvm_call_hyp_req() on the host side. + */ +static int errno_to_smccc(int ret) +{ + struct pkvm_hyp_req req = { .type = PKVM_HYP_NO_REQ }; + + switch (ret) { + case -ENOMEM: { + u32 nr_pages = hyp_alloc_topup_needed(); + + if (nr_pages) { + req.type = PKVM_HYP_REQ_HYP_ALLOC; + req.mem.nr_pages = nr_pages; + } + break; + } + } + + pkvm_hyp_req_to_smccc(host_data_ptr(host_ctxt), &req); + + return ret; +} + /* Number of implemented GICv3 LRs. Used by flush_hyp_vcpu(). */ unsigned int hyp_gicv3_nr_lr; void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt); +typedef void (*hyp_entry_exit_handler_fn)(struct pkvm_hyp_vcpu *); + +static bool pvm_sys64_is_write(u64 esr) +{ + return (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_WRITE; +} + +static void handle_pvm_entry_wfx(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + /* Exceptions have priority; the host injects none on WFx. */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) + return; + + if (vcpu_get_flag(host_vcpu, INCREMENT_PC)) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } +} + +static void handle_pvm_entry_sys64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + bool pc_update; + + /* Exceptions have priority over anything else */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) { + /* A host-requested exception on SYS64 is always an UNDEF. */ + u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT) | ESR_ELx_IL; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); + return; + } + + /* Handle PC increment on a host-emulated access */ + pc_update = vcpu_get_flag(host_vcpu, INCREMENT_PC); + if (pc_update) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } + + /* If the host emulated a read access, update the register */ + if (pc_update && !pvm_sys64_is_write(kvm_vcpu_get_esr(vcpu))) { + /* r0 as transfer register between the guest and the host. */ + u64 rt_val = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); + int rt = kvm_vcpu_sys_get_rt(vcpu); + + vcpu_set_reg(vcpu, rt, rt_val); + } +} + +static void handle_pvm_entry_iabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + unsigned long cpsr = *vcpu_cpsr(vcpu); + u32 esr = ESR_ELx_IL; + + if (!vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) + return; + + /* The host's only IABT injection: an external abort. */ + if ((cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) + esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT); + else + esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT); + + esr |= ESR_ELx_FSC_EXTABT; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + __vcpu_assign_sys_reg(vcpu, FAR_EL1, kvm_vcpu_get_hfar(vcpu)); + + /* Injected by __kvm_adjust_pc() on entry. */ + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); +} + +/* + * Clamp MMIO data to the access width, so a write does not leak the + * register's upper bits and a read takes no bits beyond the load. The + * host applies endianness. + */ +static inline u64 kvm_mmio_clamp_data(struct kvm_vcpu *vcpu, u64 val) +{ + unsigned int len = kvm_vcpu_dabt_get_as(vcpu); + + return val & GENMASK_U64(len * 8 - 1, 0); +} + +/* + * Complete an MMIO load: sign-extend from EL2's own syndrome, as the + * architecture does. + */ +static inline u64 kvm_mmio_read_data(struct kvm_vcpu *vcpu, u64 val) +{ + val = kvm_mmio_clamp_data(vcpu, val); + + if (kvm_vcpu_dabt_issext(vcpu)) + val = sign_extend64(val, kvm_vcpu_dabt_get_as(vcpu) * 8 - 1); + + if (!kvm_vcpu_dabt_issf(vcpu)) + val &= GENMASK_U64(31, 0); + + return val; +} + +static void handle_pvm_entry_dabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + bool pc_update; + + /* Exceptions have priority over anything else */ + if (vcpu_get_flag(host_vcpu, PENDING_EXCEPTION)) { + unsigned long cpsr = *vcpu_cpsr(vcpu); + u32 esr = ESR_ELx_IL; + + if ((cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) + esr |= (ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT); + else + esr |= (ESR_ELx_EC_DABT_CUR << ESR_ELx_EC_SHIFT); + + esr |= ESR_ELx_FSC_EXTABT; + + __vcpu_assign_sys_reg(vcpu, ESR_EL1, esr); + __vcpu_assign_sys_reg(vcpu, FAR_EL1, kvm_vcpu_get_hfar(vcpu)); + + /* Injected by __kvm_adjust_pc() on entry. */ + kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); + + /* Cancel any in-flight MMIO */ + vcpu->mmio_needed = false; + return; + } + + /* Handle PC increment on MMIO, or on a CMO the host skipped */ + pc_update = vcpu_get_flag(host_vcpu, INCREMENT_PC) && + (vcpu->mmio_needed || esr_dabt_is_cm(kvm_vcpu_get_esr(vcpu))); + if (pc_update) { + vcpu_clear_flag(vcpu, PC_UPDATE_REQ); + kvm_incr_pc(vcpu); + } + + /* If the host emulated an MMIO read, update the register */ + if (pc_update && vcpu->mmio_needed && !kvm_vcpu_dabt_iswrite(vcpu)) { + /* r0 as transfer register between the guest and the host. */ + u64 rd_val = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); + int rd = kvm_vcpu_dabt_get_rd(vcpu); + + rd_val = kvm_mmio_read_data(vcpu, rd_val); + vcpu_set_reg(vcpu, rd, rd_val); + } + + vcpu->mmio_needed = false; +} + +static void handle_pvm_entry_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 ret = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); + u32 psci_fn = smccc_get_function(vcpu); + + switch (psci_fn) { + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + /* + * Roll back a CPU_ON the host failed, unless the target + * already reached ON: it is running, and the guest sees + * SUCCESS. + */ + if (ret != PSCI_RET_SUCCESS) { + unsigned long cpu_id = smccc_get_arg1(vcpu); + struct pkvm_hyp_vcpu *target_vcpu; + struct pkvm_hyp_vm *hyp_vm; + int prev; + + hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + target_vcpu = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); + + /* + * pvm_psci_vcpu_on() resolved this MPIDR and vcpus[] + * entries are never removed, so the lookup cannot miss. + * The release orders this vCPU's reset_state writes + * before OFF, for the next CPU_ON. + */ + prev = cmpxchg_release(&target_vcpu->power_state, + PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, + PSCI_0_2_AFFINITY_LEVEL_OFF); + switch (prev) { + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + /* + * Leave reset_state.reset set: a clear races a + * fresh CPU_ON's publish. The stale pc/r0/be are + * the guest's own. ALREADY_ON is PSCI's retry + * signal for a CPU_ON that raced the CPU_OFF. + */ + if (ret != PSCI_RET_ALREADY_ON) + ret = PSCI_RET_INTERNAL_FAILURE; + break; + case PSCI_0_2_AFFINITY_LEVEL_ON: + case PSCI_0_2_AFFINITY_LEVEL_OFF: + /* Target already ran (and may have stopped). */ + ret = PSCI_RET_SUCCESS; + break; + default: + ret = PSCI_RET_INTERNAL_FAILURE; + break; + } + } + + break; + default: + break; + } + + vcpu_set_reg(vcpu, 0, ret); +} + +/* The host's view of a syndrome: the guest register index is withheld. */ +static u64 pvm_host_esr(u64 esr) +{ + switch (ESR_ELx_EC(esr)) { + case ESR_ELx_EC_WFx: + return esr & ~ESR_ELx_WFx_ISS_RN; + case ESR_ELx_EC_SYS64: + return esr & ~ESR_ELx_SYS64_ISS_RT_MASK; + case ESR_ELx_EC_DABT_LOW: + return esr & ~ESR_ELx_SRT_MASK; + default: + return esr; + } +} + +/* + * The host's view of PSTATE: the mode, with SErrors masked so that the host + * pends an SError through HCR_EL2.VSE rather than emulating the entry. + */ +static unsigned long pvm_host_pstate(unsigned long pstate) +{ + return (pstate & PSR_MODE_MASK) | PSR_A_BIT; +} + +static void handle_pvm_exit_wfx(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); +} + +static void handle_pvm_exit_sys64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 esr = kvm_vcpu_get_esr(vcpu); + + /* The mode is required for the host to emulate some sysregs */ + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); + + /* r0 as transfer register between the guest and the host. */ + if (pvm_sys64_is_write(esr)) { + int rt = kvm_vcpu_sys_get_rt(vcpu); + u64 rt_val = vcpu_get_reg(vcpu, rt); + + vcpu_set_reg(host_vcpu, 0, rt_val); + } +} + +static void handle_pvm_exit_iabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + host_vcpu->arch.fault.hpfar_el2 = vcpu->arch.fault.hpfar_el2; +} + +static void handle_pvm_exit_dabt(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u64 sctlr; + + /* + * EL2 has no memslot view: a decodable data abort is prepared as MMIO + * for the host to resolve. On unbacked memory the host injects an SEA + * for one with ISV clear (LDP/STP, atomics), which EL2 does not + * decode, and skips a cache maintenance operation, as for any guest. + */ + vcpu->mmio_needed = kvm_vcpu_dabt_isvalid(vcpu); + + /* r0 as transfer register between the guest and the host. */ + if (vcpu->mmio_needed && kvm_vcpu_dabt_iswrite(vcpu)) { + int rt = kvm_vcpu_dabt_get_rd(vcpu); + u64 rt_val = vcpu_get_reg(vcpu, rt); + + rt_val = kvm_mmio_clamp_data(vcpu, rt_val); + vcpu_set_reg(host_vcpu, 0, rt_val); + } + + *vcpu_cpsr(host_vcpu) = pvm_host_pstate(*vcpu_cpsr(vcpu)); + host_vcpu->arch.fault.far_el2 = vcpu->arch.fault.far_el2 & GENMASK(11, 0); + host_vcpu->arch.fault.hpfar_el2 = vcpu->arch.fault.hpfar_el2; + sctlr = __vcpu_sys_reg(vcpu, SCTLR_EL1) & (SCTLR_ELx_EE | SCTLR_EL1_E0E); + __vcpu_assign_sys_reg(host_vcpu, SCTLR_EL1, sctlr); +} + +static void handle_pvm_exit_hvc64(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + int n, i; + + switch (smccc_get_function(vcpu)) { + /* + * CPU_ON: the host uses only the target MPIDR (x1). EL2 resets the + * target from its own copy of the entry point and context id. + */ + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + n = 2; + break; + + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + n = 1; + break; + + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + n = 3; + break; + + /* Unreachable: kvm_handle_pvm_hvc64() forwards only the calls above. */ + default: + hyp_panic(); + } + + /* Pass the HVC function id (r0) and its arguments. */ + for (i = 0; i < n; i++) + vcpu_set_reg(host_vcpu, i, vcpu_get_reg(vcpu, i)); +} + +static const hyp_entry_exit_handler_fn entry_hyp_pvm_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_WFx] = handle_pvm_entry_wfx, + [ESR_ELx_EC_SYS64] = handle_pvm_entry_sys64, + [ESR_ELx_EC_IABT_LOW] = handle_pvm_entry_iabt, + [ESR_ELx_EC_DABT_LOW] = handle_pvm_entry_dabt, + [ESR_ELx_EC_HVC64] = handle_pvm_entry_hvc64, +}; + +static const hyp_entry_exit_handler_fn exit_hyp_pvm_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = NULL, + [ESR_ELx_EC_WFx] = handle_pvm_exit_wfx, + [ESR_ELx_EC_SYS64] = handle_pvm_exit_sys64, + [ESR_ELx_EC_IABT_LOW] = handle_pvm_exit_iabt, + [ESR_ELx_EC_DABT_LOW] = handle_pvm_exit_dabt, + [ESR_ELx_EC_HVC64] = handle_pvm_exit_hvc64, +}; + static void __hyp_sve_save_guest(struct kvm_vcpu *vcpu) { __vcpu_assign_sys_reg(vcpu, ZCR_EL1, read_sysreg_el1(SYS_ZCR)); @@ -143,6 +585,35 @@ static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; } +static void flush_hyp_timer_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + + /* A hyp vcpu has no offset, and sees vtime == ptime. */ + write_sysreg(0, cntvoff_el2); + write_sysreg_el0(__vcpu_sys_reg(vcpu, CNTV_CVAL_EL0), SYS_CNTV_CVAL); + isb(); + write_sysreg_el0(__vcpu_sys_reg(vcpu, CNTV_CTL_EL0), SYS_CNTV_CTL); +} + +static void sync_hyp_timer_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + + /* + * Preserve the vtimer state so that it is always correct, + * even if the host tries to make a mess. + */ + __vcpu_assign_sys_reg(vcpu, CNTV_CVAL_EL0, read_sysreg_el0(SYS_CNTV_CVAL)); + __vcpu_assign_sys_reg(vcpu, CNTV_CTL_EL0, read_sysreg_el0(SYS_CNTV_CTL)); +} + static void __copy_vcpu_state(const struct kvm_vcpu *from_vcpu, struct kvm_vcpu *to_vcpu) { @@ -189,6 +660,9 @@ static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + hyp_vcpu->vcpu.arch.debug_owner = host_vcpu->arch.debug_owner; if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) { @@ -207,6 +681,9 @@ static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) host_vcpu->arch.vcpu_debug_state = hyp_vcpu->vcpu.arch.vcpu_debug_state; else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) @@ -217,6 +694,8 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; u64 host_hcr_mask = PKVM_HCR_EL2_HOST_PVM; + hyp_entry_exit_handler_fn ec_handler; + u8 esr_ec; fpsimd_sve_flush(); flush_debug_state(hyp_vcpu); @@ -229,42 +708,75 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) flush_hyp_vcpu_state(hyp_vcpu); + + hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; + hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; host_hcr_mask = PKVM_HCR_EL2_HOST_NPVM; - } else { - hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; } /* __hyp_running_vcpu must be NULL in a guest context. */ hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; - hyp_vcpu->vcpu.arch.mdcr_el2 = host_vcpu->arch.mdcr_el2; /* - * HCR_EL2.VSE is host-owned (a pending virtual SError to inject), not a - * trap-control bit, so it must flow to the hyp vCPU alongside TWI/TWE - * for the vSError to be delivered. sync_hyp_vcpu() reflects it back. + * A host-injected vSError is masked by the guest's own PSTATE.A, so it + * applies to protected guests too. */ hyp_vcpu->vcpu.arch.hcr_el2 &= ~host_hcr_mask; hyp_vcpu->vcpu.arch.hcr_el2 |= READ_ONCE(host_vcpu->arch.hcr_el2) & host_hcr_mask; - - hyp_vcpu->vcpu.arch.iflags = host_vcpu->arch.iflags; - - hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; + hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; flush_hyp_vgic_state(hyp_vcpu); + flush_hyp_timer_state(hyp_vcpu); hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; + + switch (ARM_EXCEPTION_CODE(hyp_vcpu->exit_code)) { + case ARM_EXCEPTION_IRQ: + case ARM_EXCEPTION_EL1_SERROR: + case ARM_EXCEPTION_IL: + break; + case ARM_EXCEPTION_TRAP: + /* Nothing was marshalled for this trap, see sync_hyp_vcpu(). */ + if (ARM_SERROR_PENDING(hyp_vcpu->exit_code)) + break; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); + ec_handler = entry_hyp_pvm_handlers[esr_ec]; + if (ec_handler) + ec_handler(hyp_vcpu); + } + break; + default: + BUG(); + } + + hyp_vcpu->exit_code = 0; } -static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) +static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, u32 exit_reason) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + hyp_entry_exit_handler_fn ec_handler; + u8 esr_ec; fpsimd_sve_sync(&hyp_vcpu->vcpu); sync_debug_state(hyp_vcpu); if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { - host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; + /* + * Protected: the host sees ESR_EL2 as EL2 took it, register + * index withheld; the fault addresses stay withheld unless the + * EC handler below adds them. + */ + host_vcpu->arch.fault = (struct kvm_vcpu_fault_info) { + .esr_el2 = pvm_host_esr(hyp_vcpu->vcpu.arch.fault.esr_el2), + .disr_el1 = hyp_vcpu->vcpu.arch.fault.disr_el1, + }; } else { + /* Non-protected: the host gets the full fault. */ + host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; + host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; /* * PC feeds trace_kvm_exit(), PSTATE.SS the host software-step * machine, and both run before the next on-demand ctxt sync. @@ -273,18 +785,45 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) host_vcpu->arch.ctxt.regs.pstate = hyp_vcpu->vcpu.arch.ctxt.regs.pstate; } - host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; + switch (ARM_EXCEPTION_CODE(exit_reason)) { + case ARM_EXCEPTION_IRQ: + break; + case ARM_EXCEPTION_TRAP: + /* SError pending: not handled at EL2, the guest replays it. */ + if (ARM_SERROR_PENDING(exit_reason)) + break; - host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; + /* Per-EC marshalling is for protected guests only. */ + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + esr_ec = ESR_ELx_EC(kvm_vcpu_get_esr(&hyp_vcpu->vcpu)); + ec_handler = exit_hyp_pvm_handlers[esr_ec]; + if (ec_handler) + ec_handler(hyp_vcpu); + } + break; + case ARM_EXCEPTION_EL1_SERROR: + case ARM_EXCEPTION_IL: + break; + default: + BUG(); + } + + /* Cleared by hardware once the guest takes the vSError. */ + host_vcpu->arch.hcr_el2 &= ~HCR_VSE; + host_vcpu->arch.hcr_el2 |= hyp_vcpu->vcpu.arch.hcr_el2 & HCR_VSE; sync_hyp_vgic_state(hyp_vcpu); + sync_hyp_timer_state(hyp_vcpu); + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + vcpu_clear_flag(host_vcpu, PC_UPDATE_REQ); + + hyp_vcpu->exit_code = exit_reason; } -static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_vcpu_load, + pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(unsigned int, vcpu_idx, host_ctxt, 2); - DECLARE_REG(u64, hcr_el2, host_ctxt, 3); struct pkvm_hyp_vcpu *hyp_vcpu; hyp_vcpu = pkvm_load_hyp_vcpu(handle, vcpu_idx); @@ -292,16 +831,24 @@ static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) return; if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + u64 dfr0 = read_sysreg(id_aa64dfr0_el1); + /* Propagate WFx trapping flags */ hyp_vcpu->vcpu.arch.hcr_el2 &= ~(HCR_TWE | HCR_TWI); hyp_vcpu->vcpu.arch.hcr_el2 |= hcr_el2 & (HCR_TWE | HCR_TWI); + + /* HPMN == 0 is reserved without FEAT_HPMN0. */ + if (pmuv3_implemented(SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, dfr0))) + u64p_replace_bits(&hyp_vcpu->vcpu.arch.mdcr_el2, + FIELD_GET(ARMV8_PMU_PMCR_N, read_sysreg(pmcr_el0)), + MDCR_EL2_HPMN); } else { memcpy(&hyp_vcpu->vcpu.arch.fgt, hyp_vcpu->host_vcpu->arch.fgt, sizeof(hyp_vcpu->vcpu.arch.fgt)); } } -static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put) { struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); @@ -317,7 +864,7 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) } } -static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state) { struct pkvm_hyp_vcpu *hyp_vcpu; @@ -328,10 +875,9 @@ static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) sync_hyp_vcpu_state(hyp_vcpu); } -static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, +static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *host_vcpu, struct pkvm_hyp_vcpu **hyp_vcpup) { - struct kvm_vcpu *host_vcpu = kern_hyp_va(arg); struct pkvm_hyp_vcpu *hyp_vcpu = NULL; if (unlikely(is_protected_kvm_enabled())) { @@ -347,34 +893,26 @@ static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, return host_vcpu; } -#define get_host_hyp_vcpus(ctxt, regnr, hyp_vcpup) \ - ({ \ - DECLARE_REG(struct kvm_vcpu *, __vcpu, ctxt, regnr); \ - __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ - }) +static struct kvm_vcpu * +__get_host_hyp_vcpus_from_vgic_v3_cpu_if(struct vgic_v3_cpu_if __kern *cpu_if, + struct pkvm_hyp_vcpu **hyp_vcpup) +{ + struct vgic_v3_cpu_if *host_cpu_if = kern_hyp_va_host(cpu_if); + struct kvm_vcpu *host_vcpu = container_of(host_cpu_if, struct kvm_vcpu, + arch.vgic_cpu.vgic_v3); -#define get_host_hyp_vcpus_from_vgic_v3_cpu_if(ctxt, regnr, hyp_vcpup) \ - ({ \ - DECLARE_REG(struct vgic_v3_cpu_if *, cif, ctxt, regnr);\ - struct kvm_vcpu *__vcpu = container_of(cif, \ - struct kvm_vcpu, \ - arch.vgic_cpu.vgic_v3); \ - \ - __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ - }) + return __get_host_hyp_vcpus(host_vcpu, hyp_vcpup); +} -static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __kvm_vcpu_run, struct kvm_vcpu __kern *, vcpu) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - int ret; + int ret = ARM_EXCEPTION_IL; - host_vcpu = get_host_hyp_vcpus(host_ctxt, 1, &hyp_vcpu); - - if (!host_vcpu) { - ret = -EINVAL; - goto out; - } + host_vcpu = __get_host_hyp_vcpus(kern_hyp_va_host(vcpu), &hyp_vcpu); + if (!host_vcpu) + return ret; if (unlikely(hyp_vcpu)) { /* @@ -383,213 +921,245 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) * loading a vcpu. Therefore, if SME features enabled the host * is misbehaving. */ - if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) { - ret = -EINVAL; - goto out; + if (unlikely(system_supports_sme() && read_sysreg_s(SYS_SVCR))) + return ret; + + /* + * ON has a single writer, pkvm_reset_vcpu() on this CPU, so + * READ_ONCE suffices. ON_PENDING takes the reset; -ECANCELED + * is a rollback that raced it. + */ + switch (READ_ONCE(hyp_vcpu->power_state)) { + case PSCI_0_2_AFFINITY_LEVEL_ON: + break; + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + if (pkvm_reset_vcpu(hyp_vcpu)) + return ret; + break; + default: + return ret; } flush_hyp_vcpu(hyp_vcpu); ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); - sync_hyp_vcpu(hyp_vcpu); + sync_hyp_vcpu(hyp_vcpu, ret); } else { /* The host is fully trusted, run its vCPU directly. */ fpsimd_lazy_switch_to_guest(host_vcpu); ret = __kvm_vcpu_run(host_vcpu); fpsimd_lazy_switch_to_host(host_vcpu); } -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } static int pkvm_refill_memcache(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; - return refill_memcache(&hyp_vcpu->vcpu.arch.pkvm_memcache, - host_vcpu->arch.pkvm_memcache.nr_pages, - &host_vcpu->arch.pkvm_memcache); + return refill_memcache(&hyp_vcpu->vcpu.arch.stage2_mc, + host_vcpu->arch.stage2_mc.nr_pages, + &host_vcpu->arch.stage2_mc); } -static void handle___pkvm_host_donate_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest, + u64, pfn, u64, gfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; + int ret; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || !pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; ret = pkvm_refill_memcache(hyp_vcpu); if (ret) - goto out; + return ret; - ret = __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_donate_guest(pfn, gfn, hyp_vcpu); } -static void handle___pkvm_host_share_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_guest, + u64, pfn, u64, gfn, u64, nr_pages, u64, prot) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 4); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; + int ret; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; ret = pkvm_refill_memcache(hyp_vcpu); if (ret) - goto out; + return ret; - ret = __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_share_guest(pfn, gfn, nr_pages, hyp_vcpu, prot); } -static void handle___pkvm_host_unshare_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_unshare_guest(gfn, nr_pages, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_relax_perms_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest, + u64, gfn, u64, prot) { - DECLARE_REG(u64, gfn, host_ctxt, 1); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 2); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; - ret = __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_relax_perms_guest(gfn, hyp_vcpu, prot); } -static void handle___pkvm_host_wrprotect_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_wrprotect_guest(gfn, nr_pages, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_test_clear_young_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest, + pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - DECLARE_REG(u64, nr_pages, host_ctxt, 3); - DECLARE_REG(bool, mkold, host_ctxt, 4); struct pkvm_hyp_vm *hyp_vm; - int ret = -EINVAL; + int ret; hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) - goto out; + return -EINVAL; ret = __pkvm_host_test_clear_young_guest(gfn, nr_pages, mkold, hyp_vm); put_pkvm_hyp_vm(hyp_vm); -out: - cpu_reg(host_ctxt, 1) = ret; + + return ret; } -static void handle___pkvm_host_mkyoung_guest(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest, + u64, gfn) { - DECLARE_REG(u64, gfn, host_ctxt, 1); struct pkvm_hyp_vcpu *hyp_vcpu; - int ret = -EINVAL; hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) - goto out; + return -EINVAL; - ret = __pkvm_host_mkyoung_guest(gfn, hyp_vcpu); -out: - cpu_reg(host_ctxt, 1) = ret; + return __pkvm_host_mkyoung_guest(gfn, hyp_vcpu); } -static void handle___kvm_adjust_pc(struct kvm_cpu_context *host_ctxt) +/* + * PKVM_HOST_STATE_DIRTY names the authoritative copy, the host's when set. + * A loaded protected vCPU takes the request at its next entry instead. + */ +struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu) { - DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1); + struct pkvm_hyp_vcpu *hyp_vcpu; - __kvm_adjust_pc(kern_hyp_va(vcpu)); + if (!is_protected_kvm_enabled()) + return vcpu; + + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu || vcpu == &hyp_vcpu->vcpu) + return vcpu; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return NULL; + + if (vcpu_get_flag(vcpu, PKVM_HOST_STATE_DIRTY)) + return vcpu; + + vcpu_copy_flag(&hyp_vcpu->vcpu, vcpu, PC_UPDATE_REQ); + return &hyp_vcpu->vcpu; } -static void handle___kvm_flush_vm_context(struct kvm_cpu_context *host_ctxt) +/* Reflect the consumed request back, otherwise it stays pending. */ +void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, struct kvm_vcpu *target) +{ + if (target != vcpu) + vcpu_copy_flag(vcpu, target, PC_UPDATE_REQ); +} + +DEFINE_KVM_HOST_HCALL(void, __kvm_adjust_pc, + struct kvm_vcpu __kern *, vcpu) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; + + host_vcpu = __get_host_hyp_vcpus(kern_hyp_va_host(vcpu), &hyp_vcpu); + if (host_vcpu) { + __kvm_adjust_pc(host_vcpu); + return; + } + + /* + * With no hyp vCPU loaded for it, the host vCPU may be unpinned, + * and so unmapped at EL2: its first run pins it. A pin fails only + * for memory the host isn't sharing, a bad pointer, so the request + * is dropped. + */ + host_vcpu = kern_hyp_va_host(vcpu); + if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) + return; + + __kvm_adjust_pc(host_vcpu); + hyp_unpin_shared_mem(host_vcpu, host_vcpu + 1); +} + +DEFINE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context) { __kvm_flush_vm_context(); } -static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); - DECLARE_REG(int, level, host_ctxt, 3); - - __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); + __kvm_tlb_flush_vmid_ipa(kern_hyp_va_host(mmu), ipa, level); } -static void handle___kvm_tlb_flush_vmid_ipa_nsh(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, ipa, host_ctxt, 2); - DECLARE_REG(int, level, host_ctxt, 3); - - __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va(mmu), ipa, level); + __kvm_tlb_flush_vmid_ipa_nsh(kern_hyp_va_host(mmu), ipa, level); } -static void -handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range, + struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - DECLARE_REG(phys_addr_t, start, host_ctxt, 2); - DECLARE_REG(unsigned long, pages, host_ctxt, 3); - - __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, pages); + __kvm_tlb_flush_vmid_range(kern_hyp_va_host(mmu), start, pages); } -static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid, + struct kvm_s2_mmu __kern *, mmu) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - - __kvm_tlb_flush_vmid(kern_hyp_va(mmu)); + __kvm_tlb_flush_vmid(kern_hyp_va_host(mmu)); } -static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); struct pkvm_hyp_vm *hyp_vm = get_np_pkvm_hyp_vm(handle); if (!hyp_vm) @@ -599,19 +1169,19 @@ static void handle___pkvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) put_pkvm_hyp_vm(hyp_vm); } -static void handle___kvm_flush_cpu_context(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context, + struct kvm_s2_mmu __kern *, mmu) { - DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); - - __kvm_flush_cpu_context(kern_hyp_va(mmu)); + __kvm_flush_cpu_context(kern_hyp_va_host(mmu)); } -static void handle___kvm_timer_set_cntvoff(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff, + u64, cntvoff) { - __kvm_timer_set_cntvoff(cpu_reg(host_ctxt, 1)); + __kvm_timer_set_cntvoff(cntvoff); } -static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs) { u64 tmp; @@ -620,23 +1190,23 @@ static void handle___kvm_enable_ssbs(struct kvm_cpu_context *host_ctxt) write_sysreg_el2(tmp, SYS_SCTLR); } -static void handle___vgic_v3_get_gic_config(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config) { - cpu_reg(host_ctxt, 1) = __vgic_v3_get_gic_config(); + return __vgic_v3_get_gic_config(); } -static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs) { __vgic_v3_init_lrs(); } -static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs, + struct vgic_v3_cpu_if __kern *, cpu_if) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, - &hyp_vcpu); + host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu); if (!host_vcpu) return; @@ -658,13 +1228,13 @@ static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) } } -static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs, + struct vgic_v3_cpu_if __kern *, cpu_if) { struct pkvm_hyp_vcpu *hyp_vcpu; struct kvm_vcpu *host_vcpu; - host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, - &hyp_vcpu); + host_vcpu = __get_host_hyp_vcpus_from_vgic_v3_cpu_if(cpu_if, &hyp_vcpu); if (!host_vcpu) return; @@ -691,48 +1261,39 @@ static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt } } -static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init, + phys_addr_t, phys, unsigned long, size, + unsigned long *, per_cpu_base, u32, hyp_va_bits) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - DECLARE_REG(unsigned long, size, host_ctxt, 2); - DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 3); - DECLARE_REG(u32, hyp_va_bits, host_ctxt, 4); - /* * __pkvm_init() will return only if an error occurred, otherwise it * will tail-call in __pkvm_init_finalise() which will have to deal * with the host context directly. */ - cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, per_cpu_base, hyp_va_bits); + return __pkvm_init(phys, size, per_cpu_base, hyp_va_bits); } -static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector, + enum arm64_hyp_spectre_vector, slot) { - DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot); + return pkvm_cpu_set_vector(slot); } -static void handle___pkvm_host_share_hyp(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp, + u64, pfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_share_hyp(pfn); + return __pkvm_host_share_hyp(pfn); } -static void handle___pkvm_host_unshare_hyp(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp, + u64, pfn) { - DECLARE_REG(u64, pfn, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_unshare_hyp(pfn); + return __pkvm_host_unshare_hyp(pfn); } -static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping, + phys_addr_t, phys, size_t, size, u64, prot) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - DECLARE_REG(size_t, size, host_ctxt, 2); - DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3); - /* * __pkvm_create_private_mapping() populates a pointer with the * hypervisor start address of the allocation. @@ -743,161 +1304,237 @@ static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ct * Instead pass the allocation address as the return value (or return * ERR_PTR() on failure). */ - unsigned long haddr; + ulong haddr; int err = __pkvm_create_private_mapping(phys, size, prot, &haddr); if (err) - haddr = (unsigned long)ERR_PTR(err); + haddr = (ulong)ERR_PTR(err); - cpu_reg(host_ctxt, 1) = haddr; + return haddr; } -static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize) { - cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize(); + return __pkvm_prot_finalize(); } -static void handle___pkvm_reserve_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm) { - cpu_reg(host_ctxt, 1) = __pkvm_reserve_vm(); + return __pkvm_reserve_vm(); } -static void handle___pkvm_unreserve_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - __pkvm_unreserve_vm(handle); } -static void handle___pkvm_init_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vm, + struct kvm __kern *, host_kvm, void __kern *, pgd_hva) { - DECLARE_REG(struct kvm *, host_kvm, host_ctxt, 1); - DECLARE_REG(unsigned long, vm_hva, host_ctxt, 2); - DECLARE_REG(unsigned long, pgd_hva, host_ctxt, 3); - - host_kvm = kern_hyp_va(host_kvm); - cpu_reg(host_ctxt, 1) = __pkvm_init_vm(host_kvm, vm_hva, pgd_hva); + return errno_to_smccc(__pkvm_init_vm(kern_hyp_va_host(host_kvm), pgd_hva)); } -static void handle___pkvm_init_vcpu(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_init_vcpu, + pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 2); - DECLARE_REG(unsigned long, vcpu_hva, host_ctxt, 3); - - host_vcpu = kern_hyp_va(host_vcpu); - cpu_reg(host_ctxt, 1) = __pkvm_init_vcpu(handle, host_vcpu, vcpu_hva); + return errno_to_smccc(__pkvm_init_vcpu(handle, kern_hyp_va_host(host_vcpu))); } -static void handle___pkvm_vcpu_in_poison_fault(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault) { - int ret; struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - ret = hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL; - cpu_reg(host_ctxt, 1) = ret; + return hyp_vcpu ? __pkvm_vcpu_in_poison_fault(hyp_vcpu) : -EINVAL; } -static void handle___pkvm_force_reclaim_guest_page(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page, + phys_addr_t, phys) { - DECLARE_REG(phys_addr_t, phys, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_host_force_reclaim_page_guest(phys); + return __pkvm_host_force_reclaim_page_guest(phys); } -static void handle___pkvm_reclaim_dying_guest_page(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page, + pkvm_handle_t, handle, u64, gfn) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - DECLARE_REG(u64, gfn, host_ctxt, 2); - - cpu_reg(host_ctxt, 1) = __pkvm_reclaim_dying_guest_page(handle, gfn); + return __pkvm_reclaim_dying_guest_page(handle, gfn); } -static void handle___pkvm_start_teardown_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_start_teardown_vm(handle); + return __pkvm_start_teardown_vm(handle); } -static void handle___pkvm_finalize_teardown_vm(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm, + pkvm_handle_t, handle) { - DECLARE_REG(pkvm_handle_t, handle, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __pkvm_finalize_teardown_vm(handle); + return __pkvm_finalize_teardown_vm(handle); } -static void handle___tracing_load(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest) { - DECLARE_REG(unsigned long, desc_hva, host_ctxt, 1); - DECLARE_REG(size_t, desc_size, host_ctxt, 2); + struct pkvm_hyp_req req = { .type = PKVM_HYP_NO_REQ }; + int ret = -EPERM; - cpu_reg(host_ctxt, 1) = __tracing_load(desc_hva, desc_size); +#ifdef CONFIG_NVHE_EL2_DEBUG + ret = hyp_allocator_selftest(); + if (ret == -ENOMEM) { + req.type = PKVM_HYP_REQ_HYP_ALLOC_SELFTEST; + req.mem.nr_pages = hyp_alloc_selftest_topup_needed(); + } +#endif + pkvm_hyp_req_to_smccc(host_data_ptr(host_ctxt), &req); + + return ret; } -static void handle___tracing_unload(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_topup, + enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages) +{ + struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); + struct kvm_hyp_memcache host_mc = { + .head = head, + .nr_pages = nr_pages, + }; + int ret; + + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + ret = hyp_alloc_topup(&host_mc); + break; + case PKVM_TOPUP_HYP_ALLOC_SELFTEST: + ret = hyp_alloc_selftest_topup(&host_mc); + break; + default: + ret = -EINVAL; + } + + cpu_reg(host_ctxt, 2) = host_mc.head; + cpu_reg(host_ctxt, 3) = host_mc.nr_pages; + + return ret; +} + +DEFINE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim, + enum pkvm_topup_id, id, unsigned long, target) +{ + struct kvm_cpu_context *host_ctxt = host_data_ptr(host_ctxt); + struct kvm_hyp_memcache host_mc = {}; + int ret = 0; + + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + hyp_alloc_reclaim(&host_mc, target); + break; + case PKVM_TOPUP_HYP_ALLOC_SELFTEST: + hyp_alloc_selftest_reclaim(&host_mc, target); + break; + default: + ret = -EINVAL; + } + + cpu_reg(host_ctxt, 2) = host_mc.head; + cpu_reg(host_ctxt, 3) = host_mc.nr_pages; + + return ret; +} + +DEFINE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable, + enum pkvm_topup_id, id) +{ + switch (id) { + case PKVM_TOPUP_HYP_ALLOC: + return hyp_alloc_reclaimable(); + default: + return 0; + } +} + +DEFINE_KVM_HOST_HCALL(int, __tracing_load, + void __kern *, desc_hva, size_t, desc_size) +{ + return errno_to_smccc(__tracing_load(desc_hva, desc_size)); +} + +DEFINE_KVM_HOST_HCALL0(void, __tracing_unload) { __tracing_unload(); } -static void handle___tracing_enable(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_enable, + bool, enable) { - DECLARE_REG(bool, enable, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_enable(enable); + return __tracing_enable(enable); } -static void handle___tracing_swap_reader(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_swap_reader, + unsigned int, cpu) { - DECLARE_REG(unsigned int, cpu, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_swap_reader(cpu); + return __tracing_swap_reader(cpu); } -static void handle___tracing_update_clock(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __tracing_update_clock, + u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc) { - DECLARE_REG(u32, mult, host_ctxt, 1); - DECLARE_REG(u32, shift, host_ctxt, 2); - DECLARE_REG(u64, epoch_ns, host_ctxt, 3); - DECLARE_REG(u64, epoch_cyc, host_ctxt, 4); - __tracing_update_clock(mult, shift, epoch_ns, epoch_cyc); } -static void handle___tracing_reset(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_reset, + unsigned int, cpu) { - DECLARE_REG(unsigned int, cpu, host_ctxt, 1); - - cpu_reg(host_ctxt, 1) = __tracing_reset(cpu); + return __tracing_reset(cpu); } -static void handle___tracing_enable_event(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(int, __tracing_enable_event, + unsigned short, id, bool, enable) { - DECLARE_REG(unsigned short, id, host_ctxt, 1); - DECLARE_REG(bool, enable, host_ctxt, 2); - - cpu_reg(host_ctxt, 1) = __tracing_enable_event(id, enable); + return __tracing_enable_event(id, enable); } -static void handle___tracing_write_event(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __tracing_write_event, + u64, id) { - DECLARE_REG(u64, id, host_ctxt, 1); - trace_selftest(id); } -static void handle___vgic_v5_save_apr(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_make_resident, + struct vgic_v5_cpu_if __kern *, cpu_if) { - DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); + if (unlikely(is_protected_kvm_enabled())) + return; - __vgic_v5_save_apr(kern_hyp_va(cpu_if)); + __vgic_v5_make_resident(kern_hyp_va_host(cpu_if)); } -static void handle___vgic_v5_restore_vmcr_apr(struct kvm_cpu_context *host_ctxt) +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident, + struct vgic_v5_cpu_if __kern *, cpu_if) { - DECLARE_REG(struct vgic_v5_cpu_if *, cpu_if, host_ctxt, 1); + if (unlikely(is_protected_kvm_enabled())) + return; - __vgic_v5_restore_vmcr_apr(kern_hyp_va(cpu_if)); + __vgic_v5_make_non_resident(kern_hyp_va_host(cpu_if)); +} + +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_save_apr, + struct vgic_v5_cpu_if __kern *, cpu_if) +{ + __vgic_v5_save_apr(kern_hyp_va_host(cpu_if)); +} + +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr, + struct vgic_v5_cpu_if __kern *, cpu_if) +{ + __vgic_v5_restore_vmcr_apr(kern_hyp_va_host(cpu_if)); +} + +DEFINE_KVM_HOST_HCALL(void, __vgic_v5_vdpend, + u32, intid, bool, pending, u16, vm) +{ + if (unlikely(is_protected_kvm_enabled())) + return; + + __vgic_v5_vdpend(intid, pending, vm); } typedef void (*hcall_t)(struct kvm_cpu_context *); @@ -912,6 +1549,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__kvm_enable_ssbs), HANDLE_FUNC(__vgic_v3_init_lrs), HANDLE_FUNC(__vgic_v3_get_gic_config), + HANDLE_FUNC(__pkvm_hyp_alloc_selftest), HANDLE_FUNC(__pkvm_prot_finalize), HANDLE_FUNC(__kvm_adjust_pc), @@ -933,11 +1571,18 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__tracing_write_event), HANDLE_FUNC(__vgic_v3_save_aprs), HANDLE_FUNC(__vgic_v3_restore_vmcr_aprs), + HANDLE_FUNC(__vgic_v5_make_resident), + HANDLE_FUNC(__vgic_v5_make_non_resident), + HANDLE_FUNC(__vgic_v5_vdpend), HANDLE_FUNC(__vgic_v5_save_apr), HANDLE_FUNC(__vgic_v5_restore_vmcr_apr), + HANDLE_FUNC(__pkvm_hyp_topup), + HANDLE_FUNC(__pkvm_hyp_reclaim), + HANDLE_FUNC(__pkvm_hyp_reclaimable), HANDLE_FUNC(__pkvm_host_share_hyp), HANDLE_FUNC(__pkvm_host_unshare_hyp), + HANDLE_FUNC(__pkvm_host_donate_guest), HANDLE_FUNC(__pkvm_host_share_guest), HANDLE_FUNC(__pkvm_host_unshare_guest), diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S index 7a02837203d1..89222425359a 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S @@ -17,7 +17,6 @@ SECTIONS { HYP_SECTION(.data..ro_after_init) HYP_SECTION(.rodata) #ifdef CONFIG_NVHE_EL2_TRACING - . = ALIGN(PAGE_SIZE); BEGIN_HYP_SECTION(.event_ids) *(SORT(.hyp.event_ids.*)) END_HYP_SECTION diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index a6a47c1e058b..c3035aa5a470 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -540,7 +540,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range) int ret; hyp_assert_lock_held(&host_mmu.lock); - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level, 0); if (ret) return ret; @@ -930,7 +930,7 @@ static int get_valid_guest_pte(struct pkvm_hyp_vm *vm, u64 ipa, kvm_pte_t *ptep, s8 level; int ret; - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) return ret; if (guest_pte_is_poisoned(pte)) @@ -979,7 +979,7 @@ int __pkvm_vcpu_in_poison_fault(struct pkvm_hyp_vcpu *hyp_vcpu) ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(&hyp_vcpu->vcpu)); guest_lock_component(vm); - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) goto unlock; @@ -1043,7 +1043,7 @@ int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn) ret = 0; WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_SHARED_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); WARN_ON(__host_set_page_state_range(phys, PAGE_SIZE, PKVM_PAGE_SHARED_BORROWED)); unlock: guest_unlock_component(vm); @@ -1078,7 +1078,7 @@ int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn) PKVM_ID_GUEST, meta)); WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); host_unlock_component(); @@ -1335,7 +1335,7 @@ static int host_stage2_get_guest_info(phys_addr_t phys, struct pkvm_hyp_vm **vm, return -EPERM; } - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level, 0); if (ret) return ret; @@ -1442,7 +1442,7 @@ static int __guest_check_pgtable_memcache(struct pkvm_hyp_vcpu *vcpu) { struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu); - if (vcpu->vcpu.arch.pkvm_memcache.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu)) + if (vcpu->vcpu.arch.stage2_mc.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu)) return -ENOMEM; return 0; @@ -1476,7 +1476,7 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu) PKVM_ID_GUEST, meta)); WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys, pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); @@ -1547,7 +1547,7 @@ int __pkvm_host_share_guest(u64 pfn, u64 gfn, u64 nr_pages, struct pkvm_hyp_vcpu WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys, pkvm_mkstate(prot, PKVM_PAGE_SHARED_BORROWED), - &vcpu->vcpu.arch.pkvm_memcache, 0)); + &vcpu->vcpu.arch.stage2_mc, 0)); unlock: guest_unlock_component(vm); @@ -1564,7 +1564,7 @@ static int __check_host_shared_guest(struct pkvm_hyp_vm *vm, u64 *__phys, u64 ip s8 level; int ret; - ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level); + ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0); if (ret) return ret; if (!kvm_pte_valid(pte)) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index 29ab5ee9d57f..b95d98cfe57d 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -91,6 +91,59 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) return ret; } +/** + * pkvm_map_private_va_range() - Map a physical range into the private VA range + * @haddr: The virtual address in the private range. + * @phys: The physical address to map. + * @size: The size of the range to map. + * + * The hypervisor VA @haddr must have been first allocated with + * pkvm_alloc_private_va_range(). The created mapping can later be removed + * with pkvm_remove_mappings(). + * + * Return: 0 on success, negative error code on failure. + */ +int pkvm_map_private_va_range(void *haddr, phys_addr_t phys, size_t size) +{ + unsigned long addr = (unsigned long)haddr; + int ret; + + if (!PAGE_ALIGNED(addr | phys | size)) + return -EINVAL; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + + if (addr < __io_map_base || addr >= __io_map_next) + return -EINVAL; + + if (size > __io_map_next - addr) + return -EINVAL; + + ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, PAGE_HYP); + if (ret) + kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size); + + return ret; +} + +/** + * pkvm_private_va_range_pa - Translate a private VA to a physical address. + * @va: The private virtual address to translate. + * + * Return: The physical address corresponding to @va. + */ +phys_addr_t pkvm_private_va_range_pa(void *va) +{ + kvm_pte_t pte; + s8 level; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + WARN_ON(kvm_pgtable_get_leaf(&pkvm_pgtable, (u64)va, &pte, &level, 0)); + WARN_ON(!kvm_pte_valid(pte)); + + return kvm_pte_to_phys(pte) + ((u64)va & (kvm_granule_size(level) - 1)); +} + int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, enum kvm_pgtable_prot prot, unsigned long *haddr) @@ -147,6 +200,30 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot) return ret; } +/** + * pkvm_remove_mappings - Remove mappings from the hypervisor page-table + * @from: The starting virtual address of the range to remove + * @to: The ending virtual address of the range to remove + * + * The range [@from, @to) can be a subset of the range previously mapped by + * pkvm_create_mappings(), as the latter enforces PTE-level mappings. However, + * it cannot be a subset of a range mapped by pkvm_map_private_va_range() as + * that function allows block mappings which the hypervisor page-table does not + * split. + */ +void pkvm_remove_mappings(void *from, void *to) +{ + u64 size; + + WARN_ON(from > to); + to = PTR_ALIGN(to, PAGE_SIZE); + from = PTR_ALIGN_DOWN(from, PAGE_SIZE); + size = (u64)to - (u64)from; + + guard(hyp_spinlock)(&pkvm_pgd_lock); + WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, (u64)from, size) != size); +} + int hyp_back_vmemmap(phys_addr_t back) { unsigned long i, start, size, end = 0; @@ -486,7 +563,7 @@ static int check_page_ownership(phys_addr_t phys) return -EPERM; } - ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL); + ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL, 0); if (ret) return ret; diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 6290c4b62659..5168321713d9 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -5,17 +5,25 @@ */ #include +#include #include #include #include +#include +#include #include #include #include #include +/* The host's Spectre mitigation state, for SMCCC_ARCH_FEATURES. */ +enum mitigation_state spectre_v2_state; +enum mitigation_state spectre_v4_state; +enum mitigation_state spectre_bhb_state; + /* Used by icache_is_aliasing(). */ unsigned long __icache_flags; @@ -109,7 +117,7 @@ static void pvm_init_traps_mdcr(struct kvm_vcpu *vcpu) if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) { val |= MDCR_EL2_TPM | MDCR_EL2_TPMCR; - val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME | MDCR_EL2_HPMN_MASK); + val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME); } if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, IMP)) @@ -285,7 +293,7 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, } hyp_vcpu->loaded_hyp_vcpu = this_cpu_ptr(&loaded_hyp_vcpu); - hyp_page_ref_inc(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_inc(hyp_vm); unlock: hyp_spin_unlock(&vm_table_lock); @@ -301,7 +309,7 @@ void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_spin_lock(&vm_table_lock); hyp_vcpu->loaded_hyp_vcpu = NULL; __this_cpu_write(loaded_hyp_vcpu, NULL); - hyp_page_ref_dec(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_dec(hyp_vm); hyp_spin_unlock(&vm_table_lock); } @@ -311,6 +319,43 @@ struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void) } +static struct pkvm_hyp_vm *pkvm_get_loaded_hyp_vm(struct kvm_vcpu *vcpu) +{ + struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + + if (hyp_vcpu && + (vcpu == &hyp_vcpu->vcpu || vcpu == hyp_vcpu->host_vcpu)) + return pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + + return NULL; +} + +/* + * A loaded vCPU's VM is the hyp VM. An unloaded host vCPU's is mapped at + * EL2 only while pinned, so it's read once and pinned; the pin fails for + * memory the host isn't sharing. + */ +struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_get_loaded_hyp_vm(vcpu); + struct kvm *kvm; + + if (hyp_vm) + return &hyp_vm->kvm; + + kvm = kern_hyp_va(READ_ONCE(vcpu->kvm)); + if (hyp_pin_shared_mem(kvm, kvm + 1)) + return NULL; + + return kvm; +} + +void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm) +{ + if (kvm && !pkvm_get_loaded_hyp_vm(vcpu)) + hyp_unpin_shared_mem(kvm, kvm + 1); +} + struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) { struct pkvm_hyp_vm *hyp_vm; @@ -318,7 +363,7 @@ struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) hyp_spin_lock(&vm_table_lock); hyp_vm = get_vm_by_handle(handle); if (hyp_vm) - hyp_page_ref_inc(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_inc(hyp_vm); hyp_spin_unlock(&vm_table_lock); return hyp_vm; @@ -327,7 +372,7 @@ struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle) void put_pkvm_hyp_vm(struct pkvm_hyp_vm *hyp_vm) { hyp_spin_lock(&vm_table_lock); - hyp_page_ref_dec(hyp_virt_to_page(hyp_vm)); + pkvm_hyp_vm_ref_dec(hyp_vm); hyp_spin_unlock(&vm_table_lock); } @@ -373,21 +418,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE)) kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_MTE_ENABLED); - bitmap_zero(allowed_features, KVM_VCPU_MAX_FEATURES); - - set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3)) - set_bit(KVM_ARM_VCPU_PMU_V3, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS)) - set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) - set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features); - - if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) - set_bit(KVM_ARM_VCPU_SVE, allowed_features); + kvm_pkvm_vcpu_allowed_features(kvm, allowed_features); bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features, allowed_features, KVM_VCPU_MAX_FEATURES); @@ -396,6 +427,40 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE)); } +static int pkvm_vcpu_init_psci(struct pkvm_hyp_vcpu *hyp_vcpu, u32 mp_state) +{ + struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + struct kvm_vcpu *host_vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + /* The host manages a non-protected vCPU: always ON at EL2. */ + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON; + return 0; + } + + if (mp_state != KVM_MP_STATE_RUNNABLE && mp_state != KVM_MP_STATE_STOPPED) + return -EINVAL; + + if (mp_state == KVM_MP_STATE_STOPPED) { + reset_state->reset = false; + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_OFF; + return 0; + } + + hyp_assert_lock_held(&vm_table_lock); + if (hyp_vm->primary_vcpu) + return -EINVAL; + hyp_vm->primary_vcpu = hyp_vcpu; + + host_vcpu = hyp_vcpu->host_vcpu; + reset_state->pc = READ_ONCE(*vcpu_pc(host_vcpu)); + reset_state->r0 = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]); + reset_state->reset = true; + hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING; + return 0; +} + static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu) { if (host_vcpu) @@ -440,7 +505,11 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, hyp_vm->host_kvm = host_kvm; hyp_vm->kvm.created_vcpus = nr_vcpus; - hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected); + if (READ_ONCE(host_kvm->arch.vm_flavor) == VM_PROTECTED_PKVM) + hyp_vm->kvm.arch.vm_flavor = VM_PROTECTED_PKVM; + else + hyp_vm->kvm.arch.vm_flavor = VM_PKVM; + hyp_vm->kvm.arch.flags = 0; pkvm_init_features_from_host(hyp_vm, host_kvm); @@ -518,10 +587,12 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, struct kvm_vcpu *host_vcpu) { int ret = 0; + u32 mp_state; if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1)) return -EBUSY; + mp_state = READ_ONCE(host_vcpu->arch.mp_state.mp_state); hyp_vcpu->host_vcpu = host_vcpu; hyp_vcpu->vcpu.kvm = &hyp_vm->kvm; @@ -530,7 +601,6 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu; hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags); - hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { /* @@ -555,9 +625,17 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu, goto done; ret = pkvm_vcpu_init_sve(hyp_vcpu, host_vcpu); -done: if (ret) + goto done; + + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); + ret = pkvm_vcpu_init_psci(hyp_vcpu, mp_state); +done: + if (ret) { unpin_host_vcpu(host_vcpu); + unpin_host_sve_state(hyp_vcpu); + } return ret; } @@ -661,9 +739,9 @@ static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus) size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus)); } -static void *map_donated_memory_noclear(unsigned long host_va, size_t size) +static void *map_donated_memory_noclear(void __kern *host_va, size_t size) { - void *va = (void *)kern_hyp_va(host_va); + void *va = kern_hyp_va_host(host_va); if (!PAGE_ALIGNED(va)) return NULL; @@ -675,16 +753,6 @@ static void *map_donated_memory_noclear(unsigned long host_va, size_t size) return va; } -static void *map_donated_memory(unsigned long host_va, size_t size) -{ - void *va = map_donated_memory_noclear(host_va, size); - - if (va) - memset(va, 0, size); - - return va; -} - static void __unmap_donated_memory(void *va, size_t size) { kvm_flush_dcache_to_poc(va, size); @@ -787,7 +855,7 @@ struct pkvm_hyp_vcpu *init_selftest_vm(void *virt) continue; p[i].refcount = 1; if (seeded < min_pages) { - push_hyp_memcache(&selftest_vcpu.vcpu.arch.pkvm_memcache, + push_hyp_memcache(&selftest_vcpu.vcpu.arch.stage2_mc, hyp_page_to_virt(&p[i]), hyp_virt_to_phys); seeded++; } else { @@ -814,16 +882,13 @@ void teardown_selftest_vm(void) * Unmap the donated memory from the host at stage 2. * * host_kvm: A pointer to the host's struct kvm. - * vm_hva: The host va of the area being donated for the VM state. - * Must be page aligned. * pgd_hva: The host va of the area being donated for the stage-2 PGD for * the VM. Must be page aligned. Its size is implied by the VM's * VTCR. * * Return 0 success, negative error code on failure. */ -int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, - unsigned long pgd_hva) +int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva) { struct pkvm_hyp_vm *hyp_vm = NULL; size_t vm_size, pgd_size; @@ -851,15 +916,17 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, vm_size = pkvm_get_hyp_vm_size(nr_vcpus); pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.mmu.vtcr); - ret = -ENOMEM; + hyp_vm = hyp_alloc(vm_size); + if (!hyp_vm) { + ret = hyp_alloc_errno(); + goto err_unpin_kvm; + } - hyp_vm = map_donated_memory(vm_hva, vm_size); - if (!hyp_vm) - goto err_remove_mappings; + ret = -ENOMEM; pgd = map_donated_memory_noclear(pgd_hva, pgd_size); if (!pgd) - goto err_remove_mappings; + goto err_free_hyp_vm; init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus, handle); @@ -877,8 +944,9 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, err_destroy_stage2: kvm_guest_destroy_stage2(hyp_vm); err_remove_mappings: - unmap_donated_memory(hyp_vm, vm_size); unmap_donated_memory(pgd, pgd_size); +err_free_hyp_vm: + hyp_free(hyp_vm); err_unpin_kvm: hyp_unpin_shared_mem(host_kvm, host_kvm + 1); return ret; @@ -913,16 +981,15 @@ static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, return 0; } -int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, - unsigned long vcpu_hva) +int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu) { struct pkvm_hyp_vcpu *hyp_vcpu; struct pkvm_hyp_vm *hyp_vm; int ret; - hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu)); + hyp_vcpu = hyp_alloc(sizeof(*hyp_vcpu)); if (!hyp_vcpu) - return -ENOMEM; + return hyp_alloc_errno(); hyp_spin_lock(&vm_table_lock); @@ -934,33 +1001,30 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu); if (ret) - goto unlock; + goto unclaim; ret = register_hyp_vcpu(hyp_vm, hyp_vcpu); if (ret) { unpin_host_vcpu(host_vcpu); unpin_host_sve_state(hyp_vcpu); + goto unclaim; } + goto unlock; +unclaim: + /* + * Under vm_table_lock, so no other claim can have landed: undo + * this one. + */ + if (hyp_vm->primary_vcpu == hyp_vcpu) + hyp_vm->primary_vcpu = NULL; unlock: hyp_spin_unlock(&vm_table_lock); - if (ret) - unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu)); + hyp_free(hyp_vcpu); + return ret; } -static void -teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size) -{ - size = PAGE_ALIGN(size); - memset(addr, 0, size); - - for (void *start = addr; start < addr + size; start += PAGE_SIZE) - push_hyp_memcache(mc, start, hyp_virt_to_phys); - - unmap_donated_memory_noclear(addr, size); -} - int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn) { struct pkvm_hyp_vm *hyp_vm = get_pkvm_hyp_vm(handle); @@ -983,7 +1047,7 @@ static struct pkvm_hyp_vm *get_pkvm_unref_hyp_vm_locked(pkvm_handle_t handle) hyp_assert_lock_held(&vm_table_lock); hyp_vm = get_vm_by_handle(handle); - if (!hyp_vm || hyp_page_count(hyp_vm)) + if (!hyp_vm || hyp_vm->refcount) return NULL; return hyp_vm; @@ -1010,11 +1074,10 @@ int __pkvm_start_teardown_vm(pkvm_handle_t handle) int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) { - struct kvm_hyp_memcache *mc, *stage2_mc; + struct kvm_hyp_memcache *stage2_mc; struct pkvm_hyp_vm *hyp_vm; struct kvm *host_kvm; unsigned int idx; - size_t vm_size; int err; hyp_spin_lock(&vm_table_lock); @@ -1032,12 +1095,11 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) hyp_spin_unlock(&vm_table_lock); /* Reclaim guest pages (including page-table pages) */ - mc = &host_kvm->arch.pkvm.teardown_mc; stage2_mc = &host_kvm->arch.pkvm.stage2_teardown_mc; reclaim_pgtable_pages(hyp_vm, stage2_mc); unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus); - /* Push the metadata pages to the teardown memcache */ + /* Push the stage-2 pages to the teardown memcache */ for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) { struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx]; struct kvm_hyp_memcache *vcpu_mc; @@ -1045,7 +1107,7 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) if (!hyp_vcpu) continue; - vcpu_mc = &hyp_vcpu->vcpu.arch.pkvm_memcache; + vcpu_mc = &hyp_vcpu->vcpu.arch.stage2_mc; while (vcpu_mc->nr_pages) { void *addr = pop_hyp_memcache(vcpu_mc, hyp_phys_to_virt); @@ -1054,11 +1116,10 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle) unmap_donated_memory_noclear(addr, PAGE_SIZE); } - teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu)); + hyp_free(hyp_vcpu); } - vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus); - teardown_donated_memory(mc, hyp_vm, vm_size); + hyp_free(hyp_vm); hyp_unpin_shared_mem(host_kvm, host_kvm + 1); return 0; @@ -1132,6 +1193,238 @@ static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu) ret[0] = SMCCC_RET_SUCCESS; } +/* + * Reset the vCPU to its power-on state and commit ON_PENDING -> ON, on the + * target's own CPU. Returns -ECANCELED, with no side effects, if a rollback + * raced the reset. + * + * The requestor's PSCI CPU_ON, in pvm_psci_vcpu_on(): + * + * cmpxchg_relaxed(power_state): OFF -> ON_PENDING + * + * write_reset_state(); + * smp_store_release(reset_state->reset, true); + * + * The target coming online here, running, then its PSCI CPU_OFF in + * pvm_psci_vcpu_off(): + * + * smp_load_acquire(reset_state->reset) == true; + * cmpxchg_relaxed(power_state): ON_PENDING -> ON; + * read_reset_state(); + * + * + * + * WRITE_ONCE(reset_state->reset, false); + * smp_store_release(power_state, OFF); + */ +int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state; + + /* + * Pairs with smp_store_release(&reset_state->reset, true) in + * pvm_psci_vcpu_on(). The acquire must precede the cmpxchg: reversed, a + * winning cmpxchg with a false acquire would leave power_state == ON + * with the reset skipped. + */ + if (!smp_load_acquire(&reset_state->reset)) + return -ECANCELED; + + if (cmpxchg_relaxed(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_ON_PENDING, + PSCI_0_2_AFFINITY_LEVEL_ON) != PSCI_0_2_AFFINITY_LEVEL_ON_PENDING) + return -ECANCELED; + + kvm_reset_vcpu_core(&hyp_vcpu->vcpu); + kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu); + + /* Must be done after resetting sys registers. */ + kvm_reset_vcpu_psci(&hyp_vcpu->vcpu, reset_state); + + hyp_vcpu->exit_code = 0; + return 0; +} + +struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, unsigned long mpidr) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + int i; + + mpidr &= MPIDR_HWID_BITMASK; + + for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) { + /* Pairs with smp_store_release() in register_hyp_vcpu(). */ + hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[i]); + + if (!hyp_vcpu) + continue; + + if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu)) + return hyp_vcpu; + } + + return NULL; +} + +/* + * Returns true when handled at EL2, false when the host must wake the target + * vCPU. + */ +static bool pvm_psci_vcpu_on(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu); + struct vcpu_reset_state *reset_state; + struct pkvm_hyp_vcpu *target; + unsigned long cpu_id, ret; + int power_state; + + cpu_id = smccc_get_arg1(&hyp_vcpu->vcpu); + if (!kvm_psci_valid_affinity(&hyp_vcpu->vcpu, cpu_id)) { + ret = PSCI_RET_INVALID_PARAMS; + goto error; + } + + target = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id); + if (!target) { + ret = PSCI_RET_INVALID_PARAMS; + goto error; + } + + /* + * vCPUs race to power on the same target. The writes below depend on + * the cmpxchg reading OFF, and that control dependency with the release + * of OFF in pvm_psci_vcpu_off() orders them after the target's accesses + * to reset_state, so relaxed suffices. + */ + power_state = cmpxchg_relaxed(&target->power_state, + PSCI_0_2_AFFINITY_LEVEL_OFF, + PSCI_0_2_AFFINITY_LEVEL_ON_PENDING); + switch (power_state) { + case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING: + ret = PSCI_RET_ON_PENDING; + goto error; + case PSCI_0_2_AFFINITY_LEVEL_ON: + ret = PSCI_RET_ALREADY_ON; + goto error; + case PSCI_0_2_AFFINITY_LEVEL_OFF: + break; + default: + ret = PSCI_RET_INTERNAL_FAILURE; + goto error; + } + + reset_state = &target->vcpu.arch.reset_state; + WRITE_ONCE(reset_state->pc, smccc_get_arg2(&hyp_vcpu->vcpu)); + WRITE_ONCE(reset_state->r0, smccc_get_arg3(&hyp_vcpu->vcpu)); + WRITE_ONCE(reset_state->be, kvm_vcpu_is_be(&hyp_vcpu->vcpu)); + /* + * Publish reset_state.{pc, r0, be} to the target vCPU. Pairs with + * smp_load_acquire(&reset_state->reset) in pkvm_reset_vcpu(). + */ + smp_store_release(&reset_state->reset, true); + + /* The host requests KVM_REQ_VCPU_RESET and wakes the target. */ + return false; + +error: + smccc_set_retval(&hyp_vcpu->vcpu, ret, 0, 0, 0); + return true; +} + +/* + * Returns true when handled at EL2, false when the host must stop scheduling + * the vCPU. + */ +static bool pvm_psci_vcpu_off(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + /* No other writer runs while this vCPU is ON and executing. */ + WARN_ON(READ_ONCE(hyp_vcpu->power_state) != PSCI_0_2_AFFINITY_LEVEL_ON); + + WRITE_ONCE(hyp_vcpu->vcpu.arch.reset_state.reset, false); + + /* + * Orders pkvm_reset_vcpu()'s accesses to reset_state, and the clear + * above, before OFF. Pairs with the control dependency in + * pvm_psci_vcpu_on(). + */ + smp_store_release(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_OFF); + + /* Return to the host so that it can finish powering off the vcpu. */ + return false; +} + +static bool pvm_psci_version(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + /* Nothing to be handled by the host. Go back to the guest. */ + smccc_set_retval(&hyp_vcpu->vcpu, KVM_ARM_PSCI_1_1, 0, 0, 0); + return true; +} + +static bool pvm_psci_features(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u32 feature = smccc_get_arg1(vcpu); + unsigned long val; + + switch (feature) { + case PSCI_0_2_FN_PSCI_VERSION: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + case PSCI_0_2_FN_CPU_OFF: + case PSCI_0_2_FN_CPU_ON: + case PSCI_0_2_FN64_CPU_ON: + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_1_0_FN_PSCI_FEATURES: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + case ARM_SMCCC_VERSION_FUNC_ID: + val = PSCI_RET_SUCCESS; + break; + default: + val = PSCI_RET_NOT_SUPPORTED; + break; + } + + /* Nothing to be handled by the host. Go back to the guest. */ + smccc_set_retval(vcpu, val, 0, 0, 0); + return true; +} + +static bool pkvm_handle_psci(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu; + u32 psci_fn = smccc_get_function(vcpu); + + switch (psci_fn) { + case PSCI_0_2_FN_CPU_ON: + kvm_psci_narrow_to_32bit(vcpu); + fallthrough; + case PSCI_0_2_FN64_CPU_ON: + return pvm_psci_vcpu_on(hyp_vcpu); + case PSCI_0_2_FN_CPU_OFF: + return pvm_psci_vcpu_off(hyp_vcpu); + case PSCI_0_2_FN_PSCI_VERSION: + return pvm_psci_version(hyp_vcpu); + case PSCI_1_0_FN_PSCI_FEATURES: + return pvm_psci_features(hyp_vcpu); + case PSCI_0_2_FN_AFFINITY_INFO: + case PSCI_0_2_FN64_AFFINITY_INFO: + case PSCI_0_2_FN_SYSTEM_RESET: + case PSCI_0_2_FN_CPU_SUSPEND: + case PSCI_0_2_FN64_CPU_SUSPEND: + case PSCI_0_2_FN_SYSTEM_OFF: + case PSCI_1_1_FN_SYSTEM_RESET2: + case PSCI_1_1_FN64_SYSTEM_RESET2: + return false; /* Handled by the host. */ + default: + /* Unknown PSCI calls are handled here, not forwarded. */ + smccc_set_retval(vcpu, PSCI_RET_NOT_SUPPORTED, 0, 0, 0); + return true; + } +} + /* * Handler for protected VM HVC calls. * @@ -1141,10 +1434,73 @@ static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu) */ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) { + struct pkvm_hyp_vcpu *hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu); u64 val[4] = { SMCCC_RET_INVALID_PARAMETER }; bool handled = true; + u32 feature; + uuid_t uuid; switch (smccc_get_function(vcpu)) { + case ARM_SMCCC_VERSION_FUNC_ID: + /* Nothing to be handled by the host. Go back to the guest. */ + val[0] = ARM_SMCCC_VERSION_1_1; + break; + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + /* The workaround calls themselves are handled on hyp entry. */ + val[0] = SMCCC_RET_NOT_SUPPORTED; + feature = smccc_get_arg1(vcpu); + switch (feature) { + case ARM_SMCCC_VERSION_FUNC_ID: + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + val[0] = SMCCC_RET_SUCCESS; + break; + case ARM_SMCCC_ARCH_WORKAROUND_1: + switch (spectre_v2_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + val[0] = SMCCC_RET_SUCCESS; + break; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; + break; + } + break; + case ARM_SMCCC_ARCH_WORKAROUND_2: + switch (spectre_v4_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + /* With SSBS advertised the guest's default is safe. */ + if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP)) + break; + fallthrough; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_RET_NOT_REQUIRED; + break; + } + break; + case ARM_SMCCC_ARCH_WORKAROUND_3: + switch (spectre_bhb_state) { + case SPECTRE_VULNERABLE: + break; + case SPECTRE_MITIGATED: + val[0] = SMCCC_RET_SUCCESS; + break; + case SPECTRE_UNAFFECTED: + val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED; + break; + } + break; + } + break; + case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: + uuid = ARM_SMCCC_VENDOR_HYP_UID_KVM; + val[0] = smccc_uuid_to_reg(&uuid, 0); + val[1] = smccc_uuid_to_reg(&uuid, 1); + val[2] = smccc_uuid_to_reg(&uuid, 2); + val[3] = smccc_uuid_to_reg(&uuid, 3); + break; case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: val[0] = BIT(ARM_SMCCC_KVM_FUNC_FEATURES); val[0] |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO); @@ -1177,8 +1533,7 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code) pkvm_memunshare_call(val, vcpu); break; default: - /* Punt everything else back to the host, for now. */ - handled = false; + return pkvm_handle_psci(hyp_vcpu); } if (handled) diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 45ac5f2ba4f7..7bbc26b28faa 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -377,6 +378,11 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_bas if (ret) return ret; + /* Arbitrary size, unmapped at boot and only consume VA-space */ + ret = hyp_alloc_init(SZ_128M); + if (ret) + return ret; + update_nvhe_init_params(); /* Jump in the idmap page to switch to the new page-tables */ diff --git a/arch/arm64/kvm/hyp/nvhe/stacktrace.c b/arch/arm64/kvm/hyp/nvhe/stacktrace.c index 7c832d60d22b..11fadbebbf1d 100644 --- a/arch/arm64/kvm/hyp/nvhe/stacktrace.c +++ b/arch/arm64/kvm/hyp/nvhe/stacktrace.c @@ -8,6 +8,7 @@ #include #include #include +#include DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) __aligned(16); @@ -35,8 +36,6 @@ static void hyp_prepare_backtrace(unsigned long fp, unsigned long pc) } #ifdef CONFIG_PKVM_STACKTRACE -#include - DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], pkvm_stacktrace); static struct stack_info stackinfo_get_overflow(void) diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 7318e3e6a5f3..91e1edf40bcd 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -203,15 +203,36 @@ static const exit_handler_fn hyp_exit_handlers[] = { [ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops, }; +/* WFI/WFE exit to the host, which emulates them. WFxT is not advertised. */ +static bool kvm_handle_pvm_wfx(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (kvm_vcpu_get_esr(vcpu) & ESR_ELx_WFx_ISS_WFxT) + return kvm_handle_pvm_restricted(vcpu, exit_code); + + return false; +} + +/* Lazy FP/SIMD switch, or the UNDEF that would otherwise be left to the host. */ +static bool kvm_handle_pvm_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) +{ + if (kvm_hyp_handle_fpsimd(vcpu, exit_code)) + return true; + + return kvm_handle_pvm_restricted(vcpu, exit_code); +} + +/* + * A class not listed takes an UNDEF at EL2: the host has no way to + * inject one into a protected vCPU. + */ static const exit_handler_fn pvm_exit_handlers[] = { - [0 ... ESR_ELx_EC_MAX] = NULL, + [0 ... ESR_ELx_EC_MAX] = kvm_handle_pvm_restricted, + [ESR_ELx_EC_WFx] = kvm_handle_pvm_wfx, [ESR_ELx_EC_HVC64] = kvm_handle_pvm_hvc64, [ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64, - [ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted, - [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, + [ESR_ELx_EC_FP_ASIMD] = kvm_handle_pvm_fpsimd, [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, - [ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low, [ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops, }; diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8758c6801776..32e82653ebf7 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -121,6 +121,9 @@ static const struct pvm_ftr_bits pvmid_aa64mmfr0[] = { MAX_FEAT(ID_AA64MMFR0_EL1, SNSMEM, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, BIGENDEL0, IMP), MAX_FEAT(ID_AA64MMFR0_EL1, EXS, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN4, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN16, IMP), + MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN64, IMP), FEAT_END }; @@ -525,6 +528,66 @@ static const struct sys_reg_desc pvm_sys_reg_descs[] = { /* Performance Monitoring Registers are restricted. */ }; +struct sys_reg_desc_reset { + int reg; + void (*reset)(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *rd); + u64 value; +}; + +static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *r) +{ + __vcpu_assign_sys_reg(vcpu, r->reg, kvm_calculate_mpidr(vcpu)); +} + +static void reset_value(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *r) +{ + __vcpu_assign_sys_reg(vcpu, r->reg, r->value); +} + +#define RESET_VAL(REG, VAL) { REG, reset_value, VAL } + +#define RESET_ZERO(REG) RESET_VAL(REG, 0) + +#define RESET_UNKNOWN(REG) RESET_VAL(REG, 0x1de7ec7edbadc0deULL) + +#define RESET_FUNC(REG, FN) { REG, FN, 0 } + +static const struct sys_reg_desc_reset pvm_sys_reg_reset_vals[] = { + RESET_FUNC(MPIDR_EL1, reset_mpidr), + RESET_UNKNOWN(TPIDR_EL0), + RESET_UNKNOWN(TPIDRRO_EL0), + RESET_UNKNOWN(TPIDR_EL1), + RESET_ZERO(CNTKCTL_EL1), + RESET_UNKNOWN(PAR_EL1), + RESET_ZERO(DISR_EL1), + RESET_ZERO(CPACR_EL1), + RESET_VAL(CONTEXTIDR_EL1, 0x00000000dbadc0deULL), + RESET_VAL(SCTLR_EL1, 0x00C50078ULL), + RESET_ZERO(TCR_EL1), + RESET_UNKNOWN(AFSR0_EL1), + RESET_UNKNOWN(AFSR1_EL1), + RESET_UNKNOWN(ESR_EL1), + RESET_UNKNOWN(MAIR_EL1), + RESET_ZERO(AMAIR_EL1), + RESET_ZERO(MDSCR_EL1), + RESET_ZERO(CNTV_CTL_EL0), + RESET_UNKNOWN(TTBR0_EL1), + RESET_UNKNOWN(TTBR1_EL1), + RESET_UNKNOWN(FAR_EL1), + RESET_VAL(VBAR_EL1, 0x1de7ec7edbadc000ULL), +}; + +void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu) +{ + unsigned long i; + + for (i = 0; i < ARRAY_SIZE(pvm_sys_reg_reset_vals); i++) { + const struct sys_reg_desc_reset *r = &pvm_sys_reg_reset_vals[i]; + + r->reset(vcpu, r); + } +} + /* * Initializes feature registers for protected vms. */ diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index 65be6c9fb379..0d7e4db0c3e1 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -4,6 +4,7 @@ * Author: Vincent Donnefort */ +#include #include #include #include @@ -62,18 +63,34 @@ static void __release_host_mem(void *start, u64 size) WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT)); } -static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_buffer, - struct hyp_trace_desc *desc) +static int hyp_trace_buffer_alloc_bpages(struct hyp_trace_buffer *trace_buffer, + struct hyp_trace_desc *desc) { - void *start = (void *)kern_hyp_va(desc->bpages_backing_start); - size_t size = desc->bpages_backing_size; + void *start; + size_t size; int ret; - ret = __admit_host_mem(start, size); - if (ret) - return ret; + if (is_protected_kvm_enabled()) { + struct ring_buffer_desc *rb_desc; + int cpu; - memset(start, 0, size); + size = 0; + for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) + size += rb_desc->nr_page_va * sizeof(struct simple_buffer_page); + + start = hyp_alloc(size); + if (!start) + return hyp_alloc_errno(); + } else { + start = (void *)kern_hyp_va(desc->bpages_backing_start); + size = desc->bpages_backing_size; + + ret = __admit_host_mem(start, size); + if (ret) + return ret; + + memset(start, 0, size); + } trace_buffer->bpages_backing_start = start; trace_buffer->bpages_backing_size = size; @@ -81,7 +98,7 @@ static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_bu return 0; } -static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace_buffer) +static void hyp_trace_buffer_free_bpages(struct hyp_trace_buffer *trace_buffer) { void *start = trace_buffer->bpages_backing_start; size_t size = trace_buffer->bpages_backing_size; @@ -89,11 +106,14 @@ static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace if (!size) return; - memset(start, 0, size); + if (is_protected_kvm_enabled()) { + hyp_free(start); + } else { + memset(start, 0, size); + __release_host_mem(start, size); + } - __release_host_mem(start, size); - - trace_buffer->bpages_backing_start = 0; + trace_buffer->bpages_backing_start = NULL; trace_buffer->bpages_backing_size = 0; } @@ -128,7 +148,7 @@ static void hyp_trace_buffer_unload(struct hyp_trace_buffer *trace_buffer) simple_ring_buffer_unload_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu), __unpin_shared_page); - hyp_trace_buffer_unload_bpage_backing(trace_buffer); + hyp_trace_buffer_free_bpages(trace_buffer); } static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer, @@ -143,7 +163,7 @@ static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer, if (hyp_trace_buffer_loaded(trace_buffer)) return -EINVAL; - ret = hyp_trace_buffer_load_bpage_backing(trace_buffer, desc); + ret = hyp_trace_buffer_alloc_bpages(trace_buffer, desc); if (ret) return ret; @@ -168,47 +188,43 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz { struct ring_buffer_desc *rb_desc; unsigned int cpu; - size_t nr_bpages; void *desc_end; if (!is_protected_kvm_enabled()) return true; /* - * Both desc_size and bpages_backing_size are untrusted host-provided - * values. We rely on __pkvm_host_donate_hyp() to enforce their validity. + * desc_size is an untrusted host-provided value. We rely on + * __pkvm_host_donate_hyp() to enforce its validity. */ desc_end = (void *)desc + desc_size; - nr_bpages = desc->bpages_backing_size / sizeof(struct simple_buffer_page); + + if (desc->trace_buffer_desc.nr_cpus != hyp_nr_cpus) + return false; for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) { /* Can we read nr_page_va? */ if ((void *)rb_desc + struct_size(rb_desc, page_va, 0) > desc_end) return false; + /* simple_ring_buffer_init_mm() expects at least 3 pages */ + if (rb_desc->nr_page_va < 3) + return false; + /* Overflow desc? */ if ((void *)rb_desc + struct_size(rb_desc, page_va, rb_desc->nr_page_va) > desc_end) return false; - /* Overflow bpages backing memory? */ - if (nr_bpages < rb_desc->nr_page_va) - return false; - - if (cpu >= hyp_nr_cpus) - return false; - if (cpu != rb_desc->cpu) return false; - - nr_bpages -= rb_desc->nr_page_va; } return true; } -int __tracing_load(unsigned long desc_hva, size_t desc_size) +int __tracing_load(void __kern *desc_hva, size_t desc_size) { - struct hyp_trace_desc *desc = (struct hyp_trace_desc *)kern_hyp_va(desc_hva); + struct hyp_trace_desc *desc = kern_hyp_va_host(desc_hva); int ret; ret = __admit_host_mem(desc, desc_size); diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index b74dd5ce1efd..9c83837b184d 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -8,6 +8,7 @@ */ #include +#include #include #include @@ -298,12 +299,13 @@ static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx, } int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr, - kvm_pte_t *ptep, s8 *level) + kvm_pte_t *ptep, s8 *level, + enum kvm_pgtable_walk_flags flags) { struct leaf_walk_data data; struct kvm_pgtable_walker walker = { .cb = leaf_walker, - .flags = KVM_PGTABLE_WALK_LEAF, + .flags = flags | KVM_PGTABLE_WALK_LEAF, .arg = &data, }; int ret; diff --git a/arch/arm64/kvm/hyp/vgic-v5-sr.c b/arch/arm64/kvm/hyp/vgic-v5-sr.c index 6d69dfe89a96..94746317ca79 100644 --- a/arch/arm64/kvm/hyp/vgic-v5-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v5-sr.c @@ -7,6 +7,41 @@ #include +void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if) +{ + write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2); + isb(); + + /* Catch any faults */ + cpu_if->vgic_contextr = read_sysreg_s(SYS_ICH_CONTEXTR_EL2); + if (!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr)) + return; + + cpu_if->gicv5_vpe.resident = true; +} + +void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if) +{ + /* + * Clear the db_fired state to ensure that we're ready for the next + * doorbell when it is requested. If a doorbell firing caused us to + * enter the guest, then we've already consumed that state at this + * point, so this is safe to clear. Use WRITE_ONCE() to ensure we're not + * racing with the doorbell firing and setting the state true again. + */ + WRITE_ONCE(cpu_if->gicv5_vpe.db_fired, false); + + /* + * Make as non-resident before actually making non-resident. Avoids race + * with doorbell arriving. + */ + cpu_if->gicv5_vpe.resident = false; + dsb(st); + + write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2); + isb(); +} + void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if) { cpu_if->vgic_apr = read_sysreg_s(SYS_ICH_APR_EL2); @@ -114,3 +149,13 @@ void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if) { write_sysreg_s(cpu_if->vgic_icsr, SYS_ICC_ICSR_EL1); } + +void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm) +{ + u64 value; + + value = intid & (GICV5_GIC_VDPEND_ID_MASK | GICV5_GIC_VDPEND_TYPE_MASK); + value |= FIELD_PREP(GICV5_GIC_VDPEND_PENDING_MASK, pending); + value |= FIELD_PREP(GICV5_GIC_VDPEND_VM_MASK, vm); + gic_insn(value, VDPEND); +} diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 7875911c0506..4c965f9b1e1d 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -175,7 +175,8 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) offset = read_sysreg_s(SYS_CNTPOFF_EL2); if (map.direct_ptimer && offset) { - write_sysreg_el0(val + offset, SYS_CNTP_CVAL); + val = timer_apply_offset(val, offset, arch_timer_read_cntpct_el0()); + write_sysreg_el0(val, SYS_CNTP_CVAL); isb(); } } @@ -296,10 +297,10 @@ static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code) break; case SYS_CNTP_CVAL_EL0: if (vcpu_el2_e2h_is_set(vcpu)) { - val = read_sysreg_el0(SYS_CNTP_CVAL); - - if (!has_cntpoff()) - val -= timer_get_offset(vcpu_hptimer(vcpu)); + if (!has_cntpoff() && timer_get_offset(vcpu_hptimer(vcpu))) + val = __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2); + else + val = read_sysreg_el0(SYS_CNTP_CVAL); } else { val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0); } @@ -321,10 +322,14 @@ static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code) val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); break; case SYS_CNTV_CVAL_EL0: - if (vcpu_el2_e2h_is_set(vcpu)) - val = read_sysreg_el0(SYS_CNTV_CVAL); - else + if (vcpu_el2_e2h_is_set(vcpu)) { + if (has_broken_cntvoff() && timer_get_offset(vcpu_hvtimer(vcpu))) + val = __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2); + else + val = read_sysreg_el0(SYS_CNTV_CVAL); + } else { val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0); + } break; case SYS_CNTVCT_EL0: case SYS_CNTVCTSS_EL0: diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c index 9644c424819b..b9404496f977 100644 --- a/arch/arm64/kvm/hyp_trace.c +++ b/arch/arm64/kvm/hyp_trace.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "hyp_trace.h" @@ -59,14 +60,8 @@ static void __hyp_clock_work(struct work_struct *work) if (hyp_clock->mult) { u64 err, cur = delta_cycles; - if (WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64)) { - __uint128_t tmp = (__uint128_t)cur * hyp_clock->mult; - - cur = tmp >> hyp_clock->shift; - } else { - cur *= hyp_clock->mult; - cur >>= hyp_clock->shift; - } + WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64); + cur = mul_u64_u32_shr(cur, hyp_clock->mult, hyp_clock->shift); cur += hyp_clock->boot; err = abs_diff(cur, boot); @@ -84,7 +79,7 @@ static void __hyp_clock_work(struct work_struct *work) err / NSEC_PER_USEC); } - rate = div64_u64(delta_cycles * NSEC_PER_SEC, delta_boot); + rate = mul_u64_u64_div_u64(delta_cycles, NSEC_PER_SEC, delta_boot); clocks_calc_mult_shift(&hyp_clock->mult, &hyp_clock->shift, rate, NSEC_PER_SEC, CLOCK_MAX_CONVERSION_S); @@ -151,11 +146,18 @@ static void __unshare_page(unsigned long va) static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size) { - int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1; + unsigned long nr_bpages; size_t backing_size; void *start; int ret; + if (is_protected_kvm_enabled()) { + trace_buffer->desc->bpages_backing_start = 0; + trace_buffer->desc->bpages_backing_size = 0; + return 0; + } + + nr_bpages = __calc_nr_pages_ring_buffer_desc(size); backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages * num_possible_cpus()); @@ -177,6 +179,9 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_ static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer) { + if (!trace_buffer->desc->bpages_backing_start) + return; + free_pages_exact((void *)trace_buffer->desc->bpages_backing_start, trace_buffer->desc->bpages_backing_size); } @@ -269,7 +274,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv) if (ret) goto err_free_buffer; - ret = kvm_call_hyp_nvhe(__tracing_load, (unsigned long)desc, desc_size); + ret = pkvm_call_hyp_req(__tracing_load, desc, desc_size); if (ret) goto err_unload_pages; @@ -402,7 +407,10 @@ static struct trace_remote_callbacks trace_remote_callbacks = { static const char *__hyp_enter_exit_reason_str(u8 reason); -#include +struct remote_event_format_hyp_printk; +static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq); + +#include "define_hypevents.h" static const char *__hyp_enter_exit_reason_str(u8 reason) { @@ -421,6 +429,61 @@ static const char *__hyp_enter_exit_reason_str(u8 reason) return strs[min(reason, HYP_REASON_UNKNOWN)]; } +static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq) +{ + const struct hyp_string_fmt *hyp_str_fmt; + const char *fmt; + + trace_seq_putc(seq, ' '); + + if (entry->fmt_id >= (__hyp_string_fmts_end - __hyp_string_fmts_start)) { + trace_seq_printf(seq, "Unknown hyp_string_fmt ID %u\n", entry->fmt_id); + return; + } + + hyp_str_fmt = &__hyp_string_fmts_start[entry->fmt_id]; + fmt = hyp_str_fmt->fmt; + + switch (hyp_str_fmt->nr_args) { + case 0: + trace_seq_puts(seq, fmt); + break; + case 1: + trace_seq_printf(seq, fmt, entry->args[0]); + break; + case 2: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1]); + break; + case 3: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2]); + break; + case 4: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3]); + break; + case 5: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4]); + break; + case 6: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4], entry->args[5]); + break; + case 7: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2], + entry->args[3], entry->args[4], entry->args[5], entry->args[6]); + break; + default: + trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], + entry->args[2], entry->args[3], entry->args[4], entry->args[5], + entry->args[6], entry->args[7]); + break; + } + + if (trace_seq_used(seq) && seq->buffer[trace_seq_used(seq) - 1] != '\n') + trace_seq_putc(seq, '\n'); +} + static void __init hyp_trace_init_events(void) { struct hyp_event_id *hyp_event_id = __hyp_event_ids_start; @@ -430,6 +493,9 @@ static void __init hyp_trace_init_events(void) /* Events on both sides hypervisor are sorted */ for (; event < __hyp_events_end; event++, hyp_event_id++, id++) event->id = hyp_event_id->id = id; + + WARN(__hyp_string_fmts_end - __hyp_string_fmts_start > U16_MAX + 1, + "Too many trace_hyp_printk() callsites\n"); } int __init kvm_hyp_trace_init(void) diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index dfa25bb6f25d..f4b932b7bf12 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -294,6 +294,10 @@ int kvm_smccc_call_handler(struct kvm_vcpu *vcpu) case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: feature = smccc_get_arg1(vcpu); switch (feature) { + case ARM_SMCCC_VERSION_FUNC_ID: + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + val[0] = SMCCC_RET_SUCCESS; + break; case ARM_SMCCC_ARCH_WORKAROUND_1: switch (arm64_get_spectre_v2_state()) { case SPECTRE_VULNERABLE: @@ -581,6 +585,13 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) wants_02 = vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2); + /* + * EL2 advertises PSCI 1.1; the host handles the forwarded calls + * by this version. + */ + if (vcpu_is_protected(vcpu) && val < KVM_ARM_PSCI_1_1) + return -EINVAL; + switch (val) { case KVM_ARM_PSCI_0_1: if (wants_02) diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index d6c4fc16f879..96bf926ca520 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -371,15 +371,15 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr) } /* - * Emulate the exception entry if SErrors are unmasked. This is useful if - * the vCPU is in a nested context w/ vSErrors enabled then we've already - * delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2, - * VDISR_EL2) to the guest hypervisor. + * With NV, emulate the exception entry if SErrors are unmasked: in a + * nested context with vSErrors enabled, the hardware vSError context + * (i.e. HCR_EL2.VSE, VSESR_EL2, VDISR_EL2) is delegated to the guest + * hypervisor. Otherwise, HCR_EL2.VSE delivers it once it is unmasked. * * As we're emulating the SError injection we need to explicitly populate * ESR_ELx.EC because hardware will not do it on our behalf. */ - if (!serror_is_masked(vcpu)) { + if (vcpu_has_nv(vcpu) && !serror_is_masked(vcpu)) { pend_serror_exception(vcpu); esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL; vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu)); diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index d1c3a352d5a2..9cd1d3d8f707 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -6,10 +6,12 @@ #include #include +#include #include #include "trace.h" +#if ARM64_S390_COMMON == 1 void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data) { void *datap = NULL; @@ -258,3 +260,4 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa) run->exit_reason = KVM_EXIT_MMIO; return 0; } +#endif /* ARM64_S390_COMMON == 1 */ diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index bf6d6526639f..745cb86f9e04 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -5,8 +5,10 @@ */ #include +#include #include #include +#include #include #include #include @@ -37,6 +39,8 @@ static unsigned long __ro_after_init io_map_base; #define KVM_PGT_FN(fn) (!is_protected_kvm_enabled() ? fn : p ## fn) +static int kvm_vm_init_vm_s2_ops(struct kvm *kvm); + static phys_addr_t __stage2_range_addr_end(phys_addr_t addr, phys_addr_t end, phys_addr_t size) { @@ -175,6 +179,18 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) return memslot->dirty_bitmap && !(memslot->flags & KVM_MEM_READONLY); } +static int pkvm_flush_remote_tlbs(struct kvm *kvm) +{ + kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); + return 0; +} + +static int kvm_vm_flush_remote_tlbs(struct kvm *kvm) +{ + kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); + return 0; +} + /** * kvm_arch_flush_remote_tlbs() - flush all VM TLB entries for v7/8 * @kvm: pointer to kvm structure. @@ -183,26 +199,31 @@ static bool memslot_is_logging(struct kvm_memory_slot *memslot) */ int kvm_arch_flush_remote_tlbs(struct kvm *kvm) { - if (is_protected_kvm_enabled()) - kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); - else - kvm_call_hyp(__kvm_tlb_flush_vmid, &kvm->arch.mmu); - return 0; + return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs(kvm); } -int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, - gfn_t gfn, u64 nr_pages) +static int pkvm_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) +{ + return pkvm_flush_remote_tlbs(kvm); +} + +static int kvm_vm_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) { u64 size = nr_pages << PAGE_SHIFT; u64 addr = gfn << PAGE_SHIFT; - if (is_protected_kvm_enabled()) - kvm_call_hyp_nvhe(__pkvm_tlb_flush_vmid, kvm->arch.pkvm.handle); - else - kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size); + kvm_tlb_flush_vmid_range(&kvm->arch.mmu, addr, size); return 0; } +int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, + gfn_t gfn, u64 nr_pages) +{ + return kvm->arch.vm_s2_ops->vm_flush_remote_tlbs_range(kvm, gfn, nr_pages); +} + static void *stage2_memcache_zalloc_page(void *arg) { struct kvm_mmu_memory_cache *mc = arg; @@ -298,6 +319,37 @@ static void invalidate_icache_guest_page(void *va, size_t size) __invalidate_icache_guest_page(va, size); } +static int kvm_pgtable_stage2_unmap_tracked(struct kvm_pgtable *pgt, + u64 addr, u64 size); + +static void kvm_vm_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, + u64 size, bool may_block) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + int (*fn)(struct kvm_pgtable *, u64, u64); + + if (kvm_is_nested_s2_mmu(kvm, mmu)) + fn = kvm_pgtable_stage2_unmap_tracked; + else + fn = kvm_pgtable_stage2_unmap; + + WARN_ON(stage2_apply_range(mmu, start, start + size, fn, may_block)); +} + +static void pkvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, + u64 size, bool may_block) +{ + WARN_ON(stage2_apply_range(mmu, start, start + size, + pkvm_pgtable_stage2_unmap, may_block)); +} + +static void no_stage2_unmap_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, u64 size, bool may_block) +{ +} + /* * Unmapping vs dcache management: * @@ -322,37 +374,39 @@ static void invalidate_icache_guest_page(void *va, size_t size) * we then fully enforce cacheability of RAM, no matter what the guest * does. */ + +static int kvm_pgtable_stage2_unmap_tracked(struct kvm_pgtable *pgt, u64 addr, u64 size) +{ + int ret; + + ret = kvm_pgtable_stage2_unmap(pgt, addr, size); + if (ret) + return ret; + + kvm_remove_guest_s2_mappings(pgt->mmu, addr, size); + return 0; +} + /** - * __unmap_stage2_range -- Clear stage2 page table entries to unmap a range + * kvm_stage2_unmap_range -- Clear stage2 page table entries to unmap a range * @mmu: The KVM stage-2 MMU pointer * @start: The intermediate physical base address of the range to unmap * @size: The size of the area to unmap * @may_block: Whether or not we are permitted to block * * Clear a range of stage-2 mappings, lowering the various ref-counts. Must - * be called while holding mmu_lock (unless for freeing the stage2 pgd before - * destroying the VM), otherwise another faulting VCPU may come in and mess - * with things behind our backs. + * be called while holding mmu_lock otherwise another faulting VCPU may + * come in and mess with things behind our backs. */ -static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, - bool may_block) -{ - struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - phys_addr_t end = start + size; - - lockdep_assert_held_write(&kvm->mmu_lock); - WARN_ON(size & ~PAGE_MASK); - WARN_ON(stage2_apply_range(mmu, start, end, KVM_PGT_FN(kvm_pgtable_stage2_unmap), - may_block)); -} - void kvm_stage2_unmap_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size, bool may_block) { - if (kvm_vm_is_protected(kvm_s2_mmu_to_kvm(mmu))) - return; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - __unmap_stage2_range(mmu, start, size, may_block); + lockdep_assert_held_write(&kvm->mmu_lock); + WARN_ON(size & ~PAGE_MASK); + + kvm->arch.vm_s2_ops->vm_stage2_unmap_range(mmu, start, size, may_block); } void kvm_stage2_flush_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end) @@ -875,7 +929,7 @@ static int get_user_mapping_size(struct kvm *kvm, u64 addr) * IPI-ing threads). */ local_irq_save(flags); - ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level); + ret = kvm_pgtable_get_leaf(&pgt, addr, &pte, &level, 0); local_irq_restore(flags); if (ret) @@ -992,6 +1046,12 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t int cpu, err; struct kvm_pgtable *pgt; + /* Initialize the VM ops for the VM instance for the first time */ + if (mmu == &kvm->arch.mmu) { + err = kvm_vm_init_vm_s2_ops(kvm); + if (err) + return err; + } /* * If we already have our page tables in place, and that the * MMU context is the canonical one, we have a bug somewhere, @@ -1042,6 +1102,8 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu, unsigned long t mmu->pgd_phys = __pa(pgt->pgd); + mmu->guest_s2_mappings = RB_ROOT_CACHED; + if (kvm_is_nested_s2_mmu(kvm, mmu)) kvm_init_nested_s2_mmu(mmu); @@ -1131,10 +1193,25 @@ void stage2_unmap_vm(struct kvm *kvm) srcu_read_unlock(&kvm->srcu, idx); } +static void guest_s2_tracking_destroy(struct rb_root_cached *tree) +{ + struct kvm_guest_s2_mapping *mapping; + struct interval_tree_node *node; + + while ((node = interval_tree_iter_first(tree, 0, ULONG_MAX))) { + interval_tree_remove(node, tree); + mapping = container_of(node, struct kvm_guest_s2_mapping, + canonical); + kfree(mapping); + cond_resched(); + } +} + void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) { struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); struct kvm_pgtable *pgt = NULL; + struct rb_root_cached mappings_tree; write_lock(&kvm->mmu_lock); pgt = mmu->pgt; @@ -1147,12 +1224,18 @@ void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu) if (kvm_is_nested_s2_mmu(kvm, mmu)) kvm_init_nested_s2_mmu(mmu); + mappings_tree = mmu->guest_s2_mappings; + mmu->guest_s2_mappings = RB_ROOT_CACHED; + write_unlock(&kvm->mmu_lock); if (pgt) { kvm_stage2_destroy(pgt); kfree(pgt); } + + if (!kvm_is_nested_s2_mmu(kvm, mmu)) + guest_s2_tracking_destroy(&mappings_tree); } static void hyp_mc_free_fn(void *addr, void *mc) @@ -1169,8 +1252,10 @@ static void *hyp_mc_alloc_fn(void *mc) { struct kvm_hyp_memcache *memcache = mc; void *addr; + gfp_t gfp; - addr = (void *)__get_free_page(GFP_KERNEL_ACCOUNT); + gfp = memcache->flags & HYP_MEMCACHE_ACCOUNT_KMEMCG ? GFP_KERNEL_ACCOUNT : GFP_KERNEL; + addr = (void *)__get_free_page(gfp); if (addr && memcache->flags & HYP_MEMCACHE_ACCOUNT_STAGE2) kvm_account_pgtable_pages(addr, 1); @@ -1223,7 +1308,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, KVM_PGTABLE_PROT_R | (writable ? KVM_PGTABLE_PROT_W : 0); - if (is_protected_kvm_enabled()) + if (kvm_vm_hyp_is_distrusting(kvm)) return -EPERM; size += offset_in_page(guest_ipa); @@ -1362,21 +1447,53 @@ static void kvm_send_hwpoison_signal(unsigned long address, short lsb) send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current); } -static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, - unsigned long hva, +struct kvm_s2_fault_desc { + struct kvm_vcpu *vcpu; + phys_addr_t fault_ipa; + struct kvm_s2_trans *nested; + struct kvm_memory_slot *memslot; + unsigned long hva; + unsigned long esr; + struct kvm_s2_mmu *mmu; +}; + +struct kvm_s2_fault_vma_info { + unsigned long mmu_seq; + long vma_pagesize; + vm_flags_t vm_flags; + unsigned long max_map_size; + struct page *page; + kvm_pfn_t pfn; + gfn_t gfn; + bool device; + bool mte_allowed; + bool is_vma_cacheable; + bool map_writable; + bool map_non_cacheable; +}; + +struct kvm_s2_fault_result { + unsigned long mapping_size; +}; + +static bool fault_supports_stage2_huge_mapping(const struct kvm_s2_fault_desc *s2fd, unsigned long map_size) { - gpa_t gpa_start; + struct kvm_memory_slot *memslot = s2fd->memslot; + unsigned long hva = s2fd->hva; hva_t uaddr_start, uaddr_end; + gpa_t gpa_start; size_t size; /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ if (map_size == PAGE_SIZE) return true; - /* pKVM only supports PMD_SIZE huge-mappings */ - if (is_protected_kvm_enabled() && map_size != PMD_SIZE) - return false; + /* pKVM only supports PMD_SIZE huge-mappings for non-protected VMs */ + if (is_protected_kvm_enabled()) { + if (vcpu_is_protected(s2fd->vcpu) || map_size != PMD_SIZE) + return false; + } size = memslot->npages * PAGE_SIZE; @@ -1436,9 +1553,9 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, * Returns the size of the mapping. */ static long -transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, - unsigned long hva, kvm_pfn_t *pfnp, gfn_t *gfnp) +transparent_hugepage_adjust(const struct kvm_s2_fault_desc *s2fd, kvm_pfn_t *pfnp, gfn_t *gfnp) { + struct kvm *kvm = s2fd->vcpu->kvm; kvm_pfn_t pfn = *pfnp; gfn_t gfn = *gfnp; @@ -1447,8 +1564,8 @@ transparent_hugepage_adjust(struct kvm *kvm, struct kvm_memory_slot *memslot, * sure that the HVA and IPA are sufficiently aligned and that the * block map is contained within the memslot. */ - if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { - int sz = get_user_mapping_size(kvm, hva); + if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) { + int sz = get_user_mapping_size(kvm, s2fd->hva); if (sz < 0) return sz; @@ -1539,12 +1656,12 @@ static void *get_mmu_memcache(struct kvm_vcpu *vcpu) if (!is_protected_kvm_enabled()) return &vcpu->arch.mmu_page_cache; else - return &vcpu->arch.pkvm_memcache; + return &vcpu->arch.stage2_mc; } -static int topup_mmu_memcache(struct kvm_vcpu *vcpu, void *memcache) +static int topup_mmu_memcache(struct kvm_s2_mmu *mmu, void *memcache) { - int min_pages = kvm_mmu_cache_min_pages(vcpu->arch.hw_mmu); + int min_pages = kvm_mmu_cache_min_pages(mmu); if (!is_protected_kvm_enabled()) return kvm_mmu_topup_memory_cache(memcache, min_pages); @@ -1585,55 +1702,85 @@ static enum kvm_pgtable_prot adjust_nested_exec_perms(struct kvm *kvm, return prot; } -struct kvm_s2_fault_desc { - struct kvm_vcpu *vcpu; - phys_addr_t fault_ipa; - struct kvm_s2_trans *nested; - struct kvm_memory_slot *memslot; - unsigned long hva; -}; +static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_fsc_is_permission_fault(s2fd->esr); +} -static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) +static bool kvm_s2_fault_is_exec(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_abt_is_exec_fault(s2fd->esr); +} + +static bool kvm_s2_fault_is_write(const struct kvm_s2_fault_desc *s2fd) +{ + return esr_abt_is_write_fault(s2fd->esr); +} + +static u64 kvm_s2_perm_fault_granule(const struct kvm_s2_fault_desc *s2fd) +{ + u64 level; + + if (!kvm_s2_fault_is_perm(s2fd)) + return 0; + level = s2fd->esr & ESR_ELx_FSC_LEVEL; + return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(level)); +} + +static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_vma_info *s2vi); + +static gfn_t get_canonical_gfn(const struct kvm_s2_fault_desc *s2fd, + const struct kvm_s2_fault_vma_info *s2vi); + +static int gmem_abort(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_result *result) { bool write_fault, exec_fault; - bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); + bool perm_fault = kvm_s2_fault_is_perm(s2fd); enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R; - struct kvm_pgtable *pgt = s2fd->vcpu->arch.hw_mmu->pgt; - unsigned long mmu_seq; - struct page *page; + struct kvm_pgtable *pgt = s2fd->mmu->pgt; + struct kvm_guest_s2_mapping *mapping = NULL; + struct kvm_s2_fault_vma_info s2vi = {}; struct kvm *kvm = s2fd->vcpu->kvm; void *memcache = NULL; - kvm_pfn_t pfn; - gfn_t gfn; + gfn_t canonical_gfn; int ret; if (!perm_fault) { memcache = get_mmu_memcache(s2fd->vcpu); - ret = topup_mmu_memcache(s2fd->vcpu, memcache); + ret = topup_mmu_memcache(s2fd->mmu, memcache); if (ret) return ret; + if (kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + mapping = kmalloc_obj(struct kvm_guest_s2_mapping, GFP_KERNEL_ACCOUNT); + if (!mapping) + return -ENOMEM; + } } - if (s2fd->nested) - gfn = kvm_s2_trans_output(s2fd->nested) >> PAGE_SHIFT; - else - gfn = s2fd->fault_ipa >> PAGE_SHIFT; + s2vi.vma_pagesize = PAGE_SIZE; + s2vi.gfn = ALIGN_DOWN(s2fd->fault_ipa, s2vi.vma_pagesize) >> PAGE_SHIFT; + canonical_gfn = get_canonical_gfn(s2fd, &s2vi); - write_fault = kvm_is_write_fault(s2fd->vcpu); - exec_fault = kvm_vcpu_trap_is_exec_fault(s2fd->vcpu); + write_fault = kvm_s2_fault_is_write(s2fd); + exec_fault = kvm_s2_fault_is_exec(s2fd); VM_WARN_ON_ONCE(write_fault && exec_fault); - mmu_seq = kvm->mmu_invalidate_seq; + s2vi.mmu_seq = kvm->mmu_invalidate_seq; /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */ smp_rmb(); - ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(kvm, s2fd->memslot, canonical_gfn, &s2vi.pfn, NULL); if (ret) { - kvm_prepare_memory_fault_exit(s2fd->vcpu, s2fd->fault_ipa, PAGE_SIZE, - write_fault, exec_fault, false); - return ret; + /* If result is non-NULL this is a synthetic fault. */ + if (!result) + kvm_prepare_memory_fault_exit(s2fd->vcpu, gfn_to_gpa(canonical_gfn), + s2vi.vma_pagesize, write_fault, exec_fault, + false); + kfree(mapping); } if (!(s2fd->memslot->flags & KVM_MEM_READONLY)) @@ -1649,7 +1796,7 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) prot = adjust_nested_exec_perms(kvm, s2fd->nested, prot); kvm_fault_lock(kvm); - if (mmu_invalidate_retry(kvm, mmu_seq)) { + if (mmu_invalidate_retry(kvm, s2vi.mmu_seq)) { ret = -EAGAIN; goto out_unlock; } @@ -1660,61 +1807,67 @@ static int gmem_abort(const struct kvm_s2_fault_desc *s2fd) * PTE, which will be preserved. */ prot &= ~KVM_NV_GUEST_MAP_SZ; - ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, s2fd->fault_ipa, + ret = KVM_PGT_FN(kvm_pgtable_stage2_relax_perms)(pgt, gfn_to_gpa(s2vi.gfn), prot, flags); } else { - ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, s2fd->fault_ipa, PAGE_SIZE, - __pfn_to_phys(pfn), prot, + ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(s2vi.gfn), + s2vi.vma_pagesize, + __pfn_to_phys(s2vi.pfn), prot, memcache, flags); + /* + * -EAGAIN from kvm_pgtable_stage2_map() can install mappings. + * We don't know which subrange is installed, track the whole + * thing. + */ + if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + kvm_record_guest_s2_mapping(pgt->mmu, canonical_gfn << PAGE_SHIFT, + s2fd->fault_ipa, s2vi.vma_pagesize, mapping); + mapping = NULL; + } } out_unlock: - kvm_release_faultin_page(kvm, page, !!ret, prot & KVM_PGTABLE_PROT_W); kvm_fault_unlock(kvm); + kfree(mapping); if ((prot & KVM_PGTABLE_PROT_W) && !ret) - mark_page_dirty_in_slot(kvm, s2fd->memslot, gfn); + mark_page_dirty_in_slot(kvm, s2fd->memslot, canonical_gfn); - return ret != -EAGAIN ? ret : 0; + if (ret == -EAGAIN) + return result ? ret : 0; + + if (result && !ret) + result->mapping_size = s2vi.vma_pagesize; + + return ret; } -struct kvm_s2_fault_vma_info { - unsigned long mmu_seq; - long vma_pagesize; - vm_flags_t vm_flags; - unsigned long max_map_size; - struct page *page; - kvm_pfn_t pfn; - gfn_t gfn; - bool device; - bool mte_allowed; - bool is_vma_cacheable; - bool map_writable; - bool map_non_cacheable; -}; - -static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) +static int protected_pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) { unsigned int flags = FOLL_HWPOISON | FOLL_LONGTERM | FOLL_WRITE; + struct kvm_s2_fault_vma_info s2vi = {}; struct kvm_vcpu *vcpu = s2fd->vcpu; - struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt; + struct kvm_pgtable *pgt = s2fd->mmu->pgt; struct mm_struct *mm = current->mm; struct kvm *kvm = vcpu->kvm; void *hyp_memcache; - struct page *page; int ret; hyp_memcache = get_mmu_memcache(vcpu); - ret = topup_mmu_memcache(vcpu, hyp_memcache); + ret = topup_mmu_memcache(s2fd->mmu, hyp_memcache); if (ret) return -ENOMEM; + ret = kvm_s2_fault_get_vma_info(s2fd, &s2vi); + if (ret) + return ret; + ret = account_locked_vm(mm, 1, true); if (ret) return ret; mmap_read_lock(mm); - ret = pin_user_pages(s2fd->hva, 1, flags, &page); + ret = pin_user_pages(s2fd->hva, 1, flags, &s2vi.page); mmap_read_unlock(mm); if (ret == -EHWPOISON) { @@ -1724,7 +1877,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) } else if (ret != 1) { ret = -EFAULT; goto dec_account; - } else if (!folio_test_swapbacked(page_folio(page))) { + } else if (!folio_test_swapbacked(page_folio(s2vi.page))) { /* * We really can't deal with page-cache pages returned by GUP * because (a) we may trigger writeback of a page for which we @@ -1744,8 +1897,8 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) } write_lock(&kvm->mmu_lock); - ret = pkvm_pgtable_stage2_map(pgt, s2fd->fault_ipa, PAGE_SIZE, - page_to_phys(page), KVM_PGTABLE_PROT_RWX, + ret = pkvm_pgtable_stage2_map(pgt, gfn_to_gpa(s2vi.gfn), PAGE_SIZE, + page_to_phys(s2vi.page), KVM_PGTABLE_PROT_RWX, hyp_memcache, 0); write_unlock(&kvm->mmu_lock); if (ret) { @@ -1756,7 +1909,7 @@ static int pkvm_mem_abort(const struct kvm_s2_fault_desc *s2fd) return 0; unpin: - unpin_user_pages(&page, 1); + unpin_user_page(s2vi.page); dec_account: account_locked_vm(mm, 1, false); return ret; @@ -1779,7 +1932,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, switch (vma_shift) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SHIFT: - if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PUD_SIZE)) + if (fault_supports_stage2_huge_mapping(s2fd, PUD_SIZE)) break; fallthrough; #endif @@ -1787,7 +1940,7 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, vma_shift = PMD_SHIFT; fallthrough; case PMD_SHIFT: - if (fault_supports_stage2_huge_mapping(s2fd->memslot, s2fd->hva, PMD_SIZE)) + if (fault_supports_stage2_huge_mapping(s2fd, PMD_SIZE)) break; fallthrough; case CONT_PTE_SHIFT: @@ -1828,11 +1981,6 @@ static short kvm_s2_resolve_vma_size(const struct kvm_s2_fault_desc *s2fd, return vma_shift; } -static bool kvm_s2_fault_is_perm(const struct kvm_s2_fault_desc *s2fd) -{ - return kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); -} - static int kvm_s2_fault_get_vma_info(const struct kvm_s2_fault_desc *s2fd, struct kvm_s2_fault_vma_info *s2vi) { @@ -1898,13 +2046,11 @@ static int kvm_s2_fault_pin_pfn(const struct kvm_s2_fault_desc *s2fd, return ret; s2vi->pfn = __kvm_faultin_pfn(s2fd->memslot, get_canonical_gfn(s2fd, s2vi), - kvm_is_write_fault(s2fd->vcpu) ? FOLL_WRITE : 0, + kvm_s2_fault_is_write(s2fd) ? FOLL_WRITE : 0, &s2vi->map_writable, &s2vi->page); if (unlikely(is_error_noslot_pfn(s2vi->pfn))) { - if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) { - kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi->vma_pagesize)); - return 0; - } + if (s2vi->pfn == KVM_PFN_ERR_HWPOISON) + return -EHWPOISON; return -EFAULT; } @@ -1946,7 +2092,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, { struct kvm *kvm = s2fd->vcpu->kvm; - if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu) && s2vi->map_non_cacheable) + if (kvm_s2_fault_is_exec(s2fd) && s2vi->map_non_cacheable) return -ENOEXEC; /* @@ -1955,7 +2101,7 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, * and trigger the exception here. Since the memslot is valid, inject * the fault back to the guest. */ - if (esr_fsc_is_excl_atomic_fault(kvm_vcpu_get_esr(s2fd->vcpu))) { + if (esr_fsc_is_excl_atomic_fault(s2fd->esr)) { kvm_inject_dabt_excl_atomic(s2fd->vcpu, kvm_vcpu_get_hfar(s2fd->vcpu)); return 1; } @@ -1964,13 +2110,13 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, if (s2vi->map_writable && (s2vi->device || !memslot_is_logging(s2fd->memslot) || - kvm_is_write_fault(s2fd->vcpu))) + kvm_s2_fault_is_write(s2fd))) *prot |= KVM_PGTABLE_PROT_W; if (s2fd->nested) *prot = adjust_nested_fault_perms(s2fd->nested, *prot); - if (kvm_vcpu_trap_is_exec_fault(s2fd->vcpu)) + if (kvm_s2_fault_is_exec(s2fd)) *prot |= KVM_PGTABLE_PROT_X; if (s2vi->map_non_cacheable) @@ -1994,11 +2140,14 @@ static int kvm_s2_fault_compute_prot(const struct kvm_s2_fault_desc *s2fd, static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, const struct kvm_s2_fault_vma_info *s2vi, enum kvm_pgtable_prot prot, - void *memcache) + void *memcache, + struct kvm_s2_fault_result *result) { enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED; + struct kvm_guest_s2_mapping *mapping = NULL; bool writable = prot & KVM_PGTABLE_PROT_W; struct kvm *kvm = s2fd->vcpu->kvm; + phys_addr_t canonical_ipa; struct kvm_pgtable *pgt; long perm_fault_granule; long mapping_size; @@ -2006,17 +2155,26 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, gfn_t gfn; int ret; + if (kvm_is_nested_s2_mmu(kvm, s2fd->mmu)) { + mapping = kmalloc_obj(struct kvm_guest_s2_mapping, + GFP_KERNEL_ACCOUNT); + if (!mapping) { + kvm_release_page_unused(s2vi->page); + return -ENOMEM; + } + } + kvm_fault_lock(kvm); - pgt = s2fd->vcpu->arch.hw_mmu->pgt; + pgt = s2fd->mmu->pgt; ret = -EAGAIN; if (mmu_invalidate_retry(kvm, s2vi->mmu_seq)) goto out_unlock; - perm_fault_granule = (kvm_s2_fault_is_perm(s2fd) ? - kvm_vcpu_trap_get_perm_fault_granule(s2fd->vcpu) : 0); + perm_fault_granule = kvm_s2_perm_fault_granule(s2fd); mapping_size = s2vi->vma_pagesize; pfn = s2vi->pfn; gfn = s2vi->gfn; + canonical_ipa = gfn_to_gpa(get_canonical_gfn(s2fd, s2vi)); /* * If we are not forced to use page mapping, check if we are @@ -2027,14 +2185,13 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, if (perm_fault_granule > PAGE_SIZE) { mapping_size = perm_fault_granule; } else { - mapping_size = transparent_hugepage_adjust(kvm, s2fd->memslot, - s2fd->hva, &pfn, - &gfn); + mapping_size = transparent_hugepage_adjust(s2fd, &pfn, &gfn); if (mapping_size < 0) { ret = mapping_size; goto out_unlock; } } + canonical_ipa = ALIGN_DOWN(canonical_ipa, mapping_size); } if (!perm_fault_granule && !s2vi->map_non_cacheable && kvm_has_mte(kvm)) @@ -2057,31 +2214,45 @@ static int kvm_s2_fault_map(const struct kvm_s2_fault_desc *s2fd, ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, gfn_to_gpa(gfn), mapping_size, __pfn_to_phys(pfn), prot, memcache, flags); + /* + * -EAGAIN from kvm_pgtable_stage2_map() can install mappings. + * We don't know which subrange is installed, track the whole + * thing. + */ + if ((ret == 0 || ret == -EAGAIN) && kvm_is_nested_s2_mmu(kvm, pgt->mmu)) { + kvm_record_guest_s2_mapping(pgt->mmu, canonical_ipa, + gfn_to_gpa(gfn), mapping_size, mapping); + mapping = NULL; + } } out_unlock: kvm_release_faultin_page(kvm, s2vi->page, !!ret, writable); kvm_fault_unlock(kvm); + kfree(mapping); /* * Mark the page dirty only if the fault is handled successfully, * making sure we adjust the canonical IPA if the mapping size has * been updated (via a THP upgrade, for example). */ - if (writable && !ret) { - phys_addr_t ipa = gfn_to_gpa(get_canonical_gfn(s2fd, s2vi)); - ipa &= ~(mapping_size - 1); - mark_page_dirty_in_slot(kvm, s2fd->memslot, gpa_to_gfn(ipa)); - } + if (writable && !ret) + mark_page_dirty_in_slot(kvm, s2fd->memslot, + gpa_to_gfn(canonical_ipa)); - if (ret != -EAGAIN) - return ret; - return 0; + if (ret == -EAGAIN) + return result ? ret : 0; + + if (result && !ret) + result->mapping_size = mapping_size; + + return ret; } -static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) +static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd, + struct kvm_s2_fault_result *result) { - bool perm_fault = kvm_vcpu_trap_is_permission_fault(s2fd->vcpu); + bool perm_fault = kvm_s2_fault_is_perm(s2fd); struct kvm_s2_fault_vma_info s2vi = {}; enum kvm_pgtable_prot prot; void *memcache; @@ -2099,12 +2270,19 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) memcache = get_mmu_memcache(s2fd->vcpu); if (!perm_fault || memslot_is_logging(s2fd->memslot) || is_protected_kvm_enabled()) { - ret = topup_mmu_memcache(s2fd->vcpu, memcache); + ret = topup_mmu_memcache(s2fd->mmu, memcache); if (ret) return ret; } ret = kvm_s2_fault_pin_pfn(s2fd, &s2vi); + if (ret == -EHWPOISON) { + /* If result is specified, let the caller handle this. */ + if (result) + return -EHWPOISON; + kvm_send_hwpoison_signal(s2fd->hva, __ffs(s2vi.vma_pagesize)); + return 0; + } if (ret != 1) return ret; @@ -2114,7 +2292,19 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) return ret; } - return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache); + return kvm_s2_fault_map(s2fd, &s2vi, prot, memcache, result); +} + +static int kvm_vm_mem_abort(const struct kvm_s2_fault_desc *s2fd) +{ + VM_WARN_ON_ONCE(kvm_s2_fault_is_perm(s2fd) && + !esr_abt_is_write_fault(s2fd->esr) && + !kvm_s2_fault_is_exec(s2fd)); + + if (kvm_slot_has_gmem(s2fd->memslot)) + return gmem_abort(s2fd, NULL); + else + return user_mem_abort(s2fd, NULL); } /* Resolve the access fault by making the page young again. */ @@ -2224,7 +2414,9 @@ int kvm_handle_guest_sea(struct kvm_vcpu *vcpu) int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) { struct kvm_s2_trans nested_trans, *nested = NULL; - unsigned long esr; + unsigned long esr = kvm_vcpu_get_esr(vcpu); + struct kvm_s2_mmu *mmu = vcpu->arch.hw_mmu; + struct kvm *kvm = vcpu->kvm; phys_addr_t fault_ipa; /* The address we faulted on */ phys_addr_t ipa; /* Always the IPA in the L1 guest phys space */ struct kvm_memory_slot *memslot; @@ -2233,20 +2425,18 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) gfn_t gfn; int ret, idx; - if (kvm_vcpu_abt_issea(vcpu)) + if (esr_abt_is_sea(esr)) return kvm_handle_guest_sea(vcpu); - esr = kvm_vcpu_get_esr(vcpu); - /* * The fault IPA should be reliable at this point as we're not dealing * with an SEA. */ ipa = fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); - if (KVM_BUG_ON(ipa == INVALID_GPA, vcpu->kvm)) + if (KVM_BUG_ON(ipa == INVALID_GPA, kvm)) return -EFAULT; - is_iabt = kvm_vcpu_trap_is_iabt(vcpu); + is_iabt = esr_trap_is_iabt(esr); if (esr_fsc_is_translation_fault(esr)) { /* Beyond sanitised PARange (which is the IPA limit) */ @@ -2256,14 +2446,14 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } /* Falls between the IPA range and the PARange? */ - if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(vcpu->arch.hw_mmu->vtcr))) { + if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(mmu->vtcr))) { fault_ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(vcpu)); return kvm_inject_sea(vcpu, is_iabt, fault_ipa); } } - trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu), + trace_kvm_guest_fault(*vcpu_pc(vcpu), esr, kvm_vcpu_get_hfar(vcpu), fault_ipa); /* Check the stage-2 fault is trans. fault or write fault */ @@ -2271,14 +2461,14 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) !esr_fsc_is_permission_fault(esr) && !esr_fsc_is_access_flag_fault(esr) && !esr_fsc_is_excl_atomic_fault(esr)) { - kvm_err("Unsupported FSC: EC=%#x xFSC=%#lx ESR_EL2=%#lx\n", - kvm_vcpu_trap_get_class(vcpu), - (unsigned long)kvm_vcpu_trap_get_fault(vcpu), - (unsigned long)kvm_vcpu_get_esr(vcpu)); + kvm_err("Unsupported FSC: EC=%#lx xFSC=%#lx ESR_EL2=%#lx\n", + ESR_ELx_EC(esr), + (unsigned long)(esr & ESR_ELx_FSC), + (unsigned long)esr); return -EFAULT; } - idx = srcu_read_lock(&vcpu->kvm->srcu); + idx = srcu_read_lock(&kvm->srcu); /* * We may have faulted on a shadow stage 2 page table if we are @@ -2293,8 +2483,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * nothing to walk and we treat it as a 1:1 before going through the * canonical translation. */ - if (kvm_is_nested_s2_mmu(vcpu->kvm,vcpu->arch.hw_mmu) && - vcpu->arch.hw_mmu->nested_stage2_enabled) { + if (kvm_is_nested_s2_mmu(kvm, mmu) && mmu->nested_stage2_enabled) { u32 esr; ret = kvm_walk_nested_s2(vcpu, fault_ipa, &nested_trans); @@ -2321,9 +2510,9 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } gfn = ipa >> PAGE_SHIFT; - memslot = gfn_to_memslot(vcpu->kvm, gfn); + memslot = gfn_to_memslot(kvm, gfn); hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); - write_fault = kvm_is_write_fault(vcpu); + write_fault = esr_abt_is_write_fault(esr); if (kvm_is_error_hva(hva) || (write_fault && !writable)) { /* * The guest has put either its instructions or its page-tables @@ -2336,7 +2525,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) goto out; } - if (kvm_vcpu_abt_iss1tw(vcpu)) { + if (esr_abt_is_s1ptw(esr)) { ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu)); goto out_unlock; } @@ -2351,7 +2540,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) * So let's assume that the guest is just being * cautious, and skip the instruction. */ - if (kvm_is_error_hva(hva) && kvm_vcpu_dabt_is_cm(vcpu)) { + if (kvm_is_error_hva(hva) && esr_dabt_is_cm(esr)) { kvm_incr_pc(vcpu); ret = 1; goto out_unlock; @@ -2369,7 +2558,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) } /* Userspace should not be able to register out-of-bounds IPAs */ - VM_BUG_ON(ipa >= kvm_phys_size(vcpu->arch.hw_mmu)); + VM_BUG_ON(ipa >= kvm_phys_size(mmu)); if (esr_fsc_is_access_flag_fault(esr)) { handle_access_fault(vcpu, fault_ipa); @@ -2383,72 +2572,98 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) .nested = nested, .memslot = memslot, .hva = hva, + .esr = esr, + .mmu = mmu, }; - if (kvm_vm_is_protected(vcpu->kvm)) { - ret = pkvm_mem_abort(&s2fd); - } else { - VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) && - !write_fault && - !kvm_vcpu_trap_is_exec_fault(vcpu)); - - if (kvm_slot_has_gmem(memslot)) - ret = gmem_abort(&s2fd); - else - ret = user_mem_abort(&s2fd); - } - + ret = kvm->arch.vm_s2_ops->vm_mem_abort(&s2fd); if (ret == 0) ret = 1; out: if (ret == -ENOEXEC) ret = kvm_inject_sea_iabt(vcpu, kvm_vcpu_get_hfar(vcpu)); out_unlock: - srcu_read_unlock(&vcpu->kvm->srcu, idx); + srcu_read_unlock(&kvm->srcu, idx); return ret; } bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) + gpa_t gpa = range->start << PAGE_SHIFT; + size_t size = (range->end - range->start) << PAGE_SHIFT; + bool may_block = range->may_block; + + if (!kvm->arch.mmu.pgt) return false; - __unmap_stage2_range(&kvm->arch.mmu, range->start << PAGE_SHIFT, - (range->end - range->start) << PAGE_SHIFT, - range->may_block); + kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, size, may_block); + kvm_nested_unmap_cipa_range(kvm, gpa, size, may_block); - kvm_nested_s2_unmap(kvm, range->may_block); + return false; +} + +static bool kvm_vm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, true); +} + +static bool pkvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, true); +} + +static bool no_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + /* The hypervisor doesn't support aging */ return false; } bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - u64 size = (range->end - range->start) << PAGE_SHIFT; - - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) + if (!kvm->arch.mmu.pgt) return false; - return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt, - range->start << PAGE_SHIFT, - size, true); + return kvm->arch.vm_s2_ops->vm_age_gfn(kvm, range); /* * TODO: Handle nested_mmu structures here using the reverse mapping in * a later version of patch series. */ } -bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +static bool kvm_vm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { u64 size = (range->end - range->start) << PAGE_SHIFT; - if (!kvm->arch.mmu.pgt || kvm_vm_is_protected(kvm)) - return false; - - return KVM_PGT_FN(kvm_pgtable_stage2_test_clear_young)(kvm->arch.mmu.pgt, + return kvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, range->start << PAGE_SHIFT, size, false); } +static bool pkvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + u64 size = (range->end - range->start) << PAGE_SHIFT; + + return pkvm_pgtable_stage2_test_clear_young(kvm->arch.mmu.pgt, + range->start << PAGE_SHIFT, + size, false); +} + +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + if (!kvm->arch.mmu.pgt) + return false; + + return kvm->arch.vm_s2_ops->vm_test_age_gfn(kvm, range); +} + phys_addr_t kvm_mmu_get_httbr(void) { return __pa(hyp_pgtable->pgd); @@ -2699,7 +2914,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, write_lock(&kvm->mmu_lock); kvm_stage2_unmap_range(&kvm->arch.mmu, gpa, size, true); - kvm_nested_s2_unmap(kvm, true); + kvm_nested_unmap_cipa_range(kvm, gpa, size, true); write_unlock(&kvm->mmu_lock); } @@ -2770,3 +2985,193 @@ void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled) trace_kvm_toggle_cache(*vcpu_pc(vcpu), was_enabled, now_enabled); } + +/* + * Try to walk to the specified GPA in canonical mmu - if unmapped returns 0, if + * mapped returns the granule size, otherwise returns an error. + */ +static long kvm_walk_s2(struct kvm_pgtable *pgt, + gpa_t gpa, s8 *level) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(pgt->mmu); + kvm_pte_t pte; + long ret; + + guard(read_lock)(&kvm->mmu_lock); + + ret = kvm_pgtable_get_leaf(pgt, gpa, &pte, level, + KVM_PGTABLE_WALK_SHARED); + if (ret) + return ret; + /* Unpopulated, must fault. */ + if (!kvm_pte_valid(pte)) + return 0; + return kvm_granule_size(*level); +} + +/* Synthesised data abort at specified page table level. */ +#define PRE_FAULT_ESR(level) \ + ((ESR_ELx_EC_DABT_LOW << ESR_ELx_EC_SHIFT) | \ + ESR_ELx_IL | ESR_ELx_FSC_FAULT_L(level)) + +/* Retrieve either a read-only or a read/write hva. */ +static hva_t gfn_to_hva_memslot_read(struct kvm_memory_slot *slot, gfn_t gfn) +{ + return gfn_to_hva_memslot_prot(slot, gfn, /*writable=*/NULL); +} + +static long __pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu, + gpa_t gpa, struct kvm_memory_slot *memslot, s8 level) +{ + const bool is_gmem = kvm_slot_has_gmem(memslot); + const gfn_t gfn = gpa_to_gfn(gpa); + const hva_t hva = is_gmem ? 0 : gfn_to_hva_memslot_read(memslot, gfn); + const struct kvm_s2_fault_desc s2fd = { + .vcpu = vcpu, + .fault_ipa = gpa, + .nested = NULL, + .memslot = memslot, + .hva = hva, + .esr = PRE_FAULT_ESR(level), + .mmu = mmu, + }; + struct kvm_s2_fault_result result = {}; + long ret; + + if (kvm_is_error_hva(hva)) + return -EFAULT; + + if (is_gmem) + ret = gmem_abort(&s2fd, &result); + else + ret = user_mem_abort(&s2fd, &result); + if (IS_ERR_VALUE(ret)) + return ret; + return result.mapping_size; +} + +static long pre_fault_s2(struct kvm_s2_mmu *mmu, struct kvm_vcpu *vcpu, + gpa_t gpa, struct kvm_memory_slot *memslot) +{ + s8 level; + long ret; + + /* Try a walk first. */ + ret = kvm_walk_s2(mmu->pgt, gpa, &level); + if (ret) + return ret; + /* OK, have to fault page in. */ + return __pre_fault_s2(mmu, vcpu, gpa, memslot, level); +} + +static unsigned long +pre_fault_bytes_consumed(gpa_t gpa, unsigned long granule_size, + unsigned long bytes_remaining) +{ + /* Granules are always a power-of-2. */ + const unsigned long granule_bytes_remaining = + granule_size - (gpa % granule_size); + + return min(granule_bytes_remaining, bytes_remaining); +} + +/* If you lose the race this many times, time to give up. */ +#define MAX_PRE_FAULT_RETRIES 3 + +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (is_protected_kvm_enabled()) + return -EOPNOTSUPP; + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + return 0; +} + +/** + * kvm_arch_vcpu_pre_fault_memory - pre-fault stage-2 page tables for the + * specified GPA. + * @vcpu: The VCPU pointer + * @range: {gpa, size, flags} tuple + * + * The mapping performed is always best-effort - faulting in is necessarily + * racey. The ranges faulted in are canonical, nested page tables are ignored. + * + * @range->gpa specifies the GPA to pre-fault, @range->size specifies how many + * bytes remain to be pre-faulted and @range->flags is reserved and must be 0. + * + * Returns: the number of bytes the pre-fault consumed, or an error. + */ +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, + struct kvm_pre_fault_memory *range) +{ + struct kvm *kvm = vcpu->kvm; + const u64 bytes_remaining = range->size; + struct kvm_s2_mmu *mmu = &kvm->arch.mmu; /* Canonical. */ + struct kvm_memory_slot *memslot; + const gpa_t gpa = range->gpa; + int num_retries = 0; + long ret; + + memslot = gfn_to_memslot(kvm, gpa_to_gfn(gpa)); + if (!memslot) + return -ENOENT; + /* SRCU must be released for progress and only userland can do that. */ + if (memslot->flags & KVM_MEMSLOT_INVALID) + return -EAGAIN; + + do { + ret = pre_fault_s2(mmu, vcpu, gpa, memslot); + } while (ret == -EAGAIN && num_retries++ < MAX_PRE_FAULT_RETRIES); + + if (IS_ERR_VALUE(ret)) + return ret; + return pre_fault_bytes_consumed(gpa, ret, bytes_remaining); +} + +static const struct kvm_vm_s2_ops protected_pkvm_vm_s2_ops = { + .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range, + .vm_age_gfn = no_age_gfn, + .vm_test_age_gfn = no_age_gfn, + .vm_stage2_unmap_range = no_stage2_unmap_range, + .vm_mem_abort = protected_pkvm_mem_abort, +}; + +static const struct kvm_vm_s2_ops pkvm_vm_s2_ops = { + .vm_flush_remote_tlbs = pkvm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = pkvm_flush_remote_tlbs_range, + .vm_age_gfn = pkvm_age_gfn, + .vm_test_age_gfn = pkvm_test_age_gfn, + .vm_stage2_unmap_range = pkvm_stage2_unmap_range, + .vm_mem_abort = kvm_vm_mem_abort, +}; + +static const struct kvm_vm_s2_ops kvm_default_vm_s2_ops = { + .vm_flush_remote_tlbs = kvm_vm_flush_remote_tlbs, + .vm_flush_remote_tlbs_range = kvm_vm_flush_remote_tlbs_range, + .vm_age_gfn = kvm_vm_age_gfn, + .vm_test_age_gfn = kvm_vm_test_age_gfn, + .vm_stage2_unmap_range = kvm_vm_stage2_unmap_range, + .vm_mem_abort = kvm_vm_mem_abort, +}; + +#define KVM_VM_S2_OPS(flavor, ops) \ + [(flavor)] = &(ops) + +static const struct kvm_vm_s2_ops *arm64_vm_s2_ops[] = { + KVM_VM_S2_OPS(VM_VHE, kvm_default_vm_s2_ops), + KVM_VM_S2_OPS(VM_NVHE, kvm_default_vm_s2_ops), + KVM_VM_S2_OPS(VM_PKVM, pkvm_vm_s2_ops), + KVM_VM_S2_OPS(VM_PROTECTED_PKVM, protected_pkvm_vm_s2_ops), +}; + +static int kvm_vm_init_vm_s2_ops(struct kvm *kvm) +{ + BUILD_BUG_ON(ARRAY_SIZE(arm64_vm_s2_ops) != VM_FLAVOR_MAX); + + kvm->arch.vm_s2_ops = arm64_vm_s2_ops[kvm->arch.vm_flavor]; + if (WARN_ON(!kvm->arch.vm_s2_ops)) + return -EINVAL; + return 0; +} diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index b191365d97cc..f1c414f1c3fa 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -5,6 +5,7 @@ */ #include +#include #include #include @@ -44,14 +45,16 @@ struct vncr_tlb { * will invalidate them more often). */ #define S2_MMU_PER_VCPU 2 +#define S2_MMU_PER_VM (KVM_MAX_VCPUS * S2_MMU_PER_VCPU) int kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *, - KVM_MAX_VCPUS * S2_MMU_PER_VCPU, + S2_MMU_PER_VM, GFP_KERNEL_ACCOUNT); kvm->arch.nested_mmus_size = 0; atomic_set(&kvm->arch.vncr_tlb_count, 0); + spin_lock_init(&kvm->arch.guest_s2_tracking_lock); return kvm->arch.nested_mmus ? 0 : -ENOMEM; } @@ -124,10 +127,15 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu) return ret; } + for (i = 0; i < S2_MMU_PER_VCPU; i++) + kvm_nested_s2_ptdump_create_debugfs(&tmp[i], i + kvm->arch.nested_mmus_size); + guard(write_lock)(&kvm->mmu_lock); - for (i = 0; i < S2_MMU_PER_VCPU; i++) + for (i = 0; i < S2_MMU_PER_VCPU; i++) { + tmp[i].s2_mmu_idx = i + kvm->arch.nested_mmus_size; kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i]; + } kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU; } @@ -657,7 +665,7 @@ static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr) return 0; tmp &= ~(sz - 1); - if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL)) + if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL, 0)) goto again; if (!(pte & PTE_VALID)) goto again; @@ -837,10 +845,8 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size; /* Make sure we don't forget to do the laundry */ - if (kvm_s2_mmu_valid(s2_mmu)) { - kvm_nested_s2_ptdump_remove_debugfs(s2_mmu); + if (kvm_s2_mmu_valid(s2_mmu)) s2_mmu->pending_unmap = true; - } /* * The virtual VMID (modulo CnP) will be used as a key when matching @@ -854,8 +860,6 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2); s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM; - kvm_nested_s2_ptdump_create_debugfs(s2_mmu); - out: atomic_inc(&s2_mmu->refcnt); @@ -871,6 +875,87 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu) return s2_mmu; } +#define S2_MMU_IDX_MASK GENMASK_ULL(11, 0) + +static void tag_s2_mapping_mmu(struct kvm_guest_s2_mapping *mapping, + struct kvm_s2_mmu *mmu) +{ + BUILD_BUG_ON(S2_MMU_PER_VM > SZ_4K); + mapping->nested.start &= ~S2_MMU_IDX_MASK; + mapping->nested.start |= mmu->s2_mmu_idx; +} + +static struct kvm_s2_mmu *s2_mapping_to_mmu(struct kvm *kvm, + struct kvm_guest_s2_mapping *mapping) +{ + return kvm->arch.nested_mmus[mapping->nested.start & S2_MMU_IDX_MASK]; +} + +static unsigned long s2_mapping_to_nested_start(struct kvm_guest_s2_mapping *mapping) +{ + return mapping->nested.start & ~S2_MMU_IDX_MASK; +} + +void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa, + gpa_t nested_ipa, size_t map_size, + struct kvm_guest_s2_mapping *mapping) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + lockdep_assert_held_read(&kvm->mmu_lock); + + canonical_ipa = ALIGN_DOWN(canonical_ipa, map_size); + nested_ipa = ALIGN_DOWN(nested_ipa, map_size); + + mapping->canonical.start = canonical_ipa; + mapping->canonical.last = canonical_ipa + map_size - 1; + + mapping->nested.start = nested_ipa; + mapping->nested.last = nested_ipa + map_size - 1; + + tag_s2_mapping_mmu(mapping, mmu); + + guard(spinlock)(&kvm->arch.guest_s2_tracking_lock); + interval_tree_insert(&mapping->nested, &mmu->guest_s2_mappings); + interval_tree_insert(&mapping->canonical, &kvm->arch.mmu.guest_s2_mappings); +} + +void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa, + size_t size) +{ + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + struct interval_tree_node *node, *next; + struct kvm_guest_s2_mapping *mapping; + gpa_t nipa_end = nipa + size - 1; + + /* + * See kvm_nested_unmap_cipa_range() for why guest_s2_tracking_lock + * isn't taken here. + */ + lockdep_assert_held_write(&kvm->mmu_lock); + + node = interval_tree_iter_first(&mmu->guest_s2_mappings, nipa, nipa_end); + while (node) { + unsigned long nested_start; + + next = interval_tree_iter_next(node, nipa, nipa_end); + mapping = container_of(node, struct kvm_guest_s2_mapping, + nested); + /* + * Tracking must be conservative on removal, only remove + * mappings that are within the unmap range. + */ + nested_start = s2_mapping_to_nested_start(mapping); + if (nipa <= nested_start && nipa_end >= mapping->nested.last) { + interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); + interval_tree_remove(&mapping->canonical, + &kvm->arch.mmu.guest_s2_mappings); + kfree(mapping); + } + node = next; + } +} + void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu) { /* CnP being set denotes an invalid entry */ @@ -1293,6 +1378,56 @@ void kvm_nested_s2_wp(struct kvm *kvm) kvm_invalidate_vncr_ipa_all(kvm); } +void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, size_t unmap_size, + bool may_block) +{ + gpa_t cipa_end = cipa + unmap_size - 1; + struct kvm_guest_s2_mapping *mapping; + struct interval_tree_node *node; + size_t mapping_size; + + /* + * Guest s2 tracking interval trees are only accessed while holding the + * mmu_lock, hence we don't have to take guest_s2_tracking_lock if the + * mmu_lock is held for write. This saves us from having to manually + * lock/unlock guest_s2_tracking_lock below around + * cond_resched_rwlock_write(). + */ + lockdep_assert_held_write(&kvm->mmu_lock); + + if (!kvm->arch.nested_mmus_size) + return; + + while ((node = interval_tree_iter_first(&kvm->arch.mmu.guest_s2_mappings, + cipa, cipa_end))) { + unsigned long nested_start; + struct kvm_s2_mmu *mmu; + + mapping = container_of(node, struct kvm_guest_s2_mapping, + canonical); + nested_start = s2_mapping_to_nested_start(mapping); + mmu = s2_mapping_to_mmu(kvm, mapping); + + /* We could race against MMU teardown, which frees mmu->pgt. */ + if (mmu->pgt) { + mapping_size = mapping->nested.last - nested_start + 1; + + if (WARN_ON_ONCE(kvm_pgtable_stage2_unmap(mmu->pgt, nested_start, + mapping_size))) + return; + + interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings); + } + interval_tree_remove(node, &kvm->arch.mmu.guest_s2_mappings); + kfree(mapping); + + if (may_block) + cond_resched_rwlock_write(&kvm->mmu_lock); + } + + kvm_invalidate_vncr_ipa(kvm, cipa, cipa + unmap_size); +} + void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block) { int i; @@ -1424,7 +1559,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) bool write_fault, writable; unsigned long mmu_seq; struct vncr_tlb *vt; - struct page *page; + struct page *page = NULL; u64 va, pfn, gfn; int ret; @@ -1480,7 +1615,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) return -EFAULT; } } else { - ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, &page, NULL); + ret = kvm_gmem_get_pfn(vcpu->kvm, memslot, gfn, &pfn, NULL); if (ret) { kvm_prepare_memory_fault_exit(vcpu, vt->wr.pa, PAGE_SIZE, write_fault, false, false); diff --git a/arch/arm64/kvm/pauth.c b/arch/arm64/kvm/pauth.c index d5eb3ae876be..bb6c42e78058 100644 --- a/arch/arm64/kvm/pauth.c +++ b/arch/arm64/kvm/pauth.c @@ -5,7 +5,7 @@ * * Primitive PAuth emulation for ERETAA/ERETAB. * - * This code assumes that is is run from EL2, and that it is part of + * This code assumes that it is run from EL2, and that it is part of * the emulation of ERETAx for a guest hypervisor. That's a lot of * baked-in assumptions and shortcuts. * diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 8e4c6e4bec12..380dfb1db0be 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -16,6 +16,9 @@ #include "hyp_constants.h" +#define CREATE_TRACE_POINTS +#include "trace_pkvm.h" + DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory); @@ -85,6 +88,77 @@ void __init kvm_hyp_reserve(void) hyp_mem_base); } +static int pkvm_hyp_topup(enum pkvm_topup_id id, unsigned long nr_pages) +{ + struct kvm_hyp_memcache mc; + struct arm_smccc_res res; + int ret; + + init_hyp_memcache(&mc); + ret = topup_hyp_memcache(&mc, nr_pages); + if (ret) + goto err; + + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_topup), id, mc.head, + mc.nr_pages, &res); + if (WARN_ON_ONCE(res.a0 != SMCCC_RET_SUCCESS)) { + ret = -EINVAL; + goto err; + } + + ret = res.a1; + mc.head = res.a2; + mc.nr_pages = res.a3; + +err: + free_hyp_memcache(&mc); + return ret; +} + +static unsigned long __pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target) +{ + struct kvm_hyp_memcache mc; + struct arm_smccc_res res; + unsigned long reclaimed; + + arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_reclaim), id, target, &res); + if (WARN_ON_ONCE(res.a0 != SMCCC_RET_SUCCESS) || WARN_ON_ONCE(res.a1)) + return 0; + + init_hyp_memcache(&mc); + mc.head = res.a2; + mc.nr_pages = reclaimed = res.a3; + free_hyp_memcache(&mc); + + return reclaimed; +} + +static unsigned long pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target) +{ + unsigned long reclaimed = 0; + + while (reclaimed < target) { + /* Arbitrary limit to avoid blocking in EL2 for too long */ + unsigned long r = __pkvm_hyp_reclaim(id, min(target - reclaimed, 16)); + + if (!r) + break; + + reclaimed += r; + if (reclaimed >= target) + break; + + cond_resched(); + } + + return reclaimed; +} + +static unsigned long pkvm_hyp_reclaimable(enum pkvm_topup_id id) +{ + return kvm_call_hyp_nvhe(__pkvm_hyp_reclaimable, id); +} + static void __pkvm_destroy_hyp_vm(struct kvm *kvm) { if (pkvm_hyp_vm_is_created(kvm)) { @@ -100,30 +174,34 @@ static void __pkvm_destroy_hyp_vm(struct kvm *kvm) kvm->arch.pkvm.handle = 0; kvm->arch.pkvm.is_created = false; - free_hyp_memcache(&kvm->arch.pkvm.teardown_mc); free_hyp_memcache(&kvm->arch.pkvm.stage2_teardown_mc); } static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) { - size_t hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE); pkvm_handle_t handle = vcpu->kvm->arch.pkvm.handle; - void *hyp_vcpu; int ret; - vcpu->arch.pkvm_memcache.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2; + init_hyp_stage2_memcache(&vcpu->arch.stage2_mc); - hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT); - if (!hyp_vcpu) - return -ENOMEM; + ret = pkvm_call_hyp_req(__pkvm_init_vcpu, handle, vcpu); + if (ret) + return ret; - ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, vcpu, hyp_vcpu); - if (!ret) - vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); - else - free_pages_exact(hyp_vcpu, hyp_vcpu_sz); + /* + * Mirror EL2's seeding of power_state from mp_state. The hyp vCPU is + * published, so take mp_state_lock against kvm_psci_vcpu_on(). + */ + if (vcpu_is_protected(vcpu)) { + spin_lock(&vcpu->arch.mp_state_lock); + if (kvm_arm_vcpu_stopped(vcpu)) + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_UNINITIALIZED); + spin_unlock(&vcpu->arch.mp_state_lock); + } - return ret; + vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED); + + return 0; } /* @@ -138,8 +216,8 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu) */ static int __pkvm_create_hyp_vm(struct kvm *kvm) { - size_t pgd_sz, hyp_vm_sz; - void *pgd, *hyp_vm; + size_t pgd_sz; + void *pgd; int ret; if (kvm->created_vcpus < 1) @@ -156,28 +234,15 @@ static int __pkvm_create_hyp_vm(struct kvm *kvm) if (!pgd) return -ENOMEM; - /* Allocate memory to donate to hyp for vm and vcpu pointers. */ - hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE, - size_mul(sizeof(void *), - kvm->created_vcpus))); - hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT); - if (!hyp_vm) { - ret = -ENOMEM; - goto free_pgd; - } - - /* Donate the VM memory to hyp and let hyp initialize it. */ - ret = kvm_call_hyp_nvhe(__pkvm_init_vm, kvm, hyp_vm, pgd); + ret = pkvm_call_hyp_req(__pkvm_init_vm, kvm, pgd); if (ret) - goto free_vm; + goto free_pgd; kvm->arch.pkvm.is_created = true; - kvm->arch.pkvm.stage2_teardown_mc.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2; + init_hyp_stage2_memcache(&kvm->arch.pkvm.stage2_teardown_mc); kvm_account_pgtable_pages(pgd, pgd_sz / PAGE_SIZE); return 0; -free_vm: - free_pages_exact(hyp_vm, hyp_vm_sz); free_pgd: free_pages_exact(pgd, pgd_sz); return ret; @@ -229,10 +294,9 @@ void pkvm_destroy_hyp_vm(struct kvm *kvm) mutex_unlock(&kvm->arch.config_lock); } -int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) +int pkvm_init_host_vm(struct kvm *kvm) { int ret; - bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; /* Reserve the VM in hyp and obtain a hyp handle for the VM. */ ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm); @@ -240,8 +304,7 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) return ret; kvm->arch.pkvm.handle = ret; - kvm->arch.pkvm.is_protected = protected; - if (protected) { + if (kvm_vm_is_protected(kvm)) { pr_warn_once("kvm: protected VMs are experimental and for development only, tainting kernel\n"); add_taint(TAINT_USER, LOCKDEP_STILL_OK); } @@ -270,8 +333,55 @@ static int __init pkvm_drop_host_privileges(void) return ret; } +void __init pkvm_selftests(void) +{ +#ifdef CONFIG_NVHE_EL2_DEBUG + int ret = pkvm_call_hyp_req(__pkvm_hyp_alloc_selftest); + unsigned long reclaimed; + + reclaimed = pkvm_hyp_reclaim(PKVM_TOPUP_HYP_ALLOC_SELFTEST, ULONG_MAX); + + /* On failure, not all the pages may be reclaimable */ + if (!ret) + WARN_ON(reclaimed != 6 /* SELFTEST_MAX_PAGES */); + else + kvm_err("pKVM hyp allocator selftest failed (%d)\n", ret); +#endif +} + +static unsigned long pkvm_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long reclaimable = 0; + int id; + + for (id = 0; id < NR_PKVM_TOPUP_HYP_IDS; id++) + reclaimable += pkvm_hyp_reclaimable(id); + + return reclaimable ?: SHRINK_EMPTY; +} + +static unsigned long pkvm_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long reclaimed = 0; + int id; + + sc->nr_scanned = 0; + + for (id = 0; id < NR_PKVM_TOPUP_HYP_IDS; id++) { + unsigned long r = pkvm_hyp_reclaim(id, sc->nr_to_scan - sc->nr_scanned); + + reclaimed += r; + sc->nr_scanned += r; + if (sc->nr_scanned >= sc->nr_to_scan) + break; + } + + return reclaimed ?: SHRINK_STOP; +} + static int __init finalize_pkvm(void) { + struct shrinker *pkvm_shrinker; int ret; if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised()) @@ -287,10 +397,21 @@ static int __init finalize_pkvm(void) kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size); ret = pkvm_drop_host_privileges(); - if (ret) + if (ret) { pr_err("Failed to finalize Hyp protection: %d\n", ret); + return ret; + } - return ret; + pkvm_shrinker = shrinker_alloc(0, "pkvm"); + if (pkvm_shrinker) { + pkvm_shrinker->count_objects = pkvm_shrinker_count; + pkvm_shrinker->scan_objects = pkvm_shrinker_scan; + shrinker_register(pkvm_shrinker); + } else { + kvm_err("Failed to register shrinker for pKVM\n"); + } + + return 0; } device_initcall_sync(finalize_pkvm); @@ -414,6 +535,9 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 end = addr + size; int ret; + if (WARN_ON_ONCE(!PAGE_ALIGNED(addr | size))) + return -EINVAL; + lockdep_assert_held_write(&kvm->mmu_lock); mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1); @@ -594,3 +718,31 @@ bool pkvm_force_reclaim_guest_page(phys_addr_t phys) return !ret || ret == -EAGAIN; } + +static int pkvm_handle_hyp_req(struct pkvm_hyp_req *req) +{ + int ret = -EINVAL; + + switch (req->type) { + case PKVM_HYP_REQ_HYP_ALLOC: + ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC, req->mem.nr_pages); + break; + case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST: + ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC_SELFTEST, req->mem.nr_pages); + break; + } + + trace_kvm_handle_pkvm_hyp_req(req, ret); + + return ret; +} + +int __pkvm_handle_smccc_req(struct arm_smccc_res *res) +{ + struct pkvm_hyp_req req; + + if (smccc_to_pkvm_hyp_req(&req, res)) + return pkvm_handle_hyp_req(&req); + + return res->a1; +} diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 5b1af7e2176f..e2b28dbc0451 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -50,8 +50,11 @@ static u32 __kvm_pmu_event_mask(unsigned int pmuver) case ID_AA64DFR0_EL1_PMUVer_V3P5: case ID_AA64DFR0_EL1_PMUVer_V3P7: return GENMASK(15, 0); - default: /* Shouldn't be here, just for sanity */ + default: + /* Shouldn't be here, just for sanity */ WARN_ONCE(1, "Unknown PMU version %d\n", pmuver); + fallthrough; + case ID_AA64DFR0_EL1_PMUVer_NI: return 0; } } diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index e3db84400d1f..b01f8d44033f 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -63,7 +63,12 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) return PSCI_RET_INVALID_PARAMS; spin_lock(&vcpu->arch.mp_state_lock); - if (!kvm_arm_vcpu_stopped(vcpu)) { + /* + * A protected vCPU is off once the host has acted on EL2's CPU_OFF. + * STOPPED before that is the VMM's pause. + */ + if (!kvm_arm_vcpu_stopped(vcpu) || + (vcpu_is_protected(vcpu) && !kvm_pkvm_vcpu_is_powered_off(vcpu))) { if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) ret = PSCI_RET_ALREADY_ON; else @@ -161,7 +166,9 @@ static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags) */ kvm_for_each_vcpu(i, tmp, vcpu->kvm) { spin_lock(&tmp->arch.mp_state_lock); - WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + /* Keep EL2's off record for the guest's next CPU_ON. */ + if (!kvm_pkvm_vcpu_is_powered_off(tmp)) + WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); spin_unlock(&tmp->arch.mp_state_lock); } kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP); diff --git a/arch/arm64/kvm/ptdump.c b/arch/arm64/kvm/ptdump.c index 69899797dbad..9f0aebacea5b 100644 --- a/arch/arm64/kvm/ptdump.c +++ b/arch/arm64/kvm/ptdump.c @@ -17,7 +17,7 @@ #define MARKERS_LEN 2 #define KVM_PGTABLE_MAX_LEVELS (KVM_PGTABLE_LAST_LEVEL + 1) -#define S2FNAMESZ sizeof("0x0123456789abcdef-0x0123456789abcdef-s2-disabled") +#define S2FNAMESZ sizeof("nested_mmu0000") struct kvm_ptdump_guest_state { struct kvm_s2_mmu *mmu; @@ -115,15 +115,28 @@ static int kvm_ptdump_build_levels(struct ptdump_pg_level *level, u32 start_lvl) static struct kvm_ptdump_guest_state *kvm_ptdump_parser_create(struct kvm_s2_mmu *mmu) { + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); struct kvm_ptdump_guest_state *st; - struct kvm_pgtable *pgtable = mmu->pgt; + s8 start_level; int ret; + /* + * We only need the pgt start level to initialize the ptdump, get it + * while holding the mmu_lock. It's fine if the pgt gets freed + * afterwards, we'll check again when doing the actual dump. + */ + scoped_guard(read_lock, &kvm->mmu_lock) { + if (mmu->pgt) + start_level = mmu->pgt->start_level; + else + return ERR_PTR(-ENOENT); + } + st = kzalloc_obj(struct kvm_ptdump_guest_state, GFP_KERNEL_ACCOUNT); if (!st) return ERR_PTR(-ENOMEM); - ret = kvm_ptdump_build_levels(&st->level[0], pgtable->start_level); + ret = kvm_ptdump_build_levels(&st->level[0], start_level); if (ret) { kfree(st); return ERR_PTR(ret); @@ -149,6 +162,9 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused) }; guard(write_lock)(&kvm->mmu_lock); + if (!mmu->pgt) + return 0; + st->parser_state = (struct ptdump_pg_state) { .marker = &st->ipa_marker[0], .end_address = BIT(mmu->pgt->ia_bits), @@ -157,6 +173,15 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused) .seq = m, }; + if (kvm_is_nested_s2_mmu(kvm, mmu)) { + if (kvm_s2_mmu_valid(mmu)) + seq_printf(m, "VTCR: 0x%016llx VTTBR: 0x%016llx s2: %s\n", + mmu->tlb_vtcr, mmu->tlb_vttbr, + mmu->nested_stage2_enabled ? "enabled" : "disabled"); + else + return 0; + } + ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker); if (ret) return ret; @@ -211,17 +236,27 @@ static const struct file_operations kvm_ptdump_guest_fops = { static int kvm_pgtable_range_show(struct seq_file *m, void *unused) { - struct kvm_pgtable *pgtable = m->private; + struct kvm_s2_mmu *mmu = m->private; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + guard(read_lock)(&kvm->mmu_lock); + + if (mmu->pgt) + seq_printf(m, "%2u\n", mmu->pgt->ia_bits); - seq_printf(m, "%2u\n", pgtable->ia_bits); return 0; } static int kvm_pgtable_levels_show(struct seq_file *m, void *unused) { - struct kvm_pgtable *pgtable = m->private; + struct kvm_s2_mmu *mmu = m->private; + struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); + + guard(read_lock)(&kvm->mmu_lock); + + if (mmu->pgt) + seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - mmu->pgt->start_level); - seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - pgtable->start_level); return 0; } @@ -230,15 +265,12 @@ static int kvm_pgtable_debugfs_open(struct inode *m, struct file *file, { struct kvm_s2_mmu *mmu = m->i_private; struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu); - struct kvm_pgtable *pgtable; int ret; if (!kvm_get_kvm_safe(kvm)) return -ENOENT; - pgtable = mmu->pgt; - - ret = single_open(file, show, pgtable); + ret = single_open(file, show, mmu); if (ret < 0) kvm_put_kvm(kvm); return ret; @@ -276,30 +308,23 @@ static const struct file_operations kvm_pgtable_levels_fops = { .release = kvm_pgtable_debugfs_close, }; -void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) +void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx) { - struct dentry *dent; char file_name[S2FNAMESZ]; - snprintf(file_name, sizeof(file_name), "0x%016llx-0x%016llx-s2-%sabled", - mmu->tlb_vttbr, - mmu->tlb_vtcr, - mmu->nested_stage2_enabled ? "en" : "dis"); + snprintf(file_name, sizeof(file_name), "nested_mmu%d", idx); - dent = debugfs_create_file(file_name, 0400, - mmu->arch->debugfs_nv_dentry, mmu, - &kvm_ptdump_guest_fops); - - mmu->shadow_pt_debugfs_dentry = dent; -} - -void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) -{ - debugfs_remove(mmu->shadow_pt_debugfs_dentry); + debugfs_create_file(file_name, 0400, + mmu->arch->debugfs_nv_dentry, mmu, + &kvm_ptdump_guest_fops); } void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) { + /* pKVM handles the page-tables in the hypervisor */ + if (is_protected_kvm_enabled()) + return; + debugfs_create_file("stage2_page_tables", 0400, kvm->debugfs_dentry, &kvm->arch.mmu, &kvm_ptdump_guest_fops); debugfs_create_file("ipa_range", 0400, kvm->debugfs_dentry, diff --git a/arch/arm64/kvm/pvtime.c b/arch/arm64/kvm/pvtime.c index 4ceabaa4c30b..053fe831fa54 100644 --- a/arch/arm64/kvm/pvtime.c +++ b/arch/arm64/kvm/pvtime.c @@ -67,9 +67,9 @@ gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu) return base; } -bool kvm_arm_pvtime_supported(void) +bool kvm_arm_pvtime_supported(struct kvm *kvm) { - return !!sched_info_on(); + return !!sched_info_on() && (!kvm || !kvm_vm_is_protected(kvm)); } int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, @@ -81,7 +81,7 @@ int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu, int ret = 0; int idx; - if (!kvm_arm_pvtime_supported() || + if (!kvm_arm_pvtime_supported(kvm) || attr->attr != KVM_ARM_VCPU_PVTIME_IPA) return -ENXIO; @@ -110,7 +110,7 @@ int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu, u64 __user *user = (u64 __user *)attr->addr; u64 ipa; - if (!kvm_arm_pvtime_supported() || + if (!kvm_arm_pvtime_supported(vcpu->kvm) || attr->attr != KVM_ARM_VCPU_PVTIME_IPA) return -ENXIO; @@ -126,7 +126,7 @@ int kvm_arm_pvtime_has_attr(struct kvm_vcpu *vcpu, { switch (attr->attr) { case KVM_ARM_VCPU_PVTIME_IPA: - if (kvm_arm_pvtime_supported()) + if (kvm_arm_pvtime_supported(vcpu->kvm)) return 0; } return -ENXIO; diff --git a/arch/arm64/kvm/stacktrace.c b/arch/arm64/kvm/stacktrace.c index 9724c320126b..69377195e18b 100644 --- a/arch/arm64/kvm/stacktrace.c +++ b/arch/arm64/kvm/stacktrace.c @@ -198,9 +198,6 @@ static void hyp_dump_backtrace(unsigned long hyp_offset) } #ifdef CONFIG_PKVM_STACKTRACE -DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], - pkvm_stacktrace); - /* * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace. * diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5e6b04c5567d..edfc3be1d4b3 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1029,7 +1029,7 @@ static unsigned int hidden_visibility(const struct kvm_vcpu *vcpu, static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - if (kvm_vcpu_has_pmu(vcpu)) + if (kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) return 0; return REG_HIDDEN; @@ -1059,10 +1059,6 @@ static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - /* This thing will UNDEF, who cares about the reset value? */ - if (!kvm_vcpu_has_pmu(vcpu)) - return 0; - reset_unknown(vcpu, r); __vcpu_rmw_sys_reg(vcpu, r->reg, &=, kvm_pmu_evtyper_mask(vcpu->kvm)); @@ -2514,9 +2510,10 @@ static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, /* * Once the VM has started the ID registers are immutable. Reject any - * write that does not match the final register value. + * write that does not match the final register value once we have + * got far enough into first running the VM to use the values. */ - if (kvm_vm_has_ran_once(vcpu->kvm)) { + if (kvm_id_regs_final(vcpu->kvm)) { if (val != read_id_reg(vcpu, rd)) ret = -EBUSY; else @@ -2550,7 +2547,7 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val) lockdep_assert_held(&kvm->arch.config_lock); - if (KVM_BUG_ON(kvm_vm_has_ran_once(kvm) || !p, kvm)) + if (KVM_BUG_ON(kvm_id_regs_final(kvm) || !p, kvm)) return; *p = val; @@ -3246,10 +3243,10 @@ static int set_imp_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, return -EINVAL; /* - * Once the VM has started the ID registers are immutable. Reject the - * write if userspace tries to change it. + * Once we have been far enough into starting the VM the ID registers + * are immutable. Reject the write if userspace tries to change it. */ - if (kvm_vm_has_ran_once(kvm)) + if (kvm_id_regs_final(kvm)) return -EBUSY; /* @@ -5864,26 +5861,17 @@ void kvm_calculate_traps(struct kvm_vcpu *vcpu) } /* - * Perform last adjustments to the ID registers that are implied by the + * Do system register finalization that is shared by the whole guest. This + * includes last adjustments to the ID registers that are implied by the * configuration outside of the ID regs themselves, as well as any * initialisation that directly depend on these ID registers (such as * RES0/RES1 behaviours). This is not the place to configure traps though. - * - * Because this can be called once per CPU, changes must be idempotent. */ -int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) +static int kvm_vm_finalize_sys_regs(struct kvm *kvm) { - struct kvm *kvm = vcpu->kvm; + lockdep_assert_held(&kvm->arch.config_lock); - guard(mutex)(&kvm->arch.config_lock); - - if (vcpu_has_nv(vcpu)) { - int ret = kvm_init_nv_sysregs(vcpu); - if (ret) - return ret; - } - - if (kvm_vm_has_ran_once(kvm)) + if (kvm_id_regs_final(kvm)) return 0; /* @@ -5931,12 +5919,37 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu) kvm_vgic_finalize_idregs(kvm); } + set_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &kvm->arch.flags); + + return 0; +} + +/* + * Because this can be called once per CPU, changes must be idempotent. + */ +int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + int ret; + + guard(mutex)(&kvm->arch.config_lock); + + ret = kvm_vm_finalize_sys_regs(kvm); + if (ret) + return ret; + + if (vcpu_has_nv(vcpu)) { + ret = kvm_init_nv_sysregs(vcpu); + if (ret) + return ret; + } + return 0; } int __init kvm_sys_reg_table_init(void) { - const struct sys_reg_desc *gicv3_regs; + const struct sys_reg_desc *gicv3_regs, *gicv5_regs; bool valid = true; unsigned int i, sz; int ret = 0; @@ -5949,8 +5962,12 @@ int __init kvm_sys_reg_table_init(void) valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), false); valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false); + /* The GICv3 system registers... */ gicv3_regs = vgic_v3_get_sysreg_table(&sz); valid &= check_sysreg_table(gicv3_regs, sz, false); + /* ...and the GICv5 system registers. */ + gicv5_regs = vgic_v5_get_sysreg_table(&sz); + valid &= check_sysreg_table(gicv5_regs, sz, false); if (!valid) return -EINVAL; diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index bd56a45abbf9..a3cccad2766f 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -254,7 +254,7 @@ int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg, bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index); -int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu); +int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu); #define AA32(_x) .aarch32_map = AA32_##_x #define Op0(_x) .Op0 = _x diff --git a/arch/arm64/kvm/trace_pkvm.h b/arch/arm64/kvm/trace_pkvm.h new file mode 100644 index 000000000000..493225e3282d --- /dev/null +++ b/arch/arm64/kvm/trace_pkvm.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(_TRACE_PKVM_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_PKVM_ARM64_KVM_H + +#include +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +TRACE_DEFINE_ENUM(PKVM_HYP_NO_REQ); +TRACE_DEFINE_ENUM(PKVM_HYP_REQ_HYP_ALLOC); + +#define PKVM_HYP_REQ_TYPES \ + { PKVM_HYP_NO_REQ, "NO_REQ" }, \ + { PKVM_HYP_REQ_HYP_ALLOC, "HYP_ALLOC" } + +TRACE_EVENT(kvm_handle_pkvm_hyp_req, + TP_PROTO(struct pkvm_hyp_req *req, int ret), + TP_ARGS(req, ret), + + TP_STRUCT__entry( + __field(u8, type) + __field(int, ret) + ), + + TP_fast_assign( + __entry->type = req->type; + __entry->ret = ret; + ), + + TP_printk("type: %s ret: %d", + __print_symbolic(__entry->type, PKVM_HYP_REQ_TYPES), + __entry->ret) +); + +#endif /* _TRACE_PKVM_ARM64_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace_pkvm + +/* This part must be outside protection */ +#include diff --git a/arch/arm64/kvm/vgic-sys-reg-v5.c b/arch/arm64/kvm/vgic-sys-reg-v5.c new file mode 100644 index 000000000000..6b255eacbf60 --- /dev/null +++ b/arch/arm64/kvm/vgic-sys-reg-v5.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +/* + * VGICv5 system registers handling functions for AArch64 mode + */ + +#include + +#include +#include +#include + +#include + +#include "vgic/vgic.h" +#include "sys_regs.h" + +#define ICC_PPI_PRIORITYR_PRIORITY_MASK REPEAT_BYTE(0x1f) + +static int set_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* The upper 32 bits are RES0 */ + cpu_if->vgic_apr = val & ~ICC_APR_EL1_RES0; + + return 0; +} + +static int get_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = cpu_if->vgic_apr; + + return 0; +} + +static int set_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* + * We only support setting the ICC_CR0_EL1.En bit, which is actually + * stored in the VMCR. + */ + FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_EN, &cpu_if->vgic_vmcr, + FIELD_GET(ICC_CR0_EL1_EN, val)); + + return 0; +} + +static int get_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* + * PID only applies if EL3 is present. Same applies to IPPT. Hence, + * those fields are always presented as 0. + * + * We always present the link as connected and idle: + * (LINK = 1, LINK_IDLE = 1). + */ + *val = FIELD_PREP(ICC_CR0_EL1_EN, + FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, cpu_if->vgic_vmcr)); + *val |= ICC_CR0_EL1_LINK_MASK; + *val |= ICC_CR0_EL1_LINK_IDLE_MASK; + + return 0; +} + +static int set_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + /* Set the VPMR field in the VMCR */ + FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_VPMR, &cpu_if->vgic_vmcr, + FIELD_GET(ICC_PCR_EL1_PRIORITY, val)); + + return 0; +} + +static int get_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = FIELD_PREP(ICC_PCR_EL1_PRIORITY, + FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, cpu_if->vgic_vmcr)); + + return 0; +} + +static int set_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + cpu_if->vgic_icsr = val & ~ICC_ICSR_EL1_RES0; + + return 0; +} + +static int get_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + + *val = cpu_if->vgic_icsr; + + return 0; +} + +/* + * Helper macro to iterate over a range of PPIs and execute some code (to either + * extract or set the vgic_irq state). This is used when `get`-ing the PPI + * ENABLER, ACTIVER, PENDR and when setting the PRIORITYR state. + * + * vcpu: Pointer to struct kvm_vcpu (to which these PPIs belong) + * r: The register index. 0 or 1 for all except PRIORITYR (which is 0-15) + * nr: The number of PPIs iterated over. 64 for all but PRIORITYR (which is 8) + * code: The code snippet to execute for each vgic_irq + */ +#define for_ppi_state(vcpu, r, nr, code) \ + do { \ + struct kvm_vcpu *__vcpu = (vcpu); \ + int __r = (r); \ + int __nr = (nr); \ + \ + for (int i = 0; i < __nr; i++) { \ + u32 id = vgic_v5_make_ppi(__r * __nr + i); \ + struct vgic_irq *irq; \ + \ + irq = vgic_get_vcpu_irq(__vcpu, id); \ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { \ + code; \ + } \ + vgic_put_irq(__vcpu->kvm, irq); \ + } \ + } while (0) + +static int set_gic_ppi_enabler(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int i, start, end, reg = r->Op2 % 2; + + /* + * If we're only handling architected PPIs and the guest writes to the + * enable for the non-architected PPIs, we just return as there's + * nothing to do at all. We don't even allocate the storage for them in + * this case. + */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Merge the raw guest write into our bitmap at an offset of either 0 or + * 64. + * + * Note that there is *NO* masking applied - the enable state is written + * unfiltered. The assumption is that userspace uses this interface to + * set initial state before the guest runs, and then the exposed PPI + * mask is applied later, when vgic_v5_finalize_ppi_state() runs on + * first entry to each vCPU. If userspace chooses to set the enabler + * state later, it is fully capable of breaking the illusion we provided + * to the guest by exposing register state (and PPIs) to the guest that + * were not initially exposed. Good luck! + */ + bitmap_write(cpu_if->vgic_ppi_enabler, val, 64 * reg, 64); + + /* + * Sync the change in enable states to the vgic_irqs for the written + * register slice. + */ + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) + irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler); + + vgic_put_irq(vcpu->kvm, irq); + } + + return 0; +} + +static int get_gic_ppi_enabler(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long enabler = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the ENABLER value. */ + for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &enabler, irq->enabled)); + + *val = enabler; + + return 0; +} + +static int set_gic_ppi_activer(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int i, start, end, reg = r->Op2 % 2; + + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Store the raw guest write. The exposed PPI mask is applied later, + * when vgic_v5_finalize_ppi_state() runs on first entry to each + * vCPU. See comment on set_gic_ppi_enabler() for details. + */ + bitmap_write(cpu_if->vgic_ppi_activer, val, 64 * reg, 64); + + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) + irq->active = test_bit(i, cpu_if->vgic_ppi_activer); + + vgic_put_irq(vcpu->kvm, irq); + } + + return 0; +} + +static int get_gic_ppi_activer(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long activer = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the ACTIVER value. */ + for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &activer, irq->active)); + + *val = activer; + + return 0; +} + +static int set_gic_ppi_pendr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + int i, start, end, reg = r->Op2 % 2; + + /* If we only support architected PPIs, return */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) + return 0; + + /* + * Update each struct vgic_irq with the pending state, treating Level + * and Edge interrupts differently. The exposed PPI mask is applied + * later, when vgic_v5_finalize_ppi_state() runs on first entry to each + * vCPU. See comment on set_gic_ppi_enabler() for details. + */ + start = reg * 64; + end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS); + for (i = start; i < end; i++) { + u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; + + irq = vgic_get_vcpu_irq(vcpu, intid); + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + bool level = !!(val & BIT_ULL(i - start)); + + if (irq->config == VGIC_CONFIG_LEVEL) + irq->line_level = level; + else + irq->pending_latch = level; + } + + vgic_put_irq(vcpu->kvm, irq); + } + + /* + * The pending state is generated from the vgic_irqs on each guest + * entry. Therefore, we don't store the raw value written anywhere in + * the case of userspace PPI_PENDRx_EL1 writes. + */ + + return 0; +} + +static int get_gic_ppi_pendr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + unsigned long pendr = 0; + int reg = r->Op2 % 2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) { + *val = 0; + return 0; + } + + /* Iterate over each struct vgic_irq to build the PENDR value. */ + for_ppi_state(vcpu, reg, 64, { + if (irq_is_pending(irq)) + __assign_bit(i % 64, &pendr, 1); + }); + + *val = pendr; + + return 0; +} + +static int set_gic_ppi_priorityr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int reg = ((r->CRm & 0x1) << 3) + r->Op2; + + /* If we only support architected PPIs, return */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7) + return 0; + + val &= ICC_PPI_PRIORITYR_PRIORITY_MASK; + + /* + * Although priorities are not regularly synced back to the vgic_irq + * state, they are explicitly synced back here. This is to ensure that + * any pending PPIs are evaluated correctly when first running the guest + * after setting the state. + */ + for_ppi_state(vcpu, reg, 8, + irq->priority = (u8)(val >> (8 * i)); + ); + + /* + * Update the state that will be written to the ICH_PPI_PRIORITYRx_EL2 + * on next guest entry. + */ + cpu_if->vgic_ppi_priorityr[reg] = val; + + return 0; +} + +static int get_gic_ppi_priorityr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *r, u64 *val) +{ + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + int reg = ((r->CRm & 0x1) << 3) + r->Op2; + + /* If we only support architected PPIs, return 0 */ + if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7) { + *val = 0; + return 0; + } + + /* + * The priorities are only synced back to the vgic_irq state when the + * vcpu is entering WFI (KVM only needs to know the priorities when + * evaluating if there are pending PPI interrupts for a vcpu). The raw + * register ICH_PPI_PRIORITYRx_EL1 state is simply saved and restored + * blindly. This state is just returned as it contains the most recent + * priorities written by the guest. + */ + *val = cpu_if->vgic_ppi_priorityr[reg]; + + return 0; +} + +/* + * The following registers are NOT supported: + * + * - ICC_HAPR_EL1 + * The value of this is directly generated by the GICv5 hardware based on + * the ICC_APR_EL1 when the guest is running. + * - ICC_IAFFIDR_EL1 + * The IAFFID for a GICv5 VPE is the same as the VPE ID, which is the index + * into the in-memory VPE Table. This is not configurable, and instead we + * rely on userspace recreating the VPEs in the same order prior to + * restoring guest state. + * - ICC_PPI_CACTIVER_EL1 + * Only raw state writes are supported via the S(et) variant. + * - ICC_PPI_CPENDR_EL1 + * Only raw state writes are supported via the S(et) variant. + */ +static const struct sys_reg_desc gic_v5_icc_reg_descs[] = { + { SYS_DESC(SYS_ICC_ICSR_EL1), + .set_user = set_gic_icsr, .get_user = get_gic_icsr, }, + { SYS_DESC(SYS_ICC_PPI_ENABLER0_EL1), + .set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, }, + { SYS_DESC(SYS_ICC_PPI_ENABLER1_EL1), + .set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, }, + /* + * Only ICC_SACTIVER_EL1 is exposed to the guest. This is treated as + * a *RAW* write of register state for writes. + */ + { SYS_DESC(SYS_ICC_PPI_SACTIVER0_EL1), + .set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, }, + { SYS_DESC(SYS_ICC_PPI_SACTIVER1_EL1), + .set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, }, + /* + * Only ICC_SPENDR_EL1 is exposed to the guest. This is treated as + * a *RAW* write of register state for writes. + */ + { SYS_DESC(SYS_ICC_PPI_SPENDR0_EL1), + .set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, }, + { SYS_DESC(SYS_ICC_PPI_SPENDR1_EL1), + .set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR0_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR1_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR2_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR3_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR4_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR5_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR6_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR7_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR8_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR9_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR10_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR11_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR12_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR13_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR14_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_PPI_PRIORITYR15_EL1), + .set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, }, + { SYS_DESC(SYS_ICC_APR_EL1), + .set_user = set_gic_apr, .get_user = get_gic_apr, }, + { SYS_DESC(SYS_ICC_CR0_EL1), + .set_user = set_gic_cr0, .get_user = get_gic_cr0, }, + { SYS_DESC(SYS_ICC_PCR_EL1), + .set_user = set_gic_pcr, .get_user = get_gic_pcr, }, +}; + +const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz) +{ + *sz = ARRAY_SIZE(gic_v5_icc_reg_descs); + return gic_v5_icc_reg_descs; +} + +static u64 attr_to_id(u64 attr) +{ + return ARM64_SYS_REG(FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP0_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP1_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRN_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRM_MASK, attr), + FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP2_MASK, attr)); +} + +int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) +{ + const struct sys_reg_desc *r; + + r = get_reg_by_id(attr_to_id(attr->attr), gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); + + if (r && !sysreg_hidden(vcpu, r)) + return 0; + + return -ENXIO; +} + +int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr, + bool is_write) +{ + struct kvm_one_reg reg = { + .id = attr_to_id(attr->attr), + .addr = attr->addr, + }; + + if (is_write) + return kvm_sys_reg_set_user(vcpu, ®, gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); + else + return kvm_sys_reg_get_user(vcpu, ®, gic_v5_icc_reg_descs, + ARRAY_SIZE(gic_v5_icc_reg_descs)); +} diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index a58575df36e9..979301c3dbd8 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -84,6 +84,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) !kvm_vgic_global_state.can_emulate_gicv2) return -ENODEV; + if (kvm_vm_hyp_is_distrusting(kvm) && type != KVM_DEV_TYPE_ARM_VGIC_V3) + return -ENODEV; /* * Ensure mutual exclusion with vCPU creation and any vCPU ioctls by: * @@ -119,19 +121,34 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) goto out_unlock; } - kvm_for_each_vcpu(i, vcpu, kvm) { - if (vcpu_has_run_once(vcpu)) - goto out_unlock; - } + if (kvm_id_regs_final(kvm)) + goto out_unlock; ret = 0; - if (type == KVM_DEV_TYPE_ARM_VGIC_V2) + switch (type) { + case KVM_DEV_TYPE_ARM_VGIC_V2: kvm->max_vcpus = VGIC_V2_MAX_CPUS; - else if (type == KVM_DEV_TYPE_ARM_VGIC_V3) + break; + case KVM_DEV_TYPE_ARM_VGIC_V3: kvm->max_vcpus = VGIC_V3_MAX_CPUS; - else if (type == KVM_DEV_TYPE_ARM_VGIC_V5) - kvm->max_vcpus = min(VGIC_V5_MAX_CPUS, - kvm_vgic_global_state.max_gic_vcpus); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + kvm->max_vcpus = kvm_vgic_global_state.max_gicv5_vcpus; + break; + } + + /* + * KVM_CREATE_VCPU enforces the model-specific limit if the VGIC has + * already been created. Apply the same limit to any existing vCPUs so + * that the result does not depend on the order in which userspace + * creates them. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vcpu->vcpu_id >= kvm->max_vcpus) { + ret = -E2BIG; + goto out_unlock; + } + } if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) { ret = -E2BIG; @@ -150,6 +167,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) case KVM_DEV_TYPE_ARM_VGIC_V3: INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions); break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL; } /* @@ -164,29 +183,49 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) break; } - if (ret) { - kvm_for_each_vcpu(i, vcpu, kvm) { - struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; - kfree(vgic_cpu->private_irqs); - vgic_cpu->private_irqs = NULL; - } - - kvm->arch.vgic.vgic_model = 0; - kvm->arch.vgic.in_kernel = false; - goto out_unlock; - } + if (ret) + goto out_free_private_irqs; if (type == KVM_DEV_TYPE_ARM_VGIC_V3) kvm->arch.vgic.nassgicap = system_supports_direct_sgis(); - /* - * We now know that we have a GICv5. The Arch Timer PPI interrupts may - * have been initialised at this stage, but will have done so assuming - * that we have an older GIC, meaning that the IntIDs won't be - * correct. We init them again, and this time they will be correct. - */ - if (type == KVM_DEV_TYPE_ARM_VGIC_V5) + if (type == KVM_DEV_TYPE_ARM_VGIC_V5) { + /* Allocate a vIRS for GICv5 systems */ + kvm->arch.vgic.vgic_v5_irs_data = kzalloc_obj(struct vgic_v5_irs, + GFP_KERNEL_ACCOUNT); + if (!kvm->arch.vgic.vgic_v5_irs_data) { + ret = -ENOMEM; + goto out_free_private_irqs; + } + + /* + * Initialization happens later, for now just explicitly + * disable the device and undef its base address. + */ + kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base = VGIC_ADDR_UNDEF; + + /* + * We now know that we have a GICv5. The Arch Timer PPI + * interrupts may have been initialised at this stage, but will + * have done so assuming that we have an older GIC, meaning that + * the IntIDs won't be correct. We init them again, and this + * time they will be correct. + */ kvm_timer_init_vm(kvm); + } + + goto out_unlock; + +out_free_private_irqs: + kvm_for_each_vcpu(i, vcpu, kvm) { + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + + kfree(vgic_cpu->private_irqs); + vgic_cpu->private_irqs = NULL; + } + + kvm->arch.vgic.in_kernel = false; + kvm->arch.vgic.vgic_model = 0; out_unlock: mutex_unlock(&kvm->arch.config_lock); @@ -393,15 +432,27 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu) if (ret) return ret; - /* - * If we are creating a VCPU with a GICv3 we must also register the - * KVM io device for the redistributor that belongs to this VCPU. - */ - if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V3: + /* + * If we are creating a VCPU with a GICv3 we must also register + * the KVM io device for the redistributor that belongs to this + * VCPU. + */ mutex_lock(&vcpu->kvm->slots_lock); ret = vgic_register_redist_iodev(vcpu); mutex_unlock(&vcpu->kvm->slots_lock); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + /* + * Ensure that it is possible to represent the + * userspace-allocated vcpu_id in the hardware-limited (or + * KVM-capped) VPE table used by GICv5. + */ + if (vcpu->vcpu_id >= kvm_vgic_global_state.max_gicv5_vcpus) + return -EINVAL; } + return ret; } @@ -461,6 +512,9 @@ int vgic_init(struct kvm *kvm) return ret; } } else { + if (!dist->nr_spis) + dist->nr_spis = VGIC_V5_DEFAULT_NR_SPIS; + ret = vgic_v5_init(kvm); if (ret) return ret; @@ -470,8 +524,12 @@ int vgic_init(struct kvm *kvm) kvm_vgic_vcpu_reset(vcpu); ret = kvm_vgic_setup_default_irq_routing(kvm); - if (ret) + if (ret) { + if (vgic_is_v5(kvm)) + vgic_v5_teardown(kvm); + return ret; + } vgic_debug_init(kvm); dist->initialized = true; @@ -492,16 +550,24 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm) dist->nr_spis = 0; dist->vgic_dist_base = VGIC_ADDR_UNDEF; - if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) { + switch (dist->vgic_model) { + case KVM_DEV_TYPE_ARM_VGIC_V2: + dist->vgic_cpu_base = VGIC_ADDR_UNDEF; + break; + case KVM_DEV_TYPE_ARM_VGIC_V3: list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list) vgic_v3_free_redist_region(kvm, rdreg); INIT_LIST_HEAD(&dist->rd_regions); - } else { - dist->vgic_cpu_base = VGIC_ADDR_UNDEF; - } - if (vgic_supports_direct_irqs(kvm)) - vgic_v4_teardown(kvm); + if (vgic_supports_direct_irqs(kvm)) + vgic_v4_teardown(kvm); + break; + case KVM_DEV_TYPE_ARM_VGIC_V5: + vgic_v5_teardown(kvm); + kfree(dist->vgic_v5_irs_data); + dist->vgic_v5_irs_data = NULL; + break; + } xa_destroy(&dist->lpi_xa); } @@ -620,9 +686,8 @@ int vgic_lazy_init(struct kvm *kvm) int kvm_vgic_map_resources(struct kvm *kvm) { struct vgic_dist *dist = &kvm->arch.vgic; - bool needs_dist = true; enum vgic_type type; - gpa_t dist_base; + gpa_t dist_base, irs_base; int ret = 0; if (likely(smp_load_acquire(&dist->ready))) @@ -645,13 +710,12 @@ int kvm_vgic_map_resources(struct kvm *kvm) } else { ret = vgic_v5_map_resources(kvm); type = VGIC_V5; - needs_dist = false; } if (ret) goto out; - if (needs_dist) { + if (type != VGIC_V5) { dist_base = dist->vgic_dist_base; mutex_unlock(&kvm->arch.config_lock); @@ -661,7 +725,19 @@ int kvm_vgic_map_resources(struct kvm *kvm) goto out_slots; } } else { + irs_base = dist->vgic_v5_irs_data->vgic_v5_irs_base; mutex_unlock(&kvm->arch.config_lock); + + if (IS_VGIC_ADDR_UNDEF(irs_base)) { + ret = -ENXIO; + goto out_slots; + } + + ret = vgic_v5_register_irs_iodev(kvm, irs_base); + if (ret) { + kvm_err("Unable to register VGIC IRS MMIO regions\n"); + goto out_slots; + } } smp_store_release(&dist->ready, true); diff --git a/arch/arm64/kvm/vgic/vgic-irqfd.c b/arch/arm64/kvm/vgic/vgic-irqfd.c index 19a1094536e6..ee1a1b214f03 100644 --- a/arch/arm64/kvm/vgic/vgic-irqfd.c +++ b/arch/arm64/kvm/vgic/vgic-irqfd.c @@ -19,9 +19,14 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) { - unsigned int spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS; + unsigned int spi_id; int ret; + if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5) + spi_id = vgic_v5_make_spi(e->irqchip.pin); + else + spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS; + if (!vgic_valid_spi(kvm, spi_id)) return -EINVAL; @@ -45,6 +50,13 @@ int kvm_set_routing_entry(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) { + /* + * The common routing table reserves enough entries for all GICv5 SPI + * pins. KVM limits GICv5 to 1024 SPI pins, while GICv2 and GICv3 + * only support SPIs 32..1019, leaving 988 pins. + */ + unsigned int max_irqchip_pins = vgic_is_v5(kvm) ? + VGIC_V5_MAX_NR_SPIS : VGIC_MAX_SPI - VGIC_NR_PRIVATE_IRQS + 1; int r = -EINVAL; switch (ue->type) { @@ -52,8 +64,8 @@ int kvm_set_routing_entry(struct kvm *kvm, e->set = vgic_irqfd_set_irq; e->irqchip.irqchip = ue->u.irqchip.irqchip; e->irqchip.pin = ue->u.irqchip.pin; - if ((e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) || - (e->irqchip.irqchip >= KVM_NR_IRQCHIPS)) + if (e->irqchip.pin >= max_irqchip_pins || + e->irqchip.irqchip >= KVM_NR_IRQCHIPS) goto out; break; case KVM_IRQ_ROUTING_MSI: diff --git a/arch/arm64/kvm/vgic/vgic-irs-v5.c b/arch/arm64/kvm/vgic/vgic-irs-v5.c new file mode 100644 index 000000000000..1874c3c289a6 --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-irs-v5.c @@ -0,0 +1,1217 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 ARM Limited, All Rights Reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vgic.h" +#include "vgic-mmio.h" +#include "vgic-v5-tables.h" + +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static struct vgic_dist *vgic_v5_get_vgic(struct kvm_vcpu *vcpu) +{ + return &vcpu->kvm->arch.vgic; +} + +static struct vgic_v5_irs *vgic_v5_get_irs(struct kvm_vcpu *vcpu) +{ + return vcpu->kvm->arch.vgic.vgic_v5_irs_data; +} + +static unsigned long vgic_v5_mmio_read_irs_misc(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct kvm_vcpu *target_vcpu; + u8 vpe_id_bits; + u64 value = 0; + + switch (offset) { + case GICV5_IRS_IDR0: + value = FIELD_PREP(GICV5_IRS_IDR0_INT_DOM, irs->idr0.domain); + value |= FIELD_PREP(GICV5_IRS_IDR0_PA_RANGE, irs->idr0.pa_range); + if (irs->idr0.virt) + value |= GICV5_IRS_IDR0_VIRT; + if (irs->idr0.setlpi) + value |= GICV5_IRS_IDR0_SETLPI; + if (irs->idr0.mec) + value |= GICV5_IRS_IDR0_MEC; + if (irs->idr0.mpam) + value |= GICV5_IRS_IDR0_MPAM; + if (irs->idr0.swe) + value |= GICV5_IRS_IDR0_SWE; + value |= FIELD_PREP(GICV5_IRS_IDR0_IRSID, irs->idr0.irs_id); + break; + case GICV5_IRS_IDR1: + value = FIELD_PREP(GICV5_IRS_IDR1_PE_CNT, + atomic_read(&vcpu->kvm->online_vcpus)); + /* + * IRS_IDR1 encodes IAFFID_BITS as N - 1. + */ + vpe_id_bits = vgic_v5_vmte_vpe_id_bits(vcpu); + value |= FIELD_PREP(GICV5_IRS_IDR1_IAFFID_BITS, vpe_id_bits - 1); + value |= FIELD_PREP(GICV5_IRS_IDR1_PRIORITY_BITS, irs->idr1.priority_bits); + break; + case GICV5_IRS_IDR2: + value = FIELD_PREP(GICV5_IRS_IDR2_ISTMD_SZ, irs->idr2.istmd_sz); + if (irs->idr2.istmd) + value |= GICV5_IRS_IDR2_ISTMD; + value |= FIELD_PREP(GICV5_IRS_IDR2_IST_L2SZ, irs->idr2.ist_l2sz); + if (irs->idr2.ist_levels) + value |= GICV5_IRS_IDR2_IST_LEVELS; + value |= FIELD_PREP(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, irs->idr2.min_lpi_id_bits); + value |= GICV5_IRS_IDR2_LPI; + value |= FIELD_PREP(GICV5_IRS_IDR2_ID_BITS, irs->idr2.id_bits); + break; + case GICV5_IRS_IDR5: + value = FIELD_PREP(GICV5_IRS_IDR5_SPI_RANGE, irs->idr5.spi_range); + break; + case GICV5_IRS_IDR6: + value = FIELD_PREP(GICV5_IRS_IDR6_SPI_IRS_RANGE, irs->idr6.spi_irs_range); + break; + case GICV5_IRS_IDR7: + value = FIELD_PREP(GICV5_IRS_IDR7_SPI_BASE, irs->idr7.spi_base); + break; + case GICV5_IRS_IIDR: + /* Revision, Variant, ProductID are implementation defined */ + value = FIELD_PREP(GICV5_IRS_IIDR_PRODUCT_ID, PRODUCT_ID_KVM); + value |= FIELD_PREP(GICV5_IRS_IIDR_VARIANT, 0); + value |= FIELD_PREP(GICV5_IRS_IIDR_REVISION, 0); + value |= FIELD_PREP(GICV5_IRS_IIDR_IMPLEMENTER, IMPLEMENTER_ARM); + break; + case GICV5_IRS_AIDR: + value = FIELD_PREP(GICV5_IRS_AIDR_COMPONENT, + GICV5_AIDR_COMPONENT_IRS); + value |= FIELD_PREP(GICV5_IRS_AIDR_ARCHMAJORREV, + GICV5_AIDR_ARCH_MAJ_REV_V5); + value |= FIELD_PREP(GICV5_IRS_AIDR_ARCHMINORREV, + GICV5_AIDR_ARCH_MIN_REV_V0); + break; + case GICV5_IRS_CR0: + /* + * The IRS is ALWAYS idle as we handle things instantaneously + * from a guest's viewpoint. + */ + value = GICV5_IRS_CR0_IDLE; + if (READ_ONCE(vcpu->kvm->arch.vgic.enabled)) + value |= GICV5_IRS_CR0_IRSEN; + break; + case GICV5_IRS_CR1: + if (irs->cr1.vped_wa) + value |= GICV5_IRS_CR1_VPED_WA; + if (irs->cr1.vped_ra) + value |= GICV5_IRS_CR1_VPED_RA; + if (irs->cr1.vmd_wa) + value |= GICV5_IRS_CR1_VMD_WA; + if (irs->cr1.vmd_ra) + value |= GICV5_IRS_CR1_VMD_RA; + if (irs->cr1.vpet_ra) + value |= GICV5_IRS_CR1_VPET_RA; + if (irs->cr1.vmt_ra) + value |= GICV5_IRS_CR1_VMT_RA; + if (irs->cr1.ist_wa) + value |= GICV5_IRS_CR1_IST_WA; + if (irs->cr1.ist_ra) + value |= GICV5_IRS_CR1_IST_RA; + value |= FIELD_PREP(GICV5_IRS_CR1_IC, irs->cr1.ic); + value |= FIELD_PREP(GICV5_IRS_CR1_OC, irs->cr1.oc); + value |= FIELD_PREP(GICV5_IRS_CR1_SH, irs->cr1.sh); + break; + case GICV5_IRS_SYNC_STATUSR: + value = GICV5_IRS_SYNC_STATUSR_IDLE; + break; + case GICV5_IRS_PE_SELR: + value = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, + READ_ONCE(irs->pe_selr.iaffid)); + break; + case GICV5_IRS_PE_STATUSR: + /* We assume that the PE is Online if present. Always IDLE too */ + value = GICV5_IRS_PE_STATUSR_IDLE; + + /* Set ONLINE and V if IAFFID selects a present PE */ + if (kvm_get_vcpu_by_id(vcpu->kvm, + READ_ONCE(irs->pe_selr.iaffid))) { + value |= GICV5_IRS_PE_STATUSR_ONLINE; + value |= GICV5_IRS_PE_STATUSR_V; + } + break; + case GICV5_IRS_PE_CR0: { + u32 iaffid = READ_ONCE(irs->pe_selr.iaffid); + + /* + * Make sure that we are doing something reasonable first. + * Remember, the IAFFID is the same as the VPE_ID + */ + target_vcpu = kvm_get_vcpu_by_id(vcpu->kvm, iaffid); + if (!target_vcpu) { + kvm_debug("Guest programmed invalid IAFFID (0x%x) into the IRS_PE_SELR\n", + iaffid); + break; + } + + value = GICV5_IRS_PE_CR0_DPS; + break; + } + default: + return 0; + } + + return value; +} + +static void vgic_v5_mmio_write_irs_misc(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + struct vgic_dist *vgic = vgic_v5_get_vgic(vcpu); + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_IRS_CR0: + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + bool was_enabled = vgic->enabled; + + WRITE_ONCE(vgic->enabled, !!(val & GICV5_IRS_CR0_IRSEN)); + + /* Reload other vCPUs on an edge */ + if (was_enabled != !!(val & GICV5_IRS_CR0_IRSEN)) + kvm_make_all_cpus_request(vcpu->kvm, + KVM_REQ_RELOAD_GICv5); + } + return; + case GICV5_IRS_CR1: + irs->cr1.sh = FIELD_GET(GICV5_IRS_CR1_SH, val); + irs->cr1.oc = FIELD_GET(GICV5_IRS_CR1_OC, val); + irs->cr1.ic = FIELD_GET(GICV5_IRS_CR1_IC, val); + irs->cr1.ist_ra = !!(val & GICV5_IRS_CR1_IST_RA); + irs->cr1.ist_wa = !!(val & GICV5_IRS_CR1_IST_WA); + irs->cr1.vmt_ra = !!(val & GICV5_IRS_CR1_VMT_RA); + irs->cr1.vpet_ra = !!(val & GICV5_IRS_CR1_VPET_RA); + irs->cr1.vmd_ra = !!(val & GICV5_IRS_CR1_VMD_RA); + irs->cr1.vmd_wa = !!(val & GICV5_IRS_CR1_VMD_WA); + irs->cr1.vped_ra = !!(val & GICV5_IRS_CR1_VPED_RA); + irs->cr1.vped_wa = !!(val & GICV5_IRS_CR1_VPED_WA); + return; + case GICV5_IRS_PE_SELR: + WRITE_ONCE(irs->pe_selr.iaffid, + FIELD_GET(GICV5_IRS_PE_SELR_IAFFID, val)); + return; + case GICV5_IRS_PE_CR0: + /* + * We actually have nothing to do here as we don't support + * 1-of-N routing. The only thing that the guest can correctly + * write here is 0x1. However, there's no way to fault if it + * writes something else. This is effectively a WI in our case, + * but we keep it here for the purposes of documenting it. + */ + return; + default: + return; + } +} + +static bool vgic_v5_is_spi_selr_valid(struct vgic_v5_irs *irs, u32 id) +{ + /* Invalid - we don't have any SPIs at all */ + if (irs->idr5.spi_range == 0) + return false; + + /* Invalid - we don't have any on this IRS */ + if (irs->idr6.spi_irs_range == 0) + return false; + + /* Invalid - ID is less than min */ + if (id < irs->idr7.spi_base) + return false; + + /* Invalid - ID is greater than max */ + if (id >= (irs->idr7.spi_base + irs->idr6.spi_irs_range)) + return false; + + return true; +} + +static unsigned long vgic_v5_mmio_read_irs_spi(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct vgic_irq *irq; + u64 value = 0; + + switch (offset) { + case GICV5_IRS_SPI_SELR: + /* Return whatever was last written */ + value = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, READ_ONCE(irs->spi_selr.id)); + break; + case GICV5_IRS_SPI_STATUSR: + /* We assume that we can always claim to be idle */ + value = GICV5_IRS_SPI_STATUSR_IDLE; + if (vgic_v5_is_spi_selr_valid(irs, READ_ONCE(irs->spi_selr.id))) + value |= GICV5_IRS_SPI_STATUSR_V; + break; + case GICV5_IRS_SPI_DOMAINR: + value = FIELD_PREP(GICV5_IRS_SPI_DOMAINR_DOMAIN, + GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE); + break; + case GICV5_IRS_SPI_CFGR: { + u32 id = READ_ONCE(irs->spi_selr.id); + + if (!vgic_v5_is_spi_selr_valid(irs, id)) { + /* Fault with IRS_SPI_SELR; return 0*/ + value = 0; + break; + } + + irq = vgic_get_irq(vcpu->kvm, vgic_v5_make_spi(id)); + if (!irq) { + kvm_debug_ratelimited("Guest trying to access SPI not backed by KVM\n"); + value = 0; + break; + } + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (irq->config == VGIC_CONFIG_LEVEL) + value = GICV5_IRS_SPI_CFGR_TM; + } + + vgic_put_irq(vcpu->kvm, irq); + + break; + } + default: + return 0; + } + + return value; +} + +static void vgic_v5_mmio_write_irs_spi(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + struct vgic_irq *irq; + + switch (offset) { + case GICV5_IRS_SPI_SELR: + WRITE_ONCE(irs->spi_selr.id, + FIELD_GET(GICV5_IRS_SPI_SELR_ID, val)); + return; + case GICV5_IRS_SPI_CFGR: { + u32 id = READ_ONCE(irs->spi_selr.id); + + if (!vgic_v5_is_spi_selr_valid(irs, id)) + return; + + /* + * Find KVM's representation of the interrupt - we need to make + * sure that KVM's view agrees with the guest's, else interrupt + * injection won't work properly for level-triggered interrupts + * (we fail to handle the clearing of the pending state if KVM + * thinks that the interrupt is edge-triggered, which is the + * default.) + */ + irq = vgic_get_irq(vcpu->kvm, vgic_v5_make_spi(id)); + if (!irq) + return; + + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (val & GICV5_IRS_SPI_CFGR_TM) + irq->config = VGIC_CONFIG_LEVEL; + else + irq->config = VGIC_CONFIG_EDGE; + } + + vgic_put_irq(vcpu->kvm, irq); + + return; + } + default: + return; + } +} + +static bool vgic_v5_ist_cfgr_valid(struct vgic_v5_irs *irs) +{ + unsigned int expected_istsz; + + if (irs->ist_cfgr.lpi_id_bits < irs->idr2.min_lpi_id_bits || + irs->ist_cfgr.lpi_id_bits > irs->idr2.id_bits) + return false; + + if (!irs->idr2.istmd) + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_4; + else if (irs->ist_cfgr.lpi_id_bits >= irs->idr2.istmd_sz) + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_16; + else + expected_istsz = GICV5_IRS_IST_CFGR_ISTSZ_8; + + if (irs->ist_cfgr.istsz != expected_istsz) + return false; + + if (irs->ist_cfgr.structure && !irs->idr2.ist_levels) + return false; + + if (!irs->ist_cfgr.structure) + return true; + + return irs->ist_cfgr.l2sz == irs->idr2.ist_l2sz; +} + +static u64 vgic_v5_read_irs_ist_baser(struct vgic_v5_irs *irs) +{ + u64 value; + + value = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + irs->ist_baser.addr >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + if (irs->ist_baser.valid) + value |= GICV5_IRS_IST_BASER_VALID; + + return value; +} + +static bool vgic_v5_ist_cfgr_matches(const struct vgic_v5_irs *irs, + unsigned long val) +{ + u8 lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + u8 l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + u8 istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + bool structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + + return irs->ist_cfgr.lpi_id_bits == lpi_id_bits && + irs->ist_cfgr.l2sz == l2sz && + irs->ist_cfgr.istsz == istsz && + irs->ist_cfgr.structure == structure; +} + +static unsigned long vgic_v5_mmio_read_irs_ist(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + u64 value = 0; + + switch (offset) { + case GICV5_IRS_IST_STATUSR: + return GICV5_IRS_IST_STATUSR_IDLE; + case GICV5_IRS_IST_CFGR: + if (irs->ist_cfgr.structure) + value |= GICV5_IRS_IST_CFGR_STRUCTURE; + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, irs->ist_cfgr.istsz); + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, irs->ist_cfgr.l2sz); + value |= FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, irs->ist_cfgr.lpi_id_bits); + break; + case GICV5_IRS_IST_BASER: + case GICV5_IRS_IST_BASER + 4: + value = vgic_v5_read_irs_ist_baser(irs); + return extract_bytes(value, offset & 7, len); + default: + return 0; + } + + return value; +} + +static bool vgic_v5_ist_baser_matches(const struct vgic_v5_irs *irs, + unsigned long val) +{ + u64 addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, val) + << GICV5_IRS_IST_BASER_ADDR_SHIFT; + bool valid = !!(val & GICV5_IRS_IST_BASER_VALID); + + return irs->ist_baser.addr == addr && irs->ist_baser.valid == valid; +} + +static void vgic_v5_update_irs_ist_baser(struct vgic_v5_irs *irs, + unsigned long val) +{ + irs->ist_baser.valid = !!(val & GICV5_IRS_IST_BASER_VALID); + irs->ist_baser.addr = FIELD_GET(GICV5_IRS_IST_BASER_ADDR_MASK, val) + << GICV5_IRS_IST_BASER_ADDR_SHIFT; +} + +static int vgic_v5_write_irs_ist_baser(struct kvm_vcpu *vcpu, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + enum gicv5_vcpu_cmd cmd = LPI_VIST_MAKE_INVALID; + bool valid = !!(val & GICV5_IRS_IST_BASER_VALID); + int rc; + + /* The address cannot be changed while the IST is valid. */ + if (irs->ist_baser.valid && valid) + return 0; + + /* Valid -> Invalid */ + if (irs->ist_baser.valid && !valid) { + /* Make the LPI IST invalid and then ... */ + rc = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu), &cmd); + if (rc) + return rc; + + vgic_v5_update_irs_ist_baser(irs, val); + + /* + * ... reflect that in the emulated BASER before freeing the + * host IST. If the free fails, the guest-visible valid bit + * still matches the hardware state. + */ + return vgic_v5_lpi_ist_free(vcpu->kvm); + } else if (!irs->ist_baser.valid && valid) { /* Invalid -> Valid */ + if (!vgic_v5_ist_cfgr_valid(irs)) + return -EINVAL; + + rc = vgic_v5_lpi_ist_alloc(vcpu->kvm, irs->ist_cfgr.lpi_id_bits); + if (rc) + return rc; + } + + /* Now that we've handled the edges, update the valid bit and addr */ + vgic_v5_update_irs_ist_baser(irs, val); + + return 0; +} + +static void vgic_v5_mmio_write_irs_ist(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_v5_irs *irs = vgic_v5_get_irs(vcpu); + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_IRS_IST_CFGR: + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + if (irs->ist_baser.valid) + return; + + irs->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + irs->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + irs->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + irs->ist_cfgr.structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + } + return; + case GICV5_IRS_IST_BASER: + case GICV5_IRS_IST_BASER + 4: { + scoped_guard(mutex, &vcpu->kvm->arch.config_lock) { + u64 baser; + + baser = update_64bit_reg(vgic_v5_read_irs_ist_baser(irs), + offset & 4, len, val); + vgic_v5_write_irs_ist_baser(vcpu, baser); + } + return; + } + default: + return; + } +} + +static unsigned long vgic_v5_coresight_read(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_CORESIGHT_DEVARCH: + return GICV5_CORESIGHT_DEVARCH_VAL; + case GICV5_CORESIGHT_PIDR4: + return GICV5_CORESIGHT_PIDR4_JEP106_CONT; + case GICV5_CORESIGHT_PIDR5: + return GICV5_CORESIGHT_PIDR5_RES0; + case GICV5_CORESIGHT_PIDR6: + return GICV5_CORESIGHT_PIDR6_RES0; + case GICV5_CORESIGHT_PIDR7: + return GICV5_CORESIGHT_PIDR7_RES0; + case GICV5_CORESIGHT_PIDR0: + return GICV5_CORESIGHT_PIDR0_PART_0; + case GICV5_CORESIGHT_PIDR1: + return GICV5_CORESIGHT_PIDR1_DES_0_PART_1; + case GICV5_CORESIGHT_PIDR2: + return GICV5_CORESIGHT_PIDR2_DES_1; + case GICV5_CORESIGHT_PIDR3: + return GICV5_CORESIGHT_PIDR3_REVAND_CMOD; + case GICV5_CORESIGHT_CIDR0: + return GICV5_CORESIGHT_CIDR0_VAL; + case GICV5_CORESIGHT_CIDR1: + return GICV5_CORESIGHT_CIDR1_VAL; + case GICV5_CORESIGHT_CIDR2: + return GICV5_CORESIGHT_CIDR2_VAL; + case GICV5_CORESIGHT_CIDR3: + return GICV5_CORESIGHT_CIDR3_VAL; + default: + return 0; + } +} + +static unsigned long vgic_v5_mmio_uaccess_read_irs_status(struct kvm_vcpu *vcpu, + gpa_t addr, + unsigned int len) +{ + const size_t offset = addr & (SZ_64K - 1); + + switch (offset) { + case GICV5_IRS_SYNC_STATUSR: + return GICV5_IRS_SYNC_STATUSR_IDLE; + case GICV5_IRS_SPI_STATUSR: + return GICV5_IRS_SPI_STATUSR_IDLE; + case GICV5_IRS_PE_STATUSR: + return GICV5_IRS_PE_STATUSR_IDLE; + case GICV5_IRS_IST_STATUSR: + return GICV5_IRS_IST_STATUSR_IDLE; + default: + return 0; + } +} + +static int vgic_v5_mmio_uaccess_write_irs(struct kvm_vcpu *vcpu, gpa_t addr, + unsigned int len, unsigned long val) +{ + struct vgic_dist *vgic = &vcpu->kvm->arch.vgic; + struct vgic_v5_irs *irs_data = vgic->vgic_v5_irs_data; + size_t offset = addr & (SZ_64K - 1); + int ret; + + /* + * The following registers are ONLY settable via uaccesses. The guest + * cannot write them! + */ + + switch (offset) { + case GICV5_IRS_IDR0: + if (FIELD_GET(GICV5_IRS_IDR0_INT_DOM, val) != + GICV5_IRS_IDR0_INT_DOM_NON_SECURE) + return -EINVAL; + + if ((val & GICV5_IRS_IDR0_VIRT) || + (val & GICV5_IRS_IDR0_ONE_N) || + (val & GICV5_IRS_IDR0_VIRT_ONE_N) || + (val & GICV5_IRS_IDR0_SETLPI) || + (val & GICV5_IRS_IDR0_MEC) || + (val & GICV5_IRS_IDR0_MPAM) || + (val & GICV5_IRS_IDR0_SWE)) + return -EINVAL; + + irs_data->idr0.domain = FIELD_GET(GICV5_IRS_IDR0_INT_DOM, val); + irs_data->idr0.pa_range = FIELD_GET(GICV5_IRS_IDR0_PA_RANGE, val); + irs_data->idr0.virt = !!(val & GICV5_IRS_IDR0_VIRT); + irs_data->idr0.setlpi = !!(val & GICV5_IRS_IDR0_SETLPI); + irs_data->idr0.mec = !!(val & GICV5_IRS_IDR0_MEC); + irs_data->idr0.mpam = !!(val & GICV5_IRS_IDR0_MPAM); + irs_data->idr0.swe = !!(val & GICV5_IRS_IDR0_SWE); + irs_data->idr0.irs_id = FIELD_GET(GICV5_IRS_IDR0_IRSID, val); + break; + case GICV5_IRS_IDR1: { + unsigned int iaffid_bits, priority_bits; + u8 vpe_id_bits; + + /* Ignore writes to PE_CNT as this is populated from num vcpus */ + iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, val); + priority_bits = FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, val); + + /* IAFFID_BITS is derived from the VMTE and encoded as N - 1. */ + vpe_id_bits = vgic_v5_vmte_vpe_id_bits(vcpu); + if (iaffid_bits != vpe_id_bits - 1) + return -EINVAL; + + if (priority_bits > gicv5_global_data.irs_pri_bits - 1) + return -EINVAL; + + irs_data->idr1.priority_bits = priority_bits; + break; + } + case GICV5_IRS_IDR2: + /* We always support LPIs */ + if (!(val & GICV5_IRS_IDR2_LPI)) + return -EINVAL; + + /* We only support LPIs with linear, non-metadata guest ISTs */ + if (val & GICV5_IRS_IDR2_IST_LEVELS) + return -EINVAL; + + if ((val & GICV5_IRS_IDR2_ISTMD) || + FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, val)) + return -EINVAL; + + /* We can't present more bits than we have support for in HW */ + if (FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val) > + vgic_v5_irs_ist_id_bits(&irs_caps)) + return -EINVAL; + + /* Min LPI ID bits must be greater than or equal to the HW */ + if (FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val) < + vgic_v5_irs_min_lpi_id_bits(&irs_caps)) + return -EINVAL; + + if (FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val) > + FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val)) + return -EINVAL; + + irs_data->idr2.istmd_sz = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, val); + irs_data->idr2.istmd = !!(val & GICV5_IRS_IDR2_ISTMD); + irs_data->idr2.ist_l2sz = FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, val); + irs_data->idr2.ist_levels = !!(val & GICV5_IRS_IDR2_IST_LEVELS); + irs_data->idr2.min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, val); + irs_data->idr2.id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, val); + break; + case GICV5_IRS_IDR5: + if (FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, val) != irs_data->idr5.spi_range) + return -EINVAL; + break; + case GICV5_IRS_IDR6: + if (FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, val) != irs_data->idr6.spi_irs_range) + return -EINVAL; + break; + case GICV5_IRS_IDR7: + if (FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, val) != irs_data->idr7.spi_base) + return -EINVAL; + break; + case GICV5_IRS_IST_BASER: + ret = vgic_v5_lpi_ist_exists(vcpu->kvm); + if (ret < 0) + return ret; + + if (ret && !vgic_v5_ist_baser_matches(irs_data, val)) + return -EINVAL; + + if (!irs_data->ist_baser.valid && + (val & GICV5_IRS_IST_BASER_VALID)) + irs_data->lpi_ist_restore_pending = true; + + vgic_v5_update_irs_ist_baser(irs_data, val); + break; + case GICV5_IRS_IST_CFGR: + ret = vgic_v5_lpi_ist_exists(vcpu->kvm); + if (ret < 0) + return ret; + + if (ret && !vgic_v5_ist_cfgr_matches(irs_data, val)) + return -EINVAL; + + irs_data->ist_cfgr.lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, val); + irs_data->ist_cfgr.l2sz = FIELD_GET(GICV5_IRS_IST_CFGR_L2SZ, val); + irs_data->ist_cfgr.istsz = FIELD_GET(GICV5_IRS_IST_CFGR_ISTSZ, val); + irs_data->ist_cfgr.structure = !!(val & GICV5_IRS_IST_CFGR_STRUCTURE); + break; + case GICV5_IRS_CR0: + vgic->enabled = !!(val & GICV5_IRS_CR0_IRSEN); + break; + case GICV5_IRS_SPI_CFGR: + break; + case GICV5_IRS_IIDR: + fallthrough; + case GICV5_IRS_AIDR: + break; + default: + return -EINVAL; + } + + return 0; +} + +static const struct vgic_register_region vgic_v5_irs_registers[] = { + /* + * This is the IRS_CONFIG_FRAME. + */ + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR0, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR1, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR2, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR3, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_IDR4, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR5, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR6, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IDR7, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IIDR, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_AIDR, vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_CR0, + vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_CR1, vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SYNCR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SYNC_STATUSR, + vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_SELR, vgic_v5_mmio_read_irs_spi, + vgic_v5_mmio_write_irs_spi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_DOMAINR, vgic_v5_mmio_read_irs_spi, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SPI_RESAMPLER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SPI_CFGR, + vgic_v5_mmio_read_irs_spi, + vgic_v5_mmio_write_irs_spi, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_SPI_STATUSR, + vgic_v5_mmio_read_irs_spi, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_SELR, + vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_PE_STATUSR, + vgic_v5_mmio_read_irs_misc, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_PE_CR0, vgic_v5_mmio_read_irs_misc, + vgic_v5_mmio_write_irs_misc, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_BASER, + vgic_v5_mmio_read_irs_ist, + vgic_v5_mmio_write_irs_ist, NULL, + vgic_v5_mmio_uaccess_write_irs, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_CFGR, + vgic_v5_mmio_read_irs_ist, + vgic_v5_mmio_write_irs_ist, NULL, + vgic_v5_mmio_uaccess_write_irs, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH_UACCESS(GICV5_IRS_IST_STATUSR, + vgic_v5_mmio_read_irs_ist, + vgic_mmio_write_wi, + vgic_v5_mmio_uaccess_read_irs_status, + vgic_mmio_uaccess_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MAP_L2_ISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + + /* + * The following registers are only for running VMs. They are not yet + * supported as we don't currently support nested, so expose them as + * read-as-zero/write-ignored. + */ + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_BASER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_CFGR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMT_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_SELR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_DBR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_HPPIR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_CR0, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VPE_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_DBR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_SELR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VM_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_L2_VMTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_L2_VISTR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_VMAP_VPER, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SAVE_VMR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SAVE_VM_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + + /* MEC, MPAM, SWERR - all unimplemented */ + + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MEC_IDR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MEC_MECID_R, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MPAM_IDR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_MPAM_PARTID_R, vgic_mmio_read_raz, + vgic_mmio_write_wi, 4, VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_STATUSR, vgic_mmio_read_raz, + vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER0, + vgic_mmio_read_raz, vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_IRS_SWERR_SYNDROMER1, + vgic_mmio_read_raz, vgic_mmio_write_wi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + + /* CoreSight identification registers */ + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_DEVARCH, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR4, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR5, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR6, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR7, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR0, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR1, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR2, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_PIDR3, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR0, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR1, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR2, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICV5_CORESIGHT_CIDR3, + vgic_v5_coresight_read, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), +}; + +unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev) +{ + dev->regions = vgic_v5_irs_registers; + dev->nr_regions = ARRAY_SIZE(vgic_v5_irs_registers); + + kvm_iodevice_init(&dev->dev, &kvm_io_gic_ops); + + /* We represent both of the IRS frames back to back, so this is 128K */ + return KVM_VGIC_V5_IRS_SIZE; +} + +int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address) +{ + struct vgic_io_device *io_device = &kvm->arch.vgic.vgic_v5_irs_data->iodev; + unsigned int len; + + /* + * Design choice: Force MMIO region to be 64k aligned. Simplifies + * pulling out registers. + */ + if (!IS_ALIGNED(irs_base_address, SZ_64K)) + return -EINVAL; + + len = vgic_v5_init_irs_iodev(io_device); + + io_device->base_addr = irs_base_address; + io_device->iodev_type = IODEV_GICV5_IRS; + io_device->redist_vcpu = NULL; + + return kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, irs_base_address, len, + &io_device->dev); +} + +/** + * kvm_vgic_v5_irs_init: initialize the IRS data structures + * @kvm: kvm struct pointer + * @nr_spis: number of spis, frozen by caller + */ +int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct vgic_v5_irs *irs = dist->vgic_v5_irs_data; + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + size_t nr_spi_bits; + u64 mmfr0; + int ret, i; + + /* + * We (KVM) allocate an Interrupt State Table (IST) for SPIs. The + * hardware mandates that lower 6 bits of the address are 0. Each ISTE + * is 4 bytes in size (or larger if metadata storage is required), so 16 + * entries would be enough for alignment. We require userspace to + * provide at least 32 SPIs, so we always meet alignment requirements. + */ + if (nr_spis) { + dist->spis = kcalloc(nr_spis, sizeof(struct vgic_irq), + GFP_KERNEL_ACCOUNT); + if (!dist->spis) + return -ENOMEM; + + /* + * In the following code we do not take the irq struct lock since + * no other action on irq structs can happen while the VGIC is + * not initialized yet. + */ + for (i = 0; i < nr_spis; i++) { + struct vgic_irq *irq = &dist->spis[i]; + + irq->intid = vgic_v5_make_spi(i); + INIT_LIST_HEAD(&irq->ap_list); + raw_spin_lock_init(&irq->irq_lock); + irq->vcpu = NULL; + irq->target_vcpu = vcpu0; + refcount_set(&irq->refcount, 0); + /* + * The guest controls the enable state, and again it is + * directly handled by the hardware. From our point of + * view it is always enabled. + */ + irq->enabled = 1; + vgic_v5_set_spi_ops(irq); + } + + nr_spi_bits = fls(roundup_pow_of_two(nr_spis)) - 1; + + ret = vgic_v5_spi_ist_alloc(kvm, nr_spi_bits); + if (ret) { + kfree(dist->spis); + dist->spis = NULL; + return ret; + } + } + + /* Set sane initial state for the IRS MMIO registers */ + + irs->idr0.domain = GICV5_IRS_IDR0_INT_DOM_NON_SECURE; + + mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); + irs->idr0.pa_range = cpuid_feature_extract_unsigned_field(mmfr0, + ID_AA64MMFR0_EL1_PARANGE_SHIFT); + + irs->idr0.virt = 0; + irs->idr0.setlpi = 0; + irs->idr0.mec = 0; + irs->idr0.mpam = 0; + irs->idr0.swe = 0; + irs->idr0.irs_id = 0; + + irs->idr1.priority_bits = gicv5_global_data.irs_pri_bits - 1; + + /* + * Support 16-bits of ID space for the IRS. This should be sufficient + * for most applications, and the CPUIF is guaranteed to have at least + * 16-bits of ID space support (we actually present 16-bits there, even + * if the hardware supports more). Warn if the hardware doesn't support + * 16 bits, and use the smaller value. YMMV! + * + * As for the minimum number of ID bits, we match the hardware's + * capability. + */ + if (vgic_v5_irs_ist_id_bits(&irs_caps) < 16) + pr_warn_ratelimited("Host IRS supports fewer than 16 ID bits for ISTs (%u)\n", + vgic_v5_irs_ist_id_bits(&irs_caps)); + + irs->idr2.id_bits = min(16, vgic_v5_irs_ist_id_bits(&irs_caps)); + irs->idr2.min_lpi_id_bits = vgic_v5_irs_min_lpi_id_bits(&irs_caps); + + /* Only allow the guest to create Linear ISTs - simplifies Save/Restore */ + irs->idr2.ist_levels = 0; + irs->idr2.ist_l2sz = GICV5_IRS_IST_CFGR_L2SZ_4K; + irs->idr2.istmd = 0; + irs->idr2.istmd_sz = 0; + + /* We have a single IRS, only. All SPIs reside here! */ + irs->idr5.spi_range = nr_spis; + irs->idr6.spi_irs_range = nr_spis; + irs->idr7.spi_base = 0; + + irs->cr1.sh = 0; + irs->cr1.oc = 0; + irs->cr1.ic = 0; + irs->cr1.ist_ra = 0; + irs->cr1.ist_wa = 0; + irs->cr1.vmt_ra = 0; + irs->cr1.vpet_ra = 0; + irs->cr1.vmd_ra = 0; + irs->cr1.vmd_wa = 0; + irs->cr1.vped_ra = 0; + irs->cr1.vped_wa = 0; + + irs->spi_selr.id = -1; + + irs->pe_selr.iaffid = -1; + + irs->ist_cfgr.lpi_id_bits = 0; + irs->ist_cfgr.l2sz = 0; + irs->ist_cfgr.istsz = 0; + irs->ist_cfgr.structure = 0; + + irs->ist_baser.valid = 0; + irs->ist_baser.addr = 0; + irs->lpi_ist_restore_pending = false; + + return 0; +} + +int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits) +{ + struct vgic_v5_irs *irs = kvm->arch.vgic.vgic_v5_irs_data; + + if (!irs) + return -ENXIO; + + if (!irs->ist_baser.valid) + return 0; + + if (!vgic_v5_ist_cfgr_valid(irs)) + return -EINVAL; + + *id_bits = irs->ist_cfgr.lpi_id_bits; + + return 1; +} + +int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr) +{ + const struct vgic_register_region *region; + struct vgic_reg_attr reg_attr; + struct kvm_vcpu *vcpu; + gpa_t addr, offset; + int ret, align; + + ret = vgic_v5_parse_attr(dev, attr, ®_attr); + if (ret) + return ret; + + vcpu = reg_attr.vcpu; + addr = reg_attr.addr; + + if (attr->group == KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS) + return vgic_v5_has_cpu_sysregs_attr(vcpu, attr); + + offset = attr->attr; + + region = vgic_find_mmio_region(vgic_v5_irs_registers, + ARRAY_SIZE(vgic_v5_irs_registers), + offset); + if (!region) + return -ENXIO; + + align = region->access_flags & VGIC_ACCESS_64bit ? 0x7 : 0x3; + if (offset & align) + return -EINVAL; + + return 0; +} + +/* + * Access the IRS MMIO Regs. Relevant locks have been taken by the calling code. + */ +int vgic_v5_irs_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + u64 *reg, bool is_write) +{ + const struct vgic_register_region *region; + gpa_t addr, offset; + unsigned int len; + int align, ret = 0; + + offset = attr->attr; + + if (IS_VGIC_ADDR_UNDEF(dev->kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base)) + return -ENXIO; + + region = vgic_find_mmio_region(vgic_v5_irs_registers, + ARRAY_SIZE(vgic_v5_irs_registers), + offset); + if (!region) + return -ENXIO; + + /* + * Although the spec supports upper/lower 32-bit accesses to + * 64-bit IRS registers, the userspace ABI requires 64-bit + * accesses to all 64-bit wide registers. We therefore only + * support 32-bit accesses to 32-bit-wide registers. + */ + align = region->access_flags & VGIC_ACCESS_64bit ? 0x7 : 0x3; + len = region->access_flags & VGIC_ACCESS_64bit ? 8 : 4; + + if (offset & align) + return -EINVAL; + + addr = dev->kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base + offset; + + if (is_write) { + if (region->uaccess_write) + ret = region->uaccess_write(kvm_get_vcpu(dev->kvm, 0), + addr, len, *reg); + else + region->write(kvm_get_vcpu(dev->kvm, 0), addr, len, *reg); + } else { + if (region->uaccess_read) + *reg = region->uaccess_read(kvm_get_vcpu(dev->kvm, 0), + addr, len); + else + *reg = region->read(kvm_get_vcpu(dev->kvm, 0), addr, len); + } + + return ret; +} diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index c10a11373e10..e5657cb60356 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2134,9 +2134,9 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev, u64 val; /* - * MAPC with V=0 keeps the ITEs mapped but drops their collection, - * and with it the ICID. Save a zeroed entry, which the restore path - * reads back as invalid. + * MAPC with V=0 and clearing GITS_BASER.Valid both keep the + * ITEs mapped while dropping their collection, and with it the ICID. + * Save a zeroed entry, which the restore path reads back as invalid. */ if (!ite->collection) return vgic_its_write_entry_lock(its, gpa, 0ULL, ite); diff --git a/arch/arm64/kvm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c index 90be99443df3..1c205cb1361f 100644 --- a/arch/arm64/kvm/vgic/vgic-kvm-device.c +++ b/arch/arm64/kvm/vgic/vgic-kvm-device.c @@ -181,6 +181,14 @@ static int kvm_vgic_addr(struct kvm *kvm, struct kvm_device_attr *attr, bool wri addr |= (u64)rdreg->count << KVM_VGIC_V3_RDIST_COUNT_SHIFT; goto out; } + case KVM_VGIC_V5_ADDR_TYPE_IRS: + r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V5); + if (r) + break; + addr_ptr = &vgic->vgic_v5_irs_data->vgic_v5_irs_base; + alignment = SZ_64K; + size = KVM_VGIC_V5_IRS_SIZE; + break; default: r = -ENODEV; } @@ -224,31 +232,48 @@ static int vgic_set_common_attr(struct kvm_device *dev, if (get_user(val, uaddr)) return -EFAULT; - /* - * We require: - * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs - * - at most 1024 interrupts - * - a multiple of 32 interrupts - */ - if (val < (VGIC_NR_PRIVATE_IRQS + 32) || - val > VGIC_MAX_RESERVED || - (val & 31)) - return -EINVAL; + if (!vgic_is_v5(dev->kvm)) { + /* + * We require: + * - at least 32 SPIs on top of the 16 SGIs and 16 PPIs + * - at most 1024 interrupts + * - a multiple of 32 interrupts + */ + if (val < (VGIC_NR_PRIVATE_IRQS + 32) || + val > VGIC_MAX_RESERVED || (val & 31)) + return -EINVAL; - mutex_lock(&dev->kvm->arch.config_lock); + mutex_lock(&dev->kvm->arch.config_lock); - /* - * Either userspace has already configured NR_IRQS or - * the vgic has already been initialized and vgic_init() - * supplied a default amount of SPIs. - */ - if (dev->kvm->arch.vgic.nr_spis) - ret = -EBUSY; - else - dev->kvm->arch.vgic.nr_spis = - val - VGIC_NR_PRIVATE_IRQS; + /* + * Either userspace has already configured NR_IRQS or + * the vgic has already been initialized and vgic_init() + * supplied a default amount of SPIs. + */ + if (dev->kvm->arch.vgic.nr_spis) + ret = -EBUSY; + else + dev->kvm->arch.vgic.nr_spis = + val - VGIC_NR_PRIVATE_IRQS; - mutex_unlock(&dev->kvm->arch.config_lock); + mutex_unlock(&dev->kvm->arch.config_lock); + } else { + /* + * GICv5 reports a number of SPIs, not a total number of + * interrupts. Require a multiple of 32 SPIs. + */ + if (val < VGIC_V5_DEFAULT_NR_SPIS || + val > VGIC_V5_MAX_NR_SPIS || + (val & 31)) + return -EINVAL; + + mutex_lock(&dev->kvm->arch.config_lock); + if (vgic_initialized(dev->kvm) || dev->kvm->arch.vgic.nr_spis) + ret = -EBUSY; + else + dev->kvm->arch.vgic.nr_spis = val; + mutex_unlock(&dev->kvm->arch.config_lock); + } return ret; } @@ -299,9 +324,14 @@ static int vgic_get_common_attr(struct kvm_device *dev, return (r == -ENODEV) ? -ENXIO : r; case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: { u32 __user *uaddr = (u32 __user *)(long)attr->addr; - - r = put_user(dev->kvm->arch.vgic.nr_spis + - VGIC_NR_PRIVATE_IRQS, uaddr); + /* Older GICs */ + if (!vgic_is_v5(dev->kvm)) { + r = put_user(dev->kvm->arch.vgic.nr_spis + + VGIC_NR_PRIVATE_IRQS, + uaddr); + } else { + r = put_user(dev->kvm->arch.vgic.nr_spis, uaddr); + } break; } } @@ -512,7 +542,7 @@ int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, * Allow access to certain ID-like registers prior to VGIC initialization, * thereby allowing the VMM to provision the features / sizing of the VGIC. */ -static bool reg_allowed_pre_init(struct kvm_device_attr *attr) +static bool v3_reg_allowed_pre_init(struct kvm_device_attr *attr) { if (attr->group != KVM_DEV_ARM_VGIC_GRP_DIST_REGS) return false; @@ -575,7 +605,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev, mutex_lock(&dev->kvm->arch.config_lock); - if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) { + if (!(vgic_initialized(dev->kvm) || v3_reg_allowed_pre_init(attr))) { ret = -EBUSY; goto out; } @@ -743,26 +773,178 @@ static int vgic_v5_get_userspace_ppis(struct kvm_device *dev, return ret; } +int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, + struct vgic_reg_attr *reg_attr) +{ + unsigned long vgic_mpidr, mpidr_reg; + + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + vgic_mpidr = (attr->attr & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) >> + KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT; + + mpidr_reg = VGIC_TO_MPIDR(vgic_mpidr); + reg_attr->vcpu = kvm_mpidr_to_vcpu(dev->kvm, mpidr_reg); + break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + reg_attr->vcpu = kvm_get_vcpu(dev->kvm, 0); + break; + default: + return -EINVAL; + } + + if (!reg_attr->vcpu) + return -EINVAL; + + reg_attr->addr = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK; + + return 0; +} + +/* + * Some registers can potentially be read before the core GIC & IRS has been + * initialised. Right now, everything is required to be post-init. + */ +static bool v5_reg_allowed_pre_init(struct kvm_device_attr *attr) +{ + return false; +} + +static bool vgic_v5_vm_has_run_once(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vcpu_has_run_once(vcpu)) + return true; + } + + return false; +} + +/* + * vgic_v5_attr_regs_access - allows user space to access VGIC v5 state + * + * @dev: kvm device handle + * @attr: kvm device attribute + * @is_write: true if userspace is writing a register + */ +static int vgic_v5_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + bool is_write) +{ + u64 __user *uaddr = (u64 __user *)(unsigned long)attr->addr; + struct vgic_reg_attr reg_attr; + struct kvm_vcpu *vcpu; + bool uaccess; + u64 val; + int ret; + + ret = vgic_v5_parse_attr(dev, attr, ®_attr); + if (ret) + return ret; + + vcpu = reg_attr.vcpu; + + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + /* Sysregs uaccess is performed by the sysreg handling code */ + uaccess = false; + break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; + default: + uaccess = true; + } + + if (uaccess && is_write) { + if (get_user(val, uaddr)) + return -EFAULT; + } + + mutex_lock(&dev->kvm->lock); + + if (kvm_trylock_all_vcpus(dev->kvm)) { + mutex_unlock(&dev->kvm->lock); + return -EBUSY; + } + + mutex_lock(&dev->kvm->arch.config_lock); + + if (!(vgic_initialized(dev->kvm) || v5_reg_allowed_pre_init(attr))) { + ret = -EBUSY; + goto out; + } + + if (is_write && vgic_v5_vm_has_run_once(dev->kvm)) { + ret = -EBUSY; + goto out; + } + + switch (attr->group) { + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + ret = vgic_v5_cpu_sysregs_uaccess(vcpu, attr, is_write); + break; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + /* + * The IRS registers are a mixture of 32-bit and 64-bit + * registers. Internally, we always perform the correctly sized + * access, but the UAPI is defined in such a way that we are + * always provided a __u64 by userspace. When userspace writes, + * the upper 32-bits are ignored for 32-bit accesses, and on a + * read any 32-bit accesses are written back to user memory + * using the full 64-bits. + */ + ret = vgic_v5_irs_attr_regs_access(dev, attr, &val, is_write); + break; + default: + ret = -EINVAL; + break; + } + +out: + mutex_unlock(&dev->kvm->arch.config_lock); + kvm_unlock_all_vcpus(dev->kvm); + mutex_unlock(&dev->kvm->lock); + + if (!ret && uaccess && !is_write) + ret = put_user(val, uaddr); + + return ret; +} + static int vgic_v5_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: + break; + case KVM_DEV_ARM_VGIC_GRP_IST: + if (attr->attr) + return -ENXIO; + + return vgic_v5_irs_restore_ists(dev->kvm, attr); + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + return vgic_v5_attr_regs_access(dev, attr, true); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: - return -ENXIO; + break; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: - return vgic_set_common_attr(dev, attr); + break; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: default: return -ENXIO; } + break; default: return -ENXIO; } + return vgic_set_common_attr(dev, attr); } static int vgic_v5_get_attr(struct kvm_device *dev, @@ -770,21 +952,33 @@ static int vgic_v5_get_attr(struct kvm_device *dev, { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: + break; + case KVM_DEV_ARM_VGIC_GRP_IST: + if (attr->attr) + return -ENXIO; + + return vgic_v5_irs_save_ists(dev->kvm, attr); + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + return vgic_v5_attr_regs_access(dev, attr, false); case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: - return -ENXIO; + break; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: - return vgic_get_common_attr(dev, attr); + break; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: return vgic_v5_get_userspace_ppis(dev, attr); default: return -ENXIO; } + break; default: return -ENXIO; } + + return vgic_get_common_attr(dev, attr); } static int vgic_v5_has_attr(struct kvm_device *dev, @@ -792,18 +986,30 @@ static int vgic_v5_has_attr(struct kvm_device *dev, { switch (attr->group) { case KVM_DEV_ARM_VGIC_GRP_ADDR: - case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: - case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: + switch (attr->attr) { + case KVM_VGIC_V5_ADDR_TYPE_IRS: + return 0; + } return -ENXIO; + case KVM_DEV_ARM_VGIC_GRP_IRS_REGS: + fallthrough; + case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS: + return vgic_v5_has_attr_regs(dev, attr); + case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: + return 0; case KVM_DEV_ARM_VGIC_GRP_CTRL: switch (attr->attr) { case KVM_DEV_ARM_VGIC_CTRL_INIT: return 0; case KVM_DEV_ARM_VGIC_USERSPACE_PPIS: return 0; + case KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES: default: return -ENXIO; } + break; + case KVM_DEV_ARM_VGIC_GRP_IST: + return attr->attr ? -ENXIO : 0; default: return -ENXIO; } diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 5913a20d8301..5ab789accc9d 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -818,16 +818,21 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu) rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers); rd_dev->redist_vcpu = vcpu; + /* Protected by slots_lock */ + rdreg->free_index++; + mutex_unlock(&kvm->arch.config_lock); ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base, 2 * SZ_64K, &rd_dev->dev); - if (ret) - return ret; + if (ret) { + guard(mutex)(&vcpu->kvm->arch.config_lock); + vgic_cpu->rdreg->free_index--; + vgic_cpu->rdreg = NULL; + vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF; + } - /* Protected by slots_lock */ - rdreg->free_index++; - return 0; + return ret; out_unlock: mutex_unlock(&kvm->arch.config_lock); @@ -841,7 +846,8 @@ void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu) kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev); } -static int vgic_register_all_redist_iodevs(struct kvm *kvm) +static int vgic_register_all_redist_iodevs(struct kvm *kvm, + struct vgic_redist_region *rdreg) { struct kvm_vcpu *vcpu; unsigned long c; @@ -861,7 +867,14 @@ static int vgic_register_all_redist_iodevs(struct kvm *kvm) for (i = 0; i < c; i++) { vcpu = kvm_get_vcpu(kvm, i); + if (vcpu->arch.vgic_cpu.rdreg != rdreg) + continue; + vgic_unregister_redist_iodev(vcpu); + + guard(mutex)(&vcpu->kvm->arch.config_lock); + vcpu->arch.vgic_cpu.rdreg = NULL; + vcpu->arch.vgic_cpu.rd_iodev.base_addr = VGIC_ADDR_UNDEF; } } @@ -970,10 +983,13 @@ void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdre int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count) { + struct vgic_redist_region *rdreg; int ret; mutex_lock(&kvm->arch.config_lock); ret = vgic_v3_alloc_redist_region(kvm, index, addr, count); + if (!ret) + rdreg = vgic_v3_rdist_region_from_index(kvm, index); mutex_unlock(&kvm->arch.config_lock); if (ret) return ret; @@ -982,12 +998,9 @@ int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count) * Register iodevs for each existing VCPU. Adding more VCPUs * afterwards will register the iodevs when needed. */ - ret = vgic_register_all_redist_iodevs(kvm); + ret = vgic_register_all_redist_iodevs(kvm, rdreg); if (ret) { - struct vgic_redist_region *rdreg; - mutex_lock(&kvm->arch.config_lock); - rdreg = vgic_v3_rdist_region_from_index(kvm, index); vgic_v3_free_redist_region(kvm, rdreg); mutex_unlock(&kvm->arch.config_lock); return ret; diff --git a/arch/arm64/kvm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c index 74d76dec9730..fddb9da0403d 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.c +++ b/arch/arm64/kvm/vgic/vgic-mmio.c @@ -1065,6 +1065,9 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, case IODEV_ITS: data = region->its_read(vcpu->kvm, iodev->its, addr, len); break; + case IODEV_GICV5_IRS: + data = region->read(vcpu, addr, len); + break; } vgic_data_host_to_mmio_bus(val, len, data); @@ -1095,6 +1098,9 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, case IODEV_ITS: region->its_write(vcpu->kvm, iodev->its, addr, len, data); break; + case IODEV_GICV5_IRS: + region->write(vcpu, addr, len, data); + break; } return 0; diff --git a/arch/arm64/kvm/vgic/vgic-mmio.h b/arch/arm64/kvm/vgic/vgic-mmio.h index 50dc80220b0f..38ed730d68ac 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio.h +++ b/arch/arm64/kvm/vgic/vgic-mmio.h @@ -217,6 +217,8 @@ unsigned int vgic_v2_init_cpuif_iodev(struct vgic_io_device *dev); unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev); +unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev); + u64 vgic_sanitise_outer_cacheability(u64 reg); u64 vgic_sanitise_inner_cacheability(u64 reg); u64 vgic_sanitise_shareability(u64 reg); diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.c b/arch/arm64/kvm/vgic/vgic-v5-tables.c new file mode 100644 index 000000000000..d577e485ebe9 --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.c @@ -0,0 +1,1872 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vgic.h" +#include "vgic-v5-tables.h" + +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static struct vgic_v5_vmt *vmt_info; + +/* Serialises IRS MMIO interaction (commands) and accesses to IRS tables. */ +DEFINE_RAW_SPINLOCK(vgic_v5_irs_lock); + +/* Serialises lazy installation of shared second-level VMTs. */ +static DEFINE_MUTEX(vmt_l2_lock); + +static DEFINE_XARRAY(vm_info); + +/* Level 1 Virtual Machine Table Entry */ +#define GICV5_VMTEL1E_VALID BIT_ULL(0) +/* Note that there is no shift for the address by design */ +#define GICV5_VMTEL1E_L2_ADDR GENMASK(51, 12) + +#define GICV5_VMTEL2E_SIZE 32ULL +/* An L2 table (two-level VMT) is ALWAYS 4kB! */ +#define GICV5_VMT_L2_TABLE_SIZE 4096ULL +#define GICV5_VMT_L2_TABLE_ENTRIES (GICV5_VMT_L2_TABLE_SIZE / GICV5_VMTEL2E_SIZE) + +/* + * As the L2 VMTE is a large data structure, we are splitting it into 4 parts. + * We only mask and shift WITHIN each part for simplicity. + */ +/* First 64-bit chunk */ +#define GICV5_VMTEL2E_VALID BIT_ULL(0) +#define GICV5_VMTEL2E_VMD_ADDR_SHIFT 3ULL +#define GICV5_VMTEL2E_VMD_ADDR GENMASK_ULL(55, 3) +/* Second 64-bit chunk */ +#define GICV5_VMTEL2E_VPET_ADDR_SHIFT 3ULL +#define GICV5_VMTEL2E_VPET_ADDR GENMASK_ULL(55, 3) +#define GICV5_VMTEL2E_VPE_ID_BITS GENMASK_ULL(63, 59) +/* Third & fourth 64-bit chunks (the encodings are the same for each) */ +#define GICV5_VMTEL2E_IST_VALID BIT_ULL(0) +#define GICV5_VMTEL2E_IST_L2SZ GENMASK_ULL(2, 1) +#define GICV5_VMTEL2E_IST_ADDR_SHIFT 6ULL +#define GICV5_VMTEL2E_IST_ADDR GENMASK_ULL(55, 6) +#define GICV5_VMTEL2E_IST_ISTSZ GENMASK_ULL(57, 56) +#define GICV5_VMTEL2E_IST_STRUCTURE BIT_ULL(58) +#define GICV5_VMTEL2E_IST_ID_BITS GENMASK_ULL(63, 59) + +/* Virtual PE Table Entry */ +#define GICV5_VPE_VALID BIT_ULL(0) +/* Note that there is no shift for the address by design. */ +#define GICV5_VPED_ADDR GENMASK_ULL(55, 3) + +/* L2 Interrupt State Table Entry */ +#define GICV5_ISTL2E_PENDING BIT(0) +#define GICV5_ISTL2E_ACTIVE BIT(1) +#define GICV5_ISTL2E_HM BIT(2) +#define GICV5_ISTL2E_ENABLE BIT(3) +#define GICV5_ISTL2E_IRM BIT(4) +#define GICV5_ISTL2E_HWU GENMASK(10, 9) +#define GICV5_ISTL2E_PRIORITY GENMASK(15, 11) +#define GICV5_ISTL2E_IAFFID GENMASK(31, 16) + +#define GICV5_ISTE_SIZE(istsz) BIT((istsz) + 2) +#define GICV5_LINEAR_IST_SIZE(id_bits, istsz) \ + (BIT(id_bits) * GICV5_ISTE_SIZE(istsz)) + +/* + * The LPI and SPI configuration is stored in the 2nd and 3rd 64-bit chunks of + * the VMTE (0-based). We call this a section here in an attempt to simplify the + * code. + */ +#define GICV5_VMTEL2_LPI_SECTION 2 +#define GICV5_VMTEL2_SPI_SECTION 3 + +struct vgic_v5_ist_desc { + struct vgic_v5_vm_info *vmi; + void *base; + unsigned int id_bits; + unsigned int istsz; + unsigned int l2sz; + size_t iste_size; + bool present; +}; + +struct vgic_v5_two_level_ist_shape { + size_t l1_entries; + size_t l2_entries; +}; + +struct vgic_v5_pending_irq { + u32 irq; + struct list_head next; +}; + +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, + unsigned int istsz); +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2_split); +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2_split); +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi); +static int vgic_v5_spi_ist_free(struct kvm *kvm); + +/* + * Our IRS might be coherent or non-coherent. If coherent, we can just emit a + * DSB to ensure that we're in sync. However, when non-coherent, we need to + * manage our cached data explicitly. + * + * This helper is used to handle both coherent and non-coherent IRSes, and + * handles all combinations of cleaning and invalidating to the PoC. + */ +static void vgic_v5_clean_inval(void *va, size_t size) +{ + unsigned long base = (unsigned long)va; + + dsb(ishst); + + if (kvm_vgic_global_state.vgic_v5_irs_caps.non_coherent) + dcache_clean_inval_poc(base, base + size); +} + +static void vgic_v5_drain_pending_irqs(struct kvm *kvm, + struct vgic_v5_vm_info *vmi, + bool reinject) +{ + struct vgic_v5_pending_irq *pirq, *tmp; + + list_for_each_entry_safe(pirq, tmp, &vmi->pending_irqs, next) { + if (reinject) + kvm_call_hyp(__vgic_v5_vdpend, pirq->irq, true, + vgic_v5_vm_id(kvm)); + + list_del(&pirq->next); + kfree(pirq); + } +} + +/* + * Create a linear VM Table. Directly using the number of entries supplied as + * the size of an L2 VMTE (32 bytes) guarantees that our allocation is aligned per + * the GICv5 requirements for the IRS_VMT_BASER. + */ +static int vgic_v5_alloc_vmt_linear(unsigned int num_entries) +{ + vmt_info->linear.vmt_base = kzalloc_objs(*vmt_info->linear.vmt_base, + num_entries); + if (!vmt_info->linear.vmt_base) + return -ENOMEM; + + vgic_v5_clean_inval(vmt_info->linear.vmt_base, + num_entries * sizeof(struct vmtl2_entry)); + + return 0; +} + +/* + * Allocate the first level of a two-level VM table. The second-level VM tables + * are allocated on demand (by vgic_v5_alloc_l2_vmt()). + */ +static int vgic_v5_alloc_vmt_two_level(unsigned int num_entries) +{ + /* + * Each L2 VMT array is always 4k-sized (covering 128 VMs). This is + * mandated by the GICv5 specification (GICv5 EAC0 Specification rule + * D_LSPBK). Hence, round up the number of entries to be at least 128 + * (or the next highest power of two as we give the HW the number of VM + * ID bits). + */ + if (num_entries < GICV5_VMT_L2_TABLE_ENTRIES) + num_entries = GICV5_VMT_L2_TABLE_ENTRIES; + num_entries = roundup_pow_of_two(num_entries); + + vmt_info->l2.num_l1_ents = (num_entries / GICV5_VMT_L2_TABLE_ENTRIES); + vmt_info->l2.vmt_base = kzalloc_objs(*vmt_info->l2.vmt_base, + vmt_info->l2.num_l1_ents); + if (!vmt_info->l2.vmt_base) + return -ENOMEM; + + vmt_info->l2.l2ptrs = kzalloc_objs(*vmt_info->l2.l2ptrs, + vmt_info->l2.num_l1_ents, + GFP_KERNEL); + if (!vmt_info->l2.l2ptrs) { + kfree(vmt_info->l2.vmt_base); + return -ENOMEM; + } + + vgic_v5_clean_inval(vmt_info->l2.vmt_base, + vmt_info->l2.num_l1_ents * sizeof(vmtl1_entry)); + + return 0; +} + +/* + * Allocate a second level VMT, if required. This can be called eagerly, and + * will only perform the allocation if required. + */ +static int vgic_v5_alloc_l2_vmt(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + u32 vm_id = vgic_v5_vm_id(kvm); + enum gicv5_vcpu_cmd cmd = VMT_L2_MAP; + struct vmtl2_entry *l2_table; + unsigned int l1_index; + int ret; + + /* Nothing to do if we have linear tables! */ + if (!vmt_info->two_level) + return 0; + + if (vm_id == VGIC_V5_VM_ID_INVAL) + return -EINVAL; + + /* + * We have 4k-sized L2 tables - this is mandated by the spec for + * two-level VMTs (GICv5 EAC0 Specification rule D_LSPBK). This means + * that we have 128 entries per L1 VMTE. + */ + l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES; + + guard(mutex)(&vmt_l2_lock); + + /* Already valid? Great! */ + if (vmt_info->l2.l2ptrs[l1_index]) + return 0; + + l2_table = kzalloc_objs(*l2_table, GICV5_VMT_L2_TABLE_ENTRIES); + if (!l2_table) + return -ENOMEM; + + /* The VMT is shared between all VMs. */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(l2_table, GICV5_VMT_L2_TABLE_SIZE); + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + + WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], + cpu_to_le64(virt_to_phys(l2_table))); + + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + + } + + /* + * VMAP in the L2 VMT via the IRS. We use any of the VM's CPUs as a + * conduit for interacting with the host's IRS. In the current case, + * this lets us resolve the VM ID to pass to the hardware. + */ + ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + + /* We've failed to make the L2 VMT valid - things are very broken! */ + if (ret) { + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* Remove the pointer from L1 table */ + WRITE_ONCE(vmt_info->l2.vmt_base[l1_index], 0); + + vgic_v5_clean_inval(vmt_info->l2.vmt_base + l1_index, + sizeof(vmtl1_entry)); + } + + kfree(l2_table); + return ret; + } + + vmt_info->l2.l2ptrs[l1_index] = l2_table; + + return 0; +} + +/* + * Allocate the top-level VMT. This can either be linear or two-level. + */ +int vgic_v5_vmt_allocate(unsigned int max_vpes) +{ + int ret; + + /* Allocate the tracking structure */ + vmt_info = kzalloc_obj(*vmt_info, GFP_KERNEL); + if (!vmt_info) + return -ENOMEM; + + ida_init(&vmt_info->vm_id_ida); + vmt_info->max_vpes = max_vpes; + vmt_info->vmd_size = vgic_v5_irs_vmd_size(&irs_caps); + vmt_info->vped_size = vgic_v5_irs_vped_size(&irs_caps); + vmt_info->two_level = vgic_v5_irs_two_level_vmt_support(&irs_caps); + vmt_info->num_entries = vgic_v5_irs_max_vms(&irs_caps); + + if (vmt_info->two_level) + ret = vgic_v5_alloc_vmt_two_level(vmt_info->num_entries); + else + ret = vgic_v5_alloc_vmt_linear(vmt_info->num_entries); + + /* If anything failed, free our tracking structure before returning */ + if (ret) { + kfree(vmt_info); + vmt_info = NULL; + } + + return ret; +} + +/* + * Free the VMT and associated tracking structures. This isn't strictly expected + * to be called in general operation, but instead exists for completeness. + */ +int vgic_v5_vmt_free(void) +{ + if (!vmt_info) + return 0; + + if (!vmt_info->two_level) { + kfree(vmt_info->linear.vmt_base); + } else { + /* Free the L2 tables; kfree(NULL) is safe */ + for (int i = 0; i < vmt_info->l2.num_l1_ents; ++i) + kfree(vmt_info->l2.l2ptrs[i]); + kfree(vmt_info->l2.l2ptrs); + + /* And now free the L1 table */ + kfree(vmt_info->l2.vmt_base); + } + + ida_destroy(&vmt_info->vm_id_ida); + kfree(vmt_info); + vmt_info = NULL; + + return 0; +} + +/* + * Look up a VMT Entry by VM ID. + */ +static struct vmtl2_entry *vgic_v5_get_l2_vmte(u32 vm_id) +{ + unsigned int l1_index, l2_index; + struct vmtl2_entry *l2_table; + + if (vm_id == VGIC_V5_VM_ID_INVAL) + return ERR_PTR(-EINVAL); + + if (!vmt_info->two_level) + return &vmt_info->linear.vmt_base[vm_id]; + + l1_index = vm_id / GICV5_VMT_L2_TABLE_ENTRIES; + l2_index = vm_id % GICV5_VMT_L2_TABLE_ENTRIES; + + if (l1_index >= vmt_info->l2.num_l1_ents) + return ERR_PTR(-E2BIG); + + if (!vmt_info->l2.l2ptrs[l1_index]) + return ERR_PTR(-EINVAL); + + l2_table = vmt_info->l2.l2ptrs[l1_index]; + return &l2_table[l2_index]; +} + +/* + * Zero a VMT Entry, and flush & invalidate to the PoC, if required. + */ +static int vgic_v5_reset_vmte(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* + * The VMT is normal memory shared with the IRS. Invalidate before + * rewriting the entry so that cacheline-granular maintenance cannot + * later push stale data for neighbouring IRS-visible state back to + * memory. + */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + + /* + * Prevent the compiler from eliding the individual VMTE + * stores. Ordering and visibility to the IRS are provided by the + * surrounding cache maintenance and command protocol, not by + * WRITE_ONCE(). + * + * The same compiler-access constraint applies to READ_ONCE() users in + * this file: when inspecting IRS-visible table entries, read the field + * exactly once and prevent the compiler from reusing, merging or + * tearing the access. Coherency and freshness for non-coherent IRSes + * still come from the surrounding cache maintenance. + */ + WRITE_ONCE(vmte->val[0], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[1], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[2], cpu_to_le64(0ULL)); + WRITE_ONCE(vmte->val[3], cpu_to_le64(0ULL)); + + /* And make our write visible to the IRS (if non-coherent) */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* + * Use the IDA to allocate a new VM ID, and track it in the gicv5_vm data + * structure. If we're out of VM IDs, the IDA catches that, and we return the + * error (-ENOSPC). If we've previously allocated a VM ID, we catch that too and + * return -EBUSY. + */ +int vgic_v5_allocate_vm_id(struct kvm *kvm) +{ + int id; + + if (kvm->arch.vgic.gicv5_vm.vm_id != VGIC_V5_VM_ID_INVAL) + return -EBUSY; + + id = ida_alloc_max(&vmt_info->vm_id_ida, vmt_info->num_entries - 1u, + GFP_KERNEL); + if (id < 0) + return id; + + kvm->arch.vgic.gicv5_vm.vm_id = id; + + return 0; +} + +/* + * Release the VM ID to allow it to be reallocated in the future. + */ +void vgic_v5_release_vm_id(struct kvm *kvm) +{ + if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL) + return; + + ida_free(&vmt_info->vm_id_ida, kvm->arch.vgic.gicv5_vm.vm_id); + kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL; +} + +/* + * Initialise an entry in the VMT based on the index of the VM. + * + * Note: We don't mark the VMTE as valid as this needs to be done by + * the hardware. + */ +int vgic_v5_vmte_init(struct kvm *kvm) +{ + size_t vmd_alloc_size, vpet_alloc_size, vped_alloc_size; + void *vped_base = NULL, *vmd = NULL; + struct vgic_v5_vm_info *vmi = NULL; + u64 tmp, vmte_val0 = 0, vmte_val1; + u32 vm_id = vgic_v5_vm_id(kvm); + int ret, nr_cpus, nr_vcpus; + bool vmi_inserted = false; + struct vmtl2_entry *vmte; + vpe_entry *vpet = NULL; + struct kvm_vcpu *vcpu; + u16 max_vpe_id = 0; + unsigned long i; + + nr_vcpus = atomic_read(&kvm->online_vcpus); + if (nr_vcpus > vmt_info->max_vpes) + return -E2BIG; + + /* + * If we're using two-level VMTs, L2 is allocated on demand. For linear + * VMTs, this is a NOP. + */ + ret = vgic_v5_alloc_l2_vmt(kvm); + if (ret) + return ret; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + /* If the entry is already valid, something went wrong */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EINVAL; + } + + ret = vgic_v5_reset_vmte(kvm); + if (ret) + return ret; + + vmi = kzalloc_obj(*vmi); + if (!vmi) { + ret = -ENOMEM; + goto out_fail; + } + + ret = xa_insert(&vm_info, vm_id, vmi, GFP_KERNEL); + if (ret) + goto out_fail; + vmi_inserted = true; + + /* + * If we are restoring the state of a guest, we need to re-inject any + * IRQs that were pending when the state of the guest was originally + * saved. We use the pending_irqs list for this. + */ + INIT_LIST_HEAD(&vmi->pending_irqs); + + /* Allocate and assign the VM Descriptor, if required. */ + if (vmt_info->vmd_size != 0) { + vmd_alloc_size = round_up(vmt_info->vmd_size, + dma_get_cache_alignment()); + vmd = kzalloc(vmd_alloc_size, GFP_KERNEL); + if (!vmd) { + ret = -ENOMEM; + goto out_fail; + } + + /* Stash the VA so we can free it later */ + vmi->vmd_base = vmd; + + tmp = FIELD_PREP(GICV5_VMTEL2E_VMD_ADDR, + virt_to_phys(vmd) >> GICV5_VMTEL2E_VMD_ADDR_SHIFT); + vmte_val0 = tmp; + } + + /* + * Allocate and assign the VPE Table. + * + * First of all, iterate over all vcpus to find the highest VPE ID we + * require - we need to ensure that we have enough storage for all + * vcpu_id values that userspace has picked and not just the total + * number of vcpus. This gives us the number of VPEs required for the + * VM. + * + * Round up the number of VPEs to a whole power of two as we cannot + * describe non-powers-of-two in the VMTE field as it conveys the number + * of ID bits used and not the number of vPEs. IRS_IDR1.IAFFID_BITS is + * encoded as N - 1, so expose at least one VPE ID bit even for a + * single-vCPU VM to keep the views consistent. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + u16 vpe_id = vgic_v5_vpe_id(vcpu); + + if (vpe_id > max_vpe_id) + max_vpe_id = vpe_id; + } + + nr_cpus = max(2UL, roundup_pow_of_two(max_vpe_id + 1)); + vmi->vpe_id_bits = fls(nr_cpus) - 1; + + vpet_alloc_size = round_up((size_t)nr_cpus * sizeof(*vpet), + dma_get_cache_alignment()); + vpet = kzalloc(vpet_alloc_size, GFP_KERNEL); + if (!vpet) { + ret = -ENOMEM; + goto out_fail; + } + + /* Stash the VA so we can free it later */ + vmi->vpet_base = vpet; + + tmp = FIELD_PREP(GICV5_VMTEL2E_VPET_ADDR, + virt_to_phys(vpet) >> GICV5_VMTEL2E_VPET_ADDR_SHIFT); + tmp |= FIELD_PREP(GICV5_VMTEL2E_VPE_ID_BITS, vmi->vpe_id_bits); + vmte_val1 = tmp; + + /* + * Allocate a dense VPED array indexed by vcpu_idx. The VPET is indexed + * by the potentially sparse vcpu_id, but using that ID here would waste + * memory. Given that this is not userspace visible, we can cheat a bit + * and use the dense index instead. This is the ONLY place that we do + * this. + * + * Round the requested size up to a whole cacheline. kzalloc() gurantees + * natural alignment, so we ensure that the cachelines cannot be shared + * with unrelated slab objects. VPED and cacheline sizes are powers of + * two, so this also preserves the required VPED alignment when a VPED + * is larger than a cacheline. + */ + vped_alloc_size = round_up((size_t)nr_vcpus * vmt_info->vped_size, + dma_get_cache_alignment()); + vped_base = kzalloc(vped_alloc_size, GFP_KERNEL); + if (!vped_base) { + ret = -ENOMEM; + goto out_fail; + } + vmi->vped_base = vped_base; + + if (vmd) + vgic_v5_clean_inval(vmd, vmd_alloc_size); + vgic_v5_clean_inval(vpet, vpet_alloc_size); + vgic_v5_clean_inval(vped_base, vped_alloc_size); + + /* Publish the VMTE while serialising access to the shared VMT. */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[0], cpu_to_le64(vmte_val0)); + WRITE_ONCE(vmte->val[1], cpu_to_le64(vmte_val1)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + kvm->arch.vgic.gicv5_vm.vmte_allocated = true; + + return 0; + +out_fail: + /* kfree(NULL) is safe so we can just kfree() at leisure */ + kfree(vmd); + kfree(vpet); + kfree(vped_base); + if (vmi_inserted) + xa_erase(&vm_info, vm_id); + kfree(vmi); + + vgic_v5_reset_vmte(kvm); + + return ret; +} + +/* + * Release the VMT Entry, freeing up any allocated data structures before + * zeroing the VMTE. + * + * The VMTE must be marked as invalid before it is released. + */ +int vgic_v5_vmte_release(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + int ret; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + /* Reject if the VMTE has not been marked as invalid! */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EINVAL; + } + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + goto no_vmi; + + /* Unlikely, but possible. Avoid leaking the memory. */ + vgic_v5_drain_pending_irqs(kvm, vmi, false); + + /* If we have an LPI IST, free it */ + if (vmi->h_lpi_ist) { + ret = vgic_v5_lpi_ist_free(kvm); + if (ret) + return ret; + } + vmi->h_lpi_ist = NULL; + + /* If we have an SPI IST, free it */ + if (vmi->h_spi_ist) { + ret = vgic_v5_spi_ist_free(kvm); + if (ret) + return ret; + } + vmi->h_spi_ist = NULL; + + kfree(vmi->vped_base); + kfree(vmi->vpet_base); + kfree(vmi->vmd_base); + + xa_erase(&vm_info, vm_id); + kfree(vmi); + +no_vmi: + /* + * If we didn't get far enough into allocating a VMTE to create the VM + * info structure, then we just zero the VMTE and move on. There's + * nothing else we can realistically do here. + */ + ret = vgic_v5_reset_vmte(kvm); + if (ret) + return ret; + + kvm->arch.vgic.gicv5_vm.vmte_allocated = false; + + return 0; +} + +/* + * Provide a way for the IRS MMIO emulation to correctly populate the number of + * IAFFID bits (which correspond to our vpe_id_bits. + */ +u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return 0; + + return vmi->vpe_id_bits; +} + +/* Provide the preallocated VPE descriptor to the hardware via the VPE Table. */ +int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + u16 vpe_id = vgic_v5_vpe_id(vcpu); + struct vgic_v5_vm_info *vmi; + vpe_entry *vpet_base; + void *vped; + + /* Make sure we're not over what the hardware supports */ + if (vpe_id >= vmt_info->max_vpes) + return -E2BIG; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vpe_id >= 1 << vmi->vpe_id_bits) + return -E2BIG; + + vpet_base = vmi->vpet_base; + + /* If the VPETE for this CPU is already valid we've gone wrong */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(&vpet_base[vpe_id], sizeof(*vpet_base)); + if (le64_to_cpu(READ_ONCE(vpet_base[vpe_id])) & GICV5_VPE_VALID) + return -EBUSY; + } + + vped = (u8 *)vmi->vped_base + + (size_t)vcpu->vcpu_idx * vmt_info->vped_size; + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + phys_addr_t paddr = virt_to_phys(vped) & GICV5_VPED_ADDR; + WRITE_ONCE(vpet_base[vpe_id], cpu_to_le64(paddr)); + vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); + } + + return 0; +} + +/* Clear the VPE's table entry after the VMTE has been made invalid. */ +int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu) +{ + u32 vm_id = vgic_v5_vm_id(vcpu->kvm); + u16 vpe_id = vgic_v5_vpe_id(vcpu); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + vpe_entry *vpet_base; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[0])) & GICV5_VMTEL2E_VALID) + return -EBUSY; + } + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vpe_id >= 1 << vmi->vpe_id_bits) + return -E2BIG; + + vpet_base = vmi->vpet_base; + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vpet_base[vpe_id], 0ULL); + vgic_v5_clean_inval(vpet_base + vpe_id, sizeof(vpe_entry)); + } + + return 0; +} + +phys_addr_t vgic_v5_get_vmt_base(void) +{ + phys_addr_t vmt_base; + + if (!vmt_info->two_level) + vmt_base = virt_to_phys(vmt_info->linear.vmt_base); + else + vmt_base = virt_to_phys(vmt_info->l2.vmt_base); + + return vmt_base; +} + +/* + * Assign an already allocated IST to the VM by populating the fields in the + * corresponding VMTE. We re-use this code for both an SPI IST and LPI IST, even + * if the paths to reach it might be vastly different. + */ +static int vgic_v5_vmte_assign_ist(struct kvm *kvm, phys_addr_t ist_base, + bool two_level, unsigned int id_bits, + unsigned int l2sz, unsigned int istsz, + bool spi_ist) +{ + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + u32 vm_id = vgic_v5_vm_id(kvm); + enum gicv5_vcpu_cmd cmd; + struct vmtl2_entry *vmte; + unsigned int section; + u64 tmp; + int ret; + + /* + * The L2 VMTE comprises four 64-bit "sections", where sections 2 & 3 + * describe the LPI and SPI ISTs, respectively. Both the LPI and SPI + * sections have the same layout, and as we are either operating on SPIs + * or LPIs we pick a section of the VMTE to modify up-front. + * + * See the GICv5 EAC0 Specification 11.2.2 for more details about the + * VMTE layout. + */ + section = spi_ist ? GICV5_VMTEL2_SPI_SECTION : GICV5_VMTEL2_LPI_SECTION; + + if (ist_base & ~GICV5_VMTEL2E_IST_ADDR) { + pr_err_ratelimited("kvm [%i]: IST misaligned: address 0x%llx, mask 0x%llx\n", + task_pid_nr(current), ist_base, + GICV5_VMTEL2E_IST_ADDR); + return -EINVAL; + } + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + tmp = FIELD_PREP(GICV5_VMTEL2E_IST_L2SZ, l2sz); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ADDR, + ist_base >> GICV5_VMTEL2E_IST_ADDR_SHIFT); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ISTSZ, istsz); + tmp |= FIELD_PREP(GICV5_VMTEL2E_IST_ID_BITS, id_bits); + if (two_level) + tmp |= GICV5_VMTEL2E_IST_STRUCTURE; + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + /* Bail if already allocated */ + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + if (le64_to_cpu(READ_ONCE(vmte->val[section])) & + GICV5_VMTEL2E_IST_VALID) + return -EINVAL; + + WRITE_ONCE(vmte->val[section], cpu_to_le64(tmp)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + /* Finally, mark the entry as valid */ + cmd = spi_ist ? SPI_VIST_MAKE_VALID : LPI_VIST_MAKE_VALID; + ret = irq_set_vcpu_affinity(vgic_v5_vpe_db(vcpu0), &cmd); + if (ret) { + /* + * A timeout does not tell us whether the IRS consumed the + * command, so we're in some indeterminate and potentially + * transient state. It may still make the VIST valid and access + * the backing memory, so leave both the VMTE and allocation + * intact. Killing the VM ensures that teardown is the only path + * that can reclaim them, after it has successfully made the + * complete VMTE invalid. + */ + if (ret == -ETIMEDOUT) { + kvm_vm_dead(kvm); + return ret; + } + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], 0ULL); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return ret; + } + + return 0; +} + +/* + * Allocate a Linear IST - always used for SPIs and potentially LPIs. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. + * + * NOTE: istsz is the FIELD used by GICv5, not the actual size (or log2() of the + * size). + */ +static int vgic_v5_alloc_linear_ist(struct kvm *kvm, bool spi_ist, + unsigned int id_bits, unsigned int istsz) +{ + const size_t n = max(5, id_bits + 1 + istsz); + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + __le64 *ist; + u32 l1sz; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + /* + * Allocate the IST. We only have one level, so we just use the L2 ISTE. + */ + l1sz = BIT(n + 1); + ist = kzalloc(l1sz, GFP_KERNEL_ACCOUNT); + if (!ist) + return -ENOMEM; + + if (spi_ist) { + vmi->h_spi_ist = ist; + } else { + vmi->h_lpi_ist_structure = false; + vmi->h_lpi_ist = ist; + } + + vgic_v5_clean_inval(ist, l1sz); + + return 0; +} + +/* + * Allocate the first level of a two-level IST - LPI, only. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. + * + * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or + * log2() of the sizes). + */ +static int vgic_v5_alloc_l1_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + u32 vm_id = vgic_v5_vm_id(kvm); + const u32 l1_size = BIT(n + 1); + struct vgic_v5_vm_info *vmi; + __le64 *ist; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + ist = kzalloc(l1_size, GFP_KERNEL_ACCOUNT); + if (!ist) + return -ENOMEM; + + vmi->h_lpi_ist_structure = true; + vmi->h_lpi_ist = ist; + + vgic_v5_clean_inval(ist, l1_size); + + return 0; +} + +/* + * Allocate ALL of the second level ISTs for a two-level IST - LPI, only. + * + * The calculation for n has been taken from section 11.2.2 of the GICv5 EAC0 + * spec. The l2_size calculation is from section 11.2.3 of the same document. + * + * NOTE: istsz and l2sz are the FIELDS used by GICv5, not the actual sizes (or + * log2() of the sizes). + */ +static int vgic_v5_alloc_l2_ists(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + const size_t l2_size = BIT(11 + (2 * l2sz) + 1); + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + __le64 *l2ist; + __le64 *l1ist; + int index; + u64 val; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + l1ist = vmi->h_lpi_ist; + + /* + * Allocate the storage for the pointers to the L2 ISTs (used when + * freeing later). + */ + vmi->h_lpi_l2_ists = kzalloc_objs(*vmi->h_lpi_l2_ists, l1_entries, + GFP_KERNEL_ACCOUNT); + if (!vmi->h_lpi_l2_ists) + return -ENOMEM; + + /* Allocate the L2 IST for each L1 IST entry */ + for (index = 0; index < l1_entries; ++index) { + l2ist = kzalloc(l2_size, GFP_KERNEL_ACCOUNT); + if (!l2ist) { + while (--index >= 0) + kfree(vmi->h_lpi_l2_ists[index]); + + kfree(vmi->h_lpi_l2_ists); + vmi->h_lpi_l2_ists = NULL; + + return -ENOMEM; + } + + /* + * We are not doing on-demand allocation of the L2 ISTs, and are + * instead provisioning the whole IST up front. This means that + * we are able to mark the L2 ISTs as valid in the L1 ISTEs as + * the overall IST is not yet valid. + */ + val = (virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK) | + GICV5_ISTL1E_VALID; + l1ist[index] = cpu_to_le64(val); + + vmi->h_lpi_l2_ists[index] = l2ist; + + vgic_v5_clean_inval(l2ist, l2_size); + } + + /* Handle CMOs for the whole L1 IST in one go */ + vgic_v5_clean_inval(l1ist, l1_entries * sizeof(*l1ist)); + + return 0; +} + +/* Allocate a two-level IST - LPIs, only */ +static int vgic_v5_alloc_two_level_lpi_ist(struct kvm *kvm, unsigned int id_bits, + unsigned int istsz, unsigned int l2sz) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + int ret; + + /* + * Allocate the L1 IST first, then all of the L2s. Everything + * is preallocated and we do no on-demand IST allocation. This + * is to avoid needing to track if and when the guest is doing + * on-demand IST allocation. + */ + ret = vgic_v5_alloc_l1_ist(kvm, id_bits, istsz, l2sz); + if (ret) + return ret; + + ret = vgic_v5_alloc_l2_ists(kvm, id_bits, istsz, l2sz); + if (ret) { + /* Free the L1 IST again */ + vmi = xa_load(&vm_info, vm_id); + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; + + return ret; + } + + return 0; +} + +static void vgic_v5_free_allocated_lpi_ist(struct vgic_v5_vm_info *vmi, + unsigned int id_bits, + unsigned int istsz, + unsigned int l2sz) +{ + if (!vmi->h_lpi_ist_structure) { + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; + return; + } + + if (vmi->h_lpi_l2_ists) { + const size_t n = max(5, id_bits - ((10 - istsz) + (2 * l2sz)) + 3 - 1); + const int l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + int index; + + for (index = 0; index < l1_entries; ++index) + kfree(vmi->h_lpi_l2_ists[index]); + + kfree(vmi->h_lpi_l2_ists); + vmi->h_lpi_l2_ists = NULL; + } + + kfree(vmi->h_lpi_ist); + vmi->h_lpi_ist = NULL; +} + +static void vgic_v5_free_allocated_spi_ist(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return; + + kfree(vmi->h_spi_ist); + vmi->h_spi_ist = NULL; +} + +/* + * Free a Linear IST. Can only happen once the VM is dead. + */ +static int vgic_v5_linear_ist_free(struct kvm *kvm, bool spi) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + struct vgic_v5_vm_info *vmi; + int section; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + if (spi) { + section = GICV5_VMTEL2_SPI_SECTION; + vgic_v5_free_allocated_spi_ist(kvm); + } else { + section = GICV5_VMTEL2_LPI_SECTION; + vgic_v5_free_allocated_lpi_ist(vmi, 0, 0, 0); + } + + /* The VM should be dead here, so we can just zero the VMT section */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], cpu_to_le64(0)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* + * Free a Two-Level IST. Can only happen once the VM is dead. + */ +static int vgic_v5_two_level_ist_free(struct kvm *kvm, bool spi) +{ + unsigned int id_bits, istsz, l2sz; + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + struct vmtl2_entry *vmte; + u64 tmp; + int section; + + /* We don't create two-level SPI ISTs, so freeing is a bad idea! */ + if (spi) + return -EINVAL; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + section = GICV5_VMTEL2_LPI_SECTION; + + if (!vmi->h_lpi_ist_structure) + return -EINVAL; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + tmp = le64_to_cpu(READ_ONCE(vmte->val[section])); + } + + id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, tmp); + istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, tmp); + l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, tmp); + + vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); + + /* The VM must be dead, so we can just zero the VMT section */ + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + WRITE_ONCE(vmte->val[section], cpu_to_le64(0)); + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + } + + return 0; +} + +/* Helper to determine ISTE size based on metadata requirements */ +static unsigned int vgic_v5_ist_istsz(unsigned int id_bits) +{ + if (!vgic_v5_irs_istmd(&irs_caps)) + return GICV5_IRS_IST_CFGR_ISTSZ_4; + + if (id_bits >= vgic_v5_irs_istmd_sz(&irs_caps)) + return GICV5_IRS_IST_CFGR_ISTSZ_16; + + return GICV5_IRS_IST_CFGR_ISTSZ_8; +} + +/* + * Allocate an IST for SPIs. + * + * We don't anticipate a large number of SPIs being allocated. Therefore, we + * always allocate a Linear IST for SPIs. This will need to be revisited should + * that assumption no longer hold. + */ +int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + phys_addr_t base_addr; + unsigned int istsz; + int ret; + + lockdep_assert_held(&kvm->arch.config_lock); + + istsz = vgic_v5_ist_istsz(id_bits); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vmi->h_spi_ist) + return -EBUSY; + + ret = vgic_v5_alloc_linear_ist(kvm, true, id_bits, istsz); + if (ret) + return ret; + base_addr = virt_to_phys(vmi->h_spi_ist); + + ret = vgic_v5_vmte_assign_ist(kvm, base_addr, false, id_bits, 0, istsz, + true); + if (ret) { + if (ret != -ETIMEDOUT) + vgic_v5_free_allocated_spi_ist(kvm); + return ret; + } + + return 0; +} + +/* + * Free the IST for SPIs. Should only happen once the VM is dead. + */ +static int vgic_v5_spi_ist_free(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->arch.config_lock); + + return vgic_v5_linear_ist_free(kvm, true); +} + +int vgic_v5_lpi_ist_exists(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + return !!vmi->h_lpi_ist; +} + +/* + * Allocate an IST for LPIs. + * + * Unlike with SPIs, we anticipate that the guest will allocate a relatively + * large number of LPIs. Therefore, while we support doing a linear LPI IST, it + * is expected that LPI ISTs will be two-level. + */ +int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + unsigned int istsz, l2sz; + phys_addr_t phys_addr; + bool two_level; + int ret; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -EINVAL; + + if (vmi->h_lpi_ist) + return -EBUSY; + + istsz = vgic_v5_ist_istsz(id_bits); + l2sz = gicv5_irs_l2_sz(vgic_v5_irs_ist_l2sz(&irs_caps)); + + /* + * Determine if we want to create a Linear or a Two-Level IST. + * + * A two-level IST is only required when a single L2 IST cannot cover + * the requested ID space. This depends on the L2 IST size selected for + * the IRS, not PAGE_SIZE. Using PAGE_SIZE here would switch to + * two-level too early when the selected L2 IST is larger than a page, + * and the allocation sizing arithmetic would underflow. + */ + two_level = vgic_v5_irs_ist_levels(&irs_caps) && + id_bits > ((10 - istsz) + (2 * l2sz)); + + if (!two_level) + ret = vgic_v5_alloc_linear_ist(kvm, false /* LPIs, not SPIs */, + id_bits, istsz); + else + ret = vgic_v5_alloc_two_level_lpi_ist(kvm, id_bits, istsz, + l2sz); + + if (ret) + return ret; + + phys_addr = virt_to_phys(vmi->h_lpi_ist); + ret = vgic_v5_vmte_assign_ist(kvm, phys_addr, two_level, id_bits, l2sz, + istsz, false); + if (ret && ret != -ETIMEDOUT) + vgic_v5_free_allocated_lpi_ist(vmi, id_bits, istsz, l2sz); + + return ret; +} + +/* Free the LPI IST again */ +int vgic_v5_lpi_ist_free(struct kvm *kvm) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + if (!vmi->h_lpi_ist_structure) + return vgic_v5_linear_ist_free(kvm, false); + else + return vgic_v5_two_level_ist_free(kvm, false); +} + +static struct vgic_v5_two_level_ist_shape +vgic_v5_two_level_ist_shape(const struct vgic_v5_ist_desc *ist) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t l2bits, n; + + l2bits = (10 - ist->istsz) + (2 * ist->l2sz); + n = max(2, ist->id_bits - l2bits + 3 - 1); + + shape.l1_entries = BIT(n + 1) / GICV5_IRS_ISTL1E_SIZE; + shape.l2_entries = BIT(l2bits); + + return shape; +} + +static int vgic_v5_read_vm_ist_desc(struct kvm *kvm, unsigned int section, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vmtl2_entry *vmte; + u64 vmte_ist_section; + + vmte = vgic_v5_get_l2_vmte(vm_id); + if (IS_ERR(vmte)) + return PTR_ERR(vmte); + + scoped_guard(raw_spinlock_irqsave, &vgic_v5_irs_lock) { + vgic_v5_clean_inval(vmte, sizeof(*vmte)); + vmte_ist_section = le64_to_cpu(READ_ONCE(vmte->val[section])); + } + + ist->id_bits = FIELD_GET(GICV5_VMTEL2E_IST_ID_BITS, vmte_ist_section); + ist->istsz = FIELD_GET(GICV5_VMTEL2E_IST_ISTSZ, vmte_ist_section); + ist->l2sz = FIELD_GET(GICV5_VMTEL2E_IST_L2SZ, vmte_ist_section); + ist->iste_size = GICV5_ISTE_SIZE(ist->istsz); + + return !!(vmte_ist_section & GICV5_VMTEL2E_IST_VALID); +} + +static int vgic_v5_get_spi_ist_desc(struct kvm *kvm, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + int ret; + + memset(ist, 0, sizeof(*ist)); + + ist->vmi = xa_load(&vm_info, vm_id); + if (!ist->vmi) + return -ENXIO; + + ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_SPI_SECTION, ist); + if (ret < 0) + return ret; + + ist->base = ist->vmi->h_spi_ist; + if (!ret || !ist->base) + return -ENXIO; + + ist->present = true; + return 0; +} + +static int vgic_v5_get_lpi_ist_desc(struct kvm *kvm, + struct vgic_v5_ist_desc *ist) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + bool guest_valid, host_valid; + int ret; + + memset(ist, 0, sizeof(*ist)); + + ist->vmi = xa_load(&vm_info, vm_id); + if (WARN_ON_ONCE(!ist->vmi)) + return -ENXIO; + + ret = vgic_v5_read_vm_ist_desc(kvm, GICV5_VMTEL2_LPI_SECTION, ist); + if (ret < 0) + return ret; + + host_valid = ret; + guest_valid = kvm->arch.vgic.vgic_v5_irs_data->ist_baser.valid; + ist->base = ist->vmi->h_lpi_ist; + + /* If there is no IST to save/restore, return without error. */ + if (!guest_valid && !host_valid && !ist->base) + return 0; + + /* Mismatched combination of valid state */ + if (!guest_valid || !host_valid || !ist->base) + return -ENXIO; + + if (ist->vmi->h_lpi_ist_structure && !ist->vmi->h_lpi_l2_ists) + return -ENXIO; + + ist->present = true; + return 0; +} + +/* + * Save a linear host IST to userspace memory. + * + * Only the architected 32-bit ISTE state is stored. Metadata is skipped when + * striding through the host IST. + */ +static int vgic_v5_save_linear_ist(const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, size_t nr_entries) +{ + size_t index; + int ret; + + vgic_v5_clean_inval(ist->base, + GICV5_LINEAR_IST_SIZE(ist->id_bits, ist->istsz)); + + for (index = 0; index < nr_entries; index++) { + __le32 *h_iste_addr = ist->base + index * ist->iste_size; + u32 h_iste; + + h_iste = le32_to_cpu(READ_ONCE(*h_iste_addr)); + ret = put_user(h_iste, uaddr); + if (ret) + return ret; + + uaddr++; + } + + return 0; +} + +/* + * Save a two-level host IST to userspace memory. + * + * Only the architected 32-bit ISTE state is stored. Metadata is skipped when + * striding through the host IST. + */ +static int vgic_v5_save_two_level_ist(const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t h_l1_index, h_l2_index; + void *h_l2_ist_base; + int ret; + + shape = vgic_v5_two_level_ist_shape(ist); + + vgic_v5_clean_inval(ist->base, + shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist)); + + for (h_l1_index = 0; h_l1_index < shape.l1_entries; h_l1_index++) { + u64 l1_iste; + + /* + * Host L2 ISTs are preallocated. Any invalid L1 entry means the + * host IST state is inconsistent. + */ + l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index])); + if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste)) + return -ENXIO; + + h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index]; + if (!h_l2_ist_base) + return -ENXIO; + + vgic_v5_clean_inval(h_l2_ist_base, + shape.l2_entries * ist->iste_size); + + for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) { + void *h_iste_addr = h_l2_ist_base + h_l2_index * ist->iste_size; + u32 h_iste = le32_to_cpu(*(__le32 *)h_iste_addr); + + ret = put_user(h_iste, uaddr); + if (ret) + return ret; + + uaddr++; + } + } + + return 0; +} + +/* + * Save the SPI IST to userspace-provided memory. + */ +int vgic_v5_save_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + struct vgic_v5_ist_desc ist; + u32 __user *uaddr; + int ret; + + ret = vgic_v5_get_spi_ist_desc(kvm, &ist); + if (ret) + return ret; + + uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr; + + /* The host SPI IST is always linear. */ + return vgic_v5_save_linear_ist(&ist, uaddr, + kvm->arch.vgic.nr_spis); +} + +/* + * Save the LPI IST to userspace memory. + * + * The LPI IST may be linear or two-level, so host iteration depends on the + * allocated host shape. + */ +int vgic_v5_save_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + struct vgic_v5_ist_desc ist; + u32 __user *uaddr; + int ret; + + ret = vgic_v5_get_lpi_ist_desc(kvm, &ist); + if (ret) + return ret; + + if (!ist.present) + return 0; + + /* + * Userspace sized the buffer from the guest-visible configuration. + * Refuse to walk a host IST with a different number of entries. + */ + if (ist.id_bits != + kvm->arch.vgic.vgic_v5_irs_data->ist_cfgr.lpi_id_bits) + return -EINVAL; + + uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr; + + if (!ist.vmi->h_lpi_ist_structure) + return vgic_v5_save_linear_ist(&ist, uaddr, + BIT(ist.id_bits)); + + return vgic_v5_save_two_level_ist(&ist, uaddr); +} + +/* + * Track any SPIs and LPIs that were marked as pending at the point where the + * IST was restored. + * + * Restored pending state is cleared from the host ISTE and replayed with VDPEND + * before the VM first runs. + */ +static int vgic_v5_track_pending_irq(struct list_head *pending_irqs, u32 intid, + u32 type) +{ + struct vgic_v5_pending_irq *pirq; + + pirq = kzalloc_obj(*pirq, GFP_KERNEL); + if (!pirq) + return -ENOMEM; + + /* Encode the interrupt as a GICv5 IntID. */ + pirq->irq = FIELD_PREP(GICV5_HWIRQ_TYPE, type) | + FIELD_PREP(GICV5_HWIRQ_ID, intid); + + INIT_LIST_HEAD(&pirq->next); + list_add_tail(&pirq->next, pending_irqs); + + return 0; +} + +/* + * Process and sanitise each restored ISTE. + * + * HWU is for hardware use and must not survive migration. Pending state is + * tracked, cleared from the ISTE, and replayed before the VM first runs. + */ +static int vgic_v5_process_iste(__le32 *iste, struct list_head *pending_irqs, + u32 intid, u32 type) +{ + u32 iste_data = le32_to_cpu(READ_ONCE(*iste)); + int ret; + + /* Pending state is replayed later with VDPEND. */ + if (iste_data & GICV5_ISTL2E_PENDING) { + ret = vgic_v5_track_pending_irq(pending_irqs, intid, type); + if (ret) + return ret; + } + + iste_data &= ~GICV5_ISTL2E_PENDING; + iste_data &= ~GICV5_ISTL2E_HWU; + + WRITE_ONCE(*iste, cpu_to_le32(iste_data)); + + return 0; +} + +static void vgic_v5_restore_spi_config(struct kvm *kvm, __le32 iste, u32 spi) +{ + u32 iste_data = le32_to_cpu(iste); + bool pending = iste_data & GICV5_ISTL2E_PENDING; + struct vgic_irq *irq; + unsigned long flags; + + irq = vgic_get_irq(kvm, vgic_v5_make_spi(spi)); + if (WARN_ON_ONCE(!irq)) + return; + + raw_spin_lock_irqsave(&irq->irq_lock, flags); + + if (iste_data & GICV5_ISTL2E_HM) + irq->config = VGIC_CONFIG_LEVEL; + else + irq->config = VGIC_CONFIG_EDGE; + + if (irq->config == VGIC_CONFIG_EDGE) + irq->pending_latch = pending; + else if (pending) + irq->pending_latch = true; + else if (!irq->active) + irq->pending_latch = false; + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + vgic_put_irq(kvm, irq); +} + +static int vgic_v5_restore_ist_entry(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + void *h_iste_addr, __le32 h_iste, + u32 intid, u32 intid_type) +{ + __le32 raw_iste = h_iste; + int ret; + + /* + * Sanitise the IST, clearing HWU & pending fields. Pending state is + * later replayed via GIC VDPEND. + */ + ret = vgic_v5_process_iste(&h_iste, &ist->vmi->pending_irqs, + intid, intid_type); + if (ret) + return ret; + + if (intid_type == GICV5_HWIRQ_TYPE_SPI) + vgic_v5_restore_spi_config(kvm, raw_iste, intid); + + /* + * Zero the full ISTE (incl metadata), and write back the non-metadata + * region, only. + */ + memset(h_iste_addr, 0, ist->iste_size); + WRITE_ONCE(*(__le32 *)h_iste_addr, h_iste); + vgic_v5_clean_inval(h_iste_addr, ist->iste_size); + + return 0; +} + +/* + * Restore a userspace IST image to a linear host IST. + * + * The userspace IST image is a linear array of 32-bit ISTEs. + */ +static int vgic_v5_restore_linear_ist(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, size_t nr_entries, + u32 intid_type) +{ + size_t index; + int ret; + + for (index = 0; index < nr_entries; index++) { + void *h_iste_addr = ist->base + index * ist->iste_size; + u32 h_iste; + + ret = get_user(h_iste, uaddr); + if (ret) + return ret; + + ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr, + cpu_to_le32(h_iste), index, intid_type); + if (ret) + return ret; + + uaddr++; + } + + return 0; +} + +/* + * Restore a userspace IST image to a two-level host IST. + * + * The userspace IST image is a linear array of 32-bit ISTEs. + */ +static int vgic_v5_restore_two_level_ist(struct kvm *kvm, + const struct vgic_v5_ist_desc *ist, + u32 __user *uaddr, u32 intid_type) +{ + struct vgic_v5_two_level_ist_shape shape; + size_t h_l1_index, h_l2_index; + void *h_l2_ist_base; + int ret; + + shape = vgic_v5_two_level_ist_shape(ist); + + vgic_v5_clean_inval(ist->vmi->h_lpi_ist, + shape.l1_entries * sizeof(*ist->vmi->h_lpi_ist)); + + for (h_l1_index = 0; h_l1_index < shape.l1_entries; ++h_l1_index) { + u64 l1_iste; + + /* + * Host L2 ISTs are preallocated. Any invalid L1 entry means the + * host IST state is inconsistent. + */ + l1_iste = le64_to_cpu(READ_ONCE(ist->vmi->h_lpi_ist[h_l1_index])); + if (!FIELD_GET(GICV5_ISTL1E_VALID, l1_iste)) + return -ENXIO; + + h_l2_ist_base = ist->vmi->h_lpi_l2_ists[h_l1_index]; + if (!h_l2_ist_base) + return -ENXIO; + + for (h_l2_index = 0; h_l2_index < shape.l2_entries; h_l2_index++) { + void *h_iste_addr = h_l2_ist_base + + h_l2_index * ist->iste_size; + u32 intid = h_l1_index * shape.l2_entries + h_l2_index; + u32 h_iste; + + ret = get_user(h_iste, uaddr); + if (ret) + return ret; + + ret = vgic_v5_restore_ist_entry(kvm, ist, h_iste_addr, + cpu_to_le32(h_iste), intid, + intid_type); + if (ret) + return ret; + + uaddr++; + } + } + + return 0; +} + +/* + * Restore the SPI IST from userspace-provided buffer to the host-allocated IST. + */ +int vgic_v5_restore_spi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + u32 __user *uaddr; + struct vgic_v5_ist_desc ist; + int ret; + + ret = vgic_v5_get_spi_ist_desc(kvm, &ist); + if (ret) + return ret; + + uaddr = (u32 __user *)(unsigned long)ist_attr->spi_ist_addr; + + /* The host SPI IST is always linear. */ + return vgic_v5_restore_linear_ist(kvm, &ist, uaddr, + kvm->arch.vgic.nr_spis, + GICV5_HWIRQ_TYPE_SPI); +} + +/* + * Restore the LPI IST from userspace memory to the host-allocated LPI IST. + * + * The host LPI IST may be linear or two-level, so host iteration depends on the + * host IST's shape. + */ +int vgic_v5_restore_lpi_ist(struct kvm *kvm, struct kvm_vgic_v5_ist *ist_attr) +{ + u32 __user *uaddr; + struct vgic_v5_ist_desc ist; + int ret; + + ret = vgic_v5_get_lpi_ist_desc(kvm, &ist); + if (ret) + return ret; + + if (!ist.present) + return 0; + + uaddr = (u32 __user *)(unsigned long)ist_attr->lpi_ist_addr; + + if (!ist.vmi->h_lpi_ist_structure) + return vgic_v5_restore_linear_ist(kvm, &ist, uaddr, + BIT(ist.id_bits), + GICV5_HWIRQ_TYPE_LPI); + + return vgic_v5_restore_two_level_ist(kvm, &ist, uaddr, + GICV5_HWIRQ_TYPE_LPI); +} + +/* + * Process the pending IRQs removing them from the list and optionally injecting + * them. + */ +static int vgic_v5_process_pending_irqs(struct kvm *kvm, bool inject) +{ + u32 vm_id = vgic_v5_vm_id(kvm); + struct vgic_v5_vm_info *vmi; + + lockdep_assert_held(&kvm->arch.config_lock); + + vmi = xa_load(&vm_info, vm_id); + if (!vmi) + return -ENXIO; + + vgic_v5_drain_pending_irqs(kvm, vmi, inject); + + return 0; +} + +/* Replay pending state that was cleared while restoring guest IST state. */ +int vgic_v5_restore_pending_irqs(struct kvm *kvm) +{ + return vgic_v5_process_pending_irqs(kvm, true); +} + +/* Drop pending state collected by a failed IST restore. */ +void vgic_v5_discard_pending_irqs(struct kvm *kvm) +{ + vgic_v5_process_pending_irqs(kvm, false); +} diff --git a/arch/arm64/kvm/vgic/vgic-v5-tables.h b/arch/arm64/kvm/vgic/vgic-v5-tables.h new file mode 100644 index 000000000000..89096926563f --- /dev/null +++ b/arch/arm64/kvm/vgic/vgic-v5-tables.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025, 2026 Arm Ltd. + */ + +#ifndef __KVM_ARM_VGICV5_TABLES_H__ +#define __KVM_ARM_VGICV5_TABLES_H__ + +#include +#include +#include +#include + +/* Level 1 Virtual Machine Table Entry */ +typedef __le64 vmtl1_entry; + +/* Level 2 Virtual Machine Table Entry */ +struct vmtl2_entry { + __le64 val[4]; +}; + +/* Virtual PE Table Entry */ +typedef __le64 vpe_entry; + +struct vgic_v5_vm_info { + void *vmd_base; + vpe_entry *vpet_base; + void *vped_base; + u8 vpe_id_bits; + + /* + * Both the LPI and SPI ISTs are allocated by the hypervisor. While it + * would be possible to track and access them by iterating over the ISTs + * themselves, it makes more sense to store pointers to the ISTs. + * + * The LPI IST can either be two-level or linear. Hence, we keep track + * of the structure. If it is two-level, we retain pointers to the L1 + * IST and to each L2 IST array. If it is linear, we just store the base + * address of the IST array. + * + * The SPI IST is linear, and therefore we just store the base address + * of the SPI IST array. + */ + bool h_lpi_ist_structure; + __le64 *h_lpi_ist; + __le64 **h_lpi_l2_ists; + __le64 *h_spi_ist; + + /* Tracking of pending interrupts as part of IST restore */ + struct list_head pending_irqs; +}; + +struct vgic_v5_vmt { + union { + struct { + struct vmtl2_entry *vmt_base; + unsigned int num_ents; + } linear; + struct { + vmtl1_entry *vmt_base; + struct vmtl2_entry **l2ptrs; + unsigned int num_l1_ents; + } l2; + }; + bool two_level; + unsigned int num_entries; + unsigned int max_vpes; + size_t vmd_size; + size_t vped_size; + struct ida vm_id_ida; +}; + +static inline u32 vgic_v5_vm_id(struct kvm *kvm) +{ + return kvm->arch.vgic.gicv5_vm.vm_id; +} + +/* + * For vGICv5, we need to consolidate two views: + * - The vcpu ID assigned by userspace, which it can assign however it + * sees fit. + * - The index into the VPET, which is also presented to the guest as + * the IAFFID for the VPE in question. + * + * Ideally, we don't want to intercept the guest setting an interrupt's affinity + * - this would somewhat defeat the point of allowing the hardware to manage the + * interrupt directly. Therefore, we need to use vcpu->vcpu_id as our VPE ID, + * which ensures that both userspace and the GICv5 hardware have a consistent + * view. + * + * This means two things: + * - Our VPET might be sparse, depending on the IDs picked by userspace. + * - We need to check the vcpu ID provided by userspace, and reject anything + * we cannot handle. + * + * In order to at least ensure consistent usage in the places that we need to + * use the VPE ID, we provide this helper. + */ +static inline u16 vgic_v5_vpe_id(struct kvm_vcpu *vcpu) +{ + return vcpu->vcpu_id; +} + +static inline int vgic_v5_vpe_db(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db; +} + +extern raw_spinlock_t vgic_v5_irs_lock; + +int vgic_v5_vmt_allocate(unsigned int max_vpes); +int vgic_v5_vmt_free(void); +phys_addr_t vgic_v5_get_vmt_base(void); + +int vgic_v5_allocate_vm_id(struct kvm *kvm); +void vgic_v5_release_vm_id(struct kvm *kvm); + +int vgic_v5_vmte_init(struct kvm *kvm); +int vgic_v5_vmte_release(struct kvm *kvm); +u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu); +int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu); +int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu); + +int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_exists(struct kvm *kvm); +int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits); +int vgic_v5_lpi_ist_free(struct kvm *kvm); + +int vgic_v5_save_spi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_save_lpi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_spi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_lpi_ist(struct kvm *kvm, + struct kvm_vgic_v5_ist *ist_attr); +int vgic_v5_restore_pending_irqs(struct kvm *kvm); +void vgic_v5_discard_pending_irqs(struct kvm *kvm); + +#endif diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 4e3e4db24e77..91810fbd99af 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -7,10 +7,18 @@ #include #include +#include +#include +#include +#include "vgic-v5-tables.h" #include "vgic.h" #define ppi_caps kvm_vgic_global_state.vgic_v5_ppi_caps +#define irs_caps kvm_vgic_global_state.vgic_v5_irs_caps + +static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, phys_addr_t vmt_base); +static int vgic_v5_irs_clear_vmt(void); /* * Not all PPIs are guaranteed to be implemented for GICv5. Deterermine which @@ -34,6 +42,36 @@ static void vgic_v5_get_implemented_ppis(void) __assign_bit(GICV5_ARCH_PPI_PMUIRQ, ppi_caps.impl_ppi_mask, system_supports_pmuv3()); } +static u32 irs_readl_relaxed(const u32 reg_offset) +{ + return readl_relaxed(irs_caps.irs_base + reg_offset); +} + +static void irs_writel_relaxed(const u32 val, const u32 reg_offset) +{ + writel_relaxed(val, irs_caps.irs_base + reg_offset); +} + +static u64 irs_readq_relaxed(const u32 reg_offset) +{ + return readq_relaxed(irs_caps.irs_base + reg_offset); +} + +static void irs_writeq_relaxed(const u64 val, const u32 reg_offset) +{ + writeq_relaxed(val, irs_caps.irs_base + reg_offset); +} + +static void vgic_v5_irs_cache_id_regs(const struct gic_kvm_info *info) +{ + irs_caps.irs_base = info->gicv5_irs.base; + irs_caps.non_coherent = info->gicv5_irs.non_coherent; + + irs_caps.idr2 = irs_readl_relaxed(GICV5_IRS_IDR2); + irs_caps.idr3 = irs_readl_relaxed(GICV5_IRS_IDR3); + irs_caps.idr4 = irs_readl_relaxed(GICV5_IRS_IDR4); +} + /* * Probe for a vGICv5 compatible interrupt controller, returning 0 on success. */ @@ -43,6 +81,8 @@ int vgic_v5_probe(const struct gic_kvm_info *info) int ret; kvm_vgic_global_state.type = VGIC_V5; + kvm_vgic_global_state.max_gic_vcpus = 0; + kvm_vgic_global_state.max_gicv5_vcpus = 0; kvm_vgic_global_state.vcpu_base = 0; kvm_vgic_global_state.vctrl_base = NULL; @@ -60,17 +100,65 @@ int vgic_v5_probe(const struct gic_kvm_info *info) goto skip_v5; } - kvm_vgic_global_state.max_gic_vcpus = VGIC_V5_MAX_CPUS; + if (!gicv5_global_data.lpi_domain) { + kvm_err("GICv5 LPI domain unavailable\n"); + return -ENODEV; + } + vgic_v5_irs_cache_id_regs(info); vgic_v5_get_implemented_ppis(); + /* + * Even if the HW supports more per-VM vCPUs, artificially cap as we + * can't use them all. + */ + kvm_vgic_global_state.max_gicv5_vcpus = min(vgic_v5_irs_max_vpes(&irs_caps), + VGIC_V5_MAX_CPUS); + + /* + * GICv5 requires a set of tables to be allocated in order to manage + * VMs. We allocate them in advance here, which alas means that we + * already have to make a decisions regarding the maximum number of VMs + * we want to run. For now, we match the maximum number offered by the + * hardware, but this might not be a wise choice in the long term. + */ + ret = vgic_v5_vmt_allocate(kvm_vgic_global_state.max_gicv5_vcpus); + if (ret) { + kvm_err("Failed to allocate the GICv5 VM tables; no GICv5 support\n"); + return -ENODEV; + } + + /* + * We've now allocated the VM table, but the host's IRS doesn't know + * about it yet. Provide the base address of the VMT to the IRS, as well + * as the number of ID bits that it covers and the structure used + * (linear/two-level). + */ + ret = vgic_v5_irs_assign_vmt(vgic_v5_irs_two_level_vmt_support(&irs_caps), + ilog2(vgic_v5_irs_max_vms(&irs_caps)), + vgic_v5_get_vmt_base()); + if (ret) { + kvm_err("Failed to assign the GICv5 VM tables to the IRS; no GICv5 support\n"); + if (!vgic_v5_irs_clear_vmt()) + vgic_v5_vmt_free(); + return -ENODEV; + } + ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V5); if (ret) { kvm_err("Cannot register GICv5 KVM device.\n"); - goto skip_v5; + /* + * Don't free the VMT itself if the hardware still has a valid + * pointer to it. + */ + if (!vgic_v5_irs_clear_vmt()) + vgic_v5_vmt_free(); + return -ENODEV; } v5_registered = true; + kvm_vgic_global_state.max_gic_vcpus = + kvm_vgic_global_state.max_gicv5_vcpus; kvm_info("GCIE system register CPU interface\n"); skip_v5: @@ -92,12 +180,12 @@ int vgic_v5_probe(const struct gic_kvm_info *info) ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3); if (ret) { kvm_err("Cannot register GICv3-legacy KVM device.\n"); - return ret; + /* vGICv5 should still work */ + return v5_registered ? 0 : ret; } - /* We potentially limit the max VCPUs further than we need to here */ - kvm_vgic_global_state.max_gic_vcpus = min(VGIC_V3_MAX_CPUS, - VGIC_V5_MAX_CPUS); + kvm_vgic_global_state.max_gic_vcpus = max(kvm_vgic_global_state.max_gic_vcpus, + VGIC_V3_MAX_CPUS); static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif); kvm_info("GCIE legacy system register CPU interface\n"); @@ -107,6 +195,411 @@ int vgic_v5_probe(const struct gic_kvm_info *info) return 0; } +/* + * Wait for completion of a change in any of IRS_VMT_BASER, IRS_VMAP_L2_VMTR, + * IRS_VMAP_VMR, IRS_VMAP_VPER, IRS_VMAP_VISTR, IRS_VMAP_L2_VISTR. + */ +static int vgic_v5_irs_wait_for_vm_op(void) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_VMT_STATUSR, + GICV5_IRS_VMT_STATUSR_IDLE, + NULL); +} + +/* + * Wait for completion of a change in any of IRS_VPE_SELR, IRS_VPE_DBR, + * IRS_VPE_CR0. + */ +static int vgic_v5_irs_wait_for_vpe_op(void) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_VPE_STATUSR, + GICV5_IRS_VPE_STATUSR_IDLE, + NULL); +} + +/* + * Wait for a write to IRS_SAVE_VMR to complete. + */ +static int vgic_v5_irs_wait_for_save_vm_op(u32 *statusr) +{ + return gicv5_wait_for_op_atomic(irs_caps.irs_base, + GICV5_IRS_SAVE_VM_STATUSR, + GICV5_IRS_SAVE_VM_STATUSR_IDLE, + statusr); +} + +static int vgic_v5_irs_write_vm_mmio_reg(u64 val, u32 offset) +{ + int ret; + + lockdep_assert_held(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + irs_writeq_relaxed(val, offset); + + return vgic_v5_irs_wait_for_vm_op(); +} + +static int vgic_v5_irs_assign_vmt(bool two_level, u8 vm_id_bits, + phys_addr_t vmt_base) +{ + u64 vmt_baser; + u32 vmt_cfgr; + int ret; + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + vmt_baser = irs_readq_relaxed(GICV5_IRS_VMT_BASER); + if (!!FIELD_GET(GICV5_IRS_VMT_BASER_VALID, vmt_baser)) + return -EBUSY; + + vmt_cfgr = FIELD_PREP(GICV5_IRS_VMT_CFGR_VM_ID_BITS, vm_id_bits); + if (two_level) + vmt_cfgr |= FIELD_PREP(GICV5_IRS_VMT_CFGR_STRUCTURE, + GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL); + + irs_writel_relaxed(vmt_cfgr, GICV5_IRS_VMT_CFGR); + + /* The base address is intentionally only masked and not shifted */ + vmt_baser = FIELD_PREP(GICV5_IRS_VMT_BASER_VALID, true) | + (vmt_base & GICV5_IRS_VMT_BASER_ADDR); + irs_writeq_relaxed(vmt_baser, GICV5_IRS_VMT_BASER); + + return vgic_v5_irs_wait_for_vm_op(); +} + +static int vgic_v5_irs_clear_vmt(void) +{ + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + return vgic_v5_irs_write_vm_mmio_reg(0, GICV5_IRS_VMT_BASER); +} + +static int vgic_v5_irs_vmap_l2_vmt(u16 vm_id) +{ + u64 val = FIELD_PREP(GICV5_IRS_VMAP_L2_VMTR_VM_ID, vm_id) | + GICV5_IRS_VMAP_L2_VMTR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_L2_VMTR); +} + +static int __vgic_v5_irs_vmap_vm(u16 vm_id, bool unmap) +{ + u64 val = FIELD_PREP(GICV5_IRS_VMAP_VMR_VM_ID, vm_id) | + FIELD_PREP(GICV5_IRS_VMAP_VMR_U, unmap) | + GICV5_IRS_VMAP_VMR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VMR); +} + +static int vgic_v5_irs_set_vm_valid(u16 vm_id) +{ + return __vgic_v5_irs_vmap_vm(vm_id, false); +} + +static int vgic_v5_irs_set_vm_invalid(u16 vm_id) +{ + return __vgic_v5_irs_vmap_vm(vm_id, true); +} + +static int __vgic_v5_irs_update_vist_validity(u16 vm_id, bool spi_ist, bool unmap) +{ + u8 type = spi_ist ? 0b011 : 0b010; + u64 val = FIELD_PREP(GICV5_IRS_VMAP_VISTR_TYPE, type) | + FIELD_PREP(GICV5_IRS_VMAP_VISTR_VM_ID, vm_id) | + FIELD_PREP(GICV5_IRS_VMAP_VISTR_U, unmap) | + GICV5_IRS_VMAP_VISTR_M; + + return vgic_v5_irs_write_vm_mmio_reg(val, GICV5_IRS_VMAP_VISTR); +} + +static int vgic_v5_irs_set_vist_valid(u16 vm_id, bool spi_ist) +{ + return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, false); +} + +/* + * LPI ISTs can be invalidated explicitly. SPI ISTs are invalidated by making + * the VMTE invalid during teardown. + */ +static int vgic_v5_irs_set_vist_invalid(u16 vm_id, bool spi_ist) +{ + return __vgic_v5_irs_update_vist_validity(vm_id, spi_ist, true); +} + +static int vgic_v5_irs_set_up_vpe(u16 vm_id, u16 vpe_id, + irq_hw_number_t db_hwirq) +{ + u64 vmap_vper, dbr, selr; + u32 statusr, cr0; + int ret; + + lockdep_assert_held(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + /* Mark the VPE as valid */ + vmap_vper = FIELD_PREP(GICV5_IRS_VMAP_VPER_VPE_ID, vpe_id) | + FIELD_PREP(GICV5_IRS_VMAP_VPER_VM_ID, vm_id) | + GICV5_IRS_VMAP_VPER_M; + irs_writeq_relaxed(vmap_vper, GICV5_IRS_VMAP_VPER); + + /* Wait for the VPE to be marked valid in the VPET */ + ret = vgic_v5_irs_wait_for_vm_op(); + if (ret) + return ret; + + selr = FIELD_PREP(GICV5_IRS_VPE_SELR_VPE_ID, vpe_id) | + FIELD_PREP(GICV5_IRS_VPE_SELR_VM_ID, vm_id) | + GICV5_IRS_VPE_SELR_S; + irs_writeq_relaxed(selr, GICV5_IRS_VPE_SELR); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + statusr = irs_readl_relaxed(GICV5_IRS_VPE_STATUSR); + if (!FIELD_GET(GICV5_IRS_VPE_STATUSR_V, statusr)) + return -EINVAL; + + /* Set targeted only routing (disable 1ofN vPE selection) */ + cr0 = GICV5_IRS_VPE_CR0_DPS; + irs_writel_relaxed(cr0, GICV5_IRS_VPE_CR0); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + /* + * The VPE has not yet run. Therefore, make sure that all interrupts + * will generate a doorbell. + */ + dbr = FIELD_PREP(GICV5_IRS_VPE_DBR_INTID, db_hwirq) | + GICV5_IRS_VPE_DBR_DBV; + irs_writeq_relaxed(dbr, GICV5_IRS_VPE_DBR); + + ret = vgic_v5_irs_wait_for_vpe_op(); + if (ret) + return ret; + + return 0; +} + +static int vgic_v5_irs_save_vm_op(u16 vm_id, bool save, u32 *statusr) +{ + u64 save_vmr; + int ret; + + save_vmr = FIELD_PREP(GICV5_IRS_SAVE_VMR_VM_ID, vm_id); + save_vmr |= GICV5_IRS_SAVE_VMR_Q; + save_vmr |= FIELD_PREP(GICV5_IRS_SAVE_VMR_S, save); + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + /* Make sure that we are idle to begin with. */ + ret = vgic_v5_irs_wait_for_save_vm_op(NULL); + if (ret) + return ret; + + irs_writeq_relaxed(save_vmr, GICV5_IRS_SAVE_VMR); + + return vgic_v5_irs_wait_for_save_vm_op(statusr); +} + +static irqreturn_t db_handler(int irq, void *data) +{ + struct kvm_vcpu *vcpu = data; + + WRITE_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired, true); + + kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu); + kvm_vcpu_kick(vcpu); + + return IRQ_HANDLED; +} + +static int vgic_v5_send_command(struct kvm_vcpu *vcpu, enum gicv5_vcpu_cmd cmd) +{ + int irq = vgic_v5_vpe_db(vcpu); + + if (!irq) + return -ENXIO; + + return irq_set_vcpu_affinity(irq, &cmd); +} + +static int vgic_v5_db_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) +{ + struct vgic_v5_vm *vm = data->domain->host_data; + enum gicv5_vcpu_cmd *cmd = vcpu_info; + unsigned int vcpu_idx = data->hwirq; + struct kvm_vcpu *vcpu; + u16 vpe_id; + + guard(raw_spinlock_irqsave)(&vgic_v5_irs_lock); + + switch (*cmd) { + case VMT_L2_MAP: + return vgic_v5_irs_vmap_l2_vmt(vm->vm_id); + case VMTE_MAKE_VALID: + return vgic_v5_irs_set_vm_valid(vm->vm_id); + case VMTE_MAKE_INVALID: + return vgic_v5_irs_set_vm_invalid(vm->vm_id); + case VPE_MAKE_VALID: + /* + * The index in the doorbell domain aligns with our flat + * vcpu_idx index. However, we need the actual VPE ID, which + * means we first need to resolve the actual vcpu. + */ + vcpu = kvm_get_vcpu(vm->kvm, vcpu_idx); + if (!vcpu) + return -EINVAL; + + vpe_id = vgic_v5_vpe_id(vcpu); + + /* + * We need the actual LPI ID which lives in the top-most parent + * domain. This hwirq won't include the type (LPI) but that's + * not required for the IRS_VPE_DBR. + */ + while (data->parent_data) + data = data->parent_data; + return vgic_v5_irs_set_up_vpe(vm->vm_id, vpe_id, data->hwirq); + case SPI_VIST_MAKE_VALID: + return vgic_v5_irs_set_vist_valid(vm->vm_id, true); + case LPI_VIST_MAKE_VALID: + return vgic_v5_irs_set_vist_valid(vm->vm_id, false); + case LPI_VIST_MAKE_INVALID: + return vgic_v5_irs_set_vist_invalid(vm->vm_id, false); + default: + return -EINVAL; + } +} + +/* + * This set of irq_chip functions is specific for doorbells. + */ +static const struct irq_chip vgic_v5_db_irq_chip = { + .name = "GICv5-DB", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .irq_set_vcpu_affinity = vgic_v5_db_set_vcpu_affinity, + .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static void vgic_v5_irq_db_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static int vgic_v5_irq_db_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *arg) +{ + const struct irq_chip *chip = &vgic_v5_db_irq_chip; + struct irq_data *irqd; + int ret; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL); + if (ret) + return ret; + + for (int i = 0; i < nr_irqs; i++) { + irq_domain_set_hwirq_and_chip(domain, virq + i, i, chip, + domain->host_data); + irqd = irq_desc_get_irq_data(irq_to_desc(virq + i)); + irqd_set_single_target(irqd); + } + + return 0; +} + +static const struct irq_domain_ops vgic_v5_irq_db_domain_ops = { + .alloc = vgic_v5_irq_db_domain_alloc, + .free = vgic_v5_irq_db_domain_free, +}; + +static int vgic_v5_create_per_vm_domain(struct kvm *kvm) +{ + struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm; + int nr_vcpus = atomic_read(&kvm->online_vcpus); + int id = task_pid_nr(current); + int ret, db_virq = 0; + + vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv5-vpe-db", id); + if (!vm->fwnode) + return -ENOMEM; + + vm->domain = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain, + 0, nr_vcpus, vm->fwnode, + &vgic_v5_irq_db_domain_ops, vm); + if (!vm->domain) { + ret = -ENOMEM; + goto err; + } + + db_virq = irq_domain_alloc_irqs(vm->domain, nr_vcpus, NUMA_NO_NODE, vm); + if (db_virq <= 0) { + ret = db_virq; + goto err; + } + + kvm->arch.vgic.gicv5_vm.vpe_db_base = db_virq; + + return 0; + +err: + if (vm->domain) + irq_domain_remove(vm->domain); + if (vm->fwnode) + irq_domain_free_fwnode(vm->fwnode); + + kvm->arch.vgic.gicv5_vm.vpe_db_base = 0; + vm->domain = NULL; + vm->fwnode = NULL; + + return ret; +} + +static void vgic_v5_teardown_per_vm_domain(struct vgic_v5_vm *vm) +{ + if (!vm->domain) + return; + + irq_domain_remove(vm->domain); + irq_domain_free_fwnode(vm->fwnode); + vm->domain = NULL; + vm->fwnode = NULL; +} + void vgic_v5_reset(struct kvm_vcpu *vcpu) { /* @@ -120,17 +613,117 @@ void vgic_v5_reset(struct kvm_vcpu *vcpu) * CPUIF (but potentially fewer in the IRS). */ vcpu->arch.vgic_cpu.num_pri_bits = 5; + + /* Make the VPE valid in the VPET */ + if (WARN_ON(vgic_v5_send_command(vcpu, VPE_MAKE_VALID))) + return; } +static void vgic_v5_free_doorbells(struct kvm *kvm, unsigned int nr_dbs) +{ + struct vgic_v5_vm *vm = &kvm->arch.vgic.gicv5_vm; + struct kvm_vcpu *vcpu; + unsigned long i; + int db; + + for (i = 0; i < nr_dbs; i++) { + vcpu = kvm_get_vcpu(kvm, i); + db = vgic_v5_vpe_db(vcpu); + if (!db) + continue; + + free_irq(db, vcpu); + vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = 0; + } + + if (vm->vpe_db_base) { + irq_domain_free_irqs(vm->vpe_db_base, + atomic_read(&kvm->online_vcpus)); + vm->vpe_db_base = 0; + } +} + +void vgic_v5_teardown(struct kvm *kvm) +{ + struct vgic_dist *dist = &kvm->arch.vgic; + struct kvm_vcpu *vcpu, *vcpu0; + bool release_vm_id = true; + unsigned long i; + int rc; + + lockdep_assert_held(&kvm->arch.config_lock); + + /* + * If the VM's ID isn't valid, then we either failed init very early or + * we've been called a second time. Nothing to do here in either case. + */ + if (kvm->arch.vgic.gicv5_vm.vm_id == VGIC_V5_VM_ID_INVAL) + return; + + if (kvm->arch.vgic.gicv5_vm.vmte_allocated) { + /* Make the VM invalid */ + vcpu0 = kvm_get_vcpu(kvm, 0); + rc = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID); + if (rc) { + kvm_err("could not make VMTE invalid\n"); + release_vm_id = false; + goto out_free_doorbells; + } + + kvm_for_each_vcpu(i, vcpu, kvm) { + if (vgic_v5_vmte_free_vpe(vcpu)) { + kvm_err("Failed to free VPE\n"); + release_vm_id = false; + goto out_free_doorbells; + } + } + + if (vgic_v5_vmte_release(kvm)) { + kvm_err("Failed to release VM 0x%x\n", dist->gicv5_vm.vm_id); + release_vm_id = false; + } + } + +out_free_doorbells: + vgic_v5_free_doorbells(kvm, atomic_read(&kvm->online_vcpus)); + vgic_v5_teardown_per_vm_domain(&kvm->arch.vgic.gicv5_vm); + + /* + * We only release the VM ID itself if we didn't fail earlier. It does + * mean that we might lose the VM ID (and associated VMTE, etc), but + * given that we've failed to tear them down correctly there's no way to + * safely reuse them. The VM ID allocating IDA will make sure we don't + * accidentally reuse this partially torn down state. + */ + if (release_vm_id) + vgic_v5_release_vm_id(kvm); +} + +/* + * Claim and populate a VMTE (optionally making a new L2 VMT valid), create VPE + * doorbells, allocate VPET and populate for each VPE. Finally, we also init the + * vIRS, which means allocating and making the virtual SPI IST valid. + * + * Note: We do need to put the cart before the horse here. The VPE doorbells are + * our conduit for communication with the IRS, which means we need to have those + * before making the VMTE valid. + * + * On failure, we clean up in the teardown path (vgic_v5_teardown()). + */ int vgic_v5_init(struct kvm *kvm) { - struct kvm_vcpu *vcpu; - unsigned long idx; + struct kvm_vcpu *vcpu, *vcpu0; + int nr_vcpus, ret = 0; + unsigned int db_virq; + unsigned long i; - if (vgic_initialized(kvm)) - return 0; + lockdep_assert_held(&kvm->arch.config_lock); - kvm_for_each_vcpu(idx, vcpu, kvm) { + nr_vcpus = atomic_read(&kvm->online_vcpus); + if (nr_vcpus == 0) + return -ENODEV; + + kvm_for_each_vcpu(i, vcpu, kvm) { if (vcpu_has_nv(vcpu)) { kvm_err("Nested GICv5 VMs are currently unsupported\n"); return -EINVAL; @@ -145,20 +738,84 @@ int vgic_v5_init(struct kvm *kvm) kvm->arch.vgic.gicv5_vm.userspace_ppis, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + ret = vgic_v5_allocate_vm_id(kvm); + if (ret) + return ret; + + /* + * Stash a backpointer to struct kvm. It is required to resolve the VPE + * ID from the doorbell index, which matches the flat vcpu_idx and not + * the vcpu_id. + */ + kvm->arch.vgic.gicv5_vm.kvm = kvm; + + ret = vgic_v5_create_per_vm_domain(kvm); + if (ret) + goto err; + + db_virq = kvm->arch.vgic.gicv5_vm.vpe_db_base; + kvm_for_each_vcpu(i, vcpu, kvm) { + ret = request_irq(db_virq + i, db_handler, 0, "vcpu", vcpu); + if (ret) + goto err; + + /* Stash it with the VCPU for easy retrieval */ + vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db = db_virq + i; + } + + /* Populate VMTE (with VPET and VM descriptor) */ + ret = vgic_v5_vmte_init(kvm); + if (ret) + goto err; + + /* We pick the first vcpu to make the VMTE valid - any would do */ + vcpu0 = kvm_get_vcpu(kvm, 0); + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID); + if (ret) + goto err; + + /* Populate the VPETE for each VPE. */ + kvm_for_each_vcpu(i, vcpu, kvm) { + ret = vgic_v5_vmte_alloc_vpe(vcpu); + if (ret) + goto err; + } + + /* Init IRS (and alloc SPI IST) */ + ret = kvm_vgic_v5_irs_init(kvm, kvm->arch.vgic.nr_spis); + if (ret) + goto err; + return 0; + +err: + /* + * Explicitly tear everything down on failure. The teardown function is + * written to handle any partial state we might have, so we don't need + * to do any clean-up first. Teardown will be called a second time on VM + * destruction, but that's fine - it is better to leave things in a + * clean state now, and doubly so because userspace could actually go + * and retry init. + */ + vgic_v5_teardown(kvm); + + return ret; } int vgic_v5_map_resources(struct kvm *kvm) { if (!vgic_initialized(kvm)) return -EBUSY; + if (kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending) + return -EINVAL; return 0; } -int vgic_v5_finalize_ppi_state(struct kvm *kvm) +int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu) { - struct kvm_vcpu *vcpu0; + struct kvm *kvm = vcpu->kvm; + struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; int i; if (!vgic_is_v5(kvm)) @@ -167,35 +824,65 @@ int vgic_v5_finalize_ppi_state(struct kvm *kvm) guard(mutex)(&kvm->arch.config_lock); /* - * If SW_PPI has been advertised, then we know we already - * initialised the whole thing, and we can return early. Yes, - * this is pretty hackish as far as state tracking goes... + * Discover the set of PPIs that are exposed to the guest once per VM. + * Once known, apply that mask to each VCPU's restored PPI state as the + * VCPUs are first run. */ - if (test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask)) - return 0; + if (!test_bit(GICV5_ARCH_PPI_SW_PPI, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask)) { + bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, + VGIC_V5_NR_PRIVATE_IRQS); + bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, + VGIC_V5_NR_PRIVATE_IRQS); - /* The PPI state for all VCPUs should be the same. Pick the first. */ - vcpu0 = kvm_get_vcpu(kvm, 0); + for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) { + const u32 intid = vgic_v5_make_ppi(i); + struct vgic_irq *irq; - bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); - bitmap_zero(kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS); + irq = vgic_get_vcpu_irq(vcpu, intid); - for_each_set_bit(i, ppi_caps.impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) { + /* Expose PPIs with an owner or the SW_PPI, only */ + scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { + if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) { + __set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); + __assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, + irq->config == VGIC_CONFIG_LEVEL); + } + } + + vgic_put_irq(kvm, irq); + } + } + + /* + * Apply the mask to Enable, Active. Skip pending as that's calculated + * on guest entry. + */ + bitmap_and(cpu_if->vgic_ppi_enabler, cpu_if->vgic_ppi_enabler, + kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + bitmap_and(cpu_if->vgic_ppi_activer, cpu_if->vgic_ppi_activer, + kvm->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); + + /* Also update the vgic_irqs */ + for (i = 0; i < VGIC_V5_NR_PRIVATE_IRQS; i++) { + bool visible = test_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); const u32 intid = vgic_v5_make_ppi(i); struct vgic_irq *irq; - irq = vgic_get_vcpu_irq(vcpu0, intid); + irq = vgic_get_vcpu_irq(vcpu, intid); - /* Expose PPIs with an owner or the SW_PPI, only */ scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { - if (irq->owner || i == GICV5_ARCH_PPI_SW_PPI) { - __set_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_mask); - __assign_bit(i, kvm->arch.vgic.gicv5_vm.vgic_ppi_hmr, - irq->config == VGIC_CONFIG_LEVEL); + if (!visible) { + irq->enabled = false; + irq->active = false; + irq->pending_latch = false; + irq->line_level = false; + } else { + irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler); + irq->active = test_bit(i, cpu_if->vgic_ppi_activer); } } - vgic_put_irq(vcpu0->kvm, irq); + vgic_put_irq(kvm, irq); } return 0; @@ -436,9 +1123,53 @@ void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu) VGIC_V5_NR_PRIVATE_IRQS); } +static bool vgic_v5_set_spi_pending_state(struct kvm_vcpu *vcpu, + struct vgic_irq *irq) +{ + vgic_v5_set_irq_pend(irq->target_vcpu, irq); + return true; +} + +bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, + struct vgic_irq *irq, + unsigned long flags) + __releases(&irq->irq_lock) +{ + lockdep_assert_held(&irq->irq_lock); + + raw_spin_unlock_irqrestore(&irq->irq_lock, flags); + return true; +} + +static const struct irq_ops vgic_v5_spi_irq_ops = { + .set_pending_state = vgic_v5_set_spi_pending_state, + .queue_irq_unlock = vgic_v5_spi_queue_irq_unlock, +}; + +void vgic_v5_set_spi_ops(struct vgic_irq *irq) +{ + if (WARN_ON(!irq) || WARN_ON(irq->ops)) + return; + + irq->ops = &vgic_v5_spi_irq_ops; +} + +/* Set the pending state for GICv5 SPIs and LPIs */ +void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq) +{ + if (WARN_ON(__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, irq->intid))) + return; + + kvm_call_hyp(__vgic_v5_vdpend, irq->intid, irq_is_pending(irq), + vcpu->kvm->arch.vgic.gicv5_vm.vm_id); +} + void vgic_v5_load(struct kvm_vcpu *vcpu) { + bool irichppidis = !READ_ONCE(vcpu->kvm->arch.vgic.enabled); struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5; + u16 vm = vgic_v5_vm_id(vcpu->kvm); + u16 vpe = vgic_v5_vpe_id(vcpu); /* * On the WFI path, vgic_load is called a second time. The first is when @@ -451,7 +1182,16 @@ void vgic_v5_load(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_restore_vmcr_apr, cpu_if); - cpu_if->gicv5_vpe.resident = true; + cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_V, true) | + FIELD_PREP(ICH_CONTEXTR_EL2_IRICHPPIDIS, irichppidis) | + FIELD_PREP(ICH_CONTEXTR_EL2_VPE, vpe) | + FIELD_PREP(ICH_CONTEXTR_EL2_VM, vm); + + kvm_call_hyp(__vgic_v5_make_resident, cpu_if); + + /* Failed to make the VPE resident? Bang! */ + if (WARN_ON(!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr))) + kvm_vm_dead(vcpu->kvm); } void vgic_v5_put(struct kvm_vcpu *vcpu) @@ -469,7 +1209,58 @@ void vgic_v5_put(struct kvm_vcpu *vcpu) kvm_call_hyp(__vgic_v5_save_apr, cpu_if); - cpu_if->gicv5_vpe.resident = false; + cpu_if->vgic_contextr = 0; + + /* Request a doorbell if entering WFI, unless the IRS is disabled */ + if (vcpu_get_flag(vcpu, IN_WFI) && + READ_ONCE(vcpu->kvm->arch.vgic.enabled)) { + u32 priority_mask; + int dbpm; + + /* + * Find the virtual running priority and use this to calculate + * the doorbell priority mask. We combine the highest active + * priority and the CPU's priority mask. The guest can't handle + * interrupts with priorities less than or equal to the virtual + * running priority, so there's literally no point in waking the + * guest for these. + * + * The priority needs to be higher than the mask to signal, so + * pick the next higher priority (subtract 1). + */ + priority_mask = vgic_v5_get_effective_priority_mask(vcpu); + + /* + * Request a doorbell *unless* the priority is 0, indicating + * that no interrupt can wake the CPU up. + */ + if (priority_mask) { + int db_irq = vgic_v5_vpe_db(vcpu); + struct irq_data *d = irq_get_irq_data(db_irq); + const struct cpumask *aff = irq_data_get_effective_affinity_mask(d); + int cpu = smp_processor_id(); + + dbpm = priority_mask - 1; + cpu_if->vgic_contextr = FIELD_PREP(ICH_CONTEXTR_EL2_DB, 1) | + FIELD_PREP(ICH_CONTEXTR_EL2_DBPM, dbpm); + + /* + * Make the doorbell affine to this CPU, if it isn't + * already. Actively check the cpumask first as it is + * cheaper than changing the affinity every time. + */ + if (!cpumask_test_cpu(cpu, aff)) { + int ret; + + ret = irq_set_affinity(db_irq, cpumask_of(cpu)); + if (ret) + kvm_debug_ratelimited("Failed to move doorbell IRQ %d to CPU %d: %d\n", + db_irq, cpu, ret); + } + } + } + + kvm_call_hyp(__vgic_v5_make_non_resident, cpu_if); /* The shadow priority is only updated on entering WFI */ if (vcpu_get_flag(vcpu, IN_WFI)) @@ -513,3 +1304,300 @@ void vgic_v5_save_state(struct kvm_vcpu *vcpu) __vgic_v5_save_ppi_state(cpu_if); dsb(sy); } + +static int vgic_v5_irs_status_is_quiesced(u32 statusr) +{ + if (statusr & GICV5_IRS_SAVE_VM_STATUSR_Q) + return 0; + + return -EBUSY; +} + +static int vgic_v5_irs_is_quiesced(u16 vm_id) +{ + u32 statusr; + int ret; + + /* + * Ensure that the IST snapshot is complete before asking the IRS + * whether the VM remained quiescent while it was copied. + */ + mb(); + + ret = vgic_v5_irs_save_vm_op(vm_id, false, &statusr); + if (ret) + return ret; + + return vgic_v5_irs_status_is_quiesced(statusr); +} + +static int vgic_v5_copy_ist_attr(struct kvm_device_attr *attr, + struct kvm_vgic_v5_ist *ist_attr) +{ + void __user *uaddr = (void __user *)(unsigned long)attr->addr; + + if (!uaddr) + return -EINVAL; + + if (copy_from_user(ist_attr, uaddr, sizeof(*ist_attr))) + return -EFAULT; + + return 0; +} + +static int vgic_v5_validate_ist_user_buffer(__u64 addr, __u64 size, + size_t expected) +{ + if (!addr || size != expected) + return -EINVAL; + + return 0; +} + +static int vgic_v5_validate_ist_attr(struct kvm *kvm, + const struct kvm_vgic_v5_ist *ist_attr) +{ + unsigned int id_bits; + int ret; + + /* We always have SPIs to save */ + ret = vgic_v5_validate_ist_user_buffer(ist_attr->spi_ist_addr, + ist_attr->spi_ist_size, + kvm->arch.vgic.nr_spis * sizeof(__u32)); + if (ret) + return ret; + + /* We don't always have LPIs to save */ + ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits); + if (ret < 0) + return ret; + + /* No LPI IST */ + if (!ret) { + if (ist_attr->lpi_ist_addr || ist_attr->lpi_ist_size) + return -EINVAL; + + return 0; + } + + return vgic_v5_validate_ist_user_buffer(ist_attr->lpi_ist_addr, + ist_attr->lpi_ist_size, + BIT(id_bits) * sizeof(__u32)); +} + +int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr) +{ + struct kvm_vgic_v5_ist ist_attr; + u16 vm_id = vgic_v5_vm_id(kvm); + u32 statusr; + int ret = 0; + + mutex_lock(&kvm->lock); + + if (kvm_trylock_all_vcpus(kvm)) { + mutex_unlock(&kvm->lock); + return -EBUSY; + } + + mutex_lock(&kvm->arch.config_lock); + + if (!vgic_initialized(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + ret = vgic_v5_copy_ist_attr(attr, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_validate_ist_attr(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_save_vm_op(vm_id, true, &statusr); + if (ret) { + kvm_err("Failed to save GICv5 IRS VM state: %d\n", ret); + goto out_unlock; + } + + ret = vgic_v5_irs_status_is_quiesced(statusr); + if (ret) + goto out_unlock; + + /* Save the SPI IST to the userspace buffer. */ + ret = vgic_v5_save_spi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_is_quiesced(vm_id); + if (ret) + goto out_unlock; + + /* Save the LPI IST to the userspace buffer. */ + ret = vgic_v5_save_lpi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_irs_is_quiesced(vm_id); + if (ret) + goto out_unlock; + +out_unlock: + mutex_unlock(&kvm->arch.config_lock); + kvm_unlock_all_vcpus(kvm); + mutex_unlock(&kvm->lock); + + return ret; +} + +/* Allocate the LPI IST to restore into */ +static int vgic_v5_restore_lpi_ist_alloc(struct kvm *kvm, bool *allocated) +{ + unsigned int id_bits; + int ret; + + *allocated = false; + + ret = vgic_v5_irs_lpi_ist_id_bits(kvm, &id_bits); + if (ret <= 0) + return ret; + + ret = vgic_v5_lpi_ist_alloc(kvm, id_bits); + if (ret) + return ret; + + *allocated = true; + + return 0; +} + +/* + * Clean up the LPI IST if we allocated it, and restore the VMTE to the + * original, valid state. + */ +static void vgic_v5_restore_cleanup(struct kvm *kvm, + struct kvm_vcpu *vcpu, + bool lpi_ist_allocated) +{ + /* + * We are on the restore failure path, so we do a best-effort + * cleanup. These commands might fail, but at this stage this is the + * best we can realistically do. + */ + if (lpi_ist_allocated) { + if (!vgic_v5_send_command(vcpu, VMTE_MAKE_INVALID)) + vgic_v5_lpi_ist_free(kvm); + } + + vgic_v5_send_command(vcpu, VMTE_MAKE_VALID); +} + +int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr) +{ + bool lpi_ist_allocated = false, vmte_invalid = false; + struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0); + struct kvm_vgic_v5_ist ist_attr; + int ret = 0; + + mutex_lock(&kvm->lock); + + if (kvm_trylock_all_vcpus(kvm)) { + mutex_unlock(&kvm->lock); + return -EBUSY; + } + + mutex_lock(&kvm->arch.config_lock); + + if (!vgic_initialized(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + if (kvm_vm_has_ran_once(kvm)) { + ret = -EBUSY; + goto out_unlock; + } + + ret = vgic_v5_copy_ist_attr(attr, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_validate_ist_attr(kvm, &ist_attr); + if (ret) + goto out_unlock; + + ret = vgic_v5_lpi_ist_exists(kvm); + if (ret) { + if (ret > 0) + ret = -EBUSY; + goto out_unlock; + } + + /* + * If the guest has previously allocated an IST (which we check based on + * the IRS_IST_BASER), extract the number of LPI ID bits from the + * IRS_IST_CFGR. Else, do nothing. + * + * We do this before making the VMTE invalid as we rely on + * IRS_VMAP_VISTR to mark the IST as valid in the VMTE. This can only + * happen while the VMTE is valid. + */ + ret = vgic_v5_restore_lpi_ist_alloc(kvm, &lpi_ist_allocated); + if (ret) + goto out_unlock; + + /* + * Host ISTs are updated while the VMTE is invalid, so the GIC cannot + * observe partially restored state. + */ + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_INVALID); + if (ret) { + /* + * If invalidation fails, the restore cannot safely update host + * IST state. + */ + goto out_unlock; + } + vmte_invalid = true; + + /* Restore the SPI IST from the userspace buffer. */ + ret = vgic_v5_restore_spi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + + /* Restore the LPI IST from the userspace buffer. */ + if (lpi_ist_allocated) { + ret = vgic_v5_restore_lpi_ist(kvm, &ist_attr); + if (ret) + goto out_unlock; + } + + /* And make the VM Valid again */ + ret = vgic_v5_send_command(vcpu0, VMTE_MAKE_VALID); + if (ret) + goto out_unlock; + vmte_invalid = false; + + /* + * As part of restoring the ISTs, and previously pending interrupts have + * been tracked and made non-pending. Now that the ISTs have been + * restored, and the VM is valid again, restore the pending interrupts. + */ + ret = vgic_v5_restore_pending_irqs(kvm); + if (ret) + goto out_unlock; + + kvm->arch.vgic.vgic_v5_irs_data->lpi_ist_restore_pending = false; + +out_unlock: + if (ret && (vmte_invalid || lpi_ist_allocated)) { + vgic_v5_discard_pending_irqs(kvm); + vgic_v5_restore_cleanup(kvm, vcpu0, lpi_ist_allocated); + } + + mutex_unlock(&kvm->arch.config_lock); + kvm_unlock_all_vcpus(kvm); + mutex_unlock(&kvm->lock); + + return ret; +} diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 5cf5a1ef86cd..c2726d4c06e6 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -86,20 +86,32 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid) */ struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid) { - /* Non-private IRQs are not yet implemented for GICv5 */ - if (vgic_is_v5(kvm)) - return NULL; + enum kvm_device_type type = kvm->arch.vgic.vgic_model; /* SPIs */ - if (intid >= VGIC_NR_PRIVATE_IRQS && - intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) { - intid -= VGIC_NR_PRIVATE_IRQS; - intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); - return &kvm->arch.vgic.spis[intid]; + if (__irq_is_spi(type, intid)) { + switch (type) { + case KVM_DEV_TYPE_ARM_VGIC_V5: + intid = vgic_v5_get_hwirq_id(intid); + + if (intid >= kvm->arch.vgic.nr_spis) + return NULL; + + intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); + return &kvm->arch.vgic.spis[intid]; + default: { + u32 max_intid = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS; + + if (intid < max_intid) { + intid -= VGIC_NR_PRIVATE_IRQS; + intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis); + return &kvm->arch.vgic.spis[intid]; + } + }} } /* LPIs */ - if (irq_is_lpi(kvm, intid)) + if (__irq_is_lpi(type, intid)) return vgic_get_lpi(kvm, intid); return NULL; @@ -565,6 +577,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, else irq->pending_latch = true; + if (irq->ops && irq->ops->set_pending_state) + WARN_ON_ONCE(!irq->ops->set_pending_state(vcpu, irq)); + vgic_queue_irq_unlock(kvm, irq, flags); vgic_put_irq(kvm, irq); @@ -1247,8 +1262,12 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu) unsigned long flags; struct vgic_vmcr vmcr; - if (vgic_is_v5(vcpu->kvm)) + if (vgic_is_v5(vcpu->kvm)) { + if (READ_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired)) + return true; + return vgic_v5_has_pending_ppi(vcpu); + } if (!vcpu->kvm->arch.vgic.enabled) return false; diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index b71d486ae514..97fc3bdf520e 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -269,6 +269,8 @@ struct ap_list_summary { #define irqs_active_outside_lrs(s) \ ((s)->nr_act && irqs_outside_lrs(s)) +int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, + struct vgic_reg_attr *reg_attr); int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, struct vgic_reg_attr *reg_attr); int vgic_v2_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, @@ -383,8 +385,14 @@ void vgic_debug_destroy(struct kvm *kvm); int vgic_v5_probe(const struct gic_kvm_info *info); void vgic_v5_reset(struct kvm_vcpu *vcpu); int vgic_v5_init(struct kvm *kvm); +int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis); +void vgic_v5_teardown(struct kvm *kvm); int vgic_v5_map_resources(struct kvm *kvm); void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid); +void vgic_v5_set_spi_ops(struct vgic_irq *irq); +bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, + unsigned long flags); +void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq); bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu); void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu); void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu); @@ -394,6 +402,19 @@ void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); void vgic_v5_restore_state(struct kvm_vcpu *vcpu); void vgic_v5_save_state(struct kvm_vcpu *vcpu); +int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address); +int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits); + +int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr, bool is_write); +int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); +const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz); +int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr); +int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr); +int vgic_v5_irs_attr_regs_access(struct kvm_device *dev, + struct kvm_device_attr *attr, + u64 *reg, bool is_write); +int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr); #define for_each_visible_v5_ppi(__i, __k) \ for_each_set_bit(__i, (__k)->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index b15a59b9e95d..997c62ab72e6 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -401,6 +401,57 @@ Res0 3:1 Field 0 SO EndSysreg +SysregFields SPSR_ELx +Res0 63:37 +Field 36 UINJ +Field 35 PACM +Field 34 EXLOCK +Res0 33 +Field 32 PM +Field 31 N +Field 30 Z +Field 29 C +Field 28 V +Res0 27:26 +Field 25 TCO +Field 24 DIT +Field 23 UAO +Field 22 PAN +Field 21 SS +Field 20 IL +Res0 19:14 +Field 13 ALLINT +Field 12 SSBS +Field 11:10 BTYPE +Field 9 D +Field 8 A +Field 7 I +Field 6 F +Res0 5 +Field 4 M4 +Enum 3:0 M + 0b0000 EL0 + 0b0100 EL1t + 0b0101 EL1h + 0b1000 EL2t + 0b1001 EL2h + 0b1100 EL3t + 0b1101 EL3h +EndEnum +EndSysregFields + +Sysreg SPSR_EL1 3 0 4 0 0 +Fields SPSR_ELx +EndSysreg + +Sysreg SPSR_EL2 3 4 4 0 0 +Fields SPSR_ELx +EndSysreg + +Sysreg SPSR_EL12 3 5 4 0 0 +Mapping SPSR_EL1 +EndSysreg + Sysreg ID_PFR0_EL1 3 0 0 1 0 Res0 63:32 UnsignedEnum 31:28 RAS diff --git a/arch/loongarch/include/asm/kvm_dmsintc.h b/arch/loongarch/include/asm/kvm_dmsintc.h index 7c0158764d84..e19b3f5b3c52 100644 --- a/arch/loongarch/include/asm/kvm_dmsintc.h +++ b/arch/loongarch/include/asm/kvm_dmsintc.h @@ -22,7 +22,6 @@ struct dmsintc_state { int kvm_loongarch_register_dmsintc_device(void); void kvm_loongarch_unregister_dmsintc_device(void); void dmsintc_inject_irq(struct kvm_vcpu *vcpu); -int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level); -int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 vector, int level); +int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data); #endif diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 65d91c3ce313..b0692f89b243 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -26,6 +26,8 @@ #include #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM /* Loongarch KVM register ids */ #define KVM_GET_IOC_CSR_IDX(id) ((id & KVM_CSR_IDX_MASK) >> LOONGARCH_REG_SHIFT) @@ -79,6 +81,7 @@ struct kvm_arch_memory_slot { #define HOST_MAX_PMNUM 16 struct kvm_context { unsigned long vpid_cache; + unsigned long vmid_cache; struct kvm_vcpu *last_vcpu; /* Host PMU CSR */ u64 perf_ctrl[HOST_MAX_PMNUM]; @@ -133,6 +136,8 @@ struct kvm_arch { unsigned long kvm_features; s64 time_offset; + cpumask_t tlb_flush_pending; + unsigned long vmid[NR_CPUS]; struct kvm_context __percpu *vmcs; struct loongarch_ipi *ipi; struct loongarch_dmsintc *dmsintc; @@ -164,11 +169,11 @@ enum emulation_result { #define KVM_LARCH_SWCSR_LATEST (0x1 << 3) #define KVM_LARCH_HWCSR_USABLE (0x1 << 4) -#define LOONGARCH_PV_FEAT_UPDATED BIT_ULL(63) #define LOONGARCH_PV_FEAT_MASK (BIT(KVM_FEATURE_IPI) | \ BIT(KVM_FEATURE_PREEMPT) | \ BIT(KVM_FEATURE_STEAL_TIME) | \ BIT(KVM_FEATURE_USER_HCALL) | \ + BIT(KVM_FEATURE_CRASH) | \ BIT(KVM_FEATURE_VIRT_EXTIOI)) struct kvm_vcpu_arch { @@ -233,14 +238,16 @@ struct kvm_vcpu_arch { /* Cache for pages needed inside spinlock regions */ struct kvm_mmu_memory_cache mmu_page_cache; - /* vcpu's vpid */ - u64 vpid; + /* vCPU's vpid & tgid */ + unsigned long vpid, tgid; gpa_t flush_gpa; /* Frequency of stable timer in Hz */ u64 timer_mhz; ktime_t expire; + /* vCPU has run at least once */ + bool has_run; /* Last CPU the vCPU state was loaded on */ int last_sched_cpu; /* mp state */ @@ -318,6 +325,8 @@ int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu); /* MMU handling */ void kvm_flush_tlb_all(void); +void kvm_flush_tlb_all_stage1(void); +void kvm_flush_tlb_all_stage2(void); void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa); int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long badv, bool write, int ecode); @@ -349,6 +358,8 @@ static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS +int kvm_arch_flush_remote_tlbs(struct kvm *kvm); void kvm_check_vpid(struct kvm_vcpu *vcpu); enum hrtimer_restart kvm_swtimer_wakeup(struct hrtimer *timer); void kvm_init_vmcs(struct kvm *kvm); diff --git a/arch/loongarch/include/asm/kvm_para.h b/arch/loongarch/include/asm/kvm_para.h index fb17ba0fa101..2f4b9ccf2afb 100644 --- a/arch/loongarch/include/asm/kvm_para.h +++ b/arch/loongarch/include/asm/kvm_para.h @@ -14,6 +14,7 @@ #define KVM_HCALL_CODE_SERVICE 0 #define KVM_HCALL_CODE_SWDBG 1 #define KVM_HCALL_CODE_USER_SERVICE 2 +#define KVM_HCALL_CODE_CRASH 3 #define KVM_HCALL_SERVICE HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_SERVICE) #define KVM_HCALL_FUNC_IPI 1 @@ -23,6 +24,8 @@ #define KVM_HCALL_USER_SERVICE HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_USER_SERVICE) +#define KVM_HCALL_CRASH HYPERCALL_ENCODE(HYPERVISOR_KVM, KVM_HCALL_CODE_CRASH) + /* * LoongArch hypercall return code */ diff --git a/arch/loongarch/include/asm/kvm_pch_pic.h b/arch/loongarch/include/asm/kvm_pch_pic.h index 887b0431fd20..19ed1494c114 100644 --- a/arch/loongarch/include/asm/kvm_pch_pic.h +++ b/arch/loongarch/include/asm/kvm_pch_pic.h @@ -72,6 +72,6 @@ struct kvm_kernel_irq_routing_entry; int kvm_loongarch_register_pch_pic_device(void); void kvm_loongarch_unregister_pch_pic_device(void); void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level); -int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level); +int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e); #endif /* __ASM_KVM_PCH_PIC_H */ diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index 2a6bc99177d8..88e3bdfd4971 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -138,6 +138,9 @@ #define CPUCFG6_PMBITS_SHIFT 8 #define CPUCFG6_UPM BIT(14) +#define LOONGARCH_CPUCFG8 0x8 +#define CPUCFG8_S2TGID BIT(8) + #define LOONGARCH_CPUCFG16 0x10 #define CPUCFG16_L1_IUPRE BIT(0) #define CPUCFG16_L1_IUUNIFY BIT(1) @@ -350,6 +353,8 @@ #define CSR_GTLBC_TGID_WIDTH 8 #define CSR_GTLBC_TGID_SHIFT_END (CSR_GTLBC_TGID_SHIFT + CSR_GTLBC_TGID_WIDTH - 1) #define CSR_GTLBC_TGID (_ULCAST_(0xff) << CSR_GTLBC_TGID_SHIFT) +#define CSR_GTLBC_USES2TGID_SHIFT 14 +#define CSR_GTLBC_USES2TGID (_ULCAST_(0x1) << CSR_GTLBC_USES2TGID_SHIFT) #define CSR_GTLBC_TOTI_SHIFT 13 #define CSR_GTLBC_TOTI (_ULCAST_(0x1) << CSR_GTLBC_TOTI_SHIFT) #define CSR_GTLBC_USETGID_SHIFT 12 diff --git a/arch/loongarch/include/uapi/asm/kvm.h b/arch/loongarch/include/uapi/asm/kvm.h index cd0b5c11ca9c..b222bfc48042 100644 --- a/arch/loongarch/include/uapi/asm/kvm.h +++ b/arch/loongarch/include/uapi/asm/kvm.h @@ -106,6 +106,7 @@ struct kvm_fpu { #define KVM_LOONGARCH_VM_FEAT_PTW 8 #define KVM_LOONGARCH_VM_FEAT_MSGINT 9 #define KVM_LOONGARCH_VM_FEAT_PV_PREEMPT 10 +#define KVM_LOONGARCH_VM_FEAT_CRASH 11 /* Device Control API on vcpu fd */ #define KVM_LOONGARCH_VCPU_CPUCFG 0 diff --git a/arch/loongarch/include/uapi/asm/kvm_para.h b/arch/loongarch/include/uapi/asm/kvm_para.h index d28cbcadd276..bf1a0c826ca6 100644 --- a/arch/loongarch/include/uapi/asm/kvm_para.h +++ b/arch/loongarch/include/uapi/asm/kvm_para.h @@ -16,6 +16,7 @@ #define KVM_FEATURE_IPI 1 #define KVM_FEATURE_STEAL_TIME 2 #define KVM_FEATURE_PREEMPT 3 +#define KVM_FEATURE_CRASH 4 /* BIT 24 - 31 are features configurable by user space vmm */ #define KVM_FEATURE_VIRT_EXTIOI 24 #define KVM_FEATURE_USER_HCALL 25 diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c index 1b861cbc5e10..7afeff852bfe 100644 --- a/arch/loongarch/kernel/asm-offsets.c +++ b/arch/loongarch/kernel/asm-offsets.c @@ -301,6 +301,7 @@ static void __used output_kvm_defines(void) OFFSET(KVM_ARCH_HTP, kvm_vcpu_arch, host_tp); OFFSET(KVM_ARCH_HPGD, kvm_vcpu_arch, host_pgd); OFFSET(KVM_ARCH_KVMPGD, kvm_vcpu_arch, kvm_pgd); + OFFSET(KVM_ARCH_TGID, kvm_vcpu_arch, tgid); OFFSET(KVM_ARCH_HANDLE_EXIT, kvm_vcpu_arch, handle_exit); OFFSET(KVM_ARCH_HEENTRY, kvm_vcpu_arch, host_eentry); OFFSET(KVM_ARCH_GEENTRY, kvm_vcpu_arch, guest_eentry); diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 74d31f260dfd..7121e7483498 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c @@ -241,6 +241,10 @@ static void cpu_probe_common(struct cpuinfo_loongarch *c) if (config & CPUCFG6_PMP) c->options |= LOONGARCH_CPU_PMP; + config = read_cpucfg(LOONGARCH_CPUCFG8); + if (config & CPUCFG8_S2TGID) + c->options |= LOONGARCH_CPU_GUESTID; + config = csr_read32(LOONGARCH_CSR_ASID); config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT; asid_mask = GENMASK(config - 1, 0); diff --git a/arch/loongarch/kvm/Kconfig b/arch/loongarch/kvm/Kconfig index 15da2d88c0c1..fd387326d2f2 100644 --- a/arch/loongarch/kvm/Kconfig +++ b/arch/loongarch/kvm/Kconfig @@ -23,12 +23,9 @@ config KVM select HAVE_KVM_DIRTY_RING_ACQ_REL select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_IRQCHIP - select HAVE_KVM_MSI - select HAVE_KVM_READONLY_MEM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING - select KVM_MMIO select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index 4f58e6e2cf86..3b6daa946c76 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -911,6 +911,17 @@ static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode) vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE; ret = RESUME_HOST; break; + case KVM_HCALL_CRASH: + if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_CRASH)) { + kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE); + break; + } + + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_CRASH; + vcpu->run->system_event.ndata = 0; + ret = RESUME_HOST; + break; case KVM_HCALL_SWDBG: /* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */ if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) { diff --git a/arch/loongarch/kvm/intc/dmsintc.c b/arch/loongarch/kvm/intc/dmsintc.c index 89f980d867be..ff327b4c05f7 100644 --- a/arch/loongarch/kvm/intc/dmsintc.c +++ b/arch/loongarch/kvm/intc/dmsintc.c @@ -14,9 +14,6 @@ void dmsintc_inject_irq(struct kvm_vcpu *vcpu) unsigned long vector[4], old; struct dmsintc_state *ds = &vcpu->arch.dmsintc_state; - if (!ds) - return; - for (i = 0; i < 4; i++) { old = atomic64_read(&(ds->vector_map[i])); vector[i] = old ? atomic64_xchg(&(ds->vector_map[i]), 0) : 0; @@ -43,42 +40,25 @@ void dmsintc_inject_irq(struct kvm_vcpu *vcpu) } } -int dmsintc_deliver_msi_to_vcpu(struct kvm *kvm, - struct kvm_vcpu *vcpu, u32 vector, int level) -{ - struct dmsintc_state *ds = &vcpu->arch.dmsintc_state; - - if (!level) - return 0; - if (!vcpu || vector >= 256) - return -EINVAL; - if (!ds) - return -ENODEV; - - if (!kvm_guest_has_msgint(&vcpu->arch)) - return -EINVAL; - - set_bit(vector, (unsigned long *)&ds->vector_map); - kvm_queue_irq(vcpu, INT_AVEC); - kvm_vcpu_kick(vcpu); - - return 0; -} - -int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data, int level) +int dmsintc_set_irq(struct kvm *kvm, u64 addr, int data) { unsigned int irq, cpu; struct kvm_vcpu *vcpu; irq = (addr >> AVEC_IRQ_SHIFT) & AVEC_IRQ_MASK; cpu = (addr >> AVEC_CPU_SHIFT) & kvm->arch.dmsintc->cpu_mask; - if (cpu >= KVM_MAX_VCPUS) - return -EINVAL; vcpu = kvm_get_vcpu_by_cpuid(kvm, cpu); if (!vcpu) return -EINVAL; - return dmsintc_deliver_msi_to_vcpu(kvm, vcpu, irq, level); + if (!kvm_guest_has_msgint(&vcpu->arch)) + return -EINVAL; + + set_bit(irq, (unsigned long *)&vcpu->arch.dmsintc_state.vector_map); + kvm_queue_irq(vcpu, INT_AVEC); + kvm_vcpu_kick(vcpu); + + return 0; } static int kvm_dmsintc_ctrl_access(struct kvm_device *dev, diff --git a/arch/loongarch/kvm/intc/ipi.c b/arch/loongarch/kvm/intc/ipi.c index 7b333a4a0430..80faa770c0cd 100644 --- a/arch/loongarch/kvm/intc/ipi.c +++ b/arch/loongarch/kvm/intc/ipi.c @@ -423,15 +423,12 @@ static int kvm_ipi_create(struct kvm_device *dev, u32 type) mutex_unlock(&kvm->slots_lock); if (ret < 0) { kvm_pr_unimpl("%s: Initialize IOCSR dev failed, ret = %d\n", __func__, ret); - goto err; + kfree(s); + return ret; } kvm->arch.ipi = s; return 0; - -err: - kfree(s); - return -EFAULT; } static void kvm_ipi_destroy(struct kvm_device *dev) diff --git a/arch/loongarch/kvm/intc/pch_pic.c b/arch/loongarch/kvm/intc/pch_pic.c index 2b63b0c2c7ce..220fc23ffe8c 100644 --- a/arch/loongarch/kvm/intc/pch_pic.c +++ b/arch/loongarch/kvm/intc/pch_pic.c @@ -68,20 +68,20 @@ void pch_pic_set_irq(struct loongarch_pch_pic *s, int irq, int level) } /* msi irq handler */ -int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, int level) +int pch_msi_set_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e) { u64 msg_addr = (((u64)e->msi.address_hi) << 32) | e->msi.address_lo; if (cpu_has_msgint && kvm->arch.dmsintc && msg_addr >= kvm->arch.dmsintc->msg_addr_base && msg_addr < (kvm->arch.dmsintc->msg_addr_base + kvm->arch.dmsintc->msg_addr_size)) { - return dmsintc_set_irq(kvm, msg_addr, e->msi.data, level); + return dmsintc_set_irq(kvm, msg_addr, e->msi.data); } if (e->msi.data >= EIOINTC_IRQS) return -EINVAL; - eiointc_set_irq(kvm->arch.eiointc, e->msi.data, level); + eiointc_set_irq(kvm->arch.eiointc, e->msi.data, 1); return 0; } diff --git a/arch/loongarch/kvm/irqfd.c b/arch/loongarch/kvm/irqfd.c index 40ed1081c4b6..2e9c32023752 100644 --- a/arch/loongarch/kvm/irqfd.c +++ b/arch/loongarch/kvm/irqfd.c @@ -29,7 +29,7 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, if (!level) return -1; - return pch_msi_set_irq(kvm, e, level); + return pch_msi_set_irq(kvm, e); } /* @@ -78,7 +78,7 @@ int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, pch_pic_set_irq(kvm->arch.pch_pic, e->irqchip.pin, level); return 0; case KVM_IRQ_ROUTING_MSI: - return pch_msi_set_irq(kvm, e, level); + return pch_msi_set_irq(kvm, e); default: return -EWOULDBLOCK; } diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 236523d2449d..c65ae2fa4c4b 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -218,14 +218,17 @@ static void kvm_update_vpid(struct kvm_vcpu *vcpu, int cpu) ++vpid; /* vpid 0 reserved for root */ /* start new vpid cycle */ - kvm_flush_tlb_all(); + if (!cpu_has_guestid) + kvm_flush_tlb_all(); + else + kvm_flush_tlb_all_stage1(); } context->vpid_cache = vpid; vcpu->arch.vpid = vpid; } -void kvm_check_vpid(struct kvm_vcpu *vcpu) +static void __kvm_check_vpid(struct kvm_vcpu *vcpu) { int cpu; bool migrated; @@ -253,7 +256,6 @@ void kvm_check_vpid(struct kvm_vcpu *vcpu) kvm_update_vpid(vcpu, cpu); trace_kvm_vpid_change(vcpu, vcpu->arch.vpid); vcpu->cpu = cpu; - kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); /* * LLBCTL is a separated guest CSR register from host, a general @@ -279,6 +281,62 @@ void kvm_check_vpid(struct kvm_vcpu *vcpu) } } +static void kvm_update_vmid(struct kvm_vcpu *vcpu, int cpu) +{ + unsigned long vmid; + struct kvm_context *context; + + context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); + vmid = context->vmid_cache + 1; + if (!(vmid & vpid_mask)) { + /* finish round of vmid loop */ + if (unlikely(!vmid)) + vmid = vpid_mask + 1; + + ++vmid; /* vmid 0 reserved for root */ + + /* start new vmid cycle */ + kvm_flush_tlb_all_stage2(); + } + + context->vmid_cache = vmid; + vcpu->kvm->arch.vmid[cpu] = vmid; +} + +static void __kvm_check_vmid(struct kvm_vcpu *vcpu) +{ + int cpu; + unsigned long ver, old, vmid; + + /* On CPUs like Loongson-3A5000, vmid must be the same as vpid */ + if (!cpu_has_guestid) { + vmid = vcpu->arch.vpid & vpid_mask; + if (vcpu->arch.tgid != vmid) { + vcpu->arch.tgid = vcpu->arch.vpid & vpid_mask; + kvm_clear_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + } + return; + } + + cpu = smp_processor_id(); + if (cpumask_test_and_clear_cpu(cpu, &vcpu->kvm->arch.tlb_flush_pending)) + vcpu->kvm->arch.vmid[cpu] = 0; + + /* Check if our vmid is of an older version */ + ver = vcpu->kvm->arch.vmid[cpu] & ~vpid_mask; + old = this_cpu_ptr(vcpu->kvm->arch.vmcs)->vmid_cache & ~vpid_mask; + if (ver != old) + kvm_update_vmid(vcpu, cpu); + + vcpu->arch.tgid = vcpu->kvm->arch.vmid[cpu] & vpid_mask; +} + +void kvm_check_vpid(struct kvm_vcpu *vcpu) +{ + __kvm_check_vpid(vcpu); + __kvm_check_vmid(vcpu); +} + void kvm_init_vmcs(struct kvm *kvm) { kvm->arch.vmcs = vmcs; @@ -300,7 +358,7 @@ int kvm_arch_enable_virtualization_cpu(void) write_csr_gcfg(0); write_csr_gstat(0); write_csr_gintc(0); - clear_csr_gtlbc(CSR_GTLBC_USETGID | CSR_GTLBC_TOTI); + clear_csr_gtlbc(CSR_GTLBC_TOTI | CSR_GTLBC_USETGID | CSR_GTLBC_USES2TGID); /* * Enable virtualization features granting guest direct control of @@ -323,6 +381,8 @@ int kvm_arch_enable_virtualization_cpu(void) /* Enable using TGID */ set_csr_gtlbc(CSR_GTLBC_USETGID); + if (cpu_has_guestid) + set_csr_gtlbc(CSR_GTLBC_USES2TGID); kvm_debug("GCFG:%lx GSTAT:%lx GINTC:%lx GTLBC:%lx", read_csr_gcfg(), read_csr_gstat(), read_csr_gintc(), read_csr_gtlbc()); @@ -341,7 +401,7 @@ void kvm_arch_disable_virtualization_cpu(void) write_csr_gcfg(0); write_csr_gstat(0); write_csr_gintc(0); - clear_csr_gtlbc(CSR_GTLBC_USETGID | CSR_GTLBC_TOTI); + clear_csr_gtlbc(CSR_GTLBC_TOTI | CSR_GTLBC_USETGID | CSR_GTLBC_USES2TGID); /* Flush any remaining guest TLB entries */ kvm_flush_tlb_all(); @@ -376,6 +436,7 @@ static int kvm_loongarch_env_init(void) for_each_possible_cpu(cpu) { context = per_cpu_ptr(vmcs, cpu); context->vpid_cache = vpid_mask + 1; + context->vmid_cache = vpid_mask + 1; context->last_vcpu = NULL; } diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c index 3e9a0b285fd2..64fddd9f0ada 100644 --- a/arch/loongarch/kvm/mmu.c +++ b/arch/loongarch/kvm/mmu.c @@ -903,7 +903,12 @@ static int kvm_map_page(struct kvm_vcpu *vcpu, unsigned long gpa, bool write) * there is invalid tlb with small page * need flush these invalid tlbs for current vcpu */ - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + if (!cpu_has_ptw) { + if (!cpu_has_guestid) + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + else + cpumask_set_cpu(vcpu->cpu, &vcpu->kvm->arch.tlb_flush_pending); + } ++kvm->stat.hugepages; } else if (kvm_pte_huge(*ptep) && write) ptep = kvm_split_huge(vcpu, ptep, gfn); @@ -939,8 +944,11 @@ int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long gpa, bool write, in * * With SW PTW, invalid TLB is added in TLB refill exception. */ - vcpu->arch.flush_gpa = gpa; - kvm_make_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + if (!cpu_has_guestid) { + vcpu->arch.flush_gpa = gpa; + kvm_make_request(KVM_REQ_TLB_FLUSH_GPA, vcpu); + } else + cpumask_set_cpu(vcpu->cpu, &vcpu->kvm->arch.tlb_flush_pending); } return 0; @@ -949,3 +957,14 @@ int kvm_handle_mm_fault(struct kvm_vcpu *vcpu, unsigned long gpa, bool write, in void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { } + +int kvm_arch_flush_remote_tlbs(struct kvm *kvm) +{ + /* Queue a TLB flush for each CPU to perform on next vCPU loading */ + if (cpu_has_guestid) { + cpumask_setall(&kvm->arch.tlb_flush_pending); + smp_wmb(); /* Be sure that other CPUS can watch the changes */ + } + + return 1; /* Return 1 continue to send IPI to running vCPUs */ +} diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S index 936e4ae3e408..e17e9edb5dd4 100644 --- a/arch/loongarch/kvm/switch.S +++ b/arch/loongarch/kvm/switch.S @@ -66,9 +66,8 @@ ld.d t0, a2, KVM_ARCH_KVMPGD csrwr t0, LOONGARCH_CSR_PGDL - /* Mix GID and RID */ - csrrd t1, LOONGARCH_CSR_GSTAT - bstrpick.w t1, t1, CSR_GSTAT_GID_SHIFT_END, CSR_GSTAT_GID_SHIFT + /* Set VMID for gpa --> hpa mapping */ + ld.d t1, a2, KVM_ARCH_TGID csrrd t0, LOONGARCH_CSR_GTLBC bstrins.w t0, t1, CSR_GTLBC_TGID_SHIFT_END, CSR_GTLBC_TGID_SHIFT csrwr t0, LOONGARCH_CSR_GTLBC diff --git a/arch/loongarch/kvm/tlb.c b/arch/loongarch/kvm/tlb.c index ebdbe9264e9c..994216c98ea2 100644 --- a/arch/loongarch/kvm/tlb.c +++ b/arch/loongarch/kvm/tlb.c @@ -21,9 +21,26 @@ void kvm_flush_tlb_all(void) local_irq_restore(flags); } -void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) +/* Invalidate all stage1 TLB entries including GVA-->GPA mappings */ +void kvm_flush_tlb_all_stage1(void) { lockdep_assert_irqs_disabled(); - gpa &= (PAGE_MASK << 1); - invtlb(INVTLB_GID_ADDR, read_csr_gstat() & CSR_GSTAT_GID, gpa); + invtlb_all(INVGTLB_ALLGID_GVA_TO_GPA, 0, 0); +} + +/* Invalidate all stage2 TLB entries including GPA-->HPA mappings */ +void kvm_flush_tlb_all_stage2(void) +{ + lockdep_assert_irqs_disabled(); + invtlb_all(INVTLB_ALLGID_GPA_TO_HPA, 0, 0); +} + +void kvm_flush_tlb_gpa(struct kvm_vcpu *vcpu, unsigned long gpa) +{ + unsigned int vmid; + + lockdep_assert_irqs_disabled(); + gpa &= (PAGE_MASK << 1); + vmid = (vcpu->arch.tgid << CSR_GSTAT_GID_SHIFT) & CSR_GSTAT_GID; + invtlb(INVTLB_GID_ADDR, vmid, gpa); } diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 8e028be3f0a9..4026b9bf9f2d 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -1166,12 +1166,17 @@ static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, /* All vCPUs need set the same PV features */ spin_lock(&kvm->arch.pv_setting_lock); - if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED) - && ((kvm->arch.pv_features & valid) != val)) { + if ((kvm->arch.pv_features & valid) == val) { + spin_unlock(&kvm->arch.pv_setting_lock); + return 0; + } + + if (vcpu->arch.has_run) { spin_unlock(&kvm->arch.pv_setting_lock); return -EINVAL; } - kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED; + + kvm->arch.pv_features = val; spin_unlock(&kvm->arch.pv_setting_lock); return 0; default: @@ -1858,6 +1863,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) int r = -EINTR; struct kvm_run *run = vcpu->run; + /* Mark vCPU has run at least once */ + if (!vcpu->arch.has_run) + vcpu->arch.has_run = true; + if (vcpu->mmio_needed) { if (!vcpu->mmio_is_write) kvm_complete_mmio_read(vcpu, run); diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index 6dabb227a732..e4c1c65fe20b 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c @@ -50,7 +50,9 @@ static void kvm_vm_init_features(struct kvm *kvm) /* Enable all PV features by default */ kvm->arch.pv_features |= BIT(KVM_FEATURE_IPI); + kvm->arch.pv_features |= BIT(KVM_FEATURE_CRASH); kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_PV_IPI); + kvm->arch.kvm_features |= BIT(KVM_LOONGARCH_VM_FEAT_CRASH); if (kvm_pvtime_supported()) { kvm->arch.pv_features |= BIT(KVM_FEATURE_PREEMPT); kvm->arch.pv_features |= BIT(KVM_FEATURE_STEAL_TIME); diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index c14b10821817..d481275f38eb 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -893,5 +893,6 @@ static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS +#define __KVM_HAVE_READONLY_MEM #endif /* __MIPS_KVM_HOST_H__ */ diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index b1b9a1d67758..849ce7c7c4df 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -22,9 +22,7 @@ config KVM select EXPORT_UASM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT - select KVM_MMIO select KVM_GENERIC_HARDWARE_ENABLING - select HAVE_KVM_READONLY_MEM help Support for hosting Guest kernels. diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 2d139c807577..708a32f89776 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -29,6 +29,10 @@ #define __KVM_HAVE_ARCH_VCPU_DEBUGFS +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) && !defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE) +#define __KVM_NO_MMIO +#endif + #define KVM_MAX_VCPUS NR_CPUS #define KVM_MAX_VCORES NR_CPUS @@ -49,6 +53,9 @@ #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ #define __KVM_HAVE_ARCH_INTC_INITIALIZED +#ifdef CONFIG_KVM_MPIC +#define __KVM_HAVE_MSI +#endif #define KVM_HALT_POLL_NS_DEFAULT 10000 /* 10 us */ diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index b6bc2fc86dca..cacc22e34afd 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -30,7 +30,6 @@ config KVM_BOOK3S_HANDLER config KVM_BOOK3S_32_HANDLER bool select KVM_BOOK3S_HANDLER - select KVM_MMIO config KVM_BOOK3S_64_HANDLER bool @@ -38,7 +37,6 @@ config KVM_BOOK3S_64_HANDLER config KVM_BOOK3S_PR_POSSIBLE bool - select KVM_MMIO config KVM_BOOK3S_HV_POSSIBLE bool @@ -201,7 +199,6 @@ config KVM_E500V2 depends on PPC_E500 && !PPC_E500MC depends on !CONTEXT_TRACKING_USER select KVM - select KVM_MMIO help Support running unmodified E500 guest kernels in virtual machines on E500v2 host processors. @@ -216,7 +213,6 @@ config KVM_E500MC depends on PPC_E500MC depends on !CONTEXT_TRACKING_USER select KVM - select KVM_MMIO select KVM_BOOKE_HV help Support running unmodified E500MC/E5500/E6500 guest kernels in @@ -232,7 +228,6 @@ config KVM_MPIC depends on KVM && PPC_E500 select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_MSI help Enable support for emulating MPIC devices inside the host kernel, rather than relying on userspace to emulate. diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 140eadb76dfe..22283d46e8fe 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -269,7 +269,7 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) * stolen time for a vcore when it is inactive, or for a vcpu * when it is in state RUNNING or NOTREADY. NOTREADY is a bit of * a misnomer; it means that the vcpu task is not executing in - * the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in + * the KVM_RUN ioctl, i.e. it is in userspace or elsewhere in * the kernel. We don't have any way of dividing up that time * between time that the vcpu is genuinely stopped, time that * the task is actively working on behalf of the vcpu, and time diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index a30600579231..e28f058a94d7 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -47,6 +47,8 @@ #define KVM_REQ_STEAL_UPDATE KVM_ARCH_REQ(6) #define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE +#define __KVM_HAVE_MSI +#define __KVM_HAVE_READONLY_MEM #define KVM_HAVE_MMU_RWLOCK diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index ec2cee0a39e0..2e3c4e6c1cd7 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -22,13 +22,10 @@ config KVM depends on RISCV_SBI && MMU select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_MSI - select HAVE_KVM_READONLY_MEM select HAVE_KVM_DIRTY_RING_ACQ_REL select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING - select KVM_MMIO select VIRT_XFER_TO_GUEST_WORK select SCHED_INFO select GUEST_PERF_EVENTS if PERF_EVENTS diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index a8b891cda4a7..2c46cfc526a9 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -181,7 +181,7 @@ config S390 select GENERIC_ENTRY select GENERIC_GETTIMEOFDAY select GENERIC_SMP_IDLE_THREAD - select GENERIC_IOREMAP if PCI + select GENERIC_IOREMAP select GLOB select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL @@ -766,9 +766,6 @@ config PCI_NR_FUNCTIONS endif # PCI -config HAS_IOMEM - def_bool PCI - config CHSC_SCH def_tristate m prompt "Support for CHSC subchannels" diff --git a/arch/s390/include/arm64/kvm_emulate.h b/arch/s390/include/arm64/kvm_emulate.h new file mode 100644 index 000000000000..416c4f96db05 --- /dev/null +++ b/arch/s390/include/arm64/kvm_emulate.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_KVM_EMULATE_H__ +#define __S390_ARM64_KVM_EMULATE_H__ + +#include +#include + +#include +#include +#include +#include + +static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.pc; +} + +static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.pstate; +} + +static __always_inline unsigned long *vcpu_sp_el0(const struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.sae_block.sp_el0; +} + +static __always_inline u64 *vcpu_sp_el1(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.sp_el1; +} + +static __always_inline __vector128 *vcpu_vreg(struct kvm_vcpu *vcpu, int off) +{ + return &vcpu->arch.ctxt.vregs[off]; +} + +static __always_inline u32 *vcpu_fpsr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.fpsr; +} + +static __always_inline u32 *vcpu_fpcr(struct kvm_vcpu *vcpu) +{ + return &vcpu->arch.sae_block.fpcr; +} + +static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu) +{ + return false; +} + +static __always_inline u64 kvm_vcpu_get_esr(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.esr_el2; +} + +static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) +{ + return (unsigned long *)&vcpu->arch.hcr_el2; +} + +static __always_inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.far_el2; +} + +static __always_inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu) +{ + return gfn_to_gpa(vcpu->arch.sae_block.hai.teid.addr); +} + +static inline u16 kvm_vcpu_fault_pic(const struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.pic & PGM_INT_CODE_MASK; +} + +static __always_inline +bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_fault_pic(vcpu) == PGM_PROTECTION; +} + +static __always_inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu) +{ + return true; +} + +static __always_inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu) +{ + return false; +} + +static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu) +{ + return false; +} + +static inline int kvm_vcpu_abt_gltl(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.sae_block.hai.gltl; +} + +static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu) +{ + u32 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK; + + return mode != PSR_MODE_EL0t; +} + +static inline void kvm_skip_instr(struct kvm_vcpu *vcpu) +{ + *vcpu_pc(vcpu) += 4; + *vcpu_cpsr(vcpu) &= ~PSR_BTYPE_MASK; + + /* advance the singlestep state machine */ + *vcpu_cpsr(vcpu) &= ~SPSR_ELx_SS; +} + +static inline void kvm_reset_fpsimd(struct kvm_vcpu *vcpu) +{ + memset(vcpu->arch.ctxt.vregs, 0, sizeof(vcpu->arch.ctxt.vregs)); + vcpu->arch.sae_block.fpsr = 0; + vcpu->arch.sae_block.fpcr = 0; +} + +#include + +#endif /* __S390_ARM64_KVM_EMULATE_H__ */ diff --git a/arch/s390/include/arm64/kvm_nested.h b/arch/s390/include/arm64/kvm_nested.h new file mode 100644 index 000000000000..e950b1a10c41 --- /dev/null +++ b/arch/s390/include/arm64/kvm_nested.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef ASM_KVM_NESTED_H +#define ASM_KVM_NESTED_H + +static inline bool vcpu_has_nv(const struct kvm_vcpu *vcpu) +{ + return false; +} + +#endif /* ASM_KVM_NESTED_H */ diff --git a/arch/s390/include/arm64/ptrace.h b/arch/s390/include/arm64/ptrace.h new file mode 100644 index 000000000000..449b85582c97 --- /dev/null +++ b/arch/s390/include/arm64/ptrace.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_PTRACE_H__ +#define __S390_ARM64_PTRACE_H__ + +#include +#include + +#include + +#endif /* __S390_ARM64_PTRACE_H__ */ diff --git a/arch/s390/include/arm64/sysreg.h b/arch/s390/include/arm64/sysreg.h new file mode 100644 index 000000000000..ac6b638fef56 --- /dev/null +++ b/arch/s390/include/arm64/sysreg.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __S390_ARM64_SYSREG_H__ +#define __S390_ARM64_SYSREG_H__ + +#include +#include +#include +#include + +#include + +#endif /* __S390_ARM64_SYSREG_H__ */ diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index d4da4436d02b..faa2afb259a1 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #endif /* _ASM_S390_PROTOTYPES_H */ diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index bb63fa4d20bb..ad3108ecfb07 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h @@ -123,6 +123,7 @@ enum { HWCAP_NR_NNPA = 20, HWCAP_NR_PCI_MIO = 21, HWCAP_NR_SIE = 22, + HWCAP_NR_SAE = 23, HWCAP_NR_MAX }; @@ -150,6 +151,7 @@ enum { #define HWCAP_NNPA BIT(HWCAP_NR_NNPA) #define HWCAP_PCI_MIO BIT(HWCAP_NR_PCI_MIO) #define HWCAP_SIE BIT(HWCAP_NR_SIE) +#define HWCAP_SAE BIT(HWCAP_NR_SAE) /* * These are used to set parameters in the core dumps. diff --git a/arch/s390/include/asm/io.h b/arch/s390/include/asm/io.h index faddb9aef3b8..ef0490edeccd 100644 --- a/arch/s390/include/asm/io.h +++ b/arch/s390/include/asm/io.h @@ -27,14 +27,8 @@ void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr); /* * I/O memory mapping functions. */ -#define ioremap_prot ioremap_prot -#define iounmap iounmap - #define _PAGE_IOREMAP pgprot_val(PAGE_KERNEL) -#define ioremap_wc(addr, size) \ - ioremap_prot((addr), (size), pgprot_writecombine(PAGE_KERNEL)) - static inline void __iomem *ioport_map(unsigned long port, unsigned int nr) { return NULL; @@ -46,6 +40,12 @@ static inline void ioport_unmap(void __iomem *p) #ifdef CONFIG_PCI +#define ioremap_prot ioremap_prot +#define iounmap iounmap + +#define ioremap_wc(addr, size) \ + ioremap_prot((addr), (size), pgprot_writecombine(PAGE_KERNEL)) + /* * s390 needs a private implementation of pci_iomap since ioremap with its * offset parameter isn't sufficient. That's because BAR spaces are not diff --git a/arch/s390/include/asm/kvm.h b/arch/s390/include/asm/kvm.h new file mode 100644 index 000000000000..aeb11dc631d5 --- /dev/null +++ b/arch/s390/include/asm/kvm.h @@ -0,0 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef KVM_S390_ARM64 +#include +#else +#include +#endif diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 147325978225..ac3440a617c1 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -3,7 +3,11 @@ #ifndef ASM_KVM_HOST_H #define ASM_KVM_HOST_H +#ifdef KVM_S390_ARM64 +#include +#else #include +#endif #define PGM_OPERATION 0x01 #define PGM_PRIVILEGED_OP 0x02 diff --git a/arch/s390/include/asm/kvm_host_arm64.h b/arch/s390/include/asm/kvm_host_arm64.h new file mode 100644 index 000000000000..82343e723ebf --- /dev/null +++ b/arch/s390/include/asm/kvm_host_arm64.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_KVM_HOST_ARM64_H +#define ASM_KVM_HOST_ARM64_H + +#include + +#include +#include + +#define vcpu_gp_regs(v) ((v)->arch.sae_block.gpr) + +#include +#include + +#include + +#define KVM_HAVE_MMU_RWLOCK +#define KVM_MAX_VCPUS 1 +#define __KVM_HAVE_VCPU_RUN_PID_CHANGE + +#define KVM_S390_ARM64_IMPL_FEATURES 0 +#define KVM_HALT_POLL_NS_DEFAULT 50000 + +#define KVM_S390_MANAGES_S390_GUEST 0 + +/* Minimal (=no) vgic definitions */ +#define KVM_IRQCHIP_NUM_PINS 1 +#define irqchip_in_kernel(_k) false + +#define __ctxt_sys_reg(ctx, reg) NULL +struct kvm_cpu_context { + /* + * These are just for 32 bit, which we don't have, making them RES0. + * They are exposed to user space. + */ + u64 spsr_abt; + u64 spsr_und; + u64 spsr_irq; + u64 spsr_fiq; + + __vector128 __aligned(16) vregs[32]; +}; + +struct kvm_vcpu_arch { + struct kvm_sae_block sae_block; + struct kvm_sae_save_area save_area; + struct kvm_cpu_context ctxt; + + u32 host_acrs[NUM_ACRS]; + + /* Hypervisor Configuration Register */ + u64 hcr_el2; + + /* Configuration flags, set once and for all before the vcpu can run */ + u8 cflags; + + /* Input flags to the hypervisor code, potentially cleared after use */ + u8 iflags; + + /* State flags for kernel bookkeeping, unused by the hypervisor code */ + u8 sflags; + + /* + * Don't run the guest (internal implementation need). + * + * Contrary to the flags above, this is set/cleared outside of + * a vcpu context, and thus cannot be mixed with the flags + * themselves (or the flag accesses need to be made atomic). + */ + bool pause; + + /* vcpu power state */ + struct kvm_mp_state mp_state; + /* lock for mp_state & reset_state.reset */ + spinlock_t mp_state_lock; + + /* vcpu reset state */ + struct vcpu_reset_state reset_state; + + struct kvm_s390_mmu_cache *mc; + + void *debugfs_state_data; +}; + +struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; + /* ARM64 stats */ + u64 hvc_exit_stat; + u64 wfe_exit_stat; + u64 wfi_exit_stat; + u64 mmio_exit_user; + u64 mmio_exit_kernel; + u64 signal_exits; + u64 exits; + /* GMAP stats */ + u64 pfault_sync; +}; + +#define kvm_has_mte(_kvm) false +#define vcpu_has_sve(_vcpu) false +#define vcpu_has_ptrauth(_vcpu) false + +struct kvm_arch_memory_slot { +}; + +struct kvm_arch { + struct gmap *gmap; + u64 guest_phys_size; + + /* VM-wide vCPU feature set */ + unsigned long flags; + + /* Protects VM-scoped configuration data */ + struct mutex config_lock; + + debug_info_t *dbf; + + DECLARE_BITMAP(vcpu_features, KVM_VCPU_MAX_FEATURES); + + unsigned long mem_limit; +}; + +static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature) +{ + return test_bit(feature, ka->vcpu_features); +} + +struct kvm_vm_stat { + struct kvm_vm_stat_generic generic; +}; + +static inline bool system_has_full_ptr_auth(void) +{ + return true; +} + +#define kvm_vm_is_protected(_kvm) false +#define vcpu_is_protected(_vcpu) false + +#define vcpu_is_loaded(_vcpu) ((_vcpu)->cpu != -1) + +#define KVM_HVA_ERR_BAD (-1UL) +#define KVM_HVA_ERR_RO_BAD (-2UL) + +static inline bool kvm_is_error_hva(unsigned long addr) +{ + return IS_ERR_VALUE(addr); +} + +u32 get_kvm_ipa_limit(void); + +/* unused, but required functions */ +static inline void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} +static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} +static inline void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) {} +static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} +static inline void kvm_arch_sync_events(struct kvm *kvm) {} +static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} +static inline void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, + unsigned long mask) {} + +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu); +void kvm_reset_vcpu(struct kvm_vcpu *vcpu); + +/* arm64 guests do not use async-pf. Defined because Kbuild requires it as s390 kvm turns it on. */ +#define ASYNC_PF_PER_VCPU 0 +struct kvm_arch_async_pf { + unsigned long pfault_token; +}; + +#define __unsupp_async_call(fn) WARN_ONCE(true, "async not supported on kvm-arm64 %s", fn) + +static inline bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) +{ + __unsupp_async_call(__func__); + return false; +}; + +static inline void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); +}; + +static inline bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); + return false; +}; + +static inline void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, + struct kvm_async_pf *work) +{ + __unsupp_async_call(__func__); +}; + +static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) +{ + __unsupp_async_call(__func__); +}; + +#define kvm_supports_32bit_el0() false + +#endif /* ASM_KVM_HOST_ARM64_H */ diff --git a/arch/s390/include/asm/kvm_host_arm64_types.h b/arch/s390/include/asm/kvm_host_arm64_types.h new file mode 100644 index 000000000000..8783a66d3029 --- /dev/null +++ b/arch/s390/include/asm/kvm_host_arm64_types.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_KVM_HOST_ARM64_TYPES_H +#define ASM_KVM_HOST_ARM64_TYPES_H + +#include +#include +#include +#include +#include + +struct kvm_sae_block { + u64 _0000[16]; /* 0x0000 */ +#define SAE_ICPTR_SPURIOUS 0x00 +#define SAE_ICPTR_VALIDITY 0x01 +#define SAE_ICPTR_HOST_ACCESS_EXCEPTION 0x02 +#define SAE_ICPTR_SYNCHRONOUS_EXCEPTION 0x03 +#define SAE_ICPTR_TIMER 0x04 +#define SAE_ICPTR_PE_INTERCOMM 0x05 +#define SAE_ICPTR_GUEST_ADDRESS_SIZE 0x06 +#define SAE_ICPTR_STOP 0x07 +#define SAE_ICPTR_MIO_ADDRESS 0x08 +#define SAE_ICPTR_PMU 0x09 +#define SAE_ICPTR_MAINTENANCE 0x0a + u8 icptr; /* 0x0080 */ + u8 _0081[7]; /* 0x0081 */ + u64 scad; /* 0x0088 */ + u64 _0090[16]; /* 0x0090 */ + u32 cntp_ctl; /* 0x0110 */ + u32 cntv_ctl; /* 0x0114 */ + u8 irq_ctl; /* 0x0118 */ + u8 _0119[7]; /* 0x0119 */ + struct { + u64 ich_hcr_el2; /* 0x0120 */ + u64 ich_vmcr_el2; /* 0x0128 */ + u64 ich_ap0r0_el2; /* 0x0130 */ + u64 ich_ap1r0_el2; /* 0x0138 */ + u64 _0140[2]; /* 0x0140 */ + u64 ich_lrn_el2[4]; /* 0x0150 */ + u64 _0170[4]; /* 0x0170 */ + } ic_regs; + u64 _0190[12]; /* 0x0190 */ + u64 contextidr_el1; /* 0x01f0 */ + u32 wip; /* 0x01f8 */ + u32 _01fc; /* 0x01fc */ +#define SAE_SD_FORMAT_0 0x00 + u8 sdf; /* 0x0200 */ + u8 _0201[7]; /* 0x0201 */ + u64 mso; /* 0x0208 */ + u64 msl; /* 0x0210 */ + u64 hbasce; /* 0x0218 */ + u64 _0220; /* 0x0220 */ + u64 gpto; /* 0x0228 */ + u64 ic; /* 0x0230 */ + u64 ec; /* 0x0238 */ + u64 save_area; /* 0x0240 */ + u64 _0248[7]; /* 0x0248 */ + u8 _0280[6]; /* 0x0280 */ + u16 lrcpua; /* 0x0286 */ + u64 pstate; /* 0x0288 */ + u64 pc; /* 0x0290 */ + u64 sp_el0; /* 0x0298 */ + u64 sp_el1; /* 0x02a0 */ + u64 _02a8; /* 0x02a8 */ + struct { /* 0x02b0 */ + u32 _fpcr_res0; + u32 fpcr; + }; + struct { /* 0x02b8 */ + u32 _fpsr_res0; + u32 fpsr; + }; + u16 sve_pregs[16]; /* 0x02c0 */ + u16 sve_ffr; /* 0x02e0 */ + u8 _02e2[6]; /* 0x02e2 */ + u64 _02e8[3]; /* 0x02e8 */ + + u64 gpr[31]; /* 0x0300 */ + u64 _03f8; /* 0x03f8 */ + + union { + u64 icptd[8]; /* 0x0400 */ + /* validity-interception reason; icptr 0x01 */ + u16 vir; /* 0x0400 */ + /* host access interception details; icptr 0x02 */ + struct { + u64 esr_el2; /* 0x0400 */ + u8 _0408[6]; /* 0x0408 */ + u16 pic; /* 0x040e */ + union teid teid; /* 0x0410 */ + gva_t far_el2; /* 0x0418 */ + gva_t vaddr; /* 0x0420 */ + u64 suppl; /* 0x0428 */ + u8 gltl; /* 0x0430 */ + u8 _0431[7]; /* 0x0431 */ + u64 _0438; /* 0x0438 */ + } hai; + /* exception-interception details; icptr 0x03 */ + struct { + u64 esr_el2; /* 0x0400 */ + u64 _0408[2]; /* 0x0408 */ + gva_t far_el2; /* 0x0418 */ + } trap; + /* timer-interception reason; icptr 0x04 */ +#define SAE_IR_TIMER_ID_VIRT BIT(6) +#define SAE_IR_TIMER_ID_PHYS BIT(7) + u8 tir; /* 0x0400 */ + }; + u64 _0440[376]; /* 0x0440 */ +} __packed __aligned(PAGE_SIZE); +static_assert(sizeof(struct kvm_sae_block) == PAGE_SIZE); + +struct kvm_sae_save_area { +#define SAE_SAVE_AREA_FORMAT_0 0x00 + u8 saf; /* 0x0000 */ + u8 _0001[5]; /* 0x0001 */ +#define SAE_SAS_VALID BIT(0) + u16 sas; /* 0x0006 */ + u64 sdo; /* 0x0008 */ + u64 _0010[2]; /* 0x0010 */ + u64 regs[508]; /* 0x0020 */ +} __packed __aligned(PAGE_SIZE); +static_assert(sizeof(struct kvm_sae_save_area) == PAGE_SIZE); + +#endif /* ASM_KVM_HOST_ARM64_TYPES_H */ diff --git a/arch/s390/include/asm/kvm_host_s390.h b/arch/s390/include/asm/kvm_host_s390.h index cd692f8fb764..f7802600b3fb 100644 --- a/arch/s390/include/asm/kvm_host_s390.h +++ b/arch/s390/include/asm/kvm_host_s390.h @@ -27,13 +27,20 @@ #include #include +#define kvm_file_to_kvm_arch s390 + #define KVM_HAVE_MMU_RWLOCK #define KVM_MAX_VCPUS 255 #define KVM_INTERNAL_MEM_SLOTS 1 +#define __KVM_HAVE_INVALID_WAKEUPS +#define __KVM_HAVE_NO_POLL + #define KVM_S390_MANAGES_S390_GUEST 1 +#define __KVM_NO_MMIO + /* * These seem to be used for allocating ->chip in the routing table, which we * don't use. 1 is as small as we can get to reduce the needed memory. If we @@ -720,7 +727,7 @@ static inline void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) {} void kvm_arch_free_vm(struct kvm *kvm); struct zpci_kvm_hook { - int (*kvm_register)(void *opaque, struct kvm *kvm); + int (*kvm_register)(void *opaque, struct file *kvm_file); void (*kvm_unregister)(void *opaque); }; diff --git a/arch/s390/include/asm/kvm_host_types.h b/arch/s390/include/asm/kvm_host_types.h index e5bdba07cab0..d308739ebcf5 100644 --- a/arch/s390/include/asm/kvm_host_types.h +++ b/arch/s390/include/asm/kvm_host_types.h @@ -3,6 +3,10 @@ #ifndef ASM_KVM_HOST_TYPES_H #define ASM_KVM_HOST_TYPES_H +#ifdef KVM_S390_ARM64 +#include +#else #include +#endif /* KVM_S390_ARM64 */ #endif /* ASM_KVM_HOST_TYPES_H */ diff --git a/arch/s390/include/asm/sae.h b/arch/s390/include/asm/sae.h new file mode 100644 index 000000000000..26d89fae0836 --- /dev/null +++ b/arch/s390/include/asm/sae.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_S390_SAE_H +#define __ASM_S390_SAE_H + +#include +#include + +#ifndef __ASSEMBLER__ + +/* defined in arch/s390/kernel/entry.S */ +asmlinkage void sae(phys_addr_t sae_block_phys); + +#include +#include + +/** + * stiasrm() - STore and Invalidate Arm System Register Multiple + * @save_area: Pointer to SAE save area + * + * Store the guest system registers to the save area. + * The values in the guest are no longer valid. + */ +static __always_inline void stiasrm(struct kvm_sae_save_area *save_area) +{ + asm_inline volatile( + " .insn rre,0xb9a70000,%[r1],0\n" + : "+m" (*save_area) + : [r1] "a" (save_area) + ); +} + +/** + * lasrm() - Load Arm System Register Multiple + * + * @save_area: Pointer to SAE save area + * + * Load the system registers from save_area into the guest. + */ +static __always_inline void lasrm(struct kvm_sae_save_area *save_area) +{ + asm_inline volatile( + " .insn rre,0xb9a60000,%[r1],0\n" + : + : "m" (*save_area), [r1] "a" (save_area) + ); +} + +#endif /* !__ASSEMBLER__ */ +#endif /* __ASM_S390_SAE_H */ diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 44066550b9b1..68f61b71a2d5 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -56,7 +56,9 @@ struct sclp_core_entry { u8 siif : 1; u8 sigpif : 1; u8 : 3; - u8 reserved2[3]; + u8 aef: 1; + u8 : 7; + u8 reserved2[2]; u8 : 2; u8 ib : 1; u8 cei : 1; @@ -109,6 +111,7 @@ struct sclp_info { unsigned char has_aeni : 1; unsigned char has_aisi : 1; unsigned char has_astfleie2 : 1; + unsigned char has_aef : 1; unsigned int ibc; unsigned int mtid; unsigned int mtid_cp; diff --git a/arch/s390/include/asm/stacktrace.h b/arch/s390/include/asm/stacktrace.h index ac3606c3babe..2d332d7c8145 100644 --- a/arch/s390/include/asm/stacktrace.h +++ b/arch/s390/include/asm/stacktrace.h @@ -59,6 +59,7 @@ static inline bool on_stack(struct stack_info *info, struct stack_frame { union { unsigned long empty[9]; + /* SIE stack frame */ struct { unsigned long sie_control_block; unsigned long sie_savearea; @@ -68,6 +69,10 @@ struct stack_frame { unsigned long sie_guest_asce; unsigned long sie_irq; }; + /* SAE stack frame */ + struct { + unsigned long sae_bear; + }; }; unsigned long gprs[10]; unsigned long back_chain; diff --git a/arch/s390/include/uapi/Kbuild b/arch/s390/include/uapi/Kbuild new file mode 100644 index 000000000000..81dcaaf42c40 --- /dev/null +++ b/arch/s390/include/uapi/Kbuild @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 + +# The ARM64 uapi headers under generated/uapi/arm64/ are private copies used +# internally by the s390 KVM ARM64 implementation. They contain #line +# directives and rewritten include paths and must not be shipped to userspace. +no-export-headers += arm64/% diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index f6dd2b67dcee..f9f9c2246074 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -70,6 +70,7 @@ int main(void) OFFSET(__SF_SIE_CONTROL_PHYS, stack_frame, sie_control_block_phys); OFFSET(__SF_SIE_GUEST_ASCE, stack_frame, sie_guest_asce); OFFSET(__SF_SIE_IRQ, stack_frame, sie_irq); + OFFSET(__SF_SAE_BEAR, stack_frame, sae_bear); DEFINE(STACK_FRAME_OVERHEAD, sizeof(struct stack_frame)); BLANK(); OFFSET(__SFUSER_BACKCHAIN, stack_frame_user, back_chain); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 4242cd7ab2fe..999dc7d0c637 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -245,6 +245,28 @@ EXPORT_SYMBOL(__sie64a) EXPORT_SYMBOL(sie_exit) #endif +#if IS_ENABLED(CONFIG_KVM) +/* + * SAE calling convention: + * %r2 physical address of the SAE control block + */ +SYM_FUNC_START(sae) + stmg %r6,%r14,__SF_GPRS(%r15) # store kernel registers + STBEAR __SF_SAE_BEAR(%r15) # save breaking event address register + .insn rre,0xb9a50000,%r2,0 # Start Arm Execution + LBEAR __SF_SAE_BEAR(%r15) # restore breaking event address register + lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers + xgr %r0,%r0 # clear guest registers to + xgr %r1,%r1 # prevent speculative use + xgr %r2,%r2 + xgr %r3,%r3 + xgr %r4,%r4 + xgr %r5,%r5 + BR_EX %r14 +SYM_FUNC_END(sae) +EXPORT_SYMBOL(sae) +#endif + /* * SVC interrupt handler routine. System calls are synchronous events and * are entered with interrupts disabled. diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c index e33a3eccda56..6da55a158027 100644 --- a/arch/s390/kernel/processor.c +++ b/arch/s390/kernel/processor.c @@ -150,6 +150,7 @@ static void show_cpu_summary(struct seq_file *m, void *v) [HWCAP_NR_NNPA] = "nnpa", [HWCAP_NR_PCI_MIO] = "pcimio", [HWCAP_NR_SIE] = "sie", + [HWCAP_NR_SAE] = "sae", }; int i, cpu; @@ -254,6 +255,8 @@ static int __init setup_hwcaps(void) /* virtualization support */ if (sclp.has_sief2) elf_hwcap |= HWCAP_SIE; + if (sclp.has_aef) + elf_hwcap |= HWCAP_SAE; return 0; } diff --git a/arch/s390/kernel/uv.c b/arch/s390/kernel/uv.c index 8ea9dd7704ff..abb94720c79b 100644 --- a/arch/s390/kernel/uv.c +++ b/arch/s390/kernel/uv.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -242,34 +243,38 @@ void uv_free_stor_var(void *stor_var) } EXPORT_SYMBOL_FOR_MODULES(uv_free_stor_var, "kvm"); -static int uv_alloc_range_cb(pte_t *ptep, unsigned long addr, void *data) -{ - struct page *page; - pte_t pte; - - page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!page) - return -ENOMEM; - pte = __pte(page_to_phys(page) | pgprot_val(PAGE_KERNEL)); - set_pte(ptep, pte); - return 0; -} - void *uv_alloc_stor_var(unsigned long size) { + unsigned long i, nr_pages, addr; struct vm_struct *area; - unsigned long addr; + struct page **pages; size = PAGE_ALIGN(size); + nr_pages = size >> PAGE_SHIFT; area = get_vm_area(size, VM_SPARSE); if (!area) return NULL; + pages = kvcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL_ACCOUNT); + if (!pages) + goto free_area; + for (i = 0; i < nr_pages; i++) { + pages[i] = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!pages[i]) + goto free_pages; + } addr = (unsigned long)area->addr; - if (apply_to_page_range(&init_mm, addr, size, uv_alloc_range_cb, NULL)) - goto out; + if (vm_area_map_pages(area, addr, addr + size, pages)) + goto free_pages; + kvfree(pages); return area->addr; -out: - uv_free_stor_var(area->addr); +free_pages: + for (i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kvfree(pages); +free_area: + free_vm_area(area); return NULL; } EXPORT_SYMBOL_FOR_MODULES(uv_alloc_stor_var, "kvm"); diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig index 8d3ee17a1bcb..e7b525d04c6d 100644 --- a/arch/s390/kvm/Kconfig +++ b/arch/s390/kvm/Kconfig @@ -25,22 +25,27 @@ config KVM select KVM_COMMON select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_INVALID_WAKEUPS - select HAVE_KVM_NO_POLL select KVM_VFIO select VIRT_XFER_TO_GUEST_WORK select KVM_MMU_LOCKLESS_AGING select KVM_GENERIC_PRE_FAULT_MEMORY help - Support hosting paravirtualized guest machines using the SIE - virtualization capability on the mainframe. This should work - on any 64bit machine. + Enable support for hosting virtualized, hardware-accelerated + virtual machines via two modules: kvm and kvm_arm64. - This module provides access to the hardware capabilities through - a character device node named /dev/kvm. + The kvm module provides access to host hardware-accelerated s390 + guests using the Start Interpretive Execution (SIE) instruction via a + character device node named /dev/kvm. This requires a 64-bit machine + (hardware model). - To compile this as a module, choose M here: the module - will be called kvm. + The kvm_arm64 module provides access to host hardware-accelerated + arm64 guests using the Start ARM Execution (SAE) instruction via a + character device node named /dev/kvm_arm64. This requires hardware + models that support the Arm execution facility (AEF). + + To compile this driver as a module, choose M here. The module will be + called kvm for the SIE acceleration and kvm_arm64 for the SAE + acceleration. If unsure, say N. diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index a4e3875b5bdd..380db443a0e9 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -4,3 +4,4 @@ # Copyright IBM Corp. 2008 obj-$(CONFIG_KVM) += s390/ +obj-$(CONFIG_KVM) += arm64/ diff --git a/arch/s390/kvm/arm64/.gitignore b/arch/s390/kvm/arm64/.gitignore new file mode 100644 index 000000000000..bd9d90694676 --- /dev/null +++ b/arch/s390/kvm/arm64/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +generated/* diff --git a/arch/s390/kvm/arm64/Makefile b/arch/s390/kvm/arm64/Makefile new file mode 100644 index 000000000000..3f0fe5c9f0a1 --- /dev/null +++ b/arch/s390/kvm/arm64/Makefile @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: GPL-2.0 + +KVM := ../../../../virt/kvm +KVM_DEV_NAME = kvm_arm64 +KVM_DEV_MINOR = MISC_DYNAMIC_MINOR +include $(srctree)/virt/kvm/Makefile.kvm +include $(srctree)/arch/$(SRCARCH)/kvm/gmap/Makefile +include $(src)/Makefile.gen + +ccflags-y += -I $(src) -I $(objtree)/$(obj) -I$(srctree)/arch/$(SRCARCH)/kvm/gmap -DKVM_S390_ARM64 + +kvm-arm64-local := \ + arm.o \ + guest.o \ + handle_exit.o \ + inject_fault.o \ + mmio.o \ + mmu.o \ + reset.o + +kvm-arm64-obj := $(kvm-arm64-local) +kvm-arm64-obj += $(patsubst %.o,%-arm64.o,$(kvm-y)) +kvm-arm64-obj += $(patsubst %.o,%-arm64.o,$(gmap-y)) + +targets += $(kvm-arm64-obj) + +$(obj)/%-arm64.o: $(src)/%.c FORCE + @mkdir -p $(dir $@) + $(call if_changed_rule,cc_o_c) + $(call cmd,force_checksrc) + +# %-arm64.o stems don't match any kvm-arm64-y entry so Kbuild derives wrong +# KBUILD_MODNAME/KBUILD_MODFILE. Force include a generated header to override. +KVM_ARM64_MODNAME_H := $(obj)/generated/kvm_arm64_modname.h +ccflags-y += -include $(KVM_ARM64_MODNAME_H) + +$(addprefix $(obj)/,$(kvm-arm64-obj)): $(KVM_ARM64_MODNAME_H) +$(foreach sfx, i s ll lst, \ + $(eval $(patsubst %.o,%.$(sfx),$(addprefix $(obj)/,$(kvm-arm64-local))): $(KVM_ARM64_MODNAME_H))) + +targets += generated/kvm_arm64_modname.h + +define filechk_kvm_arm64_modname + echo '#ifndef __KVM_ARM64_MODNAME_H'; \ + echo '#define __KVM_ARM64_MODNAME_H'; \ + echo '#undef KBUILD_MODNAME'; \ + echo '#define KBUILD_MODNAME "$(KVM_DEV_NAME)"'; \ + echo '#undef __KBUILD_MODNAME'; \ + echo '#define __KBUILD_MODNAME $(KVM_DEV_NAME)'; \ + echo '#undef KBUILD_MODFILE'; \ + echo '#define KBUILD_MODFILE "$(obj)/kvm-arm64"'; \ + echo '#endif /* __KVM_ARM64_MODNAME_H */' +endef + +$(KVM_ARM64_MODNAME_H): FORCE + $(call filechk,kvm_arm64_modname) + +# Members are not in real-obj-y. Set part-of-builtin to enable instrumentation. +$(addprefix $(obj)/,$(kvm-arm64-obj)): private part-of-builtin := y + +ifeq ($(CONFIG_KVM),m) + +kvm-arm64-y = $(kvm-arm64-obj) + +else ifeq ($(CONFIG_KVM),y) + +prereq-o-cmd = $(foreach o, $(filter %.o, $^), $(dir $(o)).$(notdir $(o)).cmd) +cmd_gen_symversions_o = \ + grep --no-filename "^\#SYMVER" $(prereq-o-cmd) >> $(dot-target).cmd || true + +cmd_ld_o_o = $(cmd_ld) +define rule_ld_o_o + $(call cmd_and_savecmd,ld) + $(call cmd,gen_symversions_o) +endef + +LDFLAGS_kvm-unnamespaced.o := -r +$(obj)/kvm-unnamespaced.o: $(addprefix $(obj)/,$(kvm-arm64-obj)) FORCE + $(call if_changed_rule,ld_o_o) + +quiet_cmd_nm_filter = NMFLTR $@ + cmd_nm_filter = $(NM) --defined-only --format=just-symbols $< | sort -u > $@ + +$(obj)/kvm_symbol_list: $(obj)/kvm-unnamespaced.o FORCE + $(call if_changed,nm_filter) + +# cmd_oc_o_o carries -DKBUILD_MODFILE so generate_builtin_ranges.awk +# can attribute kvm-namespaced.o to kvm_arm64 via its .cmd file. +cmd_oc_o_o = $(cmd_objcopy) -DKBUILD_MODFILE='"$(obj)/kvm-arm64"' +define rule_oc_o_o + $(call cmd,objcopy) + @printf '%s\n' 'savedcmd_$@ := $(call make-cmd,oc_o_o)' > $(dot-target).cmd + $(call cmd,gen_objtooldep) + $(call cmd,gen_symversions_o) +endef + +OBJCOPYFLAGS_kvm-namespaced.o := -O $(LD_BFD) --localize-symbols=$(obj)/kvm_symbol_list +$(obj)/kvm-namespaced.o: $(obj)/kvm-unnamespaced.o $(obj)/kvm_symbol_list FORCE + $(call if_changed_rule,oc_o_o) + +kvm-arm64-y = kvm-namespaced.o + +targets += kvm-unnamespaced.o kvm_symbol_list kvm-namespaced.o + +endif + +clean-files += kvm-unnamespaced.o kvm_symbol_list kvm-namespaced.o + +obj-$(CONFIG_KVM) += kvm-arm64.o + diff --git a/arch/s390/kvm/arm64/Makefile.gen b/arch/s390/kvm/arm64/Makefile.gen new file mode 100644 index 000000000000..9c33b068fab5 --- /dev/null +++ b/arch/s390/kvm/arm64/Makefile.gen @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Extracts ARM64 marked sections to .inc files that are included by main source +# files + +# List of ARM64 C files to extract +ARM64_CFILES := \ + arm.c \ + guest.c \ + mmio.c + +quiet_cmd_extract_inc = GEN $@ + cmd_extract_inc = mkdir -p $(dir $@); \ + $(AWK) -f $(src)/copy-arm64c.awk $< > $@ || \ + { echo "Error: No or unbalanced ARM64_S390_COMMON markers in $<" >&2; rm -f $@; exit 1; } + +ARM64_INC_FILES := $(foreach cfile,$(ARM64_CFILES),generated/$(basename $(cfile)).inc) + +$(addprefix $(obj)/,$(ARM64_INC_FILES)): $(obj)/generated/%.inc: $(srctree)/arch/arm64/kvm/%.c $(src)/copy-arm64c.awk FORCE + $(call if_changed,extract_inc) + +targets += $(ARM64_INC_FILES) +clean-files += generated/*.inc + +$(foreach cfile,$(basename $(ARM64_CFILES)),\ + $(eval $(obj)/$(cfile).o $(obj)/$(cfile).i $(obj)/$(cfile).s \ + $(obj)/$(cfile).ll $(obj)/$(cfile).lst: \ + $(obj)/generated/$(cfile).inc)) diff --git a/arch/s390/kvm/arm64/arm.c b/arch/s390/kvm/arm64/arm.c new file mode 100644 index 000000000000..bd9d85235f43 --- /dev/null +++ b/arch/s390/kvm/arm64/arm.c @@ -0,0 +1,754 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +#include "arm.h" +#include "handle_exit.h" + +#define CREATE_TRACE_POINTS +#include "trace.h" + +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu); + +#define __INCL_GEN_ARM_FILE +#include "generated/arm.inc" +#undef __INCL_GEN_ARM_FILE + +#define KVM_PHYS_SHIFT (40) + +int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) +{ + int ret; + + switch (ext) { + case KVM_CAP_NR_VCPUS: + case KVM_CAP_MAX_VCPUS: + case KVM_CAP_MAX_VCPU_ID: + ret = KVM_MAX_VCPUS; + break; + case KVM_CAP_ARM_VM_IPA_SIZE: + ret = get_kvm_ipa_limit(); + break; + case KVM_CAP_IOEVENTFD: + case KVM_CAP_PRE_FAULT_MEMORY: + ret = 1; + break; + default: + ret = 0; + } + + return ret; +} + +static u64 kvm_max_guest_address(void) +{ + u64 max_addr; + + max_addr = min_t(u64, TASK_SIZE_MAX, sclp.hamax); + max_addr = max_t(u64, max_addr, SZ_1G - 1); + return ALIGN_DOWN(max_addr + 1, SZ_1G) - 1; +} + +static int kvm_gmap_init(struct kvm *kvm) +{ + struct crst_table *table; + + kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.guest_phys_size)); + + if (!kvm->arch.gmap) + return -ENOMEM; + + /* arm64 (on s390) do not have pfault */ + clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags); + set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags); + + table = dereference_asce(kvm->arch.gmap->asce); + crst_table_init((void *)table, _CRSTE_HOLE(table->crstes[0].h.tt).val); + + return 0; +} + +static int kvm_vm_type_ipa_size_shift(unsigned long type) +{ + int phys_shift; + + phys_shift = KVM_VM_TYPE_ARM_IPA_SIZE(type); + if (phys_shift) { + if (phys_shift > get_kvm_ipa_limit() || + phys_shift < ARM64_MIN_PARANGE_BITS) + return -EINVAL; + } else { + phys_shift = KVM_PHYS_SHIFT; + if (phys_shift > get_kvm_ipa_limit()) { + pr_warn_once("%s using unsupported default IPA limit\n", + current->comm); + return -EINVAL; + } + } + + return phys_shift; +} + +int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) +{ + char debug_name[32]; + int ret; + + if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK) + return -EINVAL; + + ret = kvm_vm_type_ipa_size_shift(type); + if (ret < 0) + return ret; + kvm->arch.guest_phys_size = 1UL << ret; + + mutex_init(&kvm->arch.config_lock); + bitmap_zero(kvm->arch.vcpu_features, KVM_VCPU_MAX_FEATURES); + + snprintf(debug_name, sizeof(debug_name), "kvm-arm64-%u-%u", + current->pid, get_random_u32()); + kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long)); + if (!kvm->arch.dbf) + return -ENOMEM; + debug_register_view(kvm->arch.dbf, &debug_sprintf_view); + + ret = kvm_gmap_init(kvm); + if (ret) + goto out_err; + kvm->arch.mem_limit = kvm->arch.guest_phys_size; + + VM_EVENT(kvm, 3, "vm created with type %lu", type); + return 0; + +out_err: + debug_unregister(kvm->arch.dbf); + + return ret; +} + +vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) +{ + return VM_FAULT_SIGBUS; +} + +long kvm_arch_dev_ioctl(struct file *filp, + unsigned int ioctl, unsigned long arg) +{ + return -EINVAL; +} + +void kvm_arch_destroy_vm(struct kvm *kvm) +{ + kvm_destroy_vcpus(kvm); + debug_unregister(kvm->arch.dbf); + kvm->arch.gmap = gmap_put(kvm->arch.gmap); +} + +u32 get_kvm_ipa_limit(void) +{ + return fls64(kvm_max_guest_address() + 1) - 1; +} + +int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) +{ + return 0; +} + +int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) +{ + struct kvm_sae_block *sae_block = &vcpu->arch.sae_block; + struct kvm_sae_save_area *save_area = &vcpu->arch.save_area; + + spin_lock_init(&vcpu->arch.mp_state_lock); + + /* Force users to call KVM_ARM_VCPU_INIT */ + vcpu_clear_flag(vcpu, VCPU_INITIALIZED); + + vcpu->arch.mc = kvm_s390_new_mmu_cache(); + if (!vcpu->arch.mc) + return -ENOMEM; + + save_area->sdo = virt_to_phys(sae_block); + + sae_block->hbasce = vcpu->kvm->arch.gmap->asce.val; + sae_block->mso = 0L; + sae_block->msl = vcpu->kvm->arch.mem_limit - 1; + sae_block->save_area = virt_to_phys(save_area); + + VM_EVENT(vcpu->kvm, 3, + "create cpu %d at 0x%p, sae block at 0x%p, satellite at 0x%p", + vcpu->vcpu_id, vcpu, &vcpu->arch.sae_block, + &vcpu->arch.save_area); + return 0; +} + +void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) +{ +} + +void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) +{ + kvm_s390_free_mmu_cache(vcpu->arch.mc); + + VCPU_EVENT(vcpu, 3, "%s", "free cpu"); +} + +void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +{ + save_access_regs(&vcpu->arch.host_acrs[0]); + vcpu->cpu = cpu; + + lasrm(&vcpu->arch.save_area); +} + +void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) +{ + stiasrm(&vcpu->arch.save_area); + + vcpu->cpu = -1; + restore_access_regs(&vcpu->arch.host_acrs[0]); +} + +int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + *mp_state = READ_ONCE(vcpu->arch.mp_state); + return 0; +} + +int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, + struct kvm_mp_state *mp_state) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) +{ + return 0; +} + +static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu) +{ + unsigned long features = KVM_S390_ARM64_IMPL_FEATURES; + + return features; +} + +bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) +{ + return vcpu_mode_priv(vcpu); +} + +int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + if (!kvm_arm_vcpu_is_finalized(vcpu)) + return -EPERM; + + if (likely(READ_ONCE(vcpu->pid))) + return 0; + + scoped_guard(mutex, &kvm->arch.config_lock) + set_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags); + + return 0; +} + +/** + * check_vcpu_requests - check and handle pending vCPU requests + * @vcpu: the VCPU pointer + * + * Return: 1 if we should enter the guest + * 0 if we should exit to userspace + * < 0 if we should exit to userspace, where the return value indicates + * an error + */ +static int check_vcpu_requests(struct kvm_vcpu *vcpu) +{ + if (kvm_request_pending(vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) + return -EIO; + if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) + kvm_reset_vcpu(vcpu); + /* + * Clear IRQ_PENDING requests that were made to guarantee + * that a VCPU sees new virtual interrupts. + */ + kvm_check_request(KVM_REQ_IRQ_PENDING, vcpu); + + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) + asce_flush_tlb(vcpu->kvm->arch.gmap->asce); + } + + return 1; +} + +static int kvm_vcpu_initialize(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + unsigned long features = init->features[0]; + struct kvm *kvm = vcpu->kvm; + + guard(mutex)(&kvm->arch.config_lock); + if (test_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags) && + kvm_vcpu_init_changed(vcpu, init)) + return -EINVAL; + + bitmap_copy(kvm->arch.vcpu_features, &features, KVM_VCPU_MAX_FEATURES); + + kvm_reset_vcpu(vcpu); + + set_bit(KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED, &kvm->arch.flags); + vcpu_set_flag(vcpu, VCPU_INITIALIZED); + + return 0; +} + +static int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, + const struct kvm_vcpu_init *init) +{ + int ret; + + if (init->target != KVM_ARM_TARGET_GENERIC_V8) + return -EINVAL; + + ret = kvm_vcpu_init_check_features(vcpu, init); + if (ret) + return ret; + + if (!kvm_vcpu_initialized(vcpu)) + return kvm_vcpu_initialize(vcpu, init); + + if (kvm_vcpu_init_changed(vcpu, init)) + return -EINVAL; + + kvm_reset_vcpu(vcpu); + + return 0; +} + +static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, + struct kvm_vcpu_init *init) +{ + struct kvm_sae_save_area *save_area = &vcpu->arch.save_area; + struct kvm_sae_block *sae_block = &vcpu->arch.sae_block; + int ret; + + sae_block->save_area = virt_to_phys(save_area); + save_area->sdo = virt_to_phys(sae_block); + + ret = kvm_vcpu_set_target(vcpu, init); + if (ret) + return ret; + + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); + } + + return 0; +} + +int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, + bool line_status) +{ + /* stub for now*/ + return -EINVAL; +} + +static void adjust_pc(struct kvm_vcpu *vcpu) +{ + if (vcpu_get_flag(vcpu, INCREMENT_PC)) { + kvm_skip_instr(vcpu); + vcpu_clear_flag(vcpu, INCREMENT_PC); + } +} + +static noinstr void arm_vcpu_enter_exit(phys_addr_t sae_block) +{ + guest_timing_enter_irqoff(); + guest_state_enter_irqoff(); + + /* + * External interrupts may occur while the SAE instruction is executing + * and must preempt guest execution to maintain responsiveness of the + * host. + * + * As a temporary workaround, run SAE with interrupts enabled so that any + * interrupt is taken via the normal host interrupt path rather than + * being subsumed inside the SAE window. + */ + raw_local_irq_enable(); + sae(sae_block); + raw_local_irq_disable(); + + guest_state_exit_irqoff(); + guest_timing_exit_irqoff(); +} + +/** kvm_arch_vcpu_ioctl_run() - run arm64 vCPU + * + * Execute arm64 guest instructions using SAE. + * + * Returns: + * 1 enter the guest (should not be observed by userspace) + * 0 exit to userspace + * < 0 exit to userspace, where the return value indicates an error + * + * + */ +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) +{ + DECLARE_KERNEL_FPU_ONSTACK32(fpu_save); + struct kvm_run *kvm_run = vcpu->run; + int ret; + + if (kvm_run->exit_reason == KVM_EXIT_MMIO) { + ret = kvm_handle_mmio_return(vcpu); + if (ret <= 0) + return ret; + } + + vcpu_load(vcpu); + + kernel_fpu_begin(&fpu_save, KERNEL_FPC | KERNEL_VXR); + load_vx_regs((vcpu->arch.ctxt.vregs)); + + if (!vcpu->wants_to_run) { + ret = -EINTR; + goto out; + } + + kvm_sigset_activate(vcpu); + + might_fault(); + + ret = 1; + kvm_run->exit_reason = KVM_EXIT_UNKNOWN; + while (ret > 0) { + ret = kvm_xfer_to_guest_mode_handle_work(vcpu); + if (!ret) + ret = 1; + + if (ret > 0) + ret = check_vcpu_requests(vcpu); + + vcpu->arch.sae_block.icptr = 0; + + local_irq_disable(); + xfer_to_guest_mode_prepare(); + /* + * The full memory barrier in smp_store_mb() orders the mode + * store against the kvm_request_pending() check below, so + * any pending kick is visible before we decide to enter the + * guest. + */ + smp_store_mb(vcpu->mode, IN_GUEST_MODE); + + if (ret <= 0 || kvm_request_pending(vcpu) || + xfer_to_guest_mode_work_pending()) { + vcpu->mode = OUTSIDE_GUEST_MODE; + local_irq_enable(); + continue; + } + + trace_kvm_entry(*vcpu_pc(vcpu)); + + adjust_pc(vcpu); + + arm_vcpu_enter_exit(virt_to_phys(&vcpu->arch.sae_block)); + + vcpu->mode = OUTSIDE_GUEST_MODE; + vcpu->stat.exits++; + + local_irq_enable(); + trace_kvm_exit(kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); + + ret = handle_exit(vcpu); + } + + kvm_sigset_deactivate(vcpu); +out: + if (unlikely(vcpu_get_flag(vcpu, INCREMENT_PC))) + adjust_pc(vcpu); + + save_vx_regs(vcpu->arch.ctxt.vregs); + kernel_fpu_end(&fpu_save, KERNEL_FPC | KERNEL_VXR); + vcpu_put(vcpu); + + return ret; +} + +static int kvm_arm_vcpu_set_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int kvm_arm_vcpu_get_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +static int kvm_arm_vcpu_has_attr(struct kvm_vcpu *vcpu, + struct kvm_device_attr *attr) +{ + int ret; + + switch (attr->group) { + default: + ret = -ENXIO; + break; + } + + return ret; +} + +long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct kvm_vcpu *vcpu = filp->private_data; + void __user *argp = (void __user *)arg; + struct kvm_device_attr attr; + int ret; + + switch (ioctl) { + case KVM_ARM_VCPU_INIT: { + struct kvm_vcpu_init init; + + ret = -EFAULT; + if (copy_from_user(&init, argp, sizeof(init))) + break; + + ret = kvm_arch_vcpu_ioctl_vcpu_init(vcpu, &init); + break; + } + case KVM_SET_ONE_REG: + case KVM_GET_ONE_REG: { + struct kvm_one_reg reg; + + ret = -ENOEXEC; + if (unlikely(!kvm_vcpu_initialized(vcpu))) + break; + + ret = -EFAULT; + if (copy_from_user(®, argp, sizeof(reg))) + break; + + if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) + kvm_reset_vcpu(vcpu); + + if (ioctl == KVM_SET_ONE_REG) + ret = kvm_arm_set_reg(vcpu, ®); + else + ret = kvm_arm_get_reg(vcpu, ®); + break; + } + case KVM_GET_REG_LIST: { + struct kvm_reg_list __user *user_list = argp; + struct kvm_reg_list reg_list; + unsigned int n; + + ret = -ENOEXEC; + if (unlikely(!kvm_vcpu_initialized(vcpu))) + break; + ret = -EPERM; + if (!kvm_arm_vcpu_is_finalized(vcpu)) + break; + ret = -EFAULT; + if (copy_from_user(®_list, user_list, sizeof(reg_list))) + break; + n = reg_list.n; + reg_list.n = kvm_arm_num_regs(vcpu); + if (copy_to_user(user_list, ®_list, sizeof(reg_list))) + break; + ret = -E2BIG; + if (n < reg_list.n) + break; + ret = kvm_arm_copy_reg_indices(vcpu, user_list->reg); + break; + } + case KVM_ARM_VCPU_FINALIZE: { + int what; + + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + if (get_user(what, (const int __user *)argp)) + return -EFAULT; + + ret = kvm_arm_vcpu_finalize(vcpu, what); + break; + } + case KVM_SET_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_set_attr(vcpu, &attr); + break; + } + case KVM_GET_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_get_attr(vcpu, &attr); + break; + } + case KVM_HAS_DEVICE_ATTR: { + ret = -EFAULT; + if (copy_from_user(&attr, argp, sizeof(attr))) + break; + ret = kvm_arm_vcpu_has_attr(vcpu, &attr); + break; + } + default: + ret = -EINVAL; + } + + return ret; +} + +int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, + struct kvm_dirty_log *log) +{ + return s390_kvm_mmu_get_dirty_log(kvm, log); +} + +bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + scoped_guard(read_lock, &kvm->mmu_lock) + return gmap_age_gfn(kvm->arch.gmap, range->start, range->end); +} + +void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) +{ + gfn_t last_gfn = memslot->base_gfn + memslot->npages; + + scoped_guard(read_lock, &kvm->mmu_lock) + gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn); +} + +int kvm_arch_prepare_memory_region(struct kvm *kvm, + const struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ + return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change); +} + +void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, + const struct kvm_memory_slot *new, + enum kvm_mr_change change) +{ +} + +bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end); +} + +bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) +{ + scoped_guard(read_lock, &kvm->mmu_lock) + return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end); +} + +int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + + switch (ioctl) { + case KVM_ARM_PREFERRED_TARGET: { + struct kvm_vcpu_init init = { + .target = KVM_ARM_TARGET_GENERIC_V8, + }; + + if (copy_to_user(argp, &init, sizeof(init))) + return -EFAULT; + + return 0; + } + + default: + return -EINVAL; + } +} + +bool kvm_arch_irqchip_in_kernel(struct kvm *kvm) +{ + return false; +} + +int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, + int irq_source_id, int level, bool line_status) +{ + return -EINVAL; +} + +int kvm_set_routing_entry(struct kvm *kvm, + struct kvm_kernel_irq_routing_entry *e, + const struct kvm_irq_routing_entry *ue) +{ + return -EINVAL; +} + +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return 0; +} + +long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + return -ENOIOCTLCMD; +} + +static int __init kvm_s390_arm64_init(void) +{ + if (!sclp.has_aef) + return -ENODEV; + + return kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); +} + +static __exit void kvm_s390_arm64_exit(void) +{ + kvm_exit(); +} + +module_init(kvm_s390_arm64_init); +module_exit(kvm_s390_arm64_exit); diff --git a/arch/s390/kvm/arm64/arm.h b/arch/s390/kvm/arm64/arm.h new file mode 100644 index 000000000000..760230685ca1 --- /dev/null +++ b/arch/s390/kvm/arm64/arm.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_S390_KVM_ARM64_H +#define ARCH_S390_KVM_ARM64_H + +#include +#include + +#define VM_EVENT(d_kvm, d_loglevel, d_string, d_args...) \ + debug_sprintf_event((d_kvm)->arch.dbf, d_loglevel, \ + __stringify(KVM_DEV_NAME) ": " d_string "\n", \ + d_args) + +#define VCPU_EVENT(d_vcpu, d_loglevel, d_string, d_args...) \ + debug_sprintf_event((d_vcpu)->kvm->arch.dbf, d_loglevel, \ + __stringify(KVM_DEV_NAME) " %02d[%016llx-%016llx]: " d_string "\n", \ + (d_vcpu)->vcpu_id, (d_vcpu)->arch.sae_block.pstate, \ + (d_vcpu)->arch.sae_block.pc, d_args) + +static __always_inline int kvm_is_ucontrol(struct kvm *kvm) +{ + return 0; +} + +static __always_inline int __kvm_s390_pv_destroy_page(struct page *page) +{ + return 0; +} + +static __always_inline void kvm_s390_vsie_gmap_notifier(struct gmap *gmap, gpa_t start, gpa_t end) +{ +} + +static __always_inline int kvm_s390_pv_get_handle(struct kvm *kvm) +{ + return 0; +} + +static __always_inline int kvm_s390_is_migration_mode(struct kvm *kvm) +{ + return false; +} + +static __always_inline bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) +{ + return false; +} + +static __always_inline void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old) +{ +} + +/* should never be called */ +static __always_inline int kvm_s390_vm_stop_migration(struct kvm *kvm) +{ + return -EINVAL; +} + +#endif /* ARCH_S390_KVM_ARM64_H */ diff --git a/arch/s390/kvm/arm64/copy-arm64c.awk b/arch/s390/kvm/arm64/copy-arm64c.awk new file mode 100644 index 000000000000..640a7bcb7e23 --- /dev/null +++ b/arch/s390/kvm/arm64/copy-arm64c.awk @@ -0,0 +1,82 @@ +#!/usr/bin/awk -f +# SPDX-License-Identifier: GPL-2.0 +# +# Extract marked sections from ARM64 C files for sharing with s390 KVM +# +# Usage: copy-arm64c.awk +# +# Extracts all sections between start/end markers. If no markers found, signals failure. + +BEGIN { + # Constants + start_pattern = "^#if ARM64_S390_COMMON == 1$" + end_pattern = "^#endif /\\* ARM64_S390_COMMON == 1 \\*/$" + + # State variables + copying = found_marker = 0 + file_header_done = 0 + depth = 0 +} + +!file_header_done { + if (/^\/\*/ || /^\/\/ SPDX-License-Identifier:/) { + print + next + } + if (/[[:space:]]\*([[:space:]]|$)/) { + print + next + } + if (/\*\//) { + print " *" + } else { + print "/*" + } + + print " * This file was automatically generated from " FILENAME + print " * Do not modify this file directly." + print " */" + print "" + print "#ifndef __INCL_GEN_ARM_FILE" + print "#error do not include this file directly." + print "#endif /* __INCL_GEN_ARM_FILE */" + print "#undef __INCL_GEN_ARM_FILE" + print "" + + file_header_done = 1 +} + +$0 ~ start_pattern { + if (copying) { error = 1; exit } + depth = copying = found_marker = 1 + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " (NR + 1) " \"" FILENAME "\"" + next +} + +copying && /^[ \t]*#[ \t]*if(def|ndef)?([ \t]|$)/ { depth++ } + +$0 ~ /^[ \t]*#[ \t]*endif/ { + depth-- + if (depth == 0) { + if ($0 ~ end_pattern) { + copying = 0 + next + } + error = 1; exit + } +} + +/ARM64_S390_COMMON/ { error = 1; exit } + +copying { + gsub(/#include +#include + +#include +#include + +#define __INCL_GEN_ARM_FILE +#include "generated/guest.inc" +#undef __INCL_GEN_ARM_FILE + +const struct kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS() +}; + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + /* ARM64 stats */ + STATS_DESC_COUNTER(VCPU, hvc_exit_stat), + STATS_DESC_COUNTER(VCPU, wfe_exit_stat), + STATS_DESC_COUNTER(VCPU, wfi_exit_stat), + STATS_DESC_COUNTER(VCPU, mmio_exit_user), + STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), + STATS_DESC_COUNTER(VCPU, signal_exits), + STATS_DESC_COUNTER(VCPU, exits), + /* GMAP stats */ + STATS_DESC_COUNTER(VCPU, pfault_sync), +}; + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), +}; + +int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) +{ + int ret; + + ret = copy_core_reg_indices(vcpu, uindices); + if (ret < 0) + return ret; + uindices += ret; + + return 0; +} + +unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu) +{ + return num_core_regs(vcpu); +} + +int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + /* We currently use nothing arch-specific in upper 32 bits */ + if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) + return -EINVAL; + + switch (reg->id & KVM_REG_ARM_COPROC_MASK) { + case KVM_REG_ARM_CORE: + return get_core_reg(vcpu, reg); + default: + return -EINVAL; + } +} + +int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + /* We currently use nothing arch-specific in upper 32 bits */ + if ((reg->id & ~KVM_REG_SIZE_MASK) >> 32 != KVM_REG_ARM64 >> 32) + return -EINVAL; + + switch (reg->id & KVM_REG_ARM_COPROC_MASK) { + case KVM_REG_ARM_CORE: + return set_core_reg(vcpu, reg); + default: + return -EINVAL; + } +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, + struct kvm_translation *tr) +{ + return -EINVAL; +} + +int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, + struct kvm_guest_debug *dbg) +{ + return -EINVAL; +} diff --git a/arch/s390/kvm/arm64/handle_exit.c b/arch/s390/kvm/arm64/handle_exit.c new file mode 100644 index 000000000000..e0c7773c2165 --- /dev/null +++ b/arch/s390/kvm/arm64/handle_exit.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include +#include + +#include "handle_exit.h" + +typedef int (*exit_handle_fn)(struct kvm_vcpu *); + +#define PSCI_0_2_FN_SYSTEM_OFF 0x84000008 +#define PSCI_RET_NOT_SUPPORTED -1 +#define PSCI_RET_INTERNAL_FAILURE -6 +/* + * Temporary smc/hvc handler. Non-compliant implementation (features missing). + * Implements only system off so that test programs are able to end their execution. + * Multi-vCPU power state management and KVM_REQ_SLEEP handling are deferred + * to future SMP series. + */ +static int kvm_smccc_call_handler(struct kvm_vcpu *vcpu) +{ + u32 func_id = vcpu_get_reg(vcpu, 0); + u64 val = PSCI_RET_NOT_SUPPORTED; + int ret = 1; + + if (func_id == PSCI_0_2_FN_SYSTEM_OFF) { + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); + } + memset(&vcpu->run->system_event, 0, + sizeof(vcpu->run->system_event)); + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SHUTDOWN; + vcpu->run->system_event.ndata = 1; + vcpu->run->system_event.data[0] = 0; + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + val = PSCI_RET_INTERNAL_FAILURE; + ret = 0; + } + vcpu_set_reg(vcpu, 0, val); + + return ret; +} + +static int handle_hvc(struct kvm_vcpu *vcpu) +{ + vcpu->stat.hvc_exit_stat++; + return kvm_smccc_call_handler(vcpu); +} + +static int kvm_handle_unknown_ec(struct kvm_vcpu *vcpu) +{ + u64 esr = kvm_vcpu_get_esr(vcpu); + + kvm_pr_unimpl("Unknown exception class: esr: %#016llx -- %s\n", + esr, esr_get_class_string(esr)); + + kvm_inject_undefined(vcpu); + return 1; +} + +static exit_handle_fn arm_exit_handlers[] = { + [0 ... ESR_ELx_EC_MAX] = kvm_handle_unknown_ec, + [ESR_ELx_EC_HVC64] = handle_hvc, + [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort, + [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort, +}; + +static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) +{ + u64 esr = kvm_vcpu_get_esr(vcpu); + u8 esr_ec = ESR_ELx_EC(esr); + + return arm_exit_handlers[esr_ec]; +} + +/* + * We may be single-stepping an emulated instruction. If the emulation + * has been completed in the kernel, we can return to userspace with a + * KVM_EXIT_DEBUG, otherwise userspace needs to complete its + * emulation first. + */ +static int handle_trap_exceptions(struct kvm_vcpu *vcpu) +{ + exit_handle_fn exit_handler; + + exit_handler = kvm_get_exit_handler(vcpu); + return exit_handler(vcpu); +} + +/* + * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on + * proper exit to userspace. + */ +int handle_exit(struct kvm_vcpu *vcpu) +{ + u8 icptr = vcpu->arch.sae_block.icptr; + int ret = 1; + + switch (icptr) { + case SAE_ICPTR_SPURIOUS: + break; + case SAE_ICPTR_VALIDITY: + WARN_ONCE(true, "SAE: validity intercept. vir: 0x%04x", + vcpu->arch.sae_block.vir); + vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; + ret = 0; + break; + case SAE_ICPTR_GUEST_ADDRESS_SIZE: + case SAE_ICPTR_HOST_ACCESS_EXCEPTION: + case SAE_ICPTR_SYNCHRONOUS_EXCEPTION: + ret = handle_trap_exceptions(vcpu); + break; + default: + vcpu->run->exit_reason = KVM_EXIT_FAIL_ENTRY; + ret = 0; + } + return ret; +} + +/* manually copied from arch/arm64/kernel/traps.c */ +static const char * const esr_class_str[] = { + [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", + [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", + [ESR_ELx_EC_WFx] = "WFI/WFE", + [ESR_ELx_EC_FP_ASIMD] = "ASIMD", + [ESR_ELx_EC_PAC] = "PAC", + [ESR_ELx_EC_BTI] = "BTI", + [ESR_ELx_EC_ILL] = "PSTATE.IL", + [ESR_ELx_EC_SVC64] = "SVC (AArch64)", + [ESR_ELx_EC_HVC64] = "HVC (AArch64)", + [ESR_ELx_EC_SMC64] = "SMC (AArch64)", + [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", + [ESR_ELx_EC_SVE] = "SVE", + [ESR_ELx_EC_ERET] = "ERET/ERETAA/ERETAB", + [ESR_ELx_EC_FPAC] = "FPAC", + [ESR_ELx_EC_SME] = "SME", + [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", + [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", + [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", + [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", + [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", + [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", + [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", + [ESR_ELx_EC_MOPS] = "MOPS", + [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", + [ESR_ELx_EC_GCS] = "Guarded Control Stack", + [ESR_ELx_EC_SERROR] = "SError", + [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", + [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", + [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", + [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", + [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", + [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", + [ESR_ELx_EC_BRK64] = "BRK (AArch64)", +}; + +const char *esr_get_class_string(unsigned long esr) +{ + return esr_class_str[ESR_ELx_EC(esr)]; +} diff --git a/arch/s390/kvm/arm64/handle_exit.h b/arch/s390/kvm/arm64/handle_exit.h new file mode 100644 index 000000000000..65ae58721537 --- /dev/null +++ b/arch/s390/kvm/arm64/handle_exit.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef KVM_ARM64_HANDLE_EXIT_H +#define KVM_ARM64_HANDLE_EXIT_H + +#include + +int handle_exit(struct kvm_vcpu *vcpu); + +#endif /* KVM_ARM64_HANDLE_EXIT_H */ diff --git a/arch/s390/kvm/arm64/inject_fault.c b/arch/s390/kvm/arm64/inject_fault.c new file mode 100644 index 000000000000..425dbeaa421c --- /dev/null +++ b/arch/s390/kvm/arm64/inject_fault.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/** + * kvm_inject_undefined - inject an undefined instruction into the guest + * @vcpu: The vCPU in which to inject the exception + * + * It is assumed that this code is called from the VCPU thread and that the + * VCPU therefore is not currently executing guest code. + */ +void kvm_inject_undefined(struct kvm_vcpu *vcpu) +{ + /* Stub until s390 supports arm64 sysregs TODO sysregs*/ +} + +int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr) +{ + /* Stub until s390 supports arm64 sysregs TODO sysregs*/ + return 1; +} diff --git a/arch/s390/kvm/arm64/mmio.c b/arch/s390/kvm/arm64/mmio.c new file mode 100644 index 000000000000..908a6b43ddf7 --- /dev/null +++ b/arch/s390/kvm/arm64/mmio.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include + +#include +#include + +#include "trace.h" + +#define __INCL_GEN_ARM_FILE +#include "generated/mmio.inc" +#undef __INCL_GEN_ARM_FILE diff --git a/arch/s390/kvm/arm64/mmu.c b/arch/s390/kvm/arm64/mmu.c new file mode 100644 index 000000000000..31e0d036896f --- /dev/null +++ b/arch/s390/kvm/arm64/mmu.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include + +#include + +#include "trace.h" + +static inline bool kvm_s390_cur_gmap_fault_is_write(struct kvm_vcpu *vcpu) +{ + return (vcpu->arch.sae_block.hai.pic & PGM_INT_CODE_MASK) == PGM_PROTECTION || + vcpu->arch.sae_block.hai.teid.fsi == TEID_FSI_STORE; +} + +/* + * user_mem_abort() - handle a dat fault for the gmap of a vcpu + * + * Return: 0 on success, < 0 in case of error. + * Context: The mm lock must not be held before calling. May sleep. + */ +static int user_mem_abort(struct kvm_vcpu *vcpu, gpa_t fault_ipa, + struct kvm_memory_slot *slot, hva_t hva) +{ + struct guest_fault f = { }; + int ret; + + if (kvm_s390_cur_gmap_fault_is_write(vcpu)) + f.write_attempt = FOLL_WRITE; + f.gfn = gpa_to_gfn(fault_ipa); + + ret = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (ret <= 0) + return ret; + if (ret == PGM_ADDRESSING) + /* + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + return -ENOEXEC; + KVM_BUG_ON(ret, vcpu->kvm); + return -EINVAL; +} + +static int kvm_handle_pic(struct kvm_vcpu *vcpu, bool *translation) +{ + switch (kvm_vcpu_fault_pic(vcpu)) { + /* expected cases: */ + case PGM_ASCE_TYPE: + case PGM_REGION_FIRST_TRANS: + case PGM_REGION_SECOND_TRANS: + case PGM_REGION_THIRD_TRANS: + case PGM_SEGMENT_TRANSLATION: + case PGM_PAGE_TRANSLATION: + *translation = true; + break; + case PGM_PROTECTION: + break; + /* unexpected cases: */ + case 0: + KVM_BUG(1, vcpu->kvm, "On MMU fault path but no fault occurred"); + return -EFAULT; + default: + KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx", + vcpu->arch.sae_block.hai.pic, vcpu->arch.sae_block.hai.teid.val); + send_sig(SIGSEGV, current, 0); + return -EFAULT; + } + + return 0; +} + +int kvm_handle_guest_abort(struct kvm_vcpu *vcpu) +{ + struct kvm_memory_slot *memslot; + bool translation = false; + phys_addr_t fault_ipa; + unsigned long hva; + unsigned long esr; + bool write_fault; + bool guest_size_err; + bool writable; + bool is_iabt; + int ret; + gfn_t gfn; + + esr = kvm_vcpu_get_esr(vcpu); + fault_ipa = kvm_vcpu_get_fault_ipa(vcpu); + is_iabt = kvm_vcpu_trap_is_iabt(vcpu); + guest_size_err = vcpu->arch.sae_block.icptr == SAE_ICPTR_GUEST_ADDRESS_SIZE; + + if (guest_size_err) { + translation = true; + } else { + ret = kvm_handle_pic(vcpu, &translation); + if (ret) + return ret; + } + + if (translation) { + /* + * For both cases: + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + if (fault_ipa >= BIT_ULL(get_kvm_ipa_limit())) + return -ENOEXEC; + + if (fault_ipa >= vcpu->kvm->arch.guest_phys_size) + return -ENOEXEC; + } + + trace_kvm_guest_fault(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu), + kvm_vcpu_get_hfar(vcpu), fault_ipa); + + kvm_vcpu_srcu_read_lock(vcpu); + + gfn = fault_ipa >> PAGE_SHIFT; + + memslot = gfn_to_memslot(vcpu->kvm, gfn); + hva = gfn_to_hva_memslot_prot(memslot, gfn, &writable); + write_fault = kvm_s390_cur_gmap_fault_is_write(vcpu); + if (kvm_is_error_hva(hva) || (write_fault && !writable)) { + ret = -ENOEXEC; + /* + * The guest has put either its instructions or its page-tables + * somewhere it shouldn't have. Userspace won't be able to do + * anything about this (there's no syndrome for a start). + * + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + if (is_iabt) + goto out_unlock; + + if (kvm_vcpu_abt_iss1tw(vcpu)) { + /* + * Without the relevant sysregs we cannot do anything for now. + * Go back to userspace with an error. TODO sysreg handling + */ + goto out_unlock; + } + + /* + * Check for a cache maintenance operation. Assume the guest is + * cautious and skip instruction + */ + if (kvm_is_error_hva(hva) && esr_dabt_is_cm(esr)) { + kvm_incr_pc(vcpu); + ret = 1; + goto out_unlock; + } + + /* + * The IPA is reported as [MAX:12], so we need to + * complement it with the bottom 12 bits from the + * faulting VA. This is always 12 bits, irrespective + * of the page size. + */ + fault_ipa |= kvm_vcpu_get_hfar(vcpu) & ((1 << 12) - 1); + ret = io_mem_abort(vcpu, fault_ipa); + goto out_unlock; + } + + /* Userspace should not be able to register out-of-bounds IPAs */ + VM_BUG_ON(fault_ipa >= vcpu->kvm->arch.guest_phys_size); + /* + * Proper guest size faults have been injected. + * In theory it's fine to have device memory higher than MSL, + * even if not currently possible, but that would have been handled above. + * So if we get here with a guest size intercept, we have a bug somewhere. + */ + VM_BUG_ON(guest_size_err); + + ret = user_mem_abort(vcpu, fault_ipa, memslot, hva); + if (!ret) + ret = 1; +out_unlock: + kvm_vcpu_srcu_read_unlock(vcpu); + return ret; +} + +long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) +{ + struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), }; + gpa_t end; + int rc; + + rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); + if (rc == PGM_ADDRESSING) + return -ENOENT; + if (rc > 0) + return -EIO; + if (rc < 0) + return rc; + + if (f.ptep) + return PAGE_SIZE; + + end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE); + return min(range->size, end - range->gpa); +} + +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (!kvm_vcpu_initialized(vcpu)) + return -ENOEXEC; + + return 0; +} diff --git a/arch/s390/kvm/arm64/reset.c b/arch/s390/kvm/arm64/reset.c new file mode 100644 index 000000000000..e4cdce1dbf9d --- /dev/null +++ b/arch/s390/kvm/arm64/reset.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include +#include + +bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu) +{ + return true; +} + +static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) +{ + vcpu->arch.hcr_el2 = HCR_EL2_E2H | HCR_EL2_RW | HCR_EL2_AMO | + HCR_EL2_IMO | HCR_EL2_FMO | HCR_EL2_PTW; + /* traps */ + vcpu->arch.hcr_el2 |= HCR_EL2_TSC | HCR_EL2_TID1 | HCR_EL2_TID2 | + HCR_EL2_TID3 | HCR_EL2_TID4 | HCR_EL2_TID5 | + HCR_EL2_TIDCP | HCR_EL2_TLOR; +} + +void kvm_reset_vcpu(struct kvm_vcpu *vcpu) +{ + struct vcpu_reset_state reset_state; + bool loaded; + + scoped_guard(spinlock, &vcpu->arch.mp_state_lock) { + reset_state = vcpu->arch.reset_state; + vcpu->arch.reset_state.reset = false; + } + + /* + * Disable preemption around the vcpu reset as we might otherwise race with + * preempt notifiers which call stiasrm/lasrm from put/load + */ + preempt_disable(); + + /* The reset must run with an unloaded save area */ + loaded = vcpu_is_loaded(vcpu); + if (loaded) + vcpu_put(vcpu); + + kvm_reset_vcpu_core(vcpu); + + /* Reset special registers */ + vcpu_reset_hcr(vcpu); + + if (reset_state.reset) { + *vcpu_pc(vcpu) = reset_state.pc; + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); + vcpu_clear_flag(vcpu, EXCEPT_MASK); + vcpu_clear_flag(vcpu, INCREMENT_PC); + vcpu_set_reg(vcpu, 0, reset_state.r0); + } + + /* Load new vx-regs into HW if they are currently loaded */ + if (current->thread.kfpu_flags) + load_vx_regs(vcpu->arch.ctxt.vregs); + + if (loaded) + vcpu_load(vcpu); + + preempt_enable(); +} + +int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature) +{ + return -EINVAL; +} diff --git a/arch/s390/kvm/arm64/trace.h b/arch/s390/kvm/arm64/trace.h new file mode 100644 index 000000000000..ee232c4f5a6d --- /dev/null +++ b/arch/s390/kvm/arm64/trace.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_ARM64_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define KVM_ARM64_TRACE_KVM_H + +#include + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm_arm64 +#undef TRACE_SYSTEM_VAR +#define TRACE_SYSTEM_VAR kvm_arm64 + +TRACE_EVENT(kvm_entry, + TP_PROTO(unsigned long vcpu_pc), + TP_ARGS(vcpu_pc), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + ), + + TP_fast_assign(__entry->vcpu_pc = vcpu_pc;), + + TP_printk("PC: 0x%016lx", __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_exit, + TP_PROTO(unsigned int esr_ec, unsigned long vcpu_pc), + TP_ARGS(esr_ec, vcpu_pc), + + TP_STRUCT__entry( + __field(unsigned int, esr_ec ) + __field(unsigned long, vcpu_pc ) + ), + + TP_fast_assign( + __entry->esr_ec = esr_ec; + __entry->vcpu_pc = vcpu_pc; + ), + + TP_printk("HSR_EC: 0x%04x, PC: 0x%016lx", + __entry->esr_ec, + __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_mmio_nisv, + TP_PROTO(unsigned long vcpu_pc, unsigned long esr, + unsigned long far, unsigned long ipa), + TP_ARGS(vcpu_pc, esr, far, ipa), + + TP_STRUCT__entry( + __field( unsigned long, vcpu_pc ) + __field( unsigned long, esr ) + __field( unsigned long, far ) + __field( unsigned long, ipa ) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->esr = esr; + __entry->far = far; + __entry->ipa = ipa; + ), + + TP_printk("ipa %#016lx, esr %#016lx, far %#016lx, pc %#016lx", + __entry->ipa, __entry->esr, + __entry->far, __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_guest_fault, + TP_PROTO(unsigned long vcpu_pc, unsigned long hsr, + unsigned long hxfar, + unsigned long long ipa), + TP_ARGS(vcpu_pc, hsr, hxfar, ipa), + + TP_STRUCT__entry( + __field( unsigned long, vcpu_pc ) + __field( unsigned long, hsr ) + __field( unsigned long, hxfar ) + __field( unsigned long long, ipa ) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->hsr = hsr; + __entry->hxfar = hxfar; + __entry->ipa = ipa; + ), + + TP_printk("ipa %#llx, hsr %#08lx, hxfar %#08lx, pc %#016lx", + __entry->ipa, __entry->hsr, + __entry->hxfar, __entry->vcpu_pc) +); + +#endif /* KVM_ARM64_TRACE_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include diff --git a/arch/s390/kvm/gmap/faultin.c b/arch/s390/kvm/gmap/faultin.c index 1dc79807012c..2e93bd563d5c 100644 --- a/arch/s390/kvm/gmap/faultin.c +++ b/arch/s390/kvm/gmap/faultin.c @@ -10,6 +10,11 @@ #include "gmap.h" #include "faultin.h" +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else +#include "s390.h" +#endif bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu); #define CREATE_TRACE_POINTS diff --git a/arch/s390/kvm/gmap/gmap.c b/arch/s390/kvm/gmap/gmap.c index 3f3fa864cc36..bf91e3d5013b 100644 --- a/arch/s390/kvm/gmap/gmap.c +++ b/arch/s390/kvm/gmap/gmap.c @@ -21,7 +21,11 @@ #include "dat.h" #include "gmap.h" +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else #include "s390.h" +#endif #include "faultin.h" static int gmap_limit_to_type(gfn_t limit) diff --git a/arch/s390/kvm/gmap/kvm_mmu.c b/arch/s390/kvm/gmap/kvm_mmu.c index c2ffb5e59ec6..4e2da60b9859 100644 --- a/arch/s390/kvm/gmap/kvm_mmu.c +++ b/arch/s390/kvm/gmap/kvm_mmu.c @@ -3,7 +3,11 @@ #include #include +#ifdef KVM_S390_ARM64 +#include "arm.h" +#else #include "s390.h" +#endif #include "gmap.h" #include "dat.h" #include "kvm_mmu.h" @@ -76,7 +80,7 @@ int s390_kvm_mmu_prepare_memory_region(struct kvm *kvm, return -EINVAL; } - if (kvm->arch.migration_mode) { + if (kvm_s390_is_migration_mode(kvm)) { /* * Turn off migration mode when: * - userspace creates a new memslot with dirty logging off, diff --git a/arch/s390/kvm/gmap/trace_gmap.h b/arch/s390/kvm/gmap/trace_gmap.h index 15ec88bdce55..1dce3ef3b0d9 100644 --- a/arch/s390/kvm/gmap/trace_gmap.h +++ b/arch/s390/kvm/gmap/trace_gmap.h @@ -4,13 +4,27 @@ #include +#undef UNPACK_KVM +#define UNPACK_KVM(name) name #undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm +#define TRACE_SYSTEM UNPACK_KVM(KVM_DEV_NAME) #undef TRACE_INCLUDE_PATH #define TRACE_INCLUDE_PATH ../gmap #undef TRACE_INCLUDE_FILE #define TRACE_INCLUDE_FILE trace_gmap +#ifdef KVM_S390_ARM64 +#define __KVM_FIELDS \ + __field(unsigned long, pstate) \ + __field(unsigned long, pc) +#define __KVM_ASSIGN ({\ + __entry->pstate = vcpu->arch.sae_block.pstate; \ + __entry->pc = vcpu->arch.sae_block.pc; \ + }) +#define __KVM_PRINT \ + __entry->pstate, \ + __entry->pc +#else #define __KVM_FIELDS \ __field(unsigned long, pswmask) \ __field(unsigned long, pswaddr) @@ -21,6 +35,7 @@ #define __KVM_PRINT \ __entry->pswmask,\ __entry->pswaddr +#endif TRACE_EVENT(kvm_s390_major_guest_pfault, TP_PROTO(struct kvm_vcpu *vcpu), diff --git a/arch/s390/kvm/s390/Makefile b/arch/s390/kvm/s390/Makefile index 762a63826423..b91334db90a2 100644 --- a/arch/s390/kvm/s390/Makefile +++ b/arch/s390/kvm/s390/Makefile @@ -4,7 +4,7 @@ KVM := ../../../../virt/kvm include $(srctree)/virt/kvm/Makefile.kvm include $(srctree)/arch/s390/kvm/gmap/Makefile -ccflags-y := -I$(src) -I$(srctree)/arch/s390/kvm/gmap +ccflags-y += -I$(src) -I$(srctree)/arch/s390/kvm/gmap kvm-y += s390.o intercept.o interrupt.o priv.o sigp.o kvm-y += diag.o gaccess.o guestdbg.o vsie.o pv.o diff --git a/arch/s390/kvm/s390/intercept.c b/arch/s390/kvm/s390/intercept.c index ca1205dfac8b..adfde2a656e7 100644 --- a/arch/s390/kvm/s390/intercept.c +++ b/arch/s390/kvm/s390/intercept.c @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include @@ -411,9 +413,9 @@ static int handle_partial_execution(struct kvm_vcpu *vcpu) */ int handle_sthyi(struct kvm_vcpu *vcpu) { + struct sthyi_sctns *sctns __free(kfree) = NULL; int reg1, reg2, cc = 0, r = 0; u64 code, addr, rc = 0; - struct sthyi_sctns *sctns = NULL; if (!test_kvm_facility(vcpu->kvm, 74)) return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); @@ -430,37 +432,32 @@ int handle_sthyi(struct kvm_vcpu *vcpu) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); if (code & 0xffff) { - cc = 3; - rc = 4; - goto out; + vcpu->run->s.regs.gprs[reg2 + 1] = 4; + kvm_s390_set_psw_cc(vcpu, 3); + return 0; } if (!kvm_s390_pv_cpu_is_protected(vcpu) && (addr & ~PAGE_MASK)) return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); - sctns = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + sctns = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!sctns) return -ENOMEM; cc = sthyi_fill(sctns, &rc); - if (cc < 0) { - free_page((unsigned long)sctns); + if (cc < 0) return cc; - } -out: + if (!cc) { if (kvm_s390_pv_cpu_is_protected(vcpu)) { memcpy(sida_addr(vcpu->arch.sie_block), sctns, PAGE_SIZE); } else { r = write_guest(vcpu, addr, reg2, sctns, PAGE_SIZE); - if (r) { - free_page((unsigned long)sctns); + if (r) return kvm_s390_inject_prog_cond(vcpu, r); - } } } - free_page((unsigned long)sctns); vcpu->run->s.regs.gprs[reg2 + 1] = rc; kvm_s390_set_psw_cc(vcpu, cc); return r; @@ -536,6 +533,15 @@ static int handle_pv_sclp(struct kvm_vcpu *vcpu) set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); clear_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); spin_unlock_irqrestore(&fi->lock, flags); + + /* + * We missed the floating IRQ kick since we can only inject + * when we end up here and not when the irq was injected via + * the FLIC. + * + * Now that we have cleared the masking we can kick cpus. + */ + kvm_s390_pv_sclp_kick(vcpu); return 0; } diff --git a/arch/s390/kvm/s390/interrupt.c b/arch/s390/kvm/s390/interrupt.c index f892f4307883..31a6074feffc 100644 --- a/arch/s390/kvm/s390/interrupt.c +++ b/arch/s390/kvm/s390/interrupt.c @@ -1918,49 +1918,126 @@ static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) return 0; } +static u64 inti_to_irq_pend_mask(u64 type, int isc) +{ + switch (type) { + case KVM_S390_MCHK: + /* Only repressible machine checks are floating */ + return BIT(IRQ_PEND_MCHK_REP); + case KVM_S390_INT_VIRTIO: + return BIT(IRQ_PEND_VIRTIO); + case KVM_S390_INT_SERVICE: + return BIT(IRQ_PEND_EXT_SERVICE) | + BIT(IRQ_PEND_EXT_SERVICE_EV); + case KVM_S390_INT_PFAULT_DONE: + return BIT(IRQ_PEND_PFAULT_DONE); + case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + return BIT(isc_to_irq_type(isc)); + default: + return 0; + } +} + /* - * Find a destination VCPU for a floating irq and kick it. + * Setup intervention masks to catch running vcpus that hopefully open + * their masks soonish and kick sleeping vcpus to motivate them to + * take IRQs. */ -static void __floating_irq_kick(struct kvm *kvm, u64 type) +static void vcpu_intervention_kick(struct kvm_vcpu *vcpu, u64 type) +{ + /* make the VCPU drop out of the SIE, or wake it up if sleeping */ + switch (type) { + case KVM_S390_MCHK: + kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT); + break; + case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + if (!(type & KVM_S390_INT_IO_AI_MASK && + vcpu->kvm->arch.gisa_int.origin) || + kvm_s390_pv_cpu_get_handle(vcpu)) + kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT); + break; + default: + kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); + break; + } + kvm_s390_vcpu_wakeup(vcpu); +} + +static void kick_cpu_irq(struct kvm *kvm, u64 type, u64 parm) { struct kvm_vcpu *dst_vcpu; int sigcpu, online_vcpus, nr_tries = 0; + u64 irq_pend_mask; + unsigned long i; online_vcpus = atomic_read(&kvm->online_vcpus); if (!online_vcpus) return; + irq_pend_mask = inti_to_irq_pend_mask(type, parm); for (sigcpu = kvm->arch.float_int.last_sleep_cpu; ; sigcpu++) { sigcpu %= online_vcpus; dst_vcpu = kvm_get_vcpu(kvm, sigcpu); - if (!is_vcpu_stopped(dst_vcpu)) + if (!is_vcpu_stopped(dst_vcpu) && + deliverable_irqs(dst_vcpu) & irq_pend_mask) break; /* avoid endless loops if all vcpus are stopped */ - if (nr_tries++ >= online_vcpus) - return; + if (nr_tries++ >= online_vcpus) { + dst_vcpu = NULL; + break; + } } - /* make the VCPU drop out of the SIE, or wake it up if sleeping */ - switch (type) { - case KVM_S390_MCHK: - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT); - break; - case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: - if (!(type & KVM_S390_INT_IO_AI_MASK && - kvm->arch.gisa_int.origin) || - kvm_s390_pv_cpu_get_handle(dst_vcpu)) - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT); - break; - default: - kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT); - break; + /* Nobody was enabled, time to wake all of them */ + if (!dst_vcpu) { + kvm_for_each_vcpu(i, dst_vcpu, kvm) + vcpu_intervention_kick(dst_vcpu, type); + return; } - kvm_s390_vcpu_wakeup(dst_vcpu); + + vcpu_intervention_kick(dst_vcpu, type); +} + +void kvm_s390_pv_sclp_kick(struct kvm_vcpu *vcpu) +{ + /* + * The cpu that called sclp likely will also take the IRQ, no + * need to kick anyone. + */ + if (deliverable_irqs(vcpu) & BIT(IRQ_PEND_EXT_SERVICE)) + return; + + /* + * For the other cases we might have sleeping cpus with open + * masks. Time to find and kick them. + */ + kick_cpu_irq(vcpu->kvm, KVM_S390_INT_SERVICE, -1); +} + +/* + * Find a destination VCPU for a floating irq and kick it. + */ +static void __floating_irq_kick(struct kvm *kvm, u64 type, u64 parm) +{ + struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; + + /* + * No need to kick on non-ev service IRQs for PV VMs, we're + * not allowed to inject anyway. We need to wait for the sclp + * instruction notification AFTER re-entry of the vcpu that + * handled the instruction intercept. + */ + if (type == KVM_S390_INT_SERVICE && !(parm & SCCB_EVENT_PENDING) && + test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs)) + return; + + kick_cpu_irq(kvm, type, parm); } static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) { u64 type = READ_ONCE(inti->type); + u64 parm = -1; int rc; switch (type) { @@ -1971,12 +2048,15 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) rc = __inject_virtio(kvm, inti); break; case KVM_S390_INT_SERVICE: + parm = inti->ext.ext_params & SCCB_EVENT_PENDING; rc = __inject_service(kvm, inti); break; case KVM_S390_INT_PFAULT_DONE: rc = __inject_pfault_done(kvm, inti); break; case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: + /* Grab isc here since __inject_io() might free inti */ + parm = int_word_to_isc(inti->io.io_int_word); rc = __inject_io(kvm, inti); break; default: @@ -1985,7 +2065,7 @@ static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) if (rc) return rc; - __floating_irq_kick(kvm, type); + __floating_irq_kick(kvm, type, parm); return 0; } @@ -3690,7 +3770,7 @@ void kvm_s390_gib_destroy(void) } chsc_sgib(0); unregister_adapter_interrupt(&gib_alert_irq); - free_page((unsigned long)gib); + kfree(gib); gib = NULL; } @@ -3704,7 +3784,7 @@ int __init kvm_s390_gib_init(u8 nisc) goto out; } - gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA); + gib = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT | GFP_DMA); if (!gib) { rc = -ENOMEM; goto out; @@ -3723,7 +3803,7 @@ int __init kvm_s390_gib_init(u8 nisc) gib_origin = virt_to_phys(gib); if (chsc_sgib(gib_origin)) { pr_err("Associating the GIB with the AIV facility failed\n"); - free_page((unsigned long)gib); + kfree(gib); gib = NULL; rc = -EIO; goto out_unreg_gal; @@ -3743,7 +3823,7 @@ int __init kvm_s390_gib_init(u8 nisc) out_unreg_gal: unregister_adapter_interrupt(&gib_alert_irq); out_free_gib: - free_page((unsigned long)gib); + kfree(gib); gib = NULL; out: return rc; diff --git a/arch/s390/kvm/s390/pci.c b/arch/s390/kvm/s390/pci.c index 82892e1e03d9..d79bd3bdc68e 100644 --- a/arch/s390/kvm/s390/pci.c +++ b/arch/s390/kvm/s390/pci.c @@ -498,17 +498,22 @@ static void kvm_s390_pci_dev_release(struct zpci_dev *zdev) * available, enable them and let userspace indicate whether or not they will * be used (specify SHM bit to disable). */ -static int kvm_s390_pci_register_kvm(void *opaque, struct kvm *kvm) +static int kvm_s390_pci_register_kvm(void *opaque, struct file *kvm_file) { struct zpci_dev *zdev = opaque; + struct kvm *kvm; int rc; if (!zdev) return -EINVAL; + kvm = file_to_kvm_s390(kvm_file); + if (!kvm) + return -ENOENT; + mutex_lock(&zdev->kzdev_lock); - if (zdev->kzdev || zdev->gisa != 0 || !kvm) { + if (zdev->kzdev || zdev->gisa != 0) { mutex_unlock(&zdev->kzdev_lock); return -EINVAL; } diff --git a/arch/s390/kvm/s390/priv.c b/arch/s390/kvm/s390/priv.c index b3cb2c2c3aa7..856939acf012 100644 --- a/arch/s390/kvm/s390/priv.c +++ b/arch/s390/kvm/s390/priv.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -869,7 +870,7 @@ static int handle_stsi(struct kvm_vcpu *vcpu) int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; int sel1 = vcpu->run->s.regs.gprs[0] & 0xff; int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff; - unsigned long mem = 0; + void *mem = NULL; u64 operand2; int rc = 0; u8 ar; @@ -911,19 +912,19 @@ static int handle_stsi(struct kvm_vcpu *vcpu) switch (fc) { case 1: /* same handling for 1 and 2 */ case 2: - mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); + mem = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!mem) goto out_no_data; - if (stsi((void *) mem, fc, sel1, sel2)) + if (stsi(mem, fc, sel1, sel2)) goto out_no_data; break; case 3: if (sel1 != 2 || sel2 != 2) goto out_no_data; - mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); + mem = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!mem) goto out_no_data; - handle_stsi_3_2_2(vcpu, (void *) mem); + handle_stsi_3_2_2(vcpu, mem); break; case 15: /* fc 15 is fully handled in userspace */ insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); @@ -931,10 +932,10 @@ static int handle_stsi(struct kvm_vcpu *vcpu) return -EREMOTE; } if (kvm_s390_pv_cpu_is_protected(vcpu)) { - memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE); + memcpy(sida_addr(vcpu->arch.sie_block), mem, PAGE_SIZE); rc = 0; } else { - rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); + rc = write_guest(vcpu, operand2, ar, mem, PAGE_SIZE); } if (rc) { rc = kvm_s390_inject_prog_cond(vcpu, rc); @@ -945,14 +946,14 @@ static int handle_stsi(struct kvm_vcpu *vcpu) rc = -EREMOTE; } trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); - free_page(mem); + kfree(mem); kvm_s390_set_psw_cc(vcpu, 0); vcpu->run->s.regs.gprs[0] = 0; return rc; out_no_data: kvm_s390_set_psw_cc(vcpu, 3); out: - free_page(mem); + kfree(mem); return rc; } diff --git a/arch/s390/kvm/s390/s390.c b/arch/s390/kvm/s390/s390.c index e4912db67adc..18fb200f854a 100644 --- a/arch/s390/kvm/s390/s390.c +++ b/arch/s390/kvm/s390/s390.c @@ -3273,8 +3273,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) goto out_err; BUILD_BUG_ON(sizeof(struct sie_page2) != 4096); - kvm->arch.sie_page2 = - (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA); + kvm->arch.sie_page2 = kzalloc_obj(*kvm->arch.sie_page2, + GFP_KERNEL_ACCOUNT | GFP_DMA); if (!kvm->arch.sie_page2) goto out_err; @@ -3369,7 +3369,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) return 0; out_err: - free_page((unsigned long)kvm->arch.sie_page2); + kfree(kvm->arch.sie_page2); debug_unregister(kvm->arch.dbf); sca_dispose(kvm); KVM_EVENT(3, "creation of vm failed: %d", rc); @@ -3427,7 +3427,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm) mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm); debug_unregister(kvm->arch.dbf); - free_page((unsigned long)kvm->arch.sie_page2); + kfree(kvm->arch.sie_page2); kvm_s390_destroy_adapters(kvm); kvm_s390_clear_float_irqs(kvm); kvm_s390_vsie_destroy(kvm); @@ -3656,13 +3656,13 @@ static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) { if (vcpu->arch.sie_block->cbrlo) - free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo)); + kfree(phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK)); vcpu->arch.sie_block->cbrlo = 0; } int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) { - void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); + void *cbrlo_page = kzalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); if (!cbrlo_page) return -ENOMEM; @@ -5780,6 +5780,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old, { } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (kvm_is_ucontrol(vcpu->kvm)) + return -EINVAL; + + return 0; +} + /** * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables * @vcpu: the vcpu that shall appear to have generated the fault-in. @@ -5806,9 +5814,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_ gpa_t end; int rc; - if (kvm_is_ucontrol(vcpu->kvm)) - return -EINVAL; - rc = kvm_s390_faultin_gfn(vcpu, NULL, &f); if (rc == PGM_ADDRESSING) return -ENOENT; diff --git a/arch/s390/kvm/s390/s390.h b/arch/s390/kvm/s390/s390.h index aa0d1d062f8d..9ed1af5583e3 100644 --- a/arch/s390/kvm/s390/s390.h +++ b/arch/s390/kvm/s390/s390.h @@ -375,6 +375,7 @@ enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer); int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu); void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu); void kvm_s390_clear_float_irqs(struct kvm *kvm); +void kvm_s390_pv_sclp_kick(struct kvm_vcpu *vcpu); int __must_check kvm_s390_inject_vm(struct kvm *kvm, struct kvm_s390_interrupt *s390int, struct kvm_s390_interrupt_info *inti); diff --git a/arch/s390/tools/Makefile b/arch/s390/tools/Makefile index ba1e42d005e7..e72b48e236ad 100644 --- a/arch/s390/tools/Makefile +++ b/arch/s390/tools/Makefile @@ -3,14 +3,18 @@ # Makefile for s390 specific build tools # +include $(srctree)/arch/s390/tools/Makefile.arm64h + kapi := arch/$(ARCH)/include/generated/asm kapi-hdrs-y := $(kapi)/dis-defs.h kapi-hdrs-y += $(kapi)/facility-defs.h kapi-hdrs-y += $(kapi)/pgmcheck-defs.h +kapi-hdrs-$(CONFIG_KVM) += $(ARM64_TARGETS) + PHONY += kapi -kapi: $(kapi-hdrs-y) +kapi: $(kapi-hdrs-y) $(kapi-hdrs-m) hostprogs += gen_bitmaps hostprogs += gen_opcode_table diff --git a/arch/s390/tools/Makefile.arm64h b/arch/s390/tools/Makefile.arm64h new file mode 100644 index 000000000000..f3f8e9d9bdd8 --- /dev/null +++ b/arch/s390/tools/Makefile.arm64h @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Share ARM64 headers with s390 KVM +# + +arm64api := $(objtree)/arch/$(ARCH)/include/generated/arm64 +arm64uapi := $(objtree)/arch/$(ARCH)/include/generated/uapi/arm64 + +# List of ARM64 headers to share with s390 KVM +# Format: [uapi:]header.h or header-gen.h +# +# * header.h - from arch/arm64/include/asm/ to arm64/ +# * uapi:header.h - from arch/arm64/include/uapi/asm/ to uapi/arm64/ +# * header-gen.h - generated as header-gen.h from ARM64 header header.h +# If a header contains partial markers (#if ARM64_S390_COMMON == 1) only this part will be copied. +# If the file does not contain any markers the whole file will be copied. +ARM64_SHARED_HEADERS := \ + brk-imm.h \ + esr.h \ + kvm_arm.h \ + kvm_emulate-gen.h \ + kvm_host.h \ + ptrace-gen.h \ + sysreg-gen.h \ + uapi:hwcap.h \ + uapi:kvm.h \ + uapi:ptrace.h \ + uapi:sve_context.h + +filechk_gen_arm_hdr = \ + $(AWK) -f $(src)/copy-arm64h.awk $< || \ + $(AWK) -f $(src)/copy-arm64h-full.awk $< || \ + { echo "Error: Unbalanced ARM64_S390_COMMON markers in $<" >&2; exit 1; } + +arm64-type = $(firstword $(subst :, ,$(1))) +arm64-base = $(or $(word 2,$(subst :, ,$(1))),$(1)) +arm64-src-base = $(patsubst %-gen.h,%.h,$(call arm64-base,$(1))) + +arm64-src = $(srctree)/arch/arm64/include/$(if $(filter uapi,$(call arm64-type,$(1))),uapi/)asm/$(call arm64-src-base,$(1)) +arm64-dst = $(if $(filter uapi,$(call arm64-type,$(1))),$(arm64uapi),$(arm64api))/$(call arm64-base,$(1)) + +$(foreach hdr,$(ARM64_SHARED_HEADERS),\ + $(eval $(call arm64-dst,$(hdr)): $(call arm64-src,$(hdr)) $(src)/copy-arm64h.awk $(src)/copy-arm64h-full.awk FORCE)) + +ARM64_TARGETS := $(sort $(foreach hdr,$(ARM64_SHARED_HEADERS),$(call arm64-dst,$(hdr)))) +$(ARM64_TARGETS): + $(call filechk,gen_arm_hdr) + +filechk_gen_sysreg_arm64 = \ + $(AWK) -f $(srctree)/arch/arm64/tools/gen-sysreg.awk $(srctree)/arch/arm64/tools/sysreg + +$(arm64api)/sysreg-defs.h: $(srctree)/arch/arm64/tools/gen-sysreg.awk $(srctree)/arch/arm64/tools/sysreg FORCE + $(call filechk,gen_sysreg_arm64) + +ARM64_TARGETS += $(arm64api)/sysreg-defs.h diff --git a/arch/s390/tools/copy-arm64h-full.awk b/arch/s390/tools/copy-arm64h-full.awk new file mode 100644 index 000000000000..ad7bb98dc4a6 --- /dev/null +++ b/arch/s390/tools/copy-arm64h-full.awk @@ -0,0 +1,50 @@ +#!/usr/bin/awk -f +# SPDX-License-Identifier: GPL-2.0 +# +# Process entire ARM64 headers for sharing with s390 KVM +# +# Usage: copy-arm64h-full.awk +# +# This processes the entire file (unlike copy-arm64h.awk which only extracts marked sections) + +BEGIN { + max_guard_line = 25 + guard_found = 0 + header_added = 0 +} + +# Find and process the include guard in the first few lines +NR <= max_guard_line && !guard_found && /^#ifndef [A-Z_0-9]+$/ { + guard_name = $2 + guard_found = 1 + print "/* This header was copied from " FILENAME " */" + print "" + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " NR " \"" FILENAME "\"" + print + header_added = 1 + next +} + +# Transform include directives +/^#include +# +# Extracts all sections between start/end markers. If no markers found, signals for fallback. +BEGIN { + # Constants + start_pattern = "^#if ARM64_S390_COMMON == 1$" + end_pattern = "^#endif \\/\\* ARM64_S390_COMMON == 1 \\*\\/$" + max_guard_line = 25 + + # State variables + copying = found_marker = error = 0 + guard_name = "" + file_header_done = 0 + depth = 0 +} + +NR <= max_guard_line && !guard_name && /^#ifndef [A-Za-z0-9_]+$/ { + guard_name = $2 + next +} + +NR > max_guard_line && !guard_name && !file_header_done { + print "error: no include guard found in first " max_guard_line " lines" > "/dev/stderr" + file_header_done = 1 + error = 1; exit +} + +$0 ~ start_pattern { + if (copying) { error = 1; exit } + if (!found_marker) { + print "/*" + print " * This header was automatically generated from " FILENAME + print " * Do not modify this file directly." + print " */" + print "#ifndef " guard_name + print "#define " guard_name + print "" + } + depth = copying= found_marker = 1 + # Map subsequent lines back to the original ARM64 source file so that + # compiler diagnostics point there rather than at the generated copy. + print "#line " (NR + 1) " \"" FILENAME "\"" + next +} + +guard_name && !found_marker { + next +} + +$0 ~ /^[ \t]*#[ \t]*endif/ { + depth-- + if (depth == 0) { + if ($0 ~ end_pattern ) { + copying = 0 + next + } + error = 1; exit + } +} + +/ARM64_S390_COMMON/ { error = 1; exit } + +copying { + if ($0 ~ /^[ \t]*#[ \t]*if(def|ndef)?([ \t]|$)/) depth++ + gsub(/#include #include #include +#include #include #include @@ -44,7 +45,12 @@ #include +#define kvm_file_to_kvm_arch x86 + #define __KVM_HAVE_ARCH_VCPU_DEBUGFS +#define __KVM_HAVE_MSI +#define __KVM_HAVE_NO_POLL +#define __KVM_HAVE_READONLY_MEM /* * CONFIG_KVM_MAX_NR_VCPUS is defined iff CONFIG_KVM!=n, provide a dummy max if @@ -779,6 +785,8 @@ struct kvm_vcpu_arch { */ struct kvm_mmu_memory_cache mmu_external_spt_cache; + struct page *pml_page; + /* * QEMU userspace and the guest each have their own FPU state. * In vcpu_run, we switch between the user and guest FPU contexts. @@ -855,7 +863,7 @@ struct kvm_vcpu_arch { gpa_t time; s8 pvclock_tsc_shift; u32 pvclock_tsc_mul; - unsigned int hw_tsc_khz; + u64 hw_tsc_hz; struct gfn_to_pfn_cache pv_time; /* set guest stopped flag in pvclock flags field */ bool pvclock_set_guest_stopped_request; @@ -964,7 +972,7 @@ struct kvm_vcpu_arch { bool pv_unhalted; } pv; - int pending_ioapic_eoi; + u8 pending_ioapic_eoi; int pending_external_vector; int highest_stale_pending_ioapic_eoi; @@ -1104,7 +1112,7 @@ struct kvm_xen { bool runstate_update_flag; u8 upcall_vector; struct gfn_to_pfn_cache shinfo_cache; - struct idr evtchn_ports; + struct xarray evtchn_ports; unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; struct kvm_xen_hvm_config hvm_config; @@ -1156,6 +1164,7 @@ struct kvm_arch { u8 mmu_valid_gen; u8 vm_type; bool has_private_mem; + bool has_protected_page_tables; bool has_protected_state; bool has_protected_eoi; bool has_protected_pmu; @@ -1227,6 +1236,7 @@ struct kvm_arch { u64 last_tsc_write; u32 last_tsc_khz; u64 last_tsc_offset; + u64 last_tsc_scaling_ratio; u64 cur_tsc_nsec; u64 cur_tsc_write; u64 cur_tsc_offset; @@ -1242,6 +1252,9 @@ struct kvm_arch { u64 master_kernel_ns; u64 master_cycle_now; struct ratelimit_state kvmclock_update_rs; + u64 master_tsc_scaling_ratio; + s8 master_tsc_shift; + u32 master_tsc_mul; #ifdef CONFIG_KVM_HYPERV struct kvm_hv hyperv; @@ -1397,6 +1410,7 @@ struct kvm_arch { struct kvm_mmu_memory_cache split_desc_cache; gfn_t gfn_direct_bits; + int mirror_root_level; /* * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A Zero @@ -1569,7 +1583,7 @@ struct kvm_x86_ops { * Can potentially get non-canonical addresses through INVLPGs, which * the implementation may choose to ignore if appropriate. */ - void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr, bool *full); + void (*flush_tlb_gva)(struct kvm_vcpu *vcpu, gva_t addr); /* * Flush any TLB entries created by the guest. Like tlb_flush_gva(), @@ -1598,6 +1612,13 @@ struct kvm_x86_ops { int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); + /* + * Force the vCPU out of any hardware-tracked halted/inactive state so + * that it will fetch and execute instructions on the next VM-Enter. + * Only needed by VMX, where VMCS.GUEST_ACTIVITY_STATE persists across + * VM-Exit/VM-Enter; SVM has no equivalent VMCB field. + */ + void (*clear_hlt)(struct kvm_vcpu *vcpu); /* Whether or not a virtual NMI is pending in hardware. */ bool (*is_vnmi_pending)(struct kvm_vcpu *vcpu); /* @@ -1661,7 +1682,7 @@ struct kvm_x86_ops { struct x86_exception *exception); void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); - void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); + void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu, bool enable); void (*vcpu_blocking)(struct kvm_vcpu *vcpu); void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); @@ -1724,7 +1745,7 @@ struct kvm_x86_ops { int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +#if defined(CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT) || defined(CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM) void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE @@ -1843,9 +1864,14 @@ enum kvm_intr_type { ((vcpu) && (vcpu)->arch.handling_intr_from_guest && \ (!!in_nmi() == ((vcpu)->arch.handling_intr_from_guest == KVM_HANDLING_NMI))) -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#if defined(CONFIG_KVM_SW_PROTECTED_VM) || \ + defined(CONFIG_KVM_INTEL_TDX) || \ + defined(CONFIG_KVM_AMD_SEV) #define kvm_arch_has_private_mem(kvm) ((kvm)->arch.has_private_mem) #endif +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() (!!kvm_x86_ops.gmem_make_private) +#endif #define kvm_arch_has_readonly_mem(kvm) (!(kvm)->arch.has_protected_state) diff --git a/arch/x86/include/asm/kvm_page_track.h b/arch/x86/include/asm/kvm_page_track.h index 3d040741044b..ea885dd5c9af 100644 --- a/arch/x86/include/asm/kvm_page_track.h +++ b/arch/x86/include/asm/kvm_page_track.h @@ -44,13 +44,15 @@ struct kvm_page_track_notifier_node { struct kvm_page_track_notifier_node *node); }; -int kvm_page_track_register_notifier(struct kvm *kvm, +struct file; + +int kvm_page_track_register_notifier(struct file *file, struct kvm_page_track_notifier_node *n); -void kvm_page_track_unregister_notifier(struct kvm *kvm, +void kvm_page_track_unregister_notifier(struct file *file, struct kvm_page_track_notifier_node *n); -int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn); -int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn); +int kvm_write_track_add_gfn(struct file *file, gfn_t gfn); +int kvm_write_track_remove_gfn(struct file *file, gfn_t gfn); #else /* * Allow defining a node in a structure even if page tracking is disabled, e.g. diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index aa63431ba92c..83fd49b26a4b 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -165,7 +165,10 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u8 reserved_9[22]; u64 allowed_sev_features; /* Offset 0x138 */ u64 guest_sev_features; /* Offset 0x140 */ - u8 reserved_10[664]; + u8 reserved_10[128]; + u64 pml_addr; /* Offset 0x1c8 */ + u16 pml_index; /* Offset 0x1d0 */ + u8 reserved_11[526]; /* * Offset 0x3e0, 32 bytes reserved * for use by hypervisor/software. @@ -240,10 +243,11 @@ struct __attribute__ ((__packed__)) vmcb_control_area { #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT) #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT) -#define SVM_MISC_ENABLE_NP BIT(0) -#define SVM_MISC_ENABLE_SEV BIT(1) -#define SVM_MISC_ENABLE_SEV_ES BIT(2) -#define SVM_MISC_ENABLE_GMET BIT(3) +#define SVM_MISC_ENABLE_NP BIT_ULL(0) +#define SVM_MISC_ENABLE_SEV BIT_ULL(1) +#define SVM_MISC_ENABLE_SEV_ES BIT_ULL(2) +#define SVM_MISC_ENABLE_GMET BIT_ULL(3) +#define SVM_MISC_ENABLE_PML BIT_ULL(11) #define SVM_MISC2_ENABLE_V_LBR BIT_ULL(0) #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE BIT_ULL(1) @@ -637,8 +641,8 @@ static inline void __unused_size_checks(void) #define SVM_EVTINJ_TYPE_EXEPT (3 << SVM_EVTINJ_TYPE_SHIFT) #define SVM_EVTINJ_TYPE_SOFT (4 << SVM_EVTINJ_TYPE_SHIFT) -#define SVM_EVTINJ_VALID (1 << 31) -#define SVM_EVTINJ_VALID_ERR (1 << 11) +#define SVM_EVTINJ_VALID BIT(31) +#define SVM_EVTINJ_VALID_ERR BIT(11) #define SVM_EVTINJ_RESERVED_BITS ~(SVM_EVTINJ_VEC_MASK | SVM_EVTINJ_TYPE_MASK | \ SVM_EVTINJ_VALID_ERR | SVM_EVTINJ_VALID) diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 3f1b3096ff04..e49b836c7313 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -20,6 +20,14 @@ #include #include +/* + * UNDEFINED is a KVM-defined value used to indicate that the basic exit reason + * is undefined/invalid, e.g. on VM-Fail or if TDX never attempted VM-Enter. + * The value itself is arbitrary, it just needs to not collide with a real exit + * reason. + */ +#define EXIT_REASON_UNDEFINED 0xdead + struct vmcs_hdr { u32 revision_id:31; u32 shadow_vmcs:1; diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 1585ec804066..00ce265391e1 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -45,9 +45,7 @@ #define __KVM_HAVE_PIT #define __KVM_HAVE_IOAPIC #define __KVM_HAVE_IRQ_LINE -#define __KVM_HAVE_MSI #define __KVM_HAVE_USER_NMI -#define __KVM_HAVE_MSIX #define __KVM_HAVE_MCE #define __KVM_HAVE_PIT_STATE2 #define __KVM_HAVE_XEN_HVM diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 010a45c9f614..e80676185092 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -101,6 +101,7 @@ #define SVM_EXIT_AVIC_INCOMPLETE_IPI 0x401 #define SVM_EXIT_AVIC_UNACCELERATED_ACCESS 0x402 #define SVM_EXIT_VMGEXIT 0x403 +#define SVM_EXIT_PML_FULL 0x407 /* SEV-ES software-defined VMGEXIT events */ #define SVM_VMGEXIT_MMIO_READ 0x80000001ull @@ -236,6 +237,7 @@ { SVM_EXIT_AVIC_INCOMPLETE_IPI, "avic_incomplete_ipi" }, \ { SVM_EXIT_AVIC_UNACCELERATED_ACCESS, "avic_unaccelerated_access" }, \ { SVM_EXIT_VMGEXIT, "vmgexit" }, \ + { SVM_EXIT_PML_FULL, "pml_full" }, \ { SVM_VMGEXIT_MMIO_READ, "vmgexit_mmio_read" }, \ { SVM_VMGEXIT_MMIO_WRITE, "vmgexit_mmio_write" }, \ { SVM_VMGEXIT_NMI_COMPLETE, "vmgexit_nmi_complete" }, \ diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c index 580b252e07cc..d867ab90d42a 100644 --- a/arch/x86/kernel/cpu/scattered.c +++ b/arch/x86/kernel/cpu/scattered.c @@ -55,6 +55,7 @@ static const struct cpuid_bit cpuid_bits[] = { { X86_FEATURE_AMD_FAST_CPPC, CPUID_EDX, 15, 0x80000007, 0 }, { X86_FEATURE_CPPC_PERF_PRIO, CPUID_EDX, 16, 0x80000007, 0 }, { X86_FEATURE_MBA, CPUID_EBX, 6, 0x80000008, 0 }, + { X86_FEATURE_PML, CPUID_ECX, 4, 0x8000000a, 0 }, { X86_FEATURE_X2AVIC_EXT, CPUID_ECX, 6, 0x8000000a, 0 }, { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 }, { X86_FEATURE_SMBA, CPUID_EBX, 2, 0x80000020, 0 }, diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 538ed1e80332..1719059570fb 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -28,18 +28,14 @@ config KVM_X86 select HAVE_KVM_DIRTY_RING_ACQ_REL select HAVE_KVM_IRQ_BYPASS select HAVE_KVM_IRQ_ROUTING - select HAVE_KVM_READONLY_MEM select VHOST_TASK select KVM_ASYNC_PF select USER_RETURN_NOTIFIER - select KVM_MMIO select SCHED_INFO select PERF_EVENTS select GUEST_PERF_EVENTS select PERF_GUEST_MEDIATED_PMU - select HAVE_KVM_MSI select HAVE_KVM_CPU_RELAX_INTERCEPT - select HAVE_KVM_NO_POLL select VIRT_XFER_TO_GUEST_WORK select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_VFIO @@ -80,11 +76,24 @@ config KVM_WERROR If in doubt, say "N". +config KVM_PRIVATE_MEMATTR_VM + bool "Enable per-VM PRIVATE vs. SHARED attributes (for CoCo VMs)" + depends on KVM_SW_PROTECTED_VM || KVM_INTEL_TDX || KVM_AMD_SEV + select KVM_VM_MEMORY_ATTRIBUTES + default KVM_INTEL_TDX || KVM_AMD_SEV + help + Enable support for tracking PRIVATE vs. SHARED memory using per-VM + memory attributes. Using per-VM attributes is deprecated in favor of + tracking PRIVATE state in guest_memfd. Select this if you need to run + CoCo VMs using a VMM that doesn't support guest_memfd memory + attributes. + + If unsure, say Y. + config KVM_SW_PROTECTED_VM bool "Enable support for KVM software-protected VMs" depends on EXPERT depends on KVM_X86 && X86_64 - select KVM_GENERIC_MEMORY_ATTRIBUTES help Enable support for KVM software-protected VMs. Currently, software- protected VMs are purely a development and testing vehicle for @@ -135,7 +144,6 @@ config KVM_INTEL_TDX bool "Intel Trust Domain Extensions (TDX) support" default y depends on INTEL_TDX_HOST - select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_POPULATE help Provides support for launching Intel Trust Domain Extensions (TDX) @@ -159,7 +167,6 @@ config KVM_AMD_SEV depends on KVM_AMD && X86_64 depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) select ARCH_HAS_CC_PLATFORM - select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_CONVERT select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 0474604ab8a1..c6bf463f5032 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -61,9 +61,7 @@ exports_grep_trailer := --include='*.[ch]' -nrw $(srctree)/virt/kvm $(srctree)/a -e kvm_page_track_unregister_notifier \ -e kvm_write_track_add_gfn \ -e kvm_write_track_remove_gfn \ - -e kvm_get_kvm \ - -e kvm_get_kvm_safe \ - -e kvm_put_kvm + -e kvm_file_to_kvm_fn # Force grep to emit a goofy group separator that can in turn be replaced with # the above newline macro (newlines in Make are a nightmare). Note, grep only diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 851f151efb35..adafd6fa2c54 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -483,20 +484,6 @@ int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu) return 36; } -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu) -{ - struct kvm_cpuid_entry2 *best; - - best = kvm_find_cpuid_entry(vcpu, 0x80000000); - if (!best || best->eax < 0x80000008) - goto not_found; - best = kvm_find_cpuid_entry(vcpu, 0x80000008); - if (best) - return (best->eax >> 16) & 0xff; -not_found: - return 0; -} - /* * This "raw" version returns the reserved GPA bits without any adjustments for * encryption technologies that usurp bits. The raw mask should be used if and @@ -1095,7 +1082,7 @@ void kvm_initialize_cpu_caps(void) ); kvm_cpu_cap_init(CPUID_24_1_ECX, - F(AVX10_VNNI_INT), + F(AVX10_V1_AUX), ); kvm_cpu_cap_init(CPUID_8000_0001_ECX, @@ -1171,6 +1158,10 @@ void kvm_initialize_cpu_caps(void) F(AMD_STIBP), F(AMD_STIBP_ALWAYS_ON), F(AMD_IBRS_SAME_MODE), + /* + * Vendor code also sets EFER_LMSLE_MBZ if KVM itself + * can't support EFER.LMSLE, e.g. if nested SVM is disabled. + */ PASSTHROUGH_F(EFER_LMSLE_MBZ), F(AMD_PSFD), F(AMD_IBPB_RET), @@ -1415,6 +1406,22 @@ static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 ind if (kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) entry->ecx = feature_bit(RDPID); return 1; + case 0x80000008: + /* + * Honor the guest's EFER_LMSLE_MBZ even if the underlying CPU + * allows setting EFER.LMSLE, e.g. to allow migrating a vCPU + * between hosts with and without EFER.LMSLE support. To avoid + * breaking existing setups that reflect KVM's supported CPUID + * into the guest, KVM doesn't advertise EFER_LMSLE_MBZ unless + * KVM *can't* support EFER.LMSLE=1. + */ + if (include_partially_emulated && + !kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) { + entry->ebx |= feature_bit(EFER_LMSLE_MBZ); + return 1; + } + /* Nothing in 0x80000008 is fully emulated, don't emit an entry. */ + return 0; default: return 0; } diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 8d863f45585d..46bfe8699e67 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -68,7 +68,6 @@ void __init kvm_init_xstate_sizes(void); u32 xstate_required_size(u64 xstate_bv, bool compacted); int cpuid_query_maxphyaddr(struct kvm_vcpu *vcpu); -int cpuid_query_maxguestphyaddr(struct kvm_vcpu *vcpu); u64 kvm_vcpu_reserved_gpa_bits_raw(struct kvm_vcpu *vcpu); static inline int cpuid_maxphyaddr(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 604651cb2739..5d2e3617c952 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -802,7 +802,9 @@ static int synic_deliver_msg(struct kvm_vcpu_hv_synic *synic, u32 sint, if (no_retry) return 0; + hv_hdr.message_flags.asu8 = 0; hv_hdr.message_flags.msg_pending = 1; + r = kvm_vcpu_write_guest_page(vcpu, msg_page_gfn, &hv_hdr.message_flags, msg_off + @@ -1995,7 +1997,6 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; int i, j, count; gva_t gva; - bool full = false; if (!tdp_enabled) return -EINVAL; @@ -2006,7 +2007,7 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); - for (i = 0; i < count && !full; i++) { + for (i = 0; i < count; i++) { if (entries[i] == KVM_HV_TLB_FLUSHALL_ENTRY) goto out_flush_all; @@ -2015,11 +2016,11 @@ int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) * pages to flush. */ gva = entries[i] & PAGE_MASK; - for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1 && !full; j++) { + for (j = 0; j < (entries[i] & ~PAGE_MASK) + 1; j++) { if (is_noncanonical_invlpg_address(gva + j * PAGE_SIZE, vcpu)) continue; - kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE, &full); + kvm_x86_call(flush_tlb_gva)(vcpu, gva + j * PAGE_SIZE); } ++vcpu->stat.tlb_flush; diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 3e375af15c03..10886bea82c9 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -354,8 +354,8 @@ struct x86_emulate_ctxt { bool have_exception; struct x86_exception exception; - /* GPA available */ - bool gpa_available; + /* Saved GPA and permitted emulated accesses. */ + u64 gpa_access; gpa_t gpa_val; /* diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index e1f3cea14765..ecf611845fb1 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1552,7 +1552,7 @@ static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector) return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors); } -static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector) +static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, u8 vector) { int __maybe_unused trigger_mode; @@ -1622,7 +1622,7 @@ static int apic_set_eoi(struct kvm_lapic *apic) * this interface assumes a trap-like exit, which has already finished * desired side effect including vISR and vPPR update. */ -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector) +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector) { struct kvm_lapic *apic = vcpu->arch.apic; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index bd1098c89d99..db071d0b172d 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -177,7 +177,7 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data); void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset); -void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector); +void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, u8 vector); int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr); void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 46c033d87d39..dc2aef72ffdd 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -868,11 +868,12 @@ void untrack_possible_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp, list_del_init(&sp->possible_nx_huge_page_link); } -static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp) +static void unaccount_nx_huge_page(struct kvm *kvm, struct kvm_mmu_page *sp, + enum kvm_mmu_type mmu_type) { sp->nx_huge_page_disallowed = false; - untrack_possible_nx_huge_page(kvm, sp, KVM_SHADOW_MMU); + untrack_possible_nx_huge_page(kvm, sp, mmu_type); } static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, @@ -899,6 +900,7 @@ static struct kvm_memory_slot *gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu */ #define KVM_RMAP_MANY BIT(0) +#ifndef CONFIG_PREEMPT_RT /* * rmaps and PTE lists are mostly protected by mmu_lock (the shadow MMU always * operates with mmu_lock held for write), but rmaps can be walked without @@ -1012,7 +1014,8 @@ static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head) * actual locking is the same, but the caller is disallowed from modifying the * rmap, and so the unlock flow is a nop if the rmap is/was empty. */ -static unsigned long kvm_rmap_lock_readonly(struct kvm_rmap_head *rmap_head) +static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) { unsigned long rmap_val; @@ -1036,6 +1039,35 @@ static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head, __kvm_rmap_unlock(rmap_head, old_val); preempt_enable(); } +#else +static unsigned long kvm_rmap_get(struct kvm_rmap_head *rmap_head) +{ + return atomic_long_read(&rmap_head->val); +} +static unsigned long kvm_rmap_lock(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) +{ + lockdep_assert_held_write(&kvm->mmu_lock); + return kvm_rmap_get(rmap_head); +} + +static void kvm_rmap_unlock(struct kvm *kvm, + struct kvm_rmap_head *rmap_head, + unsigned long new_val) +{ + atomic_long_set_release(&rmap_head->val, new_val); +} + +static unsigned long kvm_rmap_lock_readonly(struct kvm *kvm, + struct kvm_rmap_head *rmap_head) +{ + lockdep_assert_held_read(&kvm->mmu_lock); + return kvm_rmap_get(rmap_head); +} + +static void kvm_rmap_unlock_readonly(struct kvm_rmap_head *rmap_head, + unsigned long old_val) { } +#endif /* * Returns the number of pointers in the rmap chain, not counting the new one. @@ -1749,11 +1781,24 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, gfn_t gfn; int level; + BUILD_BUG_ON(!IS_ENABLED(CONFIG_KVM_MMU_LOCKLESS_AGING)); + + /* + * For realtime kernels, do aging under mmu_lock (per-rmap locking is + * compiled out), as realtime deployments are unlikely to benefit from + * increased aging throughput and reduced jitter for memory-overcommitted + * nested VMs, whereas keeping preemption enabled is extremely valuable + * (mmu_lock becomes a sleepable lock on realtime kernels). + */ +#ifdef CONFIG_PREEMPT_RT + guard(read_lock)(&kvm->mmu_lock); +#endif + for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { for (gfn = range->start; gfn < range->end; gfn += KVM_PAGES_PER_HPAGE(level)) { rmap_head = gfn_to_rmap(gfn, level, range->slot); - rmap_val = kvm_rmap_lock_readonly(rmap_head); + rmap_val = kvm_rmap_lock_readonly(kvm, rmap_head); for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) { if (!is_accessed_spte(old_spte)) @@ -2795,7 +2840,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, } if (sp->nx_huge_page_disallowed) - unaccount_nx_huge_page(kvm, sp); + unaccount_nx_huge_page(kvm, sp, KVM_SHADOW_MMU); sp->role.invalid = 1; @@ -3031,6 +3076,9 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, if (prefetch) return -EEXIST; + if (KVM_BUG_ON(sp->role.level != PG_LEVEL_4K, kvm)) + continue; + /* * TDP MMU page faults require an additional spinlock as they * run with mmu_lock held for read, not write, and the unsync @@ -3054,7 +3102,6 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, continue; } - WARN_ON_ONCE(sp->role.level != PG_LEVEL_4K); kvm_unsync_page(kvm, sp); } if (locked) @@ -3431,7 +3478,7 @@ int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault, is_private = fault->is_private; } else { max_level = PG_LEVEL_NUM; - is_private = kvm_mem_is_private(kvm, gfn); + is_private = kvm_is_private_gfn(kvm, gfn); } max_level = min(max_level, max_huge_page_level); @@ -3541,8 +3588,8 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) fault->req_level >= it.level); } - if (WARN_ON_ONCE(it.level != fault->goal_level)) - return -EFAULT; + if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm)) + return -EIO; ret = mmu_set_spte(vcpu, fault->slot, it.sptep, access, base_gfn, fault->pfn, fault); @@ -3580,6 +3627,7 @@ static int kvm_handle_error_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa return RET_PF_RETRY; } + kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return -EFAULT; } @@ -3644,13 +3692,13 @@ static bool page_fault_can_be_fast(struct kvm *kvm, struct kvm_page_fault *fault * guest spinning on a #PF indefinitely, so don't attempt the fast path * in this case. * - * Note that the kvm_mem_is_private() check might race with an + * Note that the kvm_is_private_gfn() check might race with an * attribute update, but this will either result in the guest spinning * on RET_PF_SPURIOUS until the update completes, or an actual spurious * case might go down the slow path. Either case will resolve itself. */ if (kvm->arch.has_private_mem && - fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) + fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) return false; /* @@ -4638,7 +4686,7 @@ static int kvm_mmu_faultin_pfn_gmem(struct kvm_vcpu *vcpu, } r = kvm_gmem_get_pfn(vcpu->kvm, fault->slot, fault->gfn, &fault->pfn, - &fault->refcounted_page, &max_order); + &max_order); if (r) { kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return r; @@ -4718,7 +4766,7 @@ static int kvm_mmu_faultin_pfn(struct kvm_vcpu *vcpu, * Now that we have a snapshot of mmu_invalidate_seq we can check for a * private vs. shared mismatch. */ - if (fault->is_private != kvm_mem_is_private(kvm, fault->gfn)) { + if (fault->is_private != kvm_is_private_gfn(kvm, fault->gfn)) { kvm_mmu_prepare_memory_fault_exit(vcpu, fault); return -EFAULT; } @@ -4905,8 +4953,8 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, #ifndef CONFIG_X86_64 /* A 64-bit CR2 should be impossible on 32-bit KVM. */ - if (WARN_ON_ONCE(fault_address >> 32)) - return -EFAULT; + if (KVM_BUG_ON(fault_address >> 32, vcpu->kvm)) + return -EIO; #endif /* * Legacy #PF exception only have a 32-bit error code. Simply drop the @@ -5071,6 +5119,10 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; + r = kvm_mmu_reload(vcpu); + if (r) + return r; + cond_resched(); r = kvm_mmu_do_page_fault(vcpu, gpa, error_code, true, NULL, level); } while (r == RET_PF_RETRY); @@ -5096,6 +5148,14 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, } } +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu) +{ + if (!vcpu->kvm->arch.pre_fault_allowed) + return -EOPNOTSUPP; + + return 0; +} + long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) { @@ -5105,23 +5165,12 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, u64 end; int r; - if (!vcpu->kvm->arch.pre_fault_allowed) - return -EOPNOTSUPP; - if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa))) return -EINVAL; - /* - * reload is efficient when called repeatedly, so we can do it on - * every iteration. - */ - r = kvm_mmu_reload(vcpu); - if (r) - return r; - direct_bits = 0; if (kvm_arch_has_private_mem(vcpu->kvm) && - kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(range->gpa))) + kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(range->gpa))) error_code |= PFERR_PRIVATE_ACCESS; else direct_bits = gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm)); @@ -5223,14 +5272,6 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_gfn_is_write_tracked(kvm, fault.slot, fault.gfn)) return -EPERM; - r = kvm_mmu_reload(vcpu); - if (r) - return r; - - r = mmu_topup_memory_caches(vcpu, false); - if (r) - return r; - do { if (signal_pending(current)) return -EINTR; @@ -5238,10 +5279,38 @@ int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; + r = kvm_mmu_reload(vcpu); + if (r) + return r; + + r = mmu_topup_memory_caches(vcpu, false); + if (r) + return r; + cond_resched(); guard(read_lock)(&kvm->mmu_lock); + /* + * Because slots_lock is held, it should be impossible for *any* + * roots to be invalidated after the initial MMU reload. WARN, + * but continue on; the above MMU reload will do the right thing + * if the current root is actually invalid. + */ + WARN_ON_ONCE(kvm_test_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu)); + + /* + * Snapshot the invalidation sequence counter after acquiring + * mmu_lock, as guest_memfd guarantees the validity of the pfn, + * i.e. any concurrent invalidations are guaranteed to be + * irrelevant. + */ + fault.mmu_seq = vcpu->kvm->mmu_invalidate_seq; + if (is_page_fault_stale(vcpu, &fault)) { + r = RET_PF_RETRY; + continue; + } + r = kvm_tdp_mmu_map(vcpu, &fault); } while (r == RET_PF_RETRY); @@ -5920,6 +5989,9 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu, return role; } + if (KVM_BUG_ON(____is_efer_lma(regs) && !____is_cr4_pae(regs), vcpu->kvm)) + *(u64 *)®s->efer &= ~EFER_LMA; + role.base.efer_nx = ____is_efer_nx(regs); role.base.cr0_wp = ____is_cr0_wp(regs); role.base.cr4_smep = ____is_cr4_smep(regs); @@ -5963,19 +6035,22 @@ void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu, static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) { - int maxpa; - - if (vcpu->kvm->arch.vm_type == KVM_X86_TDX_VM) - maxpa = cpuid_query_maxguestphyaddr(vcpu); - else - maxpa = cpuid_maxphyaddr(vcpu); - /* tdp_root_level is architecture forced level, use it if nonzero */ if (tdp_root_level) return tdp_root_level; + /* + * If the VM has mirror roots, then the root level is predefined as the + * mirror root (and by extension the normal root) needs to match the + * root level that was configured for the external page tables that are + * being mirrored by KVM. + */ + if (kvm_has_mirrored_tdp(vcpu->kvm) && + !WARN_ON_ONCE(!vcpu->kvm->arch.mirror_root_level)) + return vcpu->kvm->arch.mirror_root_level; + /* Use 5-level TDP if and only if it's useful/necessary. */ - if (max_tdp_level == 5 && maxpa <= 48) + if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48) return 4; return max_tdp_level; @@ -6594,13 +6669,13 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err if (IS_ENABLED(CONFIG_KVM_SW_PROTECTED_VM) && !(error_code & PFERR_RSVD_MASK) && vcpu->kvm->arch.vm_type == KVM_X86_SW_PROTECTED_VM && - kvm_mem_is_private(vcpu->kvm, gpa_to_gfn(cr2_or_gpa))) + kvm_is_private_gfn(vcpu->kvm, gpa_to_gfn(cr2_or_gpa))) error_code |= PFERR_PRIVATE_ACCESS; r = RET_PF_INVALID; if (unlikely(error_code & PFERR_RSVD_MASK)) { - if (WARN_ON_ONCE(error_code & PFERR_PRIVATE_ACCESS)) - return -EFAULT; + if (KVM_BUG_ON(error_code & PFERR_PRIVATE_ACCESS, vcpu->kvm)) + return -EIO; r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct); if (r == RET_PF_EMULATE) @@ -6642,6 +6717,9 @@ int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 err return r; emulate: + if (error_code & PFERR_WRITE_MASK) + emulation_type |= EMULTYPE_PF_WRITE; + return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn, insn_len); } @@ -6714,7 +6792,7 @@ void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w, if (is_noncanonical_invlpg_address(addr, vcpu)) return; - kvm_x86_call(flush_tlb_gva)(vcpu, addr, NULL); + kvm_x86_call(flush_tlb_gva)(vcpu, addr); if (tdp_enabled) return; @@ -6844,7 +6922,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, struct k * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit * KVM; that horror is handled on-demand by mmu_alloc_special_roots(). */ - if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) + if (tdp_enabled && kvm_mmu_get_max_tdp_level() > PT32E_ROOT_LEVEL) return 0; page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); @@ -7127,6 +7205,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) if (WARN_ON_ONCE(gfn_end <= gfn_start)) return; + guard(srcu)(&kvm->srcu); + write_lock(&kvm->mmu_lock); kvm_mmu_invalidate_start(kvm); @@ -7995,7 +8075,7 @@ static void kvm_recover_nx_huge_pages(struct kvm *kvm, WARN_ON_ONCE(!sp->nx_huge_page_disallowed); WARN_ON_ONCE(!sp->role.direct); - unaccount_nx_huge_page(kvm, sp); + unaccount_nx_huge_page(kvm, sp, mmu_type); if (is_tdp_mmu) spin_unlock(tdp_mmu_pages_lock); @@ -8107,7 +8187,7 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm) vhost_task_stop(kvm->arch.nx_huge_page_recovery_thread); } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES static bool hugepage_test_mixed(struct kvm_memory_slot *slot, gfn_t gfn, int level) { @@ -8126,8 +8206,8 @@ static void hugepage_set_mixed(struct kvm_memory_slot *slot, gfn_t gfn, lpage_info_slot(gfn, slot, level)->disallow_lpage |= KVM_LPAGE_MIXED_FLAG; } -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { struct kvm_memory_slot *slot = range->slot; int level; @@ -8196,18 +8276,18 @@ static bool hugepage_has_attrs(struct kvm *kvm, struct kvm_memory_slot *slot, const unsigned long end = start + KVM_PAGES_PER_HPAGE(level); if (level == PG_LEVEL_2M) - return kvm_range_has_memory_attributes(kvm, start, end, ~0, attrs); + return kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attrs); for (gfn = start; gfn < end; gfn += KVM_PAGES_PER_HPAGE(level - 1)) { if (hugepage_test_mixed(slot, gfn, level - 1) || - attrs != kvm_get_memory_attributes(kvm, gfn)) + attrs != kvm_get_vm_memory_attributes(kvm, gfn)) return false; } return true; } -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { unsigned long attrs = range->arg.attributes; struct kvm_memory_slot *slot = range->slot; @@ -8300,7 +8380,7 @@ void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm, * be manually checked as the attributes may already be mixed. */ for (gfn = start; gfn < end; gfn += nr_pages) { - unsigned long attrs = kvm_get_memory_attributes(kvm, gfn); + unsigned long attrs = kvm_get_vm_memory_attributes(kvm, gfn); if (hugepage_has_attrs(kvm, slot, gfn, level, attrs)) hugepage_clear_mixed(slot, gfn, level); diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 7e8195a311bb..f12558dfcd81 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -16,6 +16,8 @@ #include #include +#include + #include "mmu.h" #include "mmu_internal.h" #include "page_track.h" @@ -237,10 +239,11 @@ static int kvm_enable_external_write_tracking(struct kvm *kvm) * register the notifier so that event interception for the tracked guest * pages can be received. */ -int kvm_page_track_register_notifier(struct kvm *kvm, +int kvm_page_track_register_notifier(struct file *file, struct kvm_page_track_notifier_node *n) { struct kvm_page_track_notifier_head *head; + struct kvm *kvm = file_to_kvm_x86(file); int r; if (!kvm || kvm->mm != current->mm) @@ -252,7 +255,7 @@ int kvm_page_track_register_notifier(struct kvm *kvm, return r; } - kvm_get_kvm(kvm); + get_file(file); head = &kvm->arch.track_notifier_head; @@ -267,10 +270,11 @@ EXPORT_SYMBOL_GPL(kvm_page_track_register_notifier); * stop receiving the event interception. It is the opposed operation of * kvm_page_track_register_notifier(). */ -void kvm_page_track_unregister_notifier(struct kvm *kvm, +void kvm_page_track_unregister_notifier(struct file *file, struct kvm_page_track_notifier_node *n) { struct kvm_page_track_notifier_head *head; + struct kvm *kvm = file_to_kvm_x86(file); head = &kvm->arch.track_notifier_head; @@ -279,7 +283,7 @@ void kvm_page_track_unregister_notifier(struct kvm *kvm, write_unlock(&kvm->mmu_lock); synchronize_srcu(&head->track_srcu); - kvm_put_kvm(kvm); + fput(file); } EXPORT_SYMBOL_GPL(kvm_page_track_unregister_notifier); @@ -336,11 +340,12 @@ void kvm_page_track_delete_slot(struct kvm *kvm, struct kvm_memory_slot *slot) * add guest page to the tracking pool so that corresponding access on that * page will be intercepted. * - * @kvm: the guest instance we are interested in. + * @file: the VM file of the guest instance we are interested in. * @gfn: the guest page. */ -int kvm_write_track_add_gfn(struct kvm *kvm, gfn_t gfn) +int kvm_write_track_add_gfn(struct file *file, gfn_t gfn) { + struct kvm *kvm = file_to_kvm_x86(file); struct kvm_memory_slot *slot; int idx; @@ -366,11 +371,12 @@ EXPORT_SYMBOL_GPL(kvm_write_track_add_gfn); * remove the guest page from the tracking pool which stops the interception * of corresponding access on that page. * - * @kvm: the guest instance we are interested in. + * @file: the VM file of the guest instance we are interested in. * @gfn: the guest page. */ -int kvm_write_track_remove_gfn(struct kvm *kvm, gfn_t gfn) +int kvm_write_track_remove_gfn(struct file *file, gfn_t gfn) { + struct kvm *kvm = file_to_kvm_x86(file); struct kvm_memory_slot *slot; int idx; diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index e5450faa669e..dc86c2ae70c9 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -367,13 +367,14 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, pte_access = ~0; /* - * Queue a page fault for injection if this assertion fails, as callers - * assume that walker.fault contains sane info on a walk failure. I.e. - * avoid making the situation worse by inducing even worse badness - * between when the assertion fails and when KVM kicks the vCPU out to - * userspace (because the VM is bugged). + * Queue a page fault for injection if any of the below assertions fail, + * as callers assume that walker.fault contains sane info on a walk + * failure. I.e. avoid making the situation worse by inducing even + * worse badness between when the assertion fails and when KVM kicks + * the vCPU out to userspace (because the VM is bugged). */ - if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm)) + if (KVM_BUG_ON(is_long_mode(vcpu) && !is_pae(vcpu), vcpu->kvm) || + KVM_BUG_ON(walker->max_level > PT_MAX_FULL_LEVELS, vcpu->kvm)) goto error; ++walker->level; @@ -390,7 +391,9 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, offset = index * sizeof(pt_element_t); pte_gpa = gfn_to_gpa(table_gfn) + offset; - BUG_ON(walker->level < 1); + if (KVM_BUG_ON(walker->level < 1, vcpu->kvm)) + goto error; + walker->table_gfn[walker->level - 1] = table_gfn; walker->pte_gpa[walker->level - 1] = pte_gpa; @@ -774,8 +777,8 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, fault->req_level >= it.level); } - if (WARN_ON_ONCE(it.level != fault->goal_level)) - return -EFAULT; + if (KVM_BUG_ON(it.level != fault->goal_level, vcpu->kvm)) + return -EIO; ret = mmu_set_spte(vcpu, fault->slot, it.sptep, gw->pte_access, base_gfn, fault->pfn, fault); diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index dd3bb04878ca..e91650af44c2 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -598,9 +598,19 @@ static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX)) return false; - return true; + /* + * EFER_LMSLE_MBZ is a "defeature" bit, i.e. is set when the CPU does + * *not* support long mode segment limits, and so is the only EFER + * check whose polarity is inverted: EFER.LMSLE is legal if and only if + * the guest does *not* have the defeature. + */ + if (efer & EFER_LMSLE && + guest_cpu_cap_has(vcpu, X86_FEATURE_EFER_LMSLE_MBZ)) + return false; + return true; } + bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) { if (efer & ~kvm_caps.supported_efer_bits) @@ -1631,7 +1641,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (msr_info->host_initiated) { kvm_synchronize_tsc(vcpu, &data); } else if (!vcpu->arch.guest_tsc_protected) { - u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; + u64 adj = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data) - + vcpu->arch.l1_tsc_offset; adjust_tsc_offset_guest(vcpu, adj); vcpu->arch.ia32_tsc_adjust_msr += adj; } diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index d2fd47ee5ec8..eef2f792dc9e 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -662,7 +662,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) DECLARE_BITMAP(bitmap, X86_PMC_IDX_MAX); struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; - int bit; + int bit, r; bitmap_copy(bitmap, pmu->reprogram_pmi, X86_PMC_IDX_MAX); @@ -676,12 +676,14 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) kvm_for_each_pmc(pmu, pmc, bit, bitmap) { /* - * If reprogramming fails, e.g. due to contention, re-set the - * reprogram bit, i.e. opportunistically try again on the next - * PMU refresh. Don't make a new request as doing so can stall - * the guest if reprogramming repeatedly fails. + * If reprogramming fails on a transient condition, e.g. due to + * contention, re-set the reprogram bit, i.e. opportunistically + * try again on the next PMU refresh. Don't make a new request + * as doing so can stall the guest if reprogramming repeatedly + * fails. */ - if (reprogram_counter(pmc)) + r = reprogram_counter(pmc); + if (r == -EBUSY || r == -ENOMEM) set_bit(pmc->idx, pmu->reprogram_pmi); } diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index 8f66438989e4..3621f1a47d37 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -440,9 +440,14 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); +static bool kvm_is_valid_cr8(unsigned long cr8) +{ + return !(cr8 & CR8_RESERVED_BITS); +} + int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) { - if (cr8 & CR8_RESERVED_BITS) + if (!kvm_is_valid_cr8(cr8)) return 1; if (lapic_in_kernel(vcpu)) kvm_lapic_set_tpr(vcpu, cr8); @@ -565,6 +570,7 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) return kvm_is_valid_cr4(vcpu, sregs->cr4) && kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_is_valid_cr8(sregs->cr8) && kvm_valid_efer(vcpu, sregs->efer); } @@ -766,17 +772,19 @@ void kvm_update_dr7(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); -static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu) { - u64 fixed = DR6_FIXED_1; + unsigned long fixed = DR6_ACTIVE_LOW & ~DR6_VOLATILE; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) fixed |= DR6_RTM; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) fixed |= DR6_BUS_LOCK; + return fixed; } +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr6_fixed_1); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) { @@ -792,7 +800,7 @@ int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) case 6: if (!kvm_dr6_valid(val)) return 1; /* #GP */ - vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); + vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_get_dr6_fixed_1(vcpu); break; case 5: default: /* 7 */ diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 447f0ec3e63e..97c17b120a57 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -38,16 +38,14 @@ static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS)); #define DR6_BT (1 << 15) #define DR6_RTM (1 << 16) /* - * DR6_ACTIVE_LOW combines fixed-1 and active-low bits. - * We can regard all the bits in DR6_FIXED_1 as active_low bits; - * they will never be 0 for now, but when they are defined - * in the future it will require no code change. + * DR6_ACTIVE_LOW combines fixed-1 and active-low bits (bits that are currently + * fixed-1 are guaranteed to have active-low semantics if they are ever used to + * support a new feature). * - * DR6_ACTIVE_LOW is also used as the init/reset value for DR6. + * Note, DR6_ACTIVE_LOW is also the INIT/RESET value for DR6. */ #define DR6_ACTIVE_LOW 0xffff0ff0 #define DR6_VOLATILE 0x0001e80f -#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE) #define DR7_BP_EN_MASK 0x000000ff #define DR7_GE (1 << 9) @@ -62,6 +60,7 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8); int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val); unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr); +unsigned long kvm_get_dr6_fixed_1(struct kvm_vcpu *vcpu); unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu); void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw); int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3); diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h index 2ad25781cefb..89345af0f7ad 100644 --- a/arch/x86/kvm/reverse_cpuid.h +++ b/arch/x86/kvm/reverse_cpuid.h @@ -66,7 +66,7 @@ #define X86_FEATURE_AVX10_512 KVM_X86_FEATURE(CPUID_24_0_EBX, 18) /* Intel-defined sub-features, CPUID level 0x00000024:1 (ECX) */ -#define X86_FEATURE_AVX10_VNNI_INT KVM_X86_FEATURE(CPUID_24_1_ECX, 2) +#define X86_FEATURE_AVX10_V1_AUX KVM_X86_FEATURE(CPUID_24_1_ECX, 2) /* CPUID level 0x80000007 (EDX). */ #define KVM_X86_FEATURE_CONSTANT_TSC KVM_X86_FEATURE(CPUID_8000_0007_EDX, 8) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index f9090b601efa..af0ef678433d 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -527,12 +527,16 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, /* Always clear misc_ctl bits that the guest cannot use */ to->misc_ctl = from->misc_ctl; + to->misc_ctl2 = from->misc_ctl2; if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT)) to->misc_ctl &= ~SVM_MISC_ENABLE_NP; if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET)) to->misc_ctl &= ~SVM_MISC_ENABLE_GMET; + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV)) + to->misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; + to->iopm_base_pa = from->iopm_base_pa & PAGE_MASK; to->msrpm_base_pa = from->msrpm_base_pa & PAGE_MASK; to->tsc_offset = from->tsc_offset; @@ -550,7 +554,6 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, to->event_inj_err = from->event_inj_err; to->next_rip = from->next_rip; to->nested_cr3 = from->nested_cr3; - to->misc_ctl2 = from->misc_ctl2; to->pause_filter_count = from->pause_filter_count; to->pause_filter_thresh = from->pause_filter_thresh; @@ -735,12 +738,6 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, return 0; } -static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) -{ - return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (to_svm(vcpu)->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); -} - static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) { struct vmcb_ctrl_area_cached *control = &svm->nested.ctl; @@ -789,6 +786,10 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED); + /* SVM ignores EFER.LMA if EFER.LME=0 (instead of failing VMRUN). */ + if (!(svm->nested.save.efer & EFER_LME)) + svm->nested.save.efer &= ~EFER_LMA; + svm_set_efer(vcpu, svm->nested.save.efer); svm_set_cr0(vcpu, svm->nested.save.cr0); @@ -807,17 +808,17 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm) if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_DR))) { vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; - svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; + svm->vcpu.arch.dr6 = svm->nested.save.dr6 | kvm_get_dr6_fixed_1(vcpu); vmcb_mark_dirty(vmcb02, VMCB_DR); } - if (nested_vmcb12_has_lbrv(vcpu)) { + if (control->misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) { /* * Reserved bits of DEBUGCTL are ignored. Be consistent with * svm_set_msr's definition of reserved bits. */ svm_copy_lbrs(&vmcb02->save, save); - vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; + vmcb02->save.dbgctl &= svm_get_supported_debugctl(vcpu); } else { svm_copy_lbrs(&vmcb02->save, &vmcb01->save); } @@ -901,6 +902,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP); + /* + * PML is never enabled in hardware for L2. Make sure that an + * unexpected PML write would trigger a PML_FULL VM-Exit. + */ + if (pml) + vmcb02->control.pml_index = -1; + /* * Stash vmcb02's counter if the guest hasn't moved past the guilty * instruction; otherwise, reset the counter to '0'. @@ -1199,7 +1207,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) } /* Copy state save area fields which are handled by VMRUN */ -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save) { to_save->es = from_save->es; @@ -1226,7 +1234,7 @@ void svm_copy_vmrun_state(struct vmcb_save_area *to_save, if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) { svm_copy_lbrs(to_save, from_save); - to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS; + to_save->dbgctl &= svm_get_supported_debugctl(vcpu); } } @@ -1297,7 +1305,7 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) vmcb12->control.next_rip = vmcb02->control.next_rip; - if (nested_vmcb12_has_lbrv(vcpu)) + if (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR) svm_copy_lbrs(&vmcb12->save, &vmcb02->save); vmcb12->control.event_inj = 0; @@ -1374,7 +1382,7 @@ void nested_svm_vmexit(struct vcpu_svm *svm) if (!nested_exit_on_intr(svm)) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - if (!nested_vmcb12_has_lbrv(vcpu)) { + if (!(svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)) { svm_copy_lbrs(&vmcb01->save, &vmcb02->save); vmcb_mark_dirty(vmcb01, VMCB_LBR); } @@ -1494,6 +1502,7 @@ int svm_allocate_nested(struct vcpu_svm *svm) if (!svm->nested.msrpm) goto err_free_vmcb02; + svm->nested.vmcb02.cpu = -1; svm->nested.initialized = true; return 0; @@ -1820,6 +1829,13 @@ int nested_svm_exit_special(struct vcpu_svm *svm) if (nested_svm_is_l2_tlb_flush_hcall(vcpu)) return NESTED_EXIT_HOST; break; + case SVM_EXIT_PML_FULL: + /* + * All PML full exits are handled by KVM. KVM emulates PML in + * software for L1, but never enables PML in hardware on behalf + * of L1. + */ + return NESTED_EXIT_HOST; default: break; } @@ -2028,6 +2044,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, if (!(save->cr0 & X86_CR0_PG) || !(save->cr0 & X86_CR0_PE) || (save->rflags & X86_EFLAGS_VM) || + ((save->efer & EFER_LMA) && !(save->efer & EFER_LME)) || !nested_vmcb_check_save(vcpu, &save_cached, false)) goto out_free; @@ -2064,7 +2081,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, svm->nested.vmcb12_gpa = kvm_state->hdr.svm.vmcb_pa; - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, save); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, save); nested_copy_vmcb_control_to_cache(svm, ctl); svm_switch_vmcb(svm, &svm->nested.vmcb02); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 558eb0180219..b90c5f14c1ac 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -573,6 +573,7 @@ static int __sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp, INIT_LIST_HEAD(&sev->regions_list); INIT_LIST_HEAD(&sev->mirror_vms); sev->need_init = false; + kvm->arch.has_protected_page_tables = true; kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_SEV); @@ -2058,12 +2059,14 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) dst->pages_locked = src->pages_locked; dst->es_active = src->es_active; dst->vmsa_features = src->vmsa_features; + dst_kvm->arch.has_protected_page_tables = true; src->asid = 0; src->active = false; src->handle = 0; src->pages_locked = 0; src->es_active = false; + src_kvm->arch.has_protected_page_tables = false; list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); @@ -2158,10 +2161,10 @@ int sev_vm_move_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (fd_empty(f)) return -EBADF; - if (!file_is_kvm(fd_file(f))) + source_kvm = file_to_kvm_x86(fd_file(f)); + if (!source_kvm) return -EBADF; - source_kvm = fd_file(f)->private_data; ret = sev_lock_two_vms(kvm, source_kvm); if (ret) return ret; @@ -2416,7 +2419,7 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, */ if (ret && !snp_page_reclaim(kvm, pfn) && sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID && - sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) { + sev_populate_args->fw_error == SEV_RET_INVALID_PARAM && src_page) { void *src_vaddr = kmap_local_page(src_page); void *dst_vaddr = kmap_local_pfn(pfn); @@ -2449,8 +2452,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) if (copy_from_user(¶ms, u64_to_user_ptr(argp->data), sizeof(params))) return -EFAULT; - pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d\n", __func__, - params.gfn_start, params.len, params.type, params.flags); + pr_debug("%s: GFN start 0x%llx length 0x%llx type %d flags %d src %llx\n", __func__, + params.gfn_start, params.len, params.type, params.flags, params.uaddr); if (!params.len || !PAGE_ALIGNED(params.len) || params.flags || (params.type != KVM_SEV_SNP_PAGE_TYPE_NORMAL && @@ -2462,7 +2465,8 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO) src = NULL; - else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr)) + else if ((!gmem_in_place_conversion && !params.uaddr) || + !PAGE_ALIGNED(params.uaddr)) return -EINVAL; else src = u64_to_user_ptr(params.uaddr); @@ -2509,7 +2513,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) params.gfn_start += count; params.len -= count * PAGE_SIZE; - if (params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) + if (src && params.type != KVM_SEV_SNP_PAGE_TYPE_ZERO) params.uaddr += count * PAGE_SIZE; if (copy_to_user(u64_to_user_ptr(argp->data), ¶ms, sizeof(params))) @@ -2879,10 +2883,10 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) if (fd_empty(f)) return -EBADF; - if (!file_is_kvm(fd_file(f))) + source_kvm = file_to_kvm_x86(fd_file(f)); + if (!source_kvm) return -EBADF; - source_kvm = fd_file(f)->private_data; ret = sev_lock_two_vms(kvm, source_kvm); if (ret) return ret; @@ -2919,6 +2923,7 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) mutex_unlock(&sev_mirror_lock); mirror_sev->active = true; + kvm->arch.has_protected_page_tables = true; mirror_sev->asid = source_sev->asid; mirror_sev->fd = source_sev->fd; mirror_sev->es_active = source_sev->es_active; @@ -2977,6 +2982,7 @@ void sev_vm_init(struct kvm *kvm) kvm->arch.has_protected_state = true; fallthrough; case KVM_X86_SEV_VM: + kvm->arch.has_protected_page_tables = true; kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem; to_kvm_sev_info(kvm)->need_init = true; break; @@ -4049,7 +4055,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) struct kvm *kvm = vcpu->kvm; gfn_t gfn = gpa_to_gfn(gpa); unsigned long mmu_seq; - struct page *page; kvm_pfn_t pfn; lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); @@ -4093,7 +4098,7 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) * The new VMSA will be private memory guest memory, so retrieve the * PFN from the gmem backend. */ - if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL)) + if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, NULL)) return; read_lock(&kvm->mmu_lock); @@ -4109,8 +4114,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) else svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); read_unlock(&kvm->mmu_lock); - - kvm_release_page_clean(page); } /* @@ -4238,6 +4241,7 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ struct kvm *kvm = svm->vcpu.kvm; struct kvm_sev_info *sev = to_kvm_sev_info(kvm); sev_ret_code fw_err = 0; + u8 tickle = 0; int ret; if (!is_sev_snp_guest(&svm->vcpu)) @@ -4245,8 +4249,21 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ guard(mutex)(&sev->guest_req_mutex); - if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE)) - return -EIO; + if (kvm_read_guest(kvm, req_gpa, sev->guest_req_buf, PAGE_SIZE)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } + + /* + * "Tickle" the response buffer to verify it's writable before sending + * the request to firmware, which will modify the VMPCK sequence number. + * Note, this is a best effort check and doesn't guard against TOCTOU + * issues. See below for more information. + */ + if (kvm_write_guest(kvm, resp_gpa, &tickle, sizeof(tickle))) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } data.gctx_paddr = __psp_pa(sev->snp_context); data.req_paddr = __psp_pa(sev->guest_req_buf); @@ -4261,6 +4278,13 @@ static int snp_handle_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t resp_ if (ret && !fw_err) return ret; + /* + * Exit to userspace if writing the response fails, e.g. if the mapping + * changed between the initial tickle and the actual write, to avoid + * creating an ambiguous failure ABI with the guest. Don't return an + * error to the guest because firmware already incremented the VMPCK + * sequence number, but there's no way to communicate that to the guest. + */ if (kvm_write_guest(kvm, resp_gpa, sev->guest_resp_buf, PAGE_SIZE)) return -EIO; @@ -4312,8 +4336,10 @@ static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t r return -EINVAL; if (kvm_read_guest(kvm, req_gpa + offsetof(struct snp_guest_msg_hdr, msg_type), - &msg_type, 1)) - return -EIO; + &msg_type, 1)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } /* * As per GHCB spec, requests of type MSG_REPORT_REQ also allow for @@ -4352,8 +4378,10 @@ static int snp_handle_ext_guest_req(struct vcpu_svm *svm, gpa_t req_gpa, gpa_t r * As per GHCB spec (see "SNP Extended Guest Request"), the * certificate table is terminated by 24-bytes of zeroes. */ - if (data_npages && kvm_clear_guest(kvm, data_gpa, 24)) - return -EIO; + if (data_npages && kvm_clear_guest(kvm, data_gpa, 24)) { + svm_vmgexit_bad_input(svm, GHCB_ERR_INVALID_INPUT); + return 1; + } } return snp_handle_guest_req(svm, req_gpa, resp_gpa); @@ -5011,7 +5039,7 @@ struct page *snp_safe_alloc_page_node(int node, gfp_t gfp) * Allocate an SNP-safe page to workaround the SNP erratum where * the CPU will incorrectly signal an RMP violation #PF if a * hugepage (2MB or 1GB) collides with the RMP entry of a - * 2MB-aligned VMCB, VMSA, or AVIC backing page. + * 2MB-aligned VMCB, VMSA, PML or AVIC backing page. * * Allocate one extra page, choose a page which is not * 2MB-aligned, and free the other. @@ -5036,7 +5064,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) struct kvm_memory_slot *slot; struct kvm *kvm = vcpu->kvm; int order, rmp_level, ret; - struct page *page; + unsigned long mmu_seq; bool assigned; kvm_pfn_t pfn; gfn_t gfn; @@ -5050,7 +5078,7 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) * userspace via KVM_EXIT_MEMORY_FAULT events, however, so RMP faults * for shared pages should not end up here. */ - if (!kvm_mem_is_private(kvm, gfn)) { + if (!kvm_is_private_gfn(kvm, gfn)) { pr_warn_ratelimited("SEV: Unexpected RMP fault for non-private GPA 0x%llx\n", gpa); return; @@ -5063,7 +5091,10 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) return; } - ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &page, &order); + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + + ret = kvm_gmem_get_pfn(kvm, slot, gfn, &pfn, &order); if (ret) { pr_warn_ratelimited("SEV: Unexpected RMP fault, no backing page for private GPA 0x%llx\n", gpa); @@ -5072,9 +5103,13 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) ret = snp_lookup_rmpentry(pfn, &assigned, &rmp_level); if (ret || !assigned) { - pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", - gpa, pfn, ret); - goto out_no_trace; + guard(read_lock)(&kvm->mmu_lock); + + if (!mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) + pr_warn_ratelimited("SEV: Unexpected RMP fault, no assigned RMP entry found for GPA 0x%llx PFN 0x%llx error %d\n", + gpa, pfn, ret); + + return; } /* @@ -5102,26 +5137,31 @@ void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) if (rmp_level == PG_LEVEL_4K) goto out; - ret = snp_rmptable_psmash(pfn); - if (ret) { - /* - * Look it up again. If it's 4K now then the PSMASH may have - * raced with another process and the issue has already resolved - * itself. - */ - if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && - assigned && rmp_level == PG_LEVEL_4K) + scoped_guard(read_lock, &kvm->mmu_lock) { + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) goto out; - pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", - gpa, pfn, ret); + ret = snp_rmptable_psmash(pfn); + if (ret) { + /* + * Look it up again. If it's 4K now then the PSMASH may + * have raced with another process and the issue has + * already resolved itself. If it's not assigned, then + * this must have raced with another process that made + * this page shared. + */ + if (!snp_lookup_rmpentry(pfn, &assigned, &rmp_level) && + ((assigned && rmp_level == PG_LEVEL_4K) || !assigned)) + goto out; + + pr_warn_ratelimited("SEV: Unable to split RMP entry for GPA 0x%llx PFN 0x%llx ret %d\n", + gpa, pfn, ret); + } } kvm_zap_gfn_range(kvm, gfn, gfn + PTRS_PER_PMD); out: trace_kvm_rmp_fault(vcpu, gpa, pfn, error_code, rmp_level, ret); -out_no_trace: - kvm_release_page_unused(page); } static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index c054803d4673..136069dfcd44 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -179,6 +179,9 @@ module_param(vnmi, bool, 0444); module_param(enable_mediated_pmu, bool, 0444); +bool __ro_after_init pml = true; +module_param(pml, bool, 0444); + static bool __ro_after_init svm_gp_erratum_intercept = true; static u8 rsm_ins_bytes[] = "\x0f\xaa"; @@ -530,6 +533,15 @@ static int svm_check_processor_compat(void) static void __svm_write_tsc_multiplier(u64 multiplier) { + /* + * Fallback to the default ratio if KVM is buggy and tries to program + * an unsupported scaling ratio, e.g. so that the guest has a chance of + * surviving, so that the cache isn't stale/corrupted, and so that KVM + * doesn't leak state across VMs. + */ + if (WARN_ON_ONCE(multiplier & SVM_TSC_RATIO_RSVD)) + multiplier = SVM_TSC_RATIO_DEFAULT; + if (multiplier == __this_cpu_read(current_tsc_ratio)) return; @@ -872,13 +884,22 @@ static void __svm_disable_lbrv(struct kvm_vcpu *vcpu) to_svm(vcpu)->vmcb->control.misc_ctl2 &= ~SVM_MISC2_ENABLE_V_LBR; } +static bool svm_need_lbr_virtualization(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (svm->vmcb->save.dbgctl & (DEBUGCTLMSR_LBR | DEBUGCTLMSR_BUS_LOCK_DETECT)) + return true; + + return is_guest_mode(vcpu) && + (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR); +} + void svm_update_lbrv(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool current_enable_lbrv = svm->vmcb->control.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR; - bool enable_lbrv = (svm->vmcb->save.dbgctl & DEBUGCTLMSR_LBR) || - (is_guest_mode(vcpu) && guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (svm->nested.ctl.misc_ctl2 & SVM_MISC2_ENABLE_V_LBR)); + bool enable_lbrv = svm_need_lbr_virtualization(vcpu); if (enable_lbrv && !current_enable_lbrv) __svm_enable_lbrv(vcpu); @@ -1258,6 +1279,23 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) if (vcpu->kvm->arch.bus_lock_detection_enabled) svm_set_intercept(svm, INTERCEPT_BUSLOCK); + if (pml) { + /* + * Populate the page address and index here, PML is enabled + * when dirty logging is enabled on the memslot through + * svm_update_cpu_dirty_logging() + */ + control->pml_addr = (u64)__sme_set(page_to_phys(vcpu->arch.pml_page)); + + /* + * The VMCB control area remains valid after an intercepted SHUTDOWN + * (AMD APM Rev 3.45+), so only initialize PML index on reset to avoid + * discarding already-logged entries that haven't been flushed. + */ + if (!init_event) + control->pml_index = PML_HEAD_INDEX; + } + if (is_sev_guest(vcpu)) sev_init_vmcb(svm, init_event); @@ -1317,9 +1355,15 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) if (!vmcb01_page) goto out; + if (pml) { + vcpu->arch.pml_page = snp_safe_alloc_page(); + if (!vcpu->arch.pml_page) + goto error_free_vmcb_page; + } + err = sev_vcpu_create(vcpu); if (err) - goto error_free_vmcb_page; + goto error_free_pml_page; err = avic_init_vcpu(svm); if (err) @@ -1346,6 +1390,9 @@ static int svm_vcpu_create(struct kvm_vcpu *vcpu) avic_vcpu_free(vcpu); error_free_sev: sev_free_vcpu(vcpu); +error_free_pml_page: + if (vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); error_free_vmcb_page: __free_page(vmcb01_page); out: @@ -1365,6 +1412,9 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu) sev_free_vcpu(vcpu); + if (pml && vcpu->arch.pml_page) + __free_page(vcpu->arch.pml_page); + __free_page(__sme_pa_to_page(svm->vmcb01.pa)); svm_vcpu_free_msrpm(svm->msrpm); } @@ -2753,10 +2803,19 @@ static int efer_trap(struct kvm_vcpu *vcpu) * bit in svm_set_efer(), but __kvm_valid_efer() checks it against * whether the guest has X86_FEATURE_SVM - this avoids a failure if * the guest doesn't have X86_FEATURE_SVM. + * + * Clear EFER_LMSLE for a related reason: EFER writes are *trapped*, + * not intercepted, i.e. hardware has already committed the write by + * the time KVM gains control, and the trap is enabled if and only if + * the guest is SEV-ES, whose EFER lives in the encrypted VMSA and so + * can't be fixed up by KVM. Rejecting EFER.LMSLE=1 would inject a #GP + * *and* leave EFER.LMSLE set in the guest, which is strictly worse + * than honoring a write that hardware itself allowed. */ msr_info.host_initiated = false; msr_info.index = MSR_EFER; - msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; + msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & + ~(EFER_SVME | EFER_LMSLE); ret = kvm_set_msr_common(vcpu, &msr_info); return kvm_complete_insn_gp(vcpu, ret); @@ -3169,7 +3228,7 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) data &= ~DEBUGCTLMSR_BTF; } - if (data & DEBUGCTL_RESERVED_BITS) + if (data & ~svm_get_supported_debugctl(vcpu)) return 1; if (svm->vmcb->save.dbgctl == data) @@ -3338,6 +3397,45 @@ static int vmmcall_interception(struct kvm_vcpu *vcpu) return kvm_emulate_hypercall(vcpu); } +static void svm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb01 = svm->vmcb01.ptr; + + if (enable) + vmcb01->control.misc_ctl |= SVM_MISC_ENABLE_PML; + else + vmcb01->control.misc_ctl &= ~SVM_MISC_ENABLE_PML; + + vmcb_mark_dirty(vmcb01, VMCB_NPT); +} + +static void svm_flush_pml_buffer(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb_control_area *control = &svm->vmcb->control; + + /* Do nothing if PML buffer is empty */ + if (control->pml_index == PML_HEAD_INDEX) + return; + + kvm_flush_pml_buffer(vcpu, control->pml_index); + + /* Reset the PML index */ + control->pml_index = PML_HEAD_INDEX; +} + +static int pml_full_interception(struct kvm_vcpu *vcpu) +{ + trace_kvm_pml_full(vcpu->vcpu_id); + + /* + * PML buffer is already flushed at the beginning of svm_handle_exit(). + * Nothing to do here. + */ + return 1; +} + static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { [SVM_EXIT_READ_CR0] = cr_interception, [SVM_EXIT_READ_CR3] = cr_interception, @@ -3415,6 +3513,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { #ifdef CONFIG_KVM_AMD_SEV [SVM_EXIT_VMGEXIT] = sev_handle_vmgexit, #endif + [SVM_EXIT_PML_FULL] = pml_full_interception, }; static void dump_vmcb(struct kvm_vcpu *vcpu) @@ -3464,8 +3563,14 @@ static void dump_vmcb(struct kvm_vcpu *vcpu) pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); - pr_err("%-20s%lld\n", "misc_ctl:", control->misc_ctl); + pr_err("%-20s%llx\n", "misc_ctl:", control->misc_ctl); pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); + + if (pml) { + pr_err("%-20s%016llx\n", "pml_addr:", control->pml_addr); + pr_err("%-20s%04x\n", "pml_index:", control->pml_index); + } + pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); pr_err("%-20s%08x\n", "event_inj:", control->event_inj); @@ -3643,6 +3748,13 @@ int svm_invoke_exit_handler(struct kvm_vcpu *vcpu, u64 __exit_code) (u64)exit_code != __exit_code) goto unexpected_vmexit; + /* + * PML is never enabled when running L2, bail immediately if a PML full + * exit occurs as something is horribly wrong. + */ + if (unlikely(is_guest_mode(vcpu) && exit_code == SVM_EXIT_PML_FULL)) + goto unexpected_vmexit; + #ifdef CONFIG_MITIGATION_RETPOLINE if (exit_code == SVM_EXIT_MSR) return msr_interception(vcpu); @@ -3711,6 +3823,14 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; + /* + * Opportunistically flush the PML buffer on VM exit. This keeps the + * dirty bitmap current by processing logged GPAs rather than waiting for + * PML_FULL exit. + */ + if (vcpu->kvm->arch.cpu_dirty_log_size && !is_guest_mode(vcpu)) + svm_flush_pml_buffer(vcpu); + if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE)) return 0; @@ -3759,14 +3879,25 @@ static int pre_svm_run(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); /* - * If the previous vmrun of the vmcb occurred on a different physical - * cpu, then mark the vmcb dirty and assign a new asid. Hardware's - * vmcb clean bits are per logical CPU, as are KVM's asid assignments. + * If the previous VMRUN of the VMCB occurred on a different physical + * cpu, then mark the VMCB dirty as hardware's clean bits are per pCPU. + * + * Reset the ASID generation in both VMCBs. This will lead to assigning + * a new ASID now, and then again when switching to the other VMCB. + * However, this is needed as the ASID is shared between the VMCBs, and + * otherwise it would be possible to use an ASID allocated on one pCPU + * on another, for example: + * - vCPU migrates from pCPU A to pCPU B, allocates a new ASID. + * - vCPU migrates back to pCPU A, and then switches the VMCB. + * - The new VMCB does not detect a pCPU change and runs on pCPU A with + * the new ASID allocated on pCPU B, which is potentially used by + * another vCPU/VM. */ if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { - svm->current_vmcb->asid_generation = 0; vmcb_mark_all_dirty(svm->vmcb); svm->current_vmcb->cpu = vcpu->cpu; + svm->vmcb01.asid_generation = 0; + svm->nested.vmcb02.asid_generation = 0; } if (is_sev_guest(vcpu)) @@ -4239,7 +4370,7 @@ static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) svm_flush_tlb_asid(vcpu); } -static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) +static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) { struct vcpu_svm *svm = to_svm(vcpu); @@ -4255,8 +4386,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva, bool *full) } svm_flush_tlb_guest(vcpu); - if (full) - *full = true; } static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu) @@ -5053,7 +5182,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); - svm_copy_vmrun_state(m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); + svm_copy_vmrun_state(vcpu, m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); return 0; } @@ -5093,7 +5222,7 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) * used during SMM (see svm_enter_smm()) */ - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, m_save.map.hva + 0x400); + svm_copy_vmrun_state(vcpu, &svm->vmcb01.ptr->save, m_save.map.hva + 0x400); /* * Enter the nested guest now @@ -5312,6 +5441,9 @@ static int svm_vm_init(struct kvm *kvm) if (!pause_filter_count || !pause_filter_thresh) kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); + if (pml) + kvm->arch.cpu_dirty_log_size = PML_LOG_NR_ENTRIES; + svm_srso_vm_init(); return 0; } @@ -5430,6 +5562,8 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .check_intercept = svm_check_intercept, .handle_exit_irqoff = svm_handle_exit_irqoff, + .update_cpu_dirty_logging = svm_update_cpu_dirty_logging, + .deliver_interrupt = svm_deliver_interrupt, .pi_update_irte = avic_pi_update_irte, .setup_mce = svm_setup_mce, @@ -5572,6 +5706,16 @@ static __init void svm_set_cpu_caps(void) boot_cpu_has(X86_FEATURE_AMD_SSBD)) kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); + /* + * Tell userspace that EFER.LMSLE must be zero if nested SVM is + * disabled, as KVM allows EFER.LMSLE if and only if nested SVM is + * supported (a historical artifact of commit eec4b140c924 ("KVM: SVM: + * Allow EFER.LMSLE to be set with nested svm"), not an architectural + * requirement). + */ + if (!nested) + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + if (enable_pmu) { /* * Enumerate support for PERFCTR_CORE if and only if KVM has @@ -5596,9 +5740,17 @@ static __init void svm_set_cpu_caps(void) * Clear capabilities that are automatically configured by common code, * but that require explicit SVM support (that isn't yet implemented). */ - kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); kvm_cpu_cap_clear(X86_FEATURE_MSR_IMM); + /* + * LBR Virtualization must be enabled to support BusLockTrap inside the + * guest, since BusLockTrap is enabled through MSR_IA32_DEBUGCTLMSR and + * MSR_IA32_DEBUGCTLMSR is virtualized only if LBR Virtualization is + * enabled. + */ + if (!lbrv) + kvm_cpu_cap_clear(X86_FEATURE_BUS_LOCK_DETECT); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } @@ -5621,7 +5773,8 @@ static __init int svm_hardware_setup(void) XFEATURE_MASK_BNDCSR); if (tsc_scaling) { - if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR)) { + if (!boot_cpu_has(X86_FEATURE_TSCRATEMSR) || + !boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { tsc_scaling = false; } else { pr_info("TSC scaling supported\n"); @@ -5690,6 +5843,10 @@ static __init int svm_hardware_setup(void) nrips = nrips && boot_cpu_has(X86_FEATURE_NRIPS); + pml = pml && npt_enabled && cpu_feature_enabled(X86_FEATURE_PML); + if (pml) + pr_info("Page modification logging supported\n"); + if (lbrv) { if (!boot_cpu_has(X86_FEATURE_LBRV)) lbrv = false; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 29bde741874a..6d0242966dc8 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -53,6 +53,7 @@ extern int vgif; extern bool intercept_smi; extern bool vnmi; extern int lbrv; +extern bool pml; extern int tsc_aux_uret_slot __ro_after_init; @@ -786,7 +787,15 @@ BUILD_SVM_MSR_BITMAP_HELPERS(bool, test, test) BUILD_SVM_MSR_BITMAP_HELPERS(void, clear, __clear) BUILD_SVM_MSR_BITMAP_HELPERS(void, set, __set) -#define DEBUGCTL_RESERVED_BITS (~DEBUGCTLMSR_LBR) +static inline u64 svm_get_supported_debugctl(struct kvm_vcpu *vcpu) +{ + u64 debugctl = DEBUGCTLMSR_LBR; + + if (guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; + + return debugctl; +} /* svm.c */ extern bool dump_invalid_vmcb; @@ -876,7 +885,7 @@ void svm_leave_nested(struct kvm_vcpu *vcpu); void svm_free_nested(struct vcpu_svm *svm); int svm_allocate_nested(struct vcpu_svm *svm); int nested_svm_vmrun(struct kvm_vcpu *vcpu); -void svm_copy_vmrun_state(struct vmcb_save_area *to_save, +void svm_copy_vmrun_state(struct kvm_vcpu *vcpu, struct vmcb_save_area *to_save, struct vmcb_save_area *from_save); void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb); void nested_svm_vmexit(struct vcpu_svm *svm); diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h index 08005676702c..b2905335a2d2 100644 --- a/arch/x86/kvm/vmx/common.h +++ b/arch/x86/kvm/vmx/common.h @@ -2,10 +2,13 @@ #ifndef __KVM_X86_VMX_COMMON_H #define __KVM_X86_VMX_COMMON_H +#include #include #include +#include #include "mmu.h" +#include "vmx_ops.h" union vmx_exit_reason { struct { @@ -55,6 +58,8 @@ struct vcpu_vt { #endif }; +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu); + #ifdef CONFIG_KVM_INTEL_TDX static __always_inline bool is_td(struct kvm *kvm) @@ -74,14 +79,62 @@ static __always_inline bool is_td_vcpu(struct kvm_vcpu *vcpu) { return false; } #endif +struct vcpu_vmx_tdx { + struct kvm_vcpu vcpu; + struct vcpu_vt vt; +}; + +static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) +{ + return &(container_of(vcpu, struct vcpu_vmx_tdx, vcpu)->vt); +} + +static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) +{ + return &(container_of(vt, struct vcpu_vmx_tdx, vt)->vcpu); +} + +static __always_inline union vmx_exit_reason vt_get_exit_reason(struct kvm_vcpu *vcpu) +{ + return to_vt(vcpu)->exit_reason; +} + +static __always_inline unsigned long vt_get_exit_qual(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + + return vt->exit_qualification; +} + +static __always_inline u32 vt_get_intr_info(struct kvm_vcpu *vcpu) +{ + struct vcpu_vt *vt = to_vt(vcpu); + + if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && + !WARN_ON_ONCE(is_td_vcpu(vcpu))) + vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + + return vt->exit_intr_info; +} + +static inline bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_fpu.fpstate->xfd && + !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); +} + static inline bool vt_is_tdx_private_gpa(struct kvm *kvm, gpa_t gpa) { /* For TDX the direct mask is the shared mask. */ return !kvm_is_addr_direct(kvm, gpa); } -static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, - unsigned long exit_qualification) +static inline int __vt_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa, + unsigned long exit_qualification) { u64 error_code; @@ -164,8 +217,8 @@ static inline void kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu, * Post an interrupt to a vCPU's PIR and trigger the vCPU to process the * interrupt if necessary. */ -static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, - struct pi_desc *pi_desc, int vector) +static inline void __vt_deliver_posted_interrupt(struct kvm_vcpu *vcpu, + struct pi_desc *pi_desc, int vector) { if (pi_test_and_set_pir(vector, pi_desc)) return; @@ -183,6 +236,33 @@ static inline void __vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, kvm_vcpu_trigger_posted_interrupt(vcpu, POSTED_INTR_VECTOR); } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu); +static inline int __vt_handle_notify(struct kvm_vcpu *vcpu, + unsigned long exit_qual) +{ + bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; + + ++vcpu->stat.notify_window_exits; + + if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || + context_invalid) { + vcpu->run->exit_reason = KVM_EXIT_NOTIFY; + vcpu->run->notify.flags = context_invalid ? + KVM_NOTIFY_CONTEXT_INVALID : 0; + return 0; + } + + return 1; +} + +static inline int vt_handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) +{ + /* + * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK + * VM-Exits. Unconditionally set the flag here and leave the handling + * to .handle_exit() callback. + */ + to_vt(vcpu)->exit_reason.bus_lock_detected = true; + return 1; +} #endif /* __KVM_X86_VMX_COMMON_H */ diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 4c52ab8d0786..d75170a6199e 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include "x86_ops.h" @@ -10,8 +11,12 @@ #include "tdx.h" #include "tdx_arch.h" +#pragma GCC poison to_vmx + +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_vmx, vt)); + #ifdef CONFIG_KVM_INTEL_TDX -static_assert(offsetof(struct vcpu_vmx, vt) == offsetof(struct vcpu_tdx, vt)); +static_assert(offsetof(struct vcpu_vmx_tdx, vt) == offsetof(struct vcpu_tdx, vt)); static void vt_disable_virtualization_cpu(void) { @@ -108,7 +113,7 @@ static void vt_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load(vcpu, cpu); } -static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { /* * Basic TDX does not support feature PML. KVM does not enable PML in @@ -117,7 +122,7 @@ static void vt_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) if (WARN_ON_ONCE(is_td_vcpu(vcpu))) return; - vmx_update_cpu_dirty_logging(vcpu); + vmx_update_cpu_dirty_logging(vcpu, enable); } static void vt_prepare_switch_to_guest(struct kvm_vcpu *vcpu) @@ -154,15 +159,6 @@ static fastpath_t vt_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) return vmx_vcpu_run(vcpu, run_flags); } -static int vt_handle_exit(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion fastpath) -{ - if (is_td_vcpu(vcpu)) - return tdx_handle_exit(vcpu, fastpath); - - return vmx_handle_exit(vcpu, fastpath); -} - static bool vt_unhandleable_emulation_required(struct kvm_vcpu *vcpu) { if (is_td_vcpu(vcpu)) { @@ -543,12 +539,12 @@ static void vt_flush_tlb_current(struct kvm_vcpu *vcpu) vmx_flush_tlb_current(vcpu); } -static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +static void vt_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { if (is_td_vcpu(vcpu)) return; - vmx_flush_tlb_gva(vcpu, addr, full); + vmx_flush_tlb_gva(vcpu, addr); } static void vt_flush_tlb_guest(struct kvm_vcpu *vcpu) @@ -601,6 +597,18 @@ static void vt_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) vmx_set_nmi_mask(vcpu, masked); } +static void vt_clear_hlt(struct kvm_vcpu *vcpu) +{ + /* + * TDX doesn't support disabling HLT-exiting, and KVM can't access a + * TD's VMCS, so there is never any hardware halted state to clear. + */ + if (is_td_vcpu(vcpu)) + return; + + vmx_clear_hlt(vcpu); +} + static void vt_enable_nmi_window(struct kvm_vcpu *vcpu) { /* Refer to the comments in tdx_inject_nmi(). */ @@ -807,7 +815,6 @@ static void vt_write_tsc_multiplier(struct kvm_vcpu *vcpu) vmx_write_tsc_multiplier(vcpu); } -#ifdef CONFIG_X86_64 static int vt_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired) { @@ -826,7 +833,6 @@ static void vt_cancel_hv_timer(struct kvm_vcpu *vcpu) vmx_cancel_hv_timer(vcpu); } -#endif static void vt_setup_mce(struct kvm_vcpu *vcpu) { @@ -876,6 +882,112 @@ static int vt_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, #define vt_op_tdx_only(name) NULL #endif /* CONFIG_KVM_INTEL_TDX */ +noinstr void vt_handle_nmi(struct kvm_vcpu *vcpu) +{ + if ((u16)vt_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || + !is_nmi(vt_get_intr_info(vcpu))) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); + x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); + kvm_after_interrupt(vcpu); +} + +static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) +{ + /* + * Save xfd_err to guest_fpu before interrupt is enabled, so the + * MSR value is not clobbered by the host activity before the guest + * has chance to consume it. + * + * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM + * interception may have been caused by L1 interception. Per the SDM, + * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. + * + * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. + * unlike CR2 and DR6, the value is not a payload that is attached to + * the #NM exception. + */ + if (is_xfd_nm_fault(vcpu)) + rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); +} + +static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) +{ + /* if exit due to PF check for async PF */ + if (is_page_fault(intr_info)) + vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); + /* if exit due to NM, handle before interrupts are enabled */ + else if (is_nm_fault(intr_info)) + handle_nm_fault_irqoff(vcpu); + /* Handle machine checks before interrupts are enabled */ + else if (is_machine_check(intr_info)) + kvm_machine_check(); +} + +static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, + u32 intr_info) +{ + unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; + + if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, + "unexpected VM-Exit interrupt info: 0x%x", intr_info)) + return; + + kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); + x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); + kvm_after_interrupt(vcpu); + + vcpu->arch.at_instruction_boundary = true; +} + +static void vt_handle_exit_irqoff(struct kvm_vcpu *vcpu) +{ + if (to_vt(vcpu)->emulation_required) + return; + + switch (vt_get_exit_reason(vcpu).basic) { + case EXIT_REASON_EXTERNAL_INTERRUPT: + handle_external_interrupt_irqoff(vcpu, vt_get_intr_info(vcpu)); + break; + case EXIT_REASON_EXCEPTION_NMI: + handle_exception_irqoff(vcpu, vt_get_intr_info(vcpu)); + break; + case EXIT_REASON_MCE_DURING_VMENTRY: + kvm_machine_check(); + break; + default: + break; + } +} + +static int vt_handle_exit(struct kvm_vcpu *vcpu, + enum exit_fastpath_completion fastpath) +{ + int ret; + +#ifdef CONFIG_KVM_INTEL_TDX + if (is_td_vcpu(vcpu)) + ret = tdx_handle_exit(vcpu, fastpath); + else +#endif + ret = vmx_handle_exit(vcpu, fastpath); + + /* + * Exit to user space when bus lock detected to inform that there is + * a bus lock in guest. + */ + if (vt_get_exit_reason(vcpu).bus_lock_detected) { + if (ret > 0) { + vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; + ret = 0; + } + + vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; + } + return ret; +} + #define VMX_REQUIRED_APICV_INHIBITS \ (BIT(APICV_INHIBIT_REASON_DISABLED) | \ BIT(APICV_INHIBIT_REASON_ABSENT) | \ @@ -949,7 +1061,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .vcpu_needs_initialization = vt_op_tdx_only(vcpu_needs_initialization), .vcpu_run = vt_op(vcpu_run), - .handle_exit = vt_op(handle_exit), + .handle_exit = vt_handle_exit, .skip_emulated_instruction = vmx_skip_emulated_instruction, .update_emulated_instruction = vmx_update_emulated_instruction, .unhandleable_emulation_required = vt_op(unhandleable_emulation_required), @@ -964,6 +1076,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .nmi_allowed = vt_op(nmi_allowed), .get_nmi_mask = vt_op(get_nmi_mask), .set_nmi_mask = vt_op(set_nmi_mask), + .clear_hlt = vt_op(clear_hlt), .enable_nmi_window = vt_op(enable_nmi_window), .enable_irq_window = vt_op(enable_irq_window), .update_cr8_intercept = vt_op(update_cr8_intercept), @@ -1000,12 +1113,12 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .load_mmu_pgd = vt_op(load_mmu_pgd), .check_intercept = vmx_check_intercept, - .handle_exit_irqoff = vmx_handle_exit_irqoff, + .handle_exit_irqoff = vt_handle_exit_irqoff, .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), - .pi_update_irte = vmx_pi_update_irte, - .pi_start_bypass = vmx_pi_start_bypass, + .pi_update_irte = vt_pi_update_irte, + .pi_start_bypass = vt_pi_start_bypass, #ifdef CONFIG_X86_64 .set_hv_timer = vt_op(set_hv_timer), diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 5165275128fc..516ba03bf608 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -446,7 +446,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * "NMI unblocking due to IRET", i.e. the bit can be propagated * as-is from the original EXIT_QUALIFICATION. */ - exit_qualification = vmx_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; + exit_qualification = vt_get_exit_qual(vcpu) & INTR_INFO_UNBLOCK_NMI; } else { if (fault->error_code & PFERR_RSVD_MASK) { vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; @@ -472,7 +472,7 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, * and walks the faulting GPA. */ if (from_hardware) - exit_qualification |= vmx_get_exit_qual(vcpu) & mask; + exit_qualification |= vt_get_exit_qual(vcpu) & mask; else exit_qualification |= fault->exit_qualification & mask; @@ -4750,7 +4750,7 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, { /* update exit information fields: */ vmcs12->vm_exit_reason = vm_exit_reason; - if (vmx_get_exit_reason(vcpu).enclave_mode) + if (vt_get_exit_reason(vcpu).enclave_mode) vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; vmcs12->exit_qualification = exit_qualification; @@ -5376,7 +5376,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, struct x86_exception e; int r; - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmcs_read32(VMX_INSTRUCTION_INFO), false, sizeof(*vmpointer), &gva)) { *ret = 1; @@ -5668,7 +5668,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5774,7 +5774,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5900,6 +5900,12 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; + /* Forbid normal VMPTRLD if Enlightened version was used */ + if (nested_vmx_is_evmptr12_valid(vmx)) { + kvm_queue_exception(vcpu, UD_VECTOR); + return 1; + } + if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) return r; @@ -5909,10 +5915,6 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) if (vmptr == vmx->nested.vmxon_ptr) return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); - /* Forbid normal VMPTRLD if Enlightened version was used */ - if (nested_vmx_is_evmptr12_valid(vmx)) - return 1; - if (vmx->nested.current_vmptr != vmptr) { struct gfn_to_hva_cache *ghc = &vmx->nested.vmcs12_cache; struct vmcs_hdr hdr; @@ -5963,9 +5965,9 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) /* Emulate the VMPTRST instruction */ static int handle_vmptrst(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); + unsigned long exit_qual = vt_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); - gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; + gpa_t current_vmptr; struct x86_exception e; gva_t gva; int r; @@ -5973,8 +5975,16 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; - if (unlikely(nested_vmx_is_evmptr12_valid(to_vmx(vcpu)))) - return 1; + /* + * Hyper-V TLFS does not specify the behavior of VMPTRST when eVMCS is used + * but genuine Hyper-V seems to be returning eVMCS GPA. + */ +#ifdef CONFIG_KVM_HYPERV + if (nested_vmx_is_evmptr12_valid(to_vmx(vcpu))) + current_vmptr = to_vmx(vcpu)->nested.hv_evmcs_vmptr; + else +#endif + current_vmptr = to_vmx(vcpu)->nested.current_vmptr; if (get_vmx_mem_address(vcpu, exit_qual, instr_info, true, sizeof(gpa_t), &gva)) @@ -6024,7 +6034,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) /* According to the Intel VMX instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6107,7 +6117,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) /* according to the intel vmx instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); @@ -6239,8 +6249,8 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) * EXIT_REASON_VMFUNC as the exit reason. */ nested_vmx_vmexit(vcpu, vmx->vt.exit_reason.full, - vmx_get_intr_info(vcpu), - vmx_get_exit_qual(vcpu)); + vt_get_intr_info(vcpu), + vt_get_exit_qual(vcpu)); return 1; } @@ -6291,7 +6301,7 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; @@ -6317,7 +6327,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, if (exit_reason.basic == EXIT_REASON_MSR_READ_IMM || exit_reason.basic == EXIT_REASON_MSR_WRITE_IMM) - msr_index = vmx_get_exit_qual(vcpu); + msr_index = vt_get_exit_qual(vcpu); else msr_index = kvm_ecx_read(vcpu); @@ -6353,7 +6363,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int cr = exit_qualification & 15; int reg; unsigned long val; @@ -6487,7 +6497,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6570,7 +6580,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, switch ((u16)exit_reason.basic) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -6740,14 +6750,14 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) * need to be synthesized by querying the in-kernel LAPIC, but external * interrupts are never reflected to L1 so it's a non-issue. */ - exit_intr_info = vmx_get_intr_info(vcpu); + exit_intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(exit_intr_info)) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); vmcs12->vm_exit_intr_error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); } - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); reflect_vmexit: nested_vmx_vmexit(vcpu, exit_reason.full, exit_intr_info, exit_qual); diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 4a6d9a17da23..2644add54b76 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -14,6 +14,8 @@ #include "vmx.h" #include "tdx.h" +#pragma GCC poison to_vmx + /* * Maintain a per-CPU list of vCPUs that need to be awakened by wakeup_handler() * when a WAKEUP_VECTOR interrupted is posted. vCPUs are added to the list when @@ -54,7 +56,7 @@ static int pi_try_set_control(struct pi_desc *pi_desc, u64 *pold, u64 new) return 0; } -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); struct vcpu_vt *vt = to_vt(vcpu); @@ -144,11 +146,11 @@ void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu) pi_set_on(pi_desc); } -static bool vmx_can_use_vtd_pi(struct kvm *kvm) +static bool vt_can_use_vtd_pi(struct kvm *kvm) { /* * Note, reading the number of possible bypass IRQs can race with a - * bypass IRQ being attached to the VM. vmx_pi_start_bypass() ensures + * bypass IRQ being attached to the VM. vt_pi_start_bypass() ensures * blockng vCPUs will see an elevated count or get KVM_REQ_UNBLOCK. */ return irqchip_in_kernel(kvm) && kvm_arch_has_irq_bypass() && @@ -206,7 +208,7 @@ static void pi_enable_wakeup_handler(struct kvm_vcpu *vcpu) __apic_send_IPI_self(POSTED_INTR_WAKEUP_VECTOR); } -static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) +static bool vt_needs_pi_wakeup(struct kvm_vcpu *vcpu) { /* * The default posted interrupt vector does nothing when @@ -217,14 +219,14 @@ static bool vmx_needs_pi_wakeup(struct kvm_vcpu *vcpu) * back to the pi_wakeup_handler() function. */ return (vmx_can_use_ipiv(vcpu) && !is_td_vcpu(vcpu)) || - vmx_can_use_vtd_pi(vcpu->kvm); + vt_can_use_vtd_pi(vcpu->kvm); } -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu) { struct pi_desc *pi_desc = vcpu_to_pi_desc(vcpu); - if (!vmx_needs_pi_wakeup(vcpu)) + if (!vt_needs_pi_wakeup(vcpu)) return; /* @@ -292,17 +294,17 @@ bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu) /* * Kick all vCPUs when the first possible bypass IRQ is attached to a VM, as * blocking vCPUs may scheduled out without reconfiguring PID.NV to the wakeup - * vector, i.e. if the bypass IRQ came along after vmx_vcpu_pi_put(). + * vector, i.e. if the bypass IRQ came along after vt_vcpu_pi_put(). */ -void vmx_pi_start_bypass(struct kvm *kvm) +void vt_pi_start_bypass(struct kvm *kvm) { - if (WARN_ON_ONCE(!vmx_can_use_vtd_pi(kvm))) + if (WARN_ON_ONCE(!vt_can_use_vtd_pi(kvm))) return; kvm_make_all_cpus_request(kvm, KVM_REQ_UNBLOCK); } -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector) { diff --git a/arch/x86/kvm/vmx/posted_intr.h b/arch/x86/kvm/vmx/posted_intr.h index a4af39948cf0..c5860ea828ae 100644 --- a/arch/x86/kvm/vmx/posted_intr.h +++ b/arch/x86/kvm/vmx/posted_intr.h @@ -8,16 +8,16 @@ #include -void vmx_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); -void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu); +void vt_vcpu_pi_load(struct kvm_vcpu *vcpu, int cpu); +void vt_vcpu_pi_put(struct kvm_vcpu *vcpu); void pi_wakeup_handler(void); void __init pi_init_cpu(int cpu); void pi_apicv_pre_state_restore(struct kvm_vcpu *vcpu); bool pi_has_pending_interrupt(struct kvm_vcpu *vcpu); -int vmx_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, +int vt_pi_update_irte(struct kvm_kernel_irqfd *irqfd, struct kvm *kvm, unsigned int host_irq, uint32_t guest_irq, struct kvm_vcpu *vcpu, u32 vector); -void vmx_pi_start_bypass(struct kvm *kvm); +void vt_pi_start_bypass(struct kvm *kvm); static inline int pi_find_highest_vector(struct pi_desc *pi_desc) { diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index aac79fe08005..8a593159c9b7 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -56,9 +56,6 @@ bool enable_tdx __ro_after_init; module_param_named(tdx, enable_tdx, bool, 0444); -#define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51)) -#define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47)) - static const struct tdx_sys_info *tdx_sysinfo; void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err) @@ -619,6 +616,7 @@ int tdx_vm_init(struct kvm *kvm) { struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); + kvm->arch.has_protected_page_tables = true; kvm->arch.has_protected_state = true; /* * TDX Module doesn't allow the hypervisor to modify the EOI-bitmap, @@ -718,7 +716,7 @@ void tdx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_tdx *tdx = to_tdx(vcpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); if (vcpu->cpu == cpu || !is_hkid_assigned(to_kvm_tdx(vcpu->kvm))) return; @@ -743,7 +741,7 @@ bool tdx_interrupt_allowed(struct kvm_vcpu *vcpu) * interrupt is always allowed unless TDX guest calls TDVMCALL with HLT, * which passes the interrupt blocked flag. */ - return vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + return vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || !to_tdx(vcpu)->vp_enter_args.r12; } @@ -761,7 +759,7 @@ static bool tdx_protected_apic_has_interrupt(struct kvm_vcpu *vcpu) * otherwise the interrupt would have been serviced at the instruction * boundary. */ - if (vmx_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || + if (vt_get_exit_reason(vcpu).basic != EXIT_REASON_HLT || to_tdx(vcpu)->vp_enter_args.r12) return false; @@ -828,7 +826,7 @@ static void tdx_prepare_switch_to_host(struct kvm_vcpu *vcpu) void tdx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); tdx_prepare_switch_to_host(vcpu); } @@ -925,36 +923,51 @@ static __always_inline u32 tdcall_to_vmx_exit_reason(struct kvm_vcpu *vcpu) return EXIT_REASON_TDCALL; } -static __always_inline u32 tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +static __always_inline bool tdx_is_exit_reason_valid(u64 vp_enter_ret) { - struct vcpu_tdx *tdx = to_tdx(vcpu); - u32 exit_reason; - - switch (tdx->vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { + switch (vp_enter_ret & TDX_SEAMCALL_STATUS_MASK) { case TDX_SUCCESS: case TDX_NON_RECOVERABLE_VCPU: case TDX_NON_RECOVERABLE_TD: case TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE: case TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE: - break; + return true; default: - return -1u; + return false; } +} - exit_reason = tdx->vp_enter_ret; +static __always_inline union vmx_exit_reason tdx_to_vmx_exit_reason(struct kvm_vcpu *vcpu) +{ + struct vcpu_tdx *tdx = to_tdx(vcpu); + union vmx_exit_reason exit_reason; - switch (exit_reason) { + /* + * Return the synthesized invalid Exit Reason, as the TDX module + * never attempted to run the vCPU, i.e. the Exit Reason is undefined, + * but this is NOT a failed VM-Enter. + */ + if (!tdx_is_exit_reason_valid(tdx->vp_enter_ret)) + return (union vmx_exit_reason) { + .basic = EXIT_REASON_UNDEFINED, + }; + + exit_reason.full = (u32)tdx->vp_enter_ret; + + switch (exit_reason.basic) { case EXIT_REASON_TDCALL: if (tdvmcall_exit_type(vcpu)) - return EXIT_REASON_VMCALL; - - return tdcall_to_vmx_exit_reason(vcpu); + exit_reason.basic = EXIT_REASON_VMCALL; + else + exit_reason.basic = tdcall_to_vmx_exit_reason(vcpu); + break; case EXIT_REASON_EPT_MISCONFIG: /* * Defer KVM_BUG_ON() until tdx_handle_exit() because this is in * non-instrumentable code with interrupts disabled. */ - return -1u; + exit_reason.basic = EXIT_REASON_UNDEFINED; + break; default: break; } @@ -971,22 +984,21 @@ static noinstr void tdx_vcpu_enter_exit(struct kvm_vcpu *vcpu) tdx->vp_enter_ret = tdh_vp_enter(&tdx->vp, &tdx->vp_enter_args); - vt->exit_reason.full = tdx_to_vmx_exit_reason(vcpu); + vt->exit_reason = tdx_to_vmx_exit_reason(vcpu); vt->exit_qualification = tdx->vp_enter_args.rcx; tdx->ext_exit_qualification = tdx->vp_enter_args.rdx; tdx->exit_gpa = tdx->vp_enter_args.r8; vt->exit_intr_info = tdx->vp_enter_args.r9; - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); guest_state_exit_irqoff(); } static bool tdx_failed_vmentry(struct kvm_vcpu *vcpu) { - return vmx_get_exit_reason(vcpu).failed_vmentry && - vmx_get_exit_reason(vcpu).full != -1u; + return vt_get_exit_reason(vcpu).failed_vmentry; } static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) @@ -1075,8 +1087,16 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * allowing vCPU entry to avoid contention with tdh_vp_enter() and * TDCALLs. */ - if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) + if (unlikely(READ_ONCE(to_kvm_tdx(vcpu->kvm)->wait_for_sept_zap))) { + /* + * The vCPU never entered the guest, but this looks like a + * handled exit to the caller. Synthesize an invalid exit + * reason so the previous exit's stale value isn't consumed + * a second time. + */ + vt->exit_reason.full = EXIT_REASON_UNDEFINED; return EXIT_FASTPATH_EXIT_HANDLED; + } trace_kvm_entry(vcpu, run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT); @@ -1135,7 +1155,7 @@ void tdx_inject_nmi(struct kvm_vcpu *vcpu) static int tdx_handle_exception_nmi(struct kvm_vcpu *vcpu) { - u32 intr_info = vmx_get_intr_info(vcpu); + u32 intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -1613,10 +1633,7 @@ static int handle_tdvmcall(struct kvm_vcpu *vcpu) void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level) { - u64 shared_bit = (pgd_level == 5) ? TDX_SHARED_BIT_PWL_5 : - TDX_SHARED_BIT_PWL_4; - - if (KVM_BUG_ON(shared_bit != kvm_gfn_direct_bits(vcpu->kvm), vcpu->kvm)) + if (KVM_BUG_ON(pgd_level != vcpu->kvm->arch.mirror_root_level, vcpu->kvm)) return; td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa); @@ -1744,7 +1761,7 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve return ret; /* - * Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory() + * Ensure pre_fault_allowed is read by kvm_arch_pre_fault_allowed() * before kvm_tdx->state. Userspace must not be allowed to pre-fault * arbitrary memory until the initial memory image is finalized. Pairs * with the smp_wmb() in tdx_td_finalize(). @@ -1939,7 +1956,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, struct vcpu_tdx *tdx = to_tdx(vcpu); /* TDX supports only posted interrupt. No lapic emulation. */ - __vmx_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &tdx->vt.pi_desc, vector); trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); } @@ -1947,7 +1964,7 @@ void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcpu) { u64 eeq_type = to_tdx(vcpu)->ext_exit_qualification & TDX_EXT_EXIT_QUAL_TYPE_MASK; - u64 eq = vmx_get_exit_qual(vcpu); + u64 eq = vt_get_exit_qual(vcpu); if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION) return false; @@ -1983,7 +2000,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) /* Only private GPA triggers zero-step mitigation */ local_retry = true; } else { - exit_qual = vmx_get_exit_qual(vcpu); + exit_qual = vt_get_exit_qual(vcpu); /* * EPT violation due to instruction fetch should never be * triggered from shared memory in TDX guest. If such EPT @@ -2031,7 +2048,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) while (1) { struct kvm_memory_slot *slot; - ret = __vmx_handle_ept_violation(vcpu, gpa, exit_qual); + ret = __vt_handle_ept_violation(vcpu, gpa, exit_qual); if (ret != RET_PF_RETRY || !local_retry) break; @@ -2065,7 +2082,7 @@ int tdx_complete_emulated_msr(struct kvm_vcpu *vcpu, int err) return 1; } - if (vmx_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) + if (vt_get_exit_reason(vcpu).basic == EXIT_REASON_MSR_READ) tdvmcall_set_return_val(vcpu, kvm_read_edx_eax(vcpu)); return 1; @@ -2076,7 +2093,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) { struct vcpu_tdx *tdx = to_tdx(vcpu); u64 vp_enter_ret = tdx->vp_enter_ret; - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); if (fastpath != EXIT_FASTPATH_NONE) return 1; @@ -2170,6 +2187,11 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) * - If it's not an MSMI, no need to do anything here. */ return 1; + case EXIT_REASON_NOTIFY: + /* NMI blocking state is handled by TDX module */ + return __vt_handle_notify(vcpu, vt_get_exit_qual(vcpu)); + case EXIT_REASON_BUS_LOCK: + return vt_handle_bus_lock_vmexit(vcpu); default: break; } @@ -2185,10 +2207,10 @@ void tdx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, struct vcpu_tdx *tdx = to_tdx(vcpu); *reason = tdx->vt.exit_reason.full; - if (*reason != -1u) { - *info1 = vmx_get_exit_qual(vcpu); + if (tdx_is_exit_reason_valid(tdx->vp_enter_ret)) { + *info1 = vt_get_exit_qual(vcpu); *info2 = tdx->ext_exit_qualification; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); } else { *info1 = 0; *info2 = 0; @@ -2794,6 +2816,14 @@ DEFINE_CLASS(tdx_vm_state_guard, tdx_vm_state_guard_t, if (!IS_ERR(_T)) tdx_release_vm_state_locks(_T), tdx_acquire_vm_state_locks(kvm), struct kvm *kvm); +static __always_inline void tdx_set_mirror_root_level(struct kvm *kvm, int level) +{ + BUILD_BUG_ON(level != 4 && level != 5); + + kvm->arch.mirror_root_level = level; + kvm->arch.gfn_direct_bits = gpa_to_gfn(BIT_ULL(level == 4 ? 47 : 51)); +} + static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) { struct kvm_tdx_init_vm __user *user_data = u64_to_user_ptr(cmd->data); @@ -2857,9 +2887,9 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd) kvm_tdx->xfam = td_params->xfam; if (td_params->config_flags & TDX_CONFIG_FLAGS_MAX_GPAW) - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_5; + tdx_set_mirror_root_level(kvm, 5); else - kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4; + tdx_set_mirror_root_level(kvm, 4); kvm_tdx->state = TD_STATE_INITIALIZED; out: @@ -3186,6 +3216,17 @@ static int tdx_vcpu_init(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *cmd) td_vmcs_write64(tdx, POSTED_INTR_DESC_ADDR, __pa(&tdx->vt.pi_desc)); td_vmcs_setbit32(tdx, PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_POSTED_INTR); + if (kvm_notify_vmexit_enabled(vcpu->kvm)) { + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_NOTIFY_VM_EXITING); + td_vmcs_write32(tdx, NOTIFY_WINDOW, + vcpu->kvm->arch.notify_window); + } + + if (vcpu->kvm->arch.bus_lock_detection_enabled) + td_vmcs_setbit32(tdx, SECONDARY_VM_EXEC_CONTROL, + SECONDARY_EXEC_BUS_LOCK_DETECTION); + tdx->state = VCPU_TD_STATE_INITIALIZED; return 0; @@ -3221,7 +3262,7 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm)) return -EIO; - kvm_tdx->page_add_src = src_page; + kvm_tdx->page_add_src = src_page ?: pfn_to_page(pfn); ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn); kvm_tdx->page_add_src = NULL; @@ -3251,6 +3292,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); struct kvm_tdx_init_mem_region region; struct tdx_gmem_post_populate_arg arg; + gpa_t nr_bytes, end_gpa; long gmem_ret; int ret; @@ -3267,11 +3309,15 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c if (copy_from_user(®ion, u64_to_user_ptr(cmd->data), sizeof(region))) return -EFAULT; - if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr || - !PAGE_ALIGNED(region.gpa) || !region.nr_pages || - region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || + if (!PAGE_ALIGNED(region.source_addr) || + (!gmem_in_place_conversion && !region.source_addr) || + !PAGE_ALIGNED(region.gpa) || !region.nr_pages) + return -EINVAL; + + if (check_shl_overflow(region.nr_pages, PAGE_SHIFT, &nr_bytes) || + check_add_overflow(region.gpa, nr_bytes - 1, &end_gpa) || !vt_is_tdx_private_gpa(kvm, region.gpa) || - !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1)) + !vt_is_tdx_private_gpa(kvm, end_gpa)) return -EINVAL; ret = 0; @@ -3298,7 +3344,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c break; } - region.source_addr += PAGE_SIZE; + if (region.source_addr) + region.source_addr += PAGE_SIZE; region.gpa += PAGE_SIZE; region.nr_pages--; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 2d0443562a16..82eb891f684c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1577,12 +1577,12 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) vmx_vcpu_load_vmcs(vcpu, cpu); - vmx_vcpu_pi_load(vcpu, cpu); + vt_vcpu_pi_load(vcpu, cpu); } void vmx_vcpu_put(struct kvm_vcpu *vcpu) { - vmx_vcpu_pi_put(vcpu); + vt_vcpu_pi_put(vcpu); vmx_prepare_switch_to_host(to_vmx(vcpu)); } @@ -1791,7 +1791,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, * so that guest userspace can't DoS the guest simply by triggering * emulation (enclaves are CPL3 only). */ - if (vmx_get_exit_reason(vcpu).enclave_mode) { + if (vt_get_exit_reason(vcpu).enclave_mode) { kvm_queue_exception(vcpu, UD_VECTOR); return X86EMUL_PROPAGATE_FAULT; } @@ -1806,7 +1806,7 @@ int vmx_check_emulate_instruction(struct kvm_vcpu *vcpu, int emul_type, static int skip_emulated_instruction(struct kvm_vcpu *vcpu) { - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); unsigned long rip, orig_rip; u32 instr_len; @@ -1909,7 +1909,7 @@ int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu) return skip_emulated_instruction(vcpu); } -static void vmx_clear_hlt(struct kvm_vcpu *vcpu) +void vmx_clear_hlt(struct kvm_vcpu *vcpu) { /* * Ensure that we clear the HLT state in the VMCS. We don't need to @@ -2843,6 +2843,16 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, if (!cpu_has_sgx()) _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING; + /* + * KVM doesn't re-derive the TSC scaling ratio when the host TSC + * frequency changes, so TSC scaling is only usable with a constant + * TSC. Clear the control here rather than in vmx_hardware_setup() so + * that the per-CPU configs recomputed by vmx_check_processor_compat() + * stay consistent with the golden vmcs_config. + */ + if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) + _cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_TSC_SCALING; + if (_cpu_based_exec_control & CPU_BASED_ACTIVATE_TERTIARY_CONTROLS) _cpu_based_3rd_exec_control = adjust_vmx_controls64(KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL, @@ -3379,7 +3389,7 @@ void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) vpid_sync_context(vmx_get_current_vpid(vcpu)); } -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full) +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { /* * vpid_sync_vcpu_addr() is a nop if vpid==0, see the comment in @@ -4426,7 +4436,7 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) if (!vcpu->arch.apic->apicv_active) return -1; - __vmx_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); + __vt_deliver_posted_interrupt(vcpu, &vt->pi_desc, vector); return 0; } @@ -4883,10 +4893,11 @@ int vmx_vcpu_precreate(struct kvm *kvm) #define VMX_XSS_EXIT_BITMAP 0 -static void init_vmcs(struct vcpu_vmx *vmx) +static void init_vmcs(struct kvm_vcpu *vcpu) { - struct kvm *kvm = vmx->vcpu.kvm; + struct kvm *kvm = vcpu->kvm; struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); + struct vcpu_vmx *vmx = to_vmx(vcpu); if (nested) nested_vmx_set_vmcs_shadowing_bitmap(); @@ -4911,7 +4922,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (cpu_has_tertiary_exec_ctrls()) tertiary_exec_controls_set(vmx, vmx_tertiary_exec_control(vmx)); - if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { + if (enable_apicv && lapic_in_kernel(vcpu)) { vmcs_write64(EOI_EXIT_BITMAP0, 0); vmcs_write64(EOI_EXIT_BITMAP1, 0); vmcs_write64(EOI_EXIT_BITMAP2, 0); @@ -4923,7 +4934,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(POSTED_INTR_DESC_ADDR, __pa((&vmx->vt.pi_desc))); } - if (vmx_can_use_ipiv(&vmx->vcpu)) { + if (vmx_can_use_ipiv(vcpu)) { vmcs_write64(PID_POINTER_TABLE, __pa(kvm_vmx->pid_table)); vmcs_write16(LAST_PID_POINTER_INDEX, kvm->arch.max_vcpu_ids - 1); } @@ -4958,15 +4969,15 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val)); if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) - vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); + vmcs_write64(GUEST_IA32_PAT, vcpu->arch.pat); vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl()); /* 22.2.1, 20.8.1 */ vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl()); - vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); - vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits); + vcpu->arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); + vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits); set_cr4_guest_host_mask(vmx); @@ -4977,11 +4988,11 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_write64(XSS_EXIT_BITMAP, VMX_XSS_EXIT_BITMAP); if (enable_pml) { - vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); + vmcs_write64(PML_ADDRESS, page_to_phys(vcpu->arch.pml_page)); vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); } - vmx_write_encls_bitmap(&vmx->vcpu, NULL); + vmx_write_encls_bitmap(vcpu, NULL); if (vmx_pt_mode_is_host_guest()) { memset(&vmx->pt_desc, 0, sizeof(vmx->pt_desc)); @@ -4994,13 +5005,13 @@ static void init_vmcs(struct vcpu_vmx *vmx) vmcs_writel(GUEST_SYSENTER_ESP, 0); vmcs_writel(GUEST_SYSENTER_EIP, 0); - vmx_guest_debugctl_write(&vmx->vcpu, 0); + vmx_guest_debugctl_write(vcpu, 0); if (cpu_has_vmx_tpr_shadow()) { vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, 0); - if (cpu_need_tpr_shadow(&vmx->vcpu)) + if (cpu_need_tpr_shadow(vcpu)) vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, - __pa(vmx->vcpu.arch.apic->regs)); + __pa(vcpu->arch.apic->regs)); vmcs_write32(TPR_THRESHOLD, 0); } @@ -5011,7 +5022,7 @@ static void __vmx_vcpu_reset(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - init_vmcs(vmx); + init_vmcs(vcpu); if (nested && kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) @@ -5381,15 +5392,9 @@ bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) (kvm_get_rflags(vcpu) & X86_EFLAGS_AC); } -static bool is_xfd_nm_fault(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.guest_fpu.fpstate->xfd && - !kvm_is_cr0_bit_set(vcpu, X86_CR0_TS); -} - static int vmx_handle_page_fault(struct kvm_vcpu *vcpu, u32 error_code) { - unsigned long cr2 = vmx_get_exit_qual(vcpu); + unsigned long cr2 = vt_get_exit_qual(vcpu); if (vcpu->arch.apf.host_apf_flags) goto handle_pf; @@ -5437,7 +5442,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) u32 vect_info; vect_info = vmx->idt_vectoring_info; - intr_info = vmx_get_intr_info(vcpu); + intr_info = vt_get_intr_info(vcpu); /* * Machine checks are handled by handle_exception_irqoff(), or by @@ -5518,7 +5523,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) switch (ex_no) { case DB_VECTOR: - dr6 = vmx_get_exit_qual(vcpu); + dr6 = vt_get_exit_qual(vcpu); if (!(vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { /* @@ -5594,7 +5599,7 @@ static int handle_io(struct kvm_vcpu *vcpu) int size, in, string; unsigned port; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); string = (exit_qualification & 16) != 0; ++vcpu->stat.io_exits; @@ -5683,7 +5688,7 @@ static int handle_cr(struct kvm_vcpu *vcpu) int err; int ret; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); cr = exit_qualification & 15; reg = (exit_qualification >> 8) & 15; switch ((exit_qualification >> 4) & 3) { @@ -5761,7 +5766,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) int dr, dr7, reg; int err = 1; - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); dr = exit_qualification & DEBUG_REG_ACCESS_NUM; /* First, if DR does not exist, trigger UD */ @@ -5857,7 +5862,7 @@ static int handle_interrupt_window(struct kvm_vcpu *vcpu) static int handle_invlpg(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); kvm_mmu_invlpg(vcpu, exit_qualification); return kvm_skip_emulated_instruction(vcpu); @@ -5866,7 +5871,7 @@ static int handle_invlpg(struct kvm_vcpu *vcpu) static int handle_apic_access(struct kvm_vcpu *vcpu) { if (likely(fasteoi)) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); int access_type, offset; access_type = exit_qualification & APIC_ACCESS_TYPE; @@ -5887,17 +5892,16 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); - int vector = exit_qualification & 0xff; + unsigned long exit_qualification = vt_get_exit_qual(vcpu); /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ - kvm_apic_set_eoi_accelerated(vcpu, vector); + kvm_apic_set_eoi_accelerated(vcpu, exit_qualification & 0xff); return 1; } static int handle_apic_write(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); /* * APIC-write VM-Exit is trap-like, KVM doesn't need to advance RIP and @@ -5925,7 +5929,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); reason = (u32)exit_qualification >> 30; if (reason == TASK_SWITCH_GATE && idt_v) { @@ -5971,7 +5975,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) static int handle_ept_violation(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmx_get_exit_qual(vcpu); + unsigned long exit_qualification = vt_get_exit_qual(vcpu); gpa_t gpa; /* @@ -5999,7 +6003,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) if (unlikely(allow_smaller_maxphyaddr && !kvm_vcpu_is_legal_gpa(vcpu, gpa))) return kvm_emulate_instruction(vcpu, 0); - return __vmx_handle_ept_violation(vcpu, gpa, exit_qualification); + return __vt_handle_ept_violation(vcpu, gpa, exit_qualification); } static int handle_ept_misconfig(struct kvm_vcpu *vcpu) @@ -6162,7 +6166,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) /* According to the Intel instruction reference, the memory operand * is read even if it isn't needed (e.g., for type==all) */ - if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), + if (get_vmx_mem_address(vcpu, vt_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; @@ -6176,7 +6180,7 @@ static int handle_pml_full(struct kvm_vcpu *vcpu) trace_kvm_pml_full(vcpu->vcpu_id); - exit_qualification = vmx_get_exit_qual(vcpu); + exit_qualification = vt_get_exit_qual(vcpu); /* * PML buffer FULL happened while executing iret from NMI, @@ -6267,23 +6271,9 @@ static int handle_encls(struct kvm_vcpu *vcpu) } #endif /* CONFIG_X86_SGX_KVM */ -static int handle_bus_lock_vmexit(struct kvm_vcpu *vcpu) -{ - /* - * Hardware may or may not set the BUS_LOCK_DETECTED flag on BUS_LOCK - * VM-Exits. Unconditionally set the flag here and leave the handling to - * vmx_handle_exit(). - */ - to_vt(vcpu)->exit_reason.bus_lock_detected = true; - return 1; -} - static int handle_notify(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmx_get_exit_qual(vcpu); - bool context_invalid = exit_qual & NOTIFY_VM_CONTEXT_INVALID; - - ++vcpu->stat.notify_window_exits; + unsigned long exit_qual = vt_get_exit_qual(vcpu); /* * Notify VM exit happened while executing iret from NMI, @@ -6293,15 +6283,7 @@ static int handle_notify(struct kvm_vcpu *vcpu) vmcs_set_bits(GUEST_INTERRUPTIBILITY_INFO, GUEST_INTR_STATE_NMI); - if (vcpu->kvm->arch.notify_vmexit_flags & KVM_X86_NOTIFY_VMEXIT_USER || - context_invalid) { - vcpu->run->exit_reason = KVM_EXIT_NOTIFY; - vcpu->run->notify.flags = context_invalid ? - KVM_NOTIFY_CONTEXT_INVALID : 0; - return 0; - } - - return 1; + return __vt_handle_notify(vcpu, exit_qual); } static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) @@ -6311,13 +6293,13 @@ static int vmx_get_msr_imm_reg(struct kvm_vcpu *vcpu) static int handle_rdmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_rdmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_rdmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } static int handle_wrmsr_imm(struct kvm_vcpu *vcpu) { - return kvm_emulate_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return kvm_emulate_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); } @@ -6377,7 +6359,7 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { [EXIT_REASON_VMFUNC] = handle_vmx_instruction, [EXIT_REASON_PREEMPTION_TIMER] = handle_preemption_timer, [EXIT_REASON_ENCLS] = handle_encls, - [EXIT_REASON_BUS_LOCK] = handle_bus_lock_vmexit, + [EXIT_REASON_BUS_LOCK] = vt_handle_bus_lock_vmexit, [EXIT_REASON_NOTIFY] = handle_notify, [EXIT_REASON_SEAMCALL] = handle_tdx_instruction, [EXIT_REASON_TDCALL] = handle_tdx_instruction, @@ -6388,16 +6370,18 @@ static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { static const int kvm_vmx_max_exit_handlers = ARRAY_SIZE(kvm_vmx_exit_handlers); +static_assert(EXIT_REASON_UNDEFINED >= ARRAY_SIZE(kvm_vmx_exit_handlers)); + void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason, u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code) { struct vcpu_vmx *vmx = to_vmx(vcpu); *reason = vmx->vt.exit_reason.full; - *info1 = vmx_get_exit_qual(vcpu); + *info1 = vt_get_exit_qual(vcpu); if (!(vmx->vt.exit_reason.failed_vmentry)) { *info2 = vmx->idt_vectoring_info; - *intr_info = vmx_get_intr_info(vcpu); + *intr_info = vt_get_intr_info(vcpu); if (is_exception_with_error_code(*intr_info)) *error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); else @@ -6418,48 +6402,25 @@ void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code) *error_code = 0; } -static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) +static void vmx_destroy_pml_buffer(struct kvm_vcpu *vcpu) { - if (vmx->pml_pg) { - __free_page(vmx->pml_pg); - vmx->pml_pg = NULL; + if (vcpu->arch.pml_page) { + __free_page(vcpu->arch.pml_page); + vcpu->arch.pml_page = NULL; } } static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) { - struct vcpu_vmx *vmx = to_vmx(vcpu); - u16 pml_idx, pml_tail_index; - u64 *pml_buf; - int i; + u16 pml_idx; pml_idx = vmcs_read16(GUEST_PML_INDEX); /* Do nothing if PML buffer is empty */ if (pml_idx == PML_HEAD_INDEX) return; - /* - * PML index always points to the next available PML buffer entity - * unless PML log has just overflowed. - */ - pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; - /* - * PML log is written backwards: the CPU first writes the entry 511 - * then the entry 510, and so on. - * - * Read the entries in the same order they were written, to ensure that - * the dirty ring is filled in the same order the CPU wrote them. - */ - pml_buf = page_address(vmx->pml_pg); - - for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { - u64 gpa; - - gpa = pml_buf[i]; - WARN_ON(gpa & (PAGE_SIZE - 1)); - kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); - } + kvm_flush_pml_buffer(vcpu, pml_idx); /* reset PML index */ vmcs_write16(GUEST_PML_INDEX, PML_HEAD_INDEX); @@ -6705,10 +6666,10 @@ void dump_vmcs(struct kvm_vcpu *vcpu) * The guest has exited. See if we can fix it or if we need userspace * assistance. */ -static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) +int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_vmx *vmx = to_vmx(vcpu); - union vmx_exit_reason exit_reason = vmx_get_exit_reason(vcpu); + union vmx_exit_reason exit_reason = vt_get_exit_reason(vcpu); u32 vectoring_info = vmx->idt_vectoring_info; u16 exit_handler_index; @@ -6864,24 +6825,6 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) return 0; } -int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) -{ - int ret = __vmx_handle_exit(vcpu, exit_fastpath); - - /* - * Exit to user space when bus lock detected to inform that there is - * a bus lock in guest. - */ - if (vmx_get_exit_reason(vcpu).bus_lock_detected) { - if (ret > 0) - vcpu->run->exit_reason = KVM_EXIT_X86_BUS_LOCK; - - vcpu->run->flags |= KVM_RUN_X86_BUS_LOCK; - return 0; - } - return ret; -} - void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) { int tpr_threshold; @@ -7145,74 +7088,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap) vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]); } -static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu) -{ - /* - * Save xfd_err to guest_fpu before interrupt is enabled, so the - * MSR value is not clobbered by the host activity before the guest - * has chance to consume it. - * - * Update the guest's XFD_ERR if and only if XFD is enabled, as the #NM - * interception may have been caused by L1 interception. Per the SDM, - * XFD_ERR is not modified for non-XFD #NM, i.e. if CR0.TS=1. - * - * Note, XFD_ERR is updated _before_ the #NM interception check, i.e. - * unlike CR2 and DR6, the value is not a payload that is attached to - * the #NM exception. - */ - if (is_xfd_nm_fault(vcpu)) - rdmsrq(MSR_IA32_XFD_ERR, vcpu->arch.guest_fpu.xfd_err); -} - -static void handle_exception_irqoff(struct kvm_vcpu *vcpu, u32 intr_info) -{ - /* if exit due to PF check for async PF */ - if (is_page_fault(intr_info)) - vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); - /* if exit due to NM, handle before interrupts are enabled */ - else if (is_nm_fault(intr_info)) - handle_nm_fault_irqoff(vcpu); - /* Handle machine checks before interrupts are enabled */ - else if (is_machine_check(intr_info)) - kvm_machine_check(); -} - -static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu, - u32 intr_info) -{ - unsigned int vector = intr_info & INTR_INFO_VECTOR_MASK; - - if (KVM_BUG(!is_external_intr(intr_info), vcpu->kvm, - "unexpected VM-Exit interrupt info: 0x%x", intr_info)) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ); - x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector); - kvm_after_interrupt(vcpu); - - vcpu->arch.at_instruction_boundary = true; -} - -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) -{ - if (to_vt(vcpu)->emulation_required) - return; - - switch (vmx_get_exit_reason(vcpu).basic) { - case EXIT_REASON_EXTERNAL_INTERRUPT: - handle_external_interrupt_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_EXCEPTION_NMI: - handle_exception_irqoff(vcpu, vmx_get_intr_info(vcpu)); - break; - case EXIT_REASON_MCE_DURING_VMENTRY: - kvm_machine_check(); - break; - default: - break; - } -} - /* * The kvm parameter can be NULL (module initialization, or invocation before * VM creation). Be sure to check the kvm parameter before using it. @@ -7252,7 +7127,7 @@ static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) if (vmx->loaded_vmcs->nmi_known_unmasked) return; - exit_intr_info = vmx_get_intr_info(&vmx->vcpu); + exit_intr_info = vt_get_intr_info(&vmx->vcpu); unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; vector = exit_intr_info & INTR_INFO_VECTOR_MASK; /* @@ -7421,14 +7296,14 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, * the fastpath even, all other exits must use the slow path. */ if (is_guest_mode(vcpu) && - vmx_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) + vt_get_exit_reason(vcpu).basic != EXIT_REASON_PREEMPTION_TIMER) return EXIT_FASTPATH_NONE; - switch (vmx_get_exit_reason(vcpu).basic) { + switch (vt_get_exit_reason(vcpu).basic) { case EXIT_REASON_MSR_WRITE: return handle_fastpath_wrmsr(vcpu); case EXIT_REASON_MSR_WRITE_IMM: - return handle_fastpath_wrmsr_imm(vcpu, vmx_get_exit_qual(vcpu), + return handle_fastpath_wrmsr_imm(vcpu, vt_get_exit_qual(vcpu), vmx_get_msr_imm_reg(vcpu)); case EXIT_REASON_PREEMPTION_TIMER: return handle_fastpath_preemption_timer(vcpu, force_immediate_exit); @@ -7441,17 +7316,6 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu, } } -noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu) -{ - if ((u16)vmx_get_exit_reason(vcpu).basic != EXIT_REASON_EXCEPTION_NMI || - !is_nmi(vmx_get_intr_info(vcpu))) - return; - - kvm_before_interrupt(vcpu, KVM_HANDLING_NMI); - x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR); - kvm_after_interrupt(vcpu); -} - static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, unsigned int flags) { @@ -7476,15 +7340,15 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu, vmx_enable_fb_clear(vmx); if (unlikely(vmx->fail)) { - vmx->vt.exit_reason.full = 0xdead; + vmx->vt.exit_reason.full = EXIT_REASON_UNDEFINED; goto out; } vmx->vt.exit_reason.full = vmcs_read32(VM_EXIT_REASON); - if (likely(!vmx_get_exit_reason(vcpu).failed_vmentry)) + if (likely(!vt_get_exit_reason(vcpu).failed_vmentry)) vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); - vmx_handle_nmi(vcpu); + vt_handle_nmi(vcpu); out: guest_state_exit_irqoff(); @@ -7621,7 +7485,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) * checking. */ if (vcpu->arch.nested_run_pending && - !vmx_get_exit_reason(vcpu).failed_vmentry) + !vt_get_exit_reason(vcpu).failed_vmentry) ++vcpu->stat.nested_run; vcpu->arch.nested_run_pending = 0; @@ -7632,7 +7496,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) trace_kvm_exit(vcpu, KVM_ISA_VMX); - if (unlikely(vmx_get_exit_reason(vcpu).failed_vmentry)) + if (unlikely(vt_get_exit_reason(vcpu).failed_vmentry)) return EXIT_FASTPATH_NONE; vmx->loaded_vmcs->launched = 1; @@ -7650,7 +7514,7 @@ void vmx_vcpu_free(struct kvm_vcpu *vcpu) struct vcpu_vmx *vmx = to_vmx(vcpu); if (enable_pml) - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid(vmx->vpid); nested_vmx_free_vcpu(vcpu); free_loaded_vmcs(vmx->loaded_vmcs); @@ -7682,8 +7546,8 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) * for the guest), etc. */ if (enable_pml) { - vmx->pml_pg = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); - if (!vmx->pml_pg) + vcpu->arch.pml_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!vcpu->arch.pml_page) goto free_vpid; } @@ -7754,7 +7618,7 @@ int vmx_vcpu_create(struct kvm_vcpu *vcpu) free_vmcs: free_loaded_vmcs(vmx->loaded_vmcs); free_pml: - vmx_destroy_pml_buffer(vmx); + vmx_destroy_pml_buffer(vcpu); free_vpid: free_vpid(vmx->vpid); return err; @@ -8139,6 +8003,13 @@ static __init void vmx_set_cpu_caps(void) kvm_cpu_cap_clear(X86_FEATURE_IBT); } + /* + * CPUID 0x80000008. Tell userspace that EFER.LMSLE must be zero; KVM + * never allows EFER.LMSLE to be set on Intel CPUs, as KVM supports long + * mode segment limits only in conjunction with nested SVM. + */ + kvm_cpu_cap_set(X86_FEATURE_EFER_LMSLE_MBZ); + kvm_setup_xss_caps(); kvm_finalize_cpu_caps(); } @@ -8442,21 +8313,13 @@ static void vmx_update_hv_timer(struct kvm_vcpu *vcpu, bool force_immediate_exit } #endif -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (WARN_ON_ONCE(!enable_pml)) - return; - guard(vmx_vmcs01)(vcpu); - /* - * Note, nr_memslots_dirty_logging can be changed concurrent with this - * code, but in that case another update request will be made and so - * the guest will never run with a stale PML value. - */ - if (atomic_read(&vcpu->kvm->nr_memslots_dirty_logging)) + if (enable) secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); else secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 1db461060c7e..74cd50f29fa4 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -265,13 +265,6 @@ struct vcpu_vmx { unsigned int ple_window; bool ple_window_dirty; - /* Support for PML */ -#define PML_LOG_NR_ENTRIES 512 - /* PML is written backwards: this is the first entry written by the CPU */ -#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) - - struct page *pml_pg; - /* apic deadline value in host tsc */ u64 hv_deadline_tsc; @@ -304,43 +297,6 @@ struct kvm_vmx { u64 *pid_table; }; -static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu) -{ - return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt); -} - -static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt) -{ - return &(container_of(vt, struct vcpu_vmx, vt)->vcpu); -} - -static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu) -{ - return to_vt(vcpu)->exit_reason; -} - -static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); - - return vt->exit_qualification; -} - -static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) -{ - struct vcpu_vt *vt = to_vt(vcpu); - - if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) && - !WARN_ON_ONCE(is_td_vcpu(vcpu))) - vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - - return vt->exit_intr_info; -} - void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu); int allocate_vpid(void); void free_vpid(int vpid); @@ -397,7 +353,7 @@ u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated); bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated); diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h index 054fd14bb2e1..cd72684d075d 100644 --- a/arch/x86/kvm/vmx/x86_ops.h +++ b/arch/x86/kvm/vmx/x86_ops.h @@ -27,7 +27,6 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event); void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu); void vmx_vcpu_put(struct kvm_vcpu *vcpu); int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath); -void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu); int vmx_skip_emulated_instruction(struct kvm_vcpu *vcpu); void vmx_update_emulated_instruction(struct kvm_vcpu *vcpu); bool vmx_unhandleable_emulation_required(struct kvm_vcpu *vcpu); @@ -82,7 +81,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); bool vmx_get_if_flag(struct kvm_vcpu *vcpu); void vmx_flush_tlb_all(struct kvm_vcpu *vcpu); void vmx_flush_tlb_current(struct kvm_vcpu *vcpu); -void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr, bool *full); +void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr); void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu); void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask); u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu); @@ -95,6 +94,7 @@ int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection); int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection); bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu); void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked); +void vmx_clear_hlt(struct kvm_vcpu *vcpu); void vmx_enable_nmi_window(struct kvm_vcpu *vcpu); void vmx_enable_irq_window(struct kvm_vcpu *vcpu); void vmx_update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr); @@ -118,7 +118,7 @@ u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu); u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); void vmx_write_tsc_offset(struct kvm_vcpu *vcpu); void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu); -void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu, bool enable); #ifdef CONFIG_X86_64 int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, bool *expired); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index aad065d035fb..32563a91fba9 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -35,6 +35,7 @@ #include "smm.h" #include +#include #include #include #include @@ -61,6 +62,7 @@ #include #include #include +#include #include #include @@ -399,30 +401,37 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, switch (ex->vector) { case DB_VECTOR: /* - * "Certain debug exceptions may clear bit 0-3. The - * remaining contents of the DR6 register are never - * cleared by the processor". + * DR6 is a mess. Reserved/unused bits are fixed-to-1, and so + * to maintain backwards compatibility with existing software, + * features that use previously-reserved bits have active-low + * semantics, i.e. clear the bit when the feature is present in + * the payload. + * + * Further complicating matters, some DR6 bits are preserved by + * hardware, while others are explicitly modified on every #DB. + * The trap bits are always set based on the payload, as is the + * RTM flag (but it's active low). All other bits are modified + * if and only if a relevant debug exception occurs, e.g. BD, + * BS, and BT are never cleared by hardware, and BLD is never + * set by hardware (when supported, excepting RESET). + * + * Lastly, the payload does NOT have active-low semantics, e.g. + * so that it's compatible VMX's pending debug exceptions and + * qualification fields, and to avoid bleeding the DR6 madness + * into other KVM code. + * + * To compute DR6: + * + * 1. "Reset" the bits that are modified on all #DBs + * 2. Clear active-low bits that are present in the payload. + * 3. Set active-high bits that are present in the payload. + * 4. Clear fixed-0 bits. + * 5. Set fixed-1 bits. */ vcpu->arch.dr6 &= ~DR_TRAP_BITS; - /* - * In order to reflect the #DB exception payload in guest - * dr6, three components need to be considered: active low - * bit, FIXED_1 bits and active high bits (e.g. DR6_BD, - * DR6_BS and DR6_BT) - * DR6_ACTIVE_LOW contains the FIXED_1 and active low bits. - * In the target guest dr6: - * FIXED_1 bits should always be set. - * Active low bits should be cleared if 1-setting in payload. - * Active high bits should be set if 1-setting in payload. - * - * Note, the payload is compatible with the pending debug - * exceptions/exit qualification under VMX, that active_low bits - * are active high in payload. - * So they need to be flipped for DR6. - */ - vcpu->arch.dr6 |= DR6_ACTIVE_LOW; - vcpu->arch.dr6 |= ex->payload; - vcpu->arch.dr6 ^= ex->payload & DR6_ACTIVE_LOW; + vcpu->arch.dr6 |= DR6_RTM; + vcpu->arch.dr6 &= ~(ex->payload & DR6_ACTIVE_LOW); + vcpu->arch.dr6 |= (ex->payload & ~DR6_ACTIVE_LOW); /* * The #DB payload is defined as compatible with the 'pending @@ -431,6 +440,7 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, * breakpoint), it is reserved and must be zero in DR6. */ vcpu->arch.dr6 &= ~BIT(12); + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); break; case PF_VECTOR: vcpu->arch.cr2 = ex->payload; @@ -448,6 +458,8 @@ static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vecto { struct kvm_queued_exception *ex = &vcpu->arch.exception_vmexit; + kvm_make_request(KVM_REQ_EVENT, vcpu); + ex->vector = vector; ex->injected = false; ex->pending = true; @@ -925,19 +937,13 @@ static void update_pvclock_gtod(struct timekeeper *tk) write_seqcount_end(&vdata->seq); } +#endif static s64 get_kvmclock_base_ns(void) { /* Count up from boot time, but with the frequency of the raw clock. */ - return ktime_to_ns(ktime_add(ktime_get_raw(), pvclock_gtod_data.offs_boot)); + return ktime_to_ns(ktime_mono_to_any(ktime_get_raw(), TK_OFFS_BOOT)); } -#else -static s64 get_kvmclock_base_ns(void) -{ - /* Master clock not used, so we can just use CLOCK_BOOTTIME. */ - return ktime_get_boottime_ns(); -} -#endif static uint32_t div_frac(uint32_t dividend, uint32_t divisor) { @@ -945,32 +951,57 @@ static uint32_t div_frac(uint32_t dividend, uint32_t divisor) return dividend; } -static void kvm_get_time_scale(uint64_t scaled_hz, uint64_t base_hz, +static void kvm_get_time_scale(u64 scaled_hz, u64 base_hz, s8 *pshift, u32 *pmultiplier) { - uint64_t scaled64; - int32_t shift = 0; - uint64_t tps64; - uint32_t tps32; + u64 scaled_hz_u64 = scaled_hz; + s32 shift = 0; + u64 base_hz_u64; + u32 base32; - tps64 = base_hz; - scaled64 = scaled_hz; - while (tps64 > scaled64*2 || tps64 & 0xffffffff00000000ULL) { - tps64 >>= 1; + /* + * This function calculates a fixed-point multiplier and shift such + * that: + * time_ns = (tsc_cycles << shift) * multiplier >> 32 + * + * Where tsc_cycles tick at base_hz, and time_ns should count at + * scaled_hz (typically NSEC_PER_SEC for a TSC→nanoseconds conversion). + * + * The multiplier is: (scaled_hz << 32) / base_hz, adjusted by shift + * to keep everything in range. + */ + + base_hz_u64 = base_hz; + + /* + * Start by shifting base_hz right until it fits in 32 bits, and + * is lower than double the target rate. This introduces a negative + * shift value which would result in pvclock_scale_delta() shifting + * the actual tick count right before performing the multiplication. + */ + while (base_hz_u64 > scaled_hz_u64 * 2 || base_hz_u64 >> 32) { + base_hz_u64 >>= 1; shift--; } - tps32 = (uint32_t)tps64; - while (tps32 <= scaled64 || scaled64 & 0xffffffff00000000ULL) { - if (scaled64 & 0xffffffff00000000ULL || tps32 & 0x80000000) - scaled64 >>= 1; + /* Now the shifted base_hz fits in 32 bits. */ + base32 = (u32)base_hz_u64; + + /* + * Next, shift scaled_hz right until it fits in 32 bits, and ensure + * that the shifted base_hz is strictly larger (so that the result of the + * final division also fits in 32 bits). + */ + while (base32 <= scaled_hz_u64 || scaled_hz_u64 >> 32) { + if (scaled_hz_u64 >> 32 || base32 & BIT(31)) + scaled_hz_u64 >>= 1; else - tps32 <<= 1; + base32 <<= 1; shift++; } *pshift = shift; - *pmultiplier = div_frac(scaled64, tps32); + *pmultiplier = div_frac(scaled_hz_u64, base32); } #ifdef CONFIG_X86_64 @@ -1061,11 +1092,15 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) static u64 compute_guest_tsc(struct kvm_vcpu *vcpu, s64 kernel_ns) { - u64 tsc = pvclock_scale_delta(kernel_ns-vcpu->arch.this_tsc_nsec, - vcpu->arch.virtual_tsc_mult, - vcpu->arch.virtual_tsc_shift); - tsc += vcpu->arch.this_tsc_write; - return tsc; + s64 delta_ns = kernel_ns - vcpu->arch.this_tsc_nsec; + u64 tsc; + + /* Handle negative deltas gracefully (master clock ref may be earlier) */ + tsc = pvclock_scale_delta(abs(delta_ns), + vcpu->arch.virtual_tsc_mult, + vcpu->arch.virtual_tsc_shift); + + return vcpu->arch.this_tsc_write + (delta_ns >= 0 ? tsc : -tsc); } #ifdef CONFIG_X86_64 @@ -1131,11 +1166,12 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio) return _tsc; } -u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) +u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, + u64 target_tsc) { u64 tsc; - tsc = kvm_scale_tsc(rdtsc(), vcpu->arch.l1_tsc_scaling_ratio); + tsc = kvm_scale_tsc(host_tsc, vcpu->arch.l1_tsc_scaling_ratio); return target_tsc - tsc; } @@ -1164,11 +1200,58 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_offset); u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) { - if (l2_multiplier != kvm_caps.default_tsc_scaling_ratio) - return mul_u64_u64_shr(l1_multiplier, l2_multiplier, - kvm_caps.tsc_scaling_ratio_frac_bits); + u8 frac_bits = kvm_caps.tsc_scaling_ratio_frac_bits; + u64 nested_multiplier; - return l1_multiplier; + if (l2_multiplier == kvm_caps.default_tsc_scaling_ratio) + return l1_multiplier; + + /* + * The shift is fixed on both AMD and Intel, and operates on a 64-bit + * value. I.e. a shift greater than 63 is completely nonsensical. + */ + if (WARN_ON_ONCE(frac_bits > 63)) + return l1_multiplier; + + /* + * If the resulting multiplier can't be programmed into hardware, run + * L2 at the minimum/maximum frequency supported by hardware, i.e. + * saturate L2's frequency on both sides. Because L2's frequency needs + * to be distilled down to a single multiplier to get from: + * + * L2 = (((L0 * L1_mult) >> frac) * L2_mult) >> frac) + * + * to: + * + * L2 = (L0 * mult) >> frac + * + * very small/large L1 and L2 multipliers can underflow/overflow the + * minimum/maximum multiplier supported by hardware when combined into + * a single value. + * + * Manually check for the case where the result would overflow a u64, + * i.e. if the multiplier would be silently truncated before the "too + * large" check. Avoid doing the multiply twice in the common case + * where the compiler natively supports 128-bit values. + */ +#ifdef CONFIG_ARCH_SUPPORTS_INT128 + unsigned __int128 m = (unsigned __int128)l1_multiplier * l2_multiplier; + + if (m >> (64 + frac_bits)) + return kvm_caps.max_tsc_scaling_ratio; + + nested_multiplier = m >> frac_bits; +#else + if (mul_u64_u64_shr(l1_multiplier, l2_multiplier, 64 + frac_bits)) + return kvm_caps.max_tsc_scaling_ratio; + + nested_multiplier = mul_u64_u64_shr(l1_multiplier, l2_multiplier, frac_bits); +#endif + if (nested_multiplier > kvm_caps.max_tsc_scaling_ratio) + return kvm_caps.max_tsc_scaling_ratio; + + /* The minimum multiplier is '1' on both AMD and Intel. */ + return nested_multiplier ?: 1; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_calc_nested_tsc_multiplier); @@ -1254,6 +1337,7 @@ static void __kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 offset, u64 tsc, kvm->arch.last_tsc_write = tsc; kvm->arch.last_tsc_khz = vcpu->arch.virtual_tsc_khz; kvm->arch.last_tsc_offset = offset; + kvm->arch.last_tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; vcpu->arch.last_guest_tsc = tsc; @@ -1296,7 +1380,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) bool synchronizing = false; raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); - offset = kvm_compute_l1_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); ns = get_kvmclock_base_ns(); elapsed = ns - kvm->arch.last_tsc_nsec; @@ -1345,7 +1429,7 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) } else { u64 delta = nsec_to_cycles(vcpu, elapsed); data += delta; - offset = kvm_compute_l1_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), data); } matched = true; } @@ -1356,126 +1440,27 @@ void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 *user_value) #ifdef CONFIG_X86_64 -static u64 read_tsc(void) +static bool kvm_snapshot_has_tsc(struct system_time_snapshot *snap, + u64 *tsc_timestamp) { - u64 ret = (u64)rdtsc_ordered(); - u64 last = pvclock_gtod_data.clock.cycle_last; - - if (likely(ret >= last)) - return ret; - /* - * GCC likes to generate cmov here, but this branch is extremely - * predictable (it's just a function of time and the likely is - * very likely) and there's a data dependence, so force GCC - * to generate a branch instead. I don't barrier() because - * we don't actually need a barrier, and if this function - * ever gets inlined it will generate worse code. + * ktime_get_snapshot_id() cannot fail for standard clock IDs + * (only for invalid/aux clocks or during suspend, with a WARN). */ - asm volatile (""); - return last; -} + if (!snap->valid) + return false; -static inline u64 vgettsc(struct pvclock_clock *clock, u64 *tsc_timestamp, - int *mode) -{ - u64 tsc_pg_val; - long v; - - switch (clock->vclock_mode) { - case VDSO_CLOCKMODE_HVCLOCK: - if (hv_read_tsc_page_tsc(hv_get_tsc_page(), - tsc_timestamp, &tsc_pg_val)) { - /* TSC page valid */ - *mode = VDSO_CLOCKMODE_HVCLOCK; - v = (tsc_pg_val - clock->cycle_last) & - clock->mask; - } else { - /* TSC page invalid */ - *mode = VDSO_CLOCKMODE_NONE; - } - break; - case VDSO_CLOCKMODE_TSC: - *mode = VDSO_CLOCKMODE_TSC; - *tsc_timestamp = read_tsc(); - v = (*tsc_timestamp - clock->cycle_last) & - clock->mask; - break; - default: - *mode = VDSO_CLOCKMODE_NONE; + if (snap->cs_id == CSID_X86_TSC) { + *tsc_timestamp = snap->cycles; + return true; } - if (*mode == VDSO_CLOCKMODE_NONE) - *tsc_timestamp = v = 0; + if (snap->hw_csid == CSID_X86_TSC && snap->hw_cycles) { + *tsc_timestamp = snap->hw_cycles; + return true; + } - return v * clock->mult; -} - -/* - * As with get_kvmclock_base_ns(), this counts from boot time, at the - * frequency of CLOCK_MONOTONIC_RAW (hence adding gtos->offs_boot). - */ -static int do_kvmclock_base(s64 *t, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ns = gtod->raw_clock.base_cycles; - ns += vgettsc(>od->raw_clock, tsc_timestamp, &mode); - ns >>= gtod->raw_clock.shift; - ns += ktime_to_ns(ktime_add(gtod->raw_clock.offset, gtod->offs_boot)); - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - *t = ns; - - return mode; -} - -/* - * This calculates CLOCK_MONOTONIC at the time of the TSC snapshot, with - * no boot time offset. - */ -static int do_monotonic(s64 *t, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ns = gtod->clock.base_cycles; - ns += vgettsc(>od->clock, tsc_timestamp, &mode); - ns >>= gtod->clock.shift; - ns += ktime_to_ns(gtod->clock.offset); - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - *t = ns; - - return mode; -} - -static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) -{ - struct pvclock_gtod_data *gtod = &pvclock_gtod_data; - unsigned long seq; - int mode; - u64 ns; - - do { - seq = read_seqcount_begin(>od->seq); - ts->tv_sec = gtod->wall_time_sec; - ns = gtod->clock.base_cycles; - ns += vgettsc(>od->clock, tsc_timestamp, &mode); - ns >>= gtod->clock.shift; - } while (unlikely(read_seqcount_retry(>od->seq, seq))); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return mode; + return false; } /* @@ -1485,12 +1470,14 @@ static int do_realtime(struct timespec64 *ts, u64 *tsc_timestamp) */ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_MONOTONIC_RAW, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_kvmclock_base(kernel_ns, - tsc_timestamp)); + *kernel_ns = ktime_to_ns(ktime_mono_to_any(snap.systime, TK_OFFS_BOOT)); + return true; } /* @@ -1499,12 +1486,14 @@ static bool kvm_get_time_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) */ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_MONOTONIC, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_monotonic(kernel_ns, - tsc_timestamp)); + *kernel_ns = ktime_to_ns(snap.systime); + return true; } /* @@ -1517,11 +1506,14 @@ bool kvm_get_monotonic_and_clockread(s64 *kernel_ns, u64 *tsc_timestamp) static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, u64 *tsc_timestamp) { - /* checked again under seqlock below */ - if (!gtod_is_based_on_tsc(pvclock_gtod_data.clock.vclock_mode)) + struct system_time_snapshot snap = {}; + + ktime_get_snapshot_id(CLOCK_REALTIME, &snap); + if (!kvm_snapshot_has_tsc(&snap, tsc_timestamp)) return false; - return gtod_is_based_on_tsc(do_realtime(ts, tsc_timestamp)); + *ts = ktime_to_timespec64(snap.systime); + return true; } #endif @@ -1566,6 +1558,8 @@ static bool kvm_get_walltime_and_clockread(struct timespec64 *ts, * */ +static unsigned long get_cpu_tsc_khz(void); + static void pvclock_update_vm_gtod_copy(struct kvm *kvm) { #ifdef CONFIG_X86_64 @@ -1589,9 +1583,30 @@ static void pvclock_update_vm_gtod_copy(struct kvm *kvm) && !ka->backwards_tsc_observed && !ka->boot_vcpu_runs_old_kvmclock; - if (ka->use_master_clock) + if (ka->use_master_clock) { + u64 tsc_hz; + atomic_set(&kvm_guest_has_master_clock, 1); + /* + * Copy the scaling ratio and precompute the mul/shift for + * converting guest TSC to nanoseconds. These are used by + * get_kvmclock() to compute kvmclock from the host TSC + * without needing a vCPU reference. + */ + ka->master_tsc_scaling_ratio = ka->last_tsc_scaling_ratio; + tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + if (tsc_hz && kvm_caps.has_tsc_control) + tsc_hz = kvm_scale_tsc(tsc_hz, + ka->master_tsc_scaling_ratio); + if (tsc_hz) + kvm_get_time_scale(NSEC_PER_SEC, tsc_hz, + &ka->master_tsc_shift, + &ka->master_tsc_mul); + else + ka->use_master_clock = false; + } + vclock_mode = pvclock_gtod_data.clock.vclock_mode; trace_kvm_update_master_clock(ka->use_master_clock, vclock_mode, vcpus_matched); @@ -1658,39 +1673,49 @@ static unsigned long get_cpu_tsc_khz(void) } /* Called within read_seqcount_begin/retry for kvm->pvclock_sc. */ -static void __get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) +static bool __get_kvmclock_master_clock(struct kvm *kvm, + struct kvm_clock_data *data) { +#ifdef CONFIG_X86_64 struct kvm_arch *ka = &kvm->arch; struct pvclock_vcpu_time_info hv_clock; + struct timespec64 ts; - /* both __this_cpu_read() and rdtsc() should be on the same cpu */ - get_cpu(); + if (!ka->use_master_clock) + return false; - data->flags = 0; - if (ka->use_master_clock && - (cpu_feature_enabled(X86_FEATURE_CONSTANT_TSC) || __this_cpu_read(cpu_tsc_khz))) { -#ifdef CONFIG_X86_64 - struct timespec64 ts; + if (!kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) + return false; - if (kvm_get_walltime_and_clockread(&ts, &data->host_tsc)) { - data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; - data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC; - } else -#endif - data->host_tsc = rdtsc(); + data->realtime = ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec; + data->flags |= KVM_CLOCK_REALTIME | KVM_CLOCK_HOST_TSC | + KVM_CLOCK_TSC_STABLE; - data->flags |= KVM_CLOCK_TSC_STABLE; - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - kvm_get_time_scale(NSEC_PER_SEC, get_cpu_tsc_khz() * 1000LL, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); + hv_clock.tsc_timestamp = ka->master_cycle_now; + hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; + + /* + * Use the precomputed guest-TSC-based mul/shift so that the kvmclock + * value matches what the guest computes from its own TSC. + */ + hv_clock.tsc_shift = ka->master_tsc_shift; + hv_clock.tsc_to_system_mul = ka->master_tsc_mul; + + if (kvm_caps.has_tsc_control) { + u64 tsc_delta = data->host_tsc - ka->master_cycle_now; + + tsc_delta = kvm_scale_tsc(tsc_delta, ka->master_tsc_scaling_ratio); + data->clock = hv_clock.system_time + + pvclock_scale_delta(tsc_delta, + hv_clock.tsc_to_system_mul, + hv_clock.tsc_shift); } else { - data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; + data->clock = __pvclock_read_cycles(&hv_clock, data->host_tsc); } - - put_cpu(); + return true; +#else + return false; +#endif } static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) @@ -1699,8 +1724,11 @@ static void get_kvmclock(struct kvm *kvm, struct kvm_clock_data *data) unsigned seq; do { + data->flags = 0; + seq = read_seqcount_begin(&ka->pvclock_sc); - __get_kvmclock(kvm, data); + if (!__get_kvmclock_master_clock(kvm, data)) + data->clock = get_kvmclock_base_ns() + ka->kvmclock_offset; } while (read_seqcount_retry(&ka->pvclock_sc, seq)); } @@ -1762,36 +1790,45 @@ static void kvm_setup_guest_pvclock(struct pvclock_vcpu_time_info *ref_hv_clock, int kvm_guest_time_update(struct kvm_vcpu *v) { + u64 tgt_tsc_hz, tsc_timestamp, host_tsc, master_tsc, master_ns; + struct kvm_arch *ka __maybe_unused = &v->kvm->arch; struct pvclock_vcpu_time_info hv_clock = {}; - unsigned long flags, tgt_tsc_khz; - unsigned seq; struct kvm_vcpu_arch *vcpu = &v->arch; - struct kvm_arch *ka = &v->kvm->arch; s64 kernel_ns; - u64 tsc_timestamp, host_tsc; - bool use_master_clock; - - kernel_ns = 0; - host_tsc = 0; /* * If the host uses TSC clock, then passthrough TSC as stable * to the guest. */ +#ifdef CONFIG_X86_64 + bool use_master_clock; + unsigned int seq; + do { seq = read_seqcount_begin(&ka->pvclock_sc); use_master_clock = ka->use_master_clock; - if (use_master_clock) { - host_tsc = ka->master_cycle_now; - kernel_ns = ka->master_kernel_ns; - } - } while (read_seqcount_retry(&ka->pvclock_sc, seq)); + if (!use_master_clock) + continue; - /* Keep irq disabled to prevent changes to the clock */ - local_irq_save(flags); - tgt_tsc_khz = get_cpu_tsc_khz(); - if (unlikely(tgt_tsc_khz == 0)) { - local_irq_restore(flags); + if (!kvm_get_time_and_clockread(&kernel_ns, &host_tsc)) { + use_master_clock = false; + continue; + } + + master_tsc = ka->master_cycle_now; + master_ns = ka->master_kernel_ns; + } while (read_seqcount_retry(&ka->pvclock_sc, seq)); +#else + const bool use_master_clock = false; +#endif + /* + * Ensure reading the TSC+frequency pair is done on the same CPU. When + * NOT using the master clock, the TSC frequency may vary between CPUs. + */ + preempt_disable(); + tgt_tsc_hz = (u64)get_cpu_tsc_khz() * HZ_PER_KHZ; + if (unlikely(tgt_tsc_hz == 0)) { + preempt_enable(); kvm_make_request(KVM_REQ_CLOCK_UPDATE, v); return 1; } @@ -1820,28 +1857,43 @@ int kvm_guest_time_update(struct kvm_vcpu *v) } } - local_irq_restore(flags); + /* + * Refresh L1's last "observed" TSC to match the PV clock's timestamp, + * e.g. so that the guest can't see a TSC that's behind the reference. + */ + vcpu->last_guest_tsc = tsc_timestamp; + + preempt_enable(); /* With all the info we got, fill in the values */ if (kvm_caps.has_tsc_control) { - tgt_tsc_khz = kvm_scale_tsc(tgt_tsc_khz, + tgt_tsc_hz = kvm_scale_tsc(tgt_tsc_hz, v->arch.l1_tsc_scaling_ratio); - tgt_tsc_khz = tgt_tsc_khz ? : 1; + tgt_tsc_hz = tgt_tsc_hz ? : 1; } - if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) { - kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL, + if (unlikely(vcpu->hw_tsc_hz != tgt_tsc_hz)) { + kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_hz, &vcpu->pvclock_tsc_shift, &vcpu->pvclock_tsc_mul); - vcpu->hw_tsc_khz = tgt_tsc_khz; + vcpu->hw_tsc_hz = tgt_tsc_hz; } hv_clock.tsc_shift = vcpu->pvclock_tsc_shift; hv_clock.tsc_to_system_mul = vcpu->pvclock_tsc_mul; - hv_clock.tsc_timestamp = tsc_timestamp; - hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; - vcpu->last_guest_tsc = tsc_timestamp; + /* + * If the master clock is NOT in use, the reference time placed in the + * hv_clock is "now". If master clock is in use, the reference time is + * the master clock's snapshot from some time in the past, not "now". + */ + if (use_master_clock) { + hv_clock.tsc_timestamp = kvm_read_l1_tsc(v, master_tsc); + hv_clock.system_time = master_ns + v->kvm->arch.kvmclock_offset; + } else { + hv_clock.tsc_timestamp = tsc_timestamp; + hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; + } /* If the host uses TSC clocksource, then it is stable */ hv_clock.flags = 0; @@ -1903,63 +1955,22 @@ int kvm_guest_time_update(struct kvm_vcpu *v) * wallclock and kvmclock times, and subtracting one from the other. * * Fall back to using their values at slightly different moments by - * calling ktime_get_real_ns() and get_kvmclock_ns() separately. + * calling ktime_get_real_ns() and get_kvmclock() separately. */ uint64_t kvm_get_wall_clock_epoch(struct kvm *kvm) { -#ifdef CONFIG_X86_64 - struct pvclock_vcpu_time_info hv_clock; - struct kvm_arch *ka = &kvm->arch; - unsigned long seq, local_tsc_khz; - struct timespec64 ts; - uint64_t host_tsc; + struct kvm_clock_data data; - do { - seq = read_seqcount_begin(&ka->pvclock_sc); - - local_tsc_khz = 0; - if (!ka->use_master_clock) - break; - - /* - * The TSC read and the call to get_cpu_tsc_khz() must happen - * on the same CPU. - */ - get_cpu(); - - local_tsc_khz = get_cpu_tsc_khz(); - - if (local_tsc_khz && - !kvm_get_walltime_and_clockread(&ts, &host_tsc)) - local_tsc_khz = 0; /* Fall back to old method */ - - put_cpu(); - - /* - * These values must be snapshotted within the seqcount loop. - * After that, it's just mathematics which can happen on any - * CPU at any time. - */ - hv_clock.tsc_timestamp = ka->master_cycle_now; - hv_clock.system_time = ka->master_kernel_ns + ka->kvmclock_offset; - - } while (read_seqcount_retry(&ka->pvclock_sc, seq)); + get_kvmclock(kvm, &data); /* - * If the conditions were right, and obtaining the wallclock+TSC was - * successful, calculate the KVM clock at the corresponding time and - * subtract one from the other to get the guest's epoch in nanoseconds - * since 1970-01-01. + * If get_kvmclock() captured both wallclock and kvmclock from the + * same TSC reading, use them for a precise epoch calculation. */ - if (local_tsc_khz) { - kvm_get_time_scale(NSEC_PER_SEC, local_tsc_khz * NSEC_PER_USEC, - &hv_clock.tsc_shift, - &hv_clock.tsc_to_system_mul); - return ts.tv_nsec + NSEC_PER_SEC * ts.tv_sec - - __pvclock_read_cycles(&hv_clock, host_tsc); - } -#endif - return ktime_get_real_ns() - get_kvmclock_ns(kvm); + if (data.flags & KVM_CLOCK_REALTIME) + return data.realtime - data.clock; + + return ktime_get_real_ns() - data.clock; } /* @@ -2583,7 +2594,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) mark_tsc_unstable("KVM discovered backwards TSC"); if (kvm_check_tsc_unstable()) { - u64 offset = kvm_compute_l1_tsc_offset(vcpu, + u64 offset = kvm_compute_l1_tsc_offset(vcpu, rdtsc(), vcpu->arch.last_guest_tsc); kvm_vcpu_write_tsc_offset(vcpu, offset); if (!vcpu->arch.guest_tsc_protected) @@ -3717,7 +3728,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, user_tsc_khz = (u32)arg; if (kvm_caps.has_tsc_control && - user_tsc_khz >= kvm_caps.max_guest_tsc_khz) + user_tsc_khz > kvm_caps.max_guest_tsc_khz) goto out; if (user_tsc_khz == 0) @@ -3941,6 +3952,37 @@ void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) kvm_vcpu_kick(vcpu); } +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx) +{ + u16 pml_tail_index; + u64 *pml_buf; + int i; + + /* + * PML index always points to the next available PML buffer entity + * unless PML log has just overflowed. + */ + pml_tail_index = (pml_idx >= PML_LOG_NR_ENTRIES) ? 0 : pml_idx + 1; + + /* + * PML log is written backwards: the CPU first writes the entry 511 + * then the entry 510, and so on. + * + * Read the entries in the same order they were written, to ensure that + * the dirty ring is filled in the same order the CPU wrote them. + */ + pml_buf = page_address(vcpu->arch.pml_page); + + for (i = PML_HEAD_INDEX; i >= pml_tail_index; i--) { + u64 gpa; + + gpa = pml_buf[i]; + WARN_ON_ONCE(gpa & (PAGE_SIZE - 1)); + kvm_vcpu_mark_page_dirty(vcpu, gpa >> PAGE_SHIFT); + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_pml_buffer); + int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) { @@ -4653,7 +4695,7 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) user_tsc_khz = (u32)arg; if (kvm_caps.has_tsc_control && - user_tsc_khz >= kvm_caps.max_guest_tsc_khz) + user_tsc_khz > kvm_caps.max_guest_tsc_khz) goto out; if (user_tsc_khz == 0) @@ -5089,7 +5131,8 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, * operation using rep will only have the initial GPA from the NPF * occurred. */ - if (ctxt->gpa_available && emulator_can_use_gpa(ctxt) && + if ((ctxt->gpa_access & (write ? ACC_WRITE_MASK : ACC_READ_MASK)) && + emulator_can_use_gpa(ctxt) && (addr & ~PAGE_MASK) == (ctxt->gpa_val & ~PAGE_MASK)) { gpa = ctxt->gpa_val; ret = vcpu_is_mmio_gpa(vcpu, addr, gpa, write); @@ -5938,7 +5981,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu) kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - ctxt->gpa_available = false; + ctxt->gpa_access = 0; ctxt->eflags = kvm_get_rflags(vcpu); ctxt->tf = (ctxt->eflags & X86_EFLAGS_TF) != 0; @@ -6454,7 +6497,17 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, /* With shadow page tables, cr2 contains a GVA or nGPA. */ if (vcpu->arch.mmu->root_role.direct) { - ctxt->gpa_available = true; + ctxt->gpa_access = ACC_READ_MASK; + /* + * Always allow writes for guests with protected page + * tables, as KVM can't walk the guest's page tables, + * i.e. KVM can't get the RMW protections for a given + * GVA to see if the write side of a RMW operation + * should be allowed. + */ + if ((emulation_type & EMULTYPE_PF_WRITE) || + vcpu->kvm->arch.has_protected_page_tables) + ctxt->gpa_access |= ACC_WRITE_MASK; ctxt->gpa_val = cr2_or_gpa; } } else { @@ -6930,7 +6983,14 @@ static void kvm_setup_efer_caps(void) if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { kvm_caps.supported_efer_bits |= EFER_SVME; - if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) + + /* + * Enumerating EFER_LMSLE_MBZ and allowing EFER.LMSLE=1 + * would be nonsensical. Note, vendor code sets the defeature + * if KVM can't support EFER.LMSLE for any reason, i.e. this + * needs to consult KVM's capabilities, not just raw CPUID. + */ + if (!kvm_cpu_cap_has(X86_FEATURE_EFER_LMSLE_MBZ)) kvm_caps.supported_efer_bits |= EFER_LMSLE; } } @@ -7131,7 +7191,8 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) if (kvm_caps.has_tsc_control) { /* * Make sure the user can only configure tsc_khz values that - * fit into a signed integer. + * fit into a signed integer, otherwise KVM_GET_TSC_KHZ would + * return a negative value and confuse userspace. * A min value is not calculated because it will always * be 1 on all machines. */ @@ -7925,10 +7986,8 @@ void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, kvm->arch.apicv_inhibit_reasons = new; if (new) { unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); - int idx = srcu_read_lock(&kvm->srcu); kvm_zap_gfn_range(kvm, gfn, gfn+1); - srcu_read_unlock(&kvm->srcu, idx); } } else { kvm->arch.apicv_inhibit_reasons = new; @@ -8042,6 +8101,21 @@ static void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) kvm_x86_call(set_apic_access_page_addr)(vcpu); } +static void kvm_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) +{ + /* + * Note, nr_memslots_dirty_logging can be changed concurrent with this + * code, but in that case another update request will be made and so + * the guest will never run with a stale PML value. + */ + bool enable = atomic_read(&vcpu->kvm->nr_memslots_dirty_logging); + + if (WARN_ON_ONCE(!vcpu->kvm->arch.cpu_dirty_log_size)) + return; + + kvm_x86_call(update_cpu_dirty_logging)(vcpu, enable); +} + /* * Called within kvm->srcu read side. * Returns 1 to let vcpu_run() continue the guest execution loop without @@ -8155,7 +8229,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_NMI, vcpu)) process_nmi(vcpu); if (kvm_check_request(KVM_REQ_IOAPIC_EOI_EXIT, vcpu)) { - BUG_ON(vcpu->arch.pending_ioapic_eoi > 255); if (test_bit(vcpu->arch.pending_ioapic_eoi, vcpu->arch.ioapic_handled_vectors)) { vcpu->run->exit_reason = KVM_EXIT_IOAPIC_EOI; @@ -8212,7 +8285,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_call(recalc_intercepts)(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) - kvm_x86_call(update_cpu_dirty_logging)(vcpu); + kvm_update_cpu_dirty_logging(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_PROTECTED_GUEST_STATE, vcpu)) { kvm_vcpu_reset(vcpu, true); @@ -8401,6 +8474,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) WARN_ON(vcpu->arch.switch_db_regs & KVM_DEBUGREG_AUTO_SWITCH); kvm_x86_call(sync_dirty_debug_regs)(vcpu); kvm_update_dr0123(vcpu); + vcpu->arch.dr6 |= kvm_get_dr6_fixed_1(vcpu); kvm_update_dr7(vcpu); } @@ -9062,6 +9136,19 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, } kvm_set_mp_state(vcpu, mp_state->mp_state); + + /* + * Force the vCPU out of any hardware-tracked halted state, e.g. VMX's + * GUEST_ACTIVITY_STATE=HLT, when userspace puts the vCPU into a state + * other than HALTED. The hardware state is sticky across VM-Exit and + * VM-Enter and is not touched by any other ioctl, so a vCPU that halted + * with HLT-exiting disabled stays wedged even after userspace rewrites + * its registers, e.g. when a VMM emulates a machine reset. + */ + if (kvm_hlt_in_guest(vcpu->kvm) && + mp_state->mp_state != KVM_MP_STATE_HALTED) + kvm_x86_call(clear_hlt)(vcpu); + kvm_make_request(KVM_REQ_EVENT, vcpu); ret = 0; @@ -9444,6 +9531,8 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) return; vcpu_load(vcpu); kvm_synchronize_tsc(vcpu, NULL); + if (kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu)) + kvm_update_masterclock(vcpu->kvm); vcpu_put(vcpu); /* poll control enabled by default */ @@ -10144,7 +10233,7 @@ static int kvm_alloc_memslot_metadata(struct kvm *kvm, } } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES kvm_mmu_init_memslot_memory_attributes(kvm, slot); #endif @@ -10641,14 +10730,15 @@ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) } #ifdef CONFIG_KVM_GUEST_MEMFD -/* - * KVM doesn't yet support initializing guest_memfd memory as shared for VMs - * with private memory (the private vs. shared tracking needs to be moved into - * guest_memfd). - */ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) { - return !kvm_arch_has_private_mem(kvm); + /* + * INIT_SHARED is supported if in-place conversion is enabled, or if + * the VM doesn't support private memory. If the VM has private memory + * and in-place conversion is disabled, then guest_memfd can _only_ be + * used for private memory. + */ + return gmem_in_place_conversion || !kvm_arch_has_private_mem(kvm); } #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT @@ -10657,6 +10747,11 @@ int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, { return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); } + +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +{ + kvm_x86_call(gmem_make_shared)(pfn, nr_pages); +} #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index f72897ae56bc..436e2fb1396c 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -335,7 +335,8 @@ u64 kvm_scale_tsc(u64 tsc, u64 ratio); u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc); u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier); u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier); -u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc); +u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 host_tsc, + u64 target_tsc); void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset); static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, @@ -407,6 +408,8 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * EMULTYPE_PF - Set when an intercepted #PF triggers the emulation, in which case * the CR2/GPA value pass on the stack is valid. * + * EMULTYPE_PF_WRITE - Set with EMULTYPE_PF when hardware reports a write access. + * * EMULTYPE_COMPLETE_USER_EXIT - Set when the emulator should update interruptibility * state and inject single-step #DBs after skipping * an instruction (after completing userspace I/O). @@ -445,6 +448,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, #define EMULTYPE_COMPLETE_USER_EXIT (1 << 7) #define EMULTYPE_WRITE_PF_TO_SP (1 << 8) #define EMULTYPE_SKIP_SOFT_INT (1 << 9) +#define EMULTYPE_PF_WRITE (1 << 10) #define EMULTYPE_SET_SOFT_INT_VECTOR(v) ((u32)((v) & 0xff) << 16) #define EMULTYPE_GET_SOFT_INT_VECTOR(e) (((e) >> 16) & 0xff) @@ -900,4 +904,11 @@ int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); +/* Support for PML */ +#define PML_LOG_NR_ENTRIES 512 +/* PML is written backwards: this is the first entry written by the CPU */ +#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1) + +void kvm_flush_pml_buffer(struct kvm_vcpu *vcpu, u16 pml_idx); + #endif diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index d113bd0f1c3d..6aa00f746cba 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -73,7 +73,7 @@ static int kvm_xen_shared_info_init(struct kvm *kvm) BUILD_BUG_ON(offsetof(struct shared_info, wc) != 0xc00); BUILD_BUG_ON(offsetof(struct shared_info, wc_sec_hi) != 0xc0c); - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(kvm)) { struct shared_info *shinfo = gpc->khva; wc_sec_hi = &shinfo->wc_sec_hi; @@ -389,7 +389,7 @@ static void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, bool atomic) BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) != sizeof(vx->runstate_times)); - if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(v->kvm)) { user_len = sizeof(struct vcpu_runstate_info); times_ofs = offsetof(struct vcpu_runstate_info, state_entry_time); @@ -676,28 +676,32 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) } /* Now gpc->khva is a valid kernel address for the vcpu_info */ - if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + if (kvm_xen_has_64bit_shinfo(v->kvm)) { struct vcpu_info *vi = gpc->khva; + void *vi_pending_sel = &vi->evtchn_pending_sel; - asm volatile(LOCK_PREFIX "orq %0, %1\n" - "notq %0\n" - LOCK_PREFIX "andq %0, %2\n" - : "=r" (evtchn_pending_sel), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel)); + if (IS_ALIGNED((unsigned long)vi_pending_sel, sizeof(u64))) { + atomic64_or(evtchn_pending_sel, vi_pending_sel); + } else { + atomic_or(evtchn_pending_sel, vi_pending_sel); + /* + * The cast keeps the shift well-defined on 32-bit, + * where evtchn_pending_sel is 32 bits wide and this + * branch is unreachable anyway (this is inside + * kvm_xen_has_64bit_shinfo(), which is gated on + * IS_ENABLED(CONFIG_64BIT)). + */ + atomic_or((u64)evtchn_pending_sel >> 32, + vi_pending_sel + 4); + } + + atomic64_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } else { - u32 evtchn_pending_sel32 = evtchn_pending_sel; struct compat_vcpu_info *vi = gpc->khva; - asm volatile(LOCK_PREFIX "orl %0, %1\n" - "notl %0\n" - LOCK_PREFIX "andl %0, %2\n" - : "=r" (evtchn_pending_sel32), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel32)); + atomic_or(evtchn_pending_sel, (void *)&vi->evtchn_pending_sel); + atomic_andnot(evtchn_pending_sel, (void *)&v->arch.xen.evtchn_pending_sel); WRITE_ONCE(vi->evtchn_upcall_pending, 1); } @@ -728,6 +732,16 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) BUILD_BUG_ON(sizeof(rc) != sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending)); + /* + * kvm_gpc_check() checks the memslot generation, so kvm->srcu must be + * held. Most callers hold it already, but this is also reached from + * kvm_emulate_halt() on the VM-Exit path and from kvm_vcpu_block(), + * where vcpu_enter_guest() has already dropped the vCPU's SRCU lock. + * Taking SRCU does not sleep, so it is safe even in the atomic case + * which is handled below. + */ + guard(srcu)(&v->kvm->srcu); + read_lock_irqsave(&gpc->lock, flags); while (!kvm_gpc_check(gpc, sizeof(struct vcpu_info))) { read_unlock_irqrestore(&gpc->lock, flags); @@ -941,6 +955,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + r = -ENXIO; + if (!IS_ALIGNED(data->u.gpa, sizeof(u32))) + break; + r = kvm_gpc_activate(&vcpu->arch.xen.vcpu_info_cache, data->u.gpa, sizeof(struct vcpu_info)); } else { @@ -950,6 +968,10 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } + r = -ENXIO; + if (!IS_ALIGNED(data->u.hva, sizeof(u32))) + break; + r = kvm_gpc_activate_hva(&vcpu->arch.xen.vcpu_info_cache, data->u.hva, sizeof(struct vcpu_info)); } @@ -993,7 +1015,7 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) * address, that's actually OK. kvm_xen_update_runstate_guest() * will cope. */ - if (IS_ENABLED(CONFIG_64BIT) && vcpu->kvm->arch.xen.long_mode) + if (kvm_xen_has_64bit_shinfo(vcpu->kvm)) sz = sizeof(struct vcpu_runstate_info); else sz = sizeof(struct compat_vcpu_runstate_info); @@ -1439,16 +1461,21 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result); } -static inline int max_evtchn_port(struct kvm *kvm) +static inline int max_evtchn_port(bool has_64bit_shinfo) { - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) + if (has_64bit_shinfo) return EVTCHN_2L_NR_CHANNELS; else return COMPAT_EVTCHN_2L_NR_CHANNELS; } -static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, - evtchn_port_t *ports) +static inline int kvm_max_evtchn_port(struct kvm *kvm) +{ + return max_evtchn_port(kvm_xen_has_64bit_shinfo(kvm)); +} + +static bool wait_pending_event(struct kvm_vcpu *vcpu, bool has_64bit_shinfo, + int nr_ports, evtchn_port_t *ports) { struct kvm *kvm = vcpu->kvm; struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; @@ -1463,7 +1490,7 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, goto out_rcu; ret = false; - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (has_64bit_shinfo) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; } else { @@ -1485,9 +1512,10 @@ static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, return ret; } -static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, +static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool is_64bit, u64 param, u64 *r) { + bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(vcpu->kvm); struct sched_poll sched_poll; evtchn_port_t port, *ports; struct x86_exception e; @@ -1497,7 +1525,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, !(vcpu->kvm->arch.xen.hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND)) return false; - if (IS_ENABLED(CONFIG_64BIT) && !longmode) { + if (IS_ENABLED(CONFIG_64BIT) && !is_64bit) { struct compat_sched_poll sp32; /* Sanity check that the compat struct definition is correct */ @@ -1546,20 +1574,20 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, } for (i = 0; i < sched_poll.nr_ports; i++) { - if (ports[i] >= max_evtchn_port(vcpu->kvm)) { + if (ports[i] >= max_evtchn_port(has_64bit_shinfo)) { *r = -EINVAL; goto out; } } if (sched_poll.nr_ports == 1) - vcpu->arch.xen.poll_evtchn = port; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, port); else - vcpu->arch.xen.poll_evtchn = -1; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, -1); set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask); - if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) { + if (!wait_pending_event(vcpu, has_64bit_shinfo, sched_poll.nr_ports, ports)) { kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); if (sched_poll.timeout) @@ -1574,7 +1602,7 @@ static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); } - vcpu->arch.xen.poll_evtchn = 0; + WRITE_ONCE(vcpu->arch.xen.poll_evtchn, 0); *r = 0; out: /* Really, this is only needed in case of timeout */ @@ -1594,12 +1622,12 @@ static void cancel_evtchn_poll(struct timer_list *t) kvm_vcpu_kick(vcpu); } -static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool longmode, +static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd, u64 param, u64 *r) { switch (cmd) { case SCHEDOP_poll: - if (kvm_xen_schedop_poll(vcpu, longmode, param, r)) + if (kvm_xen_schedop_poll(vcpu, is_64bit, param, r)) return true; fallthrough; case SCHEDOP_yield: @@ -1618,7 +1646,7 @@ struct compat_vcpu_set_singleshot_timer { uint32_t flags; } __attribute__((packed)); -static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, +static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool is_64bit, int cmd, int vcpu_id, u64 param, u64 *r) { struct vcpu_set_singleshot_timer oneshot; @@ -1662,7 +1690,7 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, flags) != sizeof_field(struct vcpu_set_singleshot_timer, flags)); - if (kvm_read_guest_virt(vcpu, param, &oneshot, longmode ? sizeof(oneshot) : + if (kvm_read_guest_virt(vcpu, param, &oneshot, is_64bit ? sizeof(oneshot) : sizeof(struct compat_vcpu_set_singleshot_timer), &e)) { *r = -EFAULT; return true; @@ -1694,7 +1722,7 @@ static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout, int kvm_xen_hypercall(struct kvm_vcpu *vcpu) { - bool longmode; + bool is_64bit; u64 input, params[6], r = -ENOSYS; bool handled = false; u8 cpl; @@ -1704,8 +1732,8 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) kvm_hv_hypercall_enabled(vcpu)) return kvm_hv_hypercall(vcpu); - longmode = is_64_bit_hypercall(vcpu); - if (!longmode) { + is_64bit = is_64_bit_hypercall(vcpu); + if (!is_64bit) { input = kvm_eax_read(vcpu); params[0] = kvm_ebx_read(vcpu); params[1] = kvm_ecx_read(vcpu); @@ -1751,17 +1779,17 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r); break; case __HYPERVISOR_sched_op: - handled = kvm_xen_hcall_sched_op(vcpu, longmode, params[0], + handled = kvm_xen_hcall_sched_op(vcpu, is_64bit, params[0], params[1], &r); break; case __HYPERVISOR_vcpu_op: - handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1], + handled = kvm_xen_hcall_vcpu_op(vcpu, is_64bit, params[0], params[1], params[2], &r); break; case __HYPERVISOR_set_timer_op: { u64 timeout = params[0]; /* In 32-bit mode, the 64-bit timeout is in two 32-bit params. */ - if (!longmode) + if (!is_64bit) timeout |= params[1] << 32; handled = kvm_xen_hcall_set_timer_op(vcpu, timeout, &r); break; @@ -1776,7 +1804,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handle_in_userspace: vcpu->run->exit_reason = KVM_EXIT_XEN; vcpu->run->xen.type = KVM_EXIT_XEN_HCALL; - vcpu->run->xen.u.hcall.longmode = longmode; + vcpu->run->xen.u.hcall.longmode = is_64bit; vcpu->run->xen.u.hcall.cpl = cpl; vcpu->run->xen.u.hcall.input = input; vcpu->run->xen.u.hcall.params[0] = params[0]; @@ -1794,7 +1822,7 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) { - int poll_evtchn = vcpu->arch.xen.poll_evtchn; + int poll_evtchn = READ_ONCE(vcpu->arch.xen.poll_evtchn); if ((poll_evtchn == port || poll_evtchn == -1) && test_and_clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.xen.poll_mask)) { @@ -1816,8 +1844,9 @@ static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) { struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; + bool has_64bit_shinfo = kvm_xen_has_64bit_shinfo(kvm); + unsigned long *pending_bits, *mask_bits, vi_pending_sel_ofs; struct kvm_vcpu *vcpu; - unsigned long *pending_bits, *mask_bits; unsigned long flags; int port_word_bit; bool kick_vcpu = false; @@ -1833,7 +1862,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) WRITE_ONCE(xe->vcpu_idx, vcpu->vcpu_idx); } - if (xe->port >= max_evtchn_port(kvm)) + if (xe->port >= max_evtchn_port(has_64bit_shinfo)) return -EINVAL; rc = -EWOULDBLOCK; @@ -1844,16 +1873,23 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) if (!kvm_gpc_check(gpc, PAGE_SIZE)) goto out_rcu; - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + if (has_64bit_shinfo) { struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; port_word_bit = xe->port / 64; + + vi_pending_sel_ofs = offsetof(struct vcpu_info, evtchn_pending_sel); } else { struct compat_shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; port_word_bit = xe->port / 32; + + vi_pending_sel_ofs = offsetof(struct compat_vcpu_info, evtchn_pending_sel); + + /* test_and_set_bit() needs 64-bit alignment, but that's OK */ + BUILD_BUG_ON(offsetof(struct compat_shared_info, evtchn_pending) & 7); } /* @@ -1869,6 +1905,8 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) rc = -ENOTCONN; /* Masked */ kvm_xen_check_poller(vcpu, xe->port); } else { + bool old; + rc = 1; /* Delivered to the bitmap in shared_info. */ /* Now switch to the vCPU's vcpu_info to set the index and pending_sel */ read_unlock_irqrestore(&gpc->lock, flags); @@ -1885,19 +1923,29 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) goto out_rcu; } - if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { - struct vcpu_info *vcpu_info = gpc->khva; - if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { - WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); - kick_vcpu = true; - } - } else { - struct compat_vcpu_info *vcpu_info = gpc->khva; - if (!test_and_set_bit(port_word_bit, - (unsigned long *)&vcpu_info->evtchn_pending_sel)) { - WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); - kick_vcpu = true; - } + /* + * Explicitly use a 32-bit btsl instead of test_and_set_bit(), + * which would use btsq on x86-64. The vcpu_info is guest- + * controlled and only required to be 32-bit aligned, so a + * 64-bit access could generate a split-lock #AC. + * + * Note, this does not apply to the test_and_set_bit() on + * pending_bits above: that is in the per-VM shared_info, which + * is page aligned, so the access is guaranteed to be 64-bit + * aligned. + */ + old = GEN_BINARY_RMWcc(LOCK_PREFIX "btsl", + *(u32 *)(gpc->khva + vi_pending_sel_ofs), + c, "Ir", port_word_bit); + if (!old) { + struct vcpu_info *vi = gpc->khva; + + /* No need for compat handling */ + BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) != + offsetof(struct compat_vcpu_info, evtchn_upcall_pending)); + + WRITE_ONCE(vi->evtchn_upcall_pending, 1); + kick_vcpu = true; } /* For the per-vCPU lapic vector, deliver it as MSI. */ @@ -1995,7 +2043,7 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, struct kvm_vcpu *vcpu; /* - * Don't check for the port being within range of max_evtchn_port(). + * Don't check for the port being within range of kvm_max_evtchn_port(). * Userspace can configure what ever targets it likes; events just won't * be delivered if/while the target is invalid, just like userspace can * configure MSIs which target non-existent APICs. @@ -2004,8 +2052,8 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, * can be restored *independently* of other things like creating vCPUs, * without imposing an ordering dependency on userspace. In this * particular case, the problematic ordering would be with setting the - * Xen 'long mode' flag, which changes max_evtchn_port() to allow 4096 - * instead of 1024 event channels. + * Xen 'long mode' flag, which changes kvm_max_evtchn_port() to allow + * 4096 instead of 1024 event channels. */ /* We only support 2 level event channels for now */ @@ -2042,7 +2090,7 @@ int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn * struct kvm_xen_evtchn e; int ret; - if (!uxe->port || uxe->port >= max_evtchn_port(kvm)) + if (!uxe->port || uxe->port >= kvm_max_evtchn_port(kvm)) return -EINVAL; /* We only support 2 level event channels for now */ @@ -2093,7 +2141,7 @@ static int kvm_xen_eventfd_update(struct kvm *kvm, /* Protect writes to evtchnfd as well as the idr lookup. */ mutex_lock(&kvm->arch.xen.xen_lock); - evtchnfd = idr_find(&kvm->arch.xen.evtchn_ports, port); + evtchnfd = xa_load(&kvm->arch.xen.evtchn_ports, port); ret = -ENOENT; if (!evtchnfd) @@ -2152,7 +2200,7 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, case EVTCHNSTAT_interdomain: if (data->u.evtchn.deliver.port.port) { - if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm)) + if (data->u.evtchn.deliver.port.port >= kvm_max_evtchn_port(kvm)) goto out_noeventfd; /* -EINVAL */ } else { eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd); @@ -2187,13 +2235,13 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, } mutex_lock(&kvm->arch.xen.xen_lock); - ret = idr_alloc(&kvm->arch.xen.evtchn_ports, evtchnfd, port, port + 1, + ret = xa_insert(&kvm->arch.xen.evtchn_ports, port, evtchnfd, GFP_KERNEL); mutex_unlock(&kvm->arch.xen.xen_lock); - if (ret >= 0) + if (!ret) return 0; - if (ret == -ENOSPC) + if (ret == -EBUSY) ret = -EEXIST; out: if (eventfd) @@ -2208,7 +2256,7 @@ static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port) struct evtchnfd *evtchnfd; mutex_lock(&kvm->arch.xen.xen_lock); - evtchnfd = idr_remove(&kvm->arch.xen.evtchn_ports, port); + evtchnfd = xa_erase(&kvm->arch.xen.evtchn_ports, port); mutex_unlock(&kvm->arch.xen.xen_lock); if (!evtchnfd) @@ -2224,7 +2272,7 @@ static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port) static int kvm_xen_eventfd_reset(struct kvm *kvm) { struct evtchnfd *evtchnfd, **all_evtchnfds; - int i; + unsigned long i; int n = 0; mutex_lock(&kvm->arch.xen.xen_lock); @@ -2234,7 +2282,7 @@ static int kvm_xen_eventfd_reset(struct kvm *kvm) * critical section, first collect all the evtchnfd objects * in an array as they are removed from evtchn_ports. */ - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) n++; all_evtchnfds = kmalloc_objs(struct evtchnfd *, n); @@ -2244,9 +2292,9 @@ static int kvm_xen_eventfd_reset(struct kvm *kvm) } n = 0; - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) { all_evtchnfds[n++] = evtchnfd; - idr_remove(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port); + xa_erase(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port); } mutex_unlock(&kvm->arch.xen.xen_lock); @@ -2270,7 +2318,7 @@ static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data if (data->u.evtchn.flags == KVM_XEN_EVTCHN_RESET) return kvm_xen_eventfd_reset(kvm); - if (!port || port >= max_evtchn_port(kvm)) + if (!port || port >= kvm_max_evtchn_port(kvm)) return -EINVAL; if (data->u.evtchn.flags == KVM_XEN_EVTCHN_DEASSIGN) @@ -2297,12 +2345,10 @@ static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r) } /* - * evtchnfd is protected by kvm->srcu; the idr lookup instead - * is protected by RCU. + * evtchnfd is protected by kvm->srcu; the xa_load is RCU-safe + * internally, no explicit rcu_read_lock() needed. */ - rcu_read_lock(); - evtchnfd = idr_find(&vcpu->kvm->arch.xen.evtchn_ports, send.port); - rcu_read_unlock(); + evtchnfd = xa_load(&vcpu->kvm->arch.xen.evtchn_ports, send.port); if (!evtchnfd) return false; @@ -2349,23 +2395,23 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) void kvm_xen_init_vm(struct kvm *kvm) { mutex_init(&kvm->arch.xen.xen_lock); - idr_init(&kvm->arch.xen.evtchn_ports); + xa_init(&kvm->arch.xen.evtchn_ports); kvm_gpc_init(&kvm->arch.xen.shinfo_cache, kvm); } void kvm_xen_destroy_vm(struct kvm *kvm) { struct evtchnfd *evtchnfd; - int i; + unsigned long i; kvm_gpc_deactivate(&kvm->arch.xen.shinfo_cache); - idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { + xa_for_each(&kvm->arch.xen.evtchn_ports, i, evtchnfd) { if (!evtchnfd->deliver.port.port) eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx); kfree(evtchnfd); } - idr_destroy(&kvm->arch.xen.evtchn_ports); + xa_destroy(&kvm->arch.xen.evtchn_ports); if (kvm->arch.xen.hvm_config.msr) static_branch_slow_dec_deferred(&kvm_xen_enabled); diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index f372855857a8..9d04e350bdb1 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -235,6 +235,11 @@ struct compat_shared_info { #define COMPAT_EVTCHN_2L_NR_CHANNELS (8 * \ sizeof_field(struct compat_shared_info, \ evtchn_pending)) + +/* Latched VM-wide mode; the KVM equivalent of Xen's !has_32bit_shinfo(). */ +#define kvm_xen_has_64bit_shinfo(kvm) \ + (IS_ENABLED(CONFIG_64BIT) && READ_ONCE((kvm)->arch.xen.long_mode)) + struct compat_vcpu_runstate_info { int state; uint64_t state_entry_time; diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c index fb1cecef613b..9d60f474aa4f 100644 --- a/drivers/irqchip/irq-gic-v5-irs.c +++ b/drivers/irqchip/irq-gic-v5-irs.c @@ -21,8 +21,6 @@ */ #define LPI_ID_BITS_LINEAR 12 -#define IRS_FLAGS_NON_COHERENT BIT(0) - static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data); static LIST_HEAD(irs_nodes); @@ -50,6 +48,11 @@ static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data, writeq_relaxed(val, irs_data->irs_base + reg_offset); } +struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void) +{ + return per_cpu(per_cpu_irs_data, 0); +} + /* * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register * provides the memory barriers (through MMIO accessors) @@ -266,24 +269,24 @@ int gicv5_irs_iste_alloc(const u32 lpi) * itself is not supported) again serves to make it easier to find physically * contiguous blocks of memory. */ -static unsigned int gicv5_irs_l2_sz(u32 idr2) +unsigned int gicv5_irs_l2_sz(u32 l2sz) { switch (PAGE_SIZE) { case SZ_64K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_64K; fallthrough; case SZ_4K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_4K; fallthrough; case SZ_16K: - if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_16K; break; } - if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz)) return GICV5_IRS_IST_CFGR_L2SZ_4K; return GICV5_IRS_IST_CFGR_L2SZ_64K; @@ -331,7 +334,7 @@ static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); if (two_levels) - l2sz = gicv5_irs_l2_sz(idr2); + l2sz = gicv5_irs_l2_sz(FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, idr2)); istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2); diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index 74be71be479a..8c0303af71e7 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -1122,8 +1122,10 @@ static void gicv5_set_cpuif_idbits(void) #ifdef CONFIG_KVM static struct gic_kvm_info gic_v5_kvm_info __initdata; -static void __init gic_of_setup_kvm_info(struct device_node *node) +static int __init gic_setup_kvm_info(unsigned int maint_irq) { + struct gicv5_irs_chip_data *irs_data = gicv5_irs_get_chip_data(); + /* * If we don't have native GICv5 virtualisation support, then * we also don't have FEAT_GCIE_LEGACY - the architecture @@ -1131,25 +1133,120 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) */ if (!gicv5_global_data.virt_capable) { pr_info("GIC implementation is not virtualization capable\n"); - return; + return -ENODEV; } + if (WARN_ON(!irs_data)) + return -ENODEV; + gic_v5_kvm_info.type = GIC_V5; + gic_v5_kvm_info.gicv5_irs.base = irs_data->irs_base; + gic_v5_kvm_info.gicv5_irs.non_coherent = !!(irs_data->flags & IRS_FLAGS_NON_COHERENT); /* GIC Virtual CPU interface maintenance interrupt */ gic_v5_kvm_info.no_maint_irq_mask = false; - gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); - if (!gic_v5_kvm_info.maint_irq) { - pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n"); + gic_v5_kvm_info.maint_irq = maint_irq; + + /* + * We require an MI if we have legacy support, but don't, otherwise. + * Given that there's an existing flag to convey that an MI isn't + * needed, we (ab)use it to tell KVM that the MI isn't needed if we + * don't support legacy. + * + * The check for ARM64_HAS_GICV5_LEGACY explicitly doesn't use + * cpus_have_final_cap() here as we run too early. + */ + if (!cpus_have_cap(ARM64_HAS_GICV5_LEGACY) && !gic_v5_kvm_info.maint_irq) + gic_v5_kvm_info.no_maint_irq_mask = true; + + vgic_set_kvm_info(&gic_v5_kvm_info); + return 0; +} + +static void __init gic_of_setup_kvm_info(struct device_node *node) +{ + unsigned int maint_irq; + + /* GIC Virtual CPU interface maintenance interrupt */ + maint_irq = irq_of_parse_and_map(node, 0); + if (gic_setup_kvm_info(maint_irq)) + irq_dispose_mapping(maint_irq); +} + +#ifdef CONFIG_ACPI +struct gicv5_acpi_kvm_info { + u32 maint_irq; + bool valid; +}; + +static struct gicv5_acpi_kvm_info acpi_v5_kvm_info __initdata; + +static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_generic_interrupt *gicc = + (struct acpi_madt_generic_interrupt *)header; + + if (!(gicc->flags & + (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) + return 0; + + if (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) + pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n"); + + if (!acpi_v5_kvm_info.valid) { + acpi_v5_kvm_info.valid = true; + acpi_v5_kvm_info.maint_irq = gicc->vgic_interrupt; + return 0; + } + + if (acpi_v5_kvm_info.maint_irq != gicc->vgic_interrupt) + pr_warn_once(FW_BUG "MI not the same on all CPUs\n"); + + return 0; +} + +static bool __init gic_acpi_collect_virt_info(void) +{ + acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, + gic_acpi_parse_virt_madt_gicc, 0); + + return acpi_v5_kvm_info.valid; +} + +static void __init gic_acpi_setup_kvm_info(void) +{ + int irq = 0; + + if (!gic_acpi_collect_virt_info()) { + pr_warn("Unable to get hardware information used for virtualization\n"); return; } - vgic_set_kvm_info(&gic_v5_kvm_info); + if (acpi_v5_kvm_info.maint_irq) { + irq = acpi_register_gsi(NULL, acpi_v5_kvm_info.maint_irq, + ACPI_LEVEL_SENSITIVE, + ACPI_ACTIVE_HIGH); + if (irq <= 0) { + pr_warn("Failed to register GSI for GICv5 maintenance IRQ\n"); + irq = 0; + } + } + + if (gic_setup_kvm_info(irq) && irq) + acpi_unregister_gsi(acpi_v5_kvm_info.maint_irq); } +#endif // CONFIG_ACPI #else static inline void __init gic_of_setup_kvm_info(struct device_node *node) { } + +#ifdef CONFIG_ACPI +static inline void __init gic_acpi_setup_kvm_info(void) +{ +} +#endif #endif // CONFIG_KVM static int __init gicv5_init_common(struct fwnode_handle *parent_domain) @@ -1256,6 +1353,8 @@ static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsig acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id, gic_v5_get_gsi_handle); + gic_acpi_setup_kvm_info(); + return 0; out_irs: diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 821b61727d1f..1ffcd4c3f360 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -96,6 +96,7 @@ static void __init sclp_early_facilities_detect(void) sclp.has_ib = cpue->ib; sclp.has_cei = cpue->cei; sclp.has_skey = cpue->skey; + sclp.has_aef = cpue->aef; break; } diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 4db878c18f41..373ce574c431 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1822,17 +1822,27 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { /** * vfio_ap_mdev_set_kvm - sets all data for @matrix_mdev that are needed - * to manage AP resources for the guest whose state is represented by @kvm + * to manage AP resources for the guest whose state is represented by + * @kvm_file * * @matrix_mdev: a mediated matrix device - * @kvm: reference to KVM instance + * @kvm_file: the KVM VM file this vfio device is associated with * - * Return: 0 if no other mediated matrix device has a reference to @kvm; + * Return: 0 if no other mediated matrix device has a reference to the VM; * otherwise, returns an -EPERM. */ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, - struct kvm *kvm) + struct file *kvm_file) { + struct kvm *kvm; + + if (!kvm_file) + return -ENOENT; + + kvm = file_to_kvm_s390(kvm_file); + if (!kvm) + return -ENOENT; + if (kvm->arch.crypto.crycbd) { get_update_locks_for_kvm(kvm); if (kvm->arch.crypto.pqap_hook) { @@ -1841,7 +1851,6 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, } kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; - kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; vfio_ap_mdev_update_guest_apcb(matrix_mdev); release_update_locks_for_kvm(kvm); @@ -1894,7 +1903,6 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) matrix_mdev->kvm = NULL; release_update_locks_for_kvm(kvm); - kvm_put_kvm(kvm); } } diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c index 1d9515c967b0..c1c26d7a9aec 100644 --- a/drivers/vfio/device_cdev.c +++ b/drivers/vfio/device_cdev.c @@ -65,7 +65,7 @@ int vfio_device_fops_cdev_open(struct inode *inode, struct file *filep) static void vfio_df_get_kvm_safe(struct vfio_device_file *df) { spin_lock(&df->kvm_ref_lock); - vfio_device_get_kvm_safe(df->device, df->kvm); + vfio_device_get_kvm(df->device, df->kvm); spin_unlock(&df->kvm_ref_lock); } diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index b2299e5bc6df..185f359f75d3 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c @@ -163,7 +163,7 @@ static int vfio_group_ioctl_set_container(struct vfio_group *group, static void vfio_device_group_get_kvm_safe(struct vfio_device *device) { spin_lock(&device->group->kvm_ref_lock); - vfio_device_get_kvm_safe(device, device->group->kvm); + vfio_device_get_kvm(device, device->group->kvm); spin_unlock(&device->group->kvm_ref_lock); } @@ -860,11 +860,20 @@ bool vfio_group_enforced_coherent(struct vfio_group *group) return ret; } -void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm) +void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm) { + struct file *old; + + if (kvm) + get_file(kvm); + spin_lock(&group->kvm_ref_lock); + old = group->kvm; group->kvm = kvm; spin_unlock(&group->kvm_ref_lock); + + if (old) + fput(old); } /** diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h index 7728bc99b63d..2eeb526a8da5 100644 --- a/drivers/vfio/vfio.h +++ b/drivers/vfio/vfio.h @@ -23,7 +23,7 @@ struct vfio_device_file { u8 access_granted; u32 devid; /* only valid when iommufd is valid */ spinlock_t kvm_ref_lock; /* protect kvm field */ - struct kvm *kvm; + struct file *kvm; struct iommufd_ctx *iommufd; /* protected by struct vfio_device_set::lock */ }; @@ -88,7 +88,7 @@ struct vfio_group { #endif enum vfio_group_type type; struct mutex group_lock; - struct kvm *kvm; + struct file *kvm; struct file *opened_file; struct iommufd_ctx *iommufd; spinlock_t kvm_ref_lock; @@ -107,7 +107,7 @@ void vfio_device_group_unuse_iommu(struct vfio_device *device); void vfio_df_group_close(struct vfio_device_file *df); struct vfio_group *vfio_group_from_file(struct file *file); bool vfio_group_enforced_coherent(struct vfio_group *group); -void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm); +void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm); bool vfio_device_has_container(struct vfio_device *device); int __init vfio_group_init(void); void vfio_group_cleanup(void); @@ -165,7 +165,7 @@ static inline bool vfio_group_enforced_coherent(struct vfio_group *group) return true; } -static inline void vfio_group_set_kvm(struct vfio_group *group, struct kvm *kvm) +static inline void vfio_group_set_kvm(struct vfio_group *group, struct file *kvm) { } @@ -429,11 +429,11 @@ static inline void vfio_virqfd_exit(void) #endif #if IS_ENABLED(CONFIG_KVM) -void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm); +void vfio_device_get_kvm(struct vfio_device *device, struct file *kvm); void vfio_device_put_kvm(struct vfio_device *device); #else -static inline void vfio_device_get_kvm_safe(struct vfio_device *device, - struct kvm *kvm) +static inline void vfio_device_get_kvm(struct vfio_device *device, + struct file *kvm) { } diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 423ead48aafe..ea8e6cc13c45 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -472,36 +472,14 @@ void vfio_unregister_group_dev(struct vfio_device *device) EXPORT_SYMBOL_GPL(vfio_unregister_group_dev); #if IS_ENABLED(CONFIG_KVM) -void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm) +void vfio_device_get_kvm(struct vfio_device *device, struct file *kvm) { - void (*pfn)(struct kvm *kvm); - bool (*fn)(struct kvm *kvm); - bool ret; - lockdep_assert_held(&device->dev_set->lock); if (!kvm) return; - pfn = symbol_get(kvm_put_kvm); - if (WARN_ON(!pfn)) - return; - - fn = symbol_get(kvm_get_kvm_safe); - if (WARN_ON(!fn)) { - symbol_put(kvm_put_kvm); - return; - } - - ret = fn(kvm); - symbol_put(kvm_get_kvm_safe); - if (!ret) { - symbol_put(kvm_put_kvm); - return; - } - - device->put_kvm = pfn; - device->kvm = kvm; + device->kvm = get_file(kvm); } void vfio_device_put_kvm(struct vfio_device *device) @@ -511,14 +489,7 @@ void vfio_device_put_kvm(struct vfio_device *device) if (!device->kvm) return; - if (WARN_ON(!device->put_kvm)) - goto clear; - - device->put_kvm(device->kvm); - device->put_kvm = NULL; - symbol_put(kvm_put_kvm); - -clear: + fput(device->kvm); device->kvm = NULL; } #endif @@ -1544,9 +1515,13 @@ bool vfio_file_enforced_coherent(struct file *file) } EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent); -static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) +static void vfio_device_file_set_kvm(struct file *file, struct file *kvm) { struct vfio_device_file *df = file->private_data; + struct file *old; + + if (kvm) + get_file(kvm); /* * The kvm is first recorded in the vfio_device_file, and will @@ -1554,28 +1529,32 @@ static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm) * iommufd successfully in the vfio device cdev path. */ spin_lock(&df->kvm_ref_lock); + old = df->kvm; df->kvm = kvm; spin_unlock(&df->kvm_ref_lock); + + if (old) + fput(old); } /** * vfio_file_set_kvm - Link a kvm with VFIO drivers - * @file: VFIO group file or VFIO device file - * @kvm: KVM to link + * @vfio_file: VFIO group file or VFIO device file + * @kvm: KVM file to link * * When a VFIO device is first opened the KVM will be available in * device->kvm if one was associated with the file. */ -void vfio_file_set_kvm(struct file *file, struct kvm *kvm) +void vfio_file_set_kvm(struct file *vfio_file, struct file *kvm) { struct vfio_group *group; - group = vfio_group_from_file(file); + group = vfio_group_from_file(vfio_file); if (group) vfio_group_set_kvm(group, kvm); - if (vfio_device_from_file(file)) - vfio_device_file_set_kvm(file, kvm); + if (vfio_device_from_file(vfio_file)) + vfio_device_file_set_kvm(vfio_file, kvm); } EXPORT_SYMBOL_GPL(vfio_file_set_kvm); diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index bc6f2fdd7ad3..80f96ea59f1a 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -195,4 +195,19 @@ static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset) WRITE_ONCE(*ctxt->offset.vm_offset, offset); } +/* + * CVAL to program when the hardware won't apply the timer's offset, so that + * its compare against the raw counter matches the guest's at 'now'. An + * expired timer stays expired; one past the wrap never fires. + */ +static inline u64 timer_apply_offset(u64 cval, u64 offset, u64 now) +{ + u64 hw = cval + offset; + + if (now - offset >= cval) + return 0; + + return hw > now ? hw : U64_MAX; +} + #endif diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h index f86a006d6713..06c20612e9e7 100644 --- a/include/kvm/arm_psci.h +++ b/include/kvm/arm_psci.h @@ -10,6 +10,8 @@ #include #include +#include + #define KVM_ARM_PSCI_0_1 PSCI_VERSION(0, 1) #define KVM_ARM_PSCI_0_2 PSCI_VERSION(0, 2) #define KVM_ARM_PSCI_1_0 PSCI_VERSION(1, 0) diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 086b7578e5f3..7be81b936bf1 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -31,7 +31,12 @@ #define VGIC_MAX_SPI 1019 #define VGIC_MAX_RESERVED 1023 #define VGIC_MIN_LPI 8192 -#define KVM_IRQCHIP_NUM_PINS (1020 - 32) + +/* + * The common IRQ routing table must accommodate all GIC types. GICv2 and + * GICv3 need 988 pins, while GICv5 needs 1024 pins. + */ +#define KVM_IRQCHIP_NUM_PINS 1024 /* * GICv5 supports 128 PPIs, but only the first 64 are architected. We only @@ -40,6 +45,8 @@ * in KVM for now. At a future stage, this can be bumped up to 128, if required. */ #define VGIC_V5_NR_PRIVATE_IRQS 64 +#define VGIC_V5_DEFAULT_NR_SPIS 32 +#define VGIC_V5_MAX_NR_SPIS BIT(10) #define is_v5_type(t, i) (FIELD_GET(GICV5_HWIRQ_TYPE, (i)) == (t)) @@ -132,6 +139,19 @@ enum vgic_type { VGIC_V5, /* Newer, fancier GICv5 */ }; +struct vgic_v5_irs_caps { + /* Base address of the host IRS's CONFIG_FRAME */ + void __iomem *irs_base; + + /* Raw host IRS ID registers */ + u32 idr2; + u32 idr3; + u32 idr4; + + /* Is the IRS coherent with us, or not? */ + bool non_coherent; +}; + /* same for all guests, as depending only on the _host's_ GIC model */ struct vgic_global { /* type of the host GIC */ @@ -159,9 +179,16 @@ struct vgic_global { /* Maintenance IRQ number */ unsigned int maint_irq; - /* maximum number of VCPUs allowed (GICv2 limits us to 8) */ + /* + * Maximum number of VCPUs exposed before userspace has selected a + * VGIC model. Individual VGIC models can impose a lower limit + * (GICv2 limits us to 8). + */ int max_gic_vcpus; + /* Maximum number of VCPUs allowed for a GICv5 VM. */ + int max_gicv5_vcpus; + /* Only needed for the legacy KVM_CREATE_IRQCHIP */ bool can_emulate_gicv2; @@ -182,6 +209,9 @@ struct vgic_global { struct { DECLARE_BITMAP(impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); } vgic_v5_ppi_caps; + + /* GICv5 IRS capabilities */ + struct vgic_v5_irs_caps vgic_v5_irs_caps; }; extern struct vgic_global kvm_vgic_global_state; @@ -216,6 +246,12 @@ struct irq_ops { */ bool (*get_input_level)(int vintid); + /* + * Function pointer to directly update hardware pending state after the + * VGIC shadow pending state has changed. + */ + bool (*set_pending_state)(struct kvm_vcpu *vcpu, struct vgic_irq *irq); + /* * Function pointer to override the queuing of an IRQ. */ @@ -287,7 +323,8 @@ enum iodev_type { IODEV_CPUIF, IODEV_DIST, IODEV_REDIST, - IODEV_ITS + IODEV_ITS, + IODEV_GICV5_IRS }; struct vgic_io_device { @@ -345,7 +382,11 @@ struct vgic_redist_region { struct list_head list; }; +#define VGIC_V5_VM_ID_INVAL (-1) + struct vgic_v5_vm { + struct kvm *kvm; + /* * We only expose a subset of PPIs to the guest. This subset is a * combination of the PPIs that are actually implemented and what we @@ -363,6 +404,96 @@ struct vgic_v5_vm { * convenient way to do that). */ DECLARE_BITMAP(vgic_ppi_hmr, VGIC_V5_NR_PRIVATE_IRQS); + + struct fwnode_handle *fwnode; + struct irq_domain *domain; + int vpe_db_base; + u32 vm_id; + bool vmte_allocated; +}; + +/*** GICv5 ***/ +struct vgic_v5_irs { + /* base addresses in guest physical address space: */ + gpa_t vgic_v5_irs_base; + + struct vgic_io_device iodev; + struct kvm_device *dev; + + /* IRS state - used for registers etc */ + struct { + u8 domain; + u8 pa_range; + bool virt; + bool setlpi; + bool mec; + bool mpam; + bool swe; + u16 irs_id; + } idr0; + + struct { + /* PE_CNT is populated from online_vcpus at runtime */ + u8 priority_bits; + } idr1; + + struct { + u8 id_bits; + u8 min_lpi_id_bits; + bool ist_levels; + u8 ist_l2sz; + bool istmd; + u8 istmd_sz; + } idr2; + + struct { + u32 spi_range; + } idr5; + + struct { + u32 spi_irs_range; + } idr6; + + struct { + u32 spi_base; + } idr7; + + struct { + u8 sh; + u8 oc; + u8 ic; + bool ist_ra; + bool ist_wa; + bool vmt_ra; + bool vpet_ra; + bool vmd_ra; + bool vmd_wa; + bool vped_ra; + bool vped_wa; + } cr1; + + struct { + u32 id; + } spi_selr; + + struct { + u32 iaffid; + } pe_selr; + + struct { + u8 lpi_id_bits; + u8 l2sz; + u8 istsz; + bool structure; + } ist_cfgr; + + struct { + bool valid; + u64 addr; + } ist_baser; + + /* A valid userspace-restored BASER still needs its host LPI IST. */ + bool lpi_ist_restore_pending; }; struct vgic_dist { @@ -442,6 +573,11 @@ struct vgic_dist { * GICv5 per-VM data. */ struct vgic_v5_vm gicv5_vm; + + /* + * GICv5 IRS data. Dynamically allocated due to the size. + */ + struct vgic_v5_irs *vgic_v5_irs_data; }; struct vgic_v2_cpu_if { @@ -492,6 +628,9 @@ struct vgic_v5_cpu_if { */ u64 vgic_icsr; + /* The contextr used to make VPEs resident and non-resident */ + u64 vgic_contextr; + struct gicv5_vpe gicv5_vpe; }; @@ -598,10 +737,11 @@ void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu); void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1); /** - * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW + * kvm_vgic_get_max_vcpus - Get the pre-VGIC-selection VCPU limit * - * The host's GIC naturally limits the maximum amount of VCPUs a guest - * can use. + * Userspace can query KVM_CAP_MAX_VCPUS before selecting a VGIC model, so + * expose the highest model-specific limit and let kvm_vgic_create() enforce + * the selected model's actual limit. */ static inline int kvm_vgic_get_max_vcpus(void) { @@ -627,7 +767,66 @@ int vgic_v4_load(struct kvm_vcpu *vcpu); void vgic_v4_commit(struct kvm_vcpu *vcpu); int vgic_v4_put(struct kvm_vcpu *vcpu); -int vgic_v5_finalize_ppi_state(struct kvm *kvm); +static inline u8 vgic_v5_irs_ist_id_bits(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_ID_BITS, caps->idr2); +} + +static inline u8 vgic_v5_irs_min_lpi_id_bits(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, caps->idr2); +} + +static inline bool vgic_v5_irs_ist_levels(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr2 & GICV5_IRS_IDR2_IST_LEVELS); +} + +static inline u8 vgic_v5_irs_ist_l2sz(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, caps->idr2); +} + +static inline bool vgic_v5_irs_istmd(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr2 & GICV5_IRS_IDR2_ISTMD); +} + +static inline u8 vgic_v5_irs_istmd_sz(const struct vgic_v5_irs_caps *caps) +{ + return FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, caps->idr2); +} + +static inline u32 vgic_v5_irs_max_vms(const struct vgic_v5_irs_caps *caps) +{ + return BIT(FIELD_GET(GICV5_IRS_IDR3_VM_ID_BITS, caps->idr3)); +} + +static inline bool vgic_v5_irs_two_level_vmt_support(const struct vgic_v5_irs_caps *caps) +{ + return !!(caps->idr3 & GICV5_IRS_IDR3_VMT_LEVELS); +} + +static inline u16 vgic_v5_irs_vmd_size(const struct vgic_v5_irs_caps *caps) +{ + if (!(caps->idr3 & GICV5_IRS_IDR3_VMD)) + return 0; + + return BIT(FIELD_GET(GICV5_IRS_IDR3_VMD_SZ, caps->idr3)); +} + +static inline u16 vgic_v5_irs_vped_size(const struct vgic_v5_irs_caps *caps) +{ + return BIT(FIELD_GET(GICV5_IRS_IDR4_VPED_SZ, caps->idr4)); +} + +static inline u32 vgic_v5_irs_max_vpes(const struct vgic_v5_irs_caps *caps) +{ + /* Field stores VPE_ID_BITS - 1 */ + return BIT(FIELD_GET(GICV5_IRS_IDR4_VPE_ID_BITS, caps->idr4) + 1); +} + +int vgic_v5_finalize_ppi_state(struct kvm_vcpu *vcpu); bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, unsigned long flags); void vgic_v5_set_ppi_dvi(struct kvm_vcpu *vcpu, struct vgic_irq *irq, bool dvi); diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 2c2fb39f049c..299e75bc9525 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -63,20 +63,34 @@ #define GICV5_OUTER_SHARE 0b10 #define GICV5_INNER_SHARE 0b11 +#define GICV5_AIDR_COMPONENT_IRS 0b00 +#define GICV5_AIDR_COMPONENT_ITS 0b01 +#define GICV5_AIDR_COMPONENT_IWB 0b10 + +#define GICV5_AIDR_ARCH_MAJ_REV_V5 0 +#define GICV5_AIDR_ARCH_MIN_REV_V0 0 + /* * IRS registers and tables structures */ #define GICV5_IRS_IDR0 0x0000 #define GICV5_IRS_IDR1 0x0004 #define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 #define GICV5_IRS_IDR5 0x0014 #define GICV5_IRS_IDR6 0x0018 #define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 #define GICV5_IRS_CR0 0x0080 #define GICV5_IRS_CR1 0x0084 #define GICV5_IRS_SYNCR 0x00c0 #define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 #define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 #define GICV5_IRS_SPI_CFGR 0x0114 #define GICV5_IRS_SPI_STATUSR 0x0118 #define GICV5_IRS_PE_SELR 0x0140 @@ -86,11 +100,51 @@ #define GICV5_IRS_IST_CFGR 0x0190 #define GICV5_IRS_IST_STATUSR 0x0194 #define GICV5_IRS_MAP_L2_ISTR 0x01c0 +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 +#define GICV5_IRS_IDR0_IRSID GENMASK(31, 16) +#define GICV5_IRS_IDR0_SWE BIT(12) +#define GICV5_IRS_IDR0_MPAM BIT(11) +#define GICV5_IRS_IDR0_MEC BIT(10) +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_VIRT_ONE_N BIT(8) +#define GICV5_IRS_IDR0_ONE_N BIT(7) #define GICV5_IRS_IDR0_VIRT BIT(6) +#define GICV5_IRS_IDR0_PA_RANGE GENMASK(5, 2) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) + +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 +#define GICV5_IRS_IDR0_INT_DOM_NON_SECURE 0b01 +#define GICV5_IRS_IDR0_INT_DOM_EL3 0b10 +#define GICV5_IRS_IDR0_INT_DOM_REALM 0b11 #define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) #define GICV5_IRS_IDR1_IAFFID_BITS GENMASK(19, 16) +#define GICV5_IRS_IDR1_PE_CNT GENMASK(15, 0) #define GICV5_IRS_IDR1_PRIORITY_BITS_1BITS 0b000 #define GICV5_IRS_IDR1_PRIORITY_BITS_2BITS 0b001 @@ -106,13 +160,30 @@ #define GICV5_IRS_IDR2_LPI BIT(5) #define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(0), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(1), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(2), (r)) + +#define GICV5_IRS_IDR3_VMT_LEVELS BIT(10) +#define GICV5_IRS_IDR3_VM_ID_BITS GENMASK(9, 5) +#define GICV5_IRS_IDR3_VMD_SZ GENMASK(4, 1) +#define GICV5_IRS_IDR3_VMD BIT(0) + +#define GICV5_IRS_IDR4_VPE_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR4_VPED_SZ GENMASK(5, 0) + #define GICV5_IRS_IDR5_SPI_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR6_SPI_IRS_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR7_SPI_BASE GENMASK(23, 0) -#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) +#define GICV5_IRS_IIDR_PRODUCT_ID GENMASK(31, 20) +#define GICV5_IRS_IIDR_VARIANT GENMASK(19, 16) +#define GICV5_IRS_IIDR_REVISION GENMASK(15, 12) +#define GICV5_IRS_IIDR_IMPLEMENTER GENMASK(11, 0) + +#define GICV5_IRS_AIDR_COMPONENT GENMASK(11, 8) +#define GICV5_IRS_AIDR_ARCHMAJORREV GENMASK(7, 4) +#define GICV5_IRS_AIDR_ARCHMINORREV GENMASK(3, 0) #define GICV5_IRS_CR0_IDLE BIT(1) #define GICV5_IRS_CR0_IRSEN BIT(0) @@ -135,21 +206,39 @@ #define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) -#define GICV5_IRS_SPI_STATUSR_V BIT(1) -#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_VMR_VIRT BIT_ULL(63) +#define GICV5_IRS_SPI_VMR_VM_ID GENMASK_ULL(15, 0) #define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_DOMAINR_DOMAIN GENMASK(1, 0) + +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_SECURE 0b00 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE 0b01 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_EL3 0b10 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_REALM 0b11 + +#define GICV5_IRS_SPI_RESAMPLER_ID GENMASK(23, 0) + #define GICV5_IRS_SPI_CFGR_TM BIT(0) +#define GICV5_IRS_SPI_CFGR_TM_EDGE 0b0 +#define GICV5_IRS_SPI_CFGR_TM_LEVEL 0b1 + +#define GICV5_IRS_SPI_STATUSR_V BIT(1) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) + #define GICV5_IRS_PE_SELR_IAFFID GENMASK(15, 0) +#define GICV5_IRS_PE_STATUSR_ONLINE BIT(2) #define GICV5_IRS_PE_STATUSR_V BIT(1) #define GICV5_IRS_PE_STATUSR_IDLE BIT(0) #define GICV5_IRS_PE_CR0_DPS BIT(0) -#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL #define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) #define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) @@ -167,15 +256,111 @@ #define GICV5_IRS_IST_CFGR_L2SZ_16K 0b01 #define GICV5_IRS_IST_CFGR_L2SZ_64K 0b10 -#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) -#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) #define GICV5_IRS_MAP_L2_ISTR_ID GENMASK(23, 0) +#define GICV5_IRS_VMT_BASER_ADDR GENMASK_ULL(55, 3) +#define GICV5_IRS_VMT_BASER_ADDR_SHIFT 3ULL +#define GICV5_IRS_VMT_BASER_VALID BIT_ULL(0) + +#define GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL 0b1 +#define GICV5_IRS_VMT_CFGR_STRUCTURE_LINEAR 0b0 + +#define GICV5_IRS_VMT_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_VMT_CFGR_VM_ID_BITS GENMASK(4, 0) + +#define GICV5_IRS_VMT_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VPE_SELR_S BIT_ULL(63) +#define GICV5_IRS_VPE_SELR_VPE_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VPE_SELR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VPE_DBR_DBV BIT_ULL(63) +#define GICV5_IRS_VPE_DBR_REQ_DB BIT_ULL(62) +#define GICV5_IRS_VPE_DBR_DBPM GENMASK_ULL(36, 32) +#define GICV5_IRS_VPE_DBR_INTID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_HPPIR_HPPIV BIT_ULL(32) +#define GICV5_IRS_VPE_HPPIR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VPE_HPPIR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_CR0_DPS BIT(0) + +#define GICV5_IRS_VPE_STATUSR_V BIT(1) +#define GICV5_IRS_VPE_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VM_DBR_EN BIT_ULL(63) +#define GICV5_IRS_VM_DBR_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VM_SELR_VM_ID GENMASK(15, 0) + +#define GICV5_IRS_VM_STATUSR_V BIT(1) +#define GICV5_IRS_VM_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VMAP_L2_VMTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VMTR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VMR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VMR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VMR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VISTR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VISTR_TYPE GENMASK_ULL(31, 29) + +#define GICV5_IRS_VMAP_L2_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_L2_VISTR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VMAP_L2_VISTR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VMAP_VPER_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VPER_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VPER_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SAVE_VMR_VM_ID GENMASK_ULL(15, 0) +#define GICV5_IRS_SAVE_VMR_Q BIT_ULL(62) +#define GICV5_IRS_SAVE_VMR_S BIT_ULL(63) + +#define GICV5_IRS_SAVE_VM_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SAVE_VM_STATUSR_Q BIT(1) + +#define GICV5_IRS_MEC_IDR_MECIDSIZE GENMASK(3, 0) + +#define GICV5_IRS_MEC_MECID_R_MECID GENMASK(15, 0) + +#define GICV5_IRS_MPAM_IDR_HAS_MPAM_SP BIT(24) +#define GICV5_IRS_MPAM_IDR_PMG_MAX GENMASK(23, 16) +#define GICV5_IRS_MPAM_IDR_PARTID_MAX GENMASK(15, 0) + +#define GICV5_IRS_MPAM_PARTID_R_IDLE BIT(31) +#define GICV5_IRS_MPAM_PARTID_R_MPAM_SP GENMASK(25, 24) +#define GICV5_IRS_MPAM_PARTID_R_PMG GENMASK(23, 16) +#define GICV5_IRS_MPAM_PARTID_R_PARTID GENMASK(15, 0) + +#define GICV5_IRS_SWERR_STATUSR_IMP_EC GENMASK_ULL(31, 24) +#define GICV5_IRS_SWERR_STATUSR_EC GENMASK_ULL(23, 16) +#define GICV5_IRS_SWERR_STATUSR_OF BIT_ULL(3) +#define GICV5_IRS_SWERR_STATUSR_S1V BIT_ULL(2) +#define GICV5_IRS_SWERR_STATUSR_S0V BIT_ULL(1) +#define GICV5_IRS_SWERR_STATUSR_V BIT_ULL(0) + +#define GICV5_IRS_SWERR_SYNDROMER0_VIRTUAL BIT_ULL(63) +#define GICV5_IRS_SWERR_SYNDROMER0_TYPE GENMASK_ULL(62, 60) +#define GICV5_IRS_SWERR_SYNDROMER0_ID GENMASK_ULL(55, 32) +#define GICV5_IRS_SWERR_SYNDROMER0_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SWERR_SYNDROMER1_ADDR GENMASK_ULL(55, 3) + #define GICV5_ISTL1E_VALID BIT_ULL(0) +#define GICV5_IRS_ISTL1E_SIZE 8UL #define GICV5_ISTL1E_L2_ADDR_MASK GENMASK_ULL(55, 12) +#define GICV5_IRS_SETLPIR 0x0000 +#define GICV5_IRS_SETLPIR_ID GENMASK(23, 0) + /* * ITS registers and tables structures */ @@ -297,6 +482,42 @@ #define GICV5_GSI_IWB_FRAME_ID GENMASK(28, 16) #define GICV5_GSI_IWB_WIRE GENMASK(15, 0) +/* + * CoreSight identification registers - ordered by increasing offset. + */ +#define GICV5_CORESIGHT_DEVARCH 0xffbc +#define GICV5_CORESIGHT_PIDR4 0xffd0 +#define GICV5_CORESIGHT_PIDR5 0xffd4 +#define GICV5_CORESIGHT_PIDR6 0xffd8 +#define GICV5_CORESIGHT_PIDR7 0xffdc +#define GICV5_CORESIGHT_PIDR0 0xffe0 +#define GICV5_CORESIGHT_PIDR1 0xffe4 +#define GICV5_CORESIGHT_PIDR2 0xffe8 +#define GICV5_CORESIGHT_PIDR3 0xffec +#define GICV5_CORESIGHT_CIDR0 0xfff0 +#define GICV5_CORESIGHT_CIDR1 0xfff4 +#define GICV5_CORESIGHT_CIDR2 0xfff8 +#define GICV5_CORESIGHT_CIDR3 0xfffc + +#define GICV5_CORESIGHT_DEVARCH_VAL \ + (FIELD_PREP(GENMASK(31, 21), 0x23b) | \ + BIT(20) | \ + 0x5a19) + +#define GICV5_CORESIGHT_PIDR4_JEP106_CONT 0x04 +#define GICV5_CORESIGHT_PIDR5_RES0 0x00 +#define GICV5_CORESIGHT_PIDR6_RES0 0x00 +#define GICV5_CORESIGHT_PIDR7_RES0 0x00 +#define GICV5_CORESIGHT_PIDR0_PART_0 0x4b +#define GICV5_CORESIGHT_PIDR1_DES_0_PART_1 0xb0 +#define GICV5_CORESIGHT_PIDR2_DES_1 0x0b +#define GICV5_CORESIGHT_PIDR3_REVAND_CMOD 0x00 + +#define GICV5_CORESIGHT_CIDR0_VAL 0x0d +#define GICV5_CORESIGHT_CIDR1_VAL 0xf0 +#define GICV5_CORESIGHT_CIDR2_VAL 0x05 +#define GICV5_CORESIGHT_CIDR3_VAL 0xb1 + /* * Global Data structures and functions */ @@ -332,6 +553,8 @@ struct gicv5_irs_chip_data { raw_spinlock_t spi_config_lock; }; +#define IRS_FLAGS_NON_COHERENT BIT(0) + static inline int gicv5_wait_for_op_s_atomic(void __iomem *addr, u32 offset, const char *reg_s, u32 mask, u32 *val) @@ -379,6 +602,7 @@ void __init gicv5_free_lpi_domain(void); int gicv5_irs_of_probe(struct device_node *parent); int gicv5_irs_acpi_probe(void); +struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void); void gicv5_irs_remove(void); int gicv5_irs_enable(void); void gicv5_irs_its_probe(void); @@ -387,10 +611,13 @@ int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid); struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id); int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type); int gicv5_irs_iste_alloc(u32 lpi); +unsigned int gicv5_irs_l2_sz(u32 l2sz); void gicv5_irs_syncr(void); /* Embedded in kvm.arch */ struct gicv5_vpe { + int db; + bool db_fired; bool resident; }; @@ -429,4 +656,15 @@ void gicv5_deinit_lpis(void); void __init gicv5_its_of_probe(struct device_node *parent); void __init gicv5_its_acpi_probe(void); + +enum gicv5_vcpu_cmd { + VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ + VMTE_MAKE_VALID, /* Make the VMTE valid */ + VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ + VPE_MAKE_VALID, /* No corresponding invalid */ + SPI_VIST_MAKE_VALID, /* No corresponding invalid */ + LPI_VIST_MAKE_VALID, /* Triggered by a guest */ + LPI_VIST_MAKE_INVALID, /* Triggered by a guest */ +}; + #endif diff --git a/include/linux/irqchip/arm-vgic-info.h b/include/linux/irqchip/arm-vgic-info.h index 67d9d960273b..f05370e2debf 100644 --- a/include/linux/irqchip/arm-vgic-info.h +++ b/include/linux/irqchip/arm-vgic-info.h @@ -38,6 +38,11 @@ struct gic_kvm_info { bool has_v4_1; /* Deactivation impared, subpar stuff */ bool no_hw_deactivation; + /* GICv5 IRS base */ + struct { + void __iomem *base; + bool non_coherent; + } gicv5_irs; }; #ifdef CONFIG_KVM diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index c844def6dd95..e7fc94e2910a 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -45,6 +45,10 @@ #include #include +#define __KVM_DEV_STRING(name, x) __stringify(name ## _ ## x) +#define _KVM_DEV_STRING(name, x) __KVM_DEV_STRING(name, x) +#define KVM_DEV_STRING(x) _KVM_DEV_STRING(KVM_DEV_NAME, x) + #ifndef KVM_MAX_VCPU_IDS #define KVM_MAX_VCPU_IDS KVM_MAX_VCPUS #endif @@ -562,7 +566,7 @@ static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) /* * The memory barrier ensures a previous write to vcpu->requests cannot * be reordered with the read of vcpu->mode. It pairs with the general - * memory barrier following the write of vcpu->mode in VCPU RUN. + * memory barrier following the write of vcpu->mode in KVM_RUN. */ smp_mb__before_atomic(); return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); @@ -722,31 +726,14 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) } #endif -#ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static inline bool kvm_arch_has_private_mem(struct kvm *kvm) -{ - return false; -} -#endif - -#ifdef CONFIG_KVM_GUEST_MEMFD -bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); - -static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) -{ - u64 flags = GUEST_MEMFD_FLAG_MMAP; - - if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) - flags |= GUEST_MEMFD_FLAG_INIT_SHARED; - - return flags; -} -#endif - #ifndef kvm_arch_has_readonly_mem static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm) { - return IS_ENABLED(CONFIG_HAVE_KVM_READONLY_MEM); +#ifdef __KVM_HAVE_READONLY_MEM + return true; +#else + return false; +#endif } #endif @@ -784,6 +771,12 @@ struct kvm { * kvm_swap_active_memslots(). */ struct mutex slots_arch_lock; + /* + * Back-reference to the VM file for subsystems (e.g., VFIO). Holds no + * reference to avoid pinning the VM. Callers storing the file must + * take their own reference. + */ + struct file *file; struct mm_struct *mm; /* userspace tied to this vm */ unsigned long nr_memslot_pages; /* The two memslot sets - active and inactive (per address space) */ @@ -832,7 +825,7 @@ struct kvm { struct kvm_vm_stat stat; struct kvm_arch arch; refcount_t users_count; -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; spinlock_t ring_lock; struct list_head coalesced_zones; @@ -854,6 +847,8 @@ struct kvm { gfn_t mmu_invalidate_range_start; gfn_t mmu_invalidate_range_end; + unsigned long gpc_invalidate_seq; + struct list_head devices; u64 manual_dirty_log_protect; struct dentry *debugfs_dentry; @@ -871,7 +866,7 @@ struct kvm { #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER struct notifier_block pm_notifier; #endif -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* Protected by slots_lock (for writes) and RCU (for reads) */ struct xarray mem_attr_array; #endif @@ -1080,7 +1075,18 @@ void kvm_exit(void); void kvm_get_kvm(struct kvm *kvm); bool kvm_get_kvm_safe(struct kvm *kvm); void kvm_put_kvm(struct kvm *kvm); -bool file_is_kvm(struct file *file); + +/* + * Architectures define kvm_file_to_kvm_arch to + * to get a typed, arch-namespaced helper: + * + * struct kvm *file_to_kvm_(struct file *file) + */ +#ifdef kvm_file_to_kvm_arch +#define kvm_file_to_kvm_fn CONCATENATE(file_to_kvm_, kvm_file_to_kvm_arch) +struct kvm *kvm_file_to_kvm_fn(struct file *file); +#endif /* kvm_file_to_kvm_arch */ + void kvm_put_kvm_no_destroy(struct kvm *kvm); static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) @@ -2464,7 +2470,7 @@ void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd, struct kvm_kernel_irq_routing_entry *new); #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ -#ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS +#ifdef __KVM_HAVE_INVALID_WAKEUPS /* If we wakeup during the poll time, was it a sucessful poll? */ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) { @@ -2476,9 +2482,9 @@ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) { return true; } -#endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */ +#endif /* __KVM_HAVE_INVALID_WAKEUPS */ -#ifdef CONFIG_HAVE_KVM_NO_POLL +#ifdef __KVM_HAVE_NO_POLL /* Callback that tells if we must not poll */ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu); #else @@ -2486,18 +2492,18 @@ static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) { return false; } -#endif /* CONFIG_HAVE_KVM_NO_POLL */ +#endif /* __KVM_HAVE_NO_POLL */ void kvm_arch_guest_memory_reclaimed(struct kvm *kvm); -#ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE +#ifdef __KVM_HAVE_VCPU_RUN_PID_CHANGE int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu); #else static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) { return 0; } -#endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */ +#endif /* __KVM_HAVE_VCPU_RUN_PID_CHANGE */ #ifdef CONFIG_VIRT_XFER_TO_GUEST_WORK static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu) @@ -2564,48 +2570,79 @@ static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot) return slot->flags & KVM_MEMSLOT_GMEM_ONLY; } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +static inline unsigned long kvm_get_vm_memory_attributes(struct kvm *kvm, gfn_t gfn) { return xa_to_value(xa_load(&kvm->mem_attr_array, gfn)); } -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs); -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs); +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM +extern bool gmem_in_place_conversion; +typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn); +DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t); + +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { - return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; + return static_call(__kvm_is_private_gfn)(kvm, gfn); } + +#elif defined kvm_arch_has_private_mem +#define gmem_in_place_conversion true +#define kvm_is_private_gfn kvm_gmem_is_private_gfn + #else -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +#define gmem_in_place_conversion false + +static inline bool kvm_arch_has_private_mem(struct kvm *kvm) { return false; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ + +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return false; +} +#endif /* kvm_arch_has_private_mem */ #ifdef CONFIG_KVM_GUEST_MEMFD +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn); +bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); + +static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) +{ + u64 flags = GUEST_MEMFD_FLAG_MMAP; + + if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) + flags |= GUEST_MEMFD_FLAG_INIT_SHARED; + + return flags; +} + int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order); + gfn_t gfn, kvm_pfn_t *pfn, int *max_order); #else static inline int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, - kvm_pfn_t *pfn, struct page **page, - int *max_order) + kvm_pfn_t *pfn, int *max_order) { KVM_BUG_ON(1, kvm); return -EIO; } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#ifndef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() false #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE @@ -2647,6 +2684,7 @@ void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu); long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range); #endif diff --git a/include/linux/trace_remote_event.h b/include/linux/trace_remote_event.h index c8ae1e1f5e72..e4cc2d4497bc 100644 --- a/include/linux/trace_remote_event.h +++ b/include/linux/trace_remote_event.h @@ -3,6 +3,8 @@ #ifndef _LINUX_TRACE_REMOTE_EVENTS_H #define _LINUX_TRACE_REMOTE_EVENTS_H +#include + struct trace_remote; struct trace_event_fields; struct trace_seq; diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 45f08986359e..0cc91c6f96d2 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -54,7 +54,7 @@ struct vfio_device { struct list_head dev_set_list; unsigned int migration_flags; u8 precopy_info_v2; - struct kvm *kvm; + struct file *kvm; /* Members below here are private, not for driver use */ unsigned int index; @@ -66,7 +66,6 @@ struct vfio_device { unsigned int open_count; struct completion comp; struct iommufd_access *iommufd_access; - void (*put_kvm)(struct kvm *kvm); struct inode *inode; #if IS_ENABLED(CONFIG_IOMMUFD) struct iommufd_device *iommufd_device; @@ -378,7 +377,7 @@ static inline bool vfio_file_has_dev(struct file *file, struct vfio_device *devi #endif bool vfio_file_is_valid(struct file *file); bool vfio_file_enforced_coherent(struct file *file); -void vfio_file_set_kvm(struct file *file, struct kvm *kvm); +void vfio_file_set_kvm(struct file *vfio_file, struct file *kvm); #define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long)) diff --git a/include/trace/define_remote_events.h b/include/trace/define_remote_events.h index 676e803dc144..78346ac77585 100644 --- a/include/trace/define_remote_events.h +++ b/include/trace/define_remote_events.h @@ -35,17 +35,26 @@ do { \ #define RE_PRINTK(__args...) __args -#define REMOTE_EVENT(__name, __id, __struct, __printk) \ - REMOTE_EVENT_FORMAT(__name, __struct); \ +#define REMOTE_EVENT_PRINT_FUNC(__name, __printk) \ static void remote_event_print_##__name(void *evt, struct trace_seq *seq) \ { \ struct remote_event_format_##__name __maybe_unused *__entry = evt; \ trace_seq_puts(seq, #__name); \ - remote_printk(__printk); \ + __printk; \ } + +#define REMOTE_EVENT(__name, __id, __struct, __printk) \ + REMOTE_EVENT_FORMAT(__name, __struct); \ + REMOTE_EVENT_PRINT_FUNC(__name, remote_printk(__printk)) + +#define REMOTE_EVENT_CUSTOM_PRINTK(__name, __id, __struct, __printk) \ + REMOTE_EVENT_FORMAT(__name, __struct); \ + REMOTE_EVENT_PRINT_FUNC(__name, __printk) + #include REMOTE_EVENT_INCLUDE(REMOTE_EVENT_INCLUDE_FILE) #undef REMOTE_EVENT +#undef REMOTE_EVENT_CUSTOM_PRINTK #undef RE_PRINTK #undef re_field #define re_field(__type, __field) \ @@ -70,4 +79,8 @@ do { \ .print_fmt = remote_event_print_fmt_##__name, \ .print = remote_event_print_##__name, \ } + +#define REMOTE_EVENT_CUSTOM_PRINTK(__name, __id, __struct, __printk) \ + REMOTE_EVENT(__name, __id, RE_STRUCT(__struct), "\"Unknown\"") + #include REMOTE_EVENT_INCLUDE(REMOTE_EVENT_INCLUDE_FILE) diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h index b282e3a86769..07013f88e655 100644 --- a/include/trace/events/kvm.h +++ b/include/trace/events/kvm.h @@ -4,8 +4,14 @@ #include +#undef UNPACK_KVM +#define UNPACK_KVM(name) name #undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm +#define TRACE_SYSTEM UNPACK_KVM(KVM_DEV_NAME) +#undef TRACE_SYSTEM_VAR +#define TRACE_SYSTEM_VAR kvm +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE kvm #define ERSN(x) { KVM_EXIT_##x, "KVM_EXIT_" #x } @@ -358,13 +364,13 @@ TRACE_EVENT(kvm_dirty_ring_exit, TP_printk("vcpu %d", __entry->vcpu_id) ); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* * @start: Starting address of guest memory range * @end: End address of guest memory range * @attr: The value of the attribute being set. */ -TRACE_EVENT(kvm_vm_set_mem_attributes, +TRACE_EVENT(kvm_set_vm_mem_attributes, TP_PROTO(gfn_t start, gfn_t end, unsigned long attr), TP_ARGS(start, end, attr), @@ -383,7 +389,7 @@ TRACE_EVENT(kvm_vm_set_mem_attributes, TP_printk("%#016llx -- %#016llx [0x%lx]", __entry->start, __entry->end, __entry->attr) ); -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ TRACE_EVENT(kvm_unmap_hva_range, TP_PROTO(unsigned long start, unsigned long end), diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index ac2d77d14963..8dff2fc1972e 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -999,6 +999,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_HPAGE_2G 249 #define KVM_CAP_PPC_COMPAT_CAPS 250 #define KVM_CAP_ARM_PMU_V3_STRICT 251 +#define KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES 252 struct kvm_irq_routing_irqchip { __u32 irqchip; @@ -1653,6 +1654,21 @@ struct kvm_memory_attributes { __u64 flags; }; +/* Available with KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES */ +#define KVM_SET_MEMORY_ATTRIBUTES2 _IOWR(KVMIO, 0xd2, struct kvm_memory_attributes2) + +struct kvm_memory_attributes2 { + union { + __u64 address; + __u64 offset; + }; + __u64 size; + __u64 attributes; + __u64 flags; + __u64 error_offset; + __u64 reserved[11]; +}; + #define KVM_MEMORY_ATTRIBUTE_PRIVATE (1ULL << 3) #define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd) diff --git a/tools/arch/arm64/include/uapi/asm/kvm.h b/tools/arch/arm64/include/uapi/asm/kvm.h index 1c13bfa2d38a..9998252ea8ec 100644 --- a/tools/arch/arm64/include/uapi/asm/kvm.h +++ b/tools/arch/arm64/include/uapi/asm/kvm.h @@ -97,6 +97,11 @@ struct kvm_regs { #define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) #define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K) +/* Supported VGICv5 address types */ +#define KVM_VGIC_V5_ADDR_TYPE_IRS 6 + +#define KVM_VGIC_V5_IRS_SIZE (2 * SZ_64K) + #define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ #define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ #define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */ @@ -417,6 +422,16 @@ enum { #define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7 #define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8 #define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9 +#define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10 +#define KVM_DEV_ARM_VGIC_GRP_IST 11 + +struct kvm_vgic_v5_ist { + __u64 spi_ist_addr; + __u64 spi_ist_size; + __u64 lpi_ist_addr; + __u64 lpi_ist_size; +}; + #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10 #define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \ (0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT) diff --git a/tools/arch/x86/include/uapi/asm/kvm.h b/tools/arch/x86/include/uapi/asm/kvm.h index 1585ec804066..00ce265391e1 100644 --- a/tools/arch/x86/include/uapi/asm/kvm.h +++ b/tools/arch/x86/include/uapi/asm/kvm.h @@ -45,9 +45,7 @@ #define __KVM_HAVE_PIT #define __KVM_HAVE_IOAPIC #define __KVM_HAVE_IRQ_LINE -#define __KVM_HAVE_MSI #define __KVM_HAVE_USER_NMI -#define __KVM_HAVE_MSIX #define __KVM_HAVE_MCE #define __KVM_HAVE_PIT_STATE2 #define __KVM_HAVE_XEN_HVM diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index f2b223072b62..03d91f00092f 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -3,7 +3,7 @@ top_srcdir = ../../../.. include $(top_srcdir)/scripts/subarch.include ARCH ?= $(SUBARCH) -ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch)) +ifeq ($(ARCH),$(filter $(ARCH),arm64 s390 riscv x86 x86_64 loongarch powerpc)) # Top-level selftests allows ARCH=x86_64 :-( ifeq ($(ARCH),x86_64) override ARCH := x86 diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index f43549a56f0b..3cb35a6963d5 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -6,7 +6,6 @@ all: LIBKVM += lib/assert.c LIBKVM += lib/elf.c LIBKVM += lib/guest_modes.c -LIBKVM += lib/io.c LIBKVM += lib/kvm_util.c LIBKVM += lib/lru_gen_util.c LIBKVM += lib/memstress.c @@ -53,6 +52,11 @@ LIBKVM_loongarch += lib/loongarch/processor.c LIBKVM_loongarch += lib/loongarch/ucall.c LIBKVM_loongarch += lib/loongarch/exception.S +LIBKVM_powerpc += lib/powerpc/handlers.S +LIBKVM_powerpc += lib/powerpc/processor.c +LIBKVM_powerpc += lib/powerpc/ucall.c +LIBKVM_powerpc += lib/powerpc/hcall.c + # Non-compiled test targets TEST_PROGS_x86 += x86/nx_huge_pages_test.sh @@ -118,7 +122,7 @@ TEST_GEN_PROGS_x86 += x86/state_test TEST_GEN_PROGS_x86 += x86/vmx_preemption_timer_test TEST_GEN_PROGS_x86 += x86/svm_vmcall_test TEST_GEN_PROGS_x86 += x86/svm_int_ctl_test -TEST_GEN_PROGS_x86 += x86/svm_nested_clear_efer_svme +TEST_GEN_PROGS_x86 += x86/svm_nested_efer_test TEST_GEN_PROGS_x86 += x86/svm_nested_shutdown_test TEST_GEN_PROGS_x86 += x86/svm_nested_soft_inject_test TEST_GEN_PROGS_x86 += x86/svm_nested_vmcb12_gpa @@ -156,6 +160,7 @@ TEST_GEN_PROGS_x86 += x86/max_vcpuid_cap_test TEST_GEN_PROGS_x86 += x86/triple_fault_event_test TEST_GEN_PROGS_x86 += x86/recalc_apic_map_test TEST_GEN_PROGS_x86 += x86/aperfmperf_test +TEST_GEN_PROGS_x86 += x86/guest_memfd_conversions_test TEST_GEN_PROGS_x86 += access_tracking_perf_test TEST_GEN_PROGS_x86 += coalesced_io_test TEST_GEN_PROGS_x86 += dirty_log_perf_test @@ -179,6 +184,7 @@ TEST_GEN_PROGS_arm64 += arm64/debug-exceptions TEST_GEN_PROGS_arm64 += arm64/hello_el2 TEST_GEN_PROGS_arm64 += arm64/host_sve TEST_GEN_PROGS_arm64 += arm64/hypercalls +TEST_GEN_PROGS_arm64 += arm64/nv_pre_fault_memory_test TEST_GEN_PROGS_arm64 += arm64/external_aborts TEST_GEN_PROGS_arm64 += arm64/mmio_sign_ext TEST_GEN_PROGS_arm64 += arm64/page_fault_test @@ -207,6 +213,7 @@ TEST_GEN_PROGS_arm64 += guest_memfd_test TEST_GEN_PROGS_arm64 += mmu_stress_test TEST_GEN_PROGS_arm64 += rseq_test TEST_GEN_PROGS_arm64 += steal_time +TEST_GEN_PROGS_arm64 += pre_fault_memory_test TEST_GEN_PROGS_s390 = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_s390 += s390/memop @@ -224,6 +231,7 @@ TEST_GEN_PROGS_s390 += rseq_test TEST_GEN_PROGS_s390 += s390/irq_routing TEST_GEN_PROGS_s390 += mmu_stress_test TEST_GEN_PROGS_s390 += pre_fault_memory_test +TEST_GEN_PROGS_s390 += s390/irq_injection TEST_GEN_PROGS_riscv = $(TEST_GEN_PROGS_COMMON) TEST_GEN_PROGS_riscv += riscv/sbi_pmu_test @@ -254,6 +262,10 @@ TEST_GEN_PROGS_loongarch += memslot_perf_test TEST_GEN_PROGS_loongarch += set_memory_region_test TEST_GEN_PROGS_loongarch += steal_time +TEST_GEN_PROGS_powerpc = $(TEST_GEN_PROGS_COMMON) +TEST_GEN_PROGS_powerpc += access_tracking_perf_test +TEST_GEN_PROGS_powerpc += hardware_disable_test + SPLIT_TESTS += arch_timer SPLIT_TESTS += get-reg-list diff --git a/tools/testing/selftests/kvm/access_tracking_perf_test.c b/tools/testing/selftests/kvm/access_tracking_perf_test.c index 4415c94b2866..eb6939065e79 100644 --- a/tools/testing/selftests/kvm/access_tracking_perf_test.c +++ b/tools/testing/selftests/kvm/access_tracking_perf_test.c @@ -471,6 +471,16 @@ static int access_tracking_unreliable(void) return 1; } #endif + /* + * PowerPC: Nested KVM (pSeries LPAR) can hide accessed/idle, since KVM + * doesn't explicitely flush the TLB for access bit update. L0 keeps + * translations cached so the guest won't see the idle bit cleared. + */ + if (kvm_is_ppc_pseries_lpar()) { + puts("Skipping idle page count sanity check, because the test is run nested"); + return 1; + } + /* * When NUMA balancing is enabled, guest memory will be unmapped to get * NUMA faults, dropping the Accessed bits. diff --git a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c index d9c9377a6325..d46799ff91dc 100644 --- a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c +++ b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c @@ -787,6 +787,30 @@ static void test_timers_in_the_past(enum arch_timer timer) } } +/* + * A cval within long_wait_ms of the counter's wrap while the counter sits at + * wait_ms: a deadline further away than the counter can reach. Unmasked, so + * an early ISTATUS shows up as an IRQ. + */ +static void test_timers_past_counter_wrap(enum arch_timer timer) +{ + u64 cval = -msec_to_cycles(test_args.long_wait_ms); + int i; + + for (i = 0; i < ARRAY_SIZE(sleep_method); i++) { + sleep_method_t sm = sleep_method[i]; + + local_irq_disable(); + set_counter(timer, msec_to_cycles(test_args.wait_ms)); + set_cval_irq(timer, cval, CTL_ENABLE); + sm(timer, msecs_to_usecs(test_args.wait_ms) + TIMEOUT_NO_IRQ_US); + local_irq_enable(); + isb(); + assert_irqs_handled(0); + timer_set_ctl(timer, CTL_IMASK); + } +} + static void test_long_timer_delays(enum arch_timer timer) { s32 tval = (s32)msec_to_cycles(test_args.long_wait_ms); @@ -806,6 +830,7 @@ static void guest_run_iteration(enum arch_timer timer) test_basic_functionality(timer); test_timers_sanity_checks(timer); + test_timers_past_counter_wrap(timer); test_timers_above_tval_max(timer); test_timers_in_the_past(timer); diff --git a/tools/testing/selftests/kvm/arm64/debug-exceptions.c b/tools/testing/selftests/kvm/arm64/debug-exceptions.c index 7dc5f0b4f6ad..9da6dc5bcfe6 100644 --- a/tools/testing/selftests/kvm/arm64/debug-exceptions.c +++ b/tools/testing/selftests/kvm/arm64/debug-exceptions.c @@ -387,7 +387,7 @@ static void guest_code_ss(int test_cnt) * Enable Single Step execution. Note! This _must_ be a bare * ucall as the ucall() path uses atomic operations to manage * the ucall structures, and the built-in "atomics" are usually - * implemented via exclusive access instructions. The exlusive + * implemented via exclusive access instructions. The exclusive * monitor is cleared on ERET, and so taking debug exceptions * during a LDREX=>STREX sequence will prevent forward progress * and hang the guest/test. diff --git a/tools/testing/selftests/kvm/arm64/external_aborts.c b/tools/testing/selftests/kvm/arm64/external_aborts.c index d8fe17a6cc59..1a858d0d8296 100644 --- a/tools/testing/selftests/kvm/arm64/external_aborts.c +++ b/tools/testing/selftests/kvm/arm64/external_aborts.c @@ -75,6 +75,17 @@ static void vcpu_inject_serror(struct kvm_vcpu *vcpu) vcpu_events_set(vcpu, &events); } +static void vcpu_assert_serror_pending(struct kvm_vcpu *vcpu, bool pending) +{ + struct kvm_vcpu_events events; + + vcpu_events_get(vcpu, &events); + TEST_ASSERT_EQ(events.exception.serror_pending, pending); + + if (pending && vcpu_has_ras(vcpu)) + TEST_ASSERT_EQ(events.exception.serror_esr, EXPECTED_SERROR_ISS); +} + static void __vcpu_run_expect(struct kvm_vcpu *vcpu, unsigned int cmd) { struct ucall uc; @@ -134,6 +145,7 @@ static void test_mmio_abort(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -162,6 +174,7 @@ static void test_mmio_nisv(void) TEST_ASSERT(_vcpu_run(vcpu), "Expected nonzero return code from KVM_RUN"); TEST_ASSERT_EQ(errno, ENOSYS); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -185,6 +198,7 @@ static void test_mmio_nisv_abort(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -212,6 +226,7 @@ static void test_serror_masked(void) vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, true); kvm_vm_free(vm); } @@ -247,6 +262,7 @@ static void test_serror(void) vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -288,6 +304,7 @@ static void test_s1ptw_abort(void) *ptep |= bad_pa; vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -313,6 +330,7 @@ static void test_serror_emulated(void) vcpu_run_expect_sync(vcpu); vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } @@ -339,7 +357,7 @@ static void test_mmio_ease(void) pfr1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR1_EL1)); if (!SYS_FIELD_GET(ID_AA64PFR1_EL1, DF2, pfr1)) { pr_debug("Skipping %s\n", __func__); - return; + goto done; } /* @@ -356,6 +374,8 @@ static void test_mmio_ease(void) vcpu_inject_sea(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); +done: kvm_vm_free(vm); } @@ -394,6 +414,7 @@ static void test_serror_amo(void) vcpu_run_expect_sync(vcpu); vcpu_inject_serror(vcpu); vcpu_run_expect_done(vcpu); + vcpu_assert_serror_pending(vcpu, false); kvm_vm_free(vm); } diff --git a/tools/testing/selftests/kvm/arm64/hypercalls.c b/tools/testing/selftests/kvm/arm64/hypercalls.c index 5d96cdf382c4..87eade24ac5d 100644 --- a/tools/testing/selftests/kvm/arm64/hypercalls.c +++ b/tools/testing/selftests/kvm/arm64/hypercalls.c @@ -1,12 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only -/* hypercalls: Check the ARM64's psuedo-firmware bitmap register interface. +/* hypercalls: Check the ARM64's pseudo-firmware bitmap register interface. * * The test validates the basic hypercall functionalities that are exposed - * via the psuedo-firmware bitmap register. This includes the registers' + * via the pseudo-firmware bitmap register. This includes the registers' * read/write behavior before and after the VM has started, and if the * hypercalls are properly masked or unmasked to the guest when disabled or - * enabled from the KVM userspace, respectively. + * enabled from the KVM userspace, respectively. It also checks the two + * SMCCC_ARCH_FEATURES queries whose answer SMCCC pins to SUCCESS. */ #include #include @@ -97,6 +98,12 @@ static const struct test_hvc_info false_hvc_info[] = { TEST_HVC_INFO(ARM_SMCCC_HV_PV_TIME_FEATURES, ARM_SMCCC_TRNG_RND64), }; +/* SMCCC pins the answer to SUCCESS, whatever the bitmaps hold. */ +static const struct test_hvc_info always_hvc_info[] = { + TEST_HVC_INFO(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, ARM_SMCCC_VERSION_FUNC_ID), + TEST_HVC_INFO(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, ARM_SMCCC_ARCH_FEATURES_FUNC_ID), +}; + static void guest_test_hvc(const struct test_hvc_info *hc_info) { unsigned int i; @@ -128,6 +135,23 @@ static void guest_test_hvc(const struct test_hvc_info *hc_info) } } +static void guest_test_always_hvc(void) +{ + struct arm_smccc_res res; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(always_hvc_info); i++) { + memset(&res, 0, sizeof(res)); + do_smccc(always_hvc_info[i].func_id, always_hvc_info[i].arg1, + 0, 0, 0, 0, 0, 0, &res); + + __GUEST_ASSERT(res.a0 == SMCCC_RET_SUCCESS, + "a0 = 0x%lx, func_id = 0x%x, arg1 = 0x%lx, stage = %u", + res.a0, always_hvc_info[i].func_id, + always_hvc_info[i].arg1, stage); + } +} + static void guest_code(void) { while (stage != TEST_STAGE_END) { @@ -137,6 +161,7 @@ static void guest_code(void) case TEST_STAGE_HVC_IFACE_FEAT_DISABLED: case TEST_STAGE_HVC_IFACE_FEAT_ENABLED: guest_test_hvc(hvc_info); + guest_test_always_hvc(); break; case TEST_STAGE_HVC_IFACE_FALSE_INFO: guest_test_hvc(false_hvc_info); @@ -162,7 +187,7 @@ struct st_time { static void steal_time_init(struct kvm_vcpu *vcpu) { - u64 st_ipa = (ulong)ST_GPA_BASE; + u64 st_ipa = (unsigned long)ST_GPA_BASE; unsigned int gpages; gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE); diff --git a/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c b/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c new file mode 100644 index 000000000000..09c1db303856 --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/nv_pre_fault_memory_test.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * nv_pre_fault_memory_test - Test KVM_PRE_FAULT_MEMORY on a vCPU whose + * last-run context is nested. + * + * The guest enters vEL2, sets up its EL2 translation configuration into the + * real EL1 registers then ERETs to vEL1 and exits to userspace so its vCPU + * last-run context is nested backed by a shadow stage 2 MMU. + * + * Assert that pre-faulting ignores that and targets the canonical stage-2 + * page tables only. + */ +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" +#include "ucall.h" + +#include +#include + +#define TEST_MEM_SLOT 10 +#define TEST_MEM_SIZE SZ_2M +#define TEST_MEM_GPA SZ_1G + +static void guest_el1_code(void) +{ + u64 offset; + + GUEST_ASSERT_EQ(get_current_el(), 1); + + /* Exit to userspace with the vEL1 (nested) context live. */ + GUEST_SYNC(1); + + /* + * Touch the prefaulted range. vstage-2 is disabled, so the shadow + * stage-2 is a 1:1 view of the canonical IPA space. + */ + for (offset = 0; offset < TEST_MEM_SIZE; offset += SZ_4K) + READ_ONCE(*(u64 *)(TEST_MEM_GPA + offset)); + + GUEST_DONE(); +} + +static void guest_code(void) +{ + u64 sp; + + GUEST_ASSERT_EQ(get_current_el(), 2); + + /* + * Mirror the EL2 translation regime into the real EL1 registers so + * that vEL1 runs on the test's stage-1 page tables. With E2H=1, the + * _EL1 accessors read the EL2 registers, and the _EL12 accessors + * write the real EL1 registers. + */ + write_sysreg_s(read_sysreg(sctlr_el1), SYS_SCTLR_EL12); + write_sysreg_s(read_sysreg(tcr_el1), SYS_TCR_EL12); + write_sysreg_s(read_sysreg(ttbr0_el1), SYS_TTBR0_EL12); + write_sysreg_s(read_sysreg(mair_el1), SYS_MAIR_EL12); + write_sysreg_s(read_sysreg(cpacr_el1), SYS_CPACR_EL12); + + /* Run vEL1 on the same stack. */ + asm volatile("mov %0, sp" : "=r"(sp)); + write_sysreg(sp, sp_el1); + + /* + * Drop TGE so that vEL1 is a nested context rather than host EL0. + * KVM backs it with a shadow stage-2 MMU even though vstage-2 is + * disabled (HCR_EL2.VM=0). + */ + write_sysreg(read_sysreg(hcr_el2) & ~HCR_EL2_TGE, hcr_el2); + isb(); + + write_sysreg(PSR_MODE_EL1h | PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | + PSR_D_BIT, spsr_el2); + write_sysreg((u64)guest_el1_code, elr_el2); + asm volatile("eret"); + + GUEST_ASSERT(false); +} + +static void pre_fault(struct kvm_vcpu *vcpu, u64 gpa, u64 size) +{ + struct kvm_pre_fault_memory range = { + .gpa = gpa, + .size = size, + }; + int ret; + + do { + ret = __vcpu_ioctl(vcpu, KVM_PRE_FAULT_MEMORY, &range); + } while ((!ret && range.size) || + (ret < 0 && (errno == EINTR || errno == EAGAIN))); + + TEST_ASSERT(!ret, "KVM_PRE_FAULT_MEMORY failed, ret: %d errno: %d", + ret, errno); + TEST_ASSERT_EQ(range.size, 0); +} + +int main(void) +{ + struct kvm_vcpu_init init; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + u64 npages; + + TEST_REQUIRE(test_supports_el2()); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY)); + + vm = vm_create(1); + + kvm_get_default_vcpu_target(vm, &init); + init.features[0] |= BIT(KVM_ARM_VCPU_HAS_EL2); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + kvm_arch_vm_finalize_vcpus(vm); + + npages = TEST_MEM_SIZE / vm->page_size; + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, TEST_MEM_GPA, + TEST_MEM_SLOT, npages, 0); + virt_map(vm, TEST_MEM_GPA, TEST_MEM_GPA, npages); + + /* Run the guest until it has ERET'd from vEL2 to vEL1. */ + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_SYNC: + TEST_ASSERT_EQ(uc.args[1], 1); + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Unhandled ucall: %ld", uc.cmd); + } + + /* + * The vCPU's last-run context is vEL1, so its hw_mmu is a shadow + * stage-2 MMU. + * + * Pre-faulting must ignore that and populate the canonical stage-2. + */ + pre_fault(vcpu, TEST_MEM_GPA, TEST_MEM_SIZE); + + /* Resume at vEL1 and touch the prefaulted range. */ + vcpu_run(vcpu); + switch (get_ucall(vcpu, &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + default: + TEST_FAIL("Unhandled ucall: %ld", uc.cmd); + } + + kvm_vm_free(vm); + return 0; +} diff --git a/tools/testing/selftests/kvm/arm64/page_fault_test.c b/tools/testing/selftests/kvm/arm64/page_fault_test.c index 6bb3d82906b2..b0e3fc8cd76a 100644 --- a/tools/testing/selftests/kvm/arm64/page_fault_test.c +++ b/tools/testing/selftests/kvm/arm64/page_fault_test.c @@ -597,18 +597,16 @@ static void setup_memslots(struct kvm_vm *vm, struct test_params *p) vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, CODE_AND_DATA_MEMSLOT, code_npages, 0); - vm->memslots[MEM_REGION_CODE] = CODE_AND_DATA_MEMSLOT; - vm->memslots[MEM_REGION_DATA] = CODE_AND_DATA_MEMSLOT; + ____vm_override_mem_region(vm, MEM_REGION_CODE, CODE_AND_DATA_MEMSLOT); + ____vm_override_mem_region(vm, MEM_REGION_DATA, CODE_AND_DATA_MEMSLOT); - vm_userspace_mem_region_add(vm, p->src_type, data_gpa - pt_size, - PAGE_TABLE_MEMSLOT, pt_size / guest_page_size, - p->test_desc->pt_memslot_flags); - vm->memslots[MEM_REGION_PT] = PAGE_TABLE_MEMSLOT; + __vm_override_mem_region(vm, MEM_REGION_PT, p->src_type, data_gpa - pt_size, + PAGE_TABLE_MEMSLOT, pt_size / guest_page_size, + p->test_desc->pt_memslot_flags); - vm_userspace_mem_region_add(vm, p->src_type, data_gpa, TEST_DATA_MEMSLOT, - data_size / guest_page_size, - p->test_desc->data_memslot_flags); - vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT; + __vm_override_mem_region(vm, MEM_REGION_TEST_DATA, p->src_type, data_gpa, + TEST_DATA_MEMSLOT, data_size / guest_page_size, + p->test_desc->data_memslot_flags); } static void setup_ucall(struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/arm64/sea_to_user.c b/tools/testing/selftests/kvm/arm64/sea_to_user.c index e96d8982c28b..d4af6e0eed28 100644 --- a/tools/testing/selftests/kvm/arm64/sea_to_user.c +++ b/tools/testing/selftests/kvm/arm64/sea_to_user.c @@ -81,25 +81,46 @@ static u64 translate_hva_to_hpa(unsigned long hva) static void write_einj_entry(const char *einj_path, u64 val) { - char cmd[256] = {0}; - FILE *cmdfile = NULL; + char buf[32]; + int fd, len, ret; - sprintf(cmd, "echo %#lx > %s", val, einj_path); - cmdfile = popen(cmd, "r"); + fd = open(einj_path, O_WRONLY); + if (fd < 0) + ksft_exit_fail_perror(einj_path); - if (pclose(cmdfile) == 0) - ksft_print_msg("echo %#lx > %s - done\n", val, einj_path); - else - ksft_exit_fail_perror("Failed to write EINJ entry"); + len = snprintf(buf, sizeof(buf), "%#lx\n", val); + ret = write(fd, buf, len); + if (ret != len) + ksft_exit_fail_perror(einj_path); + + close(fd); + ksft_print_msg("%#lx > %s - done\n", val, einj_path); } static void inject_uer(u64 hpa) { - if (access("/sys/firmware/acpi/tables/EINJ", R_OK) == -1) - ksft_test_result_skip("EINJ table no available in firmware"); + int fd; - if (access(EINJ_ETYPE, R_OK | W_OK) == -1) - ksft_test_result_skip("EINJ module probably not loaded?"); + /* + * EINJ is exposed only through debugfs, and opening it tells us why it + * is unusable far more reliably than access()-ing the ACPI EINJ table: + * reading that table needs CAP_SYS_ADMIN, so access() gives a false + * negative for unprivileged runs. Open the injection interface and let + * errno say whether the test cannot run (skip) or genuinely failed. + */ + fd = open(EINJ_ETYPE, O_WRONLY); + if (fd < 0) { + switch (errno) { + case ENOENT: + ksft_exit_skip("need CONFIG_ACPI_APEI_EINJ and firmware EINJ support\n"); + case EACCES: + case EPERM: + ksft_exit_skip("EINJ requires running as root\n"); + default: + ksft_exit_fail_perror(EINJ_ETYPE); + } + } + close(fd); write_einj_entry(EINJ_ETYPE, ERROR_TYPE_MEMORY_UER); write_einj_entry(EINJ_FLAGS, MASK_MEMORY_UER); @@ -194,13 +215,33 @@ static void run_vm(struct kvm_vm *vm, struct kvm_vcpu *vcpu) ksft_print_msg("Dump kvm_run info about KVM_EXIT_%s\n", exit_reason_str(run->exit_reason)); + + /* + * The guest's read of the injected location is expected to trap to KVM + * as an SEA. If it does not, the injected error was never placed as + * consumable poison: some firmware honours EINJ's notrigger request by + * arming the poison only as part of the (now skipped) trigger step, so + * nothing is left in memory for the guest to consume. The guest then + * reads back the sentinel value and reports it via GUEST_FAIL, which + * arm64 delivers as a ucall over MMIO (hence a KVM_EXIT_MMIO here). + * Treat that as "this platform cannot host the test" and skip, matching + * the requirement documented at the top of this file, rather than + * failing on a hardware/firmware limitation the test cannot control. + */ + if (run->exit_reason != KVM_EXIT_ARM_SEA && + get_ucall(vcpu, &uc) == UCALL_ABORT) { + ksft_print_msg("Guest consumed no SEA: %s", uc.buffer); + ksft_exit_skip("EINJ notrigger placed no consumable poison on this platform\n"); + } + + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_SEA); + + /* arm_sea holds valid data only for a KVM_EXIT_ARM_SEA exit. */ ksft_print_msg("kvm_run.arm_sea: esr=%#llx, flags=%#llx\n", run->arm_sea.esr, run->arm_sea.flags); ksft_print_msg("kvm_run.arm_sea: gva=%#llx, gpa=%#llx\n", run->arm_sea.gva, run->arm_sea.gpa); - TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_SEA); - esr = run->arm_sea.esr; TEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_LOW); TEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT); @@ -260,6 +301,12 @@ static struct kvm_vm *vm_create_with_sea_handler(struct kvm_vcpu **vcpu) alignment = max(backing_page_size, guest_page_size); num_guest_pages = VM_MEM_SIZE / guest_page_size; + /* + * The region is backed by 1GB hugepages; skip gracefully rather than + * failing with mmap() -ENOMEM if the host has none reserved. + */ + TEST_REQUIRE(get_free_hugepages(backing_page_size) >= VM_MEM_SIZE); + vm = __vm_create_with_one_vcpu(vcpu, num_guest_pages, guest_code); vm_init_descriptor_tables(vm); vcpu_init_descriptor_tables(*vcpu); diff --git a/tools/testing/selftests/kvm/arm64/set_id_regs.c b/tools/testing/selftests/kvm/arm64/set_id_regs.c index 7429a1055df5..10849d21c0dd 100644 --- a/tools/testing/selftests/kvm/arm64/set_id_regs.c +++ b/tools/testing/selftests/kvm/arm64/set_id_regs.c @@ -13,6 +13,7 @@ #include "kvm_util.h" #include "processor.h" #include "test_util.h" +#include "vgic.h" #include enum ftr_type { @@ -803,6 +804,107 @@ static void test_reset_preserves_id_regs(struct kvm_vcpu *vcpu) ksft_test_result_pass("%s\n", __func__); } +/* + * ID registers must stay immutable even when a vCPU's first KVM_RUN fails + * after finalization but before KVM_ARCH_FLAG_HAS_RAN_ONCE is set. + */ +static void test_idreg_frozen_after_failed_run(void) +{ + static const u32 imp_id_regs[] = { + SYS_MIDR_EL1, + SYS_REVIDR_EL1, + SYS_AIDR_EL1, + }; + struct kvm_vcpu_init init; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + int r; + + if (!kvm_has_cap(KVM_CAP_ARM_PMU_V3)) { + ksft_print_msg("PMUv3 unsupported, cannot fail the first run\n"); + ksft_test_result_skip("%s\n", __func__); + return; + } + + /* Skip the default vGIC so the KVM_CREATE_DEVICE gate is reachable. */ + test_disable_default_vgic(); + + vm = vm_create(1); + vm_enable_cap(vm, KVM_CAP_ARM_WRITABLE_IMP_ID_REGS, 0); + kvm_get_default_vcpu_target(vm, &init); + init.features[0] |= (1 << KVM_ARM_VCPU_PMU_V3); + vcpu = aarch64_vcpu_add(vm, 0, &init, guest_code); + kvm_arch_vm_finalize_vcpus(vm); + + /* + * A PMUv3 vCPU left without PMU init is rejected by + * kvm_arm_pmu_v3_enable(), which runs after sysreg finalization. + */ + r = _vcpu_run(vcpu); + TEST_ASSERT(r < 0 && errno == EINVAL, + "first KVM_RUN should fail post-finalization: r=%d errno=%d", + r, errno); + + /* + * Feature ID registers: use values that would have been accepted before + * finalization, so that a rejection means the registers are final + * rather than the value being invalid. + */ + for (int i = 0; i < ARRAY_SIZE(test_regs); i++) { + const struct reg_ftr_bits *ftr_bits = test_regs[i].ftr_bits; + u64 reg = KVM_ARM64_SYS_REG(test_regs[i].reg); + u64 val = vcpu_get_reg(vcpu, reg); + + for (int j = 0; ftr_bits[j].type != FTR_END; j++) { + u64 ftr = (val & ftr_bits[j].mask) >> ftr_bits[j].shift; + u64 safe = get_safe_value(&ftr_bits[j], ftr); + u64 new_val; + + if (safe == ftr) + continue; + + new_val = (val & ~ftr_bits[j].mask) | + (safe << ftr_bits[j].shift); + + r = __vcpu_set_reg(vcpu, reg, new_val); + TEST_ASSERT(r < 0 && errno == EBUSY, + "%s write after failed first run: r=%d errno=%d", + ftr_bits[j].name, r, errno); + TEST_ASSERT_EQ(vcpu_get_reg(vcpu, reg), val); + } + + /* A write matching the finalized value is still accepted. */ + vcpu_set_reg(vcpu, reg, val); + } + + /* + * The VM-wide implementation ID registers are gated separately. Bit 0 + * is within the writable mask of all three, so flipping it is a change + * KVM would otherwise accept. + */ + for (int i = 0; i < ARRAY_SIZE(imp_id_regs); i++) { + u64 reg = KVM_ARM64_SYS_REG(imp_id_regs[i]); + u64 val = vcpu_get_reg(vcpu, reg); + + r = __vcpu_set_reg(vcpu, reg, val ^ 1); + TEST_ASSERT(r < 0 && errno == EBUSY, + "implementation ID reg write after failed first run: r=%d errno=%d", + r, errno); + TEST_ASSERT_EQ(vcpu_get_reg(vcpu, reg), val); + } + + /* Creating an in-kernel irqchip would change the ID registers too. */ + if (kvm_supports_vgic_v3()) { + r = __kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3); + TEST_ASSERT(r < 0 && errno == EBUSY, + "vGIC creation after failed first run: r=%d errno=%d", + r, errno); + } + + kvm_vm_free(vm); + ksft_test_result_pass("%s\n", __func__); +} + int main(void) { struct kvm_vcpu *vcpu; @@ -828,7 +930,7 @@ int main(void) ksft_print_header(); - test_cnt = 3 + MPAM_IDREG_TEST + MTE_IDREG_TEST; + test_cnt = 4 + MPAM_IDREG_TEST + MTE_IDREG_TEST; for (i = 0; i < ARRAY_SIZE(test_regs); i++) for (j = 0; test_regs[i].ftr_bits[j].type != FTR_END; j++) test_cnt++; @@ -847,5 +949,7 @@ int main(void) kvm_vm_free(vm); + test_idreg_frozen_after_failed_run(); + ksft_finished(); } diff --git a/tools/testing/selftests/kvm/arm64/vgic_init.c b/tools/testing/selftests/kvm/arm64/vgic_init.c index 47e34b43afb2..5a30f3cb039b 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_init.c +++ b/tools/testing/selftests/kvm/arm64/vgic_init.c @@ -5,6 +5,7 @@ * Copyright (C) 2020, Red Hat, Inc. */ #include +#include #include #include #include @@ -13,12 +14,21 @@ #include "test_util.h" #include "kvm_util.h" +#include "gic.h" #include "processor.h" #include "vgic.h" #include "gic_v3.h" #define NR_VCPUS 4 +#define REDIST_RETRY_REGION0_BASE GICR_BASE_GPA +#define REDIST_RETRY_REGION1_BASE \ + (REDIST_RETRY_REGION0_BASE + 2 * KVM_VGIC_V3_REDIST_SIZE) +#define REDIST_RETRY_DIST_BASE \ + (REDIST_RETRY_REGION1_BASE + KVM_VGIC_V3_REDIST_SIZE) +#define REDIST_RETRY_REGION2_BASE \ + (REDIST_RETRY_DIST_BASE + KVM_VGIC_V3_DIST_SIZE) + #define REG_OFFSET(vcpu, offset) (((u64)vcpu << 32) | offset) #define VGIC_DEV_IS_V2(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V2) @@ -65,6 +75,23 @@ static void guest_code(void) GUEST_DONE(); } +static void guest_check_redist_retry(void) +{ + unsigned int i; + + /* The first three redistributors span adjacent regions 0 and 1. */ + for (i = 0; i < NR_VCPUS; i++) { + u64 base = i < 3 ? REDIST_RETRY_REGION0_BASE + + i * KVM_VGIC_V3_REDIST_SIZE : + REDIST_RETRY_REGION2_BASE; + u64 typer = readq((void *)(unsigned long)(base + GICR_TYPER)); + + GUEST_ASSERT_EQ(GICR_TYPER_CPU_NUMBER(typer), i); + } + + GUEST_DONE(); +} + /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) { @@ -73,6 +100,7 @@ static int run_vcpu(struct kvm_vcpu *vcpu) static struct vm_gic vm_gic_create_with_vcpus(u32 gic_dev_type, u32 nr_vcpus, + void *guest_code, struct kvm_vcpu *vcpus[]) { struct vm_gic v; @@ -338,7 +366,7 @@ static void test_vgic_then_vcpus(u32 gic_dev_type) struct vm_gic v; int ret, i; - v = vm_gic_create_with_vcpus(gic_dev_type, 1, vcpus); + v = vm_gic_create_with_vcpus(gic_dev_type, 1, guest_code, vcpus); subtest_dist_rdist(&v); @@ -359,7 +387,8 @@ static void test_vcpus_then_vgic(u32 gic_dev_type) struct vm_gic v; int ret; - v = vm_gic_create_with_vcpus(gic_dev_type, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(gic_dev_type, NR_VCPUS, guest_code, + vcpus); subtest_dist_rdist(&v); @@ -411,7 +440,8 @@ static void test_v3_new_redist_regions(void) u64 addr; int ret; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); @@ -422,7 +452,8 @@ static void test_v3_new_redist_regions(void) /* step2 */ - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); addr = REDIST_REGION_ATTR_ADDR(1, 0x280000, 0, 2); @@ -436,7 +467,8 @@ static void test_v3_new_redist_regions(void) /* step 3 */ - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); subtest_v3_redist_regions(&v); ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, @@ -457,6 +489,70 @@ static void test_v3_new_redist_regions(void) vm_gic_destroy(&v); } +static void test_v3_redist_region_retry(void) +{ + struct kvm_vcpu *vcpus[NR_VCPUS]; + struct vm_gic v; + struct ucall uc; + u64 addr; + int ret; + + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_check_redist_retry, vcpus); + + addr = REDIST_REGION_ATTR_ADDR(2, REDIST_RETRY_REGION0_BASE, 0, 0); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_REGION1_BASE, 0, 1); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + addr = REDIST_RETRY_DIST_BASE; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_DIST, &addr); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_DIST_BASE, 0, 2); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, + &addr); + TEST_ASSERT(ret && errno == EINVAL, + "register redist region colliding with dist"); + + addr = REDIST_REGION_ATTR_ADDR(1, REDIST_RETRY_REGION2_BASE, 0, 2); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V3_ADDR_TYPE_REDIST_REGION, &addr); + + virt_map(v.vm, REDIST_RETRY_REGION0_BASE, REDIST_RETRY_REGION0_BASE, + vm_calc_num_guest_pages(v.vm->mode, + 3 * KVM_VGIC_V3_REDIST_SIZE)); + virt_map(v.vm, REDIST_RETRY_REGION2_BASE, REDIST_RETRY_REGION2_BASE, + vm_calc_num_guest_pages(v.vm->mode, + KVM_VGIC_V3_REDIST_SIZE)); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + vcpu_run(vcpus[0]); + switch (get_ucall(vcpus[0], &uc)) { + case UCALL_DONE: + break; + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_NONE: + if (vcpus[0]->run->exit_reason == KVM_EXIT_MMIO) + TEST_FAIL("Unexpected MMIO exit at 0x%llx", + vcpus[0]->run->mmio.phys_addr); + fallthrough; + default: + TEST_FAIL("Unexpected ucall %lu, exit_reason %u", + uc.cmd, vcpus[0]->run->exit_reason); + } + + vm_gic_destroy(&v); +} + static void test_v3_typer_accesses(void) { struct vm_gic v; @@ -608,7 +704,8 @@ static void test_v3_redist_ipa_range_check_at_vcpu_run(void) int ret, i; u64 addr; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, 1, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, 1, guest_code, + vcpus); /* Set space for 3 redists, we have 1 vcpu, so this succeeds. */ addr = max_phys_size - (3 * 2 * 0x10000); @@ -641,7 +738,8 @@ static void test_v3_its_region(void) u64 addr; int its_fd, ret; - v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + v = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); its_fd = kvm_create_device(v.vm, KVM_DEV_TYPE_ARM_VGIC_ITS); addr = 0x401000; @@ -684,7 +782,8 @@ static void test_v3_nassgicap(void) u32 typer2; int ret; - vm = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus); + vm = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, + guest_code, vcpus); kvm_device_attr_get(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS, GICD_TYPER2, &typer2); has_nassgicap = typer2 & GICD_TYPER2_nASSGIcap; @@ -978,6 +1077,7 @@ void run_tests(u32 gic_dev_type) if (VGIC_DEV_IS_V3(gic_dev_type)) { test_v3_new_redist_regions(); + test_v3_redist_region_retry(); test_v3_typer_accesses(); test_v3_last_bit_redist_regions(); test_v3_last_bit_single_rdist(); diff --git a/tools/testing/selftests/kvm/arm64/vgic_its_save.c b/tools/testing/selftests/kvm/arm64/vgic_its_save.c index 864da01539f3..fae8a6c45f88 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_its_save.c +++ b/tools/testing/selftests/kvm/arm64/vgic_its_save.c @@ -279,15 +279,15 @@ static void setup_memslot(size_t sz) size_t pages = sz / vm->page_size; gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base, - TEST_MEMSLOT_INDEX, pages, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + gpa_base, TEST_MEMSLOT_INDEX, pages); } static gpa_t alloc_64k(size_t nr) { size_t pages_per_64k = vm_calc_num_guest_pages(vm->mode, SZ_64K); - gpa_t gpa = vm_phy_pages_alloc(vm, nr * pages_per_64k, gpa_base, - TEST_MEMSLOT_INDEX); + gpa_t gpa = vm_phy_pages_alloc(vm, nr * pages_per_64k, + MEM_REGION_TEST_EXTRA); TEST_ASSERT(IS_ALIGNED(gpa, SZ_64K), "Allocation at 0x%lx is not 64K aligned, GITS_BASER cannot address it", diff --git a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c index 9a69c439a0f8..45a87ba5a737 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c +++ b/tools/testing/selftests/kvm/arm64/vgic_lpi_stress.c @@ -23,7 +23,6 @@ #define GIC_LPI_OFFSET 8192 static size_t nr_iterations = 1000; -static gpa_t gpa_base; static struct kvm_vm *vm; static struct kvm_vcpu **vcpus; @@ -143,6 +142,7 @@ static void guest_code(size_t nr_lpis) static void setup_memslot(void) { + gpa_t gpa_base; size_t pages; size_t sz; @@ -164,8 +164,8 @@ static void setup_memslot(void) pages = sz / vm->page_size; gpa_base = ((vm_compute_max_gfn(vm) + 1) * vm->page_size) - sz; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa_base, - TEST_MEMSLOT_INDEX, pages, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + gpa_base, TEST_MEMSLOT_INDEX, pages); } #define LPI_PROP_DEFAULT_PRIO 0xa0 @@ -191,28 +191,25 @@ static void setup_test_data(void) gpa_t cmdq_base; test_data.device_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, - TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); test_data.collection_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, - TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); - cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, gpa_base, - TEST_MEMSLOT_INDEX); + cmdq_base = vm_phy_pages_alloc(vm, pages_per_64k, MEM_REGION_TEST_EXTRA); virt_map(vm, cmdq_base, cmdq_base, pages_per_64k); test_data.cmdq_base = cmdq_base; test_data.cmdq_base_va = (void *)cmdq_base; test_data.itt_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_devices, - gpa_base, TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); test_data.lpi_prop_table = vm_phy_pages_alloc(vm, pages_per_64k, - gpa_base, TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); configure_lpis(); test_data.lpi_pend_tables = vm_phy_pages_alloc(vm, pages_per_64k * nr_cpus, - gpa_base, TEST_MEMSLOT_INDEX); + MEM_REGION_TEST_EXTRA); sync_global_to_guest(vm, test_data); } diff --git a/tools/testing/selftests/kvm/arm64/vgic_v5.c b/tools/testing/selftests/kvm/arm64/vgic_v5.c index 96cfd6bb32f6..d5df742b2300 100644 --- a/tools/testing/selftests/kvm/arm64/vgic_v5.c +++ b/tools/testing/selftests/kvm/arm64/vgic_v5.c @@ -13,6 +13,32 @@ #include "vgic.h" #define NR_VCPUS 1 +#define VGIC_V5_LPI_NR_VCPUS 2 +#define VGIC_V5_SPARSE_NR_VCPUS 2 +#define VGIC_V5_SPARSE_VCPU0_ID 17 +#define VGIC_V5_SPARSE_VCPU1_ID 3 +#define VGIC_V5_LPI_MEMSLOT 1 +#define VGIC_V5_NR_PRIVATE_IRQS 64 +#define VGIC_V5_DEFAULT_NR_SPIS 32 +#define VGIC_V5_MAX_NR_SPIS BIT(10) +#define VGIC_V5_IRS_SIZE 0x20000 +#define VGIC_V5_IRS_WAIT_RETRIES 1000000 +#define VGIC_V5_LPI_IST_BASE_GPA (GICV5_IRS_CONFIG_BASE_GPA + VGIC_V5_IRS_SIZE) +#define VGIC_V5_LPI_IST_SIZE 0x10000 +#define LPI_TEST_TO_VPE1 0 +#define LPI_TEST_TO_VPE0 1 +#define RESTORE_TEST_LPI_PENDING 0 +#define RESTORE_TEST_LPI_ENABLED 1 +#define RESTORE_TEST_SPI_PRIO_HIGH 0 +#define RESTORE_TEST_SPI_PRIO_LOW 1 +#define RESTORE_TEST_SPI_PENDING_DISABLED 2 +#define RESTORE_TEST_SPI_ENABLED 3 +#define VGIC_V5_IST_PRIO_HIGH 0x08 +#define VGIC_V5_IST_PRIO_LOW 0x10 + +static u64 max_phys_size; + +#define VGIC_V5_TEST_IST_BASER_GPA 0x10000000ULL struct vm_gic { struct kvm_vm *vm; @@ -24,8 +50,200 @@ struct vm_gic { #define GUEST_CMD_IRQ_DIEOI 11 #define GUEST_CMD_IS_AWAKE 12 #define GUEST_CMD_IS_READY 13 +#define GUEST_CMD_LPI_SENT 14 +#define GUEST_CMD_LPI_REPLIED 15 +#define GUEST_CMD_IST_SOURCE_READY 16 +#define GUEST_CMD_IST_SPI_INJECT_READY 17 +#define GUEST_CMD_IST_LPI_INJECT_READY 18 -static void guest_irq_handler(struct ex_regs *regs) +static struct kvm_vgic_v5_ist vgic_v5_ist_attr(void *spi_ist, size_t spi_size, + void *lpi_ist, size_t lpi_size) +{ + return (struct kvm_vgic_v5_ist) { + .spi_ist_addr = (uintptr_t)spi_ist, + .spi_ist_size = spi_size, + .lpi_ist_addr = (uintptr_t)lpi_ist, + .lpi_ist_size = lpi_size, + }; +} + +static u32 spi_line_expected; +static bool spi_line_level_sensitive; +static bool lpi_ist_ready; +static u32 ist_restore_irq_count[VGIC_V5_LPI_NR_VCPUS]; + +static u64 gicv5_hwirq(u32 type, u32 intid) +{ + return FIELD_PREP(GICV5_HWIRQ_TYPE, type) | + FIELD_PREP(GICV5_HWIRQ_ID, intid); +} + +static u64 gicv5_lpi_hwirq(u32 lpi) +{ + return gicv5_hwirq(GICV5_HWIRQ_TYPE_LPI, lpi); +} + +static u64 gicv5_spi_hwirq(u32 spi) +{ + return gicv5_hwirq(GICV5_HWIRQ_TYPE_SPI, spi); +} + +static void gicv5_setup_and_enable_hwirq(u64 hwirq, u32 target_vpe) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, + GICV5_IRQ_DEFAULT_PRI); + gic_insn(val, CDPRI); + + val = hwirq | FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, target_vpe); + gic_insn(val, CDAFF); + + gic_insn(hwirq, CDEN); +} + +static void gicv5_set_hwirq_priority(u64 hwirq, u32 priority) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority); + gic_insn(val, CDPRI); +} + +static void gicv5_set_hwirq_affinity(u64 hwirq, u32 target_vpe) +{ + u64 val; + + val = hwirq | FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, target_vpe); + gic_insn(val, CDAFF); +} + +static void gicv5_enable_hwirq(u64 hwirq) +{ + gic_insn(hwirq, CDEN); +} + +static void gicv5_enable_spi(u32 spi, u32 target_vpe) +{ + gicv5_setup_and_enable_hwirq(gicv5_spi_hwirq(spi), target_vpe); +} + +static void gicv5_enable_lpi(u32 lpi, u32 target_vpe) +{ + gicv5_setup_and_enable_hwirq(gicv5_lpi_hwirq(lpi), target_vpe); +} + +static void gicv5_send_lpi(u32 lpi) +{ + u64 hwirq = gicv5_lpi_hwirq(lpi); + + gic_insn(hwirq | GICV5_GIC_CDPEND_PENDING_MASK, CDPEND); +} + +static void gicv5_wait_for_irs_idle(u32 reg, u32 idle) +{ + int i; + + for (i = 0; i < VGIC_V5_IRS_WAIT_RETRIES; i++) { + if (readl(GICV5_IRS_CONFIG_BASE_GVA + reg) & idle) + return; + + cpu_relax(); + } + + GUEST_FAIL("IRS operation did not become idle"); +} + +static void gicv5_configure_spi(u32 spi, bool level) +{ + u32 val; + + val = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, spi); + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_SPI_SELR); + gicv5_wait_for_irs_idle(GICV5_IRS_SPI_STATUSR, + GICV5_IRS_SPI_STATUSR_IDLE); + + val = level ? GICV5_IRS_SPI_CFGR_TM : 0; + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_SPI_CFGR); + gicv5_wait_for_irs_idle(GICV5_IRS_SPI_STATUSR, + GICV5_IRS_SPI_STATUSR_IDLE); +} + +static u32 gicv5_lpi_istsz(u32 idr2, u32 lpi_id_bits) +{ + if (!(idr2 & GICV5_IRS_IDR2_ISTMD)) + return GICV5_IRS_IST_CFGR_ISTSZ_4; + + if (lpi_id_bits >= FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2)) + return GICV5_IRS_IST_CFGR_ISTSZ_16; + + return GICV5_IRS_IST_CFGR_ISTSZ_8; +} + +static void gicv5_configure_lpi_ist(void) +{ + u32 idr2, min_lpi_id_bits, lpi_id_bits, id_bits, istsz; + u64 val; + + idr2 = readl(GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IDR2); + GUEST_ASSERT(idr2 & GICV5_IRS_IDR2_LPI); + + min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2); + id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2); + /* Default to 32 LPIs, unless the min requirement is higher */ + lpi_id_bits = max(5U, min_lpi_id_bits); + GUEST_ASSERT(lpi_id_bits <= id_bits); + + istsz = gicv5_lpi_istsz(idr2, lpi_id_bits); + /* + * We allocate a fixed-size IST buffer on the host to avoid dynamic + * allocation from the guest. Make sure it fits the linear IST described + * by IDR2 before programming IRS_IST_BASER. + */ + GUEST_ASSERT((BIT_ULL(lpi_id_bits) << (istsz + 2)) <= VGIC_V5_LPI_IST_SIZE); + + val = FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits) | + FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz); + writel(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_CFGR); + + val = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + VGIC_V5_LPI_IST_BASE_GPA >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + val |= GICV5_IRS_IST_BASER_VALID; + writeq(val, GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_BASER); + + GUEST_ASSERT_EQ(readl(GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_STATUSR), + GICV5_IRS_IST_STATUSR_IDLE); + + /* The address is immutable while the BASER is valid. */ + writeq(val ^ FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1), + GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_IST_BASER); + GUEST_ASSERT_EQ(readq(GICV5_IRS_CONFIG_BASE_GVA + + GICV5_IRS_IST_BASER), val); +} + +static void gicv5_enable_irs(void) +{ + writel(GICV5_IRS_CR0_IRSEN, + GICV5_IRS_CONFIG_BASE_GVA + GICV5_IRS_CR0); + gicv5_wait_for_irs_idle(GICV5_IRS_CR0, GICV5_IRS_CR0_IDLE); +} + +static void gicv5_configure_test_lpis(void) +{ + gicv5_enable_lpi(LPI_TEST_TO_VPE1, 1); + gicv5_enable_lpi(LPI_TEST_TO_VPE0, 0); +} + +static void gicv5_set_pcr(u32 priority) +{ + u64 pcr; + + pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, priority); + write_sysreg_s(pcr, SYS_ICC_PCR_EL1); + isb(); +} + +static void guest_ppi_irq_handler(struct ex_regs *regs) { bool valid; u32 hwirq; @@ -78,9 +296,284 @@ static void guest_code(void) GUEST_SYNC(GUEST_CMD_IS_READY); /* Loop forever waiting for interrupts */ - while (1); + for (;;) + cpu_relax(); } +static void guest_iaffid_code(void) +{ + GUEST_SYNC(FIELD_GET(ICC_IAFFIDR_EL1_IAFFID, + read_sysreg_s(SYS_ICC_IAFFIDR_EL1))); + GUEST_DONE(); +} + +static void guest_spi_irq_handler(struct ex_regs *regs) +{ + bool valid; + u32 hwirq; + u64 ia; + + ia = gicr_insn(CDIA); + valid = GICV5_GICR_CDIA_VALID(ia); + + if (!valid) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + + GUEST_ASSERT_EQ(hwirq, gicv5_spi_hwirq(READ_ONCE(spi_line_expected))); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + + GUEST_DONE(); +} + +static void guest_spi_line_code(void) +{ + local_irq_disable(); + + gicv5_enable_irs(); + gicv5_cpu_enable_interrupts(); + gicv5_configure_spi(READ_ONCE(spi_line_expected), + READ_ONCE(spi_line_level_sensitive)); + gicv5_enable_spi(READ_ONCE(spi_line_expected), 0); + + local_irq_enable(); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + while (1) + wfi(); +} + +static void guest_lpi_irq_handler(struct ex_regs *regs) +{ + u32 vcpu_id = guest_get_vcpuid(); + u32 expected_lpi = vcpu_id ? LPI_TEST_TO_VPE1 : LPI_TEST_TO_VPE0; + u64 expected_hwirq = gicv5_lpi_hwirq(expected_lpi); + u32 hwirq; + u64 ia; + + ia = gicr_insn(CDIA); + if (!GICV5_GICR_CDIA_VALID(ia)) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + GUEST_ASSERT_EQ(hwirq, expected_hwirq); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + + if (vcpu_id) { + gicv5_send_lpi(LPI_TEST_TO_VPE0); + GUEST_SYNC(GUEST_CMD_LPI_REPLIED); + while (1) + wfi(); + } + + GUEST_DONE(); +} + +static void guest_unexpected_irq_handler(struct ex_regs *regs) +{ + GUEST_FAIL("Unexpected IRQ"); +} + +static void guest_lpi_code(void) +{ + u32 vcpu_id = guest_get_vcpuid(); + + local_irq_disable(); + + if (!vcpu_id) { + gicv5_enable_irs(); + gicv5_configure_lpi_ist(); + gicv5_configure_test_lpis(); + WRITE_ONCE(lpi_ist_ready, true); + } + + /* Go bang if we run VCPU1 before VCPU0 */ + GUEST_ASSERT(READ_ONCE(lpi_ist_ready)); + + gicv5_cpu_enable_interrupts(); + local_irq_enable(); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + if (!vcpu_id) { + gicv5_send_lpi(LPI_TEST_TO_VPE1); + GUEST_SYNC(GUEST_CMD_LPI_SENT); + } + + while (1) + wfi(); +} + +static void restore_test_setup_hwirq(u64 hwirq, u32 target_vpe, + u32 priority, bool enable) +{ + gicv5_set_hwirq_priority(hwirq, priority); + gicv5_set_hwirq_affinity(hwirq, target_vpe); + + if (enable) + gicv5_enable_hwirq(hwirq); +} + +/* + * The source VM configures SPI and LPI state for the IST save/restore test. + * It leaves interrupts disabled, allowing userspace to save the state before + * the restore VM enables interrupts and checks delivery. + */ +static void guest_ist_source_code(void) +{ + local_irq_disable(); + + gicv5_enable_irs(); + gicv5_configure_lpi_ist(); + + restore_test_setup_hwirq(gicv5_lpi_hwirq(RESTORE_TEST_LPI_PENDING), + 1, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_lpi_hwirq(RESTORE_TEST_LPI_ENABLED), + 1, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_HIGH), + 0, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_LOW), + 0, VGIC_V5_IST_PRIO_LOW, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_ENABLED), + 0, VGIC_V5_IST_PRIO_HIGH, true); + restore_test_setup_hwirq(gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED), + 0, VGIC_V5_IST_PRIO_HIGH, false); + + gicv5_send_lpi(RESTORE_TEST_LPI_PENDING); + + GUEST_SYNC(GUEST_CMD_IST_SOURCE_READY); + + while (1) + wfi(); +} + +/* + * This is the restore VM for the IST save/restore test. The IRS regs and ISTs + * have been restored before this code runs. Based on which VPE is running, a + * different set of interrupts is expected. We check that we get what we + * expected before terminating the test. + * + * The expected interrupts for VPE0 are: + * - High-priority SPI, pending and enabled before save. + * - Low-priority SPI, pending and enabled before save. This is only delivered + * after the guest lowers the masked priority via ICC_PCR_EL1. + * - SPI that was pending but disabled before save. This is only delivered + * after the guest enables it after restore. + * - SPI that was enabled but not pending before save. Userspace injects this + * after restore to check that the enable state and affinity were restored. + * + * The expected interrupts for VPE1 are: + * - LPI that was pending and enabled before save. + * - LPI that was enabled but not pending before save. The guest injects this + * after restore to check that the enable state and affinity were restored. + */ +static void guest_ist_restore_irq_handler(struct ex_regs *regs) +{ + u32 vcpu_id = guest_get_vcpuid(); + u32 count = ist_restore_irq_count[vcpu_id]; + u64 hwirq, expected_hwirq = 0; + u64 ia; + + ia = gicr_insn(CDIA); + if (!GICV5_GICR_CDIA_VALID(ia)) + return; + + gsb_ack(); + isb(); + + hwirq = FIELD_GET(GICV5_GICR_CDIA_INTID, ia); + + if (!vcpu_id) { + switch (count) { + case 0: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_HIGH); + break; + case 1: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PRIO_LOW); + break; + case 2: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED); + break; + case 3: + expected_hwirq = gicv5_spi_hwirq(RESTORE_TEST_SPI_ENABLED); + break; + default: + GUEST_FAIL("Unexpected VPE0 IST restore interrupt"); + } + } else { + switch (count) { + case 0: + expected_hwirq = gicv5_lpi_hwirq(RESTORE_TEST_LPI_PENDING); + break; + case 1: + expected_hwirq = gicv5_lpi_hwirq(RESTORE_TEST_LPI_ENABLED); + break; + default: + GUEST_FAIL("Unexpected VPE1 IST restore interrupt"); + } + } + + GUEST_ASSERT_EQ(hwirq, expected_hwirq); + + gic_insn(hwirq, CDDI); + gic_insn(0, CDEOI); + ist_restore_irq_count[vcpu_id]++; + + if (!vcpu_id) { + switch (count) { + case 0: + gicv5_set_pcr(VGIC_V5_IST_PRIO_LOW); + break; + case 1: + gicv5_enable_hwirq( + gicv5_spi_hwirq(RESTORE_TEST_SPI_PENDING_DISABLED)); + break; + case 2: + GUEST_SYNC(GUEST_CMD_IST_SPI_INJECT_READY); + break; + case 3: + GUEST_DONE(); + } + } else { + switch (count) { + case 0: + gicv5_send_lpi(RESTORE_TEST_LPI_ENABLED); + GUEST_SYNC(GUEST_CMD_IST_LPI_INJECT_READY); + break; + case 1: + GUEST_DONE(); + } + } +} + +static void guest_ist_restore_code(void) +{ + local_irq_disable(); + + gicv5_cpu_enable_interrupts(); + + if (!guest_get_vcpuid()) + gicv5_set_pcr(VGIC_V5_IST_PRIO_HIGH); + + GUEST_SYNC(GUEST_CMD_IS_READY); + + local_irq_enable(); + + while (1) + wfi(); +} /* we don't want to assert on run execution, hence that helper */ static int run_vcpu(struct kvm_vcpu *vcpu) @@ -94,12 +587,1005 @@ static void vm_gic_destroy(struct vm_gic *v) kvm_vm_free(v->vm); } +static void vgic_v5_map_irs(struct kvm_vm *vm) +{ + unsigned int nr_irs_pages; + + nr_irs_pages = vm_calc_num_guest_pages(vm->mode, VGIC_V5_IRS_SIZE); + + /* Map the IRS at VA == IPA so guest MMIO writes hit the IRS IODEV. */ + virt_map(vm, GICV5_IRS_CONFIG_BASE_GPA, + GICV5_IRS_CONFIG_BASE_GPA, nr_irs_pages); +} + +static u32 vgic_v5_irq_line_payload(u32 type, u32 num) +{ + return (type << KVM_ARM_IRQ_TYPE_SHIFT) | + FIELD_PREP(KVM_ARM_IRQ_NUM_MASK, num); +} + +static int __vgic_v5_irq_line(struct kvm_vm *vm, u32 type, u32 num, int level) +{ + return _kvm_irq_line(vm, vgic_v5_irq_line_payload(type, num), level); +} + +static void vgic_v5_spi_line(struct kvm_vm *vm, u32 spi, int level) +{ + int ret = __vgic_v5_irq_line(vm, KVM_ARM_IRQ_TYPE_SPI, spi, level); + + TEST_ASSERT(!ret, "KVM_IRQ_LINE failed for SPI %u level %d", + spi, level); +} + +static void vgic_v5_spi_line_vm_create(struct vm_gic *v, + struct kvm_vcpu **vcpu, + u32 nr_spis) +{ + u64 attr; + + v->gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v->vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v->gic_fd = kvm_create_device(v->vm, v->gic_dev_type); + *vcpu = vm_vcpu_add(v->vm, 0, guest_spi_line_code); + TEST_ASSERT(*vcpu, "Failed to create vCPU"); + + vm_init_descriptor_tables(v->vm); + vm_install_exception_handler(v->vm, VECTOR_IRQ_CURRENT, + guest_spi_irq_handler); + vcpu_init_descriptor_tables(*vcpu); + + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0, + &nr_spis); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v->vm); + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); +} + +static void vgic_v5_lpi_ist_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS], + void (*guest_code)(void), + void (*irq_handler)(struct ex_regs *)) +{ + unsigned int nr_lpi_ist_pages; + u64 attr; + int i; + + v->gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v->vm = __vm_create(VM_SHAPE_DEFAULT, VGIC_V5_LPI_NR_VCPUS, 0); + v->gic_fd = kvm_create_device(v->vm, v->gic_dev_type); + + for (i = 0; i < VGIC_V5_LPI_NR_VCPUS; i++) { + vcpus[i] = vm_vcpu_add(v->vm, i, guest_code); + TEST_ASSERT(vcpus[i], "Failed to create vCPU"); + } + + vm_init_descriptor_tables(v->vm); + vm_install_exception_handler(v->vm, VECTOR_IRQ_CURRENT, + irq_handler); + + for (i = 0; i < VGIC_V5_LPI_NR_VCPUS; i++) + vcpu_init_descriptor_tables(vcpus[i]); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v->vm); + + nr_lpi_ist_pages = vm_calc_num_guest_pages(v->vm->mode, + VGIC_V5_LPI_IST_SIZE); + vm_userspace_mem_region_add(v->vm, VM_MEM_SRC_ANONYMOUS, + VGIC_V5_LPI_IST_BASE_GPA, + VGIC_V5_LPI_MEMSLOT, + nr_lpi_ist_pages, 0); + /* + * Map the IST at VA == IPA so the guest can program the same BASER + * address. + */ + virt_map(v->vm, VGIC_V5_LPI_IST_BASE_GPA, + VGIC_V5_LPI_IST_BASE_GPA, nr_lpi_ist_pages); + + kvm_device_attr_set(v->gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); +} + +static void vgic_v5_lpi_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]) +{ + vgic_v5_lpi_ist_vm_create(v, vcpus, guest_lpi_code, + guest_lpi_irq_handler); +} + +static void vgic_v5_ist_restore_vm_create(struct vm_gic *v, + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS], + void (*guest_code)(void), + void (*irq_handler)(struct ex_regs *)) +{ + vgic_v5_lpi_ist_vm_create(v, vcpus, guest_code, irq_handler); +} + +static void vgic_v5_run_spi_line_test(u32 nr_spis, u32 expected_spi, + bool level_sensitive, + bool lower_before_run) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + struct ucall uc; + int ret; + + spi_line_expected = expected_spi; + spi_line_level_sensitive = level_sensitive; + + vgic_v5_spi_line_vm_create(&v, &vcpu, nr_spis); + + sync_global_to_guest(v.vm, spi_line_expected); + sync_global_to_guest(v.vm, spi_line_level_sensitive); + + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 IRQ_LINE VM"); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 IRQ_LINE guest did not become ready"); + + vgic_v5_spi_line(v.vm, expected_spi, 1); + /* + * For edge-triggered SPIs, a following low transition must be ignored: + * once the edge has made the SPI pending, it cannot be recalled. This + * allows us to test that. + */ + if (lower_before_run) + vgic_v5_spi_line(v.vm, expected_spi, 0); + + while (1) { + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 IRQ_LINE VM"); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + break; + case UCALL_DONE: + goto done; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } + +done: + vm_gic_destroy(&v); +} + +static void vgic_v5_expect_sync(struct kvm_vcpu *vcpu, u64 cmd, + const char *msg) +{ + struct ucall uc; + + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && uc.args[1] == cmd, + "%s", msg); +} + +struct vgic_region_attr { + u64 attr; + u64 size; + u64 alignment; +}; + +static const struct vgic_region_attr gic_v5_irs_region = { + .attr = KVM_VGIC_V5_ADDR_TYPE_IRS, + .size = GICV5_IRS_SIZE, + .alignment = GICV5_IRS_ALIGN, +}; + +struct vgic_sysreg_attr { + const char *name; + u32 encoding; +}; + +struct vgic_irs_reg_attr { + const char *name; + u64 attr; +}; + +#define PACK_SR(r) \ + ((sys_reg_Op0(r) << 14) | \ + (sys_reg_Op1(r) << 11) | \ + (sys_reg_CRn(r) << 7) | \ + (sys_reg_CRm(r) << 3) | \ + (sys_reg_Op2(r))) + +#define SR(r) \ + { \ + .name = #r, \ + .encoding = r, \ + } + +#define IRS_REG(r) \ + { \ + .name = #r, \ + .attr = r, \ + } + +static const struct vgic_irs_reg_attr gic_v5_irs_regs[] = { + IRS_REG(GICV5_IRS_IDR0), + IRS_REG(GICV5_IRS_IDR1), + IRS_REG(GICV5_IRS_IDR2), + IRS_REG(GICV5_IRS_IDR3), + IRS_REG(GICV5_IRS_IDR4), + IRS_REG(GICV5_IRS_IDR5), + IRS_REG(GICV5_IRS_IDR6), + IRS_REG(GICV5_IRS_IDR7), + IRS_REG(GICV5_IRS_IIDR), + IRS_REG(GICV5_IRS_AIDR), + IRS_REG(GICV5_IRS_CR0), + IRS_REG(GICV5_IRS_CR1), + IRS_REG(GICV5_IRS_SYNCR), + IRS_REG(GICV5_IRS_SYNC_STATUSR), + IRS_REG(GICV5_IRS_SPI_VMR), + IRS_REG(GICV5_IRS_SPI_SELR), + IRS_REG(GICV5_IRS_SPI_DOMAINR), + IRS_REG(GICV5_IRS_SPI_RESAMPLER), + IRS_REG(GICV5_IRS_SPI_CFGR), + IRS_REG(GICV5_IRS_SPI_STATUSR), + IRS_REG(GICV5_IRS_PE_SELR), + IRS_REG(GICV5_IRS_PE_STATUSR), + IRS_REG(GICV5_IRS_PE_CR0), + IRS_REG(GICV5_IRS_IST_BASER), + IRS_REG(GICV5_IRS_IST_CFGR), + IRS_REG(GICV5_IRS_IST_STATUSR), + IRS_REG(GICV5_IRS_MAP_L2_ISTR), + IRS_REG(GICV5_IRS_VMT_BASER), + IRS_REG(GICV5_IRS_VMT_CFGR), + IRS_REG(GICV5_IRS_VMT_STATUSR), + IRS_REG(GICV5_IRS_VPE_SELR), + IRS_REG(GICV5_IRS_VPE_DBR), + IRS_REG(GICV5_IRS_VPE_HPPIR), + IRS_REG(GICV5_IRS_VPE_CR0), + IRS_REG(GICV5_IRS_VPE_STATUSR), + IRS_REG(GICV5_IRS_VM_DBR), + IRS_REG(GICV5_IRS_VM_SELR), + IRS_REG(GICV5_IRS_VM_STATUSR), + IRS_REG(GICV5_IRS_VMAP_L2_VMTR), + IRS_REG(GICV5_IRS_VMAP_VMR), + IRS_REG(GICV5_IRS_VMAP_VISTR), + IRS_REG(GICV5_IRS_VMAP_L2_VISTR), + IRS_REG(GICV5_IRS_VMAP_VPER), + IRS_REG(GICV5_IRS_SAVE_VMR), + IRS_REG(GICV5_IRS_SAVE_VM_STATUSR), + IRS_REG(GICV5_IRS_MEC_IDR), + IRS_REG(GICV5_IRS_MEC_MECID_R), + IRS_REG(GICV5_IRS_MPAM_IDR), + IRS_REG(GICV5_IRS_MPAM_PARTID_R), + IRS_REG(GICV5_IRS_SWERR_STATUSR), + IRS_REG(GICV5_IRS_SWERR_SYNDROMER0), + IRS_REG(GICV5_IRS_SWERR_SYNDROMER1), +}; + +static const struct vgic_sysreg_attr gic_v5_cpu_sysregs[] = { + SR(SYS_ICC_ICSR_EL1), + SR(SYS_ICC_PPI_ENABLER0_EL1), + SR(SYS_ICC_PPI_ENABLER1_EL1), + SR(SYS_ICC_PPI_SACTIVER0_EL1), + SR(SYS_ICC_PPI_SACTIVER1_EL1), + SR(SYS_ICC_PPI_SPENDR0_EL1), + SR(SYS_ICC_PPI_SPENDR1_EL1), + SR(SYS_ICC_PPI_PRIORITYR0_EL1), + SR(SYS_ICC_PPI_PRIORITYR1_EL1), + SR(SYS_ICC_PPI_PRIORITYR2_EL1), + SR(SYS_ICC_PPI_PRIORITYR3_EL1), + SR(SYS_ICC_PPI_PRIORITYR4_EL1), + SR(SYS_ICC_PPI_PRIORITYR5_EL1), + SR(SYS_ICC_PPI_PRIORITYR6_EL1), + SR(SYS_ICC_PPI_PRIORITYR7_EL1), + SR(SYS_ICC_PPI_PRIORITYR8_EL1), + SR(SYS_ICC_PPI_PRIORITYR9_EL1), + SR(SYS_ICC_PPI_PRIORITYR10_EL1), + SR(SYS_ICC_PPI_PRIORITYR11_EL1), + SR(SYS_ICC_PPI_PRIORITYR12_EL1), + SR(SYS_ICC_PPI_PRIORITYR13_EL1), + SR(SYS_ICC_PPI_PRIORITYR14_EL1), + SR(SYS_ICC_PPI_PRIORITYR15_EL1), + SR(SYS_ICC_APR_EL1), + SR(SYS_ICC_CR0_EL1), + SR(SYS_ICC_PCR_EL1), +}; + +static void vgic_v5_save_irs_regs(struct vm_gic *v, u64 *regs) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + ret = __kvm_device_attr_get(v->gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + gic_v5_irs_regs[i].attr, + ®s[i]); + TEST_ASSERT(!ret, "GICv5 IRS_REGS save failed for %s", + gic_v5_irs_regs[i].name); + } +} + +static void vgic_v5_restore_irs_regs(struct vm_gic *v, u64 *regs) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + ret = __kvm_device_attr_set(v->gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + gic_v5_irs_regs[i].attr, + ®s[i]); + TEST_ASSERT(!ret, "GICv5 IRS_REGS restore failed for %s", + gic_v5_irs_regs[i].name); + } +} + +static u64 vgic_v5_saved_irs_reg(u64 *regs, u64 attr) +{ + for (int i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + if (gic_v5_irs_regs[i].attr == attr) + return regs[i]; + } + + TEST_FAIL("GICv5 IRS register 0x%lx was not saved", attr); +} + +static size_t vgic_v5_lpi_ist_state_size(u64 *irs_regs) +{ + u64 cfgr = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_CFGR); + u32 lpi_id_bits = FIELD_GET(GICV5_IRS_IST_CFGR_LPI_ID_BITS, cfgr); + + return BIT(lpi_id_bits) * sizeof(__u32); +} + +static void test_vgic_v5_addr_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 addr; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + + /* Check existing group/attributes */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, gic_v5_irs_region.attr); + + /* check non existing attribute */ + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, -1); + TEST_ASSERT(ret && errno == ENXIO, "attribute not supported"); + + /* get IRS base address before setting*/ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret && addr == (-1ULL), "GICv5 IRS returns VGIC_ADDR_UNDEF"); + + /* misaligned IRS address settings */ + addr = gic_v5_irs_region.alignment / 0x10; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == EINVAL, "GIC IRS base not aligned"); + + /* out of range address */ + addr = max_phys_size; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == E2BIG, "IRS address beyond IPA limit"); + + /* Space for half an IRS (an IRS is: 2 * irs.alignment). */ + addr = max_phys_size - gic_v5_irs_region.alignment; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == E2BIG, + "half of the IRS is beyond IPA limit"); + + /* set IRS base address @0x0*/ + addr = 0x00000; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret, "GICv5 IRS base correctly set"); + + /* get IRS base address */ + addr = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(!ret && addr == 0, "GICv5 IRS base correctly set"); + + /* Attempt to create a second IRS region */ + addr = 0xE0000; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &addr); + TEST_ASSERT(ret && errno == EEXIST, "GICv5 IRS base set again"); + + vm_gic_destroy(&v); + + /* Try running a VM without ever setting the IRS base addr */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + ret = run_vcpu(vcpu); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS base not set"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_nr_irqs_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u32 nr_irqs; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0); + + /* Before userspace sets NR_IRQS, no SPI count has been selected. */ + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == 0, "GICv5 NR_IRQS defaults to 0 before init"); + + /* Too few SPIs */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS - 1; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS below minimum"); + + /* Not a multiple of 32 */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS + 1; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS not 32-aligned"); + + /* Larger than KVM's supported VGICv5 SPI count */ + nr_irqs = VGIC_V5_MAX_NR_SPIS + 32; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 NR_IRQS above maximum"); + + /* Valid custom SPI count */ + nr_irqs = VGIC_V5_MAX_NR_SPIS; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret, "GICv5 NR_IRQS accepts valid custom SPI count"); + + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == VGIC_V5_MAX_NR_SPIS, + "GICv5 NR_IRQS returns SPI count only"); + + /* A second successful configuration attempt must be rejected. */ + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 NR_IRQS set twice"); + + /* The maximum supported count must also initialize successfully. */ + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + vm_gic_destroy(&v); + + /* If userspace does not set NR_IRQS, init selects the default. */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + nr_irqs = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(!ret && nr_irqs == VGIC_V5_DEFAULT_NR_SPIS, + "GICv5 NR_IRQS defaults to 32 SPIs after init"); + + nr_irqs = VGIC_V5_DEFAULT_NR_SPIS * 2; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, + 0, &nr_irqs); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 NR_IRQS set after init"); + + vm_gic_destroy(&v); + +} + +static void test_vgic_v5_irs_regs_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 attr, val; + int ret, i; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* IRS_REGS attributes can be probed before the IRS base is set. */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0); + + /* IRS_REGS are not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IRS_REGS get before init"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IRS_REGS set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Read all supported IRS regs and write the value back. */ + for (i = 0; i < ARRAY_SIZE(gic_v5_irs_regs); i++) { + attr = gic_v5_irs_regs[i].attr; + ret = __kvm_has_device_attr(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(!ret, "GICv5 IRS_REGS missing %s", + gic_v5_irs_regs[i].name); + + val = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get failed for %s", + gic_v5_irs_regs[i].name); + + ret = __kvm_device_attr_set(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS set failed for %s", + gic_v5_irs_regs[i].name); + } + + /* Check bad offsets */ + attr = 0x10000; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS_REGS accepted bad offset"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr, &val); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IRS_REGS get bad offset"); + + /* Check alignment for 32-bit and 64-bit IRS regs. */ + attr = GICV5_IRS_IDR0 + 2; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unaligned 32-bit attr"); + + attr = GICV5_IRS_IST_BASER + 4; + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unaligned 64-bit attr"); + + /* Check bad user pointers */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IRS_REGS get with bad pointer"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IRS_REGS set with bad pointer"); + + /* ID restore validation rejects unsupported values. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR0 failed"); + val &= ~GICV5_IRS_IDR0_INT_DOM; + val |= FIELD_PREP(GICV5_IRS_IDR0_INT_DOM, + GICV5_IRS_IDR0_INT_DOM_SECURE); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR0 domain"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR0 failed"); + val |= GICV5_IRS_IDR0_SETLPI; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR0, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted unsupported IDR0"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR1, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR1 failed"); + val |= FIELD_PREP(GICV5_IRS_IDR1_PRIORITY_BITS, 0x7); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR1, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR1"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR2, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR2"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR5, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR5 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR5, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR5"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR6, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR6 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR6, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR6"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR7, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS get IDR7 failed"); + val++; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IDR7, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IRS_REGS accepted bad IDR7"); + + /* Status registers read as idle through userspace accessors. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_SYNC_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_SYNC_STATUSR_IDLE, + "GICv5 IRS_REGS SYNC_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_SPI_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_SPI_STATUSR_IDLE, + "GICv5 IRS_REGS SPI_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_PE_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_PE_STATUSR_IDLE, + "GICv5 IRS_REGS PE_STATUSR is not idle"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_STATUSR, &val); + TEST_ASSERT(!ret && val == GICV5_IRS_IST_STATUSR_IDLE, + "GICv5 IRS_REGS IST_STATUSR is not idle"); + + /* + * Userspace restores IST_BASER without allocating or accessing an IST. + * Supply a synthetic, aligned guest address only to verify the register + * round trip. + */ + val = FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, + VGIC_V5_TEST_IST_BASER_GPA >> GICV5_IRS_IST_BASER_ADDR_SHIFT); + val |= GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &val); + TEST_ASSERT(!ret, "GICv5 IRS_REGS failed to restore IST_BASER"); + + attr = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &attr); + TEST_ASSERT(!ret && attr == val, "GICv5 IRS_REGS IST_BASER restore mismatch"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_ist_attrs(void) +{ + struct kvm_vgic_v5_ist ist_attr; + struct kvm_vcpu *vcpu; + struct vm_gic v; + u32 spi_ist[VGIC_V5_DEFAULT_NR_SPIS]; + u64 attr; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* Check existing group/attribute */ + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, 0); + + /* Check non-existing attribute */ + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, 1); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST accepted bad attr"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 1, &ist_attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST get accepted bad attr"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 1, &ist_attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 IST set accepted bad attr"); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + + /* IST save/restore is not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST get before init"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* A VM with SPIs must provide a userspace IST descriptor. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, NULL); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted NULL descriptor"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, NULL); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted NULL descriptor"); + + /* Check bad userspace IST descriptors. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, (void *)1); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IST get with bad descriptor"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, (void *)1); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 IST set with bad descriptor"); + + /* Check missing and incorrectly sized SPI IST buffers. */ + ist_attr = vgic_v5_ist_attr(NULL, 0, NULL, 0); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted missing SPI buffer"); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted missing SPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist) - sizeof(__u32), + NULL, 0); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted bad SPI size"); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted bad SPI size"); + + /* LPI storage must be absent when no LPI IST is configured. */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), spi_ist, + sizeof(__u32)); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST get accepted unexpected LPI buffer"); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST set accepted unexpected LPI buffer"); + + vm_gic_destroy(&v); + + /* IST restore is rejected after the VM has run. */ + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, guest_code); + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + ret = run_vcpu(vcpu); + TEST_ASSERT(!ret, "Failed to run GICv5 VM before IST restore test"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 IST restore after run"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_userspace_ppis_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 user_ppis[2]; + int ret; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT); + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + user_ppis[0] = 0; + user_ppis[1] = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, user_ppis); + TEST_ASSERT(!ret, "GICv5 USERSPACE_PPIS get failed"); + TEST_ASSERT(user_ppis[0] & BIT(GICV5_ARCH_PPI_SW_PPI), + "GICv5 USERSPACE_PPIS does not expose SW_PPI"); + TEST_ASSERT(!user_ppis[1], "GICv5 USERSPACE_PPIS upper word is not zero"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, user_ppis); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 USERSPACE_PPIS set accepted"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_USERSPACE_PPIS, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 USERSPACE_PPIS get with bad pointer"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_cpu_sysreg_attrs(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + u64 attr, val; + int ret, i; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + vcpu = vm_vcpu_add(v.vm, 0, NULL); + TEST_ASSERT(vcpu, "Failed to create vCPU"); + + /* Check existing group/attribute */ + attr = PACK_SR(SYS_ICC_CR0_EL1); + kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, attr); + + /* CPU sysregs are not accessible before the VGIC is initialized. */ + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 CPU_SYSREGS get before init"); + + val = 0; + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EBUSY, "GICv5 CPU_SYSREGS set before init"); + + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Read all exposed CPU sysregs and write the value back. */ + for (i = 0; i < ARRAY_SIZE(gic_v5_cpu_sysregs); i++) { + attr = PACK_SR(gic_v5_cpu_sysregs[i].encoding); + ret = __kvm_has_device_attr(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS missing %s", + gic_v5_cpu_sysregs[i].name); + + val = 0xbad; + ret = __kvm_device_attr_get(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS get failed for %s", + gic_v5_cpu_sysregs[i].name); + + ret = __kvm_device_attr_set(v.gic_fd, + KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(!ret, "GICv5 CPU_SYSREGS set failed for %s", + gic_v5_cpu_sysregs[i].name); + } + + /* Check non existent GICv3 sysreg */ + attr = PACK_SR(SYS_ICC_CTLR_EL1); + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(ret && errno == ENXIO, "GICv5 CPU_SYSREGS accepted bad sysreg"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == ENOENT, "GICv5 CPU_SYSREGS get bad sysreg"); + + /* Check non existing vCPU */ + attr = PACK_SR(SYS_ICC_CR0_EL1) | + (1ULL << KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT); + ret = __kvm_has_device_attr(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 CPU_SYSREGS accepted bad MPIDR"); + + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, &val); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 CPU_SYSREGS get bad MPIDR"); + + /* Check bad user pointers */ + attr = PACK_SR(SYS_ICC_CR0_EL1); + ret = __kvm_device_attr_get(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 CPU_SYSREGS get with bad pointer"); + + ret = __kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS, + attr, NULL); + TEST_ASSERT(ret && errno == EFAULT, "GICv5 CPU_SYSREGS set with bad pointer"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_sparse_vcpu_ids(void) +{ + struct kvm_vcpu *vcpus[VGIC_V5_SPARSE_NR_VCPUS]; + const u32 vcpu_ids[VGIC_V5_SPARSE_NR_VCPUS] = { + VGIC_V5_SPARSE_VCPU0_ID, + VGIC_V5_SPARSE_VCPU1_ID, + }; + int ret, i; + struct ucall uc; + struct vm_gic v; + u64 attr; + + v.gic_dev_type = KVM_DEV_TYPE_ARM_VGIC_V5; + v.vm = __vm_create(VM_SHAPE_DEFAULT, VGIC_V5_SPARSE_NR_VCPUS, 0); + v.gic_fd = kvm_create_device(v.vm, v.gic_dev_type); + + /* Create two vCPUs with legal, albeit sparse, IDs. */ + for (i = 0; i < ARRAY_SIZE(vcpus); i++) { + vcpus[i] = vm_vcpu_add(v.vm, vcpu_ids[i], guest_iaffid_code); + TEST_ASSERT(vcpus[i], "Failed to create vCPU %u", vcpu_ids[i]); + } + + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + vgic_v5_map_irs(v.vm); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, + KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); + + /* Check both are runnable, and read back their correct IAFFIDs */ + for (i = 0; i < ARRAY_SIZE(vcpus); i++) { + /* IAFFID is the VPE ID and must match the userspace vCPU ID. */ + ret = run_vcpu(vcpus[i]); + TEST_ASSERT(!ret, "Failed to run GICv5 vCPU %u", vcpu_ids[i]); + TEST_ASSERT(get_ucall(vcpus[i], &uc) == UCALL_SYNC && + uc.args[1] == vcpu_ids[i], + "GICv5 vCPU %u IAFFID mismatch", vcpu_ids[i]); + + ret = run_vcpu(vcpus[i]); + TEST_ASSERT(!ret, "Failed to complete GICv5 vCPU %u", vcpu_ids[i]); + TEST_ASSERT(get_ucall(vcpus[i], NULL) == UCALL_DONE, + "GICv5 vCPU %u did not complete", vcpu_ids[i]); + } + + vm_gic_destroy(&v); +} + static void test_vgic_v5_ppis(u32 gic_dev_type) { struct kvm_vcpu *vcpus[NR_VCPUS]; struct ucall uc; u64 user_ppis[2]; struct vm_gic v; + uint64_t attr; int ret, i; v.gic_dev_type = gic_dev_type; @@ -111,11 +1597,17 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) vcpus[i] = vm_vcpu_add(v.vm, i, guest_code); vm_init_descriptor_tables(v.vm); - vm_install_exception_handler(v.vm, VECTOR_IRQ_CURRENT, guest_irq_handler); + vm_install_exception_handler(v.vm, VECTOR_IRQ_CURRENT, + guest_ppi_irq_handler); for (i = 0; i < NR_VCPUS; i++) vcpu_init_descriptor_tables(vcpus[i]); + /* Set the address of the IRS before initialising the GIC */ + attr = GICV5_IRS_CONFIG_BASE_GPA; + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_ADDR, + KVM_VGIC_V5_ADDR_TYPE_IRS, &attr); + kvm_device_attr_set(v.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL, KVM_DEV_ARM_VGIC_CTRL_INIT, NULL); @@ -127,6 +1619,20 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) TEST_ASSERT(user_ppis[0] & BIT(GICV5_ARCH_PPI_SW_PPI), "SW_PPI is not drivable by userspace"); + /* PPIs not explicitly exposed to userspace must be rejected. */ + for (i = 0; i < VGIC_V5_NR_PRIVATE_IRQS; i++) { + if (user_ppis[i / 64] & BIT_ULL(i % 64)) + continue; + + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_PPI, i, 1); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted non-userspace PPI %d", i); + } + + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_PPI, + VGIC_V5_NR_PRIVATE_IRQS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted out-of-range PPI"); + while (1) { ret = run_vcpu(vcpus[0]); if (ret) @@ -148,9 +1654,9 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) kvm_irq_line(v.vm, irq, level); } else if (uc.args[1] == GUEST_CMD_IS_AWAKE) { - pr_info("Guest skipping WFI due to pending IRQ\n"); + pr_debug("Guest skipping WFI due to pending IRQ\n"); } else if (uc.args[1] == GUEST_CMD_IRQ_CDIA) { - pr_info("Guest acknowledged IRQ\n"); + pr_debug("Guest acknowledged IRQ\n"); } continue; @@ -170,6 +1676,304 @@ static void test_vgic_v5_ppis(u32 gic_dev_type) vm_gic_destroy(&v); } +static void test_vgic_v5_spis(void) +{ + struct kvm_vcpu *vcpu; + struct vm_gic v; + int ret; + + /* Default NR_IRQS exposes 32 SPIs, numbered 0..31 in KVM_IRQ_LINE. */ + vgic_v5_spi_line_vm_create(&v, &vcpu, VGIC_V5_DEFAULT_NR_SPIS); + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_SPI, + VGIC_V5_DEFAULT_NR_SPIS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted first invalid SPI"); + + vm_gic_destroy(&v); + + /* Basic SPI injection through KVM_IRQ_LINE. */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 0, false, false); + + /* The last valid SPI in the maximum configured range is injectable. */ + vgic_v5_run_spi_line_test(VGIC_V5_MAX_NR_SPIS, + VGIC_V5_MAX_NR_SPIS - 1, false, false); + + vgic_v5_spi_line_vm_create(&v, &vcpu, VGIC_V5_MAX_NR_SPIS); + ret = __vgic_v5_irq_line(v.vm, KVM_ARM_IRQ_TYPE_SPI, + VGIC_V5_MAX_NR_SPIS, 1); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 accepted configured invalid SPI"); + vm_gic_destroy(&v); + + /* + * Edge SPIs remain pending after the line is lowered. The low + * transition is injected before the guest runs and must be ignored: + * once an edge interrupt is pending, lowering the line cannot recall it. + */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 1, false, true); + + /* Level SPIs can be raised and delivered through KVM_IRQ_LINE. */ + vgic_v5_run_spi_line_test(VGIC_V5_DEFAULT_NR_SPIS, 2, true, false); +} + +static void test_vgic_v5_lpis(void) +{ + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]; + struct ucall uc; + struct vm_gic v; + int ret; + + lpi_ist_ready = false; + vgic_v5_lpi_vm_create(&v, vcpus); + sync_global_to_guest(v.vm, lpi_ist_ready); + + /* VPE0 programs a linear LPI IST from the virtual IRS ID registers. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 LPI vCPU0 did not become ready"); + + /* VPE1 waits for the shared LPI setup and enables its CPU interface. */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU1"); + TEST_ASSERT(get_ucall(vcpus[1], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_IS_READY, + "GICv5 LPI vCPU1 did not become ready"); + + /* VPE0 sends an LPI to VPE1. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0 sender"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_LPI_SENT, + "GICv5 LPI vCPU0 did not send LPI"); + + /* VPE1 consumes the LPI and replies with another LPI. */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU1 receiver"); + TEST_ASSERT(get_ucall(vcpus[1], &uc) == UCALL_SYNC && + uc.args[1] == GUEST_CMD_LPI_REPLIED, + "GICv5 LPI vCPU1 did not reply"); + + /* VPE0 consumes the reply LPI and ends the test. */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 LPI vCPU0 receiver"); + TEST_ASSERT(get_ucall(vcpus[0], &uc) == UCALL_DONE, + "GICv5 LPI vCPU0 did not complete"); + + vm_gic_destroy(&v); +} + +static void test_vgic_v5_ist_save_restore(void) +{ + static u32 lpi_ist[VGIC_V5_LPI_IST_SIZE / sizeof(u32)]; + static u32 lpi_ist_resave[VGIC_V5_LPI_IST_SIZE / sizeof(u32)]; + static u32 spi_ist[VGIC_V5_DEFAULT_NR_SPIS]; + static u32 spi_ist_resave[VGIC_V5_DEFAULT_NR_SPIS]; + static u64 irs_regs[ARRAY_SIZE(gic_v5_irs_regs)]; + struct kvm_vcpu *vcpus[VGIC_V5_LPI_NR_VCPUS]; + struct kvm_vgic_v5_ist ist_attr; + struct vm_gic src, dst; + u64 ist_baser, bad_ist_baser; + u64 ist_cfgr, bad_ist_cfgr; + size_t lpi_ist_size; + int ret; + + memset(ist_restore_irq_count, 0, sizeof(ist_restore_irq_count)); + + vgic_v5_ist_restore_vm_create(&src, vcpus, guest_ist_source_code, + guest_unexpected_irq_handler); + + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST save source VM"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IST_SOURCE_READY, + "GICv5 IST source guest did not become ready"); + + /* + * The guest configured enable/priority/affinity state. Make selected + * SPIs pending from userspace so the SPI IST save path sees them. + */ + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PRIO_HIGH, 1); + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PRIO_LOW, 1); + vgic_v5_spi_line(src.vm, RESTORE_TEST_SPI_PENDING_DISABLED, 1); + + vgic_v5_save_irs_regs(&src, irs_regs); + lpi_ist_size = vgic_v5_lpi_ist_state_size(irs_regs); + + /* + * VCPU0 has configured the initial state. KVM must reject missing or + * incorrectly sized LPI storage before accepting the complete IST image. + */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST save accepted missing LPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size - sizeof(__u32)); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST save accepted bad LPI size"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_get(src.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST save failed"); + + vm_gic_destroy(&src); + + /* Create the guest that we are restoring state into */ + memset(ist_restore_irq_count, 0, sizeof(ist_restore_irq_count)); + vgic_v5_ist_restore_vm_create(&dst, vcpus, guest_ist_restore_code, + guest_ist_restore_irq_handler); + sync_global_to_guest(dst.vm, ist_restore_irq_count); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore before IRS state"); + + vgic_v5_restore_irs_regs(&dst, irs_regs); + + /* BASER remains mutable until the host LPI IST has been allocated. */ + ist_baser = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_BASER); + bad_ist_baser = ist_baser ^ + FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected changed unallocated IST_BASER"); + + bad_ist_baser = ist_baser & ~GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected invalidated unallocated IST_BASER"); + + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &ist_baser); + TEST_ASSERT(!ret, "GICv5 failed to restore IST_BASER"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), NULL, 0); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore accepted missing LPI buffer"); + + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size - sizeof(__u32)); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(ret && errno == EINVAL, "GICv5 IST restore accepted bad LPI size"); + + /* Restore the VMM-owned SPI and LPI IST images. */ + ist_attr = vgic_v5_ist_attr(spi_ist, sizeof(spi_ist), lpi_ist, + lpi_ist_size); + ret = __kvm_device_attr_set(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST restore failed"); + + /* A live IST makes BASER and the guest-visible configuration immutable. */ + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &ist_baser); + TEST_ASSERT(!ret, "GICv5 rejected unchanged live IST_BASER"); + + bad_ist_baser = ist_baser ^ + FIELD_PREP(GICV5_IRS_IST_BASER_ADDR_MASK, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted changed live IST_BASER"); + + bad_ist_baser = ist_baser & ~GICV5_IRS_IST_BASER_VALID; + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_BASER, &bad_ist_baser); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted invalidated live IST_BASER"); + + ist_cfgr = vgic_v5_saved_irs_reg(irs_regs, GICV5_IRS_IST_CFGR); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_CFGR, &ist_cfgr); + TEST_ASSERT(!ret, "GICv5 rejected unchanged live IST_CFGR"); + + bad_ist_cfgr = ist_cfgr ^ + FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, 1); + ret = __kvm_device_attr_set(dst.gic_fd, + KVM_DEV_ARM_VGIC_GRP_IRS_REGS, + GICV5_IRS_IST_CFGR, &bad_ist_cfgr); + TEST_ASSERT(ret && errno == EINVAL, + "GICv5 accepted changed live IST_CFGR"); + + /* + * Re-save the restored IST image before the destination VM runs, so a + * dropped serialized field fails independently of later interrupt + * delivery. + */ + ist_attr = vgic_v5_ist_attr(spi_ist_resave, sizeof(spi_ist_resave), + lpi_ist_resave, lpi_ist_size); + ret = __kvm_device_attr_get(dst.gic_fd, KVM_DEV_ARM_VGIC_GRP_IST, + 0, &ist_attr); + TEST_ASSERT(!ret, "GICv5 IST re-save failed"); + TEST_ASSERT(!memcmp(spi_ist, spi_ist_resave, sizeof(spi_ist)), + "GICv5 SPI IST state changed across restore"); + TEST_ASSERT(!memcmp(lpi_ist, lpi_ist_resave, lpi_ist_size), + "GICv5 LPI IST state changed across restore"); + + /* + * At this stage, the guest's IST state should match what was saved. Run + * each vcpu once to let it enable interrupts, etc. + */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST restore vCPU0"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IS_READY, + "GICv5 IST restore vCPU0 did not become ready"); + + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to run GICv5 IST restore vCPU1"); + vgic_v5_expect_sync(vcpus[1], GUEST_CMD_IS_READY, + "GICv5 IST restore vCPU1 did not become ready"); + + /* + * VPE0 consumes two pending SPIs in restored priority order, then + * enables a restored pending-but-disabled SPI. + */ + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to consume restored GICv5 SPIs"); + vgic_v5_expect_sync(vcpus[0], GUEST_CMD_IST_SPI_INJECT_READY, + "GICv5 IST restore vCPU0 did not consume restored SPIs"); + + /* + * This SPI was enabled before save, but not pending. Inject it after + * restore to prove enable, priority and affinity were restored. + */ + vgic_v5_spi_line(dst.vm, RESTORE_TEST_SPI_ENABLED, 1); + + ret = run_vcpu(vcpus[0]); + TEST_ASSERT(!ret, "Failed to consume post-restore GICv5 SPI"); + TEST_ASSERT(get_ucall(vcpus[0], NULL) == UCALL_DONE, + "GICv5 IST restore vCPU0 did not complete"); + + /* + * VPE1 consumes a restored pending LPI, then pends another LPI that + * was enabled before save but not pending. + */ + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to consume restored GICv5 LPI"); + vgic_v5_expect_sync(vcpus[1], GUEST_CMD_IST_LPI_INJECT_READY, + "GICv5 IST restore vCPU1 did not consume restored LPI"); + + ret = run_vcpu(vcpus[1]); + TEST_ASSERT(!ret, "Failed to consume post-restore GICv5 LPI"); + TEST_ASSERT(get_ucall(vcpus[1], NULL) == UCALL_DONE, + "GICv5 IST restore vCPU1 did not complete"); + + vm_gic_destroy(&dst); +} + /* * Returns 0 if it's possible to create GIC device of a given type (V5). */ @@ -201,13 +2005,46 @@ int test_kvm_device(u32 gic_dev_type) void run_tests(u32 gic_dev_type) { + pr_info("Test VGICv5 address attrs\n"); + test_vgic_v5_addr_attrs(); + + pr_info("Test VGICv5 NR_IRQS attrs\n"); + test_vgic_v5_nr_irqs_attrs(); + + pr_info("Test VGICv5 IRS_REGS attrs\n"); + test_vgic_v5_irs_regs_attrs(); + + pr_info("Test VGICv5 IST attrs\n"); + test_vgic_v5_ist_attrs(); + + pr_info("Test VGICv5 userspace PPI attrs\n"); + test_vgic_v5_userspace_ppis_attrs(); + + pr_info("Test VGICv5 CPU sysreg attrs\n"); + test_vgic_v5_cpu_sysreg_attrs(); + + pr_info("Test VGICv5 sparse vCPU IDs\n"); + test_vgic_v5_sparse_vcpu_ids(); + pr_info("Test VGICv5 PPIs\n"); test_vgic_v5_ppis(gic_dev_type); + + pr_info("Test VGICv5 SPIs\n"); + test_vgic_v5_spis(); + + pr_info("Test VGICv5 LPIs\n"); + test_vgic_v5_lpis(); + + pr_info("Test VGICv5 IST save/restore\n"); + test_vgic_v5_ist_save_restore(); } int main(int ac, char **av) { - int ret; + int pa_bits, ret; + + pa_bits = vm_guest_mode_params[VM_MODE_DEFAULT].pa_bits; + max_phys_size = 1ULL << pa_bits; test_disable_default_vgic(); diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c index f8b3d0b68830..619a0b2be45c 100644 --- a/tools/testing/selftests/kvm/demand_paging_test.c +++ b/tools/testing/selftests/kvm/demand_paging_test.c @@ -297,8 +297,7 @@ int main(int argc, char *argv[]) p.single_uffd = true; break; case 'd': - p.uffd_delay = strtoul(optarg, NULL, 0); - TEST_ASSERT(p.uffd_delay >= 0, "A negative UFFD delay is not supported."); + p.uffd_delay = atoi_non_negative("UFFD delay", optarg); break; case 'b': guest_percpu_mem_size = parse_size(optarg); diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index af5eb0334a74..8189eaac713c 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -44,8 +44,11 @@ #define TEST_MIN_WRITES_PER_ITERATION 0x100 /* Dirty bitmaps are always little endian, so we need to swap on big endian */ -#if defined(__s390x__) +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ # define BITOP_LE_SWIZZLE ((BITS_PER_LONG-1) & ~0x7) +#else +# define BITOP_LE_SWIZZLE 0 +#endif # define test_bit_le(nr, addr) \ test_bit((nr) ^ BITOP_LE_SWIZZLE, addr) # define __set_bit_le(nr, addr) \ @@ -56,13 +59,6 @@ __test_and_set_bit((nr) ^ BITOP_LE_SWIZZLE, addr) # define __test_and_clear_bit_le(nr, addr) \ __test_and_clear_bit((nr) ^ BITOP_LE_SWIZZLE, addr) -#else -# define test_bit_le test_bit -# define __set_bit_le __set_bit -# define __clear_bit_le __clear_bit -# define __test_and_set_bit_le __test_and_set_bit -# define __test_and_clear_bit_le __test_and_clear_bit -#endif #define TEST_DIRTY_RING_COUNT 65536 diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index 2233d871a38f..cd5df88bc642 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -80,7 +80,7 @@ static void test_mbind(int fd, size_t total_size) { const unsigned long nodemask_0 = 1; /* nid: 0 */ unsigned long nodemask = 0; - unsigned long maxnode = BITS_PER_TYPE(nodemask); + unsigned long maxnode = MAXNODE_FOR_MASK(nodemask); int policy; char *mem; int ret; diff --git a/tools/testing/selftests/kvm/hardware_disable_test.c b/tools/testing/selftests/kvm/hardware_disable_test.c index 1c20892d6782..ef004afd74a5 100644 --- a/tools/testing/selftests/kvm/hardware_disable_test.c +++ b/tools/testing/selftests/kvm/hardware_disable_test.c @@ -130,6 +130,9 @@ int main(int argc, char **argv) int s, r, cpu, i; pid_t pid; + /* PowerPC PAPR L0 cannot guarantee 512 overlapping H_GUEST_CREATE from fork+SIGKILL. */ + TEST_REQUIRE(!kvm_is_ppc_pseries_lpar()); + kvm_sched_getaffinity(0, sizeof(cpu_set_t), &allowed_cpu_set); for (i = 0; i < NR_VCPUS && CPU_COUNT(&allowed_cpu_set); i++) { diff --git a/tools/testing/selftests/kvm/include/arm64/gic_v5.h b/tools/testing/selftests/kvm/include/arm64/gic_v5.h index eb523d9277cf..2d455515563e 100644 --- a/tools/testing/selftests/kvm/include/arm64/gic_v5.h +++ b/tools/testing/selftests/kvm/include/arm64/gic_v5.h @@ -10,6 +10,106 @@ #include "processor.h" +/* GIC component base address is guest PA space */ +#define GICV5_IRS_CONFIG_BASE_GPA 0x8000000ULL +#define GICV5_IRS_CONFIG_BASE_GVA ((volatile void *)GICV5_IRS_CONFIG_BASE_GPA) + +#define GICV5_IRS_SIZE 0x20000 +#define GICV5_IRS_ALIGN 0x10000 + +/* + * IRS register offsets. Keep this selftest copy scoped to the offsets and + * fields exercised by the userspace API tests. + */ +#define GICV5_IRS_IDR0 0x0000 +#define GICV5_IRS_IDR1 0x0004 +#define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 +#define GICV5_IRS_IDR5 0x0014 +#define GICV5_IRS_IDR6 0x0018 +#define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 +#define GICV5_IRS_CR0 0x0080 +#define GICV5_IRS_CR1 0x0084 +#define GICV5_IRS_SYNCR 0x00c0 +#define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 +#define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 +#define GICV5_IRS_SPI_CFGR 0x0114 +#define GICV5_IRS_SPI_STATUSR 0x0118 +#define GICV5_IRS_PE_SELR 0x0140 +#define GICV5_IRS_PE_STATUSR 0x0144 +#define GICV5_IRS_PE_CR0 0x0148 +#define GICV5_IRS_IST_BASER 0x0180 +#define GICV5_IRS_IST_CFGR 0x0190 +#define GICV5_IRS_IST_STATUSR 0x0194 +#define GICV5_IRS_MAP_L2_ISTR 0x01c0 +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 + +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 + +#define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) + +#define GICV5_IRS_IDR2_ISTMD_SZ GENMASK(19, 15) +#define GICV5_IRS_IDR2_ISTMD BIT(14) +#define GICV5_IRS_IDR2_IST_L2SZ GENMASK(13, 11) +#define GICV5_IRS_IDR2_IST_LEVELS BIT(10) +#define GICV5_IRS_IDR2_MIN_LPI_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR2_LPI BIT(5) +#define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) + +#define GICV5_IRS_CR0_IDLE BIT(1) +#define GICV5_IRS_CR0_IRSEN BIT(0) + +#define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_CFGR_TM BIT(0) + +#define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_PE_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL +#define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) +#define GICV5_IRS_IST_CFGR_L2SZ GENMASK(6, 5) +#define GICV5_IRS_IST_CFGR_LPI_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_CFGR_ISTSZ_4 0b00 +#define GICV5_IRS_IST_CFGR_ISTSZ_8 0b01 +#define GICV5_IRS_IST_CFGR_ISTSZ_16 0b10 + /* * Definitions for GICv5 instructions for the Current Domain */ @@ -77,6 +177,11 @@ #define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29) #define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE GENMASK_ULL(31, 29) +#define GICV5_HWIRQ_ID GENMASK_ULL(23, 0) +#define GICV5_HWIRQ_TYPE_LPI 0x2 +#define GICV5_HWIRQ_TYPE_SPI 0x3 + #define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn) #define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 5a7a455b5387..e2618877619e 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -14,6 +14,7 @@ #include #include "linux/rbtree.h" #include +#include #include #include @@ -33,6 +34,8 @@ #define NSEC_PER_SEC 1000000000L +#define KVM_INVALID_MEMSLOT UINT_MAX + struct userspace_mem_region { struct kvm_userspace_memory_region2 region; struct sparsebit *unused_phy_pages; @@ -80,11 +83,45 @@ struct userspace_mem_regions { DECLARE_HASHTABLE(slot_hash, 9); }; +/* + * Memory region types are passed to various page allocators to communicate + * various properties and metadata related to the allocation. Note, the + * descriptions below described the primary usage of each type. Individual + * tests may allocate memory for other purposes. + * + * By default, all regions except TEST_EXTRA are mapped to memslot '0'. The + * TEST_EXTRA region is left unmapped as it's intended to be used only for test + * specific allocations, i.e. should never be used by core/infrastructure code. + * Tests can override the memslot for any or all types, e.g. so that all test + * data is allocated from a curated memslot. + */ enum kvm_mem_region_type { + /* + * The CODE region is used by lib/elf when loading the test's code into + * guest memory. + */ MEM_REGION_CODE, + /* + * The DATA region is used to allocate core data structures, e.g. vCPU + * stacks, VM exception tables, x86's TSS, etc. + */ MEM_REGION_DATA, + /* + * The PT region, a.k.a. Page Table region, is used to allocate page + * table pages. + */ MEM_REGION_PT, + /* + * The TEST_DATA region is used for allocating test data that is either + * test specific, and/or isn't considered a "core" data structure. + */ MEM_REGION_TEST_DATA, + /* + * The TEST_EXTRA region is for special snowflakes, where a test wants + * to create and use a one-off memslot, without impacting "normal" test + * data allocations. + */ + MEM_REGION_TEST_EXTRA, NR_MEM_REGIONS, }; @@ -127,11 +164,6 @@ struct kvm_vm { struct kvm_binary_stats stats; - /* - * KVM region slots. These are the default memslots used by page - * allocators, e.g., lib/elf uses the memslots[MEM_REGION_CODE] - * memslot. - */ u32 memslots[NR_MEM_REGIONS]; }; @@ -169,7 +201,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot); static inline struct userspace_mem_region *vm_get_mem_region(struct kvm_vm *vm, enum kvm_mem_region_type type) { - assert(type < NR_MEM_REGIONS); + TEST_ASSERT(type < NR_MEM_REGIONS, + "Invalid memory region type '%u'", type); return memslot2region(vm, vm->memslots[type]); } @@ -209,6 +242,9 @@ enum vm_guest_mode { VM_MODE_P41V48_4K, VM_MODE_P41V39_4K, + VM_MODE_P52V52_4K, /* For powerpc64 */ + VM_MODE_P52V52_64K, + NUM_VM_MODES, }; @@ -268,6 +304,12 @@ extern enum vm_guest_mode vm_mode_default; #define MIN_PAGE_SHIFT 12U #define ptes_per_page(page_size) ((page_size) / 8) +#elif defined(__powerpc64__) + +#define VM_MODE_DEFAULT vm_mode_default +#define MIN_PAGE_SHIFT 12U +#define ptes_per_page(page_size) ((page_size) / 8) + #endif #define VM_SHAPE_DEFAULT VM_SHAPE(VM_MODE_DEFAULT) @@ -392,6 +434,16 @@ static __always_inline void static_assert_is_vcpu(struct kvm_vcpu *vcpu) { } __TEST_ASSERT_VM_VCPU_IOCTL(!ret, #cmd, ret, (vcpu)->vm); \ }) +#define __gmem_ioctl(gmem_fd, cmd, arg) \ + kvm_do_ioctl(gmem_fd, cmd, arg) + +#define gmem_ioctl(gmem_fd, cmd, arg) \ +({ \ + int ret = __gmem_ioctl(gmem_fd, cmd, arg); \ + \ + TEST_ASSERT(!ret, __KVM_IOCTL_ERROR(#cmd, ret)); \ +}) + /* * Looks up and returns the value corresponding to the capability * (KVM_CAP_*) given by cap. @@ -418,8 +470,19 @@ static inline void vm_enable_cap(struct kvm_vm *vm, u32 cap, u64 arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes); + +/* + * KVM_SET_MEMORY_ATTRIBUTES{,2} overwrites _all_ attributes. These + * flows need significant enhancements to support multiple attributes. + */ +#define TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes) \ + TEST_ASSERT(!(attributes) || (attributes) == KVM_MEMORY_ATTRIBUTE_PRIVATE, \ + "Update me to support multiple attributes!") + static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, - u64 size, u64 attributes) + size_t size, u64 attributes) { struct kvm_memory_attributes attr = { .attributes = attributes, @@ -428,27 +491,109 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, .flags = 0, }; - /* - * KVM_SET_MEMORY_ATTRIBUTES overwrites _all_ attributes. These flows - * need significant enhancements to support multiple attributes. - */ - TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, - "Update me to support multiple attributes!"); + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); } +static inline int __gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes, + u64 *error_offset) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + .error_offset = 0, + }; + int r; + + r = __gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); + + /* Copy error_offset regardless of r so caller can check. */ + if (error_offset) + *error_offset = attr.error_offset; + + return r; +} + +static inline int __gmem_set_private(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE, + error_offset); +} + +static inline int __gmem_set_shared(int fd, u64 offset, size_t size, + u64 *error_offset) +{ + return __gmem_set_memory_attributes(fd, offset, size, 0, + error_offset); +} + +static inline void gmem_set_memory_attributes(int fd, u64 offset, + size_t size, u64 attributes) +{ + struct kvm_memory_attributes2 attr = { + .attributes = attributes, + .offset = offset, + .size = size, + .flags = 0, + }; + + TEST_ASSERT_SUPPORTED_ATTRIBUTES(attributes); + + __TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) > 0, + "No valid attributes for guest_memfd ioctl!"); + + gmem_ioctl(fd, KVM_SET_MEMORY_ATTRIBUTES2, &attr); +} + +static inline void gmem_set_private(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); +} + +static inline void gmem_set_shared(int fd, u64 offset, size_t size) +{ + gmem_set_memory_attributes(fd, offset, size, 0); +} + +static inline void vm_mem_set_memory_attributes(struct kvm_vm *vm, gpa_t gpa, + size_t size, u64 attrs) +{ + if (kvm_has_gmem_attributes) { + gpa_t end = gpa + size; + off_t fd_offset; + gpa_t addr; + size_t len; + int fd; + + for (addr = gpa; addr < end; addr += len) { + fd = kvm_gpa_to_guest_memfd(vm, addr, &fd_offset, &len); + len = min(end - addr, len); + + gmem_set_memory_attributes(fd, fd_offset, len, attrs); + } + } else { + vm_set_memory_attributes(vm, gpa, size, attrs); + } +} static inline void vm_mem_set_private(struct kvm_vm *vm, gpa_t gpa, - u64 size) + size_t size) { - vm_set_memory_attributes(vm, gpa, size, KVM_MEMORY_ATTRIBUTE_PRIVATE); + vm_mem_set_memory_attributes(vm, gpa, size, + KVM_MEMORY_ATTRIBUTE_PRIVATE); } static inline void vm_mem_set_shared(struct kvm_vm *vm, gpa_t gpa, - u64 size) + size_t size) { - vm_set_memory_attributes(vm, gpa, size, 0); + vm_mem_set_memory_attributes(vm, gpa, size, 0); } void vm_guest_mem_fallocate(struct kvm_vm *vm, gpa_t gpa, u64 size, @@ -690,17 +835,47 @@ int __vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva); void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset); + u32 gmem_fd, u64 gmem_offset); void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags); void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd_fd, u64 guest_memfd_offset); + int gmem_fd, u64 gmem_offset, u64 gmem_flags); + + +static inline void ____vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + u32 slot) +{ + TEST_ASSERT(vm->memslots[type] == KVM_INVALID_MEMSLOT, + "Memory region type '%u' was already overridden with slot=%u", + type, vm->memslots[type]); + + vm->memslots[type] = slot; +} + +static inline void __vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + enum vm_mem_backing_src_type src_type, + gpa_t gpa, u32 slot, u64 npages, + u32 flags) +{ + ____vm_override_mem_region(vm, type, slot); + vm_userspace_mem_region_add(vm, src_type, gpa, slot, npages, flags); +} + +static inline void vm_override_mem_region(struct kvm_vm *vm, + enum kvm_mem_region_type type, + enum vm_mem_backing_src_type src_type, + gpa_t gpa, u32 slot, u64 npages) +{ + __vm_override_mem_region(vm, type, src_type, gpa, slot, npages, 0); +} #ifndef vm_arch_has_protected_memory static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) @@ -732,6 +907,11 @@ void *addr_gva2hva(struct kvm_vm *vm, gva_t gva); gpa_t addr_hva2gpa(struct kvm_vm *vm, void *hva); void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa); +static inline int kvm_slot_to_fd(struct kvm_vm *vm, u32 slot) +{ + return memslot2region(vm, slot)->fd; +} + #ifndef vcpu_arch_put_guest #define vcpu_arch_put_guest(mem, val) do { (mem) = (val); } while (0) #endif @@ -990,23 +1170,57 @@ void kvm_gsi_routing_write(struct kvm_vm *vm, struct kvm_irq_routing *routing); const char *exit_reason_str(unsigned int exit_reason); -gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot); -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, gpa_t min_gpa, - u32 memslot, bool protected); -gpa_t vm_alloc_page_table(struct kvm_vm *vm); +bool kvm_arch_needs_naturally_aligned_page_tables(void); -static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, - gpa_t min_gpa, u32 memslot) +/* + * True on powerpc when this host is a pseries LPAR (nested case). + * PowerPC's PAPR L0 does not guarantee H_GUEST_CREATE / H_GUEST_CREATE_VCPU up + * to MAX_VCPUS or creating hundreds of VMs - due to limited resouces at L0 for + * guest management space. This helper allows us to skip such tests when we are + * running nested. + */ +static inline bool kvm_is_ppc_pseries_lpar(void) +{ +#ifdef __powerpc64__ + return !access("/proc/device-tree/rtas/ibm,hypertas-functions", F_OK); +#else + return false; +#endif +} + +gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected, bool naturally_aligned); +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, + enum kvm_mem_region_type type, bool protected); + +static inline gpa_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, + enum kvm_mem_region_type type) { /* * By default, allocate memory as protected for VMs that support * protected memory, as the majority of memory for such VMs is * protected, i.e. using shared memory is effectively opt-in. */ - return __vm_phy_pages_alloc(vm, num, min_gpa, memslot, + return __vm_phy_pages_alloc(vm, nr_pages, type, vm_arch_has_protected_memory(vm)); } +static inline gpa_t vm_phy_page_alloc(struct kvm_vm *vm, + enum kvm_mem_region_type type) +{ + return vm_phy_pages_alloc(vm, 1, type); +} + +static inline gpa_t vm_alloc_page_table_pages(struct kvm_vm *vm, size_t nr_pages) +{ + return vm_phy_pages_alloc(vm, nr_pages, MEM_REGION_PT); +} + +static inline gpa_t vm_alloc_page_table(struct kvm_vm *vm) +{ + return vm_alloc_page_table_pages(vm, 1); +} + /* * ____vm_create() does KVM_CREATE_VM and little else. __vm_create() also * loads the test binary into guest memory and creates an IRQ chip (x86 only). @@ -1123,6 +1337,9 @@ static inline int pin_self_to_any_cpu(void) return pin_task_to_any_cpu(pthread_self()); } +void pin_pages(void *vaddr, uint64_t size); +void unpin_pages(void); + void kvm_print_vcpu_pinning_help(void); void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[], int nr_vcpus); diff --git a/tools/testing/selftests/kvm/include/loongarch/processor.h b/tools/testing/selftests/kvm/include/loongarch/processor.h index 0e59daa8024e..8d9fabdc89e3 100644 --- a/tools/testing/selftests/kvm/include/loongarch/processor.h +++ b/tools/testing/selftests/kvm/include/loongarch/processor.h @@ -79,6 +79,7 @@ #define CPUCFG2_LASX BIT(7) #define CPUCFG2_LLFTP BIT(14) #define CPUCFG2_LLFTPREV GENMASK(17, 15) +#define CPUCFG2_PTW BIT(24) /* LoongArch Basic CSR registers */ #define LOONGARCH_CSR_CRMD 0x0 /* Current mode info */ @@ -128,6 +129,8 @@ #define LOONGARCH_CSR_PGD 0x1b #define LOONGARCH_CSR_PWCTL0 0x1c #define LOONGARCH_CSR_PWCTL1 0x1d +#define CSR_PWCTL1_PTW_SHIFT 24 +#define CSR_PWCTL1_PTW BIT_ULL(CSR_PWCTL1_PTW_SHIFT) #define LOONGARCH_CSR_STLBPGSIZE 0x1e #define LOONGARCH_CSR_CPUID 0x20 #define LOONGARCH_CSR_KS0 0x30 diff --git a/tools/testing/selftests/kvm/include/numaif.h b/tools/testing/selftests/kvm/include/numaif.h index 29572a6d789c..0945500c6322 100644 --- a/tools/testing/selftests/kvm/include/numaif.h +++ b/tools/testing/selftests/kvm/include/numaif.h @@ -6,6 +6,7 @@ #include +#include #include #include "kvm_syscalls.h" @@ -30,6 +31,48 @@ KVM_SYSCALL_DEFINE(mbind, 6, void *, addr, unsigned long, size, int, mode, const unsigned long *, nodemask, unsigned long, maxnode, unsigned int, flags); +/* + * Calculate the @maxnode param for the above syscalls given the mask that will + * be passed to the kernel, to account for a longstanding off-by-one bug in the + * kernel that isn't properly documented in the manpages. The manpages say + * that @maxnode is "the maximum node ID plus one", but the kernel's actual + * behavior is "the number of bits in the mask plus one", i.e. "the maximum + * node ID plus two". + */ +#define MAXNODE_FOR_MASK(mask) (BITS_PER_TYPE(mask) + 1) + +static inline int kvm_get_numa_memory_nodes(unsigned long *nodemask) +{ + int r; + + *nodemask = 0; + + r = get_mempolicy(NULL, nodemask, MAXNODE_FOR_MASK(*nodemask), 0, + MPOL_F_MEMS_ALLOWED); + TEST_ASSERT(!r || errno == ENOSYS || errno == EPERM, + "Unexpected get_mempolicy() failure"); + return __builtin_popcountl(*nodemask); +} + +/* + * Return the node ID of the next NUMA node in the mask, starting at @from+1. + * Guarantees a node is found, and that the found node is not @from. Pass -1 + * to find the first node in the mask. + */ +static inline int kvm_get_next_numa_node(unsigned long nodemask, int from) +{ + const unsigned long nr_bits = BITS_PER_TYPE(nodemask); + int to; + + to = find_next_bit(&nodemask, nr_bits, from + 1); + if (to == nr_bits) + to = find_next_bit(&nodemask, nr_bits, 0); + + TEST_ASSERT(to != nr_bits && to != from, + "Unabled to find second NUMA node (from = %d, to = %d)", from, to); + return to; +} + static inline int get_max_numa_node(void) { struct dirent *de; diff --git a/tools/testing/selftests/kvm/include/powerpc/hcall.h b/tools/testing/selftests/kvm/include/powerpc/hcall.h new file mode 100644 index 000000000000..21b2bf9180f1 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/hcall.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc hcall defines + */ +#ifndef SELFTEST_KVM_HCALL_H +#define SELFTEST_KVM_HCALL_H + +#include + +/* Ucalls use unimplemented PAPR hcall 0 which exits KVM */ +#define H_UCALL 0 + +s64 hcall0(u64 token); +s64 hcall1(u64 token, u64 arg1); +s64 hcall2(u64 token, u64 arg1, u64 arg2); + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h b/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h new file mode 100644 index 000000000000..5d45c25cd299 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/kvm_util_arch.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_UTIL_ARCH_H +#define SELFTEST_KVM_UTIL_ARCH_H + +#include + +#include "kvm_util_types.h" + +struct kvm_mmu_arch {}; + +/* Page table fragment cache for guest page tables < page size */ +struct vm_pt_frag_cache { + gpa_t page; + size_t page_nr_used; +}; + +struct kvm_vm_arch { + gpa_t prtb; /* process table */ + struct vm_pt_frag_cache pt_frag_cache[2]; /* 256B and 4KB PT caches */ +}; + +#endif /* SELFTEST_KVM_UTIL_ARCH_H */ diff --git a/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h b/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h new file mode 100644 index 000000000000..b9df64659792 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/ppc_asm.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc asm specific defines + */ +#ifndef SELFTEST_KVM_PPC_ASM_H +#define SELFTEST_KVM_PPC_ASM_H + +#define STACK_FRAME_MIN_SIZE 112 /* Could be 32 on ELFv2 */ +#define STACK_REDZONE_SIZE 512 + +#define INT_FRAME_SIZE (STACK_FRAME_MIN_SIZE + STACK_REDZONE_SIZE) + +#define SPR_SRR0 0x01a +#define SPR_SRR1 0x01b +#define SPR_CFAR 0x01c + +#define MSR_SF 0x8000000000000000ULL +#define MSR_HV 0x1000000000000000ULL +#define MSR_VEC 0x0000000002000000ULL +#define MSR_VSX 0x0000000000800000ULL +#define MSR_EE 0x0000000000008000ULL +#define MSR_PR 0x0000000000004000ULL +#define MSR_FP 0x0000000000002000ULL +#define MSR_ME 0x0000000000001000ULL +#define MSR_IR 0x0000000000000020ULL +#define MSR_DR 0x0000000000000010ULL +#define MSR_RI 0x0000000000000002ULL +#define MSR_LE 0x0000000000000001ULL + +#define LPCR_ILE 0x0000000002000000ULL + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/processor.h b/tools/testing/selftests/kvm/include/powerpc/processor.h new file mode 100644 index 000000000000..f0ee21a8e2d4 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/processor.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * powerpc processor specific defines + */ +#ifndef SELFTEST_KVM_PROCESSOR_H +#define SELFTEST_KVM_PROCESSOR_H + +#include +#include "ppc_asm.h" +#include "kvm_util_types.h" + +extern unsigned char __interrupts_start[]; +extern unsigned char __interrupts_end[]; + +struct kvm_vm; +struct kvm_vcpu; + +struct ex_regs { + u64 gprs[32]; + u64 nia; + u64 msr; + u64 cfar; + u64 lr; + u64 ctr; + u64 xer; + u32 cr; + u32 trap; + gva_t gva; /* gva of this struct */ +}; + +void vm_install_exception_handler(struct kvm_vm *vm, int vector, + void (*handler)(struct ex_regs *)); + +static inline void cpu_relax(void) +{ + asm volatile("" ::: "memory"); +} + +#endif diff --git a/tools/testing/selftests/kvm/include/powerpc/ucall.h b/tools/testing/selftests/kvm/include/powerpc/ucall.h new file mode 100644 index 000000000000..e0dbe91e8848 --- /dev/null +++ b/tools/testing/selftests/kvm/include/powerpc/ucall.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_UCALL_H +#define SELFTEST_KVM_UCALL_H + +#include "hcall.h" + +#define UCALL_EXIT_REASON KVM_EXIT_PAPR_HCALL + +#define UCALL_R4_UCALL 0x5715 /* regular ucall, r5 contains ucall pointer */ +#define UCALL_R4_SIMPLE 0x0000 /* simple exit usable by asm with no ucall data */ + +static inline void ucall_arch_init(struct kvm_vm *vm, gpa_t mmio_gpa) +{ +} + +static inline void ucall_arch_do_ucall(gva_t uc) +{ + hcall2(H_UCALL, UCALL_R4_UCALL, (uintptr_t)(uc)); +} + +#endif diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a6a3e1657895..c80019e24b2d 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -49,10 +49,6 @@ do { \ #define TEST_REQUIRE(f) __TEST_REQUIRE(f, "Requirement not met: %s", #f) -ssize_t test_write(int fd, const void *buf, size_t count); -ssize_t test_read(int fd, void *buf, size_t count); -int test_seq_read(const char *path, char **bufp, size_t *sizep); - void __printf(5, 6) test_assert(bool exp, const char *exp_str, const char *file, unsigned int line, const char *fmt, ...); @@ -83,21 +79,23 @@ do { \ __builtin_unreachable(); \ } while (0) -extern sigjmp_buf expect_sigbus_jmpbuf; -void expect_sigbus_handler(int signum); +extern __thread sigjmp_buf expect_sigbus_jmpbuf; +extern __thread volatile sig_atomic_t expecting_sigbus; +void catchall_signal_handler(int signum); -#define TEST_EXPECT_SIGBUS(action) \ -do { \ - struct sigaction sa_old, sa_new = { \ - .sa_handler = expect_sigbus_handler, \ - }; \ - \ - sigaction(SIGBUS, &sa_new, &sa_old); \ - if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ - action; \ - TEST_FAIL("'%s' should have triggered SIGBUS", #action); \ - } \ - sigaction(SIGBUS, &sa_old, NULL); \ +#define TEST_EXPECT_SIGBUS(action) \ +do { \ + struct sigaction __sa = {}; \ + \ + TEST_ASSERT_EQ(sigaction(SIGBUS, NULL, &__sa), 0); \ + TEST_ASSERT_EQ(__sa.sa_handler, &catchall_signal_handler); \ + \ + expecting_sigbus = true; \ + if (sigsetjmp(expect_sigbus_jmpbuf, 1) == 0) { \ + action; \ + TEST_FAIL("'%s' should have triggered SIGBUS", #action);\ + } \ + expecting_sigbus = false; \ } while (0) size_t parse_size(const char *size); @@ -116,6 +114,8 @@ struct kvm_random_state { extern u32 kvm_random_seed; extern struct kvm_random_state kvm_rng; +extern bool kvm_has_gmem_attributes; + struct kvm_random_state new_kvm_random_state(u32 seed); u32 kvm_random_u32(struct kvm_random_state *state); @@ -172,6 +172,7 @@ struct vm_mem_backing_src_alias { bool thp_configured(void); size_t get_trans_hugepagesz(void); size_t get_def_hugetlb_pagesz(void); +size_t get_free_hugepages(size_t page_size); const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(u32 i); size_t get_backing_src_pagesz(u32 i); bool is_backing_src_hugetlb(u32 i); @@ -238,7 +239,6 @@ static inline u32 atoi_non_negative(const char *name, const char *num_str) } int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args); -__printf(3, 4) int guest_snprintf(char *buf, int n, const char *fmt, ...); char *strdup_printf(const char *fmt, ...) __attribute__((format(printf, 1, 2), nonnull(1))); diff --git a/tools/testing/selftests/kvm/include/x86/evmcs.h b/tools/testing/selftests/kvm/include/x86/evmcs.h index 82a8ea6b661f..491fb8107473 100644 --- a/tools/testing/selftests/kvm/include/x86/evmcs.h +++ b/tools/testing/selftests/kvm/include/x86/evmcs.h @@ -245,32 +245,18 @@ static inline void evmcs_enable(void) enable_evmcs = true; } -static inline int evmcs_vmptrld(u64 vmcs_pa, void *vmcs) +static inline void evmcs_vmptrld(u64 vmcs_pa, void *vmcs) { current_vp_assist->current_nested_vmcs = vmcs_pa; current_vp_assist->enlighten_vmentry = 1; current_evmcs = vmcs; - - return 0; } -static inline bool load_evmcs(struct hyperv_test_pages *hv) +static inline void load_evmcs(struct hyperv_test_pages *hv) { - if (evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs)) - return false; - + evmcs_vmptrld(hv->enlightened_vmcs_gpa, hv->enlightened_vmcs); current_evmcs->revision_id = EVMCS_VERSION; - - return true; -} - -static inline int evmcs_vmptrst(u64 *value) -{ - *value = current_vp_assist->current_nested_vmcs & - ~HV_X64_MSR_VP_ASSIST_PAGE_ENABLE; - - return 0; } static inline int evmcs_vmread(u64 encoding, u64 *value) @@ -1201,62 +1187,4 @@ static inline int evmcs_vmwrite(u64 encoding, u64 value) return 0; } -static inline int evmcs_vmlaunch(void) -{ - int ret; - - current_evmcs->hv_clean_fields = 0; - - __asm__ __volatile__("push $0;" - "mov %%rsp, (%[host_rsp]);" - "lea 1f(%%rip), %%rax;" - "mov %%rax, (%[host_rip]);" - VMX_SWITCH_GPRS_ASM - "vmlaunch;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r" - ((u64)¤t_evmcs->host_rsp), - [host_rip]"r" - ((u64)¤t_evmcs->host_rip), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; -} - -/* - * No guest state (e.g. GPRs) is established by this vmresume. - */ -static inline int evmcs_vmresume(void) -{ - int ret; - - /* HOST_RIP */ - current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1; - /* HOST_RSP */ - current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER; - - __asm__ __volatile__("push $0;" - "mov %%rsp, (%[host_rsp]);" - "lea 1f(%%rip), %%rax;" - "mov %%rax, (%[host_rip]);" - VMX_SWITCH_GPRS_ASM - "vmresume;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r" - ((u64)¤t_evmcs->host_rsp), - [host_rip]"r" - ((u64)¤t_evmcs->host_rip), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; -} - #endif /* !SELFTEST_KVM_EVMCS_H */ diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 6e6f70035508..5898a277a2e0 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -23,6 +23,8 @@ extern bool host_cpu_is_intel; extern bool host_cpu_is_amd; extern bool host_cpu_is_hygon; extern bool host_cpu_is_amd_compatible; +extern bool host_cpu_is_zhaoxin; +extern bool host_cpu_is_intel_compatible; extern u64 guest_tsc_khz; extern struct kvm_mmu guest_mmu; @@ -216,6 +218,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_INVTSC KVM_X86_CPU_FEATURE(0x80000007, 0, EDX, 8) #define X86_FEATURE_RDPRU KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 4) #define X86_FEATURE_AMD_IBPB KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 12) +#define X86_FEATURE_EFER_LMSLE_MBZ KVM_X86_CPU_FEATURE(0x80000008, 0, EBX, 20) #define X86_FEATURE_NPT KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 0) #define X86_FEATURE_LBRV KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 1) #define X86_FEATURE_NRIPS KVM_X86_CPU_FEATURE(0x8000000A, 0, EDX, 3) @@ -782,6 +785,12 @@ static inline bool this_cpu_is_hygon(void) return this_cpu_vendor_string_is("HygonGenuine"); } +static inline bool this_cpu_is_zhaoxin(void) +{ + return this_cpu_vendor_string_is("CentaurHauls") || + this_cpu_vendor_string_is(" Shanghai "); +} + static inline u32 __this_cpu_has(u32 function, u32 index, u8 reg, u8 lo, u8 hi) { u32 gprs[4]; @@ -1172,7 +1181,6 @@ static inline void vcpu_set_cpuid(struct kvm_vcpu *vcpu) void vcpu_set_cpuid_property(struct kvm_vcpu *vcpu, struct kvm_x86_cpu_property property, u32 value); -void vcpu_set_cpuid_maxphyaddr(struct kvm_vcpu *vcpu, u8 maxphyaddr); void vcpu_clear_cpuid_entry(struct kvm_vcpu *vcpu, u32 function); @@ -1318,7 +1326,7 @@ gva_t vm_alloc_stack(struct kvm_vm *vm, int nr_pages); "lea 1f(%%rip), %%r10\n\t" \ "lea 2f(%%rip), %%r11\n\t" \ fep "1: " insn "\n\t" \ - "xor %%r9, %%r9\n\t" \ + "mov $0, %%r9d\n\t" \ "2:\n\t" \ "mov %%r9b, %[vector]\n\t" \ "mov %%r10, %[error_code]\n\t" @@ -1446,7 +1454,7 @@ static inline bool kvm_is_pmu_enabled(void) static inline bool kvm_is_mediated_pmu_enabled(void) { - if (host_cpu_is_intel) + if (host_cpu_is_intel_compatible) return get_kvm_intel_param_bool("enable_mediated_pmu"); return get_kvm_amd_param_bool("enable_mediated_pmu"); @@ -1472,6 +1480,19 @@ static inline bool kvm_is_lbrv_enabled(void) return !!get_kvm_amd_param_integer("lbrv"); } +/* + * Do NOT use this to check for nVMX or nSVM support. Querying kvm_cpu_has() + * for either of X86_FEATURE_{VMX,SVM} is the idiomatic way to check for nested + * virtualization support. Use this *only* to validate KVM's own enumeration. + */ +static inline bool kvm_is_nested_virtualization_enabled(void) +{ + if (host_cpu_is_intel) + return get_kvm_intel_param_bool("nested"); + + return get_kvm_amd_param_integer("nested"); +} + u64 *vm_get_pte(struct kvm_vm *vm, gva_t gva); u64 kvm_hypercall(u64 nr, u64 a0, u64 a1, u64 a2, u64 a3); diff --git a/tools/testing/selftests/kvm/include/x86/smm.h b/tools/testing/selftests/kvm/include/x86/smm.h index 2d1afa09819b..15faaa060126 100644 --- a/tools/testing/selftests/kvm/include/x86/smm.h +++ b/tools/testing/selftests/kvm/include/x86/smm.h @@ -8,7 +8,7 @@ #define SMRAM_MEMSLOT ((1 << 16) | 1) #define SMRAM_PAGES (SMRAM_SIZE / PAGE_SIZE) -void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa, +void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, const void *smi_handler, size_t handler_size); void inject_smi(struct kvm_vcpu *vcpu); diff --git a/tools/testing/selftests/kvm/include/x86/vmx.h b/tools/testing/selftests/kvm/include/x86/vmx.h index 04f5e34dea3a..8fd97434eee5 100644 --- a/tools/testing/selftests/kvm/include/x86/vmx.h +++ b/tools/testing/selftests/kvm/include/x86/vmx.h @@ -309,125 +309,67 @@ struct vmx_msr_entry { #include "evmcs.h" -static inline int vmxon(u64 phys) -{ - u8 ret; +#define GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, __r, __pa) \ + __GUEST_ASSERT(!__r, __stringify(insn) "[0x%lx] hit %s", \ + __pa, __r < 0 ? "VM-Fail" : ex_str(__r)) - __asm__ __volatile__ ("vmxon %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(phys) - : "cc", "memory"); - - return ret; +#define BUILD_VMCS_ASM_HELPERS(insn) \ +static inline int __##insn(u64 vmcs_pa) \ +{ \ + u64 error_code; \ + u8 vector; \ + u8 failed; \ + \ + asm volatile(KVM_ASM_SAFE(__stringify(insn) " %[pa]") \ + "\n\tsetna %[failed]" \ + : KVM_ASM_SAFE_OUTPUTS(vector, error_code), \ + [failed]"=qm"(failed) \ + : [pa]"m"(vmcs_pa) \ + : "cc", "memory", KVM_ASM_SAFE_CLOBBERS); \ + \ + return vector ? vector : failed ? -EINVAL : 0; \ +} \ + \ +static inline void insn(u64 vmcs_pa) \ +{ \ + int ret = __##insn(vmcs_pa); \ + \ + GUEST_ASSERT_VMX_INSN_SUCCEEDED(insn, ret, vmcs_pa); \ } +BUILD_VMCS_ASM_HELPERS(vmxon) +BUILD_VMCS_ASM_HELPERS(vmptrld) +BUILD_VMCS_ASM_HELPERS(vmclear) + static inline void vmxoff(void) { __asm__ __volatile__("vmxoff"); } -static inline int vmclear(u64 vmcs_pa) -{ - u8 ret; - - __asm__ __volatile__ ("vmclear %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(vmcs_pa) - : "cc", "memory"); - - return ret; -} - -static inline int vmptrld(u64 vmcs_pa) -{ - u8 ret; - - if (enable_evmcs) - return -1; - - __asm__ __volatile__ ("vmptrld %[pa]; setna %[ret]" - : [ret]"=rm"(ret) - : [pa]"m"(vmcs_pa) - : "cc", "memory"); - - return ret; -} - -static inline int vmptrst(u64 *value) -{ - u64 tmp; - u8 ret; - - if (enable_evmcs) - return evmcs_vmptrst(value); - - __asm__ __volatile__("vmptrst %[value]; setna %[ret]" - : [value]"=m"(tmp), [ret]"=rm"(ret) - : : "cc", "memory"); - - *value = tmp; - return ret; -} - -/* - * A wrapper around vmptrst that ignores errors and returns zero if the - * vmptrst instruction fails. - */ -static inline u64 vmptrstz(void) +static inline u64 vmptrst(void) { u64 value = 0; - vmptrst(&value); + u8 ret; + + __asm__ __volatile__("vmptrst %[value]; setna %[ret]" + : [value]"=m"(value), [ret]"=rm"(ret) + : : "cc", "memory"); + + __GUEST_ASSERT(!ret, "vmptrst failed"); return value; } -static inline int vmlaunch(void) +int __vmlaunch(void); +int __vmresume(void); + +static inline void vmlaunch(void) { - int ret; - - if (enable_evmcs) - return evmcs_vmlaunch(); - - __asm__ __volatile__("push $0;" - "vmwrite %%rsp, %[host_rsp];" - "lea 1f(%%rip), %%rax;" - "vmwrite %%rax, %[host_rip];" - VMX_SWITCH_GPRS_ASM - "vmlaunch;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r"((u64)HOST_RSP), - [host_rip]"r"((u64)HOST_RIP), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; + __GUEST_ASSERT(!__vmlaunch(), "vmlaunch hit VM-Fail"); } -static inline int vmresume(void) +static inline void vmresume(void) { - int ret; - - if (enable_evmcs) - return evmcs_vmresume(); - - __asm__ __volatile__("push $0;" - "vmwrite %%rsp, %[host_rsp];" - "lea 1f(%%rip), %%rax;" - "vmwrite %%rax, %[host_rip];" - VMX_SWITCH_GPRS_ASM - "vmresume;" - "incq (%%rsp);" - "1: ;" - VMX_SWITCH_GPRS_ASM - "pop %%rax;" - : [ret]"=&a"(ret) - : [host_rsp]"r"((u64)HOST_RSP), - [host_rip]"r"((u64)HOST_RIP), - GUEST_REGS_OFFSETS - : "memory", "cc"); - return ret; + __GUEST_ASSERT(!__vmresume(), "vmresume hit VM-Fail"); } static inline void vmcall(void) @@ -444,7 +386,7 @@ static inline void vmcall(void) "r10", "r11", "r12", "r13", "r14", "r15"); } -static inline int vmread(u64 encoding, u64 *value) +static inline int __vmread(u64 encoding, u64 *value) { u64 tmp; u8 ret; @@ -457,22 +399,21 @@ static inline int vmread(u64 encoding, u64 *value) : [encoding]"r"(encoding) : "cc", "memory"); - *value = tmp; + if (!ret) + *value = tmp; return ret; } -/* - * A wrapper around vmread that ignores errors and returns zero if the - * vmread instruction fails. - */ -static inline u64 vmreadz(u64 encoding) +static inline u64 vmread(u64 encoding) { u64 value = 0; - vmread(encoding, &value); + + __GUEST_ASSERT(!__vmread(encoding, &value), + "vmread[0x%lx] hit VM-Fail", encoding); return value; } -static inline int vmwrite(u64 encoding, u64 value) +static __always_inline int __vmwrite(u64 encoding, u64 value) { u8 ret; @@ -487,6 +428,12 @@ static inline int vmwrite(u64 encoding, u64 value) return ret; } +static inline void vmwrite(u64 encoding, u64 value) +{ + __GUEST_ASSERT(!__vmwrite(encoding, value), + "vmwrite[0x%lx] = 0x%lx hit VM-Fail", encoding, value); +} + static inline u32 vmcs_revision(void) { return rdmsr(MSR_IA32_VMX_BASIC); @@ -550,13 +497,24 @@ union vmx_ctrl_msr { }; struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva); -bool prepare_for_vmx_operation(struct vmx_pages *vmx); +void prepare_for_vmx_operation(struct vmx_pages *vmx); void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip); -bool load_vmcs(struct vmx_pages *vmx); +void load_vmcs(struct vmx_pages *vmx); bool ept_1g_pages_supported(void); -bool kvm_cpu_has_ept(void); +bool kvm_cpu_has_secondary_exec_control(u32 ctrl); + +static inline bool kvm_cpu_has_ept(void) +{ + return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_ENABLE_EPT); +} + +static inline bool kvm_cpu_has_vmx_virtualize_apic_accesses(void) +{ + return kvm_cpu_has_secondary_exec_control(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES); +} + void vm_enable_ept(struct kvm_vm *vm); void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm); diff --git a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c index c5310736ed06..22279c638d06 100644 --- a/tools/testing/selftests/kvm/kvm_create_max_vcpus.c +++ b/tools/testing/selftests/kvm/kvm_create_max_vcpus.c @@ -39,6 +39,12 @@ int main(int argc, char *argv[]) int kvm_max_vcpu_id = kvm_check_cap(KVM_CAP_MAX_VCPU_ID); int kvm_max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); + /* + * PowerPC's PAPR L0 does not guarantee KVM_CAP_MAX_VCPUS when running + * on pseries LPAR (nested kvm-on-powervm). + */ + TEST_REQUIRE(!kvm_is_ppc_pseries_lpar()); + pr_info("KVM_CAP_MAX_VCPU_ID: %d\n", kvm_max_vcpu_id); pr_info("KVM_CAP_MAX_VCPUS: %d\n", kvm_max_vcpus); @@ -58,7 +64,13 @@ int main(int argc, char *argv[]) test_vcpu_creation(0, kvm_max_vcpus); - if (kvm_max_vcpu_id > kvm_max_vcpus) + /* + * Skip the vCPU ID test when running on PowerPC with SMT support, in + * which case system topology is encoded into the vCPU ID and so a VM + * can't use the full advertised vCPU ID range without crafting a valid + * platform specific topology. + */ + if (kvm_max_vcpu_id > kvm_max_vcpus && !kvm_check_cap(KVM_CAP_PPC_SMT)) test_vcpu_creation( kvm_max_vcpu_id - kvm_max_vcpus, kvm_max_vcpus); diff --git a/tools/testing/selftests/kvm/lib/arm64/processor.c b/tools/testing/selftests/kvm/lib/arm64/processor.c index 01325bf4d36f..2dcaa2c4a038 100644 --- a/tools/testing/selftests/kvm/lib/arm64/processor.c +++ b/tools/testing/selftests/kvm/lib/arm64/processor.c @@ -115,9 +115,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - vm->mmu.pgd = vm_phy_pages_alloc(vm, nr_pages, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + vm->mmu.pgd = vm_alloc_page_table_pages(vm, nr_pages); vm->mmu.pgd_created = true; } diff --git a/tools/testing/selftests/kvm/lib/elf.c b/tools/testing/selftests/kvm/lib/elf.c index 1924a9895834..d5adb68fd7c0 100644 --- a/tools/testing/selftests/kvm/lib/elf.c +++ b/tools/testing/selftests/kvm/lib/elf.c @@ -12,6 +12,44 @@ #include "kvm_util.h" +static ssize_t elf_read(int fd, void *buf, size_t count) +{ + ssize_t rc; + ssize_t num_read = 0; + size_t num_left = count; + char *ptr = buf; + + TEST_ASSERT(count, "Count must be non-zero"); + + do { + rc = read(fd, ptr, num_left); + + switch (rc) { + case -1: + TEST_ASSERT(errno == EAGAIN || errno == EINTR, + "Unexpected read failure,\n" + " rc: %zi errno: %i", rc, errno); + break; + + case 0: + TEST_FAIL("Unexpected EOF,\n" + " rc: %zi num_read: %zi num_left: %zu", + rc, num_read, num_left); + break; + + default: + TEST_ASSERT(rc > 0, "Unexpected ret from read,\n" + " rc: %zi errno: %i", rc, errno); + num_read += rc; + num_left -= rc; + ptr += rc; + break; + } + } while (num_read < count); + + return num_read; +} + static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) { off_t offset_rv; @@ -31,7 +69,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) * the real size of the ELF header. */ unsigned char ident[EI_NIDENT]; - test_read(fd, ident, sizeof(ident)); + elf_read(fd, ident, sizeof(ident)); TEST_ASSERT((ident[EI_MAG0] == ELFMAG0) && (ident[EI_MAG1] == ELFMAG1) && (ident[EI_MAG2] == ELFMAG2) && (ident[EI_MAG3] == ELFMAG3), "ELF MAGIC Mismatch,\n" @@ -79,7 +117,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) offset_rv = lseek(fd, 0, SEEK_SET); TEST_ASSERT(offset_rv == 0, "Seek to ELF header failed,\n" " rv: %zi expected: %i", offset_rv, 0); - test_read(fd, hdrp, sizeof(*hdrp)); + elf_read(fd, hdrp, sizeof(*hdrp)); TEST_ASSERT(hdrp->e_phentsize == sizeof(Elf64_Phdr), "Unexpected physical header size,\n" " hdrp->e_phentsize: %x\n" @@ -146,7 +184,7 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) /* Read in the program header. */ Elf64_Phdr phdr; - test_read(fd, &phdr, sizeof(phdr)); + elf_read(fd, &phdr, sizeof(phdr)); /* Skip if this header doesn't describe a loadable segment. */ if (phdr.p_type != PT_LOAD) @@ -186,8 +224,8 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) " expected: 0x%jx", n1, errno, (intmax_t) offset_rv, (intmax_t) phdr.p_offset); - test_read(fd, addr_gva2hva(vm, phdr.p_vaddr), - phdr.p_filesz); + elf_read(fd, addr_gva2hva(vm, phdr.p_vaddr), + phdr.p_filesz); } } close(fd); diff --git a/tools/testing/selftests/kvm/lib/guest_modes.c b/tools/testing/selftests/kvm/lib/guest_modes.c index 7a96c43b5704..439766fad693 100644 --- a/tools/testing/selftests/kvm/lib/guest_modes.c +++ b/tools/testing/selftests/kvm/lib/guest_modes.c @@ -4,16 +4,20 @@ */ #include "guest_modes.h" -#if defined(__aarch64__) || defined(__riscv) +#if defined(__aarch64__) || defined(__riscv) || defined(__powerpc64__) #include "processor.h" enum vm_guest_mode vm_mode_default; #endif +#if defined(__powerpc64__) +#include +#endif + struct guest_mode guest_modes[NUM_VM_MODES]; void guest_modes_append_default(void) { -#if !defined(__aarch64__) && !defined(__riscv) +#if !defined(__aarch64__) && !defined(__riscv) && !defined(__powerpc64__) guest_mode_append(VM_MODE_DEFAULT, true); #endif @@ -108,6 +112,18 @@ void guest_modes_append_default(void) TEST_ASSERT(vm_mode_default != NUM_VM_MODES, "No supported mode!"); } #endif +#ifdef __powerpc64__ + { + TEST_REQUIRE(kvm_has_cap(KVM_CAP_PPC_MMU_RADIX)); + /* Radix guest EA and RA are 52-bit on POWER9 and POWER10 */ + if (sysconf(_SC_PAGESIZE) == 4096) + vm_mode_default = VM_MODE_P52V52_4K; + else + vm_mode_default = VM_MODE_P52V52_64K; + guest_mode_append(VM_MODE_P52V52_4K, true); + guest_mode_append(VM_MODE_P52V52_64K, true); + } +#endif } void for_each_guest_mode(void (*func)(enum vm_guest_mode, void *), void *arg) diff --git a/tools/testing/selftests/kvm/lib/guest_sprintf.c b/tools/testing/selftests/kvm/lib/guest_sprintf.c index 7a33965349a7..6caebb456f3d 100644 --- a/tools/testing/selftests/kvm/lib/guest_sprintf.c +++ b/tools/testing/selftests/kvm/lib/guest_sprintf.c @@ -300,15 +300,3 @@ int guest_vsnprintf(char *buf, int n, const char *fmt, va_list args) *str = '\0'; return str - buf; } - -int guest_snprintf(char *buf, int n, const char *fmt, ...) -{ - va_list va; - int len; - - va_start(va, fmt); - len = guest_vsnprintf(buf, n, fmt, va); - va_end(va); - - return len; -} diff --git a/tools/testing/selftests/kvm/lib/io.c b/tools/testing/selftests/kvm/lib/io.c deleted file mode 100644 index fedb2a741f0b..000000000000 --- a/tools/testing/selftests/kvm/lib/io.c +++ /dev/null @@ -1,157 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * tools/testing/selftests/kvm/lib/io.c - * - * Copyright (C) 2018, Google LLC. - */ - -#include "test_util.h" - -/* Test Write - * - * A wrapper for write(2), that automatically handles the following - * special conditions: - * - * + Interrupted system call (EINTR) - * + Write of less than requested amount - * + Non-block return (EAGAIN) - * - * For each of the above, an additional write is performed to automatically - * continue writing the requested data. - * There are also many cases where write(2) can return an unexpected - * error (e.g. EIO). Such errors cause a TEST_ASSERT failure. - * - * Note, for function signature compatibility with write(2), this function - * returns the number of bytes written, but that value will always be equal - * to the number of requested bytes. All other conditions in this and - * future enhancements to this function either automatically issue another - * write(2) or cause a TEST_ASSERT failure. - * - * Args: - * fd - Opened file descriptor to file to be written. - * count - Number of bytes to write. - * - * Output: - * buf - Starting address of data to be written. - * - * Return: - * On success, number of bytes written. - * On failure, a TEST_ASSERT failure is caused. - */ -ssize_t test_write(int fd, const void *buf, size_t count) -{ - ssize_t rc; - ssize_t num_written = 0; - size_t num_left = count; - const char *ptr = buf; - - /* Note: Count of zero is allowed (see "RETURN VALUE" portion of - * write(2) manpage for details. - */ - TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count); - - do { - rc = write(fd, ptr, num_left); - - switch (rc) { - case -1: - TEST_ASSERT(errno == EAGAIN || errno == EINTR, - "Unexpected write failure,\n" - " rc: %zi errno: %i", rc, errno); - continue; - - case 0: - TEST_FAIL("Unexpected EOF,\n" - " rc: %zi num_written: %zi num_left: %zu", - rc, num_written, num_left); - break; - - default: - TEST_ASSERT(rc >= 0, "Unexpected ret from write,\n" - " rc: %zi errno: %i", rc, errno); - num_written += rc; - num_left -= rc; - ptr += rc; - break; - } - } while (num_written < count); - - return num_written; -} - -/* Test Read - * - * A wrapper for read(2), that automatically handles the following - * special conditions: - * - * + Interrupted system call (EINTR) - * + Read of less than requested amount - * + Non-block return (EAGAIN) - * - * For each of the above, an additional read is performed to automatically - * continue reading the requested data. - * There are also many cases where read(2) can return an unexpected - * error (e.g. EIO). Such errors cause a TEST_ASSERT failure. Note, - * it is expected that the file opened by fd at the current file position - * contains at least the number of requested bytes to be read. A TEST_ASSERT - * failure is produced if an End-Of-File condition occurs, before all the - * data is read. It is the callers responsibility to assure that sufficient - * data exists. - * - * Note, for function signature compatibility with read(2), this function - * returns the number of bytes read, but that value will always be equal - * to the number of requested bytes. All other conditions in this and - * future enhancements to this function either automatically issue another - * read(2) or cause a TEST_ASSERT failure. - * - * Args: - * fd - Opened file descriptor to file to be read. - * count - Number of bytes to read. - * - * Output: - * buf - Starting address of where to write the bytes read. - * - * Return: - * On success, number of bytes read. - * On failure, a TEST_ASSERT failure is caused. - */ -ssize_t test_read(int fd, void *buf, size_t count) -{ - ssize_t rc; - ssize_t num_read = 0; - size_t num_left = count; - char *ptr = buf; - - /* Note: Count of zero is allowed (see "If count is zero" portion of - * read(2) manpage for details. - */ - TEST_ASSERT(count >= 0, "Unexpected count, count: %li", count); - - do { - rc = read(fd, ptr, num_left); - - switch (rc) { - case -1: - TEST_ASSERT(errno == EAGAIN || errno == EINTR, - "Unexpected read failure,\n" - " rc: %zi errno: %i", rc, errno); - break; - - case 0: - TEST_FAIL("Unexpected EOF,\n" - " rc: %zi num_read: %zi num_left: %zu", - rc, num_read, num_left); - break; - - default: - TEST_ASSERT(rc > 0, "Unexpected ret from read,\n" - " rc: %zi errno: %i", rc, errno); - num_read += rc; - num_left -= rc; - ptr += rc; - break; - } - } while (num_read < count); - - return num_read; -} diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9ddc047d5c27..d42e201eba76 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -17,8 +17,11 @@ #include #include #include +#include #include +#include "../../../../../mm/gup_test.h" + #define KVM_UTIL_MIN_PFN 2 u32 kvm_random_seed; @@ -32,6 +35,8 @@ static void kvm_seed_rng(u32 seed) kvm_rng = new_kvm_random_state(kvm_random_seed); } +bool kvm_has_gmem_attributes; + static size_t vcpu_mmap_sz(void); int __open_path_or_exit(const char *path, int flags, const char *enoent_help) @@ -226,6 +231,8 @@ const char *vm_guest_mode_string(u32 i) [VM_MODE_P41V57_4K] = "PA-bits:41, VA-bits:57, 4K pages", [VM_MODE_P41V48_4K] = "PA-bits:41, VA-bits:48, 4K pages", [VM_MODE_P41V39_4K] = "PA-bits:41, VA-bits:39, 4K pages", + [VM_MODE_P52V52_4K] = "PA-bits:52, VA-bits:52, 4K pages", + [VM_MODE_P52V52_64K] = "PA-bits:52, VA-bits:52, 64K pages", }; _Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES, "Missing new mode strings?"); @@ -262,6 +269,8 @@ const struct vm_guest_mode_params vm_guest_mode_params[] = { [VM_MODE_P41V57_4K] = { 41, 57, 0x1000, 12 }, [VM_MODE_P41V48_4K] = { 41, 48, 0x1000, 12 }, [VM_MODE_P41V39_4K] = { 41, 39, 0x1000, 12 }, + [VM_MODE_P52V52_4K] = { 52, 52, 0x1000, 12 }, + [VM_MODE_P52V52_64K] = { 52, 52, 0x10000, 16 }, }; _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES, "Missing new mode params?"); @@ -287,6 +296,7 @@ __weak void vm_populate_gva_bitmap(struct kvm_vm *vm) struct kvm_vm *____vm_create(struct vm_shape shape) { struct kvm_vm *vm; + int i; vm = calloc(1, sizeof(*vm)); TEST_ASSERT(vm != NULL, "Insufficient Memory"); @@ -295,6 +305,8 @@ struct kvm_vm *____vm_create(struct vm_shape shape) vm->regions.gpa_tree = RB_ROOT; vm->regions.hva_tree = RB_ROOT; hash_init(vm->regions.slot_hash); + for (i = 0; i < NR_MEM_REGIONS; i++) + vm->memslots[i] = KVM_INVALID_MEMSLOT; vm->mode = shape.mode; vm->type = shape.type; @@ -379,6 +391,10 @@ struct kvm_vm *____vm_create(struct vm_shape shape) case VM_MODE_P41V39_4K: vm->mmu.pgtable_levels = 3; break; + case VM_MODE_P52V52_4K: + case VM_MODE_P52V52_64K: + vm->mmu.pgtable_levels = 4; + break; default: TEST_FAIL("Unknown guest mode: 0x%x", vm->mode); } @@ -490,8 +506,9 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, u64 nr_pages = vm_nr_pages_required(shape.mode, nr_runnable_vcpus, nr_extra_pages); struct userspace_mem_region *slot0; + u64 gmem_flags = 0; struct kvm_vm *vm; - int i, flags; + int flags; kvm_set_files_rlimit(nr_runnable_vcpus); @@ -505,12 +522,24 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, * for CoCo VMs that require GUEST_MEMFD backed private memory. */ flags = 0; - if (is_guest_memfd_required(shape)) + if (is_guest_memfd_required(shape)) { flags |= KVM_MEM_GUEST_MEMFD; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags); - for (i = 0; i < NR_MEM_REGIONS; i++) - vm->memslots[i] = 0; + /* + * Allow mmap() and initialize the gmem instance as SHARED if + * in-place conversions are supported, so that the memory can + * be populated/initialized like any other memory region. + */ + if (kvm_has_gmem_attributes) + gmem_flags |= GUEST_MEMFD_FLAG_INIT_SHARED | + GUEST_MEMFD_FLAG_MMAP; + } + + vm_mem_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, nr_pages, flags, -1, 0, gmem_flags); + ____vm_override_mem_region(vm, MEM_REGION_CODE, 0); + ____vm_override_mem_region(vm, MEM_REGION_PT, 0); + ____vm_override_mem_region(vm, MEM_REGION_DATA, 0); + ____vm_override_mem_region(vm, MEM_REGION_TEST_DATA, 0); kvm_vm_elf_load(vm, program_invocation_name); @@ -527,6 +556,7 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, kvm_seed_rng(kvm_random_seed); sync_global_to_guest(vm, kvm_rng); + sync_global_to_guest(vm, kvm_has_gmem_attributes); kvm_arch_vm_post_create(vm, nr_runnable_vcpus); @@ -642,6 +672,29 @@ int __pin_task_to_cpu(pthread_t task, int cpu) return pthread_setaffinity_np(task, sizeof(cpuset), &cpuset); } +static int gup_test_fd = -1; + +void pin_pages(void *vaddr, uint64_t size) +{ + const struct pin_longterm_test args = { + .addr = (uint64_t)vaddr, + .size = size, + .flags = PIN_LONGTERM_TEST_FLAG_USE_WRITE, + }; + + gup_test_fd = __open_path_or_exit("/sys/kernel/debug/gup_test", O_RDWR, + "Is CONFIG_GUP_TEST enabled?"); + + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_START, &args), 0); +} + +void unpin_pages(void) +{ + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_STOP), 0); + + kvm_free_fd(gup_test_fd); +} + static u32 parse_pcpu(const char *cpu_str, const cpu_set_t *allowed_mask) { u32 pcpu = atoi_non_negative("CPU number", cpu_str); @@ -970,7 +1023,7 @@ void vm_set_user_memory_region(struct kvm_vm *vm, u32 slot, u32 flags, int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset) + u32 gmem_fd, u64 gmem_offset) { struct kvm_userspace_memory_region2 region = { .slot = slot, @@ -978,8 +1031,8 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, .guest_phys_addr = gpa, .memory_size = size, .userspace_addr = (uintptr_t)hva, - .guest_memfd = guest_memfd, - .guest_memfd_offset = guest_memfd_offset, + .guest_memfd = gmem_fd, + .guest_memfd_offset = gmem_offset, }; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -989,10 +1042,10 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, gpa_t gpa, u64 size, void *hva, - u32 guest_memfd, u64 guest_memfd_offset) + u32 gmem_fd, u64 gmem_offset) { int ret = __vm_set_user_memory_region2(vm, slot, flags, gpa, size, hva, - guest_memfd, guest_memfd_offset); + gmem_fd, gmem_offset); TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION2 failed, errno = %d (%s)", errno, strerror(errno)); @@ -1002,13 +1055,16 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, u32 slot, u32 flags, /* FIXME: This thing needs to be ripped apart and rewritten. */ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags, - int guest_memfd, u64 guest_memfd_offset) + int gmem_fd, u64 gmem_offset, u64 gmem_flags) { int ret; struct userspace_mem_region *region; - size_t backing_src_pagesz = get_backing_src_pagesz(src_type); size_t mem_size = npages * vm->page_size; - size_t alignment = 1; + size_t backing_src_pagesz; + off_t mmap_offset; + bool is_gmem_mmap; + size_t alignment; + int mmap_flags; TEST_REQUIRE_SET_USER_MEMORY_REGION2(); @@ -1060,58 +1116,40 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, /* Allocate and initialize new mem region structure. */ region = calloc(1, sizeof(*region)); TEST_ASSERT(region != NULL, "Insufficient Memory"); - region->mmap_size = mem_size; - /* - * When using THP mmap is not guaranteed to returned a hugepage aligned - * address so we have to pad the mmap. Padding is not needed for HugeTLB - * because mmap will always return an address aligned to the HugeTLB - * page size. - */ - if (src_type == VM_MEM_SRC_ANONYMOUS_THP) - alignment = max(backing_src_pagesz, alignment); + is_gmem_mmap = (flags & KVM_MEM_GUEST_MEMFD) && + (gmem_flags & GUEST_MEMFD_FLAG_MMAP); + + if (is_gmem_mmap) { + backing_src_pagesz = getpagesize(); + alignment = 1; + mmap_flags = MAP_SHARED; + mmap_offset = gmem_offset; + } else { + backing_src_pagesz = get_backing_src_pagesz(src_type); + /* + * When using THP mmap is not guaranteed to returned a hugepage aligned + * address so we have to pad the mmap. Padding is not needed for HugeTLB + * because mmap will always return an address aligned to the HugeTLB + * page size. + */ + alignment = src_type == VM_MEM_SRC_ANONYMOUS_THP ? backing_src_pagesz : 1; + mmap_flags = vm_mem_backing_src_alias(src_type)->flag; + mmap_offset = 0; + } TEST_ASSERT_EQ(gpa, align_up(gpa, backing_src_pagesz)); + region->mmap_size = mem_size; /* Add enough memory to align up if necessary */ if (alignment > 1) region->mmap_size += alignment; - region->fd = -1; - if (backing_src_is_shared(src_type)) - region->fd = kvm_memfd_alloc(region->mmap_size, - src_type == VM_MEM_SRC_SHARED_HUGETLB); - - region->mmap_start = kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd); - - TEST_ASSERT(!is_backing_src_hugetlb(src_type) || - region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz), - "mmap_start %p is not aligned to HugeTLB page size 0x%lx", - region->mmap_start, backing_src_pagesz); - - /* Align host address */ - region->host_mem = align_ptr_up(region->mmap_start, alignment); - - /* As needed perform madvise */ - if ((src_type == VM_MEM_SRC_ANONYMOUS || - src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { - ret = madvise(region->host_mem, mem_size, - src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); - TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", - region->host_mem, mem_size, - vm_mem_backing_src_alias(src_type)->name); - } - - region->backing_src_type = src_type; - if (flags & KVM_MEM_GUEST_MEMFD) { - if (guest_memfd < 0) { - u32 guest_memfd_flags = 0; - TEST_ASSERT(!guest_memfd_offset, + if (gmem_fd < 0) { + TEST_ASSERT(!gmem_offset, "Offset must be zero when creating new guest_memfd"); - guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags); + gmem_fd = vm_create_guest_memfd(vm, mem_size, gmem_flags); } else { /* * Install a unique fd for each memslot so that the fd @@ -1119,15 +1157,51 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, * needing to track if the fd is owned by the framework * or by the caller. */ - guest_memfd = kvm_dup(guest_memfd); + gmem_fd = kvm_dup(gmem_fd); } - region->region.guest_memfd = guest_memfd; - region->region.guest_memfd_offset = guest_memfd_offset; + region->region.guest_memfd = gmem_fd; + region->region.guest_memfd_offset = gmem_offset; } else { region->region.guest_memfd = -1; } + region->fd = -1; + if (is_gmem_mmap) { + region->fd = kvm_dup(gmem_fd); + } else if (backing_src_is_shared(src_type)) { + region->fd = kvm_memfd_alloc(region->mmap_size, + src_type == VM_MEM_SRC_SHARED_HUGETLB); + } + + region->mmap_start = __kvm_mmap(region->mmap_size, PROT_READ | PROT_WRITE, + mmap_flags, region->fd, mmap_offset); + + /* Align host address */ + region->host_mem = align_ptr_up(region->mmap_start, alignment); + + if (!is_gmem_mmap) { + TEST_ASSERT(!is_backing_src_hugetlb(src_type) || + region->mmap_start == + align_ptr_up(region->mmap_start, backing_src_pagesz), + "mmap_start %p is not aligned to HugeTLB page size 0x%lx", + region->mmap_start, backing_src_pagesz); + + /* As needed perform madvise */ + if ((src_type == VM_MEM_SRC_ANONYMOUS || + src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { + int advice = src_type == VM_MEM_SRC_ANONYMOUS ? + MADV_NOHUGEPAGE : MADV_HUGEPAGE; + + ret = madvise(region->host_mem, mem_size, advice); + TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", + region->host_mem, mem_size, + vm_mem_backing_src_alias(src_type)->name); + } + } + + region->backing_src_type = src_type; + region->unused_phy_pages = sparsebit_alloc(); if (vm_arch_has_protected_memory(vm)) region->protected_phy_pages = sparsebit_alloc(); @@ -1152,10 +1226,10 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, /* If shared memory, create an alias. */ if (region->fd >= 0) { - region->mmap_alias = kvm_mmap(region->mmap_size, - PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd); + region->mmap_alias = __kvm_mmap(region->mmap_size, + PROT_READ | PROT_WRITE, + mmap_flags, region->fd, + mmap_offset); /* Align host alias address */ region->host_alias = align_ptr_up(region->mmap_alias, alignment); @@ -1166,7 +1240,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, gpa_t gpa, u32 slot, u64 npages, u32 flags) { - vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0); + vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0, 0); } /* @@ -1189,6 +1263,8 @@ memslot2region(struct kvm_vm *vm, u32 memslot) { struct userspace_mem_region *region; + TEST_ASSERT(memslot != KVM_INVALID_MEMSLOT, "vm->memslots[] unpopulated?"); + hash_for_each_possible(vm->regions.slot_hash, region, slot_node, memslot) if (region->region.slot == memslot) @@ -1299,27 +1375,20 @@ void vm_guest_mem_fallocate(struct kvm_vm *vm, u64 base, u64 size, bool punch_hole) { const int mode = FALLOC_FL_KEEP_SIZE | (punch_hole ? FALLOC_FL_PUNCH_HOLE : 0); - struct userspace_mem_region *region; u64 end = base + size; - gpa_t gpa, len; off_t fd_offset; - int ret; + int fd, ret; + size_t len; + gpa_t gpa; for (gpa = base; gpa < end; gpa += len) { - u64 offset; + fd = kvm_gpa_to_guest_memfd(vm, gpa, &fd_offset, &len); + len = min(end - gpa, len); - region = userspace_mem_region_find(vm, gpa, gpa); - TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, - "Private memory region not found for GPA 0x%lx", gpa); - - offset = gpa - region->region.guest_phys_addr; - fd_offset = region->region.guest_memfd_offset + offset; - len = min_t(u64, end - gpa, region->region.memory_size - offset); - - ret = fallocate(region->region.guest_memfd, mode, fd_offset, len); + ret = fallocate(fd, mode, fd_offset, len); TEST_ASSERT(!ret, "fallocate() failed to %s at %lx (len = %lu), fd = %d, mode = %x, offset = %lx", punch_hole ? "punch hole" : "allocate", gpa, len, - region->region.guest_memfd, mode, fd_offset); + fd, mode, fd_offset); } } @@ -1465,9 +1534,7 @@ static gva_t ____vm_alloc(struct kvm_vm *vm, size_t sz, gva_t min_gva, u64 pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0); virt_pgd_alloc(vm); - gpa_t gpa = __vm_phy_pages_alloc(vm, pages, - KVM_UTIL_MIN_PFN * vm->page_size, - vm->memslots[type], protected); + gpa_t gpa = __vm_phy_pages_alloc(vm, pages, type, protected); /* * Find an unused range of virtual page addresses of at least @@ -1656,6 +1723,22 @@ void *addr_gpa2alias(struct kvm_vm *vm, gpa_t gpa) return (void *) ((uintptr_t) region->host_alias + offset); } +int kvm_gpa_to_guest_memfd(struct kvm_vm *vm, gpa_t gpa, off_t *fd_offset, + size_t *nr_bytes) +{ + struct userspace_mem_region *region; + gpa_t gpa_offset; + + region = userspace_mem_region_find(vm, gpa, gpa); + TEST_ASSERT(region && region->region.flags & KVM_MEM_GUEST_MEMFD, + "guest_memfd memory region not found for GPA 0x%lx", gpa); + + gpa_offset = gpa - region->region.guest_phys_addr; + *fd_offset = region->region.guest_memfd_offset + gpa_offset; + *nr_bytes = region->region.memory_size - gpa_offset; + return region->region.guest_memfd; +} + /* Create an interrupt controller chip for the specified VM. */ void vm_create_irqchip(struct kvm_vm *vm) { @@ -1727,6 +1810,7 @@ struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vcpu *vcpu) TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0"); reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64)); + TEST_ASSERT(reg_list, "Failed to allocate reg_list"); reg_list->n = reg_list_n.n; vcpu_ioctl(vcpu, KVM_GET_REG_LIST, reg_list); return reg_list; @@ -2025,33 +2109,21 @@ const char *exit_reason_str(unsigned int exit_reason) } /* - * Physical Contiguous Page Allocator + * Allocate contiguous (guest) physical pages in a given memory region, at or + * above the minimum specified GPA. If the memory is protected/private, also + * add the allocated pages to the region's set of protected pages, e.g. so that + * arch code knows which pages need to be encrypted when launching the VM. * - * Input Args: - * vm - Virtual Machine - * num - number of pages - * min_gpa - Physical address minimum - * memslot - Memory region to allocate page from - * protected - True if the pages will be used as protected/private memory - * - * Output Args: None - * - * Return: - * Starting physical address - * - * Within the VM specified by vm, locates a range of available physical - * pages at or above min_gpa. If found, the pages are marked as in use - * and their base address is returned. A TEST_ASSERT failure occurs if - * not enough pages are available at or above min_gpa. + * Note, success is guaranteed! */ -gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, - gpa_t min_gpa, u32 memslot, - bool protected) +gpa_t ____vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, gpa_t min_gpa, + u32 memslot, bool protected, bool naturally_aligned) { + size_t alignment = naturally_aligned ? nr_pages : 1; struct userspace_mem_region *region; sparsebit_idx_t pg, base; - TEST_ASSERT(num > 0, "Must allocate at least one page"); + TEST_ASSERT(nr_pages, "Must allocate at least one page"); TEST_ASSERT((min_gpa % vm->page_size) == 0, "Min physical address " "not divisible by page size.\n" @@ -2062,43 +2134,78 @@ gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, TEST_ASSERT(!protected || region->protected_phy_pages, "Region doesn't support protected memory"); - base = pg = min_gpa >> vm->page_shift; - do { - for (; pg < base + num; ++pg) { - if (!sparsebit_is_set(region->unused_phy_pages, pg)) { - base = pg = sparsebit_next_set(region->unused_phy_pages, pg); - break; - } + base = min_gpa >> vm->page_shift; +again: + base = ALIGN(base, alignment); + for (pg = base; pg < base + nr_pages; ++pg) { + if (!sparsebit_is_set(region->unused_phy_pages, pg)) { + base = sparsebit_next_set(region->unused_phy_pages, pg); + if (!base) + goto enomem; + goto again; } - } while (pg && pg != base + num); - - if (pg == 0) { - fprintf(stderr, "No guest physical page available, " - "min_gpa: 0x%lx page_size: 0x%x memslot: %u\n", - min_gpa, vm->page_size, memslot); - fputs("---- vm dump ----\n", stderr); - vm_dump(stderr, vm, 2); - abort(); } - for (pg = base; pg < base + num; ++pg) { + for (pg = base; pg < base + nr_pages; ++pg) { sparsebit_clear(region->unused_phy_pages, pg); if (protected) sparsebit_set(region->protected_phy_pages, pg); } return base * vm->page_size; + +enomem: + fprintf(stderr, "No guest physical page available, min_gpa: 0x%lx page_size: 0x%x memslot: %u\n", + min_gpa, vm->page_size, memslot); + fputs("---- vm dump ----\n", stderr); + vm_dump(stderr, vm, 2); + abort(); + __builtin_unreachable(); } -gpa_t vm_phy_page_alloc(struct kvm_vm *vm, gpa_t min_gpa, u32 memslot) +__weak bool kvm_arch_needs_naturally_aligned_page_tables(void) { - return vm_phy_pages_alloc(vm, 1, min_gpa, memslot); + return false; } -gpa_t vm_alloc_page_table(struct kvm_vm *vm) +gpa_t __vm_phy_pages_alloc(struct kvm_vm *vm, size_t nr_pages, + enum kvm_mem_region_type type, bool protected) { - return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + struct userspace_mem_region *region = vm_get_mem_region(vm, type); + bool naturally_aligned = false; + gpa_t min_gpa; + + TEST_ASSERT(region, "No region for type '%u', memslot '%u'", + type, vm->memslots[type]); + + switch (type) { + case MEM_REGION_CODE: + case MEM_REGION_DATA: + case MEM_REGION_TEST_DATA: + /* + * If the region is backed by the default memslot (id=0), use + * selftests' hardcoded minimum PFN, otherwise use the base of + * the custom memory slot that backs the region. + */ + if (!vm->memslots[type]) + min_gpa = KVM_UTIL_MIN_PFN * vm->page_size; + else + min_gpa = region->region.guest_phys_addr; + break; + case MEM_REGION_PT: + min_gpa = KVM_GUEST_PAGE_TABLE_MIN_PADDR; + naturally_aligned = kvm_arch_needs_naturally_aligned_page_tables(); + break; + case MEM_REGION_TEST_EXTRA: + min_gpa = region->region.guest_phys_addr; + break; + default: + TEST_FAIL("Invalid memory region type '%u'", type); + break; + } + + return ____vm_phy_pages_alloc(vm, nr_pages, min_gpa, vm->memslots[type], + protected, naturally_aligned); } /* @@ -2274,13 +2381,20 @@ __weak void kvm_selftest_arch_init(void) { } -static void report_unexpected_signal(int signum) +__thread sigjmp_buf expect_sigbus_jmpbuf; +__thread volatile sig_atomic_t expecting_sigbus; + +void catchall_signal_handler(int signum) { + switch (signum) { + case SIGBUS: { + if (expecting_sigbus) + siglongjmp(expect_sigbus_jmpbuf, 1); + + TEST_FAIL("Unexpected SIGBUS (%d)\n", signum); + } #define KVM_CASE_SIGNUM(sig) \ case sig: TEST_FAIL("Unexpected " #sig " (%d)\n", signum) - - switch (signum) { - KVM_CASE_SIGNUM(SIGBUS); KVM_CASE_SIGNUM(SIGSEGV); KVM_CASE_SIGNUM(SIGILL); KVM_CASE_SIGNUM(SIGFPE); @@ -2292,12 +2406,13 @@ static void report_unexpected_signal(int signum) void __attribute((constructor)) kvm_selftest_init(void) { struct sigaction sig_sa = { - .sa_handler = report_unexpected_signal, + .sa_handler = catchall_signal_handler, }; /* Tell stdout not to buffer its content. */ setbuf(stdout, NULL); + expecting_sigbus = false; sigaction(SIGBUS, &sig_sa, NULL); sigaction(SIGSEGV, &sig_sa, NULL); sigaction(SIGILL, &sig_sa, NULL); @@ -2306,6 +2421,8 @@ void __attribute((constructor)) kvm_selftest_init(void) srandom(time(0)); kvm_seed_rng(random()); + kvm_has_gmem_attributes = kvm_has_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES); + kvm_selftest_arch_init(); } diff --git a/tools/testing/selftests/kvm/lib/loongarch/processor.c b/tools/testing/selftests/kvm/lib/loongarch/processor.c index 47bbde3e205a..83e53ff705ca 100644 --- a/tools/testing/selftests/kvm/lib/loongarch/processor.c +++ b/tools/testing/selftests/kvm/lib/loongarch/processor.c @@ -9,7 +9,6 @@ #include "processor.h" #include "ucall_common.h" -#define LOONGARCH_PAGE_TABLE_PHYS_MIN 0x200000 #define LOONGARCH_GUEST_STACK_VADDR_MIN 0x200000 static gpa_t invalid_pgtable[4]; @@ -57,9 +56,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) child = table = 0; for (i = 0; i < vm->mmu.pgtable_levels; i++) { invalid_pgtable[i] = child; - table = vm_phy_page_alloc(vm, LOONGARCH_PAGE_TABLE_PHYS_MIN, - vm->memslots[MEM_REGION_PT]); - TEST_ASSERT(table, "Fail to allocate page tale at level %d\n", i); + table = vm_alloc_page_table(vm); virt_set_pgtable(vm, table, child); child = table; } @@ -278,6 +275,7 @@ static void loongarch_set_csr(struct kvm_vcpu *vcpu, u64 id, u64 val) void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) { + bool has_ptw; int ret, width; unsigned int cfg; unsigned long val; @@ -300,6 +298,12 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) ret = __kvm_has_device_attr(vm->fd, KVM_LOONGARCH_VM_FEAT_CTRL, KVM_LOONGARCH_VM_FEAT_LASX); if (!ret) cfg |= CPUCFG2_LASX; + ret = __kvm_has_device_attr(vm->fd, KVM_LOONGARCH_VM_FEAT_CTRL, KVM_LOONGARCH_VM_FEAT_PTW); + if (!ret) + cfg |= CPUCFG2_PTW; + + has_ptw = !ret; + loongarch_set_cpucfg(vcpu, LOONGARCH_CPUCFG2, cfg); cfg = read_cpucfg(LOONGARCH_CPUCFG6); loongarch_set_cpucfg(vcpu, LOONGARCH_CPUCFG6, cfg); @@ -338,6 +342,9 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) /* PGD page shift and width */ val = (vm->page_shift + width * (vm->mmu.pgtable_levels - 1)) | width << 6; + if (has_ptw) + val |= CSR_PWCTL1_PTW; + loongarch_set_csr(vcpu, LOONGARCH_CSR_PWCTL1, val); loongarch_set_csr(vcpu, LOONGARCH_CSR_PGDL, vm->mmu.pgd); @@ -345,8 +352,10 @@ void loongarch_vcpu_setup(struct kvm_vcpu *vcpu) * Refill exception runs on real mode * Entry address should be physical address */ - val = addr_gva2gpa(vm, (unsigned long)handle_tlb_refill); - loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBRENTRY, val); + if (!has_ptw) { + val = addr_gva2gpa(vm, (unsigned long)handle_tlb_refill); + loongarch_set_csr(vcpu, LOONGARCH_CSR_TLBRENTRY, val); + } /* * General exception runs on page-enabled mode diff --git a/tools/testing/selftests/kvm/lib/powerpc/handlers.S b/tools/testing/selftests/kvm/lib/powerpc/handlers.S new file mode 100644 index 000000000000..b860f6a520a1 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/handlers.S @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include + +.macro INTERRUPT vec +. = __interrupts_start + \vec + std %r0,(0*8)(%r13) + std %r3,(3*8)(%r13) + mfspr %r0,SPR_CFAR + li %r3,\vec + b handle_interrupt +.endm + +.balign 0x1000 +.global __interrupts_start +__interrupts_start: +INTERRUPT 0x100 +INTERRUPT 0x200 +INTERRUPT 0x300 +INTERRUPT 0x380 +INTERRUPT 0x400 +INTERRUPT 0x480 +INTERRUPT 0x500 +INTERRUPT 0x600 +INTERRUPT 0x700 +INTERRUPT 0x800 +INTERRUPT 0x900 +INTERRUPT 0xa00 +INTERRUPT 0xc00 +INTERRUPT 0xd00 +INTERRUPT 0xf00 +INTERRUPT 0xf20 +INTERRUPT 0xf40 +INTERRUPT 0xf60 + +virt_handle_interrupt: + stdu %r1,-INT_FRAME_SIZE(%r1) + mr %r3,%r31 + bl route_interrupt + ld %r4,(32*8)(%r31) /* NIA */ + ld %r5,(33*8)(%r31) /* MSR */ + ld %r6,(35*8)(%r31) /* LR */ + ld %r7,(36*8)(%r31) /* CTR */ + ld %r8,(37*8)(%r31) /* XER */ + lwz %r9,(38*8)(%r31) /* CR */ + mtspr SPR_SRR0,%r4 + mtspr SPR_SRR1,%r5 + mtlr %r6 + mtctr %r7 + mtxer %r8 + mtcr %r9 +reg=4 + ld %r0,(0*8)(%r31) + ld %r3,(3*8)(%r31) +.rept 28 + ld reg,(reg*8)(%r31) + reg=reg+1 +.endr + addi %r1,%r1,INT_FRAME_SIZE + rfid + +virt_handle_interrupt_p: + .llong virt_handle_interrupt + +handle_interrupt: +reg=4 +.rept 28 + std reg,(reg*8)(%r13) + reg=reg+1 +.endr + mfspr %r4,SPR_SRR0 + mfspr %r5,SPR_SRR1 + mflr %r6 + mfctr %r7 + mfxer %r8 + mfcr %r9 + std %r4,(32*8)(%r13) /* NIA */ + std %r5,(33*8)(%r13) /* MSR */ + std %r0,(34*8)(%r13) /* CFAR */ + std %r6,(35*8)(%r13) /* LR */ + std %r7,(36*8)(%r13) /* CTR */ + std %r8,(37*8)(%r13) /* XER */ + stw %r9,(38*8 + 0)(%r13) /* CR */ + stw %r3,(38*8 + 4)(%r13) /* TRAP */ + + ld %r31,(39*8)(%r13) /* vaddr */ + ld %r4,virt_handle_interrupt_p - __interrupts_start(0) + mtspr SPR_SRR0,%r4 + /* Reuse SRR1 */ + + rfid +.global __interrupts_end +__interrupts_end: diff --git a/tools/testing/selftests/kvm/lib/powerpc/hcall.c b/tools/testing/selftests/kvm/lib/powerpc/hcall.c new file mode 100644 index 000000000000..d5f5282654ee --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/hcall.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PAPR (pseries) hcall support. + */ +#include "kvm_util.h" +#include "hcall.h" + +s64 hcall0(u64 token) +{ + register uintptr_t r3 asm ("r3") = token; + + asm volatile("sc 1" : "+r"(r3) : + : "r0", "r4", "r5", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +} + +s64 hcall1(u64 token, u64 arg1) +{ + register uintptr_t r3 asm ("r3") = token; + register uintptr_t r4 asm ("r4") = arg1; + + asm volatile("sc 1" : "+r"(r3), "+r"(r4) : + : "r0", "r5", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +} + +s64 hcall2(u64 token, u64 arg1, u64 arg2) +{ + register uintptr_t r3 asm ("r3") = token; + register uintptr_t r4 asm ("r4") = arg1; + register uintptr_t r5 asm ("r5") = arg2; + + asm volatile("sc 1" : "+r"(r3), "+r"(r4), "+r"(r5) : + : "r0", "r6", "r7", "r8", "r9", + "r10", "r11", "r12", "ctr", "xer", + "cc", "memory"); + + return r3; +} diff --git a/tools/testing/selftests/kvm/lib/powerpc/processor.c b/tools/testing/selftests/kvm/lib/powerpc/processor.c new file mode 100644 index 000000000000..301455024571 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/processor.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KVM selftest powerpc library code - CPU-related functions (page tables...) + */ + +#include + +#include "processor.h" +#include "kvm_util.h" +#include "ucall_common.h" +#include "guest_modes.h" +#include "hcall.h" + +#define RADIX_TREE_SIZE ((0x2UL << 61) | (0x5UL << 5)) /* 52-bits */ +#define RADIX_PGD_INDEX_SIZE 13 + +/* + * A radix table must be aligned to its size, and the tables are larger than + * the guest base page size for several level/page size combinations. + */ +bool kvm_arch_needs_naturally_aligned_page_tables(void) +{ + return true; +} + +static void set_proc_table(struct kvm_vm *vm, int pid, u64 dw0, u64 dw1) +{ + u64 *proc_table; + + proc_table = addr_gpa2hva(vm, vm->arch.prtb); + proc_table[pid * 2 + 0] = cpu_to_be64(dw0); + proc_table[pid * 2 + 1] = cpu_to_be64(dw1); +} + +static void set_radix_proc_table(struct kvm_vm *vm, int pid, gpa_t pgd) +{ + set_proc_table(vm, pid, pgd | RADIX_TREE_SIZE | RADIX_PGD_INDEX_SIZE, 0); +} + +void virt_arch_pgd_alloc(struct kvm_vm *vm) +{ + struct kvm_ppc_mmuv3_cfg mmu_cfg; + gpa_t prtb, pgtb; + size_t pgd_pages; + + TEST_ASSERT((vm->mode == VM_MODE_P52V52_4K) || + (vm->mode == VM_MODE_P52V52_64K), + "Unsupported guest mode, mode: 0x%x", vm->mode); + + if (vm->mmu.pgd_created) + return; + + prtb = vm_alloc_page_table(vm); + vm->arch.prtb = prtb; + + pgd_pages = (1UL << (RADIX_PGD_INDEX_SIZE + 3)) >> vm->page_shift; + if (!pgd_pages) + pgd_pages = 1; + pgtb = vm_alloc_page_table_pages(vm, pgd_pages); + vm->mmu.pgd = pgtb; + + /* Set the base page directory in the proc table */ + set_radix_proc_table(vm, 0, pgtb); + + if (vm->mode == VM_MODE_P52V52_4K) + mmu_cfg.process_table = prtb | 0x8000000000000000UL | 0x0; /* 4K size */ + else /* vm->mode == VM_MODE_P52V52_64K */ + mmu_cfg.process_table = prtb | 0x8000000000000000UL | 0x4; /* 64K size */ + mmu_cfg.flags = KVM_PPC_MMUV3_RADIX | KVM_PPC_MMUV3_GTSE; + + vm_ioctl(vm, KVM_PPC_CONFIGURE_V3_MMU, &mmu_cfg); + + vm->mmu.pgd_created = true; +} + +static int pt_shift(struct kvm_vm *vm, int level) +{ + switch (level) { + case 1: + return 13; + case 2: + case 3: + return 9; + case 4: + if (vm->mode == VM_MODE_P52V52_4K) + return 9; + else /* vm->mode == VM_MODE_P52V52_64K */ + return 5; + default: + TEST_ASSERT(false, "Invalid page table level %d\n", level); + return 0; + } +} + +static u64 pt_entry_coverage(struct kvm_vm *vm, int level) +{ + u64 size = vm->page_size; + + if (level == 4) + return size; + size <<= pt_shift(vm, 4); + if (level == 3) + return size; + size <<= pt_shift(vm, 3); + if (level == 2) + return size; + size <<= pt_shift(vm, 2); + return size; +} + +static int pt_idx(struct kvm_vm *vm, gva_t gva, int level, u64 *nls) +{ + switch (level) { + case 1: + if (nls) + *nls = 0x9; + return (gva >> 39) & 0x1fff; + case 2: + if (nls) + *nls = 0x9; + return (gva >> 30) & 0x1ff; + case 3: + if (vm->mode == VM_MODE_P52V52_4K) { + if (nls) + *nls = 0x9; + } else { /* vm->mode == VM_MODE_P52V52_64K */ + if (nls) + *nls = 0x5; + } + return (gva >> 21) & 0x1ff; + case 4: + if (vm->mode == VM_MODE_P52V52_4K) + return (gva >> 12) & 0x1ff; + else /* vm->mode == VM_MODE_P52V52_64K */ + return (gva >> 16) & 0x1f; + default: + TEST_ASSERT(false, "Invalid page table level %d\n", level); + return 0; + } +} + +static u64 *virt_get_pte(struct kvm_vm *vm, gpa_t pt, + gva_t gva, int level, u64 *nls) +{ + int idx = pt_idx(vm, gva, level, nls); + u64 *ptep = addr_gpa2hva(vm, pt + idx * 8); + + return ptep; +} + +#define PTE_VALID 0x8000000000000000ull +#define PTE_LEAF 0x4000000000000000ull +#define PTE_REFERENCED 0x0000000000000100ull +#define PTE_CHANGED 0x0000000000000080ull +#define PTE_PRIV 0x0000000000000008ull +#define PTE_READ 0x0000000000000004ull +#define PTE_RW 0x0000000000000002ull +#define PTE_EXEC 0x0000000000000001ull +#define PTE_PAGE_MASK 0x01fffffffffff000ull + +#define PDE_VALID PTE_VALID +#define PDE_NLS 0x0000000000000011ull +#define PDE_PT_MASK 0x0fffffffffffff00ull + +static gpa_t __vm_alloc_pt(struct kvm_vm *vm, u64 pt_shift) +{ + gpa_t pt; + + if (pt_shift >= vm->page_shift) { + size_t pt_pages = 1ULL << (pt_shift - vm->page_shift); + + pt = vm_alloc_page_table_pages(vm, pt_pages); + } else { + struct vm_pt_frag_cache *pt_frag_cache; + + if (pt_shift == 8) { + pt_frag_cache = &vm->arch.pt_frag_cache[0]; + } else if (pt_shift == 12) { + pt_frag_cache = &vm->arch.pt_frag_cache[1]; + } else { + TEST_ASSERT(0, "Invalid pt_shift:%lu\n", pt_shift); + return 0; + } + + if (!pt_frag_cache->page) + pt_frag_cache->page = vm_alloc_page_table(vm); + pt = pt_frag_cache->page + pt_frag_cache->page_nr_used; + pt_frag_cache->page_nr_used += (1 << pt_shift); + if (pt_frag_cache->page_nr_used == vm->page_size) { + pt_frag_cache->page = 0; + pt_frag_cache->page_nr_used = 0; + } + } + + return pt; +} + +void virt_arch_pg_map(struct kvm_vm *vm, gva_t gva, gpa_t gpa) +{ + gpa_t pt = vm->mmu.pgd; + u64 *ptep, pte; + int level; + + for (level = 1; level <= 3; level++) { + u64 nls; + u64 *pdep = virt_get_pte(vm, pt, gva, level, &nls); + u64 pde = be64_to_cpu(*pdep); + + if (pde) { + TEST_ASSERT((pde & PDE_VALID) && !(pde & PTE_LEAF), + "Invalid PDE at level: %u gva: 0x%lx pde:0x%lx\n", + level, gva, pde); + pt = pde & PDE_PT_MASK; + continue; + } + + pt = __vm_alloc_pt(vm, nls + 3); + pde = PDE_VALID | nls | pt; + *pdep = cpu_to_be64(pde); + } + + ptep = virt_get_pte(vm, pt, gva, level, NULL); + pte = be64_to_cpu(*ptep); + + TEST_ASSERT(!pte, "PTE already present at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + pte = PTE_VALID | PTE_LEAF | PTE_REFERENCED | PTE_CHANGED | PTE_PRIV | + PTE_READ | PTE_RW | PTE_EXEC | (gpa & PTE_PAGE_MASK); + *ptep = cpu_to_be64(pte); +} + +gpa_t addr_arch_gva2gpa(struct kvm_vm *vm, gva_t gva) +{ + gpa_t pt = vm->mmu.pgd; + u64 *ptep, pte; + int level; + + for (level = 1; level <= 3; level++) { + u64 nls; + u64 *pdep = virt_get_pte(vm, pt, gva, level, &nls); + u64 pde = be64_to_cpu(*pdep); + + TEST_ASSERT((pde & PDE_VALID) && !(pde & PTE_LEAF), + "PDE not present at level: %u gva: 0x%lx pde:0x%lx\n", + level, gva, pde); + pt = pde & PDE_PT_MASK; + } + + ptep = virt_get_pte(vm, pt, gva, level, NULL); + pte = be64_to_cpu(*ptep); + + TEST_ASSERT(pte, + "PTE not present at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + TEST_ASSERT((pte & PTE_VALID) && (pte & PTE_LEAF), + "PTE not valid at level: %u gva: 0x%lx pte:0x%lx\n", + level, gva, pte); + + return (pte & PTE_PAGE_MASK) + (gva & (vm->page_size - 1)); +} + +static void virt_dump_pt(FILE *stream, struct kvm_vm *vm, gpa_t pt, + gva_t va, int level, u8 indent) +{ + int size, idx; + + size = 1U << (pt_shift(vm, level) + 3); + + for (idx = 0; idx < size; idx += 8, va += pt_entry_coverage(vm, level)) { + u64 *page_table = addr_gpa2hva(vm, pt + idx); + u64 pte = be64_to_cpu(*page_table); + + if (!(pte & PTE_VALID)) + continue; + + if (pte & PTE_LEAF) { + fprintf(stream, + "%*s PTE[%d] gVA:0x%016lx -> gRA:0x%016llx\n", + indent, "", idx / 8, va, pte & PTE_PAGE_MASK); + } else { + fprintf(stream, "%*sPDE%d[%d] gVA:0x%016lx\n", + indent, "", level, idx / 8, va); + virt_dump_pt(stream, vm, pte & PDE_PT_MASK, va, + level + 1, indent + 2); + } + } + +} + +void virt_arch_dump(FILE *stream, struct kvm_vm *vm, u8 indent) +{ + gpa_t pt = vm->mmu.pgd; + + if (!vm->mmu.pgd_created) + return; + + virt_dump_pt(stream, vm, pt, 0, 1, indent); +} + +static unsigned long get_r2(void) +{ + unsigned long r2; + + asm("mr %0,%%r2" : "=r"(r2)); + + return r2; +} + +/* + * Under ELFv1 a function pointer is the address of a descriptor holding the + * entry point and the TOC, not the entry point itself. ELFv2 dropped + * descriptors, i.e. the pointer is already the entry point. Dereference the + * descriptor as needed so that the guest starts executing code and not data. + */ +static uintptr_t function_entry(void *fn) +{ +#if defined(_CALL_ELF) && _CALL_ELF == 2 + return (uintptr_t)fn; +#else + return ((const u64 *)fn)[0]; +#endif +} + +void vcpu_arch_set_entry_point(struct kvm_vcpu *vcpu, void *guest_code) +{ + uintptr_t entry = function_entry(guest_code); + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + regs.pc = entry; + regs.gpr[12] = entry; + vcpu_regs_set(vcpu, ®s); +} + +struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, u32 vcpu_id) +{ + const size_t stack_size = SZ_64K; + gva_t stack_vaddr, ex_regs_vaddr; + gpa_t ex_regs_paddr; + struct ex_regs *ex_regs; + struct kvm_regs regs; + struct kvm_vcpu *vcpu; + u64 lpcr; + + stack_vaddr = __vm_alloc(vm, stack_size, + DEFAULT_GUEST_STACK_VADDR_MIN, + MEM_REGION_DATA); + + ex_regs_vaddr = __vm_alloc(vm, stack_size, + DEFAULT_GUEST_STACK_VADDR_MIN, + MEM_REGION_DATA); + ex_regs_paddr = addr_gva2gpa(vm, ex_regs_vaddr); + ex_regs = addr_gpa2hva(vm, ex_regs_paddr); + ex_regs->gva = ex_regs_vaddr; + + vcpu = __vm_vcpu_add(vm, vcpu_id); + + vcpu_enable_cap(vcpu, KVM_CAP_PPC_PAPR, 1); + + /* Setup guest registers */ + vcpu_regs_get(vcpu, ®s); + lpcr = vcpu_get_reg(vcpu, KVM_REG_PPC_LPCR_64); + + regs.gpr[1] = stack_vaddr + stack_size - 256; + regs.gpr[2] = (uintptr_t)get_r2(); + regs.gpr[13] = (uintptr_t)ex_regs_paddr; + + regs.msr = MSR_SF | MSR_VEC | MSR_VSX | MSR_FP | + MSR_ME | MSR_IR | MSR_DR | MSR_RI; + + if (BYTE_ORDER == LITTLE_ENDIAN) { + regs.msr |= MSR_LE; + lpcr |= LPCR_ILE; + } else { + lpcr &= ~LPCR_ILE; + } + + vcpu_regs_set(vcpu, ®s); + vcpu_set_reg(vcpu, KVM_REG_PPC_LPCR_64, lpcr); + + return vcpu; +} + +void vcpu_args_set(struct kvm_vcpu *vcpu, unsigned int num, ...) +{ + va_list ap; + struct kvm_regs regs; + int i; + + TEST_ASSERT(num >= 1 && num <= 5, "Unsupported number of args: %u\n", + num); + + va_start(ap, num); + vcpu_regs_get(vcpu, ®s); + + for (i = 0; i < num; i++) + regs.gpr[i + 3] = va_arg(ap, u64); + + vcpu_regs_set(vcpu, ®s); + va_end(ap); +} + +void vcpu_arch_dump(FILE *stream, struct kvm_vcpu *vcpu, u8 indent) +{ + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + + fprintf(stream, "%*sNIA: 0x%016llx MSR: 0x%016llx\n", + indent, "", regs.pc, regs.msr); + fprintf(stream, "%*sLR: 0x%016llx CTR :0x%016llx\n", + indent, "", regs.lr, regs.ctr); + fprintf(stream, "%*sCR: 0x%08llx XER :0x%016llx\n", + indent, "", regs.cr, regs.xer); +} + +void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) +{ + size_t excp_size = __interrupts_end - __interrupts_start; + size_t excp_pages = (excp_size + vm->page_size - 1) / vm->page_size; + gpa_t excp_paddr; + void *mem; + + /* + * Guest interrupts are taken in real-mode, i.e. the vectors must live + * at gPA 0. Use the innermost allocator to override the minimum GPA + * that MEM_REGION_DATA would otherwise impose. + */ + excp_paddr = ____vm_phy_pages_alloc(vm, excp_pages, 0, + vm->memslots[MEM_REGION_DATA], + vm_arch_has_protected_memory(vm), + false); + + TEST_ASSERT(excp_paddr == 0, + "Interrupt vectors not allocated at gPA address 0: (0x%lx)", + excp_paddr); + + mem = addr_gpa2hva(vm, excp_paddr); + memcpy(mem, __interrupts_start, excp_size); +} + +void assert_on_unhandled_exception(struct kvm_vcpu *vcpu) +{ + struct ucall uc; + + if (get_ucall(vcpu, &uc) == UCALL_UNHANDLED) { + gpa_t ex_regs_paddr; + struct ex_regs *ex_regs; + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + ex_regs_paddr = (gpa_t)regs.gpr[13]; + ex_regs = addr_gpa2hva(vcpu->vm, ex_regs_paddr); + + TEST_FAIL("Unexpected interrupt in guest NIA:0x%016lx MSR:0x%016lx TRAP:0x%04x", + ex_regs->nia, ex_regs->msr, ex_regs->trap); + } +} + +struct handler { + void (*fn)(struct ex_regs *regs); + int trap; +}; + +#define NR_HANDLERS 10 +static struct handler handlers[NR_HANDLERS]; + +void route_interrupt(struct ex_regs *regs) +{ + int i; + + for (i = 0; i < NR_HANDLERS; i++) { + if (handlers[i].trap == regs->trap) { + handlers[i].fn(regs); + return; + } + } + + ucall(UCALL_UNHANDLED, 0); +} + +void vm_install_exception_handler(struct kvm_vm *vm, int trap, + void (*fn)(struct ex_regs *)) +{ + int i; + + for (i = 0; i < NR_HANDLERS; i++) { + if (!handlers[i].trap || handlers[i].trap == trap) { + if (fn == NULL) + trap = 0; /* Clear handler */ + handlers[i].trap = trap; + handlers[i].fn = fn; + sync_global_to_guest(vm, handlers[i]); + return; + } + } + + TEST_FAIL("Out of exception handlers"); +} + +void kvm_selftest_arch_init(void) +{ + TEST_REQUIRE(kvm_has_cap(KVM_CAP_PPC_MMU_RADIX)); + + /* + * powerpc default mode is set by host page size and not static, + * so start by computing that early. + */ + guest_modes_append_default(); +} diff --git a/tools/testing/selftests/kvm/lib/powerpc/ucall.c b/tools/testing/selftests/kvm/lib/powerpc/ucall.c new file mode 100644 index 000000000000..3481a7a0b850 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/powerpc/ucall.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ucall support. A ucall is a "hypercall to host userspace". + */ +#include "kvm_util.h" +#include "ucall_common.h" +#include "hcall.h" + +void *ucall_arch_get_ucall(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + if (run->exit_reason == UCALL_EXIT_REASON && + run->papr_hcall.nr == H_UCALL) { + struct kvm_regs regs; + + vcpu_regs_get(vcpu, ®s); + if (regs.gpr[4] == UCALL_R4_UCALL) + return (void *)regs.gpr[5]; + } + return NULL; +} diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index ded5429f3448..c175dcd14114 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -69,9 +69,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - vm->mmu.pgd = vm_phy_pages_alloc(vm, nr_pages, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + vm->mmu.pgd = vm_alloc_page_table_pages(vm, nr_pages); vm->mmu.pgd_created = true; } diff --git a/tools/testing/selftests/kvm/lib/s390/processor.c b/tools/testing/selftests/kvm/lib/s390/processor.c index a9adb3782b35..33f49d743b5f 100644 --- a/tools/testing/selftests/kvm/lib/s390/processor.c +++ b/tools/testing/selftests/kvm/lib/s390/processor.c @@ -20,9 +20,7 @@ void virt_arch_pgd_alloc(struct kvm_vm *vm) if (vm->mmu.pgd_created) return; - gpa = vm_phy_pages_alloc(vm, PAGES_PER_REGION, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, - vm->memslots[MEM_REGION_PT]); + gpa = vm_alloc_page_table_pages(vm, PAGES_PER_REGION); memset(addr_gpa2hva(vm, gpa), 0xff, PAGES_PER_REGION * vm->page_size); vm->mmu.pgd = gpa; @@ -38,8 +36,7 @@ static u64 virt_alloc_region(struct kvm_vm *vm, int ri) { u64 taddr; - taddr = vm_phy_pages_alloc(vm, ri < 4 ? PAGES_PER_REGION : 1, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0); + taddr = vm_alloc_page_table_pages(vm, ri < 4 ? PAGES_PER_REGION : 1); memset(addr_gpa2hva(vm, taddr), 0xff, PAGES_PER_REGION * vm->page_size); return (taddr & REGION_ENTRY_ORIGIN) diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index 4dff72f6bd34..f5f7189f96a8 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -19,13 +19,6 @@ #include "test_util.h" #include "kvm_syscalls.h" -sigjmp_buf expect_sigbus_jmpbuf; - -void __attribute__((used)) expect_sigbus_handler(int signum) -{ - siglongjmp(expect_sigbus_jmpbuf, 1); -} - /* * Random number generator that is usable from guest code. This is the * Park-Miller LCG using standard constants. @@ -255,6 +248,21 @@ size_t get_def_hugetlb_pagesz(void) TEST_FAIL("Error in reading /proc/meminfo"); } +size_t get_free_hugepages(size_t page_size) +{ + char path[128]; + size_t free; + + snprintf(path, sizeof(path), + "/sys/kernel/mm/hugepages/hugepages-%zukB/free_hugepages", + page_size >> 10); + if (!test_sysfs_path(path)) + return 0; + + free = get_sysfs_val(path); + return free * page_size; +} + #define ANON_FLAGS (MAP_PRIVATE | MAP_ANONYMOUS) #define ANON_HUGE_FLAGS (ANON_FLAGS | MAP_HUGETLB) diff --git a/tools/testing/selftests/kvm/lib/userfaultfd_util.c b/tools/testing/selftests/kvm/lib/userfaultfd_util.c index f7ce5a6ddcc2..49eff05b7d39 100644 --- a/tools/testing/selftests/kvm/lib/userfaultfd_util.c +++ b/tools/testing/selftests/kvm/lib/userfaultfd_util.c @@ -119,13 +119,13 @@ struct uffd_desc *uffd_setup_demand_paging(int uffd_mode, useconds_t delay, uffd_desc = malloc(sizeof(struct uffd_desc)); TEST_ASSERT(uffd_desc, "Failed to malloc uffd descriptor"); - uffd_desc->pipefds = calloc(sizeof(int), num_readers); + uffd_desc->pipefds = calloc(num_readers, sizeof(int)); TEST_ASSERT(uffd_desc->pipefds, "Failed to alloc pipes"); - uffd_desc->readers = calloc(sizeof(pthread_t), num_readers); + uffd_desc->readers = calloc(num_readers, sizeof(pthread_t)); TEST_ASSERT(uffd_desc->readers, "Failed to alloc reader threads"); - uffd_desc->reader_args = calloc(sizeof(struct uffd_reader_args), num_readers); + uffd_desc->reader_args = calloc(num_readers, sizeof(struct uffd_reader_args)); TEST_ASSERT(uffd_desc->reader_args, "Failed to alloc reader_args"); uffd_desc->num_readers = num_readers; diff --git a/tools/testing/selftests/kvm/lib/x86/memstress.c b/tools/testing/selftests/kvm/lib/x86/memstress.c index e19e8b5a09c5..9bfea36144a7 100644 --- a/tools/testing/selftests/kvm/lib/x86/memstress.c +++ b/tools/testing/selftests/kvm/lib/x86/memstress.c @@ -33,15 +33,15 @@ __asm__( static void l1_vmx_code(struct vmx_pages *vmx, u64 vcpu_id) { GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); GUEST_ASSERT(ept_1g_pages_supported()); *(u64 *)vmx->stack = vcpu_id; prepare_vmcs(vmx, memstress_l2_guest_entry); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); + vmlaunch(); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_VMCALL); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/lib/x86/processor.c b/tools/testing/selftests/kvm/lib/x86/processor.c index d31fa81ea075..8374afd08baf 100644 --- a/tools/testing/selftests/kvm/lib/x86/processor.c +++ b/tools/testing/selftests/kvm/lib/x86/processor.c @@ -26,6 +26,8 @@ bool host_cpu_is_amd; bool host_cpu_is_intel; bool host_cpu_is_hygon; bool host_cpu_is_amd_compatible; +bool host_cpu_is_zhaoxin; +bool host_cpu_is_intel_compatible; bool is_forced_emulation_enabled; u64 guest_tsc_khz; struct kvm_mmu guest_mmu; @@ -158,7 +160,7 @@ static void sregs_dump(FILE *stream, struct kvm_sregs *sregs, u8 indent) bool kvm_is_tdp_enabled(void) { - if (host_cpu_is_intel) + if (host_cpu_is_intel_compatible) return get_kvm_intel_param_bool("ept"); else return get_kvm_amd_param_bool("npt"); @@ -529,7 +531,10 @@ void tdp_identity_map_default_memslots(struct kvm_vm *vm) /* Only memslot 0 is mapped here, ensure it's the only one being used */ for (s = 0; s < NR_MEM_REGIONS; s++) - TEST_ASSERT_EQ(vm->memslots[s], 0); + TEST_ASSERT(!vm->memslots[s] || + vm->memslots[s] == KVM_INVALID_MEMSLOT, + "Unhandled memslot '%u' for type '%u'", + vm->memslots[s], s); i = (region->region.guest_phys_addr >> vm->page_shift) - 1; last = i + (region->region.memory_size >> vm->page_shift); @@ -819,6 +824,8 @@ void kvm_arch_vm_post_create(struct kvm_vm *vm, unsigned int nr_vcpus) sync_global_to_guest(vm, host_cpu_is_amd); sync_global_to_guest(vm, host_cpu_is_hygon); sync_global_to_guest(vm, host_cpu_is_amd_compatible); + sync_global_to_guest(vm, host_cpu_is_zhaoxin); + sync_global_to_guest(vm, host_cpu_is_intel_compatible); sync_global_to_guest(vm, is_forced_emulation_enabled); sync_global_to_guest(vm, pmu_errata_mask); @@ -1444,6 +1451,8 @@ void kvm_selftest_arch_init(void) host_cpu_is_amd = this_cpu_is_amd(); host_cpu_is_hygon = this_cpu_is_hygon(); host_cpu_is_amd_compatible = host_cpu_is_amd || host_cpu_is_hygon; + host_cpu_is_zhaoxin = this_cpu_is_zhaoxin(); + host_cpu_is_intel_compatible = host_cpu_is_intel || host_cpu_is_zhaoxin; is_forced_emulation_enabled = kvm_is_forced_emulation_enabled(); kvm_init_pmu_errata(); @@ -1465,13 +1474,13 @@ bool kvm_arch_has_default_irqchip(void) return true; } -void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, u64 smram_gpa, +void setup_smram(struct kvm_vm *vm, struct kvm_vcpu *vcpu, gpa_t smram_gpa, const void *smi_handler, size_t handler_size) { - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, smram_gpa, - SMRAM_MEMSLOT, SMRAM_PAGES, 0); - TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, smram_gpa, - SMRAM_MEMSLOT) == smram_gpa, + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + smram_gpa, SMRAM_MEMSLOT, SMRAM_PAGES); + + TEST_ASSERT(vm_phy_pages_alloc(vm, SMRAM_PAGES, MEM_REGION_TEST_EXTRA) == smram_gpa, "Could not allocate guest physical addresses for SMRAM"); memset(addr_gpa2hva(vm, smram_gpa), 0x0, SMRAM_SIZE); diff --git a/tools/testing/selftests/kvm/lib/x86/vmx.c b/tools/testing/selftests/kvm/lib/x86/vmx.c index 089e1a8af53f..5a06746ebea4 100644 --- a/tools/testing/selftests/kvm/lib/x86/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86/vmx.c @@ -125,7 +125,7 @@ vcpu_alloc_vmx(struct kvm_vm *vm, gva_t *p_vmx_gva) return vmx; } -bool prepare_for_vmx_operation(struct vmx_pages *vmx) +void prepare_for_vmx_operation(struct vmx_pages *vmx) { u64 feature_control; u64 required; @@ -163,28 +163,75 @@ bool prepare_for_vmx_operation(struct vmx_pages *vmx) /* Enter VMX root operation. */ *(u32 *)(vmx->vmxon) = vmcs_revision(); - if (vmxon(vmx->vmxon_gpa)) - return false; - - return true; + vmxon(vmx->vmxon_gpa); } -bool load_vmcs(struct vmx_pages *vmx) +void load_vmcs(struct vmx_pages *vmx) { /* Load a VMCS. */ *(u32 *)(vmx->vmcs) = vmcs_revision(); - if (vmclear(vmx->vmcs_gpa)) - return false; + vmclear(vmx->vmcs_gpa); - if (vmptrld(vmx->vmcs_gpa)) - return false; + vmptrld(vmx->vmcs_gpa); /* Setup shadow VMCS, do not load it yet. */ *(u32 *)(vmx->shadow_vmcs) = vmcs_revision() | 0x80000000ul; - if (vmclear(vmx->shadow_vmcs_gpa)) - return false; + vmclear(vmx->shadow_vmcs_gpa); +} - return true; +#define __BUILD_VMX_VM_ENTRY_HELPER(insn, prefix, vmwrite_insn, vmwrite_operand, \ + __host_rsp, __host_rip) \ +static int __##prefix##_##insn(void) \ +{ \ + int ret; \ + \ + __asm__ __volatile__("push $0;" \ + __stringify(vmwrite_insn) " %%rsp, %[host_rsp];" \ + "lea 1f(%%rip), %%rax;" \ + __stringify(vmwrite_insn) " %%rax, %[host_rip];" \ + VMX_SWITCH_GPRS_ASM \ + __stringify(insn)";" \ + "incq (%%rsp);" \ + "1: ;" \ + VMX_SWITCH_GPRS_ASM \ + "pop %%rax;" \ + : [ret]"=&a"(ret) \ + : [host_rsp]__stringify(vmwrite_operand)(__host_rsp), \ + [host_rip]__stringify(vmwrite_operand)(__host_rip), \ + GUEST_REGS_OFFSETS \ + : "memory", "cc"); \ + return ret; \ +} + +#define BUILD_VMX_VM_ENTRY_HELPER(insn) \ + __BUILD_VMX_VM_ENTRY_HELPER(insn, _, vmwrite, r, (u64)HOST_RSP, (u64)HOST_RIP) \ + __BUILD_VMX_VM_ENTRY_HELPER(insn, __evmcs, mov, m, \ + current_evmcs->host_rsp, current_evmcs->host_rip) + +BUILD_VMX_VM_ENTRY_HELPER(vmlaunch) +BUILD_VMX_VM_ENTRY_HELPER(vmresume) + +int __vmlaunch(void) +{ + if (enable_evmcs) { + current_evmcs->hv_clean_fields = 0; + return ____evmcs_vmlaunch(); + } + + return ____vmlaunch(); +} + +int __vmresume(void) +{ + if (enable_evmcs) { + /* HOST_RIP */ + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1; + /* HOST_RSP */ + current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER; + return ____evmcs_vmresume(); + } + + return ____vmresume(); } static bool ept_vpid_cap_supported(u64 mask) @@ -204,8 +251,8 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) { u32 sec_exec_ctl = 0; - vmwrite(VIRTUAL_PROCESSOR_ID, 0); - vmwrite(POSTED_INTR_NV, 0); + __vmwrite(VIRTUAL_PROCESSOR_ID, 0); + __vmwrite(POSTED_INTR_NV, 0); vmwrite(PIN_BASED_VM_EXEC_CONTROL, rdmsr(MSR_IA32_VMX_TRUE_PINBASED_CTLS)); @@ -222,7 +269,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) sec_exec_ctl |= SECONDARY_EXEC_ENABLE_EPT; } - if (!vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_exec_ctl)) + if (!__vmwrite(SECONDARY_VM_EXEC_CONTROL, sec_exec_ctl)) vmwrite(CPU_BASED_VM_EXEC_CONTROL, rdmsr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS); else { @@ -242,16 +289,16 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) VM_ENTRY_IA32E_MODE); /* 64-bit guest */ vmwrite(VM_ENTRY_MSR_LOAD_COUNT, 0); vmwrite(VM_ENTRY_INTR_INFO_FIELD, 0); - vmwrite(TPR_THRESHOLD, 0); + __vmwrite(TPR_THRESHOLD, 0); vmwrite(CR0_GUEST_HOST_MASK, 0); vmwrite(CR4_GUEST_HOST_MASK, 0); vmwrite(CR0_READ_SHADOW, get_cr0()); vmwrite(CR4_READ_SHADOW, get_cr4()); - vmwrite(MSR_BITMAP, vmx->msr_gpa); - vmwrite(VMREAD_BITMAP, vmx->vmread_gpa); - vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa); + __vmwrite(MSR_BITMAP, vmx->msr_gpa); + __vmwrite(VMREAD_BITMAP, vmx->vmread_gpa); + __vmwrite(VMWRITE_BITMAP, vmx->vmwrite_gpa); } /* @@ -261,7 +308,7 @@ static inline void init_vmcs_control_fields(struct vmx_pages *vmx) */ static inline void init_vmcs_host_state(void) { - u32 exit_controls = vmreadz(VM_EXIT_CONTROLS); + u32 exit_controls = vmread(VM_EXIT_CONTROLS); vmwrite(HOST_ES_SELECTOR, get_es()); vmwrite(HOST_CS_SELECTOR, get_cs()); @@ -301,23 +348,23 @@ static inline void init_vmcs_host_state(void) */ static inline void init_vmcs_guest_state(void *rip, void *rsp) { - vmwrite(GUEST_ES_SELECTOR, vmreadz(HOST_ES_SELECTOR)); - vmwrite(GUEST_CS_SELECTOR, vmreadz(HOST_CS_SELECTOR)); - vmwrite(GUEST_SS_SELECTOR, vmreadz(HOST_SS_SELECTOR)); - vmwrite(GUEST_DS_SELECTOR, vmreadz(HOST_DS_SELECTOR)); - vmwrite(GUEST_FS_SELECTOR, vmreadz(HOST_FS_SELECTOR)); - vmwrite(GUEST_GS_SELECTOR, vmreadz(HOST_GS_SELECTOR)); + vmwrite(GUEST_ES_SELECTOR, vmread(HOST_ES_SELECTOR)); + vmwrite(GUEST_CS_SELECTOR, vmread(HOST_CS_SELECTOR)); + vmwrite(GUEST_SS_SELECTOR, vmread(HOST_SS_SELECTOR)); + vmwrite(GUEST_DS_SELECTOR, vmread(HOST_DS_SELECTOR)); + vmwrite(GUEST_FS_SELECTOR, vmread(HOST_FS_SELECTOR)); + vmwrite(GUEST_GS_SELECTOR, vmread(HOST_GS_SELECTOR)); vmwrite(GUEST_LDTR_SELECTOR, 0); - vmwrite(GUEST_TR_SELECTOR, vmreadz(HOST_TR_SELECTOR)); - vmwrite(GUEST_INTR_STATUS, 0); - vmwrite(GUEST_PML_INDEX, 0); + vmwrite(GUEST_TR_SELECTOR, vmread(HOST_TR_SELECTOR)); + __vmwrite(GUEST_INTR_STATUS, 0); + __vmwrite(GUEST_PML_INDEX, 0); vmwrite(VMCS_LINK_POINTER, -1ll); vmwrite(GUEST_IA32_DEBUGCTL, 0); - vmwrite(GUEST_IA32_PAT, vmreadz(HOST_IA32_PAT)); - vmwrite(GUEST_IA32_EFER, vmreadz(HOST_IA32_EFER)); - vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL, - vmreadz(HOST_IA32_PERF_GLOBAL_CTRL)); + __vmwrite(GUEST_IA32_PAT, vmread(HOST_IA32_PAT)); + __vmwrite(GUEST_IA32_EFER, vmread(HOST_IA32_EFER)); + __vmwrite(GUEST_IA32_PERF_GLOBAL_CTRL, + vmread(HOST_IA32_PERF_GLOBAL_CTRL)); vmwrite(GUEST_ES_LIMIT, -1); vmwrite(GUEST_CS_LIMIT, -1); @@ -330,42 +377,42 @@ static inline void init_vmcs_guest_state(void *rip, void *rsp) vmwrite(GUEST_GDTR_LIMIT, 0xffff); vmwrite(GUEST_IDTR_LIMIT, 0xffff); vmwrite(GUEST_ES_AR_BYTES, - vmreadz(GUEST_ES_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_ES_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_CS_AR_BYTES, 0xa09b); vmwrite(GUEST_SS_AR_BYTES, 0xc093); vmwrite(GUEST_DS_AR_BYTES, - vmreadz(GUEST_DS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_DS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_FS_AR_BYTES, - vmreadz(GUEST_FS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_FS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_GS_AR_BYTES, - vmreadz(GUEST_GS_SELECTOR) == 0 ? 0x10000 : 0xc093); + vmread(GUEST_GS_SELECTOR) == 0 ? 0x10000 : 0xc093); vmwrite(GUEST_LDTR_AR_BYTES, 0x10000); vmwrite(GUEST_TR_AR_BYTES, 0x8b); vmwrite(GUEST_INTERRUPTIBILITY_INFO, 0); vmwrite(GUEST_ACTIVITY_STATE, 0); - vmwrite(GUEST_SYSENTER_CS, vmreadz(HOST_IA32_SYSENTER_CS)); - vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0); + vmwrite(GUEST_SYSENTER_CS, vmread(HOST_IA32_SYSENTER_CS)); + __vmwrite(VMX_PREEMPTION_TIMER_VALUE, 0); - vmwrite(GUEST_CR0, vmreadz(HOST_CR0)); - vmwrite(GUEST_CR3, vmreadz(HOST_CR3)); - vmwrite(GUEST_CR4, vmreadz(HOST_CR4)); + vmwrite(GUEST_CR0, vmread(HOST_CR0)); + vmwrite(GUEST_CR3, vmread(HOST_CR3)); + vmwrite(GUEST_CR4, vmread(HOST_CR4)); vmwrite(GUEST_ES_BASE, 0); vmwrite(GUEST_CS_BASE, 0); vmwrite(GUEST_SS_BASE, 0); vmwrite(GUEST_DS_BASE, 0); - vmwrite(GUEST_FS_BASE, vmreadz(HOST_FS_BASE)); - vmwrite(GUEST_GS_BASE, vmreadz(HOST_GS_BASE)); + vmwrite(GUEST_FS_BASE, vmread(HOST_FS_BASE)); + vmwrite(GUEST_GS_BASE, vmread(HOST_GS_BASE)); vmwrite(GUEST_LDTR_BASE, 0); - vmwrite(GUEST_TR_BASE, vmreadz(HOST_TR_BASE)); - vmwrite(GUEST_GDTR_BASE, vmreadz(HOST_GDTR_BASE)); - vmwrite(GUEST_IDTR_BASE, vmreadz(HOST_IDTR_BASE)); + vmwrite(GUEST_TR_BASE, vmread(HOST_TR_BASE)); + vmwrite(GUEST_GDTR_BASE, vmread(HOST_GDTR_BASE)); + vmwrite(GUEST_IDTR_BASE, vmread(HOST_IDTR_BASE)); vmwrite(GUEST_DR7, 0x400); vmwrite(GUEST_RSP, (u64)rsp); vmwrite(GUEST_RIP, (u64)rip); vmwrite(GUEST_RFLAGS, X86_EFLAGS_FIXED); vmwrite(GUEST_PENDING_DBG_EXCEPTIONS, 0); - vmwrite(GUEST_SYSENTER_ESP, vmreadz(HOST_IA32_SYSENTER_ESP)); - vmwrite(GUEST_SYSENTER_EIP, vmreadz(HOST_IA32_SYSENTER_EIP)); + vmwrite(GUEST_SYSENTER_ESP, vmread(HOST_IA32_SYSENTER_ESP)); + vmwrite(GUEST_SYSENTER_EIP, vmread(HOST_IA32_SYSENTER_EIP)); } void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip) @@ -375,19 +422,19 @@ void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip) init_vmcs_guest_state(guest_rip, vmx->stack); } -bool kvm_cpu_has_ept(void) +bool kvm_cpu_has_secondary_exec_control(u32 ctrl) { - u64 ctrl; + u64 ctrl_msr; if (!kvm_cpu_has(X86_FEATURE_VMX)) return false; - ctrl = kvm_get_feature_msr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32; - if (!(ctrl & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) + ctrl_msr = kvm_get_feature_msr(MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32; + if (!(ctrl_msr & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) return false; - ctrl = kvm_get_feature_msr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32; - return ctrl & SECONDARY_EXEC_ENABLE_EPT; + ctrl_msr = kvm_get_feature_msr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32; + return ctrl_msr & ctrl; } void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/memslot_perf_test.c b/tools/testing/selftests/kvm/memslot_perf_test.c index e1299611be32..db2fd3c59d49 100644 --- a/tools/testing/selftests/kvm/memslot_perf_test.c +++ b/tools/testing/selftests/kvm/memslot_perf_test.c @@ -30,7 +30,7 @@ #define MEM_EXTRA_SIZE SZ_64K #define MEM_SIZE (SZ_512M + MEM_EXTRA_SIZE) -#define MEM_GPA SZ_256M +#define MEM_GPA SZ_512M #define MEM_AUX_GPA MEM_GPA #define MEM_SYNC_GPA MEM_AUX_GPA #define MEM_TEST_GPA (MEM_AUX_GPA + MEM_EXTRA_SIZE) @@ -152,7 +152,7 @@ static void *vcpu_worker(void *__data) case UCALL_SYNC: TEST_ASSERT(uc.args[1] == 0, "Unexpected sync ucall, got %lx", - (ulong)uc.args[1]); + (unsigned long)uc.args[1]); sem_post(&vcpu_ready); continue; case UCALL_NONE: @@ -339,7 +339,8 @@ static bool prepare_vm(struct vm_data *data, int nslots, u64 *maxslots, if (slot == data->nslots) npages += rempages; - gpa = vm_phy_pages_alloc(data->vm, npages, guest_addr, slot); + gpa = ____vm_phy_pages_alloc(data->vm, npages, guest_addr, slot, + false, false); TEST_ASSERT(gpa == guest_addr, "vm_phy_pages_alloc() failed"); diff --git a/tools/testing/selftests/kvm/pre_fault_memory_test.c b/tools/testing/selftests/kvm/pre_fault_memory_test.c index c57631aab3d3..3082fe09b95f 100644 --- a/tools/testing/selftests/kvm/pre_fault_memory_test.c +++ b/tools/testing/selftests/kvm/pre_fault_memory_test.c @@ -12,19 +12,29 @@ #include #include #include +#include /* Arbitrarily chosen values */ -#define TEST_SIZE (SZ_2M + PAGE_SIZE) -#define TEST_NPAGES (TEST_SIZE / PAGE_SIZE) +#define TEST_BASE_SIZE SZ_2M #define TEST_SLOT 10 +/* Storage of test info to share with guest code */ +struct test_config { + u64 page_size; + u64 test_size; + u64 test_num_pages; +}; + +static struct test_config test_config; + static void guest_code(u64 base_gva) { volatile u64 val __used; + struct test_config *config = &test_config; int i; - for (i = 0; i < TEST_NPAGES; i++) { - u64 *src = (u64 *)(base_gva + i * PAGE_SIZE); + for (i = 0; i < config->test_num_pages; i++) { + u64 *src = (u64 *)(base_gva + i * config->page_size); val = *src; } @@ -36,6 +46,7 @@ struct slot_worker_data { struct kvm_vm *vm; gpa_t gpa; u32 flags; + enum vm_mem_backing_src_type mem_backing_src; bool worker_ready; bool prefault_ready; bool recreate_slot; @@ -56,14 +67,16 @@ static void *delete_slot_worker(void *__data) while (!READ_ONCE(data->recreate_slot)) cpu_relax(); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, data->gpa, - TEST_SLOT, TEST_NPAGES, data->flags); + vm_userspace_mem_region_add(vm, data->mem_backing_src, data->gpa, + TEST_SLOT, test_config.test_num_pages, data->flags); return NULL; } static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, - u64 size, u64 expected_left, bool private) + u64 size, u64 expected_left, + enum vm_mem_backing_src_type mem_backing_src, + bool private) { struct kvm_pre_fault_memory range = { .gpa = base_gpa + offset, @@ -74,6 +87,7 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, .vm = vcpu->vm, .gpa = base_gpa, .flags = private ? KVM_MEM_GUEST_MEMFD : 0, + .mem_backing_src = mem_backing_src, }; bool slot_recreated = false; pthread_t slot_worker; @@ -150,8 +164,8 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, /* * Assert success if prefaulting the entire range should succeed, i.e. * complete with no bytes remaining. Otherwise prefaulting should have - * failed due to ENOENT (due to RET_PF_EMULATE for emulated MMIO when - * no memslot exists). + * failed due to ENOENT (no memslot exists for the GPA; on x86 this + * surfaces via RET_PF_EMULATE). */ if (!expected_left) TEST_ASSERT_VM_VCPU_IOCTL(!ret, KVM_PRE_FAULT_MEMORY, ret, vcpu->vm); @@ -160,39 +174,82 @@ static void pre_fault_memory(struct kvm_vcpu *vcpu, u64 base_gpa, u64 offset, KVM_PRE_FAULT_MEMORY, ret, vcpu->vm); } -static void __test_pre_fault_memory(unsigned long vm_type, bool private) +struct test_params { + unsigned long vm_type; + bool private; + enum vm_mem_backing_src_type mem_backing_src; +}; + +static void __test_pre_fault_memory(enum vm_guest_mode guest_mode, void *arg) { - gpa_t gpa, gva, alignment, guest_page_size; + gpa_t gpa, gva, alignment, guest_page_size, host_page_size; + gpa_t backing_src_pagesz, mem_page_size; + struct test_params *p = arg; const struct vm_shape shape = { - .mode = VM_MODE_DEFAULT, - .type = vm_type, + .mode = guest_mode, + .type = p->vm_type, }; struct kvm_vcpu *vcpu; + struct kvm_run *run; struct kvm_vm *vm; struct ucall uc; + pr_info("Testing guest mode: %s\n", vm_guest_mode_string(guest_mode)); + pr_info("Testing memory backing src type: %s\n", + vm_mem_backing_src_alias(p->mem_backing_src)->name); + vm = vm_create_shape_with_one_vcpu(shape, &vcpu, guest_code); - alignment = guest_page_size = vm_guest_mode_params[VM_MODE_DEFAULT].page_size; - gpa = (vm->max_gfn - TEST_NPAGES) * guest_page_size; + guest_page_size = vm_guest_mode_params[guest_mode].page_size; + host_page_size = getpagesize(); + backing_src_pagesz = get_backing_src_pagesz(p->mem_backing_src); + mem_page_size = max(host_page_size, backing_src_pagesz); + + test_config.page_size = guest_page_size; + test_config.test_size = align_up(TEST_BASE_SIZE + test_config.page_size, + mem_page_size); + test_config.test_num_pages = vm_calc_num_guest_pages(vm->mode, test_config.test_size); + + gpa = (vm->max_gfn - test_config.test_num_pages) * test_config.page_size; alignment = SZ_2M; + alignment = max(alignment, mem_page_size); gpa = align_down(gpa, alignment); gva = gpa & ((1ULL << (vm->va_bits - 1)) - 1); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, gpa, TEST_SLOT, - TEST_NPAGES, private ? KVM_MEM_GUEST_MEMFD : 0); - virt_map(vm, gva, gpa, TEST_NPAGES); + vm_userspace_mem_region_add(vm, p->mem_backing_src, + gpa, TEST_SLOT, test_config.test_num_pages, + p->private ? KVM_MEM_GUEST_MEMFD : 0); + virt_map(vm, gva, gpa, test_config.test_num_pages); - if (private) - vm_mem_set_private(vm, gpa, TEST_SIZE); + if (p->private) + vm_mem_set_private(vm, gpa, test_config.test_size); - pre_fault_memory(vcpu, gpa, 0, SZ_2M, 0, private); - pre_fault_memory(vcpu, gpa, SZ_2M, PAGE_SIZE * 2, PAGE_SIZE, private); - pre_fault_memory(vcpu, gpa, TEST_SIZE, PAGE_SIZE, PAGE_SIZE, private); + pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, + p->mem_backing_src, p->private); + /* Retry the same range after the first prefault attempt. */ + pre_fault_memory(vcpu, gpa, 0, test_config.test_size, 0, + p->mem_backing_src, p->private); + pre_fault_memory(vcpu, gpa, + test_config.test_size - host_page_size, + host_page_size * 2, host_page_size, + p->mem_backing_src, p->private); + pre_fault_memory(vcpu, gpa, test_config.test_size, + host_page_size, host_page_size, + p->mem_backing_src, p->private); vcpu_args_set(vcpu, 1, gva); + + /* Export the shared variables to the guest. */ + sync_global_to_guest(vm, test_config); + vcpu_run(vcpu); + run = vcpu->run; + TEST_ASSERT(run->exit_reason == UCALL_EXIT_REASON, + "Wanted %s, got exit reason: %u (%s)", + exit_reason_str(UCALL_EXIT_REASON), + run->exit_reason, exit_reason_str(run->exit_reason)); + switch (get_ucall(vcpu, &uc)) { case UCALL_ABORT: REPORT_GUEST_ASSERT(uc); @@ -207,24 +264,61 @@ static void __test_pre_fault_memory(unsigned long vm_type, bool private) kvm_vm_free(vm); } -static void test_pre_fault_memory(unsigned long vm_type, bool private) +static void test_pre_fault_memory(unsigned long vm_type, enum vm_mem_backing_src_type backing_src, + bool private) { + struct test_params p = { + .vm_type = vm_type, + .private = private, + .mem_backing_src = backing_src, + }; + if (vm_type && !(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(vm_type))) { pr_info("Skipping tests for vm_type 0x%lx\n", vm_type); return; } - __test_pre_fault_memory(vm_type, private); + for_each_guest_mode(__test_pre_fault_memory, &p); +} + +static void help(char *name) +{ + puts(""); + printf("usage: %s [-h] [-m mode] [-s mem-type]\n", name); + puts(""); + guest_modes_help(); + backing_src_help("-s"); + puts(""); } int main(int argc, char *argv[]) { + enum vm_mem_backing_src_type backing = DEFAULT_VM_MEM_SRC; + int opt; + + guest_modes_append_default(); + + while ((opt = getopt(argc, argv, "hm:s:")) != -1) { + switch (opt) { + case 'm': + guest_modes_cmdline(optarg); + break; + case 's': + backing = parse_backing_src_type(optarg); + break; + case 'h': + default: + help(argv[0]); + exit(0); + } + } + TEST_REQUIRE(kvm_check_cap(KVM_CAP_PRE_FAULT_MEMORY)); - test_pre_fault_memory(0, false); + test_pre_fault_memory(0, backing, false); #ifdef __x86_64__ - test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, false); - test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, true); + test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, backing, false); + test_pre_fault_memory(KVM_X86_SW_PROTECTED_VM, backing, true); #endif return 0; } diff --git a/tools/testing/selftests/kvm/s390/cmma_test.c b/tools/testing/selftests/kvm/s390/cmma_test.c index 15d81b2ed7ad..73387cc3bca7 100644 --- a/tools/testing/selftests/kvm/s390/cmma_test.c +++ b/tools/testing/selftests/kvm/s390/cmma_test.c @@ -102,24 +102,17 @@ static void guest_dirty_test_data(void) static void create_main_memslot(struct kvm_vm *vm) { - int i; - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, 0, 0, MAIN_PAGE_COUNT, 0); - /* set the array of memslots to zero like __vm_create does */ - for (i = 0; i < NR_MEM_REGIONS; i++) - vm->memslots[i] = 0; + ____vm_override_mem_region(vm, MEM_REGION_CODE, 0); + ____vm_override_mem_region(vm, MEM_REGION_PT, 0); + ____vm_override_mem_region(vm, MEM_REGION_DATA, 0); } static void create_test_memslot(struct kvm_vm *vm) { - vm_userspace_mem_region_add(vm, - VM_MEM_SRC_ANONYMOUS, - TEST_DATA_START_GFN << vm->page_shift, - TEST_DATA_MEMSLOT, - TEST_DATA_PAGE_COUNT, - 0 - ); - vm->memslots[MEM_REGION_TEST_DATA] = TEST_DATA_MEMSLOT; + vm_override_mem_region(vm, MEM_REGION_TEST_DATA, VM_MEM_SRC_ANONYMOUS, + TEST_DATA_START_GFN << vm->page_shift, + TEST_DATA_MEMSLOT, TEST_DATA_PAGE_COUNT); } static void create_memslots(struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/s390/irq_injection.c b/tools/testing/selftests/kvm/s390/irq_injection.c new file mode 100644 index 000000000000..75e0528f9b01 --- /dev/null +++ b/tools/testing/selftests/kvm/s390/irq_injection.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Test for KVM_S390_IRQ - s390x IRQ injection API + * + * Verifies that VM-level interrupt types are correctly rejected when + * injected via the vCPU file descriptor, and that invalid arguments + * to vCPU-level interrupts are also rejected. + * + * Copyright IBM Corp. 2026 + * Author: Gabriella Seifert + */ +#include +#include +#include +#include +#include +#include +#include "kvm_util.h" +#include "test_util.h" +#include + +static void test_vm_irq_on_vcpu_fails(struct kvm_vcpu *vcpu, __u64 type) +{ + struct kvm_s390_irq irq = { .type = type }; + int rc; + + rc = __vcpu_ioctl(vcpu, KVM_S390_IRQ, &irq); + ksft_test_result(rc == -1 && errno == EINVAL, + "VM on VCPU IRQ type 0x%llx: expected %d, got %d\n", + type, EINVAL, errno); +} + +static void test_vcpu_irq_bad_param_fails(struct kvm_vcpu *vcpu, + struct kvm_s390_irq *irq) +{ + int rc; + + rc = __vcpu_ioctl(vcpu, KVM_S390_IRQ, irq); + ksft_test_result(rc == -1 && errno == EINVAL, + "VCPU IRQ bad param type 0x%llx: expected %d got %d\n", + irq->type, EINVAL, errno); +} + +int main(void) +{ + struct kvm_s390_irq irq; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + + ksft_print_header(); + ksft_set_plan(6); + vm = vm_create_with_one_vcpu(&vcpu, NULL); + + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_VIRTIO); + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_SERVICE); + test_vm_irq_on_vcpu_fails(vcpu, KVM_S390_INT_IO_MIN); + + memset(&irq, 0, sizeof(irq)); + irq.type = KVM_S390_INT_EMERGENCY; + irq.u.emerg.code = 0xffff; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + memset(&irq, 0, sizeof(irq)); + irq.type = KVM_S390_INT_EXTERNAL_CALL; + irq.u.extcall.code = 0xffff; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + memset(&irq, 0, sizeof(irq)); + irq.type = 0xdeadbeef; + test_vcpu_irq_bad_param_fails(vcpu, &irq); + + kvm_vm_free(vm); + ksft_finished(); +} diff --git a/tools/testing/selftests/kvm/s390/irq_routing.c b/tools/testing/selftests/kvm/s390/irq_routing.c index f3839284ac08..cd26cc1aed80 100644 --- a/tools/testing/selftests/kvm/s390/irq_routing.c +++ b/tools/testing/selftests/kvm/s390/irq_routing.c @@ -36,7 +36,7 @@ static void test(void) }; vm = vm_create_with_one_vcpu(&vcpu, guest_code); - mem = vm_phy_pages_alloc(vm, 2, 4096 * 42, 0); + mem = ____vm_phy_pages_alloc(vm, 2, 4096 * 42, 0, false, false); routing = kvm_gsi_routing_create(); routing->nr = 1; diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index dffe9654f45b..bfa5ac8ce029 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -116,15 +116,15 @@ static struct kvm_vm *spawn_vm(struct kvm_vcpu **vcpu, pthread_t *vcpu_thread, vm = vm_create_with_one_vcpu(vcpu, guest_code); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP, - MEM_REGION_GPA, MEM_REGION_SLOT, - MEM_REGION_SIZE / getpagesize(), 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS_THP, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / getpagesize()); /* * Allocate and map two pages so that the GPA accessed by guest_code() * stays valid across the memslot move. */ - gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT); + gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_TEST_EXTRA); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n"); virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2); diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index 785d19f9ee8f..82b94a82dd03 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -76,12 +76,12 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); - vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_MSR_ENABLED); + vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (unsigned long)st_gva[i] | KVM_MSR_ENABLED); } static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct kvm_steal_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" steal: %lld\n", st->steal); @@ -105,7 +105,7 @@ static void check_steal_time_uapi(void) vm = vm_create_with_one_vcpu(&vcpu, NULL); ret = _vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, - (ulong)ST_GPA_BASE | KVM_STEAL_RESERVED_MASK); + (unsigned long)ST_GPA_BASE | KVM_STEAL_RESERVED_MASK); TEST_ASSERT(ret == 0, "Bad GPA didn't fail"); kvm_vm_free(vm); @@ -154,7 +154,7 @@ static void guest_code(int cpu) status = smccc(PV_TIME_ST, 0); GUEST_ASSERT_NE(status, -1); - GUEST_ASSERT_EQ(status, (ulong)st_gva[cpu]); + GUEST_ASSERT_EQ(status, (unsigned long)st_gva[cpu]); st = (struct st_time *)status; GUEST_SYNC(0); @@ -192,13 +192,13 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) /* ST_GPA_BASE is identity mapped */ WRITE_AND_SYNC_TO_GUEST(vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); - st_ipa = (ulong)st_gva[i]; + st_ipa = (unsigned long)st_gva[i]; vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); } static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct st_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct st_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" rev: %d\n", st->rev); @@ -222,11 +222,11 @@ static void check_steal_time_uapi(void) vm = vm_create_steal_time(1, NULL, &vcpu); vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &dev); - st_ipa = (ulong)ST_GPA_BASE | 1; + st_ipa = (unsigned long)ST_GPA_BASE | 1; ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); TEST_ASSERT(ret == -1 && errno == EINVAL, "Bad IPA didn't report EINVAL"); - st_ipa = (ulong)ST_GPA_BASE; + st_ipa = (unsigned long)ST_GPA_BASE; vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); ret = __vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); @@ -315,7 +315,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct sta_struct *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct sta_struct *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); int i; pr_info("VCPU%d:\n", vcpu_idx); @@ -451,7 +451,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) { - struct kvm_steal_time *st = addr_gva2hva(vm, (ulong)st_gva[vcpu_idx]); + struct kvm_steal_time *st = addr_gva2hva(vm, (unsigned long)st_gva[vcpu_idx]); ksft_print_msg("VCPU%d:\n", vcpu_idx); ksft_print_msg(" steal: %lld\n", st->steal); diff --git a/tools/testing/selftests/kvm/x86/amx_test.c b/tools/testing/selftests/kvm/x86/amx_test.c index 4e63da2b1889..66b6e442c41c 100644 --- a/tools/testing/selftests/kvm/x86/amx_test.c +++ b/tools/testing/selftests/kvm/x86/amx_test.c @@ -325,7 +325,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } break; case UCALL_DONE: diff --git a/tools/testing/selftests/kvm/x86/aperfmperf_test.c b/tools/testing/selftests/kvm/x86/aperfmperf_test.c index 845cb685f174..1ca6495a446f 100644 --- a/tools/testing/selftests/kvm/x86/aperfmperf_test.c +++ b/tools/testing/selftests/kvm/x86/aperfmperf_test.c @@ -70,8 +70,8 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); @@ -80,10 +80,10 @@ static void l1_vmx_code(struct vmx_pages *vmx) * in the VMCS by prepare_vmcs()), as MSR exiting mandatory on Intel. */ vmwrite(CPU_BASED_VM_EXEC_CONTROL, - vmreadz(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); + vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmlaunch()); + vmwrite(GUEST_RIP, (u64)l2_guest_code); + vmlaunch(); } static void guest_code(void *nested_test_data) diff --git a/tools/testing/selftests/kvm/x86/cpuid_test.c b/tools/testing/selftests/kvm/x86/cpuid_test.c index ef0ddd240887..dcd99a0501ee 100644 --- a/tools/testing/selftests/kvm/x86/cpuid_test.c +++ b/tools/testing/selftests/kvm/x86/cpuid_test.c @@ -128,7 +128,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage + 1, "Stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); return; case UCALL_DONE: return; diff --git a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c index aa7f3b405fd3..4a65eb02349c 100644 --- a/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c +++ b/tools/testing/selftests/kvm/x86/evmcs_smm_controls_test.c @@ -55,11 +55,11 @@ static void guest_code(struct vmx_pages *vmx_pages, enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist); evmcs_enable(); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_evmcs(hv_pages)); + prepare_for_vmx_operation(vmx_pages); + load_evmcs(hv_pages); prepare_vmcs(vmx_pages, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 exits via vmcall if test fails */ sync_with_host(2); diff --git a/tools/testing/selftests/kvm/x86/feature_msrs_test.c b/tools/testing/selftests/kvm/x86/feature_msrs_test.c index 158550701771..d56d0ddb3a57 100644 --- a/tools/testing/selftests/kvm/x86/feature_msrs_test.c +++ b/tools/testing/selftests/kvm/x86/feature_msrs_test.c @@ -59,7 +59,7 @@ static void test_feature_msr(u32 msr) * but initializes the vCPU's revision ID to an arbitrary value. */ if (msr == MSR_IA32_UCODE_REV) - reset_value = host_cpu_is_intel ? 0x100000000ULL : 0x01000065; + reset_value = host_cpu_is_intel_compatible ? 0x100000000ULL : 0x01000065; /* * For quirked MSRs, KVM's ABI is to initialize the vCPU's value to the diff --git a/tools/testing/selftests/kvm/x86/fix_hypercall_test.c b/tools/testing/selftests/kvm/x86/fix_hypercall_test.c index 4931ec22768e..f11fcc520f24 100644 --- a/tools/testing/selftests/kvm/x86/fix_hypercall_test.c +++ b/tools/testing/selftests/kvm/x86/fix_hypercall_test.c @@ -48,7 +48,7 @@ static void guest_main(void) const u8 *other_hypercall_insn; u64 ret; - if (host_cpu_is_intel) { + if (host_cpu_is_intel_compatible) { native_hypercall_insn = vmx_vmcall; other_hypercall_insn = svm_vmmcall; } else if (host_cpu_is_amd_compatible) { diff --git a/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c new file mode 100644 index 000000000000..e1c84df0f059 --- /dev/null +++ b/tools/testing/selftests/kvm/x86/guest_memfd_conversions_test.c @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, Google LLC. + */ +#include +#include +#include +#include + +#include +#include +#include + +#include "kvm_util.h" +#include "kvm_syscalls.h" +#include "kselftest_harness.h" +#include "test_util.h" +#include "ucall_common.h" + +FIXTURE(gmem_conversions) { + struct kvm_vcpu *vcpu; + int gmem_fd; + /* HVA of the first byte of the memory mmap()-ed from gmem_fd. */ + char *mem; +}; + +typedef FIXTURE_DATA(gmem_conversions) test_data_t; + +FIXTURE_SETUP(gmem_conversions) { } + +static size_t page_size; + +static void guest_do_rmw(void); +#define GUEST_MEMFD_SHARING_TEST_GVA 0x90000000ULL + +/* + * Defer setup until the individual test is invoked so that tests can specify + * the number of pages and flags for the guest_memfd instance. + */ +static void gmem_conversions_do_setup(test_data_t *t, int nr_pages, + int gmem_flags) +{ + const struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, + }; + /* + * Use high GPA above APIC_DEFAULT_PHYS_BASE to avoid clashing with + * APIC_DEFAULT_PHYS_BASE. + */ + const gpa_t gpa = SZ_4G; + const u32 slot = 1; + struct kvm_vm *vm; + + vm = __vm_create_shape_with_one_vcpu(shape, &t->vcpu, nr_pages, guest_do_rmw); + + vm_mem_add(vm, VM_MEM_SRC_SHMEM, gpa, slot, nr_pages, + KVM_MEM_GUEST_MEMFD, -1, 0, gmem_flags); + + t->gmem_fd = kvm_slot_to_fd(vm, slot); + t->mem = addr_gpa2hva(vm, gpa); + virt_map(vm, GUEST_MEMFD_SHARING_TEST_GVA, gpa, nr_pages); +} + +static void gmem_conversions_do_teardown(test_data_t *t) +{ + /* Use NULL to avoid second free in FIXTURE_TEARDOWN (multipage tests). */ + if (!t->vcpu) + return; + + /* No need to close gmem_fd, it's owned by the VM structure. */ + kvm_vm_free(t->vcpu->vm); + t->vcpu = NULL; +} + +FIXTURE_TEARDOWN(gmem_conversions) +{ + gmem_conversions_do_teardown(self); +} + +/* + * In these test definition macros, __nr_pages and nr_pages is used to set up + * the total number of pages in the guest_memfd under test. This will be + * available in the test definitions as nr_pages. + */ + +#define __GMEM_CONVERSION_TEST(test, __nr_pages, flags) \ +static void __gmem_conversions_##test(test_data_t *t, int nr_pages); \ + \ +TEST_F(gmem_conversions, test) \ +{ \ + const int nr = (__nr_pages); \ + \ + gmem_conversions_do_setup(self, nr, flags); \ + __gmem_conversions_##test(self, nr); \ +} \ +static void __gmem_conversions_##test(test_data_t *t, int nr_pages) \ + +#define GMEM_CONVERSION_TEST(test, __nr_pages, flags) \ + __GMEM_CONVERSION_TEST(test, __nr_pages, (flags) | GUEST_MEMFD_FLAG_MMAP) + +#define __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, __nr_pages) \ + GMEM_CONVERSION_TEST(test, __nr_pages, 0) + +#define GMEM_CONVERSION_TEST_INIT_PRIVATE(test) \ + __GMEM_CONVERSION_TEST_INIT_PRIVATE(test, 1) + +#define __GMEM_CONVERSION_TEST_INIT_SHARED(test, __nr_pages) \ + GMEM_CONVERSION_TEST(test, __nr_pages, GUEST_MEMFD_FLAG_INIT_SHARED) + +#define GMEM_CONVERSION_TEST_INIT_SHARED(test) \ + __GMEM_CONVERSION_TEST_INIT_SHARED(test, 1) + +/* + * Repeats test over nr_pages in a guest_memfd of size nr_pages, providing each + * test iteration with test_page, the index of the page under test in + * guest_memfd. test_page takes values 0..(nr_pages - 1) inclusive. + */ +#define GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(test, __nr_pages) \ +static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \ + const int test_page); \ + \ +TEST_F(gmem_conversions, test) \ +{ \ + const u64 flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; \ + const int nr = (__nr_pages); \ + int i; \ + \ + for (i = 0; i < nr; ++i) { \ + gmem_conversions_do_setup(self, nr, flags); \ + __gmem_conversions_multipage_##test(self, nr, i); \ + gmem_conversions_do_teardown(self); \ + } \ +} \ +static void __gmem_conversions_multipage_##test(test_data_t *t, int nr_pages, \ + const int test_page) + +struct guest_check_data { + void *mem; + char expected_val; + char write_val; +}; +static struct guest_check_data guest_data; + +static void guest_do_rmw(void) +{ + for (;;) { + char *mem = READ_ONCE(guest_data.mem); + + GUEST_ASSERT_EQ(READ_ONCE(*mem), READ_ONCE(guest_data.expected_val)); + WRITE_ONCE(*mem, READ_ONCE(guest_data.write_val)); + + GUEST_SYNC(0); + } +} + +static void run_guest_do_rmw(struct kvm_vcpu *vcpu, u64 pgoff, + char expected_val, char write_val) +{ + struct ucall uc; + int r; + + guest_data.mem = (void *)GUEST_MEMFD_SHARING_TEST_GVA + pgoff * page_size; + guest_data.expected_val = expected_val; + guest_data.write_val = write_val; + sync_global_to_guest(vcpu->vm, guest_data); + + do { + r = __vcpu_run(vcpu); + } while (r == -1 && errno == EINTR); + + TEST_ASSERT_EQ(r, 0); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_SYNC: + break; + default: + TEST_FAIL("Unexpected ucall %lu", uc.cmd); + } +} + +static void host_do_rmw(char *mem, u64 pgoff, char expected_val, + char write_val) +{ + TEST_ASSERT_EQ(READ_ONCE(mem[pgoff * page_size]), expected_val); + WRITE_ONCE(mem[pgoff * page_size], write_val); +} + +static void test_private(test_data_t *t, u64 pgoff, char starting_val, + char write_val) +{ + TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[pgoff * page_size], write_val)); + run_guest_do_rmw(t->vcpu, pgoff, starting_val, write_val); + TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[pgoff * page_size])); +} + +static void test_convert_to_private(test_data_t *t, u64 pgoff, + char starting_val, char write_val) +{ + gmem_set_private(t->gmem_fd, pgoff * page_size, page_size); + test_private(t, pgoff, starting_val, write_val); +} + +static void test_shared(test_data_t *t, u64 pgoff, char starting_val, + char host_write_val, char write_val) +{ + host_do_rmw(t->mem, pgoff, starting_val, host_write_val); + run_guest_do_rmw(t->vcpu, pgoff, host_write_val, write_val); + TEST_ASSERT_EQ(READ_ONCE(t->mem[pgoff * page_size]), write_val); +} + +static void test_convert_to_shared(test_data_t *t, u64 pgoff, + char starting_val, char host_write_val, + char write_val) +{ + gmem_set_shared(t->gmem_fd, pgoff * page_size, page_size); + test_shared(t, pgoff, starting_val, host_write_val, write_val); +} + +GMEM_CONVERSION_TEST_INIT_PRIVATE(init_private) +{ + test_private(t, 0, 0, 'A'); + test_convert_to_shared(t, 0, 'A', 'B', 'C'); + test_convert_to_private(t, 0, 'C', 'E'); +} + +GMEM_CONVERSION_TEST_INIT_SHARED(init_shared) +{ + test_shared(t, 0, 0, 'A', 'B'); + test_convert_to_private(t, 0, 'B', 'C'); + test_convert_to_shared(t, 0, 'C', 'D', 'E'); +} + +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(indexing, 4) +{ + int i; + + /* Get a char that varies with both i and n. */ +#define combine(x, n) (((x) << 4) + (n)) +#define i_(n) (combine(i, n)) +#define t_(n) (combine(test_page, n)) + + /* + * Start with the highest index, to catch any errors when, perhaps, the + * first page is returned even for the last index. + */ + for (i = nr_pages - 1; i >= 0; --i) + test_shared(t, i, 0, i_(0), i_(2)); + + test_convert_to_private(t, test_page, t_(2), t_(3)); + + for (i = 0; i < nr_pages; ++i) { + if (i == test_page) + test_private(t, test_page, t_(3), t_(4)); + else + test_shared(t, i, i_(2), i_(3), i_(4)); + } + + test_convert_to_shared(t, test_page, t_(4), t_(5), t_(6)); + + for (i = 0; i < nr_pages; ++i) { + char expected = i == test_page ? t_(6) : i_(4); + + test_shared(t, i, expected, i_(7), i_(8)); + } + +#undef t_ +#undef i_ +#undef combine +} + +/* + * Test that even if there are no folios yet, conversion requests are recorded + * in guest_memfd. + */ +GMEM_CONVERSION_TEST_INIT_SHARED(before_allocation_shared) +{ + test_convert_to_private(t, 0, 0, 'A'); +} + +GMEM_CONVERSION_TEST_INIT_PRIVATE(before_allocation_private) +{ + test_convert_to_shared(t, 0, 0, 'A', 'B'); +} + +/* + * Test that when some of the folios in the conversion range are allocated, + * conversion requests are handled correctly in guest_memfd. Vary the ranges + * allocated before conversion, using test_page, to cover various layouts of + * contiguous allocated and unallocated regions. + */ +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(unallocated_folios, 8) +{ + const int second_page_to_fault = 4; + int i; + + /* + * Fault 2 of the pages to test filemap range operations except when + * test_page == second_page_to_fault. + */ + host_do_rmw(t->mem, test_page, 0, 'A'); + if (test_page != second_page_to_fault) + host_do_rmw(t->mem, second_page_to_fault, 0, 'A'); + + gmem_set_private(t->gmem_fd, 0, nr_pages * page_size); + for (i = 0; i < nr_pages; ++i) { + char expected = (i == test_page || i == second_page_to_fault) ? 'A' : 0; + + test_private(t, i, expected, 'B'); + } + + for (i = 0; i < nr_pages; ++i) + test_convert_to_shared(t, i, 'B', 'C', 'D'); +} + +/* Truncation should not affect shared/private status. */ +GMEM_CONVERSION_TEST_INIT_SHARED(truncate) +{ + host_do_rmw(t->mem, 0, 0, 'A'); + kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size); + host_do_rmw(t->mem, 0, 0, 'A'); + + test_convert_to_private(t, 0, 'A', 'B'); + + kvm_fallocate(t->gmem_fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, page_size); + test_private(t, 0, 0, 'A'); +} + +/* Test that shared/private memory protections work and are seen from any process. */ +GMEM_CONVERSION_TEST_INIT_SHARED(forked_accesses) +{ + enum test_state { + STATE_INIT, + STATE_CHECK_SHARED, + STATE_DONE_CHECKING_SHARED, + STATE_CHECK_PRIVATE, + STATE_DONE_CHECKING_PRIVATE, + }; + + struct sync_state { + pthread_mutex_t mutex; + pthread_cond_t cond; + enum test_state step; + } *sync; + + pthread_mutexattr_t mattr; + pthread_condattr_t cattr; + pid_t child_pid, parent_pid; + int status; + + sync = kvm_mmap(sizeof(*sync), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1); + + pthread_mutexattr_init(&mattr); + pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); + pthread_mutex_init(&sync->mutex, &mattr); + pthread_mutexattr_destroy(&mattr); + + pthread_condattr_init(&cattr); + pthread_condattr_setpshared(&cattr, PTHREAD_PROCESS_SHARED); + pthread_cond_init(&sync->cond, &cattr); + pthread_condattr_destroy(&cattr); + + sync->step = STATE_INIT; + +#define TEST_STATE_AWAIT(__state) \ + do { \ + pthread_mutex_lock(&sync->mutex); \ + while (sync->step != (__state)) { \ + struct timespec ts, stop; \ + int ret; \ + \ + clock_gettime(CLOCK_REALTIME, &ts); \ + stop = timespec_add_ns(ts, 100 * 1000000UL); \ + \ + ret = pthread_cond_timedwait(&sync->cond, &sync->mutex, &stop); \ + if (ret == ETIMEDOUT) { \ + bool alive = (child_pid == 0) ? \ + (getppid() == parent_pid) : \ + (waitpid(child_pid, NULL, WNOHANG) == 0); \ + TEST_ASSERT(alive, "Other process exited prematurely"); \ + } else { \ + TEST_ASSERT(!ret, "pthread_cond_timedwait failed"); \ + } \ + } \ + pthread_mutex_unlock(&sync->mutex); \ + } while (0) + +#define TEST_STATE_SET(__state) \ + do { \ + pthread_mutex_lock(&sync->mutex); \ + sync->step = (__state); \ + pthread_cond_broadcast(&sync->cond); \ + pthread_mutex_unlock(&sync->mutex); \ + } while (0) + + parent_pid = getpid(); + child_pid = fork(); + TEST_ASSERT(child_pid != -1, "fork failed"); + + if (child_pid == 0) { + const char inconsequential = 0xdd; + + TEST_STATE_AWAIT(STATE_CHECK_SHARED); + + /* + * This maps the pages into the child process as well, and tests + * that the conversion process will unmap the guest_memfd memory + * from all processes. + */ + host_do_rmw(t->mem, 0, 0xB, 0xC); + + TEST_STATE_SET(STATE_DONE_CHECKING_SHARED); + TEST_STATE_AWAIT(STATE_CHECK_PRIVATE); + + TEST_EXPECT_SIGBUS(READ_ONCE(t->mem[0])); + TEST_EXPECT_SIGBUS(WRITE_ONCE(t->mem[0], inconsequential)); + + TEST_STATE_SET(STATE_DONE_CHECKING_PRIVATE); + exit(0); + } + + test_shared(t, 0, 0, 0xA, 0xB); + + TEST_STATE_SET(STATE_CHECK_SHARED); + TEST_STATE_AWAIT(STATE_DONE_CHECKING_SHARED); + + test_convert_to_private(t, 0, 0xC, 0xD); + + TEST_STATE_SET(STATE_CHECK_PRIVATE); + TEST_STATE_AWAIT(STATE_DONE_CHECKING_PRIVATE); + + TEST_ASSERT_EQ(waitpid(child_pid, &status, 0), child_pid); + TEST_ASSERT(WIFEXITED(status) && WEXITSTATUS(status) == 0, + "Child exited with unexpected status"); + + pthread_mutex_destroy(&sync->mutex); + pthread_cond_destroy(&sync->cond); + kvm_munmap(sync, sizeof(*sync)); + +#undef TEST_STATE_SET +#undef TEST_STATE_AWAIT +} + +static void test_convert_to_private_fails(test_data_t *t, u64 pgoff, + size_t nr_pages, + u64 expected_error_offset) +{ + /* +1 to make it anything but expected_error_offset. */ + u64 error_offset = expected_error_offset + 1; + u64 offset = pgoff * page_size; + int ret; + + do { + ret = __gmem_set_private(t->gmem_fd, offset, + nr_pages * page_size, &error_offset); + } while (ret == -1 && errno == EINTR); + TEST_ASSERT(ret == -1 && errno == EAGAIN, + "Wanted EAGAIN on page %lu, got %d (ret = %d)", pgoff, + errno, ret); + TEST_ASSERT_EQ(error_offset, expected_error_offset); +} + +GMEM_CONVERSION_MULTIPAGE_TEST_INIT_SHARED(elevated_refcount, 4) +{ + int i; + + pin_pages(t->mem + test_page * page_size, page_size); + + for (i = 0; i < nr_pages; i++) + test_shared(t, i, 0, 'A', 'B'); + + /* + * Converting in bulk should fail as long any page in the range has + * unexpected refcounts. + */ + test_convert_to_private_fails(t, 0, nr_pages, test_page * page_size); + + for (i = 0; i < nr_pages; i++) { + /* + * Converting page-wise should also fail as long any page in the + * range has unexpected refcounts. + */ + if (i == test_page) + test_convert_to_private_fails(t, i, 1, test_page * page_size); + else + test_convert_to_private(t, i, 'B', 'C'); + } + + unpin_pages(); + + gmem_set_private(t->gmem_fd, 0, nr_pages * page_size); + + for (i = 0; i < nr_pages; i++) { + char expected = i == test_page ? 'B' : 'C'; + + test_private(t, i, expected, 'D'); + } +} + +int main(int argc, char *argv[]) +{ + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES) & + KVM_MEMORY_ATTRIBUTE_PRIVATE); + + page_size = getpagesize(); + + return test_harness_run(argc, argv); +} diff --git a/tools/testing/selftests/kvm/x86/hyperv_clock.c b/tools/testing/selftests/kvm/x86/hyperv_clock.c index d5d779623cc6..7a1b3b5d1e62 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_clock.c +++ b/tools/testing/selftests/kvm/x86/hyperv_clock.c @@ -248,7 +248,7 @@ int main(void) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); /* Reset kvmclock triggering TSC page update */ if (stage == 7 || stage == 8 || stage == 10) { diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 63ea1533e4ea..c81f2dd4f2a1 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -23,7 +23,11 @@ static int ud_count; static void guest_ud_handler(struct ex_regs *regs) { ud_count++; - regs->rip += 3; /* VMLAUNCH */ + /* + * VMLAUNCH insn can't be easily covered by KVM_ASM_SAFE framework but + * luckily the instruction is always three bytes. + */ + regs->rip += 3; } static void guest_nmi_handler(struct ex_regs *regs) @@ -89,24 +93,23 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, enable_vp_assist(hv_pages->vp_assist_gpa, hv_pages->vp_assist); evmcs_enable(); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_SYNC(3); - GUEST_ASSERT(load_evmcs(hv_pages)); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); + load_evmcs(hv_pages); + /* VMPTRST returns -1 until VMLAUNCH with eVMCS ptr set */ + GUEST_ASSERT(vmptrst() == -1); GUEST_SYNC(4); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); prepare_vmcs(vmx_pages, l2_guest_code); GUEST_SYNC(5); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); current_evmcs->revision_id = -1u; - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); current_evmcs->revision_id = EVMCS_VERSION; GUEST_SYNC(6); - vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmreadz(PIN_BASED_VM_EXEC_CONTROL) | + vmwrite(PIN_BASED_VM_EXEC_CONTROL, vmread(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_NMI_EXITING); /* L2 TLB flush setup */ @@ -117,71 +120,79 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, current_vp_assist->nested_control.features.directhypercall = 1; *(u32 *)(hv_pages->partition_assist) = 0; - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR); - GUEST_ASSERT(vmptrstz() == hv_pages->enlightened_vmcs_gpa); + vmlaunch(); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ((vmread(VM_EXIT_INTR_INFO) & 0xff), NMI_VECTOR); + GUEST_ASSERT(vmptrst() == hv_pages->enlightened_vmcs_gpa); /* * NMI forces L2->L1 exit, resuming L2 and hope that EVMCS is * up-to-date (RIP points where it should and not at the beginning * of l2_guest_code(). GUEST_SYNC(9) checks that. */ - GUEST_ASSERT(!vmresume()); + vmresume(); GUEST_SYNC(10); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); current_evmcs->guest_rip += 3; /* vmcall */ /* Intercept RDMSR 0xc0000100 */ - vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | + vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_USE_MSR_BITMAPS); __set_bit(MSR_FS_BASE & 0x1fff, vmx_pages->msr + 0x400); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Enable enlightened MSR bitmap */ current_evmcs->hv_enlightenments_control.msr_bitmap = 1; - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Intercept RDMSR 0xc0000101 without telling KVM about it */ __set_bit(MSR_GS_BASE & 0x1fff, vmx_pages->msr + 0x400); /* Make sure HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP is set */ current_evmcs->hv_clean_fields |= HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; - GUEST_ASSERT(!vmresume()); + vmresume(); /* Make sure we don't see EXIT_REASON_MSR_READ here so eMSR bitmap works */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); current_evmcs->guest_rip += 3; /* vmcall */ /* Now tell KVM we've changed MSR-Bitmap */ current_evmcs->hv_clean_fields &= ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP; - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* * L2 TLB flush test. First VMCALL should be handled directly by L0, * no VMCALL exit expected. */ - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_MSR_READ); current_evmcs->guest_rip += 2; /* rdmsr */ /* Enable synthetic vmexit */ *(u32 *)(hv_pages->partition_assist) = 1; - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == HV_VMX_SYNTHETIC_EXIT_REASON_TRAP_AFTER_FLUSH); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(11); - /* Try enlightened vmptrld with an incorrect GPA */ + /* VMPTRLD instruction causes #UD after enlightened VMLAUNCH */ + GUEST_ASSERT(__vmptrld(hv_pages->enlightened_vmcs_gpa) == UD_VECTOR); + + /* + * Try enlightened vmptrld with an incorrect GPA. GUEST_SYNC(12) signals + * the host to enable guest_ud_handler() which cannot be enabled beforehand + * to not override the default fixup handler from KVM_ASM_SAFE(). + */ + GUEST_SYNC(12); evmcs_vmptrld(0xdeadbeef, hv_pages->enlightened_vmcs); - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); GUEST_ASSERT(ud_count == 1); GUEST_DONE(); } @@ -221,7 +232,7 @@ static struct kvm_vcpu *save_restore_vm(struct kvm_vm *vm, vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); return vcpu; } @@ -253,7 +264,6 @@ int main(int argc, char *argv[]) vcpu_args_set(vcpu, 3, vmx_pages_gva, hv_pages_gva, addr_gva2gpa(vm, hcall_page)); vcpu_set_msr(vcpu, HV_X64_MSR_VP_INDEX, vcpu->id); - vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); pr_info("Running L1 which uses EVMCS to run L2\n"); @@ -277,13 +287,13 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); vcpu = save_restore_vm(vm, vcpu); /* Force immediate L2->L1 exit before resuming */ if (stage == 8) { - pr_info("Injecting NMI into L1 before L2 had a chance to run after restore\n"); + pr_debug("Injecting NMI into L1 before L2 had a chance to run after restore\n"); inject_nmi(vcpu); } @@ -293,9 +303,14 @@ int main(int argc, char *argv[]) * KVM_STATE_NESTED_EVMCS is not lost. */ if (stage == 9) { - pr_info("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n"); + pr_debug("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n"); vcpu = save_restore_vm(vm, vcpu); } + + if (stage == 12) { + pr_debug("Trying enlightened VMLAUNCH with an invalid PTR\n"); + vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); + } } done: diff --git a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c index 1f74b0fa9b83..3a3dc31072e8 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_svm_test.c +++ b/tools/testing/selftests/kvm/x86/hyperv_svm_test.c @@ -186,7 +186,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); } diff --git a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c index 25a182be00a9..e1d2730b16ba 100644 --- a/tools/testing/selftests/kvm/x86/kvm_buslock_test.c +++ b/tools/testing/selftests/kvm/x86/kvm_buslock_test.c @@ -42,13 +42,13 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmlaunch()); + vmwrite(GUEST_RIP, (u64)l2_guest_code); + vmlaunch(); } static void guest_code(void *test_data) diff --git a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c index b974cfb347d6..1f4a5e44a6ef 100644 --- a/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c +++ b/tools/testing/selftests/kvm/x86/nested_close_kvm_test.c @@ -30,13 +30,13 @@ static void l2_guest_code(void) static void l1_vmx_code(struct vmx_pages *vmx_pages) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_ASSERT(0); } diff --git a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c index 26b474bf1353..7e5f9ae1cb00 100644 --- a/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86/nested_dirty_log_test.c @@ -93,8 +93,8 @@ void l1_vmx_code(struct vmx_pages *vmx) void *l2_rip; GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); if (vmx->eptp_gpa) l2_rip = l2_guest_code_tdp_enabled; @@ -104,9 +104,9 @@ void l1_vmx_code(struct vmx_pages *vmx) prepare_vmcs(vmx, l2_rip); GUEST_SYNC(TEST_SYNC_NO_FAULT); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); GUEST_SYNC(TEST_SYNC_NO_FAULT); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_VMCALL); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/nested_emulation_test.c b/tools/testing/selftests/kvm/x86/nested_emulation_test.c index e08c6b0697e5..68d9ce9dc6ba 100644 --- a/tools/testing/selftests/kvm/x86/nested_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/nested_emulation_test.c @@ -66,15 +66,15 @@ static void guest_code(void *test_data) BIT_ULL(INTERCEPT_HLT); vmcb->control.intercept_exceptions = 0; } else { - GUEST_ASSERT(prepare_for_vmx_operation(test_data)); - GUEST_ASSERT(load_vmcs(test_data)); + prepare_for_vmx_operation(test_data); + load_vmcs(test_data); prepare_vmcs(test_data, NULL); - GUEST_ASSERT(!vmwrite(GUEST_IDTR_LIMIT, 0)); - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_guest_code)); - GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, 0)); + vmwrite(GUEST_IDTR_LIMIT, 0); + vmwrite(GUEST_RIP, (u64)l2_guest_code); + vmwrite(EXCEPTION_BITMAP, 0); - vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmreadz(CPU_BASED_VM_EXEC_CONTROL) | + vmwrite(CPU_BASED_VM_EXEC_CONTROL, vmread(CPU_BASED_VM_EXEC_CONTROL) | CPU_BASED_PAUSE_EXITING | CPU_BASED_HLT_EXITING); } @@ -102,10 +102,13 @@ static void guest_code(void *test_data) exit_insn_len = vmcb->control.next_rip - vmcb->save.rip; GUEST_ASSERT_EQ(vmcb->save.rip, (u64)l2_instruction); } else { - GUEST_ASSERT_EQ(i ? vmresume() : vmlaunch(), 0); - exit_reason = vmreadz(VM_EXIT_REASON); - exit_insn_len = vmreadz(VM_EXIT_INSTRUCTION_LEN); - GUEST_ASSERT_EQ(vmreadz(GUEST_RIP), (u64)l2_instruction); + if (!i) + vmlaunch(); + else + vmresume(); + exit_reason = vmread(VM_EXIT_REASON); + exit_insn_len = vmread(VM_EXIT_INSTRUCTION_LEN); + GUEST_ASSERT_EQ(vmread(GUEST_RIP), (u64)l2_instruction); } __GUEST_ASSERT(exit_reason == insn->exit_reason[f], diff --git a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c index aeec3121c8e8..f47432526ea1 100644 --- a/tools/testing/selftests/kvm/x86/nested_exceptions_test.c +++ b/tools/testing/selftests/kvm/x86/nested_exceptions_test.c @@ -38,6 +38,7 @@ */ #define GP_ERROR_CODE_AMD ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG) #define GP_ERROR_CODE_INTEL ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG | ERROR_CODE_EXT_FLAG) +#define GP_ERROR_CODE_ZHAOXIN ((SS_VECTOR * 8) | ERROR_CODE_IDT_FLAG) /* * Intel and AMD both shove '0' into the error code on #DF, regardless of what @@ -110,43 +111,47 @@ static void l1_svm_code(struct svm_test_data *svm) static void vmx_run_l2(void *l2_code, int vector, u32 error_code) { - GUEST_ASSERT(!vmwrite(GUEST_RIP, (u64)l2_code)); + vmwrite(GUEST_RIP, (u64)l2_code); - GUEST_ASSERT_EQ(vector == SS_VECTOR ? vmlaunch() : vmresume(), 0); + if (vector == SS_VECTOR) + vmlaunch(); + else + vmresume(); if (vector == FAKE_TRIPLE_FAULT_VECTOR) return; - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ((vmreadz(VM_EXIT_INTR_INFO) & 0xff), vector); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_ERROR_CODE), error_code); - GUEST_ASSERT(!vmreadz(GUEST_INTERRUPTIBILITY_INFO)); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ((vmread(VM_EXIT_INTR_INFO) & 0xff), vector); + GUEST_ASSERT_EQ(vmread(VM_EXIT_INTR_ERROR_CODE), error_code); + GUEST_ASSERT(!vmread(GUEST_INTERRUPTIBILITY_INFO)); } static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + prepare_for_vmx_operation(vmx); - GUEST_ASSERT_EQ(load_vmcs(vmx), true); + load_vmcs(vmx); prepare_vmcs(vmx, NULL); - GUEST_ASSERT_EQ(vmwrite(GUEST_IDTR_LIMIT, 0), 0); + vmwrite(GUEST_IDTR_LIMIT, 0); /* * VMX disallows injecting an exception with error_code[31:16] != 0, * and hardware will never generate a VM-Exit with bits 31:16 set. * KVM should likewise truncate the "bad" userspace value. */ - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_GP_DF); vmx_run_l2(l2_ss_pending_test, SS_VECTOR, (u16)SS_ERROR_CODE); - vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, GP_ERROR_CODE_INTEL); + vmx_run_l2(l2_ss_injected_gp_test, GP_VECTOR, + host_cpu_is_zhaoxin ? GP_ERROR_CODE_ZHAOXIN : GP_ERROR_CODE_INTEL); - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS_DF); vmx_run_l2(l2_ss_injected_df_test, DF_VECTOR, DF_ERROR_CODE); - GUEST_ASSERT_EQ(vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS), 0); + vmwrite(EXCEPTION_BITMAP, INTERCEPT_SS); vmx_run_l2(l2_ss_injected_tf_test, FAKE_TRIPLE_FAULT_VECTOR, 0); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c index 8c2ba9674558..6e147826bc90 100644 --- a/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c +++ b/tools/testing/selftests/kvm/x86/nested_invalid_cr3_test.c @@ -40,22 +40,22 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) { uintptr_t save_cr3; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); /* Try to run L2 with invalid CR3 and make sure it fails */ - save_cr3 = vmreadz(GUEST_CR3); + save_cr3 = vmread(GUEST_CR3); vmwrite(GUEST_CR3, -1ull); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == (EXIT_REASON_FAILED_VMENTRY | EXIT_REASON_INVALID_STATE)); /* Now restore CR3 and make sure L2 runs successfully */ vmwrite(GUEST_CR3, save_cr3); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c index 2e04563790ff..fae77d843b13 100644 --- a/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tdp_fault_test.c @@ -55,25 +55,25 @@ static void l1_vmx_code(struct vmx_pages *vmx, u64 expected_fault_gpa, u64 exit_qual; GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, l2_entry); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* Verify we got an EPT violation exit */ - __GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION, + __GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION, "Expected EPT violation (0x%x), got 0x%lx", EXIT_REASON_EPT_VIOLATION, - vmreadz(VM_EXIT_REASON)); + vmread(VM_EXIT_REASON)); - __GUEST_ASSERT(vmreadz(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa, + __GUEST_ASSERT(vmread(GUEST_PHYSICAL_ADDRESS) == expected_fault_gpa, "Expected guest_physical_address = 0x%lx, got 0x%lx", expected_fault_gpa, - vmreadz(GUEST_PHYSICAL_ADDRESS)); + vmread(GUEST_PHYSICAL_ADDRESS)); - exit_qual = vmreadz(EXIT_QUALIFICATION); + exit_qual = vmread(EXIT_QUALIFICATION); /* * Note, EPT page table accesses are always read+write, e.g. so that diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c index cb79d7b9619c..6a1a88e926f8 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_adjust_test.c @@ -86,17 +86,17 @@ static void l1_guest_code(void *data) struct vmx_pages *vmx_pages = data; u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); vmwrite(TSC_OFFSET, TSC_OFFSET_VALUE); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); } else { struct svm_test_data *svm = data; diff --git a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c index 18f765835bf4..bfb1ff441c78 100644 --- a/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c +++ b/tools/testing/selftests/kvm/x86/nested_tsc_scaling_test.c @@ -1,14 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * vmx_nested_tsc_scaling_test - * * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. * * This test case verifies that nested TSC scaling behaves as expected when - * both L1 and L2 are scaled using different ratios. For this test we scale - * L1 down and scale L2 up. + * both L1 and L2 are scaled using different ratios. */ - +#include #include #include "kvm_util.h" @@ -16,16 +13,11 @@ #include "svm_util.h" #include "kselftest.h" -/* L2 is scaled up (from L1's perspective) by this factor */ -#define L2_SCALE_FACTOR 4ULL - -#define TSC_OFFSET_L2 ((u64)-33125236320908) -#define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48) - -enum { USLEEP, UCHECK_L1, UCHECK_L2 }; -#define GUEST_SLEEP(sec) ucall(UCALL_SYNC, 2, USLEEP, sec) -#define GUEST_CHECK(level, freq) ucall(UCALL_SYNC, 2, level, freq) +/* L2's TSC multiplier and offset, relative to L1. */ +static u64 l2_multiplier; +static u64 l2_offset; +enum ucall_cmd { UCALL_SLEEP, UCALL_CHECK_L1, UCALL_CHECK_L2 }; /* * This function checks whether the "actual" TSC frequency of a guest matches @@ -33,25 +25,20 @@ enum { USLEEP, UCHECK_L1, UCHECK_L2 }; * measurements, a difference of 1% between the actual and the expected value * is tolerated. */ -static void compare_tsc_freq(u64 actual, u64 expected) +static void host_check_tsc_freq(int level, u64 actual, u64 expected) { u64 tolerance, thresh_low, thresh_high; - tolerance = expected / 100; + tolerance = max(expected / 100, (u64)1); thresh_low = expected - tolerance; thresh_high = expected + tolerance; - TEST_ASSERT(thresh_low < actual, - "TSC freq is expected to be between %"PRIu64" and %"PRIu64 - " but it actually is %"PRIu64, - thresh_low, thresh_high, actual); - TEST_ASSERT(thresh_high > actual, - "TSC freq is expected to be between %"PRIu64" and %"PRIu64 - " but it actually is %"PRIu64, - thresh_low, thresh_high, actual); + TEST_ASSERT(thresh_low <= actual && thresh_high >= actual, + "L%u TSC freq is '%lu', expected to be between %lu and %lu", + level, actual, thresh_low, thresh_high); } -static void check_tsc_freq(int level) +static void guest_check_tsc_freq(enum ucall_cmd check_level) { u64 tsc_start, tsc_end, tsc_freq; @@ -62,17 +49,17 @@ static void check_tsc_freq(int level) * be good enough for the purposes of this test. */ tsc_start = rdmsr(MSR_IA32_TSC); - GUEST_SLEEP(1); + GUEST_SYNC2(UCALL_SLEEP, 1); tsc_end = rdmsr(MSR_IA32_TSC); tsc_freq = tsc_end - tsc_start; - GUEST_CHECK(level, tsc_freq); + GUEST_SYNC2(check_level, tsc_freq); } static void l2_guest_code(void) { - check_tsc_freq(UCHECK_L2); + guest_check_tsc_freq(UCALL_CHECK_L2); /* exit to L1 */ __asm__ __volatile__("vmcall"); @@ -81,19 +68,20 @@ static void l2_guest_code(void) static void l1_svm_code(struct svm_test_data *svm) { /* check that L1's frequency looks alright before launching L2 */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); generic_svm_setup(svm, l2_guest_code); + svm->vmcb->control.tsc_offset = l2_offset; /* enable TSC scaling for L2 */ - wrmsr(MSR_AMD64_TSC_RATIO, L2_SCALE_FACTOR << 32); + wrmsr(MSR_AMD64_TSC_RATIO, l2_multiplier); /* launch L2 */ run_guest(svm->vmcb, svm->vmcb_gpa); GUEST_ASSERT(svm->vmcb->control.exit_code == SVM_EXIT_VMMCALL); /* check that L1's frequency still looks good */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); GUEST_DONE(); } @@ -103,33 +91,32 @@ static void l1_vmx_code(struct vmx_pages *vmx_pages) u32 control; /* check that L1's frequency looks alright before launching L2 */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* prepare the VMCS for L2 execution */ prepare_vmcs(vmx_pages, l2_guest_code); /* enable TSC offsetting and TSC scaling for L2 */ - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control |= SECONDARY_EXEC_TSC_SCALING; vmwrite(SECONDARY_VM_EXEC_CONTROL, control); - vmwrite(TSC_OFFSET, TSC_OFFSET_L2); - vmwrite(TSC_MULTIPLIER, TSC_MULTIPLIER_L2); - vmwrite(TSC_MULTIPLIER_HIGH, TSC_MULTIPLIER_L2 >> 32); + vmwrite(TSC_OFFSET, l2_offset); + vmwrite(TSC_MULTIPLIER, l2_multiplier); /* launch L2 */ - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* check that L1's frequency still looks good */ - check_tsc_freq(UCHECK_L1); + guest_check_tsc_freq(UCALL_CHECK_L1); GUEST_DONE(); } @@ -142,44 +129,24 @@ static void l1_guest_code(void *data) l1_svm_code(data); } -int main(int argc, char *argv[]) +static void test_tsc_scaling(u64 l0_tsc_freq, u64 l1_tsc_freq, u64 l2_tsc_freq, + u64 __l2_multiplier) { struct kvm_vcpu *vcpu; struct kvm_vm *vm; - gva_t guest_gva = 0; + gva_t guest_gva; - u64 tsc_start, tsc_end; - u64 tsc_khz; - u64 l1_scale_factor; - u64 l0_tsc_freq = 0; - u64 l1_tsc_freq = 0; - u64 l2_tsc_freq = 0; - - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || - kvm_cpu_has(X86_FEATURE_SVM)); - TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL)); - TEST_REQUIRE(sys_clocksource_is_based_on_tsc()); - - /* - * We set L1's scale factor to be a random number from 2 to 10. - * Ideally we would do the same for L2's factor but that one is - * referenced by both main() and l1_guest_code() and using a global - * variable does not work. - */ - srand(time(NULL)); - l1_scale_factor = (rand() % 9) + 2; - printf("L1's scale down factor is: %"PRIu64"\n", l1_scale_factor); - printf("L2's scale up factor is: %llu\n", L2_SCALE_FACTOR); - - tsc_start = rdtsc(); - sleep(1); - tsc_end = rdtsc(); - - l0_tsc_freq = tsc_end - tsc_start; - printf("real TSC frequency is around: %"PRIu64"\n", l0_tsc_freq); + printf("Testing L0 freq = %lu, L1 freq = %lu, L2 freq = %lu, L2 mult = 0x%lx\n", + l0_tsc_freq, l1_tsc_freq, l2_tsc_freq, __l2_multiplier); vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + l2_offset = kvm_random_u64(&kvm_rng); + sync_global_to_guest(vm, l2_offset); + + l2_multiplier = __l2_multiplier; + sync_global_to_guest(vm, l2_multiplier); + if (kvm_cpu_has(X86_FEATURE_VMX)) vcpu_alloc_vmx(vm, &guest_gva); else @@ -187,11 +154,7 @@ int main(int argc, char *argv[]) vcpu_args_set(vcpu, 1, guest_gva); - tsc_khz = __vcpu_ioctl(vcpu, KVM_GET_TSC_KHZ, NULL); - TEST_ASSERT(tsc_khz != -1, "vcpu ioctl KVM_GET_TSC_KHZ failed"); - - /* scale down L1's TSC frequency */ - vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *) (tsc_khz / l1_scale_factor)); + vcpu_ioctl(vcpu, KVM_SET_TSC_KHZ, (void *)(l1_tsc_freq / 1000)); for (;;) { struct ucall uc; @@ -204,24 +167,16 @@ int main(int argc, char *argv[]) REPORT_GUEST_ASSERT(uc); case UCALL_SYNC: switch (uc.args[0]) { - case USLEEP: + case UCALL_SLEEP: sleep(uc.args[1]); break; - case UCHECK_L1: - l1_tsc_freq = uc.args[1]; - printf("L1's TSC frequency is around: %"PRIu64 - "\n", l1_tsc_freq); - - compare_tsc_freq(l1_tsc_freq, - l0_tsc_freq / l1_scale_factor); + case UCALL_CHECK_L1: + printf("L1's observed TSC frequency: %lu\n", uc.args[1]); + host_check_tsc_freq(1, uc.args[1], l1_tsc_freq); break; - case UCHECK_L2: - l2_tsc_freq = uc.args[1]; - printf("L2's TSC frequency is around: %"PRIu64 - "\n", l2_tsc_freq); - - compare_tsc_freq(l2_tsc_freq, - l1_tsc_freq * L2_SCALE_FACTOR); + case UCALL_CHECK_L2: + printf("L2's observed TSC frequency: %lu\n", uc.args[1]); + host_check_tsc_freq(2, uc.args[1], l2_tsc_freq); break; } break; @@ -234,5 +189,108 @@ int main(int argc, char *argv[]) done: kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + u64 min_freq, min_multiplier, max_freq, max_multiplier, l1_max_freq, l1_min_freq; + u64 l0_tsc_freq, tsc_start, tsc_end, l1_scale, l2_scale; + u8 frac_bits = kvm_cpu_has(X86_FEATURE_VMX) ? 48 : 32; + struct kvm_vm *vm; + + TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || + kvm_cpu_has(X86_FEATURE_SVM)); + TEST_REQUIRE(kvm_has_cap(KVM_CAP_TSC_CONTROL)); + TEST_REQUIRE(sys_clocksource_is_based_on_tsc()); + + /* + * Create a dummy VM to get KVM's default TSC frequency. All CPUs that + * support TSC scaling should have a constant TSC, i.e. there's no need + * to calibrate the "real" TSC. But do sanity check that the observed + * TSC is within range of KVM's reported TSC frequency. + */ + vm = vm_create_barebones(); + l0_tsc_freq = (u64)__vm_ioctl(vm, KVM_GET_TSC_KHZ, NULL) * 1000; + TEST_ASSERT(l0_tsc_freq, "vcpu ioctl KVM_GET_TSC_KHZ failed"); + kvm_vm_free(vm); + + printf("L0 TSC frequency is: %lu\n", l0_tsc_freq); + + tsc_start = rdtsc(); + sleep(1); + tsc_end = rdtsc(); + host_check_tsc_freq(0, tsc_end - tsc_start, l0_tsc_freq); + + /* Sanity check the frequency reported by KVM_GET_TSC_KHZ. */ + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq, l0_tsc_freq, BIT_ULL(frac_bits)); + + /* + * Scale L1 and L2 up and down at random factors from 2 to 10. Current + * CPUs have two full orders of magnitude of "breathing room" before an + * ultrafast L0 TSC frequency multiplied by 10x will encounter KVM's + * signed 32-bit limit on L1's frequency, and it's highly unlikely a + * CPU that supports TSC scaling will show up running at 100MHz, i.e. + * underflowing KVM's minimum 1KHz frequency is extremely unlikely. + * + * For L2, the frequency is limited only by what hardware can support, + * not by KVM's limits. SVM provides 8 bits of integer scale up, and + * 32 bits of fractional scale down, i.e. can scale up 255x and down a + * comical amount, so scaling up 100x and down 1/100 is well within + * hardware's capabilities (VMX provides 16 bits of "up" and 48 bits of + * "down"). + */ + l1_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; + l2_scale = (kvm_random_u32(&kvm_rng) % 9) + 2; + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale, + mul_u64_u64_div64(l0_tsc_freq, l2_scale, l1_scale), + (l2_scale << frac_bits)); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale, + mul_u64_u64_div64(l0_tsc_freq, l1_scale, l2_scale), + (1ull << frac_bits) / l2_scale); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq / l1_scale, + l0_tsc_freq / l1_scale / l2_scale, + (1ull << frac_bits) / l2_scale); + + test_tsc_scaling(l0_tsc_freq, l0_tsc_freq * l1_scale, + l0_tsc_freq * l1_scale * l2_scale, + (l2_scale << frac_bits)); + + /* + * Test that KVM saturates L2's frequency on both ends if the resulting + * L2 TSC frequency would be below or above what hardware can support. + * Because L2 = L0 * (L1_mult >> frac) * (L2_mult >> frac) needs to be + * distilled down to a single multiplier, very small/large multipliers + * will underflow/overflow the minimum/maximum multiplier supported by + * hardware when L1 and L2 multipliers are combined. KVM's behavior is + * saturate on {under,over}flow, i.e. to run at the min/max frequency. + * + * Note, userspace can only program L1's frequency in KHz, i.e. can't + * specify an exact multiplier. As a result, the minimum and maximum + * frequencies are different for L1 vs L2, because L1 is constrained by + * hardware *and* KVM, whereas L2 is constrained only by hardware. + */ + min_multiplier = 1; + min_freq = mul_u64_u64_div64(l0_tsc_freq, min_multiplier, BIT_ULL(frac_bits)); + min_freq = max(min_freq, (u64)1); + l1_min_freq = max(min_freq, (u64)1 * 1000); + test_tsc_scaling(l0_tsc_freq, l1_min_freq, min_freq, 1); + + /* + * SVM takes a 40-bit value (right shifted by 32), while VMX takes a + * 64-bit value (right shifted by 48). mul_u64_u64_shr() isn't (yet) + * available in selftests, but mul_u64_u64_div64() does nicely since, + * albeit more slowly (performance is obviously not a concern). Note, + * because KVM_GET_TSC_KHZ returns a signed 32-bit integer, KVM limits + * KVM_SET_TSC_KHZ to INT_MAX, even though hardware (both SVM and VMX) + * supports much higher frequencies. + */ + max_multiplier = kvm_cpu_has(X86_FEATURE_VMX) ? -1ull : GENMASK_U64(39, 0); + max_freq = mul_u64_u64_div64(l0_tsc_freq, max_multiplier, BIT_ULL(frac_bits)); + l1_max_freq = min(max_freq, (u64)INT32_MAX * 1000); + test_tsc_scaling(l0_tsc_freq, l1_max_freq, max_freq, max_multiplier); + return 0; } diff --git a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c index ce204ce29a9c..2ae698c9aa1c 100644 --- a/tools/testing/selftests/kvm/x86/nested_x2apic_test.c +++ b/tools/testing/selftests/kvm/x86/nested_x2apic_test.c @@ -61,57 +61,55 @@ static void l1_vmx_code(struct vmx_pages *vmx, struct hyperv_test_pages *hv_page evmcs_enable(); } - GUEST_ASSERT_EQ(prepare_for_vmx_operation(vmx), true); + prepare_for_vmx_operation(vmx); if (hv_pages) { - GUEST_ASSERT(load_evmcs(hv_pages)); + load_evmcs(hv_pages); current_evmcs->hv_enlightenments_control.msr_bitmap = 1; } else { - GUEST_ASSERT(load_vmcs(vmx)); + load_vmcs(vmx); } prepare_vmcs(vmx, NULL); - GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, (unsigned long)l2_guest_code), 0); + vmwrite(GUEST_RIP, (unsigned long)l2_guest_code); - control = vmreadz(PIN_BASED_VM_EXEC_CONTROL); + control = vmread(PIN_BASED_VM_EXEC_CONTROL); control |= PIN_BASED_EXT_INTR_MASK; vmwrite(PIN_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_TPR_SHADOW; - GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control |= SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | SECONDARY_EXEC_APIC_REGISTER_VIRT | SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY; control &= (rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2) >> 32); - GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0); + vmwrite(SECONDARY_VM_EXEC_CONTROL, control); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); - GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, - vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0); + vmlaunch(); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_CPUID); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); } static void l1_vmx_code_part2(void) { u64 control; - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control &= ~CPU_BASED_TPR_SHADOW; - GUEST_ASSERT_EQ(vmwrite(CPU_BASED_VM_EXEC_CONTROL, control), 0); + vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control &= ~(SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | SECONDARY_EXEC_APIC_REGISTER_VIRT | SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY); - GUEST_ASSERT_EQ(vmwrite(SECONDARY_VM_EXEC_CONTROL, control), 0); + vmwrite(SECONDARY_VM_EXEC_CONTROL, control); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_CPUID); - GUEST_ASSERT_EQ(vmwrite(GUEST_RIP, - vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)), 0); + vmresume(); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_CPUID); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); } static void l1_test_x2apic_intercepts(void) diff --git a/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c b/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c index 70950067b989..e3b20033a884 100644 --- a/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c +++ b/tools/testing/selftests/kvm/x86/nx_huge_pages_test.c @@ -120,7 +120,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages, } else { TEST_ASSERT(r == -1 && errno == EPERM, "This process should not have permission to disable NX huge pages"); - return; + goto done; } } @@ -213,6 +213,7 @@ void run_test(int reclaim_period_ms, bool disable_nx_huge_pages, check_2m_page_count(vm, disable_nx_huge_pages ? 3 : 2); check_split_count(vm, 0); +done: kvm_vm_free(vm); } diff --git a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c index 3aa0673f79f3..33b610383693 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_conversions_test.c @@ -202,15 +202,18 @@ static void guest_test_explicit_conversion(u64 base_gpa, bool do_fallocate) guest_sync_shared(gpa, size, p3, p4); memcmp_g(gpa, p4, size); - /* Reset the shared memory back to the initial pattern. */ - memset((void *)gpa, init_p, size); - /* * Free (via PUNCH_HOLE) *all* private memory so that the next * iteration starts from a clean slate, e.g. with respect to * whether or not there are pages/folios in guest_mem. */ guest_map_shared(base_gpa, PER_CPU_DATA_SIZE, true); + + /* + * Hole-punching above zeroed private memory. Reset shared + * memory in preparation for the next GUEST_STAGE. + */ + memset((void *)base_gpa, init_p, PER_CPU_DATA_SIZE); } } @@ -303,9 +306,12 @@ static void handle_exit_hypercall(struct kvm_vcpu *vcpu) if (do_fallocate) vm_guest_mem_fallocate(vm, gpa, size, map_shared); - if (set_attributes) - vm_set_memory_attributes(vm, gpa, size, - map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE); + if (set_attributes) { + u64 attrs = map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE; + + vm_mem_set_memory_attributes(vm, gpa, size, attrs); + } + run->hypercall.ret = 0; } @@ -349,8 +355,21 @@ static void *__test_mem_conversions(void *__vcpu) size_t nr_bytes = min_t(size_t, vm->page_size, size - i); u8 *hva = addr_gpa2hva(vm, gpa + i); - /* In all cases, the host should observe the shared data. */ - memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + if (kvm_has_gmem_attributes && + uc.args[0] == SYNC_PRIVATE) { + TEST_EXPECT_SIGBUS(READ_ONCE(*hva)); + } else { + /* + * If not testing in-place conversion + * (dual backing), the host should + * always observe shared data, since the + * shared memory is a separate page. + * + * For SYNC_SHARED, test that the host + * can see shared memory. + */ + memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + } /* For shared, write the new pattern to guest memory. */ if (uc.args[0] == SYNC_SHARED) @@ -366,20 +385,29 @@ static void *__test_mem_conversions(void *__vcpu) } } +/* Align each vCPU's chunk of memory naturally to the size of the backing store. */ +static size_t compute_per_cpu_size(enum vm_mem_backing_src_type src_type) +{ + size_t alignment; + + if (kvm_has_gmem_attributes) + alignment = getpagesize(); + else + alignment = get_backing_src_pagesz(src_type); + + return align_up(PER_CPU_DATA_SIZE, max_t(size_t, SZ_2M, alignment)); +} + static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_vcpus, u32 nr_memslots) { - /* - * Allocate enough memory so that each vCPU's chunk of memory can be - * naturally aligned with respect to the size of the backing store. - */ - const size_t alignment = max_t(size_t, SZ_2M, get_backing_src_pagesz(src_type)); - const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment); + const size_t per_cpu_size = compute_per_cpu_size(src_type); const size_t memfd_size = per_cpu_size * nr_vcpus; const size_t slot_size = memfd_size / nr_memslots; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; pthread_t threads[KVM_MAX_VCPUS]; struct kvm_vm *vm; + u64 gmem_flags; int memfd, i; const struct vm_shape shape = { @@ -394,12 +422,16 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, u32 nr_v vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE)); - memfd = vm_create_guest_memfd(vm, memfd_size, 0); + gmem_flags = 0; + if (kvm_has_gmem_attributes) + gmem_flags = GUEST_MEMFD_FLAG_MMAP | GUEST_MEMFD_FLAG_INIT_SHARED; + + memfd = vm_create_guest_memfd(vm, memfd_size, gmem_flags); for (i = 0; i < nr_memslots; i++) vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i, BASE_DATA_SLOT + i, slot_size / vm->page_size, - KVM_MEM_GUEST_MEMFD, memfd, slot_size * i); + KVM_MEM_GUEST_MEMFD, memfd, slot_size * i, gmem_flags); for (i = 0; i < nr_vcpus; i++) { gpa_t gpa = BASE_DATA_GPA + i * per_cpu_size; @@ -459,6 +491,8 @@ int main(int argc, char *argv[]) while ((opt = getopt(argc, argv, "hm:s:n:")) != -1) { switch (opt) { case 's': + TEST_ASSERT(!kvm_has_gmem_attributes, + "src_type is only configurable when testing without in-place conversion"); src_type = parse_backing_src_type(optarg); break; case 'n': diff --git a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c index e6d16e9d6120..f7228e0f1ac6 100644 --- a/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c +++ b/tools/testing/selftests/kvm/x86/private_mem_kvm_exits_test.c @@ -62,8 +62,9 @@ static void test_private_access_memslot_deleted(void) virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES); - /* Request to access page privately */ - vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); + /* Request to access page privately. */ + if (!kvm_has_gmem_attributes) + vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE); kvm_pthread_create(&vm_thread, NULL, (pthread_fn_t)run_vcpu_get_exit_reason, (void *)vcpu); @@ -73,10 +74,26 @@ static void test_private_access_memslot_deleted(void) kvm_pthread_join(vm_thread, &thread_return); exit_reason = (u32)(u64)thread_return; - TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); - TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); - TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); - TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + /* + * If attributes are tracked per-gmem, deleting the memslot that points + * at the gmem instance effectively makes the memory shared, and so the + * read should trigger emulated MMIO. + * + * If attributes are tracked per-VM, deleting the memslot shouldn't + * affect the private attribute, and so KVM should generate a memory + * fault exit (emulated MMIO on private GPAs is disallowed). + */ + if (kvm_has_gmem_attributes) { + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MMIO); + TEST_ASSERT_EQ(vcpu->run->mmio.phys_addr, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->mmio.len, sizeof(u64)); + TEST_ASSERT_EQ(vcpu->run->mmio.is_write, false); + } else { + TEST_ASSERT_EQ(exit_reason, KVM_EXIT_MEMORY_FAULT); + TEST_ASSERT_EQ(vcpu->run->memory_fault.flags, KVM_MEMORY_EXIT_FLAG_PRIVATE); + TEST_ASSERT_EQ(vcpu->run->memory_fault.gpa, EXITS_TEST_GPA); + TEST_ASSERT_EQ(vcpu->run->memory_fault.size, EXITS_TEST_SIZE); + } kvm_vm_free(vm); } @@ -87,6 +104,13 @@ static void test_private_access_memslot_not_private(void) struct kvm_vcpu *vcpu; u32 exit_reason; + /* + * Accessing non-private memory as private with a software-protected VM + * isn't possible when doing in-place conversions. + */ + if (kvm_has_gmem_attributes) + return; + vm = vm_create_shape_with_one_vcpu(protected_vm_shape, &vcpu, guest_repeatedly_read); diff --git a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c index 507391ab2c93..24056921110c 100644 --- a/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c +++ b/tools/testing/selftests/kvm/x86/save_restore_pf_stress_test.c @@ -98,17 +98,17 @@ static void l1_svm_code(struct svm_test_data *svm) static void l1_vmx_code(struct vmx_pages *vmx) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, guest_access_memory); - GUEST_ASSERT(!vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR))); + vmwrite(EXCEPTION_BITMAP, BIT(UD_VECTOR)); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); while (1) { - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); - GUEST_ASSERT_EQ(vmreadz(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR); - GUEST_ASSERT(!vmresume()); + GUEST_ASSERT_EQ(vmread(VM_EXIT_REASON), EXIT_REASON_EXCEPTION_NMI); + GUEST_ASSERT_EQ(vmread(VM_EXIT_INTR_INFO) & 0xff, UD_VECTOR); + vmresume(); } } diff --git a/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c b/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c index 8e3898646c69..909cd9e83fda 100644 --- a/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c +++ b/tools/testing/selftests/kvm/x86/set_boot_cpu_id.c @@ -69,7 +69,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu) TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage + 1, "Stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); test_set_bsp_busy(vcpu, "while running vm"); break; case UCALL_DONE: diff --git a/tools/testing/selftests/kvm/x86/set_sregs_test.c b/tools/testing/selftests/kvm/x86/set_sregs_test.c index 603226ffe437..5afc4187f5eb 100644 --- a/tools/testing/selftests/kvm/x86/set_sregs_test.c +++ b/tools/testing/selftests/kvm/x86/set_sregs_test.c @@ -135,6 +135,17 @@ static void test_cr_bits(struct kvm_vcpu *vcpu, u64 cr4) /* NW without CD is illegal, as is PG without PE. */ TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_NW); TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_PG); + + /* CR8 bits 3:0 are writable; bits 63:4 are reserved. */ + vcpu_sregs_get(vcpu, &sregs); + sregs.cr8 = 0xf; + vcpu_sregs_set(vcpu, &sregs); + + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT_EQ(sregs.cr8, 0xf); + + for (i = 4; i < 64; i++) + TEST_INVALID_SREG_BIT(vcpu, cr8, sregs, BIT_ULL(i)); } static void test_efer_bits(struct kvm_vcpu *vcpu, u64 efer) diff --git a/tools/testing/selftests/kvm/x86/sev_dbg_test.c b/tools/testing/selftests/kvm/x86/sev_dbg_test.c index eaa8201b937d..0277e38e3556 100644 --- a/tools/testing/selftests/kvm/x86/sev_dbg_test.c +++ b/tools/testing/selftests/kvm/x86/sev_dbg_test.c @@ -14,9 +14,11 @@ static u8 *data; static u8 src[BUFFER_SIZE] __aligned(PAGE_SIZE); static u8 dst[BUFFER_SIZE] __aligned(PAGE_SIZE); -static void validate_dst(int i, int nr_bytes, u8 pattern) +static void validate_dst(int offset, int nr_bytes, u8 pattern) { - for ( ; i < nr_bytes; i++) + int i; + + for (i = offset; i < offset + nr_bytes; i++) TEST_ASSERT(dst[i] == pattern, "Expected 0x%x at byte %u, got 0x%x", pattern, i, dst[i]); diff --git a/tools/testing/selftests/kvm/x86/sev_smoke_test.c b/tools/testing/selftests/kvm/x86/sev_smoke_test.c index bf27b6187afa..19f76f7d8ca6 100644 --- a/tools/testing/selftests/kvm/x86/sev_smoke_test.c +++ b/tools/testing/selftests/kvm/x86/sev_smoke_test.c @@ -178,7 +178,7 @@ static void test_sev(void *guest_code, u32 type, u64 policy) case UCALL_SYNC: continue; case UCALL_DONE: - return; + goto done; case UCALL_ABORT: REPORT_GUEST_ASSERT(uc); default: @@ -187,6 +187,7 @@ static void test_sev(void *guest_code, u32 type, u64 policy) } } +done: kvm_vm_free(vm); } diff --git a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c index 3dca85e95478..8d1822717abd 100644 --- a/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c +++ b/tools/testing/selftests/kvm/x86/smaller_maxphyaddr_emulation_test.c @@ -62,11 +62,12 @@ int main(int argc, char *argv[]) TEST_ASSERT(rc, "KVM_CAP_EXIT_ON_EMULATION_FAILURE is unavailable"); vm_enable_cap(vm, KVM_CAP_EXIT_ON_EMULATION_FAILURE, 1); - vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, - MEM_REGION_GPA, MEM_REGION_SLOT, - MEM_REGION_SIZE / PAGE_SIZE, 0); + vm_override_mem_region(vm, MEM_REGION_TEST_EXTRA, VM_MEM_SRC_ANONYMOUS, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / PAGE_SIZE); + gpa = vm_phy_pages_alloc(vm, MEM_REGION_SIZE / PAGE_SIZE, - MEM_REGION_GPA, MEM_REGION_SLOT); + MEM_REGION_TEST_EXTRA); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc"); virt_map(vm, MEM_REGION_GVA, MEM_REGION_GPA, 1); hva = addr_gpa2hva(vm, MEM_REGION_GPA); diff --git a/tools/testing/selftests/kvm/x86/smm_test.c b/tools/testing/selftests/kvm/x86/smm_test.c index d1edafd5af75..377df09e582a 100644 --- a/tools/testing/selftests/kvm/x86/smm_test.c +++ b/tools/testing/selftests/kvm/x86/smm_test.c @@ -78,8 +78,8 @@ static void guest_code(void *arg) if (this_cpu_has(X86_FEATURE_SVM)) { generic_svm_setup(svm, l2_guest_code); } else { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); } diff --git a/tools/testing/selftests/kvm/x86/state_test.c b/tools/testing/selftests/kvm/x86/state_test.c index 4a1056a6cb8d..500b09762555 100644 --- a/tools/testing/selftests/kvm/x86/state_test.c +++ b/tools/testing/selftests/kvm/x86/state_test.c @@ -59,13 +59,13 @@ void vmx_l2_guest_code(void) vmcall(); /* L1 has now set up a shadow VMCS for us. */ - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); GUEST_SYNC(10); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); - GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0fffee)); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); + vmwrite(GUEST_RIP, 0xc0fffee); GUEST_SYNC(11); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0fffee); - GUEST_ASSERT(!vmwrite(GUEST_RIP, 0xc0ffffee)); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0fffee); + vmwrite(GUEST_RIP, 0xc0ffffee); GUEST_SYNC(12); /* Done, exit to L1 and never come back. */ @@ -75,61 +75,61 @@ void vmx_l2_guest_code(void) static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) { GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); GUEST_SYNC(3); - GUEST_ASSERT(load_vmcs(vmx_pages)); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + load_vmcs(vmx_pages); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); GUEST_SYNC(4); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); prepare_vmcs(vmx_pages, vmx_l2_guest_code); GUEST_SYNC(5); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); + vmlaunch(); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); /* Check that the launched state is preserved. */ - GUEST_ASSERT(vmlaunch()); + GUEST_ASSERT(__vmlaunch()); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_SYNC(7); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + 3); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + 3); vmwrite(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); vmwrite(VMCS_LINK_POINTER, vmx_pages->shadow_vmcs_gpa); - GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa)); - GUEST_ASSERT(vmlaunch()); + vmptrld(vmx_pages->shadow_vmcs_gpa); + GUEST_ASSERT(__vmlaunch()); GUEST_SYNC(8); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); vmwrite(GUEST_RIP, 0xc0ffee); GUEST_SYNC(9); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffee); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffee); - GUEST_ASSERT(!vmptrld(vmx_pages->vmcs_gpa)); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmptrld(vmx_pages->vmcs_gpa); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - GUEST_ASSERT(!vmptrld(vmx_pages->shadow_vmcs_gpa)); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + vmptrld(vmx_pages->shadow_vmcs_gpa); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffffee); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); GUEST_SYNC(13); - GUEST_ASSERT(vmreadz(GUEST_RIP) == 0xc0ffffee); - GUEST_ASSERT(vmlaunch()); - GUEST_ASSERT(vmresume()); + GUEST_ASSERT(vmread(GUEST_RIP) == 0xc0ffffee); + GUEST_ASSERT(__vmlaunch()); + GUEST_ASSERT(__vmresume()); } static void __attribute__((__flatten__)) guest_code(void *arg) @@ -296,7 +296,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); state = vcpu_save_state(vcpu); memset(®s1, 0, sizeof(regs1)); @@ -343,7 +343,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } done: diff --git a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c b/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c deleted file mode 100644 index 6bc301207cbc..000000000000 --- a/tools/testing/selftests/kvm/x86/svm_nested_clear_efer_svme.c +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2026, Google LLC. - */ -#include "kvm_util.h" -#include "vmx.h" -#include "svm_util.h" -#include "kselftest.h" - - -static void l2_guest_code(void) -{ - unsigned long efer = rdmsr(MSR_EFER); - - /* generic_svm_setup() initializes EFER_SVME set for L2 */ - GUEST_ASSERT(efer & EFER_SVME); - wrmsr(MSR_EFER, efer & ~EFER_SVME); - - /* Unreachable, L1 should be shutdown */ - GUEST_ASSERT(0); -} - -static void l1_guest_code(struct svm_test_data *svm) -{ - generic_svm_setup(svm, l2_guest_code); - run_guest(svm->vmcb, svm->vmcb_gpa); - - /* Unreachable, L1 should be shutdown */ - GUEST_ASSERT(0); -} - -int main(int argc, char *argv[]) -{ - struct kvm_vcpu *vcpu; - struct kvm_vm *vm; - gva_t nested_gva = 0; - - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_SVM)); - - vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); - - vcpu_alloc_svm(vm, &nested_gva); - vcpu_args_set(vcpu, 1, nested_gva); - - vcpu_run(vcpu); - TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_SHUTDOWN); - - kvm_vm_free(vm); - return 0; -} diff --git a/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c new file mode 100644 index 000000000000..fa0d76a9c390 --- /dev/null +++ b/tools/testing/selftests/kvm/x86/svm_nested_efer_test.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Tests for KVM's handling of EFER bits whose behavior is tied to nested SVM. + * + * Copyright (C) 2026, Google LLC. + */ +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "svm_util.h" +#include "kselftest.h" + +static bool l2_ran; + +static void l2_clear_efer_svme(void) +{ + u64 efer = rdmsr(MSR_EFER); + + /* generic_svm_setup() initializes EFER_SVME set for L2 */ + GUEST_ASSERT(efer & EFER_SVME); + wrmsr(MSR_EFER, efer & ~EFER_SVME); + + /* Unreachable, L1 should be shutdown */ + GUEST_ASSERT(0); +} + +static void l1_clear_efer_svme(struct svm_test_data *svm) +{ + generic_svm_setup(svm, l2_clear_efer_svme); + run_guest(svm->vmcb, svm->vmcb_gpa); + + /* Unreachable, L1 should be shutdown */ + GUEST_ASSERT(0); +} + +static void l2_lmsle(void) +{ + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + l2_ran = true; + vmmcall(); +} + +static void l1_lmsle(struct svm_test_data *svm) +{ + bool lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + struct vmcb *vmcb = svm->vmcb; + u64 efer = rdmsr(MSR_EFER); + + /* + * Selftests' vCPUs are created with EFER.LMSLE clear; the sub-tests + * below need to start from a clean slate. + */ + GUEST_ASSERT(!(efer & EFER_LMSLE)); + GUEST_ASSERT(!l2_ran); + + /* + * Per the APM, if EFER_LMSLE_MBZ is enumerated in CPUID, "64-bit mode + * segment limit checking is not supported and attempting to set + * EFER.LMSLE = 1 causes a #GP exception". + */ + if (lmsle_mbz) { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), GP_VECTOR); + GUEST_ASSERT(!(rdmsr(MSR_EFER) & EFER_LMSLE)); + } else { + GUEST_ASSERT_EQ(wrmsr_safe(MSR_EFER, efer | EFER_LMSLE), 0); + GUEST_ASSERT(rdmsr(MSR_EFER) & EFER_LMSLE); + + /* + * Restore EFER so that generic_svm_setup() doesn't propagate + * EFER.LMSLE into vmcb12 on its own, i.e. so that the VMRUN + * sub-test actually tests what it thinks it's testing. + */ + wrmsr(MSR_EFER, efer); + } + + /* + * VMRUN's consistency checks reject "any MBZ bit of EFER", i.e. a + * vmcb12 with EFER.LMSLE set must generate VMEXIT_INVALID when the + * defeature is enumerated. + */ + generic_svm_setup(svm, l2_lmsle); + vmcb->save.efer |= EFER_LMSLE; + run_guest(vmcb, svm->vmcb_gpa); + + if (lmsle_mbz) { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_ERR); + GUEST_ASSERT(!l2_ran); + } else { + GUEST_ASSERT_EQ(vmcb->control.exit_code, SVM_EXIT_VMMCALL); + GUEST_ASSERT(l2_ran); + GUEST_ASSERT(vmcb->save.efer & EFER_LMSLE); + } + + GUEST_DONE(); +} + +static struct kvm_vcpu *create_l1_vcpu(struct kvm_vm **vm, void *l1_guest_code) +{ + struct kvm_vcpu *vcpu; + gva_t svm_gva; + + *vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); + + vcpu_alloc_svm(*vm, &svm_gva); + vcpu_args_set(vcpu, 1, svm_gva); + + return vcpu; +} + +static void test_enumeration(void) +{ + bool lmsle_mbz; + + /* + * EFER_LMSLE_MBZ, CPUID.80000008H:EBX[bit 20], is a "defeature" bit, + * i.e. is set when the CPU does *not* support long mode segment + * limits. KVM enumerates the defeature if and only if KVM refuses to + * set EFER.LMSLE, i.e. if the CPU doesn't support LMSLE, or if KVM + * doesn't support nested SVM. Derive the expectation from raw CPUID + * and kvm_amd's "nested" module param rather than from + * kvm_cpu_has(X86_FEATURE_SVM), so that the assertion doesn't simply + * compare KVM's enumeration to itself. + */ + lmsle_mbz = this_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ) || + !this_cpu_has(X86_FEATURE_SVM) || + !kvm_is_nested_virtualization_enabled(); + + TEST_ASSERT_EQ(kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ), lmsle_mbz); + + ksft_test_result_pass("KVM enumerates EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_clear_efer_svme(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + + vcpu = create_l1_vcpu(&vm, l1_clear_efer_svme); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_SHUTDOWN); + + kvm_vm_free(vm); + ksft_test_result_pass("L2 clearing EFER.SVME shuts down L1\n"); +} + +static void test_lmsle(bool lmsle_mbz) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + + vcpu = create_l1_vcpu(&vm, l1_lmsle); + + vcpu_set_or_clear_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ, + lmsle_mbz); + + vcpu_run(vcpu); + TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO); + + switch (get_ucall(vcpu, &uc)) { + case UCALL_ABORT: + REPORT_GUEST_ASSERT(uc); + case UCALL_DONE: + break; + default: + TEST_FAIL("Unexpected ucall: %lu", uc.cmd); + } + + kvm_vm_free(vm); + ksft_test_result_pass("Guest EFER_LMSLE_MBZ=%d\n", lmsle_mbz); +} + +static void test_host_initiated_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + u64 efer; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * EFER_LMSLE_MBZ is a guest CPUID consistency check, not a host + * capability, i.e. must not be enforced against host-initiated writes, + * so that userspace can set MSRs before it sets guest CPUID. + */ + efer = vcpu_get_msr(vcpu, MSR_EFER); + TEST_ASSERT(!(efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + vcpu_set_msr(vcpu, MSR_EFER, efer | EFER_LMSLE); + TEST_ASSERT_EQ(vcpu_get_msr(vcpu, MSR_EFER), efer | EFER_LMSLE); + + kvm_vm_free(vm); + ksft_test_result_pass("Host-initiated EFER.LMSLE=1 is allowed\n"); +} + +static void test_sregs_lmsle(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_sregs sregs; + struct kvm_vm *vm; + int rc; + + vm = vm_create_with_one_vcpu(&vcpu, NULL); + vcpu_set_cpuid_feature(vcpu, X86_FEATURE_EFER_LMSLE_MBZ); + + /* + * Unlike KVM_SET_MSRS, KVM_SET_SREGS runs the full set of guest CPUID + * checks, i.e. rejects EFER.LMSLE even though it's host-initiated. + */ + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT(!(sregs.efer & EFER_LMSLE), "EFER.LMSLE unexpectedly set"); + + sregs.efer |= EFER_LMSLE; + rc = _vcpu_sregs_set(vcpu, &sregs); + TEST_ASSERT(rc, "KVM allowed EFER.LMSLE with EFER_LMSLE_MBZ set"); + + kvm_vm_free(vm); + ksft_test_result_pass("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); +} + +int main(int argc, char *argv[]) +{ + bool has_nested_svm, has_lmsle; + + ksft_print_header(); + ksft_set_plan(6); + + test_enumeration(); + + /* + * The sub-tests below need to actually run a nested guest, and the + * EFER.LMSLE sub-tests additionally need KVM to allow EFER.LMSLE. + * It's KVM's view of the world, not raw CPUID, that dictates whether + * EFER.LMSLE is allowed, i.e. whether the defeature is emulated. + */ + has_nested_svm = kvm_cpu_has(X86_FEATURE_SVM); + has_lmsle = has_nested_svm && + !kvm_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ); + + if (!has_nested_svm) + ksft_print_msg("Nested SVM unsupported\n"); + else if (!has_lmsle) + ksft_print_msg("KVM doesn't support EFER.LMSLE\n"); + + if (has_nested_svm) { + test_clear_efer_svme(); + test_lmsle(true); + } else { + ksft_test_result_skip("L2 clearing EFER.SVME shuts down L1\n"); + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=1\n"); + } + + if (has_lmsle) { + test_lmsle(false); + test_host_initiated_lmsle(); + test_sregs_lmsle(); + } else { + ksft_test_result_skip("Guest EFER_LMSLE_MBZ=0\n"); + ksft_test_result_skip("Host-initiated EFER.LMSLE=1 is allowed\n"); + ksft_test_result_skip("KVM_SET_SREGS rejects EFER.LMSLE=1\n"); + } + + ksft_finished(); +} diff --git a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c index 0d83516f4bd0..b2b796fd9ffb 100644 --- a/tools/testing/selftests/kvm/x86/triple_fault_event_test.c +++ b/tools/testing/selftests/kvm/x86/triple_fault_event_test.c @@ -25,14 +25,14 @@ void l1_guest_code_vmx(struct vmx_pages *vmx) { GUEST_ASSERT(vmx->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx)); - GUEST_ASSERT(load_vmcs(vmx)); + prepare_for_vmx_operation(vmx); + load_vmcs(vmx); prepare_vmcs(vmx, l2_guest_code); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 should triple fault after a triple fault event injected. */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/tsc_msrs_test.c b/tools/testing/selftests/kvm/x86/tsc_msrs_test.c index 91583969a14f..55779773614b 100644 --- a/tools/testing/selftests/kvm/x86/tsc_msrs_test.c +++ b/tools/testing/selftests/kvm/x86/tsc_msrs_test.c @@ -78,7 +78,7 @@ static void run_vcpu(struct kvm_vcpu *vcpu, int stage) else ksft_test_result_fail( "stage %d: Unexpected register values vmexit, got %lx", - stage + 1, (ulong)uc.args[1]); + stage + 1, (unsigned long)uc.args[1]); return; case UCALL_DONE: ksft_test_result_pass("stage %d passed\n", stage + 1); diff --git a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c index 463f73aa9159..1d7a529c8297 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apic_access_test.c @@ -38,30 +38,30 @@ static void l1_guest_code(struct vmx_pages *vmx_pages, unsigned long high_gpa) { u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); - control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control = vmread(SECONDARY_VM_EXEC_CONTROL); control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; vmwrite(SECONDARY_VM_EXEC_CONTROL, control); vmwrite(APIC_ACCESS_ADDR, vmx_pages->apic_access_gpa); /* Try to launch L2 with the memory-backed APIC-access address. */ - GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR)); - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_SYNC(vmread(APIC_ACCESS_ADDR)); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); vmwrite(APIC_ACCESS_ADDR, high_gpa); /* Try to resume L2 with the unbacked APIC-access address. */ - GUEST_SYNC(vmreadz(APIC_ACCESS_ADDR)); - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + GUEST_SYNC(vmread(APIC_ACCESS_ADDR)); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_DONE(); } @@ -77,7 +77,7 @@ int main(int argc, char *argv[]) struct kvm_vcpu *vcpu; struct kvm_vm *vm; - TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX)); + TEST_REQUIRE(kvm_cpu_has_vmx_virtualize_apic_accesses()); vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c index f9b88a6f6113..cda9e43e8599 100644 --- a/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_apicv_updates_test.c @@ -33,12 +33,12 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) { u32 control; - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); - control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control = vmread(CPU_BASED_VM_EXEC_CONTROL); control |= CPU_BASED_USE_MSR_BITMAPS; vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); @@ -62,9 +62,9 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * Run L2 to switch to x2APIC mode, which in turn will uninhibit APICv, * as KVM should force the APIC ID back to its default. */ - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); GUEST_ASSERT(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD); /* @@ -90,8 +90,8 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * handles the x2APIC => xAPIC transition and inhibits APICv while L2 * is active. */ - GUEST_ASSERT(!vmresume()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmresume(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); GUEST_ASSERT(!(rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_EXTD)); xapic_write_reg(APIC_ICR, APIC_DEST_SELF | APIC_DM_FIXED | GOOD_IPI_VECTOR); diff --git a/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c index 2cae86d9d5e2..08a0caa9c52d 100644 --- a/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_exception_with_invalid_guest_state.c @@ -109,7 +109,7 @@ int main(int argc, char *argv[]) struct kvm_vcpu *vcpu; struct kvm_vm *vm; - TEST_REQUIRE(host_cpu_is_intel); + TEST_REQUIRE(host_cpu_is_intel_compatible); TEST_REQUIRE(!kvm_is_unrestricted_guest_enabled()); vm = vm_create_with_one_vcpu(&vcpu, guest_code); diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c index c8379124b317..bbd188949412 100644 --- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c @@ -41,8 +41,8 @@ static void l2_guest_code(void) static void l1_guest_code(struct vmx_pages *vmx_pages) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); /* Prepare the VMCS for L2 execution. */ prepare_vmcs(vmx_pages, l2_guest_code); @@ -52,13 +52,13 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) * library hasn't enabled it. Because KVM selftests jump directly to * 64-bit mode, unrestricted guest support isn't required. */ - GUEST_ASSERT(!(vmreadz(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) || - !(vmreadz(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST)); + GUEST_ASSERT(!(vmread(CPU_BASED_VM_EXEC_CONTROL) & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) || + !(vmread(SECONDARY_VM_EXEC_CONTROL) & SECONDARY_EXEC_UNRESTRICTED_GUEST)); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); /* L2 should triple fault after main() stuffs invalid guest state. */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_TRIPLE_FAULT); GUEST_DONE(); } diff --git a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c index 75073efa926d..7dcee11877ea 100644 --- a/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_nested_la57_state_test.c @@ -37,8 +37,8 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(rdmsr(MSR_GS_BASE) == LA57_GS_BASE); GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); prepare_vmcs(vmx_pages, l2_guest_code); @@ -52,13 +52,13 @@ static void l1_guest_code(struct vmx_pages *vmx_pages) pml4_pa = pml5[0] & PHYSICAL_PAGE_MASK; vmwrite(GUEST_CR3, pml4_pa); - guest_cr4 = vmreadz(GUEST_CR4); + guest_cr4 = vmread(GUEST_CR4); guest_cr4 &= ~X86_CR4_LA57; vmwrite(GUEST_CR4, guest_cr4); - GUEST_ASSERT(!vmlaunch()); + vmlaunch(); - exit_reason = vmreadz(VM_EXIT_REASON); + exit_reason = vmread(VM_EXIT_REASON); GUEST_ASSERT(exit_reason == EXIT_REASON_VMCALL); } @@ -109,7 +109,7 @@ int main(int argc, char *argv[]) } TEST_ASSERT(uc.args[1] == stage, - "Expected stage %d, got stage %lu", stage, (ulong)uc.args[1]); + "Expected stage %d, got stage %lu", stage, (unsigned long)uc.args[1]); if (stage == 1) { pr_info("L2 is active; performing save/restore.\n"); state = vcpu_save_state(vcpu); diff --git a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c index eb8021c33cd4..b9ee9d3c1da4 100644 --- a/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86/vmx_preemption_timer_test.c @@ -71,9 +71,9 @@ void l1_guest_code(struct vmx_pages *vmx_pages) u64 l1_tsc_deadline, l2_tsc_deadline; GUEST_ASSERT(vmx_pages->vmcs_gpa); - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); - GUEST_ASSERT(load_vmcs(vmx_pages)); - GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + prepare_for_vmx_operation(vmx_pages); + load_vmcs(vmx_pages); + GUEST_ASSERT(vmptrst() == vmx_pages->vmcs_gpa); prepare_vmcs(vmx_pages, l2_guest_code); @@ -90,19 +90,17 @@ void l1_guest_code(struct vmx_pages *vmx_pages) !(ctrl_exit_rev.clr & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) return; - GUEST_ASSERT(!vmlaunch()); - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); - vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); + vmlaunch(); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmwrite(GUEST_RIP, vmread(GUEST_RIP) + vmread(VM_EXIT_INSTRUCTION_LEN)); /* * Turn on PIN control and resume the guest */ - GUEST_ASSERT(!vmwrite(PIN_BASED_VM_EXEC_CONTROL, - vmreadz(PIN_BASED_VM_EXEC_CONTROL) | - PIN_BASED_VMX_PREEMPTION_TIMER)); + vmwrite(PIN_BASED_VM_EXEC_CONTROL, + vmread(PIN_BASED_VM_EXEC_CONTROL) | PIN_BASED_VMX_PREEMPTION_TIMER); - GUEST_ASSERT(!vmwrite(VMX_PREEMPTION_TIMER_VALUE, - PREEMPTION_TIMER_VALUE)); + vmwrite(VMX_PREEMPTION_TIMER_VALUE, PREEMPTION_TIMER_VALUE); vmx_pt_rate = rdmsr(MSR_IA32_VMX_MISC) & 0x1F; @@ -110,7 +108,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) l1_vmx_pt_start = (rdtsc() >> vmx_pt_rate) << vmx_pt_rate; - GUEST_ASSERT(!vmresume()); + vmresume(); l1_vmx_pt_finish = rdtsc(); @@ -123,7 +121,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) /* * Ensure the exit from L2 is due to preemption timer expiry */ - GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER); + GUEST_ASSERT(vmread(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER); l1_tsc_deadline = l1_vmx_pt_start + (PREEMPTION_TIMER_VALUE << vmx_pt_rate); @@ -193,7 +191,7 @@ int main(int argc, char *argv[]) /* UCALL_SYNC is handled here. */ TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", - stage, (ulong)uc.args[1]); + stage, (unsigned long)uc.args[1]); /* * If this stage 2 then we should verify the vmx pt expiry * is as expected. @@ -234,7 +232,7 @@ int main(int argc, char *argv[]) vcpu_regs_get(vcpu, ®s2); TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + (unsigned long)regs2.rdi, (unsigned long)regs2.rsi); } done: diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 469e3ab16460..2362bf272e80 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -233,13 +233,10 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, { long pages_not_moved; unsigned long nodemask = 0; - unsigned long nodemasks[sizeof(nodemask) * 8]; int nodes = 0; time_t start_time, last_update, now; time_t interval_secs = 1; - int i; int from, to; - unsigned long bit; u64 hlt_count; u64 wake_count; u64 ipis_sent; @@ -247,32 +244,19 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, fprintf(stderr, "Calling migrate_pages every %d microseconds\n", delay_usecs); - /* Get set of first 64 numa nodes available */ - kvm_get_mempolicy(NULL, &nodemask, sizeof(nodemask) * 8, - 0, MPOL_F_MEMS_ALLOWED); + nodes = kvm_get_numa_memory_nodes(&nodemask); + TEST_ASSERT(nodes > 1, + "NUMA nodes disappeared? nodemask = 0x%lx", nodemask); fprintf(stderr, "Numa nodes found amongst first %lu possible nodes " "(each 1-bit indicates node is present): %#lx\n", - sizeof(nodemask) * 8, nodemask); - - /* Init array of masks containing a single-bit in each, one for each - * available node. migrate_pages called below requires specifying nodes - * as bit masks. - */ - for (i = 0, bit = 1; i < sizeof(nodemask) * 8; i++, bit <<= 1) { - if (nodemask & bit) { - nodemasks[nodes] = nodemask & bit; - nodes++; - } - } - - TEST_ASSERT(nodes > 1, - "Did not find at least 2 numa nodes. Can't do migration"); + BITS_PER_TYPE(nodemask), nodemask); fprintf(stderr, "Migrating amongst %d nodes found\n", nodes); - from = 0; - to = 1; + from = kvm_get_next_numa_node(nodemask, -1); + to = kvm_get_next_numa_node(nodemask, from); + start_time = time(NULL); last_update = start_time; @@ -281,6 +265,9 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, wake_count = data->wake_count; while ((int)(time(NULL) - start_time) < run_secs) { + unsigned long from_mask = BIT(from); + unsigned long to_mask = BIT(to); + data->migrations_attempted++; /* @@ -291,9 +278,8 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, * KVM_CREATE_VCPU ioctl. If that assumption ever changes this * test may break or give a false positive signal. */ - pages_not_moved = migrate_pages(0, sizeof(nodemasks[from]), - &nodemasks[from], - &nodemasks[to]); + pages_not_moved = migrate_pages(0, MAXNODE_FOR_MASK(from_mask), + &from_mask, &to_mask); if (pages_not_moved < 0) fprintf(stderr, "migrate_pages failed, errno=%d\n", errno); @@ -305,9 +291,7 @@ void do_migrations(struct test_data_page *data, int run_secs, int delay_usecs, data->migrations_completed++; from = to; - to++; - if (to == nodes) - to = 0; + to = kvm_get_next_numa_node(nodemask, from); now = time(NULL); if (((now - start_time) % interval_secs == 0) && @@ -366,26 +350,17 @@ void get_cmdline_args(int argc, char *argv[], int *run_secs, } } -int main(int argc, char *argv[]) +static void test_xapic_ipi(int run_secs, int delay_usecs, bool migrate) { int wait_secs; const int max_halter_wait = 10; - int run_secs = 0; - int delay_usecs = 0; struct test_data_page *data; gva_t test_data_page_gva; - bool migrate = false; pthread_t threads[2]; struct thread_params params[2]; struct kvm_vm *vm; u64 *pipis_rcvd; - get_cmdline_args(argc, argv, &run_secs, &migrate, &delay_usecs); - if (run_secs <= 0) - run_secs = DEFAULT_RUN_SECS; - if (delay_usecs <= 0) - delay_usecs = DEFAULT_DELAY_USECS; - vm = vm_create_with_one_vcpu(¶ms[0].vcpu, halter_guest_code); vm_install_exception_handler(vm, IPI_VECTOR, guest_ipi_handler); @@ -473,5 +448,30 @@ int main(int argc, char *argv[]) kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + bool force_migrate = false; + unsigned long nodemask; + int run_secs = 0; + int delay_usecs = 0; + + get_cmdline_args(argc, argv, &run_secs, &force_migrate, &delay_usecs); + if (run_secs <= 0) + run_secs = DEFAULT_RUN_SECS; + if (delay_usecs <= 0) + delay_usecs = DEFAULT_DELAY_USECS; + + if (!force_migrate) + test_xapic_ipi(run_secs, delay_usecs, false); + + if (kvm_get_numa_memory_nodes(&nodemask) > 1) + test_xapic_ipi(run_secs, delay_usecs, true); + else + __TEST_REQUIRE(!force_migrate, + "Need at least 2 NUMA nodes to do migration (nodemask = 0x%lx)", + nodemask); + return 0; } diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index c3c0ee253fc7..99c9969e77a4 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -40,9 +40,6 @@ config NEED_KVM_DIRTY_RING_WITH_BITMAP bool depends on HAVE_KVM_DIRTY_RING -config KVM_MMIO - bool - config KVM_ASYNC_PF bool @@ -50,21 +47,12 @@ config KVM_ASYNC_PF config KVM_ASYNC_PF_SYNC bool -config HAVE_KVM_MSI - bool - -config HAVE_KVM_READONLY_MEM - bool - config HAVE_KVM_CPU_RELAX_INTERCEPT bool config KVM_VFIO bool -config HAVE_KVM_INVALID_WAKEUPS - bool - config KVM_GENERIC_DIRTYLOG_READ_PROTECT bool @@ -79,12 +67,6 @@ config HAVE_KVM_IRQ_BYPASS tristate select IRQ_BYPASS_MANAGER -config HAVE_KVM_VCPU_RUN_PID_CHANGE - bool - -config HAVE_KVM_NO_POLL - bool - config VIRT_XFER_TO_GUEST_WORK bool @@ -100,9 +82,6 @@ config KVM_ELIDE_TLB_FLUSH_IF_YOUNG config KVM_MMU_LOCKLESS_AGING bool -config KVM_GENERIC_MEMORY_ATTRIBUTES - bool - config KVM_GUEST_MEMFD select XARRAY_MULTI bool @@ -122,3 +101,6 @@ config HAVE_KVM_ARCH_GMEM_INVALIDATE config HAVE_KVM_ARCH_GMEM_POPULATE bool depends on KVM_GUEST_MEMFD + +config KVM_VM_MEMORY_ATTRIBUTES + bool diff --git a/virt/kvm/Makefile.kvm b/virt/kvm/Makefile.kvm index d047d4cf58c9..ec7da5942354 100644 --- a/virt/kvm/Makefile.kvm +++ b/virt/kvm/Makefile.kvm @@ -5,9 +5,14 @@ KVM ?= ../../../virt/kvm -kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o +KVM_DEV_NAME ?= kvm +KVM_DEV_MINOR ?= KVM_MINOR + +ccflags-y +=-DKVM_DEV_NAME=$(KVM_DEV_NAME) -DKVM_DEV_MINOR=$(KVM_DEV_MINOR) + +kvm-y := $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o \ + $(KVM)/coalesced_mmio.o kvm-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o -kvm-$(CONFIG_KVM_MMIO) += $(KVM)/coalesced_mmio.o kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o kvm-$(CONFIG_HAVE_KVM_IRQ_ROUTING) += $(KVM)/irqchip.o kvm-$(CONFIG_HAVE_KVM_DIRTY_RING) += $(KVM)/dirty_ring.o diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index b8aaa96b799b..6b3b6353cb9d 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -21,7 +21,10 @@ static struct kmem_cache *async_pf_cache; int kvm_async_pf_init(void) { - async_pf_cache = KMEM_CACHE(kvm_async_pf, 0); + async_pf_cache = kmem_cache_create(KVM_DEV_STRING(async_pf), + sizeof(struct kvm_async_pf), + __alignof__(struct kvm_async_pf), + 0, NULL); if (!async_pf_cache) return -ENOMEM; diff --git a/virt/kvm/coalesced_mmio.c b/virt/kvm/coalesced_mmio.c index 6b1d90161099..3b5430036cae 100644 --- a/virt/kvm/coalesced_mmio.c +++ b/virt/kvm/coalesced_mmio.c @@ -15,6 +15,8 @@ #include #include +#ifndef __KVM_NO_MMIO + #include "coalesced_mmio.h" static inline struct kvm_coalesced_mmio_dev *to_mmio(struct kvm_io_device *dev) @@ -188,3 +190,5 @@ int kvm_vm_ioctl_unregister_coalesced_mmio(struct kvm *kvm, */ return 0; } + +#endif /* !__KVM_NO_MMIO */ diff --git a/virt/kvm/coalesced_mmio.h b/virt/kvm/coalesced_mmio.h index 36f84264ed25..77b811dee27c 100644 --- a/virt/kvm/coalesced_mmio.h +++ b/virt/kvm/coalesced_mmio.h @@ -11,7 +11,7 @@ * */ -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO #include diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 93ad2ebc963f..f43b8b25bc85 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -711,7 +711,8 @@ bool kvm_notify_irqfd_resampler(struct kvm *kvm, */ int kvm_irqfd_init(void) { - irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", WQ_PERCPU, 0); + irqfd_cleanup_wq = alloc_workqueue(__stringify(KVM_DEV_NAME) "-irqfd-cleanup", + WQ_PERCPU, 0); if (!irqfd_cleanup_wq) return -ENOMEM; diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 5c084d5393a6..8c97a69b16dc 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -4,9 +4,11 @@ #include #include #include +#include #include #include #include +#include #include "kvm_mm.h" #include "guest_memfd.h" @@ -34,6 +36,13 @@ struct gmem_inode { struct list_head gmem_file_list; u64 flags; + /* + * Every index in this inode, whether memory is populated or + * not, is tracked in attributes. The entire range of indices, + * corresponding to the size of this inode, is represented in + * this maple tree. + */ + struct maple_tree attributes; }; static __always_inline struct gmem_inode *GMEM_I(struct inode *inode) @@ -61,9 +70,28 @@ static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn) return gfn - slot->base_gfn + slot->gmem.pgoff; } +static u64 kvm_gmem_get_default_attributes(struct inode *inode) +{ + bool init_shared = GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED; + + return init_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE; +} + +static u64 kvm_gmem_get_attributes(struct inode *inode, void *entry) +{ + if (WARN_ON_ONCE(!entry)) + return kvm_gmem_get_default_attributes(inode); + + return xa_to_value(entry); +} + static bool kvm_gmem_is_private_mem(struct inode *inode, pgoff_t index) { - return !(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED); + struct maple_tree *mt = &GMEM_I(inode)->attributes; + void *entry = mtree_load(mt, index); + + return kvm_gmem_get_attributes(inode, entry) & + KVM_MEMORY_ATTRIBUTE_PRIVATE; } static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) @@ -71,6 +99,24 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) return !kvm_gmem_is_private_mem(inode, index); } +static bool kvm_gmem_range_has_attributes(struct inode *inode, + pgoff_t index, size_t nr_pages, + u64 attributes) +{ + struct maple_tree *mt = &GMEM_I(inode)->attributes; + pgoff_t end = index + nr_pages - 1; + void *entry; + + lockdep_assert(mt_lock_is_held(mt)); + + mt_for_each(mt, entry, index, end) { + if (kvm_gmem_get_attributes(inode, entry) != attributes) + return false; + } + + return true; +} + /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -112,8 +158,11 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) return folio; } -static enum kvm_gfn_range_filter kvm_gmem_get_invalidate_filter(struct inode *inode) +static enum kvm_gfn_range_filter kvm_gmem_get_all_gfns_filter(struct inode *inode) { + if (gmem_in_place_conversion) + return KVM_FILTER_SHARED | KVM_FILTER_PRIVATE; + if (GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED) return KVM_FILTER_SHARED; @@ -162,15 +211,13 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, } static void kvm_gmem_invalidate_start(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, + enum kvm_gfn_range_filter filter) { - enum kvm_gfn_range_filter attr_filter; struct gmem_file *f; - attr_filter = kvm_gmem_get_invalidate_filter(inode); - kvm_gmem_for_each_file(f, inode) - __kvm_gmem_invalidate_start(f, start, end, attr_filter); + __kvm_gmem_invalidate_start(f, start, end, filter); } static void __kvm_gmem_invalidate_end(struct gmem_file *f, pgoff_t start, @@ -196,6 +243,7 @@ static void kvm_gmem_invalidate_end(struct inode *inode, pgoff_t start, static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { + enum kvm_gfn_range_filter filter = kvm_gmem_get_all_gfns_filter(inode); pgoff_t start = offset >> PAGE_SHIFT; pgoff_t end = (offset + len) >> PAGE_SHIFT; @@ -205,7 +253,7 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) */ filemap_invalidate_lock(inode->i_mapping); - kvm_gmem_invalidate_start(inode, start, end); + kvm_gmem_invalidate_start(inode, start, end, filter); truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1); @@ -300,17 +348,56 @@ static int kvm_gmem_release(struct inode *inode, struct file *file) * dereferencing the slot for existing bindings needs to be protected * against memslot updates, specifically so that unbind doesn't race * and free the memslot (kvm_gmem_get_file() will return NULL). - * - * Since .release is called only when the reference count is zero, - * after which file_ref_get() and get_file_active() fail, - * kvm_gmem_get_pfn() cannot be using the file concurrently. - * file_ref_put() provides a full barrier, and get_file_active() the - * matching acquire barrier. */ mutex_lock(&kvm->slots_lock); filemap_invalidate_lock(inode->i_mapping); + /* + * Note! synchronize_srcu() is _not_ needed after nullifying memslot + * bindings as slot->gmem.file cannot be set back to a non-null value + * without the memslot first being deleted. I.e. this relies on the + * synchronize_srcu_expedited() in kvm_swap_active_memslots() to ensure + * kvm_gmem_get_pfn() (which runs with kvm->srcu held for read) can't + * grab a reference to slot->gmem.file even if the struct file object + * is reallocated (for use as a different file). + * + * file_ref_put() provides a full barrier, and __get_file_rcu() the + * matching acquire barrier, to ensure that kvm_gmem_get_file() (via + * __get_file_rcu()) sees refcount==0 or fails the "file reloaded" + * check (file != NULL due to nullifying the file pointer here). + * + * Unlike most other users of get_file_rcu(), where callers don't care + * if they race with a write, only that they have a reference to _a_ + * live file, kvm_gmem_get_pfn() needs to get the exact file that is + * associated with the memslot. Without the aforementioned SRCU + * synchronization, the following could happen: + * + * CPU0 CPU1 + * kvm_gmem_get_pfn() + * slot->gmem.file == A + * kvm_gmem_release()) + * slot->gmem.file = NULL + * + * kvm_set_memory_region() + * slot deleted + * + * kvm_set_memory_region() + * slot created + * slot->gmem.file = B (alloc the same object) + * + * get_file_active() + * file = B + * file_reloaded = B + * + * + * + * Obviously KVM would be broken in many places if the synchronization + * were omitted, but it's important to note that get_file_active() does + * NOT guarantee a reference to the "original" file was obtained, only + * that it grabbed a reference for the returned file, i.e. didn't grab + * a reference for file A, but then returned a pointer to file B. + */ xa_for_each(&f->bindings, index, slot) WRITE_ONCE(slot->gmem.file, NULL); @@ -320,7 +407,7 @@ static int kvm_gmem_release(struct inode *inode, struct file *file) * memory, as its lifetime is associated with the inode, not the file. */ __kvm_gmem_invalidate_start(f, 0, -1ul, - kvm_gmem_get_invalidate_filter(inode)); + kvm_gmem_get_all_gfns_filter(inode)); __kvm_gmem_invalidate_end(f, 0, -1ul); list_del(&f->entry); @@ -364,10 +451,13 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) return VM_FAULT_SIGBUS; - if (!kvm_gmem_is_shared_mem(inode, vmf->pgoff)) - return VM_FAULT_SIGBUS; + filemap_invalidate_lock_shared(inode->i_mapping); + if (kvm_gmem_is_shared_mem(inode, vmf->pgoff)) + folio = kvm_gmem_get_folio(inode, vmf->pgoff); + else + folio = ERR_PTR(-EACCES); + filemap_invalidate_unlock_shared(inode->i_mapping); - folio = kvm_gmem_get_folio(inode, vmf->pgoff); if (IS_ERR(folio)) { if (PTR_ERR(folio) == -EAGAIN) return VM_FAULT_RETRY; @@ -448,11 +538,280 @@ static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) return 0; } +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn); + struct inode *inode; + + if (!slot) + return 0; + + CLASS(gmem_get_file, file)(slot); + if (!file) + return 0; + + inode = file_inode(file); + + /* + * Rely on the maple tree's internal RCU lock to ensure a stable result. + * This result can become stale as soon as the lock is dropped, so the + * caller _must_ protect consumption of private vs. shared either by + * holding guest_memfd's invalidate lock for the entire duration, or by + * checking mmu_invalidate_retry_gfn() under mmu_lock to serialize + * against concurrent attribute updates. + */ + return kvm_gmem_is_private_mem(inode, kvm_gmem_get_index(slot, gfn)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_is_private_gfn); + +/* + * Preallocate memory for attributes to be stored on a maple tree, pointed to + * by mas. Adjacent ranges with attributes identical to the new attributes + * will be merged. Also sets mas's bounds up for storing attributes. + * + * This maintains the invariant that ranges with the same attributes will + * always be merged. + */ +static int kvm_gmem_mas_preallocate(struct ma_state *mas, u64 attributes, + pgoff_t start, size_t nr_pages) +{ + pgoff_t end = start + nr_pages; + pgoff_t last = end - 1; + void *entry; + + /* Try extending range. entry is NULL on overflow/wrap-around. */ + mas_set(mas, end); + entry = mas_find(mas, end); + if (entry && xa_to_value(entry) == attributes) + last = mas->last; + + if (start > 0) { + mas_set(mas, start - 1); + entry = mas_find(mas, start - 1); + if (entry && xa_to_value(entry) == attributes) + start = mas->index; + } + + mas_set_range(mas, start, last); + return mas_preallocate(mas, xa_mk_value(attributes), GFP_KERNEL); +} + +static bool __folio_has_outstanding_references(struct folio *folio, + enum lru_cache_drained *drained) +{ + if (folio_maybe_dma_pinned(folio) || folio_mapped(folio)) + return true; + + /* 1 reference held by filemap_get_folios() in the folio batch. */ + lru_cache_drain_for_folio(folio, 1, drained); + + /* + * Outstanding references are anything other than those from the page + * cache, plus 1 temporary reference held by filemap_get_folios() in the + * folio batch. + */ + return folio_ref_count(folio) != folio_nr_pages(folio) + 1; +} + +static bool kvm_gmem_has_outstanding_references(struct inode *inode, + pgoff_t start, size_t nr_pages, + pgoff_t *err_index) +{ + enum lru_cache_drained drained = LRU_CACHE_NOT_DRAINED; + struct address_space *mapping = inode->i_mapping; + pgoff_t last = start + nr_pages - 1; + struct folio_batch fbatch; + pgoff_t next; + int i; + + folio_batch_init(&fbatch); + + next = start; + while (filemap_get_folios(mapping, &next, last, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + + if (__folio_has_outstanding_references(folio, &drained)) { + *err_index = max(start, folio->index); + folio_batch_release(&fbatch); + return true; + } + } + + folio_batch_release(&fbatch); + cond_resched(); + } + + return false; +} + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) +{ + struct folio_batch fbatch; + pgoff_t next = start; + int i; + + folio_batch_init(&fbatch); + while (filemap_get_folios(inode->i_mapping, &next, end - 1, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + pgoff_t start_index, end_index; + kvm_pfn_t start_pfn; + kvm_pfn_t nr_pages; + + start_index = max(start, folio->index); + end_index = min(end, folio_next_index(folio)); + /* + * end_index is either in folio or points to + * the first page of the next folio. Hence, + * all pages in range [start_index, end_index) + * are contiguous. + */ + start_pfn = folio_file_pfn(folio, start_index); + nr_pages = end_index - start_index; + + kvm_arch_gmem_make_shared(start_pfn, nr_pages); + } + + folio_batch_release(&fbatch); + cond_resched(); + } +} +#else +static void kvm_gmem_make_shared(struct inode *inode, pgoff_t start, pgoff_t end) {} +#endif + +static int __kvm_gmem_set_attributes(struct inode *inode, pgoff_t start, + size_t nr_pages, uint64_t attrs, + pgoff_t *err_index) +{ + bool to_private = attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE; + struct address_space *mapping = inode->i_mapping; + struct gmem_inode *gi = GMEM_I(inode); + enum kvm_gfn_range_filter filter; + pgoff_t end = start + nr_pages; + struct maple_tree *mt; + struct ma_state mas; + int r = 0; + + mt = &gi->attributes; + + filemap_invalidate_lock(mapping); + + if (kvm_gmem_range_has_attributes(inode, start, nr_pages, attrs)) + goto out; + + mas_init(&mas, mt, start); + r = kvm_gmem_mas_preallocate(&mas, attrs, start, nr_pages); + if (r) { + *err_index = start; + goto out; + } + + if (to_private) { + /* + * Forcefully unmap the pages from all userspace page tables, + * and then verify there are no outstanding references, e.g. + * acquired via GUP or similar. Tell userspace to try again if + * there are outstanding references and hope that whatever has + * pinned the page will put its reference "soon". + */ + unmap_mapping_pages(mapping, start, nr_pages, false); + + if (kvm_gmem_has_outstanding_references(inode, start, nr_pages, + err_index)) { + mas_destroy(&mas); + r = -EAGAIN; + goto out; + } + } + + /* + * From this point on guest_memfd has performed necessary + * checks and can proceed to do guest-breaking changes. + */ + + filter = to_private ? KVM_FILTER_SHARED : KVM_FILTER_PRIVATE; + kvm_gmem_invalidate_start(inode, start, end, filter); + + if (!to_private && kvm_arch_has_gmem_convert()) + kvm_gmem_make_shared(inode, start, end); + + mas_store_prealloc(&mas, xa_mk_value(attrs)); + + kvm_gmem_invalidate_end(inode, start, end); +out: + filemap_invalidate_unlock(mapping); + return r; +} + +static long kvm_gmem_set_attributes(struct file *file, void __user *argp) +{ + struct gmem_file *f = file->private_data; + struct inode *inode = file_inode(file); + struct kvm_memory_attributes2 attrs; + pgoff_t err_index; + size_t nr_pages; + pgoff_t index; + int i, r; + + if (copy_from_user(&attrs, argp, sizeof(attrs))) + return -EFAULT; + + if (attrs.flags) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(attrs.reserved); i++) { + if (attrs.reserved[i]) + return -EINVAL; + } + if (!kvm_arch_has_private_mem(f->kvm)) + return -EINVAL; + if (attrs.attributes & ~KVM_MEMORY_ATTRIBUTE_PRIVATE) + return -EINVAL; + if (attrs.size == 0 || attrs.offset + attrs.size < attrs.offset) + return -EINVAL; + if (!PAGE_ALIGNED(attrs.offset) || !PAGE_ALIGNED(attrs.size)) + return -EINVAL; + + if (attrs.offset >= i_size_read(inode) || + attrs.offset + attrs.size > i_size_read(inode)) + return -EINVAL; + + nr_pages = attrs.size >> PAGE_SHIFT; + index = attrs.offset >> PAGE_SHIFT; + r = __kvm_gmem_set_attributes(inode, index, nr_pages, attrs.attributes, + &err_index); + if (r) { + attrs.error_offset = ((uint64_t)err_index) << PAGE_SHIFT; + + if (copy_to_user(argp, &attrs, sizeof(attrs))) + return -EFAULT; + } + + return r; +} + +static long kvm_gmem_ioctl(struct file *file, unsigned int ioctl, + unsigned long arg) +{ + switch (ioctl) { + case KVM_SET_MEMORY_ATTRIBUTES2: + if (!gmem_in_place_conversion) + return -ENOTTY; + + return kvm_gmem_set_attributes(file, (void __user *)arg); + default: + return -ENOTTY; + } +} + static struct file_operations kvm_gmem_fops = { .mmap = kvm_gmem_mmap, .open = generic_file_open, .release = kvm_gmem_release, .fallocate = kvm_gmem_fallocate, + .unlocked_ioctl = kvm_gmem_ioctl, }; static int kvm_gmem_migrate_folio(struct address_space *mapping, @@ -465,14 +824,27 @@ static int kvm_gmem_migrate_folio(struct address_space *mapping, static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *folio) { + struct inode *inode = mapping->host; + enum kvm_gfn_range_filter filter; pgoff_t start, end; - filemap_invalidate_lock_shared(mapping); + /* + * memory_failure() invokes ->error_remove_folio() while holding the + * poisoned folio lock. Do not block on mapping->invalidate_lock, as + * a writer that already holds the invalidate lock can be waiting for + * the same folio, e.g. truncation from MADV_REMOVE/FALLOC_FL_PUNCH_HOLE. + * + * If the invalidate lock is contended, fail the recovery instead of + * risking a deadlock in the memory-failure path. + */ + if (!filemap_invalidate_trylock_shared(mapping)) + return -EBUSY; start = folio->index; end = start + folio_nr_pages(folio); - kvm_gmem_invalidate_start(mapping->host, start, end); + filter = kvm_gmem_get_all_gfns_filter(inode); + kvm_gmem_invalidate_start(inode, start, end, filter); /* * Do not truncate the range, what action is taken in response to the @@ -520,6 +892,51 @@ bool __weak kvm_arch_supports_gmem_init_shared(struct kvm *kvm) return true; } +static int kvm_gmem_init_inode(struct inode *inode, loff_t size, u64 flags) +{ + struct gmem_inode *gi = GMEM_I(inode); + MA_STATE(mas, &gi->attributes, 0, (size >> PAGE_SHIFT) - 1); + u64 attrs; + int r; + + inode->i_op = &kvm_gmem_iops; + inode->i_mapping->a_ops = &kvm_gmem_aops; + inode->i_mode |= S_IFREG; + inode->i_size = size; + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + + /* + * guest_memfd memory is neither migratable nor swappable: set + * inaccessible to gate off both. + */ + mapping_set_inaccessible(inode->i_mapping); + WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); + + gi->flags = flags; + + mt_set_external_lock(&gi->attributes, + &inode->i_mapping->invalidate_lock); + + /* + * Store default attributes for the entire gmem instance. Ensuring every + * index is represented in the maple tree at all times simplifies the + * conversion and merging logic. + */ + attrs = kvm_gmem_get_default_attributes(inode); + + /* + * Acquire the invalidation lock purely to make lockdep happy. The + * maple tree library expects all stores to be protected via the lock, + * and the library can't know when the tree is reachable only by the + * caller, as is the case here. + */ + filemap_invalidate_lock(inode->i_mapping); + r = mas_store_gfp(&mas, xa_mk_value(attrs), GFP_KERNEL); + filemap_invalidate_unlock(inode->i_mapping); + + return r; +} + static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { static const char *name = "[kvm-gmem]"; @@ -550,16 +967,9 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) goto err_fops; } - inode->i_op = &kvm_gmem_iops; - inode->i_mapping->a_ops = &kvm_gmem_aops; - inode->i_mode |= S_IFREG; - inode->i_size = size; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); - mapping_set_inaccessible(inode->i_mapping); - /* Unmovable mappings are supposed to be marked unevictable as well. */ - WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); - - GMEM_I(inode)->flags = flags; + err = kvm_gmem_init_inode(inode, size, flags); + if (err) + goto err_inode; file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, &kvm_gmem_fops); if (IS_ERR(file)) { @@ -603,19 +1013,21 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) return __kvm_gmem_create(kvm, size, flags); } -int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset) +int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset) { uoff_t size = slot->npages << PAGE_SHIFT; - unsigned long start, end; struct gmem_file *f; struct inode *inode; struct file *file; - int r = -EINVAL; + BUILD_BUG_ON(sizeof(gpa_t) != sizeof(offset)); BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff)); + if (WARN_ON_ONCE(slot->flags & KVM_MEMSLOT_GMEM_ONLY)) + return -EINVAL; + file = fget(fd); if (!file) return -EBADF; @@ -632,84 +1044,93 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, if (!PAGE_ALIGNED(offset) || offset + size > i_size_read(inode)) goto err; - filemap_invalidate_lock(inode->i_mapping); - - start = offset >> PAGE_SHIFT; - end = start + slot->npages; - - if (!xa_empty(&f->bindings) && - xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) { - r = -EEXIST; - filemap_invalidate_unlock(inode->i_mapping); - goto err; - } - /* * memslots of flag KVM_MEM_GUEST_MEMFD are immutable to change, so * kvm_gmem_bind() must occur on a new memslot. Because the memslot * is not visible yet, kvm_gmem_get_pfn() is guaranteed to see the file. */ - WRITE_ONCE(slot->gmem.file, file); - slot->gmem.pgoff = start; - if (kvm_gmem_supports_mmap(inode)) + slot->gmem.file = file; + slot->gmem.pgoff = offset >> PAGE_SHIFT; + if (gmem_in_place_conversion || kvm_gmem_supports_mmap(inode)) slot->flags |= KVM_MEMSLOT_GMEM_ONLY; - xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL); - filemap_invalidate_unlock(inode->i_mapping); - /* - * Drop the reference to the file, even on success. The file pins KVM, - * not the other way 'round. Active bindings are invalidated if the - * file is closed before memslots are destroyed. + * Gift the caller a reference to the file. The reference will be + * dropped after bindings are established, or if installing the new + * memslot ultimately fails. */ - r = 0; + return 0; + err: fput(file); - return r; + return -EINVAL; } -static void __kvm_gmem_unbind(struct kvm_memory_slot *slot, struct gmem_file *f) +int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot) { - unsigned long start = slot->gmem.pgoff; - unsigned long end = start + slot->npages; + struct gmem_file *f = slot->gmem.file->private_data; + struct inode *inode = file_inode(slot->gmem.file); + unsigned long start, end; + int r; - xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + if (WARN_ON_ONCE(slot->gmem.file->f_op != &kvm_gmem_fops)) + return -EIO; - /* - * synchronize_srcu(&kvm->srcu) ensured that kvm_gmem_get_pfn() - * cannot see this memslot. - */ - WRITE_ONCE(slot->gmem.file, NULL); + filemap_invalidate_lock(inode->i_mapping); + + start = slot->gmem.pgoff; + end = start + slot->npages; + + if (!xa_empty(&f->bindings) && + xa_find(&f->bindings, &start, end - 1, XA_PRESENT)) { + filemap_invalidate_unlock(inode->i_mapping); + return -EEXIST; + } + + r = xa_err(xa_store_range(&f->bindings, start, end - 1, slot, GFP_KERNEL)); + if (r) + xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + + filemap_invalidate_unlock(inode->i_mapping); + + return r; } void kvm_gmem_unbind(struct kvm_memory_slot *slot) { + struct file *file = slot->gmem.file; + unsigned long start = slot->gmem.pgoff; + unsigned long end = start + slot->npages; + struct gmem_file *f; + /* * Nothing to do if the underlying file was _already_ closed, as * kvm_gmem_release() invalidates and nullifies all bindings. */ - if (!slot->gmem.file) + if (!file) return; - CLASS(gmem_get_file, file)(slot); - /* * However, if the file is _being_ closed, then the bindings need to be * removed as kvm_gmem_release() might not run until after the memslot - * is freed. Note, modifying the bindings is safe even though the file - * is dying as kvm_gmem_release() nullifies slot->gmem.file under - * slots_lock, and only puts its reference to KVM after destroying all - * bindings. I.e. reaching this point means kvm_gmem_release() hasn't - * yet destroyed the bindings or freed the gmem_file, and can't do so - * until the caller drops slots_lock. + * is freed. Modifying the bindings is safe even if the file is dying + * as kvm_gmem_release() nullifies slot->gmem.file under slots_lock, + * and only puts its reference to KVM after destroying all bindings. + * I.e. reaching this point means kvm_gmem_release() hasn't destroyed + * the bindings or freed the gmem_file and can't do so until the caller + * drops slots_lock, so there's no need to verify the file is live. */ - if (!file) { - __kvm_gmem_unbind(slot, slot->gmem.file->private_data); - return; - } + f = file->private_data; filemap_invalidate_lock(file->f_mapping); - __kvm_gmem_unbind(slot, file->private_data); + xa_store_range(&f->bindings, start, end - 1, NULL, GFP_KERNEL); + + /* + * Note, the caller is responsible for ensuring the slot is unreachable + * before unbinding, e.g. by synchronizing SRCU after deleting the slot, + * to guarantee kvm_gmem_get_pfn() can't see the slot+file. + */ + WRITE_ONCE(slot->gmem.file, NULL); filemap_invalidate_unlock(file->f_mapping); } @@ -743,6 +1164,11 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, return ERR_PTR(-EHWPOISON); } + if (!folio_test_uptodate(folio)) { + clear_highpage(folio_page(folio, 0)); + folio_mark_uptodate(folio); + } + *pfn = folio_file_pfn(folio, index); if (max_order) *max_order = 0; @@ -751,8 +1177,7 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, } int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order) + gfn_t gfn, kvm_pfn_t *pfn, int *max_order) { pgoff_t index = kvm_gmem_get_index(slot, gfn); struct folio *folio; @@ -764,28 +1189,24 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, if (!file) return -EFAULT; - folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order); - if (IS_ERR(folio)) - return PTR_ERR(folio); + filemap_invalidate_lock_shared(file_inode(file)->i_mapping); - if (!folio_test_uptodate(folio)) { - clear_highpage(folio_page(folio, 0)); - folio_mark_uptodate(folio); + folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order); + if (IS_ERR(folio)) { + r = PTR_ERR(folio); + goto out; } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - if (kvm_gmem_is_private_mem(file_inode(file), index)) + if (kvm_arch_has_gmem_convert() && + kvm_gmem_is_private_mem(file_inode(file), index)) r = kvm_arch_gmem_make_private(kvm, gfn, *pfn, (kvm_pfn_t)1 << *max_order); -#endif folio_unlock(folio); + folio_put(folio); - if (!r) - *page = folio_file_page(folio, index); - else - folio_put(folio); - +out: + filemap_invalidate_unlock_shared(file_inode(file)->i_mapping); return r; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn); @@ -811,16 +1232,12 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, folio_unlock(folio); - if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1, - KVM_MEMORY_ATTRIBUTE_PRIVATE, - KVM_MEMORY_ATTRIBUTE_PRIVATE)) { + if (!kvm_is_private_gfn(kvm, gfn)) { ret = -EINVAL; goto out_put_folio; } ret = post_populate(kvm, gfn, pfn, src_page, opaque); - if (!ret) - folio_mark_uptodate(folio); out_put_folio: folio_put(folio); @@ -917,6 +1334,15 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) mpol_shared_policy_init(&gi->policy, NULL); + /* + * Memory attributes are protected by the filemap invalidation lock, but + * the lock structure isn't available at this time. Immediately mark + * maple tree as using external locking so that accessing the tree + * before it's fully initialized results in NULL pointer dereferences + * and not more subtle bugs. + */ + mt_init_flags(&gi->attributes, MT_FLAGS_LOCK_EXTERN | MT_FLAGS_USE_RCU); + gi->flags = 0; INIT_LIST_HEAD(&gi->gmem_file_list); return &gi->vfs_inode; @@ -924,7 +1350,26 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) static void kvm_gmem_destroy_inode(struct inode *inode) { - mpol_free_shared_policy(&GMEM_I(inode)->policy); + struct gmem_inode *gi = GMEM_I(inode); + + mpol_free_shared_policy(&gi->policy); + + /* + * Note! Checking for an empty tree is functionally necessary + * to avoid explosions if the tree hasn't been fully + * initialized, i.e. if the inode is being destroyed before + * guest_memfd can set the external lock, lockdep would find + * that the tree's internal ma_lock was not held. + */ + if (!mtree_empty(&gi->attributes)) { + /* + * Acquire the invalidation lock purely to make lockdep happy, + * the inode is unreachable at this point. + */ + filemap_invalidate_lock(inode->i_mapping); + __mt_destroy(&gi->attributes); + filemap_invalidate_unlock(inode->i_mapping); + } } static void kvm_gmem_free_inode(struct inode *inode) diff --git a/virt/kvm/guest_memfd.h b/virt/kvm/guest_memfd.h index 0f9c6f840838..01bd359d27e3 100644 --- a/virt/kvm/guest_memfd.h +++ b/virt/kvm/guest_memfd.h @@ -8,8 +8,9 @@ int kvm_gmem_init(struct module *module); void kvm_gmem_exit(void); int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args); -int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset); +int kvm_gmem_prepare_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset); +int kvm_gmem_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *slot); void kvm_gmem_unbind(struct kvm_memory_slot *slot); #else static inline int kvm_gmem_init(struct module *module) @@ -17,9 +18,17 @@ static inline int kvm_gmem_init(struct module *module) return 0; } static inline void kvm_gmem_exit(void) {}; -static inline int kvm_gmem_bind(struct kvm *kvm, - struct kvm_memory_slot *slot, - unsigned int fd, uoff_t offset) + +static inline int kvm_gmem_prepare_memory_region(struct kvm *kvm, + struct kvm_memory_slot *slot, + unsigned int fd, uoff_t offset) +{ + WARN_ON_ONCE(1); + return -EIO; +} + +static inline int kvm_gmem_commit_memory_region(struct kvm *kvm, + struct kvm_memory_slot *slot) { WARN_ON_ONCE(1); return -EIO; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 5a782849521c..cf4cb586f140 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -10,6 +10,8 @@ * Yaniv Kamay */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include #include @@ -102,6 +104,12 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink); static bool __ro_after_init allow_unsafe_mappings; module_param(allow_unsafe_mappings, bool, 0444); +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM +bool __ro_after_init gmem_in_place_conversion = false; +module_param(gmem_in_place_conversion, bool, 0444); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(gmem_in_place_conversion); +#endif + /* * Ordering of locks: * @@ -470,7 +478,7 @@ static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu) kvm_dirty_ring_free(&vcpu->dirty_ring); /* - * No need for rcu_read_lock as VCPU_RUN is the only place that changes + * No need for rcu_read_lock as KVM_RUN is the only place that changes * the vcpu->pid pointer, and at destruction time all file descriptors * are already gone. */ @@ -813,6 +821,16 @@ static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, /* Pairs with the increment in range_start(). */ spin_lock(&kvm->mn_invalidate_lock); + kvm->gpc_invalidate_seq++; + + /* + * As with the MMU sequence counter and mmu_invalidate_in_progress, the + * GPC sequence increase must be visible before the invalidate count + * goes to zero. Pairs with the smp_rmb() in + * mmu_notifier_retry_cache(). + */ + smp_wmb(); + if (!WARN_ON_ONCE(!kvm->mn_active_invalidate_count)) --kvm->mn_active_invalidate_count; wake = !kvm->mn_active_invalidate_count; @@ -1013,7 +1031,7 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, const char *fdname) mutex_lock(&kvm_debugfs_lock); dent = debugfs_lookup(dir_name, kvm_debugfs_dir); if (dent) { - pr_warn_ratelimited("KVM: debugfs: duplicate directory %s\n", dir_name); + pr_warn_ratelimited("debugfs: duplicate directory %s\n", dir_name); dput(dent); mutex_unlock(&kvm_debugfs_lock); return 0; @@ -1116,7 +1134,7 @@ static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) spin_lock_init(&kvm->mn_invalidate_lock); rcuwait_init(&kvm->mn_memslots_update_rcuwait); xa_init(&kvm->vcpu_array); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES xa_init_flags(&kvm->mem_attr_array, XA_FLAGS_ACCOUNT); #endif @@ -1301,7 +1319,7 @@ static void kvm_destroy_vm(struct kvm *kvm) cleanup_srcu_struct(&kvm->irq_srcu); srcu_barrier(&kvm->srcu); cleanup_srcu_struct(&kvm->srcu); -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES xa_destroy(&kvm->mem_attr_array); #endif kvm_arch_free_vm(kvm); @@ -1314,7 +1332,7 @@ void kvm_get_kvm(struct kvm *kvm) { refcount_inc(&kvm->users_count); } -EXPORT_SYMBOL_GPL(kvm_get_kvm); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_kvm); /* * Make sure the vm is not during destruction, which is a safe version of @@ -1324,14 +1342,14 @@ bool kvm_get_kvm_safe(struct kvm *kvm) { return refcount_inc_not_zero(&kvm->users_count); } -EXPORT_SYMBOL_GPL(kvm_get_kvm_safe); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_kvm_safe); void kvm_put_kvm(struct kvm *kvm) { if (refcount_dec_and_test(&kvm->users_count)) kvm_destroy_vm(kvm); } -EXPORT_SYMBOL_GPL(kvm_put_kvm); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_put_kvm); /* * Used to put a reference that was taken on behalf of an object associated @@ -1352,6 +1370,8 @@ static int kvm_vm_release(struct inode *inode, struct file *filp) kvm_irqfd_release(kvm); + WRITE_ONCE(kvm->file, NULL); + kvm_put_kvm(kvm); return 0; } @@ -1748,10 +1768,10 @@ static void kvm_commit_memory_region(struct kvm *kvm, kvm_destroy_dirty_bitmap(old); /* - * Unbind the guest_memfd instance as needed; the @new slot has - * already created its own binding. TODO: Drop the WARN when - * dirty logging guest_memfd memslots is supported. Until then, - * flags-only changes on guest_memfd slots should be impossible. + * TODO: Drop the WARN and do the unbind() call only for MOVE + * when dirty logging guest_memfd memslots is supported. Until + * then, flags-only changes on guest_memfd slots should also be + * impossible; unbind the old memslot for defense-in-depth. */ if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD)) kvm_gmem_unbind(old); @@ -1937,20 +1957,16 @@ static int kvm_set_memslot(struct kvm *kvm, } r = kvm_prepare_memory_region(kvm, old, new, change); - if (r) { - /* - * For DELETE/MOVE, revert the above INVALID change. No - * modifications required since the original slot was preserved - * in the inactive slots. Changing the active memslots also - * release slots_arch_lock. - */ - if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { - kvm_activate_memslot(kvm, invalid_slot, old); - kfree(invalid_slot); - } else { - mutex_unlock(&kvm->slots_arch_lock); + if (r) + goto err; + + if (change == KVM_MR_CREATE && (new->flags & KVM_MEM_GUEST_MEMFD)) { + r = kvm_gmem_commit_memory_region(kvm, new); + if (r) { + kvm_arch_free_memslot(kvm, new); + kvm_destroy_dirty_bitmap(new); + goto err; } - return r; } /* @@ -1983,6 +1999,20 @@ static int kvm_set_memslot(struct kvm *kvm, kvm_commit_memory_region(kvm, old, new, change); return 0; + +err: + /* + * For DELETE/MOVE, revert the above INVALID change. No modifications + * required since the original slot was preserved in the inactive slots. + * Changing the active memslots also release slots_arch_lock. + */ + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { + kvm_activate_memslot(kvm, invalid_slot, old); + kfree(invalid_slot); + } else { + mutex_unlock(&kvm->slots_arch_lock); + } + return r; } static bool kvm_check_memslot_overlap(struct kvm_memslots *slots, int id, @@ -2111,21 +2141,30 @@ static int kvm_set_memory_region(struct kvm *kvm, new->npages = npages; new->flags = mem->flags; new->userspace_addr = mem->userspace_addr; - if (mem->flags & KVM_MEM_GUEST_MEMFD) { - r = kvm_gmem_bind(kvm, new, mem->guest_memfd, mem->guest_memfd_offset); + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) { + r = kvm_gmem_prepare_memory_region(kvm, new, mem->guest_memfd, + mem->guest_memfd_offset); if (r) goto out; } r = kvm_set_memslot(kvm, old, new, change); + + /* + * Drop the reference to the gmem file, even on success. The file pins + * KVM, not the other way 'round. Active bindings are invalidated if + * the file is closed before memslots are destroyed. + */ +#ifdef CONFIG_KVM_GUEST_MEMFD + if (change == KVM_MR_CREATE && (mem->flags & KVM_MEM_GUEST_MEMFD)) + fput(new->gmem.file); +#endif + if (r) - goto out_unbind; + goto out; return 0; -out_unbind: - if (mem->flags & KVM_MEM_GUEST_MEMFD) - kvm_gmem_unbind(new); out: kfree(new); return r; @@ -2134,7 +2173,7 @@ static int kvm_set_memory_region(struct kvm *kvm, int kvm_set_internal_memslot(struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem) { - if (WARN_ON_ONCE(mem->slot < KVM_USER_MEM_SLOTS)) + if (WARN_ON_ONCE((u16)mem->slot < KVM_USER_MEM_SLOTS)) return -EINVAL; if (WARN_ON_ONCE(mem->flags)) @@ -2425,32 +2464,41 @@ static int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, } #endif /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static u64 kvm_supported_mem_attributes(struct kvm *kvm) +#ifdef kvm_arch_has_private_mem +static u64 kvm_supports_private_mem(struct kvm *kvm) { - if (!kvm || kvm_arch_has_private_mem(kvm)) - return KVM_MEMORY_ATTRIBUTE_PRIVATE; + return !kvm || kvm_arch_has_private_mem(kvm); +} +#else +#define kvm_supports_private_mem(kvm) false +#endif - return 0; +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +static u64 kvm_supported_vm_mem_attributes(struct kvm *kvm) +{ + if (gmem_in_place_conversion || !kvm_supports_private_mem(kvm)) + return 0; + + return KVM_MEMORY_ATTRIBUTE_PRIVATE; } /* * Returns true if _all_ gfns in the range [@start, @end) have attributes * such that the bits in @mask match @attrs. */ -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs) +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs) { XA_STATE(xas, &kvm->mem_attr_array, start); unsigned long index; void *entry; - mask &= kvm_supported_mem_attributes(kvm); + mask &= kvm_supported_vm_mem_attributes(kvm); if (attrs & ~mask) return false; if (end == start + 1) - return (kvm_get_memory_attributes(kvm, start) & mask) == attrs; + return (kvm_get_vm_memory_attributes(kvm, start) & mask) == attrs; guard(rcu)(); @@ -2543,8 +2591,8 @@ static __always_inline void kvm_handle_gfn_range(struct kvm *kvm, KVM_MMU_UNLOCK(kvm); } -static bool kvm_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range) +static bool kvm_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range) { /* * Unconditionally add the range to the invalidation set, regardless of @@ -2559,18 +2607,18 @@ static bool kvm_pre_set_memory_attributes(struct kvm *kvm, */ kvm_mmu_invalidate_range_add(kvm, range->start, range->end); - return kvm_arch_pre_set_memory_attributes(kvm, range); + return kvm_arch_pre_set_vm_memory_attributes(kvm, range); } /* Set @attributes for the gfn range [@start, @end). */ -static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, +static int kvm_set_vm_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, unsigned long attributes) { struct kvm_mmu_notifier_range pre_set_range = { .start = start, .end = end, .arg.attributes = attributes, - .handler = kvm_pre_set_memory_attributes, + .handler = kvm_pre_set_vm_memory_attributes, .on_lock = kvm_mmu_invalidate_start, .flush_on_ret = true, .may_block = true, @@ -2579,7 +2627,7 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, .start = start, .end = end, .arg.attributes = attributes, - .handler = kvm_arch_post_set_memory_attributes, + .handler = kvm_arch_post_set_vm_memory_attributes, .on_lock = kvm_mmu_invalidate_end, .may_block = true, }; @@ -2589,12 +2637,12 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, entry = attributes ? xa_mk_value(attributes) : NULL; - trace_kvm_vm_set_mem_attributes(start, end, attributes); + trace_kvm_set_vm_mem_attributes(start, end, attributes); mutex_lock(&kvm->slots_lock); /* Nothing to do if the entire range has the desired attributes. */ - if (kvm_range_has_memory_attributes(kvm, start, end, ~0, attributes)) + if (kvm_range_has_vm_memory_attributes(kvm, start, end, ~0, attributes)) goto out_unlock; /* @@ -2634,7 +2682,7 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, /* flags is currently not used. */ if (attrs->flags) return -EINVAL; - if (attrs->attributes & ~kvm_supported_mem_attributes(kvm)) + if (attrs->attributes & ~kvm_supported_vm_mem_attributes(kvm)) return -EINVAL; if (attrs->size == 0 || attrs->address + attrs->size < attrs->address) return -EINVAL; @@ -2651,9 +2699,29 @@ static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, */ BUILD_BUG_ON(sizeof(attrs->attributes) != sizeof(unsigned long)); - return kvm_vm_set_mem_attributes(kvm, start, end, attrs->attributes); + return kvm_set_vm_mem_attributes(kvm, start, end, attrs->attributes); } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ + +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM +DEFINE_STATIC_CALL_RET0(__kvm_is_private_gfn, kvm_is_private_gfn_t); +EXPORT_STATIC_CALL_GPL(__kvm_is_private_gfn); + +static bool kvm_vm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return kvm_get_vm_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; +} + +static void kvm_init_memory_attributes(void) +{ + if (gmem_in_place_conversion) + static_call_update(__kvm_is_private_gfn, kvm_gmem_is_private_gfn); + else + static_call_update(__kvm_is_private_gfn, kvm_vm_is_private_gfn); +} +#else +static void kvm_init_memory_attributes(void) { } +#endif struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { @@ -4088,7 +4156,7 @@ static vm_fault_t kvm_vcpu_fault(struct vm_fault *vmf) else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET) page = virt_to_page(vcpu->arch.pio_data); #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET) page = virt_to_page(vcpu->kvm->coalesced_mmio_ring); #endif @@ -4142,9 +4210,9 @@ static struct file_operations kvm_vcpu_fops = { */ static int create_vcpu_fd(struct kvm_vcpu *vcpu) { - char name[8 + 1 + ITOA_MAX_LEN + 1]; + char name[sizeof(__stringify(KVM_DEV_NAME) "-vcpu:") + ITOA_MAX_LEN + 1]; - snprintf(name, sizeof(name), "kvm-vcpu:%d", vcpu->vcpu_id); + snprintf(name, sizeof(name), __stringify(KVM_DEV_NAME) "-vcpu:%d", vcpu->vcpu_id); return anon_inode_getfd(name, &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC); } @@ -4338,9 +4406,9 @@ static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) { int fd; struct file *file; - char name[15 + ITOA_MAX_LEN + 1]; + char name[sizeof(__stringify(KVM_DEV_NAME) "-vcpu-stats:") + ITOA_MAX_LEN + 1]; - snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id); + snprintf(name, sizeof(name), __stringify(KVM_DEV_NAME) "-vcpu-stats:%d", vcpu->vcpu_id); fd = get_unused_fd_flags(O_CLOEXEC); if (fd < 0) @@ -4363,7 +4431,7 @@ static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range) { - int idx; + int idx, err; long r; u64 full_size; @@ -4375,6 +4443,14 @@ static int kvm_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, range->gpa + range->size <= range->gpa) return -EINVAL; + /* + * Certain architectures (e.g. arm64) need to reject the ioctl 'early' + * before vcpu_load(). + */ + err = kvm_arch_pre_fault_allowed(vcpu); + if (err) + return err; + vcpu_load(vcpu); idx = srcu_read_lock(&vcpu->kvm->srcu); @@ -4899,7 +4975,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_DESTROY_MEMORY_REGION_WORKS: case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS: case KVM_CAP_INTERNAL_ERROR_DATA: -#ifdef CONFIG_HAVE_KVM_MSI +#ifdef __KVM_HAVE_MSI case KVM_CAP_SIGNAL_MSI: #endif #ifdef CONFIG_HAVE_KVM_IRQCHIP @@ -4910,7 +4986,7 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_ENABLE_CAP_VM: case KVM_CAP_HALT_POLL: return 1; -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO case KVM_CAP_COALESCED_MMIO: return KVM_COALESCED_MMIO_PAGE_OFFSET; case KVM_CAP_COALESCED_PIO: @@ -4951,15 +5027,20 @@ static int kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_SYSTEM_EVENT_DATA: case KVM_CAP_DEVICE_CTRL: return 1; -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_CAP_MEMORY_ATTRIBUTES: - return kvm_supported_mem_attributes(kvm); + return kvm_supported_vm_mem_attributes(kvm); #endif #ifdef CONFIG_KVM_GUEST_MEMFD case KVM_CAP_GUEST_MEMFD: return 1; case KVM_CAP_GUEST_MEMFD_FLAGS: return kvm_gmem_get_supported_flags(kvm); + case KVM_CAP_GUEST_MEMFD_MEMORY_ATTRIBUTES: + if (!gmem_in_place_conversion || !kvm_supports_private_mem(kvm)) + return 0; + + return KVM_MEMORY_ATTRIBUTE_PRIVATE; #endif default: break; @@ -5153,7 +5234,7 @@ static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm) if (fd < 0) return fd; - file = anon_inode_getfile_fmode("kvm-vm-stats", + file = anon_inode_getfile_fmode(__stringify(KVM_DEV_NAME) "-vm-stats", &kvm_vm_stats_fops, kvm, O_RDONLY, FMODE_PREAD); if (IS_ERR(file)) { put_unused_fd(fd); @@ -5251,7 +5332,7 @@ static long kvm_vm_ioctl(struct file *filp, break; } #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO case KVM_REGISTER_COALESCED_MMIO: { struct kvm_coalesced_mmio_zone zone; @@ -5289,7 +5370,7 @@ static long kvm_vm_ioctl(struct file *filp, r = kvm_ioeventfd(kvm, &data); break; } -#ifdef CONFIG_HAVE_KVM_MSI +#ifdef __KVM_HAVE_MSI case KVM_SIGNAL_MSI: { struct kvm_msi msi; @@ -5355,7 +5436,7 @@ static long kvm_vm_ioctl(struct file *filp, break; } #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES case KVM_SET_MEMORY_ATTRIBUTES: { struct kvm_memory_attributes attrs; @@ -5366,7 +5447,7 @@ static long kvm_vm_ioctl(struct file *filp, r = kvm_vm_ioctl_set_mem_attributes(kvm, &attrs); break; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ case KVM_CREATE_DEVICE: { struct kvm_create_device cd; @@ -5500,11 +5581,16 @@ static struct file_operations kvm_vm_fops = { KVM_COMPAT(kvm_vm_compat_ioctl), }; -bool file_is_kvm(struct file *file) +#ifdef kvm_file_to_kvm_arch +struct kvm *kvm_file_to_kvm_fn(struct file *file) { - return file && file->f_op == &kvm_vm_fops; + if (!file || file->f_op != &kvm_vm_fops) + return NULL; + + return file->private_data; } -EXPORT_SYMBOL_FOR_KVM_INTERNAL(file_is_kvm); +EXPORT_SYMBOL_GPL(kvm_file_to_kvm_fn); +#endif static int kvm_dev_ioctl_create_vm(unsigned long type) { @@ -5525,12 +5611,14 @@ static int kvm_dev_ioctl_create_vm(unsigned long type) goto put_fd; } - file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); + file = anon_inode_getfile(__stringify(KVM_DEV_NAME) "-vm", &kvm_vm_fops, kvm, O_RDWR); if (IS_ERR(file)) { r = PTR_ERR(file); goto put_kvm; } + kvm->file = file; + /* * Don't call kvm_put_kvm anymore at this point; file->f_op is * already set, with ->release() being kvm_vm_release(). In error @@ -5573,7 +5661,7 @@ static long kvm_dev_ioctl(struct file *filp, #ifdef CONFIG_X86 r += PAGE_SIZE; /* pio data page */ #endif -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO r += PAGE_SIZE; /* coalesced mmio ring page */ #endif break; @@ -5591,8 +5679,8 @@ static struct file_operations kvm_chardev_ops = { }; static struct miscdevice kvm_dev = { - KVM_MINOR, - "kvm", + KVM_DEV_MINOR, + __stringify(KVM_DEV_NAME), &kvm_chardev_ops, }; @@ -5626,7 +5714,7 @@ static int kvm_enable_virtualization_cpu(void) return 0; if (kvm_arch_enable_virtualization_cpu()) { - pr_info("kvm: enabling virtualization on CPU%d failed\n", + pr_info("enabling virtualization on CPU%d failed\n", raw_smp_processor_id()); return -EIO; } @@ -5673,7 +5761,7 @@ static void kvm_shutdown(void *data) * is used for both shutdown and reboot scenarios, i.e. neither name is * 100% comprehensive. */ - pr_info("kvm: exiting hardware virtualization\n"); + pr_info("exiting hardware virtualization\n"); on_each_cpu(kvm_disable_virtualization_cpu, NULL, 1); } @@ -6084,7 +6172,7 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx, * attached devices. Otherwise, destroy the caller's device only. */ if (!new_bus) { - pr_err("kvm: failed to shrink bus, removing it completely\n"); + pr_err("failed to shrink bus, removing it completely\n"); kvm_io_bus_destroy(bus); return -ENOMEM; } @@ -6371,7 +6459,7 @@ static void kvm_init_debug(void) const struct kvm_stats_desc *pdesc; int i; - kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); + kvm_debugfs_dir = debugfs_create_dir(__stringify(KVM_DEV_NAME), NULL); for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) { pdesc = &kvm_vm_stats_desc[i]; @@ -6516,7 +6604,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) if (!vcpu_align) vcpu_align = __alignof__(struct kvm_vcpu); kvm_vcpu_cache = - kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align, + kmem_cache_create_usercopy(KVM_DEV_STRING(vcpu), vcpu_size, vcpu_align, SLAB_ACCOUNT, offsetof(struct kvm_vcpu, arch), offsetofend(struct kvm_vcpu, stats_id) @@ -6549,6 +6637,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) kvm_preempt_ops.sched_in = kvm_sched_in; kvm_preempt_ops.sched_out = kvm_sched_out; + kvm_init_memory_attributes(); kvm_init_debug(); r = kvm_vfio_ops_init(); @@ -6569,7 +6658,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) */ r = misc_register(&kvm_dev); if (r) { - pr_err("kvm: misc device register failed\n"); + pr_err("misc device register failed\n"); goto err_register; } diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index 728d2c1b488a..3659686b97c2 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -124,7 +124,7 @@ static void gpc_unmap(kvm_pfn_t pfn, void *khva) #endif } -static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_seq) +static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long gpc_seq) { /* * mn_active_invalidate_count acts for all intents and purposes @@ -136,20 +136,20 @@ static inline bool mmu_notifier_retry_cache(struct kvm *kvm, unsigned long mmu_s * Note, it does not matter that mn_active_invalidate_count * is not protected by gpc->lock. It is guaranteed to * be elevated before the mmu_notifier acquires gpc->lock, and - * isn't dropped until after mmu_invalidate_seq is updated. + * isn't dropped until after gpc_invalidate_seq is updated. */ if (kvm->mn_active_invalidate_count) return true; /* * Ensure mn_active_invalidate_count is read before - * mmu_invalidate_seq. This pairs with the smp_wmb() in - * mmu_notifier_invalidate_range_end() to guarantee either the + * gpc_invalidate_seq. This pairs with the smp_wmb() in + * kvm_mmu_notifier_invalidate_range_end() to guarantee either the * old (non-zero) value of mn_active_invalidate_count or the - * new (incremented) value of mmu_invalidate_seq is observed. + * new (incremented) value of gpc_invalidate_seq is observed. */ smp_rmb(); - return kvm->mmu_invalidate_seq != mmu_seq; + return kvm->gpc_invalidate_seq != gpc_seq; } static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) @@ -158,7 +158,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) void *old_khva = (void *)PAGE_ALIGN_DOWN((uintptr_t)gpc->khva); kvm_pfn_t new_pfn = KVM_PFN_ERR_FAULT; void *new_khva = NULL; - unsigned long mmu_seq; + unsigned long gpc_seq; struct page *page; struct kvm_follow_pfn kfp = { @@ -181,7 +181,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) gpc->valid = false; do { - mmu_seq = gpc->kvm->mmu_invalidate_seq; + gpc_seq = gpc->kvm->gpc_invalidate_seq; smp_rmb(); write_unlock_irq(&gpc->lock); @@ -232,7 +232,7 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc) * attempting to refresh. */ WARN_ON_ONCE(gpc->valid); - } while (mmu_notifier_retry_cache(gpc->kvm, mmu_seq)); + } while (mmu_notifier_retry_cache(gpc->kvm, gpc_seq)); gpc->valid = true; gpc->pfn = new_pfn; diff --git a/virt/kvm/vfio.c b/virt/kvm/vfio.c index 6cdc4e9a333a..19548a430942 100644 --- a/virt/kvm/vfio.c +++ b/virt/kvm/vfio.c @@ -35,15 +35,15 @@ struct kvm_vfio { bool noncoherent; }; -static void kvm_vfio_file_set_kvm(struct file *file, struct kvm *kvm) +static void kvm_vfio_file_set_kvm(struct file *vfio_file, struct file *kvm) { - void (*fn)(struct file *file, struct kvm *kvm); + void (*fn)(struct file *vfio_file, struct file *kvm); fn = symbol_get(vfio_file_set_kvm); if (!fn) return; - fn(file, kvm); + fn(vfio_file, kvm); symbol_put(vfio_file_set_kvm); } @@ -144,6 +144,7 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) { struct kvm_vfio *kv = dev->private; struct kvm_vfio_file *kvf; + struct file *kvm_file __free(fput) = NULL; struct file *filp __free(fput) = NULL; filp = fget(fd); @@ -154,6 +155,10 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) if (!kvm_vfio_file_is_valid(filp)) return -EINVAL; + kvm_file = get_file_active(&dev->kvm->file); + if (!kvm_file) + return -ENOENT; + guard(mutex)(&kv->lock); list_for_each_entry(kvf, &kv->file_list, node) { @@ -168,7 +173,7 @@ static int kvm_vfio_file_add(struct kvm_device *dev, unsigned int fd) kvf->file = get_file(filp); list_add_tail(&kvf->node, &kv->file_list); - kvm_vfio_file_set_kvm(kvf->file, dev->kvm); + kvm_vfio_file_set_kvm(kvf->file, kvm_file); kvm_vfio_update_coherency(dev); return 0;