github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

KVM/arm64 updates for 7.4

Browse Source 
* New features
 
   - pKVM state isolation: years after pKVM started being enabled
     upstream, we finally get the basics in the form of state
     isolation. The vcpu state is now off-limit for the host for
     protected VMs.
 
   - pKVM also grows a hypervisor-private memory allocator, allowing it
     to deal with sub-page allocation, much like everywhere else.
 
   - pKVM also lands a trace_printk() equivalent, which hopefully will
     make debugging a bit easier.
 
   - First shot at annotating portions of the KVM/arm64 code to be
     extracted and built on s390. The real fun is yet to come.
 
   - Allow pre-faulting of stage-2 canonical mappings from userspace,
     joining x86 and s390 in implementing this feature.
 
   - Add a reverse mapping infrastructure for shadow stage-2,
     significantly improving performance when MMU notifiers kick in,
     and avoiding full unmap of all shadow tables for this VM.
 
   - Next step of GICv5 enablement, adding support for the IRS block,
     and providing both SPI and LPI support.  This is the first point
     where KVM/arm64 can support an SMP GICv5 guest.
 
 * Improvements
 
   - As a preliminary to eventually enabling CCA, add a bunch of
     infrastructure to KVM/arm64, splitting some of the basic
     operations into individual backends (normal VM, normal pKVM VM,
     protected pKVM VM). This allows CCA to later slot in without
     polluting the rest of KVM with its firmware-driven nonsense.
 
   - Make ID registers immutable from the point where a vcpu is able
     to run, even if it fails to run. This prevents KVM from sampling
     untrusted values.
 
   - Restore some type checking to the EL2 object. KVM/arm64 originally
     had some strong type checking between the kernel and the EL2
     object, but lost it along the way when the private hypercall
     interface was (somewhat pointlessly) converted to SMCCC. Type
     checking is back, syscall style, and with added sparse goodies.
 
   - Expand the FFA support to FFA_NOTIFICATION_*, and add some
     sanitisation of the protocol that pKVM relays to the "trusted"
     runtime.
 
 * Fixes
 
   - Make GICv3 RD assignment more robust to userspace nonsense
 
   - Convert the remaining MMU fault handler backends to
     kvm_s2_fault_vma_info and co, fixing a number of bugs in the
     process.
 
   - Fix the SError/SEA selftests to be both more robust and less
     braindead.
 
   - Fix a smattering of documentation bugs, some of them as a result
     of our AI friends spitting out credible assertions based on
     non-existing symbols.
 
   - The usual collection of low impact bug fixes (SMCCC, timers, PMU).
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEn9UcU+C1Yxj9lZw9I9DQutE9ekMFAmrLpqsACgkQI9DQutE9
 ekN2fA//audkwURfYzgi/TKSUYu0LI3PRDcSMBd+chHYjSEHdE8yUL/6cX9R8SY4
 jkz9rRj5ypYAzhCDmaC4aY4RYaEeZJEdNkeUccdR4oNWGxwYbHItxDzZdT3UEtpp
 FcekDvKOQFStAY5emMqf1TUZ2bcOiMaHgnfasG8YdDXJe7QNNV8mQ9TaZVBsg3c0
 xEtbCSbj6yJ/udy3kmUIciA3uB5+b+n0pH13rWtyMTfOZkHcLp76HMMTjVUx9F78
 Ov3gFof8VQd/5egR1KBNTEqsiGeBXe1eXDOvmwmqClK3t5ycfhdRC6xc8T9zaja0
 GQ1CTxJE88RWBcuqsxlipwjGTF2e4aG6+ipFjedtBSU8y5/CjVyFf3J7eG+pyyr3
 uVMG6nsgbkCrLR5UA5/5FUNt05xL6f+o0EK0rTr9yzjSBDcrwtXBnlWGD7/SF2gt
 EwPZTF24vaH/qcfveLtfN8shMlmFpZfqmXyPUHpd7CUAthQ0ENta7DRCA6OXg6X5
 Gp7nJgkp5SjsdD1skQdzmttMHIdq3feU4HZwOHwlwB19OwhagcpKK3MzHxKlRD1m
 LdV5zIT3bkNrl+mINlap8HlIZUCgKl9H+/ckoShjtge3CbxYrkU9iZleBMQGhAAD
 HoE02ol/fN78m8wiKqtDXpPbpe5eV3SDwBfU1ialpxcxQ/uCKQ4=
 =C3da
 -----END PGP SIGNATURE-----

Merge tag 'kvmarm-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 updates for 7.4

* New features

  - pKVM state isolation: years after pKVM started being enabled
    upstream, we finally get the basics in the form of state
    isolation. The vcpu state is now off-limit for the host for
    protected VMs.

  - pKVM also grows a hypervisor-private memory allocator, allowing it
    to deal with sub-page allocation, much like everywhere else.

  - pKVM also lands a trace_printk() equivalent, which hopefully will
    make debugging a bit easier.

  - First shot at annotating portions of the KVM/arm64 code to be
    extracted and built on s390. The real fun is yet to come.

  - Allow pre-faulting of stage-2 canonical mappings from userspace,
    joining x86 and s390 in implementing this feature.

  - Add a reverse mapping infrastructure for shadow stage-2,
    significantly improving performance when MMU notifiers kick in,
    and avoiding full unmap of all shadow tables for this VM.

  - Next step of GICv5 enablement, adding support for the IRS block,
    and providing both SPI and LPI support.  This is the first point
    where KVM/arm64 can support an SMP GICv5 guest.

* Improvements

  - As a preliminary to eventually enabling CCA, add a bunch of
    infrastructure to KVM/arm64, splitting some of the basic
    operations into individual backends (normal VM, normal pKVM VM,
    protected pKVM VM). This allows CCA to later slot in without
    polluting the rest of KVM with its firmware-driven nonsense.

  - Make ID registers immutable from the point where a vcpu is able
    to run, even if it fails to run. This prevents KVM from sampling
    untrusted values.

  - Restore some type checking to the EL2 object. KVM/arm64 originally
    had some strong type checking between the kernel and the EL2
    object, but lost it along the way when the private hypercall
    interface was (somewhat pointlessly) converted to SMCCC. Type
    checking is back, syscall style, and with added sparse goodies.

  - Expand the FFA support to FFA_NOTIFICATION_*, and add some
    sanitisation of the protocol that pKVM relays to the "trusted"
    runtime.

* Fixes

  - Make GICv3 RD assignment more robust to userspace nonsense

  - Convert the remaining MMU fault handler backends to
    kvm_s2_fault_vma_info and co, fixing a number of bugs in the
    process.

  - Fix the SError/SEA selftests to be both more robust and less
    braindead.

  - Fix a smattering of documentation bugs, some of them as a result
    of our AI friends spitting out credible assertions based on
    non-existing symbols.

  - The usual collection of low impact bug fixes (SMCCC, timers, PMU).
This commit is contained in:
Paolo Bonzini 2026-10-14 18:26:28 +02:00
commit 439d8f9cb0
122 changed files with 14789 additions and 1324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
Documentation/virt/kvm/api.rst
View File

@ -918,7 +918,8 @@ The irq_type field has the following values:
- KVM_ARM_IRQ_TYPE_SPI:
in-kernel GICv2/GICv3: SPI, irq_id between 32 and 1019 (incl.)
(the vcpu_index field is ignored)
in-kernel GICv5: SPI, irq_id between 0 and 65535 (incl.)
in-kernel GICv5: SPI, irq_id between 0 and the configured
number of SPIs minus one (1023 maximum)
- KVM_ARM_IRQ_TYPE_PPI:
in-kernel GICv2/GICv3: PPI, irq_id between 16 and 31 (incl.)
in-kernel GICv5: PPI, irq_id between 0 and 127 (incl.)
@ -1322,6 +1323,9 @@ guests, across different userspace implementations. Nevertheless, userspace
can still emulate all Arm exceptions by manipulating individual registers
using the KVM_SET_ONE_REG API.
For a protected VM, setting ext_dabt_pending returns -EPERM; see
Documentation/virt/kvm/arm/pkvm.rst. Injecting an SError is unaffected.
See KVM_GET_VCPU_EVENTS for the data structure.
Calling this ioctl on a vCPU that hasn't been initialized will return
@ -1545,7 +1549,9 @@ Possible values are:
KVM_MP_STATE_RUNNABLE the vcpu is currently running
[x86,arm64,riscv,loongarch]
KVM_MP_STATE_UNINITIALIZED the vcpu is an application processor (AP)
which has not yet received an INIT signal [x86]
which has not yet received an INIT signal
[x86], or a protected VM's vcpu that the
guest holds powered off [arm64]
KVM_MP_STATE_INIT_RECEIVED the vcpu has received an INIT signal, and is
now ready for a SIPI [x86]
KVM_MP_STATE_HALTED the vcpu has executed a HLT instruction and
@ -1624,6 +1630,12 @@ For arm64/riscv:
The only states that are valid are KVM_MP_STATE_STOPPED and
KVM_MP_STATE_RUNNABLE which reflect if the vcpu should be paused or not.
On arm64, once a protected VM's vcpu has run, it reads as
KVM_MP_STATE_UNINITIALIZED while the guest holds it powered off, with
CPU_OFF or by not yet bringing it online with CPU_ON, and
KVM_MP_STATE_RUNNABLE and KVM_MP_STATE_SUSPENDED return -EPERM on it: only
the guest can power it on. See Documentation/virt/kvm/arm/pkvm.rst.
On LoongArch, only the KVM_MP_STATE_RUNNABLE state is used to reflect
whether the vcpu is runnable.
@ -2324,7 +2336,8 @@ Errors:
ENOENT no such register
EINVAL invalid register ID, or no such register or used with VMs in
protected virtualization mode on s390
EPERM (arm64) register access not allowed before vcpu finalization
EPERM (arm64) register access not allowed before vcpu
finalization, or after a protected VM's vcpu has run
EBUSY (riscv) changing register value not allowed after the vcpu
has run at least once
====== ============================================================
@ -2949,7 +2962,8 @@ Errors include:
ENOENT no such register
EINVAL invalid register ID, or no such register or used with VMs in
protected virtualization mode on s390
EPERM (arm64) register access not allowed before vcpu finalization
EPERM (arm64) register access not allowed before vcpu
finalization, or after a protected VM's vcpu has run
======== ============================================================
(These error codes are indicative only: do not rely on a specific error
@ -3484,6 +3498,7 @@ Errors:
====== =================================================================
EINVAL the target is unknown, or the combination of features is invalid.
ENOENT a features bit specified is unknown.
EPERM the vcpu belongs to a protected VM and has already run.
====== =================================================================
This tells KVM what type of CPU to present to the guest, and what
@ -3512,6 +3527,10 @@ after the vcpu has been run. This will reset the vcpu to its initial
state. All calls to this function after the initial call must use the same
target and same set of feature flags, otherwise EINVAL will be returned.
For a protected VM this ioctl returns EPERM once the vcpu has run, and the
features it accepts are restricted: KVM_ARM_VCPU_PSCI_0_2 is required. See
Documentation/virt/kvm/arm/pkvm.rst.
Possible features:
- KVM_ARM_VCPU_POWER_OFF: Starts the CPU in a power-off state.
@ -3600,20 +3619,14 @@ Possible features:
:Parameters: struct kvm_vcpu_init (out)
:Returns: 0 on success; -1 on error
Errors:
====== ==========================================
ENODEV no preferred target available for the host
====== ==========================================
This queries KVM for preferred CPU target type which can be emulated
by KVM on underlying host.
The ioctl returns struct kvm_vcpu_init instance containing information
about preferred CPU target type and recommended features for it. The
kvm_vcpu_init->features bitmap returned will have feature bits set if
the preferred target recommends setting these features, but this is
not mandatory.
The ioctl returns a struct kvm_vcpu_init instance containing the
preferred CPU target type. The kvm_vcpu_init->features bitmap is
returned empty: userspace selects the vCPU features itself, and their
availability is reported by the capabilities listed under
KVM_ARM_VCPU_INIT.
The information returned by this ioctl can be used to prepare an instance
of struct kvm_vcpu_init for KVM_ARM_VCPU_INIT ioctl which will result in
@ -3676,7 +3689,7 @@ type KVM_X86_REG_TYPE_MSR, but are NOT enumerated via KVM_GET_REG_LIST.
:Capability: KVM_CAP_ARM_SET_DEVICE_ADDR
:Architectures: arm64
:Type: vm ioctl
:Parameters: struct kvm_arm_device_address (in)
:Parameters: struct kvm_arm_device_addr (in)
:Returns: 0 on success, -1 on error
Errors:
@ -3790,6 +3803,9 @@ can be determined by querying the KVM_CAP_GUEST_DEBUG_HW_BPS and
KVM_CAP_GUEST_DEBUG_HW_WPS capabilities which return a positive number
indicating the number of supported registers.
On arm64, this ioctl returns -EPERM for a protected VM: debugging a
protected guest is not supported. See Documentation/virt/kvm/arm/pkvm.rst.
For ppc, the KVM_CAP_PPC_GUEST_DEBUG_SSTEP capability indicates whether
the single-step debug event (KVM_GUESTDBG_SINGLESTEP) is supported.
@ -6494,7 +6510,7 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details.
---------------------------
:Capability: KVM_CAP_PRE_FAULT_MEMORY
:Architectures: none
:Architectures: x86, s390, arm64
:Type: vcpu ioctl
:Parameters: struct kvm_pre_fault_memory (in/out)
:Returns: 0 if at least one page is processed, < 0 on error
@ -6502,12 +6518,15 @@ See KVM_SET_USER_MEMORY_REGION2 for additional details.
Errors:
========== ===============================================================
EAGAIN A race occurred before progress was made, but a retry may succeed.
EINVAL The specified `gpa` and `size` were invalid (e.g. not
page aligned, causes an overflow, or size is zero), or the VM
is UCONTROL (s390).
ENOENT The specified `gpa` is outside defined memslots.
ENOEXEC The vCPU has not been initialised (arm64).
EINTR An unmasked signal is pending and no page was processed.
EFAULT The parameter address was invalid.
EHWPOISON A poisoned host page was encountered.
EOPNOTSUPP Mapping memory for a GPA is unsupported by the
hypervisor, and/or for the current vCPU state/mode.
EIO unexpected error conditions (also causes a WARN)
@ -6527,7 +6546,17 @@ Errors:
KVM_PRE_FAULT_MEMORY populates KVM's stage-2 page tables used to map memory
for the current vCPU state. KVM maps memory as if the vCPU generated a
stage-2 read page fault, e.g. faults in memory as needed, but doesn't break
CoW. On x86, KVM does not mark any newly created stage-2 PTE as Accessed.
CoW. On arm64, KVM marks newly created stage-2 PTEs as Accessed, as it
does for any stage-2 fault, but leaves the Accessed state of existing PTEs
unchanged. On x86, KVM does not mark any newly created stage-2 PTE as
Accessed, and for s390 it is not applicable.
On arm64, a GPA is interpreted as an IPA, and never interpreted as the IPA
of a nested guest. Pre-faulting only populates canonical stage-2 page
tables.
The feature is not supported on arm64 if the protected KVM (pKVM) feature
is enabled.
In the case of confidential VM types where there is an initial set up of
private guest memory before the guest is 'finalized'/measured, this ioctl
@ -6542,7 +6571,7 @@ When the ioctl returns, the input values are updated to point to the
remaining range. If `size` > 0 on return, the caller can just issue
the ioctl again with the same `struct kvm_map_memory` argument.
Shadow page tables cannot support this ioctl because they
On x86, shadow page tables cannot support this ioctl because they
are indexed by virtual address or nested guest physical address.
Calling this ioctl when the guest is using shadow page tables (for
example because it is running a nested guest with nested page tables)

2
Documentation/virt/kvm/arm/fw-pseudo-registers.rst
View File

@ -33,6 +33,8 @@ The following registers are defined:
highest PSCI version implemented by KVM and compatible with v0.2)
- Allows any PSCI version implemented by KVM and compatible with
v0.2 to be set with SET_ONE_REG
- Returns -EINVAL for a version below v1.1 on a protected VM (see
Documentation/virt/kvm/arm/pkvm.rst)
- Affects the whole VM (even if the register view is per-vcpu)
* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1:

164
Documentation/virt/kvm/arm/pkvm.rst
View File

@ -15,7 +15,9 @@ Overview
Booting a host kernel with '``kvm-arm.mode=protected``' enables
"Protected KVM" (pKVM). During boot, pKVM installs a stage-2 identity
map page-table for the host and uses it to isolate the hypervisor
running at EL2 from the rest of the host running at EL1/0.
running at EL2 from the rest of the host running at EL1/0. pKVM supports
GICv3 only: GICv2 is not supported in protected mode, and neither are
GICv5-based guests.
pKVM permits creation of protected virtual machines (pVMs) by passing
the ``KVM_VM_TYPE_ARM_PROTECTED`` machine type identifier to the
@ -31,7 +33,7 @@ See hypercalls.rst for more details.
Isolation mechanisms
====================
pKVM relies on a number of mechanisms to isolate PVMs from the host:
pKVM relies on a number of mechanisms to isolate pVMs from the host:
CPU memory isolation
--------------------
@ -50,8 +52,9 @@ stage-2 identity map as part of the donation procedure. This gives rise
to some user-visible differences when compared to non-protected VMs,
largely due to the lack of MMU notifiers:
* Memslots cannot be moved or deleted once the pVM has started running.
* Read-only memslots and dirty logging are not supported.
* Memslots cannot be moved or deleted once the pVM has started running
(``-EPERM``).
* Read-only memslots and dirty logging are not supported (``-EPERM``).
* With the exception of swap, file-backed pages cannot be mapped into a
pVM.
* Donated pages are accounted against ``RLIMIT_MLOCK`` and so the VMM
@ -67,12 +70,19 @@ largely due to the lack of MMU notifiers:
then it will either return ``-EFAULT`` or forcefully reclaim the
memory pages. Reclaimed memory is zeroed by the hypervisor and a
subsequent attempt to access it in the pVM will return ``-EFAULT``
from the ``VCPU_RUN`` ioctl().
from the ``KVM_RUN`` ioctl().
CPU state isolation
-------------------
Status: **Unimplemented.**
Status: CPU register state of protected vCPUs is managed entirely at EL2.
pKVM performs the complete context switch for protected vCPUs within the
hypervisor. The hypervisor sets a protected vCPU's initial state, not the
host, and only what each exit needs is synchronised back.
The user-visible consequences are described under `API behaviour for
protected VMs`_.
DMA isolation using an IOMMU
----------------------------
@ -89,13 +99,153 @@ The FF-A proxy ensures that the host cannot share pVM or hypervisor
memory with Trustzone as part of a "confused deputy" attack.
The PSCI proxy ensures that CPUs always have the stage-2 identity map
installed when they are executing in the host.
installed when they are executing in the host. This proxy is distinct
from the PSCI handling provided to protected guests, which is described
under `API behaviour for protected VMs`_.
Protected VM firmware (pvmfw)
-----------------------------
Status: **Unimplemented.**
API behaviour for protected VMs
===============================
Protected vCPU state is owned by EL2 (see `CPU state isolation`_). The VMM
configures a vCPU before its first ``KVM_RUN``; afterwards the state is
private to the guest and the ioctls that access it return ``-EPERM``. The
vCPU ioctls' errors follow one rule: ``-EPERM`` means the host asked for
state that the guest owns, ``-EINVAL`` means the request is not valid for a
protected VM, and a feature pKVM doesn't support for protected VMs is
rejected the way KVM rejects one that isn't implemented (``-ENXIO`` from
``KVM_ARM_VCPU_PVTIME_CTRL``, for example). The ioctls themselves are
described in Documentation/virt/kvm/api.rst.
Boot
----
A protected VM boots from a single primary vCPU. Before the first
``KVM_RUN``, the VMM prepares the boot state:
* Set ``KVM_MP_STATE_RUNNABLE`` on the primary vCPU and
``KVM_MP_STATE_STOPPED`` on every other vCPU. EL2 allows only one
RUNNABLE primary per protected VM. A second RUNNABLE vCPU fails at its
first ``KVM_RUN``.
* Set the primary vCPU's boot state with ``KVM_SET_ONE_REG``: the kernel
entry address in ``PC`` and the DTB pointer in ``x0``.
``PC`` and ``x0`` are the only registers EL2 takes from the host. Other
pre-run writes are accepted, but the guest starts from the state the
hypervisor resets it to.
Secondary vCPUs are started by the guest itself through PSCI ``CPU_ON``
(see `Power state`_), which supplies their entry point and context ID.
The VMM cannot choose where they boot.
vCPU state
----------
* ``KVM_GET_ONE_REG`` and ``KVM_SET_ONE_REG`` return ``-EPERM`` once the
vCPU has run. Before that, they access the host-side copy from which
EL2 builds the guest's boot state (see `Boot`_).
* ``KVM_ARM_VCPU_INIT`` accepts only the vCPU features that a protected
guest supports and returns ``-EINVAL`` otherwise.
``KVM_ARM_VCPU_PSCI_0_2`` is required, as EL2 implements PSCI 1.1 for
the guest (see `Power state`_). ``KVM_ARM_VCPU_EL1_32BIT`` is not
supported: protected guests run in AArch64 only and see no AArch32
support in ``ID_AA64PFR0_EL1``. Once the vCPU has run,
``KVM_ARM_VCPU_INIT`` returns ``-EPERM``, as re-initialising it would
reset the host-side copy alone.
* ``KVM_SET_VCPU_EVENTS`` returns ``-EPERM`` for external-abort injection
(``ext_dabt_pending``), and ``KVM_CAP_ARM_INJECT_EXT_DABT`` is not
advertised to a protected VM. SError injection is unaffected.
* ``KVM_SET_GUEST_DEBUG`` returns ``-EPERM``, and ``KVM_CAP_SET_GUEST_DEBUG``
is not advertised to a protected VM (see `Debug`_).
Power state
-----------
EL2 implements PSCI 1.1 for a protected guest. The calls that move a
vCPU's power state, ``CPU_ON`` and ``CPU_OFF``, are handled at EL2, and
the host cannot change the outcome: for ``CPU_ON`` the host only
schedules the target, which EL2 has already reset to the entry point
the guest chose, and for ``CPU_OFF`` it only stops scheduling it.
``AFFINITY_INFO`` is handled by the host, which returns ``OFF`` once
it has stopped scheduling every vCPU the query covers, as for a
non-protected VM. A vCPU becomes a valid ``CPU_ON`` target at its first
``KVM_RUN``, and before that EL2 returns ``INVALID_PARAMETERS``. The
platform calls, ``CPU_SUSPEND``, ``SYSTEM_OFF``, ``SYSTEM_RESET`` and
``SYSTEM_RESET2``, are forwarded to the host and behave as for a
non-protected VM, with the ``SYSTEM_*`` calls exiting to the VMM as
``KVM_EXIT_SYSTEM_EVENT``; the ``SYSTEM_RESET2`` reset type and cookie
are in the guest's registers, which ``KVM_GET_ONE_REG`` rejects once
the vCPU has run. Any other function returns ``NOT_SUPPORTED``, and
``PSCI_FEATURES`` reports the same set. Because the host handles those
forwarded calls, ``KVM_SET_ONE_REG`` on ``KVM_REG_ARM_PSCI_VERSION``
returns ``-EINVAL`` for a version below 1.1.
Once a vCPU has run, its power state follows the guest's PSCI calls, not
the VMM's. A vCPU that the guest has powered off with ``CPU_OFF``, or has
not yet brought online with ``CPU_ON``, reads as
``KVM_MP_STATE_UNINITIALIZED``, and only an in-guest ``CPU_ON`` can power
it on: ``KVM_SET_MP_STATE`` returns ``-EPERM`` for ``KVM_MP_STATE_RUNNABLE``
and ``KVM_MP_STATE_SUSPENDED`` on it, accepts ``KVM_MP_STATE_STOPPED`` and
leaves it as it is, and never takes ``KVM_MP_STATE_UNINITIALIZED`` from
the VMM. ``KVM_MP_STATE_STOPPED`` still stops a vCPU EL2 has powered on,
and ``KVM_MP_STATE_RUNNABLE`` resumes it, so the VMM can pause it.
Other interface differences
---------------------------
* ``KVM_CHECK_EXTENSION`` reports only the arm64 capabilities that pKVM
supports for protected guests, and ``KVM_ENABLE_CAP`` accepts only
those. Query them on the VM file descriptor: the system file descriptor
has no VM to filter against. The capabilities the generic KVM code
answers itself (``KVM_CAP_SYNC_MMU`` or ``KVM_CAP_DEVICE_CTRL``, for
example) are reported as for any VM, and don't mean the feature works
for a protected one. The filter does not cover every interface either:
device-fd configuration (for example the VGIC after
``KVM_CREATE_DEVICE``) and vCPU attributes are unfiltered, so a feature
the filter excludes is rejected by the attribute itself.
``KVM_ARM_VCPU_PVTIME_CTRL`` returns ``-ENXIO``, for example, as it does
where steal time isn't implemented.
* The vGIC of a protected VM remains host-managed: device creation,
configuration and interrupt injection all work as they do for a
non-protected VM.
* ``KVM_ARM_SET_COUNTER_OFFSET`` and ``KVM_ARM_GET_REG_WRITABLE_MASKS``
return ``-EINVAL``: their capabilities are not advertised to a protected
VM, whose counter offset and ID registers are set by EL2. A protected
guest sees the physical timebase.
* A protected guest's first access to each page of memory exits to the
host, since the hypervisor cannot tell memory from a device before the
page is mapped. For a store, the host sees the value of the register
the syndrome names, clamped to the access width, and nothing else from
the register file. For a load, the register the syndrome names takes
the value the host returns, as for any emulated access.
* A protected guest that uses a feature it was not given, or executes an
``SMC``, takes an undefined instruction exception from the hypervisor;
the host is not involved.
* Apart from PSCI, the hypervisor handles a protected guest's SMCCC
calls itself and does not involve the host. A function it does not
implement returns ``NOT_SUPPORTED``, and the host's SMCCC filter does
not apply: the VM device attribute ioctls (``KVM_ARM_VM_SMCCC_CTRL``)
return ``-EINVAL`` for a protected VM.
* The hypervisor can decode a trapped guest access only from the CPU's
instruction syndrome, which is provided only for a load or store of a
single general-purpose register. An access without one (for example a
load/store pair or a SIMD/FP access) cannot be decoded. For a
non-protected VM it can exit to the VMM as ``KVM_EXIT_ARM_NISV``. For a
protected VM it cannot be emulated by the VMM, so the guest takes a
synchronous external abort instead.
Debug
-----
Hardware-assisted debugging is not available to protected guests.
``MDSCR_EL1``, ``OSLAR_EL1``, ``OSLSR_EL1``, ``OSDLR_EL1``, and breakpoint
and watchpoint 0 are RAZ/WI; any other debug register access takes an
undefined instruction exception.
Resources
=========

271
Documentation/virt/kvm/devices/arm-vgic-v5.rst
View File

@ -12,12 +12,39 @@ Only one VGIC instance may be instantiated through this API. The created VGIC
will act as the VM interrupt controller, requiring emulated user-space devices
to inject interrupts to the VGIC instead of directly to CPUs.
Creating a guest GICv5 device requires a GICv5 host. The current VGICv5 device
only supports PPI interrupts. These can either be injected from emulated
in-kernel devices (such as the Arch Timer, or PMU), or via the KVM_IRQ_LINE
ioctl.
Creating a guest GICv5 device requires a GICv5 host. The VGICv5 device supports
PPI, SPI, and LPI interrupts. The PPI and SPI interrupts can either be injected
from emulated in-kernel devices (such as the Arch Timer, or PMU), or via the
KVM_IRQ_LINE ioctl. LPIs are not externally injected, but are handled in
hardware via the LPI IST. Their pending state is driven directly by the guest.
Groups:
KVM_DEV_ARM_VGIC_GRP_ADDR
Attributes:
KVM_VGIC_V5_ADDR_TYPE_IRS (rw, 64-bit)
Base address in the guest physical address space of the GICv5 IRS
(Interrupt Routing Service) register mappings. Only valid for
KVM_DEV_TYPE_ARM_VGIC_V5. This address needs to be 64K aligned and the
region covers 128 KByte - the IRS has a CONFIG_FRAME and a SETLPI_FRAME,
each of which is 64 KBytes in size.
Setting the address of the IRS in GPA space is mandatory before VGIC
resources are mapped, as the IRS is responsible for handling SPIs and
LPIs. Failure to set the IRS address before the first vCPU run results in
an error.
KVM_DEV_ARM_VGIC_GRP_NR_IRQS
Attributes:
A value describing the number of SPIs for this GIC instance. This is
GICv5-specific: unlike GICv2/v3, the value does not include SGIs or PPIs.
The value ranges from 32 to KVM's VGICv5 maximum of 1024 SPIs, in
increments of 32. If userspace does not set this attribute, KVM uses 32
SPIs by default.
kvm_device_attr.addr points to a __u32 value.
KVM_DEV_ARM_VGIC_GRP_CTRL
Attributes:
@ -48,3 +75,239 @@ Groups:
-EFAULT Invalid guest ram access
-EBUSY One or more VCPUS are running
======= ========================================================
KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS
Attributes:
The attr field of kvm_device_attr encodes two values::
bits: | 63 .... 32 | 31 .... 16 | 15 .... 0 |
values: | mpidr | RES | instr |
The mpidr field encodes the CPU ID based on the affinity information in the
architecture defined MPIDR, and the field is encoded as follows::
| 63 .... 56 | 55 .... 48 | 47 .... 40 | 39 .... 32 |
| Aff3 | Aff2 | Aff1 | Aff0 |
The instr field encodes the system register to access based on the fields
defined in the A64 instruction set encoding for system register access
(RES means the bits are reserved for future use and should be zero)::
| 15 ... 14 | 13 ... 11 | 10 ... 7 | 6 ... 3 | 2 ... 0 |
| Op 0 | Op1 | CRn | CRm | Op2 |
All system regs accessed through this API are (rw, 64-bit) and
kvm_device_attr.addr points to a __u64 value.
KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS accesses the CPU interface registers for the
CPU specified by the mpidr field.
The available registers are:
======================= ===================================================
ICC_ICSR_EL1
ICC_PPI_ENABLER0_EL1
ICC_PPI_ENABLER1_EL1
ICC_PPI_SACTIVER0_EL1 ICC_PPI_CACTIVER0_EL1 is not supported. Writes to
ICC_PPI_SACTIVER0_EL1 are treated as RAW writes of
the underlying state.
ICC_PPI_SACTIVER1_EL1 ICC_PPI_CACTIVER1_EL1 is not supported. Writes to
ICC_PPI_SACTIVER1_EL1 are treated as RAW writes of
the underlying state.
ICC_PPI_SPENDR0_EL1 ICC_PPI_CPENDR0_EL1 is not supported. Writes to
ICC_PPI_SPENDR0_EL1 are treated as RAW writes of
the underlying state.
ICC_PPI_SPENDR1_EL1 ICC_PPI_CPENDR1_EL1 is not supported. Writes to
ICC_PPI_SPENDR1_EL1 are treated as RAW writes of
the underlying state.
ICC_PPI_PRIORITYR0_EL1
ICC_PPI_PRIORITYR1_EL1
ICC_PPI_PRIORITYR2_EL1
ICC_PPI_PRIORITYR3_EL1
ICC_PPI_PRIORITYR4_EL1
ICC_PPI_PRIORITYR5_EL1
ICC_PPI_PRIORITYR6_EL1
ICC_PPI_PRIORITYR7_EL1
ICC_PPI_PRIORITYR8_EL1
ICC_PPI_PRIORITYR9_EL1
ICC_PPI_PRIORITYR10_EL1
ICC_PPI_PRIORITYR11_EL1
ICC_PPI_PRIORITYR12_EL1
ICC_PPI_PRIORITYR13_EL1
ICC_PPI_PRIORITYR14_EL1
ICC_PPI_PRIORITYR15_EL1
ICC_APR_EL1
ICC_CR0_EL1
ICC_PCR_EL1
======================= ===================================================
Errors:
======= =============================================================
-ENXIO Getting or setting this register is not supported
-EBUSY VCPU is running, or write attempted after a VCPU has run
-EINVAL Invalid mpidr or register value supplied
======= =============================================================
KVM_DEV_ARM_VGIC_GRP_IRS_REGS
Attributes:
The attr field of kvm_device_attr encodes the offset of the IRS register,
relative to the IRS CONFIG_FRAME base address. This is the address that
was provided via KVM_VGIC_V5_ADDR_TYPE_IRS when creating VGICv5 in the
first place.
kvm_device_attr.addr points to a __u64 value whatever the width
of the addressed register (32/64 bits). 64 bit registers can only
be accessed with full length.
Writes to read-only registers are ignored by the kernel except for
IRS_IDR0 - IRS_IDR2 and IRS_IDR5 - IRS_IDR7, which are sanity checked to
ensure that they match a sane config.
IRS_IDR3 and IRS_IDR4 are RAZ/WI, as nested virtualization is not
supported.
For registers without dedicated userspace accessors, getting or setting a
register uses the same emulated MMIO handlers as guest reads/writes.
Dedicated userspace accessors may instead save or restore migration state
without triggering guest-visible side effects. For example, restoring
IRS_IST_BASER only restores the emulated register state; any host LPI IST
allocation based on the restored IRS_IST_CFGR and IRS_IST_BASER state
happens when KVM_DEV_ARM_VGIC_GRP_IST is restored.
Once an LPI IST has been allocated, IRS_IST_CFGR is immutable. Userspace
may write back its current value, but setting a different value is
rejected with -EINVAL.
Once an LPI IST has been allocated, IRS_IST_BASER is immutable.
Userspace may write back its current value, but setting a different value
is rejected with -EINVAL.
Errors:
======= =================================================================
-ENXIO Offset does not correspond to any supported register
-EFAULT Invalid user pointer for attr->addr
-EINVAL Offset is not 32-bit aligned for 32-bit MMIO registers, not
64-bit aligned for 64-bit registers, or the supplied register
value is not compatible with the configured VGICv5 IRS state
-EBUSY VGIC is not initialized, one or more VCPUs are running, or a
write is attempted after a VCPU has run
======= =================================================================
KVM_DEV_ARM_VGIC_GRP_IST
Attributes:
This interface is used to either save the state of the IRS's Interrupt
State Tables (ISTs), or to restore them. A get operation saves IST state,
and a set operation restores IST state. kvm_device_attr.attr is reserved
and must be zero.
The VGIC must be initialized before using this interface. Restore must be
performed before the VM has run. For restore, userspace must have already
restored the IRS state needed to describe any guest LPI IST.
Saving first asks the IRS to save and quiesce the VM so that interrupt
state has been written back to the ISTs. KVM checks that the VM remains
quiesced while copying out the SPI and LPI IST state.
kvm_device_attr.addr points to a struct kvm_vgic_v5_ist::
struct kvm_vgic_v5_ist {
__u64 spi_ist_addr;
__u64 spi_ist_size;
__u64 lpi_ist_addr;
__u64 lpi_ist_size;
};
The SPI and LPI IST buffers contain one little-endian 32-bit IST entry per
interrupt, in interrupt number order. Only the architected 32-bit ISTE
state is exposed to userspace; host IST layout and metadata are private to
KVM.
The SPI IST buffer is required and its size must be:
nr_spis * sizeof(__u32)
where nr_spis is the value returned by KVM_DEV_ARM_VGIC_GRP_NR_IRQS for
the VGICv5 device. For VGICv5 this value is the number of SPIs, not the
total number of interrupts.
The LPI IST buffer is required if the guest's IRS_IST_CFGR and
IRS_IST_BASER state describes a valid LPI IST, and its size must be:
BIT(lpi_id_bits) * sizeof(__u32)
where lpi_id_bits is the LPI ID width configured in IRS_IST_CFGR. If no
LPI IST is configured, lpi_ist_addr and lpi_ist_size must both be zero.
Errors:
=========== ============================================================
-EBUSY One or more VCPUs are running, the VGIC is not initialized,
restore was requested after the VM has run, an LPI IST
already exists, or the save operation completed but the VM
did not remain quiesced
-EINVAL attr->addr is NULL, the userspace IST descriptor is missing
a required buffer, supplies a non-zero LPI buffer when no
LPI IST is configured, has a buffer size that does not match
the configured VGICv5 IST state, or an internal VM table
operation rejected the VM state
-EFAULT Invalid user pointer for attr->addr, spi_ist_addr, or
lpi_ist_addr
-ENXIO Required per-VM VGICv5/IST backing state is missing or
inconsistent
-ENOMEM Restoring IST state failed while allocating the host LPI IST
or tracking pending interrupts
-ETIMEDOUT An IRS save/VM operation timed out
=========== ============================================================
IRS Save Sequence:
------------------
The following operations are required when saving the virtual GICv5 IRS:
a) Save the ISTs by issuing KVM_GET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_IST.
b) Save the IRS MMIO register state by issuing KVM_GET_DEVICE_ATTR on
KVM_DEV_ARM_VGIC_GRP_IRS_REGS.
These two steps may be performed in either order. Saving the ISTs writes all
IST state to userspace-provided buffers and does not update guest memory.
IRS Restore Sequence:
---------------------
The following ordering must be followed when restoring the virtual GICv5 and
IRS:
a) Create vCPUs.
b) Provide the IRS base address by issuing KVM_SET_DEVICE_ATTR on
KVM_DEV_ARM_VGIC_GRP_ADDR
c) Restore the number of SPIs by issuing KVM_SET_DEVICE_ATTR on
KVM_DEV_ARM_VGIC_GRP_NR_IRQS.
d) Initialise the GIC - this sets up the default state and creates the SPI
IST - by issuing KVM_SET_DEVICE_ATTR on KVM_DEV_ARM_VGIC_GRP_CTRL with
KVM_DEV_ARM_VGIC_CTRL_INIT
e) Restore the IRS MMIO register state by issuing KVM_SET_DEVICE_ATTR on
KVM_DEV_ARM_VGIC_GRP_IRS_REGS. The IRS registers may be restored in any
order, but any IRS_IDR* state and the IRS_IST_CFGR/IRS_IST_BASER state
describing an LPI IST must be restored before restoring the ISTs. KVM uses
the restored IRS_IST_CFGR and IRS_IST_BASER state to allocate the LPI IST
during the following step.
f) Restore the ISTs by issuing KVM_SET_DEVICE_ATTR on
KVM_DEV_ARM_VGIC_GRP_IST.
The number of SPIs must be restored before VGIC initialization because
initialization allocates the SPI state and fixes the SPI range exposed by the
IRS ID registers.
The IRS's MMIO registers must be restored prior to restoring the ISTs as these
are used to convey the number of LPIs the guest has configured to KVM.
The various ``*_STATUSR`` registers are observational state in the current KVM
implementation. Userspace may save them for validation or debugging purposes,
but they are not required as restore input and do not need to be replayed during
restore.
Then vCPUs can be started.

5
Documentation/virt/kvm/devices/vcpu.rst
View File

@ -135,7 +135,7 @@ after selecting a hardware PMU.
Request that the VCPU uses the specified hardware PMU when creating guest events
for the purpose of PMU emulation. The PMU identifier can be read from the "type"
file for the desired PMU instance under /sys/devices (or, equivalent,
/sys/bus/even_source). This attribute is particularly useful on heterogeneous
/sys/bus/event_source). This attribute is particularly useful on heterogeneous
systems where there are at least two CPU PMUs on the system. The PMU that is set
for one VCPU will be used by all the other VCPUs. It isn't possible to set a PMU
if a PMU event filter is already present.
@ -231,7 +231,8 @@ Returns:
Specifies the base address of the stolen time structure for this VCPU. The
base address must be 64 byte aligned and exist within a valid guest memory
region. See Documentation/virt/kvm/arm/pvtime.rst for more information
including the layout of the stolen time structure.
including the layout of the stolen time structure. Stolen time is not
supported for protected VMs (see Documentation/virt/kvm/arm/pkvm.rst).
4. GROUP: KVM_VCPU_TSC_CTRL
===========================

3
arch/arm64/Makefile
View File

@ -45,6 +45,9 @@ KBUILD_CFLAGS += $(CC_FLAGS_NO_FPU) \
KBUILD_CFLAGS += $(call cc-disable-warning, psabi)
KBUILD_AFLAGS += $(compat_vdso)
# Enable all code shared to s390
KBUILD_CPPFLAGS += -DARM64_S390_COMMON=1
ifeq ($(call rustc-min-version, 108500),y)
KBUILD_RUSTFLAGS += --target=aarch64-unknown-none-softfloat
else

1
arch/arm64/include/asm/brk-imm.h
View File

@ -2,6 +2,7 @@
/*
* Copyright (C) 2012 ARM Ltd.
*/
/* Whole file is shared with s390 */
#ifndef __ASM_BRK_IMM_H
#define __ASM_BRK_IMM_H

34
arch/arm64/include/asm/esr.h
View File

@ -3,11 +3,12 @@
* Copyright (C) 2013 - ARM Ltd
* Author: Marc Zyngier <marc.zyngier@arm.com>
*/
/* Whole file is shared with s390 */
#ifndef __ASM_ESR_H
#define __ASM_ESR_H
#include <asm/memory.h>
#include <asm/brk-imm.h>
#include <asm/sysreg.h>
#define ESR_ELx_EC_UNKNOWN UL(0x00)
@ -435,7 +436,36 @@
#define ESR_ELx_IT_GCSPOPX 7
#ifndef __ASSEMBLER__
#include <asm/types.h>
#include <linux/types.h>
static __always_inline bool esr_trap_is_iabt(unsigned long esr)
{
return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
}
/* Always check for S1PTW *before* using this. */
static __always_inline bool esr_dabt_is_write(unsigned long esr)
{
return esr & ESR_ELx_WNR;
}
static __always_inline bool esr_dabt_is_cm(unsigned long esr)
{
return esr & ESR_ELx_CM;
}
static __always_inline bool esr_abt_is_sea(unsigned long esr)
{
switch (esr & ESR_ELx_FSC) {
case ESR_ELx_FSC_EXTABT:
case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3):
case ESR_ELx_FSC_SECC:
case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3):
return true;
default:
return false;
}
}
static inline unsigned long esr_brk_comment(unsigned long esr)
{

6
arch/arm64/include/asm/kvm_arm.h
View File

@ -3,14 +3,16 @@
* Copyright (C) 2012,2013 - ARM Ltd
* Author: Marc Zyngier <marc.zyngier@arm.com>
*/
/* Whole file is shared with s390 */
#ifndef __ARM64_KVM_ARM_H__
#define __ARM64_KVM_ARM_H__
#include <linux/const.h>
#include <linux/bits.h>
#include <linux/types.h>
#include <asm/esr.h>
#include <asm/memory.h>
#include <asm/sysreg.h>
#include <asm/types.h>
/*
* Because I'm terribly lazy and that repainting the whole of the KVM

11
arch/arm64/include/asm/kvm_asm.h
View File

@ -62,6 +62,7 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___kvm_enable_ssbs,
__KVM_HOST_SMCCC_FUNC___vgic_v3_init_lrs,
__KVM_HOST_SMCCC_FUNC___vgic_v3_get_gic_config,
__KVM_HOST_SMCCC_FUNC___pkvm_hyp_alloc_selftest,
MARKER(__KVM_HOST_SMCCC_FUNC_MIN_PKVM),
@ -87,14 +88,20 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___tracing_write_event,
__KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs,
__KVM_HOST_SMCCC_FUNC___vgic_v3_restore_vmcr_aprs,
__KVM_HOST_SMCCC_FUNC___vgic_v5_make_resident,
__KVM_HOST_SMCCC_FUNC___vgic_v5_make_non_resident,
__KVM_HOST_SMCCC_FUNC___vgic_v5_vdpend,
__KVM_HOST_SMCCC_FUNC___vgic_v5_save_apr,
__KVM_HOST_SMCCC_FUNC___vgic_v5_restore_vmcr_apr,
__KVM_HOST_SMCCC_FUNC___pkvm_hyp_topup,
__KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaim,
__KVM_HOST_SMCCC_FUNC___pkvm_hyp_reclaimable,
__KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp,
__KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp,
MARKER(__KVM_HOST_SMCCC_FUNC_PKVM_ONLY),
/* Hypercalls that are available only when pKVM has finalised. */
__KVM_HOST_SMCCC_FUNC___pkvm_host_share_hyp,
__KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_hyp,
__KVM_HOST_SMCCC_FUNC___pkvm_host_donate_guest,
__KVM_HOST_SMCCC_FUNC___pkvm_host_share_guest,
__KVM_HOST_SMCCC_FUNC___pkvm_host_unshare_guest,

124
arch/arm64/include/asm/kvm_emulate.h
View File

@ -44,6 +44,7 @@ enum exception_type {
bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
void kvm_skip_instr32(struct kvm_vcpu *vcpu);
#if ARM64_S390_COMMON == 1
void kvm_inject_undefined(struct kvm_vcpu *vcpu);
void kvm_inject_sync(struct kvm_vcpu *vcpu, u64 esr);
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr);
@ -75,6 +76,8 @@ static inline int kvm_inject_serror(struct kvm_vcpu *vcpu)
void kvm_vcpu_wfi(struct kvm_vcpu *vcpu);
#endif /* ARM64_S390_COMMON == 1 */
void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);
@ -133,12 +136,37 @@ static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr)
static __always_inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
{
return (unsigned long *)&vcpu_gp_regs(vcpu)->pc;
return (unsigned long *)&vcpu->arch.ctxt.regs.pc;
}
static __always_inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu)
{
return (unsigned long *)&vcpu_gp_regs(vcpu)->pstate;
return (unsigned long *)&vcpu->arch.ctxt.regs.pstate;
}
static __always_inline unsigned long *vcpu_sp_el0(struct kvm_vcpu *vcpu)
{
return (unsigned long *)&vcpu->arch.ctxt.regs.sp;
}
static __always_inline u64 *vcpu_sp_el1(struct kvm_vcpu *vcpu)
{
return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1);
}
static __always_inline __u128 *vcpu_vreg(struct kvm_vcpu *vcpu, int n)
{
return &vcpu->arch.ctxt.fp_regs.vregs[n];
}
static __always_inline __u32 *vcpu_fpsr(struct kvm_vcpu *vcpu)
{
return &vcpu->arch.ctxt.fp_regs.fpsr;
}
static __always_inline __u32 *vcpu_fpcr(struct kvm_vcpu *vcpu)
{
return &vcpu->arch.ctxt.fp_regs.fpcr;
}
static __always_inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
@ -159,6 +187,7 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
*vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT;
}
#if ARM64_S390_COMMON == 1
/*
* vcpu_get_reg and vcpu_set_reg should always be passed a register number
* coming from a read of ESR_EL2. Otherwise, it may give the wrong result on
@ -167,16 +196,18 @@ static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
static __always_inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu,
u8 reg_num)
{
return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs[reg_num];
return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)[reg_num];
}
static __always_inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num,
unsigned long val)
{
if (reg_num != 31)
vcpu_gp_regs(vcpu)->regs[reg_num] = val;
vcpu_gp_regs(vcpu)[reg_num] = val;
}
#endif /* ARM64_S390_COMMON == 1 */
static inline bool vcpu_is_el2_ctxt(const struct kvm_cpu_context *ctxt)
{
switch (ctxt->regs.pstate & (PSR_MODE32_BIT | PSR_MODE_MASK)) {
@ -379,6 +410,12 @@ static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu)
return vcpu->arch.fault.disr_el1;
}
#if ARM64_S390_COMMON == 1
static __always_inline bool esr_abt_is_s1ptw(unsigned long esr)
{
return esr & ESR_ELx_S1PTW;
}
static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_get_esr(vcpu) & ESR_ELx_xVC_IMM_MASK;
@ -411,18 +448,13 @@ static __always_inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu)
{
return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_S1PTW);
return esr_abt_is_s1ptw(kvm_vcpu_get_esr(vcpu));
}
/* Always check for S1PTW *before* using this. */
static __always_inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_get_esr(vcpu) & ESR_ELx_WNR;
}
static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
{
return !!(kvm_vcpu_get_esr(vcpu) & ESR_ELx_CM);
return esr_dabt_is_write(kvm_vcpu_get_esr(vcpu));
}
static __always_inline unsigned int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
@ -443,12 +475,7 @@ static __always_inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
}
static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu)
{
return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu);
return esr_trap_is_iabt(kvm_vcpu_get_esr(vcpu));
}
static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
@ -456,6 +483,13 @@ static __always_inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
return kvm_vcpu_get_esr(vcpu) & ESR_ELx_FSC;
}
#endif /* ARM64_S390_COMMON == 1 */
static __always_inline bool esr_abt_is_exec_fault(unsigned long esr)
{
return esr_trap_is_iabt(esr) && !esr_abt_is_s1ptw(esr);
}
static inline
bool kvm_vcpu_trap_is_permission_fault(const struct kvm_vcpu *vcpu)
{
@ -468,26 +502,9 @@ bool kvm_vcpu_trap_is_translation_fault(const struct kvm_vcpu *vcpu)
return esr_fsc_is_translation_fault(kvm_vcpu_get_esr(vcpu));
}
static inline
u64 kvm_vcpu_trap_get_perm_fault_granule(const struct kvm_vcpu *vcpu)
{
unsigned long esr = kvm_vcpu_get_esr(vcpu);
BUG_ON(!esr_fsc_is_permission_fault(esr));
return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(esr & ESR_ELx_FSC_LEVEL));
}
static __always_inline bool kvm_vcpu_abt_issea(const struct kvm_vcpu *vcpu)
{
switch (kvm_vcpu_trap_get_fault(vcpu)) {
case ESR_ELx_FSC_EXTABT:
case ESR_ELx_FSC_SEA_TTW(-1) ... ESR_ELx_FSC_SEA_TTW(3):
case ESR_ELx_FSC_SECC:
case ESR_ELx_FSC_SECC_TTW(-1) ... ESR_ELx_FSC_SECC_TTW(3):
return true;
default:
return false;
}
return esr_abt_is_sea(kvm_vcpu_get_esr(vcpu));
}
static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
@ -496,9 +513,10 @@ static __always_inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
return ESR_ELx_SYS64_ISS_RT(esr);
}
static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
#if ARM64_S390_COMMON == 1
static inline bool esr_abt_is_write_fault(unsigned long esr)
{
if (kvm_vcpu_abt_iss1tw(vcpu)) {
if (esr_abt_is_s1ptw(esr)) {
/*
* Only a permission fault on a S1PTW should be
* considered as a write. Otherwise, page tables baked
@ -511,15 +529,22 @@ static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
* first), then a permission fault to allow the flags
* to be set.
*/
return kvm_vcpu_trap_is_permission_fault(vcpu);
return esr_fsc_is_permission_fault(esr);
}
if (kvm_vcpu_trap_is_iabt(vcpu))
if (esr_trap_is_iabt(esr))
return false;
return kvm_vcpu_dabt_iswrite(vcpu);
return esr_dabt_is_write(esr);
}
static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
{
return esr_abt_is_write_fault(kvm_vcpu_get_esr(vcpu));
}
#endif /* ARM64_S390_COMMON == 1 */
static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
{
return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
@ -561,6 +586,7 @@ static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
return vcpu_read_sys_reg(vcpu, r) & bit;
}
#if ARM64_S390_COMMON == 1
static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,
unsigned long data,
unsigned int len)
@ -636,6 +662,8 @@ static __always_inline void kvm_incr_pc(struct kvm_vcpu *vcpu)
vcpu_set_flag((v), e); \
} while (0)
#endif /* ARM64_S390_COMMON == 1 */
/*
* Returns a 'sanitised' view of CPTR_EL2, translating from nVHE to the VHE
* format if E2H isn't set.
@ -737,6 +765,12 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu)
}
}
static inline void kvm_reset_fpsimd(struct kvm_vcpu *vcpu)
{
memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs));
}
#if ARM64_S390_COMMON == 1
/* Reset a vcpu's core registers. */
static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu)
{
@ -750,15 +784,19 @@ static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu)
pstate = VCPU_RESET_PSTATE_EL1;
/* Reset core registers */
memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu)));
memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs));
memset(vcpu_gp_regs(vcpu), 0, sizeof(vcpu_gp_regs(vcpu)));
*vcpu_pc(vcpu) = 0;
*vcpu_sp_el0(vcpu) = 0;
kvm_reset_fpsimd(vcpu);
vcpu->arch.ctxt.spsr_abt = 0;
vcpu->arch.ctxt.spsr_und = 0;
vcpu->arch.ctxt.spsr_irq = 0;
vcpu->arch.ctxt.spsr_fiq = 0;
vcpu_gp_regs(vcpu)->pstate = pstate;
*vcpu_cpsr(vcpu) = pstate;
}
#endif /* ARM64_S390_COMMON == 1 */
/* PSCI reset handling for a vcpu. */
static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu,
struct vcpu_reset_state *reset_state)

258
arch/arm64/include/asm/kvm_hcall.h Normal file
View File

@ -0,0 +1,258 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* The host<->hyp hypercall interface.
*
* Copyright (C) 2026 Google LLC
* Author: Fuad Tabba <fuad.tabba@linux.dev>
*/
#ifndef __ARM64_KVM_HCALL_H__
#define __ARM64_KVM_HCALL_H__
#include <linux/args.h>
#include <linux/arm-smccc.h>
#include <linux/bug.h>
#include <linux/errno.h>
#include <linux/types.h>
#include <asm/barrier.h>
#include <asm/kvm_asm.h>
#include <asm/spectre.h>
#include <asm/virt.h>
typedef u16 pkvm_handle_t;
enum pkvm_topup_id {
PKVM_TOPUP_HYP_ALLOC,
NR_PKVM_TOPUP_HYP_IDS,
PKVM_TOPUP_HYP_ALLOC_SELFTEST,
};
struct kvm;
struct kvm_s2_mmu;
struct kvm_vcpu;
struct vgic_v3_cpu_if;
struct vgic_v5_cpu_if;
/*
* A host VA carried by a hypercall argument. At EL2 such a pointer must not
* be dereferenced until it is translated with kern_hyp_va_host(); sparse
* flags any use that skips the translation. The tag describes the EL2 view
* only: the host dereferences its own VAs freely.
*/
#if defined(__KVM_NVHE_HYPERVISOR__) && defined(__CHECKER__)
#define __kern __attribute__((noderef, address_space(__kern)))
#else
#define __kern
#endif
/*
* Hypercall signatures are declared as (type, name) argument pairs.
* __KVM_HCALL_MAP() applies a macro to each pair, in the mold of __MAP()
* in <linux/syscalls.h>. The ladder is indexed by list entries, two per
* argument; __KVM_HCALL_MAP_N() takes that count explicitly.
*/
#define __KVM_HCALL_MAP2(m, t, a, ...) m(t, a)
#define __KVM_HCALL_MAP4(m, t, a, ...) m(t, a), __KVM_HCALL_MAP2(m, __VA_ARGS__)
#define __KVM_HCALL_MAP6(m, t, a, ...) m(t, a), __KVM_HCALL_MAP4(m, __VA_ARGS__)
#define __KVM_HCALL_MAP8(m, t, a, ...) m(t, a), __KVM_HCALL_MAP6(m, __VA_ARGS__)
#define __KVM_HCALL_MAP10(m, t, a, ...) m(t, a), __KVM_HCALL_MAP8(m, __VA_ARGS__)
#define __KVM_HCALL_MAP12(m, t, a, ...) m(t, a), __KVM_HCALL_MAP10(m, __VA_ARGS__)
#define __KVM_HCALL_MAP_N(n, m, ...) CONCATENATE(__KVM_HCALL_MAP, n)(m, __VA_ARGS__)
#define __KVM_HCALL_MAP(m, ...) __KVM_HCALL_MAP_N(COUNT_ARGS(__VA_ARGS__), m, __VA_ARGS__)
#define __KVM_HCALL_DECL(t, a) t a
#define __KVM_HCALL_LONG(t, a) unsigned long a
#define __KVM_HCALL_CAST(t, a) (__force t) a
#define __KVM_HCALL_ARGS(t, a) a
#ifndef __KVM_NVHE_HYPERVISOR__
#define __kvm_call_hyp_nvhe(f, ...) \
({ \
struct arm_smccc_res res; \
\
arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \
##__VA_ARGS__, &res); \
if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \
res.a1 = -EOPNOTSUPP; \
\
res.a1; \
})
/*
* Generate a typed stub for each declared hypercall. kvm_call_hyp_nvhe()
* resolves to the stub, so a call with the wrong argument count or types
* fails to compile instead of being silently truncated to an SMCCC function
* number and a pile of registers. The stub inlines to the same SMCCC call
* the untyped macro used to make.
*/
#define DECLARE_KVM_HOST_HCALL(ret, name, ...) \
static __always_inline \
ret nvhe_hvc_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__)) \
{ \
return (ret)__kvm_call_hyp_nvhe(name, \
__KVM_HCALL_MAP(__KVM_HCALL_ARGS, __VA_ARGS__));\
}
#define DECLARE_KVM_HOST_HCALL0(ret, name) \
static __always_inline ret nvhe_hvc_##name(void) \
{ \
return (ret)__kvm_call_hyp_nvhe(name); \
}
#define kvm_call_hyp_nvhe(f, ...) nvhe_hvc_##f(__VA_ARGS__)
/*
* The isb() below is there to guarantee the same behaviour on VHE as on !VHE,
* where the eret to EL1 acts as a context synchronization event.
*/
#define kvm_call_hyp(f, ...) \
do { \
if (has_vhe()) { \
f(__VA_ARGS__); \
isb(); \
} else { \
kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \
} \
} while (0)
#define kvm_call_hyp_ret(f, ...) \
({ \
typeof(f(__VA_ARGS__)) ret; \
\
if (has_vhe()) { \
ret = f(__VA_ARGS__); \
} else { \
ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \
} \
\
ret; \
})
#else /* __KVM_NVHE_HYPERVISOR__ */
#define kvm_call_hyp(f, ...) f(__VA_ARGS__)
#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__)
#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__)
/*
* At EL2 each declaration emits the canonical signature of the hypercall,
* which DEFINE_KVM_HOST_HCALL() in hyp-main.c checks the handler
* definition against.
*/
#define DECLARE_KVM_HOST_HCALL(ret, name, ...) \
typedef ret kvm_host_hcall_sig_##name(__KVM_HCALL_MAP(__KVM_HCALL_DECL, __VA_ARGS__));
#define DECLARE_KVM_HOST_HCALL0(ret, name) \
typedef ret kvm_host_hcall_sig_##name(void);
#endif /* __KVM_NVHE_HYPERVISOR__ */
/* Hypercalls that are unavailable once pKVM has finalised. */
DECLARE_KVM_HOST_HCALL(int, __pkvm_init,
phys_addr_t, phys, unsigned long, size,
unsigned long *, per_cpu_base, u32, hyp_va_bits)
DECLARE_KVM_HOST_HCALL(ulong, __pkvm_create_private_mapping,
phys_addr_t, phys, size_t, size, u64, prot)
DECLARE_KVM_HOST_HCALL(int, __pkvm_cpu_set_vector,
enum arm64_hyp_spectre_vector, slot)
DECLARE_KVM_HOST_HCALL0(void, __kvm_enable_ssbs)
DECLARE_KVM_HOST_HCALL0(void, __vgic_v3_init_lrs)
DECLARE_KVM_HOST_HCALL0(u64, __vgic_v3_get_gic_config)
DECLARE_KVM_HOST_HCALL0(int, __pkvm_hyp_alloc_selftest)
DECLARE_KVM_HOST_HCALL0(int, __pkvm_prot_finalize)
/* Hypercalls that are always available and common to [nh]VHE/pKVM. */
DECLARE_KVM_HOST_HCALL(void, __kvm_adjust_pc,
struct kvm_vcpu __kern *, vcpu)
DECLARE_KVM_HOST_HCALL(int, __kvm_vcpu_run,
struct kvm_vcpu __kern *, vcpu)
DECLARE_KVM_HOST_HCALL0(void, __kvm_flush_vm_context)
DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa,
struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level)
DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_ipa_nsh,
struct kvm_s2_mmu __kern *, mmu, phys_addr_t, ipa, int, level)
DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid,
struct kvm_s2_mmu __kern *, mmu)
DECLARE_KVM_HOST_HCALL(void, __kvm_tlb_flush_vmid_range,
struct kvm_s2_mmu __kern *, mmu, phys_addr_t, start, unsigned long, pages)
DECLARE_KVM_HOST_HCALL(void, __kvm_flush_cpu_context,
struct kvm_s2_mmu __kern *, mmu)
DECLARE_KVM_HOST_HCALL(void, __kvm_timer_set_cntvoff,
u64, cntvoff)
DECLARE_KVM_HOST_HCALL(int, __tracing_load,
void __kern *, desc_hva, size_t, desc_size)
DECLARE_KVM_HOST_HCALL0(void, __tracing_unload)
DECLARE_KVM_HOST_HCALL(int, __tracing_enable,
bool, enable)
DECLARE_KVM_HOST_HCALL(int, __tracing_swap_reader,
unsigned int, cpu)
DECLARE_KVM_HOST_HCALL(void, __tracing_update_clock,
u32, mult, u32, shift, u64, epoch_ns, u64, epoch_cyc)
DECLARE_KVM_HOST_HCALL(int, __tracing_reset,
unsigned int, cpu)
DECLARE_KVM_HOST_HCALL(int, __tracing_enable_event,
unsigned short, id, bool, enable)
DECLARE_KVM_HOST_HCALL(void, __tracing_write_event,
u64, id)
DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_resident,
struct vgic_v5_cpu_if __kern *, cpu_if);
DECLARE_KVM_HOST_HCALL(void, __vgic_v5_make_non_resident,
struct vgic_v5_cpu_if __kern *, cpu_if);
DECLARE_KVM_HOST_HCALL(void, __vgic_v3_save_aprs,
struct vgic_v3_cpu_if __kern *, cpu_if)
DECLARE_KVM_HOST_HCALL(void, __vgic_v3_restore_vmcr_aprs,
struct vgic_v3_cpu_if __kern *, cpu_if)
DECLARE_KVM_HOST_HCALL(void, __vgic_v5_save_apr,
struct vgic_v5_cpu_if __kern *, cpu_if)
DECLARE_KVM_HOST_HCALL(void, __vgic_v5_restore_vmcr_apr,
struct vgic_v5_cpu_if __kern *, cpu_if)
DECLARE_KVM_HOST_HCALL(void, __vgic_v5_vdpend,
u32, intid, bool, pending, u16, vm);
DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_topup,
enum pkvm_topup_id, id, phys_addr_t, head, unsigned long, nr_pages)
DECLARE_KVM_HOST_HCALL(int, __pkvm_hyp_reclaim,
enum pkvm_topup_id, id, unsigned long, target)
DECLARE_KVM_HOST_HCALL(ulong, __pkvm_hyp_reclaimable,
enum pkvm_topup_id, id)
/* Hypercalls that are available only when pKVM has finalised. */
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_hyp,
u64, pfn)
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_hyp,
u64, pfn)
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_donate_guest,
u64, pfn, u64, gfn)
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_share_guest,
u64, pfn, u64, gfn, u64, nr_pages, u64, prot)
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_unshare_guest,
pkvm_handle_t, handle, u64, gfn, u64, nr_pages)
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_relax_perms_guest,
u64, gfn, u64, prot)
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_wrprotect_guest,
pkvm_handle_t, handle, u64, gfn, u64, nr_pages)
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_test_clear_young_guest,
pkvm_handle_t, handle, u64, gfn, u64, nr_pages, bool, mkold)
DECLARE_KVM_HOST_HCALL(int, __pkvm_host_mkyoung_guest,
u64, gfn)
DECLARE_KVM_HOST_HCALL0(int, __pkvm_reserve_vm)
DECLARE_KVM_HOST_HCALL(void, __pkvm_unreserve_vm,
pkvm_handle_t, handle)
DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vm,
struct kvm __kern *, host_kvm, void __kern *, pgd_hva)
DECLARE_KVM_HOST_HCALL(int, __pkvm_init_vcpu,
pkvm_handle_t, handle, struct kvm_vcpu __kern *, host_vcpu)
DECLARE_KVM_HOST_HCALL0(int, __pkvm_vcpu_in_poison_fault)
DECLARE_KVM_HOST_HCALL(int, __pkvm_force_reclaim_guest_page,
phys_addr_t, phys)
DECLARE_KVM_HOST_HCALL(int, __pkvm_reclaim_dying_guest_page,
pkvm_handle_t, handle, u64, gfn)
DECLARE_KVM_HOST_HCALL(int, __pkvm_start_teardown_vm,
pkvm_handle_t, handle)
DECLARE_KVM_HOST_HCALL(int, __pkvm_finalize_teardown_vm,
pkvm_handle_t, handle)
DECLARE_KVM_HOST_HCALL(void, __pkvm_vcpu_load,
pkvm_handle_t, handle, unsigned int, vcpu_idx, u64, hcr_el2)
DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_put)
DECLARE_KVM_HOST_HCALL0(void, __pkvm_vcpu_sync_state)
DECLARE_KVM_HOST_HCALL(void, __pkvm_tlb_flush_vmid,
pkvm_handle_t, handle)
#endif /* __ARM64_KVM_HCALL_H__ */

298
arch/arm64/include/asm/kvm_host.h
View File

@ -14,6 +14,7 @@
#include <linux/arm-smccc.h>
#include <linux/bitmap.h>
#include <linux/types.h>
#include <linux/interval_tree.h>
#include <linux/jump_label.h>
#include <linux/kvm_types.h>
#include <linux/maple_tree.h>
@ -27,6 +28,7 @@
#include <asm/fpsimd.h>
#include <asm/kvm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_hcall.h>
#include <asm/vncr_mapping.h>
#define __KVM_HAVE_ARCH_INTC_INITIALIZED
@ -42,6 +44,7 @@
#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
#if ARM64_S390_COMMON == 1
#define KVM_VCPU_MAX_FEATURES 10
#define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1)
@ -58,6 +61,51 @@
#define KVM_REQ_GUEST_HYP_IRQ_PENDING KVM_ARCH_REQ(9)
#define KVM_REQ_MAP_L1_VNCR_EL2 KVM_ARCH_REQ(10)
#define KVM_REQ_VGIC_PROCESS_UPDATE KVM_ARCH_REQ(11)
#define KVM_REQ_RELOAD_GICv5 KVM_ARCH_REQ(12)
/* Bit indices for kvm_arch::flags */
enum {
/*
* If we encounter a data abort without valid instruction syndrome
* information, report this to user space. User space can (and
* should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is
* supported.
*/
KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
/* Memory Tagging Extension enabled for the guest */
KVM_ARCH_FLAG_MTE_ENABLED,
/* At least one vCPU has ran in the VM */
KVM_ARCH_FLAG_HAS_RAN_ONCE,
/* The vCPU feature set for the VM is configured */
KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED,
/* PSCI SYSTEM_SUSPEND enabled for the guest */
KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED,
/* VM counter offset */
KVM_ARCH_FLAG_VM_COUNTER_OFFSET,
/* Timer PPIs made immutable */
KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE,
/* Initial ID reg values loaded */
KVM_ARCH_FLAG_ID_REGS_INITIALIZED,
/* Fine-Grained UNDEF initialised */
KVM_ARCH_FLAG_FGU_INITIALIZED,
/* SVE exposed to guest */
KVM_ARCH_FLAG_GUEST_HAS_SVE,
/* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */
KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS,
/* Unhandled SEAs are taken to userspace */
KVM_ARCH_FLAG_EXIT_SEA,
/* No further ID register changes possible */
KVM_ARCH_FLAG_ID_REGS_FINAL,
};
struct vcpu_reset_state {
unsigned long pc;
unsigned long r0;
bool be;
bool reset;
};
#endif /* ARM64_S390_COMMON == 1 */
#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
KVM_DIRTY_LOG_INITIALLY_SET)
@ -94,9 +142,22 @@ struct kvm_hyp_memcache {
struct pkvm_mapping *mapping; /* only used from EL1 */
#define HYP_MEMCACHE_ACCOUNT_STAGE2 BIT(1)
#define HYP_MEMCACHE_ACCOUNT_KMEMCG BIT(2)
unsigned long flags;
};
static inline void init_hyp_memcache(struct kvm_hyp_memcache *mc)
{
memset(mc, 0, sizeof(*mc));
mc->mapping = ZERO_SIZE_PTR; /* Prevent allocation, solely useful for stage2 memcache */
}
static inline void init_hyp_stage2_memcache(struct kvm_hyp_memcache *mc)
{
memset(mc, 0, sizeof(*mc));
mc->flags = HYP_MEMCACHE_ACCOUNT_STAGE2 | HYP_MEMCACHE_ACCOUNT_KMEMCG;
}
static inline void push_hyp_memcache(struct kvm_hyp_memcache *mc,
phys_addr_t *p,
phys_addr_t (*to_pa)(void *virt))
@ -153,6 +214,36 @@ struct kvm_vmid {
atomic64_t id;
};
/*
* Record of a guest stage-2 mapping, storing canonical and nested IPA
* ranges. Both ranges have the same size. The lower bits of nested.start
* store the index of the nested mmu this mapping belongs to.
*/
struct kvm_guest_s2_mapping {
struct interval_tree_node canonical;
struct interval_tree_node nested;
};
struct kvm_vcpu_ops {
void (*vcpu_load)(struct kvm_vcpu *vcpu);
void (*vcpu_put)(struct kvm_vcpu *vcpu);
};
struct kvm_gfn_range;
struct kvm_s2_fault_desc;
struct kvm_vm_s2_ops {
bool (*vm_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range);
bool (*vm_test_age_gfn)(struct kvm *kvm, struct kvm_gfn_range *range);
int (*vm_flush_remote_tlbs)(struct kvm *kvm);
int (*vm_flush_remote_tlbs_range)(struct kvm *kvm, gfn_t gfn,
u64 nr_pages);
void (*vm_stage2_unmap_range)(struct kvm_s2_mmu *mmu,
phys_addr_t start, u64 size,
bool may_block);
int (*vm_mem_abort)(const struct kvm_s2_fault_desc *s2fd);
};
struct kvm_s2_mmu {
struct kvm_vmid vmid;
@ -214,22 +305,24 @@ struct kvm_s2_mmu {
u64 tlb_vttbr;
u64 tlb_vtcr;
/* Guest s2 mapping records indexed in this MMU's IPA space. */
struct rb_root_cached guest_s2_mappings;
/*
* true when this represents a nested context where virtual
* HCR_EL2.VM == 1
*/
bool nested_stage2_enabled;
#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
struct dentry *shadow_pt_debugfs_dentry;
#endif
/*
* true when this MMU needs to be unmapped before being used for a new
* purpose.
*/
bool pending_unmap;
/* Index in the S2 MMU array, only valid for a shadow S2 */
u16 s2_mmu_idx;
/*
* 0: Nobody is currently using this, check vttbr for validity
* >0: Somebody is actively using this.
@ -254,13 +347,9 @@ struct kvm_smccc_features {
unsigned long vendor_hyp_bmap_2; /* Function numbers 64-127 */
};
typedef u16 pkvm_handle_t;
struct kvm_protected_vm {
pkvm_handle_t handle;
struct kvm_hyp_memcache teardown_mc;
struct kvm_hyp_memcache stage2_teardown_mc;
bool is_protected;
bool is_created;
/*
@ -309,8 +398,24 @@ enum fgt_group_id {
__NR_FGT_GROUP_IDS__
};
enum kvm_arm_vm_flavor {
VM_NVHE,
VM_VHE,
/* VMs running on a hyp that doesn't trust */
MARKER(__VM_DISTRUSTING_HYP),
VM_PKVM, /* Normal guests on pKVM */
MARKER(__VM_PROTECTED),
VM_PROTECTED_PKVM, /* Protected VM */
VM_FLAVOR_MAX
};
struct kvm_arch {
struct kvm_s2_mmu mmu;
const struct kvm_vm_s2_ops *vm_s2_ops;
enum kvm_arm_vm_flavor vm_flavor;
/* Mandated version of PSCI */
u32 psci_version;
/*
* Fine-Grained UNDEF, mimicking the FGT layout defined by the
@ -329,47 +434,21 @@ struct kvm_arch {
size_t nested_mmus_size;
int nested_mmus_next;
/*
* Serializes guest s2 tracking trees access when the mmu_lock
* is only held for read.
*/
spinlock_t guest_s2_tracking_lock;
/* Interrupt controller */
struct vgic_dist vgic;
/* Timers */
struct arch_timer_vm_data timer_data;
/* Mandated version of PSCI */
u32 psci_version;
/* Protects VM-scoped configuration data */
struct mutex config_lock;
/*
* If we encounter a data abort without valid instruction syndrome
* information, report this to user space. User space can (and
* should) opt in to this feature if KVM_CAP_ARM_NISV_TO_USER is
* supported.
*/
#define KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER 0
/* Memory Tagging Extension enabled for the guest */
#define KVM_ARCH_FLAG_MTE_ENABLED 1
/* At least one vCPU has ran in the VM */
#define KVM_ARCH_FLAG_HAS_RAN_ONCE 2
/* The vCPU feature set for the VM is configured */
#define KVM_ARCH_FLAG_VCPU_FEATURES_CONFIGURED 3
/* PSCI SYSTEM_SUSPEND enabled for the guest */
#define KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED 4
/* VM counter offset */
#define KVM_ARCH_FLAG_VM_COUNTER_OFFSET 5
/* Timer PPIs made immutable */
#define KVM_ARCH_FLAG_TIMER_PPIS_IMMUTABLE 6
/* Initial ID reg values loaded */
#define KVM_ARCH_FLAG_ID_REGS_INITIALIZED 7
/* Fine-Grained UNDEF initialised */
#define KVM_ARCH_FLAG_FGU_INITIALIZED 8
/* SVE exposed to guest */
#define KVM_ARCH_FLAG_GUEST_HAS_SVE 9
/* MIDR_EL1, REVIDR_EL1, and AIDR_EL1 are writable from userspace */
#define KVM_ARCH_FLAG_WRITABLE_IMP_ID_REGS 10
/* Unhandled SEAs are taken to userspace */
#define KVM_ARCH_FLAG_EXIT_SEA 11
unsigned long flags;
/* VM-wide vCPU feature set */
@ -838,17 +917,11 @@ extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS];
#define hyp_cpu_logical_map CHOOSE_NVHE_SYM(hyp_cpu_logical_map)
struct vcpu_reset_state {
unsigned long pc;
unsigned long r0;
bool be;
bool reset;
};
struct vncr_tlb;
struct kvm_vcpu_arch {
struct kvm_cpu_context ctxt;
const struct kvm_vcpu_ops *vcpu_ops;
/*
* Guest floating point state
@ -929,8 +1002,8 @@ struct kvm_vcpu_arch {
/* Cache some mmu pages needed inside spinlock regions */
struct kvm_mmu_memory_cache mmu_page_cache;
/* Pages to top-up the pKVM/EL2 guest pool */
struct kvm_hyp_memcache pkvm_memcache;
/* Pages to top-up the pKVM/EL2 guest stage-2 pool */
struct kvm_hyp_memcache stage2_mc;
/* Virtual SError ESR to restore when HCR_EL2.VSE is set */
u64 vsesr_el2;
@ -954,6 +1027,7 @@ struct kvm_vcpu_arch {
pid_t pid;
};
#if ARM64_S390_COMMON == 1
/*
* Each 'flag' is composed of a comma-separated triplet:
*
@ -1036,11 +1110,28 @@ struct kvm_vcpu_arch {
set; \
})
#define __vcpu_copy_flag(vt, vs, flagset, f, m) \
do { \
typeof(vs->arch.flagset) tmp, val; \
\
__build_check_flag(vs, flagset, f, m); \
\
val = READ_ONCE(vs->arch.flagset); \
val &= (m); \
__vcpu_flags_preempt_disable(); \
tmp = READ_ONCE(vt->arch.flagset); \
tmp &= ~(m); \
tmp |= val; \
WRITE_ONCE(vt->arch.flagset, tmp); \
__vcpu_flags_preempt_enable(); \
} while (0)
#define vcpu_get_flag(v, ...) __vcpu_get_flag((v), __VA_ARGS__)
#define vcpu_set_flag(v, ...) __vcpu_set_flag((v), __VA_ARGS__)
#define vcpu_clear_flag(v, ...) __vcpu_clear_flag((v), __VA_ARGS__)
#define vcpu_test_and_clear_flag(v, ...) \
__vcpu_test_and_clear_flag((v), __VA_ARGS__)
#define vcpu_copy_flag(vt, vs, ...) __vcpu_copy_flag((vt), (vs), __VA_ARGS__)
/* KVM_ARM_VCPU_INIT completed */
#define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0))
@ -1049,6 +1140,11 @@ struct kvm_vcpu_arch {
/* pKVM VCPU setup completed */
#define VCPU_PKVM_FINALIZED __vcpu_single_flag(cflags, BIT(2))
int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature);
bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu);
#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED)
/* Exception pending */
#define PENDING_EXCEPTION __vcpu_single_flag(iflags, BIT(0))
/*
@ -1058,6 +1154,8 @@ struct kvm_vcpu_arch {
#define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1))
/* Target EL/MODE (not a single flag, but let's abuse the macro) */
#define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1))
/* Cover both PENDING_EXCEPTION and EXCEPT_MASK for global operations */
#define PC_UPDATE_REQ __vcpu_single_flag(iflags, GENMASK(3, 0))
/* Host-set: the hyp flushes the non-protected vCPU state in on entry */
#define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(4))
@ -1107,6 +1205,8 @@ struct kvm_vcpu_arch {
/* KVM is currently emulating an L2 to L1 exception */
#define IN_NESTED_EXCEPTION __vcpu_single_flag(sflags, BIT(9))
#endif /* ARM64_S390_COMMON == 1 */
#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl)
#define vcpu_sve_zcr_elx(vcpu) \
@ -1152,6 +1252,9 @@ struct kvm_vcpu_arch {
#define vcpu_has_ptrauth(vcpu) false
#endif
#define kvm_id_regs_final(kvm) \
test_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &(kvm)->arch.flags)
#define vcpu_on_unsupported_cpu(vcpu) \
vcpu_get_flag(vcpu, ON_UNSUPPORTED_CPU)
@ -1161,7 +1264,7 @@ struct kvm_vcpu_arch {
#define vcpu_clear_on_unsupported_cpu(vcpu) \
vcpu_clear_flag(vcpu, ON_UNSUPPORTED_CPU)
#define vcpu_gp_regs(v) (&(v)->arch.ctxt.regs)
#define vcpu_gp_regs(v) ((v)->arch.ctxt.regs.regs)
/*
* Only use __vcpu_sys_reg/ctxt_sys_reg if you know you want the
@ -1242,14 +1345,6 @@ struct kvm_vcpu_stat {
u64 exits;
};
unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu);
int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices);
int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
struct kvm_vcpu_events *events);
@ -1261,51 +1356,6 @@ void kvm_arm_resume_guest(struct kvm *kvm);
#define vcpu_has_run_once(vcpu) (!!READ_ONCE((vcpu)->pid))
#ifndef __KVM_NVHE_HYPERVISOR__
#define kvm_call_hyp_nvhe(f, ...) \
({ \
struct arm_smccc_res res; \
\
arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), \
##__VA_ARGS__, &res); \
if (WARN_ON(res.a0 != SMCCC_RET_SUCCESS)) \
res.a1 = -EOPNOTSUPP; \
\
res.a1; \
})
/*
* The isb() below is there to guarantee the same behaviour on VHE as on !VHE,
* where the eret to EL1 acts as a context synchronization event.
*/
#define kvm_call_hyp(f, ...) \
do { \
if (has_vhe()) { \
f(__VA_ARGS__); \
isb(); \
} else { \
kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \
} \
} while(0)
#define kvm_call_hyp_ret(f, ...) \
({ \
typeof(f(__VA_ARGS__)) ret; \
\
if (has_vhe()) { \
ret = f(__VA_ARGS__); \
} else { \
ret = kvm_call_hyp_nvhe(f, ##__VA_ARGS__); \
} \
\
ret; \
})
#else /* __KVM_NVHE_HYPERVISOR__ */
#define kvm_call_hyp(f, ...) f(__VA_ARGS__)
#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__)
#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__)
#endif /* __KVM_NVHE_HYPERVISOR__ */
int handle_exit(struct kvm_vcpu *vcpu, int exception_index);
void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index);
@ -1328,6 +1378,15 @@ int __init populate_nv_trap_config(void);
void kvm_calculate_traps(struct kvm_vcpu *vcpu);
#if ARM64_S390_COMMON == 1
unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg);
unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu);
int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices);
/* MMIO helpers */
void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data);
unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len);
@ -1335,6 +1394,8 @@ unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len);
int kvm_handle_mmio_return(struct kvm_vcpu *vcpu);
int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa);
#endif /* ARM64_S390_COMMON == 1 */
/*
* Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event,
* arrived in guest context. For arm64, any event that arrives while a vCPU is
@ -1349,7 +1410,7 @@ long kvm_hypercall_pv_features(struct kvm_vcpu *vcpu);
gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu);
void kvm_update_stolen_time(struct kvm_vcpu *vcpu);
bool kvm_arm_pvtime_supported(void);
bool kvm_arm_pvtime_supported(struct kvm *kvm);
int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr);
int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu,
@ -1507,14 +1568,30 @@ struct kvm *kvm_arch_alloc_vm(void);
#define __KVM_HAVE_ARCH_FLUSH_REMOTE_TLBS_RANGE
#define kvm_vm_is_protected(kvm) (is_protected_kvm_enabled() && (kvm)->arch.pkvm.is_protected)
#ifdef __KVM_NVHE_HYPERVISOR__
#define kvm_vm_is_protected(kvm) \
(is_protected_kvm_enabled() && ((kvm)->arch.vm_flavor == VM_PROTECTED_PKVM))
/*
* Accessing vcpu->kvm from nVHE hyp stub is tricky, as we need to convert the
* pointer to the hyp VA. With pKVM, the nVHE code runs with the hyp_vcpu,
* which is populated correctly and is gated on is_protected_kvm_enabled().
*/
#define vcpu_is_protected(vcpu) \
({ \
struct kvm *__kvm = READ_ONCE((vcpu)->kvm); \
\
(__kvm && kvm_vm_is_protected(__kvm)); \
})
#else
#define kvm_vm_is_protected(kvm) ((kvm)->arch.vm_flavor >= __VM_PROTECTED)
#define vcpu_is_protected(vcpu) kvm_vm_is_protected((vcpu)->kvm)
int kvm_arm_vcpu_finalize(struct kvm_vcpu *vcpu, int feature);
bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu);
#define kvm_vm_hyp_is_distrusting(kvm) ((kvm)->arch.vm_flavor >= __VM_DISTRUSTING_HYP)
#define kvm_arm_vcpu_sve_finalized(vcpu) vcpu_get_flag(vcpu, VCPU_SVE_FINALIZED)
#endif /* __KVM_NVHE_HYPERVISOR__ */
#define kvm_has_mte(kvm) \
(system_supports_mte() && \
@ -1532,11 +1609,15 @@ static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature)
return test_bit(feature, ka->vcpu_features);
}
#if ARM64_S390_COMMON == 1
#define kvm_vcpu_has_feature(k, f) __vcpu_has_feature(&(k)->arch, (f))
#define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f))
#define kvm_vcpu_initialized(v) vcpu_get_flag(v, VCPU_INITIALIZED)
#endif /* ARM64_S390_COMMON == 1 */
int kvm_trng_call(struct kvm_vcpu *vcpu);
#ifdef CONFIG_KVM
extern phys_addr_t hyp_mem_base;
@ -1548,6 +1629,7 @@ static inline void kvm_hyp_reserve(void) { }
void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu);
bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu);
bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu);
static inline u64 *__vm_id_reg(struct kvm_arch *ka, u32 reg)
{

7
arch/arm64/include/asm/kvm_hyp.h
View File

@ -10,6 +10,7 @@
#include <linux/compiler.h>
#include <linux/kvm_host.h>
#include <asm/alternative.h>
#include <asm/spectre.h>
#include <asm/sysreg.h>
DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
@ -88,6 +89,8 @@ void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if);
int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu);
/* GICv5 */
void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if);
void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if);
void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if);
void __vgic_v5_restore_vmcr_apr(struct vgic_v5_cpu_if *cpu_if);
/* No hypercalls for the following */
@ -95,6 +98,7 @@ void __vgic_v5_save_ppi_state(struct vgic_v5_cpu_if *cpu_if);
void __vgic_v5_restore_ppi_state(struct vgic_v5_cpu_if *cpu_if);
void __vgic_v5_save_state(struct vgic_v5_cpu_if *cpu_if);
void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if);
void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm);
#ifdef __KVM_NVHE_HYPERVISOR__
void __timer_enable_traps(struct kvm_vcpu *vcpu);
@ -150,6 +154,9 @@ extern u64 kvm_nvhe_sym(id_aa64smfr0_el1_sys_val);
extern unsigned long kvm_nvhe_sym(__icache_flags);
extern unsigned int kvm_nvhe_sym(kvm_arm_vmid_bits);
extern enum mitigation_state kvm_nvhe_sym(spectre_v2_state);
extern enum mitigation_state kvm_nvhe_sym(spectre_v4_state);
extern enum mitigation_state kvm_nvhe_sym(spectre_bhb_state);
extern unsigned int kvm_nvhe_sym(kvm_host_sve_max_vl);
extern unsigned long kvm_nvhe_sym(hyp_nr_cpus);
extern unsigned int kvm_nvhe_sym(hyp_gicv3_nr_lr);

14
arch/arm64/include/asm/kvm_hypevents.h
View File

@ -58,4 +58,18 @@ HYP_EVENT(selftest,
),
RE_PRINTK("id=%llu", __entry->id)
);
/*
* trace_hyp_printk() has too many specificities to be declared with HYP_EVENT().
* However, we can use a REMOTE_EVENT macro to automatically do the declaration
* for the kernel side.
*/
REMOTE_EVENT_CUSTOM_PRINTK(hyp_printk,
0, /* id will be overwritten during hyp event init */
RE_STRUCT(
re_field(u16, fmt_id)
re_field(u64, args[])
),
__hyp_trace_printk(evt, seq)
);
#endif

9
arch/arm64/include/asm/kvm_hyptrace.h
View File

@ -23,4 +23,13 @@ extern struct remote_event __hyp_events_end[];
extern struct hyp_event_id __hyp_event_ids_start[];
extern struct hyp_event_id __hyp_event_ids_end[];
#define HYP_STRING_FMT_MAX_SIZE 127
struct hyp_string_fmt {
const char fmt[HYP_STRING_FMT_MAX_SIZE];
u8 nr_args;
};
extern const struct hyp_string_fmt __hyp_string_fmts_start[];
extern const struct hyp_string_fmt __hyp_string_fmts_end[];
#endif

19
arch/arm64/include/asm/kvm_mmu.h
View File

@ -7,6 +7,8 @@
#ifndef __ARM64_KVM_MMU_H__
#define __ARM64_KVM_MMU_H__
#include <linux/compiler.h>
#include <asm/page.h>
#include <asm/memory.h>
#include <asm/mmu.h>
@ -126,6 +128,9 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v)
* replace the instructions with `nop`s.
*/
#ifndef __KVM_VHE_HYPERVISOR__
if (is_protected_kvm_enabled() && !is_ttbr1_addr(v))
return v;
asm volatile(ALTERNATIVE_CB("and %0, %0, #1\n" /* mask with va_mask */
"ror %0, %0, #1\n" /* rotate to the first tag bit */
"add %0, %0, #0\n" /* insert the low 12 bits of the tag */
@ -140,6 +145,14 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v)
#define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v))))
/*
* Translate a __kern-tagged host VA, dropping the tag: the only sanctioned
* unwrap. Translation only, no ownership or bounds validation; the result
* carries the pointee type stripped of the tag and of any cv-qualifiers.
*/
#define kern_hyp_va_host(v) \
((TYPEOF_UNQUAL(*(v)) *)__kern_hyp_va((unsigned long)(__force void *)(v)))
extern u32 __hyp_va_bits;
/*
@ -392,12 +405,10 @@ static inline bool kvm_supports_cacheable_pfnmap(void)
#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
void kvm_s2_ptdump_create_debugfs(struct kvm *kvm);
void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu);
void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu);
void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx);
#else
static inline void kvm_s2_ptdump_create_debugfs(struct kvm *kvm) {}
static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu) {}
static inline void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu) {}
static inline void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx) {}
#endif /* CONFIG_PTDUMP_STAGE2_DEBUGFS */
#endif /* __ASSEMBLER__ */

7
arch/arm64/include/asm/kvm_nested.h
View File

@ -78,6 +78,11 @@ extern void kvm_s2_mmu_iterate_by_vmid(struct kvm *kvm, u16 vmid,
const union tlbi_info *info,
void (*)(struct kvm_s2_mmu *,
const union tlbi_info *));
extern void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa,
gpa_t nested_ipa, size_t map_size,
struct kvm_guest_s2_mapping *mapping);
extern void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu,
gpa_t nipa, size_t size);
extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
@ -166,6 +171,8 @@ extern int kvm_s2_handle_perm_fault(struct kvm_vcpu *vcpu,
struct kvm_s2_trans *trans);
extern int kvm_inject_s2_fault(struct kvm_vcpu *vcpu, u64 esr_el2);
extern void kvm_nested_s2_wp(struct kvm *kvm);
extern void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa,
size_t unmap_size, bool may_block);
extern void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block);
extern void kvm_nested_s2_flush(struct kvm *kvm);

17
arch/arm64/include/asm/kvm_pgtable.h
View File

@ -7,10 +7,18 @@
#ifndef __ARM64_KVM_PGTABLE_H__
#define __ARM64_KVM_PGTABLE_H__
#include <linux/bitfield.h>
#include <linux/bits.h>
#include <linux/kvm_host.h>
#include <linux/kvm_types.h>
#include <linux/rbtree_types.h>
#include <linux/rcupdate.h>
#include <linux/types.h>
#include <asm/memory.h>
#include <asm/page.h>
struct kvm_s2_mmu;
#define KVM_PGTABLE_FIRST_LEVEL -1
#define KVM_PGTABLE_LAST_LEVEL 3
@ -532,7 +540,7 @@ u64 kvm_pgtable_hyp_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
* kvm_get_vtcr() - Helper to construct VTCR_EL2
* @mmfr0: Sanitized value of SYS_ID_AA64MMFR0_EL1 register.
* @mmfr1: Sanitized value of SYS_ID_AA64MMFR1_EL1 register.
* @phys_shfit: Value to set in VTCR_EL2.T0SZ.
* @phys_shift: IPA size, in bits.
*
* The VTCR value is common across all the physical CPUs on the system.
* We use system wide sanitised values to fill in different fields,
@ -859,6 +867,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
* @addr: Input address for the start of the walk.
* @ptep: Pointer to storage for the retrieved PTE.
* @level: Pointer to storage for the level of the retrieved PTE.
* @flags: Flags to control the page-table walk
* (see struct kvm_pgtable_visit_ctx).
*
* The offset of @addr within a page is ignored.
*
@ -869,7 +879,8 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
* Return: 0 on success, negative error code on failure.
*/
int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
kvm_pte_t *ptep, s8 *level);
kvm_pte_t *ptep, s8 *level,
enum kvm_pgtable_walk_flags flags);
/**
* kvm_pgtable_stage2_pte_prot() - Retrieve the protection attributes of a

152
arch/arm64/include/asm/kvm_pkvm.h
View File

@ -7,9 +7,9 @@
#define __ARM64_KVM_PKVM_H__
#include <linux/arm_ffa.h>
#include <linux/kvm_host.h>
#include <linux/memblock.h>
#include <linux/scatterlist.h>
#include <asm/kvm_host.h>
#include <asm/kvm_pgtable.h>
/* Maximum number of VMs that can co-exist under pKVM. */
@ -17,7 +17,8 @@
#define HYP_MEMBLOCK_REGIONS 128
int pkvm_init_host_vm(struct kvm *kvm, unsigned long type);
void pkvm_selftests(void);
int pkvm_init_host_vm(struct kvm *kvm);
int pkvm_create_hyp_vm(struct kvm *kvm);
bool pkvm_hyp_vm_is_created(struct kvm *kvm);
void pkvm_destroy_hyp_vm(struct kvm *kvm);
@ -42,17 +43,50 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext)
case KVM_CAP_ARM_VM_IPA_SIZE:
case KVM_CAP_ARM_PTRAUTH_ADDRESS:
case KVM_CAP_ARM_PTRAUTH_GENERIC:
case KVM_CAP_ONE_REG:
case KVM_CAP_MP_STATE:
case KVM_CAP_VCPU_EVENTS:
case KVM_CAP_VCPU_ATTRIBUTES:
case KVM_CAP_IMMEDIATE_EXIT:
case KVM_CAP_IOEVENTFD:
case KVM_CAP_IRQFD_RESAMPLE:
case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
case KVM_CAP_ARM_INJECT_SERROR_ESR:
return true;
case KVM_CAP_ARM_MTE:
return false;
case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
case KVM_CAP_PRE_FAULT_MEMORY:
return false;
default:
return !kvm || !kvm_vm_is_protected(kvm);
return !kvm || (kvm->arch.vm_flavor == VM_PKVM);
}
}
/*
* The vCPU features a protected VM may use: checked by the host at
* KVM_ARM_VCPU_INIT, applied by EL2 when the hyp VM is created.
*/
static inline void kvm_pkvm_vcpu_allowed_features(struct kvm *kvm,
unsigned long *allowed)
{
bitmap_zero(allowed, KVM_VCPU_MAX_FEATURES);
set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed);
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3))
set_bit(KVM_ARM_VCPU_PMU_V3, allowed);
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS))
set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed);
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC))
set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed);
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE))
set_bit(KVM_ARM_VCPU_SVE, allowed);
}
/*
* Check whether the KVM VM IOCTL is allowed in pKVM.
*
@ -204,6 +238,116 @@ struct pkvm_mapping {
u64 __subtree_last; /* Internal member for interval tree */
};
enum pkvm_hyp_req_type {
PKVM_HYP_NO_REQ = 0,
PKVM_HYP_REQ_HYP_ALLOC,
PKVM_HYP_REQ_HYP_ALLOC_SELFTEST,
__PKVM_HYP_REQ_TYPE_MAX,
};
#define PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX \
(sizeof(struct arm_smccc_res) - offsetof(struct arm_smccc_res, a2) - 1)
struct pkvm_hyp_req {
u8 type;
union {
struct {
u32 nr_pages;
} mem;
struct {
/* Helper for SMCCC encoding/decoding */
u8 args[PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX];
} args;
};
};
static inline size_t pkvm_hyp_req_arg_size(u8 type)
{
struct pkvm_hyp_req *req;
switch (type) {
case PKVM_HYP_NO_REQ:
return 0;
case PKVM_HYP_REQ_HYP_ALLOC:
case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST:
return sizeof(req->mem);
default:
WARN_ON(1);
}
return 0;
}
/* Encode the pending pkvm_hyp_req type into the SMCCC args */
static inline void
pkvm_hyp_req_to_smccc(struct kvm_cpu_context *host_ctxt, struct pkvm_hyp_req *req)
{
u8 *dst, type = req->type;
size_t size;
if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) {
host_ctxt->regs.regs[2] = 0;
return;
}
size = pkvm_hyp_req_arg_size(type);
if (WARN_ON(size > PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX))
return;
dst = (u8 *)&host_ctxt->regs.regs[2];
*dst = type;
memcpy(dst + 1, &req->args, size);
}
/* Return true if a pkvm_hyp_req has been decoded from the SMCCC args */
static inline bool smccc_to_pkvm_hyp_req(struct pkvm_hyp_req *req, struct arm_smccc_res *res)
{
u8 *src = (u8 *)res + offsetof(struct arm_smccc_res, a2);
u8 type = *src;
if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX)
return false;
req->type = type;
memcpy(&req->args, src + 1, pkvm_hyp_req_arg_size(type));
return true;
}
int __pkvm_handle_smccc_req(struct arm_smccc_res *res);
/**
* pkvm_call_hyp_req() - Issue an HVC that can return hypervisor requests
* @f: Hypervisor function symbol to call.
* @...: Arguments to pass to the hypercall.
*
* Re-issue an HVC and process any pending hypervisor request until completion
* or error.
*
* Only use this helper for HVCs whose hypervisor handlers format their return
* registers with pkvm_hyp_req_to_smccc().
*
* Return: Result of the hypercall or a negative error if the hyp request
* handling failed.
*/
#define pkvm_call_hyp_req(f, ...) \
({ \
struct arm_smccc_res __res; \
int __ret; \
do { \
__ret = -1; \
arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), ##__VA_ARGS__, &__res); \
if (WARN_ON(__res.a0 != SMCCC_RET_SUCCESS)) \
break; \
__ret = __res.a1; \
if (!__ret) \
break; \
__ret = __pkvm_handle_smccc_req(&__res); \
} while (!__ret); \
__ret; \
})
int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
struct kvm_pgtable_mm_ops *mm_ops);
void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,

64
arch/arm64/include/asm/ptrace.h
View File

@ -26,6 +26,8 @@
#define GIC_PRIO_PSR_I_SET GICV3_PRIO_PSR_I_SET
#if ARM64_S390_COMMON == 1
/* Additional SPSR bits not exposed in the UABI */
#define PSR_MODE_THREAD_BIT (1 << 0)
#define PSR_IL_BIT (1 << 20)
@ -66,6 +68,8 @@
#define PSR_AA32_IT_MASK 0x0600fc00 /* If-Then execution state mask */
#define PSR_AA32_GE_MASK 0x000f0000
#endif /* ARM64_S390_COMMON == 1 */
#ifdef CONFIG_CPU_BIG_ENDIAN
#define PSR_AA32_ENDSTATE PSR_AA32_E_BIT
#else
@ -102,26 +106,48 @@
#define COMPAT_USER_SZ 296
/* Architecturally defined mapping between AArch32 and AArch64 registers */
enum {
__compat_fp = 11,
__compat_sp = 13,
__compat_lr = 14,
__compat_sp_hyp = 15,
__compat_lr_irq = 16,
__compat_sp_irq = 17,
__compat_lr_svc = 18,
__compat_sp_svc = 19,
__compat_lr_abt = 20,
__compat_sp_abt = 21,
__compat_lr_und = 22,
__compat_sp_und = 23,
__compat_r8_fiq = 24,
__compat_r9_fiq = 25,
__compat_r10_fiq = 26,
__compat_r11_fiq = 27,
__compat_r12_fiq = 28,
__compat_sp_fiq = 29,
__compat_lr_fiq = 30,
};
#define compat_usr(x) regs[(x)]
#define compat_fp regs[11]
#define compat_sp regs[13]
#define compat_lr regs[14]
#define compat_sp_hyp regs[15]
#define compat_lr_irq regs[16]
#define compat_sp_irq regs[17]
#define compat_lr_svc regs[18]
#define compat_sp_svc regs[19]
#define compat_lr_abt regs[20]
#define compat_sp_abt regs[21]
#define compat_lr_und regs[22]
#define compat_sp_und regs[23]
#define compat_r8_fiq regs[24]
#define compat_r9_fiq regs[25]
#define compat_r10_fiq regs[26]
#define compat_r11_fiq regs[27]
#define compat_r12_fiq regs[28]
#define compat_sp_fiq regs[29]
#define compat_lr_fiq regs[30]
#define compat_fp regs[__compat_fp]
#define compat_sp regs[__compat_sp]
#define compat_lr regs[__compat_lr]
#define compat_sp_hyp regs[__compat_sp_hyp]
#define compat_lr_irq regs[__compat_lr_irq]
#define compat_sp_irq regs[__compat_sp_irq]
#define compat_lr_svc regs[__compat_lr_svc]
#define compat_sp_svc regs[__compat_sp_svc]
#define compat_lr_abt regs[__compat_lr_abt]
#define compat_sp_abt regs[__compat_sp_abt]
#define compat_lr_und regs[__compat_lr_und]
#define compat_sp_und regs[__compat_sp_und]
#define compat_r8_fiq regs[__compat_r8_fiq]
#define compat_r9_fiq regs[__compat_r9_fiq]
#define compat_r10_fiq regs[__compat_r10_fiq]
#define compat_r11_fiq regs[__compat_r11_fiq]
#define compat_r12_fiq regs[__compat_r12_fiq]
#define compat_sp_fiq regs[__compat_sp_fiq]
#define compat_lr_fiq regs[__compat_lr_fiq]
static inline unsigned long compat_psr_to_pstate(const unsigned long psr)
{

10
arch/arm64/include/asm/stacktrace/nvhe.h
View File

@ -37,6 +37,14 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state,
state->pc = pc;
}
DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info);
#ifdef CONFIG_PKVM_STACKTRACE
DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
pkvm_stacktrace);
#endif
#ifndef __KVM_NVHE_HYPERVISOR__
/*
* Conventional (non-protected) nVHE HYP stack unwinder
@ -45,8 +53,6 @@ static inline void kvm_nvhe_unwind_init(struct unwind_state *state,
* (by the host in EL1).
*/
DECLARE_KVM_NVHE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_stacktrace_info, kvm_stacktrace_info);
DECLARE_PER_CPU(unsigned long, kvm_arm_hyp_stack_base);
void kvm_nvhe_dump_backtrace(unsigned long hyp_offset);

33
arch/arm64/include/asm/sysreg.h
View File

@ -16,6 +16,8 @@
#include <asm/gpr-num.h>
#if ARM64_S390_COMMON == 1
/*
* ARMv8 ARM reserves the following encoding for system registers:
* (Ref: ARMv8 ARM, Section: "System instruction class encoding overview",
@ -50,6 +52,8 @@
#define sys_reg_CRm(id) (((id) >> CRm_shift) & CRm_mask)
#define sys_reg_Op2(id) (((id) >> Op2_shift) & Op2_mask)
#endif /* ARM64_S390_COMMON == 1 */
#ifndef CONFIG_BROKEN_GAS_INST
#ifdef __ASSEMBLER__
@ -123,6 +127,8 @@
#define GSB_SYS_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 0, 31)
#define GSB_ACK_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 1, 31)
#if ARM64_S390_COMMON == 1
/* Data cache zero operations */
#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)
#define SYS_DC_IGSW sys_insn(1, 0, 7, 6, 4)
@ -295,7 +301,6 @@
#define SYS_APGAKEYLO_EL1 sys_reg(3, 0, 2, 3, 0)
#define SYS_APGAKEYHI_EL1 sys_reg(3, 0, 2, 3, 1)
#define SYS_SPSR_EL1 sys_reg(3, 0, 4, 0, 0)
#define SYS_ELR_EL1 sys_reg(3, 0, 4, 0, 1)
#define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
@ -518,7 +523,6 @@
#define SYS_VTTBR_EL2 sys_reg(3, 4, 2, 1, 0)
#define SYS_HAFGRTR_EL2 sys_reg(3, 4, 3, 1, 6)
#define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0)
#define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1)
#define SYS_SP_EL1 sys_reg(3, 4, 4, 1, 0)
#define SYS_SPSR_irq sys_reg(3, 4, 4, 3, 0)
@ -604,7 +608,6 @@
#define SYS_BRBCR_EL12 sys_reg(2, 5, 9, 0, 0)
#define SYS_TTBR0_EL12 sys_reg(3, 5, 2, 0, 0)
#define SYS_TTBR1_EL12 sys_reg(3, 5, 2, 0, 1)
#define SYS_SPSR_EL12 sys_reg(3, 5, 4, 0, 0)
#define SYS_ELR_EL12 sys_reg(3, 5, 4, 0, 1)
#define SYS_AFSR0_EL12 sys_reg(3, 5, 5, 1, 0)
#define SYS_AFSR1_EL12 sys_reg(3, 5, 5, 1, 1)
@ -835,6 +838,8 @@
#define SCTLR_ELx_A (BIT(1))
#define SCTLR_ELx_M (BIT(0))
#endif /* ARM64_S390_COMMON == 1 */
#ifdef CONFIG_CPU_BIG_ENDIAN
#define ENDIAN_SET_EL2 SCTLR_ELx_EE
#else
@ -869,6 +874,7 @@
SCTLR_EL1_LSMAOE | SCTLR_EL1_nTLSMD | SCTLR_EL1_EIS | \
SCTLR_EL1_TSCXT | SCTLR_EL1_EOS)
#if ARM64_S390_COMMON == 1
/* MAIR_ELx memory attributes (used by Linux) */
#define MAIR_ATTR_DEVICE_nGnRnE UL(0x00)
#define MAIR_ATTR_DEVICE_nGnRE UL(0x04)
@ -1040,7 +1046,7 @@
#define GCS_CAP(x) ((((unsigned long)x) & GCS_CAP_ADDR_MASK) | \
GCS_CAP_VALID_TOKEN)
/*
* Definitions for GICv5 instructions
* Definitions for GICv5 instructions for the Current Domain
*/
#define GICV5_OP_GIC_CDAFF sys_insn(1, 0, 12, 1, 3)
#define GICV5_OP_GIC_CDDI sys_insn(1, 0, 12, 2, 0)
@ -1105,6 +1111,19 @@
#define GICV5_GICR_CDNMIA_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GICR_CDNMIA_ID_MASK GENMASK_ULL(23, 0)
/*
* Definitions for GICv5 instructions for the Virtual Domain
*/
#define GICV5_OP_GIC_VDPEND sys_insn(1, 4, 12, 1, 4)
/* Shift and mask definitions for GIC VDPEND */
#define GICV5_GIC_VDPEND_PENDING_MASK BIT_ULL(63)
#define GICV5_GIC_VDPEND_VM_MASK GENMASK_ULL(47, 32)
#define GICV5_GIC_VDPEND_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_VDPEND_ID_MASK GENMASK_ULL(23, 0)
#endif /* ARM64_S390_COMMON == 1 */
#define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn)
#define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn)
@ -1254,6 +1273,8 @@
par; \
})
#if ARM64_S390_COMMON == 1
#define SYS_FIELD_VALUE(reg, field, val) reg##_##field##_##val
#define SYS_FIELD_GET(reg, field, val) \
@ -1266,6 +1287,8 @@
FIELD_PREP(reg##_##field##_MASK, \
SYS_FIELD_VALUE(reg, field, val))
#endif
#endif /* ARM64_S390_COMMON == 1 */
#endif /* __ASSEMBLER__ */
#endif /* __ASM_SYSREG_H */

15
arch/arm64/include/uapi/asm/kvm.h
View File

@ -97,6 +97,11 @@ struct kvm_regs {
#define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K)
#define KVM_VGIC_V3_ITS_SIZE (2 * SZ_64K)
/* Supported VGICv5 address types */
#define KVM_VGIC_V5_ADDR_TYPE_IRS 6
#define KVM_VGIC_V5_IRS_SIZE (2 * SZ_64K)
#define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */
#define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */
#define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */
@ -418,6 +423,16 @@ enum {
#define KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO 7
#define KVM_DEV_ARM_VGIC_GRP_ITS_REGS 8
#define KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ 9
#define KVM_DEV_ARM_VGIC_GRP_IRS_REGS 10
#define KVM_DEV_ARM_VGIC_GRP_IST 11
struct kvm_vgic_v5_ist {
__u64 spi_ist_addr;
__u64 spi_ist_size;
__u64 lpi_ist_addr;
__u64 lpi_ist_size;
};
#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT 10
#define KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_MASK \
(0x3fffffULL << KVM_DEV_ARM_VGIC_LINE_LEVEL_INFO_SHIFT)

1
arch/arm64/kernel/image-vars.h
View File

@ -143,6 +143,7 @@ KVM_NVHE_ALIAS(__hyp_rodata_end);
#ifdef CONFIG_NVHE_EL2_TRACING
KVM_NVHE_ALIAS(__hyp_event_ids_start);
KVM_NVHE_ALIAS(__hyp_event_ids_end);
KVM_NVHE_ALIAS(__hyp_string_fmts_start);
#endif
/* pKVM static key */

1
arch/arm64/kernel/pi/Makefile
View File

@ -34,6 +34,7 @@ $(obj)/lib-%.pi.o: OBJCOPYFLAGS += --prefix-alloc-sections=.init
$(obj)/lib-%.o: $(srctree)/lib/%.c FORCE
$(call if_changed_rule,cc_o_c)
$(call cmd,force_checksrc)
obj-y := idreg-override.pi.o \
map_kernel.pi.o map_range.pi.o \

10
arch/arm64/kernel/vmlinux.lds.S
View File

@ -15,7 +15,7 @@
#ifdef CONFIG_NVHE_EL2_TRACING
#define HYPERVISOR_EVENT_IDS \
. = ALIGN(PAGE_SIZE); \
. = ALIGN(SZ_8); \
__hyp_event_ids_start = .; \
*(HYP_SECTION_NAME(.event_ids)) \
__hyp_event_ids_end = .;
@ -214,6 +214,14 @@ SECTIONS
HYPERVISOR_RODATA_SECTIONS
#ifdef CONFIG_NVHE_EL2_TRACING
.rodata.hyp_string_fmts : {
__hyp_string_fmts_start = .;
*(_hyp_string_fmts)
__hyp_string_fmts_end = .;
}
#endif
.got : { *(.got) }
/*
* Make sure that the .got.plt is either completely empty or it

1
arch/arm64/kvm/Kconfig
View File

@ -33,6 +33,7 @@ menuconfig KVM
select SCHED_INFO
select GUEST_PERF_EVENTS if PERF_EVENTS
select KVM_GUEST_MEMFD
select KVM_GENERIC_PRE_FAULT_MEMORY
help
Support hosting virtualized guest machines.

3
arch/arm64/kvm/Makefile
View File

@ -24,7 +24,8 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \
vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \
vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \
vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \
vgic/vgic-v5.o
vgic/vgic-v5.o vgic/vgic-v5-tables.o vgic/vgic-irs-v5.o \
vgic-sys-reg-v5.o
kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o
kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o

39
arch/arm64/kvm/arch_timer.c
View File

@ -528,17 +528,11 @@ static void timer_save_state(struct arch_timer_context *ctx)
goto out;
switch (index) {
u64 cval;
case TIMER_VTIMER:
case TIMER_HVTIMER:
timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTV_CTL));
cval = read_sysreg_el0(SYS_CNTV_CVAL);
if (has_broken_cntvoff())
cval -= timer_get_offset(ctx);
timer_set_cval(ctx, cval);
if (!has_broken_cntvoff() || !timer_get_offset(ctx))
timer_set_cval(ctx, read_sysreg_el0(SYS_CNTV_CVAL));
/* Disable the timer */
write_sysreg_el0(0, SYS_CNTV_CTL);
@ -564,11 +558,12 @@ static void timer_save_state(struct arch_timer_context *ctx)
case TIMER_PTIMER:
case TIMER_HPTIMER:
timer_set_ctl(ctx, read_sysreg_el0(SYS_CNTP_CTL));
cval = read_sysreg_el0(SYS_CNTP_CVAL);
cval -= timer_get_offset(ctx);
timer_set_cval(ctx, cval);
/*
* With an offset, memory already holds the guest's CVAL (the
* trap handler or __deactivate_traps() wrote it).
*/
if (!timer_get_offset(ctx))
timer_set_cval(ctx, read_sysreg_el0(SYS_CNTP_CVAL));
/* Disable the timer */
write_sysreg_el0(0, SYS_CNTP_CTL);
@ -647,7 +642,8 @@ static void timer_restore_state(struct arch_timer_context *ctx)
offset = timer_get_offset(ctx);
if (has_broken_cntvoff()) {
set_cntvoff(0);
cval += offset;
if (offset)
cval = timer_apply_offset(cval, offset, kvm_phys_timer_read());
} else {
set_cntvoff(offset);
}
@ -660,7 +656,8 @@ static void timer_restore_state(struct arch_timer_context *ctx)
cval = timer_get_cval(ctx);
offset = timer_get_offset(ctx);
set_cntpoff(offset);
cval += offset;
if (offset)
cval = timer_apply_offset(cval, offset, kvm_phys_timer_read());
write_sysreg_el0(cval, SYS_CNTP_CVAL);
isb();
write_sysreg_el0(timer_get_ctl(ctx), SYS_CNTP_CTL);
@ -1110,8 +1107,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
timer_context_init(vcpu, i);
/* Synchronize offsets across timers of a VM if not already provided */
if (!vcpu_is_protected(vcpu) &&
!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) {
if (!test_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &vcpu->kvm->arch.flags)) {
timer_set_offset(vcpu_vtimer(vcpu), kvm_phys_timer_read());
timer_set_offset(vcpu_ptimer(vcpu), 0);
}
@ -1133,6 +1129,15 @@ void kvm_timer_init_vm(struct kvm *kvm)
*/
for (int i = 0; i < NR_KVM_TIMERS; i++)
kvm->arch.timer_data.ppi[i] = get_vgic_ppi(kvm, default_ppi[i]);
/*
* Protected VMs don't allow the userspace to set counter offsets,
* either set via counter register writes or the dedicated ioctls.
* Pretend the offset has already been set and rely on the default
* offset being 0.
*/
if (kvm_vm_is_protected(kvm))
set_bit(KVM_ARCH_FLAG_VM_COUNTER_OFFSET, &kvm->arch.flags);
}
void kvm_timer_cpu_up(void)

359
arch/arm64/kvm/arm.c
View File

@ -93,6 +93,7 @@ static const struct kvm_ioctl_cap_map vm_ioctl_caps[] = {
{ KVM_ARM_PREFERRED_TARGET, KVM_CAP_ARM_BASIC },
};
static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu);
/*
* Set *ext to the capability.
* Return 0 if found, or -EINVAL if no IOCTL matches.
@ -214,6 +215,26 @@ static int kvm_arm_default_max_vcpus(void)
return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS;
}
static int kvm_init_vm_flavor(struct kvm *kvm, unsigned long type)
{
bool protected = type & KVM_VM_TYPE_ARM_PROTECTED;
if (is_protected_kvm_enabled()) {
if (protected)
kvm->arch.vm_flavor = VM_PROTECTED_PKVM;
else
kvm->arch.vm_flavor = VM_PKVM;
} else if (protected) {
return -EINVAL;
} else if (has_vhe()) {
kvm->arch.vm_flavor = VM_VHE;
} else {
kvm->arch.vm_flavor = VM_NVHE;
}
return 0;
}
/**
* kvm_arch_init_vm - initializes a VM data structure
* @kvm: pointer to the KVM struct
@ -236,6 +257,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
mutex_unlock(&kvm->lock);
#endif
ret = kvm_init_vm_flavor(kvm, type);
if (ret)
return ret;
ret = kvm_share_hyp(kvm, kvm + 1);
if (ret)
return ret;
@ -259,12 +284,9 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
* If any failures occur after this is successful, make sure to
* call __pkvm_unreserve_vm to unreserve the VM in hyp.
*/
ret = pkvm_init_host_vm(kvm, type);
ret = pkvm_init_host_vm(kvm);
if (ret)
goto err_uninit_mmu;
} else if (type & KVM_VM_TYPE_ARM_PROTECTED) {
ret = -EINVAL;
goto err_uninit_mmu;
}
kvm_vgic_early_init(kvm);
@ -410,6 +432,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
case KVM_CAP_COUNTER_OFFSET:
case KVM_CAP_ARM_WRITABLE_IMP_ID_REGS:
case KVM_CAP_ARM_SEA_TO_USER:
case KVM_CAP_PRE_FAULT_MEMORY:
r = 1;
break;
case KVM_CAP_SET_GUEST_DEBUG2:
@ -451,7 +474,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
r = system_supports_mte();
break;
case KVM_CAP_STEAL_TIME:
r = kvm_arm_pvtime_supported();
r = kvm_arm_pvtime_supported(kvm);
break;
case KVM_CAP_ARM_EL1_32BIT:
r = cpus_have_final_cap(ARM64_HAS_32BIT_EL1);
@ -552,6 +575,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
mutex_unlock(&vcpu->mutex);
#endif
kvm_init_vcpu_ops(vcpu);
/* Force users to call KVM_ARM_VCPU_INIT */
vcpu_clear_flag(vcpu, VCPU_INITIALIZED);
@ -595,7 +620,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
if (!is_protected_kvm_enabled())
kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
else
free_hyp_memcache(&vcpu->arch.pkvm_memcache);
free_hyp_memcache(&vcpu->arch.stage2_mc);
kvm_timer_vcpu_terminate(vcpu);
kvm_pmu_vcpu_destroy(vcpu);
kvm_vgic_vcpu_destroy(vcpu);
@ -614,33 +639,34 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu)
static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
{
if (vcpu_has_ptrauth(vcpu) && !is_protected_kvm_enabled()) {
/*
* Either we're running an L2 guest, and the API/APK bits come
* from L1's HCR_EL2, or API/APK are both set.
*/
if (unlikely(is_nested_ctxt(vcpu))) {
u64 val;
if (!vcpu_has_ptrauth(vcpu))
return;
val = __vcpu_sys_reg(vcpu, HCR_EL2);
val &= (HCR_API | HCR_APK);
vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
vcpu->arch.hcr_el2 |= val;
} else {
vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
}
/*
* Either we're running an L2 guest, and the API/APK bits come
* from L1's HCR_EL2, or API/APK are both set.
*/
if (unlikely(is_nested_ctxt(vcpu))) {
u64 val;
/*
* Save the host keys if there is any chance for the guest
* to use pauth, as the entry code will reload the guest
* keys in that case.
*/
if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
struct kvm_cpu_context *ctxt;
val = __vcpu_sys_reg(vcpu, HCR_EL2);
val &= (HCR_API | HCR_APK);
vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
vcpu->arch.hcr_el2 |= val;
} else {
vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
}
ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
ptrauth_save_keys(ctxt);
}
/*
* Save the host keys if there is any chance for the guest
* to use pauth, as the entry code will reload the guest
* keys in that case.
*/
if (vcpu->arch.hcr_el2 & (HCR_API | HCR_APK)) {
struct kvm_cpu_context *ctxt;
ctxt = this_cpu_ptr_hyp_sym(kvm_hyp_ctxt);
ptrauth_save_keys(ctxt);
}
}
@ -666,14 +692,11 @@ static bool kvm_vcpu_should_clear_twe(struct kvm_vcpu *vcpu)
return single_task_running();
}
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
static void vcpu_prepare_mmu(struct kvm_vcpu *vcpu)
{
struct kvm_s2_mmu *mmu;
int *last_ran;
if (is_protected_kvm_enabled())
goto nommu;
if (vcpu_has_nv(vcpu))
kvm_vcpu_load_hw_mmu(vcpu);
@ -703,25 +726,15 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
kvm_call_hyp(__kvm_flush_cpu_context, mmu);
*last_ran = vcpu->vcpu_idx;
}
}
nommu:
vcpu->cpu = cpu;
/*
* The timer must be loaded before the vgic to correctly set up physical
* interrupt deactivation in nested state (e.g. timer interrupt).
*/
kvm_timer_vcpu_load(vcpu);
kvm_vgic_load(vcpu);
kvm_vcpu_load_debug(vcpu);
kvm_vcpu_load_fgt(vcpu);
if (has_vhe())
kvm_vcpu_load_vhe(vcpu);
kvm_arch_vcpu_load_fp(vcpu);
kvm_vcpu_pmu_restore_guest(vcpu);
if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
static void vcpu_put_mmu(struct kvm_vcpu *vcpu)
{
kvm_arm_vmid_clear_active();
}
static void vcpu_set_wfx_traps(struct kvm_vcpu *vcpu)
{
if (kvm_vcpu_should_clear_twe(vcpu))
vcpu->arch.hcr_el2 &= ~HCR_TWE;
else
@ -731,16 +744,73 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
vcpu->arch.hcr_el2 &= ~HCR_TWI;
else
vcpu->arch.hcr_el2 |= HCR_TWI;
}
static void vcpu_load_pvtime(struct kvm_vcpu *vcpu)
{
if (kvm_arm_is_pvtime_enabled(&vcpu->arch))
kvm_make_request(KVM_REQ_RECORD_STEAL, vcpu);
}
static void vhe_vcpu_load(struct kvm_vcpu *vcpu)
{
vcpu_prepare_mmu(vcpu);
/*
* For VHE, the timer must be loaded before the vgic to correctly
* set up physical interrupt deactivation in nested state (e.g. timer
* interrupt).
*/
kvm_timer_vcpu_load(vcpu);
kvm_vgic_load(vcpu);
kvm_vcpu_load_debug(vcpu);
kvm_vcpu_load_fgt(vcpu);
kvm_vcpu_load_vhe(vcpu);
kvm_arch_vcpu_load_fp(vcpu);
kvm_vcpu_pmu_restore_guest(vcpu);
vcpu_load_pvtime(vcpu);
vcpu_set_wfx_traps(vcpu);
vcpu_set_pauth_traps(vcpu);
}
if (is_protected_kvm_enabled()) {
kvm_call_hyp_nvhe(__pkvm_vcpu_load,
vcpu->kvm->arch.pkvm.handle,
vcpu->vcpu_idx, vcpu->arch.hcr_el2);
kvm_call_hyp(__vgic_v3_restore_vmcr_aprs,
&vcpu->arch.vgic_cpu.vgic_v3);
}
static void nvhe_vcpu_load(struct kvm_vcpu *vcpu)
{
vcpu_prepare_mmu(vcpu);
kvm_timer_vcpu_load(vcpu);
kvm_vgic_load(vcpu);
kvm_vcpu_load_debug(vcpu);
kvm_vcpu_load_fgt(vcpu);
kvm_arch_vcpu_load_fp(vcpu);
kvm_vcpu_pmu_restore_guest(vcpu);
vcpu_load_pvtime(vcpu);
vcpu_set_wfx_traps(vcpu);
vcpu_set_pauth_traps(vcpu);
}
static void pkvm_vcpu_load(struct kvm_vcpu *vcpu)
{
kvm_timer_vcpu_load(vcpu);
kvm_vgic_load(vcpu);
kvm_vcpu_load_debug(vcpu);
kvm_vcpu_load_fgt(vcpu);
kvm_arch_vcpu_load_fp(vcpu);
kvm_vcpu_pmu_restore_guest(vcpu);
vcpu_load_pvtime(vcpu);
vcpu_set_wfx_traps(vcpu);
kvm_call_hyp_nvhe(__pkvm_vcpu_load,
vcpu->kvm->arch.pkvm.handle,
vcpu->vcpu_idx, vcpu->arch.hcr_el2);
kvm_call_hyp_nvhe(__vgic_v3_restore_vmcr_aprs,
&vcpu->arch.vgic_cpu.vgic_v3);
}
void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
{
vcpu->cpu = cpu;
vcpu->arch.vcpu_ops->vcpu_load(vcpu);
if (!cpumask_test_cpu(cpu, vcpu->kvm->arch.supported_cpus))
vcpu_set_on_unsupported_cpu(vcpu);
@ -748,49 +818,76 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
vcpu->arch.pid = pid_nr(vcpu->pid);
}
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
static void vhe_vcpu_put(struct kvm_vcpu *vcpu)
{
if (is_protected_kvm_enabled()) {
kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3);
kvm_call_hyp_nvhe(__pkvm_vcpu_put);
/* __pkvm_vcpu_put implies a sync of the state */
if (!kvm_vm_is_protected(vcpu->kvm))
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
}
kvm_vcpu_put_debug(vcpu);
kvm_arch_vcpu_put_fp(vcpu);
if (has_vhe())
kvm_vcpu_put_vhe(vcpu);
kvm_vcpu_put_vhe(vcpu);
kvm_timer_vcpu_put(vcpu);
kvm_vgic_put(vcpu);
kvm_vcpu_pmu_restore_host(vcpu);
if (vcpu_has_nv(vcpu))
kvm_vcpu_put_hw_mmu(vcpu);
kvm_arm_vmid_clear_active();
vcpu_put_mmu(vcpu);
}
static void nvhe_vcpu_put(struct kvm_vcpu *vcpu)
{
kvm_vcpu_put_debug(vcpu);
kvm_arch_vcpu_put_fp(vcpu);
kvm_timer_vcpu_put(vcpu);
kvm_vgic_put(vcpu);
kvm_vcpu_pmu_restore_host(vcpu);
vcpu_put_mmu(vcpu);
}
static void pkvm_vcpu_put(struct kvm_vcpu *vcpu)
{
kvm_call_hyp_nvhe(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3);
kvm_call_hyp_nvhe(__pkvm_vcpu_put);
/* __pkvm_vcpu_put implies a sync of the state */
if (vcpu->kvm->arch.vm_flavor == VM_PKVM)
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
kvm_vcpu_put_debug(vcpu);
kvm_arch_vcpu_put_fp(vcpu);
kvm_timer_vcpu_put(vcpu);
kvm_vgic_put(vcpu);
kvm_vcpu_pmu_restore_host(vcpu);
}
void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
{
vcpu->arch.vcpu_ops->vcpu_put(vcpu);
vcpu_clear_on_unsupported_cpu(vcpu);
vcpu->cpu = -1;
}
static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
static void __kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu, u32 mp_state)
{
WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
WRITE_ONCE(vcpu->arch.mp_state.mp_state, mp_state);
kvm_make_request(KVM_REQ_SLEEP, vcpu);
kvm_vcpu_kick(vcpu);
}
/* The guest's own CPU_OFF: EL2 has already powered the vCPU off. */
void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu)
{
u32 mp_state = vcpu_is_protected(vcpu) ? KVM_MP_STATE_UNINITIALIZED : KVM_MP_STATE_STOPPED;
spin_lock(&vcpu->arch.mp_state_lock);
__kvm_arm_vcpu_power_off(vcpu);
__kvm_arm_vcpu_power_off(vcpu, mp_state);
spin_unlock(&vcpu->arch.mp_state_lock);
}
bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu)
{
return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED;
u32 mp_state = READ_ONCE(vcpu->arch.mp_state.mp_state);
return mp_state == KVM_MP_STATE_STOPPED || mp_state == KVM_MP_STATE_UNINITIALIZED;
}
static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu)
@ -805,6 +902,12 @@ static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu)
return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_SUSPENDED;
}
/* Only the guest's CPU_ON may start a vCPU EL2 holds powered off. */
bool kvm_pkvm_vcpu_is_powered_off(struct kvm_vcpu *vcpu)
{
return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_UNINITIALIZED;
}
int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
struct kvm_mp_state *mp_state)
{
@ -822,12 +925,24 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
switch (mp_state->mp_state) {
case KVM_MP_STATE_RUNNABLE:
if (kvm_pkvm_vcpu_is_powered_off(vcpu)) {
ret = -EPERM;
break;
}
WRITE_ONCE(vcpu->arch.mp_state, *mp_state);
break;
case KVM_MP_STATE_STOPPED:
__kvm_arm_vcpu_power_off(vcpu);
/* Keep EL2's off record: STOPPED over it would let RUNNABLE through. */
if (!kvm_pkvm_vcpu_is_powered_off(vcpu))
__kvm_arm_vcpu_power_off(vcpu, KVM_MP_STATE_STOPPED);
break;
case KVM_MP_STATE_SUSPENDED:
if (kvm_pkvm_vcpu_is_powered_off(vcpu)) {
ret = -EPERM;
break;
}
kvm_arm_vcpu_suspend(vcpu);
break;
default:
@ -953,7 +1068,7 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
return ret;
}
ret = kvm_finalize_sys_regs(vcpu);
ret = kvm_vcpu_finalize_sys_regs(vcpu);
if (ret)
return ret;
@ -983,13 +1098,13 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
return ret;
}
ret = vgic_v5_finalize_ppi_state(kvm);
ret = vgic_v5_finalize_ppi_state(vcpu);
if (ret)
return ret;
if (is_protected_kvm_enabled()) {
/* Start with the vcpu in a dirty state */
if (!kvm_vm_is_protected(vcpu->kvm))
if (vcpu->kvm->arch.vm_flavor == VM_PKVM)
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
ret = pkvm_create_hyp_vm(kvm);
if (ret)
@ -1163,6 +1278,14 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu)
preempt_enable();
}
if (kvm_check_request(KVM_REQ_RELOAD_GICv5, vcpu)) {
/* The IRS enable bit was changed */
preempt_disable();
vgic_v5_put(vcpu);
vgic_v5_load(vcpu);
preempt_enable();
}
if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu))
kvm_vcpu_reload_pmu(vcpu);
@ -1267,7 +1390,7 @@ static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu)
* kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
* @vcpu: The VCPU pointer
*
* This function is called through the VCPU_RUN ioctl called from user space. It
* This function is called through the KVM_RUN ioctl called from user space. It
* will execute VM code in a loop until the time slice for the process is used
* or some emulation is needed from user space in which case the function will
* return with return value 0 and with the kvm_run structure filled in with the
@ -1581,10 +1704,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
return -EINVAL;
}
static unsigned long system_supported_vcpu_features(void)
static unsigned long system_supported_vcpu_features(struct kvm_vcpu *vcpu)
{
unsigned long features = KVM_VCPU_VALID_FEATURES;
if (vcpu->kvm->arch.vm_flavor == VM_PROTECTED_PKVM)
kvm_pkvm_vcpu_allowed_features(vcpu->kvm, &features);
if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
@ -1607,6 +1733,7 @@ static unsigned long system_supported_vcpu_features(void)
return features;
}
#if ARM64_S390_COMMON == 1
static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
const struct kvm_vcpu_init *init)
{
@ -1621,7 +1748,12 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
return -ENOENT;
}
if (features & ~system_supported_vcpu_features())
if (features & ~system_supported_vcpu_features(vcpu))
return -EINVAL;
/* EL2 implements PSCI 1.1; the host must not dispatch as 0.1. */
if (vcpu_is_protected(vcpu) &&
!test_bit(KVM_ARM_VCPU_PSCI_0_2, &features))
return -EINVAL;
/*
@ -1660,6 +1792,8 @@ static bool kvm_vcpu_init_changed(struct kvm_vcpu *vcpu,
KVM_VCPU_MAX_FEATURES);
}
#endif /* ARM64_S390_COMMON == 1 */
static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = vcpu->kvm;
@ -1779,7 +1913,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu,
spin_lock(&vcpu->arch.mp_state_lock);
if (power_off)
__kvm_arm_vcpu_power_off(vcpu);
__kvm_arm_vcpu_power_off(vcpu, KVM_MP_STATE_STOPPED);
else
WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE);
@ -1856,6 +1990,24 @@ static int kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
return __kvm_arm_vcpu_set_events(vcpu, events);
}
/*
* Once a protected vCPU has run, the host copy holds the boot state the VMM
* wrote plus what the exit handlers copy out, and EL2 has read mp_state,
* which it does only at hyp vCPU creation.
*/
static long pkvm_filter_vcpu_ioctl(struct kvm_vcpu *vcpu, unsigned int ioctl)
{
switch (ioctl) {
case KVM_ARM_VCPU_INIT:
case KVM_SET_ONE_REG:
case KVM_GET_ONE_REG:
if (vcpu_is_protected(vcpu) && vcpu_has_run_once(vcpu))
return -EPERM;
}
return 0;
}
long kvm_arch_vcpu_ioctl(struct file *filp,
unsigned int ioctl, unsigned long arg)
{
@ -1864,6 +2016,10 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
struct kvm_device_attr attr;
long r;
r = pkvm_filter_vcpu_ioctl(vcpu, ioctl);
if (r)
return r;
switch (ioctl) {
case KVM_ARM_VCPU_INIT: {
struct kvm_vcpu_init init;
@ -2109,6 +2265,37 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
}
}
static const struct kvm_vcpu_ops vhe_vcpu_ops = {
.vcpu_load = vhe_vcpu_load,
.vcpu_put = vhe_vcpu_put,
};
static const struct kvm_vcpu_ops nvhe_vcpu_ops = {
.vcpu_load = nvhe_vcpu_load,
.vcpu_put = nvhe_vcpu_put,
};
static const struct kvm_vcpu_ops pkvm_vcpu_ops = {
.vcpu_load = pkvm_vcpu_load,
.vcpu_put = pkvm_vcpu_put,
};
#define KVM_VCPU_OPS(flavor, ops) \
[(flavor)] = &(ops)
static const struct kvm_vcpu_ops *arm64_vcpu_ops[] = {
KVM_VCPU_OPS(VM_NVHE, nvhe_vcpu_ops),
KVM_VCPU_OPS(VM_VHE, vhe_vcpu_ops),
KVM_VCPU_OPS(VM_PKVM, pkvm_vcpu_ops),
KVM_VCPU_OPS(VM_PROTECTED_PKVM, pkvm_vcpu_ops),
};
static void kvm_init_vcpu_ops(struct kvm_vcpu *vcpu)
{
BUILD_BUG_ON(ARRAY_SIZE(arm64_vcpu_ops) != VM_FLAVOR_MAX);
vcpu->arch.vcpu_ops = arm64_vcpu_ops[vcpu->kvm->arch.vm_flavor];
}
static unsigned long nvhe_percpu_size(void)
{
return (unsigned long)CHOOSE_NVHE_SYM(__per_cpu_end) -
@ -2626,6 +2813,10 @@ static void kvm_hyp_init_symbols(void)
kvm_nvhe_sym(__icache_flags) = __icache_flags;
kvm_nvhe_sym(kvm_arm_vmid_bits) = kvm_arm_vmid_bits;
kvm_nvhe_sym(spectre_v2_state) = arm64_get_spectre_v2_state();
kvm_nvhe_sym(spectre_v4_state) = arm64_get_spectre_v4_state();
kvm_nvhe_sym(spectre_bhb_state) = arm64_get_spectre_bhb_state();
/* Propagate the FGT state to the nVHE side */
kvm_nvhe_sym(hfgrtr_masks) = hfgrtr_masks;
kvm_nvhe_sym(hfgwtr_masks) = hfgwtr_masks;
@ -2918,6 +3109,8 @@ static int __init init_hyp_mode(void)
kvm_err("Failed to init hyp memory protection\n");
goto out_err;
}
pkvm_selftests();
}
return 0;

0
arch/arm64/include/asm/kvm_define_hypevents.h → arch/arm64/kvm/define_hypevents.h
View File

33
arch/arm64/kvm/guest.c
View File

@ -62,6 +62,7 @@ const struct kvm_stats_header kvm_vcpu_stats_header = {
sizeof(kvm_vcpu_stats_desc),
};
#if ARM64_S390_COMMON == 1
static bool core_reg_offset_is_vreg(u64 off)
{
return off >= KVM_REG_ARM_CORE_REG(fp_regs.vregs) &&
@ -134,19 +135,19 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
KVM_REG_ARM_CORE_REG(regs.regs[30]):
off -= KVM_REG_ARM_CORE_REG(regs.regs[0]);
off /= 2;
return &vcpu->arch.ctxt.regs.regs[off];
return &vcpu_gp_regs(vcpu)[off];
case KVM_REG_ARM_CORE_REG(regs.sp):
return &vcpu->arch.ctxt.regs.sp;
return vcpu_sp_el0(vcpu);
case KVM_REG_ARM_CORE_REG(regs.pc):
return &vcpu->arch.ctxt.regs.pc;
return vcpu_pc(vcpu);
case KVM_REG_ARM_CORE_REG(regs.pstate):
return &vcpu->arch.ctxt.regs.pstate;
return vcpu_cpsr(vcpu);
case KVM_REG_ARM_CORE_REG(sp_el1):
return __ctxt_sys_reg(&vcpu->arch.ctxt, SP_EL1);
return vcpu_sp_el1(vcpu);
case KVM_REG_ARM_CORE_REG(elr_el1):
return __ctxt_sys_reg(&vcpu->arch.ctxt, ELR_EL1);
@ -170,13 +171,13 @@ static void *core_reg_addr(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]):
off -= KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]);
off /= 4;
return &vcpu->arch.ctxt.fp_regs.vregs[off];
return vcpu_vreg(vcpu, off);
case KVM_REG_ARM_CORE_REG(fp_regs.fpsr):
return &vcpu->arch.ctxt.fp_regs.fpsr;
return vcpu_fpsr(vcpu);
case KVM_REG_ARM_CORE_REG(fp_regs.fpcr):
return &vcpu->arch.ctxt.fp_regs.fpcr;
return vcpu_fpcr(vcpu);
default:
return NULL;
@ -306,6 +307,8 @@ static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
return err;
}
#endif /* ARM64_S390_COMMON == 1 */
#define vq_word(vq) (((vq) - SVE_VQ_MIN) / 64)
#define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64)
#define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq)))
@ -543,6 +546,7 @@ int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
return -EINVAL;
}
#if ARM64_S390_COMMON == 1
static int copy_core_reg_indices(const struct kvm_vcpu *vcpu,
u64 __user *uindices)
{
@ -591,6 +595,8 @@ static unsigned long num_core_regs(const struct kvm_vcpu *vcpu)
return copy_core_reg_indices(vcpu, NULL);
}
#endif /* ARM64_S390_COMMON == 1 */
static unsigned long num_sve_regs(const struct kvm_vcpu *vcpu)
{
const unsigned int slices = vcpu_sve_slices(vcpu);
@ -786,6 +792,13 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
u64 esr = events->exception.serror_esr;
int ret = 0;
/*
* EL2 injects an external abort only to complete a forwarded abort.
* SError injection is forwarded.
*/
if (vcpu_is_protected(vcpu) && ext_dabt_pending)
return -EPERM;
/*
* Immediately commit the pending SEA to the vCPU's architectural
* state which is necessary since we do not return a pending SEA
@ -883,6 +896,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
{
trace_kvm_set_guest_debug(vcpu, dbg->control);
/* A protected guest's debug state is not exposed to the host. */
if (vcpu_is_protected(vcpu))
return -EPERM;
if (dbg->control & ~KVM_GUESTDBG_VALID_MASK)
return -EINVAL;

2
arch/arm64/kvm/hyp/entry.S
View File

@ -46,7 +46,7 @@ alternative_else_nop_endif
// Ensure that __guest_enter() always provides a context
// synchronization event so that callers don't need ISBs for anything
// that would usually be synchonized by the ERET.
// that would usually be synchronized by the ERET.
isb
mov x0, #ARM_EXCEPTION_IRQ
ret

30
arch/arm64/kvm/hyp/exception.c
View File

@ -70,6 +70,7 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
enum exception_type type)
{
unsigned long sctlr, vbar, old, new, mode;
struct kvm *kvm;
u64 exc_offset;
mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
@ -109,8 +110,10 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
new |= (old & PSR_C_BIT);
new |= (old & PSR_V_BIT);
if (kvm_has_mte(kern_hyp_va(vcpu->kvm)))
kvm = vcpu_get_kvm(vcpu);
if (kvm && kvm_has_mte(kvm))
new |= PSR_TCO_BIT;
vcpu_put_kvm(vcpu, kvm);
new |= (old & PSR_DIT_BIT);
@ -276,12 +279,12 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
switch(mode) {
case PSR_AA32_MODE_ABT:
__vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr));
vcpu_gp_regs(vcpu)->compat_lr_abt = return_address;
vcpu_gp_regs(vcpu)[__compat_lr_abt] = return_address;
break;
case PSR_AA32_MODE_UND:
__vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr));
vcpu_gp_regs(vcpu)->compat_lr_und = return_address;
vcpu_gp_regs(vcpu)[__compat_lr_und] = return_address;
break;
}
@ -350,12 +353,19 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu)
*/
void __kvm_adjust_pc(struct kvm_vcpu *vcpu)
{
if (vcpu_get_flag(vcpu, PENDING_EXCEPTION)) {
kvm_inject_exception(vcpu);
vcpu_clear_flag(vcpu, PENDING_EXCEPTION);
vcpu_clear_flag(vcpu, EXCEPT_MASK);
} else if (vcpu_get_flag(vcpu, INCREMENT_PC)) {
kvm_skip_instr(vcpu);
vcpu_clear_flag(vcpu, INCREMENT_PC);
struct kvm_vcpu *target = kvm_adjust_pc_get(vcpu);
if (!target)
return;
if (vcpu_get_flag(target, PENDING_EXCEPTION)) {
kvm_inject_exception(target);
vcpu_clear_flag(target, PENDING_EXCEPTION);
vcpu_clear_flag(target, EXCEPT_MASK);
} else if (vcpu_get_flag(target, INCREMENT_PC)) {
kvm_skip_instr(target);
vcpu_clear_flag(target, INCREMENT_PC);
}
kvm_adjust_pc_put(vcpu, target);
}

2
arch/arm64/kvm/hyp/hyp-constants.c
View File

@ -7,7 +7,5 @@
int main(void)
{
DEFINE(STRUCT_HYP_PAGE_SIZE, sizeof(struct hyp_page));
DEFINE(PKVM_HYP_VM_SIZE, sizeof(struct pkvm_hyp_vm));
DEFINE(PKVM_HYP_VCPU_SIZE, sizeof(struct pkvm_hyp_vcpu));
return 0;
}

52
arch/arm64/kvm/hyp/include/hyp/adjust_pc.h
View File

@ -13,6 +13,34 @@
#include <asm/kvm_emulate.h>
#include <asm/kvm_host.h>
#ifdef __KVM_NVHE_HYPERVISOR__
#include <nvhe/pkvm.h>
/* Under pKVM a host vCPU's ->kvm is host-writable. */
static inline struct kvm *vcpu_get_kvm(struct kvm_vcpu *vcpu)
{
if (is_protected_kvm_enabled())
return pkvm_vcpu_get_kvm(vcpu);
return kern_hyp_va(vcpu->kvm);
}
static inline void vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm)
{
if (is_protected_kvm_enabled())
pkvm_vcpu_put_kvm(vcpu, kvm);
}
#else
static inline struct kvm *vcpu_get_kvm(struct kvm_vcpu *vcpu)
{
return vcpu->kvm;
}
static inline void vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm)
{
}
#endif
static inline void kvm_skip_instr(struct kvm_vcpu *vcpu)
{
if (vcpu_mode_is_32bit(vcpu)) {
@ -33,11 +61,11 @@ static inline void kvm_skip_instr(struct kvm_vcpu *vcpu)
static inline void __kvm_skip_instr(struct kvm_vcpu *vcpu)
{
*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
vcpu_gp_regs(vcpu)->pstate = read_sysreg_el2(SYS_SPSR);
*vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR);
kvm_skip_instr(vcpu);
write_sysreg_el2(vcpu_gp_regs(vcpu)->pstate, SYS_SPSR);
write_sysreg_el2(*vcpu_cpsr(vcpu), SYS_SPSR);
write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
}
@ -50,4 +78,24 @@ static inline void kvm_skip_host_instr(void)
write_sysreg_el2(read_sysreg_el2(SYS_ELR) + 4, SYS_ELR);
}
/*
* Under pKVM, the vCPU __kvm_adjust_pc() adjusts for @vcpu (NULL leaves the
* request on @vcpu for its next entry), and the copy of the consumed
* PC_UPDATE_REQ back to @vcpu.
*/
#ifdef __KVM_NVHE_HYPERVISOR__
struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu);
void kvm_adjust_pc_put(struct kvm_vcpu *vcpu, struct kvm_vcpu *target);
#else
static inline struct kvm_vcpu *kvm_adjust_pc_get(struct kvm_vcpu *vcpu)
{
return vcpu;
}
static inline void kvm_adjust_pc_put(struct kvm_vcpu *vcpu,
struct kvm_vcpu *target)
{
}
#endif
#endif

4
arch/arm64/kvm/hyp/include/hyp/switch.h
View File

@ -470,7 +470,7 @@ static inline bool kvm_hyp_handle_mops(struct kvm_vcpu *vcpu, u64 *exit_code)
u64 spsr;
*vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
arm64_mops_reset_regs(vcpu_gp_regs(vcpu), vcpu->arch.fault.esr_el2);
arm64_mops_reset_regs(&vcpu->arch.ctxt.regs, vcpu->arch.fault.esr_el2);
write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
/*
@ -902,7 +902,7 @@ static inline void synchronize_vcpu_pstate(struct kvm_vcpu *vcpu)
/*
* Check for the conditions of Cortex-A510's #2077057. When these occur
* SPSR_EL2 can't be trusted, but isn't needed either as it is
* unchanged from the value in vcpu_gp_regs(vcpu)->pstate.
* unchanged from the value in vcpu_cpsr(vcpu).
* Are we single-stepping the guest, and took a PAC exception from the
* active-not-pending state?
*/

28
arch/arm64/kvm/hyp/include/nvhe/alloc.h Normal file
View File

@ -0,0 +1,28 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __KVM_NVHE_ALLOC__
#define __KVM_NVHE_ALLOC__
#include <linux/types.h>
#include <asm/kvm_host.h>
void *hyp_alloc(size_t size);
int hyp_alloc_errno(void);
u32 hyp_alloc_topup_needed(void);
void hyp_free(void *addr);
int hyp_alloc_init(size_t size);
int hyp_alloc_topup(struct kvm_hyp_memcache *host_mc);
unsigned long hyp_alloc_reclaimable(void);
void hyp_alloc_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target);
#ifdef CONFIG_NVHE_EL2_DEBUG
int hyp_allocator_selftest(void);
u32 hyp_alloc_selftest_topup_needed(void);
int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc);
void hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target);
#else
static inline int hyp_alloc_selftest_topup(struct kvm_hyp_memcache *host_mc) { return 0; };
static inline void
hyp_alloc_selftest_reclaim(struct kvm_hyp_memcache *host_mc, unsigned long target) { };
#endif
#endif

2
arch/arm64/kvm/hyp/include/nvhe/define_events.h
View File

@ -10,5 +10,3 @@
#define HYP_EVENT_MULTI_READ
#include <asm/kvm_hypevents.h>
#undef HYP_EVENT_MULTI_READ
#undef HYP_EVENT

3
arch/arm64/kvm/hyp/include/nvhe/mm.h
View File

@ -25,11 +25,14 @@ int hyp_back_vmemmap(phys_addr_t back);
int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot);
int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot);
void pkvm_remove_mappings(void *from, void *to);
int __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
enum kvm_pgtable_prot prot,
unsigned long *haddr);
int pkvm_create_stack(phys_addr_t phys, unsigned long *haddr);
int pkvm_check_host_ownership(void);
int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr);
int pkvm_map_private_va_range(void *haddr, phys_addr_t phys, size_t size);
phys_addr_t pkvm_private_va_range_pa(void *va);
#endif /* __KVM_HYP_MM_H */

40
arch/arm64/kvm/hyp/include/nvhe/pkvm.h
View File

@ -7,6 +7,7 @@
#ifndef __ARM64_KVM_NVHE_PKVM_H__
#define __ARM64_KVM_NVHE_PKVM_H__
#include <asm/kvm_hcall.h>
#include <asm/kvm_pkvm.h>
#include <nvhe/gfp.h>
@ -35,6 +36,15 @@ struct pkvm_hyp_vcpu {
* per-cpu pointer tracking us. Otherwise, NULL if not loaded.
*/
struct pkvm_hyp_vcpu **loaded_hyp_vcpu;
/* The previous exit's ARM_EXCEPTION_* code. */
u32 exit_code;
/*
* PSCI_0_2_AFFINITY_LEVEL_{OFF, ON_PENDING, ON}. A non-protected
* vCPU is always ON.
*/
int power_state;
};
/*
@ -50,8 +60,15 @@ struct pkvm_hyp_vm {
struct kvm_pgtable pgt;
struct kvm_pgtable_mm_ops mm_ops;
struct hyp_pool pool;
unsigned short refcount;
hyp_spinlock_t lock;
/*
* The vCPU initialised RUNNABLE: claimed under vm_table_lock,
* released only if its own init fails.
*/
struct pkvm_hyp_vcpu *primary_vcpu;
/* Array of the hyp vCPU structures for this VM. */
struct pkvm_hyp_vcpu *vcpus[];
};
@ -74,14 +91,24 @@ static inline bool pkvm_hyp_vm_is_protected(struct pkvm_hyp_vm *hyp_vm)
return kvm_vm_is_protected(&hyp_vm->kvm);
}
static inline void pkvm_hyp_vm_ref_inc(struct pkvm_hyp_vm *hyp_vm)
{
BUG_ON(hyp_vm->refcount == USHRT_MAX);
hyp_vm->refcount++;
}
static inline void pkvm_hyp_vm_ref_dec(struct pkvm_hyp_vm *hyp_vm)
{
BUG_ON(!hyp_vm->refcount);
hyp_vm->refcount--;
}
void pkvm_hyp_vm_table_init(void *tbl);
int __pkvm_reserve_vm(void);
void __pkvm_unreserve_vm(pkvm_handle_t handle);
int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
unsigned long pgd_hva);
int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
unsigned long vcpu_hva);
int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva);
int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu);
int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn);
int __pkvm_start_teardown_vm(pkvm_handle_t handle);
@ -92,6 +119,8 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
unsigned int vcpu_idx);
void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu);
struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void);
struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu);
void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm);
struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle);
struct pkvm_hyp_vm *get_np_pkvm_hyp_vm(pkvm_handle_t handle);
@ -101,6 +130,9 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code);
bool kvm_handle_pvm_sysreg(struct kvm_vcpu *vcpu, u64 *exit_code);
bool kvm_handle_pvm_restricted(struct kvm_vcpu *vcpu, u64 *exit_code);
void kvm_init_pvm_id_regs(struct kvm_vcpu *vcpu);
void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu);
int kvm_check_pvm_sysreg_table(void);
int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu);
struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *vm, unsigned long mpidr);
#endif /* __ARM64_KVM_NVHE_PKVM_H__ */

4
arch/arm64/kvm/hyp/include/nvhe/spinlock.h
View File

@ -17,6 +17,8 @@
#include <asm/lse.h>
#include <asm/rwonce.h>
#include <linux/cleanup.h>
typedef union hyp_spinlock {
u32 __val;
struct {
@ -122,4 +124,6 @@ static inline void hyp_assert_lock_held(hyp_spinlock_t *lock)
static inline void hyp_assert_lock_held(hyp_spinlock_t *lock) { }
#endif
DEFINE_LOCK_GUARD_1(hyp_spinlock, hyp_spinlock_t, hyp_spin_lock(_T->lock),
hyp_spin_unlock(_T->lock))
#endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */

71
arch/arm64/kvm/hyp/include/nvhe/trace.h
View File

@ -4,6 +4,7 @@
#include <linux/trace_remote_event.h>
#include <asm/kvm_hcall.h>
#include <asm/kvm_hyptrace.h>
static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt)
@ -46,7 +47,71 @@ static inline pid_t __tracing_get_vcpu_pid(struct kvm_cpu_context *host_ctxt)
void *tracing_reserve_entry(unsigned long length);
void tracing_commit_entry(void);
int __tracing_load(unsigned long desc_va, size_t desc_size);
/*
* The trace_hyp_printk boilerplate is too fiddly to be declared with
* HYP_EVENT():
*
* The string format is stored into a kernel-accessible ELF section. The
* hypervisor only writes the format ID.
*
* The function has a variadic prototype. We have no easy way to know each
* argument width so they must all cast to u64.
*/
#define REMOTE_EVENT_CUSTOM_PRINTK(...)
#define __TO_U64_0()
#define __TO_U64_1(x) , (u64)(x)
#define __TO_U64_2(x, ...) , (u64)(x) __TO_U64_1(__VA_ARGS__)
#define __TO_U64_3(x, ...) , (u64)(x) __TO_U64_2(__VA_ARGS__)
#define __TO_U64_4(x, ...) , (u64)(x) __TO_U64_3(__VA_ARGS__)
#define __TO_U64_5(x, ...) , (u64)(x) __TO_U64_4(__VA_ARGS__)
#define __TO_U64_6(x, ...) , (u64)(x) __TO_U64_5(__VA_ARGS__)
#define __TO_U64_7(x, ...) , (u64)(x) __TO_U64_6(__VA_ARGS__)
#define __TO_U64_8(x, ...) , (u64)(x) __TO_U64_7(__VA_ARGS__)
#define __TO_U64_X(N, ...) CONCATENATE(__TO_U64_, N)(__VA_ARGS__)
#define __TO_U64(...) __TO_U64_X(COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__)
REMOTE_EVENT_FORMAT(hyp_printk, HE_STRUCT(he_field(u16, fmt_id) he_field(u64, args[])));
extern struct hyp_event_id hyp_event_id_hyp_printk;
static __always_inline void __trace_hyp_printk(const struct hyp_string_fmt *fmt, int nr_args, ...)
{
struct remote_event_format_hyp_printk *entry;
va_list va;
int i;
if (!atomic_read(&hyp_event_id_hyp_printk.enabled))
return;
entry = tracing_reserve_entry(struct_size(entry, args, nr_args));
if (!entry)
return;
entry->hdr.id = hyp_event_id_hyp_printk.id;
entry->fmt_id = fmt - __hyp_string_fmts_start;
va_start(va, nr_args);
for (i = 0; i < nr_args; i++)
entry->args[i] = va_arg(va, u64);
va_end(va);
tracing_commit_entry();
}
#define trace_hyp_printk(__fmt, __args...) \
do { \
static const struct hyp_string_fmt __used __section("_hyp_string_fmts") \
__hyp_str_fmt = { \
.fmt = __fmt, \
.nr_args = COUNT_ARGS(__args), \
}; \
BUILD_BUG_ON(sizeof(__fmt) > HYP_STRING_FMT_MAX_SIZE); \
/* __TO_U64 prepends a comma if there are arguments */ \
__trace_hyp_printk(&__hyp_str_fmt, COUNT_ARGS(__args) __TO_U64(__args)); \
} while (0)
int __tracing_load(void __kern *desc_va, size_t desc_size);
void __tracing_unload(void);
int __tracing_enable(bool enable);
int __tracing_swap_reader(unsigned int cpu);
@ -58,8 +123,10 @@ static inline void *tracing_reserve_entry(unsigned long length) { return NULL; }
static inline void tracing_commit_entry(void) { }
#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
static inline void trace_##__name(__proto) {}
#define REMOTE_EVENT_CUSTOM_PRINTK(...)
#define trace_hyp_printk(fmt, args...) do { } while (0)
static inline int __tracing_load(unsigned long desc_va, size_t desc_size) { return -ENODEV; }
static inline int __tracing_load(void __kern *desc_va, size_t desc_size) { return -ENODEV; }
static inline void __tracing_unload(void) { }
static inline int __tracing_enable(bool enable) { return -ENODEV; }
static inline int __tracing_swap_reader(unsigned int cpu) { return -ENODEV; }

3
arch/arm64/kvm/hyp/nvhe/Makefile
View File

@ -23,7 +23,7 @@ lib-objs := $(addprefix ../../../lib/, $(lib-objs))
CFLAGS_switch.nvhe.o += -Wno-override-init
hyp-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
hyp-main.o hyp-smp.o psci-relay.o early_alloc.o page_alloc.o \
hyp-main.o hyp-smp.o psci-relay.o alloc.o early_alloc.o page_alloc.o \
cache.o setup.o mm.o mem_protect.o sys_regs.o pkvm.o stacktrace.o ffa.o
hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../hyp-entry.o ../exception.o ../pgtable.o ../vgic-v5-sr.o
@ -49,6 +49,7 @@ targets += $(hyp-obj) kvm_nvhe.tmp.o kvm_nvhe.rel.o hyp.lds hyp-reloc.S hyp-relo
# avoids file name clashes for files shared with VHE.
$(obj)/%.nvhe.o: $(src)/%.c FORCE
$(call if_changed_rule,cc_o_c)
$(call cmd,force_checksrc)
$(obj)/%.nvhe.o: $(src)/%.S FORCE
$(call if_changed_rule,as_o_S)

1212
arch/arm64/kvm/hyp/nvhe/alloc.c Normal file
View File

File diff suppressed because it is too large Load Diff

8
arch/arm64/kvm/hyp/nvhe/events.c
View File

@ -7,8 +7,16 @@
#include <nvhe/mm.h>
#include <nvhe/trace.h>
#include <asm/kvm_hypevents.h>
#include <nvhe/define_events.h>
/*
* The hyp_printk event is not declared with HYP_EVENT in kvm_hypevents.h,
* so we manually add the boilerplate here.
*/
HYP_EVENT(hyp_printk, 0, 0, 0, 0);
int __tracing_enable_event(unsigned short id, bool enable)
{
struct hyp_event_id *event_id = &__hyp_event_ids_start[id];

220
arch/arm64/kvm/hyp/nvhe/ffa.c
View File

@ -42,6 +42,9 @@
*/
#define HOST_FFA_ID 0
#define FFA_NOTIF_RECEIVER_ENDP_MASK GENMASK(15, 0)
#define FFA_NOTIF_SENDER_ENDP_MASK GENMASK(31, 16)
/*
* A buffer to hold the maximum descriptor size we can see from the host,
* which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP
@ -71,6 +74,21 @@ static u32 hyp_ffa_version;
static bool has_version_negotiated;
static hyp_spinlock_t version_lock;
static bool ffa_check_unused_args_sbz(struct kvm_cpu_context *ctxt, int first_reg)
{
DECLARE_REG(u32, func_id, ctxt, 0);
int reg, end_reg = 7;
if (hyp_ffa_version >= FFA_VERSION_1_2 && ARM_SMCCC_IS_64(func_id))
end_reg = 17;
for (reg = first_reg; reg <= end_reg; reg++) {
if (cpu_reg(ctxt, reg))
return true;
}
return false;
}
static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno)
{
*res = (struct arm_smccc_1_2_regs) {
@ -239,6 +257,11 @@ static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res,
int ret = 0;
void *rx_virt, *tx_virt;
if (ffa_check_unused_args_sbz(ctxt, 4)) {
ret = FFA_RET_INVALID_PARAMETERS;
goto out;
}
if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) {
ret = FFA_RET_INVALID_PARAMETERS;
goto out;
@ -315,6 +338,11 @@ static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res,
DECLARE_REG(u32, id, ctxt, 1);
int ret = 0;
if (ffa_check_unused_args_sbz(ctxt, 2)) {
ret = FFA_RET_INVALID_PARAMETERS;
goto out;
}
if (id != HOST_FFA_ID) {
ret = FFA_RET_INVALID_PARAMETERS;
goto out;
@ -421,6 +449,11 @@ static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res,
int ret = FFA_RET_INVALID_PARAMETERS;
u32 nr_ranges;
if (ffa_check_unused_args_sbz(ctxt, 5)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)
goto out;
@ -483,6 +516,11 @@ static void __do_ffa_mem_xfer(const u64 func_id,
int ret = 0;
size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version);
if (ffa_check_unused_args_sbz(ctxt, 5)) {
ret = FFA_RET_INVALID_PARAMETERS;
goto out;
}
if (addr_mbz || npages_mbz || fraglen > len ||
fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
ret = FFA_RET_INVALID_PARAMETERS;
@ -586,6 +624,11 @@ static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res,
int ret = 0;
u64 handle;
if (ffa_check_unused_args_sbz(ctxt, 4)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
handle = PACK_HANDLE(handle_lo, handle_hi);
hyp_spin_lock(&host_buffers.lock);
@ -691,14 +734,6 @@ static bool ffa_call_supported(u64 func_id)
case FFA_RXTX_MAP:
case FFA_MEM_DONATE:
case FFA_MEM_RETRIEVE_REQ:
/* Optional notification interfaces added in FF-A 1.1 */
case FFA_NOTIFICATION_BITMAP_CREATE:
case FFA_NOTIFICATION_BITMAP_DESTROY:
case FFA_NOTIFICATION_BIND:
case FFA_NOTIFICATION_UNBIND:
case FFA_NOTIFICATION_SET:
case FFA_NOTIFICATION_GET:
case FFA_NOTIFICATION_INFO_GET:
/* Optional interfaces added in FF-A 1.2 */
case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */
case FFA_MSG_SEND_DIRECT_RESP2: /* Optional per 7.5.1 */
@ -785,6 +820,11 @@ static void do_ffa_version(struct arm_smccc_1_2_regs *res,
{
DECLARE_REG(u32, ffa_req_version, ctxt, 1);
if (ffa_check_unused_args_sbz(ctxt, 2)) {
res->a0 = FFA_RET_NOT_SUPPORTED;
return;
}
if (FFA_MAJOR_VERSION(ffa_req_version) != 1) {
res->a0 = FFA_RET_NOT_SUPPORTED;
return;
@ -834,6 +874,11 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
DECLARE_REG(u32, flags, ctxt, 5);
u32 count, partition_sz, copy_sz;
if (ffa_check_unused_args_sbz(ctxt, 6)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
hyp_spin_lock(&host_buffers.lock);
if (!host_buffers.rx) {
ffa_to_smccc_res(res, FFA_RET_BUSY);
@ -878,6 +923,140 @@ static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
hyp_spin_unlock(&host_buffers.lock);
}
static void do_ffa_notif_bitmap(struct arm_smccc_1_2_regs *res,
struct kvm_cpu_context *ctxt)
{
DECLARE_REG(u32, func_id, ctxt, 0);
DECLARE_REG(u32, vmid, ctxt, 1);
struct arm_smccc_1_2_regs *args;
if (ffa_check_unused_args_sbz(ctxt, func_id == FFA_NOTIFICATION_BITMAP_CREATE ? 3 : 2)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (vmid != HOST_FFA_ID) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
args = (void *)&ctxt->regs.regs[0];
hyp_smccc_1_2_smc(args, res);
}
static void do_ffa_notif_bind(struct arm_smccc_1_2_regs *res,
struct kvm_cpu_context *ctxt)
{
DECLARE_REG(u32, endp_id, ctxt, 1);
DECLARE_REG(u32, flags, ctxt, 2);
struct arm_smccc_1_2_regs *args;
if (ffa_check_unused_args_sbz(ctxt, 5)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (flags > 1) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
args = (void *)&ctxt->regs.regs[0];
hyp_smccc_1_2_smc(args, res);
}
static void do_ffa_notif_unbind(struct arm_smccc_1_2_regs *res,
struct kvm_cpu_context *ctxt)
{
DECLARE_REG(u32, endp_id, ctxt, 1);
DECLARE_REG(u32, reserved, ctxt, 2);
struct arm_smccc_1_2_regs *args;
if (ffa_check_unused_args_sbz(ctxt, 5) || reserved) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
args = (void *)&ctxt->regs.regs[0];
hyp_smccc_1_2_smc(args, res);
}
static void do_ffa_notif_set(struct arm_smccc_1_2_regs *res,
struct kvm_cpu_context *ctxt)
{
DECLARE_REG(u32, endp_id, ctxt, 1);
DECLARE_REG(u32, flags, ctxt, 2);
struct arm_smccc_1_2_regs *args;
if (FIELD_GET(FFA_NOTIF_SENDER_ENDP_MASK, endp_id) != HOST_FFA_ID) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (ffa_check_unused_args_sbz(ctxt, 5)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (flags & GENMASK(15, 2)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
args = (void *)&ctxt->regs.regs[0];
hyp_smccc_1_2_smc(args, res);
}
static void do_ffa_notif_get(struct arm_smccc_1_2_regs *res,
struct kvm_cpu_context *ctxt)
{
DECLARE_REG(u32, endp_id, ctxt, 1);
DECLARE_REG(u32, flags, ctxt, 2);
struct arm_smccc_1_2_regs *args;
if (FIELD_GET(FFA_NOTIF_RECEIVER_ENDP_MASK, endp_id) != HOST_FFA_ID) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (ffa_check_unused_args_sbz(ctxt, 3)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
if (flags & GENMASK(31, 4)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
args = (void *)&ctxt->regs.regs[0];
hyp_smccc_1_2_smc(args, res);
}
static void do_ffa_notif_info_get(struct arm_smccc_1_2_regs *res,
struct kvm_cpu_context *ctxt)
{
struct arm_smccc_1_2_regs *args;
if (ffa_check_unused_args_sbz(ctxt, 1)) {
ffa_to_smccc_res(res, FFA_RET_INVALID_PARAMETERS);
return;
}
args = (void *)&ctxt->regs.regs[0];
hyp_smccc_1_2_smc(args, res);
}
bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
{
struct arm_smccc_1_2_regs res = {0};
@ -906,6 +1085,11 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
switch (func_id) {
case FFA_FEATURES:
if (ffa_check_unused_args_sbz(host_ctxt, 3)) {
ffa_to_smccc_res(&res, FFA_RET_INVALID_PARAMETERS);
goto out_handled;
}
if (!do_ffa_features(&res, host_ctxt))
return false;
goto out_handled;
@ -936,6 +1120,26 @@ bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
case FFA_PARTITION_INFO_GET:
do_ffa_part_get(&res, host_ctxt);
goto out_handled;
case FFA_NOTIFICATION_BITMAP_CREATE:
case FFA_NOTIFICATION_BITMAP_DESTROY:
do_ffa_notif_bitmap(&res, host_ctxt);
goto out_handled;
case FFA_NOTIFICATION_BIND:
do_ffa_notif_bind(&res, host_ctxt);
goto out_handled;
case FFA_NOTIFICATION_UNBIND:
do_ffa_notif_unbind(&res, host_ctxt);
goto out_handled;
case FFA_NOTIFICATION_SET:
do_ffa_notif_set(&res, host_ctxt);
goto out_handled;
case FFA_NOTIFICATION_GET:
do_ffa_notif_get(&res, host_ctxt);
goto out_handled;
case FFA_NOTIFICATION_INFO_GET:
case FFA_FN64_NOTIFICATION_INFO_GET:
do_ffa_notif_info_get(&res, host_ctxt);
goto out_handled;
}
if (ffa_call_supported(func_id))

1173
arch/arm64/kvm/hyp/nvhe/hyp-main.c
View File

File diff suppressed because it is too large Load Diff

1
arch/arm64/kvm/hyp/nvhe/hyp.lds.S
View File

@ -17,7 +17,6 @@ SECTIONS {
HYP_SECTION(.data..ro_after_init)
HYP_SECTION(.rodata)
#ifdef CONFIG_NVHE_EL2_TRACING
. = ALIGN(PAGE_SIZE);
BEGIN_HYP_SECTION(.event_ids)
*(SORT(.hyp.event_ids.*))
END_HYP_SECTION

20
arch/arm64/kvm/hyp/nvhe/mem_protect.c
View File

@ -540,7 +540,7 @@ static int host_stage2_adjust_range(u64 addr, struct kvm_mem_range *range)
int ret;
hyp_assert_lock_held(&host_mmu.lock);
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level);
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, addr, &pte, &level, 0);
if (ret)
return ret;
@ -930,7 +930,7 @@ static int get_valid_guest_pte(struct pkvm_hyp_vm *vm, u64 ipa, kvm_pte_t *ptep,
s8 level;
int ret;
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
if (ret)
return ret;
if (guest_pte_is_poisoned(pte))
@ -979,7 +979,7 @@ int __pkvm_vcpu_in_poison_fault(struct pkvm_hyp_vcpu *hyp_vcpu)
ipa |= FAR_TO_FIPA_OFFSET(kvm_vcpu_get_hfar(&hyp_vcpu->vcpu));
guest_lock_component(vm);
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
if (ret)
goto unlock;
@ -1043,7 +1043,7 @@ int __pkvm_guest_share_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn)
ret = 0;
WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys,
pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_SHARED_OWNED),
&vcpu->vcpu.arch.pkvm_memcache, 0));
&vcpu->vcpu.arch.stage2_mc, 0));
WARN_ON(__host_set_page_state_range(phys, PAGE_SIZE, PKVM_PAGE_SHARED_BORROWED));
unlock:
guest_unlock_component(vm);
@ -1078,7 +1078,7 @@ int __pkvm_guest_unshare_host(struct pkvm_hyp_vcpu *vcpu, u64 gfn)
PKVM_ID_GUEST, meta));
WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys,
pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED),
&vcpu->vcpu.arch.pkvm_memcache, 0));
&vcpu->vcpu.arch.stage2_mc, 0));
unlock:
guest_unlock_component(vm);
host_unlock_component();
@ -1335,7 +1335,7 @@ static int host_stage2_get_guest_info(phys_addr_t phys, struct pkvm_hyp_vm **vm,
return -EPERM;
}
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level);
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, &level, 0);
if (ret)
return ret;
@ -1442,7 +1442,7 @@ static int __guest_check_pgtable_memcache(struct pkvm_hyp_vcpu *vcpu)
{
struct pkvm_hyp_vm *vm = pkvm_hyp_vcpu_to_hyp_vm(vcpu);
if (vcpu->vcpu.arch.pkvm_memcache.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu))
if (vcpu->vcpu.arch.stage2_mc.nr_pages < kvm_mmu_cache_min_pages(vm->pgt.mmu))
return -ENOMEM;
return 0;
@ -1476,7 +1476,7 @@ int __pkvm_host_donate_guest(u64 pfn, u64 gfn, struct pkvm_hyp_vcpu *vcpu)
PKVM_ID_GUEST, meta));
WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, PAGE_SIZE, phys,
pkvm_mkstate(KVM_PGTABLE_PROT_RWX, PKVM_PAGE_OWNED),
&vcpu->vcpu.arch.pkvm_memcache, 0));
&vcpu->vcpu.arch.stage2_mc, 0));
unlock:
guest_unlock_component(vm);
@ -1547,7 +1547,7 @@ int __pkvm_host_share_guest(u64 pfn, u64 gfn, u64 nr_pages, struct pkvm_hyp_vcpu
WARN_ON(kvm_pgtable_stage2_map(&vm->pgt, ipa, size, phys,
pkvm_mkstate(prot, PKVM_PAGE_SHARED_BORROWED),
&vcpu->vcpu.arch.pkvm_memcache, 0));
&vcpu->vcpu.arch.stage2_mc, 0));
unlock:
guest_unlock_component(vm);
@ -1564,7 +1564,7 @@ static int __check_host_shared_guest(struct pkvm_hyp_vm *vm, u64 *__phys, u64 ip
s8 level;
int ret;
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level);
ret = kvm_pgtable_get_leaf(&vm->pgt, ipa, &pte, &level, 0);
if (ret)
return ret;
if (!kvm_pte_valid(pte))

79
arch/arm64/kvm/hyp/nvhe/mm.c
View File

@ -91,6 +91,59 @@ int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr)
return ret;
}
/**
* pkvm_map_private_va_range() - Map a physical range into the private VA range
* @haddr: The virtual address in the private range.
* @phys: The physical address to map.
* @size: The size of the range to map.
*
* The hypervisor VA @haddr must have been first allocated with
* pkvm_alloc_private_va_range(). The created mapping can later be removed
* with pkvm_remove_mappings().
*
* Return: 0 on success, negative error code on failure.
*/
int pkvm_map_private_va_range(void *haddr, phys_addr_t phys, size_t size)
{
unsigned long addr = (unsigned long)haddr;
int ret;
if (!PAGE_ALIGNED(addr | phys | size))
return -EINVAL;
guard(hyp_spinlock)(&pkvm_pgd_lock);
if (addr < __io_map_base || addr >= __io_map_next)
return -EINVAL;
if (size > __io_map_next - addr)
return -EINVAL;
ret = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, PAGE_HYP);
if (ret)
kvm_pgtable_hyp_unmap(&pkvm_pgtable, addr, size);
return ret;
}
/**
* pkvm_private_va_range_pa - Translate a private VA to a physical address.
* @va: The private virtual address to translate.
*
* Return: The physical address corresponding to @va.
*/
phys_addr_t pkvm_private_va_range_pa(void *va)
{
kvm_pte_t pte;
s8 level;
guard(hyp_spinlock)(&pkvm_pgd_lock);
WARN_ON(kvm_pgtable_get_leaf(&pkvm_pgtable, (u64)va, &pte, &level, 0));
WARN_ON(!kvm_pte_valid(pte));
return kvm_pte_to_phys(pte) + ((u64)va & (kvm_granule_size(level) - 1));
}
int __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
enum kvm_pgtable_prot prot,
unsigned long *haddr)
@ -147,6 +200,30 @@ int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
return ret;
}
/**
* pkvm_remove_mappings - Remove mappings from the hypervisor page-table
* @from: The starting virtual address of the range to remove
* @to: The ending virtual address of the range to remove
*
* The range [@from, @to) can be a subset of the range previously mapped by
* pkvm_create_mappings(), as the latter enforces PTE-level mappings. However,
* it cannot be a subset of a range mapped by pkvm_map_private_va_range() as
* that function allows block mappings which the hypervisor page-table does not
* split.
*/
void pkvm_remove_mappings(void *from, void *to)
{
u64 size;
WARN_ON(from > to);
to = PTR_ALIGN(to, PAGE_SIZE);
from = PTR_ALIGN_DOWN(from, PAGE_SIZE);
size = (u64)to - (u64)from;
guard(hyp_spinlock)(&pkvm_pgd_lock);
WARN_ON(kvm_pgtable_hyp_unmap(&pkvm_pgtable, (u64)from, size) != size);
}
int hyp_back_vmemmap(phys_addr_t back)
{
unsigned long i, start, size, end = 0;
@ -486,7 +563,7 @@ static int check_page_ownership(phys_addr_t phys)
return -EPERM;
}
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL);
ret = kvm_pgtable_get_leaf(&host_mmu.pgt, phys, &pte, NULL, 0);
if (ret)
return ret;

507
arch/arm64/kvm/hyp/nvhe/pkvm.c
View File

@ -5,17 +5,25 @@
*/
#include <kvm/arm_hypercalls.h>
#include <kvm/arm_psci.h>
#include <linux/kvm_host.h>
#include <linux/mm.h>
#include <asm/kvm_emulate.h>
#include <asm/spectre.h>
#include <nvhe/alloc.h>
#include <nvhe/mem_protect.h>
#include <nvhe/memory.h>
#include <nvhe/pkvm.h>
#include <nvhe/trap_handler.h>
/* The host's Spectre mitigation state, for SMCCC_ARCH_FEATURES. */
enum mitigation_state spectre_v2_state;
enum mitigation_state spectre_v4_state;
enum mitigation_state spectre_bhb_state;
/* Used by icache_is_aliasing(). */
unsigned long __icache_flags;
@ -109,7 +117,7 @@ static void pvm_init_traps_mdcr(struct kvm_vcpu *vcpu)
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP)) {
val |= MDCR_EL2_TPM | MDCR_EL2_TPMCR;
val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME | MDCR_EL2_HPMN_MASK);
val &= ~(MDCR_EL2_HPME | MDCR_EL2_MTPME);
}
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, IMP))
@ -285,7 +293,7 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle,
}
hyp_vcpu->loaded_hyp_vcpu = this_cpu_ptr(&loaded_hyp_vcpu);
hyp_page_ref_inc(hyp_virt_to_page(hyp_vm));
pkvm_hyp_vm_ref_inc(hyp_vm);
unlock:
hyp_spin_unlock(&vm_table_lock);
@ -301,7 +309,7 @@ void pkvm_put_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
hyp_spin_lock(&vm_table_lock);
hyp_vcpu->loaded_hyp_vcpu = NULL;
__this_cpu_write(loaded_hyp_vcpu, NULL);
hyp_page_ref_dec(hyp_virt_to_page(hyp_vm));
pkvm_hyp_vm_ref_dec(hyp_vm);
hyp_spin_unlock(&vm_table_lock);
}
@ -311,6 +319,43 @@ struct pkvm_hyp_vcpu *pkvm_get_loaded_hyp_vcpu(void)
}
static struct pkvm_hyp_vm *pkvm_get_loaded_hyp_vm(struct kvm_vcpu *vcpu)
{
struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
if (hyp_vcpu &&
(vcpu == &hyp_vcpu->vcpu || vcpu == hyp_vcpu->host_vcpu))
return pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
return NULL;
}
/*
* A loaded vCPU's VM is the hyp VM. An unloaded host vCPU's is mapped at
* EL2 only while pinned, so it's read once and pinned; the pin fails for
* memory the host isn't sharing.
*/
struct kvm *pkvm_vcpu_get_kvm(struct kvm_vcpu *vcpu)
{
struct pkvm_hyp_vm *hyp_vm = pkvm_get_loaded_hyp_vm(vcpu);
struct kvm *kvm;
if (hyp_vm)
return &hyp_vm->kvm;
kvm = kern_hyp_va(READ_ONCE(vcpu->kvm));
if (hyp_pin_shared_mem(kvm, kvm + 1))
return NULL;
return kvm;
}
void pkvm_vcpu_put_kvm(struct kvm_vcpu *vcpu, struct kvm *kvm)
{
if (kvm && !pkvm_get_loaded_hyp_vm(vcpu))
hyp_unpin_shared_mem(kvm, kvm + 1);
}
struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle)
{
struct pkvm_hyp_vm *hyp_vm;
@ -318,7 +363,7 @@ struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle)
hyp_spin_lock(&vm_table_lock);
hyp_vm = get_vm_by_handle(handle);
if (hyp_vm)
hyp_page_ref_inc(hyp_virt_to_page(hyp_vm));
pkvm_hyp_vm_ref_inc(hyp_vm);
hyp_spin_unlock(&vm_table_lock);
return hyp_vm;
@ -327,7 +372,7 @@ struct pkvm_hyp_vm *get_pkvm_hyp_vm(pkvm_handle_t handle)
void put_pkvm_hyp_vm(struct pkvm_hyp_vm *hyp_vm)
{
hyp_spin_lock(&vm_table_lock);
hyp_page_ref_dec(hyp_virt_to_page(hyp_vm));
pkvm_hyp_vm_ref_dec(hyp_vm);
hyp_spin_unlock(&vm_table_lock);
}
@ -373,21 +418,7 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_MTE))
kvm->arch.flags |= host_arch_flags & BIT(KVM_ARCH_FLAG_MTE_ENABLED);
bitmap_zero(allowed_features, KVM_VCPU_MAX_FEATURES);
set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed_features);
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3))
set_bit(KVM_ARM_VCPU_PMU_V3, allowed_features);
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS))
set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed_features);
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC))
set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed_features);
if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE))
set_bit(KVM_ARM_VCPU_SVE, allowed_features);
kvm_pkvm_vcpu_allowed_features(kvm, allowed_features);
bitmap_and(kvm->arch.vcpu_features, host_kvm->arch.vcpu_features,
allowed_features, KVM_VCPU_MAX_FEATURES);
@ -396,6 +427,40 @@ static void pkvm_init_features_from_host(struct pkvm_hyp_vm *hyp_vm, const struc
kvm_vcpu_has_feature(kvm, KVM_ARM_VCPU_SVE));
}
static int pkvm_vcpu_init_psci(struct pkvm_hyp_vcpu *hyp_vcpu, u32 mp_state)
{
struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state;
struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
struct kvm_vcpu *host_vcpu;
if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) {
/* The host manages a non-protected vCPU: always ON at EL2. */
hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON;
return 0;
}
if (mp_state != KVM_MP_STATE_RUNNABLE && mp_state != KVM_MP_STATE_STOPPED)
return -EINVAL;
if (mp_state == KVM_MP_STATE_STOPPED) {
reset_state->reset = false;
hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_OFF;
return 0;
}
hyp_assert_lock_held(&vm_table_lock);
if (hyp_vm->primary_vcpu)
return -EINVAL;
hyp_vm->primary_vcpu = hyp_vcpu;
host_vcpu = hyp_vcpu->host_vcpu;
reset_state->pc = READ_ONCE(*vcpu_pc(host_vcpu));
reset_state->r0 = READ_ONCE(vcpu_gp_regs(host_vcpu)[0]);
reset_state->reset = true;
hyp_vcpu->power_state = PSCI_0_2_AFFINITY_LEVEL_ON_PENDING;
return 0;
}
static void unpin_host_vcpu(struct kvm_vcpu *host_vcpu)
{
if (host_vcpu)
@ -440,7 +505,11 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
hyp_vm->host_kvm = host_kvm;
hyp_vm->kvm.created_vcpus = nr_vcpus;
hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected);
if (READ_ONCE(host_kvm->arch.vm_flavor) == VM_PROTECTED_PKVM)
hyp_vm->kvm.arch.vm_flavor = VM_PROTECTED_PKVM;
else
hyp_vm->kvm.arch.vm_flavor = VM_PKVM;
hyp_vm->kvm.arch.flags = 0;
pkvm_init_features_from_host(hyp_vm, host_kvm);
@ -518,10 +587,12 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu,
struct kvm_vcpu *host_vcpu)
{
int ret = 0;
u32 mp_state;
if (hyp_pin_shared_mem(host_vcpu, host_vcpu + 1))
return -EBUSY;
mp_state = READ_ONCE(host_vcpu->arch.mp_state.mp_state);
hyp_vcpu->host_vcpu = host_vcpu;
hyp_vcpu->vcpu.kvm = &hyp_vm->kvm;
@ -530,7 +601,6 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu,
hyp_vcpu->vcpu.arch.hw_mmu = &hyp_vm->kvm.arch.mmu;
hyp_vcpu->vcpu.arch.cflags = READ_ONCE(host_vcpu->arch.cflags);
hyp_vcpu->vcpu.arch.mp_state.mp_state = KVM_MP_STATE_STOPPED;
if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) {
/*
@ -555,9 +625,17 @@ static int init_pkvm_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu,
goto done;
ret = pkvm_vcpu_init_sve(hyp_vcpu, host_vcpu);
done:
if (ret)
goto done;
if (pkvm_hyp_vcpu_is_protected(hyp_vcpu))
kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu);
ret = pkvm_vcpu_init_psci(hyp_vcpu, mp_state);
done:
if (ret) {
unpin_host_vcpu(host_vcpu);
unpin_host_sve_state(hyp_vcpu);
}
return ret;
}
@ -661,9 +739,9 @@ static size_t pkvm_get_hyp_vm_size(unsigned int nr_vcpus)
size_mul(sizeof(struct pkvm_hyp_vcpu *), nr_vcpus));
}
static void *map_donated_memory_noclear(unsigned long host_va, size_t size)
static void *map_donated_memory_noclear(void __kern *host_va, size_t size)
{
void *va = (void *)kern_hyp_va(host_va);
void *va = kern_hyp_va_host(host_va);
if (!PAGE_ALIGNED(va))
return NULL;
@ -675,16 +753,6 @@ static void *map_donated_memory_noclear(unsigned long host_va, size_t size)
return va;
}
static void *map_donated_memory(unsigned long host_va, size_t size)
{
void *va = map_donated_memory_noclear(host_va, size);
if (va)
memset(va, 0, size);
return va;
}
static void __unmap_donated_memory(void *va, size_t size)
{
kvm_flush_dcache_to_poc(va, size);
@ -787,7 +855,7 @@ struct pkvm_hyp_vcpu *init_selftest_vm(void *virt)
continue;
p[i].refcount = 1;
if (seeded < min_pages) {
push_hyp_memcache(&selftest_vcpu.vcpu.arch.pkvm_memcache,
push_hyp_memcache(&selftest_vcpu.vcpu.arch.stage2_mc,
hyp_page_to_virt(&p[i]), hyp_virt_to_phys);
seeded++;
} else {
@ -814,16 +882,13 @@ void teardown_selftest_vm(void)
* Unmap the donated memory from the host at stage 2.
*
* host_kvm: A pointer to the host's struct kvm.
* vm_hva: The host va of the area being donated for the VM state.
* Must be page aligned.
* pgd_hva: The host va of the area being donated for the stage-2 PGD for
* the VM. Must be page aligned. Its size is implied by the VM's
* VTCR.
*
* Return 0 success, negative error code on failure.
*/
int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
unsigned long pgd_hva)
int __pkvm_init_vm(struct kvm *host_kvm, void __kern *pgd_hva)
{
struct pkvm_hyp_vm *hyp_vm = NULL;
size_t vm_size, pgd_size;
@ -851,15 +916,17 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
vm_size = pkvm_get_hyp_vm_size(nr_vcpus);
pgd_size = kvm_pgtable_stage2_pgd_size(host_mmu.arch.mmu.vtcr);
ret = -ENOMEM;
hyp_vm = hyp_alloc(vm_size);
if (!hyp_vm) {
ret = hyp_alloc_errno();
goto err_unpin_kvm;
}
hyp_vm = map_donated_memory(vm_hva, vm_size);
if (!hyp_vm)
goto err_remove_mappings;
ret = -ENOMEM;
pgd = map_donated_memory_noclear(pgd_hva, pgd_size);
if (!pgd)
goto err_remove_mappings;
goto err_free_hyp_vm;
init_pkvm_hyp_vm(host_kvm, hyp_vm, nr_vcpus, handle);
@ -877,8 +944,9 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva,
err_destroy_stage2:
kvm_guest_destroy_stage2(hyp_vm);
err_remove_mappings:
unmap_donated_memory(hyp_vm, vm_size);
unmap_donated_memory(pgd, pgd_size);
err_free_hyp_vm:
hyp_free(hyp_vm);
err_unpin_kvm:
hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
return ret;
@ -913,16 +981,15 @@ static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm,
return 0;
}
int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
unsigned long vcpu_hva)
int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu)
{
struct pkvm_hyp_vcpu *hyp_vcpu;
struct pkvm_hyp_vm *hyp_vm;
int ret;
hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu));
hyp_vcpu = hyp_alloc(sizeof(*hyp_vcpu));
if (!hyp_vcpu)
return -ENOMEM;
return hyp_alloc_errno();
hyp_spin_lock(&vm_table_lock);
@ -934,33 +1001,30 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu,
ret = init_pkvm_hyp_vcpu(hyp_vcpu, hyp_vm, host_vcpu);
if (ret)
goto unlock;
goto unclaim;
ret = register_hyp_vcpu(hyp_vm, hyp_vcpu);
if (ret) {
unpin_host_vcpu(host_vcpu);
unpin_host_sve_state(hyp_vcpu);
goto unclaim;
}
goto unlock;
unclaim:
/*
* Under vm_table_lock, so no other claim can have landed: undo
* this one.
*/
if (hyp_vm->primary_vcpu == hyp_vcpu)
hyp_vm->primary_vcpu = NULL;
unlock:
hyp_spin_unlock(&vm_table_lock);
if (ret)
unmap_donated_memory(hyp_vcpu, sizeof(*hyp_vcpu));
hyp_free(hyp_vcpu);
return ret;
}
static void
teardown_donated_memory(struct kvm_hyp_memcache *mc, void *addr, size_t size)
{
size = PAGE_ALIGN(size);
memset(addr, 0, size);
for (void *start = addr; start < addr + size; start += PAGE_SIZE)
push_hyp_memcache(mc, start, hyp_virt_to_phys);
unmap_donated_memory_noclear(addr, size);
}
int __pkvm_reclaim_dying_guest_page(pkvm_handle_t handle, u64 gfn)
{
struct pkvm_hyp_vm *hyp_vm = get_pkvm_hyp_vm(handle);
@ -983,7 +1047,7 @@ static struct pkvm_hyp_vm *get_pkvm_unref_hyp_vm_locked(pkvm_handle_t handle)
hyp_assert_lock_held(&vm_table_lock);
hyp_vm = get_vm_by_handle(handle);
if (!hyp_vm || hyp_page_count(hyp_vm))
if (!hyp_vm || hyp_vm->refcount)
return NULL;
return hyp_vm;
@ -1010,11 +1074,10 @@ int __pkvm_start_teardown_vm(pkvm_handle_t handle)
int __pkvm_finalize_teardown_vm(pkvm_handle_t handle)
{
struct kvm_hyp_memcache *mc, *stage2_mc;
struct kvm_hyp_memcache *stage2_mc;
struct pkvm_hyp_vm *hyp_vm;
struct kvm *host_kvm;
unsigned int idx;
size_t vm_size;
int err;
hyp_spin_lock(&vm_table_lock);
@ -1032,12 +1095,11 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle)
hyp_spin_unlock(&vm_table_lock);
/* Reclaim guest pages (including page-table pages) */
mc = &host_kvm->arch.pkvm.teardown_mc;
stage2_mc = &host_kvm->arch.pkvm.stage2_teardown_mc;
reclaim_pgtable_pages(hyp_vm, stage2_mc);
unpin_host_vcpus(hyp_vm->vcpus, hyp_vm->kvm.created_vcpus);
/* Push the metadata pages to the teardown memcache */
/* Push the stage-2 pages to the teardown memcache */
for (idx = 0; idx < hyp_vm->kvm.created_vcpus; ++idx) {
struct pkvm_hyp_vcpu *hyp_vcpu = hyp_vm->vcpus[idx];
struct kvm_hyp_memcache *vcpu_mc;
@ -1045,7 +1107,7 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle)
if (!hyp_vcpu)
continue;
vcpu_mc = &hyp_vcpu->vcpu.arch.pkvm_memcache;
vcpu_mc = &hyp_vcpu->vcpu.arch.stage2_mc;
while (vcpu_mc->nr_pages) {
void *addr = pop_hyp_memcache(vcpu_mc, hyp_phys_to_virt);
@ -1054,11 +1116,10 @@ int __pkvm_finalize_teardown_vm(pkvm_handle_t handle)
unmap_donated_memory_noclear(addr, PAGE_SIZE);
}
teardown_donated_memory(mc, hyp_vcpu, sizeof(*hyp_vcpu));
hyp_free(hyp_vcpu);
}
vm_size = pkvm_get_hyp_vm_size(hyp_vm->kvm.created_vcpus);
teardown_donated_memory(mc, hyp_vm, vm_size);
hyp_free(hyp_vm);
hyp_unpin_shared_mem(host_kvm, host_kvm + 1);
return 0;
@ -1132,6 +1193,238 @@ static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu)
ret[0] = SMCCC_RET_SUCCESS;
}
/*
* Reset the vCPU to its power-on state and commit ON_PENDING -> ON, on the
* target's own CPU. Returns -ECANCELED, with no side effects, if a rollback
* raced the reset.
*
* The requestor's PSCI CPU_ON, in pvm_psci_vcpu_on():
*
* cmpxchg_relaxed(power_state): OFF -> ON_PENDING
* <ctrl>
* write_reset_state();
* smp_store_release(reset_state->reset, true);
*
* The target coming online here, running, then its PSCI CPU_OFF in
* pvm_psci_vcpu_off():
*
* smp_load_acquire(reset_state->reset) == true;
* cmpxchg_relaxed(power_state): ON_PENDING -> ON;
* read_reset_state();
*
* <run guest>
*
* WRITE_ONCE(reset_state->reset, false);
* smp_store_release(power_state, OFF);
*/
int pkvm_reset_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
{
struct vcpu_reset_state *reset_state = &hyp_vcpu->vcpu.arch.reset_state;
/*
* Pairs with smp_store_release(&reset_state->reset, true) in
* pvm_psci_vcpu_on(). The acquire must precede the cmpxchg: reversed, a
* winning cmpxchg with a false acquire would leave power_state == ON
* with the reset skipped.
*/
if (!smp_load_acquire(&reset_state->reset))
return -ECANCELED;
if (cmpxchg_relaxed(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_ON_PENDING,
PSCI_0_2_AFFINITY_LEVEL_ON) != PSCI_0_2_AFFINITY_LEVEL_ON_PENDING)
return -ECANCELED;
kvm_reset_vcpu_core(&hyp_vcpu->vcpu);
kvm_reset_pvm_sys_regs(&hyp_vcpu->vcpu);
/* Must be done after resetting sys registers. */
kvm_reset_vcpu_psci(&hyp_vcpu->vcpu, reset_state);
hyp_vcpu->exit_code = 0;
return 0;
}
struct pkvm_hyp_vcpu *pkvm_mpidr_to_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, unsigned long mpidr)
{
struct pkvm_hyp_vcpu *hyp_vcpu;
int i;
mpidr &= MPIDR_HWID_BITMASK;
for (i = 0; i < hyp_vm->kvm.created_vcpus; i++) {
/* Pairs with smp_store_release() in register_hyp_vcpu(). */
hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[i]);
if (!hyp_vcpu)
continue;
if (mpidr == kvm_vcpu_get_mpidr_aff(&hyp_vcpu->vcpu))
return hyp_vcpu;
}
return NULL;
}
/*
* Returns true when handled at EL2, false when the host must wake the target
* vCPU.
*/
static bool pvm_psci_vcpu_on(struct pkvm_hyp_vcpu *hyp_vcpu)
{
struct pkvm_hyp_vm *hyp_vm = pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu);
struct vcpu_reset_state *reset_state;
struct pkvm_hyp_vcpu *target;
unsigned long cpu_id, ret;
int power_state;
cpu_id = smccc_get_arg1(&hyp_vcpu->vcpu);
if (!kvm_psci_valid_affinity(&hyp_vcpu->vcpu, cpu_id)) {
ret = PSCI_RET_INVALID_PARAMS;
goto error;
}
target = pkvm_mpidr_to_hyp_vcpu(hyp_vm, cpu_id);
if (!target) {
ret = PSCI_RET_INVALID_PARAMS;
goto error;
}
/*
* vCPUs race to power on the same target. The writes below depend on
* the cmpxchg reading OFF, and that control dependency with the release
* of OFF in pvm_psci_vcpu_off() orders them after the target's accesses
* to reset_state, so relaxed suffices.
*/
power_state = cmpxchg_relaxed(&target->power_state,
PSCI_0_2_AFFINITY_LEVEL_OFF,
PSCI_0_2_AFFINITY_LEVEL_ON_PENDING);
switch (power_state) {
case PSCI_0_2_AFFINITY_LEVEL_ON_PENDING:
ret = PSCI_RET_ON_PENDING;
goto error;
case PSCI_0_2_AFFINITY_LEVEL_ON:
ret = PSCI_RET_ALREADY_ON;
goto error;
case PSCI_0_2_AFFINITY_LEVEL_OFF:
break;
default:
ret = PSCI_RET_INTERNAL_FAILURE;
goto error;
}
reset_state = &target->vcpu.arch.reset_state;
WRITE_ONCE(reset_state->pc, smccc_get_arg2(&hyp_vcpu->vcpu));
WRITE_ONCE(reset_state->r0, smccc_get_arg3(&hyp_vcpu->vcpu));
WRITE_ONCE(reset_state->be, kvm_vcpu_is_be(&hyp_vcpu->vcpu));
/*
* Publish reset_state.{pc, r0, be} to the target vCPU. Pairs with
* smp_load_acquire(&reset_state->reset) in pkvm_reset_vcpu().
*/
smp_store_release(&reset_state->reset, true);
/* The host requests KVM_REQ_VCPU_RESET and wakes the target. */
return false;
error:
smccc_set_retval(&hyp_vcpu->vcpu, ret, 0, 0, 0);
return true;
}
/*
* Returns true when handled at EL2, false when the host must stop scheduling
* the vCPU.
*/
static bool pvm_psci_vcpu_off(struct pkvm_hyp_vcpu *hyp_vcpu)
{
/* No other writer runs while this vCPU is ON and executing. */
WARN_ON(READ_ONCE(hyp_vcpu->power_state) != PSCI_0_2_AFFINITY_LEVEL_ON);
WRITE_ONCE(hyp_vcpu->vcpu.arch.reset_state.reset, false);
/*
* Orders pkvm_reset_vcpu()'s accesses to reset_state, and the clear
* above, before OFF. Pairs with the control dependency in
* pvm_psci_vcpu_on().
*/
smp_store_release(&hyp_vcpu->power_state, PSCI_0_2_AFFINITY_LEVEL_OFF);
/* Return to the host so that it can finish powering off the vcpu. */
return false;
}
static bool pvm_psci_version(struct pkvm_hyp_vcpu *hyp_vcpu)
{
/* Nothing to be handled by the host. Go back to the guest. */
smccc_set_retval(&hyp_vcpu->vcpu, KVM_ARM_PSCI_1_1, 0, 0, 0);
return true;
}
static bool pvm_psci_features(struct pkvm_hyp_vcpu *hyp_vcpu)
{
struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
u32 feature = smccc_get_arg1(vcpu);
unsigned long val;
switch (feature) {
case PSCI_0_2_FN_PSCI_VERSION:
case PSCI_0_2_FN_CPU_SUSPEND:
case PSCI_0_2_FN64_CPU_SUSPEND:
case PSCI_0_2_FN_CPU_OFF:
case PSCI_0_2_FN_CPU_ON:
case PSCI_0_2_FN64_CPU_ON:
case PSCI_0_2_FN_AFFINITY_INFO:
case PSCI_0_2_FN64_AFFINITY_INFO:
case PSCI_0_2_FN_SYSTEM_OFF:
case PSCI_0_2_FN_SYSTEM_RESET:
case PSCI_1_0_FN_PSCI_FEATURES:
case PSCI_1_1_FN_SYSTEM_RESET2:
case PSCI_1_1_FN64_SYSTEM_RESET2:
case ARM_SMCCC_VERSION_FUNC_ID:
val = PSCI_RET_SUCCESS;
break;
default:
val = PSCI_RET_NOT_SUPPORTED;
break;
}
/* Nothing to be handled by the host. Go back to the guest. */
smccc_set_retval(vcpu, val, 0, 0, 0);
return true;
}
static bool pkvm_handle_psci(struct pkvm_hyp_vcpu *hyp_vcpu)
{
struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
u32 psci_fn = smccc_get_function(vcpu);
switch (psci_fn) {
case PSCI_0_2_FN_CPU_ON:
kvm_psci_narrow_to_32bit(vcpu);
fallthrough;
case PSCI_0_2_FN64_CPU_ON:
return pvm_psci_vcpu_on(hyp_vcpu);
case PSCI_0_2_FN_CPU_OFF:
return pvm_psci_vcpu_off(hyp_vcpu);
case PSCI_0_2_FN_PSCI_VERSION:
return pvm_psci_version(hyp_vcpu);
case PSCI_1_0_FN_PSCI_FEATURES:
return pvm_psci_features(hyp_vcpu);
case PSCI_0_2_FN_AFFINITY_INFO:
case PSCI_0_2_FN64_AFFINITY_INFO:
case PSCI_0_2_FN_SYSTEM_RESET:
case PSCI_0_2_FN_CPU_SUSPEND:
case PSCI_0_2_FN64_CPU_SUSPEND:
case PSCI_0_2_FN_SYSTEM_OFF:
case PSCI_1_1_FN_SYSTEM_RESET2:
case PSCI_1_1_FN64_SYSTEM_RESET2:
return false; /* Handled by the host. */
default:
/* Unknown PSCI calls are handled here, not forwarded. */
smccc_set_retval(vcpu, PSCI_RET_NOT_SUPPORTED, 0, 0, 0);
return true;
}
}
/*
* Handler for protected VM HVC calls.
*
@ -1141,10 +1434,73 @@ static void pkvm_memunshare_call(u64 *ret, struct kvm_vcpu *vcpu)
*/
bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code)
{
struct pkvm_hyp_vcpu *hyp_vcpu = container_of(vcpu, struct pkvm_hyp_vcpu, vcpu);
u64 val[4] = { SMCCC_RET_INVALID_PARAMETER };
bool handled = true;
u32 feature;
uuid_t uuid;
switch (smccc_get_function(vcpu)) {
case ARM_SMCCC_VERSION_FUNC_ID:
/* Nothing to be handled by the host. Go back to the guest. */
val[0] = ARM_SMCCC_VERSION_1_1;
break;
case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
/* The workaround calls themselves are handled on hyp entry. */
val[0] = SMCCC_RET_NOT_SUPPORTED;
feature = smccc_get_arg1(vcpu);
switch (feature) {
case ARM_SMCCC_VERSION_FUNC_ID:
case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
val[0] = SMCCC_RET_SUCCESS;
break;
case ARM_SMCCC_ARCH_WORKAROUND_1:
switch (spectre_v2_state) {
case SPECTRE_VULNERABLE:
break;
case SPECTRE_MITIGATED:
val[0] = SMCCC_RET_SUCCESS;
break;
case SPECTRE_UNAFFECTED:
val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED;
break;
}
break;
case ARM_SMCCC_ARCH_WORKAROUND_2:
switch (spectre_v4_state) {
case SPECTRE_VULNERABLE:
break;
case SPECTRE_MITIGATED:
/* With SSBS advertised the guest's default is safe. */
if (kvm_has_feat(vcpu->kvm, ID_AA64PFR1_EL1, SSBS, IMP))
break;
fallthrough;
case SPECTRE_UNAFFECTED:
val[0] = SMCCC_RET_NOT_REQUIRED;
break;
}
break;
case ARM_SMCCC_ARCH_WORKAROUND_3:
switch (spectre_bhb_state) {
case SPECTRE_VULNERABLE:
break;
case SPECTRE_MITIGATED:
val[0] = SMCCC_RET_SUCCESS;
break;
case SPECTRE_UNAFFECTED:
val[0] = SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED;
break;
}
break;
}
break;
case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID:
uuid = ARM_SMCCC_VENDOR_HYP_UID_KVM;
val[0] = smccc_uuid_to_reg(&uuid, 0);
val[1] = smccc_uuid_to_reg(&uuid, 1);
val[2] = smccc_uuid_to_reg(&uuid, 2);
val[3] = smccc_uuid_to_reg(&uuid, 3);
break;
case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID:
val[0] = BIT(ARM_SMCCC_KVM_FUNC_FEATURES);
val[0] |= BIT(ARM_SMCCC_KVM_FUNC_HYP_MEMINFO);
@ -1177,8 +1533,7 @@ bool kvm_handle_pvm_hvc64(struct kvm_vcpu *vcpu, u64 *exit_code)
pkvm_memunshare_call(val, vcpu);
break;
default:
/* Punt everything else back to the host, for now. */
handled = false;
return pkvm_handle_psci(hyp_vcpu);
}
if (handled)

6
arch/arm64/kvm/hyp/nvhe/setup.c
View File

@ -10,6 +10,7 @@
#include <asm/kvm_pgtable.h>
#include <asm/kvm_pkvm.h>
#include <nvhe/alloc.h>
#include <nvhe/early_alloc.h>
#include <nvhe/ffa.h>
#include <nvhe/gfp.h>
@ -377,6 +378,11 @@ int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long *per_cpu_bas
if (ret)
return ret;
/* Arbitrary size, unmapped at boot and only consume VA-space */
ret = hyp_alloc_init(SZ_128M);
if (ret)
return ret;
update_nvhe_init_params();
/* Jump in the idmap page to switch to the new page-tables */

3
arch/arm64/kvm/hyp/nvhe/stacktrace.c
View File

@ -8,6 +8,7 @@
#include <asm/kvm_hyp.h>
#include <asm/memory.h>
#include <asm/percpu.h>
#include <asm/stacktrace/nvhe.h>
DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
__aligned(16);
@ -35,8 +36,6 @@ static void hyp_prepare_backtrace(unsigned long fp, unsigned long pc)
}
#ifdef CONFIG_PKVM_STACKTRACE
#include <asm/stacktrace/nvhe.h>
DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], pkvm_stacktrace);
static struct stack_info stackinfo_get_overflow(void)

29
arch/arm64/kvm/hyp/nvhe/switch.c
View File

@ -203,15 +203,36 @@ static const exit_handler_fn hyp_exit_handlers[] = {
[ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops,
};
/* WFI/WFE exit to the host, which emulates them. WFxT is not advertised. */
static bool kvm_handle_pvm_wfx(struct kvm_vcpu *vcpu, u64 *exit_code)
{
if (kvm_vcpu_get_esr(vcpu) & ESR_ELx_WFx_ISS_WFxT)
return kvm_handle_pvm_restricted(vcpu, exit_code);
return false;
}
/* Lazy FP/SIMD switch, or the UNDEF that would otherwise be left to the host. */
static bool kvm_handle_pvm_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code)
{
if (kvm_hyp_handle_fpsimd(vcpu, exit_code))
return true;
return kvm_handle_pvm_restricted(vcpu, exit_code);
}
/*
* A class not listed takes an UNDEF at EL2: the host has no way to
* inject one into a protected vCPU.
*/
static const exit_handler_fn pvm_exit_handlers[] = {
[0 ... ESR_ELx_EC_MAX] = NULL,
[0 ... ESR_ELx_EC_MAX] = kvm_handle_pvm_restricted,
[ESR_ELx_EC_WFx] = kvm_handle_pvm_wfx,
[ESR_ELx_EC_HVC64] = kvm_handle_pvm_hvc64,
[ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64,
[ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted,
[ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd,
[ESR_ELx_EC_FP_ASIMD] = kvm_handle_pvm_fpsimd,
[ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low,
[ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low,
[ESR_ELx_EC_WATCHPT_LOW] = kvm_hyp_handle_watchpt_low,
[ESR_ELx_EC_MOPS] = kvm_hyp_handle_mops,
};

63
arch/arm64/kvm/hyp/nvhe/sys_regs.c
View File

@ -121,6 +121,9 @@ static const struct pvm_ftr_bits pvmid_aa64mmfr0[] = {
MAX_FEAT(ID_AA64MMFR0_EL1, SNSMEM, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, BIGENDEL0, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, EXS, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN4, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN16, IMP),
MAX_FEAT(ID_AA64MMFR0_EL1, TGRAN64, IMP),
FEAT_END
};
@ -525,6 +528,66 @@ static const struct sys_reg_desc pvm_sys_reg_descs[] = {
/* Performance Monitoring Registers are restricted. */
};
struct sys_reg_desc_reset {
int reg;
void (*reset)(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *rd);
u64 value;
};
static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *r)
{
__vcpu_assign_sys_reg(vcpu, r->reg, kvm_calculate_mpidr(vcpu));
}
static void reset_value(struct kvm_vcpu *vcpu, const struct sys_reg_desc_reset *r)
{
__vcpu_assign_sys_reg(vcpu, r->reg, r->value);
}
#define RESET_VAL(REG, VAL) { REG, reset_value, VAL }
#define RESET_ZERO(REG) RESET_VAL(REG, 0)
#define RESET_UNKNOWN(REG) RESET_VAL(REG, 0x1de7ec7edbadc0deULL)
#define RESET_FUNC(REG, FN) { REG, FN, 0 }
static const struct sys_reg_desc_reset pvm_sys_reg_reset_vals[] = {
RESET_FUNC(MPIDR_EL1, reset_mpidr),
RESET_UNKNOWN(TPIDR_EL0),
RESET_UNKNOWN(TPIDRRO_EL0),
RESET_UNKNOWN(TPIDR_EL1),
RESET_ZERO(CNTKCTL_EL1),
RESET_UNKNOWN(PAR_EL1),
RESET_ZERO(DISR_EL1),
RESET_ZERO(CPACR_EL1),
RESET_VAL(CONTEXTIDR_EL1, 0x00000000dbadc0deULL),
RESET_VAL(SCTLR_EL1, 0x00C50078ULL),
RESET_ZERO(TCR_EL1),
RESET_UNKNOWN(AFSR0_EL1),
RESET_UNKNOWN(AFSR1_EL1),
RESET_UNKNOWN(ESR_EL1),
RESET_UNKNOWN(MAIR_EL1),
RESET_ZERO(AMAIR_EL1),
RESET_ZERO(MDSCR_EL1),
RESET_ZERO(CNTV_CTL_EL0),
RESET_UNKNOWN(TTBR0_EL1),
RESET_UNKNOWN(TTBR1_EL1),
RESET_UNKNOWN(FAR_EL1),
RESET_VAL(VBAR_EL1, 0x1de7ec7edbadc000ULL),
};
void kvm_reset_pvm_sys_regs(struct kvm_vcpu *vcpu)
{
unsigned long i;
for (i = 0; i < ARRAY_SIZE(pvm_sys_reg_reset_vals); i++) {
const struct sys_reg_desc_reset *r = &pvm_sys_reg_reset_vals[i];
r->reset(vcpu, r);
}
}
/*
* Initializes feature registers for protected vms.
*/

76
arch/arm64/kvm/hyp/nvhe/trace.c
View File

@ -4,6 +4,7 @@
* Author: Vincent Donnefort <vdonnefort@google.com>
*/
#include <nvhe/alloc.h>
#include <nvhe/clock.h>
#include <nvhe/mem_protect.h>
#include <nvhe/mm.h>
@ -62,18 +63,34 @@ static void __release_host_mem(void *start, u64 size)
WARN_ON(__pkvm_hyp_donate_host(hyp_virt_to_pfn(start), size >> PAGE_SHIFT));
}
static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_buffer,
struct hyp_trace_desc *desc)
static int hyp_trace_buffer_alloc_bpages(struct hyp_trace_buffer *trace_buffer,
struct hyp_trace_desc *desc)
{
void *start = (void *)kern_hyp_va(desc->bpages_backing_start);
size_t size = desc->bpages_backing_size;
void *start;
size_t size;
int ret;
ret = __admit_host_mem(start, size);
if (ret)
return ret;
if (is_protected_kvm_enabled()) {
struct ring_buffer_desc *rb_desc;
int cpu;
memset(start, 0, size);
size = 0;
for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc)
size += rb_desc->nr_page_va * sizeof(struct simple_buffer_page);
start = hyp_alloc(size);
if (!start)
return hyp_alloc_errno();
} else {
start = (void *)kern_hyp_va(desc->bpages_backing_start);
size = desc->bpages_backing_size;
ret = __admit_host_mem(start, size);
if (ret)
return ret;
memset(start, 0, size);
}
trace_buffer->bpages_backing_start = start;
trace_buffer->bpages_backing_size = size;
@ -81,7 +98,7 @@ static int hyp_trace_buffer_load_bpage_backing(struct hyp_trace_buffer *trace_bu
return 0;
}
static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace_buffer)
static void hyp_trace_buffer_free_bpages(struct hyp_trace_buffer *trace_buffer)
{
void *start = trace_buffer->bpages_backing_start;
size_t size = trace_buffer->bpages_backing_size;
@ -89,11 +106,14 @@ static void hyp_trace_buffer_unload_bpage_backing(struct hyp_trace_buffer *trace
if (!size)
return;
memset(start, 0, size);
if (is_protected_kvm_enabled()) {
hyp_free(start);
} else {
memset(start, 0, size);
__release_host_mem(start, size);
}
__release_host_mem(start, size);
trace_buffer->bpages_backing_start = 0;
trace_buffer->bpages_backing_start = NULL;
trace_buffer->bpages_backing_size = 0;
}
@ -128,7 +148,7 @@ static void hyp_trace_buffer_unload(struct hyp_trace_buffer *trace_buffer)
simple_ring_buffer_unload_mm(per_cpu_ptr(trace_buffer->simple_rbs, cpu),
__unpin_shared_page);
hyp_trace_buffer_unload_bpage_backing(trace_buffer);
hyp_trace_buffer_free_bpages(trace_buffer);
}
static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer,
@ -143,7 +163,7 @@ static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer,
if (hyp_trace_buffer_loaded(trace_buffer))
return -EINVAL;
ret = hyp_trace_buffer_load_bpage_backing(trace_buffer, desc);
ret = hyp_trace_buffer_alloc_bpages(trace_buffer, desc);
if (ret)
return ret;
@ -168,47 +188,43 @@ static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_siz
{
struct ring_buffer_desc *rb_desc;
unsigned int cpu;
size_t nr_bpages;
void *desc_end;
if (!is_protected_kvm_enabled())
return true;
/*
* Both desc_size and bpages_backing_size are untrusted host-provided
* values. We rely on __pkvm_host_donate_hyp() to enforce their validity.
* desc_size is an untrusted host-provided value. We rely on
* __pkvm_host_donate_hyp() to enforce its validity.
*/
desc_end = (void *)desc + desc_size;
nr_bpages = desc->bpages_backing_size / sizeof(struct simple_buffer_page);
if (desc->trace_buffer_desc.nr_cpus != hyp_nr_cpus)
return false;
for_each_ring_buffer_desc(rb_desc, cpu, &desc->trace_buffer_desc) {
/* Can we read nr_page_va? */
if ((void *)rb_desc + struct_size(rb_desc, page_va, 0) > desc_end)
return false;
/* simple_ring_buffer_init_mm() expects at least 3 pages */
if (rb_desc->nr_page_va < 3)
return false;
/* Overflow desc? */
if ((void *)rb_desc + struct_size(rb_desc, page_va, rb_desc->nr_page_va) > desc_end)
return false;
/* Overflow bpages backing memory? */
if (nr_bpages < rb_desc->nr_page_va)
return false;
if (cpu >= hyp_nr_cpus)
return false;
if (cpu != rb_desc->cpu)
return false;
nr_bpages -= rb_desc->nr_page_va;
}
return true;
}
int __tracing_load(unsigned long desc_hva, size_t desc_size)
int __tracing_load(void __kern *desc_hva, size_t desc_size)
{
struct hyp_trace_desc *desc = (struct hyp_trace_desc *)kern_hyp_va(desc_hva);
struct hyp_trace_desc *desc = kern_hyp_va_host(desc_hva);
int ret;
ret = __admit_host_mem(desc, desc_size);

6
arch/arm64/kvm/hyp/pgtable.c
View File

@ -8,6 +8,7 @@
*/
#include <linux/bitfield.h>
#include <linux/kvm_host.h>
#include <asm/kvm_pgtable.h>
#include <asm/stage2_pgtable.h>
@ -298,12 +299,13 @@ static int leaf_walker(const struct kvm_pgtable_visit_ctx *ctx,
}
int kvm_pgtable_get_leaf(struct kvm_pgtable *pgt, u64 addr,
kvm_pte_t *ptep, s8 *level)
kvm_pte_t *ptep, s8 *level,
enum kvm_pgtable_walk_flags flags)
{
struct leaf_walk_data data;
struct kvm_pgtable_walker walker = {
.cb = leaf_walker,
.flags = KVM_PGTABLE_WALK_LEAF,
.flags = flags | KVM_PGTABLE_WALK_LEAF,
.arg = &data,
};
int ret;

45
arch/arm64/kvm/hyp/vgic-v5-sr.c
View File

@ -7,6 +7,41 @@
#include <asm/kvm_hyp.h>
void __vgic_v5_make_resident(struct vgic_v5_cpu_if *cpu_if)
{
write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2);
isb();
/* Catch any faults */
cpu_if->vgic_contextr = read_sysreg_s(SYS_ICH_CONTEXTR_EL2);
if (!!FIELD_GET(ICH_CONTEXTR_EL2_F, cpu_if->vgic_contextr))
return;
cpu_if->gicv5_vpe.resident = true;
}
void __vgic_v5_make_non_resident(struct vgic_v5_cpu_if *cpu_if)
{
/*
* Clear the db_fired state to ensure that we're ready for the next
* doorbell when it is requested. If a doorbell firing caused us to
* enter the guest, then we've already consumed that state at this
* point, so this is safe to clear. Use WRITE_ONCE() to ensure we're not
* racing with the doorbell firing and setting the state true again.
*/
WRITE_ONCE(cpu_if->gicv5_vpe.db_fired, false);
/*
* Make as non-resident before actually making non-resident. Avoids race
* with doorbell arriving.
*/
cpu_if->gicv5_vpe.resident = false;
dsb(st);
write_sysreg_s(cpu_if->vgic_contextr, SYS_ICH_CONTEXTR_EL2);
isb();
}
void __vgic_v5_save_apr(struct vgic_v5_cpu_if *cpu_if)
{
cpu_if->vgic_apr = read_sysreg_s(SYS_ICH_APR_EL2);
@ -114,3 +149,13 @@ void __vgic_v5_restore_state(struct vgic_v5_cpu_if *cpu_if)
{
write_sysreg_s(cpu_if->vgic_icsr, SYS_ICC_ICSR_EL1);
}
void __vgic_v5_vdpend(u32 intid, bool pending, u16 vm)
{
u64 value;
value = intid & (GICV5_GIC_VDPEND_ID_MASK | GICV5_GIC_VDPEND_TYPE_MASK);
value |= FIELD_PREP(GICV5_GIC_VDPEND_PENDING_MASK, pending);
value |= FIELD_PREP(GICV5_GIC_VDPEND_VM_MASK, vm);
gic_insn(value, VDPEND);
}

21
arch/arm64/kvm/hyp/vhe/switch.c
View File

@ -175,7 +175,8 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
offset = read_sysreg_s(SYS_CNTPOFF_EL2);
if (map.direct_ptimer && offset) {
write_sysreg_el0(val + offset, SYS_CNTP_CVAL);
val = timer_apply_offset(val, offset, arch_timer_read_cntpct_el0());
write_sysreg_el0(val, SYS_CNTP_CVAL);
isb();
}
}
@ -296,10 +297,10 @@ static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code)
break;
case SYS_CNTP_CVAL_EL0:
if (vcpu_el2_e2h_is_set(vcpu)) {
val = read_sysreg_el0(SYS_CNTP_CVAL);
if (!has_cntpoff())
val -= timer_get_offset(vcpu_hptimer(vcpu));
if (!has_cntpoff() && timer_get_offset(vcpu_hptimer(vcpu)))
val = __vcpu_sys_reg(vcpu, CNTHP_CVAL_EL2);
else
val = read_sysreg_el0(SYS_CNTP_CVAL);
} else {
val = __vcpu_sys_reg(vcpu, CNTP_CVAL_EL0);
}
@ -321,10 +322,14 @@ static bool kvm_hyp_handle_timer(struct kvm_vcpu *vcpu, u64 *exit_code)
val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);
break;
case SYS_CNTV_CVAL_EL0:
if (vcpu_el2_e2h_is_set(vcpu))
val = read_sysreg_el0(SYS_CNTV_CVAL);
else
if (vcpu_el2_e2h_is_set(vcpu)) {
if (has_broken_cntvoff() && timer_get_offset(vcpu_hvtimer(vcpu)))
val = __vcpu_sys_reg(vcpu, CNTHV_CVAL_EL2);
else
val = read_sysreg_el0(SYS_CNTV_CVAL);
} else {
val = __vcpu_sys_reg(vcpu, CNTV_CVAL_EL0);
}
break;
case SYS_CNTVCT_EL0:
case SYS_CNTVCTSS_EL0:

90
arch/arm64/kvm/hyp_trace.c
View File

@ -13,6 +13,7 @@
#include <asm/kvm_host.h>
#include <asm/kvm_hyptrace.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_pkvm.h>
#include "hyp_trace.h"
@ -59,14 +60,8 @@ static void __hyp_clock_work(struct work_struct *work)
if (hyp_clock->mult) {
u64 err, cur = delta_cycles;
if (WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64)) {
__uint128_t tmp = (__uint128_t)cur * hyp_clock->mult;
cur = tmp >> hyp_clock->shift;
} else {
cur *= hyp_clock->mult;
cur >>= hyp_clock->shift;
}
WARN_ON_ONCE(cur >= hyp_clock->cyc_overflow64);
cur = mul_u64_u32_shr(cur, hyp_clock->mult, hyp_clock->shift);
cur += hyp_clock->boot;
err = abs_diff(cur, boot);
@ -84,7 +79,7 @@ static void __hyp_clock_work(struct work_struct *work)
err / NSEC_PER_USEC);
}
rate = div64_u64(delta_cycles * NSEC_PER_SEC, delta_boot);
rate = mul_u64_u64_div_u64(delta_cycles, NSEC_PER_SEC, delta_boot);
clocks_calc_mult_shift(&hyp_clock->mult, &hyp_clock->shift,
rate, NSEC_PER_SEC, CLOCK_MAX_CONVERSION_S);
@ -151,11 +146,18 @@ static void __unshare_page(unsigned long va)
static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_buffer, size_t size)
{
int nr_bpages = (PAGE_ALIGN(size) / PAGE_SIZE) + 1;
unsigned long nr_bpages;
size_t backing_size;
void *start;
int ret;
if (is_protected_kvm_enabled()) {
trace_buffer->desc->bpages_backing_start = 0;
trace_buffer->desc->bpages_backing_size = 0;
return 0;
}
nr_bpages = __calc_nr_pages_ring_buffer_desc(size);
backing_size = PAGE_ALIGN(sizeof(struct simple_buffer_page) * nr_bpages *
num_possible_cpus());
@ -177,6 +179,9 @@ static int hyp_trace_buffer_alloc_bpages_backing(struct hyp_trace_buffer *trace_
static void hyp_trace_buffer_free_bpages_backing(struct hyp_trace_buffer *trace_buffer)
{
if (!trace_buffer->desc->bpages_backing_start)
return;
free_pages_exact((void *)trace_buffer->desc->bpages_backing_start,
trace_buffer->desc->bpages_backing_size);
}
@ -269,7 +274,7 @@ static struct trace_buffer_desc *hyp_trace_load(unsigned long size, void *priv)
if (ret)
goto err_free_buffer;
ret = kvm_call_hyp_nvhe(__tracing_load, (unsigned long)desc, desc_size);
ret = pkvm_call_hyp_req(__tracing_load, desc, desc_size);
if (ret)
goto err_unload_pages;
@ -402,7 +407,10 @@ static struct trace_remote_callbacks trace_remote_callbacks = {
static const char *__hyp_enter_exit_reason_str(u8 reason);
#include <asm/kvm_define_hypevents.h>
struct remote_event_format_hyp_printk;
static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq);
#include "define_hypevents.h"
static const char *__hyp_enter_exit_reason_str(u8 reason)
{
@ -421,6 +429,61 @@ static const char *__hyp_enter_exit_reason_str(u8 reason)
return strs[min(reason, HYP_REASON_UNKNOWN)];
}
static void __hyp_trace_printk(struct remote_event_format_hyp_printk *entry, struct trace_seq *seq)
{
const struct hyp_string_fmt *hyp_str_fmt;
const char *fmt;
trace_seq_putc(seq, ' ');
if (entry->fmt_id >= (__hyp_string_fmts_end - __hyp_string_fmts_start)) {
trace_seq_printf(seq, "Unknown hyp_string_fmt ID %u\n", entry->fmt_id);
return;
}
hyp_str_fmt = &__hyp_string_fmts_start[entry->fmt_id];
fmt = hyp_str_fmt->fmt;
switch (hyp_str_fmt->nr_args) {
case 0:
trace_seq_puts(seq, fmt);
break;
case 1:
trace_seq_printf(seq, fmt, entry->args[0]);
break;
case 2:
trace_seq_printf(seq, fmt, entry->args[0], entry->args[1]);
break;
case 3:
trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2]);
break;
case 4:
trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2],
entry->args[3]);
break;
case 5:
trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2],
entry->args[3], entry->args[4]);
break;
case 6:
trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2],
entry->args[3], entry->args[4], entry->args[5]);
break;
case 7:
trace_seq_printf(seq, fmt, entry->args[0], entry->args[1], entry->args[2],
entry->args[3], entry->args[4], entry->args[5], entry->args[6]);
break;
default:
trace_seq_printf(seq, fmt, entry->args[0], entry->args[1],
entry->args[2], entry->args[3], entry->args[4], entry->args[5],
entry->args[6], entry->args[7]);
break;
}
if (trace_seq_used(seq) && seq->buffer[trace_seq_used(seq) - 1] != '\n')
trace_seq_putc(seq, '\n');
}
static void __init hyp_trace_init_events(void)
{
struct hyp_event_id *hyp_event_id = __hyp_event_ids_start;
@ -430,6 +493,9 @@ static void __init hyp_trace_init_events(void)
/* Events on both sides hypervisor are sorted */
for (; event < __hyp_events_end; event++, hyp_event_id++, id++)
event->id = hyp_event_id->id = id;
WARN(__hyp_string_fmts_end - __hyp_string_fmts_start > U16_MAX + 1,
"Too many trace_hyp_printk() callsites\n");
}
int __init kvm_hyp_trace_init(void)

11
arch/arm64/kvm/hypercalls.c
View File

@ -294,6 +294,10 @@ int kvm_smccc_call_handler(struct kvm_vcpu *vcpu)
case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
feature = smccc_get_arg1(vcpu);
switch (feature) {
case ARM_SMCCC_VERSION_FUNC_ID:
case ARM_SMCCC_ARCH_FEATURES_FUNC_ID:
val[0] = SMCCC_RET_SUCCESS;
break;
case ARM_SMCCC_ARCH_WORKAROUND_1:
switch (arm64_get_spectre_v2_state()) {
case SPECTRE_VULNERABLE:
@ -581,6 +585,13 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
wants_02 = vcpu_has_feature(vcpu, KVM_ARM_VCPU_PSCI_0_2);
/*
* EL2 advertises PSCI 1.1; the host handles the forwarded calls
* by this version.
*/
if (vcpu_is_protected(vcpu) && val < KVM_ARM_PSCI_1_1)
return -EINVAL;
switch (val) {
case KVM_ARM_PSCI_0_1:
if (wants_02)

10
arch/arm64/kvm/inject_fault.c
View File

@ -371,15 +371,15 @@ int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr)
}
/*
* Emulate the exception entry if SErrors are unmasked. This is useful if
* the vCPU is in a nested context w/ vSErrors enabled then we've already
* delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2,
* VDISR_EL2) to the guest hypervisor.
* With NV, emulate the exception entry if SErrors are unmasked: in a
* nested context with vSErrors enabled, the hardware vSError context
* (i.e. HCR_EL2.VSE, VSESR_EL2, VDISR_EL2) is delegated to the guest
* hypervisor. Otherwise, HCR_EL2.VSE delivers it once it is unmasked.
*
* As we're emulating the SError injection we need to explicitly populate
* ESR_ELx.EC because hardware will not do it on our behalf.
*/
if (!serror_is_masked(vcpu)) {
if (vcpu_has_nv(vcpu) && !serror_is_masked(vcpu)) {
pend_serror_exception(vcpu);
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR) | ESR_ELx_IL;
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));

3
arch/arm64/kvm/mmio.c
View File

@ -6,10 +6,12 @@
#include <linux/kvm_host.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_mmu.h>
#include <trace/events/kvm.h>
#include "trace.h"
#if ARM64_S390_COMMON == 1
void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data)
{
void *datap = NULL;
@ -258,3 +260,4 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
run->exit_reason = KVM_EXIT_MMIO;
return 0;
}
#endif /* ARM64_S390_COMMON == 1 */

784
arch/arm64/kvm/mmu.c
View File

File diff suppressed because it is too large Load Diff

151
arch/arm64/kvm/nested.c
View File

@ -5,6 +5,7 @@
*/
#include <linux/bitfield.h>
#include <linux/interval_tree.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
@ -44,14 +45,16 @@ struct vncr_tlb {
* will invalidate them more often).
*/
#define S2_MMU_PER_VCPU 2
#define S2_MMU_PER_VM (KVM_MAX_VCPUS * S2_MMU_PER_VCPU)
int kvm_init_nested(struct kvm *kvm)
{
kvm->arch.nested_mmus = kvmalloc_objs(struct kvm_s2_mmu *,
KVM_MAX_VCPUS * S2_MMU_PER_VCPU,
S2_MMU_PER_VM,
GFP_KERNEL_ACCOUNT);
kvm->arch.nested_mmus_size = 0;
atomic_set(&kvm->arch.vncr_tlb_count, 0);
spin_lock_init(&kvm->arch.guest_s2_tracking_lock);
return kvm->arch.nested_mmus ? 0 : -ENOMEM;
}
@ -124,10 +127,15 @@ int kvm_vcpu_init_nested(struct kvm_vcpu *vcpu)
return ret;
}
for (i = 0; i < S2_MMU_PER_VCPU; i++)
kvm_nested_s2_ptdump_create_debugfs(&tmp[i], i + kvm->arch.nested_mmus_size);
guard(write_lock)(&kvm->mmu_lock);
for (i = 0; i < S2_MMU_PER_VCPU; i++)
for (i = 0; i < S2_MMU_PER_VCPU; i++) {
tmp[i].s2_mmu_idx = i + kvm->arch.nested_mmus_size;
kvm->arch.nested_mmus[i + kvm->arch.nested_mmus_size] = &tmp[i];
}
kvm->arch.nested_mmus_size += S2_MMU_PER_VCPU;
}
@ -657,7 +665,7 @@ static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
return 0;
tmp &= ~(sz - 1);
if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL, 0))
goto again;
if (!(pte & PTE_VALID))
goto again;
@ -837,10 +845,8 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
kvm->arch.nested_mmus_next = (i + 1) % kvm->arch.nested_mmus_size;
/* Make sure we don't forget to do the laundry */
if (kvm_s2_mmu_valid(s2_mmu)) {
kvm_nested_s2_ptdump_remove_debugfs(s2_mmu);
if (kvm_s2_mmu_valid(s2_mmu))
s2_mmu->pending_unmap = true;
}
/*
* The virtual VMID (modulo CnP) will be used as a key when matching
@ -854,8 +860,6 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
s2_mmu->tlb_vtcr = vcpu_read_sys_reg(vcpu, VTCR_EL2);
s2_mmu->nested_stage2_enabled = vcpu_read_sys_reg(vcpu, HCR_EL2) & HCR_VM;
kvm_nested_s2_ptdump_create_debugfs(s2_mmu);
out:
atomic_inc(&s2_mmu->refcnt);
@ -871,6 +875,87 @@ static struct kvm_s2_mmu *get_s2_mmu_nested(struct kvm_vcpu *vcpu)
return s2_mmu;
}
#define S2_MMU_IDX_MASK GENMASK_ULL(11, 0)
static void tag_s2_mapping_mmu(struct kvm_guest_s2_mapping *mapping,
struct kvm_s2_mmu *mmu)
{
BUILD_BUG_ON(S2_MMU_PER_VM > SZ_4K);
mapping->nested.start &= ~S2_MMU_IDX_MASK;
mapping->nested.start |= mmu->s2_mmu_idx;
}
static struct kvm_s2_mmu *s2_mapping_to_mmu(struct kvm *kvm,
struct kvm_guest_s2_mapping *mapping)
{
return kvm->arch.nested_mmus[mapping->nested.start & S2_MMU_IDX_MASK];
}
static unsigned long s2_mapping_to_nested_start(struct kvm_guest_s2_mapping *mapping)
{
return mapping->nested.start & ~S2_MMU_IDX_MASK;
}
void kvm_record_guest_s2_mapping(struct kvm_s2_mmu *mmu, gpa_t canonical_ipa,
gpa_t nested_ipa, size_t map_size,
struct kvm_guest_s2_mapping *mapping)
{
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
lockdep_assert_held_read(&kvm->mmu_lock);
canonical_ipa = ALIGN_DOWN(canonical_ipa, map_size);
nested_ipa = ALIGN_DOWN(nested_ipa, map_size);
mapping->canonical.start = canonical_ipa;
mapping->canonical.last = canonical_ipa + map_size - 1;
mapping->nested.start = nested_ipa;
mapping->nested.last = nested_ipa + map_size - 1;
tag_s2_mapping_mmu(mapping, mmu);
guard(spinlock)(&kvm->arch.guest_s2_tracking_lock);
interval_tree_insert(&mapping->nested, &mmu->guest_s2_mappings);
interval_tree_insert(&mapping->canonical, &kvm->arch.mmu.guest_s2_mappings);
}
void kvm_remove_guest_s2_mappings(struct kvm_s2_mmu *mmu, gpa_t nipa,
size_t size)
{
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
struct interval_tree_node *node, *next;
struct kvm_guest_s2_mapping *mapping;
gpa_t nipa_end = nipa + size - 1;
/*
* See kvm_nested_unmap_cipa_range() for why guest_s2_tracking_lock
* isn't taken here.
*/
lockdep_assert_held_write(&kvm->mmu_lock);
node = interval_tree_iter_first(&mmu->guest_s2_mappings, nipa, nipa_end);
while (node) {
unsigned long nested_start;
next = interval_tree_iter_next(node, nipa, nipa_end);
mapping = container_of(node, struct kvm_guest_s2_mapping,
nested);
/*
* Tracking must be conservative on removal, only remove
* mappings that are within the unmap range.
*/
nested_start = s2_mapping_to_nested_start(mapping);
if (nipa <= nested_start && nipa_end >= mapping->nested.last) {
interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings);
interval_tree_remove(&mapping->canonical,
&kvm->arch.mmu.guest_s2_mappings);
kfree(mapping);
}
node = next;
}
}
void kvm_init_nested_s2_mmu(struct kvm_s2_mmu *mmu)
{
/* CnP being set denotes an invalid entry */
@ -1293,6 +1378,56 @@ void kvm_nested_s2_wp(struct kvm *kvm)
kvm_invalidate_vncr_ipa_all(kvm);
}
void kvm_nested_unmap_cipa_range(struct kvm *kvm, gpa_t cipa, size_t unmap_size,
bool may_block)
{
gpa_t cipa_end = cipa + unmap_size - 1;
struct kvm_guest_s2_mapping *mapping;
struct interval_tree_node *node;
size_t mapping_size;
/*
* Guest s2 tracking interval trees are only accessed while holding the
* mmu_lock, hence we don't have to take guest_s2_tracking_lock if the
* mmu_lock is held for write. This saves us from having to manually
* lock/unlock guest_s2_tracking_lock below around
* cond_resched_rwlock_write().
*/
lockdep_assert_held_write(&kvm->mmu_lock);
if (!kvm->arch.nested_mmus_size)
return;
while ((node = interval_tree_iter_first(&kvm->arch.mmu.guest_s2_mappings,
cipa, cipa_end))) {
unsigned long nested_start;
struct kvm_s2_mmu *mmu;
mapping = container_of(node, struct kvm_guest_s2_mapping,
canonical);
nested_start = s2_mapping_to_nested_start(mapping);
mmu = s2_mapping_to_mmu(kvm, mapping);
/* We could race against MMU teardown, which frees mmu->pgt. */
if (mmu->pgt) {
mapping_size = mapping->nested.last - nested_start + 1;
if (WARN_ON_ONCE(kvm_pgtable_stage2_unmap(mmu->pgt, nested_start,
mapping_size)))
return;
interval_tree_remove(&mapping->nested, &mmu->guest_s2_mappings);
}
interval_tree_remove(node, &kvm->arch.mmu.guest_s2_mappings);
kfree(mapping);
if (may_block)
cond_resched_rwlock_write(&kvm->mmu_lock);
}
kvm_invalidate_vncr_ipa(kvm, cipa, cipa + unmap_size);
}
void kvm_nested_s2_unmap(struct kvm *kvm, bool may_block)
{
int i;

2
arch/arm64/kvm/pauth.c
View File

@ -5,7 +5,7 @@
*
* Primitive PAuth emulation for ERETAA/ERETAB.
*
* This code assumes that is is run from EL2, and that it is part of
* This code assumes that it is run from EL2, and that it is part of
* the emulation of ERETAx for a guest hypervisor. That's a lot of
* baked-in assumptions and shortcuts.
*

226
arch/arm64/kvm/pkvm.c
View File

@ -16,6 +16,9 @@
#include "hyp_constants.h"
#define CREATE_TRACE_POINTS
#include "trace_pkvm.h"
DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory);
@ -85,6 +88,77 @@ void __init kvm_hyp_reserve(void)
hyp_mem_base);
}
static int pkvm_hyp_topup(enum pkvm_topup_id id, unsigned long nr_pages)
{
struct kvm_hyp_memcache mc;
struct arm_smccc_res res;
int ret;
init_hyp_memcache(&mc);
ret = topup_hyp_memcache(&mc, nr_pages);
if (ret)
goto err;
arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_topup), id, mc.head,
mc.nr_pages, &res);
if (WARN_ON_ONCE(res.a0 != SMCCC_RET_SUCCESS)) {
ret = -EINVAL;
goto err;
}
ret = res.a1;
mc.head = res.a2;
mc.nr_pages = res.a3;
err:
free_hyp_memcache(&mc);
return ret;
}
static unsigned long __pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target)
{
struct kvm_hyp_memcache mc;
struct arm_smccc_res res;
unsigned long reclaimed;
arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__pkvm_hyp_reclaim), id, target, &res);
if (WARN_ON_ONCE(res.a0 != SMCCC_RET_SUCCESS) || WARN_ON_ONCE(res.a1))
return 0;
init_hyp_memcache(&mc);
mc.head = res.a2;
mc.nr_pages = reclaimed = res.a3;
free_hyp_memcache(&mc);
return reclaimed;
}
static unsigned long pkvm_hyp_reclaim(enum pkvm_topup_id id, unsigned long target)
{
unsigned long reclaimed = 0;
while (reclaimed < target) {
/* Arbitrary limit to avoid blocking in EL2 for too long */
unsigned long r = __pkvm_hyp_reclaim(id, min(target - reclaimed, 16));
if (!r)
break;
reclaimed += r;
if (reclaimed >= target)
break;
cond_resched();
}
return reclaimed;
}
static unsigned long pkvm_hyp_reclaimable(enum pkvm_topup_id id)
{
return kvm_call_hyp_nvhe(__pkvm_hyp_reclaimable, id);
}
static void __pkvm_destroy_hyp_vm(struct kvm *kvm)
{
if (pkvm_hyp_vm_is_created(kvm)) {
@ -100,30 +174,34 @@ static void __pkvm_destroy_hyp_vm(struct kvm *kvm)
kvm->arch.pkvm.handle = 0;
kvm->arch.pkvm.is_created = false;
free_hyp_memcache(&kvm->arch.pkvm.teardown_mc);
free_hyp_memcache(&kvm->arch.pkvm.stage2_teardown_mc);
}
static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu)
{
size_t hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE);
pkvm_handle_t handle = vcpu->kvm->arch.pkvm.handle;
void *hyp_vcpu;
int ret;
vcpu->arch.pkvm_memcache.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2;
init_hyp_stage2_memcache(&vcpu->arch.stage2_mc);
hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT);
if (!hyp_vcpu)
return -ENOMEM;
ret = pkvm_call_hyp_req(__pkvm_init_vcpu, handle, vcpu);
if (ret)
return ret;
ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, vcpu, hyp_vcpu);
if (!ret)
vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED);
else
free_pages_exact(hyp_vcpu, hyp_vcpu_sz);
/*
* Mirror EL2's seeding of power_state from mp_state. The hyp vCPU is
* published, so take mp_state_lock against kvm_psci_vcpu_on().
*/
if (vcpu_is_protected(vcpu)) {
spin_lock(&vcpu->arch.mp_state_lock);
if (kvm_arm_vcpu_stopped(vcpu))
WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_UNINITIALIZED);
spin_unlock(&vcpu->arch.mp_state_lock);
}
return ret;
vcpu_set_flag(vcpu, VCPU_PKVM_FINALIZED);
return 0;
}
/*
@ -138,8 +216,8 @@ static int __pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu)
*/
static int __pkvm_create_hyp_vm(struct kvm *kvm)
{
size_t pgd_sz, hyp_vm_sz;
void *pgd, *hyp_vm;
size_t pgd_sz;
void *pgd;
int ret;
if (kvm->created_vcpus < 1)
@ -156,28 +234,15 @@ static int __pkvm_create_hyp_vm(struct kvm *kvm)
if (!pgd)
return -ENOMEM;
/* Allocate memory to donate to hyp for vm and vcpu pointers. */
hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE,
size_mul(sizeof(void *),
kvm->created_vcpus)));
hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT);
if (!hyp_vm) {
ret = -ENOMEM;
goto free_pgd;
}
/* Donate the VM memory to hyp and let hyp initialize it. */
ret = kvm_call_hyp_nvhe(__pkvm_init_vm, kvm, hyp_vm, pgd);
ret = pkvm_call_hyp_req(__pkvm_init_vm, kvm, pgd);
if (ret)
goto free_vm;
goto free_pgd;
kvm->arch.pkvm.is_created = true;
kvm->arch.pkvm.stage2_teardown_mc.flags |= HYP_MEMCACHE_ACCOUNT_STAGE2;
init_hyp_stage2_memcache(&kvm->arch.pkvm.stage2_teardown_mc);
kvm_account_pgtable_pages(pgd, pgd_sz / PAGE_SIZE);
return 0;
free_vm:
free_pages_exact(hyp_vm, hyp_vm_sz);
free_pgd:
free_pages_exact(pgd, pgd_sz);
return ret;
@ -229,10 +294,9 @@ void pkvm_destroy_hyp_vm(struct kvm *kvm)
mutex_unlock(&kvm->arch.config_lock);
}
int pkvm_init_host_vm(struct kvm *kvm, unsigned long type)
int pkvm_init_host_vm(struct kvm *kvm)
{
int ret;
bool protected = type & KVM_VM_TYPE_ARM_PROTECTED;
/* Reserve the VM in hyp and obtain a hyp handle for the VM. */
ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm);
@ -240,8 +304,7 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type)
return ret;
kvm->arch.pkvm.handle = ret;
kvm->arch.pkvm.is_protected = protected;
if (protected) {
if (kvm_vm_is_protected(kvm)) {
pr_warn_once("kvm: protected VMs are experimental and for development only, tainting kernel\n");
add_taint(TAINT_USER, LOCKDEP_STILL_OK);
}
@ -270,8 +333,55 @@ static int __init pkvm_drop_host_privileges(void)
return ret;
}
void __init pkvm_selftests(void)
{
#ifdef CONFIG_NVHE_EL2_DEBUG
int ret = pkvm_call_hyp_req(__pkvm_hyp_alloc_selftest);
unsigned long reclaimed;
reclaimed = pkvm_hyp_reclaim(PKVM_TOPUP_HYP_ALLOC_SELFTEST, ULONG_MAX);
/* On failure, not all the pages may be reclaimable */
if (!ret)
WARN_ON(reclaimed != 6 /* SELFTEST_MAX_PAGES */);
else
kvm_err("pKVM hyp allocator selftest failed (%d)\n", ret);
#endif
}
static unsigned long pkvm_shrinker_count(struct shrinker *shrink, struct shrink_control *sc)
{
unsigned long reclaimable = 0;
int id;
for (id = 0; id < NR_PKVM_TOPUP_HYP_IDS; id++)
reclaimable += pkvm_hyp_reclaimable(id);
return reclaimable ?: SHRINK_EMPTY;
}
static unsigned long pkvm_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc)
{
unsigned long reclaimed = 0;
int id;
sc->nr_scanned = 0;
for (id = 0; id < NR_PKVM_TOPUP_HYP_IDS; id++) {
unsigned long r = pkvm_hyp_reclaim(id, sc->nr_to_scan - sc->nr_scanned);
reclaimed += r;
sc->nr_scanned += r;
if (sc->nr_scanned >= sc->nr_to_scan)
break;
}
return reclaimed ?: SHRINK_STOP;
}
static int __init finalize_pkvm(void)
{
struct shrinker *pkvm_shrinker;
int ret;
if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised())
@ -287,10 +397,21 @@ static int __init finalize_pkvm(void)
kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size);
ret = pkvm_drop_host_privileges();
if (ret)
if (ret) {
pr_err("Failed to finalize Hyp protection: %d\n", ret);
return ret;
}
return ret;
pkvm_shrinker = shrinker_alloc(0, "pkvm");
if (pkvm_shrinker) {
pkvm_shrinker->count_objects = pkvm_shrinker_count;
pkvm_shrinker->scan_objects = pkvm_shrinker_scan;
shrinker_register(pkvm_shrinker);
} else {
kvm_err("Failed to register shrinker for pKVM\n");
}
return 0;
}
device_initcall_sync(finalize_pkvm);
@ -414,6 +535,9 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
u64 end = addr + size;
int ret;
if (WARN_ON_ONCE(!PAGE_ALIGNED(addr | size)))
return -EINVAL;
lockdep_assert_held_write(&kvm->mmu_lock);
mapping = pkvm_mapping_iter_first(&pgt->pkvm_mappings, addr, end - 1);
@ -594,3 +718,31 @@ bool pkvm_force_reclaim_guest_page(phys_addr_t phys)
return !ret || ret == -EAGAIN;
}
static int pkvm_handle_hyp_req(struct pkvm_hyp_req *req)
{
int ret = -EINVAL;
switch (req->type) {
case PKVM_HYP_REQ_HYP_ALLOC:
ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC, req->mem.nr_pages);
break;
case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST:
ret = pkvm_hyp_topup(PKVM_TOPUP_HYP_ALLOC_SELFTEST, req->mem.nr_pages);
break;
}
trace_kvm_handle_pkvm_hyp_req(req, ret);
return ret;
}
int __pkvm_handle_smccc_req(struct arm_smccc_res *res)
{
struct pkvm_hyp_req req;
if (smccc_to_pkvm_hyp_req(&req, res))
return pkvm_handle_hyp_req(&req);
return res->a1;
}

5
arch/arm64/kvm/pmu-emul.c
View File

@ -50,8 +50,11 @@ static u32 __kvm_pmu_event_mask(unsigned int pmuver)
case ID_AA64DFR0_EL1_PMUVer_V3P5:
case ID_AA64DFR0_EL1_PMUVer_V3P7:
return GENMASK(15, 0);
default: /* Shouldn't be here, just for sanity */
default:
/* Shouldn't be here, just for sanity */
WARN_ONCE(1, "Unknown PMU version %d\n", pmuver);
fallthrough;
case ID_AA64DFR0_EL1_PMUVer_NI:
return 0;
}
}

11
arch/arm64/kvm/psci.c
View File

@ -63,7 +63,12 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu)
return PSCI_RET_INVALID_PARAMS;
spin_lock(&vcpu->arch.mp_state_lock);
if (!kvm_arm_vcpu_stopped(vcpu)) {
/*
* A protected vCPU is off once the host has acted on EL2's CPU_OFF.
* STOPPED before that is the VMM's pause.
*/
if (!kvm_arm_vcpu_stopped(vcpu) ||
(vcpu_is_protected(vcpu) && !kvm_pkvm_vcpu_is_powered_off(vcpu))) {
if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1)
ret = PSCI_RET_ALREADY_ON;
else
@ -161,7 +166,9 @@ static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags)
*/
kvm_for_each_vcpu(i, tmp, vcpu->kvm) {
spin_lock(&tmp->arch.mp_state_lock);
WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
/* Keep EL2's off record for the guest's next CPU_ON. */
if (!kvm_pkvm_vcpu_is_powered_off(tmp))
WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED);
spin_unlock(&tmp->arch.mp_state_lock);
}
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP);

79
arch/arm64/kvm/ptdump.c
View File

@ -17,7 +17,7 @@
#define MARKERS_LEN 2
#define KVM_PGTABLE_MAX_LEVELS (KVM_PGTABLE_LAST_LEVEL + 1)
#define S2FNAMESZ sizeof("0x0123456789abcdef-0x0123456789abcdef-s2-disabled")
#define S2FNAMESZ sizeof("nested_mmu0000")
struct kvm_ptdump_guest_state {
struct kvm_s2_mmu *mmu;
@ -115,15 +115,28 @@ static int kvm_ptdump_build_levels(struct ptdump_pg_level *level, u32 start_lvl)
static struct kvm_ptdump_guest_state *kvm_ptdump_parser_create(struct kvm_s2_mmu *mmu)
{
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
struct kvm_ptdump_guest_state *st;
struct kvm_pgtable *pgtable = mmu->pgt;
s8 start_level;
int ret;
/*
* We only need the pgt start level to initialize the ptdump, get it
* while holding the mmu_lock. It's fine if the pgt gets freed
* afterwards, we'll check again when doing the actual dump.
*/
scoped_guard(read_lock, &kvm->mmu_lock) {
if (mmu->pgt)
start_level = mmu->pgt->start_level;
else
return ERR_PTR(-ENOENT);
}
st = kzalloc_obj(struct kvm_ptdump_guest_state, GFP_KERNEL_ACCOUNT);
if (!st)
return ERR_PTR(-ENOMEM);
ret = kvm_ptdump_build_levels(&st->level[0], pgtable->start_level);
ret = kvm_ptdump_build_levels(&st->level[0], start_level);
if (ret) {
kfree(st);
return ERR_PTR(ret);
@ -149,6 +162,9 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused)
};
guard(write_lock)(&kvm->mmu_lock);
if (!mmu->pgt)
return 0;
st->parser_state = (struct ptdump_pg_state) {
.marker = &st->ipa_marker[0],
.end_address = BIT(mmu->pgt->ia_bits),
@ -157,6 +173,15 @@ static int kvm_ptdump_guest_show(struct seq_file *m, void *unused)
.seq = m,
};
if (kvm_is_nested_s2_mmu(kvm, mmu)) {
if (kvm_s2_mmu_valid(mmu))
seq_printf(m, "VTCR: 0x%016llx VTTBR: 0x%016llx s2: %s\n",
mmu->tlb_vtcr, mmu->tlb_vttbr,
mmu->nested_stage2_enabled ? "enabled" : "disabled");
else
return 0;
}
ret = kvm_pgtable_walk(mmu->pgt, 0, BIT(mmu->pgt->ia_bits), &walker);
if (ret)
return ret;
@ -211,17 +236,27 @@ static const struct file_operations kvm_ptdump_guest_fops = {
static int kvm_pgtable_range_show(struct seq_file *m, void *unused)
{
struct kvm_pgtable *pgtable = m->private;
struct kvm_s2_mmu *mmu = m->private;
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
guard(read_lock)(&kvm->mmu_lock);
if (mmu->pgt)
seq_printf(m, "%2u\n", mmu->pgt->ia_bits);
seq_printf(m, "%2u\n", pgtable->ia_bits);
return 0;
}
static int kvm_pgtable_levels_show(struct seq_file *m, void *unused)
{
struct kvm_pgtable *pgtable = m->private;
struct kvm_s2_mmu *mmu = m->private;
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
guard(read_lock)(&kvm->mmu_lock);
if (mmu->pgt)
seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - mmu->pgt->start_level);
seq_printf(m, "%1d\n", KVM_PGTABLE_MAX_LEVELS - pgtable->start_level);
return 0;
}
@ -230,15 +265,12 @@ static int kvm_pgtable_debugfs_open(struct inode *m, struct file *file,
{
struct kvm_s2_mmu *mmu = m->i_private;
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
struct kvm_pgtable *pgtable;
int ret;
if (!kvm_get_kvm_safe(kvm))
return -ENOENT;
pgtable = mmu->pgt;
ret = single_open(file, show, pgtable);
ret = single_open(file, show, mmu);
if (ret < 0)
kvm_put_kvm(kvm);
return ret;
@ -276,30 +308,23 @@ static const struct file_operations kvm_pgtable_levels_fops = {
.release = kvm_pgtable_debugfs_close,
};
void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu)
void kvm_nested_s2_ptdump_create_debugfs(struct kvm_s2_mmu *mmu, int idx)
{
struct dentry *dent;
char file_name[S2FNAMESZ];
snprintf(file_name, sizeof(file_name), "0x%016llx-0x%016llx-s2-%sabled",
mmu->tlb_vttbr,
mmu->tlb_vtcr,
mmu->nested_stage2_enabled ? "en" : "dis");
snprintf(file_name, sizeof(file_name), "nested_mmu%d", idx);
dent = debugfs_create_file(file_name, 0400,
mmu->arch->debugfs_nv_dentry, mmu,
&kvm_ptdump_guest_fops);
mmu->shadow_pt_debugfs_dentry = dent;
}
void kvm_nested_s2_ptdump_remove_debugfs(struct kvm_s2_mmu *mmu)
{
debugfs_remove(mmu->shadow_pt_debugfs_dentry);
debugfs_create_file(file_name, 0400,
mmu->arch->debugfs_nv_dentry, mmu,
&kvm_ptdump_guest_fops);
}
void kvm_s2_ptdump_create_debugfs(struct kvm *kvm)
{
/* pKVM handles the page-tables in the hypervisor */
if (is_protected_kvm_enabled())
return;
debugfs_create_file("stage2_page_tables", 0400, kvm->debugfs_dentry,
&kvm->arch.mmu, &kvm_ptdump_guest_fops);
debugfs_create_file("ipa_range", 0400, kvm->debugfs_dentry,

10
arch/arm64/kvm/pvtime.c
View File

@ -67,9 +67,9 @@ gpa_t kvm_init_stolen_time(struct kvm_vcpu *vcpu)
return base;
}
bool kvm_arm_pvtime_supported(void)
bool kvm_arm_pvtime_supported(struct kvm *kvm)
{
return !!sched_info_on();
return !!sched_info_on() && (!kvm || !kvm_vm_is_protected(kvm));
}
int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu,
@ -81,7 +81,7 @@ int kvm_arm_pvtime_set_attr(struct kvm_vcpu *vcpu,
int ret = 0;
int idx;
if (!kvm_arm_pvtime_supported() ||
if (!kvm_arm_pvtime_supported(kvm) ||
attr->attr != KVM_ARM_VCPU_PVTIME_IPA)
return -ENXIO;
@ -110,7 +110,7 @@ int kvm_arm_pvtime_get_attr(struct kvm_vcpu *vcpu,
u64 __user *user = (u64 __user *)attr->addr;
u64 ipa;
if (!kvm_arm_pvtime_supported() ||
if (!kvm_arm_pvtime_supported(vcpu->kvm) ||
attr->attr != KVM_ARM_VCPU_PVTIME_IPA)
return -ENXIO;
@ -126,7 +126,7 @@ int kvm_arm_pvtime_has_attr(struct kvm_vcpu *vcpu,
{
switch (attr->attr) {
case KVM_ARM_VCPU_PVTIME_IPA:
if (kvm_arm_pvtime_supported())
if (kvm_arm_pvtime_supported(vcpu->kvm))
return 0;
}
return -ENXIO;

3
arch/arm64/kvm/stacktrace.c
View File

@ -198,9 +198,6 @@ static void hyp_dump_backtrace(unsigned long hyp_offset)
}
#ifdef CONFIG_PKVM_STACKTRACE
DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
pkvm_stacktrace);
/*
* pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace.
*

69
arch/arm64/kvm/sys_regs.c
View File

@ -1029,7 +1029,7 @@ static unsigned int hidden_visibility(const struct kvm_vcpu *vcpu,
static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r)
{
if (kvm_vcpu_has_pmu(vcpu))
if (kvm_has_feat(vcpu->kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
return 0;
return REG_HIDDEN;
@ -1059,10 +1059,6 @@ static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
{
/* This thing will UNDEF, who cares about the reset value? */
if (!kvm_vcpu_has_pmu(vcpu))
return 0;
reset_unknown(vcpu, r);
__vcpu_rmw_sys_reg(vcpu, r->reg, &=, kvm_pmu_evtyper_mask(vcpu->kvm));
@ -2512,9 +2508,10 @@ static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
/*
* Once the VM has started the ID registers are immutable. Reject any
* write that does not match the final register value.
* write that does not match the final register value once we have
* got far enough into first running the VM to use the values.
*/
if (kvm_vm_has_ran_once(vcpu->kvm)) {
if (kvm_id_regs_final(vcpu->kvm)) {
if (val != read_id_reg(vcpu, rd))
ret = -EBUSY;
else
@ -2548,7 +2545,7 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val)
lockdep_assert_held(&kvm->arch.config_lock);
if (KVM_BUG_ON(kvm_vm_has_ran_once(kvm) || !p, kvm))
if (KVM_BUG_ON(kvm_id_regs_final(kvm) || !p, kvm))
return;
*p = val;
@ -3244,10 +3241,10 @@ static int set_imp_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
return -EINVAL;
/*
* Once the VM has started the ID registers are immutable. Reject the
* write if userspace tries to change it.
* Once we have been far enough into starting the VM the ID registers
* are immutable. Reject the write if userspace tries to change it.
*/
if (kvm_vm_has_ran_once(kvm))
if (kvm_id_regs_final(kvm))
return -EBUSY;
/*
@ -5862,26 +5859,17 @@ void kvm_calculate_traps(struct kvm_vcpu *vcpu)
}
/*
* Perform last adjustments to the ID registers that are implied by the
* Do system register finalization that is shared by the whole guest. This
* includes last adjustments to the ID registers that are implied by the
* configuration outside of the ID regs themselves, as well as any
* initialisation that directly depend on these ID registers (such as
* RES0/RES1 behaviours). This is not the place to configure traps though.
*
* Because this can be called once per CPU, changes must be idempotent.
*/
int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu)
static int kvm_vm_finalize_sys_regs(struct kvm *kvm)
{
struct kvm *kvm = vcpu->kvm;
lockdep_assert_held(&kvm->arch.config_lock);
guard(mutex)(&kvm->arch.config_lock);
if (vcpu_has_nv(vcpu)) {
int ret = kvm_init_nv_sysregs(vcpu);
if (ret)
return ret;
}
if (kvm_vm_has_ran_once(kvm))
if (kvm_id_regs_final(kvm))
return 0;
/*
@ -5929,12 +5917,37 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu)
kvm_vgic_finalize_idregs(kvm);
}
set_bit(KVM_ARCH_FLAG_ID_REGS_FINAL, &kvm->arch.flags);
return 0;
}
/*
* Because this can be called once per CPU, changes must be idempotent.
*/
int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = vcpu->kvm;
int ret;
guard(mutex)(&kvm->arch.config_lock);
ret = kvm_vm_finalize_sys_regs(kvm);
if (ret)
return ret;
if (vcpu_has_nv(vcpu)) {
ret = kvm_init_nv_sysregs(vcpu);
if (ret)
return ret;
}
return 0;
}
int __init kvm_sys_reg_table_init(void)
{
const struct sys_reg_desc *gicv3_regs;
const struct sys_reg_desc *gicv3_regs, *gicv5_regs;
bool valid = true;
unsigned int i, sz;
int ret = 0;
@ -5947,8 +5960,12 @@ int __init kvm_sys_reg_table_init(void)
valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), false);
valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false);
/* The GICv3 system registers... */
gicv3_regs = vgic_v3_get_sysreg_table(&sz);
valid &= check_sysreg_table(gicv3_regs, sz, false);
/* ...and the GICv5 system registers. */
gicv5_regs = vgic_v5_get_sysreg_table(&sz);
valid &= check_sysreg_table(gicv5_regs, sz, false);
if (!valid)
return -EINVAL;

2
arch/arm64/kvm/sys_regs.h
View File

@ -254,7 +254,7 @@ int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
bool triage_sysreg_trap(struct kvm_vcpu *vcpu, int *sr_index);
int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu);
int kvm_vcpu_finalize_sys_regs(struct kvm_vcpu *vcpu);
#define AA32(_x) .aarch32_map = AA32_##_x
#define Op0(_x) .Op0 = _x

45
arch/arm64/kvm/trace_pkvm.h Normal file
View File

@ -0,0 +1,45 @@
/* SPDX-License-Identifier: GPL-2.0 */
#if !defined(_TRACE_PKVM_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_PKVM_ARM64_KVM_H
#include <linux/tracepoint.h>
#include <asm/kvm_pkvm.h>
#undef TRACE_SYSTEM
#define TRACE_SYSTEM kvm
TRACE_DEFINE_ENUM(PKVM_HYP_NO_REQ);
TRACE_DEFINE_ENUM(PKVM_HYP_REQ_HYP_ALLOC);
#define PKVM_HYP_REQ_TYPES \
{ PKVM_HYP_NO_REQ, "NO_REQ" }, \
{ PKVM_HYP_REQ_HYP_ALLOC, "HYP_ALLOC" }
TRACE_EVENT(kvm_handle_pkvm_hyp_req,
TP_PROTO(struct pkvm_hyp_req *req, int ret),
TP_ARGS(req, ret),
TP_STRUCT__entry(
__field(u8, type)
__field(int, ret)
),
TP_fast_assign(
__entry->type = req->type;
__entry->ret = ret;
),
TP_printk("type: %s ret: %d",
__print_symbolic(__entry->type, PKVM_HYP_REQ_TYPES),
__entry->ret)
);
#endif /* _TRACE_PKVM_ARM64_KVM_H */
#undef TRACE_INCLUDE_PATH
#define TRACE_INCLUDE_PATH .
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE trace_pkvm
/* This part must be outside protection */
#include <trace/define_trace.h>

519
arch/arm64/kvm/vgic-sys-reg-v5.c Normal file
View File

@ -0,0 +1,519 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2025, 2026 Arm Ltd.
*/
/*
* VGICv5 system registers handling functions for AArch64 mode
*/
#include <linux/irqchip/arm-gic-v5.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/wordpart.h>
#include <asm/kvm_emulate.h>
#include "vgic/vgic.h"
#include "sys_regs.h"
#define ICC_PPI_PRIORITYR_PRIORITY_MASK REPEAT_BYTE(0x1f)
static int set_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
/* The upper 32 bits are RES0 */
cpu_if->vgic_apr = val & ~ICC_APR_EL1_RES0;
return 0;
}
static int get_gic_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
*val = cpu_if->vgic_apr;
return 0;
}
static int set_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
/*
* We only support setting the ICC_CR0_EL1.En bit, which is actually
* stored in the VMCR.
*/
FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_EN, &cpu_if->vgic_vmcr,
FIELD_GET(ICC_CR0_EL1_EN, val));
return 0;
}
static int get_gic_cr0(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
/*
* PID only applies if EL3 is present. Same applies to IPPT. Hence,
* those fields are always presented as 0.
*
* We always present the link as connected and idle:
* (LINK = 1, LINK_IDLE = 1).
*/
*val = FIELD_PREP(ICC_CR0_EL1_EN,
FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_EN, cpu_if->vgic_vmcr));
*val |= ICC_CR0_EL1_LINK_MASK;
*val |= ICC_CR0_EL1_LINK_IDLE_MASK;
return 0;
}
static int set_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
/* Set the VPMR field in the VMCR */
FIELD_MODIFY(FEAT_GCIE_ICH_VMCR_EL2_VPMR, &cpu_if->vgic_vmcr,
FIELD_GET(ICC_PCR_EL1_PRIORITY, val));
return 0;
}
static int get_gic_pcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
*val = FIELD_PREP(ICC_PCR_EL1_PRIORITY,
FIELD_GET(FEAT_GCIE_ICH_VMCR_EL2_VPMR, cpu_if->vgic_vmcr));
return 0;
}
static int set_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
cpu_if->vgic_icsr = val & ~ICC_ICSR_EL1_RES0;
return 0;
}
static int get_gic_icsr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
*val = cpu_if->vgic_icsr;
return 0;
}
/*
* Helper macro to iterate over a range of PPIs and execute some code (to either
* extract or set the vgic_irq state). This is used when `get`-ing the PPI
* ENABLER, ACTIVER, PENDR and when setting the PRIORITYR state.
*
* vcpu: Pointer to struct kvm_vcpu (to which these PPIs belong)
* r: The register index. 0 or 1 for all except PRIORITYR (which is 0-15)
* nr: The number of PPIs iterated over. 64 for all but PRIORITYR (which is 8)
* code: The code snippet to execute for each vgic_irq
*/
#define for_ppi_state(vcpu, r, nr, code) \
do { \
struct kvm_vcpu *__vcpu = (vcpu); \
int __r = (r); \
int __nr = (nr); \
\
for (int i = 0; i < __nr; i++) { \
u32 id = vgic_v5_make_ppi(__r * __nr + i); \
struct vgic_irq *irq; \
\
irq = vgic_get_vcpu_irq(__vcpu, id); \
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) { \
code; \
} \
vgic_put_irq(__vcpu->kvm, irq); \
} \
} while (0)
static int set_gic_ppi_enabler(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r, u64 val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
int i, start, end, reg = r->Op2 % 2;
/*
* If we're only handling architected PPIs and the guest writes to the
* enable for the non-architected PPIs, we just return as there's
* nothing to do at all. We don't even allocate the storage for them in
* this case.
*/
if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1)
return 0;
/*
* Merge the raw guest write into our bitmap at an offset of either 0 or
* 64.
*
* Note that there is *NO* masking applied - the enable state is written
* unfiltered. The assumption is that userspace uses this interface to
* set initial state before the guest runs, and then the exposed PPI
* mask is applied later, when vgic_v5_finalize_ppi_state() runs on
* first entry to each vCPU. If userspace chooses to set the enabler
* state later, it is fully capable of breaking the illusion we provided
* to the guest by exposing register state (and PPIs) to the guest that
* were not initially exposed. Good luck!
*/
bitmap_write(cpu_if->vgic_ppi_enabler, val, 64 * reg, 64);
/*
* Sync the change in enable states to the vgic_irqs for the written
* register slice.
*/
start = reg * 64;
end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS);
for (i = start; i < end; i++) {
u32 intid = vgic_v5_make_ppi(i);
struct vgic_irq *irq;
irq = vgic_get_vcpu_irq(vcpu, intid);
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
irq->enabled = test_bit(i, cpu_if->vgic_ppi_enabler);
vgic_put_irq(vcpu->kvm, irq);
}
return 0;
}
static int get_gic_ppi_enabler(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r, u64 *val)
{
unsigned long enabler = 0;
int reg = r->Op2 % 2;
/* If we only support architected PPIs, return 0 */
if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) {
*val = 0;
return 0;
}
/* Iterate over each struct vgic_irq to build the ENABLER value. */
for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &enabler, irq->enabled));
*val = enabler;
return 0;
}
static int set_gic_ppi_activer(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r, u64 val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
int i, start, end, reg = r->Op2 % 2;
if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1)
return 0;
/*
* Store the raw guest write. The exposed PPI mask is applied later,
* when vgic_v5_finalize_ppi_state() runs on first entry to each
* vCPU. See comment on set_gic_ppi_enabler() for details.
*/
bitmap_write(cpu_if->vgic_ppi_activer, val, 64 * reg, 64);
start = reg * 64;
end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS);
for (i = start; i < end; i++) {
u32 intid = vgic_v5_make_ppi(i);
struct vgic_irq *irq;
irq = vgic_get_vcpu_irq(vcpu, intid);
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
irq->active = test_bit(i, cpu_if->vgic_ppi_activer);
vgic_put_irq(vcpu->kvm, irq);
}
return 0;
}
static int get_gic_ppi_activer(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r, u64 *val)
{
unsigned long activer = 0;
int reg = r->Op2 % 2;
/* If we only support architected PPIs, return 0 */
if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) {
*val = 0;
return 0;
}
/* Iterate over each struct vgic_irq to build the ACTIVER value. */
for_ppi_state(vcpu, reg, 64, __assign_bit(i % 64, &activer, irq->active));
*val = activer;
return 0;
}
static int set_gic_ppi_pendr(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r, u64 val)
{
int i, start, end, reg = r->Op2 % 2;
/* If we only support architected PPIs, return */
if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1)
return 0;
/*
* Update each struct vgic_irq with the pending state, treating Level
* and Edge interrupts differently. The exposed PPI mask is applied
* later, when vgic_v5_finalize_ppi_state() runs on first entry to each
* vCPU. See comment on set_gic_ppi_enabler() for details.
*/
start = reg * 64;
end = min(start + 64, VGIC_V5_NR_PRIVATE_IRQS);
for (i = start; i < end; i++) {
u32 intid = vgic_v5_make_ppi(i);
struct vgic_irq *irq;
irq = vgic_get_vcpu_irq(vcpu, intid);
scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
bool level = !!(val & BIT_ULL(i - start));
if (irq->config == VGIC_CONFIG_LEVEL)
irq->line_level = level;
else
irq->pending_latch = level;
}
vgic_put_irq(vcpu->kvm, irq);
}
/*
* The pending state is generated from the vgic_irqs on each guest
* entry. Therefore, we don't store the raw value written anywhere in
* the case of userspace PPI_PENDRx_EL1 writes.
*/
return 0;
}
static int get_gic_ppi_pendr(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r, u64 *val)
{
unsigned long pendr = 0;
int reg = r->Op2 % 2;
/* If we only support architected PPIs, return 0 */
if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg == 1) {
*val = 0;
return 0;
}
/* Iterate over each struct vgic_irq to build the PENDR value. */
for_ppi_state(vcpu, reg, 64, {
if (irq_is_pending(irq))
__assign_bit(i % 64, &pendr, 1);
});
*val = pendr;
return 0;
}
static int set_gic_ppi_priorityr(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r, u64 val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
int reg = ((r->CRm & 0x1) << 3) + r->Op2;
/* If we only support architected PPIs, return */
if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7)
return 0;
val &= ICC_PPI_PRIORITYR_PRIORITY_MASK;
/*
* Although priorities are not regularly synced back to the vgic_irq
* state, they are explicitly synced back here. This is to ensure that
* any pending PPIs are evaluated correctly when first running the guest
* after setting the state.
*/
for_ppi_state(vcpu, reg, 8,
irq->priority = (u8)(val >> (8 * i));
);
/*
* Update the state that will be written to the ICH_PPI_PRIORITYRx_EL2
* on next guest entry.
*/
cpu_if->vgic_ppi_priorityr[reg] = val;
return 0;
}
static int get_gic_ppi_priorityr(struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r, u64 *val)
{
struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
int reg = ((r->CRm & 0x1) << 3) + r->Op2;
/* If we only support architected PPIs, return 0 */
if (VGIC_V5_NR_PRIVATE_IRQS == 64 && reg > 7) {
*val = 0;
return 0;
}
/*
* The priorities are only synced back to the vgic_irq state when the
* vcpu is entering WFI (KVM only needs to know the priorities when
* evaluating if there are pending PPI interrupts for a vcpu). The raw
* register ICH_PPI_PRIORITYRx_EL1 state is simply saved and restored
* blindly. This state is just returned as it contains the most recent
* priorities written by the guest.
*/
*val = cpu_if->vgic_ppi_priorityr[reg];
return 0;
}
/*
* The following registers are NOT supported:
*
* - ICC_HAPR_EL1
* The value of this is directly generated by the GICv5 hardware based on
* the ICC_APR_EL1 when the guest is running.
* - ICC_IAFFIDR_EL1
* The IAFFID for a GICv5 VPE is the same as the VPE ID, which is the index
* into the in-memory VPE Table. This is not configurable, and instead we
* rely on userspace recreating the VPEs in the same order prior to
* restoring guest state.
* - ICC_PPI_CACTIVER<n>_EL1
* Only raw state writes are supported via the S(et) variant.
* - ICC_PPI_CPENDR<n>_EL1
* Only raw state writes are supported via the S(et) variant.
*/
static const struct sys_reg_desc gic_v5_icc_reg_descs[] = {
{ SYS_DESC(SYS_ICC_ICSR_EL1),
.set_user = set_gic_icsr, .get_user = get_gic_icsr, },
{ SYS_DESC(SYS_ICC_PPI_ENABLER0_EL1),
.set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, },
{ SYS_DESC(SYS_ICC_PPI_ENABLER1_EL1),
.set_user = set_gic_ppi_enabler, .get_user = get_gic_ppi_enabler, },
/*
* Only ICC_SACTIVER<n>_EL1 is exposed to the guest. This is treated as
* a *RAW* write of register state for writes.
*/
{ SYS_DESC(SYS_ICC_PPI_SACTIVER0_EL1),
.set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, },
{ SYS_DESC(SYS_ICC_PPI_SACTIVER1_EL1),
.set_user = set_gic_ppi_activer, .get_user = get_gic_ppi_activer, },
/*
* Only ICC_SPENDR<n>_EL1 is exposed to the guest. This is treated as
* a *RAW* write of register state for writes.
*/
{ SYS_DESC(SYS_ICC_PPI_SPENDR0_EL1),
.set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, },
{ SYS_DESC(SYS_ICC_PPI_SPENDR1_EL1),
.set_user = set_gic_ppi_pendr, .get_user = get_gic_ppi_pendr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR0_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR1_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR2_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR3_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR4_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR5_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR6_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR7_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR8_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR9_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR10_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR11_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR12_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR13_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR14_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_PPI_PRIORITYR15_EL1),
.set_user = set_gic_ppi_priorityr, .get_user = get_gic_ppi_priorityr, },
{ SYS_DESC(SYS_ICC_APR_EL1),
.set_user = set_gic_apr, .get_user = get_gic_apr, },
{ SYS_DESC(SYS_ICC_CR0_EL1),
.set_user = set_gic_cr0, .get_user = get_gic_cr0, },
{ SYS_DESC(SYS_ICC_PCR_EL1),
.set_user = set_gic_pcr, .get_user = get_gic_pcr, },
};
const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz)
{
*sz = ARRAY_SIZE(gic_v5_icc_reg_descs);
return gic_v5_icc_reg_descs;
}
static u64 attr_to_id(u64 attr)
{
return ARM64_SYS_REG(FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP0_MASK, attr),
FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP1_MASK, attr),
FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRN_MASK, attr),
FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_CRM_MASK, attr),
FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP2_MASK, attr));
}
int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
{
const struct sys_reg_desc *r;
r = get_reg_by_id(attr_to_id(attr->attr), gic_v5_icc_reg_descs,
ARRAY_SIZE(gic_v5_icc_reg_descs));
if (r && !sysreg_hidden(vcpu, r))
return 0;
return -ENXIO;
}
int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr,
bool is_write)
{
struct kvm_one_reg reg = {
.id = attr_to_id(attr->attr),
.addr = attr->addr,
};
if (is_write)
return kvm_sys_reg_set_user(vcpu, &reg, gic_v5_icc_reg_descs,
ARRAY_SIZE(gic_v5_icc_reg_descs));
else
return kvm_sys_reg_get_user(vcpu, &reg, gic_v5_icc_reg_descs,
ARRAY_SIZE(gic_v5_icc_reg_descs));
}

162
arch/arm64/kvm/vgic/vgic-init.c
View File

@ -84,6 +84,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
!kvm_vgic_global_state.can_emulate_gicv2)
return -ENODEV;
if (kvm_vm_hyp_is_distrusting(kvm) && type != KVM_DEV_TYPE_ARM_VGIC_V3)
return -ENODEV;
/*
* Ensure mutual exclusion with vCPU creation and any vCPU ioctls by:
*
@ -123,19 +125,34 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
goto out_unlock;
}
kvm_for_each_vcpu(i, vcpu, kvm) {
if (vcpu_has_run_once(vcpu))
goto out_unlock;
}
if (kvm_id_regs_final(kvm))
goto out_unlock;
ret = 0;
if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
switch (type) {
case KVM_DEV_TYPE_ARM_VGIC_V2:
kvm->max_vcpus = VGIC_V2_MAX_CPUS;
else if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
break;
case KVM_DEV_TYPE_ARM_VGIC_V3:
kvm->max_vcpus = VGIC_V3_MAX_CPUS;
else if (type == KVM_DEV_TYPE_ARM_VGIC_V5)
kvm->max_vcpus = min(VGIC_V5_MAX_CPUS,
kvm_vgic_global_state.max_gic_vcpus);
break;
case KVM_DEV_TYPE_ARM_VGIC_V5:
kvm->max_vcpus = kvm_vgic_global_state.max_gicv5_vcpus;
break;
}
/*
* KVM_CREATE_VCPU enforces the model-specific limit if the VGIC has
* already been created. Apply the same limit to any existing vCPUs so
* that the result does not depend on the order in which userspace
* creates them.
*/
kvm_for_each_vcpu(i, vcpu, kvm) {
if (vcpu->vcpu_id >= kvm->max_vcpus) {
ret = -E2BIG;
goto out_unlock;
}
}
if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) {
ret = -E2BIG;
@ -154,6 +171,8 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
case KVM_DEV_TYPE_ARM_VGIC_V3:
INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
break;
case KVM_DEV_TYPE_ARM_VGIC_V5:
kvm->arch.vgic.gicv5_vm.vm_id = VGIC_V5_VM_ID_INVAL;
}
/*
@ -168,29 +187,49 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
break;
}
if (ret) {
kvm_for_each_vcpu(i, vcpu, kvm) {
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
kfree(vgic_cpu->private_irqs);
vgic_cpu->private_irqs = NULL;
}
kvm->arch.vgic.vgic_model = 0;
kvm->arch.vgic.in_kernel = false;
goto out_unlock;
}
if (ret)
goto out_free_private_irqs;
if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
kvm->arch.vgic.nassgicap = system_supports_direct_sgis();
/*
* We now know that we have a GICv5. The Arch Timer PPI interrupts may
* have been initialised at this stage, but will have done so assuming
* that we have an older GIC, meaning that the IntIDs won't be
* correct. We init them again, and this time they will be correct.
*/
if (type == KVM_DEV_TYPE_ARM_VGIC_V5)
if (type == KVM_DEV_TYPE_ARM_VGIC_V5) {
/* Allocate a vIRS for GICv5 systems */
kvm->arch.vgic.vgic_v5_irs_data = kzalloc_obj(struct vgic_v5_irs,
GFP_KERNEL_ACCOUNT);
if (!kvm->arch.vgic.vgic_v5_irs_data) {
ret = -ENOMEM;
goto out_free_private_irqs;
}
/*
* Initialization happens later, for now just explicitly
* disable the device and undef its base address.
*/
kvm->arch.vgic.vgic_v5_irs_data->vgic_v5_irs_base = VGIC_ADDR_UNDEF;
/*
* We now know that we have a GICv5. The Arch Timer PPI
* interrupts may have been initialised at this stage, but will
* have done so assuming that we have an older GIC, meaning that
* the IntIDs won't be correct. We init them again, and this
* time they will be correct.
*/
kvm_timer_init_vm(kvm);
}
goto out_unlock;
out_free_private_irqs:
kvm_for_each_vcpu(i, vcpu, kvm) {
struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
kfree(vgic_cpu->private_irqs);
vgic_cpu->private_irqs = NULL;
}
kvm->arch.vgic.in_kernel = false;
kvm->arch.vgic.vgic_model = 0;
out_unlock:
mutex_unlock(&kvm->arch.config_lock);
@ -397,15 +436,27 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
if (ret)
return ret;
/*
* If we are creating a VCPU with a GICv3 we must also register the
* KVM io device for the redistributor that belongs to this VCPU.
*/
if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
switch (dist->vgic_model) {
case KVM_DEV_TYPE_ARM_VGIC_V3:
/*
* If we are creating a VCPU with a GICv3 we must also register
* the KVM io device for the redistributor that belongs to this
* VCPU.
*/
mutex_lock(&vcpu->kvm->slots_lock);
ret = vgic_register_redist_iodev(vcpu);
mutex_unlock(&vcpu->kvm->slots_lock);
break;
case KVM_DEV_TYPE_ARM_VGIC_V5:
/*
* Ensure that it is possible to represent the
* userspace-allocated vcpu_id in the hardware-limited (or
* KVM-capped) VPE table used by GICv5.
*/
if (vcpu->vcpu_id >= kvm_vgic_global_state.max_gicv5_vcpus)
return -EINVAL;
}
return ret;
}
@ -465,6 +516,9 @@ int vgic_init(struct kvm *kvm)
return ret;
}
} else {
if (!dist->nr_spis)
dist->nr_spis = VGIC_V5_DEFAULT_NR_SPIS;
ret = vgic_v5_init(kvm);
if (ret)
return ret;
@ -474,8 +528,12 @@ int vgic_init(struct kvm *kvm)
kvm_vgic_vcpu_reset(vcpu);
ret = kvm_vgic_setup_default_irq_routing(kvm);
if (ret)
if (ret) {
if (vgic_is_v5(kvm))
vgic_v5_teardown(kvm);
return ret;
}
vgic_debug_init(kvm);
dist->initialized = true;
@ -496,16 +554,24 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)
dist->nr_spis = 0;
dist->vgic_dist_base = VGIC_ADDR_UNDEF;
if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
switch (dist->vgic_model) {
case KVM_DEV_TYPE_ARM_VGIC_V2:
dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
break;
case KVM_DEV_TYPE_ARM_VGIC_V3:
list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list)
vgic_v3_free_redist_region(kvm, rdreg);
INIT_LIST_HEAD(&dist->rd_regions);
} else {
dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
}
if (vgic_supports_direct_irqs(kvm))
vgic_v4_teardown(kvm);
if (vgic_supports_direct_irqs(kvm))
vgic_v4_teardown(kvm);
break;
case KVM_DEV_TYPE_ARM_VGIC_V5:
vgic_v5_teardown(kvm);
kfree(dist->vgic_v5_irs_data);
dist->vgic_v5_irs_data = NULL;
break;
}
xa_destroy(&dist->lpi_xa);
}
@ -624,9 +690,8 @@ int vgic_lazy_init(struct kvm *kvm)
int kvm_vgic_map_resources(struct kvm *kvm)
{
struct vgic_dist *dist = &kvm->arch.vgic;
bool needs_dist = true;
enum vgic_type type;
gpa_t dist_base;
gpa_t dist_base, irs_base;
int ret = 0;
if (likely(smp_load_acquire(&dist->ready)))
@ -649,13 +714,12 @@ int kvm_vgic_map_resources(struct kvm *kvm)
} else {
ret = vgic_v5_map_resources(kvm);
type = VGIC_V5;
needs_dist = false;
}
if (ret)
goto out;
if (needs_dist) {
if (type != VGIC_V5) {
dist_base = dist->vgic_dist_base;
mutex_unlock(&kvm->arch.config_lock);
@ -665,7 +729,19 @@ int kvm_vgic_map_resources(struct kvm *kvm)
goto out_slots;
}
} else {
irs_base = dist->vgic_v5_irs_data->vgic_v5_irs_base;
mutex_unlock(&kvm->arch.config_lock);
if (IS_VGIC_ADDR_UNDEF(irs_base)) {
ret = -ENXIO;
goto out_slots;
}
ret = vgic_v5_register_irs_iodev(kvm, irs_base);
if (ret) {
kvm_err("Unable to register VGIC IRS MMIO regions\n");
goto out_slots;
}
}
smp_store_release(&dist->ready, true);

18
arch/arm64/kvm/vgic/vgic-irqfd.c
View File

@ -19,9 +19,14 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e,
struct kvm *kvm, int irq_source_id,
int level, bool line_status)
{
unsigned int spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS;
unsigned int spi_id;
int ret;
if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5)
spi_id = vgic_v5_make_spi(e->irqchip.pin);
else
spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS;
if (!vgic_valid_spi(kvm, spi_id))
return -EINVAL;
@ -45,6 +50,13 @@ int kvm_set_routing_entry(struct kvm *kvm,
struct kvm_kernel_irq_routing_entry *e,
const struct kvm_irq_routing_entry *ue)
{
/*
* The common routing table reserves enough entries for all GICv5 SPI
* pins. KVM limits GICv5 to 1024 SPI pins, while GICv2 and GICv3
* only support SPIs 32..1019, leaving 988 pins.
*/
unsigned int max_irqchip_pins = vgic_is_v5(kvm) ?
VGIC_V5_MAX_NR_SPIS : VGIC_MAX_SPI - VGIC_NR_PRIVATE_IRQS + 1;
int r = -EINVAL;
switch (ue->type) {
@ -52,8 +64,8 @@ int kvm_set_routing_entry(struct kvm *kvm,
e->set = vgic_irqfd_set_irq;
e->irqchip.irqchip = ue->u.irqchip.irqchip;
e->irqchip.pin = ue->u.irqchip.pin;
if ((e->irqchip.pin >= KVM_IRQCHIP_NUM_PINS) ||
(e->irqchip.irqchip >= KVM_NR_IRQCHIPS))
if (e->irqchip.pin >= max_irqchip_pins ||
e->irqchip.irqchip >= KVM_NR_IRQCHIPS)
goto out;
break;
case KVM_IRQ_ROUTING_MSI:

1217
arch/arm64/kvm/vgic/vgic-irs-v5.c Normal file
View File

File diff suppressed because it is too large Load Diff

6
arch/arm64/kvm/vgic/vgic-its.c
View File

@ -2134,9 +2134,9 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,
u64 val;
/*
* MAPC with V=0 keeps the ITEs mapped but drops their collection,
* and with it the ICID. Save a zeroed entry, which the restore path
* reads back as invalid.
* MAPC with V=0 and clearing GITS_BASER<coll>.Valid both keep the
* ITEs mapped while dropping their collection, and with it the ICID.
* Save a zeroed entry, which the restore path reads back as invalid.
*/
if (!ite->collection)
return vgic_its_write_entry_lock(its, gpa, 0ULL, ite);

272
arch/arm64/kvm/vgic/vgic-kvm-device.c
View File

@ -181,6 +181,14 @@ static int kvm_vgic_addr(struct kvm *kvm, struct kvm_device_attr *attr, bool wri
addr |= (u64)rdreg->count << KVM_VGIC_V3_RDIST_COUNT_SHIFT;
goto out;
}
case KVM_VGIC_V5_ADDR_TYPE_IRS:
r = vgic_check_type(kvm, KVM_DEV_TYPE_ARM_VGIC_V5);
if (r)
break;
addr_ptr = &vgic->vgic_v5_irs_data->vgic_v5_irs_base;
alignment = SZ_64K;
size = KVM_VGIC_V5_IRS_SIZE;
break;
default:
r = -ENODEV;
}
@ -224,31 +232,48 @@ static int vgic_set_common_attr(struct kvm_device *dev,
if (get_user(val, uaddr))
return -EFAULT;
/*
* We require:
* - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
* - at most 1024 interrupts
* - a multiple of 32 interrupts
*/
if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
val > VGIC_MAX_RESERVED ||
(val & 31))
return -EINVAL;
if (!vgic_is_v5(dev->kvm)) {
/*
* We require:
* - at least 32 SPIs on top of the 16 SGIs and 16 PPIs
* - at most 1024 interrupts
* - a multiple of 32 interrupts
*/
if (val < (VGIC_NR_PRIVATE_IRQS + 32) ||
val > VGIC_MAX_RESERVED || (val & 31))
return -EINVAL;
mutex_lock(&dev->kvm->arch.config_lock);
mutex_lock(&dev->kvm->arch.config_lock);
/*
* Either userspace has already configured NR_IRQS or
* the vgic has already been initialized and vgic_init()
* supplied a default amount of SPIs.
*/
if (dev->kvm->arch.vgic.nr_spis)
ret = -EBUSY;
else
dev->kvm->arch.vgic.nr_spis =
val - VGIC_NR_PRIVATE_IRQS;
/*
* Either userspace has already configured NR_IRQS or
* the vgic has already been initialized and vgic_init()
* supplied a default amount of SPIs.
*/
if (dev->kvm->arch.vgic.nr_spis)
ret = -EBUSY;
else
dev->kvm->arch.vgic.nr_spis =
val - VGIC_NR_PRIVATE_IRQS;
mutex_unlock(&dev->kvm->arch.config_lock);
mutex_unlock(&dev->kvm->arch.config_lock);
} else {
/*
* GICv5 reports a number of SPIs, not a total number of
* interrupts. Require a multiple of 32 SPIs.
*/
if (val < VGIC_V5_DEFAULT_NR_SPIS ||
val > VGIC_V5_MAX_NR_SPIS ||
(val & 31))
return -EINVAL;
mutex_lock(&dev->kvm->arch.config_lock);
if (vgic_initialized(dev->kvm) || dev->kvm->arch.vgic.nr_spis)
ret = -EBUSY;
else
dev->kvm->arch.vgic.nr_spis = val;
mutex_unlock(&dev->kvm->arch.config_lock);
}
return ret;
}
@ -299,9 +324,14 @@ static int vgic_get_common_attr(struct kvm_device *dev,
return (r == -ENODEV) ? -ENXIO : r;
case KVM_DEV_ARM_VGIC_GRP_NR_IRQS: {
u32 __user *uaddr = (u32 __user *)(long)attr->addr;
r = put_user(dev->kvm->arch.vgic.nr_spis +
VGIC_NR_PRIVATE_IRQS, uaddr);
/* Older GICs */
if (!vgic_is_v5(dev->kvm)) {
r = put_user(dev->kvm->arch.vgic.nr_spis +
VGIC_NR_PRIVATE_IRQS,
uaddr);
} else {
r = put_user(dev->kvm->arch.vgic.nr_spis, uaddr);
}
break;
}
}
@ -512,7 +542,7 @@ int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
* Allow access to certain ID-like registers prior to VGIC initialization,
* thereby allowing the VMM to provision the features / sizing of the VGIC.
*/
static bool reg_allowed_pre_init(struct kvm_device_attr *attr)
static bool v3_reg_allowed_pre_init(struct kvm_device_attr *attr)
{
if (attr->group != KVM_DEV_ARM_VGIC_GRP_DIST_REGS)
return false;
@ -575,7 +605,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
mutex_lock(&dev->kvm->arch.config_lock);
if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) {
if (!(vgic_initialized(dev->kvm) || v3_reg_allowed_pre_init(attr))) {
ret = -EBUSY;
goto out;
}
@ -743,26 +773,178 @@ static int vgic_v5_get_userspace_ppis(struct kvm_device *dev,
return ret;
}
int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
struct vgic_reg_attr *reg_attr)
{
unsigned long vgic_mpidr, mpidr_reg;
switch (attr->group) {
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
vgic_mpidr = (attr->attr & KVM_DEV_ARM_VGIC_V3_MPIDR_MASK) >>
KVM_DEV_ARM_VGIC_V3_MPIDR_SHIFT;
mpidr_reg = VGIC_TO_MPIDR(vgic_mpidr);
reg_attr->vcpu = kvm_mpidr_to_vcpu(dev->kvm, mpidr_reg);
break;
case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
reg_attr->vcpu = kvm_get_vcpu(dev->kvm, 0);
break;
default:
return -EINVAL;
}
if (!reg_attr->vcpu)
return -EINVAL;
reg_attr->addr = attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK;
return 0;
}
/*
* Some registers can potentially be read before the core GIC & IRS has been
* initialised. Right now, everything is required to be post-init.
*/
static bool v5_reg_allowed_pre_init(struct kvm_device_attr *attr)
{
return false;
}
static bool vgic_v5_vm_has_run_once(struct kvm *kvm)
{
struct kvm_vcpu *vcpu;
unsigned long i;
kvm_for_each_vcpu(i, vcpu, kvm) {
if (vcpu_has_run_once(vcpu))
return true;
}
return false;
}
/*
* vgic_v5_attr_regs_access - allows user space to access VGIC v5 state
*
* @dev: kvm device handle
* @attr: kvm device attribute
* @is_write: true if userspace is writing a register
*/
static int vgic_v5_attr_regs_access(struct kvm_device *dev,
struct kvm_device_attr *attr,
bool is_write)
{
u64 __user *uaddr = (u64 __user *)(unsigned long)attr->addr;
struct vgic_reg_attr reg_attr;
struct kvm_vcpu *vcpu;
bool uaccess;
u64 val;
int ret;
ret = vgic_v5_parse_attr(dev, attr, &reg_attr);
if (ret)
return ret;
vcpu = reg_attr.vcpu;
switch (attr->group) {
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
/* Sysregs uaccess is performed by the sysreg handling code */
uaccess = false;
break;
case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
fallthrough;
default:
uaccess = true;
}
if (uaccess && is_write) {
if (get_user(val, uaddr))
return -EFAULT;
}
mutex_lock(&dev->kvm->lock);
if (kvm_trylock_all_vcpus(dev->kvm)) {
mutex_unlock(&dev->kvm->lock);
return -EBUSY;
}
mutex_lock(&dev->kvm->arch.config_lock);
if (!(vgic_initialized(dev->kvm) || v5_reg_allowed_pre_init(attr))) {
ret = -EBUSY;
goto out;
}
if (is_write && vgic_v5_vm_has_run_once(dev->kvm)) {
ret = -EBUSY;
goto out;
}
switch (attr->group) {
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
ret = vgic_v5_cpu_sysregs_uaccess(vcpu, attr, is_write);
break;
case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
/*
* The IRS registers are a mixture of 32-bit and 64-bit
* registers. Internally, we always perform the correctly sized
* access, but the UAPI is defined in such a way that we are
* always provided a __u64 by userspace. When userspace writes,
* the upper 32-bits are ignored for 32-bit accesses, and on a
* read any 32-bit accesses are written back to user memory
* using the full 64-bits.
*/
ret = vgic_v5_irs_attr_regs_access(dev, attr, &val, is_write);
break;
default:
ret = -EINVAL;
break;
}
out:
mutex_unlock(&dev->kvm->arch.config_lock);
kvm_unlock_all_vcpus(dev->kvm);
mutex_unlock(&dev->kvm->lock);
if (!ret && uaccess && !is_write)
ret = put_user(val, uaddr);
return ret;
}
static int vgic_v5_set_attr(struct kvm_device *dev,
struct kvm_device_attr *attr)
{
switch (attr->group) {
case KVM_DEV_ARM_VGIC_GRP_ADDR:
break;
case KVM_DEV_ARM_VGIC_GRP_IST:
if (attr->attr)
return -ENXIO;
return vgic_v5_irs_restore_ists(dev->kvm, attr);
case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
fallthrough;
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
return vgic_v5_attr_regs_access(dev, attr, true);
case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
return -ENXIO;
break;
case KVM_DEV_ARM_VGIC_GRP_CTRL:
switch (attr->attr) {
case KVM_DEV_ARM_VGIC_CTRL_INIT:
return vgic_set_common_attr(dev, attr);
break;
case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
default:
return -ENXIO;
}
break;
default:
return -ENXIO;
}
return vgic_set_common_attr(dev, attr);
}
static int vgic_v5_get_attr(struct kvm_device *dev,
@ -770,21 +952,33 @@ static int vgic_v5_get_attr(struct kvm_device *dev,
{
switch (attr->group) {
case KVM_DEV_ARM_VGIC_GRP_ADDR:
break;
case KVM_DEV_ARM_VGIC_GRP_IST:
if (attr->attr)
return -ENXIO;
return vgic_v5_irs_save_ists(dev->kvm, attr);
case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
fallthrough;
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
return vgic_v5_attr_regs_access(dev, attr, false);
case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
return -ENXIO;
break;
case KVM_DEV_ARM_VGIC_GRP_CTRL:
switch (attr->attr) {
case KVM_DEV_ARM_VGIC_CTRL_INIT:
return vgic_get_common_attr(dev, attr);
break;
case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
return vgic_v5_get_userspace_ppis(dev, attr);
default:
return -ENXIO;
}
break;
default:
return -ENXIO;
}
return vgic_get_common_attr(dev, attr);
}
static int vgic_v5_has_attr(struct kvm_device *dev,
@ -792,18 +986,30 @@ static int vgic_v5_has_attr(struct kvm_device *dev,
{
switch (attr->group) {
case KVM_DEV_ARM_VGIC_GRP_ADDR:
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
switch (attr->attr) {
case KVM_VGIC_V5_ADDR_TYPE_IRS:
return 0;
}
return -ENXIO;
case KVM_DEV_ARM_VGIC_GRP_IRS_REGS:
fallthrough;
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
return vgic_v5_has_attr_regs(dev, attr);
case KVM_DEV_ARM_VGIC_GRP_NR_IRQS:
return 0;
case KVM_DEV_ARM_VGIC_GRP_CTRL:
switch (attr->attr) {
case KVM_DEV_ARM_VGIC_CTRL_INIT:
return 0;
case KVM_DEV_ARM_VGIC_USERSPACE_PPIS:
return 0;
case KVM_DEV_ARM_VGIC_SAVE_PENDING_TABLES:
default:
return -ENXIO;
}
break;
case KVM_DEV_ARM_VGIC_GRP_IST:
return attr->attr ? -ENXIO : 0;
default:
return -ENXIO;
}

33
arch/arm64/kvm/vgic/vgic-mmio-v3.c
View File

@ -818,16 +818,21 @@ int vgic_register_redist_iodev(struct kvm_vcpu *vcpu)
rd_dev->nr_regions = ARRAY_SIZE(vgic_v3_rd_registers);
rd_dev->redist_vcpu = vcpu;
/* Protected by slots_lock */
rdreg->free_index++;
mutex_unlock(&kvm->arch.config_lock);
ret = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, rd_base,
2 * SZ_64K, &rd_dev->dev);
if (ret)
return ret;
if (ret) {
guard(mutex)(&vcpu->kvm->arch.config_lock);
vgic_cpu->rdreg->free_index--;
vgic_cpu->rdreg = NULL;
vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
}
/* Protected by slots_lock */
rdreg->free_index++;
return 0;
return ret;
out_unlock:
mutex_unlock(&kvm->arch.config_lock);
@ -841,7 +846,8 @@ void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu)
kvm_io_bus_unregister_dev(vcpu->kvm, KVM_MMIO_BUS, &rd_dev->dev);
}
static int vgic_register_all_redist_iodevs(struct kvm *kvm)
static int vgic_register_all_redist_iodevs(struct kvm *kvm,
struct vgic_redist_region *rdreg)
{
struct kvm_vcpu *vcpu;
unsigned long c;
@ -861,7 +867,14 @@ static int vgic_register_all_redist_iodevs(struct kvm *kvm)
for (i = 0; i < c; i++) {
vcpu = kvm_get_vcpu(kvm, i);
if (vcpu->arch.vgic_cpu.rdreg != rdreg)
continue;
vgic_unregister_redist_iodev(vcpu);
guard(mutex)(&vcpu->kvm->arch.config_lock);
vcpu->arch.vgic_cpu.rdreg = NULL;
vcpu->arch.vgic_cpu.rd_iodev.base_addr = VGIC_ADDR_UNDEF;
}
}
@ -970,10 +983,13 @@ void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdre
int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count)
{
struct vgic_redist_region *rdreg;
int ret;
mutex_lock(&kvm->arch.config_lock);
ret = vgic_v3_alloc_redist_region(kvm, index, addr, count);
if (!ret)
rdreg = vgic_v3_rdist_region_from_index(kvm, index);
mutex_unlock(&kvm->arch.config_lock);
if (ret)
return ret;
@ -982,12 +998,9 @@ int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count)
* Register iodevs for each existing VCPU. Adding more VCPUs
* afterwards will register the iodevs when needed.
*/
ret = vgic_register_all_redist_iodevs(kvm);
ret = vgic_register_all_redist_iodevs(kvm, rdreg);
if (ret) {
struct vgic_redist_region *rdreg;
mutex_lock(&kvm->arch.config_lock);
rdreg = vgic_v3_rdist_region_from_index(kvm, index);
vgic_v3_free_redist_region(kvm, rdreg);
mutex_unlock(&kvm->arch.config_lock);
return ret;

6
arch/arm64/kvm/vgic/vgic-mmio.c
View File

@ -1065,6 +1065,9 @@ static int dispatch_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
case IODEV_ITS:
data = region->its_read(vcpu->kvm, iodev->its, addr, len);
break;
case IODEV_GICV5_IRS:
data = region->read(vcpu, addr, len);
break;
}
vgic_data_host_to_mmio_bus(val, len, data);
@ -1095,6 +1098,9 @@ static int dispatch_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev,
case IODEV_ITS:
region->its_write(vcpu->kvm, iodev->its, addr, len, data);
break;
case IODEV_GICV5_IRS:
region->write(vcpu, addr, len, data);
break;
}
return 0;

2
arch/arm64/kvm/vgic/vgic-mmio.h
View File

@ -217,6 +217,8 @@ unsigned int vgic_v2_init_cpuif_iodev(struct vgic_io_device *dev);
unsigned int vgic_v3_init_dist_iodev(struct vgic_io_device *dev);
unsigned int vgic_v5_init_irs_iodev(struct vgic_io_device *dev);
u64 vgic_sanitise_outer_cacheability(u64 reg);
u64 vgic_sanitise_inner_cacheability(u64 reg);
u64 vgic_sanitise_shareability(u64 reg);

1872
arch/arm64/kvm/vgic/vgic-v5-tables.c Normal file
View File

File diff suppressed because it is too large Load Diff

140
arch/arm64/kvm/vgic/vgic-v5-tables.h Normal file
View File

@ -0,0 +1,140 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2025, 2026 Arm Ltd.
*/
#ifndef __KVM_ARM_VGICV5_TABLES_H__
#define __KVM_ARM_VGICV5_TABLES_H__
#include <linux/spinlock.h>
#include <linux/idr.h>
#include <linux/irqchip/arm-gic-v5.h>
#include <linux/list.h>
/* Level 1 Virtual Machine Table Entry */
typedef __le64 vmtl1_entry;
/* Level 2 Virtual Machine Table Entry */
struct vmtl2_entry {
__le64 val[4];
};
/* Virtual PE Table Entry */
typedef __le64 vpe_entry;
struct vgic_v5_vm_info {
void *vmd_base;
vpe_entry *vpet_base;
void *vped_base;
u8 vpe_id_bits;
/*
* Both the LPI and SPI ISTs are allocated by the hypervisor. While it
* would be possible to track and access them by iterating over the ISTs
* themselves, it makes more sense to store pointers to the ISTs.
*
* The LPI IST can either be two-level or linear. Hence, we keep track
* of the structure. If it is two-level, we retain pointers to the L1
* IST and to each L2 IST array. If it is linear, we just store the base
* address of the IST array.
*
* The SPI IST is linear, and therefore we just store the base address
* of the SPI IST array.
*/
bool h_lpi_ist_structure;
__le64 *h_lpi_ist;
__le64 **h_lpi_l2_ists;
__le64 *h_spi_ist;
/* Tracking of pending interrupts as part of IST restore */
struct list_head pending_irqs;
};
struct vgic_v5_vmt {
union {
struct {
struct vmtl2_entry *vmt_base;
unsigned int num_ents;
} linear;
struct {
vmtl1_entry *vmt_base;
struct vmtl2_entry **l2ptrs;
unsigned int num_l1_ents;
} l2;
};
bool two_level;
unsigned int num_entries;
unsigned int max_vpes;
size_t vmd_size;
size_t vped_size;
struct ida vm_id_ida;
};
static inline u32 vgic_v5_vm_id(struct kvm *kvm)
{
return kvm->arch.vgic.gicv5_vm.vm_id;
}
/*
* For vGICv5, we need to consolidate two views:
* - The vcpu ID assigned by userspace, which it can assign however it
* sees fit.
* - The index into the VPET, which is also presented to the guest as
* the IAFFID for the VPE in question.
*
* Ideally, we don't want to intercept the guest setting an interrupt's affinity
* - this would somewhat defeat the point of allowing the hardware to manage the
* interrupt directly. Therefore, we need to use vcpu->vcpu_id as our VPE ID,
* which ensures that both userspace and the GICv5 hardware have a consistent
* view.
*
* This means two things:
* - Our VPET might be sparse, depending on the IDs picked by userspace.
* - We need to check the vcpu ID provided by userspace, and reject anything
* we cannot handle.
*
* In order to at least ensure consistent usage in the places that we need to
* use the VPE ID, we provide this helper.
*/
static inline u16 vgic_v5_vpe_id(struct kvm_vcpu *vcpu)
{
return vcpu->vcpu_id;
}
static inline int vgic_v5_vpe_db(struct kvm_vcpu *vcpu)
{
return vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db;
}
extern raw_spinlock_t vgic_v5_irs_lock;
int vgic_v5_vmt_allocate(unsigned int max_vpes);
int vgic_v5_vmt_free(void);
phys_addr_t vgic_v5_get_vmt_base(void);
int vgic_v5_allocate_vm_id(struct kvm *kvm);
void vgic_v5_release_vm_id(struct kvm *kvm);
int vgic_v5_vmte_init(struct kvm *kvm);
int vgic_v5_vmte_release(struct kvm *kvm);
u8 vgic_v5_vmte_vpe_id_bits(struct kvm_vcpu *vcpu);
int vgic_v5_vmte_alloc_vpe(struct kvm_vcpu *vcpu);
int vgic_v5_vmte_free_vpe(struct kvm_vcpu *vcpu);
int vgic_v5_spi_ist_alloc(struct kvm *kvm, unsigned int id_bits);
int vgic_v5_lpi_ist_exists(struct kvm *kvm);
int vgic_v5_lpi_ist_alloc(struct kvm *kvm, unsigned int id_bits);
int vgic_v5_lpi_ist_free(struct kvm *kvm);
int vgic_v5_save_spi_ist(struct kvm *kvm,
struct kvm_vgic_v5_ist *ist_attr);
int vgic_v5_save_lpi_ist(struct kvm *kvm,
struct kvm_vgic_v5_ist *ist_attr);
int vgic_v5_restore_spi_ist(struct kvm *kvm,
struct kvm_vgic_v5_ist *ist_attr);
int vgic_v5_restore_lpi_ist(struct kvm *kvm,
struct kvm_vgic_v5_ist *ist_attr);
int vgic_v5_restore_pending_irqs(struct kvm *kvm);
void vgic_v5_discard_pending_irqs(struct kvm *kvm);
#endif

1152
arch/arm64/kvm/vgic/vgic-v5.c
View File

File diff suppressed because it is too large Load Diff

39
arch/arm64/kvm/vgic/vgic.c
View File

@ -86,20 +86,32 @@ static struct vgic_irq *vgic_get_lpi(struct kvm *kvm, u32 intid)
*/
struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid)
{
/* Non-private IRQs are not yet implemented for GICv5 */
if (vgic_is_v5(kvm))
return NULL;
enum kvm_device_type type = kvm->arch.vgic.vgic_model;
/* SPIs */
if (intid >= VGIC_NR_PRIVATE_IRQS &&
intid < (kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) {
intid -= VGIC_NR_PRIVATE_IRQS;
intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis);
return &kvm->arch.vgic.spis[intid];
if (__irq_is_spi(type, intid)) {
switch (type) {
case KVM_DEV_TYPE_ARM_VGIC_V5:
intid = vgic_v5_get_hwirq_id(intid);
if (intid >= kvm->arch.vgic.nr_spis)
return NULL;
intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis);
return &kvm->arch.vgic.spis[intid];
default: {
u32 max_intid = kvm->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS;
if (intid < max_intid) {
intid -= VGIC_NR_PRIVATE_IRQS;
intid = array_index_nospec(intid, kvm->arch.vgic.nr_spis);
return &kvm->arch.vgic.spis[intid];
}
}}
}
/* LPIs */
if (irq_is_lpi(kvm, intid))
if (__irq_is_lpi(type, intid))
return vgic_get_lpi(kvm, intid);
return NULL;
@ -565,6 +577,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
else
irq->pending_latch = true;
if (irq->ops && irq->ops->set_pending_state)
WARN_ON_ONCE(!irq->ops->set_pending_state(vcpu, irq));
vgic_queue_irq_unlock(kvm, irq, flags);
vgic_put_irq(kvm, irq);
@ -1247,8 +1262,12 @@ int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu)
unsigned long flags;
struct vgic_vmcr vmcr;
if (vgic_is_v5(vcpu->kvm))
if (vgic_is_v5(vcpu->kvm)) {
if (READ_ONCE(vcpu->arch.vgic_cpu.vgic_v5.gicv5_vpe.db_fired))
return true;
return vgic_v5_has_pending_ppi(vcpu);
}
if (!vcpu->kvm->arch.vgic.enabled)
return false;

21
arch/arm64/kvm/vgic/vgic.h
View File

@ -269,6 +269,8 @@ struct ap_list_summary {
#define irqs_active_outside_lrs(s) \
((s)->nr_act && irqs_outside_lrs(s))
int vgic_v5_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
struct vgic_reg_attr *reg_attr);
int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
struct vgic_reg_attr *reg_attr);
int vgic_v2_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
@ -383,8 +385,14 @@ void vgic_debug_destroy(struct kvm *kvm);
int vgic_v5_probe(const struct gic_kvm_info *info);
void vgic_v5_reset(struct kvm_vcpu *vcpu);
int vgic_v5_init(struct kvm *kvm);
int kvm_vgic_v5_irs_init(struct kvm *kvm, unsigned int nr_spis);
void vgic_v5_teardown(struct kvm *kvm);
int vgic_v5_map_resources(struct kvm *kvm);
void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid);
void vgic_v5_set_spi_ops(struct vgic_irq *irq);
bool vgic_v5_spi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
unsigned long flags);
void vgic_v5_set_irq_pend(struct kvm_vcpu *vcpu, struct vgic_irq *irq);
bool vgic_v5_has_pending_ppi(struct kvm_vcpu *vcpu);
void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu);
void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu);
@ -394,6 +402,19 @@ void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
void vgic_v5_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
void vgic_v5_restore_state(struct kvm_vcpu *vcpu);
void vgic_v5_save_state(struct kvm_vcpu *vcpu);
int vgic_v5_register_irs_iodev(struct kvm *kvm, gpa_t irs_base_address);
int vgic_v5_irs_lpi_ist_id_bits(struct kvm *kvm, unsigned int *id_bits);
int vgic_v5_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr, bool is_write);
int vgic_v5_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
const struct sys_reg_desc *vgic_v5_get_sysreg_table(unsigned int *sz);
int vgic_v5_irs_save_ists(struct kvm *kvm, struct kvm_device_attr *attr);
int vgic_v5_irs_restore_ists(struct kvm *kvm, struct kvm_device_attr *attr);
int vgic_v5_irs_attr_regs_access(struct kvm_device *dev,
struct kvm_device_attr *attr,
u64 *reg, bool is_write);
int vgic_v5_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr);
#define for_each_visible_v5_ppi(__i, __k) \
for_each_set_bit(__i, (__k)->arch.vgic.gicv5_vm.vgic_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS)

51
arch/arm64/tools/sysreg
View File

@ -401,6 +401,57 @@ Res0 3:1
Field 0 SO
EndSysreg
SysregFields SPSR_ELx
Res0 63:37
Field 36 UINJ
Field 35 PACM
Field 34 EXLOCK
Res0 33
Field 32 PM
Field 31 N
Field 30 Z
Field 29 C
Field 28 V
Res0 27:26
Field 25 TCO
Field 24 DIT
Field 23 UAO
Field 22 PAN
Field 21 SS
Field 20 IL
Res0 19:14
Field 13 ALLINT
Field 12 SSBS
Field 11:10 BTYPE
Field 9 D
Field 8 A
Field 7 I
Field 6 F
Res0 5
Field 4 M4
Enum 3:0 M
0b0000 EL0
0b0100 EL1t
0b0101 EL1h
0b1000 EL2t
0b1001 EL2h
0b1100 EL3t
0b1101 EL3h
EndEnum
EndSysregFields
Sysreg SPSR_EL1 3 0 4 0 0
Fields SPSR_ELx
EndSysreg
Sysreg SPSR_EL2 3 4 4 0 0
Fields SPSR_ELx
EndSysreg
Sysreg SPSR_EL12 3 5 4 0 0
Mapping SPSR_EL1
EndSysreg
Sysreg ID_PFR0_EL1 3 0 0 1 0
Res0 63:32
UnsignedEnum 31:28 RAS

2
arch/powerpc/kvm/book3s_hv.c
View File

@ -269,7 +269,7 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
* stolen time for a vcore when it is inactive, or for a vcpu
* when it is in state RUNNING or NOTREADY. NOTREADY is a bit of
* a misnomer; it means that the vcpu task is not executing in
* the KVM_VCPU_RUN ioctl, i.e. it is in userspace or elsewhere in
* the KVM_RUN ioctl, i.e. it is in userspace or elsewhere in
* the kernel. We don't have any way of dividing up that time
* between time that the vcpu is genuinely stopped, time that
* the task is actively working on behalf of the vcpu, and time

11
arch/s390/kvm/s390/s390.c
View File

@ -5783,6 +5783,14 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old,
{
}
int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu)
{
if (kvm_is_ucontrol(vcpu->kvm))
return -EINVAL;
return 0;
}
/**
* kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables
* @vcpu: the vcpu that shall appear to have generated the fault-in.
@ -5809,9 +5817,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_
gpa_t end;
int rc;
if (kvm_is_ucontrol(vcpu->kvm))
return -EINVAL;
rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
if (rc == PGM_ADDRESSING)
return -ENOENT;

11
arch/x86/kvm/mmu/mmu.c
View File

@ -5086,6 +5086,14 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa,
}
}
int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu)
{
if (!vcpu->kvm->arch.pre_fault_allowed)
return -EOPNOTSUPP;
return 0;
}
long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
struct kvm_pre_fault_memory *range)
{
@ -5095,9 +5103,6 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu,
u64 end;
int r;
if (!vcpu->kvm->arch.pre_fault_allowed)
return -EOPNOTSUPP;
if (kvm_is_gfn_alias(vcpu->kvm, gpa_to_gfn(range->gpa)))
return -EINVAL;

2
arch/x86/kvm/vmx/tdx.c
View File

@ -1708,7 +1708,7 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve
WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK);
/*
* Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory()
* Ensure pre_fault_allowed is read by kvm_arch_pre_fault_allowed()
* before kvm_tdx->state. Userspace must not be allowed to pre-fault
* arbitrary memory until the initial memory image is finalized. Pairs
* with the smp_wmb() in tdx_td_finalize().

19
drivers/irqchip/irq-gic-v5-irs.c
View File

@ -21,8 +21,6 @@
*/
#define LPI_ID_BITS_LINEAR 12
#define IRS_FLAGS_NON_COHERENT BIT(0)
static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data);
static LIST_HEAD(irs_nodes);
@ -50,6 +48,11 @@ static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data,
writeq_relaxed(val, irs_data->irs_base + reg_offset);
}
struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void)
{
return per_cpu(per_cpu_irs_data, 0);
}
/*
* The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register
* provides the memory barriers (through MMIO accessors)
@ -266,24 +269,24 @@ int gicv5_irs_iste_alloc(const u32 lpi)
* itself is not supported) again serves to make it easier to find physically
* contiguous blocks of memory.
*/
static unsigned int gicv5_irs_l2_sz(u32 idr2)
unsigned int gicv5_irs_l2_sz(u32 l2sz)
{
switch (PAGE_SIZE) {
case SZ_64K:
if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2))
if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(l2sz))
return GICV5_IRS_IST_CFGR_L2SZ_64K;
fallthrough;
case SZ_4K:
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz))
return GICV5_IRS_IST_CFGR_L2SZ_4K;
fallthrough;
case SZ_16K:
if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2))
if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(l2sz))
return GICV5_IRS_IST_CFGR_L2SZ_16K;
break;
}
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(l2sz))
return GICV5_IRS_IST_CFGR_L2SZ_4K;
return GICV5_IRS_IST_CFGR_L2SZ_64K;
@ -331,7 +334,7 @@ static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data)
lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits);
if (two_levels)
l2sz = gicv5_irs_l2_sz(idr2);
l2sz = gicv5_irs_l2_sz(FIELD_GET(GICV5_IRS_IDR2_IST_L2SZ, idr2));
istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);

111
drivers/irqchip/irq-gic-v5.c
View File

@ -1122,8 +1122,10 @@ static void gicv5_set_cpuif_idbits(void)
#ifdef CONFIG_KVM
static struct gic_kvm_info gic_v5_kvm_info __initdata;
static void __init gic_of_setup_kvm_info(struct device_node *node)
static int __init gic_setup_kvm_info(unsigned int maint_irq)
{
struct gicv5_irs_chip_data *irs_data = gicv5_irs_get_chip_data();
/*
* If we don't have native GICv5 virtualisation support, then
* we also don't have FEAT_GCIE_LEGACY - the architecture
@ -1131,25 +1133,120 @@ static void __init gic_of_setup_kvm_info(struct device_node *node)
*/
if (!gicv5_global_data.virt_capable) {
pr_info("GIC implementation is not virtualization capable\n");
return;
return -ENODEV;
}
if (WARN_ON(!irs_data))
return -ENODEV;
gic_v5_kvm_info.type = GIC_V5;
gic_v5_kvm_info.gicv5_irs.base = irs_data->irs_base;
gic_v5_kvm_info.gicv5_irs.non_coherent = !!(irs_data->flags & IRS_FLAGS_NON_COHERENT);
/* GIC Virtual CPU interface maintenance interrupt */
gic_v5_kvm_info.no_maint_irq_mask = false;
gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0);
if (!gic_v5_kvm_info.maint_irq) {
pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n");
gic_v5_kvm_info.maint_irq = maint_irq;
/*
* We require an MI if we have legacy support, but don't, otherwise.
* Given that there's an existing flag to convey that an MI isn't
* needed, we (ab)use it to tell KVM that the MI isn't needed if we
* don't support legacy.
*
* The check for ARM64_HAS_GICV5_LEGACY explicitly doesn't use
* cpus_have_final_cap() here as we run too early.
*/
if (!cpus_have_cap(ARM64_HAS_GICV5_LEGACY) && !gic_v5_kvm_info.maint_irq)
gic_v5_kvm_info.no_maint_irq_mask = true;
vgic_set_kvm_info(&gic_v5_kvm_info);
return 0;
}
static void __init gic_of_setup_kvm_info(struct device_node *node)
{
unsigned int maint_irq;
/* GIC Virtual CPU interface maintenance interrupt */
maint_irq = irq_of_parse_and_map(node, 0);
if (gic_setup_kvm_info(maint_irq))
irq_dispose_mapping(maint_irq);
}
#ifdef CONFIG_ACPI
struct gicv5_acpi_kvm_info {
u32 maint_irq;
bool valid;
};
static struct gicv5_acpi_kvm_info acpi_v5_kvm_info __initdata;
static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header,
const unsigned long end)
{
struct acpi_madt_generic_interrupt *gicc =
(struct acpi_madt_generic_interrupt *)header;
if (!(gicc->flags &
(ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE)))
return 0;
if (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE)
pr_warn_once(FW_BUG "MI wrongly advertised as Edge-triggered\n");
if (!acpi_v5_kvm_info.valid) {
acpi_v5_kvm_info.valid = true;
acpi_v5_kvm_info.maint_irq = gicc->vgic_interrupt;
return 0;
}
if (acpi_v5_kvm_info.maint_irq != gicc->vgic_interrupt)
pr_warn_once(FW_BUG "MI not the same on all CPUs\n");
return 0;
}
static bool __init gic_acpi_collect_virt_info(void)
{
acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
gic_acpi_parse_virt_madt_gicc, 0);
return acpi_v5_kvm_info.valid;
}
static void __init gic_acpi_setup_kvm_info(void)
{
int irq = 0;
if (!gic_acpi_collect_virt_info()) {
pr_warn("Unable to get hardware information used for virtualization\n");
return;
}
vgic_set_kvm_info(&gic_v5_kvm_info);
if (acpi_v5_kvm_info.maint_irq) {
irq = acpi_register_gsi(NULL, acpi_v5_kvm_info.maint_irq,
ACPI_LEVEL_SENSITIVE,
ACPI_ACTIVE_HIGH);
if (irq <= 0) {
pr_warn("Failed to register GSI for GICv5 maintenance IRQ\n");
irq = 0;
}
}
if (gic_setup_kvm_info(irq) && irq)
acpi_unregister_gsi(acpi_v5_kvm_info.maint_irq);
}
#endif // CONFIG_ACPI
#else
static inline void __init gic_of_setup_kvm_info(struct device_node *node)
{
}
#ifdef CONFIG_ACPI
static inline void __init gic_acpi_setup_kvm_info(void)
{
}
#endif
#endif // CONFIG_KVM
static int __init gicv5_init_common(struct fwnode_handle *parent_domain)
@ -1259,6 +1356,8 @@ static int __init gic_acpi_init(union acpi_subtable_headers *header, const unsig
acpi_set_irq_model(ACPI_IRQ_MODEL_GIC_V5, gic_v5_get_gsi_domain_id,
gic_v5_get_gsi_handle);
gic_acpi_setup_kvm_info();
return 0;
out_irs:

15
include/kvm/arm_arch_timer.h
View File

@ -195,4 +195,19 @@ static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset)
WRITE_ONCE(*ctxt->offset.vm_offset, offset);
}
/*
* CVAL to program when the hardware won't apply the timer's offset, so that
* its compare against the raw counter matches the guest's at 'now'. An
* expired timer stays expired; one past the wrap never fires.
*/
static inline u64 timer_apply_offset(u64 cval, u64 offset, u64 now)
{
u64 hw = cval + offset;
if (now - offset >= cval)
return 0;
return hw > now ? hw : U64_MAX;
}
#endif

2
include/kvm/arm_psci.h
View File

@ -10,6 +10,8 @@
#include <linux/kvm_host.h>
#include <uapi/linux/psci.h>
#include <asm/kvm_emulate.h>
#define KVM_ARM_PSCI_0_1 PSCI_VERSION(0, 1)
#define KVM_ARM_PSCI_0_2 PSCI_VERSION(0, 2)
#define KVM_ARM_PSCI_1_0 PSCI_VERSION(1, 0)

Some files were not shown because too many files have changed in this diff Show More