diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index f39c311fd756..0474604ab8a1 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -5,7 +5,7 @@ ccflags-$(CONFIG_KVM_WERROR) += -Werror include $(srctree)/virt/kvm/Makefile.kvm -kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o regs.o \ +kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o msrs.o pmu.o regs.o \ mtrr.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c new file mode 100644 index 000000000000..07f2a22d2607 --- /dev/null +++ b/arch/x86/kvm/msrs.c @@ -0,0 +1,2732 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include + +#include "hyperv.h" +#include "lapic.h" +#include "msrs.h" +#include "pmu.h" +#include "trace.h" +#include "vmx/vmx.h" +#include "xen.h" +#include "x86.h" + +bool __read_mostly ignore_msrs = 0; +module_param(ignore_msrs, bool, 0644); + +bool __read_mostly report_ignored_msrs = true; +module_param(report_ignored_msrs, bool, 0644); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs); + +/* EFER defaults: + * - enable syscall per default because its emulated by KVM + * - enable LME and LMA per default on 64 bit KVM + */ +#ifdef CONFIG_X86_64 +static +u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA)); +#else +static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); +#endif + +#define MAX_IO_MSRS 256 + +/* + * Restoring the host value for MSRs that are only consumed when running in + * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU + * returns to userspace, i.e. the kernel can run with the guest's value. + */ +#define KVM_MAX_NR_USER_RETURN_MSRS 16 + +struct kvm_user_return_msrs { + struct user_return_notifier urn; + bool registered; + struct kvm_user_return_msr_values { + u64 host; + u64 curr; + } values[KVM_MAX_NR_USER_RETURN_MSRS]; +}; + +u32 __read_mostly kvm_nr_uret_msrs; +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_nr_uret_msrs); +static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS]; +static DEFINE_PER_CPU(struct kvm_user_return_msrs, user_return_msrs); + +void kvm_destroy_user_return_msrs(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + WARN_ON_ONCE(per_cpu(user_return_msrs, cpu).registered); + + kvm_nr_uret_msrs = 0; +} + +static void kvm_on_user_return(struct user_return_notifier *urn) +{ + unsigned slot; + struct kvm_user_return_msrs *msrs + = container_of(urn, struct kvm_user_return_msrs, urn); + struct kvm_user_return_msr_values *values; + + msrs->registered = false; + user_return_notifier_unregister(urn); + + for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) { + values = &msrs->values[slot]; + if (values->host != values->curr) { + wrmsrq(kvm_uret_msrs_list[slot], values->host); + values->curr = values->host; + } + } +} + +static int kvm_probe_user_return_msr(u32 msr) +{ + u64 val; + int ret; + + preempt_disable(); + ret = rdmsrq_safe(msr, &val); + if (ret) + goto out; + ret = wrmsrq_safe(msr, val); +out: + preempt_enable(); + return ret; +} + +int kvm_add_user_return_msr(u32 msr) +{ + BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS); + + if (kvm_probe_user_return_msr(msr)) + return -1; + + kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr; + return kvm_nr_uret_msrs++; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_add_user_return_msr); + +int kvm_find_user_return_msr(u32 msr) +{ + int i; + + for (i = 0; i < kvm_nr_uret_msrs; ++i) { + if (kvm_uret_msrs_list[i] == msr) + return i; + } + return -1; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_find_user_return_msr); + +void kvm_user_return_msr_cpu_online(void) +{ + struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); + u64 value; + int i; + + for (i = 0; i < kvm_nr_uret_msrs; ++i) { + rdmsrq_safe(kvm_uret_msrs_list[i], &value); + msrs->values[i].host = value; + msrs->values[i].curr = value; + } +} + +static void kvm_user_return_register_notifier(struct kvm_user_return_msrs *msrs) +{ + if (!msrs->registered) { + msrs->urn.on_user_return = kvm_on_user_return; + user_return_notifier_register(&msrs->urn); + msrs->registered = true; + } +} + +int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask) +{ + struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); + int err; + + value = (value & mask) | (msrs->values[slot].host & ~mask); + if (value == msrs->values[slot].curr) + return 0; + err = wrmsrq_safe(kvm_uret_msrs_list[slot], value); + if (err) + return 1; + + msrs->values[slot].curr = value; + kvm_user_return_register_notifier(msrs); + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_user_return_msr); + +u64 kvm_get_user_return_msr(unsigned int slot) +{ + return this_cpu_ptr(&user_return_msrs)->values[slot].curr; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_user_return_msr); + +void drop_user_return_notifiers(void) +{ + struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); + + if (msrs->registered) + kvm_on_user_return(&msrs->urn); +} + +/* + * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) track + * the set of MSRs that KVM exposes to userspace through KVM_GET_MSRS, + * KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. msrs_to_save holds MSRs that + * require host support, i.e. should be probed via RDMSR. emulated_msrs holds + * MSRs that KVM emulates without strictly requiring host support. + * msr_based_features holds MSRs that enumerate features, i.e. are effectively + * CPUID leafs. Note, msr_based_features isn't mutually exclusive with + * msrs_to_save and emulated_msrs. + */ + +static const u32 msrs_to_save_base[] = { + MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, + MSR_STAR, +#ifdef CONFIG_X86_64 + MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, +#endif + MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA, + MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX, + MSR_IA32_SPEC_CTRL, MSR_IA32_TSX_CTRL, + MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH, + MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK, + MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B, + MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B, + MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B, + MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B, + MSR_IA32_UMWAIT_CONTROL, + + MSR_IA32_XFD, MSR_IA32_XFD_ERR, MSR_IA32_XSS, + + MSR_IA32_U_CET, MSR_IA32_S_CET, + MSR_IA32_PL0_SSP, MSR_IA32_PL1_SSP, MSR_IA32_PL2_SSP, + MSR_IA32_PL3_SSP, MSR_IA32_INT_SSP_TAB, + MSR_IA32_DEBUGCTLMSR, + MSR_IA32_LASTBRANCHFROMIP, MSR_IA32_LASTBRANCHTOIP, + MSR_IA32_LASTINTFROMIP, MSR_IA32_LASTINTTOIP, +}; + +static const u32 msrs_to_save_pmu[] = { + MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1, + MSR_ARCH_PERFMON_FIXED_CTR0 + 2, + MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS, + MSR_CORE_PERF_GLOBAL_CTRL, + MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG, + + /* This part of MSRs should match KVM_MAX_NR_INTEL_GP_COUNTERS. */ + MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1, + MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3, + MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5, + MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7, + MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1, + MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3, + MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5, + MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7, + + MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3, + MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3, + + /* This part of MSRs should match KVM_MAX_NR_AMD_GP_COUNTERS. */ + MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2, + MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5, + MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2, + MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5, + + MSR_AMD64_PERF_CNTR_GLOBAL_CTL, + MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, + MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, + MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET, +}; + +static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) + + ARRAY_SIZE(msrs_to_save_pmu)]; +static unsigned num_msrs_to_save; + +static const u32 emulated_msrs_all[] = { + MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, + MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, + +#ifdef CONFIG_KVM_HYPERV + HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, + HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC, + HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY, + HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2, + HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL, + HV_X64_MSR_RESET, + HV_X64_MSR_VP_INDEX, + HV_X64_MSR_VP_RUNTIME, + HV_X64_MSR_SCONTROL, + HV_X64_MSR_STIMER0_CONFIG, + HV_X64_MSR_VP_ASSIST_PAGE, + HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL, + HV_X64_MSR_TSC_EMULATION_STATUS, HV_X64_MSR_TSC_INVARIANT_CONTROL, + HV_X64_MSR_SYNDBG_OPTIONS, + HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS, + HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER, + HV_X64_MSR_SYNDBG_PENDING_BUFFER, +#endif + + MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, + MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK, + + MSR_IA32_TSC_ADJUST, + MSR_IA32_TSC_DEADLINE, + MSR_IA32_ARCH_CAPABILITIES, + MSR_IA32_PERF_CAPABILITIES, + MSR_IA32_MISC_ENABLE, + MSR_IA32_MCG_STATUS, + MSR_IA32_MCG_CTL, + MSR_IA32_MCG_EXT_CTL, + MSR_IA32_SMBASE, + MSR_SMI_COUNT, + MSR_PLATFORM_INFO, + MSR_MISC_FEATURES_ENABLES, + MSR_AMD64_VIRT_SPEC_CTRL, + MSR_AMD64_TSC_RATIO, + MSR_IA32_POWER_CTL, + MSR_IA32_UCODE_REV, + + /* + * KVM always supports the "true" VMX control MSRs, even if the host + * does not. The VMX MSRs as a whole are considered "emulated" as KVM + * doesn't strictly require them to exist in the host (ignoring that + * KVM would refuse to load in the first place if the core set of MSRs + * aren't supported). + */ + MSR_IA32_VMX_BASIC, + MSR_IA32_VMX_TRUE_PINBASED_CTLS, + MSR_IA32_VMX_TRUE_PROCBASED_CTLS, + MSR_IA32_VMX_TRUE_EXIT_CTLS, + MSR_IA32_VMX_TRUE_ENTRY_CTLS, + MSR_IA32_VMX_MISC, + MSR_IA32_VMX_CR0_FIXED0, + MSR_IA32_VMX_CR4_FIXED0, + MSR_IA32_VMX_VMCS_ENUM, + MSR_IA32_VMX_PROCBASED_CTLS2, + MSR_IA32_VMX_EPT_VPID_CAP, + MSR_IA32_VMX_VMFUNC, + + MSR_K7_HWCR, + MSR_KVM_POLL_CONTROL, +}; + +static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)]; +static unsigned num_emulated_msrs; + +/* + * List of MSRs that control the existence of MSR-based features, i.e. MSRs + * that are effectively CPUID leafs. VMX MSRs are also included in the set of + * feature MSRs, but are handled separately to allow expedited lookups. + */ +static const u32 msr_based_features_all_except_vmx[] = { + MSR_AMD64_DE_CFG, + MSR_IA32_UCODE_REV, + MSR_IA32_ARCH_CAPABILITIES, + MSR_IA32_PERF_CAPABILITIES, + MSR_PLATFORM_INFO, +}; + +static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all_except_vmx) + + (KVM_LAST_EMULATED_VMX_MSR - KVM_FIRST_EMULATED_VMX_MSR + 1)]; +static unsigned int num_msr_based_features; + +int kvm_get_msr_index_list(struct kvm_msr_list __user *user_msr_list) +{ + struct kvm_msr_list msr_list; + unsigned int n; + + if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list))) + return -EFAULT; + + n = msr_list.nmsrs; + msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs; + if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list))) + return -EFAULT; + + if (n < msr_list.nmsrs) + return -E2BIG; + + if (copy_to_user(user_msr_list->indices, &msrs_to_save, + num_msrs_to_save * sizeof(u32))) + return -EFAULT; + + if (copy_to_user(user_msr_list->indices + num_msrs_to_save, + &emulated_msrs, num_emulated_msrs * sizeof(u32))) + return -EFAULT; + + return 0; +} + +int kvm_get_feature_msr_index_list(struct kvm_msr_list __user *user_msr_list) +{ + struct kvm_msr_list msr_list; + unsigned int n; + + if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list))) + return -EFAULT; + + n = msr_list.nmsrs; + msr_list.nmsrs = num_msr_based_features; + if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list))) + return -EFAULT; + + if (n < msr_list.nmsrs) + return -E2BIG; + + if (copy_to_user(user_msr_list->indices, &msr_based_features, + num_msr_based_features * sizeof(u32))) + return -EFAULT; + + return 0; +} + +/* + * All feature MSRs except uCode revID, which tracks the currently loaded uCode + * patch, are immutable once the vCPU model is defined. + */ +static bool kvm_is_immutable_feature_msr(u32 msr) +{ + int i; + + if (msr >= KVM_FIRST_EMULATED_VMX_MSR && msr <= KVM_LAST_EMULATED_VMX_MSR) + return true; + + for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) { + if (msr == msr_based_features_all_except_vmx[i]) + return msr != MSR_IA32_UCODE_REV; + } + + return false; +} + +static bool kvm_is_advertised_msr(u32 msr_index) +{ + unsigned int i; + + for (i = 0; i < num_msrs_to_save; i++) { + if (msrs_to_save[i] == msr_index) + return true; + } + + for (i = 0; i < num_emulated_msrs; i++) { + if (emulated_msrs[i] == msr_index) + return true; + } + + return false; +} + + +/* + * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM + * does not yet virtualize. These include: + * 10 - MISC_PACKAGE_CTRLS + * 11 - ENERGY_FILTERING_CTL + * 12 - DOITM + * 18 - FB_CLEAR_CTRL + * 21 - XAPIC_DISABLE_STATUS + * 23 - OVERCLOCKING_STATUS + */ + +#define KVM_SUPPORTED_ARCH_CAP \ + (ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \ + ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \ + ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \ + ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \ + ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO | ARCH_CAP_GDS_NO | \ + ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO | ARCH_CAP_ITS_NO) + +u64 kvm_get_arch_capabilities(void) +{ + u64 data = kvm_host.arch_capabilities & KVM_SUPPORTED_ARCH_CAP; + + /* + * If nx_huge_pages is enabled, KVM's shadow paging will ensure that + * the nested hypervisor runs with NX huge pages. If it is not, + * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other + * L1 guests, so it need not worry about its own (L2) guests. + */ + data |= ARCH_CAP_PSCHANGE_MC_NO; + + /* + * If we're doing cache flushes (either "always" or "cond") + * we will do one whenever the guest does a vmlaunch/vmresume. + * If an outer hypervisor is doing the cache flush for us + * (ARCH_CAP_SKIP_VMENTRY_L1DFLUSH), we can safely pass that + * capability to the guest too, and if EPT is disabled we're not + * vulnerable. Overall, only VMENTER_L1D_FLUSH_NEVER will + * require a nested hypervisor to do a flush of its own. + */ + if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER) + data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH; + + if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN)) + data |= ARCH_CAP_RDCL_NO; + if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) + data |= ARCH_CAP_SSB_NO; + if (!boot_cpu_has_bug(X86_BUG_MDS)) + data |= ARCH_CAP_MDS_NO; + if (!boot_cpu_has_bug(X86_BUG_RFDS)) + data |= ARCH_CAP_RFDS_NO; + if (!boot_cpu_has_bug(X86_BUG_ITS)) + data |= ARCH_CAP_ITS_NO; + + if (!boot_cpu_has(X86_FEATURE_RTM)) { + /* + * If RTM=0 because the kernel has disabled TSX, the host might + * have TAA_NO or TSX_CTRL. Clear TAA_NO (the guest sees RTM=0 + * and therefore knows that there cannot be TAA) but keep + * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts, + * and we want to allow migrating those guests to tsx=off hosts. + */ + data &= ~ARCH_CAP_TAA_NO; + } else if (!boot_cpu_has_bug(X86_BUG_TAA)) { + data |= ARCH_CAP_TAA_NO; + } else { + /* + * Nothing to do here; we emulate TSX_CTRL if present on the + * host so the guest can choose between disabling TSX or + * using VERW to clear CPU buffers. + */ + } + + if (!boot_cpu_has_bug(X86_BUG_GDS) || gds_ucode_mitigated()) + data |= ARCH_CAP_GDS_NO; + + return data; +} + +static int kvm_get_feature_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, + bool host_initiated) +{ + WARN_ON_ONCE(!host_initiated); + + switch (index) { + case MSR_IA32_ARCH_CAPABILITIES: + *data = kvm_get_arch_capabilities(); + break; + case MSR_IA32_PERF_CAPABILITIES: + *data = kvm_caps.supported_perf_cap; + break; + case MSR_PLATFORM_INFO: + *data = MSR_PLATFORM_INFO_CPUID_FAULT; + break; + case MSR_IA32_UCODE_REV: + rdmsrq_safe(index, data); + break; + default: + return kvm_x86_call(get_feature_msr)(index, data); + } + return 0; +} + +typedef int (*msr_access_t)(struct kvm_vcpu *vcpu, u32 index, u64 *data, + bool host_initiated); + +static __always_inline int kvm_do_msr_access(struct kvm_vcpu *vcpu, u32 msr, + u64 *data, bool host_initiated, + enum kvm_msr_access rw, + msr_access_t msr_access_fn) +{ + const char *op = rw == MSR_TYPE_W ? "wrmsr" : "rdmsr"; + int ret; + + BUILD_BUG_ON(rw != MSR_TYPE_R && rw != MSR_TYPE_W); + + /* + * Zero the data on read failures to avoid leaking stack data to the + * guest and/or userspace, e.g. if the failure is ignored below. + */ + ret = msr_access_fn(vcpu, msr, data, host_initiated); + if (ret && rw == MSR_TYPE_R) + *data = 0; + + if (ret != KVM_MSR_RET_UNSUPPORTED) + return ret; + + /* + * Userspace is allowed to read MSRs, and write '0' to MSRs, that KVM + * advertises to userspace, even if an MSR isn't fully supported. + * Simply check that @data is '0', which covers both the write '0' case + * and all reads (in which case @data is zeroed on failure; see above). + */ + if (host_initiated && !*data && kvm_is_advertised_msr(msr)) + return 0; + + if (!ignore_msrs) { + kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n", + op, msr, *data); + return ret; + } + + if (report_ignored_msrs) + kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n", op, msr, *data); + + return 0; +} + +static int do_get_feature_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) +{ + return kvm_do_msr_access(vcpu, index, data, true, MSR_TYPE_R, + kvm_get_feature_msr); +} + +static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) +{ + if (efer & EFER_AUTOIBRS && !guest_cpu_cap_has(vcpu, X86_FEATURE_AUTOIBRS)) + return false; + + if (efer & EFER_FFXSR && !guest_cpu_cap_has(vcpu, X86_FEATURE_FXSR_OPT)) + return false; + + if (efer & EFER_SVME && !guest_cpu_cap_has(vcpu, X86_FEATURE_SVM)) + return false; + + if (efer & (EFER_LME | EFER_LMA) && + !guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) + return false; + + if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX)) + return false; + + return true; + +} +bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) +{ + if (efer & efer_reserved_bits) + return false; + + return __kvm_valid_efer(vcpu, efer); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_valid_efer); + +static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) +{ + u64 old_efer = vcpu->arch.efer; + u64 efer = msr_info->data; + int r; + + if (efer & efer_reserved_bits) + return 1; + + if (!msr_info->host_initiated) { + if (!__kvm_valid_efer(vcpu, efer)) + return 1; + + if (is_paging(vcpu) && + (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME)) + return 1; + } + + efer &= ~EFER_LMA; + efer |= vcpu->arch.efer & EFER_LMA; + + r = kvm_x86_call(set_efer)(vcpu, efer); + if (r) { + WARN_ON(r > 0); + return r; + } + + if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS) + kvm_mmu_reset_context(vcpu); + + if (!static_cpu_has(X86_FEATURE_XSAVES) && + (efer & EFER_SVME)) + kvm_hv_xsaves_xsavec_maybe_warn(vcpu); + + return 0; +} + +void kvm_enable_efer_bits(u64 mask) +{ + efer_reserved_bits &= ~mask; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_enable_efer_bits); + +bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type) +{ + struct kvm_x86_msr_filter *msr_filter; + struct msr_bitmap_range *ranges; + struct kvm *kvm = vcpu->kvm; + bool allowed; + int idx; + u32 i; + + /* x2APIC MSRs do not support filtering. */ + if (index >= 0x800 && index <= 0x8ff) + return true; + + idx = srcu_read_lock(&kvm->srcu); + + msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu); + if (!msr_filter) { + allowed = true; + goto out; + } + + allowed = msr_filter->default_allow; + ranges = msr_filter->ranges; + + for (i = 0; i < msr_filter->count; i++) { + u32 start = ranges[i].base; + u32 end = start + ranges[i].nmsrs; + u32 flags = ranges[i].flags; + unsigned long *bitmap = ranges[i].bitmap; + + if ((index >= start) && (index < end) && (flags & type)) { + allowed = test_bit(index - start, bitmap); + break; + } + } + +out: + srcu_read_unlock(&kvm->srcu, idx); + + return allowed; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_allowed); + +/* + * Write @data into the MSR specified by @index. Select MSR specific fault + * checks are bypassed if @host_initiated is %true. + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, + bool host_initiated) +{ + struct msr_data msr; + + switch (index) { + case MSR_FS_BASE: + case MSR_GS_BASE: + case MSR_KERNEL_GS_BASE: + case MSR_CSTAR: + case MSR_LSTAR: + if (is_noncanonical_msr_address(data, vcpu)) + return 1; + break; + case MSR_IA32_SYSENTER_EIP: + case MSR_IA32_SYSENTER_ESP: + /* + * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if + * non-canonical address is written on Intel but not on + * AMD (which ignores the top 32-bits, because it does + * not implement 64-bit SYSENTER). + * + * 64-bit code should hence be able to write a non-canonical + * value on AMD. Making the address canonical ensures that + * vmentry does not fail on Intel after writing a non-canonical + * value, and that something deterministic happens if the guest + * invokes 64-bit SYSENTER. + */ + data = __canonical_address(data, max_host_virt_addr_bits()); + break; + case MSR_TSC_AUX: + if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) + return 1; + + if (!host_initiated && + !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) && + !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID)) + return 1; + + /* + * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has + * incomplete and conflicting architectural behavior. Current + * AMD CPUs completely ignore bits 63:32, i.e. they aren't + * reserved and always read as zeros. Enforce Intel's reserved + * bits check if the guest CPU is Intel compatible, otherwise + * clear the bits. This ensures cross-vendor migration will + * provide consistent behavior for the guest. + */ + if (guest_cpuid_is_intel_compatible(vcpu) && (data >> 32) != 0) + return 1; + + data = (u32)data; + break; + case MSR_IA32_U_CET: + case MSR_IA32_S_CET: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && + !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT)) + return KVM_MSR_RET_UNSUPPORTED; + if (!kvm_is_valid_u_s_cet(vcpu, data)) + return 1; + break; + case MSR_KVM_INTERNAL_GUEST_SSP: + if (!host_initiated) + return 1; + fallthrough; + /* + * Note that the MSR emulation here is flawed when a vCPU + * doesn't support the Intel 64 architecture. The expected + * architectural behavior in this case is that the upper 32 + * bits do not exist and should always read '0'. However, + * because the actual hardware on which the virtual CPU is + * running does support Intel 64, XRSTORS/XSAVES in the + * guest could observe behavior that violates the + * architecture. Intercepting XRSTORS/XSAVES for this + * special case isn't deemed worthwhile. + */ + case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) + return KVM_MSR_RET_UNSUPPORTED; + /* + * MSR_IA32_INT_SSP_TAB is not present on processors that do + * not support Intel 64 architecture. + */ + if (index == MSR_IA32_INT_SSP_TAB && !guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) + return KVM_MSR_RET_UNSUPPORTED; + if (is_noncanonical_msr_address(data, vcpu)) + return 1; + /* All SSP MSRs except MSR_IA32_INT_SSP_TAB must be 4-byte aligned */ + if (index != MSR_IA32_INT_SSP_TAB && !IS_ALIGNED(data, 4)) + return 1; + break; + } + + msr.data = data; + msr.index = index; + msr.host_initiated = host_initiated; + + return kvm_x86_call(set_msr)(vcpu, &msr); +} + +static int _kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, + bool host_initiated) +{ + return __kvm_set_msr(vcpu, index, *data, host_initiated); +} + +static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu, + u32 index, u64 data, bool host_initiated) +{ + return kvm_do_msr_access(vcpu, index, &data, host_initiated, MSR_TYPE_W, + _kvm_set_msr); +} + +/* + * Read the MSR specified by @index into @data. Select MSR specific fault + * checks are bypassed if @host_initiated is %true. + * Returns 0 on success, non-0 otherwise. + * Assumes vcpu_load() was already called. + */ +static int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, + bool host_initiated) +{ + struct msr_data msr; + int ret; + + switch (index) { + case MSR_TSC_AUX: + if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) + return 1; + + if (!host_initiated && + !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) && + !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID)) + return 1; + break; + case MSR_IA32_U_CET: + case MSR_IA32_S_CET: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && + !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT)) + return KVM_MSR_RET_UNSUPPORTED; + break; + case MSR_KVM_INTERNAL_GUEST_SSP: + if (!host_initiated) + return 1; + fallthrough; + case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) + return KVM_MSR_RET_UNSUPPORTED; + break; + } + + msr.index = index; + msr.host_initiated = host_initiated; + + ret = kvm_x86_call(get_msr)(vcpu, &msr); + if (!ret) + *data = msr.data; + return ret; +} + +static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu, + u32 index, u64 *data, bool host_initiated) +{ + return kvm_do_msr_access(vcpu, index, data, host_initiated, MSR_TYPE_R, + __kvm_get_msr); +} + +int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) +{ + return __kvm_set_msr(vcpu, index, data, true); +} + +int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) +{ + return __kvm_get_msr(vcpu, index, data, true); +} + +int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) +{ + return kvm_get_msr_ignored_check(vcpu, index, data, false); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_read); + +int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) +{ + return kvm_set_msr_ignored_check(vcpu, index, data, false); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_write); + +int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) +{ + if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ)) + return KVM_MSR_RET_FILTERED; + + return __kvm_emulate_msr_read(vcpu, index, data); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_read); + +int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) +{ + if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE)) + return KVM_MSR_RET_FILTERED; + + return __kvm_emulate_msr_write(vcpu, index, data); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_write); + +static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) +{ + if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu)) + return EXIT_FASTPATH_NONE; + + switch (msr) { + case APIC_BASE_MSR + (APIC_ICR >> 4): + if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic) || + kvm_x2apic_icr_write_fast(vcpu->arch.apic, data)) + return EXIT_FASTPATH_NONE; + break; + case MSR_IA32_TSC_DEADLINE: + kvm_set_lapic_tscdeadline_msr(vcpu, data); + break; + default: + return EXIT_FASTPATH_NONE; + } + + trace_kvm_msr_write(msr, data); + + if (!kvm_skip_emulated_instruction(vcpu)) + return EXIT_FASTPATH_EXIT_USERSPACE; + + return EXIT_FASTPATH_REENTER_GUEST; +} + +fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu) +{ + return __handle_fastpath_wrmsr(vcpu, kvm_ecx_read(vcpu), + kvm_read_edx_eax(vcpu)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr); + +fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) +{ + return __handle_fastpath_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr_imm); + +static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu) +{ + if (!vcpu->run->msr.error) { + kvm_eax_write(vcpu, vcpu->run->msr.data); + kvm_edx_write(vcpu, vcpu->run->msr.data >> 32); + } +} + +static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err) +{ + if (err) { + kvm_inject_gp(vcpu, 0); + return 1; + } + + return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP | + EMULTYPE_COMPLETE_USER_EXIT); +} + +static int complete_emulated_msr_access(struct kvm_vcpu *vcpu) +{ + return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error); +} + +static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) +{ + complete_userspace_rdmsr(vcpu); + return complete_emulated_msr_access(vcpu); +} + +static int complete_fast_msr_access(struct kvm_vcpu *vcpu) +{ + return kvm_x86_call(complete_emulated_msr)(vcpu, vcpu->run->msr.error); +} + +static int complete_fast_rdmsr(struct kvm_vcpu *vcpu) +{ + complete_userspace_rdmsr(vcpu); + return complete_fast_msr_access(vcpu); +} + +static int complete_fast_rdmsr_imm(struct kvm_vcpu *vcpu) +{ + if (!vcpu->run->msr.error) + kvm_register_write(vcpu, vcpu->arch.cui_rdmsr_imm_reg, + vcpu->run->msr.data); + + return complete_fast_msr_access(vcpu); +} + +static u64 kvm_msr_reason(int r) +{ + switch (r) { + case KVM_MSR_RET_UNSUPPORTED: + return KVM_MSR_EXIT_REASON_UNKNOWN; + case KVM_MSR_RET_FILTERED: + return KVM_MSR_EXIT_REASON_FILTER; + default: + return KVM_MSR_EXIT_REASON_INVAL; + } +} + +static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index, + u32 exit_reason, u64 data, + int (*completion)(struct kvm_vcpu *vcpu), + int r) +{ + u64 msr_reason = kvm_msr_reason(r); + + /* Check if the user wanted to know about this MSR fault */ + if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason)) + return 0; + + vcpu->run->exit_reason = exit_reason; + vcpu->run->msr.error = 0; + memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad)); + vcpu->run->msr.reason = msr_reason; + vcpu->run->msr.index = index; + vcpu->run->msr.data = data; + vcpu->arch.complete_userspace_io = completion; + + return 1; +} + +static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg, + int (*complete_rdmsr)(struct kvm_vcpu *)) +{ + u64 data; + int r; + + r = kvm_emulate_msr_read(vcpu, msr, &data); + + if (!r) { + trace_kvm_msr_read(msr, data); + + if (reg < 0) { + kvm_eax_write(vcpu, data); + kvm_edx_write(vcpu, data >> 32); + } else { + kvm_register_write(vcpu, reg, data); + } + } else { + /* MSR read failed? See if we should ask user space */ + if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_RDMSR, 0, + complete_rdmsr, r)) + return 0; + trace_kvm_msr_read_ex(msr); + } + + return kvm_x86_call(complete_emulated_msr)(vcpu, r); +} + +int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) +{ + return __kvm_emulate_rdmsr(vcpu, kvm_ecx_read(vcpu), -1, + complete_fast_rdmsr); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr); + +int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) +{ + vcpu->arch.cui_rdmsr_imm_reg = reg; + + return __kvm_emulate_rdmsr(vcpu, msr, reg, complete_fast_rdmsr_imm); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr_imm); + +static int __kvm_emulate_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) +{ + int r; + + r = kvm_emulate_msr_write(vcpu, msr, data); + if (!r) { + trace_kvm_msr_write(msr, data); + } else { + /* MSR write failed? See if we should ask user space */ + if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_WRMSR, data, + complete_fast_msr_access, r)) + return 0; + /* Signal all other negative errors to userspace */ + if (r < 0) + return r; + trace_kvm_msr_write_ex(msr, data); + } + + return kvm_x86_call(complete_emulated_msr)(vcpu, r); +} + +int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) +{ + return __kvm_emulate_wrmsr(vcpu, kvm_ecx_read(vcpu), + kvm_read_edx_eax(vcpu)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr); + +int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) +{ + return __kvm_emulate_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr_imm); + +int kvm_emulator_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index, + u64 *pdata) +{ + int r; + + r = kvm_emulate_msr_read(vcpu, msr_index, pdata); + if (r < 0) + return X86EMUL_UNHANDLEABLE; + + if (r) { + if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, + complete_emulated_rdmsr, r)) + return X86EMUL_IO_NEEDED; + + trace_kvm_msr_read_ex(msr_index); + return X86EMUL_PROPAGATE_FAULT; + } + + trace_kvm_msr_read(msr_index, *pdata); + return X86EMUL_CONTINUE; +} + +int kvm_emulator_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index, + u64 data) +{ + int r; + + r = kvm_emulate_msr_write(vcpu, msr_index, data); + if (r < 0) + return X86EMUL_UNHANDLEABLE; + + if (r) { + if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, + complete_emulated_msr_access, r)) + return X86EMUL_IO_NEEDED; + + trace_kvm_msr_write_ex(msr_index, data); + return X86EMUL_PROPAGATE_FAULT; + } + + trace_kvm_msr_write(msr_index, data); + return X86EMUL_CONTINUE; +} + +int kvm_emulator_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata) +{ + /* + * Treat emulator accesses to the current shadow stack pointer as host- + * initiated, as they aren't true MSR accesses (SSP is a "just a reg"), + * and this API is used only for implicit accesses, i.e. not RDMSR, and + * so the index is fully KVM-controlled. + */ + if (unlikely(msr_index == MSR_KVM_INTERNAL_GUEST_SSP)) + return kvm_msr_read(vcpu, msr_index, pdata); + + return __kvm_emulate_msr_read(vcpu, msr_index, pdata); +} + +/* + * Returns true if the MSR in question is managed via XSTATE, i.e. is context + * switched with the rest of guest FPU state. + * + * Note, S_CET is _not_ saved/restored via XSAVES/XRSTORS. + */ +static bool is_xstate_managed_msr(struct kvm_vcpu *vcpu, u32 msr) +{ + if (!vcpu) + return false; + + switch (msr) { + case MSR_IA32_U_CET: + return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) || + guest_cpu_cap_has(vcpu, X86_FEATURE_IBT); + case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: + return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); + default: + return false; + } +} + +/* + * Lock (and if necessary, re-load) the guest FPU, i.e. XSTATE, and access an + * MSR that is managed via XSTATE. Note, the caller is responsible for doing + * the initial FPU load, this helper only ensures that guest state is resident + * in hardware (the kernel can load its FPU state in IRQ context). + * + * Note, loading guest values for U_CET and PL[0-3]_SSP while executing in the + * kernel is safe, as U_CET is specific to userspace, and PL[0-3]_SSP are only + * consumed when transitioning to lower privilege levels, i.e. are effectively + * only consumed by userspace as well. + */ +static __always_inline void kvm_access_xstate_msr(struct kvm_vcpu *vcpu, + struct msr_data *msr_info, + int access) +{ + BUILD_BUG_ON(access != MSR_TYPE_R && access != MSR_TYPE_W); + + KVM_BUG_ON(!is_xstate_managed_msr(vcpu, msr_info->index), vcpu->kvm); + KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm); + + kvm_fpu_get(); + if (access == MSR_TYPE_R) + rdmsrq(msr_info->index, msr_info->data); + else + wrmsrq(msr_info->index, msr_info->data); + kvm_fpu_put(); +} + +static void kvm_set_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) +{ + kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_W); +} + +static void kvm_get_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) +{ + kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_R); +} + +static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs) +{ + int version; + int r; + struct pvclock_wall_clock wc; + u32 wc_sec_hi; + u64 wall_nsec; + + if (!wall_clock) + return; + + r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version)); + if (r) + return; + + if (version & 1) + ++version; /* first time write, random junk */ + + ++version; + + if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version))) + return; + + wall_nsec = kvm_get_wall_clock_epoch(kvm); + + wc.nsec = do_div(wall_nsec, NSEC_PER_SEC); + wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */ + wc.version = version; + + kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc)); + + if (sec_hi_ofs) { + wc_sec_hi = wall_nsec >> 32; + kvm_write_guest(kvm, wall_clock + sec_hi_ofs, + &wc_sec_hi, sizeof(wc_sec_hi)); + } + + version++; + kvm_write_guest(kvm, wall_clock, &version, sizeof(version)); +} + +static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time, + bool old_msr, bool host_initiated) +{ + struct kvm_arch *ka = &vcpu->kvm->arch; + + if (vcpu->vcpu_id == 0 && !host_initiated) { + if (ka->boot_vcpu_runs_old_kvmclock != old_msr) + kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); + + ka->boot_vcpu_runs_old_kvmclock = old_msr; + } + + vcpu->arch.time = system_time; + kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); + + /* we verify if the enable bit is set... */ + if (system_time & 1) + kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL, + sizeof(struct pvclock_vcpu_time_info)); + else + kvm_gpc_deactivate(&vcpu->arch.pv_time); + + return; +} + +/* These helpers are safe iff @msr is known to be an MCx bank MSR. */ +static bool is_mci_control_msr(u32 msr) +{ + return (msr & 3) == 0; +} +static bool is_mci_status_msr(u32 msr) +{ + return (msr & 3) == 1; +} + +/* + * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP. + */ +static bool can_set_mci_status(struct kvm_vcpu *vcpu) +{ + /* McStatusWrEn enabled? */ + if (guest_cpuid_is_amd_compatible(vcpu)) + return !!(vcpu->arch.msr_hwcr & BIT_ULL(18)); + + return false; +} + +static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info) +{ + u64 mcg_cap = vcpu->arch.mcg_cap; + unsigned bank_num = mcg_cap & 0xff; + u32 msr = msr_info->index; + u64 data = msr_info->data; + u32 offset, last_msr; + + switch (msr) { + case MSR_IA32_MCG_STATUS: + vcpu->arch.mcg_status = data; + break; + case MSR_IA32_MCG_CTL: + if (!(mcg_cap & MCG_CTL_P) && + (data || !msr_info->host_initiated)) + return 1; + if (data != 0 && data != ~(u64)0) + return 1; + vcpu->arch.mcg_ctl = data; + break; + case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: + last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1; + if (msr > last_msr) + return 1; + + if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated)) + return 1; + /* An attempt to write a 1 to a reserved bit raises #GP */ + if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK)) + return 1; + offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2, + last_msr + 1 - MSR_IA32_MC0_CTL2); + vcpu->arch.mci_ctl2_banks[offset] = data; + break; + case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: + last_msr = MSR_IA32_MCx_CTL(bank_num) - 1; + if (msr > last_msr) + return 1; + + /* + * Only 0 or all 1s can be written to IA32_MCi_CTL, all other + * values are architecturally undefined. But, some Linux + * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB + * issue on AMD K8s, allow bit 10 to be clear when setting all + * other bits in order to avoid an uncaught #GP in the guest. + * + * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable, + * single-bit ECC data errors. + */ + if (is_mci_control_msr(msr) && + data != 0 && (data | (1 << 10) | 1) != ~(u64)0) + return 1; + + /* + * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR. + * AMD-based CPUs allow non-zero values, but if and only if + * HWCR[McStatusWrEn] is set. + */ + if (!msr_info->host_initiated && is_mci_status_msr(msr) && + data != 0 && !can_set_mci_status(vcpu)) + return 1; + + offset = array_index_nospec(msr - MSR_IA32_MC0_CTL, + last_msr + 1 - MSR_IA32_MC0_CTL); + vcpu->arch.mce_banks[offset] = data; + break; + default: + return 1; + } + return 0; +} + +static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) +{ + gpa_t gpa = data & ~0x3f; + + /* Bits 4:5 are reserved, Should be zero */ + if (data & 0x30) + return 1; + + if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) && + (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT)) + return 1; + + if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) && + (data & KVM_ASYNC_PF_DELIVERY_AS_INT)) + return 1; + + if (!lapic_in_kernel(vcpu)) + return data ? 1 : 0; + + if (__kvm_pv_async_pf_enabled(data) && + kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, + sizeof(u64))) + return 1; + + vcpu->arch.apf.msr_en_val = data; + + if (__kvm_pv_async_pf_enabled(data)) { + kvm_async_pf_wakeup_all(vcpu); + } else { + kvm_clear_async_pf_completion_queue(vcpu); + kvm_async_pf_hash_reset(vcpu); + } + return 0; +} + +static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data) +{ + /* Bits 8-63 are reserved */ + if (data >> 8) + return 1; + + if (!lapic_in_kernel(vcpu)) + return 1; + + vcpu->arch.apf.msr_int_val = data; + + vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK; + + return 0; +} + +#ifdef CONFIG_X86_64 +static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC; +} +#endif + +int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) +{ + u32 msr = msr_info->index; + u64 data = msr_info->data; + + /* + * Do not allow host-initiated writes to trigger the Xen hypercall + * page setup; it could incur locking paths which are not expected + * if userspace sets the MSR in an unusual location. + */ + if (kvm_xen_is_hypercall_page_msr(vcpu->kvm, msr) && + !msr_info->host_initiated) + return kvm_xen_write_hypercall_page(vcpu, data); + + switch (msr) { + case MSR_AMD64_NB_CFG: + case MSR_IA32_UCODE_WRITE: + case MSR_VM_HSAVE_PA: + case MSR_AMD64_PATCH_LOADER: + case MSR_AMD64_BU_CFG2: + case MSR_AMD64_DC_CFG: + case MSR_AMD64_TW_CFG: + case MSR_F15H_EX_CFG: + break; + + case MSR_IA32_UCODE_REV: + if (msr_info->host_initiated) + vcpu->arch.microcode_version = data; + break; + case MSR_IA32_ARCH_CAPABILITIES: + if (!msr_info->host_initiated || + !guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES)) + return KVM_MSR_RET_UNSUPPORTED; + vcpu->arch.arch_capabilities = data; + break; + case MSR_IA32_PERF_CAPABILITIES: + if (!msr_info->host_initiated || + !guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM)) + return KVM_MSR_RET_UNSUPPORTED; + + if (data & ~kvm_caps.supported_perf_cap) + return 1; + + /* + * Note, this is not just a performance optimization! KVM + * disallows changing feature MSRs after the vCPU has run; PMU + * refresh will bug the VM if called after the vCPU has run. + */ + if (vcpu->arch.perf_capabilities == data) + break; + + vcpu->arch.perf_capabilities = data; + kvm_pmu_refresh(vcpu); + kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu); + break; + case MSR_IA32_PRED_CMD: { + u64 reserved_bits = ~(PRED_CMD_IBPB | PRED_CMD_SBPB); + + if (!msr_info->host_initiated) { + if ((!guest_has_pred_cmd_msr(vcpu))) + return 1; + + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SPEC_CTRL) && + !guest_cpu_cap_has(vcpu, X86_FEATURE_AMD_IBPB)) + reserved_bits |= PRED_CMD_IBPB; + + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SBPB)) + reserved_bits |= PRED_CMD_SBPB; + } + + if (!boot_cpu_has(X86_FEATURE_IBPB)) + reserved_bits |= PRED_CMD_IBPB; + + if (!boot_cpu_has(X86_FEATURE_SBPB)) + reserved_bits |= PRED_CMD_SBPB; + + if (data & reserved_bits) + return 1; + + if (!data) + break; + + wrmsrq(MSR_IA32_PRED_CMD, data); + break; + } + case MSR_IA32_FLUSH_CMD: + if (!msr_info->host_initiated && + !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D)) + return 1; + + if (!boot_cpu_has(X86_FEATURE_FLUSH_L1D) || (data & ~L1D_FLUSH)) + return 1; + if (!data) + break; + + wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH); + break; + case MSR_EFER: + return set_efer(vcpu, msr_info); + case MSR_K7_HWCR: { + /* + * Allow McStatusWrEn and TscFreqSel. (Linux guests from v3.2 + * through at least v6.6 whine if TscFreqSel is clear, + * depending on F/M/S. + */ + u64 valid = BIT_ULL(18) | BIT_ULL(24); + + data &= ~(u64)0x40; /* ignore flush filter disable */ + data &= ~(u64)0x100; /* ignore ignne emulation enable */ + data &= ~(u64)0x8; /* ignore TLB cache disable */ + + if (guest_cpu_cap_has(vcpu, X86_FEATURE_GP_ON_USER_CPUID)) + valid |= MSR_K7_HWCR_CPUID_USER_DIS; + + if (data & ~valid) { + kvm_pr_unimpl_wrmsr(vcpu, msr, data); + return 1; + } + vcpu->arch.msr_hwcr = data; + break; + } + case MSR_FAM10H_MMIO_CONF_BASE: + if (data != 0) { + kvm_pr_unimpl_wrmsr(vcpu, msr, data); + return 1; + } + break; + case MSR_IA32_CR_PAT: + if (!kvm_pat_valid(data)) + return 1; + + vcpu->arch.pat = data; + break; + case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000: + case MSR_MTRRdefType: + return kvm_mtrr_set_msr(vcpu, msr, data); + case MSR_IA32_APICBASE: + return kvm_apic_set_base(vcpu, data, msr_info->host_initiated); + case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: + return kvm_x2apic_msr_write(vcpu, msr, data); + case MSR_IA32_TSC_DEADLINE: + kvm_set_lapic_tscdeadline_msr(vcpu, data); + break; + case MSR_IA32_TSC_ADJUST: + if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSC_ADJUST)) { + if (!msr_info->host_initiated) { + s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr; + adjust_tsc_offset_guest(vcpu, adj); + /* Before back to guest, tsc_timestamp must be adjusted + * as well, otherwise guest's percpu pvclock time could jump. + */ + kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); + } + vcpu->arch.ia32_tsc_adjust_msr = data; + } + break; + case MSR_IA32_MISC_ENABLE: { + u64 old_val = vcpu->arch.ia32_misc_enable_msr; + + if (!msr_info->host_initiated) { + /* RO bits */ + if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK) + return 1; + + /* R bits, i.e. writes are ignored, but don't fault. */ + data = data & ~MSR_IA32_MISC_ENABLE_EMON; + data |= old_val & MSR_IA32_MISC_ENABLE_EMON; + } + + if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) && + ((old_val ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) { + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XMM3)) + return 1; + vcpu->arch.ia32_misc_enable_msr = data; + vcpu->arch.cpuid_dynamic_bits_dirty = true; + } else { + vcpu->arch.ia32_misc_enable_msr = data; + } + break; + } + case MSR_IA32_SMBASE: + if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated) + return 1; + vcpu->arch.smbase = data; + break; + case MSR_IA32_POWER_CTL: + vcpu->arch.msr_ia32_power_ctl = data; + break; + case MSR_IA32_TSC: + if (msr_info->host_initiated) { + kvm_synchronize_tsc(vcpu, &data); + } else if (!vcpu->arch.guest_tsc_protected) { + u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; + adjust_tsc_offset_guest(vcpu, adj); + vcpu->arch.ia32_tsc_adjust_msr += adj; + } + break; + case MSR_IA32_XSS: + if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) + return KVM_MSR_RET_UNSUPPORTED; + + if (data & ~vcpu->arch.guest_supported_xss) + return 1; + if (vcpu->arch.ia32_xss == data) + break; + vcpu->arch.ia32_xss = data; + vcpu->arch.cpuid_dynamic_bits_dirty = true; + break; + case MSR_SMI_COUNT: + if (!msr_info->host_initiated) + return 1; + vcpu->arch.smi_count = data; + break; + case MSR_KVM_WALL_CLOCK_NEW: + if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) + return KVM_MSR_RET_UNSUPPORTED; + + vcpu->kvm->arch.wall_clock = data; + kvm_write_wall_clock(vcpu->kvm, data, 0); + break; + case MSR_KVM_WALL_CLOCK: + if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) + return KVM_MSR_RET_UNSUPPORTED; + + vcpu->kvm->arch.wall_clock = data; + kvm_write_wall_clock(vcpu->kvm, data, 0); + break; + case MSR_KVM_SYSTEM_TIME_NEW: + if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) + return KVM_MSR_RET_UNSUPPORTED; + + kvm_write_system_time(vcpu, data, false, msr_info->host_initiated); + break; + case MSR_KVM_SYSTEM_TIME: + if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) + return KVM_MSR_RET_UNSUPPORTED; + + kvm_write_system_time(vcpu, data, true, msr_info->host_initiated); + break; + case MSR_KVM_ASYNC_PF_EN: + if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) + return KVM_MSR_RET_UNSUPPORTED; + + if (kvm_pv_enable_async_pf(vcpu, data)) + return 1; + break; + case MSR_KVM_ASYNC_PF_INT: + if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) + return KVM_MSR_RET_UNSUPPORTED; + + if (kvm_pv_enable_async_pf_int(vcpu, data)) + return 1; + break; + case MSR_KVM_ASYNC_PF_ACK: + if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) + return KVM_MSR_RET_UNSUPPORTED; + if (data & 0x1) { + /* + * Pairs with the smp_mb__after_atomic() in + * kvm_arch_async_page_present_queued(). + */ + smp_store_mb(vcpu->arch.apf.pageready_pending, false); + + kvm_check_async_pf_completion(vcpu); + } + break; + case MSR_KVM_STEAL_TIME: + if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) + return KVM_MSR_RET_UNSUPPORTED; + + if (unlikely(!sched_info_on())) + return 1; + + if (data & KVM_STEAL_RESERVED_MASK) + return 1; + + vcpu->arch.st.msr_val = data; + + if (!(data & KVM_MSR_ENABLED)) + break; + + kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); + + break; + case MSR_KVM_PV_EOI_EN: + if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) + return KVM_MSR_RET_UNSUPPORTED; + + if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) + return 1; + break; + + case MSR_KVM_POLL_CONTROL: + if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) + return KVM_MSR_RET_UNSUPPORTED; + + /* only enable bit supported */ + if (data & (-1ULL << 1)) + return 1; + + vcpu->arch.msr_kvm_poll_control = data; + break; + + case MSR_IA32_MCG_CTL: + case MSR_IA32_MCG_STATUS: + case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: + case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: + return set_msr_mce(vcpu, msr_info); + + case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: + case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: + case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: + case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: + if (kvm_pmu_is_valid_msr(vcpu, msr)) + return kvm_pmu_set_msr(vcpu, msr_info); + + if (data) + kvm_pr_unimpl_wrmsr(vcpu, msr, data); + break; + case MSR_K7_CLK_CTL: + /* + * Ignore all writes to this no longer documented MSR. + * Writes are only relevant for old K7 processors, + * all pre-dating SVM, but a recommended workaround from + * AMD for these chips. It is possible to specify the + * affected processor models on the command line, hence + * the need to ignore the workaround. + */ + break; +#ifdef CONFIG_KVM_HYPERV + case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: + case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: + case HV_X64_MSR_SYNDBG_OPTIONS: + case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: + case HV_X64_MSR_CRASH_CTL: + case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: + case HV_X64_MSR_REENLIGHTENMENT_CONTROL: + case HV_X64_MSR_TSC_EMULATION_CONTROL: + case HV_X64_MSR_TSC_EMULATION_STATUS: + case HV_X64_MSR_TSC_INVARIANT_CONTROL: + return kvm_hv_set_msr_common(vcpu, msr, data, + msr_info->host_initiated); +#endif + case MSR_IA32_BBL_CR_CTL3: + /* Drop writes to this legacy MSR -- see rdmsr + * counterpart for further detail. + */ + kvm_pr_unimpl_wrmsr(vcpu, msr, data); + break; + case MSR_AMD64_OSVW_ID_LENGTH: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) + return 1; + vcpu->arch.osvw.length = data; + break; + case MSR_AMD64_OSVW_STATUS: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) + return 1; + vcpu->arch.osvw.status = data; + break; + case MSR_PLATFORM_INFO: + if (!msr_info->host_initiated) + return 1; + vcpu->arch.msr_platform_info = data; + break; + case MSR_MISC_FEATURES_ENABLES: + if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT || + (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT && + !(vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT))) + return 1; + vcpu->arch.msr_misc_features_enables = data; + break; +#ifdef CONFIG_X86_64 + case MSR_IA32_XFD: + if (!msr_info->host_initiated && + !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) + return 1; + + if (data & ~kvm_guest_supported_xfd(vcpu)) + return 1; + + fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data); + break; + case MSR_IA32_XFD_ERR: + if (!msr_info->host_initiated && + !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) + return 1; + + if (data & ~kvm_guest_supported_xfd(vcpu)) + return 1; + + vcpu->arch.guest_fpu.xfd_err = data; + break; +#endif + case MSR_IA32_U_CET: + case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: + kvm_set_xstate_msr(vcpu, msr_info); + break; + default: + if (kvm_pmu_is_valid_msr(vcpu, msr)) + return kvm_pmu_set_msr(vcpu, msr_info); + + return KVM_MSR_RET_UNSUPPORTED; + } + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_msr_common); + +static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) +{ + u64 data; + u64 mcg_cap = vcpu->arch.mcg_cap; + unsigned bank_num = mcg_cap & 0xff; + u32 offset, last_msr; + + switch (msr) { + case MSR_IA32_P5_MC_ADDR: + case MSR_IA32_P5_MC_TYPE: + data = 0; + break; + case MSR_IA32_MCG_CAP: + data = vcpu->arch.mcg_cap; + break; + case MSR_IA32_MCG_CTL: + if (!(mcg_cap & MCG_CTL_P) && !host) + return 1; + data = vcpu->arch.mcg_ctl; + break; + case MSR_IA32_MCG_STATUS: + data = vcpu->arch.mcg_status; + break; + case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: + last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1; + if (msr > last_msr) + return 1; + + if (!(mcg_cap & MCG_CMCI_P) && !host) + return 1; + offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2, + last_msr + 1 - MSR_IA32_MC0_CTL2); + data = vcpu->arch.mci_ctl2_banks[offset]; + break; + case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: + last_msr = MSR_IA32_MCx_CTL(bank_num) - 1; + if (msr > last_msr) + return 1; + + offset = array_index_nospec(msr - MSR_IA32_MC0_CTL, + last_msr + 1 - MSR_IA32_MC0_CTL); + data = vcpu->arch.mce_banks[offset]; + break; + default: + return 1; + } + *pdata = data; + return 0; +} + +int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) +{ + switch (msr_info->index) { + case MSR_IA32_PLATFORM_ID: + case MSR_IA32_EBL_CR_POWERON: + case MSR_IA32_LASTBRANCHFROMIP: + case MSR_IA32_LASTBRANCHTOIP: + case MSR_IA32_LASTINTFROMIP: + case MSR_IA32_LASTINTTOIP: + case MSR_AMD64_SYSCFG: + case MSR_K8_TSEG_ADDR: + case MSR_K8_TSEG_MASK: + case MSR_VM_HSAVE_PA: + case MSR_K8_INT_PENDING_MSG: + case MSR_AMD64_NB_CFG: + case MSR_FAM10H_MMIO_CONF_BASE: + case MSR_AMD64_BU_CFG2: + case MSR_IA32_PERF_CTL: + case MSR_AMD64_DC_CFG: + case MSR_AMD64_TW_CFG: + case MSR_F15H_EX_CFG: + /* + * Intel Sandy Bridge CPUs must support the RAPL (running average power + * limit) MSRs. Just return 0, as we do not want to expose the host + * data here. Do not conditionalize this on CPUID, as KVM does not do + * so for existing CPU-specific MSRs. + */ + case MSR_RAPL_POWER_UNIT: + case MSR_PP0_ENERGY_STATUS: /* Power plane 0 (core) */ + case MSR_PP1_ENERGY_STATUS: /* Power plane 1 (graphics uncore) */ + case MSR_PKG_ENERGY_STATUS: /* Total package */ + case MSR_DRAM_ENERGY_STATUS: /* DRAM controller */ + msr_info->data = 0; + break; + case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: + case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: + case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: + case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: + if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) + return kvm_pmu_get_msr(vcpu, msr_info); + msr_info->data = 0; + break; + case MSR_IA32_UCODE_REV: + msr_info->data = vcpu->arch.microcode_version; + break; + case MSR_IA32_ARCH_CAPABILITIES: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES)) + return KVM_MSR_RET_UNSUPPORTED; + msr_info->data = vcpu->arch.arch_capabilities; + break; + case MSR_IA32_PERF_CAPABILITIES: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM)) + return KVM_MSR_RET_UNSUPPORTED; + msr_info->data = vcpu->arch.perf_capabilities; + break; + case MSR_IA32_POWER_CTL: + msr_info->data = vcpu->arch.msr_ia32_power_ctl; + break; + case MSR_IA32_TSC: { + /* + * Intel SDM states that MSR_IA32_TSC read adds the TSC offset + * even when not intercepted. AMD manual doesn't explicitly + * state this but appears to behave the same. + * + * On userspace reads and writes, however, we unconditionally + * return L1's TSC value to ensure backwards-compatible + * behavior for migration. + */ + u64 offset, ratio; + + if (msr_info->host_initiated) { + offset = vcpu->arch.l1_tsc_offset; + ratio = vcpu->arch.l1_tsc_scaling_ratio; + } else { + offset = vcpu->arch.tsc_offset; + ratio = vcpu->arch.tsc_scaling_ratio; + } + + msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset; + break; + } + case MSR_IA32_CR_PAT: + msr_info->data = vcpu->arch.pat; + break; + case MSR_MTRRcap: + case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000: + case MSR_MTRRdefType: + return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data); + case 0xcd: /* fsb frequency */ + msr_info->data = 3; + break; + /* + * MSR_EBC_FREQUENCY_ID + * Conservative value valid for even the basic CPU models. + * Models 0,1: 000 in bits 23:21 indicating a bus speed of + * 100MHz, model 2 000 in bits 18:16 indicating 100MHz, + * and 266MHz for model 3, or 4. Set Core Clock + * Frequency to System Bus Frequency Ratio to 1 (bits + * 31:24) even though these are only valid for CPU + * models > 2, however guests may end up dividing or + * multiplying by zero otherwise. + */ + case MSR_EBC_FREQUENCY_ID: + msr_info->data = 1 << 24; + break; + case MSR_IA32_APICBASE: + msr_info->data = vcpu->arch.apic_base; + break; + case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: + return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data); + case MSR_IA32_TSC_DEADLINE: + msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu); + break; + case MSR_IA32_TSC_ADJUST: + msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr; + break; + case MSR_IA32_MISC_ENABLE: + msr_info->data = vcpu->arch.ia32_misc_enable_msr; + break; + case MSR_IA32_SMBASE: + if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated) + return 1; + msr_info->data = vcpu->arch.smbase; + break; + case MSR_SMI_COUNT: + msr_info->data = vcpu->arch.smi_count; + break; + case MSR_IA32_PERF_STATUS: + /* TSC increment by tick */ + msr_info->data = 1000ULL; + /* CPU multiplier */ + msr_info->data |= (((uint64_t)4ULL) << 40); + break; + case MSR_EFER: + msr_info->data = vcpu->arch.efer; + break; + case MSR_KVM_WALL_CLOCK: + if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->kvm->arch.wall_clock; + break; + case MSR_KVM_WALL_CLOCK_NEW: + if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->kvm->arch.wall_clock; + break; + case MSR_KVM_SYSTEM_TIME: + if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->arch.time; + break; + case MSR_KVM_SYSTEM_TIME_NEW: + if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->arch.time; + break; + case MSR_KVM_ASYNC_PF_EN: + if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->arch.apf.msr_en_val; + break; + case MSR_KVM_ASYNC_PF_INT: + if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->arch.apf.msr_int_val; + break; + case MSR_KVM_ASYNC_PF_ACK: + if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = 0; + break; + case MSR_KVM_STEAL_TIME: + if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->arch.st.msr_val; + break; + case MSR_KVM_PV_EOI_EN: + if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->arch.pv_eoi.msr_val; + break; + case MSR_KVM_POLL_CONTROL: + if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) + return KVM_MSR_RET_UNSUPPORTED; + + msr_info->data = vcpu->arch.msr_kvm_poll_control; + break; + case MSR_IA32_P5_MC_ADDR: + case MSR_IA32_P5_MC_TYPE: + case MSR_IA32_MCG_CAP: + case MSR_IA32_MCG_CTL: + case MSR_IA32_MCG_STATUS: + case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: + case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: + return get_msr_mce(vcpu, msr_info->index, &msr_info->data, + msr_info->host_initiated); + case MSR_IA32_XSS: + if (!msr_info->host_initiated && + !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) + return 1; + msr_info->data = vcpu->arch.ia32_xss; + break; + case MSR_K7_CLK_CTL: + /* + * Provide expected ramp-up count for K7. All other + * are set to zero, indicating minimum divisors for + * every field. + * + * This prevents guest kernels on AMD host with CPU + * type 6, model 8 and higher from exploding due to + * the rdmsr failing. + */ + msr_info->data = 0x20000000; + break; +#ifdef CONFIG_KVM_HYPERV + case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: + case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: + case HV_X64_MSR_SYNDBG_OPTIONS: + case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: + case HV_X64_MSR_CRASH_CTL: + case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: + case HV_X64_MSR_REENLIGHTENMENT_CONTROL: + case HV_X64_MSR_TSC_EMULATION_CONTROL: + case HV_X64_MSR_TSC_EMULATION_STATUS: + case HV_X64_MSR_TSC_INVARIANT_CONTROL: + return kvm_hv_get_msr_common(vcpu, + msr_info->index, &msr_info->data, + msr_info->host_initiated); +#endif + case MSR_IA32_BBL_CR_CTL3: + /* This legacy MSR exists but isn't fully documented in current + * silicon. It is however accessed by winxp in very narrow + * scenarios where it sets bit #19, itself documented as + * a "reserved" bit. Best effort attempt to source coherent + * read data here should the balance of the register be + * interpreted by the guest: + * + * L2 cache control register 3: 64GB range, 256KB size, + * enabled, latency 0x1, configured + */ + msr_info->data = 0xbe702111; + break; + case MSR_AMD64_OSVW_ID_LENGTH: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) + return 1; + msr_info->data = vcpu->arch.osvw.length; + break; + case MSR_AMD64_OSVW_STATUS: + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) + return 1; + msr_info->data = vcpu->arch.osvw.status; + break; + case MSR_PLATFORM_INFO: + if (!msr_info->host_initiated && + !vcpu->kvm->arch.guest_can_read_msr_platform_info) + return 1; + msr_info->data = vcpu->arch.msr_platform_info; + break; + case MSR_MISC_FEATURES_ENABLES: + msr_info->data = vcpu->arch.msr_misc_features_enables; + break; + case MSR_K7_HWCR: + msr_info->data = vcpu->arch.msr_hwcr; + break; +#ifdef CONFIG_X86_64 + case MSR_IA32_XFD: + if (!msr_info->host_initiated && + !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) + return 1; + + msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd; + break; + case MSR_IA32_XFD_ERR: + if (!msr_info->host_initiated && + !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) + return 1; + + msr_info->data = vcpu->arch.guest_fpu.xfd_err; + break; +#endif + case MSR_IA32_U_CET: + case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: + kvm_get_xstate_msr(vcpu, msr_info); + break; + default: + if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) + return kvm_pmu_get_msr(vcpu, msr_info); + + return KVM_MSR_RET_UNSUPPORTED; + } + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_msr_common); + +static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) +{ + return kvm_get_msr_ignored_check(vcpu, index, data, true); +} + +static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) +{ + u64 val; + + /* + * Reject writes to immutable feature MSRs if the vCPU model is frozen, + * as KVM doesn't support modifying the guest vCPU model on the fly, + * e.g. changing the VMX capabilities MSRs while L2 is active is + * nonsensical. Allow writes of the same value, e.g. so that userspace + * can blindly stuff all MSRs when emulating RESET. + */ + if (!kvm_can_set_cpuid_and_feature_msrs(vcpu) && + kvm_is_immutable_feature_msr(index) && + (do_get_msr(vcpu, index, &val) || *data != val)) + return -EINVAL; + + return kvm_set_msr_ignored_check(vcpu, index, *data, true); +} + +/* + * Read or write a bunch of msrs. All parameters are kernel addresses. + * + * @return number of msrs set successfully. + */ +static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, + struct kvm_msr_entry *entries, + int (*do_msr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data)) +{ + bool fpu_loaded = false; + int i; + + for (i = 0; i < msrs->nmsrs; ++i) { + /* + * If userspace is accessing one or more XSTATE-managed MSRs, + * temporarily load the guest's FPU state so that the guest's + * MSR value(s) is resident in hardware and thus can be accessed + * via RDMSR/WRMSR. + */ + if (!fpu_loaded && is_xstate_managed_msr(vcpu, entries[i].index)) { + kvm_load_guest_fpu(vcpu); + fpu_loaded = true; + } + if (do_msr(vcpu, entries[i].index, &entries[i].data)) + break; + } + if (fpu_loaded) + kvm_put_guest_fpu(vcpu); + + return i; +} + +/* + * Read or write a bunch of msrs. Parameters are user addresses. + * + * @return number of msrs set successfully. + */ +static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, + int (*do_msr)(struct kvm_vcpu *vcpu, + unsigned index, u64 *data), + int writeback) +{ + struct kvm_msrs msrs; + struct kvm_msr_entry *entries; + unsigned size; + int r; + + r = -EFAULT; + if (copy_from_user(&msrs, user_msrs, sizeof(msrs))) + goto out; + + r = -E2BIG; + if (msrs.nmsrs >= MAX_IO_MSRS) + goto out; + + size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; + entries = memdup_user(user_msrs->entries, size); + if (IS_ERR(entries)) { + r = PTR_ERR(entries); + goto out; + } + + r = __msr_io(vcpu, &msrs, entries, do_msr); + + if (writeback && copy_to_user(user_msrs->entries, entries, size)) + r = -EFAULT; + + kfree(entries); +out: + return r; +} + +int kvm_get_feature_msrs(struct kvm_msrs __user *user_msrs) +{ + return msr_io(NULL, user_msrs, do_get_feature_msr, 1); +} + +int kvm_get_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs) +{ + guard(srcu)(&vcpu->kvm->srcu); + + return msr_io(vcpu, user_msrs, do_get_msr, 1); +} + +int kvm_set_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs) +{ + guard(srcu)(&vcpu->kvm->srcu); + + return msr_io(vcpu, user_msrs, do_set_msr, 0); +} + +static int kvm_get_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val) +{ + u64 val; + + if (do_get_msr(vcpu, msr, &val)) + return -EINVAL; + + if (put_user(val, user_val)) + return -EFAULT; + + return 0; +} + +static int kvm_set_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val) +{ + u64 val; + + if (get_user(val, user_val)) + return -EFAULT; + + if (do_set_msr(vcpu, msr, &val)) + return -EINVAL; + + return 0; +} + +struct kvm_x86_reg_id { + __u32 index; + __u8 type; + __u8 rsvd1; + __u8 rsvd2:4; + __u8 size:4; + __u8 x86; +}; + +static int kvm_translate_kvm_reg(struct kvm_vcpu *vcpu, + struct kvm_x86_reg_id *reg) +{ + switch (reg->index) { + case KVM_REG_GUEST_SSP: + /* + * FIXME: If host-initiated accesses are ever exempted from + * ignore_msrs (in kvm_do_msr_access()), drop this manual check + * and rely on KVM's standard checks to reject accesses to regs + * that don't exist. + */ + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) + return -EINVAL; + + reg->type = KVM_X86_REG_TYPE_MSR; + reg->index = MSR_KVM_INTERNAL_GUEST_SSP; + break; + default: + return -EINVAL; + } + return 0; +} + +int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl, + void __user *argp) +{ + struct kvm_one_reg one_reg; + struct kvm_x86_reg_id *reg; + u64 __user *user_val; + bool load_fpu; + int r; + + if (copy_from_user(&one_reg, argp, sizeof(one_reg))) + return -EFAULT; + + if ((one_reg.id & KVM_REG_ARCH_MASK) != KVM_REG_X86) + return -EINVAL; + + reg = (struct kvm_x86_reg_id *)&one_reg.id; + if (reg->rsvd1 || reg->rsvd2) + return -EINVAL; + + if (reg->type == KVM_X86_REG_TYPE_KVM) { + r = kvm_translate_kvm_reg(vcpu, reg); + if (r) + return r; + } + + if (reg->type != KVM_X86_REG_TYPE_MSR) + return -EINVAL; + + if ((one_reg.id & KVM_REG_SIZE_MASK) != KVM_REG_SIZE_U64) + return -EINVAL; + + guard(srcu)(&vcpu->kvm->srcu); + + load_fpu = is_xstate_managed_msr(vcpu, reg->index); + if (load_fpu) + kvm_load_guest_fpu(vcpu); + + user_val = u64_to_user_ptr(one_reg.addr); + if (ioctl == KVM_GET_ONE_REG) + r = kvm_get_one_msr(vcpu, reg->index, user_val); + else + r = kvm_set_one_msr(vcpu, reg->index, user_val); + + if (load_fpu) + kvm_put_guest_fpu(vcpu); + return r; +} + +int kvm_get_reg_list(struct kvm_vcpu *vcpu, + struct kvm_reg_list __user *user_list) +{ + u64 nr_regs = guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) ? 1 : 0; + u64 user_nr_regs; + + if (get_user(user_nr_regs, &user_list->n)) + return -EFAULT; + + if (put_user(nr_regs, &user_list->n)) + return -EFAULT; + + if (user_nr_regs < nr_regs) + return -E2BIG; + + if (nr_regs && + put_user(KVM_X86_REG_KVM(KVM_REG_GUEST_SSP), &user_list->reg[0])) + return -EFAULT; + + return 0; +} + +static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow) +{ + struct kvm_x86_msr_filter *msr_filter; + + msr_filter = kzalloc_obj(*msr_filter, GFP_KERNEL_ACCOUNT); + if (!msr_filter) + return NULL; + + msr_filter->default_allow = default_allow; + return msr_filter; +} + +void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter) +{ + u32 i; + + if (!msr_filter) + return; + + for (i = 0; i < msr_filter->count; i++) + kfree(msr_filter->ranges[i].bitmap); + + kfree(msr_filter); +} + +static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter, + struct kvm_msr_filter_range *user_range) +{ + unsigned long *bitmap; + size_t bitmap_size; + + if (!user_range->nmsrs) + return 0; + + if (user_range->flags & ~KVM_MSR_FILTER_RANGE_VALID_MASK) + return -EINVAL; + + if (!user_range->flags) + return -EINVAL; + + bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long); + if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE) + return -EINVAL; + + bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size); + if (IS_ERR(bitmap)) + return PTR_ERR(bitmap); + + msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) { + .flags = user_range->flags, + .base = user_range->base, + .nmsrs = user_range->nmsrs, + .bitmap = bitmap, + }; + + msr_filter->count++; + return 0; +} + +int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, struct kvm_msr_filter *filter) +{ + struct kvm_x86_msr_filter *new_filter, *old_filter; + bool default_allow; + bool empty = true; + int r; + u32 i; + + if (filter->flags & ~KVM_MSR_FILTER_VALID_MASK) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) + empty &= !filter->ranges[i].nmsrs; + + default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY); + if (empty && !default_allow) + return -EINVAL; + + new_filter = kvm_alloc_msr_filter(default_allow); + if (!new_filter) + return -ENOMEM; + + for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) { + r = kvm_add_msr_filter(new_filter, &filter->ranges[i]); + if (r) { + kvm_free_msr_filter(new_filter); + return r; + } + } + + mutex_lock(&kvm->lock); + old_filter = rcu_replace_pointer(kvm->arch.msr_filter, new_filter, + mutex_is_locked(&kvm->lock)); + mutex_unlock(&kvm->lock); + synchronize_srcu(&kvm->srcu); + + kvm_free_msr_filter(old_filter); + + /* + * Recalc MSR intercepts as userspace may want to intercept accesses to + * MSRs that KVM would otherwise pass through to the guest. + */ + kvm_make_all_cpus_request(kvm, KVM_REQ_RECALC_INTERCEPTS); + + return 0; +} + + +static void kvm_probe_feature_msr(u32 msr_index) +{ + u64 data; + + if (kvm_get_feature_msr(NULL, msr_index, &data, true)) + return; + + msr_based_features[num_msr_based_features++] = msr_index; +} + +static void kvm_probe_msr_to_save(u32 msr_index) +{ + u32 dummy[2]; + + if (rdmsr_safe(msr_index, &dummy[0], &dummy[1])) + return; + + /* + * Even MSRs that are valid in the host may not be exposed to guests in + * some cases. + */ + switch (msr_index) { + case MSR_IA32_BNDCFGS: + if (!kvm_mpx_supported()) + return; + break; + case MSR_TSC_AUX: + if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) && + !kvm_cpu_cap_has(X86_FEATURE_RDPID)) + return; + break; + case MSR_IA32_UMWAIT_CONTROL: + if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG)) + return; + break; + case MSR_IA32_RTIT_CTL: + case MSR_IA32_RTIT_STATUS: + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) + return; + break; + case MSR_IA32_RTIT_CR3_MATCH: + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || + !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering)) + return; + break; + case MSR_IA32_RTIT_OUTPUT_BASE: + case MSR_IA32_RTIT_OUTPUT_MASK: + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || + (!intel_pt_validate_hw_cap(PT_CAP_topa_output) && + !intel_pt_validate_hw_cap(PT_CAP_single_range_output))) + return; + break; + case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: + if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || + (msr_index - MSR_IA32_RTIT_ADDR0_A >= + intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)) + return; + break; + case MSR_ARCH_PERFMON_PERFCTR0 ... + MSR_ARCH_PERFMON_PERFCTR0 + KVM_MAX_NR_GP_COUNTERS - 1: + if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >= + kvm_pmu_cap.num_counters_gp) + return; + break; + case MSR_ARCH_PERFMON_EVENTSEL0 ... + MSR_ARCH_PERFMON_EVENTSEL0 + KVM_MAX_NR_GP_COUNTERS - 1: + if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >= + kvm_pmu_cap.num_counters_gp) + return; + break; + case MSR_ARCH_PERFMON_FIXED_CTR0 ... + MSR_ARCH_PERFMON_FIXED_CTR0 + KVM_MAX_NR_FIXED_COUNTERS - 1: + if (msr_index - MSR_ARCH_PERFMON_FIXED_CTR0 >= + kvm_pmu_cap.num_counters_fixed) + return; + break; + case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: + case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: + if (!kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) + return; + break; + case MSR_IA32_XFD: + case MSR_IA32_XFD_ERR: + if (!kvm_cpu_cap_has(X86_FEATURE_XFD)) + return; + break; + case MSR_IA32_TSX_CTRL: + if (!(kvm_get_arch_capabilities() & ARCH_CAP_TSX_CTRL_MSR)) + return; + break; + case MSR_IA32_XSS: + if (!kvm_caps.supported_xss) + return; + break; + case MSR_IA32_U_CET: + case MSR_IA32_S_CET: + if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && + !kvm_cpu_cap_has(X86_FEATURE_IBT)) + return; + break; + case MSR_IA32_INT_SSP_TAB: + if (!kvm_cpu_cap_has(X86_FEATURE_LM)) + return; + fallthrough; + case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: + if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK)) + return; + break; + default: + break; + } + + msrs_to_save[num_msrs_to_save++] = msr_index; +} + +void kvm_init_msr_lists(void) +{ + unsigned i; + + BUILD_BUG_ON_MSG(KVM_MAX_NR_FIXED_COUNTERS != 3, + "Please update the fixed PMCs in msrs_to_save_pmu[]"); + + num_msrs_to_save = 0; + num_emulated_msrs = 0; + num_msr_based_features = 0; + + for (i = 0; i < ARRAY_SIZE(msrs_to_save_base); i++) + kvm_probe_msr_to_save(msrs_to_save_base[i]); + + if (enable_pmu) { + for (i = 0; i < ARRAY_SIZE(msrs_to_save_pmu); i++) + kvm_probe_msr_to_save(msrs_to_save_pmu[i]); + } + + for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) { + if (!kvm_x86_call(has_emulated_msr)(NULL, + emulated_msrs_all[i])) + continue; + + emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i]; + } + + for (i = KVM_FIRST_EMULATED_VMX_MSR; i <= KVM_LAST_EMULATED_VMX_MSR; i++) + kvm_probe_feature_msr(i); + + for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) + kvm_probe_feature_msr(msr_based_features_all_except_vmx[i]); +} + +int kvm_spec_ctrl_test_value(u64 value) +{ + /* + * test that setting IA32_SPEC_CTRL to given value + * is allowed by the host processor + */ + + u64 saved_value; + unsigned long flags; + int ret = 0; + + local_irq_save(flags); + + if (rdmsrq_safe(MSR_IA32_SPEC_CTRL, &saved_value)) + ret = 1; + else if (wrmsrq_safe(MSR_IA32_SPEC_CTRL, value)) + ret = 1; + else + wrmsrq(MSR_IA32_SPEC_CTRL, saved_value); + + local_irq_restore(flags); + + return ret; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_spec_ctrl_test_value); diff --git a/arch/x86/kvm/msrs.h b/arch/x86/kvm/msrs.h new file mode 100644 index 000000000000..25b75fe39b80 --- /dev/null +++ b/arch/x86/kvm/msrs.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_X86_KVM_MSR_H +#define ARCH_X86_KVM_MSR_H + +#include +#include + +#include "cpuid.h" +#include "regs.h" + +extern bool report_ignored_msrs; +extern bool ignore_msrs; + +static inline void kvm_pr_unimpl_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) +{ + if (report_ignored_msrs) + vcpu_unimpl(vcpu, "Unhandled WRMSR(0x%x) = 0x%llx\n", msr, data); +} + +static inline void kvm_pr_unimpl_rdmsr(struct kvm_vcpu *vcpu, u32 msr) +{ + if (report_ignored_msrs) + vcpu_unimpl(vcpu, "Unhandled RDMSR(0x%x)\n", msr); +} + +/* + * The first...last VMX feature MSRs that are emulated by KVM. This may or may + * not cover all known VMX MSRs, as KVM doesn't emulate an MSR until there's an + * associated feature that KVM supports for nested virtualization. + */ +#define KVM_FIRST_EMULATED_VMX_MSR MSR_IA32_VMX_BASIC +#define KVM_LAST_EMULATED_VMX_MSR MSR_IA32_VMX_VMFUNC + +/* + * KVM's internal, non-ABI indices for synthetic MSRs. The values themselves + * are arbitrary and have no meaning, the only requirement is that they don't + * conflict with "real" MSRs that KVM supports. Use values at the upper end + * of KVM's reserved paravirtual MSR range to minimize churn, i.e. these values + * will be usable until KVM exhausts its supply of paravirtual MSR indices. + */ +#define MSR_KVM_INTERNAL_GUEST_SSP 0x4b564dff + +#define MSR_IA32_CR_PAT_DEFAULT \ + PAT_VALUE(WB, WT, UC_MINUS, UC, WB, WT, UC_MINUS, UC) + +void kvm_init_msr_lists(void); +int kvm_get_msr_index_list(struct kvm_msr_list __user *user_msr_list); +int kvm_get_feature_msr_index_list(struct kvm_msr_list __user *user_msr_list); +int kvm_get_feature_msrs(struct kvm_msrs __user *user_msrs); + +int kvm_get_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs); +int kvm_set_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs); + +int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl, + void __user *argp); +int kvm_get_reg_list(struct kvm_vcpu *vcpu, + struct kvm_reg_list __user *user_list); + +void kvm_user_return_msr_cpu_online(void); +void drop_user_return_notifiers(void); +void kvm_destroy_user_return_msrs(void); + +fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu); +fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg); + +int kvm_emulator_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index, + u64 *pdata); +int kvm_emulator_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index, + u64 data); +int kvm_emulator_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); + +bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type); + +enum kvm_msr_access { + MSR_TYPE_R = BIT(0), + MSR_TYPE_W = BIT(1), + MSR_TYPE_RW = MSR_TYPE_R | MSR_TYPE_W, +}; + +/* + * Internal error codes that are used to indicate that MSR emulation encountered + * an error that should result in #GP in the guest, unless userspace handles it. + * Note, '1', '0', and negative numbers are off limits, as they are used by KVM + * as part of KVM's lightly documented internal KVM_RUN return codes. + * + * UNSUPPORTED - The MSR isn't supported, either because it is completely + * unknown to KVM, or because the MSR should not exist according + * to the vCPU model. + * + * FILTERED - Access to the MSR is denied by a userspace MSR filter. + */ +#define KVM_MSR_RET_UNSUPPORTED 2 +#define KVM_MSR_RET_FILTERED 3 + +int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, struct kvm_msr_filter *filter); +void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter); + +int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); +int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); + +u64 kvm_get_arch_capabilities(void); +int kvm_spec_ctrl_test_value(u64 value); + +#define CET_US_RESERVED_BITS GENMASK(9, 6) +#define CET_US_SHSTK_MASK_BITS GENMASK(1, 0) +#define CET_US_IBT_MASK_BITS (GENMASK_ULL(5, 2) | GENMASK_ULL(63, 10)) +#define CET_US_LEGACY_BITMAP_BASE(data) ((data) >> 12) + +static inline bool kvm_is_valid_u_s_cet(struct kvm_vcpu *vcpu, u64 data) +{ + if (data & CET_US_RESERVED_BITS) + return false; + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && + (data & CET_US_SHSTK_MASK_BITS)) + return false; + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) && + (data & CET_US_IBT_MASK_BITS)) + return false; + if (!IS_ALIGNED(CET_US_LEGACY_BITMAP_BASE(data), 4)) + return false; + /* IBT can be suppressed iff the TRACKER isn't WAIT_ENDBR. */ + if ((data & CET_SUPPRESS) && (data & CET_WAIT_ENDBR)) + return false; + + return true; +} + +#endif diff --git a/arch/x86/kvm/mtrr.c b/arch/x86/kvm/mtrr.c index 6f74e2b27c1e..c4ec024943bb 100644 --- a/arch/x86/kvm/mtrr.c +++ b/arch/x86/kvm/mtrr.c @@ -19,7 +19,7 @@ #include #include "cpuid.h" -#include "x86.h" +#include "msrs.h" static u64 *find_mtrr(struct kvm_vcpu *vcpu, unsigned int msr) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c579dddbf7d7..fe61d024e6dc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -80,7 +80,6 @@ #include #include #include -#include #include #include #include @@ -90,8 +89,6 @@ #define CREATE_TRACE_POINTS #include "trace.h" -#define MAX_IO_MSRS 256 - /* * Note, kvm_caps fields should *never* have default values, all fields must be * recomputed from scratch during vendor module load, e.g. to account for a @@ -108,17 +105,6 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_host); #define emul_to_vcpu(ctxt) \ ((struct kvm_vcpu *)(ctxt)->vcpu) -/* EFER defaults: - * - enable syscall per default because its emulated by KVM - * - enable LME and LMA per default on 64 bit KVM - */ -#ifdef CONFIG_X86_64 -static -u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA)); -#else -static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); -#endif - #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE @@ -146,13 +132,6 @@ EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits); EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg); EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl); -static bool __read_mostly ignore_msrs = 0; -module_param(ignore_msrs, bool, 0644); - -bool __read_mostly report_ignored_msrs = true; -module_param(report_ignored_msrs, bool, 0644); -EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs); - unsigned int min_timer_period_us = 200; module_param(min_timer_period_us, uint, 0644); @@ -179,27 +158,6 @@ module_param(pi_inject_timer, bint, 0644); static bool __read_mostly mitigate_smt_rsb; module_param(mitigate_smt_rsb, bool, 0444); -/* - * Restoring the host value for MSRs that are only consumed when running in - * usermode, e.g. SYSCALL MSRs and TSC_AUX, can be deferred until the CPU - * returns to userspace, i.e. the kernel can run with the guest's value. - */ -#define KVM_MAX_NR_USER_RETURN_MSRS 16 - -struct kvm_user_return_msrs { - struct user_return_notifier urn; - bool registered; - struct kvm_user_return_msr_values { - u64 host; - u64 curr; - } values[KVM_MAX_NR_USER_RETURN_MSRS]; -}; - -u32 __read_mostly kvm_nr_uret_msrs; -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_nr_uret_msrs); -static u32 __read_mostly kvm_uret_msrs_list[KVM_MAX_NR_USER_RETURN_MSRS]; -static DEFINE_PER_CPU(struct kvm_user_return_msrs, user_return_msrs); - #define KVM_SUPPORTED_XCR0 (XFEATURE_MASK_FP | XFEATURE_MASK_SSE \ | XFEATURE_MASK_YMM | XFEATURE_MASK_BNDREGS \ | XFEATURE_MASK_BNDCSR | XFEATURE_MASK_AVX512 \ @@ -302,249 +260,6 @@ const struct kvm_stats_header kvm_vcpu_stats_header = { static struct kmem_cache *x86_emulator_cache; -/* - * The three MSR lists(msrs_to_save, emulated_msrs, msr_based_features) track - * the set of MSRs that KVM exposes to userspace through KVM_GET_MSRS, - * KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. msrs_to_save holds MSRs that - * require host support, i.e. should be probed via RDMSR. emulated_msrs holds - * MSRs that KVM emulates without strictly requiring host support. - * msr_based_features holds MSRs that enumerate features, i.e. are effectively - * CPUID leafs. Note, msr_based_features isn't mutually exclusive with - * msrs_to_save and emulated_msrs. - */ - -static const u32 msrs_to_save_base[] = { - MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP, - MSR_STAR, -#ifdef CONFIG_X86_64 - MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, -#endif - MSR_IA32_TSC, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA, - MSR_IA32_FEAT_CTL, MSR_IA32_BNDCFGS, MSR_TSC_AUX, - MSR_IA32_SPEC_CTRL, MSR_IA32_TSX_CTRL, - MSR_IA32_RTIT_CTL, MSR_IA32_RTIT_STATUS, MSR_IA32_RTIT_CR3_MATCH, - MSR_IA32_RTIT_OUTPUT_BASE, MSR_IA32_RTIT_OUTPUT_MASK, - MSR_IA32_RTIT_ADDR0_A, MSR_IA32_RTIT_ADDR0_B, - MSR_IA32_RTIT_ADDR1_A, MSR_IA32_RTIT_ADDR1_B, - MSR_IA32_RTIT_ADDR2_A, MSR_IA32_RTIT_ADDR2_B, - MSR_IA32_RTIT_ADDR3_A, MSR_IA32_RTIT_ADDR3_B, - MSR_IA32_UMWAIT_CONTROL, - - MSR_IA32_XFD, MSR_IA32_XFD_ERR, MSR_IA32_XSS, - - MSR_IA32_U_CET, MSR_IA32_S_CET, - MSR_IA32_PL0_SSP, MSR_IA32_PL1_SSP, MSR_IA32_PL2_SSP, - MSR_IA32_PL3_SSP, MSR_IA32_INT_SSP_TAB, - MSR_IA32_DEBUGCTLMSR, - MSR_IA32_LASTBRANCHFROMIP, MSR_IA32_LASTBRANCHTOIP, - MSR_IA32_LASTINTFROMIP, MSR_IA32_LASTINTTOIP, -}; - -static const u32 msrs_to_save_pmu[] = { - MSR_ARCH_PERFMON_FIXED_CTR0, MSR_ARCH_PERFMON_FIXED_CTR1, - MSR_ARCH_PERFMON_FIXED_CTR0 + 2, - MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS, - MSR_CORE_PERF_GLOBAL_CTRL, - MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG, - - /* This part of MSRs should match KVM_MAX_NR_INTEL_GP_COUNTERS. */ - MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1, - MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3, - MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5, - MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7, - MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1, - MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3, - MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5, - MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7, - - MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3, - MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3, - - /* This part of MSRs should match KVM_MAX_NR_AMD_GP_COUNTERS. */ - MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2, - MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5, - MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2, - MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5, - - MSR_AMD64_PERF_CNTR_GLOBAL_CTL, - MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, - MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, - MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET, -}; - -static u32 msrs_to_save[ARRAY_SIZE(msrs_to_save_base) + - ARRAY_SIZE(msrs_to_save_pmu)]; -static unsigned num_msrs_to_save; - -static const u32 emulated_msrs_all[] = { - MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, - MSR_KVM_SYSTEM_TIME_NEW, MSR_KVM_WALL_CLOCK_NEW, - -#ifdef CONFIG_KVM_HYPERV - HV_X64_MSR_GUEST_OS_ID, HV_X64_MSR_HYPERCALL, - HV_X64_MSR_TIME_REF_COUNT, HV_X64_MSR_REFERENCE_TSC, - HV_X64_MSR_TSC_FREQUENCY, HV_X64_MSR_APIC_FREQUENCY, - HV_X64_MSR_CRASH_P0, HV_X64_MSR_CRASH_P1, HV_X64_MSR_CRASH_P2, - HV_X64_MSR_CRASH_P3, HV_X64_MSR_CRASH_P4, HV_X64_MSR_CRASH_CTL, - HV_X64_MSR_RESET, - HV_X64_MSR_VP_INDEX, - HV_X64_MSR_VP_RUNTIME, - HV_X64_MSR_SCONTROL, - HV_X64_MSR_STIMER0_CONFIG, - HV_X64_MSR_VP_ASSIST_PAGE, - HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL, - HV_X64_MSR_TSC_EMULATION_STATUS, HV_X64_MSR_TSC_INVARIANT_CONTROL, - HV_X64_MSR_SYNDBG_OPTIONS, - HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS, - HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER, - HV_X64_MSR_SYNDBG_PENDING_BUFFER, -#endif - - MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, - MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK, - - MSR_IA32_TSC_ADJUST, - MSR_IA32_TSC_DEADLINE, - MSR_IA32_ARCH_CAPABILITIES, - MSR_IA32_PERF_CAPABILITIES, - MSR_IA32_MISC_ENABLE, - MSR_IA32_MCG_STATUS, - MSR_IA32_MCG_CTL, - MSR_IA32_MCG_EXT_CTL, - MSR_IA32_SMBASE, - MSR_SMI_COUNT, - MSR_PLATFORM_INFO, - MSR_MISC_FEATURES_ENABLES, - MSR_AMD64_VIRT_SPEC_CTRL, - MSR_AMD64_TSC_RATIO, - MSR_IA32_POWER_CTL, - MSR_IA32_UCODE_REV, - - /* - * KVM always supports the "true" VMX control MSRs, even if the host - * does not. The VMX MSRs as a whole are considered "emulated" as KVM - * doesn't strictly require them to exist in the host (ignoring that - * KVM would refuse to load in the first place if the core set of MSRs - * aren't supported). - */ - MSR_IA32_VMX_BASIC, - MSR_IA32_VMX_TRUE_PINBASED_CTLS, - MSR_IA32_VMX_TRUE_PROCBASED_CTLS, - MSR_IA32_VMX_TRUE_EXIT_CTLS, - MSR_IA32_VMX_TRUE_ENTRY_CTLS, - MSR_IA32_VMX_MISC, - MSR_IA32_VMX_CR0_FIXED0, - MSR_IA32_VMX_CR4_FIXED0, - MSR_IA32_VMX_VMCS_ENUM, - MSR_IA32_VMX_PROCBASED_CTLS2, - MSR_IA32_VMX_EPT_VPID_CAP, - MSR_IA32_VMX_VMFUNC, - - MSR_K7_HWCR, - MSR_KVM_POLL_CONTROL, -}; - -static u32 emulated_msrs[ARRAY_SIZE(emulated_msrs_all)]; -static unsigned num_emulated_msrs; - -/* - * List of MSRs that control the existence of MSR-based features, i.e. MSRs - * that are effectively CPUID leafs. VMX MSRs are also included in the set of - * feature MSRs, but are handled separately to allow expedited lookups. - */ -static const u32 msr_based_features_all_except_vmx[] = { - MSR_AMD64_DE_CFG, - MSR_IA32_UCODE_REV, - MSR_IA32_ARCH_CAPABILITIES, - MSR_IA32_PERF_CAPABILITIES, - MSR_PLATFORM_INFO, -}; - -static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all_except_vmx) + - (KVM_LAST_EMULATED_VMX_MSR - KVM_FIRST_EMULATED_VMX_MSR + 1)]; -static unsigned int num_msr_based_features; - -/* - * All feature MSRs except uCode revID, which tracks the currently loaded uCode - * patch, are immutable once the vCPU model is defined. - */ -static bool kvm_is_immutable_feature_msr(u32 msr) -{ - int i; - - if (msr >= KVM_FIRST_EMULATED_VMX_MSR && msr <= KVM_LAST_EMULATED_VMX_MSR) - return true; - - for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) { - if (msr == msr_based_features_all_except_vmx[i]) - return msr != MSR_IA32_UCODE_REV; - } - - return false; -} - -static bool kvm_is_advertised_msr(u32 msr_index) -{ - unsigned int i; - - for (i = 0; i < num_msrs_to_save; i++) { - if (msrs_to_save[i] == msr_index) - return true; - } - - for (i = 0; i < num_emulated_msrs; i++) { - if (emulated_msrs[i] == msr_index) - return true; - } - - return false; -} - -typedef int (*msr_access_t)(struct kvm_vcpu *vcpu, u32 index, u64 *data, - bool host_initiated); - -static __always_inline int kvm_do_msr_access(struct kvm_vcpu *vcpu, u32 msr, - u64 *data, bool host_initiated, - enum kvm_msr_access rw, - msr_access_t msr_access_fn) -{ - const char *op = rw == MSR_TYPE_W ? "wrmsr" : "rdmsr"; - int ret; - - BUILD_BUG_ON(rw != MSR_TYPE_R && rw != MSR_TYPE_W); - - /* - * Zero the data on read failures to avoid leaking stack data to the - * guest and/or userspace, e.g. if the failure is ignored below. - */ - ret = msr_access_fn(vcpu, msr, data, host_initiated); - if (ret && rw == MSR_TYPE_R) - *data = 0; - - if (ret != KVM_MSR_RET_UNSUPPORTED) - return ret; - - /* - * Userspace is allowed to read MSRs, and write '0' to MSRs, that KVM - * advertises to userspace, even if an MSR isn't fully supported. - * Simply check that @data is '0', which covers both the write '0' case - * and all reads (in which case @data is zeroed on failure; see above). - */ - if (host_initiated && !*data && kvm_is_advertised_msr(msr)) - return 0; - - if (!ignore_msrs) { - kvm_debug_ratelimited("unhandled %s: 0x%x data 0x%llx\n", - op, msr, *data); - return ret; - } - - if (report_ignored_msrs) - kvm_pr_unimpl("ignored %s: 0x%x data 0x%llx\n", op, msr, *data); - - return 0; -} - static struct kmem_cache *kvm_alloc_emulator_cache(void) { unsigned int useroffset = offsetof(struct x86_emulate_ctxt, src); @@ -558,128 +273,6 @@ static struct kmem_cache *kvm_alloc_emulator_cache(void) static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); -static void kvm_destroy_user_return_msrs(void) -{ - int cpu; - - for_each_possible_cpu(cpu) - WARN_ON_ONCE(per_cpu(user_return_msrs, cpu).registered); - - kvm_nr_uret_msrs = 0; -} - -static void kvm_on_user_return(struct user_return_notifier *urn) -{ - unsigned slot; - struct kvm_user_return_msrs *msrs - = container_of(urn, struct kvm_user_return_msrs, urn); - struct kvm_user_return_msr_values *values; - - msrs->registered = false; - user_return_notifier_unregister(urn); - - for (slot = 0; slot < kvm_nr_uret_msrs; ++slot) { - values = &msrs->values[slot]; - if (values->host != values->curr) { - wrmsrq(kvm_uret_msrs_list[slot], values->host); - values->curr = values->host; - } - } -} - -static int kvm_probe_user_return_msr(u32 msr) -{ - u64 val; - int ret; - - preempt_disable(); - ret = rdmsrq_safe(msr, &val); - if (ret) - goto out; - ret = wrmsrq_safe(msr, val); -out: - preempt_enable(); - return ret; -} - -int kvm_add_user_return_msr(u32 msr) -{ - BUG_ON(kvm_nr_uret_msrs >= KVM_MAX_NR_USER_RETURN_MSRS); - - if (kvm_probe_user_return_msr(msr)) - return -1; - - kvm_uret_msrs_list[kvm_nr_uret_msrs] = msr; - return kvm_nr_uret_msrs++; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_add_user_return_msr); - -int kvm_find_user_return_msr(u32 msr) -{ - int i; - - for (i = 0; i < kvm_nr_uret_msrs; ++i) { - if (kvm_uret_msrs_list[i] == msr) - return i; - } - return -1; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_find_user_return_msr); - -static void kvm_user_return_msr_cpu_online(void) -{ - struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); - u64 value; - int i; - - for (i = 0; i < kvm_nr_uret_msrs; ++i) { - rdmsrq_safe(kvm_uret_msrs_list[i], &value); - msrs->values[i].host = value; - msrs->values[i].curr = value; - } -} - -static void kvm_user_return_register_notifier(struct kvm_user_return_msrs *msrs) -{ - if (!msrs->registered) { - msrs->urn.on_user_return = kvm_on_user_return; - user_return_notifier_register(&msrs->urn); - msrs->registered = true; - } -} - -int kvm_set_user_return_msr(unsigned slot, u64 value, u64 mask) -{ - struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); - int err; - - value = (value & mask) | (msrs->values[slot].host & ~mask); - if (value == msrs->values[slot].curr) - return 0; - err = wrmsrq_safe(kvm_uret_msrs_list[slot], value); - if (err) - return 1; - - msrs->values[slot].curr = value; - kvm_user_return_register_notifier(msrs); - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_user_return_msr); - -u64 kvm_get_user_return_msr(unsigned int slot) -{ - return this_cpu_ptr(&user_return_msrs)->values[slot].curr; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_user_return_msr); - -static void drop_user_return_notifiers(void) -{ - struct kvm_user_return_msrs *msrs = this_cpu_ptr(&user_return_msrs); - - if (msrs->registered) - kvm_on_user_return(&msrs->urn); -} - /* * Handle a fault on a hardware virtualization (VMX or SVM) instruction. * @@ -934,17 +527,6 @@ int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_complete_insn_gp); -static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err) -{ - if (err) { - kvm_inject_gp(vcpu, 0); - return 1; - } - - return kvm_emulate_instruction(vcpu, EMULTYPE_NO_DECODE | EMULTYPE_SKIP | - EMULTYPE_COMPLETE_USER_EXIT); -} - void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault, bool from_hardware) { @@ -1051,13 +633,6 @@ static void kvm_load_host_pkru(struct kvm_vcpu *vcpu) } } -#ifdef CONFIG_X86_64 -static inline u64 kvm_guest_supported_xfd(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.guest_supported_xcr0 & XFEATURE_MASK_USER_DYNAMIC; -} -#endif - int __kvm_set_xcr(struct kvm_vcpu *vcpu, u32 index, u64 xcr) { u64 xcr0 = xcr; @@ -1176,595 +751,6 @@ int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdpmc); -/* - * Some IA32_ARCH_CAPABILITIES bits have dependencies on MSRs that KVM - * does not yet virtualize. These include: - * 10 - MISC_PACKAGE_CTRLS - * 11 - ENERGY_FILTERING_CTL - * 12 - DOITM - * 18 - FB_CLEAR_CTRL - * 21 - XAPIC_DISABLE_STATUS - * 23 - OVERCLOCKING_STATUS - */ - -#define KVM_SUPPORTED_ARCH_CAP \ - (ARCH_CAP_RDCL_NO | ARCH_CAP_IBRS_ALL | ARCH_CAP_RSBA | \ - ARCH_CAP_SKIP_VMENTRY_L1DFLUSH | ARCH_CAP_SSB_NO | ARCH_CAP_MDS_NO | \ - ARCH_CAP_PSCHANGE_MC_NO | ARCH_CAP_TSX_CTRL_MSR | ARCH_CAP_TAA_NO | \ - ARCH_CAP_SBDR_SSDP_NO | ARCH_CAP_FBSDP_NO | ARCH_CAP_PSDP_NO | \ - ARCH_CAP_FB_CLEAR | ARCH_CAP_RRSBA | ARCH_CAP_PBRSB_NO | ARCH_CAP_GDS_NO | \ - ARCH_CAP_RFDS_NO | ARCH_CAP_RFDS_CLEAR | ARCH_CAP_BHI_NO | ARCH_CAP_ITS_NO) - -static u64 kvm_get_arch_capabilities(void) -{ - u64 data = kvm_host.arch_capabilities & KVM_SUPPORTED_ARCH_CAP; - - /* - * If nx_huge_pages is enabled, KVM's shadow paging will ensure that - * the nested hypervisor runs with NX huge pages. If it is not, - * L1 is anyway vulnerable to ITLB_MULTIHIT exploits from other - * L1 guests, so it need not worry about its own (L2) guests. - */ - data |= ARCH_CAP_PSCHANGE_MC_NO; - - /* - * If we're doing cache flushes (either "always" or "cond") - * we will do one whenever the guest does a vmlaunch/vmresume. - * If an outer hypervisor is doing the cache flush for us - * (ARCH_CAP_SKIP_VMENTRY_L1DFLUSH), we can safely pass that - * capability to the guest too, and if EPT is disabled we're not - * vulnerable. Overall, only VMENTER_L1D_FLUSH_NEVER will - * require a nested hypervisor to do a flush of its own. - */ - if (l1tf_vmx_mitigation != VMENTER_L1D_FLUSH_NEVER) - data |= ARCH_CAP_SKIP_VMENTRY_L1DFLUSH; - - if (!boot_cpu_has_bug(X86_BUG_CPU_MELTDOWN)) - data |= ARCH_CAP_RDCL_NO; - if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS)) - data |= ARCH_CAP_SSB_NO; - if (!boot_cpu_has_bug(X86_BUG_MDS)) - data |= ARCH_CAP_MDS_NO; - if (!boot_cpu_has_bug(X86_BUG_RFDS)) - data |= ARCH_CAP_RFDS_NO; - if (!boot_cpu_has_bug(X86_BUG_ITS)) - data |= ARCH_CAP_ITS_NO; - - if (!boot_cpu_has(X86_FEATURE_RTM)) { - /* - * If RTM=0 because the kernel has disabled TSX, the host might - * have TAA_NO or TSX_CTRL. Clear TAA_NO (the guest sees RTM=0 - * and therefore knows that there cannot be TAA) but keep - * TSX_CTRL: some buggy userspaces leave it set on tsx=on hosts, - * and we want to allow migrating those guests to tsx=off hosts. - */ - data &= ~ARCH_CAP_TAA_NO; - } else if (!boot_cpu_has_bug(X86_BUG_TAA)) { - data |= ARCH_CAP_TAA_NO; - } else { - /* - * Nothing to do here; we emulate TSX_CTRL if present on the - * host so the guest can choose between disabling TSX or - * using VERW to clear CPU buffers. - */ - } - - if (!boot_cpu_has_bug(X86_BUG_GDS) || gds_ucode_mitigated()) - data |= ARCH_CAP_GDS_NO; - - return data; -} - -static int kvm_get_feature_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, - bool host_initiated) -{ - WARN_ON_ONCE(!host_initiated); - - switch (index) { - case MSR_IA32_ARCH_CAPABILITIES: - *data = kvm_get_arch_capabilities(); - break; - case MSR_IA32_PERF_CAPABILITIES: - *data = kvm_caps.supported_perf_cap; - break; - case MSR_PLATFORM_INFO: - *data = MSR_PLATFORM_INFO_CPUID_FAULT; - break; - case MSR_IA32_UCODE_REV: - rdmsrq_safe(index, data); - break; - default: - return kvm_x86_call(get_feature_msr)(index, data); - } - return 0; -} - -static int do_get_feature_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) -{ - return kvm_do_msr_access(vcpu, index, data, true, MSR_TYPE_R, - kvm_get_feature_msr); -} - -static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) -{ - if (efer & EFER_AUTOIBRS && !guest_cpu_cap_has(vcpu, X86_FEATURE_AUTOIBRS)) - return false; - - if (efer & EFER_FFXSR && !guest_cpu_cap_has(vcpu, X86_FEATURE_FXSR_OPT)) - return false; - - if (efer & EFER_SVME && !guest_cpu_cap_has(vcpu, X86_FEATURE_SVM)) - return false; - - if (efer & (EFER_LME | EFER_LMA) && - !guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) - return false; - - if (efer & EFER_NX && !guest_cpu_cap_has(vcpu, X86_FEATURE_NX)) - return false; - - return true; - -} -bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) -{ - if (efer & efer_reserved_bits) - return false; - - return __kvm_valid_efer(vcpu, efer); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_valid_efer); - -static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) -{ - u64 old_efer = vcpu->arch.efer; - u64 efer = msr_info->data; - int r; - - if (efer & efer_reserved_bits) - return 1; - - if (!msr_info->host_initiated) { - if (!__kvm_valid_efer(vcpu, efer)) - return 1; - - if (is_paging(vcpu) && - (vcpu->arch.efer & EFER_LME) != (efer & EFER_LME)) - return 1; - } - - efer &= ~EFER_LMA; - efer |= vcpu->arch.efer & EFER_LMA; - - r = kvm_x86_call(set_efer)(vcpu, efer); - if (r) { - WARN_ON(r > 0); - return r; - } - - if ((efer ^ old_efer) & KVM_MMU_EFER_ROLE_BITS) - kvm_mmu_reset_context(vcpu); - - if (!static_cpu_has(X86_FEATURE_XSAVES) && - (efer & EFER_SVME)) - kvm_hv_xsaves_xsavec_maybe_warn(vcpu); - - return 0; -} - -void kvm_enable_efer_bits(u64 mask) -{ - efer_reserved_bits &= ~mask; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_enable_efer_bits); - -bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type) -{ - struct kvm_x86_msr_filter *msr_filter; - struct msr_bitmap_range *ranges; - struct kvm *kvm = vcpu->kvm; - bool allowed; - int idx; - u32 i; - - /* x2APIC MSRs do not support filtering. */ - if (index >= 0x800 && index <= 0x8ff) - return true; - - idx = srcu_read_lock(&kvm->srcu); - - msr_filter = srcu_dereference(kvm->arch.msr_filter, &kvm->srcu); - if (!msr_filter) { - allowed = true; - goto out; - } - - allowed = msr_filter->default_allow; - ranges = msr_filter->ranges; - - for (i = 0; i < msr_filter->count; i++) { - u32 start = ranges[i].base; - u32 end = start + ranges[i].nmsrs; - u32 flags = ranges[i].flags; - unsigned long *bitmap = ranges[i].bitmap; - - if ((index >= start) && (index < end) && (flags & type)) { - allowed = test_bit(index - start, bitmap); - break; - } - } - -out: - srcu_read_unlock(&kvm->srcu, idx); - - return allowed; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_msr_allowed); - -/* - * Write @data into the MSR specified by @index. Select MSR specific fault - * checks are bypassed if @host_initiated is %true. - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -static int __kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data, - bool host_initiated) -{ - struct msr_data msr; - - switch (index) { - case MSR_FS_BASE: - case MSR_GS_BASE: - case MSR_KERNEL_GS_BASE: - case MSR_CSTAR: - case MSR_LSTAR: - if (is_noncanonical_msr_address(data, vcpu)) - return 1; - break; - case MSR_IA32_SYSENTER_EIP: - case MSR_IA32_SYSENTER_ESP: - /* - * IA32_SYSENTER_ESP and IA32_SYSENTER_EIP cause #GP if - * non-canonical address is written on Intel but not on - * AMD (which ignores the top 32-bits, because it does - * not implement 64-bit SYSENTER). - * - * 64-bit code should hence be able to write a non-canonical - * value on AMD. Making the address canonical ensures that - * vmentry does not fail on Intel after writing a non-canonical - * value, and that something deterministic happens if the guest - * invokes 64-bit SYSENTER. - */ - data = __canonical_address(data, max_host_virt_addr_bits()); - break; - case MSR_TSC_AUX: - if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) - return 1; - - if (!host_initiated && - !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) && - !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID)) - return 1; - - /* - * Per Intel's SDM, bits 63:32 are reserved, but AMD's APM has - * incomplete and conflicting architectural behavior. Current - * AMD CPUs completely ignore bits 63:32, i.e. they aren't - * reserved and always read as zeros. Enforce Intel's reserved - * bits check if the guest CPU is Intel compatible, otherwise - * clear the bits. This ensures cross-vendor migration will - * provide consistent behavior for the guest. - */ - if (guest_cpuid_is_intel_compatible(vcpu) && (data >> 32) != 0) - return 1; - - data = (u32)data; - break; - case MSR_IA32_U_CET: - case MSR_IA32_S_CET: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && - !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT)) - return KVM_MSR_RET_UNSUPPORTED; - if (!kvm_is_valid_u_s_cet(vcpu, data)) - return 1; - break; - case MSR_KVM_INTERNAL_GUEST_SSP: - if (!host_initiated) - return 1; - fallthrough; - /* - * Note that the MSR emulation here is flawed when a vCPU - * doesn't support the Intel 64 architecture. The expected - * architectural behavior in this case is that the upper 32 - * bits do not exist and should always read '0'. However, - * because the actual hardware on which the virtual CPU is - * running does support Intel 64, XRSTORS/XSAVES in the - * guest could observe behavior that violates the - * architecture. Intercepting XRSTORS/XSAVES for this - * special case isn't deemed worthwhile. - */ - case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) - return KVM_MSR_RET_UNSUPPORTED; - /* - * MSR_IA32_INT_SSP_TAB is not present on processors that do - * not support Intel 64 architecture. - */ - if (index == MSR_IA32_INT_SSP_TAB && !guest_cpu_cap_has(vcpu, X86_FEATURE_LM)) - return KVM_MSR_RET_UNSUPPORTED; - if (is_noncanonical_msr_address(data, vcpu)) - return 1; - /* All SSP MSRs except MSR_IA32_INT_SSP_TAB must be 4-byte aligned */ - if (index != MSR_IA32_INT_SSP_TAB && !IS_ALIGNED(data, 4)) - return 1; - break; - } - - msr.data = data; - msr.index = index; - msr.host_initiated = host_initiated; - - return kvm_x86_call(set_msr)(vcpu, &msr); -} - -static int _kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, - bool host_initiated) -{ - return __kvm_set_msr(vcpu, index, *data, host_initiated); -} - -static int kvm_set_msr_ignored_check(struct kvm_vcpu *vcpu, - u32 index, u64 data, bool host_initiated) -{ - return kvm_do_msr_access(vcpu, index, &data, host_initiated, MSR_TYPE_W, - _kvm_set_msr); -} - -/* - * Read the MSR specified by @index into @data. Select MSR specific fault - * checks are bypassed if @host_initiated is %true. - * Returns 0 on success, non-0 otherwise. - * Assumes vcpu_load() was already called. - */ -static int __kvm_get_msr(struct kvm_vcpu *vcpu, u32 index, u64 *data, - bool host_initiated) -{ - struct msr_data msr; - int ret; - - switch (index) { - case MSR_TSC_AUX: - if (!kvm_is_supported_user_return_msr(MSR_TSC_AUX)) - return 1; - - if (!host_initiated && - !guest_cpu_cap_has(vcpu, X86_FEATURE_RDTSCP) && - !guest_cpu_cap_has(vcpu, X86_FEATURE_RDPID)) - return 1; - break; - case MSR_IA32_U_CET: - case MSR_IA32_S_CET: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && - !guest_cpu_cap_has(vcpu, X86_FEATURE_IBT)) - return KVM_MSR_RET_UNSUPPORTED; - break; - case MSR_KVM_INTERNAL_GUEST_SSP: - if (!host_initiated) - return 1; - fallthrough; - case MSR_IA32_PL0_SSP ... MSR_IA32_INT_SSP_TAB: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) - return KVM_MSR_RET_UNSUPPORTED; - break; - } - - msr.index = index; - msr.host_initiated = host_initiated; - - ret = kvm_x86_call(get_msr)(vcpu, &msr); - if (!ret) - *data = msr.data; - return ret; -} - -int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) -{ - return __kvm_set_msr(vcpu, index, data, true); -} - -int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) -{ - return __kvm_get_msr(vcpu, index, data, true); -} - -static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu, - u32 index, u64 *data, bool host_initiated) -{ - return kvm_do_msr_access(vcpu, index, data, host_initiated, MSR_TYPE_R, - __kvm_get_msr); -} - -int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) -{ - return kvm_get_msr_ignored_check(vcpu, index, data, false); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_read); - -int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) -{ - return kvm_set_msr_ignored_check(vcpu, index, data, false); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_emulate_msr_write); - -int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data) -{ - if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_READ)) - return KVM_MSR_RET_FILTERED; - - return __kvm_emulate_msr_read(vcpu, index, data); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_read); - -int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data) -{ - if (!kvm_msr_allowed(vcpu, index, KVM_MSR_FILTER_WRITE)) - return KVM_MSR_RET_FILTERED; - - return __kvm_emulate_msr_write(vcpu, index, data); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_msr_write); - - -static void complete_userspace_rdmsr(struct kvm_vcpu *vcpu) -{ - if (!vcpu->run->msr.error) { - kvm_eax_write(vcpu, vcpu->run->msr.data); - kvm_edx_write(vcpu, vcpu->run->msr.data >> 32); - } -} - -static int complete_emulated_msr_access(struct kvm_vcpu *vcpu) -{ - return complete_emulated_insn_gp(vcpu, vcpu->run->msr.error); -} - -static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu) -{ - complete_userspace_rdmsr(vcpu); - return complete_emulated_msr_access(vcpu); -} - -static int complete_fast_msr_access(struct kvm_vcpu *vcpu) -{ - return kvm_x86_call(complete_emulated_msr)(vcpu, vcpu->run->msr.error); -} - -static int complete_fast_rdmsr(struct kvm_vcpu *vcpu) -{ - complete_userspace_rdmsr(vcpu); - return complete_fast_msr_access(vcpu); -} - -static int complete_fast_rdmsr_imm(struct kvm_vcpu *vcpu) -{ - if (!vcpu->run->msr.error) - kvm_register_write(vcpu, vcpu->arch.cui_rdmsr_imm_reg, - vcpu->run->msr.data); - - return complete_fast_msr_access(vcpu); -} - -static u64 kvm_msr_reason(int r) -{ - switch (r) { - case KVM_MSR_RET_UNSUPPORTED: - return KVM_MSR_EXIT_REASON_UNKNOWN; - case KVM_MSR_RET_FILTERED: - return KVM_MSR_EXIT_REASON_FILTER; - default: - return KVM_MSR_EXIT_REASON_INVAL; - } -} - -static int kvm_msr_user_space(struct kvm_vcpu *vcpu, u32 index, - u32 exit_reason, u64 data, - int (*completion)(struct kvm_vcpu *vcpu), - int r) -{ - u64 msr_reason = kvm_msr_reason(r); - - /* Check if the user wanted to know about this MSR fault */ - if (!(vcpu->kvm->arch.user_space_msr_mask & msr_reason)) - return 0; - - vcpu->run->exit_reason = exit_reason; - vcpu->run->msr.error = 0; - memset(vcpu->run->msr.pad, 0, sizeof(vcpu->run->msr.pad)); - vcpu->run->msr.reason = msr_reason; - vcpu->run->msr.index = index; - vcpu->run->msr.data = data; - vcpu->arch.complete_userspace_io = completion; - - return 1; -} - -static int __kvm_emulate_rdmsr(struct kvm_vcpu *vcpu, u32 msr, int reg, - int (*complete_rdmsr)(struct kvm_vcpu *)) -{ - u64 data; - int r; - - r = kvm_emulate_msr_read(vcpu, msr, &data); - - if (!r) { - trace_kvm_msr_read(msr, data); - - if (reg < 0) { - kvm_eax_write(vcpu, data); - kvm_edx_write(vcpu, data >> 32); - } else { - kvm_register_write(vcpu, reg, data); - } - } else { - /* MSR read failed? See if we should ask user space */ - if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_RDMSR, 0, - complete_rdmsr, r)) - return 0; - trace_kvm_msr_read_ex(msr); - } - - return kvm_x86_call(complete_emulated_msr)(vcpu, r); -} - -int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu) -{ - return __kvm_emulate_rdmsr(vcpu, kvm_ecx_read(vcpu), -1, - complete_fast_rdmsr); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr); - -int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) -{ - vcpu->arch.cui_rdmsr_imm_reg = reg; - - return __kvm_emulate_rdmsr(vcpu, msr, reg, complete_fast_rdmsr_imm); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_rdmsr_imm); - -static int __kvm_emulate_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) -{ - int r; - - r = kvm_emulate_msr_write(vcpu, msr, data); - if (!r) { - trace_kvm_msr_write(msr, data); - } else { - /* MSR write failed? See if we should ask user space */ - if (kvm_msr_user_space(vcpu, msr, KVM_EXIT_X86_WRMSR, data, - complete_fast_msr_access, r)) - return 0; - /* Signal all other negative errors to userspace */ - if (r < 0) - return r; - trace_kvm_msr_write_ex(msr, data); - } - - return kvm_x86_call(complete_emulated_msr)(vcpu, r); -} - -int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) -{ - return __kvm_emulate_wrmsr(vcpu, kvm_ecx_read(vcpu), - kvm_read_edx_eax(vcpu)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr); - -int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) -{ - return __kvm_emulate_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_wrmsr_imm); - int kvm_emulate_as_nop(struct kvm_vcpu *vcpu) { return kvm_skip_emulated_instruction(vcpu); @@ -1836,72 +822,6 @@ static inline bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu) kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending(); } -static fastpath_t __handle_fastpath_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) -{ - if (!kvm_pmu_is_fastpath_emulation_allowed(vcpu)) - return EXIT_FASTPATH_NONE; - - switch (msr) { - case APIC_BASE_MSR + (APIC_ICR >> 4): - if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(vcpu->arch.apic) || - kvm_x2apic_icr_write_fast(vcpu->arch.apic, data)) - return EXIT_FASTPATH_NONE; - break; - case MSR_IA32_TSC_DEADLINE: - kvm_set_lapic_tscdeadline_msr(vcpu, data); - break; - default: - return EXIT_FASTPATH_NONE; - } - - trace_kvm_msr_write(msr, data); - - if (!kvm_skip_emulated_instruction(vcpu)) - return EXIT_FASTPATH_EXIT_USERSPACE; - - return EXIT_FASTPATH_REENTER_GUEST; -} - -fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu) -{ - return __handle_fastpath_wrmsr(vcpu, kvm_ecx_read(vcpu), - kvm_read_edx_eax(vcpu)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr); - -fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg) -{ - return __handle_fastpath_wrmsr(vcpu, msr, kvm_register_read(vcpu, reg)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(handle_fastpath_wrmsr_imm); - -/* - * Adapt set_msr() to msr_io()'s calling convention - */ -static int do_get_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) -{ - return kvm_get_msr_ignored_check(vcpu, index, data, true); -} - -static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data) -{ - u64 val; - - /* - * Reject writes to immutable feature MSRs if the vCPU model is frozen, - * as KVM doesn't support modifying the guest vCPU model on the fly, - * e.g. changing the VMX capabilities MSRs while L2 is active is - * nonsensical. Allow writes of the same value, e.g. so that userspace - * can blindly stuff all MSRs when emulating RESET. - */ - if (!kvm_can_set_cpuid_and_feature_msrs(vcpu) && - kvm_is_immutable_feature_msr(index) && - (do_get_msr(vcpu, index, &val) || *data != val)) - return -EINVAL; - - return kvm_set_msr_ignored_check(vcpu, index, *data, true); -} - #ifdef CONFIG_X86_64 struct pvclock_clock { int vclock_mode; @@ -1968,72 +888,6 @@ static s64 get_kvmclock_base_ns(void) } #endif -static void kvm_write_wall_clock(struct kvm *kvm, gpa_t wall_clock, int sec_hi_ofs) -{ - int version; - int r; - struct pvclock_wall_clock wc; - u32 wc_sec_hi; - u64 wall_nsec; - - if (!wall_clock) - return; - - r = kvm_read_guest(kvm, wall_clock, &version, sizeof(version)); - if (r) - return; - - if (version & 1) - ++version; /* first time write, random junk */ - - ++version; - - if (kvm_write_guest(kvm, wall_clock, &version, sizeof(version))) - return; - - wall_nsec = kvm_get_wall_clock_epoch(kvm); - - wc.nsec = do_div(wall_nsec, NSEC_PER_SEC); - wc.sec = (u32)wall_nsec; /* overflow in 2106 guest time */ - wc.version = version; - - kvm_write_guest(kvm, wall_clock, &wc, sizeof(wc)); - - if (sec_hi_ofs) { - wc_sec_hi = wall_nsec >> 32; - kvm_write_guest(kvm, wall_clock + sec_hi_ofs, - &wc_sec_hi, sizeof(wc_sec_hi)); - } - - version++; - kvm_write_guest(kvm, wall_clock, &version, sizeof(version)); -} - -static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time, - bool old_msr, bool host_initiated) -{ - struct kvm_arch *ka = &vcpu->kvm->arch; - - if (vcpu->vcpu_id == 0 && !host_initiated) { - if (ka->boot_vcpu_runs_old_kvmclock != old_msr) - kvm_make_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu); - - ka->boot_vcpu_runs_old_kvmclock = old_msr; - } - - vcpu->arch.time = system_time; - kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); - - /* we verify if the enable bit is set... */ - if (system_time & 1) - kvm_gpc_activate(&vcpu->arch.pv_time, system_time & ~1ULL, - sizeof(struct pvclock_vcpu_time_info)); - else - kvm_gpc_deactivate(&vcpu->arch.pv_time); - - return; -} - static uint32_t div_frac(uint32_t dividend, uint32_t divisor) { do_shl32_div32(dividend, divisor); @@ -3078,151 +1932,6 @@ static void kvm_gen_kvmclock_update(struct kvm_vcpu *v) } } -/* These helpers are safe iff @msr is known to be an MCx bank MSR. */ -static bool is_mci_control_msr(u32 msr) -{ - return (msr & 3) == 0; -} -static bool is_mci_status_msr(u32 msr) -{ - return (msr & 3) == 1; -} - -/* - * On AMD, HWCR[McStatusWrEn] controls whether setting MCi_STATUS results in #GP. - */ -static bool can_set_mci_status(struct kvm_vcpu *vcpu) -{ - /* McStatusWrEn enabled? */ - if (guest_cpuid_is_amd_compatible(vcpu)) - return !!(vcpu->arch.msr_hwcr & BIT_ULL(18)); - - return false; -} - -static int set_msr_mce(struct kvm_vcpu *vcpu, struct msr_data *msr_info) -{ - u64 mcg_cap = vcpu->arch.mcg_cap; - unsigned bank_num = mcg_cap & 0xff; - u32 msr = msr_info->index; - u64 data = msr_info->data; - u32 offset, last_msr; - - switch (msr) { - case MSR_IA32_MCG_STATUS: - vcpu->arch.mcg_status = data; - break; - case MSR_IA32_MCG_CTL: - if (!(mcg_cap & MCG_CTL_P) && - (data || !msr_info->host_initiated)) - return 1; - if (data != 0 && data != ~(u64)0) - return 1; - vcpu->arch.mcg_ctl = data; - break; - case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: - last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1; - if (msr > last_msr) - return 1; - - if (!(mcg_cap & MCG_CMCI_P) && (data || !msr_info->host_initiated)) - return 1; - /* An attempt to write a 1 to a reserved bit raises #GP */ - if (data & ~(MCI_CTL2_CMCI_EN | MCI_CTL2_CMCI_THRESHOLD_MASK)) - return 1; - offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2, - last_msr + 1 - MSR_IA32_MC0_CTL2); - vcpu->arch.mci_ctl2_banks[offset] = data; - break; - case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: - last_msr = MSR_IA32_MCx_CTL(bank_num) - 1; - if (msr > last_msr) - return 1; - - /* - * Only 0 or all 1s can be written to IA32_MCi_CTL, all other - * values are architecturally undefined. But, some Linux - * kernels clear bit 10 in bank 4 to workaround a BIOS/GART TLB - * issue on AMD K8s, allow bit 10 to be clear when setting all - * other bits in order to avoid an uncaught #GP in the guest. - * - * UNIXWARE clears bit 0 of MC1_CTL to ignore correctable, - * single-bit ECC data errors. - */ - if (is_mci_control_msr(msr) && - data != 0 && (data | (1 << 10) | 1) != ~(u64)0) - return 1; - - /* - * All CPUs allow writing 0 to MCi_STATUS MSRs to clear the MSR. - * AMD-based CPUs allow non-zero values, but if and only if - * HWCR[McStatusWrEn] is set. - */ - if (!msr_info->host_initiated && is_mci_status_msr(msr) && - data != 0 && !can_set_mci_status(vcpu)) - return 1; - - offset = array_index_nospec(msr - MSR_IA32_MC0_CTL, - last_msr + 1 - MSR_IA32_MC0_CTL); - vcpu->arch.mce_banks[offset] = data; - break; - default: - return 1; - } - return 0; -} - -static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) -{ - gpa_t gpa = data & ~0x3f; - - /* Bits 4:5 are reserved, Should be zero */ - if (data & 0x30) - return 1; - - if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_VMEXIT) && - (data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT)) - return 1; - - if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT) && - (data & KVM_ASYNC_PF_DELIVERY_AS_INT)) - return 1; - - if (!lapic_in_kernel(vcpu)) - return data ? 1 : 0; - - if (__kvm_pv_async_pf_enabled(data) && - kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, - sizeof(u64))) - return 1; - - vcpu->arch.apf.msr_en_val = data; - - if (__kvm_pv_async_pf_enabled(data)) { - kvm_async_pf_wakeup_all(vcpu); - } else { - kvm_clear_async_pf_completion_queue(vcpu); - kvm_async_pf_hash_reset(vcpu); - } - return 0; -} - -static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data) -{ - /* Bits 8-63 are reserved */ - if (data >> 8) - return 1; - - if (!lapic_in_kernel(vcpu)) - return 1; - - vcpu->arch.apf.msr_int_val = data; - - vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK; - - return 0; -} - static void kvmclock_reset(struct kvm_vcpu *vcpu) { kvm_gpc_deactivate(&vcpu->arch.pv_time); @@ -3383,899 +2092,6 @@ static void record_steal_time(struct kvm_vcpu *vcpu) mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); } -/* - * Returns true if the MSR in question is managed via XSTATE, i.e. is context - * switched with the rest of guest FPU state. - * - * Note, S_CET is _not_ saved/restored via XSAVES/XRSTORS. - */ -static bool is_xstate_managed_msr(struct kvm_vcpu *vcpu, u32 msr) -{ - if (!vcpu) - return false; - - switch (msr) { - case MSR_IA32_U_CET: - return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) || - guest_cpu_cap_has(vcpu, X86_FEATURE_IBT); - case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: - return guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK); - default: - return false; - } -} - -/* - * Lock (and if necessary, re-load) the guest FPU, i.e. XSTATE, and access an - * MSR that is managed via XSTATE. Note, the caller is responsible for doing - * the initial FPU load, this helper only ensures that guest state is resident - * in hardware (the kernel can load its FPU state in IRQ context). - * - * Note, loading guest values for U_CET and PL[0-3]_SSP while executing in the - * kernel is safe, as U_CET is specific to userspace, and PL[0-3]_SSP are only - * consumed when transitioning to lower privilege levels, i.e. are effectively - * only consumed by userspace as well. - */ -static __always_inline void kvm_access_xstate_msr(struct kvm_vcpu *vcpu, - struct msr_data *msr_info, - int access) -{ - BUILD_BUG_ON(access != MSR_TYPE_R && access != MSR_TYPE_W); - - KVM_BUG_ON(!is_xstate_managed_msr(vcpu, msr_info->index), vcpu->kvm); - KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm); - - kvm_fpu_get(); - if (access == MSR_TYPE_R) - rdmsrq(msr_info->index, msr_info->data); - else - wrmsrq(msr_info->index, msr_info->data); - kvm_fpu_put(); -} - -static void kvm_set_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) -{ - kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_W); -} - -static void kvm_get_xstate_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) -{ - kvm_access_xstate_msr(vcpu, msr_info, MSR_TYPE_R); -} - -int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) -{ - u32 msr = msr_info->index; - u64 data = msr_info->data; - - /* - * Do not allow host-initiated writes to trigger the Xen hypercall - * page setup; it could incur locking paths which are not expected - * if userspace sets the MSR in an unusual location. - */ - if (kvm_xen_is_hypercall_page_msr(vcpu->kvm, msr) && - !msr_info->host_initiated) - return kvm_xen_write_hypercall_page(vcpu, data); - - switch (msr) { - case MSR_AMD64_NB_CFG: - case MSR_IA32_UCODE_WRITE: - case MSR_VM_HSAVE_PA: - case MSR_AMD64_PATCH_LOADER: - case MSR_AMD64_BU_CFG2: - case MSR_AMD64_DC_CFG: - case MSR_AMD64_TW_CFG: - case MSR_F15H_EX_CFG: - break; - - case MSR_IA32_UCODE_REV: - if (msr_info->host_initiated) - vcpu->arch.microcode_version = data; - break; - case MSR_IA32_ARCH_CAPABILITIES: - if (!msr_info->host_initiated || - !guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES)) - return KVM_MSR_RET_UNSUPPORTED; - vcpu->arch.arch_capabilities = data; - break; - case MSR_IA32_PERF_CAPABILITIES: - if (!msr_info->host_initiated || - !guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM)) - return KVM_MSR_RET_UNSUPPORTED; - - if (data & ~kvm_caps.supported_perf_cap) - return 1; - - /* - * Note, this is not just a performance optimization! KVM - * disallows changing feature MSRs after the vCPU has run; PMU - * refresh will bug the VM if called after the vCPU has run. - */ - if (vcpu->arch.perf_capabilities == data) - break; - - vcpu->arch.perf_capabilities = data; - kvm_pmu_refresh(vcpu); - kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu); - break; - case MSR_IA32_PRED_CMD: { - u64 reserved_bits = ~(PRED_CMD_IBPB | PRED_CMD_SBPB); - - if (!msr_info->host_initiated) { - if ((!guest_has_pred_cmd_msr(vcpu))) - return 1; - - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SPEC_CTRL) && - !guest_cpu_cap_has(vcpu, X86_FEATURE_AMD_IBPB)) - reserved_bits |= PRED_CMD_IBPB; - - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SBPB)) - reserved_bits |= PRED_CMD_SBPB; - } - - if (!boot_cpu_has(X86_FEATURE_IBPB)) - reserved_bits |= PRED_CMD_IBPB; - - if (!boot_cpu_has(X86_FEATURE_SBPB)) - reserved_bits |= PRED_CMD_SBPB; - - if (data & reserved_bits) - return 1; - - if (!data) - break; - - wrmsrq(MSR_IA32_PRED_CMD, data); - break; - } - case MSR_IA32_FLUSH_CMD: - if (!msr_info->host_initiated && - !guest_cpu_cap_has(vcpu, X86_FEATURE_FLUSH_L1D)) - return 1; - - if (!boot_cpu_has(X86_FEATURE_FLUSH_L1D) || (data & ~L1D_FLUSH)) - return 1; - if (!data) - break; - - wrmsrq(MSR_IA32_FLUSH_CMD, L1D_FLUSH); - break; - case MSR_EFER: - return set_efer(vcpu, msr_info); - case MSR_K7_HWCR: { - /* - * Allow McStatusWrEn and TscFreqSel. (Linux guests from v3.2 - * through at least v6.6 whine if TscFreqSel is clear, - * depending on F/M/S. - */ - u64 valid = BIT_ULL(18) | BIT_ULL(24); - - data &= ~(u64)0x40; /* ignore flush filter disable */ - data &= ~(u64)0x100; /* ignore ignne emulation enable */ - data &= ~(u64)0x8; /* ignore TLB cache disable */ - - if (guest_cpu_cap_has(vcpu, X86_FEATURE_GP_ON_USER_CPUID)) - valid |= MSR_K7_HWCR_CPUID_USER_DIS; - - if (data & ~valid) { - kvm_pr_unimpl_wrmsr(vcpu, msr, data); - return 1; - } - vcpu->arch.msr_hwcr = data; - break; - } - case MSR_FAM10H_MMIO_CONF_BASE: - if (data != 0) { - kvm_pr_unimpl_wrmsr(vcpu, msr, data); - return 1; - } - break; - case MSR_IA32_CR_PAT: - if (!kvm_pat_valid(data)) - return 1; - - vcpu->arch.pat = data; - break; - case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000: - case MSR_MTRRdefType: - return kvm_mtrr_set_msr(vcpu, msr, data); - case MSR_IA32_APICBASE: - return kvm_apic_set_base(vcpu, data, msr_info->host_initiated); - case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: - return kvm_x2apic_msr_write(vcpu, msr, data); - case MSR_IA32_TSC_DEADLINE: - kvm_set_lapic_tscdeadline_msr(vcpu, data); - break; - case MSR_IA32_TSC_ADJUST: - if (guest_cpu_cap_has(vcpu, X86_FEATURE_TSC_ADJUST)) { - if (!msr_info->host_initiated) { - s64 adj = data - vcpu->arch.ia32_tsc_adjust_msr; - adjust_tsc_offset_guest(vcpu, adj); - /* Before back to guest, tsc_timestamp must be adjusted - * as well, otherwise guest's percpu pvclock time could jump. - */ - kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); - } - vcpu->arch.ia32_tsc_adjust_msr = data; - } - break; - case MSR_IA32_MISC_ENABLE: { - u64 old_val = vcpu->arch.ia32_misc_enable_msr; - - if (!msr_info->host_initiated) { - /* RO bits */ - if ((old_val ^ data) & MSR_IA32_MISC_ENABLE_PMU_RO_MASK) - return 1; - - /* R bits, i.e. writes are ignored, but don't fault. */ - data = data & ~MSR_IA32_MISC_ENABLE_EMON; - data |= old_val & MSR_IA32_MISC_ENABLE_EMON; - } - - if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) && - ((old_val ^ data) & MSR_IA32_MISC_ENABLE_MWAIT)) { - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_XMM3)) - return 1; - vcpu->arch.ia32_misc_enable_msr = data; - vcpu->arch.cpuid_dynamic_bits_dirty = true; - } else { - vcpu->arch.ia32_misc_enable_msr = data; - } - break; - } - case MSR_IA32_SMBASE: - if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated) - return 1; - vcpu->arch.smbase = data; - break; - case MSR_IA32_POWER_CTL: - vcpu->arch.msr_ia32_power_ctl = data; - break; - case MSR_IA32_TSC: - if (msr_info->host_initiated) { - kvm_synchronize_tsc(vcpu, &data); - } else if (!vcpu->arch.guest_tsc_protected) { - u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; - adjust_tsc_offset_guest(vcpu, adj); - vcpu->arch.ia32_tsc_adjust_msr += adj; - } - break; - case MSR_IA32_XSS: - if (!guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) - return KVM_MSR_RET_UNSUPPORTED; - - if (data & ~vcpu->arch.guest_supported_xss) - return 1; - if (vcpu->arch.ia32_xss == data) - break; - vcpu->arch.ia32_xss = data; - vcpu->arch.cpuid_dynamic_bits_dirty = true; - break; - case MSR_SMI_COUNT: - if (!msr_info->host_initiated) - return 1; - vcpu->arch.smi_count = data; - break; - case MSR_KVM_WALL_CLOCK_NEW: - if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return KVM_MSR_RET_UNSUPPORTED; - - vcpu->kvm->arch.wall_clock = data; - kvm_write_wall_clock(vcpu->kvm, data, 0); - break; - case MSR_KVM_WALL_CLOCK: - if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return KVM_MSR_RET_UNSUPPORTED; - - vcpu->kvm->arch.wall_clock = data; - kvm_write_wall_clock(vcpu->kvm, data, 0); - break; - case MSR_KVM_SYSTEM_TIME_NEW: - if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return KVM_MSR_RET_UNSUPPORTED; - - kvm_write_system_time(vcpu, data, false, msr_info->host_initiated); - break; - case MSR_KVM_SYSTEM_TIME: - if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return KVM_MSR_RET_UNSUPPORTED; - - kvm_write_system_time(vcpu, data, true, msr_info->host_initiated); - break; - case MSR_KVM_ASYNC_PF_EN: - if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) - return KVM_MSR_RET_UNSUPPORTED; - - if (kvm_pv_enable_async_pf(vcpu, data)) - return 1; - break; - case MSR_KVM_ASYNC_PF_INT: - if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return KVM_MSR_RET_UNSUPPORTED; - - if (kvm_pv_enable_async_pf_int(vcpu, data)) - return 1; - break; - case MSR_KVM_ASYNC_PF_ACK: - if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return KVM_MSR_RET_UNSUPPORTED; - if (data & 0x1) { - /* - * Pairs with the smp_mb__after_atomic() in - * kvm_arch_async_page_present_queued(). - */ - smp_store_mb(vcpu->arch.apf.pageready_pending, false); - - kvm_check_async_pf_completion(vcpu); - } - break; - case MSR_KVM_STEAL_TIME: - if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) - return KVM_MSR_RET_UNSUPPORTED; - - if (unlikely(!sched_info_on())) - return 1; - - if (data & KVM_STEAL_RESERVED_MASK) - return 1; - - vcpu->arch.st.msr_val = data; - - if (!(data & KVM_MSR_ENABLED)) - break; - - kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); - - break; - case MSR_KVM_PV_EOI_EN: - if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) - return KVM_MSR_RET_UNSUPPORTED; - - if (kvm_lapic_set_pv_eoi(vcpu, data, sizeof(u8))) - return 1; - break; - - case MSR_KVM_POLL_CONTROL: - if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) - return KVM_MSR_RET_UNSUPPORTED; - - /* only enable bit supported */ - if (data & (-1ULL << 1)) - return 1; - - vcpu->arch.msr_kvm_poll_control = data; - break; - - case MSR_IA32_MCG_CTL: - case MSR_IA32_MCG_STATUS: - case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: - case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: - return set_msr_mce(vcpu, msr_info); - - case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: - case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: - case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: - case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: - if (kvm_pmu_is_valid_msr(vcpu, msr)) - return kvm_pmu_set_msr(vcpu, msr_info); - - if (data) - kvm_pr_unimpl_wrmsr(vcpu, msr, data); - break; - case MSR_K7_CLK_CTL: - /* - * Ignore all writes to this no longer documented MSR. - * Writes are only relevant for old K7 processors, - * all pre-dating SVM, but a recommended workaround from - * AMD for these chips. It is possible to specify the - * affected processor models on the command line, hence - * the need to ignore the workaround. - */ - break; -#ifdef CONFIG_KVM_HYPERV - case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: - case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: - case HV_X64_MSR_SYNDBG_OPTIONS: - case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: - case HV_X64_MSR_CRASH_CTL: - case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: - case HV_X64_MSR_REENLIGHTENMENT_CONTROL: - case HV_X64_MSR_TSC_EMULATION_CONTROL: - case HV_X64_MSR_TSC_EMULATION_STATUS: - case HV_X64_MSR_TSC_INVARIANT_CONTROL: - return kvm_hv_set_msr_common(vcpu, msr, data, - msr_info->host_initiated); -#endif - case MSR_IA32_BBL_CR_CTL3: - /* Drop writes to this legacy MSR -- see rdmsr - * counterpart for further detail. - */ - kvm_pr_unimpl_wrmsr(vcpu, msr, data); - break; - case MSR_AMD64_OSVW_ID_LENGTH: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) - return 1; - vcpu->arch.osvw.length = data; - break; - case MSR_AMD64_OSVW_STATUS: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) - return 1; - vcpu->arch.osvw.status = data; - break; - case MSR_PLATFORM_INFO: - if (!msr_info->host_initiated) - return 1; - vcpu->arch.msr_platform_info = data; - break; - case MSR_MISC_FEATURES_ENABLES: - if (data & ~MSR_MISC_FEATURES_ENABLES_CPUID_FAULT || - (data & MSR_MISC_FEATURES_ENABLES_CPUID_FAULT && - !(vcpu->arch.msr_platform_info & MSR_PLATFORM_INFO_CPUID_FAULT))) - return 1; - vcpu->arch.msr_misc_features_enables = data; - break; -#ifdef CONFIG_X86_64 - case MSR_IA32_XFD: - if (!msr_info->host_initiated && - !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) - return 1; - - if (data & ~kvm_guest_supported_xfd(vcpu)) - return 1; - - fpu_update_guest_xfd(&vcpu->arch.guest_fpu, data); - break; - case MSR_IA32_XFD_ERR: - if (!msr_info->host_initiated && - !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) - return 1; - - if (data & ~kvm_guest_supported_xfd(vcpu)) - return 1; - - vcpu->arch.guest_fpu.xfd_err = data; - break; -#endif - case MSR_IA32_U_CET: - case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: - kvm_set_xstate_msr(vcpu, msr_info); - break; - default: - if (kvm_pmu_is_valid_msr(vcpu, msr)) - return kvm_pmu_set_msr(vcpu, msr_info); - - return KVM_MSR_RET_UNSUPPORTED; - } - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_msr_common); - -static int get_msr_mce(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) -{ - u64 data; - u64 mcg_cap = vcpu->arch.mcg_cap; - unsigned bank_num = mcg_cap & 0xff; - u32 offset, last_msr; - - switch (msr) { - case MSR_IA32_P5_MC_ADDR: - case MSR_IA32_P5_MC_TYPE: - data = 0; - break; - case MSR_IA32_MCG_CAP: - data = vcpu->arch.mcg_cap; - break; - case MSR_IA32_MCG_CTL: - if (!(mcg_cap & MCG_CTL_P) && !host) - return 1; - data = vcpu->arch.mcg_ctl; - break; - case MSR_IA32_MCG_STATUS: - data = vcpu->arch.mcg_status; - break; - case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: - last_msr = MSR_IA32_MCx_CTL2(bank_num) - 1; - if (msr > last_msr) - return 1; - - if (!(mcg_cap & MCG_CMCI_P) && !host) - return 1; - offset = array_index_nospec(msr - MSR_IA32_MC0_CTL2, - last_msr + 1 - MSR_IA32_MC0_CTL2); - data = vcpu->arch.mci_ctl2_banks[offset]; - break; - case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: - last_msr = MSR_IA32_MCx_CTL(bank_num) - 1; - if (msr > last_msr) - return 1; - - offset = array_index_nospec(msr - MSR_IA32_MC0_CTL, - last_msr + 1 - MSR_IA32_MC0_CTL); - data = vcpu->arch.mce_banks[offset]; - break; - default: - return 1; - } - *pdata = data; - return 0; -} - -int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) -{ - switch (msr_info->index) { - case MSR_IA32_PLATFORM_ID: - case MSR_IA32_EBL_CR_POWERON: - case MSR_IA32_LASTBRANCHFROMIP: - case MSR_IA32_LASTBRANCHTOIP: - case MSR_IA32_LASTINTFROMIP: - case MSR_IA32_LASTINTTOIP: - case MSR_AMD64_SYSCFG: - case MSR_K8_TSEG_ADDR: - case MSR_K8_TSEG_MASK: - case MSR_VM_HSAVE_PA: - case MSR_K8_INT_PENDING_MSG: - case MSR_AMD64_NB_CFG: - case MSR_FAM10H_MMIO_CONF_BASE: - case MSR_AMD64_BU_CFG2: - case MSR_IA32_PERF_CTL: - case MSR_AMD64_DC_CFG: - case MSR_AMD64_TW_CFG: - case MSR_F15H_EX_CFG: - /* - * Intel Sandy Bridge CPUs must support the RAPL (running average power - * limit) MSRs. Just return 0, as we do not want to expose the host - * data here. Do not conditionalize this on CPUID, as KVM does not do - * so for existing CPU-specific MSRs. - */ - case MSR_RAPL_POWER_UNIT: - case MSR_PP0_ENERGY_STATUS: /* Power plane 0 (core) */ - case MSR_PP1_ENERGY_STATUS: /* Power plane 1 (graphics uncore) */ - case MSR_PKG_ENERGY_STATUS: /* Total package */ - case MSR_DRAM_ENERGY_STATUS: /* DRAM controller */ - msr_info->data = 0; - break; - case MSR_K7_EVNTSEL0 ... MSR_K7_EVNTSEL3: - case MSR_K7_PERFCTR0 ... MSR_K7_PERFCTR3: - case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: - case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: - if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) - return kvm_pmu_get_msr(vcpu, msr_info); - msr_info->data = 0; - break; - case MSR_IA32_UCODE_REV: - msr_info->data = vcpu->arch.microcode_version; - break; - case MSR_IA32_ARCH_CAPABILITIES: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_ARCH_CAPABILITIES)) - return KVM_MSR_RET_UNSUPPORTED; - msr_info->data = vcpu->arch.arch_capabilities; - break; - case MSR_IA32_PERF_CAPABILITIES: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM)) - return KVM_MSR_RET_UNSUPPORTED; - msr_info->data = vcpu->arch.perf_capabilities; - break; - case MSR_IA32_POWER_CTL: - msr_info->data = vcpu->arch.msr_ia32_power_ctl; - break; - case MSR_IA32_TSC: { - /* - * Intel SDM states that MSR_IA32_TSC read adds the TSC offset - * even when not intercepted. AMD manual doesn't explicitly - * state this but appears to behave the same. - * - * On userspace reads and writes, however, we unconditionally - * return L1's TSC value to ensure backwards-compatible - * behavior for migration. - */ - u64 offset, ratio; - - if (msr_info->host_initiated) { - offset = vcpu->arch.l1_tsc_offset; - ratio = vcpu->arch.l1_tsc_scaling_ratio; - } else { - offset = vcpu->arch.tsc_offset; - ratio = vcpu->arch.tsc_scaling_ratio; - } - - msr_info->data = kvm_scale_tsc(rdtsc(), ratio) + offset; - break; - } - case MSR_IA32_CR_PAT: - msr_info->data = vcpu->arch.pat; - break; - case MSR_MTRRcap: - case MTRRphysBase_MSR(0) ... MSR_MTRRfix4K_F8000: - case MSR_MTRRdefType: - return kvm_mtrr_get_msr(vcpu, msr_info->index, &msr_info->data); - case 0xcd: /* fsb frequency */ - msr_info->data = 3; - break; - /* - * MSR_EBC_FREQUENCY_ID - * Conservative value valid for even the basic CPU models. - * Models 0,1: 000 in bits 23:21 indicating a bus speed of - * 100MHz, model 2 000 in bits 18:16 indicating 100MHz, - * and 266MHz for model 3, or 4. Set Core Clock - * Frequency to System Bus Frequency Ratio to 1 (bits - * 31:24) even though these are only valid for CPU - * models > 2, however guests may end up dividing or - * multiplying by zero otherwise. - */ - case MSR_EBC_FREQUENCY_ID: - msr_info->data = 1 << 24; - break; - case MSR_IA32_APICBASE: - msr_info->data = vcpu->arch.apic_base; - break; - case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff: - return kvm_x2apic_msr_read(vcpu, msr_info->index, &msr_info->data); - case MSR_IA32_TSC_DEADLINE: - msr_info->data = kvm_get_lapic_tscdeadline_msr(vcpu); - break; - case MSR_IA32_TSC_ADJUST: - msr_info->data = (u64)vcpu->arch.ia32_tsc_adjust_msr; - break; - case MSR_IA32_MISC_ENABLE: - msr_info->data = vcpu->arch.ia32_misc_enable_msr; - break; - case MSR_IA32_SMBASE: - if (!IS_ENABLED(CONFIG_KVM_SMM) || !msr_info->host_initiated) - return 1; - msr_info->data = vcpu->arch.smbase; - break; - case MSR_SMI_COUNT: - msr_info->data = vcpu->arch.smi_count; - break; - case MSR_IA32_PERF_STATUS: - /* TSC increment by tick */ - msr_info->data = 1000ULL; - /* CPU multiplier */ - msr_info->data |= (((uint64_t)4ULL) << 40); - break; - case MSR_EFER: - msr_info->data = vcpu->arch.efer; - break; - case MSR_KVM_WALL_CLOCK: - if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->kvm->arch.wall_clock; - break; - case MSR_KVM_WALL_CLOCK_NEW: - if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->kvm->arch.wall_clock; - break; - case MSR_KVM_SYSTEM_TIME: - if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->arch.time; - break; - case MSR_KVM_SYSTEM_TIME_NEW: - if (!guest_pv_has(vcpu, KVM_FEATURE_CLOCKSOURCE2)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->arch.time; - break; - case MSR_KVM_ASYNC_PF_EN: - if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->arch.apf.msr_en_val; - break; - case MSR_KVM_ASYNC_PF_INT: - if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->arch.apf.msr_int_val; - break; - case MSR_KVM_ASYNC_PF_ACK: - if (!guest_pv_has(vcpu, KVM_FEATURE_ASYNC_PF_INT)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = 0; - break; - case MSR_KVM_STEAL_TIME: - if (!guest_pv_has(vcpu, KVM_FEATURE_STEAL_TIME)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->arch.st.msr_val; - break; - case MSR_KVM_PV_EOI_EN: - if (!guest_pv_has(vcpu, KVM_FEATURE_PV_EOI)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->arch.pv_eoi.msr_val; - break; - case MSR_KVM_POLL_CONTROL: - if (!guest_pv_has(vcpu, KVM_FEATURE_POLL_CONTROL)) - return KVM_MSR_RET_UNSUPPORTED; - - msr_info->data = vcpu->arch.msr_kvm_poll_control; - break; - case MSR_IA32_P5_MC_ADDR: - case MSR_IA32_P5_MC_TYPE: - case MSR_IA32_MCG_CAP: - case MSR_IA32_MCG_CTL: - case MSR_IA32_MCG_STATUS: - case MSR_IA32_MC0_CTL ... MSR_IA32_MCx_CTL(KVM_MAX_MCE_BANKS) - 1: - case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1: - return get_msr_mce(vcpu, msr_info->index, &msr_info->data, - msr_info->host_initiated); - case MSR_IA32_XSS: - if (!msr_info->host_initiated && - !guest_cpuid_has(vcpu, X86_FEATURE_XSAVES)) - return 1; - msr_info->data = vcpu->arch.ia32_xss; - break; - case MSR_K7_CLK_CTL: - /* - * Provide expected ramp-up count for K7. All other - * are set to zero, indicating minimum divisors for - * every field. - * - * This prevents guest kernels on AMD host with CPU - * type 6, model 8 and higher from exploding due to - * the rdmsr failing. - */ - msr_info->data = 0x20000000; - break; -#ifdef CONFIG_KVM_HYPERV - case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: - case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: - case HV_X64_MSR_SYNDBG_OPTIONS: - case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: - case HV_X64_MSR_CRASH_CTL: - case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: - case HV_X64_MSR_REENLIGHTENMENT_CONTROL: - case HV_X64_MSR_TSC_EMULATION_CONTROL: - case HV_X64_MSR_TSC_EMULATION_STATUS: - case HV_X64_MSR_TSC_INVARIANT_CONTROL: - return kvm_hv_get_msr_common(vcpu, - msr_info->index, &msr_info->data, - msr_info->host_initiated); -#endif - case MSR_IA32_BBL_CR_CTL3: - /* This legacy MSR exists but isn't fully documented in current - * silicon. It is however accessed by winxp in very narrow - * scenarios where it sets bit #19, itself documented as - * a "reserved" bit. Best effort attempt to source coherent - * read data here should the balance of the register be - * interpreted by the guest: - * - * L2 cache control register 3: 64GB range, 256KB size, - * enabled, latency 0x1, configured - */ - msr_info->data = 0xbe702111; - break; - case MSR_AMD64_OSVW_ID_LENGTH: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) - return 1; - msr_info->data = vcpu->arch.osvw.length; - break; - case MSR_AMD64_OSVW_STATUS: - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_OSVW)) - return 1; - msr_info->data = vcpu->arch.osvw.status; - break; - case MSR_PLATFORM_INFO: - if (!msr_info->host_initiated && - !vcpu->kvm->arch.guest_can_read_msr_platform_info) - return 1; - msr_info->data = vcpu->arch.msr_platform_info; - break; - case MSR_MISC_FEATURES_ENABLES: - msr_info->data = vcpu->arch.msr_misc_features_enables; - break; - case MSR_K7_HWCR: - msr_info->data = vcpu->arch.msr_hwcr; - break; -#ifdef CONFIG_X86_64 - case MSR_IA32_XFD: - if (!msr_info->host_initiated && - !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) - return 1; - - msr_info->data = vcpu->arch.guest_fpu.fpstate->xfd; - break; - case MSR_IA32_XFD_ERR: - if (!msr_info->host_initiated && - !guest_cpu_cap_has(vcpu, X86_FEATURE_XFD)) - return 1; - - msr_info->data = vcpu->arch.guest_fpu.xfd_err; - break; -#endif - case MSR_IA32_U_CET: - case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: - kvm_get_xstate_msr(vcpu, msr_info); - break; - default: - if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) - return kvm_pmu_get_msr(vcpu, msr_info); - - return KVM_MSR_RET_UNSUPPORTED; - } - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_msr_common); - -/* - * Read or write a bunch of msrs. All parameters are kernel addresses. - * - * @return number of msrs set successfully. - */ -static int __msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs *msrs, - struct kvm_msr_entry *entries, - int (*do_msr)(struct kvm_vcpu *vcpu, - unsigned index, u64 *data)) -{ - bool fpu_loaded = false; - int i; - - for (i = 0; i < msrs->nmsrs; ++i) { - /* - * If userspace is accessing one or more XSTATE-managed MSRs, - * temporarily load the guest's FPU state so that the guest's - * MSR value(s) is resident in hardware and thus can be accessed - * via RDMSR/WRMSR. - */ - if (!fpu_loaded && is_xstate_managed_msr(vcpu, entries[i].index)) { - kvm_load_guest_fpu(vcpu); - fpu_loaded = true; - } - if (do_msr(vcpu, entries[i].index, &entries[i].data)) - break; - } - if (fpu_loaded) - kvm_put_guest_fpu(vcpu); - - return i; -} - -/* - * Read or write a bunch of msrs. Parameters are user addresses. - * - * @return number of msrs set successfully. - */ -static int msr_io(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs, - int (*do_msr)(struct kvm_vcpu *vcpu, - unsigned index, u64 *data), - int writeback) -{ - struct kvm_msrs msrs; - struct kvm_msr_entry *entries; - unsigned size; - int r; - - r = -EFAULT; - if (copy_from_user(&msrs, user_msrs, sizeof(msrs))) - goto out; - - r = -E2BIG; - if (msrs.nmsrs >= MAX_IO_MSRS) - goto out; - - size = sizeof(struct kvm_msr_entry) * msrs.nmsrs; - entries = memdup_user(user_msrs->entries, size); - if (IS_ERR(entries)) { - r = PTR_ERR(entries); - goto out; - } - - r = __msr_io(vcpu, &msrs, entries, do_msr); - - if (writeback && copy_to_user(user_msrs->entries, entries, size)) - r = -EFAULT; - - kfree(entries); -out: - return r; -} - static inline bool kvm_can_mwait_in_guest(void) { return boot_cpu_has(X86_FEATURE_MWAIT) && @@ -4587,61 +2403,6 @@ static int kvm_x86_dev_has_attr(struct kvm_device_attr *attr) return __kvm_x86_dev_get_attr(attr, &val); } -static int kvm_get_msr_index_list(struct kvm_msr_list __user *user_msr_list) -{ - struct kvm_msr_list msr_list; - unsigned int n; - - if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list))) - return -EFAULT; - - n = msr_list.nmsrs; - msr_list.nmsrs = num_msrs_to_save + num_emulated_msrs; - if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list))) - return -EFAULT; - - if (n < msr_list.nmsrs) - return -E2BIG; - - if (copy_to_user(user_msr_list->indices, &msrs_to_save, - num_msrs_to_save * sizeof(u32))) - return -EFAULT; - - if (copy_to_user(user_msr_list->indices + num_msrs_to_save, - &emulated_msrs, num_emulated_msrs * sizeof(u32))) - return -EFAULT; - - return 0; -} - -static int kvm_get_feature_msr_index_list(struct kvm_msr_list __user *user_msr_list) -{ - struct kvm_msr_list msr_list; - unsigned int n; - - if (copy_from_user(&msr_list, user_msr_list, sizeof(msr_list))) - return -EFAULT; - - n = msr_list.nmsrs; - msr_list.nmsrs = num_msr_based_features; - if (copy_to_user(user_msr_list, &msr_list, sizeof(msr_list))) - return -EFAULT; - - if (n < msr_list.nmsrs) - return -E2BIG; - - if (copy_to_user(user_msr_list->indices, &msr_based_features, - num_msr_based_features * sizeof(u32))) - return -EFAULT; - - return 0; -} - -static int kvm_get_feature_msrs(struct kvm_msrs __user *user_msrs) -{ - return msr_io(NULL, user_msrs, do_get_feature_msr, 1); -} - long kvm_arch_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -5588,148 +3349,6 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, } } -struct kvm_x86_reg_id { - __u32 index; - __u8 type; - __u8 rsvd1; - __u8 rsvd2:4; - __u8 size:4; - __u8 x86; -}; - -static int kvm_translate_kvm_reg(struct kvm_vcpu *vcpu, - struct kvm_x86_reg_id *reg) -{ - switch (reg->index) { - case KVM_REG_GUEST_SSP: - /* - * FIXME: If host-initiated accesses are ever exempted from - * ignore_msrs (in kvm_do_msr_access()), drop this manual check - * and rely on KVM's standard checks to reject accesses to regs - * that don't exist. - */ - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) - return -EINVAL; - - reg->type = KVM_X86_REG_TYPE_MSR; - reg->index = MSR_KVM_INTERNAL_GUEST_SSP; - break; - default: - return -EINVAL; - } - return 0; -} - -static int kvm_get_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val) -{ - u64 val; - - if (do_get_msr(vcpu, msr, &val)) - return -EINVAL; - - if (put_user(val, user_val)) - return -EFAULT; - - return 0; -} - -static int kvm_set_one_msr(struct kvm_vcpu *vcpu, u32 msr, u64 __user *user_val) -{ - u64 val; - - if (get_user(val, user_val)) - return -EFAULT; - - if (do_set_msr(vcpu, msr, &val)) - return -EINVAL; - - return 0; -} - -static int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl, - void __user *argp) -{ - struct kvm_one_reg one_reg; - struct kvm_x86_reg_id *reg; - u64 __user *user_val; - bool load_fpu; - int r; - - if (copy_from_user(&one_reg, argp, sizeof(one_reg))) - return -EFAULT; - - if ((one_reg.id & KVM_REG_ARCH_MASK) != KVM_REG_X86) - return -EINVAL; - - reg = (struct kvm_x86_reg_id *)&one_reg.id; - if (reg->rsvd1 || reg->rsvd2) - return -EINVAL; - - if (reg->type == KVM_X86_REG_TYPE_KVM) { - r = kvm_translate_kvm_reg(vcpu, reg); - if (r) - return r; - } - - if (reg->type != KVM_X86_REG_TYPE_MSR) - return -EINVAL; - - if ((one_reg.id & KVM_REG_SIZE_MASK) != KVM_REG_SIZE_U64) - return -EINVAL; - - guard(srcu)(&vcpu->kvm->srcu); - - load_fpu = is_xstate_managed_msr(vcpu, reg->index); - if (load_fpu) - kvm_load_guest_fpu(vcpu); - - user_val = u64_to_user_ptr(one_reg.addr); - if (ioctl == KVM_GET_ONE_REG) - r = kvm_get_one_msr(vcpu, reg->index, user_val); - else - r = kvm_set_one_msr(vcpu, reg->index, user_val); - - if (load_fpu) - kvm_put_guest_fpu(vcpu); - return r; -} - -static int kvm_get_reg_list(struct kvm_vcpu *vcpu, - struct kvm_reg_list __user *user_list) -{ - u64 nr_regs = guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) ? 1 : 0; - u64 user_nr_regs; - - if (get_user(user_nr_regs, &user_list->n)) - return -EFAULT; - - if (put_user(nr_regs, &user_list->n)) - return -EFAULT; - - if (user_nr_regs < nr_regs) - return -E2BIG; - - if (nr_regs && - put_user(KVM_X86_REG_KVM(KVM_REG_GUEST_SSP), &user_list->reg[0])) - return -EFAULT; - - return 0; -} - -static int kvm_get_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs) -{ - guard(srcu)(&vcpu->kvm->srcu); - - return msr_io(vcpu, user_msrs, do_get_msr, 1); -} - -static int kvm_set_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs) -{ - guard(srcu)(&vcpu->kvm->srcu); - - return msr_io(vcpu, user_msrs, do_set_msr, 0); -} - long kvm_arch_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -6532,113 +4151,6 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, return r; } -static struct kvm_x86_msr_filter *kvm_alloc_msr_filter(bool default_allow) -{ - struct kvm_x86_msr_filter *msr_filter; - - msr_filter = kzalloc_obj(*msr_filter, GFP_KERNEL_ACCOUNT); - if (!msr_filter) - return NULL; - - msr_filter->default_allow = default_allow; - return msr_filter; -} - -static void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter) -{ - u32 i; - - if (!msr_filter) - return; - - for (i = 0; i < msr_filter->count; i++) - kfree(msr_filter->ranges[i].bitmap); - - kfree(msr_filter); -} - -static int kvm_add_msr_filter(struct kvm_x86_msr_filter *msr_filter, - struct kvm_msr_filter_range *user_range) -{ - unsigned long *bitmap; - size_t bitmap_size; - - if (!user_range->nmsrs) - return 0; - - if (user_range->flags & ~KVM_MSR_FILTER_RANGE_VALID_MASK) - return -EINVAL; - - if (!user_range->flags) - return -EINVAL; - - bitmap_size = BITS_TO_LONGS(user_range->nmsrs) * sizeof(long); - if (!bitmap_size || bitmap_size > KVM_MSR_FILTER_MAX_BITMAP_SIZE) - return -EINVAL; - - bitmap = memdup_user((__user u8*)user_range->bitmap, bitmap_size); - if (IS_ERR(bitmap)) - return PTR_ERR(bitmap); - - msr_filter->ranges[msr_filter->count] = (struct msr_bitmap_range) { - .flags = user_range->flags, - .base = user_range->base, - .nmsrs = user_range->nmsrs, - .bitmap = bitmap, - }; - - msr_filter->count++; - return 0; -} - -static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, - struct kvm_msr_filter *filter) -{ - struct kvm_x86_msr_filter *new_filter, *old_filter; - bool default_allow; - bool empty = true; - int r; - u32 i; - - if (filter->flags & ~KVM_MSR_FILTER_VALID_MASK) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) - empty &= !filter->ranges[i].nmsrs; - - default_allow = !(filter->flags & KVM_MSR_FILTER_DEFAULT_DENY); - if (empty && !default_allow) - return -EINVAL; - - new_filter = kvm_alloc_msr_filter(default_allow); - if (!new_filter) - return -ENOMEM; - - for (i = 0; i < ARRAY_SIZE(filter->ranges); i++) { - r = kvm_add_msr_filter(new_filter, &filter->ranges[i]); - if (r) { - kvm_free_msr_filter(new_filter); - return r; - } - } - - mutex_lock(&kvm->lock); - old_filter = rcu_replace_pointer(kvm->arch.msr_filter, new_filter, - mutex_is_locked(&kvm->lock)); - mutex_unlock(&kvm->lock); - synchronize_srcu(&kvm->srcu); - - kvm_free_msr_filter(old_filter); - - /* - * Recalc MSR intercepts as userspace may want to intercept accesses to - * MSRs that KVM would otherwise pass through to the guest. - */ - kvm_make_all_cpus_request(kvm, KVM_REQ_RECALC_INTERCEPTS); - - return 0; -} - #ifdef CONFIG_KVM_COMPAT /* for KVM_X86_SET_MSR_FILTER */ struct kvm_msr_filter_range_compat { @@ -7159,157 +4671,6 @@ int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) return r; } -static void kvm_probe_feature_msr(u32 msr_index) -{ - u64 data; - - if (kvm_get_feature_msr(NULL, msr_index, &data, true)) - return; - - msr_based_features[num_msr_based_features++] = msr_index; -} - -static void kvm_probe_msr_to_save(u32 msr_index) -{ - u32 dummy[2]; - - if (rdmsr_safe(msr_index, &dummy[0], &dummy[1])) - return; - - /* - * Even MSRs that are valid in the host may not be exposed to guests in - * some cases. - */ - switch (msr_index) { - case MSR_IA32_BNDCFGS: - if (!kvm_mpx_supported()) - return; - break; - case MSR_TSC_AUX: - if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP) && - !kvm_cpu_cap_has(X86_FEATURE_RDPID)) - return; - break; - case MSR_IA32_UMWAIT_CONTROL: - if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG)) - return; - break; - case MSR_IA32_RTIT_CTL: - case MSR_IA32_RTIT_STATUS: - if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) - return; - break; - case MSR_IA32_RTIT_CR3_MATCH: - if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || - !intel_pt_validate_hw_cap(PT_CAP_cr3_filtering)) - return; - break; - case MSR_IA32_RTIT_OUTPUT_BASE: - case MSR_IA32_RTIT_OUTPUT_MASK: - if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || - (!intel_pt_validate_hw_cap(PT_CAP_topa_output) && - !intel_pt_validate_hw_cap(PT_CAP_single_range_output))) - return; - break; - case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: - if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || - (msr_index - MSR_IA32_RTIT_ADDR0_A >= - intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2)) - return; - break; - case MSR_ARCH_PERFMON_PERFCTR0 ... - MSR_ARCH_PERFMON_PERFCTR0 + KVM_MAX_NR_GP_COUNTERS - 1: - if (msr_index - MSR_ARCH_PERFMON_PERFCTR0 >= - kvm_pmu_cap.num_counters_gp) - return; - break; - case MSR_ARCH_PERFMON_EVENTSEL0 ... - MSR_ARCH_PERFMON_EVENTSEL0 + KVM_MAX_NR_GP_COUNTERS - 1: - if (msr_index - MSR_ARCH_PERFMON_EVENTSEL0 >= - kvm_pmu_cap.num_counters_gp) - return; - break; - case MSR_ARCH_PERFMON_FIXED_CTR0 ... - MSR_ARCH_PERFMON_FIXED_CTR0 + KVM_MAX_NR_FIXED_COUNTERS - 1: - if (msr_index - MSR_ARCH_PERFMON_FIXED_CTR0 >= - kvm_pmu_cap.num_counters_fixed) - return; - break; - case MSR_AMD64_PERF_CNTR_GLOBAL_CTL: - case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS: - case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR: - case MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_SET: - if (!kvm_cpu_cap_has(X86_FEATURE_PERFMON_V2)) - return; - break; - case MSR_IA32_XFD: - case MSR_IA32_XFD_ERR: - if (!kvm_cpu_cap_has(X86_FEATURE_XFD)) - return; - break; - case MSR_IA32_TSX_CTRL: - if (!(kvm_get_arch_capabilities() & ARCH_CAP_TSX_CTRL_MSR)) - return; - break; - case MSR_IA32_XSS: - if (!kvm_caps.supported_xss) - return; - break; - case MSR_IA32_U_CET: - case MSR_IA32_S_CET: - if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK) && - !kvm_cpu_cap_has(X86_FEATURE_IBT)) - return; - break; - case MSR_IA32_INT_SSP_TAB: - if (!kvm_cpu_cap_has(X86_FEATURE_LM)) - return; - fallthrough; - case MSR_IA32_PL0_SSP ... MSR_IA32_PL3_SSP: - if (!kvm_cpu_cap_has(X86_FEATURE_SHSTK)) - return; - break; - default: - break; - } - - msrs_to_save[num_msrs_to_save++] = msr_index; -} - -static void kvm_init_msr_lists(void) -{ - unsigned i; - - BUILD_BUG_ON_MSG(KVM_MAX_NR_FIXED_COUNTERS != 3, - "Please update the fixed PMCs in msrs_to_save_pmu[]"); - - num_msrs_to_save = 0; - num_emulated_msrs = 0; - num_msr_based_features = 0; - - for (i = 0; i < ARRAY_SIZE(msrs_to_save_base); i++) - kvm_probe_msr_to_save(msrs_to_save_base[i]); - - if (enable_pmu) { - for (i = 0; i < ARRAY_SIZE(msrs_to_save_pmu); i++) - kvm_probe_msr_to_save(msrs_to_save_pmu[i]); - } - - for (i = 0; i < ARRAY_SIZE(emulated_msrs_all); i++) { - if (!kvm_x86_call(has_emulated_msr)(NULL, - emulated_msrs_all[i])) - continue; - - emulated_msrs[num_emulated_msrs++] = emulated_msrs_all[i]; - } - - for (i = KVM_FIRST_EMULATED_VMX_MSR; i <= KVM_LAST_EMULATED_VMX_MSR; i++) - kvm_probe_feature_msr(i); - - for (i = 0; i < ARRAY_SIZE(msr_based_features_all_except_vmx); i++) - kvm_probe_feature_msr(msr_based_features_all_except_vmx[i]); -} - static int vcpu_mmio_write(struct kvm_vcpu *vcpu, gpa_t addr, int len, void *__v) { @@ -8247,61 +5608,22 @@ static int emulator_get_msr_with_filter(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 *pdata) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); - int r; - r = kvm_emulate_msr_read(vcpu, msr_index, pdata); - if (r < 0) - return X86EMUL_UNHANDLEABLE; - - if (r) { - if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_RDMSR, 0, - complete_emulated_rdmsr, r)) - return X86EMUL_IO_NEEDED; - - trace_kvm_msr_read_ex(msr_index); - return X86EMUL_PROPAGATE_FAULT; - } - - trace_kvm_msr_read(msr_index, *pdata); - return X86EMUL_CONTINUE; + return kvm_emulator_get_msr_with_filter(vcpu, msr_index, pdata); } static int emulator_set_msr_with_filter(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 data) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); - int r; - r = kvm_emulate_msr_write(vcpu, msr_index, data); - if (r < 0) - return X86EMUL_UNHANDLEABLE; - - if (r) { - if (kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data, - complete_emulated_msr_access, r)) - return X86EMUL_IO_NEEDED; - - trace_kvm_msr_write_ex(msr_index, data); - return X86EMUL_PROPAGATE_FAULT; - } - - trace_kvm_msr_write(msr_index, data); - return X86EMUL_CONTINUE; + return kvm_emulator_set_msr_with_filter(vcpu, msr_index, data); } static int emulator_get_msr(struct x86_emulate_ctxt *ctxt, u32 msr_index, u64 *pdata) { - /* - * Treat emulator accesses to the current shadow stack pointer as host- - * initiated, as they aren't true MSR accesses (SSP is a "just a reg"), - * and this API is used only for implicit accesses, i.e. not RDMSR, and - * so the index is fully KVM-controlled. - */ - if (unlikely(msr_index == MSR_KVM_INTERNAL_GUEST_SSP)) - return kvm_msr_read(emul_to_vcpu(ctxt), msr_index, pdata); - - return __kvm_emulate_msr_read(emul_to_vcpu(ctxt), msr_index, pdata); + return kvm_emulator_get_msr(emul_to_vcpu(ctxt), msr_index, pdata); } static int emulator_check_rdpmc_early(struct x86_emulate_ctxt *ctxt, u32 pmc) @@ -13248,32 +10570,6 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) #endif #endif -int kvm_spec_ctrl_test_value(u64 value) -{ - /* - * test that setting IA32_SPEC_CTRL to given value - * is allowed by the host processor - */ - - u64 saved_value; - unsigned long flags; - int ret = 0; - - local_irq_save(flags); - - if (rdmsrq_safe(MSR_IA32_SPEC_CTRL, &saved_value)) - ret = 1; - else if (wrmsrq_safe(MSR_IA32_SPEC_CTRL, value)) - ret = 1; - else - wrmsrq(MSR_IA32_SPEC_CTRL, saved_value); - - local_irq_restore(flags); - - return ret; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_spec_ctrl_test_value); - void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code) { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 31e67b060148..fd3d0a196526 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -6,6 +6,7 @@ #include #include #include +#include "msrs.h" #include "mmu.h" #include "regs.h" #include "kvm_emulate.h" @@ -45,14 +46,6 @@ do { \ failed; \ }) -/* - * The first...last VMX feature MSRs that are emulated by KVM. This may or may - * not cover all known VMX MSRs, as KVM doesn't emulate an MSR until there's an - * associated feature that KVM supports for nested virtualization. - */ -#define KVM_FIRST_EMULATED_VMX_MSR MSR_IA32_VMX_BASIC -#define KVM_LAST_EMULATED_VMX_MSR MSR_IA32_VMX_VMFUNC - #define KVM_DEFAULT_PLE_GAP 128 #define KVM_VMX_DEFAULT_PLE_WINDOW 4096 #define KVM_DEFAULT_PLE_WINDOW_GROW 2 @@ -61,16 +54,6 @@ do { \ #define KVM_SVM_DEFAULT_PLE_WINDOW_MAX USHRT_MAX #define KVM_SVM_DEFAULT_PLE_WINDOW 3000 -/* - * KVM's internal, non-ABI indices for synthetic MSRs. The values themselves - * are arbitrary and have no meaning, the only requirement is that they don't - * conflict with "real" MSRs that KVM supports. Use values at the upper end - * of KVM's reserved paravirtual MSR range to minimize churn, i.e. these values - * will be usable until KVM exhausts its supply of paravirtual MSR indices. - */ - -#define MSR_KVM_INTERNAL_GUEST_SSP 0x4b564dff - static inline unsigned int __grow_ple_window(unsigned int val, unsigned int base, unsigned int modifier, unsigned int max) { @@ -101,9 +84,6 @@ static inline unsigned int __shrink_ple_window(unsigned int val, return max(val, min); } -#define MSR_IA32_CR_PAT_DEFAULT \ - PAT_VALUE(WB, WT, UC_MINUS, UC, WB, WT, UC_MINUS, UC) - void kvm_service_local_tlb_flush_requests(struct kvm_vcpu *vcpu); int kvm_check_nested_events(struct kvm_vcpu *vcpu); @@ -378,15 +358,12 @@ void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, struct kvm_queued_exception *ex); void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu); -int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); -int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_code); int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type, void *insn, int insn_len); int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, int emulation_type, void *insn, int insn_len); -fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu); -fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg); + fastpath_t handle_fastpath_hlt(struct kvm_vcpu *vcpu); fastpath_t handle_fastpath_invd(struct kvm_vcpu *vcpu); @@ -432,20 +409,6 @@ extern bool enable_vmware_backdoor; extern int pi_inject_timer; -extern bool report_ignored_msrs; - -static inline void kvm_pr_unimpl_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data) -{ - if (report_ignored_msrs) - vcpu_unimpl(vcpu, "Unhandled WRMSR(0x%x) = 0x%llx\n", msr, data); -} - -static inline void kvm_pr_unimpl_rdmsr(struct kvm_vcpu *vcpu, u32 msr) -{ - if (report_ignored_msrs) - vcpu_unimpl(vcpu, "Unhandled RDMSR(0x%x)\n", msr); -} - static inline u64 nsec_to_cycles(struct kvm_vcpu *vcpu, u64 nsec) { return pvclock_scale_delta(nsec, vcpu->arch.virtual_tsc_mult, @@ -563,33 +526,10 @@ static inline void kvm_machine_check(void) #endif } -int kvm_spec_ctrl_test_value(u64 value); int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, struct x86_exception *e); void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid); int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva); -bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type); - -enum kvm_msr_access { - MSR_TYPE_R = BIT(0), - MSR_TYPE_W = BIT(1), - MSR_TYPE_RW = MSR_TYPE_R | MSR_TYPE_W, -}; - -/* - * Internal error codes that are used to indicate that MSR emulation encountered - * an error that should result in #GP in the guest, unless userspace handles it. - * Note, '1', '0', and negative numbers are off limits, as they are used by KVM - * as part of KVM's lightly documented internal KVM_RUN return codes. - * - * UNSUPPORTED - The MSR isn't supported, either because it is completely - * unknown to KVM, or because the MSR should not exist according - * to the vCPU model. - * - * FILTERED - Access to the MSR is denied by a userspace MSR filter. - */ -#define KVM_MSR_RET_UNSUPPORTED 2 -#define KVM_MSR_RET_FILTERED 3 int kvm_sev_es_mmio(struct kvm_vcpu *vcpu, bool is_write, gpa_t gpa, unsigned int bytes, void *data); @@ -649,27 +589,4 @@ int ____kvm_emulate_hypercall(struct kvm_vcpu *vcpu, int cpl, int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); -#define CET_US_RESERVED_BITS GENMASK(9, 6) -#define CET_US_SHSTK_MASK_BITS GENMASK(1, 0) -#define CET_US_IBT_MASK_BITS (GENMASK_ULL(5, 2) | GENMASK_ULL(63, 10)) -#define CET_US_LEGACY_BITMAP_BASE(data) ((data) >> 12) - -static inline bool kvm_is_valid_u_s_cet(struct kvm_vcpu *vcpu, u64 data) -{ - if (data & CET_US_RESERVED_BITS) - return false; - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) && - (data & CET_US_SHSTK_MASK_BITS)) - return false; - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) && - (data & CET_US_IBT_MASK_BITS)) - return false; - if (!IS_ALIGNED(CET_US_LEGACY_BITMAP_BASE(data), 4)) - return false; - /* IBT can be suppressed iff the TRACKER isn't WAIT_ENDBR. */ - if ((data & CET_SUPPRESS) && (data & CET_WAIT_ENDBR)) - return false; - - return true; -} #endif