From f1a9761fbb00639c5e73835f7f373ef834bb867f Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 16 Mar 2022 00:55:37 +0000 Subject: [PATCH 001/319] KVM: x86: Allow userspace to opt out of hypercall patching KVM handles the VMCALL/VMMCALL instructions very strangely. Even though both of these instructions really should #UD when executed on the wrong vendor's hardware (i.e. VMCALL on SVM, VMMCALL on VMX), KVM replaces the guest's instruction with the appropriate instruction for the vendor. Nonetheless, older guest kernels without commit c1118b3602c2 ("x86: kvm: use alternatives for VMCALL vs. VMMCALL if kernel text is read-only") do not patch in the appropriate instruction using alternatives, likely motivating KVM's intervention. Add a quirk allowing userspace to opt out of hypercall patching. If the quirk is disabled, KVM synthesizes a #UD in the guest. Signed-off-by: Oliver Upton Message-Id: <20220316005538.2282772-2-oupton@google.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 9 +++++++++ arch/x86/include/asm/kvm_host.h | 3 ++- arch/x86/include/uapi/asm/kvm.h | 11 ++++++----- arch/x86/kvm/x86.c | 11 +++++++++++ 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index d13fa6600467..700df350332b 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -7134,6 +7134,15 @@ The valid bits in cap.args[0] are: Additionally, when this quirk is disabled, KVM clears CPUID.01H:ECX[bit 3] if IA32_MISC_ENABLE[bit 18] is cleared. + + KVM_X86_QUIRK_FIX_HYPERCALL_INSN By default, KVM rewrites guest + VMMCALL/VMCALL instructions to match the + vendor's hypercall instruction for the + system. When this quirk is disabled, KVM + will no longer rewrite invalid guest + hypercall instructions. Executing the + incorrect hypercall instruction will + generate a #UD within the guest. =================================== ============================================ 8. Other capabilities. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 93a2671a57cf..d9a825182b84 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1987,6 +1987,7 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages); KVM_X86_QUIRK_CD_NW_CLEARED | \ KVM_X86_QUIRK_LAPIC_MMIO_HOLE | \ KVM_X86_QUIRK_OUT_7E_INC_RIP | \ - KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT) + KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT | \ + KVM_X86_QUIRK_FIX_HYPERCALL_INSN) #endif /* _ASM_X86_KVM_HOST_H */ diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index bf6e96011dfe..21614807a2cb 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -428,11 +428,12 @@ struct kvm_sync_regs { struct kvm_vcpu_events events; }; -#define KVM_X86_QUIRK_LINT0_REENABLED (1 << 0) -#define KVM_X86_QUIRK_CD_NW_CLEARED (1 << 1) -#define KVM_X86_QUIRK_LAPIC_MMIO_HOLE (1 << 2) -#define KVM_X86_QUIRK_OUT_7E_INC_RIP (1 << 3) -#define KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT (1 << 4) +#define KVM_X86_QUIRK_LINT0_REENABLED (1 << 0) +#define KVM_X86_QUIRK_CD_NW_CLEARED (1 << 1) +#define KVM_X86_QUIRK_LAPIC_MMIO_HOLE (1 << 2) +#define KVM_X86_QUIRK_OUT_7E_INC_RIP (1 << 3) +#define KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT (1 << 4) +#define KVM_X86_QUIRK_FIX_HYPERCALL_INSN (1 << 5) #define KVM_STATE_NESTED_FORMAT_VMX 0 #define KVM_STATE_NESTED_FORMAT_SVM 1 diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 498cd4aaa2cb..c9964b78fa2e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9267,6 +9267,17 @@ static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt) char instruction[3]; unsigned long rip = kvm_rip_read(vcpu); + /* + * If the quirk is disabled, synthesize a #UD and let the guest pick up + * the pieces. + */ + if (!kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) { + ctxt->exception.error_code_valid = false; + ctxt->exception.vector = UD_VECTOR; + ctxt->have_exception = true; + return X86EMUL_PROPAGATE_FAULT; + } + static_call(kvm_x86_patch_hypercall)(vcpu, instruction); return emulator_write_emulated(ctxt, rip, instruction, 3, From 6c2fa8b20d0cad3719b024ba0f25a50199a90c9a Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 16 Mar 2022 00:55:38 +0000 Subject: [PATCH 002/319] selftests: KVM: Test KVM_X86_QUIRK_FIX_HYPERCALL_INSN Add a test that asserts KVM rewrites guest hypercall instructions to match the running architecture (VMCALL on VMX, VMMCALL on SVM). Additionally, test that with the quirk disabled, KVM no longer rewrites guest instructions and instead injects a #UD. Signed-off-by: Oliver Upton Message-Id: <20220316005538.2282772-3-oupton@google.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/.gitignore | 1 + tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/x86_64/fix_hypercall_test.c | 170 ++++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 9b67343dc4ab..1f1b6c978bf7 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -15,6 +15,7 @@ /x86_64/debug_regs /x86_64/evmcs_test /x86_64/emulator_error_test +/x86_64/fix_hypercall_test /x86_64/get_msr_index_features /x86_64/kvm_clock_test /x86_64/kvm_pv_test diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 04099f453b59..03662666b30d 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -48,6 +48,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/cr4_cpuid_sync_test TEST_GEN_PROGS_x86_64 += x86_64/get_msr_index_features TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test TEST_GEN_PROGS_x86_64 += x86_64/emulator_error_test +TEST_GEN_PROGS_x86_64 += x86_64/fix_hypercall_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_clock TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid TEST_GEN_PROGS_x86_64 += x86_64/hyperv_features diff --git a/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c b/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c new file mode 100644 index 000000000000..1f5c32146f3d --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020, Google LLC. + * + * Tests for KVM paravirtual feature disablement + */ +#include +#include +#include +#include + +#include "apic.h" +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" + +#define VCPU_ID 0 + +static bool ud_expected; + +static void guest_ud_handler(struct ex_regs *regs) +{ + GUEST_ASSERT(ud_expected); + GUEST_DONE(); +} + +extern unsigned char svm_hypercall_insn; +static uint64_t svm_do_sched_yield(uint8_t apic_id) +{ + uint64_t ret; + + asm volatile("mov %1, %%rax\n\t" + "mov %2, %%rbx\n\t" + "svm_hypercall_insn:\n\t" + "vmmcall\n\t" + "mov %%rax, %0\n\t" + : "=r"(ret) + : "r"((uint64_t)KVM_HC_SCHED_YIELD), "r"((uint64_t)apic_id) + : "rax", "rbx", "memory"); + + return ret; +} + +extern unsigned char vmx_hypercall_insn; +static uint64_t vmx_do_sched_yield(uint8_t apic_id) +{ + uint64_t ret; + + asm volatile("mov %1, %%rax\n\t" + "mov %2, %%rbx\n\t" + "vmx_hypercall_insn:\n\t" + "vmcall\n\t" + "mov %%rax, %0\n\t" + : "=r"(ret) + : "r"((uint64_t)KVM_HC_SCHED_YIELD), "r"((uint64_t)apic_id) + : "rax", "rbx", "memory"); + + return ret; +} + +static void assert_hypercall_insn(unsigned char *exp_insn, unsigned char *obs_insn) +{ + uint32_t exp = 0, obs = 0; + + memcpy(&exp, exp_insn, sizeof(exp)); + memcpy(&obs, obs_insn, sizeof(obs)); + + GUEST_ASSERT_EQ(exp, obs); +} + +static void guest_main(void) +{ + unsigned char *native_hypercall_insn, *hypercall_insn; + uint8_t apic_id; + + apic_id = GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID)); + + if (is_intel_cpu()) { + native_hypercall_insn = &vmx_hypercall_insn; + hypercall_insn = &svm_hypercall_insn; + svm_do_sched_yield(apic_id); + } else if (is_amd_cpu()) { + native_hypercall_insn = &svm_hypercall_insn; + hypercall_insn = &vmx_hypercall_insn; + vmx_do_sched_yield(apic_id); + } else { + GUEST_ASSERT(0); + /* unreachable */ + return; + } + + GUEST_ASSERT(!ud_expected); + assert_hypercall_insn(native_hypercall_insn, hypercall_insn); + GUEST_DONE(); +} + +static void setup_ud_vector(struct kvm_vm *vm) +{ + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vm, VCPU_ID); + vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); +} + +static void enter_guest(struct kvm_vm *vm) +{ + struct kvm_run *run; + struct ucall uc; + + run = vcpu_state(vm, VCPU_ID); + + vcpu_run(vm, VCPU_ID); + switch (get_ucall(vm, VCPU_ID, &uc)) { + case UCALL_SYNC: + pr_info("%s: %016lx\n", (const char *)uc.args[2], uc.args[3]); + break; + case UCALL_DONE: + return; + case UCALL_ABORT: + TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], __FILE__, uc.args[1]); + default: + TEST_FAIL("Unhandled ucall: %ld\nexit_reason: %u (%s)", + uc.cmd, run->exit_reason, exit_reason_str(run->exit_reason)); + } +} + +static void test_fix_hypercall(void) +{ + struct kvm_vm *vm; + + vm = vm_create_default(VCPU_ID, 0, guest_main); + setup_ud_vector(vm); + + ud_expected = false; + sync_global_to_guest(vm, ud_expected); + + virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); + + enter_guest(vm); +} + +static void test_fix_hypercall_disabled(void) +{ + struct kvm_enable_cap cap = {0}; + struct kvm_vm *vm; + + vm = vm_create_default(VCPU_ID, 0, guest_main); + setup_ud_vector(vm); + + cap.cap = KVM_CAP_DISABLE_QUIRKS2; + cap.args[0] = KVM_X86_QUIRK_FIX_HYPERCALL_INSN; + vm_enable_cap(vm, &cap); + + ud_expected = true; + sync_global_to_guest(vm, ud_expected); + + virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); + + enter_guest(vm); +} + +int main(void) +{ + if (!(kvm_check_cap(KVM_CAP_DISABLE_QUIRKS2) & KVM_X86_QUIRK_FIX_HYPERCALL_INSN)) { + print_skip("KVM_X86_QUIRK_HYPERCALL_INSN not supported"); + exit(KSFT_SKIP); + } + + test_fix_hypercall(); + test_fix_hypercall_disabled(); +} From b9f3973ab3a80141a6decc34f4740e6494ca7d51 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 1 Mar 2022 16:36:46 +0200 Subject: [PATCH 003/319] KVM: x86: nSVM: implement nested VMLOAD/VMSAVE This was tested by booting L1,L2,L3 (all Linux) and checking that no VMLOAD/VMSAVE vmexits happened. Signed-off-by: Maxim Levitsky Message-Id: <20220301143650.143749-4-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 35 ++++++++++++++++++++++++++++------- arch/x86/kvm/svm/svm.c | 7 +++++++ arch/x86/kvm/svm/svm.h | 8 +++++++- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 96bab464967f..d736ec6514ca 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -121,6 +121,20 @@ static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; } +static bool nested_vmcb_needs_vls_intercept(struct vcpu_svm *svm) +{ + if (!svm->v_vmload_vmsave_enabled) + return true; + + if (!nested_npt_enabled(svm)) + return true; + + if (!(svm->nested.ctl.virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK)) + return true; + + return false; +} + void recalc_intercepts(struct vcpu_svm *svm) { struct vmcb_control_area *c, *h; @@ -162,8 +176,17 @@ void recalc_intercepts(struct vcpu_svm *svm) if (!intercept_smi) vmcb_clr_intercept(c, INTERCEPT_SMI); - vmcb_set_intercept(c, INTERCEPT_VMLOAD); - vmcb_set_intercept(c, INTERCEPT_VMSAVE); + if (nested_vmcb_needs_vls_intercept(svm)) { + /* + * If the virtual VMLOAD/VMSAVE is not enabled for the L2, + * we must intercept these instructions to correctly + * emulate them in case L1 doesn't intercept them. + */ + vmcb_set_intercept(c, INTERCEPT_VMLOAD); + vmcb_set_intercept(c, INTERCEPT_VMSAVE); + } else { + WARN_ON(!(c->virt_ext & VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK)); + } } /* @@ -454,11 +477,6 @@ static void nested_save_pending_event_to_vmcb12(struct vcpu_svm *svm, vmcb12->control.exit_int_info = exit_int_info; } -static inline bool nested_npt_enabled(struct vcpu_svm *svm) -{ - return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE; -} - static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) { /* @@ -621,6 +639,9 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; + if (!nested_vmcb_needs_vls_intercept(svm)) + svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; + nested_svm_transition_tlb_flush(vcpu); /* Enter Guest-Mode */ diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index bd4c64b362d2..af1706144bac 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -960,6 +960,8 @@ static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu) set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); + + svm->v_vmload_vmsave_enabled = false; } else { /* * If hardware supports Virtual VMLOAD VMSAVE then enable it @@ -3947,6 +3949,8 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) svm->tsc_scaling_enabled = tsc_scaling && guest_cpuid_has(vcpu, X86_FEATURE_TSCRATEMSR); + svm->v_vmload_vmsave_enabled = vls && guest_cpuid_has(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD); + svm_recalc_instruction_intercepts(vcpu, svm); /* For sev guests, the memory encryption bit is not reserved in CR3. */ @@ -4695,6 +4699,9 @@ static __init void svm_set_cpu_caps(void) if (tsc_scaling) kvm_cpu_cap_set(X86_FEATURE_TSCRATEMSR); + if (vls) + kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD); + /* Nested VM can receive #VMEXIT instead of triggering #GP */ kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index f77a7d2d39dd..468f149556dd 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -231,9 +231,10 @@ struct vcpu_svm { unsigned int3_injected; unsigned long int3_rip; - /* cached guest cpuid flags for faster access */ + /* optional nested SVM features that are enabled for this guest */ bool nrips_enabled : 1; bool tsc_scaling_enabled : 1; + bool v_vmload_vmsave_enabled : 1; u32 ldr_reg; u32 dfr_reg; @@ -481,6 +482,11 @@ static inline bool gif_set(struct vcpu_svm *svm) return !!(svm->vcpu.arch.hflags & HF_GIF_MASK); } +static inline bool nested_npt_enabled(struct vcpu_svm *svm) +{ + return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE; +} + /* svm.c */ #define MSR_INVALID 0xffffffffU From edf721238576544ed8daf9943ab398bb27da8e6b Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 1 Mar 2022 16:36:49 +0200 Subject: [PATCH 004/319] KVM: x86: SVM: allow to force AVIC to be enabled Apparently on some systems AVIC is disabled in CPUID but still usable. Allow the user to override the CPUID if the user is willing to take the risk. Signed-off-by: Maxim Levitsky Message-Id: <20220301143650.143749-7-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index af1706144bac..f1cb8e991749 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -189,6 +189,9 @@ module_param(tsc_scaling, int, 0444); static bool avic; module_param(avic, bool, 0444); +static bool force_avic; +module_param_unsafe(force_avic, bool, 0444); + bool __read_mostly dump_invalid_vmcb; module_param(dump_invalid_vmcb, bool, 0644); @@ -4813,10 +4816,14 @@ static __init int svm_hardware_setup(void) nrips = false; } - enable_apicv = avic = avic && npt_enabled && boot_cpu_has(X86_FEATURE_AVIC); + enable_apicv = avic = avic && npt_enabled && (boot_cpu_has(X86_FEATURE_AVIC) || force_avic); if (enable_apicv) { - pr_info("AVIC enabled\n"); + if (!boot_cpu_has(X86_FEATURE_AVIC)) { + pr_warn("AVIC is not supported in CPUID but force enabled"); + pr_warn("Your system might crash and burn"); + } else + pr_info("AVIC enabled\n"); amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); } else { From 249f32493304e2ab04ed53cc634909629b8dc4c6 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 1 Mar 2022 15:55:23 +0200 Subject: [PATCH 005/319] KVM: x86: mark synthetic SMM vmexit as SVM_EXIT_SW Use a dummy unused vmexit reason to mark the 'VM exit' that is happening when kvm exits to handle SMM, which is not a real VM exit. This makes it a bit easier to read the KVM trace, and avoids other potential problems due to a stale vmexit reason in the vmcb. If SVM_EXIT_SW somehow reaches svm_invoke_exit_handler(), instead, svm_check_exit_valid() will return false and a WARN will be logged. Signed-off-by: Maxim Levitsky Message-Id: <20220301135526.136554-2-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index f1cb8e991749..462f691fb614 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4231,7 +4231,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate) svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; - ret = nested_svm_vmexit(svm); + ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW); if (ret) return ret; From a795cd43c5b5819b8ee94690f22caa5e2e4d8620 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:13 +0000 Subject: [PATCH 006/319] KVM: x86/xen: Use gfn_to_pfn_cache for runstate area Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-4-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 +- arch/x86/kvm/x86.c | 1 + arch/x86/kvm/xen.c | 107 ++++++++++++++++---------------- arch/x86/kvm/xen.h | 6 +- 4 files changed, 59 insertions(+), 58 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index d9a825182b84..3263147a166f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -608,10 +608,9 @@ struct kvm_vcpu_xen { u32 current_runstate; bool vcpu_info_set; bool vcpu_time_info_set; - bool runstate_set; struct gfn_to_hva_cache vcpu_info_cache; struct gfn_to_hva_cache vcpu_time_info_cache; - struct gfn_to_hva_cache runstate_cache; + struct gfn_to_pfn_cache runstate_cache; u64 last_steal; u64 runstate_entry_time; u64 runstate_times[4]; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c9964b78fa2e..7042495eb089 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11316,6 +11316,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); fpu_free_guest_fpstate(&vcpu->arch.guest_fpu); + kvm_xen_destroy_vcpu(vcpu); kvm_hv_vcpu_uninit(vcpu); kvm_pmu_destroy(vcpu); kfree(vcpu->arch.mce_banks); diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index bf6cc25eee76..de1b3be7239e 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -133,27 +133,36 @@ static void kvm_xen_update_runstate(struct kvm_vcpu *v, int state) void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) { struct kvm_vcpu_xen *vx = &v->arch.xen; - struct gfn_to_hva_cache *ghc = &vx->runstate_cache; - struct kvm_memslots *slots = kvm_memslots(v->kvm); - bool atomic = (state == RUNSTATE_runnable); - uint64_t state_entry_time; - int __user *user_state; - uint64_t __user *user_times; + struct gfn_to_pfn_cache *gpc = &vx->runstate_cache; + uint64_t *user_times; + unsigned long flags; + size_t user_len; + int *user_state; kvm_xen_update_runstate(v, state); - if (!vx->runstate_set) + if (!vx->runstate_cache.active) return; - if (unlikely(slots->generation != ghc->generation || kvm_is_error_hva(ghc->hva)) && - kvm_gfn_to_hva_cache_init(v->kvm, ghc, ghc->gpa, ghc->len)) - return; + if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) + user_len = sizeof(struct vcpu_runstate_info); + else + user_len = sizeof(struct compat_vcpu_runstate_info); - /* We made sure it fits in a single page */ - BUG_ON(!ghc->memslot); + read_lock_irqsave(&gpc->lock, flags); + while (!kvm_gfn_to_pfn_cache_check(v->kvm, gpc, gpc->gpa, + user_len)) { + read_unlock_irqrestore(&gpc->lock, flags); - if (atomic) - pagefault_disable(); + /* When invoked from kvm_sched_out() we cannot sleep */ + if (state == RUNSTATE_runnable) + return; + + if (kvm_gfn_to_pfn_cache_refresh(v->kvm, gpc, gpc->gpa, user_len)) + return; + + read_lock_irqsave(&gpc->lock, flags); + } /* * The only difference between 32-bit and 64-bit versions of the @@ -167,38 +176,33 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) */ BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) != 0); BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state) != 0); - user_state = (int __user *)ghc->hva; - BUILD_BUG_ON(sizeof(struct compat_vcpu_runstate_info) != 0x2c); - - user_times = (uint64_t __user *)(ghc->hva + - offsetof(struct compat_vcpu_runstate_info, - state_entry_time)); #ifdef CONFIG_X86_64 BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) != offsetof(struct compat_vcpu_runstate_info, state_entry_time) + 4); BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, time) != offsetof(struct compat_vcpu_runstate_info, time) + 4); - - if (v->kvm->arch.xen.long_mode) - user_times = (uint64_t __user *)(ghc->hva + - offsetof(struct vcpu_runstate_info, - state_entry_time)); #endif + + user_state = gpc->khva; + + if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) + user_times = gpc->khva + offsetof(struct vcpu_runstate_info, + state_entry_time); + else + user_times = gpc->khva + offsetof(struct compat_vcpu_runstate_info, + state_entry_time); + /* * First write the updated state_entry_time at the appropriate * location determined by 'offset'. */ - state_entry_time = vx->runstate_entry_time; - state_entry_time |= XEN_RUNSTATE_UPDATE; - BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state_entry_time) != - sizeof(state_entry_time)); + sizeof(user_times[0])); BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state_entry_time) != - sizeof(state_entry_time)); + sizeof(user_times[0])); - if (__put_user(state_entry_time, user_times)) - goto out; + user_times[0] = vx->runstate_entry_time | XEN_RUNSTATE_UPDATE; smp_wmb(); /* @@ -212,8 +216,7 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state) != sizeof(vx->current_runstate)); - if (__put_user(vx->current_runstate, user_state)) - goto out; + *user_state = vx->current_runstate; /* * Write the actual runstate times immediately after the @@ -228,23 +231,19 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) != sizeof(vx->runstate_times)); - if (__copy_to_user(user_times + 1, vx->runstate_times, sizeof(vx->runstate_times))) - goto out; + memcpy(user_times + 1, vx->runstate_times, sizeof(vx->runstate_times)); smp_wmb(); /* * Finally, clear the XEN_RUNSTATE_UPDATE bit in the guest's * runstate_entry_time field. */ - state_entry_time &= ~XEN_RUNSTATE_UPDATE; - __put_user(state_entry_time, user_times); + user_times[0] &= ~XEN_RUNSTATE_UPDATE; smp_wmb(); - out: - mark_page_dirty_in_slot(v->kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); + read_unlock_irqrestore(&gpc->lock, flags); - if (atomic) - pagefault_enable(); + mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); } int __kvm_xen_has_interrupt(struct kvm_vcpu *v) @@ -507,24 +506,16 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } if (data->u.gpa == GPA_INVALID) { - vcpu->arch.xen.runstate_set = false; + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, + &vcpu->arch.xen.runstate_cache); r = 0; break; } - /* It must fit within a single page */ - if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct vcpu_runstate_info) > PAGE_SIZE) { - r = -EINVAL; - break; - } - - r = kvm_gfn_to_hva_cache_init(vcpu->kvm, + r = kvm_gfn_to_pfn_cache_init(vcpu->kvm, &vcpu->arch.xen.runstate_cache, - data->u.gpa, + NULL, KVM_HOST_USES_PFN, data->u.gpa, sizeof(struct vcpu_runstate_info)); - if (!r) { - vcpu->arch.xen.runstate_set = true; - } break; case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_CURRENT: @@ -659,7 +650,7 @@ int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) r = -EOPNOTSUPP; break; } - if (vcpu->arch.xen.runstate_set) { + if (vcpu->arch.xen.runstate_cache.active) { data->u.gpa = vcpu->arch.xen.runstate_cache.gpa; r = 0; } @@ -1056,3 +1047,9 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, return 0; } + +void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) +{ + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, + &vcpu->arch.xen.runstate_cache); +} diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index adbcc9ed59db..54b2bf4c3001 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -23,7 +23,7 @@ int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data); int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc); void kvm_xen_init_vm(struct kvm *kvm); void kvm_xen_destroy_vm(struct kvm *kvm); - +void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu); int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm); int kvm_xen_setup_evtchn(struct kvm *kvm, @@ -65,6 +65,10 @@ static inline void kvm_xen_destroy_vm(struct kvm *kvm) { } +static inline void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) +{ +} + static inline bool kvm_xen_msr_enabled(struct kvm *kvm) { return false; From 916d3608df8265a2e3f6214200dbb0b39a5ca383 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:14 +0000 Subject: [PATCH 007/319] KVM: x86: Use gfn_to_pfn_cache for pv_time Add a new kvm_setup_guest_pvclock() which parallels the existing kvm_setup_pvclock_page(). The latter will be removed once we convert all users to the gfn_to_pfn_cache version. Using the new cache, we can potentially let kvm_set_guest_paused() set the PVCLOCK_GUEST_STOPPED bit directly rather than having to delegate to the vCPU via KVM_REQ_CLOCK_UPDATE. But not yet. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-5-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 +- arch/x86/kvm/x86.c | 76 +++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3263147a166f..5a0ff6a07431 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -751,8 +751,7 @@ struct kvm_vcpu_arch { gpa_t time; struct pvclock_vcpu_time_info hv_clock; unsigned int hw_tsc_khz; - struct gfn_to_hva_cache pv_time; - bool pv_time_enabled; + struct gfn_to_pfn_cache pv_time; /* set guest stopped flag in pvclock flags field */ bool pvclock_set_guest_stopped_request; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7042495eb089..e5d43b441c03 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2249,14 +2249,13 @@ static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time, kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu); /* we verify if the enable bit is set... */ - vcpu->arch.pv_time_enabled = false; - if (!(system_time & 1)) - return; - - if (!kvm_gfn_to_hva_cache_init(vcpu->kvm, - &vcpu->arch.pv_time, system_time & ~1ULL, - sizeof(struct pvclock_vcpu_time_info))) - vcpu->arch.pv_time_enabled = true; + if (system_time & 1) { + kvm_gfn_to_pfn_cache_init(vcpu->kvm, &vcpu->arch.pv_time, vcpu, + KVM_HOST_USES_PFN, system_time & ~1ULL, + sizeof(struct pvclock_vcpu_time_info)); + } else { + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.pv_time); + } return; } @@ -2961,6 +2960,57 @@ u64 get_kvmclock_ns(struct kvm *kvm) return data.clock; } +static void kvm_setup_guest_pvclock(struct kvm_vcpu *v, + struct gfn_to_pfn_cache *gpc, + unsigned int offset) +{ + struct kvm_vcpu_arch *vcpu = &v->arch; + struct pvclock_vcpu_time_info *guest_hv_clock; + unsigned long flags; + + read_lock_irqsave(&gpc->lock, flags); + while (!kvm_gfn_to_pfn_cache_check(v->kvm, gpc, gpc->gpa, + offset + sizeof(*guest_hv_clock))) { + read_unlock_irqrestore(&gpc->lock, flags); + + if (kvm_gfn_to_pfn_cache_refresh(v->kvm, gpc, gpc->gpa, + offset + sizeof(*guest_hv_clock))) + return; + + read_lock_irqsave(&gpc->lock, flags); + } + + guest_hv_clock = (void *)(gpc->khva + offset); + + /* + * This VCPU is paused, but it's legal for a guest to read another + * VCPU's kvmclock, so we really have to follow the specification where + * it says that version is odd if data is being modified, and even after + * it is consistent. + */ + + guest_hv_clock->version = vcpu->hv_clock.version = (guest_hv_clock->version + 1) | 1; + smp_wmb(); + + /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */ + vcpu->hv_clock.flags |= (guest_hv_clock->flags & PVCLOCK_GUEST_STOPPED); + + if (vcpu->pvclock_set_guest_stopped_request) { + vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED; + vcpu->pvclock_set_guest_stopped_request = false; + } + + memcpy(guest_hv_clock, &vcpu->hv_clock, sizeof(*guest_hv_clock)); + smp_wmb(); + + guest_hv_clock->version = ++vcpu->hv_clock.version; + + mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); + read_unlock_irqrestore(&gpc->lock, flags); + + trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock); +} + static void kvm_setup_pvclock_page(struct kvm_vcpu *v, struct gfn_to_hva_cache *cache, unsigned int offset) @@ -3106,8 +3156,8 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) vcpu->hv_clock.flags = pvclock_flags; - if (vcpu->pv_time_enabled) - kvm_setup_pvclock_page(v, &vcpu->pv_time, 0); + if (vcpu->pv_time.active) + kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0); if (vcpu->xen.vcpu_info_set) kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_info_cache, offsetof(struct compat_vcpu_info, time)); @@ -3301,7 +3351,7 @@ static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data) static void kvmclock_reset(struct kvm_vcpu *vcpu) { - vcpu->arch.pv_time_enabled = false; + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.pv_time); vcpu->arch.time = 0; } @@ -5103,7 +5153,7 @@ static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu, */ static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) { - if (!vcpu->arch.pv_time_enabled) + if (!vcpu->arch.pv_time.active) return -EINVAL; vcpu->arch.pvclock_set_guest_stopped_request = true; kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); @@ -6187,7 +6237,7 @@ static int kvm_arch_suspend_notifier(struct kvm *kvm) mutex_lock(&kvm->lock); kvm_for_each_vcpu(i, vcpu, kvm) { - if (!vcpu->arch.pv_time_enabled) + if (!vcpu->arch.pv_time.active) continue; ret = kvm_set_guest_paused(vcpu); From 7caf9571563eade546976d600dd023674b1c3185 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:15 +0000 Subject: [PATCH 008/319] KVM: x86/xen: Use gfn_to_pfn_cache for vcpu_info Currently, the fast path of kvm_xen_set_evtchn_fast() doesn't set the index bits in the target vCPU's evtchn_pending_sel, because it only has a userspace virtual address with which to do so. It just sets them in the kernel, and kvm_xen_has_interrupt() then completes the delivery to the actual vcpu_info structure when the vCPU runs. Using a gfn_to_pfn_cache allows kvm_xen_set_evtchn_fast() to do the full delivery in the common case. Clean up the fallback case too, by moving the deferred delivery out into a separate kvm_xen_inject_pending_events() function which isn't ever called in atomic contexts as __kvm_xen_has_interrupt() is. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-6-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 +- arch/x86/kvm/x86.c | 12 +- arch/x86/kvm/xen.c | 242 +++++++++++++++++--------------- arch/x86/kvm/xen.h | 20 ++- 4 files changed, 157 insertions(+), 120 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5a0ff6a07431..fce4dca16e5b 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -606,9 +606,8 @@ struct kvm_vcpu_hv { struct kvm_vcpu_xen { u64 hypercall_rip; u32 current_runstate; - bool vcpu_info_set; bool vcpu_time_info_set; - struct gfn_to_hva_cache vcpu_info_cache; + struct gfn_to_pfn_cache vcpu_info_cache; struct gfn_to_hva_cache vcpu_time_info_cache; struct gfn_to_pfn_cache runstate_cache; u64 last_steal; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e5d43b441c03..d67487efdba2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3158,9 +3158,9 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) if (vcpu->pv_time.active) kvm_setup_guest_pvclock(v, &vcpu->pv_time, 0); - if (vcpu->xen.vcpu_info_set) - kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_info_cache, - offsetof(struct compat_vcpu_info, time)); + if (vcpu->xen.vcpu_info_cache.active) + kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache, + offsetof(struct compat_vcpu_info, time)); if (vcpu->xen.vcpu_time_info_set) kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_time_info_cache, 0); if (!v->vcpu_idx) @@ -10424,6 +10424,9 @@ static int vcpu_run(struct kvm_vcpu *vcpu) break; kvm_clear_request(KVM_REQ_UNBLOCK, vcpu); + if (kvm_xen_has_pending_events(vcpu)) + kvm_xen_inject_pending_events(vcpu); + if (kvm_cpu_has_pending_timer(vcpu)) kvm_inject_pending_timer_irqs(vcpu); @@ -12236,6 +12239,9 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) kvm_x86_ops.nested_ops->hv_timer_pending(vcpu)) return true; + if (kvm_xen_has_pending_events(vcpu)) + return true; + return false; } diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index de1b3be7239e..56e8a6980b65 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -8,6 +8,7 @@ #include "x86.h" #include "xen.h" +#include "lapic.h" #include "hyperv.h" #include @@ -246,23 +247,79 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); } -int __kvm_xen_has_interrupt(struct kvm_vcpu *v) +/* + * On event channel delivery, the vcpu_info may not have been accessible. + * In that case, there are bits in vcpu->arch.xen.evtchn_pending_sel which + * need to be marked into the vcpu_info (and evtchn_upcall_pending set). + * Do so now that we can sleep in the context of the vCPU to bring the + * page in, and refresh the pfn cache for it. + */ +void kvm_xen_inject_pending_events(struct kvm_vcpu *v) { unsigned long evtchn_pending_sel = READ_ONCE(v->arch.xen.evtchn_pending_sel); - bool atomic = in_atomic() || !task_is_running(current); - int err; + struct gfn_to_pfn_cache *gpc = &v->arch.xen.vcpu_info_cache; + unsigned long flags; + + if (!evtchn_pending_sel) + return; + + /* + * Yes, this is an open-coded loop. But that's just what put_user() + * does anyway. Page it in and retry the instruction. We're just a + * little more honest about it. + */ + read_lock_irqsave(&gpc->lock, flags); + while (!kvm_gfn_to_pfn_cache_check(v->kvm, gpc, gpc->gpa, + sizeof(struct vcpu_info))) { + read_unlock_irqrestore(&gpc->lock, flags); + + if (kvm_gfn_to_pfn_cache_refresh(v->kvm, gpc, gpc->gpa, + sizeof(struct vcpu_info))) + return; + + read_lock_irqsave(&gpc->lock, flags); + } + + /* Now gpc->khva is a valid kernel address for the vcpu_info */ + if (IS_ENABLED(CONFIG_64BIT) && v->kvm->arch.xen.long_mode) { + struct vcpu_info *vi = gpc->khva; + + asm volatile(LOCK_PREFIX "orq %0, %1\n" + "notq %0\n" + LOCK_PREFIX "andq %0, %2\n" + : "=r" (evtchn_pending_sel), + "+m" (vi->evtchn_pending_sel), + "+m" (v->arch.xen.evtchn_pending_sel) + : "0" (evtchn_pending_sel)); + WRITE_ONCE(vi->evtchn_upcall_pending, 1); + } else { + u32 evtchn_pending_sel32 = evtchn_pending_sel; + struct compat_vcpu_info *vi = gpc->khva; + + asm volatile(LOCK_PREFIX "orl %0, %1\n" + "notl %0\n" + LOCK_PREFIX "andl %0, %2\n" + : "=r" (evtchn_pending_sel32), + "+m" (vi->evtchn_pending_sel), + "+m" (v->arch.xen.evtchn_pending_sel) + : "0" (evtchn_pending_sel32)); + WRITE_ONCE(vi->evtchn_upcall_pending, 1); + } + read_unlock_irqrestore(&gpc->lock, flags); + + mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); +} + +int __kvm_xen_has_interrupt(struct kvm_vcpu *v) +{ + struct gfn_to_pfn_cache *gpc = &v->arch.xen.vcpu_info_cache; + unsigned long flags; u8 rc = 0; /* * If the global upcall vector (HVMIRQ_callback_vector) is set and * the vCPU's evtchn_upcall_pending flag is set, the IRQ is pending. */ - struct gfn_to_hva_cache *ghc = &v->arch.xen.vcpu_info_cache; - struct kvm_memslots *slots = kvm_memslots(v->kvm); - bool ghc_valid = slots->generation == ghc->generation && - !kvm_is_error_hva(ghc->hva) && ghc->memslot; - - unsigned int offset = offsetof(struct vcpu_info, evtchn_upcall_pending); /* No need for compat handling here */ BUILD_BUG_ON(offsetof(struct vcpu_info, evtchn_upcall_pending) != @@ -272,101 +329,35 @@ int __kvm_xen_has_interrupt(struct kvm_vcpu *v) BUILD_BUG_ON(sizeof(rc) != sizeof_field(struct compat_vcpu_info, evtchn_upcall_pending)); - /* - * For efficiency, this mirrors the checks for using the valid - * cache in kvm_read_guest_offset_cached(), but just uses - * __get_user() instead. And falls back to the slow path. - */ - if (!evtchn_pending_sel && ghc_valid) { - /* Fast path */ - pagefault_disable(); - err = __get_user(rc, (u8 __user *)ghc->hva + offset); - pagefault_enable(); - if (!err) - return rc; - } + read_lock_irqsave(&gpc->lock, flags); + while (!kvm_gfn_to_pfn_cache_check(v->kvm, gpc, gpc->gpa, + sizeof(struct vcpu_info))) { + read_unlock_irqrestore(&gpc->lock, flags); - /* Slow path */ + /* + * This function gets called from kvm_vcpu_block() after setting the + * task to TASK_INTERRUPTIBLE, to see if it needs to wake immediately + * from a HLT. So we really mustn't sleep. If the page ended up absent + * at that point, just return 1 in order to trigger an immediate wake, + * and we'll end up getting called again from a context where we *can* + * fault in the page and wait for it. + */ + if (in_atomic() || !task_is_running(current)) + return 1; - /* - * This function gets called from kvm_vcpu_block() after setting the - * task to TASK_INTERRUPTIBLE, to see if it needs to wake immediately - * from a HLT. So we really mustn't sleep. If the page ended up absent - * at that point, just return 1 in order to trigger an immediate wake, - * and we'll end up getting called again from a context where we *can* - * fault in the page and wait for it. - */ - if (atomic) - return 1; - - if (!ghc_valid) { - err = kvm_gfn_to_hva_cache_init(v->kvm, ghc, ghc->gpa, ghc->len); - if (err || !ghc->memslot) { + if (kvm_gfn_to_pfn_cache_refresh(v->kvm, gpc, gpc->gpa, + sizeof(struct vcpu_info))) { /* * If this failed, userspace has screwed up the * vcpu_info mapping. No interrupts for you. */ return 0; } + read_lock_irqsave(&gpc->lock, flags); } - /* - * Now we have a valid (protected by srcu) userspace HVA in - * ghc->hva which points to the struct vcpu_info. If there - * are any bits in the in-kernel evtchn_pending_sel then - * we need to write those to the guest vcpu_info and set - * its evtchn_upcall_pending flag. If there aren't any bits - * to add, we only want to *check* evtchn_upcall_pending. - */ - if (evtchn_pending_sel) { - bool long_mode = v->kvm->arch.xen.long_mode; - - if (!user_access_begin((void __user *)ghc->hva, sizeof(struct vcpu_info))) - return 0; - - if (IS_ENABLED(CONFIG_64BIT) && long_mode) { - struct vcpu_info __user *vi = (void __user *)ghc->hva; - - /* Attempt to set the evtchn_pending_sel bits in the - * guest, and if that succeeds then clear the same - * bits in the in-kernel version. */ - asm volatile("1:\t" LOCK_PREFIX "orq %0, %1\n" - "\tnotq %0\n" - "\t" LOCK_PREFIX "andq %0, %2\n" - "2:\n" - _ASM_EXTABLE_UA(1b, 2b) - : "=r" (evtchn_pending_sel), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel)); - } else { - struct compat_vcpu_info __user *vi = (void __user *)ghc->hva; - u32 evtchn_pending_sel32 = evtchn_pending_sel; - - /* Attempt to set the evtchn_pending_sel bits in the - * guest, and if that succeeds then clear the same - * bits in the in-kernel version. */ - asm volatile("1:\t" LOCK_PREFIX "orl %0, %1\n" - "\tnotl %0\n" - "\t" LOCK_PREFIX "andl %0, %2\n" - "2:\n" - _ASM_EXTABLE_UA(1b, 2b) - : "=r" (evtchn_pending_sel32), - "+m" (vi->evtchn_pending_sel), - "+m" (v->arch.xen.evtchn_pending_sel) - : "0" (evtchn_pending_sel32)); - } - rc = 1; - unsafe_put_user(rc, (u8 __user *)ghc->hva + offset, err); - - err: - user_access_end(); - - mark_page_dirty_in_slot(v->kvm, ghc->memslot, ghc->gpa >> PAGE_SHIFT); - } else { - __get_user(rc, (u8 __user *)ghc->hva + offset); - } - + rc = ((struct vcpu_info *)gpc->khva)->evtchn_upcall_pending; + read_unlock_irqrestore(&gpc->lock, flags); return rc; } @@ -456,25 +447,18 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) offsetof(struct compat_vcpu_info, time)); if (data->u.gpa == GPA_INVALID) { - vcpu->arch.xen.vcpu_info_set = false; + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.xen.vcpu_info_cache); r = 0; break; } - /* It must fit within a single page */ - if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct vcpu_info) > PAGE_SIZE) { - r = -EINVAL; - break; - } - - r = kvm_gfn_to_hva_cache_init(vcpu->kvm, + r = kvm_gfn_to_pfn_cache_init(vcpu->kvm, &vcpu->arch.xen.vcpu_info_cache, - data->u.gpa, + NULL, KVM_HOST_USES_PFN, data->u.gpa, sizeof(struct vcpu_info)); - if (!r) { - vcpu->arch.xen.vcpu_info_set = true; + if (!r) kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); - } + break; case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO: @@ -630,7 +614,7 @@ int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) switch (data->type) { case KVM_XEN_VCPU_ATTR_TYPE_VCPU_INFO: - if (vcpu->arch.xen.vcpu_info_set) + if (vcpu->arch.xen.vcpu_info_cache.active) data->u.gpa = vcpu->arch.xen.vcpu_info_cache.gpa; else data->u.gpa = GPA_INVALID; @@ -903,16 +887,17 @@ int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, if (!vcpu) return -1; - if (!vcpu->arch.xen.vcpu_info_set) + if (!vcpu->arch.xen.vcpu_info_cache.active) return -1; if (e->xen_evtchn.port >= max_evtchn_port(kvm)) return -1; rc = -EWOULDBLOCK; - read_lock_irqsave(&gpc->lock, flags); idx = srcu_read_lock(&kvm->srcu); + + read_lock_irqsave(&gpc->lock, flags); if (!kvm_gfn_to_pfn_cache_check(kvm, gpc, gpc->gpa, PAGE_SIZE)) goto out_rcu; @@ -940,17 +925,44 @@ int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, } else if (test_bit(e->xen_evtchn.port, mask_bits)) { rc = -1; /* Masked */ } else { - rc = 1; /* Delivered. But was the vCPU waking already? */ - if (!test_and_set_bit(port_word_bit, &vcpu->arch.xen.evtchn_pending_sel)) - kick_vcpu = true; + rc = 1; /* Delivered to the bitmap in shared_info. */ + /* Now switch to the vCPU's vcpu_info to set the index and pending_sel */ + read_unlock_irqrestore(&gpc->lock, flags); + gpc = &vcpu->arch.xen.vcpu_info_cache; + + read_lock_irqsave(&gpc->lock, flags); + if (!kvm_gfn_to_pfn_cache_check(kvm, gpc, gpc->gpa, sizeof(struct vcpu_info))) { + /* + * Could not access the vcpu_info. Set the bit in-kernel + * and prod the vCPU to deliver it for itself. + */ + if (!test_and_set_bit(port_word_bit, &vcpu->arch.xen.evtchn_pending_sel)) + kick_vcpu = true; + goto out_rcu; + } + + if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + struct vcpu_info *vcpu_info = gpc->khva; + if (!test_and_set_bit(port_word_bit, &vcpu_info->evtchn_pending_sel)) { + WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); + kick_vcpu = true; + } + } else { + struct compat_vcpu_info *vcpu_info = gpc->khva; + if (!test_and_set_bit(port_word_bit, + (unsigned long *)&vcpu_info->evtchn_pending_sel)) { + WRITE_ONCE(vcpu_info->evtchn_upcall_pending, 1); + kick_vcpu = true; + } + } } out_rcu: - srcu_read_unlock(&kvm->srcu, idx); read_unlock_irqrestore(&gpc->lock, flags); + srcu_read_unlock(&kvm->srcu, idx); if (kick_vcpu) { - kvm_make_request(KVM_REQ_EVENT, vcpu); + kvm_make_request(KVM_REQ_UNBLOCK, vcpu); kvm_vcpu_kick(vcpu); } @@ -1052,4 +1064,6 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) { kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.xen.runstate_cache); + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, + &vcpu->arch.xen.vcpu_info_cache); } diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index 54b2bf4c3001..7dd0590f93e1 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -15,6 +15,7 @@ extern struct static_key_false_deferred kvm_xen_enabled; int __kvm_xen_has_interrupt(struct kvm_vcpu *vcpu); +void kvm_xen_inject_pending_events(struct kvm_vcpu *vcpu); int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data); int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data); int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data); @@ -46,11 +47,19 @@ static inline bool kvm_xen_hypercall_enabled(struct kvm *kvm) static inline int kvm_xen_has_interrupt(struct kvm_vcpu *vcpu) { if (static_branch_unlikely(&kvm_xen_enabled.key) && - vcpu->arch.xen.vcpu_info_set && vcpu->kvm->arch.xen.upcall_vector) + vcpu->arch.xen.vcpu_info_cache.active && + vcpu->kvm->arch.xen.upcall_vector) return __kvm_xen_has_interrupt(vcpu); return 0; } + +static inline bool kvm_xen_has_pending_events(struct kvm_vcpu *vcpu) +{ + return static_branch_unlikely(&kvm_xen_enabled.key) && + vcpu->arch.xen.evtchn_pending_sel; +} + #else static inline int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data) { @@ -83,6 +92,15 @@ static inline int kvm_xen_has_interrupt(struct kvm_vcpu *vcpu) { return 0; } + +static inline void kvm_xen_inject_pending_events(struct kvm_vcpu *vcpu) +{ +} + +static inline bool kvm_xen_has_pending_events(struct kvm_vcpu *vcpu) +{ + return false; +} #endif int kvm_xen_hypercall(struct kvm_vcpu *vcpu); From 69d413cfcf77920bb4d7c4bbfbf305781fa53a0e Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:16 +0000 Subject: [PATCH 009/319] KVM: x86/xen: Use gfn_to_pfn_cache for vcpu_time_info This switches the final pvclock to kvm_setup_pvclock_pfncache() and now the old kvm_setup_pvclock_page() can be removed. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-7-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 +- arch/x86/kvm/x86.c | 63 ++------------------------------- arch/x86/kvm/xen.c | 21 +++++------ 3 files changed, 11 insertions(+), 76 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index fce4dca16e5b..7808491ebba8 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -606,9 +606,8 @@ struct kvm_vcpu_hv { struct kvm_vcpu_xen { u64 hypercall_rip; u32 current_runstate; - bool vcpu_time_info_set; struct gfn_to_pfn_cache vcpu_info_cache; - struct gfn_to_hva_cache vcpu_time_info_cache; + struct gfn_to_pfn_cache vcpu_time_info_cache; struct gfn_to_pfn_cache runstate_cache; u64 last_steal; u64 runstate_entry_time; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d67487efdba2..feefa1fb32eb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3011,65 +3011,6 @@ static void kvm_setup_guest_pvclock(struct kvm_vcpu *v, trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock); } -static void kvm_setup_pvclock_page(struct kvm_vcpu *v, - struct gfn_to_hva_cache *cache, - unsigned int offset) -{ - struct kvm_vcpu_arch *vcpu = &v->arch; - struct pvclock_vcpu_time_info guest_hv_clock; - - if (unlikely(kvm_read_guest_offset_cached(v->kvm, cache, - &guest_hv_clock, offset, sizeof(guest_hv_clock)))) - return; - - /* This VCPU is paused, but it's legal for a guest to read another - * VCPU's kvmclock, so we really have to follow the specification where - * it says that version is odd if data is being modified, and even after - * it is consistent. - * - * Version field updates must be kept separate. This is because - * kvm_write_guest_cached might use a "rep movs" instruction, and - * writes within a string instruction are weakly ordered. So there - * are three writes overall. - * - * As a small optimization, only write the version field in the first - * and third write. The vcpu->pv_time cache is still valid, because the - * version field is the first in the struct. - */ - BUILD_BUG_ON(offsetof(struct pvclock_vcpu_time_info, version) != 0); - - if (guest_hv_clock.version & 1) - ++guest_hv_clock.version; /* first time write, random junk */ - - vcpu->hv_clock.version = guest_hv_clock.version + 1; - kvm_write_guest_offset_cached(v->kvm, cache, - &vcpu->hv_clock, offset, - sizeof(vcpu->hv_clock.version)); - - smp_wmb(); - - /* retain PVCLOCK_GUEST_STOPPED if set in guest copy */ - vcpu->hv_clock.flags |= (guest_hv_clock.flags & PVCLOCK_GUEST_STOPPED); - - if (vcpu->pvclock_set_guest_stopped_request) { - vcpu->hv_clock.flags |= PVCLOCK_GUEST_STOPPED; - vcpu->pvclock_set_guest_stopped_request = false; - } - - trace_kvm_pvclock_update(v->vcpu_id, &vcpu->hv_clock); - - kvm_write_guest_offset_cached(v->kvm, cache, - &vcpu->hv_clock, offset, - sizeof(vcpu->hv_clock)); - - smp_wmb(); - - vcpu->hv_clock.version++; - kvm_write_guest_offset_cached(v->kvm, cache, - &vcpu->hv_clock, offset, - sizeof(vcpu->hv_clock.version)); -} - static int kvm_guest_time_update(struct kvm_vcpu *v) { unsigned long flags, tgt_tsc_khz; @@ -3161,8 +3102,8 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) if (vcpu->xen.vcpu_info_cache.active) kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_info_cache, offsetof(struct compat_vcpu_info, time)); - if (vcpu->xen.vcpu_time_info_set) - kvm_setup_pvclock_page(v, &vcpu->xen.vcpu_time_info_cache, 0); + if (vcpu->xen.vcpu_time_info_cache.active) + kvm_setup_guest_pvclock(v, &vcpu->xen.vcpu_time_info_cache, 0); if (!v->vcpu_idx) kvm_hv_setup_tsc_page(v->kvm, &vcpu->hv_clock); return 0; diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 56e8a6980b65..65ffba89441a 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -463,25 +463,18 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO: if (data->u.gpa == GPA_INVALID) { - vcpu->arch.xen.vcpu_time_info_set = false; + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, + &vcpu->arch.xen.vcpu_time_info_cache); r = 0; break; } - /* It must fit within a single page */ - if ((data->u.gpa & ~PAGE_MASK) + sizeof(struct pvclock_vcpu_time_info) > PAGE_SIZE) { - r = -EINVAL; - break; - } - - r = kvm_gfn_to_hva_cache_init(vcpu->kvm, + r = kvm_gfn_to_pfn_cache_init(vcpu->kvm, &vcpu->arch.xen.vcpu_time_info_cache, - data->u.gpa, + NULL, KVM_HOST_USES_PFN, data->u.gpa, sizeof(struct pvclock_vcpu_time_info)); - if (!r) { - vcpu->arch.xen.vcpu_time_info_set = true; + if (!r) kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); - } break; case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADDR: @@ -622,7 +615,7 @@ int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO: - if (vcpu->arch.xen.vcpu_time_info_set) + if (vcpu->arch.xen.vcpu_time_info_cache.active) data->u.gpa = vcpu->arch.xen.vcpu_time_info_cache.gpa; else data->u.gpa = GPA_INVALID; @@ -1066,4 +1059,6 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) &vcpu->arch.xen.runstate_cache); kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.xen.vcpu_info_cache); + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, + &vcpu->arch.xen.vcpu_time_info_cache); } From 8733068b9bdbc7a54f02dcc59eb0e4789cd60942 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:17 +0000 Subject: [PATCH 010/319] KVM: x86/xen: Make kvm_xen_set_evtchn() reusable from other places Clean it up to return -errno on error consistently, while still being compatible with the return conventions for kvm_arch_set_irq_inatomic() and the kvm_set_irq() callback. We use -ENOTCONN to indicate when the port is masked. No existing users care, except that it's negative. Also allow it to optimise the vCPU lookup. Unless we abuse the lapic map, there is no quick lookup from APIC ID to a vCPU; the logic in kvm_get_vcpu_by_id() will just iterate over all vCPUs till it finds the one it wants. So do that just once and stash the result in the struct kvm_xen_evtchn for next time. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-8-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/irq_comm.c | 2 +- arch/x86/kvm/xen.c | 83 ++++++++++++++++++++++++++++------------ arch/x86/kvm/xen.h | 2 +- include/linux/kvm_host.h | 3 +- 4 files changed, 62 insertions(+), 28 deletions(-) diff --git a/arch/x86/kvm/irq_comm.c b/arch/x86/kvm/irq_comm.c index 6e0dab04320e..0687162c4f22 100644 --- a/arch/x86/kvm/irq_comm.c +++ b/arch/x86/kvm/irq_comm.c @@ -181,7 +181,7 @@ int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, if (!level) return -1; - return kvm_xen_set_evtchn_fast(e, kvm); + return kvm_xen_set_evtchn_fast(&e->xen_evtchn, kvm); #endif default: break; diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 65ffba89441a..9c87263a5be2 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -859,13 +859,16 @@ static inline int max_evtchn_port(struct kvm *kvm) } /* - * This follows the kvm_set_irq() API, so it returns: + * The return value from this function is propagated to kvm_set_irq() API, + * so it returns: * < 0 Interrupt was ignored (masked or not delivered for other reasons) * = 0 Interrupt was coalesced (previous irq is still pending) * > 0 Number of CPUs interrupt was delivered to + * + * It is also called directly from kvm_arch_set_irq_inatomic(), where the + * only check on its return value is a comparison with -EWOULDBLOCK'. */ -int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, - struct kvm *kvm) +int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) { struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; struct kvm_vcpu *vcpu; @@ -873,18 +876,23 @@ int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, unsigned long flags; int port_word_bit; bool kick_vcpu = false; - int idx; - int rc; + int vcpu_idx, idx, rc; - vcpu = kvm_get_vcpu_by_id(kvm, e->xen_evtchn.vcpu); - if (!vcpu) - return -1; + vcpu_idx = READ_ONCE(xe->vcpu_idx); + if (vcpu_idx >= 0) + vcpu = kvm_get_vcpu(kvm, vcpu_idx); + else { + vcpu = kvm_get_vcpu_by_id(kvm, xe->vcpu_id); + if (!vcpu) + return -EINVAL; + WRITE_ONCE(xe->vcpu_idx, kvm_vcpu_get_idx(vcpu)); + } if (!vcpu->arch.xen.vcpu_info_cache.active) - return -1; + return -EINVAL; - if (e->xen_evtchn.port >= max_evtchn_port(kvm)) - return -1; + if (xe->port >= max_evtchn_port(kvm)) + return -EINVAL; rc = -EWOULDBLOCK; @@ -898,12 +906,12 @@ int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, struct shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; - port_word_bit = e->xen_evtchn.port / 64; + port_word_bit = xe->port / 64; } else { struct compat_shared_info *shinfo = gpc->khva; pending_bits = (unsigned long *)&shinfo->evtchn_pending; mask_bits = (unsigned long *)&shinfo->evtchn_mask; - port_word_bit = e->xen_evtchn.port / 32; + port_word_bit = xe->port / 32; } /* @@ -913,10 +921,10 @@ int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, * already set, then we kick the vCPU in question to write to the * *real* evtchn_pending_sel in its own guest vcpu_info struct. */ - if (test_and_set_bit(e->xen_evtchn.port, pending_bits)) { + if (test_and_set_bit(xe->port, pending_bits)) { rc = 0; /* It was already raised */ - } else if (test_bit(e->xen_evtchn.port, mask_bits)) { - rc = -1; /* Masked */ + } else if (test_bit(xe->port, mask_bits)) { + rc = -ENOTCONN; /* Masked */ } else { rc = 1; /* Delivered to the bitmap in shared_info. */ /* Now switch to the vCPU's vcpu_info to set the index and pending_sel */ @@ -962,17 +970,12 @@ int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, return rc; } -/* This is the version called from kvm_set_irq() as the .set function */ -static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, - int irq_source_id, int level, bool line_status) +static int kvm_xen_set_evtchn(struct kvm_xen_evtchn *xe, struct kvm *kvm) { bool mm_borrowed = false; int rc; - if (!level) - return -1; - - rc = kvm_xen_set_evtchn_fast(e, kvm); + rc = kvm_xen_set_evtchn_fast(xe, kvm); if (rc != -EWOULDBLOCK) return rc; @@ -1016,7 +1019,7 @@ static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; int idx; - rc = kvm_xen_set_evtchn_fast(e, kvm); + rc = kvm_xen_set_evtchn_fast(xe, kvm); if (rc != -EWOULDBLOCK) break; @@ -1033,11 +1036,27 @@ static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm return rc; } +/* This is the version called from kvm_set_irq() as the .set function */ +static int evtchn_set_fn(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, + int irq_source_id, int level, bool line_status) +{ + if (!level) + return -EINVAL; + + return kvm_xen_set_evtchn(&e->xen_evtchn, kvm); +} + +/* + * Set up an event channel interrupt from the KVM IRQ routing table. + * Used for e.g. PIRQ from passed through physical devices. + */ int kvm_xen_setup_evtchn(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, const struct kvm_irq_routing_entry *ue) { + struct kvm_vcpu *vcpu; + if (ue->u.xen_evtchn.port >= max_evtchn_port(kvm)) return -EINVAL; @@ -1045,8 +1064,22 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, if (ue->u.xen_evtchn.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) return -EINVAL; + /* + * Xen gives us interesting mappings from vCPU index to APIC ID, + * which means kvm_get_vcpu_by_id() has to iterate over all vCPUs + * to find it. Do that once at setup time, instead of every time. + * But beware that on live update / live migration, the routing + * table might be reinstated before the vCPU threads have finished + * recreating their vCPUs. + */ + vcpu = kvm_get_vcpu_by_id(kvm, ue->u.xen_evtchn.vcpu); + if (vcpu) + e->xen_evtchn.vcpu_idx = kvm_vcpu_get_idx(vcpu); + else + e->xen_evtchn.vcpu_idx = -1; + e->xen_evtchn.port = ue->u.xen_evtchn.port; - e->xen_evtchn.vcpu = ue->u.xen_evtchn.vcpu; + e->xen_evtchn.vcpu_id = ue->u.xen_evtchn.vcpu; e->xen_evtchn.priority = ue->u.xen_evtchn.priority; e->set = evtchn_set_fn; diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index 7dd0590f93e1..e28feb32add6 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -25,7 +25,7 @@ int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc); void kvm_xen_init_vm(struct kvm *kvm); void kvm_xen_destroy_vm(struct kvm *kvm); void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu); -int kvm_xen_set_evtchn_fast(struct kvm_kernel_irq_routing_entry *e, +int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm); int kvm_xen_setup_evtchn(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 3f9b22c4983a..252ee4a61b58 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -611,7 +611,8 @@ struct kvm_hv_sint { struct kvm_xen_evtchn { u32 port; - u32 vcpu; + u32 vcpu_id; + int vcpu_idx; u32 priority; }; From 35025735a79eaa894c43837b94fd33c9d6b122df Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:18 +0000 Subject: [PATCH 011/319] KVM: x86/xen: Support direct injection of event channel events This adds a KVM_XEN_HVM_EVTCHN_SEND ioctl which allows direct injection of events given an explicit { vcpu, port, priority } in precisely the same form that those fields are given in the IRQ routing table. Userspace is currently able to inject 2-level events purely by setting the bits in the shared_info and vcpu_info, but FIFO event channels are harder to deal with; we will need the kernel to take sole ownership of delivery when we support those. A patch advertising this feature with a new bit in the KVM_CAP_XEN_HVM ioctl will be added in a subsequent patch. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-9-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 9 +++++++++ arch/x86/kvm/xen.c | 32 ++++++++++++++++++++++++++++++++ arch/x86/kvm/xen.h | 1 + include/uapi/linux/kvm.h | 3 +++ 4 files changed, 45 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index feefa1fb32eb..e6bc0b654bbb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6505,6 +6505,15 @@ long kvm_arch_vm_ioctl(struct file *filp, r = kvm_xen_hvm_set_attr(kvm, &xha); break; } + case KVM_XEN_HVM_EVTCHN_SEND: { + struct kvm_irq_routing_xen_evtchn uxe; + + r = -EFAULT; + if (copy_from_user(&uxe, argp, sizeof(uxe))) + goto out; + r = kvm_xen_hvm_evtchn_send(kvm, &uxe); + break; + } #endif case KVM_SET_CLOCK: r = kvm_vm_ioctl_set_clock(kvm, argp); diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 9c87263a5be2..f0f0011c4617 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1086,6 +1086,38 @@ int kvm_xen_setup_evtchn(struct kvm *kvm, return 0; } +/* + * Explicit event sending from userspace with KVM_XEN_HVM_EVTCHN_SEND ioctl. + */ +int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn *uxe) +{ + struct kvm_xen_evtchn e; + int ret; + + if (!uxe->port || uxe->port >= max_evtchn_port(kvm)) + return -EINVAL; + + /* We only support 2 level event channels for now */ + if (uxe->priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) + return -EINVAL; + + e.port = uxe->port; + e.vcpu_id = uxe->vcpu; + e.vcpu_idx = -1; + e.priority = uxe->priority; + + ret = kvm_xen_set_evtchn(&e, kvm); + + /* + * None of that 'return 1 if it actually got delivered' nonsense. + * We don't care if it was masked (-ENOTCONN) either. + */ + if (ret > 0 || ret == -ENOTCONN) + ret = 0; + + return ret; +} + void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) { kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index e28feb32add6..852286de574e 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -20,6 +20,7 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data); int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data); int kvm_xen_hvm_get_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data); +int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn *evt); int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data); int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc); void kvm_xen_init_vm(struct kvm *kvm); diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 91a6fe4e02c0..49cd2e9e0f6a 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1699,6 +1699,9 @@ struct kvm_xen_hvm_attr { #define KVM_XEN_VCPU_GET_ATTR _IOWR(KVMIO, 0xca, struct kvm_xen_vcpu_attr) #define KVM_XEN_VCPU_SET_ATTR _IOW(KVMIO, 0xcb, struct kvm_xen_vcpu_attr) +/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ +#define KVM_XEN_HVM_EVTCHN_SEND _IOW(KVMIO, 0xd0, struct kvm_irq_routing_xen_evtchn) + #define KVM_GET_SREGS2 _IOR(KVMIO, 0xcc, struct kvm_sregs2) #define KVM_SET_SREGS2 _IOW(KVMIO, 0xcd, struct kvm_sregs2) From 2fd6df2f2b47d4301b1ee0fe9d627d1c061a5988 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Thu, 3 Mar 2022 15:41:19 +0000 Subject: [PATCH 012/319] KVM: x86/xen: intercept EVTCHNOP_send from guests Userspace registers a sending @port to either deliver to an @eventfd or directly back to a local event channel port. After binding events the guest or host may wish to bind those events to a particular vcpu. This is usually done for unbound and and interdomain events. Update requests are handled via the KVM_XEN_EVTCHN_UPDATE flag. Unregistered ports are handled by the emulator. Co-developed-by: Ankur Arora Co-developed-By: David Woodhouse Signed-off-by: Joao Martins Signed-off-by: Ankur Arora Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-10-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/xen.c | 295 ++++++++++++++++++++++++++++++-- include/uapi/linux/kvm.h | 28 +++ 3 files changed, 309 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 7808491ebba8..b20f7d99d702 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1024,6 +1024,7 @@ struct kvm_xen { bool long_mode; u8 upcall_vector; struct gfn_to_pfn_cache shinfo_cache; + struct idr evtchn_ports; }; enum kvm_irqchip_mode { diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index f0f0011c4617..3d95167028ba 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -11,6 +11,7 @@ #include "lapic.h" #include "hyperv.h" +#include #include #include @@ -21,6 +22,9 @@ #include "trace.h" +static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data); +static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r); + DEFINE_STATIC_KEY_DEFERRED_FALSE(kvm_xen_enabled, HZ); static int kvm_xen_shared_info_init(struct kvm *kvm, gfn_t gfn) @@ -365,36 +369,44 @@ int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data) { int r = -ENOENT; - mutex_lock(&kvm->lock); switch (data->type) { case KVM_XEN_ATTR_TYPE_LONG_MODE: if (!IS_ENABLED(CONFIG_64BIT) && data->u.long_mode) { r = -EINVAL; } else { + mutex_lock(&kvm->lock); kvm->arch.xen.long_mode = !!data->u.long_mode; + mutex_unlock(&kvm->lock); r = 0; } break; case KVM_XEN_ATTR_TYPE_SHARED_INFO: + mutex_lock(&kvm->lock); r = kvm_xen_shared_info_init(kvm, data->u.shared_info.gfn); + mutex_unlock(&kvm->lock); break; case KVM_XEN_ATTR_TYPE_UPCALL_VECTOR: if (data->u.vector && data->u.vector < 0x10) r = -EINVAL; else { + mutex_lock(&kvm->lock); kvm->arch.xen.upcall_vector = data->u.vector; + mutex_unlock(&kvm->lock); r = 0; } break; + case KVM_XEN_ATTR_TYPE_EVTCHN: + r = kvm_xen_setattr_evtchn(kvm, data); + break; + default: break; } - mutex_unlock(&kvm->lock); return r; } @@ -770,18 +782,6 @@ int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc) return 0; } -void kvm_xen_init_vm(struct kvm *kvm) -{ -} - -void kvm_xen_destroy_vm(struct kvm *kvm) -{ - kvm_gfn_to_pfn_cache_destroy(kvm, &kvm->arch.xen.shinfo_cache); - - if (kvm->arch.xen_hvm_config.msr) - static_branch_slow_dec_deferred(&kvm_xen_enabled); -} - static int kvm_xen_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) { kvm_rax_write(vcpu, result); @@ -801,7 +801,8 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) int kvm_xen_hypercall(struct kvm_vcpu *vcpu) { bool longmode; - u64 input, params[6]; + u64 input, params[6], r = -ENOSYS; + bool handled = false; input = (u64)kvm_register_read(vcpu, VCPU_REGS_RAX); @@ -832,6 +833,19 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) trace_kvm_xen_hypercall(input, params[0], params[1], params[2], params[3], params[4], params[5]); + switch (input) { + case __HYPERVISOR_event_channel_op: + if (params[0] == EVTCHNOP_send) + handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r); + break; + + default: + break; + } + + if (handled) + return kvm_xen_hypercall_set_result(vcpu, r); + vcpu->run->exit_reason = KVM_EXIT_XEN; vcpu->run->xen.type = KVM_EXIT_XEN_HCALL; vcpu->run->xen.u.hcall.longmode = longmode; @@ -1118,6 +1132,234 @@ int kvm_xen_hvm_evtchn_send(struct kvm *kvm, struct kvm_irq_routing_xen_evtchn * return ret; } +/* + * Support for *outbound* event channel events via the EVTCHNOP_send hypercall. + */ +struct evtchnfd { + u32 send_port; + u32 type; + union { + struct kvm_xen_evtchn port; + struct { + u32 port; /* zero */ + struct eventfd_ctx *ctx; + } eventfd; + } deliver; +}; + +/* + * Update target vCPU or priority for a registered sending channel. + */ +static int kvm_xen_eventfd_update(struct kvm *kvm, + struct kvm_xen_hvm_attr *data) +{ + u32 port = data->u.evtchn.send_port; + struct evtchnfd *evtchnfd; + + if (!port || port >= max_evtchn_port(kvm)) + return -EINVAL; + + mutex_lock(&kvm->lock); + evtchnfd = idr_find(&kvm->arch.xen.evtchn_ports, port); + mutex_unlock(&kvm->lock); + + if (!evtchnfd) + return -ENOENT; + + /* For an UPDATE, nothing may change except the priority/vcpu */ + if (evtchnfd->type != data->u.evtchn.type) + return -EINVAL; + + /* + * Port cannot change, and if it's zero that was an eventfd + * which can't be changed either. + */ + if (!evtchnfd->deliver.port.port || + evtchnfd->deliver.port.port != data->u.evtchn.deliver.port.port) + return -EINVAL; + + /* We only support 2 level event channels for now */ + if (data->u.evtchn.deliver.port.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) + return -EINVAL; + + mutex_lock(&kvm->lock); + evtchnfd->deliver.port.priority = data->u.evtchn.deliver.port.priority; + if (evtchnfd->deliver.port.vcpu_id != data->u.evtchn.deliver.port.vcpu) { + evtchnfd->deliver.port.vcpu_id = data->u.evtchn.deliver.port.vcpu; + evtchnfd->deliver.port.vcpu_idx = -1; + } + mutex_unlock(&kvm->lock); + return 0; +} + +/* + * Configure the target (eventfd or local port delivery) for sending on + * a given event channel. + */ +static int kvm_xen_eventfd_assign(struct kvm *kvm, + struct kvm_xen_hvm_attr *data) +{ + u32 port = data->u.evtchn.send_port; + struct eventfd_ctx *eventfd = NULL; + struct evtchnfd *evtchnfd = NULL; + int ret = -EINVAL; + + if (!port || port >= max_evtchn_port(kvm)) + return -EINVAL; + + evtchnfd = kzalloc(sizeof(struct evtchnfd), GFP_KERNEL); + if (!evtchnfd) + return -ENOMEM; + + switch(data->u.evtchn.type) { + case EVTCHNSTAT_ipi: + /* IPI must map back to the same port# */ + if (data->u.evtchn.deliver.port.port != data->u.evtchn.send_port) + goto out; /* -EINVAL */ + break; + + case EVTCHNSTAT_interdomain: + if (data->u.evtchn.deliver.port.port) { + if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm)) + goto out; /* -EINVAL */ + } else { + eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd); + if (IS_ERR(eventfd)) { + ret = PTR_ERR(eventfd); + goto out; + } + } + break; + + case EVTCHNSTAT_virq: + case EVTCHNSTAT_closed: + case EVTCHNSTAT_unbound: + case EVTCHNSTAT_pirq: + default: /* Unknown event channel type */ + goto out; /* -EINVAL */ + } + + evtchnfd->send_port = data->u.evtchn.send_port; + evtchnfd->type = data->u.evtchn.type; + if (eventfd) { + evtchnfd->deliver.eventfd.ctx = eventfd; + } else { + /* We only support 2 level event channels for now */ + if (data->u.evtchn.deliver.port.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) + goto out; /* -EINVAL; */ + + evtchnfd->deliver.port.port = data->u.evtchn.deliver.port.port; + evtchnfd->deliver.port.vcpu_id = data->u.evtchn.deliver.port.vcpu; + evtchnfd->deliver.port.vcpu_idx = -1; + evtchnfd->deliver.port.priority = data->u.evtchn.deliver.port.priority; + } + + mutex_lock(&kvm->lock); + ret = idr_alloc(&kvm->arch.xen.evtchn_ports, evtchnfd, port, port + 1, + GFP_KERNEL); + mutex_unlock(&kvm->lock); + if (ret >= 0) + return 0; + + if (ret == -ENOSPC) + ret = -EEXIST; +out: + if (eventfd) + eventfd_ctx_put(eventfd); + kfree(evtchnfd); + return ret; +} + +static int kvm_xen_eventfd_deassign(struct kvm *kvm, u32 port) +{ + struct evtchnfd *evtchnfd; + + mutex_lock(&kvm->lock); + evtchnfd = idr_remove(&kvm->arch.xen.evtchn_ports, port); + mutex_unlock(&kvm->lock); + + if (!evtchnfd) + return -ENOENT; + + if (kvm) + synchronize_srcu(&kvm->srcu); + if (!evtchnfd->deliver.port.port) + eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx); + kfree(evtchnfd); + return 0; +} + +static int kvm_xen_eventfd_reset(struct kvm *kvm) +{ + struct evtchnfd *evtchnfd; + int i; + + mutex_lock(&kvm->lock); + idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { + idr_remove(&kvm->arch.xen.evtchn_ports, evtchnfd->send_port); + synchronize_srcu(&kvm->srcu); + if (!evtchnfd->deliver.port.port) + eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx); + kfree(evtchnfd); + } + mutex_unlock(&kvm->lock); + + return 0; +} + +static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data) +{ + u32 port = data->u.evtchn.send_port; + + if (data->u.evtchn.flags == KVM_XEN_EVTCHN_RESET) + return kvm_xen_eventfd_reset(kvm); + + if (!port || port >= max_evtchn_port(kvm)) + return -EINVAL; + + if (data->u.evtchn.flags == KVM_XEN_EVTCHN_DEASSIGN) + return kvm_xen_eventfd_deassign(kvm, port); + if (data->u.evtchn.flags == KVM_XEN_EVTCHN_UPDATE) + return kvm_xen_eventfd_update(kvm, data); + if (data->u.evtchn.flags) + return -EINVAL; + + return kvm_xen_eventfd_assign(kvm, data); +} + +static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r) +{ + struct evtchnfd *evtchnfd; + struct evtchn_send send; + gpa_t gpa; + int idx; + + idx = srcu_read_lock(&vcpu->kvm->srcu); + gpa = kvm_mmu_gva_to_gpa_system(vcpu, param, NULL); + srcu_read_unlock(&vcpu->kvm->srcu, idx); + + if (!gpa || kvm_vcpu_read_guest(vcpu, gpa, &send, sizeof(send))) { + *r = -EFAULT; + return true; + } + + /* The evtchn_ports idr is protected by vcpu->kvm->srcu */ + evtchnfd = idr_find(&vcpu->kvm->arch.xen.evtchn_ports, send.port); + if (!evtchnfd) + return false; + + if (evtchnfd->deliver.port.port) { + int ret = kvm_xen_set_evtchn(&evtchnfd->deliver.port, vcpu->kvm); + if (ret < 0 && ret != -ENOTCONN) + return false; + } else { + eventfd_signal(evtchnfd->deliver.eventfd.ctx, 1); + } + + *r = 0; + return true; +} + void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) { kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, @@ -1127,3 +1369,26 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.xen.vcpu_time_info_cache); } + +void kvm_xen_init_vm(struct kvm *kvm) +{ + idr_init(&kvm->arch.xen.evtchn_ports); +} + +void kvm_xen_destroy_vm(struct kvm *kvm) +{ + struct evtchnfd *evtchnfd; + int i; + + kvm_gfn_to_pfn_cache_destroy(kvm, &kvm->arch.xen.shinfo_cache); + + idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { + if (!evtchnfd->deliver.port.port) + eventfd_ctx_put(evtchnfd->deliver.eventfd.ctx); + kfree(evtchnfd); + } + idr_destroy(&kvm->arch.xen.evtchn_ports); + + if (kvm->arch.xen_hvm_config.msr) + static_branch_slow_dec_deferred(&kvm_xen_enabled); +} diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 49cd2e9e0f6a..623ed2cb228f 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1686,6 +1686,32 @@ struct kvm_xen_hvm_attr { struct { __u64 gfn; } shared_info; + struct { + __u32 send_port; + __u32 type; /* EVTCHNSTAT_ipi / EVTCHNSTAT_interdomain */ + __u32 flags; +#define KVM_XEN_EVTCHN_DEASSIGN (1 << 0) +#define KVM_XEN_EVTCHN_UPDATE (1 << 1) +#define KVM_XEN_EVTCHN_RESET (1 << 2) + /* + * Events sent by the guest are either looped back to + * the guest itself (potentially on a different port#) + * or signalled via an eventfd. + */ + union { + struct { + __u32 port; + __u32 vcpu; + __u32 priority; + } port; + struct { + __u32 port; /* Zero for eventfd */ + __s32 fd; + } eventfd; + __u32 padding[4]; + } deliver; + } evtchn; + __u64 pad[8]; } u; }; @@ -1694,6 +1720,8 @@ struct kvm_xen_hvm_attr { #define KVM_XEN_ATTR_TYPE_LONG_MODE 0x0 #define KVM_XEN_ATTR_TYPE_SHARED_INFO 0x1 #define KVM_XEN_ATTR_TYPE_UPCALL_VECTOR 0x2 +/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ +#define KVM_XEN_ATTR_TYPE_EVTCHN 0x3 /* Per-vCPU Xen attributes */ #define KVM_XEN_VCPU_GET_ATTR _IOWR(KVMIO, 0xca, struct kvm_xen_vcpu_attr) From 0ec6c5c5bb6585320a2e51a316c2fd381fdec836 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Thu, 3 Mar 2022 15:41:20 +0000 Subject: [PATCH 013/319] KVM: x86/xen: handle PV IPI vcpu yield Cooperative Linux guests after an IPI-many may yield vcpu if any of the IPI'd vcpus were preempted (i.e. runstate is 'runnable'.) Support SCHEDOP_yield for handling yield. Signed-off-by: Joao Martins Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-11-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 3d95167028ba..0dae583c4a1d 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "trace.h" @@ -798,6 +799,20 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result); } +static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, int cmd, u64 param, u64 *r) +{ + switch (cmd) { + case SCHEDOP_yield: + kvm_vcpu_on_spin(vcpu, true); + *r = 0; + return true; + default: + break; + } + + return false; +} + int kvm_xen_hypercall(struct kvm_vcpu *vcpu) { bool longmode; @@ -838,7 +853,9 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) if (params[0] == EVTCHNOP_send) handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r); break; - + case __HYPERVISOR_sched_op: + handled = kvm_xen_hcall_sched_op(vcpu, params[0], params[1], &r); + break; default: break; } From 942c2490c23f2800ad8143f5eb84a79b859aa743 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:21 +0000 Subject: [PATCH 014/319] KVM: x86/xen: Add KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID In order to intercept hypercalls such as VCPUOP_set_singleshot_timer, we need to be aware of the Xen CPU numbering. This looks a lot like the Hyper-V handling of vpidx, for obvious reasons. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-12-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/x86.c | 1 + arch/x86/kvm/xen.c | 19 +++++++++++++++++++ arch/x86/kvm/xen.h | 5 +++++ include/uapi/linux/kvm.h | 3 +++ 5 files changed, 29 insertions(+) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b20f7d99d702..9e3408542b41 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -613,6 +613,7 @@ struct kvm_vcpu_xen { u64 runstate_entry_time; u64 runstate_times[4]; unsigned long evtchn_pending_sel; + u32 vcpu_id; /* The Xen / ACPI vCPU ID */ }; struct kvm_vcpu_arch { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e6bc0b654bbb..3d7b65f33bd8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11262,6 +11262,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.arch_capabilities = kvm_get_arch_capabilities(); vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT; + kvm_xen_init_vcpu(vcpu); kvm_vcpu_mtrr_init(vcpu); vcpu_load(vcpu); kvm_set_tsc_khz(vcpu, max_tsc_khz); diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 0dae583c4a1d..244cf3cd858a 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -603,6 +603,15 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) r = 0; break; + case KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID: + if (data->u.vcpu_id >= KVM_MAX_VCPUS) + r = -EINVAL; + else { + vcpu->arch.xen.vcpu_id = data->u.vcpu_id; + r = 0; + } + break; + default: break; } @@ -678,6 +687,11 @@ int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) r = -EINVAL; break; + case KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID: + data->u.vcpu_id = vcpu->arch.xen.vcpu_id; + r = 0; + break; + default: break; } @@ -1377,6 +1391,11 @@ static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r) return true; } +void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu) +{ + vcpu->arch.xen.vcpu_id = vcpu->vcpu_idx; +} + void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) { kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index 852286de574e..54d587aae85b 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -25,6 +25,7 @@ int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data); int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc); void kvm_xen_init_vm(struct kvm *kvm); void kvm_xen_destroy_vm(struct kvm *kvm); +void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu); void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu); int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm); @@ -75,6 +76,10 @@ static inline void kvm_xen_destroy_vm(struct kvm *kvm) { } +static inline void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu) +{ +} + static inline void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) { } diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 623ed2cb228f..4b65e9f0a4d9 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1747,6 +1747,7 @@ struct kvm_xen_vcpu_attr { __u64 time_blocked; __u64 time_offline; } runstate; + __u32 vcpu_id; } u; }; @@ -1757,6 +1758,8 @@ struct kvm_xen_vcpu_attr { #define KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_CURRENT 0x3 #define KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_DATA 0x4 #define KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST 0x5 +/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ +#define KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID 0x6 /* Secure Encrypted Virtualization command */ enum sev_cmd_id { From 536395260582be7443b0b35b0bbb89ffe3947f62 Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Thu, 3 Mar 2022 15:41:22 +0000 Subject: [PATCH 015/319] KVM: x86/xen: handle PV timers oneshot mode If the guest has offloaded the timer virq, handle the following hypercalls for programming the timer: VCPUOP_set_singleshot_timer VCPUOP_stop_singleshot_timer set_timer_op(timestamp_ns) The event channel corresponding to the timer virq is then used to inject events once timer deadlines are met. For now we back the PV timer with hrtimer. [ dwmw2: Add save/restore, 32-bit compat mode, immediate delivery, don't check timer in kvm_vcpu_has_event() ] Signed-off-by: Joao Martins Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-13-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 4 + arch/x86/kvm/irq.c | 12 +- arch/x86/kvm/xen.c | 211 ++++++++++++++++++++++++++++++++ arch/x86/kvm/xen.h | 28 +++++ include/uapi/linux/kvm.h | 6 + 5 files changed, 258 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 9e3408542b41..8193d5285544 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -614,6 +614,10 @@ struct kvm_vcpu_xen { u64 runstate_times[4]; unsigned long evtchn_pending_sel; u32 vcpu_id; /* The Xen / ACPI vCPU ID */ + u32 timer_virq; + u64 timer_expires; /* In guest epoch */ + atomic_t timer_pending; + struct hrtimer timer; }; struct kvm_vcpu_arch { diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 172b05343cfd..f371f1292ca3 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -22,10 +22,14 @@ */ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) { - if (lapic_in_kernel(vcpu)) - return apic_has_pending_timer(vcpu); + int r = 0; - return 0; + if (lapic_in_kernel(vcpu)) + r = apic_has_pending_timer(vcpu); + if (kvm_xen_timer_enabled(vcpu)) + r += kvm_xen_has_pending_timer(vcpu); + + return r; } EXPORT_SYMBOL(kvm_cpu_has_pending_timer); @@ -143,6 +147,8 @@ void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu) { if (lapic_in_kernel(vcpu)) kvm_inject_apic_timer_irqs(vcpu); + if (kvm_xen_timer_enabled(vcpu)) + kvm_xen_inject_timer_irqs(vcpu); } EXPORT_SYMBOL_GPL(kvm_inject_pending_timer_irqs); diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 244cf3cd858a..0e1595dc8c5f 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -23,6 +23,7 @@ #include "trace.h" +static int kvm_xen_set_evtchn(struct kvm_xen_evtchn *xe, struct kvm *kvm); static int kvm_xen_setattr_evtchn(struct kvm *kvm, struct kvm_xen_hvm_attr *data); static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r); @@ -108,6 +109,66 @@ static int kvm_xen_shared_info_init(struct kvm *kvm, gfn_t gfn) return ret; } +void kvm_xen_inject_timer_irqs(struct kvm_vcpu *vcpu) +{ + if (atomic_read(&vcpu->arch.xen.timer_pending) > 0) { + struct kvm_xen_evtchn e; + + e.vcpu_id = vcpu->vcpu_id; + e.vcpu_idx = vcpu->vcpu_idx; + e.port = vcpu->arch.xen.timer_virq; + e.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL; + + kvm_xen_set_evtchn(&e, vcpu->kvm); + + vcpu->arch.xen.timer_expires = 0; + atomic_set(&vcpu->arch.xen.timer_pending, 0); + } +} + +static enum hrtimer_restart xen_timer_callback(struct hrtimer *timer) +{ + struct kvm_vcpu *vcpu = container_of(timer, struct kvm_vcpu, + arch.xen.timer); + if (atomic_read(&vcpu->arch.xen.timer_pending)) + return HRTIMER_NORESTART; + + atomic_inc(&vcpu->arch.xen.timer_pending); + kvm_make_request(KVM_REQ_UNBLOCK, vcpu); + kvm_vcpu_kick(vcpu); + + return HRTIMER_NORESTART; +} + +static void kvm_xen_start_timer(struct kvm_vcpu *vcpu, u64 guest_abs, s64 delta_ns) +{ + atomic_set(&vcpu->arch.xen.timer_pending, 0); + vcpu->arch.xen.timer_expires = guest_abs; + + if (delta_ns <= 0) { + xen_timer_callback(&vcpu->arch.xen.timer); + } else { + ktime_t ktime_now = ktime_get(); + hrtimer_start(&vcpu->arch.xen.timer, + ktime_add_ns(ktime_now, delta_ns), + HRTIMER_MODE_ABS_HARD); + } +} + +static void kvm_xen_stop_timer(struct kvm_vcpu *vcpu) +{ + hrtimer_cancel(&vcpu->arch.xen.timer); + vcpu->arch.xen.timer_expires = 0; + atomic_set(&vcpu->arch.xen.timer_pending, 0); +} + +static void kvm_xen_init_timer(struct kvm_vcpu *vcpu) +{ + hrtimer_init(&vcpu->arch.xen.timer, CLOCK_MONOTONIC, + HRTIMER_MODE_ABS_HARD); + vcpu->arch.xen.timer.function = xen_timer_callback; +} + static void kvm_xen_update_runstate(struct kvm_vcpu *v, int state) { struct kvm_vcpu_xen *vx = &v->arch.xen; @@ -612,6 +673,28 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) } break; + case KVM_XEN_VCPU_ATTR_TYPE_TIMER: + if (data->u.timer.port) { + if (data->u.timer.priority != KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL) { + r = -EINVAL; + break; + } + vcpu->arch.xen.timer_virq = data->u.timer.port; + kvm_xen_init_timer(vcpu); + + /* Restart the timer if it's set */ + if (data->u.timer.expires_ns) + kvm_xen_start_timer(vcpu, data->u.timer.expires_ns, + data->u.timer.expires_ns - + get_kvmclock_ns(vcpu->kvm)); + } else if (kvm_xen_timer_enabled(vcpu)) { + kvm_xen_stop_timer(vcpu); + vcpu->arch.xen.timer_virq = 0; + } + + r = 0; + break; + default: break; } @@ -692,6 +775,13 @@ int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) r = 0; break; + case KVM_XEN_VCPU_ATTR_TYPE_TIMER: + data->u.timer.port = vcpu->arch.xen.timer_virq; + data->u.timer.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL; + data->u.timer.expires_ns = vcpu->arch.xen.timer_expires; + r = 0; + break; + default: break; } @@ -827,6 +917,112 @@ static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, int cmd, u64 param, u6 return false; } +struct compat_vcpu_set_singleshot_timer { + uint64_t timeout_abs_ns; + uint32_t flags; +} __attribute__((packed)); + +static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, + int vcpu_id, u64 param, u64 *r) +{ + struct vcpu_set_singleshot_timer oneshot; + s64 delta; + gpa_t gpa; + int idx; + + if (!kvm_xen_timer_enabled(vcpu)) + return false; + + switch (cmd) { + case VCPUOP_set_singleshot_timer: + if (vcpu->arch.xen.vcpu_id != vcpu_id) { + *r = -EINVAL; + return true; + } + idx = srcu_read_lock(&vcpu->kvm->srcu); + gpa = kvm_mmu_gva_to_gpa_system(vcpu, param, NULL); + srcu_read_unlock(&vcpu->kvm->srcu, idx); + + /* + * The only difference for 32-bit compat is the 4 bytes of + * padding after the interesting part of the structure. So + * for a faithful emulation of Xen we have to *try* to copy + * the padding and return -EFAULT if we can't. Otherwise we + * might as well just have copied the 12-byte 32-bit struct. + */ + BUILD_BUG_ON(offsetof(struct compat_vcpu_set_singleshot_timer, timeout_abs_ns) != + offsetof(struct vcpu_set_singleshot_timer, timeout_abs_ns)); + BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, timeout_abs_ns) != + sizeof_field(struct vcpu_set_singleshot_timer, timeout_abs_ns)); + BUILD_BUG_ON(offsetof(struct compat_vcpu_set_singleshot_timer, flags) != + offsetof(struct vcpu_set_singleshot_timer, flags)); + BUILD_BUG_ON(sizeof_field(struct compat_vcpu_set_singleshot_timer, flags) != + sizeof_field(struct vcpu_set_singleshot_timer, flags)); + + if (!gpa || + kvm_vcpu_read_guest(vcpu, gpa, &oneshot, longmode ? sizeof(oneshot) : + sizeof(struct compat_vcpu_set_singleshot_timer))) { + *r = -EFAULT; + return true; + } + + delta = oneshot.timeout_abs_ns - get_kvmclock_ns(vcpu->kvm); + if ((oneshot.flags & VCPU_SSHOTTMR_future) && delta < 0) { + *r = -ETIME; + return true; + } + + kvm_xen_start_timer(vcpu, oneshot.timeout_abs_ns, delta); + *r = 0; + return true; + + case VCPUOP_stop_singleshot_timer: + if (vcpu->arch.xen.vcpu_id != vcpu_id) { + *r = -EINVAL; + return true; + } + kvm_xen_stop_timer(vcpu); + *r = 0; + return true; + } + + return false; +} + +static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout, + u64 *r) +{ + if (!kvm_xen_timer_enabled(vcpu)) + return false; + + if (timeout) { + uint64_t guest_now = get_kvmclock_ns(vcpu->kvm); + int64_t delta = timeout - guest_now; + + /* Xen has a 'Linux workaround' in do_set_timer_op() which + * checks for negative absolute timeout values (caused by + * integer overflow), and for values about 13 days in the + * future (2^50ns) which would be caused by jiffies + * overflow. For those cases, it sets the timeout 100ms in + * the future (not *too* soon, since if a guest really did + * set a long timeout on purpose we don't want to keep + * churning CPU time by waking it up). + */ + if (unlikely((int64_t)timeout < 0 || + (delta > 0 && (uint32_t) (delta >> 50) != 0))) { + delta = 100 * NSEC_PER_MSEC; + timeout = guest_now + delta; + } + + kvm_xen_start_timer(vcpu, timeout, delta); + } else { + kvm_xen_stop_timer(vcpu); + } + + *r = 0; + return true; +} + int kvm_xen_hypercall(struct kvm_vcpu *vcpu) { bool longmode; @@ -870,6 +1066,18 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) case __HYPERVISOR_sched_op: handled = kvm_xen_hcall_sched_op(vcpu, params[0], params[1], &r); break; + case __HYPERVISOR_vcpu_op: + handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1], + params[2], &r); + break; + case __HYPERVISOR_set_timer_op: { + u64 timeout = params[0]; + /* In 32-bit mode, the 64-bit timeout is in two 32-bit params. */ + if (!longmode) + timeout |= params[1] << 32; + handled = kvm_xen_hcall_set_timer_op(vcpu, timeout, &r); + break; + } default: break; } @@ -1398,6 +1606,9 @@ void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu) void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) { + if (kvm_xen_timer_enabled(vcpu)) + kvm_xen_stop_timer(vcpu); + kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.xen.runstate_cache); kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, diff --git a/arch/x86/kvm/xen.h b/arch/x86/kvm/xen.h index 54d587aae85b..ee5c4ae0755c 100644 --- a/arch/x86/kvm/xen.h +++ b/arch/x86/kvm/xen.h @@ -62,6 +62,20 @@ static inline bool kvm_xen_has_pending_events(struct kvm_vcpu *vcpu) vcpu->arch.xen.evtchn_pending_sel; } +static inline bool kvm_xen_timer_enabled(struct kvm_vcpu *vcpu) +{ + return !!vcpu->arch.xen.timer_virq; +} + +static inline int kvm_xen_has_pending_timer(struct kvm_vcpu *vcpu) +{ + if (kvm_xen_hypercall_enabled(vcpu->kvm) && kvm_xen_timer_enabled(vcpu)) + return atomic_read(&vcpu->arch.xen.timer_pending); + + return 0; +} + +void kvm_xen_inject_timer_irqs(struct kvm_vcpu *vcpu); #else static inline int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data) { @@ -107,6 +121,20 @@ static inline bool kvm_xen_has_pending_events(struct kvm_vcpu *vcpu) { return false; } + +static inline int kvm_xen_has_pending_timer(struct kvm_vcpu *vcpu) +{ + return 0; +} + +static inline void kvm_xen_inject_timer_irqs(struct kvm_vcpu *vcpu) +{ +} + +static inline bool kvm_xen_timer_enabled(struct kvm_vcpu *vcpu) +{ + return false; +} #endif int kvm_xen_hypercall(struct kvm_vcpu *vcpu); diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 4b65e9f0a4d9..cb223e425223 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1748,6 +1748,11 @@ struct kvm_xen_vcpu_attr { __u64 time_offline; } runstate; __u32 vcpu_id; + struct { + __u32 port; + __u32 priority; + __u64 expires_ns; + } timer; } u; }; @@ -1760,6 +1765,7 @@ struct kvm_xen_vcpu_attr { #define KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST 0x5 /* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ #define KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID 0x6 +#define KVM_XEN_VCPU_ATTR_TYPE_TIMER 0x7 /* Secure Encrypted Virtualization command */ enum sev_cmd_id { From 28d1629f751c4a5f9437fbaa0ee4ed81d1a8e587 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:23 +0000 Subject: [PATCH 016/319] KVM: x86/xen: Kernel acceleration for XENVER_version Turns out this is a fast path for PV guests because they use it to trigger the event channel upcall. So letting it bounce all the way up to userspace is not great. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-14-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/xen.c | 19 +++++++++++++++++++ include/uapi/linux/kvm.h | 3 ++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 8193d5285544..f3fba9d8ddc6 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1026,6 +1026,7 @@ struct msr_bitmap_range { /* Xen emulation context */ struct kvm_xen { + u32 xen_version; bool long_mode; u8 upcall_vector; struct gfn_to_pfn_cache shinfo_cache; diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 0e1595dc8c5f..98438f27f6b3 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -465,6 +466,13 @@ int kvm_xen_hvm_set_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data) r = kvm_xen_setattr_evtchn(kvm, data); break; + case KVM_XEN_ATTR_TYPE_XEN_VERSION: + mutex_lock(&kvm->lock); + kvm->arch.xen.xen_version = data->u.xen_version; + mutex_unlock(&kvm->lock); + r = 0; + break; + default: break; } @@ -497,6 +505,11 @@ int kvm_xen_hvm_get_attr(struct kvm *kvm, struct kvm_xen_hvm_attr *data) r = 0; break; + case KVM_XEN_ATTR_TYPE_XEN_VERSION: + data->u.xen_version = kvm->arch.xen.xen_version; + r = 0; + break; + default: break; } @@ -1059,6 +1072,12 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) params[3], params[4], params[5]); switch (input) { + case __HYPERVISOR_xen_version: + if (params[0] == XENVER_version && vcpu->kvm->arch.xen.xen_version) { + r = vcpu->kvm->arch.xen.xen_version; + handled = true; + } + break; case __HYPERVISOR_event_channel_op: if (params[0] == EVTCHNOP_send) handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r); diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index cb223e425223..4dda3896ed71 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1711,7 +1711,7 @@ struct kvm_xen_hvm_attr { __u32 padding[4]; } deliver; } evtchn; - + __u32 xen_version; __u64 pad[8]; } u; }; @@ -1722,6 +1722,7 @@ struct kvm_xen_hvm_attr { #define KVM_XEN_ATTR_TYPE_UPCALL_VECTOR 0x2 /* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ #define KVM_XEN_ATTR_TYPE_EVTCHN 0x3 +#define KVM_XEN_ATTR_TYPE_XEN_VERSION 0x4 /* Per-vCPU Xen attributes */ #define KVM_XEN_VCPU_GET_ATTR _IOWR(KVMIO, 0xca, struct kvm_xen_vcpu_attr) From fde0451be8fb3208d4d146b8602d99ee8139e515 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:24 +0000 Subject: [PATCH 017/319] KVM: x86/xen: Support per-vCPU event channel upcall via local APIC Windows uses a per-vCPU vector, and it's delivered via the local APIC basically like an MSI (with associated EOI) unlike the traditional guest-wide vector which is just magically asserted by Xen (and in the KVM case by kvm_xen_has_interrupt() / kvm_cpu_get_extint()). Now that the kernel is able to raise event channel events for itself, being able to do so for Windows guests is also going to be useful. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-15-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/xen.c | 40 +++++++++++++++++++++++++++++++++ include/uapi/linux/kvm.h | 2 ++ 3 files changed, 43 insertions(+) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f3fba9d8ddc6..998caf7a3ce9 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -606,6 +606,7 @@ struct kvm_vcpu_hv { struct kvm_vcpu_xen { u64 hypercall_rip; u32 current_runstate; + u8 upcall_vector; struct gfn_to_pfn_cache vcpu_info_cache; struct gfn_to_pfn_cache vcpu_time_info_cache; struct gfn_to_pfn_cache runstate_cache; diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 98438f27f6b3..5afaf7b59944 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -314,6 +314,22 @@ void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state) mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); } +static void kvm_xen_inject_vcpu_vector(struct kvm_vcpu *v) +{ + struct kvm_lapic_irq irq = { }; + int r; + + irq.dest_id = v->vcpu_id; + irq.vector = v->arch.xen.upcall_vector; + irq.dest_mode = APIC_DEST_PHYSICAL; + irq.shorthand = APIC_DEST_NOSHORT; + irq.delivery_mode = APIC_DM_FIXED; + irq.level = 1; + + /* The fast version will always work for physical unicast */ + WARN_ON_ONCE(!kvm_irq_delivery_to_apic_fast(v->kvm, NULL, &irq, &r, NULL)); +} + /* * On event channel delivery, the vcpu_info may not have been accessible. * In that case, there are bits in vcpu->arch.xen.evtchn_pending_sel which @@ -374,6 +390,10 @@ void kvm_xen_inject_pending_events(struct kvm_vcpu *v) } read_unlock_irqrestore(&gpc->lock, flags); + /* For the per-vCPU lapic vector, deliver it as MSI. */ + if (v->arch.xen.upcall_vector) + kvm_xen_inject_vcpu_vector(v); + mark_page_dirty_in_slot(v->kvm, gpc->memslot, gpc->gpa >> PAGE_SHIFT); } @@ -708,6 +728,15 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) r = 0; break; + case KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR: + if (data->u.vector && data->u.vector < 0x10) + r = -EINVAL; + else { + vcpu->arch.xen.upcall_vector = data->u.vector; + r = 0; + } + break; + default: break; } @@ -795,6 +824,11 @@ int kvm_xen_vcpu_get_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) r = 0; break; + case KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR: + data->u.vector = vcpu->arch.xen.upcall_vector; + r = 0; + break; + default: break; } @@ -1228,6 +1262,12 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) kick_vcpu = true; } } + + /* For the per-vCPU lapic vector, deliver it as MSI. */ + if (kick_vcpu && vcpu->arch.xen.upcall_vector) { + kvm_xen_inject_vcpu_vector(vcpu); + kick_vcpu = false; + } } out_rcu: diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 4dda3896ed71..a9ba690c4f37 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1754,6 +1754,7 @@ struct kvm_xen_vcpu_attr { __u32 priority; __u64 expires_ns; } timer; + __u8 vector; } u; }; @@ -1767,6 +1768,7 @@ struct kvm_xen_vcpu_attr { /* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ #define KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID 0x6 #define KVM_XEN_VCPU_ATTR_TYPE_TIMER 0x7 +#define KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR 0x8 /* Secure Encrypted Virtualization command */ enum sev_cmd_id { From 661a20fab7d156cf6b9a407c946a1e558a633151 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:25 +0000 Subject: [PATCH 018/319] KVM: x86/xen: Advertise and document KVM_XEN_HVM_CONFIG_EVTCHN_SEND MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the end of the patch series adding this batch of event channel acceleration features, finally add the feature bit which advertises them and document it all. For SCHEDOP_poll we need to wake a polling vCPU when a given port is triggered, even when it's masked — and we want to implement that in the kernel, for efficiency. So we want the kernel to know that it has sole ownership of event channel delivery. Thus, we allow userspace to make the 'promise' by setting the corresponding feature bit in its KVM_XEN_HVM_CONFIG call. As we implement SCHEDOP_poll bypass later, we will do so only if that promise has been made by userspace. Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-16-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 129 ++++++++++++++++++++++++++++++--- arch/x86/kvm/x86.c | 3 +- arch/x86/kvm/xen.c | 6 +- include/uapi/linux/kvm.h | 1 + 4 files changed, 127 insertions(+), 12 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 700df350332b..fdfd9e1bc7f7 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -982,12 +982,22 @@ memory. __u8 pad2[30]; }; -If the KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL flag is returned from the -KVM_CAP_XEN_HVM check, it may be set in the flags field of this ioctl. -This requests KVM to generate the contents of the hypercall page -automatically; hypercalls will be intercepted and passed to userspace -through KVM_EXIT_XEN. In this case, all of the blob size and address -fields must be zero. +If certain flags are returned from the KVM_CAP_XEN_HVM check, they may +be set in the flags field of this ioctl: + +The KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL flag requests KVM to generate +the contents of the hypercall page automatically; hypercalls will be +intercepted and passed to userspace through KVM_EXIT_XEN. In this +ase, all of the blob size and address fields must be zero. + +The KVM_XEN_HVM_CONFIG_EVTCHN_SEND flag indicates to KVM that userspace +will always use the KVM_XEN_HVM_EVTCHN_SEND ioctl to deliver event +channel interrupts rather than manipulating the guest's shared_info +structures directly. This, in turn, may allow KVM to enable features +such as intercepting the SCHEDOP_poll hypercall to accelerate PV +spinlock operation for the guest. Userspace may still use the ioctl +to deliver events if it was advertised, even if userspace does not +send this indication that it will always do so No other flags are currently valid in the struct kvm_xen_hvm_config. @@ -5216,7 +5226,25 @@ have deterministic behavior. struct { __u64 gfn; } shared_info; - __u64 pad[4]; + struct { + __u32 send_port; + __u32 type; /* EVTCHNSTAT_ipi / EVTCHNSTAT_interdomain */ + __u32 flags; + union { + struct { + __u32 port; + __u32 vcpu; + __u32 priority; + } port; + struct { + __u32 port; /* Zero for eventfd */ + __s32 fd; + } eventfd; + __u32 padding[4]; + } deliver; + } evtchn; + __u32 xen_version; + __u64 pad[8]; } u; }; @@ -5247,6 +5275,30 @@ KVM_XEN_ATTR_TYPE_SHARED_INFO KVM_XEN_ATTR_TYPE_UPCALL_VECTOR Sets the exception vector used to deliver Xen event channel upcalls. + This is the HVM-wide vector injected directly by the hypervisor + (not through the local APIC), typically configured by a guest via + HVM_PARAM_CALLBACK_IRQ. + +KVM_XEN_ATTR_TYPE_EVTCHN + This attribute is available when the KVM_CAP_XEN_HVM ioctl indicates + support for KVM_XEN_HVM_CONFIG_EVTCHN_SEND features. It configures + an outbound port number for interception of EVTCHNOP_send requests + from the guest. A given sending port number may be directed back + to a specified vCPU (by APIC ID) / port / priority on the guest, + or to trigger events on an eventfd. The vCPU and priority can be + changed by setting KVM_XEN_EVTCHN_UPDATE in a subsequent call, + but other fields cannot change for a given sending port. A port + mapping is removed by using KVM_XEN_EVTCHN_DEASSIGN in the flags + field. + +KVM_XEN_ATTR_TYPE_XEN_VERSION + This attribute is available when the KVM_CAP_XEN_HVM ioctl indicates + support for KVM_XEN_HVM_CONFIG_EVTCHN_SEND features. It configures + the 32-bit version code returned to the guest when it invokes the + XENVER_version call; typically (XEN_MAJOR << 16 | XEN_MINOR). PV + Xen guests will often use this to as a dummy hypercall to trigger + event channel delivery, so responding within the kernel without + exiting to userspace is beneficial. 4.127 KVM_XEN_HVM_GET_ATTR -------------------------- @@ -5258,7 +5310,8 @@ KVM_XEN_ATTR_TYPE_UPCALL_VECTOR :Returns: 0 on success, < 0 on error Allows Xen VM attributes to be read. For the structure and types, -see KVM_XEN_HVM_SET_ATTR above. +see KVM_XEN_HVM_SET_ATTR above. The KVM_XEN_ATTR_TYPE_EVTCHN +attribute cannot be read. 4.128 KVM_XEN_VCPU_SET_ATTR --------------------------- @@ -5285,6 +5338,13 @@ see KVM_XEN_HVM_SET_ATTR above. __u64 time_blocked; __u64 time_offline; } runstate; + __u32 vcpu_id; + struct { + __u32 port; + __u32 priority; + __u64 expires_ns; + } timer; + __u8 vector; } u; }; @@ -5326,6 +5386,27 @@ KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST or RUNSTATE_offline) to set the current accounted state as of the adjusted state_entry_time. +KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID + This attribute is available when the KVM_CAP_XEN_HVM ioctl indicates + support for KVM_XEN_HVM_CONFIG_EVTCHN_SEND features. It sets the Xen + vCPU ID of the given vCPU, to allow timer-related VCPU operations to + be intercepted by KVM. + +KVM_XEN_VCPU_ATTR_TYPE_TIMER + This attribute is available when the KVM_CAP_XEN_HVM ioctl indicates + support for KVM_XEN_HVM_CONFIG_EVTCHN_SEND features. It sets the + event channel port/priority for the VIRQ_TIMER of the vCPU, as well + as allowing a pending timer to be saved/restored. + +KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR + This attribute is available when the KVM_CAP_XEN_HVM ioctl indicates + support for KVM_XEN_HVM_CONFIG_EVTCHN_SEND features. It sets the + per-vCPU local APIC upcall vector, configured by a Xen guest with + the HVMOP_set_evtchn_upcall_vector hypercall. This is typically + used by Windows guests, and is distinct from the HVM-wide upcall + vector configured with HVM_PARAM_CALLBACK_IRQ. + + 4.129 KVM_XEN_VCPU_GET_ATTR --------------------------- @@ -5645,6 +5726,25 @@ enabled with ``arch_prctl()``, but this may change in the future. The offsets of the state save areas in struct kvm_xsave follow the contents of CPUID leaf 0xD on the host. +4.135 KVM_XEN_HVM_EVTCHN_SEND +----------------------------- + +:Capability: KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND +:Architectures: x86 +:Type: vm ioctl +:Parameters: struct kvm_irq_routing_xen_evtchn +:Returns: 0 on success, < 0 on error + + +:: + + struct kvm_irq_routing_xen_evtchn { + __u32 port; + __u32 vcpu; + __u32 priority; + }; + +This ioctl injects an event channel interrupt directly to the guest vCPU. 5. The kvm_run structure ======================== @@ -7620,8 +7720,9 @@ PVHVM guests. Valid flags are:: #define KVM_XEN_HVM_CONFIG_HYPERCALL_MSR (1 << 0) #define KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL (1 << 1) #define KVM_XEN_HVM_CONFIG_SHARED_INFO (1 << 2) - #define KVM_XEN_HVM_CONFIG_RUNSTATE (1 << 2) - #define KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL (1 << 3) + #define KVM_XEN_HVM_CONFIG_RUNSTATE (1 << 3) + #define KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL (1 << 4) + #define KVM_XEN_HVM_CONFIG_EVTCHN_SEND (1 << 5) The KVM_XEN_HVM_CONFIG_HYPERCALL_MSR flag indicates that the KVM_XEN_HVM_CONFIG ioctl is available, for the guest to set its hypercall page. @@ -7645,6 +7746,14 @@ The KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL flag indicates that IRQ routing entries of the type KVM_IRQ_ROUTING_XEN_EVTCHN are supported, with the priority field set to indicate 2 level event channel delivery. +The KVM_XEN_HVM_CONFIG_EVTCHN_SEND flag indicates that KVM supports +injecting event channel events directly into the guest with the +KVM_XEN_HVM_EVTCHN_SEND ioctl. It also indicates support for the +KVM_XEN_ATTR_TYPE_EVTCHN/XEN_VERSION HVM attributes and the +KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID/TIMER/UPCALL_VECTOR vCPU attributes. +related to event channel delivery, timers, and the XENVER_version +interception. + 8.31 KVM_CAP_PPC_MULTITCE ------------------------- diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3d7b65f33bd8..1a7d1b5f7232 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4276,7 +4276,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = KVM_XEN_HVM_CONFIG_HYPERCALL_MSR | KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | KVM_XEN_HVM_CONFIG_SHARED_INFO | - KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL; + KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL | + KVM_XEN_HVM_CONFIG_EVTCHN_SEND; if (sched_info_on()) r |= KVM_XEN_HVM_CONFIG_RUNSTATE; break; diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 5afaf7b59944..53232c9ff89c 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -909,7 +909,11 @@ int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data) int kvm_xen_hvm_config(struct kvm *kvm, struct kvm_xen_hvm_config *xhc) { - if (xhc->flags & ~KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL) + /* Only some feature flags need to be *enabled* by userspace */ + u32 permitted_flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL | + KVM_XEN_HVM_CONFIG_EVTCHN_SEND; + + if (xhc->flags & ~permitted_flags) return -EINVAL; /* diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index a9ba690c4f37..ee5cc9e2a837 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1232,6 +1232,7 @@ struct kvm_x86_mce { #define KVM_XEN_HVM_CONFIG_SHARED_INFO (1 << 2) #define KVM_XEN_HVM_CONFIG_RUNSTATE (1 << 3) #define KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL (1 << 4) +#define KVM_XEN_HVM_CONFIG_EVTCHN_SEND (1 << 5) struct kvm_xen_hvm_config { __u32 flags; From 1a65105a5aba9f7c6ea68cf687e569d055a94685 Mon Sep 17 00:00:00 2001 From: Boris Ostrovsky Date: Thu, 3 Mar 2022 15:41:26 +0000 Subject: [PATCH 019/319] KVM: x86/xen: handle PV spinlocks slowpath Add support for SCHEDOP_poll hypercall. This implementation is optimized for polling for a single channel, which is what Linux does. Polling for multiple channels is not especially efficient (and has not been tested). PV spinlocks slow path uses this hypercall, and explicitly crash if it's not supported. [ dwmw2: Rework to use kvm_vcpu_halt(), not supported for 32-bit guests ] Signed-off-by: Boris Ostrovsky Signed-off-by: David Woodhouse Signed-off-by: Paolo Bonzini Message-Id: <20220303154127.202856-17-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 + arch/x86/kvm/xen.c | 158 +++++++++++++++++- .../selftests/kvm/x86_64/xen_shinfo_test.c | 6 + 3 files changed, 165 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 998caf7a3ce9..5370744b789c 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -619,6 +619,8 @@ struct kvm_vcpu_xen { u64 timer_expires; /* In guest epoch */ atomic_t timer_pending; struct hrtimer timer; + int poll_evtchn; + struct timer_list poll_timer; }; struct kvm_vcpu_arch { @@ -1032,6 +1034,7 @@ struct kvm_xen { u8 upcall_vector; struct gfn_to_pfn_cache shinfo_cache; struct idr evtchn_ports; + unsigned long poll_mask[BITS_TO_LONGS(KVM_MAX_VCPUS)]; }; enum kvm_irqchip_mode { diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 53232c9ff89c..7e7c8a5bff52 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -10,6 +10,7 @@ #include "xen.h" #include "lapic.h" #include "hyperv.h" +#include "lapic.h" #include #include @@ -954,9 +955,146 @@ static int kvm_xen_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_xen_hypercall_set_result(vcpu, run->xen.u.hcall.result); } -static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, int cmd, u64 param, u64 *r) +static bool wait_pending_event(struct kvm_vcpu *vcpu, int nr_ports, + evtchn_port_t *ports) +{ + struct kvm *kvm = vcpu->kvm; + struct gfn_to_pfn_cache *gpc = &kvm->arch.xen.shinfo_cache; + unsigned long *pending_bits; + unsigned long flags; + bool ret = true; + int idx, i; + + read_lock_irqsave(&gpc->lock, flags); + idx = srcu_read_lock(&kvm->srcu); + if (!kvm_gfn_to_pfn_cache_check(kvm, gpc, gpc->gpa, PAGE_SIZE)) + goto out_rcu; + + ret = false; + if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode) { + struct shared_info *shinfo = gpc->khva; + pending_bits = (unsigned long *)&shinfo->evtchn_pending; + } else { + struct compat_shared_info *shinfo = gpc->khva; + pending_bits = (unsigned long *)&shinfo->evtchn_pending; + } + + for (i = 0; i < nr_ports; i++) { + if (test_bit(ports[i], pending_bits)) { + ret = true; + break; + } + } + + out_rcu: + srcu_read_unlock(&kvm->srcu, idx); + read_unlock_irqrestore(&gpc->lock, flags); + + return ret; +} + +static bool kvm_xen_schedop_poll(struct kvm_vcpu *vcpu, bool longmode, + u64 param, u64 *r) +{ + int idx, i; + struct sched_poll sched_poll; + evtchn_port_t port, *ports; + gpa_t gpa; + + if (!longmode || !lapic_in_kernel(vcpu) || + !(vcpu->kvm->arch.xen_hvm_config.flags & KVM_XEN_HVM_CONFIG_EVTCHN_SEND)) + return false; + + idx = srcu_read_lock(&vcpu->kvm->srcu); + gpa = kvm_mmu_gva_to_gpa_system(vcpu, param, NULL); + srcu_read_unlock(&vcpu->kvm->srcu, idx); + + if (!gpa || kvm_vcpu_read_guest(vcpu, gpa, &sched_poll, + sizeof(sched_poll))) { + *r = -EFAULT; + return true; + } + + if (unlikely(sched_poll.nr_ports > 1)) { + /* Xen (unofficially) limits number of pollers to 128 */ + if (sched_poll.nr_ports > 128) { + *r = -EINVAL; + return true; + } + + ports = kmalloc_array(sched_poll.nr_ports, + sizeof(*ports), GFP_KERNEL); + if (!ports) { + *r = -ENOMEM; + return true; + } + } else + ports = &port; + + for (i = 0; i < sched_poll.nr_ports; i++) { + idx = srcu_read_lock(&vcpu->kvm->srcu); + gpa = kvm_mmu_gva_to_gpa_system(vcpu, + (gva_t)(sched_poll.ports + i), + NULL); + srcu_read_unlock(&vcpu->kvm->srcu, idx); + + if (!gpa || kvm_vcpu_read_guest(vcpu, gpa, + &ports[i], sizeof(port))) { + *r = -EFAULT; + goto out; + } + } + + if (sched_poll.nr_ports == 1) + vcpu->arch.xen.poll_evtchn = port; + else + vcpu->arch.xen.poll_evtchn = -1; + + set_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.xen.poll_mask); + + if (!wait_pending_event(vcpu, sched_poll.nr_ports, ports)) { + vcpu->arch.mp_state = KVM_MP_STATE_HALTED; + + if (sched_poll.timeout) + mod_timer(&vcpu->arch.xen.poll_timer, + jiffies + nsecs_to_jiffies(sched_poll.timeout)); + + kvm_vcpu_halt(vcpu); + + if (sched_poll.timeout) + del_timer(&vcpu->arch.xen.poll_timer); + + vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; + kvm_clear_request(KVM_REQ_UNHALT, vcpu); + } + + vcpu->arch.xen.poll_evtchn = 0; + *r = 0; +out: + /* Really, this is only needed in case of timeout */ + clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.xen.poll_mask); + + if (unlikely(sched_poll.nr_ports > 1)) + kfree(ports); + return true; +} + +static void cancel_evtchn_poll(struct timer_list *t) +{ + struct kvm_vcpu *vcpu = from_timer(vcpu, t, arch.xen.poll_timer); + + kvm_make_request(KVM_REQ_UNBLOCK, vcpu); + kvm_vcpu_kick(vcpu); +} + +static bool kvm_xen_hcall_sched_op(struct kvm_vcpu *vcpu, bool longmode, + int cmd, u64 param, u64 *r) { switch (cmd) { + case SCHEDOP_poll: + if (kvm_xen_schedop_poll(vcpu, longmode, param, r)) + return true; + fallthrough; case SCHEDOP_yield: kvm_vcpu_on_spin(vcpu, true); *r = 0; @@ -1121,7 +1259,8 @@ int kvm_xen_hypercall(struct kvm_vcpu *vcpu) handled = kvm_xen_hcall_evtchn_send(vcpu, params[1], &r); break; case __HYPERVISOR_sched_op: - handled = kvm_xen_hcall_sched_op(vcpu, params[0], params[1], &r); + handled = kvm_xen_hcall_sched_op(vcpu, longmode, params[0], + params[1], &r); break; case __HYPERVISOR_vcpu_op: handled = kvm_xen_hcall_vcpu_op(vcpu, longmode, params[0], params[1], @@ -1168,6 +1307,17 @@ static inline int max_evtchn_port(struct kvm *kvm) return COMPAT_EVTCHN_2L_NR_CHANNELS; } +static void kvm_xen_check_poller(struct kvm_vcpu *vcpu, int port) +{ + int poll_evtchn = vcpu->arch.xen.poll_evtchn; + + if ((poll_evtchn == port || poll_evtchn == -1) && + test_and_clear_bit(kvm_vcpu_get_idx(vcpu), vcpu->kvm->arch.xen.poll_mask)) { + kvm_make_request(KVM_REQ_UNBLOCK, vcpu); + kvm_vcpu_kick(vcpu); + } +} + /* * The return value from this function is propagated to kvm_set_irq() API, * so it returns: @@ -1235,6 +1385,7 @@ int kvm_xen_set_evtchn_fast(struct kvm_xen_evtchn *xe, struct kvm *kvm) rc = 0; /* It was already raised */ } else if (test_bit(xe->port, mask_bits)) { rc = -ENOTCONN; /* Masked */ + kvm_xen_check_poller(vcpu, xe->port); } else { rc = 1; /* Delivered to the bitmap in shared_info. */ /* Now switch to the vCPU's vcpu_info to set the index and pending_sel */ @@ -1665,6 +1816,8 @@ static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r) void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu) { vcpu->arch.xen.vcpu_id = vcpu->vcpu_idx; + vcpu->arch.xen.poll_evtchn = 0; + timer_setup(&vcpu->arch.xen.poll_timer, cancel_evtchn_poll, 0); } void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) @@ -1678,6 +1831,7 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) &vcpu->arch.xen.vcpu_info_cache); kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.xen.vcpu_time_info_cache); + del_timer_sync(&vcpu->arch.xen.poll_timer); } void kvm_xen_init_vm(struct kvm *kvm) diff --git a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c index 865e17146815..376c611443cd 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c @@ -233,6 +233,12 @@ int main(int argc, char *argv[]) .flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL, .msr = XEN_HYPERCALL_MSR, }; + + /* Let the kernel know that we *will* use it for sending all + * event channels, which lets it intercept SCHEDOP_poll */ + if (xen_caps & KVM_XEN_HVM_CONFIG_EVTCHN_SEND) + hvmc.flags |= KVM_XEN_HVM_CONFIG_EVTCHN_SEND; + vm_ioctl(vm, KVM_XEN_HVM_CONFIG, &hvmc); struct kvm_xen_hvm_attr lm = { From 25eaeebe710c8c3aa588ee0830155fefb2548c28 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 3 Mar 2022 15:41:27 +0000 Subject: [PATCH 020/319] KVM: x86/xen: Add self tests for KVM_XEN_HVM_CONFIG_EVTCHN_SEND Test a combination of event channel send, poll and timer operations. Signed-off-by: David Woodhouse Message-Id: <20220303154127.202856-18-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/x86_64/xen_shinfo_test.c | 333 +++++++++++++++++- 1 file changed, 320 insertions(+), 13 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c index 376c611443cd..63b0ca7685b0 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c @@ -39,12 +39,36 @@ #define EVTCHN_VECTOR 0x10 +#define EVTCHN_TEST1 15 +#define EVTCHN_TEST2 66 +#define EVTCHN_TIMER 13 + static struct kvm_vm *vm; #define XEN_HYPERCALL_MSR 0x40000000 #define MIN_STEAL_TIME 50000 +#define __HYPERVISOR_set_timer_op 15 +#define __HYPERVISOR_sched_op 29 +#define __HYPERVISOR_event_channel_op 32 + +#define SCHEDOP_poll 3 + +#define EVTCHNOP_send 4 + +#define EVTCHNSTAT_interdomain 2 + +struct evtchn_send { + u32 port; +}; + +struct sched_poll { + u32 *ports; + unsigned int nr_ports; + u64 timeout; +}; + struct pvclock_vcpu_time_info { u32 version; u32 pad0; @@ -107,15 +131,25 @@ struct { struct kvm_irq_routing_entry entries[2]; } irq_routes; +bool guest_saw_irq; + static void evtchn_handler(struct ex_regs *regs) { struct vcpu_info *vi = (void *)VCPU_INFO_VADDR; vi->evtchn_upcall_pending = 0; vi->evtchn_pending_sel = 0; + guest_saw_irq = true; GUEST_SYNC(0x20); } +static void guest_wait_for_irq(void) +{ + while (!guest_saw_irq) + __asm__ __volatile__ ("rep nop" : : : "memory"); + guest_saw_irq = false; +} + static void guest_code(void) { struct vcpu_runstate_info *rs = (void *)RUNSTATE_VADDR; @@ -128,6 +162,8 @@ static void guest_code(void) /* Trigger an interrupt injection */ GUEST_SYNC(0); + guest_wait_for_irq(); + /* Test having the host set runstates manually */ GUEST_SYNC(RUNSTATE_runnable); GUEST_ASSERT(rs->time[RUNSTATE_runnable] != 0); @@ -168,14 +204,127 @@ static void guest_code(void) /* Now deliver an *unmasked* interrupt */ GUEST_SYNC(8); - while (!si->evtchn_pending[1]) - __asm__ __volatile__ ("rep nop" : : : "memory"); + guest_wait_for_irq(); /* Change memslots and deliver an interrupt */ GUEST_SYNC(9); - for (;;) - __asm__ __volatile__ ("rep nop" : : : "memory"); + guest_wait_for_irq(); + + /* Deliver event channel with KVM_XEN_HVM_EVTCHN_SEND */ + GUEST_SYNC(10); + + guest_wait_for_irq(); + + GUEST_SYNC(11); + + /* Our turn. Deliver event channel (to ourselves) with + * EVTCHNOP_send hypercall. */ + unsigned long rax; + struct evtchn_send s = { .port = 127 }; + __asm__ __volatile__ ("vmcall" : + "=a" (rax) : + "a" (__HYPERVISOR_event_channel_op), + "D" (EVTCHNOP_send), + "S" (&s)); + + GUEST_ASSERT(rax == 0); + + guest_wait_for_irq(); + + GUEST_SYNC(12); + + /* Deliver "outbound" event channel to an eventfd which + * happens to be one of our own irqfds. */ + s.port = 197; + __asm__ __volatile__ ("vmcall" : + "=a" (rax) : + "a" (__HYPERVISOR_event_channel_op), + "D" (EVTCHNOP_send), + "S" (&s)); + + GUEST_ASSERT(rax == 0); + + guest_wait_for_irq(); + + GUEST_SYNC(13); + + /* Set a timer 100ms in the future. */ + __asm__ __volatile__ ("vmcall" : + "=a" (rax) : + "a" (__HYPERVISOR_set_timer_op), + "D" (rs->state_entry_time + 100000000)); + GUEST_ASSERT(rax == 0); + + GUEST_SYNC(14); + + /* Now wait for the timer */ + guest_wait_for_irq(); + + GUEST_SYNC(15); + + /* The host has 'restored' the timer. Just wait for it. */ + guest_wait_for_irq(); + + GUEST_SYNC(16); + + /* Poll for an event channel port which is already set */ + u32 ports[1] = { EVTCHN_TIMER }; + struct sched_poll p = { + .ports = ports, + .nr_ports = 1, + .timeout = 0, + }; + + __asm__ __volatile__ ("vmcall" : + "=a" (rax) : + "a" (__HYPERVISOR_sched_op), + "D" (SCHEDOP_poll), + "S" (&p)); + + GUEST_ASSERT(rax == 0); + + GUEST_SYNC(17); + + /* Poll for an unset port and wait for the timeout. */ + p.timeout = 100000000; + __asm__ __volatile__ ("vmcall" : + "=a" (rax) : + "a" (__HYPERVISOR_sched_op), + "D" (SCHEDOP_poll), + "S" (&p)); + + GUEST_ASSERT(rax == 0); + + GUEST_SYNC(18); + + /* A timer will wake the masked port we're waiting on, while we poll */ + p.timeout = 0; + __asm__ __volatile__ ("vmcall" : + "=a" (rax) : + "a" (__HYPERVISOR_sched_op), + "D" (SCHEDOP_poll), + "S" (&p)); + + GUEST_ASSERT(rax == 0); + + GUEST_SYNC(19); + + /* A timer wake an *unmasked* port which should wake us with an + * actual interrupt, while we're polling on a different port. */ + ports[0]++; + p.timeout = 0; + __asm__ __volatile__ ("vmcall" : + "=a" (rax) : + "a" (__HYPERVISOR_sched_op), + "D" (SCHEDOP_poll), + "S" (&p)); + + GUEST_ASSERT(rax == 0); + + guest_wait_for_irq(); + + GUEST_SYNC(20); } static int cmp_timespec(struct timespec *a, struct timespec *b) @@ -191,9 +340,13 @@ static int cmp_timespec(struct timespec *a, struct timespec *b) else return 0; } +struct vcpu_info *vinfo; static void handle_alrm(int sig) { + if (vinfo) + printf("evtchn_upcall_pending 0x%x\n", vinfo->evtchn_upcall_pending); + vcpu_dump(stdout, vm, VCPU_ID, 0); TEST_FAIL("IRQ delivery timed out"); } @@ -213,6 +366,7 @@ int main(int argc, char *argv[]) bool do_runstate_tests = !!(xen_caps & KVM_XEN_HVM_CONFIG_RUNSTATE); bool do_eventfd_tests = !!(xen_caps & KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL); + bool do_evtchn_tests = do_eventfd_tests && !!(xen_caps & KVM_XEN_HVM_CONFIG_EVTCHN_SEND); clock_gettime(CLOCK_REALTIME, &min_ts); @@ -236,7 +390,7 @@ int main(int argc, char *argv[]) /* Let the kernel know that we *will* use it for sending all * event channels, which lets it intercept SCHEDOP_poll */ - if (xen_caps & KVM_XEN_HVM_CONFIG_EVTCHN_SEND) + if (do_evtchn_tests) hvmc.flags |= KVM_XEN_HVM_CONFIG_EVTCHN_SEND; vm_ioctl(vm, KVM_XEN_HVM_CONFIG, &hvmc); @@ -301,7 +455,7 @@ int main(int argc, char *argv[]) /* Unexpected, but not a KVM failure */ if (irq_fd[0] == -1 || irq_fd[1] == -1) - do_eventfd_tests = false; + do_evtchn_tests = do_eventfd_tests = false; } if (do_eventfd_tests) { @@ -309,13 +463,13 @@ int main(int argc, char *argv[]) irq_routes.entries[0].gsi = 32; irq_routes.entries[0].type = KVM_IRQ_ROUTING_XEN_EVTCHN; - irq_routes.entries[0].u.xen_evtchn.port = 15; + irq_routes.entries[0].u.xen_evtchn.port = EVTCHN_TEST1; irq_routes.entries[0].u.xen_evtchn.vcpu = VCPU_ID; irq_routes.entries[0].u.xen_evtchn.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL; irq_routes.entries[1].gsi = 33; irq_routes.entries[1].type = KVM_IRQ_ROUTING_XEN_EVTCHN; - irq_routes.entries[1].u.xen_evtchn.port = 66; + irq_routes.entries[1].u.xen_evtchn.port = EVTCHN_TEST2; irq_routes.entries[1].u.xen_evtchn.vcpu = VCPU_ID; irq_routes.entries[1].u.xen_evtchn.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL; @@ -336,7 +490,39 @@ int main(int argc, char *argv[]) sigaction(SIGALRM, &sa, NULL); } - struct vcpu_info *vinfo = addr_gpa2hva(vm, VCPU_INFO_VADDR); + struct kvm_xen_vcpu_attr tmr = { + .type = KVM_XEN_VCPU_ATTR_TYPE_TIMER, + .u.timer.port = EVTCHN_TIMER, + .u.timer.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL, + .u.timer.expires_ns = 0 + }; + + if (do_evtchn_tests) { + struct kvm_xen_hvm_attr inj = { + .type = KVM_XEN_ATTR_TYPE_EVTCHN, + .u.evtchn.send_port = 127, + .u.evtchn.type = EVTCHNSTAT_interdomain, + .u.evtchn.flags = 0, + .u.evtchn.deliver.port.port = EVTCHN_TEST1, + .u.evtchn.deliver.port.vcpu = VCPU_ID + 1, + .u.evtchn.deliver.port.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL, + }; + vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &inj); + + /* Test migration to a different vCPU */ + inj.u.evtchn.flags = KVM_XEN_EVTCHN_UPDATE; + inj.u.evtchn.deliver.port.vcpu = VCPU_ID; + vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &inj); + + inj.u.evtchn.send_port = 197; + inj.u.evtchn.deliver.eventfd.port = 0; + inj.u.evtchn.deliver.eventfd.fd = irq_fd[1]; + inj.u.evtchn.flags = 0; + vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &inj); + + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &tmr); + } + vinfo = addr_gpa2hva(vm, VCPU_INFO_VADDR); vinfo->evtchn_upcall_pending = 0; struct vcpu_runstate_info *rs = addr_gpa2hva(vm, RUNSTATE_ADDR); @@ -429,7 +615,7 @@ int main(int argc, char *argv[]) goto done; if (verbose) printf("Testing masked event channel\n"); - shinfo->evtchn_mask[0] = 0x8000; + shinfo->evtchn_mask[0] = 1UL << EVTCHN_TEST1; eventfd_write(irq_fd[0], 1UL); alarm(1); break; @@ -446,6 +632,9 @@ int main(int argc, char *argv[]) break; case 9: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + shinfo->evtchn_pending[1] = 0; if (verbose) printf("Testing event channel after memslot change\n"); vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, @@ -455,12 +644,129 @@ int main(int argc, char *argv[]) alarm(1); break; + case 10: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + if (!do_evtchn_tests) + goto done; + + shinfo->evtchn_pending[0] = 0; + if (verbose) + printf("Testing injection with KVM_XEN_HVM_EVTCHN_SEND\n"); + + struct kvm_irq_routing_xen_evtchn e; + e.port = EVTCHN_TEST2; + e.vcpu = VCPU_ID; + e.priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL; + + vm_ioctl(vm, KVM_XEN_HVM_EVTCHN_SEND, &e); + evtchn_irq_expected = true; + alarm(1); + break; + + case 11: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + shinfo->evtchn_pending[1] = 0; + + if (verbose) + printf("Testing guest EVTCHNOP_send direct to evtchn\n"); + evtchn_irq_expected = true; + alarm(1); + break; + + case 12: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + shinfo->evtchn_pending[0] = 0; + + if (verbose) + printf("Testing guest EVTCHNOP_send to eventfd\n"); + evtchn_irq_expected = true; + alarm(1); + break; + + case 13: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + shinfo->evtchn_pending[1] = 0; + + if (verbose) + printf("Testing guest oneshot timer\n"); + break; + + case 14: + memset(&tmr, 0, sizeof(tmr)); + tmr.type = KVM_XEN_VCPU_ATTR_TYPE_TIMER, + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_GET_ATTR, &tmr); + TEST_ASSERT(tmr.u.timer.port == EVTCHN_TIMER, + "Timer port not returned"); + TEST_ASSERT(tmr.u.timer.priority == KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL, + "Timer priority not returned"); + TEST_ASSERT(tmr.u.timer.expires_ns > rs->state_entry_time, + "Timer expiry not returned"); + evtchn_irq_expected = true; + alarm(1); + break; + + case 15: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + shinfo->evtchn_pending[0] = 0; + + if (verbose) + printf("Testing restored oneshot timer\n"); + + tmr.u.timer.expires_ns = rs->state_entry_time + 100000000, + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &tmr); + evtchn_irq_expected = true; + alarm(1); + break; + + case 16: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + + if (verbose) + printf("Testing SCHEDOP_poll with already pending event\n"); + shinfo->evtchn_pending[0] = shinfo->evtchn_mask[0] = 1UL << EVTCHN_TIMER; + alarm(1); + break; + + case 17: + if (verbose) + printf("Testing SCHEDOP_poll timeout\n"); + shinfo->evtchn_pending[0] = 0; + alarm(1); + break; + + case 18: + if (verbose) + printf("Testing SCHEDOP_poll wake on masked event\n"); + + tmr.u.timer.expires_ns = rs->state_entry_time + 100000000, + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &tmr); + break; + + case 19: + shinfo->evtchn_pending[0] = shinfo->evtchn_mask[0] = 0; + if (verbose) + printf("Testing SCHEDOP_poll wake on unmasked event\n"); + + tmr.u.timer.expires_ns = rs->state_entry_time + 100000000, + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &tmr); + evtchn_irq_expected = true; + break; + + case 20: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + shinfo->evtchn_pending[1] = 0; + goto done; + case 0x20: TEST_ASSERT(evtchn_irq_expected, "Unexpected event channel IRQ"); evtchn_irq_expected = false; - if (shinfo->evtchn_pending[1] && - shinfo->evtchn_pending[0]) - goto done; break; } break; @@ -473,6 +779,7 @@ int main(int argc, char *argv[]) } done: + alarm(0); clock_gettime(CLOCK_REALTIME, &max_ts); /* From a29833e36b43b326e6371c181474119069d6073a Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Wed, 9 Mar 2022 14:38:35 +0000 Subject: [PATCH 021/319] KVM: x86/xen: Update self test for Xen PV timers Add test cases for timers in the past, and reading the status of a timer which has already fired. Signed-off-by: David Woodhouse Message-Id: <20220309143835.253911-3-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/x86_64/xen_shinfo_test.c | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c index 63b0ca7685b0..d9d9d1deec45 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c @@ -325,6 +325,11 @@ static void guest_code(void) guest_wait_for_irq(); GUEST_SYNC(20); + + /* Timer should have fired already */ + guest_wait_for_irq(); + + GUEST_SYNC(21); } static int cmp_timespec(struct timespec *a, struct timespec *b) @@ -746,6 +751,7 @@ int main(int argc, char *argv[]) tmr.u.timer.expires_ns = rs->state_entry_time + 100000000, vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &tmr); + alarm(1); break; case 19: @@ -753,15 +759,38 @@ int main(int argc, char *argv[]) if (verbose) printf("Testing SCHEDOP_poll wake on unmasked event\n"); - tmr.u.timer.expires_ns = rs->state_entry_time + 100000000, - vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &tmr); evtchn_irq_expected = true; + tmr.u.timer.expires_ns = rs->state_entry_time + 100000000; + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &tmr); + + /* Read it back and check the pending time is reported correctly */ + tmr.u.timer.expires_ns = 0; + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_GET_ATTR, &tmr); + TEST_ASSERT(tmr.u.timer.expires_ns == rs->state_entry_time + 100000000, + "Timer not reported pending"); + alarm(1); break; case 20: TEST_ASSERT(!evtchn_irq_expected, "Expected event channel IRQ but it didn't happen"); - shinfo->evtchn_pending[1] = 0; + /* Read timer and check it is no longer pending */ + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_GET_ATTR, &tmr); + TEST_ASSERT(!tmr.u.timer.expires_ns, "Timer still reported pending"); + + shinfo->evtchn_pending[0] = 0; + if (verbose) + printf("Testing timer in the past\n"); + + evtchn_irq_expected = true; + tmr.u.timer.expires_ns = rs->state_entry_time - 100000000ULL; + vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_SET_ATTR, &tmr); + alarm(1); + break; + + case 21: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); goto done; case 0x20: From 1421211ae1de8cc547753080e9c9ffae8b67790f Mon Sep 17 00:00:00 2001 From: Zeng Guang Date: Tue, 15 Mar 2022 22:58:36 +0800 Subject: [PATCH 022/319] KVM: VMX: Prepare VMCS setting for posted interrupt enabling when APICv is available Currently KVM setup posted interrupt VMCS only depending on per-vcpu APICv activation status at the vCPU creation time. However, this status can be toggled dynamically under some circumstance. So potentially, later posted interrupt enabling may be problematic without VMCS readiness. To fix this, always settle the VMCS setting for posted interrupt as long as APICv is available and lapic locates in kernel. Signed-off-by: Zeng Guang Message-Id: <20220315145836.9910-1-guang.zeng@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 04d170c4b61e..c654c9d76e09 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4380,7 +4380,7 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (cpu_has_secondary_exec_ctrls()) secondary_exec_controls_set(vmx, vmx_secondary_exec_control(vmx)); - if (kvm_vcpu_apicv_active(&vmx->vcpu)) { + if (enable_apicv && lapic_in_kernel(&vmx->vcpu)) { vmcs_write64(EOI_EXIT_BITMAP0, 0); vmcs_write64(EOI_EXIT_BITMAP1, 0); vmcs_write64(EOI_EXIT_BITMAP2, 0); From fe3787a0d14920fa98c38a21431001c9d4edfe93 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Tue, 1 Mar 2022 20:02:17 +0800 Subject: [PATCH 023/319] KVM: x86/i8259: Remove a dead store of irq in a conditional block The [clang-analyzer-deadcode.DeadStores] helper reports that the value stored to 'irq' is never read. Signed-off-by: Like Xu Message-Id: <20220301120217.38092-1-likexu@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/i8259.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c index be99dc86293d..e1bb6218bb96 100644 --- a/arch/x86/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c @@ -252,7 +252,6 @@ int kvm_pic_read_irq(struct kvm *kvm) */ irq2 = 7; intno = s->pics[1].irq_base + irq2; - irq = irq2 + 8; } else intno = s->pics[0].irq_base + irq; } else { From ffbb61d09fc56c85e28b110494f3788d0ed4d1f8 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 25 Feb 2022 14:53:02 +0000 Subject: [PATCH 024/319] KVM: x86: Accept KVM_[GS]ET_TSC_KHZ as a VM ioctl. This sets the default TSC frequency for subsequently created vCPUs. Signed-off-by: David Woodhouse Message-Id: <20220225145304.36166-2-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 11 +++++++---- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/x86.c | 26 +++++++++++++++++++++++++- include/uapi/linux/kvm.h | 4 +++- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index fdfd9e1bc7f7..b102ba7cf903 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -1897,22 +1897,25 @@ the future. 4.55 KVM_SET_TSC_KHZ -------------------- -:Capability: KVM_CAP_TSC_CONTROL +:Capability: KVM_CAP_TSC_CONTROL / KVM_CAP_VM_TSC_CONTROL :Architectures: x86 -:Type: vcpu ioctl +:Type: vcpu ioctl / vm ioctl :Parameters: virtual tsc_khz :Returns: 0 on success, -1 on error Specifies the tsc frequency for the virtual machine. The unit of the frequency is KHz. +If the KVM_CAP_VM_TSC_CONTROL capability is advertised, this can also +be used as a vm ioctl to set the initial tsc frequency of subsequently +created vCPUs. 4.56 KVM_GET_TSC_KHZ -------------------- -:Capability: KVM_CAP_GET_TSC_KHZ +:Capability: KVM_CAP_GET_TSC_KHZ / KVM_CAP_VM_TSC_CONTROL :Architectures: x86 -:Type: vcpu ioctl +:Type: vcpu ioctl / vm ioctl :Parameters: none :Returns: virtual tsc-khz on success, negative value on error diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5370744b789c..fac990cc189d 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1126,6 +1126,8 @@ struct kvm_arch { u64 cur_tsc_generation; int nr_vcpus_matched_tsc; + u32 default_tsc_khz; + seqcount_raw_spinlock_t pvclock_sc; bool use_master_clock; u64 master_kernel_ns; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1a7d1b5f7232..c069b97a8523 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4324,6 +4324,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = boot_cpu_has(X86_FEATURE_XSAVE); break; case KVM_CAP_TSC_CONTROL: + case KVM_CAP_VM_TSC_CONTROL: r = kvm_has_tsc_control; break; case KVM_CAP_X2APIC_API: @@ -6522,6 +6523,28 @@ long kvm_arch_vm_ioctl(struct file *filp, case KVM_GET_CLOCK: r = kvm_vm_ioctl_get_clock(kvm, argp); break; + case KVM_SET_TSC_KHZ: { + u32 user_tsc_khz; + + r = -EINVAL; + user_tsc_khz = (u32)arg; + + if (kvm_has_tsc_control && + user_tsc_khz >= kvm_max_guest_tsc_khz) + goto out; + + if (user_tsc_khz == 0) + user_tsc_khz = tsc_khz; + + WRITE_ONCE(kvm->arch.default_tsc_khz, user_tsc_khz); + r = 0; + + goto out; + } + case KVM_GET_TSC_KHZ: { + r = READ_ONCE(kvm->arch.default_tsc_khz); + goto out; + } case KVM_MEMORY_ENCRYPT_OP: { r = -ENOTTY; if (!kvm_x86_ops.mem_enc_ioctl) @@ -11266,7 +11289,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) kvm_xen_init_vcpu(vcpu); kvm_vcpu_mtrr_init(vcpu); vcpu_load(vcpu); - kvm_set_tsc_khz(vcpu, max_tsc_khz); + kvm_set_tsc_khz(vcpu, vcpu->kvm->arch.default_tsc_khz); kvm_vcpu_reset(vcpu, false); kvm_init_mmu(vcpu); vcpu_put(vcpu); @@ -11714,6 +11737,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) pvclock_update_vm_gtod_copy(kvm); raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); + kvm->arch.default_tsc_khz = max_tsc_khz; kvm->arch.guest_can_read_msr_platform_info = true; kvm->arch.enable_pmu = enable_pmu; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index ee5cc9e2a837..8616af85dc5d 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1144,6 +1144,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_S390_MEM_OP_EXTENSION 211 #define KVM_CAP_PMU_CAPABILITY 212 #define KVM_CAP_DISABLE_QUIRKS2 213 +#define KVM_CAP_VM_TSC_CONTROL 214 #ifdef KVM_CAP_IRQ_ROUTING @@ -1471,7 +1472,8 @@ struct kvm_s390_ucas_mapping { #define KVM_SET_PIT2 _IOW(KVMIO, 0xa0, struct kvm_pit_state2) /* Available with KVM_CAP_PPC_GET_PVINFO */ #define KVM_PPC_GET_PVINFO _IOW(KVMIO, 0xa1, struct kvm_ppc_pvinfo) -/* Available with KVM_CAP_TSC_CONTROL */ +/* Available with KVM_CAP_TSC_CONTROL for a vCPU, or with +* KVM_CAP_VM_TSC_CONTROL to set defaults for a VM */ #define KVM_SET_TSC_KHZ _IO(KVMIO, 0xa2) #define KVM_GET_TSC_KHZ _IO(KVMIO, 0xa3) /* Available with KVM_CAP_PCI_2_3 */ From 741e511b42086a100c05dbe8fd1baeec42e7c584 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 25 Feb 2022 14:53:03 +0000 Subject: [PATCH 025/319] KVM: x86: Don't snapshot "max" TSC if host TSC is constant Don't snapshot tsc_khz into max_tsc_khz during KVM initialization if the host TSC is constant, in which case the actual TSC frequency will never change and thus capturing the "max" TSC during initialization is unnecessary, KVM can simply use tsc_khz during VM creation. On CPUs with constant TSC, but not a hardware-specified TSC frequency, snapshotting max_tsc_khz and using that to set a VM's default TSC frequency can lead to KVM thinking it needs to manually scale the guest's TSC if refining the TSC completes after KVM snapshots tsc_khz. The actual frequency never changes, only the kernel's calculation of what that frequency is changes. On systems without hardware TSC scaling, this either puts KVM into "always catchup" mode (extremely inefficient), or prevents creating VMs altogether. Ideally, KVM would not be able to race with TSC refinement, or would have a hook into tsc_refine_calibration_work() to get an alert when refinement is complete. Avoiding the race altogether isn't practical as refinement takes a relative eternity; it's deliberately put on a work queue outside of the normal boot sequence to avoid unnecessarily delaying boot. Adding a hook is doable, but somewhat gross due to KVM's ability to be built as a module. And if the TSC is constant, which is likely the case for every VMX/SVM-capable CPU produced in the last decade, the race can be hit if and only if userspace is able to create a VM before TSC refinement completes; refinement is slow, but not that slow. For now, punt on a proper fix, as not taking a snapshot can help some uses cases and not taking a snapshot is arguably correct irrespective of the race with refinement. [ dwmw2: Rebase on top of KVM-wide default_tsc_khz to ensure that all vCPUs get the same frequency even if we hit the race. ] Cc: Suleiman Souhlal Cc: Anton Romanov Signed-off-by: Sean Christopherson Signed-off-by: David Woodhouse Message-Id: <20220225145304.36166-3-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c069b97a8523..dad13fd6b61e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8814,22 +8814,22 @@ static int kvmclock_cpu_online(unsigned int cpu) static void kvm_timer_init(void) { - max_tsc_khz = tsc_khz; - if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { -#ifdef CONFIG_CPU_FREQ - struct cpufreq_policy *policy; - int cpu; + max_tsc_khz = tsc_khz; - cpu = get_cpu(); - policy = cpufreq_cpu_get(cpu); - if (policy) { - if (policy->cpuinfo.max_freq) - max_tsc_khz = policy->cpuinfo.max_freq; - cpufreq_cpu_put(policy); + if (IS_ENABLED(CONFIG_CPU_FREQ)) { + struct cpufreq_policy *policy; + int cpu; + + cpu = get_cpu(); + policy = cpufreq_cpu_get(cpu); + if (policy) { + if (policy->cpuinfo.max_freq) + max_tsc_khz = policy->cpuinfo.max_freq; + cpufreq_cpu_put(policy); + } + put_cpu(); } - put_cpu(); -#endif cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, CPUFREQ_TRANSITION_NOTIFIER); } @@ -11737,7 +11737,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) pvclock_update_vm_gtod_copy(kvm); raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags); - kvm->arch.default_tsc_khz = max_tsc_khz; + kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz; kvm->arch.guest_can_read_msr_platform_info = true; kvm->arch.enable_pmu = enable_pmu; From e467b0de82b260a4ee2c3ea53ef76ac40259498f Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 25 Feb 2022 14:53:04 +0000 Subject: [PATCH 026/319] KVM: x86: Test case for TSC scaling and offset sync Signed-off-by: David Woodhouse Message-Id: <20220225145304.36166-4-dwmw2@infradead.org> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/x86_64/tsc_scaling_sync.c | 119 ++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 tools/testing/selftests/kvm/x86_64/tsc_scaling_sync.c diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 03662666b30d..c9cdbd248727 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -66,6 +66,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/state_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_preemption_timer_test TEST_GEN_PROGS_x86_64 += x86_64/svm_vmcall_test TEST_GEN_PROGS_x86_64 += x86_64/svm_int_ctl_test +TEST_GEN_PROGS_x86_64 += x86_64/tsc_scaling_sync TEST_GEN_PROGS_x86_64 += x86_64/sync_regs_test TEST_GEN_PROGS_x86_64 += x86_64/userspace_io_test TEST_GEN_PROGS_x86_64 += x86_64/userspace_msr_exit_test diff --git a/tools/testing/selftests/kvm/x86_64/tsc_scaling_sync.c b/tools/testing/selftests/kvm/x86_64/tsc_scaling_sync.c new file mode 100644 index 000000000000..f0083d8cfe98 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/tsc_scaling_sync.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * svm_vmcall_test + * + * Copyright © 2021 Amazon.com, Inc. or its affiliates. + * + * Xen shared_info / pvclock testing + */ + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" + +#include +#include +#include +#include +#include + +#define NR_TEST_VCPUS 20 + +static struct kvm_vm *vm; +pthread_spinlock_t create_lock; + +#define TEST_TSC_KHZ 2345678UL +#define TEST_TSC_OFFSET 200000000 + +uint64_t tsc_sync; +static void guest_code(void) +{ + uint64_t start_tsc, local_tsc, tmp; + + start_tsc = rdtsc(); + do { + tmp = READ_ONCE(tsc_sync); + local_tsc = rdtsc(); + WRITE_ONCE(tsc_sync, local_tsc); + if (unlikely(local_tsc < tmp)) + GUEST_SYNC_ARGS(0, local_tsc, tmp, 0, 0); + + } while (local_tsc - start_tsc < 5000 * TEST_TSC_KHZ); + + GUEST_DONE(); +} + + +static void *run_vcpu(void *_cpu_nr) +{ + unsigned long cpu = (unsigned long)_cpu_nr; + unsigned long failures = 0; + static bool first_cpu_done; + + /* The kernel is fine, but vm_vcpu_add_default() needs locking */ + pthread_spin_lock(&create_lock); + + vm_vcpu_add_default(vm, cpu, guest_code); + + if (!first_cpu_done) { + first_cpu_done = true; + vcpu_set_msr(vm, cpu, MSR_IA32_TSC, TEST_TSC_OFFSET); + } + + pthread_spin_unlock(&create_lock); + + for (;;) { + volatile struct kvm_run *run = vcpu_state(vm, cpu); + struct ucall uc; + + vcpu_run(vm, cpu); + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Got exit_reason other than KVM_EXIT_IO: %u (%s)\n", + run->exit_reason, + exit_reason_str(run->exit_reason)); + + switch (get_ucall(vm, cpu, &uc)) { + case UCALL_DONE: + goto out; + + case UCALL_SYNC: + printf("Guest %ld sync %lx %lx %ld\n", cpu, uc.args[2], uc.args[3], uc.args[2] - uc.args[3]); + failures++; + break; + + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } + out: + return (void *)failures; +} + +int main(int argc, char *argv[]) +{ + if (!kvm_check_cap(KVM_CAP_VM_TSC_CONTROL)) { + print_skip("KVM_CAP_VM_TSC_CONTROL not available"); + exit(KSFT_SKIP); + } + + vm = vm_create_default_with_vcpus(0, DEFAULT_STACK_PGS * NR_TEST_VCPUS, 0, guest_code, NULL); + vm_ioctl(vm, KVM_SET_TSC_KHZ, (void *) TEST_TSC_KHZ); + + pthread_spin_init(&create_lock, PTHREAD_PROCESS_PRIVATE); + pthread_t cpu_threads[NR_TEST_VCPUS]; + unsigned long cpu; + for (cpu = 0; cpu < NR_TEST_VCPUS; cpu++) + pthread_create(&cpu_threads[cpu], NULL, run_vcpu, (void *)cpu); + + unsigned long failures = 0; + for (cpu = 0; cpu < NR_TEST_VCPUS; cpu++) { + void *this_cpu_failures; + pthread_join(cpu_threads[cpu], &this_cpu_failures); + failures += (unsigned long)this_cpu_failures; + } + + TEST_ASSERT(!failures, "TSC sync failed"); + pthread_spin_destroy(&create_lock); + kvm_vm_free(vm); + return 0; +} From d063de55f4799261858676d5cfa49a1612cd57ea Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 9 Mar 2022 16:46:50 +0800 Subject: [PATCH 027/319] KVM: x86: Support the vCPU preemption check with nopvspin and realtime hint If guest kernel is configured with nopvspin, or CONFIG_PARAVIRT_SPINLOCK is disabled, or guest find its has dedicated pCPUs from realtime hint feature, the pvspinlock will be disabled, and vCPU preemption check is disabled too. Hoever, KVM still can emulating HLT for vCPU for both cases. Checking if a vCPU is preempted or not can still boost performance in IPI-heavy scenarios such as unixbench file copy and pipe-based context switching tests: Here the vCPU is running with a dedicated pCPU, so the guest kernel has nopvspin but is emulating HLT for the vCPU: Testcase Base with patch System Benchmarks Index Values INDEX INDEX Dhrystone 2 using register variables 3278.4 3277.7 Double-Precision Whetstone 822.8 825.8 Execl Throughput 1296.5 941.1 File Copy 1024 bufsize 2000 maxblocks 2124.2 2142.7 File Copy 256 bufsize 500 maxblocks 1335.9 1353.6 File Copy 4096 bufsize 8000 maxblocks 4256.3 4760.3 Pipe Throughput 1050.1 1054.0 Pipe-based Context Switching 243.3 352.0 Process Creation 820.1 814.4 Shell Scripts (1 concurrent) 2169.0 2086.0 Shell Scripts (8 concurrent) 7710.3 7576.3 System Call Overhead 672.4 673.9 ======== ======= System Benchmarks Index Score 1467.2 1483.0 Move the setting of pv_ops.lock.vcpu_is_preempted to kvm_guest_init, so that it does not depend on pvspinlock. Signed-off-by: Li RongQing Message-Id: <1646815610-43315-1-git-send-email-lirongqing@baidu.com> Signed-off-by: Paolo Bonzini --- arch/x86/kernel/asm-offsets_64.c | 4 +- arch/x86/kernel/kvm.c | 75 ++++++++++++++++---------------- 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index b14533af7676..9b698215d261 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c @@ -5,7 +5,7 @@ #include -#if defined(CONFIG_KVM_GUEST) && defined(CONFIG_PARAVIRT_SPINLOCKS) +#if defined(CONFIG_KVM_GUEST) #include #endif @@ -20,7 +20,7 @@ int main(void) BLANK(); #endif -#if defined(CONFIG_KVM_GUEST) && defined(CONFIG_PARAVIRT_SPINLOCKS) +#if defined(CONFIG_KVM_GUEST) OFFSET(KVM_STEAL_TIME_preempted, kvm_steal_time, preempted); BLANK(); #endif diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index ed8a13ac4ab2..774d924aeda8 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -752,6 +752,41 @@ static void kvm_crash_shutdown(struct pt_regs *regs) } #endif +#if defined(CONFIG_X86_32) || !defined(CONFIG_SMP) +bool __kvm_vcpu_is_preempted(long cpu); + +__visible bool __kvm_vcpu_is_preempted(long cpu) +{ + struct kvm_steal_time *src = &per_cpu(steal_time, cpu); + + return !!(src->preempted & KVM_VCPU_PREEMPTED); +} +PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted); + +#else + +#include + +extern bool __raw_callee_save___kvm_vcpu_is_preempted(long); + +/* + * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and + * restoring to/from the stack. + */ +asm( +".pushsection .text;" +".global __raw_callee_save___kvm_vcpu_is_preempted;" +".type __raw_callee_save___kvm_vcpu_is_preempted, @function;" +"__raw_callee_save___kvm_vcpu_is_preempted:" +"movq __per_cpu_offset(,%rdi,8), %rax;" +"cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);" +"setne %al;" +"ret;" +".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;" +".popsection"); + +#endif + static void __init kvm_guest_init(void) { int i; @@ -764,6 +799,9 @@ static void __init kvm_guest_init(void) if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { has_steal_clock = 1; static_call_update(pv_steal_clock, kvm_steal_clock); + + pv_ops.lock.vcpu_is_preempted = + PV_CALLEE_SAVE(__kvm_vcpu_is_preempted); } if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) @@ -1005,39 +1043,6 @@ static void kvm_wait(u8 *ptr, u8 val) } } -#ifdef CONFIG_X86_32 -__visible bool __kvm_vcpu_is_preempted(long cpu) -{ - struct kvm_steal_time *src = &per_cpu(steal_time, cpu); - - return !!(src->preempted & KVM_VCPU_PREEMPTED); -} -PV_CALLEE_SAVE_REGS_THUNK(__kvm_vcpu_is_preempted); - -#else - -#include - -extern bool __raw_callee_save___kvm_vcpu_is_preempted(long); - -/* - * Hand-optimize version for x86-64 to avoid 8 64-bit register saving and - * restoring to/from the stack. - */ -asm( -".pushsection .text;" -".global __raw_callee_save___kvm_vcpu_is_preempted;" -".type __raw_callee_save___kvm_vcpu_is_preempted, @function;" -"__raw_callee_save___kvm_vcpu_is_preempted:" -"movq __per_cpu_offset(,%rdi,8), %rax;" -"cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);" -"setne %al;" -"ret;" -".size __raw_callee_save___kvm_vcpu_is_preempted, .-__raw_callee_save___kvm_vcpu_is_preempted;" -".popsection"); - -#endif - /* * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present. */ @@ -1081,10 +1086,6 @@ void __init kvm_spinlock_init(void) pv_ops.lock.wait = kvm_wait; pv_ops.lock.kick = kvm_kick_cpu; - if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { - pv_ops.lock.vcpu_is_preempted = - PV_CALLEE_SAVE(__kvm_vcpu_is_preempted); - } /* * When PV spinlock is enabled which is preferred over * virt_spin_lock(), virt_spin_lock_key's value is meaningless. From 1ee73a332f80ddb05b9becde53a644d6efc666c7 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 18:24:43 +0100 Subject: [PATCH 028/319] KVM: x86: SVM: use vmcb01 in init_vmcb Clarify that this function is not used to initialize any part of the vmcb02. No functional change intended. Signed-off-by: Maxim Levitsky Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/avic.c | 3 +-- arch/x86/kvm/svm/svm.c | 11 ++++++----- arch/x86/kvm/svm/svm.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index a1cf9c31273b..b7dabc01d644 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -165,9 +165,8 @@ int avic_vm_init(struct kvm *kvm) return err; } -void avic_init_vmcb(struct vcpu_svm *svm) +void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb) { - struct vmcb *vmcb = svm->vmcb; struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm); phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page)); phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page)); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 462f691fb614..dd2abed45511 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -984,8 +984,9 @@ static inline void init_vmcb_after_set_cpuid(struct kvm_vcpu *vcpu) static void init_vmcb(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - struct vmcb_control_area *control = &svm->vmcb->control; - struct vmcb_save_area *save = &svm->vmcb->save; + struct vmcb *vmcb = svm->vmcb01.ptr; + struct vmcb_control_area *control = &vmcb->control; + struct vmcb_save_area *save = &vmcb->save; svm_set_intercept(svm, INTERCEPT_CR0_READ); svm_set_intercept(svm, INTERCEPT_CR3_READ); @@ -1109,7 +1110,7 @@ static void init_vmcb(struct kvm_vcpu *vcpu) set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); if (kvm_vcpu_apicv_active(vcpu)) - avic_init_vmcb(svm); + avic_init_vmcb(svm, vmcb); if (vgif) { svm_clr_intercept(svm, INTERCEPT_STGI); @@ -1127,10 +1128,10 @@ static void init_vmcb(struct kvm_vcpu *vcpu) } } - svm_hv_init_vmcb(svm->vmcb); + svm_hv_init_vmcb(vmcb); init_vmcb_after_set_cpuid(vcpu); - vmcb_mark_all_dirty(svm->vmcb); + vmcb_mark_all_dirty(vmcb); enable_gif(svm); } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 468f149556dd..57bde9569902 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -580,7 +580,7 @@ extern struct kvm_x86_nested_ops svm_nested_ops; int avic_ga_log_notifier(u32 ga_tag); void avic_vm_destroy(struct kvm *kvm); int avic_vm_init(struct kvm *kvm); -void avic_init_vmcb(struct vcpu_svm *svm); +void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb); int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu); int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu); int avic_init_vcpu(struct vcpu_svm *svm); From db663af4a0012a1ab0f966621a62617ce5907b35 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:24:44 +0200 Subject: [PATCH 029/319] kvm: x86: SVM: use vmcb* instead of svm->vmcb where it makes sense This makes the code a bit shorter and cleaner. No functional change intended. Signed-off-by: Maxim Levitsky Message-Id: <20220322172449.235575-4-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 179 ++++++++++++++++++++------------------ 1 file changed, 94 insertions(+), 85 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index d736ec6514ca..1c381c6a7b51 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -36,40 +36,43 @@ static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, struct x86_exception *fault) { struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb = svm->vmcb; - if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) { + if (vmcb->control.exit_code != SVM_EXIT_NPF) { /* * TODO: track the cause of the nested page fault, and * correctly fill in the high bits of exit_info_1. */ - svm->vmcb->control.exit_code = SVM_EXIT_NPF; - svm->vmcb->control.exit_code_hi = 0; - svm->vmcb->control.exit_info_1 = (1ULL << 32); - svm->vmcb->control.exit_info_2 = fault->address; + vmcb->control.exit_code = SVM_EXIT_NPF; + vmcb->control.exit_code_hi = 0; + vmcb->control.exit_info_1 = (1ULL << 32); + vmcb->control.exit_info_2 = fault->address; } - svm->vmcb->control.exit_info_1 &= ~0xffffffffULL; - svm->vmcb->control.exit_info_1 |= fault->error_code; + vmcb->control.exit_info_1 &= ~0xffffffffULL; + vmcb->control.exit_info_1 |= fault->error_code; nested_svm_vmexit(svm); } static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) { - struct vcpu_svm *svm = to_svm(vcpu); - WARN_ON(!is_guest_mode(vcpu)); + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *vmcb = svm->vmcb; + + WARN_ON(!is_guest_mode(vcpu)); if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && - !svm->nested.nested_run_pending) { - svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; - svm->vmcb->control.exit_code_hi = 0; - svm->vmcb->control.exit_info_1 = fault->error_code; - svm->vmcb->control.exit_info_2 = fault->address; - nested_svm_vmexit(svm); - } else { - kvm_inject_page_fault(vcpu, fault); - } + !svm->nested.nested_run_pending) { + vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; + vmcb->control.exit_code_hi = 0; + vmcb->control.exit_info_1 = fault->error_code; + vmcb->control.exit_info_2 = fault->address; + nested_svm_vmexit(svm); + } else { + kvm_inject_page_fault(vcpu, fault); + } } static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) @@ -533,6 +536,7 @@ void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) { bool new_vmcb12 = false; + struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; nested_vmcb02_compute_g_pat(svm); @@ -544,18 +548,18 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 } if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_SEG))) { - svm->vmcb->save.es = vmcb12->save.es; - svm->vmcb->save.cs = vmcb12->save.cs; - svm->vmcb->save.ss = vmcb12->save.ss; - svm->vmcb->save.ds = vmcb12->save.ds; - svm->vmcb->save.cpl = vmcb12->save.cpl; - vmcb_mark_dirty(svm->vmcb, VMCB_SEG); + vmcb02->save.es = vmcb12->save.es; + vmcb02->save.cs = vmcb12->save.cs; + vmcb02->save.ss = vmcb12->save.ss; + vmcb02->save.ds = vmcb12->save.ds; + vmcb02->save.cpl = vmcb12->save.cpl; + vmcb_mark_dirty(vmcb02, VMCB_SEG); } if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DT))) { - svm->vmcb->save.gdtr = vmcb12->save.gdtr; - svm->vmcb->save.idtr = vmcb12->save.idtr; - vmcb_mark_dirty(svm->vmcb, VMCB_DT); + vmcb02->save.gdtr = vmcb12->save.gdtr; + vmcb02->save.idtr = vmcb12->save.idtr; + vmcb_mark_dirty(vmcb02, VMCB_DT); } kvm_set_rflags(&svm->vcpu, vmcb12->save.rflags | X86_EFLAGS_FIXED); @@ -572,15 +576,15 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 kvm_rip_write(&svm->vcpu, vmcb12->save.rip); /* In case we don't even reach vcpu_run, the fields are not updated */ - svm->vmcb->save.rax = vmcb12->save.rax; - svm->vmcb->save.rsp = vmcb12->save.rsp; - svm->vmcb->save.rip = vmcb12->save.rip; + vmcb02->save.rax = vmcb12->save.rax; + vmcb02->save.rsp = vmcb12->save.rsp; + vmcb02->save.rip = vmcb12->save.rip; /* These bits will be set properly on the first execution when new_vmc12 is true */ if (unlikely(new_vmcb12 || vmcb_is_dirty(vmcb12, VMCB_DR))) { - svm->vmcb->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; + vmcb02->save.dr7 = svm->nested.save.dr7 | DR7_FIXED_1; svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; - vmcb_mark_dirty(svm->vmcb, VMCB_DR); + vmcb_mark_dirty(vmcb02, VMCB_DR); } } @@ -592,6 +596,8 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) const u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; struct kvm_vcpu *vcpu = &svm->vcpu; + struct vmcb *vmcb01 = svm->vmcb01.ptr; + struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; /* * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2, @@ -605,14 +611,14 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); /* Copied from vmcb01. msrpm_base can be overwritten later. */ - svm->vmcb->control.nested_ctl = svm->vmcb01.ptr->control.nested_ctl; - svm->vmcb->control.iopm_base_pa = svm->vmcb01.ptr->control.iopm_base_pa; - svm->vmcb->control.msrpm_base_pa = svm->vmcb01.ptr->control.msrpm_base_pa; + vmcb02->control.nested_ctl = vmcb01->control.nested_ctl; + vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa; + vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa; /* Done at vmrun: asid. */ /* Also overwritten later if necessary. */ - svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; + vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; /* nested_cr3. */ if (nested_npt_enabled(svm)) @@ -623,24 +629,24 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) svm->nested.ctl.tsc_offset, svm->tsc_ratio_msr); - svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; + vmcb02->control.tsc_offset = vcpu->arch.tsc_offset; if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { WARN_ON(!svm->tsc_scaling_enabled); nested_svm_update_tsc_ratio_msr(vcpu); } - svm->vmcb->control.int_ctl = + vmcb02->control.int_ctl = (svm->nested.ctl.int_ctl & int_ctl_vmcb12_bits) | - (svm->vmcb01.ptr->control.int_ctl & int_ctl_vmcb01_bits); + (vmcb01->control.int_ctl & int_ctl_vmcb01_bits); - svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; - svm->vmcb->control.int_state = svm->nested.ctl.int_state; - svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; - svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; + vmcb02->control.int_vector = svm->nested.ctl.int_vector; + vmcb02->control.int_state = svm->nested.ctl.int_state; + vmcb02->control.event_inj = svm->nested.ctl.event_inj; + vmcb02->control.event_inj_err = svm->nested.ctl.event_inj_err; if (!nested_vmcb_needs_vls_intercept(svm)) - svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; + vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; nested_svm_transition_tlb_flush(vcpu); @@ -719,6 +725,7 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) struct vmcb *vmcb12; struct kvm_host_map map; u64 vmcb12_gpa; + struct vmcb *vmcb01 = svm->vmcb01.ptr; if (!svm->nested.hsave_msr) { kvm_inject_gp(vcpu, 0); @@ -762,14 +769,14 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) * Since vmcb01 is not in use, we can use it to store some of the L1 * state. */ - svm->vmcb01.ptr->save.efer = vcpu->arch.efer; - svm->vmcb01.ptr->save.cr0 = kvm_read_cr0(vcpu); - svm->vmcb01.ptr->save.cr4 = vcpu->arch.cr4; - svm->vmcb01.ptr->save.rflags = kvm_get_rflags(vcpu); - svm->vmcb01.ptr->save.rip = kvm_rip_read(vcpu); + vmcb01->save.efer = vcpu->arch.efer; + vmcb01->save.cr0 = kvm_read_cr0(vcpu); + vmcb01->save.cr4 = vcpu->arch.cr4; + vmcb01->save.rflags = kvm_get_rflags(vcpu); + vmcb01->save.rip = kvm_rip_read(vcpu); if (!npt_enabled) - svm->vmcb01.ptr->save.cr3 = kvm_read_cr3(vcpu); + vmcb01->save.cr3 = kvm_read_cr3(vcpu); svm->nested.nested_run_pending = 1; @@ -835,8 +842,9 @@ void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) int nested_svm_vmexit(struct vcpu_svm *svm) { struct kvm_vcpu *vcpu = &svm->vcpu; + struct vmcb *vmcb01 = svm->vmcb01.ptr; + struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; struct vmcb *vmcb12; - struct vmcb *vmcb = svm->vmcb; struct kvm_host_map map; int rc; @@ -864,36 +872,36 @@ int nested_svm_vmexit(struct vcpu_svm *svm) /* Give the current vmcb to the guest */ - vmcb12->save.es = vmcb->save.es; - vmcb12->save.cs = vmcb->save.cs; - vmcb12->save.ss = vmcb->save.ss; - vmcb12->save.ds = vmcb->save.ds; - vmcb12->save.gdtr = vmcb->save.gdtr; - vmcb12->save.idtr = vmcb->save.idtr; + vmcb12->save.es = vmcb02->save.es; + vmcb12->save.cs = vmcb02->save.cs; + vmcb12->save.ss = vmcb02->save.ss; + vmcb12->save.ds = vmcb02->save.ds; + vmcb12->save.gdtr = vmcb02->save.gdtr; + vmcb12->save.idtr = vmcb02->save.idtr; vmcb12->save.efer = svm->vcpu.arch.efer; vmcb12->save.cr0 = kvm_read_cr0(vcpu); vmcb12->save.cr3 = kvm_read_cr3(vcpu); - vmcb12->save.cr2 = vmcb->save.cr2; + vmcb12->save.cr2 = vmcb02->save.cr2; vmcb12->save.cr4 = svm->vcpu.arch.cr4; vmcb12->save.rflags = kvm_get_rflags(vcpu); vmcb12->save.rip = kvm_rip_read(vcpu); vmcb12->save.rsp = kvm_rsp_read(vcpu); vmcb12->save.rax = kvm_rax_read(vcpu); - vmcb12->save.dr7 = vmcb->save.dr7; + vmcb12->save.dr7 = vmcb02->save.dr7; vmcb12->save.dr6 = svm->vcpu.arch.dr6; - vmcb12->save.cpl = vmcb->save.cpl; + vmcb12->save.cpl = vmcb02->save.cpl; - vmcb12->control.int_state = vmcb->control.int_state; - vmcb12->control.exit_code = vmcb->control.exit_code; - vmcb12->control.exit_code_hi = vmcb->control.exit_code_hi; - vmcb12->control.exit_info_1 = vmcb->control.exit_info_1; - vmcb12->control.exit_info_2 = vmcb->control.exit_info_2; + vmcb12->control.int_state = vmcb02->control.int_state; + vmcb12->control.exit_code = vmcb02->control.exit_code; + vmcb12->control.exit_code_hi = vmcb02->control.exit_code_hi; + vmcb12->control.exit_info_1 = vmcb02->control.exit_info_1; + vmcb12->control.exit_info_2 = vmcb02->control.exit_info_2; if (vmcb12->control.exit_code != SVM_EXIT_ERR) nested_save_pending_event_to_vmcb12(svm, vmcb12); if (svm->nrips_enabled) - vmcb12->control.next_rip = vmcb->control.next_rip; + vmcb12->control.next_rip = vmcb02->control.next_rip; vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; vmcb12->control.tlb_ctl = svm->nested.ctl.tlb_ctl; @@ -909,12 +917,12 @@ int nested_svm_vmexit(struct vcpu_svm *svm) * no event can be injected in L1. */ svm_set_gif(svm, false); - svm->vmcb->control.exit_int_info = 0; + vmcb01->control.exit_int_info = 0; svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; - if (svm->vmcb->control.tsc_offset != svm->vcpu.arch.tsc_offset) { - svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; - vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); + if (vmcb01->control.tsc_offset != svm->vcpu.arch.tsc_offset) { + vmcb01->control.tsc_offset = svm->vcpu.arch.tsc_offset; + vmcb_mark_dirty(vmcb01, VMCB_INTERCEPTS); } if (svm->tsc_ratio_msr != kvm_default_tsc_scaling_ratio) { @@ -928,13 +936,13 @@ int nested_svm_vmexit(struct vcpu_svm *svm) /* * Restore processor state that had been saved in vmcb01 */ - kvm_set_rflags(vcpu, svm->vmcb->save.rflags); - svm_set_efer(vcpu, svm->vmcb->save.efer); - svm_set_cr0(vcpu, svm->vmcb->save.cr0 | X86_CR0_PE); - svm_set_cr4(vcpu, svm->vmcb->save.cr4); - kvm_rax_write(vcpu, svm->vmcb->save.rax); - kvm_rsp_write(vcpu, svm->vmcb->save.rsp); - kvm_rip_write(vcpu, svm->vmcb->save.rip); + kvm_set_rflags(vcpu, vmcb01->save.rflags); + svm_set_efer(vcpu, vmcb01->save.efer); + svm_set_cr0(vcpu, vmcb01->save.cr0 | X86_CR0_PE); + svm_set_cr4(vcpu, vmcb01->save.cr4); + kvm_rax_write(vcpu, vmcb01->save.rax); + kvm_rsp_write(vcpu, vmcb01->save.rsp); + kvm_rip_write(vcpu, vmcb01->save.rip); svm->vcpu.arch.dr7 = DR7_FIXED_1; kvm_update_dr7(&svm->vcpu); @@ -952,7 +960,7 @@ int nested_svm_vmexit(struct vcpu_svm *svm) nested_svm_uninit_mmu_context(vcpu); - rc = nested_svm_load_cr3(vcpu, svm->vmcb->save.cr3, false, true); + rc = nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true); if (rc) return 1; @@ -970,7 +978,7 @@ int nested_svm_vmexit(struct vcpu_svm *svm) * right now so that it an be accounted for before we execute * L1's next instruction. */ - if (unlikely(svm->vmcb->save.rflags & X86_EFLAGS_TF)) + if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF)) kvm_queue_exception(&(svm->vcpu), DB_VECTOR); return 0; @@ -1183,12 +1191,13 @@ static bool nested_exit_on_exception(struct vcpu_svm *svm) static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) { unsigned int nr = svm->vcpu.arch.exception.nr; + struct vmcb *vmcb = svm->vmcb; - svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; - svm->vmcb->control.exit_code_hi = 0; + vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; + vmcb->control.exit_code_hi = 0; if (svm->vcpu.arch.exception.has_error_code) - svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; + vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; /* * EXITINFO2 is undefined for all exception intercepts other @@ -1196,11 +1205,11 @@ static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) */ if (nr == PF_VECTOR) { if (svm->vcpu.arch.exception.nested_apf) - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; + vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; else if (svm->vcpu.arch.exception.has_payload) - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; + vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; else - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; + vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; } else if (nr == DB_VECTOR) { /* See inject_pending_event. */ kvm_deliver_exception_payload(&svm->vcpu); From ea91559b005443ddd9ea180ffdec4b4fda21d477 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:24:49 +0200 Subject: [PATCH 030/319] KVM: x86: SVM: remove vgif_enabled() KVM always uses vgif when allowed, thus there is no need to query current vmcb for it Suggested-by: Paolo Bonzini Signed-off-by: Maxim Levitsky Message-Id: <20220322172449.235575-9-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 12 ++++++------ arch/x86/kvm/svm/svm.h | 12 ++++-------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index dd2abed45511..c39f2655ceeb 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -172,7 +172,7 @@ static int vls = true; module_param(vls, int, 0444); /* enable/disable Virtual GIF */ -static int vgif = true; +int vgif = true; module_param(vgif, int, 0444); /* enable/disable LBR virtualization */ @@ -2148,7 +2148,7 @@ void svm_set_gif(struct vcpu_svm *svm, bool value) * Likewise, clear the VINTR intercept, we will set it * again while processing KVM_REQ_EVENT if needed. */ - if (vgif_enabled(svm)) + if (vgif) svm_clr_intercept(svm, INTERCEPT_STGI); if (svm_is_intercept(svm, INTERCEPT_VINTR)) svm_clear_vintr(svm); @@ -2166,7 +2166,7 @@ void svm_set_gif(struct vcpu_svm *svm, bool value) * in use, we still rely on the VINTR intercept (rather than * STGI) to detect an open interrupt window. */ - if (!vgif_enabled(svm)) + if (!vgif) svm_clear_vintr(svm); } } @@ -3502,7 +3502,7 @@ static void svm_enable_irq_window(struct kvm_vcpu *vcpu) * enabled, the STGI interception will not occur. Enable the irq * window under the assumption that the hardware will set the GIF. */ - if (vgif_enabled(svm) || gif_set(svm)) { + if (vgif || gif_set(svm)) { /* * IRQ window is not needed when AVIC is enabled, * unless we have pending ExtINT since it cannot be injected @@ -3522,7 +3522,7 @@ static void svm_enable_nmi_window(struct kvm_vcpu *vcpu) return; /* IRET will cause a vm exit */ if (!gif_set(svm)) { - if (vgif_enabled(svm)) + if (vgif) svm_set_intercept(svm, INTERCEPT_STGI); return; /* STGI will cause a vm exit */ } @@ -4329,7 +4329,7 @@ static void svm_enable_smi_window(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); if (!gif_set(svm)) { - if (vgif_enabled(svm)) + if (vgif) svm_set_intercept(svm, INTERCEPT_STGI); /* STGI will cause a vm exit */ } else { diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 57bde9569902..aad3a678c5ab 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -33,6 +33,7 @@ #define MSRPM_OFFSETS 16 extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; extern bool npt_enabled; +extern int vgif; extern bool intercept_smi; /* @@ -453,14 +454,9 @@ static inline bool svm_is_intercept(struct vcpu_svm *svm, int bit) return vmcb_is_intercept(&svm->vmcb->control, bit); } -static inline bool vgif_enabled(struct vcpu_svm *svm) -{ - return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK); -} - static inline void enable_gif(struct vcpu_svm *svm) { - if (vgif_enabled(svm)) + if (vgif) svm->vmcb->control.int_ctl |= V_GIF_MASK; else svm->vcpu.arch.hflags |= HF_GIF_MASK; @@ -468,7 +464,7 @@ static inline void enable_gif(struct vcpu_svm *svm) static inline void disable_gif(struct vcpu_svm *svm) { - if (vgif_enabled(svm)) + if (vgif) svm->vmcb->control.int_ctl &= ~V_GIF_MASK; else svm->vcpu.arch.hflags &= ~HF_GIF_MASK; @@ -476,7 +472,7 @@ static inline void disable_gif(struct vcpu_svm *svm) static inline bool gif_set(struct vcpu_svm *svm) { - if (vgif_enabled(svm)) + if (vgif) return !!(svm->vmcb->control.int_ctl & V_GIF_MASK); else return !!(svm->vcpu.arch.hflags & HF_GIF_MASK); From 1d5a1b5860ed6f623071329112e5935db043eb55 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:40:45 +0200 Subject: [PATCH 031/319] KVM: x86: nSVM: correctly virtualize LBR msrs when L2 is running When L2 is running without LBR virtualization, we should ensure that L1's LBR msrs continue to update as usual. Signed-off-by: Maxim Levitsky Message-Id: <20220322174050.241850-2-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 12 +++++ arch/x86/kvm/svm/svm.c | 98 +++++++++++++++++++++++++++++++-------- arch/x86/kvm/svm/svm.h | 2 + 3 files changed, 93 insertions(+), 19 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 1c381c6a7b51..98647f5dec93 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -536,6 +536,7 @@ void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) { bool new_vmcb12 = false; + struct vmcb *vmcb01 = svm->vmcb01.ptr; struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; nested_vmcb02_compute_g_pat(svm); @@ -586,6 +587,9 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 svm->vcpu.arch.dr6 = svm->nested.save.dr6 | DR6_ACTIVE_LOW; vmcb_mark_dirty(vmcb02, VMCB_DR); } + + if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) + svm_copy_lbrs(vmcb02, vmcb01); } static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) @@ -645,6 +649,9 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) vmcb02->control.event_inj = svm->nested.ctl.event_inj; vmcb02->control.event_inj_err = svm->nested.ctl.event_inj_err; + vmcb02->control.virt_ext = vmcb01->control.virt_ext & + LBR_CTL_ENABLE_MASK; + if (!nested_vmcb_needs_vls_intercept(svm)) vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; @@ -912,6 +919,11 @@ int nested_svm_vmexit(struct vcpu_svm *svm) svm_switch_vmcb(svm, &svm->vmcb01); + if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) { + svm_copy_lbrs(vmcb01, vmcb02); + svm_update_lbrv(vcpu); + } + /* * On vmexit the GIF is set to false and * no event can be injected in L1. diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index c39f2655ceeb..699a367af107 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -793,6 +793,17 @@ static void init_msrpm_offsets(void) } } +void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb) +{ + to_vmcb->save.dbgctl = from_vmcb->save.dbgctl; + to_vmcb->save.br_from = from_vmcb->save.br_from; + to_vmcb->save.br_to = from_vmcb->save.br_to; + to_vmcb->save.last_excp_from = from_vmcb->save.last_excp_from; + to_vmcb->save.last_excp_to = from_vmcb->save.last_excp_to; + + vmcb_mark_dirty(to_vmcb, VMCB_LBR); +} + static void svm_enable_lbrv(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -802,6 +813,10 @@ static void svm_enable_lbrv(struct kvm_vcpu *vcpu) set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); + + /* Move the LBR msrs to the vmcb02 so that the guest can see them. */ + if (is_guest_mode(vcpu)) + svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr); } static void svm_disable_lbrv(struct kvm_vcpu *vcpu) @@ -813,6 +828,63 @@ static void svm_disable_lbrv(struct kvm_vcpu *vcpu) set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0); + + /* + * Move the LBR msrs back to the vmcb01 to avoid copying them + * on nested guest entries. + */ + if (is_guest_mode(vcpu)) + svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb); +} + +static int svm_get_lbr_msr(struct vcpu_svm *svm, u32 index) +{ + /* + * If the LBR virtualization is disabled, the LBR msrs are always + * kept in the vmcb01 to avoid copying them on nested guest entries. + * + * If nested, and the LBR virtualization is enabled/disabled, the msrs + * are moved between the vmcb01 and vmcb02 as needed. + */ + struct vmcb *vmcb = + (svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) ? + svm->vmcb : svm->vmcb01.ptr; + + switch (index) { + case MSR_IA32_DEBUGCTLMSR: + return vmcb->save.dbgctl; + case MSR_IA32_LASTBRANCHFROMIP: + return vmcb->save.br_from; + case MSR_IA32_LASTBRANCHTOIP: + return vmcb->save.br_to; + case MSR_IA32_LASTINTFROMIP: + return vmcb->save.last_excp_from; + case MSR_IA32_LASTINTTOIP: + return vmcb->save.last_excp_to; + default: + KVM_BUG(false, svm->vcpu.kvm, + "%s: Unknown MSR 0x%x", __func__, index); + return 0; + } +} + +void svm_update_lbrv(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + bool enable_lbrv = svm_get_lbr_msr(svm, MSR_IA32_DEBUGCTLMSR) & + DEBUGCTLMSR_LBR; + + bool current_enable_lbrv = !!(svm->vmcb->control.virt_ext & + LBR_CTL_ENABLE_MASK); + + if (enable_lbrv == current_enable_lbrv) + return; + + if (enable_lbrv) + svm_enable_lbrv(vcpu); + else + svm_disable_lbrv(vcpu); } void disable_nmi_singlestep(struct vcpu_svm *svm) @@ -2581,25 +2653,12 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_TSC_AUX: msr_info->data = svm->tsc_aux; break; - /* - * Nobody will change the following 5 values in the VMCB so we can - * safely return them on rdmsr. They will always be 0 until LBRV is - * implemented. - */ case MSR_IA32_DEBUGCTLMSR: - msr_info->data = svm->vmcb->save.dbgctl; - break; case MSR_IA32_LASTBRANCHFROMIP: - msr_info->data = svm->vmcb->save.br_from; - break; case MSR_IA32_LASTBRANCHTOIP: - msr_info->data = svm->vmcb->save.br_to; - break; case MSR_IA32_LASTINTFROMIP: - msr_info->data = svm->vmcb->save.last_excp_from; - break; case MSR_IA32_LASTINTTOIP: - msr_info->data = svm->vmcb->save.last_excp_to; + msr_info->data = svm_get_lbr_msr(svm, msr_info->index); break; case MSR_VM_HSAVE_PA: msr_info->data = svm->nested.hsave_msr; @@ -2845,12 +2904,13 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) if (data & DEBUGCTL_RESERVED_BITS) return 1; - svm->vmcb->save.dbgctl = data; - vmcb_mark_dirty(svm->vmcb, VMCB_LBR); - if (data & (1ULL<<0)) - svm_enable_lbrv(vcpu); + if (svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) + svm->vmcb->save.dbgctl = data; else - svm_disable_lbrv(vcpu); + svm->vmcb01.ptr->save.dbgctl = data; + + svm_update_lbrv(vcpu); + break; case MSR_VM_HSAVE_PA: /* diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index aad3a678c5ab..47d4f389bf91 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -492,6 +492,8 @@ u32 svm_msrpm_offset(u32 msr); u32 *svm_vcpu_alloc_msrpm(void); void svm_vcpu_init_msrpm(struct kvm_vcpu *vcpu, u32 *msrpm); void svm_vcpu_free_msrpm(u32 *msrpm); +void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb); +void svm_update_lbrv(struct kvm_vcpu *vcpu); int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer); void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); From d20c796ca3709801f8a7fa36e8770a3dd8ebd34e Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:40:46 +0200 Subject: [PATCH 032/319] KVM: x86: nSVM: implement nested LBR virtualization This was tested with kvm-unit-test that was developed for this purpose. Signed-off-by: Maxim Levitsky Message-Id: <20220322174050.241850-3-mlevitsk@redhat.com> [Copy all of DEBUGCTL except for reserved bits. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 20 ++++++++++++++++++-- arch/x86/kvm/svm/svm.c | 9 +++++++-- arch/x86/kvm/svm/svm.h | 3 +++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 98647f5dec93..f1332d802ec8 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -588,8 +588,18 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 vmcb_mark_dirty(vmcb02, VMCB_DR); } - if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) + if (unlikely(svm->lbrv_enabled && (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) { + /* + * Reserved bits of DEBUGCTL are ignored. Be consistent with + * svm_set_msr's definition of reserved bits. + */ + svm_copy_lbrs(vmcb02, vmcb12); + vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; + svm_update_lbrv(&svm->vcpu); + + } else if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) { svm_copy_lbrs(vmcb02, vmcb01); + } } static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) @@ -651,6 +661,9 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) vmcb02->control.virt_ext = vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK; + if (svm->lbrv_enabled) + vmcb02->control.virt_ext |= + (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK); if (!nested_vmcb_needs_vls_intercept(svm)) vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; @@ -919,7 +932,10 @@ int nested_svm_vmexit(struct vcpu_svm *svm) svm_switch_vmcb(svm, &svm->vmcb01); - if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) { + if (unlikely(svm->lbrv_enabled && (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) { + svm_copy_lbrs(vmcb12, vmcb02); + svm_update_lbrv(vcpu); + } else if (unlikely(vmcb01->control.virt_ext & LBR_CTL_ENABLE_MASK)) { svm_copy_lbrs(vmcb01, vmcb02); svm_update_lbrv(vcpu); } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 699a367af107..a6282be4e419 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -62,8 +62,6 @@ MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id); #define SEG_TYPE_LDT 2 #define SEG_TYPE_BUSY_TSS16 3 -#define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) - static bool erratum_383_found __read_mostly; u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; @@ -878,6 +876,10 @@ void svm_update_lbrv(struct kvm_vcpu *vcpu) bool current_enable_lbrv = !!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK); + if (unlikely(is_guest_mode(vcpu) && svm->lbrv_enabled)) + if (unlikely(svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK)) + enable_lbrv = true; + if (enable_lbrv == current_enable_lbrv) return; @@ -4012,6 +4014,7 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) guest_cpuid_has(vcpu, X86_FEATURE_NRIPS); svm->tsc_scaling_enabled = tsc_scaling && guest_cpuid_has(vcpu, X86_FEATURE_TSCRATEMSR); + svm->lbrv_enabled = lbrv && guest_cpuid_has(vcpu, X86_FEATURE_LBRV); svm->v_vmload_vmsave_enabled = vls && guest_cpuid_has(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD); @@ -4765,6 +4768,8 @@ static __init void svm_set_cpu_caps(void) if (vls) kvm_cpu_cap_set(X86_FEATURE_V_VMSAVE_VMLOAD); + if (lbrv) + kvm_cpu_cap_set(X86_FEATURE_LBRV); /* Nested VM can receive #VMEXIT instead of triggering #GP */ kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 47d4f389bf91..b687393e86ad 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -236,6 +236,7 @@ struct vcpu_svm { bool nrips_enabled : 1; bool tsc_scaling_enabled : 1; bool v_vmload_vmsave_enabled : 1; + bool lbrv_enabled : 1; u32 ldr_reg; u32 dfr_reg; @@ -486,6 +487,8 @@ static inline bool nested_npt_enabled(struct vcpu_svm *svm) /* svm.c */ #define MSR_INVALID 0xffffffffU +#define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) + extern bool dump_invalid_vmcb; u32 svm_msrpm_offset(u32 msr); From 74fd41ed16fd71725e69e2cb90b755505326c2e6 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:40:47 +0200 Subject: [PATCH 033/319] KVM: x86: nSVM: support PAUSE filtering when L0 doesn't intercept PAUSE Expose the pause filtering and threshold in the guest CPUID and support PAUSE filtering when possible: - If the L0 doesn't intercept PAUSE (cpu_pm=on), then allow L1 to have full control over PAUSE filtering. - if the L1 doesn't intercept PAUSE, use host values and update the adaptive count/threshold even when running nested. - Otherwise always exit to L1; it is not really possible to merge the fields correctly. It is expected that in this case, userspace will not enable this feature in the guest CPUID, to avoid having the guest update both fields pointlessly. Signed-off-by: Maxim Levitsky Message-Id: <20220322174050.241850-4-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 26 ++++++++++++++++++++++++++ arch/x86/kvm/svm/svm.c | 22 +++++++++++++++++++--- arch/x86/kvm/svm/svm.h | 2 ++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index f1332d802ec8..fe0011771b10 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -668,6 +668,29 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) if (!nested_vmcb_needs_vls_intercept(svm)) vmcb02->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; + if (kvm_pause_in_guest(svm->vcpu.kvm)) { + /* use guest values since host doesn't use them */ + vmcb02->control.pause_filter_count = + svm->pause_filter_enabled ? + svm->nested.ctl.pause_filter_count : 0; + + vmcb02->control.pause_filter_thresh = + svm->pause_threshold_enabled ? + svm->nested.ctl.pause_filter_thresh : 0; + + } else if (!vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_PAUSE)) { + /* use host values when guest doesn't use them */ + vmcb02->control.pause_filter_count = vmcb01->control.pause_filter_count; + vmcb02->control.pause_filter_thresh = vmcb01->control.pause_filter_thresh; + } else { + /* + * Intercept every PAUSE otherwise and + * ignore both host and guest values + */ + vmcb02->control.pause_filter_count = 0; + vmcb02->control.pause_filter_thresh = 0; + } + nested_svm_transition_tlb_flush(vcpu); /* Enter Guest-Mode */ @@ -928,6 +951,9 @@ int nested_svm_vmexit(struct vcpu_svm *svm) vmcb12->control.event_inj = svm->nested.ctl.event_inj; vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; + if (!kvm_pause_in_guest(vcpu->kvm) && vmcb02->control.pause_filter_count) + vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count; + nested_svm_copy_common_state(svm->nested.vmcb02.ptr, svm->vmcb01.ptr); svm_switch_vmcb(svm, &svm->vmcb01); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index a6282be4e419..30b00a87bc98 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -908,6 +908,9 @@ static void grow_ple_window(struct kvm_vcpu *vcpu) struct vmcb_control_area *control = &svm->vmcb->control; int old = control->pause_filter_count; + if (kvm_pause_in_guest(vcpu->kvm) || !old) + return; + control->pause_filter_count = __grow_ple_window(old, pause_filter_count, pause_filter_count_grow, @@ -926,6 +929,9 @@ static void shrink_ple_window(struct kvm_vcpu *vcpu) struct vmcb_control_area *control = &svm->vmcb->control; int old = control->pause_filter_count; + if (kvm_pause_in_guest(vcpu->kvm) || !old) + return; + control->pause_filter_count = __shrink_ple_window(old, pause_filter_count, @@ -2982,7 +2988,6 @@ static int interrupt_window_interception(struct kvm_vcpu *vcpu) static int pause_interception(struct kvm_vcpu *vcpu) { bool in_kernel; - /* * CPL is not made available for an SEV-ES guest, therefore * vcpu->arch.preempted_in_kernel can never be true. Just @@ -2990,8 +2995,7 @@ static int pause_interception(struct kvm_vcpu *vcpu) */ in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0; - if (!kvm_pause_in_guest(vcpu->kvm)) - grow_ple_window(vcpu); + grow_ple_window(vcpu); kvm_vcpu_on_spin(vcpu, in_kernel); return kvm_skip_emulated_instruction(vcpu); @@ -4018,6 +4022,12 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) svm->v_vmload_vmsave_enabled = vls && guest_cpuid_has(vcpu, X86_FEATURE_V_VMSAVE_VMLOAD); + svm->pause_filter_enabled = kvm_cpu_cap_has(X86_FEATURE_PAUSEFILTER) && + guest_cpuid_has(vcpu, X86_FEATURE_PAUSEFILTER); + + svm->pause_threshold_enabled = kvm_cpu_cap_has(X86_FEATURE_PFTHRESHOLD) && + guest_cpuid_has(vcpu, X86_FEATURE_PFTHRESHOLD); + svm_recalc_instruction_intercepts(vcpu, svm); /* For sev guests, the memory encryption bit is not reserved in CR3. */ @@ -4771,6 +4781,12 @@ static __init void svm_set_cpu_caps(void) if (lbrv) kvm_cpu_cap_set(X86_FEATURE_LBRV); + if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) + kvm_cpu_cap_set(X86_FEATURE_PAUSEFILTER); + + if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) + kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD); + /* Nested VM can receive #VMEXIT instead of triggering #GP */ kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index b687393e86ad..50b95e8115dd 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -237,6 +237,8 @@ struct vcpu_svm { bool tsc_scaling_enabled : 1; bool v_vmload_vmsave_enabled : 1; bool lbrv_enabled : 1; + bool pause_filter_enabled : 1; + bool pause_threshold_enabled : 1; u32 ldr_reg; u32 dfr_reg; From 0b349662184ba76b335af84579f3ed50baf54607 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:40:48 +0200 Subject: [PATCH 034/319] KVM: x86: nSVM: implement nested vGIF In case L1 enables vGIF for L2, the L2 cannot affect L1's GIF, regardless of STGI/CLGI intercepts, and since VM entry enables GIF, this means that L1's GIF is always 1 while L2 is running. Thus in this case leave L1's vGIF in vmcb01, while letting L2 control the vGIF thus implementing nested vGIF. Also allow KVM to toggle L1's GIF during nested entry/exit by always using vmcb01. Signed-off-by: Maxim Levitsky Message-Id: <20220322174050.241850-5-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 15 +++++++++++---- arch/x86/kvm/svm/svm.c | 5 +++++ arch/x86/kvm/svm/svm.h | 35 +++++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index fe0011771b10..a99a34bf0305 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -439,6 +439,10 @@ void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) */ mask &= ~V_IRQ_MASK; } + + if (nested_vgif_enabled(svm)) + mask |= V_GIF_MASK; + svm->nested.ctl.int_ctl &= ~mask; svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; } @@ -604,10 +608,8 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) { - const u32 int_ctl_vmcb01_bits = - V_INTR_MASKING_MASK | V_GIF_MASK | V_GIF_ENABLE_MASK; - - const u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; + u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK; + u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; struct kvm_vcpu *vcpu = &svm->vcpu; struct vmcb *vmcb01 = svm->vmcb01.ptr; @@ -624,6 +626,11 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) */ WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); + if (svm->vgif_enabled && (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK)) + int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); + else + int_ctl_vmcb01_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); + /* Copied from vmcb01. msrpm_base can be overwritten later. */ vmcb02->control.nested_ctl = vmcb01->control.nested_ctl; vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 30b00a87bc98..772efdff46b3 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4028,6 +4028,8 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) svm->pause_threshold_enabled = kvm_cpu_cap_has(X86_FEATURE_PFTHRESHOLD) && guest_cpuid_has(vcpu, X86_FEATURE_PFTHRESHOLD); + svm->vgif_enabled = vgif && guest_cpuid_has(vcpu, X86_FEATURE_VGIF); + svm_recalc_instruction_intercepts(vcpu, svm); /* For sev guests, the memory encryption bit is not reserved in CR3. */ @@ -4787,6 +4789,9 @@ static __init void svm_set_cpu_caps(void) if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD); + if (vgif) + kvm_cpu_cap_set(X86_FEATURE_VGIF); + /* Nested VM can receive #VMEXIT instead of triggering #GP */ kvm_cpu_cap_set(X86_FEATURE_SVME_ADDR_CHK); } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 50b95e8115dd..6b822f790106 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -239,6 +239,7 @@ struct vcpu_svm { bool lbrv_enabled : 1; bool pause_filter_enabled : 1; bool pause_threshold_enabled : 1; + bool vgif_enabled : 1; u32 ldr_reg; u32 dfr_reg; @@ -457,26 +458,48 @@ static inline bool svm_is_intercept(struct vcpu_svm *svm, int bit) return vmcb_is_intercept(&svm->vmcb->control, bit); } +static inline bool nested_vgif_enabled(struct vcpu_svm *svm) +{ + return svm->vgif_enabled && (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK); +} + +static inline struct vmcb *get_vgif_vmcb(struct vcpu_svm *svm) +{ + if (!vgif) + return NULL; + + if (is_guest_mode(&svm->vcpu) && !nested_vgif_enabled(svm)) + return svm->nested.vmcb02.ptr; + else + return svm->vmcb01.ptr; +} + static inline void enable_gif(struct vcpu_svm *svm) { - if (vgif) - svm->vmcb->control.int_ctl |= V_GIF_MASK; + struct vmcb *vmcb = get_vgif_vmcb(svm); + + if (vmcb) + vmcb->control.int_ctl |= V_GIF_MASK; else svm->vcpu.arch.hflags |= HF_GIF_MASK; } static inline void disable_gif(struct vcpu_svm *svm) { - if (vgif) - svm->vmcb->control.int_ctl &= ~V_GIF_MASK; + struct vmcb *vmcb = get_vgif_vmcb(svm); + + if (vmcb) + vmcb->control.int_ctl &= ~V_GIF_MASK; else svm->vcpu.arch.hflags &= ~HF_GIF_MASK; } static inline bool gif_set(struct vcpu_svm *svm) { - if (vgif) - return !!(svm->vmcb->control.int_ctl & V_GIF_MASK); + struct vmcb *vmcb = get_vgif_vmcb(svm); + + if (vmcb) + return !!(vmcb->control.int_ctl & V_GIF_MASK); else return !!(svm->vcpu.arch.hflags & HF_GIF_MASK); } From d5fa597ed87047117dc62ce1d38ba1d007442359 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:40:49 +0200 Subject: [PATCH 035/319] KVM: x86: allow per cpu apicv inhibit reasons Add optional callback .vcpu_get_apicv_inhibit_reasons returning extra inhibit reasons that prevent APICv from working on this vCPU. Suggested-by: Paolo Bonzini Signed-off-by: Maxim Levitsky Message-Id: <20220322174050.241850-6-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm-x86-ops.h | 1 + arch/x86/include/asm/kvm_host.h | 6 ++++++ arch/x86/kvm/x86.c | 13 +++++++++++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 3c368b639c04..96e4e9842dfc 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -126,6 +126,7 @@ KVM_X86_OP_OPTIONAL(migrate_timers) KVM_X86_OP(msr_filter_changed) KVM_X86_OP(complete_emulated_msr) KVM_X86_OP(vcpu_deliver_sipi_vector) +KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons); #undef KVM_X86_OP #undef KVM_X86_OP_OPTIONAL diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index fac990cc189d..676705ad1e23 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1507,6 +1507,11 @@ struct kvm_x86_ops { int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err); void (*vcpu_deliver_sipi_vector)(struct kvm_vcpu *vcpu, u8 vector); + + /* + * Returns vCPU specific APICv inhibit reasons + */ + unsigned long (*vcpu_get_apicv_inhibit_reasons)(struct kvm_vcpu *vcpu); }; struct kvm_x86_nested_ops { @@ -1807,6 +1812,7 @@ gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception); bool kvm_apicv_activated(struct kvm *kvm); +bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu); void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu); void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, enum kvm_apicv_inhibit reason, bool set); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index dad13fd6b61e..cb9c0197c414 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9114,6 +9114,14 @@ bool kvm_apicv_activated(struct kvm *kvm) } EXPORT_SYMBOL_GPL(kvm_apicv_activated); +bool kvm_vcpu_apicv_activated(struct kvm_vcpu *vcpu) +{ + ulong vm_reasons = READ_ONCE(vcpu->kvm->arch.apicv_inhibit_reasons); + ulong vcpu_reasons = static_call(kvm_x86_vcpu_get_apicv_inhibit_reasons)(vcpu); + + return (vm_reasons | vcpu_reasons) == 0; +} +EXPORT_SYMBOL_GPL(kvm_vcpu_apicv_activated); static void set_or_clear_apicv_inhibit(unsigned long *inhibits, enum kvm_apicv_inhibit reason, bool set) @@ -9799,7 +9807,8 @@ void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) down_read(&vcpu->kvm->arch.apicv_update_lock); - activate = kvm_apicv_activated(vcpu->kvm); + activate = kvm_vcpu_apicv_activated(vcpu); + if (vcpu->arch.apicv_active == activate) goto out; @@ -10200,7 +10209,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) * per-VM state, and responsing vCPUs must wait for the update * to complete before servicing KVM_REQ_APICV_UPDATE. */ - WARN_ON_ONCE(kvm_apicv_activated(vcpu->kvm) != kvm_vcpu_apicv_active(vcpu)); + WARN_ON_ONCE(kvm_vcpu_apicv_activated(vcpu) != kvm_vcpu_apicv_active(vcpu)); exit_fastpath = static_call(kvm_x86_vcpu_run)(vcpu); if (likely(exit_fastpath != EXIT_FASTPATH_REENTER_GUEST)) From f44509f849fe55bbd81bd3f099b9aecd19002243 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:40:50 +0200 Subject: [PATCH 036/319] KVM: x86: SVM: allow AVIC to co-exist with a nested guest running Inhibit the AVIC of the vCPU that is running nested for the duration of the nested run, so that all interrupts arriving from both its vCPU siblings and from KVM are delivered using normal IPIs and cause that vCPU to vmexit. Note that unlike normal AVIC inhibition, there is no need to update the AVIC mmio memslot, because the nested guest uses its own set of paging tables. That also means that AVIC doesn't need to be inhibited VM wide. Signed-off-by: Maxim Levitsky Message-Id: <20220322174050.241850-7-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/avic.c | 7 +++++++ arch/x86/kvm/svm/nested.c | 16 ++++++++++------ arch/x86/kvm/svm/svm.c | 30 +++++++++++++++++++----------- arch/x86/kvm/svm/svm.h | 1 + 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index b7dabc01d644..6c81abe3153b 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -356,6 +356,13 @@ int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) return 1; } +unsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu) +{ + if (is_guest_mode(vcpu)) + return APICV_INHIBIT_REASON_NESTED; + return 0; +} + static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat) { struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index a99a34bf0305..73b545278f5f 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -620,12 +620,6 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) * exit_int_info, exit_int_info_err, next_rip, insn_len, insn_bytes. */ - /* - * Also covers avic_vapic_bar, avic_backing_page, avic_logical_id, - * avic_physical_id. - */ - WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); - if (svm->vgif_enabled && (svm->nested.ctl.int_ctl & V_GIF_ENABLE_MASK)) int_ctl_vmcb12_bits |= (V_GIF_MASK | V_GIF_ENABLE_MASK); else @@ -765,6 +759,9 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, svm_set_gif(svm, true); + if (kvm_vcpu_apicv_active(vcpu)) + kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); + return 0; } @@ -1042,6 +1039,13 @@ int nested_svm_vmexit(struct vcpu_svm *svm) if (unlikely(vmcb01->save.rflags & X86_EFLAGS_TF)) kvm_queue_exception(&(svm->vcpu), DB_VECTOR); + /* + * Un-inhibit the AVIC right away, so that other vCPUs can start + * to benefit from it right away. + */ + if (kvm_apicv_activated(vcpu->kvm)) + kvm_vcpu_update_apicv(vcpu); + return 0; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 772efdff46b3..95b26dbfd561 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1466,7 +1466,7 @@ static void svm_set_vintr(struct vcpu_svm *svm) /* * The following fields are ignored when AVIC is enabled */ - WARN_ON(kvm_apicv_activated(svm->vcpu.kvm)); + WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu)); svm_set_intercept(svm, INTERCEPT_VINTR); @@ -2975,9 +2975,16 @@ static int interrupt_window_interception(struct kvm_vcpu *vcpu) svm_clear_vintr(to_svm(vcpu)); /* - * For AVIC, the only reason to end up here is ExtINTs. + * If not running nested, for AVIC, the only reason to end up here is ExtINTs. * In this case AVIC was temporarily disabled for * requesting the IRQ window and we have to re-enable it. + * + * If running nested, still remove the VM wide AVIC inhibit to + * support case in which the interrupt window was requested when the + * vCPU was not running nested. + + * All vCPUs which run still run nested, will remain to have their + * AVIC still inhibited due to per-cpu AVIC inhibition. */ kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); @@ -3572,10 +3579,16 @@ static void svm_enable_irq_window(struct kvm_vcpu *vcpu) /* * IRQ window is not needed when AVIC is enabled, * unless we have pending ExtINT since it cannot be injected - * via AVIC. In such case, we need to temporarily disable AVIC, + * via AVIC. In such case, KVM needs to temporarily disable AVIC, * and fallback to injecting IRQ via V_IRQ. + * + * If running nested, AVIC is already locally inhibited + * on this vCPU, therefore there is no need to request + * the VM wide AVIC inhibition. */ - kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); + if (!is_guest_mode(vcpu)) + kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); + svm_set_vintr(svm); } } @@ -4046,13 +4059,6 @@ static void svm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) */ if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC)) kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_X2APIC); - - /* - * Currently, AVIC does not work with nested virtualization. - * So, we disable AVIC when cpuid for SVM is set in the L1 guest. - */ - if (nested && guest_cpuid_has(vcpu, X86_FEATURE_SVM)) - kvm_set_apicv_inhibit(kvm, APICV_INHIBIT_REASON_NESTED); } init_vmcb_after_set_cpuid(vcpu); } @@ -4715,6 +4721,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .complete_emulated_msr = svm_complete_emulated_msr, .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, + .vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons, }; /* @@ -4916,6 +4923,7 @@ static __init int svm_hardware_setup(void) } else { svm_x86_ops.vcpu_blocking = NULL; svm_x86_ops.vcpu_unblocking = NULL; + svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL; } if (vls) { diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 6b822f790106..e246793cbeae 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -624,6 +624,7 @@ int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq, void avic_vcpu_blocking(struct kvm_vcpu *vcpu); void avic_vcpu_unblocking(struct kvm_vcpu *vcpu); void avic_ring_doorbell(struct kvm_vcpu *vcpu); +unsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu); /* sev.c */ From 945024d764a18b1484428f0055439072ea58e349 Mon Sep 17 00:00:00 2001 From: Jon Kohler Date: Wed, 23 Mar 2022 20:44:39 -0400 Subject: [PATCH 037/319] KVM: x86: optimize PKU branching in kvm_load_{guest|host}_xsave_state kvm_load_{guest|host}_xsave_state handles xsave on vm entry and exit, part of which is managing memory protection key state. The latest arch.pkru is updated with a rdpkru, and if that doesn't match the base host_pkru (which about 70% of the time), we issue a __write_pkru. To improve performance, implement the following optimizations: 1. Reorder if conditions prior to wrpkru in both kvm_load_{guest|host}_xsave_state. Flip the ordering of the || condition so that XFEATURE_MASK_PKRU is checked first, which when instrumented in our environment appeared to be always true and less overall work than kvm_read_cr4_bits. For kvm_load_guest_xsave_state, hoist arch.pkru != host_pkru ahead one position. When instrumented, I saw this be true roughly ~70% of the time vs the other conditions which were almost always true. With this change, we will avoid 3rd condition check ~30% of the time. 2. Wrap PKU sections with CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS, as if the user compiles out this feature, we should not have these branches at all. Signed-off-by: Jon Kohler Message-Id: <20220324004439.6709-1-jon@nutanix.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index cb9c0197c414..7a066cf92692 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -961,11 +961,13 @@ void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu) wrmsrl(MSR_IA32_XSS, vcpu->arch.ia32_xss); } +#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS if (static_cpu_has(X86_FEATURE_PKU) && - (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || - (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU)) && - vcpu->arch.pkru != vcpu->arch.host_pkru) + vcpu->arch.pkru != vcpu->arch.host_pkru && + ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || + kvm_read_cr4_bits(vcpu, X86_CR4_PKE))) write_pkru(vcpu->arch.pkru); +#endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */ } EXPORT_SYMBOL_GPL(kvm_load_guest_xsave_state); @@ -974,13 +976,15 @@ void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu) if (vcpu->arch.guest_state_protected) return; +#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS if (static_cpu_has(X86_FEATURE_PKU) && - (kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || - (vcpu->arch.xcr0 & XFEATURE_MASK_PKRU))) { + ((vcpu->arch.xcr0 & XFEATURE_MASK_PKRU) || + kvm_read_cr4_bits(vcpu, X86_CR4_PKE))) { vcpu->arch.pkru = rdpkru(); if (vcpu->arch.pkru != vcpu->arch.host_pkru) write_pkru(vcpu->arch.host_pkru); } +#endif /* CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS */ if (kvm_read_cr4_bits(vcpu, X86_CR4_OSXSAVE)) { From 8d5678a76689acbf91245a3791fe853ab773090f Mon Sep 17 00:00:00 2001 From: Hou Wenlong Date: Tue, 15 Mar 2022 17:35:13 +0800 Subject: [PATCH 038/319] KVM: x86/mmu: Don't rebuild page when the page is synced and no tlb flushing is required Before Commit c3e5e415bc1e6 ("KVM: X86: Change kvm_sync_page() to return true when remote flush is needed"), the return value of kvm_sync_page() indicates whether the page is synced, and kvm_mmu_get_page() would rebuild page when the sync fails. But now, kvm_sync_page() returns false when the page is synced and no tlb flushing is required, which leads to rebuild page in kvm_mmu_get_page(). So return the return value of mmu->sync_page() directly and check it in kvm_mmu_get_page(). If the sync fails, the page will be zapped and the invalid_list is not empty, so set flush as true is accepted in mmu_sync_children(). Cc: stable@vger.kernel.org Fixes: c3e5e415bc1e6 ("KVM: X86: Change kvm_sync_page() to return true when remote flush is needed") Signed-off-by: Hou Wenlong Acked-by: Lai Jiangshan Message-Id: <0dabeeb789f57b0d793f85d073893063e692032d.1647336064.git.houwenlong.hwl@antgroup.com> [mmu_sync_children should not flush if the page is zapped. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8f19ea752704..dbf46dd98618 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1866,17 +1866,14 @@ static void kvm_mmu_commit_zap_page(struct kvm *kvm, &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)]) \ if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else -static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, +static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, struct list_head *invalid_list) { int ret = vcpu->arch.mmu->sync_page(vcpu, sp); - if (ret < 0) { + if (ret < 0) kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); - return false; - } - - return !!ret; + return ret; } static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm, @@ -1998,7 +1995,7 @@ static int mmu_sync_children(struct kvm_vcpu *vcpu, for_each_sp(pages, sp, parents, i) { kvm_unlink_unsync_page(vcpu->kvm, sp); - flush |= kvm_sync_page(vcpu, sp, &invalid_list); + flush |= kvm_sync_page(vcpu, sp, &invalid_list) > 0; mmu_pages_clear_parents(&parents); } if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) { @@ -2039,6 +2036,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, struct hlist_head *sp_list; unsigned quadrant; struct kvm_mmu_page *sp; + int ret; int collisions = 0; LIST_HEAD(invalid_list); @@ -2091,11 +2089,13 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, * If the sync fails, the page is zapped. If so, break * in order to rebuild it. */ - if (!kvm_sync_page(vcpu, sp, &invalid_list)) + ret = kvm_sync_page(vcpu, sp, &invalid_list); + if (ret < 0) break; WARN_ON(!list_empty(&invalid_list)); - kvm_flush_remote_tlbs(vcpu->kvm); + if (ret > 0) + kvm_flush_remote_tlbs(vcpu->kvm); } __clear_sp_write_flooding_count(sp); From 265a3bf486d4aac913d1bcbe81b01e271328a265 Mon Sep 17 00:00:00 2001 From: Kunihiko Hayashi Date: Wed, 30 Mar 2022 14:17:03 +0900 Subject: [PATCH 039/319] dt-bindings: gpio: uniphier: Add hogs parsing Allow parsing GPIO controller children nodes with GPIO hogs to fix the following warning: uniphier-ld11-ref.dtb: gpio@55000000: 'xirq0-hog' does not match any of the regexes: 'pinctrl-[0-9]+' From schema: Documentation/devicetree/bindings/gpio/socionext,uniphier-gpio.yaml Signed-off-by: Kunihiko Hayashi Reviewed-by: Krzysztof Kozlowski Signed-off-by: Bartosz Golaszewski --- .../bindings/gpio/socionext,uniphier-gpio.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Documentation/devicetree/bindings/gpio/socionext,uniphier-gpio.yaml b/Documentation/devicetree/bindings/gpio/socionext,uniphier-gpio.yaml index bcafa494ed7a..228fa27ffdc3 100644 --- a/Documentation/devicetree/bindings/gpio/socionext,uniphier-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/socionext,uniphier-gpio.yaml @@ -52,6 +52,23 @@ properties: triplets. $ref: /schemas/types.yaml#/definitions/uint32-matrix +patternProperties: + "^.+-hog(-[0-9]+)?$": + type: object + properties: + gpio-hog: true + gpios: true + input: true + output-high: true + output-low: true + line-name: true + + required: + - gpio-hog + - gpios + + additionalProperties: false + required: - compatible - reg From 1cef8b5019769d46725932eeace7a383bca97905 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Mar 2022 18:06:20 +0300 Subject: [PATCH 040/319] gpiolib: Get rid of redundant 'else' In the snippets like the following if (...) return / goto / break / continue ...; else ... the 'else' is redundant. Get rid of it. In case of IOCTLs use switch-case pattern that seems the usual in such cases. While at it, clarify necessity of else in gpiod_direction_output() by attaching else if to the closing curly brace on a previous line. Signed-off-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-cdev.c | 66 ++++++++++++++++++++----------------- drivers/gpio/gpiolib.c | 12 +++---- 2 files changed, 40 insertions(+), 38 deletions(-) diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index ffa0256cad5a..c2900b1be69d 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -197,16 +197,15 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, void __user *ip = (void __user *)arg; struct gpiohandle_data ghd; DECLARE_BITMAP(vals, GPIOHANDLES_MAX); - int i; + unsigned int i; + int ret; - if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) { - /* NOTE: It's ok to read values of output lines. */ - int ret = gpiod_get_array_value_complex(false, - true, - lh->num_descs, - lh->descs, - NULL, - vals); + switch (cmd) { + case GPIOHANDLE_GET_LINE_VALUES_IOCTL: + /* NOTE: It's okay to read values of output lines */ + ret = gpiod_get_array_value_complex(false, true, + lh->num_descs, lh->descs, + NULL, vals); if (ret) return ret; @@ -218,7 +217,7 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, return -EFAULT; return 0; - } else if (cmd == GPIOHANDLE_SET_LINE_VALUES_IOCTL) { + case GPIOHANDLE_SET_LINE_VALUES_IOCTL: /* * All line descriptors were created at once with the same * flags so just check if the first one is really output. @@ -240,10 +239,11 @@ static long linehandle_ioctl(struct file *file, unsigned int cmd, lh->descs, NULL, vals); - } else if (cmd == GPIOHANDLE_SET_CONFIG_IOCTL) { + case GPIOHANDLE_SET_CONFIG_IOCTL: return linehandle_set_config(lh, ip); + default: + return -EINVAL; } - return -EINVAL; } #ifdef CONFIG_COMPAT @@ -1188,14 +1188,16 @@ static long linereq_ioctl(struct file *file, unsigned int cmd, struct linereq *lr = file->private_data; void __user *ip = (void __user *)arg; - if (cmd == GPIO_V2_LINE_GET_VALUES_IOCTL) + switch (cmd) { + case GPIO_V2_LINE_GET_VALUES_IOCTL: return linereq_get_values(lr, ip); - else if (cmd == GPIO_V2_LINE_SET_VALUES_IOCTL) + case GPIO_V2_LINE_SET_VALUES_IOCTL: return linereq_set_values(lr, ip); - else if (cmd == GPIO_V2_LINE_SET_CONFIG_IOCTL) + case GPIO_V2_LINE_SET_CONFIG_IOCTL: return linereq_set_config(lr, ip); - - return -EINVAL; + default: + return -EINVAL; + } } #ifdef CONFIG_COMPAT @@ -2113,28 +2115,30 @@ static long gpio_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return -ENODEV; /* Fill in the struct and pass to userspace */ - if (cmd == GPIO_GET_CHIPINFO_IOCTL) { + switch (cmd) { + case GPIO_GET_CHIPINFO_IOCTL: return chipinfo_get(cdev, ip); #ifdef CONFIG_GPIO_CDEV_V1 - } else if (cmd == GPIO_GET_LINEHANDLE_IOCTL) { + case GPIO_GET_LINEHANDLE_IOCTL: return linehandle_create(gdev, ip); - } else if (cmd == GPIO_GET_LINEEVENT_IOCTL) { + case GPIO_GET_LINEEVENT_IOCTL: return lineevent_create(gdev, ip); - } else if (cmd == GPIO_GET_LINEINFO_IOCTL || - cmd == GPIO_GET_LINEINFO_WATCH_IOCTL) { - return lineinfo_get_v1(cdev, ip, - cmd == GPIO_GET_LINEINFO_WATCH_IOCTL); + case GPIO_GET_LINEINFO_IOCTL: + return lineinfo_get_v1(cdev, ip, false); + case GPIO_GET_LINEINFO_WATCH_IOCTL: + return lineinfo_get_v1(cdev, ip, true); #endif /* CONFIG_GPIO_CDEV_V1 */ - } else if (cmd == GPIO_V2_GET_LINEINFO_IOCTL || - cmd == GPIO_V2_GET_LINEINFO_WATCH_IOCTL) { - return lineinfo_get(cdev, ip, - cmd == GPIO_V2_GET_LINEINFO_WATCH_IOCTL); - } else if (cmd == GPIO_V2_GET_LINE_IOCTL) { + case GPIO_V2_GET_LINEINFO_IOCTL: + return lineinfo_get(cdev, ip, false); + case GPIO_V2_GET_LINEINFO_WATCH_IOCTL: + return lineinfo_get(cdev, ip, true); + case GPIO_V2_GET_LINE_IOCTL: return linereq_create(gdev, ip); - } else if (cmd == GPIO_GET_LINEINFO_UNWATCH_IOCTL) { + case GPIO_GET_LINEINFO_UNWATCH_IOCTL: return lineinfo_unwatch(cdev, ip); + default: + return -EINVAL; } - return -EINVAL; } #ifdef CONFIG_COMPAT diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index e59884cc12a7..3ffe7b6cebbf 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -189,9 +189,8 @@ static int gpiochip_find_base(int ngpio) /* found a free space? */ if (gdev->base + gdev->ngpio <= base) break; - else - /* nope, check the space right before the chip */ - base = gdev->base - ngpio; + /* nope, check the space right before the chip */ + base = gdev->base - ngpio; } if (gpio_is_valid(base)) { @@ -2401,8 +2400,7 @@ int gpiod_direction_output(struct gpio_desc *desc, int value) ret = gpiod_direction_input(desc); goto set_output_flag; } - } - else if (test_bit(FLAG_OPEN_SOURCE, &desc->flags)) { + } else if (test_bit(FLAG_OPEN_SOURCE, &desc->flags)) { ret = gpio_set_config(desc, PIN_CONFIG_DRIVE_OPEN_SOURCE); if (!ret) goto set_output_value; @@ -2559,9 +2557,9 @@ static int gpiod_get_raw_value_commit(const struct gpio_desc *desc) static int gpio_chip_get_multiple(struct gpio_chip *gc, unsigned long *mask, unsigned long *bits) { - if (gc->get_multiple) { + if (gc->get_multiple) return gc->get_multiple(gc, mask, bits); - } else if (gc->get) { + if (gc->get) { int i, value; for_each_set_bit(i, mask, gc->ngpio) { From afd24a50c73c8fa6619fee093ceee2ab40bb2146 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 4 Apr 2022 17:29:48 +0200 Subject: [PATCH 041/319] dt-bindings: gpio: renesas,rcar-gpio: Add r8a779f0 support Document support for the GPIO controller blocks in the Renesas R-Car S4-8 (R8A779F0) SoC, including a new family-specific compatible value for the R-Car Gen4 family. Signed-off-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Signed-off-by: Bartosz Golaszewski --- .../devicetree/bindings/gpio/renesas,rcar-gpio.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml index f2541739ee3b..0681a4790cd6 100644 --- a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml @@ -51,6 +51,11 @@ properties: - items: - const: renesas,gpio-r8a779a0 # R-Car V3U + - items: + - enum: + - renesas,gpio-r8a779f0 # R-Car S4-8 + - const: renesas,rcar-gen4-gpio # R-Car Gen4 + reg: maxItems: 1 From 43ebbb92e43fc9d85f6ff8b2a01c20ab5cf08678 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 4 Apr 2022 17:29:49 +0200 Subject: [PATCH 042/319] gpio: rcar: Add R-Car Gen4 support R-Car V3U (R8A779A0) was the first member of the R-Car Gen4 family. Generalize the support for R-Car V3U to other SoCs in the R-Car Gen4 family by adding a family-specific compatible value. Signed-off-by: Geert Uytterhoeven Reviewed-by: Wolfram Sang Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-rcar.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c index 3a76538f27fa..356aac4de17c 100644 --- a/drivers/gpio/gpio-rcar.c +++ b/drivers/gpio/gpio-rcar.c @@ -411,7 +411,7 @@ static const struct gpio_rcar_info gpio_rcar_info_gen3 = { .has_inen = false, }; -static const struct gpio_rcar_info gpio_rcar_info_v3u = { +static const struct gpio_rcar_info gpio_rcar_info_gen4 = { .has_outdtsel = true, .has_both_edge_trigger = true, .has_always_in = true, @@ -421,7 +421,7 @@ static const struct gpio_rcar_info gpio_rcar_info_v3u = { static const struct of_device_id gpio_rcar_of_table[] = { { .compatible = "renesas,gpio-r8a779a0", - .data = &gpio_rcar_info_v3u, + .data = &gpio_rcar_info_gen4, }, { .compatible = "renesas,rcar-gen1-gpio", .data = &gpio_rcar_info_gen1, @@ -431,6 +431,9 @@ static const struct of_device_id gpio_rcar_of_table[] = { }, { .compatible = "renesas,rcar-gen3-gpio", .data = &gpio_rcar_info_gen3, + }, { + .compatible = "renesas,rcar-gen4-gpio", + .data = &gpio_rcar_info_gen4, }, { .compatible = "renesas,gpio-rcar", .data = &gpio_rcar_info_gen1, From a60707d74bd1d119cf7bcc5101cda912fc46d5e3 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 15 Feb 2022 19:45:57 +0800 Subject: [PATCH 043/319] sched: Move child_runs_first sysctls to fair.c move child_runs_first sysctls to fair.c and use the new register_sysctl_init() to register the sysctl interface. Signed-off-by: Zhen Ni Signed-off-by: Luis Chamberlain --- include/linux/sched/sysctl.h | 2 -- kernel/sched/fair.c | 19 +++++++++++++++++++ kernel/sched/sched.h | 2 ++ kernel/sysctl.c | 7 ------- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index c1076b5e17fb..1d2ff3cd1728 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -14,8 +14,6 @@ extern unsigned long sysctl_hung_task_timeout_secs; enum { sysctl_hung_task_timeout_secs = 0 }; #endif -extern unsigned int sysctl_sched_child_runs_first; - enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, SCHED_TUNABLESCALING_LOG, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d4bd299d67ab..788b1d6a3248 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -109,6 +109,25 @@ static unsigned int sched_nr_latency = 8; * parent will (try to) run first. */ unsigned int sysctl_sched_child_runs_first __read_mostly; +#ifdef CONFIG_SYSCTL +static struct ctl_table sched_child_runs_first_sysctls[] = { + { + .procname = "sched_child_runs_first", + .data = &sysctl_sched_child_runs_first, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + {} +}; + +static int __init sched_child_runs_first_sysctl_init(void) +{ + register_sysctl_init("kernel", sched_child_runs_first_sysctls); + return 0; +} +late_initcall(sched_child_runs_first_sysctl_init); +#endif /* * SCHED_OTHER wake-up granularity. diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 58263f90c559..767fc1de9646 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -108,6 +108,8 @@ extern __read_mostly int scheduler_running; extern unsigned long calc_load_update; extern atomic_long_t calc_load_tasks; +extern unsigned int sysctl_sched_child_runs_first; + extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq, long adjust); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 830aaf8ca08e..6bbb8e1af675 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1659,13 +1659,6 @@ int proc_do_static_key(struct ctl_table *table, int write, } static struct ctl_table kern_table[] = { - { - .procname = "sched_child_runs_first", - .data = &sysctl_sched_child_runs_first, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, #ifdef CONFIG_SCHEDSTATS { .procname = "sched_schedstats", From f5ef06d58be8311a9425e6a54a053ecb350952f3 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 15 Feb 2022 19:45:58 +0800 Subject: [PATCH 044/319] sched: Move schedstats sysctls to core.c move schedstats sysctls to core.c and use the new register_sysctl_init() to register the sysctl interface. Signed-off-by: Zhen Ni Signed-off-by: Luis Chamberlain --- include/linux/sched/sysctl.h | 2 -- kernel/sched/core.c | 22 +++++++++++++++++++++- kernel/sysctl.c | 11 ----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 1d2ff3cd1728..6c7a6850559b 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -64,8 +64,6 @@ int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int sysctl_numa_balancing(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); -int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) extern unsigned int sysctl_sched_energy_aware; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d575b4914925..04440da5955f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4430,7 +4430,7 @@ static int __init setup_schedstats(char *str) __setup("schedstats=", setup_schedstats); #ifdef CONFIG_PROC_SYSCTL -int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, +static int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { struct ctl_table t; @@ -4449,6 +4449,26 @@ int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, set_schedstats(state); return err; } + +static struct ctl_table sched_schedstats_sysctls[] = { + { + .procname = "sched_schedstats", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_schedstats, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + {} +}; + +static int __init sched_schedstats_sysctl_init(void) +{ + register_sysctl_init("kernel", sched_schedstats_sysctls); + return 0; +} +late_initcall(sched_schedstats_sysctl_init); #endif /* CONFIG_PROC_SYSCTL */ #endif /* CONFIG_SCHEDSTATS */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 6bbb8e1af675..fc0eeca20718 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1659,17 +1659,6 @@ int proc_do_static_key(struct ctl_table *table, int write, } static struct ctl_table kern_table[] = { -#ifdef CONFIG_SCHEDSTATS - { - .procname = "sched_schedstats", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sysctl_schedstats, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, -#endif /* CONFIG_SCHEDSTATS */ #ifdef CONFIG_TASK_DELAY_ACCT { .procname = "task_delayacct", From d9ab0e63fa7f8405fbb19e28c5191e0880a7f2db Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 15 Feb 2022 19:45:59 +0800 Subject: [PATCH 045/319] sched: Move rt_period/runtime sysctls to rt.c move rt_period/runtime sysctls to rt.c and use the new register_sysctl_init() to register the sysctl interface. Signed-off-by: Zhen Ni Signed-off-by: Luis Chamberlain --- include/linux/sched/sysctl.h | 11 --------- kernel/rcu/rcu.h | 2 ++ kernel/sched/core.c | 13 ----------- kernel/sched/rt.c | 43 +++++++++++++++++++++++++++++++++++- kernel/sched/sched.h | 4 ++++ kernel/sysctl.c | 14 ------------ 6 files changed, 48 insertions(+), 39 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 6c7a6850559b..4391c1307945 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -31,15 +31,6 @@ extern int sysctl_numa_balancing_mode; #define sysctl_numa_balancing_mode 0 #endif -/* - * control realtime throttling: - * - * /proc/sys/kernel/sched_rt_period_us - * /proc/sys/kernel/sched_rt_runtime_us - */ -extern unsigned int sysctl_sched_rt_period; -extern int sysctl_sched_rt_runtime; - extern unsigned int sysctl_sched_dl_period_max; extern unsigned int sysctl_sched_dl_period_min; @@ -58,8 +49,6 @@ extern int sched_rr_timeslice; int sched_rr_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); -int sched_rt_handler(struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int sysctl_numa_balancing(struct ctl_table *table, int write, void *buffer, diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 24b5f2c2de87..7812c740b3bf 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -23,6 +23,8 @@ #define RCU_SEQ_CTR_SHIFT 2 #define RCU_SEQ_STATE_MASK ((1 << RCU_SEQ_CTR_SHIFT) - 1) +extern int sysctl_sched_rt_runtime; + /* * Return the counter portion of a sequence number previously returned * by rcu_seq_snap() or rcu_seq_current(). diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 04440da5955f..774f3229db37 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -145,12 +145,6 @@ const_debug unsigned int sysctl_sched_nr_migrate = 8; const_debug unsigned int sysctl_sched_nr_migrate = 32; #endif -/* - * period over which we measure -rt task CPU usage in us. - * default: 1s - */ -unsigned int sysctl_sched_rt_period = 1000000; - __read_mostly int scheduler_running; #ifdef CONFIG_SCHED_CORE @@ -444,13 +438,6 @@ sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { } #endif /* CONFIG_SCHED_CORE */ -/* - * part of the period that we allow rt tasks to run in us. - * default: 0.95s - */ -int sysctl_sched_rt_runtime = 950000; - - /* * Serialization rules: * diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index a32c46889af8..5663bb5ff890 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -13,6 +13,47 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); struct rt_bandwidth def_rt_bandwidth; +/* + * period over which we measure -rt task CPU usage in us. + * default: 1s + */ +unsigned int sysctl_sched_rt_period = 1000000; + +/* + * part of the period that we allow rt tasks to run in us. + * default: 0.95s + */ +int sysctl_sched_rt_runtime = 950000; + +static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos); +#ifdef CONFIG_SYSCTL +static struct ctl_table sched_rt_sysctls[] = { + { + .procname = "sched_rt_period_us", + .data = &sysctl_sched_rt_period, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_rt_handler, + }, + { + .procname = "sched_rt_runtime_us", + .data = &sysctl_sched_rt_runtime, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = sched_rt_handler, + }, + {} +}; + +static int __init sched_rt_sysctl_init(void) +{ + register_sysctl_init("kernel", sched_rt_sysctls); + return 0; +} +late_initcall(sched_rt_sysctl_init); +#endif + static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer) { struct rt_bandwidth *rt_b = @@ -2925,7 +2966,7 @@ static void sched_rt_do_global(void) raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); } -int sched_rt_handler(struct ctl_table *table, int write, void *buffer, +static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { int old_period, old_runtime; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 767fc1de9646..3b406c78a8e9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -114,6 +114,10 @@ extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq, long adjust); extern void call_trace_sched_update_nr_running(struct rq *rq, int count); + +extern unsigned int sysctl_sched_rt_period; +extern int sysctl_sched_rt_runtime; + /* * Helpers for converting nanosecond timing to jiffy resolution */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index fc0eeca20718..029bfe06c68d 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1681,20 +1681,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_FOUR, }, #endif /* CONFIG_NUMA_BALANCING */ - { - .procname = "sched_rt_period_us", - .data = &sysctl_sched_rt_period, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_rt_handler, - }, - { - .procname = "sched_rt_runtime_us", - .data = &sysctl_sched_rt_runtime, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = sched_rt_handler, - }, { .procname = "sched_deadline_period_max_us", .data = &sysctl_sched_dl_period_max, From 84227c12888b1105725cd2de14705b029bcbb4b2 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 15 Feb 2022 19:46:00 +0800 Subject: [PATCH 046/319] sched: Move deadline_period sysctls to deadline.c move deadline_period sysctls to deadline.c and use the new register_sysctl_init() to register the sysctl interface. Signed-off-by: Zhen Ni Signed-off-by: Luis Chamberlain --- include/linux/sched/sysctl.h | 3 --- kernel/sched/deadline.c | 42 +++++++++++++++++++++++++++++------- kernel/sysctl.c | 14 ------------ 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 4391c1307945..7da9b94c5e1c 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -31,9 +31,6 @@ extern int sysctl_numa_balancing_mode; #define sysctl_numa_balancing_mode 0 #endif -extern unsigned int sysctl_sched_dl_period_max; -extern unsigned int sysctl_sched_dl_period_min; - #ifdef CONFIG_UCLAMP_TASK extern unsigned int sysctl_sched_uclamp_util_min; extern unsigned int sysctl_sched_uclamp_util_max; diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index fb4255ae0b2c..82e10b74a7b2 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -16,6 +16,40 @@ * Fabio Checconi */ +/* + * Default limits for DL period; on the top end we guard against small util + * tasks still getting ridiculously long effective runtimes, on the bottom end we + * guard against timer DoS. + */ +static unsigned int sysctl_sched_dl_period_max = 1 << 22; /* ~4 seconds */ +static unsigned int sysctl_sched_dl_period_min = 100; /* 100 us */ +#ifdef CONFIG_SYSCTL +static struct ctl_table sched_dl_sysctls[] = { + { + .procname = "sched_deadline_period_max_us", + .data = &sysctl_sched_dl_period_max, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "sched_deadline_period_min_us", + .data = &sysctl_sched_dl_period_min, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + {} +}; + +static int __init sched_dl_sysctl_init(void) +{ + register_sysctl_init("kernel", sched_dl_sysctls); + return 0; +} +late_initcall(sched_dl_sysctl_init); +#endif + static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) { return container_of(dl_se, struct task_struct, dl); @@ -2879,14 +2913,6 @@ void __getparam_dl(struct task_struct *p, struct sched_attr *attr) attr->sched_flags |= dl_se->flags; } -/* - * Default limits for DL period; on the top end we guard against small util - * tasks still getting ridiculously long effective runtimes, on the bottom end we - * guard against timer DoS. - */ -unsigned int sysctl_sched_dl_period_max = 1 << 22; /* ~4 seconds */ -unsigned int sysctl_sched_dl_period_min = 100; /* 100 us */ - /* * This function validates the new parameters of a -deadline task. * We ask for the deadline not being zero, and greater or equal diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 029bfe06c68d..2c8c75e11a37 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1681,20 +1681,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_FOUR, }, #endif /* CONFIG_NUMA_BALANCING */ - { - .procname = "sched_deadline_period_max_us", - .data = &sysctl_sched_dl_period_max, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { - .procname = "sched_deadline_period_min_us", - .data = &sysctl_sched_dl_period_min, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, { .procname = "sched_rr_timeslice_ms", .data = &sysctl_sched_rr_timeslice, From dafd7a9dad22fadcb290b24dff54e2eae3b89776 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 15 Feb 2022 19:46:01 +0800 Subject: [PATCH 047/319] sched: Move rr_timeslice sysctls to rt.c move rr_timeslice sysctls to rt.c and use the new register_sysctl_init() to register the sysctl interface. Signed-off-by: Zhen Ni Signed-off-by: Luis Chamberlain --- include/linux/sched/sysctl.h | 5 ----- kernel/sched/rt.c | 13 +++++++++++-- kernel/sched/sched.h | 1 + kernel/sysctl.c | 7 ------- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 7da9b94c5e1c..3d307e512d1f 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -41,11 +41,6 @@ extern unsigned int sysctl_sched_uclamp_util_min_rt_default; extern unsigned int sysctl_sched_cfs_bandwidth_slice; #endif -extern int sysctl_sched_rr_timeslice; -extern int sched_rr_timeslice; - -int sched_rr_handler(struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int sysctl_numa_balancing(struct ctl_table *table, int write, void *buffer, diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 5663bb5ff890..71791be36065 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -5,7 +5,7 @@ */ int sched_rr_timeslice = RR_TIMESLICE; -int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE; +static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE; /* More than 4 hours if BW_SHIFT equals 20. */ static const u64 max_rt_runtime = MAX_BW; @@ -27,6 +27,8 @@ int sysctl_sched_rt_runtime = 950000; static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); +static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos); #ifdef CONFIG_SYSCTL static struct ctl_table sched_rt_sysctls[] = { { @@ -43,6 +45,13 @@ static struct ctl_table sched_rt_sysctls[] = { .mode = 0644, .proc_handler = sched_rt_handler, }, + { + .procname = "sched_rr_timeslice_ms", + .data = &sysctl_sched_rr_timeslice, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = sched_rr_handler, + }, {} }; @@ -3005,7 +3014,7 @@ static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, return ret; } -int sched_rr_handler(struct ctl_table *table, int write, void *buffer, +static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { int ret; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3b406c78a8e9..ae0f6e5a76f9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -117,6 +117,7 @@ extern void call_trace_sched_update_nr_running(struct rq *rq, int count); extern unsigned int sysctl_sched_rt_period; extern int sysctl_sched_rt_runtime; +extern int sched_rr_timeslice; /* * Helpers for converting nanosecond timing to jiffy resolution diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 2c8c75e11a37..b074f70a3e11 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1681,13 +1681,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_FOUR, }, #endif /* CONFIG_NUMA_BALANCING */ - { - .procname = "sched_rr_timeslice_ms", - .data = &sysctl_sched_rr_timeslice, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = sched_rr_handler, - }, #ifdef CONFIG_UCLAMP_TASK { .procname = "sched_util_clamp_min", From 28f152cd0926596e69d412467b11b6fe6fe4e864 Mon Sep 17 00:00:00 2001 From: Baisong Zhong Date: Fri, 18 Mar 2022 10:54:17 +0800 Subject: [PATCH 048/319] sched/rt: fix build error when CONFIG_SYSCTL is disable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoid random build errors which do not select CONFIG_SYSCTL by depending on it in Kconfig. This fixes the following warning: In file included from kernel/sched/build_policy.c:43: At top level: kernel/sched/rt.c:3017:12: error: ‘sched_rr_handler’ defined but not used [-Werror=unused-function] 3017 | static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, | ^~~~~~~~~~~~~~~~ kernel/sched/rt.c:2978:12: error: ‘sched_rt_handler’ defined but not used [-Werror=unused-function] 2978 | static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, | ^~~~~~~~~~~~~~~~ cc1: all warnings being treated as errors make[2]: *** [scripts/Makefile.build:310: kernel/sched/build_policy.o] Error 1 make[1]: *** [scripts/Makefile.build:638: kernel/sched] Error 2 make[1]: *** Waiting for unfinished jobs.... Reported-by: Hulk Robot Signed-off-by: Baisong Zhong [mcgrof: small build fix, we need sched_rt_can_attach() even when CONFIG_SYSCTL is disabled] Signed-off-by: Luis Chamberlain --- kernel/sched/rt.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 71791be36065..b491a0f8c25d 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -5,7 +5,6 @@ */ int sched_rr_timeslice = RR_TIMESLICE; -static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE; /* More than 4 hours if BW_SHIFT equals 20. */ static const u64 max_rt_runtime = MAX_BW; @@ -25,11 +24,12 @@ unsigned int sysctl_sched_rt_period = 1000000; */ int sysctl_sched_rt_runtime = 950000; +#ifdef CONFIG_SYSCTL +static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE; static int sched_rt_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); -#ifdef CONFIG_SYSCTL static struct ctl_table sched_rt_sysctls[] = { { .procname = "sched_rt_period_us", @@ -2911,6 +2911,7 @@ long sched_group_rt_period(struct task_group *tg) return rt_period_us; } +#ifdef CONFIG_SYSCTL static int sched_rt_global_constraints(void) { int ret = 0; @@ -2921,6 +2922,7 @@ static int sched_rt_global_constraints(void) return ret; } +#endif /* CONFIG_SYSCTL */ int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) { @@ -2932,6 +2934,8 @@ int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) } #else /* !CONFIG_RT_GROUP_SCHED */ + +#ifdef CONFIG_SYSCTL static int sched_rt_global_constraints(void) { unsigned long flags; @@ -2949,8 +2953,10 @@ static int sched_rt_global_constraints(void) return 0; } +#endif /* CONFIG_SYSCTL */ #endif /* CONFIG_RT_GROUP_SCHED */ +#ifdef CONFIG_SYSCTL static int sched_rt_global_validate(void) { if (sysctl_sched_rt_period <= 0) @@ -3035,6 +3041,7 @@ static int sched_rr_handler(struct ctl_table *table, int write, void *buffer, return ret; } +#endif /* CONFIG_SYSCTL */ #ifdef CONFIG_SCHED_DEBUG void print_rt_stats(struct seq_file *m, int cpu) From 3267e0156c3341ac25b37a0f60551cdae1634b60 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 15 Feb 2022 19:46:02 +0800 Subject: [PATCH 049/319] sched: Move uclamp_util sysctls to core.c move uclamp_util sysctls to core.c and use the new register_sysctl_init() to register the sysctl interface. Signed-off-by: Zhen Ni Signed-off-by: Luis Chamberlain --- include/linux/sched/sysctl.h | 8 ------ kernel/sched/core.c | 48 +++++++++++++++++++++++++++--------- kernel/sysctl.c | 23 ----------------- 3 files changed, 37 insertions(+), 42 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 3d307e512d1f..0934b21a57a4 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -31,18 +31,10 @@ extern int sysctl_numa_balancing_mode; #define sysctl_numa_balancing_mode 0 #endif -#ifdef CONFIG_UCLAMP_TASK -extern unsigned int sysctl_sched_uclamp_util_min; -extern unsigned int sysctl_sched_uclamp_util_max; -extern unsigned int sysctl_sched_uclamp_util_min_rt_default; -#endif - #ifdef CONFIG_CFS_BANDWIDTH extern unsigned int sysctl_sched_cfs_bandwidth_slice; #endif -int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); int sysctl_numa_balancing(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 774f3229db37..ef31751c5799 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1306,10 +1306,10 @@ static void set_load_weight(struct task_struct *p, bool update_load) static DEFINE_MUTEX(uclamp_mutex); /* Max allowed minimum utilization */ -unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; +static unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; /* Max allowed maximum utilization */ -unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; +static unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; /* * By default RT tasks run at the maximum performance point/capacity of the @@ -1326,7 +1326,7 @@ unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; * This knob will not override the system default sched_util_clamp_min defined * above. */ -unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; +static unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE; /* All clamps are required to be less or equal than these values */ static struct uclamp_se uclamp_default[UCLAMP_CNT]; @@ -1779,7 +1779,7 @@ static void uclamp_update_root_tg(void) static void uclamp_update_root_tg(void) { } #endif -int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, +static int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { bool update_root_tg = false; @@ -4436,8 +4436,12 @@ static int sysctl_schedstats(struct ctl_table *table, int write, void *buffer, set_schedstats(state); return err; } +#endif /* CONFIG_PROC_SYSCTL */ +#endif /* CONFIG_SCHEDSTATS */ -static struct ctl_table sched_schedstats_sysctls[] = { +#ifdef CONFIG_SYSCTL +static struct ctl_table sched_core_sysctls[] = { +#ifdef CONFIG_SCHEDSTATS { .procname = "sched_schedstats", .data = NULL, @@ -4447,17 +4451,39 @@ static struct ctl_table sched_schedstats_sysctls[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, +#endif /* CONFIG_SCHEDSTATS */ +#ifdef CONFIG_UCLAMP_TASK + { + .procname = "sched_util_clamp_min", + .data = &sysctl_sched_uclamp_util_min, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_sched_uclamp_handler, + }, + { + .procname = "sched_util_clamp_max", + .data = &sysctl_sched_uclamp_util_max, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_sched_uclamp_handler, + }, + { + .procname = "sched_util_clamp_min_rt_default", + .data = &sysctl_sched_uclamp_util_min_rt_default, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_sched_uclamp_handler, + }, +#endif /* CONFIG_UCLAMP_TASK */ {} }; - -static int __init sched_schedstats_sysctl_init(void) +static int __init sched_core_sysctl_init(void) { - register_sysctl_init("kernel", sched_schedstats_sysctls); + register_sysctl_init("kernel", sched_core_sysctls); return 0; } -late_initcall(sched_schedstats_sysctl_init); -#endif /* CONFIG_PROC_SYSCTL */ -#endif /* CONFIG_SCHEDSTATS */ +late_initcall(sched_core_sysctl_init); +#endif /* CONFIG_SYSCTL */ /* * fork()/clone()-time setup: diff --git a/kernel/sysctl.c b/kernel/sysctl.c index b074f70a3e11..a48c090d57f9 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1681,29 +1681,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_FOUR, }, #endif /* CONFIG_NUMA_BALANCING */ -#ifdef CONFIG_UCLAMP_TASK - { - .procname = "sched_util_clamp_min", - .data = &sysctl_sched_uclamp_util_min, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sysctl_sched_uclamp_handler, - }, - { - .procname = "sched_util_clamp_max", - .data = &sysctl_sched_uclamp_util_max, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sysctl_sched_uclamp_handler, - }, - { - .procname = "sched_util_clamp_min_rt_default", - .data = &sysctl_sched_uclamp_util_min_rt_default, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sysctl_sched_uclamp_handler, - }, -#endif #ifdef CONFIG_CFS_BANDWIDTH { .procname = "sched_cfs_bandwidth_slice_us", From d4ae80ffa64f87b9c355692b680b603add084e96 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 15 Feb 2022 19:46:03 +0800 Subject: [PATCH 050/319] sched: Move cfs_bandwidth_slice sysctls to fair.c move cfs_bandwidth_slice sysctls to fair.c and use the new register_sysctl_init() to register the sysctl interface. Signed-off-by: Zhen Ni Signed-off-by: Luis Chamberlain --- include/linux/sched/sysctl.h | 4 --- kernel/sched/fair.c | 51 ++++++++++++++++++++++-------------- kernel/sysctl.c | 10 ------- 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 0934b21a57a4..198f77c8a873 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -31,10 +31,6 @@ extern int sysctl_numa_balancing_mode; #define sysctl_numa_balancing_mode 0 #endif -#ifdef CONFIG_CFS_BANDWIDTH -extern unsigned int sysctl_sched_cfs_bandwidth_slice; -#endif - int sysctl_numa_balancing(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 788b1d6a3248..265bf7a75a37 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -109,25 +109,6 @@ static unsigned int sched_nr_latency = 8; * parent will (try to) run first. */ unsigned int sysctl_sched_child_runs_first __read_mostly; -#ifdef CONFIG_SYSCTL -static struct ctl_table sched_child_runs_first_sysctls[] = { - { - .procname = "sched_child_runs_first", - .data = &sysctl_sched_child_runs_first, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - {} -}; - -static int __init sched_child_runs_first_sysctl_init(void) -{ - register_sysctl_init("kernel", sched_child_runs_first_sysctls); - return 0; -} -late_initcall(sched_child_runs_first_sysctl_init); -#endif /* * SCHED_OTHER wake-up granularity. @@ -192,7 +173,37 @@ int __weak arch_asym_cpu_priority(int cpu) * * (default: 5 msec, units: microseconds) */ -unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; +static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL; +#endif + +#ifdef CONFIG_SYSCTL +static struct ctl_table sched_fair_sysctls[] = { + { + .procname = "sched_child_runs_first", + .data = &sysctl_sched_child_runs_first, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#ifdef CONFIG_CFS_BANDWIDTH + { + .procname = "sched_cfs_bandwidth_slice_us", + .data = &sysctl_sched_cfs_bandwidth_slice, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + }, +#endif + {} +}; + +static int __init sched_fair_sysctl_init(void) +{ + register_sysctl_init("kernel", sched_fair_sysctls); + return 0; +} +late_initcall(sched_fair_sysctl_init); #endif static inline void update_load_add(struct load_weight *lw, unsigned long inc) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index a48c090d57f9..10ab81c7c457 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1681,16 +1681,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_FOUR, }, #endif /* CONFIG_NUMA_BALANCING */ -#ifdef CONFIG_CFS_BANDWIDTH - { - .procname = "sched_cfs_bandwidth_slice_us", - .data = &sysctl_sched_cfs_bandwidth_slice, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ONE, - }, -#endif #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) { .procname = "sched_energy_aware", From 8a0441415b3f9b9a920a6a5086580ea3daa7b884 Mon Sep 17 00:00:00 2001 From: Zhen Ni Date: Tue, 15 Feb 2022 19:46:04 +0800 Subject: [PATCH 051/319] sched: Move energy_aware sysctls to topology.c move energy_aware sysctls to topology.c and use the new register_sysctl_init() to register the sysctl interface. Signed-off-by: Zhen Ni Signed-off-by: Luis Chamberlain --- include/linux/sched/sysctl.h | 6 ------ kernel/sched/topology.c | 25 +++++++++++++++++++++++-- kernel/sysctl.c | 11 ----------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 198f77c8a873..e650946816d0 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -34,10 +34,4 @@ extern int sysctl_numa_balancing_mode; int sysctl_numa_balancing(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); -#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) -extern unsigned int sysctl_sched_energy_aware; -int sched_energy_aware_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); -#endif - #endif /* _LINUX_SCHED_SYSCTL_H */ diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 810750e62118..05b6c2ad90b9 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -206,7 +206,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) DEFINE_STATIC_KEY_FALSE(sched_energy_present); -unsigned int sysctl_sched_energy_aware = 1; +static unsigned int sysctl_sched_energy_aware = 1; DEFINE_MUTEX(sched_energy_mutex); bool sched_energy_update; @@ -220,7 +220,7 @@ void rebuild_sched_domains_energy(void) } #ifdef CONFIG_PROC_SYSCTL -int sched_energy_aware_handler(struct ctl_table *table, int write, +static int sched_energy_aware_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { int ret, state; @@ -237,6 +237,27 @@ int sched_energy_aware_handler(struct ctl_table *table, int write, return ret; } + +static struct ctl_table sched_energy_aware_sysctls[] = { + { + .procname = "sched_energy_aware", + .data = &sysctl_sched_energy_aware, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sched_energy_aware_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + {} +}; + +static int __init sched_energy_aware_sysctl_init(void) +{ + register_sysctl_init("kernel", sched_energy_aware_sysctls); + return 0; +} + +late_initcall(sched_energy_aware_sysctl_init); #endif static void free_pd(struct perf_domain *pd) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 10ab81c7c457..8241c5401ee8 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1681,17 +1681,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_FOUR, }, #endif /* CONFIG_NUMA_BALANCING */ -#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL) - { - .procname = "sched_energy_aware", - .data = &sysctl_sched_energy_aware, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sched_energy_aware_handler, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, -#endif #ifdef CONFIG_PROVE_LOCKING { .procname = "prove_locking", From 06d177662fb86b80c7fc2290667b9a14cb0bd925 Mon Sep 17 00:00:00 2001 From: tangmeng Date: Thu, 17 Feb 2022 12:23:21 +0800 Subject: [PATCH 052/319] kernel/reboot: move reboot sysctls to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. All filesystem syctls now get reviewed by fs folks. This commit follows the commit of fs, move the poweroff_cmd and ctrl-alt-del sysctls to its own file, kernel/reboot.c. Signed-off-by: tangmeng Signed-off-by: Luis Chamberlain --- include/linux/reboot.h | 4 ---- kernel/reboot.c | 34 ++++++++++++++++++++++++++++++++-- kernel/sysctl.c | 14 -------------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/include/linux/reboot.h b/include/linux/reboot.h index af907a3d68d1..a2429648d831 100644 --- a/include/linux/reboot.h +++ b/include/linux/reboot.h @@ -71,12 +71,8 @@ extern void kernel_restart(char *cmd); extern void kernel_halt(void); extern void kernel_power_off(void); -extern int C_A_D; /* for sysctl */ void ctrl_alt_del(void); -#define POWEROFF_CMD_PATH_LEN 256 -extern char poweroff_cmd[POWEROFF_CMD_PATH_LEN]; - extern void orderly_poweroff(bool force); extern void orderly_reboot(void); void hw_protection_shutdown(const char *reason, int ms_until_forced); diff --git a/kernel/reboot.c b/kernel/reboot.c index 6bcc5d6a6572..ed4e6dfb7d44 100644 --- a/kernel/reboot.c +++ b/kernel/reboot.c @@ -23,7 +23,7 @@ * this indicates whether you can reboot with ctrl-alt-del: the default is yes */ -int C_A_D = 1; +static int C_A_D = 1; struct pid *cad_pid; EXPORT_SYMBOL(cad_pid); @@ -417,9 +417,37 @@ void ctrl_alt_del(void) kill_cad_pid(SIGINT, 1); } -char poweroff_cmd[POWEROFF_CMD_PATH_LEN] = "/sbin/poweroff"; +#define POWEROFF_CMD_PATH_LEN 256 +static char poweroff_cmd[POWEROFF_CMD_PATH_LEN] = "/sbin/poweroff"; static const char reboot_cmd[] = "/sbin/reboot"; +#ifdef CONFIG_SYSCTL +static struct ctl_table kern_reboot_table[] = { + { + .procname = "poweroff_cmd", + .data = &poweroff_cmd, + .maxlen = POWEROFF_CMD_PATH_LEN, + .mode = 0644, + .proc_handler = proc_dostring, + }, + { + .procname = "ctrl-alt-del", + .data = &C_A_D, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { } +}; + +static void __init kernel_reboot_sysctls_init(void) +{ + register_sysctl_init("kernel", kern_reboot_table); +} +#else +#define kernel_reboot_sysctls_init() do { } while (0) +#endif /* CONFIG_SYSCTL */ + static int run_cmd(const char *cmd) { char **argv; @@ -886,6 +914,8 @@ static int __init reboot_ksysfs_init(void) return ret; } + kernel_reboot_sysctls_init(); + return 0; } late_initcall(reboot_ksysfs_init); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 8241c5401ee8..5e43569ce2be 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1798,13 +1798,6 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec, }, #endif - { - .procname = "ctrl-alt-del", - .data = &C_A_D, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, #ifdef CONFIG_FUNCTION_TRACER { .procname = "ftrace_enabled", @@ -2111,13 +2104,6 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec, }, #endif - { - .procname = "poweroff_cmd", - .data = &poweroff_cmd, - .maxlen = POWEROFF_CMD_PATH_LEN, - .mode = 0644, - .proc_handler = proc_dostring, - }, #ifdef CONFIG_KEYS { .procname = "keys", From 43fe219aa56a2fdd8f0623c9470a32b14b0617a5 Mon Sep 17 00:00:00 2001 From: sujiaxun Date: Thu, 17 Feb 2022 18:51:48 -0800 Subject: [PATCH 053/319] mm: move oom_kill sysctls to their own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the oom_kill sysctls to their own file, mm/oom_kill.c [sfr@canb.auug.org.au: null-terminate the array] Link: https://lkml.kernel.org/r/20220216193202.28838626@canb.auug.org.au Link: https://lkml.kernel.org/r/20220215093203.31032-1-sujiaxun@uniontech.com Signed-off-by: sujiaxun Signed-off-by: Stephen Rothwell Cc: Kees Cook Cc: Iurii Zaikin Cc: Michal Hocko Signed-off-by: Andrew Morton Signed-off-by: Luis Chamberlain --- include/linux/oom.h | 4 ---- kernel/sysctl.c | 23 ----------------------- mm/oom_kill.c | 38 +++++++++++++++++++++++++++++++++++--- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/include/linux/oom.h b/include/linux/oom.h index 2db9a1432511..02d1e7bbd8cd 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -123,8 +123,4 @@ extern void oom_killer_enable(void); extern struct task_struct *find_lock_task_mm(struct task_struct *p); -/* sysctls */ -extern int sysctl_oom_dump_tasks; -extern int sysctl_oom_kill_allocating_task; -extern int sysctl_panic_on_oom; #endif /* _INCLUDE_LINUX_OOM_H */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 5e43569ce2be..a21c0ea396f3 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2241,29 +2241,6 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_TWO, }, - { - .procname = "panic_on_oom", - .data = &sysctl_panic_on_oom, - .maxlen = sizeof(sysctl_panic_on_oom), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_TWO, - }, - { - .procname = "oom_kill_allocating_task", - .data = &sysctl_oom_kill_allocating_task, - .maxlen = sizeof(sysctl_oom_kill_allocating_task), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { - .procname = "oom_dump_tasks", - .data = &sysctl_oom_dump_tasks, - .maxlen = sizeof(sysctl_oom_dump_tasks), - .mode = 0644, - .proc_handler = proc_dointvec, - }, { .procname = "overcommit_ratio", .data = &sysctl_overcommit_ratio, diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 7ec38194f8e1..7cc338a9e9e4 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -52,9 +52,38 @@ #define CREATE_TRACE_POINTS #include -int sysctl_panic_on_oom; -int sysctl_oom_kill_allocating_task; -int sysctl_oom_dump_tasks = 1; +static int sysctl_panic_on_oom; +static int sysctl_oom_kill_allocating_task; +static int sysctl_oom_dump_tasks = 1; + +#ifdef CONFIG_SYSCTL +static struct ctl_table vm_oom_kill_table[] = { + { + .procname = "panic_on_oom", + .data = &sysctl_panic_on_oom, + .maxlen = sizeof(sysctl_panic_on_oom), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, + { + .procname = "oom_kill_allocating_task", + .data = &sysctl_oom_kill_allocating_task, + .maxlen = sizeof(sysctl_oom_kill_allocating_task), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "oom_dump_tasks", + .data = &sysctl_oom_dump_tasks, + .maxlen = sizeof(sysctl_oom_dump_tasks), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + {} +}; +#endif /* * Serializes oom killer invocations (out_of_memory()) from all contexts to @@ -677,6 +706,9 @@ static void wake_oom_reaper(struct task_struct *tsk) static int __init oom_init(void) { oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper"); +#ifdef CONFIG_SYSCTL + register_sysctl_init("vm", vm_oom_kill_table); +#endif return 0; } subsys_initcall(oom_init) From aa779e5102195e1d9ade95dcbc0bfbd8f916eb59 Mon Sep 17 00:00:00 2001 From: zhanglianjie Date: Thu, 17 Feb 2022 18:51:51 -0800 Subject: [PATCH 054/319] mm: move page-writeback sysctls to their own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. So move the page-writeback sysctls to its own file. [akpm@linux-foundation.org: coding-style cleanups] akpm@linux-foundation.org: fix CONFIG_SYSCTL=n warnings] Link: https://lkml.kernel.org/r/20220129012955.26594-1-zhanglianjie@uniontech.com Signed-off-by: zhanglianjie Cc: Kees Cook Cc: Iurii Zaikin Cc: Luis Chamberlain Signed-off-by: Andrew Morton Signed-off-by: Luis Chamberlain --- include/linux/writeback.h | 15 ------ kernel/sysctl.c | 69 ------------------------- mm/page-writeback.c | 103 ++++++++++++++++++++++++++++++++++---- 3 files changed, 93 insertions(+), 94 deletions(-) diff --git a/include/linux/writeback.h b/include/linux/writeback.h index fec248ab1fec..dc2b94e6a94f 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -345,28 +345,13 @@ void wb_domain_exit(struct wb_domain *dom); extern struct wb_domain global_wb_domain; /* These are exported to sysctl. */ -extern int dirty_background_ratio; -extern unsigned long dirty_background_bytes; -extern int vm_dirty_ratio; -extern unsigned long vm_dirty_bytes; extern unsigned int dirty_writeback_interval; extern unsigned int dirty_expire_interval; extern unsigned int dirtytime_expire_interval; -extern int vm_highmem_is_dirtyable; extern int laptop_mode; -int dirty_background_ratio_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); -int dirty_background_bytes_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); -int dirty_ratio_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); -int dirty_bytes_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); int dirtytime_interval_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); -int dirty_writeback_centisecs_handler(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty); unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index a21c0ea396f3..36bfe1c92d44 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -100,8 +100,6 @@ static const int six_hundred_forty_kb = 640 * 1024; #endif -/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */ -static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE; static const int ngroups_max = NGROUPS_MAX; static const int cap_last_cap = CAP_LAST_CAP; @@ -2263,55 +2261,6 @@ static struct ctl_table vm_table[] = { .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, }, - { - .procname = "dirty_background_ratio", - .data = &dirty_background_ratio, - .maxlen = sizeof(dirty_background_ratio), - .mode = 0644, - .proc_handler = dirty_background_ratio_handler, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE_HUNDRED, - }, - { - .procname = "dirty_background_bytes", - .data = &dirty_background_bytes, - .maxlen = sizeof(dirty_background_bytes), - .mode = 0644, - .proc_handler = dirty_background_bytes_handler, - .extra1 = SYSCTL_LONG_ONE, - }, - { - .procname = "dirty_ratio", - .data = &vm_dirty_ratio, - .maxlen = sizeof(vm_dirty_ratio), - .mode = 0644, - .proc_handler = dirty_ratio_handler, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE_HUNDRED, - }, - { - .procname = "dirty_bytes", - .data = &vm_dirty_bytes, - .maxlen = sizeof(vm_dirty_bytes), - .mode = 0644, - .proc_handler = dirty_bytes_handler, - .extra1 = (void *)&dirty_bytes_min, - }, - { - .procname = "dirty_writeback_centisecs", - .data = &dirty_writeback_interval, - .maxlen = sizeof(dirty_writeback_interval), - .mode = 0644, - .proc_handler = dirty_writeback_centisecs_handler, - }, - { - .procname = "dirty_expire_centisecs", - .data = &dirty_expire_interval, - .maxlen = sizeof(dirty_expire_interval), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - }, { .procname = "dirtytime_expire_seconds", .data = &dirtytime_expire_interval, @@ -2483,13 +2432,6 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, }, #endif - { - .procname = "laptop_mode", - .data = &laptop_mode, - .maxlen = sizeof(laptop_mode), - .mode = 0644, - .proc_handler = proc_dointvec_jiffies, - }, { .procname = "vfs_cache_pressure", .data = &sysctl_vfs_cache_pressure, @@ -2587,17 +2529,6 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, }, #endif -#ifdef CONFIG_HIGHMEM - { - .procname = "highmem_is_dirtyable", - .data = &vm_highmem_is_dirtyable, - .maxlen = sizeof(vm_highmem_is_dirtyable), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, -#endif #ifdef CONFIG_MEMORY_FAILURE { .procname = "memory_failure_early_kill", diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 7e2da284e427..438762173a59 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -70,30 +70,33 @@ static long ratelimit_pages = 32; /* * Start background writeback (via writeback threads) at this percentage */ -int dirty_background_ratio = 10; +static int dirty_background_ratio = 10; /* * dirty_background_bytes starts at 0 (disabled) so that it is a function of * dirty_background_ratio * the amount of dirtyable memory */ -unsigned long dirty_background_bytes; +static unsigned long dirty_background_bytes; /* * free highmem will not be subtracted from the total free memory * for calculating free ratios if vm_highmem_is_dirtyable is true */ -int vm_highmem_is_dirtyable; +static int vm_highmem_is_dirtyable; /* * The generator of dirty data starts writeback at this percentage */ -int vm_dirty_ratio = 20; +static int vm_dirty_ratio = 20; + +/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */ +static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE; /* * vm_dirty_bytes starts at 0 (disabled) so that it is a function of * vm_dirty_ratio * the amount of dirtyable memory */ -unsigned long vm_dirty_bytes; +static unsigned long vm_dirty_bytes; /* * The interval between `kupdate'-style writebacks @@ -491,7 +494,8 @@ bool node_dirty_ok(struct pglist_data *pgdat) return nr_pages <= limit; } -int dirty_background_ratio_handler(struct ctl_table *table, int write, +#ifdef CONFIG_SYSCTL +static int dirty_background_ratio_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { int ret; @@ -502,7 +506,7 @@ int dirty_background_ratio_handler(struct ctl_table *table, int write, return ret; } -int dirty_background_bytes_handler(struct ctl_table *table, int write, +static int dirty_background_bytes_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { int ret; @@ -513,7 +517,7 @@ int dirty_background_bytes_handler(struct ctl_table *table, int write, return ret; } -int dirty_ratio_handler(struct ctl_table *table, int write, void *buffer, +static int dirty_ratio_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { int old_ratio = vm_dirty_ratio; @@ -527,7 +531,7 @@ int dirty_ratio_handler(struct ctl_table *table, int write, void *buffer, return ret; } -int dirty_bytes_handler(struct ctl_table *table, int write, +static int dirty_bytes_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { unsigned long old_bytes = vm_dirty_bytes; @@ -540,6 +544,7 @@ int dirty_bytes_handler(struct ctl_table *table, int write, } return ret; } +#endif static unsigned long wp_next_time(unsigned long cur_time) { @@ -1981,10 +1986,11 @@ bool wb_over_bg_thresh(struct bdi_writeback *wb) return false; } +#ifdef CONFIG_SYSCTL /* * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs */ -int dirty_writeback_centisecs_handler(struct ctl_table *table, int write, +static int dirty_writeback_centisecs_handler(struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos) { unsigned int old_interval = dirty_writeback_interval; @@ -2005,6 +2011,7 @@ int dirty_writeback_centisecs_handler(struct ctl_table *table, int write, return ret; } +#endif void laptop_mode_timer_fn(struct timer_list *t) { @@ -2069,6 +2076,79 @@ static int page_writeback_cpu_online(unsigned int cpu) return 0; } +#ifdef CONFIG_SYSCTL +static struct ctl_table vm_page_writeback_sysctls[] = { + { + .procname = "dirty_background_ratio", + .data = &dirty_background_ratio, + .maxlen = sizeof(dirty_background_ratio), + .mode = 0644, + .proc_handler = dirty_background_ratio_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, + { + .procname = "dirty_background_bytes", + .data = &dirty_background_bytes, + .maxlen = sizeof(dirty_background_bytes), + .mode = 0644, + .proc_handler = dirty_background_bytes_handler, + .extra1 = SYSCTL_LONG_ONE, + }, + { + .procname = "dirty_ratio", + .data = &vm_dirty_ratio, + .maxlen = sizeof(vm_dirty_ratio), + .mode = 0644, + .proc_handler = dirty_ratio_handler, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE_HUNDRED, + }, + { + .procname = "dirty_bytes", + .data = &vm_dirty_bytes, + .maxlen = sizeof(vm_dirty_bytes), + .mode = 0644, + .proc_handler = dirty_bytes_handler, + .extra1 = (void *)&dirty_bytes_min, + }, + { + .procname = "dirty_writeback_centisecs", + .data = &dirty_writeback_interval, + .maxlen = sizeof(dirty_writeback_interval), + .mode = 0644, + .proc_handler = dirty_writeback_centisecs_handler, + }, + { + .procname = "dirty_expire_centisecs", + .data = &dirty_expire_interval, + .maxlen = sizeof(dirty_expire_interval), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + }, +#ifdef CONFIG_HIGHMEM + { + .procname = "highmem_is_dirtyable", + .data = &vm_highmem_is_dirtyable, + .maxlen = sizeof(vm_highmem_is_dirtyable), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +#endif + { + .procname = "laptop_mode", + .data = &laptop_mode, + .maxlen = sizeof(laptop_mode), + .mode = 0644, + .proc_handler = proc_dointvec_jiffies, + }, + {} +}; +#endif + /* * Called early on to tune the page writeback dirty limits. * @@ -2093,6 +2173,9 @@ void __init page_writeback_init(void) page_writeback_cpu_online, NULL); cpuhp_setup_state(CPUHP_MM_WRITEBACK_DEAD, "mm/writeback:dead", NULL, page_writeback_cpu_online); +#ifdef CONFIG_SYSCTL + register_sysctl_init("vm", vm_page_writeback_sysctls); +#endif } /** From f79c9b8ae8bde10126586c1bb55b5fd027276d8e Mon Sep 17 00:00:00 2001 From: tangmeng Date: Fri, 18 Feb 2022 18:58:57 +0800 Subject: [PATCH 055/319] kernel/lockdep: move lockdep sysctls to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. All filesystem syctls now get reviewed by fs folks. This commit follows the commit of fs, move the prove_locking and lock_stat sysctls to its own file, kernel/lockdep.c. Signed-off-by: tangmeng Signed-off-by: Luis Chamberlain --- include/linux/lockdep.h | 4 ---- kernel/locking/lockdep.c | 35 +++++++++++++++++++++++++++++++++-- kernel/sysctl.c | 21 --------------------- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 467b94257105..37951c17908e 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -16,10 +16,6 @@ struct task_struct; -/* for sysctl */ -extern int prove_locking; -extern int lock_stat; - #ifdef CONFIG_LOCKDEP #include diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index c06cab6546ed..a4382ae1be59 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -64,19 +64,50 @@ #include #ifdef CONFIG_PROVE_LOCKING -int prove_locking = 1; +static int prove_locking = 1; module_param(prove_locking, int, 0644); #else #define prove_locking 0 #endif #ifdef CONFIG_LOCK_STAT -int lock_stat = 1; +static int lock_stat = 1; module_param(lock_stat, int, 0644); #else #define lock_stat 0 #endif +#ifdef CONFIG_SYSCTL +static struct ctl_table kern_lockdep_table[] = { +#ifdef CONFIG_PROVE_LOCKING + { + .procname = "prove_locking", + .data = &prove_locking, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif /* CONFIG_PROVE_LOCKING */ +#ifdef CONFIG_LOCK_STAT + { + .procname = "lock_stat", + .data = &lock_stat, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif /* CONFIG_LOCK_STAT */ + { } +}; + +static __init int kernel_lockdep_sysctls_init(void) +{ + register_sysctl_init("kernel", kern_lockdep_table); + return 0; +} +late_initcall(kernel_lockdep_sysctls_init); +#endif /* CONFIG_SYSCTL */ + DEFINE_PER_CPU(unsigned int, lockdep_recursion); EXPORT_PER_CPU_SYMBOL_GPL(lockdep_recursion); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 36bfe1c92d44..95380d250c8c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -88,9 +88,6 @@ #ifdef CONFIG_RT_MUTEXES #include #endif -#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_LOCK_STAT) -#include -#endif #if defined(CONFIG_SYSCTL) @@ -1679,24 +1676,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_FOUR, }, #endif /* CONFIG_NUMA_BALANCING */ -#ifdef CONFIG_PROVE_LOCKING - { - .procname = "prove_locking", - .data = &prove_locking, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif -#ifdef CONFIG_LOCK_STAT - { - .procname = "lock_stat", - .data = &lock_stat, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif { .procname = "panic", .data = &panic_timeout, From 9df918698408fd914493aba0b7858fef50eba63a Mon Sep 17 00:00:00 2001 From: tangmeng Date: Fri, 18 Feb 2022 18:59:12 +0800 Subject: [PATCH 056/319] kernel/panic: move panic sysctls to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. All filesystem syctls now get reviewed by fs folks. This commit follows the commit of fs, move the oops_all_cpu_backtrace sysctl to its own file, kernel/panic.c. Signed-off-by: tangmeng Signed-off-by: Luis Chamberlain --- include/linux/panic.h | 6 ------ kernel/panic.c | 26 +++++++++++++++++++++++++- kernel/sysctl.c | 11 ----------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/include/linux/panic.h b/include/linux/panic.h index f5844908a089..e71161da69c4 100644 --- a/include/linux/panic.h +++ b/include/linux/panic.h @@ -15,12 +15,6 @@ extern void oops_enter(void); extern void oops_exit(void); extern bool oops_may_print(void); -#ifdef CONFIG_SMP -extern unsigned int sysctl_oops_all_cpu_backtrace; -#else -#define sysctl_oops_all_cpu_backtrace 0 -#endif /* CONFIG_SMP */ - extern int panic_timeout; extern unsigned long panic_print; extern int panic_on_oops; diff --git a/kernel/panic.c b/kernel/panic.c index eb4dfb932c85..eb3f2fe4f6d7 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -43,7 +43,9 @@ * Should we dump all CPUs backtraces in an oops event? * Defaults to 0, can be changed via sysctl. */ -unsigned int __read_mostly sysctl_oops_all_cpu_backtrace; +static unsigned int __read_mostly sysctl_oops_all_cpu_backtrace; +#else +#define sysctl_oops_all_cpu_backtrace 0 #endif /* CONFIG_SMP */ int panic_on_oops = CONFIG_PANIC_ON_OOPS_VALUE; @@ -73,6 +75,28 @@ ATOMIC_NOTIFIER_HEAD(panic_notifier_list); EXPORT_SYMBOL(panic_notifier_list); +#if defined(CONFIG_SMP) && defined(CONFIG_SYSCTL) +static struct ctl_table kern_panic_table[] = { + { + .procname = "oops_all_cpu_backtrace", + .data = &sysctl_oops_all_cpu_backtrace, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { } +}; + +static __init int kernel_panic_sysctls_init(void) +{ + register_sysctl_init("kernel", kern_panic_table); + return 0; +} +late_initcall(kernel_panic_sysctls_init); +#endif + static long no_blink(int state) { return 0; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 95380d250c8c..90fc2212b536 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1922,17 +1922,6 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec, }, #endif -#ifdef CONFIG_SMP - { - .procname = "oops_all_cpu_backtrace", - .data = &sysctl_oops_all_cpu_backtrace, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, -#endif /* CONFIG_SMP */ { .procname = "pid_max", .data = &pid_max, From 801b501439d1b366d524dee4fc1e6b3473a95b9a Mon Sep 17 00:00:00 2001 From: tangmeng Date: Fri, 18 Feb 2022 18:59:23 +0800 Subject: [PATCH 057/319] kernel/acct: move acct sysctls to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. All filesystem syctls now get reviewed by fs folks. This commit follows the commit of fs, move the acct sysctl to its own file, kernel/acct.c. Signed-off-by: tangmeng Signed-off-by: Luis Chamberlain --- include/linux/acct.h | 1 - kernel/acct.c | 22 +++++++++++++++++++++- kernel/sysctl.c | 12 ------------ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/include/linux/acct.h b/include/linux/acct.h index bc70e81895c0..2718c4854815 100644 --- a/include/linux/acct.h +++ b/include/linux/acct.h @@ -21,7 +21,6 @@ #ifdef CONFIG_BSD_PROCESS_ACCT struct pid_namespace; -extern int acct_parm[]; /* for sysctl */ extern void acct_collect(long exitcode, int group_dead); extern void acct_process(void); extern void acct_exit_ns(struct pid_namespace *); diff --git a/kernel/acct.c b/kernel/acct.c index 3df53cf1dcd5..13706356ec54 100644 --- a/kernel/acct.c +++ b/kernel/acct.c @@ -70,11 +70,31 @@ * Turned into sysctl-controllable parameters. AV, 12/11/98 */ -int acct_parm[3] = {4, 2, 30}; +static int acct_parm[3] = {4, 2, 30}; #define RESUME (acct_parm[0]) /* >foo% free space - resume */ #define SUSPEND (acct_parm[1]) /* #endif -#ifdef CONFIG_BSD_PROCESS_ACCT -#include -#endif #ifdef CONFIG_RT_MUTEXES #include #endif @@ -1856,15 +1853,6 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dostring, }, #endif -#ifdef CONFIG_BSD_PROCESS_ACCT - { - .procname = "acct", - .data = &acct_parm, - .maxlen = 3*sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif #ifdef CONFIG_MAGIC_SYSRQ { .procname = "sysrq", From 1186618a6a35d43a865448472a261184b608d13c Mon Sep 17 00:00:00 2001 From: tangmeng Date: Fri, 18 Feb 2022 18:59:36 +0800 Subject: [PATCH 058/319] kernel/delayacct: move delayacct sysctls to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. All filesystem syctls now get reviewed by fs folks. This commit follows the commit of fs, move the delayacct sysctl to its own file, kernel/delayacct.c. Signed-off-by: tangmeng Signed-off-by: Luis Chamberlain --- include/linux/delayacct.h | 3 --- kernel/delayacct.c | 22 +++++++++++++++++++++- kernel/sysctl.c | 12 ------------ 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/include/linux/delayacct.h b/include/linux/delayacct.h index 3e03d010bd2e..6b16a6930a19 100644 --- a/include/linux/delayacct.h +++ b/include/linux/delayacct.h @@ -61,9 +61,6 @@ extern int delayacct_on; /* Delay accounting turned on/off */ extern struct kmem_cache *delayacct_cache; extern void delayacct_init(void); -extern int sysctl_delayacct(struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); - extern void __delayacct_tsk_init(struct task_struct *); extern void __delayacct_tsk_exit(struct task_struct *); extern void __delayacct_blkio_start(void); diff --git a/kernel/delayacct.c b/kernel/delayacct.c index c5e8cea9e05f..2c1e18f7c5cf 100644 --- a/kernel/delayacct.c +++ b/kernel/delayacct.c @@ -44,7 +44,7 @@ void delayacct_init(void) } #ifdef CONFIG_PROC_SYSCTL -int sysctl_delayacct(struct ctl_table *table, int write, void *buffer, +static int sysctl_delayacct(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { int state = delayacct_on; @@ -63,6 +63,26 @@ int sysctl_delayacct(struct ctl_table *table, int write, void *buffer, set_delayacct(state); return err; } + +static struct ctl_table kern_delayacct_table[] = { + { + .procname = "task_delayacct", + .data = NULL, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = sysctl_delayacct, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { } +}; + +static __init int kernel_delayacct_sysctls_init(void) +{ + register_sysctl_init("kernel", kern_delayacct_table); + return 0; +} +late_initcall(kernel_delayacct_sysctls_init); #endif void __delayacct_tsk_init(struct task_struct *tsk) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 5421e28dbb25..9b74ba12a711 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -67,7 +67,6 @@ #include #include #include -#include #include "../lib/kstrtox.h" @@ -1651,17 +1650,6 @@ int proc_do_static_key(struct ctl_table *table, int write, } static struct ctl_table kern_table[] = { -#ifdef CONFIG_TASK_DELAY_ACCT - { - .procname = "task_delayacct", - .data = NULL, - .maxlen = sizeof(unsigned int), - .mode = 0644, - .proc_handler = sysctl_delayacct, - .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, - }, -#endif /* CONFIG_TASK_DELAY_ACCT */ #ifdef CONFIG_NUMA_BALANCING { .procname = "numa_balancing", From d772cc2c321900f3f463a124eebeb7218e66dda6 Mon Sep 17 00:00:00 2001 From: tangmeng Date: Fri, 18 Feb 2022 18:59:49 +0800 Subject: [PATCH 059/319] kernel/do_mount_initrd: move real_root_dev sysctls to its own file kernel/sysctl.c is a kitchen sink where everyone leaves their dirty dishes, this makes it very difficult to maintain. To help with this maintenance let's start by moving sysctls to places where they actually belong. The proc sysctl maintainers do not want to know what sysctl knobs you wish to add for your own piece of code, we just care about the core logic. All filesystem syctls now get reviewed by fs folks. This commit follows the commit of fs, move the real_root_dev sysctl to its own file, kernel/do_mount_initrd.c. Signed-off-by: tangmeng Signed-off-by: Luis Chamberlain --- include/linux/initrd.h | 2 -- init/do_mounts_initrd.c | 22 +++++++++++++++++++++- kernel/sysctl.c | 9 --------- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/include/linux/initrd.h b/include/linux/initrd.h index 1bbe9af48dc3..f1a1f4c92ded 100644 --- a/include/linux/initrd.h +++ b/include/linux/initrd.h @@ -29,8 +29,6 @@ static inline void wait_for_initramfs(void) {} extern phys_addr_t phys_initrd_start; extern unsigned long phys_initrd_size; -extern unsigned int real_root_dev; - extern char __initramfs_start[]; extern unsigned long __initramfs_size; diff --git a/init/do_mounts_initrd.c b/init/do_mounts_initrd.c index 533d81ed74d4..327962ea354c 100644 --- a/init/do_mounts_initrd.c +++ b/init/do_mounts_initrd.c @@ -14,12 +14,32 @@ unsigned long initrd_start, initrd_end; int initrd_below_start_ok; -unsigned int real_root_dev; /* do_proc_dointvec cannot handle kdev_t */ +static unsigned int real_root_dev; /* do_proc_dointvec cannot handle kdev_t */ static int __initdata mount_initrd = 1; phys_addr_t phys_initrd_start __initdata; unsigned long phys_initrd_size __initdata; +#ifdef CONFIG_SYSCTL +static struct ctl_table kern_do_mounts_initrd_table[] = { + { + .procname = "real-root-dev", + .data = &real_root_dev, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { } +}; + +static __init int kernel_do_mounts_initrd_sysctls_init(void) +{ + register_sysctl_init("kernel", kern_do_mounts_initrd_table); + return 0; +} +late_initcall(kernel_do_mounts_initrd_sysctls_init); +#endif /* CONFIG_SYSCTL */ + static int __init no_initrd(char *str) { mount_initrd = 0; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 9b74ba12a711..10a551f8fcab 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1693,15 +1693,6 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = sysctl_latencytop, }, -#endif -#ifdef CONFIG_BLK_DEV_INITRD - { - .procname = "real-root-dev", - .data = &real_root_dev, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, #endif { .procname = "print-fatal-signals", From 8e4e83b2278bdfb55cb2b13de07cf0a721ce8af7 Mon Sep 17 00:00:00 2001 From: Wei Xiao Date: Wed, 23 Feb 2022 19:11:53 +0800 Subject: [PATCH 060/319] ftrace: move sysctl_ftrace_enabled to ftrace.c This moves ftrace_enabled to trace/ftrace.c. We move sysctls to places where features actually belong to improve the readability of the code and reduce the risk of code merge conflicts. At the same time, the proc-sysctl maintainers do not want to know what sysctl knobs you wish to add for your owner piece of code, we just care about the core logic. Signed-off-by: Wei Xiao Acked-by: Steven Rostedt (Google) Signed-off-by: Luis Chamberlain --- include/linux/ftrace.h | 3 --- kernel/sysctl.c | 9 --------- kernel/trace/ftrace.c | 22 +++++++++++++++++++++- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 4816b7e11047..088b915853dd 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -101,9 +101,6 @@ static inline int ftrace_mod_get_kallsym(unsigned int symnum, unsigned long *val #ifdef CONFIG_FUNCTION_TRACER extern int ftrace_enabled; -extern int -ftrace_enable_sysctl(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos); #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 10a551f8fcab..21172d3dad6e 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -1751,15 +1751,6 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec, }, #endif -#ifdef CONFIG_FUNCTION_TRACER - { - .procname = "ftrace_enabled", - .data = &ftrace_enabled, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = ftrace_enable_sysctl, - }, -#endif #ifdef CONFIG_STACK_TRACER { .procname = "stack_tracer_enabled", diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 4f1d2f5e7263..a5efbbc289b4 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -7921,7 +7921,8 @@ static bool is_permanent_ops_registered(void) return false; } -int +#ifdef CONFIG_SYSCTL +static int ftrace_enable_sysctl(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -7964,3 +7965,22 @@ ftrace_enable_sysctl(struct ctl_table *table, int write, mutex_unlock(&ftrace_lock); return ret; } + +static struct ctl_table ftrace_sysctls[] = { + { + .procname = "ftrace_enabled", + .data = &ftrace_enabled, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = ftrace_enable_sysctl, + }, + {} +}; + +static int __init ftrace_sysctl_init(void) +{ + register_sysctl_init("kernel", ftrace_sysctls); + return 0; +} +late_initcall(ftrace_sysctl_init); +#endif From 85ebb1a6bd62147ebcfa70500d513331a8daf9e0 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:35:52 +0300 Subject: [PATCH 061/319] gpiolib: Introduce for_each_gpiochip_node() loop helper Introduce for_each_gpiochip_node() loop helper which iterates over the GPIO controller child nodes of a given device. Signed-off-by: Andy Shevchenko Reviewed-by: Geert Uytterhoeven Tested-by: Geert Uytterhoeven Acked-by: Bartosz Golaszewski --- include/linux/gpio/driver.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 98c93510640e..bfc91f122d5f 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -3,13 +3,14 @@ #define __LINUX_GPIO_DRIVER_H #include -#include #include #include #include #include #include #include +#include +#include struct gpio_desc; struct of_phandle_args; @@ -750,4 +751,8 @@ static inline void gpiochip_unlock_as_irq(struct gpio_chip *gc, } #endif /* CONFIG_GPIOLIB */ +#define for_each_gpiochip_node(dev, child) \ + device_for_each_child_node(dev, child) \ + if (!fwnode_property_present(child, "gpio-controller")) {} else + #endif /* __LINUX_GPIO_DRIVER_H */ From 0b19dde90ad004592792a928c75e80612be3e2e8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:35:53 +0300 Subject: [PATCH 062/319] gpiolib: Introduce gpiochip_node_count() helper The gpiochip_node_count() helper iterates over the device child nodes that have the "gpio-controller" property set. It returns the number of such nodes under a given device. Signed-off-by: Andy Shevchenko Reviewed-by: Geert Uytterhoeven Tested-by: Geert Uytterhoeven Acked-by: Bartosz Golaszewski --- include/linux/gpio/driver.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index bfc91f122d5f..12de0b22b4ef 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -755,4 +755,15 @@ static inline void gpiochip_unlock_as_irq(struct gpio_chip *gc, device_for_each_child_node(dev, child) \ if (!fwnode_property_present(child, "gpio-controller")) {} else +static inline unsigned int gpiochip_node_count(struct device *dev) +{ + struct fwnode_handle *child; + unsigned int count = 0; + + for_each_gpiochip_node(dev, child) + count++; + + return count; +} + #endif /* __LINUX_GPIO_DRIVER_H */ From d9463201ec0824037a9ca84b9bb03a79d4460cfb Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:35:54 +0300 Subject: [PATCH 063/319] pinctrl: stm32: Replace custom code by gpiochip_node_count() call Since we have generic function to count GPIO controller nodes under a given device, there is no need to open code it. Replace custom code by gpiochip_node_count() call. Signed-off-by: Andy Shevchenko Reviewed-by: Fabien Dessenne --- drivers/pinctrl/stm32/pinctrl-stm32.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c index 9ed764731570..91b9a64b649e 100644 --- a/drivers/pinctrl/stm32/pinctrl-stm32.c +++ b/drivers/pinctrl/stm32/pinctrl-stm32.c @@ -1423,7 +1423,8 @@ int stm32_pctl_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct stm32_pinctrl *pctl; struct pinctrl_pin_desc *pins; - int i, ret, hwlock_id, banks = 0; + int i, ret, hwlock_id; + unsigned int banks; if (!np) return -EINVAL; @@ -1513,10 +1514,7 @@ int stm32_pctl_probe(struct platform_device *pdev) return PTR_ERR(pctl->pctl_dev); } - for_each_available_child_of_node(np, child) - if (of_property_read_bool(child, "gpio-controller")) - banks++; - + banks = gpiochip_node_count(dev); if (!banks) { dev_err(dev, "at least one GPIO bank is required\n"); return -EINVAL; From bb949ed9b16ba4575c76f7be55d4279e35ddccc1 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:35:55 +0300 Subject: [PATCH 064/319] pinctrl: stm32: Switch to use for_each_gpiochip_node() helper Switch the code to use for_each_gpiochip_node() helper. While at it, in order to avoid additional churn in the future, switch to fwnode APIs where it makes sense. Signed-off-by: Andy Shevchenko Reviewed-by: Fabien Dessenne --- drivers/pinctrl/stm32/pinctrl-stm32.c | 72 ++++++++++++--------------- 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/drivers/pinctrl/stm32/pinctrl-stm32.c b/drivers/pinctrl/stm32/pinctrl-stm32.c index 91b9a64b649e..09952c463f67 100644 --- a/drivers/pinctrl/stm32/pinctrl-stm32.c +++ b/drivers/pinctrl/stm32/pinctrl-stm32.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1215,13 +1216,12 @@ static const struct pinconf_ops stm32_pconf_ops = { .pin_config_dbg_show = stm32_pconf_dbg_show, }; -static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl, - struct device_node *np) +static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl, struct fwnode_handle *fwnode) { struct stm32_gpio_bank *bank = &pctl->banks[pctl->nbanks]; int bank_ioport_nr; struct pinctrl_gpio_range *range = &bank->range; - struct of_phandle_args args; + struct fwnode_reference_args args; struct device *dev = pctl->dev; struct resource res; int npins = STM32_GPIO_PINS_PER_BANK; @@ -1230,7 +1230,7 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl, if (!IS_ERR(bank->rstc)) reset_control_deassert(bank->rstc); - if (of_address_to_resource(np, 0, &res)) + if (of_address_to_resource(to_of_node(fwnode), 0, &res)) return -ENODEV; bank->base = devm_ioremap_resource(dev, &res); @@ -1245,15 +1245,15 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl, bank->gpio_chip = stm32_gpio_template; - of_property_read_string(np, "st,bank-name", &bank->gpio_chip.label); + fwnode_property_read_string(fwnode, "st,bank-name", &bank->gpio_chip.label); - if (!of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, i, &args)) { + if (!fwnode_property_get_reference_args(fwnode, "gpio-ranges", NULL, 3, i, &args)) { bank_nr = args.args[1] / STM32_GPIO_PINS_PER_BANK; bank->gpio_chip.base = args.args[1]; /* get the last defined gpio line (offset + nb of pins) */ npins = args.args[0] + args.args[2]; - while (!of_parse_phandle_with_fixed_args(np, "gpio-ranges", 3, ++i, &args)) + while (!fwnode_property_get_reference_args(fwnode, "gpio-ranges", NULL, 3, ++i, &args)) npins = max(npins, (int)(args.args[0] + args.args[2])); } else { bank_nr = pctl->nbanks; @@ -1268,20 +1268,20 @@ static int stm32_gpiolib_register_bank(struct stm32_pinctrl *pctl, &pctl->banks[bank_nr].range); } - if (of_property_read_u32(np, "st,bank-ioport", &bank_ioport_nr)) + if (fwnode_property_read_u32(fwnode, "st,bank-ioport", &bank_ioport_nr)) bank_ioport_nr = bank_nr; bank->gpio_chip.base = bank_nr * STM32_GPIO_PINS_PER_BANK; bank->gpio_chip.ngpio = npins; - bank->gpio_chip.of_node = np; + bank->gpio_chip.fwnode = fwnode; bank->gpio_chip.parent = dev; bank->bank_nr = bank_nr; bank->bank_ioport_nr = bank_ioport_nr; spin_lock_init(&bank->lock); /* create irq hierarchical domain */ - bank->fwnode = of_node_to_fwnode(np); + bank->fwnode = fwnode; bank->domain = irq_domain_create_hierarchy(pctl->domain, 0, STM32_GPIO_IRQ_LINE, bank->fwnode, @@ -1418,7 +1418,7 @@ static int stm32_pctrl_create_pins_tab(struct stm32_pinctrl *pctl, int stm32_pctl_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; - struct device_node *child; + struct fwnode_handle *child; const struct of_device_id *match; struct device *dev = &pdev->dev; struct stm32_pinctrl *pctl; @@ -1525,40 +1525,34 @@ int stm32_pctl_probe(struct platform_device *pdev) return -ENOMEM; i = 0; - for_each_available_child_of_node(np, child) { + for_each_gpiochip_node(dev, child) { struct stm32_gpio_bank *bank = &pctl->banks[i]; + struct device_node *np = to_of_node(child); - if (of_property_read_bool(child, "gpio-controller")) { - bank->rstc = of_reset_control_get_exclusive(child, - NULL); - if (PTR_ERR(bank->rstc) == -EPROBE_DEFER) { - of_node_put(child); - return -EPROBE_DEFER; - } - - bank->clk = of_clk_get_by_name(child, NULL); - if (IS_ERR(bank->clk)) { - if (PTR_ERR(bank->clk) != -EPROBE_DEFER) - dev_err(dev, - "failed to get clk (%ld)\n", - PTR_ERR(bank->clk)); - of_node_put(child); - return PTR_ERR(bank->clk); - } - i++; + bank->rstc = of_reset_control_get_exclusive(np, NULL); + if (PTR_ERR(bank->rstc) == -EPROBE_DEFER) { + fwnode_handle_put(child); + return -EPROBE_DEFER; } + + bank->clk = of_clk_get_by_name(np, NULL); + if (IS_ERR(bank->clk)) { + if (PTR_ERR(bank->clk) != -EPROBE_DEFER) + dev_err(dev, "failed to get clk (%ld)\n", PTR_ERR(bank->clk)); + fwnode_handle_put(child); + return PTR_ERR(bank->clk); + } + i++; } - for_each_available_child_of_node(np, child) { - if (of_property_read_bool(child, "gpio-controller")) { - ret = stm32_gpiolib_register_bank(pctl, child); - if (ret) { - of_node_put(child); - return ret; - } - - pctl->nbanks++; + for_each_gpiochip_node(dev, child) { + ret = stm32_gpiolib_register_bank(pctl, child); + if (ret) { + fwnode_handle_put(child); + return ret; } + + pctl->nbanks++; } dev_info(dev, "Pinctrl STM32 initialized\n"); From 1e0afd470e26f83c674ff239dd5b5347cdc24fdb Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:35:57 +0300 Subject: [PATCH 065/319] pinctrl: renesas: rza1: Replace custom code by gpiochip_node_count() call Since we have generic function to count GPIO controller nodes under a given device, there is no need to open code it. Replace custom code by gpiochip_node_count() call. Signed-off-by: Andy Shevchenko Reviewed-by: Geert Uytterhoeven Tested-by: Geert Uytterhoeven Acked-by: Geert Uytterhoeven --- drivers/pinctrl/renesas/pinctrl-rza1.c | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/drivers/pinctrl/renesas/pinctrl-rza1.c b/drivers/pinctrl/renesas/pinctrl-rza1.c index c1d6e9512c7a..acc00b1a955d 100644 --- a/drivers/pinctrl/renesas/pinctrl-rza1.c +++ b/drivers/pinctrl/renesas/pinctrl-rza1.c @@ -1154,21 +1154,6 @@ static const struct pinmux_ops rza1_pinmux_ops = { * RZ/A1 pin controller driver operations */ -static unsigned int rza1_count_gpio_chips(struct device_node *np) -{ - struct device_node *child; - unsigned int count = 0; - - for_each_child_of_node(np, child) { - if (!of_property_read_bool(child, "gpio-controller")) - continue; - - count++; - } - - return count; -} - /** * rza1_parse_gpiochip() - parse and register a gpio chip and pin range * @@ -1255,7 +1240,7 @@ static int rza1_gpio_register(struct rza1_pinctrl *rza1_pctl) unsigned int i; int ret; - ngpiochips = rza1_count_gpio_chips(np); + ngpiochips = gpiochip_node_count(rza1_pctl->dev); if (ngpiochips == 0) { dev_dbg(rza1_pctl->dev, "No gpiochip registered\n"); return 0; From 5e455dd93397b3cad0b6767654d29919d28d3a89 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:35:58 +0300 Subject: [PATCH 066/319] pinctrl: renesas: rza1: Switch to use for_each_gpiochip_node() helper Switch the code to use for_each_gpiochip_node() helper. While at it, in order to avoid additional churn in the future, switch to fwnode APIs where it makes sense. Signed-off-by: Andy Shevchenko Reviewed-by: Geert Uytterhoeven Tested-by: Geert Uytterhoeven Acked-by: Geert Uytterhoeven --- drivers/pinctrl/renesas/pinctrl-rza1.c | 30 +++++++++++--------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/drivers/pinctrl/renesas/pinctrl-rza1.c b/drivers/pinctrl/renesas/pinctrl-rza1.c index acc00b1a955d..529c0fc4ec06 100644 --- a/drivers/pinctrl/renesas/pinctrl-rza1.c +++ b/drivers/pinctrl/renesas/pinctrl-rza1.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "../core.h" @@ -1161,22 +1162,22 @@ static const struct pinmux_ops rza1_pinmux_ops = { * defined by gpio device tree binding documentation. * * @rza1_pctl: RZ/A1 pin controller device - * @np: of gpio-controller node + * @fwnode: gpio-controller firmware node * @chip: gpio chip to register to gpiolib * @range: pin range to register to pinctrl core */ static int rza1_parse_gpiochip(struct rza1_pinctrl *rza1_pctl, - struct device_node *np, + struct fwnode_handle *fwnode, struct gpio_chip *chip, struct pinctrl_gpio_range *range) { const char *list_name = "gpio-ranges"; - struct of_phandle_args of_args; + struct fwnode_reference_args args; unsigned int gpioport; u32 pinctrl_base; int ret; - ret = of_parse_phandle_with_fixed_args(np, list_name, 3, 0, &of_args); + ret = fwnode_property_get_reference_args(fwnode, list_name, NULL, 3, 0, &args); if (ret) { dev_err(rza1_pctl->dev, "Unable to parse %s list property\n", list_name); @@ -1187,7 +1188,7 @@ static int rza1_parse_gpiochip(struct rza1_pinctrl *rza1_pctl, * Find out on which port this gpio-chip maps to by inspecting the * second argument of the "gpio-ranges" property. */ - pinctrl_base = of_args.args[1]; + pinctrl_base = args.args[1]; gpioport = RZA1_PIN_ID_TO_PORT(pinctrl_base); if (gpioport >= RZA1_NPORTS) { dev_err(rza1_pctl->dev, @@ -1197,19 +1198,18 @@ static int rza1_parse_gpiochip(struct rza1_pinctrl *rza1_pctl, *chip = rza1_gpiochip_template; chip->base = -1; - chip->label = devm_kasprintf(rza1_pctl->dev, GFP_KERNEL, "%pOFn", - np); + chip->ngpio = args.args[2]; + chip->label = devm_kasprintf(rza1_pctl->dev, GFP_KERNEL, "%pfwP", fwnode); if (!chip->label) return -ENOMEM; - chip->ngpio = of_args.args[2]; - chip->of_node = np; + chip->fwnode = fwnode; chip->parent = rza1_pctl->dev; range->id = gpioport; range->name = chip->label; range->pin_base = range->base = pinctrl_base; - range->npins = of_args.args[2]; + range->npins = args.args[2]; range->gc = chip; ret = devm_gpiochip_add_data(rza1_pctl->dev, chip, @@ -1232,10 +1232,9 @@ static int rza1_parse_gpiochip(struct rza1_pinctrl *rza1_pctl, */ static int rza1_gpio_register(struct rza1_pinctrl *rza1_pctl) { - struct device_node *np = rza1_pctl->dev->of_node; struct pinctrl_gpio_range *gpio_ranges; struct gpio_chip *gpio_chips; - struct device_node *child; + struct fwnode_handle *child; unsigned int ngpiochips; unsigned int i; int ret; @@ -1254,14 +1253,11 @@ static int rza1_gpio_register(struct rza1_pinctrl *rza1_pctl) return -ENOMEM; i = 0; - for_each_child_of_node(np, child) { - if (!of_property_read_bool(child, "gpio-controller")) - continue; - + for_each_gpiochip_node(rza1_pctl->dev, child) { ret = rza1_parse_gpiochip(rza1_pctl, child, &gpio_chips[i], &gpio_ranges[i]); if (ret) { - of_node_put(child); + fwnode_handle_put(child); return ret; } From 0173ce55e50800a1a59dddcb972fe459cff0fee4 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:35:59 +0300 Subject: [PATCH 067/319] pinctrl: npcm7xx: Switch to use for_each_gpiochip_node() helper Switch the code to use for_each_gpiochip_node() helper. While at it, in order to avoid additional churn in the future, do the following: - use a temporary variable for struct device pointer to shorten a few lines - get rid of a temporary variable for vIRQ number, assign it directly - switch to fwnode APIs where it makes sense Signed-off-by: Andy Shevchenko --- drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c | 138 ++++++++++------------ 1 file changed, 60 insertions(+), 78 deletions(-) diff --git a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c index 9557fac5d11c..3cf0f8a43c37 100644 --- a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c +++ b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c @@ -17,6 +17,7 @@ #include #include #include +#include #include /* GCR registers */ @@ -1862,88 +1863,69 @@ static int npcm7xx_gpio_of(struct npcm7xx_pinctrl *pctrl) { int ret = -ENXIO; struct resource res; - int id = 0, irq; - struct device_node *np; - struct of_phandle_args pinspec; + struct device *dev = pctrl->dev; + struct fwnode_reference_args args; + struct fwnode_handle *child; + int id = 0; - for_each_available_child_of_node(pctrl->dev->of_node, np) - if (of_find_property(np, "gpio-controller", NULL)) { - ret = of_address_to_resource(np, 0, &res); - if (ret < 0) { - dev_err(pctrl->dev, - "Resource fail for GPIO bank %u\n", id); - return ret; - } + for_each_gpiochip_node(dev, child) { + struct device_node *np = to_of_node(child); - pctrl->gpio_bank[id].base = - ioremap(res.start, resource_size(&res)); - - irq = irq_of_parse_and_map(np, 0); - if (irq < 0) { - dev_err(pctrl->dev, - "No IRQ for GPIO bank %u\n", id); - ret = irq; - return ret; - } - - ret = bgpio_init(&pctrl->gpio_bank[id].gc, - pctrl->dev, 4, - pctrl->gpio_bank[id].base + - NPCM7XX_GP_N_DIN, - pctrl->gpio_bank[id].base + - NPCM7XX_GP_N_DOUT, - NULL, - NULL, - pctrl->gpio_bank[id].base + - NPCM7XX_GP_N_IEM, - BGPIOF_READ_OUTPUT_REG_SET); - if (ret) { - dev_err(pctrl->dev, "bgpio_init() failed\n"); - return ret; - } - - ret = of_parse_phandle_with_fixed_args(np, - "gpio-ranges", 3, - 0, &pinspec); - if (ret < 0) { - dev_err(pctrl->dev, - "gpio-ranges fail for GPIO bank %u\n", - id); - return ret; - } - - pctrl->gpio_bank[id].irq = irq; - pctrl->gpio_bank[id].irq_chip = npcmgpio_irqchip; - pctrl->gpio_bank[id].gc.parent = pctrl->dev; - pctrl->gpio_bank[id].irqbase = - id * NPCM7XX_GPIO_PER_BANK; - pctrl->gpio_bank[id].pinctrl_id = pinspec.args[0]; - pctrl->gpio_bank[id].gc.base = pinspec.args[1]; - pctrl->gpio_bank[id].gc.ngpio = pinspec.args[2]; - pctrl->gpio_bank[id].gc.owner = THIS_MODULE; - pctrl->gpio_bank[id].gc.label = - devm_kasprintf(pctrl->dev, GFP_KERNEL, "%pOF", - np); - if (pctrl->gpio_bank[id].gc.label == NULL) - return -ENOMEM; - - pctrl->gpio_bank[id].gc.dbg_show = npcmgpio_dbg_show; - pctrl->gpio_bank[id].direction_input = - pctrl->gpio_bank[id].gc.direction_input; - pctrl->gpio_bank[id].gc.direction_input = - npcmgpio_direction_input; - pctrl->gpio_bank[id].direction_output = - pctrl->gpio_bank[id].gc.direction_output; - pctrl->gpio_bank[id].gc.direction_output = - npcmgpio_direction_output; - pctrl->gpio_bank[id].request = - pctrl->gpio_bank[id].gc.request; - pctrl->gpio_bank[id].gc.request = npcmgpio_gpio_request; - pctrl->gpio_bank[id].gc.free = npcmgpio_gpio_free; - pctrl->gpio_bank[id].gc.of_node = np; - id++; + ret = of_address_to_resource(np, 0, &res); + if (ret < 0) { + dev_err(dev, "Resource fail for GPIO bank %u\n", id); + return ret; } + pctrl->gpio_bank[id].base = ioremap(res.start, resource_size(&res)); + + ret = bgpio_init(&pctrl->gpio_bank[id].gc, dev, 4, + pctrl->gpio_bank[id].base + NPCM7XX_GP_N_DIN, + pctrl->gpio_bank[id].base + NPCM7XX_GP_N_DOUT, + NULL, + NULL, + pctrl->gpio_bank[id].base + NPCM7XX_GP_N_IEM, + BGPIOF_READ_OUTPUT_REG_SET); + if (ret) { + dev_err(dev, "bgpio_init() failed\n"); + return ret; + } + + ret = fwnode_property_get_reference_args(child, "gpio-ranges", NULL, 3, 0, &args); + if (ret < 0) { + dev_err(dev, "gpio-ranges fail for GPIO bank %u\n", id); + return ret; + } + + ret = irq_of_parse_and_map(np, 0); + if (ret < 0) { + dev_err(dev, "No IRQ for GPIO bank %u\n", id); + return ret; + } + pctrl->gpio_bank[id].irq = ret; + pctrl->gpio_bank[id].irq_chip = npcmgpio_irqchip; + pctrl->gpio_bank[id].irqbase = id * NPCM7XX_GPIO_PER_BANK; + pctrl->gpio_bank[id].pinctrl_id = args.args[0]; + pctrl->gpio_bank[id].gc.base = args.args[1]; + pctrl->gpio_bank[id].gc.ngpio = args.args[2]; + pctrl->gpio_bank[id].gc.owner = THIS_MODULE; + pctrl->gpio_bank[id].gc.parent = dev; + pctrl->gpio_bank[id].gc.fwnode = child; + pctrl->gpio_bank[id].gc.label = devm_kasprintf(dev, GFP_KERNEL, "%pfw", child); + if (pctrl->gpio_bank[id].gc.label == NULL) + return -ENOMEM; + + pctrl->gpio_bank[id].gc.dbg_show = npcmgpio_dbg_show; + pctrl->gpio_bank[id].direction_input = pctrl->gpio_bank[id].gc.direction_input; + pctrl->gpio_bank[id].gc.direction_input = npcmgpio_direction_input; + pctrl->gpio_bank[id].direction_output = pctrl->gpio_bank[id].gc.direction_output; + pctrl->gpio_bank[id].gc.direction_output = npcmgpio_direction_output; + pctrl->gpio_bank[id].request = pctrl->gpio_bank[id].gc.request; + pctrl->gpio_bank[id].gc.request = npcmgpio_gpio_request; + pctrl->gpio_bank[id].gc.free = npcmgpio_gpio_free; + id++; + } + pctrl->bank_num = id; return ret; } From 24a9dbb1c1575b9890bb7d9028cc89894da6dc22 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 30 Mar 2022 17:59:10 +0300 Subject: [PATCH 068/319] gpiolib: Move error message out of a spinlock An error path is a slow path, no need to block other CPUs when printing error messages. Signed-off-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 3ffe7b6cebbf..1e4aa358e606 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -288,7 +288,6 @@ static int gpiodev_add_to_list(struct gpio_device *gdev) } } - dev_err(&gdev->dev, "GPIO integer space overlap, cannot add chip\n"); return -EBUSY; } @@ -727,6 +726,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, ret = gpiodev_add_to_list(gdev); if (ret) { spin_unlock_irqrestore(&gpio_lock, flags); + chip_err(gc, "GPIO integer space overlap, cannot add chip\n"); goto err_free_label; } From 57017edd46f835c85642fe8299f13b0db61d4c31 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 8 Apr 2022 21:18:50 +0300 Subject: [PATCH 069/319] gpiolib: Embed iterator variable into for_each_gpio_desc_with_flag() The iterator loop is used exclusively to get a descriptor, which in its turn is what is being used by the caller. Embed the iterator variable into the loop in the for_each_gpio_desc_with_flag() macro helper. Suggested-by: Bartosz Golaszewski Signed-off-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib-of.c | 3 +-- drivers/gpio/gpiolib-sysfs.c | 3 +-- drivers/gpio/gpiolib.c | 3 +-- drivers/gpio/gpiolib.h | 8 ++++---- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index ae1ce319cd78..47c0e07802d6 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -712,9 +712,8 @@ static void of_gpiochip_remove_hog(struct gpio_chip *chip, struct device_node *hog) { struct gpio_desc *desc; - unsigned int i; - for_each_gpio_desc_with_flag(i, chip, desc, FLAG_IS_HOGGED) + for_each_gpio_desc_with_flag(chip, desc, FLAG_IS_HOGGED) if (desc->hog == hog) gpiochip_free_own_desc(desc); } diff --git a/drivers/gpio/gpiolib-sysfs.c b/drivers/gpio/gpiolib-sysfs.c index d44ffea038f5..cd27bf173dec 100644 --- a/drivers/gpio/gpiolib-sysfs.c +++ b/drivers/gpio/gpiolib-sysfs.c @@ -760,7 +760,6 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) { struct gpio_desc *desc; struct gpio_chip *chip = gdev->chip; - unsigned int i; if (!gdev->mockdev) return; @@ -773,7 +772,7 @@ void gpiochip_sysfs_unregister(struct gpio_device *gdev) mutex_unlock(&sysfs_lock); /* unregister gpiod class devices owned by sysfs */ - for_each_gpio_desc_with_flag(i, chip, desc, FLAG_SYSFS) + for_each_gpio_desc_with_flag(chip, desc, FLAG_SYSFS) gpiod_free(desc); } diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 1e4aa358e606..d41041966cf5 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -4119,9 +4119,8 @@ int gpiod_hog(struct gpio_desc *desc, const char *name, static void gpiochip_free_hogs(struct gpio_chip *gc) { struct gpio_desc *desc; - int id; - for_each_gpio_desc_with_flag(id, gc, desc, FLAG_IS_HOGGED) + for_each_gpio_desc_with_flag(gc, desc, FLAG_IS_HOGGED) gpiochip_free_own_desc(desc); } diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 06f3faa9fbef..7bac62d36f0c 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -100,10 +100,10 @@ struct gpio_array { struct gpio_desc *gpiochip_get_desc(struct gpio_chip *gc, unsigned int hwnum); -#define for_each_gpio_desc_with_flag(i, gc, desc, flag) \ - for (i = 0, desc = gpiochip_get_desc(gc, i); \ - i < gc->ngpio; \ - i++, desc = gpiochip_get_desc(gc, i)) \ +#define for_each_gpio_desc_with_flag(gc, desc, flag) \ + for (unsigned int __i = 0; \ + __i < gc->ngpio && (desc = gpiochip_get_desc(gc, __i)); \ + __i++) \ if (!test_bit(flag, &desc->flags)) {} else int gpiod_get_array_value_complex(bool raw, bool can_sleep, From 66f46e370a9aec05524cdffde4062953e6ba2e08 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 8 Apr 2022 21:18:51 +0300 Subject: [PATCH 070/319] gpiolib: Split out for_each_gpio_desc() macro In some cases we want to traverse all GPIO descriptors for given chip, let's split out for_each_gpio_desc() macro for such cases. Signed-off-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 11 +++-------- drivers/gpio/gpiolib.h | 5 ++++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index d41041966cf5..2bdc1efad66e 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -308,15 +308,10 @@ static struct gpio_desc *gpio_name_to_desc(const char * const name) spin_lock_irqsave(&gpio_lock, flags); list_for_each_entry(gdev, &gpio_devices, list) { - int i; + struct gpio_desc *desc; - for (i = 0; i != gdev->ngpio; ++i) { - struct gpio_desc *desc = &gdev->descs[i]; - - if (!desc->name) - continue; - - if (!strcmp(desc->name, name)) { + for_each_gpio_desc(gdev->chip, desc) { + if (desc->name && !strcmp(desc->name, name)) { spin_unlock_irqrestore(&gpio_lock, flags); return desc; } diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h index 7bac62d36f0c..eef3ec073d9e 100644 --- a/drivers/gpio/gpiolib.h +++ b/drivers/gpio/gpiolib.h @@ -100,10 +100,13 @@ struct gpio_array { struct gpio_desc *gpiochip_get_desc(struct gpio_chip *gc, unsigned int hwnum); -#define for_each_gpio_desc_with_flag(gc, desc, flag) \ +#define for_each_gpio_desc(gc, desc) \ for (unsigned int __i = 0; \ __i < gc->ngpio && (desc = gpiochip_get_desc(gc, __i)); \ __i++) \ + +#define for_each_gpio_desc_with_flag(gc, desc, flag) \ + for_each_gpio_desc(gc, desc) \ if (!test_bit(flag, &desc->flags)) {} else int gpiod_get_array_value_complex(bool raw, bool can_sleep, From 3de69ae1c407da6cbeca75fd3ee6a40237d899dd Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 8 Apr 2022 21:18:52 +0300 Subject: [PATCH 071/319] gpiolib: Refactor gpiolib_dbg_show() with help of for_each_gpio_desc() Use for_each_gpio_desc() and since we would need to touch the entire conditionals, do the following: - rename last occurrence of gdesc to desc - use short ternary operator ?: - join two seq_printf() calls into single one - fix indentation of the seq_printf() parameters Signed-off-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 2bdc1efad66e..eda3f2f70a2d 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -4375,34 +4375,32 @@ core_initcall(gpiolib_dev_init); static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) { - unsigned i; struct gpio_chip *gc = gdev->chip; + struct gpio_desc *desc; unsigned gpio = gdev->base; - struct gpio_desc *gdesc = &gdev->descs[0]; + int value; bool is_out; bool is_irq; bool active_low; - for (i = 0; i < gdev->ngpio; i++, gpio++, gdesc++) { - if (!test_bit(FLAG_REQUESTED, &gdesc->flags)) { - if (gdesc->name) { - seq_printf(s, " gpio-%-3d (%-20.20s)\n", - gpio, gdesc->name); - } - continue; + for_each_gpio_desc(gc, desc) { + if (test_bit(FLAG_REQUESTED, &desc->flags)) { + gpiod_get_direction(desc); + is_out = test_bit(FLAG_IS_OUT, &desc->flags); + value = gc->get ? gc->get(gc, gpio_chip_hwgpio(desc)) : -EIO; + is_irq = test_bit(FLAG_USED_AS_IRQ, &desc->flags); + active_low = test_bit(FLAG_ACTIVE_LOW, &desc->flags); + seq_printf(s, " gpio-%-3d (%-20.20s|%-20.20s) %s %s %s%s\n", + gpio, desc->name ?: "", desc->label, + is_out ? "out" : "in ", + value >= 0 ? (value ? "hi" : "lo") : "? ", + is_irq ? "IRQ " : "", + active_low ? "ACTIVE LOW" : ""); + } else if (desc->name) { + seq_printf(s, " gpio-%-3d (%-20.20s)\n", gpio, desc->name); } - gpiod_get_direction(gdesc); - is_out = test_bit(FLAG_IS_OUT, &gdesc->flags); - is_irq = test_bit(FLAG_USED_AS_IRQ, &gdesc->flags); - active_low = test_bit(FLAG_ACTIVE_LOW, &gdesc->flags); - seq_printf(s, " gpio-%-3d (%-20.20s|%-20.20s) %s %s %s%s", - gpio, gdesc->name ? gdesc->name : "", gdesc->label, - is_out ? "out" : "in ", - gc->get ? (gc->get(gc, i) ? "hi" : "lo") : "? ", - is_irq ? "IRQ " : "", - active_low ? "ACTIVE LOW" : ""); - seq_printf(s, "\n"); + gpio++; } } From 234c52097ce416028854d2167afb38b7718b9a94 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 8 Apr 2022 21:18:53 +0300 Subject: [PATCH 072/319] gpiolib: Extract gpio_chip_get_value() wrapper In couple of cases we are using the same code to wrap ->get() callback. Extract that code into a helper for the sake of better maintenance. Signed-off-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpiolib.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index eda3f2f70a2d..2e18fef3847e 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -2513,6 +2513,11 @@ void gpiod_toggle_active_low(struct gpio_desc *desc) } EXPORT_SYMBOL_GPL(gpiod_toggle_active_low); +static int gpio_chip_get_value(struct gpio_chip *gc, const struct gpio_desc *desc) +{ + return gc->get ? gc->get(gc, gpio_chip_hwgpio(desc)) : -EIO; +} + /* I/O calls are only valid after configuration completed; the relevant * "is this a valid GPIO" error checks should already have been done. * @@ -2538,12 +2543,10 @@ EXPORT_SYMBOL_GPL(gpiod_toggle_active_low); static int gpiod_get_raw_value_commit(const struct gpio_desc *desc) { struct gpio_chip *gc; - int offset; int value; gc = desc->gdev->chip; - offset = gpio_chip_hwgpio(desc); - value = gc->get ? gc->get(gc, offset) : -EIO; + value = gpio_chip_get_value(gc, desc); value = value < 0 ? value : !!value; trace_gpio_value(desc_to_gpio(desc), 1, value); return value; @@ -4387,7 +4390,7 @@ static void gpiolib_dbg_show(struct seq_file *s, struct gpio_device *gdev) if (test_bit(FLAG_REQUESTED, &desc->flags)) { gpiod_get_direction(desc); is_out = test_bit(FLAG_IS_OUT, &desc->flags); - value = gc->get ? gc->get(gc, gpio_chip_hwgpio(desc)) : -EIO; + value = gpio_chip_get_value(gc, desc); is_irq = test_bit(FLAG_USED_AS_IRQ, &desc->flags); active_low = test_bit(FLAG_ACTIVE_LOW, &desc->flags); seq_printf(s, " gpio-%-3d (%-20.20s|%-20.20s) %s %s %s%s\n", From 06a6a774f6b8ed94aefcf60caa56fe92d3a18e17 Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 9 Apr 2022 21:55:46 +0200 Subject: [PATCH 073/319] dt-bindings: gpio: realtek-otto: Add rtl9300 compatible Add the "realtek,rlt9300-gpio", "realtek,otto-gpio" compatible for GPIO nodes on the RTL930x SoC series. This SoC requires an extra register range, defining the interrupt CPU mapping registers. Signed-off-by: Sander Vanheule Reviewed-by: Krzysztof Kozlowski Signed-off-by: Bartosz Golaszewski --- .../bindings/gpio/realtek,otto-gpio.yaml | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml b/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml index 100f20cebd76..3c511e9af377 100644 --- a/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml @@ -28,10 +28,10 @@ properties: - enum: - realtek,rtl8380-gpio - realtek,rtl8390-gpio + - realtek,rtl9300-gpio - const: realtek,otto-gpio - reg: - maxItems: 1 + reg: true "#gpio-cells": const: 2 @@ -50,6 +50,23 @@ properties: interrupts: maxItems: 1 +if: + properties: + compatible: + contains: + const: realtek,rtl9300-gpio +then: + properties: + reg: + items: + - description: GPIO and interrupt control + - description: interrupt CPU map +else: + properties: + reg: + items: + - description: GPIO and interrupt control + required: - compatible - reg @@ -74,5 +91,17 @@ examples: interrupt-parent = <&rtlintc>; interrupts = <23>; }; + - | + gpio@3300 { + compatible = "realtek,rtl9300-gpio", "realtek,otto-gpio"; + reg = <0x3300 0x1c>, <0x3338 0x8>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <24>; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&rtlintc>; + interrupts = <13>; + }; ... From 512c5be35223d9baa2629efa1084cf5210eaee80 Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 9 Apr 2022 21:55:47 +0200 Subject: [PATCH 074/319] gpio: realtek-otto: Support reversed port layouts The GPIO port layout on the RTL930x SoC series is reversed compared to the RTL838x and RTL839x SoC series. Add new port offset calculator functions to ensure the correct order is used when reading port IRQ data, and ensure bgpio uses the right byte ordering. Signed-off-by: Sander Vanheule Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-realtek-otto.c | 55 +++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c index bd75401b549d..c838ad8ce55f 100644 --- a/drivers/gpio/gpio-realtek-otto.c +++ b/drivers/gpio/gpio-realtek-otto.c @@ -58,6 +58,8 @@ struct realtek_gpio_ctrl { raw_spinlock_t lock; u16 intr_mask[REALTEK_GPIO_PORTS_PER_BANK]; u16 intr_type[REALTEK_GPIO_PORTS_PER_BANK]; + unsigned int (*port_offset_u8)(unsigned int port); + unsigned int (*port_offset_u16)(unsigned int port); }; /* Expand with more flags as devices with other quirks are added */ @@ -69,6 +71,11 @@ enum realtek_gpio_flags { * line the IRQ handler was assigned to, causing uncaught interrupts. */ GPIO_INTERRUPTS_DISABLED = BIT(0), + /* + * Port order is reversed, meaning DCBA register layout for 1-bit + * fields, and [BA, DC] for 2-bit fields. + */ + GPIO_PORTS_REVERSED = BIT(1), }; static struct realtek_gpio_ctrl *irq_data_to_ctrl(struct irq_data *data) @@ -86,21 +93,50 @@ static struct realtek_gpio_ctrl *irq_data_to_ctrl(struct irq_data *data) * port. The two interrupt mask registers store two bits per GPIO, so use u16 * values. */ +static unsigned int realtek_gpio_port_offset_u8(unsigned int port) +{ + return port; +} + +static unsigned int realtek_gpio_port_offset_u16(unsigned int port) +{ + return 2 * port; +} + +/* + * Reversed port order register access + * + * For registers with one bit per GPIO, all ports are stored as u8-s in one + * register in reversed order. The two interrupt mask registers store two bits + * per GPIO, so use u16 values. The first register contains ports 1 and 0, the + * second ports 3 and 2. + */ +static unsigned int realtek_gpio_port_offset_u8_rev(unsigned int port) +{ + return 3 - port; +} + +static unsigned int realtek_gpio_port_offset_u16_rev(unsigned int port) +{ + return 2 * (port ^ 1); +} + static void realtek_gpio_write_imr(struct realtek_gpio_ctrl *ctrl, unsigned int port, u16 irq_type, u16 irq_mask) { - iowrite16(irq_type & irq_mask, ctrl->base + REALTEK_GPIO_REG_IMR + 2 * port); + iowrite16(irq_type & irq_mask, + ctrl->base + REALTEK_GPIO_REG_IMR + ctrl->port_offset_u16(port)); } static void realtek_gpio_clear_isr(struct realtek_gpio_ctrl *ctrl, unsigned int port, u8 mask) { - iowrite8(mask, ctrl->base + REALTEK_GPIO_REG_ISR + port); + iowrite8(mask, ctrl->base + REALTEK_GPIO_REG_ISR + ctrl->port_offset_u8(port)); } static u8 realtek_gpio_read_isr(struct realtek_gpio_ctrl *ctrl, unsigned int port) { - return ioread8(ctrl->base + REALTEK_GPIO_REG_ISR + port); + return ioread8(ctrl->base + REALTEK_GPIO_REG_ISR + ctrl->port_offset_u8(port)); } /* Set the rising and falling edge mask bits for a GPIO port pin */ @@ -250,6 +286,7 @@ MODULE_DEVICE_TABLE(of, realtek_gpio_of_match); static int realtek_gpio_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; + unsigned long bgpio_flags; unsigned int dev_flags; struct gpio_irq_chip *girq; struct realtek_gpio_ctrl *ctrl; @@ -277,10 +314,20 @@ static int realtek_gpio_probe(struct platform_device *pdev) raw_spin_lock_init(&ctrl->lock); + if (dev_flags & GPIO_PORTS_REVERSED) { + bgpio_flags = 0; + ctrl->port_offset_u8 = realtek_gpio_port_offset_u8_rev; + ctrl->port_offset_u16 = realtek_gpio_port_offset_u16_rev; + } else { + bgpio_flags = BGPIOF_BIG_ENDIAN_BYTE_ORDER; + ctrl->port_offset_u8 = realtek_gpio_port_offset_u8; + ctrl->port_offset_u16 = realtek_gpio_port_offset_u16; + } + err = bgpio_init(&ctrl->gc, dev, 4, ctrl->base + REALTEK_GPIO_REG_DATA, NULL, NULL, ctrl->base + REALTEK_GPIO_REG_DIR, NULL, - BGPIOF_BIG_ENDIAN_BYTE_ORDER); + bgpio_flags); if (err) { dev_err(dev, "unable to init generic GPIO"); return err; From 95fa6dbe58f286a8f87cb37b7516232eb678de2d Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 9 Apr 2022 21:55:48 +0200 Subject: [PATCH 075/319] gpio: realtek-otto: Support per-cpu interrupts On SoCs with multiple cores, it is possible that the GPIO interrupt controller supports assigning specific pins to one or more cores. IRQ balancing can be performed on a line-by-line basis if the parent interrupt is routed to all available cores, which is the default upon initialisation. Signed-off-by: Sander Vanheule Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-realtek-otto.c | 75 +++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c index c838ad8ce55f..dd1b7656d23a 100644 --- a/drivers/gpio/gpio-realtek-otto.c +++ b/drivers/gpio/gpio-realtek-otto.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only #include +#include #include #include #include @@ -55,6 +56,8 @@ struct realtek_gpio_ctrl { struct gpio_chip gc; void __iomem *base; + void __iomem *cpumask_base; + struct cpumask cpu_irq_maskable; raw_spinlock_t lock; u16 intr_mask[REALTEK_GPIO_PORTS_PER_BANK]; u16 intr_type[REALTEK_GPIO_PORTS_PER_BANK]; @@ -76,6 +79,11 @@ enum realtek_gpio_flags { * fields, and [BA, DC] for 2-bit fields. */ GPIO_PORTS_REVERSED = BIT(1), + /* + * Interrupts can be enabled per cpu. This requires a secondary IO + * range, where the per-cpu enable masks are located. + */ + GPIO_INTERRUPTS_PER_CPU = BIT(2), }; static struct realtek_gpio_ctrl *irq_data_to_ctrl(struct irq_data *data) @@ -247,14 +255,61 @@ static void realtek_gpio_irq_handler(struct irq_desc *desc) chained_irq_exit(irq_chip, desc); } +static inline void __iomem *realtek_gpio_irq_cpu_mask(struct realtek_gpio_ctrl *ctrl, + unsigned int port, int cpu) +{ + return ctrl->cpumask_base + ctrl->port_offset_u8(port) + + REALTEK_GPIO_PORTS_PER_BANK * cpu; +} + +static int realtek_gpio_irq_set_affinity(struct irq_data *data, + const struct cpumask *dest, bool force) +{ + struct realtek_gpio_ctrl *ctrl = irq_data_to_ctrl(data); + unsigned int line = irqd_to_hwirq(data); + unsigned int port = line / 8; + unsigned int port_pin = line % 8; + void __iomem *irq_cpu_mask; + unsigned long flags; + int cpu; + u8 v; + + if (!ctrl->cpumask_base) + return -ENXIO; + + raw_spin_lock_irqsave(&ctrl->lock, flags); + + for_each_cpu(cpu, &ctrl->cpu_irq_maskable) { + irq_cpu_mask = realtek_gpio_irq_cpu_mask(ctrl, port, cpu); + v = ioread8(irq_cpu_mask); + + if (cpumask_test_cpu(cpu, dest)) + v |= BIT(port_pin); + else + v &= ~BIT(port_pin); + + iowrite8(v, irq_cpu_mask); + } + + raw_spin_unlock_irqrestore(&ctrl->lock, flags); + + irq_data_update_effective_affinity(data, dest); + + return 0; +} + static int realtek_gpio_irq_init(struct gpio_chip *gc) { struct realtek_gpio_ctrl *ctrl = gpiochip_get_data(gc); unsigned int port; + int cpu; for (port = 0; (port * 8) < gc->ngpio; port++) { realtek_gpio_write_imr(ctrl, port, 0, 0); realtek_gpio_clear_isr(ctrl, port, GENMASK(7, 0)); + + for_each_cpu(cpu, &ctrl->cpu_irq_maskable) + iowrite8(GENMASK(7, 0), realtek_gpio_irq_cpu_mask(ctrl, port, cpu)); } return 0; @@ -266,6 +321,7 @@ static struct irq_chip realtek_gpio_irq_chip = { .irq_mask = realtek_gpio_irq_mask, .irq_unmask = realtek_gpio_irq_unmask, .irq_set_type = realtek_gpio_irq_set_type, + .irq_set_affinity = realtek_gpio_irq_set_affinity, }; static const struct of_device_id realtek_gpio_of_match[] = { @@ -290,8 +346,10 @@ static int realtek_gpio_probe(struct platform_device *pdev) unsigned int dev_flags; struct gpio_irq_chip *girq; struct realtek_gpio_ctrl *ctrl; + struct resource *res; u32 ngpios; - int err, irq; + unsigned int nr_cpus; + int cpu, err, irq; ctrl = devm_kzalloc(dev, sizeof(*ctrl), GFP_KERNEL); if (!ctrl) @@ -352,6 +410,21 @@ static int realtek_gpio_probe(struct platform_device *pdev) girq->init_hw = realtek_gpio_irq_init; } + cpumask_clear(&ctrl->cpu_irq_maskable); + + if ((dev_flags & GPIO_INTERRUPTS_PER_CPU) && irq > 0) { + ctrl->cpumask_base = devm_platform_get_and_ioremap_resource(pdev, 1, &res); + if (IS_ERR(ctrl->cpumask_base)) + return dev_err_probe(dev, PTR_ERR(ctrl->cpumask_base), + "missing CPU IRQ mask registers"); + + nr_cpus = resource_size(res) / REALTEK_GPIO_PORTS_PER_BANK; + nr_cpus = min(nr_cpus, num_present_cpus()); + + for (cpu = 0; cpu < nr_cpus; cpu++) + cpumask_set_cpu(cpu, &ctrl->cpu_irq_maskable); + } + return devm_gpiochip_add_data(dev, &ctrl->gc, ctrl); } From deaf1cecdeb052cdb5e92fd642016198724b44a4 Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 9 Apr 2022 21:55:49 +0200 Subject: [PATCH 076/319] gpio: realtek-otto: Add RTL930x support The RTL930x SoC series has support for 24 GPIOs, with the port order reversed compared to RTL838x and RTL839x. The RTL930x series also has two CPUs (VPEs) and can distribute individual GPIO interrupts between them. Signed-off-by: Sander Vanheule Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-realtek-otto.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c index dd1b7656d23a..3ddaa17accff 100644 --- a/drivers/gpio/gpio-realtek-otto.c +++ b/drivers/gpio/gpio-realtek-otto.c @@ -335,6 +335,10 @@ static const struct of_device_id realtek_gpio_of_match[] = { { .compatible = "realtek,rtl8390-gpio", }, + { + .compatible = "realtek,rtl9300-gpio", + .data = (void *)(GPIO_PORTS_REVERSED | GPIO_INTERRUPTS_PER_CPU) + }, {} }; MODULE_DEVICE_TABLE(of, realtek_gpio_of_match); From d0b55b6912f3c249097977eba8f3ed580d5f1c6b Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 9 Apr 2022 21:55:50 +0200 Subject: [PATCH 077/319] dt-bindings: gpio: realtek-otto: Add rtl9310 compatible Add the "realtek,rlt9310-gpio", "realtek,otto-gpio" compatible for GPIO nodes on the RTL931x SoC series. Signed-off-by: Sander Vanheule Acked-by: Krzysztof Kozlowski Signed-off-by: Bartosz Golaszewski --- Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml b/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml index 3c511e9af377..39fd959c45d2 100644 --- a/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/realtek,otto-gpio.yaml @@ -29,6 +29,7 @@ properties: - realtek,rtl8380-gpio - realtek,rtl8390-gpio - realtek,rtl9300-gpio + - realtek,rtl9310-gpio - const: realtek,otto-gpio reg: true From d3bf3dc4bbbf6109bd9b4bd60089d36205ec4a37 Mon Sep 17 00:00:00 2001 From: Sander Vanheule Date: Sat, 9 Apr 2022 21:55:51 +0200 Subject: [PATCH 078/319] gpio: realtek-otto: Add RTL931x support The RTL931x SoC series has support for 32 GPIOs, although not all lines may be broken out to a physical pad. The GPIO bank's parent interrupt can be routed to either or both of the SoC's CPU cores by the GIC. Line-by-line IRQ balancing is not possible on these SoCs. Signed-off-by: Sander Vanheule Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-realtek-otto.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpio/gpio-realtek-otto.c b/drivers/gpio/gpio-realtek-otto.c index 3ddaa17accff..c52b2cb1acae 100644 --- a/drivers/gpio/gpio-realtek-otto.c +++ b/drivers/gpio/gpio-realtek-otto.c @@ -339,6 +339,9 @@ static const struct of_device_id realtek_gpio_of_match[] = { .compatible = "realtek,rtl9300-gpio", .data = (void *)(GPIO_PORTS_REVERSED | GPIO_INTERRUPTS_PER_CPU) }, + { + .compatible = "realtek,rtl9310-gpio", + }, {} }; MODULE_DEVICE_TABLE(of, realtek_gpio_of_match); From 04c975121cae872acb553e1ad0a132f9b4a2640b Mon Sep 17 00:00:00 2001 From: Like Xu Date: Wed, 6 Apr 2022 14:37:12 +0800 Subject: [PATCH 079/319] KVM: x86/xen: Remove the redundantly included header file lapic.h The header lapic.h is included more than once, remove one of them. Signed-off-by: Like Xu Message-Id: <20220406063715.55625-2-likexu@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 7e7c8a5bff52..610beba35907 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -8,7 +8,6 @@ #include "x86.h" #include "xen.h" -#include "lapic.h" #include "hyperv.h" #include "lapic.h" From 42c35fdc340ffbf6b3a8ffbdd5b53d856ecf924b Mon Sep 17 00:00:00 2001 From: Like Xu Date: Wed, 6 Apr 2022 14:37:14 +0800 Subject: [PATCH 080/319] selftests: kvm/x86/xen: Replace a comma in the xen_shinfo_test with semicolon +WARNING: Possible comma where semicolon could be used +#397: FILE: tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c:700: ++ tmr.type = KVM_XEN_VCPU_ATTR_TYPE_TIMER, ++ vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_GET_ATTR, &tmr); Fixes: 25eaeebe710c ("KVM: x86/xen: Add self tests for KVM_XEN_HVM_CONFIG_EVTCHN_SEND") Signed-off-by: Like Xu Message-Id: <20220406063715.55625-4-likexu@tencent.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c index d9d9d1deec45..5e6f40633ea6 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c @@ -702,7 +702,7 @@ int main(int argc, char *argv[]) case 14: memset(&tmr, 0, sizeof(tmr)); - tmr.type = KVM_XEN_VCPU_ATTR_TYPE_TIMER, + tmr.type = KVM_XEN_VCPU_ATTR_TYPE_TIMER; vcpu_ioctl(vm, VCPU_ID, KVM_XEN_VCPU_GET_ATTR, &tmr); TEST_ASSERT(tmr.u.timer.port == EVTCHN_TIMER, "Timer port not returned"); From 8176472563fb1a233f46f3f0441738c82afd88da Mon Sep 17 00:00:00 2001 From: Peng Hao Date: Mon, 28 Feb 2022 11:07:49 +0800 Subject: [PATCH 081/319] kvm: x86: Adjust the location of pkru_mask of kvm_mmu to reduce memory Adjust the field pkru_mask to the back of direct_map to make up 8-byte alignment.This reduces the size of kvm_mmu by 8 bytes. Signed-off-by: Peng Hao Message-Id: <20220228030749.88353-1-flyingpeng@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f1dfa066068d..e9c9a23d6623 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -443,14 +443,6 @@ struct kvm_mmu { u8 shadow_root_level; u8 ept_ad; bool direct_map; - struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; - - /* - * Bitmap; bit set = permission fault - * Byte index: page fault error code [4:1] - * Bit index: pte permissions in ACC_* format - */ - u8 permissions[16]; /* * The pkru_mask indicates if protection key checks are needed. It @@ -460,6 +452,15 @@ struct kvm_mmu { */ u32 pkru_mask; + struct kvm_mmu_root_info prev_roots[KVM_MMU_NUM_PREV_ROOTS]; + + /* + * Bitmap; bit set = permission fault + * Byte index: page fault error code [4:1] + * Bit index: pte permissions in ACC_* format + */ + u8 permissions[16]; + u64 *pae_root; u64 *pml4_root; u64 *pml5_root; From 6e97b2b822902f80142be26614fc0f6a8053ac45 Mon Sep 17 00:00:00 2001 From: Peng Hao Date: Mon, 28 Feb 2022 11:09:02 +0800 Subject: [PATCH 082/319] kvm: vmx: remove redundant parentheses Remove redundant parentheses. Signed-off-by: Peng Hao Message-Id: <20220228030902.88465-1-flyingpeng@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c654c9d76e09..ef0aadb0ab3a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2444,7 +2444,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf, &_cpu_based_exec_control) < 0) return -EIO; #ifdef CONFIG_X86_64 - if ((_cpu_based_exec_control & CPU_BASED_TPR_SHADOW)) + if (_cpu_based_exec_control & CPU_BASED_TPR_SHADOW) _cpu_based_exec_control &= ~CPU_BASED_CR8_LOAD_EXITING & ~CPU_BASED_CR8_STORE_EXITING; #endif From 77d727926607ba42f0c4ba82baaec1076e8cbebf Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Tue, 8 Mar 2022 00:18:49 -0800 Subject: [PATCH 083/319] x86/kvm: Don't waste kvmclock memory if there is nopv parameter When the "nopv" command line parameter is used, it should not waste memory for kvmclock. Signed-off-by: Wanpeng Li Message-Id: <1646727529-11774-1-git-send-email-wanpengli@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kernel/kvmclock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index c5caa7311bd8..16333ba1904b 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -239,7 +239,7 @@ static void __init kvmclock_init_mem(void) static int __init kvm_setup_vsyscall_timeinfo(void) { - if (!kvm_para_available() || !kvmclock) + if (!kvm_para_available() || !kvmclock || nopv) return 0; kvmclock_init_mem(); From aecce510fee5620bf1906f94772fdda3b9966455 Mon Sep 17 00:00:00 2001 From: SU Hang Date: Tue, 29 Mar 2022 11:01:06 +0800 Subject: [PATCH 084/319] KVM: VMX: replace 0x180 with EPT_VIOLATION_* definition Using self-expressing macro definition EPT_VIOLATION_GVA_VALIDATION and EPT_VIOLATION_GVA_TRANSLATED instead of 0x180 in FNAME(walk_addr_generic)(). Signed-off-by: SU Hang Reviewed-by: Sean Christopherson Message-Id: <20220329030108.97341-2-darcy.sh@antgroup.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/vmx.h | 2 ++ arch/x86/kvm/mmu/paging_tmpl.h | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 0ffaa3156a4e..3586d4aeaac7 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -546,6 +546,7 @@ enum vm_entry_failure_code { #define EPT_VIOLATION_READABLE_BIT 3 #define EPT_VIOLATION_WRITABLE_BIT 4 #define EPT_VIOLATION_EXECUTABLE_BIT 5 +#define EPT_VIOLATION_GVA_IS_VALID_BIT 7 #define EPT_VIOLATION_GVA_TRANSLATED_BIT 8 #define EPT_VIOLATION_ACC_READ (1 << EPT_VIOLATION_ACC_READ_BIT) #define EPT_VIOLATION_ACC_WRITE (1 << EPT_VIOLATION_ACC_WRITE_BIT) @@ -553,6 +554,7 @@ enum vm_entry_failure_code { #define EPT_VIOLATION_READABLE (1 << EPT_VIOLATION_READABLE_BIT) #define EPT_VIOLATION_WRITABLE (1 << EPT_VIOLATION_WRITABLE_BIT) #define EPT_VIOLATION_EXECUTABLE (1 << EPT_VIOLATION_EXECUTABLE_BIT) +#define EPT_VIOLATION_GVA_IS_VALID (1 << EPT_VIOLATION_GVA_IS_VALID_BIT) #define EPT_VIOLATION_GVA_TRANSLATED (1 << EPT_VIOLATION_GVA_TRANSLATED_BIT) /* diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 01fee5f67ac3..9c6bdab9f29b 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -515,7 +515,8 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, * The other bits are set to 0. */ if (!(errcode & PFERR_RSVD_MASK)) { - vcpu->arch.exit_qualification &= 0x180; + vcpu->arch.exit_qualification &= (EPT_VIOLATION_GVA_IS_VALID | + EPT_VIOLATION_GVA_TRANSLATED); if (write_fault) vcpu->arch.exit_qualification |= EPT_VIOLATION_ACC_WRITE; if (user_fault) From ca2a7c22a115d1a9a60ce4f61fe43d5fcaaaa516 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 29 Mar 2022 11:01:07 +0800 Subject: [PATCH 085/319] KVM: x86/mmu: Derive EPT violation RWX bits from EPTE RWX bits Derive the mask of RWX bits reported on EPT violations from the mask of RWX bits that are shoved into EPT entries; the layout is the same, the EPT violation bits are simply shifted by three. Use the new shift and a slight copy-paste of the mask derivation instead of completely open coding the same to convert between the EPT entry bits and the exit qualification when synthesizing a nested EPT Violation. No functional change intended. Cc: SU Hang Signed-off-by: Sean Christopherson Message-Id: <20220329030108.97341-3-darcy.sh@antgroup.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/vmx.h | 8 ++------ arch/x86/kvm/mmu/paging_tmpl.h | 8 +++++++- arch/x86/kvm/vmx/vmx.c | 4 +--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 3586d4aeaac7..6c343c6a1855 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -543,17 +543,13 @@ enum vm_entry_failure_code { #define EPT_VIOLATION_ACC_READ_BIT 0 #define EPT_VIOLATION_ACC_WRITE_BIT 1 #define EPT_VIOLATION_ACC_INSTR_BIT 2 -#define EPT_VIOLATION_READABLE_BIT 3 -#define EPT_VIOLATION_WRITABLE_BIT 4 -#define EPT_VIOLATION_EXECUTABLE_BIT 5 +#define EPT_VIOLATION_RWX_SHIFT 3 #define EPT_VIOLATION_GVA_IS_VALID_BIT 7 #define EPT_VIOLATION_GVA_TRANSLATED_BIT 8 #define EPT_VIOLATION_ACC_READ (1 << EPT_VIOLATION_ACC_READ_BIT) #define EPT_VIOLATION_ACC_WRITE (1 << EPT_VIOLATION_ACC_WRITE_BIT) #define EPT_VIOLATION_ACC_INSTR (1 << EPT_VIOLATION_ACC_INSTR_BIT) -#define EPT_VIOLATION_READABLE (1 << EPT_VIOLATION_READABLE_BIT) -#define EPT_VIOLATION_WRITABLE (1 << EPT_VIOLATION_WRITABLE_BIT) -#define EPT_VIOLATION_EXECUTABLE (1 << EPT_VIOLATION_EXECUTABLE_BIT) +#define EPT_VIOLATION_RWX_MASK (VMX_EPT_RWX_MASK << EPT_VIOLATION_RWX_SHIFT) #define EPT_VIOLATION_GVA_IS_VALID (1 << EPT_VIOLATION_GVA_IS_VALID_BIT) #define EPT_VIOLATION_GVA_TRANSLATED (1 << EPT_VIOLATION_GVA_TRANSLATED_BIT) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 9c6bdab9f29b..1dad8f3f2bb0 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -523,7 +523,13 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, vcpu->arch.exit_qualification |= EPT_VIOLATION_ACC_READ; if (fetch_fault) vcpu->arch.exit_qualification |= EPT_VIOLATION_ACC_INSTR; - vcpu->arch.exit_qualification |= (pte_access & 0x7) << 3; + + /* + * Note, pte_access holds the raw RWX bits from the EPTE, not + * ACC_*_MASK flags! + */ + vcpu->arch.exit_qualification |= (pte_access & VMX_EPT_RWX_MASK) << + EPT_VIOLATION_RWX_SHIFT; } #endif walker->fault.address = addr; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index ef0aadb0ab3a..fb0b0fa6b949 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5405,9 +5405,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR) ? PFERR_FETCH_MASK : 0; /* ept page table entry is present? */ - error_code |= (exit_qualification & - (EPT_VIOLATION_READABLE | EPT_VIOLATION_WRITABLE | - EPT_VIOLATION_EXECUTABLE)) + error_code |= (exit_qualification & EPT_VIOLATION_RWX_MASK) ? PFERR_PRESENT_MASK : 0; error_code |= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) != 0 ? From fdc298da866165ec0288fe227982f1aa0467bf5c Mon Sep 17 00:00:00 2001 From: Like Xu Date: Tue, 29 Mar 2022 23:50:51 +0000 Subject: [PATCH 086/319] KVM: x86: Move kvm_ops_static_call_update() to x86.c The kvm_ops_static_call_update() is defined in kvm_host.h. That's completely unnecessary, it should have exactly one caller, kvm_arch_hardware_setup(). Move the helper to x86.c and have it do the actual memcpy() of the ops in addition to the static call updates. This will also allow for cleanly giving kvm_pmu_ops static_call treatment. Suggested-by: Sean Christopherson Signed-off-by: Like Xu [sean: Move memcpy() into the helper and rename accordingly] Signed-off-by: Sean Christopherson Message-Id: <20220329235054.3534728-2-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 14 -------------- arch/x86/kvm/x86.c | 19 +++++++++++++++++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index e9c9a23d6623..25d1aac74c55 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1563,20 +1563,6 @@ extern struct kvm_x86_ops kvm_x86_ops; #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP #include -static inline void kvm_ops_static_call_update(void) -{ -#define __KVM_X86_OP(func) \ - static_call_update(kvm_x86_##func, kvm_x86_ops.func); -#define KVM_X86_OP(func) \ - WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func) -#define KVM_X86_OP_OPTIONAL __KVM_X86_OP -#define KVM_X86_OP_OPTIONAL_RET0(func) \ - static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \ - (void *)__static_call_return0); -#include -#undef __KVM_X86_OP -} - #define __KVM_HAVE_ARCH_VM_ALLOC static inline struct kvm *kvm_arch_alloc_vm(void) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 10ad1029f69a..d22fc56d8bb8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11618,6 +11618,22 @@ void kvm_arch_hardware_disable(void) drop_user_return_notifiers(); } +static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) +{ + memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); + +#define __KVM_X86_OP(func) \ + static_call_update(kvm_x86_##func, kvm_x86_ops.func); +#define KVM_X86_OP(func) \ + WARN_ON(!kvm_x86_ops.func); __KVM_X86_OP(func) +#define KVM_X86_OP_OPTIONAL __KVM_X86_OP +#define KVM_X86_OP_OPTIONAL_RET0(func) \ + static_call_update(kvm_x86_##func, (void *)kvm_x86_ops.func ? : \ + (void *)__static_call_return0); +#include +#undef __KVM_X86_OP +} + int kvm_arch_hardware_setup(void *opaque) { struct kvm_x86_init_ops *ops = opaque; @@ -11632,8 +11648,7 @@ int kvm_arch_hardware_setup(void *opaque) if (r != 0) return r; - memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); - kvm_ops_static_call_update(); + kvm_ops_update(ops); kvm_register_perf_callbacks(ops->handle_intel_pt_intr); From 8f969c0c3443183972ac7311d2bd656806082579 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Tue, 29 Mar 2022 23:50:52 +0000 Subject: [PATCH 087/319] KVM: x86: Copy kvm_pmu_ops by value to eliminate layer of indirection Replace the kvm_pmu_ops pointer in common x86 with an instance of the struct to save one pointer dereference when invoking functions. Copy the struct by value to set the ops during kvm_init(). Signed-off-by: Like Xu [sean: Move pmc_is_enabled(), make kvm_pmu_ops static] Signed-off-by: Sean Christopherson Message-Id: <20220329235054.3534728-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/pmu.c | 54 ++++++++++++++++++++++++++++------------------ arch/x86/kvm/pmu.h | 7 ++---- arch/x86/kvm/x86.c | 2 ++ 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index eca39f56c231..bb0b1ad0fda5 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -49,6 +49,18 @@ * * AMD: [0 .. AMD64_NUM_COUNTERS-1] <=> gp counters */ +static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; + +void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) +{ + memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); +} + +static inline bool pmc_is_enabled(struct kvm_pmc *pmc) +{ + return kvm_pmu_ops.pmc_is_enabled(pmc); +} + static void kvm_pmi_trigger_fn(struct irq_work *irq_work) { struct kvm_pmu *pmu = container_of(irq_work, struct kvm_pmu, irq_work); @@ -213,7 +225,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config = kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc); + config = kvm_pmu_ops.pmc_perf_hw_id(pmc); if (config != PERF_COUNT_HW_MAX) type = PERF_TYPE_HARDWARE; } @@ -263,7 +275,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) pmc->current_config = (u64)ctrl; pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, - kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc), + kvm_pmu_ops.pmc_perf_hw_id(pmc), !(en_field & 0x2), /* exclude user */ !(en_field & 0x1), /* exclude kernel */ pmi); @@ -272,7 +284,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter); void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx) { - struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, pmc_idx); + struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx); if (!pmc) return; @@ -294,7 +306,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) int bit; for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) { - struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, bit); + struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit); if (unlikely(!pmc || !pmc->perf_event)) { clear_bit(bit, pmu->reprogram_pmi); @@ -316,7 +328,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) /* check if idx is a valid index to access PMU */ bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { - return kvm_x86_ops.pmu_ops->is_valid_rdpmc_ecx(vcpu, idx); + return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx); } bool is_vmware_backdoor_pmc(u32 pmc_idx) @@ -366,7 +378,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) if (is_vmware_backdoor_pmc(idx)) return kvm_pmu_rdpmc_vmware(vcpu, idx, data); - pmc = kvm_x86_ops.pmu_ops->rdpmc_ecx_to_pmc(vcpu, idx, &mask); + pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask); if (!pmc) return 1; @@ -382,22 +394,22 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) { if (lapic_in_kernel(vcpu)) { - if (kvm_x86_ops.pmu_ops->deliver_pmi) - kvm_x86_ops.pmu_ops->deliver_pmi(vcpu); + if (kvm_pmu_ops.deliver_pmi) + kvm_pmu_ops.deliver_pmi(vcpu); kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); } } bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { - return kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr) || - kvm_x86_ops.pmu_ops->is_valid_msr(vcpu, msr); + return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) || + kvm_pmu_ops.is_valid_msr(vcpu, msr); } static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); - struct kvm_pmc *pmc = kvm_x86_ops.pmu_ops->msr_idx_to_pmc(vcpu, msr); + struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr); if (pmc) __set_bit(pmc->idx, pmu->pmc_in_use); @@ -405,13 +417,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { - return kvm_x86_ops.pmu_ops->get_msr(vcpu, msr_info); + return kvm_pmu_ops.get_msr(vcpu, msr_info); } int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); - return kvm_x86_ops.pmu_ops->set_msr(vcpu, msr_info); + return kvm_pmu_ops.set_msr(vcpu, msr_info); } /* refresh PMU settings. This function generally is called when underlying @@ -420,7 +432,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) */ void kvm_pmu_refresh(struct kvm_vcpu *vcpu) { - kvm_x86_ops.pmu_ops->refresh(vcpu); + kvm_pmu_ops.refresh(vcpu); } void kvm_pmu_reset(struct kvm_vcpu *vcpu) @@ -428,7 +440,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); irq_work_sync(&pmu->irq_work); - kvm_x86_ops.pmu_ops->reset(vcpu); + kvm_pmu_ops.reset(vcpu); } void kvm_pmu_init(struct kvm_vcpu *vcpu) @@ -436,7 +448,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); memset(pmu, 0, sizeof(*pmu)); - kvm_x86_ops.pmu_ops->init(vcpu); + kvm_pmu_ops.init(vcpu); init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count = 0; pmu->need_cleanup = false; @@ -468,14 +480,14 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX); for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) { - pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i); + pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i); if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc)) pmc_stop_counter(pmc); } - if (kvm_x86_ops.pmu_ops->cleanup) - kvm_x86_ops.pmu_ops->cleanup(vcpu); + if (kvm_pmu_ops.cleanup) + kvm_pmu_ops.cleanup(vcpu); bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); } @@ -505,7 +517,7 @@ static inline bool eventsel_match_perf_hw_id(struct kvm_pmc *pmc, unsigned int config; pmc->eventsel &= (ARCH_PERFMON_EVENTSEL_EVENT | ARCH_PERFMON_EVENTSEL_UMASK); - config = kvm_x86_ops.pmu_ops->pmc_perf_hw_id(pmc); + config = kvm_pmu_ops.pmc_perf_hw_id(pmc); pmc->eventsel = old_eventsel; return config == perf_hw_id; } @@ -533,7 +545,7 @@ void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id) int i; for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) { - pmc = kvm_x86_ops.pmu_ops->pmc_idx_to_pmc(pmu, i); + pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i); if (!pmc || !pmc_is_enabled(pmc) || !pmc_speculative_in_use(pmc)) continue; diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index 9e66fba1d6a3..2a53b6c9495c 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -39,6 +39,8 @@ struct kvm_pmu_ops { void (*cleanup)(struct kvm_vcpu *vcpu); }; +void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops); + static inline u64 pmc_bitmask(struct kvm_pmc *pmc) { struct kvm_pmu *pmu = pmc_to_pmu(pmc); @@ -86,11 +88,6 @@ static inline bool pmc_is_fixed(struct kvm_pmc *pmc) return pmc->type == KVM_PMC_FIXED; } -static inline bool pmc_is_enabled(struct kvm_pmc *pmc) -{ - return kvm_x86_ops.pmu_ops->pmc_is_enabled(pmc); -} - static inline bool kvm_valid_perf_global_ctrl(struct kvm_pmu *pmu, u64 data) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d22fc56d8bb8..65be0de9239c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11632,6 +11632,8 @@ static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) (void *)__static_call_return0); #include #undef __KVM_X86_OP + + kvm_pmu_ops_update(ops->runtime_ops->pmu_ops); } int kvm_arch_hardware_setup(void *opaque) From 34886e796c413273908b036df57cc9ae731badae Mon Sep 17 00:00:00 2001 From: Like Xu Date: Tue, 29 Mar 2022 23:50:53 +0000 Subject: [PATCH 088/319] KVM: x86: Move .pmu_ops to kvm_x86_init_ops and tag as __initdata The pmu_ops should be moved to kvm_x86_init_ops and tagged as __initdata. That'll save those precious few bytes, and more importantly make the original ops unreachable, i.e. make it harder to sneak in post-init modification bugs. Suggested-by: Sean Christopherson Signed-off-by: Like Xu Reviewed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20220329235054.3534728-4-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 +-- arch/x86/kvm/svm/pmu.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/pmu_intel.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/x86.c | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 25d1aac74c55..e1c695f72b8b 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1465,8 +1465,6 @@ struct kvm_x86_ops { int cpu_dirty_log_size; void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); - /* pmu operations of sub-arch */ - const struct kvm_pmu_ops *pmu_ops; const struct kvm_x86_nested_ops *nested_ops; void (*vcpu_blocking)(struct kvm_vcpu *vcpu); @@ -1542,6 +1540,7 @@ struct kvm_x86_init_ops { unsigned int (*handle_intel_pt_intr)(void); struct kvm_x86_ops *runtime_ops; + struct kvm_pmu_ops *pmu_ops; }; struct kvm_arch_async_pf { diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 24eb935b6f85..57ab4739eb19 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -319,7 +319,7 @@ static void amd_pmu_reset(struct kvm_vcpu *vcpu) } } -struct kvm_pmu_ops amd_pmu_ops = { +struct kvm_pmu_ops amd_pmu_ops __initdata = { .pmc_perf_hw_id = amd_pmc_perf_hw_id, .pmc_is_enabled = amd_pmc_is_enabled, .pmc_idx_to_pmc = amd_pmc_idx_to_pmc, diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 95b26dbfd561..22bbd69495ad 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4694,7 +4694,6 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .sched_in = svm_sched_in, - .pmu_ops = &amd_pmu_ops, .nested_ops = &svm_nested_ops, .deliver_interrupt = svm_deliver_interrupt, @@ -4988,6 +4987,7 @@ static struct kvm_x86_init_ops svm_init_ops __initdata = { .check_processor_compatibility = svm_check_processor_compat, .runtime_ops = &svm_x86_ops, + .pmu_ops = &amd_pmu_ops, }; static int __init svm_init(void) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index bc3f8512bb64..9db662399487 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -723,7 +723,7 @@ static void intel_pmu_cleanup(struct kvm_vcpu *vcpu) intel_pmu_release_guest_lbr_event(vcpu); } -struct kvm_pmu_ops intel_pmu_ops = { +struct kvm_pmu_ops intel_pmu_ops __initdata = { .pmc_perf_hw_id = intel_pmc_perf_hw_id, .pmc_is_enabled = intel_pmc_is_enabled, .pmc_idx_to_pmc = intel_pmc_idx_to_pmc, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index fb0b0fa6b949..df0b70ccd289 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7816,7 +7816,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .cpu_dirty_log_size = PML_ENTITY_NUM, .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging, - .pmu_ops = &intel_pmu_ops, .nested_ops = &vmx_nested_ops, .pi_update_irte = vmx_pi_update_irte, @@ -8070,6 +8069,7 @@ static struct kvm_x86_init_ops vmx_init_ops __initdata = { .handle_intel_pt_intr = NULL, .runtime_ops = &vmx_x86_ops, + .pmu_ops = &intel_pmu_ops, }; static void vmx_cleanup_l1d_flush(void) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 65be0de9239c..fb8153d584c8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11633,7 +11633,7 @@ static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) #include #undef __KVM_X86_OP - kvm_pmu_ops_update(ops->runtime_ops->pmu_ops); + kvm_pmu_ops_update(ops->pmu_ops); } int kvm_arch_hardware_setup(void *opaque) From 1921f3aa9263977f6c31b1c3eb2814bff2e84a12 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Tue, 29 Mar 2022 23:50:54 +0000 Subject: [PATCH 089/319] KVM: x86: Use static calls to reduce kvm_pmu_ops overhead Use static calls to improve kvm_pmu_ops performance, following the same pattern and naming scheme used by kvm-x86-ops.h. Here are the worst fenced_rdtsc() cycles numbers for the kvm_pmu_ops functions that is most often called (up to 7 digits of calls) when running a single perf test case in a guest on an ICX 2.70GHz host (mitigations=on): | legacy | static call ------------------------------------------------------------ .pmc_idx_to_pmc | 1304840 | 994872 (+23%) .pmc_is_enabled | 978670 | 1011750 (-3%) .msr_idx_to_pmc | 47828 | 41690 (+12%) .is_valid_msr | 28786 | 30108 (-4%) Signed-off-by: Like Xu [sean: Handle static call updates in pmu.c, tweak changelog] Signed-off-by: Sean Christopherson Message-Id: <20220329235054.3534728-5-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm-x86-pmu-ops.h | 31 ++++++++++++++ arch/x86/kvm/pmu.c | 56 ++++++++++++++++---------- 2 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 arch/x86/include/asm/kvm-x86-pmu-ops.h diff --git a/arch/x86/include/asm/kvm-x86-pmu-ops.h b/arch/x86/include/asm/kvm-x86-pmu-ops.h new file mode 100644 index 000000000000..fdfd8e06fee6 --- /dev/null +++ b/arch/x86/include/asm/kvm-x86-pmu-ops.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_X86_PMU_OP) || !defined(KVM_X86_PMU_OP_OPTIONAL) +BUILD_BUG_ON(1) +#endif + +/* + * KVM_X86_PMU_OP() and KVM_X86_PMU_OP_OPTIONAL() are used to help generate + * both DECLARE/DEFINE_STATIC_CALL() invocations and + * "static_call_update()" calls. + * + * KVM_X86_PMU_OP_OPTIONAL() can be used for those functions that can have + * a NULL definition, for example if "static_call_cond()" will be used + * at the call sites. + */ +KVM_X86_PMU_OP(pmc_perf_hw_id) +KVM_X86_PMU_OP(pmc_is_enabled) +KVM_X86_PMU_OP(pmc_idx_to_pmc) +KVM_X86_PMU_OP(rdpmc_ecx_to_pmc) +KVM_X86_PMU_OP(msr_idx_to_pmc) +KVM_X86_PMU_OP(is_valid_rdpmc_ecx) +KVM_X86_PMU_OP(is_valid_msr) +KVM_X86_PMU_OP(get_msr) +KVM_X86_PMU_OP(set_msr) +KVM_X86_PMU_OP(refresh) +KVM_X86_PMU_OP(init) +KVM_X86_PMU_OP(reset) +KVM_X86_PMU_OP_OPTIONAL(deliver_pmi) +KVM_X86_PMU_OP_OPTIONAL(cleanup) + +#undef KVM_X86_PMU_OP +#undef KVM_X86_PMU_OP_OPTIONAL diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index bb0b1ad0fda5..618f529f1c4d 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -51,14 +51,28 @@ static struct kvm_pmu_ops kvm_pmu_ops __read_mostly; +#define KVM_X86_PMU_OP(func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_pmu_##func, \ + *(((struct kvm_pmu_ops *)0)->func)); +#define KVM_X86_PMU_OP_OPTIONAL KVM_X86_PMU_OP +#include + void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops) { memcpy(&kvm_pmu_ops, pmu_ops, sizeof(kvm_pmu_ops)); + +#define __KVM_X86_PMU_OP(func) \ + static_call_update(kvm_x86_pmu_##func, kvm_pmu_ops.func); +#define KVM_X86_PMU_OP(func) \ + WARN_ON(!kvm_pmu_ops.func); __KVM_X86_PMU_OP(func) +#define KVM_X86_PMU_OP_OPTIONAL __KVM_X86_PMU_OP +#include +#undef __KVM_X86_PMU_OP } static inline bool pmc_is_enabled(struct kvm_pmc *pmc) { - return kvm_pmu_ops.pmc_is_enabled(pmc); + return static_call(kvm_x86_pmu_pmc_is_enabled)(pmc); } static void kvm_pmi_trigger_fn(struct irq_work *irq_work) @@ -225,7 +239,7 @@ void reprogram_gp_counter(struct kvm_pmc *pmc, u64 eventsel) ARCH_PERFMON_EVENTSEL_CMASK | HSW_IN_TX | HSW_IN_TX_CHECKPOINTED))) { - config = kvm_pmu_ops.pmc_perf_hw_id(pmc); + config = static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc); if (config != PERF_COUNT_HW_MAX) type = PERF_TYPE_HARDWARE; } @@ -275,7 +289,7 @@ void reprogram_fixed_counter(struct kvm_pmc *pmc, u8 ctrl, int idx) pmc->current_config = (u64)ctrl; pmc_reprogram_counter(pmc, PERF_TYPE_HARDWARE, - kvm_pmu_ops.pmc_perf_hw_id(pmc), + static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc), !(en_field & 0x2), /* exclude user */ !(en_field & 0x1), /* exclude kernel */ pmi); @@ -284,7 +298,7 @@ EXPORT_SYMBOL_GPL(reprogram_fixed_counter); void reprogram_counter(struct kvm_pmu *pmu, int pmc_idx) { - struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, pmc_idx); + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, pmc_idx); if (!pmc) return; @@ -306,7 +320,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) int bit; for_each_set_bit(bit, pmu->reprogram_pmi, X86_PMC_IDX_MAX) { - struct kvm_pmc *pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, bit); + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, bit); if (unlikely(!pmc || !pmc->perf_event)) { clear_bit(bit, pmu->reprogram_pmi); @@ -328,7 +342,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu) /* check if idx is a valid index to access PMU */ bool kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx) { - return kvm_pmu_ops.is_valid_rdpmc_ecx(vcpu, idx); + return static_call(kvm_x86_pmu_is_valid_rdpmc_ecx)(vcpu, idx); } bool is_vmware_backdoor_pmc(u32 pmc_idx) @@ -378,7 +392,7 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) if (is_vmware_backdoor_pmc(idx)) return kvm_pmu_rdpmc_vmware(vcpu, idx, data); - pmc = kvm_pmu_ops.rdpmc_ecx_to_pmc(vcpu, idx, &mask); + pmc = static_call(kvm_x86_pmu_rdpmc_ecx_to_pmc)(vcpu, idx, &mask); if (!pmc) return 1; @@ -394,22 +408,21 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) { if (lapic_in_kernel(vcpu)) { - if (kvm_pmu_ops.deliver_pmi) - kvm_pmu_ops.deliver_pmi(vcpu); + static_call_cond(kvm_x86_pmu_deliver_pmi)(vcpu); kvm_apic_local_deliver(vcpu->arch.apic, APIC_LVTPC); } } bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { - return kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr) || - kvm_pmu_ops.is_valid_msr(vcpu, msr); + return static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr) || + static_call(kvm_x86_pmu_is_valid_msr)(vcpu, msr); } static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); - struct kvm_pmc *pmc = kvm_pmu_ops.msr_idx_to_pmc(vcpu, msr); + struct kvm_pmc *pmc = static_call(kvm_x86_pmu_msr_idx_to_pmc)(vcpu, msr); if (pmc) __set_bit(pmc->idx, pmu->pmc_in_use); @@ -417,13 +430,13 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { - return kvm_pmu_ops.get_msr(vcpu, msr_info); + return static_call(kvm_x86_pmu_get_msr)(vcpu, msr_info); } int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { kvm_pmu_mark_pmc_in_use(vcpu, msr_info->index); - return kvm_pmu_ops.set_msr(vcpu, msr_info); + return static_call(kvm_x86_pmu_set_msr)(vcpu, msr_info); } /* refresh PMU settings. This function generally is called when underlying @@ -432,7 +445,7 @@ int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) */ void kvm_pmu_refresh(struct kvm_vcpu *vcpu) { - kvm_pmu_ops.refresh(vcpu); + static_call(kvm_x86_pmu_refresh)(vcpu); } void kvm_pmu_reset(struct kvm_vcpu *vcpu) @@ -440,7 +453,7 @@ void kvm_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); irq_work_sync(&pmu->irq_work); - kvm_pmu_ops.reset(vcpu); + static_call(kvm_x86_pmu_reset)(vcpu); } void kvm_pmu_init(struct kvm_vcpu *vcpu) @@ -448,7 +461,7 @@ void kvm_pmu_init(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); memset(pmu, 0, sizeof(*pmu)); - kvm_pmu_ops.init(vcpu); + static_call(kvm_x86_pmu_init)(vcpu); init_irq_work(&pmu->irq_work, kvm_pmi_trigger_fn); pmu->event_count = 0; pmu->need_cleanup = false; @@ -480,14 +493,13 @@ void kvm_pmu_cleanup(struct kvm_vcpu *vcpu) pmu->pmc_in_use, X86_PMC_IDX_MAX); for_each_set_bit(i, bitmask, X86_PMC_IDX_MAX) { - pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i); + pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i); if (pmc && pmc->perf_event && !pmc_speculative_in_use(pmc)) pmc_stop_counter(pmc); } - if (kvm_pmu_ops.cleanup) - kvm_pmu_ops.cleanup(vcpu); + static_call_cond(kvm_x86_pmu_cleanup)(vcpu); bitmap_zero(pmu->pmc_in_use, X86_PMC_IDX_MAX); } @@ -517,7 +529,7 @@ static inline bool eventsel_match_perf_hw_id(struct kvm_pmc *pmc, unsigned int config; pmc->eventsel &= (ARCH_PERFMON_EVENTSEL_EVENT | ARCH_PERFMON_EVENTSEL_UMASK); - config = kvm_pmu_ops.pmc_perf_hw_id(pmc); + config = static_call(kvm_x86_pmu_pmc_perf_hw_id)(pmc); pmc->eventsel = old_eventsel; return config == perf_hw_id; } @@ -545,7 +557,7 @@ void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 perf_hw_id) int i; for_each_set_bit(i, pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX) { - pmc = kvm_pmu_ops.pmc_idx_to_pmc(pmu, i); + pmc = static_call(kvm_x86_pmu_pmc_idx_to_pmc)(pmu, i); if (!pmc || !pmc_is_enabled(pmc) || !pmc_speculative_in_use(pmc)) continue; From 45846661d10422ce9e22da21f8277540b29eca22 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 7 Apr 2022 00:23:13 +0000 Subject: [PATCH 090/319] KVM: x86: Drop WARNs that assert a triple fault never "escapes" from L2 Remove WARNs that sanity check that KVM never lets a triple fault for L2 escape and incorrectly end up in L1. In normal operation, the sanity check is perfectly valid, but it incorrectly assumes that it's impossible for userspace to induce KVM_REQ_TRIPLE_FAULT without bouncing through KVM_RUN (which guarantees kvm_check_nested_state() will see and handle the triple fault). The WARN can currently be triggered if userspace injects a machine check while L2 is active and CR4.MCE=0. And a future fix to allow save/restore of KVM_REQ_TRIPLE_FAULT, e.g. so that a synthesized triple fault isn't lost on migration, will make it trivially easy for userspace to trigger the WARN. Clearing KVM_REQ_TRIPLE_FAULT when forcibly leaving guest mode is tempting, but wrong, especially if/when the request is saved/restored, e.g. if userspace restores events (including a triple fault) and then restores nested state (which may forcibly leave guest mode). Ignoring the fact that KVM doesn't currently provide the necessary APIs, it's userspace's responsibility to manage pending events during save/restore. ------------[ cut here ]------------ WARNING: CPU: 7 PID: 1399 at arch/x86/kvm/vmx/nested.c:4522 nested_vmx_vmexit+0x7fe/0xd90 [kvm_intel] Modules linked in: kvm_intel kvm irqbypass CPU: 7 PID: 1399 Comm: state_test Not tainted 5.17.0-rc3+ #808 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:nested_vmx_vmexit+0x7fe/0xd90 [kvm_intel] Call Trace: vmx_leave_nested+0x30/0x40 [kvm_intel] vmx_set_nested_state+0xca/0x3e0 [kvm_intel] kvm_arch_vcpu_ioctl+0xf49/0x13e0 [kvm] kvm_vcpu_ioctl+0x4b9/0x660 [kvm] __x64_sys_ioctl+0x83/0xb0 do_syscall_64+0x3b/0xc0 entry_SYSCALL_64_after_hwframe+0x44/0xae ---[ end trace 0000000000000000 ]--- Fixes: cb6a32c2b877 ("KVM: x86: Handle triple fault in L2 without killing L1") Cc: stable@vger.kernel.org Cc: Chenyi Qiang Signed-off-by: Sean Christopherson Message-Id: <20220407002315.78092-2-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 3 --- arch/x86/kvm/vmx/nested.c | 3 --- 2 files changed, 6 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 73b545278f5f..caa691229b71 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -895,9 +895,6 @@ int nested_svm_vmexit(struct vcpu_svm *svm) struct kvm_host_map map; int rc; - /* Triple faults in L2 should never escape. */ - WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); - rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); if (rc) { if (rc == -EINVAL) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index f18744f7ff82..8aae175d9b33 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4518,9 +4518,6 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, /* trying to cancel vmlaunch/vmresume is a bug */ WARN_ON_ONCE(vmx->nested.nested_run_pending); - /* Similarly, triple faults in L2 should never escape. */ - WARN_ON_ONCE(kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)); - if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { /* * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map From c3634d25fbee88e2368a8e0903ae0d0670eb9e71 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 7 Apr 2022 00:23:14 +0000 Subject: [PATCH 091/319] KVM: nVMX: Leave most VM-Exit info fields unmodified on failed VM-Entry Don't modify vmcs12 exit fields except EXIT_REASON and EXIT_QUALIFICATION when performing a nested VM-Exit due to failed VM-Entry. Per the SDM, only the two aformentioned fields are filled and "All other VM-exit information fields are unmodified". Fixes: 4704d0befb07 ("KVM: nVMX: Exiting from L2 to L1") Signed-off-by: Sean Christopherson Message-Id: <20220407002315.78092-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 8aae175d9b33..d1d867528698 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4202,12 +4202,12 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, if (to_vmx(vcpu)->exit_reason.enclave_mode) vmcs12->vm_exit_reason |= VMX_EXIT_REASONS_SGX_ENCLAVE_MODE; vmcs12->exit_qualification = exit_qualification; - vmcs12->vm_exit_intr_info = exit_intr_info; - - vmcs12->idt_vectoring_info_field = 0; - vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); - vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); + /* + * On VM-Exit due to a failed VM-Entry, the VMCS isn't marked launched + * and only EXIT_REASON and EXIT_QUALIFICATION are updated, all other + * exit info fields are unmodified. + */ if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) { vmcs12->launch_state = 1; @@ -4219,8 +4219,13 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, * Transfer the event that L0 or L1 may wanted to inject into * L2 to IDT_VECTORING_INFO_FIELD. */ + vmcs12->idt_vectoring_info_field = 0; vmcs12_save_pending_event(vcpu, vmcs12); + vmcs12->vm_exit_intr_info = exit_intr_info; + vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); + vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO); + /* * According to spec, there's no need to store the guest's * MSRs if the exit is due to a VM-entry failure that occurs From 9bd1f0efa859b61950d109b32ff8d529cc33a3ad Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 7 Apr 2022 00:23:15 +0000 Subject: [PATCH 092/319] KVM: nVMX: Clear IDT vectoring on nested VM-Exit for double/triple fault Clear the IDT vectoring field in vmcs12 on next VM-Exit due to a double or triple fault. Per the SDM, a VM-Exit isn't considered to occur during event delivery if the exit is due to an intercepted double fault or a triple fault. Opportunistically move the default clearing (no event "pending") into the helper so that it's more obvious that KVM does indeed handle this case. Note, the double fault case is worded rather wierdly in the SDM: The original event results in a double-fault exception that causes the VM exit directly. Temporarily ignoring injected events, double faults can _only_ occur if an exception occurs while attempting to deliver a different exception, i.e. there's _always_ an original event. And for injected double fault, while there's no original event, injected events are never subject to interception. Presumably the SDM is calling out that a the vectoring info will be valid if a different exit occurs after a double fault, e.g. if a #PF occurs and is intercepted while vectoring #DF, then the vectoring info will show the double fault. In other words, the clause can simply be read as: The VM exit is caused by a double-fault exception. Fixes: 4704d0befb07 ("KVM: nVMX: Exiting from L2 to L1") Cc: Chenyi Qiang Signed-off-by: Sean Christopherson Message-Id: <20220407002315.78092-4-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 32 ++++++++++++++++++++++++++++---- arch/x86/kvm/vmx/vmcs.h | 5 +++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index d1d867528698..838ac7ab5950 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3695,12 +3695,34 @@ vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) } static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, - struct vmcs12 *vmcs12) + struct vmcs12 *vmcs12, + u32 vm_exit_reason, u32 exit_intr_info) { u32 idt_vectoring; unsigned int nr; - if (vcpu->arch.exception.injected) { + /* + * Per the SDM, VM-Exits due to double and triple faults are never + * considered to occur during event delivery, even if the double/triple + * fault is the result of an escalating vectoring issue. + * + * Note, the SDM qualifies the double fault behavior with "The original + * event results in a double-fault exception". It's unclear why the + * qualification exists since exits due to double fault can occur only + * while vectoring a different exception (injected events are never + * subject to interception), i.e. there's _always_ an original event. + * + * The SDM also uses NMI as a confusing example for the "original event + * causes the VM exit directly" clause. NMI isn't special in any way, + * the same rule applies to all events that cause an exit directly. + * NMI is an odd choice for the example because NMIs can only occur on + * instruction boundaries, i.e. they _can't_ occur during vectoring. + */ + if ((u16)vm_exit_reason == EXIT_REASON_TRIPLE_FAULT || + ((u16)vm_exit_reason == EXIT_REASON_EXCEPTION_NMI && + is_double_fault(exit_intr_info))) { + vmcs12->idt_vectoring_info_field = 0; + } else if (vcpu->arch.exception.injected) { nr = vcpu->arch.exception.nr; idt_vectoring = nr | VECTORING_INFO_VALID_MASK; @@ -3733,6 +3755,8 @@ static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu, idt_vectoring |= INTR_TYPE_EXT_INTR; vmcs12->idt_vectoring_info_field = idt_vectoring; + } else { + vmcs12->idt_vectoring_info_field = 0; } } @@ -4219,8 +4243,8 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, * Transfer the event that L0 or L1 may wanted to inject into * L2 to IDT_VECTORING_INFO_FIELD. */ - vmcs12->idt_vectoring_info_field = 0; - vmcs12_save_pending_event(vcpu, vmcs12); + vmcs12_save_pending_event(vcpu, vmcs12, + vm_exit_reason, exit_intr_info); vmcs12->vm_exit_intr_info = exit_intr_info; vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN); diff --git a/arch/x86/kvm/vmx/vmcs.h b/arch/x86/kvm/vmx/vmcs.h index e325c290a816..2b9d7a7e83f7 100644 --- a/arch/x86/kvm/vmx/vmcs.h +++ b/arch/x86/kvm/vmx/vmcs.h @@ -104,6 +104,11 @@ static inline bool is_breakpoint(u32 intr_info) return is_exception_n(intr_info, BP_VECTOR); } +static inline bool is_double_fault(u32 intr_info) +{ + return is_exception_n(intr_info, DF_VECTOR); +} + static inline bool is_page_fault(u32 intr_info) { return is_exception_n(intr_info, PF_VECTOR); From c24a950ec7d60c4da91dc3f273295c7f438b531e Mon Sep 17 00:00:00 2001 From: Peter Gonda Date: Thu, 7 Apr 2022 14:02:33 -0700 Subject: [PATCH 093/319] KVM, SEV: Add KVM_EXIT_SHUTDOWN metadata for SEV-ES If an SEV-ES guest requests termination, exit to userspace with KVM_EXIT_SYSTEM_EVENT and a dedicated SEV_TERM type instead of -EINVAL so that userspace can take appropriate action. See AMD's GHCB spec section '4.1.13 Termination Request' for more details. Suggested-by: Sean Christopherson Suggested-by: Paolo Bonzini Cc: kvm@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Peter Gonda Reported-by: kernel test robot Message-Id: <20220407210233.782250-1-pgonda@google.com> [Add documentatino. - Paolo] Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 12 +++++++++++- arch/x86/kvm/svm/sev.c | 9 +++++++-- include/uapi/linux/kvm.h | 5 ++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index e7a0dfdc0178..72183ae628f7 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6088,8 +6088,12 @@ should put the acknowledged interrupt vector into the 'epr' field. #define KVM_SYSTEM_EVENT_SHUTDOWN 1 #define KVM_SYSTEM_EVENT_RESET 2 #define KVM_SYSTEM_EVENT_CRASH 3 + #define KVM_SYSTEM_EVENT_SEV_TERM 4 + #define KVM_SYSTEM_EVENT_NDATA_VALID (1u << 31) __u32 type; + __u32 ndata; __u64 flags; + __u64 data[16]; } system_event; If exit_reason is KVM_EXIT_SYSTEM_EVENT then the vcpu has triggered @@ -6099,7 +6103,7 @@ HVC instruction based PSCI call from the vcpu. The 'type' field describes the system-level event type. The 'flags' field describes architecture specific flags for the system-level event. -Valid values for 'type' are: +Valid values for bits 30:0 of 'type' are: - KVM_SYSTEM_EVENT_SHUTDOWN -- the guest has requested a shutdown of the VM. Userspace is not obliged to honour this, and if it does honour @@ -6112,12 +6116,18 @@ Valid values for 'type' are: has requested a crash condition maintenance. Userspace can choose to ignore the request, or to gather VM memory core dump and/or reset/shutdown of the VM. + - KVM_SYSTEM_EVENT_SEV_TERM -- an AMD SEV guest requested termination. + The guest physical address of the guest's GHCB is stored in `data[0]`. Valid flags are: - KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2 (arm64 only) -- the guest issued a SYSTEM_RESET2 call according to v1.1 of the PSCI specification. +Extra data for this event is stored in the `data[]` array, up to index +`ndata-1` included, if bit 31 is set in `type`. The data depends on the +`type` field. There is no extra data if bit 31 is clear or `ndata` is zero. + :: /* KVM_EXIT_IOAPIC_EOI */ diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 537aaddc852f..a93f0d01bb90 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2738,8 +2738,13 @@ static int sev_handle_vmgexit_msr_protocol(struct vcpu_svm *svm) pr_info("SEV-ES guest requested termination: %#llx:%#llx\n", reason_set, reason_code); - ret = -EINVAL; - break; + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_SEV_TERM | + KVM_SYSTEM_EVENT_NDATA_VALID; + vcpu->run->system_event.ndata = 1; + vcpu->run->system_event.data[1] = control->ghcb_gpa; + + return 0; } default: /* Error, keep GHCB MSR value as-is */ diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 8616af85dc5d..dd1d8167e71f 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -444,8 +444,11 @@ struct kvm_run { #define KVM_SYSTEM_EVENT_SHUTDOWN 1 #define KVM_SYSTEM_EVENT_RESET 2 #define KVM_SYSTEM_EVENT_CRASH 3 +#define KVM_SYSTEM_EVENT_SEV_TERM 4 +#define KVM_SYSTEM_EVENT_NDATA_VALID (1u << 31) __u32 type; - __u64 flags; + __u32 ndata; + __u64 data[16]; } system_event; /* KVM_EXIT_S390_STSI */ struct { From 1aa0e8b144b6474c4914439d232d15bfe883636b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Feb 2022 00:49:41 +0000 Subject: [PATCH 094/319] Kconfig: Add option for asm goto w/ tied outputs to workaround clang-13 bug Add a config option to guard (future) usage of asm_volatile_goto() that includes "tied outputs", i.e. "+" constraints that specify both an input and output parameter. clang-13 has a bug[1] that causes compilation of such inline asm to fail, and KVM wants to use a "+m" constraint to implement a uaccess form of CMPXCHG[2]. E.g. the test code fails with :1:29: error: invalid operand in inline asm: '.long (${1:l}) - .' int foo(int *x) { asm goto (".long (%l[bar]) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; } ^ :1:29: error: unknown token in expression :1:9: note: instantiated into assembly here .long () - . ^ 2 errors generated. on clang-13, but passes on gcc (with appropriate asm goto support). The bug is fixed in clang-14, but won't be backported to clang-13 as the changes are too invasive/risky. gcc also had a similar bug[3], fixed in gcc-11, where gcc failed to account for its behavior of assigning two numbers to tied outputs (one for input, one for output) when evaluating symbolic references. [1] https://github.com/ClangBuiltLinux/linux/issues/1512 [2] https://lore.kernel.org/all/YfMruK8%2F1izZ2VHS@google.com [3] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98096 Suggested-by: Nick Desaulniers Reviewed-by: Nick Desaulniers Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Message-Id: <20220202004945.2540433-2-seanjc@google.com> Signed-off-by: Paolo Bonzini --- init/Kconfig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/init/Kconfig b/init/Kconfig index ddcbefe535e9..b19e2eeaae80 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -77,6 +77,11 @@ config CC_HAS_ASM_GOTO_OUTPUT depends on CC_HAS_ASM_GOTO def_bool $(success,echo 'int foo(int x) { asm goto ("": "=r"(x) ::: bar); return x; bar: return 0; }' | $(CC) -x c - -c -o /dev/null) +config CC_HAS_ASM_GOTO_TIED_OUTPUT + depends on CC_HAS_ASM_GOTO_OUTPUT + # Detect buggy gcc and clang, fixed in gcc-11 clang-14. + def_bool $(success,echo 'int foo(int *x) { asm goto (".long (%l[bar]) - .\n": "+m"(*x) ::: bar); return *x; bar: return 0; }' | $CC -x c - -c -o /dev/null) + config TOOLS_SUPPORT_RELR def_bool $(success,env "CC=$(CC)" "LD=$(LD)" "NM=$(NM)" "OBJCOPY=$(OBJCOPY)" $(srctree)/scripts/tools-support-relr.sh) From 989b5db215a2f22f89d730b607b071d964780f10 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 2 Feb 2022 00:49:42 +0000 Subject: [PATCH 095/319] x86/uaccess: Implement macros for CMPXCHG on user addresses Add support for CMPXCHG loops on userspace addresses. Provide both an "unsafe" version for tight loops that do their own uaccess begin/end, as well as a "safe" version for use cases where the CMPXCHG is not buried in a loop, e.g. KVM will resume the guest instead of looping when emulation of a guest atomic accesses fails the CMPXCHG. Provide 8-byte versions for 32-bit kernels so that KVM can do CMPXCHG on guest PAE PTEs, which are accessed via userspace addresses. Guard the asm_volatile_goto() variation with CC_HAS_ASM_GOTO_TIED_OUTPUT, the "+m" constraint fails on some compilers that otherwise support CC_HAS_ASM_GOTO_OUTPUT. Cc: stable@vger.kernel.org Signed-off-by: Peter Zijlstra (Intel) Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20220202004945.2540433-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/uaccess.h | 142 +++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index f78e2b3501a1..35f222aa66bf 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -382,6 +382,103 @@ do { \ #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT +#ifdef CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT +#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \ + bool success; \ + __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ + __typeof__(*(_ptr)) __old = *_old; \ + __typeof__(*(_ptr)) __new = (_new); \ + asm_volatile_goto("\n" \ + "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\ + _ASM_EXTABLE_UA(1b, %l[label]) \ + : CC_OUT(z) (success), \ + [ptr] "+m" (*_ptr), \ + [old] "+a" (__old) \ + : [new] ltype (__new) \ + : "memory" \ + : label); \ + if (unlikely(!success)) \ + *_old = __old; \ + likely(success); }) + +#ifdef CONFIG_X86_32 +#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \ + bool success; \ + __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ + __typeof__(*(_ptr)) __old = *_old; \ + __typeof__(*(_ptr)) __new = (_new); \ + asm_volatile_goto("\n" \ + "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \ + _ASM_EXTABLE_UA(1b, %l[label]) \ + : CC_OUT(z) (success), \ + "+A" (__old), \ + [ptr] "+m" (*_ptr) \ + : "b" ((u32)__new), \ + "c" ((u32)((u64)__new >> 32)) \ + : "memory" \ + : label); \ + if (unlikely(!success)) \ + *_old = __old; \ + likely(success); }) +#endif // CONFIG_X86_32 +#else // !CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT +#define __try_cmpxchg_user_asm(itype, ltype, _ptr, _pold, _new, label) ({ \ + int __err = 0; \ + bool success; \ + __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ + __typeof__(*(_ptr)) __old = *_old; \ + __typeof__(*(_ptr)) __new = (_new); \ + asm volatile("\n" \ + "1: " LOCK_PREFIX "cmpxchg"itype" %[new], %[ptr]\n"\ + CC_SET(z) \ + "2:\n" \ + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, \ + %[errout]) \ + : CC_OUT(z) (success), \ + [errout] "+r" (__err), \ + [ptr] "+m" (*_ptr), \ + [old] "+a" (__old) \ + : [new] ltype (__new) \ + : "memory", "cc"); \ + if (unlikely(__err)) \ + goto label; \ + if (unlikely(!success)) \ + *_old = __old; \ + likely(success); }) + +#ifdef CONFIG_X86_32 +/* + * Unlike the normal CMPXCHG, hardcode ECX for both success/fail and error. + * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are + * hardcoded by CMPXCHG8B, leaving only ESI and EDI. If the compiler uses + * both ESI and EDI for the memory operand, compilation will fail if the error + * is an input+output as there will be no register available for input. + */ +#define __try_cmpxchg64_user_asm(_ptr, _pold, _new, label) ({ \ + int __result; \ + __typeof__(_ptr) _old = (__typeof__(_ptr))(_pold); \ + __typeof__(*(_ptr)) __old = *_old; \ + __typeof__(*(_ptr)) __new = (_new); \ + asm volatile("\n" \ + "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n" \ + "mov $0, %%ecx\n\t" \ + "setz %%cl\n" \ + "2:\n" \ + _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %%ecx) \ + : [result]"=c" (__result), \ + "+A" (__old), \ + [ptr] "+m" (*_ptr) \ + : "b" ((u32)__new), \ + "c" ((u32)((u64)__new >> 32)) \ + : "memory", "cc"); \ + if (unlikely(__result < 0)) \ + goto label; \ + if (unlikely(!__result)) \ + *_old = __old; \ + likely(__result); }) +#endif // CONFIG_X86_32 +#endif // CONFIG_CC_HAS_ASM_GOTO_TIED_OUTPUT + /* FIXME: this hack is definitely wrong -AK */ struct __large_struct { unsigned long buf[100]; }; #define __m(x) (*(struct __large_struct __user *)(x)) @@ -474,6 +571,51 @@ do { \ } while (0) #endif // CONFIG_CC_HAS_ASM_GOTO_OUTPUT +extern void __try_cmpxchg_user_wrong_size(void); + +#ifndef CONFIG_X86_32 +#define __try_cmpxchg64_user_asm(_ptr, _oldp, _nval, _label) \ + __try_cmpxchg_user_asm("q", "r", (_ptr), (_oldp), (_nval), _label) +#endif + +/* + * Force the pointer to u to match the size expected by the asm helper. + * clang/LLVM compiles all cases and only discards the unused paths after + * processing errors, which breaks i386 if the pointer is an 8-byte value. + */ +#define unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \ + bool __ret; \ + __chk_user_ptr(_ptr); \ + switch (sizeof(*(_ptr))) { \ + case 1: __ret = __try_cmpxchg_user_asm("b", "q", \ + (__force u8 *)(_ptr), (_oldp), \ + (_nval), _label); \ + break; \ + case 2: __ret = __try_cmpxchg_user_asm("w", "r", \ + (__force u16 *)(_ptr), (_oldp), \ + (_nval), _label); \ + break; \ + case 4: __ret = __try_cmpxchg_user_asm("l", "r", \ + (__force u32 *)(_ptr), (_oldp), \ + (_nval), _label); \ + break; \ + case 8: __ret = __try_cmpxchg64_user_asm((__force u64 *)(_ptr), (_oldp),\ + (_nval), _label); \ + break; \ + default: __try_cmpxchg_user_wrong_size(); \ + } \ + __ret; }) + +/* "Returns" 0 on success, 1 on failure, -EFAULT if the access faults. */ +#define __try_cmpxchg_user(_ptr, _oldp, _nval, _label) ({ \ + int __ret = -EFAULT; \ + __uaccess_begin_nospec(); \ + __ret = !unsafe_try_cmpxchg_user(_ptr, _oldp, _nval, _label); \ +_label: \ + __uaccess_end(); \ + __ret; \ + }) + /* * We want the unsafe accessors to always be inlined and use * the error labels - thus the macro games. From f122dfe4476890d60b8c679128cd2259ec96a24c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Feb 2022 00:49:43 +0000 Subject: [PATCH 096/319] KVM: x86: Use __try_cmpxchg_user() to update guest PTE A/D bits Use the recently introduced __try_cmpxchg_user() to update guest PTE A/D bits instead of mapping the PTE into kernel address space. The VM_PFNMAP path is broken as it assumes that vm_pgoff is the base pfn of the mapped VMA range, which is conceptually wrong as vm_pgoff is the offset relative to the file and has nothing to do with the pfn. The horrific hack worked for the original use case (backing guest memory with /dev/mem), but leads to accessing "random" pfns for pretty much any other VM_PFNMAP case. Fixes: bd53cb35a3e9 ("X86/KVM: Handle PFNs outside of kernel reach when touching GPTEs") Debugged-by: Tadeusz Struk Tested-by: Tadeusz Struk Reported-by: syzbot+6cde2282daa792c49ab8@syzkaller.appspotmail.com Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Message-Id: <20220202004945.2540433-4-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/paging_tmpl.h | 38 +--------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 1dad8f3f2bb0..7d4377f1ef2a 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -144,42 +144,6 @@ static bool FNAME(is_rsvd_bits_set)(struct kvm_mmu *mmu, u64 gpte, int level) FNAME(is_bad_mt_xwr)(&mmu->guest_rsvd_check, gpte); } -static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - pt_element_t __user *ptep_user, unsigned index, - pt_element_t orig_pte, pt_element_t new_pte) -{ - signed char r; - - if (!user_access_begin(ptep_user, sizeof(pt_element_t))) - return -EFAULT; - -#ifdef CMPXCHG - asm volatile("1:" LOCK_PREFIX CMPXCHG " %[new], %[ptr]\n" - "setnz %b[r]\n" - "2:" - _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %k[r]) - : [ptr] "+m" (*ptep_user), - [old] "+a" (orig_pte), - [r] "=q" (r) - : [new] "r" (new_pte) - : "memory"); -#else - asm volatile("1:" LOCK_PREFIX "cmpxchg8b %[ptr]\n" - "setnz %b[r]\n" - "2:" - _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %k[r]) - : [ptr] "+m" (*ptep_user), - [old] "+A" (orig_pte), - [r] "=q" (r) - : [new_lo] "b" ((u32)new_pte), - [new_hi] "c" ((u32)(new_pte >> 32)) - : "memory"); -#endif - - user_access_end(); - return r; -} - static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *spte, u64 gpte) @@ -278,7 +242,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu, if (unlikely(!walker->pte_writable[level - 1])) continue; - ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, orig_pte, pte); + ret = __try_cmpxchg_user(ptep_user, &orig_pte, pte, fault); if (ret) return ret; From 1c2361f667f3648855ceae25f1332c18413fdb9f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Feb 2022 00:49:44 +0000 Subject: [PATCH 097/319] KVM: x86: Use __try_cmpxchg_user() to emulate atomic accesses Use the recently introduce __try_cmpxchg_user() to emulate atomic guest accesses via the associated userspace address instead of mapping the backing pfn into kernel address space. Using kvm_vcpu_map() is unsafe as it does not coordinate with KVM's mmu_notifier to ensure the hva=>pfn translation isn't changed/unmapped in the memremap() path, i.e. when there's no struct page and thus no elevated refcount. Fixes: 42e35f8072c3 ("KVM/X86: Use kvm_vcpu_map in emulator_cmpxchg_emulated") Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Message-Id: <20220202004945.2540433-5-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index fb8153d584c8..e5b0dd24b200 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7257,15 +7257,8 @@ static int emulator_write_emulated(struct x86_emulate_ctxt *ctxt, exception, &write_emultor); } -#define CMPXCHG_TYPE(t, ptr, old, new) \ - (cmpxchg((t *)(ptr), *(t *)(old), *(t *)(new)) == *(t *)(old)) - -#ifdef CONFIG_X86_64 -# define CMPXCHG64(ptr, old, new) CMPXCHG_TYPE(u64, ptr, old, new) -#else -# define CMPXCHG64(ptr, old, new) \ - (cmpxchg64((u64 *)(ptr), *(u64 *)(old), *(u64 *)(new)) == *(u64 *)(old)) -#endif +#define emulator_try_cmpxchg_user(t, ptr, old, new) \ + (__try_cmpxchg_user((t __user *)(ptr), (t *)(old), *(t *)(new), efault ## t)) static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, unsigned long addr, @@ -7274,12 +7267,11 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, unsigned int bytes, struct x86_exception *exception) { - struct kvm_host_map map; struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); u64 page_line_mask; + unsigned long hva; gpa_t gpa; - char *kaddr; - bool exchanged; + int r; /* guests cmpxchg8b have to be emulated atomically */ if (bytes > 8 || (bytes & (bytes - 1))) @@ -7303,31 +7295,32 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask)) goto emul_write; - if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map)) + hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa)); + if (kvm_is_error_hva(addr)) goto emul_write; - kaddr = map.hva + offset_in_page(gpa); + hva += offset_in_page(gpa); switch (bytes) { case 1: - exchanged = CMPXCHG_TYPE(u8, kaddr, old, new); + r = emulator_try_cmpxchg_user(u8, hva, old, new); break; case 2: - exchanged = CMPXCHG_TYPE(u16, kaddr, old, new); + r = emulator_try_cmpxchg_user(u16, hva, old, new); break; case 4: - exchanged = CMPXCHG_TYPE(u32, kaddr, old, new); + r = emulator_try_cmpxchg_user(u32, hva, old, new); break; case 8: - exchanged = CMPXCHG64(kaddr, old, new); + r = emulator_try_cmpxchg_user(u64, hva, old, new); break; default: BUG(); } - kvm_vcpu_unmap(vcpu, &map, true); - - if (!exchanged) + if (r < 0) + goto emul_write; + if (r) return X86EMUL_CMPXCHG_FAILED; kvm_page_track_write(vcpu, gpa, new, bytes); From 5d6c7de6446e9ab3fb41d6f7d82770e50998f3de Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 2 Feb 2022 00:49:45 +0000 Subject: [PATCH 098/319] KVM: x86: Bail to userspace if emulation of atomic user access faults Exit to userspace when emulating an atomic guest access if the CMPXCHG on the userspace address faults. Emulating the access as a write and thus likely treating it as emulated MMIO is wrong, as KVM has already confirmed there is a valid, writable memslot. Signed-off-by: Sean Christopherson Message-Id: <20220202004945.2540433-6-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e5b0dd24b200..4e7f3a8da16a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7319,7 +7319,7 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, } if (r < 0) - goto emul_write; + return X86EMUL_UNHANDLEABLE; if (r) return X86EMUL_CMPXCHG_FAILED; From 954445c72fc75823b23c7046a9790e55dcf4f7f1 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 11 Apr 2022 14:49:05 +0300 Subject: [PATCH 099/319] pinctrl: samsung: Drop redundant node parameter in samsung_banks_of_node_get() The node is taken from the device pointer, which is supplied as a parameter, hence no need to have a separate parameter for node. Drop redundant node parameter in samsung_banks_of_node_get(). While at it, drop "of_" part in the samsung_banks_of_node_get() and samsung_banks_of_node_put() function names to avoid additional churn in the next changes. Signed-off-by: Andy Shevchenko Tested-by: Krzysztof Kozlowski --- drivers/pinctrl/samsung/pinctrl-samsung.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c index f610beab23a0..2ba8b43c25bd 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.c +++ b/drivers/pinctrl/samsung/pinctrl-samsung.c @@ -1002,7 +1002,7 @@ samsung_pinctrl_get_soc_data_for_of_alias(struct platform_device *pdev) return &(of_data->ctrl[id]); } -static void samsung_banks_of_node_put(struct samsung_pinctrl_drv_data *d) +static void samsung_banks_node_put(struct samsung_pinctrl_drv_data *d) { struct samsung_pin_bank *bank; unsigned int i; @@ -1016,12 +1016,11 @@ static void samsung_banks_of_node_put(struct samsung_pinctrl_drv_data *d) * Iterate over all driver pin banks to find one matching the name of node, * skipping optional "-gpio" node suffix. When found, assign node to the bank. */ -static void samsung_banks_of_node_get(struct device *dev, - struct samsung_pinctrl_drv_data *d, - struct device_node *node) +static void samsung_banks_node_get(struct device *dev, struct samsung_pinctrl_drv_data *d) { const char *suffix = "-gpio-bank"; struct samsung_pin_bank *bank; + struct device_node *node = dev->of_node; struct device_node *child; /* Pin bank names are up to 4 characters */ char node_name[20]; @@ -1061,7 +1060,6 @@ static const struct samsung_pin_ctrl * samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d, struct platform_device *pdev) { - struct device_node *node = pdev->dev.of_node; const struct samsung_pin_bank_data *bdata; const struct samsung_pin_ctrl *ctrl; struct samsung_pin_bank *bank; @@ -1125,7 +1123,7 @@ samsung_pinctrl_get_soc_data(struct samsung_pinctrl_drv_data *d, */ d->virt_base = virt_base[0]; - samsung_banks_of_node_get(&pdev->dev, d, node); + samsung_banks_node_get(&pdev->dev, d); d->pin_base = pin_base; pin_base += d->nr_pins; @@ -1186,7 +1184,7 @@ static int samsung_pinctrl_probe(struct platform_device *pdev) err_unregister: samsung_pinctrl_unregister(pdev, drvdata); err_put_banks: - samsung_banks_of_node_put(drvdata); + samsung_banks_node_put(drvdata); return ret; } From 492fca28fae8e4aa93c1b054423c152d540a36e8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Mon, 11 Apr 2022 14:53:53 +0300 Subject: [PATCH 100/319] pinctrl: samsung: Switch to use for_each_gpiochip_node() helper Switch the code to use for_each_gpiochip_node() helper. While at it, in order to avoid additional churn in the future, switch to fwnode APIs where it makes sense. Signed-off-by: Andy Shevchenko Tested-by: Krzysztof Kozlowski --- drivers/pinctrl/samsung/pinctrl-exynos.c | 8 ++++---- drivers/pinctrl/samsung/pinctrl-s3c24xx.c | 2 +- drivers/pinctrl/samsung/pinctrl-s3c64xx.c | 4 ++-- drivers/pinctrl/samsung/pinctrl-samsung.c | 20 ++++++++++---------- drivers/pinctrl/samsung/pinctrl-samsung.h | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/pinctrl/samsung/pinctrl-exynos.c b/drivers/pinctrl/samsung/pinctrl-exynos.c index c1c4ffbae6e2..6d7ca1758292 100644 --- a/drivers/pinctrl/samsung/pinctrl-exynos.c +++ b/drivers/pinctrl/samsung/pinctrl-exynos.c @@ -307,7 +307,7 @@ __init int exynos_eint_gpio_init(struct samsung_pinctrl_drv_data *d) } bank->irq_chip->chip.name = bank->name; - bank->irq_domain = irq_domain_add_linear(bank->of_node, + bank->irq_domain = irq_domain_create_linear(bank->fwnode, bank->nr_pins, &exynos_eint_irqd_ops, bank); if (!bank->irq_domain) { dev_err(dev, "gpio irq domain add failed\n"); @@ -565,7 +565,7 @@ __init int exynos_eint_wkup_init(struct samsung_pinctrl_drv_data *d) } bank->irq_chip->chip.name = bank->name; - bank->irq_domain = irq_domain_add_linear(bank->of_node, + bank->irq_domain = irq_domain_create_linear(bank->fwnode, bank->nr_pins, &exynos_eint_irqd_ops, bank); if (!bank->irq_domain) { dev_err(dev, "wkup irq domain add failed\n"); @@ -573,7 +573,7 @@ __init int exynos_eint_wkup_init(struct samsung_pinctrl_drv_data *d) return -ENXIO; } - if (!of_find_property(bank->of_node, "interrupts", NULL)) { + if (!fwnode_property_present(bank->fwnode, "interrupts")) { bank->eint_type = EINT_TYPE_WKUP_MUX; ++muxed_banks; continue; @@ -588,7 +588,7 @@ __init int exynos_eint_wkup_init(struct samsung_pinctrl_drv_data *d) } for (idx = 0; idx < bank->nr_pins; ++idx) { - irq = irq_of_parse_and_map(bank->of_node, idx); + irq = irq_of_parse_and_map(to_of_node(bank->fwnode), idx); if (!irq) { dev_err(dev, "irq number for eint-%s-%d not found\n", bank->name, idx); diff --git a/drivers/pinctrl/samsung/pinctrl-s3c24xx.c b/drivers/pinctrl/samsung/pinctrl-s3c24xx.c index ac1eba30cf40..625cb1065eaf 100644 --- a/drivers/pinctrl/samsung/pinctrl-s3c24xx.c +++ b/drivers/pinctrl/samsung/pinctrl-s3c24xx.c @@ -525,7 +525,7 @@ static int s3c24xx_eint_init(struct samsung_pinctrl_drv_data *d) ops = (bank->eint_offset == 0) ? &s3c24xx_gpf_irq_ops : &s3c24xx_gpg_irq_ops; - bank->irq_domain = irq_domain_add_linear(bank->of_node, + bank->irq_domain = irq_domain_create_linear(bank->fwnode, bank->nr_pins, ops, ddata); if (!bank->irq_domain) { dev_err(dev, "wkup irq domain add failed\n"); diff --git a/drivers/pinctrl/samsung/pinctrl-s3c64xx.c b/drivers/pinctrl/samsung/pinctrl-s3c64xx.c index c5f95a1071ae..c5d92db4fdb1 100644 --- a/drivers/pinctrl/samsung/pinctrl-s3c64xx.c +++ b/drivers/pinctrl/samsung/pinctrl-s3c64xx.c @@ -471,7 +471,7 @@ static int s3c64xx_eint_gpio_init(struct samsung_pinctrl_drv_data *d) mask = bank->eint_mask; nr_eints = fls(mask); - bank->irq_domain = irq_domain_add_linear(bank->of_node, + bank->irq_domain = irq_domain_create_linear(bank->fwnode, nr_eints, &s3c64xx_gpio_irqd_ops, bank); if (!bank->irq_domain) { dev_err(dev, "gpio irq domain add failed\n"); @@ -743,7 +743,7 @@ static int s3c64xx_eint_eint0_init(struct samsung_pinctrl_drv_data *d) return -ENOMEM; ddata->bank = bank; - bank->irq_domain = irq_domain_add_linear(bank->of_node, + bank->irq_domain = irq_domain_create_linear(bank->fwnode, nr_eints, &s3c64xx_eint0_irqd_ops, ddata); if (!bank->irq_domain) { dev_err(dev, "wkup irq domain add failed\n"); diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.c b/drivers/pinctrl/samsung/pinctrl-samsung.c index 2ba8b43c25bd..26d309d2516d 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.c +++ b/drivers/pinctrl/samsung/pinctrl-samsung.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -966,7 +967,7 @@ static int samsung_gpiolib_register(struct platform_device *pdev, gc->base = bank->grange.base; gc->ngpio = bank->nr_pins; gc->parent = &pdev->dev; - gc->of_node = bank->of_node; + gc->fwnode = bank->fwnode; gc->label = bank->name; ret = devm_gpiochip_add_data(&pdev->dev, gc, bank); @@ -1009,7 +1010,7 @@ static void samsung_banks_node_put(struct samsung_pinctrl_drv_data *d) bank = d->pin_banks; for (i = 0; i < d->nr_banks; ++i, ++bank) - of_node_put(bank->of_node); + fwnode_handle_put(bank->fwnode); } /* @@ -1020,8 +1021,7 @@ static void samsung_banks_node_get(struct device *dev, struct samsung_pinctrl_dr { const char *suffix = "-gpio-bank"; struct samsung_pin_bank *bank; - struct device_node *node = dev->of_node; - struct device_node *child; + struct fwnode_handle *child; /* Pin bank names are up to 4 characters */ char node_name[20]; unsigned int i; @@ -1037,17 +1037,17 @@ static void samsung_banks_node_get(struct device *dev, struct samsung_pinctrl_dr continue; } - for_each_child_of_node(node, child) { - if (!of_find_property(child, "gpio-controller", NULL)) - continue; - if (of_node_name_eq(child, node_name)) + for_each_gpiochip_node(dev, child) { + struct device_node *np = to_of_node(child); + + if (of_node_name_eq(np, node_name)) break; - else if (of_node_name_eq(child, bank->name)) + if (of_node_name_eq(np, bank->name)) break; } if (child) - bank->of_node = child; + bank->fwnode = child; else dev_warn(dev, "Missing node for bank %s - invalid DTB\n", bank->name); diff --git a/drivers/pinctrl/samsung/pinctrl-samsung.h b/drivers/pinctrl/samsung/pinctrl-samsung.h index 5b32d3f30fcd..fc6f5199c548 100644 --- a/drivers/pinctrl/samsung/pinctrl-samsung.h +++ b/drivers/pinctrl/samsung/pinctrl-samsung.h @@ -165,7 +165,7 @@ struct samsung_pin_bank { u32 pin_base; void *soc_priv; - struct device_node *of_node; + struct fwnode_handle *fwnode; struct samsung_pinctrl_drv_data *drvdata; struct irq_domain *irq_domain; struct gpio_chip gpio_chip; From af47d8033fc731f19600efd27ba4a7d0fdfcc77c Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 14 Apr 2022 21:42:48 +0300 Subject: [PATCH 101/319] gpiolib: Introduce a helper to get first GPIO controller node Introduce a helper to get first GPIO controller node which drivers may want to use. Signed-off-by: Andy Shevchenko Tested-by: Marek Szyprowski --- include/linux/gpio/driver.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 12de0b22b4ef..83e2d72e51bb 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -766,4 +766,14 @@ static inline unsigned int gpiochip_node_count(struct device *dev) return count; } +static inline struct fwnode_handle *gpiochip_node_get_first(struct device *dev) +{ + struct fwnode_handle *fwnode; + + for_each_gpiochip_node(dev, fwnode) + return fwnode; + + return NULL; +} + #endif /* __LINUX_GPIO_DRIVER_H */ From 5d79fa0d33258d8e79064316ce8fae37e27bd34b Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Thu, 7 Apr 2022 15:46:12 +0800 Subject: [PATCH 102/319] ftrace: Fix build warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If CONFIG_SYSCTL and CONFIG_DYNAMIC_FTRACE is n, build warns: kernel/trace/ftrace.c:7912:13: error: ‘is_permanent_ops_registered’ defined but not used [-Werror=unused-function] static bool is_permanent_ops_registered(void) ^~~~~~~~~~~~~~~~~~~~~~~~~~~ kernel/trace/ftrace.c:89:12: error: ‘last_ftrace_enabled’ defined but not used [-Werror=unused-variable] static int last_ftrace_enabled; ^~~~~~~~~~~~~~~~~~~ Move is_permanent_ops_registered() to ifdef block and mark last_ftrace_enabled as __maybe_unused to fix this. Fixes: 7cde53da38a3 ("ftrace: move sysctl_ftrace_enabled to ftrace.c") Signed-off-by: YueHaibing Acked-by: Steven Rostedt (Google) Signed-off-by: Luis Chamberlain --- kernel/trace/ftrace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index a5efbbc289b4..db8d553728b6 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -86,7 +86,7 @@ struct ftrace_ops ftrace_list_end __read_mostly = { /* ftrace_enabled is a method to turn ftrace on or off */ int ftrace_enabled __read_mostly; -static int last_ftrace_enabled; +static int __maybe_unused last_ftrace_enabled; /* Current function tracing op */ struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end; @@ -7909,6 +7909,7 @@ int unregister_ftrace_function(struct ftrace_ops *ops) } EXPORT_SYMBOL_GPL(unregister_ftrace_function); +#ifdef CONFIG_SYSCTL static bool is_permanent_ops_registered(void) { struct ftrace_ops *op; @@ -7921,7 +7922,6 @@ static bool is_permanent_ops_registered(void) return false; } -#ifdef CONFIG_SYSCTL static int ftrace_enable_sysctl(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) From 2cd01bd6b117df07b1bc2852f08694fdd29e40ed Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 16 Feb 2022 16:03:02 +0800 Subject: [PATCH 103/319] platform/chrome: cros_ec: fix error handling in cros_ec_register() Fix cros_ec_register() to unregister platform devices if blocking_notifier_chain_register() fails. Also use the single exit path to handle the platform device unregistration. Fixes: 42cd0ab476e2 ("platform/chrome: cros_ec: Query EC protocol version if EC transitions between RO/RW") Reviewed-by: Prashant Malani Signed-off-by: Tzung-Bi Shih --- drivers/platform/chrome/cros_ec.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index d49a4efe46c8..a5cc8f24299e 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -189,6 +189,8 @@ int cros_ec_register(struct cros_ec_device *ec_dev) ec_dev->max_request = sizeof(struct ec_params_hello); ec_dev->max_response = sizeof(struct ec_response_get_protocol_info); ec_dev->max_passthru = 0; + ec_dev->ec = NULL; + ec_dev->pd = NULL; ec_dev->din = devm_kzalloc(dev, ec_dev->din_size, GFP_KERNEL); if (!ec_dev->din) @@ -245,18 +247,16 @@ int cros_ec_register(struct cros_ec_device *ec_dev) if (IS_ERR(ec_dev->pd)) { dev_err(ec_dev->dev, "Failed to create CrOS PD platform device\n"); - platform_device_unregister(ec_dev->ec); - return PTR_ERR(ec_dev->pd); + err = PTR_ERR(ec_dev->pd); + goto exit; } } if (IS_ENABLED(CONFIG_OF) && dev->of_node) { err = devm_of_platform_populate(dev); if (err) { - platform_device_unregister(ec_dev->pd); - platform_device_unregister(ec_dev->ec); dev_err(dev, "Failed to register sub-devices\n"); - return err; + goto exit; } } @@ -278,7 +278,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev) err = blocking_notifier_chain_register(&ec_dev->event_notifier, &ec_dev->notifier_ready); if (err) - return err; + goto exit; } dev_info(dev, "Chrome EC device registered\n"); @@ -291,6 +291,10 @@ int cros_ec_register(struct cros_ec_device *ec_dev) cros_ec_irq_thread(0, ec_dev); return 0; +exit: + platform_device_unregister(ec_dev->ec); + platform_device_unregister(ec_dev->pd); + return err; } EXPORT_SYMBOL(cros_ec_register); From f47a6113f4e87db7ca066635822e1b3ca3ed9514 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 16 Feb 2022 16:03:03 +0800 Subject: [PATCH 104/319] platform/chrome: cros_ec: remove unused variable `was_wake_device` Reviewed-by: Prashant Malani Signed-off-by: Tzung-Bi Shih --- drivers/platform/chrome/cros_ec.c | 1 - include/linux/platform_data/cros_ec_proto.h | 3 --- 2 files changed, 4 deletions(-) diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index a5cc8f24299e..836794871443 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -342,7 +342,6 @@ int cros_ec_suspend(struct cros_ec_device *ec_dev) ec_dev->wake_enabled = !enable_irq_wake(ec_dev->irq); disable_irq(ec_dev->irq); - ec_dev->was_wake_device = ec_dev->wake_enabled; ec_dev->suspended = true; return 0; diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index df3c78c92ca2..c65971ec90ea 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -76,8 +76,6 @@ struct cros_ec_command { * struct cros_ec_device - Information about a ChromeOS EC device. * @phys_name: Name of physical comms layer (e.g. 'i2c-4'). * @dev: Device pointer for physical comms device - * @was_wake_device: True if this device was set to wake the system from - * sleep at the last suspend. * @cros_class: The class structure for this device. * @cmd_readmem: Direct read of the EC memory-mapped region, if supported. * @offset: Is within EC_LPC_ADDR_MEMMAP region. @@ -137,7 +135,6 @@ struct cros_ec_device { /* These are used by other drivers that want to talk to the EC */ const char *phys_name; struct device *dev; - bool was_wake_device; struct class *cros_class; int (*cmd_readmem)(struct cros_ec_device *ec, unsigned int offset, unsigned int bytes, void *dest); From 9fbe967d4e6e017c85c94aead6a1310b5f77db9a Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 16 Feb 2022 16:03:04 +0800 Subject: [PATCH 105/319] platform/chrome: cros_ec: determine `wake_enabled` in cros_ec_suspend() `wake_enabled` indicates cros_ec_resume() needs to call disable_irq_wake() to undo enable_irq_wake() in cros_ec_suspend(). Determine `wake_enabled` in cros_ec_suspend() instead of reset-after-used in cros_ec_resume(). Reviewed-by: Prashant Malani Signed-off-by: Tzung-Bi Shih --- drivers/platform/chrome/cros_ec.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index 836794871443..3bd2b548179d 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -340,6 +340,8 @@ int cros_ec_suspend(struct cros_ec_device *ec_dev) if (device_may_wakeup(dev)) ec_dev->wake_enabled = !enable_irq_wake(ec_dev->irq); + else + ec_dev->wake_enabled = false; disable_irq(ec_dev->irq); ec_dev->suspended = true; @@ -381,10 +383,9 @@ int cros_ec_resume(struct cros_ec_device *ec_dev) dev_dbg(ec_dev->dev, "Error %d sending resume event to ec", ret); - if (ec_dev->wake_enabled) { + if (ec_dev->wake_enabled) disable_irq_wake(ec_dev->irq); - ec_dev->wake_enabled = 0; - } + /* * Let the mfd devices know about events that occur during * suspend. This way the clients know what to do with them. From 5781a33098c69a3b08890ad78a297b81bb69ca4b Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 16 Feb 2022 16:03:05 +0800 Subject: [PATCH 106/319] platform/chrome: cros_ec: sort header inclusion alphabetically Sort header inclusion alphabetically. Reviewed-by: Guenter Roeck Reviewed-by: Prashant Malani Signed-off-by: Tzung-Bi Shih --- drivers/platform/chrome/cros_ec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index 3bd2b548179d..9a5d01bf1f76 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -9,12 +9,12 @@ * battery charging and regulator control, firmware update. */ -#include #include -#include #include +#include #include #include +#include #include #include "cros_ec.h" From 8d4668064cce8f8d52d4bd2b3b864feed33b1258 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Wed, 16 Feb 2022 16:03:06 +0800 Subject: [PATCH 107/319] platform/chrome: cros_ec: append newline to all logs To be consistent, append newline ("\n") to all logs. Reviewed-by: Guenter Roeck Reviewed-by: Prashant Malani Signed-off-by: Tzung-Bi Shih --- drivers/platform/chrome/cros_ec.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c index 9a5d01bf1f76..b3e94cdf7d1a 100644 --- a/drivers/platform/chrome/cros_ec.c +++ b/drivers/platform/chrome/cros_ec.c @@ -215,7 +215,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev) IRQF_TRIGGER_LOW | IRQF_ONESHOT, "chromeos-ec", ec_dev); if (err) { - dev_err(dev, "Failed to request IRQ %d: %d", + dev_err(dev, "Failed to request IRQ %d: %d\n", ec_dev->irq, err); return err; } @@ -266,7 +266,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev) */ err = cros_ec_sleep_event(ec_dev, 0); if (err < 0) - dev_dbg(ec_dev->dev, "Error %d clearing sleep event to ec", + dev_dbg(ec_dev->dev, "Error %d clearing sleep event to ec\n", err); if (ec_dev->mkbp_event_supported) { @@ -335,7 +335,7 @@ int cros_ec_suspend(struct cros_ec_device *ec_dev) ret = cros_ec_sleep_event(ec_dev, sleep_event); if (ret < 0) - dev_dbg(ec_dev->dev, "Error %d sending suspend event to ec", + dev_dbg(ec_dev->dev, "Error %d sending suspend event to ec\n", ret); if (device_may_wakeup(dev)) @@ -380,7 +380,7 @@ int cros_ec_resume(struct cros_ec_device *ec_dev) ret = cros_ec_sleep_event(ec_dev, sleep_event); if (ret < 0) - dev_dbg(ec_dev->dev, "Error %d sending resume event to ec", + dev_dbg(ec_dev->dev, "Error %d sending resume event to ec\n", ret); if (ec_dev->wake_enabled) From 2954ce1e452567c17e3899be2df6b145bc50a05e Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:36:03 +0300 Subject: [PATCH 108/319] pinctrl: armada-37xx: Switch to use fwnode instead of of_node GPIO library now accepts fwnode as a firmware node, so switch the driver to use it. Signed-off-by: Andy Shevchenko --- drivers/pinctrl/mvebu/pinctrl-armada-37xx.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c index 08cad14042e2..110f70bce3fe 100644 --- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c +++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -787,18 +788,13 @@ static int armada_37xx_gpiochip_register(struct platform_device *pdev, struct armada_37xx_pinctrl *info) { struct device *dev = &pdev->dev; - struct device_node *np; + struct fwnode_handle *fwnode; struct gpio_chip *gc; - int ret = -ENODEV; + int ret; - for_each_child_of_node(dev->of_node, np) { - if (of_find_property(np, "gpio-controller", NULL)) { - ret = 0; - break; - } - } - if (ret) - return ret; + fwnode = gpiochip_node_get_first(dev); + if (!fwnode) + return -ENODEV; info->gpio_chip = armada_37xx_gpiolib_chip; @@ -806,7 +802,7 @@ static int armada_37xx_gpiochip_register(struct platform_device *pdev, gc->ngpio = info->data->nr_pins; gc->parent = dev; gc->base = -1; - gc->of_node = np; + gc->fwnode = fwnode; gc->label = info->data->name; ret = armada_37xx_irqchip_register(pdev, info); From 46d34d4d502ea1030f5de434e6677ec96ca131c3 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:36:04 +0300 Subject: [PATCH 109/319] pinctrl: armada-37xx: Reuse GPIO fwnode in armada_37xx_irqchip_register() Since we have fwnode of the first found GPIO controller assigned to the struct gpio_chip, we may reuse it in the armada_37xx_irqchip_register(). Signed-off-by: Andy Shevchenko --- drivers/pinctrl/mvebu/pinctrl-armada-37xx.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c index 110f70bce3fe..ef4118e49f16 100644 --- a/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c +++ b/drivers/pinctrl/mvebu/pinctrl-armada-37xx.c @@ -727,23 +727,13 @@ static int armada_37xx_irqchip_register(struct platform_device *pdev, struct gpio_chip *gc = &info->gpio_chip; struct irq_chip *irqchip = &info->irq_chip; struct gpio_irq_chip *girq = &gc->irq; + struct device_node *np = to_of_node(gc->fwnode); struct device *dev = &pdev->dev; - struct device_node *np; - int ret = -ENODEV, i, nr_irq_parent; + unsigned int i, nr_irq_parent; - /* Check if we have at least one gpio-controller child node */ - for_each_child_of_node(dev->of_node, np) { - if (of_property_read_bool(np, "gpio-controller")) { - ret = 0; - break; - } - } - if (ret) - return dev_err_probe(dev, ret, "no gpio-controller child node\n"); - - nr_irq_parent = of_irq_count(np); spin_lock_init(&info->irq_lock); + nr_irq_parent = of_irq_count(np); if (!nr_irq_parent) { dev_err(dev, "invalid or no IRQ\n"); return 0; From 2b2dce809920b3cf92df27a484c0649dda7270b8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:36:00 +0300 Subject: [PATCH 110/319] pinctrl: meson: Rename REG_* to MESON_REG_* Currently compilation test fails on x86 due to name collision. The usual way to fix that is to move both conflicting parts to their own namespaces. Rename REG_* to MESON_REG_* as a prerequisite for enabling COMPILE_TEST. Signed-off-by: Andy Shevchenko Tested-by: Marek Szyprowski Reviewed-by: Neil Armstrong --- drivers/pinctrl/meson/pinctrl-meson.c | 24 +++++++++++------------ drivers/pinctrl/meson/pinctrl-meson.h | 28 +++++++++++++-------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c index 49851444a6e3..5b46a0979db7 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.c +++ b/drivers/pinctrl/meson/pinctrl-meson.c @@ -218,13 +218,13 @@ static int meson_pinconf_set_output(struct meson_pinctrl *pc, unsigned int pin, bool out) { - return meson_pinconf_set_gpio_bit(pc, pin, REG_DIR, !out); + return meson_pinconf_set_gpio_bit(pc, pin, MESON_REG_DIR, !out); } static int meson_pinconf_get_output(struct meson_pinctrl *pc, unsigned int pin) { - int ret = meson_pinconf_get_gpio_bit(pc, pin, REG_DIR); + int ret = meson_pinconf_get_gpio_bit(pc, pin, MESON_REG_DIR); if (ret < 0) return ret; @@ -236,13 +236,13 @@ static int meson_pinconf_set_drive(struct meson_pinctrl *pc, unsigned int pin, bool high) { - return meson_pinconf_set_gpio_bit(pc, pin, REG_OUT, high); + return meson_pinconf_set_gpio_bit(pc, pin, MESON_REG_OUT, high); } static int meson_pinconf_get_drive(struct meson_pinctrl *pc, unsigned int pin) { - return meson_pinconf_get_gpio_bit(pc, pin, REG_OUT); + return meson_pinconf_get_gpio_bit(pc, pin, MESON_REG_OUT); } static int meson_pinconf_set_output_drive(struct meson_pinctrl *pc, @@ -269,7 +269,7 @@ static int meson_pinconf_disable_bias(struct meson_pinctrl *pc, if (ret) return ret; - meson_calc_reg_and_bit(bank, pin, REG_PULLEN, ®, &bit); + meson_calc_reg_and_bit(bank, pin, MESON_REG_PULLEN, ®, &bit); ret = regmap_update_bits(pc->reg_pullen, reg, BIT(bit), 0); if (ret) return ret; @@ -288,7 +288,7 @@ static int meson_pinconf_enable_bias(struct meson_pinctrl *pc, unsigned int pin, if (ret) return ret; - meson_calc_reg_and_bit(bank, pin, REG_PULL, ®, &bit); + meson_calc_reg_and_bit(bank, pin, MESON_REG_PULL, ®, &bit); if (pull_up) val = BIT(bit); @@ -296,7 +296,7 @@ static int meson_pinconf_enable_bias(struct meson_pinctrl *pc, unsigned int pin, if (ret) return ret; - meson_calc_reg_and_bit(bank, pin, REG_PULLEN, ®, &bit); + meson_calc_reg_and_bit(bank, pin, MESON_REG_PULLEN, ®, &bit); ret = regmap_update_bits(pc->reg_pullen, reg, BIT(bit), BIT(bit)); if (ret) return ret; @@ -321,7 +321,7 @@ static int meson_pinconf_set_drive_strength(struct meson_pinctrl *pc, if (ret) return ret; - meson_calc_reg_and_bit(bank, pin, REG_DS, ®, &bit); + meson_calc_reg_and_bit(bank, pin, MESON_REG_DS, ®, &bit); if (drive_strength_ua <= 500) { ds_val = MESON_PINCONF_DRV_500UA; @@ -407,7 +407,7 @@ static int meson_pinconf_get_pull(struct meson_pinctrl *pc, unsigned int pin) if (ret) return ret; - meson_calc_reg_and_bit(bank, pin, REG_PULLEN, ®, &bit); + meson_calc_reg_and_bit(bank, pin, MESON_REG_PULLEN, ®, &bit); ret = regmap_read(pc->reg_pullen, reg, &val); if (ret) @@ -416,7 +416,7 @@ static int meson_pinconf_get_pull(struct meson_pinctrl *pc, unsigned int pin) if (!(val & BIT(bit))) { conf = PIN_CONFIG_BIAS_DISABLE; } else { - meson_calc_reg_and_bit(bank, pin, REG_PULL, ®, &bit); + meson_calc_reg_and_bit(bank, pin, MESON_REG_PULL, ®, &bit); ret = regmap_read(pc->reg_pull, reg, &val); if (ret) @@ -447,7 +447,7 @@ static int meson_pinconf_get_drive_strength(struct meson_pinctrl *pc, if (ret) return ret; - meson_calc_reg_and_bit(bank, pin, REG_DS, ®, &bit); + meson_calc_reg_and_bit(bank, pin, MESON_REG_DS, ®, &bit); ret = regmap_read(pc->reg_ds, reg, &val); if (ret) @@ -595,7 +595,7 @@ static int meson_gpio_get(struct gpio_chip *chip, unsigned gpio) if (ret) return ret; - meson_calc_reg_and_bit(bank, gpio, REG_IN, ®, &bit); + meson_calc_reg_and_bit(bank, gpio, MESON_REG_IN, ®, &bit); regmap_read(pc->reg_gpio, reg, &val); return !!(val & BIT(bit)); diff --git a/drivers/pinctrl/meson/pinctrl-meson.h b/drivers/pinctrl/meson/pinctrl-meson.h index ff5372e0a475..b197827027bd 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.h +++ b/drivers/pinctrl/meson/pinctrl-meson.h @@ -63,13 +63,13 @@ struct meson_reg_desc { * enum meson_reg_type - type of registers encoded in @meson_reg_desc */ enum meson_reg_type { - REG_PULLEN, - REG_PULL, - REG_DIR, - REG_OUT, - REG_IN, - REG_DS, - NUM_REG, + MESON_REG_PULLEN, + MESON_REG_PULL, + MESON_REG_DIR, + MESON_REG_OUT, + MESON_REG_IN, + MESON_REG_DS, + MESON_NUM_REG, }; /** @@ -102,7 +102,7 @@ struct meson_bank { unsigned int last; int irq_first; int irq_last; - struct meson_reg_desc regs[NUM_REG]; + struct meson_reg_desc regs[MESON_NUM_REG]; }; struct meson_pinctrl_data { @@ -150,12 +150,12 @@ struct meson_pinctrl { .irq_first = fi, \ .irq_last = li, \ .regs = { \ - [REG_PULLEN] = { per, peb }, \ - [REG_PULL] = { pr, pb }, \ - [REG_DIR] = { dr, db }, \ - [REG_OUT] = { or, ob }, \ - [REG_IN] = { ir, ib }, \ - [REG_DS] = { dsr, dsb }, \ + [MESON_REG_PULLEN] = { per, peb }, \ + [MESON_REG_PULL] = { pr, pb }, \ + [MESON_REG_DIR] = { dr, db }, \ + [MESON_REG_OUT] = { or, ob }, \ + [MESON_REG_IN] = { ir, ib }, \ + [MESON_REG_DS] = { dsr, dsb }, \ }, \ } From 6671d0bc17866f71b8ca7ea3fb7e6dfc2a8069d2 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:36:01 +0300 Subject: [PATCH 111/319] pinctrl: meson: Enable COMPILE_TEST Enable COMPILE_TEST for a better test coverage. Signed-off-by: Andy Shevchenko Tested-by: Marek Szyprowski Reviewed-by: Neil Armstrong --- drivers/pinctrl/meson/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pinctrl/meson/Kconfig b/drivers/pinctrl/meson/Kconfig index d1955c65b4b6..64fb9e074ac6 100644 --- a/drivers/pinctrl/meson/Kconfig +++ b/drivers/pinctrl/meson/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only menuconfig PINCTRL_MESON tristate "Amlogic SoC pinctrl drivers" - depends on ARCH_MESON + depends on ARCH_MESON || COMPILE_TEST depends on OF default y select PINMUX From edc5601db66411a8c9c6b08b3aacf7e154a34c6d Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 1 Apr 2022 13:36:02 +0300 Subject: [PATCH 112/319] pinctrl: meson: Replace custom code by gpiochip_node_count() call Since we have generic function to count GPIO controller nodes under a given device, there is no need to open code it. Replace custom code by gpiochip_node_count() call. Signed-off-by: Andy Shevchenko Tested-by: Marek Szyprowski Reviewed-by: Neil Armstrong --- drivers/pinctrl/meson/pinctrl-meson.c | 28 ++++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/drivers/pinctrl/meson/pinctrl-meson.c b/drivers/pinctrl/meson/pinctrl-meson.c index 5b46a0979db7..cc2cd73ff8f9 100644 --- a/drivers/pinctrl/meson/pinctrl-meson.c +++ b/drivers/pinctrl/meson/pinctrl-meson.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -662,27 +663,22 @@ static struct regmap *meson_map_resource(struct meson_pinctrl *pc, return devm_regmap_init_mmio(pc->dev, base, &meson_regmap_config); } -static int meson_pinctrl_parse_dt(struct meson_pinctrl *pc, - struct device_node *node) +static int meson_pinctrl_parse_dt(struct meson_pinctrl *pc) { - struct device_node *np, *gpio_np = NULL; + struct device_node *gpio_np; + unsigned int chips; - for_each_child_of_node(node, np) { - if (!of_find_property(np, "gpio-controller", NULL)) - continue; - if (gpio_np) { - dev_err(pc->dev, "multiple gpio nodes\n"); - of_node_put(np); - return -EINVAL; - } - gpio_np = np; - } - - if (!gpio_np) { + chips = gpiochip_node_count(pc->dev); + if (!chips) { dev_err(pc->dev, "no gpio node found\n"); return -EINVAL; } + if (chips > 1) { + dev_err(pc->dev, "multiple gpio nodes\n"); + return -EINVAL; + } + gpio_np = to_of_node(gpiochip_node_get_first(pc->dev)); pc->of_node = gpio_np; pc->reg_mux = meson_map_resource(pc, gpio_np, "mux"); @@ -751,7 +747,7 @@ int meson_pinctrl_probe(struct platform_device *pdev) pc->dev = dev; pc->data = (struct meson_pinctrl_data *) of_device_get_match_data(dev); - ret = meson_pinctrl_parse_dt(pc, dev->of_node); + ret = meson_pinctrl_parse_dt(pc); if (ret) return ret; From 4f3e79b36d7fdbc3315efe5612a01800931fa25d Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Wed, 6 Apr 2022 22:51:56 +0200 Subject: [PATCH 113/319] gpio: ixp4xx: Detect special machines by compatible There are some special clock amendments for two machines formerly detected by their machine_is() boardfile macro. They are now migrated to device tree so use of_machine_is_compatible() instead. Signed-off-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/Kconfig | 1 - drivers/gpio/gpio-ixp4xx.c | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 45764ec3b2eb..075a5d0feef7 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -353,7 +353,6 @@ config GPIO_IOP config GPIO_IXP4XX bool "Intel IXP4xx GPIO" - depends on ARM # For depends on ARCH_IXP4XX select GPIO_GENERIC select GPIOLIB_IRQCHIP diff --git a/drivers/gpio/gpio-ixp4xx.c b/drivers/gpio/gpio-ixp4xx.c index b3b050604e0b..1acda980d119 100644 --- a/drivers/gpio/gpio-ixp4xx.c +++ b/drivers/gpio/gpio-ixp4xx.c @@ -17,8 +17,6 @@ /* Include that go away with DT transition */ #include -#include - #define IXP4XX_REG_GPOUT 0x00 #define IXP4XX_REG_GPOE 0x04 #define IXP4XX_REG_GPIN 0x08 @@ -240,7 +238,8 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev) * Make sure GPIO 14 and 15 are NOT used as clocks but GPIO on * specific machines. */ - if (machine_is_dsmg600() || machine_is_nas100d()) + if (of_machine_is_compatible("dlink,dsm-g600-a") || + of_machine_is_compatible("iom,nas-100d")) __raw_writel(0x0, g->base + IXP4XX_REG_GPCLK); /* From 57b888ca2541785de2fcb90575b378921919b6c0 Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Fri, 18 Mar 2022 09:54:22 -0700 Subject: [PATCH 114/319] platform/chrome: Re-introduce cros_ec_cmd_xfer and use it for ioctls Commit 413dda8f2c6f ("platform/chrome: cros_ec_chardev: Use cros_ec_cmd_xfer_status helper") inadvertendly changed the userspace ABI. Previously, cros_ec ioctls would only report errors if the EC communication failed, and otherwise return success and the result of the EC communication. An EC command execution failure was reported in the EC response field. The above mentioned commit changed this behavior, and the ioctl itself would fail. This breaks userspace commands trying to analyze the EC command execution error since the actual EC command response is no longer reported to userspace. Fix the problem by re-introducing the cros_ec_cmd_xfer() helper, and use it to handle ioctl messages. Fixes: 413dda8f2c6f ("platform/chrome: cros_ec_chardev: Use cros_ec_cmd_xfer_status helper") Cc: Daisuke Nojiri Cc: Rob Barnes Cc: Rajat Jain Cc: Brian Norris Cc: Parth Malkan Reviewed-by: Daisuke Nojiri Reviewed-by: Brian Norris Signed-off-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih --- drivers/platform/chrome/cros_ec_chardev.c | 2 +- drivers/platform/chrome/cros_ec_proto.c | 50 +++++++++++++++++---- include/linux/platform_data/cros_ec_proto.h | 3 ++ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_chardev.c b/drivers/platform/chrome/cros_ec_chardev.c index e0bce869c49a..fd33de546aee 100644 --- a/drivers/platform/chrome/cros_ec_chardev.c +++ b/drivers/platform/chrome/cros_ec_chardev.c @@ -301,7 +301,7 @@ static long cros_ec_chardev_ioctl_xcmd(struct cros_ec_dev *ec, void __user *arg) } s_cmd->command += ec->cmd_offset; - ret = cros_ec_cmd_xfer_status(ec->ec_dev, s_cmd); + ret = cros_ec_cmd_xfer(ec->ec_dev, s_cmd); /* Only copy data to userland if data was received. */ if (ret < 0) goto exit; diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index c4caf2e2de82..ac1419881ff3 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -560,22 +560,28 @@ int cros_ec_query_all(struct cros_ec_device *ec_dev) EXPORT_SYMBOL(cros_ec_query_all); /** - * cros_ec_cmd_xfer_status() - Send a command to the ChromeOS EC. + * cros_ec_cmd_xfer() - Send a command to the ChromeOS EC. * @ec_dev: EC device. * @msg: Message to write. * - * Call this to send a command to the ChromeOS EC. This should be used instead of calling the EC's - * cmd_xfer() callback directly. It returns success status only if both the command was transmitted - * successfully and the EC replied with success status. + * Call this to send a command to the ChromeOS EC. This should be used instead + * of calling the EC's cmd_xfer() callback directly. This function does not + * convert EC command execution error codes to Linux error codes. Most + * in-kernel users will want to use cros_ec_cmd_xfer_status() instead since + * that function implements the conversion. * * Return: - * >=0 - The number of bytes transferred - * <0 - Linux error code + * >0 - EC command was executed successfully. The return value is the number + * of bytes returned by the EC (excluding the header). + * =0 - EC communication was successful. EC command execution results are + * reported in msg->result. The result will be EC_RES_SUCCESS if the + * command was executed successfully or report an EC command execution + * error. + * <0 - EC communication error. Return value is the Linux error code. */ -int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev, - struct cros_ec_command *msg) +int cros_ec_cmd_xfer(struct cros_ec_device *ec_dev, struct cros_ec_command *msg) { - int ret, mapped; + int ret; mutex_lock(&ec_dev->lock); if (ec_dev->proto_version == EC_PROTO_VERSION_UNKNOWN) { @@ -616,6 +622,32 @@ int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev, ret = send_command(ec_dev, msg); mutex_unlock(&ec_dev->lock); + return ret; +} +EXPORT_SYMBOL(cros_ec_cmd_xfer); + +/** + * cros_ec_cmd_xfer_status() - Send a command to the ChromeOS EC. + * @ec_dev: EC device. + * @msg: Message to write. + * + * Call this to send a command to the ChromeOS EC. This should be used instead of calling the EC's + * cmd_xfer() callback directly. It returns success status only if both the command was transmitted + * successfully and the EC replied with success status. + * + * Return: + * >=0 - The number of bytes transferred. + * <0 - Linux error code + */ +int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev, + struct cros_ec_command *msg) +{ + int ret, mapped; + + ret = cros_ec_cmd_xfer(ec_dev, msg); + if (ret < 0) + return ret; + mapped = cros_ec_map_error(msg->result); if (mapped) { dev_dbg(ec_dev->dev, "Command result (err: %d [%d])\n", diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index c65971ec90ea..138fd912c808 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -213,6 +213,9 @@ int cros_ec_prepare_tx(struct cros_ec_device *ec_dev, int cros_ec_check_result(struct cros_ec_device *ec_dev, struct cros_ec_command *msg); +int cros_ec_cmd_xfer(struct cros_ec_device *ec_dev, + struct cros_ec_command *msg); + int cros_ec_cmd_xfer_status(struct cros_ec_device *ec_dev, struct cros_ec_command *msg); From 6a437208cb942a2dd98f7e1c3fd347ed3d425ffc Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:46 +0100 Subject: [PATCH 115/319] arm64: Expand ESR_ELx_WFx_ISS_TI to match its ARMv8.7 definition Starting with FEAT_WFXT in ARMv8.7, the TI field in the ISS that is reported on a WFx trap is expanded by one bit to allow the description of WFET and WFIT. Special care is taken to exclude the WFxT bit from the mask used to match WFI so that it also matches WFIT when trapped from EL0. Reviewed-by: Joey Gouly Signed-off-by: Marc Zyngier Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/20220419182755.601427-2-maz@kernel.org --- arch/arm64/include/asm/esr.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index d52a0b269ee8..65c2201b11b2 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -133,7 +133,8 @@ #define ESR_ELx_CV (UL(1) << 24) #define ESR_ELx_COND_SHIFT (20) #define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT) -#define ESR_ELx_WFx_ISS_TI (UL(1) << 0) +#define ESR_ELx_WFx_ISS_TI (UL(3) << 0) +#define ESR_ELx_WFx_ISS_WFxT (UL(2) << 0) #define ESR_ELx_WFx_ISS_WFI (UL(0) << 0) #define ESR_ELx_WFx_ISS_WFE (UL(1) << 0) #define ESR_ELx_xVC_IMM_MASK ((1UL << 16) - 1) @@ -146,7 +147,8 @@ #define DISR_EL1_ESR_MASK (ESR_ELx_AET | ESR_ELx_EA | ESR_ELx_FSC) /* ESR value templates for specific events */ -#define ESR_ELx_WFx_MASK (ESR_ELx_EC_MASK | ESR_ELx_WFx_ISS_TI) +#define ESR_ELx_WFx_MASK (ESR_ELx_EC_MASK | \ + (ESR_ELx_WFx_ISS_TI & ~ESR_ELx_WFx_ISS_WFxT)) #define ESR_ELx_WFx_WFI_VAL ((ESR_ELx_EC_WFx << ESR_ELx_EC_SHIFT) | \ ESR_ELx_WFx_ISS_WFI) From bdcc2f280334e4e3f42a5a740494444f1026fb65 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:47 +0100 Subject: [PATCH 116/319] arm64: Add RV and RN fields for ESR_ELx_WFx_ISS The ISS field exposed by ESR_ELx contain two additional subfields with FEAT_WFxT: - RN, the register number containing the timeout - RV, indicating if the register number is valid Describe these two fields according to the arch spec. No functional change. Reviewed-by: Joey Gouly Signed-off-by: Marc Zyngier Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/20220419182755.601427-3-maz@kernel.org --- arch/arm64/include/asm/esr.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 65c2201b11b2..15156c478054 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -133,6 +133,8 @@ #define ESR_ELx_CV (UL(1) << 24) #define ESR_ELx_COND_SHIFT (20) #define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT) +#define ESR_ELx_WFx_ISS_RN (UL(0x1F) << 5) +#define ESR_ELx_WFx_ISS_RV (UL(1) << 2) #define ESR_ELx_WFx_ISS_TI (UL(3) << 0) #define ESR_ELx_WFx_ISS_WFxT (UL(2) << 0) #define ESR_ELx_WFx_ISS_WFI (UL(0) << 0) From 69bb02ebc38ace438c9cd7c5315cfe43862b51fe Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 20 Apr 2022 13:21:12 +0100 Subject: [PATCH 117/319] arm64: Add HWCAP advertising FEAT_WFXT In order to allow userspace to enjoy WFET, add a new HWCAP that advertises it when available. Signed-off-by: Marc Zyngier Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/20220419182755.601427-9-maz@kernel.org --- Documentation/arm64/cpu-feature-registers.rst | 2 ++ Documentation/arm64/elf_hwcaps.rst | 4 ++++ arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + arch/arm64/kernel/cpufeature.c | 2 ++ arch/arm64/kernel/cpuinfo.c | 1 + arch/arm64/tools/cpucaps | 1 + 7 files changed, 12 insertions(+) diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst index 749ae970c319..04ba83e1965f 100644 --- a/Documentation/arm64/cpu-feature-registers.rst +++ b/Documentation/arm64/cpu-feature-registers.rst @@ -290,6 +290,8 @@ infrastructure: +------------------------------+---------+---------+ | RPRES | [7-4] | y | +------------------------------+---------+---------+ + | WFXT | [3-0] | y | + +------------------------------+---------+---------+ Appendix I: Example diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst index a8f30963e550..af3ab524e826 100644 --- a/Documentation/arm64/elf_hwcaps.rst +++ b/Documentation/arm64/elf_hwcaps.rst @@ -264,6 +264,10 @@ HWCAP2_MTE3 Functionality implied by ID_AA64PFR1_EL1.MTE == 0b0011, as described by Documentation/arm64/memory-tagging-extension.rst. +HWCAP2_WFXT + + Functionality implied by ID_AA64ISAR2_EL1.WFXT == 0b0010. + 4. Unused AT_HWCAP bits ----------------------- diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 8db5ec0089db..909337b50e1f 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -109,6 +109,7 @@ #define KERNEL_HWCAP_AFP __khwcap2_feature(AFP) #define KERNEL_HWCAP_RPRES __khwcap2_feature(RPRES) #define KERNEL_HWCAP_MTE3 __khwcap2_feature(MTE3) +#define KERNEL_HWCAP_WFXT __khwcap2_feature(WFXT) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 99cb5d383048..9dddde1d046c 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -79,5 +79,6 @@ #define HWCAP2_AFP (1 << 20) #define HWCAP2_RPRES (1 << 21) #define HWCAP2_MTE3 (1 << 22) +#define HWCAP2_WFXT (1 << 23) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index d72c4b4d389c..a6db8ca04429 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -237,6 +237,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = { ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH), FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_GPA3_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_WFXT_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -2575,6 +2576,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = { HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV), HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP), HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES), + HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_WFXT_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_WFXT_SUPPORTED, CAP_HWCAP, KERNEL_HWCAP_WFXT), {}, }; diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 330b92ea863a..781eab314794 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -98,6 +98,7 @@ static const char *const hwcap_str[] = { [KERNEL_HWCAP_AFP] = "afp", [KERNEL_HWCAP_RPRES] = "rpres", [KERNEL_HWCAP_MTE3] = "mte3", + [KERNEL_HWCAP_WFXT] = "wfxt", }; #ifdef CONFIG_COMPAT diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 3ed418f70e3b..01f7d253dec4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -38,6 +38,7 @@ HAS_STAGE2_FWB HAS_SYSREG_GIC_CPUIF HAS_TLB_RANGE HAS_VIRT_HOST_EXTN +HAS_WFXT HW_DBM KVM_PROTECTED_MODE MISMATCHED_CACHE_TYPE From 9eae588529751f95834bca775b30b66291def7f6 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:54 +0100 Subject: [PATCH 118/319] arm64: Add wfet()/wfit() helpers Just like we have helpers for WFI and WFE, add the WFxT versions. Note that the encoding is that reported by objdump, as no currrent toolchain knows about these instructions yet. Signed-off-by: Marc Zyngier Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/20220419182755.601427-10-maz@kernel.org --- arch/arm64/include/asm/barrier.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 62217be36217..9f3e2c3d2ca0 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -16,7 +16,11 @@ #define sev() asm volatile("sev" : : : "memory") #define wfe() asm volatile("wfe" : : : "memory") +#define wfet(val) asm volatile("msr s0_3_c1_c0_0, %0" \ + : : "r" (val) : "memory") #define wfi() asm volatile("wfi" : : : "memory") +#define wfit(val) asm volatile("msr s0_3_c1_c0_1, %0" \ + : : "r" (val) : "memory") #define isb() asm volatile("isb" : : : "memory") #define dmb(opt) asm volatile("dmb " #opt : : : "memory") From 7d26b0516a0df5888fd1486054bc5159f6c0b88f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:55 +0100 Subject: [PATCH 119/319] arm64: Use WFxT for __delay() when possible Marginally optimise __delay() by using a WFIT/WFET sequence. It probably is a win if no interrupt fires during the delay. Signed-off-by: Marc Zyngier Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/20220419182755.601427-11-maz@kernel.org --- arch/arm64/lib/delay.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index 1688af0a4c97..5b7890139bc2 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -27,7 +27,17 @@ void __delay(unsigned long cycles) { cycles_t start = get_cycles(); - if (arch_timer_evtstrm_available()) { + if (cpus_have_const_cap(ARM64_HAS_WFXT)) { + u64 end = start + cycles; + + /* + * Start with WFIT. If an interrupt makes us resume + * early, use a WFET loop to complete the delay. + */ + wfit(end); + while ((get_cycles() - start) < cycles) + wfet(end); + } else if (arch_timer_evtstrm_available()) { const cycles_t timer_evt_period = USECS_TO_CYCLES(ARCH_TIMER_EVT_STREAM_PERIOD_US); From b57de4ffd7c6d1247fdaef8437450ab7c1dd4ed0 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:48 +0100 Subject: [PATCH 120/319] KVM: arm64: Simplify kvm_cpu_has_pending_timer() kvm_cpu_has_pending_timer() ends up checking all the possible timers for a wake-up cause. However, we already check for pending interrupts whenever we try to wake-up a vcpu, including the timer interrupts. Obviously, doing the same work twice is once too many. Reduce this helper to almost nothing, but keep it around, as we are going to make use of it soon. Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220419182755.601427-4-maz@kernel.org --- arch/arm64/kvm/arch_timer.c | 10 ++-------- arch/arm64/kvm/arm.c | 5 ----- include/kvm/arm_arch_timer.h | 2 -- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 6e542e2eae32..16dda1a383a6 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -350,15 +350,9 @@ static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) return cval <= now; } -bool kvm_timer_is_pending(struct kvm_vcpu *vcpu) +int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) { - struct timer_map map; - - get_timer_map(vcpu, &map); - - return kvm_timer_should_fire(map.direct_vtimer) || - kvm_timer_should_fire(map.direct_ptimer) || - kvm_timer_should_fire(map.emul_ptimer); + return 0; } /* diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 523bc934fe2f..2122c699af06 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -356,11 +356,6 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) kvm_arm_vcpu_destroy(vcpu); } -int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) -{ - return kvm_timer_is_pending(vcpu); -} - void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) { diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index 51c19381108c..cd6d8f260eab 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -76,8 +76,6 @@ int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); -bool kvm_timer_is_pending(struct kvm_vcpu *vcpu); - u64 kvm_phys_timer_read(void); void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu); From daf85a5f6be33788e18ff3efad1d7c3ad66a8cb3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:49 +0100 Subject: [PATCH 121/319] KVM: arm64: Introduce kvm_counter_compute_delta() helper Refactor kvm_timer_compute_delta() and extract a helper that compute the delta (in ns) between a given timer and an arbitrary value. No functional change expected. Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220419182755.601427-5-maz@kernel.org --- arch/arm64/kvm/arch_timer.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 16dda1a383a6..c92a68190f6a 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -208,18 +208,16 @@ static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id) return IRQ_HANDLED; } -static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx) +static u64 kvm_counter_compute_delta(struct arch_timer_context *timer_ctx, + u64 val) { - u64 cval, now; + u64 now = kvm_phys_timer_read() - timer_get_offset(timer_ctx); - cval = timer_get_cval(timer_ctx); - now = kvm_phys_timer_read() - timer_get_offset(timer_ctx); - - if (now < cval) { + if (now < val) { u64 ns; ns = cyclecounter_cyc2ns(timecounter->cc, - cval - now, + val - now, timecounter->mask, &timecounter->frac); return ns; @@ -228,6 +226,11 @@ static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx) return 0; } +static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx) +{ + return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx)); +} + static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) { WARN_ON(timer_ctx && timer_ctx->loaded); From 89f5074c503b6b6f181c0240c931f67bcaf266e9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:50 +0100 Subject: [PATCH 122/319] KVM: arm64: Handle blocking WFIT instruction When trapping a blocking WFIT instruction, take it into account when computing the deadline of the background timer. The state is tracked with a new vcpu flag, and is gated by a new CPU capability, which isn't currently enabled. Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220419182755.601427-6-maz@kernel.org --- arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/arch_timer.c | 22 ++++++++++++++++++++-- arch/arm64/kvm/arm.c | 1 + arch/arm64/kvm/handle_exit.c | 7 ++++++- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index e3b25dc6c367..9e6e8701933e 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -441,6 +441,7 @@ struct kvm_vcpu_arch { #define KVM_ARM64_DEBUG_STATE_SAVE_TRBE (1 << 13) /* Save TRBE context if active */ #define KVM_ARM64_FP_FOREIGN_FPSTATE (1 << 14) #define KVM_ARM64_ON_UNSUPPORTED_CPU (1 << 15) /* Physical CPU not in supported_cpus */ +#define KVM_ARM64_WFIT (1 << 16) /* WFIT instruction trapped */ #define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ KVM_GUESTDBG_USE_SW_BP | \ diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index c92a68190f6a..4e39ace073af 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -239,6 +239,20 @@ static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx) (ARCH_TIMER_CTRL_IT_MASK | ARCH_TIMER_CTRL_ENABLE)) == ARCH_TIMER_CTRL_ENABLE); } +static bool vcpu_has_wfit_active(struct kvm_vcpu *vcpu) +{ + return (cpus_have_final_cap(ARM64_HAS_WFXT) && + (vcpu->arch.flags & KVM_ARM64_WFIT)); +} + +static u64 wfit_delay_ns(struct kvm_vcpu *vcpu) +{ + struct arch_timer_context *ctx = vcpu_vtimer(vcpu); + u64 val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu)); + + return kvm_counter_compute_delta(ctx, val); +} + /* * Returns the earliest expiration time in ns among guest timers. * Note that it will return 0 if none of timers can fire. @@ -256,6 +270,9 @@ static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu) min_delta = min(min_delta, kvm_timer_compute_delta(ctx)); } + if (vcpu_has_wfit_active(vcpu)) + min_delta = min(min_delta, wfit_delay_ns(vcpu)); + /* If none of timers can fire, then return 0 */ if (min_delta == ULLONG_MAX) return 0; @@ -355,7 +372,7 @@ static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx) int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) { - return 0; + return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0; } /* @@ -481,7 +498,8 @@ static void kvm_timer_blocking(struct kvm_vcpu *vcpu) */ if (!kvm_timer_irq_can_fire(map.direct_vtimer) && !kvm_timer_irq_can_fire(map.direct_ptimer) && - !kvm_timer_irq_can_fire(map.emul_ptimer)) + !kvm_timer_irq_can_fire(map.emul_ptimer) && + !vcpu_has_wfit_active(vcpu)) return; /* diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 2122c699af06..e7cb8a4d2e81 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -634,6 +634,7 @@ void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) preempt_enable(); kvm_vcpu_halt(vcpu); + vcpu->arch.flags &= ~KVM_ARM64_WFIT; kvm_clear_request(KVM_REQ_UNHALT, vcpu); preempt_disable(); diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 97fe14aab1a3..4260f2cd1971 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -85,16 +85,21 @@ static int handle_no_fpsimd(struct kvm_vcpu *vcpu) * WFI: Simply call kvm_vcpu_halt(), which will halt execution of * world-switches and schedule other host processes until there is an * incoming IRQ or FIQ to the VM. + * WFIT: Same as WFI, with a timed wakeup implemented as a background timer */ static int kvm_handle_wfx(struct kvm_vcpu *vcpu) { - if (kvm_vcpu_get_esr(vcpu) & ESR_ELx_WFx_ISS_WFE) { + u64 esr = kvm_vcpu_get_esr(vcpu); + + if (esr & ESR_ELx_WFx_ISS_WFE) { trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true); vcpu->stat.wfe_exit_stat++; kvm_vcpu_on_spin(vcpu, vcpu_mode_priv(vcpu)); } else { trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false); vcpu->stat.wfi_exit_stat++; + if ((esr & (ESR_ELx_WFx_ISS_RV | ESR_ELx_WFx_ISS_WFxT)) == (ESR_ELx_WFx_ISS_RV | ESR_ELx_WFx_ISS_WFxT)) + vcpu->arch.flags |= KVM_ARM64_WFIT; kvm_vcpu_wfi(vcpu); } From a3fb59651449d8bd4dc4ed5413888819932c740b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:51 +0100 Subject: [PATCH 123/319] KVM: arm64: Offer early resume for non-blocking WFxT instructions For WFxT instructions used with very small delays, it is not unlikely that the deadline is already expired by the time we reach the WFx handling code. Check for this condition as soon as possible, and return to the guest immediately if we can. Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220419182755.601427-7-maz@kernel.org --- arch/arm64/kvm/handle_exit.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 4260f2cd1971..7726b01dc09a 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -80,12 +80,14 @@ static int handle_no_fpsimd(struct kvm_vcpu *vcpu) * * @vcpu: the vcpu pointer * - * WFE: Yield the CPU and come back to this vcpu when the scheduler + * WFE[T]: Yield the CPU and come back to this vcpu when the scheduler * decides to. * WFI: Simply call kvm_vcpu_halt(), which will halt execution of * world-switches and schedule other host processes until there is an * incoming IRQ or FIQ to the VM. * WFIT: Same as WFI, with a timed wakeup implemented as a background timer + * + * WF{I,E}T can immediately return if the deadline has already expired. */ static int kvm_handle_wfx(struct kvm_vcpu *vcpu) { @@ -94,15 +96,35 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu) if (esr & ESR_ELx_WFx_ISS_WFE) { trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true); vcpu->stat.wfe_exit_stat++; - kvm_vcpu_on_spin(vcpu, vcpu_mode_priv(vcpu)); } else { trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false); vcpu->stat.wfi_exit_stat++; - if ((esr & (ESR_ELx_WFx_ISS_RV | ESR_ELx_WFx_ISS_WFxT)) == (ESR_ELx_WFx_ISS_RV | ESR_ELx_WFx_ISS_WFxT)) - vcpu->arch.flags |= KVM_ARM64_WFIT; - kvm_vcpu_wfi(vcpu); } + if (esr & ESR_ELx_WFx_ISS_WFxT) { + if (esr & ESR_ELx_WFx_ISS_RV) { + u64 val, now; + + now = kvm_arm_timer_get_reg(vcpu, KVM_REG_ARM_TIMER_CNT); + val = vcpu_get_reg(vcpu, kvm_vcpu_sys_get_rt(vcpu)); + + if (now >= val) + goto out; + } else { + /* Treat WFxT as WFx if RN is invalid */ + esr &= ~ESR_ELx_WFx_ISS_WFxT; + } + } + + if (esr & ESR_ELx_WFx_ISS_WFE) { + kvm_vcpu_on_spin(vcpu, vcpu_mode_priv(vcpu)); + } else { + if (esr & ESR_ELx_WFx_ISS_WFxT) + vcpu->arch.flags |= KVM_ARM64_WFIT; + + kvm_vcpu_wfi(vcpu); + } +out: kvm_incr_pc(vcpu); return 1; From 06e0b802583d7bbc075476d90da995ee3e6053d5 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 19 Apr 2022 19:27:52 +0100 Subject: [PATCH 124/319] KVM: arm64: Expose the WFXT feature to guests Plumb in the capability, and expose WFxT to guests when available. Signed-off-by: Marc Zyngier Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/20220419182755.601427-8-maz@kernel.org --- arch/arm64/kernel/cpufeature.c | 11 +++++++++++ arch/arm64/kvm/sys_regs.c | 2 ++ 2 files changed, 13 insertions(+) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index a6db8ca04429..945190ebadd5 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2443,6 +2443,17 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .matches = has_cpuid_feature, .min_field_value = 1, }, + { + .desc = "WFx with timeout", + .capability = ARM64_HAS_WFXT, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .sys_reg = SYS_ID_AA64ISAR2_EL1, + .sign = FTR_UNSIGNED, + .field_pos = ID_AA64ISAR2_WFXT_SHIFT, + .field_width = 4, + .matches = has_cpuid_feature, + .min_field_value = ID_AA64ISAR2_WFXT_SUPPORTED, + }, {}, }; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 7b45c040cc27..cc9a77546cc0 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1144,6 +1144,8 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, if (!vcpu_has_ptrauth(vcpu)) val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_APA3) | ARM64_FEATURE_MASK(ID_AA64ISAR2_GPA3)); + if (!cpus_have_final_cap(ARM64_HAS_WFXT)) + val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_WFXT); break; case SYS_ID_AA64DFR0_EL1: /* Limit debug to ARMv8.0 */ From f8b7d2b4c192118c37ab24c0540d1134dd0104d8 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Apr 2022 14:29:57 -0700 Subject: [PATCH 125/319] ftrace: fix building with SYSCTL=n but DYNAMIC_FTRACE=y MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One can enable dyanmic tracing but disable sysctls. When this is doen we get the compile kernel warning: CC kernel/trace/ftrace.o kernel/trace/ftrace.c:3086:13: warning: ‘ftrace_shutdown_sysctl’ defined but not used [-Wunused-function] 3086 | static void ftrace_shutdown_sysctl(void) | ^~~~~~~~~~~~~~~~~~~~~~ kernel/trace/ftrace.c:3068:13: warning: ‘ftrace_startup_sysctl’ defined but not used [-Wunused-function] 3068 | static void ftrace_startup_sysctl(void) When CONFIG_DYNAMIC_FTRACE=n the ftrace_startup_sysctl() and routines ftrace_shutdown_sysctl() still compiles, so these are actually more just used for when SYSCTL=y. Fix this then by just moving these routines to when sysctls are enabled. Fixes: 7cde53da38a3 ("ftrace: move sysctl_ftrace_enabled to ftrace.c") Reported-by: kernel test robot Acked-by: Steven Rostedt (Google) Signed-off-by: Luis Chamberlain --- kernel/trace/ftrace.c | 71 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index db8d553728b6..d9424fd9a183 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -3065,40 +3065,6 @@ int ftrace_shutdown(struct ftrace_ops *ops, int command) return 0; } -static void ftrace_startup_sysctl(void) -{ - int command; - - if (unlikely(ftrace_disabled)) - return; - - /* Force update next time */ - saved_ftrace_func = NULL; - /* ftrace_start_up is true if we want ftrace running */ - if (ftrace_start_up) { - command = FTRACE_UPDATE_CALLS; - if (ftrace_graph_active) - command |= FTRACE_START_FUNC_RET; - ftrace_startup_enable(command); - } -} - -static void ftrace_shutdown_sysctl(void) -{ - int command; - - if (unlikely(ftrace_disabled)) - return; - - /* ftrace_start_up is true if ftrace is running */ - if (ftrace_start_up) { - command = FTRACE_DISABLE_CALLS; - if (ftrace_graph_active) - command |= FTRACE_STOP_FUNC_RET; - ftrace_run_update_code(command); - } -} - static u64 ftrace_update_time; unsigned long ftrace_update_tot_cnt; unsigned long ftrace_number_of_pages; @@ -7267,9 +7233,6 @@ core_initcall(ftrace_nodyn_init); static inline int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { return 0; } static inline void ftrace_startup_all(int command) { } -# define ftrace_startup_sysctl() do { } while (0) -# define ftrace_shutdown_sysctl() do { } while (0) - static void ftrace_update_trampoline(struct ftrace_ops *ops) { } @@ -7910,6 +7873,40 @@ int unregister_ftrace_function(struct ftrace_ops *ops) EXPORT_SYMBOL_GPL(unregister_ftrace_function); #ifdef CONFIG_SYSCTL +static void ftrace_startup_sysctl(void) +{ + int command; + + if (unlikely(ftrace_disabled)) + return; + + /* Force update next time */ + saved_ftrace_func = NULL; + /* ftrace_start_up is true if we want ftrace running */ + if (ftrace_start_up) { + command = FTRACE_UPDATE_CALLS; + if (ftrace_graph_active) + command |= FTRACE_START_FUNC_RET; + ftrace_startup_enable(command); + } +} + +static void ftrace_shutdown_sysctl(void) +{ + int command; + + if (unlikely(ftrace_disabled)) + return; + + /* ftrace_start_up is true if ftrace is running */ + if (ftrace_start_up) { + command = FTRACE_DISABLE_CALLS; + if (ftrace_graph_active) + command |= FTRACE_STOP_FUNC_RET; + ftrace_run_update_code(command); + } +} + static bool is_permanent_ops_registered(void) { struct ftrace_ops *op; From 988f11e046401f8561c4afefa506a50f0203de40 Mon Sep 17 00:00:00 2001 From: liaohua Date: Thu, 7 Apr 2022 15:29:48 +0800 Subject: [PATCH 126/319] latencytop: move sysctl to its own file This moves latencytop sysctl to kernel/latencytop.c Signed-off-by: liaohua Signed-off-by: Luis Chamberlain --- include/linux/latencytop.h | 3 --- kernel/latencytop.c | 41 +++++++++++++++++++++++++++----------- kernel/sysctl.c | 10 ---------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/include/linux/latencytop.h b/include/linux/latencytop.h index abe3d95f795b..84f1053cf2a8 100644 --- a/include/linux/latencytop.h +++ b/include/linux/latencytop.h @@ -38,9 +38,6 @@ account_scheduler_latency(struct task_struct *task, int usecs, int inter) void clear_tsk_latency_tracing(struct task_struct *p); -int sysctl_latencytop(struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); - #else static inline void diff --git a/kernel/latencytop.c b/kernel/latencytop.c index 166d7bf49666..76166df011a4 100644 --- a/kernel/latencytop.c +++ b/kernel/latencytop.c @@ -55,6 +55,7 @@ #include #include #include +#include static DEFINE_RAW_SPINLOCK(latency_lock); @@ -63,6 +64,31 @@ static struct latency_record latency_record[MAXLR]; int latencytop_enabled; +#ifdef CONFIG_SYSCTL +static int sysctl_latencytop(struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + int err; + + err = proc_dointvec(table, write, buffer, lenp, ppos); + if (latencytop_enabled) + force_schedstat_enabled(); + + return err; +} + +static struct ctl_table latencytop_sysctl[] = { + { + .procname = "latencytop", + .data = &latencytop_enabled, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = sysctl_latencytop, + }, + {} +}; +#endif + void clear_tsk_latency_tracing(struct task_struct *p) { unsigned long flags; @@ -266,18 +292,9 @@ static const struct proc_ops lstats_proc_ops = { static int __init init_lstats_procfs(void) { proc_create("latency_stats", 0644, NULL, &lstats_proc_ops); +#ifdef CONFIG_SYSCTL + register_sysctl_init("kernel", latencytop_sysctl); +#endif return 0; } - -int sysctl_latencytop(struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - int err; - - err = proc_dointvec(table, write, buffer, lenp, ppos); - if (latencytop_enabled) - force_schedstat_enabled(); - - return err; -} device_initcall(init_lstats_procfs); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index c0fdf465a93d..b60345cbadf0 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -64,7 +64,6 @@ #include #include #include -#include #include #include "../lib/kstrtox.h" @@ -1623,15 +1622,6 @@ static struct ctl_table kern_table[] = { .extra1 = SYSCTL_NEG_ONE, .extra2 = SYSCTL_ONE, }, -#endif -#ifdef CONFIG_LATENCYTOP - { - .procname = "latencytop", - .data = &latencytop_enabled, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = sysctl_latencytop, - }, #endif { .procname = "print-fatal-signals", From 3c6a4cba3138d1aeeb8fd917178c6578b9b8ae29 Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Fri, 15 Apr 2022 15:08:02 -0700 Subject: [PATCH 127/319] mm: fix unused variable kernel warning when SYSCTL=n When CONFIG_SYSCTL=n the variable dirty_bytes_min which is just used as a minimum to a proc handler is not used. So just move this under the ifdef for CONFIG_SYSCTL. Fixes: aa779e510219 ("mm: move page-writeback sysctls to their own file") Reported-by: kernel test robot Acked-by: Andrew Morton Signed-off-by: Luis Chamberlain --- mm/page-writeback.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 438762173a59..4f51d0ac5043 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -89,9 +89,6 @@ static int vm_highmem_is_dirtyable; */ static int vm_dirty_ratio = 20; -/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */ -static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE; - /* * vm_dirty_bytes starts at 0 (disabled) so that it is a function of * vm_dirty_ratio * the amount of dirtyable memory @@ -2077,6 +2074,10 @@ static int page_writeback_cpu_online(unsigned int cpu) } #ifdef CONFIG_SYSCTL + +/* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */ +static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE; + static struct ctl_table vm_page_writeback_sysctls[] = { { .procname = "dirty_background_ratio", From cb55f27ac9326bfe3bcaaf0adb498778d64602b6 Mon Sep 17 00:00:00 2001 From: Meng Tang Date: Mon, 11 Apr 2022 13:12:05 +0800 Subject: [PATCH 128/319] fs/proc: Introduce list_for_each_table_entry for proc sysctl Use the list_for_each_table_entry macro to optimize the scenario of traverse ctl_table. This make the code neater and easier to understand. Suggested-by: Davidlohr Bueso Signed-off-by: Meng Tang [updated the sysctl_check_table() hunk due to some changes upstream] Signed-off-by: Luis Chamberlain --- fs/proc/proc_sysctl.c | 86 ++++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 7d9cfc730bd4..8901fe0b7e2a 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -19,6 +19,9 @@ #include #include "internal.h" +#define list_for_each_table_entry(entry, table) \ + for ((entry) = (table); (entry)->procname; (entry)++) + static const struct dentry_operations proc_sys_dentry_operations; static const struct file_operations proc_sys_file_operations; static const struct inode_operations proc_sys_inode_operations; @@ -215,15 +218,19 @@ static void init_header(struct ctl_table_header *head, INIT_HLIST_HEAD(&head->inodes); if (node) { struct ctl_table *entry; - for (entry = table; entry->procname; entry++, node++) + + list_for_each_table_entry(entry, table) { node->header = head; + node++; + } } } static void erase_header(struct ctl_table_header *head) { struct ctl_table *entry; - for (entry = head->ctl_table; entry->procname; entry++) + + list_for_each_table_entry(entry, head->ctl_table) erase_entry(head, entry); } @@ -248,7 +255,7 @@ static int insert_header(struct ctl_dir *dir, struct ctl_table_header *header) err = insert_links(header); if (err) goto fail_links; - for (entry = header->ctl_table; entry->procname; entry++) { + list_for_each_table_entry(entry, header->ctl_table) { err = insert_entry(header, entry); if (err) goto fail; @@ -1130,35 +1137,36 @@ static int sysctl_check_table_array(const char *path, struct ctl_table *table) static int sysctl_check_table(const char *path, struct ctl_table *table) { + struct ctl_table *entry; int err = 0; - for (; table->procname; table++) { - if (table->child) - err |= sysctl_err(path, table, "Not a file"); + list_for_each_table_entry(entry, table) { + if (entry->child) + err |= sysctl_err(path, entry, "Not a file"); - if ((table->proc_handler == proc_dostring) || - (table->proc_handler == proc_dointvec) || - (table->proc_handler == proc_douintvec) || - (table->proc_handler == proc_douintvec_minmax) || - (table->proc_handler == proc_dointvec_minmax) || - (table->proc_handler == proc_dou8vec_minmax) || - (table->proc_handler == proc_dointvec_jiffies) || - (table->proc_handler == proc_dointvec_userhz_jiffies) || - (table->proc_handler == proc_dointvec_ms_jiffies) || - (table->proc_handler == proc_doulongvec_minmax) || - (table->proc_handler == proc_doulongvec_ms_jiffies_minmax)) { - if (!table->data) - err |= sysctl_err(path, table, "No data"); - if (!table->maxlen) - err |= sysctl_err(path, table, "No maxlen"); + if ((entry->proc_handler == proc_dostring) || + (entry->proc_handler == proc_dointvec) || + (entry->proc_handler == proc_douintvec) || + (entry->proc_handler == proc_douintvec_minmax) || + (entry->proc_handler == proc_dointvec_minmax) || + (entry->proc_handler == proc_dou8vec_minmax) || + (entry->proc_handler == proc_dointvec_jiffies) || + (entry->proc_handler == proc_dointvec_userhz_jiffies) || + (entry->proc_handler == proc_dointvec_ms_jiffies) || + (entry->proc_handler == proc_doulongvec_minmax) || + (entry->proc_handler == proc_doulongvec_ms_jiffies_minmax)) { + if (!entry->data) + err |= sysctl_err(path, entry, "No data"); + if (!entry->maxlen) + err |= sysctl_err(path, entry, "No maxlen"); else - err |= sysctl_check_table_array(path, table); + err |= sysctl_check_table_array(path, entry); } - if (!table->proc_handler) - err |= sysctl_err(path, table, "No proc_handler"); + if (!entry->proc_handler) + err |= sysctl_err(path, entry, "No proc_handler"); - if ((table->mode & (S_IRUGO|S_IWUGO)) != table->mode) - err |= sysctl_err(path, table, "bogus .mode 0%o", - table->mode); + if ((entry->mode & (S_IRUGO|S_IWUGO)) != entry->mode) + err |= sysctl_err(path, entry, "bogus .mode 0%o", + entry->mode); } return err; } @@ -1174,7 +1182,7 @@ static struct ctl_table_header *new_links(struct ctl_dir *dir, struct ctl_table name_bytes = 0; nr_entries = 0; - for (entry = table; entry->procname; entry++) { + list_for_each_table_entry(entry, table) { nr_entries++; name_bytes += strlen(entry->procname) + 1; } @@ -1191,14 +1199,16 @@ static struct ctl_table_header *new_links(struct ctl_dir *dir, struct ctl_table node = (struct ctl_node *)(links + 1); link_table = (struct ctl_table *)(node + nr_entries); link_name = (char *)&link_table[nr_entries + 1]; + link = link_table; - for (link = link_table, entry = table; entry->procname; link++, entry++) { + list_for_each_table_entry(entry, table) { int len = strlen(entry->procname) + 1; memcpy(link_name, entry->procname, len); link->procname = link_name; link->mode = S_IFLNK|S_IRWXUGO; link->data = link_root; link_name += len; + link++; } init_header(links, dir->header.root, dir->header.set, node, link_table); links->nreg = nr_entries; @@ -1213,7 +1223,7 @@ static bool get_links(struct ctl_dir *dir, struct ctl_table *entry, *link; /* Are there links available for every entry in table? */ - for (entry = table; entry->procname; entry++) { + list_for_each_table_entry(entry, table) { const char *procname = entry->procname; link = find_entry(&head, dir, procname, strlen(procname)); if (!link) @@ -1226,7 +1236,7 @@ static bool get_links(struct ctl_dir *dir, } /* The checks passed. Increase the registration count on the links */ - for (entry = table; entry->procname; entry++) { + list_for_each_table_entry(entry, table) { const char *procname = entry->procname; link = find_entry(&head, dir, procname, strlen(procname)); head->nreg++; @@ -1329,7 +1339,7 @@ struct ctl_table_header *__register_sysctl_table( struct ctl_node *node; int nr_entries = 0; - for (entry = table; entry->procname; entry++) + list_for_each_table_entry(entry, table) nr_entries++; header = kzalloc(sizeof(struct ctl_table_header) + @@ -1456,7 +1466,7 @@ static int count_subheaders(struct ctl_table *table) if (!table || !table->procname) return 1; - for (entry = table; entry->procname; entry++) { + list_for_each_table_entry(entry, table) { if (entry->child) nr_subheaders += count_subheaders(entry->child); else @@ -1475,7 +1485,7 @@ static int register_leaf_sysctl_tables(const char *path, char *pos, int nr_dirs = 0; int err = -ENOMEM; - for (entry = table; entry->procname; entry++) { + list_for_each_table_entry(entry, table) { if (entry->child) nr_dirs++; else @@ -1492,7 +1502,9 @@ static int register_leaf_sysctl_tables(const char *path, char *pos, goto out; ctl_table_arg = files; - for (new = files, entry = table; entry->procname; entry++) { + new = files; + + list_for_each_table_entry(entry, table) { if (entry->child) continue; *new = *entry; @@ -1516,7 +1528,7 @@ static int register_leaf_sysctl_tables(const char *path, char *pos, } /* Recurse into the subdirectories. */ - for (entry = table; entry->procname; entry++) { + list_for_each_table_entry(entry, table) { char *child_pos; if (!entry->child) @@ -1670,7 +1682,7 @@ static void put_links(struct ctl_table_header *header) if (IS_ERR(core_parent)) return; - for (entry = header->ctl_table; entry->procname; entry++) { + list_for_each_table_entry(entry, header->ctl_table) { struct ctl_table_header *link_head; struct ctl_table *link; const char *name = entry->procname; From 8fd7c2144d1292f15c901211750dee021ed5079a Mon Sep 17 00:00:00 2001 From: Luis Chamberlain Date: Thu, 21 Apr 2022 11:38:43 -0700 Subject: [PATCH 129/319] ftrace: fix building with SYSCTL=y but DYNAMIC_FTRACE=n Ok so hopefully this is the last of it. 0day picked up a build failure [0] when SYSCTL=y but DYNAMIC_FTRACE=n. This can be fixed by just declaring an empty routine for the calls moved just recently. [0] https://lkml.kernel.org/r/202204161203.6dSlgKJX-lkp@intel.com Reported-by: kernel test robot Fixes: f8b7d2b4c192 ("ftrace: fix building with SYSCTL=n but DYNAMIC_FTRACE=y") Acked-by: Steven Rostedt (Google) Signed-off-by: Luis Chamberlain --- kernel/trace/ftrace.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index d9424fd9a183..1f89039f0feb 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -7873,6 +7873,8 @@ int unregister_ftrace_function(struct ftrace_ops *ops) EXPORT_SYMBOL_GPL(unregister_ftrace_function); #ifdef CONFIG_SYSCTL + +#ifdef CONFIG_DYNAMIC_FTRACE static void ftrace_startup_sysctl(void) { int command; @@ -7906,6 +7908,10 @@ static void ftrace_shutdown_sysctl(void) ftrace_run_update_code(command); } } +#else +# define ftrace_startup_sysctl() do { } while (0) +# define ftrace_shutdown_sysctl() do { } while (0) +#endif /* CONFIG_DYNAMIC_FTRACE */ static bool is_permanent_ops_registered(void) { From 3c938cc5cebcbd2291fe97f523c0705a2c24c77d Mon Sep 17 00:00:00 2001 From: Schspa Shi Date: Tue, 19 Apr 2022 09:28:10 +0800 Subject: [PATCH 130/319] gpio: use raw spinlock for gpio chip shadowed data In case of PREEMPT_RT, there is a raw_spinlock -> spinlock dependency as the lockdep report shows. __irq_set_handler irq_get_desc_buslock __irq_get_desc_lock raw_spin_lock_irqsave(&desc->lock, *flags); // raw spinlock get here __irq_do_set_handler mask_ack_irq dwapb_irq_ack spin_lock_irqsave(&gc->bgpio_lock, flags); // sleep able spinlock irq_put_desc_busunlock Replace with a raw lock to avoid BUGs. This lock is only used to access registers, and It's safe to replace with the raw lock without bad influence. [ 15.090359][ T1] ============================= [ 15.090365][ T1] [ BUG: Invalid wait context ] [ 15.090373][ T1] 5.10.59-rt52-00983-g186a6841c682-dirty #3 Not tainted [ 15.090386][ T1] ----------------------------- [ 15.090392][ T1] swapper/0/1 is trying to lock: [ 15.090402][ T1] 70ff00018507c188 (&gc->bgpio_lock){....}-{3:3}, at: _raw_spin_lock_irqsave+0x1c/0x28 [ 15.090470][ T1] other info that might help us debug this: [ 15.090477][ T1] context-{5:5} [ 15.090485][ T1] 3 locks held by swapper/0/1: [ 15.090497][ T1] #0: c2ff0001816de1a0 (&dev->mutex){....}-{4:4}, at: __device_driver_lock+0x98/0x104 [ 15.090553][ T1] #1: ffff90001485b4b8 (irq_domain_mutex){+.+.}-{4:4}, at: irq_domain_associate+0xbc/0x6d4 [ 15.090606][ T1] #2: 4bff000185d7a8e0 (lock_class){....}-{2:2}, at: _raw_spin_lock_irqsave+0x1c/0x28 [ 15.090654][ T1] stack backtrace: [ 15.090661][ T1] CPU: 4 PID: 1 Comm: swapper/0 Not tainted 5.10.59-rt52-00983-g186a6841c682-dirty #3 [ 15.090682][ T1] Hardware name: Horizon Robotics Journey 5 DVB (DT) [ 15.090692][ T1] Call trace: ...... [ 15.090811][ T1] _raw_spin_lock_irqsave+0x1c/0x28 [ 15.090828][ T1] dwapb_irq_ack+0xb4/0x300 [ 15.090846][ T1] __irq_do_set_handler+0x494/0xb2c [ 15.090864][ T1] __irq_set_handler+0x74/0x114 [ 15.090881][ T1] irq_set_chip_and_handler_name+0x44/0x58 [ 15.090900][ T1] gpiochip_irq_map+0x210/0x644 Signed-off-by: Schspa Shi Reviewed-by: Andy Shevchenko Acked-by: Linus Walleij Acked-by: Doug Berger Acked-by: Serge Semin Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-amdpt.c | 10 +++---- drivers/gpio/gpio-brcmstb.c | 12 ++++---- drivers/gpio/gpio-cadence.c | 12 ++++---- drivers/gpio/gpio-dwapb.c | 36 +++++++++++------------ drivers/gpio/gpio-grgpio.c | 30 +++++++++---------- drivers/gpio/gpio-hlwd.c | 18 ++++++------ drivers/gpio/gpio-idt3243x.c | 12 ++++---- drivers/gpio/gpio-ixp4xx.c | 4 +-- drivers/gpio/gpio-loongson1.c | 8 ++--- drivers/gpio/gpio-menz127.c | 8 ++--- drivers/gpio/gpio-mlxbf2.c | 18 ++++++------ drivers/gpio/gpio-mmio.c | 22 +++++++------- drivers/gpio/gpio-sifive.c | 12 ++++---- drivers/gpio/gpio-tb10x.c | 4 +-- drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c | 8 ++--- include/linux/gpio/driver.h | 2 +- 16 files changed, 108 insertions(+), 108 deletions(-) diff --git a/drivers/gpio/gpio-amdpt.c b/drivers/gpio/gpio-amdpt.c index 8cfb353c3abb..07c6d090058d 100644 --- a/drivers/gpio/gpio-amdpt.c +++ b/drivers/gpio/gpio-amdpt.c @@ -36,19 +36,19 @@ static int pt_gpio_request(struct gpio_chip *gc, unsigned offset) dev_dbg(gc->parent, "pt_gpio_request offset=%x\n", offset); - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); using_pins = readl(pt_gpio->reg_base + PT_SYNC_REG); if (using_pins & BIT(offset)) { dev_warn(gc->parent, "PT GPIO pin %x reconfigured\n", offset); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return -EINVAL; } writel(using_pins | BIT(offset), pt_gpio->reg_base + PT_SYNC_REG); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } @@ -59,13 +59,13 @@ static void pt_gpio_free(struct gpio_chip *gc, unsigned offset) unsigned long flags; u32 using_pins; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); using_pins = readl(pt_gpio->reg_base + PT_SYNC_REG); using_pins &= ~BIT(offset); writel(using_pins, pt_gpio->reg_base + PT_SYNC_REG); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); dev_dbg(gc->parent, "pt_gpio_free offset=%x\n", offset); } diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c index 74ef89248867..6b7439b44690 100644 --- a/drivers/gpio/gpio-brcmstb.c +++ b/drivers/gpio/gpio-brcmstb.c @@ -92,9 +92,9 @@ brcmstb_gpio_get_active_irqs(struct brcmstb_gpio_bank *bank) unsigned long status; unsigned long flags; - spin_lock_irqsave(&bank->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&bank->gc.bgpio_lock, flags); status = __brcmstb_gpio_get_active_irqs(bank); - spin_unlock_irqrestore(&bank->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&bank->gc.bgpio_lock, flags); return status; } @@ -114,14 +114,14 @@ static void brcmstb_gpio_set_imask(struct brcmstb_gpio_bank *bank, u32 imask; unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); imask = gc->read_reg(priv->reg_base + GIO_MASK(bank->id)); if (enable) imask |= mask; else imask &= ~mask; gc->write_reg(priv->reg_base + GIO_MASK(bank->id), imask); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static int brcmstb_gpio_to_irq(struct gpio_chip *gc, unsigned offset) @@ -204,7 +204,7 @@ static int brcmstb_gpio_irq_set_type(struct irq_data *d, unsigned int type) return -EINVAL; } - spin_lock_irqsave(&bank->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&bank->gc.bgpio_lock, flags); iedge_config = bank->gc.read_reg(priv->reg_base + GIO_EC(bank->id)) & ~mask; @@ -220,7 +220,7 @@ static int brcmstb_gpio_irq_set_type(struct irq_data *d, unsigned int type) bank->gc.write_reg(priv->reg_base + GIO_LEVEL(bank->id), ilevel | level); - spin_unlock_irqrestore(&bank->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&bank->gc.bgpio_lock, flags); return 0; } diff --git a/drivers/gpio/gpio-cadence.c b/drivers/gpio/gpio-cadence.c index 562f8f7e7d1f..137aea49ba02 100644 --- a/drivers/gpio/gpio-cadence.c +++ b/drivers/gpio/gpio-cadence.c @@ -41,12 +41,12 @@ static int cdns_gpio_request(struct gpio_chip *chip, unsigned int offset) struct cdns_gpio_chip *cgpio = gpiochip_get_data(chip); unsigned long flags; - spin_lock_irqsave(&chip->bgpio_lock, flags); + raw_spin_lock_irqsave(&chip->bgpio_lock, flags); iowrite32(ioread32(cgpio->regs + CDNS_GPIO_BYPASS_MODE) & ~BIT(offset), cgpio->regs + CDNS_GPIO_BYPASS_MODE); - spin_unlock_irqrestore(&chip->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&chip->bgpio_lock, flags); return 0; } @@ -55,13 +55,13 @@ static void cdns_gpio_free(struct gpio_chip *chip, unsigned int offset) struct cdns_gpio_chip *cgpio = gpiochip_get_data(chip); unsigned long flags; - spin_lock_irqsave(&chip->bgpio_lock, flags); + raw_spin_lock_irqsave(&chip->bgpio_lock, flags); iowrite32(ioread32(cgpio->regs + CDNS_GPIO_BYPASS_MODE) | (BIT(offset) & cgpio->bypass_orig), cgpio->regs + CDNS_GPIO_BYPASS_MODE); - spin_unlock_irqrestore(&chip->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&chip->bgpio_lock, flags); } static void cdns_gpio_irq_mask(struct irq_data *d) @@ -90,7 +90,7 @@ static int cdns_gpio_irq_set_type(struct irq_data *d, unsigned int type) u32 mask = BIT(d->hwirq); int ret = 0; - spin_lock_irqsave(&chip->bgpio_lock, flags); + raw_spin_lock_irqsave(&chip->bgpio_lock, flags); int_value = ioread32(cgpio->regs + CDNS_GPIO_IRQ_VALUE) & ~mask; int_type = ioread32(cgpio->regs + CDNS_GPIO_IRQ_TYPE) & ~mask; @@ -115,7 +115,7 @@ static int cdns_gpio_irq_set_type(struct irq_data *d, unsigned int type) iowrite32(int_type, cgpio->regs + CDNS_GPIO_IRQ_TYPE); err_irq_type: - spin_unlock_irqrestore(&chip->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&chip->bgpio_lock, flags); return ret; } diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c index b0f3aca61974..7130195da48d 100644 --- a/drivers/gpio/gpio-dwapb.c +++ b/drivers/gpio/gpio-dwapb.c @@ -243,9 +243,9 @@ static void dwapb_irq_ack(struct irq_data *d) u32 val = BIT(irqd_to_hwirq(d)); unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); dwapb_write(gpio, GPIO_PORTA_EOI, val); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void dwapb_irq_mask(struct irq_data *d) @@ -255,10 +255,10 @@ static void dwapb_irq_mask(struct irq_data *d) unsigned long flags; u32 val; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val = dwapb_read(gpio, GPIO_INTMASK) | BIT(irqd_to_hwirq(d)); dwapb_write(gpio, GPIO_INTMASK, val); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void dwapb_irq_unmask(struct irq_data *d) @@ -268,10 +268,10 @@ static void dwapb_irq_unmask(struct irq_data *d) unsigned long flags; u32 val; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val = dwapb_read(gpio, GPIO_INTMASK) & ~BIT(irqd_to_hwirq(d)); dwapb_write(gpio, GPIO_INTMASK, val); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void dwapb_irq_enable(struct irq_data *d) @@ -281,11 +281,11 @@ static void dwapb_irq_enable(struct irq_data *d) unsigned long flags; u32 val; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val = dwapb_read(gpio, GPIO_INTEN); val |= BIT(irqd_to_hwirq(d)); dwapb_write(gpio, GPIO_INTEN, val); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void dwapb_irq_disable(struct irq_data *d) @@ -295,11 +295,11 @@ static void dwapb_irq_disable(struct irq_data *d) unsigned long flags; u32 val; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val = dwapb_read(gpio, GPIO_INTEN); val &= ~BIT(irqd_to_hwirq(d)); dwapb_write(gpio, GPIO_INTEN, val); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static int dwapb_irq_set_type(struct irq_data *d, u32 type) @@ -309,7 +309,7 @@ static int dwapb_irq_set_type(struct irq_data *d, u32 type) irq_hw_number_t bit = irqd_to_hwirq(d); unsigned long level, polarity, flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); level = dwapb_read(gpio, GPIO_INTTYPE_LEVEL); polarity = dwapb_read(gpio, GPIO_INT_POLARITY); @@ -344,7 +344,7 @@ static int dwapb_irq_set_type(struct irq_data *d, u32 type) dwapb_write(gpio, GPIO_INTTYPE_LEVEL, level); if (type != IRQ_TYPE_EDGE_BOTH) dwapb_write(gpio, GPIO_INT_POLARITY, polarity); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } @@ -374,7 +374,7 @@ static int dwapb_gpio_set_debounce(struct gpio_chip *gc, unsigned long flags, val_deb; unsigned long mask = BIT(offset); - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val_deb = dwapb_read(gpio, GPIO_PORTA_DEBOUNCE); if (debounce) @@ -383,7 +383,7 @@ static int dwapb_gpio_set_debounce(struct gpio_chip *gc, val_deb &= ~mask; dwapb_write(gpio, GPIO_PORTA_DEBOUNCE, val_deb); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } @@ -738,7 +738,7 @@ static int dwapb_gpio_suspend(struct device *dev) unsigned long flags; int i; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); for (i = 0; i < gpio->nr_ports; i++) { unsigned int offset; unsigned int idx = gpio->ports[i].idx; @@ -765,7 +765,7 @@ static int dwapb_gpio_suspend(struct device *dev) dwapb_write(gpio, GPIO_INTMASK, ~ctx->wake_en); } } - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); clk_bulk_disable_unprepare(DWAPB_NR_CLOCKS, gpio->clks); @@ -785,7 +785,7 @@ static int dwapb_gpio_resume(struct device *dev) return err; } - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); for (i = 0; i < gpio->nr_ports; i++) { unsigned int offset; unsigned int idx = gpio->ports[i].idx; @@ -812,7 +812,7 @@ static int dwapb_gpio_resume(struct device *dev) dwapb_write(gpio, GPIO_PORTA_EOI, 0xffffffff); } } - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } diff --git a/drivers/gpio/gpio-grgpio.c b/drivers/gpio/gpio-grgpio.c index 23d447e17a67..df563616f943 100644 --- a/drivers/gpio/gpio-grgpio.c +++ b/drivers/gpio/gpio-grgpio.c @@ -145,7 +145,7 @@ static int grgpio_irq_set_type(struct irq_data *d, unsigned int type) return -EINVAL; } - spin_lock_irqsave(&priv->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&priv->gc.bgpio_lock, flags); ipol = priv->gc.read_reg(priv->regs + GRGPIO_IPOL) & ~mask; iedge = priv->gc.read_reg(priv->regs + GRGPIO_IEDGE) & ~mask; @@ -153,7 +153,7 @@ static int grgpio_irq_set_type(struct irq_data *d, unsigned int type) priv->gc.write_reg(priv->regs + GRGPIO_IPOL, ipol | pol); priv->gc.write_reg(priv->regs + GRGPIO_IEDGE, iedge | edge); - spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); return 0; } @@ -164,11 +164,11 @@ static void grgpio_irq_mask(struct irq_data *d) int offset = d->hwirq; unsigned long flags; - spin_lock_irqsave(&priv->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&priv->gc.bgpio_lock, flags); grgpio_set_imask(priv, offset, 0); - spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); } static void grgpio_irq_unmask(struct irq_data *d) @@ -177,11 +177,11 @@ static void grgpio_irq_unmask(struct irq_data *d) int offset = d->hwirq; unsigned long flags; - spin_lock_irqsave(&priv->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&priv->gc.bgpio_lock, flags); grgpio_set_imask(priv, offset, 1); - spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); } static struct irq_chip grgpio_irq_chip = { @@ -199,7 +199,7 @@ static irqreturn_t grgpio_irq_handler(int irq, void *dev) int i; int match = 0; - spin_lock_irqsave(&priv->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&priv->gc.bgpio_lock, flags); /* * For each gpio line, call its interrupt handler if it its underlying @@ -215,7 +215,7 @@ static irqreturn_t grgpio_irq_handler(int irq, void *dev) } } - spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); if (!match) dev_warn(priv->dev, "No gpio line matched irq %d\n", irq); @@ -247,13 +247,13 @@ static int grgpio_irq_map(struct irq_domain *d, unsigned int irq, dev_dbg(priv->dev, "Mapping irq %d for gpio line %d\n", irq, offset); - spin_lock_irqsave(&priv->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&priv->gc.bgpio_lock, flags); /* Request underlying irq if not already requested */ lirq->irq = irq; uirq = &priv->uirqs[lirq->index]; if (uirq->refcnt == 0) { - spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); ret = request_irq(uirq->uirq, grgpio_irq_handler, 0, dev_name(priv->dev), priv); if (ret) { @@ -262,11 +262,11 @@ static int grgpio_irq_map(struct irq_domain *d, unsigned int irq, uirq->uirq); return ret; } - spin_lock_irqsave(&priv->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&priv->gc.bgpio_lock, flags); } uirq->refcnt++; - spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); /* Setup irq */ irq_set_chip_data(irq, priv); @@ -290,7 +290,7 @@ static void grgpio_irq_unmap(struct irq_domain *d, unsigned int irq) irq_set_chip_and_handler(irq, NULL, NULL); irq_set_chip_data(irq, NULL); - spin_lock_irqsave(&priv->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&priv->gc.bgpio_lock, flags); /* Free underlying irq if last user unmapped */ index = -1; @@ -309,13 +309,13 @@ static void grgpio_irq_unmap(struct irq_domain *d, unsigned int irq) uirq = &priv->uirqs[lirq->index]; uirq->refcnt--; if (uirq->refcnt == 0) { - spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); free_irq(uirq->uirq, priv); return; } } - spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&priv->gc.bgpio_lock, flags); } static const struct irq_domain_ops grgpio_irq_domain_ops = { diff --git a/drivers/gpio/gpio-hlwd.c b/drivers/gpio/gpio-hlwd.c index 641719a96a1a..4e13e937f832 100644 --- a/drivers/gpio/gpio-hlwd.c +++ b/drivers/gpio/gpio-hlwd.c @@ -65,7 +65,7 @@ static void hlwd_gpio_irqhandler(struct irq_desc *desc) int hwirq; u32 emulated_pending; - spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags); pending = ioread32be(hlwd->regs + HW_GPIOB_INTFLAG); pending &= ioread32be(hlwd->regs + HW_GPIOB_INTMASK); @@ -93,7 +93,7 @@ static void hlwd_gpio_irqhandler(struct irq_desc *desc) /* Mark emulated interrupts as pending */ pending |= rising | falling; } - spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); chained_irq_enter(chip, desc); @@ -118,11 +118,11 @@ static void hlwd_gpio_irq_mask(struct irq_data *data) unsigned long flags; u32 mask; - spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags); mask = ioread32be(hlwd->regs + HW_GPIOB_INTMASK); mask &= ~BIT(data->hwirq); iowrite32be(mask, hlwd->regs + HW_GPIOB_INTMASK); - spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); } static void hlwd_gpio_irq_unmask(struct irq_data *data) @@ -132,11 +132,11 @@ static void hlwd_gpio_irq_unmask(struct irq_data *data) unsigned long flags; u32 mask; - spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags); mask = ioread32be(hlwd->regs + HW_GPIOB_INTMASK); mask |= BIT(data->hwirq); iowrite32be(mask, hlwd->regs + HW_GPIOB_INTMASK); - spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); } static void hlwd_gpio_irq_enable(struct irq_data *data) @@ -173,7 +173,7 @@ static int hlwd_gpio_irq_set_type(struct irq_data *data, unsigned int flow_type) unsigned long flags; u32 level; - spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_lock_irqsave(&hlwd->gpioc.bgpio_lock, flags); hlwd->edge_emulation &= ~BIT(data->hwirq); @@ -194,11 +194,11 @@ static int hlwd_gpio_irq_set_type(struct irq_data *data, unsigned int flow_type) hlwd_gpio_irq_setup_emulation(hlwd, data->hwirq, flow_type); break; default: - spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); return -EINVAL; } - spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&hlwd->gpioc.bgpio_lock, flags); return 0; } diff --git a/drivers/gpio/gpio-idt3243x.c b/drivers/gpio/gpio-idt3243x.c index 52b8b72ded77..1cafdf46f875 100644 --- a/drivers/gpio/gpio-idt3243x.c +++ b/drivers/gpio/gpio-idt3243x.c @@ -57,7 +57,7 @@ static int idt_gpio_irq_set_type(struct irq_data *d, unsigned int flow_type) if (sense == IRQ_TYPE_NONE || (sense & IRQ_TYPE_EDGE_BOTH)) return -EINVAL; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); ilevel = readl(ctrl->gpio + IDT_GPIO_ILEVEL); if (sense & IRQ_TYPE_LEVEL_HIGH) @@ -68,7 +68,7 @@ static int idt_gpio_irq_set_type(struct irq_data *d, unsigned int flow_type) writel(ilevel, ctrl->gpio + IDT_GPIO_ILEVEL); irq_set_handler_locked(d, handle_level_irq); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } @@ -86,12 +86,12 @@ static void idt_gpio_mask(struct irq_data *d) struct idt_gpio_ctrl *ctrl = gpiochip_get_data(gc); unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); ctrl->mask_cache |= BIT(d->hwirq); writel(ctrl->mask_cache, ctrl->pic + IDT_PIC_IRQ_MASK); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void idt_gpio_unmask(struct irq_data *d) @@ -100,12 +100,12 @@ static void idt_gpio_unmask(struct irq_data *d) struct idt_gpio_ctrl *ctrl = gpiochip_get_data(gc); unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); ctrl->mask_cache &= ~BIT(d->hwirq); writel(ctrl->mask_cache, ctrl->pic + IDT_PIC_IRQ_MASK); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static int idt_gpio_irq_init_hw(struct gpio_chip *gc) diff --git a/drivers/gpio/gpio-ixp4xx.c b/drivers/gpio/gpio-ixp4xx.c index 1acda980d119..76ab9bd7d84b 100644 --- a/drivers/gpio/gpio-ixp4xx.c +++ b/drivers/gpio/gpio-ixp4xx.c @@ -126,7 +126,7 @@ static int ixp4xx_gpio_irq_set_type(struct irq_data *d, unsigned int type) int_reg = IXP4XX_REG_GPIT1; } - spin_lock_irqsave(&g->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&g->gc.bgpio_lock, flags); /* Clear the style for the appropriate pin */ val = __raw_readl(g->base + int_reg); @@ -145,7 +145,7 @@ static int ixp4xx_gpio_irq_set_type(struct irq_data *d, unsigned int type) val |= BIT(d->hwirq); __raw_writel(val, g->base + IXP4XX_REG_GPOE); - spin_unlock_irqrestore(&g->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&g->gc.bgpio_lock, flags); /* This parent only accept level high (asserted) */ return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH); diff --git a/drivers/gpio/gpio-loongson1.c b/drivers/gpio/gpio-loongson1.c index 1b1ee94eeab4..5d90b3bc5a25 100644 --- a/drivers/gpio/gpio-loongson1.c +++ b/drivers/gpio/gpio-loongson1.c @@ -25,10 +25,10 @@ static int ls1x_gpio_request(struct gpio_chip *gc, unsigned int offset) { unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); __raw_writel(__raw_readl(gpio_reg_base + GPIO_CFG) | BIT(offset), gpio_reg_base + GPIO_CFG); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } @@ -37,10 +37,10 @@ static void ls1x_gpio_free(struct gpio_chip *gc, unsigned int offset) { unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); __raw_writel(__raw_readl(gpio_reg_base + GPIO_CFG) & ~BIT(offset), gpio_reg_base + GPIO_CFG); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static int ls1x_gpio_probe(struct platform_device *pdev) diff --git a/drivers/gpio/gpio-menz127.c b/drivers/gpio/gpio-menz127.c index 1e21c661d79d..a035a9bcb57c 100644 --- a/drivers/gpio/gpio-menz127.c +++ b/drivers/gpio/gpio-menz127.c @@ -64,7 +64,7 @@ static int men_z127_debounce(struct gpio_chip *gc, unsigned gpio, debounce /= 50; } - spin_lock(&gc->bgpio_lock); + raw_spin_lock(&gc->bgpio_lock); db_en = readl(priv->reg_base + MEN_Z127_DBER); @@ -79,7 +79,7 @@ static int men_z127_debounce(struct gpio_chip *gc, unsigned gpio, writel(db_en, priv->reg_base + MEN_Z127_DBER); writel(db_cnt, priv->reg_base + GPIO_TO_DBCNT_REG(gpio)); - spin_unlock(&gc->bgpio_lock); + raw_spin_unlock(&gc->bgpio_lock); return 0; } @@ -91,7 +91,7 @@ static int men_z127_set_single_ended(struct gpio_chip *gc, struct men_z127_gpio *priv = gpiochip_get_data(gc); u32 od_en; - spin_lock(&gc->bgpio_lock); + raw_spin_lock(&gc->bgpio_lock); od_en = readl(priv->reg_base + MEN_Z127_ODER); if (param == PIN_CONFIG_DRIVE_OPEN_DRAIN) @@ -101,7 +101,7 @@ static int men_z127_set_single_ended(struct gpio_chip *gc, od_en &= ~BIT(offset); writel(od_en, priv->reg_base + MEN_Z127_ODER); - spin_unlock(&gc->bgpio_lock); + raw_spin_unlock(&gc->bgpio_lock); return 0; } diff --git a/drivers/gpio/gpio-mlxbf2.c b/drivers/gpio/gpio-mlxbf2.c index 3d89912a05b8..64cb060d9d75 100644 --- a/drivers/gpio/gpio-mlxbf2.c +++ b/drivers/gpio/gpio-mlxbf2.c @@ -131,7 +131,7 @@ static int mlxbf2_gpio_lock_acquire(struct mlxbf2_gpio_context *gs) u32 arm_gpio_lock_val; mutex_lock(yu_arm_gpio_lock_param.lock); - spin_lock(&gs->gc.bgpio_lock); + raw_spin_lock(&gs->gc.bgpio_lock); arm_gpio_lock_val = readl(yu_arm_gpio_lock_param.io); @@ -139,7 +139,7 @@ static int mlxbf2_gpio_lock_acquire(struct mlxbf2_gpio_context *gs) * When lock active bit[31] is set, ModeX is write enabled */ if (YU_LOCK_ACTIVE_BIT(arm_gpio_lock_val)) { - spin_unlock(&gs->gc.bgpio_lock); + raw_spin_unlock(&gs->gc.bgpio_lock); mutex_unlock(yu_arm_gpio_lock_param.lock); return -EINVAL; } @@ -157,7 +157,7 @@ static void mlxbf2_gpio_lock_release(struct mlxbf2_gpio_context *gs) __releases(yu_arm_gpio_lock_param.lock) { writel(YU_ARM_GPIO_LOCK_RELEASE, yu_arm_gpio_lock_param.io); - spin_unlock(&gs->gc.bgpio_lock); + raw_spin_unlock(&gs->gc.bgpio_lock); mutex_unlock(yu_arm_gpio_lock_param.lock); } @@ -237,7 +237,7 @@ static void mlxbf2_gpio_irq_enable(struct irq_data *irqd) unsigned long flags; u32 val; - spin_lock_irqsave(&gs->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&gs->gc.bgpio_lock, flags); val = readl(gs->gpio_io + YU_GPIO_CAUSE_OR_CLRCAUSE); val |= BIT(offset); writel(val, gs->gpio_io + YU_GPIO_CAUSE_OR_CLRCAUSE); @@ -245,7 +245,7 @@ static void mlxbf2_gpio_irq_enable(struct irq_data *irqd) val = readl(gs->gpio_io + YU_GPIO_CAUSE_OR_EVTEN0); val |= BIT(offset); writel(val, gs->gpio_io + YU_GPIO_CAUSE_OR_EVTEN0); - spin_unlock_irqrestore(&gs->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gs->gc.bgpio_lock, flags); } static void mlxbf2_gpio_irq_disable(struct irq_data *irqd) @@ -256,11 +256,11 @@ static void mlxbf2_gpio_irq_disable(struct irq_data *irqd) unsigned long flags; u32 val; - spin_lock_irqsave(&gs->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&gs->gc.bgpio_lock, flags); val = readl(gs->gpio_io + YU_GPIO_CAUSE_OR_EVTEN0); val &= ~BIT(offset); writel(val, gs->gpio_io + YU_GPIO_CAUSE_OR_EVTEN0); - spin_unlock_irqrestore(&gs->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gs->gc.bgpio_lock, flags); } static irqreturn_t mlxbf2_gpio_irq_handler(int irq, void *ptr) @@ -307,7 +307,7 @@ mlxbf2_gpio_irq_set_type(struct irq_data *irqd, unsigned int type) return -EINVAL; } - spin_lock_irqsave(&gs->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&gs->gc.bgpio_lock, flags); if (fall) { val = readl(gs->gpio_io + YU_GPIO_CAUSE_FALL_EN); val |= BIT(offset); @@ -319,7 +319,7 @@ mlxbf2_gpio_irq_set_type(struct irq_data *irqd, unsigned int type) val |= BIT(offset); writel(val, gs->gpio_io + YU_GPIO_CAUSE_RISE_EN); } - spin_unlock_irqrestore(&gs->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gs->gc.bgpio_lock, flags); return 0; } diff --git a/drivers/gpio/gpio-mmio.c b/drivers/gpio/gpio-mmio.c index c335a0309ba3..d9dff3dc92ae 100644 --- a/drivers/gpio/gpio-mmio.c +++ b/drivers/gpio/gpio-mmio.c @@ -220,7 +220,7 @@ static void bgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) unsigned long mask = bgpio_line2mask(gc, gpio); unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); if (val) gc->bgpio_data |= mask; @@ -229,7 +229,7 @@ static void bgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) gc->write_reg(gc->reg_dat, gc->bgpio_data); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void bgpio_set_with_clear(struct gpio_chip *gc, unsigned int gpio, @@ -248,7 +248,7 @@ static void bgpio_set_set(struct gpio_chip *gc, unsigned int gpio, int val) unsigned long mask = bgpio_line2mask(gc, gpio); unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); if (val) gc->bgpio_data |= mask; @@ -257,7 +257,7 @@ static void bgpio_set_set(struct gpio_chip *gc, unsigned int gpio, int val) gc->write_reg(gc->reg_set, gc->bgpio_data); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void bgpio_multiple_get_masks(struct gpio_chip *gc, @@ -286,7 +286,7 @@ static void bgpio_set_multiple_single_reg(struct gpio_chip *gc, unsigned long flags; unsigned long set_mask, clear_mask; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); bgpio_multiple_get_masks(gc, mask, bits, &set_mask, &clear_mask); @@ -295,7 +295,7 @@ static void bgpio_set_multiple_single_reg(struct gpio_chip *gc, gc->write_reg(reg, gc->bgpio_data); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void bgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, @@ -347,7 +347,7 @@ static int bgpio_dir_in(struct gpio_chip *gc, unsigned int gpio) { unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); gc->bgpio_dir &= ~bgpio_line2mask(gc, gpio); @@ -356,7 +356,7 @@ static int bgpio_dir_in(struct gpio_chip *gc, unsigned int gpio) if (gc->reg_dir_out) gc->write_reg(gc->reg_dir_out, gc->bgpio_dir); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); return 0; } @@ -387,7 +387,7 @@ static void bgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) { unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); gc->bgpio_dir |= bgpio_line2mask(gc, gpio); @@ -396,7 +396,7 @@ static void bgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) if (gc->reg_dir_out) gc->write_reg(gc->reg_dir_out, gc->bgpio_dir); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static int bgpio_dir_out_dir_first(struct gpio_chip *gc, unsigned int gpio, @@ -610,7 +610,7 @@ int bgpio_init(struct gpio_chip *gc, struct device *dev, if (gc->bgpio_bits > BITS_PER_LONG) return -EINVAL; - spin_lock_init(&gc->bgpio_lock); + raw_spin_lock_init(&gc->bgpio_lock); gc->parent = dev; gc->label = dev_name(dev); gc->base = -1; diff --git a/drivers/gpio/gpio-sifive.c b/drivers/gpio/gpio-sifive.c index 7d82388b4ab7..03b8c4de2e91 100644 --- a/drivers/gpio/gpio-sifive.c +++ b/drivers/gpio/gpio-sifive.c @@ -44,7 +44,7 @@ static void sifive_gpio_set_ie(struct sifive_gpio *chip, unsigned int offset) unsigned long flags; unsigned int trigger; - spin_lock_irqsave(&chip->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&chip->gc.bgpio_lock, flags); trigger = (chip->irq_state & BIT(offset)) ? chip->trigger[offset] : 0; regmap_update_bits(chip->regs, SIFIVE_GPIO_RISE_IE, BIT(offset), (trigger & IRQ_TYPE_EDGE_RISING) ? BIT(offset) : 0); @@ -54,7 +54,7 @@ static void sifive_gpio_set_ie(struct sifive_gpio *chip, unsigned int offset) (trigger & IRQ_TYPE_LEVEL_HIGH) ? BIT(offset) : 0); regmap_update_bits(chip->regs, SIFIVE_GPIO_LOW_IE, BIT(offset), (trigger & IRQ_TYPE_LEVEL_LOW) ? BIT(offset) : 0); - spin_unlock_irqrestore(&chip->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&chip->gc.bgpio_lock, flags); } static int sifive_gpio_irq_set_type(struct irq_data *d, unsigned int trigger) @@ -84,13 +84,13 @@ static void sifive_gpio_irq_enable(struct irq_data *d) /* Switch to input */ gc->direction_input(gc, offset); - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); /* Clear any sticky pending interrupts */ regmap_write(chip->regs, SIFIVE_GPIO_RISE_IP, bit); regmap_write(chip->regs, SIFIVE_GPIO_FALL_IP, bit); regmap_write(chip->regs, SIFIVE_GPIO_HIGH_IP, bit); regmap_write(chip->regs, SIFIVE_GPIO_LOW_IP, bit); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); /* Enable interrupts */ assign_bit(offset, &chip->irq_state, 1); @@ -116,13 +116,13 @@ static void sifive_gpio_irq_eoi(struct irq_data *d) u32 bit = BIT(offset); unsigned long flags; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); /* Clear all pending interrupts */ regmap_write(chip->regs, SIFIVE_GPIO_RISE_IP, bit); regmap_write(chip->regs, SIFIVE_GPIO_FALL_IP, bit); regmap_write(chip->regs, SIFIVE_GPIO_HIGH_IP, bit); regmap_write(chip->regs, SIFIVE_GPIO_LOW_IP, bit); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); irq_chip_eoi_parent(d); } diff --git a/drivers/gpio/gpio-tb10x.c b/drivers/gpio/gpio-tb10x.c index 718a508d3b2f..de6afa3f9716 100644 --- a/drivers/gpio/gpio-tb10x.c +++ b/drivers/gpio/gpio-tb10x.c @@ -62,14 +62,14 @@ static inline void tb10x_set_bits(struct tb10x_gpio *gpio, unsigned int offs, u32 r; unsigned long flags; - spin_lock_irqsave(&gpio->gc.bgpio_lock, flags); + raw_spin_lock_irqsave(&gpio->gc.bgpio_lock, flags); r = tb10x_reg_read(gpio, offs); r = (r & ~mask) | (val & mask); tb10x_reg_write(gpio, offs, r); - spin_unlock_irqrestore(&gpio->gc.bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gpio->gc.bgpio_lock, flags); } static int tb10x_gpio_to_irq(struct gpio_chip *chip, unsigned offset) diff --git a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c index 9557fac5d11c..b2a0f11a658b 100644 --- a/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c +++ b/drivers/pinctrl/nuvoton/pinctrl-npcm7xx.c @@ -104,12 +104,12 @@ static void npcm_gpio_set(struct gpio_chip *gc, void __iomem *reg, unsigned long flags; unsigned long val; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val = ioread32(reg) | pinmask; iowrite32(val, reg); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void npcm_gpio_clr(struct gpio_chip *gc, void __iomem *reg, @@ -118,12 +118,12 @@ static void npcm_gpio_clr(struct gpio_chip *gc, void __iomem *reg, unsigned long flags; unsigned long val; - spin_lock_irqsave(&gc->bgpio_lock, flags); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); val = ioread32(reg) & ~pinmask; iowrite32(val, reg); - spin_unlock_irqrestore(&gc->bgpio_lock, flags); + raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } static void npcmgpio_dbg_show(struct seq_file *s, struct gpio_chip *chip) diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 98c93510640e..991d374dcf71 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -436,7 +436,7 @@ struct gpio_chip { void __iomem *reg_dir_in; bool bgpio_dir_unreadable; int bgpio_bits; - spinlock_t bgpio_lock; + raw_spinlock_t bgpio_lock; unsigned long bgpio_data; unsigned long bgpio_dir; #endif /* CONFIG_GPIO_GENERIC */ From 6588cb81e546e24f8271987ce0df120305cd4c80 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 25 Apr 2022 20:46:31 +0200 Subject: [PATCH 131/319] dt-bindings: gpio: add common consumer GPIO lines Typical GPIO lines like enable, powerdown, reset or wakeup are not documented as common, which leads to new variations of these (e.g. pwdn-gpios). Add a common schema which serves also as a documentation for preferred naming. Signed-off-by: Krzysztof Kozlowski Reviewed-by: Rob Herring Signed-off-by: Bartosz Golaszewski --- .../bindings/gpio/gpio-consumer-common.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Documentation/devicetree/bindings/gpio/gpio-consumer-common.yaml diff --git a/Documentation/devicetree/bindings/gpio/gpio-consumer-common.yaml b/Documentation/devicetree/bindings/gpio/gpio-consumer-common.yaml new file mode 100644 index 000000000000..40d0be31e200 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/gpio-consumer-common.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpio/gpio-consumer-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Common GPIO lines + +maintainers: + - Bartosz Golaszewski + - Linus Walleij + +description: + Pay attention to using proper GPIO flag (e.g. GPIO_ACTIVE_LOW) for the GPIOs + using inverted signal (e.g. RESETN). + +select: true + +properties: + enable-gpios: + maxItems: 1 + description: + GPIO connected to the enable control pin. + + reset-gpios: + description: + GPIO (or GPIOs for power sequence) connected to the device reset pin + (e.g. RESET or RESETN). + + powerdown-gpios: + maxItems: 1 + description: + GPIO connected to the power down pin (hardware power down or power cut, + e.g. PD or PWDN). + + pwdn-gpios: + maxItems: 1 + description: Use powerdown-gpios + deprecated: true + + wakeup-gpios: + maxItems: 1 + description: + GPIO connected to the pin waking up the device from suspend or other + power-saving modes. + +allOf: + - if: + properties: + compatible: + contains: + enum: + - mmc-pwrseq-simple + then: + properties: + reset-gpios: + minItems: 1 + maxItems: 32 + else: + properties: + reset-gpios: + maxItems: 1 + +additionalProperties: true From c83227a5d05ed77b634ce4c2fc5f143ae2a4d6f5 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Thu, 21 Apr 2022 22:06:54 +0200 Subject: [PATCH 132/319] irq/gpio: ixp4xx: Drop boardfile probe path The boardfiles for IXP4xx have been deleted. Delete all the quirks and code dealing with that boot path and rely solely on device tree boot. Fix some missing static keywords that the kernel test robot was complaining about while we're at it. Cc: Bartosz Golaszewski Acked-by: Marc Zyngier Signed-off-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/Kconfig | 2 + drivers/gpio/gpio-ixp4xx.c | 40 +++------ drivers/irqchip/irq-ixp4xx.c | 126 +---------------------------- include/linux/irqchip/irq-ixp4xx.h | 12 --- 4 files changed, 17 insertions(+), 163 deletions(-) delete mode 100644 include/linux/irqchip/irq-ixp4xx.h diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 075a5d0feef7..57426fdba1b6 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -354,6 +354,7 @@ config GPIO_IOP config GPIO_IXP4XX bool "Intel IXP4xx GPIO" depends on ARCH_IXP4XX + depends on OF select GPIO_GENERIC select GPIOLIB_IRQCHIP select IRQ_DOMAIN_HIERARCHY @@ -362,6 +363,7 @@ config GPIO_IXP4XX IXP4xx series of chips. If unsure, say N. + config GPIO_LOGICVC tristate "Xylon LogiCVC GPIO support" depends on MFD_SYSCON && OF diff --git a/drivers/gpio/gpio-ixp4xx.c b/drivers/gpio/gpio-ixp4xx.c index 76ab9bd7d84b..312309be0287 100644 --- a/drivers/gpio/gpio-ixp4xx.c +++ b/drivers/gpio/gpio-ixp4xx.c @@ -14,8 +14,6 @@ #include #include #include -/* Include that go away with DT transition */ -#include #define IXP4XX_REG_GPOUT 0x00 #define IXP4XX_REG_GPOE 0x04 @@ -193,6 +191,7 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev) struct resource *res; struct ixp4xx_gpio *g; struct gpio_irq_chip *girq; + struct device_node *irq_parent; int ret; g = devm_kzalloc(dev, sizeof(*g), GFP_KERNEL); @@ -205,34 +204,17 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev) if (IS_ERR(g->base)) return PTR_ERR(g->base); - /* - * When we convert to device tree we will simply look up the - * parent irqdomain using irq_find_host(parent) as parent comes - * from IRQCHIP_DECLARE(), then use of_node_to_fwnode() to get - * the fwnode. For now we need this boardfile style code. - */ - if (np) { - struct device_node *irq_parent; - - irq_parent = of_irq_find_parent(np); - if (!irq_parent) { - dev_err(dev, "no IRQ parent node\n"); - return -ENODEV; - } - parent = irq_find_host(irq_parent); - if (!parent) { - dev_err(dev, "no IRQ parent domain\n"); - return -ENODEV; - } - g->fwnode = of_node_to_fwnode(np); - } else { - parent = ixp4xx_get_irq_domain(); - g->fwnode = irq_domain_alloc_fwnode(&res->start); - if (!g->fwnode) { - dev_err(dev, "no domain base\n"); - return -ENODEV; - } + irq_parent = of_irq_find_parent(np); + if (!irq_parent) { + dev_err(dev, "no IRQ parent node\n"); + return -ENODEV; } + parent = irq_find_host(irq_parent); + if (!parent) { + dev_err(dev, "no IRQ parent domain\n"); + return -ENODEV; + } + g->fwnode = of_node_to_fwnode(np); /* * Make sure GPIO 14 and 15 are NOT used as clocks but GPIO on diff --git a/drivers/irqchip/irq-ixp4xx.c b/drivers/irqchip/irq-ixp4xx.c index fb68f8c59fbb..5fba907b9052 100644 --- a/drivers/irqchip/irq-ixp4xx.c +++ b/drivers/irqchip/irq-ixp4xx.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -106,7 +105,8 @@ static void ixp4xx_irq_unmask(struct irq_data *d) } } -asmlinkage void __exception_irq_entry ixp4xx_handle_irq(struct pt_regs *regs) +static asmlinkage void __exception_irq_entry +ixp4xx_handle_irq(struct pt_regs *regs) { struct ixp4xx_irq *ixi = &ixirq; unsigned long status; @@ -195,56 +195,6 @@ static const struct irq_domain_ops ixp4xx_irqdomain_ops = { .free = irq_domain_free_irqs_common, }; -/** - * ixp4xx_get_irq_domain() - retrieve the ixp4xx irq domain - * - * This function will go away when we transition to DT probing. - */ -struct irq_domain *ixp4xx_get_irq_domain(void) -{ - struct ixp4xx_irq *ixi = &ixirq; - - return ixi->domain; -} -EXPORT_SYMBOL_GPL(ixp4xx_get_irq_domain); - -/* - * This is the Linux IRQ to hwirq mapping table. This goes away when - * we have DT support as all IRQ resources are defined in the device - * tree. It will register all the IRQs that are not used by the hierarchical - * GPIO IRQ chip. The "holes" inbetween these IRQs will be requested by - * the GPIO driver using . This is a step-gap solution. - */ -struct ixp4xx_irq_chunk { - int irq; - int hwirq; - int nr_irqs; -}; - -static const struct ixp4xx_irq_chunk ixp4xx_irq_chunks[] = { - { - .irq = 16, - .hwirq = 0, - .nr_irqs = 6, - }, - { - .irq = 24, - .hwirq = 8, - .nr_irqs = 11, - }, - { - .irq = 46, - .hwirq = 30, - .nr_irqs = 2, - }, - /* Only on the 436 variants */ - { - .irq = 48, - .hwirq = 32, - .nr_irqs = 10, - }, -}; - /** * ixp4x_irq_setup() - Common setup code for the IXP4xx interrupt controller * @ixi: State container @@ -298,75 +248,8 @@ static int __init ixp4xx_irq_setup(struct ixp4xx_irq *ixi, return 0; } -/** - * ixp4xx_irq_init() - Function to initialize the irqchip from boardfiles - * @irqbase: physical base for the irq controller - * @is_356: if this is an IXP43x, IXP45x or IXP46x SoC variant - */ -void __init ixp4xx_irq_init(resource_size_t irqbase, - bool is_356) -{ - struct ixp4xx_irq *ixi = &ixirq; - void __iomem *base; - struct fwnode_handle *fwnode; - struct irq_fwspec fwspec; - int nr_chunks; - int ret; - int i; - - base = ioremap(irqbase, 0x100); - if (!base) { - pr_crit("IXP4XX: could not ioremap interrupt controller\n"); - return; - } - fwnode = irq_domain_alloc_fwnode(&irqbase); - if (!fwnode) { - pr_crit("IXP4XX: no domain handle\n"); - return; - } - ret = ixp4xx_irq_setup(ixi, base, fwnode, is_356); - if (ret) { - pr_crit("IXP4XX: failed to set up irqchip\n"); - irq_domain_free_fwnode(fwnode); - } - - nr_chunks = ARRAY_SIZE(ixp4xx_irq_chunks); - if (!is_356) - nr_chunks--; - - /* - * After adding OF support, this is no longer needed: irqs - * will be allocated for the respective fwnodes. - */ - for (i = 0; i < nr_chunks; i++) { - const struct ixp4xx_irq_chunk *chunk = &ixp4xx_irq_chunks[i]; - - pr_info("Allocate Linux IRQs %d..%d HW IRQs %d..%d\n", - chunk->irq, chunk->irq + chunk->nr_irqs - 1, - chunk->hwirq, chunk->hwirq + chunk->nr_irqs - 1); - fwspec.fwnode = fwnode; - fwspec.param[0] = chunk->hwirq; - fwspec.param[1] = IRQ_TYPE_LEVEL_HIGH; - fwspec.param_count = 2; - ret = __irq_domain_alloc_irqs(ixi->domain, - chunk->irq, - chunk->nr_irqs, - NUMA_NO_NODE, - &fwspec, - false, - NULL); - if (ret < 0) { - pr_crit("IXP4XX: can not allocate irqs in hierarchy %d\n", - ret); - return; - } - } -} -EXPORT_SYMBOL_GPL(ixp4xx_irq_init); - -#ifdef CONFIG_OF -int __init ixp4xx_of_init_irq(struct device_node *np, - struct device_node *parent) +static int __init ixp4xx_of_init_irq(struct device_node *np, + struct device_node *parent) { struct ixp4xx_irq *ixi = &ixirq; void __iomem *base; @@ -400,4 +283,3 @@ IRQCHIP_DECLARE(ixp45x, "intel,ixp45x-interrupt", ixp4xx_of_init_irq); IRQCHIP_DECLARE(ixp46x, "intel,ixp46x-interrupt", ixp4xx_of_init_irq); -#endif diff --git a/include/linux/irqchip/irq-ixp4xx.h b/include/linux/irqchip/irq-ixp4xx.h deleted file mode 100644 index 9395917d6936..000000000000 --- a/include/linux/irqchip/irq-ixp4xx.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __IRQ_IXP4XX_H -#define __IRQ_IXP4XX_H - -#include -struct irq_domain; - -void ixp4xx_irq_init(resource_size_t irqbase, - bool is_356); -struct irq_domain *ixp4xx_get_irq_domain(void); - -#endif /* __IRQ_IXP4XX_H */ From fae74fb5d525f979085b6e70b883d7a7049bf15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 25 Apr 2022 19:32:55 +0200 Subject: [PATCH 133/319] gpio: pcf857x: Make teardown callback return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All teardown functions return 0. Also there is little sense in returning a negative error code from an i2c remove function as this only results in emitting an error message but the device is removed nevertheless. This patch is a preparation for making i2c remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Bartosz Golaszewski --- arch/arm/mach-davinci/board-da830-evm.c | 3 +-- arch/arm/mach-davinci/board-dm644x-evm.c | 9 +++------ arch/arm/mach-davinci/board-dm646x-evm.c | 4 +--- drivers/gpio/gpio-pcf857x.c | 14 +++----------- include/linux/platform_data/pcf857x.h | 2 +- 5 files changed, 9 insertions(+), 23 deletions(-) diff --git a/arch/arm/mach-davinci/board-da830-evm.c b/arch/arm/mach-davinci/board-da830-evm.c index 823c9cc98f18..52a452eff01c 100644 --- a/arch/arm/mach-davinci/board-da830-evm.c +++ b/arch/arm/mach-davinci/board-da830-evm.c @@ -473,11 +473,10 @@ static int __init da830_evm_ui_expander_setup(struct i2c_client *client, return 0; } -static int da830_evm_ui_expander_teardown(struct i2c_client *client, int gpio, +static void da830_evm_ui_expander_teardown(struct i2c_client *client, int gpio, unsigned ngpio, void *context) { gpio_free(gpio + 6); - return 0; } static struct pcf857x_platform_data __initdata da830_evm_ui_expander_info = { diff --git a/arch/arm/mach-davinci/board-dm644x-evm.c b/arch/arm/mach-davinci/board-dm644x-evm.c index cce3a621eb20..b69fc17d6a8c 100644 --- a/arch/arm/mach-davinci/board-dm644x-evm.c +++ b/arch/arm/mach-davinci/board-dm644x-evm.c @@ -366,14 +366,13 @@ evm_led_setup(struct i2c_client *client, int gpio, unsigned ngpio, void *c) return status; } -static int +static void evm_led_teardown(struct i2c_client *client, int gpio, unsigned ngpio, void *c) { if (evm_led_dev) { platform_device_unregister(evm_led_dev); evm_led_dev = NULL; } - return 0; } static struct pcf857x_platform_data pcf_data_u2 = { @@ -428,7 +427,7 @@ evm_u18_setup(struct i2c_client *client, int gpio, unsigned ngpio, void *c) return 0; } -static int +static void evm_u18_teardown(struct i2c_client *client, int gpio, unsigned ngpio, void *c) { gpio_free(gpio + 1); @@ -439,7 +438,6 @@ evm_u18_teardown(struct i2c_client *client, int gpio, unsigned ngpio, void *c) device_remove_file(&client->dev, &dev_attr_user_sw); gpio_free(sw_gpio); } - return 0; } static struct pcf857x_platform_data pcf_data_u18 = { @@ -488,7 +486,7 @@ evm_u35_setup(struct i2c_client *client, int gpio, unsigned ngpio, void *c) return 0; } -static int +static void evm_u35_teardown(struct i2c_client *client, int gpio, unsigned ngpio, void *c) { gpio_free(gpio + 7); @@ -498,7 +496,6 @@ evm_u35_teardown(struct i2c_client *client, int gpio, unsigned ngpio, void *c) gpio_free(gpio + 2); gpio_free(gpio + 1); gpio_free(gpio + 0); - return 0; } static struct pcf857x_platform_data pcf_data_u35 = { diff --git a/arch/arm/mach-davinci/board-dm646x-evm.c b/arch/arm/mach-davinci/board-dm646x-evm.c index ee91d81ebbfd..625d2d626147 100644 --- a/arch/arm/mach-davinci/board-dm646x-evm.c +++ b/arch/arm/mach-davinci/board-dm646x-evm.c @@ -315,15 +315,13 @@ static int evm_pcf_setup(struct i2c_client *client, int gpio, return evm_led_setup(client, gpio+4, 4, c); } -static int evm_pcf_teardown(struct i2c_client *client, int gpio, +static void evm_pcf_teardown(struct i2c_client *client, int gpio, unsigned int ngpio, void *c) { BUG_ON(ngpio < 8); evm_sw_teardown(client, gpio, 4, c); evm_led_teardown(client, gpio+4, 4, c); - - return 0; } static struct pcf857x_platform_data pcf_data = { diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c index b7568ee33696..e3a53dd5df1e 100644 --- a/drivers/gpio/gpio-pcf857x.c +++ b/drivers/gpio/gpio-pcf857x.c @@ -396,20 +396,12 @@ static int pcf857x_remove(struct i2c_client *client) { struct pcf857x_platform_data *pdata = dev_get_platdata(&client->dev); struct pcf857x *gpio = i2c_get_clientdata(client); - int status = 0; - if (pdata && pdata->teardown) { - status = pdata->teardown(client, - gpio->chip.base, gpio->chip.ngpio, + if (pdata && pdata->teardown) + pdata->teardown(client, gpio->chip.base, gpio->chip.ngpio, pdata->context); - if (status < 0) { - dev_err(&client->dev, "%s --> %d\n", - "teardown", status); - return status; - } - } - return status; + return 0; } static void pcf857x_shutdown(struct i2c_client *client) diff --git a/include/linux/platform_data/pcf857x.h b/include/linux/platform_data/pcf857x.h index 11d4ed78c7f4..01d0a3ea3aef 100644 --- a/include/linux/platform_data/pcf857x.h +++ b/include/linux/platform_data/pcf857x.h @@ -36,7 +36,7 @@ struct pcf857x_platform_data { int (*setup)(struct i2c_client *client, int gpio, unsigned ngpio, void *context); - int (*teardown)(struct i2c_client *client, + void (*teardown)(struct i2c_client *client, int gpio, unsigned ngpio, void *context); void *context; From acd0b04ecc795e97b7878dccc5cb4d3d627a4c27 Mon Sep 17 00:00:00 2001 From: Vasily Averin Date: Sun, 24 Apr 2022 21:28:54 +0300 Subject: [PATCH 134/319] sysctl: minor cleanup in new_dir() Byte zeroing is not required here, since memory was allocated by kzalloc() Signed-off-by: Vasily Averin Signed-off-by: Luis Chamberlain --- fs/proc/proc_sysctl.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c index 8901fe0b7e2a..f5b12adee69d 100644 --- a/fs/proc/proc_sysctl.c +++ b/fs/proc/proc_sysctl.c @@ -985,7 +985,6 @@ static struct ctl_dir *new_dir(struct ctl_table_set *set, table = (struct ctl_table *)(node + 1); new_name = (char *)(table + 2); memcpy(new_name, name, namelen); - new_name[namelen] = '\0'; table[0].procname = new_name; table[0].mode = S_IFDIR|S_IRUGO|S_IXUGO; init_header(&new->header, set->dir.header.root, set, node, table); From a467257ffe4bdb13eacddec0137013f6a1140b81 Mon Sep 17 00:00:00 2001 From: yingelin Date: Sun, 24 Apr 2022 10:57:40 +0800 Subject: [PATCH 135/319] kernel/kexec_core: move kexec_core sysctls into its own file This move the kernel/kexec_core.c respective sysctls to its own file. kernel/sysctl.c has grown to an insane mess, We move sysctls to places where features actually belong to improve the readability and reduce merge conflicts. At the same time, the proc-sysctl maintainers can easily care about the core logic other than the sysctl knobs added for some feature. We already moved all filesystem sysctls out. This patch is part of the effort to move kexec related sysctls out. Signed-off-by: yingelin Acked-by: Baoquan He Signed-off-by: Luis Chamberlain --- kernel/kexec_core.c | 22 ++++++++++++++++++++++ kernel/sysctl.c | 13 ------------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 68480f731192..a0456baf52cc 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -936,6 +936,28 @@ int kimage_load_segment(struct kimage *image, struct kimage *kexec_image; struct kimage *kexec_crash_image; int kexec_load_disabled; +#ifdef CONFIG_SYSCTL +static struct ctl_table kexec_core_sysctls[] = { + { + .procname = "kexec_load_disabled", + .data = &kexec_load_disabled, + .maxlen = sizeof(int), + .mode = 0644, + /* only handle a transition from default "0" to "1" */ + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = SYSCTL_ONE, + }, + { } +}; + +static int __init kexec_core_sysctl_init(void) +{ + register_sysctl_init("kernel", kexec_core_sysctls); + return 0; +} +late_initcall(kexec_core_sysctl_init); +#endif /* * No panic_cpu check version of crash_kexec(). This function is called diff --git a/kernel/sysctl.c b/kernel/sysctl.c index b60345cbadf0..0f3cb61a2e39 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -61,7 +61,6 @@ #include #include #include -#include #include #include #include @@ -1712,18 +1711,6 @@ static struct ctl_table kern_table[] = { .proc_handler = tracepoint_printk_sysctl, }, #endif -#ifdef CONFIG_KEXEC_CORE - { - .procname = "kexec_load_disabled", - .data = &kexec_load_disabled, - .maxlen = sizeof(int), - .mode = 0644, - /* only handle a transition from default "0" to "1" */ - .proc_handler = proc_dointvec_minmax, - .extra1 = SYSCTL_ONE, - .extra2 = SYSCTL_ONE, - }, -#endif #ifdef CONFIG_MODULES { .procname = "modprobe", From 92abe0f81e1385afd8f1dc66206b5be9a514899b Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Wed, 20 Apr 2022 14:42:52 -0700 Subject: [PATCH 136/319] KVM: arm64: Introduce hyp_alloc_private_va_range() hyp_alloc_private_va_range() can be used to reserve private VA ranges in the nVHE hypervisor. Allocations are aligned based on the order of the requested size. This will be used to implement stack guard pages for KVM nVHE hypervisor (nVHE Hyp mode / not pKVM), in a subsequent patch in the series. Signed-off-by: Kalesh Singh Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220420214317.3303360-2-kaleshsingh@google.com --- arch/arm64/include/asm/kvm_mmu.h | 1 + arch/arm64/kvm/mmu.c | 68 +++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 74735a864eee..a50cbb5ba402 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -154,6 +154,7 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) int kvm_share_hyp(void *from, void *to); void kvm_unshare_hyp(void *from, void *to); int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot); +int hyp_alloc_private_va_range(size_t size, unsigned long *haddr); int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, void __iomem **kaddr, void __iomem **haddr); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 53ae2c0640bc..7de1e02ebfd1 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -457,23 +457,22 @@ int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot) return 0; } -static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size, - unsigned long *haddr, - enum kvm_pgtable_prot prot) + +/** + * hyp_alloc_private_va_range - Allocates a private VA range. + * @size: The size of the VA range to reserve. + * @haddr: The hypervisor virtual start address of the allocation. + * + * The private virtual address (VA) range is allocated below io_map_base + * and aligned based on the order of @size. + * + * Return: 0 on success or negative error code on failure. + */ +int hyp_alloc_private_va_range(size_t size, unsigned long *haddr) { unsigned long base; int ret = 0; - if (!kvm_host_owns_hyp_mappings()) { - base = kvm_call_hyp_nvhe(__pkvm_create_private_mapping, - phys_addr, size, prot); - if (IS_ERR_OR_NULL((void *)base)) - return PTR_ERR((void *)base); - *haddr = base; - - return 0; - } - mutex_lock(&kvm_hyp_pgd_mutex); /* @@ -484,8 +483,10 @@ static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size, * * The allocated size is always a multiple of PAGE_SIZE. */ - size = PAGE_ALIGN(size + offset_in_page(phys_addr)); - base = io_map_base - size; + base = io_map_base - PAGE_ALIGN(size); + + /* Align the allocation based on the order of its size */ + base = ALIGN_DOWN(base, PAGE_SIZE << get_order(size)); /* * Verify that BIT(VA_BITS - 1) hasn't been flipped by @@ -495,19 +496,40 @@ static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size, if ((base ^ io_map_base) & BIT(VA_BITS - 1)) ret = -ENOMEM; else - io_map_base = base; + *haddr = io_map_base = base; mutex_unlock(&kvm_hyp_pgd_mutex); - if (ret) - goto out; + return ret; +} - ret = __create_hyp_mappings(base, size, phys_addr, prot); - if (ret) - goto out; +static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size, + unsigned long *haddr, + enum kvm_pgtable_prot prot) +{ + unsigned long addr; + int ret = 0; - *haddr = base + offset_in_page(phys_addr); -out: + if (!kvm_host_owns_hyp_mappings()) { + addr = kvm_call_hyp_nvhe(__pkvm_create_private_mapping, + phys_addr, size, prot); + if (IS_ERR_VALUE(addr)) + return addr; + *haddr = addr; + + return 0; + } + + size = PAGE_ALIGN(size + offset_in_page(phys_addr)); + ret = hyp_alloc_private_va_range(size, &addr); + if (ret) + return ret; + + ret = __create_hyp_mappings(addr, size, phys_addr, prot); + if (ret) + return ret; + + *haddr = addr + offset_in_page(phys_addr); return ret; } From f922c13e778d6d5343d4576be785a8204c595113 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Wed, 20 Apr 2022 14:42:53 -0700 Subject: [PATCH 137/319] KVM: arm64: Introduce pkvm_alloc_private_va_range() pkvm_hyp_alloc_private_va_range() can be used to reserve private VA ranges in the pKVM nVHE hypervisor. Allocations are aligned based on the order of the requested size. This will be used to implement stack guard pages for pKVM nVHE hypervisor (in a subsequent patch in the series). Credits to Quentin Perret for the idea of moving private VA allocation out of __pkvm_create_private_mapping() Signed-off-by: Kalesh Singh Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220420214317.3303360-3-kaleshsingh@google.com --- arch/arm64/kvm/hyp/include/nvhe/mm.h | 6 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 18 +++++- arch/arm64/kvm/hyp/nvhe/mm.c | 84 ++++++++++++++++++---------- 3 files changed, 75 insertions(+), 33 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 2d08510c6cc1..42d8eb9bfe72 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -19,8 +19,10 @@ int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back); int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot); int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot); int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot); -unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size, - enum kvm_pgtable_prot prot); +int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, + enum kvm_pgtable_prot prot, + unsigned long *haddr); +int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr); static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size, unsigned long *start, unsigned long *end) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 5e2197db0d32..3cea4b6ac23e 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -160,7 +160,23 @@ static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ct DECLARE_REG(size_t, size, host_ctxt, 2); DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3); - cpu_reg(host_ctxt, 1) = __pkvm_create_private_mapping(phys, size, prot); + /* + * __pkvm_create_private_mapping() populates a pointer with the + * hypervisor start address of the allocation. + * + * However, handle___pkvm_create_private_mapping() hypercall crosses the + * EL1/EL2 boundary so the pointer would not be valid in this context. + * + * Instead pass the allocation address as the return value (or return + * ERR_PTR() on failure). + */ + unsigned long haddr; + int err = __pkvm_create_private_mapping(phys, size, prot, &haddr); + + if (err) + haddr = (unsigned long)ERR_PTR(err); + + cpu_reg(host_ctxt, 1) = haddr; } static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt) diff --git a/arch/arm64/kvm/hyp/nvhe/mm.c b/arch/arm64/kvm/hyp/nvhe/mm.c index cdbe8e246418..96193cb31a39 100644 --- a/arch/arm64/kvm/hyp/nvhe/mm.c +++ b/arch/arm64/kvm/hyp/nvhe/mm.c @@ -37,36 +37,60 @@ static int __pkvm_create_mappings(unsigned long start, unsigned long size, return err; } -unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size, - enum kvm_pgtable_prot prot) +/** + * pkvm_alloc_private_va_range - Allocates a private VA range. + * @size: The size of the VA range to reserve. + * @haddr: The hypervisor virtual start address of the allocation. + * + * The private virtual address (VA) range is allocated above __io_map_base + * and aligned based on the order of @size. + * + * Return: 0 on success or negative error code on failure. + */ +int pkvm_alloc_private_va_range(size_t size, unsigned long *haddr) +{ + unsigned long base, addr; + int ret = 0; + + hyp_spin_lock(&pkvm_pgd_lock); + + /* Align the allocation based on the order of its size */ + addr = ALIGN(__io_map_base, PAGE_SIZE << get_order(size)); + + /* The allocated size is always a multiple of PAGE_SIZE */ + base = addr + PAGE_ALIGN(size); + + /* Are we overflowing on the vmemmap ? */ + if (!addr || base > __hyp_vmemmap) + ret = -ENOMEM; + else { + __io_map_base = base; + *haddr = addr; + } + + hyp_spin_unlock(&pkvm_pgd_lock); + + return ret; +} + +int __pkvm_create_private_mapping(phys_addr_t phys, size_t size, + enum kvm_pgtable_prot prot, + unsigned long *haddr) { unsigned long addr; int err; - hyp_spin_lock(&pkvm_pgd_lock); - size = PAGE_ALIGN(size + offset_in_page(phys)); - addr = __io_map_base; - __io_map_base += size; + err = pkvm_alloc_private_va_range(size, &addr); + if (err) + return err; - /* Are we overflowing on the vmemmap ? */ - if (__io_map_base > __hyp_vmemmap) { - __io_map_base -= size; - addr = (unsigned long)ERR_PTR(-ENOMEM); - goto out; - } + err = __pkvm_create_mappings(addr, size, phys, prot); + if (err) + return err; - err = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, prot); - if (err) { - addr = (unsigned long)ERR_PTR(err); - goto out; - } - - addr = addr + offset_in_page(phys); -out: - hyp_spin_unlock(&pkvm_pgd_lock); - - return addr; + *haddr = addr + offset_in_page(phys); + return err; } int pkvm_create_mappings_locked(void *from, void *to, enum kvm_pgtable_prot prot) @@ -146,7 +170,8 @@ int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot) int hyp_map_vectors(void) { phys_addr_t phys; - void *bp_base; + unsigned long bp_base; + int ret; if (!kvm_system_needs_idmapped_vectors()) { __hyp_bp_vect_base = __bp_harden_hyp_vecs; @@ -154,13 +179,12 @@ int hyp_map_vectors(void) } phys = __hyp_pa(__bp_harden_hyp_vecs); - bp_base = (void *)__pkvm_create_private_mapping(phys, - __BP_HARDEN_HYP_VECS_SZ, - PAGE_HYP_EXEC); - if (IS_ERR_OR_NULL(bp_base)) - return PTR_ERR(bp_base); + ret = __pkvm_create_private_mapping(phys, __BP_HARDEN_HYP_VECS_SZ, + PAGE_HYP_EXEC, &bp_base); + if (ret) + return ret; - __hyp_bp_vect_base = bp_base; + __hyp_bp_vect_base = (void *)bp_base; return 0; } From ce3354318a57875dc59f4bb841662e95bfba03db Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Wed, 20 Apr 2022 14:42:54 -0700 Subject: [PATCH 138/319] KVM: arm64: Add guard pages for KVM nVHE hypervisor stack Map the stack pages in the flexible private VA range and allocate guard pages below the stack as unbacked VA space. The stack is aligned so that any valid stack address has PAGE_SHIFT bit as 1 - this is used for overflow detection (implemented in a subsequent patch in the series). Signed-off-by: Kalesh Singh Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220420214317.3303360-4-kaleshsingh@google.com --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_mmu.h | 2 ++ arch/arm64/kvm/arm.c | 37 +++++++++++++++++++++++++++++--- arch/arm64/kvm/mmu.c | 4 ++-- 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index d5b0386ef765..2e277f2ed671 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -169,6 +169,7 @@ struct kvm_nvhe_init_params { unsigned long tcr_el2; unsigned long tpidr_el2; unsigned long stack_hyp_va; + unsigned long stack_pa; phys_addr_t pgd_pa; unsigned long hcr_el2; unsigned long vttbr; diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index a50cbb5ba402..b208da3bebec 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -154,6 +154,8 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) int kvm_share_hyp(void *from, void *to); void kvm_unshare_hyp(void *from, void *to); int create_hyp_mappings(void *from, void *to, enum kvm_pgtable_prot prot); +int __create_hyp_mappings(unsigned long start, unsigned long size, + unsigned long phys, enum kvm_pgtable_prot prot); int hyp_alloc_private_va_range(size_t size, unsigned long *haddr); int create_hyp_io_mappings(phys_addr_t phys_addr, size_t size, void __iomem **kaddr, diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 523bc934fe2f..dd257d9f21a2 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1483,7 +1483,6 @@ static void cpu_prepare_hyp_mode(int cpu) tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET; params->tcr_el2 = tcr; - params->stack_hyp_va = kern_hyp_va(per_cpu(kvm_arm_hyp_stack_page, cpu) + PAGE_SIZE); params->pgd_pa = kvm_mmu_get_httbr(); if (is_protected_kvm_enabled()) params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS; @@ -1933,14 +1932,46 @@ static int init_hyp_mode(void) * Map the Hyp stack pages */ for_each_possible_cpu(cpu) { + struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu); char *stack_page = (char *)per_cpu(kvm_arm_hyp_stack_page, cpu); - err = create_hyp_mappings(stack_page, stack_page + PAGE_SIZE, - PAGE_HYP); + unsigned long hyp_addr; + /* + * Allocate a contiguous HYP private VA range for the stack + * and guard page. The allocation is also aligned based on + * the order of its size. + */ + err = hyp_alloc_private_va_range(PAGE_SIZE * 2, &hyp_addr); + if (err) { + kvm_err("Cannot allocate hyp stack guard page\n"); + goto out_err; + } + + /* + * Since the stack grows downwards, map the stack to the page + * at the higher address and leave the lower guard page + * unbacked. + * + * Any valid stack address now has the PAGE_SHIFT bit as 1 + * and addresses corresponding to the guard page have the + * PAGE_SHIFT bit as 0 - this is used for overflow detection. + */ + err = __create_hyp_mappings(hyp_addr + PAGE_SIZE, PAGE_SIZE, + __pa(stack_page), PAGE_HYP); if (err) { kvm_err("Cannot map hyp stack\n"); goto out_err; } + + /* + * Save the stack PA in nvhe_init_params. This will be needed + * to recreate the stack mapping in protected nVHE mode. + * __hyp_pa() won't do the right thing there, since the stack + * has been mapped in the flexible private VA space. + */ + params->stack_pa = __pa(stack_page); + + params->stack_hyp_va = hyp_addr + (2 * PAGE_SIZE); } for_each_possible_cpu(cpu) { diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 7de1e02ebfd1..088e14eae4cf 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -258,8 +258,8 @@ static bool kvm_host_owns_hyp_mappings(void) return true; } -static int __create_hyp_mappings(unsigned long start, unsigned long size, - unsigned long phys, enum kvm_pgtable_prot prot) +int __create_hyp_mappings(unsigned long start, unsigned long size, + unsigned long phys, enum kvm_pgtable_prot prot) { int err; From 1a919b17ef012ca0572bae759c27e5ea02bfb47f Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Wed, 20 Apr 2022 14:42:55 -0700 Subject: [PATCH 139/319] KVM: arm64: Add guard pages for pKVM (protected nVHE) hypervisor stack Map the stack pages in the flexible private VA range and allocate guard pages below the stack as unbacked VA space. The stack is aligned so that any valid stack address has PAGE_SHIFT bit as 1 - this is used for overflow detection (implemented in a subsequent patch in the series) Signed-off-by: Kalesh Singh Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220420214317.3303360-5-kaleshsingh@google.com --- arch/arm64/kvm/hyp/nvhe/setup.c | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 27af337f9fea..e8d4ea2fcfa0 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -99,17 +99,42 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size, return ret; for (i = 0; i < hyp_nr_cpus; i++) { + struct kvm_nvhe_init_params *params = per_cpu_ptr(&kvm_init_params, i); + unsigned long hyp_addr; + start = (void *)kern_hyp_va(per_cpu_base[i]); end = start + PAGE_ALIGN(hyp_percpu_size); ret = pkvm_create_mappings(start, end, PAGE_HYP); if (ret) return ret; - end = (void *)per_cpu_ptr(&kvm_init_params, i)->stack_hyp_va; - start = end - PAGE_SIZE; - ret = pkvm_create_mappings(start, end, PAGE_HYP); + /* + * Allocate a contiguous HYP private VA range for the stack + * and guard page. The allocation is also aligned based on + * the order of its size. + */ + ret = pkvm_alloc_private_va_range(PAGE_SIZE * 2, &hyp_addr); if (ret) return ret; + + /* + * Since the stack grows downwards, map the stack to the page + * at the higher address and leave the lower guard page + * unbacked. + * + * Any valid stack address now has the PAGE_SHIFT bit as 1 + * and addresses corresponding to the guard page have the + * PAGE_SHIFT bit as 0 - this is used for overflow detection. + */ + hyp_spin_lock(&pkvm_pgd_lock); + ret = kvm_pgtable_hyp_map(&pkvm_pgtable, hyp_addr + PAGE_SIZE, + PAGE_SIZE, params->stack_pa, PAGE_HYP); + hyp_spin_unlock(&pkvm_pgd_lock); + if (ret) + return ret; + + /* Update stack_hyp_va to end of the stack's private VA range */ + params->stack_hyp_va = hyp_addr + (2 * PAGE_SIZE); } /* From 66de19fad9ef47c5376a99bb2b00661f1c788a94 Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Wed, 20 Apr 2022 14:42:56 -0700 Subject: [PATCH 140/319] KVM: arm64: Detect and handle hypervisor stack overflows The hypervisor stacks (for both nVHE Hyp mode and nVHE protected mode) are aligned such that any valid stack address has PAGE_SHIFT bit as 1. This allows us to conveniently check for overflow in the exception entry without corrupting any GPRs. We won't recover from a stack overflow so panic the hypervisor. Signed-off-by: Kalesh Singh Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220420214317.3303360-6-kaleshsingh@google.com --- arch/arm64/kvm/hyp/nvhe/host.S | 24 ++++++++++++++++++++++++ arch/arm64/kvm/hyp/nvhe/switch.c | 7 ++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 3d613e721a75..09b5254fb497 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -153,6 +153,18 @@ SYM_FUNC_END(__host_hvc) .macro invalid_host_el2_vect .align 7 + + /* + * Test whether the SP has overflowed, without corrupting a GPR. + * nVHE hypervisor stacks are aligned so that the PAGE_SHIFT bit + * of SP should always be 1. + */ + add sp, sp, x0 // sp' = sp + x0 + sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp + tbz x0, #PAGE_SHIFT, .L__hyp_sp_overflow\@ + sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0 + sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp + /* If a guest is loaded, panic out of it. */ stp x0, x1, [sp, #-16]! get_loaded_vcpu x0, x1 @@ -165,6 +177,18 @@ SYM_FUNC_END(__host_hvc) * been partially clobbered by __host_enter. */ b hyp_panic + +.L__hyp_sp_overflow\@: + /* + * Reset SP to the top of the stack, to allow handling the hyp_panic. + * This corrupts the stack but is ok, since we won't be attempting + * any unwinding here. + */ + ldr_this_cpu x0, kvm_init_params + NVHE_INIT_STACK_HYP_VA, x1 + mov sp, x0 + + b hyp_panic_bad_stack + ASM_BUG() .endm .macro invalid_host_el1_vect diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 6410d21d8695..703a5d3f611b 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -347,7 +347,7 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu) return exit_code; } -void __noreturn hyp_panic(void) +asmlinkage void __noreturn hyp_panic(void) { u64 spsr = read_sysreg_el2(SYS_SPSR); u64 elr = read_sysreg_el2(SYS_ELR); @@ -369,6 +369,11 @@ void __noreturn hyp_panic(void) unreachable(); } +asmlinkage void __noreturn hyp_panic_bad_stack(void) +{ + hyp_panic(); +} + asmlinkage void kvm_unexpected_el2_exception(void) { return __kvm_unexpected_el2_exception(); From 6ccf9cb557bd32073b0d68baed97f1bd8a40ff1d Mon Sep 17 00:00:00 2001 From: Kalesh Singh Date: Wed, 20 Apr 2022 14:42:57 -0700 Subject: [PATCH 141/319] KVM: arm64: Symbolize the nVHE HYP addresses Reintroduce the __kvm_nvhe_ symbols in kallsyms, ignoring the local symbols in this namespace. The local symbols are not informative and can cause aliasing issues when symbolizing the addresses. With the necessary symbols now in kallsyms we can symbolize nVHE addresses using the %p print format specifier: [ 98.916444][ T426] kvm [426]: nVHE hyp panic at: [] __kvm_nvhe_overflow_stack+0x8/0x34! Signed-off-by: Kalesh Singh Tested-by: Fuad Tabba Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220420214317.3303360-7-kaleshsingh@google.com --- arch/arm64/kvm/handle_exit.c | 13 +++++-------- scripts/kallsyms.c | 3 ++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 97fe14aab1a3..a377b871bf58 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -295,13 +295,8 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, u64 elr_in_kimg = __phys_to_kimg(elr_phys); u64 hyp_offset = elr_in_kimg - kaslr_offset() - elr_virt; u64 mode = spsr & PSR_MODE_MASK; + u64 panic_addr = elr_virt + hyp_offset; - /* - * The nVHE hyp symbols are not included by kallsyms to avoid issues - * with aliasing. That means that the symbols cannot be printed with the - * "%pS" format specifier, so fall back to the vmlinux address if - * there's no better option. - */ if (mode != PSR_MODE_EL2t && mode != PSR_MODE_EL2h) { kvm_err("Invalid host exception to nVHE hyp!\n"); } else if (ESR_ELx_EC(esr) == ESR_ELx_EC_BRK64 && @@ -321,9 +316,11 @@ void __noreturn __cold nvhe_hyp_panic_handler(u64 esr, u64 spsr, if (file) kvm_err("nVHE hyp BUG at: %s:%u!\n", file, line); else - kvm_err("nVHE hyp BUG at: %016llx!\n", elr_virt + hyp_offset); + kvm_err("nVHE hyp BUG at: [<%016llx>] %pB!\n", panic_addr, + (void *)panic_addr); } else { - kvm_err("nVHE hyp panic at: %016llx!\n", elr_virt + hyp_offset); + kvm_err("nVHE hyp panic at: [<%016llx>] %pB!\n", panic_addr, + (void *)panic_addr); } /* diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index 8caabddf817c..e6906f79833d 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -111,7 +111,8 @@ static bool is_ignored_symbol(const char *name, char type) ".L", /* local labels, .LBB,.Ltmpxxx,.L__unnamed_xx,.LASANPC, etc. */ "__crc_", /* modversions */ "__efistub_", /* arm64 EFI stub namespace */ - "__kvm_nvhe_", /* arm64 non-VHE KVM namespace */ + "__kvm_nvhe_$", /* arm64 local symbols in non-VHE KVM namespace */ + "__kvm_nvhe_.L", /* arm64 local symbols in non-VHE KVM namespace */ "__AArch64ADRPThunk_", /* arm64 lld */ "__ARMV5PILongThunk_", /* arm lld */ "__ARMV7PILongThunk_", From f30903394eb62316dddea8801b357f5cec4df187 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Tue, 19 Apr 2022 15:53:52 -0500 Subject: [PATCH 142/319] x86/cpufeatures: Add virtual TSC_AUX feature bit The TSC_AUX Virtualization feature allows AMD SEV-ES guests to securely use TSC_AUX (auxiliary time stamp counter data) MSR in RDTSCP and RDPID instructions. The TSC_AUX MSR is typically initialized to APIC ID or another unique identifier so that software can quickly associate returned TSC value with the logical processor. Add the feature bit and also include it in the kvm for detection. Signed-off-by: Babu Moger Acked-by: Borislav Petkov Message-Id: <165040157111.1399644.6123821125319995316.stgit@bmoger-ubuntu> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/cpufeatures.h | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 73e643ae94b6..1bc66a17a95a 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -405,6 +405,7 @@ #define X86_FEATURE_SEV (19*32+ 1) /* AMD Secure Encrypted Virtualization */ #define X86_FEATURE_VM_PAGE_FLUSH (19*32+ 2) /* "" VM Page Flush MSR is supported */ #define X86_FEATURE_SEV_ES (19*32+ 3) /* AMD Secure Encrypted Virtualization - Encrypted State */ +#define X86_FEATURE_V_TSC_AUX (19*32+ 9) /* Virtual TSC_AUX */ #define X86_FEATURE_SME_COHERENT (19*32+10) /* "" AMD hardware-enforced cache coherency */ /* From 296d5a17e793956f7b914336422043c939263409 Mon Sep 17 00:00:00 2001 From: Babu Moger Date: Tue, 19 Apr 2022 15:54:44 -0500 Subject: [PATCH 143/319] KVM: SEV-ES: Use V_TSC_AUX if available instead of RDTSC/MSR_TSC_AUX intercepts The TSC_AUX virtualization feature allows AMD SEV-ES guests to securely use TSC_AUX (auxiliary time stamp counter data) in the RDTSCP and RDPID instructions. The TSC_AUX value is set using the WRMSR instruction to the TSC_AUX MSR (0xC0000103). It is read by the RDMSR, RDTSCP and RDPID instructions. If the read/write of the TSC_AUX MSR is intercepted, then RDTSCP and RDPID must also be intercepted when TSC_AUX virtualization is present. However, the RDPID instruction can't be intercepted. This means that when TSC_AUX virtualization is present, RDTSCP and TSC_AUX MSR read/write must not be intercepted for SEV-ES (or SEV-SNP) guests. Signed-off-by: Babu Moger Message-Id: <165040164424.1399644.13833277687385156344.stgit@bmoger-ubuntu> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/cpufeatures.h | 2 +- arch/x86/kvm/svm/sev.c | 8 ++++++++ arch/x86/kvm/svm/svm.c | 1 + arch/x86/kvm/svm/svm.h | 2 +- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 1bc66a17a95a..1d6826eac3e6 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -405,7 +405,7 @@ #define X86_FEATURE_SEV (19*32+ 1) /* AMD Secure Encrypted Virtualization */ #define X86_FEATURE_VM_PAGE_FLUSH (19*32+ 2) /* "" VM Page Flush MSR is supported */ #define X86_FEATURE_SEV_ES (19*32+ 3) /* AMD Secure Encrypted Virtualization - Encrypted State */ -#define X86_FEATURE_V_TSC_AUX (19*32+ 9) /* Virtual TSC_AUX */ +#define X86_FEATURE_V_TSC_AUX (19*32+ 9) /* "" Virtual TSC_AUX */ #define X86_FEATURE_SME_COHERENT (19*32+10) /* "" AMD hardware-enforced cache coherency */ /* diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 9ce162119b23..b67ce873d5d2 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2926,6 +2926,14 @@ void sev_es_init_vmcb(struct vcpu_svm *svm) set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); + + if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && + (guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDTSCP) || + guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDPID))) { + set_msr_interception(vcpu, svm->msrpm, MSR_TSC_AUX, 1, 1); + if (guest_cpuid_has(&svm->vcpu, X86_FEATURE_RDTSCP)) + svm_clr_intercept(svm, INTERCEPT_RDTSCP); + } } void sev_es_vcpu_reset(struct vcpu_svm *svm) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 22bbd69495ad..b83af9ecf85a 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -99,6 +99,7 @@ static const struct svm_direct_access_msrs { { .index = MSR_EFER, .always = false }, { .index = MSR_IA32_CR_PAT, .always = false }, { .index = MSR_AMD64_SEV_ES_GHCB, .always = true }, + { .index = MSR_TSC_AUX, .always = false }, { .index = MSR_INVALID, .always = false }, }; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index e246793cbeae..32220a1b0ea2 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -29,7 +29,7 @@ #define IOPM_SIZE PAGE_SIZE * 3 #define MSRPM_SIZE PAGE_SIZE * 2 -#define MAX_DIRECT_ACCESS_MSRS 20 +#define MAX_DIRECT_ACCESS_MSRS 21 #define MSRPM_OFFSETS 16 extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; extern bool npt_enabled; From 65936229d35883a1637730e628c3c84d3600025d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 15 Apr 2022 00:49:09 +0000 Subject: [PATCH 144/319] KVM: x86/mmu: Check for host MMIO exclusion from mem encrypt iff necessary When determining whether or not a SPTE needs to have SME/SEV's memory encryption flag set, do the moderately expensive host MMIO pfn check if and only if the memory encryption mask is non-zero. Note, KVM could further optimize the host MMIO checks by making a single call to kvm_is_mmio_pfn(), but the tdp_enabled path (for EPT's memtype handling) will likely be split out to a separate flow[*]. At that point, a better approach would be to shove the call to kvm_is_mmio_pfn() into VMX code so that AMD+NPT without SME doesn't get hit with an unnecessary lookup. [*] https://lkml.kernel.org/r/20220321224358.1305530-3-bgardon@google.com Signed-off-by: Sean Christopherson Message-Id: <20220415004909.2216670-1-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/spte.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c index 4739b53c9734..aab78574e03d 100644 --- a/arch/x86/kvm/mmu/spte.c +++ b/arch/x86/kvm/mmu/spte.c @@ -139,7 +139,7 @@ bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, else pte_access &= ~ACC_WRITE_MASK; - if (!kvm_is_mmio_pfn(pfn)) + if (shadow_me_mask && !kvm_is_mmio_pfn(pfn)) spte |= shadow_me_mask; spte |= (u64)pfn << PAGE_SHIFT; From 8b9e74bfbf8c7020498a9ea600bd4c0f1915134d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 20 Apr 2022 00:27:47 +0000 Subject: [PATCH 145/319] KVM: x86/mmu: Use enable_mmio_caching to track if MMIO caching is enabled Clear enable_mmio_caching if hardware can't support MMIO caching and use the dedicated flag to detect if MMIO caching is enabled instead of assuming shadow_mmio_value==0 means MMIO caching is disabled. TDX will use a zero value even when caching is enabled, and is_mmio_spte() isn't so hot that it needs to avoid an extra memory access, i.e. there's no reason to be super clever. And the clever approach may not even be more performant, e.g. gcc-11 lands the extra check on a non-zero value inline, but puts the enable_mmio_caching out-of-line, i.e. avoids the few extra uops for non-MMIO SPTEs. Cc: Isaku Yamahata Cc: Kai Huang Signed-off-by: Sean Christopherson Message-Id: <20220420002747.3287931-1-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/mmu/spte.c | 5 ++++- arch/x86/kvm/mmu/spte.h | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 77785587332e..7b088417d978 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3036,7 +3036,7 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa * and only if L1's MAXPHYADDR is inaccurate with respect to * the hardware's). */ - if (unlikely(!shadow_mmio_value) || + if (unlikely(!enable_mmio_caching) || unlikely(fault->gfn > kvm_mmu_max_gfn())) { *ret_val = RET_PF_EMULATE; return true; diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c index aab78574e03d..3d611f07eee8 100644 --- a/arch/x86/kvm/mmu/spte.c +++ b/arch/x86/kvm/mmu/spte.c @@ -19,7 +19,7 @@ #include #include -static bool __read_mostly enable_mmio_caching = true; +bool __read_mostly enable_mmio_caching = true; module_param_named(mmio_caching, enable_mmio_caching, bool, 0444); u64 __read_mostly shadow_host_writable_mask; @@ -351,6 +351,9 @@ void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask) WARN_ON(mmio_value && (REMOVED_SPTE & mmio_mask) == mmio_value)) mmio_value = 0; + if (!mmio_value) + enable_mmio_caching = false; + shadow_mmio_value = mmio_value; shadow_mmio_mask = mmio_mask; shadow_mmio_access_mask = access_mask; diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h index e4abeb5df1b1..43eceb821b31 100644 --- a/arch/x86/kvm/mmu/spte.h +++ b/arch/x86/kvm/mmu/spte.h @@ -5,6 +5,8 @@ #include "mmu_internal.h" +extern bool __read_mostly enable_mmio_caching; + /* * A MMU present SPTE is backed by actual memory and may or may not be present * in hardware. E.g. MMIO SPTEs are not considered present. Use bit 11, as it @@ -204,7 +206,7 @@ extern u64 __read_mostly shadow_nonpresent_or_rsvd_lower_gfn_mask; static inline bool is_mmio_spte(u64 spte) { return (spte & shadow_mmio_mask) == shadow_mmio_value && - likely(shadow_mmio_value); + likely(enable_mmio_caching); } static inline bool is_shadow_present_pte(u64 pte) From daed87b8767c005cb2b385c80e265edf82a2f64b Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:30:08 -0500 Subject: [PATCH 146/319] KVM: x86/mmu: nested EPT cannot be used in SMM The role.base.smm flag is always zero when setting up shadow EPT, do not bother copying it over from vcpu->arch.root_mmu. Reviewed-by: David Matlack Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 7b088417d978..3a2d39f7f6eb 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4953,9 +4953,11 @@ kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, { union kvm_mmu_role role = {0}; - /* SMM flag is inherited from root_mmu */ - role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm; - + /* + * KVM does not support SMM transfer monitors, and consequently does not + * support the "entry to SMM" control either. role.base.smm is always 0. + */ + WARN_ON_ONCE(is_smm(vcpu)); role.base.level = level; role.base.has_4_byte_gpte = false; role.base.direct = false; From 82ffa13f7933cdc9aa3d22c9bc5c8d9c9f610af1 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:30:20 -0500 Subject: [PATCH 147/319] KVM: x86/mmu: constify uses of struct kvm_mmu_role_regs struct kvm_mmu_role_regs is computed just once and then accessed. Use const to make this clearer, even though the const fields of struct kvm_mmu_role_regs already prevent (or make it harder...) to modify the contents of the struct. Reviewed-by: David Matlack Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3a2d39f7f6eb..5d5e5b062be0 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -197,7 +197,8 @@ struct kvm_mmu_role_regs { * the single source of truth for the MMU's state. */ #define BUILD_MMU_ROLE_REGS_ACCESSOR(reg, name, flag) \ -static inline bool __maybe_unused ____is_##reg##_##name(struct kvm_mmu_role_regs *regs)\ +static inline bool __maybe_unused \ +____is_##reg##_##name(const struct kvm_mmu_role_regs *regs) \ { \ return !!(regs->reg & flag); \ } @@ -244,7 +245,7 @@ static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu) return regs; } -static int role_regs_to_root_level(struct kvm_mmu_role_regs *regs) +static int role_regs_to_root_level(const struct kvm_mmu_role_regs *regs) { if (!____is_cr0_pg(regs)) return 0; @@ -4748,7 +4749,7 @@ static void paging32_init_context(struct kvm_mmu *context) } static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu, - struct kvm_mmu_role_regs *regs) + const struct kvm_mmu_role_regs *regs) { union kvm_mmu_extended_role ext = {0}; @@ -4771,7 +4772,7 @@ static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu, } static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, - struct kvm_mmu_role_regs *regs, + const struct kvm_mmu_role_regs *regs, bool base_only) { union kvm_mmu_role role = {0}; @@ -4807,7 +4808,8 @@ static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) static union kvm_mmu_role kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, - struct kvm_mmu_role_regs *regs, bool base_only) + const struct kvm_mmu_role_regs *regs, + bool base_only) { union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only); @@ -4853,7 +4855,8 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) static union kvm_mmu_role kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, - struct kvm_mmu_role_regs *regs, bool base_only) + const struct kvm_mmu_role_regs *regs, + bool base_only) { union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only); @@ -4866,7 +4869,8 @@ kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, static union kvm_mmu_role kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, - struct kvm_mmu_role_regs *regs, bool base_only) + const struct kvm_mmu_role_regs *regs, + bool base_only) { union kvm_mmu_role role = kvm_calc_shadow_root_page_role_common(vcpu, regs, base_only); @@ -4884,7 +4888,7 @@ kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, } static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context, - struct kvm_mmu_role_regs *regs, + const struct kvm_mmu_role_regs *regs, union kvm_mmu_role new_role) { if (new_role.as_u64 == context->mmu_role.as_u64) @@ -4907,7 +4911,7 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte } static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, - struct kvm_mmu_role_regs *regs) + const struct kvm_mmu_role_regs *regs) { struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_mmu_role new_role = @@ -4918,7 +4922,7 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, static union kvm_mmu_role kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu, - struct kvm_mmu_role_regs *regs) + const struct kvm_mmu_role_regs *regs) { union kvm_mmu_role role = kvm_calc_shadow_root_page_role_common(vcpu, regs, false); @@ -5018,7 +5022,7 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu) } static union kvm_mmu_role -kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu, struct kvm_mmu_role_regs *regs) +kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { union kvm_mmu_role role; From 39e7e2bf321ac7784823e61abcbdf1459aeaee94 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:30:31 -0500 Subject: [PATCH 148/319] KVM: x86/mmu: pull computation of kvm_mmu_role_regs to kvm_init_mmu The init_kvm_*mmu functions, with the exception of shadow NPT, do not need to know the full values of CR0/CR4/EFER; they only need to know the bits that make up the "role". This cleanup however will take quite a few incremental steps. As a start, pull the common computation of the struct kvm_mmu_role_regs into their caller: all of them extract the struct from the vcpu as the very first step. Reviewed-by: David Matlack Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 5d5e5b062be0..3e49902561f9 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4821,12 +4821,12 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, return role; } -static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) +static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, + const struct kvm_mmu_role_regs *regs) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); union kvm_mmu_role new_role = - kvm_calc_tdp_mmu_root_page_role(vcpu, ®s, false); + kvm_calc_tdp_mmu_root_page_role(vcpu, regs, false); if (new_role.as_u64 == context->mmu_role.as_u64) return; @@ -4840,7 +4840,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; - context->root_level = role_regs_to_root_level(®s); + context->root_level = role_regs_to_root_level(regs); if (!is_cr0_pg(context)) context->gva_to_gpa = nonpaging_gva_to_gpa; @@ -5009,12 +5009,12 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, } EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu); -static void init_kvm_softmmu(struct kvm_vcpu *vcpu) +static void init_kvm_softmmu(struct kvm_vcpu *vcpu, + const struct kvm_mmu_role_regs *regs) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); - kvm_init_shadow_mmu(vcpu, ®s); + kvm_init_shadow_mmu(vcpu, regs); context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; @@ -5038,10 +5038,10 @@ kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs * return role; } -static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) +static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu, + const struct kvm_mmu_role_regs *regs) { - struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); - union kvm_mmu_role new_role = kvm_calc_nested_mmu_role(vcpu, ®s); + union kvm_mmu_role new_role = kvm_calc_nested_mmu_role(vcpu, regs); struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; if (new_role.as_u64 == g_context->mmu_role.as_u64) @@ -5081,12 +5081,14 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) void kvm_init_mmu(struct kvm_vcpu *vcpu) { + struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); + if (mmu_is_nested(vcpu)) - init_kvm_nested_mmu(vcpu); + init_kvm_nested_mmu(vcpu, ®s); else if (tdp_enabled) - init_kvm_tdp_mmu(vcpu); + init_kvm_tdp_mmu(vcpu, ®s); else - init_kvm_softmmu(vcpu); + init_kvm_softmmu(vcpu, ®s); } EXPORT_SYMBOL_GPL(kvm_init_mmu); From 25cc05652cd6be7349c84abbea3886b5483330cd Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:30:40 -0500 Subject: [PATCH 149/319] KVM: x86/mmu: rephrase unclear comment If accessed bits are not supported there simple isn't any distinction between accessed and non-accessed gPTEs, so the comment does not make much sense. Rephrase it in terms of what happens if accessed bits *are* supported. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/paging_tmpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 7d4377f1ef2a..07a7832f96cb 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -151,7 +151,7 @@ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu, if (!FNAME(is_present_gpte)(gpte)) goto no_present; - /* if accessed bit is not supported prefetch non accessed gpte */ + /* Prefetch only accessed entries (unless A/D bits are disabled). */ if (PT_HAVE_ACCESSED_DIRTY(vcpu->arch.mmu) && !(gpte & PT_GUEST_ACCESSED_MASK)) goto no_present; From 6819af7597d87d40769b47bb377472877a6b56c0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 3 Mar 2022 20:20:17 -0800 Subject: [PATCH 150/319] KVM: x86: Clean up and document nested #PF workaround Replace the per-vendor hack-a-fix for KVM's #PF => #PF => #DF workaround with an explicit, common workaround in kvm_inject_emulated_page_fault(). Aside from being a hack, the current approach is brittle and incomplete, e.g. nSVM's KVM_SET_NESTED_STATE fails to set ->inject_page_fault(), and nVMX fails to apply the workaround when VMX is intercepting #PF due to allow_smaller_maxphyaddr=1. Signed-off-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/svm/nested.c | 18 +++++++++--------- arch/x86/kvm/vmx/nested.c | 15 ++++++--------- arch/x86/kvm/x86.c | 21 ++++++++++++++++++++- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index e1c695f72b8b..e46bd289e5df 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1516,6 +1516,8 @@ struct kvm_x86_ops { struct kvm_x86_nested_ops { void (*leave_nested)(struct kvm_vcpu *vcpu); int (*check_events)(struct kvm_vcpu *vcpu); + bool (*handle_page_fault_workaround)(struct kvm_vcpu *vcpu, + struct x86_exception *fault); bool (*hv_timer_pending)(struct kvm_vcpu *vcpu); void (*triple_fault)(struct kvm_vcpu *vcpu); int (*get_state)(struct kvm_vcpu *vcpu, diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index caa691229b71..bed5e1692cef 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -55,24 +55,26 @@ static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, nested_svm_vmexit(svm); } -static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) +static bool nested_svm_handle_page_fault_workaround(struct kvm_vcpu *vcpu, + struct x86_exception *fault) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; - WARN_ON(!is_guest_mode(vcpu)); + WARN_ON(!is_guest_mode(vcpu)); if (vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && - !svm->nested.nested_run_pending) { - vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; + !WARN_ON_ONCE(svm->nested.nested_run_pending)) { + vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; vmcb->control.exit_code_hi = 0; vmcb->control.exit_info_1 = fault->error_code; vmcb->control.exit_info_2 = fault->address; nested_svm_vmexit(svm); - } else { - kvm_inject_page_fault(vcpu, fault); + return true; } + + return false; } static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) @@ -751,9 +753,6 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, if (ret) return ret; - if (!npt_enabled) - vcpu->arch.mmu->inject_page_fault = svm_inject_page_fault_nested; - if (!from_vmrun) kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); @@ -1659,6 +1658,7 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) struct kvm_x86_nested_ops svm_nested_ops = { .leave_nested = svm_leave_nested, .check_events = svm_check_nested_events, + .handle_page_fault_workaround = nested_svm_handle_page_fault_workaround, .triple_fault = nested_svm_triple_fault, .get_nested_state_pages = svm_get_nested_state_pages, .get_state = svm_get_nested_state, diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 838ac7ab5950..a6688663da4d 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -476,24 +476,23 @@ static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit return 0; } - -static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, - struct x86_exception *fault) +static bool nested_vmx_handle_page_fault_workaround(struct kvm_vcpu *vcpu, + struct x86_exception *fault) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); WARN_ON(!is_guest_mode(vcpu)); if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) && - !to_vmx(vcpu)->nested.nested_run_pending) { + !WARN_ON_ONCE(to_vmx(vcpu)->nested.nested_run_pending)) { vmcs12->vm_exit_intr_error_code = fault->error_code; nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, PF_VECTOR | INTR_TYPE_HARD_EXCEPTION | INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK, fault->address); - } else { - kvm_inject_page_fault(vcpu, fault); + return true; } + return false; } static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, @@ -2614,9 +2613,6 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3); } - if (!enable_ept) - vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested; - if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL) && WARN_ON_ONCE(kvm_set_msr(vcpu, MSR_CORE_PERF_GLOBAL_CTRL, vmcs12->guest_ia32_perf_global_ctrl))) { @@ -6830,6 +6826,7 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) struct kvm_x86_nested_ops vmx_nested_ops = { .leave_nested = vmx_leave_nested, .check_events = vmx_check_nested_events, + .handle_page_fault_workaround = nested_vmx_handle_page_fault_workaround, .hv_timer_pending = nested_vmx_preemption_timer_pending, .triple_fault = nested_vmx_triple_fault, .get_state = vmx_get_nested_state, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1bc67995cc8a..b2b17728a7a4 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -748,6 +748,7 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) } EXPORT_SYMBOL_GPL(kvm_inject_page_fault); +/* Returns true if the page fault was immediately morphed into a VM-Exit. */ bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) { @@ -766,8 +767,26 @@ bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address, fault_mmu->root.hpa); + /* + * A workaround for KVM's bad exception handling. If KVM injected an + * exception into L2, and L2 encountered a #PF while vectoring the + * injected exception, manually check to see if L1 wants to intercept + * #PF, otherwise queuing the #PF will lead to #DF or a lost exception. + * In all other cases, defer the check to nested_ops->check_events(), + * which will correctly handle priority (this does not). Note, other + * exceptions, e.g. #GP, are theoretically affected, #PF is simply the + * most problematic, e.g. when L0 and L1 are both intercepting #PF for + * shadow paging. + * + * TODO: Rewrite exception handling to track injected and pending + * (VM-Exit) exceptions separately. + */ + if (unlikely(vcpu->arch.exception.injected && is_guest_mode(vcpu)) && + kvm_x86_ops.nested_ops->handle_page_fault_workaround(vcpu, fault)) + return true; + fault_mmu->inject_page_fault(vcpu, fault); - return fault->nested_page_fault; + return false; } EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault); From b89805082adf1f502b1f0b993063e938e1bcb098 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:30:58 -0500 Subject: [PATCH 151/319] KVM: x86/mmu: remove "bool base_only" arguments The argument is always false now that kvm_mmu_calc_root_page_role has been removed. Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 66 +++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 43 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3e49902561f9..8b49b1e58a98 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4748,47 +4748,30 @@ static void paging32_init_context(struct kvm_mmu *context) context->direct_map = false; } -static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) -{ - union kvm_mmu_extended_role ext = {0}; - - if (____is_cr0_pg(regs)) { - ext.cr0_pg = 1; - ext.cr4_pae = ____is_cr4_pae(regs); - ext.cr4_smep = ____is_cr4_smep(regs); - ext.cr4_smap = ____is_cr4_smap(regs); - ext.cr4_pse = ____is_cr4_pse(regs); - - /* PKEY and LA57 are active iff long mode is active. */ - ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs); - ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs); - ext.efer_lma = ____is_efer_lma(regs); - } - - ext.valid = 1; - - return ext; -} - static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs, - bool base_only) + const struct kvm_mmu_role_regs *regs) { union kvm_mmu_role role = {0}; role.base.access = ACC_ALL; if (____is_cr0_pg(regs)) { + role.ext.cr0_pg = 1; role.base.efer_nx = ____is_efer_nx(regs); role.base.cr0_wp = ____is_cr0_wp(regs); + + role.ext.cr4_pae = ____is_cr4_pae(regs); + role.ext.cr4_smep = ____is_cr4_smep(regs); + role.ext.cr4_smap = ____is_cr4_smap(regs); + role.ext.cr4_pse = ____is_cr4_pse(regs); + + /* PKEY and LA57 are active iff long mode is active. */ + role.ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs); + role.ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs); + role.ext.efer_lma = ____is_efer_lma(regs); } role.base.smm = is_smm(vcpu); role.base.guest_mode = is_guest_mode(vcpu); - - if (base_only) - return role; - - role.ext = kvm_calc_mmu_role_ext(vcpu, regs); + role.ext.valid = 1; return role; } @@ -4808,10 +4791,9 @@ static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) static union kvm_mmu_role kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs, - bool base_only) + const struct kvm_mmu_role_regs *regs) { - union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only); + union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs); role.base.ad_disabled = (shadow_accessed_mask == 0); role.base.level = kvm_mmu_get_tdp_level(vcpu); @@ -4826,7 +4808,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, { struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_mmu_role new_role = - kvm_calc_tdp_mmu_root_page_role(vcpu, regs, false); + kvm_calc_tdp_mmu_root_page_role(vcpu, regs); if (new_role.as_u64 == context->mmu_role.as_u64) return; @@ -4855,10 +4837,9 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, static union kvm_mmu_role kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs, - bool base_only) + const struct kvm_mmu_role_regs *regs) { - union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only); + union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs); role.base.smep_andnot_wp = role.ext.cr4_smep && !____is_cr0_wp(regs); role.base.smap_andnot_wp = role.ext.cr4_smap && !____is_cr0_wp(regs); @@ -4869,11 +4850,10 @@ kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, static union kvm_mmu_role kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs, - bool base_only) + const struct kvm_mmu_role_regs *regs) { union kvm_mmu_role role = - kvm_calc_shadow_root_page_role_common(vcpu, regs, base_only); + kvm_calc_shadow_root_page_role_common(vcpu, regs); role.base.direct = !____is_cr0_pg(regs); @@ -4915,7 +4895,7 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, { struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_mmu_role new_role = - kvm_calc_shadow_mmu_root_page_role(vcpu, regs, false); + kvm_calc_shadow_mmu_root_page_role(vcpu, regs); shadow_mmu_init_context(vcpu, context, regs, new_role); } @@ -4925,7 +4905,7 @@ kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { union kvm_mmu_role role = - kvm_calc_shadow_root_page_role_common(vcpu, regs, false); + kvm_calc_shadow_root_page_role_common(vcpu, regs); role.base.direct = false; role.base.level = kvm_mmu_get_tdp_level(vcpu); @@ -5026,7 +5006,7 @@ kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs * { union kvm_mmu_role role; - role = kvm_calc_shadow_root_page_role_common(vcpu, regs, false); + role = kvm_calc_shadow_root_page_role_common(vcpu, regs); /* * Nested MMUs are used only for walking L2's gva->gpa, they never have From e5ed0fb01004f93ddf8a0c632cbc8a3f1ea5b518 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 11 Feb 2022 06:50:11 -0500 Subject: [PATCH 152/319] KVM: x86/mmu: split cpu_role from mmu_role Snapshot the state of the processor registers that govern page walk into a new field of struct kvm_mmu. This is a more natural representation than having it *mostly* in mmu_role but not exclusively; the delta right now is represented in other fields, such as root_level. The nested MMU now has only the CPU role; and in fact the new function kvm_calc_cpu_role is analogous to the previous kvm_calc_nested_mmu_role, except that it has role.base.direct equal to !CR0.PG. For a walk-only MMU, "direct" has no meaning, but we set it to !CR0.PG so that role.ext.cr0_pg can go away in a future patch. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/mmu/mmu.c | 109 ++++++++++++++++++++------------ arch/x86/kvm/mmu/paging_tmpl.h | 2 +- 3 files changed, 70 insertions(+), 42 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index e46bd289e5df..50edf52a3ef6 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -438,6 +438,7 @@ struct kvm_mmu { struct kvm_mmu_page *sp); void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa); struct kvm_mmu_root_info root; + union kvm_mmu_role cpu_role; union kvm_mmu_role mmu_role; u8 root_level; u8 shadow_root_level; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8b49b1e58a98..81c9e5e59a7e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -222,7 +222,7 @@ BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA); #define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name) \ static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu) \ { \ - return !!(mmu->mmu_role. base_or_ext . reg##_##name); \ + return !!(mmu->cpu_role. base_or_ext . reg##_##name); \ } BUILD_MMU_ROLE_ACCESSOR(ext, cr0, pg); BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp); @@ -4748,6 +4748,41 @@ static void paging32_init_context(struct kvm_mmu *context) context->direct_map = false; } +static union kvm_mmu_role +kvm_calc_cpu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) +{ + union kvm_mmu_role role = {0}; + + role.base.access = ACC_ALL; + role.base.smm = is_smm(vcpu); + role.base.guest_mode = is_guest_mode(vcpu); + role.ext.valid = 1; + + if (!____is_cr0_pg(regs)) { + role.base.direct = 1; + return role; + } + + role.base.efer_nx = ____is_efer_nx(regs); + role.base.cr0_wp = ____is_cr0_wp(regs); + role.base.smep_andnot_wp = ____is_cr4_smep(regs) && !____is_cr0_wp(regs); + role.base.smap_andnot_wp = ____is_cr4_smap(regs) && !____is_cr0_wp(regs); + role.base.has_4_byte_gpte = !____is_cr4_pae(regs); + role.base.level = role_regs_to_root_level(regs); + + role.ext.cr0_pg = 1; + role.ext.cr4_pae = ____is_cr4_pae(regs); + role.ext.cr4_smep = ____is_cr4_smep(regs); + role.ext.cr4_smap = ____is_cr4_smap(regs); + role.ext.cr4_pse = ____is_cr4_pse(regs); + + /* PKEY and LA57 are active iff long mode is active. */ + role.ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs); + role.ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs); + role.ext.efer_lma = ____is_efer_lma(regs); + return role; +} + static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { @@ -4807,13 +4842,16 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - union kvm_mmu_role new_role = + union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); + union kvm_mmu_role mmu_role = kvm_calc_tdp_mmu_root_page_role(vcpu, regs); - if (new_role.as_u64 == context->mmu_role.as_u64) + if (cpu_role.as_u64 == context->cpu_role.as_u64 && + mmu_role.as_u64 == context->mmu_role.as_u64) return; - context->mmu_role.as_u64 = new_role.as_u64; + context->cpu_role.as_u64 = cpu_role.as_u64; + context->mmu_role.as_u64 = mmu_role.as_u64; context->page_fault = kvm_tdp_page_fault; context->sync_page = nonpaging_sync_page; context->invlpg = NULL; @@ -4868,13 +4906,15 @@ kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, } static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context, - const struct kvm_mmu_role_regs *regs, - union kvm_mmu_role new_role) + union kvm_mmu_role cpu_role, + union kvm_mmu_role mmu_role) { - if (new_role.as_u64 == context->mmu_role.as_u64) + if (cpu_role.as_u64 == context->cpu_role.as_u64 && + mmu_role.as_u64 == context->mmu_role.as_u64) return; - context->mmu_role.as_u64 = new_role.as_u64; + context->cpu_role.as_u64 = cpu_role.as_u64; + context->mmu_role.as_u64 = mmu_role.as_u64; if (!is_cr0_pg(context)) nonpaging_init_context(context); @@ -4882,10 +4922,10 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte paging64_init_context(context); else paging32_init_context(context); - context->root_level = role_regs_to_root_level(regs); + context->root_level = cpu_role.base.level; reset_guest_paging_metadata(vcpu, context); - context->shadow_root_level = new_role.base.level; + context->shadow_root_level = mmu_role.base.level; reset_shadow_zero_bits_mask(vcpu, context); } @@ -4894,10 +4934,11 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - union kvm_mmu_role new_role = + union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); + union kvm_mmu_role mmu_role = kvm_calc_shadow_mmu_root_page_role(vcpu, regs); - shadow_mmu_init_context(vcpu, context, regs, new_role); + shadow_mmu_init_context(vcpu, context, cpu_role, mmu_role); } static union kvm_mmu_role @@ -4922,11 +4963,10 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, .cr4 = cr4 & ~X86_CR4_PKE, .efer = efer, }; - union kvm_mmu_role new_role; + union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s); + union kvm_mmu_role mmu_role = kvm_calc_shadow_npt_root_page_role(vcpu, ®s); - new_role = kvm_calc_shadow_npt_root_page_role(vcpu, ®s); - - shadow_mmu_init_context(vcpu, context, ®s, new_role); + shadow_mmu_init_context(vcpu, context, cpu_role, mmu_role); kvm_mmu_new_pgd(vcpu, nested_cr3); } EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu); @@ -4949,7 +4989,6 @@ kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, role.base.guest_mode = true; role.base.access = ACC_ALL; - /* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */ role.ext.word = 0; role.ext.execonly = execonly; role.ext.valid = 1; @@ -4963,12 +5002,14 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, { struct kvm_mmu *context = &vcpu->arch.guest_mmu; u8 level = vmx_eptp_page_walk_level(new_eptp); - union kvm_mmu_role new_role = + union kvm_mmu_role new_mode = kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, execonly, level); - if (new_role.as_u64 != context->mmu_role.as_u64) { - context->mmu_role.as_u64 = new_role.as_u64; + if (new_mode.as_u64 != context->cpu_role.as_u64) { + /* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */ + context->cpu_role.as_u64 = new_mode.as_u64; + context->mmu_role.as_u64 = new_mode.as_u64; context->shadow_root_level = level; @@ -5001,37 +5042,20 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu, context->inject_page_fault = kvm_inject_page_fault; } -static union kvm_mmu_role -kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) -{ - union kvm_mmu_role role; - - role = kvm_calc_shadow_root_page_role_common(vcpu, regs); - - /* - * Nested MMUs are used only for walking L2's gva->gpa, they never have - * shadow pages of their own and so "direct" has no meaning. Set it - * to "true" to try to detect bogus usage of the nested MMU. - */ - role.base.direct = true; - role.base.level = role_regs_to_root_level(regs); - return role; -} - static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { - union kvm_mmu_role new_role = kvm_calc_nested_mmu_role(vcpu, regs); + union kvm_mmu_role new_mode = kvm_calc_cpu_role(vcpu, regs); struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; - if (new_role.as_u64 == g_context->mmu_role.as_u64) + if (new_mode.as_u64 == g_context->cpu_role.as_u64) return; - g_context->mmu_role.as_u64 = new_role.as_u64; + g_context->cpu_role.as_u64 = new_mode.as_u64; g_context->get_guest_pgd = get_cr3; g_context->get_pdptr = kvm_pdptr_read; g_context->inject_page_fault = kvm_inject_page_fault; - g_context->root_level = new_role.base.level; + g_context->root_level = new_mode.base.level; /* * L2 page tables are never shadowed, so there is no need to sync @@ -5089,6 +5113,9 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) vcpu->arch.root_mmu.mmu_role.ext.valid = 0; vcpu->arch.guest_mmu.mmu_role.ext.valid = 0; vcpu->arch.nested_mmu.mmu_role.ext.valid = 0; + vcpu->arch.root_mmu.cpu_role.ext.valid = 0; + vcpu->arch.guest_mmu.cpu_role.ext.valid = 0; + vcpu->arch.nested_mmu.cpu_role.ext.valid = 0; kvm_mmu_reset_context(vcpu); /* diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 07a7832f96cb..56544c542d05 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -281,7 +281,7 @@ static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu, * is not reserved and does not indicate a large page at this level, * so clear PT_PAGE_SIZE_MASK in gpte if that is the case. */ - gpte &= level - (PT32_ROOT_LEVEL + mmu->mmu_role.ext.cr4_pse); + gpte &= level - (PT32_ROOT_LEVEL + mmu->cpu_role.ext.cr4_pse); #endif /* * PG_LEVEL_4K always terminates. The RHS has bit 7 set From 60f3cb60a59d3f155d7b5b7f8339754b2aba8815 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:32:40 -0500 Subject: [PATCH 153/319] KVM: x86/mmu: do not recompute root level from kvm_mmu_role_regs The root_level can be found in the cpu_role (in fact the field is superfluous and could be removed, but one thing at a time). Since there is only one usage left of role_regs_to_root_level, inline it into kvm_calc_cpu_role. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 81c9e5e59a7e..2ced494527f0 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -245,19 +245,6 @@ static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu) return regs; } -static int role_regs_to_root_level(const struct kvm_mmu_role_regs *regs) -{ - if (!____is_cr0_pg(regs)) - return 0; - else if (____is_efer_lma(regs)) - return ____is_cr4_la57(regs) ? PT64_ROOT_5LEVEL : - PT64_ROOT_4LEVEL; - else if (____is_cr4_pae(regs)) - return PT32E_ROOT_LEVEL; - else - return PT32_ROOT_LEVEL; -} - static inline bool kvm_available_flush_tlb_with_range(void) { return kvm_x86_ops.tlb_remote_flush_with_range; @@ -4768,7 +4755,14 @@ kvm_calc_cpu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) role.base.smep_andnot_wp = ____is_cr4_smep(regs) && !____is_cr0_wp(regs); role.base.smap_andnot_wp = ____is_cr4_smap(regs) && !____is_cr0_wp(regs); role.base.has_4_byte_gpte = !____is_cr4_pae(regs); - role.base.level = role_regs_to_root_level(regs); + + if (____is_efer_lma(regs)) + role.base.level = ____is_cr4_la57(regs) ? PT64_ROOT_5LEVEL + : PT64_ROOT_4LEVEL; + else if (____is_cr4_pae(regs)) + role.base.level = PT32E_ROOT_LEVEL; + else + role.base.level = PT32_ROOT_LEVEL; role.ext.cr0_pg = 1; role.ext.cr4_pae = ____is_cr4_pae(regs); @@ -4860,7 +4854,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; - context->root_level = role_regs_to_root_level(regs); + context->root_level = cpu_role.base.level; if (!is_cr0_pg(context)) context->gva_to_gpa = nonpaging_gva_to_gpa; From ec283cb1dcb934d1a861e5e25ce843e033ee4ab7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:32:51 -0500 Subject: [PATCH 154/319] KVM: x86/mmu: remove ept_ad field The ept_ad field is used during page walk to determine if the guest PTEs have accessed and dirty bits. In the MMU role, the ad_disabled bit represents whether the *shadow* PTEs have the bits, so it would be incorrect to replace PT_HAVE_ACCESSED_DIRTY with just !mmu->mmu_role.base.ad_disabled. However, the similar field in the CPU mode, ad_disabled, is initialized correctly: to the opposite value of ept_ad for shadow EPT, and zero for non-EPT guest paging modes (which always have A/D bits). It is therefore possible to compute PT_HAVE_ACCESSED_DIRTY from the CPU mode, like other page-format fields; it just has to be inverted to account for the different polarity. In fact, now that the CPU mode is distinct from the MMU roles, it would even be possible to remove PT_HAVE_ACCESSED_DIRTY macro altogether, and use !mmu->cpu_role.base.ad_disabled instead. I am not doing this because the macro has a small effect in terms of dead code elimination: text data bss dec hex 103544 16665 112 120321 1d601 # as of this patch 103746 16665 112 120523 1d6cb # without PT_HAVE_ACCESSED_DIRTY Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/mmu/mmu.c | 1 - arch/x86/kvm/mmu/paging_tmpl.h | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 50edf52a3ef6..a299236cfde5 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -442,7 +442,6 @@ struct kvm_mmu { union kvm_mmu_role mmu_role; u8 root_level; u8 shadow_root_level; - u8 ept_ad; bool direct_map; /* diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 2ced494527f0..86d4ee6ea0f9 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5007,7 +5007,6 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, context->shadow_root_level = level; - context->ept_ad = accessed_dirty; context->page_fault = ept_page_fault; context->gva_to_gpa = ept_gva_to_gpa; context->sync_page = ept_sync_page; diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 56544c542d05..298e502286cf 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -63,7 +63,7 @@ #define PT_LEVEL_BITS PT64_LEVEL_BITS #define PT_GUEST_DIRTY_SHIFT 9 #define PT_GUEST_ACCESSED_SHIFT 8 - #define PT_HAVE_ACCESSED_DIRTY(mmu) ((mmu)->ept_ad) + #define PT_HAVE_ACCESSED_DIRTY(mmu) (!(mmu)->cpu_role.base.ad_disabled) #ifdef CONFIG_X86_64 #define CMPXCHG "cmpxchgq" #endif From 19b5dcc3be2efe93760372f444dec44e77f67146 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:33:01 -0500 Subject: [PATCH 155/319] KVM: x86/mmu: remove kvm_calc_shadow_root_page_role_common kvm_calc_shadow_root_page_role_common is the same as kvm_calc_cpu_role except for the level, which is overwritten afterwards in kvm_calc_shadow_mmu_root_page_role and kvm_calc_shadow_npt_root_page_role. role.base.direct is already set correctly for the CPU role, and CR0.PG=1 is required for VMRUN so it will also be correct for nested NPT. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 86d4ee6ea0f9..f65b7e1a5758 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4867,28 +4867,14 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, reset_tdp_shadow_zero_bits_mask(context); } -static union kvm_mmu_role -kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) -{ - union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs); - - role.base.smep_andnot_wp = role.ext.cr4_smep && !____is_cr0_wp(regs); - role.base.smap_andnot_wp = role.ext.cr4_smap && !____is_cr0_wp(regs); - role.base.has_4_byte_gpte = ____is_cr0_pg(regs) && !____is_cr4_pae(regs); - - return role; -} - static union kvm_mmu_role kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { - union kvm_mmu_role role = - kvm_calc_shadow_root_page_role_common(vcpu, regs); - - role.base.direct = !____is_cr0_pg(regs); + union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); + union kvm_mmu_role role; + role = cpu_role; if (!____is_efer_lma(regs)) role.base.level = PT32E_ROOT_LEVEL; else if (____is_cr4_la57(regs)) @@ -4939,10 +4925,11 @@ static union kvm_mmu_role kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { - union kvm_mmu_role role = - kvm_calc_shadow_root_page_role_common(vcpu, regs); + union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); + union kvm_mmu_role role; - role.base.direct = false; + WARN_ON_ONCE(cpu_role.base.direct); + role = cpu_role; role.base.level = kvm_mmu_get_tdp_level(vcpu); return role; From 2ba676774dfc9369f67f30cd3b4f58ccda907c24 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:33:08 -0500 Subject: [PATCH 156/319] KVM: x86/mmu: cleanup computation of MMU roles for two-dimensional paging Inline kvm_calc_mmu_role_common into its sole caller, and simplify it by removing the computation of unnecessary bits. Extended bits are unnecessary because page walking uses the CPU role, and EFER.NX/CR0.WP can be set to one unconditionally---matching the format of shadow pages rather than the format of guest pages. The MMU role for two dimensional paging does still depend on the CPU role, even if only barely so, due to SMM and guest mode; for consistency, pass it down to kvm_calc_tdp_mmu_root_page_role instead of querying the vcpu with is_smm or is_guest_mode. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f65b7e1a5758..0b97aee6b735 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4777,34 +4777,6 @@ kvm_calc_cpu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) return role; } -static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) -{ - union kvm_mmu_role role = {0}; - - role.base.access = ACC_ALL; - if (____is_cr0_pg(regs)) { - role.ext.cr0_pg = 1; - role.base.efer_nx = ____is_efer_nx(regs); - role.base.cr0_wp = ____is_cr0_wp(regs); - - role.ext.cr4_pae = ____is_cr4_pae(regs); - role.ext.cr4_smep = ____is_cr4_smep(regs); - role.ext.cr4_smap = ____is_cr4_smap(regs); - role.ext.cr4_pse = ____is_cr4_pse(regs); - - /* PKEY and LA57 are active iff long mode is active. */ - role.ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs); - role.ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs); - role.ext.efer_lma = ____is_efer_lma(regs); - } - role.base.smm = is_smm(vcpu); - role.base.guest_mode = is_guest_mode(vcpu); - role.ext.valid = 1; - - return role; -} - static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) { /* tdp_root_level is architecture forced level, use it if nonzero */ @@ -4820,14 +4792,20 @@ static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) static union kvm_mmu_role kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) + union kvm_mmu_role cpu_role) { - union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs); + union kvm_mmu_role role = {0}; + role.base.access = ACC_ALL; + role.base.cr0_wp = true; + role.base.efer_nx = true; + role.base.smm = cpu_role.base.smm; + role.base.guest_mode = cpu_role.base.guest_mode; role.base.ad_disabled = (shadow_accessed_mask == 0); role.base.level = kvm_mmu_get_tdp_level(vcpu); role.base.direct = true; role.base.has_4_byte_gpte = false; + role.ext.valid = true; return role; } @@ -4837,8 +4815,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, { struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); - union kvm_mmu_role mmu_role = - kvm_calc_tdp_mmu_root_page_role(vcpu, regs); + union kvm_mmu_role mmu_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role); if (cpu_role.as_u64 == context->cpu_role.as_u64 && mmu_role.as_u64 == context->mmu_role.as_u64) From f417e1459a1299bb4984cbffc03d8746cab9f8a8 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:33:26 -0500 Subject: [PATCH 157/319] KVM: x86/mmu: cleanup computation of MMU roles for shadow paging Pass the already-computed CPU role, instead of redoing it. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 0b97aee6b735..3f4777849040 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4846,15 +4846,14 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, static union kvm_mmu_role kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) + union kvm_mmu_role cpu_role) { - union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); union kvm_mmu_role role; role = cpu_role; - if (!____is_efer_lma(regs)) + if (!cpu_role.ext.efer_lma) role.base.level = PT32E_ROOT_LEVEL; - else if (____is_cr4_la57(regs)) + else if (cpu_role.ext.cr4_la57) role.base.level = PT64_ROOT_5LEVEL; else role.base.level = PT64_ROOT_4LEVEL; @@ -4893,16 +4892,15 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); union kvm_mmu_role mmu_role = - kvm_calc_shadow_mmu_root_page_role(vcpu, regs); + kvm_calc_shadow_mmu_root_page_role(vcpu, cpu_role); shadow_mmu_init_context(vcpu, context, cpu_role, mmu_role); } static union kvm_mmu_role kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) + union kvm_mmu_role cpu_role) { - union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); union kvm_mmu_role role; WARN_ON_ONCE(cpu_role.base.direct); @@ -4922,7 +4920,7 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, .efer = efer, }; union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s); - union kvm_mmu_role mmu_role = kvm_calc_shadow_npt_root_page_role(vcpu, ®s); + union kvm_mmu_role mmu_role = kvm_calc_shadow_npt_root_page_role(vcpu, cpu_role); shadow_mmu_init_context(vcpu, context, cpu_role, mmu_role); kvm_mmu_new_pgd(vcpu, nested_cr3); From 362505deb8e2ce27993a1dee0b3c5755d4d6646e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:33:38 -0500 Subject: [PATCH 158/319] KVM: x86/mmu: store shadow EFER.NX in the MMU role Now that the MMU role is separate from the CPU role, it can be a truthful description of the format of the shadow pages. This includes whether the shadow pages use the NX bit; so force the efer_nx field of the MMU role when TDP is disabled, and remove the hardcoding it in the callers of reset_shadow_zero_bits_mask. In fact, the initialization of reserved SPTE bits can now be made common to shadow paging and shadow NPT; move it to shadow_mmu_init_context. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3f4777849040..6a18660616f4 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4473,16 +4473,6 @@ static inline u64 reserved_hpa_bits(void) static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context) { - /* - * KVM uses NX when TDP is disabled to handle a variety of scenarios, - * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and - * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0. - * The iTLB multi-hit workaround can be toggled at any time, so assume - * NX can be used by any non-nested shadow MMU to avoid having to reset - * MMU contexts. Note, KVM forces EFER.NX=1 when TDP is disabled. - */ - bool uses_nx = is_efer_nx(context) || !tdp_enabled; - /* @amd adds a check on bit of SPTEs, which KVM shouldn't use anyways. */ bool is_amd = true; /* KVM doesn't use 2-level page tables for the shadow MMU. */ @@ -4494,7 +4484,8 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, shadow_zero_check = &context->shadow_zero_check; __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(), - context->shadow_root_level, uses_nx, + context->shadow_root_level, + context->mmu_role.base.efer_nx, guest_can_use_gbpages(vcpu), is_pse, is_amd); if (!shadow_me_mask) @@ -4858,6 +4849,16 @@ kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, else role.base.level = PT64_ROOT_4LEVEL; + /* + * KVM forces EFER.NX=1 when TDP is disabled, reflect it in the MMU role. + * KVM uses NX when TDP is disabled to handle a variety of scenarios, + * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and + * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0. + * The iTLB multi-hit workaround can be toggled at any time, so assume + * NX can be used by any non-nested shadow MMU to avoid having to reset + * MMU contexts. + */ + role.base.efer_nx = true; return role; } From 7a458f0e1ba150a6ea012171a43c4b947f1d825d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 14 Feb 2022 08:46:24 -0500 Subject: [PATCH 159/319] KVM: x86/mmu: remove extended bits from mmu_role, rename field mmu_role represents the role of the root of the page tables. It does not need any extended bits, as those govern only KVM's page table walking; the is_* functions used for page table walking always use the CPU role. ext.valid is not present anymore in the MMU role, but an all-zero MMU role is impossible because the level field is never zero in the MMU role. So just zap the whole mmu_role in order to force invalidation after CPUID is updated. While making this change, which requires touching almost every occurrence of "mmu_role", rename it to "root_role". Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/mmu/mmu.c | 86 ++++++++++++++++----------------- arch/x86/kvm/mmu/paging_tmpl.h | 4 +- arch/x86/kvm/mmu/tdp_mmu.c | 2 +- 4 files changed, 46 insertions(+), 48 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a299236cfde5..c81221d03a1b 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -439,7 +439,7 @@ struct kvm_mmu { void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa); struct kvm_mmu_root_info root; union kvm_mmu_role cpu_role; - union kvm_mmu_role mmu_role; + union kvm_mmu_page_role root_role; u8 root_level; u8 shadow_root_level; bool direct_map; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 6a18660616f4..9ddec2f9c5e3 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -193,7 +193,7 @@ struct kvm_mmu_role_regs { /* * Yes, lot's of underscores. They're a hint that you probably shouldn't be - * reading from the role_regs. Once the mmu_role is constructed, it becomes + * reading from the role_regs. Once the root_role is constructed, it becomes * the single source of truth for the MMU's state. */ #define BUILD_MMU_ROLE_REGS_ACCESSOR(reg, name, flag) \ @@ -2028,7 +2028,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, int collisions = 0; LIST_HEAD(invalid_list); - role = vcpu->arch.mmu->mmu_role.base; + role = vcpu->arch.mmu->root_role; role.level = level; role.direct = direct; role.access = access; @@ -3315,7 +3315,7 @@ void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu) * This should not be called while L2 is active, L2 can't invalidate * _only_ its own roots, e.g. INVVPID unconditionally exits. */ - WARN_ON_ONCE(mmu->mmu_role.base.guest_mode); + WARN_ON_ONCE(mmu->root_role.guest_mode); for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { root_hpa = mmu->prev_roots[i].hpa; @@ -4226,7 +4226,7 @@ static bool fast_pgd_switch(struct kvm *kvm, struct kvm_mmu *mmu, void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd) { struct kvm_mmu *mmu = vcpu->arch.mmu; - union kvm_mmu_page_role new_role = mmu->mmu_role.base; + union kvm_mmu_page_role new_role = mmu->root_role; if (!fast_pgd_switch(vcpu->kvm, mmu, new_pgd, new_role)) { /* kvm_mmu_ensure_valid_pgd will set up a new root. */ @@ -4485,7 +4485,7 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, shadow_zero_check = &context->shadow_zero_check; __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(), context->shadow_root_level, - context->mmu_role.base.efer_nx, + context->root_role.efer_nx, guest_can_use_gbpages(vcpu), is_pse, is_amd); if (!shadow_me_mask) @@ -4781,22 +4781,21 @@ static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) return max_tdp_level; } -static union kvm_mmu_role +static union kvm_mmu_page_role kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, union kvm_mmu_role cpu_role) { - union kvm_mmu_role role = {0}; + union kvm_mmu_page_role role = {0}; - role.base.access = ACC_ALL; - role.base.cr0_wp = true; - role.base.efer_nx = true; - role.base.smm = cpu_role.base.smm; - role.base.guest_mode = cpu_role.base.guest_mode; - role.base.ad_disabled = (shadow_accessed_mask == 0); - role.base.level = kvm_mmu_get_tdp_level(vcpu); - role.base.direct = true; - role.base.has_4_byte_gpte = false; - role.ext.valid = true; + role.access = ACC_ALL; + role.cr0_wp = true; + role.efer_nx = true; + role.smm = cpu_role.base.smm; + role.guest_mode = cpu_role.base.guest_mode; + role.ad_disabled = (shadow_accessed_mask == 0); + role.level = kvm_mmu_get_tdp_level(vcpu); + role.direct = true; + role.has_4_byte_gpte = false; return role; } @@ -4806,14 +4805,14 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, { struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); - union kvm_mmu_role mmu_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role); + union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role); if (cpu_role.as_u64 == context->cpu_role.as_u64 && - mmu_role.as_u64 == context->mmu_role.as_u64) + root_role.word == context->root_role.word) return; context->cpu_role.as_u64 = cpu_role.as_u64; - context->mmu_role.as_u64 = mmu_role.as_u64; + context->root_role.word = root_role.word; context->page_fault = kvm_tdp_page_fault; context->sync_page = nonpaging_sync_page; context->invlpg = NULL; @@ -4835,19 +4834,19 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, reset_tdp_shadow_zero_bits_mask(context); } -static union kvm_mmu_role +static union kvm_mmu_page_role kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, union kvm_mmu_role cpu_role) { - union kvm_mmu_role role; + union kvm_mmu_page_role role; - role = cpu_role; + role = cpu_role.base; if (!cpu_role.ext.efer_lma) - role.base.level = PT32E_ROOT_LEVEL; + role.level = PT32E_ROOT_LEVEL; else if (cpu_role.ext.cr4_la57) - role.base.level = PT64_ROOT_5LEVEL; + role.level = PT64_ROOT_5LEVEL; else - role.base.level = PT64_ROOT_4LEVEL; + role.level = PT64_ROOT_4LEVEL; /* * KVM forces EFER.NX=1 when TDP is disabled, reflect it in the MMU role. @@ -4858,20 +4857,20 @@ kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, * NX can be used by any non-nested shadow MMU to avoid having to reset * MMU contexts. */ - role.base.efer_nx = true; + role.efer_nx = true; return role; } static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context, union kvm_mmu_role cpu_role, - union kvm_mmu_role mmu_role) + union kvm_mmu_page_role root_role) { if (cpu_role.as_u64 == context->cpu_role.as_u64 && - mmu_role.as_u64 == context->mmu_role.as_u64) + root_role.word == context->root_role.word) return; context->cpu_role.as_u64 = cpu_role.as_u64; - context->mmu_role.as_u64 = mmu_role.as_u64; + context->root_role.word = root_role.word; if (!is_cr0_pg(context)) nonpaging_init_context(context); @@ -4882,7 +4881,7 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte context->root_level = cpu_role.base.level; reset_guest_paging_metadata(vcpu, context); - context->shadow_root_level = mmu_role.base.level; + context->shadow_root_level = root_role.level; reset_shadow_zero_bits_mask(vcpu, context); } @@ -4892,22 +4891,21 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, { struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); - union kvm_mmu_role mmu_role = + union kvm_mmu_page_role root_role = kvm_calc_shadow_mmu_root_page_role(vcpu, cpu_role); - shadow_mmu_init_context(vcpu, context, cpu_role, mmu_role); + shadow_mmu_init_context(vcpu, context, cpu_role, root_role); } -static union kvm_mmu_role +static union kvm_mmu_page_role kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu, union kvm_mmu_role cpu_role) { - union kvm_mmu_role role; + union kvm_mmu_page_role role; WARN_ON_ONCE(cpu_role.base.direct); - role = cpu_role; - role.base.level = kvm_mmu_get_tdp_level(vcpu); - + role = cpu_role.base; + role.level = kvm_mmu_get_tdp_level(vcpu); return role; } @@ -4921,9 +4919,9 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, .efer = efer, }; union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s); - union kvm_mmu_role mmu_role = kvm_calc_shadow_npt_root_page_role(vcpu, cpu_role); + union kvm_mmu_page_role root_role = kvm_calc_shadow_npt_root_page_role(vcpu, cpu_role); - shadow_mmu_init_context(vcpu, context, cpu_role, mmu_role); + shadow_mmu_init_context(vcpu, context, cpu_role, root_role); kvm_mmu_new_pgd(vcpu, nested_cr3); } EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu); @@ -4966,7 +4964,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, if (new_mode.as_u64 != context->cpu_role.as_u64) { /* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */ context->cpu_role.as_u64 = new_mode.as_u64; - context->mmu_role.as_u64 = new_mode.as_u64; + context->root_role.word = new_mode.base.word; context->shadow_root_level = level; @@ -5066,9 +5064,9 @@ void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) * problem is swept under the rug; KVM's CPUID API is horrific and * it's all but impossible to solve it without introducing a new API. */ - vcpu->arch.root_mmu.mmu_role.ext.valid = 0; - vcpu->arch.guest_mmu.mmu_role.ext.valid = 0; - vcpu->arch.nested_mmu.mmu_role.ext.valid = 0; + vcpu->arch.root_mmu.root_role.word = 0; + vcpu->arch.guest_mmu.root_role.word = 0; + vcpu->arch.nested_mmu.root_role.word = 0; vcpu->arch.root_mmu.cpu_role.ext.valid = 0; vcpu->arch.guest_mmu.cpu_role.ext.valid = 0; vcpu->arch.nested_mmu.cpu_role.ext.valid = 0; diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 298e502286cf..24157f637bd7 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -988,7 +988,7 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, */ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { - union kvm_mmu_page_role mmu_role = vcpu->arch.mmu->mmu_role.base; + union kvm_mmu_page_role root_role = vcpu->arch.mmu->root_role; int i; bool host_writable; gpa_t first_pte_gpa; @@ -1016,7 +1016,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) * reserved bits checks will be wrong, etc... */ if (WARN_ON_ONCE(sp->role.direct || - (sp->role.word ^ mmu_role.word) & ~sync_role_ign.word)) + (sp->role.word ^ root_role.word) & ~sync_role_ign.word)) return -1; first_pte_gpa = FNAME(get_level1_sp_gpa)(sp); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index edc68538819b..fe0d688af93a 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -310,7 +310,7 @@ static void tdp_mmu_init_child_sp(struct kvm_mmu_page *child_sp, hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu) { - union kvm_mmu_page_role role = vcpu->arch.mmu->mmu_role.base; + union kvm_mmu_page_role role = vcpu->arch.mmu->root_role; struct kvm *kvm = vcpu->kvm; struct kvm_mmu_page *root; From 7a7ae8292391c4d53c4340e606bf48776c3449e7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:38:32 -0500 Subject: [PATCH 160/319] KVM: x86/mmu: rename kvm_mmu_role union It is quite confusing that the "full" union is called kvm_mmu_role but is used for the "cpu_role" field of struct kvm_mmu. Rename it to kvm_cpu_role. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 6 +++--- arch/x86/kvm/mmu/mmu.c | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index c81221d03a1b..6bc5550ae530 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -281,7 +281,7 @@ struct kvm_kernel_irq_routing_entry; /* * kvm_mmu_page_role tracks the properties of a shadow page (where shadow page * also includes TDP pages) to determine whether or not a page can be used in - * the given MMU context. This is a subset of the overall kvm_mmu_role to + * the given MMU context. This is a subset of the overall kvm_cpu_role to * minimize the size of kvm_memory_slot.arch.gfn_track, i.e. allows allocating * 2 bytes per gfn instead of 4 bytes per gfn. * @@ -378,7 +378,7 @@ union kvm_mmu_extended_role { }; }; -union kvm_mmu_role { +union kvm_cpu_role { u64 as_u64; struct { union kvm_mmu_page_role base; @@ -438,7 +438,7 @@ struct kvm_mmu { struct kvm_mmu_page *sp); void (*invlpg)(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root_hpa); struct kvm_mmu_root_info root; - union kvm_mmu_role cpu_role; + union kvm_cpu_role cpu_role; union kvm_mmu_page_role root_role; u8 root_level; u8 shadow_root_level; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 9ddec2f9c5e3..810c9e5854fe 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4726,10 +4726,10 @@ static void paging32_init_context(struct kvm_mmu *context) context->direct_map = false; } -static union kvm_mmu_role +static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { - union kvm_mmu_role role = {0}; + union kvm_cpu_role role = {0}; role.base.access = ACC_ALL; role.base.smm = is_smm(vcpu); @@ -4783,7 +4783,7 @@ static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) static union kvm_mmu_page_role kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, - union kvm_mmu_role cpu_role) + union kvm_cpu_role cpu_role) { union kvm_mmu_page_role role = {0}; @@ -4804,7 +4804,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); + union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role); if (cpu_role.as_u64 == context->cpu_role.as_u64 && @@ -4836,7 +4836,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, static union kvm_mmu_page_role kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, - union kvm_mmu_role cpu_role) + union kvm_cpu_role cpu_role) { union kvm_mmu_page_role role; @@ -4862,7 +4862,7 @@ kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, } static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context, - union kvm_mmu_role cpu_role, + union kvm_cpu_role cpu_role, union kvm_mmu_page_role root_role) { if (cpu_role.as_u64 == context->cpu_role.as_u64 && @@ -4890,7 +4890,7 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); + union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); union kvm_mmu_page_role root_role = kvm_calc_shadow_mmu_root_page_role(vcpu, cpu_role); @@ -4899,7 +4899,7 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, static union kvm_mmu_page_role kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu, - union kvm_mmu_role cpu_role) + union kvm_cpu_role cpu_role) { union kvm_mmu_page_role role; @@ -4918,7 +4918,7 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, .cr4 = cr4 & ~X86_CR4_PKE, .efer = efer, }; - union kvm_mmu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s); + union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s); union kvm_mmu_page_role root_role = kvm_calc_shadow_npt_root_page_role(vcpu, cpu_role); shadow_mmu_init_context(vcpu, context, cpu_role, root_role); @@ -4926,11 +4926,11 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, } EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu); -static union kvm_mmu_role +static union kvm_cpu_role kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, bool execonly, u8 level) { - union kvm_mmu_role role = {0}; + union kvm_cpu_role role = {0}; /* * KVM does not support SMM transfer monitors, and consequently does not @@ -4957,7 +4957,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, { struct kvm_mmu *context = &vcpu->arch.guest_mmu; u8 level = vmx_eptp_page_walk_level(new_eptp); - union kvm_mmu_role new_mode = + union kvm_cpu_role new_mode = kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, execonly, level); @@ -4999,7 +4999,7 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu, static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) { - union kvm_mmu_role new_mode = kvm_calc_cpu_role(vcpu, regs); + union kvm_cpu_role new_mode = kvm_calc_cpu_role(vcpu, regs); struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; if (new_mode.as_u64 == g_context->cpu_role.as_u64) @@ -6276,7 +6276,7 @@ int kvm_mmu_vendor_module_init(void) */ BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32)); BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32)); - BUILD_BUG_ON(sizeof(union kvm_mmu_role) != sizeof(u64)); + BUILD_BUG_ON(sizeof(union kvm_cpu_role) != sizeof(u64)); kvm_mmu_reset_all_pte_masks(); From faf729621c9609367a0714f5383df67fdd8d021c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:38:51 -0500 Subject: [PATCH 161/319] KVM: x86/mmu: remove redundant bits from extended role Before the separation of the CPU and the MMU role, CR0.PG was not available in the base MMU role, because two-dimensional paging always used direct=1 in the MMU role. However, now that the raw role is snapshotted in mmu->cpu_role, the value of CR0.PG always matches both !cpu_role.base.direct and cpu_role.base.level > 0. There is no need to store it again in union kvm_mmu_extended_role; instead, write an is_cr0_pg accessor by hand that takes care of the conversion. Use cpu_role.base.level since the future of the direct field is unclear. Likewise, CR4.PAE is now always present in the CPU role as !cpu_role.base.has_4_byte_gpte. The inversion makes certain tests on the MMU role easier, and is easily hidden by the is_cr4_pae accessor when operating on the CPU role. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 -- arch/x86/kvm/mmu/mmu.c | 14 ++++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 6bc5550ae530..52ceeadbed28 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -367,8 +367,6 @@ union kvm_mmu_extended_role { struct { unsigned int valid:1; unsigned int execonly:1; - unsigned int cr0_pg:1; - unsigned int cr4_pae:1; unsigned int cr4_pse:1; unsigned int cr4_pke:1; unsigned int cr4_smap:1; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 810c9e5854fe..e8188668d9f4 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -224,16 +224,24 @@ static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu) \ { \ return !!(mmu->cpu_role. base_or_ext . reg##_##name); \ } -BUILD_MMU_ROLE_ACCESSOR(ext, cr0, pg); BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp); BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pse); -BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pae); BUILD_MMU_ROLE_ACCESSOR(ext, cr4, smep); BUILD_MMU_ROLE_ACCESSOR(ext, cr4, smap); BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pke); BUILD_MMU_ROLE_ACCESSOR(ext, cr4, la57); BUILD_MMU_ROLE_ACCESSOR(base, efer, nx); +static inline bool is_cr0_pg(struct kvm_mmu *mmu) +{ + return mmu->cpu_role.base.level > 0; +} + +static inline bool is_cr4_pae(struct kvm_mmu *mmu) +{ + return !mmu->cpu_role.base.has_4_byte_gpte; +} + static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu) { struct kvm_mmu_role_regs regs = { @@ -4755,8 +4763,6 @@ kvm_calc_cpu_role(struct kvm_vcpu *vcpu, const struct kvm_mmu_role_regs *regs) else role.base.level = PT32_ROOT_LEVEL; - role.ext.cr0_pg = 1; - role.ext.cr4_pae = ____is_cr4_pae(regs); role.ext.cr4_smep = ____is_cr4_smep(regs); role.ext.cr4_smap = ____is_cr4_smap(regs); role.ext.cr4_pse = ____is_cr4_pse(regs); From 56b321f9e332082b9c5f7519495f068b147fe2c0 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:39:50 -0500 Subject: [PATCH 162/319] KVM: x86/mmu: simplify and/or inline computation of shadow MMU roles Shadow MMUs compute their role from cpu_role.base, simply by adjusting the root level. It's one line of code, so do not place it in a separate function. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 67 ++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 42 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index e8188668d9f4..3a535f682fcd 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -231,6 +231,7 @@ BUILD_MMU_ROLE_ACCESSOR(ext, cr4, smap); BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pke); BUILD_MMU_ROLE_ACCESSOR(ext, cr4, la57); BUILD_MMU_ROLE_ACCESSOR(base, efer, nx); +BUILD_MMU_ROLE_ACCESSOR(ext, efer, lma); static inline bool is_cr0_pg(struct kvm_mmu *mmu) { @@ -4840,33 +4841,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, reset_tdp_shadow_zero_bits_mask(context); } -static union kvm_mmu_page_role -kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, - union kvm_cpu_role cpu_role) -{ - union kvm_mmu_page_role role; - - role = cpu_role.base; - if (!cpu_role.ext.efer_lma) - role.level = PT32E_ROOT_LEVEL; - else if (cpu_role.ext.cr4_la57) - role.level = PT64_ROOT_5LEVEL; - else - role.level = PT64_ROOT_4LEVEL; - - /* - * KVM forces EFER.NX=1 when TDP is disabled, reflect it in the MMU role. - * KVM uses NX when TDP is disabled to handle a variety of scenarios, - * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and - * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0. - * The iTLB multi-hit workaround can be toggled at any time, so assume - * NX can be used by any non-nested shadow MMU to avoid having to reset - * MMU contexts. - */ - role.efer_nx = true; - return role; -} - static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context, union kvm_cpu_role cpu_role, union kvm_mmu_page_role root_role) @@ -4897,24 +4871,27 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, { struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); - union kvm_mmu_page_role root_role = - kvm_calc_shadow_mmu_root_page_role(vcpu, cpu_role); + union kvm_mmu_page_role root_role; + + root_role = cpu_role.base; + + /* KVM uses PAE paging whenever the guest isn't using 64-bit paging. */ + root_role.level = max_t(u32, root_role.level, PT32E_ROOT_LEVEL); + + /* + * KVM forces EFER.NX=1 when TDP is disabled, reflect it in the MMU role. + * KVM uses NX when TDP is disabled to handle a variety of scenarios, + * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and + * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0. + * The iTLB multi-hit workaround can be toggled at any time, so assume + * NX can be used by any non-nested shadow MMU to avoid having to reset + * MMU contexts. + */ + root_role.efer_nx = true; shadow_mmu_init_context(vcpu, context, cpu_role, root_role); } -static union kvm_mmu_page_role -kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu, - union kvm_cpu_role cpu_role) -{ - union kvm_mmu_page_role role; - - WARN_ON_ONCE(cpu_role.base.direct); - role = cpu_role.base; - role.level = kvm_mmu_get_tdp_level(vcpu); - return role; -} - void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, unsigned long cr4, u64 efer, gpa_t nested_cr3) { @@ -4925,7 +4902,13 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, .efer = efer, }; union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s); - union kvm_mmu_page_role root_role = kvm_calc_shadow_npt_root_page_role(vcpu, cpu_role); + union kvm_mmu_page_role root_role; + + /* NPT requires CR0.PG=1. */ + WARN_ON_ONCE(cpu_role.base.direct); + + root_role = cpu_role.base; + root_role.level = kvm_mmu_get_tdp_level(vcpu); shadow_mmu_init_context(vcpu, context, cpu_role, root_role); kvm_mmu_new_pgd(vcpu, nested_cr3); From a7f1de9b6066cc62e0142302d8021731d56b9559 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:57:21 -0500 Subject: [PATCH 163/319] KVM: x86/mmu: pull CPU mode computation to kvm_init_mmu Do not lead init_kvm_*mmu into the temptation of poking into struct kvm_mmu_role_regs, by passing to it directly the CPU mode. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3a535f682fcd..4f43078a2b40 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4808,10 +4808,9 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, } static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) + union kvm_cpu_role cpu_role) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); union kvm_mmu_page_role root_role = kvm_calc_tdp_mmu_root_page_role(vcpu, cpu_role); if (cpu_role.as_u64 == context->cpu_role.as_u64 && @@ -4867,10 +4866,9 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte } static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) + union kvm_cpu_role cpu_role) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, regs); union kvm_mmu_page_role root_role; root_role = cpu_role.base; @@ -4974,11 +4972,11 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu); static void init_kvm_softmmu(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) + union kvm_cpu_role cpu_role) { struct kvm_mmu *context = &vcpu->arch.root_mmu; - kvm_init_shadow_mmu(vcpu, regs); + kvm_init_shadow_mmu(vcpu, cpu_role); context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; @@ -4986,9 +4984,8 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu, } static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu, - const struct kvm_mmu_role_regs *regs) + union kvm_cpu_role new_mode) { - union kvm_cpu_role new_mode = kvm_calc_cpu_role(vcpu, regs); struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; if (new_mode.as_u64 == g_context->cpu_role.as_u64) @@ -5029,13 +5026,14 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu, void kvm_init_mmu(struct kvm_vcpu *vcpu) { struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); + union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, ®s); if (mmu_is_nested(vcpu)) - init_kvm_nested_mmu(vcpu, ®s); + init_kvm_nested_mmu(vcpu, cpu_role); else if (tdp_enabled) - init_kvm_tdp_mmu(vcpu, ®s); + init_kvm_tdp_mmu(vcpu, cpu_role); else - init_kvm_softmmu(vcpu, ®s); + init_kvm_softmmu(vcpu, cpu_role); } EXPORT_SYMBOL_GPL(kvm_init_mmu); From a972e29c1d6c957242a23b42f355b1fdf721cbd4 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:41:19 -0500 Subject: [PATCH 164/319] KVM: x86/mmu: replace shadow_root_level with root_role.level root_role.level is always the same value as shadow_level: - it's kvm_mmu_get_tdp_level(vcpu) when going through init_kvm_tdp_mmu - it's the level argument when going through kvm_init_shadow_ept_mmu - it's assigned directly from new_role.base.level when going through shadow_mmu_init_context Remove the duplication and get the level directly from the role. Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/mmu.h | 2 +- arch/x86/kvm/mmu/mmu.c | 33 ++++++++++++++------------------- arch/x86/kvm/mmu/tdp_mmu.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 52ceeadbed28..35056341799d 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -439,7 +439,6 @@ struct kvm_mmu { union kvm_cpu_role cpu_role; union kvm_mmu_page_role root_role; u8 root_level; - u8 shadow_root_level; bool direct_map; /* diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index a335e7f1f69e..c242e26cfa80 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -138,7 +138,7 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu) return; static_call(kvm_x86_load_mmu_pgd)(vcpu, root_hpa, - vcpu->arch.mmu->shadow_root_level); + vcpu->arch.mmu->root_role.level); } struct kvm_page_fault { diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 4f43078a2b40..fee202e2b98e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2129,7 +2129,7 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato { iterator->addr = addr; iterator->shadow_addr = root; - iterator->level = vcpu->arch.mmu->shadow_root_level; + iterator->level = vcpu->arch.mmu->root_role.level; if (iterator->level >= PT64_ROOT_4LEVEL && vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL && @@ -3367,7 +3367,7 @@ static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva, static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) { struct kvm_mmu *mmu = vcpu->arch.mmu; - u8 shadow_root_level = mmu->shadow_root_level; + u8 shadow_root_level = mmu->root_role.level; hpa_t root; unsigned i; int r; @@ -3517,7 +3517,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) */ if (mmu->root_level >= PT64_ROOT_4LEVEL) { root = mmu_alloc_root(vcpu, root_gfn, 0, - mmu->shadow_root_level, false); + mmu->root_role.level, false); mmu->root.hpa = root; goto set_root_pgd; } @@ -3533,7 +3533,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * the shadow page table may be a PAE or a long mode page table. */ pm_mask = PT_PRESENT_MASK | shadow_me_mask; - if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { + if (mmu->root_role.level >= PT64_ROOT_4LEVEL) { pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; if (WARN_ON_ONCE(!mmu->pml4_root)) { @@ -3542,7 +3542,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) } mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask; - if (mmu->shadow_root_level == PT64_ROOT_5LEVEL) { + if (mmu->root_role.level == PT64_ROOT_5LEVEL) { if (WARN_ON_ONCE(!mmu->pml5_root)) { r = -EIO; goto out_unlock; @@ -3567,9 +3567,9 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) mmu->pae_root[i] = root | pm_mask; } - if (mmu->shadow_root_level == PT64_ROOT_5LEVEL) + if (mmu->root_role.level == PT64_ROOT_5LEVEL) mmu->root.hpa = __pa(mmu->pml5_root); - else if (mmu->shadow_root_level == PT64_ROOT_4LEVEL) + else if (mmu->root_role.level == PT64_ROOT_4LEVEL) mmu->root.hpa = __pa(mmu->pml4_root); else mmu->root.hpa = __pa(mmu->pae_root); @@ -3585,7 +3585,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu) { struct kvm_mmu *mmu = vcpu->arch.mmu; - bool need_pml5 = mmu->shadow_root_level > PT64_ROOT_4LEVEL; + bool need_pml5 = mmu->root_role.level > PT64_ROOT_4LEVEL; u64 *pml5_root = NULL; u64 *pml4_root = NULL; u64 *pae_root; @@ -3597,7 +3597,7 @@ static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu) * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare. */ if (mmu->direct_map || mmu->root_level >= PT64_ROOT_4LEVEL || - mmu->shadow_root_level < PT64_ROOT_4LEVEL) + mmu->root_role.level < PT64_ROOT_4LEVEL) return 0; /* @@ -4489,18 +4489,18 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct rsvd_bits_validate *shadow_zero_check; int i; - WARN_ON_ONCE(context->shadow_root_level < PT32E_ROOT_LEVEL); + WARN_ON_ONCE(context->root_role.level < PT32E_ROOT_LEVEL); shadow_zero_check = &context->shadow_zero_check; __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(), - context->shadow_root_level, + context->root_role.level, context->root_role.efer_nx, guest_can_use_gbpages(vcpu), is_pse, is_amd); if (!shadow_me_mask) return; - for (i = context->shadow_root_level; --i >= 0;) { + for (i = context->root_role.level; --i >= 0;) { shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; } @@ -4527,7 +4527,7 @@ reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context) if (boot_cpu_is_amd()) __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(), - context->shadow_root_level, false, + context->root_role.level, false, boot_cpu_has(X86_FEATURE_GBPAGES), false, true); else @@ -4538,7 +4538,7 @@ reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context) if (!shadow_me_mask) return; - for (i = context->shadow_root_level; --i >= 0;) { + for (i = context->root_role.level; --i >= 0;) { shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; } @@ -4822,7 +4822,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, context->page_fault = kvm_tdp_page_fault; context->sync_page = nonpaging_sync_page; context->invlpg = NULL; - context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu); context->direct_map = true; context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; @@ -4860,8 +4859,6 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte context->root_level = cpu_role.base.level; reset_guest_paging_metadata(vcpu, context); - context->shadow_root_level = root_role.level; - reset_shadow_zero_bits_mask(vcpu, context); } @@ -4953,8 +4950,6 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, context->cpu_role.as_u64 = new_mode.as_u64; context->root_role.word = new_mode.base.word; - context->shadow_root_level = level; - context->page_fault = ept_page_fault; context->gva_to_gpa = ept_gva_to_gpa; context->sync_page = ept_sync_page; diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index fe0d688af93a..4fabb2cd0ba9 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1835,7 +1835,7 @@ int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, gfn_t gfn = addr >> PAGE_SHIFT; int leaf = -1; - *root_level = vcpu->arch.mmu->shadow_root_level; + *root_level = vcpu->arch.mmu->root_role.level; tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) { leaf = iter.level; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index b83af9ecf85a..75b4f3ac8b1a 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3951,7 +3951,7 @@ static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, hv_track_root_tdp(vcpu, root_hpa); cr3 = vcpu->arch.cr3; - } else if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { + } else if (vcpu->arch.mmu->root_role.level >= PT64_ROOT_4LEVEL) { cr3 = __sme_set(root_hpa) | kvm_get_active_pcid(vcpu); } else { /* PCID in the guest should be impossible with a 32-bit MMU. */ diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index df0b70ccd289..cf8581978bce 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2948,7 +2948,7 @@ static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) if (enable_ept) ept_sync_context(construct_eptp(vcpu, root_hpa, - mmu->shadow_root_level)); + mmu->root_role.level)); else vpid_sync_context(vmx_get_current_vpid(vcpu)); } From 4d25502aa12ef1fb01e599cbfd341a8d436f4b8b Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 07:42:22 -0500 Subject: [PATCH 165/319] KVM: x86/mmu: replace root_level with cpu_role.base.level Remove another duplicate field of struct kvm_mmu. This time it's the root level for page table walking; the separate field is always initialized as cpu_role.base.level, so its users can look up the CPU mode directly instead. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/mmu/mmu.c | 18 +++++++----------- arch/x86/kvm/mmu/paging_tmpl.h | 4 ++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 35056341799d..aea39fe03a99 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -438,7 +438,6 @@ struct kvm_mmu { struct kvm_mmu_root_info root; union kvm_cpu_role cpu_role; union kvm_mmu_page_role root_role; - u8 root_level; bool direct_map; /* diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index fee202e2b98e..9752f713b025 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2132,7 +2132,7 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato iterator->level = vcpu->arch.mmu->root_role.level; if (iterator->level >= PT64_ROOT_4LEVEL && - vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL && + vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL && !vcpu->arch.mmu->direct_map) iterator->level = PT32E_ROOT_LEVEL; @@ -3491,7 +3491,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * On SVM, reading PDPTRs might access guest memory, which might fault * and thus might sleep. Grab the PDPTRs before acquiring mmu_lock. */ - if (mmu->root_level == PT32E_ROOT_LEVEL) { + if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) { for (i = 0; i < 4; ++i) { pdptrs[i] = mmu->get_pdptr(vcpu, i); if (!(pdptrs[i] & PT_PRESENT_MASK)) @@ -3515,7 +3515,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * Do we shadow a long mode page table? If so we need to * write-protect the guests page table root. */ - if (mmu->root_level >= PT64_ROOT_4LEVEL) { + if (mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) { root = mmu_alloc_root(vcpu, root_gfn, 0, mmu->root_role.level, false); mmu->root.hpa = root; @@ -3554,7 +3554,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) for (i = 0; i < 4; ++i) { WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i])); - if (mmu->root_level == PT32E_ROOT_LEVEL) { + if (mmu->cpu_role.base.level == PT32E_ROOT_LEVEL) { if (!(pdptrs[i] & PT_PRESENT_MASK)) { mmu->pae_root[i] = INVALID_PAE_ROOT; continue; @@ -3596,7 +3596,7 @@ static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu) * equivalent level in the guest's NPT to shadow. Allocate the tables * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare. */ - if (mmu->direct_map || mmu->root_level >= PT64_ROOT_4LEVEL || + if (mmu->direct_map || mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL || mmu->root_role.level < PT64_ROOT_4LEVEL) return 0; @@ -3701,7 +3701,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); - if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) { + if (vcpu->arch.mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL) { hpa_t root = vcpu->arch.mmu->root.hpa; sp = to_shadow_page(root); @@ -4417,7 +4417,7 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, { __reset_rsvds_bits_mask(&context->guest_rsvd_check, vcpu->arch.reserved_gpa_bits, - context->root_level, is_efer_nx(context), + context->cpu_role.base.level, is_efer_nx(context), guest_can_use_gbpages(vcpu), is_cr4_pse(context), guest_cpuid_is_amd_or_hygon(vcpu)); @@ -4826,7 +4826,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; - context->root_level = cpu_role.base.level; if (!is_cr0_pg(context)) context->gva_to_gpa = nonpaging_gva_to_gpa; @@ -4856,7 +4855,6 @@ static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *conte paging64_init_context(context); else paging32_init_context(context); - context->root_level = cpu_role.base.level; reset_guest_paging_metadata(vcpu, context); reset_shadow_zero_bits_mask(vcpu, context); @@ -4954,7 +4952,6 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, context->gva_to_gpa = ept_gva_to_gpa; context->sync_page = ept_sync_page; context->invlpg = ept_invlpg; - context->root_level = level; context->direct_map = false; update_permission_bitmask(context, true); context->pkru_mask = 0; @@ -4990,7 +4987,6 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu, g_context->get_guest_pgd = get_cr3; g_context->get_pdptr = kvm_pdptr_read; g_context->inject_page_fault = kvm_inject_page_fault; - g_context->root_level = new_mode.base.level; /* * L2 page tables are never shadowed, so there is no need to sync diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 24157f637bd7..66f1acf153c4 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -319,7 +319,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, trace_kvm_mmu_pagetable_walk(addr, access); retry_walk: - walker->level = mmu->root_level; + walker->level = mmu->cpu_role.base.level; pte = mmu->get_guest_pgd(vcpu); have_ad = PT_HAVE_ACCESSED_DIRTY(mmu); @@ -621,7 +621,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, WARN_ON_ONCE(gw->gfn != base_gfn); direct_access = gw->pte_access; - top_level = vcpu->arch.mmu->root_level; + top_level = vcpu->arch.mmu->cpu_role.base.level; if (top_level == PT32E_ROOT_LEVEL) top_level = PT32_ROOT_LEVEL; /* From 347a0d0ded16a2e59c35b43ace7ad2b53fb6df57 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 10 Feb 2022 08:00:56 -0500 Subject: [PATCH 166/319] KVM: x86/mmu: replace direct_map with root_role.direct direct_map is always equal to the direct field of the root page's role: - for shadow paging, direct_map is true if CR0.PG=0 and root_role.direct is copied from cpu_role.base.direct - for TDP, it is always true and root_role.direct is also always true - for shadow TDP, it is always false and root_role.direct is also always false Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/mmu/mmu.c | 27 ++++++++++++--------------- arch/x86/kvm/x86.c | 12 ++++++------ 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index aea39fe03a99..752a6d2357ce 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -438,7 +438,6 @@ struct kvm_mmu { struct kvm_mmu_root_info root; union kvm_cpu_role cpu_role; union kvm_mmu_page_role root_role; - bool direct_map; /* * The pkru_mask indicates if protection key checks are needed. It diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 9752f713b025..085e9b82f71d 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2028,7 +2028,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, int direct, unsigned int access) { - bool direct_mmu = vcpu->arch.mmu->direct_map; + bool direct_mmu = vcpu->arch.mmu->root_role.direct; union kvm_mmu_page_role role; struct hlist_head *sp_list; unsigned quadrant; @@ -2133,7 +2133,7 @@ static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterato if (iterator->level >= PT64_ROOT_4LEVEL && vcpu->arch.mmu->cpu_role.base.level < PT64_ROOT_4LEVEL && - !vcpu->arch.mmu->direct_map) + !vcpu->arch.mmu->root_role.direct) iterator->level = PT32E_ROOT_LEVEL; if (iterator->level == PT32E_ROOT_LEVEL) { @@ -2515,7 +2515,7 @@ static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) gpa_t gpa; int r; - if (vcpu->arch.mmu->direct_map) + if (vcpu->arch.mmu->root_role.direct) return 0; gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL); @@ -3596,7 +3596,8 @@ static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu) * equivalent level in the guest's NPT to shadow. Allocate the tables * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare. */ - if (mmu->direct_map || mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL || + if (mmu->root_role.direct || + mmu->cpu_role.base.level >= PT64_ROOT_4LEVEL || mmu->root_role.level < PT64_ROOT_4LEVEL) return 0; @@ -3693,7 +3694,7 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) int i; struct kvm_mmu_page *sp; - if (vcpu->arch.mmu->direct_map) + if (vcpu->arch.mmu->root_role.direct) return; if (!VALID_PAGE(vcpu->arch.mmu->root.hpa)) @@ -3923,7 +3924,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, arch.token = alloc_apf_token(vcpu); arch.gfn = gfn; - arch.direct_map = vcpu->arch.mmu->direct_map; + arch.direct_map = vcpu->arch.mmu->root_role.direct; arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); return kvm_setup_async_pf(vcpu, cr2_or_gpa, @@ -4141,7 +4142,6 @@ static void nonpaging_init_context(struct kvm_mmu *context) context->gva_to_gpa = nonpaging_gva_to_gpa; context->sync_page = nonpaging_sync_page; context->invlpg = NULL; - context->direct_map = true; } static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd, @@ -4723,7 +4723,6 @@ static void paging64_init_context(struct kvm_mmu *context) context->gva_to_gpa = paging64_gva_to_gpa; context->sync_page = paging64_sync_page; context->invlpg = paging64_invlpg; - context->direct_map = false; } static void paging32_init_context(struct kvm_mmu *context) @@ -4732,7 +4731,6 @@ static void paging32_init_context(struct kvm_mmu *context) context->gva_to_gpa = paging32_gva_to_gpa; context->sync_page = paging32_sync_page; context->invlpg = paging32_invlpg; - context->direct_map = false; } static union kvm_cpu_role @@ -4822,7 +4820,6 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu, context->page_fault = kvm_tdp_page_fault; context->sync_page = nonpaging_sync_page; context->invlpg = NULL; - context->direct_map = true; context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; @@ -4952,7 +4949,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, context->gva_to_gpa = ept_gva_to_gpa; context->sync_page = ept_sync_page; context->invlpg = ept_invlpg; - context->direct_map = false; + update_permission_bitmask(context, true); context->pkru_mask = 0; reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level); @@ -5068,13 +5065,13 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) { int r; - r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->direct_map); + r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->root_role.direct); if (r) goto out; r = mmu_alloc_special_roots(vcpu); if (r) goto out; - if (vcpu->arch.mmu->direct_map) + if (vcpu->arch.mmu->root_role.direct) r = mmu_alloc_direct_roots(vcpu); else r = mmu_alloc_shadow_roots(vcpu); @@ -5331,7 +5328,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, void *insn, int insn_len) { int r, emulation_type = EMULTYPE_PF; - bool direct = vcpu->arch.mmu->direct_map; + bool direct = vcpu->arch.mmu->root_role.direct; if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root.hpa))) return RET_PF_RETRY; @@ -5362,7 +5359,7 @@ int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, * paging in both guests. If true, we simply unprotect the page * and resume the guest. */ - if (vcpu->arch.mmu->direct_map && + if (vcpu->arch.mmu->root_role.direct && (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) { kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa)); return 1; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index b2b17728a7a4..ee774c734066 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8101,7 +8101,7 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, WARN_ON_ONCE(!(emulation_type & EMULTYPE_PF))) return false; - if (!vcpu->arch.mmu->direct_map) { + if (!vcpu->arch.mmu->root_role.direct) { /* * Write permission should be allowed since only * write access need to be emulated. @@ -8134,7 +8134,7 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, kvm_release_pfn_clean(pfn); /* The instructions are well-emulated on direct mmu. */ - if (vcpu->arch.mmu->direct_map) { + if (vcpu->arch.mmu->root_role.direct) { unsigned int indirect_shadow_pages; write_lock(&vcpu->kvm->mmu_lock); @@ -8202,7 +8202,7 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt, vcpu->arch.last_retry_eip = ctxt->eip; vcpu->arch.last_retry_addr = cr2_or_gpa; - if (!vcpu->arch.mmu->direct_map) + if (!vcpu->arch.mmu->root_role.direct) gpa = kvm_mmu_gva_to_gpa_write(vcpu, cr2_or_gpa, NULL); kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(gpa)); @@ -8482,7 +8482,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, ctxt->exception.address = cr2_or_gpa; /* With shadow page tables, cr2 contains a GVA or nGPA. */ - if (vcpu->arch.mmu->direct_map) { + if (vcpu->arch.mmu->root_role.direct) { ctxt->gpa_available = true; ctxt->gpa_val = cr2_or_gpa; } @@ -12366,7 +12366,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) { int r; - if ((vcpu->arch.mmu->direct_map != work->arch.direct_map) || + if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) || work->wakeup_all) return; @@ -12374,7 +12374,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) if (unlikely(r)) return; - if (!vcpu->arch.mmu->direct_map && + if (!vcpu->arch.mmu->root_role.direct && work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu)) return; From 7223fd2d5338d5eeef1775abd3440f8b499f3c0a Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 20 Apr 2022 10:49:53 -0500 Subject: [PATCH 167/319] KVM: SVM: Use target APIC ID to complete AVIC IRQs when possible Currently, an AVIC-enabled VM suffers from performance bottleneck when scaling to large number of vCPUs for I/O intensive workloads. In such case, a vCPU often executes halt instruction to get into idle state waiting for interrupts, in which KVM would de-schedule the vCPU from physical CPU. When AVIC HW tries to deliver interrupt to the halting vCPU, it would result in AVIC incomplete IPI #vmexit to notify KVM to reschedule the target vCPU into running state. Investigation has shown the main hotspot is in the kvm_apic_match_dest() in the following call stack where it tries to find target vCPUs corresponding to the information in the ICRH/ICRL registers. - handle_exit - svm_invoke_exit_handler - avic_incomplete_ipi_interception - kvm_apic_match_dest However, AVIC provides hints in the #vmexit info, which can be used to retrieve the destination guest physical APIC ID. In addition, since QEMU defines guest physical APIC ID to be the same as vCPU ID, it can be used to quickly identify the target vCPU to deliver IPI, and avoid the overhead from searching through all vCPUs to match the target vCPU. Reviewed-by: Maxim Levitsky Signed-off-by: Suravee Suthikulpanit Signed-off-by: Paolo Bonzini Message-Id: <20220420154954.19305-2-suravee.suthikulpanit@amd.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/avic.c | 72 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 9b859218af59..7249d766519f 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -284,11 +284,75 @@ void avic_ring_doorbell(struct kvm_vcpu *vcpu) put_cpu(); } -static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, - u32 icrl, u32 icrh) +/* + * A fast-path version of avic_kick_target_vcpus(), which attempts to match + * destination APIC ID to vCPU without looping through all vCPUs. + */ +static int avic_kick_target_vcpus_fast(struct kvm *kvm, struct kvm_lapic *source, + u32 icrl, u32 icrh, u32 index) { + u32 dest, apic_id; struct kvm_vcpu *vcpu; + int dest_mode = icrl & APIC_DEST_MASK; + int shorthand = icrl & APIC_SHORT_MASK; + struct kvm_svm *kvm_svm = to_kvm_svm(kvm); + u32 *avic_logical_id_table = page_address(kvm_svm->avic_logical_id_table_page); + + if (shorthand != APIC_DEST_NOSHORT) + return -EINVAL; + + /* + * The AVIC incomplete IPI #vmexit info provides index into + * the physical APIC ID table, which can be used to derive + * guest physical APIC ID. + */ + if (dest_mode == APIC_DEST_PHYSICAL) { + apic_id = index; + } else { + if (!apic_x2apic_mode(source)) { + /* For xAPIC logical mode, the index is for logical APIC table. */ + apic_id = avic_logical_id_table[index] & 0x1ff; + } else { + return -EINVAL; + } + } + + /* + * Assuming vcpu ID is the same as physical apic ID, + * and use it to retrieve the target vCPU. + */ + vcpu = kvm_get_vcpu_by_id(kvm, apic_id); + if (!vcpu) + return -EINVAL; + + if (apic_x2apic_mode(vcpu->arch.apic)) + dest = icrh; + else + dest = GET_APIC_DEST_FIELD(icrh); + + /* + * Try matching the destination APIC ID with the vCPU. + */ + if (kvm_apic_match_dest(vcpu, source, shorthand, dest, dest_mode)) { + vcpu->arch.apic->irr_pending = true; + svm_complete_interrupt_delivery(vcpu, + icrl & APIC_MODE_MASK, + icrl & APIC_INT_LEVELTRIG, + icrl & APIC_VECTOR_MASK); + return 0; + } + + return -EINVAL; +} + +static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, + u32 icrl, u32 icrh, u32 index) +{ unsigned long i; + struct kvm_vcpu *vcpu; + + if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index)) + return; /* * Wake any target vCPUs that are blocking, i.e. waiting for a wake @@ -315,7 +379,7 @@ int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) u32 icrh = svm->vmcb->control.exit_info_1 >> 32; u32 icrl = svm->vmcb->control.exit_info_1; u32 id = svm->vmcb->control.exit_info_2 >> 32; - u32 index = svm->vmcb->control.exit_info_2 & 0xFF; + u32 index = svm->vmcb->control.exit_info_2 & 0x1FF; struct kvm_lapic *apic = vcpu->arch.apic; trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index); @@ -342,7 +406,7 @@ int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu) * set the appropriate IRR bits on the valid target * vcpus. So, we just need to kick the appropriate vcpu. */ - avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh); + avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index); break; case AVIC_IPI_FAILURE_INVALID_TARGET: break; From 9f084f7c2ed7e6b8cc3176b9ece2f6cb0641c385 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 20 Apr 2022 10:49:54 -0500 Subject: [PATCH 168/319] KVM: SVM: Introduce trace point for the slow-path of avic_kic_target_vcpus This can help identify potential performance issues when handles AVIC incomplete IPI due vCPU not running. Reviewed-by: Maxim Levitsky Signed-off-by: Suravee Suthikulpanit Signed-off-by: Paolo Bonzini Message-Id: <20220420154954.19305-3-suravee.suthikulpanit@amd.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/avic.c | 2 ++ arch/x86/kvm/trace.h | 20 ++++++++++++++++++++ arch/x86/kvm/x86.c | 1 + 3 files changed, 23 insertions(+) diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 7249d766519f..54fe03714f8a 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -354,6 +354,8 @@ static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source, if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index)) return; + trace_kvm_avic_kick_vcpu_slowpath(icrh, icrl, index); + /* * Wake any target vCPUs that are blocking, i.e. waiting for a wake * event. There's no need to signal doorbells, as hardware has handled diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index e3a24b8f04be..de4762517569 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -1459,6 +1459,26 @@ TRACE_EVENT(kvm_avic_ga_log, __entry->vmid, __entry->vcpuid) ); +TRACE_EVENT(kvm_avic_kick_vcpu_slowpath, + TP_PROTO(u32 icrh, u32 icrl, u32 index), + TP_ARGS(icrh, icrl, index), + + TP_STRUCT__entry( + __field(u32, icrh) + __field(u32, icrl) + __field(u32, index) + ), + + TP_fast_assign( + __entry->icrh = icrh; + __entry->icrl = icrl; + __entry->index = index; + ), + + TP_printk("icrh:icrl=%#08x:%08x, index=%u", + __entry->icrh, __entry->icrl, __entry->index) +); + TRACE_EVENT(kvm_hv_timer_state, TP_PROTO(unsigned int vcpu_id, unsigned int hv_timer_in_use), TP_ARGS(vcpu_id, hv_timer_in_use), diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ee774c734066..1abce22b14d7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -13072,6 +13072,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pi_irte_update); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_unaccelerated_access); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_incomplete_ipi); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_ga_log); +EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_avic_kick_vcpu_slowpath); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_apicv_accept_irq); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_enter); EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_vmgexit_exit); From 767d8d8d503f946c7a67735e77dea7e1c9cb6b51 Mon Sep 17 00:00:00 2001 From: Lai Jiangshan Date: Wed, 20 Apr 2022 21:12:03 +0800 Subject: [PATCH 169/319] KVM: X86/MMU: Add sp_has_gptes() Add sp_has_gptes() which equals to !sp->role.direct currently. Shadow page having gptes needs to be write-protected, accounted and responded to kvm_mmu_pte_write(). Use it in these places to replace !sp->role.direct and rename for_each_gfn_indirect_valid_sp. Signed-off-by: Lai Jiangshan Message-Id: <20220420131204.2850-2-jiangshanlai@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 085e9b82f71d..caa208a3fcdc 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1853,15 +1853,23 @@ static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, static void kvm_mmu_commit_zap_page(struct kvm *kvm, struct list_head *invalid_list); +static bool sp_has_gptes(struct kvm_mmu_page *sp) +{ + if (sp->role.direct) + return false; + + return true; +} + #define for_each_valid_sp(_kvm, _sp, _list) \ hlist_for_each_entry(_sp, _list, hash_link) \ if (is_obsolete_sp((_kvm), (_sp))) { \ } else -#define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn) \ +#define for_each_gfn_valid_sp_with_gptes(_kvm, _sp, _gfn) \ for_each_valid_sp(_kvm, _sp, \ &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)]) \ - if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else + if ((_sp)->gfn != (_gfn) || !sp_has_gptes(_sp)) {} else static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, struct list_head *invalid_list) @@ -2109,7 +2117,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, sp->gfn = gfn; sp->role = role; hlist_add_head(&sp->hash_link, sp_list); - if (!direct) { + if (sp_has_gptes(sp)) { account_shadowed(vcpu->kvm, sp); if (level == PG_LEVEL_4K && kvm_vcpu_write_protect_gfn(vcpu, gfn)) kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1); @@ -2318,7 +2326,7 @@ static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm, /* Zapping children means active_mmu_pages has become unstable. */ list_unstable = *nr_zapped; - if (!sp->role.invalid && !sp->role.direct) + if (!sp->role.invalid && sp_has_gptes(sp)) unaccount_shadowed(kvm, sp); if (sp->unsync) @@ -2498,7 +2506,7 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) pgprintk("%s: looking for gfn %llx\n", __func__, gfn); r = 0; write_lock(&kvm->mmu_lock); - for_each_gfn_indirect_valid_sp(kvm, sp, gfn) { + for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) { pgprintk("%s: gfn %llx role %x\n", __func__, gfn, sp->role.word); r = 1; @@ -2560,7 +2568,7 @@ int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot, * that case, KVM must complete emulation of the guest TLB flush before * allowing shadow pages to become unsync (writable by the guest). */ - for_each_gfn_indirect_valid_sp(kvm, sp, gfn) { + for_each_gfn_valid_sp_with_gptes(kvm, sp, gfn) { if (!can_unsync) return -EPERM; @@ -5298,7 +5306,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, ++vcpu->kvm->stat.mmu_pte_write; - for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { + for_each_gfn_valid_sp_with_gptes(vcpu->kvm, sp, gfn) { if (detect_write_misaligned(sp, gpa, bytes) || detect_write_flooding(sp)) { kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list); From 84e5ffd045f33e4fa32370135436d987478d0bf7 Mon Sep 17 00:00:00 2001 From: Lai Jiangshan Date: Wed, 20 Apr 2022 21:12:04 +0800 Subject: [PATCH 170/319] KVM: X86/MMU: Fix shadowing 5-level NPT for 4-level NPT L1 guest When shadowing 5-level NPT for 4-level NPT L1 guest, the root_sp is allocated with role.level = 5 and the guest pagetable's root gfn. And root_sp->spt[0] is also allocated with the same gfn and the same role except role.level = 4. Luckily that they are different shadow pages, but only root_sp->spt[0] is the real translation of the guest pagetable. Here comes a problem: If the guest switches from gCR4_LA57=0 to gCR4_LA57=1 (or vice verse) and uses the same gfn as the root page for nested NPT before and after switching gCR4_LA57. The host (hCR4_LA57=1) might use the same root_sp for the guest even the guest switches gCR4_LA57. The guest will see unexpected page mapped and L2 may exploit the bug and hurt L1. It is lucky that the problem can't hurt L0. And three special cases need to be handled: The root_sp should be like role.direct=1 sometimes: its contents are not backed by gptes, root_sp->gfns is meaningless. (For a normal high level sp in shadow paging, sp->gfns is often unused and kept zero, but it could be relevant and meaningful if sp->gfns is used because they are backed by concrete gptes.) For such root_sp in the case, root_sp is just a portal to contribute root_sp->spt[0], and root_sp->gfns should not be used and root_sp->spt[0] should not be dropped if gpte[0] of the guest root pagetable is changed. Such root_sp should not be accounted too. So add role.passthrough to distinguish the shadow pages in the hash when gCR4_LA57 is toggled and fix above special cases by using it in kvm_mmu_page_{get|set}_gfn() and sp_has_gptes(). Signed-off-by: Lai Jiangshan Message-Id: <20220420131204.2850-3-jiangshanlai@gmail.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/x86/mmu.rst | 4 ++++ arch/x86/include/asm/kvm_host.h | 5 +++-- arch/x86/kvm/mmu/mmu.c | 16 ++++++++++++++++ arch/x86/kvm/mmu/paging_tmpl.h | 1 + 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/Documentation/virt/kvm/x86/mmu.rst b/Documentation/virt/kvm/x86/mmu.rst index 5b1ebad24c77..8739120f4300 100644 --- a/Documentation/virt/kvm/x86/mmu.rst +++ b/Documentation/virt/kvm/x86/mmu.rst @@ -202,6 +202,10 @@ Shadow pages contain the following information: Is 1 if the MMU instance cannot use A/D bits. EPT did not have A/D bits before Haswell; shadow EPT page tables also cannot use A/D bits if the L1 hypervisor does not enable them. + role.passthrough: + The page is not backed by a guest page table, but its first entry + points to one. This is set if NPT uses 5-level page tables (host + CR4.LA57=1) and is shadowing L1's 4-level NPT (L1 CR4.LA57=1). gfn: Either the guest page table containing the translations shadowed by this page, or the base page frame for linear translations. See role.direct. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 752a6d2357ce..f164c6c1514a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -285,7 +285,7 @@ struct kvm_kernel_irq_routing_entry; * minimize the size of kvm_memory_slot.arch.gfn_track, i.e. allows allocating * 2 bytes per gfn instead of 4 bytes per gfn. * - * Indirect upper-level shadow pages are tracked for write-protection via + * Upper-level shadow pages having gptes are tracked for write-protection via * gfn_track. As above, gfn_track is a 16 bit counter, so KVM must not create * more than 2^16-1 upper-level shadow pages at a single gfn, otherwise * gfn_track will overflow and explosions will ensure. @@ -331,7 +331,8 @@ union kvm_mmu_page_role { unsigned smap_andnot_wp:1; unsigned ad_disabled:1; unsigned guest_mode:1; - unsigned :6; + unsigned passthrough:1; + unsigned :5; /* * This is left at the top of the word so that diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index caa208a3fcdc..abdce06f6880 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -734,6 +734,9 @@ static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc) static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index) { + if (sp->role.passthrough) + return sp->gfn; + if (!sp->role.direct) return sp->gfns[index]; @@ -742,6 +745,11 @@ static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index) static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn) { + if (sp->role.passthrough) { + WARN_ON_ONCE(gfn != sp->gfn); + return; + } + if (!sp->role.direct) { sp->gfns[index] = gfn; return; @@ -1858,6 +1866,9 @@ static bool sp_has_gptes(struct kvm_mmu_page *sp) if (sp->role.direct) return false; + if (sp->role.passthrough) + return false; + return true; } @@ -2054,6 +2065,8 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1; role.quadrant = quadrant; } + if (level <= vcpu->arch.mmu->cpu_role.base.level) + role.passthrough = 0; sp_list = &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)]; for_each_valid_sp(vcpu->kvm, sp, sp_list) { @@ -4907,6 +4920,9 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, root_role = cpu_role.base; root_role.level = kvm_mmu_get_tdp_level(vcpu); + if (root_role.level == PT64_ROOT_5LEVEL && + cpu_role.base.level == PT64_ROOT_4LEVEL) + root_role.passthrough = 1; shadow_mmu_init_context(vcpu, context, cpu_role, root_role); kvm_mmu_new_pgd(vcpu, nested_cr3); diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 66f1acf153c4..b025decf610d 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -1007,6 +1007,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) .level = 0xf, .access = 0x7, .quadrant = 0x3, + .passthrough = 0x1, }; /* From 2e9cf8458d3fcf0823dd02d3f9ffa2c62d5f6a2c Mon Sep 17 00:00:00 2001 From: Alexander Shiyan Date: Wed, 20 Apr 2022 09:51:02 +0300 Subject: [PATCH 171/319] gpio: syscon: Remove usage of syscon_regmap_lookup_by_compatible Since version 5.13, the standard syscon bindings have been added to all clps711x DT nodes, so we can now use the more general syscon_regmap_lookup_by_phandle function to get the syscon pointer. This patch removes the usage of the syscon_regmap_lookup_by_compatible function as it is no longer used in the driver. Signed-off-by: Alexander Shiyan Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-syscon.c | 41 ++++++++++++++------------------------ 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/drivers/gpio/gpio-syscon.c b/drivers/gpio/gpio-syscon.c index fdd3d497b535..6076937b18e7 100644 --- a/drivers/gpio/gpio-syscon.c +++ b/drivers/gpio/gpio-syscon.c @@ -38,7 +38,6 @@ */ struct syscon_gpio_data { - const char *compatible; unsigned int flags; unsigned int bit_count; unsigned int dat_bit_offset; @@ -125,7 +124,6 @@ static int syscon_gpio_dir_out(struct gpio_chip *chip, unsigned offset, int val) static const struct syscon_gpio_data clps711x_mctrl_gpio = { /* ARM CLPS711X SYSFLG1 Bits 8-10 */ - .compatible = "cirrus,ep7209-syscon1", .flags = GPIO_SYSCON_FEAT_IN, .bit_count = 3, .dat_bit_offset = 0x40 * 8 + 8, @@ -182,7 +180,6 @@ static void keystone_gpio_set(struct gpio_chip *chip, unsigned offset, int val) static const struct syscon_gpio_data keystone_dsp_gpio = { /* ARM Keystone 2 */ - .compatible = NULL, .flags = GPIO_SYSCON_FEAT_OUT, .bit_count = 28, .dat_bit_offset = 4, @@ -219,33 +216,25 @@ static int syscon_gpio_probe(struct platform_device *pdev) priv->data = of_device_get_match_data(dev); - if (priv->data->compatible) { - priv->syscon = syscon_regmap_lookup_by_compatible( - priv->data->compatible); - if (IS_ERR(priv->syscon)) - return PTR_ERR(priv->syscon); - } else { - priv->syscon = - syscon_regmap_lookup_by_phandle(np, "gpio,syscon-dev"); - if (IS_ERR(priv->syscon) && np->parent) - priv->syscon = syscon_node_to_regmap(np->parent); - if (IS_ERR(priv->syscon)) - return PTR_ERR(priv->syscon); + priv->syscon = syscon_regmap_lookup_by_phandle(np, "gpio,syscon-dev"); + if (IS_ERR(priv->syscon) && np->parent) + priv->syscon = syscon_node_to_regmap(np->parent); + if (IS_ERR(priv->syscon)) + return PTR_ERR(priv->syscon); - ret = of_property_read_u32_index(np, "gpio,syscon-dev", 1, - &priv->dreg_offset); - if (ret) - dev_err(dev, "can't read the data register offset!\n"); + ret = of_property_read_u32_index(np, "gpio,syscon-dev", 1, + &priv->dreg_offset); + if (ret) + dev_err(dev, "can't read the data register offset!\n"); - priv->dreg_offset <<= 3; + priv->dreg_offset <<= 3; - ret = of_property_read_u32_index(np, "gpio,syscon-dev", 2, - &priv->dir_reg_offset); - if (ret) - dev_dbg(dev, "can't read the dir register offset!\n"); + ret = of_property_read_u32_index(np, "gpio,syscon-dev", 2, + &priv->dir_reg_offset); + if (ret) + dev_dbg(dev, "can't read the dir register offset!\n"); - priv->dir_reg_offset <<= 3; - } + priv->dir_reg_offset <<= 3; priv->chip.parent = dev; priv->chip.owner = THIS_MODULE; From 30a35c07d9e9affaebd557c454df98e5ba269776 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 25 Apr 2022 19:11:35 +0800 Subject: [PATCH 172/319] gpio: vf610: drop the SOC_VF610 dependency for GPIO_VF610 i.MX7ULP, i.MX8 and i.MX9 use this driver, so drop the SOC_VF610 dependcy to make the driver could be built for i.MX platform. Signed-off-by: Peng Fan Signed-off-by: Bartosz Golaszewski --- drivers/gpio/Kconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 57426fdba1b6..b01961999ced 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -675,10 +675,10 @@ config GPIO_UNIPHIER config GPIO_VF610 def_bool y - depends on ARCH_MXC && SOC_VF610 + depends on ARCH_MXC select GPIOLIB_IRQCHIP help - Say yes here to support Vybrid vf610 GPIOs. + Say yes here to support i.MX or Vybrid vf610 GPIOs. config GPIO_VISCONTI tristate "Toshiba Visconti GPIO support" From 6fcee03df6a1a3101a77344be37bb85c6142d56c Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 22 Mar 2022 19:24:42 +0200 Subject: [PATCH 173/319] KVM: x86: avoid loading a vCPU after .vm_destroy was called This can cause various unexpected issues, since VM is partially destroyed at that point. For example when AVIC is enabled, this causes avic_vcpu_load to access physical id page entry which is already freed by .vm_destroy. Fixes: 8221c1370056 ("svm: Manage vcpu load/unload when enable AVIC") Cc: stable@vger.kernel.org Signed-off-by: Maxim Levitsky Message-Id: <20220322172449.235575-2-mlevitsk@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1abce22b14d7..ba4faeb32437 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -11816,20 +11816,15 @@ static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) vcpu_put(vcpu); } -static void kvm_free_vcpus(struct kvm *kvm) +static void kvm_unload_vcpu_mmus(struct kvm *kvm) { unsigned long i; struct kvm_vcpu *vcpu; - /* - * Unpin any mmu pages first. - */ kvm_for_each_vcpu(i, vcpu, kvm) { kvm_clear_async_pf_completion_queue(vcpu); kvm_unload_vcpu_mmu(vcpu); } - - kvm_destroy_vcpus(kvm); } void kvm_arch_sync_events(struct kvm *kvm) @@ -11935,11 +11930,12 @@ void kvm_arch_destroy_vm(struct kvm *kvm) __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0); mutex_unlock(&kvm->slots_lock); } + kvm_unload_vcpu_mmus(kvm); static_call_cond(kvm_x86_vm_destroy)(kvm); kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1)); kvm_pic_destroy(kvm); kvm_ioapic_destroy(kvm); - kvm_free_vcpus(kvm); + kvm_destroy_vcpus(kvm); kvfree(rcu_dereference_check(kvm->arch.apic_map, 1)); kfree(srcu_dereference_check(kvm->arch.pmu_event_filter, &kvm->srcu, 1)); kvm_mmu_uninit_vm(kvm); From f502cc568de95e5ed9cc9e6133fa454fbe0c5c01 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 4 Mar 2022 11:48:38 -0800 Subject: [PATCH 174/319] KVM: Add max_vcpus field in common 'struct kvm' For TDX guests, the maximum number of vcpus needs to be specified when the TDX guest VM is initialized (creating the TDX data corresponding to TDX guest) before creating vcpu. It needs to record the maximum number of vcpus on VM creation (KVM_CREATE_VM) and return error if the number of vcpus exceeds it Because there is already max_vcpu member in arm64 struct kvm_arch, move it to common struct kvm and initialize it to KVM_MAX_VCPUS before kvm_arch_init_vm() instead of adding it to x86 struct kvm_arch. Signed-off-by: Sean Christopherson Signed-off-by: Isaku Yamahata Message-Id: Signed-off-by: Paolo Bonzini --- arch/arm64/include/asm/kvm_host.h | 3 --- arch/arm64/kvm/arm.c | 6 +++--- arch/arm64/kvm/vgic/vgic-init.c | 6 +++--- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 3 ++- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 94a27a7520f4..27ebb2929e0c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -107,9 +107,6 @@ struct kvm_arch { /* VTCR_EL2 value for this VM */ u64 vtcr; - /* The maximum number of vCPUs depends on the used GIC model */ - int max_vcpus; - /* Interrupt controller */ struct vgic_dist vgic; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 523bc934fe2f..7fceb855fa71 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -153,7 +153,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm_vgic_early_init(kvm); /* The maximum number of VCPUs is limited by the host's GIC model */ - kvm->arch.max_vcpus = kvm_arm_default_max_vcpus(); + kvm->max_vcpus = kvm_arm_default_max_vcpus(); set_default_spectre(kvm); @@ -230,7 +230,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_MAX_VCPUS: case KVM_CAP_MAX_VCPU_ID: if (kvm) - r = kvm->arch.max_vcpus; + r = kvm->max_vcpus; else r = kvm_arm_default_max_vcpus(); break; @@ -306,7 +306,7 @@ int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) if (irqchip_in_kernel(kvm) && vgic_initialized(kvm)) return -EBUSY; - if (id >= kvm->arch.max_vcpus) + if (id >= kvm->max_vcpus) return -EINVAL; return 0; diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index fc00304fe7d8..77feafd5c0e3 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -98,11 +98,11 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) ret = 0; if (type == KVM_DEV_TYPE_ARM_VGIC_V2) - kvm->arch.max_vcpus = VGIC_V2_MAX_CPUS; + kvm->max_vcpus = VGIC_V2_MAX_CPUS; else - kvm->arch.max_vcpus = VGIC_V3_MAX_CPUS; + kvm->max_vcpus = VGIC_V3_MAX_CPUS; - if (atomic_read(&kvm->online_vcpus) > kvm->arch.max_vcpus) { + if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) { ret = -E2BIG; goto out_unlock; } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 252ee4a61b58..f94f72bbd2d3 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -725,6 +725,7 @@ struct kvm { * and is accessed atomically. */ atomic_t online_vcpus; + int max_vcpus; int created_vcpus; int last_boosted_vcpu; struct list_head vm_list; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index ac57fc2c935f..655365b2cbe8 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1078,6 +1078,7 @@ static struct kvm *kvm_create_vm(unsigned long type) spin_lock_init(&kvm->gpc_lock); INIT_LIST_HEAD(&kvm->devices); + kvm->max_vcpus = KVM_MAX_VCPUS; BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); @@ -3732,7 +3733,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) return -EINVAL; mutex_lock(&kvm->lock); - if (kvm->created_vcpus == KVM_MAX_VCPUS) { + if (kvm->created_vcpus >= kvm->max_vcpus) { mutex_unlock(&kvm->lock); return -EINVAL; } From c180269d27bfadc9e79e20c233b06e49c3127732 Mon Sep 17 00:00:00 2001 From: Yuan Yao Date: Fri, 4 Mar 2022 11:49:33 -0800 Subject: [PATCH 175/319] KVM: VMX: Use vcpu_to_pi_desc() uniformly in posted_intr.c The helper function, vcpu_to_pi_desc(), is defined to get the posted interrupt descriptor from vcpu. There is one place that doesn't use it, and instead references vmx_vcpu->pi_desc directly. Remove the inconsistency. Signed-off-by: Yuan Yao Signed-off-by: Isaku Yamahata Message-Id: Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/posted_intr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 3834bb30ce54..37d0bfff269d 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -311,7 +311,7 @@ int vmx_pi_update_irte(struct kvm *kvm, unsigned int host_irq, continue; } - vcpu_info.pi_desc_addr = __pa(&to_vmx(vcpu)->pi_desc); + vcpu_info.pi_desc_addr = __pa(vcpu_to_pi_desc(vcpu)); vcpu_info.vector = irq.vector; trace_kvm_pi_irte_update(host_irq, vcpu->vcpu_id, e->gsi, From 6a5d778edaa39dfa07a61d487a70f2deb1017c0f Mon Sep 17 00:00:00 2001 From: "Dustin L. Howett" Date: Thu, 17 Feb 2022 10:59:29 -0600 Subject: [PATCH 176/319] platform/chrome: cros_ec_lpcs: detect the Framework Laptop The Framework Laptop identifies itself in DMI with manufacturer "Framework" and product "Laptop". Signed-off-by: Dustin L. Howett Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220217165930.15081-2-dustin@howett.net --- drivers/platform/chrome/cros_ec_lpc.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/platform/chrome/cros_ec_lpc.c b/drivers/platform/chrome/cros_ec_lpc.c index 7651417b4a25..1ec12d19c779 100644 --- a/drivers/platform/chrome/cros_ec_lpc.c +++ b/drivers/platform/chrome/cros_ec_lpc.c @@ -502,6 +502,14 @@ static const struct dmi_system_id cros_ec_lpc_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "Glimmer"), }, }, + /* A small number of non-Chromebook/box machines also use the ChromeOS EC */ + { + /* the Framework Laptop */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Framework"), + DMI_MATCH(DMI_PRODUCT_NAME, "Laptop"), + }, + }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(dmi, cros_ec_lpc_dmi_table); From c9bc1a0ef9f613a7bc1adfff4c67dc5e5d7d1709 Mon Sep 17 00:00:00 2001 From: "Dustin L. Howett" Date: Thu, 17 Feb 2022 10:59:30 -0600 Subject: [PATCH 177/319] platform/chrome: cros_ec_lpcs: reserve the MEC LPC I/O ports first Some ChromeOS EC devices (such as the Framework Laptop) only map I/O ports 0x800-0x807. Making the larger reservation required by the non-MEC LPC (the 0xFF ports for the memory map, and the 0xFF ports for the parameter region) is non-viable on these devices. Since we probe the MEC EC first, we can get away with a smaller reservation that covers the MEC EC ports. If we fall back to classic LPC, we can grow the reservation to cover the memory map and the parameter region. cros_ec_lpc_probe also interacted with I/O ports 0x800-0x807 without a reservation. Restructuring the code to request the MEC LPC region first obviates the need to do so. Signed-off-by: Dustin L. Howett Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220217165930.15081-3-dustin@howett.net --- drivers/platform/chrome/cros_ec_lpc.c | 39 ++++++++++++------- .../linux/platform_data/cros_ec_commands.h | 10 +++-- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_lpc.c b/drivers/platform/chrome/cros_ec_lpc.c index 1ec12d19c779..8eeef85a96b1 100644 --- a/drivers/platform/chrome/cros_ec_lpc.c +++ b/drivers/platform/chrome/cros_ec_lpc.c @@ -341,9 +341,14 @@ static int cros_ec_lpc_probe(struct platform_device *pdev) u8 buf[2]; int irq, ret; - if (!devm_request_region(dev, EC_LPC_ADDR_MEMMAP, EC_MEMMAP_SIZE, - dev_name(dev))) { - dev_err(dev, "couldn't reserve memmap region\n"); + /* + * The Framework Laptop (and possibly other non-ChromeOS devices) + * only exposes the eight I/O ports that are required for the Microchip EC. + * Requesting a larger reservation will fail. + */ + if (!devm_request_region(dev, EC_HOST_CMD_REGION0, + EC_HOST_CMD_MEC_REGION_SIZE, dev_name(dev))) { + dev_err(dev, "couldn't reserve MEC region\n"); return -EBUSY; } @@ -357,6 +362,12 @@ static int cros_ec_lpc_probe(struct platform_device *pdev) cros_ec_lpc_ops.write = cros_ec_lpc_mec_write_bytes; cros_ec_lpc_ops.read(EC_LPC_ADDR_MEMMAP + EC_MEMMAP_ID, 2, buf); if (buf[0] != 'E' || buf[1] != 'C') { + if (!devm_request_region(dev, EC_LPC_ADDR_MEMMAP, EC_MEMMAP_SIZE, + dev_name(dev))) { + dev_err(dev, "couldn't reserve memmap region\n"); + return -EBUSY; + } + /* Re-assign read/write operations for the non MEC variant */ cros_ec_lpc_ops.read = cros_ec_lpc_read_bytes; cros_ec_lpc_ops.write = cros_ec_lpc_write_bytes; @@ -366,17 +377,19 @@ static int cros_ec_lpc_probe(struct platform_device *pdev) dev_err(dev, "EC ID not detected\n"); return -ENODEV; } - } - if (!devm_request_region(dev, EC_HOST_CMD_REGION0, - EC_HOST_CMD_REGION_SIZE, dev_name(dev))) { - dev_err(dev, "couldn't reserve region0\n"); - return -EBUSY; - } - if (!devm_request_region(dev, EC_HOST_CMD_REGION1, - EC_HOST_CMD_REGION_SIZE, dev_name(dev))) { - dev_err(dev, "couldn't reserve region1\n"); - return -EBUSY; + /* Reserve the remaining I/O ports required by the non-MEC protocol. */ + if (!devm_request_region(dev, EC_HOST_CMD_REGION0 + EC_HOST_CMD_MEC_REGION_SIZE, + EC_HOST_CMD_REGION_SIZE - EC_HOST_CMD_MEC_REGION_SIZE, + dev_name(dev))) { + dev_err(dev, "couldn't reserve remainder of region0\n"); + return -EBUSY; + } + if (!devm_request_region(dev, EC_HOST_CMD_REGION1, + EC_HOST_CMD_REGION_SIZE, dev_name(dev))) { + dev_err(dev, "couldn't reserve region1\n"); + return -EBUSY; + } } ec_dev = devm_kzalloc(dev, sizeof(*ec_dev), GFP_KERNEL); diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index c23554531961..8cfa8cfca77e 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -51,10 +51,14 @@ /* * The actual block is 0x800-0x8ff, but some BIOSes think it's 0x880-0x8ff * and they tell the kernel that so we have to think of it as two parts. + * + * Other BIOSes report only the I/O port region spanned by the Microchip + * MEC series EC; an attempt to address a larger region may fail. */ -#define EC_HOST_CMD_REGION0 0x800 -#define EC_HOST_CMD_REGION1 0x880 -#define EC_HOST_CMD_REGION_SIZE 0x80 +#define EC_HOST_CMD_REGION0 0x800 +#define EC_HOST_CMD_REGION1 0x880 +#define EC_HOST_CMD_REGION_SIZE 0x80 +#define EC_HOST_CMD_MEC_REGION_SIZE 0x8 /* EC command register bit functions */ #define EC_LPC_CMDR_DATA BIT(0) /* Data ready for host to read */ From 001bb819994cd1bd037b6aefdb233f1720ee2126 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 3 May 2022 06:01:59 +0000 Subject: [PATCH 178/319] KVM: arm64: Return a bool from emulate_cp() KVM indicates success/failure in several ways, but generally an integer is used when conditionally bouncing to userspace is involved. That is not the case from emulate_cp(); just use a bool instead. No functional change intended. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220503060205.2823727-2-oupton@google.com --- arch/arm64/kvm/sys_regs.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 7b45c040cc27..36895c163eae 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2246,27 +2246,27 @@ static void perform_access(struct kvm_vcpu *vcpu, * @table: array of trap descriptors * @num: size of the trap descriptor array * - * Return 0 if the access has been handled, and -1 if not. + * Return true if the access has been handled, false if not. */ -static int emulate_cp(struct kvm_vcpu *vcpu, - struct sys_reg_params *params, - const struct sys_reg_desc *table, - size_t num) +static bool emulate_cp(struct kvm_vcpu *vcpu, + struct sys_reg_params *params, + const struct sys_reg_desc *table, + size_t num) { const struct sys_reg_desc *r; if (!table) - return -1; /* Not handled */ + return false; /* Not handled */ r = find_reg(params, table, num); if (r) { perform_access(vcpu, params, r); - return 0; + return true; } /* Not handled */ - return -1; + return false; } static void unhandled_cp_access(struct kvm_vcpu *vcpu, @@ -2330,7 +2330,7 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, * potential register operation in the case of a read and return * with success. */ - if (!emulate_cp(vcpu, ¶ms, global, nr_global)) { + if (emulate_cp(vcpu, ¶ms, global, nr_global)) { /* Split up the value between registers for the read side */ if (!params.is_write) { vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval)); @@ -2365,7 +2365,7 @@ static int kvm_handle_cp_32(struct kvm_vcpu *vcpu, params.Op1 = (esr >> 14) & 0x7; params.Op2 = (esr >> 17) & 0x7; - if (!emulate_cp(vcpu, ¶ms, global, nr_global)) { + if (emulate_cp(vcpu, ¶ms, global, nr_global)) { if (!params.is_write) vcpu_set_reg(vcpu, Rt, params.regval); return 1; From 28eda7b5e82489b9dcffc630af68c207552b4f4d Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 3 May 2022 06:02:00 +0000 Subject: [PATCH 179/319] KVM: arm64: Don't write to Rt unless sys_reg emulation succeeds emulate_sys_reg() returns 1 unconditionally, even though a a system register access can fail. Furthermore, kvm_handle_sys_reg() writes to Rt for every register read, regardless of if it actually succeeded. Though this pattern is safe (as params.regval is initialized with the current value of Rt) it is a bit ugly. Indicate failure if the register access could not be emulated and only write to Rt on success. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220503060205.2823727-3-oupton@google.com --- arch/arm64/kvm/sys_regs.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 36895c163eae..f0a076e5cc1c 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2401,7 +2401,14 @@ static bool is_imp_def_sys_reg(struct sys_reg_params *params) return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011; } -static int emulate_sys_reg(struct kvm_vcpu *vcpu, +/** + * emulate_sys_reg - Emulate a guest access to an AArch64 system register + * @vcpu: The VCPU pointer + * @params: Decoded system register parameters + * + * Return: true if the system register access was successful, false otherwise. + */ +static bool emulate_sys_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *params) { const struct sys_reg_desc *r; @@ -2410,7 +2417,10 @@ static int emulate_sys_reg(struct kvm_vcpu *vcpu, if (likely(r)) { perform_access(vcpu, params, r); - } else if (is_imp_def_sys_reg(params)) { + return true; + } + + if (is_imp_def_sys_reg(params)) { kvm_inject_undefined(vcpu); } else { print_sys_reg_msg(params, @@ -2418,7 +2428,7 @@ static int emulate_sys_reg(struct kvm_vcpu *vcpu, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu)); kvm_inject_undefined(vcpu); } - return 1; + return false; } /** @@ -2446,18 +2456,18 @@ int kvm_handle_sys_reg(struct kvm_vcpu *vcpu) struct sys_reg_params params; unsigned long esr = kvm_vcpu_get_esr(vcpu); int Rt = kvm_vcpu_sys_get_rt(vcpu); - int ret; trace_kvm_handle_sys_reg(esr); params = esr_sys64_to_params(esr); params.regval = vcpu_get_reg(vcpu, Rt); - ret = emulate_sys_reg(vcpu, ¶ms); + if (!emulate_sys_reg(vcpu, ¶ms)) + return 1; if (!params.is_write) vcpu_set_reg(vcpu, Rt, params.regval); - return ret; + return 1; } /****************************************************************************** From e65197666773f39e4378161925e5a1c7771cff29 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 3 May 2022 06:02:01 +0000 Subject: [PATCH 180/319] KVM: arm64: Wire up CP15 feature registers to their AArch64 equivalents KVM currently does not trap ID register accesses from an AArch32 EL1. This is painful for a couple of reasons. Certain unimplemented features are visible to AArch32 EL1, as we limit PMU to version 3 and the debug architecture to v8.0. Additionally, we attempt to paper over heterogeneous systems by using register values that are safe system-wide. All this hard work is completely sidestepped because KVM does not set TID3 for AArch32 guests. Fix up handling of CP15 feature registers by simply rerouting to their AArch64 aliases. Punt setting HCR_EL2.TID3 to a later change, as we need to fix up the oddball CP10 feature registers still. Signed-off-by: Oliver Upton Reviewed-by: Reiji Watanabe Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220503060205.2823727-4-oupton@google.com --- arch/arm64/kvm/sys_regs.c | 86 ++++++++++++++++++++++++++++++++------- arch/arm64/kvm/sys_regs.h | 7 ++++ 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index f0a076e5cc1c..f403ea47b8a3 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2344,34 +2344,73 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, return 1; } +static bool emulate_sys_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *params); + +/** + * kvm_emulate_cp15_id_reg() - Handles an MRC trap on a guest CP15 access where + * CRn=0, which corresponds to the AArch32 feature + * registers. + * @vcpu: the vCPU pointer + * @params: the system register access parameters. + * + * Our cp15 system register tables do not enumerate the AArch32 feature + * registers. Conveniently, our AArch64 table does, and the AArch32 system + * register encoding can be trivially remapped into the AArch64 for the feature + * registers: Append op0=3, leaving op1, CRn, CRm, and op2 the same. + * + * According to DDI0487G.b G7.3.1, paragraph "Behavior of VMSAv8-32 32-bit + * System registers with (coproc=0b1111, CRn==c0)", read accesses from this + * range are either UNKNOWN or RES0. Rerouting remains architectural as we + * treat undefined registers in this range as RAZ. + */ +static int kvm_emulate_cp15_id_reg(struct kvm_vcpu *vcpu, + struct sys_reg_params *params) +{ + int Rt = kvm_vcpu_sys_get_rt(vcpu); + + /* Treat impossible writes to RO registers as UNDEFINED */ + if (params->is_write) { + unhandled_cp_access(vcpu, params); + return 1; + } + + params->Op0 = 3; + + /* + * All registers where CRm > 3 are known to be UNKNOWN/RAZ from AArch32. + * Avoid conflicting with future expansion of AArch64 feature registers + * and simply treat them as RAZ here. + */ + if (params->CRm > 3) + params->regval = 0; + else if (!emulate_sys_reg(vcpu, params)) + return 1; + + vcpu_set_reg(vcpu, Rt, params->regval); + return 1; +} + /** * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access * @vcpu: The VCPU pointer * @run: The kvm_run struct */ static int kvm_handle_cp_32(struct kvm_vcpu *vcpu, + struct sys_reg_params *params, const struct sys_reg_desc *global, size_t nr_global) { - struct sys_reg_params params; - u32 esr = kvm_vcpu_get_esr(vcpu); int Rt = kvm_vcpu_sys_get_rt(vcpu); - params.CRm = (esr >> 1) & 0xf; - params.regval = vcpu_get_reg(vcpu, Rt); - params.is_write = ((esr & 1) == 0); - params.CRn = (esr >> 10) & 0xf; - params.Op0 = 0; - params.Op1 = (esr >> 14) & 0x7; - params.Op2 = (esr >> 17) & 0x7; + params->regval = vcpu_get_reg(vcpu, Rt); - if (emulate_cp(vcpu, ¶ms, global, nr_global)) { - if (!params.is_write) - vcpu_set_reg(vcpu, Rt, params.regval); + if (emulate_cp(vcpu, params, global, nr_global)) { + if (!params->is_write) + vcpu_set_reg(vcpu, Rt, params->regval); return 1; } - unhandled_cp_access(vcpu, ¶ms); + unhandled_cp_access(vcpu, params); return 1; } @@ -2382,7 +2421,20 @@ int kvm_handle_cp15_64(struct kvm_vcpu *vcpu) int kvm_handle_cp15_32(struct kvm_vcpu *vcpu) { - return kvm_handle_cp_32(vcpu, cp15_regs, ARRAY_SIZE(cp15_regs)); + struct sys_reg_params params; + + params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu)); + + /* + * Certain AArch32 ID registers are handled by rerouting to the AArch64 + * system register table. Registers in the ID range where CRm=0 are + * excluded from this scheme as they do not trivially map into AArch64 + * system register encodings. + */ + if (params.Op1 == 0 && params.CRn == 0 && params.CRm) + return kvm_emulate_cp15_id_reg(vcpu, ¶ms); + + return kvm_handle_cp_32(vcpu, ¶ms, cp15_regs, ARRAY_SIZE(cp15_regs)); } int kvm_handle_cp14_64(struct kvm_vcpu *vcpu) @@ -2392,7 +2444,11 @@ int kvm_handle_cp14_64(struct kvm_vcpu *vcpu) int kvm_handle_cp14_32(struct kvm_vcpu *vcpu) { - return kvm_handle_cp_32(vcpu, cp14_regs, ARRAY_SIZE(cp14_regs)); + struct sys_reg_params params; + + params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu)); + + return kvm_handle_cp_32(vcpu, ¶ms, cp14_regs, ARRAY_SIZE(cp14_regs)); } static bool is_imp_def_sys_reg(struct sys_reg_params *params) diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index cc0cc95a0280..0d31a12b640c 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -35,6 +35,13 @@ struct sys_reg_params { .Op2 = ((esr) >> 17) & 0x7, \ .is_write = !((esr) & 1) }) +#define esr_cp1x_32_to_params(esr) \ + ((struct sys_reg_params){ .Op1 = ((esr) >> 14) & 0x7, \ + .CRn = ((esr) >> 10) & 0xf, \ + .CRm = ((esr) >> 1) & 0xf, \ + .Op2 = ((esr) >> 17) & 0x7, \ + .is_write = !((esr) & 1) }) + struct sys_reg_desc { /* Sysreg string for debug */ const char *name; From 9369bc5c5e35985f38d04bd98c6d28a032e84b17 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 3 May 2022 06:02:02 +0000 Subject: [PATCH 181/319] KVM: arm64: Plumb cp10 ID traps through the AArch64 sysreg handler In order to enable HCR_EL2.TID3 for AArch32 guests KVM needs to handle traps where ESR_EL2.EC=0x8, which corresponds to an attempted VMRS access from an ID group register. Specifically, the MVFR{0-2} registers are accessed this way from AArch32. Conveniently, these registers are architecturally mapped to MVFR{0-2}_EL1 in AArch64. Furthermore, KVM already handles reads to these aliases in AArch64. Plumb VMRS read traps through to the general AArch64 system register handler. Signed-off-by: Oliver Upton Reviewed-by: Reiji Watanabe Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220503060205.2823727-5-oupton@google.com --- arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/handle_exit.c | 1 + arch/arm64/kvm/sys_regs.c | 71 +++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 94a27a7520f4..05081b9b7369 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -683,6 +683,7 @@ int kvm_handle_cp14_64(struct kvm_vcpu *vcpu); int kvm_handle_cp15_32(struct kvm_vcpu *vcpu); int kvm_handle_cp15_64(struct kvm_vcpu *vcpu); int kvm_handle_sys_reg(struct kvm_vcpu *vcpu); +int kvm_handle_cp10_id(struct kvm_vcpu *vcpu); void kvm_reset_sys_regs(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 97fe14aab1a3..5088a86ace5b 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -167,6 +167,7 @@ static exit_handle_fn arm_exit_handlers[] = { [ESR_ELx_EC_CP15_64] = kvm_handle_cp15_64, [ESR_ELx_EC_CP14_MR] = kvm_handle_cp14_32, [ESR_ELx_EC_CP14_LS] = kvm_handle_cp14_load_store, + [ESR_ELx_EC_CP10_ID] = kvm_handle_cp10_id, [ESR_ELx_EC_CP14_64] = kvm_handle_cp14_64, [ESR_ELx_EC_HVC32] = handle_hvc, [ESR_ELx_EC_SMC32] = handle_smc, diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index f403ea47b8a3..586b292ca94f 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2346,6 +2346,77 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, static bool emulate_sys_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *params); +/* + * The CP10 ID registers are architecturally mapped to AArch64 feature + * registers. Abuse that fact so we can rely on the AArch64 handler for accesses + * from AArch32. + */ +static bool kvm_esr_cp10_id_to_sys64(u32 esr, struct sys_reg_params *params) +{ + u8 reg_id = (esr >> 10) & 0xf; + bool valid; + + params->is_write = ((esr & 1) == 0); + params->Op0 = 3; + params->Op1 = 0; + params->CRn = 0; + params->CRm = 3; + + /* CP10 ID registers are read-only */ + valid = !params->is_write; + + switch (reg_id) { + /* MVFR0 */ + case 0b0111: + params->Op2 = 0; + break; + /* MVFR1 */ + case 0b0110: + params->Op2 = 1; + break; + /* MVFR2 */ + case 0b0101: + params->Op2 = 2; + break; + default: + valid = false; + } + + if (valid) + return true; + + kvm_pr_unimpl("Unhandled cp10 register %s: %u\n", + params->is_write ? "write" : "read", reg_id); + return false; +} + +/** + * kvm_handle_cp10_id() - Handles a VMRS trap on guest access to a 'Media and + * VFP Register' from AArch32. + * @vcpu: The vCPU pointer + * + * MVFR{0-2} are architecturally mapped to the AArch64 MVFR{0-2}_EL1 registers. + * Work out the correct AArch64 system register encoding and reroute to the + * AArch64 system register emulation. + */ +int kvm_handle_cp10_id(struct kvm_vcpu *vcpu) +{ + int Rt = kvm_vcpu_sys_get_rt(vcpu); + u32 esr = kvm_vcpu_get_esr(vcpu); + struct sys_reg_params params; + + /* UNDEF on any unhandled register access */ + if (!kvm_esr_cp10_id_to_sys64(esr, ¶ms)) { + kvm_inject_undefined(vcpu); + return 1; + } + + if (emulate_sys_reg(vcpu, ¶ms)) + vcpu_set_reg(vcpu, Rt, params.regval); + + return 1; +} + /** * kvm_emulate_cp15_id_reg() - Handles an MRC trap on a guest CP15 access where * CRn=0, which corresponds to the AArch32 feature From fd1264c4ca610a99d52c35a37e5551eec442723d Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Tue, 3 May 2022 06:02:03 +0000 Subject: [PATCH 182/319] KVM: arm64: Start trapping ID registers for 32 bit guests To date KVM has not trapped ID register accesses from AArch32, meaning that guests get an unconstrained view of what hardware supports. This can be a serious problem because we try to base the guest's feature registers on values that are safe system-wide. Furthermore, KVM does not implement the latest ISA in the PMU and Debug architecture, so we constrain these fields to supported values. Since KVM now correctly handles CP15 and CP10 register traps, we no longer need to clear HCR_EL2.TID3 for 32 bit guests and will instead emulate reads with their safe values. Signed-off-by: Oliver Upton Reviewed-by: Reiji Watanabe Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220503060205.2823727-6-oupton@google.com --- arch/arm64/include/asm/kvm_arm.h | 3 ++- arch/arm64/include/asm/kvm_emulate.h | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 1767ded83888..b5de102928d8 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -80,11 +80,12 @@ * FMO: Override CPSR.F and enable signaling with VF * SWIO: Turn set/way invalidates into set/way clean+invalidate * PTW: Take a stage2 fault if a stage1 walk steps in device memory + * TID3: Trap EL1 reads of group 3 ID registers */ #define HCR_GUEST_FLAGS (HCR_TSC | HCR_TSW | HCR_TWE | HCR_TWI | HCR_VM | \ HCR_BSU_IS | HCR_FB | HCR_TACR | \ HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \ - HCR_FMO | HCR_IMO | HCR_PTW ) + HCR_FMO | HCR_IMO | HCR_PTW | HCR_TID3 ) #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF) #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA) #define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 7496deab025a..ab5c66b77bb0 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -86,13 +86,6 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) if (vcpu_el1_is_32bit(vcpu)) vcpu->arch.hcr_el2 &= ~HCR_RW; - else - /* - * TID3: trap feature register accesses that we virtualise. - * For now this is conditional, since no AArch32 feature regs - * are currently virtualised. - */ - vcpu->arch.hcr_el2 |= HCR_TID3; if (cpus_have_const_cap(ARM64_MISMATCHED_CACHE_TYPE) || vcpu_el1_is_32bit(vcpu)) From a9e192cd4fc738469448803693c9dc730898b8f1 Mon Sep 17 00:00:00 2001 From: Alexandru Elisei Date: Tue, 3 May 2022 06:02:04 +0000 Subject: [PATCH 183/319] KVM: arm64: Hide AArch32 PMU registers when not available commit 11663111cd49 ("KVM: arm64: Hide PMU registers from userspace when not available") hid the AArch64 PMU registers from userspace and guest when the PMU VCPU feature was not set. Do the same when the PMU registers are accessed by an AArch32 guest. While we're at it, rename the previously unused AA32_ZEROHIGH to AA32_DIRECT to match the behavior of get_access_mask(). Now that KVM emulates ID_DFR0 and hides the PMU from the guest when the feature is not set, it is safe to inject to inject an undefined exception when the PMU is not present, as that corresponds to the architected behaviour. Signed-off-by: Alexandru Elisei [Oliver - Add AA32_DIRECT to match the zero value of the enum] Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220503060205.2823727-7-oupton@google.com --- arch/arm64/kvm/sys_regs.c | 60 ++++++++++++++++++++------------------- arch/arm64/kvm/sys_regs.h | 2 +- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 586b292ca94f..f3235eafdadc 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2014,20 +2014,22 @@ static const struct sys_reg_desc cp14_64_regs[] = { { Op1( 0), CRm( 2), .access = trap_raz_wi }, }; +#define CP15_PMU_SYS_REG(_map, _Op1, _CRn, _CRm, _Op2) \ + AA32(_map), \ + Op1(_Op1), CRn(_CRn), CRm(_CRm), Op2(_Op2), \ + .visibility = pmu_visibility + /* Macro to expand the PMEVCNTRn register */ #define PMU_PMEVCNTR(n) \ - /* PMEVCNTRn */ \ - { Op1(0), CRn(0b1110), \ - CRm((0b1000 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \ - access_pmu_evcntr } + { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \ + (0b1000 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \ + .access = access_pmu_evcntr } /* Macro to expand the PMEVTYPERn register */ #define PMU_PMEVTYPER(n) \ - /* PMEVTYPERn */ \ - { Op1(0), CRn(0b1110), \ - CRm((0b1100 | (((n) >> 3) & 0x3))), Op2(((n) & 0x7)), \ - access_pmu_evtyper } - + { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \ + (0b1100 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \ + .access = access_pmu_evtyper } /* * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding, * depending on the way they are accessed (as a 32bit or a 64bit @@ -2067,25 +2069,25 @@ static const struct sys_reg_desc cp15_regs[] = { { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw }, /* PMU */ - { Op1( 0), CRn( 9), CRm(12), Op2( 0), access_pmcr }, - { Op1( 0), CRn( 9), CRm(12), Op2( 1), access_pmcnten }, - { Op1( 0), CRn( 9), CRm(12), Op2( 2), access_pmcnten }, - { Op1( 0), CRn( 9), CRm(12), Op2( 3), access_pmovs }, - { Op1( 0), CRn( 9), CRm(12), Op2( 4), access_pmswinc }, - { Op1( 0), CRn( 9), CRm(12), Op2( 5), access_pmselr }, - { AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 6), access_pmceid }, - { AA32(LO), Op1( 0), CRn( 9), CRm(12), Op2( 7), access_pmceid }, - { Op1( 0), CRn( 9), CRm(13), Op2( 0), access_pmu_evcntr }, - { Op1( 0), CRn( 9), CRm(13), Op2( 1), access_pmu_evtyper }, - { Op1( 0), CRn( 9), CRm(13), Op2( 2), access_pmu_evcntr }, - { Op1( 0), CRn( 9), CRm(14), Op2( 0), access_pmuserenr }, - { Op1( 0), CRn( 9), CRm(14), Op2( 1), access_pminten }, - { Op1( 0), CRn( 9), CRm(14), Op2( 2), access_pminten }, - { Op1( 0), CRn( 9), CRm(14), Op2( 3), access_pmovs }, - { AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 4), access_pmceid }, - { AA32(HI), Op1( 0), CRn( 9), CRm(14), Op2( 5), access_pmceid }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 0), .access = access_pmcr }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 1), .access = access_pmcnten }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 2), .access = access_pmcnten }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 3), .access = access_pmovs }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 4), .access = access_pmswinc }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 5), .access = access_pmselr }, + { CP15_PMU_SYS_REG(LO, 0, 9, 12, 6), .access = access_pmceid }, + { CP15_PMU_SYS_REG(LO, 0, 9, 12, 7), .access = access_pmceid }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 0), .access = access_pmu_evcntr }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 1), .access = access_pmu_evtyper }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 2), .access = access_pmu_evcntr }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 0), .access = access_pmuserenr }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 1), .access = access_pminten }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 2), .access = access_pminten }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 3), .access = access_pmovs }, + { CP15_PMU_SYS_REG(HI, 0, 9, 14, 4), .access = access_pmceid }, + { CP15_PMU_SYS_REG(HI, 0, 9, 14, 5), .access = access_pmceid }, /* PMMIR */ - { Op1( 0), CRn( 9), CRm(14), Op2( 6), trap_raz_wi }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = trap_raz_wi }, /* PRRR/MAIR0 */ { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 }, @@ -2170,7 +2172,7 @@ static const struct sys_reg_desc cp15_regs[] = { PMU_PMEVTYPER(29), PMU_PMEVTYPER(30), /* PMCCFILTR */ - { Op1(0), CRn(14), CRm(15), Op2(7), access_pmu_evtyper }, + { CP15_PMU_SYS_REG(DIRECT, 0, 14, 15, 7), .access = access_pmu_evtyper }, { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr }, { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr }, @@ -2179,7 +2181,7 @@ static const struct sys_reg_desc cp15_regs[] = { static const struct sys_reg_desc cp15_64_regs[] = { { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 }, - { Op1( 0), CRn( 0), CRm( 9), Op2( 0), access_pmu_evcntr }, + { CP15_PMU_SYS_REG(DIRECT, 0, 0, 9, 0), .access = access_pmu_evcntr }, { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */ { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 }, { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */ diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index 0d31a12b640c..aee8ea054f0d 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -47,7 +47,7 @@ struct sys_reg_desc { const char *name; enum { - AA32_ZEROHIGH, + AA32_DIRECT, AA32_LO, AA32_HI, } aarch32_map; From 85fbe08e4da862dc64fc10071c4a03e51b6361d0 Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:45 +0000 Subject: [PATCH 184/319] KVM: arm64: Factor out firmware register handling from psci.c Common hypercall firmware register handing is currently employed by psci.c. Since the upcoming patches add more of these registers, it's better to move the generic handling to hypercall.c for a cleaner presentation. While we are at it, collect all the firmware registers under fw_reg_ids[] to help implement kvm_arm_get_fw_num_regs() and kvm_arm_copy_fw_reg_indices() in a generic way. Also, define KVM_REG_FEATURE_LEVEL_MASK using a GENMASK instead. No functional change intended. Signed-off-by: Raghavendra Rao Ananta Reviewed-by: Oliver Upton Reviewed-by: Gavin Shan [maz: fixed KVM_REG_FEATURE_LEVEL_MASK] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-2-rananta@google.com --- arch/arm64/kvm/guest.c | 2 +- arch/arm64/kvm/hypercalls.c | 184 +++++++++++++++++++++++++++++++++++ arch/arm64/kvm/psci.c | 183 ---------------------------------- include/kvm/arm_hypercalls.h | 7 ++ include/kvm/arm_psci.h | 7 -- 5 files changed, 192 insertions(+), 191 deletions(-) diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 7e15b03fbdf8..0d5cca56cbda 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index 202b8c455724..dd4e20b8d656 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -158,3 +158,187 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) smccc_set_retval(vcpu, val[0], val[1], val[2], val[3]); return 1; } + +static const u64 kvm_arm_fw_reg_ids[] = { + KVM_REG_ARM_PSCI_VERSION, + KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1, + KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, + KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, +}; + +int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu) +{ + return ARRAY_SIZE(kvm_arm_fw_reg_ids); +} + +int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(kvm_arm_fw_reg_ids); i++) { + if (put_user(kvm_arm_fw_reg_ids[i], uindices++)) + return -EFAULT; + } + + return 0; +} + +#define KVM_REG_FEATURE_LEVEL_MASK GENMASK(3, 0) + +/* + * Convert the workaround level into an easy-to-compare number, where higher + * values mean better protection. + */ +static int get_kernel_wa_level(u64 regid) +{ + switch (regid) { + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: + switch (arm64_get_spectre_v2_state()) { + case SPECTRE_VULNERABLE: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL; + case SPECTRE_MITIGATED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_AVAIL; + case SPECTRE_UNAFFECTED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_REQUIRED; + } + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL; + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2: + switch (arm64_get_spectre_v4_state()) { + case SPECTRE_MITIGATED: + /* + * As for the hypercall discovery, we pretend we + * don't have any FW mitigation if SSBS is there at + * all times. + */ + if (cpus_have_final_cap(ARM64_SSBS)) + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL; + fallthrough; + case SPECTRE_UNAFFECTED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED; + case SPECTRE_VULNERABLE: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL; + } + break; + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: + switch (arm64_get_spectre_bhb_state()) { + case SPECTRE_VULNERABLE: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL; + case SPECTRE_MITIGATED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL; + case SPECTRE_UNAFFECTED: + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED; + } + return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL; + } + + return -EINVAL; +} + +int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + void __user *uaddr = (void __user *)(long)reg->addr; + u64 val; + + switch (reg->id) { + case KVM_REG_ARM_PSCI_VERSION: + val = kvm_psci_version(vcpu); + break; + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: + val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK; + break; + default: + return -ENOENT; + } + + if (copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id))) + return -EFAULT; + + return 0; +} + +int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) +{ + void __user *uaddr = (void __user *)(long)reg->addr; + u64 val; + int wa_level; + + if (copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id))) + return -EFAULT; + + switch (reg->id) { + case KVM_REG_ARM_PSCI_VERSION: + { + bool wants_02; + + wants_02 = test_bit(KVM_ARM_VCPU_PSCI_0_2, vcpu->arch.features); + + switch (val) { + case KVM_ARM_PSCI_0_1: + if (wants_02) + return -EINVAL; + vcpu->kvm->arch.psci_version = val; + return 0; + case KVM_ARM_PSCI_0_2: + case KVM_ARM_PSCI_1_0: + case KVM_ARM_PSCI_1_1: + if (!wants_02) + return -EINVAL; + vcpu->kvm->arch.psci_version = val; + return 0; + } + break; + } + + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: + if (val & ~KVM_REG_FEATURE_LEVEL_MASK) + return -EINVAL; + + if (get_kernel_wa_level(reg->id) < val) + return -EINVAL; + + return 0; + + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2: + if (val & ~(KVM_REG_FEATURE_LEVEL_MASK | + KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED)) + return -EINVAL; + + /* The enabled bit must not be set unless the level is AVAIL. */ + if ((val & KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED) && + (val & KVM_REG_FEATURE_LEVEL_MASK) != KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL) + return -EINVAL; + + /* + * Map all the possible incoming states to the only two we + * really want to deal with. + */ + switch (val & KVM_REG_FEATURE_LEVEL_MASK) { + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_UNKNOWN: + wa_level = KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL; + break; + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL: + case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED: + wa_level = KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED; + break; + default: + return -EINVAL; + } + + /* + * We can deal with NOT_AVAIL on NOT_REQUIRED, but not the + * other way around. + */ + if (get_kernel_wa_level(reg->id) < wa_level) + return -EINVAL; + + return 0; + default: + return -ENOENT; + } + + return -EINVAL; +} diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index baac2b405f23..346535169faa 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -436,186 +436,3 @@ int kvm_psci_call(struct kvm_vcpu *vcpu) return -EINVAL; } } - -int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu) -{ - return 4; /* PSCI version and three workaround registers */ -} - -int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) -{ - if (put_user(KVM_REG_ARM_PSCI_VERSION, uindices++)) - return -EFAULT; - - if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1, uindices++)) - return -EFAULT; - - if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, uindices++)) - return -EFAULT; - - if (put_user(KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, uindices++)) - return -EFAULT; - - return 0; -} - -#define KVM_REG_FEATURE_LEVEL_WIDTH 4 -#define KVM_REG_FEATURE_LEVEL_MASK (BIT(KVM_REG_FEATURE_LEVEL_WIDTH) - 1) - -/* - * Convert the workaround level into an easy-to-compare number, where higher - * values mean better protection. - */ -static int get_kernel_wa_level(u64 regid) -{ - switch (regid) { - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: - switch (arm64_get_spectre_v2_state()) { - case SPECTRE_VULNERABLE: - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL; - case SPECTRE_MITIGATED: - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_AVAIL; - case SPECTRE_UNAFFECTED: - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_REQUIRED; - } - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1_NOT_AVAIL; - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2: - switch (arm64_get_spectre_v4_state()) { - case SPECTRE_MITIGATED: - /* - * As for the hypercall discovery, we pretend we - * don't have any FW mitigation if SSBS is there at - * all times. - */ - if (cpus_have_final_cap(ARM64_SSBS)) - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL; - fallthrough; - case SPECTRE_UNAFFECTED: - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED; - case SPECTRE_VULNERABLE: - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL; - } - break; - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: - switch (arm64_get_spectre_bhb_state()) { - case SPECTRE_VULNERABLE: - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL; - case SPECTRE_MITIGATED: - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_AVAIL; - case SPECTRE_UNAFFECTED: - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_REQUIRED; - } - return KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3_NOT_AVAIL; - } - - return -EINVAL; -} - -int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - void __user *uaddr = (void __user *)(long)reg->addr; - u64 val; - - switch (reg->id) { - case KVM_REG_ARM_PSCI_VERSION: - val = kvm_psci_version(vcpu); - break; - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2: - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: - val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK; - break; - default: - return -ENOENT; - } - - if (copy_to_user(uaddr, &val, KVM_REG_SIZE(reg->id))) - return -EFAULT; - - return 0; -} - -int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) -{ - void __user *uaddr = (void __user *)(long)reg->addr; - u64 val; - int wa_level; - - if (copy_from_user(&val, uaddr, KVM_REG_SIZE(reg->id))) - return -EFAULT; - - switch (reg->id) { - case KVM_REG_ARM_PSCI_VERSION: - { - bool wants_02; - - wants_02 = test_bit(KVM_ARM_VCPU_PSCI_0_2, vcpu->arch.features); - - switch (val) { - case KVM_ARM_PSCI_0_1: - if (wants_02) - return -EINVAL; - vcpu->kvm->arch.psci_version = val; - return 0; - case KVM_ARM_PSCI_0_2: - case KVM_ARM_PSCI_1_0: - case KVM_ARM_PSCI_1_1: - if (!wants_02) - return -EINVAL; - vcpu->kvm->arch.psci_version = val; - return 0; - } - break; - } - - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1: - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: - if (val & ~KVM_REG_FEATURE_LEVEL_MASK) - return -EINVAL; - - if (get_kernel_wa_level(reg->id) < val) - return -EINVAL; - - return 0; - - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2: - if (val & ~(KVM_REG_FEATURE_LEVEL_MASK | - KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED)) - return -EINVAL; - - /* The enabled bit must not be set unless the level is AVAIL. */ - if ((val & KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_ENABLED) && - (val & KVM_REG_FEATURE_LEVEL_MASK) != KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL) - return -EINVAL; - - /* - * Map all the possible incoming states to the only two we - * really want to deal with. - */ - switch (val & KVM_REG_FEATURE_LEVEL_MASK) { - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL: - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_UNKNOWN: - wa_level = KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_AVAIL; - break; - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_AVAIL: - case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED: - wa_level = KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED; - break; - default: - return -EINVAL; - } - - /* - * We can deal with NOT_AVAIL on NOT_REQUIRED, but not the - * other way around. - */ - if (get_kernel_wa_level(reg->id) < wa_level) - return -EINVAL; - - return 0; - default: - return -ENOENT; - } - - return -EINVAL; -} diff --git a/include/kvm/arm_hypercalls.h b/include/kvm/arm_hypercalls.h index 0e2509d27910..5d38628a8d04 100644 --- a/include/kvm/arm_hypercalls.h +++ b/include/kvm/arm_hypercalls.h @@ -40,4 +40,11 @@ static inline void smccc_set_retval(struct kvm_vcpu *vcpu, vcpu_set_reg(vcpu, 3, a3); } +struct kvm_one_reg; + +int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu); +int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); +int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); +int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); + #endif diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h index 68b96c3826c3..6e55b9283789 100644 --- a/include/kvm/arm_psci.h +++ b/include/kvm/arm_psci.h @@ -39,11 +39,4 @@ static inline int kvm_psci_version(struct kvm_vcpu *vcpu) int kvm_psci_call(struct kvm_vcpu *vcpu); -struct kvm_one_reg; - -int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu); -int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); -int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); -int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); - #endif /* __KVM_ARM_PSCI_H__ */ From 05714cab7d63b189894235cf310fae7d6ffc2e9b Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:46 +0000 Subject: [PATCH 185/319] KVM: arm64: Setup a framework for hypercall bitmap firmware registers KVM regularly introduces new hypercall services to the guests without any consent from the userspace. This means, the guests can observe hypercall services in and out as they migrate across various host kernel versions. This could be a major problem if the guest discovered a hypercall, started using it, and after getting migrated to an older kernel realizes that it's no longer available. Depending on how the guest handles the change, there's a potential chance that the guest would just panic. As a result, there's a need for the userspace to elect the services that it wishes the guest to discover. It can elect these services based on the kernels spread across its (migration) fleet. To remedy this, extend the existing firmware pseudo-registers, such as KVM_REG_ARM_PSCI_VERSION, but by creating a new COPROC register space for all the hypercall services available. These firmware registers are categorized based on the service call owners, but unlike the existing firmware pseudo-registers, they hold the features supported in the form of a bitmap. During the VM initialization, the registers are set to upper-limit of the features supported by the corresponding registers. It's expected that the VMMs discover the features provided by each register via GET_ONE_REG, and write back the desired values using SET_ONE_REG. KVM allows this modification only until the VM has started. Some of the standard features are not mapped to any bits of the registers. But since they can recreate the original problem of making it available without userspace's consent, they need to be explicitly added to the case-list in kvm_hvc_call_default_allowed(). Any function-id that's not enabled via the bitmap, or not listed in kvm_hvc_call_default_allowed, will be returned as SMCCC_RET_NOT_SUPPORTED to the guest. Older userspace code can simply ignore the feature and the hypercall services will be exposed unconditionally to the guests, thus ensuring backward compatibility. In this patch, the framework adds the register only for ARM's standard secure services (owner value 4). Currently, this includes support only for ARM True Random Number Generator (TRNG) service, with bit-0 of the register representing mandatory features of v1.0. Other services are momentarily added in the upcoming patches. Signed-off-by: Raghavendra Rao Ananta Reviewed-by: Gavin Shan [maz: reduced the scope of some helpers, tidy-up bitmap max values, dropped error-only fast path] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-3-rananta@google.com --- arch/arm64/include/asm/kvm_host.h | 12 ++++ arch/arm64/include/uapi/asm/kvm.h | 13 ++++ arch/arm64/kvm/arm.c | 1 + arch/arm64/kvm/guest.c | 8 ++- arch/arm64/kvm/hypercalls.c | 104 ++++++++++++++++++++++++++++++ include/kvm/arm_hypercalls.h | 1 + 6 files changed, 137 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 94a27a7520f4..fef597af0beb 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -101,6 +101,15 @@ struct kvm_s2_mmu { struct kvm_arch_memory_slot { }; +/** + * struct kvm_smccc_features: Descriptor of the hypercall services exposed to the guests + * + * @std_bmap: Bitmap of standard secure service calls + */ +struct kvm_smccc_features { + unsigned long std_bmap; +}; + struct kvm_arch { struct kvm_s2_mmu mmu; @@ -150,6 +159,9 @@ struct kvm_arch { u8 pfr0_csv2; u8 pfr0_csv3; + + /* Hypercall features firmware registers' descriptor */ + struct kvm_smccc_features smccc_feat; }; struct kvm_vcpu_fault_info { diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index c1b6ddc02d2f..b0be5ec196b9 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -332,6 +332,19 @@ struct kvm_arm_copy_mte_tags { #define KVM_ARM64_SVE_VLS_WORDS \ ((KVM_ARM64_SVE_VQ_MAX - KVM_ARM64_SVE_VQ_MIN) / 64 + 1) +/* Bitmap feature firmware registers */ +#define KVM_REG_ARM_FW_FEAT_BMAP (0x0016 << KVM_REG_ARM_COPROC_SHIFT) +#define KVM_REG_ARM_FW_FEAT_BMAP_REG(r) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ + KVM_REG_ARM_FW_FEAT_BMAP | \ + ((r) & 0xffff)) + +#define KVM_REG_ARM_STD_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(0) + +enum { + KVM_REG_ARM_STD_BIT_TRNG_V1_0 = 0, + KVM_REG_ARM_STD_BMAP_BIT_COUNT, +}; + /* Device Control API: ARM VGIC */ #define KVM_DEV_ARM_VGIC_GRP_ADDR 0 #define KVM_DEV_ARM_VGIC_GRP_DIST_REGS 1 diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 523bc934fe2f..a37fadbd617e 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -156,6 +156,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.max_vcpus = kvm_arm_default_max_vcpus(); set_default_spectre(kvm); + kvm_arm_init_hypercalls(kvm); return ret; out_free_stage2_pgd: diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 0d5cca56cbda..8c607199cad1 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -756,7 +756,9 @@ int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) switch (reg->id & KVM_REG_ARM_COPROC_MASK) { case KVM_REG_ARM_CORE: return get_core_reg(vcpu, reg); - case KVM_REG_ARM_FW: return kvm_arm_get_fw_reg(vcpu, reg); + case KVM_REG_ARM_FW: + case KVM_REG_ARM_FW_FEAT_BMAP: + return kvm_arm_get_fw_reg(vcpu, reg); case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg); } @@ -774,7 +776,9 @@ int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) switch (reg->id & KVM_REG_ARM_COPROC_MASK) { case KVM_REG_ARM_CORE: return set_core_reg(vcpu, reg); - case KVM_REG_ARM_FW: return kvm_arm_set_fw_reg(vcpu, reg); + case KVM_REG_ARM_FW: + case KVM_REG_ARM_FW_FEAT_BMAP: + return kvm_arm_set_fw_reg(vcpu, reg); case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg); } diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index dd4e20b8d656..22f3a919e699 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -9,6 +9,9 @@ #include #include +#define KVM_ARM_SMCCC_STD_FEATURES \ + GENMASK(KVM_REG_ARM_STD_BMAP_BIT_COUNT - 1, 0) + static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val) { struct system_time_snapshot systime_snapshot; @@ -58,6 +61,56 @@ static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val) val[3] = lower_32_bits(cycles); } +static bool kvm_hvc_call_default_allowed(u32 func_id) +{ + switch (func_id) { + /* + * List of function-ids that are not gated with the bitmapped + * feature firmware registers, and are to be allowed for + * servicing the call by default. + */ + case ARM_SMCCC_VERSION_FUNC_ID: + case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: + case ARM_SMCCC_HV_PV_TIME_FEATURES: + case ARM_SMCCC_HV_PV_TIME_ST: + case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: + case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: + case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID: + return true; + default: + /* PSCI 0.2 and up is in the 0:0x1f range */ + if (ARM_SMCCC_OWNER_NUM(func_id) == ARM_SMCCC_OWNER_STANDARD && + ARM_SMCCC_FUNC_NUM(func_id) <= 0x1f) + return true; + + /* + * KVM's PSCI 0.1 doesn't comply with SMCCC, and has + * its own function-id base and range + */ + if (func_id >= KVM_PSCI_FN(0) && func_id <= KVM_PSCI_FN(3)) + return true; + + return false; + } +} + +static bool kvm_hvc_call_allowed(struct kvm_vcpu *vcpu, u32 func_id) +{ + struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat; + + switch (func_id) { + case ARM_SMCCC_TRNG_VERSION: + case ARM_SMCCC_TRNG_FEATURES: + case ARM_SMCCC_TRNG_GET_UUID: + case ARM_SMCCC_TRNG_RND32: + case ARM_SMCCC_TRNG_RND64: + return test_bit(KVM_REG_ARM_STD_BIT_TRNG_V1_0, + &smccc_feat->std_bmap); + default: + return kvm_hvc_call_default_allowed(func_id); + } +} + int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) { u32 func_id = smccc_get_function(vcpu); @@ -65,6 +118,9 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) u32 feature; gpa_t gpa; + if (!kvm_hvc_call_allowed(vcpu, func_id)) + goto out; + switch (func_id) { case ARM_SMCCC_VERSION_FUNC_ID: val[0] = ARM_SMCCC_VERSION_1_1; @@ -155,6 +211,7 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) return kvm_psci_call(vcpu); } +out: smccc_set_retval(vcpu, val[0], val[1], val[2], val[3]); return 1; } @@ -164,8 +221,16 @@ static const u64 kvm_arm_fw_reg_ids[] = { KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1, KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, + KVM_REG_ARM_STD_BMAP, }; +void kvm_arm_init_hypercalls(struct kvm *kvm) +{ + struct kvm_smccc_features *smccc_feat = &kvm->arch.smccc_feat; + + smccc_feat->std_bmap = KVM_ARM_SMCCC_STD_FEATURES; +} + int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu) { return ARRAY_SIZE(kvm_arm_fw_reg_ids); @@ -236,6 +301,7 @@ static int get_kernel_wa_level(u64 regid) int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) { + struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat; void __user *uaddr = (void __user *)(long)reg->addr; u64 val; @@ -248,6 +314,9 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) case KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3: val = get_kernel_wa_level(reg->id) & KVM_REG_FEATURE_LEVEL_MASK; break; + case KVM_REG_ARM_STD_BMAP: + val = READ_ONCE(smccc_feat->std_bmap); + break; default: return -ENOENT; } @@ -258,6 +327,39 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return 0; } +static int kvm_arm_set_fw_reg_bmap(struct kvm_vcpu *vcpu, u64 reg_id, u64 val) +{ + int ret = 0; + struct kvm *kvm = vcpu->kvm; + struct kvm_smccc_features *smccc_feat = &kvm->arch.smccc_feat; + unsigned long *fw_reg_bmap, fw_reg_features; + + switch (reg_id) { + case KVM_REG_ARM_STD_BMAP: + fw_reg_bmap = &smccc_feat->std_bmap; + fw_reg_features = KVM_ARM_SMCCC_STD_FEATURES; + break; + default: + return -ENOENT; + } + + /* Check for unsupported bit */ + if (val & ~fw_reg_features) + return -EINVAL; + + mutex_lock(&kvm->lock); + + if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags)) { + ret = -EBUSY; + goto out; + } + + WRITE_ONCE(*fw_reg_bmap, val); +out: + mutex_unlock(&kvm->lock); + return ret; +} + int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) { void __user *uaddr = (void __user *)(long)reg->addr; @@ -336,6 +438,8 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return -EINVAL; return 0; + case KVM_REG_ARM_STD_BMAP: + return kvm_arm_set_fw_reg_bmap(vcpu, reg->id, val); default: return -ENOENT; } diff --git a/include/kvm/arm_hypercalls.h b/include/kvm/arm_hypercalls.h index 5d38628a8d04..1188f116cf4e 100644 --- a/include/kvm/arm_hypercalls.h +++ b/include/kvm/arm_hypercalls.h @@ -42,6 +42,7 @@ static inline void smccc_set_retval(struct kvm_vcpu *vcpu, struct kvm_one_reg; +void kvm_arm_init_hypercalls(struct kvm *kvm); int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu); int kvm_arm_copy_fw_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices); int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg); From 428fd6788d4d0e0d390de9fb4486be3c1187310d Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:47 +0000 Subject: [PATCH 186/319] KVM: arm64: Add standard hypervisor firmware register Introduce the firmware register to hold the standard hypervisor service calls (owner value 5) as a bitmap. The bitmap represents the features that'll be enabled for the guest, as configured by the user-space. Currently, this includes support only for Paravirtualized time, represented by bit-0. Signed-off-by: Raghavendra Rao Ananta Reviewed-by: Gavin Shan [maz: tidy-up bitmap values] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-4-rananta@google.com --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/include/uapi/asm/kvm.h | 7 +++++++ arch/arm64/kvm/hypercalls.c | 23 ++++++++++++++++++++--- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index fef597af0beb..281dfcfd0a4e 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -105,9 +105,11 @@ struct kvm_arch_memory_slot { * struct kvm_smccc_features: Descriptor of the hypercall services exposed to the guests * * @std_bmap: Bitmap of standard secure service calls + * @std_hyp_bmap: Bitmap of standard hypervisor service calls */ struct kvm_smccc_features { unsigned long std_bmap; + unsigned long std_hyp_bmap; }; struct kvm_arch { diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index b0be5ec196b9..7ff5a2cf7547 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -345,6 +345,13 @@ enum { KVM_REG_ARM_STD_BMAP_BIT_COUNT, }; +#define KVM_REG_ARM_STD_HYP_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(1) + +enum { + KVM_REG_ARM_STD_HYP_BIT_PV_TIME = 0, + KVM_REG_ARM_STD_HYP_BMAP_BIT_COUNT, +}; + /* Device Control API: ARM VGIC */ #define KVM_DEV_ARM_VGIC_GRP_ADDR 0 #define KVM_DEV_ARM_VGIC_GRP_DIST_REGS 1 diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index 22f3a919e699..c43f78c33d64 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -11,6 +11,8 @@ #define KVM_ARM_SMCCC_STD_FEATURES \ GENMASK(KVM_REG_ARM_STD_BMAP_BIT_COUNT - 1, 0) +#define KVM_ARM_SMCCC_STD_HYP_FEATURES \ + GENMASK(KVM_REG_ARM_STD_HYP_BMAP_BIT_COUNT - 1, 0) static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val) { @@ -71,8 +73,6 @@ static bool kvm_hvc_call_default_allowed(u32 func_id) */ case ARM_SMCCC_VERSION_FUNC_ID: case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: - case ARM_SMCCC_HV_PV_TIME_FEATURES: - case ARM_SMCCC_HV_PV_TIME_ST: case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID: @@ -106,6 +106,10 @@ static bool kvm_hvc_call_allowed(struct kvm_vcpu *vcpu, u32 func_id) case ARM_SMCCC_TRNG_RND64: return test_bit(KVM_REG_ARM_STD_BIT_TRNG_V1_0, &smccc_feat->std_bmap); + case ARM_SMCCC_HV_PV_TIME_FEATURES: + case ARM_SMCCC_HV_PV_TIME_ST: + return test_bit(KVM_REG_ARM_STD_HYP_BIT_PV_TIME, + &smccc_feat->std_hyp_bmap); default: return kvm_hvc_call_default_allowed(func_id); } @@ -113,6 +117,7 @@ static bool kvm_hvc_call_allowed(struct kvm_vcpu *vcpu, u32 func_id) int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) { + struct kvm_smccc_features *smccc_feat = &vcpu->kvm->arch.smccc_feat; u32 func_id = smccc_get_function(vcpu); u64 val[4] = {SMCCC_RET_NOT_SUPPORTED}; u32 feature; @@ -176,7 +181,9 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) } break; case ARM_SMCCC_HV_PV_TIME_FEATURES: - val[0] = SMCCC_RET_SUCCESS; + if (test_bit(KVM_REG_ARM_STD_HYP_BIT_PV_TIME, + &smccc_feat->std_hyp_bmap)) + val[0] = SMCCC_RET_SUCCESS; break; } break; @@ -222,6 +229,7 @@ static const u64 kvm_arm_fw_reg_ids[] = { KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2, KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, KVM_REG_ARM_STD_BMAP, + KVM_REG_ARM_STD_HYP_BMAP, }; void kvm_arm_init_hypercalls(struct kvm *kvm) @@ -229,6 +237,7 @@ void kvm_arm_init_hypercalls(struct kvm *kvm) struct kvm_smccc_features *smccc_feat = &kvm->arch.smccc_feat; smccc_feat->std_bmap = KVM_ARM_SMCCC_STD_FEATURES; + smccc_feat->std_hyp_bmap = KVM_ARM_SMCCC_STD_HYP_FEATURES; } int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu) @@ -317,6 +326,9 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) case KVM_REG_ARM_STD_BMAP: val = READ_ONCE(smccc_feat->std_bmap); break; + case KVM_REG_ARM_STD_HYP_BMAP: + val = READ_ONCE(smccc_feat->std_hyp_bmap); + break; default: return -ENOENT; } @@ -339,6 +351,10 @@ static int kvm_arm_set_fw_reg_bmap(struct kvm_vcpu *vcpu, u64 reg_id, u64 val) fw_reg_bmap = &smccc_feat->std_bmap; fw_reg_features = KVM_ARM_SMCCC_STD_FEATURES; break; + case KVM_REG_ARM_STD_HYP_BMAP: + fw_reg_bmap = &smccc_feat->std_hyp_bmap; + fw_reg_features = KVM_ARM_SMCCC_STD_HYP_FEATURES; + break; default: return -ENOENT; } @@ -439,6 +455,7 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return 0; case KVM_REG_ARM_STD_BMAP: + case KVM_REG_ARM_STD_HYP_BMAP: return kvm_arm_set_fw_reg_bmap(vcpu, reg->id, val); default: return -ENOENT; From b22216e1a617ca55b41337cd1e057ebc784a65d4 Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:48 +0000 Subject: [PATCH 187/319] KVM: arm64: Add vendor hypervisor firmware register Introduce the firmware register to hold the vendor specific hypervisor service calls (owner value 6) as a bitmap. The bitmap represents the features that'll be enabled for the guest, as configured by the user-space. Currently, this includes support for KVM-vendor features along with reading the UID, represented by bit-0, and Precision Time Protocol (PTP), represented by bit-1. Signed-off-by: Raghavendra Rao Ananta Reviewed-by: Gavin Shan [maz: tidy-up bitmap values] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-5-rananta@google.com --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/include/uapi/asm/kvm.h | 8 ++++++++ arch/arm64/kvm/hypercalls.c | 25 ++++++++++++++++++++----- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 281dfcfd0a4e..35a60d766fba 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -106,10 +106,12 @@ struct kvm_arch_memory_slot { * * @std_bmap: Bitmap of standard secure service calls * @std_hyp_bmap: Bitmap of standard hypervisor service calls + * @vendor_hyp_bmap: Bitmap of vendor specific hypervisor service calls */ struct kvm_smccc_features { unsigned long std_bmap; unsigned long std_hyp_bmap; + unsigned long vendor_hyp_bmap; }; struct kvm_arch { diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 7ff5a2cf7547..e523bb6eac67 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -352,6 +352,14 @@ enum { KVM_REG_ARM_STD_HYP_BMAP_BIT_COUNT, }; +#define KVM_REG_ARM_VENDOR_HYP_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(2) + +enum { + KVM_REG_ARM_VENDOR_HYP_BIT_FUNC_FEAT = 0, + KVM_REG_ARM_VENDOR_HYP_BIT_PTP = 1, + KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_COUNT, +}; + /* Device Control API: ARM VGIC */ #define KVM_DEV_ARM_VGIC_GRP_ADDR 0 #define KVM_DEV_ARM_VGIC_GRP_DIST_REGS 1 diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index c43f78c33d64..ccbd3cefb91a 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -13,6 +13,8 @@ GENMASK(KVM_REG_ARM_STD_BMAP_BIT_COUNT - 1, 0) #define KVM_ARM_SMCCC_STD_HYP_FEATURES \ GENMASK(KVM_REG_ARM_STD_HYP_BMAP_BIT_COUNT - 1, 0) +#define KVM_ARM_SMCCC_VENDOR_HYP_FEATURES \ + GENMASK(KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_COUNT - 1, 0) static void kvm_ptp_get_time(struct kvm_vcpu *vcpu, u64 *val) { @@ -73,9 +75,6 @@ static bool kvm_hvc_call_default_allowed(u32 func_id) */ case ARM_SMCCC_VERSION_FUNC_ID: case ARM_SMCCC_ARCH_FEATURES_FUNC_ID: - case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: - case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: - case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID: return true; default: /* PSCI 0.2 and up is in the 0:0x1f range */ @@ -110,6 +109,13 @@ static bool kvm_hvc_call_allowed(struct kvm_vcpu *vcpu, u32 func_id) case ARM_SMCCC_HV_PV_TIME_ST: return test_bit(KVM_REG_ARM_STD_HYP_BIT_PV_TIME, &smccc_feat->std_hyp_bmap); + case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: + case ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID: + return test_bit(KVM_REG_ARM_VENDOR_HYP_BIT_FUNC_FEAT, + &smccc_feat->vendor_hyp_bmap); + case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID: + return test_bit(KVM_REG_ARM_VENDOR_HYP_BIT_PTP, + &smccc_feat->vendor_hyp_bmap); default: return kvm_hvc_call_default_allowed(func_id); } @@ -202,8 +208,7 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu) val[3] = ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_3; break; case ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID: - val[0] = BIT(ARM_SMCCC_KVM_FUNC_FEATURES); - val[0] |= BIT(ARM_SMCCC_KVM_FUNC_PTP); + val[0] = smccc_feat->vendor_hyp_bmap; break; case ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID: kvm_ptp_get_time(vcpu, val); @@ -230,6 +235,7 @@ static const u64 kvm_arm_fw_reg_ids[] = { KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3, KVM_REG_ARM_STD_BMAP, KVM_REG_ARM_STD_HYP_BMAP, + KVM_REG_ARM_VENDOR_HYP_BMAP, }; void kvm_arm_init_hypercalls(struct kvm *kvm) @@ -238,6 +244,7 @@ void kvm_arm_init_hypercalls(struct kvm *kvm) smccc_feat->std_bmap = KVM_ARM_SMCCC_STD_FEATURES; smccc_feat->std_hyp_bmap = KVM_ARM_SMCCC_STD_HYP_FEATURES; + smccc_feat->vendor_hyp_bmap = KVM_ARM_SMCCC_VENDOR_HYP_FEATURES; } int kvm_arm_get_fw_num_regs(struct kvm_vcpu *vcpu) @@ -329,6 +336,9 @@ int kvm_arm_get_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) case KVM_REG_ARM_STD_HYP_BMAP: val = READ_ONCE(smccc_feat->std_hyp_bmap); break; + case KVM_REG_ARM_VENDOR_HYP_BMAP: + val = READ_ONCE(smccc_feat->vendor_hyp_bmap); + break; default: return -ENOENT; } @@ -355,6 +365,10 @@ static int kvm_arm_set_fw_reg_bmap(struct kvm_vcpu *vcpu, u64 reg_id, u64 val) fw_reg_bmap = &smccc_feat->std_hyp_bmap; fw_reg_features = KVM_ARM_SMCCC_STD_HYP_FEATURES; break; + case KVM_REG_ARM_VENDOR_HYP_BMAP: + fw_reg_bmap = &smccc_feat->vendor_hyp_bmap; + fw_reg_features = KVM_ARM_SMCCC_VENDOR_HYP_FEATURES; + break; default: return -ENOENT; } @@ -456,6 +470,7 @@ int kvm_arm_set_fw_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) return 0; case KVM_REG_ARM_STD_BMAP: case KVM_REG_ARM_STD_HYP_BMAP: + case KVM_REG_ARM_VENDOR_HYP_BMAP: return kvm_arm_set_fw_reg_bmap(vcpu, reg->id, val); default: return -ENOENT; From f1ced23a9be5727c6f4cac0e2262c5411038952f Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:49 +0000 Subject: [PATCH 188/319] Docs: KVM: Rename psci.rst to hypercalls.rst Since the doc also covers general hypercalls' details, rather than just PSCI, and the fact that the bitmap firmware registers' details will be added to this doc, rename the file to a more appropriate name- hypercalls.rst. Signed-off-by: Raghavendra Rao Ananta Reviewed-by: Gavin Shan Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-6-rananta@google.com --- Documentation/virt/kvm/arm/{psci.rst => hypercalls.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Documentation/virt/kvm/arm/{psci.rst => hypercalls.rst} (100%) diff --git a/Documentation/virt/kvm/arm/psci.rst b/Documentation/virt/kvm/arm/hypercalls.rst similarity index 100% rename from Documentation/virt/kvm/arm/psci.rst rename to Documentation/virt/kvm/arm/hypercalls.rst From fa246c68a04d46c7af6953b47dba7e16d24efbe2 Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:50 +0000 Subject: [PATCH 189/319] Docs: KVM: Add doc for the bitmap firmware registers Add the documentation for the bitmap firmware registers in hypercalls.rst and api.rst. This includes the details for KVM_REG_ARM_STD_BMAP, KVM_REG_ARM_STD_HYP_BMAP, and KVM_REG_ARM_VENDOR_HYP_BMAP registers. Since the document is growing to carry other hypercall related information, make necessary adjustments to present the document in a generic sense, rather than being PSCI focused. Signed-off-by: Raghavendra Rao Ananta Reviewed-by: Gavin Shan [maz: small scale reformat, move things about, random typo fixes] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-7-rananta@google.com --- Documentation/virt/kvm/api.rst | 18 +++++ Documentation/virt/kvm/arm/hypercalls.rst | 95 +++++++++++++++++++---- 2 files changed, 96 insertions(+), 17 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 85c7abc51af5..b5ccec4572d7 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -2601,6 +2601,24 @@ EINVAL. After the vcpu's SVE configuration is finalized, further attempts to write this register will fail with EPERM. +arm64 bitmap feature firmware pseudo-registers have the following bit pattern:: + + 0x6030 0000 0016 + +The bitmap feature firmware registers exposes the hypercall services that +are available for userspace to configure. The set bits corresponds to the +services that are available for the guests to access. By default, KVM +sets all the supported bits during VM initialization. The userspace can +discover the available services via KVM_GET_ONE_REG, and write back the +bitmap corresponding to the features that it wishes guests to see via +KVM_SET_ONE_REG. + +Note: These registers are immutable once any of the vCPUs of the VM has +run at least once. A KVM_SET_ONE_REG in such a scenario will return +a -EBUSY to userspace. + +(See Documentation/virt/kvm/arm/hypercalls.rst for more details.) + MIPS registers are mapped using the lower 32 bits. The upper 16 of that is the register group type: diff --git a/Documentation/virt/kvm/arm/hypercalls.rst b/Documentation/virt/kvm/arm/hypercalls.rst index d52c2e83b5b8..3e23084644ba 100644 --- a/Documentation/virt/kvm/arm/hypercalls.rst +++ b/Documentation/virt/kvm/arm/hypercalls.rst @@ -1,32 +1,32 @@ .. SPDX-License-Identifier: GPL-2.0 -========================================= -Power State Coordination Interface (PSCI) -========================================= +======================= +ARM Hypercall Interface +======================= -KVM implements the PSCI (Power State Coordination Interface) -specification in order to provide services such as CPU on/off, reset -and power-off to the guest. +KVM handles the hypercall services as requested by the guests. New hypercall +services are regularly made available by the ARM specification or by KVM (as +vendor services) if they make sense from a virtualization point of view. -The PSCI specification is regularly updated to provide new features, -and KVM implements these updates if they make sense from a virtualization -point of view. - -This means that a guest booted on two different versions of KVM can -observe two different "firmware" revisions. This could cause issues if -a given guest is tied to a particular PSCI revision (unlikely), or if -a migration causes a different PSCI version to be exposed out of the -blue to an unsuspecting guest. +This means that a guest booted on two different versions of KVM can observe +two different "firmware" revisions. This could cause issues if a given guest +is tied to a particular version of a hypercall service, or if a migration +causes a different version to be exposed out of the blue to an unsuspecting +guest. In order to remedy this situation, KVM exposes a set of "firmware pseudo-registers" that can be manipulated using the GET/SET_ONE_REG interface. These registers can be saved/restored by userspace, and set -to a convenient value if required. +to a convenient value as required. -The following register is defined: +The following registers are defined: * KVM_REG_ARM_PSCI_VERSION: + KVM implements the PSCI (Power State Coordination Interface) + specification in order to provide services such as CPU on/off, reset + and power-off to the guest. + - Only valid if the vcpu has the KVM_ARM_VCPU_PSCI_0_2 feature set (and thus has already been initialized) - Returns the current PSCI version on GET_ONE_REG (defaulting to the @@ -74,4 +74,65 @@ The following register is defined: KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2_NOT_REQUIRED: The workaround is always active on this vCPU or it is not needed. + +Bitmap Feature Firmware Registers +--------------------------------- + +Contrary to the above registers, the following registers exposes the +hypercall services in the form of a feature-bitmap to the userspace. This +bitmap is translated to the services that are available to the guest. +There is a register defined per service call owner and can be accessed via +GET/SET_ONE_REG interface. + +By default, these registers are set with the upper limit of the features +that are supported. This way userspace can discover all the usable +hypercall services via GET_ONE_REG. The user-space can write-back the +desired bitmap back via SET_ONE_REG. The features for the registers that +are untouched, probably because userspace isn't aware of them, will be +exposed as is to the guest. + +Note that KVM will not allow the userspace to configure the registers +anymore once any of the vCPUs has run at least once. Instead, it will +return a -EBUSY. + +The pseudo-firmware bitmap register are as follows: + +* KVM_REG_ARM_STD_BMAP: + Controls the bitmap of the ARM Standard Secure Service Calls. + + The following bits are accepted: + + Bit-0: KVM_REG_ARM_STD_BIT_TRNG_V1_0: + The bit represents the services offered under v1.0 of ARM True Random + Number Generator (TRNG) specification, ARM DEN0098. + +* KVM_REG_ARM_STD_HYP_BMAP: + Controls the bitmap of the ARM Standard Hypervisor Service Calls. + + The following bits are accepted: + + Bit-0: KVM_REG_ARM_STD_HYP_BIT_PV_TIME: + The bit represents the Paravirtualized Time service as represented by + ARM DEN0057A. + +* KVM_REG_ARM_VENDOR_HYP_BMAP: + Controls the bitmap of the Vendor specific Hypervisor Service Calls. + + The following bits are accepted: + + Bit-0: KVM_REG_ARM_VENDOR_HYP_BIT_FUNC_FEAT + The bit represents the ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID + and ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID function-ids. + + Bit-1: KVM_REG_ARM_VENDOR_HYP_BIT_PTP: + The bit represents the Precision Time Protocol KVM service. + +Errors: + + ======= ============================================================= + -ENOENT Unknown register accessed. + -EBUSY Attempt a 'write' to the register after the VM has started. + -EINVAL Invalid bitmap written to the register. + ======= ============================================================= + .. [1] https://developer.arm.com/-/media/developer/pdf/ARM_DEN_0070A_Firmware_interfaces_for_mitigating_CVE-2017-5715.pdf From ea733263949646700977feeb662a92703f514351 Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:51 +0000 Subject: [PATCH 190/319] tools: Import ARM SMCCC definitions Import the standard SMCCC definitions from include/linux/arm-smccc.h. Signed-off-by: Raghavendra Rao Ananta Reviewed-by: Gavin Shan Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-8-rananta@google.com --- tools/include/linux/arm-smccc.h | 193 ++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 tools/include/linux/arm-smccc.h diff --git a/tools/include/linux/arm-smccc.h b/tools/include/linux/arm-smccc.h new file mode 100644 index 000000000000..63ce9bebccd3 --- /dev/null +++ b/tools/include/linux/arm-smccc.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2015, Linaro Limited + */ +#ifndef __LINUX_ARM_SMCCC_H +#define __LINUX_ARM_SMCCC_H + +#include + +/* + * This file provides common defines for ARM SMC Calling Convention as + * specified in + * https://developer.arm.com/docs/den0028/latest + * + * This code is up-to-date with version DEN 0028 C + */ + +#define ARM_SMCCC_STD_CALL _AC(0,U) +#define ARM_SMCCC_FAST_CALL _AC(1,U) +#define ARM_SMCCC_TYPE_SHIFT 31 + +#define ARM_SMCCC_SMC_32 0 +#define ARM_SMCCC_SMC_64 1 +#define ARM_SMCCC_CALL_CONV_SHIFT 30 + +#define ARM_SMCCC_OWNER_MASK 0x3F +#define ARM_SMCCC_OWNER_SHIFT 24 + +#define ARM_SMCCC_FUNC_MASK 0xFFFF + +#define ARM_SMCCC_IS_FAST_CALL(smc_val) \ + ((smc_val) & (ARM_SMCCC_FAST_CALL << ARM_SMCCC_TYPE_SHIFT)) +#define ARM_SMCCC_IS_64(smc_val) \ + ((smc_val) & (ARM_SMCCC_SMC_64 << ARM_SMCCC_CALL_CONV_SHIFT)) +#define ARM_SMCCC_FUNC_NUM(smc_val) ((smc_val) & ARM_SMCCC_FUNC_MASK) +#define ARM_SMCCC_OWNER_NUM(smc_val) \ + (((smc_val) >> ARM_SMCCC_OWNER_SHIFT) & ARM_SMCCC_OWNER_MASK) + +#define ARM_SMCCC_CALL_VAL(type, calling_convention, owner, func_num) \ + (((type) << ARM_SMCCC_TYPE_SHIFT) | \ + ((calling_convention) << ARM_SMCCC_CALL_CONV_SHIFT) | \ + (((owner) & ARM_SMCCC_OWNER_MASK) << ARM_SMCCC_OWNER_SHIFT) | \ + ((func_num) & ARM_SMCCC_FUNC_MASK)) + +#define ARM_SMCCC_OWNER_ARCH 0 +#define ARM_SMCCC_OWNER_CPU 1 +#define ARM_SMCCC_OWNER_SIP 2 +#define ARM_SMCCC_OWNER_OEM 3 +#define ARM_SMCCC_OWNER_STANDARD 4 +#define ARM_SMCCC_OWNER_STANDARD_HYP 5 +#define ARM_SMCCC_OWNER_VENDOR_HYP 6 +#define ARM_SMCCC_OWNER_TRUSTED_APP 48 +#define ARM_SMCCC_OWNER_TRUSTED_APP_END 49 +#define ARM_SMCCC_OWNER_TRUSTED_OS 50 +#define ARM_SMCCC_OWNER_TRUSTED_OS_END 63 + +#define ARM_SMCCC_FUNC_QUERY_CALL_UID 0xff01 + +#define ARM_SMCCC_QUIRK_NONE 0 +#define ARM_SMCCC_QUIRK_QCOM_A6 1 /* Save/restore register a6 */ + +#define ARM_SMCCC_VERSION_1_0 0x10000 +#define ARM_SMCCC_VERSION_1_1 0x10001 +#define ARM_SMCCC_VERSION_1_2 0x10002 +#define ARM_SMCCC_VERSION_1_3 0x10003 + +#define ARM_SMCCC_1_3_SVE_HINT 0x10000 + +#define ARM_SMCCC_VERSION_FUNC_ID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + 0, 0) + +#define ARM_SMCCC_ARCH_FEATURES_FUNC_ID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + 0, 1) + +#define ARM_SMCCC_ARCH_SOC_ID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + 0, 2) + +#define ARM_SMCCC_ARCH_WORKAROUND_1 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + 0, 0x8000) + +#define ARM_SMCCC_ARCH_WORKAROUND_2 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + 0, 0x7fff) + +#define ARM_SMCCC_ARCH_WORKAROUND_3 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + 0, 0x3fff) + +#define ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_VENDOR_HYP, \ + ARM_SMCCC_FUNC_QUERY_CALL_UID) + +/* KVM UID value: 28b46fb6-2ec5-11e9-a9ca-4b564d003a74 */ +#define ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_0 0xb66fb428U +#define ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_1 0xe911c52eU +#define ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_2 0x564bcaa9U +#define ARM_SMCCC_VENDOR_HYP_UID_KVM_REG_3 0x743a004dU + +/* KVM "vendor specific" services */ +#define ARM_SMCCC_KVM_FUNC_FEATURES 0 +#define ARM_SMCCC_KVM_FUNC_PTP 1 +#define ARM_SMCCC_KVM_FUNC_FEATURES_2 127 +#define ARM_SMCCC_KVM_NUM_FUNCS 128 + +#define ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_VENDOR_HYP, \ + ARM_SMCCC_KVM_FUNC_FEATURES) + +#define SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED 1 + +/* + * ptp_kvm is a feature used for time sync between vm and host. + * ptp_kvm module in guest kernel will get service from host using + * this hypercall ID. + */ +#define ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_VENDOR_HYP, \ + ARM_SMCCC_KVM_FUNC_PTP) + +/* ptp_kvm counter type ID */ +#define KVM_PTP_VIRT_COUNTER 0 +#define KVM_PTP_PHYS_COUNTER 1 + +/* Paravirtualised time calls (defined by ARM DEN0057A) */ +#define ARM_SMCCC_HV_PV_TIME_FEATURES \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_STANDARD_HYP, \ + 0x20) + +#define ARM_SMCCC_HV_PV_TIME_ST \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_STANDARD_HYP, \ + 0x21) + +/* TRNG entropy source calls (defined by ARM DEN0098) */ +#define ARM_SMCCC_TRNG_VERSION \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x50) + +#define ARM_SMCCC_TRNG_FEATURES \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x51) + +#define ARM_SMCCC_TRNG_GET_UUID \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x52) + +#define ARM_SMCCC_TRNG_RND32 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x53) + +#define ARM_SMCCC_TRNG_RND64 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_STANDARD, \ + 0x53) + +/* + * Return codes defined in ARM DEN 0070A + * ARM DEN 0070A is now merged/consolidated into ARM DEN 0028 C + */ +#define SMCCC_RET_SUCCESS 0 +#define SMCCC_RET_NOT_SUPPORTED -1 +#define SMCCC_RET_NOT_REQUIRED -2 +#define SMCCC_RET_INVALID_PARAMETER -3 + +#endif /*__LINUX_ARM_SMCCC_H*/ From bf08515d39cb843c81f991ee67ff543eecdba0c3 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Sat, 9 Apr 2022 18:45:45 +0000 Subject: [PATCH 191/319] selftests: KVM: Rename psci_cpu_on_test to psci_test There are other interactions with PSCI worth testing; rename the PSCI test to make it more generic. No functional change intended. Signed-off-by: Oliver Upton Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220409184549.1681189-10-oupton@google.com --- tools/testing/selftests/kvm/.gitignore | 2 +- tools/testing/selftests/kvm/Makefile | 2 +- .../selftests/kvm/aarch64/{psci_cpu_on_test.c => psci_test.c} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename tools/testing/selftests/kvm/aarch64/{psci_cpu_on_test.c => psci_test.c} (100%) diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 0b0e4402bba6..1bb575dfc42e 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -2,7 +2,7 @@ /aarch64/arch_timer /aarch64/debug-exceptions /aarch64/get-reg-list -/aarch64/psci_cpu_on_test +/aarch64/psci_test /aarch64/vcpu_width_config /aarch64/vgic_init /aarch64/vgic_irq diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 681b173aa87c..c2cf4d318296 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -105,7 +105,7 @@ TEST_GEN_PROGS_x86_64 += system_counter_offset_test TEST_GEN_PROGS_aarch64 += aarch64/arch_timer TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list -TEST_GEN_PROGS_aarch64 += aarch64/psci_cpu_on_test +TEST_GEN_PROGS_aarch64 += aarch64/psci_test TEST_GEN_PROGS_aarch64 += aarch64/vcpu_width_config TEST_GEN_PROGS_aarch64 += aarch64/vgic_init TEST_GEN_PROGS_aarch64 += aarch64/vgic_irq diff --git a/tools/testing/selftests/kvm/aarch64/psci_cpu_on_test.c b/tools/testing/selftests/kvm/aarch64/psci_test.c similarity index 100% rename from tools/testing/selftests/kvm/aarch64/psci_cpu_on_test.c rename to tools/testing/selftests/kvm/aarch64/psci_test.c From e918e2bc52c8ac1cccd6ef822ac23eded41761b6 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Sat, 9 Apr 2022 18:45:46 +0000 Subject: [PATCH 192/319] selftests: KVM: Create helper for making SMCCC calls The PSCI and PV stolen time tests both need to make SMCCC calls within the guest. Create a helper for making SMCCC calls and rework the existing tests to use the library function. Signed-off-by: Oliver Upton Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220409184549.1681189-11-oupton@google.com --- .../testing/selftests/kvm/aarch64/psci_test.c | 25 ++++++------------- .../selftests/kvm/include/aarch64/processor.h | 22 ++++++++++++++++ .../selftests/kvm/lib/aarch64/processor.c | 25 +++++++++++++++++++ tools/testing/selftests/kvm/steal_time.c | 13 +++------- 4 files changed, 58 insertions(+), 27 deletions(-) diff --git a/tools/testing/selftests/kvm/aarch64/psci_test.c b/tools/testing/selftests/kvm/aarch64/psci_test.c index 4c5f6814030f..8c998f0b802c 100644 --- a/tools/testing/selftests/kvm/aarch64/psci_test.c +++ b/tools/testing/selftests/kvm/aarch64/psci_test.c @@ -26,32 +26,23 @@ static uint64_t psci_cpu_on(uint64_t target_cpu, uint64_t entry_addr, uint64_t context_id) { - register uint64_t x0 asm("x0") = PSCI_0_2_FN64_CPU_ON; - register uint64_t x1 asm("x1") = target_cpu; - register uint64_t x2 asm("x2") = entry_addr; - register uint64_t x3 asm("x3") = context_id; + struct arm_smccc_res res; - asm("hvc #0" - : "=r"(x0) - : "r"(x0), "r"(x1), "r"(x2), "r"(x3) - : "memory"); + smccc_hvc(PSCI_0_2_FN64_CPU_ON, target_cpu, entry_addr, context_id, + 0, 0, 0, 0, &res); - return x0; + return res.a0; } static uint64_t psci_affinity_info(uint64_t target_affinity, uint64_t lowest_affinity_level) { - register uint64_t x0 asm("x0") = PSCI_0_2_FN64_AFFINITY_INFO; - register uint64_t x1 asm("x1") = target_affinity; - register uint64_t x2 asm("x2") = lowest_affinity_level; + struct arm_smccc_res res; - asm("hvc #0" - : "=r"(x0) - : "r"(x0), "r"(x1), "r"(x2) - : "memory"); + smccc_hvc(PSCI_0_2_FN64_AFFINITY_INFO, target_affinity, lowest_affinity_level, + 0, 0, 0, 0, 0, &res); - return x0; + return res.a0; } static void guest_main(uint64_t target_cpu) diff --git a/tools/testing/selftests/kvm/include/aarch64/processor.h b/tools/testing/selftests/kvm/include/aarch64/processor.h index 8f9f46979a00..59ece9d4e0d1 100644 --- a/tools/testing/selftests/kvm/include/aarch64/processor.h +++ b/tools/testing/selftests/kvm/include/aarch64/processor.h @@ -185,4 +185,26 @@ static inline void local_irq_disable(void) asm volatile("msr daifset, #3" : : : "memory"); } +/** + * struct arm_smccc_res - Result from SMC/HVC call + * @a0-a3 result values from registers 0 to 3 + */ +struct arm_smccc_res { + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; +}; + +/** + * smccc_hvc - Invoke a SMCCC function using the hvc conduit + * @function_id: the SMCCC function to be called + * @arg0-arg6: SMCCC function arguments, corresponding to registers x1-x7 + * @res: pointer to write the return values from registers x0-x3 + * + */ +void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1, + uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, + uint64_t arg6, struct arm_smccc_res *res); + #endif /* SELFTEST_KVM_PROCESSOR_H */ diff --git a/tools/testing/selftests/kvm/lib/aarch64/processor.c b/tools/testing/selftests/kvm/lib/aarch64/processor.c index 9343d82519b4..6a041289fa80 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/processor.c +++ b/tools/testing/selftests/kvm/lib/aarch64/processor.c @@ -500,3 +500,28 @@ void __attribute__((constructor)) init_guest_modes(void) { guest_modes_append_default(); } + +void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1, + uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, + uint64_t arg6, struct arm_smccc_res *res) +{ + asm volatile("mov w0, %w[function_id]\n" + "mov x1, %[arg0]\n" + "mov x2, %[arg1]\n" + "mov x3, %[arg2]\n" + "mov x4, %[arg3]\n" + "mov x5, %[arg4]\n" + "mov x6, %[arg5]\n" + "mov x7, %[arg6]\n" + "hvc #0\n" + "mov %[res0], x0\n" + "mov %[res1], x1\n" + "mov %[res2], x2\n" + "mov %[res3], x3\n" + : [res0] "=r"(res->a0), [res1] "=r"(res->a1), + [res2] "=r"(res->a2), [res3] "=r"(res->a3) + : [function_id] "r"(function_id), [arg0] "r"(arg0), + [arg1] "r"(arg1), [arg2] "r"(arg2), [arg3] "r"(arg3), + [arg4] "r"(arg4), [arg5] "r"(arg5), [arg6] "r"(arg6) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"); +} diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index 62f2eb9ee3d5..8c4e811bd586 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -118,17 +118,10 @@ struct st_time { static int64_t smccc(uint32_t func, uint64_t arg) { - unsigned long ret; + struct arm_smccc_res res; - asm volatile( - "mov w0, %w1\n" - "mov x1, %2\n" - "hvc #0\n" - "mov %0, x0\n" - : "=r" (ret) : "r" (func), "r" (arg) : - "x0", "x1", "x2", "x3"); - - return ret; + smccc_hvc(func, arg, 0, 0, 0, 0, 0, 0, &res); + return res.a0; } static void check_status(struct st_time *st) From 5ca24697d54027c1c94c94a5b920a75448108ed0 Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:52 +0000 Subject: [PATCH 193/319] selftests: KVM: aarch64: Introduce hypercall ABI test Introduce a KVM selftest to check the hypercall interface for arm64 platforms. The test validates the user-space' [GET|SET]_ONE_REG interface to read/write the psuedo-firmware registers as well as its effects on the guest upon certain configurations. Signed-off-by: Raghavendra Rao Ananta Reviewed-by: Gavin Shan Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-9-rananta@google.com --- tools/testing/selftests/kvm/.gitignore | 1 + tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/aarch64/hypercalls.c | 336 ++++++++++++++++++ 3 files changed, 338 insertions(+) create mode 100644 tools/testing/selftests/kvm/aarch64/hypercalls.c diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 1bb575dfc42e..b17e464ec661 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -2,6 +2,7 @@ /aarch64/arch_timer /aarch64/debug-exceptions /aarch64/get-reg-list +/aarch64/hypercalls /aarch64/psci_test /aarch64/vcpu_width_config /aarch64/vgic_init diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index c2cf4d318296..97eef0c03d3b 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -105,6 +105,7 @@ TEST_GEN_PROGS_x86_64 += system_counter_offset_test TEST_GEN_PROGS_aarch64 += aarch64/arch_timer TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list +TEST_GEN_PROGS_aarch64 += aarch64/hypercalls TEST_GEN_PROGS_aarch64 += aarch64/psci_test TEST_GEN_PROGS_aarch64 += aarch64/vcpu_width_config TEST_GEN_PROGS_aarch64 += aarch64/vgic_init diff --git a/tools/testing/selftests/kvm/aarch64/hypercalls.c b/tools/testing/selftests/kvm/aarch64/hypercalls.c new file mode 100644 index 000000000000..41e0210b7a5e --- /dev/null +++ b/tools/testing/selftests/kvm/aarch64/hypercalls.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* hypercalls: Check the ARM64's psuedo-firmware bitmap register interface. + * + * The test validates the basic hypercall functionalities that are exposed + * via the psuedo-firmware bitmap register. This includes the registers' + * read/write behavior before and after the VM has started, and if the + * hypercalls are properly masked or unmasked to the guest when disabled or + * enabled from the KVM userspace, respectively. + */ + +#include +#include +#include +#include + +#include "processor.h" + +#define FW_REG_ULIMIT_VAL(max_feat_bit) (GENMASK(max_feat_bit, 0)) + +/* Last valid bits of the bitmapped firmware registers */ +#define KVM_REG_ARM_STD_BMAP_BIT_MAX 0 +#define KVM_REG_ARM_STD_HYP_BMAP_BIT_MAX 0 +#define KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_MAX 1 + +struct kvm_fw_reg_info { + uint64_t reg; /* Register definition */ + uint64_t max_feat_bit; /* Bit that represents the upper limit of the feature-map */ +}; + +#define FW_REG_INFO(r) \ + { \ + .reg = r, \ + .max_feat_bit = r##_BIT_MAX, \ + } + +static const struct kvm_fw_reg_info fw_reg_info[] = { + FW_REG_INFO(KVM_REG_ARM_STD_BMAP), + FW_REG_INFO(KVM_REG_ARM_STD_HYP_BMAP), + FW_REG_INFO(KVM_REG_ARM_VENDOR_HYP_BMAP), +}; + +enum test_stage { + TEST_STAGE_REG_IFACE, + TEST_STAGE_HVC_IFACE_FEAT_DISABLED, + TEST_STAGE_HVC_IFACE_FEAT_ENABLED, + TEST_STAGE_HVC_IFACE_FALSE_INFO, + TEST_STAGE_END, +}; + +static int stage = TEST_STAGE_REG_IFACE; + +struct test_hvc_info { + uint32_t func_id; + uint64_t arg1; +}; + +#define TEST_HVC_INFO(f, a1) \ + { \ + .func_id = f, \ + .arg1 = a1, \ + } + +static const struct test_hvc_info hvc_info[] = { + /* KVM_REG_ARM_STD_BMAP */ + TEST_HVC_INFO(ARM_SMCCC_TRNG_VERSION, 0), + TEST_HVC_INFO(ARM_SMCCC_TRNG_FEATURES, ARM_SMCCC_TRNG_RND64), + TEST_HVC_INFO(ARM_SMCCC_TRNG_GET_UUID, 0), + TEST_HVC_INFO(ARM_SMCCC_TRNG_RND32, 0), + TEST_HVC_INFO(ARM_SMCCC_TRNG_RND64, 0), + + /* KVM_REG_ARM_STD_HYP_BMAP */ + TEST_HVC_INFO(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, ARM_SMCCC_HV_PV_TIME_FEATURES), + TEST_HVC_INFO(ARM_SMCCC_HV_PV_TIME_FEATURES, ARM_SMCCC_HV_PV_TIME_ST), + TEST_HVC_INFO(ARM_SMCCC_HV_PV_TIME_ST, 0), + + /* KVM_REG_ARM_VENDOR_HYP_BMAP */ + TEST_HVC_INFO(ARM_SMCCC_VENDOR_HYP_KVM_FEATURES_FUNC_ID, + ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID), + TEST_HVC_INFO(ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID, 0), + TEST_HVC_INFO(ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID, KVM_PTP_VIRT_COUNTER), +}; + +/* Feed false hypercall info to test the KVM behavior */ +static const struct test_hvc_info false_hvc_info[] = { + /* Feature support check against a different family of hypercalls */ + TEST_HVC_INFO(ARM_SMCCC_TRNG_FEATURES, ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID), + TEST_HVC_INFO(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, ARM_SMCCC_TRNG_RND64), + TEST_HVC_INFO(ARM_SMCCC_HV_PV_TIME_FEATURES, ARM_SMCCC_TRNG_RND64), +}; + +static void guest_test_hvc(const struct test_hvc_info *hc_info) +{ + unsigned int i; + struct arm_smccc_res res; + unsigned int hvc_info_arr_sz; + + hvc_info_arr_sz = + hc_info == hvc_info ? ARRAY_SIZE(hvc_info) : ARRAY_SIZE(false_hvc_info); + + for (i = 0; i < hvc_info_arr_sz; i++, hc_info++) { + memset(&res, 0, sizeof(res)); + smccc_hvc(hc_info->func_id, hc_info->arg1, 0, 0, 0, 0, 0, 0, &res); + + switch (stage) { + case TEST_STAGE_HVC_IFACE_FEAT_DISABLED: + case TEST_STAGE_HVC_IFACE_FALSE_INFO: + GUEST_ASSERT_3(res.a0 == SMCCC_RET_NOT_SUPPORTED, + res.a0, hc_info->func_id, hc_info->arg1); + break; + case TEST_STAGE_HVC_IFACE_FEAT_ENABLED: + GUEST_ASSERT_3(res.a0 != SMCCC_RET_NOT_SUPPORTED, + res.a0, hc_info->func_id, hc_info->arg1); + break; + default: + GUEST_ASSERT_1(0, stage); + } + } +} + +static void guest_code(void) +{ + while (stage != TEST_STAGE_END) { + switch (stage) { + case TEST_STAGE_REG_IFACE: + break; + case TEST_STAGE_HVC_IFACE_FEAT_DISABLED: + case TEST_STAGE_HVC_IFACE_FEAT_ENABLED: + guest_test_hvc(hvc_info); + break; + case TEST_STAGE_HVC_IFACE_FALSE_INFO: + guest_test_hvc(false_hvc_info); + break; + default: + GUEST_ASSERT_1(0, stage); + } + + GUEST_SYNC(stage); + } + + GUEST_DONE(); +} + +static int set_fw_reg(struct kvm_vm *vm, uint64_t id, uint64_t val) +{ + struct kvm_one_reg reg = { + .id = id, + .addr = (uint64_t)&val, + }; + + return _vcpu_ioctl(vm, 0, KVM_SET_ONE_REG, ®); +} + +static void get_fw_reg(struct kvm_vm *vm, uint64_t id, uint64_t *addr) +{ + struct kvm_one_reg reg = { + .id = id, + .addr = (uint64_t)addr, + }; + + vcpu_ioctl(vm, 0, KVM_GET_ONE_REG, ®); +} + +struct st_time { + uint32_t rev; + uint32_t attr; + uint64_t st_time; +}; + +#define STEAL_TIME_SIZE ((sizeof(struct st_time) + 63) & ~63) +#define ST_GPA_BASE (1 << 30) + +static void steal_time_init(struct kvm_vm *vm) +{ + uint64_t st_ipa = (ulong)ST_GPA_BASE; + unsigned int gpages; + struct kvm_device_attr dev = { + .group = KVM_ARM_VCPU_PVTIME_CTRL, + .attr = KVM_ARM_VCPU_PVTIME_IPA, + .addr = (uint64_t)&st_ipa, + }; + + gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE); + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); + + vcpu_ioctl(vm, 0, KVM_SET_DEVICE_ATTR, &dev); +} + +static void test_fw_regs_before_vm_start(struct kvm_vm *vm) +{ + uint64_t val; + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(fw_reg_info); i++) { + const struct kvm_fw_reg_info *reg_info = &fw_reg_info[i]; + + /* First 'read' should be an upper limit of the features supported */ + get_fw_reg(vm, reg_info->reg, &val); + TEST_ASSERT(val == FW_REG_ULIMIT_VAL(reg_info->max_feat_bit), + "Expected all the features to be set for reg: 0x%lx; expected: 0x%lx; read: 0x%lx\n", + reg_info->reg, FW_REG_ULIMIT_VAL(reg_info->max_feat_bit), val); + + /* Test a 'write' by disabling all the features of the register map */ + ret = set_fw_reg(vm, reg_info->reg, 0); + TEST_ASSERT(ret == 0, + "Failed to clear all the features of reg: 0x%lx; ret: %d\n", + reg_info->reg, errno); + + get_fw_reg(vm, reg_info->reg, &val); + TEST_ASSERT(val == 0, + "Expected all the features to be cleared for reg: 0x%lx\n", reg_info->reg); + + /* + * Test enabling a feature that's not supported. + * Avoid this check if all the bits are occupied. + */ + if (reg_info->max_feat_bit < 63) { + ret = set_fw_reg(vm, reg_info->reg, BIT(reg_info->max_feat_bit + 1)); + TEST_ASSERT(ret != 0 && errno == EINVAL, + "Unexpected behavior or return value (%d) while setting an unsupported feature for reg: 0x%lx\n", + errno, reg_info->reg); + } + } +} + +static void test_fw_regs_after_vm_start(struct kvm_vm *vm) +{ + uint64_t val; + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(fw_reg_info); i++) { + const struct kvm_fw_reg_info *reg_info = &fw_reg_info[i]; + + /* + * Before starting the VM, the test clears all the bits. + * Check if that's still the case. + */ + get_fw_reg(vm, reg_info->reg, &val); + TEST_ASSERT(val == 0, + "Expected all the features to be cleared for reg: 0x%lx\n", + reg_info->reg); + + /* + * Since the VM has run at least once, KVM shouldn't allow modification of + * the registers and should return EBUSY. Set the registers and check for + * the expected errno. + */ + ret = set_fw_reg(vm, reg_info->reg, FW_REG_ULIMIT_VAL(reg_info->max_feat_bit)); + TEST_ASSERT(ret != 0 && errno == EBUSY, + "Unexpected behavior or return value (%d) while setting a feature while VM is running for reg: 0x%lx\n", + errno, reg_info->reg); + } +} + +static struct kvm_vm *test_vm_create(void) +{ + struct kvm_vm *vm; + + vm = vm_create_default(0, 0, guest_code); + + ucall_init(vm, NULL); + steal_time_init(vm); + + return vm; +} + +static struct kvm_vm *test_guest_stage(struct kvm_vm *vm) +{ + struct kvm_vm *ret_vm = vm; + + pr_debug("Stage: %d\n", stage); + + switch (stage) { + case TEST_STAGE_REG_IFACE: + test_fw_regs_after_vm_start(vm); + break; + case TEST_STAGE_HVC_IFACE_FEAT_DISABLED: + /* Start a new VM so that all the features are now enabled by default */ + kvm_vm_free(vm); + ret_vm = test_vm_create(); + break; + case TEST_STAGE_HVC_IFACE_FEAT_ENABLED: + case TEST_STAGE_HVC_IFACE_FALSE_INFO: + break; + default: + TEST_FAIL("Unknown test stage: %d\n", stage); + } + + stage++; + sync_global_to_guest(vm, stage); + + return ret_vm; +} + +static void test_run(void) +{ + struct kvm_vm *vm; + struct ucall uc; + bool guest_done = false; + + vm = test_vm_create(); + + test_fw_regs_before_vm_start(vm); + + while (!guest_done) { + vcpu_run(vm, 0); + + switch (get_ucall(vm, 0, &uc)) { + case UCALL_SYNC: + vm = test_guest_stage(vm); + break; + case UCALL_DONE: + guest_done = true; + break; + case UCALL_ABORT: + TEST_FAIL("%s at %s:%ld\n\tvalues: 0x%lx, 0x%lx; 0x%lx, stage: %u", + (const char *)uc.args[0], __FILE__, uc.args[1], + uc.args[2], uc.args[3], uc.args[4], stage); + break; + default: + TEST_FAIL("Unexpected guest exit\n"); + } + } + + kvm_vm_free(vm); +} + +int main(void) +{ + setbuf(stdout, NULL); + + test_run(); + return 0; +} From 920f4a55fdaa6f68b31c50cca6e51fecac5857a0 Mon Sep 17 00:00:00 2001 From: Raghavendra Rao Ananta Date: Mon, 2 May 2022 23:38:53 +0000 Subject: [PATCH 194/319] selftests: KVM: aarch64: Add the bitmap firmware registers to get-reg-list Add the psuedo-firmware registers KVM_REG_ARM_STD_BMAP, KVM_REG_ARM_STD_HYP_BMAP, and KVM_REG_ARM_VENDOR_HYP_BMAP to the base_regs[] list. Also, add the COPROC support for KVM_REG_ARM_FW_FEAT_BMAP. Signed-off-by: Raghavendra Rao Ananta Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220502233853.1233742-10-rananta@google.com --- tools/testing/selftests/kvm/aarch64/get-reg-list.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/testing/selftests/kvm/aarch64/get-reg-list.c b/tools/testing/selftests/kvm/aarch64/get-reg-list.c index 0b571f3fe64c..d3a7dbfcbb3d 100644 --- a/tools/testing/selftests/kvm/aarch64/get-reg-list.c +++ b/tools/testing/selftests/kvm/aarch64/get-reg-list.c @@ -294,6 +294,11 @@ static void print_reg(struct vcpu_config *c, __u64 id) "%s: Unexpected bits set in FW reg id: 0x%llx", config_name(c), id); printf("\tKVM_REG_ARM_FW_REG(%lld),\n", id & 0xffff); break; + case KVM_REG_ARM_FW_FEAT_BMAP: + TEST_ASSERT(id == KVM_REG_ARM_FW_FEAT_BMAP_REG(id & 0xffff), + "%s: Unexpected bits set in the bitmap feature FW reg id: 0x%llx", config_name(c), id); + printf("\tKVM_REG_ARM_FW_FEAT_BMAP_REG(%lld),\n", id & 0xffff); + break; case KVM_REG_ARM64_SVE: if (has_cap(c, KVM_CAP_ARM_SVE)) printf("\t%s,\n", sve_id_to_str(c, id)); @@ -692,6 +697,9 @@ static __u64 base_regs[] = { KVM_REG_ARM_FW_REG(1), /* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_1 */ KVM_REG_ARM_FW_REG(2), /* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_2 */ KVM_REG_ARM_FW_REG(3), /* KVM_REG_ARM_SMCCC_ARCH_WORKAROUND_3 */ + KVM_REG_ARM_FW_FEAT_BMAP_REG(0), /* KVM_REG_ARM_STD_BMAP */ + KVM_REG_ARM_FW_FEAT_BMAP_REG(1), /* KVM_REG_ARM_STD_HYP_BMAP */ + KVM_REG_ARM_FW_FEAT_BMAP_REG(2), /* KVM_REG_ARM_VENDOR_HYP_BMAP */ ARM64_SYS_REG(3, 3, 14, 3, 1), /* CNTV_CTL_EL0 */ ARM64_SYS_REG(3, 3, 14, 3, 2), /* CNTV_CVAL_EL0 */ ARM64_SYS_REG(3, 3, 14, 0, 2), From ee87a9bd657e89fe7763cb79cfe1b6bb2d0b0623 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 May 2022 08:01:05 +0100 Subject: [PATCH 195/319] KVM: arm64: Fix new instances of 32bit ESRs Fix the new instances of ESR being described as a u32, now that we consistently are using a u64 for this register. Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index f3235eafdadc..047f13e89632 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2306,7 +2306,7 @@ static int kvm_handle_cp_64(struct kvm_vcpu *vcpu, size_t nr_global) { struct sys_reg_params params; - u32 esr = kvm_vcpu_get_esr(vcpu); + u64 esr = kvm_vcpu_get_esr(vcpu); int Rt = kvm_vcpu_sys_get_rt(vcpu); int Rt2 = (esr >> 10) & 0x1f; @@ -2353,7 +2353,7 @@ static bool emulate_sys_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *params * registers. Abuse that fact so we can rely on the AArch64 handler for accesses * from AArch32. */ -static bool kvm_esr_cp10_id_to_sys64(u32 esr, struct sys_reg_params *params) +static bool kvm_esr_cp10_id_to_sys64(u64 esr, struct sys_reg_params *params) { u8 reg_id = (esr >> 10) & 0xf; bool valid; @@ -2404,7 +2404,7 @@ static bool kvm_esr_cp10_id_to_sys64(u32 esr, struct sys_reg_params *params) int kvm_handle_cp10_id(struct kvm_vcpu *vcpu) { int Rt = kvm_vcpu_sys_get_rt(vcpu); - u32 esr = kvm_vcpu_get_esr(vcpu); + u64 esr = kvm_vcpu_get_esr(vcpu); struct sys_reg_params params; /* UNDEF on any unhandled register access */ From 5bc2cb95ad03d866422d7b3f19ec42a6720f3262 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:35 +0000 Subject: [PATCH 196/319] KVM: arm64: Don't depend on fallthrough to hide SYSTEM_RESET2 Depending on a fallthrough to the default case for hiding SYSTEM_RESET2 requires that any new case statements clean up the failure path for this PSCI call. Unhitch SYSTEM_RESET2 from the default case by setting val to PSCI_RET_NOT_SUPPORTED outside of the switch statement. Apply the cleanup to both the PSCI_1_1_FN_SYSTEM_RESET2 and PSCI_1_0_FN_PSCI_FEATURES handlers. No functional change intended. Signed-off-by: Oliver Upton Reviewed-by: Reiji Watanabe Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-2-oupton@google.com --- arch/arm64/kvm/psci.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 708d80e8e60d..67fbd6ef022c 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -305,9 +305,9 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) { + unsigned long val = PSCI_RET_NOT_SUPPORTED; u32 psci_fn = smccc_get_function(vcpu); u32 arg; - unsigned long val; int ret = 1; switch(psci_fn) { @@ -320,6 +320,8 @@ static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) if (val) break; + val = PSCI_RET_NOT_SUPPORTED; + switch(arg) { case PSCI_0_2_FN_PSCI_VERSION: case PSCI_0_2_FN_CPU_SUSPEND: @@ -338,13 +340,8 @@ static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) break; case PSCI_1_1_FN_SYSTEM_RESET2: case PSCI_1_1_FN64_SYSTEM_RESET2: - if (minor >= 1) { + if (minor >= 1) val = 0; - break; - } - fallthrough; - default: - val = PSCI_RET_NOT_SUPPORTED; break; } break; @@ -365,7 +362,7 @@ static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) val = PSCI_RET_INVALID_PARAMS; break; } - fallthrough; + break; default: return kvm_psci_0_2_call(vcpu); } From 1e5794295c5dbfcc31cf5de840c9e095ae50efb7 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:36 +0000 Subject: [PATCH 197/319] KVM: arm64: Dedupe vCPU power off helpers vcpu_power_off() and kvm_psci_vcpu_off() are equivalent; rename the former and replace all callsites to the latter. No functional change intended. Signed-off-by: Oliver Upton Reviewed-by: Reiji Watanabe Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-3-oupton@google.com --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/kvm/arm.c | 6 +++--- arch/arm64/kvm/psci.c | 11 ++--------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 94a27a7520f4..490cd7f3a905 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -841,4 +841,6 @@ void __init kvm_hyp_reserve(void); static inline void kvm_hyp_reserve(void) { } #endif +void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu); + #endif /* __ARM64_KVM_HOST_H__ */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 523bc934fe2f..28c83c6ddbae 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -432,7 +432,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) vcpu->cpu = -1; } -static void vcpu_power_off(struct kvm_vcpu *vcpu) +void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) { vcpu->arch.power_off = true; kvm_make_request(KVM_REQ_SLEEP, vcpu); @@ -460,7 +460,7 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, vcpu->arch.power_off = false; break; case KVM_MP_STATE_STOPPED: - vcpu_power_off(vcpu); + kvm_arm_vcpu_power_off(vcpu); break; default: ret = -EINVAL; @@ -1124,7 +1124,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, * Handle the "start in power-off" case. */ if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features)) - vcpu_power_off(vcpu); + kvm_arm_vcpu_power_off(vcpu); else vcpu->arch.power_off = false; diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 67fbd6ef022c..9b1f3acae155 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -51,13 +51,6 @@ static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) return PSCI_RET_SUCCESS; } -static void kvm_psci_vcpu_off(struct kvm_vcpu *vcpu) -{ - vcpu->arch.power_off = true; - kvm_make_request(KVM_REQ_SLEEP, vcpu); - kvm_vcpu_kick(vcpu); -} - static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, unsigned long affinity) { @@ -245,7 +238,7 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) val = kvm_psci_vcpu_suspend(vcpu); break; case PSCI_0_2_FN_CPU_OFF: - kvm_psci_vcpu_off(vcpu); + kvm_arm_vcpu_power_off(vcpu); val = PSCI_RET_SUCCESS; break; case PSCI_0_2_FN_CPU_ON: @@ -379,7 +372,7 @@ static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) switch (psci_fn) { case KVM_PSCI_FN_CPU_OFF: - kvm_psci_vcpu_off(vcpu); + kvm_arm_vcpu_power_off(vcpu); val = PSCI_RET_SUCCESS; break; case KVM_PSCI_FN_CPU_ON: From b171f9bbb130cb323f2101edd32da2a25d43ebfa Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:37 +0000 Subject: [PATCH 198/319] KVM: arm64: Track vCPU power state using MP state values A subsequent change to KVM will add support for additional power states. Store the MP state by value rather than keeping track of it as a boolean. No functional change intended. Signed-off-by: Oliver Upton Reviewed-by: Reiji Watanabe Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-4-oupton@google.com --- arch/arm64/include/asm/kvm_host.h | 5 +++-- arch/arm64/kvm/arm.c | 22 ++++++++++++---------- arch/arm64/kvm/psci.c | 12 ++++++------ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 490cd7f3a905..f3f93d48e21a 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -365,8 +365,8 @@ struct kvm_vcpu_arch { u32 mdscr_el1; } guest_debug_preserved; - /* vcpu power-off state */ - bool power_off; + /* vcpu power state */ + struct kvm_mp_state mp_state; /* Don't run the guest (internal implementation need) */ bool pause; @@ -842,5 +842,6 @@ static inline void kvm_hyp_reserve(void) { } #endif void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu); +bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu); #endif /* __ARM64_KVM_HOST_H__ */ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 28c83c6ddbae..29e107457c4d 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -434,18 +434,20 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) void kvm_arm_vcpu_power_off(struct kvm_vcpu *vcpu) { - vcpu->arch.power_off = true; + vcpu->arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; kvm_make_request(KVM_REQ_SLEEP, vcpu); kvm_vcpu_kick(vcpu); } +bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.mp_state.mp_state == KVM_MP_STATE_STOPPED; +} + int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { - if (vcpu->arch.power_off) - mp_state->mp_state = KVM_MP_STATE_STOPPED; - else - mp_state->mp_state = KVM_MP_STATE_RUNNABLE; + *mp_state = vcpu->arch.mp_state; return 0; } @@ -457,7 +459,7 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, switch (mp_state->mp_state) { case KVM_MP_STATE_RUNNABLE: - vcpu->arch.power_off = false; + vcpu->arch.mp_state = *mp_state; break; case KVM_MP_STATE_STOPPED: kvm_arm_vcpu_power_off(vcpu); @@ -480,7 +482,7 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) { bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF); return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) - && !v->arch.power_off && !v->arch.pause); + && !kvm_arm_vcpu_stopped(v) && !v->arch.pause); } bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) @@ -597,10 +599,10 @@ static void vcpu_req_sleep(struct kvm_vcpu *vcpu) struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); rcuwait_wait_event(wait, - (!vcpu->arch.power_off) &&(!vcpu->arch.pause), + (!kvm_arm_vcpu_stopped(vcpu)) && (!vcpu->arch.pause), TASK_INTERRUPTIBLE); - if (vcpu->arch.power_off || vcpu->arch.pause) { + if (kvm_arm_vcpu_stopped(vcpu) || vcpu->arch.pause) { /* Awaken to handle a signal, request we sleep again later. */ kvm_make_request(KVM_REQ_SLEEP, vcpu); } @@ -1126,7 +1128,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, if (test_bit(KVM_ARM_VCPU_POWER_OFF, vcpu->arch.features)) kvm_arm_vcpu_power_off(vcpu); else - vcpu->arch.power_off = false; + vcpu->arch.mp_state.mp_state = KVM_MP_STATE_RUNNABLE; return 0; } diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 9b1f3acae155..2e6f060214a7 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -76,7 +76,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) */ if (!vcpu) return PSCI_RET_INVALID_PARAMS; - if (!vcpu->arch.power_off) { + if (!kvm_arm_vcpu_stopped(vcpu)) { if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) return PSCI_RET_ALREADY_ON; else @@ -100,12 +100,12 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) kvm_make_request(KVM_REQ_VCPU_RESET, vcpu); /* - * Make sure the reset request is observed if the change to - * power_off is observed. + * Make sure the reset request is observed if the RUNNABLE mp_state is + * observed. */ smp_wmb(); - vcpu->arch.power_off = false; + vcpu->arch.mp_state.mp_state = KVM_MP_STATE_RUNNABLE; kvm_vcpu_wake_up(vcpu); return PSCI_RET_SUCCESS; @@ -143,7 +143,7 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) mpidr = kvm_vcpu_get_mpidr_aff(tmp); if ((mpidr & target_affinity_mask) == target_affinity) { matching_cpus++; - if (!tmp->arch.power_off) + if (!kvm_arm_vcpu_stopped(tmp)) return PSCI_0_2_AFFINITY_LEVEL_ON; } } @@ -169,7 +169,7 @@ static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags) * re-initialized. */ kvm_for_each_vcpu(i, tmp, vcpu->kvm) - tmp->arch.power_off = true; + tmp->arch.mp_state.mp_state = KVM_MP_STATE_STOPPED; kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP); memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event)); From 1c6219e3faf12e58d520b3b2cdfa8cd5e1efc9a5 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:38 +0000 Subject: [PATCH 199/319] KVM: arm64: Rename the KVM_REQ_SLEEP handler The naming of the kvm_req_sleep function is confusing: the function itself sleeps the vCPU, it does not request such an event. Rename the function to make its purpose more clear. No functional change intended. Signed-off-by: Oliver Upton Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-5-oupton@google.com --- arch/arm64/kvm/arm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 29e107457c4d..77b8b870c0fc 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -594,7 +594,7 @@ void kvm_arm_resume_guest(struct kvm *kvm) } } -static void vcpu_req_sleep(struct kvm_vcpu *vcpu) +static void kvm_vcpu_sleep(struct kvm_vcpu *vcpu) { struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); @@ -652,7 +652,7 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) { if (kvm_request_pending(vcpu)) { if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) - vcpu_req_sleep(vcpu); + kvm_vcpu_sleep(vcpu); if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) kvm_reset_vcpu(vcpu); From 3fdd04592d38bb31a0bea567d9a66672b484bed3 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:39 +0000 Subject: [PATCH 200/319] KVM: arm64: Return a value from check_vcpu_requests() A subsequent change to KVM will introduce a vCPU request that could result in an exit to userspace. Change check_vcpu_requests() to return a value and document the function. Unconditionally return 1 for now. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-6-oupton@google.com --- arch/arm64/kvm/arm.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 77b8b870c0fc..efe54aba5cce 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -648,7 +648,16 @@ void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) preempt_enable(); } -static void check_vcpu_requests(struct kvm_vcpu *vcpu) +/** + * check_vcpu_requests - check and handle pending vCPU requests + * @vcpu: the VCPU pointer + * + * Return: 1 if we should enter the guest + * 0 if we should exit to userspace + * < 0 if we should exit to userspace, where the return value indicates + * an error + */ +static int check_vcpu_requests(struct kvm_vcpu *vcpu) { if (kvm_request_pending(vcpu)) { if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) @@ -678,6 +687,8 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) kvm_pmu_handle_pmcr(vcpu, __vcpu_sys_reg(vcpu, PMCR_EL0)); } + + return 1; } static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu) @@ -793,7 +804,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) if (!ret) ret = 1; - check_vcpu_requests(vcpu); + if (ret > 0) + ret = check_vcpu_requests(vcpu); /* * Preparing the interrupts to be injected also From 7b33a09d036ffd9a04506122840629c7e870cf08 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:40 +0000 Subject: [PATCH 201/319] KVM: arm64: Add support for userspace to suspend a vCPU Introduce a new MP state, KVM_MP_STATE_SUSPENDED, which indicates a vCPU is in a suspended state. In the suspended state the vCPU will block until a wakeup event (pending interrupt) is recognized. Add a new system event type, KVM_SYSTEM_EVENT_WAKEUP, to indicate to userspace that KVM has recognized one such wakeup event. It is the responsibility of userspace to then make the vCPU runnable, or leave it suspended until the next wakeup event. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-7-oupton@google.com --- Documentation/virt/kvm/api.rst | 37 ++++++++++++++++++++-- arch/arm64/include/asm/kvm_host.h | 1 + arch/arm64/kvm/arm.c | 51 +++++++++++++++++++++++++++++++ include/uapi/linux/kvm.h | 2 ++ 4 files changed, 89 insertions(+), 2 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 4a900cdbc62e..46ca84600dca 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -1476,14 +1476,43 @@ Possible values are: [s390] KVM_MP_STATE_LOAD the vcpu is in a special load/startup state [s390] + KVM_MP_STATE_SUSPENDED the vcpu is in a suspend state and is waiting + for a wakeup event [arm64] ========================== =============================================== On x86, this ioctl is only useful after KVM_CREATE_IRQCHIP. Without an in-kernel irqchip, the multiprocessing state must be maintained by userspace on these architectures. -For arm64/riscv: -^^^^^^^^^^^^^^^^ +For arm64: +^^^^^^^^^^ + +If a vCPU is in the KVM_MP_STATE_SUSPENDED state, KVM will emulate the +architectural execution of a WFI instruction. + +If a wakeup event is recognized, KVM will exit to userspace with a +KVM_SYSTEM_EVENT exit, where the event type is KVM_SYSTEM_EVENT_WAKEUP. If +userspace wants to honor the wakeup, it must set the vCPU's MP state to +KVM_MP_STATE_RUNNABLE. If it does not, KVM will continue to await a wakeup +event in subsequent calls to KVM_RUN. + +.. warning:: + + If userspace intends to keep the vCPU in a SUSPENDED state, it is + strongly recommended that userspace take action to suppress the + wakeup event (such as masking an interrupt). Otherwise, subsequent + calls to KVM_RUN will immediately exit with a KVM_SYSTEM_EVENT_WAKEUP + event and inadvertently waste CPU cycles. + + Additionally, if userspace takes action to suppress a wakeup event, + it is strongly recommended that it also restores the vCPU to its + original state when the vCPU is made RUNNABLE again. For example, + if userspace masked a pending interrupt to suppress the wakeup, + the interrupt should be unmasked before returning control to the + guest. + +For riscv: +^^^^^^^^^^ The only states that are valid are KVM_MP_STATE_STOPPED and KVM_MP_STATE_RUNNABLE which reflect if the vcpu is paused or not. @@ -5985,6 +6014,7 @@ should put the acknowledged interrupt vector into the 'epr' field. #define KVM_SYSTEM_EVENT_SHUTDOWN 1 #define KVM_SYSTEM_EVENT_RESET 2 #define KVM_SYSTEM_EVENT_CRASH 3 + #define KVM_SYSTEM_EVENT_WAKEUP 4 __u32 type; __u32 ndata; __u64 data[16]; @@ -6009,6 +6039,9 @@ Valid values for 'type' are: has requested a crash condition maintenance. Userspace can choose to ignore the request, or to gather VM memory core dump and/or reset/shutdown of the VM. + - KVM_SYSTEM_EVENT_WAKEUP -- the exiting vCPU is in a suspended state and + KVM has recognized a wakeup event. Userspace may honor this event by + marking the exiting vCPU as runnable, or deny it and call KVM_RUN again. If KVM_CAP_SYSTEM_EVENT_DATA is present, the 'data' field can contain architecture specific information for the system-level event. Only diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index f3f93d48e21a..46027b9b80ca 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -46,6 +46,7 @@ #define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(3) #define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4) #define KVM_REQ_RELOAD_PMU KVM_ARCH_REQ(5) +#define KVM_REQ_SUSPEND KVM_ARCH_REQ(6) #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index efe54aba5cce..abd32a84ed7a 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -444,6 +444,18 @@ bool kvm_arm_vcpu_stopped(struct kvm_vcpu *vcpu) return vcpu->arch.mp_state.mp_state == KVM_MP_STATE_STOPPED; } +static void kvm_arm_vcpu_suspend(struct kvm_vcpu *vcpu) +{ + vcpu->arch.mp_state.mp_state = KVM_MP_STATE_SUSPENDED; + kvm_make_request(KVM_REQ_SUSPEND, vcpu); + kvm_vcpu_kick(vcpu); +} + +static bool kvm_arm_vcpu_suspended(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.mp_state.mp_state == KVM_MP_STATE_SUSPENDED; +} + int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -464,6 +476,9 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, case KVM_MP_STATE_STOPPED: kvm_arm_vcpu_power_off(vcpu); break; + case KVM_MP_STATE_SUSPENDED: + kvm_arm_vcpu_suspend(vcpu); + break; default: ret = -EINVAL; } @@ -648,6 +663,39 @@ void kvm_vcpu_wfi(struct kvm_vcpu *vcpu) preempt_enable(); } +static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu) +{ + if (!kvm_arm_vcpu_suspended(vcpu)) + return 1; + + kvm_vcpu_wfi(vcpu); + + /* + * The suspend state is sticky; we do not leave it until userspace + * explicitly marks the vCPU as runnable. Request that we suspend again + * later. + */ + kvm_make_request(KVM_REQ_SUSPEND, vcpu); + + /* + * Check to make sure the vCPU is actually runnable. If so, exit to + * userspace informing it of the wakeup condition. + */ + if (kvm_arch_vcpu_runnable(vcpu)) { + memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event)); + vcpu->run->system_event.type = KVM_SYSTEM_EVENT_WAKEUP; + vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; + return 0; + } + + /* + * Otherwise, we were unblocked to process a different event, such as a + * pending signal. Return 1 and allow kvm_arch_vcpu_ioctl_run() to + * process the event. + */ + return 1; +} + /** * check_vcpu_requests - check and handle pending vCPU requests * @vcpu: the VCPU pointer @@ -686,6 +734,9 @@ static int check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu)) kvm_pmu_handle_pmcr(vcpu, __vcpu_sys_reg(vcpu, PMCR_EL0)); + + if (kvm_check_request(KVM_REQ_SUSPEND, vcpu)) + return kvm_vcpu_suspend(vcpu); } return 1; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 6a184d260c7f..7f72fb7b05f2 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -444,6 +444,7 @@ struct kvm_run { #define KVM_SYSTEM_EVENT_SHUTDOWN 1 #define KVM_SYSTEM_EVENT_RESET 2 #define KVM_SYSTEM_EVENT_CRASH 3 +#define KVM_SYSTEM_EVENT_WAKEUP 4 __u32 type; __u32 ndata; union { @@ -646,6 +647,7 @@ struct kvm_vapic_addr { #define KVM_MP_STATE_OPERATING 7 #define KVM_MP_STATE_LOAD 8 #define KVM_MP_STATE_AP_RESET_HOLD 9 +#define KVM_MP_STATE_SUSPENDED 10 struct kvm_mp_state { __u32 mp_state; From bfbab44568779e1682bc6f63688bb9c965f0e74a Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:41 +0000 Subject: [PATCH 202/319] KVM: arm64: Implement PSCI SYSTEM_SUSPEND ARM DEN0022D.b 5.19 "SYSTEM_SUSPEND" describes a PSCI call that allows software to request that a system be placed in the deepest possible low-power state. Effectively, software can use this to suspend itself to RAM. Unfortunately, there really is no good way to implement a system-wide PSCI call in KVM. Any precondition checks done in the kernel will need to be repeated by userspace since there is no good way to protect a critical section that spans an exit to userspace. SYSTEM_RESET and SYSTEM_OFF are equally plagued by this issue, although no users have seemingly cared for the relatively long time these calls have been supported. The solution is to just make the whole implementation userspace's problem. Introduce a new system event, KVM_SYSTEM_EVENT_SUSPEND, that indicates to userspace a calling vCPU has invoked PSCI SYSTEM_SUSPEND. Additionally, add a CAP to get buy-in from userspace for this new exit type. Only advertise the SYSTEM_SUSPEND PSCI call if userspace has opted in. If a vCPU calls SYSTEM_SUSPEND, punt straight to userspace. Provide explicit documentation of userspace's responsibilites for the exit and point to the PSCI specification to describe the actual PSCI call. Reviewed-by: Reiji Watanabe Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-8-oupton@google.com --- Documentation/virt/kvm/api.rst | 39 +++++++++++++++++++++++++++++++ arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/kvm/arm.c | 5 ++++ arch/arm64/kvm/psci.c | 29 +++++++++++++++++++++++ include/uapi/linux/kvm.h | 2 ++ 5 files changed, 77 insertions(+) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 46ca84600dca..d8d7859fc556 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6015,6 +6015,7 @@ should put the acknowledged interrupt vector into the 'epr' field. #define KVM_SYSTEM_EVENT_RESET 2 #define KVM_SYSTEM_EVENT_CRASH 3 #define KVM_SYSTEM_EVENT_WAKEUP 4 + #define KVM_SYSTEM_EVENT_SUSPEND 5 __u32 type; __u32 ndata; __u64 data[16]; @@ -6042,6 +6043,34 @@ Valid values for 'type' are: - KVM_SYSTEM_EVENT_WAKEUP -- the exiting vCPU is in a suspended state and KVM has recognized a wakeup event. Userspace may honor this event by marking the exiting vCPU as runnable, or deny it and call KVM_RUN again. + - KVM_SYSTEM_EVENT_SUSPEND -- the guest has requested a suspension of + the VM. + +For arm/arm64: +^^^^^^^^^^^^^^ + + KVM_SYSTEM_EVENT_SUSPEND exits are enabled with the + KVM_CAP_ARM_SYSTEM_SUSPEND VM capability. If a guest invokes the PSCI + SYSTEM_SUSPEND function, KVM will exit to userspace with this event + type. + + It is the sole responsibility of userspace to implement the PSCI + SYSTEM_SUSPEND call according to ARM DEN0022D.b 5.19 "SYSTEM_SUSPEND". + KVM does not change the vCPU's state before exiting to userspace, so + the call parameters are left in-place in the vCPU registers. + + Userspace is _required_ to take action for such an exit. It must + either: + + - Honor the guest request to suspend the VM. Userspace can request + in-kernel emulation of suspension by setting the calling vCPU's + state to KVM_MP_STATE_SUSPENDED. Userspace must configure the vCPU's + state according to the parameters passed to the PSCI function when + the calling vCPU is resumed. See ARM DEN0022D.b 5.19.1 "Intended use" + for details on the function parameters. + + - Deny the guest request to suspend the VM. See ARM DEN0022D.b 5.19.2 + "Caller responsibilities" for possible return values. If KVM_CAP_SYSTEM_EVENT_DATA is present, the 'data' field can contain architecture specific information for the system-level event. Only @@ -7767,6 +7796,16 @@ At this time, KVM_PMU_CAP_DISABLE is the only capability. Setting this capability will disable PMU virtualization for that VM. Usermode should adjust CPUID leaf 0xA to reflect that the PMU is disabled. +8.36 KVM_CAP_ARM_SYSTEM_SUSPEND +------------------------------- + +:Capability: KVM_CAP_ARM_SYSTEM_SUSPEND +:Architectures: arm64 +:Type: vm + +When enabled, KVM will exit to userspace with KVM_EXIT_SYSTEM_EVENT of +type KVM_SYSTEM_EVENT_SUSPEND to process the guest suspend request. + 9. Known KVM API problems ========================= diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 46027b9b80ca..d9df81949f76 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -137,6 +137,8 @@ struct kvm_arch { */ #define KVM_ARCH_FLAG_REG_WIDTH_CONFIGURED 3 #define KVM_ARCH_FLAG_EL1_32BIT 4 + /* PSCI SYSTEM_SUSPEND enabled for the guest */ +#define KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED 5 unsigned long flags; diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index abd32a84ed7a..f8a89ae52710 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -97,6 +97,10 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, } mutex_unlock(&kvm->lock); break; + case KVM_CAP_ARM_SYSTEM_SUSPEND: + r = 0; + set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags); + break; default: r = -EINVAL; break; @@ -210,6 +214,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_SET_GUEST_DEBUG: case KVM_CAP_VCPU_ATTRIBUTES: case KVM_CAP_PTP_KVM: + case KVM_CAP_ARM_SYSTEM_SUSPEND: r = 1; break; case KVM_CAP_SET_GUEST_DEBUG2: diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 2e6f060214a7..5de30e72ad40 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -195,6 +195,15 @@ static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu) KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2); } +static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu) +{ + struct kvm_run *run = vcpu->run; + + memset(&run->system_event, 0, sizeof(vcpu->run->system_event)); + run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND; + run->exit_reason = KVM_EXIT_SYSTEM_EVENT; +} + static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) { int i; @@ -300,6 +309,7 @@ static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) { unsigned long val = PSCI_RET_NOT_SUPPORTED; u32 psci_fn = smccc_get_function(vcpu); + struct kvm *kvm = vcpu->kvm; u32 arg; int ret = 1; @@ -331,6 +341,11 @@ static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) case ARM_SMCCC_VERSION_FUNC_ID: val = 0; break; + case PSCI_1_0_FN_SYSTEM_SUSPEND: + case PSCI_1_0_FN64_SYSTEM_SUSPEND: + if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) + val = 0; + break; case PSCI_1_1_FN_SYSTEM_RESET2: case PSCI_1_1_FN64_SYSTEM_RESET2: if (minor >= 1) @@ -338,6 +353,20 @@ static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) break; } break; + case PSCI_1_0_FN_SYSTEM_SUSPEND: + kvm_psci_narrow_to_32bit(vcpu); + fallthrough; + case PSCI_1_0_FN64_SYSTEM_SUSPEND: + /* + * Return directly to userspace without changing the vCPU's + * registers. Userspace depends on reading the SMCCC parameters + * to implement SYSTEM_SUSPEND. + */ + if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) { + kvm_psci_system_suspend(vcpu); + return 0; + } + break; case PSCI_1_1_FN_SYSTEM_RESET2: kvm_psci_narrow_to_32bit(vcpu); fallthrough; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 7f72fb7b05f2..32c56384fd08 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -445,6 +445,7 @@ struct kvm_run { #define KVM_SYSTEM_EVENT_RESET 2 #define KVM_SYSTEM_EVENT_CRASH 3 #define KVM_SYSTEM_EVENT_WAKEUP 4 +#define KVM_SYSTEM_EVENT_SUSPEND 5 __u32 type; __u32 ndata; union { @@ -1154,6 +1155,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_DISABLE_QUIRKS2 213 /* #define KVM_CAP_VM_TSC_CONTROL 214 */ #define KVM_CAP_SYSTEM_EVENT_DATA 215 +#define KVM_CAP_ARM_SYSTEM_SUSPEND 216 #ifdef KVM_CAP_IRQ_ROUTING From 6689fb8f21ecf5fd99278b622c9579ffbc2742c2 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:42 +0000 Subject: [PATCH 203/319] selftests: KVM: Rename psci_cpu_on_test to psci_test There are other interactions with PSCI worth testing; rename the PSCI test to make it more generic. No functional change intended. Signed-off-by: Oliver Upton Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-9-oupton@google.com --- tools/testing/selftests/kvm/.gitignore | 2 +- tools/testing/selftests/kvm/Makefile | 2 +- .../selftests/kvm/aarch64/{psci_cpu_on_test.c => psci_test.c} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename tools/testing/selftests/kvm/aarch64/{psci_cpu_on_test.c => psci_test.c} (100%) diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 0b0e4402bba6..1bb575dfc42e 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -2,7 +2,7 @@ /aarch64/arch_timer /aarch64/debug-exceptions /aarch64/get-reg-list -/aarch64/psci_cpu_on_test +/aarch64/psci_test /aarch64/vcpu_width_config /aarch64/vgic_init /aarch64/vgic_irq diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 681b173aa87c..c2cf4d318296 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -105,7 +105,7 @@ TEST_GEN_PROGS_x86_64 += system_counter_offset_test TEST_GEN_PROGS_aarch64 += aarch64/arch_timer TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list -TEST_GEN_PROGS_aarch64 += aarch64/psci_cpu_on_test +TEST_GEN_PROGS_aarch64 += aarch64/psci_test TEST_GEN_PROGS_aarch64 += aarch64/vcpu_width_config TEST_GEN_PROGS_aarch64 += aarch64/vgic_init TEST_GEN_PROGS_aarch64 += aarch64/vgic_irq diff --git a/tools/testing/selftests/kvm/aarch64/psci_cpu_on_test.c b/tools/testing/selftests/kvm/aarch64/psci_test.c similarity index 100% rename from tools/testing/selftests/kvm/aarch64/psci_cpu_on_test.c rename to tools/testing/selftests/kvm/aarch64/psci_test.c From 694e3dcc47471b8b409a0ef647319b746eabcb3a Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:43 +0000 Subject: [PATCH 204/319] selftests: KVM: Create helper for making SMCCC calls The PSCI and PV stolen time tests both need to make SMCCC calls within the guest. Create a helper for making SMCCC calls and rework the existing tests to use the library function. Signed-off-by: Oliver Upton Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-10-oupton@google.com --- .../testing/selftests/kvm/aarch64/psci_test.c | 25 ++++++------------- .../selftests/kvm/include/aarch64/processor.h | 22 ++++++++++++++++ .../selftests/kvm/lib/aarch64/processor.c | 25 +++++++++++++++++++ tools/testing/selftests/kvm/steal_time.c | 13 +++------- 4 files changed, 58 insertions(+), 27 deletions(-) diff --git a/tools/testing/selftests/kvm/aarch64/psci_test.c b/tools/testing/selftests/kvm/aarch64/psci_test.c index 4c5f6814030f..8c998f0b802c 100644 --- a/tools/testing/selftests/kvm/aarch64/psci_test.c +++ b/tools/testing/selftests/kvm/aarch64/psci_test.c @@ -26,32 +26,23 @@ static uint64_t psci_cpu_on(uint64_t target_cpu, uint64_t entry_addr, uint64_t context_id) { - register uint64_t x0 asm("x0") = PSCI_0_2_FN64_CPU_ON; - register uint64_t x1 asm("x1") = target_cpu; - register uint64_t x2 asm("x2") = entry_addr; - register uint64_t x3 asm("x3") = context_id; + struct arm_smccc_res res; - asm("hvc #0" - : "=r"(x0) - : "r"(x0), "r"(x1), "r"(x2), "r"(x3) - : "memory"); + smccc_hvc(PSCI_0_2_FN64_CPU_ON, target_cpu, entry_addr, context_id, + 0, 0, 0, 0, &res); - return x0; + return res.a0; } static uint64_t psci_affinity_info(uint64_t target_affinity, uint64_t lowest_affinity_level) { - register uint64_t x0 asm("x0") = PSCI_0_2_FN64_AFFINITY_INFO; - register uint64_t x1 asm("x1") = target_affinity; - register uint64_t x2 asm("x2") = lowest_affinity_level; + struct arm_smccc_res res; - asm("hvc #0" - : "=r"(x0) - : "r"(x0), "r"(x1), "r"(x2) - : "memory"); + smccc_hvc(PSCI_0_2_FN64_AFFINITY_INFO, target_affinity, lowest_affinity_level, + 0, 0, 0, 0, 0, &res); - return x0; + return res.a0; } static void guest_main(uint64_t target_cpu) diff --git a/tools/testing/selftests/kvm/include/aarch64/processor.h b/tools/testing/selftests/kvm/include/aarch64/processor.h index 8f9f46979a00..59ece9d4e0d1 100644 --- a/tools/testing/selftests/kvm/include/aarch64/processor.h +++ b/tools/testing/selftests/kvm/include/aarch64/processor.h @@ -185,4 +185,26 @@ static inline void local_irq_disable(void) asm volatile("msr daifset, #3" : : : "memory"); } +/** + * struct arm_smccc_res - Result from SMC/HVC call + * @a0-a3 result values from registers 0 to 3 + */ +struct arm_smccc_res { + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; +}; + +/** + * smccc_hvc - Invoke a SMCCC function using the hvc conduit + * @function_id: the SMCCC function to be called + * @arg0-arg6: SMCCC function arguments, corresponding to registers x1-x7 + * @res: pointer to write the return values from registers x0-x3 + * + */ +void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1, + uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, + uint64_t arg6, struct arm_smccc_res *res); + #endif /* SELFTEST_KVM_PROCESSOR_H */ diff --git a/tools/testing/selftests/kvm/lib/aarch64/processor.c b/tools/testing/selftests/kvm/lib/aarch64/processor.c index 9343d82519b4..6a041289fa80 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/processor.c +++ b/tools/testing/selftests/kvm/lib/aarch64/processor.c @@ -500,3 +500,28 @@ void __attribute__((constructor)) init_guest_modes(void) { guest_modes_append_default(); } + +void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1, + uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, + uint64_t arg6, struct arm_smccc_res *res) +{ + asm volatile("mov w0, %w[function_id]\n" + "mov x1, %[arg0]\n" + "mov x2, %[arg1]\n" + "mov x3, %[arg2]\n" + "mov x4, %[arg3]\n" + "mov x5, %[arg4]\n" + "mov x6, %[arg5]\n" + "mov x7, %[arg6]\n" + "hvc #0\n" + "mov %[res0], x0\n" + "mov %[res1], x1\n" + "mov %[res2], x2\n" + "mov %[res3], x3\n" + : [res0] "=r"(res->a0), [res1] "=r"(res->a1), + [res2] "=r"(res->a2), [res3] "=r"(res->a3) + : [function_id] "r"(function_id), [arg0] "r"(arg0), + [arg1] "r"(arg1), [arg2] "r"(arg2), [arg3] "r"(arg3), + [arg4] "r"(arg4), [arg5] "r"(arg5), [arg6] "r"(arg6) + : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"); +} diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index 62f2eb9ee3d5..8c4e811bd586 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -118,17 +118,10 @@ struct st_time { static int64_t smccc(uint32_t func, uint64_t arg) { - unsigned long ret; + struct arm_smccc_res res; - asm volatile( - "mov w0, %w1\n" - "mov x1, %2\n" - "hvc #0\n" - "mov %0, x0\n" - : "=r" (ret) : "r" (func), "r" (arg) : - "x0", "x1", "x2", "x3"); - - return ret; + smccc_hvc(func, arg, 0, 0, 0, 0, 0, 0, &res); + return res.a0; } static void check_status(struct st_time *st) From d135399a97cc3e27716a8e468a5fd1a209346831 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:44 +0000 Subject: [PATCH 205/319] selftests: KVM: Use KVM_SET_MP_STATE to power off vCPU in psci_test Setting a vCPU's MP state to KVM_MP_STATE_STOPPED has the effect of powering off the vCPU. Rather than using the vCPU init feature flag, use the KVM_SET_MP_STATE ioctl to power off the target vCPU. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-11-oupton@google.com --- tools/testing/selftests/kvm/aarch64/psci_test.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/aarch64/psci_test.c b/tools/testing/selftests/kvm/aarch64/psci_test.c index 8c998f0b802c..fe1d5d343a2f 100644 --- a/tools/testing/selftests/kvm/aarch64/psci_test.c +++ b/tools/testing/selftests/kvm/aarch64/psci_test.c @@ -60,6 +60,15 @@ static void guest_main(uint64_t target_cpu) GUEST_DONE(); } +static void vcpu_power_off(struct kvm_vm *vm, uint32_t vcpuid) +{ + struct kvm_mp_state mp_state = { + .mp_state = KVM_MP_STATE_STOPPED, + }; + + vcpu_set_mp_state(vm, vcpuid, &mp_state); +} + int main(void) { uint64_t target_mpidr, obs_pc, obs_x0; @@ -75,12 +84,12 @@ int main(void) init.features[0] |= (1 << KVM_ARM_VCPU_PSCI_0_2); aarch64_vcpu_add_default(vm, VCPU_ID_SOURCE, &init, guest_main); + aarch64_vcpu_add_default(vm, VCPU_ID_TARGET, &init, guest_main); /* * make sure the target is already off when executing the test. */ - init.features[0] |= (1 << KVM_ARM_VCPU_POWER_OFF); - aarch64_vcpu_add_default(vm, VCPU_ID_TARGET, &init, guest_main); + vcpu_power_off(vm, VCPU_ID_TARGET); get_reg(vm, VCPU_ID_TARGET, KVM_ARM64_SYS_REG(SYS_MPIDR_EL1), &target_mpidr); vcpu_args_set(vm, VCPU_ID_SOURCE, 1, target_mpidr & MPIDR_HWID_BITMASK); From 67a36a821312e9c0d2a2f7e6c2225204500cc01c Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:45 +0000 Subject: [PATCH 206/319] selftests: KVM: Refactor psci_test to make it amenable to new tests Split up the current test into several helpers that will be useful to subsequent test cases added to the PSCI test suite. Signed-off-by: Oliver Upton Reviewed-by: Andrew Jones Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-12-oupton@google.com --- .../testing/selftests/kvm/aarch64/psci_test.c | 109 +++++++++++------- 1 file changed, 66 insertions(+), 43 deletions(-) diff --git a/tools/testing/selftests/kvm/aarch64/psci_test.c b/tools/testing/selftests/kvm/aarch64/psci_test.c index fe1d5d343a2f..535130d5e97f 100644 --- a/tools/testing/selftests/kvm/aarch64/psci_test.c +++ b/tools/testing/selftests/kvm/aarch64/psci_test.c @@ -45,11 +45,64 @@ static uint64_t psci_affinity_info(uint64_t target_affinity, return res.a0; } -static void guest_main(uint64_t target_cpu) +static void vcpu_power_off(struct kvm_vm *vm, uint32_t vcpuid) +{ + struct kvm_mp_state mp_state = { + .mp_state = KVM_MP_STATE_STOPPED, + }; + + vcpu_set_mp_state(vm, vcpuid, &mp_state); +} + +static struct kvm_vm *setup_vm(void *guest_code) +{ + struct kvm_vcpu_init init; + struct kvm_vm *vm; + + vm = vm_create(VM_MODE_DEFAULT, DEFAULT_GUEST_PHY_PAGES, O_RDWR); + kvm_vm_elf_load(vm, program_invocation_name); + ucall_init(vm, NULL); + + vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init); + init.features[0] |= (1 << KVM_ARM_VCPU_PSCI_0_2); + + aarch64_vcpu_add_default(vm, VCPU_ID_SOURCE, &init, guest_code); + aarch64_vcpu_add_default(vm, VCPU_ID_TARGET, &init, guest_code); + + return vm; +} + +static void enter_guest(struct kvm_vm *vm, uint32_t vcpuid) +{ + struct ucall uc; + + vcpu_run(vm, vcpuid); + if (get_ucall(vm, vcpuid, &uc) == UCALL_ABORT) + TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], __FILE__, + uc.args[1]); +} + +static void assert_vcpu_reset(struct kvm_vm *vm, uint32_t vcpuid) +{ + uint64_t obs_pc, obs_x0; + + get_reg(vm, vcpuid, ARM64_CORE_REG(regs.pc), &obs_pc); + get_reg(vm, vcpuid, ARM64_CORE_REG(regs.regs[0]), &obs_x0); + + TEST_ASSERT(obs_pc == CPU_ON_ENTRY_ADDR, + "unexpected target cpu pc: %lx (expected: %lx)", + obs_pc, CPU_ON_ENTRY_ADDR); + TEST_ASSERT(obs_x0 == CPU_ON_CONTEXT_ID, + "unexpected target context id: %lx (expected: %lx)", + obs_x0, CPU_ON_CONTEXT_ID); +} + +static void guest_test_cpu_on(uint64_t target_cpu) { - GUEST_ASSERT(!psci_cpu_on(target_cpu, CPU_ON_ENTRY_ADDR, CPU_ON_CONTEXT_ID)); uint64_t target_state; + GUEST_ASSERT(!psci_cpu_on(target_cpu, CPU_ON_ENTRY_ADDR, CPU_ON_CONTEXT_ID)); + do { target_state = psci_affinity_info(target_cpu, 0); @@ -60,31 +113,13 @@ static void guest_main(uint64_t target_cpu) GUEST_DONE(); } -static void vcpu_power_off(struct kvm_vm *vm, uint32_t vcpuid) +static void host_test_cpu_on(void) { - struct kvm_mp_state mp_state = { - .mp_state = KVM_MP_STATE_STOPPED, - }; - - vcpu_set_mp_state(vm, vcpuid, &mp_state); -} - -int main(void) -{ - uint64_t target_mpidr, obs_pc, obs_x0; - struct kvm_vcpu_init init; + uint64_t target_mpidr; struct kvm_vm *vm; struct ucall uc; - vm = vm_create(VM_MODE_DEFAULT, DEFAULT_GUEST_PHY_PAGES, O_RDWR); - kvm_vm_elf_load(vm, program_invocation_name); - ucall_init(vm, NULL); - - vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init); - init.features[0] |= (1 << KVM_ARM_VCPU_PSCI_0_2); - - aarch64_vcpu_add_default(vm, VCPU_ID_SOURCE, &init, guest_main); - aarch64_vcpu_add_default(vm, VCPU_ID_TARGET, &init, guest_main); + vm = setup_vm(guest_test_cpu_on); /* * make sure the target is already off when executing the test. @@ -93,29 +128,17 @@ int main(void) get_reg(vm, VCPU_ID_TARGET, KVM_ARM64_SYS_REG(SYS_MPIDR_EL1), &target_mpidr); vcpu_args_set(vm, VCPU_ID_SOURCE, 1, target_mpidr & MPIDR_HWID_BITMASK); - vcpu_run(vm, VCPU_ID_SOURCE); + enter_guest(vm, VCPU_ID_SOURCE); - switch (get_ucall(vm, VCPU_ID_SOURCE, &uc)) { - case UCALL_DONE: - break; - case UCALL_ABORT: - TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], __FILE__, - uc.args[1]); - break; - default: + if (get_ucall(vm, VCPU_ID_SOURCE, &uc) != UCALL_DONE) TEST_FAIL("Unhandled ucall: %lu", uc.cmd); - } - - get_reg(vm, VCPU_ID_TARGET, ARM64_CORE_REG(regs.pc), &obs_pc); - get_reg(vm, VCPU_ID_TARGET, ARM64_CORE_REG(regs.regs[0]), &obs_x0); - - TEST_ASSERT(obs_pc == CPU_ON_ENTRY_ADDR, - "unexpected target cpu pc: %lx (expected: %lx)", - obs_pc, CPU_ON_ENTRY_ADDR); - TEST_ASSERT(obs_x0 == CPU_ON_CONTEXT_ID, - "unexpected target context id: %lx (expected: %lx)", - obs_x0, CPU_ON_CONTEXT_ID); + assert_vcpu_reset(vm, VCPU_ID_TARGET); kvm_vm_free(vm); +} + +int main(void) +{ + host_test_cpu_on(); return 0; } From b26dafc8a9e74254a390e8f21ff028a2573ee4fc Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 4 May 2022 03:24:46 +0000 Subject: [PATCH 207/319] selftests: KVM: Test SYSTEM_SUSPEND PSCI call Assert that the vCPU exits to userspace with KVM_SYSTEM_EVENT_SUSPEND if the guest calls PSCI SYSTEM_SUSPEND. Additionally, guarantee that the SMC32 and SMC64 flavors of this call are discoverable with the PSCI_FEATURES call. Signed-off-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220504032446.4133305-13-oupton@google.com --- .../testing/selftests/kvm/aarch64/psci_test.c | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tools/testing/selftests/kvm/aarch64/psci_test.c b/tools/testing/selftests/kvm/aarch64/psci_test.c index 535130d5e97f..88541de21c41 100644 --- a/tools/testing/selftests/kvm/aarch64/psci_test.c +++ b/tools/testing/selftests/kvm/aarch64/psci_test.c @@ -45,6 +45,25 @@ static uint64_t psci_affinity_info(uint64_t target_affinity, return res.a0; } +static uint64_t psci_system_suspend(uint64_t entry_addr, uint64_t context_id) +{ + struct arm_smccc_res res; + + smccc_hvc(PSCI_1_0_FN64_SYSTEM_SUSPEND, entry_addr, context_id, + 0, 0, 0, 0, 0, &res); + + return res.a0; +} + +static uint64_t psci_features(uint32_t func_id) +{ + struct arm_smccc_res res; + + smccc_hvc(PSCI_1_0_FN_PSCI_FEATURES, func_id, 0, 0, 0, 0, 0, 0, &res); + + return res.a0; +} + static void vcpu_power_off(struct kvm_vm *vm, uint32_t vcpuid) { struct kvm_mp_state mp_state = { @@ -137,8 +156,58 @@ static void host_test_cpu_on(void) kvm_vm_free(vm); } +static void enable_system_suspend(struct kvm_vm *vm) +{ + struct kvm_enable_cap cap = { + .cap = KVM_CAP_ARM_SYSTEM_SUSPEND, + }; + + vm_enable_cap(vm, &cap); +} + +static void guest_test_system_suspend(void) +{ + uint64_t ret; + + /* assert that SYSTEM_SUSPEND is discoverable */ + GUEST_ASSERT(!psci_features(PSCI_1_0_FN_SYSTEM_SUSPEND)); + GUEST_ASSERT(!psci_features(PSCI_1_0_FN64_SYSTEM_SUSPEND)); + + ret = psci_system_suspend(CPU_ON_ENTRY_ADDR, CPU_ON_CONTEXT_ID); + GUEST_SYNC(ret); +} + +static void host_test_system_suspend(void) +{ + struct kvm_run *run; + struct kvm_vm *vm; + + vm = setup_vm(guest_test_system_suspend); + enable_system_suspend(vm); + + vcpu_power_off(vm, VCPU_ID_TARGET); + run = vcpu_state(vm, VCPU_ID_SOURCE); + + enter_guest(vm, VCPU_ID_SOURCE); + + TEST_ASSERT(run->exit_reason == KVM_EXIT_SYSTEM_EVENT, + "Unhandled exit reason: %u (%s)", + run->exit_reason, exit_reason_str(run->exit_reason)); + TEST_ASSERT(run->system_event.type == KVM_SYSTEM_EVENT_SUSPEND, + "Unhandled system event: %u (expected: %u)", + run->system_event.type, KVM_SYSTEM_EVENT_SUSPEND); + + kvm_vm_free(vm); +} + int main(void) { + if (!kvm_check_cap(KVM_CAP_ARM_SYSTEM_SUSPEND)) { + print_skip("KVM_CAP_ARM_SYSTEM_SUSPEND not supported"); + exit(KSFT_SKIP); + } + host_test_cpu_on(); + host_test_system_suspend(); return 0; } From c36820b04c56db0c34b198a2af6c9a6bd7d93459 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 4 May 2022 13:13:55 +0100 Subject: [PATCH 208/319] Documentation: Fix index.rst after psci.rst renaming Fix the TOC in index.rst after psci.rst has been renamed to hypercalls.rst. Signed-off-by: Marc Zyngier Reported-by: Stephen Rothwell Link: https://lore.kernel.org/r/20220504205627.18f46380@canb.auug.org.au --- Documentation/virt/kvm/arm/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/arm/index.rst b/Documentation/virt/kvm/arm/index.rst index 78a9b670aafe..e84848432158 100644 --- a/Documentation/virt/kvm/arm/index.rst +++ b/Documentation/virt/kvm/arm/index.rst @@ -8,6 +8,6 @@ ARM :maxdepth: 2 hyp-abi - psci + hypercalls pvtime ptp_kvm From 94828468a6085e6ae148986d300b634b87f86516 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 5 Apr 2022 19:23:25 +0100 Subject: [PATCH 209/319] KVM: arm64: vgic-v3: Expose GICR_CTLR.RWP when disabling LPIs When disabling LPIs, a guest needs to poll GICR_CTLR.RWP in order to be sure that the write has taken effect. We so far reported it as 0, as we didn't advertise that LPIs could be turned off the first place. Start tracking this state during which LPIs are being disabled, and expose the 'in progress' state via the RWP bit. We also take this opportunity to disallow enabling LPIs and programming GICR_{PEND,PROP}BASER while LPI disabling is in progress, as allowed by the architecture (UNPRED behaviour). We don't advertise the feature to the guest yet (which is allowed by the architecture). Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220405182327.205520-3-maz@kernel.org --- arch/arm64/kvm/vgic/vgic-its.c | 2 +- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 36 ++++++++++++++++++++++-------- arch/arm64/kvm/vgic/vgic.h | 1 + include/kvm/arm_vgic.h | 4 ++-- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 2e13402be3bd..9d11072964fb 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -683,7 +683,7 @@ int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its, if (!vcpu) return E_ITS_INT_UNMAPPED_INTERRUPT; - if (!vcpu->arch.vgic_cpu.lpis_enabled) + if (!vgic_lpis_enabled(vcpu)) return -EBUSY; vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq); diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 58e40b4874f8..b5ef7e15bb66 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -221,6 +221,13 @@ static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu, vgic_put_irq(vcpu->kvm, irq); } +bool vgic_lpis_enabled(struct kvm_vcpu *vcpu) +{ + struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + + return atomic_read(&vgic_cpu->ctlr) == GICR_CTLR_ENABLE_LPIS; +} + static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len) { @@ -229,26 +236,38 @@ static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu, return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0; } - static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; - bool was_enabled = vgic_cpu->lpis_enabled; + u32 ctlr; if (!vgic_has_its(vcpu->kvm)) return; - vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS; + if (!(val & GICR_CTLR_ENABLE_LPIS)) { + /* + * Don't disable if RWP is set, as there already an + * ongoing disable. Funky guest... + */ + ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, + GICR_CTLR_ENABLE_LPIS, + GICR_CTLR_RWP); + if (ctlr != GICR_CTLR_ENABLE_LPIS) + return; - if (was_enabled && !vgic_cpu->lpis_enabled) { vgic_flush_pending_lpis(vcpu); vgic_its_invalidate_cache(vcpu->kvm); - } + atomic_set_release(&vgic_cpu->ctlr, 0); + } else { + ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0, + GICR_CTLR_ENABLE_LPIS); + if (ctlr != 0) + return; - if (!was_enabled && vgic_cpu->lpis_enabled) vgic_enable_lpis(vcpu); + } } static bool vgic_mmio_vcpu_rdist_is_last(struct kvm_vcpu *vcpu) @@ -478,11 +497,10 @@ static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu, unsigned long val) { struct vgic_dist *dist = &vcpu->kvm->arch.vgic; - struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; u64 old_propbaser, propbaser; /* Storing a value with LPIs already enabled is undefined */ - if (vgic_cpu->lpis_enabled) + if (vgic_lpis_enabled(vcpu)) return; do { @@ -513,7 +531,7 @@ static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, u64 old_pendbaser, pendbaser; /* Storing a value with LPIs already enabled is undefined */ - if (vgic_cpu->lpis_enabled) + if (vgic_lpis_enabled(vcpu)) return; do { diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index 3fd6c86a7ef3..a21e9b602ff2 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -308,6 +308,7 @@ static inline bool vgic_dist_overlap(struct kvm *kvm, gpa_t base, size_t size) (base < d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE); } +bool vgic_lpis_enabled(struct kvm_vcpu *vcpu); int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr); int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its, u32 devid, u32 eventid, struct vgic_irq **irq); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index bb30a6803d9f..fdf1c2c322e5 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -347,8 +347,8 @@ struct vgic_cpu { /* Contains the attributes and gpa of the LPI pending tables. */ u64 pendbaser; - - bool lpis_enabled; + /* GICR_CTLR.{ENABLE_LPIS,RWP} */ + atomic_t ctlr; /* Cache guest priority bits */ u32 num_pri_bits; From 4645d11f4a5538ec1221f36e397cfb0115718ffe Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 5 Apr 2022 19:23:26 +0100 Subject: [PATCH 210/319] KVM: arm64: vgic-v3: Implement MMIO-based LPI invalidation Since GICv4.1, it has become legal for an implementation to advertise GICR_{INVLPIR,INVALLR,SYNCR} while having an ITS, allowing for a more efficient invalidation scheme (no guest command queue contention when multiple CPUs are generating invalidations). Provide the invalidation registers as a primitive to their ITS counterpart. Note that we don't advertise them to the guest yet (the architecture allows an implementation to do this). Signed-off-by: Marc Zyngier Reviewed-by: Oliver Upton Link: https://lore.kernel.org/r/20220405182327.205520-4-maz@kernel.org --- arch/arm64/kvm/vgic/vgic-its.c | 62 ++++++++++++++++++---------- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 66 ++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.h | 4 ++ include/kvm/arm_vgic.h | 1 + 4 files changed, 112 insertions(+), 21 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 9d11072964fb..292ca5bb8a50 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -1272,6 +1272,11 @@ static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its, return 0; } +int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq) +{ + return update_lpi_config(kvm, irq, NULL, true); +} + /* * The INV command syncs the configuration bits from the memory table. * Must be called with the its_lock mutex held. @@ -1288,7 +1293,41 @@ static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its, if (!ite) return E_ITS_INV_UNMAPPED_INTERRUPT; - return update_lpi_config(kvm, ite->irq, NULL, true); + return vgic_its_inv_lpi(kvm, ite->irq); +} + +/** + * vgic_its_invall - invalidate all LPIs targetting a given vcpu + * @vcpu: the vcpu for which the RD is targetted by an invalidation + * + * Contrary to the INVALL command, this targets a RD instead of a + * collection, and we don't need to hold the its_lock, since no ITS is + * involved here. + */ +int vgic_its_invall(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + int irq_count, i = 0; + u32 *intids; + + irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids); + if (irq_count < 0) + return irq_count; + + for (i = 0; i < irq_count; i++) { + struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intids[i]); + if (!irq) + continue; + update_lpi_config(kvm, irq, vcpu, false); + vgic_put_irq(kvm, irq); + } + + kfree(intids); + + if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm) + its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe); + + return 0; } /* @@ -1305,32 +1344,13 @@ static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its, u32 coll_id = its_cmd_get_collection(its_cmd); struct its_collection *collection; struct kvm_vcpu *vcpu; - struct vgic_irq *irq; - u32 *intids; - int irq_count, i; collection = find_collection(its, coll_id); if (!its_is_collection_mapped(collection)) return E_ITS_INVALL_UNMAPPED_COLLECTION; vcpu = kvm_get_vcpu(kvm, collection->target_addr); - - irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids); - if (irq_count < 0) - return irq_count; - - for (i = 0; i < irq_count; i++) { - irq = vgic_get_irq(kvm, NULL, intids[i]); - if (!irq) - continue; - update_lpi_config(kvm, irq, vcpu, false); - vgic_put_irq(kvm, irq); - } - - kfree(intids); - - if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm) - its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe); + vgic_its_invall(vcpu); return 0; } diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index b5ef7e15bb66..9824c773427d 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -543,6 +543,63 @@ static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, pendbaser) != old_pendbaser); } +static unsigned long vgic_mmio_read_sync(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len) +{ + return !!atomic_read(&vcpu->arch.vgic_cpu.syncr_busy); +} + +static void vgic_set_rdist_busy(struct kvm_vcpu *vcpu, bool busy) +{ + if (busy) { + atomic_inc(&vcpu->arch.vgic_cpu.syncr_busy); + smp_mb__after_atomic(); + } else { + smp_mb__before_atomic(); + atomic_dec(&vcpu->arch.vgic_cpu.syncr_busy); + } +} + +static void vgic_mmio_write_invlpi(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len, + unsigned long val) +{ + struct vgic_irq *irq; + + /* + * If the guest wrote only to the upper 32bit part of the + * register, drop the write on the floor, as it is only for + * vPEs (which we don't support for obvious reasons). + * + * Also discard the access if LPIs are not enabled. + */ + if ((addr & 4) || !vgic_lpis_enabled(vcpu)) + return; + + vgic_set_rdist_busy(vcpu, true); + + irq = vgic_get_irq(vcpu->kvm, NULL, lower_32_bits(val)); + if (irq) { + vgic_its_inv_lpi(vcpu->kvm, irq); + vgic_put_irq(vcpu->kvm, irq); + } + + vgic_set_rdist_busy(vcpu, false); +} + +static void vgic_mmio_write_invall(struct kvm_vcpu *vcpu, + gpa_t addr, unsigned int len, + unsigned long val) +{ + /* See vgic_mmio_write_invlpi() for the early return rationale */ + if ((addr & 4) || !vgic_lpis_enabled(vcpu)) + return; + + vgic_set_rdist_busy(vcpu, true); + vgic_its_invall(vcpu); + vgic_set_rdist_busy(vcpu, false); +} + /* * The GICv3 per-IRQ registers are split to control PPIs and SGIs in the * redistributors, while SPIs are covered by registers in the distributor @@ -648,6 +705,15 @@ static const struct vgic_register_region vgic_v3_rd_registers[] = { REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER, vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8, VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICR_INVLPIR, + vgic_mmio_read_raz, vgic_mmio_write_invlpi, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICR_INVALLR, + vgic_mmio_read_raz, vgic_mmio_write_invall, 8, + VGIC_ACCESS_64bit | VGIC_ACCESS_32bit), + REGISTER_DESC_WITH_LENGTH(GICR_SYNCR, + vgic_mmio_read_sync, vgic_mmio_write_wi, 4, + VGIC_ACCESS_32bit), REGISTER_DESC_WITH_LENGTH(GICR_IDREGS, vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48, VGIC_ACCESS_32bit), diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index a21e9b602ff2..1d04a900f3e3 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -318,6 +318,10 @@ void vgic_lpi_translation_cache_init(struct kvm *kvm); void vgic_lpi_translation_cache_destroy(struct kvm *kvm); void vgic_its_invalidate_cache(struct kvm *kvm); +/* GICv4.1 MMIO interface */ +int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq); +int vgic_its_invall(struct kvm_vcpu *vcpu); + bool vgic_supports_direct_msis(struct kvm *kvm); int vgic_v4_init(struct kvm *kvm); void vgic_v4_teardown(struct kvm *kvm); diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index fdf1c2c322e5..401236f97cf2 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -344,6 +344,7 @@ struct vgic_cpu { struct vgic_io_device rd_iodev; struct vgic_redist_region *rdreg; u32 rdreg_index; + atomic_t syncr_busy; /* Contains the attributes and gpa of the LPI pending tables. */ u64 pendbaser; From 49a1a2c70a7fd820fe60a65a3e38ab9095082dc9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 5 Apr 2022 19:23:27 +0100 Subject: [PATCH 211/319] KVM: arm64: vgic-v3: Advertise GICR_CTLR.{IR, CES} as a new GICD_IIDR revision Since adversising GICR_CTLR.{IC,CES} is directly observable from a guest, we need to make it selectable from userspace. For that, bump the default GICD_IIDR revision and let userspace downgrade it to the previous default. For GICv2, the two distributor revisions are strictly equivalent. Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220405182327.205520-5-maz@kernel.org --- arch/arm64/kvm/vgic/vgic-init.c | 7 ++++++- arch/arm64/kvm/vgic/vgic-mmio-v2.c | 18 +++++++++++++++--- arch/arm64/kvm/vgic/vgic-mmio-v3.c | 23 +++++++++++++++++++++-- arch/arm64/kvm/vgic/vgic.h | 5 +++++ include/kvm/arm_vgic.h | 3 +++ 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index fc00304fe7d8..f84e04f334c6 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -319,7 +319,12 @@ int vgic_init(struct kvm *kvm) vgic_debug_init(kvm); - dist->implementation_rev = 2; + /* + * If userspace didn't set the GIC implementation revision, + * default to the latest and greatest. You know want it. + */ + if (!dist->implementation_rev) + dist->implementation_rev = KVM_VGIC_IMP_REV_LATEST; dist->initialized = true; out: diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v2.c b/arch/arm64/kvm/vgic/vgic-mmio-v2.c index 12e4c223e6b8..77a67e9d3d14 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v2.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v2.c @@ -73,9 +73,13 @@ static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len, unsigned long val) { + struct vgic_dist *dist = &vcpu->kvm->arch.vgic; + u32 reg; + switch (addr & 0x0c) { case GIC_DIST_IIDR: - if (val != vgic_mmio_read_v2_misc(vcpu, addr, len)) + reg = vgic_mmio_read_v2_misc(vcpu, addr, len); + if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK) return -EINVAL; /* @@ -87,8 +91,16 @@ static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu, * migration from old kernels to new kernels with legacy * userspace. */ - vcpu->kvm->arch.vgic.v2_groups_user_writable = true; - return 0; + reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg); + switch (reg) { + case KVM_VGIC_IMP_REV_2: + case KVM_VGIC_IMP_REV_3: + vcpu->kvm->arch.vgic.v2_groups_user_writable = true; + dist->implementation_rev = reg; + return 0; + default: + return -EINVAL; + } } vgic_mmio_write_v2_misc(vcpu, addr, len, val); diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 9824c773427d..f7aa7bcd6fb8 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -155,13 +155,27 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu, unsigned long val) { struct vgic_dist *dist = &vcpu->kvm->arch.vgic; + u32 reg; switch (addr & 0x0c) { case GICD_TYPER2: - case GICD_IIDR: if (val != vgic_mmio_read_v3_misc(vcpu, addr, len)) return -EINVAL; return 0; + case GICD_IIDR: + reg = vgic_mmio_read_v3_misc(vcpu, addr, len); + if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK) + return -EINVAL; + + reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg); + switch (reg) { + case KVM_VGIC_IMP_REV_2: + case KVM_VGIC_IMP_REV_3: + dist->implementation_rev = reg; + return 0; + default: + return -EINVAL; + } case GICD_CTLR: /* Not a GICv4.1? No HW SGIs */ if (!kvm_vgic_global_state.has_gicv4_1) @@ -232,8 +246,13 @@ static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu, gpa_t addr, unsigned int len) { struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + unsigned long val; - return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0; + val = atomic_read(&vgic_cpu->ctlr); + if (vgic_get_implementation_rev(vcpu) >= KVM_VGIC_IMP_REV_3) + val |= GICR_CTLR_IR | GICR_CTLR_CES; + + return val; } static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index 1d04a900f3e3..4c6bdd321faa 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -98,6 +98,11 @@ #define DEBUG_SPINLOCK_BUG_ON(p) #endif +static inline u32 vgic_get_implementation_rev(struct kvm_vcpu *vcpu) +{ + return vcpu->kvm->arch.vgic.implementation_rev; +} + /* Requires the irq_lock to be held by the caller. */ static inline bool irq_is_pending(struct vgic_irq *irq) { diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 401236f97cf2..2d8f2e90edc2 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -231,6 +231,9 @@ struct vgic_dist { /* Implementation revision as reported in the GICD_IIDR */ u32 implementation_rev; +#define KVM_VGIC_IMP_REV_2 2 /* GICv2 restorable groups */ +#define KVM_VGIC_IMP_REV_3 3 /* GICv3 GICR_CTLR.{IW,CES,RWP} */ +#define KVM_VGIC_IMP_REV_LATEST KVM_VGIC_IMP_REV_3 /* Userspace can write to GICv2 IGROUPR */ bool v2_groups_user_writable; From f1f0c0cfeaa7c10eb536d9919bf3902af0e17bce Mon Sep 17 00:00:00 2001 From: Alexandru Elisei Date: Thu, 28 Apr 2022 11:34:04 +0100 Subject: [PATCH 212/319] KVM: arm64: Don't BUG_ON() if emulated register table is unsorted To emulate a register access, KVM uses a table of registers sorted by register encoding to speed up queries using binary search. When Linux boots, KVM checks that the table is sorted and uses a BUG_ON() statement to let the user know if it's not. The unfortunate side effect is that an unsorted sysreg table brings down the whole kernel, not just KVM, even though the rest of the kernel can function just fine without KVM. To make matters worse, on machines which lack a serial console, the user is left pondering why the machine is taking so long to boot. Improve this situation by returning an error from kvm_arch_init() if the sysreg tables are not in the correct order. The machine is still very much usable for the user, with the exception of virtualization, who can now easily determine what went wrong. A minor typo has also been corrected in the check_sysreg_table() function. Signed-off-by: Alexandru Elisei Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220428103405.70884-2-alexandru.elisei@arm.com --- arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/kvm/arm.c | 8 ++++++-- arch/arm64/kvm/sys_regs.c | 33 ++++++++++++++++++------------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 94a27a7520f4..39ca91cc1f01 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -686,7 +686,7 @@ int kvm_handle_sys_reg(struct kvm_vcpu *vcpu); void kvm_reset_sys_regs(struct kvm_vcpu *vcpu); -void kvm_sys_reg_table_init(void); +int kvm_sys_reg_table_init(void); /* MMIO helpers */ void kvm_mmio_write_buf(void *buf, unsigned int len, unsigned long data); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 523bc934fe2f..ab365358747a 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1761,8 +1761,6 @@ static int init_subsystems(void) kvm_register_perf_callbacks(NULL); - kvm_sys_reg_table_init(); - out: if (err || !is_protected_kvm_enabled()) on_each_cpu(_kvm_arch_hardware_disable, NULL, 1); @@ -2089,6 +2087,12 @@ int kvm_arch_init(void *opaque) return -ENODEV; } + err = kvm_sys_reg_table_init(); + if (err) { + kvm_info("Error initializing system register tables"); + return err; + } + in_hyp_mode = is_kernel_in_hyp_mode(); if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) || diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 7b45c040cc27..57302048afd0 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2187,25 +2187,24 @@ static const struct sys_reg_desc cp15_64_regs[] = { { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer }, }; -static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n, - bool is_32) +static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n, + bool is_32) { unsigned int i; for (i = 0; i < n; i++) { if (!is_32 && table[i].reg && !table[i].reset) { - kvm_err("sys_reg table %p entry %d has lacks reset\n", - table, i); - return 1; + kvm_err("sys_reg table %p entry %d lacks reset\n", table, i); + return false; } if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) { kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1); - return 1; + return false; } } - return 0; + return true; } int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu) @@ -2860,18 +2859,22 @@ int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) return write_demux_regids(uindices); } -void kvm_sys_reg_table_init(void) +int kvm_sys_reg_table_init(void) { + bool valid = true; unsigned int i; struct sys_reg_desc clidr; /* Make sure tables are unique and in order. */ - BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false)); - BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true)); - BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true)); - BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true)); - BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true)); - BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false)); + valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false); + valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true); + valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true); + valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true); + valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true); + valid &= check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false); + + if (!valid) + return -EINVAL; /* We abuse the reset function to overwrite the table itself. */ for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) @@ -2894,4 +2897,6 @@ void kvm_sys_reg_table_init(void) break; /* Clear all higher bits. */ cache_levels &= (1 << (i*3))-1; + + return 0; } From 325031d4f39cfb1adf41708c1329e60b9abfbdac Mon Sep 17 00:00:00 2001 From: Alexandru Elisei Date: Thu, 28 Apr 2022 11:34:05 +0100 Subject: [PATCH 213/319] KVM: arm64: Print emulated register table name when it is unsorted When a sysreg table entry is out-of-order, KVM attempts to print the address of the table: [ 0.143911] kvm [1]: sys_reg table (____ptrval____) out of order (1) Printing the name of the table instead of a pointer is more helpful in this case. The message has also been slightly tweaked to be point out the offending entry (and to match the missing reset error message): [ 0.143891] kvm [1]: sys_reg table sys_reg_descs+0x50/0x7490 entry 1 out of order Signed-off-by: Alexandru Elisei Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220428103405.70884-3-alexandru.elisei@arm.com --- arch/arm64/kvm/sys_regs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 57302048afd0..a84a556eeb8f 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2194,12 +2194,12 @@ static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n, for (i = 0; i < n; i++) { if (!is_32 && table[i].reg && !table[i].reset) { - kvm_err("sys_reg table %p entry %d lacks reset\n", table, i); + kvm_err("sys_reg table %pS entry %d lacks reset\n", &table[i], i); return false; } if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) { - kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1); + kvm_err("sys_reg table %pS entry %d out of order\n", &table[i - 1], i - 1); return false; } } From 582eb04e05ddd234ca32083c8457c6d409fd7b6a Mon Sep 17 00:00:00 2001 From: Stephen Rothwell Date: Thu, 5 May 2022 20:06:34 +1000 Subject: [PATCH 214/319] Documentation: KVM: Fix title level for PSCI_SUSPEND The htmldoc build breaks in a funny way with: Sphinx parallel build error: docutils.utils.SystemMessage: /home/sfr/next/next/Documentation/virt/kvm/api.rst:6175: (SEVERE/4) Title level inconsistent: For arm/arm64: ^^^^^^^^^^^^^^ Swap the ^^s for a bunch of --s... Signed-off-by: Stephen Rothwell [maz: commit message] Signed-off-by: Marc Zyngier --- Documentation/virt/kvm/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index d8d7859fc556..25253b326730 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6047,7 +6047,7 @@ Valid values for 'type' are: the VM. For arm/arm64: -^^^^^^^^^^^^^^ +-------------- KVM_SYSTEM_EVENT_SUSPEND exits are enabled with the KVM_CAP_ARM_SYSTEM_SUSPEND VM capability. If a guest invokes the PSCI From 6d5f2207447b28dc73c25b3907e7ee32ee66bdbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 2 May 2022 19:08:27 +0200 Subject: [PATCH 215/319] gpio: max732x: Drop unused support for irq and setup code via platform data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only user of max732x_platform_data is arch/arm/mach-pxa/littleton.c and it only uses .gpio_base. So drop the other members from the data struct and simplify the driver accordingly. The motivating side effect of this change is that the .remove() callback cannot return a nonzero error code any more which prepares making i2c remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-max732x.c | 37 ++------------------------- include/linux/platform_data/max732x.h | 12 --------- 2 files changed, 2 insertions(+), 47 deletions(-) diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c index 238cbe926b9f..da6972117030 100644 --- a/drivers/gpio/gpio-max732x.c +++ b/drivers/gpio/gpio-max732x.c @@ -496,17 +496,13 @@ static int max732x_irq_setup(struct max732x_chip *chip, const struct i2c_device_id *id) { struct i2c_client *client = chip->client; - struct max732x_platform_data *pdata = dev_get_platdata(&client->dev); int has_irq = max732x_features[id->driver_data] >> 32; int irq_base = 0; int ret; - if (((pdata && pdata->irq_base) || client->irq) - && has_irq != INT_NONE) { + if (client->irq && has_irq != INT_NONE) { struct gpio_irq_chip *girq; - if (pdata) - irq_base = pdata->irq_base; chip->irq_features = has_irq; mutex_init(&chip->irq_lock); @@ -540,10 +536,9 @@ static int max732x_irq_setup(struct max732x_chip *chip, const struct i2c_device_id *id) { struct i2c_client *client = chip->client; - struct max732x_platform_data *pdata = dev_get_platdata(&client->dev); int has_irq = max732x_features[id->driver_data] >> 32; - if (((pdata && pdata->irq_base) || client->irq) && has_irq != INT_NONE) + if (client->irq && has_irq != INT_NONE) dev_warn(&client->dev, "interrupt support not compiled in\n"); return 0; @@ -703,44 +698,16 @@ static int max732x_probe(struct i2c_client *client, if (ret) return ret; - if (pdata->setup) { - ret = pdata->setup(client, chip->gpio_chip.base, - chip->gpio_chip.ngpio, pdata->context); - if (ret < 0) - dev_warn(&client->dev, "setup failed, %d\n", ret); - } - i2c_set_clientdata(client, chip); return 0; } -static int max732x_remove(struct i2c_client *client) -{ - struct max732x_platform_data *pdata = dev_get_platdata(&client->dev); - struct max732x_chip *chip = i2c_get_clientdata(client); - - if (pdata && pdata->teardown) { - int ret; - - ret = pdata->teardown(client, chip->gpio_chip.base, - chip->gpio_chip.ngpio, pdata->context); - if (ret < 0) { - dev_err(&client->dev, "%s failed, %d\n", - "teardown", ret); - return ret; - } - } - - return 0; -} - static struct i2c_driver max732x_driver = { .driver = { .name = "max732x", .of_match_table = of_match_ptr(max732x_of_table), }, .probe = max732x_probe, - .remove = max732x_remove, .id_table = max732x_id, }; diff --git a/include/linux/platform_data/max732x.h b/include/linux/platform_data/max732x.h index f231c635faec..423999207cd5 100644 --- a/include/linux/platform_data/max732x.h +++ b/include/linux/platform_data/max732x.h @@ -7,17 +7,5 @@ struct max732x_platform_data { /* number of the first GPIO */ unsigned gpio_base; - - /* interrupt base */ - int irq_base; - - void *context; /* param to setup/teardown */ - - int (*setup)(struct i2c_client *client, - unsigned gpio, unsigned ngpio, - void *context); - int (*teardown)(struct i2c_client *client, - unsigned gpio, unsigned ngpio, - void *context); }; #endif /* __LINUX_I2C_MAX732X_H */ From a9e49635e263b5f304179ef537e0b056199c66d8 Mon Sep 17 00:00:00 2001 From: Justin Chen Date: Wed, 4 May 2022 15:29:16 -0700 Subject: [PATCH 216/319] gpio: pca953xx: Add support for pca6408 Add support for pca6408 which is the 8-bit version of the pca6416. https://www.nxp.com/docs/en/data-sheet/PCA6408A.pdf Signed-off-by: Justin Chen Acked-by: Florian Fainelli Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-pca953x.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index d2fe76f3f34f..4909175e6921 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -71,6 +71,7 @@ #define PCA_CHIP_TYPE(x) ((x) & PCA_TYPE_MASK) static const struct i2c_device_id pca953x_id[] = { + { "pca6408", 8 | PCA953X_TYPE | PCA_INT, }, { "pca6416", 16 | PCA953X_TYPE | PCA_INT, }, { "pca9505", 40 | PCA953X_TYPE | PCA_INT, }, { "pca9506", 40 | PCA953X_TYPE | PCA_INT, }, @@ -1198,6 +1199,7 @@ static int pca953x_resume(struct device *dev) #define OF_957X(__nrgpio, __int) (void *)(__nrgpio | PCA957X_TYPE | __int) static const struct of_device_id pca953x_dt_ids[] = { + { .compatible = "nxp,pca6408", .data = OF_953X(8, PCA_INT), }, { .compatible = "nxp,pca6416", .data = OF_953X(16, PCA_INT), }, { .compatible = "nxp,pca9505", .data = OF_953X(40, PCA_INT), }, { .compatible = "nxp,pca9506", .data = OF_953X(40, PCA_INT), }, From 2d4fdc15f9b7c1cbc1800653f8c1a48ab615cbd8 Mon Sep 17 00:00:00 2001 From: Justin Chen Date: Wed, 4 May 2022 15:29:17 -0700 Subject: [PATCH 217/319] dt-bindings: gpio: pca95xx: add entry for pca6408 The NXP PCA5608 is the 8-bit version of PCA5616. Signed-off-by: Justin Chen Acked-by: Florian Fainelli Acked-by: Krzysztof Kozlowski Signed-off-by: Bartosz Golaszewski --- Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml index dc0fc8fc4839..977b14db09b0 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml @@ -30,6 +30,7 @@ properties: - maxim,max7325 - maxim,max7326 - maxim,max7327 + - nxp,pca6408 - nxp,pca6416 - nxp,pca9505 - nxp,pca9506 From 7f42aa7b008c611743daf830e5ac4066e8ae276f Mon Sep 17 00:00:00 2001 From: Jon Hunter Date: Wed, 4 May 2022 15:44:06 +0100 Subject: [PATCH 218/319] gpio: max77620: Make the irqchip immutable Commit 6c846d026d49 ("gpio: Don't fiddle with irqchips marked as immutable") added a warning to indicate if the gpiolib is altering the internals of irqchips. Following this change the following warning is now observed for the max77620 gpio driver ... WARNING KERN gpio gpiochip0: (max77620-gpio): not an immutable chip, please consider fixing it! Fix the above warning by making the max77620 gpio driver immutable. Signed-off-by: Jon Hunter Reviewed-by: Marc Zyngier Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-max77620.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpio/gpio-max77620.c b/drivers/gpio/gpio-max77620.c index ebf9dea6546b..c18b60e39a94 100644 --- a/drivers/gpio/gpio-max77620.c +++ b/drivers/gpio/gpio-max77620.c @@ -54,6 +54,7 @@ static void max77620_gpio_irq_mask(struct irq_data *data) struct max77620_gpio *gpio = gpiochip_get_data(chip); gpio->irq_enabled[data->hwirq] = false; + gpiochip_disable_irq(chip, data->hwirq); } static void max77620_gpio_irq_unmask(struct irq_data *data) @@ -61,6 +62,7 @@ static void max77620_gpio_irq_unmask(struct irq_data *data) struct gpio_chip *chip = irq_data_get_irq_chip_data(data); struct max77620_gpio *gpio = gpiochip_get_data(chip); + gpiochip_enable_irq(chip, data->hwirq); gpio->irq_enabled[data->hwirq] = true; } @@ -119,14 +121,15 @@ static void max77620_gpio_bus_sync_unlock(struct irq_data *data) mutex_unlock(&gpio->buslock); } -static struct irq_chip max77620_gpio_irqchip = { +static const struct irq_chip max77620_gpio_irqchip = { .name = "max77620-gpio", .irq_mask = max77620_gpio_irq_mask, .irq_unmask = max77620_gpio_irq_unmask, .irq_set_type = max77620_gpio_set_irq_type, .irq_bus_lock = max77620_gpio_bus_lock, .irq_bus_sync_unlock = max77620_gpio_bus_sync_unlock, - .flags = IRQCHIP_MASK_ON_SUSPEND, + .flags = IRQCHIP_IMMUTABLE | IRQCHIP_MASK_ON_SUSPEND, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static int max77620_gpio_dir_input(struct gpio_chip *gc, unsigned int offset) @@ -318,7 +321,7 @@ static int max77620_gpio_probe(struct platform_device *pdev) mgpio->gpio_chip.base = -1; girq = &mgpio->gpio_chip.irq; - girq->chip = &max77620_gpio_irqchip; + gpio_irq_chip_set_chip(girq, &max77620_gpio_irqchip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; From 7ee74cc7ad7ef526e4383a3a3294b7039622eb37 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 28 Apr 2022 16:03:50 +0200 Subject: [PATCH 219/319] KVM: arm64: Avoid unnecessary absolute addressing via literals There are a few cases in the nVHE code where we take the absolute address of a symbol via a literal pool entry, and subsequently translate it to another address space (PA, kimg VA, kernel linear VA, etc). Originally, this literal was needed because we relied on a different translation for absolute references, but this is no longer the case, so we can simply use relative addressing instead. This removes a couple of RELA entries pointing into the .text segment. Signed-off-by: Ard Biesheuvel Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220428140350.3303481-1-ardb@kernel.org --- arch/arm64/kvm/hyp/nvhe/host.S | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 3d613e721a75..366551594417 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -80,7 +80,7 @@ SYM_FUNC_START(__hyp_do_panic) mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ PSR_MODE_EL1h) msr spsr_el2, lr - ldr lr, =nvhe_hyp_panic_handler + adr_l lr, nvhe_hyp_panic_handler hyp_kimg_va lr, x6 msr elr_el2, lr @@ -125,13 +125,11 @@ alternative_else_nop_endif add sp, sp, #16 /* * Compute the idmap address of __kvm_handle_stub_hvc and - * jump there. Since we use kimage_voffset, do not use the - * HYP VA for __kvm_handle_stub_hvc, but the kernel VA instead - * (by loading it from the constant pool). + * jump there. * * Preserve x0-x4, which may contain stub parameters. */ - ldr x5, =__kvm_handle_stub_hvc + adr_l x5, __kvm_handle_stub_hvc hyp_pa x5, x6 br x5 SYM_FUNC_END(__host_hvc) From bd61395ae8393f28f4b084702acd6f5f02b1f7c0 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Fri, 29 Apr 2022 22:01:23 -0700 Subject: [PATCH 220/319] KVM: arm64: nvhe: Eliminate kernel-doc warnings Don't use begin-kernel-doc notation (/**) for comments that are not in kernel-doc format. This prevents these kernel-doc warnings: arch/arm64/kvm/hyp/nvhe/switch.c:126: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Disable host events, enable guest events arch/arm64/kvm/hyp/nvhe/switch.c:146: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Disable guest events, enable host events arch/arm64/kvm/hyp/nvhe/switch.c:164: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Handler for protected VM restricted exceptions. arch/arm64/kvm/hyp/nvhe/switch.c:176: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst * Handler for protected VM MSR, MRS or System instruction execution in AArch64. arch/arm64/kvm/hyp/nvhe/switch.c:196: warning: Function parameter or member 'vcpu' not described in 'kvm_handle_pvm_fpsimd' arch/arm64/kvm/hyp/nvhe/switch.c:196: warning: Function parameter or member 'exit_code' not described in 'kvm_handle_pvm_fpsimd' arch/arm64/kvm/hyp/nvhe/switch.c:196: warning: expecting prototype for Handler for protected floating(). Prototype was for kvm_handle_pvm_fpsimd() instead Fixes: 09cf57eba304 ("KVM: arm64: Split hyp/switch.c to VHE/nVHE") Fixes: 1423afcb4117 ("KVM: arm64: Trap access to pVM restricted features") Signed-off-by: Randy Dunlap Reported-by: kernel test robot Cc: Fuad Tabba Cc: Marc Zyngier Cc: David Brazdil Cc: James Morse Cc: Alexandru Elisei Cc: Suzuki K Poulose Cc: linux-arm-kernel@lists.infradead.org Cc: kvmarm@lists.cs.columbia.edu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220430050123.2844-1-rdunlap@infradead.org --- arch/arm64/kvm/hyp/nvhe/switch.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 6410d21d8695..476196ea90c4 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -120,7 +120,7 @@ static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) } } -/** +/* * Disable host events, enable guest events */ static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) @@ -140,7 +140,7 @@ static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) return (pmu->events_host || pmu->events_guest); } -/** +/* * Disable guest events, enable host events */ static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) @@ -158,7 +158,7 @@ static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) write_sysreg(pmu->events_host, pmcntenset_el0); } -/** +/* * Handler for protected VM MSR, MRS or System instruction execution in AArch64. * * Returns true if the hypervisor has handled the exit, and control should go @@ -175,7 +175,7 @@ static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code) kvm_handle_pvm_sysreg(vcpu, exit_code)); } -/** +/* * Handler for protected floating-point and Advanced SIMD accesses. * * Returns true if the hypervisor has handled the exit, and control should go From 764aaf44cd64dd1f760268ee0b22d2dc53cd5bc0 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Wed, 27 Apr 2022 20:54:01 +0800 Subject: [PATCH 221/319] reboot: Fix build warning without CONFIG_SYSCTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If CONFIG_SYSCTL is n, build warn: kernel/reboot.c:443:20: error: ‘kernel_reboot_sysctls_init’ defined but not used [-Werror=unused-function] static void __init kernel_reboot_sysctls_init(void) ^~~~~~~~~~~~~~~~~~~~~~~~~~ Move kernel_reboot_sysctls_init() to #ifdef block to fix this. Fixes: 06d177662fb8 ("kernel/reboot: move reboot sysctls to its own file") Signed-off-by: YueHaibing Signed-off-by: Luis Chamberlain --- kernel/reboot.c | 54 ++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/kernel/reboot.c b/kernel/reboot.c index ed4e6dfb7d44..ecbf09ea03c5 100644 --- a/kernel/reboot.c +++ b/kernel/reboot.c @@ -421,33 +421,6 @@ void ctrl_alt_del(void) static char poweroff_cmd[POWEROFF_CMD_PATH_LEN] = "/sbin/poweroff"; static const char reboot_cmd[] = "/sbin/reboot"; -#ifdef CONFIG_SYSCTL -static struct ctl_table kern_reboot_table[] = { - { - .procname = "poweroff_cmd", - .data = &poweroff_cmd, - .maxlen = POWEROFF_CMD_PATH_LEN, - .mode = 0644, - .proc_handler = proc_dostring, - }, - { - .procname = "ctrl-alt-del", - .data = &C_A_D, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, - { } -}; - -static void __init kernel_reboot_sysctls_init(void) -{ - register_sysctl_init("kernel", kern_reboot_table); -} -#else -#define kernel_reboot_sysctls_init() do { } while (0) -#endif /* CONFIG_SYSCTL */ - static int run_cmd(const char *cmd) { char **argv; @@ -895,6 +868,33 @@ static struct attribute *reboot_attrs[] = { NULL, }; +#ifdef CONFIG_SYSCTL +static struct ctl_table kern_reboot_table[] = { + { + .procname = "poweroff_cmd", + .data = &poweroff_cmd, + .maxlen = POWEROFF_CMD_PATH_LEN, + .mode = 0644, + .proc_handler = proc_dostring, + }, + { + .procname = "ctrl-alt-del", + .data = &C_A_D, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { } +}; + +static void __init kernel_reboot_sysctls_init(void) +{ + register_sysctl_init("kernel", kern_reboot_table); +} +#else +#define kernel_reboot_sysctls_init() do { } while (0) +#endif /* CONFIG_SYSCTL */ + static const struct attribute_group reboot_attr_group = { .attrs = reboot_attrs, }; From 494dcdf46e5cdee926c9f441d37e3ea1db57d1da Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Wed, 27 Apr 2022 21:10:02 +0800 Subject: [PATCH 222/319] sched: Fix build warning without CONFIG_SYSCTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IF CONFIG_SYSCTL is n, build warn: kernel/sched/core.c:1782:12: warning: ‘sysctl_sched_uclamp_handler’ defined but not used [-Wunused-function] static int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, ^~~~~~~~~~~~~~~~~~~~~~~~~~~ sysctl_sched_uclamp_handler() is used while CONFIG_SYSCTL enabled, wrap all related code with CONFIG_SYSCTL to fix this. Fixes: 3267e0156c33 ("sched: Move uclamp_util sysctls to core.c") Signed-off-by: YueHaibing Signed-off-by: Luis Chamberlain --- kernel/sched/core.c | 65 +++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ef31751c5799..b8d2ff09b853 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1306,10 +1306,10 @@ static void set_load_weight(struct task_struct *p, bool update_load) static DEFINE_MUTEX(uclamp_mutex); /* Max allowed minimum utilization */ -static unsigned int sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; +static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE; /* Max allowed maximum utilization */ -static unsigned int sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; +static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE; /* * By default RT tasks run at the maximum performance point/capacity of the @@ -1456,33 +1456,6 @@ static void uclamp_update_util_min_rt_default(struct task_struct *p) task_rq_unlock(rq, p, &rf); } -static void uclamp_sync_util_min_rt_default(void) -{ - struct task_struct *g, *p; - - /* - * copy_process() sysctl_uclamp - * uclamp_min_rt = X; - * write_lock(&tasklist_lock) read_lock(&tasklist_lock) - * // link thread smp_mb__after_spinlock() - * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); - * sched_post_fork() for_each_process_thread() - * __uclamp_sync_rt() __uclamp_sync_rt() - * - * Ensures that either sched_post_fork() will observe the new - * uclamp_min_rt or for_each_process_thread() will observe the new - * task. - */ - read_lock(&tasklist_lock); - smp_mb__after_spinlock(); - read_unlock(&tasklist_lock); - - rcu_read_lock(); - for_each_process_thread(g, p) - uclamp_update_util_min_rt_default(p); - rcu_read_unlock(); -} - static inline struct uclamp_se uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id) { @@ -1762,6 +1735,11 @@ uclamp_update_active_tasks(struct cgroup_subsys_state *css) } static void cpu_util_update_eff(struct cgroup_subsys_state *css); +#endif + +#ifdef CONFIG_SYSCTL +#ifdef CONFIG_UCLAMP_TASK +#ifdef CONFIG_UCLAMP_TASK_GROUP static void uclamp_update_root_tg(void) { struct task_group *tg = &root_task_group; @@ -1779,6 +1757,33 @@ static void uclamp_update_root_tg(void) static void uclamp_update_root_tg(void) { } #endif +static void uclamp_sync_util_min_rt_default(void) +{ + struct task_struct *g, *p; + + /* + * copy_process() sysctl_uclamp + * uclamp_min_rt = X; + * write_lock(&tasklist_lock) read_lock(&tasklist_lock) + * // link thread smp_mb__after_spinlock() + * write_unlock(&tasklist_lock) read_unlock(&tasklist_lock); + * sched_post_fork() for_each_process_thread() + * __uclamp_sync_rt() __uclamp_sync_rt() + * + * Ensures that either sched_post_fork() will observe the new + * uclamp_min_rt or for_each_process_thread() will observe the new + * task. + */ + read_lock(&tasklist_lock); + smp_mb__after_spinlock(); + read_unlock(&tasklist_lock); + + rcu_read_lock(); + for_each_process_thread(g, p) + uclamp_update_util_min_rt_default(p); + rcu_read_unlock(); +} + static int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) { @@ -1843,6 +1848,8 @@ static int sysctl_sched_uclamp_handler(struct ctl_table *table, int write, return result; } +#endif +#endif static int uclamp_validate(struct task_struct *p, const struct sched_attr *attr) From 4d2e469e163ec79340b2f42c2a07838b5ff30686 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Mon, 9 May 2022 16:25:58 +0000 Subject: [PATCH 223/319] KVM: arm64: pkvm: Drop unnecessary FP/SIMD trap handler The pVM-specific FP/SIMD trap handler just calls straight into the generic trap handler. Avoid the indirection and just call the hyp handler directly. Note that the BUILD_BUG_ON() pattern is repeated in pvm_init_traps_aa64pfr0(), which is likely a better home for it. No functional change intended. Signed-off-by: Oliver Upton Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220509162559.2387784-2-oupton@google.com --- arch/arm64/kvm/hyp/nvhe/switch.c | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 476196ea90c4..cba01d41c5c9 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -175,23 +175,6 @@ static bool kvm_handle_pvm_sys64(struct kvm_vcpu *vcpu, u64 *exit_code) kvm_handle_pvm_sysreg(vcpu, exit_code)); } -/* - * Handler for protected floating-point and Advanced SIMD accesses. - * - * Returns true if the hypervisor has handled the exit, and control should go - * back to the guest, or false if it hasn't. - */ -static bool kvm_handle_pvm_fpsimd(struct kvm_vcpu *vcpu, u64 *exit_code) -{ - /* Linux guests assume support for floating-point and Advanced SIMD. */ - BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_FP), - PVM_ID_AA64PFR0_ALLOW)); - BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_ASIMD), - PVM_ID_AA64PFR0_ALLOW)); - - return kvm_hyp_handle_fpsimd(vcpu, exit_code); -} - static const exit_handler_fn hyp_exit_handlers[] = { [0 ... ESR_ELx_EC_MAX] = NULL, [ESR_ELx_EC_CP15_32] = kvm_hyp_handle_cp15_32, @@ -207,7 +190,7 @@ static const exit_handler_fn pvm_exit_handlers[] = { [0 ... ESR_ELx_EC_MAX] = NULL, [ESR_ELx_EC_SYS64] = kvm_handle_pvm_sys64, [ESR_ELx_EC_SVE] = kvm_handle_pvm_restricted, - [ESR_ELx_EC_FP_ASIMD] = kvm_handle_pvm_fpsimd, + [ESR_ELx_EC_FP_ASIMD] = kvm_hyp_handle_fpsimd, [ESR_ELx_EC_IABT_LOW] = kvm_hyp_handle_iabt_low, [ESR_ELx_EC_DABT_LOW] = kvm_hyp_handle_dabt_low, [ESR_ELx_EC_PAC] = kvm_hyp_handle_ptrauth, From 249838b7660ac04a67bfb017364a7f01029370a0 Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Mon, 9 May 2022 16:25:59 +0000 Subject: [PATCH 224/319] KVM: arm64: pkvm: Don't mask already zeroed FEAT_SVE FEAT_SVE is already masked by the fixed configuration for ID_AA64PFR0_EL1; don't try and mask it at runtime. No functional change intended. Signed-off-by: Oliver Upton Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220509162559.2387784-3-oupton@google.com --- arch/arm64/kvm/hyp/nvhe/sys_regs.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 33f5181af330..3f5d7bd171c5 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -90,9 +90,6 @@ static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu) u64 set_mask = 0; u64 allow_mask = PVM_ID_AA64PFR0_ALLOW; - if (!vcpu_has_sve(vcpu)) - allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_SVE); - set_mask |= get_restricted_features_unsigned(id_aa64pfr0_el1_sys_val, PVM_ID_AA64PFR0_RESTRICT_UNSIGNED); From 7464ff8bf2d762251b9537863db0e1caf9b0e402 Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Mon, 4 Apr 2022 13:11:01 +0900 Subject: [PATCH 225/319] platform/chrome: cros_ec_typec: Check for EC driver The EC driver may not be initialized when cros_typec_probe is called, particulary when CONFIG_CROS_EC_CHARDEV=m. Signed-off-by: Akihiko Odaki Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20220404041101.6276-1-akihiko.odaki@gmail.com Signed-off-by: Prashant Malani --- drivers/platform/chrome/cros_ec_typec.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index 4bd2752c0823..7cb2e35c4ded 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -1084,6 +1084,9 @@ static int cros_typec_probe(struct platform_device *pdev) } ec_dev = dev_get_drvdata(&typec->ec->ec->dev); + if (!ec_dev) + return -EPROBE_DEFER; + typec->typec_cmd_supported = cros_ec_check_features(ec_dev, EC_FEATURE_TYPEC_CMD); typec->needs_mux_ack = cros_ec_check_features(ec_dev, EC_FEATURE_TYPEC_MUX_REQUIRE_AP_ACK); From 33fbe6befa622c082f7d417896832856814bdde0 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Thu, 12 May 2022 13:14:20 +0300 Subject: [PATCH 226/319] KVM: x86: fix typo in __try_cmpxchg_user causing non-atomicness This shows up as a TDP MMU leak when running nested. Non-working cmpxchg on L0 relies makes L1 install two different shadow pages under same spte, and one of them is leaked. Fixes: 1c2361f667f36 ("KVM: x86: Use __try_cmpxchg_user() to emulate atomic accesses") Signed-off-by: Maxim Levitsky Message-Id: <20220512101420.306759-1-mlevitsk@redhat.com> Reviewed-by: Sean Christopherson Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ba4faeb32437..8f2108e86b02 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7315,7 +7315,7 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt, goto emul_write; hva = kvm_vcpu_gfn_to_hva(vcpu, gpa_to_gfn(gpa)); - if (kvm_is_error_hva(addr)) + if (kvm_is_error_hva(hva)) goto emul_write; hva += offset_in_page(gpa); From 91ab933f7514fc0fb42b77dc6d128c450baeae0b Mon Sep 17 00:00:00 2001 From: Li RongQing Date: Wed, 6 Apr 2022 19:25:02 +0800 Subject: [PATCH 227/319] KVM: VMX: clean up pi_wakeup_handler Passing per_cpu() to list_for_each_entry() causes the macro to be evaluated N+1 times for N sleeping vCPUs. This is a very small inefficiency, and the code is cleaner if the address of the per-CPU variable is loaded earlier. Do this for both the list and the spinlock. Signed-off-by: Li RongQing Message-Id: <1649244302-6777-1-git-send-email-lirongqing@baidu.com> Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/posted_intr.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/posted_intr.c b/arch/x86/kvm/vmx/posted_intr.c index 37d0bfff269d..07e5fcf5a5aa 100644 --- a/arch/x86/kvm/vmx/posted_intr.c +++ b/arch/x86/kvm/vmx/posted_intr.c @@ -202,16 +202,17 @@ void vmx_vcpu_pi_put(struct kvm_vcpu *vcpu) void pi_wakeup_handler(void) { int cpu = smp_processor_id(); + struct list_head *wakeup_list = &per_cpu(wakeup_vcpus_on_cpu, cpu); + raw_spinlock_t *spinlock = &per_cpu(wakeup_vcpus_on_cpu_lock, cpu); struct vcpu_vmx *vmx; - raw_spin_lock(&per_cpu(wakeup_vcpus_on_cpu_lock, cpu)); - list_for_each_entry(vmx, &per_cpu(wakeup_vcpus_on_cpu, cpu), - pi_wakeup_list) { + raw_spin_lock(spinlock); + list_for_each_entry(vmx, wakeup_list, pi_wakeup_list) { if (pi_test_on(&vmx->pi_desc)) kvm_vcpu_wake_up(&vmx->vcpu); } - raw_spin_unlock(&per_cpu(wakeup_vcpus_on_cpu_lock, cpu)); + raw_spin_unlock(spinlock); } void __init pi_init_cpu(int cpu) From 54275f74cf384217d9e6e85629793f0a024d88b7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 23 Apr 2022 03:47:44 +0000 Subject: [PATCH 228/319] KVM: x86/mmu: Don't attempt fast page fault just because EPT is in use Check for A/D bits being disabled instead of the access tracking mask being non-zero when deciding whether or not to attempt to fix a page fault vian the fast path. Originally, the access tracking mask was non-zero if and only if A/D bits were disabled by _KVM_ (including not being supported by hardware), but that hasn't been true since nVMX was fixed to honor EPTP12's A/D enabling, i.e. since KVM allowed L1 to cause KVM to not use A/D bits while running L2 despite KVM using them while running L1. In other words, don't attempt the fast path just because EPT is enabled. Note, attempting the fast path for all !PRESENT faults can "fix" a very, _VERY_ tiny percentage of faults out of mmu_lock by detecting that the fault is spurious, i.e. has been fixed by a different vCPU, but again the odds of that happening are vanishingly small. E.g. booting an 8-vCPU VM gets less than 10 successes out of 30k+ faults, and that's likely one of the more favorable scenarios. Disabling dirty logging can likely lead to a rash of collisions between vCPUs for some workloads that operate on a common set of pages, but penalizing _all_ !PRESENT faults for that one case is unlikely to be a net positive, not to mention that that problem is best solved by not zapping in the first place. The number of spurious faults does scale with the number of vCPUs, e.g. a 255-vCPU VM using TDP "jumps" to ~60 spurious faults detected in the fast path (again out of 30k), but that's all of 0.2% of faults. Using legacy shadow paging does get more spurious faults, and a few more detected out of mmu_lock, but the percentage goes _down_ to 0.08% (and that's ignoring faults that are reflected into the guest), i.e. the extra detections are purely due to the sheer number of faults observed. On the other hand, getting a "negative" in the fast path takes in the neighborhood of 150-250 cycles. So while it is tempting to keep/extend the current behavior, such a change needs to come with hard numbers showing that it's actually a win in the grand scheme, or any scheme for that matter. Fixes: 995f00a61958 ("x86: kvm: mmu: use ept a/d in vmcs02 iff used in vmcs12") Signed-off-by: Sean Christopherson Message-Id: <20220423034752.1161007-5-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 45 ++++++++++++++++++++++++-------------- arch/x86/kvm/mmu/spte.h | 11 ++++++++++ arch/x86/kvm/mmu/tdp_mmu.c | 2 +- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 9826c62f8e40..f36e3cdde25d 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3056,19 +3056,20 @@ static bool page_fault_can_be_fast(struct kvm_page_fault *fault) /* * #PF can be fast if: - * 1. The shadow page table entry is not present, which could mean that - * the fault is potentially caused by access tracking (if enabled). - * 2. The shadow page table entry is present and the fault - * is caused by write-protect, that means we just need change the W - * bit of the spte which can be done out of mmu-lock. * - * However, if access tracking is disabled we know that a non-present - * page must be a genuine page fault where we have to create a new SPTE. - * So, if access tracking is disabled, we return true only for write - * accesses to a present page. + * 1. The shadow page table entry is not present and A/D bits are + * disabled _by KVM_, which could mean that the fault is potentially + * caused by access tracking (if enabled). If A/D bits are enabled + * by KVM, but disabled by L1 for L2, KVM is forced to disable A/D + * bits for L2 and employ access tracking, but the fast page fault + * mechanism only supports direct MMUs. + * 2. The shadow page table entry is present, the access is a write, + * and no reserved bits are set (MMIO SPTEs cannot be "fixed"), i.e. + * the fault was caused by a write-protection violation. If the + * SPTE is MMU-writable (determined later), the fault can be fixed + * by setting the Writable bit, which can be done out of mmu_lock. */ - - return shadow_acc_track_mask != 0 || (fault->write && fault->present); + return !kvm_ad_enabled() || (fault->write && fault->present); } /* @@ -3183,13 +3184,25 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) new_spte = spte; - if (is_access_track_spte(spte)) + /* + * KVM only supports fixing page faults outside of MMU lock for + * direct MMUs, nested MMUs are always indirect, and KVM always + * uses A/D bits for non-nested MMUs. Thus, if A/D bits are + * enabled, the SPTE can't be an access-tracked SPTE. + */ + if (unlikely(!kvm_ad_enabled()) && is_access_track_spte(spte)) new_spte = restore_acc_track_spte(new_spte); /* - * Currently, to simplify the code, write-protection can - * be removed in the fast path only if the SPTE was - * write-protected for dirty-logging or access tracking. + * To keep things simple, only SPTEs that are MMU-writable can + * be made fully writable outside of mmu_lock, e.g. only SPTEs + * that were write-protected for dirty-logging or access + * tracking are handled here. Don't bother checking if the + * SPTE is writable to prioritize running with A/D bits enabled. + * The is_access_allowed() check above handles the common case + * of the fault being spurious, and the SPTE is known to be + * shadow-present, i.e. except for access tracking restoration + * making the new SPTE writable, the check is wasteful. */ if (fault->write && is_mmu_writable_spte(spte)) { new_spte |= PT_WRITABLE_MASK; @@ -4794,7 +4807,7 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, role.efer_nx = true; role.smm = cpu_role.base.smm; role.guest_mode = cpu_role.base.guest_mode; - role.ad_disabled = (shadow_accessed_mask == 0); + role.ad_disabled = !kvm_ad_enabled(); role.level = kvm_mmu_get_tdp_level(vcpu); role.direct = true; role.has_4_byte_gpte = false; diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h index 76f12b375b5a..43f7924bc7f0 100644 --- a/arch/x86/kvm/mmu/spte.h +++ b/arch/x86/kvm/mmu/spte.h @@ -214,6 +214,17 @@ static inline bool is_shadow_present_pte(u64 pte) return !!(pte & SPTE_MMU_PRESENT_MASK); } +/* + * Returns true if A/D bits are supported in hardware and are enabled by KVM. + * When enabled, KVM uses A/D bits for all non-nested MMUs. Because L1 can + * disable A/D bits in EPTP12, SP and SPTE variants are needed to handle the + * scenario where KVM is using A/D bits for L1, but not L2. + */ +static inline bool kvm_ad_enabled(void) +{ + return !!shadow_accessed_mask; +} + static inline bool sp_ad_disabled(struct kvm_mmu_page *sp) { return sp->role.ad_disabled; diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index f84133856063..7fc4ead5ce0d 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1136,7 +1136,7 @@ static int tdp_mmu_link_sp(struct kvm *kvm, struct tdp_iter *iter, struct kvm_mmu_page *sp, bool account_nx, bool shared) { - u64 spte = make_nonleaf_spte(sp->spt, !shadow_accessed_mask); + u64 spte = make_nonleaf_spte(sp->spt, !kvm_ad_enabled()); int ret = 0; if (shared) { From 5c64aba5179f6fc6a367f76342c6290a5e06cfc4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 23 Apr 2022 03:47:45 +0000 Subject: [PATCH 229/319] KVM: x86/mmu: Drop exec/NX check from "page fault can be fast" Tweak the "page fault can be fast" logic to explicitly check for !PRESENT faults in the access tracking case, and drop the exec/NX check that becomes redundant as a result. No sane hardware will generate an access that is both an instruct fetch and a write, i.e. it's a waste of cycles. If hardware goes off the rails, or KVM runs under a misguided hypervisor, spuriously running throught fast path is benign (KVM has been uknowingly being doing exactly that for years). Signed-off-by: Sean Christopherson Message-Id: <20220423034752.1161007-6-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f36e3cdde25d..add486220dae 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3044,16 +3044,14 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa static bool page_fault_can_be_fast(struct kvm_page_fault *fault) { /* - * Do not fix the mmio spte with invalid generation number which - * need to be updated by slow page fault path. + * Page faults with reserved bits set, i.e. faults on MMIO SPTEs, only + * reach the common page fault handler if the SPTE has an invalid MMIO + * generation number. Refreshing the MMIO generation needs to go down + * the slow path. Note, EPT Misconfigs do NOT set the PRESENT flag! */ if (fault->rsvd) return false; - /* See if the page fault is due to an NX violation */ - if (unlikely(fault->exec && fault->present)) - return false; - /* * #PF can be fast if: * @@ -3069,7 +3067,14 @@ static bool page_fault_can_be_fast(struct kvm_page_fault *fault) * SPTE is MMU-writable (determined later), the fault can be fixed * by setting the Writable bit, which can be done out of mmu_lock. */ - return !kvm_ad_enabled() || (fault->write && fault->present); + if (!fault->present) + return !kvm_ad_enabled(); + + /* + * Note, instruction fetches and writes are mutually exclusive, ignore + * the "exec" flag. + */ + return fault->write; } /* From 5276c616abf149a4dfbc708df3f22da642ec3dde Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 23 Apr 2022 03:47:46 +0000 Subject: [PATCH 230/319] KVM: x86/mmu: Add RET_PF_CONTINUE to eliminate bool+int* "returns" Add RET_PF_CONTINUE and use it in handle_abnormal_pfn() and kvm_faultin_pfn() to signal that the page fault handler should continue doing its thing. Aside from being gross and inefficient, using a boolean return to signal continue vs. stop makes it extremely difficult to add more helpers and/or move existing code to a helper. E.g. hypothetically, if nested MMUs were to gain a separate page fault handler in the future, everything up to the "is self-modifying PTE" check can be shared by all shadow MMUs, but communicating up the stack whether to continue on or stop becomes a nightmare. More concretely, proposed support for private guest memory ran into a similar issue, where it'll be forced to forego a helper in order to yield sane code: https://lore.kernel.org/all/YkJbxiL%2FAz7olWlq@google.com. No functional change intended. Cc: David Matlack Cc: Chao Peng Signed-off-by: Sean Christopherson Message-Id: <20220423034752.1161007-7-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 51 ++++++++++++++------------------- arch/x86/kvm/mmu/mmu_internal.h | 9 +++++- arch/x86/kvm/mmu/mmutrace.h | 1 + arch/x86/kvm/mmu/paging_tmpl.h | 6 ++-- 4 files changed, 35 insertions(+), 32 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index add486220dae..8313c1d8e814 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3007,14 +3007,12 @@ static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn) return -EFAULT; } -static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, - unsigned int access, int *ret_val) +static int handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, + unsigned int access) { /* The pfn is invalid, report the error! */ - if (unlikely(is_error_pfn(fault->pfn))) { - *ret_val = kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn); - return true; - } + if (unlikely(is_error_pfn(fault->pfn))) + return kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn); if (unlikely(!fault->slot)) { gva_t gva = fault->is_tdp ? 0 : fault->addr; @@ -3032,13 +3030,11 @@ static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fa * the hardware's). */ if (unlikely(!enable_mmio_caching) || - unlikely(fault->gfn > kvm_mmu_max_gfn())) { - *ret_val = RET_PF_EMULATE; - return true; - } + unlikely(fault->gfn > kvm_mmu_max_gfn())) + return RET_PF_EMULATE; } - return false; + return RET_PF_CONTINUE; } static bool page_fault_can_be_fast(struct kvm_page_fault *fault) @@ -3946,7 +3942,7 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch); } -static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, int *r) +static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { struct kvm_memory_slot *slot = fault->slot; bool async; @@ -3957,7 +3953,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, * be zapped before KVM inserts a new MMIO SPTE for the gfn. */ if (slot && (slot->flags & KVM_MEMSLOT_INVALID)) - goto out_retry; + return RET_PF_RETRY; if (!kvm_is_visible_memslot(slot)) { /* Don't expose private memslots to L2. */ @@ -3965,7 +3961,7 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, fault->slot = NULL; fault->pfn = KVM_PFN_NOSLOT; fault->map_writable = false; - return false; + return RET_PF_CONTINUE; } /* * If the APIC access page exists but is disabled, go directly @@ -3974,10 +3970,8 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, * when the AVIC is re-enabled. */ if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT && - !kvm_apicv_activated(vcpu->kvm)) { - *r = RET_PF_EMULATE; - return true; - } + !kvm_apicv_activated(vcpu->kvm)) + return RET_PF_EMULATE; } async = false; @@ -3985,26 +3979,23 @@ static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, fault->write, &fault->map_writable, &fault->hva); if (!async) - return false; /* *pfn has correct page already */ + return RET_PF_CONTINUE; /* *pfn has correct page already */ if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) { trace_kvm_try_async_get_page(fault->addr, fault->gfn); if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) { trace_kvm_async_pf_doublefault(fault->addr, fault->gfn); kvm_make_request(KVM_REQ_APF_HALT, vcpu); - goto out_retry; - } else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) - goto out_retry; + return RET_PF_RETRY; + } else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn)) { + return RET_PF_RETRY; + } } fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL, fault->write, &fault->map_writable, &fault->hva); - return false; - -out_retry: - *r = RET_PF_RETRY; - return true; + return RET_PF_CONTINUE; } /* @@ -4059,10 +4050,12 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - if (kvm_faultin_pfn(vcpu, fault, &r)) + r = kvm_faultin_pfn(vcpu, fault); + if (r != RET_PF_CONTINUE) return r; - if (handle_abnormal_pfn(vcpu, fault, ACC_ALL, &r)) + r = handle_abnormal_pfn(vcpu, fault, ACC_ALL); + if (r != RET_PF_CONTINUE) return r; r = RET_PF_RETRY; diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index 1bff453f7cbe..c0e502b17ef7 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -143,6 +143,7 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head); /* * Return values of handle_mmio_page_fault, mmu.page_fault, and fast_page_fault(). * + * RET_PF_CONTINUE: So far, so good, keep handling the page fault. * RET_PF_RETRY: let CPU fault again on the address. * RET_PF_EMULATE: mmio page fault, emulate the instruction directly. * RET_PF_INVALID: the spte is invalid, let the real page fault path update it. @@ -151,9 +152,15 @@ unsigned int pte_list_count(struct kvm_rmap_head *rmap_head); * * Any names added to this enum should be exported to userspace for use in * tracepoints via TRACE_DEFINE_ENUM() in mmutrace.h + * + * Note, all values must be greater than or equal to zero so as not to encroach + * on -errno return values. Somewhat arbitrarily use '0' for CONTINUE, which + * will allow for efficient machine code when checking for CONTINUE, e.g. + * "TEST %rax, %rax, JNZ", as all "stop!" values are non-zero. */ enum { - RET_PF_RETRY = 0, + RET_PF_CONTINUE = 0, + RET_PF_RETRY, RET_PF_EMULATE, RET_PF_INVALID, RET_PF_FIXED, diff --git a/arch/x86/kvm/mmu/mmutrace.h b/arch/x86/kvm/mmu/mmutrace.h index 12247b96af01..ae86820cef69 100644 --- a/arch/x86/kvm/mmu/mmutrace.h +++ b/arch/x86/kvm/mmu/mmutrace.h @@ -54,6 +54,7 @@ { PFERR_RSVD_MASK, "RSVD" }, \ { PFERR_FETCH_MASK, "F" } +TRACE_DEFINE_ENUM(RET_PF_CONTINUE); TRACE_DEFINE_ENUM(RET_PF_RETRY); TRACE_DEFINE_ENUM(RET_PF_EMULATE); TRACE_DEFINE_ENUM(RET_PF_INVALID); diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index b025decf610d..7f8f1c8dbed2 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -838,10 +838,12 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - if (kvm_faultin_pfn(vcpu, fault, &r)) + r = kvm_faultin_pfn(vcpu, fault); + if (r != RET_PF_CONTINUE) return r; - if (handle_abnormal_pfn(vcpu, fault, walker.pte_access, &r)) + r = handle_abnormal_pfn(vcpu, fault, walker.pte_access); + if (r != RET_PF_CONTINUE) return r; /* From 8a009d5bca0af7385880123d12081713f7066904 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 23 Apr 2022 03:47:47 +0000 Subject: [PATCH 231/319] KVM: x86/mmu: Make all page fault handlers internal to the MMU Move kvm_arch_async_page_ready() to mmu.c where it belongs, and move all of the page fault handling collateral that was in mmu.h purely for the async #PF handler into mmu_internal.h, where it belongs. This will allow kvm_mmu_do_page_fault() to act on the RET_PF_* return without having to expose those enums outside of the MMU. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20220423034752.1161007-8-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu.h | 87 ------------------------------- arch/x86/kvm/mmu/mmu.c | 19 +++++++ arch/x86/kvm/mmu/mmu_internal.h | 90 ++++++++++++++++++++++++++++++++- arch/x86/kvm/x86.c | 19 ------- 4 files changed, 108 insertions(+), 107 deletions(-) diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index c242e26cfa80..da5ed1430d7e 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -141,93 +141,6 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu) vcpu->arch.mmu->root_role.level); } -struct kvm_page_fault { - /* arguments to kvm_mmu_do_page_fault. */ - const gpa_t addr; - const u32 error_code; - const bool prefetch; - - /* Derived from error_code. */ - const bool exec; - const bool write; - const bool present; - const bool rsvd; - const bool user; - - /* Derived from mmu and global state. */ - const bool is_tdp; - const bool nx_huge_page_workaround_enabled; - - /* - * Whether a >4KB mapping can be created or is forbidden due to NX - * hugepages. - */ - bool huge_page_disallowed; - - /* - * Maximum page size that can be created for this fault; input to - * FNAME(fetch), __direct_map and kvm_tdp_mmu_map. - */ - u8 max_level; - - /* - * Page size that can be created based on the max_level and the - * page size used by the host mapping. - */ - u8 req_level; - - /* - * Page size that will be created based on the req_level and - * huge_page_disallowed. - */ - u8 goal_level; - - /* Shifted addr, or result of guest page table walk if addr is a gva. */ - gfn_t gfn; - - /* The memslot containing gfn. May be NULL. */ - struct kvm_memory_slot *slot; - - /* Outputs of kvm_faultin_pfn. */ - kvm_pfn_t pfn; - hva_t hva; - bool map_writable; -}; - -int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); - -extern int nx_huge_pages; -static inline bool is_nx_huge_page_enabled(void) -{ - return READ_ONCE(nx_huge_pages); -} - -static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, - u32 err, bool prefetch) -{ - struct kvm_page_fault fault = { - .addr = cr2_or_gpa, - .error_code = err, - .exec = err & PFERR_FETCH_MASK, - .write = err & PFERR_WRITE_MASK, - .present = err & PFERR_PRESENT_MASK, - .rsvd = err & PFERR_RSVD_MASK, - .user = err & PFERR_USER_MASK, - .prefetch = prefetch, - .is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault), - .nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(), - - .max_level = KVM_MAX_HUGEPAGE_LEVEL, - .req_level = PG_LEVEL_4K, - .goal_level = PG_LEVEL_4K, - }; -#ifdef CONFIG_RETPOLINE - if (fault.is_tdp) - return kvm_tdp_page_fault(vcpu, &fault); -#endif - return vcpu->arch.mmu->page_fault(vcpu, &fault); -} - /* * Check if a given access (described through the I/D, W/R and U/S bits of a * page fault error code pfec) causes a permission fault with the given PTE diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8313c1d8e814..f22cbb2aef53 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3942,6 +3942,25 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch); } +void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) +{ + int r; + + if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) || + work->wakeup_all) + return; + + r = kvm_mmu_reload(vcpu); + if (unlikely(r)) + return; + + if (!vcpu->arch.mmu->root_role.direct && + work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu)) + return; + + kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true); +} + static int kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) { struct kvm_memory_slot *slot = fault->slot; diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index c0e502b17ef7..c0c85cbfa159 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -140,8 +140,70 @@ void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, u64 start_gfn, u64 pages); unsigned int pte_list_count(struct kvm_rmap_head *rmap_head); +extern int nx_huge_pages; +static inline bool is_nx_huge_page_enabled(void) +{ + return READ_ONCE(nx_huge_pages); +} + +struct kvm_page_fault { + /* arguments to kvm_mmu_do_page_fault. */ + const gpa_t addr; + const u32 error_code; + const bool prefetch; + + /* Derived from error_code. */ + const bool exec; + const bool write; + const bool present; + const bool rsvd; + const bool user; + + /* Derived from mmu and global state. */ + const bool is_tdp; + const bool nx_huge_page_workaround_enabled; + + /* + * Whether a >4KB mapping can be created or is forbidden due to NX + * hugepages. + */ + bool huge_page_disallowed; + + /* + * Maximum page size that can be created for this fault; input to + * FNAME(fetch), __direct_map and kvm_tdp_mmu_map. + */ + u8 max_level; + + /* + * Page size that can be created based on the max_level and the + * page size used by the host mapping. + */ + u8 req_level; + + /* + * Page size that will be created based on the req_level and + * huge_page_disallowed. + */ + u8 goal_level; + + /* Shifted addr, or result of guest page table walk if addr is a gva. */ + gfn_t gfn; + + /* The memslot containing gfn. May be NULL. */ + struct kvm_memory_slot *slot; + + /* Outputs of kvm_faultin_pfn. */ + kvm_pfn_t pfn; + hva_t hva; + bool map_writable; +}; + +int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault); + /* - * Return values of handle_mmio_page_fault, mmu.page_fault, and fast_page_fault(). + * Return values of handle_mmio_page_fault(), mmu.page_fault(), fast_page_fault(), + * and of course kvm_mmu_do_page_fault(). * * RET_PF_CONTINUE: So far, so good, keep handling the page fault. * RET_PF_RETRY: let CPU fault again on the address. @@ -167,6 +229,32 @@ enum { RET_PF_SPURIOUS, }; +static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, + u32 err, bool prefetch) +{ + struct kvm_page_fault fault = { + .addr = cr2_or_gpa, + .error_code = err, + .exec = err & PFERR_FETCH_MASK, + .write = err & PFERR_WRITE_MASK, + .present = err & PFERR_PRESENT_MASK, + .rsvd = err & PFERR_RSVD_MASK, + .user = err & PFERR_USER_MASK, + .prefetch = prefetch, + .is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault), + .nx_huge_page_workaround_enabled = is_nx_huge_page_enabled(), + + .max_level = KVM_MAX_HUGEPAGE_LEVEL, + .req_level = PG_LEVEL_4K, + .goal_level = PG_LEVEL_4K, + }; +#ifdef CONFIG_RETPOLINE + if (fault.is_tdp) + return kvm_tdp_page_fault(vcpu, &fault); +#endif + return vcpu->arch.mmu->page_fault(vcpu, &fault); +} + int kvm_mmu_max_mapping_level(struct kvm *kvm, const struct kvm_memory_slot *slot, gfn_t gfn, kvm_pfn_t pfn, int max_level); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8f2108e86b02..ac6007a476e5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12358,25 +12358,6 @@ void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) } EXPORT_SYMBOL_GPL(kvm_set_rflags); -void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) -{ - int r; - - if ((vcpu->arch.mmu->root_role.direct != work->arch.direct_map) || - work->wakeup_all) - return; - - r = kvm_mmu_reload(vcpu); - if (unlikely(r)) - return; - - if (!vcpu->arch.mmu->root_role.direct && - work->arch.cr3 != vcpu->arch.mmu->get_guest_pgd(vcpu)) - return; - - kvm_mmu_do_page_fault(vcpu, work->cr2_or_gpa, 0, true); -} - static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) { BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU)); From 8d5265b1016369836c0735ed40933e179f186630 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 23 Apr 2022 03:47:48 +0000 Subject: [PATCH 232/319] KVM: x86/mmu: Use IS_ENABLED() to avoid RETPOLINE for TDP page faults Use IS_ENABLED() instead of an #ifdef to activate the anti-RETPOLINE fast path for TDP page faults. The generated code is identical, and the #ifdef makes it dangerously difficult to extend the logic (guess who forgot to add an "else" inside the #ifdef and ran through the page fault handler twice). No functional or binary change intented. Signed-off-by: Sean Christopherson Message-Id: <20220423034752.1161007-9-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu_internal.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index c0c85cbfa159..9caa747ee033 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -248,10 +248,10 @@ static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, .req_level = PG_LEVEL_4K, .goal_level = PG_LEVEL_4K, }; -#ifdef CONFIG_RETPOLINE - if (fault.is_tdp) + + if (IS_ENABLED(CONFIG_RETPOLINE) && fault.is_tdp) return kvm_tdp_page_fault(vcpu, &fault); -#endif + return vcpu->arch.mmu->page_fault(vcpu, &fault); } From 1075d41efd598d3fd4d52a1e1116b20979975135 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Sat, 23 Apr 2022 03:47:49 +0000 Subject: [PATCH 233/319] KVM: x86/mmu: Expand and clean up page fault stats Expand and clean up the page fault stats. The current stats are at best incomplete, and at worst misleading. Differentiate between faults that are actually fixed vs those that result in an MMIO SPTE being created, track faults that are spurious, faults that trigger emulation, faults that that are fixed in the fast path, and last but not least, track the number of faults that are taken. Note, the number of faults that require emulation for write-protected shadow pages can roughly be calculated by subtracting the number of MMIO SPTEs created from the overall number of faults that trigger emulation. Signed-off-by: Sean Christopherson Message-Id: <20220423034752.1161007-10-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 5 +++++ arch/x86/kvm/mmu/mmu.c | 7 +++++-- arch/x86/kvm/mmu/mmu_internal.h | 28 ++++++++++++++++++++++++++-- arch/x86/kvm/mmu/paging_tmpl.h | 1 - arch/x86/kvm/mmu/tdp_mmu.c | 8 +------- arch/x86/kvm/x86.c | 5 +++++ 6 files changed, 42 insertions(+), 12 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f164c6c1514a..c5fb4115176d 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1269,7 +1269,12 @@ struct kvm_vm_stat { struct kvm_vcpu_stat { struct kvm_vcpu_stat_generic generic; + u64 pf_taken; u64 pf_fixed; + u64 pf_emulate; + u64 pf_spurious; + u64 pf_fast; + u64 pf_mmio_spte_created; u64 pf_guest; u64 tlb_flush; u64 invlpg; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f22cbb2aef53..59e02fb705c0 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2660,6 +2660,7 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot, *sptep, write_fault, gfn); if (unlikely(is_noslot_pfn(pfn))) { + vcpu->stat.pf_mmio_spte_created++; mark_mmio_spte(vcpu, sptep, gfn, pte_access); return RET_PF_EMULATE; } @@ -2980,7 +2981,6 @@ static int __direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) return ret; direct_pte_prefetch(vcpu, it.sptep); - ++vcpu->stat.pf_fixed; return ret; } @@ -3249,6 +3249,9 @@ static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault) trace_fast_page_fault(vcpu, fault, sptep, spte, ret); walk_shadow_page_lockless_end(vcpu); + if (ret != RET_PF_INVALID) + vcpu->stat.pf_fast++; + return ret; } @@ -5354,7 +5357,7 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, write_unlock(&vcpu->kvm->mmu_lock); } -int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, +int noinline kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, void *insn, int insn_len) { int r, emulation_type = EMULTYPE_PF; diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index 9caa747ee033..bd2a26897b97 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -248,11 +248,35 @@ static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, .req_level = PG_LEVEL_4K, .goal_level = PG_LEVEL_4K, }; + int r; + + /* + * Async #PF "faults", a.k.a. prefetch faults, are not faults from the + * guest perspective and have already been counted at the time of the + * original fault. + */ + if (!prefetch) + vcpu->stat.pf_taken++; if (IS_ENABLED(CONFIG_RETPOLINE) && fault.is_tdp) - return kvm_tdp_page_fault(vcpu, &fault); + r = kvm_tdp_page_fault(vcpu, &fault); + else + r = vcpu->arch.mmu->page_fault(vcpu, &fault); - return vcpu->arch.mmu->page_fault(vcpu, &fault); + /* + * Similar to above, prefetch faults aren't truly spurious, and the + * async #PF path doesn't do emulation. Do count faults that are fixed + * by the async #PF handler though, otherwise they'll never be counted. + */ + if (r == RET_PF_FIXED) + vcpu->stat.pf_fixed++; + else if (prefetch) + ; + else if (r == RET_PF_EMULATE) + vcpu->stat.pf_emulate++; + else if (r == RET_PF_SPURIOUS) + vcpu->stat.pf_spurious++; + return r; } int kvm_mmu_max_mapping_level(struct kvm *kvm, diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 7f8f1c8dbed2..db80f7ccaa4e 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -723,7 +723,6 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, return ret; FNAME(pte_prefetch)(vcpu, gw, it.sptep); - ++vcpu->stat.pf_fixed; return ret; out_gpte_changed: diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 7fc4ead5ce0d..841feaa48be5 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1100,6 +1100,7 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */ if (unlikely(is_mmio_spte(new_spte))) { + vcpu->stat.pf_mmio_spte_created++; trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn, new_spte); ret = RET_PF_EMULATE; @@ -1108,13 +1109,6 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, rcu_dereference(iter->sptep)); } - /* - * Increase pf_fixed in both RET_PF_EMULATE and RET_PF_FIXED to be - * consistent with legacy MMU behavior. - */ - if (ret != RET_PF_SPURIOUS) - vcpu->stat.pf_fixed++; - return ret; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ac6007a476e5..85dfa414582d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -266,7 +266,12 @@ const struct kvm_stats_header kvm_vm_stats_header = { const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { KVM_GENERIC_VCPU_STATS(), + STATS_DESC_COUNTER(VCPU, pf_taken), STATS_DESC_COUNTER(VCPU, pf_fixed), + STATS_DESC_COUNTER(VCPU, pf_emulate), + STATS_DESC_COUNTER(VCPU, pf_spurious), + STATS_DESC_COUNTER(VCPU, pf_fast), + STATS_DESC_COUNTER(VCPU, pf_mmio_spte_created), STATS_DESC_COUNTER(VCPU, pf_guest), STATS_DESC_COUNTER(VCPU, tlb_flush), STATS_DESC_COUNTER(VCPU, invlpg), From c9f3d9fbcd9a013f44a43bd74a3f40d924ce4843 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 27 Apr 2022 13:34:51 -0400 Subject: [PATCH 234/319] KVM: x86: a vCPU with a pending triple fault is runnable Reviewed-by: Maxim Levitsky Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 85dfa414582d..476ea099ebf0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12266,6 +12266,9 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) if (kvm_xen_has_pending_events(vcpu)) return true; + if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) + return true; + return false; } From c919e881ba83e2912354ffa035980f62c78cc2f3 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 19 Apr 2022 23:17:02 +1200 Subject: [PATCH 235/319] KVM: x86/mmu: Rename reset_rsvds_bits_mask() Rename reset_rsvds_bits_mask() to reset_guest_rsvds_bits_mask() to make it clearer that it resets the reserved bits check for guest's page table entries. Signed-off-by: Kai Huang Message-Id: Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 59e02fb705c0..b91d2af46d90 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4442,8 +4442,8 @@ static bool guest_can_use_gbpages(struct kvm_vcpu *vcpu) guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES); } -static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, - struct kvm_mmu *context) +static void reset_guest_rsvds_bits_mask(struct kvm_vcpu *vcpu, + struct kvm_mmu *context) { __reset_rsvds_bits_mask(&context->guest_rsvd_check, vcpu->arch.reserved_gpa_bits, @@ -4742,7 +4742,7 @@ static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu, if (!is_cr0_pg(mmu)) return; - reset_rsvds_bits_mask(vcpu, mmu); + reset_guest_rsvds_bits_mask(vcpu, mmu); update_permission_bitmask(mmu, false); update_pkru_bitmask(mmu); } From e54f1ff244ac96c919049838a5a1f03087793594 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 19 Apr 2022 23:17:03 +1200 Subject: [PATCH 236/319] KVM: x86/mmu: Add shadow_me_value and repurpose shadow_me_mask Intel Multi-Key Total Memory Encryption (MKTME) repurposes couple of high bits of physical address bits as 'KeyID' bits. Intel Trust Domain Extentions (TDX) further steals part of MKTME KeyID bits as TDX private KeyID bits. TDX private KeyID bits cannot be set in any mapping in the host kernel since they can only be accessed by software running inside a new CPU isolated mode. And unlike to AMD's SME, host kernel doesn't set any legacy MKTME KeyID bits to any mapping either. Therefore, it's not legitimate for KVM to set any KeyID bits in SPTE which maps guest memory. KVM maintains shadow_zero_check bits to represent which bits must be zero for SPTE which maps guest memory. MKTME KeyID bits should be set to shadow_zero_check. Currently, shadow_me_mask is used by AMD to set the sme_me_mask to SPTE, and shadow_me_shadow is excluded from shadow_zero_check. So initializing shadow_me_mask to represent all MKTME keyID bits doesn't work for VMX (as oppositely, they must be set to shadow_zero_check). Introduce a new 'shadow_me_value' to replace existing shadow_me_mask, and repurpose shadow_me_mask as 'all possible memory encryption bits'. The new schematic of them will be: - shadow_me_value: the memory encryption bit(s) that will be set to the SPTE (the original shadow_me_mask). - shadow_me_mask: all possible memory encryption bits (which is a super set of shadow_me_value). - For now, shadow_me_value is supposed to be set by SVM and VMX respectively, and it is a constant during KVM's life time. This perhaps doesn't fit MKTME but for now host kernel doesn't support it (and perhaps will never do). - Bits in shadow_me_mask are set to shadow_zero_check, except the bits in shadow_me_value. Introduce a new helper kvm_mmu_set_me_spte_mask() to initialize them. Replace shadow_me_mask with shadow_me_value in almost all code paths, except the one in PT64_PERM_MASK, which is used by need_remote_flush() to determine whether remote TLB flush is needed. This should still use shadow_me_mask as any encryption bit change should need a TLB flush. And for AMD, move initializing shadow_me_value/shadow_me_mask from kvm_mmu_reset_all_pte_masks() to svm_hardware_setup(). Signed-off-by: Kai Huang Message-Id: Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu.h | 1 + arch/x86/kvm/mmu/mmu.c | 16 ++++++++++++---- arch/x86/kvm/mmu/spte.c | 23 +++++++++++++++++------ arch/x86/kvm/mmu/spte.h | 1 + arch/x86/kvm/svm/svm.c | 3 +++ 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index da5ed1430d7e..1db11cd6b177 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -90,6 +90,7 @@ static inline gfn_t kvm_mmu_max_gfn(void) } void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask); +void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask); void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only); void kvm_init_mmu(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b91d2af46d90..ea4bc085bbf5 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3546,7 +3546,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * or a PAE 3-level page table. In either case we need to be aware that * the shadow page table may be a PAE or a long mode page table. */ - pm_mask = PT_PRESENT_MASK | shadow_me_mask; + pm_mask = PT_PRESENT_MASK | shadow_me_value; if (mmu->root_role.level >= PT64_ROOT_4LEVEL) { pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; @@ -4531,8 +4531,16 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, return; for (i = context->root_role.level; --i >= 0;) { - shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask; - shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask; + /* + * So far shadow_me_value is a constant during KVM's life + * time. Bits in shadow_me_value are allowed to be set. + * Bits in shadow_me_mask but not in shadow_me_value are + * not allowed to be set. + */ + shadow_zero_check->rsvd_bits_mask[0][i] |= shadow_me_mask; + shadow_zero_check->rsvd_bits_mask[1][i] |= shadow_me_mask; + shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_value; + shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_value; } } @@ -5624,7 +5632,7 @@ static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) if (!tdp_enabled) set_memory_decrypted((unsigned long)mmu->pae_root, 1); else - WARN_ON_ONCE(shadow_me_mask); + WARN_ON_ONCE(shadow_me_value); for (i = 0; i < 4; ++i) mmu->pae_root[i] = INVALID_PAE_ROOT; diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c index 800b857b3a53..12b9b3f8bd05 100644 --- a/arch/x86/kvm/mmu/spte.c +++ b/arch/x86/kvm/mmu/spte.c @@ -33,6 +33,7 @@ u64 __read_mostly shadow_mmio_value; u64 __read_mostly shadow_mmio_mask; u64 __read_mostly shadow_mmio_access_mask; u64 __read_mostly shadow_present_mask; +u64 __read_mostly shadow_me_value; u64 __read_mostly shadow_me_mask; u64 __read_mostly shadow_acc_track_mask; @@ -167,8 +168,8 @@ bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, else pte_access &= ~ACC_WRITE_MASK; - if (shadow_me_mask && !kvm_is_mmio_pfn(pfn)) - spte |= shadow_me_mask; + if (shadow_me_value && !kvm_is_mmio_pfn(pfn)) + spte |= shadow_me_value; spte |= (u64)pfn << PAGE_SHIFT; @@ -284,7 +285,7 @@ u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled) u64 spte = SPTE_MMU_PRESENT_MASK; spte |= __pa(child_pt) | shadow_present_mask | PT_WRITABLE_MASK | - shadow_user_mask | shadow_x_mask | shadow_me_mask; + shadow_user_mask | shadow_x_mask | shadow_me_value; if (ad_disabled) spte |= SPTE_TDP_AD_DISABLED_MASK; @@ -388,6 +389,17 @@ void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask) } EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask); +void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask) +{ + /* shadow_me_value must be a subset of shadow_me_mask */ + if (WARN_ON(me_value & ~me_mask)) + me_value = me_mask = 0; + + shadow_me_value = me_value; + shadow_me_mask = me_mask; +} +EXPORT_SYMBOL_GPL(kvm_mmu_set_me_spte_mask); + void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only) { shadow_user_mask = VMX_EPT_READABLE_MASK; @@ -397,8 +409,6 @@ void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only) shadow_x_mask = VMX_EPT_EXECUTABLE_MASK; shadow_present_mask = has_exec_only ? 0ull : VMX_EPT_READABLE_MASK; shadow_acc_track_mask = VMX_EPT_RWX_MASK; - shadow_me_mask = 0ull; - shadow_host_writable_mask = EPT_SPTE_HOST_WRITABLE; shadow_mmu_writable_mask = EPT_SPTE_MMU_WRITABLE; @@ -449,7 +459,8 @@ void kvm_mmu_reset_all_pte_masks(void) shadow_x_mask = 0; shadow_present_mask = PT_PRESENT_MASK; shadow_acc_track_mask = 0; - shadow_me_mask = sme_me_mask; + shadow_me_mask = 0; + shadow_me_value = 0; shadow_host_writable_mask = DEFAULT_SPTE_HOST_WRITABLE; shadow_mmu_writable_mask = DEFAULT_SPTE_MMU_WRITABLE; diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h index 43f7924bc7f0..0127bb6e3c7d 100644 --- a/arch/x86/kvm/mmu/spte.h +++ b/arch/x86/kvm/mmu/spte.h @@ -151,6 +151,7 @@ extern u64 __read_mostly shadow_mmio_value; extern u64 __read_mostly shadow_mmio_mask; extern u64 __read_mostly shadow_mmio_access_mask; extern u64 __read_mostly shadow_present_mask; +extern u64 __read_mostly shadow_me_value; extern u64 __read_mostly shadow_me_mask; /* diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 75b4f3ac8b1a..3b49337998ec 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4892,6 +4892,9 @@ static __init int svm_hardware_setup(void) get_npt_level(), PG_LEVEL_1G); pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); + /* Setup shadow_me_value and shadow_me_mask */ + kvm_mmu_set_me_spte_mask(sme_me_mask, sme_me_mask); + /* Note, SEV setup consumes npt_enabled. */ sev_hardware_setup(); From 3c5c32457d7da76d077effc08176d3ad2e7536f9 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 19 Apr 2022 23:17:04 +1200 Subject: [PATCH 237/319] KVM: VMX: Include MKTME KeyID bits in shadow_zero_check Intel MKTME KeyID bits (including Intel TDX private KeyID bits) should never be set to SPTE. Set shadow_me_value to 0 and shadow_me_mask to include all MKTME KeyID bits to include them to shadow_zero_check. Signed-off-by: Kai Huang Message-Id: <27bc10e97a3c0b58a4105ff9107448c190328239.1650363789.git.kai.huang@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu.h | 19 +++++++++++++++++++ arch/x86/kvm/mmu/spte.c | 19 ------------------- arch/x86/kvm/vmx/vmx.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 19 deletions(-) diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index 1db11cd6b177..f8192864b496 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -89,6 +89,25 @@ static inline gfn_t kvm_mmu_max_gfn(void) return (1ULL << (max_gpa_bits - PAGE_SHIFT)) - 1; } +static inline u8 kvm_get_shadow_phys_bits(void) +{ + /* + * boot_cpu_data.x86_phys_bits is reduced when MKTME or SME are detected + * in CPU detection code, but the processor treats those reduced bits as + * 'keyID' thus they are not reserved bits. Therefore KVM needs to look at + * the physical address bits reported by CPUID. + */ + if (likely(boot_cpu_data.extended_cpuid_level >= 0x80000008)) + return cpuid_eax(0x80000008) & 0xff; + + /* + * Quite weird to have VMX or SVM but not MAXPHYADDR; probably a VM with + * custom CPUID. Proceed with whatever the kernel found since these features + * aren't virtualizable (SME/SEV also require CPUIDs higher than 0x80000008). + */ + return boot_cpu_data.x86_phys_bits; +} + void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask); void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask); void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only); diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c index 12b9b3f8bd05..b5960bbde7f7 100644 --- a/arch/x86/kvm/mmu/spte.c +++ b/arch/x86/kvm/mmu/spte.c @@ -311,25 +311,6 @@ u64 kvm_mmu_changed_pte_notifier_make_spte(u64 old_spte, kvm_pfn_t new_pfn) return new_spte; } -static u8 kvm_get_shadow_phys_bits(void) -{ - /* - * boot_cpu_data.x86_phys_bits is reduced when MKTME or SME are detected - * in CPU detection code, but the processor treats those reduced bits as - * 'keyID' thus they are not reserved bits. Therefore KVM needs to look at - * the physical address bits reported by CPUID. - */ - if (likely(boot_cpu_data.extended_cpuid_level >= 0x80000008)) - return cpuid_eax(0x80000008) & 0xff; - - /* - * Quite weird to have VMX or SVM but not MAXPHYADDR; probably a VM with - * custom CPUID. Proceed with whatever the kernel found since these features - * aren't virtualizable (SME/SEV also require CPUIDs higher than 0x80000008). - */ - return boot_cpu_data.x86_phys_bits; -} - u64 mark_spte_for_access_track(u64 spte) { if (spte_ad_enabled(spte)) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index cf8581978bce..5fb37e75fd31 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7883,6 +7883,31 @@ static __init void vmx_setup_user_return_msrs(void) kvm_add_user_return_msr(vmx_uret_msrs_list[i]); } +static void __init vmx_setup_me_spte_mask(void) +{ + u64 me_mask = 0; + + /* + * kvm_get_shadow_phys_bits() returns shadow_phys_bits. Use + * the former to avoid exposing shadow_phys_bits. + * + * On pre-MKTME system, boot_cpu_data.x86_phys_bits equals to + * shadow_phys_bits. On MKTME and/or TDX capable systems, + * boot_cpu_data.x86_phys_bits holds the actual physical address + * w/o the KeyID bits, and shadow_phys_bits equals to MAXPHYADDR + * reported by CPUID. Those bits between are KeyID bits. + */ + if (boot_cpu_data.x86_phys_bits != kvm_get_shadow_phys_bits()) + me_mask = rsvd_bits(boot_cpu_data.x86_phys_bits, + kvm_get_shadow_phys_bits() - 1); + /* + * Unlike SME, host kernel doesn't support setting up any + * MKTME KeyID on Intel platforms. No memory encryption + * bits should be included into the SPTE. + */ + kvm_mmu_set_me_spte_mask(0, me_mask); +} + static struct kvm_x86_init_ops vmx_init_ops __initdata; static __init int hardware_setup(void) @@ -7985,6 +8010,12 @@ static __init int hardware_setup(void) kvm_mmu_set_ept_masks(enable_ept_ad_bits, cpu_has_vmx_ept_execute_only()); + /* + * Setup shadow_me_value/shadow_me_mask to include MKTME KeyID + * bits to shadow_zero_check. + */ + vmx_setup_me_spte_mask(); + kvm_configure_mmu(enable_ept, 0, vmx_get_max_tdp_level(), ept_caps_to_lpage_level(vmx_capability.ept)); From 6ba1e04fa60787ced4c9049cf7bd6cd0f1b80764 Mon Sep 17 00:00:00 2001 From: Vipin Sharma Date: Mon, 2 May 2022 22:03:47 +0000 Subject: [PATCH 238/319] KVM: x86/mmu: Speed up slot_rmap_walk_next for sparsely populated rmaps Avoid calling handlers on empty rmap entries and skip to the next non empty rmap entry. Empty rmap entries are noop in handlers. Signed-off-by: Vipin Sharma Suggested-by: Sean Christopherson Message-Id: <20220502220347.174664-1-vipinsh@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index ea4bc085bbf5..efe5a3dca1e0 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1483,9 +1483,11 @@ static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator) static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator) { - if (++iterator->rmap <= iterator->end_rmap) { + while (++iterator->rmap <= iterator->end_rmap) { iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level)); - return; + + if (iterator->rmap->val) + return; } if (++iterator->level > iterator->end_level) { From 0a4cad9c11ad46662ede48d94f08ecb7cd9f6916 Mon Sep 17 00:00:00 2001 From: Enric Balletbo i Serra Date: Fri, 13 May 2022 12:52:09 +0500 Subject: [PATCH 239/319] platform/chrome: Add ChromeOS ACPI device driver The x86 Chromebooks have the ChromeOS ACPI device. This driver attaches to the ChromeOS ACPI device and exports the values reported by ACPI in a sysfs directory. This data isn't present in ACPI tables when read through ACPI tools, hence a driver is needed to do it. The driver gets data from firmware using the ACPI component of the kernel. The ACPI values are presented in string form (numbers as decimal values) or binary blobs, and can be accessed as the contents of the appropriate read only files in the standard ACPI device's sysfs directory tree. This data is consumed by the ChromeOS user space. Reviewed-by: Guenter Roeck Reviewed-by: Andy Shevchenko Reviewed-by: Greg Kroah-Hartman Acked-by: Rafael J. Wysocki Signed-off-by: Enric Balletbo i Serra Co-developed-by: Muhammad Usama Anjum Signed-off-by: Muhammad Usama Anjum Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/Yn4OKYrtV35Dv+nd@debian-BULLSEYE-live-builder-AMD64 --- .../ABI/testing/sysfs-driver-chromeos-acpi | 126 ++++++ .../acpi/chromeos-acpi-device.rst | 363 ++++++++++++++++++ Documentation/firmware-guide/acpi/index.rst | 1 + drivers/platform/chrome/Kconfig | 11 + drivers/platform/chrome/Makefile | 1 + drivers/platform/chrome/chromeos_acpi.c | 257 +++++++++++++ 6 files changed, 759 insertions(+) create mode 100644 Documentation/ABI/testing/sysfs-driver-chromeos-acpi create mode 100644 Documentation/firmware-guide/acpi/chromeos-acpi-device.rst create mode 100644 drivers/platform/chrome/chromeos_acpi.c diff --git a/Documentation/ABI/testing/sysfs-driver-chromeos-acpi b/Documentation/ABI/testing/sysfs-driver-chromeos-acpi new file mode 100644 index 000000000000..5b59ef9d7b37 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-chromeos-acpi @@ -0,0 +1,126 @@ +What: /sys/bus/platform/devices/GGL0001:*/BINF.2 +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows information about the current boot of + the active EC firmware. + * 0 - Read only (recovery) firmware. + * 1 - Rewritable firmware. + +What: /sys/bus/platform/devices/GGL0001:*/BINF.3 +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows information about the current boot of + the active main firmware type. + * 0 - Recovery. + * 1 - Normal. + * 2 - Developer. + * 3 - Netboot (factory installation only). + +What: /sys/bus/platform/devices/GGL0001:*/CHSW +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the switch position for the Chrome OS specific + hardware switches. + * 0 - No changes. + * 2 - Recovery button was pressed when firmware booted. + * 4 - Recovery button was pressed when EC firmware booted. + * 32 - Developer switch was enabled when firmware booted. + * 512 - Firmware write protection was disabled when firmware + booted. + +What: /sys/bus/platform/devices/GGL0001:*/FMAP +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the physical memory address of the start of + the main processor firmware flashmap. + +What: /sys/bus/platform/devices/GGL0001:*/FRID +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the firmware version for the read-only portion + of the main processor firmware. + +What: /sys/bus/platform/devices/GGL0001:*/FWID +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the firmware version for the rewritable portion + of the main processor firmware. + +What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.0 +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the type of the GPIO signal for the Chrome OS + specific GPIO assignments. + * 1 - Recovery button. + * 2 - Developer mode switch. + * 3 - Firmware write protection switch. + * 256 to 511 - Debug header GPIO 0 to GPIO 255. + +What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.1 +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the signal attributes of the GPIO signal. + * 0 - Signal is active low. + * 1 - Signal is active high. + +What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.2 +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the GPIO number on the specified GPIO + controller. + +What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.3 +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the name of the GPIO controller. + +What: /sys/bus/platform/devices/GGL0001:*/HWID +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the hardware ID for the Chromebook. + +What: /sys/bus/platform/devices/GGL0001:*/MECK +Date: May 2022 +KernelVersion: 5.19 +Description: + This binary file returns the SHA-1 or SHA-256 hash that is + read out of the Management Engine extended registers during + boot. The hash is exported vi ACPI so the OS can verify that + the Management Engine firmware has not changed. If Management + Engine is not present, or if the firmware was unable to read the + extended registers, this buffer size can be zero. + +What: /sys/bus/platform/devices/GGL0001:*/VBNV.0 +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the offset in CMOS bank 0 of the verified boot + non-volatile storage block, counting from the first writable + CMOS byte (that is, 'offset = 0' is the byte following the 14 + bytes of clock data). + +What: /sys/bus/platform/devices/GGL0001:*/VBNV.1 +Date: May 2022 +KernelVersion: 5.19 +Description: + This file shows the size in bytes of the verified boot + non-volatile storage block. + +What: /sys/bus/platform/devices/GGL0001:*/VDAT +Date: May 2022 +KernelVersion: 5.19 +Description: + This binary file returns the verified boot data block shared + between the firmware verification step and the kernel + verification step. diff --git a/Documentation/firmware-guide/acpi/chromeos-acpi-device.rst b/Documentation/firmware-guide/acpi/chromeos-acpi-device.rst new file mode 100644 index 000000000000..f37fc90ce340 --- /dev/null +++ b/Documentation/firmware-guide/acpi/chromeos-acpi-device.rst @@ -0,0 +1,363 @@ +.. SPDX-License-Identifier: GPL-2.0 + +===================== +Chrome OS ACPI Device +===================== + +Hardware functionality specific to Chrome OS is exposed through a Chrome OS ACPI device. +The plug and play ID of a Chrome OS ACPI device is GGL0001. GGL is a valid PNP ID of Google. +PNP ID can be used with the ACPI devices according to the guidelines. The following ACPI +objects are supported: + +.. flat-table:: Supported ACPI Objects + :widths: 1 2 + :header-rows: 1 + + * - Object + - Description + + * - CHSW + - Chrome OS switch positions + + * - HWID + - Chrome OS hardware ID + + * - FWID + - Chrome OS firmware version + + * - FRID + - Chrome OS read-only firmware version + + * - BINF + - Chrome OS boot information + + * - GPIO + - Chrome OS GPIO assignments + + * - VBNV + - Chrome OS NVRAM locations + + * - VDTA + - Chrome OS verified boot data + + * - FMAP + - Chrome OS flashmap base address + + * - MLST + - Chrome OS method list + +CHSW (Chrome OS switch positions) +================================= +This control method returns the switch positions for Chrome OS specific hardware switches. + +Arguments: +---------- +None + +Result code: +------------ +An integer containing the switch positions as bitfields: + +.. flat-table:: + :widths: 1 2 + + * - 0x00000002 + - Recovery button was pressed when x86 firmware booted. + + * - 0x00000004 + - Recovery button was pressed when EC firmware booted. (required if EC EEPROM is + rewritable; otherwise optional) + + * - 0x00000020 + - Developer switch was enabled when x86 firmware booted. + + * - 0x00000200 + - Firmware write protection was disabled when x86 firmware booted. (required if + firmware write protection is controlled through x86 BIOS; otherwise optional) + +All other bits are reserved and should be set to 0. + +HWID (Chrome OS hardware ID) +============================ +This control method returns the hardware ID for the Chromebook. + +Arguments: +---------- +None + +Result code: +------------ +A null-terminated ASCII string containing the hardware ID from the Model-Specific Data area of +EEPROM. + +Note that the hardware ID can be up to 256 characters long, including the terminating null. + +FWID (Chrome OS firmware version) +================================= +This control method returns the firmware version for the rewritable portion of the main +processor firmware. + +Arguments: +---------- +None + +Result code: +------------ +A null-terminated ASCII string containing the complete firmware version for the rewritable +portion of the main processor firmware. + +FRID (Chrome OS read-only firmware version) +=========================================== +This control method returns the firmware version for the read-only portion of the main +processor firmware. + +Arguments: +---------- +None + +Result code: +------------ +A null-terminated ASCII string containing the complete firmware version for the read-only +(bootstrap + recovery ) portion of the main processor firmware. + +BINF (Chrome OS boot information) +================================= +This control method returns information about the current boot. + +Arguments: +---------- +None + +Result code: +------------ + +.. code-block:: + + Package { + Reserved1 + Reserved2 + Active EC Firmware + Active Main Firmware Type + Reserved5 + } + +.. flat-table:: + :widths: 1 1 2 + :header-rows: 1 + + * - Field + - Format + - Description + + * - Reserved1 + - DWORD + - Set to 256 (0x100). This indicates this field is no longer used. + + * - Reserved2 + - DWORD + - Set to 256 (0x100). This indicates this field is no longer used. + + * - Active EC firmware + - DWORD + - The EC firmware which was used during boot. + + - 0 - Read-only (recovery) firmware + - 1 - Rewritable firmware. + + Set to 0 if EC firmware is always read-only. + + * - Active Main Firmware Type + - DWORD + - The main firmware type which was used during boot. + + - 0 - Recovery + - 1 - Normal + - 2 - Developer + - 3 - netboot (factory installation only) + + Other values are reserved. + + * - Reserved5 + - DWORD + - Set to 256 (0x100). This indicates this field is no longer used. + +GPIO (Chrome OS GPIO assignments) +================================= +This control method returns information about Chrome OS specific GPIO assignments for +Chrome OS hardware, so the kernel can directly control that hardware. + +Arguments: +---------- +None + +Result code: +------------ +.. code-block:: + + Package { + Package { + // First GPIO assignment + Signal Type //DWORD + Attributes //DWORD + Controller Offset //DWORD + Controller Name //ASCIIZ + }, + ... + Package { + // Last GPIO assignment + Signal Type //DWORD + Attributes //DWORD + Controller Offset //DWORD + Controller Name //ASCIIZ + } + } + +Where ASCIIZ means a null-terminated ASCII string. + +.. flat-table:: + :widths: 1 1 2 + :header-rows: 1 + + * - Field + - Format + - Description + + * - Signal Type + - DWORD + - Type of GPIO signal + + - 0x00000001 - Recovery button + - 0x00000002 - Developer mode switch + - 0x00000003 - Firmware write protection switch + - 0x00000100 - Debug header GPIO 0 + - ... + - 0x000001FF - Debug header GPIO 255 + + Other values are reserved. + + * - Attributes + - DWORD + - Signal attributes as bitfields: + + - 0x00000001 - Signal is active-high (for button, a GPIO value + of 1 means the button is pressed; for switches, a GPIO value + of 1 means the switch is enabled). If this bit is 0, the signal + is active low. Set to 0 for debug header GPIOs. + + * - Controller Offset + - DWORD + - GPIO number on the specified controller. + + * - Controller Name + - ASCIIZ + - Name of the controller for the GPIO. + Currently supported names: + "NM10" - Intel NM10 chip + +VBNV (Chrome OS NVRAM locations) +================================ +This control method returns information about the NVRAM (CMOS) locations used to +communicate with the BIOS. + +Arguments: +---------- +None + +Result code: +------------ +.. code-block:: + + Package { + NV Storage Block Offset //DWORD + NV Storage Block Size //DWORD + } + +.. flat-table:: + :widths: 1 1 2 + :header-rows: 1 + + * - Field + - Format + - Description + + * - NV Storage Block Offset + - DWORD + - Offset in CMOS bank 0 of the verified boot non-volatile storage block, counting from + the first writable CMOS byte (that is, offset=0 is the byte following the 14 bytes of + clock data). + + * - NV Storage Block Size + - DWORD + - Size in bytes of the verified boot non-volatile storage block. + +FMAP (Chrome OS flashmap address) +================================= +This control method returns the physical memory address of the start of the main processor +firmware flashmap. + +Arguments: +---------- +None + +NoneResult code: +---------------- +A DWORD containing the physical memory address of the start of the main processor firmware +flashmap. + +VDTA (Chrome OS verified boot data) +=================================== +This control method returns the verified boot data block shared between the firmware +verification step and the kernel verification step. + +Arguments: +---------- +None + +Result code: +------------ +A buffer containing the verified boot data block. + +MECK (Management Engine Checksum) +================================= +This control method returns the SHA-1 or SHA-256 hash that is read out of the Management +Engine extended registers during boot. The hash is exported via ACPI so the OS can verify that +the ME firmware has not changed. If Management Engine is not present, or if the firmware was +unable to read the extended registers, this buffer can be zero. + +Arguments: +---------- +None + +Result code: +------------ +A buffer containing the ME hash. + +MLST (Chrome OS method list) +============================ +This control method returns a list of the other control methods supported by the Chrome OS +hardware device. + +Arguments: +---------- +None + +Result code: +------------ +A package containing a list of null-terminated ASCII strings, one for each control method +supported by the Chrome OS hardware device, not including the MLST method itself. +For this version of the specification, the result is: + +.. code-block:: + + Package { + "CHSW", + "FWID", + "HWID", + "FRID", + "BINF", + "GPIO", + "VBNV", + "FMAP", + "VDTA", + "MECK" + } diff --git a/Documentation/firmware-guide/acpi/index.rst b/Documentation/firmware-guide/acpi/index.rst index b053b0c3d696..b6a42f4ffe03 100644 --- a/Documentation/firmware-guide/acpi/index.rst +++ b/Documentation/firmware-guide/acpi/index.rst @@ -29,3 +29,4 @@ ACPI Support non-d0-probe extcon-intel-int3496 intel-pmc-mux + chromeos-acpi-device diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig index 75e93efd669f..717299cbccac 100644 --- a/drivers/platform/chrome/Kconfig +++ b/drivers/platform/chrome/Kconfig @@ -15,6 +15,17 @@ menuconfig CHROME_PLATFORMS if CHROME_PLATFORMS +config CHROMEOS_ACPI + tristate "ChromeOS specific ACPI extensions" + depends on ACPI + help + This driver provides the firmware interface for the services + exported through the ChromeOS interfaces when using ChromeOS + ACPI firmware. + + If you have an ACPI-compatible Chromebook, say Y or M here. + The module will be called chromeos_acpi. + config CHROMEOS_LAPTOP tristate "Chrome OS Laptop" depends on I2C && DMI && X86 diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile index 6420ca129548..52f5a2dde8b8 100644 --- a/drivers/platform/chrome/Makefile +++ b/drivers/platform/chrome/Makefile @@ -4,6 +4,7 @@ CFLAGS_cros_ec_trace.o:= -I$(src) CFLAGS_cros_ec_sensorhub_ring.o:= -I$(src) +obj-$(CONFIG_CHROMEOS_ACPI) += chromeos_acpi.o obj-$(CONFIG_CHROMEOS_LAPTOP) += chromeos_laptop.o obj-$(CONFIG_CHROMEOS_PRIVACY_SCREEN) += chromeos_privacy_screen.o obj-$(CONFIG_CHROMEOS_PSTORE) += chromeos_pstore.o diff --git a/drivers/platform/chrome/chromeos_acpi.c b/drivers/platform/chrome/chromeos_acpi.c new file mode 100644 index 000000000000..50d8a4d4352d --- /dev/null +++ b/drivers/platform/chrome/chromeos_acpi.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ChromeOS specific ACPI extensions + * + * Copyright 2022 Google LLC + * + * This driver attaches to the ChromeOS ACPI device and then exports the + * values reported by the ACPI in a sysfs directory. All values are + * presented in the string form (numbers as decimal values) and can be + * accessed as the contents of the appropriate read only files in the + * sysfs directory tree. + */ +#include +#include +#include +#include +#include + +#define ACPI_ATTR_NAME_LEN 4 + +#define DEV_ATTR(_var, _name) \ + static struct device_attribute dev_attr_##_var = \ + __ATTR(_name, 0444, chromeos_first_level_attr_show, NULL); + +#define GPIO_ATTR_GROUP(_group, _name, _num) \ + static umode_t attr_is_visible_gpio_##_num(struct kobject *kobj, \ + struct attribute *attr, int n) \ + { \ + if (_num < chromeos_acpi_gpio_groups) \ + return attr->mode; \ + return 0; \ + } \ + static ssize_t chromeos_attr_show_gpio_##_num(struct device *dev, \ + struct device_attribute *attr, \ + char *buf) \ + { \ + char name[ACPI_ATTR_NAME_LEN + 1]; \ + int ret, num; \ + \ + ret = parse_attr_name(attr->attr.name, name, &num); \ + if (ret) \ + return ret; \ + return chromeos_acpi_evaluate_method(dev, _num, num, name, buf); \ + } \ + static struct device_attribute dev_attr_0_##_group = \ + __ATTR(GPIO.0, 0444, chromeos_attr_show_gpio_##_num, NULL); \ + static struct device_attribute dev_attr_1_##_group = \ + __ATTR(GPIO.1, 0444, chromeos_attr_show_gpio_##_num, NULL); \ + static struct device_attribute dev_attr_2_##_group = \ + __ATTR(GPIO.2, 0444, chromeos_attr_show_gpio_##_num, NULL); \ + static struct device_attribute dev_attr_3_##_group = \ + __ATTR(GPIO.3, 0444, chromeos_attr_show_gpio_##_num, NULL); \ + \ + static struct attribute *attrs_##_group[] = { \ + &dev_attr_0_##_group.attr, \ + &dev_attr_1_##_group.attr, \ + &dev_attr_2_##_group.attr, \ + &dev_attr_3_##_group.attr, \ + NULL \ + }; \ + static const struct attribute_group attr_group_##_group = { \ + .name = _name, \ + .is_visible = attr_is_visible_gpio_##_num, \ + .attrs = attrs_##_group, \ + }; + +static unsigned int chromeos_acpi_gpio_groups; + +/* Parse the ACPI package and return the data related to that attribute */ +static int chromeos_acpi_handle_package(struct device *dev, union acpi_object *obj, + int pkg_num, int sub_pkg_num, char *name, char *buf) +{ + union acpi_object *element = obj->package.elements; + + if (pkg_num >= obj->package.count) + return -EINVAL; + element += pkg_num; + + if (element->type == ACPI_TYPE_PACKAGE) { + if (sub_pkg_num >= element->package.count) + return -EINVAL; + /* select sub element inside this package */ + element = element->package.elements; + element += sub_pkg_num; + } + + switch (element->type) { + case ACPI_TYPE_INTEGER: + return sysfs_emit(buf, "%d\n", (int)element->integer.value); + case ACPI_TYPE_STRING: + return sysfs_emit(buf, "%s\n", element->string.pointer); + case ACPI_TYPE_BUFFER: + return sysfs_emit(buf, "%s\n", element->buffer.pointer); + default: + dev_err(dev, "element type %d not supported\n", element->type); + return -EINVAL; + } +} + +static int chromeos_acpi_evaluate_method(struct device *dev, int pkg_num, int sub_pkg_num, + char *name, char *buf) +{ + struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; + acpi_status status; + int ret = -EINVAL; + + status = acpi_evaluate_object(ACPI_HANDLE(dev), name, NULL, &output); + if (ACPI_FAILURE(status)) { + dev_err(dev, "failed to retrieve %s. %s\n", name, acpi_format_exception(status)); + return ret; + } + + if (((union acpi_object *)output.pointer)->type == ACPI_TYPE_PACKAGE) + ret = chromeos_acpi_handle_package(dev, output.pointer, pkg_num, sub_pkg_num, + name, buf); + + kfree(output.pointer); + return ret; +} + +static int parse_attr_name(const char *name, char *attr_name, int *attr_num) +{ + int ret; + + ret = strscpy(attr_name, name, ACPI_ATTR_NAME_LEN + 1); + if (ret == -E2BIG) + return kstrtoint(&name[ACPI_ATTR_NAME_LEN + 1], 0, attr_num); + return 0; +} + +static ssize_t chromeos_first_level_attr_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + char attr_name[ACPI_ATTR_NAME_LEN + 1]; + int ret, attr_num = 0; + + ret = parse_attr_name(attr->attr.name, attr_name, &attr_num); + if (ret) + return ret; + return chromeos_acpi_evaluate_method(dev, attr_num, 0, attr_name, buf); +} + +static unsigned int get_gpio_pkg_num(struct device *dev) +{ + struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *obj; + acpi_status status; + unsigned int count = 0; + char *name = "GPIO"; + + status = acpi_evaluate_object(ACPI_HANDLE(dev), name, NULL, &output); + if (ACPI_FAILURE(status)) { + dev_err(dev, "failed to retrieve %s. %s\n", name, acpi_format_exception(status)); + return count; + } + + obj = output.pointer; + + if (obj->type == ACPI_TYPE_PACKAGE) + count = obj->package.count; + + kfree(output.pointer); + return count; +} + +DEV_ATTR(binf2, BINF.2) +DEV_ATTR(binf3, BINF.3) +DEV_ATTR(chsw, CHSW) +DEV_ATTR(fmap, FMAP) +DEV_ATTR(frid, FRID) +DEV_ATTR(fwid, FWID) +DEV_ATTR(hwid, HWID) +DEV_ATTR(meck, MECK) +DEV_ATTR(vbnv0, VBNV.0) +DEV_ATTR(vbnv1, VBNV.1) +DEV_ATTR(vdat, VDAT) + +static struct attribute *first_level_attrs[] = { + &dev_attr_binf2.attr, + &dev_attr_binf3.attr, + &dev_attr_chsw.attr, + &dev_attr_fmap.attr, + &dev_attr_frid.attr, + &dev_attr_fwid.attr, + &dev_attr_hwid.attr, + &dev_attr_meck.attr, + &dev_attr_vbnv0.attr, + &dev_attr_vbnv1.attr, + &dev_attr_vdat.attr, + NULL +}; + +static const struct attribute_group first_level_attr_group = { + .attrs = first_level_attrs, +}; + +/* + * Every platform can have a different number of GPIO attribute groups. + * Define upper limit groups. At run time, the platform decides to show + * the present number of groups only, others are hidden. + */ +GPIO_ATTR_GROUP(gpio0, "GPIO.0", 0) +GPIO_ATTR_GROUP(gpio1, "GPIO.1", 1) +GPIO_ATTR_GROUP(gpio2, "GPIO.2", 2) +GPIO_ATTR_GROUP(gpio3, "GPIO.3", 3) +GPIO_ATTR_GROUP(gpio4, "GPIO.4", 4) +GPIO_ATTR_GROUP(gpio5, "GPIO.5", 5) +GPIO_ATTR_GROUP(gpio6, "GPIO.6", 6) +GPIO_ATTR_GROUP(gpio7, "GPIO.7", 7) + +static const struct attribute_group *chromeos_acpi_all_groups[] = { + &first_level_attr_group, + &attr_group_gpio0, + &attr_group_gpio1, + &attr_group_gpio2, + &attr_group_gpio3, + &attr_group_gpio4, + &attr_group_gpio5, + &attr_group_gpio6, + &attr_group_gpio7, + NULL +}; + +static int chromeos_acpi_device_probe(struct platform_device *pdev) +{ + chromeos_acpi_gpio_groups = get_gpio_pkg_num(&pdev->dev); + + /* + * If the platform has more GPIO attribute groups than the number of + * groups this driver supports, give out a warning message. + */ + if (chromeos_acpi_gpio_groups > ARRAY_SIZE(chromeos_acpi_all_groups) - 2) + dev_warn(&pdev->dev, "Only %zu GPIO attr groups supported by the driver out of total %u.\n", + ARRAY_SIZE(chromeos_acpi_all_groups) - 2, chromeos_acpi_gpio_groups); + return 0; +} + +/* GGL is valid PNP ID of Google. PNP ID can be used with the ACPI devices. */ +static const struct acpi_device_id chromeos_device_ids[] = { + { "GGL0001", 0 }, + {} +}; +MODULE_DEVICE_TABLE(acpi, chromeos_device_ids); + +static struct platform_driver chromeos_acpi_device_driver = { + .probe = chromeos_acpi_device_probe, + .driver = { + .name = KBUILD_MODNAME, + .dev_groups = chromeos_acpi_all_groups, + .acpi_match_table = chromeos_device_ids, + } +}; +module_platform_driver(chromeos_acpi_device_driver); + +MODULE_AUTHOR("Muhammad Usama Anjum "); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ChromeOS specific ACPI extensions"); From cf8f4462e5fa6b3b0c7f3a7543473d0dfdd8d06a Mon Sep 17 00:00:00 2001 From: Moses Christopher Bollavarapu Date: Thu, 12 May 2022 09:14:15 +0200 Subject: [PATCH 240/319] gpio: zevio: drop of_gpio.h header Remove of_gpio.h header file, replace of_* functions and structs with appropriate alternatives. Signed-off-by: Moses Christopher Bollavarapu Reviewed-by: Andy Shevchenko Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-zevio.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/gpio/gpio-zevio.c b/drivers/gpio/gpio-zevio.c index f6f8a541348f..ce9d1282165c 100644 --- a/drivers/gpio/gpio-zevio.c +++ b/drivers/gpio/gpio-zevio.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -53,22 +52,23 @@ #define ZEVIO_GPIO_BIT(gpio) (gpio&7) struct zevio_gpio { + struct gpio_chip chip; spinlock_t lock; - struct of_mm_gpio_chip chip; + void __iomem *regs; }; static inline u32 zevio_gpio_port_get(struct zevio_gpio *c, unsigned pin, unsigned port_offset) { unsigned section_offset = ((pin >> 3) & 3)*ZEVIO_GPIO_SECTION_SIZE; - return readl(IOMEM(c->chip.regs + section_offset + port_offset)); + return readl(IOMEM(c->regs + section_offset + port_offset)); } static inline void zevio_gpio_port_set(struct zevio_gpio *c, unsigned pin, unsigned port_offset, u32 val) { unsigned section_offset = ((pin >> 3) & 3)*ZEVIO_GPIO_SECTION_SIZE; - writel(val, IOMEM(c->chip.regs + section_offset + port_offset)); + writel(val, IOMEM(c->regs + section_offset + port_offset)); } /* Functions for struct gpio_chip */ @@ -178,12 +178,15 @@ static int zevio_gpio_probe(struct platform_device *pdev) platform_set_drvdata(pdev, controller); /* Copy our reference */ - controller->chip.gc = zevio_gpio_chip; - controller->chip.gc.parent = &pdev->dev; + controller->chip = zevio_gpio_chip; + controller->chip.parent = &pdev->dev; - status = of_mm_gpiochip_add_data(pdev->dev.of_node, - &(controller->chip), - controller); + controller->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(controller->regs)) + return dev_err_probe(&pdev->dev, PTR_ERR(controller->regs), + "failed to ioremap memory resource\n"); + + status = devm_gpiochip_add_data(&pdev->dev, &controller->chip, controller); if (status) { dev_err(&pdev->dev, "failed to add gpiochip: %d\n", status); return status; @@ -192,10 +195,10 @@ static int zevio_gpio_probe(struct platform_device *pdev) spin_lock_init(&controller->lock); /* Disable interrupts, they only cause errors */ - for (i = 0; i < controller->chip.gc.ngpio; i += 8) + for (i = 0; i < controller->chip.ngpio; i += 8) zevio_gpio_port_set(controller, i, ZEVIO_GPIO_INT_MASK, 0xFF); - dev_dbg(controller->chip.gc.parent, "ZEVIO GPIO controller set up!\n"); + dev_dbg(controller->chip.parent, "ZEVIO GPIO controller set up!\n"); return 0; } From e993e236058981aceee0518ad17e11f3d16112f8 Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Tue, 10 May 2022 13:30:54 -0400 Subject: [PATCH 241/319] gpio: 104-dio-48e: Utilize iomap interface This driver doesn't need to access I/O ports directly via inb()/outb() and friends. This patch abstracts such access by calling ioport_map() to enable the use of more typical ioread8()/iowrite8() I/O memory accessor calls. Suggested-by: David Laight Signed-off-by: William Breathitt Gray Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-104-dio-48e.c | 63 +++++++++++++++++---------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/drivers/gpio/gpio-104-dio-48e.c b/drivers/gpio/gpio-104-dio-48e.c index 6bf41040c41f..f118ad9bcd33 100644 --- a/drivers/gpio/gpio-104-dio-48e.c +++ b/drivers/gpio/gpio-104-dio-48e.c @@ -49,7 +49,7 @@ struct dio48e_gpio { unsigned char out_state[6]; unsigned char control[2]; raw_spinlock_t lock; - unsigned int base; + void __iomem *base; unsigned char irq_mask; }; @@ -70,7 +70,7 @@ static int dio48e_gpio_direction_input(struct gpio_chip *chip, unsigned int offs struct dio48e_gpio *const dio48egpio = gpiochip_get_data(chip); const unsigned int io_port = offset / 8; const unsigned int control_port = io_port / 3; - const unsigned int control_addr = dio48egpio->base + 3 + control_port * 4; + void __iomem *const control_addr = dio48egpio->base + 3 + control_port * 4; unsigned long flags; unsigned int control; @@ -95,9 +95,9 @@ static int dio48e_gpio_direction_input(struct gpio_chip *chip, unsigned int offs } control = BIT(7) | dio48egpio->control[control_port]; - outb(control, control_addr); + iowrite8(control, control_addr); control &= ~BIT(7); - outb(control, control_addr); + iowrite8(control, control_addr); raw_spin_unlock_irqrestore(&dio48egpio->lock, flags); @@ -111,7 +111,7 @@ static int dio48e_gpio_direction_output(struct gpio_chip *chip, unsigned int off const unsigned int io_port = offset / 8; const unsigned int control_port = io_port / 3; const unsigned int mask = BIT(offset % 8); - const unsigned int control_addr = dio48egpio->base + 3 + control_port * 4; + void __iomem *const control_addr = dio48egpio->base + 3 + control_port * 4; const unsigned int out_port = (io_port > 2) ? io_port + 1 : io_port; unsigned long flags; unsigned int control; @@ -142,12 +142,12 @@ static int dio48e_gpio_direction_output(struct gpio_chip *chip, unsigned int off dio48egpio->out_state[io_port] &= ~mask; control = BIT(7) | dio48egpio->control[control_port]; - outb(control, control_addr); + iowrite8(control, control_addr); - outb(dio48egpio->out_state[io_port], dio48egpio->base + out_port); + iowrite8(dio48egpio->out_state[io_port], dio48egpio->base + out_port); control &= ~BIT(7); - outb(control, control_addr); + iowrite8(control, control_addr); raw_spin_unlock_irqrestore(&dio48egpio->lock, flags); @@ -171,7 +171,7 @@ static int dio48e_gpio_get(struct gpio_chip *chip, unsigned int offset) return -EINVAL; } - port_state = inb(dio48egpio->base + in_port); + port_state = ioread8(dio48egpio->base + in_port); raw_spin_unlock_irqrestore(&dio48egpio->lock, flags); @@ -186,7 +186,7 @@ static int dio48e_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask, struct dio48e_gpio *const dio48egpio = gpiochip_get_data(chip); unsigned long offset; unsigned long gpio_mask; - unsigned int port_addr; + void __iomem *port_addr; unsigned long port_state; /* clear bits array to a clean slate */ @@ -194,7 +194,7 @@ static int dio48e_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask, for_each_set_clump8(offset, gpio_mask, mask, ARRAY_SIZE(ports) * 8) { port_addr = dio48egpio->base + ports[offset / 8]; - port_state = inb(port_addr) & gpio_mask; + port_state = ioread8(port_addr) & gpio_mask; bitmap_set_value8(bits, port_state, offset); } @@ -217,7 +217,7 @@ static void dio48e_gpio_set(struct gpio_chip *chip, unsigned int offset, int val else dio48egpio->out_state[port] &= ~mask; - outb(dio48egpio->out_state[port], dio48egpio->base + out_port); + iowrite8(dio48egpio->out_state[port], dio48egpio->base + out_port); raw_spin_unlock_irqrestore(&dio48egpio->lock, flags); } @@ -229,7 +229,7 @@ static void dio48e_gpio_set_multiple(struct gpio_chip *chip, unsigned long offset; unsigned long gpio_mask; size_t index; - unsigned int port_addr; + void __iomem *port_addr; unsigned long bitmask; unsigned long flags; @@ -244,7 +244,7 @@ static void dio48e_gpio_set_multiple(struct gpio_chip *chip, /* update output state data and set device gpio register */ dio48egpio->out_state[index] &= ~gpio_mask; dio48egpio->out_state[index] |= bitmask; - outb(dio48egpio->out_state[index], port_addr); + iowrite8(dio48egpio->out_state[index], port_addr); raw_spin_unlock_irqrestore(&dio48egpio->lock, flags); } @@ -274,7 +274,7 @@ static void dio48e_irq_mask(struct irq_data *data) if (!dio48egpio->irq_mask) /* disable interrupts */ - inb(dio48egpio->base + 0xB); + ioread8(dio48egpio->base + 0xB); raw_spin_unlock_irqrestore(&dio48egpio->lock, flags); } @@ -294,8 +294,8 @@ static void dio48e_irq_unmask(struct irq_data *data) if (!dio48egpio->irq_mask) { /* enable interrupts */ - outb(0x00, dio48egpio->base + 0xF); - outb(0x00, dio48egpio->base + 0xB); + iowrite8(0x00, dio48egpio->base + 0xF); + iowrite8(0x00, dio48egpio->base + 0xB); } if (offset == 19) @@ -341,7 +341,7 @@ static irqreturn_t dio48e_irq_handler(int irq, void *dev_id) raw_spin_lock(&dio48egpio->lock); - outb(0x00, dio48egpio->base + 0xF); + iowrite8(0x00, dio48egpio->base + 0xF); raw_spin_unlock(&dio48egpio->lock); @@ -373,7 +373,7 @@ static int dio48e_irq_init_hw(struct gpio_chip *gc) struct dio48e_gpio *const dio48egpio = gpiochip_get_data(gc); /* Disable IRQ by default */ - inb(dio48egpio->base + 0xB); + ioread8(dio48egpio->base + 0xB); return 0; } @@ -395,6 +395,10 @@ static int dio48e_probe(struct device *dev, unsigned int id) return -EBUSY; } + dio48egpio->base = devm_ioport_map(dev, base[id], DIO48E_EXTENT); + if (!dio48egpio->base) + return -ENOMEM; + dio48egpio->chip.label = name; dio48egpio->chip.parent = dev; dio48egpio->chip.owner = THIS_MODULE; @@ -408,7 +412,6 @@ static int dio48e_probe(struct device *dev, unsigned int id) dio48egpio->chip.get_multiple = dio48e_gpio_get_multiple; dio48egpio->chip.set = dio48e_gpio_set; dio48egpio->chip.set_multiple = dio48e_gpio_set_multiple; - dio48egpio->base = base[id]; girq = &dio48egpio->chip.irq; girq->chip = &dio48e_irqchip; @@ -423,16 +426,16 @@ static int dio48e_probe(struct device *dev, unsigned int id) raw_spin_lock_init(&dio48egpio->lock); /* initialize all GPIO as output */ - outb(0x80, base[id] + 3); - outb(0x00, base[id]); - outb(0x00, base[id] + 1); - outb(0x00, base[id] + 2); - outb(0x00, base[id] + 3); - outb(0x80, base[id] + 7); - outb(0x00, base[id] + 4); - outb(0x00, base[id] + 5); - outb(0x00, base[id] + 6); - outb(0x00, base[id] + 7); + iowrite8(0x80, dio48egpio->base + 3); + iowrite8(0x00, dio48egpio->base); + iowrite8(0x00, dio48egpio->base + 1); + iowrite8(0x00, dio48egpio->base + 2); + iowrite8(0x00, dio48egpio->base + 3); + iowrite8(0x80, dio48egpio->base + 7); + iowrite8(0x00, dio48egpio->base + 4); + iowrite8(0x00, dio48egpio->base + 5); + iowrite8(0x00, dio48egpio->base + 6); + iowrite8(0x00, dio48egpio->base + 7); err = devm_gpiochip_add_data(dev, &dio48egpio->chip, dio48egpio); if (err) { From bed58069905dadc19e572131d97c4abb2b86893a Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Tue, 10 May 2022 13:30:55 -0400 Subject: [PATCH 242/319] gpio: 104-idi-48: Utilize iomap interface This driver doesn't need to access I/O ports directly via inb()/outb() and friends. This patch abstracts such access by calling ioport_map() to enable the use of more typical ioread8()/iowrite8() I/O memory accessor calls. Suggested-by: David Laight Signed-off-by: William Breathitt Gray Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-104-idi-48.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/drivers/gpio/gpio-104-idi-48.c b/drivers/gpio/gpio-104-idi-48.c index 34be7dd9f5b9..9521ece3ebef 100644 --- a/drivers/gpio/gpio-104-idi-48.c +++ b/drivers/gpio/gpio-104-idi-48.c @@ -47,7 +47,7 @@ struct idi_48_gpio { raw_spinlock_t lock; spinlock_t ack_lock; unsigned char irq_mask[6]; - unsigned base; + void __iomem *base; unsigned char cos_enb; }; @@ -66,15 +66,15 @@ static int idi_48_gpio_get(struct gpio_chip *chip, unsigned offset) struct idi_48_gpio *const idi48gpio = gpiochip_get_data(chip); unsigned i; static const unsigned int register_offset[6] = { 0, 1, 2, 4, 5, 6 }; - unsigned base_offset; + void __iomem *port_addr; unsigned mask; for (i = 0; i < 48; i += 8) if (offset < i + 8) { - base_offset = register_offset[i / 8]; + port_addr = idi48gpio->base + register_offset[i / 8]; mask = BIT(offset - i); - return !!(inb(idi48gpio->base + base_offset) & mask); + return !!(ioread8(port_addr) & mask); } /* The following line should never execute since offset < 48 */ @@ -88,7 +88,7 @@ static int idi_48_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask, unsigned long offset; unsigned long gpio_mask; static const size_t ports[] = { 0, 1, 2, 4, 5, 6 }; - unsigned int port_addr; + void __iomem *port_addr; unsigned long port_state; /* clear bits array to a clean slate */ @@ -96,7 +96,7 @@ static int idi_48_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask, for_each_set_clump8(offset, gpio_mask, mask, ARRAY_SIZE(ports) * 8) { port_addr = idi48gpio->base + ports[offset / 8]; - port_state = inb(port_addr) & gpio_mask; + port_state = ioread8(port_addr) & gpio_mask; bitmap_set_value8(bits, port_state, offset); } @@ -130,7 +130,7 @@ static void idi_48_irq_mask(struct irq_data *data) raw_spin_lock_irqsave(&idi48gpio->lock, flags); - outb(idi48gpio->cos_enb, idi48gpio->base + 7); + iowrite8(idi48gpio->cos_enb, idi48gpio->base + 7); raw_spin_unlock_irqrestore(&idi48gpio->lock, flags); } @@ -163,7 +163,7 @@ static void idi_48_irq_unmask(struct irq_data *data) raw_spin_lock_irqsave(&idi48gpio->lock, flags); - outb(idi48gpio->cos_enb, idi48gpio->base + 7); + iowrite8(idi48gpio->cos_enb, idi48gpio->base + 7); raw_spin_unlock_irqrestore(&idi48gpio->lock, flags); } @@ -204,7 +204,7 @@ static irqreturn_t idi_48_irq_handler(int irq, void *dev_id) raw_spin_lock(&idi48gpio->lock); - cos_status = inb(idi48gpio->base + 7); + cos_status = ioread8(idi48gpio->base + 7); raw_spin_unlock(&idi48gpio->lock); @@ -250,8 +250,8 @@ static int idi_48_irq_init_hw(struct gpio_chip *gc) struct idi_48_gpio *const idi48gpio = gpiochip_get_data(gc); /* Disable IRQ by default */ - outb(0, idi48gpio->base + 7); - inb(idi48gpio->base + 7); + iowrite8(0, idi48gpio->base + 7); + ioread8(idi48gpio->base + 7); return 0; } @@ -273,6 +273,10 @@ static int idi_48_probe(struct device *dev, unsigned int id) return -EBUSY; } + idi48gpio->base = devm_ioport_map(dev, base[id], IDI_48_EXTENT); + if (!idi48gpio->base) + return -ENOMEM; + idi48gpio->chip.label = name; idi48gpio->chip.parent = dev; idi48gpio->chip.owner = THIS_MODULE; @@ -283,7 +287,6 @@ static int idi_48_probe(struct device *dev, unsigned int id) idi48gpio->chip.direction_input = idi_48_gpio_direction_input; idi48gpio->chip.get = idi_48_gpio_get; idi48gpio->chip.get_multiple = idi_48_gpio_get_multiple; - idi48gpio->base = base[id]; girq = &idi48gpio->chip.irq; girq->chip = &idi_48_irqchip; From e0a574ef413b9e0b2ef6ddc540fa292eb38625b6 Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Tue, 10 May 2022 13:30:56 -0400 Subject: [PATCH 243/319] gpio: 104-idio-16: Utilize iomap interface This driver doesn't need to access I/O ports directly via inb()/outb() and friends. This patch abstracts such access by calling ioport_map() to enable the use of more typical ioread8()/iowrite8() I/O memory accessor calls. Suggested-by: David Laight Signed-off-by: William Breathitt Gray Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-104-idio-16.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/drivers/gpio/gpio-104-idio-16.c b/drivers/gpio/gpio-104-idio-16.c index c68ed1a135fa..45f7ad8573e1 100644 --- a/drivers/gpio/gpio-104-idio-16.c +++ b/drivers/gpio/gpio-104-idio-16.c @@ -44,7 +44,7 @@ struct idio_16_gpio { struct gpio_chip chip; raw_spinlock_t lock; unsigned long irq_mask; - unsigned int base; + void __iomem *base; unsigned int out_state; }; @@ -79,9 +79,9 @@ static int idio_16_gpio_get(struct gpio_chip *chip, unsigned int offset) return -EINVAL; if (offset < 24) - return !!(inb(idio16gpio->base + 1) & mask); + return !!(ioread8(idio16gpio->base + 1) & mask); - return !!(inb(idio16gpio->base + 5) & (mask>>8)); + return !!(ioread8(idio16gpio->base + 5) & (mask>>8)); } static int idio_16_gpio_get_multiple(struct gpio_chip *chip, @@ -91,9 +91,9 @@ static int idio_16_gpio_get_multiple(struct gpio_chip *chip, *bits = 0; if (*mask & GENMASK(23, 16)) - *bits |= (unsigned long)inb(idio16gpio->base + 1) << 16; + *bits |= (unsigned long)ioread8(idio16gpio->base + 1) << 16; if (*mask & GENMASK(31, 24)) - *bits |= (unsigned long)inb(idio16gpio->base + 5) << 24; + *bits |= (unsigned long)ioread8(idio16gpio->base + 5) << 24; return 0; } @@ -116,9 +116,9 @@ static void idio_16_gpio_set(struct gpio_chip *chip, unsigned int offset, idio16gpio->out_state &= ~mask; if (offset > 7) - outb(idio16gpio->out_state >> 8, idio16gpio->base + 4); + iowrite8(idio16gpio->out_state >> 8, idio16gpio->base + 4); else - outb(idio16gpio->out_state, idio16gpio->base); + iowrite8(idio16gpio->out_state, idio16gpio->base); raw_spin_unlock_irqrestore(&idio16gpio->lock, flags); } @@ -135,9 +135,9 @@ static void idio_16_gpio_set_multiple(struct gpio_chip *chip, idio16gpio->out_state |= *mask & *bits; if (*mask & 0xFF) - outb(idio16gpio->out_state, idio16gpio->base); + iowrite8(idio16gpio->out_state, idio16gpio->base); if ((*mask >> 8) & 0xFF) - outb(idio16gpio->out_state >> 8, idio16gpio->base + 4); + iowrite8(idio16gpio->out_state >> 8, idio16gpio->base + 4); raw_spin_unlock_irqrestore(&idio16gpio->lock, flags); } @@ -158,7 +158,7 @@ static void idio_16_irq_mask(struct irq_data *data) if (!idio16gpio->irq_mask) { raw_spin_lock_irqsave(&idio16gpio->lock, flags); - outb(0, idio16gpio->base + 2); + iowrite8(0, idio16gpio->base + 2); raw_spin_unlock_irqrestore(&idio16gpio->lock, flags); } @@ -177,7 +177,7 @@ static void idio_16_irq_unmask(struct irq_data *data) if (!prev_irq_mask) { raw_spin_lock_irqsave(&idio16gpio->lock, flags); - inb(idio16gpio->base + 2); + ioread8(idio16gpio->base + 2); raw_spin_unlock_irqrestore(&idio16gpio->lock, flags); } @@ -212,7 +212,7 @@ static irqreturn_t idio_16_irq_handler(int irq, void *dev_id) raw_spin_lock(&idio16gpio->lock); - outb(0, idio16gpio->base + 1); + iowrite8(0, idio16gpio->base + 1); raw_spin_unlock(&idio16gpio->lock); @@ -232,8 +232,8 @@ static int idio_16_irq_init_hw(struct gpio_chip *gc) struct idio_16_gpio *const idio16gpio = gpiochip_get_data(gc); /* Disable IRQ by default */ - outb(0, idio16gpio->base + 2); - outb(0, idio16gpio->base + 1); + iowrite8(0, idio16gpio->base + 2); + iowrite8(0, idio16gpio->base + 1); return 0; } @@ -255,6 +255,10 @@ static int idio_16_probe(struct device *dev, unsigned int id) return -EBUSY; } + idio16gpio->base = devm_ioport_map(dev, base[id], IDIO_16_EXTENT); + if (!idio16gpio->base) + return -ENOMEM; + idio16gpio->chip.label = name; idio16gpio->chip.parent = dev; idio16gpio->chip.owner = THIS_MODULE; @@ -268,7 +272,6 @@ static int idio_16_probe(struct device *dev, unsigned int id) idio16gpio->chip.get_multiple = idio_16_gpio_get_multiple; idio16gpio->chip.set = idio_16_gpio_set; idio16gpio->chip.set_multiple = idio_16_gpio_set_multiple; - idio16gpio->base = base[id]; idio16gpio->out_state = 0xFFFF; girq = &idio16gpio->chip.irq; From 54c8e25174a5c0557ee1c231fe9801790076b85f Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Tue, 10 May 2022 13:30:57 -0400 Subject: [PATCH 244/319] gpio: gpio-mm: Utilize iomap interface This driver doesn't need to access I/O ports directly via inb()/outb() and friends. This patch abstracts such access by calling ioport_map() to enable the use of more typical ioread8()/iowrite8() I/O memory accessor calls. Suggested-by: David Laight Signed-off-by: William Breathitt Gray Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-gpio-mm.c | 43 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/drivers/gpio/gpio-gpio-mm.c b/drivers/gpio/gpio-gpio-mm.c index b89b8c5ff1f5..097a06463d01 100644 --- a/drivers/gpio/gpio-gpio-mm.c +++ b/drivers/gpio/gpio-gpio-mm.c @@ -42,7 +42,7 @@ struct gpiomm_gpio { unsigned char out_state[6]; unsigned char control[2]; spinlock_t lock; - unsigned int base; + void __iomem *base; }; static int gpiomm_gpio_get_direction(struct gpio_chip *chip, @@ -64,7 +64,6 @@ static int gpiomm_gpio_direction_input(struct gpio_chip *chip, struct gpiomm_gpio *const gpiommgpio = gpiochip_get_data(chip); const unsigned int io_port = offset / 8; const unsigned int control_port = io_port / 3; - const unsigned int control_addr = gpiommgpio->base + 3 + control_port*4; unsigned long flags; unsigned int control; @@ -89,7 +88,7 @@ static int gpiomm_gpio_direction_input(struct gpio_chip *chip, } control = BIT(7) | gpiommgpio->control[control_port]; - outb(control, control_addr); + iowrite8(control, gpiommgpio->base + 3 + control_port*4); spin_unlock_irqrestore(&gpiommgpio->lock, flags); @@ -103,7 +102,6 @@ static int gpiomm_gpio_direction_output(struct gpio_chip *chip, const unsigned int io_port = offset / 8; const unsigned int control_port = io_port / 3; const unsigned int mask = BIT(offset % 8); - const unsigned int control_addr = gpiommgpio->base + 3 + control_port*4; const unsigned int out_port = (io_port > 2) ? io_port + 1 : io_port; unsigned long flags; unsigned int control; @@ -134,9 +132,9 @@ static int gpiomm_gpio_direction_output(struct gpio_chip *chip, gpiommgpio->out_state[io_port] &= ~mask; control = BIT(7) | gpiommgpio->control[control_port]; - outb(control, control_addr); + iowrite8(control, gpiommgpio->base + 3 + control_port*4); - outb(gpiommgpio->out_state[io_port], gpiommgpio->base + out_port); + iowrite8(gpiommgpio->out_state[io_port], gpiommgpio->base + out_port); spin_unlock_irqrestore(&gpiommgpio->lock, flags); @@ -160,7 +158,7 @@ static int gpiomm_gpio_get(struct gpio_chip *chip, unsigned int offset) return -EINVAL; } - port_state = inb(gpiommgpio->base + in_port); + port_state = ioread8(gpiommgpio->base + in_port); spin_unlock_irqrestore(&gpiommgpio->lock, flags); @@ -175,7 +173,7 @@ static int gpiomm_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask, struct gpiomm_gpio *const gpiommgpio = gpiochip_get_data(chip); unsigned long offset; unsigned long gpio_mask; - unsigned int port_addr; + void __iomem *port_addr; unsigned long port_state; /* clear bits array to a clean slate */ @@ -183,7 +181,7 @@ static int gpiomm_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask, for_each_set_clump8(offset, gpio_mask, mask, ARRAY_SIZE(ports) * 8) { port_addr = gpiommgpio->base + ports[offset / 8]; - port_state = inb(port_addr) & gpio_mask; + port_state = ioread8(port_addr) & gpio_mask; bitmap_set_value8(bits, port_state, offset); } @@ -207,7 +205,7 @@ static void gpiomm_gpio_set(struct gpio_chip *chip, unsigned int offset, else gpiommgpio->out_state[port] &= ~mask; - outb(gpiommgpio->out_state[port], gpiommgpio->base + out_port); + iowrite8(gpiommgpio->out_state[port], gpiommgpio->base + out_port); spin_unlock_irqrestore(&gpiommgpio->lock, flags); } @@ -219,7 +217,7 @@ static void gpiomm_gpio_set_multiple(struct gpio_chip *chip, unsigned long offset; unsigned long gpio_mask; size_t index; - unsigned int port_addr; + void __iomem *port_addr; unsigned long bitmask; unsigned long flags; @@ -234,7 +232,7 @@ static void gpiomm_gpio_set_multiple(struct gpio_chip *chip, /* update output state data and set device gpio register */ gpiommgpio->out_state[index] &= ~gpio_mask; gpiommgpio->out_state[index] |= bitmask; - outb(gpiommgpio->out_state[index], port_addr); + iowrite8(gpiommgpio->out_state[index], port_addr); spin_unlock_irqrestore(&gpiommgpio->lock, flags); } @@ -268,6 +266,10 @@ static int gpiomm_probe(struct device *dev, unsigned int id) return -EBUSY; } + gpiommgpio->base = devm_ioport_map(dev, base[id], GPIOMM_EXTENT); + if (!gpiommgpio->base) + return -ENOMEM; + gpiommgpio->chip.label = name; gpiommgpio->chip.parent = dev; gpiommgpio->chip.owner = THIS_MODULE; @@ -281,7 +283,6 @@ static int gpiomm_probe(struct device *dev, unsigned int id) gpiommgpio->chip.get_multiple = gpiomm_gpio_get_multiple; gpiommgpio->chip.set = gpiomm_gpio_set; gpiommgpio->chip.set_multiple = gpiomm_gpio_set_multiple; - gpiommgpio->base = base[id]; spin_lock_init(&gpiommgpio->lock); @@ -292,14 +293,14 @@ static int gpiomm_probe(struct device *dev, unsigned int id) } /* initialize all GPIO as output */ - outb(0x80, base[id] + 3); - outb(0x00, base[id]); - outb(0x00, base[id] + 1); - outb(0x00, base[id] + 2); - outb(0x80, base[id] + 7); - outb(0x00, base[id] + 4); - outb(0x00, base[id] + 5); - outb(0x00, base[id] + 6); + iowrite8(0x80, gpiommgpio->base + 3); + iowrite8(0x00, gpiommgpio->base); + iowrite8(0x00, gpiommgpio->base + 1); + iowrite8(0x00, gpiommgpio->base + 2); + iowrite8(0x80, gpiommgpio->base + 7); + iowrite8(0x00, gpiommgpio->base + 4); + iowrite8(0x00, gpiommgpio->base + 5); + iowrite8(0x00, gpiommgpio->base + 6); return 0; } From 5561a2b086394f554c2ff031db4195f39db5213c Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Tue, 10 May 2022 13:30:58 -0400 Subject: [PATCH 245/319] gpio: ws16c48: Utilize iomap interface This driver doesn't need to access I/O ports directly via inb()/outb() and friends. This patch abstracts such access by calling ioport_map() to enable the use of more typical ioread8()/iowrite8() I/O memory accessor calls. Suggested-by: David Laight Signed-off-by: William Breathitt Gray Reviewed-by: Linus Walleij Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-ws16c48.c | 65 +++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/drivers/gpio/gpio-ws16c48.c b/drivers/gpio/gpio-ws16c48.c index bb02a82e22f4..5078631d8014 100644 --- a/drivers/gpio/gpio-ws16c48.c +++ b/drivers/gpio/gpio-ws16c48.c @@ -47,7 +47,7 @@ struct ws16c48_gpio { raw_spinlock_t lock; unsigned long irq_mask; unsigned long flow_mask; - unsigned base; + void __iomem *base; }; static int ws16c48_gpio_get_direction(struct gpio_chip *chip, unsigned offset) @@ -73,7 +73,7 @@ static int ws16c48_gpio_direction_input(struct gpio_chip *chip, unsigned offset) ws16c48gpio->io_state[port] |= mask; ws16c48gpio->out_state[port] &= ~mask; - outb(ws16c48gpio->out_state[port], ws16c48gpio->base + port); + iowrite8(ws16c48gpio->out_state[port], ws16c48gpio->base + port); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); @@ -95,7 +95,7 @@ static int ws16c48_gpio_direction_output(struct gpio_chip *chip, ws16c48gpio->out_state[port] |= mask; else ws16c48gpio->out_state[port] &= ~mask; - outb(ws16c48gpio->out_state[port], ws16c48gpio->base + port); + iowrite8(ws16c48gpio->out_state[port], ws16c48gpio->base + port); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); @@ -118,7 +118,7 @@ static int ws16c48_gpio_get(struct gpio_chip *chip, unsigned offset) return -EINVAL; } - port_state = inb(ws16c48gpio->base + port); + port_state = ioread8(ws16c48gpio->base + port); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); @@ -131,7 +131,7 @@ static int ws16c48_gpio_get_multiple(struct gpio_chip *chip, struct ws16c48_gpio *const ws16c48gpio = gpiochip_get_data(chip); unsigned long offset; unsigned long gpio_mask; - unsigned int port_addr; + void __iomem *port_addr; unsigned long port_state; /* clear bits array to a clean slate */ @@ -139,7 +139,7 @@ static int ws16c48_gpio_get_multiple(struct gpio_chip *chip, for_each_set_clump8(offset, gpio_mask, mask, chip->ngpio) { port_addr = ws16c48gpio->base + offset / 8; - port_state = inb(port_addr) & gpio_mask; + port_state = ioread8(port_addr) & gpio_mask; bitmap_set_value8(bits, port_state, offset); } @@ -166,7 +166,7 @@ static void ws16c48_gpio_set(struct gpio_chip *chip, unsigned offset, int value) ws16c48gpio->out_state[port] |= mask; else ws16c48gpio->out_state[port] &= ~mask; - outb(ws16c48gpio->out_state[port], ws16c48gpio->base + port); + iowrite8(ws16c48gpio->out_state[port], ws16c48gpio->base + port); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); } @@ -178,7 +178,7 @@ static void ws16c48_gpio_set_multiple(struct gpio_chip *chip, unsigned long offset; unsigned long gpio_mask; size_t index; - unsigned int port_addr; + void __iomem *port_addr; unsigned long bitmask; unsigned long flags; @@ -195,7 +195,7 @@ static void ws16c48_gpio_set_multiple(struct gpio_chip *chip, /* update output state data and set device gpio register */ ws16c48gpio->out_state[index] &= ~gpio_mask; ws16c48gpio->out_state[index] |= bitmask; - outb(ws16c48gpio->out_state[index], port_addr); + iowrite8(ws16c48gpio->out_state[index], port_addr); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); } @@ -219,10 +219,10 @@ static void ws16c48_irq_ack(struct irq_data *data) port_state = ws16c48gpio->irq_mask >> (8*port); - outb(0x80, ws16c48gpio->base + 7); - outb(port_state & ~mask, ws16c48gpio->base + 8 + port); - outb(port_state | mask, ws16c48gpio->base + 8 + port); - outb(0xC0, ws16c48gpio->base + 7); + iowrite8(0x80, ws16c48gpio->base + 7); + iowrite8(port_state & ~mask, ws16c48gpio->base + 8 + port); + iowrite8(port_state | mask, ws16c48gpio->base + 8 + port); + iowrite8(0xC0, ws16c48gpio->base + 7); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); } @@ -244,9 +244,9 @@ static void ws16c48_irq_mask(struct irq_data *data) ws16c48gpio->irq_mask &= ~mask; - outb(0x80, ws16c48gpio->base + 7); - outb(ws16c48gpio->irq_mask >> (8*port), ws16c48gpio->base + 8 + port); - outb(0xC0, ws16c48gpio->base + 7); + iowrite8(0x80, ws16c48gpio->base + 7); + iowrite8(ws16c48gpio->irq_mask >> (8*port), ws16c48gpio->base + 8 + port); + iowrite8(0xC0, ws16c48gpio->base + 7); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); } @@ -268,9 +268,9 @@ static void ws16c48_irq_unmask(struct irq_data *data) ws16c48gpio->irq_mask |= mask; - outb(0x80, ws16c48gpio->base + 7); - outb(ws16c48gpio->irq_mask >> (8*port), ws16c48gpio->base + 8 + port); - outb(0xC0, ws16c48gpio->base + 7); + iowrite8(0x80, ws16c48gpio->base + 7); + iowrite8(ws16c48gpio->irq_mask >> (8*port), ws16c48gpio->base + 8 + port); + iowrite8(0xC0, ws16c48gpio->base + 7); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); } @@ -304,9 +304,9 @@ static int ws16c48_irq_set_type(struct irq_data *data, unsigned flow_type) return -EINVAL; } - outb(0x40, ws16c48gpio->base + 7); - outb(ws16c48gpio->flow_mask >> (8*port), ws16c48gpio->base + 8 + port); - outb(0xC0, ws16c48gpio->base + 7); + iowrite8(0x40, ws16c48gpio->base + 7); + iowrite8(ws16c48gpio->flow_mask >> (8*port), ws16c48gpio->base + 8 + port); + iowrite8(0xC0, ws16c48gpio->base + 7); raw_spin_unlock_irqrestore(&ws16c48gpio->lock, flags); @@ -330,20 +330,20 @@ static irqreturn_t ws16c48_irq_handler(int irq, void *dev_id) unsigned long int_id; unsigned long gpio; - int_pending = inb(ws16c48gpio->base + 6) & 0x7; + int_pending = ioread8(ws16c48gpio->base + 6) & 0x7; if (!int_pending) return IRQ_NONE; /* loop until all pending interrupts are handled */ do { for_each_set_bit(port, &int_pending, 3) { - int_id = inb(ws16c48gpio->base + 8 + port); + int_id = ioread8(ws16c48gpio->base + 8 + port); for_each_set_bit(gpio, &int_id, 8) generic_handle_domain_irq(chip->irq.domain, gpio + 8*port); } - int_pending = inb(ws16c48gpio->base + 6) & 0x7; + int_pending = ioread8(ws16c48gpio->base + 6) & 0x7; } while (int_pending); return IRQ_HANDLED; @@ -370,11 +370,11 @@ static int ws16c48_irq_init_hw(struct gpio_chip *gc) struct ws16c48_gpio *const ws16c48gpio = gpiochip_get_data(gc); /* Disable IRQ by default */ - outb(0x80, ws16c48gpio->base + 7); - outb(0, ws16c48gpio->base + 8); - outb(0, ws16c48gpio->base + 9); - outb(0, ws16c48gpio->base + 10); - outb(0xC0, ws16c48gpio->base + 7); + iowrite8(0x80, ws16c48gpio->base + 7); + iowrite8(0, ws16c48gpio->base + 8); + iowrite8(0, ws16c48gpio->base + 9); + iowrite8(0, ws16c48gpio->base + 10); + iowrite8(0xC0, ws16c48gpio->base + 7); return 0; } @@ -396,6 +396,10 @@ static int ws16c48_probe(struct device *dev, unsigned int id) return -EBUSY; } + ws16c48gpio->base = devm_ioport_map(dev, base[id], WS16C48_EXTENT); + if (!ws16c48gpio->base) + return -ENOMEM; + ws16c48gpio->chip.label = name; ws16c48gpio->chip.parent = dev; ws16c48gpio->chip.owner = THIS_MODULE; @@ -409,7 +413,6 @@ static int ws16c48_probe(struct device *dev, unsigned int id) ws16c48gpio->chip.get_multiple = ws16c48_gpio_get_multiple; ws16c48gpio->chip.set = ws16c48_gpio_set; ws16c48gpio->chip.set_multiple = ws16c48_gpio_set_multiple; - ws16c48gpio->base = base[id]; girq = &ws16c48gpio->chip.irq; girq->chip = &ws16c48_irqchip; From cae889302ebf5a9b22ca3580996118b8d20b3ae6 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 14 May 2022 11:25:24 +0100 Subject: [PATCH 246/319] KVM: arm64: vgic-v3: List M1 Pro/Max as requiring the SEIS workaround Unsusprisingly, Apple M1 Pro/Max have the exact same defect as the original M1 and generate random SErrors in the host when a guest tickles the GICv3 CPU interface the wrong way. Add the part numbers for both the CPU types found in these two new implementations, and add them to the hall of shame. This also applies to the Ultra version, as it is composed of 2 Max SoCs. Signed-off-by: Marc Zyngier Acked-by: Catalin Marinas Link: https://lore.kernel.org/r/20220514102524.3188730-1-maz@kernel.org --- arch/arm64/include/asm/cputype.h | 8 ++++++++ arch/arm64/kvm/vgic/vgic-v3.c | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index ff8f4511df71..60647bdc0b09 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -118,6 +118,10 @@ #define APPLE_CPU_PART_M1_ICESTORM 0x022 #define APPLE_CPU_PART_M1_FIRESTORM 0x023 +#define APPLE_CPU_PART_M1_ICESTORM_PRO 0x024 +#define APPLE_CPU_PART_M1_FIRESTORM_PRO 0x025 +#define APPLE_CPU_PART_M1_ICESTORM_MAX 0x028 +#define APPLE_CPU_PART_M1_FIRESTORM_MAX 0x029 #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) @@ -164,6 +168,10 @@ #define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) #define MIDR_APPLE_M1_ICESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) #define MIDR_APPLE_M1_FIRESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) +#define MIDR_APPLE_M1_ICESTORM_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_PRO) +#define MIDR_APPLE_M1_FIRESTORM_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_PRO) +#define MIDR_APPLE_M1_ICESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_MAX) +#define MIDR_APPLE_M1_FIRESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_MAX) /* Fujitsu Erratum 010001 affects A64FX 1.0 and 1.1, (v0r0 and v1r0) */ #define MIDR_FUJITSU_ERRATUM_010001 MIDR_FUJITSU_A64FX diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index b549af8b1dc2..826ff6f2a4e7 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -612,6 +612,10 @@ early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable); static const struct midr_range broken_seis[] = { MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM), MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM), + MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO), + MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_PRO), + MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_MAX), + MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_MAX), {}, }; From 3cb8a091a7016be0ee5420ff6c1928972e3ce23e Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 10 May 2022 09:57:07 +0000 Subject: [PATCH 247/319] KVM: arm64: Wrapper for getting pmu_events Eases migrating away from using hyp data and simplifies the code. No functional change intended. Reviewed-by: Oliver Upton Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220510095710.148178-2-tabba@google.com --- arch/arm64/kvm/pmu.c | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/arch/arm64/kvm/pmu.c b/arch/arm64/kvm/pmu.c index 03a6c1f4a09a..4bd38ff34221 100644 --- a/arch/arm64/kvm/pmu.c +++ b/arch/arm64/kvm/pmu.c @@ -25,21 +25,31 @@ static bool kvm_pmu_switch_needed(struct perf_event_attr *attr) return (attr->exclude_host != attr->exclude_guest); } +static struct kvm_pmu_events *kvm_get_pmu_events(void) +{ + struct kvm_host_data *ctx = this_cpu_ptr_hyp_sym(kvm_host_data); + + if (!ctx) + return NULL; + + return &ctx->pmu_events; +} + /* * Add events to track that we may want to switch at guest entry/exit * time. */ void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) { - struct kvm_host_data *ctx = this_cpu_ptr_hyp_sym(kvm_host_data); + struct kvm_pmu_events *pmu = kvm_get_pmu_events(); - if (!kvm_arm_support_pmu_v3() || !ctx || !kvm_pmu_switch_needed(attr)) + if (!kvm_arm_support_pmu_v3() || !pmu || !kvm_pmu_switch_needed(attr)) return; if (!attr->exclude_host) - ctx->pmu_events.events_host |= set; + pmu->events_host |= set; if (!attr->exclude_guest) - ctx->pmu_events.events_guest |= set; + pmu->events_guest |= set; } /* @@ -47,13 +57,13 @@ void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) */ void kvm_clr_pmu_events(u32 clr) { - struct kvm_host_data *ctx = this_cpu_ptr_hyp_sym(kvm_host_data); + struct kvm_pmu_events *pmu = kvm_get_pmu_events(); - if (!kvm_arm_support_pmu_v3() || !ctx) + if (!kvm_arm_support_pmu_v3() || !pmu) return; - ctx->pmu_events.events_host &= ~clr; - ctx->pmu_events.events_guest &= ~clr; + pmu->events_host &= ~clr; + pmu->events_guest &= ~clr; } #define PMEVTYPER_READ_CASE(idx) \ @@ -169,16 +179,16 @@ static void kvm_vcpu_pmu_disable_el0(unsigned long events) */ void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) { - struct kvm_host_data *host; + struct kvm_pmu_events *pmu; u32 events_guest, events_host; if (!kvm_arm_support_pmu_v3() || !has_vhe()) return; preempt_disable(); - host = this_cpu_ptr_hyp_sym(kvm_host_data); - events_guest = host->pmu_events.events_guest; - events_host = host->pmu_events.events_host; + pmu = kvm_get_pmu_events(); + events_guest = pmu->events_guest; + events_host = pmu->events_host; kvm_vcpu_pmu_enable_el0(events_guest); kvm_vcpu_pmu_disable_el0(events_host); @@ -190,15 +200,15 @@ void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) */ void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) { - struct kvm_host_data *host; + struct kvm_pmu_events *pmu; u32 events_guest, events_host; if (!kvm_arm_support_pmu_v3() || !has_vhe()) return; - host = this_cpu_ptr_hyp_sym(kvm_host_data); - events_guest = host->pmu_events.events_guest; - events_host = host->pmu_events.events_host; + pmu = kvm_get_pmu_events(); + events_guest = pmu->events_guest; + events_host = pmu->events_host; kvm_vcpu_pmu_enable_el0(events_host); kvm_vcpu_pmu_disable_el0(events_guest); From e987a4c60f9755b2f7a19bf1b5ef2eb74c90579b Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 10 May 2022 09:57:08 +0000 Subject: [PATCH 248/319] KVM: arm64: Repack struct kvm_pmu to reduce size struct kvm_pmu has 2 holes using 10 bytes. This is instantiated in all vcpus, so it adds up. Repack the structures to remove the holes. No functional change intended. Reviewed-by: Oliver Upton Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220510095710.148178-3-tabba@google.com --- include/kvm/arm_pmu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h index 20193416d214..eaa8290b116f 100644 --- a/include/kvm/arm_pmu.h +++ b/include/kvm/arm_pmu.h @@ -21,12 +21,12 @@ struct kvm_pmc { }; struct kvm_pmu { - int irq_num; + struct irq_work overflow_work; struct kvm_pmc pmc[ARMV8_PMU_MAX_COUNTERS]; DECLARE_BITMAP(chained, ARMV8_PMU_MAX_COUNTER_PAIRS); + int irq_num; bool created; bool irq_level; - struct irq_work overflow_work; }; struct arm_pmu_entry { From 84d751a019a9792f5b4884e1d598b603c360ec22 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 10 May 2022 09:57:09 +0000 Subject: [PATCH 249/319] KVM: arm64: Pass pmu events to hyp via vcpu Instead of the host accessing hyp data directly, pass the pmu events of the current cpu to hyp via the vcpu. This adds 64 bits (in two fields) to the vcpu that need to be synced before every vcpu run in nvhe and protected modes. However, it isolates the hypervisor from the host, which allows us to use pmu in protected mode in a subsequent patch. No visible side effects in behavior intended. Signed-off-by: Fuad Tabba Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220510095710.148178-4-tabba@google.com --- arch/arm64/include/asm/kvm_host.h | 7 +------ arch/arm64/kvm/arm.c | 15 +++++++++++++++ arch/arm64/kvm/hyp/nvhe/switch.c | 20 ++++++-------------- arch/arm64/kvm/pmu.c | 12 ++++-------- include/kvm/arm_pmu.h | 6 ++++++ 5 files changed, 32 insertions(+), 28 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 94a27a7520f4..efca5a63bdaf 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -254,14 +254,8 @@ struct kvm_cpu_context { struct kvm_vcpu *__hyp_running_vcpu; }; -struct kvm_pmu_events { - u32 events_host; - u32 events_guest; -}; - struct kvm_host_data { struct kvm_cpu_context host_ctxt; - struct kvm_pmu_events pmu_events; }; struct kvm_host_psci_config { @@ -796,6 +790,7 @@ void kvm_arch_vcpu_put_debug_state_flags(struct kvm_vcpu *vcpu); void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr); void kvm_clr_pmu_events(u32 clr); +struct kvm_pmu_events *kvm_get_pmu_events(void); void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu); void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu); #else diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 523bc934fe2f..aa1b15e9d5d9 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -751,6 +751,19 @@ static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu) return ret; } +/* + * Updates the vcpu's view of the pmu events for this cpu. + * Must be called before every vcpu run after disabling interrupts, to ensure + * that an interrupt cannot fire and update the structure. + */ +static void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) +{ + if (has_vhe() || !kvm_vcpu_has_pmu(vcpu)) + return; + + vcpu->arch.pmu.events = *kvm_get_pmu_events(); +} + /** * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code * @vcpu: The VCPU pointer @@ -815,6 +828,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_vgic_flush_hwstate(vcpu); + kvm_pmu_update_vcpu_events(vcpu); + /* * Ensure we set mode to IN_GUEST_MODE after we disable * interrupts and before the final VCPU requests check. diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 6410d21d8695..ff7b29fb9787 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -123,13 +123,9 @@ static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) /** * Disable host events, enable guest events */ -static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) +static bool __pmu_switch_to_guest(struct kvm_vcpu *vcpu) { - struct kvm_host_data *host; - struct kvm_pmu_events *pmu; - - host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); - pmu = &host->pmu_events; + struct kvm_pmu_events *pmu = &vcpu->arch.pmu.events; if (pmu->events_host) write_sysreg(pmu->events_host, pmcntenclr_el0); @@ -143,13 +139,9 @@ static bool __pmu_switch_to_guest(struct kvm_cpu_context *host_ctxt) /** * Disable guest events, enable host events */ -static void __pmu_switch_to_host(struct kvm_cpu_context *host_ctxt) +static void __pmu_switch_to_host(struct kvm_vcpu *vcpu) { - struct kvm_host_data *host; - struct kvm_pmu_events *pmu; - - host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); - pmu = &host->pmu_events; + struct kvm_pmu_events *pmu = &vcpu->arch.pmu.events; if (pmu->events_guest) write_sysreg(pmu->events_guest, pmcntenclr_el0); @@ -274,7 +266,7 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu) host_ctxt->__hyp_running_vcpu = vcpu; guest_ctxt = &vcpu->arch.ctxt; - pmu_switch_needed = __pmu_switch_to_guest(host_ctxt); + pmu_switch_needed = __pmu_switch_to_guest(vcpu); __sysreg_save_state_nvhe(host_ctxt); /* @@ -336,7 +328,7 @@ int __kvm_vcpu_run(struct kvm_vcpu *vcpu) __debug_restore_host_buffers_nvhe(vcpu); if (pmu_switch_needed) - __pmu_switch_to_host(host_ctxt); + __pmu_switch_to_host(vcpu); /* Returning to host will clear PSR.I, remask PMR if needed */ if (system_uses_irq_prio_masking()) diff --git a/arch/arm64/kvm/pmu.c b/arch/arm64/kvm/pmu.c index 4bd38ff34221..7887133d15f0 100644 --- a/arch/arm64/kvm/pmu.c +++ b/arch/arm64/kvm/pmu.c @@ -5,7 +5,8 @@ */ #include #include -#include + +static DEFINE_PER_CPU(struct kvm_pmu_events, kvm_pmu_events); /* * Given the perf event attributes and system type, determine @@ -25,14 +26,9 @@ static bool kvm_pmu_switch_needed(struct perf_event_attr *attr) return (attr->exclude_host != attr->exclude_guest); } -static struct kvm_pmu_events *kvm_get_pmu_events(void) +struct kvm_pmu_events *kvm_get_pmu_events(void) { - struct kvm_host_data *ctx = this_cpu_ptr_hyp_sym(kvm_host_data); - - if (!ctx) - return NULL; - - return &ctx->pmu_events; + return this_cpu_ptr(&kvm_pmu_events); } /* diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h index eaa8290b116f..35a0903cae32 100644 --- a/include/kvm/arm_pmu.h +++ b/include/kvm/arm_pmu.h @@ -20,8 +20,14 @@ struct kvm_pmc { struct perf_event *perf_event; }; +struct kvm_pmu_events { + u32 events_host; + u32 events_guest; +}; + struct kvm_pmu { struct irq_work overflow_work; + struct kvm_pmu_events events; struct kvm_pmc pmc[ARMV8_PMU_MAX_COUNTERS]; DECLARE_BITMAP(chained, ARMV8_PMU_MAX_COUNTER_PAIRS); int irq_num; From 722625c6f4c5b6a9953d6af04c7bb1a6e12830b3 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 10 May 2022 09:57:10 +0000 Subject: [PATCH 250/319] KVM: arm64: Reenable pmu in Protected Mode Now that the pmu code does not access hyp data, reenable it in protected mode. Once fully supported, protected VMs will not have pmu support, since that could leak information. However, non-protected VMs in protected mode should have pmu support if available. Signed-off-by: Fuad Tabba Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220510095710.148178-5-tabba@google.com --- arch/arm64/kvm/pmu-emul.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 78fdc443adc7..dc1779d4c7dd 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -756,8 +756,7 @@ void kvm_host_pmu_init(struct arm_pmu *pmu) { struct arm_pmu_entry *entry; - if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_PMUVER_IMP_DEF || - is_protected_kvm_enabled()) + if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_PMUVER_IMP_DEF) return; mutex_lock(&arm_pmus_lock); From 2cde51f1e10f260076899bf41add74ed4a6de034 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 15 May 2022 11:36:24 +0100 Subject: [PATCH 251/319] KVM: arm64: Hide KVM_REG_ARM_*_BMAP_BIT_COUNT from userspace These constants will change over time, and userspace has no business knowing about them. Hide them behind __KERNEL__. Signed-off-by: Marc Zyngier --- arch/arm64/include/uapi/asm/kvm.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index e523bb6eac67..d4722b08f0d4 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -342,14 +342,18 @@ struct kvm_arm_copy_mte_tags { enum { KVM_REG_ARM_STD_BIT_TRNG_V1_0 = 0, +#ifdef __KERNEL__ KVM_REG_ARM_STD_BMAP_BIT_COUNT, +#endif }; #define KVM_REG_ARM_STD_HYP_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(1) enum { KVM_REG_ARM_STD_HYP_BIT_PV_TIME = 0, +#ifdef __KERNEL__ KVM_REG_ARM_STD_HYP_BMAP_BIT_COUNT, +#endif }; #define KVM_REG_ARM_VENDOR_HYP_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(2) @@ -357,7 +361,9 @@ enum { enum { KVM_REG_ARM_VENDOR_HYP_BIT_FUNC_FEAT = 0, KVM_REG_ARM_VENDOR_HYP_BIT_PTP = 1, +#ifdef __KERNEL__ KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_COUNT, +#endif }; /* Device Control API: ARM VGIC */ From 42701e7c0cd2a715def2dafd22f11f25ca0f5024 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Fri, 13 May 2022 12:41:37 +0800 Subject: [PATCH 252/319] platform/chrome: cros_ec_proto: drop unneeded BUG_ON() in prepare_packet() prepare_packet() gets called if `ec_dev->proto_version` > 2. For now, it must be equivalent to EC_HOST_REQUEST_VERSION. Drop the BUG_ON(). Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220513044143.1045728-2-tzungbi@kernel.org --- drivers/platform/chrome/cros_ec_proto.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index ac1419881ff3..db1c8ba43171 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -60,7 +60,6 @@ static int prepare_packet(struct cros_ec_device *ec_dev, int i; u8 csum = 0; - BUG_ON(ec_dev->proto_version != EC_HOST_REQUEST_VERSION); BUG_ON(msg->outsize + sizeof(*request) > ec_dev->dout_size); out = ec_dev->dout; From 71d3ae7fb6404c87b498f8b7f86b8271dd74989f Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Fri, 13 May 2022 12:41:38 +0800 Subject: [PATCH 253/319] platform/chrome: correct cros_ec_prepare_tx() usage cros_ec_prepare_tx() returns either: - >= 0 for number of prepared bytes. - < 0 for -errno. Correct the comment and make sure all callers check the return code. Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220513044143.1045728-3-tzungbi@kernel.org --- drivers/platform/chrome/cros_ec_i2c.c | 2 ++ drivers/platform/chrome/cros_ec_ishtp.c | 4 +++- drivers/platform/chrome/cros_ec_lpc.c | 2 ++ drivers/platform/chrome/cros_ec_proto.c | 2 +- drivers/platform/chrome/cros_ec_rpmsg.c | 2 ++ drivers/platform/chrome/cros_ec_spi.c | 4 ++++ 6 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_i2c.c b/drivers/platform/chrome/cros_ec_i2c.c index 22feb0fd4ce7..a4f305f1eb0e 100644 --- a/drivers/platform/chrome/cros_ec_i2c.c +++ b/drivers/platform/chrome/cros_ec_i2c.c @@ -89,6 +89,8 @@ static int cros_ec_pkt_xfer_i2c(struct cros_ec_device *ec_dev, ec_dev->dout++; ret = cros_ec_prepare_tx(ec_dev, msg); + if (ret < 0) + goto done; ec_dev->dout--; /* send command to EC and read answer */ diff --git a/drivers/platform/chrome/cros_ec_ishtp.c b/drivers/platform/chrome/cros_ec_ishtp.c index 4020b8354bae..cb2031cf7106 100644 --- a/drivers/platform/chrome/cros_ec_ishtp.c +++ b/drivers/platform/chrome/cros_ec_ishtp.c @@ -521,7 +521,9 @@ static int cros_ec_pkt_xfer_ish(struct cros_ec_device *ec_dev, out_msg->hdr.status = 0; ec_dev->dout += OUT_MSG_EC_REQUEST_PREAMBLE; - cros_ec_prepare_tx(ec_dev, msg); + rv = cros_ec_prepare_tx(ec_dev, msg); + if (rv < 0) + goto end_error; ec_dev->dout -= OUT_MSG_EC_REQUEST_PREAMBLE; dev_dbg(dev, diff --git a/drivers/platform/chrome/cros_ec_lpc.c b/drivers/platform/chrome/cros_ec_lpc.c index 8eeef85a96b1..7677ab3c0ead 100644 --- a/drivers/platform/chrome/cros_ec_lpc.c +++ b/drivers/platform/chrome/cros_ec_lpc.c @@ -147,6 +147,8 @@ static int cros_ec_pkt_xfer_lpc(struct cros_ec_device *ec, u8 *dout; ret = cros_ec_prepare_tx(ec, msg); + if (ret < 0) + goto done; /* Write buffer */ cros_ec_lpc_ops.write(EC_LPC_ADDR_HOST_PACKET, ret, ec->dout); diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index db1c8ba43171..2d6d3fbfa905 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -164,7 +164,7 @@ static int send_command(struct cros_ec_device *ec_dev, * only SPI uses it. Once LPC uses the same protocol it can start using it. * I2C could use it now, with a refactor of the existing code. * - * Return: 0 on success or negative error code. + * Return: number of prepared bytes on success or negative error code. */ int cros_ec_prepare_tx(struct cros_ec_device *ec_dev, struct cros_ec_command *msg) diff --git a/drivers/platform/chrome/cros_ec_rpmsg.c b/drivers/platform/chrome/cros_ec_rpmsg.c index d96d15b8ca94..39d3b50a7c09 100644 --- a/drivers/platform/chrome/cros_ec_rpmsg.c +++ b/drivers/platform/chrome/cros_ec_rpmsg.c @@ -89,6 +89,8 @@ static int cros_ec_pkt_xfer_rpmsg(struct cros_ec_device *ec_dev, ec_msg->result = 0; len = cros_ec_prepare_tx(ec_dev, ec_msg); + if (len < 0) + return len; dev_dbg(ec_dev->dev, "prepared, len=%d\n", len); reinit_completion(&ec_rpmsg->xfer_ack); diff --git a/drivers/platform/chrome/cros_ec_spi.c b/drivers/platform/chrome/cros_ec_spi.c index 8493af0f680e..589f18e9537d 100644 --- a/drivers/platform/chrome/cros_ec_spi.c +++ b/drivers/platform/chrome/cros_ec_spi.c @@ -401,6 +401,8 @@ static int do_cros_ec_pkt_xfer_spi(struct cros_ec_device *ec_dev, unsigned long delay; len = cros_ec_prepare_tx(ec_dev, ec_msg); + if (len < 0) + return len; dev_dbg(ec_dev->dev, "prepared, len=%d\n", len); /* If it's too soon to do another transaction, wait */ @@ -544,6 +546,8 @@ static int do_cros_ec_cmd_xfer_spi(struct cros_ec_device *ec_dev, unsigned long delay; len = cros_ec_prepare_tx(ec_dev, ec_msg); + if (len < 0) + return len; dev_dbg(ec_dev->dev, "prepared, len=%d\n", len); /* If it's too soon to do another transaction, wait */ From c2dcb1b06053a1ccfb73fe84e7b54b92383401cc Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Fri, 13 May 2022 12:41:39 +0800 Subject: [PATCH 254/319] platform/chrome: cros_ec_proto: drop BUG_ON() in cros_ec_prepare_tx() It is overkill to crash the kernel if the given message is oversize. Drop the BUG_ON() and return -EINVAL instead. Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220513044143.1045728-4-tzungbi@kernel.org --- drivers/platform/chrome/cros_ec_proto.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index 2d6d3fbfa905..9ce3374846ff 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -60,7 +60,8 @@ static int prepare_packet(struct cros_ec_device *ec_dev, int i; u8 csum = 0; - BUG_ON(msg->outsize + sizeof(*request) > ec_dev->dout_size); + if (msg->outsize + sizeof(*request) > ec_dev->dout_size) + return -EINVAL; out = ec_dev->dout; request = (struct ec_host_request *)out; @@ -176,7 +177,9 @@ int cros_ec_prepare_tx(struct cros_ec_device *ec_dev, if (ec_dev->proto_version > 2) return prepare_packet(ec_dev, msg); - BUG_ON(msg->outsize > EC_PROTO2_MAX_PARAM_SIZE); + if (msg->outsize > EC_PROTO2_MAX_PARAM_SIZE) + return -EINVAL; + out = ec_dev->dout; out[0] = EC_CMD_VERSION0 + msg->version; out[1] = msg->command; From 20a264c97bc8c17d3a7dd7e8d0f72dc57b02c75e Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Fri, 13 May 2022 12:41:40 +0800 Subject: [PATCH 255/319] platform/chrome: cros_ec_proto: drop BUG_ON() in cros_ec_get_host_event() It is overkill to crash the kernel if the `ec_dev` doesn't support MKBP event but gets called into cros_ec_get_host_event(). Drop the BUG_ON() and return error (0 in the case) instead. Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220513044143.1045728-5-tzungbi@kernel.org --- drivers/platform/chrome/cros_ec_proto.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c index 9ce3374846ff..ff767dccdf0f 100644 --- a/drivers/platform/chrome/cros_ec_proto.c +++ b/drivers/platform/chrome/cros_ec_proto.c @@ -817,7 +817,8 @@ u32 cros_ec_get_host_event(struct cros_ec_device *ec_dev) { u32 host_event; - BUG_ON(!ec_dev->mkbp_event_supported); + if (!ec_dev->mkbp_event_supported) + return 0; if (ec_dev->event_data.event_type != EC_MKBP_EVENT_HOST_EVENT) return 0; From 8bff946c4199fd79f43dbff93c030b58b01bed65 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Fri, 13 May 2022 12:41:41 +0800 Subject: [PATCH 256/319] platform/chrome: cros_ec_i2c: drop BUG_ON() in cros_ec_pkt_xfer_i2c() It is overkill to crash the kernel if the given message is oversize. Drop the BUG_ON() and return -EINVAL instead. Reviewed-by: Guenter Roeck Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220513044143.1045728-6-tzungbi@kernel.org --- drivers/platform/chrome/cros_ec_i2c.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_i2c.c b/drivers/platform/chrome/cros_ec_i2c.c index a4f305f1eb0e..9f5b95763173 100644 --- a/drivers/platform/chrome/cros_ec_i2c.c +++ b/drivers/platform/chrome/cros_ec_i2c.c @@ -72,13 +72,19 @@ static int cros_ec_pkt_xfer_i2c(struct cros_ec_device *ec_dev, i2c_msg[1].flags = I2C_M_RD; packet_len = msg->insize + response_header_size; - BUG_ON(packet_len > ec_dev->din_size); + if (packet_len > ec_dev->din_size) { + ret = -EINVAL; + goto done; + } in_buf = ec_dev->din; i2c_msg[1].len = packet_len; i2c_msg[1].buf = (char *) in_buf; packet_len = msg->outsize + request_header_size; - BUG_ON(packet_len > ec_dev->dout_size); + if (packet_len > ec_dev->dout_size) { + ret = -EINVAL; + goto done; + } out_buf = ec_dev->dout; i2c_msg[0].len = packet_len; i2c_msg[0].buf = (char *) out_buf; From ddec8e9e90cea8e8430b04a01adce7fb196d95c6 Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Fri, 13 May 2022 12:41:42 +0800 Subject: [PATCH 257/319] platform/chrome: cros_ec_spi: drop unneeded BUG_ON() In the context, the following conditions are always false: - `todo` < 0 Suppose that EC_SPI_FRAME_START is found at the last byte of transfer. In the case, `ptr` == `end` - 1. As a result, `todo` must be 0. - `todo` > `ec_dev->din_size` Suppose that there is no preamble bytes. EC_SPI_FRAME_START is found at the first byte of transfer. In the case, `end` == `ptr` + EC_MSG_PREAMBLE_COUNT. As a result, `todo` == EC_MSG_PREAMBLE_COUNT - 1. However, it already checked `ec_dev->din_size` < EC_MSG_PREAMBLE_COUNT at the beginning of function. Drop the unneeded BUG_ON(). Signed-off-by: Tzung-Bi Shih Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20220513044143.1045728-7-tzungbi@kernel.org --- drivers/platform/chrome/cros_ec_spi.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_spi.c b/drivers/platform/chrome/cros_ec_spi.c index 589f18e9537d..5264615f46af 100644 --- a/drivers/platform/chrome/cros_ec_spi.c +++ b/drivers/platform/chrome/cros_ec_spi.c @@ -237,7 +237,6 @@ static int cros_ec_spi_receive_packet(struct cros_ec_device *ec_dev, * start of our buffer */ todo = end - ++ptr; - BUG_ON(todo < 0 || todo > ec_dev->din_size); todo = min(todo, need_len); memmove(ec_dev->din, ptr, todo); ptr = ec_dev->din + todo; @@ -345,7 +344,6 @@ static int cros_ec_spi_receive_response(struct cros_ec_device *ec_dev, * start of our buffer */ todo = end - ++ptr; - BUG_ON(todo < 0 || todo > ec_dev->din_size); todo = min(todo, need_len); memmove(ec_dev->din, ptr, todo); ptr = ec_dev->din + todo; From bbd43a37ec7a02e81dc0afb2c6194957518a904b Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Fri, 13 May 2022 12:41:43 +0800 Subject: [PATCH 258/319] platform/chrome: cros_ec_spi: drop BUG_ON() if `din` isn't large enough It is overkill to crash the kernel if the `din` buffer is going to full or overflow. Drop the BUG_ON() and return -EINVAL instead. Signed-off-by: Tzung-Bi Shih Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20220513044143.1045728-8-tzungbi@kernel.org --- drivers/platform/chrome/cros_ec_spi.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/platform/chrome/cros_ec_spi.c b/drivers/platform/chrome/cros_ec_spi.c index 5264615f46af..7360b3ff6e4f 100644 --- a/drivers/platform/chrome/cros_ec_spi.c +++ b/drivers/platform/chrome/cros_ec_spi.c @@ -160,7 +160,8 @@ static int receive_n_bytes(struct cros_ec_device *ec_dev, u8 *buf, int n) struct spi_message msg; int ret; - BUG_ON(buf - ec_dev->din + n > ec_dev->din_size); + if (buf - ec_dev->din + n > ec_dev->din_size) + return -EINVAL; memset(&trans, 0, sizeof(trans)); trans.cs_change = 1; @@ -197,7 +198,8 @@ static int cros_ec_spi_receive_packet(struct cros_ec_device *ec_dev, unsigned long deadline; int todo; - BUG_ON(ec_dev->din_size < EC_MSG_PREAMBLE_COUNT); + if (ec_dev->din_size < EC_MSG_PREAMBLE_COUNT) + return -EINVAL; /* Receive data until we see the header byte */ deadline = jiffies + msecs_to_jiffies(EC_MSG_DEADLINE_MS); @@ -304,7 +306,8 @@ static int cros_ec_spi_receive_response(struct cros_ec_device *ec_dev, unsigned long deadline; int todo; - BUG_ON(ec_dev->din_size < EC_MSG_PREAMBLE_COUNT); + if (ec_dev->din_size < EC_MSG_PREAMBLE_COUNT) + return -EINVAL; /* Receive data until we see the header byte */ deadline = jiffies + msecs_to_jiffies(EC_MSG_DEADLINE_MS); From 20492a62b99bd4367b79a76ca288d018f11980db Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 16 May 2022 13:02:24 +0100 Subject: [PATCH 259/319] KVM: arm64: pmu: Restore compilation when HW_PERF_EVENTS isn't selected Moving kvm_pmu_events into the vcpu (and refering to it) broke the somewhat unusual case where the kernel has no support for a PMU at all. In order to solve this, move things around a bit so that we can easily avoid refering to the pmu structure outside of PMU-aware code. As a bonus, pmu.c isn't compiled in when HW_PERF_EVENTS isn't selected. Reported-by: kernel test robot Reviewed-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/202205161814.KQHpOzsJ-lkp@intel.com --- arch/arm64/include/asm/kvm_host.h | 6 ------ arch/arm64/kvm/Makefile | 4 ++-- arch/arm64/kvm/arm.c | 13 ------------- arch/arm64/kvm/hyp/nvhe/switch.c | 5 +++++ include/kvm/arm_pmu.h | 24 ++++++++++++++++++++++++ 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index efca5a63bdaf..ef774b8955bc 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -789,10 +789,6 @@ void kvm_arch_vcpu_put_debug_state_flags(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr); void kvm_clr_pmu_events(u32 clr); - -struct kvm_pmu_events *kvm_get_pmu_events(void); -void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu); -void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu); #else static inline void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) {} static inline void kvm_clr_pmu_events(u32 clr) {} @@ -824,8 +820,6 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); #define kvm_has_mte(kvm) \ (system_supports_mte() && \ test_bit(KVM_ARCH_FLAG_MTE_ENABLED, &(kvm)->arch.flags)) -#define kvm_vcpu_has_pmu(vcpu) \ - (test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features)) int kvm_trng_call(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 261644b1a6bb..aa127ae9f675 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -13,7 +13,7 @@ obj-$(CONFIG_KVM) += hyp/ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ inject_fault.o va_layout.o handle_exit.o \ guest.o debug.o reset.o sys_regs.o \ - vgic-sys-reg-v3.o fpsimd.o pmu.o pkvm.o \ + vgic-sys-reg-v3.o fpsimd.o pkvm.o \ arch_timer.o trng.o vmid.o \ vgic/vgic.o vgic/vgic-init.o \ vgic/vgic-irqfd.o vgic/vgic-v2.o \ @@ -22,7 +22,7 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \ vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ vgic/vgic-its.o vgic/vgic-debug.o -kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o +kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o always-y := hyp_constants.h hyp-constants.s diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index aa1b15e9d5d9..b3821c430ec9 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -751,19 +751,6 @@ static int noinstr kvm_arm_vcpu_enter_exit(struct kvm_vcpu *vcpu) return ret; } -/* - * Updates the vcpu's view of the pmu events for this cpu. - * Must be called before every vcpu run after disabling interrupts, to ensure - * that an interrupt cannot fire and update the structure. - */ -static void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) -{ - if (has_vhe() || !kvm_vcpu_has_pmu(vcpu)) - return; - - vcpu->arch.pmu.events = *kvm_get_pmu_events(); -} - /** * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code * @vcpu: The VCPU pointer diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index ff7b29fb9787..c7cd2036a75e 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -123,6 +123,7 @@ static void __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) /** * Disable host events, enable guest events */ +#ifdef CONFIG_HW_PERF_EVENTS static bool __pmu_switch_to_guest(struct kvm_vcpu *vcpu) { struct kvm_pmu_events *pmu = &vcpu->arch.pmu.events; @@ -149,6 +150,10 @@ static void __pmu_switch_to_host(struct kvm_vcpu *vcpu) if (pmu->events_host) write_sysreg(pmu->events_host, pmcntenset_el0); } +#else +#define __pmu_switch_to_guest(v) ({ false; }) +#define __pmu_switch_to_host(v) do {} while (0) +#endif /** * Handler for protected VM MSR, MRS or System instruction execution in AArch64. diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h index 35a0903cae32..c0b868ce6a8f 100644 --- a/include/kvm/arm_pmu.h +++ b/include/kvm/arm_pmu.h @@ -72,6 +72,25 @@ int kvm_arm_pmu_v3_get_attr(struct kvm_vcpu *vcpu, int kvm_arm_pmu_v3_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu); + +struct kvm_pmu_events *kvm_get_pmu_events(void); +void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu); +void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu); + +#define kvm_vcpu_has_pmu(vcpu) \ + (test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features)) + +/* + * Updates the vcpu's view of the pmu events for this cpu. + * Must be called before every vcpu run after disabling interrupts, to ensure + * that an interrupt cannot fire and update the structure. + */ +#define kvm_pmu_update_vcpu_events(vcpu) \ + do { \ + if (!has_vhe() && kvm_vcpu_has_pmu(vcpu)) \ + vcpu->arch.pmu.events = *kvm_get_pmu_events(); \ + } while (0) + #else struct kvm_pmu { }; @@ -133,6 +152,11 @@ static inline u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) return 0; } +#define kvm_vcpu_has_pmu(vcpu) ({ false; }) +static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {} +static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {} +static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {} + #endif #endif From cafe7e544d4979da222eaff12141ecac07901b9c Mon Sep 17 00:00:00 2001 From: Ricardo Koller Date: Mon, 9 May 2022 17:16:30 -0700 Subject: [PATCH 260/319] KVM: arm64: vgic: Check that new ITEs could be saved in guest memory Try to improve the predictability of ITS save/restores by failing commands that would lead to failed saves. More specifically, fail any command that adds an entry into an ITS table that is not in guest memory, which would otherwise lead to a failed ITS save ioctl. There are already checks for collection and device entries, but not for ITEs. Add the corresponding check for the ITT when adding ITEs. Reviewed-by: Eric Auger Signed-off-by: Ricardo Koller Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220510001633.552496-2-ricarkol@google.com --- arch/arm64/kvm/vgic/vgic-its.c | 47 +++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 2e13402be3bd..93a5178374c9 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -894,6 +894,18 @@ static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its, return update_affinity(ite->irq, vcpu); } +static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa) +{ + gfn_t gfn = gpa >> PAGE_SHIFT; + int idx; + bool ret; + + idx = srcu_read_lock(&its->dev->kvm->srcu); + ret = kvm_is_visible_gfn(its->dev->kvm, gfn); + srcu_read_unlock(&its->dev->kvm->srcu, idx); + return ret; +} + /* * Check whether an ID can be stored into the corresponding guest table. * For a direct table this is pretty easy, but gets a bit nasty for @@ -908,9 +920,7 @@ static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id, u64 indirect_ptr, type = GITS_BASER_TYPE(baser); phys_addr_t base = GITS_BASER_ADDR_48_to_52(baser); int esz = GITS_BASER_ENTRY_SIZE(baser); - int index, idx; - gfn_t gfn; - bool ret; + int index; switch (type) { case GITS_BASER_TYPE_DEVICE: @@ -933,12 +943,11 @@ static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id, return false; addr = base + id * esz; - gfn = addr >> PAGE_SHIFT; if (eaddr) *eaddr = addr; - goto out; + return __is_visible_gfn_locked(its, addr); } /* calculate and check the index into the 1st level */ @@ -964,16 +973,30 @@ static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id, /* Find the address of the actual entry */ index = id % (SZ_64K / esz); indirect_ptr += index * esz; - gfn = indirect_ptr >> PAGE_SHIFT; if (eaddr) *eaddr = indirect_ptr; -out: - idx = srcu_read_lock(&its->dev->kvm->srcu); - ret = kvm_is_visible_gfn(its->dev->kvm, gfn); - srcu_read_unlock(&its->dev->kvm->srcu, idx); - return ret; + return __is_visible_gfn_locked(its, indirect_ptr); +} + +/* + * Check whether an event ID can be stored in the corresponding Interrupt + * Translation Table, which starts at device->itt_addr. + */ +static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device, + u32 event_id) +{ + const struct vgic_its_abi *abi = vgic_its_get_abi(its); + int ite_esz = abi->ite_esz; + gpa_t gpa; + + /* max table size is: BIT_ULL(device->num_eventid_bits) * ite_esz */ + if (event_id >= BIT_ULL(device->num_eventid_bits)) + return false; + + gpa = device->itt_addr + event_id * ite_esz; + return __is_visible_gfn_locked(its, gpa); } static int vgic_its_alloc_collection(struct vgic_its *its, @@ -1061,7 +1084,7 @@ static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its, if (!device) return E_ITS_MAPTI_UNMAPPED_DEVICE; - if (event_id >= BIT_ULL(device->num_eventid_bits)) + if (!vgic_its_check_event_id(its, device, event_id)) return E_ITS_MAPTI_ID_OOR; if (its_cmd_get_command(its_cmd) == GITS_CMD_MAPTI) From 243b1f6c8f0748bd7b03eab17323f1187e580771 Mon Sep 17 00:00:00 2001 From: Ricardo Koller Date: Mon, 9 May 2022 17:16:31 -0700 Subject: [PATCH 261/319] KVM: arm64: vgic: Add more checks when restoring ITS tables Try to improve the predictability of ITS save/restores (and debuggability of failed ITS saves) by failing early on restore when trying to read corrupted tables. Restoring the ITS tables does some checks for corrupted tables, but not as many as in a save: an overflowing device ID will be detected on save but not on restore. The consequence is that restoring a corrupted table won't be detected until the next save; including the ITS not working as expected after the restore. As an example, if the guest sets tables overlapping each other, which would most likely result in some corrupted table, this is what we would see from the host point of view: guest sets base addresses that overlap each other save ioctl restore ioctl save ioctl (fails) Ideally, we would like the first save to fail, but overlapping tables could actually be intended by the guest. So, let's at least fail on the restore with some checks: like checking that device and event IDs don't overflow their tables. Signed-off-by: Ricardo Koller Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220510001633.552496-3-ricarkol@google.com --- arch/arm64/kvm/vgic/vgic-its.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 93a5178374c9..8a7db839e3bf 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2198,6 +2198,9 @@ static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id, if (!collection) return -EINVAL; + if (!vgic_its_check_event_id(its, dev, event_id)) + return -EINVAL; + ite = vgic_its_alloc_ite(dev, collection, event_id); if (IS_ERR(ite)) return PTR_ERR(ite); @@ -2319,6 +2322,7 @@ static int vgic_its_restore_dte(struct vgic_its *its, u32 id, void *ptr, void *opaque) { struct its_device *dev; + u64 baser = its->baser_device_table; gpa_t itt_addr; u8 num_eventid_bits; u64 entry = *(u64 *)ptr; @@ -2339,6 +2343,9 @@ static int vgic_its_restore_dte(struct vgic_its *its, u32 id, /* dte entry is valid */ offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT; + if (!vgic_its_check_id(its, baser, id, NULL)) + return -EINVAL; + dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits); if (IS_ERR(dev)) return PTR_ERR(dev); From a1ccfd6f6e06eceb632cc29c4f15a32860f05a7e Mon Sep 17 00:00:00 2001 From: Ricardo Koller Date: Mon, 9 May 2022 17:16:32 -0700 Subject: [PATCH 262/319] KVM: arm64: vgic: Do not ignore vgic_its_restore_cte failures Restoring a corrupted collection entry (like an out of range ID) is being ignored and treated as success. More specifically, a vgic_its_restore_cte failure is treated as success by vgic_its_restore_collection_table. vgic_its_restore_cte uses positive and negative numbers to return error, and +1 to return success. The caller then uses "ret > 0" to check for success. Fix this by having vgic_its_restore_cte only return negative numbers on error. Do this by changing alloc_collection return codes to only return negative numbers on error. Signed-off-by: Ricardo Koller Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220510001633.552496-4-ricarkol@google.com --- arch/arm64/kvm/vgic/vgic-its.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index 8a7db839e3bf..f34e09cc86dc 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -999,15 +999,16 @@ static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *dev return __is_visible_gfn_locked(its, gpa); } +/* + * Add a new collection into the ITS collection table. + * Returns 0 on success, and a negative error value for generic errors. + */ static int vgic_its_alloc_collection(struct vgic_its *its, struct its_collection **colp, u32 coll_id) { struct its_collection *collection; - if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) - return E_ITS_MAPC_COLLECTION_OOR; - collection = kzalloc(sizeof(*collection), GFP_KERNEL_ACCOUNT); if (!collection) return -ENOMEM; @@ -1101,7 +1102,12 @@ static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its, collection = find_collection(its, coll_id); if (!collection) { - int ret = vgic_its_alloc_collection(its, &collection, coll_id); + int ret; + + if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) + return E_ITS_MAPC_COLLECTION_OOR; + + ret = vgic_its_alloc_collection(its, &collection, coll_id); if (ret) return ret; new_coll = collection; @@ -1256,6 +1262,10 @@ static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its, if (!collection) { int ret; + if (!vgic_its_check_id(its, its->baser_coll_table, + coll_id, NULL)) + return E_ITS_MAPC_COLLECTION_OOR; + ret = vgic_its_alloc_collection(its, &collection, coll_id); if (ret) @@ -2491,6 +2501,11 @@ static int vgic_its_save_cte(struct vgic_its *its, return kvm_write_guest_lock(its->dev->kvm, gpa, &val, esz); } +/* + * Restore a collection entry into the ITS collection table. + * Return +1 on success, 0 if the entry was invalid (which should be + * interpreted as end-of-table), and a negative error value for generic errors. + */ static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz) { struct its_collection *collection; @@ -2517,6 +2532,10 @@ static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz) collection = find_collection(its, coll_id); if (collection) return -EEXIST; + + if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL)) + return -EINVAL; + ret = vgic_its_alloc_collection(its, &collection, coll_id); if (ret) return ret; From 8c5e74c90bb522181dfb051fffff3dad702e704d Mon Sep 17 00:00:00 2001 From: Ricardo Koller Date: Mon, 9 May 2022 17:16:33 -0700 Subject: [PATCH 263/319] KVM: arm64: vgic: Undo work in failed ITS restores Failed ITS restores should clean up all state restored until the failure. There is some cleanup already present when failing to restore some tables, but it's not complete. Add the missing cleanup. Note that this changes the behavior in case of a failed restore of the device tables. restore ioctl: 1. restore collection tables 2. restore device tables With this commit, failures in 2. clean up everything created so far, including state created by 1. Reviewed-by: Eric Auger Signed-off-by: Ricardo Koller Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20220510001633.552496-5-ricarkol@google.com --- arch/arm64/kvm/vgic/vgic-its.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index f34e09cc86dc..b50542c98d79 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2219,8 +2219,10 @@ static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id, vcpu = kvm_get_vcpu(kvm, collection->target_addr); irq = vgic_add_lpi(kvm, lpi_id, vcpu); - if (IS_ERR(irq)) + if (IS_ERR(irq)) { + its_free_ite(kvm, ite); return PTR_ERR(irq); + } ite->irq = irq; return offset; @@ -2485,6 +2487,9 @@ static int vgic_its_restore_device_tables(struct vgic_its *its) if (ret > 0) ret = 0; + if (ret < 0) + vgic_its_free_device_list(its->dev->kvm, its); + return ret; } @@ -2615,6 +2620,9 @@ static int vgic_its_restore_collection_table(struct vgic_its *its) if (ret > 0) return 0; + if (ret < 0) + vgic_its_free_collection_list(its->dev->kvm, its); + return ret; } @@ -2646,7 +2654,10 @@ static int vgic_its_restore_tables_v0(struct vgic_its *its) if (ret) return ret; - return vgic_its_restore_device_tables(its); + ret = vgic_its_restore_device_tables(its); + if (ret) + vgic_its_free_collection_list(its->dev->kvm, its); + return ret; } static int vgic_its_commit_v0(struct vgic_its *its) From 528ada2811ba0bb2b2db5bf0f829b48c50f3c13c Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 16 May 2022 17:32:54 +0100 Subject: [PATCH 264/319] KVM: arm64: Fix hypercall bitmap writeback when vcpus have already run We generally want to disallow hypercall bitmaps being changed once vcpus have already run. But we must allow the write if the written value is unchanged so that userspace can rewrite the register file on reboot, for example. Without this, a QEMU-based VM will fail to reboot correctly. The original code was correct, and it is me that introduced the regression. Fixes: 05714cab7d63 ("KVM: arm64: Setup a framework for hypercall bitmap firmware registers") Signed-off-by: Marc Zyngier --- arch/arm64/kvm/hypercalls.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hypercalls.c b/arch/arm64/kvm/hypercalls.c index ccbd3cefb91a..c9f401fa01a9 100644 --- a/arch/arm64/kvm/hypercalls.c +++ b/arch/arm64/kvm/hypercalls.c @@ -379,7 +379,8 @@ static int kvm_arm_set_fw_reg_bmap(struct kvm_vcpu *vcpu, u64 reg_id, u64 val) mutex_lock(&kvm->lock); - if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags)) { + if (test_bit(KVM_ARCH_FLAG_HAS_RAN_ONCE, &kvm->arch.flags) && + val != *fw_reg_bmap) { ret = -EBUSY; goto out; } From a998ec3d7bae4f996bb9dcf6d0c76b6c812f267b Mon Sep 17 00:00:00 2001 From: Wan Jiabing Date: Mon, 16 May 2022 16:50:00 +0800 Subject: [PATCH 265/319] gpio: ftgpio: Remove unneeded ERROR check before clk_disable_unprepare clk_disable_unprepare() already checks ERROR by using IS_ERR_OR_NULL. Remove unneeded ERROR check for g->clk. Signed-off-by: Wan Jiabing Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-ftgpio010.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpio-ftgpio010.c b/drivers/gpio/gpio-ftgpio010.c index b90a45c939a4..f422c3e129a0 100644 --- a/drivers/gpio/gpio-ftgpio010.c +++ b/drivers/gpio/gpio-ftgpio010.c @@ -315,8 +315,8 @@ static int ftgpio_gpio_probe(struct platform_device *pdev) return 0; dis_clk: - if (!IS_ERR(g->clk)) - clk_disable_unprepare(g->clk); + clk_disable_unprepare(g->clk); + return ret; } @@ -324,8 +324,8 @@ static int ftgpio_gpio_remove(struct platform_device *pdev) { struct ftgpio_gpio *g = platform_get_drvdata(pdev); - if (!IS_ERR(g->clk)) - clk_disable_unprepare(g->clk); + clk_disable_unprepare(g->clk); + return 0; } From ac6c85e962d4c009c499d93657f25f46fd8212b9 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Sat, 9 Apr 2022 10:02:36 +0530 Subject: [PATCH 266/319] KVM: selftests: riscv: Improve unexpected guest trap handling Currently, we simply hang using "while (1) ;" upon any unexpected guest traps because the default guest trap handler is guest_hang(). The above approach is not useful to anyone because KVM selftests users will only see a hung application upon any unexpected guest trap. This patch improves unexpected guest trap handling for KVM RISC-V selftests by doing the following: 1) Return to host user-space 2) Dump VCPU registers 3) Die using TEST_ASSERT(0, ...) Signed-off-by: Anup Patel Tested-by: Mayuresh Chitale Signed-off-by: Anup Patel --- .../selftests/kvm/include/riscv/processor.h | 8 +++-- .../selftests/kvm/lib/riscv/processor.c | 9 +++--- tools/testing/selftests/kvm/lib/riscv/ucall.c | 29 +++++++++++++------ 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/tools/testing/selftests/kvm/include/riscv/processor.h b/tools/testing/selftests/kvm/include/riscv/processor.h index eca5c622efd2..4fcfd1c0389d 100644 --- a/tools/testing/selftests/kvm/include/riscv/processor.h +++ b/tools/testing/selftests/kvm/include/riscv/processor.h @@ -119,10 +119,12 @@ static inline void set_reg(struct kvm_vm *vm, uint32_t vcpuid, uint64_t id, #define SATP_ASID_SHIFT 44 #define SATP_ASID_MASK _AC(0xFFFF, UL) -#define SBI_EXT_EXPERIMENTAL_START 0x08000000 -#define SBI_EXT_EXPERIMENTAL_END 0x08FFFFFF +#define SBI_EXT_EXPERIMENTAL_START 0x08000000 +#define SBI_EXT_EXPERIMENTAL_END 0x08FFFFFF -#define KVM_RISCV_SELFTESTS_SBI_EXT SBI_EXT_EXPERIMENTAL_END +#define KVM_RISCV_SELFTESTS_SBI_EXT SBI_EXT_EXPERIMENTAL_END +#define KVM_RISCV_SELFTESTS_SBI_UCALL 0 +#define KVM_RISCV_SELFTESTS_SBI_UNEXP 1 struct sbiret { long error; diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index 3961487a4870..56e4705f7744 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -268,10 +268,11 @@ void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent) core.regs.t3, core.regs.t4, core.regs.t5, core.regs.t6); } -static void __aligned(16) guest_hang(void) +static void __aligned(16) guest_unexp_trap(void) { - while (1) - ; + sbi_ecall(KVM_RISCV_SELFTESTS_SBI_EXT, + KVM_RISCV_SELFTESTS_SBI_UNEXP, + 0, 0, 0, 0, 0, 0); } void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) @@ -310,7 +311,7 @@ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) /* Setup default exception vector of guest */ set_reg(vm, vcpuid, RISCV_CSR_REG(stvec), - (unsigned long)guest_hang); + (unsigned long)guest_unexp_trap); } void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...) diff --git a/tools/testing/selftests/kvm/lib/riscv/ucall.c b/tools/testing/selftests/kvm/lib/riscv/ucall.c index 9e42d8248fa6..8550f424d093 100644 --- a/tools/testing/selftests/kvm/lib/riscv/ucall.c +++ b/tools/testing/selftests/kvm/lib/riscv/ucall.c @@ -60,8 +60,9 @@ void ucall(uint64_t cmd, int nargs, ...) uc.args[i] = va_arg(va, uint64_t); va_end(va); - sbi_ecall(KVM_RISCV_SELFTESTS_SBI_EXT, 0, (vm_vaddr_t)&uc, - 0, 0, 0, 0, 0); + sbi_ecall(KVM_RISCV_SELFTESTS_SBI_EXT, + KVM_RISCV_SELFTESTS_SBI_UCALL, + (vm_vaddr_t)&uc, 0, 0, 0, 0, 0); } uint64_t get_ucall(struct kvm_vm *vm, uint32_t vcpu_id, struct ucall *uc) @@ -73,14 +74,24 @@ uint64_t get_ucall(struct kvm_vm *vm, uint32_t vcpu_id, struct ucall *uc) memset(uc, 0, sizeof(*uc)); if (run->exit_reason == KVM_EXIT_RISCV_SBI && - run->riscv_sbi.extension_id == KVM_RISCV_SELFTESTS_SBI_EXT && - run->riscv_sbi.function_id == 0) { - memcpy(&ucall, addr_gva2hva(vm, run->riscv_sbi.args[0]), - sizeof(ucall)); + run->riscv_sbi.extension_id == KVM_RISCV_SELFTESTS_SBI_EXT) { + switch (run->riscv_sbi.function_id) { + case KVM_RISCV_SELFTESTS_SBI_UCALL: + memcpy(&ucall, addr_gva2hva(vm, + run->riscv_sbi.args[0]), sizeof(ucall)); - vcpu_run_complete_io(vm, vcpu_id); - if (uc) - memcpy(uc, &ucall, sizeof(ucall)); + vcpu_run_complete_io(vm, vcpu_id); + if (uc) + memcpy(uc, &ucall, sizeof(ucall)); + + break; + case KVM_RISCV_SELFTESTS_SBI_UNEXP: + vcpu_dump(stderr, vm, vcpu_id, 2); + TEST_ASSERT(0, "Unexpected trap taken by guest"); + break; + default: + break; + } } return ucall.cmd; From dba90d6fb8b0657d45516bfe1eb8fe83d9e425f8 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Fri, 6 May 2022 17:45:12 +0800 Subject: [PATCH 267/319] KVM: selftests: riscv: Remove unneeded semicolon Fix the following coccicheck warnings: ./tools/testing/selftests/kvm/lib/riscv/processor.c:353:3-4: Unneeded semicolon. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Signed-off-by: Anup Patel --- tools/testing/selftests/kvm/lib/riscv/processor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/lib/riscv/processor.c b/tools/testing/selftests/kvm/lib/riscv/processor.c index 56e4705f7744..abc0ae5a4fe1 100644 --- a/tools/testing/selftests/kvm/lib/riscv/processor.c +++ b/tools/testing/selftests/kvm/lib/riscv/processor.c @@ -351,7 +351,7 @@ void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...) case 7: id = RISCV_CORE_REG(regs.a7); break; - }; + } set_reg(vm, vcpuid, id, va_arg(ap, uint64_t)); } From 26708234eb12e73a2a475296f562ba351fb39894 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 9 May 2022 10:43:30 +0530 Subject: [PATCH 268/319] RISC-V: KVM: Use G-stage name for hypervisor page table The two-stage address translation defined by the RISC-V privileged specification defines: VS-stage (guest virtual address to guest physical address) programmed by the Guest OS and G-stage (guest physical addree to host physical address) programmed by the hypervisor. To align with above terminology, we replace "stage2" with "gstage" and "Stage2" with "G-stage" name everywhere in KVM RISC-V sources. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_host.h | 28 ++-- arch/riscv/kvm/main.c | 8 +- arch/riscv/kvm/mmu.c | 222 +++++++++++++++--------------- arch/riscv/kvm/vcpu.c | 10 +- arch/riscv/kvm/vcpu_exit.c | 6 +- arch/riscv/kvm/vm.c | 8 +- arch/riscv/kvm/vmid.c | 18 +-- 7 files changed, 150 insertions(+), 150 deletions(-) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index cd4bbcecb0fb..ecb94dddd5c8 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -54,10 +54,10 @@ struct kvm_vmid { }; struct kvm_arch { - /* stage2 vmid */ + /* G-stage vmid */ struct kvm_vmid vmid; - /* stage2 page table */ + /* G-stage page table */ pgd_t *pgd; phys_addr_t pgd_phys; @@ -207,21 +207,21 @@ void __kvm_riscv_hfence_gvma_vmid(unsigned long vmid); void __kvm_riscv_hfence_gvma_gpa(unsigned long gpa_divby_4); void __kvm_riscv_hfence_gvma_all(void); -int kvm_riscv_stage2_map(struct kvm_vcpu *vcpu, +int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, gpa_t gpa, unsigned long hva, bool is_write); -int kvm_riscv_stage2_alloc_pgd(struct kvm *kvm); -void kvm_riscv_stage2_free_pgd(struct kvm *kvm); -void kvm_riscv_stage2_update_hgatp(struct kvm_vcpu *vcpu); -void kvm_riscv_stage2_mode_detect(void); -unsigned long kvm_riscv_stage2_mode(void); -int kvm_riscv_stage2_gpa_bits(void); +int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm); +void kvm_riscv_gstage_free_pgd(struct kvm *kvm); +void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu); +void kvm_riscv_gstage_mode_detect(void); +unsigned long kvm_riscv_gstage_mode(void); +int kvm_riscv_gstage_gpa_bits(void); -void kvm_riscv_stage2_vmid_detect(void); -unsigned long kvm_riscv_stage2_vmid_bits(void); -int kvm_riscv_stage2_vmid_init(struct kvm *kvm); -bool kvm_riscv_stage2_vmid_ver_changed(struct kvm_vmid *vmid); -void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu); +void kvm_riscv_gstage_vmid_detect(void); +unsigned long kvm_riscv_gstage_vmid_bits(void); +int kvm_riscv_gstage_vmid_init(struct kvm *kvm); +bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid); +void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu); void __kvm_riscv_unpriv_trap(void); diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index 2e5ca43c8c49..c374dad82eee 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -89,13 +89,13 @@ int kvm_arch_init(void *opaque) return -ENODEV; } - kvm_riscv_stage2_mode_detect(); + kvm_riscv_gstage_mode_detect(); - kvm_riscv_stage2_vmid_detect(); + kvm_riscv_gstage_vmid_detect(); kvm_info("hypervisor extension available\n"); - switch (kvm_riscv_stage2_mode()) { + switch (kvm_riscv_gstage_mode()) { case HGATP_MODE_SV32X4: str = "Sv32x4"; break; @@ -110,7 +110,7 @@ int kvm_arch_init(void *opaque) } kvm_info("using %s G-stage page table format\n", str); - kvm_info("VMID %ld bits available\n", kvm_riscv_stage2_vmid_bits()); + kvm_info("VMID %ld bits available\n", kvm_riscv_gstage_vmid_bits()); return 0; } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index f80a34fbf102..dc0520792e31 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -21,50 +21,50 @@ #include #ifdef CONFIG_64BIT -static unsigned long stage2_mode = (HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT); -static unsigned long stage2_pgd_levels = 3; -#define stage2_index_bits 9 +static unsigned long gstage_mode = (HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT); +static unsigned long gstage_pgd_levels = 3; +#define gstage_index_bits 9 #else -static unsigned long stage2_mode = (HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT); -static unsigned long stage2_pgd_levels = 2; -#define stage2_index_bits 10 +static unsigned long gstage_mode = (HGATP_MODE_SV32X4 << HGATP_MODE_SHIFT); +static unsigned long gstage_pgd_levels = 2; +#define gstage_index_bits 10 #endif -#define stage2_pgd_xbits 2 -#define stage2_pgd_size (1UL << (HGATP_PAGE_SHIFT + stage2_pgd_xbits)) -#define stage2_gpa_bits (HGATP_PAGE_SHIFT + \ - (stage2_pgd_levels * stage2_index_bits) + \ - stage2_pgd_xbits) -#define stage2_gpa_size ((gpa_t)(1ULL << stage2_gpa_bits)) +#define gstage_pgd_xbits 2 +#define gstage_pgd_size (1UL << (HGATP_PAGE_SHIFT + gstage_pgd_xbits)) +#define gstage_gpa_bits (HGATP_PAGE_SHIFT + \ + (gstage_pgd_levels * gstage_index_bits) + \ + gstage_pgd_xbits) +#define gstage_gpa_size ((gpa_t)(1ULL << gstage_gpa_bits)) -#define stage2_pte_leaf(__ptep) \ +#define gstage_pte_leaf(__ptep) \ (pte_val(*(__ptep)) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)) -static inline unsigned long stage2_pte_index(gpa_t addr, u32 level) +static inline unsigned long gstage_pte_index(gpa_t addr, u32 level) { unsigned long mask; - unsigned long shift = HGATP_PAGE_SHIFT + (stage2_index_bits * level); + unsigned long shift = HGATP_PAGE_SHIFT + (gstage_index_bits * level); - if (level == (stage2_pgd_levels - 1)) - mask = (PTRS_PER_PTE * (1UL << stage2_pgd_xbits)) - 1; + if (level == (gstage_pgd_levels - 1)) + mask = (PTRS_PER_PTE * (1UL << gstage_pgd_xbits)) - 1; else mask = PTRS_PER_PTE - 1; return (addr >> shift) & mask; } -static inline unsigned long stage2_pte_page_vaddr(pte_t pte) +static inline unsigned long gstage_pte_page_vaddr(pte_t pte) { return (unsigned long)pfn_to_virt(pte_val(pte) >> _PAGE_PFN_SHIFT); } -static int stage2_page_size_to_level(unsigned long page_size, u32 *out_level) +static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level) { u32 i; unsigned long psz = 1UL << 12; - for (i = 0; i < stage2_pgd_levels; i++) { - if (page_size == (psz << (i * stage2_index_bits))) { + for (i = 0; i < gstage_pgd_levels; i++) { + if (page_size == (psz << (i * gstage_index_bits))) { *out_level = i; return 0; } @@ -73,27 +73,27 @@ static int stage2_page_size_to_level(unsigned long page_size, u32 *out_level) return -EINVAL; } -static int stage2_level_to_page_size(u32 level, unsigned long *out_pgsize) +static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize) { - if (stage2_pgd_levels < level) + if (gstage_pgd_levels < level) return -EINVAL; - *out_pgsize = 1UL << (12 + (level * stage2_index_bits)); + *out_pgsize = 1UL << (12 + (level * gstage_index_bits)); return 0; } -static bool stage2_get_leaf_entry(struct kvm *kvm, gpa_t addr, +static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr, pte_t **ptepp, u32 *ptep_level) { pte_t *ptep; - u32 current_level = stage2_pgd_levels - 1; + u32 current_level = gstage_pgd_levels - 1; *ptep_level = current_level; ptep = (pte_t *)kvm->arch.pgd; - ptep = &ptep[stage2_pte_index(addr, current_level)]; + ptep = &ptep[gstage_pte_index(addr, current_level)]; while (ptep && pte_val(*ptep)) { - if (stage2_pte_leaf(ptep)) { + if (gstage_pte_leaf(ptep)) { *ptep_level = current_level; *ptepp = ptep; return true; @@ -102,8 +102,8 @@ static bool stage2_get_leaf_entry(struct kvm *kvm, gpa_t addr, if (current_level) { current_level--; *ptep_level = current_level; - ptep = (pte_t *)stage2_pte_page_vaddr(*ptep); - ptep = &ptep[stage2_pte_index(addr, current_level)]; + ptep = (pte_t *)gstage_pte_page_vaddr(*ptep); + ptep = &ptep[gstage_pte_index(addr, current_level)]; } else { ptep = NULL; } @@ -112,12 +112,12 @@ static bool stage2_get_leaf_entry(struct kvm *kvm, gpa_t addr, return false; } -static void stage2_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr) +static void gstage_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr) { unsigned long size = PAGE_SIZE; struct kvm_vmid *vmid = &kvm->arch.vmid; - if (stage2_level_to_page_size(level, &size)) + if (gstage_level_to_page_size(level, &size)) return; addr &= ~(size - 1); @@ -131,19 +131,19 @@ static void stage2_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr) preempt_enable(); } -static int stage2_set_pte(struct kvm *kvm, u32 level, +static int gstage_set_pte(struct kvm *kvm, u32 level, struct kvm_mmu_memory_cache *pcache, gpa_t addr, const pte_t *new_pte) { - u32 current_level = stage2_pgd_levels - 1; + u32 current_level = gstage_pgd_levels - 1; pte_t *next_ptep = (pte_t *)kvm->arch.pgd; - pte_t *ptep = &next_ptep[stage2_pte_index(addr, current_level)]; + pte_t *ptep = &next_ptep[gstage_pte_index(addr, current_level)]; if (current_level < level) return -EINVAL; while (current_level != level) { - if (stage2_pte_leaf(ptep)) + if (gstage_pte_leaf(ptep)) return -EEXIST; if (!pte_val(*ptep)) { @@ -155,23 +155,23 @@ static int stage2_set_pte(struct kvm *kvm, u32 level, *ptep = pfn_pte(PFN_DOWN(__pa(next_ptep)), __pgprot(_PAGE_TABLE)); } else { - if (stage2_pte_leaf(ptep)) + if (gstage_pte_leaf(ptep)) return -EEXIST; - next_ptep = (pte_t *)stage2_pte_page_vaddr(*ptep); + next_ptep = (pte_t *)gstage_pte_page_vaddr(*ptep); } current_level--; - ptep = &next_ptep[stage2_pte_index(addr, current_level)]; + ptep = &next_ptep[gstage_pte_index(addr, current_level)]; } *ptep = *new_pte; - if (stage2_pte_leaf(ptep)) - stage2_remote_tlb_flush(kvm, current_level, addr); + if (gstage_pte_leaf(ptep)) + gstage_remote_tlb_flush(kvm, current_level, addr); return 0; } -static int stage2_map_page(struct kvm *kvm, +static int gstage_map_page(struct kvm *kvm, struct kvm_mmu_memory_cache *pcache, gpa_t gpa, phys_addr_t hpa, unsigned long page_size, @@ -182,7 +182,7 @@ static int stage2_map_page(struct kvm *kvm, pte_t new_pte; pgprot_t prot; - ret = stage2_page_size_to_level(page_size, &level); + ret = gstage_page_size_to_level(page_size, &level); if (ret) return ret; @@ -193,9 +193,9 @@ static int stage2_map_page(struct kvm *kvm, * PTE so that software can update these bits. * * We support both options mentioned above. To achieve this, we - * always set 'A' and 'D' PTE bits at time of creating stage2 + * always set 'A' and 'D' PTE bits at time of creating G-stage * mapping. To support KVM dirty page logging with both options - * mentioned above, we will write-protect stage2 PTEs to track + * mentioned above, we will write-protect G-stage PTEs to track * dirty pages. */ @@ -213,24 +213,24 @@ static int stage2_map_page(struct kvm *kvm, new_pte = pfn_pte(PFN_DOWN(hpa), prot); new_pte = pte_mkdirty(new_pte); - return stage2_set_pte(kvm, level, pcache, gpa, &new_pte); + return gstage_set_pte(kvm, level, pcache, gpa, &new_pte); } -enum stage2_op { - STAGE2_OP_NOP = 0, /* Nothing */ - STAGE2_OP_CLEAR, /* Clear/Unmap */ - STAGE2_OP_WP, /* Write-protect */ +enum gstage_op { + GSTAGE_OP_NOP = 0, /* Nothing */ + GSTAGE_OP_CLEAR, /* Clear/Unmap */ + GSTAGE_OP_WP, /* Write-protect */ }; -static void stage2_op_pte(struct kvm *kvm, gpa_t addr, - pte_t *ptep, u32 ptep_level, enum stage2_op op) +static void gstage_op_pte(struct kvm *kvm, gpa_t addr, + pte_t *ptep, u32 ptep_level, enum gstage_op op) { int i, ret; pte_t *next_ptep; u32 next_ptep_level; unsigned long next_page_size, page_size; - ret = stage2_level_to_page_size(ptep_level, &page_size); + ret = gstage_level_to_page_size(ptep_level, &page_size); if (ret) return; @@ -239,31 +239,31 @@ static void stage2_op_pte(struct kvm *kvm, gpa_t addr, if (!pte_val(*ptep)) return; - if (ptep_level && !stage2_pte_leaf(ptep)) { - next_ptep = (pte_t *)stage2_pte_page_vaddr(*ptep); + if (ptep_level && !gstage_pte_leaf(ptep)) { + next_ptep = (pte_t *)gstage_pte_page_vaddr(*ptep); next_ptep_level = ptep_level - 1; - ret = stage2_level_to_page_size(next_ptep_level, + ret = gstage_level_to_page_size(next_ptep_level, &next_page_size); if (ret) return; - if (op == STAGE2_OP_CLEAR) + if (op == GSTAGE_OP_CLEAR) set_pte(ptep, __pte(0)); for (i = 0; i < PTRS_PER_PTE; i++) - stage2_op_pte(kvm, addr + i * next_page_size, + gstage_op_pte(kvm, addr + i * next_page_size, &next_ptep[i], next_ptep_level, op); - if (op == STAGE2_OP_CLEAR) + if (op == GSTAGE_OP_CLEAR) put_page(virt_to_page(next_ptep)); } else { - if (op == STAGE2_OP_CLEAR) + if (op == GSTAGE_OP_CLEAR) set_pte(ptep, __pte(0)); - else if (op == STAGE2_OP_WP) + else if (op == GSTAGE_OP_WP) set_pte(ptep, __pte(pte_val(*ptep) & ~_PAGE_WRITE)); - stage2_remote_tlb_flush(kvm, ptep_level, addr); + gstage_remote_tlb_flush(kvm, ptep_level, addr); } } -static void stage2_unmap_range(struct kvm *kvm, gpa_t start, +static void gstage_unmap_range(struct kvm *kvm, gpa_t start, gpa_t size, bool may_block) { int ret; @@ -274,9 +274,9 @@ static void stage2_unmap_range(struct kvm *kvm, gpa_t start, gpa_t addr = start, end = start + size; while (addr < end) { - found_leaf = stage2_get_leaf_entry(kvm, addr, + found_leaf = gstage_get_leaf_entry(kvm, addr, &ptep, &ptep_level); - ret = stage2_level_to_page_size(ptep_level, &page_size); + ret = gstage_level_to_page_size(ptep_level, &page_size); if (ret) break; @@ -284,8 +284,8 @@ static void stage2_unmap_range(struct kvm *kvm, gpa_t start, goto next; if (!(addr & (page_size - 1)) && ((end - addr) >= page_size)) - stage2_op_pte(kvm, addr, ptep, - ptep_level, STAGE2_OP_CLEAR); + gstage_op_pte(kvm, addr, ptep, + ptep_level, GSTAGE_OP_CLEAR); next: addr += page_size; @@ -299,7 +299,7 @@ static void stage2_unmap_range(struct kvm *kvm, gpa_t start, } } -static void stage2_wp_range(struct kvm *kvm, gpa_t start, gpa_t end) +static void gstage_wp_range(struct kvm *kvm, gpa_t start, gpa_t end) { int ret; pte_t *ptep; @@ -309,9 +309,9 @@ static void stage2_wp_range(struct kvm *kvm, gpa_t start, gpa_t end) unsigned long page_size; while (addr < end) { - found_leaf = stage2_get_leaf_entry(kvm, addr, + found_leaf = gstage_get_leaf_entry(kvm, addr, &ptep, &ptep_level); - ret = stage2_level_to_page_size(ptep_level, &page_size); + ret = gstage_level_to_page_size(ptep_level, &page_size); if (ret) break; @@ -319,15 +319,15 @@ static void stage2_wp_range(struct kvm *kvm, gpa_t start, gpa_t end) goto next; if (!(addr & (page_size - 1)) && ((end - addr) >= page_size)) - stage2_op_pte(kvm, addr, ptep, - ptep_level, STAGE2_OP_WP); + gstage_op_pte(kvm, addr, ptep, + ptep_level, GSTAGE_OP_WP); next: addr += page_size; } } -static void stage2_wp_memory_region(struct kvm *kvm, int slot) +static void gstage_wp_memory_region(struct kvm *kvm, int slot) { struct kvm_memslots *slots = kvm_memslots(kvm); struct kvm_memory_slot *memslot = id_to_memslot(slots, slot); @@ -335,12 +335,12 @@ static void stage2_wp_memory_region(struct kvm *kvm, int slot) phys_addr_t end = (memslot->base_gfn + memslot->npages) << PAGE_SHIFT; spin_lock(&kvm->mmu_lock); - stage2_wp_range(kvm, start, end); + gstage_wp_range(kvm, start, end); spin_unlock(&kvm->mmu_lock); kvm_flush_remote_tlbs(kvm); } -static int stage2_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, +static int gstage_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, unsigned long size, bool writable) { pte_t pte; @@ -361,12 +361,12 @@ static int stage2_ioremap(struct kvm *kvm, gpa_t gpa, phys_addr_t hpa, if (!writable) pte = pte_wrprotect(pte); - ret = kvm_mmu_topup_memory_cache(&pcache, stage2_pgd_levels); + ret = kvm_mmu_topup_memory_cache(&pcache, gstage_pgd_levels); if (ret) goto out; spin_lock(&kvm->mmu_lock); - ret = stage2_set_pte(kvm, 0, &pcache, addr, &pte); + ret = gstage_set_pte(kvm, 0, &pcache, addr, &pte); spin_unlock(&kvm->mmu_lock); if (ret) goto out; @@ -388,7 +388,7 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, phys_addr_t start = (base_gfn + __ffs(mask)) << PAGE_SHIFT; phys_addr_t end = (base_gfn + __fls(mask) + 1) << PAGE_SHIFT; - stage2_wp_range(kvm, start, end); + gstage_wp_range(kvm, start, end); } void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) @@ -411,7 +411,7 @@ void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) void kvm_arch_flush_shadow_all(struct kvm *kvm) { - kvm_riscv_stage2_free_pgd(kvm); + kvm_riscv_gstage_free_pgd(kvm); } void kvm_arch_flush_shadow_memslot(struct kvm *kvm, @@ -421,7 +421,7 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, phys_addr_t size = slot->npages << PAGE_SHIFT; spin_lock(&kvm->mmu_lock); - stage2_unmap_range(kvm, gpa, size, false); + gstage_unmap_range(kvm, gpa, size, false); spin_unlock(&kvm->mmu_lock); } @@ -436,7 +436,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, * the memory slot is write protected. */ if (change != KVM_MR_DELETE && new->flags & KVM_MEM_LOG_DIRTY_PAGES) - stage2_wp_memory_region(kvm, new->id); + gstage_wp_memory_region(kvm, new->id); } int kvm_arch_prepare_memory_region(struct kvm *kvm, @@ -458,7 +458,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, * space addressable by the KVM guest GPA space. */ if ((new->base_gfn + new->npages) >= - (stage2_gpa_size >> PAGE_SHIFT)) + (gstage_gpa_size >> PAGE_SHIFT)) return -EFAULT; hva = new->userspace_addr; @@ -514,7 +514,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, goto out; } - ret = stage2_ioremap(kvm, gpa, pa, + ret = gstage_ioremap(kvm, gpa, pa, vm_end - vm_start, writable); if (ret) break; @@ -527,7 +527,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, spin_lock(&kvm->mmu_lock); if (ret) - stage2_unmap_range(kvm, base_gpa, size, false); + gstage_unmap_range(kvm, base_gpa, size, false); spin_unlock(&kvm->mmu_lock); out: @@ -540,7 +540,7 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) if (!kvm->arch.pgd) return false; - stage2_unmap_range(kvm, range->start << PAGE_SHIFT, + gstage_unmap_range(kvm, range->start << PAGE_SHIFT, (range->end - range->start) << PAGE_SHIFT, range->may_block); return false; @@ -556,10 +556,10 @@ bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) WARN_ON(range->end - range->start != 1); - ret = stage2_map_page(kvm, NULL, range->start << PAGE_SHIFT, + ret = gstage_map_page(kvm, NULL, range->start << PAGE_SHIFT, __pfn_to_phys(pfn), PAGE_SIZE, true, true); if (ret) { - kvm_debug("Failed to map stage2 page (error %d)\n", ret); + kvm_debug("Failed to map G-stage page (error %d)\n", ret); return true; } @@ -577,7 +577,7 @@ bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PGDIR_SIZE); - if (!stage2_get_leaf_entry(kvm, range->start << PAGE_SHIFT, + if (!gstage_get_leaf_entry(kvm, range->start << PAGE_SHIFT, &ptep, &ptep_level)) return false; @@ -595,14 +595,14 @@ bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) WARN_ON(size != PAGE_SIZE && size != PMD_SIZE && size != PGDIR_SIZE); - if (!stage2_get_leaf_entry(kvm, range->start << PAGE_SHIFT, + if (!gstage_get_leaf_entry(kvm, range->start << PAGE_SHIFT, &ptep, &ptep_level)) return false; return pte_young(*ptep); } -int kvm_riscv_stage2_map(struct kvm_vcpu *vcpu, +int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, gpa_t gpa, unsigned long hva, bool is_write) { @@ -648,9 +648,9 @@ int kvm_riscv_stage2_map(struct kvm_vcpu *vcpu, } /* We need minimum second+third level pages */ - ret = kvm_mmu_topup_memory_cache(pcache, stage2_pgd_levels); + ret = kvm_mmu_topup_memory_cache(pcache, gstage_pgd_levels); if (ret) { - kvm_err("Failed to topup stage2 cache\n"); + kvm_err("Failed to topup G-stage cache\n"); return ret; } @@ -680,15 +680,15 @@ int kvm_riscv_stage2_map(struct kvm_vcpu *vcpu, if (writeable) { kvm_set_pfn_dirty(hfn); mark_page_dirty(kvm, gfn); - ret = stage2_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT, + ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT, vma_pagesize, false, true); } else { - ret = stage2_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT, + ret = gstage_map_page(kvm, pcache, gpa, hfn << PAGE_SHIFT, vma_pagesize, true, true); } if (ret) - kvm_err("Failed to map in stage2\n"); + kvm_err("Failed to map in G-stage\n"); out_unlock: spin_unlock(&kvm->mmu_lock); @@ -697,7 +697,7 @@ int kvm_riscv_stage2_map(struct kvm_vcpu *vcpu, return ret; } -int kvm_riscv_stage2_alloc_pgd(struct kvm *kvm) +int kvm_riscv_gstage_alloc_pgd(struct kvm *kvm) { struct page *pgd_page; @@ -707,7 +707,7 @@ int kvm_riscv_stage2_alloc_pgd(struct kvm *kvm) } pgd_page = alloc_pages(GFP_KERNEL | __GFP_ZERO, - get_order(stage2_pgd_size)); + get_order(gstage_pgd_size)); if (!pgd_page) return -ENOMEM; kvm->arch.pgd = page_to_virt(pgd_page); @@ -716,13 +716,13 @@ int kvm_riscv_stage2_alloc_pgd(struct kvm *kvm) return 0; } -void kvm_riscv_stage2_free_pgd(struct kvm *kvm) +void kvm_riscv_gstage_free_pgd(struct kvm *kvm) { void *pgd = NULL; spin_lock(&kvm->mmu_lock); if (kvm->arch.pgd) { - stage2_unmap_range(kvm, 0UL, stage2_gpa_size, false); + gstage_unmap_range(kvm, 0UL, gstage_gpa_size, false); pgd = READ_ONCE(kvm->arch.pgd); kvm->arch.pgd = NULL; kvm->arch.pgd_phys = 0; @@ -730,12 +730,12 @@ void kvm_riscv_stage2_free_pgd(struct kvm *kvm) spin_unlock(&kvm->mmu_lock); if (pgd) - free_pages((unsigned long)pgd, get_order(stage2_pgd_size)); + free_pages((unsigned long)pgd, get_order(gstage_pgd_size)); } -void kvm_riscv_stage2_update_hgatp(struct kvm_vcpu *vcpu) +void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu) { - unsigned long hgatp = stage2_mode; + unsigned long hgatp = gstage_mode; struct kvm_arch *k = &vcpu->kvm->arch; hgatp |= (READ_ONCE(k->vmid.vmid) << HGATP_VMID_SHIFT) & @@ -744,18 +744,18 @@ void kvm_riscv_stage2_update_hgatp(struct kvm_vcpu *vcpu) csr_write(CSR_HGATP, hgatp); - if (!kvm_riscv_stage2_vmid_bits()) + if (!kvm_riscv_gstage_vmid_bits()) __kvm_riscv_hfence_gvma_all(); } -void kvm_riscv_stage2_mode_detect(void) +void kvm_riscv_gstage_mode_detect(void) { #ifdef CONFIG_64BIT - /* Try Sv48x4 stage2 mode */ + /* Try Sv48x4 G-stage mode */ csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) { - stage2_mode = (HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); - stage2_pgd_levels = 4; + gstage_mode = (HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); + gstage_pgd_levels = 4; } csr_write(CSR_HGATP, 0); @@ -763,12 +763,12 @@ void kvm_riscv_stage2_mode_detect(void) #endif } -unsigned long kvm_riscv_stage2_mode(void) +unsigned long kvm_riscv_gstage_mode(void) { - return stage2_mode >> HGATP_MODE_SHIFT; + return gstage_mode >> HGATP_MODE_SHIFT; } -int kvm_riscv_stage2_gpa_bits(void) +int kvm_riscv_gstage_gpa_bits(void) { - return stage2_gpa_bits; + return gstage_gpa_bits; } diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 7461f964d20a..f8582599c7b2 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -137,7 +137,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) /* Cleanup VCPU timer */ kvm_riscv_vcpu_timer_deinit(vcpu); - /* Free unused pages pre-allocated for Stage2 page table mappings */ + /* Free unused pages pre-allocated for G-stage page table mappings */ kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); } @@ -635,7 +635,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) csr_write(CSR_HVIP, csr->hvip); csr_write(CSR_VSATP, csr->vsatp); - kvm_riscv_stage2_update_hgatp(vcpu); + kvm_riscv_gstage_update_hgatp(vcpu); kvm_riscv_vcpu_timer_restore(vcpu); @@ -690,7 +690,7 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) kvm_riscv_reset_vcpu(vcpu); if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) - kvm_riscv_stage2_update_hgatp(vcpu); + kvm_riscv_gstage_update_hgatp(vcpu); if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) __kvm_riscv_hfence_gvma_all(); @@ -762,7 +762,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) /* Check conditions before entering the guest */ cond_resched(); - kvm_riscv_stage2_vmid_update(vcpu); + kvm_riscv_gstage_vmid_update(vcpu); kvm_riscv_check_vcpu_requests(vcpu); @@ -800,7 +800,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) kvm_riscv_update_hvip(vcpu); if (ret <= 0 || - kvm_riscv_stage2_vmid_ver_changed(&vcpu->kvm->arch.vmid) || + kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || kvm_request_pending(vcpu)) { vcpu->mode = OUTSIDE_GUEST_MODE; local_irq_enable(); diff --git a/arch/riscv/kvm/vcpu_exit.c b/arch/riscv/kvm/vcpu_exit.c index a72c15d4b42a..dbb09afd7546 100644 --- a/arch/riscv/kvm/vcpu_exit.c +++ b/arch/riscv/kvm/vcpu_exit.c @@ -412,7 +412,7 @@ static int emulate_store(struct kvm_vcpu *vcpu, struct kvm_run *run, return 0; } -static int stage2_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, +static int gstage_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, struct kvm_cpu_trap *trap) { struct kvm_memory_slot *memslot; @@ -440,7 +440,7 @@ static int stage2_page_fault(struct kvm_vcpu *vcpu, struct kvm_run *run, }; } - ret = kvm_riscv_stage2_map(vcpu, memslot, fault_addr, hva, + ret = kvm_riscv_gstage_map(vcpu, memslot, fault_addr, hva, (trap->scause == EXC_STORE_GUEST_PAGE_FAULT) ? true : false); if (ret < 0) return ret; @@ -686,7 +686,7 @@ int kvm_riscv_vcpu_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, case EXC_LOAD_GUEST_PAGE_FAULT: case EXC_STORE_GUEST_PAGE_FAULT: if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) - ret = stage2_page_fault(vcpu, run, trap); + ret = gstage_page_fault(vcpu, run, trap); break; case EXC_SUPERVISOR_SYSCALL: if (vcpu->arch.guest_context.hstatus & HSTATUS_SPV) diff --git a/arch/riscv/kvm/vm.c b/arch/riscv/kvm/vm.c index c768f75279ef..945a2bf5e3f6 100644 --- a/arch/riscv/kvm/vm.c +++ b/arch/riscv/kvm/vm.c @@ -31,13 +31,13 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { int r; - r = kvm_riscv_stage2_alloc_pgd(kvm); + r = kvm_riscv_gstage_alloc_pgd(kvm); if (r) return r; - r = kvm_riscv_stage2_vmid_init(kvm); + r = kvm_riscv_gstage_vmid_init(kvm); if (r) { - kvm_riscv_stage2_free_pgd(kvm); + kvm_riscv_gstage_free_pgd(kvm); return r; } @@ -75,7 +75,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = KVM_USER_MEM_SLOTS; break; case KVM_CAP_VM_GPA_BITS: - r = kvm_riscv_stage2_gpa_bits(); + r = kvm_riscv_gstage_gpa_bits(); break; default: r = 0; diff --git a/arch/riscv/kvm/vmid.c b/arch/riscv/kvm/vmid.c index 2fa4f7b1813d..01fdc342ad76 100644 --- a/arch/riscv/kvm/vmid.c +++ b/arch/riscv/kvm/vmid.c @@ -20,7 +20,7 @@ static unsigned long vmid_next; static unsigned long vmid_bits; static DEFINE_SPINLOCK(vmid_lock); -void kvm_riscv_stage2_vmid_detect(void) +void kvm_riscv_gstage_vmid_detect(void) { unsigned long old; @@ -40,12 +40,12 @@ void kvm_riscv_stage2_vmid_detect(void) vmid_bits = 0; } -unsigned long kvm_riscv_stage2_vmid_bits(void) +unsigned long kvm_riscv_gstage_vmid_bits(void) { return vmid_bits; } -int kvm_riscv_stage2_vmid_init(struct kvm *kvm) +int kvm_riscv_gstage_vmid_init(struct kvm *kvm) { /* Mark the initial VMID and VMID version invalid */ kvm->arch.vmid.vmid_version = 0; @@ -54,7 +54,7 @@ int kvm_riscv_stage2_vmid_init(struct kvm *kvm) return 0; } -bool kvm_riscv_stage2_vmid_ver_changed(struct kvm_vmid *vmid) +bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid) { if (!vmid_bits) return false; @@ -63,13 +63,13 @@ bool kvm_riscv_stage2_vmid_ver_changed(struct kvm_vmid *vmid) READ_ONCE(vmid_version)); } -void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu) +void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu) { unsigned long i; struct kvm_vcpu *v; struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid; - if (!kvm_riscv_stage2_vmid_ver_changed(vmid)) + if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) return; spin_lock(&vmid_lock); @@ -78,7 +78,7 @@ void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu) * We need to re-check the vmid_version here to ensure that if * another vcpu already allocated a valid vmid for this vm. */ - if (!kvm_riscv_stage2_vmid_ver_changed(vmid)) { + if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) { spin_unlock(&vmid_lock); return; } @@ -96,7 +96,7 @@ void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu) * instances is invalid and we have force VMID re-assignement * for all Guest instances. The Guest instances that were not * running will automatically pick-up new VMIDs because will - * call kvm_riscv_stage2_vmid_update() whenever they enter + * call kvm_riscv_gstage_vmid_update() whenever they enter * in-kernel run loop. For Guest instances that are already * running, we force VM exits on all host CPUs using IPI and * flush all Guest TLBs. @@ -112,7 +112,7 @@ void kvm_riscv_stage2_vmid_update(struct kvm_vcpu *vcpu) spin_unlock(&vmid_lock); - /* Request stage2 page table update for all VCPUs */ + /* Request G-stage page table update for all VCPUs */ kvm_for_each_vcpu(i, v, vcpu->kvm) kvm_make_request(KVM_REQ_UPDATE_HGATP, v); } From b4bbb95ea6c41a616385902df81a66293f4b744c Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 9 May 2022 10:43:39 +0530 Subject: [PATCH 269/319] RISC-V: KVM: Add Sv57x4 mode support for G-stage Latest QEMU supports G-stage Sv57x4 mode so this patch extends KVM RISC-V G-stage handling to detect and use Sv57x4 mode when available. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Anup Patel --- arch/riscv/include/asm/csr.h | 1 + arch/riscv/kvm/main.c | 3 +++ arch/riscv/kvm/mmu.c | 11 ++++++++++- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index e935f27b10fd..cc40521e438b 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -117,6 +117,7 @@ #define HGATP_MODE_SV32X4 _AC(1, UL) #define HGATP_MODE_SV39X4 _AC(8, UL) #define HGATP_MODE_SV48X4 _AC(9, UL) +#define HGATP_MODE_SV57X4 _AC(10, UL) #define HGATP32_MODE_SHIFT 31 #define HGATP32_VMID_SHIFT 22 diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index c374dad82eee..1549205fe5fe 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -105,6 +105,9 @@ int kvm_arch_init(void *opaque) case HGATP_MODE_SV48X4: str = "Sv48x4"; break; + case HGATP_MODE_SV57X4: + str = "Sv57x4"; + break; default: return -ENODEV; } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index dc0520792e31..8823eb32dcde 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -751,14 +751,23 @@ void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu) void kvm_riscv_gstage_mode_detect(void) { #ifdef CONFIG_64BIT + /* Try Sv57x4 G-stage mode */ + csr_write(CSR_HGATP, HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT); + if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV57X4) { + gstage_mode = (HGATP_MODE_SV57X4 << HGATP_MODE_SHIFT); + gstage_pgd_levels = 5; + goto skip_sv48x4_test; + } + /* Try Sv48x4 G-stage mode */ csr_write(CSR_HGATP, HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); if ((csr_read(CSR_HGATP) >> HGATP_MODE_SHIFT) == HGATP_MODE_SV48X4) { gstage_mode = (HGATP_MODE_SV48X4 << HGATP_MODE_SHIFT); gstage_pgd_levels = 4; } - csr_write(CSR_HGATP, 0); +skip_sv48x4_test: + csr_write(CSR_HGATP, 0); __kvm_riscv_hfence_gvma_all(); #endif } From c7fa3c48de86053b0f1949fa5532082544e30fce Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 9 May 2022 10:43:46 +0530 Subject: [PATCH 270/319] RISC-V: KVM: Treat SBI HFENCE calls as NOPs We should treat SBI HFENCE calls as NOPs until nested virtualization is supported by KVM RISC-V. This will help us test booting a hypervisor under KVM RISC-V. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Anup Patel --- arch/riscv/kvm/vcpu_sbi_replace.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/riscv/kvm/vcpu_sbi_replace.c b/arch/riscv/kvm/vcpu_sbi_replace.c index 0f217365c287..3c1dcd38358e 100644 --- a/arch/riscv/kvm/vcpu_sbi_replace.c +++ b/arch/riscv/kvm/vcpu_sbi_replace.c @@ -117,7 +117,11 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: - /* TODO: implement for nested hypervisor case */ + /* + * Until nested virtualization is implemented, the + * SBI HFENCE calls should be treated as NOPs + */ + break; default: ret = -EOPNOTSUPP; } From 2415e46e3a9a22c80eddc41dc9cb0b06fe0cd5e9 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 9 May 2022 10:43:52 +0530 Subject: [PATCH 271/319] RISC-V: KVM: Introduce range based local HFENCE functions Various __kvm_riscv_hfence_xyz() functions implemented in the kvm/tlb.S are equivalent to corresponding HFENCE.GVMA instructions and we don't have range based local HFENCE functions. This patch provides complete set of local HFENCE functions which supports range based TLB invalidation and supports HFENCE.VVMA based functions. This is also a preparatory patch for upcoming Svinval support in KVM RISC-V. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_host.h | 25 +++- arch/riscv/kvm/mmu.c | 4 +- arch/riscv/kvm/tlb.S | 74 ----------- arch/riscv/kvm/tlb.c | 213 ++++++++++++++++++++++++++++++ arch/riscv/kvm/vcpu.c | 2 +- arch/riscv/kvm/vmid.c | 2 +- 6 files changed, 237 insertions(+), 83 deletions(-) delete mode 100644 arch/riscv/kvm/tlb.S create mode 100644 arch/riscv/kvm/tlb.c diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index ecb94dddd5c8..d5c2057f8451 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -201,11 +201,26 @@ static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} #define KVM_ARCH_WANT_MMU_NOTIFIER -void __kvm_riscv_hfence_gvma_vmid_gpa(unsigned long gpa_divby_4, - unsigned long vmid); -void __kvm_riscv_hfence_gvma_vmid(unsigned long vmid); -void __kvm_riscv_hfence_gvma_gpa(unsigned long gpa_divby_4); -void __kvm_riscv_hfence_gvma_all(void); +#define KVM_RISCV_GSTAGE_TLB_MIN_ORDER 12 + +void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, + gpa_t gpa, gpa_t gpsz, + unsigned long order); +void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid); +void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, + unsigned long order); +void kvm_riscv_local_hfence_gvma_all(void); +void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, + unsigned long asid, + unsigned long gva, + unsigned long gvsz, + unsigned long order); +void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, + unsigned long asid); +void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, + unsigned long gva, unsigned long gvsz, + unsigned long order); +void kvm_riscv_local_hfence_vvma_all(unsigned long vmid); int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 8823eb32dcde..1e07603c905b 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -745,7 +745,7 @@ void kvm_riscv_gstage_update_hgatp(struct kvm_vcpu *vcpu) csr_write(CSR_HGATP, hgatp); if (!kvm_riscv_gstage_vmid_bits()) - __kvm_riscv_hfence_gvma_all(); + kvm_riscv_local_hfence_gvma_all(); } void kvm_riscv_gstage_mode_detect(void) @@ -768,7 +768,7 @@ void kvm_riscv_gstage_mode_detect(void) skip_sv48x4_test: csr_write(CSR_HGATP, 0); - __kvm_riscv_hfence_gvma_all(); + kvm_riscv_local_hfence_gvma_all(); #endif } diff --git a/arch/riscv/kvm/tlb.S b/arch/riscv/kvm/tlb.S deleted file mode 100644 index 899f75d60bad..000000000000 --- a/arch/riscv/kvm/tlb.S +++ /dev/null @@ -1,74 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2019 Western Digital Corporation or its affiliates. - * - * Authors: - * Anup Patel - */ - -#include -#include - - .text - .altmacro - .option norelax - - /* - * Instruction encoding of hfence.gvma is: - * HFENCE.GVMA rs1, rs2 - * HFENCE.GVMA zero, rs2 - * HFENCE.GVMA rs1 - * HFENCE.GVMA - * - * rs1!=zero and rs2!=zero ==> HFENCE.GVMA rs1, rs2 - * rs1==zero and rs2!=zero ==> HFENCE.GVMA zero, rs2 - * rs1!=zero and rs2==zero ==> HFENCE.GVMA rs1 - * rs1==zero and rs2==zero ==> HFENCE.GVMA - * - * Instruction encoding of HFENCE.GVMA is: - * 0110001 rs2(5) rs1(5) 000 00000 1110011 - */ - -ENTRY(__kvm_riscv_hfence_gvma_vmid_gpa) - /* - * rs1 = a0 (GPA >> 2) - * rs2 = a1 (VMID) - * HFENCE.GVMA a0, a1 - * 0110001 01011 01010 000 00000 1110011 - */ - .word 0x62b50073 - ret -ENDPROC(__kvm_riscv_hfence_gvma_vmid_gpa) - -ENTRY(__kvm_riscv_hfence_gvma_vmid) - /* - * rs1 = zero - * rs2 = a0 (VMID) - * HFENCE.GVMA zero, a0 - * 0110001 01010 00000 000 00000 1110011 - */ - .word 0x62a00073 - ret -ENDPROC(__kvm_riscv_hfence_gvma_vmid) - -ENTRY(__kvm_riscv_hfence_gvma_gpa) - /* - * rs1 = a0 (GPA >> 2) - * rs2 = zero - * HFENCE.GVMA a0 - * 0110001 00000 01010 000 00000 1110011 - */ - .word 0x62050073 - ret -ENDPROC(__kvm_riscv_hfence_gvma_gpa) - -ENTRY(__kvm_riscv_hfence_gvma_all) - /* - * rs1 = zero - * rs2 = zero - * HFENCE.GVMA - * 0110001 00000 00000 000 00000 1110011 - */ - .word 0x62000073 - ret -ENDPROC(__kvm_riscv_hfence_gvma_all) diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c new file mode 100644 index 000000000000..e2d4fd610745 --- /dev/null +++ b/arch/riscv/kvm/tlb.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#include +#include +#include +#include +#include +#include + +/* + * Instruction encoding of hfence.gvma is: + * HFENCE.GVMA rs1, rs2 + * HFENCE.GVMA zero, rs2 + * HFENCE.GVMA rs1 + * HFENCE.GVMA + * + * rs1!=zero and rs2!=zero ==> HFENCE.GVMA rs1, rs2 + * rs1==zero and rs2!=zero ==> HFENCE.GVMA zero, rs2 + * rs1!=zero and rs2==zero ==> HFENCE.GVMA rs1 + * rs1==zero and rs2==zero ==> HFENCE.GVMA + * + * Instruction encoding of HFENCE.GVMA is: + * 0110001 rs2(5) rs1(5) 000 00000 1110011 + */ + +void kvm_riscv_local_hfence_gvma_vmid_gpa(unsigned long vmid, + gpa_t gpa, gpa_t gpsz, + unsigned long order) +{ + gpa_t pos; + + if (PTRS_PER_PTE < (gpsz >> order)) { + kvm_riscv_local_hfence_gvma_vmid_all(vmid); + return; + } + + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) { + /* + * rs1 = a0 (GPA >> 2) + * rs2 = a1 (VMID) + * HFENCE.GVMA a0, a1 + * 0110001 01011 01010 000 00000 1110011 + */ + asm volatile ("srli a0, %0, 2\n" + "add a1, %1, zero\n" + ".word 0x62b50073\n" + :: "r" (pos), "r" (vmid) + : "a0", "a1", "memory"); + } +} + +void kvm_riscv_local_hfence_gvma_vmid_all(unsigned long vmid) +{ + /* + * rs1 = zero + * rs2 = a0 (VMID) + * HFENCE.GVMA zero, a0 + * 0110001 01010 00000 000 00000 1110011 + */ + asm volatile ("add a0, %0, zero\n" + ".word 0x62a00073\n" + :: "r" (vmid) : "a0", "memory"); +} + +void kvm_riscv_local_hfence_gvma_gpa(gpa_t gpa, gpa_t gpsz, + unsigned long order) +{ + gpa_t pos; + + if (PTRS_PER_PTE < (gpsz >> order)) { + kvm_riscv_local_hfence_gvma_all(); + return; + } + + for (pos = gpa; pos < (gpa + gpsz); pos += BIT(order)) { + /* + * rs1 = a0 (GPA >> 2) + * rs2 = zero + * HFENCE.GVMA a0 + * 0110001 00000 01010 000 00000 1110011 + */ + asm volatile ("srli a0, %0, 2\n" + ".word 0x62050073\n" + :: "r" (pos) : "a0", "memory"); + } +} + +void kvm_riscv_local_hfence_gvma_all(void) +{ + /* + * rs1 = zero + * rs2 = zero + * HFENCE.GVMA + * 0110001 00000 00000 000 00000 1110011 + */ + asm volatile (".word 0x62000073" ::: "memory"); +} + +/* + * Instruction encoding of hfence.gvma is: + * HFENCE.VVMA rs1, rs2 + * HFENCE.VVMA zero, rs2 + * HFENCE.VVMA rs1 + * HFENCE.VVMA + * + * rs1!=zero and rs2!=zero ==> HFENCE.VVMA rs1, rs2 + * rs1==zero and rs2!=zero ==> HFENCE.VVMA zero, rs2 + * rs1!=zero and rs2==zero ==> HFENCE.VVMA rs1 + * rs1==zero and rs2==zero ==> HFENCE.VVMA + * + * Instruction encoding of HFENCE.VVMA is: + * 0010001 rs2(5) rs1(5) 000 00000 1110011 + */ + +void kvm_riscv_local_hfence_vvma_asid_gva(unsigned long vmid, + unsigned long asid, + unsigned long gva, + unsigned long gvsz, + unsigned long order) +{ + unsigned long pos, hgatp; + + if (PTRS_PER_PTE < (gvsz >> order)) { + kvm_riscv_local_hfence_vvma_asid_all(vmid, asid); + return; + } + + hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); + + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) { + /* + * rs1 = a0 (GVA) + * rs2 = a1 (ASID) + * HFENCE.VVMA a0, a1 + * 0010001 01011 01010 000 00000 1110011 + */ + asm volatile ("add a0, %0, zero\n" + "add a1, %1, zero\n" + ".word 0x22b50073\n" + :: "r" (pos), "r" (asid) + : "a0", "a1", "memory"); + } + + csr_write(CSR_HGATP, hgatp); +} + +void kvm_riscv_local_hfence_vvma_asid_all(unsigned long vmid, + unsigned long asid) +{ + unsigned long hgatp; + + hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); + + /* + * rs1 = zero + * rs2 = a0 (ASID) + * HFENCE.VVMA zero, a0 + * 0010001 01010 00000 000 00000 1110011 + */ + asm volatile ("add a0, %0, zero\n" + ".word 0x22a00073\n" + :: "r" (asid) : "a0", "memory"); + + csr_write(CSR_HGATP, hgatp); +} + +void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, + unsigned long gva, unsigned long gvsz, + unsigned long order) +{ + unsigned long pos, hgatp; + + if (PTRS_PER_PTE < (gvsz >> order)) { + kvm_riscv_local_hfence_vvma_all(vmid); + return; + } + + hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); + + for (pos = gva; pos < (gva + gvsz); pos += BIT(order)) { + /* + * rs1 = a0 (GVA) + * rs2 = zero + * HFENCE.VVMA a0 + * 0010001 00000 01010 000 00000 1110011 + */ + asm volatile ("add a0, %0, zero\n" + ".word 0x22050073\n" + :: "r" (pos) : "a0", "memory"); + } + + csr_write(CSR_HGATP, hgatp); +} + +void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) +{ + unsigned long hgatp; + + hgatp = csr_swap(CSR_HGATP, vmid << HGATP_VMID_SHIFT); + + /* + * rs1 = zero + * rs2 = zero + * HFENCE.VVMA + * 0010001 00000 00000 000 00000 1110011 + */ + asm volatile (".word 0x22000073" ::: "memory"); + + csr_write(CSR_HGATP, hgatp); +} diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index f8582599c7b2..24cf3bb4be3a 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -693,7 +693,7 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) kvm_riscv_gstage_update_hgatp(vcpu); if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) - __kvm_riscv_hfence_gvma_all(); + kvm_riscv_local_hfence_gvma_all(); } } diff --git a/arch/riscv/kvm/vmid.c b/arch/riscv/kvm/vmid.c index 01fdc342ad76..8987e76aa6db 100644 --- a/arch/riscv/kvm/vmid.c +++ b/arch/riscv/kvm/vmid.c @@ -33,7 +33,7 @@ void kvm_riscv_gstage_vmid_detect(void) csr_write(CSR_HGATP, old); /* We polluted local TLB so flush all guest TLB */ - __kvm_riscv_hfence_gvma_all(); + kvm_riscv_local_hfence_gvma_all(); /* We don't use VMID bits if they are not sufficient */ if ((1UL << vmid_bits) < num_possible_cpus()) From 486a38429498eef5acac90aeab68a1c3fa653a21 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 9 May 2022 10:44:00 +0530 Subject: [PATCH 272/319] RISC-V: KVM: Reduce KVM_MAX_VCPUS value Currently, the KVM_MAX_VCPUS value is 16384 for RV64 and 128 for RV32. The KVM_MAX_VCPUS value is too high for RV64 and too low for RV32 compared to other architectures (e.g. x86 sets it to 1024 and ARM64 sets it to 512). The too high value of KVM_MAX_VCPUS on RV64 also leads to VCPU mask on stack consuming 2KB. We set KVM_MAX_VCPUS to 1024 for both RV64 and RV32 to be aligned other architectures. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_host.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index d5c2057f8451..0a804021db8c 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -16,8 +16,7 @@ #include #include -#define KVM_MAX_VCPUS \ - ((HGATP_VMID_MASK >> HGATP_VMID_SHIFT) + 1) +#define KVM_MAX_VCPUS 1024 #define KVM_HALT_POLL_NS_DEFAULT 500000 From 13acfec2dbccfafff3331a3810cd7dde2fb16891 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 9 May 2022 10:44:05 +0530 Subject: [PATCH 273/319] RISC-V: KVM: Add remote HFENCE functions based on VCPU requests The generic KVM has support for VCPU requests which can be used to do arch-specific work in the run-loop. We introduce remote HFENCE functions which will internally use VCPU requests instead of host SBI calls. Advantages of doing remote HFENCEs as VCPU requests are: 1) Multiple VCPUs of a Guest may be running on different Host CPUs so it is not always possible to determine the Host CPU mask for doing Host SBI call. For example, when VCPU X wants to do HFENCE on VCPU Y, it is possible that VCPU Y is blocked or in user-space (i.e. vcpu->cpu < 0). 2) To support nested virtualization, we will be having a separate shadow G-stage for each VCPU and a common host G-stage for the entire Guest/VM. The VCPU requests based remote HFENCEs helps us easily synchronize the common host G-stage and shadow G-stage of each VCPU without any additional IPI calls. This is also a preparatory patch for upcoming nested virtualization support where we will be having a shadow G-stage page table for each Guest VCPU. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_host.h | 59 ++++++++ arch/riscv/kvm/mmu.c | 33 +++-- arch/riscv/kvm/tlb.c | 227 +++++++++++++++++++++++++++++- arch/riscv/kvm/vcpu.c | 24 +++- arch/riscv/kvm/vcpu_sbi_replace.c | 34 ++--- arch/riscv/kvm/vcpu_sbi_v01.c | 35 +++-- arch/riscv/kvm/vmid.c | 10 +- 7 files changed, 369 insertions(+), 53 deletions(-) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 0a804021db8c..01915d5f0094 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,31 @@ KVM_ARCH_REQ_FLAGS(0, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(1) #define KVM_REQ_UPDATE_HGATP KVM_ARCH_REQ(2) +#define KVM_REQ_FENCE_I \ + KVM_ARCH_REQ_FLAGS(3, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_HFENCE_GVMA_VMID_ALL KVM_REQ_TLB_FLUSH +#define KVM_REQ_HFENCE_VVMA_ALL \ + KVM_ARCH_REQ_FLAGS(4, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_HFENCE \ + KVM_ARCH_REQ_FLAGS(5, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) + +enum kvm_riscv_hfence_type { + KVM_RISCV_HFENCE_UNKNOWN = 0, + KVM_RISCV_HFENCE_GVMA_VMID_GPA, + KVM_RISCV_HFENCE_VVMA_ASID_GVA, + KVM_RISCV_HFENCE_VVMA_ASID_ALL, + KVM_RISCV_HFENCE_VVMA_GVA, +}; + +struct kvm_riscv_hfence { + enum kvm_riscv_hfence_type type; + unsigned long asid; + unsigned long order; + gpa_t addr; + gpa_t size; +}; + +#define KVM_RISCV_VCPU_MAX_HFENCE 64 struct kvm_vm_stat { struct kvm_vm_stat_generic generic; @@ -178,6 +204,12 @@ struct kvm_vcpu_arch { /* VCPU Timer */ struct kvm_vcpu_timer timer; + /* HFENCE request queue */ + spinlock_t hfence_lock; + unsigned long hfence_head; + unsigned long hfence_tail; + struct kvm_riscv_hfence hfence_queue[KVM_RISCV_VCPU_MAX_HFENCE]; + /* MMIO instruction details */ struct kvm_mmio_decode mmio_decode; @@ -221,6 +253,33 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, unsigned long order); void kvm_riscv_local_hfence_vvma_all(unsigned long vmid); +void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu); +void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu); +void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu); +void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu); + +void kvm_riscv_fence_i(struct kvm *kvm, + unsigned long hbase, unsigned long hmask); +void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + gpa_t gpa, gpa_t gpsz, + unsigned long order); +void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask); +void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long gva, unsigned long gvsz, + unsigned long order, unsigned long asid); +void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long asid); +void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long gva, unsigned long gvsz, + unsigned long order); +void kvm_riscv_hfence_vvma_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask); + int kvm_riscv_gstage_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, gpa_t gpa, unsigned long hva, bool is_write); diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 1e07603c905b..1c00695ebee7 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -18,7 +18,6 @@ #include #include #include -#include #ifdef CONFIG_64BIT static unsigned long gstage_mode = (HGATP_MODE_SV39X4 << HGATP_MODE_SHIFT); @@ -73,13 +72,25 @@ static int gstage_page_size_to_level(unsigned long page_size, u32 *out_level) return -EINVAL; } -static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize) +static int gstage_level_to_page_order(u32 level, unsigned long *out_pgorder) { if (gstage_pgd_levels < level) return -EINVAL; - *out_pgsize = 1UL << (12 + (level * gstage_index_bits)); + *out_pgorder = 12 + (level * gstage_index_bits); + return 0; +} +static int gstage_level_to_page_size(u32 level, unsigned long *out_pgsize) +{ + int rc; + unsigned long page_order = PAGE_SHIFT; + + rc = gstage_level_to_page_order(level, &page_order); + if (rc) + return rc; + + *out_pgsize = BIT(page_order); return 0; } @@ -114,21 +125,13 @@ static bool gstage_get_leaf_entry(struct kvm *kvm, gpa_t addr, static void gstage_remote_tlb_flush(struct kvm *kvm, u32 level, gpa_t addr) { - unsigned long size = PAGE_SIZE; - struct kvm_vmid *vmid = &kvm->arch.vmid; + unsigned long order = PAGE_SHIFT; - if (gstage_level_to_page_size(level, &size)) + if (gstage_level_to_page_order(level, &order)) return; - addr &= ~(size - 1); + addr &= ~(BIT(order) - 1); - /* - * TODO: Instead of cpu_online_mask, we should only target CPUs - * where the Guest/VM is running. - */ - preempt_disable(); - sbi_remote_hfence_gvma_vmid(cpu_online_mask, addr, size, - READ_ONCE(vmid->vmid)); - preempt_enable(); + kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0, addr, BIT(order), order); } static int gstage_set_pte(struct kvm *kvm, u32 level, diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index e2d4fd610745..c0f86d09c41d 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -3,11 +3,14 @@ * Copyright (c) 2022 Ventana Micro Systems Inc. */ -#include +#include +#include #include #include #include +#include #include +#include #include /* @@ -211,3 +214,225 @@ void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) csr_write(CSR_HGATP, hgatp); } + +void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) +{ + local_flush_icache_all(); +} + +void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu) +{ + struct kvm_vmid *vmid; + + vmid = &vcpu->kvm->arch.vmid; + kvm_riscv_local_hfence_gvma_vmid_all(READ_ONCE(vmid->vmid)); +} + +void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu) +{ + struct kvm_vmid *vmid; + + vmid = &vcpu->kvm->arch.vmid; + kvm_riscv_local_hfence_vvma_all(READ_ONCE(vmid->vmid)); +} + +static bool vcpu_hfence_dequeue(struct kvm_vcpu *vcpu, + struct kvm_riscv_hfence *out_data) +{ + bool ret = false; + struct kvm_vcpu_arch *varch = &vcpu->arch; + + spin_lock(&varch->hfence_lock); + + if (varch->hfence_queue[varch->hfence_head].type) { + memcpy(out_data, &varch->hfence_queue[varch->hfence_head], + sizeof(*out_data)); + varch->hfence_queue[varch->hfence_head].type = 0; + + varch->hfence_head++; + if (varch->hfence_head == KVM_RISCV_VCPU_MAX_HFENCE) + varch->hfence_head = 0; + + ret = true; + } + + spin_unlock(&varch->hfence_lock); + + return ret; +} + +static bool vcpu_hfence_enqueue(struct kvm_vcpu *vcpu, + const struct kvm_riscv_hfence *data) +{ + bool ret = false; + struct kvm_vcpu_arch *varch = &vcpu->arch; + + spin_lock(&varch->hfence_lock); + + if (!varch->hfence_queue[varch->hfence_tail].type) { + memcpy(&varch->hfence_queue[varch->hfence_tail], + data, sizeof(*data)); + + varch->hfence_tail++; + if (varch->hfence_tail == KVM_RISCV_VCPU_MAX_HFENCE) + varch->hfence_tail = 0; + + ret = true; + } + + spin_unlock(&varch->hfence_lock); + + return ret; +} + +void kvm_riscv_hfence_process(struct kvm_vcpu *vcpu) +{ + struct kvm_riscv_hfence d = { 0 }; + struct kvm_vmid *v = &vcpu->kvm->arch.vmid; + + while (vcpu_hfence_dequeue(vcpu, &d)) { + switch (d.type) { + case KVM_RISCV_HFENCE_UNKNOWN: + break; + case KVM_RISCV_HFENCE_GVMA_VMID_GPA: + kvm_riscv_local_hfence_gvma_vmid_gpa( + READ_ONCE(v->vmid), + d.addr, d.size, d.order); + break; + case KVM_RISCV_HFENCE_VVMA_ASID_GVA: + kvm_riscv_local_hfence_vvma_asid_gva( + READ_ONCE(v->vmid), d.asid, + d.addr, d.size, d.order); + break; + case KVM_RISCV_HFENCE_VVMA_ASID_ALL: + kvm_riscv_local_hfence_vvma_asid_all( + READ_ONCE(v->vmid), d.asid); + break; + case KVM_RISCV_HFENCE_VVMA_GVA: + kvm_riscv_local_hfence_vvma_gva( + READ_ONCE(v->vmid), + d.addr, d.size, d.order); + break; + default: + break; + } + } +} + +static void make_xfence_request(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned int req, unsigned int fallback_req, + const struct kvm_riscv_hfence *data) +{ + unsigned long i; + struct kvm_vcpu *vcpu; + unsigned int actual_req = req; + DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); + + bitmap_clear(vcpu_mask, 0, KVM_MAX_VCPUS); + kvm_for_each_vcpu(i, vcpu, kvm) { + if (hbase != -1UL) { + if (vcpu->vcpu_id < hbase) + continue; + if (!(hmask & (1UL << (vcpu->vcpu_id - hbase)))) + continue; + } + + bitmap_set(vcpu_mask, i, 1); + + if (!data || !data->type) + continue; + + /* + * Enqueue hfence data to VCPU hfence queue. If we don't + * have space in the VCPU hfence queue then fallback to + * a more conservative hfence request. + */ + if (!vcpu_hfence_enqueue(vcpu, data)) + actual_req = fallback_req; + } + + kvm_make_vcpus_request_mask(kvm, actual_req, vcpu_mask); +} + +void kvm_riscv_fence_i(struct kvm *kvm, + unsigned long hbase, unsigned long hmask) +{ + make_xfence_request(kvm, hbase, hmask, KVM_REQ_FENCE_I, + KVM_REQ_FENCE_I, NULL); +} + +void kvm_riscv_hfence_gvma_vmid_gpa(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + gpa_t gpa, gpa_t gpsz, + unsigned long order) +{ + struct kvm_riscv_hfence data; + + data.type = KVM_RISCV_HFENCE_GVMA_VMID_GPA; + data.asid = 0; + data.addr = gpa; + data.size = gpsz; + data.order = order; + make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, + KVM_REQ_HFENCE_GVMA_VMID_ALL, &data); +} + +void kvm_riscv_hfence_gvma_vmid_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask) +{ + make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_GVMA_VMID_ALL, + KVM_REQ_HFENCE_GVMA_VMID_ALL, NULL); +} + +void kvm_riscv_hfence_vvma_asid_gva(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long gva, unsigned long gvsz, + unsigned long order, unsigned long asid) +{ + struct kvm_riscv_hfence data; + + data.type = KVM_RISCV_HFENCE_VVMA_ASID_GVA; + data.asid = asid; + data.addr = gva; + data.size = gvsz; + data.order = order; + make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, + KVM_REQ_HFENCE_VVMA_ALL, &data); +} + +void kvm_riscv_hfence_vvma_asid_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long asid) +{ + struct kvm_riscv_hfence data; + + data.type = KVM_RISCV_HFENCE_VVMA_ASID_ALL; + data.asid = asid; + data.addr = data.size = data.order = 0; + make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, + KVM_REQ_HFENCE_VVMA_ALL, &data); +} + +void kvm_riscv_hfence_vvma_gva(struct kvm *kvm, + unsigned long hbase, unsigned long hmask, + unsigned long gva, unsigned long gvsz, + unsigned long order) +{ + struct kvm_riscv_hfence data; + + data.type = KVM_RISCV_HFENCE_VVMA_GVA; + data.asid = 0; + data.addr = gva; + data.size = gvsz; + data.order = order; + make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE, + KVM_REQ_HFENCE_VVMA_ALL, &data); +} + +void kvm_riscv_hfence_vvma_all(struct kvm *kvm, + unsigned long hbase, unsigned long hmask) +{ + make_xfence_request(kvm, hbase, hmask, KVM_REQ_HFENCE_VVMA_ALL, + KVM_REQ_HFENCE_VVMA_ALL, NULL); +} diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 24cf3bb4be3a..88b8c0fb5f5e 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -78,6 +78,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) WRITE_ONCE(vcpu->arch.irqs_pending, 0); WRITE_ONCE(vcpu->arch.irqs_pending_mask, 0); + vcpu->arch.hfence_head = 0; + vcpu->arch.hfence_tail = 0; + memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); + /* Reset the guest CSRs for hotplug usecase */ if (loaded) kvm_arch_vcpu_load(vcpu, smp_processor_id()); @@ -101,6 +105,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Setup ISA features available to VCPU */ vcpu->arch.isa = riscv_isa_extension_base(NULL) & KVM_RISCV_ISA_ALLOWED; + /* Setup VCPU hfence queue */ + spin_lock_init(&vcpu->arch.hfence_lock); + /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ cntx = &vcpu->arch.guest_reset_context; cntx->sstatus = SR_SPP | SR_SPIE; @@ -692,8 +699,21 @@ static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) kvm_riscv_gstage_update_hgatp(vcpu); - if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) - kvm_riscv_local_hfence_gvma_all(); + if (kvm_check_request(KVM_REQ_FENCE_I, vcpu)) + kvm_riscv_fence_i_process(vcpu); + + /* + * The generic KVM_REQ_TLB_FLUSH is same as + * KVM_REQ_HFENCE_GVMA_VMID_ALL + */ + if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu)) + kvm_riscv_hfence_gvma_vmid_all_process(vcpu); + + if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu)) + kvm_riscv_hfence_vvma_all_process(vcpu); + + if (kvm_check_request(KVM_REQ_HFENCE, vcpu)) + kvm_riscv_hfence_process(vcpu); } } diff --git a/arch/riscv/kvm/vcpu_sbi_replace.c b/arch/riscv/kvm/vcpu_sbi_replace.c index 3c1dcd38358e..4c034d8a606a 100644 --- a/arch/riscv/kvm/vcpu_sbi_replace.c +++ b/arch/riscv/kvm/vcpu_sbi_replace.c @@ -81,37 +81,31 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run struct kvm_cpu_trap *utrap, bool *exit) { int ret = 0; - unsigned long i; - struct cpumask cm; - struct kvm_vcpu *tmp; struct kvm_cpu_context *cp = &vcpu->arch.guest_context; unsigned long hmask = cp->a0; unsigned long hbase = cp->a1; unsigned long funcid = cp->a6; - cpumask_clear(&cm); - kvm_for_each_vcpu(i, tmp, vcpu->kvm) { - if (hbase != -1UL) { - if (tmp->vcpu_id < hbase) - continue; - if (!(hmask & (1UL << (tmp->vcpu_id - hbase)))) - continue; - } - if (tmp->cpu < 0) - continue; - cpumask_set_cpu(tmp->cpu, &cm); - } - switch (funcid) { case SBI_EXT_RFENCE_REMOTE_FENCE_I: - ret = sbi_remote_fence_i(&cm); + kvm_riscv_fence_i(vcpu->kvm, hbase, hmask); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: - ret = sbi_remote_hfence_vvma(&cm, cp->a2, cp->a3); + if (cp->a2 == 0 && cp->a3 == 0) + kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask); + else + kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask, + cp->a2, cp->a3, PAGE_SHIFT); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: - ret = sbi_remote_hfence_vvma_asid(&cm, cp->a2, - cp->a3, cp->a4); + if (cp->a2 == 0 && cp->a3 == 0) + kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, + hbase, hmask, cp->a4); + else + kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, + hbase, hmask, + cp->a2, cp->a3, + PAGE_SHIFT, cp->a4); break; case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: diff --git a/arch/riscv/kvm/vcpu_sbi_v01.c b/arch/riscv/kvm/vcpu_sbi_v01.c index da4d6c99c2cf..8a91a14e7139 100644 --- a/arch/riscv/kvm/vcpu_sbi_v01.c +++ b/arch/riscv/kvm/vcpu_sbi_v01.c @@ -23,7 +23,6 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, int i, ret = 0; u64 next_cycle; struct kvm_vcpu *rvcpu; - struct cpumask cm; struct kvm *kvm = vcpu->kvm; struct kvm_cpu_context *cp = &vcpu->arch.guest_context; @@ -80,19 +79,29 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, if (utrap->scause) break; - cpumask_clear(&cm); - for_each_set_bit(i, &hmask, BITS_PER_LONG) { - rvcpu = kvm_get_vcpu_by_id(vcpu->kvm, i); - if (rvcpu->cpu < 0) - continue; - cpumask_set_cpu(rvcpu->cpu, &cm); - } if (cp->a7 == SBI_EXT_0_1_REMOTE_FENCE_I) - ret = sbi_remote_fence_i(&cm); - else if (cp->a7 == SBI_EXT_0_1_REMOTE_SFENCE_VMA) - ret = sbi_remote_hfence_vvma(&cm, cp->a1, cp->a2); - else - ret = sbi_remote_hfence_vvma_asid(&cm, cp->a1, cp->a2, cp->a3); + kvm_riscv_fence_i(vcpu->kvm, 0, hmask); + else if (cp->a7 == SBI_EXT_0_1_REMOTE_SFENCE_VMA) { + if (cp->a1 == 0 && cp->a2 == 0) + kvm_riscv_hfence_vvma_all(vcpu->kvm, + 0, hmask); + else + kvm_riscv_hfence_vvma_gva(vcpu->kvm, + 0, hmask, + cp->a1, cp->a2, + PAGE_SHIFT); + } else { + if (cp->a1 == 0 && cp->a2 == 0) + kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, + 0, hmask, + cp->a3); + else + kvm_riscv_hfence_vvma_asid_gva(vcpu->kvm, + 0, hmask, + cp->a1, cp->a2, + PAGE_SHIFT, + cp->a3); + } break; default: ret = -EINVAL; diff --git a/arch/riscv/kvm/vmid.c b/arch/riscv/kvm/vmid.c index 8987e76aa6db..9f764df125db 100644 --- a/arch/riscv/kvm/vmid.c +++ b/arch/riscv/kvm/vmid.c @@ -11,9 +11,9 @@ #include #include #include +#include #include #include -#include static unsigned long vmid_version = 1; static unsigned long vmid_next; @@ -63,6 +63,11 @@ bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid) READ_ONCE(vmid_version)); } +static void __local_hfence_gvma_all(void *info) +{ + kvm_riscv_local_hfence_gvma_all(); +} + void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu) { unsigned long i; @@ -101,7 +106,8 @@ void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu) * running, we force VM exits on all host CPUs using IPI and * flush all Guest TLBs. */ - sbi_remote_hfence_gvma(cpu_online_mask, 0, 0); + on_each_cpu_mask(cpu_online_mask, __local_hfence_gvma_all, + NULL, 1); } vmid->vmid = vmid_next; From 92e450507d5612d399d0abee8447305a43a412cc Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 9 May 2022 10:44:11 +0530 Subject: [PATCH 274/319] RISC-V: KVM: Cleanup stale TLB entries when host CPU changes On RISC-V platforms with hardware VMID support, we share same VMID for all VCPUs of a particular Guest/VM. This means we might have stale G-stage TLB entries on the current Host CPU due to some other VCPU of the same Guest which ran previously on the current Host CPU. To cleanup stale TLB entries, we simply flush all G-stage TLB entries by VMID whenever underlying Host CPU changes for a VCPU. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Anup Patel --- arch/riscv/include/asm/kvm_host.h | 5 +++++ arch/riscv/kvm/tlb.c | 23 +++++++++++++++++++++++ arch/riscv/kvm/vcpu.c | 11 +++++++++++ 3 files changed, 39 insertions(+) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 01915d5f0094..319c8aeb42af 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -166,6 +166,9 @@ struct kvm_vcpu_arch { /* VCPU ran at least once */ bool ran_atleast_once; + /* Last Host CPU on which Guest VCPU exited */ + int last_exit_cpu; + /* ISA feature bits (similar to MISA) */ unsigned long isa; @@ -253,6 +256,8 @@ void kvm_riscv_local_hfence_vvma_gva(unsigned long vmid, unsigned long order); void kvm_riscv_local_hfence_vvma_all(unsigned long vmid); +void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu); + void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu); void kvm_riscv_hfence_gvma_vmid_all_process(struct kvm_vcpu *vcpu); void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu); diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index c0f86d09c41d..1a76d0b1907d 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -215,6 +215,29 @@ void kvm_riscv_local_hfence_vvma_all(unsigned long vmid) csr_write(CSR_HGATP, hgatp); } +void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu) +{ + unsigned long vmid; + + if (!kvm_riscv_gstage_vmid_bits() || + vcpu->arch.last_exit_cpu == vcpu->cpu) + return; + + /* + * On RISC-V platforms with hardware VMID support, we share same + * VMID for all VCPUs of a particular Guest/VM. This means we might + * have stale G-stage TLB entries on the current Host CPU due to + * some other VCPU of the same Guest which ran previously on the + * current Host CPU. + * + * To cleanup stale TLB entries, we simply flush all G-stage TLB + * entries by VMID whenever underlying Host CPU changes for a VCPU. + */ + + vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); + kvm_riscv_local_hfence_gvma_vmid_all(vmid); +} + void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) { local_flush_icache_all(); diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index 88b8c0fb5f5e..a7c8400f16b5 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -67,6 +67,8 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) if (loaded) kvm_arch_vcpu_put(vcpu); + vcpu->arch.last_exit_cpu = -1; + memcpy(csr, reset_csr, sizeof(*csr)); memcpy(cntx, reset_cntx, sizeof(*cntx)); @@ -735,6 +737,7 @@ static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu) { guest_state_enter_irqoff(); __kvm_riscv_switch_to(&vcpu->arch); + vcpu->arch.last_exit_cpu = vcpu->cpu; guest_state_exit_irqoff(); } @@ -829,6 +832,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) continue; } + /* + * Cleanup stale TLB enteries + * + * Note: This should be done after G-stage VMID has been + * updated using kvm_riscv_gstage_vmid_ver_changed() + */ + kvm_riscv_local_tlb_sanitize(vcpu); + guest_timing_enter_irqoff(); kvm_riscv_vcpu_enter_exit(vcpu); From affa28e4d094b48a7b27dd0c5b8258288b6c4a24 Mon Sep 17 00:00:00 2001 From: Atish Patra Date: Mon, 9 May 2022 11:29:37 -0700 Subject: [PATCH 275/319] RISC-V: KVM: Introduce ISA extension register Currently, there is no provision for vmm (qemu-kvm or kvmtool) to query about multiple-letter ISA extensions. The config register is only used for base single letter ISA extensions. A new ISA extension register is added that will allow the vmm to query about any ISA extension one at a time. It is enabled for both single letter or multi-letter ISA extensions. The ISA extension register is useful to if the vmm requires to retrieve/set single extension while the config register should be used if all the base ISA extension required to retrieve or set. For any multi-letter ISA extensions, the new register interface must be used. Signed-off-by: Atish Patra Signed-off-by: Anup Patel --- arch/riscv/include/uapi/asm/kvm.h | 20 +++++++ arch/riscv/kvm/vcpu.c | 99 +++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index f808ad1ce500..6119368ba6d5 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -82,6 +82,23 @@ struct kvm_riscv_timer { __u64 state; }; +/* + * ISA extension IDs specific to KVM. This is not the same as the host ISA + * extension IDs as that is internal to the host and should not be exposed + * to the guest. This should always be contiguous to keep the mapping simple + * in KVM implementation. + */ +enum KVM_RISCV_ISA_EXT_ID { + KVM_RISCV_ISA_EXT_A = 0, + KVM_RISCV_ISA_EXT_C, + KVM_RISCV_ISA_EXT_D, + KVM_RISCV_ISA_EXT_F, + KVM_RISCV_ISA_EXT_H, + KVM_RISCV_ISA_EXT_I, + KVM_RISCV_ISA_EXT_M, + KVM_RISCV_ISA_EXT_MAX, +}; + /* Possible states for kvm_riscv_timer */ #define KVM_RISCV_TIMER_STATE_OFF 0 #define KVM_RISCV_TIMER_STATE_ON 1 @@ -123,6 +140,9 @@ struct kvm_riscv_timer { #define KVM_REG_RISCV_FP_D_REG(name) \ (offsetof(struct __riscv_d_ext_state, name) / sizeof(__u64)) +/* ISA Extension registers are mapped as type 7 */ +#define KVM_REG_RISCV_ISA_EXT (0x07 << KVM_REG_RISCV_TYPE_SHIFT) + #endif #endif /* __LINUX_KVM_RISCV_H */ diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index a7c8400f16b5..7f4ad5e4373a 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -374,6 +374,101 @@ static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, return 0; } +/* Mapping between KVM ISA Extension ID & Host ISA extension ID */ +static unsigned long kvm_isa_ext_arr[] = { + RISCV_ISA_EXT_a, + RISCV_ISA_EXT_c, + RISCV_ISA_EXT_d, + RISCV_ISA_EXT_f, + RISCV_ISA_EXT_h, + RISCV_ISA_EXT_i, + RISCV_ISA_EXT_m, +}; + +static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg) +{ + unsigned long __user *uaddr = + (unsigned long __user *)(unsigned long)reg->addr; + unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | + KVM_REG_SIZE_MASK | + KVM_REG_RISCV_ISA_EXT); + unsigned long reg_val = 0; + unsigned long host_isa_ext; + + if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) + return -EINVAL; + + if (reg_num >= KVM_RISCV_ISA_EXT_MAX || reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) + return -EINVAL; + + host_isa_ext = kvm_isa_ext_arr[reg_num]; + if (__riscv_isa_extension_available(&vcpu->arch.isa, host_isa_ext)) + reg_val = 1; /* Mark the given extension as available */ + + if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) + return -EFAULT; + + return 0; +} + +static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu, + const struct kvm_one_reg *reg) +{ + unsigned long __user *uaddr = + (unsigned long __user *)(unsigned long)reg->addr; + unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | + KVM_REG_SIZE_MASK | + KVM_REG_RISCV_ISA_EXT); + unsigned long reg_val; + unsigned long host_isa_ext; + unsigned long host_isa_ext_mask; + + if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) + return -EINVAL; + + if (reg_num >= KVM_RISCV_ISA_EXT_MAX || reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) + return -EINVAL; + + if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) + return -EFAULT; + + host_isa_ext = kvm_isa_ext_arr[reg_num]; + if (!__riscv_isa_extension_available(NULL, host_isa_ext)) + return -EOPNOTSUPP; + + if (host_isa_ext >= RISCV_ISA_EXT_BASE && + host_isa_ext < RISCV_ISA_EXT_MAX) { + /* + * Multi-letter ISA extension. Currently there is no provision + * to enable/disable the multi-letter ISA extensions for guests. + * Return success if the request is to enable any ISA extension + * that is available in the hardware. + * Return -EOPNOTSUPP otherwise. + */ + if (!reg_val) + return -EOPNOTSUPP; + else + return 0; + } + + /* Single letter base ISA extension */ + if (!vcpu->arch.ran_atleast_once) { + host_isa_ext_mask = BIT_MASK(host_isa_ext); + if (!reg_val && (host_isa_ext_mask & KVM_RISCV_ISA_DISABLE_ALLOWED)) + vcpu->arch.isa &= ~host_isa_ext_mask; + else + vcpu->arch.isa |= host_isa_ext_mask; + vcpu->arch.isa &= riscv_isa_extension_base(NULL); + vcpu->arch.isa &= KVM_RISCV_ISA_ALLOWED; + kvm_riscv_vcpu_fp_reset(vcpu); + } else { + return -EOPNOTSUPP; + } + + return 0; +} + static int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) { @@ -391,6 +486,8 @@ static int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_FP_D) return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, KVM_REG_RISCV_FP_D); + else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_ISA_EXT) + return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); return -EINVAL; } @@ -412,6 +509,8 @@ static int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_FP_D) return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, KVM_REG_RISCV_FP_D); + else if ((reg->id & KVM_REG_RISCV_TYPE_MASK) == KVM_REG_RISCV_ISA_EXT) + return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); return -EINVAL; } From fed9b26b2501ea0ce41ae3a788bcc498440589c6 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 9 May 2022 11:23:16 +0530 Subject: [PATCH 276/319] MAINTAINERS: Update KVM RISC-V entry to cover selftests support We update KVM RISC-V maintainers entry to include appropriate KVM selftests directories so that RISC-V related KVM selftests patches are CC'ed to KVM RISC-V mailing list. Signed-off-by: Anup Patel --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index d6d879cb0afd..e5b7a325a7b5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10769,6 +10769,8 @@ T: git git://github.com/kvm-riscv/linux.git F: arch/riscv/include/asm/kvm* F: arch/riscv/include/uapi/asm/kvm* F: arch/riscv/kvm/ +F: tools/testing/selftests/kvm/*/riscv/ +F: tools/testing/selftests/kvm/riscv/ KERNEL VIRTUAL MACHINE for s390 (KVM/s390) M: Christian Borntraeger From 4689752c79fa30e91b49b39a9fba93c4d1f3e20c Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Mon, 16 May 2022 11:33:35 +0000 Subject: [PATCH 277/319] drivers/s390/char: Add Ultravisor io device This patch adds a new miscdevice to expose some Ultravisor functions to userspace. Userspace can send IOCTLs to the uvdevice that will then emit a corresponding Ultravisor Call and hands the result over to userspace. The uvdevice is available if the Ultravisor Call facility is present. Userspace can call the Retrieve Attestation Measurement Ultravisor Call using IOCTLs on the uvdevice. The uvdevice will do some sanity checks first. Then, copy the request data to kernel space, build the UVCB, perform the UV call, and copy the result back to userspace. Signed-off-by: Steffen Eiden Reviewed-by: Janosch Frank Reviewed-by: Claudio Imbrenda Link: https://lore.kernel.org/kvm/20220516113335.338212-1-seiden@linux.ibm.com/ Message-Id: <20220516113335.338212-1-seiden@linux.ibm.com> Signed-off-by: Janosch Frank (whitespace and tristate fixes, pick) --- MAINTAINERS | 2 + arch/s390/include/asm/uv.h | 23 ++- arch/s390/include/uapi/asm/uvdevice.h | 51 +++++ drivers/s390/char/Kconfig | 10 + drivers/s390/char/Makefile | 1 + drivers/s390/char/uvdevice.c | 257 ++++++++++++++++++++++++++ 6 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 arch/s390/include/uapi/asm/uvdevice.h create mode 100644 drivers/s390/char/uvdevice.c diff --git a/MAINTAINERS b/MAINTAINERS index 61d9f114c37f..a74488b7bb7c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10779,9 +10779,11 @@ F: Documentation/virt/kvm/s390* F: arch/s390/include/asm/gmap.h F: arch/s390/include/asm/kvm* F: arch/s390/include/uapi/asm/kvm* +F: arch/s390/include/uapi/asm/uvdevice.h F: arch/s390/kernel/uv.c F: arch/s390/kvm/ F: arch/s390/mm/gmap.c +F: drivers/s390/char/uvdevice.c F: tools/testing/selftests/kvm/*/s390x/ F: tools/testing/selftests/kvm/s390x/ diff --git a/arch/s390/include/asm/uv.h b/arch/s390/include/asm/uv.h index a2d376b8bce3..cfea7b77a5b8 100644 --- a/arch/s390/include/asm/uv.h +++ b/arch/s390/include/asm/uv.h @@ -2,7 +2,7 @@ /* * Ultravisor Interfaces * - * Copyright IBM Corp. 2019 + * Copyright IBM Corp. 2019, 2022 * * Author(s): * Vasily Gorbik @@ -52,6 +52,7 @@ #define UVC_CMD_UNPIN_PAGE_SHARED 0x0342 #define UVC_CMD_SET_SHARED_ACCESS 0x1000 #define UVC_CMD_REMOVE_SHARED_ACCESS 0x1001 +#define UVC_CMD_RETR_ATTEST 0x1020 /* Bits in installed uv calls */ enum uv_cmds_inst { @@ -76,6 +77,7 @@ enum uv_cmds_inst { BIT_UVC_CMD_UNSHARE_ALL = 20, BIT_UVC_CMD_PIN_PAGE_SHARED = 21, BIT_UVC_CMD_UNPIN_PAGE_SHARED = 22, + BIT_UVC_CMD_RETR_ATTEST = 28, }; enum uv_feat_ind { @@ -219,6 +221,25 @@ struct uv_cb_share { u64 reserved28; } __packed __aligned(8); +/* Retrieve Attestation Measurement */ +struct uv_cb_attest { + struct uv_cb_header header; /* 0x0000 */ + u64 reserved08[2]; /* 0x0008 */ + u64 arcb_addr; /* 0x0018 */ + u64 cont_token; /* 0x0020 */ + u8 reserved28[6]; /* 0x0028 */ + u16 user_data_len; /* 0x002e */ + u8 user_data[256]; /* 0x0030 */ + u32 reserved130[3]; /* 0x0130 */ + u32 meas_len; /* 0x013c */ + u64 meas_addr; /* 0x0140 */ + u8 config_uid[16]; /* 0x0148 */ + u32 reserved158; /* 0x0158 */ + u32 add_data_len; /* 0x015c */ + u64 add_data_addr; /* 0x0160 */ + u64 reserved168[4]; /* 0x0168 */ +} __packed __aligned(8); + static inline int __uv_call(unsigned long r1, unsigned long r2) { int cc; diff --git a/arch/s390/include/uapi/asm/uvdevice.h b/arch/s390/include/uapi/asm/uvdevice.h new file mode 100644 index 000000000000..10a5ac918e02 --- /dev/null +++ b/arch/s390/include/uapi/asm/uvdevice.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright IBM Corp. 2022 + * Author(s): Steffen Eiden + */ +#ifndef __S390_ASM_UVDEVICE_H +#define __S390_ASM_UVDEVICE_H + +#include + +struct uvio_ioctl_cb { + __u32 flags; + __u16 uv_rc; /* UV header rc value */ + __u16 uv_rrc; /* UV header rrc value */ + __u64 argument_addr; /* Userspace address of uvio argument */ + __u32 argument_len; + __u8 reserved14[0x40 - 0x14]; /* must be zero */ +}; + +#define UVIO_ATT_USER_DATA_LEN 0x100 +#define UVIO_ATT_UID_LEN 0x10 +struct uvio_attest { + __u64 arcb_addr; /* 0x0000 */ + __u64 meas_addr; /* 0x0008 */ + __u64 add_data_addr; /* 0x0010 */ + __u8 user_data[UVIO_ATT_USER_DATA_LEN]; /* 0x0018 */ + __u8 config_uid[UVIO_ATT_UID_LEN]; /* 0x0118 */ + __u32 arcb_len; /* 0x0128 */ + __u32 meas_len; /* 0x012c */ + __u32 add_data_len; /* 0x0130 */ + __u16 user_data_len; /* 0x0134 */ + __u16 reserved136; /* 0x0136 */ +}; + +/* + * The following max values define an upper length for the IOCTL in/out buffers. + * However, they do not represent the maximum the Ultravisor allows which is + * often way smaller. By allowing larger buffer sizes we hopefully do not need + * to update the code with every machine update. It is therefore possible for + * userspace to request more memory than actually used by kernel/UV. + */ +#define UVIO_ATT_ARCB_MAX_LEN 0x100000 +#define UVIO_ATT_MEASUREMENT_MAX_LEN 0x8000 +#define UVIO_ATT_ADDITIONAL_MAX_LEN 0x8000 + +#define UVIO_DEVICE_NAME "uv" +#define UVIO_TYPE_UVC 'u' + +#define UVIO_IOCTL_ATT _IOWR(UVIO_TYPE_UVC, 0x01, struct uvio_ioctl_cb) + +#endif /* __S390_ASM_UVDEVICE_H */ diff --git a/drivers/s390/char/Kconfig b/drivers/s390/char/Kconfig index 6cc4b19acf85..ef8f41833c1a 100644 --- a/drivers/s390/char/Kconfig +++ b/drivers/s390/char/Kconfig @@ -100,6 +100,16 @@ config SCLP_OFB This option enables the Open-for-Business interface to the s390 Service Element. +config S390_UV_UAPI + def_tristate m + prompt "Ultravisor userspace API" + help + Selecting exposes parts of the UV interface to userspace + by providing a misc character device at /dev/uv. + Using IOCTLs one can interact with the UV. + The device is only available if the Ultravisor + Facility (158) is present. + config S390_TAPE def_tristate m prompt "S/390 tape device support" diff --git a/drivers/s390/char/Makefile b/drivers/s390/char/Makefile index c6fdb81a068a..ce32270082f5 100644 --- a/drivers/s390/char/Makefile +++ b/drivers/s390/char/Makefile @@ -48,6 +48,7 @@ obj-$(CONFIG_MONREADER) += monreader.o obj-$(CONFIG_MONWRITER) += monwriter.o obj-$(CONFIG_S390_VMUR) += vmur.o obj-$(CONFIG_CRASH_DUMP) += sclp_sdias.o zcore.o +obj-$(CONFIG_S390_UV_UAPI) += uvdevice.o hmcdrv-objs := hmcdrv_mod.o hmcdrv_dev.o hmcdrv_ftp.o hmcdrv_cache.o diag_ftp.o sclp_ftp.o obj-$(CONFIG_HMC_DRV) += hmcdrv.o diff --git a/drivers/s390/char/uvdevice.c b/drivers/s390/char/uvdevice.c new file mode 100644 index 000000000000..66505d7166a6 --- /dev/null +++ b/drivers/s390/char/uvdevice.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright IBM Corp. 2022 + * Author(s): Steffen Eiden + * + * This file provides a Linux misc device to give userspace access to some + * Ultravisor (UV) functions. The device only accepts IOCTLs and will only + * be present if the Ultravisor facility (158) is present. + * + * When userspace sends a valid IOCTL uvdevice will copy the input data to + * kernel space, do some basic validity checks to avoid kernel/system + * corruption. Any other check that the Ultravisor does will not be done by + * the uvdevice to keep changes minimal when adding new functionalities + * to existing UV-calls. + * After the checks uvdevice builds a corresponding + * Ultravisor Call Control Block, and sends the request to the Ultravisor. + * Then, it copies the response, including the return codes, back to userspace. + * It is the responsibility of the userspace to check for any error issued + * by UV and to interpret the UV response. The uvdevice acts as a communication + * channel for userspace to the Ultravisor. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static int uvio_build_uvcb_attest(struct uv_cb_attest *uvcb_attest, u8 *arcb, + u8 *meas, u8 *add_data, struct uvio_attest *uvio_attest) +{ + void __user *user_buf_arcb = (void __user *)uvio_attest->arcb_addr; + + if (copy_from_user(arcb, user_buf_arcb, uvio_attest->arcb_len)) + return -EFAULT; + + uvcb_attest->header.len = sizeof(*uvcb_attest); + uvcb_attest->header.cmd = UVC_CMD_RETR_ATTEST; + uvcb_attest->arcb_addr = (u64)arcb; + uvcb_attest->cont_token = 0; + uvcb_attest->user_data_len = uvio_attest->user_data_len; + memcpy(uvcb_attest->user_data, uvio_attest->user_data, sizeof(uvcb_attest->user_data)); + uvcb_attest->meas_len = uvio_attest->meas_len; + uvcb_attest->meas_addr = (u64)meas; + uvcb_attest->add_data_len = uvio_attest->add_data_len; + uvcb_attest->add_data_addr = (u64)add_data; + + return 0; +} + +static int uvio_copy_attest_result_to_user(struct uv_cb_attest *uvcb_attest, + struct uvio_ioctl_cb *uv_ioctl, + u8 *measurement, u8 *add_data, + struct uvio_attest *uvio_attest) +{ + struct uvio_attest __user *user_uvio_attest = (void __user *)uv_ioctl->argument_addr; + void __user *user_buf_add = (void __user *)uvio_attest->add_data_addr; + void __user *user_buf_meas = (void __user *)uvio_attest->meas_addr; + void __user *user_buf_uid = &user_uvio_attest->config_uid; + + if (copy_to_user(user_buf_meas, measurement, uvio_attest->meas_len)) + return -EFAULT; + if (add_data && copy_to_user(user_buf_add, add_data, uvio_attest->add_data_len)) + return -EFAULT; + if (copy_to_user(user_buf_uid, uvcb_attest->config_uid, sizeof(uvcb_attest->config_uid))) + return -EFAULT; + return 0; +} + +static int get_uvio_attest(struct uvio_ioctl_cb *uv_ioctl, struct uvio_attest *uvio_attest) +{ + u8 __user *user_arg_buf = (u8 __user *)uv_ioctl->argument_addr; + + if (copy_from_user(uvio_attest, user_arg_buf, sizeof(*uvio_attest))) + return -EFAULT; + + if (uvio_attest->arcb_len > UVIO_ATT_ARCB_MAX_LEN) + return -EINVAL; + if (uvio_attest->arcb_len == 0) + return -EINVAL; + if (uvio_attest->meas_len > UVIO_ATT_MEASUREMENT_MAX_LEN) + return -EINVAL; + if (uvio_attest->meas_len == 0) + return -EINVAL; + if (uvio_attest->add_data_len > UVIO_ATT_ADDITIONAL_MAX_LEN) + return -EINVAL; + if (uvio_attest->reserved136) + return -EINVAL; + return 0; +} + +/** + * uvio_attestation() - Perform a Retrieve Attestation Measurement UVC. + * + * @uv_ioctl: ioctl control block + * + * uvio_attestation() does a Retrieve Attestation Measurement Ultravisor Call. + * It verifies that the given userspace addresses are valid and request sizes + * are sane. Every other check is made by the Ultravisor (UV) and won't result + * in a negative return value. It copies the input to kernelspace, builds the + * request, sends the UV-call, and copies the result to userspace. + * + * The Attestation Request has two input and two outputs. + * ARCB and User Data are inputs for the UV generated by userspace. + * Measurement and Additional Data are outputs for userspace generated by UV. + * + * The Attestation Request Control Block (ARCB) is a cryptographically verified + * and secured request to UV and User Data is some plaintext data which is + * going to be included in the Attestation Measurement calculation. + * + * Measurement is a cryptographic measurement of the callers properties, + * optional data configured by the ARCB and the user data. If specified by the + * ARCB, UV will add some Additional Data to the measurement calculation. + * This Additional Data is then returned as well. + * + * If the Retrieve Attestation Measurement UV facility is not present, + * UV will return invalid command rc. This won't be fenced in the driver + * and does not result in a negative return value. + * + * Context: might sleep + * + * Return: 0 on success or a negative error code on error. + */ +static int uvio_attestation(struct uvio_ioctl_cb *uv_ioctl) +{ + struct uv_cb_attest *uvcb_attest = NULL; + struct uvio_attest *uvio_attest = NULL; + u8 *measurement = NULL; + u8 *add_data = NULL; + u8 *arcb = NULL; + int ret; + + ret = -EINVAL; + if (uv_ioctl->argument_len != sizeof(*uvio_attest)) + goto out; + + ret = -ENOMEM; + uvio_attest = kzalloc(sizeof(*uvio_attest), GFP_KERNEL); + if (!uvio_attest) + goto out; + + ret = get_uvio_attest(uv_ioctl, uvio_attest); + if (ret) + goto out; + + ret = -ENOMEM; + arcb = kvzalloc(uvio_attest->arcb_len, GFP_KERNEL); + measurement = kvzalloc(uvio_attest->meas_len, GFP_KERNEL); + if (!arcb || !measurement) + goto out; + + if (uvio_attest->add_data_len) { + add_data = kvzalloc(uvio_attest->add_data_len, GFP_KERNEL); + if (!add_data) + goto out; + } + + uvcb_attest = kzalloc(sizeof(*uvcb_attest), GFP_KERNEL); + if (!uvcb_attest) + goto out; + + ret = uvio_build_uvcb_attest(uvcb_attest, arcb, measurement, add_data, uvio_attest); + if (ret) + goto out; + + uv_call_sched(0, (u64)uvcb_attest); + + uv_ioctl->uv_rc = uvcb_attest->header.rc; + uv_ioctl->uv_rrc = uvcb_attest->header.rrc; + + ret = uvio_copy_attest_result_to_user(uvcb_attest, uv_ioctl, measurement, add_data, + uvio_attest); +out: + kvfree(arcb); + kvfree(measurement); + kvfree(add_data); + kfree(uvio_attest); + kfree(uvcb_attest); + return ret; +} + +static int uvio_copy_and_check_ioctl(struct uvio_ioctl_cb *ioctl, void __user *argp) +{ + if (copy_from_user(ioctl, argp, sizeof(*ioctl))) + return -EFAULT; + if (ioctl->flags != 0) + return -EINVAL; + if (memchr_inv(ioctl->reserved14, 0, sizeof(ioctl->reserved14))) + return -EINVAL; + + return 0; +} + +/* + * IOCTL entry point for the Ultravisor device. + */ +static long uvio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + struct uvio_ioctl_cb uv_ioctl = { }; + long ret; + + switch (cmd) { + case UVIO_IOCTL_ATT: + ret = uvio_copy_and_check_ioctl(&uv_ioctl, argp); + if (ret) + return ret; + ret = uvio_attestation(&uv_ioctl); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + if (ret) + return ret; + + if (copy_to_user(argp, &uv_ioctl, sizeof(uv_ioctl))) + ret = -EFAULT; + + return ret; +} + +static const struct file_operations uvio_dev_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = uvio_ioctl, + .llseek = no_llseek, +}; + +static struct miscdevice uvio_dev_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = UVIO_DEVICE_NAME, + .fops = &uvio_dev_fops, +}; + +static void __exit uvio_dev_exit(void) +{ + misc_deregister(&uvio_dev_miscdev); +} + +static int __init uvio_dev_init(void) +{ + if (!test_facility(158)) + return -ENXIO; + return misc_register(&uvio_dev_miscdev); +} + +module_init(uvio_dev_init); +module_exit(uvio_dev_exit); + +MODULE_AUTHOR("IBM Corporation"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Ultravisor UAPI driver"); From cbac924200b838cfb8d8b1415113d788089dc50b Mon Sep 17 00:00:00 2001 From: Steffen Eiden Date: Tue, 10 May 2022 14:47:24 +0000 Subject: [PATCH 278/319] selftests: drivers/s390x: Add uvdevice tests Adds some selftests to test ioctl error paths of the uv-uapi. The Kconfig S390_UV_UAPI must be selected and the Ultravisor facility must be available. The test can be executed by non-root, however, the uvdevice special file /dev/uv must be accessible for reading and writing which may imply root privileges. ./test-uv-device TAP version 13 1..6 # Starting 6 tests from 3 test cases. # RUN uvio_fixture.att.fault_ioctl_arg ... # OK uvio_fixture.att.fault_ioctl_arg ok 1 uvio_fixture.att.fault_ioctl_arg # RUN uvio_fixture.att.fault_uvio_arg ... # OK uvio_fixture.att.fault_uvio_arg ok 2 uvio_fixture.att.fault_uvio_arg # RUN uvio_fixture.att.inval_ioctl_cb ... # OK uvio_fixture.att.inval_ioctl_cb ok 3 uvio_fixture.att.inval_ioctl_cb # RUN uvio_fixture.att.inval_ioctl_cmd ... # OK uvio_fixture.att.inval_ioctl_cmd ok 4 uvio_fixture.att.inval_ioctl_cmd # RUN attest_fixture.att_inval_request ... # OK attest_fixture.att_inval_request ok 5 attest_fixture.att_inval_request # RUN attest_fixture.att_inval_addr ... # OK attest_fixture.att_inval_addr ok 6 attest_fixture.att_inval_addr # PASSED: 6 / 6 tests passed. # Totals: pass:6 fail:0 xfail:0 xpass:0 skip:0 error:0 Signed-off-by: Steffen Eiden Acked-by: Janosch Frank Message-Id: <20220510144724.3321985-3-seiden@linux.ibm.com> Link: https://lore.kernel.org/kvm/20220510144724.3321985-3-seiden@linux.ibm.com/ Signed-off-by: Janosch Frank --- MAINTAINERS | 1 + tools/testing/selftests/Makefile | 1 + tools/testing/selftests/drivers/.gitignore | 1 + .../selftests/drivers/s390x/uvdevice/Makefile | 22 ++ .../selftests/drivers/s390x/uvdevice/config | 1 + .../drivers/s390x/uvdevice/test_uvdevice.c | 276 ++++++++++++++++++ 6 files changed, 302 insertions(+) create mode 100644 tools/testing/selftests/drivers/s390x/uvdevice/Makefile create mode 100644 tools/testing/selftests/drivers/s390x/uvdevice/config create mode 100644 tools/testing/selftests/drivers/s390x/uvdevice/test_uvdevice.c diff --git a/MAINTAINERS b/MAINTAINERS index a74488b7bb7c..bc6fae05ee4b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10784,6 +10784,7 @@ F: arch/s390/kernel/uv.c F: arch/s390/kvm/ F: arch/s390/mm/gmap.c F: drivers/s390/char/uvdevice.c +F: tools/testing/selftests/drivers/s390x/uvdevice/ F: tools/testing/selftests/kvm/*/s390x/ F: tools/testing/selftests/kvm/s390x/ diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 2319ec87f53d..d6b307371ef7 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -10,6 +10,7 @@ TARGETS += core TARGETS += cpufreq TARGETS += cpu-hotplug TARGETS += drivers/dma-buf +TARGETS += drivers/s390x/uvdevice TARGETS += efivarfs TARGETS += exec TARGETS += filesystems diff --git a/tools/testing/selftests/drivers/.gitignore b/tools/testing/selftests/drivers/.gitignore index ca74f2e1c719..09e23b5afa96 100644 --- a/tools/testing/selftests/drivers/.gitignore +++ b/tools/testing/selftests/drivers/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only /dma-buf/udmabuf +/s390x/uvdevice/test_uvdevice diff --git a/tools/testing/selftests/drivers/s390x/uvdevice/Makefile b/tools/testing/selftests/drivers/s390x/uvdevice/Makefile new file mode 100644 index 000000000000..5e701d2708d4 --- /dev/null +++ b/tools/testing/selftests/drivers/s390x/uvdevice/Makefile @@ -0,0 +1,22 @@ +include ../../../../../build/Build.include + +UNAME_M := $(shell uname -m) + +ifneq ($(UNAME_M),s390x) +nothing: +.PHONY: all clean run_tests install +.SILENT: +else + +TEST_GEN_PROGS := test_uvdevice + +top_srcdir ?= ../../../../../.. +KSFT_KHDR_INSTALL := 1 +khdr_dir = $(top_srcdir)/usr/include +LINUX_TOOL_ARCH_INCLUDE = $(top_srcdir)/tools/arch/$(ARCH)/include + +CFLAGS += -Wall -Werror -static -I$(khdr_dir) -I$(LINUX_TOOL_ARCH_INCLUDE) + +include ../../../lib.mk + +endif diff --git a/tools/testing/selftests/drivers/s390x/uvdevice/config b/tools/testing/selftests/drivers/s390x/uvdevice/config new file mode 100644 index 000000000000..f28a04b99eff --- /dev/null +++ b/tools/testing/selftests/drivers/s390x/uvdevice/config @@ -0,0 +1 @@ +CONFIG_S390_UV_UAPI=y diff --git a/tools/testing/selftests/drivers/s390x/uvdevice/test_uvdevice.c b/tools/testing/selftests/drivers/s390x/uvdevice/test_uvdevice.c new file mode 100644 index 000000000000..ea0cdc37b44f --- /dev/null +++ b/tools/testing/selftests/drivers/s390x/uvdevice/test_uvdevice.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * selftest for the Ultravisor UAPI device + * + * Copyright IBM Corp. 2022 + * Author(s): Steffen Eiden + */ + +#include +#include +#include +#include +#include + +#include + +#include "../../../kselftest_harness.h" + +#define UV_PATH "/dev/uv" +#define BUFFER_SIZE 0x200 +FIXTURE(uvio_fixture) { + int uv_fd; + struct uvio_ioctl_cb uvio_ioctl; + uint8_t buffer[BUFFER_SIZE]; + __u64 fault_page; +}; + +FIXTURE_VARIANT(uvio_fixture) { + unsigned long ioctl_cmd; + uint32_t arg_size; +}; + +FIXTURE_VARIANT_ADD(uvio_fixture, att) { + .ioctl_cmd = UVIO_IOCTL_ATT, + .arg_size = sizeof(struct uvio_attest), +}; + +FIXTURE_SETUP(uvio_fixture) +{ + self->uv_fd = open(UV_PATH, O_ACCMODE); + + self->uvio_ioctl.argument_addr = (__u64)self->buffer; + self->uvio_ioctl.argument_len = variant->arg_size; + self->fault_page = + (__u64)mmap(NULL, (size_t)getpagesize(), PROT_NONE, MAP_ANONYMOUS, -1, 0); +} + +FIXTURE_TEARDOWN(uvio_fixture) +{ + if (self->uv_fd) + close(self->uv_fd); + munmap((void *)self->fault_page, (size_t)getpagesize()); +} + +TEST_F(uvio_fixture, fault_ioctl_arg) +{ + int rc, errno_cache; + + rc = ioctl(self->uv_fd, variant->ioctl_cmd, NULL); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EFAULT); + + rc = ioctl(self->uv_fd, variant->ioctl_cmd, self->fault_page); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EFAULT); +} + +TEST_F(uvio_fixture, fault_uvio_arg) +{ + int rc, errno_cache; + + self->uvio_ioctl.argument_addr = 0; + rc = ioctl(self->uv_fd, variant->ioctl_cmd, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EFAULT); + + self->uvio_ioctl.argument_addr = self->fault_page; + rc = ioctl(self->uv_fd, variant->ioctl_cmd, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EFAULT); +} + +/* + * Test to verify that IOCTLs with invalid values in the ioctl_control block + * are rejected. + */ +TEST_F(uvio_fixture, inval_ioctl_cb) +{ + int rc, errno_cache; + + self->uvio_ioctl.argument_len = 0; + rc = ioctl(self->uv_fd, variant->ioctl_cmd, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EINVAL); + + self->uvio_ioctl.argument_len = (uint32_t)-1; + rc = ioctl(self->uv_fd, variant->ioctl_cmd, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EINVAL); + self->uvio_ioctl.argument_len = variant->arg_size; + + self->uvio_ioctl.flags = (uint32_t)-1; + rc = ioctl(self->uv_fd, variant->ioctl_cmd, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EINVAL); + self->uvio_ioctl.flags = 0; + + memset(self->uvio_ioctl.reserved14, 0xff, sizeof(self->uvio_ioctl.reserved14)); + rc = ioctl(self->uv_fd, variant->ioctl_cmd, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EINVAL); + + memset(&self->uvio_ioctl, 0x11, sizeof(self->uvio_ioctl)); + rc = ioctl(self->uv_fd, variant->ioctl_cmd, &self->uvio_ioctl); + ASSERT_EQ(rc, -1); +} + +TEST_F(uvio_fixture, inval_ioctl_cmd) +{ + int rc, errno_cache; + uint8_t nr = _IOC_NR(variant->ioctl_cmd); + unsigned long cmds[] = { + _IOWR('a', nr, struct uvio_ioctl_cb), + _IOWR(UVIO_TYPE_UVC, nr, int), + _IO(UVIO_TYPE_UVC, nr), + _IOR(UVIO_TYPE_UVC, nr, struct uvio_ioctl_cb), + _IOW(UVIO_TYPE_UVC, nr, struct uvio_ioctl_cb), + }; + + for (size_t i = 0; i < ARRAY_SIZE(cmds); i++) { + rc = ioctl(self->uv_fd, cmds[i], &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, ENOTTY); + } +} + +struct test_attest_buffer { + uint8_t arcb[0x180]; + uint8_t meas[64]; + uint8_t add[32]; +}; + +FIXTURE(attest_fixture) { + int uv_fd; + struct uvio_ioctl_cb uvio_ioctl; + struct uvio_attest uvio_attest; + struct test_attest_buffer attest_buffer; + __u64 fault_page; +}; + +FIXTURE_SETUP(attest_fixture) +{ + self->uv_fd = open(UV_PATH, O_ACCMODE); + + self->uvio_ioctl.argument_addr = (__u64)&self->uvio_attest; + self->uvio_ioctl.argument_len = sizeof(self->uvio_attest); + + self->uvio_attest.arcb_addr = (__u64)&self->attest_buffer.arcb; + self->uvio_attest.arcb_len = sizeof(self->attest_buffer.arcb); + + self->uvio_attest.meas_addr = (__u64)&self->attest_buffer.meas; + self->uvio_attest.meas_len = sizeof(self->attest_buffer.meas); + + self->uvio_attest.add_data_addr = (__u64)&self->attest_buffer.add; + self->uvio_attest.add_data_len = sizeof(self->attest_buffer.add); + self->fault_page = + (__u64)mmap(NULL, (size_t)getpagesize(), PROT_NONE, MAP_ANONYMOUS, -1, 0); +} + +FIXTURE_TEARDOWN(attest_fixture) +{ + if (self->uv_fd) + close(self->uv_fd); + munmap((void *)self->fault_page, (size_t)getpagesize()); +} + +static void att_inval_sizes_test(uint32_t *size, uint32_t max_size, bool test_zero, + struct __test_metadata *_metadata, + FIXTURE_DATA(attest_fixture) *self) +{ + int rc, errno_cache; + uint32_t tmp = *size; + + if (test_zero) { + *size = 0; + rc = ioctl(self->uv_fd, UVIO_IOCTL_ATT, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EINVAL); + } + *size = max_size + 1; + rc = ioctl(self->uv_fd, UVIO_IOCTL_ATT, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EINVAL); + *size = tmp; +} + +/* + * Test to verify that attestation IOCTLs with invalid values in the UVIO + * attestation control block are rejected. + */ +TEST_F(attest_fixture, att_inval_request) +{ + int rc, errno_cache; + + att_inval_sizes_test(&self->uvio_attest.add_data_len, UVIO_ATT_ADDITIONAL_MAX_LEN, + false, _metadata, self); + att_inval_sizes_test(&self->uvio_attest.meas_len, UVIO_ATT_MEASUREMENT_MAX_LEN, + true, _metadata, self); + att_inval_sizes_test(&self->uvio_attest.arcb_len, UVIO_ATT_ARCB_MAX_LEN, + true, _metadata, self); + + self->uvio_attest.reserved136 = (uint16_t)-1; + rc = ioctl(self->uv_fd, UVIO_IOCTL_ATT, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EINVAL); + + memset(&self->uvio_attest, 0x11, sizeof(self->uvio_attest)); + rc = ioctl(self->uv_fd, UVIO_IOCTL_ATT, &self->uvio_ioctl); + ASSERT_EQ(rc, -1); +} + +static void att_inval_addr_test(__u64 *addr, struct __test_metadata *_metadata, + FIXTURE_DATA(attest_fixture) *self) +{ + int rc, errno_cache; + __u64 tmp = *addr; + + *addr = 0; + rc = ioctl(self->uv_fd, UVIO_IOCTL_ATT, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EFAULT); + *addr = self->fault_page; + rc = ioctl(self->uv_fd, UVIO_IOCTL_ATT, &self->uvio_ioctl); + errno_cache = errno; + ASSERT_EQ(rc, -1); + ASSERT_EQ(errno_cache, EFAULT); + *addr = tmp; +} + +TEST_F(attest_fixture, att_inval_addr) +{ + att_inval_addr_test(&self->uvio_attest.arcb_addr, _metadata, self); + att_inval_addr_test(&self->uvio_attest.add_data_addr, _metadata, self); + att_inval_addr_test(&self->uvio_attest.meas_addr, _metadata, self); +} + +static void __attribute__((constructor)) __constructor_order_last(void) +{ + if (!__constructor_order) + __constructor_order = _CONSTRUCTOR_ORDER_BACKWARD; +} + +int main(int argc, char **argv) +{ + int fd = open(UV_PATH, O_ACCMODE); + + if (fd < 0) + ksft_exit_skip("No uv-device or cannot access " UV_PATH "\n" + "Enable CONFIG_S390_UV_UAPI and check the access rights on " + UV_PATH ".\n"); + close(fd); + return test_harness_run(argc, argv); +} From c783631b0bffe6060113ff0aafe5fdbd71bea793 Mon Sep 17 00:00:00 2001 From: Janis Schoetterl-Glausch Date: Thu, 12 May 2022 15:10:17 +0200 Subject: [PATCH 279/319] KVM: s390: Don't indicate suppression on dirtying, failing memop If user space uses a memop to emulate an instruction and that memop fails, the execution of the instruction ends. Instruction execution can end in different ways, one of which is suppression, which requires that the instruction execute like a no-op. A writing memop that spans multiple pages and fails due to key protection may have modified guest memory, as a result, the likely correct ending is termination. Therefore, do not indicate a suppressing instruction ending in this case. Signed-off-by: Janis Schoetterl-Glausch Reviewed-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Link: https://lore.kernel.org/r/20220512131019.2594948-2-scgl@linux.ibm.com Signed-off-by: Christian Borntraeger Signed-off-by: Janosch Frank --- Documentation/virt/kvm/api.rst | 6 ++++++ arch/s390/kvm/gaccess.c | 22 ++++++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index d13fa6600467..09353e27bdbc 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -3754,12 +3754,18 @@ in case of KVM_S390_MEMOP_F_CHECK_ONLY), the ioctl returns a positive error number indicating the type of exception. This exception is also raised directly at the corresponding VCPU if the flag KVM_S390_MEMOP_F_INJECT_EXCEPTION is set. +On protection exceptions, unless specified otherwise, the injected +translation-exception identifier (TEID) indicates suppression. If the KVM_S390_MEMOP_F_SKEY_PROTECTION flag is set, storage key protection is also in effect and may cause exceptions if accesses are prohibited given the access key designated by "key"; the valid range is 0..15. KVM_S390_MEMOP_F_SKEY_PROTECTION is available if KVM_CAP_S390_MEM_OP_EXTENSION is > 0. +Since the accessed memory may span multiple pages and those pages might have +different storage keys, it is possible that a protection exception occurs +after memory has been modified. In this case, if the exception is injected, +the TEID does not indicate suppression. Absolute read/write: ^^^^^^^^^^^^^^^^^^^^ diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c index d53a183c2005..227ed0009354 100644 --- a/arch/s390/kvm/gaccess.c +++ b/arch/s390/kvm/gaccess.c @@ -491,8 +491,8 @@ enum prot_type { PROT_TYPE_IEP = 4, }; -static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, - u8 ar, enum gacc_mode mode, enum prot_type prot) +static int trans_exc_ending(struct kvm_vcpu *vcpu, int code, unsigned long gva, u8 ar, + enum gacc_mode mode, enum prot_type prot, bool terminate) { struct kvm_s390_pgm_info *pgm = &vcpu->arch.pgm; struct trans_exc_code_bits *tec; @@ -520,6 +520,11 @@ static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, tec->b61 = 1; break; } + if (terminate) { + tec->b56 = 0; + tec->b60 = 0; + tec->b61 = 0; + } fallthrough; case PGM_ASCE_TYPE: case PGM_PAGE_TRANSLATION: @@ -552,6 +557,12 @@ static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, return code; } +static int trans_exc(struct kvm_vcpu *vcpu, int code, unsigned long gva, u8 ar, + enum gacc_mode mode, enum prot_type prot) +{ + return trans_exc_ending(vcpu, code, gva, ar, mode, prot, false); +} + static int get_vcpu_asce(struct kvm_vcpu *vcpu, union asce *asce, unsigned long ga, u8 ar, enum gacc_mode mode) { @@ -1109,8 +1120,11 @@ int access_guest_with_key(struct kvm_vcpu *vcpu, unsigned long ga, u8 ar, data += fragment_len; ga = kvm_s390_logical_to_effective(vcpu, ga + fragment_len); } - if (rc > 0) - rc = trans_exc(vcpu, rc, ga, ar, mode, prot); + if (rc > 0) { + bool terminate = (mode == GACC_STORE) && (idx > 0); + + rc = trans_exc_ending(vcpu, rc, ga, ar, mode, prot, terminate); + } out_unlock: if (need_ipte_lock) ipte_unlock(vcpu); From c71159648c3cf0f7127ddc0bdf3eb4d7885210df Mon Sep 17 00:00:00 2001 From: Janis Schoetterl-Glausch Date: Thu, 12 May 2022 15:10:18 +0200 Subject: [PATCH 280/319] KVM: s390: selftest: Test suppression indication on key prot exception Check that suppression is not indicated on injection of a key checked protection exception caused by a memop after it already modified guest memory, as that violates the definition of suppression. Signed-off-by: Janis Schoetterl-Glausch Reviewed-by: Christian Borntraeger Link: https://lore.kernel.org/r/20220512131019.2594948-3-scgl@linux.ibm.com Signed-off-by: Christian Borntraeger Signed-off-by: Janosch Frank --- tools/testing/selftests/kvm/s390x/memop.c | 46 ++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/s390x/memop.c b/tools/testing/selftests/kvm/s390x/memop.c index b04c2c1b3c30..49f26f544127 100644 --- a/tools/testing/selftests/kvm/s390x/memop.c +++ b/tools/testing/selftests/kvm/s390x/memop.c @@ -10,6 +10,8 @@ #include #include +#include + #include "test_util.h" #include "kvm_util.h" @@ -194,6 +196,7 @@ static int err_memop_ioctl(struct test_vcpu vcpu, struct kvm_s390_mem_op *ksmo) #define SIDA_OFFSET(o) ._sida_offset = 1, .sida_offset = (o) #define AR(a) ._ar = 1, .ar = (a) #define KEY(a) .f_key = 1, .key = (a) +#define INJECT .f_inject = 1 #define CHECK_N_DO(f, ...) ({ f(__VA_ARGS__, CHECK_ONLY); f(__VA_ARGS__); }) @@ -430,9 +433,18 @@ static void test_copy_key_fetch_prot(void) TEST_ASSERT(rv == 4, "Should result in protection exception"); \ }) +static void guest_error_key(void) +{ + GUEST_SYNC(STAGE_INITED); + set_storage_key_range(mem1, PAGE_SIZE, 0x18); + set_storage_key_range(mem1 + PAGE_SIZE, sizeof(mem1) - PAGE_SIZE, 0x98); + GUEST_SYNC(STAGE_SKEYS_SET); + GUEST_SYNC(STAGE_IDLED); +} + static void test_errors_key(void) { - struct test_default t = test_default_init(guest_copy_key_fetch_prot); + struct test_default t = test_default_init(guest_error_key); HOST_SYNC(t.vcpu, STAGE_INITED); HOST_SYNC(t.vcpu, STAGE_SKEYS_SET); @@ -446,6 +458,37 @@ static void test_errors_key(void) kvm_vm_free(t.kvm_vm); } +static void test_termination(void) +{ + struct test_default t = test_default_init(guest_error_key); + uint64_t prefix; + uint64_t teid; + uint64_t teid_mask = BIT(63 - 56) | BIT(63 - 60) | BIT(63 - 61); + uint64_t psw[2]; + + HOST_SYNC(t.vcpu, STAGE_INITED); + HOST_SYNC(t.vcpu, STAGE_SKEYS_SET); + + /* vcpu, mismatching keys after first page */ + ERR_PROT_MOP(t.vcpu, LOGICAL, WRITE, mem1, t.size, GADDR_V(mem1), KEY(1), INJECT); + /* + * The memop injected a program exception and the test needs to check the + * Translation-Exception Identification (TEID). It is necessary to run + * the guest in order to be able to read the TEID from guest memory. + * Set the guest program new PSW, so the guest state is not clobbered. + */ + prefix = t.run->s.regs.prefix; + psw[0] = t.run->psw_mask; + psw[1] = t.run->psw_addr; + MOP(t.vm, ABSOLUTE, WRITE, psw, sizeof(psw), GADDR(prefix + 464)); + HOST_SYNC(t.vcpu, STAGE_IDLED); + MOP(t.vm, ABSOLUTE, READ, &teid, sizeof(teid), GADDR(prefix + 168)); + /* Bits 56, 60, 61 form a code, 0 being the only one allowing for termination */ + ASSERT_EQ(teid & teid_mask, 0); + + kvm_vm_free(t.kvm_vm); +} + static void test_errors_key_storage_prot_override(void) { struct test_default t = test_default_init(guest_copy_key_fetch_prot); @@ -668,6 +711,7 @@ int main(int argc, char *argv[]) test_copy_key_fetch_prot(); test_copy_key_fetch_prot_override(); test_errors_key(); + test_termination(); test_errors_key_storage_prot_override(); test_errors_key_fetch_prot_override_not_enabled(); test_errors_key_fetch_prot_override_enabled(); From 7869b481025c048ec6ea5b99acb14d057547de80 Mon Sep 17 00:00:00 2001 From: Zheyu Ma Date: Fri, 20 May 2022 10:56:24 +0800 Subject: [PATCH 281/319] gpio: ml-ioh: Convert to use managed functions pcim* and devm_* When removing the module, we will get the following flaw: [ 14.204955] remove_proc_entry: removing non-empty directory 'irq/21', leaking at least 'gpio_ml_ioh' [ 14.205827] WARNING: CPU: 0 PID: 305 at fs/proc/generic.c:717 remove_proc_entry+0x389/0x3f0 ... [ 14.220613] ioh_gpio_remove+0xc5/0xe0 [gpio_ml_ioh] [ 14.221075] pci_device_remove+0x92/0x240 Fix this by using managed functions, this makes the error handling more simpler. Fixes: e971ac9a564a ("gpio: ml-ioh: use resource management for irqs") Signed-off-by: Zheyu Ma Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-ml-ioh.c | 76 ++++++++------------------------------ 1 file changed, 16 insertions(+), 60 deletions(-) diff --git a/drivers/gpio/gpio-ml-ioh.c b/drivers/gpio/gpio-ml-ioh.c index b060c4773698..48e3768a830e 100644 --- a/drivers/gpio/gpio-ml-ioh.c +++ b/drivers/gpio/gpio-ml-ioh.c @@ -409,29 +409,27 @@ static int ioh_gpio_probe(struct pci_dev *pdev, void *chip_save; int irq_base; - ret = pci_enable_device(pdev); + ret = pcim_enable_device(pdev); if (ret) { - dev_err(dev, "%s : pci_enable_device failed", __func__); - goto err_pci_enable; + dev_err(dev, "%s : pcim_enable_device failed", __func__); + return ret; } - ret = pci_request_regions(pdev, KBUILD_MODNAME); + ret = pcim_iomap_regions(pdev, BIT(1), KBUILD_MODNAME); if (ret) { - dev_err(dev, "pci_request_regions failed-%d", ret); - goto err_request_regions; + dev_err(dev, "pcim_iomap_regions failed-%d", ret); + return ret; } - base = pci_iomap(pdev, 1, 0); + base = pcim_iomap_table(pdev)[1]; if (!base) { - dev_err(dev, "%s : pci_iomap failed", __func__); - ret = -ENOMEM; - goto err_iomap; + dev_err(dev, "%s : pcim_iomap_table failed", __func__); + return -ENOMEM; } - chip_save = kcalloc(8, sizeof(*chip), GFP_KERNEL); + chip_save = devm_kcalloc(dev, 8, sizeof(*chip), GFP_KERNEL); if (chip_save == NULL) { - ret = -ENOMEM; - goto err_kzalloc; + return -ENOMEM; } chip = chip_save; @@ -442,10 +440,10 @@ static int ioh_gpio_probe(struct pci_dev *pdev, chip->ch = i; spin_lock_init(&chip->spinlock); ioh_gpio_setup(chip, num_ports[i]); - ret = gpiochip_add_data(&chip->gpio, chip); + ret = devm_gpiochip_add_data(dev, &chip->gpio, chip); if (ret) { dev_err(dev, "IOH gpio: Failed to register GPIO\n"); - goto err_gpiochip_add; + return ret; } } @@ -456,15 +454,14 @@ static int ioh_gpio_probe(struct pci_dev *pdev, if (irq_base < 0) { dev_warn(dev, "ml_ioh_gpio: Failed to get IRQ base num\n"); - ret = irq_base; - goto err_gpiochip_add; + return irq_base; } chip->irq_base = irq_base; ret = ioh_gpio_alloc_generic_chip(chip, irq_base, num_ports[j]); if (ret) - goto err_gpiochip_add; + return ret; } chip = chip_save; @@ -472,52 +469,12 @@ static int ioh_gpio_probe(struct pci_dev *pdev, IRQF_SHARED, KBUILD_MODNAME, chip); if (ret != 0) { dev_err(dev, "%s request_irq failed\n", __func__); - goto err_gpiochip_add; + return ret; } pci_set_drvdata(pdev, chip); return 0; - -err_gpiochip_add: - chip = chip_save; - while (--i >= 0) { - gpiochip_remove(&chip->gpio); - chip++; - } - kfree(chip_save); - -err_kzalloc: - pci_iounmap(pdev, base); - -err_iomap: - pci_release_regions(pdev); - -err_request_regions: - pci_disable_device(pdev); - -err_pci_enable: - - dev_err(dev, "%s Failed returns %d\n", __func__, ret); - return ret; -} - -static void ioh_gpio_remove(struct pci_dev *pdev) -{ - int i; - struct ioh_gpio *chip = pci_get_drvdata(pdev); - void *chip_save; - - chip_save = chip; - - for (i = 0; i < 8; i++, chip++) - gpiochip_remove(&chip->gpio); - - chip = chip_save; - pci_iounmap(pdev, chip->base); - pci_release_regions(pdev); - pci_disable_device(pdev); - kfree(chip); } static int __maybe_unused ioh_gpio_suspend(struct device *dev) @@ -558,7 +515,6 @@ static struct pci_driver ioh_gpio_driver = { .name = "ml_ioh_gpio", .id_table = ioh_gpio_pcidev_id, .probe = ioh_gpio_probe, - .remove = ioh_gpio_remove, .driver = { .pm = &ioh_gpio_pm_ops, }, From cd285535b8eeed314813783ba9b5a851bdf6bca8 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sat, 2 Apr 2022 17:55:50 +0200 Subject: [PATCH 282/319] dt-bindings: mailbox: qcom-ipcc: simplify the example Consumer examples in the bindings of resource providers are trivial, useless and duplicating code. Additionally the incomplete qcom,smp2p example triggers DT schema warnings. Cleanup the example by removing the consumer part and fixing the indentation to DT schema convention. Reported-by: Rob Herring Signed-off-by: Krzysztof Kozlowski Reviewed-by: Rob Herring Signed-off-by: Jassi Brar --- .../bindings/mailbox/qcom-ipcc.yaml | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml index 866efb278813..dfdc72345a2a 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml @@ -61,23 +61,14 @@ additionalProperties: false examples: - | - #include - #include + #include + #include - mailbox@408000 { - compatible = "qcom,sm8250-ipcc", "qcom,ipcc"; - reg = <0x408000 0x1000>; - interrupts = ; - interrupt-controller; - #interrupt-cells = <3>; - #mbox-cells = <2>; - }; - - smp2p-modem { - compatible = "qcom,smp2p"; - interrupts-extended = <&ipcc_mproc IPCC_CLIENT_MPSS - IPCC_MPROC_SIGNAL_SMP2P IRQ_TYPE_EDGE_RISING>; - mboxes = <&ipcc_mproc IPCC_CLIENT_MPSS IPCC_MPROC_SIGNAL_SMP2P>; - - /* Other SMP2P fields */ - }; + mailbox@408000 { + compatible = "qcom,sm8250-ipcc", "qcom,ipcc"; + reg = <0x408000 0x1000>; + interrupts = ; + interrupt-controller; + #interrupt-cells = <3>; + #mbox-cells = <2>; + }; From 262190a8ca2b1e1ec75b8a4f1c7f07e585facd6f Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Sun, 3 Apr 2022 10:53:04 -0400 Subject: [PATCH 283/319] mailbox: imx: remove redundant initializer Smatch reports this issue imx-mailbox.c:887:10: warning: Initializer entry defined twice imx-mailbox.c:889:10: also defined here .rxdb = imx_mu_generic_rxdb, Is listed twice, so remove one. Signed-off-by: Tom Rix Signed-off-by: Jassi Brar --- drivers/mailbox/imx-mailbox.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index e88f544a1548..df8a785be324 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -886,7 +886,6 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { .rx = imx_mu_generic_rx, .rxdb = imx_mu_generic_rxdb, .init = imx_mu_init_generic, - .rxdb = imx_mu_generic_rxdb, .type = IMX_MU_V2, .xTR = 0x200, .xRR = 0x280, From 1b0070aca35ed70f9cc6f97b5d9f7f98cb4771fd Mon Sep 17 00:00:00 2001 From: Xiaomeng Tong Date: Tue, 5 Apr 2022 16:05:56 +0800 Subject: [PATCH 284/319] mailbox: remove an unneeded NULL check on list iterator The list iterator is always non-NULL so it doesn't need to be checked. Thus just remove the unnecessary NULL check. Signed-off-by: Xiaomeng Tong Signed-off-by: Jassi Brar --- drivers/mailbox/tegra-hsp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mailbox/tegra-hsp.c b/drivers/mailbox/tegra-hsp.c index 78f7265039c6..a55d6df99697 100644 --- a/drivers/mailbox/tegra-hsp.c +++ b/drivers/mailbox/tegra-hsp.c @@ -804,7 +804,7 @@ static int __maybe_unused tegra_hsp_resume(struct device *dev) struct tegra_hsp_doorbell *db; list_for_each_entry(db, &hsp->doorbells, list) { - if (db && db->channel.chan) + if (db->channel.chan) tegra_hsp_doorbell_startup(db->channel.chan); } From 1b3418ac6451b771b677e049b8a48d96e258e922 Mon Sep 17 00:00:00 2001 From: Yongqiang Niu Date: Wed, 6 Apr 2022 17:12:13 +0800 Subject: [PATCH 285/319] dt-bindings: gce: add the GCE header file for MT8186 Add the GCE header file to define GCE subsys ids, hardware event ids and constants for MT8186. Signed-off-by: Yongqiang Niu Acked-by: Rob Herring Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Jassi Brar --- include/dt-bindings/gce/mt8186-gce.h | 421 +++++++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100644 include/dt-bindings/gce/mt8186-gce.h diff --git a/include/dt-bindings/gce/mt8186-gce.h b/include/dt-bindings/gce/mt8186-gce.h new file mode 100644 index 000000000000..f12e3cb586ce --- /dev/null +++ b/include/dt-bindings/gce/mt8186-gce.h @@ -0,0 +1,421 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* + * Copyright (C) 2022 MediaTek Inc. + * Author: Yongqiang Niu + */ + +#ifndef _DT_BINDINGS_GCE_MT8186_H +#define _DT_BINDINGS_GCE_MT8186_H + +/* assign timeout 0 also means default */ +#define CMDQ_NO_TIMEOUT 0xffffffff +#define CMDQ_TIMEOUT_DEFAULT 1000 + +/* GCE thread priority */ +#define CMDQ_THR_PRIO_LOWEST 0 +#define CMDQ_THR_PRIO_1 1 +#define CMDQ_THR_PRIO_2 2 +#define CMDQ_THR_PRIO_3 3 +#define CMDQ_THR_PRIO_4 4 +#define CMDQ_THR_PRIO_5 5 +#define CMDQ_THR_PRIO_6 6 +#define CMDQ_THR_PRIO_HIGHEST 7 + +/* CPR count in 32bit register */ +#define GCE_CPR_COUNT 1312 + +/* GCE subsys table */ +#define SUBSYS_1300XXXX 0 +#define SUBSYS_1400XXXX 1 +#define SUBSYS_1401XXXX 2 +#define SUBSYS_1402XXXX 3 +#define SUBSYS_1502XXXX 4 +#define SUBSYS_1582XXXX 5 +#define SUBSYS_1B00XXXX 6 +#define SUBSYS_1C00XXXX 7 +#define SUBSYS_1C10XXXX 8 +#define SUBSYS_1000XXXX 9 +#define SUBSYS_1001XXXX 10 +#define SUBSYS_1020XXXX 11 +#define SUBSYS_1021XXXX 12 +#define SUBSYS_1022XXXX 13 +#define SUBSYS_1023XXXX 14 +#define SUBSYS_1060XXXX 15 +#define SUBSYS_1602XXXX 16 +#define SUBSYS_1608XXXX 17 +#define SUBSYS_1700XXXX 18 +#define SUBSYS_1701XXXX 19 +#define SUBSYS_1702XXXX 20 +#define SUBSYS_1703XXXX 21 +#define SUBSYS_1706XXXX 22 +#define SUBSYS_1A00XXXX 23 +#define SUBSYS_1A01XXXX 24 +#define SUBSYS_1A02XXXX 25 +#define SUBSYS_1A03XXXX 26 +#define SUBSYS_1A04XXXX 27 +#define SUBSYS_1A05XXXX 28 +#define SUBSYS_1A06XXXX 29 +#define SUBSYS_NO_SUPPORT 99 + +/* GCE General Purpose Register (GPR) support + * Leave note for scenario usage here + */ +/* GCE: write mask */ +#define GCE_GPR_R00 0x00 +#define GCE_GPR_R01 0x01 +/* MDP: P1: JPEG dest */ +#define GCE_GPR_R02 0x02 +#define GCE_GPR_R03 0x03 +/* MDP: PQ color */ +#define GCE_GPR_R04 0x04 +/* MDP: 2D sharpness */ +#define GCE_GPR_R05 0x05 +/* DISP: poll esd */ +#define GCE_GPR_R06 0x06 +#define GCE_GPR_R07 0x07 +/* MDP: P4: 2D sharpness dst */ +#define GCE_GPR_R08 0x08 +#define GCE_GPR_R09 0x09 +/* VCU: poll with timeout for GPR timer */ +#define GCE_GPR_R10 0x0A +#define GCE_GPR_R11 0x0B +/* CMDQ: debug */ +#define GCE_GPR_R12 0x0C +#define GCE_GPR_R13 0x0D +/* CMDQ: P7: debug */ +#define GCE_GPR_R14 0x0E +#define GCE_GPR_R15 0x0F + +/* GCE hardware events */ +/* VDEC */ +#define CMDQ_EVENT_LINE_COUNT_THRESHOLD_INTERRUPT 0 +#define CMDQ_EVENT_VDEC_INT 1 +#define CMDQ_EVENT_VDEC_PAUSE 2 +#define CMDQ_EVENT_VDEC_DEC_ERROR 3 +#define CMDQ_EVENT_MDEC_TIMEOUT 4 +#define CMDQ_EVENT_DRAM_ACCESS_DONE 5 +#define CMDQ_EVENT_INI_FETCH_RDY 6 +#define CMDQ_EVENT_PROCESS_FLAG 7 +#define CMDQ_EVENT_SEARCH_START_CODE_DONE 8 +#define CMDQ_EVENT_REF_REORDER_DONE 9 +#define CMDQ_EVENT_WP_TBLE_DONE 10 +#define CMDQ_EVENT_COUNT_SRAM_CLR_DONE 11 +#define CMDQ_EVENT_GCE_CNT_OP_THRESHOLD 15 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_0 16 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_1 17 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_2 18 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_3 19 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_4 20 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_5 21 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_6 22 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_7 23 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_8 24 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_9 25 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_10 26 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_11 27 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_12 28 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_13 29 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_14 30 +#define CMDQ_EVENT_VDEC_MINI_MDP_EVENT_15 31 +#define CMDQ_EVENT_WPE_GCE_FRAME_DONE 32 + +/* CAM */ +#define CMDQ_EVENT_ISP_FRAME_DONE_A 65 +#define CMDQ_EVENT_ISP_FRAME_DONE_B 66 +#define CMDQ_EVENT_CAMSV1_PASS1_DONE 70 +#define CMDQ_EVENT_CAMSV2_PASS1_DONE 71 +#define CMDQ_EVENT_CAMSV3_PASS1_DONE 72 +#define CMDQ_EVENT_MRAW_0_PASS1_DONE 73 +#define CMDQ_EVENT_SENINF_CAM0_FIFO_FULL 75 +#define CMDQ_EVENT_SENINF_CAM1_FIFO_FULL 76 +#define CMDQ_EVENT_SENINF_CAM2_FIFO_FULL 77 +#define CMDQ_EVENT_SENINF_CAM3_FIFO_FULL 78 +#define CMDQ_EVENT_SENINF_CAM4_FIFO_FULL 79 +#define CMDQ_EVENT_SENINF_CAM5_FIFO_FULL 80 +#define CMDQ_EVENT_SENINF_CAM6_FIFO_FULL 81 +#define CMDQ_EVENT_SENINF_CAM7_FIFO_FULL 82 +#define CMDQ_EVENT_SENINF_CAM8_FIFO_FULL 83 +#define CMDQ_EVENT_SENINF_CAM9_FIFO_FULL 84 +#define CMDQ_EVENT_SENINF_CAM10_FIFO_FULL 85 +#define CMDQ_EVENT_SENINF_CAM11_FIFO_FULL 86 +#define CMDQ_EVENT_SENINF_CAM12_FIFO_FULL 87 +#define CMDQ_EVENT_TG_OVRUN_A_INT 88 +#define CMDQ_EVENT_DMA_R1_ERROR_A_INT 89 +#define CMDQ_EVENT_TG_OVRUN_B_INT 90 +#define CMDQ_EVENT_DMA_R1_ERROR_B_INT 91 +#define CMDQ_EVENT_TG_OVRUN_M0_INT 94 +#define CMDQ_EVENT_R1_ERROR_M0_INT 95 +#define CMDQ_EVENT_TG_GRABERR_M0_INT 96 +#define CMDQ_EVENT_TG_GRABERR_A_INT 98 +#define CMDQ_EVENT_CQ_VR_SNAP_A_INT 99 +#define CMDQ_EVENT_TG_GRABERR_B_INT 100 +#define CMDQ_EVENT_CQ_VR_SNAP_B_INT 101 +/* VENC */ +#define CMDQ_EVENT_VENC_CMDQ_FRAME_DONE 129 +#define CMDQ_EVENT_VENC_CMDQ_PAUSE_DONE 130 +#define CMDQ_EVENT_JPGENC_CMDQ_DONE 131 +#define CMDQ_EVENT_VENC_CMDQ_MB_DONE 132 +#define CMDQ_EVENT_VENC_CMDQ_128BYTE_CNT_DONE 133 +#define CMDQ_EVENT_VENC_CMDQ_PPS_DONE 136 +#define CMDQ_EVENT_VENC_CMDQ_SPS_DONE 137 +#define CMDQ_EVENT_VENC_CMDQ_VPS_DONE 138 +/* IPE */ +#define CMDQ_EVENT_FDVT_DONE 161 +#define CMDQ_EVENT_FE_DONE 162 +#define CMDQ_EVENT_RSC_DONE 163 +#define CMDQ_EVENT_DVS_DONE_ASYNC_SHOT 164 +#define CMDQ_EVENT_DVP_DONE_ASYNC_SHOT 165 +/* IMG2 */ +#define CMDQ_EVENT_GCE_IMG2_EVENT0 193 +#define CMDQ_EVENT_GCE_IMG2_EVENT1 194 +#define CMDQ_EVENT_GCE_IMG2_EVENT2 195 +#define CMDQ_EVENT_GCE_IMG2_EVENT3 196 +#define CMDQ_EVENT_GCE_IMG2_EVENT4 197 +#define CMDQ_EVENT_GCE_IMG2_EVENT5 198 +#define CMDQ_EVENT_GCE_IMG2_EVENT6 199 +#define CMDQ_EVENT_GCE_IMG2_EVENT7 200 +#define CMDQ_EVENT_GCE_IMG2_EVENT8 201 +#define CMDQ_EVENT_GCE_IMG2_EVENT9 202 +#define CMDQ_EVENT_GCE_IMG2_EVENT10 203 +#define CMDQ_EVENT_GCE_IMG2_EVENT11 204 +#define CMDQ_EVENT_GCE_IMG2_EVENT12 205 +#define CMDQ_EVENT_GCE_IMG2_EVENT13 206 +#define CMDQ_EVENT_GCE_IMG2_EVENT14 207 +#define CMDQ_EVENT_GCE_IMG2_EVENT15 208 +#define CMDQ_EVENT_GCE_IMG2_EVENT16 209 +#define CMDQ_EVENT_GCE_IMG2_EVENT17 210 +#define CMDQ_EVENT_GCE_IMG2_EVENT18 211 +#define CMDQ_EVENT_GCE_IMG2_EVENT19 212 +#define CMDQ_EVENT_GCE_IMG2_EVENT20 213 +#define CMDQ_EVENT_GCE_IMG2_EVENT21 214 +#define CMDQ_EVENT_GCE_IMG2_EVENT22 215 +#define CMDQ_EVENT_GCE_IMG2_EVENT23 216 +/* IMG1 */ +#define CMDQ_EVENT_GCE_IMG1_EVENT0 225 +#define CMDQ_EVENT_GCE_IMG1_EVENT1 226 +#define CMDQ_EVENT_GCE_IMG1_EVENT2 227 +#define CMDQ_EVENT_GCE_IMG1_EVENT3 228 +#define CMDQ_EVENT_GCE_IMG1_EVENT4 229 +#define CMDQ_EVENT_GCE_IMG1_EVENT5 230 +#define CMDQ_EVENT_GCE_IMG1_EVENT6 231 +#define CMDQ_EVENT_GCE_IMG1_EVENT7 232 +#define CMDQ_EVENT_GCE_IMG1_EVENT8 233 +#define CMDQ_EVENT_GCE_IMG1_EVENT9 234 +#define CMDQ_EVENT_GCE_IMG1_EVENT10 235 +#define CMDQ_EVENT_GCE_IMG1_EVENT11 236 +#define CMDQ_EVENT_GCE_IMG1_EVENT12 237 +#define CMDQ_EVENT_GCE_IMG1_EVENT13 238 +#define CMDQ_EVENT_GCE_IMG1_EVENT14 239 +#define CMDQ_EVENT_GCE_IMG1_EVENT15 240 +#define CMDQ_EVENT_GCE_IMG1_EVENT16 241 +#define CMDQ_EVENT_GCE_IMG1_EVENT17 242 +#define CMDQ_EVENT_GCE_IMG1_EVENT18 243 +#define CMDQ_EVENT_GCE_IMG1_EVENT19 244 +#define CMDQ_EVENT_GCE_IMG1_EVENT20 245 +#define CMDQ_EVENT_GCE_IMG1_EVENT21 246 +#define CMDQ_EVENT_GCE_IMG1_EVENT22 247 +#define CMDQ_EVENT_GCE_IMG1_EVENT23 248 +/* MDP */ +#define CMDQ_EVENT_MDP_RDMA0_SOF 256 +#define CMDQ_EVENT_MDP_RDMA1_SOF 257 +#define CMDQ_EVENT_MDP_AAL0_SOF 258 +#define CMDQ_EVENT_MDP_AAL1_SOF 259 +#define CMDQ_EVENT_MDP_HDR0_SOF 260 +#define CMDQ_EVENT_MDP_RSZ0_SOF 261 +#define CMDQ_EVENT_MDP_RSZ1_SOF 262 +#define CMDQ_EVENT_MDP_WROT0_SOF 263 +#define CMDQ_EVENT_MDP_WROT1_SOF 264 +#define CMDQ_EVENT_MDP_TDSHP0_SOF 265 +#define CMDQ_EVENT_MDP_TDSHP1_SOF 266 +#define CMDQ_EVENT_IMG_DL_RELAY0_SOF 267 +#define CMDQ_EVENT_IMG_DL_RELAY1_SOF 268 +#define CMDQ_EVENT_MDP_COLOR0_SOF 269 +#define CMDQ_EVENT_MDP_WROT3_FRAME_DONE 288 +#define CMDQ_EVENT_MDP_WROT2_FRAME_DONE 289 +#define CMDQ_EVENT_MDP_WROT1_FRAME_DONE 290 +#define CMDQ_EVENT_MDP_WROT0_FRAME_DONE 291 +#define CMDQ_EVENT_MDP_TDSHP3_FRAME_DONE 292 +#define CMDQ_EVENT_MDP_TDSHP2_FRAME_DONE 293 +#define CMDQ_EVENT_MDP_TDSHP1_FRAME_DONE 294 +#define CMDQ_EVENT_MDP_TDSHP0_FRAME_DONE 295 +#define CMDQ_EVENT_MDP_RSZ3_FRAME_DONE 296 +#define CMDQ_EVENT_MDP_RSZ2_FRAME_DONE 297 +#define CMDQ_EVENT_MDP_RSZ1_FRAME_DONE 298 +#define CMDQ_EVENT_MDP_RSZ0_FRAME_DONE 299 +#define CMDQ_EVENT_MDP_RDMA3_FRAME_DONE 300 +#define CMDQ_EVENT_MDP_RDMA2_FRAME_DONE 301 +#define CMDQ_EVENT_MDP_RDMA1_FRAME_DONE 302 +#define CMDQ_EVENT_MDP_RDMA0_FRAME_DONE 303 +#define CMDQ_EVENT_MDP_HDR1_FRAME_DONE 304 +#define CMDQ_EVENT_MDP_HDR0_FRAME_DONE 305 +#define CMDQ_EVENT_MDP_COLOR0_FRAME_DONE 306 +#define CMDQ_EVENT_MDP_AAL3_FRAME_DONE 307 +#define CMDQ_EVENT_MDP_AAL2_FRAME_DONE 308 +#define CMDQ_EVENT_MDP_AAL1_FRAME_DONE 309 +#define CMDQ_EVENT_MDP_AAL0_FRAME_DONE 310 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_0 320 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_1 321 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_2 322 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_3 323 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_4 324 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_5 325 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_6 326 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_7 327 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_8 328 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_9 329 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_10 330 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_11 331 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_12 332 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_13 333 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_14 334 +#define CMDQ_EVENT_MDP_STREAM_DONE_ENG_EVENT_15 335 +#define CMDQ_EVENT_MDP_WROT3_SW_RST_DONE_ENG_EVENT 336 +#define CMDQ_EVENT_MDP_WROT2_SW_RST_DONE_ENG_EVENT 337 +#define CMDQ_EVENT_MDP_WROT1_SW_RST_DONE_ENG_EVENT 338 +#define CMDQ_EVENT_MDP_WROT0_SW_RST_DONE_ENG_EVENT 339 +#define CMDQ_EVENT_MDP_RDMA3_SW_RST_DONE_ENG_EVENT 340 +#define CMDQ_EVENT_MDP_RDMA2_SW_RST_DONE_ENG_EVENT 341 +#define CMDQ_EVENT_MDP_RDMA1_SW_RST_DONE_ENG_EVENT 342 +#define CMDQ_EVENT_MDP_RDMA0_SW_RST_DONE_ENG_EVENT 343 +/* DISP */ +#define CMDQ_EVENT_DISP_OVL0_SOF 384 +#define CMDQ_EVENT_DISP_OVL0_2L_SOF 385 +#define CMDQ_EVENT_DISP_RDMA0_SOF 386 +#define CMDQ_EVENT_DISP_RSZ0_SOF 387 +#define CMDQ_EVENT_DISP_COLOR0_SOF 388 +#define CMDQ_EVENT_DISP_CCORR0_SOF 389 +#define CMDQ_EVENT_DISP_CCORR1_SOF 390 +#define CMDQ_EVENT_DISP_AAL0_SOF 391 +#define CMDQ_EVENT_DISP_GAMMA0_SOF 392 +#define CMDQ_EVENT_DISP_POSTMASK0_SOF 393 +#define CMDQ_EVENT_DISP_DITHER0_SOF 394 +#define CMDQ_EVENT_DISP_CM0_SOF 395 +#define CMDQ_EVENT_DISP_SPR0_SOF 396 +#define CMDQ_EVENT_DISP_DSC_WRAP0_SOF 397 +#define CMDQ_EVENT_DSI0_SOF 398 +#define CMDQ_EVENT_DISP_WDMA0_SOF 399 +#define CMDQ_EVENT_DISP_PWM0_SOF 400 +#define CMDQ_EVENT_DSI0_FRAME_DONE 410 +#define CMDQ_EVENT_DISP_WDMA0_FRAME_DONE 411 +#define CMDQ_EVENT_DISP_SPR0_FRAME_DONE 412 +#define CMDQ_EVENT_DISP_RSZ0_FRAME_DONE 413 +#define CMDQ_EVENT_DISP_RDMA0_FRAME_DONE 414 +#define CMDQ_EVENT_DISP_POSTMASK0_FRAME_DONE 415 +#define CMDQ_EVENT_DISP_OVL0_FRAME_DONE 416 +#define CMDQ_EVENT_DISP_OVL0_2L_FRAME_DONE 417 +#define CMDQ_EVENT_DISP_GAMMA0_FRAME_DONE 418 +#define CMDQ_EVENT_DISP_DSC_WRAP0_CORE0_FRAME_DONE 420 +#define CMDQ_EVENT_DISP_DITHER0_FRAME_DONE 421 +#define CMDQ_EVENT_DISP_COLOR0_FRAME_DONE 422 +#define CMDQ_EVENT_DISP_CM0_FRAME_DONE 423 +#define CMDQ_EVENT_DISP_CCORR1_FRAME_DONE 424 +#define CMDQ_EVENT_DISP_CCORR0_FRAME_DONE 425 +#define CMDQ_EVENT_DISP_AAL0_FRAME_DONE 426 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_0 434 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_1 435 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_2 436 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_3 437 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_4 438 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_5 439 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_6 440 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_7 441 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_8 442 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_9 443 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_10 444 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_11 445 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_12 446 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_13 447 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_14 448 +#define CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_15 449 +#define CMDQ_EVENT_DSI0_TE_ENG_EVENT 450 +#define CMDQ_EVENT_DSI0_IRQ_ENG_EVENT 451 +#define CMDQ_EVENT_DSI0_DONE_ENG_EVENT 452 +#define CMDQ_EVENT_DISP_WDMA0_SW_RST_DONE_ENG_EVENT 453 +#define CMDQ_EVENT_DISP_SMIASSERT_ENG_EVENT 454 +#define CMDQ_EVENT_DISP_POSTMASK0_RST_DONE_ENG_EVENT 455 +#define CMDQ_EVENT_DISP_OVL0_RST_DONE_ENG_EVENT 456 +#define CMDQ_EVENT_DISP_OVL0_2L_RST_DONE_ENG_EVENT 457 +#define CMDQ_EVENT_BUF_UNDERRUN_ENG_EVENT_0 458 +#define CMDQ_EVENT_BUF_UNDERRUN_ENG_EVENT_1 459 +#define CMDQ_EVENT_BUF_UNDERRUN_ENG_EVENT_2 460 +#define CMDQ_EVENT_BUF_UNDERRUN_ENG_EVENT_3 461 +#define CMDQ_EVENT_BUF_UNDERRUN_ENG_EVENT_4 462 +#define CMDQ_EVENT_BUF_UNDERRUN_ENG_EVENT_5 463 +#define CMDQ_EVENT_BUF_UNDERRUN_ENG_EVENT_6 464 +#define CMDQ_EVENT_BUF_UNDERRUN_ENG_EVENT_7 465 +#define CMDQ_EVENT_OUT_EVENT_0 898 + +/* CMDQ sw tokens + * Following definitions are gce sw token which may use by clients + * event operation API. + * Note that token 512 to 639 may set secure + */ + +/* end of hw event and begin of sw token */ +#define CMDQ_MAX_HW_EVENT 512 + +/* Config thread notify trigger thread */ +#define CMDQ_SYNC_TOKEN_CONFIG_DIRTY 640 +/* Trigger thread notify config thread */ +#define CMDQ_SYNC_TOKEN_STREAM_EOF 641 +/* Block Trigger thread until the ESD check finishes. */ +#define CMDQ_SYNC_TOKEN_ESD_EOF 642 +#define CMDQ_SYNC_TOKEN_STREAM_BLOCK 643 +/* check CABC setup finish */ +#define CMDQ_SYNC_TOKEN_CABC_EOF 644 + +/* Notify normal CMDQ there are some secure task done + * MUST NOT CHANGE, this token sync with secure world + */ +#define CMDQ_SYNC_SECURE_THR_EOF 647 + +/* CMDQ use sw token */ +#define CMDQ_SYNC_TOKEN_USER_0 649 +#define CMDQ_SYNC_TOKEN_USER_1 650 +#define CMDQ_SYNC_TOKEN_POLL_MONITOR 651 +#define CMDQ_SYNC_TOKEN_TPR_LOCK 652 + +/* ISP sw token */ +#define CMDQ_SYNC_TOKEN_MSS 665 +#define CMDQ_SYNC_TOKEN_MSF 666 + +/* DISP sw token */ +#define CMDQ_SYNC_TOKEN_SODI 671 + +/* GPR access tokens (for register backup) + * There are 15 32-bit GPR, 3 GPR form a set + * (64-bit for address, 32-bit for value) + * MUST NOT CHANGE, these tokens sync with MDP + */ +#define CMDQ_SYNC_TOKEN_GPR_SET_0 700 +#define CMDQ_SYNC_TOKEN_GPR_SET_1 701 +#define CMDQ_SYNC_TOKEN_GPR_SET_2 702 +#define CMDQ_SYNC_TOKEN_GPR_SET_3 703 +#define CMDQ_SYNC_TOKEN_GPR_SET_4 704 + +/* Resource lock event to control resource in GCE thread */ +#define CMDQ_SYNC_RESOURCE_WROT0 710 +#define CMDQ_SYNC_RESOURCE_WROT1 711 + +/* event for gpr timer, used in sleep and poll with timeout */ +#define CMDQ_TOKEN_GPR_TIMER_R0 994 +#define CMDQ_TOKEN_GPR_TIMER_R1 995 +#define CMDQ_TOKEN_GPR_TIMER_R2 996 +#define CMDQ_TOKEN_GPR_TIMER_R3 997 +#define CMDQ_TOKEN_GPR_TIMER_R4 998 +#define CMDQ_TOKEN_GPR_TIMER_R5 999 +#define CMDQ_TOKEN_GPR_TIMER_R6 1000 +#define CMDQ_TOKEN_GPR_TIMER_R7 1001 +#define CMDQ_TOKEN_GPR_TIMER_R8 1002 +#define CMDQ_TOKEN_GPR_TIMER_R9 1003 +#define CMDQ_TOKEN_GPR_TIMER_R10 1004 +#define CMDQ_TOKEN_GPR_TIMER_R11 1005 +#define CMDQ_TOKEN_GPR_TIMER_R12 1006 +#define CMDQ_TOKEN_GPR_TIMER_R13 1007 +#define CMDQ_TOKEN_GPR_TIMER_R14 1008 +#define CMDQ_TOKEN_GPR_TIMER_R15 1009 + +#define CMDQ_EVENT_MAX 0x3FF +/* CMDQ sw tokens END */ + +#endif From 8f585d14030dcf8fbec2f864d189515e8be37a80 Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 14 Apr 2022 13:05:55 +0530 Subject: [PATCH 286/319] mailbox: tegra-hsp: Add tegra_hsp_sm_ops This patch introduces tegra_hsp_sm_ops to abstract send & receive API's for shared mailboxes. Signed-off-by: Kartik Signed-off-by: Jassi Brar --- drivers/mailbox/tegra-hsp.c | 78 +++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/drivers/mailbox/tegra-hsp.c b/drivers/mailbox/tegra-hsp.c index a55d6df99697..1a4c801878b0 100644 --- a/drivers/mailbox/tegra-hsp.c +++ b/drivers/mailbox/tegra-hsp.c @@ -67,8 +67,14 @@ struct tegra_hsp_doorbell { unsigned int index; }; +struct tegra_hsp_sm_ops { + void (*send)(struct tegra_hsp_channel *channel, void *data); + void (*recv)(struct tegra_hsp_channel *channel); +}; + struct tegra_hsp_mailbox { struct tegra_hsp_channel channel; + const struct tegra_hsp_sm_ops *ops; unsigned int index; bool producer; }; @@ -208,8 +214,7 @@ static irqreturn_t tegra_hsp_shared_irq(int irq, void *data) { struct tegra_hsp *hsp = data; unsigned long bit, mask; - u32 status, value; - void *msg; + u32 status; status = tegra_hsp_readl(hsp, HSP_INT_IR) & hsp->mask; @@ -245,25 +250,8 @@ static irqreturn_t tegra_hsp_shared_irq(int irq, void *data) for_each_set_bit(bit, &mask, hsp->num_sm) { struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit]; - if (!mb->producer) { - value = tegra_hsp_channel_readl(&mb->channel, - HSP_SM_SHRD_MBOX); - value &= ~HSP_SM_SHRD_MBOX_FULL; - msg = (void *)(unsigned long)value; - mbox_chan_received_data(mb->channel.chan, msg); - - /* - * Need to clear all bits here since some producers, - * such as TCU, depend on fields in the register - * getting cleared by the consumer. - * - * The mailbox API doesn't give the consumers a way - * of doing that explicitly, so we have to make sure - * we cover all possible cases. - */ - tegra_hsp_channel_writel(&mb->channel, 0x0, - HSP_SM_SHRD_MBOX); - } + if (!mb->producer) + mb->ops->recv(&mb->channel); } return IRQ_HANDLED; @@ -372,21 +360,52 @@ static const struct mbox_chan_ops tegra_hsp_db_ops = { .shutdown = tegra_hsp_doorbell_shutdown, }; -static int tegra_hsp_mailbox_send_data(struct mbox_chan *chan, void *data) +static void tegra_hsp_sm_send32(struct tegra_hsp_channel *channel, void *data) { - struct tegra_hsp_mailbox *mb = chan->con_priv; - struct tegra_hsp *hsp = mb->channel.hsp; - unsigned long flags; u32 value; - if (WARN_ON(!mb->producer)) - return -EPERM; - /* copy data and mark mailbox full */ value = (u32)(unsigned long)data; value |= HSP_SM_SHRD_MBOX_FULL; - tegra_hsp_channel_writel(&mb->channel, value, HSP_SM_SHRD_MBOX); + tegra_hsp_channel_writel(channel, value, HSP_SM_SHRD_MBOX); +} + +static void tegra_hsp_sm_recv32(struct tegra_hsp_channel *channel) +{ + u32 value; + void *msg; + + value = tegra_hsp_channel_readl(channel, HSP_SM_SHRD_MBOX); + value &= ~HSP_SM_SHRD_MBOX_FULL; + msg = (void *)(unsigned long)value; + mbox_chan_received_data(channel->chan, msg); + + /* + * Need to clear all bits here since some producers, such as TCU, depend + * on fields in the register getting cleared by the consumer. + * + * The mailbox API doesn't give the consumers a way of doing that + * explicitly, so we have to make sure we cover all possible cases. + */ + tegra_hsp_channel_writel(channel, 0x0, HSP_SM_SHRD_MBOX); +} + +static const struct tegra_hsp_sm_ops tegra_hsp_sm_32bit_ops = { + .send = tegra_hsp_sm_send32, + .recv = tegra_hsp_sm_recv32, +}; + +static int tegra_hsp_mailbox_send_data(struct mbox_chan *chan, void *data) +{ + struct tegra_hsp_mailbox *mb = chan->con_priv; + struct tegra_hsp *hsp = mb->channel.hsp; + unsigned long flags; + + if (WARN_ON(!mb->producer)) + return -EPERM; + + mb->ops->send(&mb->channel, data); /* enable EMPTY interrupt for the shared mailbox */ spin_lock_irqsave(&hsp->lock, flags); @@ -557,6 +576,7 @@ static struct mbox_chan *tegra_hsp_sm_xlate(struct mbox_controller *mbox, return ERR_PTR(-ENODEV); mb = &hsp->mailboxes[index]; + mb->ops = &tegra_hsp_sm_32bit_ops; if ((args->args[1] & TEGRA_HSP_SM_FLAG_TX) == 0) mb->producer = false; From 58919326e72f63c380dc3271dd1cc8bdf1bbe3e4 Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 14 Apr 2022 13:05:56 +0530 Subject: [PATCH 287/319] dt-bindings: tegra186-hsp: add type for shared mailboxes Tegra234 supports sending/receiving 32-bit and 128-bit data over a shared mailbox. Based on the data size to be used, clients need to specify the type of shared mailbox in the device tree. Add a macro for 128-bit shared mailbox. Mailbox clients can use this macro as a flag in device tree to enable 128-bit data support for a shared mailbox. Signed-off-by: Kartik Acked-by: Rob Herring Signed-off-by: Jassi Brar --- .../devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml | 9 +++++++++ include/dt-bindings/mailbox/tegra186-hsp.h | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml b/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml index 9f7a7296b57f..a3e87516d637 100644 --- a/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml +++ b/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml @@ -26,6 +26,15 @@ description: | second cell is used to identify the mailbox that the client is going to use. + For shared mailboxes, the first cell composed of two fields: + - bits 15..8: + A bit mask of flags that further specifies the type of shared + mailbox to be used (based on the data size). If no flag is + specified then, 32-bit shared mailbox is used. + - bits 7..0: + Defines the type of the mailbox to be used. This field should be + TEGRA_HSP_MBOX_TYPE_SM for shared mailboxes. + For doorbells, the second cell specifies the index of the doorbell to use. diff --git a/include/dt-bindings/mailbox/tegra186-hsp.h b/include/dt-bindings/mailbox/tegra186-hsp.h index 3bdec7a84d35..b9ccae2aa9e2 100644 --- a/include/dt-bindings/mailbox/tegra186-hsp.h +++ b/include/dt-bindings/mailbox/tegra186-hsp.h @@ -15,6 +15,11 @@ #define TEGRA_HSP_MBOX_TYPE_SS 0x2 #define TEGRA_HSP_MBOX_TYPE_AS 0x3 +/* + * These define the types of shared mailbox supported based on data size. + */ +#define TEGRA_HSP_MBOX_TYPE_SM_128BIT (1 << 8) + /* * These defines represent the bit associated with the given master ID in the * doorbell registers. From 74c20dd0f89238068de5bb6ecd4e968eddab339d Mon Sep 17 00:00:00 2001 From: Kartik Date: Thu, 14 Apr 2022 13:05:57 +0530 Subject: [PATCH 288/319] mailbox: tegra-hsp: Add 128-bit shared mailbox support Add support for 128-bit shared mailboxes found on Tegra234 chips. Signed-off-by: Kartik Signed-off-by: Jassi Brar --- drivers/mailbox/tegra-hsp.c | 77 +++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/drivers/mailbox/tegra-hsp.c b/drivers/mailbox/tegra-hsp.c index 1a4c801878b0..573481e436f5 100644 --- a/drivers/mailbox/tegra-hsp.c +++ b/drivers/mailbox/tegra-hsp.c @@ -46,10 +46,18 @@ #define HSP_SM_SHRD_MBOX_FULL_INT_IE 0x04 #define HSP_SM_SHRD_MBOX_EMPTY_INT_IE 0x08 +#define HSP_SHRD_MBOX_TYPE1_TAG 0x40 +#define HSP_SHRD_MBOX_TYPE1_DATA0 0x48 +#define HSP_SHRD_MBOX_TYPE1_DATA1 0x4c +#define HSP_SHRD_MBOX_TYPE1_DATA2 0x50 +#define HSP_SHRD_MBOX_TYPE1_DATA3 0x54 + #define HSP_DB_CCPLEX 1 #define HSP_DB_BPMP 3 #define HSP_DB_MAX 7 +#define HSP_MBOX_TYPE_MASK 0xff + struct tegra_hsp_channel; struct tegra_hsp; @@ -88,6 +96,7 @@ struct tegra_hsp_db_map { struct tegra_hsp_soc { const struct tegra_hsp_db_map *map; bool has_per_mb_ie; + bool has_128_bit_mb; }; struct tegra_hsp { @@ -396,6 +405,51 @@ static const struct tegra_hsp_sm_ops tegra_hsp_sm_32bit_ops = { .recv = tegra_hsp_sm_recv32, }; +static void tegra_hsp_sm_send128(struct tegra_hsp_channel *channel, void *data) +{ + u32 value[4]; + + memcpy(value, data, sizeof(value)); + + /* Copy data */ + tegra_hsp_channel_writel(channel, value[0], HSP_SHRD_MBOX_TYPE1_DATA0); + tegra_hsp_channel_writel(channel, value[1], HSP_SHRD_MBOX_TYPE1_DATA1); + tegra_hsp_channel_writel(channel, value[2], HSP_SHRD_MBOX_TYPE1_DATA2); + tegra_hsp_channel_writel(channel, value[3], HSP_SHRD_MBOX_TYPE1_DATA3); + + /* Update tag to mark mailbox full */ + tegra_hsp_channel_writel(channel, HSP_SM_SHRD_MBOX_FULL, + HSP_SHRD_MBOX_TYPE1_TAG); +} + +static void tegra_hsp_sm_recv128(struct tegra_hsp_channel *channel) +{ + u32 value[4]; + void *msg; + + value[0] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA0); + value[1] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA1); + value[2] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA2); + value[3] = tegra_hsp_channel_readl(channel, HSP_SHRD_MBOX_TYPE1_DATA3); + + msg = (void *)(unsigned long)value; + mbox_chan_received_data(channel->chan, msg); + + /* + * Clear data registers and tag. + */ + tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_DATA0); + tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_DATA1); + tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_DATA2); + tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_DATA3); + tegra_hsp_channel_writel(channel, 0x0, HSP_SHRD_MBOX_TYPE1_TAG); +} + +static const struct tegra_hsp_sm_ops tegra_hsp_sm_128bit_ops = { + .send = tegra_hsp_sm_send128, + .recv = tegra_hsp_sm_recv128, +}; + static int tegra_hsp_mailbox_send_data(struct mbox_chan *chan, void *data) { struct tegra_hsp_mailbox *mb = chan->con_priv; @@ -571,12 +625,20 @@ static struct mbox_chan *tegra_hsp_sm_xlate(struct mbox_controller *mbox, index = args->args[1] & TEGRA_HSP_SM_MASK; - if (type != TEGRA_HSP_MBOX_TYPE_SM || !hsp->shared_irqs || - index >= hsp->num_sm) + if ((type & HSP_MBOX_TYPE_MASK) != TEGRA_HSP_MBOX_TYPE_SM || + !hsp->shared_irqs || index >= hsp->num_sm) return ERR_PTR(-ENODEV); mb = &hsp->mailboxes[index]; - mb->ops = &tegra_hsp_sm_32bit_ops; + + if (type & TEGRA_HSP_MBOX_TYPE_SM_128BIT) { + if (!hsp->soc->has_128_bit_mb) + return ERR_PTR(-ENODEV); + + mb->ops = &tegra_hsp_sm_128bit_ops; + } else { + mb->ops = &tegra_hsp_sm_32bit_ops; + } if ((args->args[1] & TEGRA_HSP_SM_FLAG_TX) == 0) mb->producer = false; @@ -853,16 +915,25 @@ static const struct tegra_hsp_db_map tegra186_hsp_db_map[] = { static const struct tegra_hsp_soc tegra186_hsp_soc = { .map = tegra186_hsp_db_map, .has_per_mb_ie = false, + .has_128_bit_mb = false, }; static const struct tegra_hsp_soc tegra194_hsp_soc = { .map = tegra186_hsp_db_map, .has_per_mb_ie = true, + .has_128_bit_mb = false, +}; + +static const struct tegra_hsp_soc tegra234_hsp_soc = { + .map = tegra186_hsp_db_map, + .has_per_mb_ie = false, + .has_128_bit_mb = true, }; static const struct of_device_id tegra_hsp_match[] = { { .compatible = "nvidia,tegra186-hsp", .data = &tegra186_hsp_soc }, { .compatible = "nvidia,tegra194-hsp", .data = &tegra194_hsp_soc }, + { .compatible = "nvidia,tegra234-hsp", .data = &tegra234_hsp_soc }, { } }; From dea27cda46116afea45fdaade52adedb732d77ab Mon Sep 17 00:00:00 2001 From: Tinghan Shen Date: Fri, 22 Apr 2022 10:39:08 +0800 Subject: [PATCH 289/319] dt-bindings: mailbox: mtk,adsp-mbox: add mt8186 compatible name Add compatible name for MediaTek MT8186 SoC ADSP mailbox. Signed-off-by: Tinghan Shen Acked-by: Rob Herring Signed-off-by: Jassi Brar --- .../devicetree/bindings/mailbox/mtk,adsp-mbox.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/mtk,adsp-mbox.yaml b/Documentation/devicetree/bindings/mailbox/mtk,adsp-mbox.yaml index fe454a1fba17..72c1d9e82c89 100644 --- a/Documentation/devicetree/bindings/mailbox/mtk,adsp-mbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/mtk,adsp-mbox.yaml @@ -11,14 +11,15 @@ maintainers: description: | The MTK ADSP mailbox Inter-Processor Communication (IPC) enables the SoC - to ommunicate with ADSP by passing messages through two mailbox channels. + to communicate with ADSP by passing messages through two mailbox channels. The MTK ADSP mailbox IPC also provides the ability for one processor to signal the other processor using interrupts. properties: compatible: - items: - - const: mediatek,mt8195-adsp-mbox + enum: + - mediatek,mt8195-adsp-mbox + - mediatek,mt8186-adsp-mbox "#mbox-cells": const: 0 From 02b5c35a617137ccad8d6988805ab54c3a9efc81 Mon Sep 17 00:00:00 2001 From: Tinghan Shen Date: Fri, 22 Apr 2022 10:39:09 +0800 Subject: [PATCH 290/319] mailbox: mediatek: support mt8186 adsp mailbox Add support of mt8186 adsp mailbox. Signed-off-by: Tinghan Shen Signed-off-by: Jassi Brar --- drivers/mailbox/mtk-adsp-mailbox.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/mailbox/mtk-adsp-mailbox.c b/drivers/mailbox/mtk-adsp-mailbox.c index 5e7378090c7b..14bc0057de81 100644 --- a/drivers/mailbox/mtk-adsp-mailbox.c +++ b/drivers/mailbox/mtk-adsp-mailbox.c @@ -149,6 +149,13 @@ static int mtk_adsp_mbox_probe(struct platform_device *pdev) return devm_mbox_controller_register(dev, &priv->mbox); } +static const struct mtk_adsp_mbox_cfg mt8186_adsp_mbox_cfg = { + .set_in = 0x00, + .set_out = 0x04, + .clr_in = 0x08, + .clr_out = 0x0C, +}; + static const struct mtk_adsp_mbox_cfg mt8195_adsp_mbox_cfg = { .set_in = 0x00, .set_out = 0x1c, @@ -157,6 +164,7 @@ static const struct mtk_adsp_mbox_cfg mt8195_adsp_mbox_cfg = { }; static const struct of_device_id mtk_adsp_mbox_of_match[] = { + { .compatible = "mediatek,mt8186-adsp-mbox", .data = &mt8186_adsp_mbox_cfg }, { .compatible = "mediatek,mt8195-adsp-mbox", .data = &mt8195_adsp_mbox_cfg }, {}, }; From 504ff5b00853b05133c9cddfc351548b48cc5bdc Mon Sep 17 00:00:00 2001 From: ran jianping Date: Wed, 27 Apr 2022 05:41:00 +0000 Subject: [PATCH 291/319] mailbox:imx: using pm_runtime_resume_and_get Using pm_runtime_resume_and_get() to replace pm_runtime_get_sync and pm_runtime_put_noidle. This change is just to simplify the code, no actual functional changes. Reported-by: Zeal Robot Signed-off-by: ran jianping Signed-off-by: Jassi Brar --- drivers/mailbox/imx-mailbox.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index df8a785be324..b10239d6ef93 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -830,11 +830,9 @@ static int imx_mu_probe(struct platform_device *pdev) pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put_noidle(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) goto disable_runtime_pm; - } ret = pm_runtime_put_sync(dev); if (ret < 0) From d9512696082bcfd068ce65476c85c5b17511d4df Mon Sep 17 00:00:00 2001 From: ran jianping Date: Thu, 28 Apr 2022 06:42:09 +0000 Subject: [PATCH 292/319] mailbox: omap: using pm_runtime_resume_and_get to simplify the code Using pm_runtime_resume_and_get() to replace pm_runtime_get_sync and pm_runtime_put_noidle. This change is just to simplify the code, no actual functional changes. Reported-by: Zeal Robot Signed-off-by: ran jianping Signed-off-by: Jassi Brar --- drivers/mailbox/omap-mailbox.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/mailbox/omap-mailbox.c b/drivers/mailbox/omap-mailbox.c index 58f3d569f095..098c82d87137 100644 --- a/drivers/mailbox/omap-mailbox.c +++ b/drivers/mailbox/omap-mailbox.c @@ -856,11 +856,9 @@ static int omap_mbox_probe(struct platform_device *pdev) platform_set_drvdata(pdev, mdev); pm_runtime_enable(mdev->dev); - ret = pm_runtime_get_sync(mdev->dev); - if (ret < 0) { - pm_runtime_put_noidle(mdev->dev); + ret = pm_runtime_resume_and_get(mdev->dev); + if (ret < 0) goto unregister; - } /* * just print the raw revision register, the format is not From a022c7c96ca1c7d8b4d2702973bf4b1a7cff5958 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Sun, 1 May 2022 12:34:27 +0200 Subject: [PATCH 293/319] mailbox: correct kerneldoc Correct kerneldoc warnings like: drivers/mailbox/arm_mhu_db.c:47: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst drivers/mailbox/qcom-ipcc.c:58: warning: Function parameter or member 'num_chans' not described in 'qcom_ipcc' Signed-off-by: Krzysztof Kozlowski Acked-by: Viresh Kumar Acked-by: Sudeep Holla Signed-off-by: Jassi Brar --- drivers/mailbox/arm_mhu_db.c | 2 +- drivers/mailbox/arm_mhuv2.c | 3 ++- drivers/mailbox/qcom-ipcc.c | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/mailbox/arm_mhu_db.c b/drivers/mailbox/arm_mhu_db.c index 8674153cc893..aa0a4d83880f 100644 --- a/drivers/mailbox/arm_mhu_db.c +++ b/drivers/mailbox/arm_mhu_db.c @@ -44,7 +44,7 @@ struct arm_mhu { }; /** - * ARM MHU Mailbox allocated channel information + * struct mhu_db_channel - ARM MHU Mailbox allocated channel information * * @mhu: Pointer to parent mailbox device * @pchan: Physical channel within which this doorbell resides in diff --git a/drivers/mailbox/arm_mhuv2.c b/drivers/mailbox/arm_mhuv2.c index d997f8ebfa98..a47aef8df52f 100644 --- a/drivers/mailbox/arm_mhuv2.c +++ b/drivers/mailbox/arm_mhuv2.c @@ -160,7 +160,8 @@ enum mhuv2_frame { * struct mhuv2 - MHUv2 mailbox controller data * * @mbox: Mailbox controller belonging to the MHU frame. - * @send/recv: Base address of the register mapping region. + * @send: Base address of the register mapping region. + * @recv: Base address of the register mapping region. * @frame: Frame type: RECEIVER_FRAME or SENDER_FRAME. * @irq: Interrupt. * @windows: Channel windows implemented by the platform. diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c index c5d963222014..881706da59c0 100644 --- a/drivers/mailbox/qcom-ipcc.c +++ b/drivers/mailbox/qcom-ipcc.c @@ -41,9 +41,10 @@ struct qcom_ipcc_chan_info { * @dev: Device associated with this instance * @base: Base address of the IPCC frame associated to APSS * @irq_domain: The irq_domain associated with this instance - * @chan: The mailbox channels array + * @chans: The mailbox channels array * @mchan: The per-mailbox channel info array * @mbox: The mailbox controller + * @num_chans: Number of @chans elements * @irq: Summary irq */ struct qcom_ipcc { From 9accf46b7fb893a46eff3a57407b02ac227a7cd7 Mon Sep 17 00:00:00 2001 From: Fabien Dessenne Date: Tue, 3 May 2022 16:55:59 +0200 Subject: [PATCH 294/319] dt-bindings: mailbox: remove the IPCC "wakeup" IRQ The stm32 ipcc mailbox driver supports only two interrupts (rx and tx), so remove the unsupported "wakeup" one. Signed-off-by: Fabien Dessenne Reviewed-by: Rob Herring Signed-off-by: Jassi Brar --- .../devicetree/bindings/mailbox/st,stm32-ipcc.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Documentation/devicetree/bindings/mailbox/st,stm32-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/st,stm32-ipcc.yaml index 8eb4bf52ea27..2c8b47285aa3 100644 --- a/Documentation/devicetree/bindings/mailbox/st,stm32-ipcc.yaml +++ b/Documentation/devicetree/bindings/mailbox/st,stm32-ipcc.yaml @@ -30,15 +30,11 @@ properties: items: - description: rx channel occupied - description: tx channel free - - description: wakeup source - minItems: 2 interrupt-names: items: - const: rx - const: tx - - const: wakeup - minItems: 2 wakeup-source: true @@ -70,10 +66,9 @@ examples: #mbox-cells = <1>; reg = <0x4c001000 0x400>; st,proc-id = <0>; - interrupts-extended = <&intc GIC_SPI 100 IRQ_TYPE_NONE>, - <&intc GIC_SPI 101 IRQ_TYPE_NONE>, - <&aiec 62 1>; - interrupt-names = "rx", "tx", "wakeup"; + interrupts-extended = <&exti 61 1>, + <&intc GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "rx", "tx"; clocks = <&rcc_clk IPCC>; wakeup-source; }; From 369e4ef87a8f5da7c348ec2c61ec5cd726e8337a Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 9 May 2022 09:17:16 -0500 Subject: [PATCH 295/319] mailbox: pcc: Fix an invalid-load caught by the address sanitizer `pcc_mailbox_probe` doesn't initialize all memory that has been allocated before the first time that one of it's members `txdone_irq` may be accessed. This leads to a an invalid load any time that this member is accessed: [ 2.429769] UBSAN: invalid-load in drivers/mailbox/pcc.c:684:22 [ 2.430324] UBSAN: invalid-load in drivers/mailbox/mailbox.c:486:12 [ 4.276782] UBSAN: invalid-load in drivers/acpi/cppc_acpi.c:314:45 Link: https://bugzilla.kernel.org/show_bug.cgi?id=215587 Fixes: ce028702ddbc ("mailbox: pcc: Move bulk of PCCT parsing into pcc_mbox_probe") Signed-off-by: Mario Limonciello Reviewed-by: Sudeep Holla Signed-off-by: Jassi Brar --- drivers/mailbox/pcc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index ed18936b8ce6..ebfa33a40fce 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -654,7 +654,7 @@ static int pcc_mbox_probe(struct platform_device *pdev) goto err; } - pcc_mbox_ctrl = devm_kmalloc(dev, sizeof(*pcc_mbox_ctrl), GFP_KERNEL); + pcc_mbox_ctrl = devm_kzalloc(dev, sizeof(*pcc_mbox_ctrl), GFP_KERNEL); if (!pcc_mbox_ctrl) { rc = -ENOMEM; goto err; From c25f77899753fdd6f43fa6999abd53fd0059496e Mon Sep 17 00:00:00 2001 From: Prasad Sodagudi Date: Tue, 17 May 2022 16:13:00 +0530 Subject: [PATCH 296/319] mailbox: qcom-ipcc: Log the pending interrupt during resume Enable logging of the pending interrupt that triggered device wakeup. This logging information helps to debug IRQs that cause periodic device wakeups by printing the detailed information of pending IPCC interrupts. Scenario: Device wakeup caused by Modem crash Logs: qcom-ipcc mailbox: virq: 182 triggered client-id: 2; signal-id: 2 From the IPCC bindings it can further be understood that the client here is IPCC_CLIENT_MPSS and the signal was IPCC_MPROC_SIGNAL_SMP2P. Reviewed-by: Manivannan Sadhasivam Signed-off-by: Prasad Sodagudi Signed-off-by: Sibi Sankar Signed-off-by: Jassi Brar --- drivers/mailbox/qcom-ipcc.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c index 881706da59c0..2583b20cdeb7 100644 --- a/drivers/mailbox/qcom-ipcc.c +++ b/drivers/mailbox/qcom-ipcc.c @@ -255,6 +255,24 @@ static int qcom_ipcc_setup_mbox(struct qcom_ipcc *ipcc, return devm_mbox_controller_register(dev, mbox); } +static int qcom_ipcc_pm_resume(struct device *dev) +{ + struct qcom_ipcc *ipcc = dev_get_drvdata(dev); + u32 hwirq; + int virq; + + hwirq = readl(ipcc->base + IPCC_REG_RECV_ID); + if (hwirq == IPCC_NO_PENDING_IRQ) + return 0; + + virq = irq_find_mapping(ipcc->irq_domain, hwirq); + + dev_dbg(dev, "virq: %d triggered client-id: %ld; signal-id: %ld\n", virq, + FIELD_GET(IPCC_CLIENT_ID_MASK, hwirq), FIELD_GET(IPCC_SIGNAL_ID_MASK, hwirq)); + + return 0; +} + static int qcom_ipcc_probe(struct platform_device *pdev) { struct qcom_ipcc *ipcc; @@ -325,6 +343,10 @@ static const struct of_device_id qcom_ipcc_of_match[] = { }; MODULE_DEVICE_TABLE(of, qcom_ipcc_of_match); +static const struct dev_pm_ops qcom_ipcc_dev_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, qcom_ipcc_pm_resume) +}; + static struct platform_driver qcom_ipcc_driver = { .probe = qcom_ipcc_probe, .remove = qcom_ipcc_remove, @@ -332,6 +354,7 @@ static struct platform_driver qcom_ipcc_driver = { .name = "qcom-ipcc", .of_match_table = qcom_ipcc_of_match, .suppress_bind_attrs = true, + .pm = pm_sleep_ptr(&qcom_ipcc_dev_pm_ops), }, }; From c680c6a814a2269427fad9ac417ab16756bceae9 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Fri, 20 May 2022 21:21:56 +0200 Subject: [PATCH 297/319] gpio: sim: Use correct order for the parameters of devm_kcalloc() We should have 'n', then 'size', not the opposite. This is harmless because the 2 values are just multiplied, but having the correct order silence a (unpublished yet) smatch warning. Fixes: cb8c474e79be ("gpio: sim: new testing module") Signed-off-by: Christophe JAILLET Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-sim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c index 8e5d87984a48..234337497826 100644 --- a/drivers/gpio/gpio-sim.c +++ b/drivers/gpio/gpio-sim.c @@ -314,8 +314,8 @@ static int gpio_sim_setup_sysfs(struct gpio_sim_chip *chip) for (i = 0; i < num_lines; i++) { attr_group = devm_kzalloc(dev, sizeof(*attr_group), GFP_KERNEL); - attrs = devm_kcalloc(dev, sizeof(*attrs), - GPIO_SIM_NUM_ATTRS, GFP_KERNEL); + attrs = devm_kcalloc(dev, GPIO_SIM_NUM_ATTRS, sizeof(*attrs), + GFP_KERNEL); val_attr = devm_kzalloc(dev, sizeof(*val_attr), GFP_KERNEL); pull_attr = devm_kzalloc(dev, sizeof(*pull_attr), GFP_KERNEL); if (!attr_group || !attrs || !val_attr || !pull_attr) From bca1a1004615efe141fd78f360ecc48c60bc4ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ard=C3=B6?= Date: Thu, 31 Mar 2022 09:01:15 +0200 Subject: [PATCH 298/319] mailbox: forward the hrtimer if not queued and under a lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit c7dacf5b0f32957b24ef29df1207dc2cd8307743, "mailbox: avoid timer start from callback" The previous commit was reverted since it lead to a race that caused the hrtimer to not be started at all. The check for hrtimer_active() in msg_submit() will return true if the callback function txdone_hrtimer() is currently running. This function could return HRTIMER_NORESTART and then the timer will not be restarted, and also msg_submit() will not start the timer. This will lead to a message actually being submitted but no timer will start to check for its compleation. The original fix that added checking hrtimer_active() was added to avoid a warning with hrtimer_forward. Looking in the kernel another solution to avoid this warning is to check hrtimer_is_queued() before calling hrtimer_forward_now() instead. This however requires a lock so the timer is not started by msg_submit() inbetween this check and the hrtimer_forward() call. Fixes: c7dacf5b0f32 ("mailbox: avoid timer start from callback") Signed-off-by: Björn Ardö Signed-off-by: Jassi Brar --- drivers/mailbox/mailbox.c | 19 +++++++++++++------ include/linux/mailbox_controller.h | 1 + 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 3e7d4b20ab34..4229b9b5da98 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -82,11 +82,11 @@ static void msg_submit(struct mbox_chan *chan) exit: spin_unlock_irqrestore(&chan->lock, flags); - /* kick start the timer immediately to avoid delays */ if (!err && (chan->txdone_method & TXDONE_BY_POLL)) { - /* but only if not already active */ - if (!hrtimer_active(&chan->mbox->poll_hrt)) - hrtimer_start(&chan->mbox->poll_hrt, 0, HRTIMER_MODE_REL); + /* kick start the timer immediately to avoid delays */ + spin_lock_irqsave(&chan->mbox->poll_hrt_lock, flags); + hrtimer_start(&chan->mbox->poll_hrt, 0, HRTIMER_MODE_REL); + spin_unlock_irqrestore(&chan->mbox->poll_hrt_lock, flags); } } @@ -120,20 +120,26 @@ static enum hrtimer_restart txdone_hrtimer(struct hrtimer *hrtimer) container_of(hrtimer, struct mbox_controller, poll_hrt); bool txdone, resched = false; int i; + unsigned long flags; for (i = 0; i < mbox->num_chans; i++) { struct mbox_chan *chan = &mbox->chans[i]; if (chan->active_req && chan->cl) { - resched = true; txdone = chan->mbox->ops->last_tx_done(chan); if (txdone) tx_tick(chan, 0); + else + resched = true; } } if (resched) { - hrtimer_forward_now(hrtimer, ms_to_ktime(mbox->txpoll_period)); + spin_lock_irqsave(&mbox->poll_hrt_lock, flags); + if (!hrtimer_is_queued(hrtimer)) + hrtimer_forward_now(hrtimer, ms_to_ktime(mbox->txpoll_period)); + spin_unlock_irqrestore(&mbox->poll_hrt_lock, flags); + return HRTIMER_RESTART; } return HRTIMER_NORESTART; @@ -500,6 +506,7 @@ int mbox_controller_register(struct mbox_controller *mbox) hrtimer_init(&mbox->poll_hrt, CLOCK_MONOTONIC, HRTIMER_MODE_REL); mbox->poll_hrt.function = txdone_hrtimer; + spin_lock_init(&mbox->poll_hrt_lock); } for (i = 0; i < mbox->num_chans; i++) { diff --git a/include/linux/mailbox_controller.h b/include/linux/mailbox_controller.h index 36d6ce673503..6fee33cb52f5 100644 --- a/include/linux/mailbox_controller.h +++ b/include/linux/mailbox_controller.h @@ -83,6 +83,7 @@ struct mbox_controller { const struct of_phandle_args *sp); /* Internal to API */ struct hrtimer poll_hrt; + spinlock_t poll_hrt_lock; struct list_head node; }; From a6c13d87b4b7ae7f7f4574014f5478f7fe92540c Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Wed, 18 May 2022 15:35:23 +0700 Subject: [PATCH 299/319] platform/chrome: Use tables for values lists of ChromeOS ACPI sysfs ABI Stephen Rothwell reported htmldocs warning when merging chrome-platform tree: Documentation/ABI/testing/sysfs-driver-chromeos-acpi:2: WARNING: Unexpected indentation. Documentation/ABI/testing/sysfs-driver-chromeos-acpi:11: WARNING: Unexpected indentation. Documentation/ABI/testing/sysfs-driver-chromeos-acpi:22: WARNING: Unexpected indentation. Documentation/ABI/testing/sysfs-driver-chromeos-acpi:56: WARNING: Unexpected indentation. The warning is due to missing blank line separator surrounding values lists. Instead of using lists, use tables to be consistent with rest of sysfs ABI documentation. Link: https://lore.kernel.org/linux-next/20220516213832.19127535@canb.auug.org.au/ Fixes: 0a4cad9c11ad ("platform/chrome: Add ChromeOS ACPI device driver") Reported-by: Stephen Rothwell Cc: Benson Leung Cc: Guenter Roeck Cc: Andy Shevchenko Cc: Greg Kroah-Hartman Cc: Rafael J. Wysocki Cc: Dmitry Torokhov Cc: Enric Balletbo i Serra Cc: Muhammad Usama Anjum Cc: Tzung-Bi Shih Cc: Linux Kernel Mailing List Cc: Linux Next Mailing List Signed-off-by: Bagas Sanjaya Reviewed-by: Muhammad Usama Anjum Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220518083524.37380-2-bagasdotme@gmail.com [tzungbi: fix the Fixes tag to 12-chars hash.] Signed-off-by: Tzung-Bi Shih --- .../ABI/testing/sysfs-driver-chromeos-acpi | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-chromeos-acpi b/Documentation/ABI/testing/sysfs-driver-chromeos-acpi index 5b59ef9d7b37..cce453951700 100644 --- a/Documentation/ABI/testing/sysfs-driver-chromeos-acpi +++ b/Documentation/ABI/testing/sysfs-driver-chromeos-acpi @@ -4,8 +4,11 @@ KernelVersion: 5.19 Description: This file shows information about the current boot of the active EC firmware. - * 0 - Read only (recovery) firmware. - * 1 - Rewritable firmware. + + == =============================== + 0 Read only (recovery) firmware. + 1 Rewritable firmware. + == =============================== What: /sys/bus/platform/devices/GGL0001:*/BINF.3 Date: May 2022 @@ -13,23 +16,28 @@ KernelVersion: 5.19 Description: This file shows information about the current boot of the active main firmware type. - * 0 - Recovery. - * 1 - Normal. - * 2 - Developer. - * 3 - Netboot (factory installation only). + + == ===================================== + 0 Recovery. + 1 Normal. + 2 Developer. + 3 Netboot (factory installation only). + == ===================================== What: /sys/bus/platform/devices/GGL0001:*/CHSW Date: May 2022 KernelVersion: 5.19 Description: This file shows the switch position for the Chrome OS specific - hardware switches. - * 0 - No changes. - * 2 - Recovery button was pressed when firmware booted. - * 4 - Recovery button was pressed when EC firmware booted. - * 32 - Developer switch was enabled when firmware booted. - * 512 - Firmware write protection was disabled when firmware - booted. + hardware switches when the firmware is booted. + + ==== =========================================== + 0 No changes. + 2 Recovery button was pressed. + 4 Recovery button was pressed (EC firmware). + 32 Developer switch was enabled. + 512 Firmware write protection was disabled. + ==== =========================================== What: /sys/bus/platform/devices/GGL0001:*/FMAP Date: May 2022 @@ -58,18 +66,24 @@ KernelVersion: 5.19 Description: This file shows the type of the GPIO signal for the Chrome OS specific GPIO assignments. - * 1 - Recovery button. - * 2 - Developer mode switch. - * 3 - Firmware write protection switch. - * 256 to 511 - Debug header GPIO 0 to GPIO 255. + + =========== ================================== + 1 Recovery button. + 2 Developer mode switch. + 3 Firmware write protection switch. + 256 to 511 Debug header GPIO 0 to GPIO 255. + =========== ================================== What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.1 Date: May 2022 KernelVersion: 5.19 Description: This file shows the signal attributes of the GPIO signal. - * 0 - Signal is active low. - * 1 - Signal is active high. + + == ======================= + 0 Signal is active low. + 1 Signal is active high. + == ======================= What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.2 Date: May 2022 From abd4fd43f2af03e2e852e6b1b98faeee9e3eae55 Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Wed, 18 May 2022 15:35:24 +0700 Subject: [PATCH 300/319] platform/chrome: Use imperative mood for ChromeOS ACPI sysfs ABI descriptions The current descriptions all begin with ("This file shows something") clause, which is incorrect wording since these mean return value from sysfs. Use imperative mood ("Returns something") instead, in line with corresponding ACPI documentation at firmware-guide/. Cc: Benson Leung Cc: Guenter Roeck Cc: Andy Shevchenko Cc: Greg Kroah-Hartman Cc: Rafael J. Wysocki Cc: Dmitry Torokhov Cc: Enric Balletbo i Serra Cc: Muhammad Usama Anjum Cc: Tzung-Bi Shih Cc: Linux Kernel Mailing List Cc: Linux Next Mailing List Signed-off-by: Bagas Sanjaya Reviewed-by: Muhammad Usama Anjum Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20220518083524.37380-3-bagasdotme@gmail.com --- .../ABI/testing/sysfs-driver-chromeos-acpi | 63 +++++++++---------- 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-chromeos-acpi b/Documentation/ABI/testing/sysfs-driver-chromeos-acpi index cce453951700..c308926e1568 100644 --- a/Documentation/ABI/testing/sysfs-driver-chromeos-acpi +++ b/Documentation/ABI/testing/sysfs-driver-chromeos-acpi @@ -2,8 +2,7 @@ What: /sys/bus/platform/devices/GGL0001:*/BINF.2 Date: May 2022 KernelVersion: 5.19 Description: - This file shows information about the current boot of - the active EC firmware. + Returns active EC firmware of current boot (boolean). == =============================== 0 Read only (recovery) firmware. @@ -14,8 +13,7 @@ What: /sys/bus/platform/devices/GGL0001:*/BINF.3 Date: May 2022 KernelVersion: 5.19 Description: - This file shows information about the current boot of - the active main firmware type. + Returns main firmware type for current boot (integer). == ===================================== 0 Recovery. @@ -28,8 +26,8 @@ What: /sys/bus/platform/devices/GGL0001:*/CHSW Date: May 2022 KernelVersion: 5.19 Description: - This file shows the switch position for the Chrome OS specific - hardware switches when the firmware is booted. + Returns switch position for Chrome OS specific hardware + switches when the firmware is booted (integer). ==== =========================================== 0 No changes. @@ -43,29 +41,29 @@ What: /sys/bus/platform/devices/GGL0001:*/FMAP Date: May 2022 KernelVersion: 5.19 Description: - This file shows the physical memory address of the start of - the main processor firmware flashmap. + Returns physical memory address of the start of the main + processor firmware flashmap. What: /sys/bus/platform/devices/GGL0001:*/FRID Date: May 2022 KernelVersion: 5.19 Description: - This file shows the firmware version for the read-only portion - of the main processor firmware. + Returns firmware version for the read-only portion of the + main processor firmware. What: /sys/bus/platform/devices/GGL0001:*/FWID Date: May 2022 KernelVersion: 5.19 Description: - This file shows the firmware version for the rewritable portion - of the main processor firmware. + Returns firmware version for the rewritable portion of the + main processor firmware. What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.0 Date: May 2022 KernelVersion: 5.19 Description: - This file shows the type of the GPIO signal for the Chrome OS - specific GPIO assignments. + Returns type of the GPIO signal for the Chrome OS specific + GPIO assignments (integer). =========== ================================== 1 Recovery button. @@ -78,7 +76,7 @@ What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.1 Date: May 2022 KernelVersion: 5.19 Description: - This file shows the signal attributes of the GPIO signal. + Returns signal attributes of the GPIO signal (integer bitfield). == ======================= 0 Signal is active low. @@ -89,52 +87,51 @@ What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.2 Date: May 2022 KernelVersion: 5.19 Description: - This file shows the GPIO number on the specified GPIO + Returns the GPIO number on the specified GPIO controller. What: /sys/bus/platform/devices/GGL0001:*/GPIO.X/GPIO.3 Date: May 2022 KernelVersion: 5.19 Description: - This file shows the name of the GPIO controller. + Returns name of the GPIO controller. What: /sys/bus/platform/devices/GGL0001:*/HWID Date: May 2022 KernelVersion: 5.19 Description: - This file shows the hardware ID for the Chromebook. + Returns hardware ID for the Chromebook. What: /sys/bus/platform/devices/GGL0001:*/MECK Date: May 2022 KernelVersion: 5.19 Description: - This binary file returns the SHA-1 or SHA-256 hash that is - read out of the Management Engine extended registers during - boot. The hash is exported vi ACPI so the OS can verify that - the Management Engine firmware has not changed. If Management - Engine is not present, or if the firmware was unable to read the - extended registers, this buffer size can be zero. + Returns the SHA-1 or SHA-256 hash that is read out of the + Management Engine extended registers during boot. The hash + is exported via ACPI so the OS can verify that the Management + Engine firmware has not changed. If Management Engine is not + present, or if the firmware was unable to read the extended registers, this buffer size can be zero. What: /sys/bus/platform/devices/GGL0001:*/VBNV.0 Date: May 2022 KernelVersion: 5.19 Description: - This file shows the offset in CMOS bank 0 of the verified boot - non-volatile storage block, counting from the first writable - CMOS byte (that is, 'offset = 0' is the byte following the 14 - bytes of clock data). + Returns offset in CMOS bank 0 of the verified boot non-volatile + storage block, counting from the first writable CMOS byte + (that is, 'offset = 0' is the byte following the 14 bytes of + clock data). What: /sys/bus/platform/devices/GGL0001:*/VBNV.1 Date: May 2022 KernelVersion: 5.19 Description: - This file shows the size in bytes of the verified boot - non-volatile storage block. + Return the size in bytes of the verified boot non-volatile + storage block. What: /sys/bus/platform/devices/GGL0001:*/VDAT Date: May 2022 KernelVersion: 5.19 Description: - This binary file returns the verified boot data block shared - between the firmware verification step and the kernel - verification step. + Returns the verified boot data block shared between the + firmware verification step and the kernel verification step + (binary). From 79f9fbe303520d2c32b70f04f2bb02cc2baaa4c3 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 23 May 2022 15:47:02 -0700 Subject: [PATCH 301/319] mailbox: qcom-ipcc: Fix -Wunused-function with CONFIG_PM_SLEEP=n When CONFIG_PM_SLEEP is not set, there is a warning that qcom_ipcc_pm_resume() is unused: drivers/mailbox/qcom-ipcc.c:258:12: error: 'qcom_ipcc_pm_resume' defined but not used [-Werror=unused-function] 258 | static int qcom_ipcc_pm_resume(struct device *dev) | ^~~~~~~~~~~~~~~~~~~ cc1: all warnings being treated as errors Commit 1a3c7bb08826 ("PM: core: Add new *_PM_OPS macros, deprecate old ones") reworked the PM_OPS macros to avoid this problem. Use NOIRQ_SYSTEM_SLEEP_PM_OPS directly so that qcom_ipcc_pm_resume() always appears to be used to the compiler, even though it will be dead code eliminated in the !CONFIG_PM_SLEEP case. Fixes: c25f77899753 ("mailbox: qcom-ipcc: Log the pending interrupt during resume") Signed-off-by: Nathan Chancellor Reviewed-by: Sibi Sankar Signed-off-by: Jassi Brar --- drivers/mailbox/qcom-ipcc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c index 2583b20cdeb7..31d58b7d55fe 100644 --- a/drivers/mailbox/qcom-ipcc.c +++ b/drivers/mailbox/qcom-ipcc.c @@ -344,7 +344,7 @@ static const struct of_device_id qcom_ipcc_of_match[] = { MODULE_DEVICE_TABLE(of, qcom_ipcc_of_match); static const struct dev_pm_ops qcom_ipcc_dev_pm_ops = { - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, qcom_ipcc_pm_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, qcom_ipcc_pm_resume) }; static struct platform_driver qcom_ipcc_driver = { From cfc2b00ebed6660a10d1be09f2dd6957556ce6a5 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 20 May 2022 12:23:18 +0200 Subject: [PATCH 302/319] gpio: dwapb: Make the irqchip immutable Commit 6c846d026d49 ("gpio: Don't fiddle with irqchips marked as immutable") added a warning to indicate if the gpiolib is altering the internals of irqchips. Following this change the following warning is now observed for the dwapb driver: gpio gpiochip0: (50200000.gpio): not an immutable chip, please consider fixing it! Fix this by making the irqchip in the dwapb driver immutable. Signed-off-by: Geert Uytterhoeven Reviewed-by: Damien Le Moal Reviewed-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-dwapb.c | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c index 7130195da48d..04afe728e187 100644 --- a/drivers/gpio/gpio-dwapb.c +++ b/drivers/gpio/gpio-dwapb.c @@ -95,7 +95,6 @@ struct dwapb_context { #endif struct dwapb_gpio_port_irqchip { - struct irq_chip irqchip; unsigned int nr_irqs; unsigned int irq[DWAPB_MAX_GPIOS]; }; @@ -252,24 +251,30 @@ static void dwapb_irq_mask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct dwapb_gpio *gpio = to_dwapb_gpio(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); unsigned long flags; u32 val; raw_spin_lock_irqsave(&gc->bgpio_lock, flags); - val = dwapb_read(gpio, GPIO_INTMASK) | BIT(irqd_to_hwirq(d)); + val = dwapb_read(gpio, GPIO_INTMASK) | BIT(hwirq); dwapb_write(gpio, GPIO_INTMASK, val); raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); + + gpiochip_disable_irq(gc, hwirq); } static void dwapb_irq_unmask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct dwapb_gpio *gpio = to_dwapb_gpio(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); unsigned long flags; u32 val; + gpiochip_enable_irq(gc, hwirq); + raw_spin_lock_irqsave(&gc->bgpio_lock, flags); - val = dwapb_read(gpio, GPIO_INTMASK) & ~BIT(irqd_to_hwirq(d)); + val = dwapb_read(gpio, GPIO_INTMASK) & ~BIT(hwirq); dwapb_write(gpio, GPIO_INTMASK, val); raw_spin_unlock_irqrestore(&gc->bgpio_lock, flags); } @@ -364,8 +369,23 @@ static int dwapb_irq_set_wake(struct irq_data *d, unsigned int enable) return 0; } +#else +#define dwapb_irq_set_wake NULL #endif +static const struct irq_chip dwapb_irq_chip = { + .name = DWAPB_DRIVER_NAME, + .irq_ack = dwapb_irq_ack, + .irq_mask = dwapb_irq_mask, + .irq_unmask = dwapb_irq_unmask, + .irq_set_type = dwapb_irq_set_type, + .irq_enable = dwapb_irq_enable, + .irq_disable = dwapb_irq_disable, + .irq_set_wake = dwapb_irq_set_wake, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + static int dwapb_gpio_set_debounce(struct gpio_chip *gc, unsigned offset, unsigned debounce) { @@ -439,16 +459,6 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio, girq->default_type = IRQ_TYPE_NONE; port->pirq = pirq; - pirq->irqchip.name = DWAPB_DRIVER_NAME; - pirq->irqchip.irq_ack = dwapb_irq_ack; - pirq->irqchip.irq_mask = dwapb_irq_mask; - pirq->irqchip.irq_unmask = dwapb_irq_unmask; - pirq->irqchip.irq_set_type = dwapb_irq_set_type; - pirq->irqchip.irq_enable = dwapb_irq_enable; - pirq->irqchip.irq_disable = dwapb_irq_disable; -#ifdef CONFIG_PM_SLEEP - pirq->irqchip.irq_set_wake = dwapb_irq_set_wake; -#endif /* * Intel ACPI-based platforms mostly have the DesignWare APB GPIO @@ -475,7 +485,7 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio, girq->parent_handler = dwapb_irq_handler; } - girq->chip = &pirq->irqchip; + gpio_irq_chip_set_chip(girq, &dwapb_irq_chip); return; From ac2f6f9385aa27b9a73156885be9765299b8e11f Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 20 May 2022 12:21:54 +0200 Subject: [PATCH 303/319] gpio: pca953x: Make the irqchip immutable Commit 6c846d026d49 ("gpio: Don't fiddle with irqchips marked as immutable") added a warning to indicate if the gpiolib is altering the internals of irqchips. Following this change the following warning is now observed for the pca953x driver: gpio gpiochip7: (0-0020): not an immutable chip, please consider fixing it! Fix this by making the irqchip in the pca953x driver immutable. Signed-off-by: Geert Uytterhoeven Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-pca953x.c | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index 4909175e6921..81d9c2113707 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -201,7 +201,6 @@ struct pca953x_chip { DECLARE_BITMAP(irq_stat, MAX_LINE); DECLARE_BITMAP(irq_trig_raise, MAX_LINE); DECLARE_BITMAP(irq_trig_fall, MAX_LINE); - struct irq_chip irq_chip; #endif atomic_t wakeup_path; @@ -629,6 +628,7 @@ static void pca953x_irq_mask(struct irq_data *d) irq_hw_number_t hwirq = irqd_to_hwirq(d); clear_bit(hwirq, chip->irq_mask); + gpiochip_disable_irq(gc, hwirq); } static void pca953x_irq_unmask(struct irq_data *d) @@ -637,6 +637,7 @@ static void pca953x_irq_unmask(struct irq_data *d) struct pca953x_chip *chip = gpiochip_get_data(gc); irq_hw_number_t hwirq = irqd_to_hwirq(d); + gpiochip_enable_irq(gc, hwirq); set_bit(hwirq, chip->irq_mask); } @@ -721,6 +722,26 @@ static void pca953x_irq_shutdown(struct irq_data *d) clear_bit(hwirq, chip->irq_trig_fall); } +static void pca953x_irq_print_chip(struct irq_data *data, struct seq_file *p) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + + seq_printf(p, dev_name(gc->parent)); +} + +static const struct irq_chip pca953x_irq_chip = { + .irq_mask = pca953x_irq_mask, + .irq_unmask = pca953x_irq_unmask, + .irq_set_wake = pca953x_irq_set_wake, + .irq_bus_lock = pca953x_irq_bus_lock, + .irq_bus_sync_unlock = pca953x_irq_bus_sync_unlock, + .irq_set_type = pca953x_irq_set_type, + .irq_shutdown = pca953x_irq_shutdown, + .irq_print_chip = pca953x_irq_print_chip, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + static bool pca953x_irq_pending(struct pca953x_chip *chip, unsigned long *pending) { struct gpio_chip *gc = &chip->gpio_chip; @@ -812,7 +833,6 @@ static irqreturn_t pca953x_irq_handler(int irq, void *devid) static int pca953x_irq_setup(struct pca953x_chip *chip, int irq_base) { struct i2c_client *client = chip->client; - struct irq_chip *irq_chip = &chip->irq_chip; DECLARE_BITMAP(reg_direction, MAX_LINE); DECLARE_BITMAP(irq_stat, MAX_LINE); struct gpio_irq_chip *girq; @@ -846,17 +866,8 @@ static int pca953x_irq_setup(struct pca953x_chip *chip, int irq_base) bitmap_and(chip->irq_stat, irq_stat, reg_direction, chip->gpio_chip.ngpio); mutex_init(&chip->irq_lock); - irq_chip->name = dev_name(&client->dev); - irq_chip->irq_mask = pca953x_irq_mask; - irq_chip->irq_unmask = pca953x_irq_unmask; - irq_chip->irq_set_wake = pca953x_irq_set_wake; - irq_chip->irq_bus_lock = pca953x_irq_bus_lock; - irq_chip->irq_bus_sync_unlock = pca953x_irq_bus_sync_unlock; - irq_chip->irq_set_type = pca953x_irq_set_type; - irq_chip->irq_shutdown = pca953x_irq_shutdown; - girq = &chip->gpio_chip.irq; - girq->chip = irq_chip; + gpio_irq_chip_set_chip(girq, &pca953x_irq_chip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; From 61550be779dcb82cb96852721bac652144a0d0b9 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 20 May 2022 12:20:57 +0200 Subject: [PATCH 304/319] gpio: pcf857x: Make the irqchip immutable Commit 6c846d026d49 ("gpio: Don't fiddle with irqchips marked as immutable") added a warning to indicate if the gpiolib is altering the internals of irqchips. Following this change the following warning is now observed for the pcf857x driver: gpio gpiochip1: (pcf8575): not an immutable chip, please consider fixing it! Fix this by making the irqchip in the pcf857x driver immutable. Signed-off-by: Geert Uytterhoeven Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-pcf857x.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c index e3a53dd5df1e..59cc27e4de51 100644 --- a/drivers/gpio/gpio-pcf857x.c +++ b/drivers/gpio/gpio-pcf857x.c @@ -71,7 +71,6 @@ MODULE_DEVICE_TABLE(of, pcf857x_of_table); */ struct pcf857x { struct gpio_chip chip; - struct irq_chip irqchip; struct i2c_client *client; struct mutex lock; /* protect 'out' */ unsigned out; /* software latch */ @@ -203,15 +202,19 @@ static int pcf857x_irq_set_wake(struct irq_data *data, unsigned int on) static void pcf857x_irq_enable(struct irq_data *data) { struct pcf857x *gpio = irq_data_get_irq_chip_data(data); + irq_hw_number_t hwirq = irqd_to_hwirq(data); - gpio->irq_enabled |= (1 << data->hwirq); + gpiochip_enable_irq(&gpio->chip, hwirq); + gpio->irq_enabled |= (1 << hwirq); } static void pcf857x_irq_disable(struct irq_data *data) { struct pcf857x *gpio = irq_data_get_irq_chip_data(data); + irq_hw_number_t hwirq = irqd_to_hwirq(data); - gpio->irq_enabled &= ~(1 << data->hwirq); + gpio->irq_enabled &= ~(1 << hwirq); + gpiochip_disable_irq(&gpio->chip, hwirq); } static void pcf857x_irq_bus_lock(struct irq_data *data) @@ -228,6 +231,20 @@ static void pcf857x_irq_bus_sync_unlock(struct irq_data *data) mutex_unlock(&gpio->lock); } +static const struct irq_chip pcf857x_irq_chip = { + .name = "pcf857x", + .irq_enable = pcf857x_irq_enable, + .irq_disable = pcf857x_irq_disable, + .irq_ack = noop, + .irq_mask = noop, + .irq_unmask = noop, + .irq_set_wake = pcf857x_irq_set_wake, + .irq_bus_lock = pcf857x_irq_bus_lock, + .irq_bus_sync_unlock = pcf857x_irq_bus_sync_unlock, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + /*-------------------------------------------------------------------------*/ static int pcf857x_probe(struct i2c_client *client, @@ -338,16 +355,6 @@ static int pcf857x_probe(struct i2c_client *client, if (client->irq) { struct gpio_irq_chip *girq; - gpio->irqchip.name = "pcf857x"; - gpio->irqchip.irq_enable = pcf857x_irq_enable; - gpio->irqchip.irq_disable = pcf857x_irq_disable; - gpio->irqchip.irq_ack = noop; - gpio->irqchip.irq_mask = noop; - gpio->irqchip.irq_unmask = noop; - gpio->irqchip.irq_set_wake = pcf857x_irq_set_wake; - gpio->irqchip.irq_bus_lock = pcf857x_irq_bus_lock; - gpio->irqchip.irq_bus_sync_unlock = pcf857x_irq_bus_sync_unlock; - status = devm_request_threaded_irq(&client->dev, client->irq, NULL, pcf857x_irq, IRQF_ONESHOT | IRQF_TRIGGER_FALLING | IRQF_SHARED, @@ -356,7 +363,7 @@ static int pcf857x_probe(struct i2c_client *client, goto fail; girq = &gpio->chip.irq; - girq->chip = &gpio->irqchip; + gpio_irq_chip_set_chip(girq, &pcf857x_irq_chip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; From 718b972d32da093ac4358c0abb099c65292dc536 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 20 May 2022 12:18:56 +0200 Subject: [PATCH 305/319] gpio: rcar: Make the irqchip immutable Commit 6c846d026d49 ("gpio: Don't fiddle with irqchips marked as immutable") added a warning to indicate if the gpiolib is altering the internals of irqchips. Following this change the following warning is now observed for the gpio-rcar driver: gpio gpiochip0: (e6050000.gpio): not an immutable chip, please consider fixing it! Fix this by making the irqchip in the gpio-rcar driver immutable. Signed-off-by: Geert Uytterhoeven Reviewed-by: Andy Shevchenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-rcar.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c index 356aac4de17c..5b117f3bd322 100644 --- a/drivers/gpio/gpio-rcar.c +++ b/drivers/gpio/gpio-rcar.c @@ -44,7 +44,6 @@ struct gpio_rcar_priv { spinlock_t lock; struct device *dev; struct gpio_chip gpio_chip; - struct irq_chip irq_chip; unsigned int irq_parent; atomic_t wakeup_path; struct gpio_rcar_info info; @@ -96,16 +95,20 @@ static void gpio_rcar_irq_disable(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct gpio_rcar_priv *p = gpiochip_get_data(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); - gpio_rcar_write(p, INTMSK, ~BIT(irqd_to_hwirq(d))); + gpio_rcar_write(p, INTMSK, ~BIT(hwirq)); + gpiochip_disable_irq(gc, hwirq); } static void gpio_rcar_irq_enable(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct gpio_rcar_priv *p = gpiochip_get_data(gc); + irq_hw_number_t hwirq = irqd_to_hwirq(d); - gpio_rcar_write(p, MSKCLR, BIT(irqd_to_hwirq(d))); + gpiochip_enable_irq(gc, hwirq); + gpio_rcar_write(p, MSKCLR, BIT(hwirq)); } static void gpio_rcar_config_interrupt_input_mode(struct gpio_rcar_priv *p, @@ -203,6 +206,17 @@ static int gpio_rcar_irq_set_wake(struct irq_data *d, unsigned int on) return 0; } +static const struct irq_chip gpio_rcar_irq_chip = { + .name = "gpio-rcar", + .irq_mask = gpio_rcar_irq_disable, + .irq_unmask = gpio_rcar_irq_enable, + .irq_set_type = gpio_rcar_irq_set_type, + .irq_set_wake = gpio_rcar_irq_set_wake, + .flags = IRQCHIP_IMMUTABLE | IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_MASK_ON_SUSPEND, + GPIOCHIP_IRQ_RESOURCE_HELPERS, +}; + static irqreturn_t gpio_rcar_irq_handler(int irq, void *dev_id) { struct gpio_rcar_priv *p = dev_id; @@ -481,7 +495,6 @@ static int gpio_rcar_probe(struct platform_device *pdev) { struct gpio_rcar_priv *p; struct gpio_chip *gpio_chip; - struct irq_chip *irq_chip; struct gpio_irq_chip *girq; struct device *dev = &pdev->dev; const char *name = dev_name(dev); @@ -531,16 +544,8 @@ static int gpio_rcar_probe(struct platform_device *pdev) gpio_chip->base = -1; gpio_chip->ngpio = npins; - irq_chip = &p->irq_chip; - irq_chip->name = "gpio-rcar"; - irq_chip->irq_mask = gpio_rcar_irq_disable; - irq_chip->irq_unmask = gpio_rcar_irq_enable; - irq_chip->irq_set_type = gpio_rcar_irq_set_type; - irq_chip->irq_set_wake = gpio_rcar_irq_set_wake; - irq_chip->flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND; - girq = &gpio_chip->irq; - girq->chip = irq_chip; + gpio_irq_chip_set_chip(girq, &gpio_rcar_irq_chip); /* This will let us handle the parent IRQ in the driver */ girq->parent_handler = NULL; girq->num_parents = 0; From 5a7cb9f3978d1fe8cfba798b4c9c054ce226e8fd Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 20 May 2022 12:24:16 +0200 Subject: [PATCH 306/319] gpio: sifive: Make the irqchip immutable Commit 6c846d026d49 ("gpio: Don't fiddle with irqchips marked as immutable") added a warning to indicate if the gpiolib is altering the internals of irqchips. Following this change the following warning is now observed for the sifive driver: gpio gpiochip1: (38001000.gpio-controller): not an immutable chip, please consider fixing it! Fix this by making the irqchip in the sifive driver immutable. Signed-off-by: Geert Uytterhoeven Reviewed-by: Andy Shevchenko Reviewed-by: Damien Le Moal Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-sifive.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/gpio/gpio-sifive.c b/drivers/gpio/gpio-sifive.c index 03b8c4de2e91..238f3210970c 100644 --- a/drivers/gpio/gpio-sifive.c +++ b/drivers/gpio/gpio-sifive.c @@ -75,10 +75,12 @@ static void sifive_gpio_irq_enable(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct sifive_gpio *chip = gpiochip_get_data(gc); - int offset = irqd_to_hwirq(d) % SIFIVE_GPIO_MAX; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + int offset = hwirq % SIFIVE_GPIO_MAX; u32 bit = BIT(offset); unsigned long flags; + gpiochip_enable_irq(gc, hwirq); irq_chip_enable_parent(d); /* Switch to input */ @@ -101,11 +103,13 @@ static void sifive_gpio_irq_disable(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); struct sifive_gpio *chip = gpiochip_get_data(gc); - int offset = irqd_to_hwirq(d) % SIFIVE_GPIO_MAX; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + int offset = hwirq % SIFIVE_GPIO_MAX; assign_bit(offset, &chip->irq_state, 0); sifive_gpio_set_ie(chip, offset); irq_chip_disable_parent(d); + gpiochip_disable_irq(gc, hwirq); } static void sifive_gpio_irq_eoi(struct irq_data *d) @@ -137,7 +141,7 @@ static int sifive_gpio_irq_set_affinity(struct irq_data *data, return -EINVAL; } -static struct irq_chip sifive_gpio_irqchip = { +static const struct irq_chip sifive_gpio_irqchip = { .name = "sifive-gpio", .irq_set_type = sifive_gpio_irq_set_type, .irq_mask = irq_chip_mask_parent, @@ -146,6 +150,8 @@ static struct irq_chip sifive_gpio_irqchip = { .irq_disable = sifive_gpio_irq_disable, .irq_eoi = sifive_gpio_irq_eoi, .irq_set_affinity = sifive_gpio_irq_set_affinity, + .flags = IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static int sifive_gpio_child_to_parent_hwirq(struct gpio_chip *gc, @@ -242,7 +248,7 @@ static int sifive_gpio_probe(struct platform_device *pdev) chip->gc.parent = dev; chip->gc.owner = THIS_MODULE; girq = &chip->gc.irq; - girq->chip = &sifive_gpio_irqchip; + gpio_irq_chip_set_chip(girq, &sifive_gpio_irqchip); girq->fwnode = of_node_to_fwnode(node); girq->parent_domain = parent; girq->child_to_parent_hwirq = sifive_gpio_child_to_parent_hwirq; From e0ac535178ad32c2a4898e2309dcc49ad546e96e Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Tue, 26 Apr 2022 01:25:51 -0700 Subject: [PATCH 307/319] KVM: LAPIC: Trace LAPIC timer expiration on every vmentry In commit ec0671d5684a ("KVM: LAPIC: Delay trace_kvm_wait_lapic_expire tracepoint to after vmexit", 2019-06-04), trace_kvm_wait_lapic_expire was moved after guest_exit_irqoff() because invoking tracepoints within kvm_guest_enter/kvm_guest_exit caused a lockdep splat. These days this is not necessary, because commit 87fa7f3e98a1 ("x86/kvm: Move context tracking where it belongs", 2020-07-09) restricted the RCU extended quiescent state to be closer to vmentry/vmexit. Moving the tracepoint back to __kvm_wait_lapic_expire is more accurate, because it will be reported even if vcpu_enter_guest causes multiple vmentries via the IPI/Timer fast paths, and it allows the removal of advance_expire_delta. Signed-off-by: Wanpeng Li Message-Id: <1650961551-38390-1-git-send-email-wanpengli@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/lapic.c | 4 ++-- arch/x86/kvm/lapic.h | 1 - arch/x86/kvm/x86.c | 8 -------- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 66b0eb0bda94..a4e9eb329e42 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1648,10 +1648,10 @@ static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu) tsc_deadline = apic->lapic_timer.expired_tscdeadline; apic->lapic_timer.expired_tscdeadline = 0; guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc()); - apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline; + trace_kvm_wait_lapic_expire(vcpu->vcpu_id, guest_tsc - tsc_deadline); if (lapic_timer_advance_dynamic) { - adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta); + adjust_lapic_timer_advance(vcpu, guest_tsc - tsc_deadline); /* * If the timer fired early, reread the TSC to account for the * overhead of the above adjustment to avoid waiting longer diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 4e4f8a22754f..65bb2a8cf145 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -38,7 +38,6 @@ struct kvm_timer { u64 tscdeadline; u64 expired_tscdeadline; u32 timer_advance_ns; - s64 advance_expire_delta; atomic_t pending; /* accumulated triggered timers */ bool hv_timer_in_use; }; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 476ea099ebf0..8b62c9b7795e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10310,14 +10310,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) */ guest_timing_exit_irqoff(); - if (lapic_in_kernel(vcpu)) { - s64 delta = vcpu->arch.apic->lapic_timer.advance_expire_delta; - if (delta != S64_MIN) { - trace_kvm_wait_lapic_expire(vcpu->vcpu_id, delta); - vcpu->arch.apic->lapic_timer.advance_expire_delta = S64_MIN; - } - } - local_irq_enable(); preempt_enable(); From 825be3b5abae1e67db45ff7d4b9a7764a2419bd9 Mon Sep 17 00:00:00 2001 From: Yang Weijiang Date: Thu, 12 May 2022 04:40:46 -0400 Subject: [PATCH 308/319] KVM: selftests: x86: Fix test failure on arch lbr capable platforms On Arch LBR capable platforms, LBR_FMT in perf capability msr is 0x3f, so the last format test will fail. Use a true invalid format(0x30) for the test if it's running on these platforms. Opportunistically change the file name to reflect the tests actually carried out. Suggested-by: Paolo Bonzini Signed-off-by: Yang Weijiang Message-Id: <20220512084046.105479-1-weijiang.yang@intel.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/Makefile | 2 +- ...vmx_pmu_msrs_test.c => vmx_pmu_caps_test.c} | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) rename tools/testing/selftests/kvm/x86_64/{vmx_pmu_msrs_test.c => vmx_pmu_caps_test.c} (83%) diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index af582d168621..d0d09ca0d495 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -83,7 +83,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/xapic_state_test TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test TEST_GEN_PROGS_x86_64 += x86_64/debug_regs TEST_GEN_PROGS_x86_64 += x86_64/tsc_msrs_test -TEST_GEN_PROGS_x86_64 += x86_64/vmx_pmu_msrs_test +TEST_GEN_PROGS_x86_64 += x86_64/vmx_pmu_caps_test TEST_GEN_PROGS_x86_64 += x86_64/xen_shinfo_test TEST_GEN_PROGS_x86_64 += x86_64/xen_vmcall_test TEST_GEN_PROGS_x86_64 += x86_64/sev_migrate_tests diff --git a/tools/testing/selftests/kvm/x86_64/vmx_pmu_msrs_test.c b/tools/testing/selftests/kvm/x86_64/vmx_pmu_caps_test.c similarity index 83% rename from tools/testing/selftests/kvm/x86_64/vmx_pmu_msrs_test.c rename to tools/testing/selftests/kvm/x86_64/vmx_pmu_caps_test.c index 2454a1f2ca0c..97b7fd4a9a3d 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_pmu_msrs_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_pmu_caps_test.c @@ -1,15 +1,14 @@ // SPDX-License-Identifier: GPL-2.0 /* - * VMX-pmu related msrs test + * Test for VMX-pmu perf capability msr * * Copyright (C) 2021 Intel Corporation * - * Test to check the effect of various CPUID settings - * on the MSR_IA32_PERF_CAPABILITIES MSR, and check that - * whatever we write with KVM_SET_MSR is _not_ modified - * in the guest and test it can be retrieved with KVM_GET_MSR. - * - * Test to check that invalid LBR formats are rejected. + * Test to check the effect of various CPUID settings on + * MSR_IA32_PERF_CAPABILITIES MSR, and check that what + * we write with KVM_SET_MSR is _not_ modified by the guest + * and check it can be retrieved with KVM_GET_MSR, also test + * the invalid LBR formats are rejected. */ #define _GNU_SOURCE /* for program_invocation_short_name */ @@ -107,8 +106,11 @@ int main(int argc, char *argv[]) ASSERT_EQ(vcpu_get_msr(vm, VCPU_ID, MSR_IA32_PERF_CAPABILITIES), (u64)host_cap.lbr_format); /* testcase 3, check invalid LBR format is rejected */ - ret = _vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, PMU_CAP_LBR_FMT); + /* Note, on Arch LBR capable platforms, LBR_FMT in perf capability msr is 0x3f, + * to avoid the failure, use a true invalid format 0x30 for the test. */ + ret = _vcpu_set_msr(vm, 0, MSR_IA32_PERF_CAPABILITIES, 0x30); TEST_ASSERT(ret == 0, "Bad PERF_CAPABILITIES didn't fail."); + printf("Completed perf capability tests.\n"); kvm_vm_free(vm); } From eb3de2d8f78d893303891d879f941c47f2f2d13d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 May 2022 15:23:21 -0400 Subject: [PATCH 309/319] s390/uv_uapi: depend on CONFIG_S390 Signed-off-by: Paolo Bonzini --- drivers/s390/char/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/s390/char/Kconfig b/drivers/s390/char/Kconfig index ef8f41833c1a..57f41efb8043 100644 --- a/drivers/s390/char/Kconfig +++ b/drivers/s390/char/Kconfig @@ -103,6 +103,7 @@ config SCLP_OFB config S390_UV_UAPI def_tristate m prompt "Ultravisor userspace API" + depends on S390 help Selecting exposes parts of the UV interface to userspace by providing a misc character device at /dev/uv. From d187ba5312307d51818beafaad87d28a7d939adf Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 4 May 2022 00:12:19 +0000 Subject: [PATCH 310/319] x86/fpu: KVM: Set the base guest FPU uABI size to sizeof(struct kvm_xsave) Set the starting uABI size of KVM's guest FPU to 'struct kvm_xsave', i.e. to KVM's historical uABI size. When saving FPU state for usersapce, KVM (well, now the FPU) sets the FP+SSE bits in the XSAVE header even if the host doesn't support XSAVE. Setting the XSAVE header allows the VM to be migrated to a host that does support XSAVE without the new host having to handle FPU state that may or may not be compatible with XSAVE. Setting the uABI size to the host's default size results in out-of-bounds writes (setting the FP+SSE bits) and data corruption (that is thankfully caught by KASAN) when running on hosts without XSAVE, e.g. on Core2 CPUs. WARN if the default size is larger than KVM's historical uABI size; all features that can push the FPU size beyond the historical size must be opt-in. ================================================================== BUG: KASAN: slab-out-of-bounds in fpu_copy_uabi_to_guest_fpstate+0x86/0x130 Read of size 8 at addr ffff888011e33a00 by task qemu-build/681 CPU: 1 PID: 681 Comm: qemu-build Not tainted 5.18.0-rc5-KASAN-amd64 #1 Hardware name: /DG35EC, BIOS ECG3510M.86A.0118.2010.0113.1426 01/13/2010 Call Trace: dump_stack_lvl+0x34/0x45 print_report.cold+0x45/0x575 kasan_report+0x9b/0xd0 fpu_copy_uabi_to_guest_fpstate+0x86/0x130 kvm_arch_vcpu_ioctl+0x72a/0x1c50 [kvm] kvm_vcpu_ioctl+0x47f/0x7b0 [kvm] __x64_sys_ioctl+0x5de/0xc90 do_syscall_64+0x31/0x50 entry_SYSCALL_64_after_hwframe+0x44/0xae Allocated by task 0: (stack is not available) The buggy address belongs to the object at ffff888011e33800 which belongs to the cache kmalloc-512 of size 512 The buggy address is located 0 bytes to the right of 512-byte region [ffff888011e33800, ffff888011e33a00) The buggy address belongs to the physical page: page:0000000089cd4adb refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x11e30 head:0000000089cd4adb order:2 compound_mapcount:0 compound_pincount:0 flags: 0x4000000000010200(slab|head|zone=1) raw: 4000000000010200 dead000000000100 dead000000000122 ffff888001041c80 raw: 0000000000000000 0000000080100010 00000001ffffffff 0000000000000000 page dumped because: kasan: bad access detected Memory state around the buggy address: ffff888011e33900: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ffff888011e33980: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 >ffff888011e33a00: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ^ ffff888011e33a80: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffff888011e33b00: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ================================================================== Disabling lock debugging due to kernel taint Fixes: be50b2065dfa ("kvm: x86: Add support for getting/setting expanded xstate buffer") Fixes: c60427dd50ba ("x86/fpu: Add uabi_size to guest_fpu") Reported-by: Zdenek Kaspar Cc: Maciej S. Szmigiero Cc: Paolo Bonzini Cc: kvm@vger.kernel.org Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Tested-by: Zdenek Kaspar Message-Id: <20220504001219.983513-1-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kernel/fpu/core.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index e28ab0ecc537..0fdc807ae13f 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -14,6 +14,8 @@ #include #include +#include + #include #include #include @@ -232,7 +234,20 @@ bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu) gfpu->fpstate = fpstate; gfpu->xfeatures = fpu_user_cfg.default_features; gfpu->perm = fpu_user_cfg.default_features; - gfpu->uabi_size = fpu_user_cfg.default_size; + + /* + * KVM sets the FP+SSE bits in the XSAVE header when copying FPU state + * to userspace, even when XSAVE is unsupported, so that restoring FPU + * state on a different CPU that does support XSAVE can cleanly load + * the incoming state using its natural XSAVE. In other words, KVM's + * uABI size may be larger than this host's default size. Conversely, + * the default size should never be larger than KVM's base uABI size; + * all features that can expand the uABI size must be opt-in. + */ + gfpu->uabi_size = sizeof(struct kvm_xsave); + if (WARN_ON_ONCE(fpu_user_cfg.default_size > gfpu->uabi_size)) + gfpu->uabi_size = fpu_user_cfg.default_size; + fpu_init_guest_permissions(gfpu); return true; From d22d2474e3953996f03528b84b7f52cc26a39403 Mon Sep 17 00:00:00 2001 From: Ashish Kalra Date: Mon, 16 May 2022 15:43:10 +0000 Subject: [PATCH 311/319] KVM: SVM: Use kzalloc for sev ioctl interfaces to prevent kernel data leak For some sev ioctl interfaces, the length parameter that is passed maybe less than or equal to SEV_FW_BLOB_MAX_SIZE, but larger than the data that PSP firmware returns. In this case, kmalloc will allocate memory that is the size of the input rather than the size of the data. Since PSP firmware doesn't fully overwrite the allocated buffer, these sev ioctl interface may return uninitialized kernel slab memory. Reported-by: Andy Nguyen Suggested-by: David Rientjes Suggested-by: Peter Gonda Cc: kvm@vger.kernel.org Cc: stable@vger.kernel.org Cc: linux-kernel@vger.kernel.org Fixes: eaf78265a4ab3 ("KVM: SVM: Move SEV code to separate file") Fixes: 2c07ded06427d ("KVM: SVM: add support for SEV attestation command") Fixes: 4cfdd47d6d95a ("KVM: SVM: Add KVM_SEV SEND_START command") Fixes: d3d1af85e2c75 ("KVM: SVM: Add KVM_SEND_UPDATE_DATA command") Fixes: eba04b20e4861 ("KVM: x86: Account a variety of miscellaneous allocations") Signed-off-by: Ashish Kalra Reviewed-by: Peter Gonda Message-Id: <20220516154310.3685678-1-Ashish.Kalra@amd.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 655770522471..39f3dd005499 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -688,7 +688,7 @@ static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp) if (params.len > SEV_FW_BLOB_MAX_SIZE) return -EINVAL; - blob = kmalloc(params.len, GFP_KERNEL_ACCOUNT); + blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT); if (!blob) return -ENOMEM; @@ -808,7 +808,7 @@ static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr, if (!IS_ALIGNED(dst_paddr, 16) || !IS_ALIGNED(paddr, 16) || !IS_ALIGNED(size, 16)) { - tpage = (void *)alloc_page(GFP_KERNEL); + tpage = (void *)alloc_page(GFP_KERNEL | __GFP_ZERO); if (!tpage) return -ENOMEM; @@ -1094,7 +1094,7 @@ static int sev_get_attestation_report(struct kvm *kvm, struct kvm_sev_cmd *argp) if (params.len > SEV_FW_BLOB_MAX_SIZE) return -EINVAL; - blob = kmalloc(params.len, GFP_KERNEL_ACCOUNT); + blob = kzalloc(params.len, GFP_KERNEL_ACCOUNT); if (!blob) return -ENOMEM; @@ -1176,7 +1176,7 @@ static int sev_send_start(struct kvm *kvm, struct kvm_sev_cmd *argp) return -EINVAL; /* allocate the memory to hold the session data blob */ - session_data = kmalloc(params.session_len, GFP_KERNEL_ACCOUNT); + session_data = kzalloc(params.session_len, GFP_KERNEL_ACCOUNT); if (!session_data) return -ENOMEM; @@ -1300,11 +1300,11 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) /* allocate memory for header and transport buffer */ ret = -ENOMEM; - hdr = kmalloc(params.hdr_len, GFP_KERNEL_ACCOUNT); + hdr = kzalloc(params.hdr_len, GFP_KERNEL_ACCOUNT); if (!hdr) goto e_unpin; - trans_data = kmalloc(params.trans_len, GFP_KERNEL_ACCOUNT); + trans_data = kzalloc(params.trans_len, GFP_KERNEL_ACCOUNT); if (!trans_data) goto e_free_hdr; From fee060cd52d69c114b62d1a2948ea9648b5131f9 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 11 Mar 2022 03:27:41 +0000 Subject: [PATCH 312/319] KVM: x86: avoid calling x86 emulator without a decoded instruction Whenever x86_decode_emulated_instruction() detects a breakpoint, it returns the value that kvm_vcpu_check_breakpoint() writes into its pass-by-reference second argument. Unfortunately this is completely bogus because the expected outcome of x86_decode_emulated_instruction is an EMULATION_* value. Then, if kvm_vcpu_check_breakpoint() does "*r = 0" (corresponding to a KVM_EXIT_DEBUG userspace exit), it is misunderstood as EMULATION_OK and x86_emulate_instruction() is called without having decoded the instruction. This causes various havoc from running with a stale emulation context. The fix is to move the call to kvm_vcpu_check_breakpoint() where it was before commit 4aa2691dcbd3 ("KVM: x86: Factor out x86 instruction emulation with decoding") introduced x86_decode_emulated_instruction(). The other caller of the function does not need breakpoint checks, because it is invoked as part of a vmexit and the processor has already checked those before executing the instruction that #GP'd. This fixes CVE-2022-1852. Reported-by: Qiuhao Li Reported-by: Gaoning Pan Reported-by: Yongkang Jia Fixes: 4aa2691dcbd3 ("KVM: x86: Factor out x86 instruction emulation with decoding") Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Message-Id: <20220311032801.3467418-2-seanjc@google.com> [Rewrote commit message according to Qiuhao's report, since a patch already existed to fix the bug. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index bc507d6414f4..b81ef4f497f4 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8296,7 +8296,7 @@ int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_skip_emulated_instruction); -static bool kvm_vcpu_check_breakpoint(struct kvm_vcpu *vcpu, int *r) +static bool kvm_vcpu_check_code_breakpoint(struct kvm_vcpu *vcpu, int *r) { if (unlikely(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) && (vcpu->arch.guest_debug_dr7 & DR7_BP_EN_MASK)) { @@ -8365,25 +8365,23 @@ static bool is_vmware_backdoor_opcode(struct x86_emulate_ctxt *ctxt) } /* - * Decode to be emulated instruction. Return EMULATION_OK if success. + * Decode an instruction for emulation. The caller is responsible for handling + * code breakpoints. Note, manually detecting code breakpoints is unnecessary + * (and wrong) when emulating on an intercepted fault-like exception[*], as + * code breakpoints have higher priority and thus have already been done by + * hardware. + * + * [*] Except #MC, which is higher priority, but KVM should never emulate in + * response to a machine check. */ int x86_decode_emulated_instruction(struct kvm_vcpu *vcpu, int emulation_type, void *insn, int insn_len) { - int r = EMULATION_OK; struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; + int r; init_emulate_ctxt(vcpu); - /* - * We will reenter on the same instruction since we do not set - * complete_userspace_io. This does not handle watchpoints yet, - * those would be handled in the emulate_ops. - */ - if (!(emulation_type & EMULTYPE_SKIP) && - kvm_vcpu_check_breakpoint(vcpu, &r)) - return r; - r = x86_decode_insn(ctxt, insn, insn_len, emulation_type); trace_kvm_emulate_insn_start(vcpu); @@ -8416,6 +8414,15 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, if (!(emulation_type & EMULTYPE_NO_DECODE)) { kvm_clear_exception_queue(vcpu); + /* + * Return immediately if RIP hits a code breakpoint, such #DBs + * are fault-like and are higher priority than any faults on + * the code fetch itself. + */ + if (!(emulation_type & EMULTYPE_SKIP) && + kvm_vcpu_check_code_breakpoint(vcpu, &r)) + return r; + r = x86_decode_emulated_instruction(vcpu, emulation_type, insn, insn_len); if (r != EMULATION_OK) { From 0547758a6de3cc71a0cfdd031a3621a30db6a68b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 19 May 2022 07:57:11 -0700 Subject: [PATCH 313/319] x86/kvm: Alloc dummy async #PF token outside of raw spinlock Drop the raw spinlock in kvm_async_pf_task_wake() before allocating the the dummy async #PF token, the allocator is preemptible on PREEMPT_RT kernels and must not be called from truly atomic contexts. Opportunistically document why it's ok to loop on allocation failure, i.e. why the function won't get stuck in an infinite loop. Reported-by: Yajun Deng Cc: stable@vger.kernel.org Signed-off-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kernel/kvm.c | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index b48ce07756ca..35b3c5836703 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -191,7 +191,7 @@ void kvm_async_pf_task_wake(u32 token) { u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; - struct kvm_task_sleep_node *n; + struct kvm_task_sleep_node *n, *dummy = NULL; if (token == ~0) { apf_task_wake_all(); @@ -203,28 +203,41 @@ void kvm_async_pf_task_wake(u32 token) n = _find_apf_task(b, token); if (!n) { /* - * async PF was not yet handled. - * Add dummy entry for the token. + * Async #PF not yet handled, add a dummy entry for the token. + * Allocating the token must be down outside of the raw lock + * as the allocator is preemptible on PREEMPT_RT kernels. */ - n = kzalloc(sizeof(*n), GFP_ATOMIC); - if (!n) { - /* - * Allocation failed! Busy wait while other cpu - * handles async PF. - */ + if (!dummy) { raw_spin_unlock(&b->lock); - cpu_relax(); + dummy = kzalloc(sizeof(*dummy), GFP_KERNEL); + + /* + * Continue looping on allocation failure, eventually + * the async #PF will be handled and allocating a new + * node will be unnecessary. + */ + if (!dummy) + cpu_relax(); + + /* + * Recheck for async #PF completion before enqueueing + * the dummy token to avoid duplicate list entries. + */ goto again; } - n->token = token; - n->cpu = smp_processor_id(); - init_swait_queue_head(&n->wq); - hlist_add_head(&n->link, &b->list); + dummy->token = token; + dummy->cpu = smp_processor_id(); + init_swait_queue_head(&dummy->wq); + hlist_add_head(&dummy->link, &b->list); + dummy = NULL; } else { apf_task_wake_one(n); } raw_spin_unlock(&b->lock); - return; + + /* A dummy token might be allocated and ultimately not used. */ + if (dummy) + kfree(dummy); } EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); From 619f51da097952194a5d4d6a6c5f9ef3b9d1b25a Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Fri, 20 May 2022 10:15:18 -0400 Subject: [PATCH 314/319] KVM: LAPIC: Drop pending LAPIC timer injection when canceling the timer The timer is disarmed when switching between TSC deadline and other modes; however, the pending timer is still in-flight, so let's accurately remove any traces of the previous mode. Fixes: 4427593258 ("KVM: x86: thoroughly disarm LAPIC timer around TSC deadline switch") Signed-off-by: Wanpeng Li Signed-off-by: Paolo Bonzini --- arch/x86/kvm/lapic.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index a4e9eb329e42..f1bdac3f5aa8 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1548,6 +1548,7 @@ static void cancel_apic_timer(struct kvm_lapic *apic) if (apic->lapic_timer.hv_timer_in_use) cancel_hv_timer(apic); preempt_enable(); + atomic_set(&apic->lapic_timer.pending, 0); } static void apic_update_lvtt(struct kvm_lapic *apic) From baec4f5a018fe2d708fc1022330dba04b38b5fe3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 24 May 2022 09:43:31 -0400 Subject: [PATCH 315/319] x86, kvm: use correct GFP flags for preemption disabled Commit ddd7ed842627 ("x86/kvm: Alloc dummy async #PF token outside of raw spinlock") leads to the following Smatch static checker warning: arch/x86/kernel/kvm.c:212 kvm_async_pf_task_wake() warn: sleeping in atomic context arch/x86/kernel/kvm.c 202 raw_spin_lock(&b->lock); 203 n = _find_apf_task(b, token); 204 if (!n) { 205 /* 206 * Async #PF not yet handled, add a dummy entry for the token. 207 * Allocating the token must be down outside of the raw lock 208 * as the allocator is preemptible on PREEMPT_RT kernels. 209 */ 210 if (!dummy) { 211 raw_spin_unlock(&b->lock); --> 212 dummy = kzalloc(sizeof(*dummy), GFP_KERNEL); ^^^^^^^^^^ Smatch thinks the caller has preempt disabled. The `smdb.py preempt kvm_async_pf_task_wake` output call tree is: sysvec_kvm_asyncpf_interrupt() <- disables preempt -> __sysvec_kvm_asyncpf_interrupt() -> kvm_async_pf_task_wake() The caller is this: arch/x86/kernel/kvm.c 290 DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_asyncpf_interrupt) 291 { 292 struct pt_regs *old_regs = set_irq_regs(regs); 293 u32 token; 294 295 ack_APIC_irq(); 296 297 inc_irq_stat(irq_hv_callback_count); 298 299 if (__this_cpu_read(apf_reason.enabled)) { 300 token = __this_cpu_read(apf_reason.token); 301 kvm_async_pf_task_wake(token); 302 __this_cpu_write(apf_reason.token, 0); 303 wrmsrl(MSR_KVM_ASYNC_PF_ACK, 1); 304 } 305 306 set_irq_regs(old_regs); 307 } The DEFINE_IDTENTRY_SYSVEC() is a wrapper that calls this function from the call_on_irqstack_cond(). It's inside the call_on_irqstack_cond() where preempt is disabled (unless it's already disabled). The irq_enter/exit_rcu() functions disable/enable preempt. Reported-by: Dan Carpenter Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kernel/kvm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 35b3c5836703..1a3658f7e6d9 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -209,7 +209,7 @@ void kvm_async_pf_task_wake(u32 token) */ if (!dummy) { raw_spin_unlock(&b->lock); - dummy = kzalloc(sizeof(*dummy), GFP_KERNEL); + dummy = kzalloc(sizeof(*dummy), GFP_ATOMIC); /* * Continue looping on allocation failure, eventually From 186af6bb40c4fd2048bb81253c1109d8b3011e42 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 20 May 2022 07:22:46 -0400 Subject: [PATCH 316/319] Documentation: kvm: reorder ARM-specific section about KVM_SYSTEM_EVENT_SUSPEND Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index d10d40867540..28b547a9d96a 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -6176,32 +6176,6 @@ Valid values for 'type' are: - KVM_SYSTEM_EVENT_SUSPEND -- the guest has requested a suspension of the VM. -For arm/arm64: --------------- - - KVM_SYSTEM_EVENT_SUSPEND exits are enabled with the - KVM_CAP_ARM_SYSTEM_SUSPEND VM capability. If a guest invokes the PSCI - SYSTEM_SUSPEND function, KVM will exit to userspace with this event - type. - - It is the sole responsibility of userspace to implement the PSCI - SYSTEM_SUSPEND call according to ARM DEN0022D.b 5.19 "SYSTEM_SUSPEND". - KVM does not change the vCPU's state before exiting to userspace, so - the call parameters are left in-place in the vCPU registers. - - Userspace is _required_ to take action for such an exit. It must - either: - - - Honor the guest request to suspend the VM. Userspace can request - in-kernel emulation of suspension by setting the calling vCPU's - state to KVM_MP_STATE_SUSPENDED. Userspace must configure the vCPU's - state according to the parameters passed to the PSCI function when - the calling vCPU is resumed. See ARM DEN0022D.b 5.19.1 "Intended use" - for details on the function parameters. - - - Deny the guest request to suspend the VM. See ARM DEN0022D.b 5.19.2 - "Caller responsibilities" for possible return values. - If KVM_CAP_SYSTEM_EVENT_DATA is present, the 'data' field can contain architecture specific information for the system-level event. Only the first `ndata` items (possibly zero) of the data array are valid. @@ -6217,6 +6191,32 @@ Previous versions of Linux defined a `flags` member in this struct. The field is now aliased to `data[0]`. Userspace can assume that it is only written if ndata is greater than 0. +For arm/arm64: +-------------- + +KVM_SYSTEM_EVENT_SUSPEND exits are enabled with the +KVM_CAP_ARM_SYSTEM_SUSPEND VM capability. If a guest invokes the PSCI +SYSTEM_SUSPEND function, KVM will exit to userspace with this event +type. + +It is the sole responsibility of userspace to implement the PSCI +SYSTEM_SUSPEND call according to ARM DEN0022D.b 5.19 "SYSTEM_SUSPEND". +KVM does not change the vCPU's state before exiting to userspace, so +the call parameters are left in-place in the vCPU registers. + +Userspace is _required_ to take action for such an exit. It must +either: + + - Honor the guest request to suspend the VM. Userspace can request + in-kernel emulation of suspension by setting the calling vCPU's + state to KVM_MP_STATE_SUSPENDED. Userspace must configure the vCPU's + state according to the parameters passed to the PSCI function when + the calling vCPU is resumed. See ARM DEN0022D.b 5.19.1 "Intended use" + for details on the function parameters. + + - Deny the guest request to suspend the VM. See ARM DEN0022D.b 5.19.2 + "Caller responsibilities" for possible return values. + :: /* KVM_EXIT_IOAPIC_EOI */ From 366d4a12cdcf3d83c8162ff6e0046c123567c754 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Thu, 19 May 2022 01:01:17 +0800 Subject: [PATCH 317/319] KVM: selftests: x86: Sync the new name of the test case to .gitignore Fixing side effect of the so-called opportunistic change in the commit. Fixes: dc8a9febbab0 ("KVM: selftests: x86: Fix test failure on arch lbr capable platforms") Signed-off-by: Like Xu Message-Id: <20220518170118.66263-2-likexu@tencent.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 54fc269c1788..4509a3a7eeae 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -55,7 +55,7 @@ /x86_64/xen_shinfo_test /x86_64/xen_vmcall_test /x86_64/xss_msr_test -/x86_64/vmx_pmu_msrs_test +/x86_64/vmx_pmu_caps_test /access_tracking_perf_test /demand_paging_test /dirty_log_test From ffd1925a596ce68bed7d81c61cb64bc35f788a9d Mon Sep 17 00:00:00 2001 From: Yanfei Xu Date: Mon, 23 May 2022 22:08:21 +0800 Subject: [PATCH 318/319] KVM: x86: Fix the intel_pt PMI handling wrongly considered from guest When kernel handles the vm-exit caused by external interrupts and NMI, it always sets kvm_intr_type to tell if it's dealing an IRQ or NMI. For the PMI scenario, it could be IRQ or NMI. However, intel_pt PMIs are only generated for HARDWARE perf events, and HARDWARE events are always configured to generate NMIs. Use kvm_handling_nmi_from_guest() to precisely identify if the intel_pt PMI came from the guest; this avoids false positives if an intel_pt PMI/NMI arrives while the host is handling an unrelated IRQ VM-Exit. Fixes: db215756ae59 ("KVM: x86: More precisely identify NMI from guest when handling PMI") Signed-off-by: Yanfei Xu Message-Id: <20220523140821.1345605-1-yanfei.xu@intel.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3e271495181e..f5aeade623d6 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7853,7 +7853,7 @@ static unsigned int vmx_handle_intel_pt_intr(void) struct kvm_vcpu *vcpu = kvm_get_running_vcpu(); /* '0' on failure so that the !PT case can use a RET0 static call. */ - if (!kvm_arch_pmi_in_guest(vcpu)) + if (!vcpu || !kvm_handling_nmi_from_guest(vcpu)) return 0; kvm_make_request(KVM_REQ_PMI, vcpu); From b1c8312c6bd70e2c41f96183936fdb6f4f07cc0e Mon Sep 17 00:00:00 2001 From: Sean Young Date: Thu, 26 May 2022 08:59:40 +0100 Subject: [PATCH 319/319] media: lirc: add missing exceptions for lirc uapi header file Commit e5499dd7253c ("media: lirc: revert removal of unused feature flags") reintroduced unused feature flags in the lirc uapi header, but failed to reintroduce the necessary exceptions for the docs. Fixes: e5499dd7253c ("media: lirc: revert removal of unused feature flags") Signed-off-by: Sean Young Signed-off-by: Linus Torvalds --- Documentation/userspace-api/media/lirc.h.rst.exceptions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Documentation/userspace-api/media/lirc.h.rst.exceptions b/Documentation/userspace-api/media/lirc.h.rst.exceptions index 913d17b49831..1aeb7d7afe13 100644 --- a/Documentation/userspace-api/media/lirc.h.rst.exceptions +++ b/Documentation/userspace-api/media/lirc.h.rst.exceptions @@ -30,6 +30,8 @@ ignore define LIRC_CAN_REC ignore define LIRC_CAN_SEND_MASK ignore define LIRC_CAN_REC_MASK +ignore define LIRC_CAN_SET_REC_FILTER +ignore define LIRC_CAN_NOTIFY_DECODE # Obsolete ioctls