From ac48017020a5f97d7ef1a5cd90278587474dd593 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:08 +0530 Subject: [PATCH 01/14] KVM: x86: Open code setting/clearing of bits in the ISR Remove __apic_test_and_set_vector() and __apic_test_and_clear_vector(), because the _only_ register that's safe to modify with a non-atomic operation is ISR, because KVM isn't running the vCPU, i.e. hardware can't service an IRQ or process an EOI for the relevant (virtual) APIC. No functional change intended. Suggested-by: Sean Christopherson Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-2-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 73418dc0ebb2..013e8681247f 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -125,16 +125,6 @@ bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) apic_test_vector(vector, apic->regs + APIC_IRR); } -static inline int __apic_test_and_set_vector(int vec, void *bitmap) -{ - return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); -} - -static inline int __apic_test_and_clear_vector(int vec, void *bitmap) -{ - return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); -} - __read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu); @@ -744,9 +734,14 @@ void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec) } EXPORT_SYMBOL_GPL(kvm_apic_clear_irr); +static void *apic_vector_to_isr(int vec, struct kvm_lapic *apic) +{ + return apic->regs + APIC_ISR + REG_POS(vec); +} + static inline void apic_set_isr(int vec, struct kvm_lapic *apic) { - if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR)) + if (__test_and_set_bit(VEC_POS(vec), apic_vector_to_isr(vec, apic))) return; /* @@ -789,7 +784,7 @@ static inline int apic_find_highest_isr(struct kvm_lapic *apic) static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) { - if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR)) + if (!__test_and_clear_bit(VEC_POS(vec), apic_vector_to_isr(vec, apic))) return; /* From 3fb7b83e2a720dc96aa275f42380cf488e6f9737 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:09 +0530 Subject: [PATCH 02/14] KVM: x86: Remove redundant parentheses around 'bitmap' When doing pointer arithmetic in apic_test_vector() and kvm_lapic_{set|clear}_vector(), remove the unnecessary parentheses surrounding the 'bitmap' parameter. No functional change intended. Reviewed-by: Borislav Petkov (AMD) Signed-off-by: Neeraj Upadhyay Link: https://lore.kernel.org/r/20250709033242.267892-3-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 2 +- arch/x86/kvm/lapic.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 013e8681247f..533daf6dd1b1 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -114,7 +114,7 @@ static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, static inline int apic_test_vector(int vec, void *bitmap) { - return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); + return test_bit(VEC_POS(vec), bitmap + REG_POS(vec)); } bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 4ce30db65828..1638a3da383a 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -150,12 +150,12 @@ u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic); static inline void kvm_lapic_clear_vector(int vec, void *bitmap) { - clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); + clear_bit(VEC_POS(vec), bitmap + REG_POS(vec)); } static inline void kvm_lapic_set_vector(int vec, void *bitmap) { - set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); + set_bit(VEC_POS(vec), bitmap + REG_POS(vec)); } static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) From dc98e3bd494bef9c8933ee9ace254aac48efe506 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 9 Jul 2025 09:02:10 +0530 Subject: [PATCH 03/14] x86/apic: KVM: Deduplicate APIC vector => register+bit math Consolidate KVM's {REG,VEC}_POS() macros and lapic_vector_set_in_irr()'s open coded equivalent logic in anticipation of the kernel gaining more usage of vector => reg+bit lookups. Use lapic_vector_set_in_irr()'s math as using divides for both the bit number and register offset makes it easier to connect the dots, and for at least one user, fixup_irqs(), "/ 32 * 0x10" generates ever so slightly better code with gcc-14 (shaves a whole 3 bytes from the code stream): ((v) >> 5) << 4: c1 ef 05 shr $0x5,%edi c1 e7 04 shl $0x4,%edi 81 c7 00 02 00 00 add $0x200,%edi (v) / 32 * 0x10: c1 ef 05 shr $0x5,%edi 83 c7 20 add $0x20,%edi c1 e7 04 shl $0x4,%edi Keep KVM's tersely named macros as "wrappers" to avoid unnecessary churn in KVM, and because the shorter names yield more readable code overall in KVM. The new macros type cast the vector parameter to "unsigned int". This is required from better code generation for cases where an "int" is passed to these macros in KVM code. int v; ((v) >> 5) << 4: c1 f8 05 sar $0x5,%eax c1 e0 04 shl $0x4,%eax ((v) / 32 * 0x10): 85 ff test %edi,%edi 8d 47 1f lea 0x1f(%rdi),%eax 0f 49 c7 cmovns %edi,%eax c1 f8 05 sar $0x5,%eax c1 e0 04 shl $0x4,%eax ((unsigned int)(v) / 32 * 0x10): c1 f8 05 sar $0x5,%eax c1 e0 04 shl $0x4,%eax (v) & (32 - 1): 89 f8 mov %edi,%eax 83 e0 1f and $0x1f,%eax (v) % 32 89 fa mov %edi,%edx c1 fa 1f sar $0x1f,%edx c1 ea 1b shr $0x1b,%edx 8d 04 17 lea (%rdi,%rdx,1),%eax 83 e0 1f and $0x1f,%eax 29 d0 sub %edx,%eax (unsigned int)(v) % 32: 89 f8 mov %edi,%eax 83 e0 1f and $0x1f,%eax Overall kvm.ko text size is impacted if "unsigned int" is not used. Bin Orig New (w/o unsigned int) New (w/ unsigned int) lapic.o 28580 28772 28580 kvm.o 670810 671002 670810 kvm.ko 708079 708271 708079 No functional change intended. [Neeraj: Type cast vec macro param to "unsigned int", provide data in commit log on "unsigned int" requirement] Signed-off-by: Neeraj Upadhyay Link: https://lore.kernel.org/r/20250709033242.267892-4-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/apic.h | 7 +++++-- arch/x86/kvm/lapic.h | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 23d86c9750b9..c84d4e86fe4e 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -488,11 +488,14 @@ static inline void apic_setup_apic_calls(void) { } extern void apic_ack_irq(struct irq_data *data); +#define APIC_VECTOR_TO_BIT_NUMBER(v) ((unsigned int)(v) % 32) +#define APIC_VECTOR_TO_REG_OFFSET(v) ((unsigned int)(v) / 32 * 0x10) + static inline bool lapic_vector_set_in_irr(unsigned int vector) { - u32 irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); + u32 irr = apic_read(APIC_IRR + APIC_VECTOR_TO_REG_OFFSET(vector)); - return !!(irr & (1U << (vector % 32))); + return !!(irr & (1U << APIC_VECTOR_TO_BIT_NUMBER(vector))); } static inline bool is_vector_pending(unsigned int vector) diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 1638a3da383a..56369d331bfc 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -145,8 +145,8 @@ void kvm_lapic_exit(void); u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic); -#define VEC_POS(v) ((v) & (32 - 1)) -#define REG_POS(v) (((v) >> 5) << 4) +#define VEC_POS(v) APIC_VECTOR_TO_BIT_NUMBER(v) +#define REG_POS(v) APIC_VECTOR_TO_REG_OFFSET(v) static inline void kvm_lapic_clear_vector(int vec, void *bitmap) { From 9cbb5fd156d777cc09f57af20505e5e17ad6263b Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:11 +0530 Subject: [PATCH 04/14] KVM: x86: Rename VEC_POS/REG_POS macro usages In preparation for moving most of the KVM's lapic helpers which use VEC_POS/REG_POS macros to common APIC header for use in Secure AVIC APIC driver, rename all VEC_POS/REG_POS macro usages to APIC_VECTOR_TO_BIT_NUMBER/APIC_VECTOR_TO_REG_OFFSET and remove VEC_POS/REG_POS. While at it, clean up line wrap in find_highest_vector(). No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-5-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 17 +++++++++-------- arch/x86/kvm/lapic.h | 7 ++----- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 533daf6dd1b1..4143f206f04d 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -114,7 +114,7 @@ static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, static inline int apic_test_vector(int vec, void *bitmap) { - return test_bit(VEC_POS(vec), bitmap + REG_POS(vec)); + return test_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); } bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) @@ -621,9 +621,8 @@ static int find_highest_vector(void *bitmap) int vec; u32 *reg; - for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; - vec >= 0; vec -= APIC_VECTORS_PER_REG) { - reg = bitmap + REG_POS(vec); + for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; vec >= 0; vec -= APIC_VECTORS_PER_REG) { + reg = bitmap + APIC_VECTOR_TO_REG_OFFSET(vec); if (*reg) return __fls(*reg) + vec; } @@ -638,7 +637,7 @@ static u8 count_vectors(void *bitmap) u8 count = 0; for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) { - reg = bitmap + REG_POS(vec); + reg = bitmap + APIC_VECTOR_TO_REG_OFFSET(vec); count += hweight32(*reg); } @@ -736,12 +735,13 @@ EXPORT_SYMBOL_GPL(kvm_apic_clear_irr); static void *apic_vector_to_isr(int vec, struct kvm_lapic *apic) { - return apic->regs + APIC_ISR + REG_POS(vec); + return apic->regs + APIC_ISR + APIC_VECTOR_TO_REG_OFFSET(vec); } static inline void apic_set_isr(int vec, struct kvm_lapic *apic) { - if (__test_and_set_bit(VEC_POS(vec), apic_vector_to_isr(vec, apic))) + if (__test_and_set_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), + apic_vector_to_isr(vec, apic))) return; /* @@ -784,7 +784,8 @@ static inline int apic_find_highest_isr(struct kvm_lapic *apic) static inline void apic_clear_isr(int vec, struct kvm_lapic *apic) { - if (!__test_and_clear_bit(VEC_POS(vec), apic_vector_to_isr(vec, apic))) + if (!__test_and_clear_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), + apic_vector_to_isr(vec, apic))) return; /* diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 56369d331bfc..eb9bda52948c 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -145,17 +145,14 @@ void kvm_lapic_exit(void); u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic); -#define VEC_POS(v) APIC_VECTOR_TO_BIT_NUMBER(v) -#define REG_POS(v) APIC_VECTOR_TO_REG_OFFSET(v) - static inline void kvm_lapic_clear_vector(int vec, void *bitmap) { - clear_bit(VEC_POS(vec), bitmap + REG_POS(vec)); + clear_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); } static inline void kvm_lapic_set_vector(int vec, void *bitmap) { - set_bit(VEC_POS(vec), bitmap + REG_POS(vec)); + set_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); } static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) From e2fa7905b293750ee75573eeee3e54575ded8217 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:12 +0530 Subject: [PATCH 05/14] KVM: x86: Change lapic regs base address to void pointer Change APIC base address from "char *" to "void *" in KVM lapic's set/get helper functions. Pointer arithmetic for "void *" and "char *" operate identically. With "void *" there is less of a chance of doing the wrong thing, e.g. neglecting to cast and reading a byte instead of the desired APIC register size. No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-6-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 6 +++--- arch/x86/kvm/lapic.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 4143f206f04d..3504077539df 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -79,7 +79,7 @@ module_param(lapic_timer_advance, bool, 0444); static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data); static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data); -static inline void __kvm_lapic_set_reg(char *regs, int reg_off, u32 val) +static inline void __kvm_lapic_set_reg(void *regs, int reg_off, u32 val) { *((u32 *) (regs + reg_off)) = val; } @@ -89,7 +89,7 @@ static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 va __kvm_lapic_set_reg(apic->regs, reg_off, val); } -static __always_inline u64 __kvm_lapic_get_reg64(char *regs, int reg) +static __always_inline u64 __kvm_lapic_get_reg64(void *regs, int reg) { BUILD_BUG_ON(reg != APIC_ICR); return *((u64 *) (regs + reg)); @@ -100,7 +100,7 @@ static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg) return __kvm_lapic_get_reg64(apic->regs, reg); } -static __always_inline void __kvm_lapic_set_reg64(char *regs, int reg, u64 val) +static __always_inline void __kvm_lapic_set_reg64(void *regs, int reg, u64 val) { BUILD_BUG_ON(reg != APIC_ICR); *((u64 *) (regs + reg)) = val; diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index eb9bda52948c..7ce89bf0b974 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -165,7 +165,7 @@ static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) apic->irr_pending = true; } -static inline u32 __kvm_lapic_get_reg(char *regs, int reg_off) +static inline u32 __kvm_lapic_get_reg(void *regs, int reg_off) { return *((u32 *) (regs + reg_off)); } From bdaccfe4e5179fe503febcd459ffe202d8097f6e Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:13 +0530 Subject: [PATCH 06/14] KVM: x86: Rename find_highest_vector() In preparation for moving kvm-internal find_highest_vector() to apic.h for use in Secure AVIC APIC driver, rename find_highest_vector() to apic_find_highest_vector() as part of the APIC API. No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-7-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 3504077539df..38bb074fddec 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -616,7 +616,7 @@ static const unsigned int apic_lvt_mask[KVM_APIC_MAX_NR_LVT_ENTRIES] = { [LVT_CMCI] = LVT_MASK | APIC_MODE_MASK }; -static int find_highest_vector(void *bitmap) +static int apic_find_highest_vector(void *bitmap) { int vec; u32 *reg; @@ -695,7 +695,7 @@ EXPORT_SYMBOL_GPL(kvm_apic_update_irr); static inline int apic_search_irr(struct kvm_lapic *apic) { - return find_highest_vector(apic->regs + APIC_IRR); + return apic_find_highest_vector(apic->regs + APIC_IRR); } static inline int apic_find_highest_irr(struct kvm_lapic *apic) @@ -776,7 +776,7 @@ static inline int apic_find_highest_isr(struct kvm_lapic *apic) if (likely(apic->highest_isr_cache != -1)) return apic->highest_isr_cache; - result = find_highest_vector(apic->regs + APIC_ISR); + result = apic_find_highest_vector(apic->regs + APIC_ISR); ASSERT(result == -1 || result >= 16); return result; From b9bd231913cf25bfaa6b5f10c2a629934697a189 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:14 +0530 Subject: [PATCH 07/14] KVM: x86: Rename lapic get/set_reg() helpers In preparation for moving kvm-internal __kvm_lapic_set_reg(), __kvm_lapic_get_reg() to apic.h for use in Secure AVIC APIC driver, rename them as part of the APIC API. No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-8-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 13 ++++++------- arch/x86/kvm/lapic.h | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 38bb074fddec..436e0b0c829e 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -79,14 +79,14 @@ module_param(lapic_timer_advance, bool, 0444); static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data); static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data); -static inline void __kvm_lapic_set_reg(void *regs, int reg_off, u32 val) +static inline void apic_set_reg(void *regs, int reg_off, u32 val) { *((u32 *) (regs + reg_off)) = val; } static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) { - __kvm_lapic_set_reg(apic->regs, reg_off, val); + apic_set_reg(apic->regs, reg_off, val); } static __always_inline u64 __kvm_lapic_get_reg64(void *regs, int reg) @@ -3080,12 +3080,12 @@ static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, if (!kvm_x86_ops.x2apic_icr_is_split) { if (set) { - icr = __kvm_lapic_get_reg(s->regs, APIC_ICR) | - (u64)__kvm_lapic_get_reg(s->regs, APIC_ICR2) << 32; + icr = apic_get_reg(s->regs, APIC_ICR) | + (u64)apic_get_reg(s->regs, APIC_ICR2) << 32; __kvm_lapic_set_reg64(s->regs, APIC_ICR, icr); } else { icr = __kvm_lapic_get_reg64(s->regs, APIC_ICR); - __kvm_lapic_set_reg(s->regs, APIC_ICR2, icr >> 32); + apic_set_reg(s->regs, APIC_ICR2, icr >> 32); } } } @@ -3101,8 +3101,7 @@ int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) * Get calculated timer current count for remaining timer period (if * any) and store it in the returned register set. */ - __kvm_lapic_set_reg(s->regs, APIC_TMCCT, - __apic_read(vcpu->arch.apic, APIC_TMCCT)); + apic_set_reg(s->regs, APIC_TMCCT, __apic_read(vcpu->arch.apic, APIC_TMCCT)); return kvm_apic_state_fixup(vcpu, s, false); } diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 7ce89bf0b974..a49e4c21db35 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -165,14 +165,14 @@ static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) apic->irr_pending = true; } -static inline u32 __kvm_lapic_get_reg(void *regs, int reg_off) +static inline u32 apic_get_reg(void *regs, int reg_off) { return *((u32 *) (regs + reg_off)); } static inline u32 kvm_lapic_get_reg(struct kvm_lapic *apic, int reg_off) { - return __kvm_lapic_get_reg(apic->regs, reg_off); + return apic_get_reg(apic->regs, reg_off); } DECLARE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); From 9c23bc4fec2b3b0324cac39ad7ec88206cc52da3 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:15 +0530 Subject: [PATCH 08/14] KVM: x86: Rename lapic get/set_reg64() helpers In preparation for moving kvm-internal __kvm_lapic_set_reg64(), __kvm_lapic_get_reg64() to apic.h for use in Secure AVIC APIC driver, rename them as part of the APIC API. No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-9-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 436e0b0c829e..b7a9d7037cab 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -89,7 +89,7 @@ static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 va apic_set_reg(apic->regs, reg_off, val); } -static __always_inline u64 __kvm_lapic_get_reg64(void *regs, int reg) +static __always_inline u64 apic_get_reg64(void *regs, int reg) { BUILD_BUG_ON(reg != APIC_ICR); return *((u64 *) (regs + reg)); @@ -97,10 +97,10 @@ static __always_inline u64 __kvm_lapic_get_reg64(void *regs, int reg) static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg) { - return __kvm_lapic_get_reg64(apic->regs, reg); + return apic_get_reg64(apic->regs, reg); } -static __always_inline void __kvm_lapic_set_reg64(void *regs, int reg, u64 val) +static __always_inline void apic_set_reg64(void *regs, int reg, u64 val) { BUILD_BUG_ON(reg != APIC_ICR); *((u64 *) (regs + reg)) = val; @@ -109,7 +109,7 @@ static __always_inline void __kvm_lapic_set_reg64(void *regs, int reg, u64 val) static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, int reg, u64 val) { - __kvm_lapic_set_reg64(apic->regs, reg, val); + apic_set_reg64(apic->regs, reg, val); } static inline int apic_test_vector(int vec, void *bitmap) @@ -3082,9 +3082,9 @@ static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, if (set) { icr = apic_get_reg(s->regs, APIC_ICR) | (u64)apic_get_reg(s->regs, APIC_ICR2) << 32; - __kvm_lapic_set_reg64(s->regs, APIC_ICR, icr); + apic_set_reg64(s->regs, APIC_ICR, icr); } else { - icr = __kvm_lapic_get_reg64(s->regs, APIC_ICR); + icr = apic_get_reg64(s->regs, APIC_ICR); apic_set_reg(s->regs, APIC_ICR2, icr >> 32); } } From b5f8980f29ce20357c6ee364a83c55f2bdf2dfde Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:16 +0530 Subject: [PATCH 09/14] KVM: x86: Rename lapic set/clear vector helpers In preparation for moving kvm-internal kvm_lapic_set_vector(), kvm_lapic_clear_vector() to apic.h for use in Secure AVIC APIC driver, rename them as part of the APIC API. No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-10-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 10 ++++------ arch/x86/kvm/lapic.h | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index b7a9d7037cab..27cc8dc1514e 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -718,10 +718,10 @@ static inline int apic_find_highest_irr(struct kvm_lapic *apic) static inline void apic_clear_irr(int vec, struct kvm_lapic *apic) { if (unlikely(apic->apicv_active)) { - kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); + apic_clear_vector(vec, apic->regs + APIC_IRR); } else { apic->irr_pending = false; - kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR); + apic_clear_vector(vec, apic->regs + APIC_IRR); if (apic_search_irr(apic) != -1) apic->irr_pending = true; } @@ -1328,11 +1328,9 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) { if (trig_mode) - kvm_lapic_set_vector(vector, - apic->regs + APIC_TMR); + apic_set_vector(vector, apic->regs + APIC_TMR); else - kvm_lapic_clear_vector(vector, - apic->regs + APIC_TMR); + apic_clear_vector(vector, apic->regs + APIC_TMR); } kvm_x86_call(deliver_interrupt)(apic, delivery_mode, diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index a49e4c21db35..c7babae8af83 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -145,19 +145,19 @@ void kvm_lapic_exit(void); u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic); -static inline void kvm_lapic_clear_vector(int vec, void *bitmap) +static inline void apic_clear_vector(int vec, void *bitmap) { clear_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); } -static inline void kvm_lapic_set_vector(int vec, void *bitmap) +static inline void apic_set_vector(int vec, void *bitmap) { set_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); } static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) { - kvm_lapic_set_vector(vec, apic->regs + APIC_IRR); + apic_set_vector(vec, apic->regs + APIC_IRR); /* * irr_pending must be true if any interrupt is pending; set it after * APIC_IRR to avoid race with apic_clear_irr From 39e81633f65eeb474215c95991c19f31b5a19a11 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:17 +0530 Subject: [PATCH 10/14] x86/apic: KVM: Move apic_find_highest_vector() to a common header In preparation for using apic_find_highest_vector() in Secure AVIC guest APIC driver, move it and associated macros to apic.h. No functional change intended. Acked-by: Sean Christopherson Signed-off-by: Neeraj Upadhyay Link: https://lore.kernel.org/r/20250709033242.267892-11-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/apic.h | 22 ++++++++++++++++++++++ arch/x86/kvm/lapic.c | 18 +----------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index c84d4e86fe4e..c7355bcbfd60 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -503,6 +503,28 @@ static inline bool is_vector_pending(unsigned int vector) return lapic_vector_set_in_irr(vector) || pi_pending_this_cpu(vector); } +#define MAX_APIC_VECTOR 256 +#define APIC_VECTORS_PER_REG 32 + +/* + * Vector states are maintained by APIC in 32-bit registers that are + * 16 bytes aligned. The status of each vector is kept in a single + * bit. + */ +static inline int apic_find_highest_vector(void *bitmap) +{ + int vec; + u32 *reg; + + for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; vec >= 0; vec -= APIC_VECTORS_PER_REG) { + reg = bitmap + APIC_VECTOR_TO_REG_OFFSET(vec); + if (*reg) + return __fls(*reg) + vec; + } + + return -1; +} + /* * Warm reset vector position: */ diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 27cc8dc1514e..ace2530b24b8 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -55,9 +56,6 @@ /* 14 is the version for Xeon and Pentium 8.4.8*/ #define APIC_VERSION 0x14UL #define LAPIC_MMIO_LENGTH (1 << 12) -/* followed define is not in apicdef.h */ -#define MAX_APIC_VECTOR 256 -#define APIC_VECTORS_PER_REG 32 /* * Enable local APIC timer advancement (tscdeadline mode only) with adaptive @@ -616,20 +614,6 @@ static const unsigned int apic_lvt_mask[KVM_APIC_MAX_NR_LVT_ENTRIES] = { [LVT_CMCI] = LVT_MASK | APIC_MODE_MASK }; -static int apic_find_highest_vector(void *bitmap) -{ - int vec; - u32 *reg; - - for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG; vec >= 0; vec -= APIC_VECTORS_PER_REG) { - reg = bitmap + APIC_VECTOR_TO_REG_OFFSET(vec); - if (*reg) - return __fls(*reg) + vec; - } - - return -1; -} - static u8 count_vectors(void *bitmap) { int vec; From 3d3a9083da1e7f5f933455411c1e96b37ae37772 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:18 +0530 Subject: [PATCH 11/14] x86/apic: KVM: Move lapic get/set helpers to common code Move the apic_get_reg(), apic_set_reg(), apic_get_reg64() and apic_set_reg64() helper functions to apic.h in order to reuse them in the Secure AVIC guest APIC driver in later patches to read/write registers from/to the APIC backing page. No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-12-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/apic.h | 22 ++++++++++++++++++++++ arch/x86/kvm/lapic.c | 17 ----------------- arch/x86/kvm/lapic.h | 7 ++----- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index c7355bcbfd60..b8b5fe875bde 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -525,6 +525,28 @@ static inline int apic_find_highest_vector(void *bitmap) return -1; } +static inline u32 apic_get_reg(void *regs, int reg_off) +{ + return *((u32 *) (regs + reg_off)); +} + +static inline void apic_set_reg(void *regs, int reg_off, u32 val) +{ + *((u32 *) (regs + reg_off)) = val; +} + +static __always_inline u64 apic_get_reg64(void *regs, int reg) +{ + BUILD_BUG_ON(reg != APIC_ICR); + return *((u64 *) (regs + reg)); +} + +static __always_inline void apic_set_reg64(void *regs, int reg, u64 val) +{ + BUILD_BUG_ON(reg != APIC_ICR); + *((u64 *) (regs + reg)) = val; +} + /* * Warm reset vector position: */ diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index ace2530b24b8..a065db3db02d 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -77,33 +77,16 @@ module_param(lapic_timer_advance, bool, 0444); static int kvm_lapic_msr_read(struct kvm_lapic *apic, u32 reg, u64 *data); static int kvm_lapic_msr_write(struct kvm_lapic *apic, u32 reg, u64 data); -static inline void apic_set_reg(void *regs, int reg_off, u32 val) -{ - *((u32 *) (regs + reg_off)) = val; -} - static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) { apic_set_reg(apic->regs, reg_off, val); } -static __always_inline u64 apic_get_reg64(void *regs, int reg) -{ - BUILD_BUG_ON(reg != APIC_ICR); - return *((u64 *) (regs + reg)); -} - static __always_inline u64 kvm_lapic_get_reg64(struct kvm_lapic *apic, int reg) { return apic_get_reg64(apic->regs, reg); } -static __always_inline void apic_set_reg64(void *regs, int reg, u64 val) -{ - BUILD_BUG_ON(reg != APIC_ICR); - *((u64 *) (regs + reg)) = val; -} - static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, int reg, u64 val) { diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index c7babae8af83..174df6996404 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -4,6 +4,8 @@ #include +#include + #include #include "hyperv.h" @@ -165,11 +167,6 @@ static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) apic->irr_pending = true; } -static inline u32 apic_get_reg(void *regs, int reg_off) -{ - return *((u32 *) (regs + reg_off)); -} - static inline u32 kvm_lapic_get_reg(struct kvm_lapic *apic, int reg_off) { return apic_get_reg(apic->regs, reg_off); From fe954bcd577e703acdef597903079c991740f6bf Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:19 +0530 Subject: [PATCH 12/14] x86/apic: KVM: Move lapic set/clear_vector() helpers to common code Move apic_clear_vector() and apic_set_vector() helper functions to apic.h in order to reuse them in the Secure AVIC guest APIC driver in later patches to atomically set/clear vectors in the APIC backing page. No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-13-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/apic.h | 10 ++++++++++ arch/x86/kvm/lapic.h | 10 ---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index b8b5fe875bde..c6d1c51f71ec 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -547,6 +547,16 @@ static __always_inline void apic_set_reg64(void *regs, int reg, u64 val) *((u64 *) (regs + reg)) = val; } +static inline void apic_clear_vector(int vec, void *bitmap) +{ + clear_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); +} + +static inline void apic_set_vector(int vec, void *bitmap) +{ + set_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); +} + /* * Warm reset vector position: */ diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 174df6996404..31284ec61a6a 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -147,16 +147,6 @@ void kvm_lapic_exit(void); u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic); -static inline void apic_clear_vector(int vec, void *bitmap) -{ - clear_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); -} - -static inline void apic_set_vector(int vec, void *bitmap) -{ - set_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); -} - static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) { apic_set_vector(vec, apic->regs + APIC_IRR); From 17776e6c203bd3a9be795fb0a2fd72191a201ac9 Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:20 +0530 Subject: [PATCH 13/14] x86/apic: KVM: Move apic_test)vector() to common code Move apic_test_vector() to apic.h in order to reuse it in the Secure AVIC guest APIC driver in later patches to test vector state in the APIC backing page. No functional change intended. Signed-off-by: Neeraj Upadhyay Acked-by: Sean Christopherson Link: https://lore.kernel.org/r/20250709033242.267892-14-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/apic.h | 5 +++++ arch/x86/kvm/lapic.c | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index c6d1c51f71ec..34e9b43d8940 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -557,6 +557,11 @@ static inline void apic_set_vector(int vec, void *bitmap) set_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); } +static inline int apic_test_vector(int vec, void *bitmap) +{ + return test_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); +} + /* * Warm reset vector position: */ diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index a065db3db02d..241b3535cb4f 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -93,11 +93,6 @@ static __always_inline void kvm_lapic_set_reg64(struct kvm_lapic *apic, apic_set_reg64(apic->regs, reg, val); } -static inline int apic_test_vector(int vec, void *bitmap) -{ - return test_bit(APIC_VECTOR_TO_BIT_NUMBER(vec), bitmap + APIC_VECTOR_TO_REG_OFFSET(vec)); -} - bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector) { struct kvm_lapic *apic = vcpu->arch.apic; From b95a9d313642c9f3abebb77a04b41bb7bdd0feef Mon Sep 17 00:00:00 2001 From: Neeraj Upadhyay Date: Wed, 9 Jul 2025 09:02:21 +0530 Subject: [PATCH 14/14] x86/apic: Rename 'reg_off' to 'reg' Rename the 'reg_off' parameter of apic_{set|get}_reg() to 'reg' to match other usages in apic.h. No functional change intended. Reviewed-by: Tianyu Lan Signed-off-by: Neeraj Upadhyay Link: https://lore.kernel.org/r/20250709033242.267892-15-Neeraj.Upadhyay@amd.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/apic.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 34e9b43d8940..07ba4935e873 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -525,14 +525,14 @@ static inline int apic_find_highest_vector(void *bitmap) return -1; } -static inline u32 apic_get_reg(void *regs, int reg_off) +static inline u32 apic_get_reg(void *regs, int reg) { - return *((u32 *) (regs + reg_off)); + return *((u32 *) (regs + reg)); } -static inline void apic_set_reg(void *regs, int reg_off, u32 val) +static inline void apic_set_reg(void *regs, int reg, u32 val) { - *((u32 *) (regs + reg_off)) = val; + *((u32 *) (regs + reg)) = val; } static __always_inline u64 apic_get_reg64(void *regs, int reg)