diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index cdfe4007443e..0591aa38fd85 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -483,17 +483,16 @@ static inline void arch_end_context_switch(struct task_struct *next) static inline void arch_enter_lazy_mmu_mode(void) { - PVOP_VCALL0(pv_ops, mmu.lazy_mode.enter); -} - -static inline void arch_leave_lazy_mmu_mode(void) -{ - PVOP_VCALL0(pv_ops, mmu.lazy_mode.leave); } static inline void arch_flush_lazy_mmu_mode(void) { - PVOP_VCALL0(pv_ops, mmu.lazy_mode.flush); + PVOP_VCALL0(pv_ops, mmu.lazy_mode_flush); +} + +static inline void arch_leave_lazy_mmu_mode(void) +{ + arch_flush_lazy_mmu_mode(); } static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx, diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index 4f5ae0068aab..b4c4a23e77a1 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -19,15 +19,6 @@ struct cpumask; struct flush_tlb_info; struct vm_area_struct; -#ifdef CONFIG_PARAVIRT_XXL -struct pv_lazy_ops { - /* Set deferred update mode, used for batching operations. */ - void (*enter)(void); - void (*leave)(void); - void (*flush)(void); -} __no_randomize_layout; -#endif - struct pv_cpu_ops { /* hooks for various privileged instructions */ #ifdef CONFIG_PARAVIRT_XXL @@ -171,7 +162,7 @@ struct pv_mmu_ops { void (*set_pgd)(pgd_t *pgdp, pgd_t pgdval); - struct pv_lazy_ops lazy_mode; + void (*lazy_mode_flush)(void); /* dom0 ops */ diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h index c2fc7869b996..f666637d773e 100644 --- a/arch/x86/include/asm/xen/hypervisor.h +++ b/arch/x86/include/asm/xen/hypervisor.h @@ -64,30 +64,7 @@ void __init xen_pvh_init(struct boot_params *boot_params); void __init mem_map_via_hcall(struct boot_params *boot_params_p); #endif -/* Lazy mode for batching updates / context switch */ -enum xen_lazy_mode { - XEN_LAZY_NONE, - XEN_LAZY_MMU, - XEN_LAZY_CPU, -}; - -DECLARE_PER_CPU(enum xen_lazy_mode, xen_lazy_mode); - -static inline void enter_lazy(enum xen_lazy_mode mode) -{ - BUG_ON(this_cpu_read(xen_lazy_mode) != XEN_LAZY_NONE); - - this_cpu_write(xen_lazy_mode, mode); -} - -static inline void leave_lazy(enum xen_lazy_mode mode) -{ - BUG_ON(this_cpu_read(xen_lazy_mode) != mode); - - this_cpu_write(xen_lazy_mode, XEN_LAZY_NONE); -} - -enum xen_lazy_mode xen_get_lazy_mode(void); +bool xen_is_cpu_lazy_mode(void); #if defined(CONFIG_XEN_DOM0) && defined(CONFIG_ACPI) void xen_sanitize_proc_cap_bits(uint32_t *buf); diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 792fa96b3233..22f72034470f 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -204,11 +204,7 @@ struct paravirt_patch_template pv_ops = { .mmu.enter_mmap = paravirt_nop, - .mmu.lazy_mode = { - .enter = paravirt_nop, - .leave = paravirt_nop, - .flush = paravirt_nop, - }, + .mmu.lazy_mode_flush = paravirt_nop, .mmu.set_fixmap = native_set_fixmap, #endif /* CONFIG_PARAVIRT_XXL */ diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig index aa4040fd9215..51b53ce66efb 100644 --- a/arch/x86/xen/Kconfig +++ b/arch/x86/xen/Kconfig @@ -65,13 +65,6 @@ config XEN_PVHVM_GUEST help Support running as a Xen PVHVM guest. -config XEN_DEBUG_FS - bool "Enable Xen debug and tuning parameters in debugfs" - depends on XEN && DEBUG_FS - help - Enable statistics output and various tuning options in debugfs. - Enabling this option may incur a significant performance overhead. - config XEN_PVH bool "Xen PVH guest support" depends on XEN && XEN_PVHVM && ACPI diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile index a9ec8c9f5c5d..717264ae269b 100644 --- a/arch/x86/xen/Makefile +++ b/arch/x86/xen/Makefile @@ -43,8 +43,6 @@ obj-$(CONFIG_XEN_PVHVM_SMP) += smp_hvm.o obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= spinlock.o -obj-$(CONFIG_XEN_DEBUG_FS) += debugfs.o - obj-$(CONFIG_XEN_DOM0) += vga.o obj-$(CONFIG_XEN_EFI) += efi.o diff --git a/arch/x86/xen/debugfs.c b/arch/x86/xen/debugfs.c deleted file mode 100644 index b8c9f2a7d9b6..000000000000 --- a/arch/x86/xen/debugfs.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include - -#include "xen-ops.h" - -static struct dentry *d_xen_debug; - -struct dentry * __init xen_init_debugfs(void) -{ - if (!d_xen_debug) - d_xen_debug = debugfs_create_dir("xen", NULL); - return d_xen_debug; -} - diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index ed2d7a3756ce..b310b91f2994 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -138,14 +138,11 @@ struct tls_descs { struct desc_struct desc[3]; }; -DEFINE_PER_CPU(enum xen_lazy_mode, xen_lazy_mode) = XEN_LAZY_NONE; +static DEFINE_PER_CPU(bool, xen_cpu_lazy_mode); -enum xen_lazy_mode xen_get_lazy_mode(void) +bool xen_is_cpu_lazy_mode(void) { - if (in_interrupt()) - return XEN_LAZY_NONE; - - return this_cpu_read(xen_lazy_mode); + return !in_interrupt() && this_cpu_read(xen_cpu_lazy_mode); } /* @@ -424,10 +421,8 @@ static void xen_start_context_switch(struct task_struct *prev) { BUG_ON(preemptible()); - if (this_cpu_read(xen_lazy_mode) == XEN_LAZY_MMU) { - arch_leave_lazy_mmu_mode(); - } - enter_lazy(XEN_LAZY_CPU); + __task_lazy_mmu_mode_pause(prev); + this_cpu_write(xen_cpu_lazy_mode, true); } static void xen_end_context_switch(struct task_struct *next) @@ -435,9 +430,8 @@ static void xen_end_context_switch(struct task_struct *next) BUG_ON(preemptible()); xen_mc_flush(); - leave_lazy(XEN_LAZY_CPU); - if (__task_lazy_mmu_mode_active(next)) - arch_enter_lazy_mmu_mode(); + this_cpu_write(xen_cpu_lazy_mode, false); + __task_lazy_mmu_mode_resume(next); } static unsigned long xen_store_tr(void) @@ -544,7 +538,7 @@ static void xen_set_ldt(const void *addr, unsigned entries) MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); - xen_mc_issue(XEN_LAZY_CPU); + xen_mc_issue(!xen_is_cpu_lazy_mode()); } static void xen_load_gdt(const struct desc_ptr *dtr) @@ -640,7 +634,7 @@ static void xen_load_tls(struct thread_struct *t, unsigned int cpu) * exception between the new %fs descriptor being loaded and * %fs being effectively cleared at __switch_to(). */ - if (xen_get_lazy_mode() == XEN_LAZY_CPU) + if (xen_is_cpu_lazy_mode()) loadsegment(fs, 0); xen_mc_batch(); @@ -649,7 +643,7 @@ static void xen_load_tls(struct thread_struct *t, unsigned int cpu) load_TLS_descriptor(t, cpu, 1); load_TLS_descriptor(t, cpu, 2); - xen_mc_issue(XEN_LAZY_CPU); + xen_mc_issue(!xen_is_cpu_lazy_mode()); } static void xen_load_gs_index(unsigned int idx) @@ -1011,7 +1005,7 @@ static void xen_load_sp0(unsigned long sp0) mcs = xen_mc_entry(0); MULTI_stack_switch(mcs.mc, __KERNEL_DS, sp0); - xen_mc_issue(XEN_LAZY_CPU); + xen_mc_issue(!xen_is_cpu_lazy_mode()); this_cpu_write(cpu_tss_rw.x86_tss.sp0, sp0); } @@ -1071,7 +1065,7 @@ static void xen_write_cr0(unsigned long cr0) MULTI_fpu_taskswitch(mcs.mc, (cr0 & X86_CR0_TS) != 0); - xen_mc_issue(XEN_LAZY_CPU); + xen_mc_issue(!xen_is_cpu_lazy_mode()); } static void xen_write_cr4(unsigned long cr4) diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c index 3eee5f84f8a7..aab5f70d407c 100644 --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -84,6 +84,14 @@ #include "xen-ops.h" +enum pt_level { + PT_PGD, + PT_P4D, + PT_PUD, + PT_PMD, + PT_PTE +}; + /* * Prototypes for functions called via PV_CALLEE_SAVE_REGS_THUNK() in order * to avoid warnings with "-Wmissing-prototypes". @@ -282,7 +290,7 @@ static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val) u.val = pmd_val_ma(val); xen_extend_mmu_update(&u); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); preempt_enable(); } @@ -316,7 +324,7 @@ static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval) { struct mmu_update u; - if (xen_get_lazy_mode() != XEN_LAZY_MMU) + if (!is_lazy_mmu_mode_active()) return false; xen_mc_batch(); @@ -325,7 +333,7 @@ static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval) u.val = pte_val_ma(pteval); xen_extend_mmu_update(&u); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); return true; } @@ -372,7 +380,7 @@ static void xen_ptep_modify_prot_commit(struct vm_area_struct *vma, u.val = pte_val_ma(pte); xen_extend_mmu_update(&u); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); } /* Assume pteval_t is equivalent to all the other *val_t types. */ @@ -466,7 +474,7 @@ static void xen_set_pud_hyper(pud_t *ptr, pud_t val) u.val = pud_val_ma(val); xen_extend_mmu_update(&u); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); preempt_enable(); } @@ -549,7 +557,7 @@ static void __init xen_set_p4d_hyper(p4d_t *ptr, p4d_t val) __xen_set_p4d_hyper(ptr, val); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); preempt_enable(); } @@ -581,7 +589,7 @@ static void xen_set_p4d(p4d_t *ptr, p4d_t val) if (user_ptr) __xen_set_p4d_hyper((p4d_t *)user_ptr, val); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); } __visible p4dval_t xen_p4d_val(p4d_t p4d) @@ -808,7 +816,7 @@ static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd) PFN_DOWN(__pa(user_pgd))); } - xen_mc_issue(0); + xen_mc_issue(true); } static void xen_pgd_pin(struct mm_struct *mm) @@ -925,7 +933,7 @@ static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd) __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT); - xen_mc_issue(0); + xen_mc_issue(true); } static void xen_pgd_unpin(struct mm_struct *mm) @@ -1301,7 +1309,7 @@ static noinline void xen_flush_tlb(void) op->cmd = MMUEXT_TLB_FLUSH_LOCAL; MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); preempt_enable(); } @@ -1321,7 +1329,7 @@ static void xen_flush_tlb_one_user(unsigned long addr) op->arg1.linear_addr = addr & PAGE_MASK; MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); preempt_enable(); } @@ -1358,7 +1366,7 @@ static void xen_flush_tlb_multi(const struct cpumask *cpus, MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); } static unsigned long xen_read_cr3(void) @@ -1417,7 +1425,7 @@ static void xen_write_cr3(unsigned long cr3) else __xen_write_cr3(false, 0); - xen_mc_issue(XEN_LAZY_CPU); /* interrupts restored */ + xen_mc_issue(!xen_is_cpu_lazy_mode()); /* interrupts restored */ } /* @@ -1452,7 +1460,7 @@ static void __init xen_write_cr3_init(unsigned long cr3) __xen_write_cr3(true, cr3); - xen_mc_issue(XEN_LAZY_CPU); /* interrupts restored */ + xen_mc_issue(!xen_is_cpu_lazy_mode()); /* interrupts restored */ } static int xen_pgd_alloc(struct mm_struct *mm) @@ -1614,7 +1622,7 @@ static inline void xen_alloc_ptpage(struct mm_struct *mm, unsigned long pfn, !pinned) __pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, pfn); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); } } @@ -1644,7 +1652,7 @@ static inline void xen_release_ptpage(unsigned long pfn, unsigned level) __set_pfn_prot(pfn, PAGE_KERNEL); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); ClearPagePinned(page); } @@ -1867,7 +1875,7 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) */ xen_mc_batch(); __xen_write_cr3(true, __pa(init_top_pgt)); - xen_mc_issue(XEN_LAZY_CPU); + xen_mc_issue(!xen_is_cpu_lazy_mode()); /* We can't that easily rip out L3 and L2, as the Xen pagetables are * set out this way: [L4], [L1], [L2], [L3], [L1], [L1] ... for @@ -2143,21 +2151,10 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) #endif } -static void xen_enter_lazy_mmu(void) -{ - preempt_disable(); - if (xen_get_lazy_mode() != XEN_LAZY_MMU) - enter_lazy(XEN_LAZY_MMU); - preempt_enable(); -} - static void xen_flush_lazy_mmu(void) { preempt_disable(); - - if (xen_get_lazy_mode() == XEN_LAZY_MMU) - xen_mc_flush(); - + xen_mc_flush(); preempt_enable(); } @@ -2181,15 +2178,6 @@ static void __init xen_post_allocator_init(void) pv_ops.mmu.write_cr3 = &xen_write_cr3; } -static void xen_leave_lazy_mmu(void) -{ - preempt_disable(); - xen_mc_flush(); - if (xen_get_lazy_mode() != XEN_LAZY_NONE) - leave_lazy(XEN_LAZY_MMU); - preempt_enable(); -} - void __init xen_init_mmu_ops(void) { x86_init.paging.pagetable_init = xen_pagetable_init; @@ -2229,9 +2217,7 @@ void __init xen_init_mmu_ops(void) pv_ops.mmu.make_p4d = PV_CALLEE_SAVE(xen_make_p4d); pv_ops.mmu.enter_mmap = xen_enter_mmap; pv_ops.mmu.exit_mmap = xen_exit_mmap; - pv_ops.mmu.lazy_mode.enter = xen_enter_lazy_mmu; - pv_ops.mmu.lazy_mode.leave = xen_leave_lazy_mmu; - pv_ops.mmu.lazy_mode.flush = xen_flush_lazy_mmu; + pv_ops.mmu.lazy_mode_flush = xen_flush_lazy_mmu; pv_ops.mmu.set_fixmap = xen_set_fixmap; memset(dummy_mapping, 0xff, PAGE_SIZE); @@ -2258,7 +2244,7 @@ static void xen_zap_pfn_range(unsigned long vaddr, unsigned int order, if (out_frames) out_frames[i] = virt_to_pfn((void *)vaddr); } - xen_mc_issue(0); + xen_mc_issue(true); } /* @@ -2301,7 +2287,7 @@ static void xen_remap_exchanged_ptes(unsigned long vaddr, int order, set_phys_to_machine(virt_to_pfn((void *)vaddr), mfn); } - xen_mc_issue(0); + xen_mc_issue(true); } /* @@ -2442,7 +2428,7 @@ static noinline void xen_flush_tlb_all(void) op->cmd = MMUEXT_TLB_FLUSH_ALL; MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); - xen_mc_issue(XEN_LAZY_MMU); + xen_mc_issue(!is_lazy_mmu_mode_active()); preempt_enable(); } diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index 2dd12b61a230..d007ccf6e7a1 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c @@ -883,48 +883,3 @@ void __init xen_add_remap_nonram(phys_addr_t maddr, phys_addr_t paddr, nr_nonram_remap++; } - -#ifdef CONFIG_XEN_DEBUG_FS -#include -static int p2m_dump_show(struct seq_file *m, void *v) -{ - static const char * const type_name[] = { - [P2M_TYPE_IDENTITY] = "identity", - [P2M_TYPE_MISSING] = "missing", - [P2M_TYPE_PFN] = "pfn", - [P2M_TYPE_UNKNOWN] = "abnormal"}; - unsigned long pfn, first_pfn; - int type, prev_type; - - prev_type = xen_p2m_elem_type(0); - first_pfn = 0; - - for (pfn = 0; pfn < xen_p2m_size; pfn++) { - type = xen_p2m_elem_type(pfn); - if (type != prev_type) { - seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn, - type_name[prev_type]); - prev_type = type; - first_pfn = pfn; - } - } - seq_printf(m, " [0x%lx->0x%lx] %s\n", first_pfn, pfn, - type_name[prev_type]); - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(p2m_dump); - -static struct dentry *d_mmu_debug; - -static int __init xen_p2m_debugfs(void) -{ - struct dentry *d_xen = xen_init_debugfs(); - - d_mmu_debug = debugfs_create_dir("mmu", d_xen); - - debugfs_create_file("p2m", 0600, d_mmu_debug, NULL, &p2m_dump_fops); - return 0; -} -fs_initcall(xen_p2m_debugfs); -#endif /* CONFIG_XEN_DEBUG_FS */ diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index f6c331b20fad..dc265bdda24d 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -12,28 +12,24 @@ #include +#ifdef CONFIG_XEN_PV + #include /* These are code, but not functions. Defined in entry.S */ extern const char xen_failsafe_callback[]; -void xen_entry_SYSENTER_compat(void); -#ifdef CONFIG_X86_64 -void xen_entry_SYSCALL_64(void); -void xen_entry_SYSCALL_compat(void); -#endif +DECLARE_PER_CPU(unsigned long, xen_cr3); extern void *xen_initial_gdt; +extern cpumask_var_t xen_cpu_initialized_map; struct trap_info; void xen_copy_trap_info(struct trap_info *traps); -DECLARE_PER_CPU_ALIGNED(struct vcpu_info, xen_vcpu_info); -DECLARE_PER_CPU(unsigned long, xen_cr3); - -extern struct start_info *xen_start_info; -extern struct shared_info xen_dummy_shared_info; -extern struct shared_info *HYPERVISOR_shared_info; +void xen_entry_SYSENTER_compat(void); +void xen_entry_SYSCALL_64(void); +void xen_entry_SYSCALL_compat(void); void xen_setup_mfn_list_list(void); void xen_build_mfn_list_list(void); @@ -44,13 +40,10 @@ void __init xen_pt_check_e820(void); void xen_mm_pin_all(void); void xen_mm_unpin_all(void); -#ifdef CONFIG_X86_64 void __init xen_relocate_p2m(void); -#endif void __init xen_do_remap_nonram(void); void __init xen_add_remap_nonram(phys_addr_t maddr, phys_addr_t paddr, unsigned long size); - void __init xen_chk_is_e820_usable(phys_addr_t start, phys_addr_t size, const char *component); unsigned long __ref xen_chk_extra_mem(unsigned long pfn); @@ -59,17 +52,121 @@ void __init xen_remap_memory(void); phys_addr_t __init xen_find_free_area(phys_addr_t size); char * __init xen_memory_setup(void); void __init xen_arch_setup(void); -void xen_banner(void); void xen_enable_syscall(void); -void xen_vcpu_restore(void); +void __init xen_build_dynamic_phys_to_machine(void); +void __init xen_vmalloc_p2m_tree(void); +void xen_init_irq_ops(void); +void xen_setup_vcpu_info_placement(void); +void __init xen_init_apic(void); +__visible void xen_irq_enable_direct(void); +__visible void xen_irq_disable_direct(void); +__visible unsigned long xen_save_fl_direct(void); + +__visible unsigned long xen_read_cr2(void); +__visible unsigned long xen_read_cr2_direct(void); + +/* These are not functions, and cannot be called normally */ +__visible void xen_iret(void); + +void xen_force_evtchn_callback(void); + +void xen_pv_pre_suspend(void); +void xen_pv_post_suspend(int suspend_cancelled); +void xen_start_kernel(struct start_info *si); + +void set_pte_mfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags); +void xen_init_mmu_ops(void); + +/* Multicalls */ +struct multicall_space +{ + struct multicall_entry *mc; + void *args; +}; + +/* Allocate room for a multicall and its args */ +struct multicall_space __xen_mc_entry(size_t args); + +DECLARE_PER_CPU(unsigned long, xen_mc_irq_flags); + +/* Call to start a batch of multiple __xen_mc_entry()s. Must be + paired with xen_mc_issue() */ +static inline void xen_mc_batch(void) +{ + unsigned long flags; + + /* need to disable interrupts until this entry is complete */ + local_irq_save(flags); + trace_xen_mc_batch(flags); + __this_cpu_write(xen_mc_irq_flags, flags); +} + +static inline struct multicall_space xen_mc_entry(size_t args) +{ + xen_mc_batch(); + return __xen_mc_entry(args); +} + +/* Flush all pending multicalls */ +void xen_mc_flush(void); + +/* Issue a multicall if we're not in a lazy mode */ +static inline void xen_mc_issue(bool flush) +{ + unsigned long flags = this_cpu_read(xen_mc_irq_flags); + + trace_xen_mc_issue(flush, flags); + + if (flush) + xen_mc_flush(); + + /* restore flags saved in xen_mc_batch */ + local_irq_restore(flags); +} + +/* Set up a callback to be called when the current batch is flushed */ +void xen_mc_callback(void (*fn)(void *), void *data); + +/* + * Try to extend the arguments of the previous multicall command. The + * previous command's op must match. If it does, then it attempts to + * extend the argument space allocated to the multicall entry by + * arg_size bytes. + * + * The returned multicall_space will return with mc pointing to the + * command on success, or NULL on failure, and args pointing to the + * newly allocated space. + */ +struct multicall_space xen_mc_extend_args(unsigned long op, size_t arg_size); + +extern bool is_xen_pmu; + +irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id); +bool pmu_msr_chk_emulated(u32 msr, u64 *val, bool is_read); +int pmu_apic_update(uint32_t reg); +u64 xen_read_pmc(int counter); + +void xen_hypercall_pv(void); + +#else + +static inline void xen_pv_pre_suspend(void) {} +static inline void xen_pv_post_suspend(int suspend_cancelled) {} + +#endif /* CONFIG_XEN_PV */ + +DECLARE_PER_CPU_ALIGNED(struct vcpu_info, xen_vcpu_info); + +extern struct start_info *xen_start_info; +extern struct shared_info xen_dummy_shared_info; +extern struct shared_info *HYPERVISOR_shared_info; + +void xen_banner(void); +void xen_vcpu_restore(void); void xen_hvm_init_shared_info(void); void xen_unplug_emulated_devices(void); -void __init xen_build_dynamic_phys_to_machine(void); -void __init xen_vmalloc_p2m_tree(void); - -void xen_init_irq_ops(void); void xen_setup_timer(int cpu); void xen_setup_runstate_info(int cpu); void xen_teardown_timer(int cpu); @@ -83,13 +180,10 @@ bool xen_vcpu_stolen(int vcpu); void xen_vcpu_setup(int cpu); void xen_vcpu_info_reset(int cpu); -void xen_setup_vcpu_info_placement(void); #ifdef CONFIG_SMP void xen_smp_init(void); void __init xen_hvm_smp_init(void); - -extern cpumask_var_t xen_cpu_initialized_map; #else static inline void xen_smp_init(void) {} static inline void xen_hvm_smp_init(void) {} @@ -125,8 +219,6 @@ static inline void __init xen_init_vga(const struct dom0_vga_console_info *info, void xen_add_preferred_consoles(void); -void __init xen_init_apic(void); - #ifdef CONFIG_XEN_EFI extern void xen_efi_init(struct boot_params *boot_params); #else @@ -135,16 +227,6 @@ static inline void __init xen_efi_init(struct boot_params *boot_params) } #endif -__visible void xen_irq_enable_direct(void); -__visible void xen_irq_disable_direct(void); -__visible unsigned long xen_save_fl_direct(void); - -__visible unsigned long xen_read_cr2(void); -__visible unsigned long xen_read_cr2_direct(void); - -/* These are not functions, and cannot be called normally */ -__visible void xen_iret(void); - extern int xen_panic_handler_init(void); int xen_cpuhp_setup(int (*cpu_up_prepare_cb)(unsigned int), @@ -153,16 +235,6 @@ int xen_cpuhp_setup(int (*cpu_up_prepare_cb)(unsigned int), void xen_pin_vcpu(int cpu); void xen_emergency_restart(void); -void xen_force_evtchn_callback(void); - -#ifdef CONFIG_XEN_PV -void xen_pv_pre_suspend(void); -void xen_pv_post_suspend(int suspend_cancelled); -void xen_start_kernel(struct start_info *si); -#else -static inline void xen_pv_pre_suspend(void) {} -static inline void xen_pv_post_suspend(int suspend_cancelled) {} -#endif #ifdef CONFIG_XEN_PVHVM void xen_hvm_post_suspend(int suspend_cancelled); @@ -184,85 +256,9 @@ static inline void xen_hvm_post_suspend(int suspend_cancelled) {} void xen_add_extra_mem(unsigned long start_pfn, unsigned long n_pfns); -struct dentry * __init xen_init_debugfs(void); - -enum pt_level { - PT_PGD, - PT_P4D, - PT_PUD, - PT_PMD, - PT_PTE -}; - bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); -void set_pte_mfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags); -unsigned long xen_read_cr2_direct(void); -void xen_init_mmu_ops(void); void xen_hvm_init_mmu_ops(void); -/* Multicalls */ -struct multicall_space -{ - struct multicall_entry *mc; - void *args; -}; - -/* Allocate room for a multicall and its args */ -struct multicall_space __xen_mc_entry(size_t args); - -DECLARE_PER_CPU(unsigned long, xen_mc_irq_flags); - -/* Call to start a batch of multiple __xen_mc_entry()s. Must be - paired with xen_mc_issue() */ -static inline void xen_mc_batch(void) -{ - unsigned long flags; - - /* need to disable interrupts until this entry is complete */ - local_irq_save(flags); - trace_xen_mc_batch(xen_get_lazy_mode()); - __this_cpu_write(xen_mc_irq_flags, flags); -} - -static inline struct multicall_space xen_mc_entry(size_t args) -{ - xen_mc_batch(); - return __xen_mc_entry(args); -} - -/* Flush all pending multicalls */ -void xen_mc_flush(void); - -/* Issue a multicall if we're not in a lazy mode */ -static inline void xen_mc_issue(unsigned mode) -{ - trace_xen_mc_issue(mode); - - if ((xen_get_lazy_mode() & mode) == 0) - xen_mc_flush(); - - /* restore flags saved in xen_mc_batch */ - local_irq_restore(this_cpu_read(xen_mc_irq_flags)); -} - -/* Set up a callback to be called when the current batch is flushed */ -void xen_mc_callback(void (*fn)(void *), void *data); - -/* - * Try to extend the arguments of the previous multicall command. The - * previous command's op must match. If it does, then it attempts to - * extend the argument space allocated to the multicall entry by - * arg_size bytes. - * - * The returned multicall_space will return with mc pointing to the - * command on success, or NULL on failure, and args pointing to the - * newly allocated space. - */ -struct multicall_space xen_mc_extend_args(unsigned long op, size_t arg_size); - -extern bool is_xen_pmu; - -irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id); #ifdef CONFIG_XEN_HAVE_VPMU void xen_pmu_init(int cpu); void xen_pmu_finish(int cpu); @@ -270,9 +266,6 @@ void xen_pmu_finish(int cpu); static inline void xen_pmu_init(int cpu) {} static inline void xen_pmu_finish(int cpu) {} #endif -bool pmu_msr_chk_emulated(u32 msr, u64 *val, bool is_read); -int pmu_apic_update(uint32_t reg); -u64 xen_read_pmc(int counter); #ifdef CONFIG_SMP @@ -321,9 +314,6 @@ static inline void xen_smp_intr_free_pv(unsigned int cpu) {} static inline void xen_smp_count_cpus(void) { } #endif /* CONFIG_SMP */ -#ifdef CONFIG_XEN_PV -void xen_hypercall_pv(void); -#endif void xen_hypercall_hvm(void); void xen_hypercall_amd(void); void xen_hypercall_intel(void); diff --git a/drivers/xen/mcelog.c b/drivers/xen/mcelog.c index 53a8720f5cae..32ab419bb503 100644 --- a/drivers/xen/mcelog.c +++ b/drivers/xen/mcelog.c @@ -54,8 +54,8 @@ #include static struct mc_info g_mi; -static struct mcinfo_logical_cpu *g_physinfo; -static uint32_t ncpus; +static struct mcinfo_logical_cpu *g_physinfo __ro_after_init; +static uint32_t ncpus __ro_after_init; static DEFINE_MUTEX(mcelog_lock); @@ -182,7 +182,7 @@ static const struct file_operations xen_mce_chrdev_ops = { .unlocked_ioctl = xen_mce_chrdev_ioctl, }; -static struct miscdevice xen_mce_chrdev_device = { +static struct miscdevice xen_mce_chrdev_device __ro_after_init = { MISC_MCELOG_MINOR, "mcelog", &xen_mce_chrdev_ops, diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c index 1db82da56db6..f2438232518c 100644 --- a/drivers/xen/platform-pci.c +++ b/drivers/xen/platform-pci.c @@ -174,11 +174,9 @@ static int platform_pci_probe(struct pci_dev *pdev, } static const struct pci_device_id platform_pci_tbl[] = { - {PCI_VENDOR_ID_XEN, PCI_DEVICE_ID_XEN_PLATFORM, - PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, - {PCI_VENDOR_ID_XEN, PCI_DEVICE_ID_XEN_PLATFORM_XS61, - PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0}, - {0,} + { PCI_VDEVICE(XEN, PCI_DEVICE_ID_XEN_PLATFORM) }, + { PCI_VDEVICE(XEN, PCI_DEVICE_ID_XEN_PLATFORM_XS61) }, + { } }; static const struct dev_pm_ops platform_pm_ops = { diff --git a/drivers/xen/xen-balloon.c b/drivers/xen/xen-balloon.c index b293d7652f15..67b0e2dbe84a 100644 --- a/drivers/xen/xen-balloon.c +++ b/drivers/xen/xen-balloon.c @@ -138,7 +138,7 @@ EXPORT_SYMBOL_GPL(xen_balloon_init); struct device_attribute *attr, \ char *buf) \ { \ - return sprintf(buf, format, ##args); \ + return sysfs_emit(buf, format, ##args); \ } \ static DEVICE_ATTR_RO(name) @@ -155,7 +155,7 @@ static DEVICE_BOOL_ATTR(scrub_pages, 0644, xen_scrub_pages); static ssize_t target_kb_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%lu\n", PAGES2KB(balloon_stats.target_pages)); + return sysfs_emit(buf, "%lu\n", PAGES2KB(balloon_stats.target_pages)); } static ssize_t target_kb_store(struct device *dev, @@ -180,7 +180,7 @@ static DEVICE_ATTR_RW(target_kb); static ssize_t target_show(struct device *dev, struct device_attribute *attr, char *buf) { - return sprintf(buf, "%llu\n", + return sysfs_emit(buf, "%llu\n", (unsigned long long)balloon_stats.target_pages << PAGE_SHIFT); } diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index eb260eceb4d2..fafb2b84fa5c 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -514,7 +514,7 @@ int xenbus_probe_node(struct xen_bus_type *bus, char devname[XEN_BUS_ID_SIZE]; int err; struct xenbus_device *xendev; - size_t stringlen; + size_t name_len, type_len; char *tmpstring; enum xenbus_state state = xenbus_read_driver_state(NULL, nodename); @@ -525,8 +525,9 @@ int xenbus_probe_node(struct xen_bus_type *bus, return 0; } - stringlen = strlen(nodename) + 1 + strlen(type) + 1; - xendev = kzalloc(sizeof(*xendev) + stringlen, GFP_KERNEL); + name_len = strlen(nodename); + type_len = strlen(type); + xendev = kzalloc(sizeof(*xendev) + name_len + 1 + type_len + 1, GFP_KERNEL); if (!xendev) return -ENOMEM; @@ -535,11 +536,11 @@ int xenbus_probe_node(struct xen_bus_type *bus, /* Copy the strings into the extra space. */ tmpstring = (char *)(xendev + 1); - strcpy(tmpstring, nodename); + memcpy(tmpstring, nodename, name_len); xendev->nodename = tmpstring; - tmpstring += strlen(tmpstring) + 1; - strcpy(tmpstring, type); + tmpstring += name_len + 1; + memcpy(tmpstring, type, type_len); xendev->devicetype = tmpstring; init_completion(&xendev->down); diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a..07483ed9b3ce 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -300,6 +300,28 @@ static inline void lazy_mmu_mode_disable(void) } +/** + * __task_lazy_mmu_mode_pause() - Pause the lazy MMU mode for a task. + * @tsk: The task to check. + * + * Pauses the lazy MMU mode of @tsk. + * + * This function only operates on the state saved in task_struct; to pause + * current lazy_mmu_mode_pause() should be used instead. + * + * This function is intended for architectures that implement the lazy MMU + * mode; it must not be called from generic code. + */ +static inline void __task_lazy_mmu_mode_pause(struct task_struct *tsk) +{ + struct lazy_mmu_state *state = &tsk->lazy_mmu_state; + + VM_WARN_ON_ONCE(state->pause_count == U8_MAX); + + if (state->pause_count++ == 0 && state->enable_count > 0) + arch_leave_lazy_mmu_mode(); +} + /** * lazy_mmu_mode_pause() - Pause the lazy MMU mode. * @@ -315,15 +337,32 @@ static inline void lazy_mmu_mode_disable(void) */ static inline void lazy_mmu_mode_pause(void) { - struct lazy_mmu_state *state = ¤t->lazy_mmu_state; - if (in_interrupt()) return; - VM_WARN_ON_ONCE(state->pause_count == U8_MAX); + __task_lazy_mmu_mode_pause(current); +} - if (state->pause_count++ == 0 && state->enable_count > 0) - arch_leave_lazy_mmu_mode(); +/** + * __task_lazy_mmu_mode_resume() - Resume the lazy MMU mode for a task. + * @tsk: The task to check. + * + * Resumes the lazy MMU mode of @tsk. + * + * This function only operates on the state saved in task_struct; to resume + * current lazy_mmu_mode_resume() should be used instead. + * + * This function is intended for architectures that implement the lazy MMU + * mode; it must not be called from generic code. + */ +static inline void __task_lazy_mmu_mode_resume(struct task_struct *tsk) +{ + struct lazy_mmu_state *state = &tsk->lazy_mmu_state; + + VM_WARN_ON_ONCE(state->pause_count == 0); + + if (--state->pause_count == 0 && state->enable_count > 0) + arch_enter_lazy_mmu_mode(); } /** @@ -341,15 +380,10 @@ static inline void lazy_mmu_mode_pause(void) */ static inline void lazy_mmu_mode_resume(void) { - struct lazy_mmu_state *state = ¤t->lazy_mmu_state; - if (in_interrupt()) return; - VM_WARN_ON_ONCE(state->pause_count == 0); - - if (--state->pause_count == 0 && state->enable_count > 0) - arch_enter_lazy_mmu_mode(); + __task_lazy_mmu_mode_resume(current); } #else static inline void lazy_mmu_mode_enable(void) {} diff --git a/include/trace/events/xen.h b/include/trace/events/xen.h index 0577f0cdd231..ad384969e2cb 100644 --- a/include/trace/events/xen.h +++ b/include/trace/events/xen.h @@ -12,24 +12,29 @@ struct multicall_entry; /* Multicalls */ -DECLARE_EVENT_CLASS(xen_mc__batch, - TP_PROTO(enum xen_lazy_mode mode), - TP_ARGS(mode), +TRACE_EVENT(xen_mc_batch, + TP_PROTO(unsigned long flags), + TP_ARGS(flags), TP_STRUCT__entry( - __field(enum xen_lazy_mode, mode) + __field(unsigned long, flags) ), - TP_fast_assign(__entry->mode = mode), - TP_printk("start batch LAZY_%s", - (__entry->mode == XEN_LAZY_MMU) ? "MMU" : - (__entry->mode == XEN_LAZY_CPU) ? "CPU" : "NONE") + TP_fast_assign(__entry->flags = flags), + TP_printk("start batch lazy flags %lx", __entry->flags) ); -#define DEFINE_XEN_MC_BATCH(name) \ - DEFINE_EVENT(xen_mc__batch, name, \ - TP_PROTO(enum xen_lazy_mode mode), \ - TP_ARGS(mode)) -DEFINE_XEN_MC_BATCH(xen_mc_batch); -DEFINE_XEN_MC_BATCH(xen_mc_issue); +TRACE_EVENT(xen_mc_issue, + TP_PROTO(bool flush, unsigned long flags), + TP_ARGS(flush, flags), + TP_STRUCT__entry( + __field(unsigned long, flags) + __field(bool, flush) + ), + TP_fast_assign(__entry->flush = flush; + __entry->flags = flags; + ), + TP_printk("flush: %s, flags %lx", + __entry->flush ? "yes" : "no", __entry->flags) + ); TRACE_DEFINE_SIZEOF(ulong); @@ -129,9 +134,10 @@ TRACE_EVENT(xen_mc_extend_args, __entry->res == XEN_MC_XE_NO_SPACE ? "NO_SPACE" : "???") ); -TRACE_DEFINE_SIZEOF(pteval_t); /* mmu */ -DECLARE_EVENT_CLASS(xen_mmu__set_pte, +TRACE_DEFINE_SIZEOF(pteval_t); + +TRACE_EVENT(xen_mmu_set_pte, TP_PROTO(pte_t *ptep, pte_t pteval), TP_ARGS(ptep, pteval), TP_STRUCT__entry( @@ -146,13 +152,6 @@ DECLARE_EVENT_CLASS(xen_mmu__set_pte, (int)sizeof(pteval_t) * 2, (unsigned long long)__entry->pteval) ); -#define DEFINE_XEN_MMU_SET_PTE(name) \ - DEFINE_EVENT(xen_mmu__set_pte, name, \ - TP_PROTO(pte_t *ptep, pte_t pteval), \ - TP_ARGS(ptep, pteval)) - -DEFINE_XEN_MMU_SET_PTE(xen_mmu_set_pte); - TRACE_DEFINE_SIZEOF(pmdval_t); TRACE_EVENT(xen_mmu_set_pmd, @@ -170,37 +169,6 @@ TRACE_EVENT(xen_mmu_set_pmd, (int)sizeof(pmdval_t) * 2, (unsigned long long)__entry->pmdval) ); -#ifdef CONFIG_X86_PAE -DEFINE_XEN_MMU_SET_PTE(xen_mmu_set_pte_atomic); - -TRACE_EVENT(xen_mmu_pte_clear, - TP_PROTO(struct mm_struct *mm, unsigned long addr, pte_t *ptep), - TP_ARGS(mm, addr, ptep), - TP_STRUCT__entry( - __field(struct mm_struct *, mm) - __field(unsigned long, addr) - __field(pte_t *, ptep) - ), - TP_fast_assign(__entry->mm = mm; - __entry->addr = addr; - __entry->ptep = ptep), - TP_printk("mm %p addr %lx ptep %p", - __entry->mm, __entry->addr, __entry->ptep) - ); - -TRACE_EVENT(xen_mmu_pmd_clear, - TP_PROTO(pmd_t *pmdp), - TP_ARGS(pmdp), - TP_STRUCT__entry( - __field(pmd_t *, pmdp) - ), - TP_fast_assign(__entry->pmdp = pmdp), - TP_printk("pmdp %p", __entry->pmdp) - ); -#endif - -#if CONFIG_PGTABLE_LEVELS >= 4 - TRACE_DEFINE_SIZEOF(pudval_t); TRACE_EVENT(xen_mmu_set_pud, @@ -236,24 +204,6 @@ TRACE_EVENT(xen_mmu_set_p4d, (int)sizeof(p4dval_t) * 2, (unsigned long long)pgd_val(native_make_pgd(__entry->p4dval)), (int)sizeof(p4dval_t) * 2, (unsigned long long)__entry->p4dval) ); -#else - -TRACE_EVENT(xen_mmu_set_pud, - TP_PROTO(pud_t *pudp, pud_t pudval), - TP_ARGS(pudp, pudval), - TP_STRUCT__entry( - __field(pud_t *, pudp) - __field(pudval_t, pudval) - ), - TP_fast_assign(__entry->pudp = pudp; - __entry->pudval = native_pud_val(pudval)), - TP_printk("pudp %p pudval %0*llx (raw %0*llx)", - __entry->pudp, - (int)sizeof(pudval_t) * 2, (unsigned long long)pgd_val(native_make_pgd(__entry->pudval)), - (int)sizeof(pudval_t) * 2, (unsigned long long)__entry->pudval) - ); - -#endif DECLARE_EVENT_CLASS(xen_mmu_ptep_modify_prot, TP_PROTO(struct mm_struct *mm, unsigned long addr, @@ -452,7 +402,6 @@ TRACE_EVENT(xen_cpu_set_ldt, __entry->addr, __entry->entries) ); - #endif /* _TRACE_XEN_H */ /* This part must be outside protection */ diff --git a/include/xen/interface/io/xs_wire.h b/include/xen/interface/io/xs_wire.h index b62365478ac0..29d0394b8154 100644 --- a/include/xen/interface/io/xs_wire.h +++ b/include/xen/interface/io/xs_wire.h @@ -51,7 +51,7 @@ struct xsd_errors const char *errstring; }; #define XSD_ERROR(x) { x, #x } -static struct xsd_errors xsd_errors[] __attribute__((unused)) = { +static const struct xsd_errors xsd_errors[] __maybe_unused = { XSD_ERROR(EINVAL), XSD_ERROR(EACCES), XSD_ERROR(EEXIST),