diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index b464a733ef37..e5a0c6e32d19 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1403,6 +1403,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, bool freed_tables) { struct flush_tlb_info info; + bool remote_flush = false; int cpu = get_cpu(); u64 new_tlb_gen; @@ -1419,9 +1420,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, if (mm_global_asid(mm)) { broadcast_tlb_flush(&info); } else if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) { - info.trim_cpumask = should_trim_cpumask(mm); - flush_tlb_multi(mm_cpumask(mm), &info); - consider_global_asid(mm); + remote_flush = true; } else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { lockdep_assert_irqs_enabled(); local_irq_disable(); @@ -1430,6 +1429,13 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, } put_cpu(); + + if (remote_flush) { + info.trim_cpumask = should_trim_cpumask(mm); + flush_tlb_multi(mm_cpumask(mm), &info); + consider_global_asid(mm); + } + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -1676,7 +1682,7 @@ EXPORT_SYMBOL_FOR_KVM(__flush_tlb_all); void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { struct flush_tlb_info info; - + bool remote_flush = false; int cpu = get_cpu(); init_flush_tlb_info(&info, NULL, 0, TLB_FLUSH_ALL, 0, false, @@ -1690,7 +1696,7 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) invlpgb_flush_all_nonglobals(); batch->unmapped_pages = false; } else if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) { - flush_tlb_multi(&batch->cpumask, &info); + remote_flush = true; } else if (cpumask_test_cpu(cpu, &batch->cpumask)) { lockdep_assert_irqs_enabled(); local_irq_disable(); @@ -1698,9 +1704,12 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) local_irq_enable(); } - cpumask_clear(&batch->cpumask); - put_cpu(); + + if (remote_flush) + flush_tlb_multi(&batch->cpumask, &info); + + cpumask_clear(&batch->cpumask); } /*