diff --git a/fs/proc/internal.h b/fs/proc/internal.h index df873df9afc5..9af1f63258b7 100644 --- a/fs/proc/internal.h +++ b/fs/proc/internal.h @@ -393,6 +393,7 @@ struct proc_maps_private { struct inode *inode; struct task_struct *task; struct vma_iterator iter; + struct vma_rcu_iter rvmi; loff_t last_pos; struct proc_maps_locking_ctx lock_ctx; #ifdef CONFIG_NUMA diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index f68673a91756..b87d3e86442d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -160,7 +160,7 @@ static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, return vma_next(&priv->iter); unlock_ctx_vma(lock_ctx); - vma = lock_next_vma(lock_ctx->mm, &priv->iter, last_pos); + vma = lock_next_vma(&priv->rvmi, last_pos); if (!IS_ERR_OR_NULL(vma)) lock_ctx->locked_vma = vma; @@ -265,7 +265,7 @@ static void *m_start(struct seq_file *m, loff_t *ppos) */ if (last_addr > 0) *ppos = last_addr = priv->last_pos; - vma_iter_init(&priv->iter, mm, (unsigned long)last_addr); + vma_rcu_iter_init(&priv->rvmi, mm, &priv->iter, (unsigned long)last_addr); hold_task_mempolicy(priv); /* * If seq_file had to flush its collected data right after m_next() set @@ -515,6 +515,7 @@ static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ct struct mm_struct *mm = lock_ctx->mm; struct vm_area_struct *vma; struct vma_iterator vmi; + struct vma_rcu_iter rvmi; if (lock_ctx->mmap_locked) return find_vma(mm, addr); @@ -522,8 +523,8 @@ static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ct /* Unlock previously locked VMA and find the next one under RCU */ unlock_ctx_vma(lock_ctx); rcu_read_lock(); - vma_iter_init(&vmi, mm, addr); - vma = lock_next_vma(mm, &vmi, addr); + vma_rcu_iter_init(&rvmi, mm, &vmi, addr); + vma = lock_next_vma(&rvmi, addr); rcu_read_unlock(); if (!vma) @@ -1416,7 +1417,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) rcu_read_lock(); reset_lock_ctx(lock_ctx); - vma_iter_init(&priv->iter, mm, 0); + vma_rcu_iter_init(&priv->rvmi, mm, &priv->iter, 0); vma = proc_get_vma(m, &pos); if (unlikely(!vma)) goto empty_set; @@ -1465,7 +1466,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v) reset_lock_ctx(lock_ctx); /* Resume from the last position. */ pos = last_vma_end; - vma_iter_init(&priv->iter, mm, pos); + vma_rcu_iter_init(&priv->rvmi, mm, &priv->iter, pos); } vma = proc_get_vma(m, &pos); } diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 11b3ffef3966..9ccbbcbd7aca 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -17,6 +17,13 @@ extern int rcuwait_wake_up(struct rcuwait *w); #define MMAP_LOCK_INITIALIZER(name) \ .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock), +/* Tie a VMA iterator to mm write lock state at iterator initialisation. */ +struct vma_rcu_iter { + struct mm_struct *mm; + struct vma_iterator *vmi; + unsigned int mm_wr_seq; +}; + DECLARE_TRACEPOINT(mmap_lock_start_locking); DECLARE_TRACEPOINT(mmap_lock_acquire_returned); DECLARE_TRACEPOINT(mmap_lock_released); @@ -509,6 +516,16 @@ static inline void vma_mark_detached(struct vm_area_struct *vma) struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, unsigned long address); +static inline void vma_rcu_iter_init(struct vma_rcu_iter *rvmi, + struct mm_struct *mm, struct vma_iterator *vmi, + unsigned long addr) +{ + vma_iter_init(vmi, mm, addr); + rvmi->mm = mm; + rvmi->vmi = vmi; + rvmi->mm_wr_seq = UINT_MAX; /* Start odd so changed by default. */ +} + /* * Locks next vma pointed by the iterator. Confirms the locked vma has not * been modified and will retry under mmap_lock protection if modification @@ -516,9 +533,8 @@ struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, * Returns either a valid locked VMA, NULL if no more VMAs or -EINTR if the * process was interrupted. */ -struct vm_area_struct *lock_next_vma(struct mm_struct *mm, - struct vma_iterator *iter, - unsigned long address); +struct vm_area_struct *lock_next_vma(struct vma_rcu_iter *rvmi, + unsigned long from_addr); static inline void vma_assert_can_modify(struct vm_area_struct *vma) { diff --git a/mm/mmap_lock.c b/mm/mmap_lock.c index 83b7d3ded0dc..834bbf86e004 100644 --- a/mm/mmap_lock.c +++ b/mm/mmap_lock.c @@ -13,6 +13,8 @@ #include #include +#include "internal.h" + EXPORT_TRACEPOINT_SYMBOL(mmap_lock_start_locking); EXPORT_TRACEPOINT_SYMBOL(mmap_lock_acquire_returned); EXPORT_TRACEPOINT_SYMBOL(mmap_lock_released); @@ -428,21 +430,53 @@ static struct vm_area_struct *lock_next_vma_under_mmap_lock(struct mm_struct *mm return vma; } -struct vm_area_struct *lock_next_vma(struct mm_struct *mm, - struct vma_iterator *vmi, +/* Is there no VMA in [start, last]? */ +static bool vma_range_is_empty(struct mm_struct *mm, unsigned long start, + unsigned long last) +{ + VMA_ITERATOR(vmi, mm, start); + + return !vma_iter_load(&vmi) && vma_iter_end(&vmi) - 1 >= last; +} + +/* Might the VMA tree have been modified since last mm speculation? */ +static bool vma_rcu_iter_maybe_changed(const struct vma_rcu_iter *rvmi) +{ + return (rvmi->mm_wr_seq & 1) || /* Odd implies write-lock. */ + mmap_lock_speculate_retry(rvmi->mm, rvmi->mm_wr_seq); +} + +/* Could the VMA iterator have skipped a VMA due to the tree changing? */ +static bool vma_rcu_iter_may_have_skipped(const struct vma_rcu_iter *rvmi, + unsigned long from, unsigned long to) +{ + return vma_rcu_iter_maybe_changed(rvmi) && + !vma_range_is_empty(rvmi->mm, from, to); +} + +struct vm_area_struct *lock_next_vma(struct vma_rcu_iter *rvmi, unsigned long from_addr) { + struct mm_struct *mm = rvmi->mm; + struct vma_iterator *vmi = rvmi->vmi; struct vm_area_struct *vma; - unsigned int mm_wr_seq; - bool mmap_unlocked; RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu read lock held"); retry: - /* Start mmap_lock speculation in case we need to verify the vma later */ - mmap_unlocked = mmap_lock_speculate_try_begin(mm, &mm_wr_seq); + /* Only trust the iterator's position if the tree cannot have changed. */ + if (vma_rcu_iter_maybe_changed(rvmi)) { + vma_iter_set(vmi, from_addr); + mmap_lock_speculate_try_begin(mm, &rvmi->mm_wr_seq); + } vma = vma_next(vmi); - if (!vma) - return NULL; + if (!vma) { + /* No reason to fallback if nothing left in range. */ + if (!vma_rcu_iter_may_have_skipped(rvmi, from_addr, ULONG_MAX)) + return NULL; + + rcu_read_unlock(); + goto fallback; + } vma = vma_start_read(mm, vma); if (IS_ERR_OR_NULL(vma)) { @@ -471,14 +505,9 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm, * installed ahead of it. Otherwise we might observe a gap that should * not be there. */ - if (from_addr < vma->vm_start) { - /* Verify only if the address space might have changed since vma lookup. */ - if (!mmap_unlocked || mmap_lock_speculate_retry(mm, mm_wr_seq)) { - vma_iter_set(vmi, from_addr); - if (vma != vma_next(vmi)) - goto fallback_unlock; - } - } + if (from_addr < vma->vm_start && + vma_rcu_iter_may_have_skipped(rvmi, from_addr, vma->vm_start - 1)) + goto fallback_unlock; return vma;